arch/s390/kernel/perf_pai_crypto.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Performance event support - Processor Activity Instrumentation Facility
   4  *
   5  *  Copyright IBM Corp. 2022
   6  *  Author(s): Thomas Richter <tmricht@linux.ibm.com>
   7  */
   8 #define KMSG_COMPONENT  "pai_crypto"
   9 #define pr_fmt(fmt)     KMSG_COMPONENT ": " fmt
  10
  11 #include <linux/kernel.h>
  12 #include <linux/kernel_stat.h>
  13 #include <linux/percpu.h>
  14 #include <linux/notifier.h>
  15 #include <linux/init.h>
  16 #include <linux/export.h>
  17 #include <linux/io.h>
  18 #include <linux/perf_event.h>
  19 #include <asm/ctlreg.h>
  20 #include <asm/pai.h>
  21 #include <asm/debug.h>
  22
  23 static debug_info_t *cfm_dbg;
  24 static unsigned int paicrypt_cnt;       /* Size of the mapped counter sets */
  25                                         /* extracted with QPACI instruction */
  26
  27 DEFINE_STATIC_KEY_FALSE(pai_key);
  28
  29 struct pai_userdata {
  30         u16 num;
  31         u64 value;
  32 } __packed;
  33
  34 struct paicrypt_map {
  35         unsigned long *page;            /* Page for CPU to store counters */
  36         struct pai_userdata *save;      /* Page to store no-zero counters */
  37         unsigned int active_events;     /* # of PAI crypto users */
  38         refcount_t refcnt;              /* Reference count mapped buffers */
  39         enum paievt_mode mode;          /* Type of event */
  40         struct perf_event *event;       /* Perf event for sampling */
  41 };
  42
  43 struct paicrypt_mapptr {
  44         struct paicrypt_map *mapptr;
  45 };
  46
  47 static struct paicrypt_root {           /* Anchor to per CPU data */
  48         refcount_t refcnt;              /* Overall active events */
  49         struct paicrypt_mapptr __percpu *mapptr;
  50 } paicrypt_root;
  51
  52 /* Free per CPU data when the last event is removed. */
  53 static void paicrypt_root_free(void)
  54 {
  55         if (refcount_dec_and_test(&paicrypt_root.refcnt)) {
  56                 free_percpu(paicrypt_root.mapptr);
  57                 paicrypt_root.mapptr = NULL;
  58         }
  59         debug_sprintf_event(cfm_dbg, 5, "%s root.refcount %d\n", __func__,
  60                             refcount_read(&paicrypt_root.refcnt));
  61 }
  62
  63 /*
  64  * On initialization of first event also allocate per CPU data dynamically.
  65  * Start with an array of pointers, the array size is the maximum number of
  66  * CPUs possible, which might be larger than the number of CPUs currently
  67  * online.
  68  */
  69 static int paicrypt_root_alloc(void)
  70 {
  71         if (!refcount_inc_not_zero(&paicrypt_root.refcnt)) {
  72                 /* The memory is already zeroed. */
  73                 paicrypt_root.mapptr = alloc_percpu(struct paicrypt_mapptr);
  74                 if (!paicrypt_root.mapptr)
  75                         return -ENOMEM;
  76                 refcount_set(&paicrypt_root.refcnt, 1);
  77         }
  78         return 0;
  79 }
  80
  81 /* Release the PMU if event is the last perf event */
  82 static DEFINE_MUTEX(pai_reserve_mutex);
  83
  84 /* Adjust usage counters and remove allocated memory when all users are
  85  * gone.
  86  */
  87 static void paicrypt_event_destroy(struct perf_event *event)
  88 {
  89         struct paicrypt_mapptr *mp = per_cpu_ptr(paicrypt_root.mapptr,
  90                                                  event->cpu);
  91         struct paicrypt_map *cpump = mp->mapptr;
  92
  93         cpump->event = NULL;
  94         static_branch_dec(&pai_key);
  95         mutex_lock(&pai_reserve_mutex);
  96         debug_sprintf_event(cfm_dbg, 5, "%s event %#llx cpu %d users %d"
  97                             " mode %d refcnt %u\n", __func__,
  98                             event->attr.config, event->cpu,
  99                             cpump->active_events, cpump->mode,
 100                             refcount_read(&cpump->refcnt));
 101         if (refcount_dec_and_test(&cpump->refcnt)) {
 102                 debug_sprintf_event(cfm_dbg, 4, "%s page %#lx save %p\n",
 103                                     __func__, (unsigned long)cpump->page,
 104                                     cpump->save);
 105                 free_page((unsigned long)cpump->page);
 106                 kvfree(cpump->save);
 107                 kfree(cpump);
 108                 mp->mapptr = NULL;
 109         }
 110         paicrypt_root_free();
 111         mutex_unlock(&pai_reserve_mutex);
 112 }
 113
 114 static u64 paicrypt_getctr(unsigned long *page, int nr, bool kernel)
 115 {
 116         if (kernel)
 117                 nr += PAI_CRYPTO_MAXCTR;
 118         return page[nr];
 119 }
 120
 121 /* Read the counter values. Return value from location in CMP. For event
 122  * CRYPTO_ALL sum up all events.
 123  */
 124 static u64 paicrypt_getdata(struct perf_event *event, bool kernel)
 125 {
 126         struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr);
 127         struct paicrypt_map *cpump = mp->mapptr;
 128         u64 sum = 0;
 129         int i;
 130
 131         if (event->attr.config != PAI_CRYPTO_BASE) {
 132                 return paicrypt_getctr(cpump->page,
 133                                        event->attr.config - PAI_CRYPTO_BASE,
 134                                        kernel);
 135         }
 136
 137         for (i = 1; i <= paicrypt_cnt; i++) {
 138                 u64 val = paicrypt_getctr(cpump->page, i, kernel);
 139
 140                 if (!val)
 141                         continue;
 142                 sum += val;
 143         }
 144         return sum;
 145 }
 146
 147 static u64 paicrypt_getall(struct perf_event *event)
 148 {
 149         u64 sum = 0;
 150
 151         if (!event->attr.exclude_kernel)
 152                 sum += paicrypt_getdata(event, true);
 153         if (!event->attr.exclude_user)
 154                 sum += paicrypt_getdata(event, false);
 155
 156         return sum;
 157 }
 158
 159 /* Used to avoid races in checking concurrent access of counting and
 160  * sampling for crypto events
 161  *
 162  * Only one instance of event pai_crypto/CRYPTO_ALL/ for sampling is
 163  * allowed and when this event is running, no counting event is allowed.
 164  * Several counting events are allowed in parallel, but no sampling event
 165  * is allowed while one (or more) counting events are running.
 166  *
 167  * This function is called in process context and it is save to block.
 168  * When the event initialization functions fails, no other call back will
 169  * be invoked.
 170  *
 171  * Allocate the memory for the event.
 172  */
 173 static struct paicrypt_map *paicrypt_busy(struct perf_event *event)
 174 {
 175         struct perf_event_attr *a = &event->attr;
 176         struct paicrypt_map *cpump = NULL;
 177         struct paicrypt_mapptr *mp;
 178         int rc;
 179
 180         mutex_lock(&pai_reserve_mutex);
 181
 182         /* Allocate root node */
 183         rc = paicrypt_root_alloc();
 184         if (rc)
 185                 goto unlock;
 186
 187         /* Allocate node for this event */
 188         mp = per_cpu_ptr(paicrypt_root.mapptr, event->cpu);
 189         cpump = mp->mapptr;
 190         if (!cpump) {                   /* Paicrypt_map allocated? */
 191                 cpump = kzalloc(sizeof(*cpump), GFP_KERNEL);
 192                 if (!cpump) {
 193                         rc = -ENOMEM;
 194                         goto free_root;
 195                 }
 196         }
 197
 198         if (a->sample_period) {         /* Sampling requested */
 199                 if (cpump->mode != PAI_MODE_NONE)
 200                         rc = -EBUSY;    /* ... sampling/counting active */
 201         } else {                        /* Counting requested */
 202                 if (cpump->mode == PAI_MODE_SAMPLING)
 203                         rc = -EBUSY;    /* ... and sampling active */
 204         }
 205         /*
 206          * This error case triggers when there is a conflict:
 207          * Either sampling requested and counting already active, or visa
 208          * versa. Therefore the struct paicrypto_map for this CPU is
 209          * needed or the error could not have occurred. Only adjust root
 210          * node refcount.
 211          */
 212         if (rc)
 213                 goto free_root;
 214
 215         /* Allocate memory for counter page and counter extraction.
 216          * Only the first counting event has to allocate a page.
 217          */
 218         if (cpump->page) {
 219                 refcount_inc(&cpump->refcnt);
 220                 goto unlock;
 221         }
 222
 223         rc = -ENOMEM;
 224         cpump->page = (unsigned long *)get_zeroed_page(GFP_KERNEL);
 225         if (!cpump->page)
 226                 goto free_paicrypt_map;
 227         cpump->save = kvmalloc_array(paicrypt_cnt + 1,
 228                                      sizeof(struct pai_userdata), GFP_KERNEL);
 229         if (!cpump->save) {
 230                 free_page((unsigned long)cpump->page);
 231                 cpump->page = NULL;
 232                 goto free_paicrypt_map;
 233         }
 234
 235         /* Set mode and reference count */
 236         rc = 0;
 237         refcount_set(&cpump->refcnt, 1);
 238         cpump->mode = a->sample_period ? PAI_MODE_SAMPLING : PAI_MODE_COUNTING;
 239         mp->mapptr = cpump;
 240         debug_sprintf_event(cfm_dbg, 5, "%s sample_period %#llx users %d"
 241                             " mode %d refcnt %u page %#lx save %p rc %d\n",
 242                             __func__, a->sample_period, cpump->active_events,
 243                             cpump->mode, refcount_read(&cpump->refcnt),
 244                             (unsigned long)cpump->page, cpump->save, rc);
 245         goto unlock;
 246
 247 free_paicrypt_map:
 248         kfree(cpump);
 249         mp->mapptr = NULL;
 250 free_root:
 251         paicrypt_root_free();
 252
 253 unlock:
 254         mutex_unlock(&pai_reserve_mutex);
 255         return rc ? ERR_PTR(rc) : cpump;
 256 }
 257
 258 /* Might be called on different CPU than the one the event is intended for. */
 259 static int paicrypt_event_init(struct perf_event *event)
 260 {
 261         struct perf_event_attr *a = &event->attr;
 262         struct paicrypt_map *cpump;
 263
 264         /* PAI crypto PMU registered as PERF_TYPE_RAW, check event type */
 265         if (a->type != PERF_TYPE_RAW && event->pmu->type != a->type)
 266                 return -ENOENT;
 267         /* PAI crypto event must be in valid range */
 268         if (a->config < PAI_CRYPTO_BASE ||
 269             a->config > PAI_CRYPTO_BASE + paicrypt_cnt)
 270                 return -EINVAL;
 271         /* Allow only CPU wide operation, no process context for now. */
 272         if ((event->attach_state & PERF_ATTACH_TASK) || event->cpu == -1)
 273                 return -ENOENT;
 274         /* Allow only CRYPTO_ALL for sampling. */
 275         if (a->sample_period && a->config != PAI_CRYPTO_BASE)
 276                 return -EINVAL;
 277
 278         cpump = paicrypt_busy(event);
 279         if (IS_ERR(cpump))
 280                 return PTR_ERR(cpump);
 281
 282         event->destroy = paicrypt_event_destroy;
 283
 284         if (a->sample_period) {
 285                 a->sample_period = 1;
 286                 a->freq = 0;
 287                 /* Register for paicrypt_sched_task() to be called */
 288                 event->attach_state |= PERF_ATTACH_SCHED_CB;
 289                 /* Add raw data which contain the memory mapped counters */
 290                 a->sample_type |= PERF_SAMPLE_RAW;
 291                 /* Turn off inheritance */
 292                 a->inherit = 0;
 293         }
 294
 295         static_branch_inc(&pai_key);
 296         return 0;
 297 }
 298
 299 static void paicrypt_read(struct perf_event *event)
 300 {
 301         u64 prev, new, delta;
 302
 303         prev = local64_read(&event->hw.prev_count);
 304         new = paicrypt_getall(event);
 305         local64_set(&event->hw.prev_count, new);
 306         delta = (prev <= new) ? new - prev
 307                               : (-1ULL - prev) + new + 1;        /* overflow */
 308         local64_add(delta, &event->count);
 309 }
 310
 311 static void paicrypt_start(struct perf_event *event, int flags)
 312 {
 313         u64 sum;
 314
 315         /* Event initialization sets last_tag to 0. When later on the events
 316          * are deleted and re-added, do not reset the event count value to zero.
 317          * Events are added, deleted and re-added when 2 or more events
 318          * are active at the same time.
 319          */
 320         if (!event->attr.sample_period) {       /* Counting */
 321                 if (!event->hw.last_tag) {
 322                         event->hw.last_tag = 1;
 323                         sum = paicrypt_getall(event);   /* Get current value */
 324                         local64_set(&event->hw.prev_count, sum);
 325                 }
 326         } else {                                /* Sampling */
 327                 perf_sched_cb_inc(event->pmu);
 328         }
 329 }
 330
 331 static int paicrypt_add(struct perf_event *event, int flags)
 332 {
 333         struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr);
 334         struct paicrypt_map *cpump = mp->mapptr;
 335         unsigned long ccd;
 336
 337         if (++cpump->active_events == 1) {
 338                 ccd = virt_to_phys(cpump->page) | PAI_CRYPTO_KERNEL_OFFSET;
 339                 WRITE_ONCE(S390_lowcore.ccd, ccd);
 340                 local_ctl_set_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT);
 341         }
 342         cpump->event = event;
 343         if (flags & PERF_EF_START)
 344                 paicrypt_start(event, PERF_EF_RELOAD);
 345         event->hw.state = 0;
 346         return 0;
 347 }
 348
 349 static void paicrypt_stop(struct perf_event *event, int flags)
 350 {
 351         if (!event->attr.sample_period) /* Counting */
 352                 paicrypt_read(event);
 353         else                            /* Sampling */
 354                 perf_sched_cb_dec(event->pmu);
 355         event->hw.state = PERF_HES_STOPPED;
 356 }
 357
 358 static void paicrypt_del(struct perf_event *event, int flags)
 359 {
 360         struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr);
 361         struct paicrypt_map *cpump = mp->mapptr;
 362
 363         paicrypt_stop(event, PERF_EF_UPDATE);
 364         if (--cpump->active_events == 0) {
 365                 local_ctl_clear_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT);
 366                 WRITE_ONCE(S390_lowcore.ccd, 0);
 367         }
 368 }
 369
 370 /* Create raw data and save it in buffer. Returns number of bytes copied.
 371  * Saves only positive counter entries of the form
 372  * 2 bytes: Number of counter
 373  * 8 bytes: Value of counter
 374  */
 375 static size_t paicrypt_copy(struct pai_userdata *userdata, unsigned long *page,
 376                             bool exclude_user, bool exclude_kernel)
 377 {
 378         int i, outidx = 0;
 379
 380         for (i = 1; i <= paicrypt_cnt; i++) {
 381                 u64 val = 0;
 382
 383                 if (!exclude_kernel)
 384                         val += paicrypt_getctr(page, i, true);
 385                 if (!exclude_user)
 386                         val += paicrypt_getctr(page, i, false);
 387                 if (val) {
 388                         userdata[outidx].num = i;
 389                         userdata[outidx].value = val;
 390                         outidx++;
 391                 }
 392         }
 393         return outidx * sizeof(struct pai_userdata);
 394 }
 395
 396 static int paicrypt_push_sample(size_t rawsize, struct paicrypt_map *cpump,
 397                                 struct perf_event *event)
 398 {
 399         struct perf_sample_data data;
 400         struct perf_raw_record raw;
 401         struct pt_regs regs;
 402         int overflow;
 403
 404         /* Setup perf sample */
 405         memset(&regs, 0, sizeof(regs));
 406         memset(&raw, 0, sizeof(raw));
 407         memset(&data, 0, sizeof(data));
 408         perf_sample_data_init(&data, 0, event->hw.last_period);
 409         if (event->attr.sample_type & PERF_SAMPLE_TID) {
 410                 data.tid_entry.pid = task_tgid_nr(current);
 411                 data.tid_entry.tid = task_pid_nr(current);
 412         }
 413         if (event->attr.sample_type & PERF_SAMPLE_TIME)
 414                 data.time = event->clock();
 415         if (event->attr.sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER))
 416                 data.id = event->id;
 417         if (event->attr.sample_type & PERF_SAMPLE_CPU) {
 418                 data.cpu_entry.cpu = smp_processor_id();
 419                 data.cpu_entry.reserved = 0;
 420         }
 421         if (event->attr.sample_type & PERF_SAMPLE_RAW) {
 422                 raw.frag.size = rawsize;
 423                 raw.frag.data = cpump->save;
 424                 perf_sample_save_raw_data(&data, &raw);
 425         }
 426
 427         overflow = perf_event_overflow(event, &data, &regs);
 428         perf_event_update_userpage(event);
 429         /* Clear lowcore page after read */
 430         memset(cpump->page, 0, PAGE_SIZE);
 431         return overflow;
 432 }
 433
 434 /* Check if there is data to be saved on schedule out of a task. */
 435 static int paicrypt_have_sample(void)
 436 {
 437         struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr);
 438         struct paicrypt_map *cpump = mp->mapptr;
 439         struct perf_event *event = cpump->event;
 440         size_t rawsize;
 441         int rc = 0;
 442
 443         if (!event)             /* No event active */
 444                 return 0;
 445         rawsize = paicrypt_copy(cpump->save, cpump->page,
 446                                 cpump->event->attr.exclude_user,
 447                                 cpump->event->attr.exclude_kernel);
 448         if (rawsize)                    /* No incremented counters */
 449                 rc = paicrypt_push_sample(rawsize, cpump, event);
 450         return rc;
 451 }
 452
 453 /* Called on schedule-in and schedule-out. No access to event structure,
 454  * but for sampling only event CRYPTO_ALL is allowed.
 455  */
 456 static void paicrypt_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in)
 457 {
 458         /* We started with a clean page on event installation. So read out
 459          * results on schedule_out and if page was dirty, clear values.
 460          */
 461         if (!sched_in)
 462                 paicrypt_have_sample();
 463 }
 464
 465 /* Attribute definitions for paicrypt interface. As with other CPU
 466  * Measurement Facilities, there is one attribute per mapped counter.
 467  * The number of mapped counters may vary per machine generation. Use
 468  * the QUERY PROCESSOR ACTIVITY COUNTER INFORMATION (QPACI) instruction
 469  * to determine the number of mapped counters. The instructions returns
 470  * a positive number, which is the highest number of supported counters.
 471  * All counters less than this number are also supported, there are no
 472  * holes. A returned number of zero means no support for mapped counters.
 473  *
 474  * The identification of the counter is a unique number. The chosen range
 475  * is 0x1000 + offset in mapped kernel page.
 476  * All CPU Measurement Facility counters identifiers must be unique and
 477  * the numbers from 0 to 496 are already used for the CPU Measurement
 478  * Counter facility. Numbers 0xb0000, 0xbc000 and 0xbd000 are already
 479  * used for the CPU Measurement Sampling facility.
 480  */
 481 PMU_FORMAT_ATTR(event, "config:0-63");
 482
 483 static struct attribute *paicrypt_format_attr[] = {
 484         &format_attr_event.attr,
 485         NULL,
 486 };
 487
 488 static struct attribute_group paicrypt_events_group = {
 489         .name = "events",
 490         .attrs = NULL                   /* Filled in attr_event_init() */
 491 };
 492
 493 static struct attribute_group paicrypt_format_group = {
 494         .name = "format",
 495         .attrs = paicrypt_format_attr,
 496 };
 497
 498 static const struct attribute_group *paicrypt_attr_groups[] = {
 499         &paicrypt_events_group,
 500         &paicrypt_format_group,
 501         NULL,
 502 };
 503
 504 /* Performance monitoring unit for mapped counters */
 505 static struct pmu paicrypt = {
 506         .task_ctx_nr  = perf_invalid_context,
 507         .event_init   = paicrypt_event_init,
 508         .add          = paicrypt_add,
 509         .del          = paicrypt_del,
 510         .start        = paicrypt_start,
 511         .stop         = paicrypt_stop,
 512         .read         = paicrypt_read,
 513         .sched_task   = paicrypt_sched_task,
 514         .attr_groups  = paicrypt_attr_groups
 515 };
 516
 517 /* List of symbolic PAI counter names. */
 518 static const char * const paicrypt_ctrnames[] = {
 519         [0] = "CRYPTO_ALL",
 520         [1] = "KM_DEA",
 521         [2] = "KM_TDEA_128",
 522         [3] = "KM_TDEA_192",
 523         [4] = "KM_ENCRYPTED_DEA",
 524         [5] = "KM_ENCRYPTED_TDEA_128",
 525         [6] = "KM_ENCRYPTED_TDEA_192",
 526         [7] = "KM_AES_128",
 527         [8] = "KM_AES_192",
 528         [9] = "KM_AES_256",
 529         [10] = "KM_ENCRYPTED_AES_128",
 530         [11] = "KM_ENCRYPTED_AES_192",
 531         [12] = "KM_ENCRYPTED_AES_256",
 532         [13] = "KM_XTS_AES_128",
 533         [14] = "KM_XTS_AES_256",
 534         [15] = "KM_XTS_ENCRYPTED_AES_128",
 535         [16] = "KM_XTS_ENCRYPTED_AES_256",
 536         [17] = "KMC_DEA",
 537         [18] = "KMC_TDEA_128",
 538         [19] = "KMC_TDEA_192",
 539         [20] = "KMC_ENCRYPTED_DEA",
 540         [21] = "KMC_ENCRYPTED_TDEA_128",
 541         [22] = "KMC_ENCRYPTED_TDEA_192",
 542         [23] = "KMC_AES_128",
 543         [24] = "KMC_AES_192",
 544         [25] = "KMC_AES_256",
 545         [26] = "KMC_ENCRYPTED_AES_128",
 546         [27] = "KMC_ENCRYPTED_AES_192",
 547         [28] = "KMC_ENCRYPTED_AES_256",
 548         [29] = "KMC_PRNG",
 549         [30] = "KMA_GCM_AES_128",
 550         [31] = "KMA_GCM_AES_192",
 551         [32] = "KMA_GCM_AES_256",
 552         [33] = "KMA_GCM_ENCRYPTED_AES_128",
 553         [34] = "KMA_GCM_ENCRYPTED_AES_192",
 554         [35] = "KMA_GCM_ENCRYPTED_AES_256",
 555         [36] = "KMF_DEA",
 556         [37] = "KMF_TDEA_128",
 557         [38] = "KMF_TDEA_192",
 558         [39] = "KMF_ENCRYPTED_DEA",
 559         [40] = "KMF_ENCRYPTED_TDEA_128",
 560         [41] = "KMF_ENCRYPTED_TDEA_192",
 561         [42] = "KMF_AES_128",
 562         [43] = "KMF_AES_192",
 563         [44] = "KMF_AES_256",
 564         [45] = "KMF_ENCRYPTED_AES_128",
 565         [46] = "KMF_ENCRYPTED_AES_192",
 566         [47] = "KMF_ENCRYPTED_AES_256",
 567         [48] = "KMCTR_DEA",
 568         [49] = "KMCTR_TDEA_128",
 569         [50] = "KMCTR_TDEA_192",
 570         [51] = "KMCTR_ENCRYPTED_DEA",
 571         [52] = "KMCTR_ENCRYPTED_TDEA_128",
 572         [53] = "KMCTR_ENCRYPTED_TDEA_192",
 573         [54] = "KMCTR_AES_128",
 574         [55] = "KMCTR_AES_192",
 575         [56] = "KMCTR_AES_256",
 576         [57] = "KMCTR_ENCRYPTED_AES_128",
 577         [58] = "KMCTR_ENCRYPTED_AES_192",
 578         [59] = "KMCTR_ENCRYPTED_AES_256",
 579         [60] = "KMO_DEA",
 580         [61] = "KMO_TDEA_128",
 581         [62] = "KMO_TDEA_192",
 582         [63] = "KMO_ENCRYPTED_DEA",
 583         [64] = "KMO_ENCRYPTED_TDEA_128",
 584         [65] = "KMO_ENCRYPTED_TDEA_192",
 585         [66] = "KMO_AES_128",
 586         [67] = "KMO_AES_192",
 587         [68] = "KMO_AES_256",
 588         [69] = "KMO_ENCRYPTED_AES_128",
 589         [70] = "KMO_ENCRYPTED_AES_192",
 590         [71] = "KMO_ENCRYPTED_AES_256",
 591         [72] = "KIMD_SHA_1",
 592         [73] = "KIMD_SHA_256",
 593         [74] = "KIMD_SHA_512",
 594         [75] = "KIMD_SHA3_224",
 595         [76] = "KIMD_SHA3_256",
 596         [77] = "KIMD_SHA3_384",
 597         [78] = "KIMD_SHA3_512",
 598         [79] = "KIMD_SHAKE_128",
 599         [80] = "KIMD_SHAKE_256",
 600         [81] = "KIMD_GHASH",
 601         [82] = "KLMD_SHA_1",
 602         [83] = "KLMD_SHA_256",
 603         [84] = "KLMD_SHA_512",
 604         [85] = "KLMD_SHA3_224",
 605         [86] = "KLMD_SHA3_256",
 606         [87] = "KLMD_SHA3_384",
 607         [88] = "KLMD_SHA3_512",
 608         [89] = "KLMD_SHAKE_128",
 609         [90] = "KLMD_SHAKE_256",
 610         [91] = "KMAC_DEA",
 611         [92] = "KMAC_TDEA_128",
 612         [93] = "KMAC_TDEA_192",
 613         [94] = "KMAC_ENCRYPTED_DEA",
 614         [95] = "KMAC_ENCRYPTED_TDEA_128",
 615         [96] = "KMAC_ENCRYPTED_TDEA_192",
 616         [97] = "KMAC_AES_128",
 617         [98] = "KMAC_AES_192",
 618         [99] = "KMAC_AES_256",
 619         [100] = "KMAC_ENCRYPTED_AES_128",
 620         [101] = "KMAC_ENCRYPTED_AES_192",
 621         [102] = "KMAC_ENCRYPTED_AES_256",
 622         [103] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_DEA",
 623         [104] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_TDEA_128",
 624         [105] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_TDEA_192",
 625         [106] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_DEA",
 626         [107] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_TDEA_128",
 627         [108] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_TDEA_192",
 628         [109] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_128",
 629         [110] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_192",
 630         [111] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_256",
 631         [112] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_128",
 632         [113] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_192",
 633         [114] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_256A",
 634         [115] = "PCC_COMPUTE_XTS_PARAMETER_USING_AES_128",
 635         [116] = "PCC_COMPUTE_XTS_PARAMETER_USING_AES_256",
 636         [117] = "PCC_COMPUTE_XTS_PARAMETER_USING_ENCRYPTED_AES_128",
 637         [118] = "PCC_COMPUTE_XTS_PARAMETER_USING_ENCRYPTED_AES_256",
 638         [119] = "PCC_SCALAR_MULTIPLY_P256",
 639         [120] = "PCC_SCALAR_MULTIPLY_P384",
 640         [121] = "PCC_SCALAR_MULTIPLY_P521",
 641         [122] = "PCC_SCALAR_MULTIPLY_ED25519",
 642         [123] = "PCC_SCALAR_MULTIPLY_ED448",
 643         [124] = "PCC_SCALAR_MULTIPLY_X25519",
 644         [125] = "PCC_SCALAR_MULTIPLY_X448",
 645         [126] = "PRNO_SHA_512_DRNG",
 646         [127] = "PRNO_TRNG_QUERY_RAW_TO_CONDITIONED_RATIO",
 647         [128] = "PRNO_TRNG",
 648         [129] = "KDSA_ECDSA_VERIFY_P256",
 649         [130] = "KDSA_ECDSA_VERIFY_P384",
 650         [131] = "KDSA_ECDSA_VERIFY_P521",
 651         [132] = "KDSA_ECDSA_SIGN_P256",
 652         [133] = "KDSA_ECDSA_SIGN_P384",
 653         [134] = "KDSA_ECDSA_SIGN_P521",
 654         [135] = "KDSA_ENCRYPTED_ECDSA_SIGN_P256",
 655         [136] = "KDSA_ENCRYPTED_ECDSA_SIGN_P384",
 656         [137] = "KDSA_ENCRYPTED_ECDSA_SIGN_P521",
 657         [138] = "KDSA_EDDSA_VERIFY_ED25519",
 658         [139] = "KDSA_EDDSA_VERIFY_ED448",
 659         [140] = "KDSA_EDDSA_SIGN_ED25519",
 660         [141] = "KDSA_EDDSA_SIGN_ED448",
 661         [142] = "KDSA_ENCRYPTED_EDDSA_SIGN_ED25519",
 662         [143] = "KDSA_ENCRYPTED_EDDSA_SIGN_ED448",
 663         [144] = "PCKMO_ENCRYPT_DEA_KEY",
 664         [145] = "PCKMO_ENCRYPT_TDEA_128_KEY",
 665         [146] = "PCKMO_ENCRYPT_TDEA_192_KEY",
 666         [147] = "PCKMO_ENCRYPT_AES_128_KEY",
 667         [148] = "PCKMO_ENCRYPT_AES_192_KEY",
 668         [149] = "PCKMO_ENCRYPT_AES_256_KEY",
 669         [150] = "PCKMO_ENCRYPT_ECC_P256_KEY",
 670         [151] = "PCKMO_ENCRYPT_ECC_P384_KEY",
 671         [152] = "PCKMO_ENCRYPT_ECC_P521_KEY",
 672         [153] = "PCKMO_ENCRYPT_ECC_ED25519_KEY",
 673         [154] = "PCKMO_ENCRYPT_ECC_ED448_KEY",
 674         [155] = "IBM_RESERVED_155",
 675         [156] = "IBM_RESERVED_156",
 676 };
 677
 678 static void __init attr_event_free(struct attribute **attrs, int num)
 679 {
 680         struct perf_pmu_events_attr *pa;
 681         int i;
 682
 683         for (i = 0; i < num; i++) {
 684                 struct device_attribute *dap;
 685
 686                 dap = container_of(attrs[i], struct device_attribute, attr);
 687                 pa = container_of(dap, struct perf_pmu_events_attr, attr);
 688                 kfree(pa);
 689         }
 690         kfree(attrs);
 691 }
 692
 693 static int __init attr_event_init_one(struct attribute **attrs, int num)
 694 {
 695         struct perf_pmu_events_attr *pa;
 696
 697         pa = kzalloc(sizeof(*pa), GFP_KERNEL);
 698         if (!pa)
 699                 return -ENOMEM;
 700
 701         sysfs_attr_init(&pa->attr.attr);
 702         pa->id = PAI_CRYPTO_BASE + num;
 703         pa->attr.attr.name = paicrypt_ctrnames[num];
 704         pa->attr.attr.mode = 0444;
 705         pa->attr.show = cpumf_events_sysfs_show;
 706         pa->attr.store = NULL;
 707         attrs[num] = &pa->attr.attr;
 708         return 0;
 709 }
 710
 711 /* Create PMU sysfs event attributes on the fly. */
 712 static int __init attr_event_init(void)
 713 {
 714         struct attribute **attrs;
 715         int ret, i;
 716
 717         attrs = kmalloc_array(ARRAY_SIZE(paicrypt_ctrnames) + 1, sizeof(*attrs),
 718                               GFP_KERNEL);
 719         if (!attrs)
 720                 return -ENOMEM;
 721         for (i = 0; i < ARRAY_SIZE(paicrypt_ctrnames); i++) {
 722                 ret = attr_event_init_one(attrs, i);
 723                 if (ret) {
 724                         attr_event_free(attrs, i - 1);
 725                         return ret;
 726                 }
 727         }
 728         attrs[i] = NULL;
 729         paicrypt_events_group.attrs = attrs;
 730         return 0;
 731 }
 732
 733 static int __init paicrypt_init(void)
 734 {
 735         struct qpaci_info_block ib;
 736         int rc;
 737
 738         if (!test_facility(196))
 739                 return 0;
 740
 741         qpaci(&ib);
 742         paicrypt_cnt = ib.num_cc;
 743         if (paicrypt_cnt == 0)
 744                 return 0;
 745         if (paicrypt_cnt >= PAI_CRYPTO_MAXCTR)
 746                 paicrypt_cnt = PAI_CRYPTO_MAXCTR - 1;
 747
 748         rc = attr_event_init();         /* Export known PAI crypto events */
 749         if (rc) {
 750                 pr_err("Creation of PMU pai_crypto /sysfs failed\n");
 751                 return rc;
 752         }
 753
 754         /* Setup s390dbf facility */
 755         cfm_dbg = debug_register(KMSG_COMPONENT, 2, 256, 128);
 756         if (!cfm_dbg) {
 757                 pr_err("Registration of s390dbf pai_crypto failed\n");
 758                 return -ENOMEM;
 759         }
 760         debug_register_view(cfm_dbg, &debug_sprintf_view);
 761
 762         rc = perf_pmu_register(&paicrypt, "pai_crypto", -1);
 763         if (rc) {
 764                 pr_err("Registering the pai_crypto PMU failed with rc=%i\n",
 765                        rc);
 766                 debug_unregister_view(cfm_dbg, &debug_sprintf_view);
 767                 debug_unregister(cfm_dbg);
 768                 return rc;
 769         }
 770         return 0;
 771 }
 772
 773 device_initcall(paicrypt_init);