1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Performance event support - Processor Activity Instrumentation Facility 4 * 5 * Copyright IBM Corp. 2022 6 * Author(s): Thomas Richter <tmricht@linux.ibm.com> 7 */ 8 #define KMSG_COMPONENT "pai_crypto" 9 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt 10 11 #include <linux/kernel.h> 12 #include <linux/kernel_stat.h> 13 #include <linux/percpu.h> 14 #include <linux/notifier.h> 15 #include <linux/init.h> 16 #include <linux/export.h> 17 #include <linux/io.h> 18 #include <linux/perf_event.h> 19 #include <asm/ctlreg.h> 20 #include <asm/pai.h> 21 #include <asm/debug.h> 22 23 static debug_info_t *cfm_dbg; 24 static unsigned int paicrypt_cnt; /* Size of the mapped counter sets */ 25 /* extracted with QPACI instruction */ 26 27 DEFINE_STATIC_KEY_FALSE(pai_key); 28 29 struct pai_userdata { 30 u16 num; 31 u64 value; 32 } __packed; 33 34 struct paicrypt_map { 35 unsigned long *page; /* Page for CPU to store counters */ 36 struct pai_userdata *save; /* Page to store no-zero counters */ 37 unsigned int active_events; /* # of PAI crypto users */ 38 refcount_t refcnt; /* Reference count mapped buffers */ 39 enum paievt_mode mode; /* Type of event */ 40 struct perf_event *event; /* Perf event for sampling */ 41 }; 42 43 struct paicrypt_mapptr { 44 struct paicrypt_map *mapptr; 45 }; 46 47 static struct paicrypt_root { /* Anchor to per CPU data */ 48 refcount_t refcnt; /* Overall active events */ 49 struct paicrypt_mapptr __percpu *mapptr; 50 } paicrypt_root; 51 52 /* Free per CPU data when the last event is removed. */ 53 static void paicrypt_root_free(void) 54 { 55 if (refcount_dec_and_test(&paicrypt_root.refcnt)) { 56 free_percpu(paicrypt_root.mapptr); 57 paicrypt_root.mapptr = NULL; 58 } 59 debug_sprintf_event(cfm_dbg, 5, "%s root.refcount %d\n", __func__, 60 refcount_read(&paicrypt_root.refcnt)); 61 } 62 63 /* 64 * On initialization of first event also allocate per CPU data dynamically. 65 * Start with an array of pointers, the array size is the maximum number of 66 * CPUs possible, which might be larger than the number of CPUs currently 67 * online. 68 */ 69 static int paicrypt_root_alloc(void) 70 { 71 if (!refcount_inc_not_zero(&paicrypt_root.refcnt)) { 72 /* The memory is already zeroed. */ 73 paicrypt_root.mapptr = alloc_percpu(struct paicrypt_mapptr); 74 if (!paicrypt_root.mapptr) 75 return -ENOMEM; 76 refcount_set(&paicrypt_root.refcnt, 1); 77 } 78 return 0; 79 } 80 81 /* Release the PMU if event is the last perf event */ 82 static DEFINE_MUTEX(pai_reserve_mutex); 83 84 /* Adjust usage counters and remove allocated memory when all users are 85 * gone. 86 */ 87 static void paicrypt_event_destroy(struct perf_event *event) 88 { 89 struct paicrypt_mapptr *mp = per_cpu_ptr(paicrypt_root.mapptr, 90 event->cpu); 91 struct paicrypt_map *cpump = mp->mapptr; 92 93 cpump->event = NULL; 94 static_branch_dec(&pai_key); 95 mutex_lock(&pai_reserve_mutex); 96 debug_sprintf_event(cfm_dbg, 5, "%s event %#llx cpu %d users %d" 97 " mode %d refcnt %u\n", __func__, 98 event->attr.config, event->cpu, 99 cpump->active_events, cpump->mode, 100 refcount_read(&cpump->refcnt)); 101 if (refcount_dec_and_test(&cpump->refcnt)) { 102 debug_sprintf_event(cfm_dbg, 4, "%s page %#lx save %p\n", 103 __func__, (unsigned long)cpump->page, 104 cpump->save); 105 free_page((unsigned long)cpump->page); 106 kvfree(cpump->save); 107 kfree(cpump); 108 mp->mapptr = NULL; 109 } 110 paicrypt_root_free(); 111 mutex_unlock(&pai_reserve_mutex); 112 } 113 114 static u64 paicrypt_getctr(unsigned long *page, int nr, bool kernel) 115 { 116 if (kernel) 117 nr += PAI_CRYPTO_MAXCTR; 118 return page[nr]; 119 } 120 121 /* Read the counter values. Return value from location in CMP. For event 122 * CRYPTO_ALL sum up all events. 123 */ 124 static u64 paicrypt_getdata(struct perf_event *event, bool kernel) 125 { 126 struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); 127 struct paicrypt_map *cpump = mp->mapptr; 128 u64 sum = 0; 129 int i; 130 131 if (event->attr.config != PAI_CRYPTO_BASE) { 132 return paicrypt_getctr(cpump->page, 133 event->attr.config - PAI_CRYPTO_BASE, 134 kernel); 135 } 136 137 for (i = 1; i <= paicrypt_cnt; i++) { 138 u64 val = paicrypt_getctr(cpump->page, i, kernel); 139 140 if (!val) 141 continue; 142 sum += val; 143 } 144 return sum; 145 } 146 147 static u64 paicrypt_getall(struct perf_event *event) 148 { 149 u64 sum = 0; 150 151 if (!event->attr.exclude_kernel) 152 sum += paicrypt_getdata(event, true); 153 if (!event->attr.exclude_user) 154 sum += paicrypt_getdata(event, false); 155 156 return sum; 157 } 158 159 /* Used to avoid races in checking concurrent access of counting and 160 * sampling for crypto events 161 * 162 * Only one instance of event pai_crypto/CRYPTO_ALL/ for sampling is 163 * allowed and when this event is running, no counting event is allowed. 164 * Several counting events are allowed in parallel, but no sampling event 165 * is allowed while one (or more) counting events are running. 166 * 167 * This function is called in process context and it is save to block. 168 * When the event initialization functions fails, no other call back will 169 * be invoked. 170 * 171 * Allocate the memory for the event. 172 */ 173 static struct paicrypt_map *paicrypt_busy(struct perf_event *event) 174 { 175 struct perf_event_attr *a = &event->attr; 176 struct paicrypt_map *cpump = NULL; 177 struct paicrypt_mapptr *mp; 178 int rc; 179 180 mutex_lock(&pai_reserve_mutex); 181 182 /* Allocate root node */ 183 rc = paicrypt_root_alloc(); 184 if (rc) 185 goto unlock; 186 187 /* Allocate node for this event */ 188 mp = per_cpu_ptr(paicrypt_root.mapptr, event->cpu); 189 cpump = mp->mapptr; 190 if (!cpump) { /* Paicrypt_map allocated? */ 191 cpump = kzalloc(sizeof(*cpump), GFP_KERNEL); 192 if (!cpump) { 193 rc = -ENOMEM; 194 goto free_root; 195 } 196 } 197 198 if (a->sample_period) { /* Sampling requested */ 199 if (cpump->mode != PAI_MODE_NONE) 200 rc = -EBUSY; /* ... sampling/counting active */ 201 } else { /* Counting requested */ 202 if (cpump->mode == PAI_MODE_SAMPLING) 203 rc = -EBUSY; /* ... and sampling active */ 204 } 205 /* 206 * This error case triggers when there is a conflict: 207 * Either sampling requested and counting already active, or visa 208 * versa. Therefore the struct paicrypto_map for this CPU is 209 * needed or the error could not have occurred. Only adjust root 210 * node refcount. 211 */ 212 if (rc) 213 goto free_root; 214 215 /* Allocate memory for counter page and counter extraction. 216 * Only the first counting event has to allocate a page. 217 */ 218 if (cpump->page) { 219 refcount_inc(&cpump->refcnt); 220 goto unlock; 221 } 222 223 rc = -ENOMEM; 224 cpump->page = (unsigned long *)get_zeroed_page(GFP_KERNEL); 225 if (!cpump->page) 226 goto free_paicrypt_map; 227 cpump->save = kvmalloc_array(paicrypt_cnt + 1, 228 sizeof(struct pai_userdata), GFP_KERNEL); 229 if (!cpump->save) { 230 free_page((unsigned long)cpump->page); 231 cpump->page = NULL; 232 goto free_paicrypt_map; 233 } 234 235 /* Set mode and reference count */ 236 rc = 0; 237 refcount_set(&cpump->refcnt, 1); 238 cpump->mode = a->sample_period ? PAI_MODE_SAMPLING : PAI_MODE_COUNTING; 239 mp->mapptr = cpump; 240 debug_sprintf_event(cfm_dbg, 5, "%s sample_period %#llx users %d" 241 " mode %d refcnt %u page %#lx save %p rc %d\n", 242 __func__, a->sample_period, cpump->active_events, 243 cpump->mode, refcount_read(&cpump->refcnt), 244 (unsigned long)cpump->page, cpump->save, rc); 245 goto unlock; 246 247 free_paicrypt_map: 248 kfree(cpump); 249 mp->mapptr = NULL; 250 free_root: 251 paicrypt_root_free(); 252 253 unlock: 254 mutex_unlock(&pai_reserve_mutex); 255 return rc ? ERR_PTR(rc) : cpump; 256 } 257 258 /* Might be called on different CPU than the one the event is intended for. */ 259 static int paicrypt_event_init(struct perf_event *event) 260 { 261 struct perf_event_attr *a = &event->attr; 262 struct paicrypt_map *cpump; 263 264 /* PAI crypto PMU registered as PERF_TYPE_RAW, check event type */ 265 if (a->type != PERF_TYPE_RAW && event->pmu->type != a->type) 266 return -ENOENT; 267 /* PAI crypto event must be in valid range */ 268 if (a->config < PAI_CRYPTO_BASE || 269 a->config > PAI_CRYPTO_BASE + paicrypt_cnt) 270 return -EINVAL; 271 /* Allow only CPU wide operation, no process context for now. */ 272 if ((event->attach_state & PERF_ATTACH_TASK) || event->cpu == -1) 273 return -ENOENT; 274 /* Allow only CRYPTO_ALL for sampling. */ 275 if (a->sample_period && a->config != PAI_CRYPTO_BASE) 276 return -EINVAL; 277 278 cpump = paicrypt_busy(event); 279 if (IS_ERR(cpump)) 280 return PTR_ERR(cpump); 281 282 event->destroy = paicrypt_event_destroy; 283 284 if (a->sample_period) { 285 a->sample_period = 1; 286 a->freq = 0; 287 /* Register for paicrypt_sched_task() to be called */ 288 event->attach_state |= PERF_ATTACH_SCHED_CB; 289 /* Add raw data which contain the memory mapped counters */ 290 a->sample_type |= PERF_SAMPLE_RAW; 291 /* Turn off inheritance */ 292 a->inherit = 0; 293 } 294 295 static_branch_inc(&pai_key); 296 return 0; 297 } 298 299 static void paicrypt_read(struct perf_event *event) 300 { 301 u64 prev, new, delta; 302 303 prev = local64_read(&event->hw.prev_count); 304 new = paicrypt_getall(event); 305 local64_set(&event->hw.prev_count, new); 306 delta = (prev <= new) ? new - prev 307 : (-1ULL - prev) + new + 1; /* overflow */ 308 local64_add(delta, &event->count); 309 } 310 311 static void paicrypt_start(struct perf_event *event, int flags) 312 { 313 u64 sum; 314 315 /* Event initialization sets last_tag to 0. When later on the events 316 * are deleted and re-added, do not reset the event count value to zero. 317 * Events are added, deleted and re-added when 2 or more events 318 * are active at the same time. 319 */ 320 if (!event->attr.sample_period) { /* Counting */ 321 if (!event->hw.last_tag) { 322 event->hw.last_tag = 1; 323 sum = paicrypt_getall(event); /* Get current value */ 324 local64_set(&event->hw.prev_count, sum); 325 } 326 } else { /* Sampling */ 327 perf_sched_cb_inc(event->pmu); 328 } 329 } 330 331 static int paicrypt_add(struct perf_event *event, int flags) 332 { 333 struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); 334 struct paicrypt_map *cpump = mp->mapptr; 335 unsigned long ccd; 336 337 if (++cpump->active_events == 1) { 338 ccd = virt_to_phys(cpump->page) | PAI_CRYPTO_KERNEL_OFFSET; 339 WRITE_ONCE(S390_lowcore.ccd, ccd); 340 local_ctl_set_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT); 341 } 342 cpump->event = event; 343 if (flags & PERF_EF_START) 344 paicrypt_start(event, PERF_EF_RELOAD); 345 event->hw.state = 0; 346 return 0; 347 } 348 349 static void paicrypt_stop(struct perf_event *event, int flags) 350 { 351 if (!event->attr.sample_period) /* Counting */ 352 paicrypt_read(event); 353 else /* Sampling */ 354 perf_sched_cb_dec(event->pmu); 355 event->hw.state = PERF_HES_STOPPED; 356 } 357 358 static void paicrypt_del(struct perf_event *event, int flags) 359 { 360 struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); 361 struct paicrypt_map *cpump = mp->mapptr; 362 363 paicrypt_stop(event, PERF_EF_UPDATE); 364 if (--cpump->active_events == 0) { 365 local_ctl_clear_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT); 366 WRITE_ONCE(S390_lowcore.ccd, 0); 367 } 368 } 369 370 /* Create raw data and save it in buffer. Returns number of bytes copied. 371 * Saves only positive counter entries of the form 372 * 2 bytes: Number of counter 373 * 8 bytes: Value of counter 374 */ 375 static size_t paicrypt_copy(struct pai_userdata *userdata, unsigned long *page, 376 bool exclude_user, bool exclude_kernel) 377 { 378 int i, outidx = 0; 379 380 for (i = 1; i <= paicrypt_cnt; i++) { 381 u64 val = 0; 382 383 if (!exclude_kernel) 384 val += paicrypt_getctr(page, i, true); 385 if (!exclude_user) 386 val += paicrypt_getctr(page, i, false); 387 if (val) { 388 userdata[outidx].num = i; 389 userdata[outidx].value = val; 390 outidx++; 391 } 392 } 393 return outidx * sizeof(struct pai_userdata); 394 } 395 396 static int paicrypt_push_sample(size_t rawsize, struct paicrypt_map *cpump, 397 struct perf_event *event) 398 { 399 struct perf_sample_data data; 400 struct perf_raw_record raw; 401 struct pt_regs regs; 402 int overflow; 403 404 /* Setup perf sample */ 405 memset(®s, 0, sizeof(regs)); 406 memset(&raw, 0, sizeof(raw)); 407 memset(&data, 0, sizeof(data)); 408 perf_sample_data_init(&data, 0, event->hw.last_period); 409 if (event->attr.sample_type & PERF_SAMPLE_TID) { 410 data.tid_entry.pid = task_tgid_nr(current); 411 data.tid_entry.tid = task_pid_nr(current); 412 } 413 if (event->attr.sample_type & PERF_SAMPLE_TIME) 414 data.time = event->clock(); 415 if (event->attr.sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER)) 416 data.id = event->id; 417 if (event->attr.sample_type & PERF_SAMPLE_CPU) { 418 data.cpu_entry.cpu = smp_processor_id(); 419 data.cpu_entry.reserved = 0; 420 } 421 if (event->attr.sample_type & PERF_SAMPLE_RAW) { 422 raw.frag.size = rawsize; 423 raw.frag.data = cpump->save; 424 perf_sample_save_raw_data(&data, &raw); 425 } 426 427 overflow = perf_event_overflow(event, &data, ®s); 428 perf_event_update_userpage(event); 429 /* Clear lowcore page after read */ 430 memset(cpump->page, 0, PAGE_SIZE); 431 return overflow; 432 } 433 434 /* Check if there is data to be saved on schedule out of a task. */ 435 static int paicrypt_have_sample(void) 436 { 437 struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); 438 struct paicrypt_map *cpump = mp->mapptr; 439 struct perf_event *event = cpump->event; 440 size_t rawsize; 441 int rc = 0; 442 443 if (!event) /* No event active */ 444 return 0; 445 rawsize = paicrypt_copy(cpump->save, cpump->page, 446 cpump->event->attr.exclude_user, 447 cpump->event->attr.exclude_kernel); 448 if (rawsize) /* No incremented counters */ 449 rc = paicrypt_push_sample(rawsize, cpump, event); 450 return rc; 451 } 452 453 /* Called on schedule-in and schedule-out. No access to event structure, 454 * but for sampling only event CRYPTO_ALL is allowed. 455 */ 456 static void paicrypt_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in) 457 { 458 /* We started with a clean page on event installation. So read out 459 * results on schedule_out and if page was dirty, clear values. 460 */ 461 if (!sched_in) 462 paicrypt_have_sample(); 463 } 464 465 /* Attribute definitions for paicrypt interface. As with other CPU 466 * Measurement Facilities, there is one attribute per mapped counter. 467 * The number of mapped counters may vary per machine generation. Use 468 * the QUERY PROCESSOR ACTIVITY COUNTER INFORMATION (QPACI) instruction 469 * to determine the number of mapped counters. The instructions returns 470 * a positive number, which is the highest number of supported counters. 471 * All counters less than this number are also supported, there are no 472 * holes. A returned number of zero means no support for mapped counters. 473 * 474 * The identification of the counter is a unique number. The chosen range 475 * is 0x1000 + offset in mapped kernel page. 476 * All CPU Measurement Facility counters identifiers must be unique and 477 * the numbers from 0 to 496 are already used for the CPU Measurement 478 * Counter facility. Numbers 0xb0000, 0xbc000 and 0xbd000 are already 479 * used for the CPU Measurement Sampling facility. 480 */ 481 PMU_FORMAT_ATTR(event, "config:0-63"); 482 483 static struct attribute *paicrypt_format_attr[] = { 484 &format_attr_event.attr, 485 NULL, 486 }; 487 488 static struct attribute_group paicrypt_events_group = { 489 .name = "events", 490 .attrs = NULL /* Filled in attr_event_init() */ 491 }; 492 493 static struct attribute_group paicrypt_format_group = { 494 .name = "format", 495 .attrs = paicrypt_format_attr, 496 }; 497 498 static const struct attribute_group *paicrypt_attr_groups[] = { 499 &paicrypt_events_group, 500 &paicrypt_format_group, 501 NULL, 502 }; 503 504 /* Performance monitoring unit for mapped counters */ 505 static struct pmu paicrypt = { 506 .task_ctx_nr = perf_invalid_context, 507 .event_init = paicrypt_event_init, 508 .add = paicrypt_add, 509 .del = paicrypt_del, 510 .start = paicrypt_start, 511 .stop = paicrypt_stop, 512 .read = paicrypt_read, 513 .sched_task = paicrypt_sched_task, 514 .attr_groups = paicrypt_attr_groups 515 }; 516 517 /* List of symbolic PAI counter names. */ 518 static const char * const paicrypt_ctrnames[] = { 519 [0] = "CRYPTO_ALL", 520 [1] = "KM_DEA", 521 [2] = "KM_TDEA_128", 522 [3] = "KM_TDEA_192", 523 [4] = "KM_ENCRYPTED_DEA", 524 [5] = "KM_ENCRYPTED_TDEA_128", 525 [6] = "KM_ENCRYPTED_TDEA_192", 526 [7] = "KM_AES_128", 527 [8] = "KM_AES_192", 528 [9] = "KM_AES_256", 529 [10] = "KM_ENCRYPTED_AES_128", 530 [11] = "KM_ENCRYPTED_AES_192", 531 [12] = "KM_ENCRYPTED_AES_256", 532 [13] = "KM_XTS_AES_128", 533 [14] = "KM_XTS_AES_256", 534 [15] = "KM_XTS_ENCRYPTED_AES_128", 535 [16] = "KM_XTS_ENCRYPTED_AES_256", 536 [17] = "KMC_DEA", 537 [18] = "KMC_TDEA_128", 538 [19] = "KMC_TDEA_192", 539 [20] = "KMC_ENCRYPTED_DEA", 540 [21] = "KMC_ENCRYPTED_TDEA_128", 541 [22] = "KMC_ENCRYPTED_TDEA_192", 542 [23] = "KMC_AES_128", 543 [24] = "KMC_AES_192", 544 [25] = "KMC_AES_256", 545 [26] = "KMC_ENCRYPTED_AES_128", 546 [27] = "KMC_ENCRYPTED_AES_192", 547 [28] = "KMC_ENCRYPTED_AES_256", 548 [29] = "KMC_PRNG", 549 [30] = "KMA_GCM_AES_128", 550 [31] = "KMA_GCM_AES_192", 551 [32] = "KMA_GCM_AES_256", 552 [33] = "KMA_GCM_ENCRYPTED_AES_128", 553 [34] = "KMA_GCM_ENCRYPTED_AES_192", 554 [35] = "KMA_GCM_ENCRYPTED_AES_256", 555 [36] = "KMF_DEA", 556 [37] = "KMF_TDEA_128", 557 [38] = "KMF_TDEA_192", 558 [39] = "KMF_ENCRYPTED_DEA", 559 [40] = "KMF_ENCRYPTED_TDEA_128", 560 [41] = "KMF_ENCRYPTED_TDEA_192", 561 [42] = "KMF_AES_128", 562 [43] = "KMF_AES_192", 563 [44] = "KMF_AES_256", 564 [45] = "KMF_ENCRYPTED_AES_128", 565 [46] = "KMF_ENCRYPTED_AES_192", 566 [47] = "KMF_ENCRYPTED_AES_256", 567 [48] = "KMCTR_DEA", 568 [49] = "KMCTR_TDEA_128", 569 [50] = "KMCTR_TDEA_192", 570 [51] = "KMCTR_ENCRYPTED_DEA", 571 [52] = "KMCTR_ENCRYPTED_TDEA_128", 572 [53] = "KMCTR_ENCRYPTED_TDEA_192", 573 [54] = "KMCTR_AES_128", 574 [55] = "KMCTR_AES_192", 575 [56] = "KMCTR_AES_256", 576 [57] = "KMCTR_ENCRYPTED_AES_128", 577 [58] = "KMCTR_ENCRYPTED_AES_192", 578 [59] = "KMCTR_ENCRYPTED_AES_256", 579 [60] = "KMO_DEA", 580 [61] = "KMO_TDEA_128", 581 [62] = "KMO_TDEA_192", 582 [63] = "KMO_ENCRYPTED_DEA", 583 [64] = "KMO_ENCRYPTED_TDEA_128", 584 [65] = "KMO_ENCRYPTED_TDEA_192", 585 [66] = "KMO_AES_128", 586 [67] = "KMO_AES_192", 587 [68] = "KMO_AES_256", 588 [69] = "KMO_ENCRYPTED_AES_128", 589 [70] = "KMO_ENCRYPTED_AES_192", 590 [71] = "KMO_ENCRYPTED_AES_256", 591 [72] = "KIMD_SHA_1", 592 [73] = "KIMD_SHA_256", 593 [74] = "KIMD_SHA_512", 594 [75] = "KIMD_SHA3_224", 595 [76] = "KIMD_SHA3_256", 596 [77] = "KIMD_SHA3_384", 597 [78] = "KIMD_SHA3_512", 598 [79] = "KIMD_SHAKE_128", 599 [80] = "KIMD_SHAKE_256", 600 [81] = "KIMD_GHASH", 601 [82] = "KLMD_SHA_1", 602 [83] = "KLMD_SHA_256", 603 [84] = "KLMD_SHA_512", 604 [85] = "KLMD_SHA3_224", 605 [86] = "KLMD_SHA3_256", 606 [87] = "KLMD_SHA3_384", 607 [88] = "KLMD_SHA3_512", 608 [89] = "KLMD_SHAKE_128", 609 [90] = "KLMD_SHAKE_256", 610 [91] = "KMAC_DEA", 611 [92] = "KMAC_TDEA_128", 612 [93] = "KMAC_TDEA_192", 613 [94] = "KMAC_ENCRYPTED_DEA", 614 [95] = "KMAC_ENCRYPTED_TDEA_128", 615 [96] = "KMAC_ENCRYPTED_TDEA_192", 616 [97] = "KMAC_AES_128", 617 [98] = "KMAC_AES_192", 618 [99] = "KMAC_AES_256", 619 [100] = "KMAC_ENCRYPTED_AES_128", 620 [101] = "KMAC_ENCRYPTED_AES_192", 621 [102] = "KMAC_ENCRYPTED_AES_256", 622 [103] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_DEA", 623 [104] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_TDEA_128", 624 [105] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_TDEA_192", 625 [106] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_DEA", 626 [107] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_TDEA_128", 627 [108] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_TDEA_192", 628 [109] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_128", 629 [110] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_192", 630 [111] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_256", 631 [112] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_128", 632 [113] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_192", 633 [114] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_256A", 634 [115] = "PCC_COMPUTE_XTS_PARAMETER_USING_AES_128", 635 [116] = "PCC_COMPUTE_XTS_PARAMETER_USING_AES_256", 636 [117] = "PCC_COMPUTE_XTS_PARAMETER_USING_ENCRYPTED_AES_128", 637 [118] = "PCC_COMPUTE_XTS_PARAMETER_USING_ENCRYPTED_AES_256", 638 [119] = "PCC_SCALAR_MULTIPLY_P256", 639 [120] = "PCC_SCALAR_MULTIPLY_P384", 640 [121] = "PCC_SCALAR_MULTIPLY_P521", 641 [122] = "PCC_SCALAR_MULTIPLY_ED25519", 642 [123] = "PCC_SCALAR_MULTIPLY_ED448", 643 [124] = "PCC_SCALAR_MULTIPLY_X25519", 644 [125] = "PCC_SCALAR_MULTIPLY_X448", 645 [126] = "PRNO_SHA_512_DRNG", 646 [127] = "PRNO_TRNG_QUERY_RAW_TO_CONDITIONED_RATIO", 647 [128] = "PRNO_TRNG", 648 [129] = "KDSA_ECDSA_VERIFY_P256", 649 [130] = "KDSA_ECDSA_VERIFY_P384", 650 [131] = "KDSA_ECDSA_VERIFY_P521", 651 [132] = "KDSA_ECDSA_SIGN_P256", 652 [133] = "KDSA_ECDSA_SIGN_P384", 653 [134] = "KDSA_ECDSA_SIGN_P521", 654 [135] = "KDSA_ENCRYPTED_ECDSA_SIGN_P256", 655 [136] = "KDSA_ENCRYPTED_ECDSA_SIGN_P384", 656 [137] = "KDSA_ENCRYPTED_ECDSA_SIGN_P521", 657 [138] = "KDSA_EDDSA_VERIFY_ED25519", 658 [139] = "KDSA_EDDSA_VERIFY_ED448", 659 [140] = "KDSA_EDDSA_SIGN_ED25519", 660 [141] = "KDSA_EDDSA_SIGN_ED448", 661 [142] = "KDSA_ENCRYPTED_EDDSA_SIGN_ED25519", 662 [143] = "KDSA_ENCRYPTED_EDDSA_SIGN_ED448", 663 [144] = "PCKMO_ENCRYPT_DEA_KEY", 664 [145] = "PCKMO_ENCRYPT_TDEA_128_KEY", 665 [146] = "PCKMO_ENCRYPT_TDEA_192_KEY", 666 [147] = "PCKMO_ENCRYPT_AES_128_KEY", 667 [148] = "PCKMO_ENCRYPT_AES_192_KEY", 668 [149] = "PCKMO_ENCRYPT_AES_256_KEY", 669 [150] = "PCKMO_ENCRYPT_ECC_P256_KEY", 670 [151] = "PCKMO_ENCRYPT_ECC_P384_KEY", 671 [152] = "PCKMO_ENCRYPT_ECC_P521_KEY", 672 [153] = "PCKMO_ENCRYPT_ECC_ED25519_KEY", 673 [154] = "PCKMO_ENCRYPT_ECC_ED448_KEY", 674 [155] = "IBM_RESERVED_155", 675 [156] = "IBM_RESERVED_156", 676 }; 677 678 static void __init attr_event_free(struct attribute **attrs, int num) 679 { 680 struct perf_pmu_events_attr *pa; 681 int i; 682 683 for (i = 0; i < num; i++) { 684 struct device_attribute *dap; 685 686 dap = container_of(attrs[i], struct device_attribute, attr); 687 pa = container_of(dap, struct perf_pmu_events_attr, attr); 688 kfree(pa); 689 } 690 kfree(attrs); 691 } 692 693 static int __init attr_event_init_one(struct attribute **attrs, int num) 694 { 695 struct perf_pmu_events_attr *pa; 696 697 pa = kzalloc(sizeof(*pa), GFP_KERNEL); 698 if (!pa) 699 return -ENOMEM; 700 701 sysfs_attr_init(&pa->attr.attr); 702 pa->id = PAI_CRYPTO_BASE + num; 703 pa->attr.attr.name = paicrypt_ctrnames[num]; 704 pa->attr.attr.mode = 0444; 705 pa->attr.show = cpumf_events_sysfs_show; 706 pa->attr.store = NULL; 707 attrs[num] = &pa->attr.attr; 708 return 0; 709 } 710 711 /* Create PMU sysfs event attributes on the fly. */ 712 static int __init attr_event_init(void) 713 { 714 struct attribute **attrs; 715 int ret, i; 716 717 attrs = kmalloc_array(ARRAY_SIZE(paicrypt_ctrnames) + 1, sizeof(*attrs), 718 GFP_KERNEL); 719 if (!attrs) 720 return -ENOMEM; 721 for (i = 0; i < ARRAY_SIZE(paicrypt_ctrnames); i++) { 722 ret = attr_event_init_one(attrs, i); 723 if (ret) { 724 attr_event_free(attrs, i - 1); 725 return ret; 726 } 727 } 728 attrs[i] = NULL; 729 paicrypt_events_group.attrs = attrs; 730 return 0; 731 } 732 733 static int __init paicrypt_init(void) 734 { 735 struct qpaci_info_block ib; 736 int rc; 737 738 if (!test_facility(196)) 739 return 0; 740 741 qpaci(&ib); 742 paicrypt_cnt = ib.num_cc; 743 if (paicrypt_cnt == 0) 744 return 0; 745 if (paicrypt_cnt >= PAI_CRYPTO_MAXCTR) 746 paicrypt_cnt = PAI_CRYPTO_MAXCTR - 1; 747 748 rc = attr_event_init(); /* Export known PAI crypto events */ 749 if (rc) { 750 pr_err("Creation of PMU pai_crypto /sysfs failed\n"); 751 return rc; 752 } 753 754 /* Setup s390dbf facility */ 755 cfm_dbg = debug_register(KMSG_COMPONENT, 2, 256, 128); 756 if (!cfm_dbg) { 757 pr_err("Registration of s390dbf pai_crypto failed\n"); 758 return -ENOMEM; 759 } 760 debug_register_view(cfm_dbg, &debug_sprintf_view); 761 762 rc = perf_pmu_register(&paicrypt, "pai_crypto", -1); 763 if (rc) { 764 pr_err("Registering the pai_crypto PMU failed with rc=%i\n", 765 rc); 766 debug_unregister_view(cfm_dbg, &debug_sprintf_view); 767 debug_unregister(cfm_dbg); 768 return rc; 769 } 770 return 0; 771 } 772 773 device_initcall(paicrypt_init); 774