1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Performance event support - Processor Activity Instrumentation Facility 4 * 5 * Copyright IBM Corp. 2022 6 * Author(s): Thomas Richter <tmricht@linux.ibm.com> 7 */ 8 #define KMSG_COMPONENT "pai_crypto" 9 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt 10 11 #include <linux/kernel.h> 12 #include <linux/kernel_stat.h> 13 #include <linux/percpu.h> 14 #include <linux/notifier.h> 15 #include <linux/init.h> 16 #include <linux/export.h> 17 #include <linux/io.h> 18 #include <linux/perf_event.h> 19 #include <asm/ctlreg.h> 20 #include <asm/pai.h> 21 #include <asm/debug.h> 22 23 static debug_info_t *cfm_dbg; 24 static unsigned int paicrypt_cnt; /* Size of the mapped counter sets */ 25 /* extracted with QPACI instruction */ 26 27 DEFINE_STATIC_KEY_FALSE(pai_key); 28 29 struct pai_userdata { 30 u16 num; 31 u64 value; 32 } __packed; 33 34 struct paicrypt_map { 35 unsigned long *page; /* Page for CPU to store counters */ 36 struct pai_userdata *save; /* Page to store no-zero counters */ 37 unsigned int active_events; /* # of PAI crypto users */ 38 refcount_t refcnt; /* Reference count mapped buffers */ 39 enum paievt_mode mode; /* Type of event */ 40 struct perf_event *event; /* Perf event for sampling */ 41 }; 42 43 struct paicrypt_mapptr { 44 struct paicrypt_map *mapptr; 45 }; 46 47 static struct paicrypt_root { /* Anchor to per CPU data */ 48 refcount_t refcnt; /* Overall active events */ 49 struct paicrypt_mapptr __percpu *mapptr; 50 } paicrypt_root; 51 52 /* Free per CPU data when the last event is removed. */ 53 static void paicrypt_root_free(void) 54 { 55 if (refcount_dec_and_test(&paicrypt_root.refcnt)) { 56 free_percpu(paicrypt_root.mapptr); 57 paicrypt_root.mapptr = NULL; 58 } 59 debug_sprintf_event(cfm_dbg, 5, "%s root.refcount %d\n", __func__, 60 refcount_read(&paicrypt_root.refcnt)); 61 } 62 63 /* 64 * On initialization of first event also allocate per CPU data dynamically. 65 * Start with an array of pointers, the array size is the maximum number of 66 * CPUs possible, which might be larger than the number of CPUs currently 67 * online. 68 */ 69 static int paicrypt_root_alloc(void) 70 { 71 if (!refcount_inc_not_zero(&paicrypt_root.refcnt)) { 72 /* The memory is already zeroed. */ 73 paicrypt_root.mapptr = alloc_percpu(struct paicrypt_mapptr); 74 if (!paicrypt_root.mapptr) 75 return -ENOMEM; 76 refcount_set(&paicrypt_root.refcnt, 1); 77 } 78 return 0; 79 } 80 81 /* Release the PMU if event is the last perf event */ 82 static DEFINE_MUTEX(pai_reserve_mutex); 83 84 /* Adjust usage counters and remove allocated memory when all users are 85 * gone. 86 */ 87 static void paicrypt_event_destroy(struct perf_event *event) 88 { 89 struct paicrypt_mapptr *mp = per_cpu_ptr(paicrypt_root.mapptr, 90 event->cpu); 91 struct paicrypt_map *cpump = mp->mapptr; 92 93 cpump->event = NULL; 94 static_branch_dec(&pai_key); 95 mutex_lock(&pai_reserve_mutex); 96 debug_sprintf_event(cfm_dbg, 5, "%s event %#llx cpu %d users %d" 97 " mode %d refcnt %u\n", __func__, 98 event->attr.config, event->cpu, 99 cpump->active_events, cpump->mode, 100 refcount_read(&cpump->refcnt)); 101 if (refcount_dec_and_test(&cpump->refcnt)) { 102 debug_sprintf_event(cfm_dbg, 4, "%s page %#lx save %p\n", 103 __func__, (unsigned long)cpump->page, 104 cpump->save); 105 free_page((unsigned long)cpump->page); 106 kvfree(cpump->save); 107 kfree(cpump); 108 mp->mapptr = NULL; 109 } 110 paicrypt_root_free(); 111 mutex_unlock(&pai_reserve_mutex); 112 } 113 114 static u64 paicrypt_getctr(struct paicrypt_map *cpump, int nr, bool kernel) 115 { 116 if (kernel) 117 nr += PAI_CRYPTO_MAXCTR; 118 return cpump->page[nr]; 119 } 120 121 /* Read the counter values. Return value from location in CMP. For event 122 * CRYPTO_ALL sum up all events. 123 */ 124 static u64 paicrypt_getdata(struct perf_event *event, bool kernel) 125 { 126 struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); 127 struct paicrypt_map *cpump = mp->mapptr; 128 u64 sum = 0; 129 int i; 130 131 if (event->attr.config != PAI_CRYPTO_BASE) { 132 return paicrypt_getctr(cpump, 133 event->attr.config - PAI_CRYPTO_BASE, 134 kernel); 135 } 136 137 for (i = 1; i <= paicrypt_cnt; i++) { 138 u64 val = paicrypt_getctr(cpump, i, kernel); 139 140 if (!val) 141 continue; 142 sum += val; 143 } 144 return sum; 145 } 146 147 static u64 paicrypt_getall(struct perf_event *event) 148 { 149 u64 sum = 0; 150 151 if (!event->attr.exclude_kernel) 152 sum += paicrypt_getdata(event, true); 153 if (!event->attr.exclude_user) 154 sum += paicrypt_getdata(event, false); 155 156 return sum; 157 } 158 159 /* Used to avoid races in checking concurrent access of counting and 160 * sampling for crypto events 161 * 162 * Only one instance of event pai_crypto/CRYPTO_ALL/ for sampling is 163 * allowed and when this event is running, no counting event is allowed. 164 * Several counting events are allowed in parallel, but no sampling event 165 * is allowed while one (or more) counting events are running. 166 * 167 * This function is called in process context and it is save to block. 168 * When the event initialization functions fails, no other call back will 169 * be invoked. 170 * 171 * Allocate the memory for the event. 172 */ 173 static struct paicrypt_map *paicrypt_busy(struct perf_event *event) 174 { 175 struct perf_event_attr *a = &event->attr; 176 struct paicrypt_map *cpump = NULL; 177 struct paicrypt_mapptr *mp; 178 int rc; 179 180 mutex_lock(&pai_reserve_mutex); 181 182 /* Allocate root node */ 183 rc = paicrypt_root_alloc(); 184 if (rc) 185 goto unlock; 186 187 /* Allocate node for this event */ 188 mp = per_cpu_ptr(paicrypt_root.mapptr, event->cpu); 189 cpump = mp->mapptr; 190 if (!cpump) { /* Paicrypt_map allocated? */ 191 cpump = kzalloc(sizeof(*cpump), GFP_KERNEL); 192 if (!cpump) { 193 rc = -ENOMEM; 194 goto free_root; 195 } 196 } 197 198 if (a->sample_period) { /* Sampling requested */ 199 if (cpump->mode != PAI_MODE_NONE) 200 rc = -EBUSY; /* ... sampling/counting active */ 201 } else { /* Counting requested */ 202 if (cpump->mode == PAI_MODE_SAMPLING) 203 rc = -EBUSY; /* ... and sampling active */ 204 } 205 /* 206 * This error case triggers when there is a conflict: 207 * Either sampling requested and counting already active, or visa 208 * versa. Therefore the struct paicrypto_map for this CPU is 209 * needed or the error could not have occurred. Only adjust root 210 * node refcount. 211 */ 212 if (rc) 213 goto free_root; 214 215 /* Allocate memory for counter page and counter extraction. 216 * Only the first counting event has to allocate a page. 217 */ 218 if (cpump->page) { 219 refcount_inc(&cpump->refcnt); 220 goto unlock; 221 } 222 223 rc = -ENOMEM; 224 cpump->page = (unsigned long *)get_zeroed_page(GFP_KERNEL); 225 if (!cpump->page) 226 goto free_paicrypt_map; 227 cpump->save = kvmalloc_array(paicrypt_cnt + 1, 228 sizeof(struct pai_userdata), GFP_KERNEL); 229 if (!cpump->save) { 230 free_page((unsigned long)cpump->page); 231 cpump->page = NULL; 232 goto free_paicrypt_map; 233 } 234 235 /* Set mode and reference count */ 236 rc = 0; 237 refcount_set(&cpump->refcnt, 1); 238 cpump->mode = a->sample_period ? PAI_MODE_SAMPLING : PAI_MODE_COUNTING; 239 mp->mapptr = cpump; 240 debug_sprintf_event(cfm_dbg, 5, "%s sample_period %#llx users %d" 241 " mode %d refcnt %u page %#lx save %p rc %d\n", 242 __func__, a->sample_period, cpump->active_events, 243 cpump->mode, refcount_read(&cpump->refcnt), 244 (unsigned long)cpump->page, cpump->save, rc); 245 goto unlock; 246 247 free_paicrypt_map: 248 kfree(cpump); 249 mp->mapptr = NULL; 250 free_root: 251 paicrypt_root_free(); 252 253 unlock: 254 mutex_unlock(&pai_reserve_mutex); 255 return rc ? ERR_PTR(rc) : cpump; 256 } 257 258 /* Might be called on different CPU than the one the event is intended for. */ 259 static int paicrypt_event_init(struct perf_event *event) 260 { 261 struct perf_event_attr *a = &event->attr; 262 struct paicrypt_map *cpump; 263 264 /* PAI crypto PMU registered as PERF_TYPE_RAW, check event type */ 265 if (a->type != PERF_TYPE_RAW && event->pmu->type != a->type) 266 return -ENOENT; 267 /* PAI crypto event must be in valid range */ 268 if (a->config < PAI_CRYPTO_BASE || 269 a->config > PAI_CRYPTO_BASE + paicrypt_cnt) 270 return -EINVAL; 271 /* Allow only CPU wide operation, no process context for now. */ 272 if ((event->attach_state & PERF_ATTACH_TASK) || event->cpu == -1) 273 return -ENOENT; 274 /* Allow only CRYPTO_ALL for sampling. */ 275 if (a->sample_period && a->config != PAI_CRYPTO_BASE) 276 return -EINVAL; 277 278 cpump = paicrypt_busy(event); 279 if (IS_ERR(cpump)) 280 return PTR_ERR(cpump); 281 282 /* Event initialization sets last_tag to 0. When later on the events 283 * are deleted and re-added, do not reset the event count value to zero. 284 * Events are added, deleted and re-added when 2 or more events 285 * are active at the same time. 286 */ 287 event->hw.last_tag = 0; 288 event->destroy = paicrypt_event_destroy; 289 290 if (a->sample_period) { 291 a->sample_period = 1; 292 a->freq = 0; 293 /* Register for paicrypt_sched_task() to be called */ 294 event->attach_state |= PERF_ATTACH_SCHED_CB; 295 /* Add raw data which contain the memory mapped counters */ 296 a->sample_type |= PERF_SAMPLE_RAW; 297 /* Turn off inheritance */ 298 a->inherit = 0; 299 } 300 301 static_branch_inc(&pai_key); 302 return 0; 303 } 304 305 static void paicrypt_read(struct perf_event *event) 306 { 307 u64 prev, new, delta; 308 309 prev = local64_read(&event->hw.prev_count); 310 new = paicrypt_getall(event); 311 local64_set(&event->hw.prev_count, new); 312 delta = (prev <= new) ? new - prev 313 : (-1ULL - prev) + new + 1; /* overflow */ 314 local64_add(delta, &event->count); 315 } 316 317 static void paicrypt_start(struct perf_event *event, int flags) 318 { 319 u64 sum; 320 321 if (!event->hw.last_tag) { 322 event->hw.last_tag = 1; 323 sum = paicrypt_getall(event); /* Get current value */ 324 local64_set(&event->hw.prev_count, sum); 325 } 326 } 327 328 static int paicrypt_add(struct perf_event *event, int flags) 329 { 330 struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); 331 struct paicrypt_map *cpump = mp->mapptr; 332 unsigned long ccd; 333 334 if (++cpump->active_events == 1) { 335 ccd = virt_to_phys(cpump->page) | PAI_CRYPTO_KERNEL_OFFSET; 336 WRITE_ONCE(S390_lowcore.ccd, ccd); 337 local_ctl_set_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT); 338 } 339 cpump->event = event; 340 if (flags & PERF_EF_START && !event->attr.sample_period) { 341 /* Only counting needs initial counter value */ 342 paicrypt_start(event, PERF_EF_RELOAD); 343 } 344 event->hw.state = 0; 345 if (event->attr.sample_period) 346 perf_sched_cb_inc(event->pmu); 347 return 0; 348 } 349 350 static void paicrypt_stop(struct perf_event *event, int flags) 351 { 352 paicrypt_read(event); 353 event->hw.state = PERF_HES_STOPPED; 354 } 355 356 static void paicrypt_del(struct perf_event *event, int flags) 357 { 358 struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); 359 struct paicrypt_map *cpump = mp->mapptr; 360 361 if (event->attr.sample_period) 362 perf_sched_cb_dec(event->pmu); 363 if (!event->attr.sample_period) 364 /* Only counting needs to read counter */ 365 paicrypt_stop(event, PERF_EF_UPDATE); 366 if (--cpump->active_events == 0) { 367 local_ctl_clear_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT); 368 WRITE_ONCE(S390_lowcore.ccd, 0); 369 } 370 } 371 372 /* Create raw data and save it in buffer. Returns number of bytes copied. 373 * Saves only positive counter entries of the form 374 * 2 bytes: Number of counter 375 * 8 bytes: Value of counter 376 */ 377 static size_t paicrypt_copy(struct pai_userdata *userdata, 378 struct paicrypt_map *cpump, 379 bool exclude_user, bool exclude_kernel) 380 { 381 int i, outidx = 0; 382 383 for (i = 1; i <= paicrypt_cnt; i++) { 384 u64 val = 0; 385 386 if (!exclude_kernel) 387 val += paicrypt_getctr(cpump, i, true); 388 if (!exclude_user) 389 val += paicrypt_getctr(cpump, i, false); 390 if (val) { 391 userdata[outidx].num = i; 392 userdata[outidx].value = val; 393 outidx++; 394 } 395 } 396 return outidx * sizeof(struct pai_userdata); 397 } 398 399 static int paicrypt_push_sample(void) 400 { 401 struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); 402 struct paicrypt_map *cpump = mp->mapptr; 403 struct perf_event *event = cpump->event; 404 struct perf_sample_data data; 405 struct perf_raw_record raw; 406 struct pt_regs regs; 407 size_t rawsize; 408 int overflow; 409 410 if (!cpump->event) /* No event active */ 411 return 0; 412 rawsize = paicrypt_copy(cpump->save, cpump, 413 cpump->event->attr.exclude_user, 414 cpump->event->attr.exclude_kernel); 415 if (!rawsize) /* No incremented counters */ 416 return 0; 417 418 /* Setup perf sample */ 419 memset(®s, 0, sizeof(regs)); 420 memset(&raw, 0, sizeof(raw)); 421 memset(&data, 0, sizeof(data)); 422 perf_sample_data_init(&data, 0, event->hw.last_period); 423 if (event->attr.sample_type & PERF_SAMPLE_TID) { 424 data.tid_entry.pid = task_tgid_nr(current); 425 data.tid_entry.tid = task_pid_nr(current); 426 } 427 if (event->attr.sample_type & PERF_SAMPLE_TIME) 428 data.time = event->clock(); 429 if (event->attr.sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER)) 430 data.id = event->id; 431 if (event->attr.sample_type & PERF_SAMPLE_CPU) { 432 data.cpu_entry.cpu = smp_processor_id(); 433 data.cpu_entry.reserved = 0; 434 } 435 if (event->attr.sample_type & PERF_SAMPLE_RAW) { 436 raw.frag.size = rawsize; 437 raw.frag.data = cpump->save; 438 perf_sample_save_raw_data(&data, &raw); 439 } 440 441 overflow = perf_event_overflow(event, &data, ®s); 442 perf_event_update_userpage(event); 443 /* Clear lowcore page after read */ 444 memset(cpump->page, 0, PAGE_SIZE); 445 return overflow; 446 } 447 448 /* Called on schedule-in and schedule-out. No access to event structure, 449 * but for sampling only event CRYPTO_ALL is allowed. 450 */ 451 static void paicrypt_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in) 452 { 453 /* We started with a clean page on event installation. So read out 454 * results on schedule_out and if page was dirty, clear values. 455 */ 456 if (!sched_in) 457 paicrypt_push_sample(); 458 } 459 460 /* Attribute definitions for paicrypt interface. As with other CPU 461 * Measurement Facilities, there is one attribute per mapped counter. 462 * The number of mapped counters may vary per machine generation. Use 463 * the QUERY PROCESSOR ACTIVITY COUNTER INFORMATION (QPACI) instruction 464 * to determine the number of mapped counters. The instructions returns 465 * a positive number, which is the highest number of supported counters. 466 * All counters less than this number are also supported, there are no 467 * holes. A returned number of zero means no support for mapped counters. 468 * 469 * The identification of the counter is a unique number. The chosen range 470 * is 0x1000 + offset in mapped kernel page. 471 * All CPU Measurement Facility counters identifiers must be unique and 472 * the numbers from 0 to 496 are already used for the CPU Measurement 473 * Counter facility. Numbers 0xb0000, 0xbc000 and 0xbd000 are already 474 * used for the CPU Measurement Sampling facility. 475 */ 476 PMU_FORMAT_ATTR(event, "config:0-63"); 477 478 static struct attribute *paicrypt_format_attr[] = { 479 &format_attr_event.attr, 480 NULL, 481 }; 482 483 static struct attribute_group paicrypt_events_group = { 484 .name = "events", 485 .attrs = NULL /* Filled in attr_event_init() */ 486 }; 487 488 static struct attribute_group paicrypt_format_group = { 489 .name = "format", 490 .attrs = paicrypt_format_attr, 491 }; 492 493 static const struct attribute_group *paicrypt_attr_groups[] = { 494 &paicrypt_events_group, 495 &paicrypt_format_group, 496 NULL, 497 }; 498 499 /* Performance monitoring unit for mapped counters */ 500 static struct pmu paicrypt = { 501 .task_ctx_nr = perf_invalid_context, 502 .event_init = paicrypt_event_init, 503 .add = paicrypt_add, 504 .del = paicrypt_del, 505 .start = paicrypt_start, 506 .stop = paicrypt_stop, 507 .read = paicrypt_read, 508 .sched_task = paicrypt_sched_task, 509 .attr_groups = paicrypt_attr_groups 510 }; 511 512 /* List of symbolic PAI counter names. */ 513 static const char * const paicrypt_ctrnames[] = { 514 [0] = "CRYPTO_ALL", 515 [1] = "KM_DEA", 516 [2] = "KM_TDEA_128", 517 [3] = "KM_TDEA_192", 518 [4] = "KM_ENCRYPTED_DEA", 519 [5] = "KM_ENCRYPTED_TDEA_128", 520 [6] = "KM_ENCRYPTED_TDEA_192", 521 [7] = "KM_AES_128", 522 [8] = "KM_AES_192", 523 [9] = "KM_AES_256", 524 [10] = "KM_ENCRYPTED_AES_128", 525 [11] = "KM_ENCRYPTED_AES_192", 526 [12] = "KM_ENCRYPTED_AES_256", 527 [13] = "KM_XTS_AES_128", 528 [14] = "KM_XTS_AES_256", 529 [15] = "KM_XTS_ENCRYPTED_AES_128", 530 [16] = "KM_XTS_ENCRYPTED_AES_256", 531 [17] = "KMC_DEA", 532 [18] = "KMC_TDEA_128", 533 [19] = "KMC_TDEA_192", 534 [20] = "KMC_ENCRYPTED_DEA", 535 [21] = "KMC_ENCRYPTED_TDEA_128", 536 [22] = "KMC_ENCRYPTED_TDEA_192", 537 [23] = "KMC_AES_128", 538 [24] = "KMC_AES_192", 539 [25] = "KMC_AES_256", 540 [26] = "KMC_ENCRYPTED_AES_128", 541 [27] = "KMC_ENCRYPTED_AES_192", 542 [28] = "KMC_ENCRYPTED_AES_256", 543 [29] = "KMC_PRNG", 544 [30] = "KMA_GCM_AES_128", 545 [31] = "KMA_GCM_AES_192", 546 [32] = "KMA_GCM_AES_256", 547 [33] = "KMA_GCM_ENCRYPTED_AES_128", 548 [34] = "KMA_GCM_ENCRYPTED_AES_192", 549 [35] = "KMA_GCM_ENCRYPTED_AES_256", 550 [36] = "KMF_DEA", 551 [37] = "KMF_TDEA_128", 552 [38] = "KMF_TDEA_192", 553 [39] = "KMF_ENCRYPTED_DEA", 554 [40] = "KMF_ENCRYPTED_TDEA_128", 555 [41] = "KMF_ENCRYPTED_TDEA_192", 556 [42] = "KMF_AES_128", 557 [43] = "KMF_AES_192", 558 [44] = "KMF_AES_256", 559 [45] = "KMF_ENCRYPTED_AES_128", 560 [46] = "KMF_ENCRYPTED_AES_192", 561 [47] = "KMF_ENCRYPTED_AES_256", 562 [48] = "KMCTR_DEA", 563 [49] = "KMCTR_TDEA_128", 564 [50] = "KMCTR_TDEA_192", 565 [51] = "KMCTR_ENCRYPTED_DEA", 566 [52] = "KMCTR_ENCRYPTED_TDEA_128", 567 [53] = "KMCTR_ENCRYPTED_TDEA_192", 568 [54] = "KMCTR_AES_128", 569 [55] = "KMCTR_AES_192", 570 [56] = "KMCTR_AES_256", 571 [57] = "KMCTR_ENCRYPTED_AES_128", 572 [58] = "KMCTR_ENCRYPTED_AES_192", 573 [59] = "KMCTR_ENCRYPTED_AES_256", 574 [60] = "KMO_DEA", 575 [61] = "KMO_TDEA_128", 576 [62] = "KMO_TDEA_192", 577 [63] = "KMO_ENCRYPTED_DEA", 578 [64] = "KMO_ENCRYPTED_TDEA_128", 579 [65] = "KMO_ENCRYPTED_TDEA_192", 580 [66] = "KMO_AES_128", 581 [67] = "KMO_AES_192", 582 [68] = "KMO_AES_256", 583 [69] = "KMO_ENCRYPTED_AES_128", 584 [70] = "KMO_ENCRYPTED_AES_192", 585 [71] = "KMO_ENCRYPTED_AES_256", 586 [72] = "KIMD_SHA_1", 587 [73] = "KIMD_SHA_256", 588 [74] = "KIMD_SHA_512", 589 [75] = "KIMD_SHA3_224", 590 [76] = "KIMD_SHA3_256", 591 [77] = "KIMD_SHA3_384", 592 [78] = "KIMD_SHA3_512", 593 [79] = "KIMD_SHAKE_128", 594 [80] = "KIMD_SHAKE_256", 595 [81] = "KIMD_GHASH", 596 [82] = "KLMD_SHA_1", 597 [83] = "KLMD_SHA_256", 598 [84] = "KLMD_SHA_512", 599 [85] = "KLMD_SHA3_224", 600 [86] = "KLMD_SHA3_256", 601 [87] = "KLMD_SHA3_384", 602 [88] = "KLMD_SHA3_512", 603 [89] = "KLMD_SHAKE_128", 604 [90] = "KLMD_SHAKE_256", 605 [91] = "KMAC_DEA", 606 [92] = "KMAC_TDEA_128", 607 [93] = "KMAC_TDEA_192", 608 [94] = "KMAC_ENCRYPTED_DEA", 609 [95] = "KMAC_ENCRYPTED_TDEA_128", 610 [96] = "KMAC_ENCRYPTED_TDEA_192", 611 [97] = "KMAC_AES_128", 612 [98] = "KMAC_AES_192", 613 [99] = "KMAC_AES_256", 614 [100] = "KMAC_ENCRYPTED_AES_128", 615 [101] = "KMAC_ENCRYPTED_AES_192", 616 [102] = "KMAC_ENCRYPTED_AES_256", 617 [103] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_DEA", 618 [104] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_TDEA_128", 619 [105] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_TDEA_192", 620 [106] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_DEA", 621 [107] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_TDEA_128", 622 [108] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_TDEA_192", 623 [109] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_128", 624 [110] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_192", 625 [111] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_256", 626 [112] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_128", 627 [113] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_192", 628 [114] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_256A", 629 [115] = "PCC_COMPUTE_XTS_PARAMETER_USING_AES_128", 630 [116] = "PCC_COMPUTE_XTS_PARAMETER_USING_AES_256", 631 [117] = "PCC_COMPUTE_XTS_PARAMETER_USING_ENCRYPTED_AES_128", 632 [118] = "PCC_COMPUTE_XTS_PARAMETER_USING_ENCRYPTED_AES_256", 633 [119] = "PCC_SCALAR_MULTIPLY_P256", 634 [120] = "PCC_SCALAR_MULTIPLY_P384", 635 [121] = "PCC_SCALAR_MULTIPLY_P521", 636 [122] = "PCC_SCALAR_MULTIPLY_ED25519", 637 [123] = "PCC_SCALAR_MULTIPLY_ED448", 638 [124] = "PCC_SCALAR_MULTIPLY_X25519", 639 [125] = "PCC_SCALAR_MULTIPLY_X448", 640 [126] = "PRNO_SHA_512_DRNG", 641 [127] = "PRNO_TRNG_QUERY_RAW_TO_CONDITIONED_RATIO", 642 [128] = "PRNO_TRNG", 643 [129] = "KDSA_ECDSA_VERIFY_P256", 644 [130] = "KDSA_ECDSA_VERIFY_P384", 645 [131] = "KDSA_ECDSA_VERIFY_P521", 646 [132] = "KDSA_ECDSA_SIGN_P256", 647 [133] = "KDSA_ECDSA_SIGN_P384", 648 [134] = "KDSA_ECDSA_SIGN_P521", 649 [135] = "KDSA_ENCRYPTED_ECDSA_SIGN_P256", 650 [136] = "KDSA_ENCRYPTED_ECDSA_SIGN_P384", 651 [137] = "KDSA_ENCRYPTED_ECDSA_SIGN_P521", 652 [138] = "KDSA_EDDSA_VERIFY_ED25519", 653 [139] = "KDSA_EDDSA_VERIFY_ED448", 654 [140] = "KDSA_EDDSA_SIGN_ED25519", 655 [141] = "KDSA_EDDSA_SIGN_ED448", 656 [142] = "KDSA_ENCRYPTED_EDDSA_SIGN_ED25519", 657 [143] = "KDSA_ENCRYPTED_EDDSA_SIGN_ED448", 658 [144] = "PCKMO_ENCRYPT_DEA_KEY", 659 [145] = "PCKMO_ENCRYPT_TDEA_128_KEY", 660 [146] = "PCKMO_ENCRYPT_TDEA_192_KEY", 661 [147] = "PCKMO_ENCRYPT_AES_128_KEY", 662 [148] = "PCKMO_ENCRYPT_AES_192_KEY", 663 [149] = "PCKMO_ENCRYPT_AES_256_KEY", 664 [150] = "PCKMO_ENCRYPT_ECC_P256_KEY", 665 [151] = "PCKMO_ENCRYPT_ECC_P384_KEY", 666 [152] = "PCKMO_ENCRYPT_ECC_P521_KEY", 667 [153] = "PCKMO_ENCRYPT_ECC_ED25519_KEY", 668 [154] = "PCKMO_ENCRYPT_ECC_ED448_KEY", 669 [155] = "IBM_RESERVED_155", 670 [156] = "IBM_RESERVED_156", 671 }; 672 673 static void __init attr_event_free(struct attribute **attrs, int num) 674 { 675 struct perf_pmu_events_attr *pa; 676 int i; 677 678 for (i = 0; i < num; i++) { 679 struct device_attribute *dap; 680 681 dap = container_of(attrs[i], struct device_attribute, attr); 682 pa = container_of(dap, struct perf_pmu_events_attr, attr); 683 kfree(pa); 684 } 685 kfree(attrs); 686 } 687 688 static int __init attr_event_init_one(struct attribute **attrs, int num) 689 { 690 struct perf_pmu_events_attr *pa; 691 692 pa = kzalloc(sizeof(*pa), GFP_KERNEL); 693 if (!pa) 694 return -ENOMEM; 695 696 sysfs_attr_init(&pa->attr.attr); 697 pa->id = PAI_CRYPTO_BASE + num; 698 pa->attr.attr.name = paicrypt_ctrnames[num]; 699 pa->attr.attr.mode = 0444; 700 pa->attr.show = cpumf_events_sysfs_show; 701 pa->attr.store = NULL; 702 attrs[num] = &pa->attr.attr; 703 return 0; 704 } 705 706 /* Create PMU sysfs event attributes on the fly. */ 707 static int __init attr_event_init(void) 708 { 709 struct attribute **attrs; 710 int ret, i; 711 712 attrs = kmalloc_array(ARRAY_SIZE(paicrypt_ctrnames) + 1, sizeof(*attrs), 713 GFP_KERNEL); 714 if (!attrs) 715 return -ENOMEM; 716 for (i = 0; i < ARRAY_SIZE(paicrypt_ctrnames); i++) { 717 ret = attr_event_init_one(attrs, i); 718 if (ret) { 719 attr_event_free(attrs, i - 1); 720 return ret; 721 } 722 } 723 attrs[i] = NULL; 724 paicrypt_events_group.attrs = attrs; 725 return 0; 726 } 727 728 static int __init paicrypt_init(void) 729 { 730 struct qpaci_info_block ib; 731 int rc; 732 733 if (!test_facility(196)) 734 return 0; 735 736 qpaci(&ib); 737 paicrypt_cnt = ib.num_cc; 738 if (paicrypt_cnt == 0) 739 return 0; 740 if (paicrypt_cnt >= PAI_CRYPTO_MAXCTR) 741 paicrypt_cnt = PAI_CRYPTO_MAXCTR - 1; 742 743 rc = attr_event_init(); /* Export known PAI crypto events */ 744 if (rc) { 745 pr_err("Creation of PMU pai_crypto /sysfs failed\n"); 746 return rc; 747 } 748 749 /* Setup s390dbf facility */ 750 cfm_dbg = debug_register(KMSG_COMPONENT, 2, 256, 128); 751 if (!cfm_dbg) { 752 pr_err("Registration of s390dbf pai_crypto failed\n"); 753 return -ENOMEM; 754 } 755 debug_register_view(cfm_dbg, &debug_sprintf_view); 756 757 rc = perf_pmu_register(&paicrypt, "pai_crypto", -1); 758 if (rc) { 759 pr_err("Registering the pai_crypto PMU failed with rc=%i\n", 760 rc); 761 debug_unregister_view(cfm_dbg, &debug_sprintf_view); 762 debug_unregister(cfm_dbg); 763 return rc; 764 } 765 return 0; 766 } 767 768 device_initcall(paicrypt_init); 769