1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Performance event support - Processor Activity Instrumentation Facility 4 * 5 * Copyright IBM Corp. 2022 6 * Author(s): Thomas Richter <tmricht@linux.ibm.com> 7 */ 8 #define KMSG_COMPONENT "pai_crypto" 9 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt 10 11 #include <linux/kernel.h> 12 #include <linux/kernel_stat.h> 13 #include <linux/percpu.h> 14 #include <linux/notifier.h> 15 #include <linux/init.h> 16 #include <linux/io.h> 17 #include <linux/perf_event.h> 18 #include <asm/ctlreg.h> 19 #include <asm/pai.h> 20 #include <asm/debug.h> 21 22 static debug_info_t *cfm_dbg; 23 static unsigned int paicrypt_cnt; /* Size of the mapped counter sets */ 24 /* extracted with QPACI instruction */ 25 26 DEFINE_STATIC_KEY_FALSE(pai_key); 27 28 struct pai_userdata { 29 u16 num; 30 u64 value; 31 } __packed; 32 33 struct paicrypt_map { 34 unsigned long *page; /* Page for CPU to store counters */ 35 struct pai_userdata *save; /* Page to store no-zero counters */ 36 unsigned int active_events; /* # of PAI crypto users */ 37 refcount_t refcnt; /* Reference count mapped buffers */ 38 struct perf_event *event; /* Perf event for sampling */ 39 struct list_head syswide_list; /* List system-wide sampling events */ 40 }; 41 42 struct paicrypt_mapptr { 43 struct paicrypt_map *mapptr; 44 }; 45 46 static struct paicrypt_root { /* Anchor to per CPU data */ 47 refcount_t refcnt; /* Overall active events */ 48 struct paicrypt_mapptr __percpu *mapptr; 49 } paicrypt_root; 50 51 /* Free per CPU data when the last event is removed. */ 52 static void paicrypt_root_free(void) 53 { 54 if (refcount_dec_and_test(&paicrypt_root.refcnt)) { 55 free_percpu(paicrypt_root.mapptr); 56 paicrypt_root.mapptr = NULL; 57 } 58 debug_sprintf_event(cfm_dbg, 5, "%s root.refcount %d\n", __func__, 59 refcount_read(&paicrypt_root.refcnt)); 60 } 61 62 /* 63 * On initialization of first event also allocate per CPU data dynamically. 64 * Start with an array of pointers, the array size is the maximum number of 65 * CPUs possible, which might be larger than the number of CPUs currently 66 * online. 67 */ 68 static int paicrypt_root_alloc(void) 69 { 70 if (!refcount_inc_not_zero(&paicrypt_root.refcnt)) { 71 /* The memory is already zeroed. */ 72 paicrypt_root.mapptr = alloc_percpu(struct paicrypt_mapptr); 73 if (!paicrypt_root.mapptr) 74 return -ENOMEM; 75 refcount_set(&paicrypt_root.refcnt, 1); 76 } 77 return 0; 78 } 79 80 /* Release the PMU if event is the last perf event */ 81 static DEFINE_MUTEX(pai_reserve_mutex); 82 83 /* Adjust usage counters and remove allocated memory when all users are 84 * gone. 85 */ 86 static void paicrypt_event_destroy_cpu(struct perf_event *event, int cpu) 87 { 88 struct paicrypt_mapptr *mp = per_cpu_ptr(paicrypt_root.mapptr, cpu); 89 struct paicrypt_map *cpump = mp->mapptr; 90 91 mutex_lock(&pai_reserve_mutex); 92 debug_sprintf_event(cfm_dbg, 5, "%s event %#llx cpu %d users %d " 93 "refcnt %u\n", __func__, event->attr.config, 94 event->cpu, cpump->active_events, 95 refcount_read(&cpump->refcnt)); 96 if (refcount_dec_and_test(&cpump->refcnt)) { 97 debug_sprintf_event(cfm_dbg, 4, "%s page %#lx save %p\n", 98 __func__, (unsigned long)cpump->page, 99 cpump->save); 100 free_page((unsigned long)cpump->page); 101 kvfree(cpump->save); 102 kfree(cpump); 103 mp->mapptr = NULL; 104 } 105 paicrypt_root_free(); 106 mutex_unlock(&pai_reserve_mutex); 107 } 108 109 static void paicrypt_event_destroy(struct perf_event *event) 110 { 111 int cpu; 112 113 static_branch_dec(&pai_key); 114 free_page(PAI_SAVE_AREA(event)); 115 if (event->cpu == -1) { 116 struct cpumask *mask = PAI_CPU_MASK(event); 117 118 for_each_cpu(cpu, mask) 119 paicrypt_event_destroy_cpu(event, cpu); 120 kfree(mask); 121 } else { 122 paicrypt_event_destroy_cpu(event, event->cpu); 123 } 124 } 125 126 static u64 paicrypt_getctr(unsigned long *page, int nr, bool kernel) 127 { 128 if (kernel) 129 nr += PAI_CRYPTO_MAXCTR; 130 return page[nr]; 131 } 132 133 /* Read the counter values. Return value from location in CMP. For event 134 * CRYPTO_ALL sum up all events. 135 */ 136 static u64 paicrypt_getdata(struct perf_event *event, bool kernel) 137 { 138 struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); 139 struct paicrypt_map *cpump = mp->mapptr; 140 u64 sum = 0; 141 int i; 142 143 if (event->attr.config != PAI_CRYPTO_BASE) { 144 return paicrypt_getctr(cpump->page, 145 event->attr.config - PAI_CRYPTO_BASE, 146 kernel); 147 } 148 149 for (i = 1; i <= paicrypt_cnt; i++) { 150 u64 val = paicrypt_getctr(cpump->page, i, kernel); 151 152 if (!val) 153 continue; 154 sum += val; 155 } 156 return sum; 157 } 158 159 static u64 paicrypt_getall(struct perf_event *event) 160 { 161 u64 sum = 0; 162 163 if (!event->attr.exclude_kernel) 164 sum += paicrypt_getdata(event, true); 165 if (!event->attr.exclude_user) 166 sum += paicrypt_getdata(event, false); 167 168 return sum; 169 } 170 171 /* Check concurrent access of counting and sampling for crypto events. 172 * This function is called in process context and it is save to block. 173 * When the event initialization functions fails, no other call back will 174 * be invoked. 175 * 176 * Allocate the memory for the event. 177 */ 178 static struct paicrypt_map *paicrypt_busy(struct perf_event *event, int cpu) 179 { 180 struct paicrypt_map *cpump = NULL; 181 struct paicrypt_mapptr *mp; 182 int rc; 183 184 mutex_lock(&pai_reserve_mutex); 185 186 /* Allocate root node */ 187 rc = paicrypt_root_alloc(); 188 if (rc) 189 goto unlock; 190 191 /* Allocate node for this event */ 192 mp = per_cpu_ptr(paicrypt_root.mapptr, cpu); 193 cpump = mp->mapptr; 194 if (!cpump) { /* Paicrypt_map allocated? */ 195 cpump = kzalloc(sizeof(*cpump), GFP_KERNEL); 196 if (!cpump) { 197 rc = -ENOMEM; 198 goto free_root; 199 } 200 INIT_LIST_HEAD(&cpump->syswide_list); 201 } 202 203 /* Allocate memory for counter page and counter extraction. 204 * Only the first counting event has to allocate a page. 205 */ 206 if (cpump->page) { 207 refcount_inc(&cpump->refcnt); 208 goto unlock; 209 } 210 211 rc = -ENOMEM; 212 cpump->page = (unsigned long *)get_zeroed_page(GFP_KERNEL); 213 if (!cpump->page) 214 goto free_paicrypt_map; 215 cpump->save = kvmalloc_array(paicrypt_cnt + 1, 216 sizeof(struct pai_userdata), GFP_KERNEL); 217 if (!cpump->save) { 218 free_page((unsigned long)cpump->page); 219 cpump->page = NULL; 220 goto free_paicrypt_map; 221 } 222 223 /* Set mode and reference count */ 224 rc = 0; 225 refcount_set(&cpump->refcnt, 1); 226 mp->mapptr = cpump; 227 debug_sprintf_event(cfm_dbg, 5, "%s users %d refcnt %u page %#lx " 228 "save %p rc %d\n", __func__, cpump->active_events, 229 refcount_read(&cpump->refcnt), 230 (unsigned long)cpump->page, cpump->save, rc); 231 goto unlock; 232 233 free_paicrypt_map: 234 /* Undo memory allocation */ 235 kfree(cpump); 236 mp->mapptr = NULL; 237 free_root: 238 paicrypt_root_free(); 239 unlock: 240 mutex_unlock(&pai_reserve_mutex); 241 return rc ? ERR_PTR(rc) : cpump; 242 } 243 244 static int paicrypt_event_init_all(struct perf_event *event) 245 { 246 struct paicrypt_map *cpump; 247 struct cpumask *maskptr; 248 int cpu, rc = -ENOMEM; 249 250 maskptr = kzalloc(sizeof(*maskptr), GFP_KERNEL); 251 if (!maskptr) 252 goto out; 253 254 for_each_online_cpu(cpu) { 255 cpump = paicrypt_busy(event, cpu); 256 if (IS_ERR(cpump)) { 257 for_each_cpu(cpu, maskptr) 258 paicrypt_event_destroy_cpu(event, cpu); 259 kfree(maskptr); 260 rc = PTR_ERR(cpump); 261 goto out; 262 } 263 cpumask_set_cpu(cpu, maskptr); 264 } 265 266 /* 267 * On error all cpumask are freed and all events have been destroyed. 268 * Save of which CPUs data structures have been allocated for. 269 * Release them in paicrypt_event_destroy call back function 270 * for this event. 271 */ 272 PAI_CPU_MASK(event) = maskptr; 273 rc = 0; 274 out: 275 return rc; 276 } 277 278 /* Might be called on different CPU than the one the event is intended for. */ 279 static int paicrypt_event_init(struct perf_event *event) 280 { 281 struct perf_event_attr *a = &event->attr; 282 struct paicrypt_map *cpump; 283 int rc = 0; 284 285 /* PAI crypto PMU registered as PERF_TYPE_RAW, check event type */ 286 if (a->type != PERF_TYPE_RAW && event->pmu->type != a->type) 287 return -ENOENT; 288 /* PAI crypto event must be in valid range */ 289 if (a->config < PAI_CRYPTO_BASE || 290 a->config > PAI_CRYPTO_BASE + paicrypt_cnt) 291 return -EINVAL; 292 /* Allow only CRYPTO_ALL for sampling */ 293 if (a->sample_period && a->config != PAI_CRYPTO_BASE) 294 return -EINVAL; 295 /* Get a page to store last counter values for sampling */ 296 if (a->sample_period) { 297 PAI_SAVE_AREA(event) = get_zeroed_page(GFP_KERNEL); 298 if (!PAI_SAVE_AREA(event)) { 299 rc = -ENOMEM; 300 goto out; 301 } 302 } 303 304 if (event->cpu >= 0) { 305 cpump = paicrypt_busy(event, event->cpu); 306 if (IS_ERR(cpump)) 307 rc = PTR_ERR(cpump); 308 } else { 309 rc = paicrypt_event_init_all(event); 310 } 311 if (rc) { 312 free_page(PAI_SAVE_AREA(event)); 313 goto out; 314 } 315 event->destroy = paicrypt_event_destroy; 316 317 if (a->sample_period) { 318 a->sample_period = 1; 319 a->freq = 0; 320 /* Register for paicrypt_sched_task() to be called */ 321 event->attach_state |= PERF_ATTACH_SCHED_CB; 322 /* Add raw data which contain the memory mapped counters */ 323 a->sample_type |= PERF_SAMPLE_RAW; 324 /* Turn off inheritance */ 325 a->inherit = 0; 326 } 327 328 static_branch_inc(&pai_key); 329 out: 330 return rc; 331 } 332 333 static void paicrypt_read(struct perf_event *event) 334 { 335 u64 prev, new, delta; 336 337 prev = local64_read(&event->hw.prev_count); 338 new = paicrypt_getall(event); 339 local64_set(&event->hw.prev_count, new); 340 delta = (prev <= new) ? new - prev 341 : (-1ULL - prev) + new + 1; /* overflow */ 342 local64_add(delta, &event->count); 343 } 344 345 static void paicrypt_start(struct perf_event *event, int flags) 346 { 347 struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); 348 struct paicrypt_map *cpump = mp->mapptr; 349 u64 sum; 350 351 if (!event->attr.sample_period) { /* Counting */ 352 sum = paicrypt_getall(event); /* Get current value */ 353 local64_set(&event->hw.prev_count, sum); 354 } else { /* Sampling */ 355 memcpy((void *)PAI_SAVE_AREA(event), cpump->page, PAGE_SIZE); 356 /* Enable context switch callback for system-wide sampling */ 357 if (!(event->attach_state & PERF_ATTACH_TASK)) { 358 list_add_tail(PAI_SWLIST(event), &cpump->syswide_list); 359 perf_sched_cb_inc(event->pmu); 360 } else { 361 cpump->event = event; 362 } 363 } 364 } 365 366 static int paicrypt_add(struct perf_event *event, int flags) 367 { 368 struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); 369 struct paicrypt_map *cpump = mp->mapptr; 370 unsigned long ccd; 371 372 if (++cpump->active_events == 1) { 373 ccd = virt_to_phys(cpump->page) | PAI_CRYPTO_KERNEL_OFFSET; 374 WRITE_ONCE(get_lowcore()->ccd, ccd); 375 local_ctl_set_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT); 376 } 377 if (flags & PERF_EF_START) 378 paicrypt_start(event, PERF_EF_RELOAD); 379 event->hw.state = 0; 380 return 0; 381 } 382 383 static void paicrypt_have_sample(struct perf_event *, struct paicrypt_map *); 384 static void paicrypt_stop(struct perf_event *event, int flags) 385 { 386 struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); 387 struct paicrypt_map *cpump = mp->mapptr; 388 389 if (!event->attr.sample_period) { /* Counting */ 390 paicrypt_read(event); 391 } else { /* Sampling */ 392 if (!(event->attach_state & PERF_ATTACH_TASK)) { 393 perf_sched_cb_dec(event->pmu); 394 list_del(PAI_SWLIST(event)); 395 } else { 396 paicrypt_have_sample(event, cpump); 397 cpump->event = NULL; 398 } 399 } 400 event->hw.state = PERF_HES_STOPPED; 401 } 402 403 static void paicrypt_del(struct perf_event *event, int flags) 404 { 405 struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); 406 struct paicrypt_map *cpump = mp->mapptr; 407 408 paicrypt_stop(event, PERF_EF_UPDATE); 409 if (--cpump->active_events == 0) { 410 local_ctl_clear_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT); 411 WRITE_ONCE(get_lowcore()->ccd, 0); 412 } 413 } 414 415 /* Create raw data and save it in buffer. Calculate the delta for each 416 * counter between this invocation and the last invocation. 417 * Returns number of bytes copied. 418 * Saves only entries with positive counter difference of the form 419 * 2 bytes: Number of counter 420 * 8 bytes: Value of counter 421 */ 422 static size_t paicrypt_copy(struct pai_userdata *userdata, unsigned long *page, 423 unsigned long *page_old, bool exclude_user, 424 bool exclude_kernel) 425 { 426 int i, outidx = 0; 427 428 for (i = 1; i <= paicrypt_cnt; i++) { 429 u64 val = 0, val_old = 0; 430 431 if (!exclude_kernel) { 432 val += paicrypt_getctr(page, i, true); 433 val_old += paicrypt_getctr(page_old, i, true); 434 } 435 if (!exclude_user) { 436 val += paicrypt_getctr(page, i, false); 437 val_old += paicrypt_getctr(page_old, i, false); 438 } 439 if (val >= val_old) 440 val -= val_old; 441 else 442 val = (~0ULL - val_old) + val + 1; 443 if (val) { 444 userdata[outidx].num = i; 445 userdata[outidx].value = val; 446 outidx++; 447 } 448 } 449 return outidx * sizeof(struct pai_userdata); 450 } 451 452 static int paicrypt_push_sample(size_t rawsize, struct paicrypt_map *cpump, 453 struct perf_event *event) 454 { 455 struct perf_sample_data data; 456 struct perf_raw_record raw; 457 struct pt_regs regs; 458 int overflow; 459 460 /* Setup perf sample */ 461 memset(®s, 0, sizeof(regs)); 462 memset(&raw, 0, sizeof(raw)); 463 memset(&data, 0, sizeof(data)); 464 perf_sample_data_init(&data, 0, event->hw.last_period); 465 if (event->attr.sample_type & PERF_SAMPLE_TID) { 466 data.tid_entry.pid = task_tgid_nr(current); 467 data.tid_entry.tid = task_pid_nr(current); 468 } 469 if (event->attr.sample_type & PERF_SAMPLE_TIME) 470 data.time = event->clock(); 471 if (event->attr.sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER)) 472 data.id = event->id; 473 if (event->attr.sample_type & PERF_SAMPLE_CPU) { 474 data.cpu_entry.cpu = smp_processor_id(); 475 data.cpu_entry.reserved = 0; 476 } 477 if (event->attr.sample_type & PERF_SAMPLE_RAW) { 478 raw.frag.size = rawsize; 479 raw.frag.data = cpump->save; 480 perf_sample_save_raw_data(&data, event, &raw); 481 } 482 483 overflow = perf_event_overflow(event, &data, ®s); 484 perf_event_update_userpage(event); 485 /* Save crypto counter lowcore page after reading event data. */ 486 memcpy((void *)PAI_SAVE_AREA(event), cpump->page, PAGE_SIZE); 487 return overflow; 488 } 489 490 /* Check if there is data to be saved on schedule out of a task. */ 491 static void paicrypt_have_sample(struct perf_event *event, 492 struct paicrypt_map *cpump) 493 { 494 size_t rawsize; 495 496 if (!event) /* No event active */ 497 return; 498 rawsize = paicrypt_copy(cpump->save, cpump->page, 499 (unsigned long *)PAI_SAVE_AREA(event), 500 event->attr.exclude_user, 501 event->attr.exclude_kernel); 502 if (rawsize) /* No incremented counters */ 503 paicrypt_push_sample(rawsize, cpump, event); 504 } 505 506 /* Check if there is data to be saved on schedule out of a task. */ 507 static void paicrypt_have_samples(void) 508 { 509 struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); 510 struct paicrypt_map *cpump = mp->mapptr; 511 struct perf_event *event; 512 513 list_for_each_entry(event, &cpump->syswide_list, hw.tp_list) 514 paicrypt_have_sample(event, cpump); 515 } 516 517 /* Called on schedule-in and schedule-out. No access to event structure, 518 * but for sampling only event CRYPTO_ALL is allowed. 519 */ 520 static void paicrypt_sched_task(struct perf_event_pmu_context *pmu_ctx, 521 struct task_struct *task, bool sched_in) 522 { 523 /* We started with a clean page on event installation. So read out 524 * results on schedule_out and if page was dirty, save old values. 525 */ 526 if (!sched_in) 527 paicrypt_have_samples(); 528 } 529 530 /* Attribute definitions for paicrypt interface. As with other CPU 531 * Measurement Facilities, there is one attribute per mapped counter. 532 * The number of mapped counters may vary per machine generation. Use 533 * the QUERY PROCESSOR ACTIVITY COUNTER INFORMATION (QPACI) instruction 534 * to determine the number of mapped counters. The instructions returns 535 * a positive number, which is the highest number of supported counters. 536 * All counters less than this number are also supported, there are no 537 * holes. A returned number of zero means no support for mapped counters. 538 * 539 * The identification of the counter is a unique number. The chosen range 540 * is 0x1000 + offset in mapped kernel page. 541 * All CPU Measurement Facility counters identifiers must be unique and 542 * the numbers from 0 to 496 are already used for the CPU Measurement 543 * Counter facility. Numbers 0xb0000, 0xbc000 and 0xbd000 are already 544 * used for the CPU Measurement Sampling facility. 545 */ 546 PMU_FORMAT_ATTR(event, "config:0-63"); 547 548 static struct attribute *paicrypt_format_attr[] = { 549 &format_attr_event.attr, 550 NULL, 551 }; 552 553 static struct attribute_group paicrypt_events_group = { 554 .name = "events", 555 .attrs = NULL /* Filled in attr_event_init() */ 556 }; 557 558 static struct attribute_group paicrypt_format_group = { 559 .name = "format", 560 .attrs = paicrypt_format_attr, 561 }; 562 563 static const struct attribute_group *paicrypt_attr_groups[] = { 564 &paicrypt_events_group, 565 &paicrypt_format_group, 566 NULL, 567 }; 568 569 /* Performance monitoring unit for mapped counters */ 570 static struct pmu paicrypt = { 571 .task_ctx_nr = perf_hw_context, 572 .event_init = paicrypt_event_init, 573 .add = paicrypt_add, 574 .del = paicrypt_del, 575 .start = paicrypt_start, 576 .stop = paicrypt_stop, 577 .read = paicrypt_read, 578 .sched_task = paicrypt_sched_task, 579 .attr_groups = paicrypt_attr_groups 580 }; 581 582 /* List of symbolic PAI counter names. */ 583 static const char * const paicrypt_ctrnames[] = { 584 [0] = "CRYPTO_ALL", 585 [1] = "KM_DEA", 586 [2] = "KM_TDEA_128", 587 [3] = "KM_TDEA_192", 588 [4] = "KM_ENCRYPTED_DEA", 589 [5] = "KM_ENCRYPTED_TDEA_128", 590 [6] = "KM_ENCRYPTED_TDEA_192", 591 [7] = "KM_AES_128", 592 [8] = "KM_AES_192", 593 [9] = "KM_AES_256", 594 [10] = "KM_ENCRYPTED_AES_128", 595 [11] = "KM_ENCRYPTED_AES_192", 596 [12] = "KM_ENCRYPTED_AES_256", 597 [13] = "KM_XTS_AES_128", 598 [14] = "KM_XTS_AES_256", 599 [15] = "KM_XTS_ENCRYPTED_AES_128", 600 [16] = "KM_XTS_ENCRYPTED_AES_256", 601 [17] = "KMC_DEA", 602 [18] = "KMC_TDEA_128", 603 [19] = "KMC_TDEA_192", 604 [20] = "KMC_ENCRYPTED_DEA", 605 [21] = "KMC_ENCRYPTED_TDEA_128", 606 [22] = "KMC_ENCRYPTED_TDEA_192", 607 [23] = "KMC_AES_128", 608 [24] = "KMC_AES_192", 609 [25] = "KMC_AES_256", 610 [26] = "KMC_ENCRYPTED_AES_128", 611 [27] = "KMC_ENCRYPTED_AES_192", 612 [28] = "KMC_ENCRYPTED_AES_256", 613 [29] = "KMC_PRNG", 614 [30] = "KMA_GCM_AES_128", 615 [31] = "KMA_GCM_AES_192", 616 [32] = "KMA_GCM_AES_256", 617 [33] = "KMA_GCM_ENCRYPTED_AES_128", 618 [34] = "KMA_GCM_ENCRYPTED_AES_192", 619 [35] = "KMA_GCM_ENCRYPTED_AES_256", 620 [36] = "KMF_DEA", 621 [37] = "KMF_TDEA_128", 622 [38] = "KMF_TDEA_192", 623 [39] = "KMF_ENCRYPTED_DEA", 624 [40] = "KMF_ENCRYPTED_TDEA_128", 625 [41] = "KMF_ENCRYPTED_TDEA_192", 626 [42] = "KMF_AES_128", 627 [43] = "KMF_AES_192", 628 [44] = "KMF_AES_256", 629 [45] = "KMF_ENCRYPTED_AES_128", 630 [46] = "KMF_ENCRYPTED_AES_192", 631 [47] = "KMF_ENCRYPTED_AES_256", 632 [48] = "KMCTR_DEA", 633 [49] = "KMCTR_TDEA_128", 634 [50] = "KMCTR_TDEA_192", 635 [51] = "KMCTR_ENCRYPTED_DEA", 636 [52] = "KMCTR_ENCRYPTED_TDEA_128", 637 [53] = "KMCTR_ENCRYPTED_TDEA_192", 638 [54] = "KMCTR_AES_128", 639 [55] = "KMCTR_AES_192", 640 [56] = "KMCTR_AES_256", 641 [57] = "KMCTR_ENCRYPTED_AES_128", 642 [58] = "KMCTR_ENCRYPTED_AES_192", 643 [59] = "KMCTR_ENCRYPTED_AES_256", 644 [60] = "KMO_DEA", 645 [61] = "KMO_TDEA_128", 646 [62] = "KMO_TDEA_192", 647 [63] = "KMO_ENCRYPTED_DEA", 648 [64] = "KMO_ENCRYPTED_TDEA_128", 649 [65] = "KMO_ENCRYPTED_TDEA_192", 650 [66] = "KMO_AES_128", 651 [67] = "KMO_AES_192", 652 [68] = "KMO_AES_256", 653 [69] = "KMO_ENCRYPTED_AES_128", 654 [70] = "KMO_ENCRYPTED_AES_192", 655 [71] = "KMO_ENCRYPTED_AES_256", 656 [72] = "KIMD_SHA_1", 657 [73] = "KIMD_SHA_256", 658 [74] = "KIMD_SHA_512", 659 [75] = "KIMD_SHA3_224", 660 [76] = "KIMD_SHA3_256", 661 [77] = "KIMD_SHA3_384", 662 [78] = "KIMD_SHA3_512", 663 [79] = "KIMD_SHAKE_128", 664 [80] = "KIMD_SHAKE_256", 665 [81] = "KIMD_GHASH", 666 [82] = "KLMD_SHA_1", 667 [83] = "KLMD_SHA_256", 668 [84] = "KLMD_SHA_512", 669 [85] = "KLMD_SHA3_224", 670 [86] = "KLMD_SHA3_256", 671 [87] = "KLMD_SHA3_384", 672 [88] = "KLMD_SHA3_512", 673 [89] = "KLMD_SHAKE_128", 674 [90] = "KLMD_SHAKE_256", 675 [91] = "KMAC_DEA", 676 [92] = "KMAC_TDEA_128", 677 [93] = "KMAC_TDEA_192", 678 [94] = "KMAC_ENCRYPTED_DEA", 679 [95] = "KMAC_ENCRYPTED_TDEA_128", 680 [96] = "KMAC_ENCRYPTED_TDEA_192", 681 [97] = "KMAC_AES_128", 682 [98] = "KMAC_AES_192", 683 [99] = "KMAC_AES_256", 684 [100] = "KMAC_ENCRYPTED_AES_128", 685 [101] = "KMAC_ENCRYPTED_AES_192", 686 [102] = "KMAC_ENCRYPTED_AES_256", 687 [103] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_DEA", 688 [104] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_TDEA_128", 689 [105] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_TDEA_192", 690 [106] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_DEA", 691 [107] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_TDEA_128", 692 [108] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_TDEA_192", 693 [109] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_128", 694 [110] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_192", 695 [111] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_256", 696 [112] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_128", 697 [113] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_192", 698 [114] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_256", 699 [115] = "PCC_COMPUTE_XTS_PARAMETER_USING_AES_128", 700 [116] = "PCC_COMPUTE_XTS_PARAMETER_USING_AES_256", 701 [117] = "PCC_COMPUTE_XTS_PARAMETER_USING_ENCRYPTED_AES_128", 702 [118] = "PCC_COMPUTE_XTS_PARAMETER_USING_ENCRYPTED_AES_256", 703 [119] = "PCC_SCALAR_MULTIPLY_P256", 704 [120] = "PCC_SCALAR_MULTIPLY_P384", 705 [121] = "PCC_SCALAR_MULTIPLY_P521", 706 [122] = "PCC_SCALAR_MULTIPLY_ED25519", 707 [123] = "PCC_SCALAR_MULTIPLY_ED448", 708 [124] = "PCC_SCALAR_MULTIPLY_X25519", 709 [125] = "PCC_SCALAR_MULTIPLY_X448", 710 [126] = "PRNO_SHA_512_DRNG", 711 [127] = "PRNO_TRNG_QUERY_RAW_TO_CONDITIONED_RATIO", 712 [128] = "PRNO_TRNG", 713 [129] = "KDSA_ECDSA_VERIFY_P256", 714 [130] = "KDSA_ECDSA_VERIFY_P384", 715 [131] = "KDSA_ECDSA_VERIFY_P521", 716 [132] = "KDSA_ECDSA_SIGN_P256", 717 [133] = "KDSA_ECDSA_SIGN_P384", 718 [134] = "KDSA_ECDSA_SIGN_P521", 719 [135] = "KDSA_ENCRYPTED_ECDSA_SIGN_P256", 720 [136] = "KDSA_ENCRYPTED_ECDSA_SIGN_P384", 721 [137] = "KDSA_ENCRYPTED_ECDSA_SIGN_P521", 722 [138] = "KDSA_EDDSA_VERIFY_ED25519", 723 [139] = "KDSA_EDDSA_VERIFY_ED448", 724 [140] = "KDSA_EDDSA_SIGN_ED25519", 725 [141] = "KDSA_EDDSA_SIGN_ED448", 726 [142] = "KDSA_ENCRYPTED_EDDSA_SIGN_ED25519", 727 [143] = "KDSA_ENCRYPTED_EDDSA_SIGN_ED448", 728 [144] = "PCKMO_ENCRYPT_DEA_KEY", 729 [145] = "PCKMO_ENCRYPT_TDEA_128_KEY", 730 [146] = "PCKMO_ENCRYPT_TDEA_192_KEY", 731 [147] = "PCKMO_ENCRYPT_AES_128_KEY", 732 [148] = "PCKMO_ENCRYPT_AES_192_KEY", 733 [149] = "PCKMO_ENCRYPT_AES_256_KEY", 734 [150] = "PCKMO_ENCRYPT_ECC_P256_KEY", 735 [151] = "PCKMO_ENCRYPT_ECC_P384_KEY", 736 [152] = "PCKMO_ENCRYPT_ECC_P521_KEY", 737 [153] = "PCKMO_ENCRYPT_ECC_ED25519_KEY", 738 [154] = "PCKMO_ENCRYPT_ECC_ED448_KEY", 739 [155] = "IBM_RESERVED_155", 740 [156] = "IBM_RESERVED_156", 741 [157] = "KM_FULL_XTS_AES_128", 742 [158] = "KM_FULL_XTS_AES_256", 743 [159] = "KM_FULL_XTS_ENCRYPTED_AES_128", 744 [160] = "KM_FULL_XTS_ENCRYPTED_AES_256", 745 [161] = "KMAC_HMAC_SHA_224", 746 [162] = "KMAC_HMAC_SHA_256", 747 [163] = "KMAC_HMAC_SHA_384", 748 [164] = "KMAC_HMAC_SHA_512", 749 [165] = "KMAC_HMAC_ENCRYPTED_SHA_224", 750 [166] = "KMAC_HMAC_ENCRYPTED_SHA_256", 751 [167] = "KMAC_HMAC_ENCRYPTED_SHA_384", 752 [168] = "KMAC_HMAC_ENCRYPTED_SHA_512", 753 [169] = "PCKMO_ENCRYPT_HMAC_512_KEY", 754 [170] = "PCKMO_ENCRYPT_HMAC_1024_KEY", 755 [171] = "PCKMO_ENCRYPT_AES_XTS_128", 756 [172] = "PCKMO_ENCRYPT_AES_XTS_256", 757 }; 758 759 static void __init attr_event_free(struct attribute **attrs, int num) 760 { 761 struct perf_pmu_events_attr *pa; 762 int i; 763 764 for (i = 0; i < num; i++) { 765 struct device_attribute *dap; 766 767 dap = container_of(attrs[i], struct device_attribute, attr); 768 pa = container_of(dap, struct perf_pmu_events_attr, attr); 769 kfree(pa); 770 } 771 kfree(attrs); 772 } 773 774 static int __init attr_event_init_one(struct attribute **attrs, int num) 775 { 776 struct perf_pmu_events_attr *pa; 777 778 /* Index larger than array_size, no counter name available */ 779 if (num >= ARRAY_SIZE(paicrypt_ctrnames)) { 780 attrs[num] = NULL; 781 return 0; 782 } 783 784 pa = kzalloc(sizeof(*pa), GFP_KERNEL); 785 if (!pa) 786 return -ENOMEM; 787 788 sysfs_attr_init(&pa->attr.attr); 789 pa->id = PAI_CRYPTO_BASE + num; 790 pa->attr.attr.name = paicrypt_ctrnames[num]; 791 pa->attr.attr.mode = 0444; 792 pa->attr.show = cpumf_events_sysfs_show; 793 pa->attr.store = NULL; 794 attrs[num] = &pa->attr.attr; 795 return 0; 796 } 797 798 /* Create PMU sysfs event attributes on the fly. */ 799 static int __init attr_event_init(void) 800 { 801 struct attribute **attrs; 802 int ret, i; 803 804 attrs = kmalloc_array(paicrypt_cnt + 2, sizeof(*attrs), GFP_KERNEL); 805 if (!attrs) 806 return -ENOMEM; 807 for (i = 0; i <= paicrypt_cnt; i++) { 808 ret = attr_event_init_one(attrs, i); 809 if (ret) { 810 attr_event_free(attrs, i); 811 return ret; 812 } 813 } 814 attrs[i] = NULL; 815 paicrypt_events_group.attrs = attrs; 816 return 0; 817 } 818 819 static int __init paicrypt_init(void) 820 { 821 struct qpaci_info_block ib; 822 int rc; 823 824 if (!test_facility(196)) 825 return 0; 826 827 qpaci(&ib); 828 paicrypt_cnt = ib.num_cc; 829 if (paicrypt_cnt == 0) 830 return 0; 831 if (paicrypt_cnt >= PAI_CRYPTO_MAXCTR) { 832 pr_err("Too many PMU pai_crypto counters %d\n", paicrypt_cnt); 833 return -E2BIG; 834 } 835 836 rc = attr_event_init(); /* Export known PAI crypto events */ 837 if (rc) { 838 pr_err("Creation of PMU pai_crypto /sysfs failed\n"); 839 return rc; 840 } 841 842 /* Setup s390dbf facility */ 843 cfm_dbg = debug_register(KMSG_COMPONENT, 2, 256, 128); 844 if (!cfm_dbg) { 845 pr_err("Registration of s390dbf pai_crypto failed\n"); 846 return -ENOMEM; 847 } 848 debug_register_view(cfm_dbg, &debug_sprintf_view); 849 850 rc = perf_pmu_register(&paicrypt, "pai_crypto", -1); 851 if (rc) { 852 pr_err("Registering the pai_crypto PMU failed with rc=%i\n", 853 rc); 854 debug_unregister_view(cfm_dbg, &debug_sprintf_view); 855 debug_unregister(cfm_dbg); 856 return rc; 857 } 858 return 0; 859 } 860 861 device_initcall(paicrypt_init); 862