1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Performance event support - Processor Activity Instrumentation Facility 4 * 5 * Copyright IBM Corp. 2026 6 * Author(s): Thomas Richter <tmricht@linux.ibm.com> 7 */ 8 #define pr_fmt(fmt) "pai: " fmt 9 10 #include <linux/kernel.h> 11 #include <linux/kernel_stat.h> 12 #include <linux/percpu.h> 13 #include <linux/notifier.h> 14 #include <linux/init.h> 15 #include <linux/io.h> 16 #include <linux/perf_event.h> 17 #include <asm/ctlreg.h> 18 #include <asm/pai.h> 19 #include <asm/debug.h> 20 21 static debug_info_t *paidbg; 22 23 DEFINE_STATIC_KEY_FALSE(pai_key); 24 25 enum { 26 PAI_PMU_CRYPTO, /* Index of PMU pai_crypto */ 27 PAI_PMU_EXT, /* Index of PMU pai_ext */ 28 PAI_PMU_MAX /* # of PAI PMUs */ 29 }; 30 31 enum { 32 PAIE1_CB_SZ = 0x200, /* Size of PAIE1 control block */ 33 PAIE1_CTRBLOCK_SZ = 0x400 /* Size of PAIE1 counter blocks */ 34 }; 35 36 struct pai_userdata { 37 u16 num; 38 u64 value; 39 } __packed; 40 41 /* Create the PAI extension 1 control block area. 42 * The PAI extension control block 1 is pointed to by lowcore 43 * address 0x1508 for each CPU. This control block is 512 bytes in size 44 * and requires a 512 byte boundary alignment. 45 */ 46 struct paiext_cb { /* PAI extension 1 control block */ 47 u64 header; /* Not used */ 48 u64 reserved1; 49 u64 acc; /* Addr to analytics counter control block */ 50 u8 reserved2[PAIE1_CTRBLOCK_SZ - 3 * sizeof(u64)]; 51 } __packed; 52 53 struct pai_map { 54 unsigned long *area; /* Area for CPU to store counters */ 55 struct pai_userdata *save; /* Page to store no-zero counters */ 56 unsigned int active_events; /* # of PAI crypto users */ 57 refcount_t refcnt; /* Reference count mapped buffers */ 58 struct perf_event *event; /* Perf event for sampling */ 59 struct list_head syswide_list; /* List system-wide sampling events */ 60 struct paiext_cb *paiext_cb; /* PAI extension control block area */ 61 bool fullpage; /* True: counter area is a full page */ 62 }; 63 64 struct pai_mapptr { 65 struct pai_map *mapptr; 66 }; 67 68 static struct pai_root { /* Anchor to per CPU data */ 69 refcount_t refcnt; /* Overall active events */ 70 atomic_t tskctx; /* Overall per-task events */ 71 struct pai_mapptr __percpu *mapptr; 72 } pai_root[PAI_PMU_MAX]; 73 74 /* This table defines the different parameters of the PAI PMUs. During 75 * initialization the machine dependent values are extracted and saved. 76 * However most of the values are static and do not change. 77 * There is one table entry per PAI PMU. 78 */ 79 struct pai_pmu { /* Define PAI PMU characteristics */ 80 const char *pmuname; /* Name of PMU */ 81 const int facility_nr; /* Facility number to check for support */ 82 unsigned int num_avail; /* # Counters defined by hardware */ 83 unsigned int num_named; /* # Counters known by name */ 84 unsigned long base; /* Counter set base number */ 85 unsigned long kernel_offset; /* Offset to kernel part in counter page */ 86 unsigned long area_size; /* Size of counter area */ 87 const char * const *names; /* List of counter names */ 88 struct pmu *pmu; /* Ptr to supporting PMU */ 89 int (*init)(struct pai_pmu *p); /* PMU support init function */ 90 void (*exit)(struct pai_pmu *p); /* PMU support exit function */ 91 struct attribute_group *event_group; /* Ptr to attribute of events */ 92 }; 93 94 static struct pai_pmu pai_pmu[]; /* Forward declaration */ 95 96 /* Free per CPU data when the last event is removed. */ 97 static void pai_root_free(int idx, int tasks) 98 { 99 if (refcount_sub_and_test(tasks, &pai_root[idx].refcnt)) { 100 free_percpu(pai_root[idx].mapptr); 101 pai_root[idx].mapptr = NULL; 102 } 103 debug_sprintf_event(paidbg, 5, "%s root[%d].refcount %d tskctx %d\n", 104 __func__, idx, refcount_read(&pai_root[idx].refcnt), 105 atomic_read(&pai_root[idx].tskctx)); 106 } 107 108 /* 109 * On initialization of first event also allocate per CPU data dynamically. 110 * Start with an array of pointers, the array size is the maximum number of 111 * CPUs possible, which might be larger than the number of CPUs currently 112 * online. 113 */ 114 static int pai_root_alloc(int idx) 115 { 116 if (!refcount_inc_not_zero(&pai_root[idx].refcnt)) { 117 /* The memory is already zeroed. */ 118 pai_root[idx].mapptr = alloc_percpu(struct pai_mapptr); 119 if (!pai_root[idx].mapptr) 120 return -ENOMEM; 121 refcount_set(&pai_root[idx].refcnt, 1); 122 } 123 return 0; 124 } 125 126 /* Release the PMU if event is the last perf event */ 127 static DEFINE_MUTEX(pai_reserve_mutex); 128 129 /* Free all memory allocated for event counting/sampling setup */ 130 static void pai_free(struct pai_mapptr *mp) 131 { 132 if (mp->mapptr->fullpage) 133 free_page((unsigned long)mp->mapptr->area); 134 else 135 kfree(mp->mapptr->area); 136 kfree(mp->mapptr->paiext_cb); 137 kvfree(mp->mapptr->save); 138 kfree(mp->mapptr); 139 mp->mapptr = NULL; 140 } 141 142 /* Called under mutex_lock */ 143 static void pai_event_destroy_cpu(int idx, int cpu, bool hotplug) 144 { 145 struct pai_mapptr *mp; 146 struct pai_map *cpump; 147 int tasks = 1; 148 149 /* Check reference count and return when all gone. 150 * 1. An event is installed on online CPU X. 151 * 2. CPU x is offlined and the per-CPU data is removed. 152 * 3. Event is destroyed via close system call. 153 */ 154 if (!refcount_read(&pai_root[idx].refcnt)) 155 return; /* No events at all */ 156 mp = per_cpu_ptr(pai_root[idx].mapptr, cpu); 157 if (!mp || !mp->mapptr) /* No events on that CPU */ 158 return; 159 160 /* When hotplug is true, invocation is from CPU hotplug callback. 161 * Delete per-CPU resource and adjust refcnt when per-task events 162 * are currently active. This can be more than one. 163 * In this case adjust counters. 164 */ 165 if (hotplug) 166 tasks = atomic_read(&pai_root[idx].tskctx); 167 168 cpump = mp->mapptr; 169 if (refcount_sub_and_test(tasks, &cpump->refcnt)) 170 pai_free(mp); 171 pai_root_free(idx, tasks); 172 } 173 174 static void pai_event_destroy(struct perf_event *event) 175 { 176 int cpu = 0, idx = PAI_PMU_IDX(event); 177 178 free_page(PAI_SAVE_AREA(event)); 179 cpus_read_lock(); 180 mutex_lock(&pai_reserve_mutex); 181 if (event->cpu == -1) { 182 atomic_dec(&pai_root[idx].tskctx); 183 for_each_online_cpu(cpu) 184 pai_event_destroy_cpu(idx, cpu, false); 185 } else { 186 pai_event_destroy_cpu(idx, event->cpu, false); 187 } 188 mutex_unlock(&pai_reserve_mutex); 189 cpus_read_unlock(); 190 } 191 192 static void paicrypt_event_destroy(struct perf_event *event) 193 { 194 static_branch_dec(&pai_key); 195 pai_event_destroy(event); 196 } 197 198 static u64 pai_getctr(unsigned long *page, int nr, unsigned long offset) 199 { 200 if (offset) 201 nr += offset / sizeof(*page); 202 return page[nr]; 203 } 204 205 static void pai_setctr(unsigned long *page, int nr, unsigned long offset, u64 v) 206 { 207 if (offset) 208 nr += offset / sizeof(*page); 209 page[nr] = v; 210 } 211 212 /* Read the counter values. Return value from location in CMP. For base 213 * event xxx_ALL sum up all events. Returns counter value. 214 */ 215 static u64 pai_getdata(struct perf_event *event, bool kernel) 216 { 217 int idx = PAI_PMU_IDX(event); 218 struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr); 219 struct pai_pmu *pp = &pai_pmu[idx]; 220 struct pai_map *cpump = mp->mapptr; 221 unsigned int i; 222 u64 sum = 0; 223 224 if (event->attr.config != pp->base) { 225 return pai_getctr(cpump->area, 226 event->attr.config - pp->base, 227 kernel ? pp->kernel_offset : 0); 228 } 229 230 for (i = 1; i <= pp->num_avail; i++) { 231 u64 val = pai_getctr(cpump->area, i, 232 kernel ? pp->kernel_offset : 0); 233 234 if (!val) 235 continue; 236 sum += val; 237 } 238 return sum; 239 } 240 241 static u64 paicrypt_getall(struct perf_event *event) 242 { 243 u64 sum = 0; 244 245 if (!event->attr.exclude_kernel) 246 sum += pai_getdata(event, true); 247 if (!event->attr.exclude_user) 248 sum += pai_getdata(event, false); 249 250 return sum; 251 } 252 253 /* Called under mutex_lock */ 254 static int pai_alloc_cpu(int idx, int cpu, bool hotplug) 255 { 256 struct pai_map *cpump = NULL; 257 bool need_paiext_cb = false; 258 struct pai_mapptr *mp; 259 int tasks = 1, rc = 0; 260 261 /* When hotplug is true, invocation is from CPU hotplug callback. 262 * Allocate per-CPU resource when per-task events are currently active. 263 * This can be more than one. In this case adjust all reference 264 * counters. Otherwise return, this ensures memory is only allocated 265 * when needed. 266 */ 267 if (hotplug) { 268 tasks = atomic_read(&pai_root[idx].tskctx); 269 if (!tasks) 270 goto out; 271 } 272 273 /* Allocate root node */ 274 rc = pai_root_alloc(idx); 275 if (rc) 276 goto out; 277 278 /* Allocate node for this event */ 279 mp = per_cpu_ptr(pai_root[idx].mapptr, cpu); 280 cpump = mp->mapptr; 281 if (!cpump) { /* Paicrypt_map allocated? */ 282 rc = -ENOMEM; 283 cpump = kzalloc_obj(*cpump); 284 if (!cpump) 285 goto undo; 286 /* Allocate memory for counter page and counter extraction. 287 * Only the first counting event has to allocate a page. 288 */ 289 mp->mapptr = cpump; 290 if (idx == PAI_PMU_CRYPTO) { 291 cpump->area = (unsigned long *)get_zeroed_page(GFP_KERNEL); 292 /* free_page() can handle 0x0 address */ 293 cpump->fullpage = true; 294 } else { /* PAI_PMU_EXT */ 295 /* 296 * Allocate memory for counter area and counter extraction. 297 * These are 298 * - a 512 byte block and requires 512 byte boundary 299 * alignment. 300 * - a 1KB byte block and requires 1KB boundary 301 * alignment. 302 * Only the first counting event has to allocate the area. 303 * 304 * Note: This works with commit 59bb47985c1d by default. 305 * Backporting this to kernels without this commit might 306 * needs adjustment. 307 */ 308 cpump->area = kzalloc(pai_pmu[idx].area_size, GFP_KERNEL); 309 cpump->paiext_cb = kzalloc(PAIE1_CB_SZ, GFP_KERNEL); 310 need_paiext_cb = true; 311 } 312 cpump->save = kvmalloc_objs(struct pai_userdata, 313 pai_pmu[idx].num_avail + 1); 314 if (!cpump->area || !cpump->save || 315 (need_paiext_cb && !cpump->paiext_cb)) { 316 pai_free(mp); 317 goto undo; 318 } 319 INIT_LIST_HEAD(&cpump->syswide_list); 320 refcount_set(&cpump->refcnt, tasks); 321 rc = 0; 322 } else { 323 refcount_add(tasks, &cpump->refcnt); 324 } 325 /* If tasks is greater than 1, we are called from CPU hotplug path 326 * and need to adjust the pai_root[idx].refcnt by the number of 327 * per-process events. Function pai_root_alloc(idx) already 328 * incremented by one. Adjust for the rest. 329 */ 330 if (tasks > 1) 331 refcount_add(tasks - 1, &pai_root[idx].refcnt); 332 333 undo: 334 if (rc) { 335 /* Error in allocation of event, decrement anchor. Since 336 * the event in not created, its destroy() function is never 337 * invoked. Adjust the reference counter for the anchor. 338 * The failure happened in the case of variable 339 * cpump == NULL branch above. The pai_root[XXX].refcnt has 340 * been incremented by one. Then the per-CPU allocation 341 * failed, so decrement it by one, regardless of tasks. 342 */ 343 pai_root_free(idx, 1); 344 } 345 out: 346 /* If rc is non-zero, no increment of counter/sampler was done. */ 347 return rc; 348 } 349 350 /* Check concurrent access of counting and sampling for PAI events. 351 * This function is called in process context and it is safe to block. 352 * When the event initialization functions fails, no other call back will 353 * be invoked. 354 * Called under mutex_lock. 355 */ 356 static int pai_alloc(struct perf_event *event) 357 { 358 int idx = PAI_PMU_IDX(event); 359 struct cpumask *maskptr; 360 int cpu, rc = -ENOMEM; 361 362 maskptr = kzalloc_obj(*maskptr); 363 if (!maskptr) 364 goto out; 365 366 for_each_online_cpu(cpu) { 367 rc = pai_alloc_cpu(idx, cpu, false); 368 if (rc) { 369 for_each_cpu(cpu, maskptr) 370 pai_event_destroy_cpu(idx, cpu, false); 371 goto undo; 372 } 373 cpumask_set_cpu(cpu, maskptr); 374 } 375 376 rc = 0; 377 /* Trace per-task events for CPU hotplug. */ 378 atomic_inc(&pai_root[idx].tskctx); 379 undo: 380 kfree(maskptr); 381 out: 382 return rc; 383 } 384 385 /* Validate event number and return error if event is not supported. 386 * On successful return, PAI_PMU_IDX(event) is set to the index of 387 * the supporting paing_support[] array element. 388 */ 389 static int pai_event_valid(struct perf_event *event, int idx) 390 { 391 struct perf_event_attr *a = &event->attr; 392 struct pai_pmu *pp = &pai_pmu[idx]; 393 394 /* PAI crypto PMU registered as PERF_TYPE_RAW, check event type */ 395 if (a->type != PERF_TYPE_RAW && event->pmu->type != a->type) 396 return -ENOENT; 397 /* Allow only CRYPTO_ALL/NNPA_ALL for sampling */ 398 if (a->sample_period && a->config != pp->base) 399 return -EINVAL; 400 /* PAI crypto event must be in valid range, try others if not */ 401 if (a->config < pp->base || a->config > pp->base + pp->num_avail) 402 return -ENOENT; 403 if (idx == PAI_PMU_EXT && a->exclude_user) 404 return -EINVAL; 405 PAI_PMU_IDX(event) = idx; 406 return 0; 407 } 408 409 /* Might be called on different CPU than the one the event is intended for. */ 410 static int pai_event_init(struct perf_event *event, int idx) 411 { 412 struct perf_event_attr *a = &event->attr; 413 int rc; 414 415 /* PAI event must be valid and in supported range */ 416 rc = pai_event_valid(event, idx); 417 if (rc) 418 goto out; 419 /* Get a page to store last counter values for sampling */ 420 if (a->sample_period) { 421 PAI_SAVE_AREA(event) = get_zeroed_page(GFP_KERNEL); 422 if (!PAI_SAVE_AREA(event)) { 423 rc = -ENOMEM; 424 goto out; 425 } 426 } 427 428 cpus_read_lock(); 429 mutex_lock(&pai_reserve_mutex); 430 if (event->cpu >= 0) 431 rc = pai_alloc_cpu(idx, event->cpu, false); 432 else 433 rc = pai_alloc(event); 434 mutex_unlock(&pai_reserve_mutex); 435 cpus_read_unlock(); 436 if (rc) { 437 free_page(PAI_SAVE_AREA(event)); 438 goto out; 439 } 440 441 if (a->sample_period) { 442 a->sample_period = 1; 443 a->freq = 0; 444 /* Register for paicrypt_sched_task() to be called */ 445 event->attach_state |= PERF_ATTACH_SCHED_CB; 446 /* Add raw data which contain the memory mapped counters */ 447 a->sample_type |= PERF_SAMPLE_RAW; 448 /* Turn off inheritance */ 449 a->inherit = 0; 450 } 451 out: 452 return rc; 453 } 454 455 static int paicrypt_event_init(struct perf_event *event) 456 { 457 int rc = pai_event_init(event, PAI_PMU_CRYPTO); 458 459 if (!rc) { 460 event->destroy = paicrypt_event_destroy; 461 static_branch_inc(&pai_key); 462 } 463 return rc; 464 } 465 466 static void pai_read(struct perf_event *event, 467 u64 (*fct)(struct perf_event *event)) 468 { 469 u64 prev, new, delta; 470 471 prev = local64_read(&event->hw.prev_count); 472 new = fct(event); 473 local64_set(&event->hw.prev_count, new); 474 delta = (prev <= new) ? new - prev : (-1ULL - prev) + new + 1; 475 local64_add(delta, &event->count); 476 } 477 478 static void paicrypt_read(struct perf_event *event) 479 { 480 pai_read(event, paicrypt_getall); 481 } 482 483 static void pai_start(struct perf_event *event, int flags, 484 u64 (*fct)(struct perf_event *event)) 485 { 486 int idx = PAI_PMU_IDX(event); 487 struct pai_pmu *pp = &pai_pmu[idx]; 488 struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr); 489 struct pai_map *cpump = mp->mapptr; 490 u64 sum; 491 492 if (!event->attr.sample_period) { /* Counting */ 493 sum = fct(event); /* Get current value */ 494 local64_set(&event->hw.prev_count, sum); 495 } else { /* Sampling */ 496 memcpy((void *)PAI_SAVE_AREA(event), cpump->area, pp->area_size); 497 /* Enable context switch callback for system-wide sampling */ 498 if (!(event->attach_state & PERF_ATTACH_TASK)) { 499 list_add_tail(PAI_SWLIST(event), &cpump->syswide_list); 500 perf_sched_cb_inc(event->pmu); 501 } else { 502 cpump->event = event; 503 } 504 } 505 event->hw.state &= ~PERF_HES_STOPPED; 506 } 507 508 static void paicrypt_start(struct perf_event *event, int flags) 509 { 510 pai_start(event, flags, paicrypt_getall); 511 } 512 513 static int pai_add(struct perf_event *event, int flags) 514 { 515 int idx = PAI_PMU_IDX(event); 516 struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr); 517 struct pai_map *cpump = mp->mapptr; 518 struct paiext_cb *pcb = cpump->paiext_cb; 519 unsigned long ccd; 520 521 if (++cpump->active_events == 1) { 522 if (!pcb) { /* PAI crypto */ 523 ccd = virt_to_phys(cpump->area) | PAI_CRYPTO_KERNEL_OFFSET; 524 WRITE_ONCE(get_lowcore()->ccd, ccd); 525 local_ctl_set_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT); 526 } else { /* PAI extension 1 */ 527 ccd = virt_to_phys(pcb); 528 WRITE_ONCE(get_lowcore()->aicd, ccd); 529 pcb->acc = virt_to_phys(cpump->area) | 0x1; 530 /* Enable CPU instruction lookup for PAIE1 control block */ 531 local_ctl_set_bit(0, CR0_PAI_EXTENSION_BIT); 532 } 533 } 534 if (flags & PERF_EF_START) 535 pai_pmu[idx].pmu->start(event, PERF_EF_RELOAD); 536 event->hw.state = 0; 537 return 0; 538 } 539 540 static int paicrypt_add(struct perf_event *event, int flags) 541 { 542 return pai_add(event, flags); 543 } 544 545 static void pai_have_sample(struct perf_event *, struct pai_map *); 546 static void pai_stop(struct perf_event *event, int flags) 547 { 548 int idx = PAI_PMU_IDX(event); 549 struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr); 550 struct pai_map *cpump = mp->mapptr; 551 552 /* Cope with multiple invocations: 553 * 1. perf_event_throttle() --> PMU->stop() 554 * 2. task schedules out --> PMU->stop() 555 * Check for event already stopped. 556 */ 557 if (event->hw.state & PERF_HES_STOPPED) 558 return; 559 if (!event->attr.sample_period) { /* Counting */ 560 pai_pmu[idx].pmu->read(event); 561 } else { /* Sampling */ 562 if (!(event->attach_state & PERF_ATTACH_TASK)) { 563 perf_sched_cb_dec(event->pmu); 564 list_del(PAI_SWLIST(event)); 565 } else { 566 pai_have_sample(event, cpump); 567 cpump->event = NULL; 568 } 569 } 570 event->hw.state = PERF_HES_STOPPED; 571 } 572 573 static void paicrypt_stop(struct perf_event *event, int flags) 574 { 575 pai_stop(event, flags); 576 } 577 578 static void pai_del(struct perf_event *event, int flags) 579 { 580 int idx = PAI_PMU_IDX(event); 581 struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr); 582 struct pai_map *cpump = mp->mapptr; 583 struct paiext_cb *pcb = cpump->paiext_cb; 584 585 pai_pmu[idx].pmu->stop(event, PERF_EF_UPDATE); 586 if (--cpump->active_events == 0) { 587 if (!pcb) { /* PAI crypto */ 588 local_ctl_clear_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT); 589 WRITE_ONCE(get_lowcore()->ccd, 0); 590 } else { /* PAI extension 1 */ 591 /* Disable CPU instruction lookup for PAIE1 control block */ 592 local_ctl_clear_bit(0, CR0_PAI_EXTENSION_BIT); 593 pcb->acc = 0; 594 WRITE_ONCE(get_lowcore()->aicd, 0); 595 } 596 } 597 } 598 599 static void paicrypt_del(struct perf_event *event, int flags) 600 { 601 pai_del(event, flags); 602 } 603 604 /* Create raw data and save it in buffer. Calculate the delta for each 605 * counter between this invocation and the last invocation. 606 * Returns number of bytes copied. 607 * After reading from PAI counter page, save the read value to the old 608 * page to calculate PAI counter deltas. 609 * Saves only entries with positive counter difference of the form 610 * 2 bytes: Number of counter 611 * 8 bytes: Value of counter 612 */ 613 static size_t pai_copy(struct pai_userdata *userdata, unsigned long *page, 614 struct pai_pmu *pp, unsigned long *page_old, 615 bool exclude_user, bool exclude_kernel) 616 { 617 int i, outidx = 0; 618 619 for (i = 1; i <= pp->num_avail; i++) { 620 u64 val = 0, val_old = 0, val_k = 0, val_old_k = 0; 621 622 if (!exclude_kernel) { 623 val_k = pai_getctr(page, i, pp->kernel_offset); 624 val_old_k = pai_getctr(page_old, i, pp->kernel_offset); 625 if (val_k != val_old_k) 626 pai_setctr(page_old, i, pp->kernel_offset, val_k); 627 } 628 if (!exclude_user) { 629 val = pai_getctr(page, i, 0); 630 val_old = pai_getctr(page_old, i, 0); 631 if (val != val_old) 632 pai_setctr(page_old, i, 0, val); 633 } 634 val += val_k; 635 val_old += val_old_k; 636 if (val >= val_old) 637 val -= val_old; 638 else 639 val = (~0ULL - val_old) + val + 1; 640 if (val) { 641 userdata[outidx].num = i; 642 userdata[outidx].value = val; 643 outidx++; 644 } 645 } 646 return outidx * sizeof(*userdata); 647 } 648 649 /* Write sample when one or more counters values are nonzero. 650 * 651 * Note: The function paicrypt_sched_task() and pai_push_sample() are not 652 * invoked after function paicrypt_del() has been called because of function 653 * perf_sched_cb_dec(). Both functions are only 654 * called when sampling is active. Function perf_sched_cb_inc() 655 * has been invoked to install function paicrypt_sched_task() as call back 656 * to run at context switch time. 657 * 658 * This causes function perf_event_context_sched_out() and 659 * perf_event_context_sched_in() to check whether the PMU has installed an 660 * sched_task() callback. That callback is not active after paicrypt_del() 661 * returns and has deleted the event on that CPU. 662 */ 663 static int pai_push_sample(size_t rawsize, struct pai_map *cpump, 664 struct perf_event *event) 665 { 666 struct perf_sample_data data; 667 struct perf_raw_record raw; 668 struct pt_regs regs; 669 int overflow; 670 671 /* Setup perf sample */ 672 memset(®s, 0, sizeof(regs)); 673 memset(&raw, 0, sizeof(raw)); 674 memset(&data, 0, sizeof(data)); 675 perf_sample_data_init(&data, 0, event->hw.last_period); 676 if (event->attr.sample_type & PERF_SAMPLE_TID) { 677 data.tid_entry.pid = task_tgid_nr(current); 678 data.tid_entry.tid = task_pid_nr(current); 679 } 680 if (event->attr.sample_type & PERF_SAMPLE_TIME) 681 data.time = event->clock(); 682 if (event->attr.sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER)) 683 data.id = event->id; 684 if (event->attr.sample_type & PERF_SAMPLE_CPU) { 685 data.cpu_entry.cpu = smp_processor_id(); 686 data.cpu_entry.reserved = 0; 687 } 688 if (event->attr.sample_type & PERF_SAMPLE_RAW) { 689 raw.frag.size = rawsize; 690 raw.frag.data = cpump->save; 691 perf_sample_save_raw_data(&data, event, &raw); 692 } 693 694 overflow = perf_event_overflow(event, &data, ®s); 695 perf_event_update_userpage(event); 696 return overflow; 697 } 698 699 /* Check if there is data to be saved on schedule out of a task. */ 700 static void pai_have_sample(struct perf_event *event, struct pai_map *cpump) 701 { 702 struct pai_pmu *pp; 703 size_t rawsize; 704 705 if (!event) /* No event active */ 706 return; 707 pp = &pai_pmu[PAI_PMU_IDX(event)]; 708 rawsize = pai_copy(cpump->save, cpump->area, pp, 709 (unsigned long *)PAI_SAVE_AREA(event), 710 event->attr.exclude_user, 711 !pp->kernel_offset ? true : event->attr.exclude_kernel); 712 if (rawsize) /* No incremented counters */ 713 pai_push_sample(rawsize, cpump, event); 714 } 715 716 /* Check if there is data to be saved on schedule out of a task. */ 717 static void pai_have_samples(int idx) 718 { 719 struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr); 720 struct pai_map *cpump = mp->mapptr; 721 struct perf_event *event, *e2; 722 723 list_for_each_entry_safe(event, e2, &cpump->syswide_list, hw.tp_list) 724 pai_have_sample(event, cpump); 725 } 726 727 /* Called on schedule-in and schedule-out. No access to event structure, 728 * but for sampling only event CRYPTO_ALL is allowed. 729 */ 730 static void paicrypt_sched_task(struct perf_event_pmu_context *pmu_ctx, 731 struct task_struct *task, bool sched_in) 732 { 733 /* We started with a clean page on event installation. So read out 734 * results on schedule_out and if page was dirty, save old values. 735 */ 736 if (!sched_in) 737 pai_have_samples(PAI_PMU_CRYPTO); 738 } 739 740 /* Prevent ioctl(fd, PERF_EVENT_IOC_PERIOD, ...) call. 741 * It sets perf_event::event_limit to a positive value and causes 742 * perf_event_overflow() to invoke pai_stop() call back function when 743 * perf_event::event_limit hits zero. This is not supported because the 744 * sample events CRYPTO_ALL and NNPA_ALL are always taken at schedule out 745 * of a task. 746 */ 747 static int pai_check_period(struct perf_event *event, u64 value) 748 { 749 return -EINVAL; 750 } 751 /* ============================= paiext ====================================*/ 752 753 static void paiext_event_destroy(struct perf_event *event) 754 { 755 pai_event_destroy(event); 756 } 757 758 /* Might be called on different CPU than the one the event is intended for. */ 759 static int paiext_event_init(struct perf_event *event) 760 { 761 int rc = pai_event_init(event, PAI_PMU_EXT); 762 763 if (!rc) { 764 event->attr.exclude_kernel = true; /* No kernel space part */ 765 event->destroy = paiext_event_destroy; 766 /* Offset of NNPA in paiext_cb */ 767 event->hw.config_base = offsetof(struct paiext_cb, acc); 768 } 769 return rc; 770 } 771 772 static u64 paiext_getall(struct perf_event *event) 773 { 774 return pai_getdata(event, false); 775 } 776 777 static void paiext_read(struct perf_event *event) 778 { 779 pai_read(event, paiext_getall); 780 } 781 782 static void paiext_start(struct perf_event *event, int flags) 783 { 784 pai_start(event, flags, paiext_getall); 785 } 786 787 static int paiext_add(struct perf_event *event, int flags) 788 { 789 return pai_add(event, flags); 790 } 791 792 static void paiext_stop(struct perf_event *event, int flags) 793 { 794 pai_stop(event, flags); 795 } 796 797 static void paiext_del(struct perf_event *event, int flags) 798 { 799 pai_del(event, flags); 800 } 801 802 /* Called on schedule-in and schedule-out. No access to event structure, 803 * but for sampling only event NNPA_ALL is allowed. 804 */ 805 static void paiext_sched_task(struct perf_event_pmu_context *pmu_ctx, 806 struct task_struct *task, bool sched_in) 807 { 808 /* We started with a clean page on event installation. So read out 809 * results on schedule_out and if page was dirty, save old values. 810 */ 811 if (!sched_in) 812 pai_have_samples(PAI_PMU_EXT); 813 } 814 815 /* Attribute definitions for paicrypt interface. As with other CPU 816 * Measurement Facilities, there is one attribute per mapped counter. 817 * The number of mapped counters may vary per machine generation. Use 818 * the QUERY PROCESSOR ACTIVITY COUNTER INFORMATION (QPACI) instruction 819 * to determine the number of mapped counters. The instructions returns 820 * a positive number, which is the highest number of supported counters. 821 * All counters less than this number are also supported, there are no 822 * holes. A returned number of zero means no support for mapped counters. 823 * 824 * The identification of the counter is a unique number. The chosen range 825 * is 0x1000 + offset in mapped kernel page. 826 * All CPU Measurement Facility counters identifiers must be unique and 827 * the numbers from 0 to 496 are already used for the CPU Measurement 828 * Counter facility. Numbers 0xb0000, 0xbc000 and 0xbd000 are already 829 * used for the CPU Measurement Sampling facility. 830 */ 831 PMU_FORMAT_ATTR(event, "config:0-63"); 832 833 static struct attribute *paicrypt_format_attr[] = { 834 &format_attr_event.attr, 835 NULL, 836 }; 837 838 static struct attribute_group paicrypt_events_group = { 839 .name = "events", 840 .attrs = NULL /* Filled in attr_event_init() */ 841 }; 842 843 static struct attribute_group paicrypt_format_group = { 844 .name = "format", 845 .attrs = paicrypt_format_attr, 846 }; 847 848 static const struct attribute_group *paicrypt_attr_groups[] = { 849 &paicrypt_events_group, 850 &paicrypt_format_group, 851 NULL, 852 }; 853 854 /* Performance monitoring unit for mapped counters */ 855 static struct pmu paicrypt = { 856 .task_ctx_nr = perf_hw_context, 857 .event_init = paicrypt_event_init, 858 .add = paicrypt_add, 859 .del = paicrypt_del, 860 .start = paicrypt_start, 861 .stop = paicrypt_stop, 862 .read = paicrypt_read, 863 .sched_task = paicrypt_sched_task, 864 .check_period = pai_check_period, 865 .attr_groups = paicrypt_attr_groups 866 }; 867 868 /* List of symbolic PAI counter names. */ 869 static const char * const paicrypt_ctrnames[] = { 870 [0] = "CRYPTO_ALL", 871 [1] = "KM_DEA", 872 [2] = "KM_TDEA_128", 873 [3] = "KM_TDEA_192", 874 [4] = "KM_ENCRYPTED_DEA", 875 [5] = "KM_ENCRYPTED_TDEA_128", 876 [6] = "KM_ENCRYPTED_TDEA_192", 877 [7] = "KM_AES_128", 878 [8] = "KM_AES_192", 879 [9] = "KM_AES_256", 880 [10] = "KM_ENCRYPTED_AES_128", 881 [11] = "KM_ENCRYPTED_AES_192", 882 [12] = "KM_ENCRYPTED_AES_256", 883 [13] = "KM_XTS_AES_128", 884 [14] = "KM_XTS_AES_256", 885 [15] = "KM_XTS_ENCRYPTED_AES_128", 886 [16] = "KM_XTS_ENCRYPTED_AES_256", 887 [17] = "KMC_DEA", 888 [18] = "KMC_TDEA_128", 889 [19] = "KMC_TDEA_192", 890 [20] = "KMC_ENCRYPTED_DEA", 891 [21] = "KMC_ENCRYPTED_TDEA_128", 892 [22] = "KMC_ENCRYPTED_TDEA_192", 893 [23] = "KMC_AES_128", 894 [24] = "KMC_AES_192", 895 [25] = "KMC_AES_256", 896 [26] = "KMC_ENCRYPTED_AES_128", 897 [27] = "KMC_ENCRYPTED_AES_192", 898 [28] = "KMC_ENCRYPTED_AES_256", 899 [29] = "KMC_PRNG", 900 [30] = "KMA_GCM_AES_128", 901 [31] = "KMA_GCM_AES_192", 902 [32] = "KMA_GCM_AES_256", 903 [33] = "KMA_GCM_ENCRYPTED_AES_128", 904 [34] = "KMA_GCM_ENCRYPTED_AES_192", 905 [35] = "KMA_GCM_ENCRYPTED_AES_256", 906 [36] = "KMF_DEA", 907 [37] = "KMF_TDEA_128", 908 [38] = "KMF_TDEA_192", 909 [39] = "KMF_ENCRYPTED_DEA", 910 [40] = "KMF_ENCRYPTED_TDEA_128", 911 [41] = "KMF_ENCRYPTED_TDEA_192", 912 [42] = "KMF_AES_128", 913 [43] = "KMF_AES_192", 914 [44] = "KMF_AES_256", 915 [45] = "KMF_ENCRYPTED_AES_128", 916 [46] = "KMF_ENCRYPTED_AES_192", 917 [47] = "KMF_ENCRYPTED_AES_256", 918 [48] = "KMCTR_DEA", 919 [49] = "KMCTR_TDEA_128", 920 [50] = "KMCTR_TDEA_192", 921 [51] = "KMCTR_ENCRYPTED_DEA", 922 [52] = "KMCTR_ENCRYPTED_TDEA_128", 923 [53] = "KMCTR_ENCRYPTED_TDEA_192", 924 [54] = "KMCTR_AES_128", 925 [55] = "KMCTR_AES_192", 926 [56] = "KMCTR_AES_256", 927 [57] = "KMCTR_ENCRYPTED_AES_128", 928 [58] = "KMCTR_ENCRYPTED_AES_192", 929 [59] = "KMCTR_ENCRYPTED_AES_256", 930 [60] = "KMO_DEA", 931 [61] = "KMO_TDEA_128", 932 [62] = "KMO_TDEA_192", 933 [63] = "KMO_ENCRYPTED_DEA", 934 [64] = "KMO_ENCRYPTED_TDEA_128", 935 [65] = "KMO_ENCRYPTED_TDEA_192", 936 [66] = "KMO_AES_128", 937 [67] = "KMO_AES_192", 938 [68] = "KMO_AES_256", 939 [69] = "KMO_ENCRYPTED_AES_128", 940 [70] = "KMO_ENCRYPTED_AES_192", 941 [71] = "KMO_ENCRYPTED_AES_256", 942 [72] = "KIMD_SHA_1", 943 [73] = "KIMD_SHA_256", 944 [74] = "KIMD_SHA_512", 945 [75] = "KIMD_SHA3_224", 946 [76] = "KIMD_SHA3_256", 947 [77] = "KIMD_SHA3_384", 948 [78] = "KIMD_SHA3_512", 949 [79] = "KIMD_SHAKE_128", 950 [80] = "KIMD_SHAKE_256", 951 [81] = "KIMD_GHASH", 952 [82] = "KLMD_SHA_1", 953 [83] = "KLMD_SHA_256", 954 [84] = "KLMD_SHA_512", 955 [85] = "KLMD_SHA3_224", 956 [86] = "KLMD_SHA3_256", 957 [87] = "KLMD_SHA3_384", 958 [88] = "KLMD_SHA3_512", 959 [89] = "KLMD_SHAKE_128", 960 [90] = "KLMD_SHAKE_256", 961 [91] = "KMAC_DEA", 962 [92] = "KMAC_TDEA_128", 963 [93] = "KMAC_TDEA_192", 964 [94] = "KMAC_ENCRYPTED_DEA", 965 [95] = "KMAC_ENCRYPTED_TDEA_128", 966 [96] = "KMAC_ENCRYPTED_TDEA_192", 967 [97] = "KMAC_AES_128", 968 [98] = "KMAC_AES_192", 969 [99] = "KMAC_AES_256", 970 [100] = "KMAC_ENCRYPTED_AES_128", 971 [101] = "KMAC_ENCRYPTED_AES_192", 972 [102] = "KMAC_ENCRYPTED_AES_256", 973 [103] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_DEA", 974 [104] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_TDEA_128", 975 [105] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_TDEA_192", 976 [106] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_DEA", 977 [107] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_TDEA_128", 978 [108] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_TDEA_192", 979 [109] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_128", 980 [110] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_192", 981 [111] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_256", 982 [112] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_128", 983 [113] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_192", 984 [114] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_256", 985 [115] = "PCC_COMPUTE_XTS_PARAMETER_USING_AES_128", 986 [116] = "PCC_COMPUTE_XTS_PARAMETER_USING_AES_256", 987 [117] = "PCC_COMPUTE_XTS_PARAMETER_USING_ENCRYPTED_AES_128", 988 [118] = "PCC_COMPUTE_XTS_PARAMETER_USING_ENCRYPTED_AES_256", 989 [119] = "PCC_SCALAR_MULTIPLY_P256", 990 [120] = "PCC_SCALAR_MULTIPLY_P384", 991 [121] = "PCC_SCALAR_MULTIPLY_P521", 992 [122] = "PCC_SCALAR_MULTIPLY_ED25519", 993 [123] = "PCC_SCALAR_MULTIPLY_ED448", 994 [124] = "PCC_SCALAR_MULTIPLY_X25519", 995 [125] = "PCC_SCALAR_MULTIPLY_X448", 996 [126] = "PRNO_SHA_512_DRNG", 997 [127] = "PRNO_TRNG_QUERY_RAW_TO_CONDITIONED_RATIO", 998 [128] = "PRNO_TRNG", 999 [129] = "KDSA_ECDSA_VERIFY_P256", 1000 [130] = "KDSA_ECDSA_VERIFY_P384", 1001 [131] = "KDSA_ECDSA_VERIFY_P521", 1002 [132] = "KDSA_ECDSA_SIGN_P256", 1003 [133] = "KDSA_ECDSA_SIGN_P384", 1004 [134] = "KDSA_ECDSA_SIGN_P521", 1005 [135] = "KDSA_ENCRYPTED_ECDSA_SIGN_P256", 1006 [136] = "KDSA_ENCRYPTED_ECDSA_SIGN_P384", 1007 [137] = "KDSA_ENCRYPTED_ECDSA_SIGN_P521", 1008 [138] = "KDSA_EDDSA_VERIFY_ED25519", 1009 [139] = "KDSA_EDDSA_VERIFY_ED448", 1010 [140] = "KDSA_EDDSA_SIGN_ED25519", 1011 [141] = "KDSA_EDDSA_SIGN_ED448", 1012 [142] = "KDSA_ENCRYPTED_EDDSA_SIGN_ED25519", 1013 [143] = "KDSA_ENCRYPTED_EDDSA_SIGN_ED448", 1014 [144] = "PCKMO_ENCRYPT_DEA_KEY", 1015 [145] = "PCKMO_ENCRYPT_TDEA_128_KEY", 1016 [146] = "PCKMO_ENCRYPT_TDEA_192_KEY", 1017 [147] = "PCKMO_ENCRYPT_AES_128_KEY", 1018 [148] = "PCKMO_ENCRYPT_AES_192_KEY", 1019 [149] = "PCKMO_ENCRYPT_AES_256_KEY", 1020 [150] = "PCKMO_ENCRYPT_ECC_P256_KEY", 1021 [151] = "PCKMO_ENCRYPT_ECC_P384_KEY", 1022 [152] = "PCKMO_ENCRYPT_ECC_P521_KEY", 1023 [153] = "PCKMO_ENCRYPT_ECC_ED25519_KEY", 1024 [154] = "PCKMO_ENCRYPT_ECC_ED448_KEY", 1025 [155] = "IBM_RESERVED_155", 1026 [156] = "IBM_RESERVED_156", 1027 [157] = "KM_FULL_XTS_AES_128", 1028 [158] = "KM_FULL_XTS_AES_256", 1029 [159] = "KM_FULL_XTS_ENCRYPTED_AES_128", 1030 [160] = "KM_FULL_XTS_ENCRYPTED_AES_256", 1031 [161] = "KMAC_HMAC_SHA_224", 1032 [162] = "KMAC_HMAC_SHA_256", 1033 [163] = "KMAC_HMAC_SHA_384", 1034 [164] = "KMAC_HMAC_SHA_512", 1035 [165] = "KMAC_HMAC_ENCRYPTED_SHA_224", 1036 [166] = "KMAC_HMAC_ENCRYPTED_SHA_256", 1037 [167] = "KMAC_HMAC_ENCRYPTED_SHA_384", 1038 [168] = "KMAC_HMAC_ENCRYPTED_SHA_512", 1039 [169] = "PCKMO_ENCRYPT_HMAC_512_KEY", 1040 [170] = "PCKMO_ENCRYPT_HMAC_1024_KEY", 1041 [171] = "PCKMO_ENCRYPT_AES_XTS_128", 1042 [172] = "PCKMO_ENCRYPT_AES_XTS_256", 1043 }; 1044 1045 static struct attribute *paiext_format_attr[] = { 1046 &format_attr_event.attr, 1047 NULL, 1048 }; 1049 1050 static struct attribute_group paiext_events_group = { 1051 .name = "events", 1052 .attrs = NULL, /* Filled in attr_event_init() */ 1053 }; 1054 1055 static struct attribute_group paiext_format_group = { 1056 .name = "format", 1057 .attrs = paiext_format_attr, 1058 }; 1059 1060 static const struct attribute_group *paiext_attr_groups[] = { 1061 &paiext_events_group, 1062 &paiext_format_group, 1063 NULL, 1064 }; 1065 1066 /* Performance monitoring unit for mapped counters */ 1067 static struct pmu paiext = { 1068 .task_ctx_nr = perf_hw_context, 1069 .event_init = paiext_event_init, 1070 .add = paiext_add, 1071 .del = paiext_del, 1072 .start = paiext_start, 1073 .stop = paiext_stop, 1074 .read = paiext_read, 1075 .sched_task = paiext_sched_task, 1076 .check_period = pai_check_period, 1077 .attr_groups = paiext_attr_groups, 1078 }; 1079 1080 /* List of symbolic PAI extension 1 NNPA counter names. */ 1081 static const char * const paiext_ctrnames[] = { 1082 [0] = "NNPA_ALL", 1083 [1] = "NNPA_ADD", 1084 [2] = "NNPA_SUB", 1085 [3] = "NNPA_MUL", 1086 [4] = "NNPA_DIV", 1087 [5] = "NNPA_MIN", 1088 [6] = "NNPA_MAX", 1089 [7] = "NNPA_LOG", 1090 [8] = "NNPA_EXP", 1091 [9] = "NNPA_IBM_RESERVED_9", 1092 [10] = "NNPA_RELU", 1093 [11] = "NNPA_TANH", 1094 [12] = "NNPA_SIGMOID", 1095 [13] = "NNPA_SOFTMAX", 1096 [14] = "NNPA_BATCHNORM", 1097 [15] = "NNPA_MAXPOOL2D", 1098 [16] = "NNPA_AVGPOOL2D", 1099 [17] = "NNPA_LSTMACT", 1100 [18] = "NNPA_GRUACT", 1101 [19] = "NNPA_CONVOLUTION", 1102 [20] = "NNPA_MATMUL_OP", 1103 [21] = "NNPA_MATMUL_OP_BCAST23", 1104 [22] = "NNPA_SMALLBATCH", 1105 [23] = "NNPA_LARGEDIM", 1106 [24] = "NNPA_SMALLTENSOR", 1107 [25] = "NNPA_1MFRAME", 1108 [26] = "NNPA_2GFRAME", 1109 [27] = "NNPA_ACCESSEXCEPT", 1110 [28] = "NNPA_TRANSFORM", 1111 [29] = "NNPA_GELU", 1112 [30] = "NNPA_MOMENTS", 1113 [31] = "NNPA_LAYERNORM", 1114 [32] = "NNPA_MATMUL_OP_BCAST1", 1115 [33] = "NNPA_SQRT", 1116 [34] = "NNPA_INVSQRT", 1117 [35] = "NNPA_NORM", 1118 [36] = "NNPA_REDUCE", 1119 }; 1120 1121 static void __init attr_event_free(struct attribute **attrs) 1122 { 1123 struct perf_pmu_events_attr *pa; 1124 unsigned int i; 1125 1126 for (i = 0; attrs[i]; i++) { 1127 struct device_attribute *dap; 1128 1129 dap = container_of(attrs[i], struct device_attribute, attr); 1130 pa = container_of(dap, struct perf_pmu_events_attr, attr); 1131 kfree(pa); 1132 } 1133 kfree(attrs); 1134 } 1135 1136 static struct attribute * __init attr_event_init_one(int num, 1137 unsigned long base, 1138 const char *name) 1139 { 1140 struct perf_pmu_events_attr *pa; 1141 1142 pa = kzalloc_obj(*pa); 1143 if (!pa) 1144 return NULL; 1145 1146 sysfs_attr_init(&pa->attr.attr); 1147 pa->id = base + num; 1148 pa->attr.attr.name = name; 1149 pa->attr.attr.mode = 0444; 1150 pa->attr.show = cpumf_events_sysfs_show; 1151 pa->attr.store = NULL; 1152 return &pa->attr.attr; 1153 } 1154 1155 static struct attribute ** __init attr_event_init(struct pai_pmu *p) 1156 { 1157 unsigned int min_attr = min_t(unsigned int, p->num_named, p->num_avail); 1158 struct attribute **attrs; 1159 unsigned int i; 1160 1161 attrs = kmalloc_objs(*attrs, min_attr + 1, GFP_KERNEL | __GFP_ZERO); 1162 if (!attrs) 1163 goto out; 1164 for (i = 0; i < min_attr; i++) { 1165 attrs[i] = attr_event_init_one(i, p->base, p->names[i]); 1166 if (!attrs[i]) { 1167 attr_event_free(attrs); 1168 attrs = NULL; 1169 goto out; 1170 } 1171 } 1172 attrs[i] = NULL; 1173 out: 1174 return attrs; 1175 } 1176 1177 static void __init pai_pmu_exit(struct pai_pmu *p) 1178 { 1179 attr_event_free(p->event_group->attrs); 1180 p->event_group->attrs = NULL; 1181 } 1182 1183 /* Add a PMU. Install its events and register the PMU device driver 1184 * call back functions. 1185 */ 1186 static int __init pai_pmu_init(struct pai_pmu *p) 1187 { 1188 int rc = -ENOMEM; 1189 1190 1191 /* Export known PAI events */ 1192 p->event_group->attrs = attr_event_init(p); 1193 if (!p->event_group->attrs) { 1194 pr_err("Creation of PMU %s /sysfs failed\n", p->pmuname); 1195 goto out; 1196 } 1197 1198 rc = perf_pmu_register(p->pmu, p->pmuname, -1); 1199 if (rc) { 1200 pai_pmu_exit(p); 1201 pr_err("Registering PMU %s failed with rc=%i\n", p->pmuname, 1202 rc); 1203 } 1204 out: 1205 return rc; 1206 } 1207 1208 /* PAI PMU characteristics table */ 1209 static struct pai_pmu pai_pmu[] __refdata = { 1210 [PAI_PMU_CRYPTO] = { 1211 .pmuname = "pai_crypto", 1212 .facility_nr = 196, 1213 .num_named = ARRAY_SIZE(paicrypt_ctrnames), 1214 .names = paicrypt_ctrnames, 1215 .base = PAI_CRYPTO_BASE, 1216 .kernel_offset = PAI_CRYPTO_KERNEL_OFFSET, 1217 .area_size = PAGE_SIZE, 1218 .init = pai_pmu_init, 1219 .exit = pai_pmu_exit, 1220 .pmu = &paicrypt, 1221 .event_group = &paicrypt_events_group 1222 }, 1223 [PAI_PMU_EXT] = { 1224 .pmuname = "pai_ext", 1225 .facility_nr = 197, 1226 .num_named = ARRAY_SIZE(paiext_ctrnames), 1227 .names = paiext_ctrnames, 1228 .base = PAI_NNPA_BASE, 1229 .kernel_offset = 0, 1230 .area_size = PAIE1_CTRBLOCK_SZ, 1231 .init = pai_pmu_init, 1232 .exit = pai_pmu_exit, 1233 .pmu = &paiext, 1234 .event_group = &paiext_events_group 1235 } 1236 }; 1237 1238 /* 1239 * Check if the PMU (via facility) is supported by machine. Try all of the 1240 * supported PAI PMUs. 1241 * Return number of successfully installed PMUs. 1242 */ 1243 static int __init paipmu_setup(void) 1244 { 1245 struct qpaci_info_block ib; 1246 int install_ok = 0, rc; 1247 struct pai_pmu *p; 1248 size_t i; 1249 1250 for (i = 0; i < ARRAY_SIZE(pai_pmu); ++i) { 1251 p = &pai_pmu[i]; 1252 1253 if (!test_facility(p->facility_nr)) 1254 continue; 1255 1256 qpaci(&ib); 1257 switch (i) { 1258 case PAI_PMU_CRYPTO: 1259 p->num_avail = ib.num_cc; 1260 if (p->num_avail >= PAI_CRYPTO_MAXCTR) { 1261 pr_err("Too many PMU %s counters %d\n", 1262 p->pmuname, p->num_avail); 1263 continue; 1264 } 1265 break; 1266 case PAI_PMU_EXT: 1267 p->num_avail = ib.num_nnpa; 1268 break; 1269 } 1270 p->num_avail += 1; /* Add xxx_ALL event */ 1271 if (p->init) { 1272 rc = p->init(p); 1273 if (!rc) 1274 ++install_ok; 1275 } 1276 } 1277 return install_ok; 1278 } 1279 1280 static int pai_online_cpu(unsigned int cpu) 1281 { 1282 int rc; 1283 1284 mutex_lock(&pai_reserve_mutex); 1285 rc = pai_alloc_cpu(PAI_PMU_CRYPTO, cpu, true); 1286 if (rc) 1287 goto out; 1288 rc = pai_alloc_cpu(PAI_PMU_EXT, cpu, true); 1289 if (rc) 1290 pai_event_destroy_cpu(PAI_PMU_CRYPTO, cpu, true); 1291 out: 1292 mutex_unlock(&pai_reserve_mutex); 1293 return rc; 1294 } 1295 1296 static int pai_offline_cpu(unsigned int cpu) 1297 { 1298 mutex_lock(&pai_reserve_mutex); 1299 pai_event_destroy_cpu(PAI_PMU_CRYPTO, cpu, true); 1300 pai_event_destroy_cpu(PAI_PMU_EXT, cpu, true); 1301 mutex_unlock(&pai_reserve_mutex); 1302 return 0; 1303 } 1304 1305 static int __init pai_init(void) 1306 { 1307 int state, rc; 1308 1309 /* Setup s390dbf facility */ 1310 paidbg = debug_register("pai", 1, 1, 128); 1311 if (!paidbg) { 1312 pr_err("Registration of s390dbf pai failed\n"); 1313 return -ENOMEM; 1314 } 1315 debug_register_view(paidbg, &debug_sprintf_view); 1316 1317 /* CPUHP_BP_PREPARE_DYN --> before CPU is brought online */ 1318 state = cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "perf/pai:prepare", 1319 pai_online_cpu, pai_offline_cpu); 1320 rc = state < 0 ? state : 0; 1321 if (rc < 0) 1322 goto out_debug; 1323 1324 rc = -ENODEV; 1325 if (!paipmu_setup()) 1326 goto out_cpuhp; 1327 return 0; 1328 1329 out_cpuhp: 1330 cpuhp_remove_state(state); 1331 out_debug: 1332 debug_unregister_view(paidbg, &debug_sprintf_view); 1333 debug_unregister(paidbg); 1334 return rc; 1335 } 1336 1337 device_initcall(pai_init); 1338