1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Performance event support - Processor Activity Instrumentation Facility 4 * 5 * Copyright IBM Corp. 2026 6 * Author(s): Thomas Richter <tmricht@linux.ibm.com> 7 */ 8 #define pr_fmt(fmt) "pai: " fmt 9 10 #include <linux/kernel.h> 11 #include <linux/kernel_stat.h> 12 #include <linux/percpu.h> 13 #include <linux/notifier.h> 14 #include <linux/init.h> 15 #include <linux/io.h> 16 #include <linux/perf_event.h> 17 #include <asm/ctlreg.h> 18 #include <asm/pai.h> 19 #include <asm/debug.h> 20 21 static debug_info_t *paidbg; 22 23 DEFINE_STATIC_KEY_FALSE(pai_key); 24 25 enum { 26 PAI_PMU_CRYPTO, /* Index of PMU pai_crypto */ 27 PAI_PMU_EXT, /* Index of PMU pai_ext */ 28 PAI_PMU_MAX /* # of PAI PMUs */ 29 }; 30 31 enum { 32 PAIE1_CB_SZ = 0x200, /* Size of PAIE1 control block */ 33 PAIE1_CTRBLOCK_SZ = 0x400 /* Size of PAIE1 counter blocks */ 34 }; 35 36 struct pai_userdata { 37 u16 num; 38 u64 value; 39 } __packed; 40 41 /* Create the PAI extension 1 control block area. 42 * The PAI extension control block 1 is pointed to by lowcore 43 * address 0x1508 for each CPU. This control block is 512 bytes in size 44 * and requires a 512 byte boundary alignment. 45 */ 46 struct paiext_cb { /* PAI extension 1 control block */ 47 u64 header; /* Not used */ 48 u64 reserved1; 49 u64 acc; /* Addr to analytics counter control block */ 50 u8 reserved2[PAIE1_CTRBLOCK_SZ - 3 * sizeof(u64)]; 51 } __packed; 52 53 struct pai_map { 54 unsigned long *area; /* Area for CPU to store counters */ 55 struct pai_userdata *save; /* Page to store no-zero counters */ 56 unsigned int active_events; /* # of PAI crypto users */ 57 refcount_t refcnt; /* Reference count mapped buffers */ 58 struct perf_event *event; /* Perf event for sampling */ 59 struct list_head syswide_list; /* List system-wide sampling events */ 60 struct paiext_cb *paiext_cb; /* PAI extension control block area */ 61 bool fullpage; /* True: counter area is a full page */ 62 }; 63 64 struct pai_mapptr { 65 struct pai_map *mapptr; 66 }; 67 68 static struct pai_root { /* Anchor to per CPU data */ 69 refcount_t refcnt; /* Overall active events */ 70 struct pai_mapptr __percpu *mapptr; 71 } pai_root[PAI_PMU_MAX]; 72 73 /* This table defines the different parameters of the PAI PMUs. During 74 * initialization the machine dependent values are extracted and saved. 75 * However most of the values are static and do not change. 76 * There is one table entry per PAI PMU. 77 */ 78 struct pai_pmu { /* Define PAI PMU characteristics */ 79 const char *pmuname; /* Name of PMU */ 80 const int facility_nr; /* Facility number to check for support */ 81 unsigned int num_avail; /* # Counters defined by hardware */ 82 unsigned int num_named; /* # Counters known by name */ 83 unsigned long base; /* Counter set base number */ 84 unsigned long kernel_offset; /* Offset to kernel part in counter page */ 85 unsigned long area_size; /* Size of counter area */ 86 const char * const *names; /* List of counter names */ 87 struct pmu *pmu; /* Ptr to supporting PMU */ 88 int (*init)(struct pai_pmu *p); /* PMU support init function */ 89 void (*exit)(struct pai_pmu *p); /* PMU support exit function */ 90 struct attribute_group *event_group; /* Ptr to attribute of events */ 91 }; 92 93 static struct pai_pmu pai_pmu[]; /* Forward declaration */ 94 95 /* Free per CPU data when the last event is removed. */ 96 static void pai_root_free(int idx) 97 { 98 if (refcount_dec_and_test(&pai_root[idx].refcnt)) { 99 free_percpu(pai_root[idx].mapptr); 100 pai_root[idx].mapptr = NULL; 101 } 102 debug_sprintf_event(paidbg, 5, "%s root[%d].refcount %d\n", __func__, 103 idx, refcount_read(&pai_root[idx].refcnt)); 104 } 105 106 /* 107 * On initialization of first event also allocate per CPU data dynamically. 108 * Start with an array of pointers, the array size is the maximum number of 109 * CPUs possible, which might be larger than the number of CPUs currently 110 * online. 111 */ 112 static int pai_root_alloc(int idx) 113 { 114 if (!refcount_inc_not_zero(&pai_root[idx].refcnt)) { 115 /* The memory is already zeroed. */ 116 pai_root[idx].mapptr = alloc_percpu(struct pai_mapptr); 117 if (!pai_root[idx].mapptr) 118 return -ENOMEM; 119 refcount_set(&pai_root[idx].refcnt, 1); 120 } 121 return 0; 122 } 123 124 /* Release the PMU if event is the last perf event */ 125 static DEFINE_MUTEX(pai_reserve_mutex); 126 127 /* Free all memory allocated for event counting/sampling setup */ 128 static void pai_free(struct pai_mapptr *mp) 129 { 130 if (mp->mapptr->fullpage) 131 free_page((unsigned long)mp->mapptr->area); 132 else 133 kfree(mp->mapptr->area); 134 kfree(mp->mapptr->paiext_cb); 135 kvfree(mp->mapptr->save); 136 kfree(mp->mapptr); 137 mp->mapptr = NULL; 138 } 139 140 /* Adjust usage counters and remove allocated memory when all users are 141 * gone. 142 */ 143 static void pai_event_destroy_cpu(struct perf_event *event, int cpu) 144 { 145 int idx = PAI_PMU_IDX(event); 146 struct pai_mapptr *mp = per_cpu_ptr(pai_root[idx].mapptr, cpu); 147 struct pai_map *cpump = mp->mapptr; 148 149 mutex_lock(&pai_reserve_mutex); 150 debug_sprintf_event(paidbg, 5, "%s event %#llx idx %d cpu %d users %d " 151 "refcnt %u\n", __func__, event->attr.config, idx, 152 event->cpu, cpump->active_events, 153 refcount_read(&cpump->refcnt)); 154 if (refcount_dec_and_test(&cpump->refcnt)) 155 pai_free(mp); 156 pai_root_free(idx); 157 mutex_unlock(&pai_reserve_mutex); 158 } 159 160 static void pai_event_destroy(struct perf_event *event) 161 { 162 int cpu; 163 164 free_page(PAI_SAVE_AREA(event)); 165 if (event->cpu == -1) { 166 struct cpumask *mask = PAI_CPU_MASK(event); 167 168 for_each_cpu(cpu, mask) 169 pai_event_destroy_cpu(event, cpu); 170 kfree(mask); 171 } else { 172 pai_event_destroy_cpu(event, event->cpu); 173 } 174 } 175 176 static void paicrypt_event_destroy(struct perf_event *event) 177 { 178 static_branch_dec(&pai_key); 179 pai_event_destroy(event); 180 } 181 182 static u64 pai_getctr(unsigned long *page, int nr, unsigned long offset) 183 { 184 if (offset) 185 nr += offset / sizeof(*page); 186 return page[nr]; 187 } 188 189 static void pai_setctr(unsigned long *page, int nr, unsigned long offset, u64 v) 190 { 191 if (offset) 192 nr += offset / sizeof(*page); 193 page[nr] = v; 194 } 195 196 /* Read the counter values. Return value from location in CMP. For base 197 * event xxx_ALL sum up all events. Returns counter value. 198 */ 199 static u64 pai_getdata(struct perf_event *event, bool kernel) 200 { 201 int idx = PAI_PMU_IDX(event); 202 struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr); 203 struct pai_pmu *pp = &pai_pmu[idx]; 204 struct pai_map *cpump = mp->mapptr; 205 unsigned int i; 206 u64 sum = 0; 207 208 if (event->attr.config != pp->base) { 209 return pai_getctr(cpump->area, 210 event->attr.config - pp->base, 211 kernel ? pp->kernel_offset : 0); 212 } 213 214 for (i = 1; i <= pp->num_avail; i++) { 215 u64 val = pai_getctr(cpump->area, i, 216 kernel ? pp->kernel_offset : 0); 217 218 if (!val) 219 continue; 220 sum += val; 221 } 222 return sum; 223 } 224 225 static u64 paicrypt_getall(struct perf_event *event) 226 { 227 u64 sum = 0; 228 229 if (!event->attr.exclude_kernel) 230 sum += pai_getdata(event, true); 231 if (!event->attr.exclude_user) 232 sum += pai_getdata(event, false); 233 234 return sum; 235 } 236 237 /* Check concurrent access of counting and sampling for crypto events. 238 * This function is called in process context and it is save to block. 239 * When the event initialization functions fails, no other call back will 240 * be invoked. 241 * 242 * Allocate the memory for the event. 243 */ 244 static int pai_alloc_cpu(struct perf_event *event, int cpu) 245 { 246 int rc, idx = PAI_PMU_IDX(event); 247 struct pai_map *cpump = NULL; 248 bool need_paiext_cb = false; 249 struct pai_mapptr *mp; 250 251 mutex_lock(&pai_reserve_mutex); 252 /* Allocate root node */ 253 rc = pai_root_alloc(idx); 254 if (rc) 255 goto unlock; 256 257 /* Allocate node for this event */ 258 mp = per_cpu_ptr(pai_root[idx].mapptr, cpu); 259 cpump = mp->mapptr; 260 if (!cpump) { /* Paicrypt_map allocated? */ 261 rc = -ENOMEM; 262 cpump = kzalloc_obj(*cpump); 263 if (!cpump) 264 goto undo; 265 /* Allocate memory for counter page and counter extraction. 266 * Only the first counting event has to allocate a page. 267 */ 268 mp->mapptr = cpump; 269 if (idx == PAI_PMU_CRYPTO) { 270 cpump->area = (unsigned long *)get_zeroed_page(GFP_KERNEL); 271 /* free_page() can handle 0x0 address */ 272 cpump->fullpage = true; 273 } else { /* PAI_PMU_EXT */ 274 /* 275 * Allocate memory for counter area and counter extraction. 276 * These are 277 * - a 512 byte block and requires 512 byte boundary 278 * alignment. 279 * - a 1KB byte block and requires 1KB boundary 280 * alignment. 281 * Only the first counting event has to allocate the area. 282 * 283 * Note: This works with commit 59bb47985c1d by default. 284 * Backporting this to kernels without this commit might 285 * needs adjustment. 286 */ 287 cpump->area = kzalloc(pai_pmu[idx].area_size, GFP_KERNEL); 288 cpump->paiext_cb = kzalloc(PAIE1_CB_SZ, GFP_KERNEL); 289 need_paiext_cb = true; 290 } 291 cpump->save = kvmalloc_objs(struct pai_userdata, 292 pai_pmu[idx].num_avail + 1); 293 if (!cpump->area || !cpump->save || 294 (need_paiext_cb && !cpump->paiext_cb)) { 295 pai_free(mp); 296 goto undo; 297 } 298 INIT_LIST_HEAD(&cpump->syswide_list); 299 refcount_set(&cpump->refcnt, 1); 300 rc = 0; 301 } else { 302 refcount_inc(&cpump->refcnt); 303 } 304 305 undo: 306 if (rc) { 307 /* Error in allocation of event, decrement anchor. Since 308 * the event in not created, its destroy() function is never 309 * invoked. Adjust the reference counter for the anchor. 310 */ 311 pai_root_free(idx); 312 } 313 unlock: 314 mutex_unlock(&pai_reserve_mutex); 315 /* If rc is non-zero, no increment of counter/sampler was done. */ 316 return rc; 317 } 318 319 static int pai_alloc(struct perf_event *event) 320 { 321 struct cpumask *maskptr; 322 int cpu, rc = -ENOMEM; 323 324 maskptr = kzalloc_obj(*maskptr); 325 if (!maskptr) 326 goto out; 327 328 for_each_online_cpu(cpu) { 329 rc = pai_alloc_cpu(event, cpu); 330 if (rc) { 331 for_each_cpu(cpu, maskptr) 332 pai_event_destroy_cpu(event, cpu); 333 kfree(maskptr); 334 goto out; 335 } 336 cpumask_set_cpu(cpu, maskptr); 337 } 338 339 /* 340 * On error all cpumask are freed and all events have been destroyed. 341 * Save of which CPUs data structures have been allocated for. 342 * Release them in pai_event_destroy call back function 343 * for this event. 344 */ 345 PAI_CPU_MASK(event) = maskptr; 346 rc = 0; 347 out: 348 return rc; 349 } 350 351 /* Validate event number and return error if event is not supported. 352 * On successful return, PAI_PMU_IDX(event) is set to the index of 353 * the supporting paing_support[] array element. 354 */ 355 static int pai_event_valid(struct perf_event *event, int idx) 356 { 357 struct perf_event_attr *a = &event->attr; 358 struct pai_pmu *pp = &pai_pmu[idx]; 359 360 /* PAI crypto PMU registered as PERF_TYPE_RAW, check event type */ 361 if (a->type != PERF_TYPE_RAW && event->pmu->type != a->type) 362 return -ENOENT; 363 /* Allow only CRYPTO_ALL/NNPA_ALL for sampling */ 364 if (a->sample_period && a->config != pp->base) 365 return -EINVAL; 366 /* PAI crypto event must be in valid range, try others if not */ 367 if (a->config < pp->base || a->config > pp->base + pp->num_avail) 368 return -ENOENT; 369 if (idx == PAI_PMU_EXT && a->exclude_user) 370 return -EINVAL; 371 PAI_PMU_IDX(event) = idx; 372 return 0; 373 } 374 375 /* Might be called on different CPU than the one the event is intended for. */ 376 static int pai_event_init(struct perf_event *event, int idx) 377 { 378 struct perf_event_attr *a = &event->attr; 379 int rc; 380 381 /* PAI event must be valid and in supported range */ 382 rc = pai_event_valid(event, idx); 383 if (rc) 384 goto out; 385 /* Get a page to store last counter values for sampling */ 386 if (a->sample_period) { 387 PAI_SAVE_AREA(event) = get_zeroed_page(GFP_KERNEL); 388 if (!PAI_SAVE_AREA(event)) { 389 rc = -ENOMEM; 390 goto out; 391 } 392 } 393 394 if (event->cpu >= 0) 395 rc = pai_alloc_cpu(event, event->cpu); 396 else 397 rc = pai_alloc(event); 398 if (rc) { 399 free_page(PAI_SAVE_AREA(event)); 400 goto out; 401 } 402 403 if (a->sample_period) { 404 a->sample_period = 1; 405 a->freq = 0; 406 /* Register for paicrypt_sched_task() to be called */ 407 event->attach_state |= PERF_ATTACH_SCHED_CB; 408 /* Add raw data which contain the memory mapped counters */ 409 a->sample_type |= PERF_SAMPLE_RAW; 410 /* Turn off inheritance */ 411 a->inherit = 0; 412 } 413 out: 414 return rc; 415 } 416 417 static int paicrypt_event_init(struct perf_event *event) 418 { 419 int rc = pai_event_init(event, PAI_PMU_CRYPTO); 420 421 if (!rc) { 422 event->destroy = paicrypt_event_destroy; 423 static_branch_inc(&pai_key); 424 } 425 return rc; 426 } 427 428 static void pai_read(struct perf_event *event, 429 u64 (*fct)(struct perf_event *event)) 430 { 431 u64 prev, new, delta; 432 433 prev = local64_read(&event->hw.prev_count); 434 new = fct(event); 435 local64_set(&event->hw.prev_count, new); 436 delta = (prev <= new) ? new - prev : (-1ULL - prev) + new + 1; 437 local64_add(delta, &event->count); 438 } 439 440 static void paicrypt_read(struct perf_event *event) 441 { 442 pai_read(event, paicrypt_getall); 443 } 444 445 static void pai_start(struct perf_event *event, int flags, 446 u64 (*fct)(struct perf_event *event)) 447 { 448 int idx = PAI_PMU_IDX(event); 449 struct pai_pmu *pp = &pai_pmu[idx]; 450 struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr); 451 struct pai_map *cpump = mp->mapptr; 452 u64 sum; 453 454 if (!event->attr.sample_period) { /* Counting */ 455 sum = fct(event); /* Get current value */ 456 local64_set(&event->hw.prev_count, sum); 457 } else { /* Sampling */ 458 memcpy((void *)PAI_SAVE_AREA(event), cpump->area, pp->area_size); 459 /* Enable context switch callback for system-wide sampling */ 460 if (!(event->attach_state & PERF_ATTACH_TASK)) { 461 list_add_tail(PAI_SWLIST(event), &cpump->syswide_list); 462 perf_sched_cb_inc(event->pmu); 463 } else { 464 cpump->event = event; 465 } 466 } 467 event->hw.state &= ~PERF_HES_STOPPED; 468 } 469 470 static void paicrypt_start(struct perf_event *event, int flags) 471 { 472 pai_start(event, flags, paicrypt_getall); 473 } 474 475 static int pai_add(struct perf_event *event, int flags) 476 { 477 int idx = PAI_PMU_IDX(event); 478 struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr); 479 struct pai_map *cpump = mp->mapptr; 480 struct paiext_cb *pcb = cpump->paiext_cb; 481 unsigned long ccd; 482 483 if (++cpump->active_events == 1) { 484 if (!pcb) { /* PAI crypto */ 485 ccd = virt_to_phys(cpump->area) | PAI_CRYPTO_KERNEL_OFFSET; 486 WRITE_ONCE(get_lowcore()->ccd, ccd); 487 local_ctl_set_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT); 488 } else { /* PAI extension 1 */ 489 ccd = virt_to_phys(pcb); 490 WRITE_ONCE(get_lowcore()->aicd, ccd); 491 pcb->acc = virt_to_phys(cpump->area) | 0x1; 492 /* Enable CPU instruction lookup for PAIE1 control block */ 493 local_ctl_set_bit(0, CR0_PAI_EXTENSION_BIT); 494 } 495 } 496 if (flags & PERF_EF_START) 497 pai_pmu[idx].pmu->start(event, PERF_EF_RELOAD); 498 event->hw.state = 0; 499 return 0; 500 } 501 502 static int paicrypt_add(struct perf_event *event, int flags) 503 { 504 return pai_add(event, flags); 505 } 506 507 static void pai_have_sample(struct perf_event *, struct pai_map *); 508 static void pai_stop(struct perf_event *event, int flags) 509 { 510 int idx = PAI_PMU_IDX(event); 511 struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr); 512 struct pai_map *cpump = mp->mapptr; 513 514 /* Cope with multiple invocations: 515 * 1. perf_event_throttle() --> PMU->stop() 516 * 2. task schedules out --> PMU->stop() 517 * Check for event already stopped. 518 */ 519 if (event->hw.state & PERF_HES_STOPPED) 520 return; 521 if (!event->attr.sample_period) { /* Counting */ 522 pai_pmu[idx].pmu->read(event); 523 } else { /* Sampling */ 524 if (!(event->attach_state & PERF_ATTACH_TASK)) { 525 perf_sched_cb_dec(event->pmu); 526 list_del(PAI_SWLIST(event)); 527 } else { 528 pai_have_sample(event, cpump); 529 cpump->event = NULL; 530 } 531 } 532 event->hw.state = PERF_HES_STOPPED; 533 } 534 535 static void paicrypt_stop(struct perf_event *event, int flags) 536 { 537 pai_stop(event, flags); 538 } 539 540 static void pai_del(struct perf_event *event, int flags) 541 { 542 int idx = PAI_PMU_IDX(event); 543 struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr); 544 struct pai_map *cpump = mp->mapptr; 545 struct paiext_cb *pcb = cpump->paiext_cb; 546 547 pai_pmu[idx].pmu->stop(event, PERF_EF_UPDATE); 548 if (--cpump->active_events == 0) { 549 if (!pcb) { /* PAI crypto */ 550 local_ctl_clear_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT); 551 WRITE_ONCE(get_lowcore()->ccd, 0); 552 } else { /* PAI extension 1 */ 553 /* Disable CPU instruction lookup for PAIE1 control block */ 554 local_ctl_clear_bit(0, CR0_PAI_EXTENSION_BIT); 555 pcb->acc = 0; 556 WRITE_ONCE(get_lowcore()->aicd, 0); 557 } 558 } 559 } 560 561 static void paicrypt_del(struct perf_event *event, int flags) 562 { 563 pai_del(event, flags); 564 } 565 566 /* Create raw data and save it in buffer. Calculate the delta for each 567 * counter between this invocation and the last invocation. 568 * Returns number of bytes copied. 569 * After reading from PAI counter page, save the read value to the old 570 * page to calculate PAI counter deltas. 571 * Saves only entries with positive counter difference of the form 572 * 2 bytes: Number of counter 573 * 8 bytes: Value of counter 574 */ 575 static size_t pai_copy(struct pai_userdata *userdata, unsigned long *page, 576 struct pai_pmu *pp, unsigned long *page_old, 577 bool exclude_user, bool exclude_kernel) 578 { 579 int i, outidx = 0; 580 581 for (i = 1; i <= pp->num_avail; i++) { 582 u64 val = 0, val_old = 0, val_k = 0, val_old_k = 0; 583 584 if (!exclude_kernel) { 585 val_k = pai_getctr(page, i, pp->kernel_offset); 586 val_old_k = pai_getctr(page_old, i, pp->kernel_offset); 587 if (val_k != val_old_k) 588 pai_setctr(page_old, i, pp->kernel_offset, val_k); 589 } 590 if (!exclude_user) { 591 val = pai_getctr(page, i, 0); 592 val_old = pai_getctr(page_old, i, 0); 593 if (val != val_old) 594 pai_setctr(page_old, i, 0, val); 595 } 596 val += val_k; 597 val_old += val_old_k; 598 if (val >= val_old) 599 val -= val_old; 600 else 601 val = (~0ULL - val_old) + val + 1; 602 if (val) { 603 userdata[outidx].num = i; 604 userdata[outidx].value = val; 605 outidx++; 606 } 607 } 608 return outidx * sizeof(*userdata); 609 } 610 611 /* Write sample when one or more counters values are nonzero. 612 * 613 * Note: The function paicrypt_sched_task() and pai_push_sample() are not 614 * invoked after function paicrypt_del() has been called because of function 615 * perf_sched_cb_dec(). Both functions are only 616 * called when sampling is active. Function perf_sched_cb_inc() 617 * has been invoked to install function paicrypt_sched_task() as call back 618 * to run at context switch time. 619 * 620 * This causes function perf_event_context_sched_out() and 621 * perf_event_context_sched_in() to check whether the PMU has installed an 622 * sched_task() callback. That callback is not active after paicrypt_del() 623 * returns and has deleted the event on that CPU. 624 */ 625 static int pai_push_sample(size_t rawsize, struct pai_map *cpump, 626 struct perf_event *event) 627 { 628 struct perf_sample_data data; 629 struct perf_raw_record raw; 630 struct pt_regs regs; 631 int overflow; 632 633 /* Setup perf sample */ 634 memset(®s, 0, sizeof(regs)); 635 memset(&raw, 0, sizeof(raw)); 636 memset(&data, 0, sizeof(data)); 637 perf_sample_data_init(&data, 0, event->hw.last_period); 638 if (event->attr.sample_type & PERF_SAMPLE_TID) { 639 data.tid_entry.pid = task_tgid_nr(current); 640 data.tid_entry.tid = task_pid_nr(current); 641 } 642 if (event->attr.sample_type & PERF_SAMPLE_TIME) 643 data.time = event->clock(); 644 if (event->attr.sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER)) 645 data.id = event->id; 646 if (event->attr.sample_type & PERF_SAMPLE_CPU) { 647 data.cpu_entry.cpu = smp_processor_id(); 648 data.cpu_entry.reserved = 0; 649 } 650 if (event->attr.sample_type & PERF_SAMPLE_RAW) { 651 raw.frag.size = rawsize; 652 raw.frag.data = cpump->save; 653 perf_sample_save_raw_data(&data, event, &raw); 654 } 655 656 overflow = perf_event_overflow(event, &data, ®s); 657 perf_event_update_userpage(event); 658 return overflow; 659 } 660 661 /* Check if there is data to be saved on schedule out of a task. */ 662 static void pai_have_sample(struct perf_event *event, struct pai_map *cpump) 663 { 664 struct pai_pmu *pp; 665 size_t rawsize; 666 667 if (!event) /* No event active */ 668 return; 669 pp = &pai_pmu[PAI_PMU_IDX(event)]; 670 rawsize = pai_copy(cpump->save, cpump->area, pp, 671 (unsigned long *)PAI_SAVE_AREA(event), 672 event->attr.exclude_user, 673 !pp->kernel_offset ? true : event->attr.exclude_kernel); 674 if (rawsize) /* No incremented counters */ 675 pai_push_sample(rawsize, cpump, event); 676 } 677 678 /* Check if there is data to be saved on schedule out of a task. */ 679 static void pai_have_samples(int idx) 680 { 681 struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr); 682 struct pai_map *cpump = mp->mapptr; 683 struct perf_event *event, *e2; 684 685 list_for_each_entry_safe(event, e2, &cpump->syswide_list, hw.tp_list) 686 pai_have_sample(event, cpump); 687 } 688 689 /* Called on schedule-in and schedule-out. No access to event structure, 690 * but for sampling only event CRYPTO_ALL is allowed. 691 */ 692 static void paicrypt_sched_task(struct perf_event_pmu_context *pmu_ctx, 693 struct task_struct *task, bool sched_in) 694 { 695 /* We started with a clean page on event installation. So read out 696 * results on schedule_out and if page was dirty, save old values. 697 */ 698 if (!sched_in) 699 pai_have_samples(PAI_PMU_CRYPTO); 700 } 701 702 /* Prevent ioctl(fd, PERF_EVENT_IOC_PERIOD, ...) call. 703 * It sets perf_event::event_limit to a positive value and causes 704 * perf_event_overflow() to invoke pai_stop() call back function when 705 * perf_event::event_limit hits zero. This is not supported because the 706 * sample events CRYPTO_ALL and NNPA_ALL are always taken at schedule out 707 * of a task. 708 */ 709 static int pai_check_period(struct perf_event *event, u64 value) 710 { 711 return -EINVAL; 712 } 713 /* ============================= paiext ====================================*/ 714 715 static void paiext_event_destroy(struct perf_event *event) 716 { 717 pai_event_destroy(event); 718 } 719 720 /* Might be called on different CPU than the one the event is intended for. */ 721 static int paiext_event_init(struct perf_event *event) 722 { 723 int rc = pai_event_init(event, PAI_PMU_EXT); 724 725 if (!rc) { 726 event->attr.exclude_kernel = true; /* No kernel space part */ 727 event->destroy = paiext_event_destroy; 728 /* Offset of NNPA in paiext_cb */ 729 event->hw.config_base = offsetof(struct paiext_cb, acc); 730 } 731 return rc; 732 } 733 734 static u64 paiext_getall(struct perf_event *event) 735 { 736 return pai_getdata(event, false); 737 } 738 739 static void paiext_read(struct perf_event *event) 740 { 741 pai_read(event, paiext_getall); 742 } 743 744 static void paiext_start(struct perf_event *event, int flags) 745 { 746 pai_start(event, flags, paiext_getall); 747 } 748 749 static int paiext_add(struct perf_event *event, int flags) 750 { 751 return pai_add(event, flags); 752 } 753 754 static void paiext_stop(struct perf_event *event, int flags) 755 { 756 pai_stop(event, flags); 757 } 758 759 static void paiext_del(struct perf_event *event, int flags) 760 { 761 pai_del(event, flags); 762 } 763 764 /* Called on schedule-in and schedule-out. No access to event structure, 765 * but for sampling only event NNPA_ALL is allowed. 766 */ 767 static void paiext_sched_task(struct perf_event_pmu_context *pmu_ctx, 768 struct task_struct *task, bool sched_in) 769 { 770 /* We started with a clean page on event installation. So read out 771 * results on schedule_out and if page was dirty, save old values. 772 */ 773 if (!sched_in) 774 pai_have_samples(PAI_PMU_EXT); 775 } 776 777 /* Attribute definitions for paicrypt interface. As with other CPU 778 * Measurement Facilities, there is one attribute per mapped counter. 779 * The number of mapped counters may vary per machine generation. Use 780 * the QUERY PROCESSOR ACTIVITY COUNTER INFORMATION (QPACI) instruction 781 * to determine the number of mapped counters. The instructions returns 782 * a positive number, which is the highest number of supported counters. 783 * All counters less than this number are also supported, there are no 784 * holes. A returned number of zero means no support for mapped counters. 785 * 786 * The identification of the counter is a unique number. The chosen range 787 * is 0x1000 + offset in mapped kernel page. 788 * All CPU Measurement Facility counters identifiers must be unique and 789 * the numbers from 0 to 496 are already used for the CPU Measurement 790 * Counter facility. Numbers 0xb0000, 0xbc000 and 0xbd000 are already 791 * used for the CPU Measurement Sampling facility. 792 */ 793 PMU_FORMAT_ATTR(event, "config:0-63"); 794 795 static struct attribute *paicrypt_format_attr[] = { 796 &format_attr_event.attr, 797 NULL, 798 }; 799 800 static struct attribute_group paicrypt_events_group = { 801 .name = "events", 802 .attrs = NULL /* Filled in attr_event_init() */ 803 }; 804 805 static struct attribute_group paicrypt_format_group = { 806 .name = "format", 807 .attrs = paicrypt_format_attr, 808 }; 809 810 static const struct attribute_group *paicrypt_attr_groups[] = { 811 &paicrypt_events_group, 812 &paicrypt_format_group, 813 NULL, 814 }; 815 816 /* Performance monitoring unit for mapped counters */ 817 static struct pmu paicrypt = { 818 .task_ctx_nr = perf_hw_context, 819 .event_init = paicrypt_event_init, 820 .add = paicrypt_add, 821 .del = paicrypt_del, 822 .start = paicrypt_start, 823 .stop = paicrypt_stop, 824 .read = paicrypt_read, 825 .sched_task = paicrypt_sched_task, 826 .check_period = pai_check_period, 827 .attr_groups = paicrypt_attr_groups 828 }; 829 830 /* List of symbolic PAI counter names. */ 831 static const char * const paicrypt_ctrnames[] = { 832 [0] = "CRYPTO_ALL", 833 [1] = "KM_DEA", 834 [2] = "KM_TDEA_128", 835 [3] = "KM_TDEA_192", 836 [4] = "KM_ENCRYPTED_DEA", 837 [5] = "KM_ENCRYPTED_TDEA_128", 838 [6] = "KM_ENCRYPTED_TDEA_192", 839 [7] = "KM_AES_128", 840 [8] = "KM_AES_192", 841 [9] = "KM_AES_256", 842 [10] = "KM_ENCRYPTED_AES_128", 843 [11] = "KM_ENCRYPTED_AES_192", 844 [12] = "KM_ENCRYPTED_AES_256", 845 [13] = "KM_XTS_AES_128", 846 [14] = "KM_XTS_AES_256", 847 [15] = "KM_XTS_ENCRYPTED_AES_128", 848 [16] = "KM_XTS_ENCRYPTED_AES_256", 849 [17] = "KMC_DEA", 850 [18] = "KMC_TDEA_128", 851 [19] = "KMC_TDEA_192", 852 [20] = "KMC_ENCRYPTED_DEA", 853 [21] = "KMC_ENCRYPTED_TDEA_128", 854 [22] = "KMC_ENCRYPTED_TDEA_192", 855 [23] = "KMC_AES_128", 856 [24] = "KMC_AES_192", 857 [25] = "KMC_AES_256", 858 [26] = "KMC_ENCRYPTED_AES_128", 859 [27] = "KMC_ENCRYPTED_AES_192", 860 [28] = "KMC_ENCRYPTED_AES_256", 861 [29] = "KMC_PRNG", 862 [30] = "KMA_GCM_AES_128", 863 [31] = "KMA_GCM_AES_192", 864 [32] = "KMA_GCM_AES_256", 865 [33] = "KMA_GCM_ENCRYPTED_AES_128", 866 [34] = "KMA_GCM_ENCRYPTED_AES_192", 867 [35] = "KMA_GCM_ENCRYPTED_AES_256", 868 [36] = "KMF_DEA", 869 [37] = "KMF_TDEA_128", 870 [38] = "KMF_TDEA_192", 871 [39] = "KMF_ENCRYPTED_DEA", 872 [40] = "KMF_ENCRYPTED_TDEA_128", 873 [41] = "KMF_ENCRYPTED_TDEA_192", 874 [42] = "KMF_AES_128", 875 [43] = "KMF_AES_192", 876 [44] = "KMF_AES_256", 877 [45] = "KMF_ENCRYPTED_AES_128", 878 [46] = "KMF_ENCRYPTED_AES_192", 879 [47] = "KMF_ENCRYPTED_AES_256", 880 [48] = "KMCTR_DEA", 881 [49] = "KMCTR_TDEA_128", 882 [50] = "KMCTR_TDEA_192", 883 [51] = "KMCTR_ENCRYPTED_DEA", 884 [52] = "KMCTR_ENCRYPTED_TDEA_128", 885 [53] = "KMCTR_ENCRYPTED_TDEA_192", 886 [54] = "KMCTR_AES_128", 887 [55] = "KMCTR_AES_192", 888 [56] = "KMCTR_AES_256", 889 [57] = "KMCTR_ENCRYPTED_AES_128", 890 [58] = "KMCTR_ENCRYPTED_AES_192", 891 [59] = "KMCTR_ENCRYPTED_AES_256", 892 [60] = "KMO_DEA", 893 [61] = "KMO_TDEA_128", 894 [62] = "KMO_TDEA_192", 895 [63] = "KMO_ENCRYPTED_DEA", 896 [64] = "KMO_ENCRYPTED_TDEA_128", 897 [65] = "KMO_ENCRYPTED_TDEA_192", 898 [66] = "KMO_AES_128", 899 [67] = "KMO_AES_192", 900 [68] = "KMO_AES_256", 901 [69] = "KMO_ENCRYPTED_AES_128", 902 [70] = "KMO_ENCRYPTED_AES_192", 903 [71] = "KMO_ENCRYPTED_AES_256", 904 [72] = "KIMD_SHA_1", 905 [73] = "KIMD_SHA_256", 906 [74] = "KIMD_SHA_512", 907 [75] = "KIMD_SHA3_224", 908 [76] = "KIMD_SHA3_256", 909 [77] = "KIMD_SHA3_384", 910 [78] = "KIMD_SHA3_512", 911 [79] = "KIMD_SHAKE_128", 912 [80] = "KIMD_SHAKE_256", 913 [81] = "KIMD_GHASH", 914 [82] = "KLMD_SHA_1", 915 [83] = "KLMD_SHA_256", 916 [84] = "KLMD_SHA_512", 917 [85] = "KLMD_SHA3_224", 918 [86] = "KLMD_SHA3_256", 919 [87] = "KLMD_SHA3_384", 920 [88] = "KLMD_SHA3_512", 921 [89] = "KLMD_SHAKE_128", 922 [90] = "KLMD_SHAKE_256", 923 [91] = "KMAC_DEA", 924 [92] = "KMAC_TDEA_128", 925 [93] = "KMAC_TDEA_192", 926 [94] = "KMAC_ENCRYPTED_DEA", 927 [95] = "KMAC_ENCRYPTED_TDEA_128", 928 [96] = "KMAC_ENCRYPTED_TDEA_192", 929 [97] = "KMAC_AES_128", 930 [98] = "KMAC_AES_192", 931 [99] = "KMAC_AES_256", 932 [100] = "KMAC_ENCRYPTED_AES_128", 933 [101] = "KMAC_ENCRYPTED_AES_192", 934 [102] = "KMAC_ENCRYPTED_AES_256", 935 [103] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_DEA", 936 [104] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_TDEA_128", 937 [105] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_TDEA_192", 938 [106] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_DEA", 939 [107] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_TDEA_128", 940 [108] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_TDEA_192", 941 [109] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_128", 942 [110] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_192", 943 [111] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_256", 944 [112] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_128", 945 [113] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_192", 946 [114] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_256", 947 [115] = "PCC_COMPUTE_XTS_PARAMETER_USING_AES_128", 948 [116] = "PCC_COMPUTE_XTS_PARAMETER_USING_AES_256", 949 [117] = "PCC_COMPUTE_XTS_PARAMETER_USING_ENCRYPTED_AES_128", 950 [118] = "PCC_COMPUTE_XTS_PARAMETER_USING_ENCRYPTED_AES_256", 951 [119] = "PCC_SCALAR_MULTIPLY_P256", 952 [120] = "PCC_SCALAR_MULTIPLY_P384", 953 [121] = "PCC_SCALAR_MULTIPLY_P521", 954 [122] = "PCC_SCALAR_MULTIPLY_ED25519", 955 [123] = "PCC_SCALAR_MULTIPLY_ED448", 956 [124] = "PCC_SCALAR_MULTIPLY_X25519", 957 [125] = "PCC_SCALAR_MULTIPLY_X448", 958 [126] = "PRNO_SHA_512_DRNG", 959 [127] = "PRNO_TRNG_QUERY_RAW_TO_CONDITIONED_RATIO", 960 [128] = "PRNO_TRNG", 961 [129] = "KDSA_ECDSA_VERIFY_P256", 962 [130] = "KDSA_ECDSA_VERIFY_P384", 963 [131] = "KDSA_ECDSA_VERIFY_P521", 964 [132] = "KDSA_ECDSA_SIGN_P256", 965 [133] = "KDSA_ECDSA_SIGN_P384", 966 [134] = "KDSA_ECDSA_SIGN_P521", 967 [135] = "KDSA_ENCRYPTED_ECDSA_SIGN_P256", 968 [136] = "KDSA_ENCRYPTED_ECDSA_SIGN_P384", 969 [137] = "KDSA_ENCRYPTED_ECDSA_SIGN_P521", 970 [138] = "KDSA_EDDSA_VERIFY_ED25519", 971 [139] = "KDSA_EDDSA_VERIFY_ED448", 972 [140] = "KDSA_EDDSA_SIGN_ED25519", 973 [141] = "KDSA_EDDSA_SIGN_ED448", 974 [142] = "KDSA_ENCRYPTED_EDDSA_SIGN_ED25519", 975 [143] = "KDSA_ENCRYPTED_EDDSA_SIGN_ED448", 976 [144] = "PCKMO_ENCRYPT_DEA_KEY", 977 [145] = "PCKMO_ENCRYPT_TDEA_128_KEY", 978 [146] = "PCKMO_ENCRYPT_TDEA_192_KEY", 979 [147] = "PCKMO_ENCRYPT_AES_128_KEY", 980 [148] = "PCKMO_ENCRYPT_AES_192_KEY", 981 [149] = "PCKMO_ENCRYPT_AES_256_KEY", 982 [150] = "PCKMO_ENCRYPT_ECC_P256_KEY", 983 [151] = "PCKMO_ENCRYPT_ECC_P384_KEY", 984 [152] = "PCKMO_ENCRYPT_ECC_P521_KEY", 985 [153] = "PCKMO_ENCRYPT_ECC_ED25519_KEY", 986 [154] = "PCKMO_ENCRYPT_ECC_ED448_KEY", 987 [155] = "IBM_RESERVED_155", 988 [156] = "IBM_RESERVED_156", 989 [157] = "KM_FULL_XTS_AES_128", 990 [158] = "KM_FULL_XTS_AES_256", 991 [159] = "KM_FULL_XTS_ENCRYPTED_AES_128", 992 [160] = "KM_FULL_XTS_ENCRYPTED_AES_256", 993 [161] = "KMAC_HMAC_SHA_224", 994 [162] = "KMAC_HMAC_SHA_256", 995 [163] = "KMAC_HMAC_SHA_384", 996 [164] = "KMAC_HMAC_SHA_512", 997 [165] = "KMAC_HMAC_ENCRYPTED_SHA_224", 998 [166] = "KMAC_HMAC_ENCRYPTED_SHA_256", 999 [167] = "KMAC_HMAC_ENCRYPTED_SHA_384", 1000 [168] = "KMAC_HMAC_ENCRYPTED_SHA_512", 1001 [169] = "PCKMO_ENCRYPT_HMAC_512_KEY", 1002 [170] = "PCKMO_ENCRYPT_HMAC_1024_KEY", 1003 [171] = "PCKMO_ENCRYPT_AES_XTS_128", 1004 [172] = "PCKMO_ENCRYPT_AES_XTS_256", 1005 }; 1006 1007 static struct attribute *paiext_format_attr[] = { 1008 &format_attr_event.attr, 1009 NULL, 1010 }; 1011 1012 static struct attribute_group paiext_events_group = { 1013 .name = "events", 1014 .attrs = NULL, /* Filled in attr_event_init() */ 1015 }; 1016 1017 static struct attribute_group paiext_format_group = { 1018 .name = "format", 1019 .attrs = paiext_format_attr, 1020 }; 1021 1022 static const struct attribute_group *paiext_attr_groups[] = { 1023 &paiext_events_group, 1024 &paiext_format_group, 1025 NULL, 1026 }; 1027 1028 /* Performance monitoring unit for mapped counters */ 1029 static struct pmu paiext = { 1030 .task_ctx_nr = perf_hw_context, 1031 .event_init = paiext_event_init, 1032 .add = paiext_add, 1033 .del = paiext_del, 1034 .start = paiext_start, 1035 .stop = paiext_stop, 1036 .read = paiext_read, 1037 .sched_task = paiext_sched_task, 1038 .check_period = pai_check_period, 1039 .attr_groups = paiext_attr_groups, 1040 }; 1041 1042 /* List of symbolic PAI extension 1 NNPA counter names. */ 1043 static const char * const paiext_ctrnames[] = { 1044 [0] = "NNPA_ALL", 1045 [1] = "NNPA_ADD", 1046 [2] = "NNPA_SUB", 1047 [3] = "NNPA_MUL", 1048 [4] = "NNPA_DIV", 1049 [5] = "NNPA_MIN", 1050 [6] = "NNPA_MAX", 1051 [7] = "NNPA_LOG", 1052 [8] = "NNPA_EXP", 1053 [9] = "NNPA_IBM_RESERVED_9", 1054 [10] = "NNPA_RELU", 1055 [11] = "NNPA_TANH", 1056 [12] = "NNPA_SIGMOID", 1057 [13] = "NNPA_SOFTMAX", 1058 [14] = "NNPA_BATCHNORM", 1059 [15] = "NNPA_MAXPOOL2D", 1060 [16] = "NNPA_AVGPOOL2D", 1061 [17] = "NNPA_LSTMACT", 1062 [18] = "NNPA_GRUACT", 1063 [19] = "NNPA_CONVOLUTION", 1064 [20] = "NNPA_MATMUL_OP", 1065 [21] = "NNPA_MATMUL_OP_BCAST23", 1066 [22] = "NNPA_SMALLBATCH", 1067 [23] = "NNPA_LARGEDIM", 1068 [24] = "NNPA_SMALLTENSOR", 1069 [25] = "NNPA_1MFRAME", 1070 [26] = "NNPA_2GFRAME", 1071 [27] = "NNPA_ACCESSEXCEPT", 1072 [28] = "NNPA_TRANSFORM", 1073 [29] = "NNPA_GELU", 1074 [30] = "NNPA_MOMENTS", 1075 [31] = "NNPA_LAYERNORM", 1076 [32] = "NNPA_MATMUL_OP_BCAST1", 1077 [33] = "NNPA_SQRT", 1078 [34] = "NNPA_INVSQRT", 1079 [35] = "NNPA_NORM", 1080 [36] = "NNPA_REDUCE", 1081 }; 1082 1083 static void __init attr_event_free(struct attribute **attrs) 1084 { 1085 struct perf_pmu_events_attr *pa; 1086 unsigned int i; 1087 1088 for (i = 0; attrs[i]; i++) { 1089 struct device_attribute *dap; 1090 1091 dap = container_of(attrs[i], struct device_attribute, attr); 1092 pa = container_of(dap, struct perf_pmu_events_attr, attr); 1093 kfree(pa); 1094 } 1095 kfree(attrs); 1096 } 1097 1098 static struct attribute * __init attr_event_init_one(int num, 1099 unsigned long base, 1100 const char *name) 1101 { 1102 struct perf_pmu_events_attr *pa; 1103 1104 pa = kzalloc_obj(*pa); 1105 if (!pa) 1106 return NULL; 1107 1108 sysfs_attr_init(&pa->attr.attr); 1109 pa->id = base + num; 1110 pa->attr.attr.name = name; 1111 pa->attr.attr.mode = 0444; 1112 pa->attr.show = cpumf_events_sysfs_show; 1113 pa->attr.store = NULL; 1114 return &pa->attr.attr; 1115 } 1116 1117 static struct attribute ** __init attr_event_init(struct pai_pmu *p) 1118 { 1119 unsigned int min_attr = min_t(unsigned int, p->num_named, p->num_avail); 1120 struct attribute **attrs; 1121 unsigned int i; 1122 1123 attrs = kmalloc_objs(*attrs, min_attr + 1, GFP_KERNEL | __GFP_ZERO); 1124 if (!attrs) 1125 goto out; 1126 for (i = 0; i < min_attr; i++) { 1127 attrs[i] = attr_event_init_one(i, p->base, p->names[i]); 1128 if (!attrs[i]) { 1129 attr_event_free(attrs); 1130 attrs = NULL; 1131 goto out; 1132 } 1133 } 1134 attrs[i] = NULL; 1135 out: 1136 return attrs; 1137 } 1138 1139 static void __init pai_pmu_exit(struct pai_pmu *p) 1140 { 1141 attr_event_free(p->event_group->attrs); 1142 p->event_group->attrs = NULL; 1143 } 1144 1145 /* Add a PMU. Install its events and register the PMU device driver 1146 * call back functions. 1147 */ 1148 static int __init pai_pmu_init(struct pai_pmu *p) 1149 { 1150 int rc = -ENOMEM; 1151 1152 1153 /* Export known PAI events */ 1154 p->event_group->attrs = attr_event_init(p); 1155 if (!p->event_group->attrs) { 1156 pr_err("Creation of PMU %s /sysfs failed\n", p->pmuname); 1157 goto out; 1158 } 1159 1160 rc = perf_pmu_register(p->pmu, p->pmuname, -1); 1161 if (rc) { 1162 pai_pmu_exit(p); 1163 pr_err("Registering PMU %s failed with rc=%i\n", p->pmuname, 1164 rc); 1165 } 1166 out: 1167 return rc; 1168 } 1169 1170 /* PAI PMU characteristics table */ 1171 static struct pai_pmu pai_pmu[] __refdata = { 1172 [PAI_PMU_CRYPTO] = { 1173 .pmuname = "pai_crypto", 1174 .facility_nr = 196, 1175 .num_named = ARRAY_SIZE(paicrypt_ctrnames), 1176 .names = paicrypt_ctrnames, 1177 .base = PAI_CRYPTO_BASE, 1178 .kernel_offset = PAI_CRYPTO_KERNEL_OFFSET, 1179 .area_size = PAGE_SIZE, 1180 .init = pai_pmu_init, 1181 .exit = pai_pmu_exit, 1182 .pmu = &paicrypt, 1183 .event_group = &paicrypt_events_group 1184 }, 1185 [PAI_PMU_EXT] = { 1186 .pmuname = "pai_ext", 1187 .facility_nr = 197, 1188 .num_named = ARRAY_SIZE(paiext_ctrnames), 1189 .names = paiext_ctrnames, 1190 .base = PAI_NNPA_BASE, 1191 .kernel_offset = 0, 1192 .area_size = PAIE1_CTRBLOCK_SZ, 1193 .init = pai_pmu_init, 1194 .exit = pai_pmu_exit, 1195 .pmu = &paiext, 1196 .event_group = &paiext_events_group 1197 } 1198 }; 1199 1200 /* 1201 * Check if the PMU (via facility) is supported by machine. Try all of the 1202 * supported PAI PMUs. 1203 * Return number of successfully installed PMUs. 1204 */ 1205 static int __init paipmu_setup(void) 1206 { 1207 struct qpaci_info_block ib; 1208 int install_ok = 0, rc; 1209 struct pai_pmu *p; 1210 size_t i; 1211 1212 for (i = 0; i < ARRAY_SIZE(pai_pmu); ++i) { 1213 p = &pai_pmu[i]; 1214 1215 if (!test_facility(p->facility_nr)) 1216 continue; 1217 1218 qpaci(&ib); 1219 switch (i) { 1220 case PAI_PMU_CRYPTO: 1221 p->num_avail = ib.num_cc; 1222 if (p->num_avail >= PAI_CRYPTO_MAXCTR) { 1223 pr_err("Too many PMU %s counters %d\n", 1224 p->pmuname, p->num_avail); 1225 continue; 1226 } 1227 break; 1228 case PAI_PMU_EXT: 1229 p->num_avail = ib.num_nnpa; 1230 break; 1231 } 1232 p->num_avail += 1; /* Add xxx_ALL event */ 1233 if (p->init) { 1234 rc = p->init(p); 1235 if (!rc) 1236 ++install_ok; 1237 } 1238 } 1239 return install_ok; 1240 } 1241 1242 static int __init pai_init(void) 1243 { 1244 /* Setup s390dbf facility */ 1245 paidbg = debug_register("pai", 1, 1, 128); 1246 if (!paidbg) { 1247 pr_err("Registration of s390dbf pai failed\n"); 1248 return -ENOMEM; 1249 } 1250 debug_register_view(paidbg, &debug_sprintf_view); 1251 1252 if (!paipmu_setup()) { 1253 /* No PMU registration, no need for debug buffer */ 1254 debug_unregister_view(paidbg, &debug_sprintf_view); 1255 debug_unregister(paidbg); 1256 return -ENODEV; 1257 } 1258 return 0; 1259 } 1260 1261 device_initcall(pai_init); 1262