1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Performance event support - Processor Activity Instrumentation Facility 4 * 5 * Copyright IBM Corp. 2026 6 * Author(s): Thomas Richter <tmricht@linux.ibm.com> 7 */ 8 #define pr_fmt(fmt) "pai: " fmt 9 10 #include <linux/kernel.h> 11 #include <linux/kernel_stat.h> 12 #include <linux/percpu.h> 13 #include <linux/notifier.h> 14 #include <linux/init.h> 15 #include <linux/io.h> 16 #include <linux/perf_event.h> 17 #include <asm/ctlreg.h> 18 #include <asm/pai.h> 19 #include <asm/debug.h> 20 21 static debug_info_t *paidbg; 22 23 DEFINE_STATIC_KEY_FALSE(pai_key); 24 25 enum { 26 PAI_PMU_CRYPTO, /* Index of PMU pai_crypto */ 27 PAI_PMU_EXT, /* Index of PMU pai_ext */ 28 PAI_PMU_MAX /* # of PAI PMUs */ 29 }; 30 31 enum { 32 PAIE1_CB_SZ = 0x200, /* Size of PAIE1 control block */ 33 PAIE1_CTRBLOCK_SZ = 0x400 /* Size of PAIE1 counter blocks */ 34 }; 35 36 struct pai_userdata { 37 u16 num; 38 u64 value; 39 } __packed; 40 41 /* Create the PAI extension 1 control block area. 42 * The PAI extension control block 1 is pointed to by lowcore 43 * address 0x1508 for each CPU. This control block is 512 bytes in size 44 * and requires a 512 byte boundary alignment. 45 */ 46 struct paiext_cb { /* PAI extension 1 control block */ 47 u64 header; /* Not used */ 48 u64 reserved1; 49 u64 acc; /* Addr to analytics counter control block */ 50 u8 reserved2[PAIE1_CTRBLOCK_SZ - 3 * sizeof(u64)]; 51 } __packed; 52 53 struct pai_map { 54 unsigned long *area; /* Area for CPU to store counters */ 55 struct pai_userdata *save; /* Page to store no-zero counters */ 56 unsigned int active_events; /* # of PAI crypto users */ 57 refcount_t refcnt; /* Reference count mapped buffers */ 58 struct perf_event *event; /* Perf event for sampling */ 59 struct list_head syswide_list; /* List system-wide sampling events */ 60 struct paiext_cb *paiext_cb; /* PAI extension control block area */ 61 bool fullpage; /* True: counter area is a full page */ 62 }; 63 64 struct pai_mapptr { 65 struct pai_map *mapptr; 66 }; 67 68 static struct pai_root { /* Anchor to per CPU data */ 69 refcount_t refcnt; /* Overall active events */ 70 struct pai_mapptr __percpu *mapptr; 71 } pai_root[PAI_PMU_MAX]; 72 73 /* This table defines the different parameters of the PAI PMUs. During 74 * initialization the machine dependent values are extracted and saved. 75 * However most of the values are static and do not change. 76 * There is one table entry per PAI PMU. 77 */ 78 struct pai_pmu { /* Define PAI PMU characteristics */ 79 const char *pmuname; /* Name of PMU */ 80 const int facility_nr; /* Facility number to check for support */ 81 unsigned int num_avail; /* # Counters defined by hardware */ 82 unsigned int num_named; /* # Counters known by name */ 83 unsigned long base; /* Counter set base number */ 84 unsigned long kernel_offset; /* Offset to kernel part in counter page */ 85 unsigned long area_size; /* Size of counter area */ 86 const char * const *names; /* List of counter names */ 87 struct pmu *pmu; /* Ptr to supporting PMU */ 88 int (*init)(struct pai_pmu *p); /* PMU support init function */ 89 void (*exit)(struct pai_pmu *p); /* PMU support exit function */ 90 struct attribute_group *event_group; /* Ptr to attribute of events */ 91 }; 92 93 static struct pai_pmu pai_pmu[]; /* Forward declaration */ 94 95 /* Free per CPU data when the last event is removed. */ 96 static void pai_root_free(int idx) 97 { 98 if (refcount_dec_and_test(&pai_root[idx].refcnt)) { 99 free_percpu(pai_root[idx].mapptr); 100 pai_root[idx].mapptr = NULL; 101 } 102 debug_sprintf_event(paidbg, 5, "%s root[%d].refcount %d\n", __func__, 103 idx, refcount_read(&pai_root[idx].refcnt)); 104 } 105 106 /* 107 * On initialization of first event also allocate per CPU data dynamically. 108 * Start with an array of pointers, the array size is the maximum number of 109 * CPUs possible, which might be larger than the number of CPUs currently 110 * online. 111 */ 112 static int pai_root_alloc(int idx) 113 { 114 if (!refcount_inc_not_zero(&pai_root[idx].refcnt)) { 115 /* The memory is already zeroed. */ 116 pai_root[idx].mapptr = alloc_percpu(struct pai_mapptr); 117 if (!pai_root[idx].mapptr) 118 return -ENOMEM; 119 refcount_set(&pai_root[idx].refcnt, 1); 120 } 121 return 0; 122 } 123 124 /* Release the PMU if event is the last perf event */ 125 static DEFINE_MUTEX(pai_reserve_mutex); 126 127 /* Free all memory allocated for event counting/sampling setup */ 128 static void pai_free(struct pai_mapptr *mp) 129 { 130 if (mp->mapptr->fullpage) 131 free_page((unsigned long)mp->mapptr->area); 132 else 133 kfree(mp->mapptr->area); 134 kfree(mp->mapptr->paiext_cb); 135 kvfree(mp->mapptr->save); 136 kfree(mp->mapptr); 137 mp->mapptr = NULL; 138 } 139 140 /* Adjust usage counters and remove allocated memory when all users are 141 * gone. 142 */ 143 static void pai_event_destroy_cpu(struct perf_event *event, int cpu) 144 { 145 int idx = PAI_PMU_IDX(event); 146 struct pai_mapptr *mp = per_cpu_ptr(pai_root[idx].mapptr, cpu); 147 struct pai_map *cpump = mp->mapptr; 148 149 mutex_lock(&pai_reserve_mutex); 150 debug_sprintf_event(paidbg, 5, "%s event %#llx idx %d cpu %d users %d " 151 "refcnt %u\n", __func__, event->attr.config, idx, 152 event->cpu, cpump->active_events, 153 refcount_read(&cpump->refcnt)); 154 if (refcount_dec_and_test(&cpump->refcnt)) 155 pai_free(mp); 156 pai_root_free(idx); 157 mutex_unlock(&pai_reserve_mutex); 158 } 159 160 static void pai_event_destroy(struct perf_event *event) 161 { 162 int cpu; 163 164 free_page(PAI_SAVE_AREA(event)); 165 if (event->cpu == -1) { 166 struct cpumask *mask = PAI_CPU_MASK(event); 167 168 for_each_cpu(cpu, mask) 169 pai_event_destroy_cpu(event, cpu); 170 kfree(mask); 171 } else { 172 pai_event_destroy_cpu(event, event->cpu); 173 } 174 } 175 176 static void paicrypt_event_destroy(struct perf_event *event) 177 { 178 static_branch_dec(&pai_key); 179 pai_event_destroy(event); 180 } 181 182 static u64 pai_getctr(unsigned long *page, int nr, unsigned long offset) 183 { 184 if (offset) 185 nr += offset / sizeof(*page); 186 return page[nr]; 187 } 188 189 static void pai_setctr(unsigned long *page, int nr, unsigned long offset, u64 v) 190 { 191 if (offset) 192 nr += offset / sizeof(*page); 193 page[nr] = v; 194 } 195 196 /* Read the counter values. Return value from location in CMP. For base 197 * event xxx_ALL sum up all events. Returns counter value. 198 */ 199 static u64 pai_getdata(struct perf_event *event, bool kernel) 200 { 201 int idx = PAI_PMU_IDX(event); 202 struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr); 203 struct pai_pmu *pp = &pai_pmu[idx]; 204 struct pai_map *cpump = mp->mapptr; 205 unsigned int i; 206 u64 sum = 0; 207 208 if (event->attr.config != pp->base) { 209 return pai_getctr(cpump->area, 210 event->attr.config - pp->base, 211 kernel ? pp->kernel_offset : 0); 212 } 213 214 for (i = 1; i <= pp->num_avail; i++) { 215 u64 val = pai_getctr(cpump->area, i, 216 kernel ? pp->kernel_offset : 0); 217 218 if (!val) 219 continue; 220 sum += val; 221 } 222 return sum; 223 } 224 225 static u64 paicrypt_getall(struct perf_event *event) 226 { 227 u64 sum = 0; 228 229 if (!event->attr.exclude_kernel) 230 sum += pai_getdata(event, true); 231 if (!event->attr.exclude_user) 232 sum += pai_getdata(event, false); 233 234 return sum; 235 } 236 237 /* Check concurrent access of counting and sampling for crypto events. 238 * This function is called in process context and it is save to block. 239 * When the event initialization functions fails, no other call back will 240 * be invoked. 241 * 242 * Allocate the memory for the event. 243 */ 244 static int pai_alloc_cpu(struct perf_event *event, int cpu) 245 { 246 int rc, idx = PAI_PMU_IDX(event); 247 struct pai_map *cpump = NULL; 248 bool need_paiext_cb = false; 249 struct pai_mapptr *mp; 250 251 mutex_lock(&pai_reserve_mutex); 252 /* Allocate root node */ 253 rc = pai_root_alloc(idx); 254 if (rc) 255 goto unlock; 256 257 /* Allocate node for this event */ 258 mp = per_cpu_ptr(pai_root[idx].mapptr, cpu); 259 cpump = mp->mapptr; 260 if (!cpump) { /* Paicrypt_map allocated? */ 261 rc = -ENOMEM; 262 cpump = kzalloc_obj(*cpump); 263 if (!cpump) 264 goto undo; 265 /* Allocate memory for counter page and counter extraction. 266 * Only the first counting event has to allocate a page. 267 */ 268 mp->mapptr = cpump; 269 if (idx == PAI_PMU_CRYPTO) { 270 cpump->area = (unsigned long *)get_zeroed_page(GFP_KERNEL); 271 /* free_page() can handle 0x0 address */ 272 cpump->fullpage = true; 273 } else { /* PAI_PMU_EXT */ 274 /* 275 * Allocate memory for counter area and counter extraction. 276 * These are 277 * - a 512 byte block and requires 512 byte boundary 278 * alignment. 279 * - a 1KB byte block and requires 1KB boundary 280 * alignment. 281 * Only the first counting event has to allocate the area. 282 * 283 * Note: This works with commit 59bb47985c1d by default. 284 * Backporting this to kernels without this commit might 285 * needs adjustment. 286 */ 287 cpump->area = kzalloc(pai_pmu[idx].area_size, GFP_KERNEL); 288 cpump->paiext_cb = kzalloc(PAIE1_CB_SZ, GFP_KERNEL); 289 need_paiext_cb = true; 290 } 291 cpump->save = kvmalloc_objs(struct pai_userdata, 292 pai_pmu[idx].num_avail + 1); 293 if (!cpump->area || !cpump->save || 294 (need_paiext_cb && !cpump->paiext_cb)) { 295 pai_free(mp); 296 goto undo; 297 } 298 INIT_LIST_HEAD(&cpump->syswide_list); 299 refcount_set(&cpump->refcnt, 1); 300 rc = 0; 301 } else { 302 refcount_inc(&cpump->refcnt); 303 } 304 305 undo: 306 if (rc) { 307 /* Error in allocation of event, decrement anchor. Since 308 * the event in not created, its destroy() function is never 309 * invoked. Adjust the reference counter for the anchor. 310 */ 311 pai_root_free(idx); 312 } 313 unlock: 314 mutex_unlock(&pai_reserve_mutex); 315 /* If rc is non-zero, no increment of counter/sampler was done. */ 316 return rc; 317 } 318 319 static int pai_alloc(struct perf_event *event) 320 { 321 struct cpumask *maskptr; 322 int cpu, rc = -ENOMEM; 323 324 maskptr = kzalloc_obj(*maskptr); 325 if (!maskptr) 326 goto out; 327 328 for_each_online_cpu(cpu) { 329 rc = pai_alloc_cpu(event, cpu); 330 if (rc) { 331 for_each_cpu(cpu, maskptr) 332 pai_event_destroy_cpu(event, cpu); 333 kfree(maskptr); 334 goto out; 335 } 336 cpumask_set_cpu(cpu, maskptr); 337 } 338 339 /* 340 * On error all cpumask are freed and all events have been destroyed. 341 * Save of which CPUs data structures have been allocated for. 342 * Release them in pai_event_destroy call back function 343 * for this event. 344 */ 345 PAI_CPU_MASK(event) = maskptr; 346 rc = 0; 347 out: 348 return rc; 349 } 350 351 /* Validate event number and return error if event is not supported. 352 * On successful return, PAI_PMU_IDX(event) is set to the index of 353 * the supporting paing_support[] array element. 354 */ 355 static int pai_event_valid(struct perf_event *event, int idx) 356 { 357 struct perf_event_attr *a = &event->attr; 358 struct pai_pmu *pp = &pai_pmu[idx]; 359 360 /* PAI crypto PMU registered as PERF_TYPE_RAW, check event type */ 361 if (a->type != PERF_TYPE_RAW && event->pmu->type != a->type) 362 return -ENOENT; 363 /* Allow only CRYPTO_ALL/NNPA_ALL for sampling */ 364 if (a->sample_period && a->config != pp->base) 365 return -EINVAL; 366 /* PAI crypto event must be in valid range, try others if not */ 367 if (a->config < pp->base || a->config > pp->base + pp->num_avail) 368 return -ENOENT; 369 if (idx == PAI_PMU_EXT && a->exclude_user) 370 return -EINVAL; 371 PAI_PMU_IDX(event) = idx; 372 return 0; 373 } 374 375 /* Might be called on different CPU than the one the event is intended for. */ 376 static int pai_event_init(struct perf_event *event, int idx) 377 { 378 struct perf_event_attr *a = &event->attr; 379 int rc; 380 381 /* PAI event must be valid and in supported range */ 382 rc = pai_event_valid(event, idx); 383 if (rc) 384 goto out; 385 /* Get a page to store last counter values for sampling */ 386 if (a->sample_period) { 387 PAI_SAVE_AREA(event) = get_zeroed_page(GFP_KERNEL); 388 if (!PAI_SAVE_AREA(event)) { 389 rc = -ENOMEM; 390 goto out; 391 } 392 } 393 394 if (event->cpu >= 0) 395 rc = pai_alloc_cpu(event, event->cpu); 396 else 397 rc = pai_alloc(event); 398 if (rc) { 399 free_page(PAI_SAVE_AREA(event)); 400 goto out; 401 } 402 403 if (a->sample_period) { 404 a->sample_period = 1; 405 a->freq = 0; 406 /* Register for paicrypt_sched_task() to be called */ 407 event->attach_state |= PERF_ATTACH_SCHED_CB; 408 /* Add raw data which contain the memory mapped counters */ 409 a->sample_type |= PERF_SAMPLE_RAW; 410 /* Turn off inheritance */ 411 a->inherit = 0; 412 } 413 out: 414 return rc; 415 } 416 417 static int paicrypt_event_init(struct perf_event *event) 418 { 419 int rc = pai_event_init(event, PAI_PMU_CRYPTO); 420 421 if (!rc) { 422 event->destroy = paicrypt_event_destroy; 423 static_branch_inc(&pai_key); 424 } 425 return rc; 426 } 427 428 static void pai_read(struct perf_event *event, 429 u64 (*fct)(struct perf_event *event)) 430 { 431 u64 prev, new, delta; 432 433 prev = local64_read(&event->hw.prev_count); 434 new = fct(event); 435 local64_set(&event->hw.prev_count, new); 436 delta = (prev <= new) ? new - prev : (-1ULL - prev) + new + 1; 437 local64_add(delta, &event->count); 438 } 439 440 static void paicrypt_read(struct perf_event *event) 441 { 442 pai_read(event, paicrypt_getall); 443 } 444 445 static void pai_start(struct perf_event *event, int flags, 446 u64 (*fct)(struct perf_event *event)) 447 { 448 int idx = PAI_PMU_IDX(event); 449 struct pai_pmu *pp = &pai_pmu[idx]; 450 struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr); 451 struct pai_map *cpump = mp->mapptr; 452 u64 sum; 453 454 if (!event->attr.sample_period) { /* Counting */ 455 sum = fct(event); /* Get current value */ 456 local64_set(&event->hw.prev_count, sum); 457 } else { /* Sampling */ 458 memcpy((void *)PAI_SAVE_AREA(event), cpump->area, pp->area_size); 459 /* Enable context switch callback for system-wide sampling */ 460 if (!(event->attach_state & PERF_ATTACH_TASK)) { 461 list_add_tail(PAI_SWLIST(event), &cpump->syswide_list); 462 perf_sched_cb_inc(event->pmu); 463 } else { 464 cpump->event = event; 465 } 466 } 467 } 468 469 static void paicrypt_start(struct perf_event *event, int flags) 470 { 471 pai_start(event, flags, paicrypt_getall); 472 } 473 474 static int pai_add(struct perf_event *event, int flags) 475 { 476 int idx = PAI_PMU_IDX(event); 477 struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr); 478 struct pai_map *cpump = mp->mapptr; 479 struct paiext_cb *pcb = cpump->paiext_cb; 480 unsigned long ccd; 481 482 if (++cpump->active_events == 1) { 483 if (!pcb) { /* PAI crypto */ 484 ccd = virt_to_phys(cpump->area) | PAI_CRYPTO_KERNEL_OFFSET; 485 WRITE_ONCE(get_lowcore()->ccd, ccd); 486 local_ctl_set_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT); 487 } else { /* PAI extension 1 */ 488 ccd = virt_to_phys(pcb); 489 WRITE_ONCE(get_lowcore()->aicd, ccd); 490 pcb->acc = virt_to_phys(cpump->area) | 0x1; 491 /* Enable CPU instruction lookup for PAIE1 control block */ 492 local_ctl_set_bit(0, CR0_PAI_EXTENSION_BIT); 493 } 494 } 495 if (flags & PERF_EF_START) 496 pai_pmu[idx].pmu->start(event, PERF_EF_RELOAD); 497 event->hw.state = 0; 498 return 0; 499 } 500 501 static int paicrypt_add(struct perf_event *event, int flags) 502 { 503 return pai_add(event, flags); 504 } 505 506 static void pai_have_sample(struct perf_event *, struct pai_map *); 507 static void pai_stop(struct perf_event *event, int flags) 508 { 509 int idx = PAI_PMU_IDX(event); 510 struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr); 511 struct pai_map *cpump = mp->mapptr; 512 513 if (!event->attr.sample_period) { /* Counting */ 514 pai_pmu[idx].pmu->read(event); 515 } else { /* Sampling */ 516 if (!(event->attach_state & PERF_ATTACH_TASK)) { 517 perf_sched_cb_dec(event->pmu); 518 list_del(PAI_SWLIST(event)); 519 } else { 520 pai_have_sample(event, cpump); 521 cpump->event = NULL; 522 } 523 } 524 event->hw.state = PERF_HES_STOPPED; 525 } 526 527 static void paicrypt_stop(struct perf_event *event, int flags) 528 { 529 pai_stop(event, flags); 530 } 531 532 static void pai_del(struct perf_event *event, int flags) 533 { 534 int idx = PAI_PMU_IDX(event); 535 struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr); 536 struct pai_map *cpump = mp->mapptr; 537 struct paiext_cb *pcb = cpump->paiext_cb; 538 539 pai_pmu[idx].pmu->stop(event, PERF_EF_UPDATE); 540 if (--cpump->active_events == 0) { 541 if (!pcb) { /* PAI crypto */ 542 local_ctl_clear_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT); 543 WRITE_ONCE(get_lowcore()->ccd, 0); 544 } else { /* PAI extension 1 */ 545 /* Disable CPU instruction lookup for PAIE1 control block */ 546 local_ctl_clear_bit(0, CR0_PAI_EXTENSION_BIT); 547 pcb->acc = 0; 548 WRITE_ONCE(get_lowcore()->aicd, 0); 549 } 550 } 551 } 552 553 static void paicrypt_del(struct perf_event *event, int flags) 554 { 555 pai_del(event, flags); 556 } 557 558 /* Create raw data and save it in buffer. Calculate the delta for each 559 * counter between this invocation and the last invocation. 560 * Returns number of bytes copied. 561 * After reading from PAI counter page, save the read value to the old 562 * page to calculate PAI counter deltas. 563 * Saves only entries with positive counter difference of the form 564 * 2 bytes: Number of counter 565 * 8 bytes: Value of counter 566 */ 567 static size_t pai_copy(struct pai_userdata *userdata, unsigned long *page, 568 struct pai_pmu *pp, unsigned long *page_old, 569 bool exclude_user, bool exclude_kernel) 570 { 571 int i, outidx = 0; 572 573 for (i = 1; i <= pp->num_avail; i++) { 574 u64 val = 0, val_old = 0, val_k = 0, val_old_k = 0; 575 576 if (!exclude_kernel) { 577 val_k = pai_getctr(page, i, pp->kernel_offset); 578 val_old_k = pai_getctr(page_old, i, pp->kernel_offset); 579 if (val_k != val_old_k) 580 pai_setctr(page_old, i, pp->kernel_offset, val_k); 581 } 582 if (!exclude_user) { 583 val = pai_getctr(page, i, 0); 584 val_old = pai_getctr(page_old, i, 0); 585 if (val != val_old) 586 pai_setctr(page_old, i, 0, val); 587 } 588 val += val_k; 589 val_old += val_old_k; 590 if (val >= val_old) 591 val -= val_old; 592 else 593 val = (~0ULL - val_old) + val + 1; 594 if (val) { 595 userdata[outidx].num = i; 596 userdata[outidx].value = val; 597 outidx++; 598 } 599 } 600 return outidx * sizeof(*userdata); 601 } 602 603 /* Write sample when one or more counters values are nonzero. 604 * 605 * Note: The function paicrypt_sched_task() and pai_push_sample() are not 606 * invoked after function paicrypt_del() has been called because of function 607 * perf_sched_cb_dec(). Both functions are only 608 * called when sampling is active. Function perf_sched_cb_inc() 609 * has been invoked to install function paicrypt_sched_task() as call back 610 * to run at context switch time. 611 * 612 * This causes function perf_event_context_sched_out() and 613 * perf_event_context_sched_in() to check whether the PMU has installed an 614 * sched_task() callback. That callback is not active after paicrypt_del() 615 * returns and has deleted the event on that CPU. 616 */ 617 static int pai_push_sample(size_t rawsize, struct pai_map *cpump, 618 struct perf_event *event) 619 { 620 struct perf_sample_data data; 621 struct perf_raw_record raw; 622 struct pt_regs regs; 623 int overflow; 624 625 /* Setup perf sample */ 626 memset(®s, 0, sizeof(regs)); 627 memset(&raw, 0, sizeof(raw)); 628 memset(&data, 0, sizeof(data)); 629 perf_sample_data_init(&data, 0, event->hw.last_period); 630 if (event->attr.sample_type & PERF_SAMPLE_TID) { 631 data.tid_entry.pid = task_tgid_nr(current); 632 data.tid_entry.tid = task_pid_nr(current); 633 } 634 if (event->attr.sample_type & PERF_SAMPLE_TIME) 635 data.time = event->clock(); 636 if (event->attr.sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER)) 637 data.id = event->id; 638 if (event->attr.sample_type & PERF_SAMPLE_CPU) { 639 data.cpu_entry.cpu = smp_processor_id(); 640 data.cpu_entry.reserved = 0; 641 } 642 if (event->attr.sample_type & PERF_SAMPLE_RAW) { 643 raw.frag.size = rawsize; 644 raw.frag.data = cpump->save; 645 perf_sample_save_raw_data(&data, event, &raw); 646 } 647 648 overflow = perf_event_overflow(event, &data, ®s); 649 perf_event_update_userpage(event); 650 return overflow; 651 } 652 653 /* Check if there is data to be saved on schedule out of a task. */ 654 static void pai_have_sample(struct perf_event *event, struct pai_map *cpump) 655 { 656 struct pai_pmu *pp; 657 size_t rawsize; 658 659 if (!event) /* No event active */ 660 return; 661 pp = &pai_pmu[PAI_PMU_IDX(event)]; 662 rawsize = pai_copy(cpump->save, cpump->area, pp, 663 (unsigned long *)PAI_SAVE_AREA(event), 664 event->attr.exclude_user, 665 !pp->kernel_offset ? true : event->attr.exclude_kernel); 666 if (rawsize) /* No incremented counters */ 667 pai_push_sample(rawsize, cpump, event); 668 } 669 670 /* Check if there is data to be saved on schedule out of a task. */ 671 static void pai_have_samples(int idx) 672 { 673 struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr); 674 struct pai_map *cpump = mp->mapptr; 675 struct perf_event *event; 676 677 list_for_each_entry(event, &cpump->syswide_list, hw.tp_list) 678 pai_have_sample(event, cpump); 679 } 680 681 /* Called on schedule-in and schedule-out. No access to event structure, 682 * but for sampling only event CRYPTO_ALL is allowed. 683 */ 684 static void paicrypt_sched_task(struct perf_event_pmu_context *pmu_ctx, 685 struct task_struct *task, bool sched_in) 686 { 687 /* We started with a clean page on event installation. So read out 688 * results on schedule_out and if page was dirty, save old values. 689 */ 690 if (!sched_in) 691 pai_have_samples(PAI_PMU_CRYPTO); 692 } 693 694 /* ============================= paiext ====================================*/ 695 696 static void paiext_event_destroy(struct perf_event *event) 697 { 698 pai_event_destroy(event); 699 } 700 701 /* Might be called on different CPU than the one the event is intended for. */ 702 static int paiext_event_init(struct perf_event *event) 703 { 704 int rc = pai_event_init(event, PAI_PMU_EXT); 705 706 if (!rc) { 707 event->attr.exclude_kernel = true; /* No kernel space part */ 708 event->destroy = paiext_event_destroy; 709 /* Offset of NNPA in paiext_cb */ 710 event->hw.config_base = offsetof(struct paiext_cb, acc); 711 } 712 return rc; 713 } 714 715 static u64 paiext_getall(struct perf_event *event) 716 { 717 return pai_getdata(event, false); 718 } 719 720 static void paiext_read(struct perf_event *event) 721 { 722 pai_read(event, paiext_getall); 723 } 724 725 static void paiext_start(struct perf_event *event, int flags) 726 { 727 pai_start(event, flags, paiext_getall); 728 } 729 730 static int paiext_add(struct perf_event *event, int flags) 731 { 732 return pai_add(event, flags); 733 } 734 735 static void paiext_stop(struct perf_event *event, int flags) 736 { 737 pai_stop(event, flags); 738 } 739 740 static void paiext_del(struct perf_event *event, int flags) 741 { 742 pai_del(event, flags); 743 } 744 745 /* Called on schedule-in and schedule-out. No access to event structure, 746 * but for sampling only event NNPA_ALL is allowed. 747 */ 748 static void paiext_sched_task(struct perf_event_pmu_context *pmu_ctx, 749 struct task_struct *task, bool sched_in) 750 { 751 /* We started with a clean page on event installation. So read out 752 * results on schedule_out and if page was dirty, save old values. 753 */ 754 if (!sched_in) 755 pai_have_samples(PAI_PMU_EXT); 756 } 757 758 /* Attribute definitions for paicrypt interface. As with other CPU 759 * Measurement Facilities, there is one attribute per mapped counter. 760 * The number of mapped counters may vary per machine generation. Use 761 * the QUERY PROCESSOR ACTIVITY COUNTER INFORMATION (QPACI) instruction 762 * to determine the number of mapped counters. The instructions returns 763 * a positive number, which is the highest number of supported counters. 764 * All counters less than this number are also supported, there are no 765 * holes. A returned number of zero means no support for mapped counters. 766 * 767 * The identification of the counter is a unique number. The chosen range 768 * is 0x1000 + offset in mapped kernel page. 769 * All CPU Measurement Facility counters identifiers must be unique and 770 * the numbers from 0 to 496 are already used for the CPU Measurement 771 * Counter facility. Numbers 0xb0000, 0xbc000 and 0xbd000 are already 772 * used for the CPU Measurement Sampling facility. 773 */ 774 PMU_FORMAT_ATTR(event, "config:0-63"); 775 776 static struct attribute *paicrypt_format_attr[] = { 777 &format_attr_event.attr, 778 NULL, 779 }; 780 781 static struct attribute_group paicrypt_events_group = { 782 .name = "events", 783 .attrs = NULL /* Filled in attr_event_init() */ 784 }; 785 786 static struct attribute_group paicrypt_format_group = { 787 .name = "format", 788 .attrs = paicrypt_format_attr, 789 }; 790 791 static const struct attribute_group *paicrypt_attr_groups[] = { 792 &paicrypt_events_group, 793 &paicrypt_format_group, 794 NULL, 795 }; 796 797 /* Performance monitoring unit for mapped counters */ 798 static struct pmu paicrypt = { 799 .task_ctx_nr = perf_hw_context, 800 .event_init = paicrypt_event_init, 801 .add = paicrypt_add, 802 .del = paicrypt_del, 803 .start = paicrypt_start, 804 .stop = paicrypt_stop, 805 .read = paicrypt_read, 806 .sched_task = paicrypt_sched_task, 807 .attr_groups = paicrypt_attr_groups 808 }; 809 810 /* List of symbolic PAI counter names. */ 811 static const char * const paicrypt_ctrnames[] = { 812 [0] = "CRYPTO_ALL", 813 [1] = "KM_DEA", 814 [2] = "KM_TDEA_128", 815 [3] = "KM_TDEA_192", 816 [4] = "KM_ENCRYPTED_DEA", 817 [5] = "KM_ENCRYPTED_TDEA_128", 818 [6] = "KM_ENCRYPTED_TDEA_192", 819 [7] = "KM_AES_128", 820 [8] = "KM_AES_192", 821 [9] = "KM_AES_256", 822 [10] = "KM_ENCRYPTED_AES_128", 823 [11] = "KM_ENCRYPTED_AES_192", 824 [12] = "KM_ENCRYPTED_AES_256", 825 [13] = "KM_XTS_AES_128", 826 [14] = "KM_XTS_AES_256", 827 [15] = "KM_XTS_ENCRYPTED_AES_128", 828 [16] = "KM_XTS_ENCRYPTED_AES_256", 829 [17] = "KMC_DEA", 830 [18] = "KMC_TDEA_128", 831 [19] = "KMC_TDEA_192", 832 [20] = "KMC_ENCRYPTED_DEA", 833 [21] = "KMC_ENCRYPTED_TDEA_128", 834 [22] = "KMC_ENCRYPTED_TDEA_192", 835 [23] = "KMC_AES_128", 836 [24] = "KMC_AES_192", 837 [25] = "KMC_AES_256", 838 [26] = "KMC_ENCRYPTED_AES_128", 839 [27] = "KMC_ENCRYPTED_AES_192", 840 [28] = "KMC_ENCRYPTED_AES_256", 841 [29] = "KMC_PRNG", 842 [30] = "KMA_GCM_AES_128", 843 [31] = "KMA_GCM_AES_192", 844 [32] = "KMA_GCM_AES_256", 845 [33] = "KMA_GCM_ENCRYPTED_AES_128", 846 [34] = "KMA_GCM_ENCRYPTED_AES_192", 847 [35] = "KMA_GCM_ENCRYPTED_AES_256", 848 [36] = "KMF_DEA", 849 [37] = "KMF_TDEA_128", 850 [38] = "KMF_TDEA_192", 851 [39] = "KMF_ENCRYPTED_DEA", 852 [40] = "KMF_ENCRYPTED_TDEA_128", 853 [41] = "KMF_ENCRYPTED_TDEA_192", 854 [42] = "KMF_AES_128", 855 [43] = "KMF_AES_192", 856 [44] = "KMF_AES_256", 857 [45] = "KMF_ENCRYPTED_AES_128", 858 [46] = "KMF_ENCRYPTED_AES_192", 859 [47] = "KMF_ENCRYPTED_AES_256", 860 [48] = "KMCTR_DEA", 861 [49] = "KMCTR_TDEA_128", 862 [50] = "KMCTR_TDEA_192", 863 [51] = "KMCTR_ENCRYPTED_DEA", 864 [52] = "KMCTR_ENCRYPTED_TDEA_128", 865 [53] = "KMCTR_ENCRYPTED_TDEA_192", 866 [54] = "KMCTR_AES_128", 867 [55] = "KMCTR_AES_192", 868 [56] = "KMCTR_AES_256", 869 [57] = "KMCTR_ENCRYPTED_AES_128", 870 [58] = "KMCTR_ENCRYPTED_AES_192", 871 [59] = "KMCTR_ENCRYPTED_AES_256", 872 [60] = "KMO_DEA", 873 [61] = "KMO_TDEA_128", 874 [62] = "KMO_TDEA_192", 875 [63] = "KMO_ENCRYPTED_DEA", 876 [64] = "KMO_ENCRYPTED_TDEA_128", 877 [65] = "KMO_ENCRYPTED_TDEA_192", 878 [66] = "KMO_AES_128", 879 [67] = "KMO_AES_192", 880 [68] = "KMO_AES_256", 881 [69] = "KMO_ENCRYPTED_AES_128", 882 [70] = "KMO_ENCRYPTED_AES_192", 883 [71] = "KMO_ENCRYPTED_AES_256", 884 [72] = "KIMD_SHA_1", 885 [73] = "KIMD_SHA_256", 886 [74] = "KIMD_SHA_512", 887 [75] = "KIMD_SHA3_224", 888 [76] = "KIMD_SHA3_256", 889 [77] = "KIMD_SHA3_384", 890 [78] = "KIMD_SHA3_512", 891 [79] = "KIMD_SHAKE_128", 892 [80] = "KIMD_SHAKE_256", 893 [81] = "KIMD_GHASH", 894 [82] = "KLMD_SHA_1", 895 [83] = "KLMD_SHA_256", 896 [84] = "KLMD_SHA_512", 897 [85] = "KLMD_SHA3_224", 898 [86] = "KLMD_SHA3_256", 899 [87] = "KLMD_SHA3_384", 900 [88] = "KLMD_SHA3_512", 901 [89] = "KLMD_SHAKE_128", 902 [90] = "KLMD_SHAKE_256", 903 [91] = "KMAC_DEA", 904 [92] = "KMAC_TDEA_128", 905 [93] = "KMAC_TDEA_192", 906 [94] = "KMAC_ENCRYPTED_DEA", 907 [95] = "KMAC_ENCRYPTED_TDEA_128", 908 [96] = "KMAC_ENCRYPTED_TDEA_192", 909 [97] = "KMAC_AES_128", 910 [98] = "KMAC_AES_192", 911 [99] = "KMAC_AES_256", 912 [100] = "KMAC_ENCRYPTED_AES_128", 913 [101] = "KMAC_ENCRYPTED_AES_192", 914 [102] = "KMAC_ENCRYPTED_AES_256", 915 [103] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_DEA", 916 [104] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_TDEA_128", 917 [105] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_TDEA_192", 918 [106] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_DEA", 919 [107] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_TDEA_128", 920 [108] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_TDEA_192", 921 [109] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_128", 922 [110] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_192", 923 [111] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_256", 924 [112] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_128", 925 [113] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_192", 926 [114] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_256", 927 [115] = "PCC_COMPUTE_XTS_PARAMETER_USING_AES_128", 928 [116] = "PCC_COMPUTE_XTS_PARAMETER_USING_AES_256", 929 [117] = "PCC_COMPUTE_XTS_PARAMETER_USING_ENCRYPTED_AES_128", 930 [118] = "PCC_COMPUTE_XTS_PARAMETER_USING_ENCRYPTED_AES_256", 931 [119] = "PCC_SCALAR_MULTIPLY_P256", 932 [120] = "PCC_SCALAR_MULTIPLY_P384", 933 [121] = "PCC_SCALAR_MULTIPLY_P521", 934 [122] = "PCC_SCALAR_MULTIPLY_ED25519", 935 [123] = "PCC_SCALAR_MULTIPLY_ED448", 936 [124] = "PCC_SCALAR_MULTIPLY_X25519", 937 [125] = "PCC_SCALAR_MULTIPLY_X448", 938 [126] = "PRNO_SHA_512_DRNG", 939 [127] = "PRNO_TRNG_QUERY_RAW_TO_CONDITIONED_RATIO", 940 [128] = "PRNO_TRNG", 941 [129] = "KDSA_ECDSA_VERIFY_P256", 942 [130] = "KDSA_ECDSA_VERIFY_P384", 943 [131] = "KDSA_ECDSA_VERIFY_P521", 944 [132] = "KDSA_ECDSA_SIGN_P256", 945 [133] = "KDSA_ECDSA_SIGN_P384", 946 [134] = "KDSA_ECDSA_SIGN_P521", 947 [135] = "KDSA_ENCRYPTED_ECDSA_SIGN_P256", 948 [136] = "KDSA_ENCRYPTED_ECDSA_SIGN_P384", 949 [137] = "KDSA_ENCRYPTED_ECDSA_SIGN_P521", 950 [138] = "KDSA_EDDSA_VERIFY_ED25519", 951 [139] = "KDSA_EDDSA_VERIFY_ED448", 952 [140] = "KDSA_EDDSA_SIGN_ED25519", 953 [141] = "KDSA_EDDSA_SIGN_ED448", 954 [142] = "KDSA_ENCRYPTED_EDDSA_SIGN_ED25519", 955 [143] = "KDSA_ENCRYPTED_EDDSA_SIGN_ED448", 956 [144] = "PCKMO_ENCRYPT_DEA_KEY", 957 [145] = "PCKMO_ENCRYPT_TDEA_128_KEY", 958 [146] = "PCKMO_ENCRYPT_TDEA_192_KEY", 959 [147] = "PCKMO_ENCRYPT_AES_128_KEY", 960 [148] = "PCKMO_ENCRYPT_AES_192_KEY", 961 [149] = "PCKMO_ENCRYPT_AES_256_KEY", 962 [150] = "PCKMO_ENCRYPT_ECC_P256_KEY", 963 [151] = "PCKMO_ENCRYPT_ECC_P384_KEY", 964 [152] = "PCKMO_ENCRYPT_ECC_P521_KEY", 965 [153] = "PCKMO_ENCRYPT_ECC_ED25519_KEY", 966 [154] = "PCKMO_ENCRYPT_ECC_ED448_KEY", 967 [155] = "IBM_RESERVED_155", 968 [156] = "IBM_RESERVED_156", 969 [157] = "KM_FULL_XTS_AES_128", 970 [158] = "KM_FULL_XTS_AES_256", 971 [159] = "KM_FULL_XTS_ENCRYPTED_AES_128", 972 [160] = "KM_FULL_XTS_ENCRYPTED_AES_256", 973 [161] = "KMAC_HMAC_SHA_224", 974 [162] = "KMAC_HMAC_SHA_256", 975 [163] = "KMAC_HMAC_SHA_384", 976 [164] = "KMAC_HMAC_SHA_512", 977 [165] = "KMAC_HMAC_ENCRYPTED_SHA_224", 978 [166] = "KMAC_HMAC_ENCRYPTED_SHA_256", 979 [167] = "KMAC_HMAC_ENCRYPTED_SHA_384", 980 [168] = "KMAC_HMAC_ENCRYPTED_SHA_512", 981 [169] = "PCKMO_ENCRYPT_HMAC_512_KEY", 982 [170] = "PCKMO_ENCRYPT_HMAC_1024_KEY", 983 [171] = "PCKMO_ENCRYPT_AES_XTS_128", 984 [172] = "PCKMO_ENCRYPT_AES_XTS_256", 985 }; 986 987 static struct attribute *paiext_format_attr[] = { 988 &format_attr_event.attr, 989 NULL, 990 }; 991 992 static struct attribute_group paiext_events_group = { 993 .name = "events", 994 .attrs = NULL, /* Filled in attr_event_init() */ 995 }; 996 997 static struct attribute_group paiext_format_group = { 998 .name = "format", 999 .attrs = paiext_format_attr, 1000 }; 1001 1002 static const struct attribute_group *paiext_attr_groups[] = { 1003 &paiext_events_group, 1004 &paiext_format_group, 1005 NULL, 1006 }; 1007 1008 /* Performance monitoring unit for mapped counters */ 1009 static struct pmu paiext = { 1010 .task_ctx_nr = perf_hw_context, 1011 .event_init = paiext_event_init, 1012 .add = paiext_add, 1013 .del = paiext_del, 1014 .start = paiext_start, 1015 .stop = paiext_stop, 1016 .read = paiext_read, 1017 .sched_task = paiext_sched_task, 1018 .attr_groups = paiext_attr_groups, 1019 }; 1020 1021 /* List of symbolic PAI extension 1 NNPA counter names. */ 1022 static const char * const paiext_ctrnames[] = { 1023 [0] = "NNPA_ALL", 1024 [1] = "NNPA_ADD", 1025 [2] = "NNPA_SUB", 1026 [3] = "NNPA_MUL", 1027 [4] = "NNPA_DIV", 1028 [5] = "NNPA_MIN", 1029 [6] = "NNPA_MAX", 1030 [7] = "NNPA_LOG", 1031 [8] = "NNPA_EXP", 1032 [9] = "NNPA_IBM_RESERVED_9", 1033 [10] = "NNPA_RELU", 1034 [11] = "NNPA_TANH", 1035 [12] = "NNPA_SIGMOID", 1036 [13] = "NNPA_SOFTMAX", 1037 [14] = "NNPA_BATCHNORM", 1038 [15] = "NNPA_MAXPOOL2D", 1039 [16] = "NNPA_AVGPOOL2D", 1040 [17] = "NNPA_LSTMACT", 1041 [18] = "NNPA_GRUACT", 1042 [19] = "NNPA_CONVOLUTION", 1043 [20] = "NNPA_MATMUL_OP", 1044 [21] = "NNPA_MATMUL_OP_BCAST23", 1045 [22] = "NNPA_SMALLBATCH", 1046 [23] = "NNPA_LARGEDIM", 1047 [24] = "NNPA_SMALLTENSOR", 1048 [25] = "NNPA_1MFRAME", 1049 [26] = "NNPA_2GFRAME", 1050 [27] = "NNPA_ACCESSEXCEPT", 1051 [28] = "NNPA_TRANSFORM", 1052 [29] = "NNPA_GELU", 1053 [30] = "NNPA_MOMENTS", 1054 [31] = "NNPA_LAYERNORM", 1055 [32] = "NNPA_MATMUL_OP_BCAST1", 1056 [33] = "NNPA_SQRT", 1057 [34] = "NNPA_INVSQRT", 1058 [35] = "NNPA_NORM", 1059 [36] = "NNPA_REDUCE", 1060 }; 1061 1062 static void __init attr_event_free(struct attribute **attrs) 1063 { 1064 struct perf_pmu_events_attr *pa; 1065 unsigned int i; 1066 1067 for (i = 0; attrs[i]; i++) { 1068 struct device_attribute *dap; 1069 1070 dap = container_of(attrs[i], struct device_attribute, attr); 1071 pa = container_of(dap, struct perf_pmu_events_attr, attr); 1072 kfree(pa); 1073 } 1074 kfree(attrs); 1075 } 1076 1077 static struct attribute * __init attr_event_init_one(int num, 1078 unsigned long base, 1079 const char *name) 1080 { 1081 struct perf_pmu_events_attr *pa; 1082 1083 pa = kzalloc_obj(*pa); 1084 if (!pa) 1085 return NULL; 1086 1087 sysfs_attr_init(&pa->attr.attr); 1088 pa->id = base + num; 1089 pa->attr.attr.name = name; 1090 pa->attr.attr.mode = 0444; 1091 pa->attr.show = cpumf_events_sysfs_show; 1092 pa->attr.store = NULL; 1093 return &pa->attr.attr; 1094 } 1095 1096 static struct attribute ** __init attr_event_init(struct pai_pmu *p) 1097 { 1098 unsigned int min_attr = min_t(unsigned int, p->num_named, p->num_avail); 1099 struct attribute **attrs; 1100 unsigned int i; 1101 1102 attrs = kmalloc_objs(*attrs, min_attr + 1, GFP_KERNEL | __GFP_ZERO); 1103 if (!attrs) 1104 goto out; 1105 for (i = 0; i < min_attr; i++) { 1106 attrs[i] = attr_event_init_one(i, p->base, p->names[i]); 1107 if (!attrs[i]) { 1108 attr_event_free(attrs); 1109 attrs = NULL; 1110 goto out; 1111 } 1112 } 1113 attrs[i] = NULL; 1114 out: 1115 return attrs; 1116 } 1117 1118 static void __init pai_pmu_exit(struct pai_pmu *p) 1119 { 1120 attr_event_free(p->event_group->attrs); 1121 p->event_group->attrs = NULL; 1122 } 1123 1124 /* Add a PMU. Install its events and register the PMU device driver 1125 * call back functions. 1126 */ 1127 static int __init pai_pmu_init(struct pai_pmu *p) 1128 { 1129 int rc = -ENOMEM; 1130 1131 1132 /* Export known PAI events */ 1133 p->event_group->attrs = attr_event_init(p); 1134 if (!p->event_group->attrs) { 1135 pr_err("Creation of PMU %s /sysfs failed\n", p->pmuname); 1136 goto out; 1137 } 1138 1139 rc = perf_pmu_register(p->pmu, p->pmuname, -1); 1140 if (rc) { 1141 pai_pmu_exit(p); 1142 pr_err("Registering PMU %s failed with rc=%i\n", p->pmuname, 1143 rc); 1144 } 1145 out: 1146 return rc; 1147 } 1148 1149 /* PAI PMU characteristics table */ 1150 static struct pai_pmu pai_pmu[] __refdata = { 1151 [PAI_PMU_CRYPTO] = { 1152 .pmuname = "pai_crypto", 1153 .facility_nr = 196, 1154 .num_named = ARRAY_SIZE(paicrypt_ctrnames), 1155 .names = paicrypt_ctrnames, 1156 .base = PAI_CRYPTO_BASE, 1157 .kernel_offset = PAI_CRYPTO_KERNEL_OFFSET, 1158 .area_size = PAGE_SIZE, 1159 .init = pai_pmu_init, 1160 .exit = pai_pmu_exit, 1161 .pmu = &paicrypt, 1162 .event_group = &paicrypt_events_group 1163 }, 1164 [PAI_PMU_EXT] = { 1165 .pmuname = "pai_ext", 1166 .facility_nr = 197, 1167 .num_named = ARRAY_SIZE(paiext_ctrnames), 1168 .names = paiext_ctrnames, 1169 .base = PAI_NNPA_BASE, 1170 .kernel_offset = 0, 1171 .area_size = PAIE1_CTRBLOCK_SZ, 1172 .init = pai_pmu_init, 1173 .exit = pai_pmu_exit, 1174 .pmu = &paiext, 1175 .event_group = &paiext_events_group 1176 } 1177 }; 1178 1179 /* 1180 * Check if the PMU (via facility) is supported by machine. Try all of the 1181 * supported PAI PMUs. 1182 * Return number of successfully installed PMUs. 1183 */ 1184 static int __init paipmu_setup(void) 1185 { 1186 struct qpaci_info_block ib; 1187 int install_ok = 0, rc; 1188 struct pai_pmu *p; 1189 size_t i; 1190 1191 for (i = 0; i < ARRAY_SIZE(pai_pmu); ++i) { 1192 p = &pai_pmu[i]; 1193 1194 if (!test_facility(p->facility_nr)) 1195 continue; 1196 1197 qpaci(&ib); 1198 switch (i) { 1199 case PAI_PMU_CRYPTO: 1200 p->num_avail = ib.num_cc; 1201 if (p->num_avail >= PAI_CRYPTO_MAXCTR) { 1202 pr_err("Too many PMU %s counters %d\n", 1203 p->pmuname, p->num_avail); 1204 continue; 1205 } 1206 break; 1207 case PAI_PMU_EXT: 1208 p->num_avail = ib.num_nnpa; 1209 break; 1210 } 1211 p->num_avail += 1; /* Add xxx_ALL event */ 1212 if (p->init) { 1213 rc = p->init(p); 1214 if (!rc) 1215 ++install_ok; 1216 } 1217 } 1218 return install_ok; 1219 } 1220 1221 static int __init pai_init(void) 1222 { 1223 /* Setup s390dbf facility */ 1224 paidbg = debug_register("pai", 32, 256, 128); 1225 if (!paidbg) { 1226 pr_err("Registration of s390dbf pai failed\n"); 1227 return -ENOMEM; 1228 } 1229 debug_register_view(paidbg, &debug_sprintf_view); 1230 1231 if (!paipmu_setup()) { 1232 /* No PMU registration, no need for debug buffer */ 1233 debug_unregister_view(paidbg, &debug_sprintf_view); 1234 debug_unregister(paidbg); 1235 return -ENODEV; 1236 } 1237 return 0; 1238 } 1239 1240 device_initcall(pai_init); 1241