1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Performance event support - Processor Activity Instrumentation Facility
4 *
5 * Copyright IBM Corp. 2026
6 * Author(s): Thomas Richter <tmricht@linux.ibm.com>
7 */
8 #define pr_fmt(fmt) "pai: " fmt
9
10 #include <linux/kernel.h>
11 #include <linux/kernel_stat.h>
12 #include <linux/percpu.h>
13 #include <linux/notifier.h>
14 #include <linux/init.h>
15 #include <linux/io.h>
16 #include <linux/perf_event.h>
17 #include <asm/ctlreg.h>
18 #include <asm/pai.h>
19 #include <asm/debug.h>
20
21 static debug_info_t *paidbg;
22
23 DEFINE_STATIC_KEY_FALSE(pai_key);
24
25 enum {
26 PAI_PMU_CRYPTO, /* Index of PMU pai_crypto */
27 PAI_PMU_EXT, /* Index of PMU pai_ext */
28 PAI_PMU_MAX /* # of PAI PMUs */
29 };
30
31 enum {
32 PAIE1_CB_SZ = 0x200, /* Size of PAIE1 control block */
33 PAIE1_CTRBLOCK_SZ = 0x400 /* Size of PAIE1 counter blocks */
34 };
35
36 struct pai_userdata {
37 u16 num;
38 u64 value;
39 } __packed;
40
41 /* Create the PAI extension 1 control block area.
42 * The PAI extension control block 1 is pointed to by lowcore
43 * address 0x1508 for each CPU. This control block is 512 bytes in size
44 * and requires a 512 byte boundary alignment.
45 */
46 struct paiext_cb { /* PAI extension 1 control block */
47 u64 header; /* Not used */
48 u64 reserved1;
49 u64 acc; /* Addr to analytics counter control block */
50 u8 reserved2[PAIE1_CTRBLOCK_SZ - 3 * sizeof(u64)];
51 } __packed;
52
53 struct pai_map {
54 unsigned long *area; /* Area for CPU to store counters */
55 struct pai_userdata *save; /* Page to store no-zero counters */
56 unsigned int active_events; /* # of PAI crypto users */
57 refcount_t refcnt; /* Reference count mapped buffers */
58 struct perf_event *event; /* Perf event for sampling */
59 struct list_head syswide_list; /* List system-wide sampling events */
60 struct paiext_cb *paiext_cb; /* PAI extension control block area */
61 bool fullpage; /* True: counter area is a full page */
62 };
63
64 struct pai_mapptr {
65 struct pai_map *mapptr;
66 };
67
68 static struct pai_root { /* Anchor to per CPU data */
69 refcount_t refcnt; /* Overall active events */
70 struct pai_mapptr __percpu *mapptr;
71 } pai_root[PAI_PMU_MAX];
72
73 /* This table defines the different parameters of the PAI PMUs. During
74 * initialization the machine dependent values are extracted and saved.
75 * However most of the values are static and do not change.
76 * There is one table entry per PAI PMU.
77 */
78 struct pai_pmu { /* Define PAI PMU characteristics */
79 const char *pmuname; /* Name of PMU */
80 const int facility_nr; /* Facility number to check for support */
81 unsigned int num_avail; /* # Counters defined by hardware */
82 unsigned int num_named; /* # Counters known by name */
83 unsigned long base; /* Counter set base number */
84 unsigned long kernel_offset; /* Offset to kernel part in counter page */
85 unsigned long area_size; /* Size of counter area */
86 const char * const *names; /* List of counter names */
87 struct pmu *pmu; /* Ptr to supporting PMU */
88 int (*init)(struct pai_pmu *p); /* PMU support init function */
89 void (*exit)(struct pai_pmu *p); /* PMU support exit function */
90 struct attribute_group *event_group; /* Ptr to attribute of events */
91 };
92
93 static struct pai_pmu pai_pmu[]; /* Forward declaration */
94
95 /* Free per CPU data when the last event is removed. */
pai_root_free(int idx)96 static void pai_root_free(int idx)
97 {
98 if (refcount_dec_and_test(&pai_root[idx].refcnt)) {
99 free_percpu(pai_root[idx].mapptr);
100 pai_root[idx].mapptr = NULL;
101 }
102 debug_sprintf_event(paidbg, 5, "%s root[%d].refcount %d\n", __func__,
103 idx, refcount_read(&pai_root[idx].refcnt));
104 }
105
106 /*
107 * On initialization of first event also allocate per CPU data dynamically.
108 * Start with an array of pointers, the array size is the maximum number of
109 * CPUs possible, which might be larger than the number of CPUs currently
110 * online.
111 */
pai_root_alloc(int idx)112 static int pai_root_alloc(int idx)
113 {
114 if (!refcount_inc_not_zero(&pai_root[idx].refcnt)) {
115 /* The memory is already zeroed. */
116 pai_root[idx].mapptr = alloc_percpu(struct pai_mapptr);
117 if (!pai_root[idx].mapptr)
118 return -ENOMEM;
119 refcount_set(&pai_root[idx].refcnt, 1);
120 }
121 return 0;
122 }
123
124 /* Release the PMU if event is the last perf event */
125 static DEFINE_MUTEX(pai_reserve_mutex);
126
127 /* Free all memory allocated for event counting/sampling setup */
pai_free(struct pai_mapptr * mp)128 static void pai_free(struct pai_mapptr *mp)
129 {
130 if (mp->mapptr->fullpage)
131 free_page((unsigned long)mp->mapptr->area);
132 else
133 kfree(mp->mapptr->area);
134 kfree(mp->mapptr->paiext_cb);
135 kvfree(mp->mapptr->save);
136 kfree(mp->mapptr);
137 mp->mapptr = NULL;
138 }
139
140 /* Adjust usage counters and remove allocated memory when all users are
141 * gone.
142 */
pai_event_destroy_cpu(struct perf_event * event,int cpu)143 static void pai_event_destroy_cpu(struct perf_event *event, int cpu)
144 {
145 int idx = PAI_PMU_IDX(event);
146 struct pai_mapptr *mp = per_cpu_ptr(pai_root[idx].mapptr, cpu);
147 struct pai_map *cpump = mp->mapptr;
148
149 mutex_lock(&pai_reserve_mutex);
150 debug_sprintf_event(paidbg, 5, "%s event %#llx idx %d cpu %d users %d "
151 "refcnt %u\n", __func__, event->attr.config, idx,
152 event->cpu, cpump->active_events,
153 refcount_read(&cpump->refcnt));
154 if (refcount_dec_and_test(&cpump->refcnt))
155 pai_free(mp);
156 pai_root_free(idx);
157 mutex_unlock(&pai_reserve_mutex);
158 }
159
pai_event_destroy(struct perf_event * event)160 static void pai_event_destroy(struct perf_event *event)
161 {
162 int cpu;
163
164 free_page(PAI_SAVE_AREA(event));
165 if (event->cpu == -1) {
166 struct cpumask *mask = PAI_CPU_MASK(event);
167
168 for_each_cpu(cpu, mask)
169 pai_event_destroy_cpu(event, cpu);
170 kfree(mask);
171 } else {
172 pai_event_destroy_cpu(event, event->cpu);
173 }
174 }
175
paicrypt_event_destroy(struct perf_event * event)176 static void paicrypt_event_destroy(struct perf_event *event)
177 {
178 static_branch_dec(&pai_key);
179 pai_event_destroy(event);
180 }
181
pai_getctr(unsigned long * page,int nr,unsigned long offset)182 static u64 pai_getctr(unsigned long *page, int nr, unsigned long offset)
183 {
184 if (offset)
185 nr += offset / sizeof(*page);
186 return page[nr];
187 }
188
pai_setctr(unsigned long * page,int nr,unsigned long offset,u64 v)189 static void pai_setctr(unsigned long *page, int nr, unsigned long offset, u64 v)
190 {
191 if (offset)
192 nr += offset / sizeof(*page);
193 page[nr] = v;
194 }
195
196 /* Read the counter values. Return value from location in CMP. For base
197 * event xxx_ALL sum up all events. Returns counter value.
198 */
pai_getdata(struct perf_event * event,bool kernel)199 static u64 pai_getdata(struct perf_event *event, bool kernel)
200 {
201 int idx = PAI_PMU_IDX(event);
202 struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr);
203 struct pai_pmu *pp = &pai_pmu[idx];
204 struct pai_map *cpump = mp->mapptr;
205 unsigned int i;
206 u64 sum = 0;
207
208 if (event->attr.config != pp->base) {
209 return pai_getctr(cpump->area,
210 event->attr.config - pp->base,
211 kernel ? pp->kernel_offset : 0);
212 }
213
214 for (i = 1; i <= pp->num_avail; i++) {
215 u64 val = pai_getctr(cpump->area, i,
216 kernel ? pp->kernel_offset : 0);
217
218 if (!val)
219 continue;
220 sum += val;
221 }
222 return sum;
223 }
224
paicrypt_getall(struct perf_event * event)225 static u64 paicrypt_getall(struct perf_event *event)
226 {
227 u64 sum = 0;
228
229 if (!event->attr.exclude_kernel)
230 sum += pai_getdata(event, true);
231 if (!event->attr.exclude_user)
232 sum += pai_getdata(event, false);
233
234 return sum;
235 }
236
237 /* Check concurrent access of counting and sampling for crypto events.
238 * This function is called in process context and it is save to block.
239 * When the event initialization functions fails, no other call back will
240 * be invoked.
241 *
242 * Allocate the memory for the event.
243 */
pai_alloc_cpu(struct perf_event * event,int cpu)244 static int pai_alloc_cpu(struct perf_event *event, int cpu)
245 {
246 int rc, idx = PAI_PMU_IDX(event);
247 struct pai_map *cpump = NULL;
248 bool need_paiext_cb = false;
249 struct pai_mapptr *mp;
250
251 mutex_lock(&pai_reserve_mutex);
252 /* Allocate root node */
253 rc = pai_root_alloc(idx);
254 if (rc)
255 goto unlock;
256
257 /* Allocate node for this event */
258 mp = per_cpu_ptr(pai_root[idx].mapptr, cpu);
259 cpump = mp->mapptr;
260 if (!cpump) { /* Paicrypt_map allocated? */
261 rc = -ENOMEM;
262 cpump = kzalloc_obj(*cpump);
263 if (!cpump)
264 goto undo;
265 /* Allocate memory for counter page and counter extraction.
266 * Only the first counting event has to allocate a page.
267 */
268 mp->mapptr = cpump;
269 if (idx == PAI_PMU_CRYPTO) {
270 cpump->area = (unsigned long *)get_zeroed_page(GFP_KERNEL);
271 /* free_page() can handle 0x0 address */
272 cpump->fullpage = true;
273 } else { /* PAI_PMU_EXT */
274 /*
275 * Allocate memory for counter area and counter extraction.
276 * These are
277 * - a 512 byte block and requires 512 byte boundary
278 * alignment.
279 * - a 1KB byte block and requires 1KB boundary
280 * alignment.
281 * Only the first counting event has to allocate the area.
282 *
283 * Note: This works with commit 59bb47985c1d by default.
284 * Backporting this to kernels without this commit might
285 * needs adjustment.
286 */
287 cpump->area = kzalloc(pai_pmu[idx].area_size, GFP_KERNEL);
288 cpump->paiext_cb = kzalloc(PAIE1_CB_SZ, GFP_KERNEL);
289 need_paiext_cb = true;
290 }
291 cpump->save = kvmalloc_objs(struct pai_userdata,
292 pai_pmu[idx].num_avail + 1);
293 if (!cpump->area || !cpump->save ||
294 (need_paiext_cb && !cpump->paiext_cb)) {
295 pai_free(mp);
296 goto undo;
297 }
298 INIT_LIST_HEAD(&cpump->syswide_list);
299 refcount_set(&cpump->refcnt, 1);
300 rc = 0;
301 } else {
302 refcount_inc(&cpump->refcnt);
303 }
304
305 undo:
306 if (rc) {
307 /* Error in allocation of event, decrement anchor. Since
308 * the event in not created, its destroy() function is never
309 * invoked. Adjust the reference counter for the anchor.
310 */
311 pai_root_free(idx);
312 }
313 unlock:
314 mutex_unlock(&pai_reserve_mutex);
315 /* If rc is non-zero, no increment of counter/sampler was done. */
316 return rc;
317 }
318
pai_alloc(struct perf_event * event)319 static int pai_alloc(struct perf_event *event)
320 {
321 struct cpumask *maskptr;
322 int cpu, rc = -ENOMEM;
323
324 maskptr = kzalloc_obj(*maskptr);
325 if (!maskptr)
326 goto out;
327
328 for_each_online_cpu(cpu) {
329 rc = pai_alloc_cpu(event, cpu);
330 if (rc) {
331 for_each_cpu(cpu, maskptr)
332 pai_event_destroy_cpu(event, cpu);
333 kfree(maskptr);
334 goto out;
335 }
336 cpumask_set_cpu(cpu, maskptr);
337 }
338
339 /*
340 * On error all cpumask are freed and all events have been destroyed.
341 * Save of which CPUs data structures have been allocated for.
342 * Release them in pai_event_destroy call back function
343 * for this event.
344 */
345 PAI_CPU_MASK(event) = maskptr;
346 rc = 0;
347 out:
348 return rc;
349 }
350
351 /* Validate event number and return error if event is not supported.
352 * On successful return, PAI_PMU_IDX(event) is set to the index of
353 * the supporting paing_support[] array element.
354 */
pai_event_valid(struct perf_event * event,int idx)355 static int pai_event_valid(struct perf_event *event, int idx)
356 {
357 struct perf_event_attr *a = &event->attr;
358 struct pai_pmu *pp = &pai_pmu[idx];
359
360 /* PAI crypto PMU registered as PERF_TYPE_RAW, check event type */
361 if (a->type != PERF_TYPE_RAW && event->pmu->type != a->type)
362 return -ENOENT;
363 /* Allow only CRYPTO_ALL/NNPA_ALL for sampling */
364 if (a->sample_period && a->config != pp->base)
365 return -EINVAL;
366 /* PAI crypto event must be in valid range, try others if not */
367 if (a->config < pp->base || a->config > pp->base + pp->num_avail)
368 return -ENOENT;
369 if (idx == PAI_PMU_EXT && a->exclude_user)
370 return -EINVAL;
371 PAI_PMU_IDX(event) = idx;
372 return 0;
373 }
374
375 /* Might be called on different CPU than the one the event is intended for. */
pai_event_init(struct perf_event * event,int idx)376 static int pai_event_init(struct perf_event *event, int idx)
377 {
378 struct perf_event_attr *a = &event->attr;
379 int rc;
380
381 /* PAI event must be valid and in supported range */
382 rc = pai_event_valid(event, idx);
383 if (rc)
384 goto out;
385 /* Get a page to store last counter values for sampling */
386 if (a->sample_period) {
387 PAI_SAVE_AREA(event) = get_zeroed_page(GFP_KERNEL);
388 if (!PAI_SAVE_AREA(event)) {
389 rc = -ENOMEM;
390 goto out;
391 }
392 }
393
394 if (event->cpu >= 0)
395 rc = pai_alloc_cpu(event, event->cpu);
396 else
397 rc = pai_alloc(event);
398 if (rc) {
399 free_page(PAI_SAVE_AREA(event));
400 goto out;
401 }
402
403 if (a->sample_period) {
404 a->sample_period = 1;
405 a->freq = 0;
406 /* Register for paicrypt_sched_task() to be called */
407 event->attach_state |= PERF_ATTACH_SCHED_CB;
408 /* Add raw data which contain the memory mapped counters */
409 a->sample_type |= PERF_SAMPLE_RAW;
410 /* Turn off inheritance */
411 a->inherit = 0;
412 }
413 out:
414 return rc;
415 }
416
paicrypt_event_init(struct perf_event * event)417 static int paicrypt_event_init(struct perf_event *event)
418 {
419 int rc = pai_event_init(event, PAI_PMU_CRYPTO);
420
421 if (!rc) {
422 event->destroy = paicrypt_event_destroy;
423 static_branch_inc(&pai_key);
424 }
425 return rc;
426 }
427
pai_read(struct perf_event * event,u64 (* fct)(struct perf_event * event))428 static void pai_read(struct perf_event *event,
429 u64 (*fct)(struct perf_event *event))
430 {
431 u64 prev, new, delta;
432
433 prev = local64_read(&event->hw.prev_count);
434 new = fct(event);
435 local64_set(&event->hw.prev_count, new);
436 delta = (prev <= new) ? new - prev : (-1ULL - prev) + new + 1;
437 local64_add(delta, &event->count);
438 }
439
paicrypt_read(struct perf_event * event)440 static void paicrypt_read(struct perf_event *event)
441 {
442 pai_read(event, paicrypt_getall);
443 }
444
pai_start(struct perf_event * event,int flags,u64 (* fct)(struct perf_event * event))445 static void pai_start(struct perf_event *event, int flags,
446 u64 (*fct)(struct perf_event *event))
447 {
448 int idx = PAI_PMU_IDX(event);
449 struct pai_pmu *pp = &pai_pmu[idx];
450 struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr);
451 struct pai_map *cpump = mp->mapptr;
452 u64 sum;
453
454 if (!event->attr.sample_period) { /* Counting */
455 sum = fct(event); /* Get current value */
456 local64_set(&event->hw.prev_count, sum);
457 } else { /* Sampling */
458 memcpy((void *)PAI_SAVE_AREA(event), cpump->area, pp->area_size);
459 /* Enable context switch callback for system-wide sampling */
460 if (!(event->attach_state & PERF_ATTACH_TASK)) {
461 list_add_tail(PAI_SWLIST(event), &cpump->syswide_list);
462 perf_sched_cb_inc(event->pmu);
463 } else {
464 cpump->event = event;
465 }
466 }
467 }
468
paicrypt_start(struct perf_event * event,int flags)469 static void paicrypt_start(struct perf_event *event, int flags)
470 {
471 pai_start(event, flags, paicrypt_getall);
472 }
473
pai_add(struct perf_event * event,int flags)474 static int pai_add(struct perf_event *event, int flags)
475 {
476 int idx = PAI_PMU_IDX(event);
477 struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr);
478 struct pai_map *cpump = mp->mapptr;
479 struct paiext_cb *pcb = cpump->paiext_cb;
480 unsigned long ccd;
481
482 if (++cpump->active_events == 1) {
483 if (!pcb) { /* PAI crypto */
484 ccd = virt_to_phys(cpump->area) | PAI_CRYPTO_KERNEL_OFFSET;
485 WRITE_ONCE(get_lowcore()->ccd, ccd);
486 local_ctl_set_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT);
487 } else { /* PAI extension 1 */
488 ccd = virt_to_phys(pcb);
489 WRITE_ONCE(get_lowcore()->aicd, ccd);
490 pcb->acc = virt_to_phys(cpump->area) | 0x1;
491 /* Enable CPU instruction lookup for PAIE1 control block */
492 local_ctl_set_bit(0, CR0_PAI_EXTENSION_BIT);
493 }
494 }
495 if (flags & PERF_EF_START)
496 pai_pmu[idx].pmu->start(event, PERF_EF_RELOAD);
497 event->hw.state = 0;
498 return 0;
499 }
500
paicrypt_add(struct perf_event * event,int flags)501 static int paicrypt_add(struct perf_event *event, int flags)
502 {
503 return pai_add(event, flags);
504 }
505
506 static void pai_have_sample(struct perf_event *, struct pai_map *);
pai_stop(struct perf_event * event,int flags)507 static void pai_stop(struct perf_event *event, int flags)
508 {
509 int idx = PAI_PMU_IDX(event);
510 struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr);
511 struct pai_map *cpump = mp->mapptr;
512
513 if (!event->attr.sample_period) { /* Counting */
514 pai_pmu[idx].pmu->read(event);
515 } else { /* Sampling */
516 if (!(event->attach_state & PERF_ATTACH_TASK)) {
517 perf_sched_cb_dec(event->pmu);
518 list_del(PAI_SWLIST(event));
519 } else {
520 pai_have_sample(event, cpump);
521 cpump->event = NULL;
522 }
523 }
524 event->hw.state = PERF_HES_STOPPED;
525 }
526
paicrypt_stop(struct perf_event * event,int flags)527 static void paicrypt_stop(struct perf_event *event, int flags)
528 {
529 pai_stop(event, flags);
530 }
531
pai_del(struct perf_event * event,int flags)532 static void pai_del(struct perf_event *event, int flags)
533 {
534 int idx = PAI_PMU_IDX(event);
535 struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr);
536 struct pai_map *cpump = mp->mapptr;
537 struct paiext_cb *pcb = cpump->paiext_cb;
538
539 pai_pmu[idx].pmu->stop(event, PERF_EF_UPDATE);
540 if (--cpump->active_events == 0) {
541 if (!pcb) { /* PAI crypto */
542 local_ctl_clear_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT);
543 WRITE_ONCE(get_lowcore()->ccd, 0);
544 } else { /* PAI extension 1 */
545 /* Disable CPU instruction lookup for PAIE1 control block */
546 local_ctl_clear_bit(0, CR0_PAI_EXTENSION_BIT);
547 pcb->acc = 0;
548 WRITE_ONCE(get_lowcore()->aicd, 0);
549 }
550 }
551 }
552
paicrypt_del(struct perf_event * event,int flags)553 static void paicrypt_del(struct perf_event *event, int flags)
554 {
555 pai_del(event, flags);
556 }
557
558 /* Create raw data and save it in buffer. Calculate the delta for each
559 * counter between this invocation and the last invocation.
560 * Returns number of bytes copied.
561 * After reading from PAI counter page, save the read value to the old
562 * page to calculate PAI counter deltas.
563 * Saves only entries with positive counter difference of the form
564 * 2 bytes: Number of counter
565 * 8 bytes: Value of counter
566 */
pai_copy(struct pai_userdata * userdata,unsigned long * page,struct pai_pmu * pp,unsigned long * page_old,bool exclude_user,bool exclude_kernel)567 static size_t pai_copy(struct pai_userdata *userdata, unsigned long *page,
568 struct pai_pmu *pp, unsigned long *page_old,
569 bool exclude_user, bool exclude_kernel)
570 {
571 int i, outidx = 0;
572
573 for (i = 1; i <= pp->num_avail; i++) {
574 u64 val = 0, val_old = 0, val_k = 0, val_old_k = 0;
575
576 if (!exclude_kernel) {
577 val_k = pai_getctr(page, i, pp->kernel_offset);
578 val_old_k = pai_getctr(page_old, i, pp->kernel_offset);
579 if (val_k != val_old_k)
580 pai_setctr(page_old, i, pp->kernel_offset, val_k);
581 }
582 if (!exclude_user) {
583 val = pai_getctr(page, i, 0);
584 val_old = pai_getctr(page_old, i, 0);
585 if (val != val_old)
586 pai_setctr(page_old, i, 0, val);
587 }
588 val += val_k;
589 val_old += val_old_k;
590 if (val >= val_old)
591 val -= val_old;
592 else
593 val = (~0ULL - val_old) + val + 1;
594 if (val) {
595 userdata[outidx].num = i;
596 userdata[outidx].value = val;
597 outidx++;
598 }
599 }
600 return outidx * sizeof(*userdata);
601 }
602
603 /* Write sample when one or more counters values are nonzero.
604 *
605 * Note: The function paicrypt_sched_task() and pai_push_sample() are not
606 * invoked after function paicrypt_del() has been called because of function
607 * perf_sched_cb_dec(). Both functions are only
608 * called when sampling is active. Function perf_sched_cb_inc()
609 * has been invoked to install function paicrypt_sched_task() as call back
610 * to run at context switch time.
611 *
612 * This causes function perf_event_context_sched_out() and
613 * perf_event_context_sched_in() to check whether the PMU has installed an
614 * sched_task() callback. That callback is not active after paicrypt_del()
615 * returns and has deleted the event on that CPU.
616 */
pai_push_sample(size_t rawsize,struct pai_map * cpump,struct perf_event * event)617 static int pai_push_sample(size_t rawsize, struct pai_map *cpump,
618 struct perf_event *event)
619 {
620 struct perf_sample_data data;
621 struct perf_raw_record raw;
622 struct pt_regs regs;
623 int overflow;
624
625 /* Setup perf sample */
626 memset(®s, 0, sizeof(regs));
627 memset(&raw, 0, sizeof(raw));
628 memset(&data, 0, sizeof(data));
629 perf_sample_data_init(&data, 0, event->hw.last_period);
630 if (event->attr.sample_type & PERF_SAMPLE_TID) {
631 data.tid_entry.pid = task_tgid_nr(current);
632 data.tid_entry.tid = task_pid_nr(current);
633 }
634 if (event->attr.sample_type & PERF_SAMPLE_TIME)
635 data.time = event->clock();
636 if (event->attr.sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER))
637 data.id = event->id;
638 if (event->attr.sample_type & PERF_SAMPLE_CPU) {
639 data.cpu_entry.cpu = smp_processor_id();
640 data.cpu_entry.reserved = 0;
641 }
642 if (event->attr.sample_type & PERF_SAMPLE_RAW) {
643 raw.frag.size = rawsize;
644 raw.frag.data = cpump->save;
645 perf_sample_save_raw_data(&data, event, &raw);
646 }
647
648 overflow = perf_event_overflow(event, &data, ®s);
649 perf_event_update_userpage(event);
650 return overflow;
651 }
652
653 /* Check if there is data to be saved on schedule out of a task. */
pai_have_sample(struct perf_event * event,struct pai_map * cpump)654 static void pai_have_sample(struct perf_event *event, struct pai_map *cpump)
655 {
656 struct pai_pmu *pp;
657 size_t rawsize;
658
659 if (!event) /* No event active */
660 return;
661 pp = &pai_pmu[PAI_PMU_IDX(event)];
662 rawsize = pai_copy(cpump->save, cpump->area, pp,
663 (unsigned long *)PAI_SAVE_AREA(event),
664 event->attr.exclude_user,
665 !pp->kernel_offset ? true : event->attr.exclude_kernel);
666 if (rawsize) /* No incremented counters */
667 pai_push_sample(rawsize, cpump, event);
668 }
669
670 /* Check if there is data to be saved on schedule out of a task. */
pai_have_samples(int idx)671 static void pai_have_samples(int idx)
672 {
673 struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr);
674 struct pai_map *cpump = mp->mapptr;
675 struct perf_event *event;
676
677 list_for_each_entry(event, &cpump->syswide_list, hw.tp_list)
678 pai_have_sample(event, cpump);
679 }
680
681 /* Called on schedule-in and schedule-out. No access to event structure,
682 * but for sampling only event CRYPTO_ALL is allowed.
683 */
paicrypt_sched_task(struct perf_event_pmu_context * pmu_ctx,struct task_struct * task,bool sched_in)684 static void paicrypt_sched_task(struct perf_event_pmu_context *pmu_ctx,
685 struct task_struct *task, bool sched_in)
686 {
687 /* We started with a clean page on event installation. So read out
688 * results on schedule_out and if page was dirty, save old values.
689 */
690 if (!sched_in)
691 pai_have_samples(PAI_PMU_CRYPTO);
692 }
693
694 /* ============================= paiext ====================================*/
695
paiext_event_destroy(struct perf_event * event)696 static void paiext_event_destroy(struct perf_event *event)
697 {
698 pai_event_destroy(event);
699 }
700
701 /* Might be called on different CPU than the one the event is intended for. */
paiext_event_init(struct perf_event * event)702 static int paiext_event_init(struct perf_event *event)
703 {
704 int rc = pai_event_init(event, PAI_PMU_EXT);
705
706 if (!rc) {
707 event->attr.exclude_kernel = true; /* No kernel space part */
708 event->destroy = paiext_event_destroy;
709 /* Offset of NNPA in paiext_cb */
710 event->hw.config_base = offsetof(struct paiext_cb, acc);
711 }
712 return rc;
713 }
714
paiext_getall(struct perf_event * event)715 static u64 paiext_getall(struct perf_event *event)
716 {
717 return pai_getdata(event, false);
718 }
719
paiext_read(struct perf_event * event)720 static void paiext_read(struct perf_event *event)
721 {
722 pai_read(event, paiext_getall);
723 }
724
paiext_start(struct perf_event * event,int flags)725 static void paiext_start(struct perf_event *event, int flags)
726 {
727 pai_start(event, flags, paiext_getall);
728 }
729
paiext_add(struct perf_event * event,int flags)730 static int paiext_add(struct perf_event *event, int flags)
731 {
732 return pai_add(event, flags);
733 }
734
paiext_stop(struct perf_event * event,int flags)735 static void paiext_stop(struct perf_event *event, int flags)
736 {
737 pai_stop(event, flags);
738 }
739
paiext_del(struct perf_event * event,int flags)740 static void paiext_del(struct perf_event *event, int flags)
741 {
742 pai_del(event, flags);
743 }
744
745 /* Called on schedule-in and schedule-out. No access to event structure,
746 * but for sampling only event NNPA_ALL is allowed.
747 */
paiext_sched_task(struct perf_event_pmu_context * pmu_ctx,struct task_struct * task,bool sched_in)748 static void paiext_sched_task(struct perf_event_pmu_context *pmu_ctx,
749 struct task_struct *task, bool sched_in)
750 {
751 /* We started with a clean page on event installation. So read out
752 * results on schedule_out and if page was dirty, save old values.
753 */
754 if (!sched_in)
755 pai_have_samples(PAI_PMU_EXT);
756 }
757
758 /* Attribute definitions for paicrypt interface. As with other CPU
759 * Measurement Facilities, there is one attribute per mapped counter.
760 * The number of mapped counters may vary per machine generation. Use
761 * the QUERY PROCESSOR ACTIVITY COUNTER INFORMATION (QPACI) instruction
762 * to determine the number of mapped counters. The instructions returns
763 * a positive number, which is the highest number of supported counters.
764 * All counters less than this number are also supported, there are no
765 * holes. A returned number of zero means no support for mapped counters.
766 *
767 * The identification of the counter is a unique number. The chosen range
768 * is 0x1000 + offset in mapped kernel page.
769 * All CPU Measurement Facility counters identifiers must be unique and
770 * the numbers from 0 to 496 are already used for the CPU Measurement
771 * Counter facility. Numbers 0xb0000, 0xbc000 and 0xbd000 are already
772 * used for the CPU Measurement Sampling facility.
773 */
774 PMU_FORMAT_ATTR(event, "config:0-63");
775
776 static struct attribute *paicrypt_format_attr[] = {
777 &format_attr_event.attr,
778 NULL,
779 };
780
781 static struct attribute_group paicrypt_events_group = {
782 .name = "events",
783 .attrs = NULL /* Filled in attr_event_init() */
784 };
785
786 static struct attribute_group paicrypt_format_group = {
787 .name = "format",
788 .attrs = paicrypt_format_attr,
789 };
790
791 static const struct attribute_group *paicrypt_attr_groups[] = {
792 &paicrypt_events_group,
793 &paicrypt_format_group,
794 NULL,
795 };
796
797 /* Performance monitoring unit for mapped counters */
798 static struct pmu paicrypt = {
799 .task_ctx_nr = perf_hw_context,
800 .event_init = paicrypt_event_init,
801 .add = paicrypt_add,
802 .del = paicrypt_del,
803 .start = paicrypt_start,
804 .stop = paicrypt_stop,
805 .read = paicrypt_read,
806 .sched_task = paicrypt_sched_task,
807 .attr_groups = paicrypt_attr_groups
808 };
809
810 /* List of symbolic PAI counter names. */
811 static const char * const paicrypt_ctrnames[] = {
812 [0] = "CRYPTO_ALL",
813 [1] = "KM_DEA",
814 [2] = "KM_TDEA_128",
815 [3] = "KM_TDEA_192",
816 [4] = "KM_ENCRYPTED_DEA",
817 [5] = "KM_ENCRYPTED_TDEA_128",
818 [6] = "KM_ENCRYPTED_TDEA_192",
819 [7] = "KM_AES_128",
820 [8] = "KM_AES_192",
821 [9] = "KM_AES_256",
822 [10] = "KM_ENCRYPTED_AES_128",
823 [11] = "KM_ENCRYPTED_AES_192",
824 [12] = "KM_ENCRYPTED_AES_256",
825 [13] = "KM_XTS_AES_128",
826 [14] = "KM_XTS_AES_256",
827 [15] = "KM_XTS_ENCRYPTED_AES_128",
828 [16] = "KM_XTS_ENCRYPTED_AES_256",
829 [17] = "KMC_DEA",
830 [18] = "KMC_TDEA_128",
831 [19] = "KMC_TDEA_192",
832 [20] = "KMC_ENCRYPTED_DEA",
833 [21] = "KMC_ENCRYPTED_TDEA_128",
834 [22] = "KMC_ENCRYPTED_TDEA_192",
835 [23] = "KMC_AES_128",
836 [24] = "KMC_AES_192",
837 [25] = "KMC_AES_256",
838 [26] = "KMC_ENCRYPTED_AES_128",
839 [27] = "KMC_ENCRYPTED_AES_192",
840 [28] = "KMC_ENCRYPTED_AES_256",
841 [29] = "KMC_PRNG",
842 [30] = "KMA_GCM_AES_128",
843 [31] = "KMA_GCM_AES_192",
844 [32] = "KMA_GCM_AES_256",
845 [33] = "KMA_GCM_ENCRYPTED_AES_128",
846 [34] = "KMA_GCM_ENCRYPTED_AES_192",
847 [35] = "KMA_GCM_ENCRYPTED_AES_256",
848 [36] = "KMF_DEA",
849 [37] = "KMF_TDEA_128",
850 [38] = "KMF_TDEA_192",
851 [39] = "KMF_ENCRYPTED_DEA",
852 [40] = "KMF_ENCRYPTED_TDEA_128",
853 [41] = "KMF_ENCRYPTED_TDEA_192",
854 [42] = "KMF_AES_128",
855 [43] = "KMF_AES_192",
856 [44] = "KMF_AES_256",
857 [45] = "KMF_ENCRYPTED_AES_128",
858 [46] = "KMF_ENCRYPTED_AES_192",
859 [47] = "KMF_ENCRYPTED_AES_256",
860 [48] = "KMCTR_DEA",
861 [49] = "KMCTR_TDEA_128",
862 [50] = "KMCTR_TDEA_192",
863 [51] = "KMCTR_ENCRYPTED_DEA",
864 [52] = "KMCTR_ENCRYPTED_TDEA_128",
865 [53] = "KMCTR_ENCRYPTED_TDEA_192",
866 [54] = "KMCTR_AES_128",
867 [55] = "KMCTR_AES_192",
868 [56] = "KMCTR_AES_256",
869 [57] = "KMCTR_ENCRYPTED_AES_128",
870 [58] = "KMCTR_ENCRYPTED_AES_192",
871 [59] = "KMCTR_ENCRYPTED_AES_256",
872 [60] = "KMO_DEA",
873 [61] = "KMO_TDEA_128",
874 [62] = "KMO_TDEA_192",
875 [63] = "KMO_ENCRYPTED_DEA",
876 [64] = "KMO_ENCRYPTED_TDEA_128",
877 [65] = "KMO_ENCRYPTED_TDEA_192",
878 [66] = "KMO_AES_128",
879 [67] = "KMO_AES_192",
880 [68] = "KMO_AES_256",
881 [69] = "KMO_ENCRYPTED_AES_128",
882 [70] = "KMO_ENCRYPTED_AES_192",
883 [71] = "KMO_ENCRYPTED_AES_256",
884 [72] = "KIMD_SHA_1",
885 [73] = "KIMD_SHA_256",
886 [74] = "KIMD_SHA_512",
887 [75] = "KIMD_SHA3_224",
888 [76] = "KIMD_SHA3_256",
889 [77] = "KIMD_SHA3_384",
890 [78] = "KIMD_SHA3_512",
891 [79] = "KIMD_SHAKE_128",
892 [80] = "KIMD_SHAKE_256",
893 [81] = "KIMD_GHASH",
894 [82] = "KLMD_SHA_1",
895 [83] = "KLMD_SHA_256",
896 [84] = "KLMD_SHA_512",
897 [85] = "KLMD_SHA3_224",
898 [86] = "KLMD_SHA3_256",
899 [87] = "KLMD_SHA3_384",
900 [88] = "KLMD_SHA3_512",
901 [89] = "KLMD_SHAKE_128",
902 [90] = "KLMD_SHAKE_256",
903 [91] = "KMAC_DEA",
904 [92] = "KMAC_TDEA_128",
905 [93] = "KMAC_TDEA_192",
906 [94] = "KMAC_ENCRYPTED_DEA",
907 [95] = "KMAC_ENCRYPTED_TDEA_128",
908 [96] = "KMAC_ENCRYPTED_TDEA_192",
909 [97] = "KMAC_AES_128",
910 [98] = "KMAC_AES_192",
911 [99] = "KMAC_AES_256",
912 [100] = "KMAC_ENCRYPTED_AES_128",
913 [101] = "KMAC_ENCRYPTED_AES_192",
914 [102] = "KMAC_ENCRYPTED_AES_256",
915 [103] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_DEA",
916 [104] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_TDEA_128",
917 [105] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_TDEA_192",
918 [106] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_DEA",
919 [107] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_TDEA_128",
920 [108] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_TDEA_192",
921 [109] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_128",
922 [110] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_192",
923 [111] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_256",
924 [112] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_128",
925 [113] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_192",
926 [114] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_256",
927 [115] = "PCC_COMPUTE_XTS_PARAMETER_USING_AES_128",
928 [116] = "PCC_COMPUTE_XTS_PARAMETER_USING_AES_256",
929 [117] = "PCC_COMPUTE_XTS_PARAMETER_USING_ENCRYPTED_AES_128",
930 [118] = "PCC_COMPUTE_XTS_PARAMETER_USING_ENCRYPTED_AES_256",
931 [119] = "PCC_SCALAR_MULTIPLY_P256",
932 [120] = "PCC_SCALAR_MULTIPLY_P384",
933 [121] = "PCC_SCALAR_MULTIPLY_P521",
934 [122] = "PCC_SCALAR_MULTIPLY_ED25519",
935 [123] = "PCC_SCALAR_MULTIPLY_ED448",
936 [124] = "PCC_SCALAR_MULTIPLY_X25519",
937 [125] = "PCC_SCALAR_MULTIPLY_X448",
938 [126] = "PRNO_SHA_512_DRNG",
939 [127] = "PRNO_TRNG_QUERY_RAW_TO_CONDITIONED_RATIO",
940 [128] = "PRNO_TRNG",
941 [129] = "KDSA_ECDSA_VERIFY_P256",
942 [130] = "KDSA_ECDSA_VERIFY_P384",
943 [131] = "KDSA_ECDSA_VERIFY_P521",
944 [132] = "KDSA_ECDSA_SIGN_P256",
945 [133] = "KDSA_ECDSA_SIGN_P384",
946 [134] = "KDSA_ECDSA_SIGN_P521",
947 [135] = "KDSA_ENCRYPTED_ECDSA_SIGN_P256",
948 [136] = "KDSA_ENCRYPTED_ECDSA_SIGN_P384",
949 [137] = "KDSA_ENCRYPTED_ECDSA_SIGN_P521",
950 [138] = "KDSA_EDDSA_VERIFY_ED25519",
951 [139] = "KDSA_EDDSA_VERIFY_ED448",
952 [140] = "KDSA_EDDSA_SIGN_ED25519",
953 [141] = "KDSA_EDDSA_SIGN_ED448",
954 [142] = "KDSA_ENCRYPTED_EDDSA_SIGN_ED25519",
955 [143] = "KDSA_ENCRYPTED_EDDSA_SIGN_ED448",
956 [144] = "PCKMO_ENCRYPT_DEA_KEY",
957 [145] = "PCKMO_ENCRYPT_TDEA_128_KEY",
958 [146] = "PCKMO_ENCRYPT_TDEA_192_KEY",
959 [147] = "PCKMO_ENCRYPT_AES_128_KEY",
960 [148] = "PCKMO_ENCRYPT_AES_192_KEY",
961 [149] = "PCKMO_ENCRYPT_AES_256_KEY",
962 [150] = "PCKMO_ENCRYPT_ECC_P256_KEY",
963 [151] = "PCKMO_ENCRYPT_ECC_P384_KEY",
964 [152] = "PCKMO_ENCRYPT_ECC_P521_KEY",
965 [153] = "PCKMO_ENCRYPT_ECC_ED25519_KEY",
966 [154] = "PCKMO_ENCRYPT_ECC_ED448_KEY",
967 [155] = "IBM_RESERVED_155",
968 [156] = "IBM_RESERVED_156",
969 [157] = "KM_FULL_XTS_AES_128",
970 [158] = "KM_FULL_XTS_AES_256",
971 [159] = "KM_FULL_XTS_ENCRYPTED_AES_128",
972 [160] = "KM_FULL_XTS_ENCRYPTED_AES_256",
973 [161] = "KMAC_HMAC_SHA_224",
974 [162] = "KMAC_HMAC_SHA_256",
975 [163] = "KMAC_HMAC_SHA_384",
976 [164] = "KMAC_HMAC_SHA_512",
977 [165] = "KMAC_HMAC_ENCRYPTED_SHA_224",
978 [166] = "KMAC_HMAC_ENCRYPTED_SHA_256",
979 [167] = "KMAC_HMAC_ENCRYPTED_SHA_384",
980 [168] = "KMAC_HMAC_ENCRYPTED_SHA_512",
981 [169] = "PCKMO_ENCRYPT_HMAC_512_KEY",
982 [170] = "PCKMO_ENCRYPT_HMAC_1024_KEY",
983 [171] = "PCKMO_ENCRYPT_AES_XTS_128",
984 [172] = "PCKMO_ENCRYPT_AES_XTS_256",
985 };
986
987 static struct attribute *paiext_format_attr[] = {
988 &format_attr_event.attr,
989 NULL,
990 };
991
992 static struct attribute_group paiext_events_group = {
993 .name = "events",
994 .attrs = NULL, /* Filled in attr_event_init() */
995 };
996
997 static struct attribute_group paiext_format_group = {
998 .name = "format",
999 .attrs = paiext_format_attr,
1000 };
1001
1002 static const struct attribute_group *paiext_attr_groups[] = {
1003 &paiext_events_group,
1004 &paiext_format_group,
1005 NULL,
1006 };
1007
1008 /* Performance monitoring unit for mapped counters */
1009 static struct pmu paiext = {
1010 .task_ctx_nr = perf_hw_context,
1011 .event_init = paiext_event_init,
1012 .add = paiext_add,
1013 .del = paiext_del,
1014 .start = paiext_start,
1015 .stop = paiext_stop,
1016 .read = paiext_read,
1017 .sched_task = paiext_sched_task,
1018 .attr_groups = paiext_attr_groups,
1019 };
1020
1021 /* List of symbolic PAI extension 1 NNPA counter names. */
1022 static const char * const paiext_ctrnames[] = {
1023 [0] = "NNPA_ALL",
1024 [1] = "NNPA_ADD",
1025 [2] = "NNPA_SUB",
1026 [3] = "NNPA_MUL",
1027 [4] = "NNPA_DIV",
1028 [5] = "NNPA_MIN",
1029 [6] = "NNPA_MAX",
1030 [7] = "NNPA_LOG",
1031 [8] = "NNPA_EXP",
1032 [9] = "NNPA_IBM_RESERVED_9",
1033 [10] = "NNPA_RELU",
1034 [11] = "NNPA_TANH",
1035 [12] = "NNPA_SIGMOID",
1036 [13] = "NNPA_SOFTMAX",
1037 [14] = "NNPA_BATCHNORM",
1038 [15] = "NNPA_MAXPOOL2D",
1039 [16] = "NNPA_AVGPOOL2D",
1040 [17] = "NNPA_LSTMACT",
1041 [18] = "NNPA_GRUACT",
1042 [19] = "NNPA_CONVOLUTION",
1043 [20] = "NNPA_MATMUL_OP",
1044 [21] = "NNPA_MATMUL_OP_BCAST23",
1045 [22] = "NNPA_SMALLBATCH",
1046 [23] = "NNPA_LARGEDIM",
1047 [24] = "NNPA_SMALLTENSOR",
1048 [25] = "NNPA_1MFRAME",
1049 [26] = "NNPA_2GFRAME",
1050 [27] = "NNPA_ACCESSEXCEPT",
1051 [28] = "NNPA_TRANSFORM",
1052 [29] = "NNPA_GELU",
1053 [30] = "NNPA_MOMENTS",
1054 [31] = "NNPA_LAYERNORM",
1055 [32] = "NNPA_MATMUL_OP_BCAST1",
1056 [33] = "NNPA_SQRT",
1057 [34] = "NNPA_INVSQRT",
1058 [35] = "NNPA_NORM",
1059 [36] = "NNPA_REDUCE",
1060 };
1061
attr_event_free(struct attribute ** attrs)1062 static void __init attr_event_free(struct attribute **attrs)
1063 {
1064 struct perf_pmu_events_attr *pa;
1065 unsigned int i;
1066
1067 for (i = 0; attrs[i]; i++) {
1068 struct device_attribute *dap;
1069
1070 dap = container_of(attrs[i], struct device_attribute, attr);
1071 pa = container_of(dap, struct perf_pmu_events_attr, attr);
1072 kfree(pa);
1073 }
1074 kfree(attrs);
1075 }
1076
attr_event_init_one(int num,unsigned long base,const char * name)1077 static struct attribute * __init attr_event_init_one(int num,
1078 unsigned long base,
1079 const char *name)
1080 {
1081 struct perf_pmu_events_attr *pa;
1082
1083 pa = kzalloc_obj(*pa);
1084 if (!pa)
1085 return NULL;
1086
1087 sysfs_attr_init(&pa->attr.attr);
1088 pa->id = base + num;
1089 pa->attr.attr.name = name;
1090 pa->attr.attr.mode = 0444;
1091 pa->attr.show = cpumf_events_sysfs_show;
1092 pa->attr.store = NULL;
1093 return &pa->attr.attr;
1094 }
1095
attr_event_init(struct pai_pmu * p)1096 static struct attribute ** __init attr_event_init(struct pai_pmu *p)
1097 {
1098 unsigned int min_attr = min_t(unsigned int, p->num_named, p->num_avail);
1099 struct attribute **attrs;
1100 unsigned int i;
1101
1102 attrs = kmalloc_objs(*attrs, min_attr + 1, GFP_KERNEL | __GFP_ZERO);
1103 if (!attrs)
1104 goto out;
1105 for (i = 0; i < min_attr; i++) {
1106 attrs[i] = attr_event_init_one(i, p->base, p->names[i]);
1107 if (!attrs[i]) {
1108 attr_event_free(attrs);
1109 attrs = NULL;
1110 goto out;
1111 }
1112 }
1113 attrs[i] = NULL;
1114 out:
1115 return attrs;
1116 }
1117
pai_pmu_exit(struct pai_pmu * p)1118 static void __init pai_pmu_exit(struct pai_pmu *p)
1119 {
1120 attr_event_free(p->event_group->attrs);
1121 p->event_group->attrs = NULL;
1122 }
1123
1124 /* Add a PMU. Install its events and register the PMU device driver
1125 * call back functions.
1126 */
pai_pmu_init(struct pai_pmu * p)1127 static int __init pai_pmu_init(struct pai_pmu *p)
1128 {
1129 int rc = -ENOMEM;
1130
1131
1132 /* Export known PAI events */
1133 p->event_group->attrs = attr_event_init(p);
1134 if (!p->event_group->attrs) {
1135 pr_err("Creation of PMU %s /sysfs failed\n", p->pmuname);
1136 goto out;
1137 }
1138
1139 rc = perf_pmu_register(p->pmu, p->pmuname, -1);
1140 if (rc) {
1141 pai_pmu_exit(p);
1142 pr_err("Registering PMU %s failed with rc=%i\n", p->pmuname,
1143 rc);
1144 }
1145 out:
1146 return rc;
1147 }
1148
1149 /* PAI PMU characteristics table */
1150 static struct pai_pmu pai_pmu[] __refdata = {
1151 [PAI_PMU_CRYPTO] = {
1152 .pmuname = "pai_crypto",
1153 .facility_nr = 196,
1154 .num_named = ARRAY_SIZE(paicrypt_ctrnames),
1155 .names = paicrypt_ctrnames,
1156 .base = PAI_CRYPTO_BASE,
1157 .kernel_offset = PAI_CRYPTO_KERNEL_OFFSET,
1158 .area_size = PAGE_SIZE,
1159 .init = pai_pmu_init,
1160 .exit = pai_pmu_exit,
1161 .pmu = &paicrypt,
1162 .event_group = &paicrypt_events_group
1163 },
1164 [PAI_PMU_EXT] = {
1165 .pmuname = "pai_ext",
1166 .facility_nr = 197,
1167 .num_named = ARRAY_SIZE(paiext_ctrnames),
1168 .names = paiext_ctrnames,
1169 .base = PAI_NNPA_BASE,
1170 .kernel_offset = 0,
1171 .area_size = PAIE1_CTRBLOCK_SZ,
1172 .init = pai_pmu_init,
1173 .exit = pai_pmu_exit,
1174 .pmu = &paiext,
1175 .event_group = &paiext_events_group
1176 }
1177 };
1178
1179 /*
1180 * Check if the PMU (via facility) is supported by machine. Try all of the
1181 * supported PAI PMUs.
1182 * Return number of successfully installed PMUs.
1183 */
paipmu_setup(void)1184 static int __init paipmu_setup(void)
1185 {
1186 struct qpaci_info_block ib;
1187 int install_ok = 0, rc;
1188 struct pai_pmu *p;
1189 size_t i;
1190
1191 for (i = 0; i < ARRAY_SIZE(pai_pmu); ++i) {
1192 p = &pai_pmu[i];
1193
1194 if (!test_facility(p->facility_nr))
1195 continue;
1196
1197 qpaci(&ib);
1198 switch (i) {
1199 case PAI_PMU_CRYPTO:
1200 p->num_avail = ib.num_cc;
1201 if (p->num_avail >= PAI_CRYPTO_MAXCTR) {
1202 pr_err("Too many PMU %s counters %d\n",
1203 p->pmuname, p->num_avail);
1204 continue;
1205 }
1206 break;
1207 case PAI_PMU_EXT:
1208 p->num_avail = ib.num_nnpa;
1209 break;
1210 }
1211 p->num_avail += 1; /* Add xxx_ALL event */
1212 if (p->init) {
1213 rc = p->init(p);
1214 if (!rc)
1215 ++install_ok;
1216 }
1217 }
1218 return install_ok;
1219 }
1220
pai_init(void)1221 static int __init pai_init(void)
1222 {
1223 /* Setup s390dbf facility */
1224 paidbg = debug_register("pai", 32, 256, 128);
1225 if (!paidbg) {
1226 pr_err("Registration of s390dbf pai failed\n");
1227 return -ENOMEM;
1228 }
1229 debug_register_view(paidbg, &debug_sprintf_view);
1230
1231 if (!paipmu_setup()) {
1232 /* No PMU registration, no need for debug buffer */
1233 debug_unregister_view(paidbg, &debug_sprintf_view);
1234 debug_unregister(paidbg);
1235 return -ENODEV;
1236 }
1237 return 0;
1238 }
1239
1240 device_initcall(pai_init);
1241