1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Performance event support - Processor Activity Instrumentation Facility
4 *
5 * Copyright IBM Corp. 2026
6 * Author(s): Thomas Richter <tmricht@linux.ibm.com>
7 */
8 #define pr_fmt(fmt) "pai: " fmt
9
10 #include <linux/kernel.h>
11 #include <linux/kernel_stat.h>
12 #include <linux/percpu.h>
13 #include <linux/notifier.h>
14 #include <linux/init.h>
15 #include <linux/io.h>
16 #include <linux/perf_event.h>
17 #include <asm/ctlreg.h>
18 #include <asm/pai.h>
19 #include <asm/debug.h>
20
21 static debug_info_t *paidbg;
22
23 DEFINE_STATIC_KEY_FALSE(pai_key);
24
25 enum {
26 PAI_PMU_CRYPTO, /* Index of PMU pai_crypto */
27 PAI_PMU_EXT, /* Index of PMU pai_ext */
28 PAI_PMU_MAX /* # of PAI PMUs */
29 };
30
31 enum {
32 PAIE1_CB_SZ = 0x200, /* Size of PAIE1 control block */
33 PAIE1_CTRBLOCK_SZ = 0x400 /* Size of PAIE1 counter blocks */
34 };
35
36 struct pai_userdata {
37 u16 num;
38 u64 value;
39 } __packed;
40
41 /* Create the PAI extension 1 control block area.
42 * The PAI extension control block 1 is pointed to by lowcore
43 * address 0x1508 for each CPU. This control block is 512 bytes in size
44 * and requires a 512 byte boundary alignment.
45 */
46 struct paiext_cb { /* PAI extension 1 control block */
47 u64 header; /* Not used */
48 u64 reserved1;
49 u64 acc; /* Addr to analytics counter control block */
50 u8 reserved2[PAIE1_CTRBLOCK_SZ - 3 * sizeof(u64)];
51 } __packed;
52
53 struct pai_map {
54 unsigned long *area; /* Area for CPU to store counters */
55 struct pai_userdata *save; /* Page to store no-zero counters */
56 unsigned int active_events; /* # of PAI crypto users */
57 refcount_t refcnt; /* Reference count mapped buffers */
58 struct perf_event *event; /* Perf event for sampling */
59 struct list_head syswide_list; /* List system-wide sampling events */
60 struct paiext_cb *paiext_cb; /* PAI extension control block area */
61 bool fullpage; /* True: counter area is a full page */
62 };
63
64 struct pai_mapptr {
65 struct pai_map *mapptr;
66 };
67
68 static struct pai_root { /* Anchor to per CPU data */
69 refcount_t refcnt; /* Overall active events */
70 atomic_t tskctx; /* Overall per-task events */
71 struct pai_mapptr __percpu *mapptr;
72 } pai_root[PAI_PMU_MAX];
73
74 /* This table defines the different parameters of the PAI PMUs. During
75 * initialization the machine dependent values are extracted and saved.
76 * However most of the values are static and do not change.
77 * There is one table entry per PAI PMU.
78 */
79 struct pai_pmu { /* Define PAI PMU characteristics */
80 const char *pmuname; /* Name of PMU */
81 const int facility_nr; /* Facility number to check for support */
82 unsigned int num_avail; /* # Counters defined by hardware */
83 unsigned int num_named; /* # Counters known by name */
84 unsigned long base; /* Counter set base number */
85 unsigned long kernel_offset; /* Offset to kernel part in counter page */
86 unsigned long area_size; /* Size of counter area */
87 const char * const *names; /* List of counter names */
88 struct pmu *pmu; /* Ptr to supporting PMU */
89 int (*init)(struct pai_pmu *p); /* PMU support init function */
90 void (*exit)(struct pai_pmu *p); /* PMU support exit function */
91 struct attribute_group *event_group; /* Ptr to attribute of events */
92 };
93
94 static struct pai_pmu pai_pmu[]; /* Forward declaration */
95
96 /* Free per CPU data when the last event is removed. */
pai_root_free(int idx,int tasks)97 static void pai_root_free(int idx, int tasks)
98 {
99 if (refcount_sub_and_test(tasks, &pai_root[idx].refcnt)) {
100 free_percpu(pai_root[idx].mapptr);
101 pai_root[idx].mapptr = NULL;
102 }
103 debug_sprintf_event(paidbg, 5, "%s root[%d].refcount %d tskctx %d\n",
104 __func__, idx, refcount_read(&pai_root[idx].refcnt),
105 atomic_read(&pai_root[idx].tskctx));
106 }
107
108 /*
109 * On initialization of first event also allocate per CPU data dynamically.
110 * Start with an array of pointers, the array size is the maximum number of
111 * CPUs possible, which might be larger than the number of CPUs currently
112 * online.
113 */
pai_root_alloc(int idx)114 static int pai_root_alloc(int idx)
115 {
116 if (!refcount_inc_not_zero(&pai_root[idx].refcnt)) {
117 /* The memory is already zeroed. */
118 pai_root[idx].mapptr = alloc_percpu(struct pai_mapptr);
119 if (!pai_root[idx].mapptr)
120 return -ENOMEM;
121 refcount_set(&pai_root[idx].refcnt, 1);
122 }
123 return 0;
124 }
125
126 /* Release the PMU if event is the last perf event */
127 static DEFINE_MUTEX(pai_reserve_mutex);
128
129 /* Free all memory allocated for event counting/sampling setup */
pai_free(struct pai_mapptr * mp)130 static void pai_free(struct pai_mapptr *mp)
131 {
132 if (mp->mapptr->fullpage)
133 free_page((unsigned long)mp->mapptr->area);
134 else
135 kfree(mp->mapptr->area);
136 kfree(mp->mapptr->paiext_cb);
137 kvfree(mp->mapptr->save);
138 kfree(mp->mapptr);
139 mp->mapptr = NULL;
140 }
141
142 /* Called under mutex_lock */
pai_event_destroy_cpu(int idx,int cpu,bool hotplug)143 static void pai_event_destroy_cpu(int idx, int cpu, bool hotplug)
144 {
145 struct pai_mapptr *mp;
146 struct pai_map *cpump;
147 int tasks = 1;
148
149 /* Check reference count and return when all gone.
150 * 1. An event is installed on online CPU X.
151 * 2. CPU x is offlined and the per-CPU data is removed.
152 * 3. Event is destroyed via close system call.
153 */
154 if (!refcount_read(&pai_root[idx].refcnt))
155 return; /* No events at all */
156 mp = per_cpu_ptr(pai_root[idx].mapptr, cpu);
157 if (!mp || !mp->mapptr) /* No events on that CPU */
158 return;
159
160 /* When hotplug is true, invocation is from CPU hotplug callback.
161 * Delete per-CPU resource and adjust refcnt when per-task events
162 * are currently active. This can be more than one.
163 * In this case adjust counters.
164 */
165 if (hotplug)
166 tasks = atomic_read(&pai_root[idx].tskctx);
167
168 cpump = mp->mapptr;
169 if (refcount_sub_and_test(tasks, &cpump->refcnt))
170 pai_free(mp);
171 pai_root_free(idx, tasks);
172 }
173
pai_event_destroy(struct perf_event * event)174 static void pai_event_destroy(struct perf_event *event)
175 {
176 int cpu = 0, idx = PAI_PMU_IDX(event);
177
178 free_page(PAI_SAVE_AREA(event));
179 cpus_read_lock();
180 mutex_lock(&pai_reserve_mutex);
181 if (event->cpu == -1) {
182 atomic_dec(&pai_root[idx].tskctx);
183 for_each_online_cpu(cpu)
184 pai_event_destroy_cpu(idx, cpu, false);
185 } else {
186 pai_event_destroy_cpu(idx, event->cpu, false);
187 }
188 mutex_unlock(&pai_reserve_mutex);
189 cpus_read_unlock();
190 }
191
paicrypt_event_destroy(struct perf_event * event)192 static void paicrypt_event_destroy(struct perf_event *event)
193 {
194 static_branch_dec(&pai_key);
195 pai_event_destroy(event);
196 }
197
pai_getctr(unsigned long * page,int nr,unsigned long offset)198 static u64 pai_getctr(unsigned long *page, int nr, unsigned long offset)
199 {
200 if (offset)
201 nr += offset / sizeof(*page);
202 return page[nr];
203 }
204
pai_setctr(unsigned long * page,int nr,unsigned long offset,u64 v)205 static void pai_setctr(unsigned long *page, int nr, unsigned long offset, u64 v)
206 {
207 if (offset)
208 nr += offset / sizeof(*page);
209 page[nr] = v;
210 }
211
212 /* Read the counter values. Return value from location in CMP. For base
213 * event xxx_ALL sum up all events. Returns counter value.
214 */
pai_getdata(struct perf_event * event,bool kernel)215 static u64 pai_getdata(struct perf_event *event, bool kernel)
216 {
217 int idx = PAI_PMU_IDX(event);
218 struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr);
219 struct pai_pmu *pp = &pai_pmu[idx];
220 struct pai_map *cpump = mp->mapptr;
221 unsigned int i;
222 u64 sum = 0;
223
224 if (event->attr.config != pp->base) {
225 return pai_getctr(cpump->area,
226 event->attr.config - pp->base,
227 kernel ? pp->kernel_offset : 0);
228 }
229
230 for (i = 1; i <= pp->num_avail; i++) {
231 u64 val = pai_getctr(cpump->area, i,
232 kernel ? pp->kernel_offset : 0);
233
234 if (!val)
235 continue;
236 sum += val;
237 }
238 return sum;
239 }
240
paicrypt_getall(struct perf_event * event)241 static u64 paicrypt_getall(struct perf_event *event)
242 {
243 u64 sum = 0;
244
245 if (!event->attr.exclude_kernel)
246 sum += pai_getdata(event, true);
247 if (!event->attr.exclude_user)
248 sum += pai_getdata(event, false);
249
250 return sum;
251 }
252
253 /* Called under mutex_lock */
pai_alloc_cpu(int idx,int cpu,bool hotplug)254 static int pai_alloc_cpu(int idx, int cpu, bool hotplug)
255 {
256 struct pai_map *cpump = NULL;
257 bool need_paiext_cb = false;
258 struct pai_mapptr *mp;
259 int tasks = 1, rc = 0;
260
261 /* When hotplug is true, invocation is from CPU hotplug callback.
262 * Allocate per-CPU resource when per-task events are currently active.
263 * This can be more than one. In this case adjust all reference
264 * counters. Otherwise return, this ensures memory is only allocated
265 * when needed.
266 */
267 if (hotplug) {
268 tasks = atomic_read(&pai_root[idx].tskctx);
269 if (!tasks)
270 goto out;
271 }
272
273 /* Allocate root node */
274 rc = pai_root_alloc(idx);
275 if (rc)
276 goto out;
277
278 /* Allocate node for this event */
279 mp = per_cpu_ptr(pai_root[idx].mapptr, cpu);
280 cpump = mp->mapptr;
281 if (!cpump) { /* Paicrypt_map allocated? */
282 rc = -ENOMEM;
283 cpump = kzalloc_obj(*cpump);
284 if (!cpump)
285 goto undo;
286 /* Allocate memory for counter page and counter extraction.
287 * Only the first counting event has to allocate a page.
288 */
289 mp->mapptr = cpump;
290 if (idx == PAI_PMU_CRYPTO) {
291 cpump->area = (unsigned long *)get_zeroed_page(GFP_KERNEL);
292 /* free_page() can handle 0x0 address */
293 cpump->fullpage = true;
294 } else { /* PAI_PMU_EXT */
295 /*
296 * Allocate memory for counter area and counter extraction.
297 * These are
298 * - a 512 byte block and requires 512 byte boundary
299 * alignment.
300 * - a 1KB byte block and requires 1KB boundary
301 * alignment.
302 * Only the first counting event has to allocate the area.
303 *
304 * Note: This works with commit 59bb47985c1d by default.
305 * Backporting this to kernels without this commit might
306 * needs adjustment.
307 */
308 cpump->area = kzalloc(pai_pmu[idx].area_size, GFP_KERNEL);
309 cpump->paiext_cb = kzalloc(PAIE1_CB_SZ, GFP_KERNEL);
310 need_paiext_cb = true;
311 }
312 cpump->save = kvmalloc_objs(struct pai_userdata,
313 pai_pmu[idx].num_avail + 1);
314 if (!cpump->area || !cpump->save ||
315 (need_paiext_cb && !cpump->paiext_cb)) {
316 pai_free(mp);
317 goto undo;
318 }
319 INIT_LIST_HEAD(&cpump->syswide_list);
320 refcount_set(&cpump->refcnt, tasks);
321 rc = 0;
322 } else {
323 refcount_add(tasks, &cpump->refcnt);
324 }
325 /* If tasks is greater than 1, we are called from CPU hotplug path
326 * and need to adjust the pai_root[idx].refcnt by the number of
327 * per-process events. Function pai_root_alloc(idx) already
328 * incremented by one. Adjust for the rest.
329 */
330 if (tasks > 1)
331 refcount_add(tasks - 1, &pai_root[idx].refcnt);
332
333 undo:
334 if (rc) {
335 /* Error in allocation of event, decrement anchor. Since
336 * the event in not created, its destroy() function is never
337 * invoked. Adjust the reference counter for the anchor.
338 * The failure happened in the case of variable
339 * cpump == NULL branch above. The pai_root[XXX].refcnt has
340 * been incremented by one. Then the per-CPU allocation
341 * failed, so decrement it by one, regardless of tasks.
342 */
343 pai_root_free(idx, 1);
344 }
345 out:
346 /* If rc is non-zero, no increment of counter/sampler was done. */
347 return rc;
348 }
349
350 /* Check concurrent access of counting and sampling for PAI events.
351 * This function is called in process context and it is safe to block.
352 * When the event initialization functions fails, no other call back will
353 * be invoked.
354 * Called under mutex_lock.
355 */
pai_alloc(struct perf_event * event)356 static int pai_alloc(struct perf_event *event)
357 {
358 int idx = PAI_PMU_IDX(event);
359 struct cpumask *maskptr;
360 int cpu, rc = -ENOMEM;
361
362 maskptr = kzalloc_obj(*maskptr);
363 if (!maskptr)
364 goto out;
365
366 for_each_online_cpu(cpu) {
367 rc = pai_alloc_cpu(idx, cpu, false);
368 if (rc) {
369 for_each_cpu(cpu, maskptr)
370 pai_event_destroy_cpu(idx, cpu, false);
371 goto undo;
372 }
373 cpumask_set_cpu(cpu, maskptr);
374 }
375
376 rc = 0;
377 /* Trace per-task events for CPU hotplug. */
378 atomic_inc(&pai_root[idx].tskctx);
379 undo:
380 kfree(maskptr);
381 out:
382 return rc;
383 }
384
385 /* Validate event number and return error if event is not supported.
386 * On successful return, PAI_PMU_IDX(event) is set to the index of
387 * the supporting paing_support[] array element.
388 */
pai_event_valid(struct perf_event * event,int idx)389 static int pai_event_valid(struct perf_event *event, int idx)
390 {
391 struct perf_event_attr *a = &event->attr;
392 struct pai_pmu *pp = &pai_pmu[idx];
393
394 /* PAI crypto PMU registered as PERF_TYPE_RAW, check event type */
395 if (a->type != PERF_TYPE_RAW && event->pmu->type != a->type)
396 return -ENOENT;
397 /* Allow only CRYPTO_ALL/NNPA_ALL for sampling */
398 if (a->sample_period && a->config != pp->base)
399 return -EINVAL;
400 /* PAI crypto event must be in valid range, try others if not */
401 if (a->config < pp->base || a->config > pp->base + pp->num_avail)
402 return -ENOENT;
403 if (idx == PAI_PMU_EXT && a->exclude_user)
404 return -EINVAL;
405 PAI_PMU_IDX(event) = idx;
406 return 0;
407 }
408
409 /* Might be called on different CPU than the one the event is intended for. */
pai_event_init(struct perf_event * event,int idx)410 static int pai_event_init(struct perf_event *event, int idx)
411 {
412 struct perf_event_attr *a = &event->attr;
413 int rc;
414
415 /* PAI event must be valid and in supported range */
416 rc = pai_event_valid(event, idx);
417 if (rc)
418 goto out;
419 /* Get a page to store last counter values for sampling */
420 if (a->sample_period) {
421 PAI_SAVE_AREA(event) = get_zeroed_page(GFP_KERNEL);
422 if (!PAI_SAVE_AREA(event)) {
423 rc = -ENOMEM;
424 goto out;
425 }
426 }
427
428 cpus_read_lock();
429 mutex_lock(&pai_reserve_mutex);
430 if (event->cpu >= 0)
431 rc = pai_alloc_cpu(idx, event->cpu, false);
432 else
433 rc = pai_alloc(event);
434 mutex_unlock(&pai_reserve_mutex);
435 cpus_read_unlock();
436 if (rc) {
437 free_page(PAI_SAVE_AREA(event));
438 goto out;
439 }
440
441 if (a->sample_period) {
442 a->sample_period = 1;
443 a->freq = 0;
444 /* Register for paicrypt_sched_task() to be called */
445 event->attach_state |= PERF_ATTACH_SCHED_CB;
446 /* Add raw data which contain the memory mapped counters */
447 a->sample_type |= PERF_SAMPLE_RAW;
448 /* Turn off inheritance */
449 a->inherit = 0;
450 }
451 out:
452 return rc;
453 }
454
paicrypt_event_init(struct perf_event * event)455 static int paicrypt_event_init(struct perf_event *event)
456 {
457 int rc = pai_event_init(event, PAI_PMU_CRYPTO);
458
459 if (!rc) {
460 event->destroy = paicrypt_event_destroy;
461 static_branch_inc(&pai_key);
462 }
463 return rc;
464 }
465
pai_read(struct perf_event * event,u64 (* fct)(struct perf_event * event))466 static void pai_read(struct perf_event *event,
467 u64 (*fct)(struct perf_event *event))
468 {
469 u64 prev, new, delta;
470
471 prev = local64_read(&event->hw.prev_count);
472 new = fct(event);
473 local64_set(&event->hw.prev_count, new);
474 delta = (prev <= new) ? new - prev : (-1ULL - prev) + new + 1;
475 local64_add(delta, &event->count);
476 }
477
paicrypt_read(struct perf_event * event)478 static void paicrypt_read(struct perf_event *event)
479 {
480 pai_read(event, paicrypt_getall);
481 }
482
pai_start(struct perf_event * event,int flags,u64 (* fct)(struct perf_event * event))483 static void pai_start(struct perf_event *event, int flags,
484 u64 (*fct)(struct perf_event *event))
485 {
486 int idx = PAI_PMU_IDX(event);
487 struct pai_pmu *pp = &pai_pmu[idx];
488 struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr);
489 struct pai_map *cpump = mp->mapptr;
490 u64 sum;
491
492 if (!event->attr.sample_period) { /* Counting */
493 sum = fct(event); /* Get current value */
494 local64_set(&event->hw.prev_count, sum);
495 } else { /* Sampling */
496 memcpy((void *)PAI_SAVE_AREA(event), cpump->area, pp->area_size);
497 /* Enable context switch callback for system-wide sampling */
498 if (!(event->attach_state & PERF_ATTACH_TASK)) {
499 list_add_tail(PAI_SWLIST(event), &cpump->syswide_list);
500 perf_sched_cb_inc(event->pmu);
501 } else {
502 cpump->event = event;
503 }
504 }
505 event->hw.state &= ~PERF_HES_STOPPED;
506 }
507
paicrypt_start(struct perf_event * event,int flags)508 static void paicrypt_start(struct perf_event *event, int flags)
509 {
510 pai_start(event, flags, paicrypt_getall);
511 }
512
pai_add(struct perf_event * event,int flags)513 static int pai_add(struct perf_event *event, int flags)
514 {
515 int idx = PAI_PMU_IDX(event);
516 struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr);
517 struct pai_map *cpump = mp->mapptr;
518 struct paiext_cb *pcb = cpump->paiext_cb;
519 unsigned long ccd;
520
521 if (++cpump->active_events == 1) {
522 if (!pcb) { /* PAI crypto */
523 ccd = virt_to_phys(cpump->area) | PAI_CRYPTO_KERNEL_OFFSET;
524 WRITE_ONCE(get_lowcore()->ccd, ccd);
525 local_ctl_set_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT);
526 } else { /* PAI extension 1 */
527 ccd = virt_to_phys(pcb);
528 WRITE_ONCE(get_lowcore()->aicd, ccd);
529 pcb->acc = virt_to_phys(cpump->area) | 0x1;
530 /* Enable CPU instruction lookup for PAIE1 control block */
531 local_ctl_set_bit(0, CR0_PAI_EXTENSION_BIT);
532 }
533 }
534 if (flags & PERF_EF_START)
535 pai_pmu[idx].pmu->start(event, PERF_EF_RELOAD);
536 event->hw.state = 0;
537 return 0;
538 }
539
paicrypt_add(struct perf_event * event,int flags)540 static int paicrypt_add(struct perf_event *event, int flags)
541 {
542 return pai_add(event, flags);
543 }
544
545 static void pai_have_sample(struct perf_event *, struct pai_map *);
pai_stop(struct perf_event * event,int flags)546 static void pai_stop(struct perf_event *event, int flags)
547 {
548 int idx = PAI_PMU_IDX(event);
549 struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr);
550 struct pai_map *cpump = mp->mapptr;
551
552 /* Cope with multiple invocations:
553 * 1. perf_event_throttle() --> PMU->stop()
554 * 2. task schedules out --> PMU->stop()
555 * Check for event already stopped.
556 */
557 if (event->hw.state & PERF_HES_STOPPED)
558 return;
559 if (!event->attr.sample_period) { /* Counting */
560 pai_pmu[idx].pmu->read(event);
561 } else { /* Sampling */
562 if (!(event->attach_state & PERF_ATTACH_TASK)) {
563 perf_sched_cb_dec(event->pmu);
564 list_del(PAI_SWLIST(event));
565 } else {
566 pai_have_sample(event, cpump);
567 cpump->event = NULL;
568 }
569 }
570 event->hw.state = PERF_HES_STOPPED;
571 }
572
paicrypt_stop(struct perf_event * event,int flags)573 static void paicrypt_stop(struct perf_event *event, int flags)
574 {
575 pai_stop(event, flags);
576 }
577
pai_del(struct perf_event * event,int flags)578 static void pai_del(struct perf_event *event, int flags)
579 {
580 int idx = PAI_PMU_IDX(event);
581 struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr);
582 struct pai_map *cpump = mp->mapptr;
583 struct paiext_cb *pcb = cpump->paiext_cb;
584
585 pai_pmu[idx].pmu->stop(event, PERF_EF_UPDATE);
586 if (--cpump->active_events == 0) {
587 if (!pcb) { /* PAI crypto */
588 local_ctl_clear_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT);
589 WRITE_ONCE(get_lowcore()->ccd, 0);
590 } else { /* PAI extension 1 */
591 /* Disable CPU instruction lookup for PAIE1 control block */
592 local_ctl_clear_bit(0, CR0_PAI_EXTENSION_BIT);
593 pcb->acc = 0;
594 WRITE_ONCE(get_lowcore()->aicd, 0);
595 }
596 }
597 }
598
paicrypt_del(struct perf_event * event,int flags)599 static void paicrypt_del(struct perf_event *event, int flags)
600 {
601 pai_del(event, flags);
602 }
603
604 /* Create raw data and save it in buffer. Calculate the delta for each
605 * counter between this invocation and the last invocation.
606 * Returns number of bytes copied.
607 * After reading from PAI counter page, save the read value to the old
608 * page to calculate PAI counter deltas.
609 * Saves only entries with positive counter difference of the form
610 * 2 bytes: Number of counter
611 * 8 bytes: Value of counter
612 */
pai_copy(struct pai_userdata * userdata,unsigned long * page,struct pai_pmu * pp,unsigned long * page_old,bool exclude_user,bool exclude_kernel)613 static size_t pai_copy(struct pai_userdata *userdata, unsigned long *page,
614 struct pai_pmu *pp, unsigned long *page_old,
615 bool exclude_user, bool exclude_kernel)
616 {
617 int i, outidx = 0;
618
619 for (i = 1; i <= pp->num_avail; i++) {
620 u64 val = 0, val_old = 0, val_k = 0, val_old_k = 0;
621
622 if (!exclude_kernel) {
623 val_k = pai_getctr(page, i, pp->kernel_offset);
624 val_old_k = pai_getctr(page_old, i, pp->kernel_offset);
625 if (val_k != val_old_k)
626 pai_setctr(page_old, i, pp->kernel_offset, val_k);
627 }
628 if (!exclude_user) {
629 val = pai_getctr(page, i, 0);
630 val_old = pai_getctr(page_old, i, 0);
631 if (val != val_old)
632 pai_setctr(page_old, i, 0, val);
633 }
634 val += val_k;
635 val_old += val_old_k;
636 if (val >= val_old)
637 val -= val_old;
638 else
639 val = (~0ULL - val_old) + val + 1;
640 if (val) {
641 userdata[outidx].num = i;
642 userdata[outidx].value = val;
643 outidx++;
644 }
645 }
646 return outidx * sizeof(*userdata);
647 }
648
649 /* Write sample when one or more counters values are nonzero.
650 *
651 * Note: The function paicrypt_sched_task() and pai_push_sample() are not
652 * invoked after function paicrypt_del() has been called because of function
653 * perf_sched_cb_dec(). Both functions are only
654 * called when sampling is active. Function perf_sched_cb_inc()
655 * has been invoked to install function paicrypt_sched_task() as call back
656 * to run at context switch time.
657 *
658 * This causes function perf_event_context_sched_out() and
659 * perf_event_context_sched_in() to check whether the PMU has installed an
660 * sched_task() callback. That callback is not active after paicrypt_del()
661 * returns and has deleted the event on that CPU.
662 */
pai_push_sample(size_t rawsize,struct pai_map * cpump,struct perf_event * event)663 static int pai_push_sample(size_t rawsize, struct pai_map *cpump,
664 struct perf_event *event)
665 {
666 struct perf_sample_data data;
667 struct perf_raw_record raw;
668 struct pt_regs regs;
669 int overflow;
670
671 /* Setup perf sample */
672 memset(®s, 0, sizeof(regs));
673 memset(&raw, 0, sizeof(raw));
674 memset(&data, 0, sizeof(data));
675 perf_sample_data_init(&data, 0, event->hw.last_period);
676 if (event->attr.sample_type & PERF_SAMPLE_TID) {
677 data.tid_entry.pid = task_tgid_nr(current);
678 data.tid_entry.tid = task_pid_nr(current);
679 }
680 if (event->attr.sample_type & PERF_SAMPLE_TIME)
681 data.time = event->clock();
682 if (event->attr.sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER))
683 data.id = event->id;
684 if (event->attr.sample_type & PERF_SAMPLE_CPU) {
685 data.cpu_entry.cpu = smp_processor_id();
686 data.cpu_entry.reserved = 0;
687 }
688 if (event->attr.sample_type & PERF_SAMPLE_RAW) {
689 raw.frag.size = rawsize;
690 raw.frag.data = cpump->save;
691 perf_sample_save_raw_data(&data, event, &raw);
692 }
693
694 overflow = perf_event_overflow(event, &data, ®s);
695 perf_event_update_userpage(event);
696 return overflow;
697 }
698
699 /* Check if there is data to be saved on schedule out of a task. */
pai_have_sample(struct perf_event * event,struct pai_map * cpump)700 static void pai_have_sample(struct perf_event *event, struct pai_map *cpump)
701 {
702 struct pai_pmu *pp;
703 size_t rawsize;
704
705 if (!event) /* No event active */
706 return;
707 pp = &pai_pmu[PAI_PMU_IDX(event)];
708 rawsize = pai_copy(cpump->save, cpump->area, pp,
709 (unsigned long *)PAI_SAVE_AREA(event),
710 event->attr.exclude_user,
711 !pp->kernel_offset ? true : event->attr.exclude_kernel);
712 if (rawsize) /* No incremented counters */
713 pai_push_sample(rawsize, cpump, event);
714 }
715
716 /* Check if there is data to be saved on schedule out of a task. */
pai_have_samples(int idx)717 static void pai_have_samples(int idx)
718 {
719 struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr);
720 struct pai_map *cpump = mp->mapptr;
721 struct perf_event *event, *e2;
722
723 list_for_each_entry_safe(event, e2, &cpump->syswide_list, hw.tp_list)
724 pai_have_sample(event, cpump);
725 }
726
727 /* Called on schedule-in and schedule-out. No access to event structure,
728 * but for sampling only event CRYPTO_ALL is allowed.
729 */
paicrypt_sched_task(struct perf_event_pmu_context * pmu_ctx,struct task_struct * task,bool sched_in)730 static void paicrypt_sched_task(struct perf_event_pmu_context *pmu_ctx,
731 struct task_struct *task, bool sched_in)
732 {
733 /* We started with a clean page on event installation. So read out
734 * results on schedule_out and if page was dirty, save old values.
735 */
736 if (!sched_in)
737 pai_have_samples(PAI_PMU_CRYPTO);
738 }
739
740 /* Prevent ioctl(fd, PERF_EVENT_IOC_PERIOD, ...) call.
741 * It sets perf_event::event_limit to a positive value and causes
742 * perf_event_overflow() to invoke pai_stop() call back function when
743 * perf_event::event_limit hits zero. This is not supported because the
744 * sample events CRYPTO_ALL and NNPA_ALL are always taken at schedule out
745 * of a task.
746 */
pai_check_period(struct perf_event * event,u64 value)747 static int pai_check_period(struct perf_event *event, u64 value)
748 {
749 return -EINVAL;
750 }
751 /* ============================= paiext ====================================*/
752
paiext_event_destroy(struct perf_event * event)753 static void paiext_event_destroy(struct perf_event *event)
754 {
755 pai_event_destroy(event);
756 }
757
758 /* Might be called on different CPU than the one the event is intended for. */
paiext_event_init(struct perf_event * event)759 static int paiext_event_init(struct perf_event *event)
760 {
761 int rc = pai_event_init(event, PAI_PMU_EXT);
762
763 if (!rc) {
764 event->attr.exclude_kernel = true; /* No kernel space part */
765 event->destroy = paiext_event_destroy;
766 /* Offset of NNPA in paiext_cb */
767 event->hw.config_base = offsetof(struct paiext_cb, acc);
768 }
769 return rc;
770 }
771
paiext_getall(struct perf_event * event)772 static u64 paiext_getall(struct perf_event *event)
773 {
774 return pai_getdata(event, false);
775 }
776
paiext_read(struct perf_event * event)777 static void paiext_read(struct perf_event *event)
778 {
779 pai_read(event, paiext_getall);
780 }
781
paiext_start(struct perf_event * event,int flags)782 static void paiext_start(struct perf_event *event, int flags)
783 {
784 pai_start(event, flags, paiext_getall);
785 }
786
paiext_add(struct perf_event * event,int flags)787 static int paiext_add(struct perf_event *event, int flags)
788 {
789 return pai_add(event, flags);
790 }
791
paiext_stop(struct perf_event * event,int flags)792 static void paiext_stop(struct perf_event *event, int flags)
793 {
794 pai_stop(event, flags);
795 }
796
paiext_del(struct perf_event * event,int flags)797 static void paiext_del(struct perf_event *event, int flags)
798 {
799 pai_del(event, flags);
800 }
801
802 /* Called on schedule-in and schedule-out. No access to event structure,
803 * but for sampling only event NNPA_ALL is allowed.
804 */
paiext_sched_task(struct perf_event_pmu_context * pmu_ctx,struct task_struct * task,bool sched_in)805 static void paiext_sched_task(struct perf_event_pmu_context *pmu_ctx,
806 struct task_struct *task, bool sched_in)
807 {
808 /* We started with a clean page on event installation. So read out
809 * results on schedule_out and if page was dirty, save old values.
810 */
811 if (!sched_in)
812 pai_have_samples(PAI_PMU_EXT);
813 }
814
815 /* Attribute definitions for paicrypt interface. As with other CPU
816 * Measurement Facilities, there is one attribute per mapped counter.
817 * The number of mapped counters may vary per machine generation. Use
818 * the QUERY PROCESSOR ACTIVITY COUNTER INFORMATION (QPACI) instruction
819 * to determine the number of mapped counters. The instructions returns
820 * a positive number, which is the highest number of supported counters.
821 * All counters less than this number are also supported, there are no
822 * holes. A returned number of zero means no support for mapped counters.
823 *
824 * The identification of the counter is a unique number. The chosen range
825 * is 0x1000 + offset in mapped kernel page.
826 * All CPU Measurement Facility counters identifiers must be unique and
827 * the numbers from 0 to 496 are already used for the CPU Measurement
828 * Counter facility. Numbers 0xb0000, 0xbc000 and 0xbd000 are already
829 * used for the CPU Measurement Sampling facility.
830 */
831 PMU_FORMAT_ATTR(event, "config:0-63");
832
833 static struct attribute *paicrypt_format_attr[] = {
834 &format_attr_event.attr,
835 NULL,
836 };
837
838 static struct attribute_group paicrypt_events_group = {
839 .name = "events",
840 .attrs = NULL /* Filled in attr_event_init() */
841 };
842
843 static struct attribute_group paicrypt_format_group = {
844 .name = "format",
845 .attrs = paicrypt_format_attr,
846 };
847
848 static const struct attribute_group *paicrypt_attr_groups[] = {
849 &paicrypt_events_group,
850 &paicrypt_format_group,
851 NULL,
852 };
853
854 /* Performance monitoring unit for mapped counters */
855 static struct pmu paicrypt = {
856 .task_ctx_nr = perf_hw_context,
857 .event_init = paicrypt_event_init,
858 .add = paicrypt_add,
859 .del = paicrypt_del,
860 .start = paicrypt_start,
861 .stop = paicrypt_stop,
862 .read = paicrypt_read,
863 .sched_task = paicrypt_sched_task,
864 .check_period = pai_check_period,
865 .attr_groups = paicrypt_attr_groups
866 };
867
868 /* List of symbolic PAI counter names. */
869 static const char * const paicrypt_ctrnames[] = {
870 [0] = "CRYPTO_ALL",
871 [1] = "KM_DEA",
872 [2] = "KM_TDEA_128",
873 [3] = "KM_TDEA_192",
874 [4] = "KM_ENCRYPTED_DEA",
875 [5] = "KM_ENCRYPTED_TDEA_128",
876 [6] = "KM_ENCRYPTED_TDEA_192",
877 [7] = "KM_AES_128",
878 [8] = "KM_AES_192",
879 [9] = "KM_AES_256",
880 [10] = "KM_ENCRYPTED_AES_128",
881 [11] = "KM_ENCRYPTED_AES_192",
882 [12] = "KM_ENCRYPTED_AES_256",
883 [13] = "KM_XTS_AES_128",
884 [14] = "KM_XTS_AES_256",
885 [15] = "KM_XTS_ENCRYPTED_AES_128",
886 [16] = "KM_XTS_ENCRYPTED_AES_256",
887 [17] = "KMC_DEA",
888 [18] = "KMC_TDEA_128",
889 [19] = "KMC_TDEA_192",
890 [20] = "KMC_ENCRYPTED_DEA",
891 [21] = "KMC_ENCRYPTED_TDEA_128",
892 [22] = "KMC_ENCRYPTED_TDEA_192",
893 [23] = "KMC_AES_128",
894 [24] = "KMC_AES_192",
895 [25] = "KMC_AES_256",
896 [26] = "KMC_ENCRYPTED_AES_128",
897 [27] = "KMC_ENCRYPTED_AES_192",
898 [28] = "KMC_ENCRYPTED_AES_256",
899 [29] = "KMC_PRNG",
900 [30] = "KMA_GCM_AES_128",
901 [31] = "KMA_GCM_AES_192",
902 [32] = "KMA_GCM_AES_256",
903 [33] = "KMA_GCM_ENCRYPTED_AES_128",
904 [34] = "KMA_GCM_ENCRYPTED_AES_192",
905 [35] = "KMA_GCM_ENCRYPTED_AES_256",
906 [36] = "KMF_DEA",
907 [37] = "KMF_TDEA_128",
908 [38] = "KMF_TDEA_192",
909 [39] = "KMF_ENCRYPTED_DEA",
910 [40] = "KMF_ENCRYPTED_TDEA_128",
911 [41] = "KMF_ENCRYPTED_TDEA_192",
912 [42] = "KMF_AES_128",
913 [43] = "KMF_AES_192",
914 [44] = "KMF_AES_256",
915 [45] = "KMF_ENCRYPTED_AES_128",
916 [46] = "KMF_ENCRYPTED_AES_192",
917 [47] = "KMF_ENCRYPTED_AES_256",
918 [48] = "KMCTR_DEA",
919 [49] = "KMCTR_TDEA_128",
920 [50] = "KMCTR_TDEA_192",
921 [51] = "KMCTR_ENCRYPTED_DEA",
922 [52] = "KMCTR_ENCRYPTED_TDEA_128",
923 [53] = "KMCTR_ENCRYPTED_TDEA_192",
924 [54] = "KMCTR_AES_128",
925 [55] = "KMCTR_AES_192",
926 [56] = "KMCTR_AES_256",
927 [57] = "KMCTR_ENCRYPTED_AES_128",
928 [58] = "KMCTR_ENCRYPTED_AES_192",
929 [59] = "KMCTR_ENCRYPTED_AES_256",
930 [60] = "KMO_DEA",
931 [61] = "KMO_TDEA_128",
932 [62] = "KMO_TDEA_192",
933 [63] = "KMO_ENCRYPTED_DEA",
934 [64] = "KMO_ENCRYPTED_TDEA_128",
935 [65] = "KMO_ENCRYPTED_TDEA_192",
936 [66] = "KMO_AES_128",
937 [67] = "KMO_AES_192",
938 [68] = "KMO_AES_256",
939 [69] = "KMO_ENCRYPTED_AES_128",
940 [70] = "KMO_ENCRYPTED_AES_192",
941 [71] = "KMO_ENCRYPTED_AES_256",
942 [72] = "KIMD_SHA_1",
943 [73] = "KIMD_SHA_256",
944 [74] = "KIMD_SHA_512",
945 [75] = "KIMD_SHA3_224",
946 [76] = "KIMD_SHA3_256",
947 [77] = "KIMD_SHA3_384",
948 [78] = "KIMD_SHA3_512",
949 [79] = "KIMD_SHAKE_128",
950 [80] = "KIMD_SHAKE_256",
951 [81] = "KIMD_GHASH",
952 [82] = "KLMD_SHA_1",
953 [83] = "KLMD_SHA_256",
954 [84] = "KLMD_SHA_512",
955 [85] = "KLMD_SHA3_224",
956 [86] = "KLMD_SHA3_256",
957 [87] = "KLMD_SHA3_384",
958 [88] = "KLMD_SHA3_512",
959 [89] = "KLMD_SHAKE_128",
960 [90] = "KLMD_SHAKE_256",
961 [91] = "KMAC_DEA",
962 [92] = "KMAC_TDEA_128",
963 [93] = "KMAC_TDEA_192",
964 [94] = "KMAC_ENCRYPTED_DEA",
965 [95] = "KMAC_ENCRYPTED_TDEA_128",
966 [96] = "KMAC_ENCRYPTED_TDEA_192",
967 [97] = "KMAC_AES_128",
968 [98] = "KMAC_AES_192",
969 [99] = "KMAC_AES_256",
970 [100] = "KMAC_ENCRYPTED_AES_128",
971 [101] = "KMAC_ENCRYPTED_AES_192",
972 [102] = "KMAC_ENCRYPTED_AES_256",
973 [103] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_DEA",
974 [104] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_TDEA_128",
975 [105] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_TDEA_192",
976 [106] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_DEA",
977 [107] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_TDEA_128",
978 [108] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_TDEA_192",
979 [109] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_128",
980 [110] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_192",
981 [111] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_256",
982 [112] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_128",
983 [113] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_192",
984 [114] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_256",
985 [115] = "PCC_COMPUTE_XTS_PARAMETER_USING_AES_128",
986 [116] = "PCC_COMPUTE_XTS_PARAMETER_USING_AES_256",
987 [117] = "PCC_COMPUTE_XTS_PARAMETER_USING_ENCRYPTED_AES_128",
988 [118] = "PCC_COMPUTE_XTS_PARAMETER_USING_ENCRYPTED_AES_256",
989 [119] = "PCC_SCALAR_MULTIPLY_P256",
990 [120] = "PCC_SCALAR_MULTIPLY_P384",
991 [121] = "PCC_SCALAR_MULTIPLY_P521",
992 [122] = "PCC_SCALAR_MULTIPLY_ED25519",
993 [123] = "PCC_SCALAR_MULTIPLY_ED448",
994 [124] = "PCC_SCALAR_MULTIPLY_X25519",
995 [125] = "PCC_SCALAR_MULTIPLY_X448",
996 [126] = "PRNO_SHA_512_DRNG",
997 [127] = "PRNO_TRNG_QUERY_RAW_TO_CONDITIONED_RATIO",
998 [128] = "PRNO_TRNG",
999 [129] = "KDSA_ECDSA_VERIFY_P256",
1000 [130] = "KDSA_ECDSA_VERIFY_P384",
1001 [131] = "KDSA_ECDSA_VERIFY_P521",
1002 [132] = "KDSA_ECDSA_SIGN_P256",
1003 [133] = "KDSA_ECDSA_SIGN_P384",
1004 [134] = "KDSA_ECDSA_SIGN_P521",
1005 [135] = "KDSA_ENCRYPTED_ECDSA_SIGN_P256",
1006 [136] = "KDSA_ENCRYPTED_ECDSA_SIGN_P384",
1007 [137] = "KDSA_ENCRYPTED_ECDSA_SIGN_P521",
1008 [138] = "KDSA_EDDSA_VERIFY_ED25519",
1009 [139] = "KDSA_EDDSA_VERIFY_ED448",
1010 [140] = "KDSA_EDDSA_SIGN_ED25519",
1011 [141] = "KDSA_EDDSA_SIGN_ED448",
1012 [142] = "KDSA_ENCRYPTED_EDDSA_SIGN_ED25519",
1013 [143] = "KDSA_ENCRYPTED_EDDSA_SIGN_ED448",
1014 [144] = "PCKMO_ENCRYPT_DEA_KEY",
1015 [145] = "PCKMO_ENCRYPT_TDEA_128_KEY",
1016 [146] = "PCKMO_ENCRYPT_TDEA_192_KEY",
1017 [147] = "PCKMO_ENCRYPT_AES_128_KEY",
1018 [148] = "PCKMO_ENCRYPT_AES_192_KEY",
1019 [149] = "PCKMO_ENCRYPT_AES_256_KEY",
1020 [150] = "PCKMO_ENCRYPT_ECC_P256_KEY",
1021 [151] = "PCKMO_ENCRYPT_ECC_P384_KEY",
1022 [152] = "PCKMO_ENCRYPT_ECC_P521_KEY",
1023 [153] = "PCKMO_ENCRYPT_ECC_ED25519_KEY",
1024 [154] = "PCKMO_ENCRYPT_ECC_ED448_KEY",
1025 [155] = "IBM_RESERVED_155",
1026 [156] = "IBM_RESERVED_156",
1027 [157] = "KM_FULL_XTS_AES_128",
1028 [158] = "KM_FULL_XTS_AES_256",
1029 [159] = "KM_FULL_XTS_ENCRYPTED_AES_128",
1030 [160] = "KM_FULL_XTS_ENCRYPTED_AES_256",
1031 [161] = "KMAC_HMAC_SHA_224",
1032 [162] = "KMAC_HMAC_SHA_256",
1033 [163] = "KMAC_HMAC_SHA_384",
1034 [164] = "KMAC_HMAC_SHA_512",
1035 [165] = "KMAC_HMAC_ENCRYPTED_SHA_224",
1036 [166] = "KMAC_HMAC_ENCRYPTED_SHA_256",
1037 [167] = "KMAC_HMAC_ENCRYPTED_SHA_384",
1038 [168] = "KMAC_HMAC_ENCRYPTED_SHA_512",
1039 [169] = "PCKMO_ENCRYPT_HMAC_512_KEY",
1040 [170] = "PCKMO_ENCRYPT_HMAC_1024_KEY",
1041 [171] = "PCKMO_ENCRYPT_AES_XTS_128",
1042 [172] = "PCKMO_ENCRYPT_AES_XTS_256",
1043 };
1044
1045 static struct attribute *paiext_format_attr[] = {
1046 &format_attr_event.attr,
1047 NULL,
1048 };
1049
1050 static struct attribute_group paiext_events_group = {
1051 .name = "events",
1052 .attrs = NULL, /* Filled in attr_event_init() */
1053 };
1054
1055 static struct attribute_group paiext_format_group = {
1056 .name = "format",
1057 .attrs = paiext_format_attr,
1058 };
1059
1060 static const struct attribute_group *paiext_attr_groups[] = {
1061 &paiext_events_group,
1062 &paiext_format_group,
1063 NULL,
1064 };
1065
1066 /* Performance monitoring unit for mapped counters */
1067 static struct pmu paiext = {
1068 .task_ctx_nr = perf_hw_context,
1069 .event_init = paiext_event_init,
1070 .add = paiext_add,
1071 .del = paiext_del,
1072 .start = paiext_start,
1073 .stop = paiext_stop,
1074 .read = paiext_read,
1075 .sched_task = paiext_sched_task,
1076 .check_period = pai_check_period,
1077 .attr_groups = paiext_attr_groups,
1078 };
1079
1080 /* List of symbolic PAI extension 1 NNPA counter names. */
1081 static const char * const paiext_ctrnames[] = {
1082 [0] = "NNPA_ALL",
1083 [1] = "NNPA_ADD",
1084 [2] = "NNPA_SUB",
1085 [3] = "NNPA_MUL",
1086 [4] = "NNPA_DIV",
1087 [5] = "NNPA_MIN",
1088 [6] = "NNPA_MAX",
1089 [7] = "NNPA_LOG",
1090 [8] = "NNPA_EXP",
1091 [9] = "NNPA_IBM_RESERVED_9",
1092 [10] = "NNPA_RELU",
1093 [11] = "NNPA_TANH",
1094 [12] = "NNPA_SIGMOID",
1095 [13] = "NNPA_SOFTMAX",
1096 [14] = "NNPA_BATCHNORM",
1097 [15] = "NNPA_MAXPOOL2D",
1098 [16] = "NNPA_AVGPOOL2D",
1099 [17] = "NNPA_LSTMACT",
1100 [18] = "NNPA_GRUACT",
1101 [19] = "NNPA_CONVOLUTION",
1102 [20] = "NNPA_MATMUL_OP",
1103 [21] = "NNPA_MATMUL_OP_BCAST23",
1104 [22] = "NNPA_SMALLBATCH",
1105 [23] = "NNPA_LARGEDIM",
1106 [24] = "NNPA_SMALLTENSOR",
1107 [25] = "NNPA_1MFRAME",
1108 [26] = "NNPA_2GFRAME",
1109 [27] = "NNPA_ACCESSEXCEPT",
1110 [28] = "NNPA_TRANSFORM",
1111 [29] = "NNPA_GELU",
1112 [30] = "NNPA_MOMENTS",
1113 [31] = "NNPA_LAYERNORM",
1114 [32] = "NNPA_MATMUL_OP_BCAST1",
1115 [33] = "NNPA_SQRT",
1116 [34] = "NNPA_INVSQRT",
1117 [35] = "NNPA_NORM",
1118 [36] = "NNPA_REDUCE",
1119 };
1120
attr_event_free(struct attribute ** attrs)1121 static void __init attr_event_free(struct attribute **attrs)
1122 {
1123 struct perf_pmu_events_attr *pa;
1124 unsigned int i;
1125
1126 for (i = 0; attrs[i]; i++) {
1127 struct device_attribute *dap;
1128
1129 dap = container_of(attrs[i], struct device_attribute, attr);
1130 pa = container_of(dap, struct perf_pmu_events_attr, attr);
1131 kfree(pa);
1132 }
1133 kfree(attrs);
1134 }
1135
attr_event_init_one(int num,unsigned long base,const char * name)1136 static struct attribute * __init attr_event_init_one(int num,
1137 unsigned long base,
1138 const char *name)
1139 {
1140 struct perf_pmu_events_attr *pa;
1141
1142 pa = kzalloc_obj(*pa);
1143 if (!pa)
1144 return NULL;
1145
1146 sysfs_attr_init(&pa->attr.attr);
1147 pa->id = base + num;
1148 pa->attr.attr.name = name;
1149 pa->attr.attr.mode = 0444;
1150 pa->attr.show = cpumf_events_sysfs_show;
1151 pa->attr.store = NULL;
1152 return &pa->attr.attr;
1153 }
1154
attr_event_init(struct pai_pmu * p)1155 static struct attribute ** __init attr_event_init(struct pai_pmu *p)
1156 {
1157 unsigned int min_attr = min_t(unsigned int, p->num_named, p->num_avail);
1158 struct attribute **attrs;
1159 unsigned int i;
1160
1161 attrs = kmalloc_objs(*attrs, min_attr + 1, GFP_KERNEL | __GFP_ZERO);
1162 if (!attrs)
1163 goto out;
1164 for (i = 0; i < min_attr; i++) {
1165 attrs[i] = attr_event_init_one(i, p->base, p->names[i]);
1166 if (!attrs[i]) {
1167 attr_event_free(attrs);
1168 attrs = NULL;
1169 goto out;
1170 }
1171 }
1172 attrs[i] = NULL;
1173 out:
1174 return attrs;
1175 }
1176
pai_pmu_exit(struct pai_pmu * p)1177 static void __init pai_pmu_exit(struct pai_pmu *p)
1178 {
1179 attr_event_free(p->event_group->attrs);
1180 p->event_group->attrs = NULL;
1181 }
1182
1183 /* Add a PMU. Install its events and register the PMU device driver
1184 * call back functions.
1185 */
pai_pmu_init(struct pai_pmu * p)1186 static int __init pai_pmu_init(struct pai_pmu *p)
1187 {
1188 int rc = -ENOMEM;
1189
1190
1191 /* Export known PAI events */
1192 p->event_group->attrs = attr_event_init(p);
1193 if (!p->event_group->attrs) {
1194 pr_err("Creation of PMU %s /sysfs failed\n", p->pmuname);
1195 goto out;
1196 }
1197
1198 rc = perf_pmu_register(p->pmu, p->pmuname, -1);
1199 if (rc) {
1200 pai_pmu_exit(p);
1201 pr_err("Registering PMU %s failed with rc=%i\n", p->pmuname,
1202 rc);
1203 }
1204 out:
1205 return rc;
1206 }
1207
1208 /* PAI PMU characteristics table */
1209 static struct pai_pmu pai_pmu[] __refdata = {
1210 [PAI_PMU_CRYPTO] = {
1211 .pmuname = "pai_crypto",
1212 .facility_nr = 196,
1213 .num_named = ARRAY_SIZE(paicrypt_ctrnames),
1214 .names = paicrypt_ctrnames,
1215 .base = PAI_CRYPTO_BASE,
1216 .kernel_offset = PAI_CRYPTO_KERNEL_OFFSET,
1217 .area_size = PAGE_SIZE,
1218 .init = pai_pmu_init,
1219 .exit = pai_pmu_exit,
1220 .pmu = &paicrypt,
1221 .event_group = &paicrypt_events_group
1222 },
1223 [PAI_PMU_EXT] = {
1224 .pmuname = "pai_ext",
1225 .facility_nr = 197,
1226 .num_named = ARRAY_SIZE(paiext_ctrnames),
1227 .names = paiext_ctrnames,
1228 .base = PAI_NNPA_BASE,
1229 .kernel_offset = 0,
1230 .area_size = PAIE1_CTRBLOCK_SZ,
1231 .init = pai_pmu_init,
1232 .exit = pai_pmu_exit,
1233 .pmu = &paiext,
1234 .event_group = &paiext_events_group
1235 }
1236 };
1237
1238 /*
1239 * Check if the PMU (via facility) is supported by machine. Try all of the
1240 * supported PAI PMUs.
1241 * Return number of successfully installed PMUs.
1242 */
paipmu_setup(void)1243 static int __init paipmu_setup(void)
1244 {
1245 struct qpaci_info_block ib;
1246 int install_ok = 0, rc;
1247 struct pai_pmu *p;
1248 size_t i;
1249
1250 for (i = 0; i < ARRAY_SIZE(pai_pmu); ++i) {
1251 p = &pai_pmu[i];
1252
1253 if (!test_facility(p->facility_nr))
1254 continue;
1255
1256 qpaci(&ib);
1257 switch (i) {
1258 case PAI_PMU_CRYPTO:
1259 p->num_avail = ib.num_cc;
1260 if (p->num_avail >= PAI_CRYPTO_MAXCTR) {
1261 pr_err("Too many PMU %s counters %d\n",
1262 p->pmuname, p->num_avail);
1263 continue;
1264 }
1265 break;
1266 case PAI_PMU_EXT:
1267 p->num_avail = ib.num_nnpa;
1268 break;
1269 }
1270 p->num_avail += 1; /* Add xxx_ALL event */
1271 if (p->init) {
1272 rc = p->init(p);
1273 if (!rc)
1274 ++install_ok;
1275 }
1276 }
1277 return install_ok;
1278 }
1279
pai_online_cpu(unsigned int cpu)1280 static int pai_online_cpu(unsigned int cpu)
1281 {
1282 int rc;
1283
1284 mutex_lock(&pai_reserve_mutex);
1285 rc = pai_alloc_cpu(PAI_PMU_CRYPTO, cpu, true);
1286 if (rc)
1287 goto out;
1288 rc = pai_alloc_cpu(PAI_PMU_EXT, cpu, true);
1289 if (rc)
1290 pai_event_destroy_cpu(PAI_PMU_CRYPTO, cpu, true);
1291 out:
1292 mutex_unlock(&pai_reserve_mutex);
1293 return rc;
1294 }
1295
pai_offline_cpu(unsigned int cpu)1296 static int pai_offline_cpu(unsigned int cpu)
1297 {
1298 mutex_lock(&pai_reserve_mutex);
1299 pai_event_destroy_cpu(PAI_PMU_CRYPTO, cpu, true);
1300 pai_event_destroy_cpu(PAI_PMU_EXT, cpu, true);
1301 mutex_unlock(&pai_reserve_mutex);
1302 return 0;
1303 }
1304
pai_init(void)1305 static int __init pai_init(void)
1306 {
1307 int state, rc;
1308
1309 /* Setup s390dbf facility */
1310 paidbg = debug_register("pai", 1, 1, 128);
1311 if (!paidbg) {
1312 pr_err("Registration of s390dbf pai failed\n");
1313 return -ENOMEM;
1314 }
1315 debug_register_view(paidbg, &debug_sprintf_view);
1316
1317 /* CPUHP_BP_PREPARE_DYN --> before CPU is brought online */
1318 state = cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "perf/pai:prepare",
1319 pai_online_cpu, pai_offline_cpu);
1320 rc = state < 0 ? state : 0;
1321 if (rc < 0)
1322 goto out_debug;
1323
1324 rc = -ENODEV;
1325 if (!paipmu_setup())
1326 goto out_cpuhp;
1327 return 0;
1328
1329 out_cpuhp:
1330 cpuhp_remove_state(state);
1331 out_debug:
1332 debug_unregister_view(paidbg, &debug_sprintf_view);
1333 debug_unregister(paidbg);
1334 return rc;
1335 }
1336
1337 device_initcall(pai_init);
1338