1 // SPDX-License-Identifier: GPL-2.0
3  * Performance event support - Processor Activity Instrumentation Extension
48 	struct pai_userdata *save;	/* Area to store non-zero counters */
53 	struct list_head syswide_list;	/* List system-wide sampling events */
92 			return -ENOMEM;  in paiext_root_alloc()
101  * counting and sampling events.
103  * sampling and counting on that cpu is zero.
108 /* Free all memory allocated for event counting/sampling setup */
111 	kfree(mp->mapptr->area);  in paiext_free()
112 	kfree(mp->mapptr->paiext_cb);  in paiext_free()
113 	kvfree(mp->mapptr->save);  in paiext_free()
114 	kfree(mp->mapptr);  in paiext_free()
115 	mp->mapptr = NULL;  in paiext_free()
122 	struct paiext_map *cpump = mp->mapptr;  in paiext_event_destroy_cpu()
125 	if (refcount_dec_and_test(&cpump->refcnt))	/* Last reference gone */  in paiext_event_destroy_cpu()
136 	if (event->cpu == -1) {  in paiext_event_destroy()
143 		paiext_event_destroy_cpu(event, event->cpu);  in paiext_event_destroy()
146 			    event->cpu);  in paiext_event_destroy()
149 /* Used to avoid races in checking concurrent access of counting and
153  * allowed and when this event is running, no counting event is allowed.
154  * Several counting events are allowed in parallel, but no sampling event
155  * is allowed while one (or more) counting events are running.
175 	cpump = mp->mapptr;  in paiext_alloc_cpu()
177 		rc = -ENOMEM;  in paiext_alloc_cpu()
184 		 * - a 512 byte block and requires 512 byte boundary alignment.  in paiext_alloc_cpu()
185 		 * - a 1KB byte block and requires 1KB boundary alignment.  in paiext_alloc_cpu()
186 		 * Only the first counting event has to allocate the area.  in paiext_alloc_cpu()
192 		mp->mapptr = cpump;  in paiext_alloc_cpu()
193 		cpump->area = kzalloc(PAIE1_CTRBLOCK_SZ, GFP_KERNEL);  in paiext_alloc_cpu()
194 		cpump->paiext_cb = kzalloc(PAIE1_CB_SZ, GFP_KERNEL);  in paiext_alloc_cpu()
195 		cpump->save = kvmalloc_array(paiext_cnt + 1,  in paiext_alloc_cpu()
198 		if (!cpump->save || !cpump->area || !cpump->paiext_cb) {  in paiext_alloc_cpu()
202 		INIT_LIST_HEAD(&cpump->syswide_list);  in paiext_alloc_cpu()
203 		refcount_set(&cpump->refcnt, 1);  in paiext_alloc_cpu()
206 		refcount_inc(&cpump->refcnt);  in paiext_alloc_cpu()
219 	/* If rc is non-zero, no increment of counter/sampler was done. */  in paiext_alloc_cpu()
226 	int cpu, rc = -ENOMEM;  in paiext_alloc()
255 /* The PAI extension 1 control block supports up to 128 entries. Return
261 	u64 cfg = event->attr.config;  in paiext_event_valid()
265 		event->hw.config_base = offsetof(struct paiext_cb, acc);  in paiext_event_valid()
268 	return -ENOENT;  in paiext_event_valid()
274 	struct perf_event_attr *a = &event->attr;  in paiext_event_init()
278 	if (a->type != PERF_TYPE_RAW && event->pmu->type != a->type)  in paiext_event_init()
279 		return -ENOENT;  in paiext_event_init()
285 	if (a->sample_period && a->config != PAI_NNPA_BASE)  in paiext_event_init()
286 		return -EINVAL;  in paiext_event_init()
288 	if (a->exclude_user)  in paiext_event_init()
289 		return -EINVAL;  in paiext_event_init()
291 	if (a->sample_period) {  in paiext_event_init()
294 			return -ENOMEM;  in paiext_event_init()
297 	if (event->cpu >= 0)  in paiext_event_init()
298 		rc = paiext_alloc_cpu(event, event->cpu);  in paiext_event_init()
305 	event->destroy = paiext_event_destroy;  in paiext_event_init()
307 	if (a->sample_period) {  in paiext_event_init()
308 		a->sample_period = 1;  in paiext_event_init()
309 		a->freq = 0;  in paiext_event_init()
311 		event->attach_state |= PERF_ATTACH_SCHED_CB;  in paiext_event_init()
313 		a->sample_type |= PERF_SAMPLE_RAW;  in paiext_event_init()
315 		a->inherit = 0;  in paiext_event_init()
327  * NNPA_ALL sum up all events.
332 	struct paiext_map *cpump = mp->mapptr;  in paiext_getdata()
336 	if (event->attr.config != PAI_NNPA_BASE)  in paiext_getdata()
337 		return paiext_getctr(cpump->area,  in paiext_getdata()
338 				     event->attr.config - PAI_NNPA_BASE);  in paiext_getdata()
341 		sum += paiext_getctr(cpump->area, i);  in paiext_getdata()
355 	prev = local64_read(&event->hw.prev_count);  in paiext_read()
357 	local64_set(&event->hw.prev_count, new);  in paiext_read()
358 	delta = new - prev;  in paiext_read()
359 	local64_add(delta, &event->count);  in paiext_read()
365 	struct paiext_map *cpump = mp->mapptr;  in paiext_start()
368 	if (!event->attr.sample_period) {	/* Counting */  in paiext_start()
370 		local64_set(&event->hw.prev_count, sum);  in paiext_start()
372 		memcpy((void *)PAI_SAVE_AREA(event), cpump->area,  in paiext_start()
374 		/* Enable context switch callback for system-wide sampling */  in paiext_start()
375 		if (!(event->attach_state & PERF_ATTACH_TASK)) {  in paiext_start()
376 			list_add_tail(PAI_SWLIST(event), &cpump->syswide_list);  in paiext_start()
377 			perf_sched_cb_inc(event->pmu);  in paiext_start()
379 			cpump->event = event;  in paiext_start()
387 	struct paiext_map *cpump = mp->mapptr;  in paiext_add()
388 	struct paiext_cb *pcb = cpump->paiext_cb;  in paiext_add()
390 	if (++cpump->active_events == 1) {  in paiext_add()
391 		get_lowcore()->aicd = virt_to_phys(cpump->paiext_cb);  in paiext_add()
392 		pcb->acc = virt_to_phys(cpump->area) | 0x1;  in paiext_add()
398 	event->hw.state = 0;  in paiext_add()
406 	struct paiext_map *cpump = mp->mapptr;  in paiext_stop()
408 	if (!event->attr.sample_period) {	/* Counting */  in paiext_stop()
411 		if (!(event->attach_state & PERF_ATTACH_TASK)) {  in paiext_stop()
413 			perf_sched_cb_dec(event->pmu);  in paiext_stop()
416 			cpump->event = NULL;  in paiext_stop()
419 	event->hw.state = PERF_HES_STOPPED;  in paiext_stop()
425 	struct paiext_map *cpump = mp->mapptr;  in paiext_del()
426 	struct paiext_cb *pcb = cpump->paiext_cb;  in paiext_del()
429 	if (--cpump->active_events == 0) {  in paiext_del()
432 		pcb->acc = 0;  in paiext_del()
433 		get_lowcore()->aicd = 0;  in paiext_del()
452 			val -= val_old;  in paiext_copy()
454 			val = (~0ULL - val_old) + val + 1;  in paiext_copy()
491 	perf_sample_data_init(&data, 0, event->hw.last_period);  in paiext_push_sample()
492 	if (event->attr.sample_type & PERF_SAMPLE_TID) {  in paiext_push_sample()
496 	if (event->attr.sample_type & PERF_SAMPLE_TIME)  in paiext_push_sample()
497 		data.time = event->clock();  in paiext_push_sample()
498 	if (event->attr.sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER))  in paiext_push_sample()
499 		data.id = event->id;  in paiext_push_sample()
500 	if (event->attr.sample_type & PERF_SAMPLE_CPU)  in paiext_push_sample()
502 	if (event->attr.sample_type & PERF_SAMPLE_RAW) {  in paiext_push_sample()
504 		raw.frag.data = cpump->save;  in paiext_push_sample()
511 	memcpy((void *)PAI_SAVE_AREA(event), cpump->area,  in paiext_push_sample()
524 	rawsize = paiext_copy(cpump->save, cpump->area,  in paiext_have_sample()
534 	struct paiext_map *cpump = mp->mapptr;  in paiext_have_samples()
537 	list_for_each_entry(event, &cpump->syswide_list, hw.tp_list)  in paiext_have_samples()
541 /* Called on schedule-in and schedule-out. No access to event structure,
572 PMU_FORMAT_ATTR(event, "config:0-63");
675 		return -ENOMEM;  in attr_event_init_one()
677 	sysfs_attr_init(&pa->attr.attr);  in attr_event_init_one()
678 	pa->id = PAI_NNPA_BASE + num;  in attr_event_init_one()
679 	pa->attr.attr.name = paiext_ctrnames[num];  in attr_event_init_one()
680 	pa->attr.attr.mode = 0444;  in attr_event_init_one()
681 	pa->attr.show = cpumf_events_sysfs_show;  in attr_event_init_one()
682 	pa->attr.store = NULL;  in attr_event_init_one()
683 	attrs[num] = &pa->attr.attr;  in attr_event_init_one()
695 		return -ENOMEM;  in attr_event_init()
711 	int rc = -ENOMEM;  in paiext_init()
733 		rc = -ENOMEM;  in paiext_init()
738 	rc = perf_pmu_register(&paiext, KMSG_COMPONENT, -1);  in paiext_init()