1 // SPDX-License-Identifier: GPL-2.0
5 #include <linux/io-64-nonatomic-lo-hi.h>
13 DEFINE_PERFMON_FORMAT_ATTR(event_category, "config:0-3");
14 DEFINE_PERFMON_FORMAT_ATTR(event, "config:4-31");
20 DEFINE_PERFMON_FORMAT_ATTR(filter_wq, "config1:0-31");
21 DEFINE_PERFMON_FORMAT_ATTR(filter_tc, "config1:32-39");
22 DEFINE_PERFMON_FORMAT_ATTR(filter_pgsz, "config1:40-43");
23 DEFINE_PERFMON_FORMAT_ATTR(filter_sz, "config1:44-51");
24 DEFINE_PERFMON_FORMAT_ATTR(filter_eng, "config1:52-59");
52 	return &idxd_pmu->pmu == event->pmu;  in is_idxd_event()
62 	max_count = idxd_pmu->n_counters;  in perfmon_collect_events()
63 	n = idxd_pmu->n_events;  in perfmon_collect_events()
66 		return -EINVAL;  in perfmon_collect_events()
69 		idxd_pmu->event_list[n] = leader;  in perfmon_collect_events()
70 		idxd_pmu->event_list[n]->hw.idx = n;  in perfmon_collect_events()
79 		    event->state <= PERF_EVENT_STATE_OFF)  in perfmon_collect_events()
83 			return -EINVAL;  in perfmon_collect_events()
85 		idxd_pmu->event_list[n] = event;  in perfmon_collect_events()
86 		idxd_pmu->event_list[n]->hw.idx = n;  in perfmon_collect_events()
96 	struct idxd_device *idxd = idxd_pmu->idxd;  in perfmon_assign_hw_event()
97 	struct hw_perf_event *hwc = &event->hw;  in perfmon_assign_hw_event()
99 	hwc->idx = idx;  in perfmon_assign_hw_event()
100 	hwc->config_base = ioread64(CNTRCFG_REG(idxd, idx));  in perfmon_assign_hw_event()
101 	hwc->event_base = ioread64(CNTRCFG_REG(idxd, idx));  in perfmon_assign_hw_event()
110 		if (!test_and_set_bit(i, idxd_pmu->used_mask))  in perfmon_assign_event()
113 	return -EINVAL;  in perfmon_assign_event()
127 	struct perf_event *leader = event->group_leader;  in perfmon_validate_group()
133 		return -ENOMEM;  in perfmon_validate_group()
135 	fake_pmu->pmu.name = pmu->pmu.name;  in perfmon_validate_group()
136 	fake_pmu->n_counters = pmu->n_counters;  in perfmon_validate_group()
144 	fake_pmu->n_events = n;  in perfmon_validate_group()
151 	fake_pmu->n_events = n;  in perfmon_validate_group()
154 		event = fake_pmu->event_list[i];  in perfmon_validate_group()
174 	event->hw.idx = -1;  in perfmon_pmu_event_init()
176 	if (event->attr.type != event->pmu->type)  in perfmon_pmu_event_init()
177 		return -ENOENT;  in perfmon_pmu_event_init()
180 	if (event->attr.sample_period)  in perfmon_pmu_event_init()
181 		return -EINVAL;  in perfmon_pmu_event_init()
183 	if (event->cpu < 0)  in perfmon_pmu_event_init()
184 		return -EINVAL;  in perfmon_pmu_event_init()
186 	if (event->pmu != &idxd->idxd_pmu->pmu)  in perfmon_pmu_event_init()
187 		return -EINVAL;  in perfmon_pmu_event_init()
189 	event->hw.event_base = ioread64(PERFMON_TABLE_OFFSET(idxd));  in perfmon_pmu_event_init()
190 	event->hw.config = event->attr.config;  in perfmon_pmu_event_init()
192 	if (event->group_leader != event)  in perfmon_pmu_event_init()
193 		 /* non-group events have themselves as leader */  in perfmon_pmu_event_init()
194 		ret = perfmon_validate_group(idxd->idxd_pmu, event);  in perfmon_pmu_event_init()
201 	struct hw_perf_event *hwc = &event->hw;  in perfmon_pmu_read_counter()
203 	int cntr = hwc->idx;  in perfmon_pmu_read_counter()
214 	int shift = 64 - idxd->idxd_pmu->counter_width;  in perfmon_pmu_event_update()
215 	struct hw_perf_event *hwc = &event->hw;  in perfmon_pmu_event_update()
217 	prev_raw_count = local64_read(&hwc->prev_count);  in perfmon_pmu_event_update()
220 	} while (!local64_try_cmpxchg(&hwc->prev_count,  in perfmon_pmu_event_update()
225 	delta = ((n - p) >> shift);  in perfmon_pmu_event_update()
227 	local64_add(delta, &event->count);  in perfmon_pmu_event_update()
236 	n_counters = min(idxd->idxd_pmu->n_counters, OVERFLOW_SIZE);  in perfmon_counter_overflow()
248 	while (ovfstatus && max_loop--) {  in perfmon_counter_overflow()
253 			/* Update event->count for overflowed counter */  in perfmon_counter_overflow()
254 			event = idxd->idxd_pmu->event_list[i];  in perfmon_counter_overflow()
256 			/* Writing 1 to OVFSTATUS bit clears it */  in perfmon_counter_overflow()
293 	struct hw_perf_event *hwc = &event->hw;  in perfmon_pmu_event_start()
301 	event->hw.idx = hwc->idx;  in perfmon_pmu_event_start()
302 	cntr = hwc->idx;  in perfmon_pmu_event_start()
305 	event_cfg.val = event->attr.config;  in perfmon_pmu_event_start()
306 	flt_cfg.val = event->attr.config1;  in perfmon_pmu_event_start()
315 	flt_eng = flt_cfg.eng;  in perfmon_pmu_event_start()
317 	if (flt_wq && test_bit(FLT_WQ, &idxd->idxd_pmu->supported_filters))  in perfmon_pmu_event_start()
319 	if (flt_tc && test_bit(FLT_TC, &idxd->idxd_pmu->supported_filters))  in perfmon_pmu_event_start()
321 	if (flt_pg_sz && test_bit(FLT_PG_SZ, &idxd->idxd_pmu->supported_filters))  in perfmon_pmu_event_start()
323 	if (flt_xfer_sz && test_bit(FLT_XFER_SZ, &idxd->idxd_pmu->supported_filters))  in perfmon_pmu_event_start()
325 	if (flt_eng && test_bit(FLT_ENG, &idxd->idxd_pmu->supported_filters))  in perfmon_pmu_event_start()
330 	local64_set(&event->hw.prev_count, cntrdata);  in perfmon_pmu_event_start()
343 	struct hw_perf_event *hwc = &event->hw;  in perfmon_pmu_event_stop()
345 	int i, cntr = hwc->idx;  in perfmon_pmu_event_stop()
351 	for (i = 0; i < idxd->idxd_pmu->n_events; i++) {  in perfmon_pmu_event_stop()
352 		if (event != idxd->idxd_pmu->event_list[i])  in perfmon_pmu_event_stop()
355 		for (++i; i < idxd->idxd_pmu->n_events; i++)  in perfmon_pmu_event_stop()
356 			idxd->idxd_pmu->event_list[i - 1] = idxd->idxd_pmu->event_list[i];  in perfmon_pmu_event_stop()
357 		--idxd->idxd_pmu->n_events;  in perfmon_pmu_event_stop()
368 	event->hw.idx = -1;  in perfmon_pmu_event_stop()
369 	clear_bit(cntr, idxd->idxd_pmu->used_mask);  in perfmon_pmu_event_stop()
380 	struct idxd_pmu *idxd_pmu = idxd->idxd_pmu;  in perfmon_pmu_event_add()
381 	struct hw_perf_event *hwc = &event->hw;  in perfmon_pmu_event_add()
388 	hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;  in perfmon_pmu_event_add()
390 		hwc->state |= PERF_HES_ARCH;  in perfmon_pmu_event_add()
401 	idxd_pmu->n_events = n;  in perfmon_pmu_event_add()
436 			perfmon_format_attrs[PERFMON_FILTERS_START + j + 1];  in skip_filter()
444 		if (!test_bit(i, &idxd_pmu->supported_filters))  in idxd_pmu_init()
448 	idxd_pmu->pmu.name		= idxd_pmu->name;  in idxd_pmu_init()
449 	idxd_pmu->pmu.attr_groups	= perfmon_attr_groups;  in idxd_pmu_init()
450 	idxd_pmu->pmu.task_ctx_nr	= perf_invalid_context;  in idxd_pmu_init()
451 	idxd_pmu->pmu.event_init	= perfmon_pmu_event_init;  in idxd_pmu_init()
452 	idxd_pmu->pmu.pmu_enable	= perfmon_pmu_enable;  in idxd_pmu_init()
453 	idxd_pmu->pmu.pmu_disable	= perfmon_pmu_disable;  in idxd_pmu_init()
454 	idxd_pmu->pmu.add		= perfmon_pmu_event_add;  in idxd_pmu_init()
455 	idxd_pmu->pmu.del		= perfmon_pmu_event_del;  in idxd_pmu_init()
456 	idxd_pmu->pmu.start		= perfmon_pmu_event_start;  in idxd_pmu_init()
457 	idxd_pmu->pmu.stop		= perfmon_pmu_event_stop;  in idxd_pmu_init()
458 	idxd_pmu->pmu.read		= perfmon_pmu_event_update;  in idxd_pmu_init()
459 	idxd_pmu->pmu.capabilities	= PERF_PMU_CAP_NO_EXCLUDE;  in idxd_pmu_init()
460 	idxd_pmu->pmu.scope		= PERF_PMU_SCOPE_SYS_WIDE;  in idxd_pmu_init()
461 	idxd_pmu->pmu.module		= THIS_MODULE;  in idxd_pmu_init()
466 	if (!idxd->idxd_pmu)  in perfmon_pmu_remove()
469 	perf_pmu_unregister(&idxd->idxd_pmu->pmu);  in perfmon_pmu_remove()
470 	kfree(idxd->idxd_pmu);  in perfmon_pmu_remove()
471 	idxd->idxd_pmu = NULL;  in perfmon_pmu_remove()
478 	int rc = -ENODEV;  in perfmon_pmu_init()
484 	if (idxd->perfmon_offset == 0)  in perfmon_pmu_init()
485 		return -ENODEV;  in perfmon_pmu_init()
489 		return -ENOMEM;  in perfmon_pmu_init()
491 	idxd_pmu->idxd = idxd;  in perfmon_pmu_init()
492 	idxd->idxd_pmu = idxd_pmu;  in perfmon_pmu_init()
494 	if (idxd->data->type == IDXD_TYPE_DSA) {  in perfmon_pmu_init()
495 		rc = sprintf(idxd_pmu->name, "dsa%d", idxd->id);  in perfmon_pmu_init()
498 	} else if (idxd->data->type == IDXD_TYPE_IAX) {  in perfmon_pmu_init()
499 		rc = sprintf(idxd_pmu->name, "iax%d", idxd->id);  in perfmon_pmu_init()
531 	 * We don't support per-counter capabilities for now.  in perfmon_pmu_init()
536 	idxd_pmu->n_event_categories = perfcap.num_event_category;  in perfmon_pmu_init()
537 	idxd_pmu->supported_event_categories = perfcap.global_event_category;  in perfmon_pmu_init()
538 	idxd_pmu->per_counter_caps_supported = perfcap.cap_per_counter;  in perfmon_pmu_init()
541 	idxd_pmu->supported_filters = perfcap.filter;  in perfmon_pmu_init()
543 		idxd_pmu->n_filters = hweight8(perfcap.filter);  in perfmon_pmu_init()
546 	idxd_pmu->n_counters = perfcap.num_perf_counter;  in perfmon_pmu_init()
547 	idxd_pmu->counter_width = perfcap.counter_width;  in perfmon_pmu_init()
551 	rc = perf_pmu_register(&idxd_pmu->pmu, idxd_pmu->name, -1);  in perfmon_pmu_init()
559 	idxd->idxd_pmu = NULL;  in perfmon_pmu_init()