1 // SPDX-License-Identifier: GPL-2.0-only
3  * Perf support for the Statistical Profiling Extension, introduced as
43  * Cache if the event is allowed to trace Context information.
54 		event->hw.flags |= SPE_PMU_HW_FLAGS_CX;  in set_spe_event_has_cx()
59 	return !!(event->hw.flags & SPE_PMU_HW_FLAGS_CX);  in get_spe_event_has_cx()
101 /* Convert a free-running index from perf into an SPE buffer offset */
103 	((idx) % ((unsigned long)(buf)->nr_pages << PAGE_SHIFT))
132 		return !!(spe_pmu->features & arm_spe_pmu_feat_caps[cap]);  in arm_spe_pmu_cap_get()
136 		return spe_pmu->counter_sz;  in arm_spe_pmu_cap_get()
138 		return spe_pmu->min_period;  in arm_spe_pmu_cap_get()
140 		return ~spe_pmu->pmsevfr_res0;  in arm_spe_pmu_cap_get()
155 	int cap = (long)ea->var;  in arm_spe_pmu_cap_show()
167 	int cap = (long)ea->var;  in arm_spe_pmu_cap_show_hex()
303 	if (attr == &format_attr_discard.attr && !(spe_pmu->features & SPE_PMU_FEAT_DISCARD))  in arm_spe_pmu_format_attr_is_visible()
306 	if (attr == &format_attr_inv_event_filter.attr && !(spe_pmu->features & SPE_PMU_FEAT_INV_FILT_EVT))  in arm_spe_pmu_format_attr_is_visible()
316 	     !(spe_pmu->features & SPE_PMU_FEAT_EFT))  in arm_spe_pmu_format_attr_is_visible()
319 	return attr->mode;  in arm_spe_pmu_format_attr_is_visible()
333 	return cpumap_print_to_pagebuf(true, buf, &spe_pmu->supported_cpus);  in cpumask_show()
356 	struct perf_event_attr *attr = &event->attr;  in arm_spe_event_to_pmscr()
363 	if (!attr->exclude_user)  in arm_spe_event_to_pmscr()
366 	if (!attr->exclude_kernel)  in arm_spe_event_to_pmscr()
377 	u64 period = event->hw.sample_period;  in arm_spe_event_sanitise_period()
381 	 * The PMSIDR_EL1.Interval field (stored in spe_pmu->min_period) is a  in arm_spe_event_sanitise_period()
392 	event->hw.sample_period = period;  in arm_spe_event_sanitise_period()
397 	struct perf_event_attr *attr = &event->attr;  in arm_spe_event_to_pmsirr()
403 	reg |= event->hw.sample_period;  in arm_spe_event_to_pmsirr()
410 	struct perf_event_attr *attr = &event->attr;  in arm_spe_event_to_pmsfcr()
441 	struct perf_event_attr *attr = &event->attr;  in arm_spe_event_to_pmsevfr()
447 	struct perf_event_attr *attr = &event->attr;  in arm_spe_event_to_pmsnevfr()
453 	struct perf_event_attr *attr = &event->attr;  in arm_spe_event_to_pmslatfr()
460 	u64 head = PERF_IDX2OFF(handle->head, buf);  in arm_spe_pmu_pad_buf()
462 	memset(buf->base + head, ARM_SPE_BUF_PAD_BYTE, len);  in arm_spe_pmu_pad_buf()
463 	if (!buf->snapshot)  in arm_spe_pmu_pad_buf()
470 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu);  in arm_spe_pmu_next_snapshot_off()
471 	u64 head = PERF_IDX2OFF(handle->head, buf);  in arm_spe_pmu_next_snapshot_off()
472 	u64 limit = buf->nr_pages * PAGE_SIZE;  in arm_spe_pmu_next_snapshot_off()
475 	 * The trace format isn't parseable in reverse, so clamp  in arm_spe_pmu_next_snapshot_off()
476 	 * the limit to half of the buffer size in snapshot mode  in arm_spe_pmu_next_snapshot_off()
477 	 * so that the worst case is half a buffer of records, as  in arm_spe_pmu_next_snapshot_off()
487 	if (limit - head < spe_pmu->max_record_sz) {  in arm_spe_pmu_next_snapshot_off()
488 		arm_spe_pmu_pad_buf(handle, limit - head);  in arm_spe_pmu_next_snapshot_off()
489 		handle->head = PERF_IDX2OFF(limit, buf);  in arm_spe_pmu_next_snapshot_off()
490 		limit = ((buf->nr_pages * PAGE_SIZE) >> 1) + handle->head;  in arm_spe_pmu_next_snapshot_off()
498 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu);  in __arm_spe_pmu_next_off()
500 	const u64 bufsize = buf->nr_pages * PAGE_SIZE;  in __arm_spe_pmu_next_off()
508 	 *    a record when generating a buffer management event.  in __arm_spe_pmu_next_off()
510 	 * 2. We used perf_aux_output_skip to consume handle->size bytes  in __arm_spe_pmu_next_off()
515 	 * moving the head index. If we run out of buffer space, we'll  in __arm_spe_pmu_next_off()
516 	 * reduce handle->size to zero and end up reporting truncation.  in __arm_spe_pmu_next_off()
518 	head = PERF_IDX2OFF(handle->head, buf);  in __arm_spe_pmu_next_off()
519 	if (!IS_ALIGNED(head, spe_pmu->align)) {  in __arm_spe_pmu_next_off()
520 		unsigned long delta = roundup(head, spe_pmu->align) - head;  in __arm_spe_pmu_next_off()
522 		delta = min(delta, handle->size);  in __arm_spe_pmu_next_off()
524 		head = PERF_IDX2OFF(handle->head, buf);  in __arm_spe_pmu_next_off()
528 	if (!handle->size)  in __arm_spe_pmu_next_off()
532 	tail = PERF_IDX2OFF(handle->head + handle->size, buf);  in __arm_spe_pmu_next_off()
533 	wakeup = PERF_IDX2OFF(handle->wakeup, buf);  in __arm_spe_pmu_next_off()
537 	 * if we see head == tail we know that the buffer is empty. If  in __arm_spe_pmu_next_off()
553 	if (handle->wakeup < (handle->head + handle->size) && head <= wakeup)  in __arm_spe_pmu_next_off()
559 	arm_spe_pmu_pad_buf(handle, handle->size);  in __arm_spe_pmu_next_off()
569 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu);  in arm_spe_pmu_next_off()
571 	u64 head = PERF_IDX2OFF(handle->head, buf);  in arm_spe_pmu_next_off()
574 	 * If the head has come too close to the end of the buffer,  in arm_spe_pmu_next_off()
577 	if (limit && (limit - head < spe_pmu->max_record_sz)) {  in arm_spe_pmu_next_off()
578 		arm_spe_pmu_pad_buf(handle, limit - head);  in arm_spe_pmu_next_off()
591 	if (ATTR_CFG_GET_FLD(&event->attr, discard)) {  in arm_spe_perf_aux_output_begin()
600 		event->hw.state |= PERF_HES_STOPPED;  in arm_spe_perf_aux_output_begin()
609 	limit = buf->snapshot ? arm_spe_pmu_next_snapshot_off(handle)  in arm_spe_perf_aux_output_begin()
614 	limit += (u64)buf->base;  in arm_spe_perf_aux_output_begin()
615 	base = (u64)buf->base + PERF_IDX2OFF(handle->head, buf);  in arm_spe_perf_aux_output_begin()
627 	offset = read_sysreg_s(SYS_PMBPTR_EL1) - (u64)buf->base;  in arm_spe_perf_aux_output_end()
628 	size = offset - PERF_IDX2OFF(handle->head, buf);  in arm_spe_perf_aux_output_end()
630 	if (buf->snapshot)  in arm_spe_perf_aux_output_end()
631 		handle->head = offset;  in arm_spe_perf_aux_output_end()
646 	/* Disable the profiling buffer */  in arm_spe_pmu_disable_and_drain_local()
686 	/* We only expect buffer management events */  in arm_spe_pmu_buf_get_fault_act()
693 		err_str = "Unexpected buffer fault";  in arm_spe_pmu_buf_get_fault_act()
700 	/* Buffer management event */  in arm_spe_pmu_buf_get_fault_act()
706 		err_str = "Unknown buffer status code";  in arm_spe_pmu_buf_get_fault_act()
724 	struct perf_event *event = handle->event;  in arm_spe_pmu_irq_handler()
736 	 * profiling buffer in response to a TRUNCATION flag.  in arm_spe_pmu_irq_handler()
744 		 * buffer enabled is a recipe waiting to happen. Since  in arm_spe_pmu_irq_handler()
746 		 * that the profiling buffer is disabled explicitly before  in arm_spe_pmu_irq_handler()
753 		 * We handled the fault (the buffer was full), so resume  in arm_spe_pmu_irq_handler()
758 		if (!(handle->aux_flags & PERF_AUX_FLAG_TRUNCATED)) {  in arm_spe_pmu_irq_handler()
768 	/* The buffer pointers are now sane, so resume profiling. */  in arm_spe_pmu_irq_handler()
777 	struct perf_event_attr *attr = &event->attr;  in arm_spe_pmu_event_init()
778 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);  in arm_spe_pmu_event_init()
780 	/* This is, of course, deeply driver-specific */  in arm_spe_pmu_event_init()
781 	if (attr->type != event->pmu->type)  in arm_spe_pmu_event_init()
782 		return -ENOENT;  in arm_spe_pmu_event_init()
784 	if (event->cpu >= 0 &&  in arm_spe_pmu_event_init()
785 	    !cpumask_test_cpu(event->cpu, &spe_pmu->supported_cpus))  in arm_spe_pmu_event_init()
786 		return -ENOENT;  in arm_spe_pmu_event_init()
788 	if (arm_spe_event_to_pmsevfr(event) & spe_pmu->pmsevfr_res0)  in arm_spe_pmu_event_init()
789 		return -EOPNOTSUPP;  in arm_spe_pmu_event_init()
791 	if (arm_spe_event_to_pmsnevfr(event) & spe_pmu->pmsevfr_res0)  in arm_spe_pmu_event_init()
792 		return -EOPNOTSUPP;  in arm_spe_pmu_event_init()
794 	if (attr->exclude_idle)  in arm_spe_pmu_event_init()
795 		return -EOPNOTSUPP;  in arm_spe_pmu_event_init()
798 	 * Feedback-directed frequency throttling doesn't work when we  in arm_spe_pmu_event_init()
799 	 * have a buffer of samples. We'd need to manually count the  in arm_spe_pmu_event_init()
800 	 * samples in the buffer when it fills up and adjust the event  in arm_spe_pmu_event_init()
804 	if (attr->freq)  in arm_spe_pmu_event_init()
805 		return -EINVAL;  in arm_spe_pmu_event_init()
809 	    !(spe_pmu->features & SPE_PMU_FEAT_FILT_EVT))  in arm_spe_pmu_event_init()
810 		return -EOPNOTSUPP;  in arm_spe_pmu_event_init()
813 	    !(spe_pmu->features & SPE_PMU_FEAT_INV_FILT_EVT))  in arm_spe_pmu_event_init()
814 		return -EOPNOTSUPP;  in arm_spe_pmu_event_init()
817 	    !(spe_pmu->features & SPE_PMU_FEAT_FILT_TYP))  in arm_spe_pmu_event_init()
818 		return -EOPNOTSUPP;  in arm_spe_pmu_event_init()
821 	    !(spe_pmu->features & SPE_PMU_FEAT_FILT_LAT))  in arm_spe_pmu_event_init()
822 		return -EOPNOTSUPP;  in arm_spe_pmu_event_init()
831 	    !(spe_pmu->features & SPE_PMU_FEAT_EFT))  in arm_spe_pmu_event_init()
832 		return -EOPNOTSUPP;  in arm_spe_pmu_event_init()
834 	if (ATTR_CFG_GET_FLD(&event->attr, discard) &&  in arm_spe_pmu_event_init()
835 	    !(spe_pmu->features & SPE_PMU_FEAT_DISCARD))  in arm_spe_pmu_event_init()
836 		return -EOPNOTSUPP;  in arm_spe_pmu_event_init()
849 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);  in arm_spe_pmu_start()
850 	struct hw_perf_event *hwc = &event->hw;  in arm_spe_pmu_start()
851 	struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle);  in arm_spe_pmu_start()
853 	hwc->state = 0;  in arm_spe_pmu_start()
855 	if (hwc->state)  in arm_spe_pmu_start()
864 	if (spe_pmu->features & SPE_PMU_FEAT_INV_FILT_EVT) {  in arm_spe_pmu_start()
876 		reg = local64_read(&hwc->period_left);  in arm_spe_pmu_start()
887 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);  in arm_spe_pmu_stop()
888 	struct hw_perf_event *hwc = &event->hw;  in arm_spe_pmu_stop()
889 	struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle);  in arm_spe_pmu_stop()
892 	if (hwc->state & PERF_HES_STOPPED)  in arm_spe_pmu_stop()
895 	/* Stop all trace generation */  in arm_spe_pmu_stop()
901 		 * to this buffer, since we might be on the context-switch  in arm_spe_pmu_stop()
919 		local64_set(&hwc->period_left, read_sysreg_s(SYS_PMSICR_EL1));  in arm_spe_pmu_stop()
920 		hwc->state |= PERF_HES_UPTODATE;  in arm_spe_pmu_stop()
923 	hwc->state |= PERF_HES_STOPPED;  in arm_spe_pmu_stop()
929 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);  in arm_spe_pmu_add()
930 	struct hw_perf_event *hwc = &event->hw;  in arm_spe_pmu_add()
931 	int cpu = event->cpu == -1 ? smp_processor_id() : event->cpu;  in arm_spe_pmu_add()
933 	if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus))  in arm_spe_pmu_add()
934 		return -ENOENT;  in arm_spe_pmu_add()
936 	hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;  in arm_spe_pmu_add()
940 		if (hwc->state & PERF_HES_STOPPED)  in arm_spe_pmu_add()
941 			ret = -EINVAL;  in arm_spe_pmu_add()
959 	int i, cpu = event->cpu;  in arm_spe_pmu_setup_aux()
969 	 * we can effectively treat the buffer as consisting of two equal  in arm_spe_pmu_setup_aux()
976 	if (cpu == -1)  in arm_spe_pmu_setup_aux()
990 	buf->base = vmap(pglist, nr_pages, VM_MAP, PAGE_KERNEL);  in arm_spe_pmu_setup_aux()
991 	if (!buf->base)  in arm_spe_pmu_setup_aux()
994 	buf->nr_pages	= nr_pages;  in arm_spe_pmu_setup_aux()
995 	buf->snapshot	= snapshot;  in arm_spe_pmu_setup_aux()
1011 	vunmap(buf->base);  in arm_spe_pmu_free_aux()
1018 	static atomic_t pmu_idx = ATOMIC_INIT(-1);  in arm_spe_pmu_perf_init()
1022 	struct device *dev = &spe_pmu->pdev->dev;  in arm_spe_pmu_perf_init()
1024 	spe_pmu->pmu = (struct pmu) {  in arm_spe_pmu_perf_init()
1026 		.parent		= &spe_pmu->pdev->dev,  in arm_spe_pmu_perf_init()
1031 		 * we can support per-task profiling (which is not possible  in arm_spe_pmu_perf_init()
1034 		 * perf_event_open, since the aux buffer is not setup until  in arm_spe_pmu_perf_init()
1037 		 * once the buffer has been created.  in arm_spe_pmu_perf_init()
1054 		return -ENOMEM;  in arm_spe_pmu_perf_init()
1057 	return perf_pmu_register(&spe_pmu->pmu, name, -1);  in arm_spe_pmu_perf_init()
1062 	perf_pmu_unregister(&spe_pmu->pmu);  in arm_spe_pmu_perf_destroy()
1070 	struct device *dev = &spe_pmu->pdev->dev;  in __arm_spe_pmu_dev_probe()
1080 	spe_pmu->pmsver = (u16)fld;  in __arm_spe_pmu_dev_probe()
1086 			"profiling buffer owned by higher exception level\n");  in __arm_spe_pmu_dev_probe()
1090 	/* Minimum alignment. If it's out-of-range, then fail the probe */  in __arm_spe_pmu_dev_probe()
1092 	spe_pmu->align = 1 << fld;  in __arm_spe_pmu_dev_probe()
1093 	if (spe_pmu->align > SZ_2K) {  in __arm_spe_pmu_dev_probe()
1102 		spe_pmu->features |= SPE_PMU_FEAT_FILT_EVT;  in __arm_spe_pmu_dev_probe()
1105 		spe_pmu->features |= SPE_PMU_FEAT_INV_FILT_EVT;  in __arm_spe_pmu_dev_probe()
1108 		spe_pmu->features |= SPE_PMU_FEAT_FILT_TYP;  in __arm_spe_pmu_dev_probe()
1111 		spe_pmu->features |= SPE_PMU_FEAT_FILT_LAT;  in __arm_spe_pmu_dev_probe()
1114 		spe_pmu->features |= SPE_PMU_FEAT_ARCH_INST;  in __arm_spe_pmu_dev_probe()
1117 		spe_pmu->features |= SPE_PMU_FEAT_LDS;  in __arm_spe_pmu_dev_probe()
1120 		spe_pmu->features |= SPE_PMU_FEAT_ERND;  in __arm_spe_pmu_dev_probe()
1122 	if (spe_pmu->pmsver >= ID_AA64DFR0_EL1_PMSVer_V1P2)  in __arm_spe_pmu_dev_probe()
1123 		spe_pmu->features |= SPE_PMU_FEAT_DISCARD;  in __arm_spe_pmu_dev_probe()
1126 		spe_pmu->features |= SPE_PMU_FEAT_EFT;  in __arm_spe_pmu_dev_probe()
1128 	/* This field has a spaced out encoding, so just use a look-up */  in __arm_spe_pmu_dev_probe()
1132 		spe_pmu->min_period = 256;  in __arm_spe_pmu_dev_probe()
1135 		spe_pmu->min_period = 512;  in __arm_spe_pmu_dev_probe()
1138 		spe_pmu->min_period = 768;  in __arm_spe_pmu_dev_probe()
1141 		spe_pmu->min_period = 1024;  in __arm_spe_pmu_dev_probe()
1144 		spe_pmu->min_period = 1536;  in __arm_spe_pmu_dev_probe()
1147 		spe_pmu->min_period = 2048;  in __arm_spe_pmu_dev_probe()
1150 		spe_pmu->min_period = 3072;  in __arm_spe_pmu_dev_probe()
1157 		spe_pmu->min_period = 4096;  in __arm_spe_pmu_dev_probe()
1160 	/* Maximum record size. If it's out-of-range, then fail the probe */  in __arm_spe_pmu_dev_probe()
1162 	spe_pmu->max_record_sz = 1 << fld;  in __arm_spe_pmu_dev_probe()
1163 	if (spe_pmu->max_record_sz > SZ_2K || spe_pmu->max_record_sz < 16) {  in __arm_spe_pmu_dev_probe()
1176 		spe_pmu->counter_sz = 12;  in __arm_spe_pmu_dev_probe()
1179 		spe_pmu->counter_sz = 16;  in __arm_spe_pmu_dev_probe()
1184 	spe_pmu->pmsevfr_res0 = ~read_sysreg_s(SYS_PMSEVFR_EL1);  in __arm_spe_pmu_dev_probe()
1188 		 spe_pmu->pmsver - 1, cpumask_pr_args(&spe_pmu->supported_cpus),  in __arm_spe_pmu_dev_probe()
1189 		 spe_pmu->max_record_sz, spe_pmu->align, spe_pmu->features);  in __arm_spe_pmu_dev_probe()
1191 	spe_pmu->features |= SPE_PMU_FEAT_DEV_PROBED;  in __arm_spe_pmu_dev_probe()
1202 	/* Reset the buffer base pointer */  in __arm_spe_pmu_reset_local()
1216 	enable_percpu_irq(spe_pmu->irq, IRQ_TYPE_NONE);  in __arm_spe_pmu_setup_one()
1223 	disable_percpu_irq(spe_pmu->irq);  in __arm_spe_pmu_stop_one()
1232 	if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus))  in arm_spe_pmu_cpu_startup()
1244 	if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus))  in arm_spe_pmu_cpu_teardown()
1254 	cpumask_t *mask = &spe_pmu->supported_cpus;  in arm_spe_pmu_dev_init()
1258 	if (ret || !(spe_pmu->features & SPE_PMU_FEAT_DEV_PROBED))  in arm_spe_pmu_dev_init()
1259 		return -ENXIO;  in arm_spe_pmu_dev_init()
1262 	ret = request_percpu_irq(spe_pmu->irq, arm_spe_pmu_irq_handler, DRVNAME,  in arm_spe_pmu_dev_init()
1263 				 spe_pmu->handle);  in arm_spe_pmu_dev_init()
1273 				       &spe_pmu->hotplug_node);  in arm_spe_pmu_dev_init()
1275 		free_percpu_irq(spe_pmu->irq, spe_pmu->handle);  in arm_spe_pmu_dev_init()
1282 	cpuhp_state_remove_instance(arm_spe_pmu_online, &spe_pmu->hotplug_node);  in arm_spe_pmu_dev_teardown()
1283 	free_percpu_irq(spe_pmu->irq, spe_pmu->handle);  in arm_spe_pmu_dev_teardown()
1289 	struct platform_device *pdev = spe_pmu->pdev;  in arm_spe_pmu_irq_probe()
1293 		return -ENXIO;  in arm_spe_pmu_irq_probe()
1296 		dev_err(&pdev->dev, "expected PPI but got SPI (%d)\n", irq);  in arm_spe_pmu_irq_probe()
1297 		return -EINVAL;  in arm_spe_pmu_irq_probe()
1300 	if (irq_get_percpu_devid_partition(irq, &spe_pmu->supported_cpus)) {  in arm_spe_pmu_irq_probe()
1301 		dev_err(&pdev->dev, "failed to get PPI partition (%d)\n", irq);  in arm_spe_pmu_irq_probe()
1302 		return -EINVAL;  in arm_spe_pmu_irq_probe()
1305 	spe_pmu->irq = irq;  in arm_spe_pmu_irq_probe()
1310 	{ .compatible = "arm,statistical-profiling-extension-v1", .data = (void *)1 },
1325 	struct device *dev = &pdev->dev;  in arm_spe_pmu_device_probe()
1329 	 * buffer will fault and prematurely terminate the AUX session.  in arm_spe_pmu_device_probe()
1332 …dev_warn_once(dev, "profiling buffer inaccessible. Try passing \"kpti=off\" on the kernel command …  in arm_spe_pmu_device_probe()
1333 		return -EPERM;  in arm_spe_pmu_device_probe()
1338 		return -ENOMEM;  in arm_spe_pmu_device_probe()
1340 	spe_pmu->handle = alloc_percpu(typeof(*spe_pmu->handle));  in arm_spe_pmu_device_probe()
1341 	if (!spe_pmu->handle)  in arm_spe_pmu_device_probe()
1342 		return -ENOMEM;  in arm_spe_pmu_device_probe()
1344 	spe_pmu->pdev = pdev;  in arm_spe_pmu_device_probe()
1364 	free_percpu(spe_pmu->handle);  in arm_spe_pmu_device_probe()
1374 	free_percpu(spe_pmu->handle);  in arm_spe_pmu_device_remove()
1415 MODULE_DESCRIPTION("Perf driver for the ARMv8.2 Statistical Profiling Extension");