1 // SPDX-License-Identifier: GPL-2.0
101 	{ .compatible = "fsl,imx8-ddr-pmu", .data = &imx8_devtype_data},
102 	{ .compatible = "fsl,imx8m-ddr-pmu", .data = &imx8m_devtype_data},
103 	{ .compatible = "fsl,imx8mq-ddr-pmu", .data = &imx8mq_devtype_data},
104 	{ .compatible = "fsl,imx8mm-ddr-pmu", .data = &imx8mm_devtype_data},
105 	{ .compatible = "fsl,imx8mn-ddr-pmu", .data = &imx8mn_devtype_data},
106 	{ .compatible = "fsl,imx8mp-ddr-pmu", .data = &imx8mp_devtype_data},
107 	{ .compatible = "fsl,imx8dxl-ddr-pmu", .data = &imx8dxl_devtype_data},
132 	return sysfs_emit(page, "%s\n", pmu->devtype_data->identifier);  in ddr_perf_identifier_show()
142 	if (!pmu->devtype_data->identifier)  in ddr_perf_identifier_attr_visible()
144 	return attr->mode;  in ddr_perf_identifier_attr_visible()
169 	u32 quirks = pmu->devtype_data->quirks;  in ddr_perf_filter_cap_get()
193 	int cap = (long)ea->var;  in ddr_perf_filter_cap_show()
223 	return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->cpu));  in ddr_perf_cpumask_show()
245 	return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);  in ddr_pmu_event_show()
254 	IMX8_DDR_PMU_EVENT_ATTR(read-accesses, 0x04),
255 	IMX8_DDR_PMU_EVENT_ATTR(write-accesses, 0x05),
256 	IMX8_DDR_PMU_EVENT_ATTR(read-queue-depth, 0x08),
257 	IMX8_DDR_PMU_EVENT_ATTR(write-queue-depth, 0x09),
258 	IMX8_DDR_PMU_EVENT_ATTR(lp-read-credit-cnt, 0x10),
259 	IMX8_DDR_PMU_EVENT_ATTR(hp-read-credit-cnt, 0x11),
260 	IMX8_DDR_PMU_EVENT_ATTR(write-credit-cnt, 0x12),
261 	IMX8_DDR_PMU_EVENT_ATTR(read-command, 0x20),
262 	IMX8_DDR_PMU_EVENT_ATTR(write-command, 0x21),
263 	IMX8_DDR_PMU_EVENT_ATTR(read-modify-write-command, 0x22),
264 	IMX8_DDR_PMU_EVENT_ATTR(hp-read, 0x23),
265 	IMX8_DDR_PMU_EVENT_ATTR(hp-req-nocredit, 0x24),
266 	IMX8_DDR_PMU_EVENT_ATTR(hp-xact-credit, 0x25),
267 	IMX8_DDR_PMU_EVENT_ATTR(lp-req-nocredit, 0x26),
268 	IMX8_DDR_PMU_EVENT_ATTR(lp-xact-credit, 0x27),
269 	IMX8_DDR_PMU_EVENT_ATTR(wr-xact-credit, 0x29),
270 	IMX8_DDR_PMU_EVENT_ATTR(read-cycles, 0x2a),
271 	IMX8_DDR_PMU_EVENT_ATTR(write-cycles, 0x2b),
272 	IMX8_DDR_PMU_EVENT_ATTR(read-write-transition, 0x30),
275 	IMX8_DDR_PMU_EVENT_ATTR(load-mode, 0x33),
276 	IMX8_DDR_PMU_EVENT_ATTR(perf-mwr, 0x34),
278 	IMX8_DDR_PMU_EVENT_ATTR(read-activate, 0x36),
281 	IMX8_DDR_PMU_EVENT_ATTR(raw-hazard, 0x39),
282 	IMX8_DDR_PMU_EVENT_ATTR(axid-read, 0x41),
283 	IMX8_DDR_PMU_EVENT_ATTR(axid-write, 0x42),
292 PMU_FORMAT_ATTR(event, "config:0-7");
293 PMU_FORMAT_ATTR(axi_id, "config1:0-15");
294 PMU_FORMAT_ATTR(axi_mask, "config1:16-31");
295 PMU_FORMAT_ATTR(axi_port, "config2:0-2");
296 PMU_FORMAT_ATTR(axi_channel, "config2:3-3");
323 	return event->attr.config == 0x41 || event->attr.config == 0x42;  in ddr_perf_is_filtered()
328 	return event->attr.config1;  in ddr_perf_filter_val()
344 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);  in ddr_perf_is_enhanced_filtered()
346 	filt = pmu->devtype_data->quirks & DDR_CAP_AXI_ID_FILTER_ENHANCED;  in ddr_perf_is_enhanced_filtered()
361 		if (pmu->events[EVENT_CYCLES_COUNTER] == NULL)  in ddr_perf_alloc_counter()
364 			return -ENOENT;  in ddr_perf_alloc_counter()
368 		if (pmu->events[i] == NULL)  in ddr_perf_alloc_counter()
372 	return -ENOENT;  in ddr_perf_alloc_counter()
377 	pmu->events[counter] = NULL;  in ddr_perf_free_counter()
382 	struct perf_event *event = pmu->events[counter];  in ddr_perf_read_counter()
383 	void __iomem *base = pmu->base;  in ddr_perf_read_counter()
387 	 * axid-read and axid-write event if PMU core supports enhanced  in ddr_perf_read_counter()
397 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);  in ddr_perf_event_init()
398 	struct hw_perf_event *hwc = &event->hw;  in ddr_perf_event_init()
401 	if (event->attr.type != event->pmu->type)  in ddr_perf_event_init()
402 		return -ENOENT;  in ddr_perf_event_init()
404 	if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)  in ddr_perf_event_init()
405 		return -EOPNOTSUPP;  in ddr_perf_event_init()
407 	if (event->cpu < 0) {  in ddr_perf_event_init()
408 		dev_warn(pmu->dev, "Can't provide per-task data!\n");  in ddr_perf_event_init()
409 		return -EOPNOTSUPP;  in ddr_perf_event_init()
415 	 * periodically read when a hrtimer aka cpu-clock leader triggers).  in ddr_perf_event_init()
417 	if (event->group_leader->pmu != event->pmu &&  in ddr_perf_event_init()
418 			!is_software_event(event->group_leader))  in ddr_perf_event_init()
419 		return -EINVAL;  in ddr_perf_event_init()
421 	if (pmu->devtype_data->quirks & DDR_CAP_AXI_ID_FILTER) {  in ddr_perf_event_init()
422 		if (!ddr_perf_filters_compatible(event, event->group_leader))  in ddr_perf_event_init()
423 			return -EINVAL;  in ddr_perf_event_init()
424 		for_each_sibling_event(sibling, event->group_leader) {  in ddr_perf_event_init()
426 				return -EINVAL;  in ddr_perf_event_init()
430 	for_each_sibling_event(sibling, event->group_leader) {  in ddr_perf_event_init()
431 		if (sibling->pmu != event->pmu &&  in ddr_perf_event_init()
433 			return -EINVAL;  in ddr_perf_event_init()
436 	event->cpu = pmu->cpu;  in ddr_perf_event_init()
437 	hwc->idx = -1;  in ddr_perf_event_init()
455 		writel(0, pmu->base + reg);  in ddr_perf_counter_enable()
464 		if (pmu->devtype_data->quirks & DDR_CAP_AXI_ID_FILTER_ENHANCED) {  in ddr_perf_counter_enable()
469 		writel(val, pmu->base + reg);  in ddr_perf_counter_enable()
472 		val = readl_relaxed(pmu->base + reg) & CNTL_EN_MASK;  in ddr_perf_counter_enable()
473 		writel(val, pmu->base + reg);  in ddr_perf_counter_enable()
481 	val = readl_relaxed(pmu->base + counter * 4 + COUNTER_CNTL);  in ddr_perf_counter_overflow()
491 	val = readl_relaxed(pmu->base + reg);  in ddr_perf_counter_clear()
493 	writel(val, pmu->base + reg);  in ddr_perf_counter_clear()
496 	writel(val, pmu->base + reg);  in ddr_perf_counter_clear()
501 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);  in ddr_perf_event_update()
502 	struct hw_perf_event *hwc = &event->hw;  in ddr_perf_event_update()
504 	int counter = hwc->idx;  in ddr_perf_event_update()
509 	if (pmu->devtype_data->quirks & DDR_CAP_AXI_ID_FILTER_ENHANCED) {  in ddr_perf_event_update()
514 	local64_add(new_raw_count, &event->count);  in ddr_perf_event_update()
525 			dev_warn_ratelimited(pmu->dev,  "events lost due to counter overflow (config 0x%llx)\n",  in ddr_perf_event_update()
526 					     event->attr.config);  in ddr_perf_event_update()
535 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);  in ddr_perf_event_start()
536 	struct hw_perf_event *hwc = &event->hw;  in ddr_perf_event_start()
537 	int counter = hwc->idx;  in ddr_perf_event_start()
539 	local64_set(&hwc->prev_count, 0);  in ddr_perf_event_start()
541 	ddr_perf_counter_enable(pmu, event->attr.config, counter, true);  in ddr_perf_event_start()
543 	if (!pmu->active_counter++)  in ddr_perf_event_start()
547 	hwc->state = 0;  in ddr_perf_event_start()
552 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);  in ddr_perf_event_add()
553 	struct hw_perf_event *hwc = &event->hw;  in ddr_perf_event_add()
555 	int cfg = event->attr.config;  in ddr_perf_event_add()
556 	int cfg1 = event->attr.config1;  in ddr_perf_event_add()
557 	int cfg2 = event->attr.config2;  in ddr_perf_event_add()
559 	if (pmu->devtype_data->quirks & DDR_CAP_AXI_ID_FILTER) {  in ddr_perf_event_add()
563 			if (pmu->events[i] &&  in ddr_perf_event_add()
564 			    !ddr_perf_filters_compatible(event, pmu->events[i]))  in ddr_perf_event_add()
565 				return -EINVAL;  in ddr_perf_event_add()
571 			writel(cfg1, pmu->base + COUNTER_DPCR1);  in ddr_perf_event_add()
577 		dev_dbg(pmu->dev, "There are not enough counters\n");  in ddr_perf_event_add()
578 		return -EOPNOTSUPP;  in ddr_perf_event_add()
581 	if (pmu->devtype_data->quirks & DDR_CAP_AXI_ID_PORT_CHANNEL_FILTER) {  in ddr_perf_event_add()
585 			writel(cfg1, pmu->base + COUNTER_MASK_COMP + ((counter - 1) << 4));  in ddr_perf_event_add()
597 			writel(cfg2, pmu->base + COUNTER_MUX_CNTL + ((counter - 1) << 4));  in ddr_perf_event_add()
601 	pmu->events[counter] = event;  in ddr_perf_event_add()
602 	hwc->idx = counter;  in ddr_perf_event_add()
604 	hwc->state |= PERF_HES_STOPPED;  in ddr_perf_event_add()
614 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);  in ddr_perf_event_stop()
615 	struct hw_perf_event *hwc = &event->hw;  in ddr_perf_event_stop()
616 	int counter = hwc->idx;  in ddr_perf_event_stop()
618 	ddr_perf_counter_enable(pmu, event->attr.config, counter, false);  in ddr_perf_event_stop()
621 	if (!--pmu->active_counter)  in ddr_perf_event_stop()
625 	hwc->state |= PERF_HES_STOPPED;  in ddr_perf_event_stop()
630 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);  in ddr_perf_event_del()
631 	struct hw_perf_event *hwc = &event->hw;  in ddr_perf_event_del()
632 	int counter = hwc->idx;  in ddr_perf_event_del()
637 	hwc->idx = -1;  in ddr_perf_event_del()
671 	pmu->id = ida_alloc(&ddr_ida, GFP_KERNEL);  in ddr_perf_init()
672 	return pmu->id;  in ddr_perf_init()
700 		if (!pmu->events[i])  in ddr_perf_irq_handler()
703 		event = pmu->events[i];  in ddr_perf_irq_handler()
721 	if (cpu != pmu->cpu)  in ddr_perf_offline_cpu()
728 	perf_pmu_migrate_context(&pmu->pmu, cpu, target);  in ddr_perf_offline_cpu()
729 	pmu->cpu = target;  in ddr_perf_offline_cpu()
731 	WARN_ON(irq_set_affinity(pmu->irq, cpumask_of(pmu->cpu)));  in ddr_perf_offline_cpu()
750 	np = pdev->dev.of_node;  in ddr_perf_probe()
752 	pmu = devm_kzalloc(&pdev->dev, sizeof(*pmu), GFP_KERNEL);  in ddr_perf_probe()
754 		return -ENOMEM;  in ddr_perf_probe()
756 	num = ddr_perf_init(pmu, base, &pdev->dev);  in ddr_perf_probe()
760 	name = devm_kasprintf(&pdev->dev, GFP_KERNEL, DDR_PERF_DEV_NAME "%d",  in ddr_perf_probe()
763 		ret = -ENOMEM;  in ddr_perf_probe()
767 	pmu->devtype_data = of_device_get_match_data(&pdev->dev);  in ddr_perf_probe()
769 	pmu->cpu = raw_smp_processor_id();  in ddr_perf_probe()
776 		dev_err(&pdev->dev, "cpuhp_setup_state_multi failed\n");  in ddr_perf_probe()
780 	pmu->cpuhp_state = ret;  in ddr_perf_probe()
783 	ret = cpuhp_state_add_instance_nocalls(pmu->cpuhp_state, &pmu->node);  in ddr_perf_probe()
785 		dev_err(&pdev->dev, "Error %d registering hotplug\n", ret);  in ddr_perf_probe()
792 		dev_err(&pdev->dev, "Failed to get irq: %d", irq);  in ddr_perf_probe()
797 	ret = devm_request_irq(&pdev->dev, irq,  in ddr_perf_probe()
803 		dev_err(&pdev->dev, "Request irq failed: %d", ret);  in ddr_perf_probe()
807 	pmu->irq = irq;  in ddr_perf_probe()
808 	ret = irq_set_affinity(pmu->irq, cpumask_of(pmu->cpu));  in ddr_perf_probe()
810 		dev_err(pmu->dev, "Failed to set interrupt affinity!\n");  in ddr_perf_probe()
814 	ret = perf_pmu_register(&pmu->pmu, name, -1);  in ddr_perf_probe()
821 	cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node);  in ddr_perf_probe()
823 	cpuhp_remove_multi_state(pmu->cpuhp_state);  in ddr_perf_probe()
825 	ida_free(&ddr_ida, pmu->id);  in ddr_perf_probe()
826 	dev_warn(&pdev->dev, "i.MX8 DDR Perf PMU failed (%d), disabled\n", ret);  in ddr_perf_probe()
834 	cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node);  in ddr_perf_remove()
835 	cpuhp_remove_multi_state(pmu->cpuhp_state);  in ddr_perf_remove()
837 	perf_pmu_unregister(&pmu->pmu);  in ddr_perf_remove()
839 	ida_free(&ddr_ida, pmu->id);  in ddr_perf_remove()
844 		.name   = "imx-ddr-pmu",