1 // SPDX-License-Identifier: GPL-2.0
42  * 32bit counters monitor counter-specific events in addition to counting reference events
59 #define to_ddr_pmu(p)		container_of(p, struct ddr_pmu, pmu)
74  * respecitively to counter 2-5.
79 	const char *identifier;		/* system PMU identifier for userspace */
84 	struct pmu pmu;  member
117 static inline bool axi_filter_v1(struct ddr_pmu *pmu)  in axi_filter_v1()  argument
119 	return pmu->devtype_data->filter_ver == DDR_PERF_AXI_FILTER_V1;  in axi_filter_v1()
122 static inline bool axi_filter_v2(struct ddr_pmu *pmu)  in axi_filter_v2()  argument
124 	return pmu->devtype_data->filter_ver == DDR_PERF_AXI_FILTER_V2;  in axi_filter_v2()
128 	{ .compatible = "fsl,imx91-ddr-pmu", .data = &imx91_devtype_data },
129 	{ .compatible = "fsl,imx93-ddr-pmu", .data = &imx93_devtype_data },
130 	{ .compatible = "fsl,imx94-ddr-pmu", .data = &imx94_devtype_data },
131 	{ .compatible = "fsl,imx95-ddr-pmu", .data = &imx95_devtype_data },
140 	struct ddr_pmu *pmu = dev_get_drvdata(dev);  in ddr_perf_identifier_show()  local
142 	return sysfs_emit(page, "%s\n", pmu->devtype_data->identifier);  in ddr_perf_identifier_show()
160 	struct ddr_pmu *pmu = dev_get_drvdata(dev);  in ddr_perf_cpumask_show()  local
162 	return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->cpu));  in ddr_perf_cpumask_show()
189 	return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);  in ddr_pmu_event_show()
323 	struct pmu *pmu = dev_get_drvdata(kobj_to_dev(kobj));  in ddr_perf_events_attrs_is_visible()  local
324 	struct ddr_pmu *ddr_pmu = to_ddr_pmu(pmu);  in ddr_perf_events_attrs_is_visible()
329 	if (!eattr->devtype_data)  in ddr_perf_events_attrs_is_visible()
330 		return attr->mode;  in ddr_perf_events_attrs_is_visible()
332 	if (eattr->devtype_data != ddr_pmu->devtype_data &&  in ddr_perf_events_attrs_is_visible()
333 	    eattr->devtype_data->filter_ver != ddr_pmu->devtype_data->filter_ver)  in ddr_perf_events_attrs_is_visible()
336 	return attr->mode;  in ddr_perf_events_attrs_is_visible()
345 PMU_FORMAT_ATTR(event, "config:0-7,16-23");
346 PMU_FORMAT_ATTR(counter, "config:8-15");
347 PMU_FORMAT_ATTR(axi_id, "config1:0-17");
348 PMU_FORMAT_ATTR(axi_mask, "config2:0-17");
371 static void ddr_perf_clear_counter(struct ddr_pmu *pmu, int counter)  in ddr_perf_clear_counter()  argument
374 		writel(0, pmu->base + PMC(counter) + 0x4);  in ddr_perf_clear_counter()
375 		writel(0, pmu->base + PMC(counter));  in ddr_perf_clear_counter()
377 		writel(0, pmu->base + PMC(counter));  in ddr_perf_clear_counter()
381 static u64 ddr_perf_read_counter(struct ddr_pmu *pmu, int counter)  in ddr_perf_read_counter()  argument
387 		val = readl_relaxed(pmu->base + PMC(counter));  in ddr_perf_read_counter()
393 		val_upper = readl_relaxed(pmu->base + PMC(counter) + 0x4);  in ddr_perf_read_counter()
394 		val_lower = readl_relaxed(pmu->base + PMC(counter));  in ddr_perf_read_counter()
395 	} while (val_upper != readl_relaxed(pmu->base + PMC(counter) + 0x4));  in ddr_perf_read_counter()
404 static void ddr_perf_counter_global_config(struct ddr_pmu *pmu, bool enable)  in ddr_perf_counter_global_config()  argument
408 	ctrl = readl_relaxed(pmu->base + PMGC0);  in ddr_perf_counter_global_config()
422 		writel(ctrl, pmu->base + PMGC0);  in ddr_perf_counter_global_config()
430 		writel(ctrl, pmu->base + PMGC0);  in ddr_perf_counter_global_config()
434 		writel(ctrl, pmu->base + PMGC0);  in ddr_perf_counter_global_config()
438 static void ddr_perf_counter_local_config(struct ddr_pmu *pmu, int config,  in ddr_perf_counter_local_config()  argument
444 	ctrl_a = readl_relaxed(pmu->base + PMLCA(counter));  in ddr_perf_counter_local_config()
449 		writel(ctrl_a, pmu->base + PMLCA(counter));  in ddr_perf_counter_local_config()
451 		ddr_perf_clear_counter(pmu, counter);  in ddr_perf_counter_local_config()
458 		writel(ctrl_a, pmu->base + PMLCA(counter));  in ddr_perf_counter_local_config()
462 		writel(ctrl_a, pmu->base + PMLCA(counter));  in ddr_perf_counter_local_config()
466 static void imx93_ddr_perf_monitor_config(struct ddr_pmu *pmu, int event,  in imx93_ddr_perf_monitor_config()  argument
476 	pmcfg1 = readl_relaxed(pmu->base + PMCFG1);  in imx93_ddr_perf_monitor_config()
479 		pmcfg1 = event == 73 ? pmcfg1 | mask[counter - 2] :  in imx93_ddr_perf_monitor_config()
480 				pmcfg1 & ~mask[counter - 2];  in imx93_ddr_perf_monitor_config()
484 	writel_relaxed(pmcfg1, pmu->base + PMCFG1);  in imx93_ddr_perf_monitor_config()
486 	pmcfg2 = readl_relaxed(pmu->base + PMCFG2);  in imx93_ddr_perf_monitor_config()
489 	writel_relaxed(pmcfg2, pmu->base + PMCFG2);  in imx93_ddr_perf_monitor_config()
492 static void imx95_ddr_perf_monitor_config(struct ddr_pmu *pmu, int event,  in imx95_ddr_perf_monitor_config()  argument
497 	pmcfg1 = readl_relaxed(pmu->base + PMCFG1);  in imx95_ddr_perf_monitor_config()
531 	writel_relaxed(pmcfg1, pmu->base + PMCFG1);  in imx95_ddr_perf_monitor_config()
534 		pmcfg = readl_relaxed(pmu->base + offset);  in imx95_ddr_perf_monitor_config()
539 		writel_relaxed(pmcfg, pmu->base + offset);  in imx95_ddr_perf_monitor_config()
545 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);  in ddr_perf_event_update()  local
546 	struct hw_perf_event *hwc = &event->hw;  in ddr_perf_event_update()
547 	int counter = hwc->idx;  in ddr_perf_event_update()
550 	new_raw_count = ddr_perf_read_counter(pmu, counter);  in ddr_perf_event_update()
551 	local64_add(new_raw_count, &event->count);  in ddr_perf_event_update()
554 	ddr_perf_clear_counter(pmu, counter);  in ddr_perf_event_update()
559 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);  in ddr_perf_event_init()  local
560 	struct hw_perf_event *hwc = &event->hw;  in ddr_perf_event_init()
563 	if (event->attr.type != event->pmu->type)  in ddr_perf_event_init()
564 		return -ENOENT;  in ddr_perf_event_init()
566 	if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)  in ddr_perf_event_init()
567 		return -EOPNOTSUPP;  in ddr_perf_event_init()
569 	if (event->cpu < 0) {  in ddr_perf_event_init()
570 		dev_warn(pmu->dev, "Can't provide per-task data!\n");  in ddr_perf_event_init()
571 		return -EOPNOTSUPP;  in ddr_perf_event_init()
577 	 * periodically read when a hrtimer aka cpu-clock leader triggers).  in ddr_perf_event_init()
579 	if (event->group_leader->pmu != event->pmu &&  in ddr_perf_event_init()
580 			!is_software_event(event->group_leader))  in ddr_perf_event_init()
581 		return -EINVAL;  in ddr_perf_event_init()
583 	for_each_sibling_event(sibling, event->group_leader) {  in ddr_perf_event_init()
584 		if (sibling->pmu != event->pmu &&  in ddr_perf_event_init()
586 			return -EINVAL;  in ddr_perf_event_init()
589 	event->cpu = pmu->cpu;  in ddr_perf_event_init()
590 	hwc->idx = -1;  in ddr_perf_event_init()
597 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);  in ddr_perf_event_start()  local
598 	struct hw_perf_event *hwc = &event->hw;  in ddr_perf_event_start()
599 	int counter = hwc->idx;  in ddr_perf_event_start()
601 	local64_set(&hwc->prev_count, 0);  in ddr_perf_event_start()
603 	ddr_perf_counter_local_config(pmu, event->attr.config, counter, true);  in ddr_perf_event_start()
604 	hwc->state = 0;  in ddr_perf_event_start()
607 static int ddr_perf_alloc_counter(struct ddr_pmu *pmu, int event, int counter)  in ddr_perf_alloc_counter()  argument
613 		if (pmu->events[CYCLES_COUNTER] == NULL)  in ddr_perf_alloc_counter()
617 		if (pmu->events[counter] == NULL)  in ddr_perf_alloc_counter()
622 			if (pmu->events[i] == NULL)  in ddr_perf_alloc_counter()
626 	return -ENOENT;  in ddr_perf_alloc_counter()
631 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);  in ddr_perf_event_add()  local
632 	struct hw_perf_event *hwc = &event->hw;  in ddr_perf_event_add()
633 	int cfg = event->attr.config;  in ddr_perf_event_add()
634 	int cfg1 = event->attr.config1;  in ddr_perf_event_add()
635 	int cfg2 = event->attr.config2;  in ddr_perf_event_add()
641 	counter = ddr_perf_alloc_counter(pmu, event_id, counter);  in ddr_perf_event_add()
643 		dev_dbg(pmu->dev, "There are not enough counters\n");  in ddr_perf_event_add()
644 		return -EOPNOTSUPP;  in ddr_perf_event_add()
647 	pmu->events[counter] = event;  in ddr_perf_event_add()
648 	pmu->active_events++;  in ddr_perf_event_add()
649 	hwc->idx = counter;  in ddr_perf_event_add()
650 	hwc->state |= PERF_HES_STOPPED;  in ddr_perf_event_add()
652 	if (axi_filter_v1(pmu))  in ddr_perf_event_add()
654 		imx93_ddr_perf_monitor_config(pmu, event_id, counter, cfg1, cfg2);  in ddr_perf_event_add()
656 	if (axi_filter_v2(pmu))  in ddr_perf_event_add()
658 		imx95_ddr_perf_monitor_config(pmu, event_id, counter, cfg1, cfg2);  in ddr_perf_event_add()
668 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);  in ddr_perf_event_stop()  local
669 	struct hw_perf_event *hwc = &event->hw;  in ddr_perf_event_stop()
670 	int counter = hwc->idx;  in ddr_perf_event_stop()
672 	ddr_perf_counter_local_config(pmu, event->attr.config, counter, false);  in ddr_perf_event_stop()
675 	hwc->state |= PERF_HES_STOPPED;  in ddr_perf_event_stop()
680 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);  in ddr_perf_event_del()  local
681 	struct hw_perf_event *hwc = &event->hw;  in ddr_perf_event_del()
682 	int counter = hwc->idx;  in ddr_perf_event_del()
686 	pmu->events[counter] = NULL;  in ddr_perf_event_del()
687 	pmu->active_events--;  in ddr_perf_event_del()
688 	hwc->idx = -1;  in ddr_perf_event_del()
691 static void ddr_perf_pmu_enable(struct pmu *pmu)  in ddr_perf_pmu_enable()  argument
693 	struct ddr_pmu *ddr_pmu = to_ddr_pmu(pmu);  in ddr_perf_pmu_enable()
698 static void ddr_perf_pmu_disable(struct pmu *pmu)  in ddr_perf_pmu_disable()  argument
700 	struct ddr_pmu *ddr_pmu = to_ddr_pmu(pmu);  in ddr_perf_pmu_disable()
705 static void ddr_perf_init(struct ddr_pmu *pmu, void __iomem *base,  in ddr_perf_init()  argument
708 	*pmu = (struct ddr_pmu) {  in ddr_perf_init()
709 		.pmu = (struct pmu) {  in ddr_perf_init()
730 	struct ddr_pmu *pmu = (struct ddr_pmu *)p;  in ddr_perf_irq_handler()  local
746 		if (!pmu->events[i])  in ddr_perf_irq_handler()
749 		event = pmu->events[i];  in ddr_perf_irq_handler()
754 	ddr_perf_counter_global_config(pmu, true);  in ddr_perf_irq_handler()
761 	struct ddr_pmu *pmu = hlist_entry_safe(node, struct ddr_pmu, node);  in ddr_perf_offline_cpu()  local
764 	if (cpu != pmu->cpu)  in ddr_perf_offline_cpu()
771 	perf_pmu_migrate_context(&pmu->pmu, cpu, target);  in ddr_perf_offline_cpu()
772 	pmu->cpu = target;  in ddr_perf_offline_cpu()
774 	WARN_ON(irq_set_affinity(pmu->irq, cpumask_of(pmu->cpu)));  in ddr_perf_offline_cpu()
781 	struct ddr_pmu *pmu;  in ddr_perf_probe()  local
790 	pmu = devm_kzalloc(&pdev->dev, sizeof(*pmu), GFP_KERNEL);  in ddr_perf_probe()
791 	if (!pmu)  in ddr_perf_probe()
792 		return -ENOMEM;  in ddr_perf_probe()
794 	ddr_perf_init(pmu, base, &pdev->dev);  in ddr_perf_probe()
796 	pmu->devtype_data = of_device_get_match_data(&pdev->dev);  in ddr_perf_probe()
798 	platform_set_drvdata(pdev, pmu);  in ddr_perf_probe()
800 	pmu->id = ida_alloc(&ddr_ida, GFP_KERNEL);  in ddr_perf_probe()
801 	name = devm_kasprintf(&pdev->dev, GFP_KERNEL, DDR_PERF_DEV_NAME "%d", pmu->id);  in ddr_perf_probe()
803 		ret = -ENOMEM;  in ddr_perf_probe()
807 	pmu->cpu = raw_smp_processor_id();  in ddr_perf_probe()
811 		dev_err(&pdev->dev, "Failed to add callbacks for multi state\n");  in ddr_perf_probe()
814 	pmu->cpuhp_state = ret;  in ddr_perf_probe()
816 	/* Register the pmu instance for cpu hotplug */  in ddr_perf_probe()
817 	ret = cpuhp_state_add_instance_nocalls(pmu->cpuhp_state, &pmu->node);  in ddr_perf_probe()
819 		dev_err(&pdev->dev, "Error %d registering hotplug\n", ret);  in ddr_perf_probe()
830 	ret = devm_request_irq(&pdev->dev, irq, ddr_perf_irq_handler,  in ddr_perf_probe()
832 			       DDR_CPUHP_CB_NAME, pmu);  in ddr_perf_probe()
834 		dev_err(&pdev->dev, "Request irq failed: %d", ret);  in ddr_perf_probe()
838 	pmu->irq = irq;  in ddr_perf_probe()
839 	ret = irq_set_affinity(pmu->irq, cpumask_of(pmu->cpu));  in ddr_perf_probe()
841 		dev_err(pmu->dev, "Failed to set interrupt affinity\n");  in ddr_perf_probe()
845 	ret = perf_pmu_register(&pmu->pmu, name, -1);  in ddr_perf_probe()
852 	cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node);  in ddr_perf_probe()
854 	cpuhp_remove_multi_state(pmu->cpuhp_state);  in ddr_perf_probe()
857 	ida_free(&ddr_ida, pmu->id);  in ddr_perf_probe()
858 	dev_warn(&pdev->dev, "i.MX9 DDR Perf PMU failed (%d), disabled\n", ret);  in ddr_perf_probe()
864 	struct ddr_pmu *pmu = platform_get_drvdata(pdev);  in ddr_perf_remove()  local
866 	cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node);  in ddr_perf_remove()
867 	cpuhp_remove_multi_state(pmu->cpuhp_state);  in ddr_perf_remove()
869 	perf_pmu_unregister(&pmu->pmu);  in ddr_perf_remove()
871 	ida_free(&ddr_ida, pmu->id);  in ddr_perf_remove()
876 		.name                = "imx9-ddr-pmu",