Lines Matching +full:rk3399 +full:- +full:dfi
1 // SPDX-License-Identifier: GPL-2.0-only
4 * Author: Lin Huang <hl@rock-chips.com>
8 #include <linux/devfreq-event.h>
69 * struct dmc_count_channel - structure to hold counter values from the DDR controller
87 * The dfi controller can monitor DDR load. It has an upper and lower threshold
121 static int rockchip_dfi_enable(struct rockchip_dfi *dfi)
123 void __iomem *dfi_regs = dfi->regs;
126 mutex_lock(&dfi->mutex);
128 dfi->usecount++;
129 if (dfi->usecount > 1)
132 ret = clk_prepare_enable(dfi->clk);
134 dev_err(&dfi->edev->dev, "failed to enable dfi clk: %d\n", ret);
138 for (i = 0; i < dfi->max_channels; i++) {
141 if (!(dfi->channel_mask & BIT(i)))
147 dfi_regs + i * dfi->ddrmon_stride + DDRMON_CTRL);
149 /* set ddr type to dfi */
150 switch (dfi->ddr_type) {
164 dfi_regs + i * dfi->ddrmon_stride + DDRMON_CTRL);
168 dfi_regs + i * dfi->ddrmon_stride + DDRMON_CTRL);
170 if (dfi->ddrmon_ctrl_single)
174 mutex_unlock(&dfi->mutex);
179 static void rockchip_dfi_disable(struct rockchip_dfi *dfi)
181 void __iomem *dfi_regs = dfi->regs;
184 mutex_lock(&dfi->mutex);
186 dfi->usecount--;
188 WARN_ON_ONCE(dfi->usecount < 0);
190 if (dfi->usecount > 0)
193 for (i = 0; i < dfi->max_channels; i++) {
194 if (!(dfi->channel_mask & BIT(i)))
198 dfi_regs + i * dfi->ddrmon_stride + DDRMON_CTRL);
200 if (dfi->ddrmon_ctrl_single)
204 clk_disable_unprepare(dfi->clk);
206 mutex_unlock(&dfi->mutex);
209 static void rockchip_dfi_read_counters(struct rockchip_dfi *dfi, struct dmc_count *res)
212 void __iomem *dfi_regs = dfi->regs;
214 for (i = 0; i < dfi->max_channels; i++) {
215 if (!(dfi->channel_mask & BIT(i)))
217 res->c[i].read_access = readl_relaxed(dfi_regs +
218 DDRMON_CH0_RD_NUM + i * dfi->ddrmon_stride);
219 res->c[i].write_access = readl_relaxed(dfi_regs +
220 DDRMON_CH0_WR_NUM + i * dfi->ddrmon_stride);
221 res->c[i].access = readl_relaxed(dfi_regs +
222 DDRMON_CH0_DFI_ACCESS_NUM + i * dfi->ddrmon_stride);
223 res->c[i].clock_cycles = readl_relaxed(dfi_regs +
224 DDRMON_CH0_COUNT_NUM + i * dfi->ddrmon_stride);
230 struct rockchip_dfi *dfi = devfreq_event_get_drvdata(edev);
232 rockchip_dfi_disable(dfi);
239 struct rockchip_dfi *dfi = devfreq_event_get_drvdata(edev);
241 return rockchip_dfi_enable(dfi);
252 struct rockchip_dfi *dfi = devfreq_event_get_drvdata(edev);
254 struct dmc_count *last = &dfi->last_event_count;
258 rockchip_dfi_read_counters(dfi, &count);
261 for (i = 0; i < dfi->max_channels; i++) {
264 if (!(dfi->channel_mask & BIT(i)))
267 a = count.c[i].access - last->c[i].access;
268 c = count.c[i].clock_cycles - last->c[i].clock_cycles;
276 edata->load_count = access * 4;
277 edata->total_count = clock_cycles;
279 dfi->last_event_count = count;
293 static void rockchip_ddr_perf_counters_add(struct rockchip_dfi *dfi,
297 const struct dmc_count *last = &dfi->last_perf_count;
300 for (i = 0; i < dfi->max_channels; i++) {
301 res->c[i].read_access = dfi->total_count.c[i].read_access +
302 (u32)(now->c[i].read_access - last->c[i].read_access);
303 res->c[i].write_access = dfi->total_count.c[i].write_access +
304 (u32)(now->c[i].write_access - last->c[i].write_access);
305 res->c[i].access = dfi->total_count.c[i].access +
306 (u32)(now->c[i].access - last->c[i].access);
307 res->c[i].clock_cycles = dfi->total_count.c[i].clock_cycles +
308 (u32)(now->c[i].clock_cycles - last->c[i].clock_cycles);
316 struct rockchip_dfi *dfi = container_of(pmu, struct rockchip_dfi, pmu);
318 return cpumap_print_to_pagebuf(true, buf, cpumask_of(dfi->cpu));
338 PMU_EVENT_ATTR_STRING(_name.scale, _var##_scale, "9.536743164e-07")
340 DFI_PMU_EVENT_ATTR(read-bytes0, ddr_pmu_read_bytes0, "event="__stringify(PERF_EVENT_READ_BYTES0));
341 DFI_PMU_EVENT_ATTR(write-bytes0, ddr_pmu_write_bytes0, "event="__stringify(PERF_EVENT_WRITE_BYTES0));
343 DFI_PMU_EVENT_ATTR(read-bytes1, ddr_pmu_read_bytes1, "event="__stringify(PERF_EVENT_READ_BYTES1));
344 DFI_PMU_EVENT_ATTR(write-bytes1, ddr_pmu_write_bytes1, "event="__stringify(PERF_EVENT_WRITE_BYTES1));
346 DFI_PMU_EVENT_ATTR(read-bytes2, ddr_pmu_read_bytes2, "event="__stringify(PERF_EVENT_READ_BYTES2));
347 DFI_PMU_EVENT_ATTR(write-bytes2, ddr_pmu_write_bytes2, "event="__stringify(PERF_EVENT_WRITE_BYTES2));
349 DFI_PMU_EVENT_ATTR(read-bytes3, ddr_pmu_read_bytes3, "event="__stringify(PERF_EVENT_READ_BYTES3));
350 DFI_PMU_EVENT_ATTR(write-bytes3, ddr_pmu_write_bytes3, "event="__stringify(PERF_EVENT_WRITE_BYTES3));
352 DFI_PMU_EVENT_ATTR(read-bytes, ddr_pmu_read_bytes, "event="__stringify(PERF_EVENT_READ_BYTES));
353 DFI_PMU_EVENT_ATTR(write-bytes, ddr_pmu_write_bytes, "event="__stringify(PERF_EVENT_WRITE_BYTES));
383 PMU_FORMAT_ATTR(event, "config:0-7");
404 struct rockchip_dfi *dfi = container_of(event->pmu, struct rockchip_dfi, pmu);
406 if (event->attr.type != event->pmu->type)
407 return -ENOENT;
409 if (event->attach_state & PERF_ATTACH_TASK)
410 return -EINVAL;
412 if (event->cpu < 0) {
413 dev_warn(dfi->dev, "Can't provide per-task data!\n");
414 return -EINVAL;
422 struct rockchip_dfi *dfi = container_of(event->pmu, struct rockchip_dfi, pmu);
423 int blen = dfi->burst_len;
429 rockchip_dfi_read_counters(dfi, &now);
432 seq = read_seqbegin(&dfi->count_seqlock);
433 rockchip_ddr_perf_counters_add(dfi, &now, &total);
434 } while (read_seqretry(&dfi->count_seqlock, seq));
436 switch (event->attr.config) {
441 for (i = 0; i < dfi->max_channels; i++)
442 count += total.c[i].read_access * blen * dfi->buswidth[i];
445 for (i = 0; i < dfi->max_channels; i++)
446 count += total.c[i].write_access * blen * dfi->buswidth[i];
449 count = total.c[0].read_access * blen * dfi->buswidth[0];
452 count = total.c[0].write_access * blen * dfi->buswidth[0];
455 count = total.c[1].read_access * blen * dfi->buswidth[1];
458 count = total.c[1].write_access * blen * dfi->buswidth[1];
461 count = total.c[2].read_access * blen * dfi->buswidth[2];
464 count = total.c[2].write_access * blen * dfi->buswidth[2];
467 count = total.c[3].read_access * blen * dfi->buswidth[3];
470 count = total.c[3].write_access * blen * dfi->buswidth[3];
473 for (i = 0; i < dfi->max_channels; i++)
474 count += total.c[i].access * blen * dfi->buswidth[i];
486 if (event->attr.config >= PERF_ACCESS_TYPE_MAX)
490 prev = local64_xchg(&event->hw.prev_count, now);
491 local64_add(now - prev, &event->count);
498 local64_set(&event->hw.prev_count, now);
503 struct rockchip_dfi *dfi = container_of(event->pmu, struct rockchip_dfi, pmu);
505 dfi->active_events++;
507 if (dfi->active_events == 1) {
508 dfi->total_count = (struct dmc_count){};
509 rockchip_dfi_read_counters(dfi, &dfi->last_perf_count);
510 hrtimer_start(&dfi->timer, ns_to_ktime(NSEC_PER_SEC), HRTIMER_MODE_REL);
526 struct rockchip_dfi *dfi = container_of(event->pmu, struct rockchip_dfi, pmu);
530 dfi->active_events--;
532 if (dfi->active_events == 0)
533 hrtimer_cancel(&dfi->timer);
538 struct rockchip_dfi *dfi = container_of(timer, struct rockchip_dfi, timer);
541 rockchip_dfi_read_counters(dfi, &now);
543 write_seqlock(&dfi->count_seqlock);
545 rockchip_ddr_perf_counters_add(dfi, &now, &total);
546 dfi->total_count = total;
547 dfi->last_perf_count = now;
549 write_sequnlock(&dfi->count_seqlock);
551 hrtimer_forward_now(&dfi->timer, ns_to_ktime(NSEC_PER_SEC));
558 struct rockchip_dfi *dfi = hlist_entry_safe(node, struct rockchip_dfi, node);
561 if (cpu != dfi->cpu)
568 perf_pmu_migrate_context(&dfi->pmu, cpu, target);
569 dfi->cpu = target;
576 struct rockchip_dfi *dfi = data;
578 cpuhp_remove_multi_state(dfi->cpuhp_state);
580 rockchip_dfi_disable(dfi);
585 struct rockchip_dfi *dfi = data;
587 cpuhp_state_remove_instance_nocalls(dfi->cpuhp_state, &dfi->node);
592 struct rockchip_dfi *dfi = data;
594 perf_pmu_unregister(&dfi->pmu);
597 static int rockchip_ddr_perf_init(struct rockchip_dfi *dfi)
599 struct pmu *pmu = &dfi->pmu;
602 seqlock_init(&dfi->count_seqlock);
604 pmu->module = THIS_MODULE;
605 pmu->capabilities = PERF_PMU_CAP_NO_EXCLUDE;
606 pmu->task_ctx_nr = perf_invalid_context;
607 pmu->attr_groups = attr_groups;
608 pmu->event_init = rockchip_ddr_perf_event_init;
609 pmu->add = rockchip_ddr_perf_event_add;
610 pmu->del = rockchip_ddr_perf_event_del;
611 pmu->start = rockchip_ddr_perf_event_start;
612 pmu->stop = rockchip_ddr_perf_event_stop;
613 pmu->read = rockchip_ddr_perf_event_update;
615 dfi->cpu = raw_smp_processor_id();
623 dev_err(dfi->dev, "cpuhp_setup_state_multi failed: %d\n", ret);
627 dfi->cpuhp_state = ret;
629 rockchip_dfi_enable(dfi);
631 ret = devm_add_action_or_reset(dfi->dev, rockchip_ddr_cpuhp_remove_state, dfi);
635 ret = cpuhp_state_add_instance_nocalls(dfi->cpuhp_state, &dfi->node);
637 dev_err(dfi->dev, "Error %d registering hotplug\n", ret);
641 ret = devm_add_action_or_reset(dfi->dev, rockchip_ddr_cpuhp_remove_instance, dfi);
645 hrtimer_setup(&dfi->timer, rockchip_dfi_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
647 switch (dfi->ddr_type) {
650 dfi->burst_len = 8;
654 dfi->burst_len = 16;
658 ret = perf_pmu_register(pmu, "rockchip_ddr", -1);
662 return devm_add_action_or_reset(dfi->dev, rockchip_ddr_perf_remove, dfi);
665 static int rockchip_ddr_perf_init(struct rockchip_dfi *dfi)
671 static int rk3399_dfi_init(struct rockchip_dfi *dfi)
673 struct regmap *regmap_pmu = dfi->regmap_pmu;
676 dfi->clk = devm_clk_get(dfi->dev, "pclk_ddr_mon");
677 if (IS_ERR(dfi->clk))
678 return dev_err_probe(dfi->dev, PTR_ERR(dfi->clk),
683 dfi->ddr_type = FIELD_GET(RK3399_PMUGRF_OS_REG2_DDRTYPE, val);
685 dfi->channel_mask = GENMASK(1, 0);
686 dfi->max_channels = 2;
688 dfi->buswidth[0] = FIELD_GET(RK3399_PMUGRF_OS_REG2_BW_CH0, val) == 0 ? 4 : 2;
689 dfi->buswidth[1] = FIELD_GET(RK3399_PMUGRF_OS_REG2_BW_CH1, val) == 0 ? 4 : 2;
691 dfi->ddrmon_stride = 0x14;
692 dfi->ddrmon_ctrl_single = true;
697 static int rk3568_dfi_init(struct rockchip_dfi *dfi)
699 struct regmap *regmap_pmu = dfi->regmap_pmu;
706 dfi->ddr_type = FIELD_GET(RK3568_PMUGRF_OS_REG2_DRAMTYPE_INFO, reg2);
713 dfi->ddr_type |= FIELD_GET(RK3568_PMUGRF_OS_REG3_DRAMTYPE_INFO_V3, reg3) << 3;
715 dfi->channel_mask = BIT(0);
716 dfi->max_channels = 1;
718 dfi->buswidth[0] = FIELD_GET(RK3568_PMUGRF_OS_REG2_BW_CH0, reg2) == 0 ? 4 : 2;
720 dfi->ddrmon_stride = 0x0; /* not relevant, we only have a single channel on this SoC */
721 dfi->ddrmon_ctrl_single = true;
726 static int rk3588_dfi_init(struct rockchip_dfi *dfi)
728 struct regmap *regmap_pmu = dfi->regmap_pmu;
736 dfi->ddr_type = FIELD_GET(RK3588_PMUGRF_OS_REG2_DRAMTYPE_INFO, reg2);
743 dfi->ddr_type |= FIELD_GET(RK3588_PMUGRF_OS_REG3_DRAMTYPE_INFO_V3, reg3) << 3;
745 dfi->buswidth[0] = FIELD_GET(RK3588_PMUGRF_OS_REG2_BW_CH0, reg2) == 0 ? 4 : 2;
746 dfi->buswidth[1] = FIELD_GET(RK3588_PMUGRF_OS_REG2_BW_CH1, reg2) == 0 ? 4 : 2;
747 dfi->buswidth[2] = FIELD_GET(RK3568_PMUGRF_OS_REG2_BW_CH0, reg4) == 0 ? 4 : 2;
748 dfi->buswidth[3] = FIELD_GET(RK3588_PMUGRF_OS_REG2_BW_CH1, reg4) == 0 ? 4 : 2;
749 dfi->channel_mask = FIELD_GET(RK3588_PMUGRF_OS_REG2_CH_INFO, reg2) |
751 dfi->max_channels = 4;
753 dfi->ddrmon_stride = 0x4000;
759 { .compatible = "rockchip,rk3399-dfi", .data = rk3399_dfi_init },
760 { .compatible = "rockchip,rk3568-dfi", .data = rk3568_dfi_init },
761 { .compatible = "rockchip,rk3588-dfi", .data = rk3588_dfi_init },
769 struct device *dev = &pdev->dev;
770 struct rockchip_dfi *dfi;
772 struct device_node *np = pdev->dev.of_node, *node;
773 int (*soc_init)(struct rockchip_dfi *dfi);
776 soc_init = of_device_get_match_data(&pdev->dev);
778 return -EINVAL;
780 dfi = devm_kzalloc(dev, sizeof(*dfi), GFP_KERNEL);
781 if (!dfi)
782 return -ENOMEM;
784 dfi->regs = devm_platform_ioremap_resource(pdev, 0);
785 if (IS_ERR(dfi->regs))
786 return PTR_ERR(dfi->regs);
790 return dev_err_probe(&pdev->dev, -ENODEV, "Can't find pmu_grf registers\n");
792 dfi->regmap_pmu = syscon_node_to_regmap(node);
794 if (IS_ERR(dfi->regmap_pmu))
795 return PTR_ERR(dfi->regmap_pmu);
797 dfi->dev = dev;
798 mutex_init(&dfi->mutex);
800 desc = &dfi->desc;
801 desc->ops = &rockchip_dfi_ops;
802 desc->driver_data = dfi;
803 desc->name = np->name;
805 ret = soc_init(dfi);
809 dfi->edev = devm_devfreq_event_add_edev(&pdev->dev, desc);
810 if (IS_ERR(dfi->edev)) {
811 dev_err(&pdev->dev,
812 "failed to add devfreq-event device\n");
813 return PTR_ERR(dfi->edev);
816 ret = rockchip_ddr_perf_init(dfi);
820 platform_set_drvdata(pdev, dfi);
828 .name = "rockchip-dfi",
836 MODULE_AUTHOR("Lin Huang <hl@rock-chips.com>");
837 MODULE_DESCRIPTION("Rockchip DFI driver");