1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved
4 */
5
6 #include <linux/cpufreq.h>
7 #include <linux/dma-mapping.h>
8 #include <linux/module.h>
9 #include <linux/of.h>
10 #include <linux/platform_device.h>
11 #include <linux/units.h>
12
13 #include <soc/tegra/bpmp.h>
14 #include <soc/tegra/bpmp-abi.h>
15
16 #define TEGRA186_NUM_CLUSTERS 2
17 #define EDVD_OFFSET_A57(core) ((SZ_64K * 6) + (0x20 + (core) * 0x4))
18 #define EDVD_OFFSET_DENVER(core) ((SZ_64K * 7) + (0x20 + (core) * 0x4))
19 #define EDVD_CORE_VOLT_FREQ_F_SHIFT 0
20 #define EDVD_CORE_VOLT_FREQ_F_MASK 0xffff
21 #define EDVD_CORE_VOLT_FREQ_V_SHIFT 16
22
23 struct tegra186_cpufreq_cpu {
24 unsigned int bpmp_cluster_id;
25 unsigned int edvd_offset;
26 };
27
28 static const struct tegra186_cpufreq_cpu tegra186_cpus[] = {
29 /* CPU0 - A57 Cluster */
30 {
31 .bpmp_cluster_id = 1,
32 .edvd_offset = EDVD_OFFSET_A57(0)
33 },
34 /* CPU1 - Denver Cluster */
35 {
36 .bpmp_cluster_id = 0,
37 .edvd_offset = EDVD_OFFSET_DENVER(0)
38 },
39 /* CPU2 - Denver Cluster */
40 {
41 .bpmp_cluster_id = 0,
42 .edvd_offset = EDVD_OFFSET_DENVER(1)
43 },
44 /* CPU3 - A57 Cluster */
45 {
46 .bpmp_cluster_id = 1,
47 .edvd_offset = EDVD_OFFSET_A57(1)
48 },
49 /* CPU4 - A57 Cluster */
50 {
51 .bpmp_cluster_id = 1,
52 .edvd_offset = EDVD_OFFSET_A57(2)
53 },
54 /* CPU5 - A57 Cluster */
55 {
56 .bpmp_cluster_id = 1,
57 .edvd_offset = EDVD_OFFSET_A57(3)
58 },
59 };
60
61 struct tegra186_cpufreq_cluster {
62 struct cpufreq_frequency_table *bpmp_lut;
63 u32 ref_clk_khz;
64 u32 div;
65 };
66
67 struct tegra186_cpufreq_data {
68 void __iomem *regs;
69 const struct tegra186_cpufreq_cpu *cpus;
70 bool icc_dram_bw_scaling;
71 struct tegra186_cpufreq_cluster clusters[];
72 };
73
tegra_cpufreq_set_bw(struct cpufreq_policy * policy,unsigned long freq_khz)74 static int tegra_cpufreq_set_bw(struct cpufreq_policy *policy, unsigned long freq_khz)
75 {
76 struct tegra186_cpufreq_data *data = cpufreq_get_driver_data();
77 struct device *dev;
78 int ret;
79
80 dev = get_cpu_device(policy->cpu);
81 if (!dev)
82 return -ENODEV;
83
84 struct dev_pm_opp *opp __free(put_opp) =
85 dev_pm_opp_find_freq_exact(dev, freq_khz * HZ_PER_KHZ, true);
86 if (IS_ERR(opp))
87 return PTR_ERR(opp);
88
89 ret = dev_pm_opp_set_opp(dev, opp);
90 if (ret)
91 data->icc_dram_bw_scaling = false;
92
93 return ret;
94 }
95
tegra_cpufreq_init_cpufreq_table(struct cpufreq_policy * policy,struct cpufreq_frequency_table * bpmp_lut,struct cpufreq_frequency_table ** opp_table)96 static int tegra_cpufreq_init_cpufreq_table(struct cpufreq_policy *policy,
97 struct cpufreq_frequency_table *bpmp_lut,
98 struct cpufreq_frequency_table **opp_table)
99 {
100 struct tegra186_cpufreq_data *data = cpufreq_get_driver_data();
101 struct cpufreq_frequency_table *freq_table = NULL;
102 struct cpufreq_frequency_table *pos;
103 struct device *cpu_dev;
104 unsigned long rate;
105 int ret, max_opps;
106 int j = 0;
107
108 cpu_dev = get_cpu_device(policy->cpu);
109 if (!cpu_dev) {
110 pr_err("%s: failed to get cpu%d device\n", __func__, policy->cpu);
111 return -ENODEV;
112 }
113
114 /* Initialize OPP table mentioned in operating-points-v2 property in DT */
115 ret = dev_pm_opp_of_add_table_indexed(cpu_dev, 0);
116 if (ret) {
117 dev_err(cpu_dev, "Invalid or empty opp table in device tree\n");
118 data->icc_dram_bw_scaling = false;
119 return ret;
120 }
121
122 max_opps = dev_pm_opp_get_opp_count(cpu_dev);
123 if (max_opps <= 0) {
124 dev_err(cpu_dev, "Failed to add OPPs\n");
125 return max_opps;
126 }
127
128 /* Disable all opps and cross-validate against LUT later */
129 for (rate = 0; ; rate++) {
130 struct dev_pm_opp *opp __free(put_opp) =
131 dev_pm_opp_find_freq_ceil(cpu_dev, &rate);
132 if (IS_ERR(opp))
133 break;
134
135 dev_pm_opp_disable(cpu_dev, rate);
136 }
137
138 freq_table = kzalloc_objs(*freq_table, (max_opps + 1));
139 if (!freq_table)
140 return -ENOMEM;
141
142 /*
143 * Cross check the frequencies from BPMP-FW LUT against the OPP's present in DT.
144 * Enable only those DT OPP's which are present in LUT also.
145 */
146 cpufreq_for_each_valid_entry(pos, bpmp_lut) {
147 struct dev_pm_opp *opp __free(put_opp) =
148 dev_pm_opp_find_freq_exact(cpu_dev, pos->frequency * HZ_PER_KHZ, false);
149 if (IS_ERR(opp))
150 continue;
151
152 ret = dev_pm_opp_enable(cpu_dev, pos->frequency * HZ_PER_KHZ);
153 if (ret < 0)
154 return ret;
155
156 freq_table[j].driver_data = pos->driver_data;
157 freq_table[j].frequency = pos->frequency;
158 j++;
159 }
160
161 freq_table[j].driver_data = pos->driver_data;
162 freq_table[j].frequency = CPUFREQ_TABLE_END;
163
164 *opp_table = &freq_table[0];
165
166 dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus);
167
168 /* Prime interconnect data */
169 tegra_cpufreq_set_bw(policy, freq_table[j - 1].frequency);
170
171 return ret;
172 }
173
tegra186_cpufreq_init(struct cpufreq_policy * policy)174 static int tegra186_cpufreq_init(struct cpufreq_policy *policy)
175 {
176 struct tegra186_cpufreq_data *data = cpufreq_get_driver_data();
177 unsigned int cluster = data->cpus[policy->cpu].bpmp_cluster_id;
178 struct cpufreq_frequency_table *freq_table;
179 struct cpufreq_frequency_table *bpmp_lut;
180 u32 cpu;
181 int ret;
182
183 policy->cpuinfo.transition_latency = 300 * 1000;
184 policy->driver_data = NULL;
185
186 /* set same policy for all cpus in a cluster */
187 for (cpu = 0; cpu < ARRAY_SIZE(tegra186_cpus); cpu++) {
188 if (data->cpus[cpu].bpmp_cluster_id == cluster)
189 cpumask_set_cpu(cpu, policy->cpus);
190 }
191
192 bpmp_lut = data->clusters[cluster].bpmp_lut;
193
194 if (data->icc_dram_bw_scaling) {
195 ret = tegra_cpufreq_init_cpufreq_table(policy, bpmp_lut, &freq_table);
196 if (!ret) {
197 policy->freq_table = freq_table;
198 return 0;
199 }
200 }
201
202 data->icc_dram_bw_scaling = false;
203 policy->freq_table = bpmp_lut;
204 pr_info("OPP tables missing from DT, EMC frequency scaling disabled\n");
205
206 return 0;
207 }
208
tegra186_cpufreq_set_target(struct cpufreq_policy * policy,unsigned int index)209 static int tegra186_cpufreq_set_target(struct cpufreq_policy *policy,
210 unsigned int index)
211 {
212 struct tegra186_cpufreq_data *data = cpufreq_get_driver_data();
213 struct cpufreq_frequency_table *tbl = policy->freq_table + index;
214 unsigned int edvd_offset;
215 u32 edvd_val = tbl->driver_data;
216 u32 cpu;
217
218 for_each_cpu(cpu, policy->cpus) {
219 edvd_offset = data->cpus[cpu].edvd_offset;
220 writel(edvd_val, data->regs + edvd_offset);
221 }
222
223 if (data->icc_dram_bw_scaling)
224 tegra_cpufreq_set_bw(policy, tbl->frequency);
225
226
227 return 0;
228 }
229
tegra186_cpufreq_get(unsigned int cpu)230 static unsigned int tegra186_cpufreq_get(unsigned int cpu)
231 {
232 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu);
233 struct tegra186_cpufreq_data *data = cpufreq_get_driver_data();
234 struct tegra186_cpufreq_cluster *cluster;
235 unsigned int edvd_offset, cluster_id;
236 u32 ndiv;
237
238 if (!policy)
239 return 0;
240
241 edvd_offset = data->cpus[policy->cpu].edvd_offset;
242 ndiv = readl(data->regs + edvd_offset) & EDVD_CORE_VOLT_FREQ_F_MASK;
243 cluster_id = data->cpus[policy->cpu].bpmp_cluster_id;
244 cluster = &data->clusters[cluster_id];
245
246 return (cluster->ref_clk_khz * ndiv) / cluster->div;
247 }
248
249 static struct cpufreq_driver tegra186_cpufreq_driver = {
250 .name = "tegra186",
251 .flags = CPUFREQ_HAVE_GOVERNOR_PER_POLICY |
252 CPUFREQ_NEED_INITIAL_FREQ_CHECK,
253 .get = tegra186_cpufreq_get,
254 .verify = cpufreq_generic_frequency_table_verify,
255 .target_index = tegra186_cpufreq_set_target,
256 .init = tegra186_cpufreq_init,
257 };
258
tegra_cpufreq_bpmp_read_lut(struct platform_device * pdev,struct tegra_bpmp * bpmp,struct tegra186_cpufreq_cluster * cluster,unsigned int cluster_id,int * num_rates)259 static struct cpufreq_frequency_table *tegra_cpufreq_bpmp_read_lut(
260 struct platform_device *pdev, struct tegra_bpmp *bpmp,
261 struct tegra186_cpufreq_cluster *cluster, unsigned int cluster_id,
262 int *num_rates)
263 {
264 struct cpufreq_frequency_table *table;
265 struct mrq_cpu_vhint_request req;
266 struct tegra_bpmp_message msg;
267 struct cpu_vhint_data *data;
268 int err, i, j;
269 dma_addr_t phys;
270 void *virt;
271
272 virt = dma_alloc_coherent(bpmp->dev, sizeof(*data), &phys,
273 GFP_KERNEL);
274 if (!virt)
275 return ERR_PTR(-ENOMEM);
276
277 data = (struct cpu_vhint_data *)virt;
278
279 memset(&req, 0, sizeof(req));
280 req.addr = phys;
281 req.cluster_id = cluster_id;
282
283 memset(&msg, 0, sizeof(msg));
284 msg.mrq = MRQ_CPU_VHINT;
285 msg.tx.data = &req;
286 msg.tx.size = sizeof(req);
287
288 err = tegra_bpmp_transfer(bpmp, &msg);
289 if (err) {
290 table = ERR_PTR(err);
291 goto free;
292 }
293 if (msg.rx.ret) {
294 table = ERR_PTR(-EINVAL);
295 goto free;
296 }
297
298 *num_rates = 0;
299 for (i = data->vfloor; i <= data->vceil; i++) {
300 u16 ndiv = data->ndiv[i];
301
302 if (ndiv < data->ndiv_min || ndiv > data->ndiv_max)
303 continue;
304
305 /* Only store lowest voltage index for each rate */
306 if (i > 0 && ndiv == data->ndiv[i - 1])
307 continue;
308
309 (*num_rates)++;
310 }
311
312 table = devm_kcalloc(&pdev->dev, *num_rates + 1, sizeof(*table),
313 GFP_KERNEL);
314 if (!table) {
315 table = ERR_PTR(-ENOMEM);
316 goto free;
317 }
318
319 cluster->ref_clk_khz = data->ref_clk_hz / 1000;
320 cluster->div = data->pdiv * data->mdiv;
321
322 for (i = data->vfloor, j = 0; i <= data->vceil; i++) {
323 struct cpufreq_frequency_table *point;
324 u16 ndiv = data->ndiv[i];
325 u32 edvd_val = 0;
326
327 if (ndiv < data->ndiv_min || ndiv > data->ndiv_max)
328 continue;
329
330 /* Only store lowest voltage index for each rate */
331 if (i > 0 && ndiv == data->ndiv[i - 1])
332 continue;
333
334 edvd_val |= i << EDVD_CORE_VOLT_FREQ_V_SHIFT;
335 edvd_val |= ndiv << EDVD_CORE_VOLT_FREQ_F_SHIFT;
336
337 point = &table[j++];
338 point->driver_data = edvd_val;
339 point->frequency = (cluster->ref_clk_khz * ndiv) / cluster->div;
340 }
341
342 table[j].frequency = CPUFREQ_TABLE_END;
343
344 free:
345 dma_free_coherent(bpmp->dev, sizeof(*data), virt, phys);
346
347 return table;
348 }
349
tegra186_cpufreq_probe(struct platform_device * pdev)350 static int tegra186_cpufreq_probe(struct platform_device *pdev)
351 {
352 struct tegra186_cpufreq_data *data;
353 struct tegra_bpmp *bpmp;
354 struct device *cpu_dev;
355 unsigned int i = 0, err, edvd_offset;
356 int num_rates = 0;
357 u32 edvd_val, cpu;
358
359 data = devm_kzalloc(&pdev->dev,
360 struct_size(data, clusters, TEGRA186_NUM_CLUSTERS),
361 GFP_KERNEL);
362 if (!data)
363 return -ENOMEM;
364
365 data->cpus = tegra186_cpus;
366
367 bpmp = tegra_bpmp_get(&pdev->dev);
368 if (IS_ERR(bpmp))
369 return PTR_ERR(bpmp);
370
371 data->regs = devm_platform_ioremap_resource(pdev, 0);
372 if (IS_ERR(data->regs)) {
373 err = PTR_ERR(data->regs);
374 goto put_bpmp;
375 }
376
377 for (i = 0; i < TEGRA186_NUM_CLUSTERS; i++) {
378 struct tegra186_cpufreq_cluster *cluster = &data->clusters[i];
379
380 cluster->bpmp_lut = tegra_cpufreq_bpmp_read_lut(pdev, bpmp, cluster, i, &num_rates);
381 if (IS_ERR(cluster->bpmp_lut)) {
382 err = PTR_ERR(cluster->bpmp_lut);
383 goto put_bpmp;
384 } else if (!num_rates) {
385 err = -EINVAL;
386 goto put_bpmp;
387 }
388
389 for (cpu = 0; cpu < ARRAY_SIZE(tegra186_cpus); cpu++) {
390 if (data->cpus[cpu].bpmp_cluster_id == i) {
391 edvd_val = cluster->bpmp_lut[num_rates - 1].driver_data;
392 edvd_offset = data->cpus[cpu].edvd_offset;
393 writel(edvd_val, data->regs + edvd_offset);
394 }
395 }
396 }
397
398 tegra186_cpufreq_driver.driver_data = data;
399
400 /* Check for optional OPPv2 and interconnect paths on CPU0 to enable ICC scaling */
401 cpu_dev = get_cpu_device(0);
402 if (!cpu_dev) {
403 err = -EPROBE_DEFER;
404 goto put_bpmp;
405 }
406
407 if (dev_pm_opp_of_get_opp_desc_node(cpu_dev)) {
408 err = dev_pm_opp_of_find_icc_paths(cpu_dev, NULL);
409 if (!err)
410 data->icc_dram_bw_scaling = true;
411 }
412
413 err = cpufreq_register_driver(&tegra186_cpufreq_driver);
414
415 put_bpmp:
416 tegra_bpmp_put(bpmp);
417
418 return err;
419 }
420
tegra186_cpufreq_remove(struct platform_device * pdev)421 static void tegra186_cpufreq_remove(struct platform_device *pdev)
422 {
423 cpufreq_unregister_driver(&tegra186_cpufreq_driver);
424 }
425
426 static const struct of_device_id tegra186_cpufreq_of_match[] = {
427 { .compatible = "nvidia,tegra186-ccplex-cluster", },
428 { }
429 };
430 MODULE_DEVICE_TABLE(of, tegra186_cpufreq_of_match);
431
432 static struct platform_driver tegra186_cpufreq_platform_driver = {
433 .driver = {
434 .name = "tegra186-cpufreq",
435 .of_match_table = tegra186_cpufreq_of_match,
436 },
437 .probe = tegra186_cpufreq_probe,
438 .remove = tegra186_cpufreq_remove,
439 };
440 module_platform_driver(tegra186_cpufreq_platform_driver);
441
442 MODULE_AUTHOR("Mikko Perttunen <mperttunen@nvidia.com>");
443 MODULE_DESCRIPTION("NVIDIA Tegra186 cpufreq driver");
444 MODULE_LICENSE("GPL v2");
445