1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved 4 */ 5 6 #include <linux/cpufreq.h> 7 #include <linux/dma-mapping.h> 8 #include <linux/module.h> 9 #include <linux/of.h> 10 #include <linux/platform_device.h> 11 #include <linux/units.h> 12 13 #include <soc/tegra/bpmp.h> 14 #include <soc/tegra/bpmp-abi.h> 15 16 #define TEGRA186_NUM_CLUSTERS 2 17 #define EDVD_OFFSET_A57(core) ((SZ_64K * 6) + (0x20 + (core) * 0x4)) 18 #define EDVD_OFFSET_DENVER(core) ((SZ_64K * 7) + (0x20 + (core) * 0x4)) 19 #define EDVD_CORE_VOLT_FREQ_F_SHIFT 0 20 #define EDVD_CORE_VOLT_FREQ_F_MASK 0xffff 21 #define EDVD_CORE_VOLT_FREQ_V_SHIFT 16 22 23 struct tegra186_cpufreq_cpu { 24 unsigned int bpmp_cluster_id; 25 unsigned int edvd_offset; 26 }; 27 28 static const struct tegra186_cpufreq_cpu tegra186_cpus[] = { 29 /* CPU0 - A57 Cluster */ 30 { 31 .bpmp_cluster_id = 1, 32 .edvd_offset = EDVD_OFFSET_A57(0) 33 }, 34 /* CPU1 - Denver Cluster */ 35 { 36 .bpmp_cluster_id = 0, 37 .edvd_offset = EDVD_OFFSET_DENVER(0) 38 }, 39 /* CPU2 - Denver Cluster */ 40 { 41 .bpmp_cluster_id = 0, 42 .edvd_offset = EDVD_OFFSET_DENVER(1) 43 }, 44 /* CPU3 - A57 Cluster */ 45 { 46 .bpmp_cluster_id = 1, 47 .edvd_offset = EDVD_OFFSET_A57(1) 48 }, 49 /* CPU4 - A57 Cluster */ 50 { 51 .bpmp_cluster_id = 1, 52 .edvd_offset = EDVD_OFFSET_A57(2) 53 }, 54 /* CPU5 - A57 Cluster */ 55 { 56 .bpmp_cluster_id = 1, 57 .edvd_offset = EDVD_OFFSET_A57(3) 58 }, 59 }; 60 61 struct tegra186_cpufreq_cluster { 62 struct cpufreq_frequency_table *bpmp_lut; 63 u32 ref_clk_khz; 64 u32 div; 65 }; 66 67 struct tegra186_cpufreq_data { 68 void __iomem *regs; 69 const struct tegra186_cpufreq_cpu *cpus; 70 bool icc_dram_bw_scaling; 71 struct tegra186_cpufreq_cluster clusters[]; 72 }; 73 74 static int tegra_cpufreq_set_bw(struct cpufreq_policy *policy, unsigned long freq_khz) 75 { 76 struct tegra186_cpufreq_data *data = cpufreq_get_driver_data(); 77 struct device *dev; 78 int ret; 79 80 dev = get_cpu_device(policy->cpu); 81 if (!dev) 82 return -ENODEV; 83 84 struct dev_pm_opp *opp __free(put_opp) = 85 dev_pm_opp_find_freq_exact(dev, freq_khz * HZ_PER_KHZ, true); 86 if (IS_ERR(opp)) 87 return PTR_ERR(opp); 88 89 ret = dev_pm_opp_set_opp(dev, opp); 90 if (ret) 91 data->icc_dram_bw_scaling = false; 92 93 return ret; 94 } 95 96 static int tegra_cpufreq_init_cpufreq_table(struct cpufreq_policy *policy, 97 struct cpufreq_frequency_table *bpmp_lut, 98 struct cpufreq_frequency_table **opp_table) 99 { 100 struct tegra186_cpufreq_data *data = cpufreq_get_driver_data(); 101 struct cpufreq_frequency_table *freq_table = NULL; 102 struct cpufreq_frequency_table *pos; 103 struct device *cpu_dev; 104 unsigned long rate; 105 int ret, max_opps; 106 int j = 0; 107 108 cpu_dev = get_cpu_device(policy->cpu); 109 if (!cpu_dev) { 110 pr_err("%s: failed to get cpu%d device\n", __func__, policy->cpu); 111 return -ENODEV; 112 } 113 114 /* Initialize OPP table mentioned in operating-points-v2 property in DT */ 115 ret = dev_pm_opp_of_add_table_indexed(cpu_dev, 0); 116 if (ret) { 117 dev_err(cpu_dev, "Invalid or empty opp table in device tree\n"); 118 data->icc_dram_bw_scaling = false; 119 return ret; 120 } 121 122 max_opps = dev_pm_opp_get_opp_count(cpu_dev); 123 if (max_opps <= 0) { 124 dev_err(cpu_dev, "Failed to add OPPs\n"); 125 return max_opps; 126 } 127 128 /* Disable all opps and cross-validate against LUT later */ 129 for (rate = 0; ; rate++) { 130 struct dev_pm_opp *opp __free(put_opp) = 131 dev_pm_opp_find_freq_ceil(cpu_dev, &rate); 132 if (IS_ERR(opp)) 133 break; 134 135 dev_pm_opp_disable(cpu_dev, rate); 136 } 137 138 freq_table = kcalloc((max_opps + 1), sizeof(*freq_table), GFP_KERNEL); 139 if (!freq_table) 140 return -ENOMEM; 141 142 /* 143 * Cross check the frequencies from BPMP-FW LUT against the OPP's present in DT. 144 * Enable only those DT OPP's which are present in LUT also. 145 */ 146 cpufreq_for_each_valid_entry(pos, bpmp_lut) { 147 struct dev_pm_opp *opp __free(put_opp) = 148 dev_pm_opp_find_freq_exact(cpu_dev, pos->frequency * HZ_PER_KHZ, false); 149 if (IS_ERR(opp)) 150 continue; 151 152 ret = dev_pm_opp_enable(cpu_dev, pos->frequency * HZ_PER_KHZ); 153 if (ret < 0) 154 return ret; 155 156 freq_table[j].driver_data = pos->driver_data; 157 freq_table[j].frequency = pos->frequency; 158 j++; 159 } 160 161 freq_table[j].driver_data = pos->driver_data; 162 freq_table[j].frequency = CPUFREQ_TABLE_END; 163 164 *opp_table = &freq_table[0]; 165 166 dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus); 167 168 /* Prime interconnect data */ 169 tegra_cpufreq_set_bw(policy, freq_table[j - 1].frequency); 170 171 return ret; 172 } 173 174 static int tegra186_cpufreq_init(struct cpufreq_policy *policy) 175 { 176 struct tegra186_cpufreq_data *data = cpufreq_get_driver_data(); 177 unsigned int cluster = data->cpus[policy->cpu].bpmp_cluster_id; 178 struct cpufreq_frequency_table *freq_table; 179 struct cpufreq_frequency_table *bpmp_lut; 180 u32 cpu; 181 int ret; 182 183 policy->cpuinfo.transition_latency = 300 * 1000; 184 policy->driver_data = NULL; 185 186 /* set same policy for all cpus in a cluster */ 187 for (cpu = 0; cpu < ARRAY_SIZE(tegra186_cpus); cpu++) { 188 if (data->cpus[cpu].bpmp_cluster_id == cluster) 189 cpumask_set_cpu(cpu, policy->cpus); 190 } 191 192 bpmp_lut = data->clusters[cluster].bpmp_lut; 193 194 if (data->icc_dram_bw_scaling) { 195 ret = tegra_cpufreq_init_cpufreq_table(policy, bpmp_lut, &freq_table); 196 if (!ret) { 197 policy->freq_table = freq_table; 198 return 0; 199 } 200 } 201 202 data->icc_dram_bw_scaling = false; 203 policy->freq_table = bpmp_lut; 204 pr_info("OPP tables missing from DT, EMC frequency scaling disabled\n"); 205 206 return 0; 207 } 208 209 static int tegra186_cpufreq_set_target(struct cpufreq_policy *policy, 210 unsigned int index) 211 { 212 struct tegra186_cpufreq_data *data = cpufreq_get_driver_data(); 213 struct cpufreq_frequency_table *tbl = policy->freq_table + index; 214 unsigned int edvd_offset; 215 u32 edvd_val = tbl->driver_data; 216 u32 cpu; 217 218 for_each_cpu(cpu, policy->cpus) { 219 edvd_offset = data->cpus[cpu].edvd_offset; 220 writel(edvd_val, data->regs + edvd_offset); 221 } 222 223 if (data->icc_dram_bw_scaling) 224 tegra_cpufreq_set_bw(policy, tbl->frequency); 225 226 227 return 0; 228 } 229 230 static unsigned int tegra186_cpufreq_get(unsigned int cpu) 231 { 232 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); 233 struct tegra186_cpufreq_data *data = cpufreq_get_driver_data(); 234 struct tegra186_cpufreq_cluster *cluster; 235 unsigned int edvd_offset, cluster_id; 236 u32 ndiv; 237 238 if (!policy) 239 return 0; 240 241 edvd_offset = data->cpus[policy->cpu].edvd_offset; 242 ndiv = readl(data->regs + edvd_offset) & EDVD_CORE_VOLT_FREQ_F_MASK; 243 cluster_id = data->cpus[policy->cpu].bpmp_cluster_id; 244 cluster = &data->clusters[cluster_id]; 245 246 return (cluster->ref_clk_khz * ndiv) / cluster->div; 247 } 248 249 static struct cpufreq_driver tegra186_cpufreq_driver = { 250 .name = "tegra186", 251 .flags = CPUFREQ_HAVE_GOVERNOR_PER_POLICY | 252 CPUFREQ_NEED_INITIAL_FREQ_CHECK, 253 .get = tegra186_cpufreq_get, 254 .verify = cpufreq_generic_frequency_table_verify, 255 .target_index = tegra186_cpufreq_set_target, 256 .init = tegra186_cpufreq_init, 257 }; 258 259 static struct cpufreq_frequency_table *tegra_cpufreq_bpmp_read_lut( 260 struct platform_device *pdev, struct tegra_bpmp *bpmp, 261 struct tegra186_cpufreq_cluster *cluster, unsigned int cluster_id, 262 int *num_rates) 263 { 264 struct cpufreq_frequency_table *table; 265 struct mrq_cpu_vhint_request req; 266 struct tegra_bpmp_message msg; 267 struct cpu_vhint_data *data; 268 int err, i, j; 269 dma_addr_t phys; 270 void *virt; 271 272 virt = dma_alloc_coherent(bpmp->dev, sizeof(*data), &phys, 273 GFP_KERNEL); 274 if (!virt) 275 return ERR_PTR(-ENOMEM); 276 277 data = (struct cpu_vhint_data *)virt; 278 279 memset(&req, 0, sizeof(req)); 280 req.addr = phys; 281 req.cluster_id = cluster_id; 282 283 memset(&msg, 0, sizeof(msg)); 284 msg.mrq = MRQ_CPU_VHINT; 285 msg.tx.data = &req; 286 msg.tx.size = sizeof(req); 287 288 err = tegra_bpmp_transfer(bpmp, &msg); 289 if (err) { 290 table = ERR_PTR(err); 291 goto free; 292 } 293 if (msg.rx.ret) { 294 table = ERR_PTR(-EINVAL); 295 goto free; 296 } 297 298 *num_rates = 0; 299 for (i = data->vfloor; i <= data->vceil; i++) { 300 u16 ndiv = data->ndiv[i]; 301 302 if (ndiv < data->ndiv_min || ndiv > data->ndiv_max) 303 continue; 304 305 /* Only store lowest voltage index for each rate */ 306 if (i > 0 && ndiv == data->ndiv[i - 1]) 307 continue; 308 309 (*num_rates)++; 310 } 311 312 table = devm_kcalloc(&pdev->dev, *num_rates + 1, sizeof(*table), 313 GFP_KERNEL); 314 if (!table) { 315 table = ERR_PTR(-ENOMEM); 316 goto free; 317 } 318 319 cluster->ref_clk_khz = data->ref_clk_hz / 1000; 320 cluster->div = data->pdiv * data->mdiv; 321 322 for (i = data->vfloor, j = 0; i <= data->vceil; i++) { 323 struct cpufreq_frequency_table *point; 324 u16 ndiv = data->ndiv[i]; 325 u32 edvd_val = 0; 326 327 if (ndiv < data->ndiv_min || ndiv > data->ndiv_max) 328 continue; 329 330 /* Only store lowest voltage index for each rate */ 331 if (i > 0 && ndiv == data->ndiv[i - 1]) 332 continue; 333 334 edvd_val |= i << EDVD_CORE_VOLT_FREQ_V_SHIFT; 335 edvd_val |= ndiv << EDVD_CORE_VOLT_FREQ_F_SHIFT; 336 337 point = &table[j++]; 338 point->driver_data = edvd_val; 339 point->frequency = (cluster->ref_clk_khz * ndiv) / cluster->div; 340 } 341 342 table[j].frequency = CPUFREQ_TABLE_END; 343 344 free: 345 dma_free_coherent(bpmp->dev, sizeof(*data), virt, phys); 346 347 return table; 348 } 349 350 static int tegra186_cpufreq_probe(struct platform_device *pdev) 351 { 352 struct tegra186_cpufreq_data *data; 353 struct tegra_bpmp *bpmp; 354 struct device *cpu_dev; 355 unsigned int i = 0, err, edvd_offset; 356 int num_rates = 0; 357 u32 edvd_val, cpu; 358 359 data = devm_kzalloc(&pdev->dev, 360 struct_size(data, clusters, TEGRA186_NUM_CLUSTERS), 361 GFP_KERNEL); 362 if (!data) 363 return -ENOMEM; 364 365 data->cpus = tegra186_cpus; 366 367 bpmp = tegra_bpmp_get(&pdev->dev); 368 if (IS_ERR(bpmp)) 369 return PTR_ERR(bpmp); 370 371 data->regs = devm_platform_ioremap_resource(pdev, 0); 372 if (IS_ERR(data->regs)) { 373 err = PTR_ERR(data->regs); 374 goto put_bpmp; 375 } 376 377 for (i = 0; i < TEGRA186_NUM_CLUSTERS; i++) { 378 struct tegra186_cpufreq_cluster *cluster = &data->clusters[i]; 379 380 cluster->bpmp_lut = tegra_cpufreq_bpmp_read_lut(pdev, bpmp, cluster, i, &num_rates); 381 if (IS_ERR(cluster->bpmp_lut)) { 382 err = PTR_ERR(cluster->bpmp_lut); 383 goto put_bpmp; 384 } else if (!num_rates) { 385 err = -EINVAL; 386 goto put_bpmp; 387 } 388 389 for (cpu = 0; cpu < ARRAY_SIZE(tegra186_cpus); cpu++) { 390 if (data->cpus[cpu].bpmp_cluster_id == i) { 391 edvd_val = cluster->bpmp_lut[num_rates - 1].driver_data; 392 edvd_offset = data->cpus[cpu].edvd_offset; 393 writel(edvd_val, data->regs + edvd_offset); 394 } 395 } 396 } 397 398 tegra186_cpufreq_driver.driver_data = data; 399 400 /* Check for optional OPPv2 and interconnect paths on CPU0 to enable ICC scaling */ 401 cpu_dev = get_cpu_device(0); 402 if (!cpu_dev) { 403 err = -EPROBE_DEFER; 404 goto put_bpmp; 405 } 406 407 if (dev_pm_opp_of_get_opp_desc_node(cpu_dev)) { 408 err = dev_pm_opp_of_find_icc_paths(cpu_dev, NULL); 409 if (!err) 410 data->icc_dram_bw_scaling = true; 411 } 412 413 err = cpufreq_register_driver(&tegra186_cpufreq_driver); 414 415 put_bpmp: 416 tegra_bpmp_put(bpmp); 417 418 return err; 419 } 420 421 static void tegra186_cpufreq_remove(struct platform_device *pdev) 422 { 423 cpufreq_unregister_driver(&tegra186_cpufreq_driver); 424 } 425 426 static const struct of_device_id tegra186_cpufreq_of_match[] = { 427 { .compatible = "nvidia,tegra186-ccplex-cluster", }, 428 { } 429 }; 430 MODULE_DEVICE_TABLE(of, tegra186_cpufreq_of_match); 431 432 static struct platform_driver tegra186_cpufreq_platform_driver = { 433 .driver = { 434 .name = "tegra186-cpufreq", 435 .of_match_table = tegra186_cpufreq_of_match, 436 }, 437 .probe = tegra186_cpufreq_probe, 438 .remove = tegra186_cpufreq_remove, 439 }; 440 module_platform_driver(tegra186_cpufreq_platform_driver); 441 442 MODULE_AUTHOR("Mikko Perttunen <mperttunen@nvidia.com>"); 443 MODULE_DESCRIPTION("NVIDIA Tegra186 cpufreq driver"); 444 MODULE_LICENSE("GPL v2"); 445