1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Apple SoC CPU cluster performance state driver 4 * 5 * Copyright The Asahi Linux Contributors 6 * 7 * Based on scpi-cpufreq.c 8 */ 9 10 #include <linux/bitfield.h> 11 #include <linux/bitops.h> 12 #include <linux/cpu.h> 13 #include <linux/cpufreq.h> 14 #include <linux/cpumask.h> 15 #include <linux/delay.h> 16 #include <linux/err.h> 17 #include <linux/io.h> 18 #include <linux/iopoll.h> 19 #include <linux/module.h> 20 #include <linux/of.h> 21 #include <linux/of_address.h> 22 #include <linux/pm_opp.h> 23 #include <linux/slab.h> 24 25 #define APPLE_DVFS_CMD 0x20 26 #define APPLE_DVFS_CMD_BUSY BIT(31) 27 #define APPLE_DVFS_CMD_SET BIT(25) 28 #define APPLE_DVFS_CMD_PS1_S5L8960X GENMASK(24, 22) 29 #define APPLE_DVFS_CMD_PS1_S5L8960X_SHIFT 22 30 #define APPLE_DVFS_CMD_PS2 GENMASK(15, 12) 31 #define APPLE_DVFS_CMD_PS1 GENMASK(4, 0) 32 #define APPLE_DVFS_CMD_PS1_SHIFT 0 33 34 /* Same timebase as CPU counter (24MHz) */ 35 #define APPLE_DVFS_LAST_CHG_TIME 0x38 36 37 /* 38 * Apple ran out of bits and had to shift this in T8112... 39 */ 40 #define APPLE_DVFS_STATUS 0x50 41 #define APPLE_DVFS_STATUS_CUR_PS_S5L8960X GENMASK(5, 3) 42 #define APPLE_DVFS_STATUS_CUR_PS_SHIFT_S5L8960X 3 43 #define APPLE_DVFS_STATUS_TGT_PS_S5L8960X GENMASK(2, 0) 44 #define APPLE_DVFS_STATUS_CUR_PS_T8103 GENMASK(7, 4) 45 #define APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8103 4 46 #define APPLE_DVFS_STATUS_TGT_PS_T8103 GENMASK(3, 0) 47 #define APPLE_DVFS_STATUS_CUR_PS_T8112 GENMASK(9, 5) 48 #define APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8112 5 49 #define APPLE_DVFS_STATUS_TGT_PS_T8112 GENMASK(4, 0) 50 51 /* 52 * Div is +1, base clock is 12MHz on existing SoCs. 53 * For documentation purposes. We use the OPP table to 54 * get the frequency. 55 */ 56 #define APPLE_DVFS_PLL_STATUS 0xc0 57 #define APPLE_DVFS_PLL_FACTOR 0xc8 58 #define APPLE_DVFS_PLL_FACTOR_MULT GENMASK(31, 16) 59 #define APPLE_DVFS_PLL_FACTOR_DIV GENMASK(15, 0) 60 61 #define APPLE_DVFS_TRANSITION_TIMEOUT 400 62 63 struct apple_soc_cpufreq_info { 64 bool has_ps2; 65 u64 max_pstate; 66 u64 cur_pstate_mask; 67 u64 cur_pstate_shift; 68 u64 ps1_mask; 69 u64 ps1_shift; 70 }; 71 72 struct apple_cpu_priv { 73 struct device *cpu_dev; 74 void __iomem *reg_base; 75 const struct apple_soc_cpufreq_info *info; 76 }; 77 78 static struct cpufreq_driver apple_soc_cpufreq_driver; 79 80 static const struct apple_soc_cpufreq_info soc_s5l8960x_info = { 81 .has_ps2 = false, 82 .max_pstate = 7, 83 .cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_S5L8960X, 84 .cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_S5L8960X, 85 .ps1_mask = APPLE_DVFS_CMD_PS1_S5L8960X, 86 .ps1_shift = APPLE_DVFS_CMD_PS1_S5L8960X_SHIFT, 87 }; 88 89 static const struct apple_soc_cpufreq_info soc_t8103_info = { 90 .has_ps2 = true, 91 .max_pstate = 15, 92 .cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8103, 93 .cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8103, 94 .ps1_mask = APPLE_DVFS_CMD_PS1, 95 .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT, 96 }; 97 98 static const struct apple_soc_cpufreq_info soc_t8112_info = { 99 .has_ps2 = false, 100 .max_pstate = 31, 101 .cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8112, 102 .cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8112, 103 .ps1_mask = APPLE_DVFS_CMD_PS1, 104 .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT, 105 }; 106 107 static const struct apple_soc_cpufreq_info soc_default_info = { 108 .has_ps2 = false, 109 .max_pstate = 15, 110 .cur_pstate_mask = 0, /* fallback */ 111 .ps1_mask = APPLE_DVFS_CMD_PS1, 112 .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT, 113 }; 114 115 static const struct of_device_id apple_soc_cpufreq_of_match[] __maybe_unused = { 116 { 117 .compatible = "apple,s5l8960x-cluster-cpufreq", 118 .data = &soc_s5l8960x_info, 119 }, 120 { 121 .compatible = "apple,t8103-cluster-cpufreq", 122 .data = &soc_t8103_info, 123 }, 124 { 125 .compatible = "apple,t8112-cluster-cpufreq", 126 .data = &soc_t8112_info, 127 }, 128 { 129 .compatible = "apple,cluster-cpufreq", 130 .data = &soc_default_info, 131 }, 132 {} 133 }; 134 135 static unsigned int apple_soc_cpufreq_get_rate(unsigned int cpu) 136 { 137 struct cpufreq_policy *policy; 138 struct apple_cpu_priv *priv; 139 struct cpufreq_frequency_table *p; 140 unsigned int pstate; 141 142 policy = cpufreq_cpu_get_raw(cpu); 143 if (unlikely(!policy)) 144 return 0; 145 146 priv = policy->driver_data; 147 148 if (priv->info->cur_pstate_mask) { 149 u32 reg = readl_relaxed(priv->reg_base + APPLE_DVFS_STATUS); 150 151 pstate = (reg & priv->info->cur_pstate_mask) >> priv->info->cur_pstate_shift; 152 } else { 153 /* 154 * For the fallback case we might not know the layout of DVFS_STATUS, 155 * so just use the command register value (which ignores boost limitations). 156 */ 157 u64 reg = readq_relaxed(priv->reg_base + APPLE_DVFS_CMD); 158 159 pstate = FIELD_GET(APPLE_DVFS_CMD_PS1, reg); 160 } 161 162 cpufreq_for_each_valid_entry(p, policy->freq_table) 163 if (p->driver_data == pstate) 164 return p->frequency; 165 166 dev_err(priv->cpu_dev, "could not find frequency for pstate %d\n", 167 pstate); 168 return 0; 169 } 170 171 static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy, 172 unsigned int index) 173 { 174 struct apple_cpu_priv *priv = policy->driver_data; 175 unsigned int pstate = policy->freq_table[index].driver_data; 176 u64 reg; 177 178 /* Fallback for newer SoCs */ 179 if (index > priv->info->max_pstate) 180 index = priv->info->max_pstate; 181 182 if (readq_poll_timeout_atomic(priv->reg_base + APPLE_DVFS_CMD, reg, 183 !(reg & APPLE_DVFS_CMD_BUSY), 2, 184 APPLE_DVFS_TRANSITION_TIMEOUT)) { 185 return -EIO; 186 } 187 188 reg &= ~priv->info->ps1_mask; 189 reg |= pstate << priv->info->ps1_shift; 190 if (priv->info->has_ps2) 191 FIELD_MODIFY(APPLE_DVFS_CMD_PS2, ®, pstate); 192 reg |= APPLE_DVFS_CMD_SET; 193 194 writeq_relaxed(reg, priv->reg_base + APPLE_DVFS_CMD); 195 196 return 0; 197 } 198 199 static unsigned int apple_soc_cpufreq_fast_switch(struct cpufreq_policy *policy, 200 unsigned int target_freq) 201 { 202 if (apple_soc_cpufreq_set_target(policy, policy->cached_resolved_idx) < 0) 203 return 0; 204 205 return policy->freq_table[policy->cached_resolved_idx].frequency; 206 } 207 208 static int apple_soc_cpufreq_find_cluster(struct cpufreq_policy *policy, 209 void __iomem **reg_base, 210 const struct apple_soc_cpufreq_info **info) 211 { 212 struct of_phandle_args args; 213 const struct of_device_id *match; 214 int ret = 0; 215 216 ret = of_perf_domain_get_sharing_cpumask(policy->cpu, "performance-domains", 217 "#performance-domain-cells", 218 policy->cpus, &args); 219 if (ret < 0) 220 return ret; 221 222 match = of_match_node(apple_soc_cpufreq_of_match, args.np); 223 of_node_put(args.np); 224 if (!match) 225 return -ENODEV; 226 227 *info = match->data; 228 229 *reg_base = of_iomap(args.np, 0); 230 if (!*reg_base) 231 return -ENOMEM; 232 233 return 0; 234 } 235 236 static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) 237 { 238 int ret, i; 239 unsigned int transition_latency; 240 void __iomem *reg_base; 241 struct device *cpu_dev; 242 struct apple_cpu_priv *priv; 243 const struct apple_soc_cpufreq_info *info; 244 struct cpufreq_frequency_table *freq_table; 245 246 cpu_dev = get_cpu_device(policy->cpu); 247 if (!cpu_dev) { 248 pr_err("failed to get cpu%d device\n", policy->cpu); 249 return -ENODEV; 250 } 251 252 ret = dev_pm_opp_of_add_table(cpu_dev); 253 if (ret < 0) { 254 dev_err(cpu_dev, "%s: failed to add OPP table: %d\n", __func__, ret); 255 return ret; 256 } 257 258 ret = apple_soc_cpufreq_find_cluster(policy, ®_base, &info); 259 if (ret) { 260 dev_err(cpu_dev, "%s: failed to get cluster info: %d\n", __func__, ret); 261 return ret; 262 } 263 264 ret = dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus); 265 if (ret) { 266 dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n", __func__, ret); 267 goto out_iounmap; 268 } 269 270 ret = dev_pm_opp_get_opp_count(cpu_dev); 271 if (ret <= 0) { 272 dev_dbg(cpu_dev, "OPP table is not ready, deferring probe\n"); 273 ret = -EPROBE_DEFER; 274 goto out_free_opp; 275 } 276 277 priv = kzalloc_obj(*priv); 278 if (!priv) { 279 ret = -ENOMEM; 280 goto out_free_opp; 281 } 282 283 ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table); 284 if (ret) { 285 dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret); 286 goto out_free_priv; 287 } 288 289 /* Get OPP levels (p-state indexes) and stash them in driver_data */ 290 for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { 291 unsigned long rate = freq_table[i].frequency * 1000 + 999; 292 struct dev_pm_opp *opp = dev_pm_opp_find_freq_floor(cpu_dev, &rate); 293 294 if (IS_ERR(opp)) { 295 ret = PTR_ERR(opp); 296 goto out_free_cpufreq_table; 297 } 298 freq_table[i].driver_data = dev_pm_opp_get_level(opp); 299 dev_pm_opp_put(opp); 300 } 301 302 priv->cpu_dev = cpu_dev; 303 priv->reg_base = reg_base; 304 priv->info = info; 305 policy->driver_data = priv; 306 policy->freq_table = freq_table; 307 308 transition_latency = dev_pm_opp_get_max_transition_latency(cpu_dev); 309 if (!transition_latency) 310 transition_latency = APPLE_DVFS_TRANSITION_TIMEOUT * NSEC_PER_USEC; 311 312 policy->cpuinfo.transition_latency = transition_latency; 313 policy->dvfs_possible_from_any_cpu = true; 314 policy->fast_switch_possible = true; 315 policy->suspend_freq = freq_table[0].frequency; 316 317 return 0; 318 319 out_free_cpufreq_table: 320 dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table); 321 out_free_priv: 322 kfree(priv); 323 out_free_opp: 324 dev_pm_opp_remove_all_dynamic(cpu_dev); 325 out_iounmap: 326 iounmap(reg_base); 327 return ret; 328 } 329 330 static void apple_soc_cpufreq_exit(struct cpufreq_policy *policy) 331 { 332 struct apple_cpu_priv *priv = policy->driver_data; 333 334 dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table); 335 dev_pm_opp_remove_all_dynamic(priv->cpu_dev); 336 iounmap(priv->reg_base); 337 kfree(priv); 338 } 339 340 static struct cpufreq_driver apple_soc_cpufreq_driver = { 341 .name = "apple-cpufreq", 342 .flags = CPUFREQ_HAVE_GOVERNOR_PER_POLICY | 343 CPUFREQ_NEED_INITIAL_FREQ_CHECK | CPUFREQ_IS_COOLING_DEV, 344 .verify = cpufreq_generic_frequency_table_verify, 345 .get = apple_soc_cpufreq_get_rate, 346 .init = apple_soc_cpufreq_init, 347 .exit = apple_soc_cpufreq_exit, 348 .target_index = apple_soc_cpufreq_set_target, 349 .fast_switch = apple_soc_cpufreq_fast_switch, 350 .register_em = cpufreq_register_em_with_opp, 351 .set_boost = cpufreq_boost_set_sw, 352 .suspend = cpufreq_generic_suspend, 353 }; 354 355 static int __init apple_soc_cpufreq_module_init(void) 356 { 357 if (!of_machine_is_compatible("apple,arm-platform")) 358 return -ENODEV; 359 360 return cpufreq_register_driver(&apple_soc_cpufreq_driver); 361 } 362 module_init(apple_soc_cpufreq_module_init); 363 364 static void __exit apple_soc_cpufreq_module_exit(void) 365 { 366 cpufreq_unregister_driver(&apple_soc_cpufreq_driver); 367 } 368 module_exit(apple_soc_cpufreq_module_exit); 369 370 MODULE_DEVICE_TABLE(of, apple_soc_cpufreq_of_match); 371 MODULE_AUTHOR("Hector Martin <marcan@marcan.st>"); 372 MODULE_DESCRIPTION("Apple SoC CPU cluster DVFS driver"); 373 MODULE_LICENSE("GPL"); 374