1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Apple SoC CPU cluster performance state driver 4 * 5 * Copyright The Asahi Linux Contributors 6 * 7 * Based on scpi-cpufreq.c 8 */ 9 10 #include <linux/bitfield.h> 11 #include <linux/bitops.h> 12 #include <linux/cpu.h> 13 #include <linux/cpufreq.h> 14 #include <linux/cpumask.h> 15 #include <linux/delay.h> 16 #include <linux/err.h> 17 #include <linux/io.h> 18 #include <linux/iopoll.h> 19 #include <linux/module.h> 20 #include <linux/of.h> 21 #include <linux/of_address.h> 22 #include <linux/pm_opp.h> 23 #include <linux/slab.h> 24 25 #define APPLE_DVFS_CMD 0x20 26 #define APPLE_DVFS_CMD_BUSY BIT(31) 27 #define APPLE_DVFS_CMD_SET BIT(25) 28 #define APPLE_DVFS_CMD_PS1_S5L8960X GENMASK(24, 22) 29 #define APPLE_DVFS_CMD_PS1_S5L8960X_SHIFT 22 30 #define APPLE_DVFS_CMD_PS2 GENMASK(15, 12) 31 #define APPLE_DVFS_CMD_PS1 GENMASK(4, 0) 32 #define APPLE_DVFS_CMD_PS1_SHIFT 0 33 34 /* Same timebase as CPU counter (24MHz) */ 35 #define APPLE_DVFS_LAST_CHG_TIME 0x38 36 37 /* 38 * Apple ran out of bits and had to shift this in T8112... 39 */ 40 #define APPLE_DVFS_STATUS 0x50 41 #define APPLE_DVFS_STATUS_CUR_PS_S5L8960X GENMASK(5, 3) 42 #define APPLE_DVFS_STATUS_CUR_PS_SHIFT_S5L8960X 3 43 #define APPLE_DVFS_STATUS_TGT_PS_S5L8960X GENMASK(2, 0) 44 #define APPLE_DVFS_STATUS_CUR_PS_T8103 GENMASK(7, 4) 45 #define APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8103 4 46 #define APPLE_DVFS_STATUS_TGT_PS_T8103 GENMASK(3, 0) 47 #define APPLE_DVFS_STATUS_CUR_PS_T8112 GENMASK(9, 5) 48 #define APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8112 5 49 #define APPLE_DVFS_STATUS_TGT_PS_T8112 GENMASK(4, 0) 50 51 /* 52 * Div is +1, base clock is 12MHz on existing SoCs. 53 * For documentation purposes. We use the OPP table to 54 * get the frequency. 55 */ 56 #define APPLE_DVFS_PLL_STATUS 0xc0 57 #define APPLE_DVFS_PLL_FACTOR 0xc8 58 #define APPLE_DVFS_PLL_FACTOR_MULT GENMASK(31, 16) 59 #define APPLE_DVFS_PLL_FACTOR_DIV GENMASK(15, 0) 60 61 #define APPLE_DVFS_TRANSITION_TIMEOUT 400 62 63 struct apple_soc_cpufreq_info { 64 bool has_ps2; 65 u64 max_pstate; 66 u64 cur_pstate_mask; 67 u64 cur_pstate_shift; 68 u64 ps1_mask; 69 u64 ps1_shift; 70 }; 71 72 struct apple_cpu_priv { 73 struct device *cpu_dev; 74 void __iomem *reg_base; 75 const struct apple_soc_cpufreq_info *info; 76 }; 77 78 static struct cpufreq_driver apple_soc_cpufreq_driver; 79 80 static const struct apple_soc_cpufreq_info soc_s5l8960x_info = { 81 .has_ps2 = false, 82 .max_pstate = 7, 83 .cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_S5L8960X, 84 .cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_S5L8960X, 85 .ps1_mask = APPLE_DVFS_CMD_PS1_S5L8960X, 86 .ps1_shift = APPLE_DVFS_CMD_PS1_S5L8960X_SHIFT, 87 }; 88 89 static const struct apple_soc_cpufreq_info soc_t8103_info = { 90 .has_ps2 = true, 91 .max_pstate = 15, 92 .cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8103, 93 .cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8103, 94 .ps1_mask = APPLE_DVFS_CMD_PS1, 95 .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT, 96 }; 97 98 static const struct apple_soc_cpufreq_info soc_t8112_info = { 99 .has_ps2 = false, 100 .max_pstate = 31, 101 .cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8112, 102 .cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8112, 103 .ps1_mask = APPLE_DVFS_CMD_PS1, 104 .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT, 105 }; 106 107 static const struct apple_soc_cpufreq_info soc_default_info = { 108 .has_ps2 = false, 109 .max_pstate = 15, 110 .cur_pstate_mask = 0, /* fallback */ 111 .ps1_mask = APPLE_DVFS_CMD_PS1, 112 .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT, 113 }; 114 115 static const struct of_device_id apple_soc_cpufreq_of_match[] __maybe_unused = { 116 { 117 .compatible = "apple,s5l8960x-cluster-cpufreq", 118 .data = &soc_s5l8960x_info, 119 }, 120 { 121 .compatible = "apple,t8103-cluster-cpufreq", 122 .data = &soc_t8103_info, 123 }, 124 { 125 .compatible = "apple,t8112-cluster-cpufreq", 126 .data = &soc_t8112_info, 127 }, 128 { 129 .compatible = "apple,cluster-cpufreq", 130 .data = &soc_default_info, 131 }, 132 {} 133 }; 134 135 static unsigned int apple_soc_cpufreq_get_rate(unsigned int cpu) 136 { 137 struct cpufreq_policy *policy; 138 struct apple_cpu_priv *priv; 139 struct cpufreq_frequency_table *p; 140 unsigned int pstate; 141 142 policy = cpufreq_cpu_get_raw(cpu); 143 if (unlikely(!policy)) 144 return 0; 145 146 priv = policy->driver_data; 147 148 if (priv->info->cur_pstate_mask) { 149 u32 reg = readl_relaxed(priv->reg_base + APPLE_DVFS_STATUS); 150 151 pstate = (reg & priv->info->cur_pstate_mask) >> priv->info->cur_pstate_shift; 152 } else { 153 /* 154 * For the fallback case we might not know the layout of DVFS_STATUS, 155 * so just use the command register value (which ignores boost limitations). 156 */ 157 u64 reg = readq_relaxed(priv->reg_base + APPLE_DVFS_CMD); 158 159 pstate = FIELD_GET(APPLE_DVFS_CMD_PS1, reg); 160 } 161 162 cpufreq_for_each_valid_entry(p, policy->freq_table) 163 if (p->driver_data == pstate) 164 return p->frequency; 165 166 dev_err(priv->cpu_dev, "could not find frequency for pstate %d\n", 167 pstate); 168 return 0; 169 } 170 171 static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy, 172 unsigned int index) 173 { 174 struct apple_cpu_priv *priv = policy->driver_data; 175 unsigned int pstate = policy->freq_table[index].driver_data; 176 u64 reg; 177 178 /* Fallback for newer SoCs */ 179 if (index > priv->info->max_pstate) 180 index = priv->info->max_pstate; 181 182 if (readq_poll_timeout_atomic(priv->reg_base + APPLE_DVFS_CMD, reg, 183 !(reg & APPLE_DVFS_CMD_BUSY), 2, 184 APPLE_DVFS_TRANSITION_TIMEOUT)) { 185 return -EIO; 186 } 187 188 reg &= ~priv->info->ps1_mask; 189 reg |= pstate << priv->info->ps1_shift; 190 if (priv->info->has_ps2) { 191 reg &= ~APPLE_DVFS_CMD_PS2; 192 reg |= FIELD_PREP(APPLE_DVFS_CMD_PS2, pstate); 193 } 194 reg |= APPLE_DVFS_CMD_SET; 195 196 writeq_relaxed(reg, priv->reg_base + APPLE_DVFS_CMD); 197 198 return 0; 199 } 200 201 static unsigned int apple_soc_cpufreq_fast_switch(struct cpufreq_policy *policy, 202 unsigned int target_freq) 203 { 204 if (apple_soc_cpufreq_set_target(policy, policy->cached_resolved_idx) < 0) 205 return 0; 206 207 return policy->freq_table[policy->cached_resolved_idx].frequency; 208 } 209 210 static int apple_soc_cpufreq_find_cluster(struct cpufreq_policy *policy, 211 void __iomem **reg_base, 212 const struct apple_soc_cpufreq_info **info) 213 { 214 struct of_phandle_args args; 215 const struct of_device_id *match; 216 int ret = 0; 217 218 ret = of_perf_domain_get_sharing_cpumask(policy->cpu, "performance-domains", 219 "#performance-domain-cells", 220 policy->cpus, &args); 221 if (ret < 0) 222 return ret; 223 224 match = of_match_node(apple_soc_cpufreq_of_match, args.np); 225 of_node_put(args.np); 226 if (!match) 227 return -ENODEV; 228 229 *info = match->data; 230 231 *reg_base = of_iomap(args.np, 0); 232 if (!*reg_base) 233 return -ENOMEM; 234 235 return 0; 236 } 237 238 static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) 239 { 240 int ret, i; 241 unsigned int transition_latency; 242 void __iomem *reg_base; 243 struct device *cpu_dev; 244 struct apple_cpu_priv *priv; 245 const struct apple_soc_cpufreq_info *info; 246 struct cpufreq_frequency_table *freq_table; 247 248 cpu_dev = get_cpu_device(policy->cpu); 249 if (!cpu_dev) { 250 pr_err("failed to get cpu%d device\n", policy->cpu); 251 return -ENODEV; 252 } 253 254 ret = dev_pm_opp_of_add_table(cpu_dev); 255 if (ret < 0) { 256 dev_err(cpu_dev, "%s: failed to add OPP table: %d\n", __func__, ret); 257 return ret; 258 } 259 260 ret = apple_soc_cpufreq_find_cluster(policy, ®_base, &info); 261 if (ret) { 262 dev_err(cpu_dev, "%s: failed to get cluster info: %d\n", __func__, ret); 263 return ret; 264 } 265 266 ret = dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus); 267 if (ret) { 268 dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n", __func__, ret); 269 goto out_iounmap; 270 } 271 272 ret = dev_pm_opp_get_opp_count(cpu_dev); 273 if (ret <= 0) { 274 dev_dbg(cpu_dev, "OPP table is not ready, deferring probe\n"); 275 ret = -EPROBE_DEFER; 276 goto out_free_opp; 277 } 278 279 priv = kzalloc(sizeof(*priv), GFP_KERNEL); 280 if (!priv) { 281 ret = -ENOMEM; 282 goto out_free_opp; 283 } 284 285 ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table); 286 if (ret) { 287 dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret); 288 goto out_free_priv; 289 } 290 291 /* Get OPP levels (p-state indexes) and stash them in driver_data */ 292 for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { 293 unsigned long rate = freq_table[i].frequency * 1000 + 999; 294 struct dev_pm_opp *opp = dev_pm_opp_find_freq_floor(cpu_dev, &rate); 295 296 if (IS_ERR(opp)) { 297 ret = PTR_ERR(opp); 298 goto out_free_cpufreq_table; 299 } 300 freq_table[i].driver_data = dev_pm_opp_get_level(opp); 301 dev_pm_opp_put(opp); 302 } 303 304 priv->cpu_dev = cpu_dev; 305 priv->reg_base = reg_base; 306 priv->info = info; 307 policy->driver_data = priv; 308 policy->freq_table = freq_table; 309 310 transition_latency = dev_pm_opp_get_max_transition_latency(cpu_dev); 311 if (!transition_latency) 312 transition_latency = APPLE_DVFS_TRANSITION_TIMEOUT * NSEC_PER_USEC; 313 314 policy->cpuinfo.transition_latency = transition_latency; 315 policy->dvfs_possible_from_any_cpu = true; 316 policy->fast_switch_possible = true; 317 policy->suspend_freq = freq_table[0].frequency; 318 319 return 0; 320 321 out_free_cpufreq_table: 322 dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table); 323 out_free_priv: 324 kfree(priv); 325 out_free_opp: 326 dev_pm_opp_remove_all_dynamic(cpu_dev); 327 out_iounmap: 328 iounmap(reg_base); 329 return ret; 330 } 331 332 static void apple_soc_cpufreq_exit(struct cpufreq_policy *policy) 333 { 334 struct apple_cpu_priv *priv = policy->driver_data; 335 336 dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table); 337 dev_pm_opp_remove_all_dynamic(priv->cpu_dev); 338 iounmap(priv->reg_base); 339 kfree(priv); 340 } 341 342 static struct cpufreq_driver apple_soc_cpufreq_driver = { 343 .name = "apple-cpufreq", 344 .flags = CPUFREQ_HAVE_GOVERNOR_PER_POLICY | 345 CPUFREQ_NEED_INITIAL_FREQ_CHECK | CPUFREQ_IS_COOLING_DEV, 346 .verify = cpufreq_generic_frequency_table_verify, 347 .get = apple_soc_cpufreq_get_rate, 348 .init = apple_soc_cpufreq_init, 349 .exit = apple_soc_cpufreq_exit, 350 .target_index = apple_soc_cpufreq_set_target, 351 .fast_switch = apple_soc_cpufreq_fast_switch, 352 .register_em = cpufreq_register_em_with_opp, 353 .set_boost = cpufreq_boost_set_sw, 354 .suspend = cpufreq_generic_suspend, 355 }; 356 357 static int __init apple_soc_cpufreq_module_init(void) 358 { 359 if (!of_machine_is_compatible("apple,arm-platform")) 360 return -ENODEV; 361 362 return cpufreq_register_driver(&apple_soc_cpufreq_driver); 363 } 364 module_init(apple_soc_cpufreq_module_init); 365 366 static void __exit apple_soc_cpufreq_module_exit(void) 367 { 368 cpufreq_unregister_driver(&apple_soc_cpufreq_driver); 369 } 370 module_exit(apple_soc_cpufreq_module_exit); 371 372 MODULE_DEVICE_TABLE(of, apple_soc_cpufreq_of_match); 373 MODULE_AUTHOR("Hector Martin <marcan@marcan.st>"); 374 MODULE_DESCRIPTION("Apple SoC CPU cluster DVFS driver"); 375 MODULE_LICENSE("GPL"); 376