1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Apple SoC CPU cluster performance state driver 4 * 5 * Copyright The Asahi Linux Contributors 6 * 7 * Based on scpi-cpufreq.c 8 */ 9 10 #include <linux/bitfield.h> 11 #include <linux/bitops.h> 12 #include <linux/cpu.h> 13 #include <linux/cpufreq.h> 14 #include <linux/cpumask.h> 15 #include <linux/delay.h> 16 #include <linux/err.h> 17 #include <linux/io.h> 18 #include <linux/iopoll.h> 19 #include <linux/module.h> 20 #include <linux/of.h> 21 #include <linux/of_address.h> 22 #include <linux/pm_opp.h> 23 #include <linux/slab.h> 24 25 #define APPLE_DVFS_CMD 0x20 26 #define APPLE_DVFS_CMD_BUSY BIT(31) 27 #define APPLE_DVFS_CMD_SET BIT(25) 28 #define APPLE_DVFS_CMD_PS1_S5L8960X GENMASK(24, 22) 29 #define APPLE_DVFS_CMD_PS1_S5L8960X_SHIFT 22 30 #define APPLE_DVFS_CMD_PS2 GENMASK(15, 12) 31 #define APPLE_DVFS_CMD_PS1 GENMASK(4, 0) 32 #define APPLE_DVFS_CMD_PS1_SHIFT 0 33 34 /* Same timebase as CPU counter (24MHz) */ 35 #define APPLE_DVFS_LAST_CHG_TIME 0x38 36 37 /* 38 * Apple ran out of bits and had to shift this in T8112... 39 */ 40 #define APPLE_DVFS_STATUS 0x50 41 #define APPLE_DVFS_STATUS_CUR_PS_S5L8960X GENMASK(5, 3) 42 #define APPLE_DVFS_STATUS_CUR_PS_SHIFT_S5L8960X 3 43 #define APPLE_DVFS_STATUS_TGT_PS_S5L8960X GENMASK(2, 0) 44 #define APPLE_DVFS_STATUS_CUR_PS_T8103 GENMASK(7, 4) 45 #define APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8103 4 46 #define APPLE_DVFS_STATUS_TGT_PS_T8103 GENMASK(3, 0) 47 #define APPLE_DVFS_STATUS_CUR_PS_T8112 GENMASK(9, 5) 48 #define APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8112 5 49 #define APPLE_DVFS_STATUS_TGT_PS_T8112 GENMASK(4, 0) 50 51 /* 52 * Div is +1, base clock is 12MHz on existing SoCs. 53 * For documentation purposes. We use the OPP table to 54 * get the frequency. 55 */ 56 #define APPLE_DVFS_PLL_STATUS 0xc0 57 #define APPLE_DVFS_PLL_FACTOR 0xc8 58 #define APPLE_DVFS_PLL_FACTOR_MULT GENMASK(31, 16) 59 #define APPLE_DVFS_PLL_FACTOR_DIV GENMASK(15, 0) 60 61 #define APPLE_DVFS_TRANSITION_TIMEOUT 400 62 63 struct apple_soc_cpufreq_info { 64 bool has_ps2; 65 u64 max_pstate; 66 u64 cur_pstate_mask; 67 u64 cur_pstate_shift; 68 u64 ps1_mask; 69 u64 ps1_shift; 70 }; 71 72 struct apple_cpu_priv { 73 struct device *cpu_dev; 74 void __iomem *reg_base; 75 const struct apple_soc_cpufreq_info *info; 76 }; 77 78 static struct cpufreq_driver apple_soc_cpufreq_driver; 79 80 static const struct apple_soc_cpufreq_info soc_s5l8960x_info = { 81 .has_ps2 = false, 82 .max_pstate = 7, 83 .cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_S5L8960X, 84 .cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_S5L8960X, 85 .ps1_mask = APPLE_DVFS_CMD_PS1_S5L8960X, 86 .ps1_shift = APPLE_DVFS_CMD_PS1_S5L8960X_SHIFT, 87 }; 88 89 static const struct apple_soc_cpufreq_info soc_t8103_info = { 90 .has_ps2 = true, 91 .max_pstate = 15, 92 .cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8103, 93 .cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8103, 94 .ps1_mask = APPLE_DVFS_CMD_PS1, 95 .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT, 96 }; 97 98 static const struct apple_soc_cpufreq_info soc_t8112_info = { 99 .has_ps2 = false, 100 .max_pstate = 31, 101 .cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8112, 102 .cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8112, 103 .ps1_mask = APPLE_DVFS_CMD_PS1, 104 .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT, 105 }; 106 107 static const struct apple_soc_cpufreq_info soc_default_info = { 108 .has_ps2 = false, 109 .max_pstate = 15, 110 .cur_pstate_mask = 0, /* fallback */ 111 .ps1_mask = APPLE_DVFS_CMD_PS1, 112 .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT, 113 }; 114 115 static const struct of_device_id apple_soc_cpufreq_of_match[] __maybe_unused = { 116 { 117 .compatible = "apple,s5l8960x-cluster-cpufreq", 118 .data = &soc_s5l8960x_info, 119 }, 120 { 121 .compatible = "apple,t8103-cluster-cpufreq", 122 .data = &soc_t8103_info, 123 }, 124 { 125 .compatible = "apple,t8112-cluster-cpufreq", 126 .data = &soc_t8112_info, 127 }, 128 { 129 .compatible = "apple,cluster-cpufreq", 130 .data = &soc_default_info, 131 }, 132 {} 133 }; 134 135 static unsigned int apple_soc_cpufreq_get_rate(unsigned int cpu) 136 { 137 struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); 138 struct apple_cpu_priv *priv = policy->driver_data; 139 struct cpufreq_frequency_table *p; 140 unsigned int pstate; 141 142 if (priv->info->cur_pstate_mask) { 143 u32 reg = readl_relaxed(priv->reg_base + APPLE_DVFS_STATUS); 144 145 pstate = (reg & priv->info->cur_pstate_mask) >> priv->info->cur_pstate_shift; 146 } else { 147 /* 148 * For the fallback case we might not know the layout of DVFS_STATUS, 149 * so just use the command register value (which ignores boost limitations). 150 */ 151 u64 reg = readq_relaxed(priv->reg_base + APPLE_DVFS_CMD); 152 153 pstate = FIELD_GET(APPLE_DVFS_CMD_PS1, reg); 154 } 155 156 cpufreq_for_each_valid_entry(p, policy->freq_table) 157 if (p->driver_data == pstate) 158 return p->frequency; 159 160 dev_err(priv->cpu_dev, "could not find frequency for pstate %d\n", 161 pstate); 162 return 0; 163 } 164 165 static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy, 166 unsigned int index) 167 { 168 struct apple_cpu_priv *priv = policy->driver_data; 169 unsigned int pstate = policy->freq_table[index].driver_data; 170 u64 reg; 171 172 /* Fallback for newer SoCs */ 173 if (index > priv->info->max_pstate) 174 index = priv->info->max_pstate; 175 176 if (readq_poll_timeout_atomic(priv->reg_base + APPLE_DVFS_CMD, reg, 177 !(reg & APPLE_DVFS_CMD_BUSY), 2, 178 APPLE_DVFS_TRANSITION_TIMEOUT)) { 179 return -EIO; 180 } 181 182 reg &= ~priv->info->ps1_mask; 183 reg |= pstate << priv->info->ps1_shift; 184 if (priv->info->has_ps2) { 185 reg &= ~APPLE_DVFS_CMD_PS2; 186 reg |= FIELD_PREP(APPLE_DVFS_CMD_PS2, pstate); 187 } 188 reg |= APPLE_DVFS_CMD_SET; 189 190 writeq_relaxed(reg, priv->reg_base + APPLE_DVFS_CMD); 191 192 return 0; 193 } 194 195 static unsigned int apple_soc_cpufreq_fast_switch(struct cpufreq_policy *policy, 196 unsigned int target_freq) 197 { 198 if (apple_soc_cpufreq_set_target(policy, policy->cached_resolved_idx) < 0) 199 return 0; 200 201 return policy->freq_table[policy->cached_resolved_idx].frequency; 202 } 203 204 static int apple_soc_cpufreq_find_cluster(struct cpufreq_policy *policy, 205 void __iomem **reg_base, 206 const struct apple_soc_cpufreq_info **info) 207 { 208 struct of_phandle_args args; 209 const struct of_device_id *match; 210 int ret = 0; 211 212 ret = of_perf_domain_get_sharing_cpumask(policy->cpu, "performance-domains", 213 "#performance-domain-cells", 214 policy->cpus, &args); 215 if (ret < 0) 216 return ret; 217 218 match = of_match_node(apple_soc_cpufreq_of_match, args.np); 219 of_node_put(args.np); 220 if (!match) 221 return -ENODEV; 222 223 *info = match->data; 224 225 *reg_base = of_iomap(args.np, 0); 226 if (!*reg_base) 227 return -ENOMEM; 228 229 return 0; 230 } 231 232 static struct freq_attr *apple_soc_cpufreq_hw_attr[] = { 233 &cpufreq_freq_attr_scaling_available_freqs, 234 NULL, /* Filled in below if boost is enabled */ 235 NULL, 236 }; 237 238 static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) 239 { 240 int ret, i; 241 unsigned int transition_latency; 242 void __iomem *reg_base; 243 struct device *cpu_dev; 244 struct apple_cpu_priv *priv; 245 const struct apple_soc_cpufreq_info *info; 246 struct cpufreq_frequency_table *freq_table; 247 248 cpu_dev = get_cpu_device(policy->cpu); 249 if (!cpu_dev) { 250 pr_err("failed to get cpu%d device\n", policy->cpu); 251 return -ENODEV; 252 } 253 254 ret = dev_pm_opp_of_add_table(cpu_dev); 255 if (ret < 0) { 256 dev_err(cpu_dev, "%s: failed to add OPP table: %d\n", __func__, ret); 257 return ret; 258 } 259 260 ret = apple_soc_cpufreq_find_cluster(policy, ®_base, &info); 261 if (ret) { 262 dev_err(cpu_dev, "%s: failed to get cluster info: %d\n", __func__, ret); 263 return ret; 264 } 265 266 ret = dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus); 267 if (ret) { 268 dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n", __func__, ret); 269 goto out_iounmap; 270 } 271 272 ret = dev_pm_opp_get_opp_count(cpu_dev); 273 if (ret <= 0) { 274 dev_dbg(cpu_dev, "OPP table is not ready, deferring probe\n"); 275 ret = -EPROBE_DEFER; 276 goto out_free_opp; 277 } 278 279 priv = kzalloc(sizeof(*priv), GFP_KERNEL); 280 if (!priv) { 281 ret = -ENOMEM; 282 goto out_free_opp; 283 } 284 285 ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table); 286 if (ret) { 287 dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret); 288 goto out_free_priv; 289 } 290 291 /* Get OPP levels (p-state indexes) and stash them in driver_data */ 292 for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { 293 unsigned long rate = freq_table[i].frequency * 1000 + 999; 294 struct dev_pm_opp *opp = dev_pm_opp_find_freq_floor(cpu_dev, &rate); 295 296 if (IS_ERR(opp)) { 297 ret = PTR_ERR(opp); 298 goto out_free_cpufreq_table; 299 } 300 freq_table[i].driver_data = dev_pm_opp_get_level(opp); 301 dev_pm_opp_put(opp); 302 } 303 304 priv->cpu_dev = cpu_dev; 305 priv->reg_base = reg_base; 306 priv->info = info; 307 policy->driver_data = priv; 308 policy->freq_table = freq_table; 309 310 transition_latency = dev_pm_opp_get_max_transition_latency(cpu_dev); 311 if (!transition_latency) 312 transition_latency = APPLE_DVFS_TRANSITION_TIMEOUT * NSEC_PER_USEC; 313 314 policy->cpuinfo.transition_latency = transition_latency; 315 policy->dvfs_possible_from_any_cpu = true; 316 policy->fast_switch_possible = true; 317 policy->suspend_freq = freq_table[0].frequency; 318 319 if (policy_has_boost_freq(policy)) { 320 ret = cpufreq_enable_boost_support(); 321 if (ret) { 322 dev_warn(cpu_dev, "failed to enable boost: %d\n", ret); 323 } else { 324 apple_soc_cpufreq_hw_attr[1] = &cpufreq_freq_attr_scaling_boost_freqs; 325 apple_soc_cpufreq_driver.boost_enabled = true; 326 } 327 } 328 329 return 0; 330 331 out_free_cpufreq_table: 332 dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table); 333 out_free_priv: 334 kfree(priv); 335 out_free_opp: 336 dev_pm_opp_remove_all_dynamic(cpu_dev); 337 out_iounmap: 338 iounmap(reg_base); 339 return ret; 340 } 341 342 static void apple_soc_cpufreq_exit(struct cpufreq_policy *policy) 343 { 344 struct apple_cpu_priv *priv = policy->driver_data; 345 346 dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table); 347 dev_pm_opp_remove_all_dynamic(priv->cpu_dev); 348 iounmap(priv->reg_base); 349 kfree(priv); 350 } 351 352 static struct cpufreq_driver apple_soc_cpufreq_driver = { 353 .name = "apple-cpufreq", 354 .flags = CPUFREQ_HAVE_GOVERNOR_PER_POLICY | 355 CPUFREQ_NEED_INITIAL_FREQ_CHECK | CPUFREQ_IS_COOLING_DEV, 356 .verify = cpufreq_generic_frequency_table_verify, 357 .get = apple_soc_cpufreq_get_rate, 358 .init = apple_soc_cpufreq_init, 359 .exit = apple_soc_cpufreq_exit, 360 .target_index = apple_soc_cpufreq_set_target, 361 .fast_switch = apple_soc_cpufreq_fast_switch, 362 .register_em = cpufreq_register_em_with_opp, 363 .attr = apple_soc_cpufreq_hw_attr, 364 .suspend = cpufreq_generic_suspend, 365 }; 366 367 static int __init apple_soc_cpufreq_module_init(void) 368 { 369 if (!of_machine_is_compatible("apple,arm-platform")) 370 return -ENODEV; 371 372 return cpufreq_register_driver(&apple_soc_cpufreq_driver); 373 } 374 module_init(apple_soc_cpufreq_module_init); 375 376 static void __exit apple_soc_cpufreq_module_exit(void) 377 { 378 cpufreq_unregister_driver(&apple_soc_cpufreq_driver); 379 } 380 module_exit(apple_soc_cpufreq_module_exit); 381 382 MODULE_DEVICE_TABLE(of, apple_soc_cpufreq_of_match); 383 MODULE_AUTHOR("Hector Martin <marcan@marcan.st>"); 384 MODULE_DESCRIPTION("Apple SoC CPU cluster DVFS driver"); 385 MODULE_LICENSE("GPL"); 386