1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * PSCI CPU idle driver. 4 * 5 * Copyright (C) 2019 ARM Ltd. 6 * Author: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com> 7 */ 8 9 #define pr_fmt(fmt) "CPUidle PSCI: " fmt 10 11 #include <linux/cpuhotplug.h> 12 #include <linux/cpu_cooling.h> 13 #include <linux/cpuidle.h> 14 #include <linux/cpumask.h> 15 #include <linux/cpu_pm.h> 16 #include <linux/kernel.h> 17 #include <linux/module.h> 18 #include <linux/of.h> 19 #include <linux/device/faux.h> 20 #include <linux/psci.h> 21 #include <linux/pm_domain.h> 22 #include <linux/pm_runtime.h> 23 #include <linux/slab.h> 24 #include <linux/string.h> 25 #include <linux/syscore_ops.h> 26 27 #include <asm/cpuidle.h> 28 #include <trace/events/power.h> 29 30 #include "cpuidle-psci.h" 31 #include "dt_idle_states.h" 32 #include "dt_idle_genpd.h" 33 34 struct psci_cpuidle_data { 35 u32 *psci_states; 36 struct device *dev; 37 }; 38 39 struct psci_cpuidle_domain_state { 40 struct generic_pm_domain *pd; 41 unsigned int state_idx; 42 u32 state; 43 }; 44 45 static DEFINE_PER_CPU_READ_MOSTLY(struct psci_cpuidle_data, psci_cpuidle_data); 46 static DEFINE_PER_CPU(struct psci_cpuidle_domain_state, psci_domain_state); 47 static bool psci_cpuidle_use_syscore; 48 49 void psci_set_domain_state(struct generic_pm_domain *pd, unsigned int state_idx, 50 u32 state) 51 { 52 struct psci_cpuidle_domain_state *ds = this_cpu_ptr(&psci_domain_state); 53 54 ds->pd = pd; 55 ds->state_idx = state_idx; 56 ds->state = state; 57 } 58 59 static inline void psci_clear_domain_state(void) 60 { 61 __this_cpu_write(psci_domain_state.state, 0); 62 } 63 64 static __cpuidle int __psci_enter_domain_idle_state(struct cpuidle_device *dev, 65 struct cpuidle_driver *drv, int idx, 66 bool s2idle) 67 { 68 struct psci_cpuidle_data *data = this_cpu_ptr(&psci_cpuidle_data); 69 u32 *states = data->psci_states; 70 struct device *pd_dev = data->dev; 71 struct psci_cpuidle_domain_state *ds; 72 u32 state = states[idx]; 73 int ret; 74 75 ret = cpu_pm_enter(); 76 if (ret) 77 return -1; 78 79 /* Do runtime PM to manage a hierarchical CPU toplogy. */ 80 if (s2idle) 81 dev_pm_genpd_suspend(pd_dev); 82 else 83 pm_runtime_put_sync_suspend(pd_dev); 84 85 ds = this_cpu_ptr(&psci_domain_state); 86 if (ds->state) 87 state = ds->state; 88 89 trace_psci_domain_idle_enter(dev->cpu, state, s2idle); 90 ret = psci_cpu_suspend_enter(state) ? -1 : idx; 91 trace_psci_domain_idle_exit(dev->cpu, state, s2idle); 92 93 if (s2idle) 94 dev_pm_genpd_resume(pd_dev); 95 else 96 pm_runtime_get_sync(pd_dev); 97 98 cpu_pm_exit(); 99 100 /* Correct domain-idlestate statistics if we failed to enter. */ 101 if (ret == -1 && ds->state) 102 pm_genpd_inc_rejected(ds->pd, ds->state_idx); 103 104 /* Clear the domain state to start fresh when back from idle. */ 105 psci_clear_domain_state(); 106 return ret; 107 } 108 109 static int psci_enter_domain_idle_state(struct cpuidle_device *dev, 110 struct cpuidle_driver *drv, int idx) 111 { 112 return __psci_enter_domain_idle_state(dev, drv, idx, false); 113 } 114 115 static int psci_enter_s2idle_domain_idle_state(struct cpuidle_device *dev, 116 struct cpuidle_driver *drv, 117 int idx) 118 { 119 return __psci_enter_domain_idle_state(dev, drv, idx, true); 120 } 121 122 static int psci_idle_cpuhp_up(unsigned int cpu) 123 { 124 struct device *pd_dev = __this_cpu_read(psci_cpuidle_data.dev); 125 126 if (pd_dev) { 127 if (!IS_ENABLED(CONFIG_PREEMPT_RT)) 128 pm_runtime_get_sync(pd_dev); 129 else 130 dev_pm_genpd_resume(pd_dev); 131 } 132 133 return 0; 134 } 135 136 static int psci_idle_cpuhp_down(unsigned int cpu) 137 { 138 struct device *pd_dev = __this_cpu_read(psci_cpuidle_data.dev); 139 140 if (pd_dev) { 141 if (!IS_ENABLED(CONFIG_PREEMPT_RT)) 142 pm_runtime_put_sync(pd_dev); 143 else 144 dev_pm_genpd_suspend(pd_dev); 145 146 /* Clear domain state to start fresh at next online. */ 147 psci_clear_domain_state(); 148 } 149 150 return 0; 151 } 152 153 static void psci_idle_syscore_switch(bool suspend) 154 { 155 bool cleared = false; 156 struct device *dev; 157 int cpu; 158 159 for_each_possible_cpu(cpu) { 160 dev = per_cpu_ptr(&psci_cpuidle_data, cpu)->dev; 161 162 if (dev && suspend) { 163 dev_pm_genpd_suspend(dev); 164 } else if (dev) { 165 dev_pm_genpd_resume(dev); 166 167 /* Account for userspace having offlined a CPU. */ 168 if (pm_runtime_status_suspended(dev)) 169 pm_runtime_set_active(dev); 170 171 /* Clear domain state to re-start fresh. */ 172 if (!cleared) { 173 psci_clear_domain_state(); 174 cleared = true; 175 } 176 } 177 } 178 } 179 180 static int psci_idle_syscore_suspend(void *data) 181 { 182 psci_idle_syscore_switch(true); 183 return 0; 184 } 185 186 static void psci_idle_syscore_resume(void *data) 187 { 188 psci_idle_syscore_switch(false); 189 } 190 191 static const struct syscore_ops psci_idle_syscore_ops = { 192 .suspend = psci_idle_syscore_suspend, 193 .resume = psci_idle_syscore_resume, 194 }; 195 196 static struct syscore psci_idle_syscore = { 197 .ops = &psci_idle_syscore_ops, 198 }; 199 200 static void psci_idle_init_syscore(void) 201 { 202 if (psci_cpuidle_use_syscore) 203 register_syscore(&psci_idle_syscore); 204 } 205 206 static void psci_idle_init_cpuhp(void) 207 { 208 int err; 209 210 err = cpuhp_setup_state_nocalls(CPUHP_AP_CPU_PM_STARTING, 211 "cpuidle/psci:online", 212 psci_idle_cpuhp_up, 213 psci_idle_cpuhp_down); 214 if (err) 215 pr_warn("Failed %d while setup cpuhp state\n", err); 216 } 217 218 static __cpuidle int psci_enter_idle_state(struct cpuidle_device *dev, 219 struct cpuidle_driver *drv, int idx) 220 { 221 u32 *state = __this_cpu_read(psci_cpuidle_data.psci_states); 222 223 return CPU_PM_CPU_IDLE_ENTER_PARAM_RCU(psci_cpu_suspend_enter, idx, state[idx]); 224 } 225 226 static const struct of_device_id psci_idle_state_match[] = { 227 { .compatible = "arm,idle-state", 228 .data = psci_enter_idle_state }, 229 { }, 230 }; 231 232 int psci_dt_parse_state_node(struct device_node *np, u32 *state) 233 { 234 int err = of_property_read_u32(np, "arm,psci-suspend-param", state); 235 236 if (err) { 237 pr_warn("%pOF missing arm,psci-suspend-param property\n", np); 238 return err; 239 } 240 241 if (!psci_power_state_is_valid(*state)) { 242 pr_warn("Invalid PSCI power state %#x\n", *state); 243 return -EINVAL; 244 } 245 246 return 0; 247 } 248 249 static int psci_dt_cpu_init_topology(struct cpuidle_driver *drv, 250 struct psci_cpuidle_data *data, 251 unsigned int state_count, int cpu) 252 { 253 /* Currently limit the hierarchical topology to be used in OSI mode. */ 254 if (!psci_has_osi_support()) 255 return 0; 256 257 data->dev = dt_idle_attach_cpu(cpu, "psci"); 258 if (IS_ERR_OR_NULL(data->dev)) 259 return PTR_ERR_OR_ZERO(data->dev); 260 261 psci_cpuidle_use_syscore = true; 262 263 /* 264 * Using the deepest state for the CPU to trigger a potential selection 265 * of a shared state for the domain, assumes the domain states are all 266 * deeper states. On PREEMPT_RT the hierarchical topology is limited to 267 * s2ram and s2idle. 268 */ 269 drv->states[state_count - 1].enter_s2idle = psci_enter_s2idle_domain_idle_state; 270 if (!IS_ENABLED(CONFIG_PREEMPT_RT)) 271 drv->states[state_count - 1].enter = psci_enter_domain_idle_state; 272 273 return 0; 274 } 275 276 static int psci_dt_cpu_init_idle(struct device *dev, struct cpuidle_driver *drv, 277 struct device_node *cpu_node, 278 unsigned int state_count, int cpu) 279 { 280 int i, ret = 0; 281 u32 *psci_states; 282 struct device_node *state_node; 283 struct psci_cpuidle_data *data = per_cpu_ptr(&psci_cpuidle_data, cpu); 284 285 state_count++; /* Add WFI state too */ 286 psci_states = devm_kcalloc(dev, state_count, sizeof(*psci_states), 287 GFP_KERNEL); 288 if (!psci_states) 289 return -ENOMEM; 290 291 for (i = 1; i < state_count; i++) { 292 state_node = of_get_cpu_state_node(cpu_node, i - 1); 293 if (!state_node) 294 break; 295 296 ret = psci_dt_parse_state_node(state_node, &psci_states[i]); 297 of_node_put(state_node); 298 299 if (ret) 300 return ret; 301 302 pr_debug("psci-power-state %#x index %d\n", psci_states[i], i); 303 } 304 305 if (i != state_count) 306 return -ENODEV; 307 308 /* Initialize optional data, used for the hierarchical topology. */ 309 ret = psci_dt_cpu_init_topology(drv, data, state_count, cpu); 310 if (ret < 0) 311 return ret; 312 313 /* Idle states parsed correctly, store them in the per-cpu struct. */ 314 data->psci_states = psci_states; 315 return 0; 316 } 317 318 static int psci_cpu_init_idle(struct device *dev, struct cpuidle_driver *drv, 319 unsigned int cpu, unsigned int state_count) 320 { 321 struct device_node *cpu_node; 322 int ret; 323 324 /* 325 * If the PSCI cpu_suspend function hook has not been initialized 326 * idle states must not be enabled, so bail out 327 */ 328 if (!psci_ops.cpu_suspend) 329 return -EOPNOTSUPP; 330 331 cpu_node = of_cpu_device_node_get(cpu); 332 if (!cpu_node) 333 return -ENODEV; 334 335 ret = psci_dt_cpu_init_idle(dev, drv, cpu_node, state_count, cpu); 336 337 of_node_put(cpu_node); 338 339 return ret; 340 } 341 342 static void psci_cpu_deinit_idle(int cpu) 343 { 344 struct psci_cpuidle_data *data = per_cpu_ptr(&psci_cpuidle_data, cpu); 345 346 dt_idle_detach_cpu(data->dev); 347 psci_cpuidle_use_syscore = false; 348 } 349 350 static int psci_idle_init_cpu(struct device *dev, int cpu) 351 { 352 struct cpuidle_driver *drv; 353 struct device_node *cpu_node; 354 const char *enable_method; 355 int ret = 0; 356 357 cpu_node = of_cpu_device_node_get(cpu); 358 if (!cpu_node) 359 return -ENODEV; 360 361 /* 362 * Check whether the enable-method for the cpu is PSCI, fail 363 * if it is not. 364 */ 365 enable_method = of_get_property(cpu_node, "enable-method", NULL); 366 if (!enable_method || (strcmp(enable_method, "psci"))) 367 ret = -ENODEV; 368 369 of_node_put(cpu_node); 370 if (ret) 371 return ret; 372 373 drv = devm_kzalloc(dev, sizeof(*drv), GFP_KERNEL); 374 if (!drv) 375 return -ENOMEM; 376 377 drv->name = "psci_idle"; 378 drv->owner = THIS_MODULE; 379 drv->cpumask = (struct cpumask *)cpumask_of(cpu); 380 381 /* 382 * PSCI idle states relies on architectural WFI to be represented as 383 * state index 0. 384 */ 385 drv->states[0].enter = psci_enter_idle_state; 386 drv->states[0].exit_latency = 1; 387 drv->states[0].target_residency = 1; 388 drv->states[0].power_usage = UINT_MAX; 389 strscpy(drv->states[0].name, "WFI"); 390 strscpy(drv->states[0].desc, "ARM WFI"); 391 392 /* 393 * If no DT idle states are detected (ret == 0) let the driver 394 * initialization fail accordingly since there is no reason to 395 * initialize the idle driver if only wfi is supported, the 396 * default archictectural back-end already executes wfi 397 * on idle entry. 398 */ 399 ret = dt_init_idle_driver(drv, psci_idle_state_match, 1); 400 if (ret <= 0) 401 return ret ? : -ENODEV; 402 403 /* 404 * Initialize PSCI idle states. 405 */ 406 ret = psci_cpu_init_idle(dev, drv, cpu, ret); 407 if (ret) { 408 pr_err("CPU %d failed to PSCI idle\n", cpu); 409 return ret; 410 } 411 412 ret = cpuidle_register(drv, NULL); 413 if (ret) 414 goto deinit; 415 416 cpuidle_cooling_register(drv); 417 418 return 0; 419 deinit: 420 psci_cpu_deinit_idle(cpu); 421 return ret; 422 } 423 424 /* 425 * psci_idle_probe - Initializes PSCI cpuidle driver 426 * 427 * Initializes PSCI cpuidle driver for all present CPUs, if any CPU fails 428 * to register cpuidle driver then rollback to cancel all CPUs 429 * registration. 430 */ 431 static int psci_cpuidle_probe(struct faux_device *fdev) 432 { 433 int cpu, ret; 434 struct cpuidle_driver *drv; 435 struct cpuidle_device *dev; 436 437 for_each_present_cpu(cpu) { 438 ret = psci_idle_init_cpu(&fdev->dev, cpu); 439 if (ret) 440 goto out_fail; 441 } 442 443 psci_idle_init_syscore(); 444 psci_idle_init_cpuhp(); 445 return 0; 446 447 out_fail: 448 while (--cpu >= 0) { 449 dev = per_cpu(cpuidle_devices, cpu); 450 drv = cpuidle_get_cpu_driver(dev); 451 cpuidle_unregister(drv); 452 psci_cpu_deinit_idle(cpu); 453 } 454 455 return ret; 456 } 457 458 static struct faux_device_ops psci_cpuidle_ops = { 459 .probe = psci_cpuidle_probe, 460 }; 461 462 static bool __init dt_idle_state_present(void) 463 { 464 struct device_node *cpu_node __free(device_node) = 465 of_cpu_device_node_get(cpumask_first(cpu_possible_mask)); 466 if (!cpu_node) 467 return false; 468 469 struct device_node *state_node __free(device_node) = 470 of_get_cpu_state_node(cpu_node, 0); 471 if (!state_node) 472 return false; 473 474 return !!of_match_node(psci_idle_state_match, state_node); 475 } 476 477 static int __init psci_idle_init(void) 478 { 479 struct faux_device *fdev; 480 481 if (!dt_idle_state_present()) 482 return 0; 483 484 fdev = faux_device_create("psci-cpuidle", NULL, &psci_cpuidle_ops); 485 if (!fdev) { 486 pr_err("Failed to create psci-cpuidle device\n"); 487 return -ENODEV; 488 } 489 490 return 0; 491 } 492 device_initcall(psci_idle_init); 493