1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * CPU idle driver for Tegra CPUs 4 * 5 * Copyright (c) 2010-2013, NVIDIA Corporation. 6 * Copyright (c) 2011 Google, Inc. 7 * Author: Colin Cross <ccross@android.com> 8 * Gary King <gking@nvidia.com> 9 * 10 * Rework for 3.3 by Peter De Schrijver <pdeschrijver@nvidia.com> 11 * 12 * Tegra20/124 driver unification by Dmitry Osipenko <digetx@gmail.com> 13 */ 14 15 #define pr_fmt(fmt) "tegra-cpuidle: " fmt 16 17 #include <linux/atomic.h> 18 #include <linux/cpuidle.h> 19 #include <linux/cpumask.h> 20 #include <linux/cpu_pm.h> 21 #include <linux/delay.h> 22 #include <linux/errno.h> 23 #include <linux/platform_device.h> 24 #include <linux/types.h> 25 26 #include <linux/clk/tegra.h> 27 #include <linux/firmware/trusted_foundations.h> 28 29 #include <soc/tegra/cpuidle.h> 30 #include <soc/tegra/flowctrl.h> 31 #include <soc/tegra/fuse.h> 32 #include <soc/tegra/irq.h> 33 #include <soc/tegra/pm.h> 34 #include <soc/tegra/pmc.h> 35 36 #include <asm/cpuidle.h> 37 #include <asm/firmware.h> 38 #include <asm/smp_plat.h> 39 #include <asm/suspend.h> 40 41 enum tegra_state { 42 TEGRA_C1, 43 TEGRA_C7, 44 TEGRA_CC6, 45 TEGRA_STATE_COUNT, 46 }; 47 48 static atomic_t tegra_idle_barrier; 49 static atomic_t tegra_abort_flag; 50 51 static void tegra_cpuidle_report_cpus_state(void) 52 { 53 unsigned long cpu, lcpu, csr; 54 55 for_each_cpu(lcpu, cpu_possible_mask) { 56 cpu = cpu_logical_map(lcpu); 57 csr = flowctrl_read_cpu_csr(cpu); 58 59 pr_err("cpu%lu: online=%d flowctrl_csr=0x%08lx\n", 60 cpu, cpu_online(lcpu), csr); 61 } 62 } 63 64 static int tegra_cpuidle_wait_for_secondary_cpus_parking(void) 65 { 66 unsigned int retries = 3; 67 68 while (retries--) { 69 unsigned int delay_us = 10; 70 unsigned int timeout_us = 500 * 1000 / delay_us; 71 72 /* 73 * The primary CPU0 core shall wait for the secondaries 74 * shutdown in order to power-off CPU's cluster safely. 75 * The timeout value depends on the current CPU frequency, 76 * it takes about 40-150us in average and over 1000us in 77 * a worst case scenario. 78 */ 79 do { 80 if (tegra_cpu_rail_off_ready()) 81 return 0; 82 83 udelay(delay_us); 84 85 } while (timeout_us--); 86 87 pr_err("secondary CPU taking too long to park\n"); 88 89 tegra_cpuidle_report_cpus_state(); 90 } 91 92 pr_err("timed out waiting secondaries to park\n"); 93 94 return -ETIMEDOUT; 95 } 96 97 static void tegra_cpuidle_unpark_secondary_cpus(void) 98 { 99 unsigned int cpu, lcpu; 100 101 for_each_cpu(lcpu, cpu_online_mask) { 102 cpu = cpu_logical_map(lcpu); 103 104 if (cpu > 0) { 105 tegra_enable_cpu_clock(cpu); 106 tegra_cpu_out_of_reset(cpu); 107 flowctrl_write_cpu_halt(cpu, 0); 108 } 109 } 110 } 111 112 static int tegra_cpuidle_cc6_enter(unsigned int cpu) 113 { 114 int ret; 115 116 if (cpu > 0) { 117 ret = cpu_suspend(cpu, tegra_pm_park_secondary_cpu); 118 } else { 119 ret = tegra_cpuidle_wait_for_secondary_cpus_parking(); 120 if (!ret) 121 ret = tegra_pm_enter_lp2(); 122 123 tegra_cpuidle_unpark_secondary_cpus(); 124 } 125 126 return ret; 127 } 128 129 static int tegra_cpuidle_c7_enter(void) 130 { 131 int err; 132 133 err = call_firmware_op(prepare_idle, TF_PM_MODE_LP2_NOFLUSH_L2); 134 if (err && err != -ENOSYS) 135 return err; 136 137 return cpu_suspend(0, tegra30_pm_secondary_cpu_suspend); 138 } 139 140 static int tegra_cpuidle_coupled_barrier(struct cpuidle_device *dev) 141 { 142 if (tegra_pending_sgi()) { 143 /* 144 * CPU got local interrupt that will be lost after GIC's 145 * shutdown because GIC driver doesn't save/restore the 146 * pending SGI state across CPU cluster PM. Abort and retry 147 * next time. 148 */ 149 atomic_set(&tegra_abort_flag, 1); 150 } 151 152 cpuidle_coupled_parallel_barrier(dev, &tegra_idle_barrier); 153 154 if (atomic_read(&tegra_abort_flag)) { 155 cpuidle_coupled_parallel_barrier(dev, &tegra_idle_barrier); 156 atomic_set(&tegra_abort_flag, 0); 157 return -EINTR; 158 } 159 160 return 0; 161 } 162 163 static __cpuidle int tegra_cpuidle_state_enter(struct cpuidle_device *dev, 164 int index, unsigned int cpu) 165 { 166 int err; 167 168 /* 169 * CC6 state is the "CPU cluster power-off" state. In order to 170 * enter this state, at first the secondary CPU cores need to be 171 * parked into offline mode, then the last CPU should clean out 172 * remaining dirty cache lines into DRAM and trigger Flow Controller 173 * logic that turns off the cluster's power domain (which includes 174 * CPU cores, GIC and L2 cache). 175 */ 176 if (index == TEGRA_CC6) { 177 err = tegra_cpuidle_coupled_barrier(dev); 178 if (err) 179 return err; 180 } 181 182 local_fiq_disable(); 183 tegra_pm_set_cpu_in_lp2(); 184 cpu_pm_enter(); 185 186 ct_cpuidle_enter(); 187 188 switch (index) { 189 case TEGRA_C7: 190 err = tegra_cpuidle_c7_enter(); 191 break; 192 193 case TEGRA_CC6: 194 err = tegra_cpuidle_cc6_enter(cpu); 195 break; 196 197 default: 198 err = -EINVAL; 199 break; 200 } 201 202 ct_cpuidle_exit(); 203 204 cpu_pm_exit(); 205 tegra_pm_clear_cpu_in_lp2(); 206 local_fiq_enable(); 207 208 return err ?: index; 209 } 210 211 static int tegra_cpuidle_adjust_state_index(int index, unsigned int cpu) 212 { 213 /* 214 * On Tegra30 CPU0 can't be power-gated separately from secondary 215 * cores because it gates the whole CPU cluster. 216 */ 217 if (cpu > 0 || index != TEGRA_C7 || tegra_get_chip_id() != TEGRA30) 218 return index; 219 220 /* put CPU0 into C1 if C7 is requested and secondaries are online */ 221 if (!IS_ENABLED(CONFIG_PM_SLEEP) || num_online_cpus() > 1) 222 index = TEGRA_C1; 223 else 224 index = TEGRA_CC6; 225 226 return index; 227 } 228 229 static __cpuidle int tegra_cpuidle_enter(struct cpuidle_device *dev, 230 struct cpuidle_driver *drv, 231 int index) 232 { 233 bool do_rcu = drv->states[index].flags & CPUIDLE_FLAG_RCU_IDLE; 234 unsigned int cpu = cpu_logical_map(dev->cpu); 235 int ret; 236 237 index = tegra_cpuidle_adjust_state_index(index, cpu); 238 if (dev->states_usage[index].disable) 239 return -1; 240 241 if (index == TEGRA_C1) { 242 if (do_rcu) 243 ct_cpuidle_enter(); 244 ret = arm_cpuidle_simple_enter(dev, drv, index); 245 if (do_rcu) 246 ct_cpuidle_exit(); 247 } else 248 ret = tegra_cpuidle_state_enter(dev, index, cpu); 249 250 if (ret < 0) { 251 if (ret != -EINTR || index != TEGRA_CC6) 252 pr_err_once("failed to enter state %d err: %d\n", 253 index, ret); 254 index = -1; 255 } else { 256 index = ret; 257 } 258 259 return index; 260 } 261 262 static int tegra114_enter_s2idle(struct cpuidle_device *dev, 263 struct cpuidle_driver *drv, 264 int index) 265 { 266 tegra_cpuidle_enter(dev, drv, index); 267 268 return 0; 269 } 270 271 /* 272 * The previous versions of Tegra CPUIDLE driver used a different "legacy" 273 * terminology for naming of the idling states, while this driver uses the 274 * new terminology. 275 * 276 * Mapping of the old terms into the new ones: 277 * 278 * Old | New 279 * --------- 280 * LP3 | C1 (CPU core clock gating) 281 * LP2 | C7 (CPU core power gating) 282 * LP2 | CC6 (CPU cluster power gating) 283 * 284 * Note that that the older CPUIDLE driver versions didn't explicitly 285 * differentiate the LP2 states because these states either used the same 286 * code path or because CC6 wasn't supported. 287 */ 288 static struct cpuidle_driver tegra_idle_driver = { 289 .name = "tegra_idle", 290 .states = { 291 [TEGRA_C1] = ARM_CPUIDLE_WFI_STATE_PWR(600), 292 [TEGRA_C7] = { 293 .enter = tegra_cpuidle_enter, 294 .exit_latency = 2000, 295 .target_residency = 2200, 296 .power_usage = 100, 297 .flags = CPUIDLE_FLAG_TIMER_STOP | 298 CPUIDLE_FLAG_RCU_IDLE, 299 .name = "C7", 300 .desc = "CPU core powered off", 301 }, 302 [TEGRA_CC6] = { 303 .enter = tegra_cpuidle_enter, 304 .exit_latency = 5000, 305 .target_residency = 10000, 306 .power_usage = 0, 307 .flags = CPUIDLE_FLAG_TIMER_STOP | 308 CPUIDLE_FLAG_RCU_IDLE | 309 CPUIDLE_FLAG_COUPLED, 310 .name = "CC6", 311 .desc = "CPU cluster powered off", 312 }, 313 }, 314 .state_count = TEGRA_STATE_COUNT, 315 .safe_state_index = TEGRA_C1, 316 }; 317 318 static inline void tegra_cpuidle_disable_state(enum tegra_state state) 319 { 320 cpuidle_driver_state_disabled(&tegra_idle_driver, state, true); 321 } 322 323 /* 324 * Tegra20 HW appears to have a bug such that PCIe device interrupts, whether 325 * they are legacy IRQs or MSI, are lost when CC6 is enabled. To work around 326 * this, simply disable CC6 if the PCI driver and DT node are both enabled. 327 */ 328 void tegra_cpuidle_pcie_irqs_in_use(void) 329 { 330 struct cpuidle_state *state_cc6 = &tegra_idle_driver.states[TEGRA_CC6]; 331 332 if ((state_cc6->flags & CPUIDLE_FLAG_UNUSABLE) || 333 tegra_get_chip_id() != TEGRA20) 334 return; 335 336 pr_info("disabling CC6 state, since PCIe IRQs are in use\n"); 337 tegra_cpuidle_disable_state(TEGRA_CC6); 338 } 339 340 static void tegra_cpuidle_setup_tegra114_c7_state(void) 341 { 342 struct cpuidle_state *s = &tegra_idle_driver.states[TEGRA_C7]; 343 344 s->enter_s2idle = tegra114_enter_s2idle; 345 s->target_residency = 1000; 346 s->exit_latency = 500; 347 } 348 349 static int tegra_cpuidle_probe(struct platform_device *pdev) 350 { 351 if (tegra_pmc_get_suspend_mode() == TEGRA_SUSPEND_NOT_READY) 352 return -EPROBE_DEFER; 353 354 /* LP2 could be disabled in device-tree */ 355 if (tegra_pmc_get_suspend_mode() < TEGRA_SUSPEND_LP2) 356 tegra_cpuidle_disable_state(TEGRA_CC6); 357 358 /* 359 * Required suspend-resume functionality, which is provided by the 360 * Tegra-arch core and PMC driver, is unavailable if PM-sleep option 361 * is disabled. 362 */ 363 if (!IS_ENABLED(CONFIG_PM_SLEEP)) { 364 tegra_cpuidle_disable_state(TEGRA_C7); 365 tegra_cpuidle_disable_state(TEGRA_CC6); 366 } 367 368 /* 369 * Generic WFI state (also known as C1 or LP3) and the coupled CPU 370 * cluster power-off (CC6 or LP2) states are common for all Tegra SoCs. 371 */ 372 switch (tegra_get_chip_id()) { 373 case TEGRA20: 374 /* Tegra20 isn't capable to power-off individual CPU cores */ 375 tegra_cpuidle_disable_state(TEGRA_C7); 376 break; 377 378 case TEGRA30: 379 break; 380 381 case TEGRA114: 382 case TEGRA124: 383 tegra_cpuidle_setup_tegra114_c7_state(); 384 385 /* coupled CC6 (LP2) state isn't implemented yet */ 386 tegra_cpuidle_disable_state(TEGRA_CC6); 387 break; 388 389 default: 390 return -EINVAL; 391 } 392 393 return cpuidle_register(&tegra_idle_driver, cpu_possible_mask); 394 } 395 396 static struct platform_driver tegra_cpuidle_driver = { 397 .probe = tegra_cpuidle_probe, 398 .driver = { 399 .name = "tegra-cpuidle", 400 }, 401 }; 402 builtin_platform_driver(tegra_cpuidle_driver); 403