1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * CPU idle driver for Tegra CPUs 4 * 5 * Copyright (c) 2010-2013, NVIDIA Corporation. 6 * Copyright (c) 2011 Google, Inc. 7 * Author: Colin Cross <ccross@android.com> 8 * Gary King <gking@nvidia.com> 9 * 10 * Rework for 3.3 by Peter De Schrijver <pdeschrijver@nvidia.com> 11 * 12 * Tegra20/124 driver unification by Dmitry Osipenko <digetx@gmail.com> 13 */ 14 15 #define pr_fmt(fmt) "tegra-cpuidle: " fmt 16 17 #include <linux/atomic.h> 18 #include <linux/cpuidle.h> 19 #include <linux/cpumask.h> 20 #include <linux/cpu_pm.h> 21 #include <linux/delay.h> 22 #include <linux/errno.h> 23 #include <linux/platform_device.h> 24 #include <linux/types.h> 25 26 #include <linux/clk/tegra.h> 27 #include <linux/firmware/trusted_foundations.h> 28 29 #include <soc/tegra/cpuidle.h> 30 #include <soc/tegra/flowctrl.h> 31 #include <soc/tegra/fuse.h> 32 #include <soc/tegra/irq.h> 33 #include <soc/tegra/pm.h> 34 #include <soc/tegra/pmc.h> 35 36 #include <asm/cpuidle.h> 37 #include <asm/firmware.h> 38 #include <asm/smp_plat.h> 39 #include <asm/suspend.h> 40 41 enum tegra_state { 42 TEGRA_C1, 43 TEGRA_C7, 44 TEGRA_CC6, 45 TEGRA_STATE_COUNT, 46 }; 47 48 static atomic_t tegra_idle_barrier; 49 static atomic_t tegra_abort_flag; 50 51 static inline bool tegra_cpuidle_using_firmware(void) 52 { 53 return firmware_ops->prepare_idle && firmware_ops->do_idle; 54 } 55 56 static void tegra_cpuidle_report_cpus_state(void) 57 { 58 unsigned long cpu, lcpu, csr; 59 60 for_each_cpu(lcpu, cpu_possible_mask) { 61 cpu = cpu_logical_map(lcpu); 62 csr = flowctrl_read_cpu_csr(cpu); 63 64 pr_err("cpu%lu: online=%d flowctrl_csr=0x%08lx\n", 65 cpu, cpu_online(lcpu), csr); 66 } 67 } 68 69 static int tegra_cpuidle_wait_for_secondary_cpus_parking(void) 70 { 71 unsigned int retries = 3; 72 73 while (retries--) { 74 unsigned int delay_us = 10; 75 unsigned int timeout_us = 500 * 1000 / delay_us; 76 77 /* 78 * The primary CPU0 core shall wait for the secondaries 79 * shutdown in order to power-off CPU's cluster safely. 80 * The timeout value depends on the current CPU frequency, 81 * it takes about 40-150us in average and over 1000us in 82 * a worst case scenario. 83 */ 84 do { 85 if (tegra_cpu_rail_off_ready()) 86 return 0; 87 88 udelay(delay_us); 89 90 } while (timeout_us--); 91 92 pr_err("secondary CPU taking too long to park\n"); 93 94 tegra_cpuidle_report_cpus_state(); 95 } 96 97 pr_err("timed out waiting secondaries to park\n"); 98 99 return -ETIMEDOUT; 100 } 101 102 static void tegra_cpuidle_unpark_secondary_cpus(void) 103 { 104 unsigned int cpu, lcpu; 105 106 for_each_cpu(lcpu, cpu_online_mask) { 107 cpu = cpu_logical_map(lcpu); 108 109 if (cpu > 0) { 110 tegra_enable_cpu_clock(cpu); 111 tegra_cpu_out_of_reset(cpu); 112 flowctrl_write_cpu_halt(cpu, 0); 113 } 114 } 115 } 116 117 static int tegra_cpuidle_cc6_enter(unsigned int cpu) 118 { 119 int ret; 120 121 if (cpu > 0) { 122 ret = cpu_suspend(cpu, tegra_pm_park_secondary_cpu); 123 } else { 124 ret = tegra_cpuidle_wait_for_secondary_cpus_parking(); 125 if (!ret) 126 ret = tegra_pm_enter_lp2(); 127 128 tegra_cpuidle_unpark_secondary_cpus(); 129 } 130 131 return ret; 132 } 133 134 static int tegra_cpuidle_c7_enter(void) 135 { 136 int err; 137 138 if (tegra_cpuidle_using_firmware()) { 139 err = call_firmware_op(prepare_idle, TF_PM_MODE_LP2_NOFLUSH_L2); 140 if (err) 141 return err; 142 143 return call_firmware_op(do_idle, 0); 144 } 145 146 return cpu_suspend(0, tegra30_pm_secondary_cpu_suspend); 147 } 148 149 static int tegra_cpuidle_coupled_barrier(struct cpuidle_device *dev) 150 { 151 if (tegra_pending_sgi()) { 152 /* 153 * CPU got local interrupt that will be lost after GIC's 154 * shutdown because GIC driver doesn't save/restore the 155 * pending SGI state across CPU cluster PM. Abort and retry 156 * next time. 157 */ 158 atomic_set(&tegra_abort_flag, 1); 159 } 160 161 cpuidle_coupled_parallel_barrier(dev, &tegra_idle_barrier); 162 163 if (atomic_read(&tegra_abort_flag)) { 164 cpuidle_coupled_parallel_barrier(dev, &tegra_idle_barrier); 165 atomic_set(&tegra_abort_flag, 0); 166 return -EINTR; 167 } 168 169 return 0; 170 } 171 172 static int tegra_cpuidle_state_enter(struct cpuidle_device *dev, 173 int index, unsigned int cpu) 174 { 175 int err; 176 177 /* 178 * CC6 state is the "CPU cluster power-off" state. In order to 179 * enter this state, at first the secondary CPU cores need to be 180 * parked into offline mode, then the last CPU should clean out 181 * remaining dirty cache lines into DRAM and trigger Flow Controller 182 * logic that turns off the cluster's power domain (which includes 183 * CPU cores, GIC and L2 cache). 184 */ 185 if (index == TEGRA_CC6) { 186 err = tegra_cpuidle_coupled_barrier(dev); 187 if (err) 188 return err; 189 } 190 191 local_fiq_disable(); 192 RCU_NONIDLE(tegra_pm_set_cpu_in_lp2()); 193 cpu_pm_enter(); 194 195 switch (index) { 196 case TEGRA_C7: 197 err = tegra_cpuidle_c7_enter(); 198 break; 199 200 case TEGRA_CC6: 201 err = tegra_cpuidle_cc6_enter(cpu); 202 break; 203 204 default: 205 err = -EINVAL; 206 break; 207 } 208 209 cpu_pm_exit(); 210 RCU_NONIDLE(tegra_pm_clear_cpu_in_lp2()); 211 local_fiq_enable(); 212 213 return err ?: index; 214 } 215 216 static int tegra_cpuidle_adjust_state_index(int index, unsigned int cpu) 217 { 218 /* 219 * On Tegra30 CPU0 can't be power-gated separately from secondary 220 * cores because it gates the whole CPU cluster. 221 */ 222 if (cpu > 0 || index != TEGRA_C7 || tegra_get_chip_id() != TEGRA30) 223 return index; 224 225 /* put CPU0 into C1 if C7 is requested and secondaries are online */ 226 if (!IS_ENABLED(CONFIG_PM_SLEEP) || num_online_cpus() > 1) 227 index = TEGRA_C1; 228 else 229 index = TEGRA_CC6; 230 231 return index; 232 } 233 234 static int tegra_cpuidle_enter(struct cpuidle_device *dev, 235 struct cpuidle_driver *drv, 236 int index) 237 { 238 unsigned int cpu = cpu_logical_map(dev->cpu); 239 int ret; 240 241 index = tegra_cpuidle_adjust_state_index(index, cpu); 242 if (dev->states_usage[index].disable) 243 return -1; 244 245 if (index == TEGRA_C1) 246 ret = arm_cpuidle_simple_enter(dev, drv, index); 247 else 248 ret = tegra_cpuidle_state_enter(dev, index, cpu); 249 250 if (ret < 0) { 251 if (ret != -EINTR || index != TEGRA_CC6) 252 pr_err_once("failed to enter state %d err: %d\n", 253 index, ret); 254 index = -1; 255 } else { 256 index = ret; 257 } 258 259 return index; 260 } 261 262 static int tegra114_enter_s2idle(struct cpuidle_device *dev, 263 struct cpuidle_driver *drv, 264 int index) 265 { 266 tegra_cpuidle_enter(dev, drv, index); 267 268 return 0; 269 } 270 271 /* 272 * The previous versions of Tegra CPUIDLE driver used a different "legacy" 273 * terminology for naming of the idling states, while this driver uses the 274 * new terminology. 275 * 276 * Mapping of the old terms into the new ones: 277 * 278 * Old | New 279 * --------- 280 * LP3 | C1 (CPU core clock gating) 281 * LP2 | C7 (CPU core power gating) 282 * LP2 | CC6 (CPU cluster power gating) 283 * 284 * Note that that the older CPUIDLE driver versions didn't explicitly 285 * differentiate the LP2 states because these states either used the same 286 * code path or because CC6 wasn't supported. 287 */ 288 static struct cpuidle_driver tegra_idle_driver = { 289 .name = "tegra_idle", 290 .states = { 291 [TEGRA_C1] = ARM_CPUIDLE_WFI_STATE_PWR(600), 292 [TEGRA_C7] = { 293 .enter = tegra_cpuidle_enter, 294 .exit_latency = 2000, 295 .target_residency = 2200, 296 .power_usage = 100, 297 .flags = CPUIDLE_FLAG_TIMER_STOP, 298 .name = "C7", 299 .desc = "CPU core powered off", 300 }, 301 [TEGRA_CC6] = { 302 .enter = tegra_cpuidle_enter, 303 .exit_latency = 5000, 304 .target_residency = 10000, 305 .power_usage = 0, 306 .flags = CPUIDLE_FLAG_TIMER_STOP | 307 CPUIDLE_FLAG_COUPLED, 308 .name = "CC6", 309 .desc = "CPU cluster powered off", 310 }, 311 }, 312 .state_count = TEGRA_STATE_COUNT, 313 .safe_state_index = TEGRA_C1, 314 }; 315 316 static inline void tegra_cpuidle_disable_state(enum tegra_state state) 317 { 318 cpuidle_driver_state_disabled(&tegra_idle_driver, state, true); 319 } 320 321 /* 322 * Tegra20 HW appears to have a bug such that PCIe device interrupts, whether 323 * they are legacy IRQs or MSI, are lost when CC6 is enabled. To work around 324 * this, simply disable CC6 if the PCI driver and DT node are both enabled. 325 */ 326 void tegra_cpuidle_pcie_irqs_in_use(void) 327 { 328 struct cpuidle_state *state_cc6 = &tegra_idle_driver.states[TEGRA_CC6]; 329 330 if ((state_cc6->flags & CPUIDLE_FLAG_UNUSABLE) || 331 tegra_get_chip_id() != TEGRA20) 332 return; 333 334 pr_info("disabling CC6 state, since PCIe IRQs are in use\n"); 335 tegra_cpuidle_disable_state(TEGRA_CC6); 336 } 337 338 static void tegra_cpuidle_setup_tegra114_c7_state(void) 339 { 340 struct cpuidle_state *s = &tegra_idle_driver.states[TEGRA_C7]; 341 342 s->enter_s2idle = tegra114_enter_s2idle; 343 s->target_residency = 1000; 344 s->exit_latency = 500; 345 } 346 347 static int tegra_cpuidle_probe(struct platform_device *pdev) 348 { 349 /* LP2 could be disabled in device-tree */ 350 if (tegra_pmc_get_suspend_mode() < TEGRA_SUSPEND_LP2) 351 tegra_cpuidle_disable_state(TEGRA_CC6); 352 353 /* 354 * Required suspend-resume functionality, which is provided by the 355 * Tegra-arch core and PMC driver, is unavailable if PM-sleep option 356 * is disabled. 357 */ 358 if (!IS_ENABLED(CONFIG_PM_SLEEP)) { 359 if (!tegra_cpuidle_using_firmware()) 360 tegra_cpuidle_disable_state(TEGRA_C7); 361 362 tegra_cpuidle_disable_state(TEGRA_CC6); 363 } 364 365 /* 366 * Generic WFI state (also known as C1 or LP3) and the coupled CPU 367 * cluster power-off (CC6 or LP2) states are common for all Tegra SoCs. 368 */ 369 switch (tegra_get_chip_id()) { 370 case TEGRA20: 371 /* Tegra20 isn't capable to power-off individual CPU cores */ 372 tegra_cpuidle_disable_state(TEGRA_C7); 373 break; 374 375 case TEGRA30: 376 break; 377 378 case TEGRA114: 379 case TEGRA124: 380 tegra_cpuidle_setup_tegra114_c7_state(); 381 382 /* coupled CC6 (LP2) state isn't implemented yet */ 383 tegra_cpuidle_disable_state(TEGRA_CC6); 384 break; 385 386 default: 387 return -EINVAL; 388 } 389 390 return cpuidle_register(&tegra_idle_driver, cpu_possible_mask); 391 } 392 393 static struct platform_driver tegra_cpuidle_driver = { 394 .probe = tegra_cpuidle_probe, 395 .driver = { 396 .name = "tegra-cpuidle", 397 }, 398 }; 399 builtin_platform_driver(tegra_cpuidle_driver); 400