1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * CPU idle driver for Tegra CPUs 4 * 5 * Copyright (c) 2010-2013, NVIDIA Corporation. 6 * Copyright (c) 2011 Google, Inc. 7 * Author: Colin Cross <ccross@android.com> 8 * Gary King <gking@nvidia.com> 9 * 10 * Rework for 3.3 by Peter De Schrijver <pdeschrijver@nvidia.com> 11 * 12 * Tegra20/124 driver unification by Dmitry Osipenko <digetx@gmail.com> 13 */ 14 15 #define pr_fmt(fmt) "tegra-cpuidle: " fmt 16 17 #include <linux/atomic.h> 18 #include <linux/cpuidle.h> 19 #include <linux/cpumask.h> 20 #include <linux/cpu_pm.h> 21 #include <linux/delay.h> 22 #include <linux/errno.h> 23 #include <linux/platform_device.h> 24 #include <linux/types.h> 25 26 #include <linux/clk/tegra.h> 27 #include <linux/firmware/trusted_foundations.h> 28 29 #include <soc/tegra/cpuidle.h> 30 #include <soc/tegra/flowctrl.h> 31 #include <soc/tegra/fuse.h> 32 #include <soc/tegra/irq.h> 33 #include <soc/tegra/pm.h> 34 #include <soc/tegra/pmc.h> 35 36 #include <asm/cpuidle.h> 37 #include <asm/firmware.h> 38 #include <asm/smp_plat.h> 39 #include <asm/suspend.h> 40 41 enum tegra_state { 42 TEGRA_C1, 43 TEGRA_C7, 44 TEGRA_CC6, 45 TEGRA_STATE_COUNT, 46 }; 47 48 static atomic_t tegra_idle_barrier; 49 static atomic_t tegra_abort_flag; 50 51 static void tegra_cpuidle_report_cpus_state(void) 52 { 53 unsigned long cpu, lcpu, csr; 54 55 for_each_cpu(lcpu, cpu_possible_mask) { 56 cpu = cpu_logical_map(lcpu); 57 csr = flowctrl_read_cpu_csr(cpu); 58 59 pr_err("cpu%lu: online=%d flowctrl_csr=0x%08lx\n", 60 cpu, cpu_online(lcpu), csr); 61 } 62 } 63 64 static int tegra_cpuidle_wait_for_secondary_cpus_parking(void) 65 { 66 unsigned int retries = 3; 67 68 while (retries--) { 69 unsigned int delay_us = 10; 70 unsigned int timeout_us = 500 * 1000 / delay_us; 71 72 /* 73 * The primary CPU0 core shall wait for the secondaries 74 * shutdown in order to power-off CPU's cluster safely. 75 * The timeout value depends on the current CPU frequency, 76 * it takes about 40-150us in average and over 1000us in 77 * a worst case scenario. 78 */ 79 do { 80 if (tegra_cpu_rail_off_ready()) 81 return 0; 82 83 udelay(delay_us); 84 85 } while (timeout_us--); 86 87 pr_err("secondary CPU taking too long to park\n"); 88 89 tegra_cpuidle_report_cpus_state(); 90 } 91 92 pr_err("timed out waiting secondaries to park\n"); 93 94 return -ETIMEDOUT; 95 } 96 97 static void tegra_cpuidle_unpark_secondary_cpus(void) 98 { 99 unsigned int cpu, lcpu; 100 101 for_each_cpu(lcpu, cpu_online_mask) { 102 cpu = cpu_logical_map(lcpu); 103 104 if (cpu > 0) { 105 tegra_enable_cpu_clock(cpu); 106 tegra_cpu_out_of_reset(cpu); 107 flowctrl_write_cpu_halt(cpu, 0); 108 } 109 } 110 } 111 112 static int tegra_cpuidle_cc6_enter(unsigned int cpu) 113 { 114 int ret; 115 116 if (cpu > 0) { 117 ret = cpu_suspend(cpu, tegra_pm_park_secondary_cpu); 118 } else { 119 ret = tegra_cpuidle_wait_for_secondary_cpus_parking(); 120 if (!ret) 121 ret = tegra_pm_enter_lp2(); 122 123 tegra_cpuidle_unpark_secondary_cpus(); 124 } 125 126 return ret; 127 } 128 129 static int tegra_cpuidle_c7_enter(void) 130 { 131 int err; 132 133 err = call_firmware_op(prepare_idle, TF_PM_MODE_LP2_NOFLUSH_L2); 134 if (err && err != -ENOSYS) 135 return err; 136 137 return cpu_suspend(0, tegra30_pm_secondary_cpu_suspend); 138 } 139 140 static int tegra_cpuidle_coupled_barrier(struct cpuidle_device *dev) 141 { 142 if (tegra_pending_sgi()) { 143 /* 144 * CPU got local interrupt that will be lost after GIC's 145 * shutdown because GIC driver doesn't save/restore the 146 * pending SGI state across CPU cluster PM. Abort and retry 147 * next time. 148 */ 149 atomic_set(&tegra_abort_flag, 1); 150 } 151 152 cpuidle_coupled_parallel_barrier(dev, &tegra_idle_barrier); 153 154 if (atomic_read(&tegra_abort_flag)) { 155 cpuidle_coupled_parallel_barrier(dev, &tegra_idle_barrier); 156 atomic_set(&tegra_abort_flag, 0); 157 return -EINTR; 158 } 159 160 return 0; 161 } 162 163 static int tegra_cpuidle_state_enter(struct cpuidle_device *dev, 164 int index, unsigned int cpu) 165 { 166 int err; 167 168 /* 169 * CC6 state is the "CPU cluster power-off" state. In order to 170 * enter this state, at first the secondary CPU cores need to be 171 * parked into offline mode, then the last CPU should clean out 172 * remaining dirty cache lines into DRAM and trigger Flow Controller 173 * logic that turns off the cluster's power domain (which includes 174 * CPU cores, GIC and L2 cache). 175 */ 176 if (index == TEGRA_CC6) { 177 err = tegra_cpuidle_coupled_barrier(dev); 178 if (err) 179 return err; 180 } 181 182 local_fiq_disable(); 183 RCU_NONIDLE(tegra_pm_set_cpu_in_lp2()); 184 cpu_pm_enter(); 185 186 switch (index) { 187 case TEGRA_C7: 188 err = tegra_cpuidle_c7_enter(); 189 break; 190 191 case TEGRA_CC6: 192 err = tegra_cpuidle_cc6_enter(cpu); 193 break; 194 195 default: 196 err = -EINVAL; 197 break; 198 } 199 200 cpu_pm_exit(); 201 RCU_NONIDLE(tegra_pm_clear_cpu_in_lp2()); 202 local_fiq_enable(); 203 204 return err ?: index; 205 } 206 207 static int tegra_cpuidle_adjust_state_index(int index, unsigned int cpu) 208 { 209 /* 210 * On Tegra30 CPU0 can't be power-gated separately from secondary 211 * cores because it gates the whole CPU cluster. 212 */ 213 if (cpu > 0 || index != TEGRA_C7 || tegra_get_chip_id() != TEGRA30) 214 return index; 215 216 /* put CPU0 into C1 if C7 is requested and secondaries are online */ 217 if (!IS_ENABLED(CONFIG_PM_SLEEP) || num_online_cpus() > 1) 218 index = TEGRA_C1; 219 else 220 index = TEGRA_CC6; 221 222 return index; 223 } 224 225 static int tegra_cpuidle_enter(struct cpuidle_device *dev, 226 struct cpuidle_driver *drv, 227 int index) 228 { 229 unsigned int cpu = cpu_logical_map(dev->cpu); 230 int ret; 231 232 index = tegra_cpuidle_adjust_state_index(index, cpu); 233 if (dev->states_usage[index].disable) 234 return -1; 235 236 if (index == TEGRA_C1) 237 ret = arm_cpuidle_simple_enter(dev, drv, index); 238 else 239 ret = tegra_cpuidle_state_enter(dev, index, cpu); 240 241 if (ret < 0) { 242 if (ret != -EINTR || index != TEGRA_CC6) 243 pr_err_once("failed to enter state %d err: %d\n", 244 index, ret); 245 index = -1; 246 } else { 247 index = ret; 248 } 249 250 return index; 251 } 252 253 static int tegra114_enter_s2idle(struct cpuidle_device *dev, 254 struct cpuidle_driver *drv, 255 int index) 256 { 257 tegra_cpuidle_enter(dev, drv, index); 258 259 return 0; 260 } 261 262 /* 263 * The previous versions of Tegra CPUIDLE driver used a different "legacy" 264 * terminology for naming of the idling states, while this driver uses the 265 * new terminology. 266 * 267 * Mapping of the old terms into the new ones: 268 * 269 * Old | New 270 * --------- 271 * LP3 | C1 (CPU core clock gating) 272 * LP2 | C7 (CPU core power gating) 273 * LP2 | CC6 (CPU cluster power gating) 274 * 275 * Note that that the older CPUIDLE driver versions didn't explicitly 276 * differentiate the LP2 states because these states either used the same 277 * code path or because CC6 wasn't supported. 278 */ 279 static struct cpuidle_driver tegra_idle_driver = { 280 .name = "tegra_idle", 281 .states = { 282 [TEGRA_C1] = ARM_CPUIDLE_WFI_STATE_PWR(600), 283 [TEGRA_C7] = { 284 .enter = tegra_cpuidle_enter, 285 .exit_latency = 2000, 286 .target_residency = 2200, 287 .power_usage = 100, 288 .flags = CPUIDLE_FLAG_TIMER_STOP, 289 .name = "C7", 290 .desc = "CPU core powered off", 291 }, 292 [TEGRA_CC6] = { 293 .enter = tegra_cpuidle_enter, 294 .exit_latency = 5000, 295 .target_residency = 10000, 296 .power_usage = 0, 297 .flags = CPUIDLE_FLAG_TIMER_STOP | 298 CPUIDLE_FLAG_COUPLED, 299 .name = "CC6", 300 .desc = "CPU cluster powered off", 301 }, 302 }, 303 .state_count = TEGRA_STATE_COUNT, 304 .safe_state_index = TEGRA_C1, 305 }; 306 307 static inline void tegra_cpuidle_disable_state(enum tegra_state state) 308 { 309 cpuidle_driver_state_disabled(&tegra_idle_driver, state, true); 310 } 311 312 /* 313 * Tegra20 HW appears to have a bug such that PCIe device interrupts, whether 314 * they are legacy IRQs or MSI, are lost when CC6 is enabled. To work around 315 * this, simply disable CC6 if the PCI driver and DT node are both enabled. 316 */ 317 void tegra_cpuidle_pcie_irqs_in_use(void) 318 { 319 struct cpuidle_state *state_cc6 = &tegra_idle_driver.states[TEGRA_CC6]; 320 321 if ((state_cc6->flags & CPUIDLE_FLAG_UNUSABLE) || 322 tegra_get_chip_id() != TEGRA20) 323 return; 324 325 pr_info("disabling CC6 state, since PCIe IRQs are in use\n"); 326 tegra_cpuidle_disable_state(TEGRA_CC6); 327 } 328 329 static void tegra_cpuidle_setup_tegra114_c7_state(void) 330 { 331 struct cpuidle_state *s = &tegra_idle_driver.states[TEGRA_C7]; 332 333 s->enter_s2idle = tegra114_enter_s2idle; 334 s->target_residency = 1000; 335 s->exit_latency = 500; 336 } 337 338 static int tegra_cpuidle_probe(struct platform_device *pdev) 339 { 340 if (tegra_pmc_get_suspend_mode() == TEGRA_SUSPEND_NOT_READY) 341 return -EPROBE_DEFER; 342 343 /* LP2 could be disabled in device-tree */ 344 if (tegra_pmc_get_suspend_mode() < TEGRA_SUSPEND_LP2) 345 tegra_cpuidle_disable_state(TEGRA_CC6); 346 347 /* 348 * Required suspend-resume functionality, which is provided by the 349 * Tegra-arch core and PMC driver, is unavailable if PM-sleep option 350 * is disabled. 351 */ 352 if (!IS_ENABLED(CONFIG_PM_SLEEP)) { 353 tegra_cpuidle_disable_state(TEGRA_C7); 354 tegra_cpuidle_disable_state(TEGRA_CC6); 355 } 356 357 /* 358 * Generic WFI state (also known as C1 or LP3) and the coupled CPU 359 * cluster power-off (CC6 or LP2) states are common for all Tegra SoCs. 360 */ 361 switch (tegra_get_chip_id()) { 362 case TEGRA20: 363 /* Tegra20 isn't capable to power-off individual CPU cores */ 364 tegra_cpuidle_disable_state(TEGRA_C7); 365 break; 366 367 case TEGRA30: 368 break; 369 370 case TEGRA114: 371 case TEGRA124: 372 tegra_cpuidle_setup_tegra114_c7_state(); 373 374 /* coupled CC6 (LP2) state isn't implemented yet */ 375 tegra_cpuidle_disable_state(TEGRA_CC6); 376 break; 377 378 default: 379 return -EINVAL; 380 } 381 382 return cpuidle_register(&tegra_idle_driver, cpu_possible_mask); 383 } 384 385 static struct platform_driver tegra_cpuidle_driver = { 386 .probe = tegra_cpuidle_probe, 387 .driver = { 388 .name = "tegra-cpuidle", 389 }, 390 }; 391 builtin_platform_driver(tegra_cpuidle_driver); 392