1 /* 2 * cpuidle.c - core cpuidle infrastructure 3 * 4 * (C) 2006-2007 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com> 5 * Shaohua Li <shaohua.li@intel.com> 6 * Adam Belay <abelay@novell.com> 7 * 8 * This code is licenced under the GPL. 9 */ 10 11 #include <linux/kernel.h> 12 #include <linux/mutex.h> 13 #include <linux/sched.h> 14 #include <linux/notifier.h> 15 #include <linux/pm_qos.h> 16 #include <linux/cpu.h> 17 #include <linux/cpuidle.h> 18 #include <linux/ktime.h> 19 #include <linux/hrtimer.h> 20 #include <linux/module.h> 21 #include <trace/events/power.h> 22 23 #include "cpuidle.h" 24 25 DEFINE_PER_CPU(struct cpuidle_device *, cpuidle_devices); 26 27 DEFINE_MUTEX(cpuidle_lock); 28 LIST_HEAD(cpuidle_detected_devices); 29 30 static int enabled_devices; 31 static int off __read_mostly; 32 static int initialized __read_mostly; 33 34 int cpuidle_disabled(void) 35 { 36 return off; 37 } 38 void disable_cpuidle(void) 39 { 40 off = 1; 41 } 42 43 #if defined(CONFIG_ARCH_HAS_CPU_IDLE_WAIT) 44 static void cpuidle_kick_cpus(void) 45 { 46 cpu_idle_wait(); 47 } 48 #elif defined(CONFIG_SMP) 49 # error "Arch needs cpu_idle_wait() equivalent here" 50 #else /* !CONFIG_ARCH_HAS_CPU_IDLE_WAIT && !CONFIG_SMP */ 51 static void cpuidle_kick_cpus(void) {} 52 #endif 53 54 static int __cpuidle_register_device(struct cpuidle_device *dev); 55 56 static inline int cpuidle_enter(struct cpuidle_device *dev, 57 struct cpuidle_driver *drv, int index) 58 { 59 struct cpuidle_state *target_state = &drv->states[index]; 60 return target_state->enter(dev, drv, index); 61 } 62 63 static inline int cpuidle_enter_tk(struct cpuidle_device *dev, 64 struct cpuidle_driver *drv, int index) 65 { 66 return cpuidle_wrap_enter(dev, drv, index, cpuidle_enter); 67 } 68 69 typedef int (*cpuidle_enter_t)(struct cpuidle_device *dev, 70 struct cpuidle_driver *drv, int index); 71 72 static cpuidle_enter_t cpuidle_enter_ops; 73 74 /** 75 * cpuidle_idle_call - the main idle loop 76 * 77 * NOTE: no locks or semaphores should be used here 78 * return non-zero on failure 79 */ 80 int cpuidle_idle_call(void) 81 { 82 struct cpuidle_device *dev = __this_cpu_read(cpuidle_devices); 83 struct cpuidle_driver *drv = cpuidle_get_driver(); 84 int next_state, entered_state; 85 86 if (off) 87 return -ENODEV; 88 89 if (!initialized) 90 return -ENODEV; 91 92 /* check if the device is ready */ 93 if (!dev || !dev->enabled) 94 return -EBUSY; 95 96 #if 0 97 /* shows regressions, re-enable for 2.6.29 */ 98 /* 99 * run any timers that can be run now, at this point 100 * before calculating the idle duration etc. 101 */ 102 hrtimer_peek_ahead_timers(); 103 #endif 104 105 /* ask the governor for the next state */ 106 next_state = cpuidle_curr_governor->select(drv, dev); 107 if (need_resched()) { 108 local_irq_enable(); 109 return 0; 110 } 111 112 trace_power_start(POWER_CSTATE, next_state, dev->cpu); 113 trace_cpu_idle(next_state, dev->cpu); 114 115 entered_state = cpuidle_enter_ops(dev, drv, next_state); 116 117 trace_power_end(dev->cpu); 118 trace_cpu_idle(PWR_EVENT_EXIT, dev->cpu); 119 120 if (entered_state >= 0) { 121 /* Update cpuidle counters */ 122 /* This can be moved to within driver enter routine 123 * but that results in multiple copies of same code. 124 */ 125 dev->states_usage[entered_state].time += 126 (unsigned long long)dev->last_residency; 127 dev->states_usage[entered_state].usage++; 128 } else { 129 dev->last_residency = 0; 130 } 131 132 /* give the governor an opportunity to reflect on the outcome */ 133 if (cpuidle_curr_governor->reflect) 134 cpuidle_curr_governor->reflect(dev, entered_state); 135 136 return 0; 137 } 138 139 /** 140 * cpuidle_install_idle_handler - installs the cpuidle idle loop handler 141 */ 142 void cpuidle_install_idle_handler(void) 143 { 144 if (enabled_devices) { 145 /* Make sure all changes finished before we switch to new idle */ 146 smp_wmb(); 147 initialized = 1; 148 } 149 } 150 151 /** 152 * cpuidle_uninstall_idle_handler - uninstalls the cpuidle idle loop handler 153 */ 154 void cpuidle_uninstall_idle_handler(void) 155 { 156 if (enabled_devices) { 157 initialized = 0; 158 cpuidle_kick_cpus(); 159 } 160 } 161 162 /** 163 * cpuidle_pause_and_lock - temporarily disables CPUIDLE 164 */ 165 void cpuidle_pause_and_lock(void) 166 { 167 mutex_lock(&cpuidle_lock); 168 cpuidle_uninstall_idle_handler(); 169 } 170 171 EXPORT_SYMBOL_GPL(cpuidle_pause_and_lock); 172 173 /** 174 * cpuidle_resume_and_unlock - resumes CPUIDLE operation 175 */ 176 void cpuidle_resume_and_unlock(void) 177 { 178 cpuidle_install_idle_handler(); 179 mutex_unlock(&cpuidle_lock); 180 } 181 182 EXPORT_SYMBOL_GPL(cpuidle_resume_and_unlock); 183 184 /** 185 * cpuidle_wrap_enter - performs timekeeping and irqen around enter function 186 * @dev: pointer to a valid cpuidle_device object 187 * @drv: pointer to a valid cpuidle_driver object 188 * @index: index of the target cpuidle state. 189 */ 190 int cpuidle_wrap_enter(struct cpuidle_device *dev, 191 struct cpuidle_driver *drv, int index, 192 int (*enter)(struct cpuidle_device *dev, 193 struct cpuidle_driver *drv, int index)) 194 { 195 ktime_t time_start, time_end; 196 s64 diff; 197 198 time_start = ktime_get(); 199 200 index = enter(dev, drv, index); 201 202 time_end = ktime_get(); 203 204 local_irq_enable(); 205 206 diff = ktime_to_us(ktime_sub(time_end, time_start)); 207 if (diff > INT_MAX) 208 diff = INT_MAX; 209 210 dev->last_residency = (int) diff; 211 212 return index; 213 } 214 215 #ifdef CONFIG_ARCH_HAS_CPU_RELAX 216 static int poll_idle(struct cpuidle_device *dev, 217 struct cpuidle_driver *drv, int index) 218 { 219 ktime_t t1, t2; 220 s64 diff; 221 222 t1 = ktime_get(); 223 local_irq_enable(); 224 while (!need_resched()) 225 cpu_relax(); 226 227 t2 = ktime_get(); 228 diff = ktime_to_us(ktime_sub(t2, t1)); 229 if (diff > INT_MAX) 230 diff = INT_MAX; 231 232 dev->last_residency = (int) diff; 233 234 return index; 235 } 236 237 static void poll_idle_init(struct cpuidle_driver *drv) 238 { 239 struct cpuidle_state *state = &drv->states[0]; 240 241 snprintf(state->name, CPUIDLE_NAME_LEN, "POLL"); 242 snprintf(state->desc, CPUIDLE_DESC_LEN, "CPUIDLE CORE POLL IDLE"); 243 state->exit_latency = 0; 244 state->target_residency = 0; 245 state->power_usage = -1; 246 state->flags = 0; 247 state->enter = poll_idle; 248 } 249 #else 250 static void poll_idle_init(struct cpuidle_driver *drv) {} 251 #endif /* CONFIG_ARCH_HAS_CPU_RELAX */ 252 253 /** 254 * cpuidle_enable_device - enables idle PM for a CPU 255 * @dev: the CPU 256 * 257 * This function must be called between cpuidle_pause_and_lock and 258 * cpuidle_resume_and_unlock when used externally. 259 */ 260 int cpuidle_enable_device(struct cpuidle_device *dev) 261 { 262 int ret, i; 263 struct cpuidle_driver *drv = cpuidle_get_driver(); 264 265 if (dev->enabled) 266 return 0; 267 if (!drv || !cpuidle_curr_governor) 268 return -EIO; 269 if (!dev->state_count) 270 return -EINVAL; 271 272 if (dev->registered == 0) { 273 ret = __cpuidle_register_device(dev); 274 if (ret) 275 return ret; 276 } 277 278 cpuidle_enter_ops = drv->en_core_tk_irqen ? 279 cpuidle_enter_tk : cpuidle_enter; 280 281 poll_idle_init(drv); 282 283 if ((ret = cpuidle_add_state_sysfs(dev))) 284 return ret; 285 286 if (cpuidle_curr_governor->enable && 287 (ret = cpuidle_curr_governor->enable(drv, dev))) 288 goto fail_sysfs; 289 290 for (i = 0; i < dev->state_count; i++) { 291 dev->states_usage[i].usage = 0; 292 dev->states_usage[i].time = 0; 293 } 294 dev->last_residency = 0; 295 296 smp_wmb(); 297 298 dev->enabled = 1; 299 300 enabled_devices++; 301 return 0; 302 303 fail_sysfs: 304 cpuidle_remove_state_sysfs(dev); 305 306 return ret; 307 } 308 309 EXPORT_SYMBOL_GPL(cpuidle_enable_device); 310 311 /** 312 * cpuidle_disable_device - disables idle PM for a CPU 313 * @dev: the CPU 314 * 315 * This function must be called between cpuidle_pause_and_lock and 316 * cpuidle_resume_and_unlock when used externally. 317 */ 318 void cpuidle_disable_device(struct cpuidle_device *dev) 319 { 320 if (!dev->enabled) 321 return; 322 if (!cpuidle_get_driver() || !cpuidle_curr_governor) 323 return; 324 325 dev->enabled = 0; 326 327 if (cpuidle_curr_governor->disable) 328 cpuidle_curr_governor->disable(cpuidle_get_driver(), dev); 329 330 cpuidle_remove_state_sysfs(dev); 331 enabled_devices--; 332 } 333 334 EXPORT_SYMBOL_GPL(cpuidle_disable_device); 335 336 /** 337 * __cpuidle_register_device - internal register function called before register 338 * and enable routines 339 * @dev: the cpu 340 * 341 * cpuidle_lock mutex must be held before this is called 342 */ 343 static int __cpuidle_register_device(struct cpuidle_device *dev) 344 { 345 int ret; 346 struct device *cpu_dev = get_cpu_device((unsigned long)dev->cpu); 347 struct cpuidle_driver *cpuidle_driver = cpuidle_get_driver(); 348 349 if (!dev) 350 return -EINVAL; 351 if (!try_module_get(cpuidle_driver->owner)) 352 return -EINVAL; 353 354 init_completion(&dev->kobj_unregister); 355 356 per_cpu(cpuidle_devices, dev->cpu) = dev; 357 list_add(&dev->device_list, &cpuidle_detected_devices); 358 if ((ret = cpuidle_add_sysfs(cpu_dev))) { 359 module_put(cpuidle_driver->owner); 360 return ret; 361 } 362 363 dev->registered = 1; 364 return 0; 365 } 366 367 /** 368 * cpuidle_register_device - registers a CPU's idle PM feature 369 * @dev: the cpu 370 */ 371 int cpuidle_register_device(struct cpuidle_device *dev) 372 { 373 int ret; 374 375 mutex_lock(&cpuidle_lock); 376 377 if ((ret = __cpuidle_register_device(dev))) { 378 mutex_unlock(&cpuidle_lock); 379 return ret; 380 } 381 382 cpuidle_enable_device(dev); 383 cpuidle_install_idle_handler(); 384 385 mutex_unlock(&cpuidle_lock); 386 387 return 0; 388 389 } 390 391 EXPORT_SYMBOL_GPL(cpuidle_register_device); 392 393 /** 394 * cpuidle_unregister_device - unregisters a CPU's idle PM feature 395 * @dev: the cpu 396 */ 397 void cpuidle_unregister_device(struct cpuidle_device *dev) 398 { 399 struct device *cpu_dev = get_cpu_device((unsigned long)dev->cpu); 400 struct cpuidle_driver *cpuidle_driver = cpuidle_get_driver(); 401 402 if (dev->registered == 0) 403 return; 404 405 cpuidle_pause_and_lock(); 406 407 cpuidle_disable_device(dev); 408 409 cpuidle_remove_sysfs(cpu_dev); 410 list_del(&dev->device_list); 411 wait_for_completion(&dev->kobj_unregister); 412 per_cpu(cpuidle_devices, dev->cpu) = NULL; 413 414 cpuidle_resume_and_unlock(); 415 416 module_put(cpuidle_driver->owner); 417 } 418 419 EXPORT_SYMBOL_GPL(cpuidle_unregister_device); 420 421 #ifdef CONFIG_SMP 422 423 static void smp_callback(void *v) 424 { 425 /* we already woke the CPU up, nothing more to do */ 426 } 427 428 /* 429 * This function gets called when a part of the kernel has a new latency 430 * requirement. This means we need to get all processors out of their C-state, 431 * and then recalculate a new suitable C-state. Just do a cross-cpu IPI; that 432 * wakes them all right up. 433 */ 434 static int cpuidle_latency_notify(struct notifier_block *b, 435 unsigned long l, void *v) 436 { 437 smp_call_function(smp_callback, NULL, 1); 438 return NOTIFY_OK; 439 } 440 441 static struct notifier_block cpuidle_latency_notifier = { 442 .notifier_call = cpuidle_latency_notify, 443 }; 444 445 static inline void latency_notifier_init(struct notifier_block *n) 446 { 447 pm_qos_add_notifier(PM_QOS_CPU_DMA_LATENCY, n); 448 } 449 450 #else /* CONFIG_SMP */ 451 452 #define latency_notifier_init(x) do { } while (0) 453 454 #endif /* CONFIG_SMP */ 455 456 /** 457 * cpuidle_init - core initializer 458 */ 459 static int __init cpuidle_init(void) 460 { 461 int ret; 462 463 if (cpuidle_disabled()) 464 return -ENODEV; 465 466 ret = cpuidle_add_interface(cpu_subsys.dev_root); 467 if (ret) 468 return ret; 469 470 latency_notifier_init(&cpuidle_latency_notifier); 471 472 return 0; 473 } 474 475 module_param(off, int, 0444); 476 core_initcall(cpuidle_init); 477