1 /* 2 * cpuidle.c - core cpuidle infrastructure 3 * 4 * (C) 2006-2007 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com> 5 * Shaohua Li <shaohua.li@intel.com> 6 * Adam Belay <abelay@novell.com> 7 * 8 * This code is licenced under the GPL. 9 */ 10 11 #include <linux/clockchips.h> 12 #include <linux/kernel.h> 13 #include <linux/mutex.h> 14 #include <linux/sched.h> 15 #include <linux/notifier.h> 16 #include <linux/pm_qos.h> 17 #include <linux/cpu.h> 18 #include <linux/cpuidle.h> 19 #include <linux/ktime.h> 20 #include <linux/hrtimer.h> 21 #include <linux/module.h> 22 #include <trace/events/power.h> 23 24 #include "cpuidle.h" 25 26 DEFINE_PER_CPU(struct cpuidle_device *, cpuidle_devices); 27 28 DEFINE_MUTEX(cpuidle_lock); 29 LIST_HEAD(cpuidle_detected_devices); 30 31 static int enabled_devices; 32 static int off __read_mostly; 33 static int initialized __read_mostly; 34 35 int cpuidle_disabled(void) 36 { 37 return off; 38 } 39 void disable_cpuidle(void) 40 { 41 off = 1; 42 } 43 44 static int __cpuidle_register_device(struct cpuidle_device *dev); 45 46 static inline int cpuidle_enter(struct cpuidle_device *dev, 47 struct cpuidle_driver *drv, int index) 48 { 49 struct cpuidle_state *target_state = &drv->states[index]; 50 return target_state->enter(dev, drv, index); 51 } 52 53 static inline int cpuidle_enter_tk(struct cpuidle_device *dev, 54 struct cpuidle_driver *drv, int index) 55 { 56 return cpuidle_wrap_enter(dev, drv, index, cpuidle_enter); 57 } 58 59 typedef int (*cpuidle_enter_t)(struct cpuidle_device *dev, 60 struct cpuidle_driver *drv, int index); 61 62 static cpuidle_enter_t cpuidle_enter_ops; 63 64 /** 65 * cpuidle_play_dead - cpu off-lining 66 * 67 * Returns in case of an error or no driver 68 */ 69 int cpuidle_play_dead(void) 70 { 71 struct cpuidle_device *dev = __this_cpu_read(cpuidle_devices); 72 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev); 73 int i; 74 75 if (!drv) 76 return -ENODEV; 77 78 /* Find lowest-power state that supports long-term idle */ 79 for (i = drv->state_count - 1; i >= CPUIDLE_DRIVER_STATE_START; i--) 80 if (drv->states[i].enter_dead) 81 return drv->states[i].enter_dead(dev, i); 82 83 return -ENODEV; 84 } 85 86 /** 87 * cpuidle_enter_state - enter the state and update stats 88 * @dev: cpuidle device for this cpu 89 * @drv: cpuidle driver for this cpu 90 * @next_state: index into drv->states of the state to enter 91 */ 92 int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv, 93 int next_state) 94 { 95 int entered_state; 96 97 entered_state = cpuidle_enter_ops(dev, drv, next_state); 98 99 if (entered_state >= 0) { 100 /* Update cpuidle counters */ 101 /* This can be moved to within driver enter routine 102 * but that results in multiple copies of same code. 103 */ 104 dev->states_usage[entered_state].time += dev->last_residency; 105 dev->states_usage[entered_state].usage++; 106 } else { 107 dev->last_residency = 0; 108 } 109 110 return entered_state; 111 } 112 113 /** 114 * cpuidle_idle_call - the main idle loop 115 * 116 * NOTE: no locks or semaphores should be used here 117 * return non-zero on failure 118 */ 119 int cpuidle_idle_call(void) 120 { 121 struct cpuidle_device *dev = __this_cpu_read(cpuidle_devices); 122 struct cpuidle_driver *drv; 123 int next_state, entered_state; 124 125 if (off) 126 return -ENODEV; 127 128 if (!initialized) 129 return -ENODEV; 130 131 /* check if the device is ready */ 132 if (!dev || !dev->enabled) 133 return -EBUSY; 134 135 drv = cpuidle_get_cpu_driver(dev); 136 137 /* ask the governor for the next state */ 138 next_state = cpuidle_curr_governor->select(drv, dev); 139 if (need_resched()) { 140 dev->last_residency = 0; 141 /* give the governor an opportunity to reflect on the outcome */ 142 if (cpuidle_curr_governor->reflect) 143 cpuidle_curr_governor->reflect(dev, next_state); 144 local_irq_enable(); 145 return 0; 146 } 147 148 trace_cpu_idle_rcuidle(next_state, dev->cpu); 149 150 if (drv->states[next_state].flags & CPUIDLE_FLAG_TIMER_STOP) 151 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, 152 &dev->cpu); 153 154 if (cpuidle_state_is_coupled(dev, drv, next_state)) 155 entered_state = cpuidle_enter_state_coupled(dev, drv, 156 next_state); 157 else 158 entered_state = cpuidle_enter_state(dev, drv, next_state); 159 160 if (drv->states[next_state].flags & CPUIDLE_FLAG_TIMER_STOP) 161 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, 162 &dev->cpu); 163 164 trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, dev->cpu); 165 166 /* give the governor an opportunity to reflect on the outcome */ 167 if (cpuidle_curr_governor->reflect) 168 cpuidle_curr_governor->reflect(dev, entered_state); 169 170 return 0; 171 } 172 173 /** 174 * cpuidle_install_idle_handler - installs the cpuidle idle loop handler 175 */ 176 void cpuidle_install_idle_handler(void) 177 { 178 if (enabled_devices) { 179 /* Make sure all changes finished before we switch to new idle */ 180 smp_wmb(); 181 initialized = 1; 182 } 183 } 184 185 /** 186 * cpuidle_uninstall_idle_handler - uninstalls the cpuidle idle loop handler 187 */ 188 void cpuidle_uninstall_idle_handler(void) 189 { 190 if (enabled_devices) { 191 initialized = 0; 192 kick_all_cpus_sync(); 193 } 194 } 195 196 /** 197 * cpuidle_pause_and_lock - temporarily disables CPUIDLE 198 */ 199 void cpuidle_pause_and_lock(void) 200 { 201 mutex_lock(&cpuidle_lock); 202 cpuidle_uninstall_idle_handler(); 203 } 204 205 EXPORT_SYMBOL_GPL(cpuidle_pause_and_lock); 206 207 /** 208 * cpuidle_resume_and_unlock - resumes CPUIDLE operation 209 */ 210 void cpuidle_resume_and_unlock(void) 211 { 212 cpuidle_install_idle_handler(); 213 mutex_unlock(&cpuidle_lock); 214 } 215 216 EXPORT_SYMBOL_GPL(cpuidle_resume_and_unlock); 217 218 /* Currently used in suspend/resume path to suspend cpuidle */ 219 void cpuidle_pause(void) 220 { 221 mutex_lock(&cpuidle_lock); 222 cpuidle_uninstall_idle_handler(); 223 mutex_unlock(&cpuidle_lock); 224 } 225 226 /* Currently used in suspend/resume path to resume cpuidle */ 227 void cpuidle_resume(void) 228 { 229 mutex_lock(&cpuidle_lock); 230 cpuidle_install_idle_handler(); 231 mutex_unlock(&cpuidle_lock); 232 } 233 234 /** 235 * cpuidle_wrap_enter - performs timekeeping and irqen around enter function 236 * @dev: pointer to a valid cpuidle_device object 237 * @drv: pointer to a valid cpuidle_driver object 238 * @index: index of the target cpuidle state. 239 */ 240 int cpuidle_wrap_enter(struct cpuidle_device *dev, 241 struct cpuidle_driver *drv, int index, 242 int (*enter)(struct cpuidle_device *dev, 243 struct cpuidle_driver *drv, int index)) 244 { 245 ktime_t time_start, time_end; 246 s64 diff; 247 248 time_start = ktime_get(); 249 250 index = enter(dev, drv, index); 251 252 time_end = ktime_get(); 253 254 local_irq_enable(); 255 256 diff = ktime_to_us(ktime_sub(time_end, time_start)); 257 if (diff > INT_MAX) 258 diff = INT_MAX; 259 260 dev->last_residency = (int) diff; 261 262 return index; 263 } 264 265 #ifdef CONFIG_ARCH_HAS_CPU_RELAX 266 static int poll_idle(struct cpuidle_device *dev, 267 struct cpuidle_driver *drv, int index) 268 { 269 ktime_t t1, t2; 270 s64 diff; 271 272 t1 = ktime_get(); 273 local_irq_enable(); 274 while (!need_resched()) 275 cpu_relax(); 276 277 t2 = ktime_get(); 278 diff = ktime_to_us(ktime_sub(t2, t1)); 279 if (diff > INT_MAX) 280 diff = INT_MAX; 281 282 dev->last_residency = (int) diff; 283 284 return index; 285 } 286 287 static void poll_idle_init(struct cpuidle_driver *drv) 288 { 289 struct cpuidle_state *state = &drv->states[0]; 290 291 snprintf(state->name, CPUIDLE_NAME_LEN, "POLL"); 292 snprintf(state->desc, CPUIDLE_DESC_LEN, "CPUIDLE CORE POLL IDLE"); 293 state->exit_latency = 0; 294 state->target_residency = 0; 295 state->power_usage = -1; 296 state->flags = 0; 297 state->enter = poll_idle; 298 state->disabled = false; 299 } 300 #else 301 static void poll_idle_init(struct cpuidle_driver *drv) {} 302 #endif /* CONFIG_ARCH_HAS_CPU_RELAX */ 303 304 /** 305 * cpuidle_enable_device - enables idle PM for a CPU 306 * @dev: the CPU 307 * 308 * This function must be called between cpuidle_pause_and_lock and 309 * cpuidle_resume_and_unlock when used externally. 310 */ 311 int cpuidle_enable_device(struct cpuidle_device *dev) 312 { 313 int ret, i; 314 struct cpuidle_driver *drv; 315 316 if (!dev) 317 return -EINVAL; 318 319 if (dev->enabled) 320 return 0; 321 322 drv = cpuidle_get_cpu_driver(dev); 323 324 if (!drv || !cpuidle_curr_governor) 325 return -EIO; 326 327 if (!dev->state_count) 328 dev->state_count = drv->state_count; 329 330 if (dev->registered == 0) { 331 ret = __cpuidle_register_device(dev); 332 if (ret) 333 return ret; 334 } 335 336 cpuidle_enter_ops = drv->en_core_tk_irqen ? 337 cpuidle_enter_tk : cpuidle_enter; 338 339 poll_idle_init(drv); 340 341 ret = cpuidle_add_device_sysfs(dev); 342 if (ret) 343 return ret; 344 345 if (cpuidle_curr_governor->enable && 346 (ret = cpuidle_curr_governor->enable(drv, dev))) 347 goto fail_sysfs; 348 349 for (i = 0; i < dev->state_count; i++) { 350 dev->states_usage[i].usage = 0; 351 dev->states_usage[i].time = 0; 352 } 353 dev->last_residency = 0; 354 355 smp_wmb(); 356 357 dev->enabled = 1; 358 359 enabled_devices++; 360 return 0; 361 362 fail_sysfs: 363 cpuidle_remove_device_sysfs(dev); 364 365 return ret; 366 } 367 368 EXPORT_SYMBOL_GPL(cpuidle_enable_device); 369 370 /** 371 * cpuidle_disable_device - disables idle PM for a CPU 372 * @dev: the CPU 373 * 374 * This function must be called between cpuidle_pause_and_lock and 375 * cpuidle_resume_and_unlock when used externally. 376 */ 377 void cpuidle_disable_device(struct cpuidle_device *dev) 378 { 379 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev); 380 381 if (!dev || !dev->enabled) 382 return; 383 384 if (!drv || !cpuidle_curr_governor) 385 return; 386 387 dev->enabled = 0; 388 389 if (cpuidle_curr_governor->disable) 390 cpuidle_curr_governor->disable(drv, dev); 391 392 cpuidle_remove_device_sysfs(dev); 393 enabled_devices--; 394 } 395 396 EXPORT_SYMBOL_GPL(cpuidle_disable_device); 397 398 /** 399 * __cpuidle_register_device - internal register function called before register 400 * and enable routines 401 * @dev: the cpu 402 * 403 * cpuidle_lock mutex must be held before this is called 404 */ 405 static int __cpuidle_register_device(struct cpuidle_device *dev) 406 { 407 int ret; 408 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev); 409 410 if (!try_module_get(drv->owner)) 411 return -EINVAL; 412 413 per_cpu(cpuidle_devices, dev->cpu) = dev; 414 list_add(&dev->device_list, &cpuidle_detected_devices); 415 ret = cpuidle_add_sysfs(dev); 416 if (ret) 417 goto err_sysfs; 418 419 ret = cpuidle_coupled_register_device(dev); 420 if (ret) 421 goto err_coupled; 422 423 dev->registered = 1; 424 return 0; 425 426 err_coupled: 427 cpuidle_remove_sysfs(dev); 428 err_sysfs: 429 list_del(&dev->device_list); 430 per_cpu(cpuidle_devices, dev->cpu) = NULL; 431 module_put(drv->owner); 432 return ret; 433 } 434 435 /** 436 * cpuidle_register_device - registers a CPU's idle PM feature 437 * @dev: the cpu 438 */ 439 int cpuidle_register_device(struct cpuidle_device *dev) 440 { 441 int ret; 442 443 if (!dev) 444 return -EINVAL; 445 446 mutex_lock(&cpuidle_lock); 447 448 if ((ret = __cpuidle_register_device(dev))) { 449 mutex_unlock(&cpuidle_lock); 450 return ret; 451 } 452 453 cpuidle_enable_device(dev); 454 cpuidle_install_idle_handler(); 455 456 mutex_unlock(&cpuidle_lock); 457 458 return 0; 459 460 } 461 462 EXPORT_SYMBOL_GPL(cpuidle_register_device); 463 464 /** 465 * cpuidle_unregister_device - unregisters a CPU's idle PM feature 466 * @dev: the cpu 467 */ 468 void cpuidle_unregister_device(struct cpuidle_device *dev) 469 { 470 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev); 471 472 if (dev->registered == 0) 473 return; 474 475 cpuidle_pause_and_lock(); 476 477 cpuidle_disable_device(dev); 478 479 cpuidle_remove_sysfs(dev); 480 list_del(&dev->device_list); 481 per_cpu(cpuidle_devices, dev->cpu) = NULL; 482 483 cpuidle_coupled_unregister_device(dev); 484 485 cpuidle_resume_and_unlock(); 486 487 module_put(drv->owner); 488 } 489 490 EXPORT_SYMBOL_GPL(cpuidle_unregister_device); 491 492 #ifdef CONFIG_SMP 493 494 static void smp_callback(void *v) 495 { 496 /* we already woke the CPU up, nothing more to do */ 497 } 498 499 /* 500 * This function gets called when a part of the kernel has a new latency 501 * requirement. This means we need to get all processors out of their C-state, 502 * and then recalculate a new suitable C-state. Just do a cross-cpu IPI; that 503 * wakes them all right up. 504 */ 505 static int cpuidle_latency_notify(struct notifier_block *b, 506 unsigned long l, void *v) 507 { 508 smp_call_function(smp_callback, NULL, 1); 509 return NOTIFY_OK; 510 } 511 512 static struct notifier_block cpuidle_latency_notifier = { 513 .notifier_call = cpuidle_latency_notify, 514 }; 515 516 static inline void latency_notifier_init(struct notifier_block *n) 517 { 518 pm_qos_add_notifier(PM_QOS_CPU_DMA_LATENCY, n); 519 } 520 521 #else /* CONFIG_SMP */ 522 523 #define latency_notifier_init(x) do { } while (0) 524 525 #endif /* CONFIG_SMP */ 526 527 /** 528 * cpuidle_init - core initializer 529 */ 530 static int __init cpuidle_init(void) 531 { 532 int ret; 533 534 if (cpuidle_disabled()) 535 return -ENODEV; 536 537 ret = cpuidle_add_interface(cpu_subsys.dev_root); 538 if (ret) 539 return ret; 540 541 latency_notifier_init(&cpuidle_latency_notifier); 542 543 return 0; 544 } 545 546 module_param(off, int, 0444); 547 core_initcall(cpuidle_init); 548