1 /* 2 * driver.c - driver support 3 * 4 * (C) 2006-2007 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com> 5 * Shaohua Li <shaohua.li@intel.com> 6 * Adam Belay <abelay@novell.com> 7 * 8 * This code is licenced under the GPL. 9 */ 10 11 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 12 13 #include <linux/mutex.h> 14 #include <linux/module.h> 15 #include <linux/sched.h> 16 #include <linux/sched/idle.h> 17 #include <linux/cpuidle.h> 18 #include <linux/cpumask.h> 19 #include <linux/tick.h> 20 #include <linux/cpu.h> 21 #include <linux/math64.h> 22 23 #include "cpuidle.h" 24 25 DEFINE_SPINLOCK(cpuidle_driver_lock); 26 27 #ifdef CONFIG_CPU_IDLE_MULTIPLE_DRIVERS 28 29 static DEFINE_PER_CPU(struct cpuidle_driver *, cpuidle_drivers); 30 31 /** 32 * __cpuidle_get_cpu_driver - return the cpuidle driver tied to a CPU. 33 * @cpu: the CPU handled by the driver 34 * 35 * Returns a pointer to struct cpuidle_driver or NULL if no driver has been 36 * registered for @cpu. 37 */ 38 static struct cpuidle_driver *__cpuidle_get_cpu_driver(int cpu) 39 { 40 return per_cpu(cpuidle_drivers, cpu); 41 } 42 43 /** 44 * __cpuidle_unset_driver - unset per CPU driver variables. 45 * @drv: a valid pointer to a struct cpuidle_driver 46 * 47 * For each CPU in the driver's CPU mask, unset the registered driver per CPU 48 * variable. If @drv is different from the registered driver, the corresponding 49 * variable is not cleared. 50 */ 51 static inline void __cpuidle_unset_driver(struct cpuidle_driver *drv) 52 { 53 int cpu; 54 55 for_each_cpu(cpu, drv->cpumask) { 56 57 if (drv != __cpuidle_get_cpu_driver(cpu)) 58 continue; 59 60 per_cpu(cpuidle_drivers, cpu) = NULL; 61 } 62 } 63 64 /** 65 * __cpuidle_set_driver - set per CPU driver variables for the given driver. 66 * @drv: a valid pointer to a struct cpuidle_driver 67 * 68 * Returns 0 on success, -EBUSY if any CPU in the cpumask have a driver 69 * different from drv already. 70 */ 71 static inline int __cpuidle_set_driver(struct cpuidle_driver *drv) 72 { 73 int cpu; 74 75 for_each_cpu(cpu, drv->cpumask) { 76 struct cpuidle_driver *old_drv; 77 78 old_drv = __cpuidle_get_cpu_driver(cpu); 79 if (old_drv && old_drv != drv) 80 return -EBUSY; 81 } 82 83 for_each_cpu(cpu, drv->cpumask) 84 per_cpu(cpuidle_drivers, cpu) = drv; 85 86 return 0; 87 } 88 89 #else 90 91 static struct cpuidle_driver *cpuidle_curr_driver; 92 93 /** 94 * __cpuidle_get_cpu_driver - return the global cpuidle driver pointer. 95 * @cpu: ignored without the multiple driver support 96 * 97 * Return a pointer to a struct cpuidle_driver object or NULL if no driver was 98 * previously registered. 99 */ 100 static inline struct cpuidle_driver *__cpuidle_get_cpu_driver(int cpu) 101 { 102 return cpuidle_curr_driver; 103 } 104 105 /** 106 * __cpuidle_set_driver - assign the global cpuidle driver variable. 107 * @drv: pointer to a struct cpuidle_driver object 108 * 109 * Returns 0 on success, -EBUSY if the driver is already registered. 110 */ 111 static inline int __cpuidle_set_driver(struct cpuidle_driver *drv) 112 { 113 if (cpuidle_curr_driver) 114 return -EBUSY; 115 116 cpuidle_curr_driver = drv; 117 118 return 0; 119 } 120 121 /** 122 * __cpuidle_unset_driver - unset the global cpuidle driver variable. 123 * @drv: a pointer to a struct cpuidle_driver 124 * 125 * Reset the global cpuidle variable to NULL. If @drv does not match the 126 * registered driver, do nothing. 127 */ 128 static inline void __cpuidle_unset_driver(struct cpuidle_driver *drv) 129 { 130 if (drv == cpuidle_curr_driver) 131 cpuidle_curr_driver = NULL; 132 } 133 134 #endif 135 136 /** 137 * cpuidle_setup_broadcast_timer - enable/disable the broadcast timer on a cpu 138 * @arg: a void pointer used to match the SMP cross call API 139 * 140 * If @arg is NULL broadcast is disabled otherwise enabled 141 * 142 * This function is executed per CPU by an SMP cross call. It's not 143 * supposed to be called directly. 144 */ 145 static void cpuidle_setup_broadcast_timer(void *arg) 146 { 147 if (arg) 148 tick_broadcast_enable(); 149 else 150 tick_broadcast_disable(); 151 } 152 153 /** 154 * __cpuidle_driver_init - initialize the driver's internal data 155 * @drv: a valid pointer to a struct cpuidle_driver 156 */ 157 static void __cpuidle_driver_init(struct cpuidle_driver *drv) 158 { 159 int i; 160 161 /* 162 * Use all possible CPUs as the default, because if the kernel boots 163 * with some CPUs offline and then we online one of them, the CPU 164 * notifier has to know which driver to assign. 165 */ 166 if (!drv->cpumask) 167 drv->cpumask = (struct cpumask *)cpu_possible_mask; 168 169 for (i = 0; i < drv->state_count; i++) { 170 struct cpuidle_state *s = &drv->states[i]; 171 172 /* 173 * Look for the timer stop flag in the different states and if 174 * it is found, indicate that the broadcast timer has to be set 175 * up. 176 */ 177 if (s->flags & CPUIDLE_FLAG_TIMER_STOP) 178 drv->bctimer = 1; 179 180 /* 181 * The core will use the target residency and exit latency 182 * values in nanoseconds, but allow drivers to provide them in 183 * microseconds too. 184 */ 185 if (s->target_residency > 0) 186 s->target_residency_ns = s->target_residency * NSEC_PER_USEC; 187 else if (s->target_residency_ns < 0) 188 s->target_residency_ns = 0; 189 else 190 s->target_residency = div_u64(s->target_residency_ns, NSEC_PER_USEC); 191 192 if (s->exit_latency > 0) 193 s->exit_latency_ns = mul_u32_u32(s->exit_latency, NSEC_PER_USEC); 194 else if (s->exit_latency_ns < 0) 195 s->exit_latency_ns = 0; 196 else 197 s->exit_latency = div_u64(s->exit_latency_ns, NSEC_PER_USEC); 198 199 /* 200 * Warn if the exit latency of a CPU idle state exceeds its 201 * target residency which is assumed to never happen in cpuidle 202 * in multiple places. 203 */ 204 if (s->exit_latency_ns > s->target_residency_ns) 205 pr_warn("Idle state %d target residency too low\n", i); 206 } 207 } 208 209 /** 210 * __cpuidle_register_driver: register the driver 211 * @drv: a valid pointer to a struct cpuidle_driver 212 * 213 * Do some sanity checks, initialize the driver, assign the driver to the 214 * global cpuidle driver variable(s) and set up the broadcast timer if the 215 * cpuidle driver has some states that shut down the local timer. 216 * 217 * Returns 0 on success, a negative error code otherwise: 218 * * -EINVAL if the driver pointer is NULL or no idle states are available 219 * * -ENODEV if the cpuidle framework is disabled 220 * * -EBUSY if the driver is already assigned to the global variable(s) 221 */ 222 static int __cpuidle_register_driver(struct cpuidle_driver *drv) 223 { 224 int ret; 225 226 if (!drv || !drv->state_count) 227 return -EINVAL; 228 229 ret = cpuidle_coupled_state_verify(drv); 230 if (ret) 231 return ret; 232 233 if (cpuidle_disabled()) 234 return -ENODEV; 235 236 __cpuidle_driver_init(drv); 237 238 ret = __cpuidle_set_driver(drv); 239 if (ret) 240 return ret; 241 242 if (drv->bctimer) 243 on_each_cpu_mask(drv->cpumask, cpuidle_setup_broadcast_timer, 244 (void *)1, 1); 245 246 return 0; 247 } 248 249 /** 250 * __cpuidle_unregister_driver - unregister the driver 251 * @drv: a valid pointer to a struct cpuidle_driver 252 * 253 * Check if the driver is no longer in use, reset the global cpuidle driver 254 * variable(s) and disable the timer broadcast notification mechanism if it was 255 * in use. 256 * 257 */ 258 static void __cpuidle_unregister_driver(struct cpuidle_driver *drv) 259 { 260 if (drv->bctimer) { 261 drv->bctimer = 0; 262 on_each_cpu_mask(drv->cpumask, cpuidle_setup_broadcast_timer, 263 NULL, 1); 264 } 265 266 __cpuidle_unset_driver(drv); 267 } 268 269 /** 270 * cpuidle_register_driver - registers a driver 271 * @drv: a pointer to a valid struct cpuidle_driver 272 * 273 * Register the driver under a lock to prevent concurrent attempts to 274 * [un]register the driver from occurring at the same time. 275 * 276 * Returns 0 on success, a negative error code (returned by 277 * __cpuidle_register_driver()) otherwise. 278 */ 279 int cpuidle_register_driver(struct cpuidle_driver *drv) 280 { 281 struct cpuidle_governor *gov; 282 int ret; 283 284 spin_lock(&cpuidle_driver_lock); 285 ret = __cpuidle_register_driver(drv); 286 spin_unlock(&cpuidle_driver_lock); 287 288 if (!ret && !strlen(param_governor) && drv->governor && 289 (cpuidle_get_driver() == drv)) { 290 mutex_lock(&cpuidle_lock); 291 gov = cpuidle_find_governor(drv->governor); 292 if (gov) { 293 cpuidle_prev_governor = cpuidle_curr_governor; 294 if (cpuidle_switch_governor(gov) < 0) 295 cpuidle_prev_governor = NULL; 296 } 297 mutex_unlock(&cpuidle_lock); 298 } 299 300 return ret; 301 } 302 EXPORT_SYMBOL_GPL(cpuidle_register_driver); 303 304 /** 305 * cpuidle_unregister_driver - unregisters a driver 306 * @drv: a pointer to a valid struct cpuidle_driver 307 * 308 * Unregisters the cpuidle driver under a lock to prevent concurrent attempts 309 * to [un]register the driver from occurring at the same time. @drv has to 310 * match the currently registered driver. 311 */ 312 void cpuidle_unregister_driver(struct cpuidle_driver *drv) 313 { 314 bool enabled = (cpuidle_get_driver() == drv); 315 316 spin_lock(&cpuidle_driver_lock); 317 __cpuidle_unregister_driver(drv); 318 spin_unlock(&cpuidle_driver_lock); 319 320 if (!enabled) 321 return; 322 323 mutex_lock(&cpuidle_lock); 324 if (cpuidle_prev_governor) { 325 if (!cpuidle_switch_governor(cpuidle_prev_governor)) 326 cpuidle_prev_governor = NULL; 327 } 328 mutex_unlock(&cpuidle_lock); 329 } 330 EXPORT_SYMBOL_GPL(cpuidle_unregister_driver); 331 332 /** 333 * cpuidle_get_driver - return the driver tied to the current CPU. 334 * 335 * Returns a struct cpuidle_driver pointer, or NULL if no driver is registered. 336 */ 337 struct cpuidle_driver *cpuidle_get_driver(void) 338 { 339 struct cpuidle_driver *drv; 340 int cpu; 341 342 cpu = get_cpu(); 343 drv = __cpuidle_get_cpu_driver(cpu); 344 put_cpu(); 345 346 return drv; 347 } 348 EXPORT_SYMBOL_GPL(cpuidle_get_driver); 349 350 /** 351 * cpuidle_get_cpu_driver - return the driver registered for a CPU. 352 * @dev: a valid pointer to a struct cpuidle_device 353 * 354 * Returns a struct cpuidle_driver pointer, or NULL if no driver is registered 355 * for the CPU associated with @dev. 356 */ 357 struct cpuidle_driver *cpuidle_get_cpu_driver(struct cpuidle_device *dev) 358 { 359 if (!dev) 360 return NULL; 361 362 return __cpuidle_get_cpu_driver(dev->cpu); 363 } 364 EXPORT_SYMBOL_GPL(cpuidle_get_cpu_driver); 365 366 /** 367 * cpuidle_driver_state_disabled - Disable or enable an idle state 368 * @drv: cpuidle driver owning the state 369 * @idx: State index 370 * @disable: Whether or not to disable the state 371 */ 372 void cpuidle_driver_state_disabled(struct cpuidle_driver *drv, int idx, 373 bool disable) 374 { 375 unsigned int cpu; 376 377 mutex_lock(&cpuidle_lock); 378 379 spin_lock(&cpuidle_driver_lock); 380 381 if (!drv->cpumask) { 382 drv->states[idx].flags |= CPUIDLE_FLAG_UNUSABLE; 383 goto unlock; 384 } 385 386 for_each_cpu(cpu, drv->cpumask) { 387 struct cpuidle_device *dev = per_cpu(cpuidle_devices, cpu); 388 389 if (!dev) 390 continue; 391 392 if (disable) 393 dev->states_usage[idx].disable |= CPUIDLE_STATE_DISABLED_BY_DRIVER; 394 else 395 dev->states_usage[idx].disable &= ~CPUIDLE_STATE_DISABLED_BY_DRIVER; 396 } 397 398 unlock: 399 spin_unlock(&cpuidle_driver_lock); 400 401 mutex_unlock(&cpuidle_lock); 402 } 403