1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * CPUFreq governor based on scheduler-provided CPU utilization data. 4 * 5 * Copyright (C) 2016, Intel Corporation 6 * Author: Rafael J. Wysocki <rafael.j.wysocki@intel.com> 7 */ 8 #include <uapi/linux/sched/types.h> 9 #include "sched.h" 10 11 #define IOWAIT_BOOST_MIN (SCHED_CAPACITY_SCALE / 8) 12 13 struct sugov_tunables { 14 struct gov_attr_set attr_set; 15 unsigned int rate_limit_us; 16 }; 17 18 struct sugov_policy { 19 struct cpufreq_policy *policy; 20 21 struct sugov_tunables *tunables; 22 struct list_head tunables_hook; 23 24 raw_spinlock_t update_lock; 25 u64 last_freq_update_time; 26 s64 freq_update_delay_ns; 27 unsigned int next_freq; 28 unsigned int cached_raw_freq; 29 30 /* The next fields are only needed if fast switch cannot be used: */ 31 struct irq_work irq_work; 32 struct kthread_work work; 33 struct mutex work_lock; 34 struct kthread_worker worker; 35 struct task_struct *thread; 36 bool work_in_progress; 37 38 bool limits_changed; 39 bool need_freq_update; 40 }; 41 42 struct sugov_cpu { 43 struct update_util_data update_util; 44 struct sugov_policy *sg_policy; 45 unsigned int cpu; 46 47 bool iowait_boost_pending; 48 unsigned int iowait_boost; 49 u64 last_update; 50 51 unsigned long util; 52 unsigned long bw_min; 53 unsigned long bw_max; 54 55 /* The field below is for single-CPU policies only: */ 56 #ifdef CONFIG_NO_HZ_COMMON 57 unsigned long saved_idle_calls; 58 #endif 59 }; 60 61 static DEFINE_PER_CPU(struct sugov_cpu, sugov_cpu); 62 63 /************************ Governor internals ***********************/ 64 65 static void sugov_update_rate_limit_us(struct sugov_policy *sg_policy) 66 { 67 /* 68 * Cast rate_limit_us before multiplication to force 64-bit arithmetic. 69 * Otherwise, on 32-bit platforms, both operands are converted to 70 * 32-bit unsigned long and the multiplication may overflow. 71 */ 72 sg_policy->freq_update_delay_ns = 73 (s64)sg_policy->tunables->rate_limit_us * NSEC_PER_USEC; 74 } 75 76 static void sugov_update_rate_limit_us_sync(struct sugov_policy *sg_policy) 77 { 78 /* 79 * Serialize with rate_limit_us_store() to prevent this path from using 80 * a stale rate limit tunable value to set the limit for the given 81 * policy. 82 */ 83 guard(mutex)(&sg_policy->tunables->attr_set.update_lock); 84 85 sugov_update_rate_limit_us(sg_policy); 86 } 87 88 static bool sugov_should_update_freq(struct sugov_policy *sg_policy, u64 time) 89 { 90 s64 delta_ns; 91 92 /* 93 * Since cpufreq_update_util() is called with rq->lock held for 94 * the @target_cpu, our per-CPU data is fully serialized. 95 * 96 * However, drivers cannot in general deal with cross-CPU 97 * requests, so while get_next_freq() will work, our 98 * sugov_update_commit() call may not for the fast switching platforms. 99 * 100 * Hence stop here for remote requests if they aren't supported 101 * by the hardware, as calculating the frequency is pointless if 102 * we cannot in fact act on it. 103 * 104 * This is needed on the slow switching platforms too to prevent CPUs 105 * going offline from leaving stale IRQ work items behind. 106 */ 107 if (!cpufreq_this_cpu_can_update(sg_policy->policy)) 108 return false; 109 110 if (unlikely(READ_ONCE(sg_policy->limits_changed))) { 111 WRITE_ONCE(sg_policy->limits_changed, false); 112 sg_policy->need_freq_update = true; 113 114 /* 115 * The above limits_changed update must occur before the reads 116 * of policy limits in cpufreq_driver_resolve_freq() or a policy 117 * limits update might be missed, so use a memory barrier to 118 * ensure it. 119 * 120 * This pairs with the write memory barrier in sugov_limits(). 121 */ 122 smp_mb(); 123 124 return true; 125 } else if (sg_policy->need_freq_update) { 126 /* ignore_dl_rate_limit() wants a new frequency to be found. */ 127 return true; 128 } 129 130 delta_ns = time - sg_policy->last_freq_update_time; 131 132 return delta_ns >= sg_policy->freq_update_delay_ns; 133 } 134 135 static bool sugov_update_next_freq(struct sugov_policy *sg_policy, u64 time, 136 unsigned int next_freq) 137 { 138 if (sg_policy->need_freq_update) { 139 sg_policy->need_freq_update = false; 140 /* 141 * The policy limits have changed, but if the return value of 142 * cpufreq_driver_resolve_freq() after applying the new limits 143 * is still equal to the previously selected frequency, the 144 * driver callback need not be invoked unless the driver 145 * specifically wants that to happen on every update of the 146 * policy limits. 147 */ 148 if (sg_policy->next_freq == next_freq && 149 !cpufreq_driver_test_flags(CPUFREQ_NEED_UPDATE_LIMITS)) 150 return false; 151 } else if (sg_policy->next_freq == next_freq) { 152 return false; 153 } 154 155 sg_policy->next_freq = next_freq; 156 sg_policy->last_freq_update_time = time; 157 158 return true; 159 } 160 161 static void sugov_deferred_update(struct sugov_policy *sg_policy) 162 { 163 if (!sg_policy->work_in_progress) { 164 sg_policy->work_in_progress = true; 165 irq_work_queue(&sg_policy->irq_work); 166 } 167 } 168 169 /** 170 * get_capacity_ref_freq - get the reference frequency that has been used to 171 * correlate frequency and compute capacity for a given cpufreq policy. We use 172 * the CPU managing it for the arch_scale_freq_ref() call in the function. 173 * @policy: the cpufreq policy of the CPU in question. 174 * 175 * Return: the reference CPU frequency to compute a capacity. 176 */ 177 static __always_inline 178 unsigned long get_capacity_ref_freq(struct cpufreq_policy *policy) 179 { 180 unsigned int freq = arch_scale_freq_ref(policy->cpu); 181 182 if (freq) 183 return freq; 184 185 if (arch_scale_freq_invariant()) 186 return policy->cpuinfo.max_freq; 187 188 /* 189 * Apply a 25% margin so that we select a higher frequency than 190 * the current one before the CPU is fully busy: 191 */ 192 return policy->cur + (policy->cur >> 2); 193 } 194 195 /** 196 * get_next_freq - Compute a new frequency for a given cpufreq policy. 197 * @sg_policy: schedutil policy object to compute the new frequency for. 198 * @util: Current CPU utilization. 199 * @max: CPU capacity. 200 * 201 * If the utilization is frequency-invariant, choose the new frequency to be 202 * proportional to it, that is 203 * 204 * next_freq = C * max_freq * util / max 205 * 206 * Otherwise, approximate the would-be frequency-invariant utilization by 207 * util_raw * (curr_freq / max_freq) which leads to 208 * 209 * next_freq = C * curr_freq * util_raw / max 210 * 211 * Take C = 1.25 for the frequency tipping point at (util / max) = 0.8. 212 * 213 * The lowest driver-supported frequency which is equal or greater than the raw 214 * next_freq (as calculated above) is returned, subject to policy min/max and 215 * cpufreq driver limitations. 216 */ 217 static unsigned int get_next_freq(struct sugov_policy *sg_policy, 218 unsigned long util, unsigned long max) 219 { 220 struct cpufreq_policy *policy = sg_policy->policy; 221 unsigned int freq; 222 223 freq = get_capacity_ref_freq(policy); 224 freq = map_util_freq(util, freq, max); 225 226 if (freq == sg_policy->cached_raw_freq && !sg_policy->need_freq_update) 227 return sg_policy->next_freq; 228 229 sg_policy->cached_raw_freq = freq; 230 return cpufreq_driver_resolve_freq(policy, freq); 231 } 232 233 unsigned long sugov_effective_cpu_perf(int cpu, unsigned long actual, 234 unsigned long min, 235 unsigned long max) 236 { 237 /* Add dvfs headroom to actual utilization */ 238 actual = map_util_perf(actual); 239 /* Actually we don't need to target the max performance */ 240 if (actual < max) 241 max = actual; 242 243 /* 244 * Ensure at least minimum performance while providing more compute 245 * capacity when possible. 246 */ 247 return max(min, max); 248 } 249 250 static void sugov_get_util(struct sugov_cpu *sg_cpu, unsigned long boost) 251 { 252 unsigned long min, max, util = scx_cpuperf_target(sg_cpu->cpu); 253 254 if (!scx_switched_all()) 255 util += cpu_util_cfs_boost(sg_cpu->cpu); 256 util = effective_cpu_util(sg_cpu->cpu, util, &min, &max); 257 util = max(util, boost); 258 sg_cpu->bw_min = min; 259 sg_cpu->bw_max = max; 260 sg_cpu->util = sugov_effective_cpu_perf(sg_cpu->cpu, util, min, max); 261 } 262 263 /** 264 * sugov_iowait_reset() - Reset the IO boost status of a CPU. 265 * @sg_cpu: the sugov data for the CPU to boost 266 * @time: the update time from the caller 267 * @set_iowait_boost: true if an IO boost has been requested 268 * 269 * The IO wait boost of a task is disabled after a tick since the last update 270 * of a CPU. If a new IO wait boost is requested after more then a tick, then 271 * we enable the boost starting from IOWAIT_BOOST_MIN, which improves energy 272 * efficiency by ignoring sporadic wakeups from IO. 273 */ 274 static bool sugov_iowait_reset(struct sugov_cpu *sg_cpu, u64 time, 275 bool set_iowait_boost) 276 { 277 s64 delta_ns = time - sg_cpu->last_update; 278 279 /* Reset boost only if a tick has elapsed since last request */ 280 if (delta_ns <= TICK_NSEC) 281 return false; 282 283 sg_cpu->iowait_boost = set_iowait_boost ? IOWAIT_BOOST_MIN : 0; 284 sg_cpu->iowait_boost_pending = set_iowait_boost; 285 286 return true; 287 } 288 289 /** 290 * sugov_iowait_boost() - Updates the IO boost status of a CPU. 291 * @sg_cpu: the sugov data for the CPU to boost 292 * @time: the update time from the caller 293 * @flags: SCHED_CPUFREQ_IOWAIT if the task is waking up after an IO wait 294 * 295 * Each time a task wakes up after an IO operation, the CPU utilization can be 296 * boosted to a certain utilization which doubles at each "frequent and 297 * successive" wakeup from IO, ranging from IOWAIT_BOOST_MIN to the utilization 298 * of the maximum OPP. 299 * 300 * To keep doubling, an IO boost has to be requested at least once per tick, 301 * otherwise we restart from the utilization of the minimum OPP. 302 */ 303 static void sugov_iowait_boost(struct sugov_cpu *sg_cpu, u64 time, 304 unsigned int flags) 305 { 306 bool set_iowait_boost = flags & SCHED_CPUFREQ_IOWAIT; 307 308 /* Reset boost if the CPU appears to have been idle enough */ 309 if (sg_cpu->iowait_boost && 310 sugov_iowait_reset(sg_cpu, time, set_iowait_boost)) 311 return; 312 313 /* Boost only tasks waking up after IO */ 314 if (!set_iowait_boost) 315 return; 316 317 /* Ensure boost doubles only one time at each request */ 318 if (sg_cpu->iowait_boost_pending) 319 return; 320 sg_cpu->iowait_boost_pending = true; 321 322 /* Double the boost at each request */ 323 if (sg_cpu->iowait_boost) { 324 sg_cpu->iowait_boost = 325 min_t(unsigned int, sg_cpu->iowait_boost << 1, SCHED_CAPACITY_SCALE); 326 return; 327 } 328 329 /* First wakeup after IO: start with minimum boost */ 330 sg_cpu->iowait_boost = IOWAIT_BOOST_MIN; 331 } 332 333 /** 334 * sugov_iowait_apply() - Apply the IO boost to a CPU. 335 * @sg_cpu: the sugov data for the cpu to boost 336 * @time: the update time from the caller 337 * @max_cap: the max CPU capacity 338 * 339 * A CPU running a task which woken up after an IO operation can have its 340 * utilization boosted to speed up the completion of those IO operations. 341 * The IO boost value is increased each time a task wakes up from IO, in 342 * sugov_iowait_boost(), and it's instead decreased by this function, 343 * each time an increase has not been requested (!iowait_boost_pending). 344 * 345 * A CPU which also appears to have been idle for at least one tick has also 346 * its IO boost utilization reset. 347 * 348 * This mechanism is designed to boost high frequently IO waiting tasks, while 349 * being more conservative on tasks which does sporadic IO operations. 350 */ 351 static unsigned long sugov_iowait_apply(struct sugov_cpu *sg_cpu, u64 time, 352 unsigned long max_cap) 353 { 354 /* No boost currently required */ 355 if (!sg_cpu->iowait_boost) 356 return 0; 357 358 /* Reset boost if the CPU appears to have been idle enough */ 359 if (sugov_iowait_reset(sg_cpu, time, false)) 360 return 0; 361 362 if (!sg_cpu->iowait_boost_pending) { 363 /* 364 * No boost pending; reduce the boost value. 365 */ 366 sg_cpu->iowait_boost >>= 1; 367 if (sg_cpu->iowait_boost < IOWAIT_BOOST_MIN) { 368 sg_cpu->iowait_boost = 0; 369 return 0; 370 } 371 } 372 373 sg_cpu->iowait_boost_pending = false; 374 375 /* 376 * sg_cpu->util is already in capacity scale; convert iowait_boost 377 * into the same scale so we can compare. 378 */ 379 return (sg_cpu->iowait_boost * max_cap) >> SCHED_CAPACITY_SHIFT; 380 } 381 382 #ifdef CONFIG_NO_HZ_COMMON 383 static bool sugov_hold_freq(struct sugov_cpu *sg_cpu) 384 { 385 unsigned long idle_calls; 386 bool ret; 387 388 /* 389 * The heuristics in this function is for the fair class. For SCX, the 390 * performance target comes directly from the BPF scheduler. Let's just 391 * follow it. 392 */ 393 if (scx_switched_all()) 394 return false; 395 396 /* if capped by uclamp_max, always update to be in compliance */ 397 if (uclamp_rq_is_capped(cpu_rq(sg_cpu->cpu))) 398 return false; 399 400 /* 401 * Maintain the frequency if the CPU has not been idle recently, as 402 * reduction is likely to be premature. 403 */ 404 idle_calls = tick_nohz_get_idle_calls_cpu(sg_cpu->cpu); 405 ret = idle_calls == sg_cpu->saved_idle_calls; 406 407 sg_cpu->saved_idle_calls = idle_calls; 408 return ret; 409 } 410 #else /* !CONFIG_NO_HZ_COMMON: */ 411 static inline bool sugov_hold_freq(struct sugov_cpu *sg_cpu) { return false; } 412 #endif /* !CONFIG_NO_HZ_COMMON */ 413 414 /* 415 * Make sugov_should_update_freq() ignore the rate limit when DL 416 * has increased the utilization. 417 */ 418 static inline void ignore_dl_rate_limit(struct sugov_cpu *sg_cpu) 419 { 420 if (cpu_bw_dl(cpu_rq(sg_cpu->cpu)) > sg_cpu->bw_min) 421 sg_cpu->sg_policy->need_freq_update = true; 422 } 423 424 static inline bool sugov_update_single_common(struct sugov_cpu *sg_cpu, 425 u64 time, unsigned long max_cap, 426 unsigned int flags) 427 { 428 unsigned long boost; 429 430 sugov_iowait_boost(sg_cpu, time, flags); 431 sg_cpu->last_update = time; 432 433 ignore_dl_rate_limit(sg_cpu); 434 435 if (!sugov_should_update_freq(sg_cpu->sg_policy, time)) 436 return false; 437 438 boost = sugov_iowait_apply(sg_cpu, time, max_cap); 439 sugov_get_util(sg_cpu, boost); 440 441 return true; 442 } 443 444 static void sugov_update_single_freq(struct update_util_data *hook, u64 time, 445 unsigned int flags) 446 { 447 struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util); 448 struct sugov_policy *sg_policy = sg_cpu->sg_policy; 449 unsigned int cached_freq = sg_policy->cached_raw_freq; 450 unsigned long max_cap; 451 unsigned int next_f; 452 453 max_cap = arch_scale_cpu_capacity(sg_cpu->cpu); 454 455 if (!sugov_update_single_common(sg_cpu, time, max_cap, flags)) 456 return; 457 458 next_f = get_next_freq(sg_policy, sg_cpu->util, max_cap); 459 460 if (sugov_hold_freq(sg_cpu) && next_f < sg_policy->next_freq && 461 !sg_policy->need_freq_update) { 462 next_f = sg_policy->next_freq; 463 464 /* Restore cached freq as next_freq has changed */ 465 sg_policy->cached_raw_freq = cached_freq; 466 } 467 468 if (!sugov_update_next_freq(sg_policy, time, next_f)) 469 return; 470 471 /* 472 * This code runs under rq->lock for the target CPU, so it won't run 473 * concurrently on two different CPUs for the same target and it is not 474 * necessary to acquire the lock in the fast switch case. 475 */ 476 if (sg_policy->policy->fast_switch_enabled) { 477 cpufreq_driver_fast_switch(sg_policy->policy, next_f); 478 } else { 479 raw_spin_lock(&sg_policy->update_lock); 480 sugov_deferred_update(sg_policy); 481 raw_spin_unlock(&sg_policy->update_lock); 482 } 483 } 484 485 static void sugov_update_single_perf(struct update_util_data *hook, u64 time, 486 unsigned int flags) 487 { 488 struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util); 489 struct sugov_policy *sg_policy = sg_cpu->sg_policy; 490 unsigned long prev_util = sg_cpu->util; 491 unsigned long max_cap; 492 493 /* 494 * Fall back to the "frequency" path if frequency invariance is not 495 * supported, because the direct mapping between the utilization and 496 * the performance levels depends on the frequency invariance. 497 */ 498 if (!arch_scale_freq_invariant()) { 499 sugov_update_single_freq(hook, time, flags); 500 return; 501 } 502 503 max_cap = arch_scale_cpu_capacity(sg_cpu->cpu); 504 505 if (!sugov_update_single_common(sg_cpu, time, max_cap, flags)) 506 return; 507 508 if (sugov_hold_freq(sg_cpu) && sg_cpu->util < prev_util) 509 sg_cpu->util = prev_util; 510 511 cpufreq_driver_adjust_perf(sg_policy->policy, sg_cpu->bw_min, 512 sg_cpu->util, sg_cpu->bw_max, max_cap); 513 514 sg_policy->need_freq_update = false; 515 sg_policy->last_freq_update_time = time; 516 } 517 518 static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu, u64 time) 519 { 520 struct sugov_policy *sg_policy = sg_cpu->sg_policy; 521 struct cpufreq_policy *policy = sg_policy->policy; 522 unsigned long util = 0, max_cap; 523 unsigned int j; 524 525 max_cap = arch_scale_cpu_capacity(sg_cpu->cpu); 526 527 for_each_cpu(j, policy->cpus) { 528 struct sugov_cpu *j_sg_cpu = &per_cpu(sugov_cpu, j); 529 unsigned long boost; 530 531 boost = sugov_iowait_apply(j_sg_cpu, time, max_cap); 532 sugov_get_util(j_sg_cpu, boost); 533 534 util = max(j_sg_cpu->util, util); 535 } 536 537 return get_next_freq(sg_policy, util, max_cap); 538 } 539 540 static void 541 sugov_update_shared(struct update_util_data *hook, u64 time, unsigned int flags) 542 { 543 struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util); 544 struct sugov_policy *sg_policy = sg_cpu->sg_policy; 545 unsigned int next_f; 546 547 raw_spin_lock(&sg_policy->update_lock); 548 549 sugov_iowait_boost(sg_cpu, time, flags); 550 sg_cpu->last_update = time; 551 552 ignore_dl_rate_limit(sg_cpu); 553 554 if (sugov_should_update_freq(sg_policy, time)) { 555 next_f = sugov_next_freq_shared(sg_cpu, time); 556 557 if (!sugov_update_next_freq(sg_policy, time, next_f)) 558 goto unlock; 559 560 if (sg_policy->policy->fast_switch_enabled) 561 cpufreq_driver_fast_switch(sg_policy->policy, next_f); 562 else 563 sugov_deferred_update(sg_policy); 564 } 565 unlock: 566 raw_spin_unlock(&sg_policy->update_lock); 567 } 568 569 static void sugov_work(struct kthread_work *work) 570 { 571 struct sugov_policy *sg_policy = container_of(work, struct sugov_policy, work); 572 unsigned int freq; 573 unsigned long flags; 574 575 /* 576 * Hold sg_policy->update_lock shortly to handle the case where: 577 * in case sg_policy->next_freq is read here, and then updated by 578 * sugov_deferred_update() just before work_in_progress is set to false 579 * here, we may miss queueing the new update. 580 * 581 * Note: If a work was queued after the update_lock is released, 582 * sugov_work() will just be called again by kthread_work code; and the 583 * request will be proceed before the sugov thread sleeps. 584 */ 585 raw_spin_lock_irqsave(&sg_policy->update_lock, flags); 586 freq = sg_policy->next_freq; 587 sg_policy->work_in_progress = false; 588 raw_spin_unlock_irqrestore(&sg_policy->update_lock, flags); 589 590 mutex_lock(&sg_policy->work_lock); 591 __cpufreq_driver_target(sg_policy->policy, freq, CPUFREQ_RELATION_L); 592 mutex_unlock(&sg_policy->work_lock); 593 } 594 595 static void sugov_irq_work(struct irq_work *irq_work) 596 { 597 struct sugov_policy *sg_policy; 598 599 sg_policy = container_of(irq_work, struct sugov_policy, irq_work); 600 601 kthread_queue_work(&sg_policy->worker, &sg_policy->work); 602 } 603 604 /************************** sysfs interface ************************/ 605 606 static struct sugov_tunables *global_tunables; 607 static DEFINE_MUTEX(global_tunables_lock); 608 609 static inline struct sugov_tunables *to_sugov_tunables(struct gov_attr_set *attr_set) 610 { 611 return container_of(attr_set, struct sugov_tunables, attr_set); 612 } 613 614 static ssize_t rate_limit_us_show(struct gov_attr_set *attr_set, char *buf) 615 { 616 struct sugov_tunables *tunables = to_sugov_tunables(attr_set); 617 618 return sysfs_emit(buf, "%u\n", tunables->rate_limit_us); 619 } 620 621 static ssize_t 622 rate_limit_us_store(struct gov_attr_set *attr_set, const char *buf, size_t count) 623 { 624 struct sugov_tunables *tunables = to_sugov_tunables(attr_set); 625 struct sugov_policy *sg_policy; 626 unsigned int rate_limit_us; 627 628 if (kstrtouint(buf, 10, &rate_limit_us)) 629 return -EINVAL; 630 631 tunables->rate_limit_us = rate_limit_us; 632 633 list_for_each_entry(sg_policy, &attr_set->policy_list, tunables_hook) 634 sugov_update_rate_limit_us(sg_policy); 635 636 return count; 637 } 638 639 static struct governor_attr rate_limit_us = __ATTR_RW(rate_limit_us); 640 641 static struct attribute *sugov_attrs[] = { 642 &rate_limit_us.attr, 643 NULL 644 }; 645 ATTRIBUTE_GROUPS(sugov); 646 647 static void sugov_tunables_free(struct kobject *kobj) 648 { 649 struct gov_attr_set *attr_set = to_gov_attr_set(kobj); 650 651 kfree(to_sugov_tunables(attr_set)); 652 } 653 654 static const struct kobj_type sugov_tunables_ktype = { 655 .default_groups = sugov_groups, 656 .sysfs_ops = &governor_sysfs_ops, 657 .release = &sugov_tunables_free, 658 }; 659 660 /********************** cpufreq governor interface *********************/ 661 662 static struct cpufreq_governor schedutil_gov; 663 664 static struct sugov_policy *sugov_policy_alloc(struct cpufreq_policy *policy) 665 { 666 struct sugov_policy *sg_policy; 667 668 sg_policy = kzalloc_obj(*sg_policy); 669 if (!sg_policy) 670 return NULL; 671 672 sg_policy->policy = policy; 673 raw_spin_lock_init(&sg_policy->update_lock); 674 return sg_policy; 675 } 676 677 static void sugov_policy_free(struct sugov_policy *sg_policy) 678 { 679 kfree(sg_policy); 680 } 681 682 static int sugov_kthread_create(struct sugov_policy *sg_policy) 683 { 684 struct task_struct *thread; 685 struct sched_attr attr = { 686 .size = sizeof(struct sched_attr), 687 .sched_policy = SCHED_DEADLINE, 688 .sched_flags = SCHED_FLAG_SUGOV, 689 .sched_nice = 0, 690 .sched_priority = 0, 691 /* 692 * Fake (unused) bandwidth; workaround to "fix" 693 * priority inheritance. 694 */ 695 .sched_runtime = NSEC_PER_MSEC, 696 .sched_deadline = 10 * NSEC_PER_MSEC, 697 .sched_period = 10 * NSEC_PER_MSEC, 698 }; 699 struct cpufreq_policy *policy = sg_policy->policy; 700 int ret; 701 702 /* kthread only required for slow path */ 703 if (policy->fast_switch_enabled) 704 return 0; 705 706 kthread_init_work(&sg_policy->work, sugov_work); 707 kthread_init_worker(&sg_policy->worker); 708 thread = kthread_create(kthread_worker_fn, &sg_policy->worker, 709 "sugov:%d", 710 cpumask_first(policy->related_cpus)); 711 if (IS_ERR(thread)) { 712 pr_err("failed to create sugov thread: %pe\n", thread); 713 return PTR_ERR(thread); 714 } 715 716 ret = sched_setattr_nocheck(thread, &attr); 717 if (ret) { 718 kthread_stop(thread); 719 pr_warn("%s: failed to set SCHED_DEADLINE\n", __func__); 720 return ret; 721 } 722 723 sg_policy->thread = thread; 724 if (policy->dvfs_possible_from_any_cpu) 725 set_cpus_allowed_ptr(thread, policy->related_cpus); 726 else 727 kthread_bind_mask(thread, policy->related_cpus); 728 729 init_irq_work(&sg_policy->irq_work, sugov_irq_work); 730 mutex_init(&sg_policy->work_lock); 731 732 wake_up_process(thread); 733 734 return 0; 735 } 736 737 static void sugov_kthread_stop(struct sugov_policy *sg_policy) 738 { 739 /* kthread only required for slow path */ 740 if (sg_policy->policy->fast_switch_enabled) 741 return; 742 743 kthread_flush_worker(&sg_policy->worker); 744 kthread_stop(sg_policy->thread); 745 mutex_destroy(&sg_policy->work_lock); 746 } 747 748 static struct sugov_tunables *sugov_tunables_alloc(struct sugov_policy *sg_policy) 749 { 750 struct sugov_tunables *tunables; 751 752 tunables = kzalloc_obj(*tunables); 753 if (tunables) { 754 gov_attr_set_init(&tunables->attr_set, &sg_policy->tunables_hook); 755 if (!have_governor_per_policy()) 756 global_tunables = tunables; 757 } 758 return tunables; 759 } 760 761 static void sugov_clear_global_tunables(void) 762 { 763 if (!have_governor_per_policy()) 764 global_tunables = NULL; 765 } 766 767 static int sugov_init(struct cpufreq_policy *policy) 768 { 769 struct sugov_policy *sg_policy; 770 struct sugov_tunables *tunables; 771 int ret = 0; 772 773 /* State should be equivalent to EXIT */ 774 if (policy->governor_data) 775 return -EBUSY; 776 777 cpufreq_enable_fast_switch(policy); 778 779 sg_policy = sugov_policy_alloc(policy); 780 if (!sg_policy) { 781 ret = -ENOMEM; 782 goto disable_fast_switch; 783 } 784 785 ret = sugov_kthread_create(sg_policy); 786 if (ret) 787 goto free_sg_policy; 788 789 mutex_lock(&global_tunables_lock); 790 791 if (global_tunables) { 792 if (WARN_ON(have_governor_per_policy())) { 793 ret = -EINVAL; 794 goto stop_kthread; 795 } 796 policy->governor_data = sg_policy; 797 sg_policy->tunables = global_tunables; 798 799 gov_attr_set_get(&global_tunables->attr_set, &sg_policy->tunables_hook); 800 goto out; 801 } 802 803 tunables = sugov_tunables_alloc(sg_policy); 804 if (!tunables) { 805 ret = -ENOMEM; 806 goto stop_kthread; 807 } 808 809 tunables->rate_limit_us = cpufreq_policy_transition_delay_us(policy); 810 811 policy->governor_data = sg_policy; 812 sg_policy->tunables = tunables; 813 814 ret = kobject_init_and_add(&tunables->attr_set.kobj, &sugov_tunables_ktype, 815 get_governor_parent_kobj(policy), "%s", 816 schedutil_gov.name); 817 if (ret) 818 goto fail; 819 820 out: 821 /* 822 * Schedutil is the preferred governor for EAS, so rebuild sched domains 823 * on governor changes to make sure the scheduler knows about them. 824 */ 825 em_rebuild_sched_domains(); 826 mutex_unlock(&global_tunables_lock); 827 return 0; 828 829 fail: 830 kobject_put(&tunables->attr_set.kobj); 831 policy->governor_data = NULL; 832 sugov_clear_global_tunables(); 833 834 stop_kthread: 835 sugov_kthread_stop(sg_policy); 836 mutex_unlock(&global_tunables_lock); 837 838 free_sg_policy: 839 sugov_policy_free(sg_policy); 840 841 disable_fast_switch: 842 cpufreq_disable_fast_switch(policy); 843 844 pr_err("initialization failed (error %d)\n", ret); 845 return ret; 846 } 847 848 static void sugov_exit(struct cpufreq_policy *policy) 849 { 850 struct sugov_policy *sg_policy = policy->governor_data; 851 struct sugov_tunables *tunables = sg_policy->tunables; 852 unsigned int count; 853 854 mutex_lock(&global_tunables_lock); 855 856 count = gov_attr_set_put(&tunables->attr_set, &sg_policy->tunables_hook); 857 policy->governor_data = NULL; 858 if (!count) 859 sugov_clear_global_tunables(); 860 861 mutex_unlock(&global_tunables_lock); 862 863 sugov_kthread_stop(sg_policy); 864 sugov_policy_free(sg_policy); 865 cpufreq_disable_fast_switch(policy); 866 867 em_rebuild_sched_domains(); 868 } 869 870 static int sugov_start(struct cpufreq_policy *policy) 871 { 872 struct sugov_policy *sg_policy = policy->governor_data; 873 void (*uu)(struct update_util_data *data, u64 time, unsigned int flags); 874 unsigned int cpu; 875 876 sugov_update_rate_limit_us_sync(sg_policy); 877 sg_policy->last_freq_update_time = 0; 878 sg_policy->next_freq = 0; 879 sg_policy->work_in_progress = false; 880 sg_policy->limits_changed = false; 881 sg_policy->cached_raw_freq = 0; 882 883 sg_policy->need_freq_update = cpufreq_driver_test_flags(CPUFREQ_NEED_UPDATE_LIMITS); 884 885 if (policy_is_shared(policy)) 886 uu = sugov_update_shared; 887 else if (policy->fast_switch_enabled && cpufreq_driver_has_adjust_perf()) 888 uu = sugov_update_single_perf; 889 else 890 uu = sugov_update_single_freq; 891 892 for_each_cpu(cpu, policy->cpus) { 893 struct sugov_cpu *sg_cpu = &per_cpu(sugov_cpu, cpu); 894 895 memset(sg_cpu, 0, sizeof(*sg_cpu)); 896 sg_cpu->cpu = cpu; 897 sg_cpu->sg_policy = sg_policy; 898 } 899 900 /* 901 * Publish the hooks only after all per-CPU data is initialized, so a 902 * shared policy's sugov_update_shared() never reads an uninitialized 903 * sibling sugov_cpu. 904 */ 905 for_each_cpu(cpu, policy->cpus) { 906 struct sugov_cpu *sg_cpu = &per_cpu(sugov_cpu, cpu); 907 908 cpufreq_add_update_util_hook(cpu, &sg_cpu->update_util, uu); 909 } 910 911 return 0; 912 } 913 914 static void sugov_stop(struct cpufreq_policy *policy) 915 { 916 struct sugov_policy *sg_policy = policy->governor_data; 917 unsigned int cpu; 918 919 for_each_cpu(cpu, policy->cpus) 920 cpufreq_remove_update_util_hook(cpu); 921 922 synchronize_rcu(); 923 924 if (!policy->fast_switch_enabled) { 925 irq_work_sync(&sg_policy->irq_work); 926 kthread_cancel_work_sync(&sg_policy->work); 927 } 928 } 929 930 static void sugov_limits(struct cpufreq_policy *policy) 931 { 932 struct sugov_policy *sg_policy = policy->governor_data; 933 934 if (!policy->fast_switch_enabled) { 935 mutex_lock(&sg_policy->work_lock); 936 cpufreq_policy_apply_limits(policy); 937 mutex_unlock(&sg_policy->work_lock); 938 } 939 940 /* 941 * The limits_changed update below must take place before the updates 942 * of policy limits in cpufreq_set_policy() or a policy limits update 943 * might be missed, so use a memory barrier to ensure it. 944 * 945 * This pairs with the memory barrier in sugov_should_update_freq(). 946 */ 947 smp_wmb(); 948 949 WRITE_ONCE(sg_policy->limits_changed, true); 950 } 951 952 static struct cpufreq_governor schedutil_gov = { 953 .name = "schedutil", 954 .owner = THIS_MODULE, 955 .flags = CPUFREQ_GOV_DYNAMIC_SWITCHING, 956 .init = sugov_init, 957 .exit = sugov_exit, 958 .start = sugov_start, 959 .stop = sugov_stop, 960 .limits = sugov_limits, 961 }; 962 963 #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_SCHEDUTIL 964 struct cpufreq_governor *cpufreq_default_governor(void) 965 { 966 return &schedutil_gov; 967 } 968 #endif 969 970 bool sugov_is_governor(struct cpufreq_policy *policy) 971 { 972 return policy->governor == &schedutil_gov; 973 } 974 975 cpufreq_governor_init(schedutil_gov); 976