1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * CPUFreq governor based on scheduler-provided CPU utilization data. 4 * 5 * Copyright (C) 2016, Intel Corporation 6 * Author: Rafael J. Wysocki <rafael.j.wysocki@intel.com> 7 */ 8 #include <uapi/linux/sched/types.h> 9 #include "sched.h" 10 11 #define IOWAIT_BOOST_MIN (SCHED_CAPACITY_SCALE / 8) 12 13 struct sugov_tunables { 14 struct gov_attr_set attr_set; 15 unsigned int rate_limit_us; 16 }; 17 18 struct sugov_policy { 19 struct cpufreq_policy *policy; 20 21 struct sugov_tunables *tunables; 22 struct list_head tunables_hook; 23 24 raw_spinlock_t update_lock; 25 u64 last_freq_update_time; 26 s64 freq_update_delay_ns; 27 unsigned int next_freq; 28 unsigned int cached_raw_freq; 29 30 /* The next fields are only needed if fast switch cannot be used: */ 31 struct irq_work irq_work; 32 struct kthread_work work; 33 struct mutex work_lock; 34 struct kthread_worker worker; 35 struct task_struct *thread; 36 bool work_in_progress; 37 38 bool limits_changed; 39 bool need_freq_update; 40 }; 41 42 struct sugov_cpu { 43 struct update_util_data update_util; 44 struct sugov_policy *sg_policy; 45 unsigned int cpu; 46 47 bool iowait_boost_pending; 48 unsigned int iowait_boost; 49 u64 last_update; 50 51 unsigned long util; 52 unsigned long bw_min; 53 unsigned long bw_max; 54 55 /* The field below is for single-CPU policies only: */ 56 #ifdef CONFIG_NO_HZ_COMMON 57 unsigned long saved_idle_calls; 58 #endif 59 }; 60 61 static DEFINE_PER_CPU(struct sugov_cpu, sugov_cpu); 62 63 /************************ Governor internals ***********************/ 64 65 static void sugov_update_rate_limit_us(struct sugov_policy *sg_policy) 66 { 67 /* 68 * Cast rate_limit_us before multiplication to force 64-bit arithmetic. 69 * Otherwise, on 32-bit platforms, both operands are converted to 70 * 32-bit unsigned long and the multiplication may overflow. 71 */ 72 sg_policy->freq_update_delay_ns = 73 (s64)sg_policy->tunables->rate_limit_us * NSEC_PER_USEC; 74 } 75 76 static bool sugov_should_update_freq(struct sugov_policy *sg_policy, u64 time) 77 { 78 s64 delta_ns; 79 80 /* 81 * Since cpufreq_update_util() is called with rq->lock held for 82 * the @target_cpu, our per-CPU data is fully serialized. 83 * 84 * However, drivers cannot in general deal with cross-CPU 85 * requests, so while get_next_freq() will work, our 86 * sugov_update_commit() call may not for the fast switching platforms. 87 * 88 * Hence stop here for remote requests if they aren't supported 89 * by the hardware, as calculating the frequency is pointless if 90 * we cannot in fact act on it. 91 * 92 * This is needed on the slow switching platforms too to prevent CPUs 93 * going offline from leaving stale IRQ work items behind. 94 */ 95 if (!cpufreq_this_cpu_can_update(sg_policy->policy)) 96 return false; 97 98 if (unlikely(READ_ONCE(sg_policy->limits_changed))) { 99 WRITE_ONCE(sg_policy->limits_changed, false); 100 sg_policy->need_freq_update = true; 101 102 /* 103 * The above limits_changed update must occur before the reads 104 * of policy limits in cpufreq_driver_resolve_freq() or a policy 105 * limits update might be missed, so use a memory barrier to 106 * ensure it. 107 * 108 * This pairs with the write memory barrier in sugov_limits(). 109 */ 110 smp_mb(); 111 112 return true; 113 } else if (sg_policy->need_freq_update) { 114 /* ignore_dl_rate_limit() wants a new frequency to be found. */ 115 return true; 116 } 117 118 delta_ns = time - sg_policy->last_freq_update_time; 119 120 return delta_ns >= sg_policy->freq_update_delay_ns; 121 } 122 123 static bool sugov_update_next_freq(struct sugov_policy *sg_policy, u64 time, 124 unsigned int next_freq) 125 { 126 if (sg_policy->need_freq_update) { 127 sg_policy->need_freq_update = false; 128 /* 129 * The policy limits have changed, but if the return value of 130 * cpufreq_driver_resolve_freq() after applying the new limits 131 * is still equal to the previously selected frequency, the 132 * driver callback need not be invoked unless the driver 133 * specifically wants that to happen on every update of the 134 * policy limits. 135 */ 136 if (sg_policy->next_freq == next_freq && 137 !cpufreq_driver_test_flags(CPUFREQ_NEED_UPDATE_LIMITS)) 138 return false; 139 } else if (sg_policy->next_freq == next_freq) { 140 return false; 141 } 142 143 sg_policy->next_freq = next_freq; 144 sg_policy->last_freq_update_time = time; 145 146 return true; 147 } 148 149 static void sugov_deferred_update(struct sugov_policy *sg_policy) 150 { 151 if (!sg_policy->work_in_progress) { 152 sg_policy->work_in_progress = true; 153 irq_work_queue(&sg_policy->irq_work); 154 } 155 } 156 157 /** 158 * get_capacity_ref_freq - get the reference frequency that has been used to 159 * correlate frequency and compute capacity for a given cpufreq policy. We use 160 * the CPU managing it for the arch_scale_freq_ref() call in the function. 161 * @policy: the cpufreq policy of the CPU in question. 162 * 163 * Return: the reference CPU frequency to compute a capacity. 164 */ 165 static __always_inline 166 unsigned long get_capacity_ref_freq(struct cpufreq_policy *policy) 167 { 168 unsigned int freq = arch_scale_freq_ref(policy->cpu); 169 170 if (freq) 171 return freq; 172 173 if (arch_scale_freq_invariant()) 174 return policy->cpuinfo.max_freq; 175 176 /* 177 * Apply a 25% margin so that we select a higher frequency than 178 * the current one before the CPU is fully busy: 179 */ 180 return policy->cur + (policy->cur >> 2); 181 } 182 183 /** 184 * get_next_freq - Compute a new frequency for a given cpufreq policy. 185 * @sg_policy: schedutil policy object to compute the new frequency for. 186 * @util: Current CPU utilization. 187 * @max: CPU capacity. 188 * 189 * If the utilization is frequency-invariant, choose the new frequency to be 190 * proportional to it, that is 191 * 192 * next_freq = C * max_freq * util / max 193 * 194 * Otherwise, approximate the would-be frequency-invariant utilization by 195 * util_raw * (curr_freq / max_freq) which leads to 196 * 197 * next_freq = C * curr_freq * util_raw / max 198 * 199 * Take C = 1.25 for the frequency tipping point at (util / max) = 0.8. 200 * 201 * The lowest driver-supported frequency which is equal or greater than the raw 202 * next_freq (as calculated above) is returned, subject to policy min/max and 203 * cpufreq driver limitations. 204 */ 205 static unsigned int get_next_freq(struct sugov_policy *sg_policy, 206 unsigned long util, unsigned long max) 207 { 208 struct cpufreq_policy *policy = sg_policy->policy; 209 unsigned int freq; 210 211 freq = get_capacity_ref_freq(policy); 212 freq = map_util_freq(util, freq, max); 213 214 if (freq == sg_policy->cached_raw_freq && !sg_policy->need_freq_update) 215 return sg_policy->next_freq; 216 217 sg_policy->cached_raw_freq = freq; 218 return cpufreq_driver_resolve_freq(policy, freq); 219 } 220 221 unsigned long sugov_effective_cpu_perf(int cpu, unsigned long actual, 222 unsigned long min, 223 unsigned long max) 224 { 225 /* Add dvfs headroom to actual utilization */ 226 actual = map_util_perf(actual); 227 /* Actually we don't need to target the max performance */ 228 if (actual < max) 229 max = actual; 230 231 /* 232 * Ensure at least minimum performance while providing more compute 233 * capacity when possible. 234 */ 235 return max(min, max); 236 } 237 238 static void sugov_get_util(struct sugov_cpu *sg_cpu, unsigned long boost) 239 { 240 unsigned long min, max, util = scx_cpuperf_target(sg_cpu->cpu); 241 242 if (!scx_switched_all()) 243 util += cpu_util_cfs_boost(sg_cpu->cpu); 244 util = effective_cpu_util(sg_cpu->cpu, util, &min, &max); 245 util = max(util, boost); 246 sg_cpu->bw_min = min; 247 sg_cpu->bw_max = max; 248 sg_cpu->util = sugov_effective_cpu_perf(sg_cpu->cpu, util, min, max); 249 } 250 251 /** 252 * sugov_iowait_reset() - Reset the IO boost status of a CPU. 253 * @sg_cpu: the sugov data for the CPU to boost 254 * @time: the update time from the caller 255 * @set_iowait_boost: true if an IO boost has been requested 256 * 257 * The IO wait boost of a task is disabled after a tick since the last update 258 * of a CPU. If a new IO wait boost is requested after more then a tick, then 259 * we enable the boost starting from IOWAIT_BOOST_MIN, which improves energy 260 * efficiency by ignoring sporadic wakeups from IO. 261 */ 262 static bool sugov_iowait_reset(struct sugov_cpu *sg_cpu, u64 time, 263 bool set_iowait_boost) 264 { 265 s64 delta_ns = time - sg_cpu->last_update; 266 267 /* Reset boost only if a tick has elapsed since last request */ 268 if (delta_ns <= TICK_NSEC) 269 return false; 270 271 sg_cpu->iowait_boost = set_iowait_boost ? IOWAIT_BOOST_MIN : 0; 272 sg_cpu->iowait_boost_pending = set_iowait_boost; 273 274 return true; 275 } 276 277 /** 278 * sugov_iowait_boost() - Updates the IO boost status of a CPU. 279 * @sg_cpu: the sugov data for the CPU to boost 280 * @time: the update time from the caller 281 * @flags: SCHED_CPUFREQ_IOWAIT if the task is waking up after an IO wait 282 * 283 * Each time a task wakes up after an IO operation, the CPU utilization can be 284 * boosted to a certain utilization which doubles at each "frequent and 285 * successive" wakeup from IO, ranging from IOWAIT_BOOST_MIN to the utilization 286 * of the maximum OPP. 287 * 288 * To keep doubling, an IO boost has to be requested at least once per tick, 289 * otherwise we restart from the utilization of the minimum OPP. 290 */ 291 static void sugov_iowait_boost(struct sugov_cpu *sg_cpu, u64 time, 292 unsigned int flags) 293 { 294 bool set_iowait_boost = flags & SCHED_CPUFREQ_IOWAIT; 295 296 /* Reset boost if the CPU appears to have been idle enough */ 297 if (sg_cpu->iowait_boost && 298 sugov_iowait_reset(sg_cpu, time, set_iowait_boost)) 299 return; 300 301 /* Boost only tasks waking up after IO */ 302 if (!set_iowait_boost) 303 return; 304 305 /* Ensure boost doubles only one time at each request */ 306 if (sg_cpu->iowait_boost_pending) 307 return; 308 sg_cpu->iowait_boost_pending = true; 309 310 /* Double the boost at each request */ 311 if (sg_cpu->iowait_boost) { 312 sg_cpu->iowait_boost = 313 min_t(unsigned int, sg_cpu->iowait_boost << 1, SCHED_CAPACITY_SCALE); 314 return; 315 } 316 317 /* First wakeup after IO: start with minimum boost */ 318 sg_cpu->iowait_boost = IOWAIT_BOOST_MIN; 319 } 320 321 /** 322 * sugov_iowait_apply() - Apply the IO boost to a CPU. 323 * @sg_cpu: the sugov data for the cpu to boost 324 * @time: the update time from the caller 325 * @max_cap: the max CPU capacity 326 * 327 * A CPU running a task which woken up after an IO operation can have its 328 * utilization boosted to speed up the completion of those IO operations. 329 * The IO boost value is increased each time a task wakes up from IO, in 330 * sugov_iowait_boost(), and it's instead decreased by this function, 331 * each time an increase has not been requested (!iowait_boost_pending). 332 * 333 * A CPU which also appears to have been idle for at least one tick has also 334 * its IO boost utilization reset. 335 * 336 * This mechanism is designed to boost high frequently IO waiting tasks, while 337 * being more conservative on tasks which does sporadic IO operations. 338 */ 339 static unsigned long sugov_iowait_apply(struct sugov_cpu *sg_cpu, u64 time, 340 unsigned long max_cap) 341 { 342 /* No boost currently required */ 343 if (!sg_cpu->iowait_boost) 344 return 0; 345 346 /* Reset boost if the CPU appears to have been idle enough */ 347 if (sugov_iowait_reset(sg_cpu, time, false)) 348 return 0; 349 350 if (!sg_cpu->iowait_boost_pending) { 351 /* 352 * No boost pending; reduce the boost value. 353 */ 354 sg_cpu->iowait_boost >>= 1; 355 if (sg_cpu->iowait_boost < IOWAIT_BOOST_MIN) { 356 sg_cpu->iowait_boost = 0; 357 return 0; 358 } 359 } 360 361 sg_cpu->iowait_boost_pending = false; 362 363 /* 364 * sg_cpu->util is already in capacity scale; convert iowait_boost 365 * into the same scale so we can compare. 366 */ 367 return (sg_cpu->iowait_boost * max_cap) >> SCHED_CAPACITY_SHIFT; 368 } 369 370 #ifdef CONFIG_NO_HZ_COMMON 371 static bool sugov_hold_freq(struct sugov_cpu *sg_cpu) 372 { 373 unsigned long idle_calls; 374 bool ret; 375 376 /* 377 * The heuristics in this function is for the fair class. For SCX, the 378 * performance target comes directly from the BPF scheduler. Let's just 379 * follow it. 380 */ 381 if (scx_switched_all()) 382 return false; 383 384 /* if capped by uclamp_max, always update to be in compliance */ 385 if (uclamp_rq_is_capped(cpu_rq(sg_cpu->cpu))) 386 return false; 387 388 /* 389 * Maintain the frequency if the CPU has not been idle recently, as 390 * reduction is likely to be premature. 391 */ 392 idle_calls = tick_nohz_get_idle_calls_cpu(sg_cpu->cpu); 393 ret = idle_calls == sg_cpu->saved_idle_calls; 394 395 sg_cpu->saved_idle_calls = idle_calls; 396 return ret; 397 } 398 #else /* !CONFIG_NO_HZ_COMMON: */ 399 static inline bool sugov_hold_freq(struct sugov_cpu *sg_cpu) { return false; } 400 #endif /* !CONFIG_NO_HZ_COMMON */ 401 402 /* 403 * Make sugov_should_update_freq() ignore the rate limit when DL 404 * has increased the utilization. 405 */ 406 static inline void ignore_dl_rate_limit(struct sugov_cpu *sg_cpu) 407 { 408 if (cpu_bw_dl(cpu_rq(sg_cpu->cpu)) > sg_cpu->bw_min) 409 sg_cpu->sg_policy->need_freq_update = true; 410 } 411 412 static inline bool sugov_update_single_common(struct sugov_cpu *sg_cpu, 413 u64 time, unsigned long max_cap, 414 unsigned int flags) 415 { 416 unsigned long boost; 417 418 sugov_iowait_boost(sg_cpu, time, flags); 419 sg_cpu->last_update = time; 420 421 ignore_dl_rate_limit(sg_cpu); 422 423 if (!sugov_should_update_freq(sg_cpu->sg_policy, time)) 424 return false; 425 426 boost = sugov_iowait_apply(sg_cpu, time, max_cap); 427 sugov_get_util(sg_cpu, boost); 428 429 return true; 430 } 431 432 static void sugov_update_single_freq(struct update_util_data *hook, u64 time, 433 unsigned int flags) 434 { 435 struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util); 436 struct sugov_policy *sg_policy = sg_cpu->sg_policy; 437 unsigned int cached_freq = sg_policy->cached_raw_freq; 438 unsigned long max_cap; 439 unsigned int next_f; 440 441 max_cap = arch_scale_cpu_capacity(sg_cpu->cpu); 442 443 if (!sugov_update_single_common(sg_cpu, time, max_cap, flags)) 444 return; 445 446 next_f = get_next_freq(sg_policy, sg_cpu->util, max_cap); 447 448 if (sugov_hold_freq(sg_cpu) && next_f < sg_policy->next_freq && 449 !sg_policy->need_freq_update) { 450 next_f = sg_policy->next_freq; 451 452 /* Restore cached freq as next_freq has changed */ 453 sg_policy->cached_raw_freq = cached_freq; 454 } 455 456 if (!sugov_update_next_freq(sg_policy, time, next_f)) 457 return; 458 459 /* 460 * This code runs under rq->lock for the target CPU, so it won't run 461 * concurrently on two different CPUs for the same target and it is not 462 * necessary to acquire the lock in the fast switch case. 463 */ 464 if (sg_policy->policy->fast_switch_enabled) { 465 cpufreq_driver_fast_switch(sg_policy->policy, next_f); 466 } else { 467 raw_spin_lock(&sg_policy->update_lock); 468 sugov_deferred_update(sg_policy); 469 raw_spin_unlock(&sg_policy->update_lock); 470 } 471 } 472 473 static void sugov_update_single_perf(struct update_util_data *hook, u64 time, 474 unsigned int flags) 475 { 476 struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util); 477 struct sugov_policy *sg_policy = sg_cpu->sg_policy; 478 unsigned long prev_util = sg_cpu->util; 479 unsigned long max_cap; 480 481 /* 482 * Fall back to the "frequency" path if frequency invariance is not 483 * supported, because the direct mapping between the utilization and 484 * the performance levels depends on the frequency invariance. 485 */ 486 if (!arch_scale_freq_invariant()) { 487 sugov_update_single_freq(hook, time, flags); 488 return; 489 } 490 491 max_cap = arch_scale_cpu_capacity(sg_cpu->cpu); 492 493 if (!sugov_update_single_common(sg_cpu, time, max_cap, flags)) 494 return; 495 496 if (sugov_hold_freq(sg_cpu) && sg_cpu->util < prev_util) 497 sg_cpu->util = prev_util; 498 499 cpufreq_driver_adjust_perf(sg_policy->policy, sg_cpu->bw_min, 500 sg_cpu->util, sg_cpu->bw_max, max_cap); 501 502 sg_policy->need_freq_update = false; 503 sg_policy->last_freq_update_time = time; 504 } 505 506 static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu, u64 time) 507 { 508 struct sugov_policy *sg_policy = sg_cpu->sg_policy; 509 struct cpufreq_policy *policy = sg_policy->policy; 510 unsigned long util = 0, max_cap; 511 unsigned int j; 512 513 max_cap = arch_scale_cpu_capacity(sg_cpu->cpu); 514 515 for_each_cpu(j, policy->cpus) { 516 struct sugov_cpu *j_sg_cpu = &per_cpu(sugov_cpu, j); 517 unsigned long boost; 518 519 boost = sugov_iowait_apply(j_sg_cpu, time, max_cap); 520 sugov_get_util(j_sg_cpu, boost); 521 522 util = max(j_sg_cpu->util, util); 523 } 524 525 return get_next_freq(sg_policy, util, max_cap); 526 } 527 528 static void 529 sugov_update_shared(struct update_util_data *hook, u64 time, unsigned int flags) 530 { 531 struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util); 532 struct sugov_policy *sg_policy = sg_cpu->sg_policy; 533 unsigned int next_f; 534 535 raw_spin_lock(&sg_policy->update_lock); 536 537 sugov_iowait_boost(sg_cpu, time, flags); 538 sg_cpu->last_update = time; 539 540 ignore_dl_rate_limit(sg_cpu); 541 542 if (sugov_should_update_freq(sg_policy, time)) { 543 next_f = sugov_next_freq_shared(sg_cpu, time); 544 545 if (!sugov_update_next_freq(sg_policy, time, next_f)) 546 goto unlock; 547 548 if (sg_policy->policy->fast_switch_enabled) 549 cpufreq_driver_fast_switch(sg_policy->policy, next_f); 550 else 551 sugov_deferred_update(sg_policy); 552 } 553 unlock: 554 raw_spin_unlock(&sg_policy->update_lock); 555 } 556 557 static void sugov_work(struct kthread_work *work) 558 { 559 struct sugov_policy *sg_policy = container_of(work, struct sugov_policy, work); 560 unsigned int freq; 561 unsigned long flags; 562 563 /* 564 * Hold sg_policy->update_lock shortly to handle the case where: 565 * in case sg_policy->next_freq is read here, and then updated by 566 * sugov_deferred_update() just before work_in_progress is set to false 567 * here, we may miss queueing the new update. 568 * 569 * Note: If a work was queued after the update_lock is released, 570 * sugov_work() will just be called again by kthread_work code; and the 571 * request will be proceed before the sugov thread sleeps. 572 */ 573 raw_spin_lock_irqsave(&sg_policy->update_lock, flags); 574 freq = sg_policy->next_freq; 575 sg_policy->work_in_progress = false; 576 raw_spin_unlock_irqrestore(&sg_policy->update_lock, flags); 577 578 mutex_lock(&sg_policy->work_lock); 579 __cpufreq_driver_target(sg_policy->policy, freq, CPUFREQ_RELATION_L); 580 mutex_unlock(&sg_policy->work_lock); 581 } 582 583 static void sugov_irq_work(struct irq_work *irq_work) 584 { 585 struct sugov_policy *sg_policy; 586 587 sg_policy = container_of(irq_work, struct sugov_policy, irq_work); 588 589 kthread_queue_work(&sg_policy->worker, &sg_policy->work); 590 } 591 592 /************************** sysfs interface ************************/ 593 594 static struct sugov_tunables *global_tunables; 595 static DEFINE_MUTEX(global_tunables_lock); 596 597 static inline struct sugov_tunables *to_sugov_tunables(struct gov_attr_set *attr_set) 598 { 599 return container_of(attr_set, struct sugov_tunables, attr_set); 600 } 601 602 static ssize_t rate_limit_us_show(struct gov_attr_set *attr_set, char *buf) 603 { 604 struct sugov_tunables *tunables = to_sugov_tunables(attr_set); 605 606 return sysfs_emit(buf, "%u\n", tunables->rate_limit_us); 607 } 608 609 static ssize_t 610 rate_limit_us_store(struct gov_attr_set *attr_set, const char *buf, size_t count) 611 { 612 struct sugov_tunables *tunables = to_sugov_tunables(attr_set); 613 struct sugov_policy *sg_policy; 614 unsigned int rate_limit_us; 615 616 if (kstrtouint(buf, 10, &rate_limit_us)) 617 return -EINVAL; 618 619 tunables->rate_limit_us = rate_limit_us; 620 621 list_for_each_entry(sg_policy, &attr_set->policy_list, tunables_hook) 622 sugov_update_rate_limit_us(sg_policy); 623 624 return count; 625 } 626 627 static struct governor_attr rate_limit_us = __ATTR_RW(rate_limit_us); 628 629 static struct attribute *sugov_attrs[] = { 630 &rate_limit_us.attr, 631 NULL 632 }; 633 ATTRIBUTE_GROUPS(sugov); 634 635 static void sugov_tunables_free(struct kobject *kobj) 636 { 637 struct gov_attr_set *attr_set = to_gov_attr_set(kobj); 638 639 kfree(to_sugov_tunables(attr_set)); 640 } 641 642 static const struct kobj_type sugov_tunables_ktype = { 643 .default_groups = sugov_groups, 644 .sysfs_ops = &governor_sysfs_ops, 645 .release = &sugov_tunables_free, 646 }; 647 648 /********************** cpufreq governor interface *********************/ 649 650 static struct cpufreq_governor schedutil_gov; 651 652 static struct sugov_policy *sugov_policy_alloc(struct cpufreq_policy *policy) 653 { 654 struct sugov_policy *sg_policy; 655 656 sg_policy = kzalloc_obj(*sg_policy); 657 if (!sg_policy) 658 return NULL; 659 660 sg_policy->policy = policy; 661 raw_spin_lock_init(&sg_policy->update_lock); 662 return sg_policy; 663 } 664 665 static void sugov_policy_free(struct sugov_policy *sg_policy) 666 { 667 kfree(sg_policy); 668 } 669 670 static int sugov_kthread_create(struct sugov_policy *sg_policy) 671 { 672 struct task_struct *thread; 673 struct sched_attr attr = { 674 .size = sizeof(struct sched_attr), 675 .sched_policy = SCHED_DEADLINE, 676 .sched_flags = SCHED_FLAG_SUGOV, 677 .sched_nice = 0, 678 .sched_priority = 0, 679 /* 680 * Fake (unused) bandwidth; workaround to "fix" 681 * priority inheritance. 682 */ 683 .sched_runtime = NSEC_PER_MSEC, 684 .sched_deadline = 10 * NSEC_PER_MSEC, 685 .sched_period = 10 * NSEC_PER_MSEC, 686 }; 687 struct cpufreq_policy *policy = sg_policy->policy; 688 int ret; 689 690 /* kthread only required for slow path */ 691 if (policy->fast_switch_enabled) 692 return 0; 693 694 kthread_init_work(&sg_policy->work, sugov_work); 695 kthread_init_worker(&sg_policy->worker); 696 thread = kthread_create(kthread_worker_fn, &sg_policy->worker, 697 "sugov:%d", 698 cpumask_first(policy->related_cpus)); 699 if (IS_ERR(thread)) { 700 pr_err("failed to create sugov thread: %pe\n", thread); 701 return PTR_ERR(thread); 702 } 703 704 ret = sched_setattr_nocheck(thread, &attr); 705 if (ret) { 706 kthread_stop(thread); 707 pr_warn("%s: failed to set SCHED_DEADLINE\n", __func__); 708 return ret; 709 } 710 711 sg_policy->thread = thread; 712 if (policy->dvfs_possible_from_any_cpu) 713 set_cpus_allowed_ptr(thread, policy->related_cpus); 714 else 715 kthread_bind_mask(thread, policy->related_cpus); 716 717 init_irq_work(&sg_policy->irq_work, sugov_irq_work); 718 mutex_init(&sg_policy->work_lock); 719 720 wake_up_process(thread); 721 722 return 0; 723 } 724 725 static void sugov_kthread_stop(struct sugov_policy *sg_policy) 726 { 727 /* kthread only required for slow path */ 728 if (sg_policy->policy->fast_switch_enabled) 729 return; 730 731 kthread_flush_worker(&sg_policy->worker); 732 kthread_stop(sg_policy->thread); 733 mutex_destroy(&sg_policy->work_lock); 734 } 735 736 static struct sugov_tunables *sugov_tunables_alloc(struct sugov_policy *sg_policy) 737 { 738 struct sugov_tunables *tunables; 739 740 tunables = kzalloc_obj(*tunables); 741 if (tunables) { 742 gov_attr_set_init(&tunables->attr_set, &sg_policy->tunables_hook); 743 if (!have_governor_per_policy()) 744 global_tunables = tunables; 745 } 746 return tunables; 747 } 748 749 static void sugov_clear_global_tunables(void) 750 { 751 if (!have_governor_per_policy()) 752 global_tunables = NULL; 753 } 754 755 static int sugov_init(struct cpufreq_policy *policy) 756 { 757 struct sugov_policy *sg_policy; 758 struct sugov_tunables *tunables; 759 int ret = 0; 760 761 /* State should be equivalent to EXIT */ 762 if (policy->governor_data) 763 return -EBUSY; 764 765 cpufreq_enable_fast_switch(policy); 766 767 sg_policy = sugov_policy_alloc(policy); 768 if (!sg_policy) { 769 ret = -ENOMEM; 770 goto disable_fast_switch; 771 } 772 773 ret = sugov_kthread_create(sg_policy); 774 if (ret) 775 goto free_sg_policy; 776 777 mutex_lock(&global_tunables_lock); 778 779 if (global_tunables) { 780 if (WARN_ON(have_governor_per_policy())) { 781 ret = -EINVAL; 782 goto stop_kthread; 783 } 784 policy->governor_data = sg_policy; 785 sg_policy->tunables = global_tunables; 786 787 gov_attr_set_get(&global_tunables->attr_set, &sg_policy->tunables_hook); 788 goto out; 789 } 790 791 tunables = sugov_tunables_alloc(sg_policy); 792 if (!tunables) { 793 ret = -ENOMEM; 794 goto stop_kthread; 795 } 796 797 tunables->rate_limit_us = cpufreq_policy_transition_delay_us(policy); 798 799 policy->governor_data = sg_policy; 800 sg_policy->tunables = tunables; 801 802 ret = kobject_init_and_add(&tunables->attr_set.kobj, &sugov_tunables_ktype, 803 get_governor_parent_kobj(policy), "%s", 804 schedutil_gov.name); 805 if (ret) 806 goto fail; 807 808 out: 809 /* 810 * Schedutil is the preferred governor for EAS, so rebuild sched domains 811 * on governor changes to make sure the scheduler knows about them. 812 */ 813 em_rebuild_sched_domains(); 814 mutex_unlock(&global_tunables_lock); 815 return 0; 816 817 fail: 818 kobject_put(&tunables->attr_set.kobj); 819 policy->governor_data = NULL; 820 sugov_clear_global_tunables(); 821 822 stop_kthread: 823 sugov_kthread_stop(sg_policy); 824 mutex_unlock(&global_tunables_lock); 825 826 free_sg_policy: 827 sugov_policy_free(sg_policy); 828 829 disable_fast_switch: 830 cpufreq_disable_fast_switch(policy); 831 832 pr_err("initialization failed (error %d)\n", ret); 833 return ret; 834 } 835 836 static void sugov_exit(struct cpufreq_policy *policy) 837 { 838 struct sugov_policy *sg_policy = policy->governor_data; 839 struct sugov_tunables *tunables = sg_policy->tunables; 840 unsigned int count; 841 842 mutex_lock(&global_tunables_lock); 843 844 count = gov_attr_set_put(&tunables->attr_set, &sg_policy->tunables_hook); 845 policy->governor_data = NULL; 846 if (!count) 847 sugov_clear_global_tunables(); 848 849 mutex_unlock(&global_tunables_lock); 850 851 sugov_kthread_stop(sg_policy); 852 sugov_policy_free(sg_policy); 853 cpufreq_disable_fast_switch(policy); 854 855 em_rebuild_sched_domains(); 856 } 857 858 static int sugov_start(struct cpufreq_policy *policy) 859 { 860 struct sugov_policy *sg_policy = policy->governor_data; 861 void (*uu)(struct update_util_data *data, u64 time, unsigned int flags); 862 unsigned int cpu; 863 864 sugov_update_rate_limit_us(sg_policy); 865 sg_policy->last_freq_update_time = 0; 866 sg_policy->next_freq = 0; 867 sg_policy->work_in_progress = false; 868 sg_policy->limits_changed = false; 869 sg_policy->cached_raw_freq = 0; 870 871 sg_policy->need_freq_update = cpufreq_driver_test_flags(CPUFREQ_NEED_UPDATE_LIMITS); 872 873 if (policy_is_shared(policy)) 874 uu = sugov_update_shared; 875 else if (policy->fast_switch_enabled && cpufreq_driver_has_adjust_perf()) 876 uu = sugov_update_single_perf; 877 else 878 uu = sugov_update_single_freq; 879 880 for_each_cpu(cpu, policy->cpus) { 881 struct sugov_cpu *sg_cpu = &per_cpu(sugov_cpu, cpu); 882 883 memset(sg_cpu, 0, sizeof(*sg_cpu)); 884 sg_cpu->cpu = cpu; 885 sg_cpu->sg_policy = sg_policy; 886 } 887 888 /* 889 * Publish the hooks only after all per-CPU data is initialized, so a 890 * shared policy's sugov_update_shared() never reads an uninitialized 891 * sibling sugov_cpu. 892 */ 893 for_each_cpu(cpu, policy->cpus) { 894 struct sugov_cpu *sg_cpu = &per_cpu(sugov_cpu, cpu); 895 896 cpufreq_add_update_util_hook(cpu, &sg_cpu->update_util, uu); 897 } 898 899 return 0; 900 } 901 902 static void sugov_stop(struct cpufreq_policy *policy) 903 { 904 struct sugov_policy *sg_policy = policy->governor_data; 905 unsigned int cpu; 906 907 for_each_cpu(cpu, policy->cpus) 908 cpufreq_remove_update_util_hook(cpu); 909 910 synchronize_rcu(); 911 912 if (!policy->fast_switch_enabled) { 913 irq_work_sync(&sg_policy->irq_work); 914 kthread_cancel_work_sync(&sg_policy->work); 915 } 916 } 917 918 static void sugov_limits(struct cpufreq_policy *policy) 919 { 920 struct sugov_policy *sg_policy = policy->governor_data; 921 922 if (!policy->fast_switch_enabled) { 923 mutex_lock(&sg_policy->work_lock); 924 cpufreq_policy_apply_limits(policy); 925 mutex_unlock(&sg_policy->work_lock); 926 } 927 928 /* 929 * The limits_changed update below must take place before the updates 930 * of policy limits in cpufreq_set_policy() or a policy limits update 931 * might be missed, so use a memory barrier to ensure it. 932 * 933 * This pairs with the memory barrier in sugov_should_update_freq(). 934 */ 935 smp_wmb(); 936 937 WRITE_ONCE(sg_policy->limits_changed, true); 938 } 939 940 static struct cpufreq_governor schedutil_gov = { 941 .name = "schedutil", 942 .owner = THIS_MODULE, 943 .flags = CPUFREQ_GOV_DYNAMIC_SWITCHING, 944 .init = sugov_init, 945 .exit = sugov_exit, 946 .start = sugov_start, 947 .stop = sugov_stop, 948 .limits = sugov_limits, 949 }; 950 951 #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_SCHEDUTIL 952 struct cpufreq_governor *cpufreq_default_governor(void) 953 { 954 return &schedutil_gov; 955 } 956 #endif 957 958 bool sugov_is_governor(struct cpufreq_policy *policy) 959 { 960 return policy->governor == &schedutil_gov; 961 } 962 963 cpufreq_governor_init(schedutil_gov); 964