xref: /linux/kernel/sched/core.c (revision ec6f6731538e0d45872eb279fdeb477bed152d6f)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  *  kernel/sched/core.c
4  *
5  *  Core kernel CPU scheduler code
6  *
7  *  Copyright (C) 1991-2002  Linus Torvalds
8  *  Copyright (C) 1998-2024  Ingo Molnar, Red Hat
9  */
10 #define INSTANTIATE_EXPORTED_MIGRATE_DISABLE
11 #include <linux/sched.h>
12 #include <linux/highmem.h>
13 #include <linux/hrtimer_api.h>
14 #include <linux/ktime_api.h>
15 #include <linux/sched/signal.h>
16 #include <linux/syscalls_api.h>
17 #include <linux/debug_locks.h>
18 #include <linux/prefetch.h>
19 #include <linux/capability.h>
20 #include <linux/pgtable_api.h>
21 #include <linux/wait_bit.h>
22 #include <linux/jiffies.h>
23 #include <linux/spinlock_api.h>
24 #include <linux/cpumask_api.h>
25 #include <linux/lockdep_api.h>
26 #include <linux/hardirq.h>
27 #include <linux/softirq.h>
28 #include <linux/refcount_api.h>
29 #include <linux/topology.h>
30 #include <linux/sched/clock.h>
31 #include <linux/sched/cond_resched.h>
32 #include <linux/sched/cputime.h>
33 #include <linux/sched/debug.h>
34 #include <linux/sched/hotplug.h>
35 #include <linux/sched/init.h>
36 #include <linux/sched/isolation.h>
37 #include <linux/sched/loadavg.h>
38 #include <linux/sched/mm.h>
39 #include <linux/sched/nohz.h>
40 #include <linux/sched/rseq_api.h>
41 #include <linux/sched/rt.h>
42 
43 #include <linux/context_tracking.h>
44 #include <linux/cpuset.h>
45 #include <linux/delayacct.h>
46 #include <linux/init_task.h>
47 #include <linux/interrupt.h>
48 #include <linux/ioprio.h>
49 #include <linux/kallsyms.h>
50 #include <linux/kcov.h>
51 #include <linux/kprobes.h>
52 #include <linux/llist_api.h>
53 #include <linux/mmu_context.h>
54 #include <linux/mmzone.h>
55 #include <linux/mutex_api.h>
56 #include <linux/nmi.h>
57 #include <linux/nospec.h>
58 #include <linux/perf_event_api.h>
59 #include <linux/profile.h>
60 #include <linux/psi.h>
61 #include <linux/rcuwait_api.h>
62 #include <linux/rseq.h>
63 #include <linux/sched/wake_q.h>
64 #include <linux/scs.h>
65 #include <linux/slab.h>
66 #include <linux/syscalls.h>
67 #include <linux/vtime.h>
68 #include <linux/wait_api.h>
69 #include <linux/workqueue_api.h>
70 #include <linux/livepatch_sched.h>
71 
72 #ifdef CONFIG_PREEMPT_DYNAMIC
73 # ifdef CONFIG_GENERIC_IRQ_ENTRY
74 #  include <linux/irq-entry-common.h>
75 # endif
76 #endif
77 
78 #include <uapi/linux/sched/types.h>
79 
80 #include <asm/irq_regs.h>
81 #include <asm/switch_to.h>
82 #include <asm/tlb.h>
83 
84 #define CREATE_TRACE_POINTS
85 #include <linux/sched/rseq_api.h>
86 #include <trace/events/sched.h>
87 #include <trace/events/ipi.h>
88 #undef CREATE_TRACE_POINTS
89 
90 #include "sched.h"
91 #include "stats.h"
92 
93 #include "autogroup.h"
94 #include "pelt.h"
95 #include "smp.h"
96 
97 #include "../workqueue_internal.h"
98 #include "../../io_uring/io-wq.h"
99 #include "../smpboot.h"
100 #include "../locking/mutex.h"
101 
102 EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpu);
103 EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpumask);
104 
105 /*
106  * Export tracepoints that act as a bare tracehook (ie: have no trace event
107  * associated with them) to allow external modules to probe them.
108  */
109 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_cfs_tp);
110 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_rt_tp);
111 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_dl_tp);
112 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_irq_tp);
113 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_se_tp);
114 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_hw_tp);
115 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_cpu_capacity_tp);
116 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_overutilized_tp);
117 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_cfs_tp);
118 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_se_tp);
119 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_update_nr_running_tp);
120 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_compute_energy_tp);
121 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_entry_tp);
122 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_exit_tp);
123 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_set_need_resched_tp);
124 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_dl_throttle_tp);
125 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_dl_replenish_tp);
126 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_dl_update_tp);
127 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_dl_server_start_tp);
128 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_dl_server_stop_tp);
129 
130 DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
131 DEFINE_PER_CPU(struct rnd_state, sched_rnd_state);
132 
133 #ifdef CONFIG_SCHED_PROXY_EXEC
134 DEFINE_STATIC_KEY_TRUE(__sched_proxy_exec);
135 static int __init setup_proxy_exec(char *str)
136 {
137 	bool proxy_enable = true;
138 
139 	if (*str && kstrtobool(str + 1, &proxy_enable)) {
140 		pr_warn("Unable to parse sched_proxy_exec=\n");
141 		return 0;
142 	}
143 
144 	if (proxy_enable) {
145 		pr_info("sched_proxy_exec enabled via boot arg\n");
146 		static_branch_enable(&__sched_proxy_exec);
147 	} else {
148 		pr_info("sched_proxy_exec disabled via boot arg\n");
149 		static_branch_disable(&__sched_proxy_exec);
150 	}
151 	return 1;
152 }
153 #else
154 static int __init setup_proxy_exec(char *str)
155 {
156 	pr_warn("CONFIG_SCHED_PROXY_EXEC=n, so it cannot be enabled or disabled at boot time\n");
157 	return 0;
158 }
159 #endif
160 __setup("sched_proxy_exec", setup_proxy_exec);
161 
162 /*
163  * Debugging: various feature bits
164  *
165  * If SCHED_DEBUG is disabled, each compilation unit has its own copy of
166  * sysctl_sched_features, defined in sched.h, to allow constants propagation
167  * at compile time and compiler optimization based on features default.
168  */
169 #define SCHED_FEAT(name, enabled)	\
170 	(1UL << __SCHED_FEAT_##name) * enabled |
171 __read_mostly unsigned int sysctl_sched_features =
172 #include "features.h"
173 	0;
174 #undef SCHED_FEAT
175 
176 /*
177  * Print a warning if need_resched is set for the given duration (if
178  * LATENCY_WARN is enabled).
179  *
180  * If sysctl_resched_latency_warn_once is set, only one warning will be shown
181  * per boot.
182  */
183 __read_mostly int sysctl_resched_latency_warn_ms = 100;
184 __read_mostly int sysctl_resched_latency_warn_once = 1;
185 
186 /*
187  * Number of tasks to iterate in a single balance run.
188  * Limited because this is done with IRQs disabled.
189  */
190 __read_mostly unsigned int sysctl_sched_nr_migrate = SCHED_NR_MIGRATE_BREAK;
191 
192 __read_mostly int scheduler_running;
193 
194 #ifdef CONFIG_SCHED_CORE
195 
196 DEFINE_STATIC_KEY_FALSE(__sched_core_enabled);
197 
198 /* kernel prio, less is more */
199 static inline int __task_prio(const struct task_struct *p)
200 {
201 	if (p->sched_class == &stop_sched_class) /* trumps deadline */
202 		return -2;
203 
204 	if (p->dl_server)
205 		return -1; /* deadline */
206 
207 	if (rt_or_dl_prio(p->prio))
208 		return p->prio; /* [-1, 99] */
209 
210 	if (p->sched_class == &idle_sched_class)
211 		return MAX_RT_PRIO + NICE_WIDTH; /* 140 */
212 
213 	if (task_on_scx(p))
214 		return MAX_RT_PRIO + MAX_NICE + 1; /* 120, squash ext */
215 
216 	return MAX_RT_PRIO + MAX_NICE; /* 119, squash fair */
217 }
218 
219 /*
220  * l(a,b)
221  * le(a,b) := !l(b,a)
222  * g(a,b)  := l(b,a)
223  * ge(a,b) := !l(a,b)
224  */
225 
226 /* real prio, less is less */
227 static inline bool prio_less(const struct task_struct *a,
228 			     const struct task_struct *b, bool in_fi)
229 {
230 
231 	int pa = __task_prio(a), pb = __task_prio(b);
232 
233 	if (-pa < -pb)
234 		return true;
235 
236 	if (-pb < -pa)
237 		return false;
238 
239 	if (pa == -1) { /* dl_prio() doesn't work because of stop_class above */
240 		const struct sched_dl_entity *a_dl, *b_dl;
241 
242 		a_dl = &a->dl;
243 		/*
244 		 * Since,'a' and 'b' can be CFS tasks served by DL server,
245 		 * __task_prio() can return -1 (for DL) even for those. In that
246 		 * case, get to the dl_server's DL entity.
247 		 */
248 		if (a->dl_server)
249 			a_dl = a->dl_server;
250 
251 		b_dl = &b->dl;
252 		if (b->dl_server)
253 			b_dl = b->dl_server;
254 
255 		return !dl_time_before(a_dl->deadline, b_dl->deadline);
256 	}
257 
258 	if (pa == MAX_RT_PRIO + MAX_NICE)	/* fair */
259 		return cfs_prio_less(a, b, in_fi);
260 
261 #ifdef CONFIG_SCHED_CLASS_EXT
262 	if (pa == MAX_RT_PRIO + MAX_NICE + 1)	/* ext */
263 		return scx_prio_less(a, b, in_fi);
264 #endif
265 
266 	return false;
267 }
268 
269 static inline bool __sched_core_less(const struct task_struct *a,
270 				     const struct task_struct *b)
271 {
272 	if (a->core_cookie < b->core_cookie)
273 		return true;
274 
275 	if (a->core_cookie > b->core_cookie)
276 		return false;
277 
278 	/* flip prio, so high prio is leftmost */
279 	if (prio_less(b, a, !!task_rq(a)->core->core_forceidle_count))
280 		return true;
281 
282 	return false;
283 }
284 
285 #define __node_2_sc(node) rb_entry((node), struct task_struct, core_node)
286 
287 static inline bool rb_sched_core_less(struct rb_node *a, const struct rb_node *b)
288 {
289 	return __sched_core_less(__node_2_sc(a), __node_2_sc(b));
290 }
291 
292 static inline int rb_sched_core_cmp(const void *key, const struct rb_node *node)
293 {
294 	const struct task_struct *p = __node_2_sc(node);
295 	unsigned long cookie = (unsigned long)key;
296 
297 	if (cookie < p->core_cookie)
298 		return -1;
299 
300 	if (cookie > p->core_cookie)
301 		return 1;
302 
303 	return 0;
304 }
305 
306 void sched_core_enqueue(struct rq *rq, struct task_struct *p)
307 {
308 	if (p->se.sched_delayed)
309 		return;
310 
311 	rq->core->core_task_seq++;
312 
313 	if (!p->core_cookie)
314 		return;
315 
316 	rb_add(&p->core_node, &rq->core_tree, rb_sched_core_less);
317 }
318 
319 void sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags)
320 {
321 	if (p->se.sched_delayed)
322 		return;
323 
324 	rq->core->core_task_seq++;
325 
326 	if (sched_core_enqueued(p)) {
327 		rb_erase(&p->core_node, &rq->core_tree);
328 		RB_CLEAR_NODE(&p->core_node);
329 	}
330 
331 	/*
332 	 * Migrating the last task off the cpu, with the cpu in forced idle
333 	 * state. Reschedule to create an accounting edge for forced idle,
334 	 * and re-examine whether the core is still in forced idle state.
335 	 */
336 	if (!(flags & DEQUEUE_SAVE) && rq->nr_running == 1 &&
337 	    rq->core->core_forceidle_count && rq->curr == rq->idle)
338 		resched_curr(rq);
339 }
340 
341 static int sched_task_is_throttled(struct task_struct *p, int cpu)
342 {
343 	if (p->sched_class->task_is_throttled)
344 		return p->sched_class->task_is_throttled(p, cpu);
345 
346 	return 0;
347 }
348 
349 static struct task_struct *sched_core_next(struct task_struct *p, unsigned long cookie)
350 {
351 	struct rb_node *node = &p->core_node;
352 	int cpu = task_cpu(p);
353 
354 	do {
355 		node = rb_next(node);
356 		if (!node)
357 			return NULL;
358 
359 		p = __node_2_sc(node);
360 		if (p->core_cookie != cookie)
361 			return NULL;
362 
363 	} while (sched_task_is_throttled(p, cpu));
364 
365 	return p;
366 }
367 
368 /*
369  * Find left-most (aka, highest priority) and unthrottled task matching @cookie.
370  * If no suitable task is found, NULL will be returned.
371  */
372 static struct task_struct *sched_core_find(struct rq *rq, unsigned long cookie)
373 {
374 	struct task_struct *p;
375 	struct rb_node *node;
376 
377 	node = rb_find_first((void *)cookie, &rq->core_tree, rb_sched_core_cmp);
378 	if (!node)
379 		return NULL;
380 
381 	p = __node_2_sc(node);
382 	if (!sched_task_is_throttled(p, rq->cpu))
383 		return p;
384 
385 	return sched_core_next(p, cookie);
386 }
387 
388 /*
389  * Magic required such that:
390  *
391  *	raw_spin_rq_lock(rq);
392  *	...
393  *	raw_spin_rq_unlock(rq);
394  *
395  * ends up locking and unlocking the _same_ lock, and all CPUs
396  * always agree on what rq has what lock.
397  *
398  * XXX entirely possible to selectively enable cores, don't bother for now.
399  */
400 
401 static DEFINE_MUTEX(sched_core_mutex);
402 static atomic_t sched_core_count;
403 static struct cpumask sched_core_mask;
404 
405 static void sched_core_lock(int cpu, unsigned long *flags)
406 	__context_unsafe(/* acquires multiple */)
407 	__acquires(&runqueues.__lock) /* overapproximation */
408 {
409 	const struct cpumask *smt_mask = cpu_smt_mask(cpu);
410 	int t, i = 0;
411 
412 	local_irq_save(*flags);
413 	for_each_cpu(t, smt_mask)
414 		raw_spin_lock_nested(&cpu_rq(t)->__lock, i++);
415 }
416 
417 static void sched_core_unlock(int cpu, unsigned long *flags)
418 	__context_unsafe(/* releases multiple */)
419 	__releases(&runqueues.__lock) /* overapproximation */
420 {
421 	const struct cpumask *smt_mask = cpu_smt_mask(cpu);
422 	int t;
423 
424 	for_each_cpu(t, smt_mask)
425 		raw_spin_unlock(&cpu_rq(t)->__lock);
426 	local_irq_restore(*flags);
427 }
428 
429 static void __sched_core_flip(bool enabled)
430 {
431 	unsigned long flags;
432 	int cpu, t;
433 
434 	cpus_read_lock();
435 
436 	/*
437 	 * Toggle the online cores, one by one.
438 	 */
439 	cpumask_copy(&sched_core_mask, cpu_online_mask);
440 	for_each_cpu(cpu, &sched_core_mask) {
441 		const struct cpumask *smt_mask = cpu_smt_mask(cpu);
442 
443 		sched_core_lock(cpu, &flags);
444 
445 		/*
446 		 * A core-wide selection may have the shared rq lock temporarily
447 		 * released by a lock-dropping ->pick_task(). Flipping would
448 		 * rebind rq_lockp() under it. Wait it out.
449 		 */
450 		while (cpu_rq(cpu)->core->core_pick_in_flight) {
451 			sched_core_unlock(cpu, &flags);
452 			cpu_relax();
453 			sched_core_lock(cpu, &flags);
454 		}
455 
456 		for_each_cpu(t, smt_mask)
457 			cpu_rq(t)->core_enabled = enabled;
458 
459 		cpu_rq(cpu)->core->core_forceidle_start = 0;
460 
461 		sched_core_unlock(cpu, &flags);
462 
463 		cpumask_andnot(&sched_core_mask, &sched_core_mask, smt_mask);
464 	}
465 
466 	/*
467 	 * Toggle the offline CPUs.
468 	 */
469 	for_each_cpu_andnot(cpu, cpu_possible_mask, cpu_online_mask)
470 		cpu_rq(cpu)->core_enabled = enabled;
471 
472 	cpus_read_unlock();
473 }
474 
475 static void sched_core_assert_empty(void)
476 {
477 	int cpu;
478 
479 	for_each_possible_cpu(cpu)
480 		WARN_ON_ONCE(!RB_EMPTY_ROOT(&cpu_rq(cpu)->core_tree));
481 }
482 
483 static void __sched_core_enable(void)
484 {
485 	static_branch_enable(&__sched_core_enabled);
486 	/*
487 	 * Ensure all previous instances of raw_spin_rq_*lock() have finished
488 	 * and future ones will observe !sched_core_disabled().
489 	 */
490 	synchronize_rcu();
491 	__sched_core_flip(true);
492 	sched_core_assert_empty();
493 }
494 
495 static void __sched_core_disable(void)
496 {
497 	sched_core_assert_empty();
498 	__sched_core_flip(false);
499 	static_branch_disable(&__sched_core_enabled);
500 }
501 
502 void sched_core_get(void)
503 {
504 	if (atomic_inc_not_zero(&sched_core_count))
505 		return;
506 
507 	mutex_lock(&sched_core_mutex);
508 	if (!atomic_read(&sched_core_count))
509 		__sched_core_enable();
510 
511 	smp_mb__before_atomic();
512 	atomic_inc(&sched_core_count);
513 	mutex_unlock(&sched_core_mutex);
514 }
515 
516 static void __sched_core_put(struct work_struct *work)
517 {
518 	if (atomic_dec_and_mutex_lock(&sched_core_count, &sched_core_mutex)) {
519 		__sched_core_disable();
520 		mutex_unlock(&sched_core_mutex);
521 	}
522 }
523 
524 void sched_core_put(void)
525 {
526 	static DECLARE_WORK(_work, __sched_core_put);
527 
528 	/*
529 	 * "There can be only one"
530 	 *
531 	 * Either this is the last one, or we don't actually need to do any
532 	 * 'work'. If it is the last *again*, we rely on
533 	 * WORK_STRUCT_PENDING_BIT.
534 	 */
535 	if (!atomic_add_unless(&sched_core_count, -1, 1))
536 		schedule_work(&_work);
537 }
538 
539 #else /* !CONFIG_SCHED_CORE: */
540 
541 static inline void sched_core_enqueue(struct rq *rq, struct task_struct *p) { }
542 static inline void
543 sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) { }
544 
545 #endif /* !CONFIG_SCHED_CORE */
546 
547 /* need a wrapper since we may need to trace from modules */
548 EXPORT_TRACEPOINT_SYMBOL(sched_set_state_tp);
549 
550 /*
551  * Call via the helper macro trace_set_current_state.
552  * Calls to this function MUST be guarded by a
553  * tracepoint_enabled(sched_set_state_tp)
554  */
555 void __trace_set_current_state(int state_value)
556 {
557 	trace_call__sched_set_state_tp(current, state_value);
558 }
559 EXPORT_SYMBOL(__trace_set_current_state);
560 
561 int task_llc(const struct task_struct *p)
562 {
563 	return per_cpu(sd_llc_id, task_cpu(p));
564 }
565 
566 /*
567  * Serialization rules:
568  *
569  * Lock order:
570  *
571  *   p->pi_lock
572  *     rq->lock
573  *       hrtimer_cpu_base->lock (hrtimer_start() for bandwidth controls)
574  *
575  *  rq1->lock
576  *    rq2->lock  where: rq1 < rq2
577  *
578  * Regular state:
579  *
580  * Normal scheduling state is serialized by rq->lock. __schedule() takes the
581  * local CPU's rq->lock, it optionally removes the task from the runqueue and
582  * always looks at the local rq data structures to find the most eligible task
583  * to run next.
584  *
585  * Task enqueue is also under rq->lock, possibly taken from another CPU.
586  * Wakeups from another LLC domain might use an IPI to transfer the enqueue to
587  * the local CPU to avoid bouncing the runqueue state around [ see
588  * ttwu_queue_wakelist() ]
589  *
590  * Task wakeup, specifically wakeups that involve migration, are horribly
591  * complicated to avoid having to take two rq->locks.
592  *
593  * Special state:
594  *
595  * System-calls and anything external will use task_rq_lock() which acquires
596  * both p->pi_lock and rq->lock. As a consequence the state they change is
597  * stable while holding either lock:
598  *
599  *  - sched_setaffinity()/
600  *    set_cpus_allowed_ptr():	p->cpus_ptr, p->nr_cpus_allowed
601  *  - set_user_nice():		p->se.load, p->*prio
602  *  - __sched_setscheduler():	p->sched_class, p->policy, p->*prio,
603  *				p->se.load, p->rt_priority,
604  *				p->dl.dl_{runtime, deadline, period, flags, bw, density}
605  *  - sched_setnuma():		p->numa_preferred_nid
606  *  - sched_move_task():	p->sched_task_group
607  *  - uclamp_update_active()	p->uclamp*
608  *
609  * p->state <- TASK_*:
610  *
611  *   is changed locklessly using set_current_state(), __set_current_state() or
612  *   set_special_state(), see their respective comments, or by
613  *   try_to_wake_up(). This latter uses p->pi_lock to serialize against
614  *   concurrent self.
615  *
616  * p->on_rq <- { 0, 1 = TASK_ON_RQ_QUEUED, 2 = TASK_ON_RQ_MIGRATING }:
617  *
618  *   is set by activate_task() and cleared by deactivate_task()/block_task(),
619  *   under rq->lock. Non-zero indicates the task is runnable, the special
620  *   ON_RQ_MIGRATING state is used for migration without holding both
621  *   rq->locks. It indicates task_cpu() is not stable, see task_rq_lock().
622  *
623  *   Additionally it is possible to be ->on_rq but still be considered not
624  *   runnable when p->se.sched_delayed is true. These tasks are on the runqueue
625  *   but will be dequeued as soon as they get picked again. See the
626  *   task_is_runnable() helper.
627  *
628  * p->on_cpu <- { 0, 1 }:
629  *
630  *   is set by prepare_task() and cleared by finish_task() such that it will be
631  *   set before p is scheduled-in and cleared after p is scheduled-out, both
632  *   under rq->lock. Non-zero indicates the task is running on its CPU.
633  *
634  *   [ The astute reader will observe that it is possible for two tasks on one
635  *     CPU to have ->on_cpu = 1 at the same time. ]
636  *
637  * p->is_blocked <- { 0, 1 }:
638  *
639  *   is set by try_to_block_task() and cleared by unblock_task() and tracks
640  *   if the task is blocked. Traditionally this would mirror p->on_rq, however
641  *   due things like DELAY_DEQUEUE and PROXY_EXEC, this can diverge.
642  *
643  * task_cpu(p): is changed by set_task_cpu(), the rules are:
644  *
645  *  - Don't call set_task_cpu() on a blocked task:
646  *
647  *    We don't care what CPU we're not running on, this simplifies hotplug,
648  *    the CPU assignment of blocked tasks isn't required to be valid.
649  *
650  *  - for try_to_wake_up(), called under p->pi_lock:
651  *
652  *    This allows try_to_wake_up() to only take one rq->lock, see its comment.
653  *
654  *  - for migration called under rq->lock:
655  *    [ see task_on_rq_migrating() in task_rq_lock() ]
656  *
657  *    o move_queued_task()
658  *    o detach_task()
659  *
660  *  - for migration called under double_rq_lock():
661  *
662  *    o __migrate_swap_task()
663  *    o push_rt_task() / pull_rt_task()
664  *    o push_dl_task() / pull_dl_task()
665  *    o dl_task_offline_migration()
666  *
667  */
668 
669 void raw_spin_rq_lock_nested(struct rq *rq, int subclass)
670 	__context_unsafe()
671 {
672 	raw_spinlock_t *lock;
673 
674 	/* Matches synchronize_rcu() in __sched_core_enable() */
675 	preempt_disable();
676 	if (sched_core_disabled()) {
677 		raw_spin_lock_nested(&rq->__lock, subclass);
678 		/* preempt_count *MUST* be > 1 */
679 		preempt_enable_no_resched();
680 		return;
681 	}
682 
683 	for (;;) {
684 		lock = __rq_lockp(rq);
685 		raw_spin_lock_nested(lock, subclass);
686 		if (likely(lock == __rq_lockp(rq))) {
687 			/* preempt_count *MUST* be > 1 */
688 			preempt_enable_no_resched();
689 			return;
690 		}
691 		raw_spin_unlock(lock);
692 	}
693 }
694 
695 bool raw_spin_rq_trylock(struct rq *rq)
696 	__context_unsafe()
697 {
698 	raw_spinlock_t *lock;
699 	bool ret;
700 
701 	/* Matches synchronize_rcu() in __sched_core_enable() */
702 	preempt_disable();
703 	if (sched_core_disabled()) {
704 		ret = raw_spin_trylock(&rq->__lock);
705 		preempt_enable();
706 		return ret;
707 	}
708 
709 	for (;;) {
710 		lock = __rq_lockp(rq);
711 		ret = raw_spin_trylock(lock);
712 		if (!ret || (likely(lock == __rq_lockp(rq)))) {
713 			preempt_enable();
714 			return ret;
715 		}
716 		raw_spin_unlock(lock);
717 	}
718 }
719 
720 /*
721  * double_rq_lock - safely lock two runqueues
722  */
723 void double_rq_lock(struct rq *rq1, struct rq *rq2)
724 {
725 	lockdep_assert_irqs_disabled();
726 
727 	if (rq_order_less(rq2, rq1))
728 		swap(rq1, rq2);
729 
730 	raw_spin_rq_lock(rq1);
731 	if (__rq_lockp(rq1) != __rq_lockp(rq2))
732 		raw_spin_rq_lock_nested(rq2, SINGLE_DEPTH_NESTING);
733 	else
734 		__acquire_ctx_lock(__rq_lockp(rq2)); /* fake acquire */
735 
736 	double_rq_clock_clear_update(rq1, rq2);
737 }
738 
739 /*
740  * ___task_rq_lock - lock the rq @p resides on.
741  */
742 struct rq *___task_rq_lock(struct task_struct *p, struct rq_flags *rf)
743 {
744 	struct rq *rq;
745 
746 	lockdep_assert_held(&p->pi_lock);
747 
748 	for (;;) {
749 		rq = task_rq(p);
750 		raw_spin_rq_lock(rq);
751 		if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) {
752 			rq_pin_lock(rq, rf);
753 			return rq;
754 		}
755 		raw_spin_rq_unlock(rq);
756 
757 		while (unlikely(task_on_rq_migrating(p)))
758 			cpu_relax();
759 	}
760 }
761 
762 /*
763  * task_rq_lock - lock p->pi_lock and lock the rq @p resides on.
764  */
765 struct rq *_task_rq_lock(struct task_struct *p, struct rq_flags *rf)
766 {
767 	struct rq *rq;
768 
769 	for (;;) {
770 		raw_spin_lock_irqsave(&p->pi_lock, rf->flags);
771 		rq = task_rq(p);
772 		raw_spin_rq_lock(rq);
773 		/*
774 		 *	move_queued_task()		task_rq_lock()
775 		 *
776 		 *	ACQUIRE (rq->lock)
777 		 *	[S] ->on_rq = MIGRATING		[L] rq = task_rq()
778 		 *	WMB (__set_task_cpu())		ACQUIRE (rq->lock);
779 		 *	[S] ->cpu = new_cpu		[L] task_rq()
780 		 *					[L] ->on_rq
781 		 *	RELEASE (rq->lock)
782 		 *
783 		 * If we observe the old CPU in task_rq_lock(), the acquire of
784 		 * the old rq->lock will fully serialize against the stores.
785 		 *
786 		 * If we observe the new CPU in task_rq_lock(), the address
787 		 * dependency headed by '[L] rq = task_rq()' and the acquire
788 		 * will pair with the WMB to ensure we then also see migrating.
789 		 */
790 		if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) {
791 			rq_pin_lock(rq, rf);
792 			return rq;
793 		}
794 		raw_spin_rq_unlock(rq);
795 		raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags);
796 
797 		while (unlikely(task_on_rq_migrating(p)))
798 			cpu_relax();
799 	}
800 }
801 
802 /*
803  * RQ-clock updating methods:
804  */
805 
806 /* Use CONFIG_PARAVIRT as this will avoid more #ifdef in arch code. */
807 #ifdef CONFIG_PARAVIRT
808 DEFINE_STATIC_KEY_FALSE(paravirt_steal_rq_enabled);
809 #endif
810 
811 static void update_rq_clock_task(struct rq *rq, s64 delta)
812 {
813 /*
814  * In theory, the compile should just see 0 here, and optimize out the call
815  * to sched_rt_avg_update. But I don't trust it...
816  */
817 	s64 __maybe_unused steal = 0, irq_delta = 0;
818 
819 #ifdef CONFIG_IRQ_TIME_ACCOUNTING
820 	if (irqtime_enabled()) {
821 		irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time;
822 
823 		/*
824 		 * Since irq_time is only updated on {soft,}irq_exit, we might run into
825 		 * this case when a previous update_rq_clock() happened inside a
826 		 * {soft,}IRQ region.
827 		 *
828 		 * When this happens, we stop ->clock_task and only update the
829 		 * prev_irq_time stamp to account for the part that fit, so that a next
830 		 * update will consume the rest. This ensures ->clock_task is
831 		 * monotonic.
832 		 *
833 		 * It does however cause some slight miss-attribution of {soft,}IRQ
834 		 * time, a more accurate solution would be to update the irq_time using
835 		 * the current rq->clock timestamp, except that would require using
836 		 * atomic ops.
837 		 */
838 		if (irq_delta > delta)
839 			irq_delta = delta;
840 
841 		rq->prev_irq_time += irq_delta;
842 		delta -= irq_delta;
843 		delayacct_irq(rq->curr, irq_delta);
844 	}
845 #endif
846 #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
847 	if (static_branch_unlikely(&paravirt_steal_rq_enabled)) {
848 		u64 prev_steal;
849 
850 		steal = prev_steal = paravirt_steal_clock(cpu_of(rq));
851 		steal -= rq->prev_steal_time_rq;
852 
853 		if (unlikely(steal > delta))
854 			steal = delta;
855 
856 		rq->prev_steal_time_rq = prev_steal;
857 		delta -= steal;
858 	}
859 #endif
860 
861 	rq->clock_task += delta;
862 
863 #ifdef CONFIG_HAVE_SCHED_AVG_IRQ
864 	if ((irq_delta + steal) && sched_feat(NONTASK_CAPACITY))
865 		update_irq_load_avg(rq, irq_delta + steal);
866 #endif
867 	update_rq_clock_pelt(rq, delta);
868 }
869 
870 void update_rq_clock(struct rq *rq)
871 {
872 	s64 delta;
873 	u64 clock;
874 
875 	lockdep_assert_rq_held(rq);
876 
877 	if (rq->clock_update_flags & RQCF_ACT_SKIP)
878 		return;
879 
880 	if (sched_feat(WARN_DOUBLE_CLOCK))
881 		WARN_ON_ONCE(rq->clock_update_flags & RQCF_UPDATED);
882 	rq->clock_update_flags |= RQCF_UPDATED;
883 
884 	clock = sched_clock_cpu(cpu_of(rq));
885 	scx_rq_clock_update(rq, clock);
886 
887 	delta = clock - rq->clock;
888 	if (delta < 0)
889 		return;
890 	rq->clock += delta;
891 
892 	update_rq_clock_task(rq, delta);
893 }
894 
895 #ifdef CONFIG_SCHED_HRTICK
896 /*
897  * Use HR-timers to deliver accurate preemption points.
898  */
899 
900 enum {
901 	HRTICK_SCHED_NONE		= 0,
902 	HRTICK_SCHED_DEFER		= BIT(1),
903 	HRTICK_SCHED_START		= BIT(2),
904 	HRTICK_SCHED_REARM_HRTIMER	= BIT(3)
905 };
906 
907 static void __used hrtick_clear(struct rq *rq)
908 {
909 	if (hrtimer_active(&rq->hrtick_timer))
910 		hrtimer_cancel(&rq->hrtick_timer);
911 }
912 
913 /*
914  * High-resolution timer tick.
915  * Runs from hardirq context with interrupts disabled.
916  */
917 static enum hrtimer_restart hrtick(struct hrtimer *timer)
918 {
919 	struct rq *rq = container_of(timer, struct rq, hrtick_timer);
920 	struct rq_flags rf;
921 
922 	WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());
923 
924 	rq_lock(rq, &rf);
925 	update_rq_clock(rq);
926 	rq->donor->sched_class->task_tick(rq, rq->donor, 1);
927 	rq_unlock(rq, &rf);
928 
929 	return HRTIMER_NORESTART;
930 }
931 
932 static inline bool hrtick_needs_rearm(struct hrtimer *timer, ktime_t expires)
933 {
934 	/*
935 	 * Queued is false when the timer is not started or currently
936 	 * running the callback. In both cases, restart. If queued check
937 	 * whether the expiry time actually changes substantially.
938 	 */
939 	return !hrtimer_is_queued(timer) ||
940 		abs(expires - hrtimer_get_expires(timer)) > 5000;
941 }
942 
943 static void hrtick_cond_restart(struct rq *rq)
944 {
945 	struct hrtimer *timer = &rq->hrtick_timer;
946 	ktime_t time = rq->hrtick_time;
947 
948 	if (hrtick_needs_rearm(timer, time))
949 		hrtimer_start(timer, time, HRTIMER_MODE_ABS_PINNED_HARD);
950 }
951 
952 /*
953  * called from hardirq (IPI) context
954  */
955 static void __hrtick_start(void *arg)
956 {
957 	struct rq *rq = arg;
958 	struct rq_flags rf;
959 
960 	rq_lock(rq, &rf);
961 	hrtick_cond_restart(rq);
962 	rq_unlock(rq, &rf);
963 }
964 
965 /*
966  * Called to set the hrtick timer state.
967  *
968  * called with rq->lock held and IRQs disabled
969  */
970 void hrtick_start(struct rq *rq, u64 delay)
971 {
972 	s64 delta;
973 
974 	/*
975 	 * Don't schedule slices shorter than 10000ns, that just
976 	 * doesn't make sense and can cause timer DoS.
977 	 */
978 	delta = max_t(s64, delay, 10000LL);
979 
980 	/*
981 	 * If this is in the middle of schedule() only note the delay
982 	 * and let hrtick_schedule_exit() deal with it.
983 	 */
984 	if (rq->hrtick_sched) {
985 		rq->hrtick_sched |= HRTICK_SCHED_START;
986 		rq->hrtick_delay = delta;
987 		return;
988 	}
989 
990 	rq->hrtick_time = ktime_add_ns(ktime_get(), delta);
991 	if (!hrtick_needs_rearm(&rq->hrtick_timer, rq->hrtick_time))
992 		return;
993 
994 	if (rq == this_rq())
995 		hrtimer_start(&rq->hrtick_timer, rq->hrtick_time, HRTIMER_MODE_ABS_PINNED_HARD);
996 	else
997 		smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd);
998 }
999 
1000 static inline void hrtick_schedule_enter(struct rq *rq)
1001 {
1002 	rq->hrtick_sched = HRTICK_SCHED_DEFER;
1003 	if (hrtimer_test_and_clear_rearm_deferred())
1004 		rq->hrtick_sched |= HRTICK_SCHED_REARM_HRTIMER;
1005 }
1006 
1007 static inline void hrtick_schedule_exit(struct rq *rq)
1008 {
1009 	if (rq->hrtick_sched & HRTICK_SCHED_START) {
1010 		rq->hrtick_time = ktime_add_ns(ktime_get(), rq->hrtick_delay);
1011 		hrtick_cond_restart(rq);
1012 	} else if (idle_rq(rq)) {
1013 		/*
1014 		 * No need for using hrtimer_is_active(). The timer is CPU local
1015 		 * and interrupts are disabled, so the callback cannot be
1016 		 * running and the queued state is valid.
1017 		 */
1018 		if (hrtimer_is_queued(&rq->hrtick_timer))
1019 			hrtimer_cancel(&rq->hrtick_timer);
1020 	}
1021 
1022 	if (rq->hrtick_sched & HRTICK_SCHED_REARM_HRTIMER)
1023 		__hrtimer_rearm_deferred();
1024 
1025 	rq->hrtick_sched = HRTICK_SCHED_NONE;
1026 }
1027 
1028 static void hrtick_rq_init(struct rq *rq)
1029 {
1030 	INIT_CSD(&rq->hrtick_csd, __hrtick_start, rq);
1031 	rq->hrtick_sched = HRTICK_SCHED_NONE;
1032 	hrtimer_setup(&rq->hrtick_timer, hrtick, CLOCK_MONOTONIC,
1033 		      HRTIMER_MODE_REL_HARD | HRTIMER_MODE_LAZY_REARM);
1034 }
1035 #else /* !CONFIG_SCHED_HRTICK: */
1036 static inline void hrtick_clear(struct rq *rq) { }
1037 static inline void hrtick_rq_init(struct rq *rq) { }
1038 static inline void hrtick_schedule_enter(struct rq *rq) { }
1039 static inline void hrtick_schedule_exit(struct rq *rq) { }
1040 #endif /* !CONFIG_SCHED_HRTICK */
1041 
1042 /*
1043  * try_cmpxchg based fetch_or() macro so it works for different integer types:
1044  */
1045 #define fetch_or(ptr, mask)						\
1046 	({								\
1047 		typeof(ptr) _ptr = (ptr);				\
1048 		typeof(mask) _mask = (mask);				\
1049 		typeof(*_ptr) _val = *_ptr;				\
1050 									\
1051 		do {							\
1052 		} while (!try_cmpxchg(_ptr, &_val, _val | _mask));	\
1053 	_val;								\
1054 })
1055 
1056 #ifdef TIF_POLLING_NRFLAG
1057 /*
1058  * Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG,
1059  * this avoids any races wrt polling state changes and thereby avoids
1060  * spurious IPIs.
1061  */
1062 static inline bool set_nr_and_not_polling(struct thread_info *ti, int tif)
1063 {
1064 	return !(fetch_or(&ti->flags, 1 << tif) & _TIF_POLLING_NRFLAG);
1065 }
1066 
1067 /*
1068  * Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set.
1069  *
1070  * If this returns true, then the idle task promises to call
1071  * sched_ttwu_pending() and reschedule soon.
1072  */
1073 static bool set_nr_if_polling(struct task_struct *p)
1074 {
1075 	struct thread_info *ti = task_thread_info(p);
1076 	typeof(ti->flags) val = READ_ONCE(ti->flags);
1077 
1078 	do {
1079 		if (!(val & _TIF_POLLING_NRFLAG))
1080 			return false;
1081 		if (val & _TIF_NEED_RESCHED)
1082 			return true;
1083 	} while (!try_cmpxchg(&ti->flags, &val, val | _TIF_NEED_RESCHED));
1084 
1085 	return true;
1086 }
1087 
1088 #else
1089 static inline bool set_nr_and_not_polling(struct thread_info *ti, int tif)
1090 {
1091 	set_ti_thread_flag(ti, tif);
1092 	return true;
1093 }
1094 
1095 static inline bool set_nr_if_polling(struct task_struct *p)
1096 {
1097 	return false;
1098 }
1099 #endif
1100 
1101 static bool __wake_q_add(struct wake_q_head *head, struct task_struct *task)
1102 {
1103 	struct wake_q_node *node = &task->wake_q;
1104 
1105 	/*
1106 	 * Atomically grab the task, if ->wake_q is !nil already it means
1107 	 * it's already queued (either by us or someone else) and will get the
1108 	 * wakeup due to that.
1109 	 *
1110 	 * In order to ensure that a pending wakeup will observe our pending
1111 	 * state, even in the failed case, an explicit smp_mb() must be used.
1112 	 */
1113 	smp_mb__before_atomic();
1114 	if (unlikely(cmpxchg_relaxed(&node->next, NULL, WAKE_Q_TAIL)))
1115 		return false;
1116 
1117 	/*
1118 	 * The head is context local, there can be no concurrency.
1119 	 */
1120 	*head->lastp = node;
1121 	head->lastp = &node->next;
1122 	return true;
1123 }
1124 
1125 /**
1126  * wake_q_add() - queue a wakeup for 'later' waking.
1127  * @head: the wake_q_head to add @task to
1128  * @task: the task to queue for 'later' wakeup
1129  *
1130  * Queue a task for later wakeup, most likely by the wake_up_q() call in the
1131  * same context, _HOWEVER_ this is not guaranteed, the wakeup can come
1132  * instantly.
1133  *
1134  * This function must be used as-if it were wake_up_process(); IOW the task
1135  * must be ready to be woken at this location.
1136  */
1137 void wake_q_add(struct wake_q_head *head, struct task_struct *task)
1138 {
1139 	if (__wake_q_add(head, task))
1140 		get_task_struct(task);
1141 }
1142 
1143 /**
1144  * wake_q_add_safe() - safely queue a wakeup for 'later' waking.
1145  * @head: the wake_q_head to add @task to
1146  * @task: the task to queue for 'later' wakeup
1147  *
1148  * Queue a task for later wakeup, most likely by the wake_up_q() call in the
1149  * same context, _HOWEVER_ this is not guaranteed, the wakeup can come
1150  * instantly.
1151  *
1152  * This function must be used as-if it were wake_up_process(); IOW the task
1153  * must be ready to be woken at this location.
1154  *
1155  * This function is essentially a task-safe equivalent to wake_q_add(). Callers
1156  * that already hold reference to @task can call the 'safe' version and trust
1157  * wake_q to do the right thing depending whether or not the @task is already
1158  * queued for wakeup.
1159  */
1160 void wake_q_add_safe(struct wake_q_head *head, struct task_struct *task)
1161 {
1162 	if (!__wake_q_add(head, task))
1163 		put_task_struct(task);
1164 }
1165 
1166 void wake_up_q(struct wake_q_head *head)
1167 {
1168 	struct wake_q_node *node = head->first;
1169 
1170 	while (node != WAKE_Q_TAIL) {
1171 		struct task_struct *task;
1172 
1173 		task = container_of(node, struct task_struct, wake_q);
1174 		node = node->next;
1175 		/* pairs with cmpxchg_relaxed() in __wake_q_add() */
1176 		WRITE_ONCE(task->wake_q.next, NULL);
1177 		/* Task can safely be re-inserted now. */
1178 
1179 		/*
1180 		 * wake_up_process() executes a full barrier, which pairs with
1181 		 * the queueing in wake_q_add() so as not to miss wakeups.
1182 		 */
1183 		wake_up_process(task);
1184 		put_task_struct(task);
1185 	}
1186 }
1187 
1188 /*
1189  * resched_curr - mark rq's current task 'to be rescheduled now'.
1190  *
1191  * On UP this means the setting of the need_resched flag, on SMP it
1192  * might also involve a cross-CPU call to trigger the scheduler on
1193  * the target CPU.
1194  */
1195 static void __resched_curr(struct rq *rq, int tif)
1196 {
1197 	struct task_struct *curr = rq->curr;
1198 	struct thread_info *cti = task_thread_info(curr);
1199 	bool need_ipi;
1200 	int cpu;
1201 
1202 	lockdep_assert_rq_held(rq);
1203 
1204 	/*
1205 	 * Always immediately preempt the idle task; no point in delaying doing
1206 	 * actual work.
1207 	 */
1208 	if (is_idle_task(curr) && tif == TIF_NEED_RESCHED_LAZY)
1209 		tif = TIF_NEED_RESCHED;
1210 
1211 	if (cti->flags & ((1 << tif) | _TIF_NEED_RESCHED))
1212 		return;
1213 
1214 	cpu = cpu_of(rq);
1215 
1216 	if (cpu == smp_processor_id()) {
1217 		set_ti_thread_flag(cti, tif);
1218 		if (tif == TIF_NEED_RESCHED)
1219 			set_preempt_need_resched();
1220 		trace_sched_set_need_resched_tp(curr, cpu, tif);
1221 		return;
1222 	}
1223 
1224 	need_ipi = set_nr_and_not_polling(cti, tif);
1225 	trace_sched_set_need_resched_tp(curr, cpu, tif);
1226 	if (need_ipi) {
1227 		if (tif == TIF_NEED_RESCHED)
1228 			smp_send_reschedule(cpu);
1229 	} else {
1230 		trace_sched_wake_idle_without_ipi(cpu);
1231 	}
1232 }
1233 
1234 /*
1235  * Calls to this function MUST be guarded by a
1236  * tracepoint_enabled(sched_set_need_resched_tp)
1237  */
1238 void __trace_set_need_resched(struct task_struct *curr, int tif)
1239 {
1240 	trace_call__sched_set_need_resched_tp(curr, smp_processor_id(), tif);
1241 }
1242 EXPORT_SYMBOL_GPL(__trace_set_need_resched);
1243 
1244 void resched_curr(struct rq *rq)
1245 {
1246 	__resched_curr(rq, TIF_NEED_RESCHED);
1247 }
1248 
1249 #ifdef CONFIG_PREEMPT_DYNAMIC
1250 static DEFINE_STATIC_KEY_FALSE(sk_dynamic_preempt_lazy);
1251 static __always_inline bool dynamic_preempt_lazy(void)
1252 {
1253 	return static_branch_unlikely(&sk_dynamic_preempt_lazy);
1254 }
1255 #else
1256 static __always_inline bool dynamic_preempt_lazy(void)
1257 {
1258 	return IS_ENABLED(CONFIG_PREEMPT_LAZY);
1259 }
1260 #endif
1261 
1262 static __always_inline int get_lazy_tif_bit(void)
1263 {
1264 	if (dynamic_preempt_lazy())
1265 		return TIF_NEED_RESCHED_LAZY;
1266 
1267 	return TIF_NEED_RESCHED;
1268 }
1269 
1270 void resched_curr_lazy(struct rq *rq)
1271 {
1272 	__resched_curr(rq, get_lazy_tif_bit());
1273 }
1274 
1275 void resched_cpu(int cpu)
1276 {
1277 	struct rq *rq = cpu_rq(cpu);
1278 	unsigned long flags;
1279 
1280 	raw_spin_rq_lock_irqsave(rq, flags);
1281 	if (cpu_online(cpu) || cpu == smp_processor_id())
1282 		resched_curr(rq);
1283 	raw_spin_rq_unlock_irqrestore(rq, flags);
1284 }
1285 
1286 #ifdef CONFIG_NO_HZ_COMMON
1287 /*
1288  * In the semi idle case, use the nearest busy CPU for migrating timers
1289  * from an idle CPU.  This is good for power-savings.
1290  *
1291  * We don't do similar optimization for completely idle system, as
1292  * selecting an idle CPU will add more delays to the timers than intended
1293  * (as that CPU's timer base may not be up to date wrt jiffies etc).
1294  */
1295 int get_nohz_timer_target(void)
1296 {
1297 	int i, cpu = smp_processor_id(), default_cpu = -1;
1298 	struct sched_domain *sd;
1299 	const struct cpumask *hk_mask;
1300 
1301 	if (housekeeping_cpu(cpu, HK_TYPE_KERNEL_NOISE)) {
1302 		if (!idle_cpu(cpu))
1303 			return cpu;
1304 		default_cpu = cpu;
1305 	}
1306 
1307 	hk_mask = housekeeping_cpumask(HK_TYPE_KERNEL_NOISE);
1308 
1309 	guard(rcu)();
1310 
1311 	for_each_domain(cpu, sd) {
1312 		for_each_cpu_and(i, sched_domain_span(sd), hk_mask) {
1313 			if (cpu == i)
1314 				continue;
1315 
1316 			if (!idle_cpu(i))
1317 				return i;
1318 		}
1319 	}
1320 
1321 	if (default_cpu == -1)
1322 		default_cpu = housekeeping_any_cpu(HK_TYPE_KERNEL_NOISE);
1323 
1324 	return default_cpu;
1325 }
1326 
1327 /*
1328  * When add_timer_on() enqueues a timer into the timer wheel of an
1329  * idle CPU then this timer might expire before the next timer event
1330  * which is scheduled to wake up that CPU. In case of a completely
1331  * idle system the next event might even be infinite time into the
1332  * future. wake_up_idle_cpu() ensures that the CPU is woken up and
1333  * leaves the inner idle loop so the newly added timer is taken into
1334  * account when the CPU goes back to idle and evaluates the timer
1335  * wheel for the next timer event.
1336  */
1337 static void wake_up_idle_cpu(int cpu)
1338 {
1339 	struct rq *rq = cpu_rq(cpu);
1340 
1341 	if (cpu == smp_processor_id())
1342 		return;
1343 
1344 	/*
1345 	 * Set TIF_NEED_RESCHED and send an IPI if in the non-polling
1346 	 * part of the idle loop. This forces an exit from the idle loop
1347 	 * and a round trip to schedule(). Now this could be optimized
1348 	 * because a simple new idle loop iteration is enough to
1349 	 * re-evaluate the next tick. Provided some re-ordering of tick
1350 	 * nohz functions that would need to follow TIF_NR_POLLING
1351 	 * clearing:
1352 	 *
1353 	 * - On most architectures, a simple fetch_or on ti::flags with a
1354 	 *   "0" value would be enough to know if an IPI needs to be sent.
1355 	 *
1356 	 * - x86 needs to perform a last need_resched() check between
1357 	 *   monitor and mwait which doesn't take timers into account.
1358 	 *   There a dedicated TIF_TIMER flag would be required to
1359 	 *   fetch_or here and be checked along with TIF_NEED_RESCHED
1360 	 *   before mwait().
1361 	 *
1362 	 * However, remote timer enqueue is not such a frequent event
1363 	 * and testing of the above solutions didn't appear to report
1364 	 * much benefits.
1365 	 */
1366 	if (set_nr_and_not_polling(task_thread_info(rq->idle), TIF_NEED_RESCHED))
1367 		smp_send_reschedule(cpu);
1368 	else
1369 		trace_sched_wake_idle_without_ipi(cpu);
1370 }
1371 
1372 static bool wake_up_full_nohz_cpu(int cpu)
1373 {
1374 	/*
1375 	 * We just need the target to call irq_exit() and re-evaluate
1376 	 * the next tick. The nohz full kick at least implies that.
1377 	 * If needed we can still optimize that later with an
1378 	 * empty IRQ.
1379 	 */
1380 	if (cpu_is_offline(cpu))
1381 		return true;  /* Don't try to wake offline CPUs. */
1382 	if (tick_nohz_full_cpu(cpu)) {
1383 		if (cpu != smp_processor_id() ||
1384 		    tick_nohz_tick_stopped())
1385 			tick_nohz_full_kick_cpu(cpu);
1386 		return true;
1387 	}
1388 
1389 	return false;
1390 }
1391 
1392 /*
1393  * Wake up the specified CPU.  If the CPU is going offline, it is the
1394  * caller's responsibility to deal with the lost wakeup, for example,
1395  * by hooking into the CPU_DEAD notifier like timers and hrtimers do.
1396  */
1397 void wake_up_nohz_cpu(int cpu)
1398 {
1399 	if (!wake_up_full_nohz_cpu(cpu))
1400 		wake_up_idle_cpu(cpu);
1401 }
1402 
1403 static void nohz_csd_func(void *info)
1404 {
1405 	struct rq *rq = info;
1406 	int cpu = cpu_of(rq);
1407 	unsigned int flags;
1408 
1409 	/*
1410 	 * Release the rq::nohz_csd.
1411 	 */
1412 	flags = atomic_fetch_andnot(NOHZ_KICK_MASK | NOHZ_NEWILB_KICK, nohz_flags(cpu));
1413 	WARN_ON(!(flags & NOHZ_KICK_MASK));
1414 
1415 	rq->idle_balance = idle_cpu(cpu);
1416 	if (rq->idle_balance) {
1417 		rq->nohz_idle_balance = flags;
1418 		__raise_softirq_irqoff(SCHED_SOFTIRQ);
1419 	}
1420 }
1421 
1422 #endif /* CONFIG_NO_HZ_COMMON */
1423 
1424 #ifdef CONFIG_NO_HZ_FULL
1425 static inline bool __need_bw_check(struct rq *rq, struct task_struct *p)
1426 {
1427 	if (rq->nr_running != 1)
1428 		return false;
1429 
1430 	if (p->sched_class != &fair_sched_class)
1431 		return false;
1432 
1433 	if (!task_on_rq_queued(p))
1434 		return false;
1435 
1436 	return true;
1437 }
1438 
1439 bool sched_can_stop_tick(struct rq *rq)
1440 {
1441 	int fifo_nr_running;
1442 
1443 	/* Deadline tasks, even if single, need the tick */
1444 	if (rq->dl.dl_nr_running)
1445 		return false;
1446 
1447 	/*
1448 	 * If there are more than one RR tasks, we need the tick to affect the
1449 	 * actual RR behaviour.
1450 	 */
1451 	if (rq->rt.rr_nr_running) {
1452 		if (rq->rt.rr_nr_running == 1)
1453 			return true;
1454 		else
1455 			return false;
1456 	}
1457 
1458 	/*
1459 	 * If there's no RR tasks, but FIFO tasks, we can skip the tick, no
1460 	 * forced preemption between FIFO tasks.
1461 	 */
1462 	fifo_nr_running = rq->rt.rt_nr_running - rq->rt.rr_nr_running;
1463 	if (fifo_nr_running)
1464 		return true;
1465 
1466 	/*
1467 	 * If there are no DL,RR/FIFO tasks, there must only be CFS or SCX tasks
1468 	 * left. For CFS, if there's more than one we need the tick for
1469 	 * involuntary preemption. For SCX, ask.
1470 	 */
1471 	if (scx_enabled() && !scx_can_stop_tick(rq))
1472 		return false;
1473 
1474 	if (rq->cfs.h_nr_queued > 1)
1475 		return false;
1476 
1477 	/*
1478 	 * If there is one task and it has CFS runtime bandwidth constraints
1479 	 * and it's on the cpu now we don't want to stop the tick.
1480 	 * This check prevents clearing the bit if a newly enqueued task here is
1481 	 * dequeued by migrating while the constrained task continues to run.
1482 	 * E.g. going from 2->1 without going through pick_next_task().
1483 	 */
1484 	if (__need_bw_check(rq, rq->curr)) {
1485 		if (cfs_task_bw_constrained(rq->curr))
1486 			return false;
1487 	}
1488 
1489 	return true;
1490 }
1491 #endif /* CONFIG_NO_HZ_FULL */
1492 
1493 #if defined(CONFIG_RT_GROUP_SCHED) || defined(CONFIG_FAIR_GROUP_SCHED)
1494 /*
1495  * Iterate task_group tree rooted at *from, calling @down when first entering a
1496  * node and @up when leaving it for the final time.
1497  *
1498  * Caller must hold rcu_lock or sufficient equivalent.
1499  */
1500 int walk_tg_tree_from(struct task_group *from,
1501 			     tg_visitor down, tg_visitor up, void *data)
1502 {
1503 	struct task_group *parent, *child;
1504 	int ret;
1505 
1506 	parent = from;
1507 
1508 down:
1509 	ret = (*down)(parent, data);
1510 	if (ret)
1511 		goto out;
1512 	list_for_each_entry_rcu(child, &parent->children, siblings) {
1513 		parent = child;
1514 		goto down;
1515 
1516 up:
1517 		continue;
1518 	}
1519 	ret = (*up)(parent, data);
1520 	if (ret || parent == from)
1521 		goto out;
1522 
1523 	child = parent;
1524 	parent = parent->parent;
1525 	if (parent)
1526 		goto up;
1527 out:
1528 	return ret;
1529 }
1530 
1531 int tg_nop(struct task_group *tg, void *data)
1532 {
1533 	return 0;
1534 }
1535 #endif
1536 
1537 void set_load_weight(struct task_struct *p, bool update_load)
1538 {
1539 	int prio = p->static_prio - MAX_RT_PRIO;
1540 	struct load_weight lw;
1541 
1542 	if (task_has_idle_policy(p)) {
1543 		lw.weight = scale_load(WEIGHT_IDLEPRIO);
1544 		lw.inv_weight = WMULT_IDLEPRIO;
1545 	} else {
1546 		lw.weight = scale_load(sched_prio_to_weight[prio]);
1547 		lw.inv_weight = sched_prio_to_wmult[prio];
1548 	}
1549 
1550 	/*
1551 	 * SCHED_OTHER tasks have to update their load when changing their
1552 	 * weight
1553 	 */
1554 	if (update_load && p->sched_class->reweight_task)
1555 		p->sched_class->reweight_task(task_rq(p), p, &lw);
1556 	else
1557 		p->se.load = lw;
1558 }
1559 
1560 #ifdef CONFIG_UCLAMP_TASK
1561 /*
1562  * Serializes updates of utilization clamp values
1563  *
1564  * The (slow-path) user-space triggers utilization clamp value updates which
1565  * can require updates on (fast-path) scheduler's data structures used to
1566  * support enqueue/dequeue operations.
1567  * While the per-CPU rq lock protects fast-path update operations, user-space
1568  * requests are serialized using a mutex to reduce the risk of conflicting
1569  * updates or API abuses.
1570  */
1571 static __maybe_unused DEFINE_MUTEX(uclamp_mutex);
1572 
1573 /* Max allowed minimum utilization */
1574 static unsigned int __maybe_unused sysctl_sched_uclamp_util_min = SCHED_CAPACITY_SCALE;
1575 
1576 /* Max allowed maximum utilization */
1577 static unsigned int __maybe_unused sysctl_sched_uclamp_util_max = SCHED_CAPACITY_SCALE;
1578 
1579 /*
1580  * By default RT tasks run at the maximum performance point/capacity of the
1581  * system. Uclamp enforces this by always setting UCLAMP_MIN of RT tasks to
1582  * SCHED_CAPACITY_SCALE.
1583  *
1584  * This knob allows admins to change the default behavior when uclamp is being
1585  * used. In battery powered devices, particularly, running at the maximum
1586  * capacity and frequency will increase energy consumption and shorten the
1587  * battery life.
1588  *
1589  * This knob only affects RT tasks that their uclamp_se->user_defined == false.
1590  *
1591  * This knob will not override the system default sched_util_clamp_min defined
1592  * above.
1593  */
1594 unsigned int sysctl_sched_uclamp_util_min_rt_default = SCHED_CAPACITY_SCALE;
1595 
1596 /* All clamps are required to be less or equal than these values */
1597 static struct uclamp_se uclamp_default[UCLAMP_CNT];
1598 
1599 /*
1600  * This static key is used to reduce the uclamp overhead in the fast path. It
1601  * primarily disables the call to uclamp_rq_{inc, dec}() in
1602  * enqueue/dequeue_task().
1603  *
1604  * This allows users to continue to enable uclamp in their kernel config with
1605  * minimum uclamp overhead in the fast path.
1606  *
1607  * As soon as userspace modifies any of the uclamp knobs, the static key is
1608  * enabled, since we have an actual users that make use of uclamp
1609  * functionality.
1610  *
1611  * The knobs that would enable this static key are:
1612  *
1613  *   * A task modifying its uclamp value with sched_setattr().
1614  *   * An admin modifying the sysctl_sched_uclamp_{min, max} via procfs.
1615  *   * An admin modifying the cgroup cpu.uclamp.{min, max}
1616  */
1617 DEFINE_STATIC_KEY_FALSE(sched_uclamp_used);
1618 
1619 static inline unsigned int
1620 uclamp_idle_value(struct rq *rq, enum uclamp_id clamp_id,
1621 		  unsigned int clamp_value)
1622 {
1623 	/*
1624 	 * Avoid blocked utilization pushing up the frequency when we go
1625 	 * idle (which drops the max-clamp) by retaining the last known
1626 	 * max-clamp.
1627 	 */
1628 	if (clamp_id == UCLAMP_MAX) {
1629 		rq->uclamp_flags |= UCLAMP_FLAG_IDLE;
1630 		return clamp_value;
1631 	}
1632 
1633 	return uclamp_none(UCLAMP_MIN);
1634 }
1635 
1636 static inline void uclamp_idle_reset(struct rq *rq, enum uclamp_id clamp_id,
1637 				     unsigned int clamp_value)
1638 {
1639 	/* Reset max-clamp retention only on idle exit */
1640 	if (!(rq->uclamp_flags & UCLAMP_FLAG_IDLE))
1641 		return;
1642 
1643 	uclamp_rq_set(rq, clamp_id, clamp_value);
1644 }
1645 
1646 static inline
1647 unsigned int uclamp_rq_max_value(struct rq *rq, enum uclamp_id clamp_id,
1648 				   unsigned int clamp_value)
1649 {
1650 	struct uclamp_bucket *bucket = rq->uclamp[clamp_id].bucket;
1651 	int bucket_id = UCLAMP_BUCKETS - 1;
1652 
1653 	/*
1654 	 * Since both min and max clamps are max aggregated, find the
1655 	 * top most bucket with tasks in.
1656 	 */
1657 	for ( ; bucket_id >= 0; bucket_id--) {
1658 		if (!bucket[bucket_id].tasks)
1659 			continue;
1660 		return bucket[bucket_id].value;
1661 	}
1662 
1663 	/* No tasks -- default clamp values */
1664 	return uclamp_idle_value(rq, clamp_id, clamp_value);
1665 }
1666 
1667 static void __uclamp_update_util_min_rt_default(struct task_struct *p)
1668 {
1669 	unsigned int default_util_min;
1670 	struct uclamp_se *uc_se;
1671 
1672 	lockdep_assert_held(&p->pi_lock);
1673 
1674 	uc_se = &p->uclamp_req[UCLAMP_MIN];
1675 
1676 	/* Only sync if user didn't override the default */
1677 	if (uc_se->user_defined)
1678 		return;
1679 
1680 	default_util_min = sysctl_sched_uclamp_util_min_rt_default;
1681 	uclamp_se_set(uc_se, default_util_min, false);
1682 }
1683 
1684 static void uclamp_update_util_min_rt_default(struct task_struct *p)
1685 {
1686 	if (!rt_task(p))
1687 		return;
1688 
1689 	/* Protect updates to p->uclamp_* */
1690 	guard(task_rq_lock)(p);
1691 	__uclamp_update_util_min_rt_default(p);
1692 }
1693 
1694 static inline struct uclamp_se
1695 uclamp_tg_restrict(struct task_struct *p, enum uclamp_id clamp_id)
1696 {
1697 	/* Copy by value as we could modify it */
1698 	struct uclamp_se uc_req = p->uclamp_req[clamp_id];
1699 #ifdef CONFIG_UCLAMP_TASK_GROUP
1700 	unsigned int tg_min, tg_max, value;
1701 
1702 	/*
1703 	 * Tasks in autogroups or root task group will be
1704 	 * restricted by system defaults.
1705 	 */
1706 	if (task_group_is_autogroup(task_group(p)))
1707 		return uc_req;
1708 	if (task_group(p) == &root_task_group)
1709 		return uc_req;
1710 
1711 	tg_min = task_group(p)->uclamp[UCLAMP_MIN].value;
1712 	tg_max = task_group(p)->uclamp[UCLAMP_MAX].value;
1713 	value = uc_req.value;
1714 	value = clamp(value, tg_min, tg_max);
1715 	uclamp_se_set(&uc_req, value, false);
1716 #endif
1717 
1718 	return uc_req;
1719 }
1720 
1721 /*
1722  * The effective clamp bucket index of a task depends on, by increasing
1723  * priority:
1724  * - the task specific clamp value, when explicitly requested from userspace
1725  * - the task group effective clamp value, for tasks not either in the root
1726  *   group or in an autogroup
1727  * - the system default clamp value, defined by the sysadmin
1728  */
1729 static inline struct uclamp_se
1730 uclamp_eff_get(struct task_struct *p, enum uclamp_id clamp_id)
1731 {
1732 	struct uclamp_se uc_req = uclamp_tg_restrict(p, clamp_id);
1733 	struct uclamp_se uc_max = uclamp_default[clamp_id];
1734 
1735 	/* System default restrictions always apply */
1736 	if (unlikely(uc_req.value > uc_max.value))
1737 		return uc_max;
1738 
1739 	return uc_req;
1740 }
1741 
1742 unsigned long uclamp_eff_value(struct task_struct *p, enum uclamp_id clamp_id)
1743 {
1744 	struct uclamp_se uc_eff;
1745 
1746 	/* Task currently refcounted: use back-annotated (effective) value */
1747 	if (p->uclamp[clamp_id].active)
1748 		return (unsigned long)p->uclamp[clamp_id].value;
1749 
1750 	uc_eff = uclamp_eff_get(p, clamp_id);
1751 
1752 	return (unsigned long)uc_eff.value;
1753 }
1754 
1755 /*
1756  * When a task is enqueued on a rq, the clamp bucket currently defined by the
1757  * task's uclamp::bucket_id is refcounted on that rq. This also immediately
1758  * updates the rq's clamp value if required.
1759  *
1760  * Tasks can have a task-specific value requested from user-space, track
1761  * within each bucket the maximum value for tasks refcounted in it.
1762  * This "local max aggregation" allows to track the exact "requested" value
1763  * for each bucket when all its RUNNABLE tasks require the same clamp.
1764  */
1765 static inline void uclamp_rq_inc_id(struct rq *rq, struct task_struct *p,
1766 				    enum uclamp_id clamp_id)
1767 {
1768 	struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id];
1769 	struct uclamp_se *uc_se = &p->uclamp[clamp_id];
1770 	struct uclamp_bucket *bucket;
1771 
1772 	lockdep_assert_rq_held(rq);
1773 
1774 	/* Update task effective clamp */
1775 	p->uclamp[clamp_id] = uclamp_eff_get(p, clamp_id);
1776 
1777 	bucket = &uc_rq->bucket[uc_se->bucket_id];
1778 	bucket->tasks++;
1779 	uc_se->active = true;
1780 
1781 	uclamp_idle_reset(rq, clamp_id, uc_se->value);
1782 
1783 	/*
1784 	 * Local max aggregation: rq buckets always track the max
1785 	 * "requested" clamp value of its RUNNABLE tasks.
1786 	 */
1787 	if (bucket->tasks == 1 || uc_se->value > bucket->value)
1788 		bucket->value = uc_se->value;
1789 
1790 	if (uc_se->value > uclamp_rq_get(rq, clamp_id))
1791 		uclamp_rq_set(rq, clamp_id, uc_se->value);
1792 }
1793 
1794 /*
1795  * When a task is dequeued from a rq, the clamp bucket refcounted by the task
1796  * is released. If this is the last task reference counting the rq's max
1797  * active clamp value, then the rq's clamp value is updated.
1798  *
1799  * Both refcounted tasks and rq's cached clamp values are expected to be
1800  * always valid. If it's detected they are not, as defensive programming,
1801  * enforce the expected state and warn.
1802  */
1803 static inline void uclamp_rq_dec_id(struct rq *rq, struct task_struct *p,
1804 				    enum uclamp_id clamp_id)
1805 {
1806 	struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id];
1807 	struct uclamp_se *uc_se = &p->uclamp[clamp_id];
1808 	struct uclamp_bucket *bucket;
1809 	unsigned int bkt_clamp;
1810 	unsigned int rq_clamp;
1811 
1812 	lockdep_assert_rq_held(rq);
1813 
1814 	/*
1815 	 * If sched_uclamp_used was enabled after task @p was enqueued,
1816 	 * we could end up with unbalanced call to uclamp_rq_dec_id().
1817 	 *
1818 	 * In this case the uc_se->active flag should be false since no uclamp
1819 	 * accounting was performed at enqueue time and we can just return
1820 	 * here.
1821 	 *
1822 	 * Need to be careful of the following enqueue/dequeue ordering
1823 	 * problem too
1824 	 *
1825 	 *	enqueue(taskA)
1826 	 *	// sched_uclamp_used gets enabled
1827 	 *	enqueue(taskB)
1828 	 *	dequeue(taskA)
1829 	 *	// Must not decrement bucket->tasks here
1830 	 *	dequeue(taskB)
1831 	 *
1832 	 * where we could end up with stale data in uc_se and
1833 	 * bucket[uc_se->bucket_id].
1834 	 *
1835 	 * The following check here eliminates the possibility of such race.
1836 	 */
1837 	if (unlikely(!uc_se->active))
1838 		return;
1839 
1840 	bucket = &uc_rq->bucket[uc_se->bucket_id];
1841 
1842 	WARN_ON_ONCE(!bucket->tasks);
1843 	if (likely(bucket->tasks))
1844 		bucket->tasks--;
1845 
1846 	uc_se->active = false;
1847 
1848 	/*
1849 	 * Keep "local max aggregation" simple and accept to (possibly)
1850 	 * overboost some RUNNABLE tasks in the same bucket.
1851 	 * The rq clamp bucket value is reset to its base value whenever
1852 	 * there are no more RUNNABLE tasks refcounting it.
1853 	 */
1854 	if (likely(bucket->tasks))
1855 		return;
1856 
1857 	rq_clamp = uclamp_rq_get(rq, clamp_id);
1858 	/*
1859 	 * Defensive programming: this should never happen. If it happens,
1860 	 * e.g. due to future modification, warn and fix up the expected value.
1861 	 */
1862 	WARN_ON_ONCE(bucket->value > rq_clamp);
1863 	if (bucket->value >= rq_clamp) {
1864 		bkt_clamp = uclamp_rq_max_value(rq, clamp_id, uc_se->value);
1865 		uclamp_rq_set(rq, clamp_id, bkt_clamp);
1866 	}
1867 }
1868 
1869 static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p, int flags)
1870 {
1871 	enum uclamp_id clamp_id;
1872 
1873 	/*
1874 	 * Avoid any overhead until uclamp is actually used by the userspace.
1875 	 *
1876 	 * The condition is constructed such that a NOP is generated when
1877 	 * sched_uclamp_used is disabled.
1878 	 */
1879 	if (!uclamp_is_used())
1880 		return;
1881 
1882 	if (unlikely(!p->sched_class->uclamp_enabled))
1883 		return;
1884 
1885 	/* Only inc the delayed task which being woken up. */
1886 	if (p->se.sched_delayed && !(flags & ENQUEUE_DELAYED))
1887 		return;
1888 
1889 	for_each_clamp_id(clamp_id)
1890 		uclamp_rq_inc_id(rq, p, clamp_id);
1891 
1892 	/* Reset clamp idle holding when there is one RUNNABLE task */
1893 	if (rq->uclamp_flags & UCLAMP_FLAG_IDLE)
1894 		rq->uclamp_flags &= ~UCLAMP_FLAG_IDLE;
1895 }
1896 
1897 static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p)
1898 {
1899 	enum uclamp_id clamp_id;
1900 
1901 	/*
1902 	 * Avoid any overhead until uclamp is actually used by the userspace.
1903 	 *
1904 	 * The condition is constructed such that a NOP is generated when
1905 	 * sched_uclamp_used is disabled.
1906 	 */
1907 	if (!uclamp_is_used())
1908 		return;
1909 
1910 	if (unlikely(!p->sched_class->uclamp_enabled))
1911 		return;
1912 
1913 	if (p->se.sched_delayed)
1914 		return;
1915 
1916 	for_each_clamp_id(clamp_id)
1917 		uclamp_rq_dec_id(rq, p, clamp_id);
1918 }
1919 
1920 static inline void uclamp_rq_reinc_id(struct rq *rq, struct task_struct *p,
1921 				      enum uclamp_id clamp_id)
1922 {
1923 	if (!p->uclamp[clamp_id].active)
1924 		return;
1925 
1926 	uclamp_rq_dec_id(rq, p, clamp_id);
1927 	uclamp_rq_inc_id(rq, p, clamp_id);
1928 
1929 	/*
1930 	 * Make sure to clear the idle flag if we've transiently reached 0
1931 	 * active tasks on rq.
1932 	 */
1933 	if (clamp_id == UCLAMP_MAX && (rq->uclamp_flags & UCLAMP_FLAG_IDLE))
1934 		rq->uclamp_flags &= ~UCLAMP_FLAG_IDLE;
1935 }
1936 
1937 static inline void
1938 uclamp_update_active(struct task_struct *p)
1939 {
1940 	enum uclamp_id clamp_id;
1941 	struct rq_flags rf;
1942 	struct rq *rq;
1943 
1944 	/*
1945 	 * Lock the task and the rq where the task is (or was) queued.
1946 	 *
1947 	 * We might lock the (previous) rq of a !RUNNABLE task, but that's the
1948 	 * price to pay to safely serialize util_{min,max} updates with
1949 	 * enqueues, dequeues and migration operations.
1950 	 * This is the same locking schema used by __set_cpus_allowed_ptr().
1951 	 */
1952 	rq = task_rq_lock(p, &rf);
1953 
1954 	/*
1955 	 * Setting the clamp bucket is serialized by task_rq_lock().
1956 	 * If the task is not yet RUNNABLE and its task_struct is not
1957 	 * affecting a valid clamp bucket, the next time it's enqueued,
1958 	 * it will already see the updated clamp bucket value.
1959 	 */
1960 	for_each_clamp_id(clamp_id)
1961 		uclamp_rq_reinc_id(rq, p, clamp_id);
1962 
1963 	task_rq_unlock(rq, p, &rf);
1964 }
1965 
1966 #ifdef CONFIG_UCLAMP_TASK_GROUP
1967 static inline void
1968 uclamp_update_active_tasks(struct cgroup_subsys_state *css)
1969 {
1970 	struct css_task_iter it;
1971 	struct task_struct *p;
1972 
1973 	css_task_iter_start(css, 0, &it);
1974 	while ((p = css_task_iter_next(&it)))
1975 		uclamp_update_active(p);
1976 	css_task_iter_end(&it);
1977 }
1978 
1979 static void cpu_util_update_eff(struct cgroup_subsys_state *css);
1980 #endif
1981 
1982 #ifdef CONFIG_SYSCTL
1983 #ifdef CONFIG_UCLAMP_TASK_GROUP
1984 static void uclamp_update_root_tg(void)
1985 {
1986 	struct task_group *tg = &root_task_group;
1987 
1988 	uclamp_se_set(&tg->uclamp_req[UCLAMP_MIN],
1989 		      sysctl_sched_uclamp_util_min, false);
1990 	uclamp_se_set(&tg->uclamp_req[UCLAMP_MAX],
1991 		      sysctl_sched_uclamp_util_max, false);
1992 
1993 	guard(rcu)();
1994 	cpu_util_update_eff(&root_task_group.css);
1995 }
1996 #else
1997 static void uclamp_update_root_tg(void) { }
1998 #endif
1999 
2000 static void uclamp_sync_util_min_rt_default(void)
2001 {
2002 	struct task_struct *g, *p;
2003 
2004 	/*
2005 	 * copy_process()			sysctl_uclamp
2006 	 *					  uclamp_min_rt = X;
2007 	 *   write_lock(&tasklist_lock)		  read_lock(&tasklist_lock)
2008 	 *   // link thread			  smp_mb__after_spinlock()
2009 	 *   write_unlock(&tasklist_lock)	  read_unlock(&tasklist_lock);
2010 	 *   sched_post_fork()			  for_each_process_thread()
2011 	 *     __uclamp_sync_rt()		    __uclamp_sync_rt()
2012 	 *
2013 	 * Ensures that either sched_post_fork() will observe the new
2014 	 * uclamp_min_rt or for_each_process_thread() will observe the new
2015 	 * task.
2016 	 */
2017 	read_lock(&tasklist_lock);
2018 	smp_mb__after_spinlock();
2019 	read_unlock(&tasklist_lock);
2020 
2021 	guard(rcu)();
2022 	for_each_process_thread(g, p)
2023 		uclamp_update_util_min_rt_default(p);
2024 }
2025 
2026 static int sysctl_sched_uclamp_handler(const struct ctl_table *table, int write,
2027 				void *buffer, size_t *lenp, loff_t *ppos)
2028 {
2029 	bool update_root_tg = false;
2030 	int old_min, old_max, old_min_rt;
2031 	int result;
2032 
2033 	guard(mutex)(&uclamp_mutex);
2034 
2035 	old_min = sysctl_sched_uclamp_util_min;
2036 	old_max = sysctl_sched_uclamp_util_max;
2037 	old_min_rt = sysctl_sched_uclamp_util_min_rt_default;
2038 
2039 	result = proc_dointvec(table, write, buffer, lenp, ppos);
2040 	if (result)
2041 		goto undo;
2042 	if (!write)
2043 		return 0;
2044 
2045 	if (sysctl_sched_uclamp_util_min > sysctl_sched_uclamp_util_max ||
2046 	    sysctl_sched_uclamp_util_max > SCHED_CAPACITY_SCALE	||
2047 	    sysctl_sched_uclamp_util_min_rt_default > SCHED_CAPACITY_SCALE) {
2048 
2049 		result = -EINVAL;
2050 		goto undo;
2051 	}
2052 
2053 	if (old_min != sysctl_sched_uclamp_util_min) {
2054 		uclamp_se_set(&uclamp_default[UCLAMP_MIN],
2055 			      sysctl_sched_uclamp_util_min, false);
2056 		update_root_tg = true;
2057 	}
2058 	if (old_max != sysctl_sched_uclamp_util_max) {
2059 		uclamp_se_set(&uclamp_default[UCLAMP_MAX],
2060 			      sysctl_sched_uclamp_util_max, false);
2061 		update_root_tg = true;
2062 	}
2063 
2064 	if (update_root_tg) {
2065 		sched_uclamp_enable();
2066 		uclamp_update_root_tg();
2067 	}
2068 
2069 	if (old_min_rt != sysctl_sched_uclamp_util_min_rt_default) {
2070 		sched_uclamp_enable();
2071 		uclamp_sync_util_min_rt_default();
2072 	}
2073 
2074 	/*
2075 	 * We update all RUNNABLE tasks only when task groups are in use.
2076 	 * Otherwise, keep it simple and do just a lazy update at each next
2077 	 * task enqueue time.
2078 	 */
2079 	return 0;
2080 
2081 undo:
2082 	sysctl_sched_uclamp_util_min = old_min;
2083 	sysctl_sched_uclamp_util_max = old_max;
2084 	sysctl_sched_uclamp_util_min_rt_default = old_min_rt;
2085 	return result;
2086 }
2087 #endif /* CONFIG_SYSCTL */
2088 
2089 static void uclamp_fork(struct task_struct *p)
2090 {
2091 	enum uclamp_id clamp_id;
2092 
2093 	/*
2094 	 * We don't need to hold task_rq_lock() when updating p->uclamp_* here
2095 	 * as the task is still at its early fork stages.
2096 	 */
2097 	for_each_clamp_id(clamp_id)
2098 		p->uclamp[clamp_id].active = false;
2099 
2100 	if (likely(!p->sched_reset_on_fork))
2101 		return;
2102 
2103 	for_each_clamp_id(clamp_id) {
2104 		uclamp_se_set(&p->uclamp_req[clamp_id],
2105 			      uclamp_none(clamp_id), false);
2106 	}
2107 }
2108 
2109 static void uclamp_post_fork(struct task_struct *p)
2110 {
2111 	uclamp_update_util_min_rt_default(p);
2112 }
2113 
2114 static void __init init_uclamp_rq(struct rq *rq)
2115 {
2116 	enum uclamp_id clamp_id;
2117 	struct uclamp_rq *uc_rq = rq->uclamp;
2118 
2119 	for_each_clamp_id(clamp_id) {
2120 		uc_rq[clamp_id] = (struct uclamp_rq) {
2121 			.value = uclamp_none(clamp_id)
2122 		};
2123 	}
2124 
2125 	rq->uclamp_flags = UCLAMP_FLAG_IDLE;
2126 }
2127 
2128 static void __init init_uclamp(void)
2129 {
2130 	struct uclamp_se uc_max = {};
2131 	enum uclamp_id clamp_id;
2132 	int cpu;
2133 
2134 	for_each_possible_cpu(cpu)
2135 		init_uclamp_rq(cpu_rq(cpu));
2136 
2137 	for_each_clamp_id(clamp_id) {
2138 		uclamp_se_set(&init_task.uclamp_req[clamp_id],
2139 			      uclamp_none(clamp_id), false);
2140 	}
2141 
2142 	/* System defaults allow max clamp values for both indexes */
2143 	uclamp_se_set(&uc_max, uclamp_none(UCLAMP_MAX), false);
2144 	for_each_clamp_id(clamp_id) {
2145 		uclamp_default[clamp_id] = uc_max;
2146 #ifdef CONFIG_UCLAMP_TASK_GROUP
2147 		root_task_group.uclamp_req[clamp_id] = uc_max;
2148 		root_task_group.uclamp[clamp_id] = uc_max;
2149 #endif
2150 	}
2151 }
2152 
2153 #else /* !CONFIG_UCLAMP_TASK: */
2154 static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p, int flags) { }
2155 static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p) { }
2156 static inline void uclamp_fork(struct task_struct *p) { }
2157 static inline void uclamp_post_fork(struct task_struct *p) { }
2158 static inline void init_uclamp(void) { }
2159 #endif /* !CONFIG_UCLAMP_TASK */
2160 
2161 bool sched_task_on_rq(struct task_struct *p)
2162 {
2163 	return task_on_rq_queued(p);
2164 }
2165 
2166 unsigned long get_wchan(struct task_struct *p)
2167 {
2168 	unsigned long ip = 0;
2169 	unsigned int state;
2170 
2171 	if (!p || p == current)
2172 		return 0;
2173 
2174 	/* Only get wchan if task is blocked and we can keep it that way. */
2175 	raw_spin_lock_irq(&p->pi_lock);
2176 	state = READ_ONCE(p->__state);
2177 	smp_rmb(); /* see try_to_wake_up() */
2178 	if (state != TASK_RUNNING && state != TASK_WAKING && !p->on_rq)
2179 		ip = __get_wchan(p);
2180 	raw_spin_unlock_irq(&p->pi_lock);
2181 
2182 	return ip;
2183 }
2184 
2185 void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
2186 {
2187 	if (!(flags & ENQUEUE_NOCLOCK))
2188 		update_rq_clock(rq);
2189 
2190 	/*
2191 	 * Can be before ->enqueue_task() because uclamp considers the
2192 	 * ENQUEUE_DELAYED task before its ->sched_delayed gets cleared
2193 	 * in ->enqueue_task().
2194 	 */
2195 	uclamp_rq_inc(rq, p, flags);
2196 
2197 	p->sched_class->enqueue_task(rq, p, flags);
2198 
2199 	psi_enqueue(p, flags);
2200 
2201 	if (!(flags & ENQUEUE_RESTORE))
2202 		sched_info_enqueue(rq, p);
2203 
2204 	if (sched_core_enabled(rq))
2205 		sched_core_enqueue(rq, p);
2206 }
2207 
2208 /*
2209  * Must only return false when DEQUEUE_SLEEP.
2210  */
2211 inline bool dequeue_task(struct rq *rq, struct task_struct *p, int flags)
2212 {
2213 	if (sched_core_enabled(rq))
2214 		sched_core_dequeue(rq, p, flags);
2215 
2216 	if (!(flags & DEQUEUE_NOCLOCK))
2217 		update_rq_clock(rq);
2218 
2219 	if (!(flags & DEQUEUE_SAVE))
2220 		sched_info_dequeue(rq, p);
2221 
2222 	psi_dequeue(p, flags);
2223 
2224 	/*
2225 	 * Must be before ->dequeue_task() because ->dequeue_task() can 'fail'
2226 	 * and mark the task ->sched_delayed.
2227 	 */
2228 	uclamp_rq_dec(rq, p);
2229 	return p->sched_class->dequeue_task(rq, p, flags);
2230 }
2231 
2232 void activate_task(struct rq *rq, struct task_struct *p, int flags)
2233 {
2234 	if (task_on_rq_migrating(p))
2235 		flags |= ENQUEUE_MIGRATED;
2236 
2237 	enqueue_task(rq, p, flags);
2238 
2239 	WRITE_ONCE(p->on_rq, TASK_ON_RQ_QUEUED);
2240 	ASSERT_EXCLUSIVE_WRITER(p->on_rq);
2241 }
2242 
2243 void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
2244 {
2245 	WARN_ON_ONCE(flags & DEQUEUE_SLEEP);
2246 
2247 	WRITE_ONCE(p->on_rq, TASK_ON_RQ_MIGRATING);
2248 	ASSERT_EXCLUSIVE_WRITER(p->on_rq);
2249 
2250 	/*
2251 	 * Code explicitly relies on TASK_ON_RQ_MIGRATING begin set *before*
2252 	 * dequeue_task() and cleared *after* enqueue_task().
2253 	 */
2254 
2255 	dequeue_task(rq, p, flags);
2256 }
2257 
2258 static bool dequeue_block_task(struct rq *rq, struct task_struct *p,
2259 			       unsigned long task_state)
2260 {
2261 	int flags = DEQUEUE_NOCLOCK;
2262 
2263 	p->sched_contributes_to_load =
2264 		(task_state & TASK_UNINTERRUPTIBLE) &&
2265 		!(task_state & TASK_NOLOAD) &&
2266 		!(task_state & TASK_FROZEN);
2267 
2268 	if (unlikely(is_special_task_state(task_state)))
2269 		flags |= DEQUEUE_SPECIAL;
2270 
2271 	/*
2272 	 * __schedule()			ttwu()
2273 	 *   prev_state = prev->state;    if (p->on_rq && ...)
2274 	 *   if (prev_state)		    goto out;
2275 	 *     p->on_rq = 0;		  smp_acquire__after_ctrl_dep();
2276 	 *				  p->state = TASK_WAKING
2277 	 *
2278 	 * Where __schedule() and ttwu() have matching control dependencies.
2279 	 *
2280 	 * Once the caller invokes __block_task(), schedule() must not care about
2281 	 * p->state any more.
2282 	 */
2283 	return dequeue_task(rq, p, DEQUEUE_SLEEP | flags);
2284 }
2285 
2286 static void block_task(struct rq *rq, struct task_struct *p, unsigned long task_state)
2287 {
2288 	if (dequeue_block_task(rq, p, task_state))
2289 		__block_task(rq, p);
2290 }
2291 
2292 /**
2293  * task_curr - is this task currently executing on a CPU?
2294  * @p: the task in question.
2295  *
2296  * Return: 1 if the task is currently executing. 0 otherwise.
2297  */
2298 inline int task_curr(const struct task_struct *p)
2299 {
2300 	return cpu_curr(task_cpu(p)) == p;
2301 }
2302 
2303 void wakeup_preempt(struct rq *rq, struct task_struct *p, int flags)
2304 {
2305 	struct task_struct *donor = rq->donor;
2306 
2307 	if (p->sched_class == rq->next_class) {
2308 		rq->next_class->wakeup_preempt(rq, p, flags);
2309 
2310 	} else if (sched_class_above(p->sched_class, rq->next_class)) {
2311 		rq->next_class->wakeup_preempt(rq, p, flags);
2312 		resched_curr(rq);
2313 		rq->next_class = p->sched_class;
2314 	}
2315 
2316 	/*
2317 	 * A queue event has occurred, and we're going to schedule.  In
2318 	 * this case, we can save a useless back to back clock update.
2319 	 */
2320 	if (task_on_rq_queued(donor) && test_tsk_need_resched(rq->curr))
2321 		rq_clock_skip_update(rq);
2322 }
2323 
2324 static __always_inline
2325 int __task_state_match(struct task_struct *p, unsigned int state)
2326 {
2327 	if (READ_ONCE(p->__state) & state)
2328 		return 1;
2329 
2330 	if (READ_ONCE(p->saved_state) & state)
2331 		return -1;
2332 
2333 	return 0;
2334 }
2335 
2336 static __always_inline
2337 int task_state_match(struct task_struct *p, unsigned int state)
2338 {
2339 	/*
2340 	 * Serialize against current_save_and_set_rtlock_wait_state(),
2341 	 * current_restore_rtlock_saved_state(), and __refrigerator().
2342 	 */
2343 	guard(raw_spinlock_irq)(&p->pi_lock);
2344 	return __task_state_match(p, state);
2345 }
2346 
2347 /*
2348  * wait_task_inactive - wait for a thread to unschedule.
2349  *
2350  * Wait for the thread to block in any of the states set in @match_state.
2351  * If it changes, i.e. @p might have woken up, then return zero.  When we
2352  * succeed in waiting for @p to be off its CPU, we return a positive number
2353  * (its total switch count).  If a second call a short while later returns the
2354  * same number, the caller can be sure that @p has remained unscheduled the
2355  * whole time.
2356  *
2357  * The caller must ensure that the task *will* unschedule sometime soon,
2358  * else this function might spin for a *long* time. This function can't
2359  * be called with interrupts off, or it may introduce deadlock with
2360  * smp_call_function() if an IPI is sent by the same process we are
2361  * waiting to become inactive.
2362  */
2363 unsigned long wait_task_inactive(struct task_struct *p, unsigned int match_state)
2364 {
2365 	int running, queued, match;
2366 	struct rq_flags rf;
2367 	unsigned long ncsw;
2368 	struct rq *rq;
2369 
2370 	for (;;) {
2371 		/*
2372 		 * We do the initial early heuristics without holding
2373 		 * any task-queue locks at all. We'll only try to get
2374 		 * the runqueue lock when things look like they will
2375 		 * work out!
2376 		 */
2377 		rq = task_rq(p);
2378 
2379 		/*
2380 		 * If the task is actively running on another CPU
2381 		 * still, just relax and busy-wait without holding
2382 		 * any locks.
2383 		 *
2384 		 * NOTE! Since we don't hold any locks, it's not
2385 		 * even sure that "rq" stays as the right runqueue!
2386 		 * But we don't care, since "task_on_cpu()" will
2387 		 * return false if the runqueue has changed and p
2388 		 * is actually now running somewhere else!
2389 		 */
2390 		while (task_on_cpu(rq, p)) {
2391 			if (!task_state_match(p, match_state))
2392 				return 0;
2393 			cpu_relax();
2394 		}
2395 
2396 		/*
2397 		 * Ok, time to look more closely! We need the rq
2398 		 * lock now, to be *sure*. If we're wrong, we'll
2399 		 * just go back and repeat.
2400 		 */
2401 		rq = task_rq_lock(p, &rf);
2402 		/*
2403 		 * If task is sched_delayed, force dequeue it, to avoid always
2404 		 * hitting the tick timeout in the queued case
2405 		 */
2406 		if (p->se.sched_delayed)
2407 			dequeue_task(rq, p, DEQUEUE_SLEEP | DEQUEUE_DELAYED);
2408 		trace_sched_wait_task(p);
2409 		running = task_on_cpu(rq, p);
2410 		queued = task_on_rq_queued(p);
2411 		ncsw = 0;
2412 		if ((match = __task_state_match(p, match_state))) {
2413 			/*
2414 			 * When matching on p->saved_state, consider this task
2415 			 * still queued so it will wait.
2416 			 */
2417 			if (match < 0)
2418 				queued = 1;
2419 			ncsw = p->nvcsw | LONG_MIN; /* sets MSB */
2420 		}
2421 		task_rq_unlock(rq, p, &rf);
2422 
2423 		/*
2424 		 * If it changed from the expected state, bail out now.
2425 		 */
2426 		if (unlikely(!ncsw))
2427 			break;
2428 
2429 		/*
2430 		 * Was it really running after all now that we
2431 		 * checked with the proper locks actually held?
2432 		 *
2433 		 * Oops. Go back and try again..
2434 		 */
2435 		if (unlikely(running)) {
2436 			cpu_relax();
2437 			continue;
2438 		}
2439 
2440 		/*
2441 		 * It's not enough that it's not actively running,
2442 		 * it must be off the runqueue _entirely_, and not
2443 		 * preempted!
2444 		 *
2445 		 * So if it was still runnable (but just not actively
2446 		 * running right now), it's preempted, and we should
2447 		 * yield - it could be a while.
2448 		 */
2449 		if (unlikely(queued)) {
2450 			ktime_t to = NSEC_PER_SEC / HZ;
2451 
2452 			set_current_state(TASK_UNINTERRUPTIBLE);
2453 			schedule_hrtimeout(&to, HRTIMER_MODE_REL_HARD);
2454 			continue;
2455 		}
2456 
2457 		/*
2458 		 * Ahh, all good. It wasn't running, and it wasn't
2459 		 * runnable, which means that it will never become
2460 		 * running in the future either. We're all done!
2461 		 */
2462 		break;
2463 	}
2464 
2465 	return ncsw;
2466 }
2467 
2468 static void
2469 do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx);
2470 
2471 static void migrate_disable_switch(struct rq *rq, struct task_struct *p)
2472 {
2473 	struct affinity_context ac = {
2474 		.new_mask  = cpumask_of(rq->cpu),
2475 		.flags     = SCA_MIGRATE_DISABLE,
2476 	};
2477 
2478 	if (likely(!p->migration_disabled))
2479 		return;
2480 
2481 	if (p->cpus_ptr != &p->cpus_mask)
2482 		return;
2483 
2484 	scoped_guard (task_rq_lock, p)
2485 		do_set_cpus_allowed(p, &ac);
2486 }
2487 
2488 void ___migrate_enable(void)
2489 {
2490 	struct task_struct *p = current;
2491 	struct affinity_context ac = {
2492 		.new_mask  = &p->cpus_mask,
2493 		.flags     = SCA_MIGRATE_ENABLE,
2494 	};
2495 
2496 	__set_cpus_allowed_ptr(p, &ac);
2497 }
2498 EXPORT_SYMBOL_GPL(___migrate_enable);
2499 
2500 void migrate_disable(void)
2501 {
2502 	__migrate_disable();
2503 }
2504 EXPORT_SYMBOL_GPL(migrate_disable);
2505 
2506 void migrate_enable(void)
2507 {
2508 	__migrate_enable();
2509 }
2510 EXPORT_SYMBOL_GPL(migrate_enable);
2511 
2512 static inline bool rq_has_pinned_tasks(struct rq *rq)
2513 {
2514 	return rq->nr_pinned;
2515 }
2516 
2517 static inline bool task_can_migrate_to_preferred(struct task_struct *p, int cpu)
2518 {
2519 	/* No need to migrate from a preferred CPU */
2520 	if (cpu_preferred(cpu))
2521 		return false;
2522 
2523 	/* Only FAIR tasks honor preferred CPU state */
2524 	if (unlikely(p->sched_class != &fair_sched_class))
2525 		return false;
2526 
2527 	/* Ignore preferred state if task affinity is changing */
2528 	if (unlikely(!cpumask_test_cpu(task_cpu(p), p->cpus_ptr)))
2529 		return false;
2530 
2531 	return cpumask_intersects_and(p->cpus_ptr, cpu_preferred_mask,
2532 				      task_cpu_possible_mask(p));
2533 }
2534 
2535 /*
2536  * Per-CPU kthreads are allowed to run on !active && online CPUs, see
2537  * __set_cpus_allowed_ptr() and select_fallback_rq().
2538  */
2539 static inline bool is_cpu_allowed(struct task_struct *p, int cpu)
2540 {
2541 	/* When not in the task's cpumask, no point in looking further. */
2542 	if (!task_allowed_on_cpu(p, cpu))
2543 		return false;
2544 
2545 	/* migrate_disabled() must be allowed to finish. */
2546 	if (is_migration_disabled(p))
2547 		return cpu_online(cpu);
2548 
2549 	/* Non kernel threads are not allowed during either online or offline. */
2550 	if (!(p->flags & PF_KTHREAD)) {
2551 		/* Try to use preferred CPU if task's affinity allows */
2552 		if (task_can_migrate_to_preferred(p, cpu))
2553 			return false;
2554 		return cpu_active(cpu);
2555 	}
2556 
2557 	/* KTHREAD_IS_PER_CPU is always allowed. */
2558 	if (kthread_is_per_cpu(p))
2559 		return cpu_online(cpu);
2560 
2561 	/* Regular kernel threads don't get to stay during offline. */
2562 	if (cpu_dying(cpu))
2563 		return false;
2564 
2565 	/* Try to keep unbound kthreads on a preferred CPU if possible. */
2566 	if (task_can_migrate_to_preferred(p, cpu))
2567 		return false;
2568 
2569 	/* Otherwise, they are allowed to run on online CPU. */
2570 	return cpu_online(cpu);
2571 }
2572 
2573 /*
2574  * This is how migration works:
2575  *
2576  * 1) we invoke migration_cpu_stop() on the target CPU using
2577  *    stop_one_cpu().
2578  * 2) stopper starts to run (implicitly forcing the migrated thread
2579  *    off the CPU)
2580  * 3) it checks whether the migrated task is still in the wrong runqueue.
2581  * 4) if it's in the wrong runqueue then the migration thread removes
2582  *    it and puts it into the right queue.
2583  * 5) stopper completes and stop_one_cpu() returns and the migration
2584  *    is done.
2585  */
2586 
2587 /*
2588  * move_queued_task - move a queued task to new rq.
2589  *
2590  * Returns (locked) new rq. Old rq's lock is released.
2591  */
2592 static struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf,
2593 				   struct task_struct *p, int new_cpu)
2594 	__must_hold(__rq_lockp(rq))
2595 {
2596 	lockdep_assert_rq_held(rq);
2597 
2598 	deactivate_task(rq, p, DEQUEUE_NOCLOCK);
2599 	set_task_cpu(p, new_cpu);
2600 	rq_unlock(rq, rf);
2601 
2602 	rq = cpu_rq(new_cpu);
2603 
2604 	rq_lock(rq, rf);
2605 	WARN_ON_ONCE(task_cpu(p) != new_cpu);
2606 	activate_task(rq, p, 0);
2607 	wakeup_preempt(rq, p, 0);
2608 
2609 	return rq;
2610 }
2611 
2612 struct migration_arg {
2613 	struct task_struct		*task;
2614 	int				dest_cpu;
2615 	struct set_affinity_pending	*pending;
2616 };
2617 
2618 /*
2619  * @refs: number of wait_for_completion()
2620  * @stop_pending: is @stop_work in use
2621  */
2622 struct set_affinity_pending {
2623 	refcount_t		refs;
2624 	unsigned int		stop_pending;
2625 	struct completion	done;
2626 	struct cpu_stop_work	stop_work;
2627 	struct migration_arg	arg;
2628 };
2629 
2630 /*
2631  * Move (not current) task off this CPU, onto the destination CPU. We're doing
2632  * this because either it can't run here any more (set_cpus_allowed()
2633  * away from this CPU, or CPU going down), or because we're
2634  * attempting to rebalance this task on exec (sched_exec).
2635  *
2636  * So we race with normal scheduler movements, but that's OK, as long
2637  * as the task is no longer on this CPU.
2638  */
2639 static struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf,
2640 				 struct task_struct *p, int dest_cpu)
2641 	__must_hold(__rq_lockp(rq))
2642 {
2643 	/* Affinity changed (again). */
2644 	if (!is_cpu_allowed(p, dest_cpu))
2645 		return rq;
2646 
2647 	rq = move_queued_task(rq, rf, p, dest_cpu);
2648 
2649 	return rq;
2650 }
2651 
2652 /*
2653  * migration_cpu_stop - this will be executed by a high-prio stopper thread
2654  * and performs thread migration by bumping thread off CPU then
2655  * 'pushing' onto another runqueue.
2656  */
2657 static int migration_cpu_stop(void *data)
2658 {
2659 	struct migration_arg *arg = data;
2660 	struct set_affinity_pending *pending = arg->pending;
2661 	struct task_struct *p = arg->task;
2662 	struct rq *rq = this_rq();
2663 	bool complete = false;
2664 	struct rq_flags rf;
2665 
2666 	/*
2667 	 * The original target CPU might have gone down and we might
2668 	 * be on another CPU but it doesn't matter.
2669 	 */
2670 	local_irq_save(rf.flags);
2671 	/*
2672 	 * We need to explicitly wake pending tasks before running
2673 	 * __migrate_task() such that we will not miss enforcing cpus_ptr
2674 	 * during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test.
2675 	 */
2676 	flush_smp_call_function_queue();
2677 
2678 	/*
2679 	 * We may change the underlying rq, but the locks held will
2680 	 * appropriately be "transferred" when switching.
2681 	 */
2682 	context_unsafe_alias(rq);
2683 
2684 	raw_spin_lock(&p->pi_lock);
2685 	rq_lock(rq, &rf);
2686 
2687 	/*
2688 	 * If we were passed a pending, then ->stop_pending was set, thus
2689 	 * p->migration_pending must have remained stable.
2690 	 */
2691 	WARN_ON_ONCE(pending && pending != p->migration_pending);
2692 
2693 	/*
2694 	 * If task_rq(p) != rq, it cannot be migrated here, because we're
2695 	 * holding rq->lock, if p->on_rq == 0 it cannot get enqueued because
2696 	 * we're holding p->pi_lock.
2697 	 */
2698 	if (task_rq(p) == rq) {
2699 		if (is_migration_disabled(p))
2700 			goto out;
2701 
2702 		if (pending) {
2703 			p->migration_pending = NULL;
2704 			complete = true;
2705 
2706 			if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask))
2707 				goto out;
2708 		}
2709 
2710 		if (task_on_rq_queued(p)) {
2711 			update_rq_clock(rq);
2712 			rq = __migrate_task(rq, &rf, p, arg->dest_cpu);
2713 		} else {
2714 			p->wake_cpu = arg->dest_cpu;
2715 		}
2716 
2717 		/*
2718 		 * XXX __migrate_task() can fail, at which point we might end
2719 		 * up running on a dodgy CPU, AFAICT this can only happen
2720 		 * during CPU hotplug, at which point we'll get pushed out
2721 		 * anyway, so it's probably not a big deal.
2722 		 */
2723 
2724 	} else if (pending) {
2725 		/*
2726 		 * This happens when we get migrated between migrate_enable()'s
2727 		 * preempt_enable() and scheduling the stopper task. At that
2728 		 * point we're a regular task again and not current anymore.
2729 		 *
2730 		 * A !PREEMPT kernel has a giant hole here, which makes it far
2731 		 * more likely.
2732 		 */
2733 
2734 		/*
2735 		 * The task moved before the stopper got to run. We're holding
2736 		 * ->pi_lock, so the allowed mask is stable - if it got
2737 		 * somewhere allowed, we're done.
2738 		 */
2739 		if (cpumask_test_cpu(task_cpu(p), p->cpus_ptr)) {
2740 			p->migration_pending = NULL;
2741 			complete = true;
2742 			goto out;
2743 		}
2744 
2745 		/*
2746 		 * When migrate_enable() hits a rq mis-match we can't reliably
2747 		 * determine is_migration_disabled() and so have to chase after
2748 		 * it.
2749 		 */
2750 		WARN_ON_ONCE(!pending->stop_pending);
2751 		preempt_disable();
2752 		rq_unlock(rq, &rf);
2753 		raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags);
2754 		stop_one_cpu_nowait(task_cpu(p), migration_cpu_stop,
2755 				    &pending->arg, &pending->stop_work);
2756 		preempt_enable();
2757 		return 0;
2758 	}
2759 out:
2760 	if (pending)
2761 		pending->stop_pending = false;
2762 	rq_unlock(rq, &rf);
2763 	raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags);
2764 
2765 	if (complete)
2766 		complete_all(&pending->done);
2767 
2768 	return 0;
2769 }
2770 
2771 int push_cpu_stop(void *arg)
2772 {
2773 	struct rq *lowest_rq = NULL, *rq = this_rq();
2774 	struct task_struct *p = arg;
2775 
2776 	raw_spin_lock_irq(&p->pi_lock);
2777 	raw_spin_rq_lock(rq);
2778 
2779 	if (task_rq(p) != rq)
2780 		goto out_unlock;
2781 
2782 	if (is_migration_disabled(p)) {
2783 		p->migration_flags |= MDF_PUSH;
2784 		goto out_unlock;
2785 	}
2786 
2787 	p->migration_flags &= ~MDF_PUSH;
2788 
2789 	if (p->sched_class->find_lock_rq)
2790 		lowest_rq = p->sched_class->find_lock_rq(p, rq);
2791 
2792 	if (!lowest_rq)
2793 		goto out_unlock;
2794 
2795 	lockdep_assert_rq_held(lowest_rq);
2796 
2797 	// XXX validate p is still the highest prio task
2798 	if (task_rq(p) == rq) {
2799 		move_queued_task_locked(rq, lowest_rq, p);
2800 		resched_curr(lowest_rq);
2801 	}
2802 
2803 	double_unlock_balance(rq, lowest_rq);
2804 
2805 out_unlock:
2806 	rq->push_busy = false;
2807 	raw_spin_rq_unlock(rq);
2808 	raw_spin_unlock_irq(&p->pi_lock);
2809 
2810 	put_task_struct(p);
2811 	return 0;
2812 }
2813 
2814 static inline void mm_update_cpus_allowed(struct mm_struct *mm, const cpumask_t *affmask);
2815 
2816 /*
2817  * sched_class::set_cpus_allowed must do the below, but is not required to
2818  * actually call this function.
2819  */
2820 void set_cpus_allowed_common(struct task_struct *p, struct affinity_context *ctx)
2821 {
2822 	if (ctx->flags & (SCA_MIGRATE_ENABLE | SCA_MIGRATE_DISABLE)) {
2823 		p->cpus_ptr = ctx->new_mask;
2824 		return;
2825 	}
2826 
2827 	cpumask_copy(&p->cpus_mask, ctx->new_mask);
2828 	p->nr_cpus_allowed = cpumask_weight(ctx->new_mask);
2829 	mm_update_cpus_allowed(p->mm, ctx->new_mask);
2830 
2831 	/*
2832 	 * Swap in a new user_cpus_ptr if SCA_USER flag set
2833 	 */
2834 	if (ctx->flags & SCA_USER)
2835 		swap(p->user_cpus_ptr, ctx->user_mask);
2836 }
2837 
2838 static void
2839 do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx)
2840 {
2841 	scoped_guard (sched_change, p, DEQUEUE_SAVE)
2842 		p->sched_class->set_cpus_allowed(p, ctx);
2843 }
2844 
2845 /*
2846  * Used for kthread_bind() and select_fallback_rq(), in both cases the user
2847  * affinity (if any) should be destroyed too.
2848  */
2849 void set_cpus_allowed_force(struct task_struct *p, const struct cpumask *new_mask)
2850 {
2851 	struct affinity_context ac = {
2852 		.new_mask  = new_mask,
2853 		.user_mask = NULL,
2854 		.flags     = SCA_USER,	/* clear the user requested mask */
2855 	};
2856 	union cpumask_rcuhead {
2857 		cpumask_t cpumask;
2858 		struct rcu_head rcu;
2859 	};
2860 
2861 	scoped_guard (__task_rq_lock, p)
2862 		do_set_cpus_allowed(p, &ac);
2863 
2864 	/*
2865 	 * Because this is called with p->pi_lock held, it is not possible
2866 	 * to use kfree() here (when PREEMPT_RT=y), therefore punt to using
2867 	 * kfree_rcu().
2868 	 */
2869 	kfree_rcu((union cpumask_rcuhead *)ac.user_mask, rcu);
2870 }
2871 
2872 int dup_user_cpus_ptr(struct task_struct *dst, struct task_struct *src,
2873 		      int node)
2874 {
2875 	cpumask_t *user_mask;
2876 	unsigned long flags;
2877 
2878 	/*
2879 	 * Always clear dst->user_cpus_ptr first as their user_cpus_ptr's
2880 	 * may differ by now due to racing.
2881 	 */
2882 	dst->user_cpus_ptr = NULL;
2883 
2884 	/*
2885 	 * This check is racy and losing the race is a valid situation.
2886 	 * It is not worth the extra overhead of taking the pi_lock on
2887 	 * every fork/clone.
2888 	 */
2889 	if (data_race(!src->user_cpus_ptr))
2890 		return 0;
2891 
2892 	user_mask = alloc_user_cpus_ptr(node);
2893 	if (!user_mask)
2894 		return -ENOMEM;
2895 
2896 	/*
2897 	 * Use pi_lock to protect content of user_cpus_ptr
2898 	 *
2899 	 * Though unlikely, user_cpus_ptr can be reset to NULL by a concurrent
2900 	 * set_cpus_allowed_force().
2901 	 */
2902 	raw_spin_lock_irqsave(&src->pi_lock, flags);
2903 	if (src->user_cpus_ptr) {
2904 		swap(dst->user_cpus_ptr, user_mask);
2905 		cpumask_copy(dst->user_cpus_ptr, src->user_cpus_ptr);
2906 	}
2907 	raw_spin_unlock_irqrestore(&src->pi_lock, flags);
2908 
2909 	if (unlikely(user_mask))
2910 		kfree(user_mask);
2911 
2912 	return 0;
2913 }
2914 
2915 static inline struct cpumask *clear_user_cpus_ptr(struct task_struct *p)
2916 {
2917 	struct cpumask *user_mask = NULL;
2918 
2919 	swap(p->user_cpus_ptr, user_mask);
2920 
2921 	return user_mask;
2922 }
2923 
2924 void release_user_cpus_ptr(struct task_struct *p)
2925 {
2926 	kfree(clear_user_cpus_ptr(p));
2927 }
2928 
2929 /*
2930  * This function is wildly self concurrent; here be dragons.
2931  *
2932  *
2933  * When given a valid mask, __set_cpus_allowed_ptr() must block until the
2934  * designated task is enqueued on an allowed CPU. If that task is currently
2935  * running, we have to kick it out using the CPU stopper.
2936  *
2937  * Migrate-Disable comes along and tramples all over our nice sandcastle.
2938  * Consider:
2939  *
2940  *     Initial conditions: P0->cpus_mask = [0, 1]
2941  *
2942  *     P0@CPU0                  P1
2943  *
2944  *     migrate_disable();
2945  *     <preempted>
2946  *                              set_cpus_allowed_ptr(P0, [1]);
2947  *
2948  * P1 *cannot* return from this set_cpus_allowed_ptr() call until P0 executes
2949  * its outermost migrate_enable() (i.e. it exits its Migrate-Disable region).
2950  * This means we need the following scheme:
2951  *
2952  *     P0@CPU0                  P1
2953  *
2954  *     migrate_disable();
2955  *     <preempted>
2956  *                              set_cpus_allowed_ptr(P0, [1]);
2957  *                                <blocks>
2958  *     <resumes>
2959  *     migrate_enable();
2960  *       __set_cpus_allowed_ptr();
2961  *       <wakes local stopper>
2962  *                         `--> <woken on migration completion>
2963  *
2964  * Now the fun stuff: there may be several P1-like tasks, i.e. multiple
2965  * concurrent set_cpus_allowed_ptr(P0, [*]) calls. CPU affinity changes of any
2966  * task p are serialized by p->pi_lock, which we can leverage: the one that
2967  * should come into effect at the end of the Migrate-Disable region is the last
2968  * one. This means we only need to track a single cpumask (i.e. p->cpus_mask),
2969  * but we still need to properly signal those waiting tasks at the appropriate
2970  * moment.
2971  *
2972  * This is implemented using struct set_affinity_pending. The first
2973  * __set_cpus_allowed_ptr() caller within a given Migrate-Disable region will
2974  * setup an instance of that struct and install it on the targeted task_struct.
2975  * Any and all further callers will reuse that instance. Those then wait for
2976  * a completion signaled at the tail of the CPU stopper callback (1), triggered
2977  * on the end of the Migrate-Disable region (i.e. outermost migrate_enable()).
2978  *
2979  *
2980  * (1) In the cases covered above. There is one more where the completion is
2981  * signaled within affine_move_task() itself: when a subsequent affinity request
2982  * occurs after the stopper bailed out due to the targeted task still being
2983  * Migrate-Disable. Consider:
2984  *
2985  *     Initial conditions: P0->cpus_mask = [0, 1]
2986  *
2987  *     CPU0		  P1				P2
2988  *     <P0>
2989  *       migrate_disable();
2990  *       <preempted>
2991  *                        set_cpus_allowed_ptr(P0, [1]);
2992  *                          <blocks>
2993  *     <migration/0>
2994  *       migration_cpu_stop()
2995  *         is_migration_disabled()
2996  *           <bails>
2997  *                                                       set_cpus_allowed_ptr(P0, [0, 1]);
2998  *                                                         <signal completion>
2999  *                          <awakes>
3000  *
3001  * Note that the above is safe vs a concurrent migrate_enable(), as any
3002  * pending affinity completion is preceded by an uninstallation of
3003  * p->migration_pending done with p->pi_lock held.
3004  */
3005 static int affine_move_task(struct rq *rq, struct task_struct *p, struct rq_flags *rf,
3006 			    int dest_cpu, unsigned int flags)
3007 	__releases(__rq_lockp(rq), &p->pi_lock)
3008 {
3009 	struct set_affinity_pending my_pending = { }, *pending = NULL;
3010 	bool stop_pending, complete = false;
3011 
3012 	/*
3013 	 * Can the task run on the task's current CPU? If so, we're done
3014 	 *
3015 	 * We are also done if the task is the current donor, boosting a lock-
3016 	 * holding proxy, (and potentially has been migrated outside its
3017 	 * current or previous affinity mask)
3018 	 */
3019 	if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask) ||
3020 	    (task_current_donor(rq, p) && !task_current(rq, p))) {
3021 		struct task_struct *push_task = NULL;
3022 
3023 		if ((flags & SCA_MIGRATE_ENABLE) &&
3024 		    (p->migration_flags & MDF_PUSH) && !rq->push_busy) {
3025 			rq->push_busy = true;
3026 			push_task = get_task_struct(p);
3027 		}
3028 
3029 		/*
3030 		 * If there are pending waiters, but no pending stop_work,
3031 		 * then complete now.
3032 		 */
3033 		pending = p->migration_pending;
3034 		if (pending && !pending->stop_pending) {
3035 			p->migration_pending = NULL;
3036 			complete = true;
3037 		}
3038 
3039 		preempt_disable();
3040 		task_rq_unlock(rq, p, rf);
3041 		if (push_task) {
3042 			stop_one_cpu_nowait(rq->cpu, push_cpu_stop,
3043 					    p, &rq->push_work);
3044 		}
3045 		preempt_enable();
3046 
3047 		if (complete)
3048 			complete_all(&pending->done);
3049 
3050 		return 0;
3051 	}
3052 
3053 	if (!(flags & SCA_MIGRATE_ENABLE)) {
3054 		/* serialized by p->pi_lock */
3055 		if (!p->migration_pending) {
3056 			/* Install the request */
3057 			refcount_set(&my_pending.refs, 1);
3058 			init_completion(&my_pending.done);
3059 			my_pending.arg = (struct migration_arg) {
3060 				.task = p,
3061 				.dest_cpu = dest_cpu,
3062 				.pending = &my_pending,
3063 			};
3064 
3065 			p->migration_pending = &my_pending;
3066 		} else {
3067 			pending = p->migration_pending;
3068 			refcount_inc(&pending->refs);
3069 			/*
3070 			 * Affinity has changed, but we've already installed a
3071 			 * pending. migration_cpu_stop() *must* see this, else
3072 			 * we risk a completion of the pending despite having a
3073 			 * task on a disallowed CPU.
3074 			 *
3075 			 * Serialized by p->pi_lock, so this is safe.
3076 			 */
3077 			pending->arg.dest_cpu = dest_cpu;
3078 		}
3079 	}
3080 	pending = p->migration_pending;
3081 	/*
3082 	 * - !MIGRATE_ENABLE:
3083 	 *   we'll have installed a pending if there wasn't one already.
3084 	 *
3085 	 * - MIGRATE_ENABLE:
3086 	 *   we're here because the current CPU isn't matching anymore,
3087 	 *   the only way that can happen is because of a concurrent
3088 	 *   set_cpus_allowed_ptr() call, which should then still be
3089 	 *   pending completion.
3090 	 *
3091 	 * Either way, we really should have a @pending here.
3092 	 */
3093 	if (WARN_ON_ONCE(!pending)) {
3094 		task_rq_unlock(rq, p, rf);
3095 		return -EINVAL;
3096 	}
3097 
3098 	if (task_on_cpu(rq, p) || READ_ONCE(p->__state) == TASK_WAKING) {
3099 		/*
3100 		 * MIGRATE_ENABLE gets here because 'p == current', but for
3101 		 * anything else we cannot do is_migration_disabled(), punt
3102 		 * and have the stopper function handle it all race-free.
3103 		 */
3104 		stop_pending = pending->stop_pending;
3105 		if (!stop_pending)
3106 			pending->stop_pending = true;
3107 
3108 		if (flags & SCA_MIGRATE_ENABLE)
3109 			p->migration_flags &= ~MDF_PUSH;
3110 
3111 		preempt_disable();
3112 		task_rq_unlock(rq, p, rf);
3113 		if (!stop_pending) {
3114 			stop_one_cpu_nowait(cpu_of(rq), migration_cpu_stop,
3115 					    &pending->arg, &pending->stop_work);
3116 		}
3117 		preempt_enable();
3118 
3119 		if (flags & SCA_MIGRATE_ENABLE)
3120 			return 0;
3121 	} else {
3122 
3123 		if (!is_migration_disabled(p)) {
3124 			if (task_on_rq_queued(p))
3125 				rq = move_queued_task(rq, rf, p, dest_cpu);
3126 
3127 			if (!pending->stop_pending) {
3128 				p->migration_pending = NULL;
3129 				complete = true;
3130 			}
3131 		}
3132 		task_rq_unlock(rq, p, rf);
3133 
3134 		if (complete)
3135 			complete_all(&pending->done);
3136 	}
3137 
3138 	wait_for_completion(&pending->done);
3139 
3140 	if (refcount_dec_and_test(&pending->refs))
3141 		wake_up_var(&pending->refs); /* No UaF, just an address */
3142 
3143 	/*
3144 	 * Block the original owner of &pending until all subsequent callers
3145 	 * have seen the completion and decremented the refcount
3146 	 */
3147 	wait_var_event(&my_pending.refs, !refcount_read(&my_pending.refs));
3148 
3149 	/* ARGH */
3150 	WARN_ON_ONCE(my_pending.stop_pending);
3151 
3152 	return 0;
3153 }
3154 
3155 /*
3156  * Called with both p->pi_lock and rq->lock held; drops both before returning.
3157  */
3158 static int __set_cpus_allowed_ptr_locked(struct task_struct *p,
3159 					 struct affinity_context *ctx,
3160 					 struct rq *rq,
3161 					 struct rq_flags *rf)
3162 	__releases(__rq_lockp(rq), &p->pi_lock)
3163 {
3164 	const struct cpumask *cpu_allowed_mask = task_cpu_possible_mask(p);
3165 	const struct cpumask *cpu_valid_mask = cpu_active_mask;
3166 	bool kthread = p->flags & PF_KTHREAD;
3167 	unsigned int dest_cpu;
3168 	int ret = 0;
3169 
3170 	if (kthread || is_migration_disabled(p)) {
3171 		/*
3172 		 * Kernel threads are allowed on online && !active CPUs,
3173 		 * however, during cpu-hot-unplug, even these might get pushed
3174 		 * away if not KTHREAD_IS_PER_CPU.
3175 		 *
3176 		 * Specifically, migration_disabled() tasks must not fail the
3177 		 * cpumask_any_and_distribute() pick below, esp. so on
3178 		 * SCA_MIGRATE_ENABLE, otherwise we'll not call
3179 		 * set_cpus_allowed_common() and actually reset p->cpus_ptr.
3180 		 */
3181 		cpu_valid_mask = cpu_online_mask;
3182 	}
3183 
3184 	if (!kthread && !cpumask_subset(ctx->new_mask, cpu_allowed_mask)) {
3185 		ret = -EINVAL;
3186 		goto out;
3187 	}
3188 
3189 	/*
3190 	 * Must re-check here, to close a race against __kthread_bind(),
3191 	 * sched_setaffinity() is not guaranteed to observe the flag.
3192 	 */
3193 	if ((ctx->flags & SCA_CHECK) && (p->flags & PF_NO_SETAFFINITY)) {
3194 		ret = -EINVAL;
3195 		goto out;
3196 	}
3197 
3198 	if (!(ctx->flags & SCA_MIGRATE_ENABLE)) {
3199 		if (cpumask_equal(&p->cpus_mask, ctx->new_mask)) {
3200 			if (ctx->flags & SCA_USER)
3201 				swap(p->user_cpus_ptr, ctx->user_mask);
3202 			goto out;
3203 		}
3204 
3205 		if (WARN_ON_ONCE(p == current &&
3206 				 is_migration_disabled(p) &&
3207 				 !cpumask_test_cpu(task_cpu(p), ctx->new_mask))) {
3208 			ret = -EBUSY;
3209 			goto out;
3210 		}
3211 	}
3212 
3213 	/*
3214 	 * Picking a ~random cpu helps in cases where we are changing affinity
3215 	 * for groups of tasks (ie. cpuset), so that load balancing is not
3216 	 * immediately required to distribute the tasks within their new mask.
3217 	 */
3218 	dest_cpu = cpumask_any_and_distribute(cpu_valid_mask, ctx->new_mask);
3219 	if (dest_cpu >= nr_cpu_ids) {
3220 		ret = -EINVAL;
3221 		goto out;
3222 	}
3223 
3224 	do_set_cpus_allowed(p, ctx);
3225 
3226 	return affine_move_task(rq, p, rf, dest_cpu, ctx->flags);
3227 
3228 out:
3229 	task_rq_unlock(rq, p, rf);
3230 
3231 	return ret;
3232 }
3233 
3234 /*
3235  * Change a given task's CPU affinity. Migrate the thread to a
3236  * proper CPU and schedule it away if the CPU it's executing on
3237  * is removed from the allowed bitmask.
3238  *
3239  * NOTE: the caller must have a valid reference to the task, the
3240  * task must not exit() & deallocate itself prematurely. The
3241  * call is not atomic; no spinlocks may be held.
3242  */
3243 int __set_cpus_allowed_ptr(struct task_struct *p, struct affinity_context *ctx)
3244 {
3245 	struct rq_flags rf;
3246 	struct rq *rq;
3247 
3248 	rq = task_rq_lock(p, &rf);
3249 	/*
3250 	 * Masking should be skipped if SCA_USER or any of the SCA_MIGRATE_*
3251 	 * flags are set.
3252 	 */
3253 	if (p->user_cpus_ptr &&
3254 	    !(ctx->flags & (SCA_USER | SCA_MIGRATE_ENABLE | SCA_MIGRATE_DISABLE)) &&
3255 	    cpumask_and(rq->scratch_mask, ctx->new_mask, p->user_cpus_ptr))
3256 		ctx->new_mask = rq->scratch_mask;
3257 
3258 	return __set_cpus_allowed_ptr_locked(p, ctx, rq, &rf);
3259 }
3260 
3261 int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
3262 {
3263 	struct affinity_context ac = {
3264 		.new_mask  = new_mask,
3265 		.flags     = 0,
3266 	};
3267 
3268 	return __set_cpus_allowed_ptr(p, &ac);
3269 }
3270 EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
3271 
3272 /*
3273  * Change a given task's CPU affinity to the intersection of its current
3274  * affinity mask and @subset_mask, writing the resulting mask to @new_mask.
3275  * If user_cpus_ptr is defined, use it as the basis for restricting CPU
3276  * affinity or use cpu_online_mask instead.
3277  *
3278  * If the resulting mask is empty, leave the affinity unchanged and return
3279  * -EINVAL.
3280  */
3281 static int restrict_cpus_allowed_ptr(struct task_struct *p,
3282 				     struct cpumask *new_mask,
3283 				     const struct cpumask *subset_mask)
3284 {
3285 	struct affinity_context ac = {
3286 		.new_mask  = new_mask,
3287 		.flags     = 0,
3288 	};
3289 	struct rq_flags rf;
3290 	struct rq *rq;
3291 	int err;
3292 
3293 	rq = task_rq_lock(p, &rf);
3294 
3295 	/*
3296 	 * Forcefully restricting the affinity of a deadline task is
3297 	 * likely to cause problems, so fail and noisily override the
3298 	 * mask entirely.
3299 	 */
3300 	if (task_has_dl_policy(p) && dl_bandwidth_enabled()) {
3301 		err = -EPERM;
3302 		goto err_unlock;
3303 	}
3304 
3305 	if (!cpumask_and(new_mask, task_user_cpus(p), subset_mask)) {
3306 		err = -EINVAL;
3307 		goto err_unlock;
3308 	}
3309 
3310 	return __set_cpus_allowed_ptr_locked(p, &ac, rq, &rf);
3311 
3312 err_unlock:
3313 	task_rq_unlock(rq, p, &rf);
3314 	return err;
3315 }
3316 
3317 /*
3318  * Restrict the CPU affinity of task @p so that it is a subset of
3319  * task_cpu_possible_mask() and point @p->user_cpus_ptr to a copy of the
3320  * old affinity mask. If the resulting mask is empty, we warn and walk
3321  * up the cpuset hierarchy until we find a suitable mask.
3322  */
3323 void force_compatible_cpus_allowed_ptr(struct task_struct *p)
3324 {
3325 	cpumask_var_t new_mask;
3326 	const struct cpumask *override_mask = task_cpu_possible_mask(p);
3327 
3328 	alloc_cpumask_var(&new_mask, GFP_KERNEL);
3329 
3330 	/*
3331 	 * __migrate_task() can fail silently in the face of concurrent
3332 	 * offlining of the chosen destination CPU, so take the hotplug
3333 	 * lock to ensure that the migration succeeds.
3334 	 */
3335 	cpus_read_lock();
3336 	if (!cpumask_available(new_mask))
3337 		goto out_set_mask;
3338 
3339 	if (!restrict_cpus_allowed_ptr(p, new_mask, override_mask))
3340 		goto out_free_mask;
3341 
3342 	/*
3343 	 * We failed to find a valid subset of the affinity mask for the
3344 	 * task, so override it based on its cpuset hierarchy.
3345 	 */
3346 	cpuset_cpus_allowed(p, new_mask);
3347 	override_mask = new_mask;
3348 
3349 out_set_mask:
3350 	if (printk_ratelimit()) {
3351 		printk_deferred("Overriding affinity for process %d (%s) to CPUs %*pbl\n",
3352 				task_pid_nr(p), p->comm,
3353 				cpumask_pr_args(override_mask));
3354 	}
3355 
3356 	WARN_ON(set_cpus_allowed_ptr(p, override_mask));
3357 out_free_mask:
3358 	cpus_read_unlock();
3359 	free_cpumask_var(new_mask);
3360 }
3361 
3362 /*
3363  * Restore the affinity of a task @p which was previously restricted by a
3364  * call to force_compatible_cpus_allowed_ptr().
3365  *
3366  * It is the caller's responsibility to serialise this with any calls to
3367  * force_compatible_cpus_allowed_ptr(@p).
3368  */
3369 void relax_compatible_cpus_allowed_ptr(struct task_struct *p)
3370 {
3371 	struct affinity_context ac = {
3372 		.new_mask  = task_user_cpus(p),
3373 		.flags     = 0,
3374 	};
3375 	int ret;
3376 
3377 	/*
3378 	 * Try to restore the old affinity mask with __sched_setaffinity().
3379 	 * Cpuset masking will be done there too.
3380 	 */
3381 	ret = __sched_setaffinity(p, &ac);
3382 	WARN_ON_ONCE(ret);
3383 }
3384 
3385 #ifdef CONFIG_SMP
3386 
3387 void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
3388 {
3389 	unsigned int state = READ_ONCE(p->__state);
3390 	bool proxy_migrated = sched_proxy_exec() && p->is_blocked &&
3391 			      task_cpu(p) != p->wake_cpu;
3392 
3393 	/*
3394 	 * We should never call set_task_cpu() on a blocked task,
3395 	 * ttwu() will sort out the placement.
3396 	 */
3397 	WARN_ON_ONCE(state != TASK_RUNNING && state != TASK_WAKING && !p->on_rq);
3398 
3399 	/*
3400 	 * Migrating fair class task must have p->on_rq = TASK_ON_RQ_MIGRATING,
3401 	 * because schedstat_wait_{start,end} rebase migrating task's wait_start
3402 	 * time relying on p->on_rq.
3403 	 */
3404 	WARN_ON_ONCE(state == TASK_RUNNING &&
3405 		     p->sched_class == &fair_sched_class &&
3406 		     (p->on_rq && !task_on_rq_migrating(p)));
3407 
3408 #ifdef CONFIG_LOCKDEP
3409 	/*
3410 	 * The caller should hold either p->pi_lock or rq->lock, when changing
3411 	 * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks.
3412 	 *
3413 	 * sched_move_task() holds both and thus holding either pins the cgroup,
3414 	 * see task_group().
3415 	 *
3416 	 * Furthermore, all task_rq users should acquire both locks, see
3417 	 * task_rq_lock().
3418 	 */
3419 	WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) ||
3420 				      lockdep_is_held(__rq_lockp(task_rq(p)))));
3421 #endif
3422 	/*
3423 	 * Clearly, migrating tasks to offline CPUs is a fairly daft thing.
3424 	 */
3425 	WARN_ON_ONCE(!cpu_online(new_cpu));
3426 
3427 	/*
3428 	 * Proxy execution can move a blocked task's scheduling context to any
3429 	 * CPU without moving its migration-disabled execution context. The
3430 	 * wakeup path will return the task to a CPU where it can execute.
3431 	 */
3432 	WARN_ON_ONCE(is_migration_disabled(p) && !proxy_migrated);
3433 
3434 	trace_sched_migrate_task(p, new_cpu);
3435 
3436 	if (task_cpu(p) != new_cpu) {
3437 		if (p->sched_class->migrate_task_rq)
3438 			p->sched_class->migrate_task_rq(p, new_cpu);
3439 		p->se.nr_migrations++;
3440 		perf_event_task_migrate(p);
3441 	}
3442 
3443 	__set_task_cpu(p, new_cpu);
3444 }
3445 #endif /* CONFIG_SMP */
3446 
3447 #ifdef CONFIG_NUMA_BALANCING
3448 static void __migrate_swap_task(struct task_struct *p, int cpu)
3449 {
3450 	if (task_on_rq_queued(p)) {
3451 		struct rq *src_rq, *dst_rq;
3452 		struct rq_flags srf, drf;
3453 
3454 		src_rq = task_rq(p);
3455 		dst_rq = cpu_rq(cpu);
3456 
3457 		rq_pin_lock(src_rq, &srf);
3458 		rq_pin_lock(dst_rq, &drf);
3459 
3460 		move_queued_task_locked(src_rq, dst_rq, p);
3461 		wakeup_preempt(dst_rq, p, 0);
3462 
3463 		rq_unpin_lock(dst_rq, &drf);
3464 		rq_unpin_lock(src_rq, &srf);
3465 
3466 	} else {
3467 		/*
3468 		 * Task isn't running anymore; make it appear like we migrated
3469 		 * it before it went to sleep. This means on wakeup we make the
3470 		 * previous CPU our target instead of where it really is.
3471 		 */
3472 		p->wake_cpu = cpu;
3473 	}
3474 }
3475 
3476 struct migration_swap_arg {
3477 	struct task_struct *src_task, *dst_task;
3478 	int src_cpu, dst_cpu;
3479 };
3480 
3481 static int migrate_swap_stop(void *data)
3482 {
3483 	struct migration_swap_arg *arg = data;
3484 	struct rq *src_rq, *dst_rq;
3485 
3486 	if (!cpu_active(arg->src_cpu) || !cpu_active(arg->dst_cpu))
3487 		return -EAGAIN;
3488 
3489 	src_rq = cpu_rq(arg->src_cpu);
3490 	dst_rq = cpu_rq(arg->dst_cpu);
3491 
3492 	guard(double_raw_spinlock)(&arg->src_task->pi_lock, &arg->dst_task->pi_lock);
3493 	guard(double_rq_lock)(src_rq, dst_rq);
3494 
3495 	if (task_cpu(arg->dst_task) != arg->dst_cpu)
3496 		return -EAGAIN;
3497 
3498 	if (task_cpu(arg->src_task) != arg->src_cpu)
3499 		return -EAGAIN;
3500 
3501 	if (!cpumask_test_cpu(arg->dst_cpu, arg->src_task->cpus_ptr))
3502 		return -EAGAIN;
3503 
3504 	if (!cpumask_test_cpu(arg->src_cpu, arg->dst_task->cpus_ptr))
3505 		return -EAGAIN;
3506 
3507 	__migrate_swap_task(arg->src_task, arg->dst_cpu);
3508 	__migrate_swap_task(arg->dst_task, arg->src_cpu);
3509 
3510 	return 0;
3511 }
3512 
3513 /*
3514  * Cross migrate two tasks
3515  */
3516 int migrate_swap(struct task_struct *cur, struct task_struct *p,
3517 		int target_cpu, int curr_cpu)
3518 {
3519 	struct migration_swap_arg arg;
3520 	int ret = -EINVAL;
3521 
3522 	arg = (struct migration_swap_arg){
3523 		.src_task = cur,
3524 		.src_cpu = curr_cpu,
3525 		.dst_task = p,
3526 		.dst_cpu = target_cpu,
3527 	};
3528 
3529 	if (arg.src_cpu == arg.dst_cpu)
3530 		goto out;
3531 
3532 	/*
3533 	 * These three tests are all lockless; this is OK since all of them
3534 	 * will be re-checked with proper locks held further down the line.
3535 	 */
3536 	if (!cpu_active(arg.src_cpu) || !cpu_active(arg.dst_cpu))
3537 		goto out;
3538 
3539 	if (!cpumask_test_cpu(arg.dst_cpu, arg.src_task->cpus_ptr))
3540 		goto out;
3541 
3542 	if (!cpumask_test_cpu(arg.src_cpu, arg.dst_task->cpus_ptr))
3543 		goto out;
3544 
3545 	trace_sched_swap_numa(cur, arg.src_cpu, p, arg.dst_cpu);
3546 	ret = stop_two_cpus(arg.dst_cpu, arg.src_cpu, migrate_swap_stop, &arg);
3547 
3548 out:
3549 	return ret;
3550 }
3551 #endif /* CONFIG_NUMA_BALANCING */
3552 
3553 /***
3554  * kick_process - kick a running thread to enter/exit the kernel
3555  * @p: the to-be-kicked thread
3556  *
3557  * Cause a process which is running on another CPU to enter
3558  * kernel-mode, without any delay. (to get signals handled.)
3559  *
3560  * NOTE: this function doesn't have to take the runqueue lock,
3561  * because all it wants to ensure is that the remote task enters
3562  * the kernel. If the IPI races and the task has been migrated
3563  * to another CPU then no harm is done and the purpose has been
3564  * achieved as well.
3565  */
3566 void kick_process(struct task_struct *p)
3567 {
3568 	guard(preempt)();
3569 	int cpu = task_cpu(p);
3570 
3571 	if ((cpu != smp_processor_id()) && task_curr(p))
3572 		smp_send_reschedule(cpu);
3573 }
3574 EXPORT_SYMBOL_GPL(kick_process);
3575 
3576 /*
3577  * ->cpus_ptr is protected by both rq->lock and p->pi_lock
3578  *
3579  * A few notes on cpu_active vs cpu_online:
3580  *
3581  *  - cpu_active must be a subset of cpu_online
3582  *
3583  *  - on CPU-up we allow per-CPU kthreads on the online && !active CPU,
3584  *    see __set_cpus_allowed_ptr(). At this point the newly online
3585  *    CPU isn't yet part of the sched domains, and balancing will not
3586  *    see it.
3587  *
3588  *  - on CPU-down we clear cpu_active() to mask the sched domains and
3589  *    avoid the load balancer to place new tasks on the to be removed
3590  *    CPU. Existing tasks will remain running there and will be taken
3591  *    off.
3592  *
3593  * This means that fallback selection must not select !active CPUs.
3594  * And can assume that any active CPU must be online. Conversely
3595  * select_task_rq() below may allow selection of !active CPUs in order
3596  * to satisfy the above rules.
3597  */
3598 static int select_fallback_rq(int cpu, struct task_struct *p)
3599 {
3600 	int nid = cpu_to_node(cpu);
3601 	const struct cpumask *nodemask = NULL;
3602 	enum { cpuset, possible, fail } state = cpuset;
3603 	int dest_cpu;
3604 
3605 	/*
3606 	 * If the node that the CPU is on has been offlined, cpu_to_node()
3607 	 * will return -1. There is no CPU on the node, and we should
3608 	 * select the CPU on the other node.
3609 	 */
3610 	if (nid != -1) {
3611 		nodemask = cpumask_of_node(nid);
3612 
3613 		/* Look for allowed, online CPU in same node. */
3614 		for_each_cpu(dest_cpu, nodemask) {
3615 			if (is_cpu_allowed(p, dest_cpu))
3616 				return dest_cpu;
3617 		}
3618 	}
3619 
3620 	for (;;) {
3621 		/* Any allowed, online CPU? */
3622 		for_each_cpu(dest_cpu, p->cpus_ptr) {
3623 			if (!is_cpu_allowed(p, dest_cpu))
3624 				continue;
3625 
3626 			goto out;
3627 		}
3628 
3629 		/* No more Mr. Nice Guy. */
3630 		switch (state) {
3631 		case cpuset:
3632 			if (cpuset_cpus_allowed_fallback(p)) {
3633 				state = possible;
3634 				break;
3635 			}
3636 			fallthrough;
3637 		case possible:
3638 			set_cpus_allowed_force(p, task_cpu_fallback_mask(p));
3639 			state = fail;
3640 			break;
3641 		case fail:
3642 			BUG();
3643 			break;
3644 		}
3645 	}
3646 
3647 out:
3648 	if (state != cpuset) {
3649 		/*
3650 		 * Don't tell them about moving exiting tasks or
3651 		 * kernel threads (both mm NULL), since they never
3652 		 * leave kernel.
3653 		 */
3654 		if (p->mm && printk_ratelimit()) {
3655 			printk_deferred("process %d (%s) no longer affine to cpu%d\n",
3656 					task_pid_nr(p), p->comm, cpu);
3657 		}
3658 	}
3659 
3660 	return dest_cpu;
3661 }
3662 
3663 /*
3664  * The caller (fork, wakeup) owns p->pi_lock, ->cpus_ptr is stable.
3665  */
3666 static inline
3667 int select_task_rq(struct task_struct *p, int cpu, int *wake_flags)
3668 {
3669 	lockdep_assert_held(&p->pi_lock);
3670 
3671 	if (p->nr_cpus_allowed > 1 && !is_migration_disabled(p)) {
3672 		cpu = p->sched_class->select_task_rq(p, cpu, *wake_flags);
3673 		*wake_flags |= WF_RQ_SELECTED;
3674 	} else {
3675 		cpu = cpumask_any(p->cpus_ptr);
3676 	}
3677 
3678 	/*
3679 	 * In order not to call set_task_cpu() on a blocking task we need
3680 	 * to rely on ttwu() to place the task on a valid ->cpus_ptr
3681 	 * CPU.
3682 	 *
3683 	 * Since this is common to all placement strategies, this lives here.
3684 	 *
3685 	 * [ this allows ->select_task() to simply return task_cpu(p) and
3686 	 *   not worry about this generic constraint ]
3687 	 */
3688 	if (unlikely(!is_cpu_allowed(p, cpu)))
3689 		cpu = select_fallback_rq(task_cpu(p), p);
3690 
3691 	return cpu;
3692 }
3693 
3694 void sched_set_stop_task(int cpu, struct task_struct *stop)
3695 {
3696 	static struct lock_class_key stop_pi_lock;
3697 	struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };
3698 	struct task_struct *old_stop = cpu_rq(cpu)->stop;
3699 
3700 	if (stop) {
3701 		/*
3702 		 * Make it appear like a SCHED_FIFO task, its something
3703 		 * userspace knows about and won't get confused about.
3704 		 *
3705 		 * Also, it will make PI more or less work without too
3706 		 * much confusion -- but then, stop work should not
3707 		 * rely on PI working anyway.
3708 		 */
3709 		sched_setscheduler_nocheck(stop, SCHED_FIFO, &param);
3710 
3711 		stop->sched_class = &stop_sched_class;
3712 
3713 		/*
3714 		 * The PI code calls rt_mutex_setprio() with ->pi_lock held to
3715 		 * adjust the effective priority of a task. As a result,
3716 		 * rt_mutex_setprio() can trigger (RT) balancing operations,
3717 		 * which can then trigger wakeups of the stop thread to push
3718 		 * around the current task.
3719 		 *
3720 		 * The stop task itself will never be part of the PI-chain, it
3721 		 * never blocks, therefore that ->pi_lock recursion is safe.
3722 		 * Tell lockdep about this by placing the stop->pi_lock in its
3723 		 * own class.
3724 		 */
3725 		lockdep_set_class(&stop->pi_lock, &stop_pi_lock);
3726 	}
3727 
3728 	cpu_rq(cpu)->stop = stop;
3729 
3730 	if (old_stop) {
3731 		/*
3732 		 * Reset it back to a normal scheduling class so that
3733 		 * it can die in pieces.
3734 		 */
3735 		old_stop->sched_class = &rt_sched_class;
3736 	}
3737 }
3738 
3739 static void
3740 ttwu_stat(struct task_struct *p, int cpu, int wake_flags)
3741 {
3742 	struct rq *rq;
3743 
3744 	if (!schedstat_enabled())
3745 		return;
3746 
3747 	rq = this_rq();
3748 
3749 	if (cpu == rq->cpu) {
3750 		__schedstat_inc(rq->ttwu_local);
3751 		__schedstat_inc(p->stats.nr_wakeups_local);
3752 	} else {
3753 		struct sched_domain *sd;
3754 
3755 		__schedstat_inc(p->stats.nr_wakeups_remote);
3756 
3757 		guard(rcu)();
3758 		for_each_domain(rq->cpu, sd) {
3759 			if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
3760 				__schedstat_inc(sd->ttwu_wake_remote);
3761 				break;
3762 			}
3763 		}
3764 	}
3765 
3766 	if (wake_flags & WF_MIGRATED)
3767 		__schedstat_inc(p->stats.nr_wakeups_migrate);
3768 
3769 	__schedstat_inc(rq->ttwu_count);
3770 	__schedstat_inc(p->stats.nr_wakeups);
3771 
3772 	if (wake_flags & WF_SYNC)
3773 		__schedstat_inc(p->stats.nr_wakeups_sync);
3774 }
3775 
3776 static inline void unblock_task(struct task_struct *p)
3777 {
3778 	if (p->is_blocked && p->in_iowait) {
3779 		delayacct_blkio_end(p);
3780 		atomic_dec(&cpu_rq(p->iowait_cpu)->nr_iowait);
3781 	}
3782 
3783 	p->is_blocked = 0;
3784 }
3785 
3786 /*
3787  * Mark the task runnable.
3788  */
3789 static inline void ttwu_do_wakeup(struct task_struct *p)
3790 {
3791 	unblock_task(p);
3792 	WRITE_ONCE(p->__state, TASK_RUNNING);
3793 	trace_sched_wakeup(p);
3794 }
3795 
3796 void update_rq_avg_idle(struct rq *rq)
3797 {
3798 	u64 idle_stamp = rq->idle_stamp;
3799 	u64 delta, max;
3800 
3801 	if (!idle_stamp)
3802 		return;
3803 
3804 	delta = rq_clock(rq) - idle_stamp;
3805 
3806 	update_avg(&rq->avg_idle, delta);
3807 
3808 	max = 2 * rq->max_idle_balance_cost;
3809 	if (rq->avg_idle > max)
3810 		rq->avg_idle = max;
3811 	rq->idle_stamp = 0;
3812 }
3813 
3814 #ifdef CONFIG_SCHED_PROXY_EXEC
3815 static void zap_balance_callbacks(struct rq *rq);
3816 
3817 static inline void proxy_reset_donor(struct rq *rq)
3818 {
3819 	WARN_ON_ONCE(rq->donor == rq->curr);
3820 
3821 	put_prev_set_next_task(rq, rq->donor, rq->curr);
3822 	rq->next_class = rq->curr->sched_class;
3823 	rq_set_donor(rq, rq->curr);
3824 	zap_balance_callbacks(rq);
3825 	resched_curr(rq);
3826 }
3827 
3828 /*
3829  * Checks to see if task p has been proxy-migrated to another rq
3830  * and needs to be returned. If so, we deactivate the task here
3831  * so that it can be properly woken up on the p->wake_cpu
3832  * (or whichever cpu select_task_rq() picks at the bottom of
3833  * try_to_wake_up()
3834  */
3835 static inline bool proxy_needs_return(struct rq *rq, struct task_struct *p)
3836 {
3837 	bool dequeued;
3838 
3839 	/*
3840 	 * Typically per __set_task_cpu(), task_cpu(p) == p->wake_cpu.
3841 	 *
3842 	 * However, proxy_set_task_cpu() is such that it preserves the
3843 	 * original cpu in p->wake_cpu while migrating p for proxy reasons
3844 	 * (possibly outside of the allowed p->cpus_ptr).
3845 	 *
3846 	 * Furthermore, migration_cpu_stop() / __migrate_swap_task(), will
3847 	 * only set p->wake_cpu when !p->on_rq, and since here p->on_rq, this
3848 	 * will not apply. But if it did, this check is the safe way around
3849 	 * and would migrate.
3850 	 */
3851 	if (task_cpu(p) == p->wake_cpu)
3852 		return false;
3853 
3854 	scoped_guard(raw_spinlock, &p->blocked_lock) {
3855 		/* Task is waking up; clear any blocked_on relationship */
3856 		__clear_task_blocked_on(p, NULL);
3857 
3858 		/* If already current, don't need to return migrate */
3859 		if (task_current(rq, p))
3860 			return false;
3861 	}
3862 
3863 	dequeued = dequeue_block_task(rq, p, TASK_WAKING);
3864 
3865 	/*
3866 	 * Dequeue @p from its scheduling class before resetting rq->donor.
3867 	 * In particular, sched_ext needs to end the donor's running session
3868 	 * and clear SCX_TASK_QUEUED before put_prev_task_scx() is called by
3869 	 * proxy_reset_donor(); otherwise it would reenqueue the blocked donor.
3870 	 *
3871 	 * Keep on_rq set until all donor references have been replaced.
3872 	 */
3873 	if (task_current_donor(rq, p))
3874 		proxy_reset_donor(rq);
3875 
3876 	if (dequeued)
3877 		__block_task(rq, p);
3878 	return true;
3879 }
3880 #else /* !CONFIG_SCHED_PROXY_EXEC */
3881 static inline bool proxy_needs_return(struct rq *rq, struct task_struct *p)
3882 {
3883 	return false;
3884 }
3885 #endif /* CONFIG_SCHED_PROXY_EXEC */
3886 
3887 static void
3888 ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags,
3889 		 struct rq_flags *rf)
3890 {
3891 	int en_flags = ENQUEUE_WAKEUP | ENQUEUE_NOCLOCK;
3892 
3893 	lockdep_assert_rq_held(rq);
3894 
3895 	if (p->sched_contributes_to_load)
3896 		rq->nr_uninterruptible--;
3897 
3898 	if (wake_flags & WF_RQ_SELECTED)
3899 		en_flags |= ENQUEUE_RQ_SELECTED;
3900 	if (wake_flags & WF_MIGRATED)
3901 		en_flags |= ENQUEUE_MIGRATED;
3902 
3903 	activate_task(rq, p, en_flags);
3904 	wakeup_preempt(rq, p, wake_flags);
3905 
3906 	ttwu_do_wakeup(p);
3907 
3908 	if (p->sched_class->task_woken) {
3909 		/*
3910 		 * Our task @p is fully woken up and running; so it's safe to
3911 		 * drop the rq->lock, hereafter rq is only used for statistics.
3912 		 */
3913 		rq_unpin_lock(rq, rf);
3914 		p->sched_class->task_woken(rq, p);
3915 		rq_repin_lock(rq, rf);
3916 	}
3917 }
3918 
3919 /*
3920  * Consider @p being inside a wait loop:
3921  *
3922  *   for (;;) {
3923  *      set_current_state(TASK_UNINTERRUPTIBLE);
3924  *
3925  *      if (CONDITION)
3926  *         break;
3927  *
3928  *      schedule();
3929  *   }
3930  *   __set_current_state(TASK_RUNNING);
3931  *
3932  * between set_current_state() and schedule(). In this case @p is still
3933  * runnable, so all that needs doing is change p->state back to TASK_RUNNING in
3934  * an atomic manner.
3935  *
3936  * By taking task_rq(p)->lock we serialize against schedule(), if @p->on_rq
3937  * then schedule() must still happen and p->state can be changed to
3938  * TASK_RUNNING. Otherwise we lost the race, schedule() has happened, and we
3939  * need to do a full wakeup with enqueue.
3940  *
3941  * Returns: %true when the wakeup is done,
3942  *          %false otherwise.
3943  */
3944 static int ttwu_runnable(struct task_struct *p, int wake_flags)
3945 {
3946 	ACQUIRE(__task_rq_lock, guard)(p);
3947 	struct rq *rq = guard.rq;
3948 
3949 	if (!task_on_rq_queued(p))
3950 		return 0;
3951 
3952 	update_rq_clock(rq);
3953 	if (p->is_blocked) {
3954 		if (p->se.sched_delayed)
3955 			enqueue_task(rq, p, ENQUEUE_NOCLOCK | ENQUEUE_DELAYED);
3956 		if (proxy_needs_return(rq, p))
3957 			return 0;
3958 	}
3959 	if (!task_on_cpu(rq, p)) {
3960 		/*
3961 		 * When on_rq && !on_cpu the task is preempted, see if
3962 		 * it should preempt the task that is current now.
3963 		 */
3964 		wakeup_preempt(rq, p, wake_flags | WF_TTWU_RQ);
3965 	}
3966 	ttwu_do_wakeup(p);
3967 	return 1;
3968 }
3969 
3970 void sched_ttwu_pending(void *arg)
3971 {
3972 	struct llist_node *llist = arg;
3973 	struct rq *rq = this_rq();
3974 	struct task_struct *p, *t;
3975 	struct rq_flags rf;
3976 
3977 	if (!llist)
3978 		return;
3979 
3980 	rq_lock_irqsave(rq, &rf);
3981 	update_rq_clock(rq);
3982 
3983 	llist_for_each_entry_safe(p, t, llist, wake_entry.llist) {
3984 		if (WARN_ON_ONCE(p->on_cpu))
3985 			smp_cond_load_acquire(&p->on_cpu, !VAL);
3986 
3987 		if (WARN_ON_ONCE(task_cpu(p) != cpu_of(rq)))
3988 			set_task_cpu(p, cpu_of(rq));
3989 
3990 		ttwu_do_activate(rq, p, p->sched_remote_wakeup ? WF_MIGRATED : 0, &rf);
3991 	}
3992 
3993 	/*
3994 	 * Must be after enqueueing at least once task such that
3995 	 * idle_cpu() does not observe a false-negative -- if it does,
3996 	 * it is possible for select_idle_siblings() to stack a number
3997 	 * of tasks on this CPU during that window.
3998 	 *
3999 	 * It is OK to clear ttwu_pending when another task pending.
4000 	 * We will receive IPI after local IRQ enabled and then enqueue it.
4001 	 * Since now nr_running > 0, idle_cpu() will always get correct result.
4002 	 */
4003 	WRITE_ONCE(rq->ttwu_pending, 0);
4004 	rq_unlock_irqrestore(rq, &rf);
4005 }
4006 
4007 /*
4008  * Prepare the scene for sending an IPI for a remote smp_call
4009  *
4010  * Returns true if the caller can proceed with sending the IPI.
4011  * Returns false otherwise.
4012  */
4013 bool call_function_single_prep_ipi(int cpu)
4014 {
4015 	if (set_nr_if_polling(cpu_rq(cpu)->idle)) {
4016 		trace_sched_wake_idle_without_ipi(cpu);
4017 		return false;
4018 	}
4019 
4020 	return true;
4021 }
4022 
4023 /*
4024  * Queue a task on the target CPUs wake_list and wake the CPU via IPI if
4025  * necessary. The wakee CPU on receipt of the IPI will queue the task
4026  * via sched_ttwu_wakeup() for activation so the wakee incurs the cost
4027  * of the wakeup instead of the waker.
4028  */
4029 static void __ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags)
4030 {
4031 	struct rq *rq = cpu_rq(cpu);
4032 
4033 	p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED);
4034 
4035 	WRITE_ONCE(rq->ttwu_pending, 1);
4036 #ifdef CONFIG_SMP
4037 	__smp_call_single_queue(cpu, &p->wake_entry.llist);
4038 #endif
4039 }
4040 
4041 void wake_up_if_idle(int cpu)
4042 {
4043 	struct rq *rq = cpu_rq(cpu);
4044 
4045 	guard(rcu)();
4046 	if (is_idle_task(rcu_dereference(rq->curr))) {
4047 		guard(rq_lock_irqsave)(rq);
4048 		if (is_idle_task(rq->curr))
4049 			resched_curr(rq);
4050 	}
4051 }
4052 
4053 bool cpus_equal_capacity(int this_cpu, int that_cpu)
4054 {
4055 	if (!sched_asym_cpucap_active())
4056 		return true;
4057 
4058 	if (this_cpu == that_cpu)
4059 		return true;
4060 
4061 	return arch_scale_cpu_capacity(this_cpu) == arch_scale_cpu_capacity(that_cpu);
4062 }
4063 
4064 bool cpus_share_cache(int this_cpu, int that_cpu)
4065 {
4066 	if (this_cpu == that_cpu)
4067 		return true;
4068 
4069 	return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu);
4070 }
4071 
4072 /*
4073  * Whether CPUs are share cache resources, which means LLC on non-cluster
4074  * machines and LLC tag or L2 on machines with clusters.
4075  */
4076 bool cpus_share_resources(int this_cpu, int that_cpu)
4077 {
4078 	if (this_cpu == that_cpu)
4079 		return true;
4080 
4081 	return per_cpu(sd_share_id, this_cpu) == per_cpu(sd_share_id, that_cpu);
4082 }
4083 
4084 static inline bool ttwu_queue_cond(struct task_struct *p, int cpu)
4085 {
4086 	int this_cpu = smp_processor_id();
4087 
4088 	/* See SCX_OPS_ALLOW_QUEUED_WAKEUP. */
4089 	if (!scx_allow_ttwu_queue(p))
4090 		return false;
4091 
4092 #ifdef CONFIG_SMP
4093 	if (p->sched_class == &stop_sched_class)
4094 		return false;
4095 #endif
4096 
4097 	/*
4098 	 * Do not complicate things with the async wake_list while the CPU is
4099 	 * in hotplug state.
4100 	 */
4101 	if (!cpu_active(cpu))
4102 		return false;
4103 
4104 	/* Ensure the task will still be allowed to run on the CPU. */
4105 	if (!cpumask_test_cpu(cpu, p->cpus_ptr))
4106 		return false;
4107 
4108 	/*
4109 	 * If the CPU does not share cache, then queue the task on the
4110 	 * remote rqs wakelist to avoid accessing remote data.
4111 	 */
4112 	if (!cpus_share_cache(this_cpu, cpu))
4113 		return true;
4114 
4115 	if (cpu == this_cpu)
4116 		return false;
4117 
4118 	/*
4119 	 * If the wakee cpu is idle, or the task is descheduling and the
4120 	 * only running task on the CPU, then use the wakelist to offload
4121 	 * the task activation to the idle (or soon-to-be-idle) CPU as
4122 	 * the current CPU is likely busy. nr_running is checked to
4123 	 * avoid unnecessary task stacking.
4124 	 *
4125 	 * Note that we can only get here with (wakee) p->on_rq=0,
4126 	 * p->on_cpu can be whatever, we've done the dequeue, so
4127 	 * the wakee has been accounted out of ->nr_running.
4128 	 */
4129 	if (!cpu_rq(cpu)->nr_running)
4130 		return true;
4131 
4132 	return false;
4133 }
4134 
4135 static bool ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags)
4136 {
4137 	if (sched_feat(TTWU_QUEUE) && ttwu_queue_cond(p, cpu)) {
4138 		sched_clock_cpu(cpu); /* Sync clocks across CPUs */
4139 		__ttwu_queue_wakelist(p, cpu, wake_flags);
4140 		return true;
4141 	}
4142 
4143 	return false;
4144 }
4145 
4146 static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags)
4147 {
4148 	struct rq *rq = cpu_rq(cpu);
4149 	struct rq_flags rf;
4150 
4151 	if (ttwu_queue_wakelist(p, cpu, wake_flags))
4152 		return;
4153 
4154 	rq_lock(rq, &rf);
4155 	update_rq_clock(rq);
4156 	ttwu_do_activate(rq, p, wake_flags, &rf);
4157 	rq_unlock(rq, &rf);
4158 }
4159 
4160 /*
4161  * Invoked from try_to_wake_up() to check whether the task can be woken up.
4162  *
4163  * The caller holds p::pi_lock if p != current or has preemption
4164  * disabled when p == current.
4165  *
4166  * The rules of saved_state:
4167  *
4168  *   The related locking code always holds p::pi_lock when updating
4169  *   p::saved_state, which means the code is fully serialized in both cases.
4170  *
4171  *   For PREEMPT_RT, the lock wait and lock wakeups happen via TASK_RTLOCK_WAIT.
4172  *   No other bits set. This allows to distinguish all wakeup scenarios.
4173  *
4174  *   For FREEZER, the wakeup happens via TASK_FROZEN. No other bits set. This
4175  *   allows us to prevent early wakeup of tasks before they can be run on
4176  *   asymmetric ISA architectures (eg ARMv9).
4177  */
4178 static __always_inline
4179 bool ttwu_state_match(struct task_struct *p, unsigned int state, int *success)
4180 {
4181 	int match;
4182 
4183 	if (IS_ENABLED(CONFIG_DEBUG_PREEMPT)) {
4184 		WARN_ON_ONCE((state & TASK_RTLOCK_WAIT) &&
4185 			     state != TASK_RTLOCK_WAIT);
4186 	}
4187 
4188 	*success = !!(match = __task_state_match(p, state));
4189 
4190 	/*
4191 	 * Saved state preserves the task state across blocking on
4192 	 * an RT lock or TASK_FREEZABLE tasks.  If the state matches,
4193 	 * set p::saved_state to TASK_RUNNING, but do not wake the task
4194 	 * because it waits for a lock wakeup or __thaw_task(). Also
4195 	 * indicate success because from the regular waker's point of
4196 	 * view this has succeeded.
4197 	 *
4198 	 * After acquiring the lock the task will restore p::__state
4199 	 * from p::saved_state which ensures that the regular
4200 	 * wakeup is not lost. The restore will also set
4201 	 * p::saved_state to TASK_RUNNING so any further tests will
4202 	 * not result in false positives vs. @success
4203 	 */
4204 	if (match < 0)
4205 		p->saved_state = TASK_RUNNING;
4206 
4207 	return match > 0;
4208 }
4209 
4210 /*
4211  * Notes on Program-Order guarantees on SMP systems.
4212  *
4213  *  MIGRATION
4214  *
4215  * The basic program-order guarantee on SMP systems is that when a task [t]
4216  * migrates, all its activity on its old CPU [c0] happens-before any subsequent
4217  * execution on its new CPU [c1].
4218  *
4219  * For migration (of runnable tasks) this is provided by the following means:
4220  *
4221  *  A) UNLOCK of the rq(c0)->lock scheduling out task t
4222  *  B) migration for t is required to synchronize *both* rq(c0)->lock and
4223  *     rq(c1)->lock (if not at the same time, then in that order).
4224  *  C) LOCK of the rq(c1)->lock scheduling in task
4225  *
4226  * Release/acquire chaining guarantees that B happens after A and C after B.
4227  * Note: the CPU doing B need not be c0 or c1
4228  *
4229  * Example:
4230  *
4231  *   CPU0            CPU1            CPU2
4232  *
4233  *   LOCK rq(0)->lock
4234  *   sched-out X
4235  *   sched-in Y
4236  *   UNLOCK rq(0)->lock
4237  *
4238  *                                   LOCK rq(0)->lock // orders against CPU0
4239  *                                   dequeue X
4240  *                                   UNLOCK rq(0)->lock
4241  *
4242  *                                   LOCK rq(1)->lock
4243  *                                   enqueue X
4244  *                                   UNLOCK rq(1)->lock
4245  *
4246  *                   LOCK rq(1)->lock // orders against CPU2
4247  *                   sched-out Z
4248  *                   sched-in X
4249  *                   UNLOCK rq(1)->lock
4250  *
4251  *
4252  *  BLOCKING -- aka. SLEEP + WAKEUP
4253  *
4254  * For blocking we (obviously) need to provide the same guarantee as for
4255  * migration. However the means are completely different as there is no lock
4256  * chain to provide order. Instead we do:
4257  *
4258  *   1) smp_store_release(X->on_cpu, 0)   -- finish_task()
4259  *   2) smp_cond_load_acquire(!X->on_cpu) -- try_to_wake_up()
4260  *
4261  * Example:
4262  *
4263  *   CPU0 (schedule)  CPU1 (try_to_wake_up) CPU2 (schedule)
4264  *
4265  *   LOCK rq(0)->lock LOCK X->pi_lock
4266  *   dequeue X
4267  *   sched-out X
4268  *   smp_store_release(X->on_cpu, 0);
4269  *
4270  *                    smp_cond_load_acquire(&X->on_cpu, !VAL);
4271  *                    X->state = WAKING
4272  *                    set_task_cpu(X,2)
4273  *
4274  *                    LOCK rq(2)->lock
4275  *                    enqueue X
4276  *                    X->state = RUNNING
4277  *                    UNLOCK rq(2)->lock
4278  *
4279  *                                          LOCK rq(2)->lock // orders against CPU1
4280  *                                          sched-out Z
4281  *                                          sched-in X
4282  *                                          UNLOCK rq(2)->lock
4283  *
4284  *                    UNLOCK X->pi_lock
4285  *   UNLOCK rq(0)->lock
4286  *
4287  *
4288  * However, for wakeups there is a second guarantee we must provide, namely we
4289  * must ensure that CONDITION=1 done by the caller can not be reordered with
4290  * accesses to the task state; see try_to_wake_up() and set_current_state().
4291  */
4292 
4293 /**
4294  * try_to_wake_up - wake up a thread
4295  * @p: the thread to be awakened
4296  * @state: the mask of task states that can be woken
4297  * @wake_flags: wake modifier flags (WF_*)
4298  *
4299  * Conceptually does:
4300  *
4301  *   If (@state & @p->state) @p->state = TASK_RUNNING.
4302  *
4303  * If the task was not queued/runnable, also place it back on a runqueue.
4304  *
4305  * This function is atomic against schedule() which would dequeue the task.
4306  *
4307  * It issues a full memory barrier before accessing @p->state, see the comment
4308  * with set_current_state().
4309  *
4310  * Uses p->pi_lock to serialize against concurrent wake-ups.
4311  *
4312  * Relies on p->pi_lock stabilizing:
4313  *  - p->sched_class
4314  *  - p->cpus_ptr
4315  *  - p->sched_task_group
4316  * in order to do migration, see its use of select_task_rq()/set_task_cpu().
4317  *
4318  * Tries really hard to only take one task_rq(p)->lock for performance.
4319  * Takes rq->lock in:
4320  *  - ttwu_runnable()    -- old rq, unavoidable, see comment there;
4321  *  - ttwu_queue()       -- new rq, for enqueue of the task;
4322  *  - psi_ttwu_dequeue() -- much sadness :-( accounting will kill us.
4323  *
4324  * As a consequence we race really badly with just about everything. See the
4325  * many memory barriers and their comments for details.
4326  *
4327  * Return: %true if @p->state changes (an actual wakeup was done),
4328  *	   %false otherwise.
4329  */
4330 int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
4331 {
4332 	guard(preempt)();
4333 	int cpu, success = 0;
4334 
4335 	wake_flags |= WF_TTWU;
4336 
4337 	if (p == current) {
4338 		/*
4339 		 * We're waking current, this means 'p->on_rq' and 'task_cpu(p)
4340 		 * == smp_processor_id()'. Together this means we can special
4341 		 * case the whole 'p->on_rq && ttwu_runnable()' case below
4342 		 * without taking any locks.
4343 		 *
4344 		 * Specifically, given current runs ttwu() we must be before
4345 		 * schedule()'s block_task(), as such this must not observe
4346 		 * sched_delayed.
4347 		 *
4348 		 * In particular:
4349 		 *  - we rely on Program-Order guarantees for all the ordering,
4350 		 *  - we're serialized against set_special_state() by virtue of
4351 		 *    it disabling IRQs (this allows not taking ->pi_lock).
4352 		 */
4353 		WARN_ON_ONCE(p->se.sched_delayed);
4354 		WARN_ON_ONCE(p->is_blocked);
4355 		/* If p is current, we know we can run here, so clear blocked_on */
4356 		clear_task_blocked_on(p, NULL);
4357 		if (!ttwu_state_match(p, state, &success))
4358 			goto out;
4359 
4360 		trace_sched_waking(p);
4361 		ttwu_do_wakeup(p);
4362 		goto out;
4363 	}
4364 
4365 	/*
4366 	 * If we are going to wake up a thread waiting for CONDITION we
4367 	 * need to ensure that CONDITION=1 done by the caller can not be
4368 	 * reordered with p->state check below. This pairs with smp_store_mb()
4369 	 * in set_current_state() that the waiting thread does.
4370 	 */
4371 	scoped_guard (raw_spinlock_irqsave, &p->pi_lock) {
4372 		smp_mb__after_spinlock();
4373 
4374 		if (!ttwu_state_match(p, state, &success))
4375 			break;
4376 
4377 		trace_sched_waking(p);
4378 
4379 		/*
4380 		 * Ensure we load p->on_rq _after_ p->state, otherwise it would
4381 		 * be possible to, falsely, observe p->on_rq == 0 and get stuck
4382 		 * in smp_cond_load_acquire() below.
4383 		 *
4384 		 * sched_ttwu_pending()			try_to_wake_up()
4385 		 *   STORE p->on_rq = 1			  LOAD p->state
4386 		 *   UNLOCK rq->lock
4387 		 *
4388 		 * __schedule() (switch to task 'p')
4389 		 *   LOCK rq->lock			  smp_rmb();
4390 		 *   smp_mb__after_spinlock();
4391 		 *   UNLOCK rq->lock
4392 		 *
4393 		 * [task p]
4394 		 *   STORE p->state = UNINTERRUPTIBLE	  LOAD p->on_rq
4395 		 *
4396 		 * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in
4397 		 * __schedule().  See the comment for smp_mb__after_spinlock().
4398 		 *
4399 		 * A similar smp_rmb() lives in __task_needs_rq_lock().
4400 		 */
4401 		smp_rmb();
4402 		if (READ_ONCE(p->on_rq) && ttwu_runnable(p, wake_flags))
4403 			break;
4404 
4405 		/*
4406 		 * Ensure we load p->on_cpu _after_ p->on_rq, otherwise it would be
4407 		 * possible to, falsely, observe p->on_cpu == 0.
4408 		 *
4409 		 * One must be running (->on_cpu == 1) in order to remove oneself
4410 		 * from the runqueue.
4411 		 *
4412 		 * __schedule() (switch to task 'p')	try_to_wake_up()
4413 		 *   STORE p->on_cpu = 1		  LOAD p->on_rq
4414 		 *   UNLOCK rq->lock
4415 		 *
4416 		 * __schedule() (put 'p' to sleep)
4417 		 *   LOCK rq->lock			  smp_rmb();
4418 		 *   smp_mb__after_spinlock();
4419 		 *   STORE p->on_rq = 0			  LOAD p->on_cpu
4420 		 *
4421 		 * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in
4422 		 * __schedule().  See the comment for smp_mb__after_spinlock().
4423 		 *
4424 		 * Form a control-dep-acquire with p->on_rq == 0 above, to ensure
4425 		 * schedule()'s block_task() has 'happened' and p will no longer
4426 		 * care about it's own p->state. See the comment in __schedule().
4427 		 */
4428 		smp_acquire__after_ctrl_dep();
4429 
4430 		/*
4431 		 * We're doing the wakeup (@success == 1), they did a dequeue (p->on_rq
4432 		 * == 0), which means we need to do an enqueue, change p->state to
4433 		 * TASK_WAKING such that we can unlock p->pi_lock before doing the
4434 		 * enqueue, such as ttwu_queue_wakelist().
4435 		 */
4436 		WRITE_ONCE(p->__state, TASK_WAKING);
4437 
4438 		/*
4439 		 * If the owning (remote) CPU is still in the middle of schedule() with
4440 		 * this task as prev, considering queueing p on the remote CPUs wake_list
4441 		 * which potentially sends an IPI instead of spinning on p->on_cpu to
4442 		 * let the waker make forward progress. This is safe because IRQs are
4443 		 * disabled and the IPI will deliver after on_cpu is cleared.
4444 		 *
4445 		 * Ensure we load task_cpu(p) after p->on_cpu:
4446 		 *
4447 		 * set_task_cpu(p, cpu);
4448 		 *   STORE p->cpu = @cpu
4449 		 * __schedule() (switch to task 'p')
4450 		 *   LOCK rq->lock
4451 		 *   smp_mb__after_spin_lock()		smp_cond_load_acquire(&p->on_cpu)
4452 		 *   STORE p->on_cpu = 1		LOAD p->cpu
4453 		 *
4454 		 * to ensure we observe the correct CPU on which the task is currently
4455 		 * scheduling.
4456 		 */
4457 		if (smp_load_acquire(&p->on_cpu) &&
4458 		    ttwu_queue_wakelist(p, task_cpu(p), wake_flags))
4459 			break;
4460 
4461 		/*
4462 		 * If the owning (remote) CPU is still in the middle of schedule() with
4463 		 * this task as prev, wait until it's done referencing the task.
4464 		 *
4465 		 * Pairs with the smp_store_release() in finish_task().
4466 		 *
4467 		 * This ensures that tasks getting woken will be fully ordered against
4468 		 * their previous state and preserve Program Order.
4469 		 */
4470 		smp_cond_load_acquire(&p->on_cpu, !VAL);
4471 
4472 		cpu = select_task_rq(p, p->wake_cpu, &wake_flags);
4473 		if (task_cpu(p) != cpu) {
4474 			wake_flags |= WF_MIGRATED;
4475 			psi_ttwu_dequeue(p);
4476 			set_task_cpu(p, cpu);
4477 		} else if (cpu != p->wake_cpu) {
4478 			/*
4479 			 * If we were proxy-migrated to cpu, then
4480 			 * select_task_rq() picks cpu instead of wake_cpu
4481 			 * to return to, we won't call set_task_cpu(),
4482 			 * leaving a stale wake_cpu pointing to where we
4483 			 * proxy-migrated from. So just fixup wake_cpu here
4484 			 * if its not correct
4485 			 */
4486 			p->wake_cpu = cpu;
4487 		}
4488 
4489 		ttwu_queue(p, cpu, wake_flags);
4490 	}
4491 out:
4492 	if (success)
4493 		ttwu_stat(p, task_cpu(p), wake_flags);
4494 
4495 	return success;
4496 }
4497 
4498 static bool __task_needs_rq_lock(struct task_struct *p)
4499 {
4500 	unsigned int state = READ_ONCE(p->__state);
4501 
4502 	/*
4503 	 * Since pi->lock blocks try_to_wake_up(), we don't need rq->lock when
4504 	 * the task is blocked. Make sure to check @state since ttwu() can drop
4505 	 * locks at the end, see ttwu_queue_wakelist().
4506 	 */
4507 	if (state == TASK_RUNNING || state == TASK_WAKING)
4508 		return true;
4509 
4510 	/*
4511 	 * Ensure we load p->on_rq after p->__state, otherwise it would be
4512 	 * possible to, falsely, observe p->on_rq == 0.
4513 	 *
4514 	 * See try_to_wake_up() for a longer comment.
4515 	 */
4516 	smp_rmb();
4517 	if (p->on_rq)
4518 		return true;
4519 
4520 	/*
4521 	 * Ensure the task has finished __schedule() and will not be referenced
4522 	 * anymore. Again, see try_to_wake_up() for a longer comment.
4523 	 */
4524 	smp_rmb();
4525 	smp_cond_load_acquire(&p->on_cpu, !VAL);
4526 
4527 	return false;
4528 }
4529 
4530 /**
4531  * task_call_func - Invoke a function on task in fixed state
4532  * @p: Process for which the function is to be invoked, can be @current.
4533  * @func: Function to invoke.
4534  * @arg: Argument to function.
4535  *
4536  * Fix the task in it's current state by avoiding wakeups and or rq operations
4537  * and call @func(@arg) on it.  This function can use task_is_runnable() and
4538  * task_curr() to work out what the state is, if required.  Given that @func
4539  * can be invoked with a runqueue lock held, it had better be quite
4540  * lightweight.
4541  *
4542  * Returns:
4543  *   Whatever @func returns
4544  */
4545 int task_call_func(struct task_struct *p, task_call_f func, void *arg)
4546 {
4547 	struct rq_flags rf;
4548 	int ret;
4549 
4550 	raw_spin_lock_irqsave(&p->pi_lock, rf.flags);
4551 
4552 	if (__task_needs_rq_lock(p)) {
4553 		struct rq *rq = __task_rq_lock(p, &rf);
4554 
4555 		/*
4556 		 * At this point the task is pinned; either:
4557 		 *  - blocked and we're holding off wakeups	 (pi->lock)
4558 		 *  - woken, and we're holding off enqueue	 (rq->lock)
4559 		 *  - queued, and we're holding off schedule	 (rq->lock)
4560 		 *  - running, and we're holding off de-schedule (rq->lock)
4561 		 *
4562 		 * The called function (@func) can use: task_curr(), p->on_rq and
4563 		 * p->__state to differentiate between these states.
4564 		 */
4565 		ret = func(p, arg);
4566 
4567 		__task_rq_unlock(rq, p, &rf);
4568 	} else {
4569 		ret = func(p, arg);
4570 	}
4571 
4572 	raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags);
4573 	return ret;
4574 }
4575 
4576 /**
4577  * cpu_curr_snapshot - Return a snapshot of the currently running task
4578  * @cpu: The CPU on which to snapshot the task.
4579  *
4580  * Returns the task_struct pointer of the task "currently" running on
4581  * the specified CPU.
4582  *
4583  * If the specified CPU was offline, the return value is whatever it
4584  * is, perhaps a pointer to the task_struct structure of that CPU's idle
4585  * task, but there is no guarantee.  Callers wishing a useful return
4586  * value must take some action to ensure that the specified CPU remains
4587  * online throughout.
4588  *
4589  * This function executes full memory barriers before and after fetching
4590  * the pointer, which permits the caller to confine this function's fetch
4591  * with respect to the caller's accesses to other shared variables.
4592  */
4593 struct task_struct *cpu_curr_snapshot(int cpu)
4594 {
4595 	struct rq *rq = cpu_rq(cpu);
4596 	struct task_struct *t;
4597 	struct rq_flags rf;
4598 
4599 	rq_lock_irqsave(rq, &rf);
4600 	smp_mb__after_spinlock(); /* Pairing determined by caller's synchronization design. */
4601 	t = rcu_dereference(cpu_curr(cpu));
4602 	rq_unlock_irqrestore(rq, &rf);
4603 	smp_mb(); /* Pairing determined by caller's synchronization design. */
4604 
4605 	return t;
4606 }
4607 
4608 /**
4609  * wake_up_process - Wake up a specific process
4610  * @p: The process to be woken up.
4611  *
4612  * Attempt to wake up the nominated process and move it to the set of runnable
4613  * processes.
4614  *
4615  * Return: 1 if the process was woken up, 0 if it was already running.
4616  *
4617  * This function executes a full memory barrier before accessing the task state.
4618  */
4619 int wake_up_process(struct task_struct *p)
4620 {
4621 	return try_to_wake_up(p, TASK_NORMAL, 0);
4622 }
4623 EXPORT_SYMBOL(wake_up_process);
4624 
4625 int wake_up_state(struct task_struct *p, unsigned int state)
4626 {
4627 	return try_to_wake_up(p, state, 0);
4628 }
4629 
4630 /*
4631  * Perform scheduler related setup for a newly forked process p.
4632  * p is forked by current.
4633  *
4634  * __sched_fork() is basic setup which is also used by sched_init() to
4635  * initialize the boot CPU's idle task.
4636  */
4637 static void __sched_fork(u64 clone_flags, struct task_struct *p)
4638 {
4639 	p->on_rq			= 0;
4640 
4641 	p->se.on_rq			= 0;
4642 	p->se.exec_start		= 0;
4643 	p->se.sum_exec_runtime		= 0;
4644 	p->se.prev_sum_exec_runtime	= 0;
4645 	p->se.nr_migrations		= 0;
4646 	p->se.vruntime			= 0;
4647 	p->se.vlag			= 0;
4648 	p->se.rel_deadline		= 0;
4649 	INIT_LIST_HEAD(&p->se.group_node);
4650 
4651 	/* A delayed task cannot be in clone(). */
4652 	WARN_ON_ONCE(p->se.sched_delayed);
4653 	WARN_ON_ONCE(p->is_blocked);
4654 
4655 #ifdef CONFIG_FAIR_GROUP_SCHED
4656 	p->se.cfs_rq			= NULL;
4657 #ifdef CONFIG_CFS_BANDWIDTH
4658 	init_cfs_throttle_work(p);
4659 #endif
4660 #endif
4661 
4662 #ifdef CONFIG_SCHEDSTATS
4663 	/* Even if schedstat is disabled, there should not be garbage */
4664 	memset(&p->stats, 0, sizeof(p->stats));
4665 #endif
4666 
4667 	init_dl_entity(&p->dl);
4668 
4669 	INIT_LIST_HEAD(&p->rt.run_list);
4670 	p->rt.timeout		= 0;
4671 	p->rt.time_slice	= sched_rr_timeslice;
4672 	p->rt.on_rq		= 0;
4673 	p->rt.on_list		= 0;
4674 
4675 #ifdef CONFIG_SCHED_CLASS_EXT
4676 	init_scx_entity(&p->scx);
4677 #endif
4678 
4679 #ifdef CONFIG_PREEMPT_NOTIFIERS
4680 	INIT_HLIST_HEAD(&p->preempt_notifiers);
4681 #endif
4682 
4683 #ifdef CONFIG_COMPACTION
4684 	p->capture_control = NULL;
4685 #endif
4686 	init_numa_balancing(clone_flags, p);
4687 	p->wake_entry.u_flags = CSD_TYPE_TTWU;
4688 	p->migration_pending = NULL;
4689 	init_sched_mm(p);
4690 }
4691 
4692 DEFINE_STATIC_KEY_FALSE(sched_numa_balancing);
4693 
4694 #ifdef CONFIG_NUMA_BALANCING
4695 
4696 int sysctl_numa_balancing_mode;
4697 
4698 static void __set_numabalancing_state(bool enabled)
4699 {
4700 	if (enabled)
4701 		static_branch_enable(&sched_numa_balancing);
4702 	else
4703 		static_branch_disable(&sched_numa_balancing);
4704 }
4705 
4706 void set_numabalancing_state(bool enabled)
4707 {
4708 	if (enabled)
4709 		sysctl_numa_balancing_mode = NUMA_BALANCING_NORMAL;
4710 	else
4711 		sysctl_numa_balancing_mode = NUMA_BALANCING_DISABLED;
4712 	__set_numabalancing_state(enabled);
4713 }
4714 
4715 #ifdef CONFIG_SYSCTL
4716 static void reset_memory_tiering(void)
4717 {
4718 	struct pglist_data *pgdat;
4719 
4720 	for_each_online_pgdat(pgdat) {
4721 		pgdat->nbp_threshold = 0;
4722 		pgdat->nbp_th_nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE);
4723 		pgdat->nbp_th_start = jiffies_to_msecs(jiffies);
4724 	}
4725 }
4726 
4727 static int sysctl_numa_balancing(const struct ctl_table *table, int write,
4728 			  void *buffer, size_t *lenp, loff_t *ppos)
4729 {
4730 	struct ctl_table t;
4731 	int err;
4732 	int state = sysctl_numa_balancing_mode;
4733 
4734 	if (write && !capable(CAP_SYS_ADMIN))
4735 		return -EPERM;
4736 
4737 	t = *table;
4738 	t.data = &state;
4739 	err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
4740 	if (err < 0)
4741 		return err;
4742 	if (write) {
4743 		if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) &&
4744 		    (state & NUMA_BALANCING_MEMORY_TIERING))
4745 			reset_memory_tiering();
4746 		sysctl_numa_balancing_mode = state;
4747 		__set_numabalancing_state(state);
4748 	}
4749 	return err;
4750 }
4751 #endif /* CONFIG_SYSCTL */
4752 #endif /* CONFIG_NUMA_BALANCING */
4753 
4754 #ifdef CONFIG_SCHEDSTATS
4755 
4756 DEFINE_STATIC_KEY_FALSE(sched_schedstats);
4757 
4758 static void set_schedstats(bool enabled)
4759 {
4760 	if (enabled)
4761 		static_branch_enable(&sched_schedstats);
4762 	else
4763 		static_branch_disable(&sched_schedstats);
4764 }
4765 
4766 void force_schedstat_enabled(void)
4767 {
4768 	if (!schedstat_enabled()) {
4769 		pr_info("kernel profiling enabled schedstats, disable via kernel.sched_schedstats.\n");
4770 		static_branch_enable(&sched_schedstats);
4771 	}
4772 }
4773 
4774 static int __init setup_schedstats(char *str)
4775 {
4776 	int ret = 0;
4777 	if (!str)
4778 		goto out;
4779 
4780 	if (!strcmp(str, "enable")) {
4781 		set_schedstats(true);
4782 		ret = 1;
4783 	} else if (!strcmp(str, "disable")) {
4784 		set_schedstats(false);
4785 		ret = 1;
4786 	}
4787 out:
4788 	if (!ret)
4789 		pr_warn("Unable to parse schedstats=\n");
4790 
4791 	return ret;
4792 }
4793 __setup("schedstats=", setup_schedstats);
4794 
4795 #ifdef CONFIG_SYSCTL
4796 static int sysctl_schedstats(const struct ctl_table *table, int write, void *buffer,
4797 		size_t *lenp, loff_t *ppos)
4798 {
4799 	struct ctl_table t;
4800 	int err;
4801 	int state = static_branch_likely(&sched_schedstats);
4802 
4803 	if (write && !capable(CAP_SYS_ADMIN))
4804 		return -EPERM;
4805 
4806 	t = *table;
4807 	t.data = &state;
4808 	err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
4809 	if (err < 0)
4810 		return err;
4811 	if (write)
4812 		set_schedstats(state);
4813 	return err;
4814 }
4815 #endif /* CONFIG_SYSCTL */
4816 #endif /* CONFIG_SCHEDSTATS */
4817 
4818 #ifdef CONFIG_SYSCTL
4819 static const struct ctl_table sched_core_sysctls[] = {
4820 #ifdef CONFIG_SCHEDSTATS
4821 	{
4822 		.procname       = "sched_schedstats",
4823 		.data           = NULL,
4824 		.maxlen         = sizeof(unsigned int),
4825 		.mode           = 0644,
4826 		.proc_handler   = sysctl_schedstats,
4827 		.extra1         = SYSCTL_ZERO,
4828 		.extra2         = SYSCTL_ONE,
4829 	},
4830 #endif /* CONFIG_SCHEDSTATS */
4831 #ifdef CONFIG_UCLAMP_TASK
4832 	{
4833 		.procname       = "sched_util_clamp_min",
4834 		.data           = &sysctl_sched_uclamp_util_min,
4835 		.maxlen         = sizeof(unsigned int),
4836 		.mode           = 0644,
4837 		.proc_handler   = sysctl_sched_uclamp_handler,
4838 	},
4839 	{
4840 		.procname       = "sched_util_clamp_max",
4841 		.data           = &sysctl_sched_uclamp_util_max,
4842 		.maxlen         = sizeof(unsigned int),
4843 		.mode           = 0644,
4844 		.proc_handler   = sysctl_sched_uclamp_handler,
4845 	},
4846 	{
4847 		.procname       = "sched_util_clamp_min_rt_default",
4848 		.data           = &sysctl_sched_uclamp_util_min_rt_default,
4849 		.maxlen         = sizeof(unsigned int),
4850 		.mode           = 0644,
4851 		.proc_handler   = sysctl_sched_uclamp_handler,
4852 	},
4853 #endif /* CONFIG_UCLAMP_TASK */
4854 #ifdef CONFIG_NUMA_BALANCING
4855 	{
4856 		.procname	= "numa_balancing",
4857 		.data		= NULL, /* filled in by handler */
4858 		.maxlen		= sizeof(unsigned int),
4859 		.mode		= 0644,
4860 		.proc_handler	= sysctl_numa_balancing,
4861 		.extra1		= SYSCTL_ZERO,
4862 		.extra2		= SYSCTL_FOUR,
4863 	},
4864 #endif /* CONFIG_NUMA_BALANCING */
4865 };
4866 static int __init sched_core_sysctl_init(void)
4867 {
4868 	register_sysctl_init("kernel", sched_core_sysctls);
4869 	return 0;
4870 }
4871 late_initcall(sched_core_sysctl_init);
4872 #endif /* CONFIG_SYSCTL */
4873 
4874 /*
4875  * fork()/clone()-time setup:
4876  */
4877 int sched_fork(u64 clone_flags, struct task_struct *p)
4878 {
4879 	__sched_fork(clone_flags, p);
4880 	/*
4881 	 * We mark the process as NEW here. This guarantees that
4882 	 * nobody will actually run it, and a signal or other external
4883 	 * event cannot wake it up and insert it on the runqueue either.
4884 	 */
4885 	p->__state = TASK_NEW;
4886 
4887 	/*
4888 	 * Make sure we do not leak PI boosting priority to the child.
4889 	 */
4890 	p->prio = current->normal_prio;
4891 
4892 	uclamp_fork(p);
4893 
4894 	/*
4895 	 * Revert to default priority/policy on fork if requested.
4896 	 */
4897 	if (unlikely(p->sched_reset_on_fork)) {
4898 		if (task_has_dl_policy(p) || task_has_rt_policy(p)) {
4899 			p->policy = SCHED_NORMAL;
4900 			p->static_prio = NICE_TO_PRIO(0);
4901 			p->rt_priority = 0;
4902 			p->timer_slack_ns = p->default_timer_slack_ns;
4903 		} else if (PRIO_TO_NICE(p->static_prio) < 0)
4904 			p->static_prio = NICE_TO_PRIO(0);
4905 
4906 		p->prio = p->normal_prio = p->static_prio;
4907 		set_load_weight(p, false);
4908 		p->se.custom_slice = 0;
4909 		p->se.slice = sysctl_sched_base_slice;
4910 
4911 		/*
4912 		 * We don't need the reset flag anymore after the fork. It has
4913 		 * fulfilled its duty:
4914 		 */
4915 		p->sched_reset_on_fork = 0;
4916 	}
4917 
4918 	if (dl_prio(p->prio))
4919 		return -EAGAIN;
4920 
4921 	scx_pre_fork(p);
4922 
4923 	if (rt_prio(p->prio)) {
4924 		p->sched_class = &rt_sched_class;
4925 #ifdef CONFIG_SCHED_CLASS_EXT
4926 	} else if (task_should_scx(p->policy)) {
4927 		p->sched_class = &ext_sched_class;
4928 #endif
4929 	} else {
4930 		p->sched_class = &fair_sched_class;
4931 	}
4932 
4933 	init_entity_runnable_average(&p->se);
4934 
4935 
4936 #ifdef CONFIG_SCHED_INFO
4937 	if (likely(sched_info_on()))
4938 		memset(&p->sched_info, 0, sizeof(p->sched_info));
4939 #endif
4940 	p->on_cpu = 0;
4941 	init_task_preempt_count(p);
4942 	plist_node_init(&p->pushable_tasks, MAX_PRIO);
4943 	RB_CLEAR_NODE(&p->pushable_dl_tasks);
4944 
4945 	return 0;
4946 }
4947 
4948 int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs)
4949 {
4950 	unsigned long flags;
4951 
4952 	/*
4953 	 * Because we're not yet on the pid-hash, p->pi_lock isn't strictly
4954 	 * required yet, but lockdep gets upset if rules are violated.
4955 	 */
4956 	raw_spin_lock_irqsave(&p->pi_lock, flags);
4957 #ifdef CONFIG_CGROUP_SCHED
4958 	if (1) {
4959 		struct task_group *tg;
4960 		tg = container_of(kargs->cset->subsys[cpu_cgrp_id],
4961 				  struct task_group, css);
4962 		tg = autogroup_task_group(p, tg);
4963 		p->sched_task_group = tg;
4964 	}
4965 #endif
4966 	/*
4967 	 * We're setting the CPU for the first time, we don't migrate,
4968 	 * so use __set_task_cpu().
4969 	 */
4970 	__set_task_cpu(p, smp_processor_id());
4971 	if (p->sched_class->task_fork)
4972 		p->sched_class->task_fork(p);
4973 	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
4974 
4975 	return scx_fork(p, kargs);
4976 }
4977 
4978 void sched_cancel_fork(struct task_struct *p)
4979 {
4980 	scx_cancel_fork(p);
4981 }
4982 
4983 static void sched_mm_cid_fork(struct task_struct *t);
4984 
4985 void sched_post_fork(struct task_struct *p)
4986 {
4987 	sched_mm_cid_fork(p);
4988 	uclamp_post_fork(p);
4989 	scx_post_fork(p);
4990 }
4991 
4992 u64 to_ratio(u64 period, u64 runtime)
4993 {
4994 	if (runtime == RUNTIME_INF)
4995 		return BW_UNIT;
4996 
4997 	/*
4998 	 * Doing this here saves a lot of checks in all
4999 	 * the calling paths, and returning zero seems
5000 	 * safe for them anyway.
5001 	 */
5002 	if (period == 0)
5003 		return 0;
5004 
5005 	return div64_u64(runtime << BW_SHIFT, period);
5006 }
5007 
5008 /*
5009  * wake_up_new_task - wake up a newly created task for the first time.
5010  *
5011  * This function will do some initial scheduler statistics housekeeping
5012  * that must be done for every newly created context, then puts the task
5013  * on the runqueue and wakes it.
5014  */
5015 void wake_up_new_task(struct task_struct *p)
5016 {
5017 	struct rq_flags rf;
5018 	struct rq *rq;
5019 	int wake_flags = WF_FORK;
5020 
5021 	raw_spin_lock_irqsave(&p->pi_lock, rf.flags);
5022 	WRITE_ONCE(p->__state, TASK_RUNNING);
5023 	/*
5024 	 * Fork balancing, do it here and not earlier because:
5025 	 *  - cpus_ptr can change in the fork path
5026 	 *  - any previously selected CPU might disappear through hotplug
5027 	 *
5028 	 * Use __set_task_cpu() to avoid calling sched_class::migrate_task_rq,
5029 	 * as we're not fully set-up yet.
5030 	 */
5031 	p->recent_used_cpu = task_cpu(p);
5032 	__set_task_cpu(p, select_task_rq(p, task_cpu(p), &wake_flags));
5033 	rq = __task_rq_lock(p, &rf);
5034 	update_rq_clock(rq);
5035 	post_init_entity_util_avg(p);
5036 
5037 	activate_task(rq, p, ENQUEUE_NOCLOCK | ENQUEUE_INITIAL);
5038 	trace_sched_wakeup_new(p);
5039 	wakeup_preempt(rq, p, wake_flags);
5040 	if (p->sched_class->task_woken) {
5041 		/*
5042 		 * Nothing relies on rq->lock after this, so it's fine to
5043 		 * drop it.
5044 		 */
5045 		rq_unpin_lock(rq, &rf);
5046 		p->sched_class->task_woken(rq, p);
5047 		rq_repin_lock(rq, &rf);
5048 	}
5049 	task_rq_unlock(rq, p, &rf);
5050 }
5051 
5052 #ifdef CONFIG_PREEMPT_NOTIFIERS
5053 
5054 static DEFINE_STATIC_KEY_FALSE(preempt_notifier_key);
5055 
5056 void preempt_notifier_inc(void)
5057 {
5058 	static_branch_inc(&preempt_notifier_key);
5059 }
5060 EXPORT_SYMBOL_GPL(preempt_notifier_inc);
5061 
5062 void preempt_notifier_dec(void)
5063 {
5064 	static_branch_dec(&preempt_notifier_key);
5065 }
5066 EXPORT_SYMBOL_GPL(preempt_notifier_dec);
5067 
5068 /**
5069  * preempt_notifier_register - tell me when current is being preempted & rescheduled
5070  * @notifier: notifier struct to register
5071  */
5072 void preempt_notifier_register(struct preempt_notifier *notifier)
5073 {
5074 	if (!static_branch_unlikely(&preempt_notifier_key))
5075 		WARN(1, "registering preempt_notifier while notifiers disabled\n");
5076 
5077 	hlist_add_head(&notifier->link, &current->preempt_notifiers);
5078 }
5079 EXPORT_SYMBOL_GPL(preempt_notifier_register);
5080 
5081 /**
5082  * preempt_notifier_unregister - no longer interested in preemption notifications
5083  * @notifier: notifier struct to unregister
5084  *
5085  * This is *not* safe to call from within a preemption notifier.
5086  */
5087 void preempt_notifier_unregister(struct preempt_notifier *notifier)
5088 {
5089 	hlist_del(&notifier->link);
5090 }
5091 EXPORT_SYMBOL_GPL(preempt_notifier_unregister);
5092 
5093 static void __fire_sched_in_preempt_notifiers(struct task_struct *curr)
5094 {
5095 	struct preempt_notifier *notifier;
5096 
5097 	hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
5098 		notifier->ops->sched_in(notifier, raw_smp_processor_id());
5099 }
5100 
5101 static __always_inline void fire_sched_in_preempt_notifiers(struct task_struct *curr)
5102 {
5103 	if (static_branch_unlikely(&preempt_notifier_key))
5104 		__fire_sched_in_preempt_notifiers(curr);
5105 }
5106 
5107 static void
5108 __fire_sched_out_preempt_notifiers(struct task_struct *curr,
5109 				   struct task_struct *next)
5110 {
5111 	struct preempt_notifier *notifier;
5112 
5113 	hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
5114 		notifier->ops->sched_out(notifier, next);
5115 }
5116 
5117 static __always_inline void
5118 fire_sched_out_preempt_notifiers(struct task_struct *curr,
5119 				 struct task_struct *next)
5120 {
5121 	if (static_branch_unlikely(&preempt_notifier_key))
5122 		__fire_sched_out_preempt_notifiers(curr, next);
5123 }
5124 
5125 #else /* !CONFIG_PREEMPT_NOTIFIERS: */
5126 
5127 static inline void fire_sched_in_preempt_notifiers(struct task_struct *curr)
5128 {
5129 }
5130 
5131 static inline void
5132 fire_sched_out_preempt_notifiers(struct task_struct *curr,
5133 				 struct task_struct *next)
5134 {
5135 }
5136 
5137 #endif /* !CONFIG_PREEMPT_NOTIFIERS */
5138 
5139 static inline void prepare_task(struct task_struct *next)
5140 {
5141 	/*
5142 	 * Claim the task as running, we do this before switching to it
5143 	 * such that any running task will have this set.
5144 	 *
5145 	 * See the smp_load_acquire(&p->on_cpu) case in ttwu() and
5146 	 * its ordering comment.
5147 	 */
5148 	WRITE_ONCE(next->on_cpu, 1);
5149 }
5150 
5151 static inline void finish_task(struct task_struct *prev)
5152 {
5153 	/*
5154 	 * This must be the very last reference to @prev from this CPU. After
5155 	 * p->on_cpu is cleared, the task can be moved to a different CPU. We
5156 	 * must ensure this doesn't happen until the switch is completely
5157 	 * finished.
5158 	 *
5159 	 * In particular, the load of prev->state in finish_task_switch() must
5160 	 * happen before this.
5161 	 *
5162 	 * Pairs with the smp_cond_load_acquire() in try_to_wake_up().
5163 	 */
5164 	smp_store_release(&prev->on_cpu, 0);
5165 }
5166 
5167 /*
5168  * Only called from __schedule context
5169  *
5170  * There are some cases where we are going to re-do the action
5171  * that added the balance callbacks. We may not be in a state
5172  * where we can run them, so just zap them so they can be
5173  * properly re-added on the next time around. This is similar
5174  * handling to running the callbacks, except we just don't call
5175  * them.
5176  */
5177 static void zap_balance_callbacks(struct rq *rq)
5178 {
5179 	struct balance_callback *next, *head;
5180 	bool found = false;
5181 
5182 	lockdep_assert_rq_held(rq);
5183 
5184 	head = rq->balance_callback;
5185 	while (head) {
5186 		if (head == &balance_push_callback)
5187 			found = true;
5188 		next = head->next;
5189 		head->next = NULL;
5190 		head = next;
5191 	}
5192 	rq->balance_callback = found ? &balance_push_callback : NULL;
5193 }
5194 
5195 static void do_balance_callbacks(struct rq *rq, struct balance_callback *head)
5196 {
5197 	void (*func)(struct rq *rq);
5198 	struct balance_callback *next;
5199 
5200 	lockdep_assert_rq_held(rq);
5201 
5202 	while (head) {
5203 		func = head->func;
5204 		next = head->next;
5205 		head->next = NULL;
5206 		head = next;
5207 
5208 		func(rq);
5209 	}
5210 }
5211 
5212 static void balance_push(struct rq *rq);
5213 
5214 /*
5215  * balance_push_callback is a right abuse of the callback interface and plays
5216  * by significantly different rules.
5217  *
5218  * Where the normal balance_callback's purpose is to be ran in the same context
5219  * that queued it (only later, when it's safe to drop rq->lock again),
5220  * balance_push_callback is specifically targeted at __schedule().
5221  *
5222  * This abuse is tolerated because it places all the unlikely/odd cases behind
5223  * a single test, namely: rq->balance_callback == NULL.
5224  */
5225 struct balance_callback balance_push_callback = {
5226 	.next = NULL,
5227 	.func = balance_push,
5228 };
5229 
5230 static inline struct balance_callback *
5231 __splice_balance_callbacks(struct rq *rq, bool split)
5232 {
5233 	struct balance_callback *head = rq->balance_callback;
5234 
5235 	if (likely(!head))
5236 		return NULL;
5237 
5238 	lockdep_assert_rq_held(rq);
5239 	/*
5240 	 * Must not take balance_push_callback off the list when
5241 	 * splice_balance_callbacks() and balance_callbacks() are not
5242 	 * in the same rq->lock section.
5243 	 *
5244 	 * In that case it would be possible for __schedule() to interleave
5245 	 * and observe the list empty.
5246 	 */
5247 	if (split && head == &balance_push_callback)
5248 		head = NULL;
5249 	else
5250 		rq->balance_callback = NULL;
5251 
5252 	return head;
5253 }
5254 
5255 struct balance_callback *splice_balance_callbacks(struct rq *rq)
5256 {
5257 	return __splice_balance_callbacks(rq, true);
5258 }
5259 
5260 void __balance_callbacks(struct rq *rq, struct rq_flags *rf)
5261 {
5262 	if (rf)
5263 		rq_unpin_lock(rq, rf);
5264 	do_balance_callbacks(rq, __splice_balance_callbacks(rq, false));
5265 	if (rf)
5266 		rq_repin_lock(rq, rf);
5267 }
5268 
5269 void balance_callbacks(struct rq *rq, struct balance_callback *head)
5270 {
5271 	unsigned long flags;
5272 
5273 	if (unlikely(head)) {
5274 		raw_spin_rq_lock_irqsave(rq, flags);
5275 		do_balance_callbacks(rq, head);
5276 		raw_spin_rq_unlock_irqrestore(rq, flags);
5277 	}
5278 }
5279 
5280 static inline void
5281 prepare_lock_switch(struct rq *rq, struct task_struct *next, struct rq_flags *rf)
5282 	__releases(__rq_lockp(rq))
5283 	__acquires(__rq_lockp(this_rq()))
5284 {
5285 	/*
5286 	 * Since the runqueue lock will be released by the next
5287 	 * task (which is an invalid locking op but in the case
5288 	 * of the scheduler it's an obvious special-case), so we
5289 	 * do an early lockdep release here:
5290 	 */
5291 	rq_unpin_lock(rq, rf);
5292 	spin_release(&__rq_lockp(rq)->dep_map, _THIS_IP_);
5293 #ifdef CONFIG_DEBUG_SPINLOCK
5294 	/* this is a valid case when another task releases the spinlock */
5295 	rq_lockp(rq)->owner = next;
5296 #endif
5297 	/*
5298 	 * Model the rq reference switcheroo.
5299 	 */
5300 	__release(__rq_lockp(rq));
5301 	__acquire(__rq_lockp(this_rq()));
5302 }
5303 
5304 static inline void finish_lock_switch(struct rq *rq)
5305 	__releases(__rq_lockp(rq))
5306 {
5307 	/*
5308 	 * If we are tracking spinlock dependencies then we have to
5309 	 * fix up the runqueue lock - which gets 'carried over' from
5310 	 * prev into current:
5311 	 */
5312 	spin_acquire(&__rq_lockp(rq)->dep_map, 0, 0, _THIS_IP_);
5313 	__balance_callbacks(rq, NULL);
5314 	hrtick_schedule_exit(rq);
5315 	raw_spin_rq_unlock_irq(rq);
5316 }
5317 
5318 /*
5319  * NOP if the arch has not defined these:
5320  */
5321 
5322 #ifndef prepare_arch_switch
5323 # define prepare_arch_switch(next)	do { } while (0)
5324 #endif
5325 
5326 #ifndef finish_arch_post_lock_switch
5327 # define finish_arch_post_lock_switch()	do { } while (0)
5328 #endif
5329 
5330 static inline void kmap_local_sched_out(void)
5331 {
5332 #ifdef CONFIG_KMAP_LOCAL
5333 	if (unlikely(current->kmap_ctrl.idx))
5334 		__kmap_local_sched_out();
5335 #endif
5336 }
5337 
5338 static inline void kmap_local_sched_in(void)
5339 {
5340 #ifdef CONFIG_KMAP_LOCAL
5341 	if (unlikely(current->kmap_ctrl.idx))
5342 		__kmap_local_sched_in();
5343 #endif
5344 }
5345 
5346 /**
5347  * prepare_task_switch - prepare to switch tasks
5348  * @rq: the runqueue preparing to switch
5349  * @prev: the current task that is being switched out
5350  * @next: the task we are going to switch to.
5351  *
5352  * This is called with the rq lock held and interrupts off. It must
5353  * be paired with a subsequent finish_task_switch after the context
5354  * switch.
5355  *
5356  * prepare_task_switch sets up locking and calls architecture specific
5357  * hooks.
5358  */
5359 static inline void
5360 prepare_task_switch(struct rq *rq, struct task_struct *prev,
5361 		    struct task_struct *next)
5362 	__must_hold(__rq_lockp(rq))
5363 {
5364 	kcov_prepare_switch(prev);
5365 	sched_info_switch(rq, prev, next);
5366 	perf_event_task_sched_out(prev, next);
5367 	fire_sched_out_preempt_notifiers(prev, next);
5368 	kmap_local_sched_out();
5369 	prepare_task(next);
5370 	prepare_arch_switch(next);
5371 }
5372 
5373 /**
5374  * finish_task_switch - clean up after a task-switch
5375  * @prev: the thread we just switched away from.
5376  *
5377  * finish_task_switch must be called after the context switch, paired
5378  * with a prepare_task_switch call before the context switch.
5379  * finish_task_switch will reconcile locking set up by prepare_task_switch,
5380  * and do any other architecture-specific cleanup actions.
5381  *
5382  * Note that we may have delayed dropping an mm in context_switch(). If
5383  * so, we finish that here outside of the runqueue lock. (Doing it
5384  * with the lock held can cause deadlocks; see schedule() for
5385  * details.)
5386  *
5387  * The context switch have flipped the stack from under us and restored the
5388  * local variables which were saved when this task called schedule() in the
5389  * past. 'prev == current' is still correct but we need to recalculate this_rq
5390  * because prev may have moved to another CPU.
5391  */
5392 static struct rq *finish_task_switch(struct task_struct *prev)
5393 	__releases(__rq_lockp(this_rq()))
5394 {
5395 	struct rq *rq = this_rq();
5396 	struct mm_struct *mm = rq->prev_mm;
5397 	unsigned int prev_state;
5398 
5399 	/*
5400 	 * The previous task will have left us with a preempt_count of 2
5401 	 * because it left us after:
5402 	 *
5403 	 *	schedule()
5404 	 *	  preempt_disable();			// 1
5405 	 *	  __schedule()
5406 	 *	    raw_spin_lock_irq(&rq->lock)	// 2
5407 	 *
5408 	 * Also, see FORK_PREEMPT_COUNT.
5409 	 */
5410 	if (WARN_ONCE(preempt_count() != 2*PREEMPT_DISABLE_OFFSET,
5411 		      "corrupted preempt_count: %s/%d/0x%x\n",
5412 		      current->comm, current->pid, preempt_count()))
5413 		preempt_count_set(FORK_PREEMPT_COUNT);
5414 
5415 	rq->prev_mm = NULL;
5416 
5417 	/*
5418 	 * A task struct has one reference for the use as "current".
5419 	 * If a task dies, then it sets TASK_DEAD in tsk->state and calls
5420 	 * schedule one last time. The schedule call will never return, and
5421 	 * the scheduled task must drop that reference.
5422 	 *
5423 	 * We must observe prev->state before clearing prev->on_cpu (in
5424 	 * finish_task), otherwise a concurrent wakeup can get prev
5425 	 * running on another CPU and we could rave with its RUNNING -> DEAD
5426 	 * transition, resulting in a double drop.
5427 	 */
5428 	prev_state = READ_ONCE(prev->__state);
5429 	vtime_task_switch(prev);
5430 	perf_event_task_sched_in(prev, current);
5431 	finish_task(prev);
5432 	tick_nohz_task_switch();
5433 	finish_lock_switch(rq);
5434 	finish_arch_post_lock_switch();
5435 	kcov_finish_switch(current);
5436 	/*
5437 	 * kmap_local_sched_out() is invoked with rq::lock held and
5438 	 * interrupts disabled. There is no requirement for that, but the
5439 	 * sched out code does not have an interrupt enabled section.
5440 	 * Restoring the maps on sched in does not require interrupts being
5441 	 * disabled either.
5442 	 */
5443 	kmap_local_sched_in();
5444 
5445 	/*
5446 	 * Any cached block-layer timestamp (plug->cur_ktime) is stale now,
5447 	 * invalidate it.
5448 	 */
5449 	blk_plug_invalidate_ts();
5450 
5451 	fire_sched_in_preempt_notifiers(current);
5452 	/*
5453 	 * When switching through a kernel thread, the loop in
5454 	 * membarrier_{private,global}_expedited() may have observed that
5455 	 * kernel thread and not issued an IPI. It is therefore possible to
5456 	 * schedule between user->kernel->user threads without passing though
5457 	 * switch_mm(). Membarrier requires a barrier after storing to
5458 	 * rq->curr, before returning to userspace, so provide them here:
5459 	 *
5460 	 * - a full memory barrier for {PRIVATE,GLOBAL}_EXPEDITED, implicitly
5461 	 *   provided by mmdrop_lazy_tlb(),
5462 	 * - a sync_core for SYNC_CORE.
5463 	 */
5464 	if (mm) {
5465 		membarrier_mm_sync_core_before_usermode(mm);
5466 		mmdrop_lazy_tlb_sched(mm);
5467 	}
5468 
5469 	if (unlikely(prev_state == TASK_DEAD)) {
5470 		if (prev->sched_class->task_dead)
5471 			prev->sched_class->task_dead(prev);
5472 
5473 		/*
5474 		 * sched_ext_dead() must come before cgroup_task_dead() to
5475 		 * prevent cgroups from being removed while its member tasks are
5476 		 * visible to SCX schedulers.
5477 		 */
5478 		sched_ext_dead(prev);
5479 		cgroup_task_dead(prev);
5480 
5481 		/* Task is done with its stack. */
5482 		put_task_stack(prev);
5483 
5484 		put_task_struct_rcu_user(prev);
5485 	}
5486 
5487 	return rq;
5488 }
5489 
5490 /**
5491  * schedule_tail - first thing a freshly forked thread must call.
5492  * @prev: the thread we just switched away from.
5493  */
5494 asmlinkage __visible void schedule_tail(struct task_struct *prev)
5495 	__releases(__rq_lockp(this_rq()))
5496 {
5497 	/*
5498 	 * New tasks start with FORK_PREEMPT_COUNT, see there and
5499 	 * finish_task_switch() for details.
5500 	 *
5501 	 * finish_task_switch() will drop rq->lock() and lower preempt_count
5502 	 * and the preempt_enable() will end up enabling preemption (on
5503 	 * PREEMPT_COUNT kernels).
5504 	 */
5505 
5506 	finish_task_switch(prev);
5507 	/*
5508 	 * This is a special case: the newly created task has just
5509 	 * switched the context for the first time. It is returning from
5510 	 * schedule for the first time in this path.
5511 	 */
5512 	trace_sched_exit_tp(true);
5513 	preempt_enable();
5514 
5515 	if (current->set_child_tid)
5516 		put_user(task_pid_vnr(current), current->set_child_tid);
5517 
5518 	calculate_sigpending();
5519 }
5520 
5521 /*
5522  * context_switch - switch to the new MM and the new thread's register state.
5523  */
5524 static __always_inline struct rq *
5525 context_switch(struct rq *rq, struct task_struct *prev,
5526 	       struct task_struct *next, struct rq_flags *rf)
5527 	__releases(__rq_lockp(rq))
5528 {
5529 	prepare_task_switch(rq, prev, next);
5530 
5531 	/*
5532 	 * For paravirt, this is coupled with an exit in switch_to to
5533 	 * combine the page table reload and the switch backend into
5534 	 * one hypercall.
5535 	 */
5536 	arch_start_context_switch(prev);
5537 
5538 	/*
5539 	 * kernel -> kernel   lazy + transfer active
5540 	 *   user -> kernel   lazy + mmgrab_lazy_tlb() active
5541 	 *
5542 	 * kernel ->   user   switch + mmdrop_lazy_tlb() active
5543 	 *   user ->   user   switch
5544 	 */
5545 	if (!next->mm) {				// to kernel
5546 		enter_lazy_tlb(prev->active_mm, next);
5547 
5548 		next->active_mm = prev->active_mm;
5549 		if (prev->mm)				// from user
5550 			mmgrab_lazy_tlb(prev->active_mm);
5551 		else
5552 			prev->active_mm = NULL;
5553 	} else {					// to user
5554 		membarrier_switch_mm(rq, prev->active_mm, next->mm);
5555 		/*
5556 		 * sys_membarrier() requires an smp_mb() between setting
5557 		 * rq->curr / membarrier_switch_mm() and returning to userspace.
5558 		 *
5559 		 * The below provides this either through switch_mm(), or in
5560 		 * case 'prev->active_mm == next->mm' through
5561 		 * finish_task_switch()'s mmdrop().
5562 		 */
5563 		switch_mm_irqs_off(prev->active_mm, next->mm, next);
5564 		lru_gen_use_mm(next->mm);
5565 
5566 		if (!prev->mm) {			// from kernel
5567 			/* will mmdrop_lazy_tlb() in finish_task_switch(). */
5568 			rq->prev_mm = prev->active_mm;
5569 			prev->active_mm = NULL;
5570 		}
5571 	}
5572 
5573 	mm_cid_switch_to(prev, next);
5574 
5575 	/*
5576 	 * Tell rseq that the task was scheduled in. Must be after
5577 	 * switch_mm_cid() to get the TIF flag set.
5578 	 */
5579 	rseq_sched_switch_event(next);
5580 
5581 	prepare_lock_switch(rq, next, rf);
5582 
5583 	/* Here we just switch the register state and the stack. */
5584 	switch_to(prev, next, prev);
5585 	barrier();
5586 
5587 	return finish_task_switch(prev);
5588 }
5589 
5590 /*
5591  * nr_running and nr_context_switches:
5592  *
5593  * externally visible scheduler statistics: current number of runnable
5594  * threads, total number of context switches performed since bootup.
5595  */
5596 unsigned int nr_running(void)
5597 {
5598 	unsigned int i, sum = 0;
5599 
5600 	for_each_online_cpu(i)
5601 		sum += cpu_rq(i)->nr_running;
5602 
5603 	return sum;
5604 }
5605 
5606 /*
5607  * Check if only the current task is running on the CPU.
5608  *
5609  * Caution: this function does not check that the caller has disabled
5610  * preemption, thus the result might have a time-of-check-to-time-of-use
5611  * race.  The caller is responsible to use it correctly, for example:
5612  *
5613  * - from a non-preemptible section (of course)
5614  *
5615  * - from a thread that is bound to a single CPU
5616  *
5617  * - in a loop with very short iterations (e.g. a polling loop)
5618  */
5619 bool single_task_running(void)
5620 {
5621 	return raw_rq()->nr_running == 1;
5622 }
5623 EXPORT_SYMBOL(single_task_running);
5624 
5625 unsigned long long nr_context_switches_cpu(int cpu)
5626 {
5627 	return cpu_rq(cpu)->nr_switches;
5628 }
5629 
5630 unsigned long long nr_context_switches(void)
5631 {
5632 	int i;
5633 	unsigned long long sum = 0;
5634 
5635 	for_each_possible_cpu(i)
5636 		sum += cpu_rq(i)->nr_switches;
5637 
5638 	return sum;
5639 }
5640 
5641 /*
5642  * Return whether any tasks are accounted as I/O waiters on this CPU.
5643  * They may wake up on a different CPU.
5644  */
5645 bool cpu_has_iowait(int cpu)
5646 {
5647 	return atomic_read(&cpu_rq(cpu)->nr_iowait) != 0;
5648 }
5649 
5650 /*
5651  * IO-wait accounting, and how it's mostly bollocks (on SMP).
5652  *
5653  * The idea behind IO-wait account is to account the idle time that we could
5654  * have spend running if it were not for IO. That is, if we were to improve the
5655  * storage performance, we'd have a proportional reduction in IO-wait time.
5656  *
5657  * This all works nicely on UP, where, when a task blocks on IO, we account
5658  * idle time as IO-wait, because if the storage were faster, it could've been
5659  * running and we'd not be idle.
5660  *
5661  * This has been extended to SMP, by doing the same for each CPU. This however
5662  * is broken.
5663  *
5664  * Imagine for instance the case where two tasks block on one CPU, only the one
5665  * CPU will have IO-wait accounted, while the other has regular idle. Even
5666  * though, if the storage were faster, both could've ran at the same time,
5667  * utilising both CPUs.
5668  *
5669  * This means, that when looking globally, the current IO-wait accounting on
5670  * SMP is a lower bound, by reason of under accounting.
5671  *
5672  * Worse, since the numbers are provided per CPU, they are sometimes
5673  * interpreted per CPU, and that is nonsensical. A blocked task isn't strictly
5674  * associated with any one particular CPU, it can wake to another CPU than it
5675  * blocked on. This means the per CPU IO-wait number is meaningless.
5676  *
5677  * Task CPU affinities can make all that even more 'interesting'.
5678  */
5679 
5680 unsigned int nr_iowait(void)
5681 {
5682 	unsigned int i, sum = 0;
5683 
5684 	for_each_possible_cpu(i)
5685 		sum += atomic_read(&cpu_rq(i)->nr_iowait);
5686 
5687 	return sum;
5688 }
5689 
5690 /*
5691  * sched_exec - execve() is a valuable balancing opportunity, because at
5692  * this point the task has the smallest effective memory and cache footprint.
5693  */
5694 void sched_exec(void)
5695 {
5696 	struct task_struct *p = current;
5697 	struct migration_arg arg;
5698 	int dest_cpu;
5699 
5700 	scoped_guard (raw_spinlock_irqsave, &p->pi_lock) {
5701 		dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), WF_EXEC);
5702 		if (dest_cpu == smp_processor_id())
5703 			return;
5704 
5705 		if (unlikely(!cpu_active(dest_cpu)))
5706 			return;
5707 
5708 		arg = (struct migration_arg){ p, dest_cpu };
5709 	}
5710 	stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg);
5711 }
5712 
5713 DEFINE_PER_CPU(struct kernel_stat, kstat);
5714 DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat) = {
5715 #ifdef CONFIG_NO_HZ_COMMON
5716 	.idle_sleeptime_seq = SEQCNT_ZERO(kernel_cpustat.idle_sleeptime_seq)
5717 #endif
5718 };
5719 
5720 EXPORT_PER_CPU_SYMBOL(kstat);
5721 EXPORT_PER_CPU_SYMBOL(kernel_cpustat);
5722 
5723 /*
5724  * The function fair_sched_class.update_curr accesses the struct curr
5725  * and its field curr->exec_start; when called from task_sched_runtime(),
5726  * we observe a high rate of cache misses in practice.
5727  * Prefetching this data results in improved performance.
5728  */
5729 static inline void prefetch_curr_exec_start(struct task_struct *p)
5730 {
5731 	struct sched_entity *curr = task_rq(p)->cfs.curr;
5732 
5733 	prefetch(curr);
5734 	prefetch(&curr->exec_start);
5735 }
5736 
5737 /*
5738  * Return accounted runtime for the task.
5739  * In case the task is currently running, return the runtime plus current's
5740  * pending runtime that have not been accounted yet.
5741  */
5742 unsigned long long task_sched_runtime(struct task_struct *p)
5743 {
5744 	struct rq_flags rf;
5745 	struct rq *rq;
5746 	u64 ns;
5747 
5748 #ifdef CONFIG_64BIT
5749 	/*
5750 	 * 64-bit doesn't need locks to atomically read a 64-bit value.
5751 	 * So we have a optimization chance when the task's delta_exec is 0.
5752 	 * Reading ->on_cpu is racy, but this is OK.
5753 	 *
5754 	 * If we race with it leaving CPU, we'll take a lock. So we're correct.
5755 	 * If we race with it entering CPU, unaccounted time is 0. This is
5756 	 * indistinguishable from the read occurring a few cycles earlier.
5757 	 * If we see ->on_cpu without ->on_rq, the task is leaving, and has
5758 	 * been accounted, so we're correct here as well.
5759 	 */
5760 	if (!p->on_cpu || !task_on_rq_queued(p))
5761 		return p->se.sum_exec_runtime;
5762 #endif
5763 
5764 	rq = task_rq_lock(p, &rf);
5765 	/*
5766 	 * Must be ->curr _and_ ->on_rq.  If dequeued, we would
5767 	 * project cycles that may never be accounted to this
5768 	 * thread, breaking clock_gettime().
5769 	 */
5770 	if (task_current_donor(rq, p) && task_on_rq_queued(p)) {
5771 		prefetch_curr_exec_start(p);
5772 		update_rq_clock(rq);
5773 		p->sched_class->update_curr(rq);
5774 	}
5775 	ns = p->se.sum_exec_runtime;
5776 	task_rq_unlock(rq, p, &rf);
5777 
5778 	return ns;
5779 }
5780 
5781 static u64 cpu_resched_latency(struct rq *rq)
5782 {
5783 	int latency_warn_ms = READ_ONCE(sysctl_resched_latency_warn_ms);
5784 	u64 resched_latency, now = rq_clock(rq);
5785 	static bool warned_once;
5786 
5787 	if (sysctl_resched_latency_warn_once && warned_once)
5788 		return 0;
5789 
5790 	if (!need_resched() || !latency_warn_ms)
5791 		return 0;
5792 
5793 	if (system_state == SYSTEM_BOOTING)
5794 		return 0;
5795 
5796 	if (!rq->last_seen_need_resched_ns) {
5797 		rq->last_seen_need_resched_ns = now;
5798 		rq->ticks_without_resched = 0;
5799 		return 0;
5800 	}
5801 
5802 	rq->ticks_without_resched++;
5803 	resched_latency = now - rq->last_seen_need_resched_ns;
5804 	if (resched_latency <= latency_warn_ms * NSEC_PER_MSEC)
5805 		return 0;
5806 
5807 	warned_once = true;
5808 
5809 	return resched_latency;
5810 }
5811 
5812 static int __init setup_resched_latency_warn_ms(char *str)
5813 {
5814 	long val;
5815 
5816 	if ((kstrtol(str, 0, &val))) {
5817 		pr_warn("Unable to set resched_latency_warn_ms\n");
5818 		return 1;
5819 	}
5820 
5821 	sysctl_resched_latency_warn_ms = val;
5822 	return 1;
5823 }
5824 __setup("resched_latency_warn_ms=", setup_resched_latency_warn_ms);
5825 
5826 /*
5827  * This function gets called by the timer code, with HZ frequency.
5828  * We call it with interrupts disabled.
5829  */
5830 void sched_tick(void)
5831 {
5832 	int cpu = smp_processor_id();
5833 	struct rq *rq = cpu_rq(cpu);
5834 	/* scheduler accounting goes to the donor task */
5835 	struct task_struct *curr, *donor;
5836 	struct rq_flags rf;
5837 	unsigned long hw_pressure;
5838 	u64 resched_latency;
5839 
5840 	if (!cpu_preferred(cpu))
5841 		sched_push_current_non_preferred_cpu(rq);
5842 
5843 	if (housekeeping_cpu(cpu, HK_TYPE_KERNEL_NOISE))
5844 		arch_scale_freq_tick();
5845 
5846 	sched_clock_tick();
5847 
5848 	rq_lock(rq, &rf);
5849 	curr = rq->curr;
5850 	donor = rq->donor;
5851 
5852 	psi_account_irqtime(rq, curr, NULL);
5853 
5854 	update_rq_clock(rq);
5855 	hw_pressure = arch_scale_hw_pressure(cpu_of(rq));
5856 	update_hw_load_avg(rq_clock_task(rq), rq, hw_pressure);
5857 
5858 	if (dynamic_preempt_lazy() && tif_test_bit(TIF_NEED_RESCHED_LAZY))
5859 		resched_curr(rq);
5860 
5861 	donor->sched_class->task_tick(rq, donor, 0);
5862 	if (sched_feat(LATENCY_WARN))
5863 		resched_latency = cpu_resched_latency(rq);
5864 	calc_global_load_tick(rq);
5865 	sched_core_tick(rq);
5866 	scx_tick(rq);
5867 
5868 	rq_unlock(rq, &rf);
5869 
5870 	if (sched_feat(LATENCY_WARN) && resched_latency)
5871 		resched_latency_warn(cpu, resched_latency);
5872 
5873 	perf_event_task_tick();
5874 
5875 	if (curr->flags & PF_WQ_WORKER)
5876 		wq_worker_tick(curr);
5877 
5878 	if (!scx_switched_all()) {
5879 		rq->idle_balance = idle_cpu(cpu);
5880 		sched_balance_trigger(rq);
5881 	}
5882 }
5883 
5884 #ifdef CONFIG_NO_HZ_FULL
5885 
5886 struct tick_work {
5887 	int			cpu;
5888 	atomic_t		state;
5889 	struct delayed_work	work;
5890 };
5891 /* Values for ->state, see diagram below. */
5892 #define TICK_SCHED_REMOTE_OFFLINE	0
5893 #define TICK_SCHED_REMOTE_OFFLINING	1
5894 #define TICK_SCHED_REMOTE_RUNNING	2
5895 
5896 /*
5897  * State diagram for ->state:
5898  *
5899  *
5900  *          TICK_SCHED_REMOTE_OFFLINE
5901  *                    |   ^
5902  *                    |   |
5903  *                    |   | sched_tick_remote()
5904  *                    |   |
5905  *                    |   |
5906  *                    +--TICK_SCHED_REMOTE_OFFLINING
5907  *                    |   ^
5908  *                    |   |
5909  * sched_tick_start() |   | sched_tick_stop()
5910  *                    |   |
5911  *                    V   |
5912  *          TICK_SCHED_REMOTE_RUNNING
5913  *
5914  *
5915  * Other transitions get WARN_ON_ONCE(), except that sched_tick_remote()
5916  * and sched_tick_start() are happy to leave the state in RUNNING.
5917  */
5918 
5919 static struct tick_work __percpu *tick_work_cpu;
5920 
5921 static void sched_tick_remote(struct work_struct *work)
5922 {
5923 	struct delayed_work *dwork = to_delayed_work(work);
5924 	struct tick_work *twork = container_of(dwork, struct tick_work, work);
5925 	int cpu = twork->cpu;
5926 	struct rq *rq = cpu_rq(cpu);
5927 	int os;
5928 
5929 	/*
5930 	 * Handle the tick only if it appears the remote CPU is running in full
5931 	 * dynticks mode. The check is racy by nature, but missing a tick or
5932 	 * having one too much is no big deal because the scheduler tick updates
5933 	 * statistics and checks timeslices in a time-independent way, regardless
5934 	 * of when exactly it is running.
5935 	 */
5936 	if (tick_nohz_tick_stopped_cpu(cpu)) {
5937 		guard(rq_lock_irq)(rq);
5938 		struct task_struct *curr = rq->curr;
5939 
5940 		if (cpu_online(cpu)) {
5941 			/*
5942 			 * Since this is a remote tick for full dynticks mode,
5943 			 * we are always sure that there is no proxy (only a
5944 			 * single task is running).
5945 			 */
5946 			WARN_ON_ONCE(rq->curr != rq->donor);
5947 			update_rq_clock(rq);
5948 
5949 			if (!is_idle_task(curr)) {
5950 				/*
5951 				 * Make sure the next tick runs within a
5952 				 * reasonable amount of time.
5953 				 */
5954 				u64 delta = rq_clock_task(rq) - curr->se.exec_start;
5955 				WARN_ON_ONCE(delta > (u64)NSEC_PER_SEC * 30);
5956 			}
5957 			curr->sched_class->task_tick(rq, curr, 0);
5958 
5959 			calc_load_nohz_remote(rq);
5960 		}
5961 	}
5962 
5963 	/*
5964 	 * Run the remote tick once per second (1Hz). This arbitrary
5965 	 * frequency is large enough to avoid overload but short enough
5966 	 * to keep scheduler internal stats reasonably up to date.  But
5967 	 * first update state to reflect hotplug activity if required.
5968 	 */
5969 	os = atomic_fetch_add_unless(&twork->state, -1, TICK_SCHED_REMOTE_RUNNING);
5970 	WARN_ON_ONCE(os == TICK_SCHED_REMOTE_OFFLINE);
5971 	if (os == TICK_SCHED_REMOTE_RUNNING)
5972 		queue_delayed_work(system_dfl_wq, dwork, HZ);
5973 }
5974 
5975 static void sched_tick_start(int cpu)
5976 {
5977 	int os;
5978 	struct tick_work *twork;
5979 
5980 	if (housekeeping_cpu(cpu, HK_TYPE_KERNEL_NOISE))
5981 		return;
5982 
5983 	WARN_ON_ONCE(!tick_work_cpu);
5984 
5985 	twork = per_cpu_ptr(tick_work_cpu, cpu);
5986 	os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_RUNNING);
5987 	WARN_ON_ONCE(os == TICK_SCHED_REMOTE_RUNNING);
5988 	if (os == TICK_SCHED_REMOTE_OFFLINE) {
5989 		twork->cpu = cpu;
5990 		INIT_DELAYED_WORK(&twork->work, sched_tick_remote);
5991 		queue_delayed_work(system_dfl_wq, &twork->work, HZ);
5992 	}
5993 }
5994 
5995 #ifdef CONFIG_HOTPLUG_CPU
5996 static void sched_tick_stop(int cpu)
5997 {
5998 	struct tick_work *twork;
5999 	int os;
6000 
6001 	if (housekeeping_cpu(cpu, HK_TYPE_KERNEL_NOISE))
6002 		return;
6003 
6004 	WARN_ON_ONCE(!tick_work_cpu);
6005 
6006 	twork = per_cpu_ptr(tick_work_cpu, cpu);
6007 	/* There cannot be competing actions, but don't rely on stop-machine. */
6008 	os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_OFFLINING);
6009 	WARN_ON_ONCE(os != TICK_SCHED_REMOTE_RUNNING);
6010 	/* Don't cancel, as this would mess up the state machine. */
6011 }
6012 #endif /* CONFIG_HOTPLUG_CPU */
6013 
6014 int __init sched_tick_offload_init(void)
6015 {
6016 	tick_work_cpu = alloc_percpu(struct tick_work);
6017 	BUG_ON(!tick_work_cpu);
6018 	return 0;
6019 }
6020 
6021 #else /* !CONFIG_NO_HZ_FULL: */
6022 static inline void sched_tick_start(int cpu) { }
6023 static inline void sched_tick_stop(int cpu) { }
6024 #endif /* !CONFIG_NO_HZ_FULL */
6025 
6026 #if defined(CONFIG_PREEMPTION) && (defined(CONFIG_DEBUG_PREEMPT) || \
6027 				defined(CONFIG_TRACE_PREEMPT_TOGGLE))
6028 /*
6029  * If the value passed in is equal to the current preempt count
6030  * then we just disabled preemption. Start timing the latency.
6031  */
6032 static inline void preempt_latency_start(int val)
6033 {
6034 	if (preempt_count() == val) {
6035 		unsigned long ip = get_lock_parent_ip();
6036 #ifdef CONFIG_DEBUG_PREEMPT
6037 		current->preempt_disable_ip = ip;
6038 #endif
6039 		trace_preempt_off(CALLER_ADDR0, ip);
6040 	}
6041 }
6042 
6043 void preempt_count_add(int val)
6044 {
6045 #ifdef CONFIG_DEBUG_PREEMPT
6046 	/*
6047 	 * Underflow?
6048 	 *
6049 	 * Cannot detect underflow based on the current preempt_count() value
6050 	 * if using HAS_SEPARATE_PREEMPT_RESCHED_BITS because preempt count takes all 32
6051 	 * bits.
6052 	 */
6053 	if (!IS_ENABLED(CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS) &&
6054 	    DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
6055 		return;
6056 #endif
6057 	__preempt_count_add(val);
6058 #ifdef CONFIG_DEBUG_PREEMPT
6059 	/*
6060 	 * Spinlock count overflowing soon?
6061 	 */
6062 	DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
6063 				PREEMPT_MASK - 10);
6064 #endif
6065 	preempt_latency_start(val);
6066 }
6067 EXPORT_SYMBOL(preempt_count_add);
6068 NOKPROBE_SYMBOL(preempt_count_add);
6069 
6070 /*
6071  * If the value passed in equals to the current preempt count
6072  * then we just enabled preemption. Stop timing the latency.
6073  */
6074 static inline void preempt_latency_stop(int val)
6075 {
6076 	if (preempt_count() == val)
6077 		trace_preempt_on(CALLER_ADDR0, get_lock_parent_ip());
6078 }
6079 
6080 void preempt_count_sub(int val)
6081 {
6082 #ifdef CONFIG_DEBUG_PREEMPT
6083 	/*
6084 	 * Underflow?
6085 	 */
6086 	unsigned int uval = val;
6087 	unsigned int pc = preempt_count();
6088 
6089 	if (DEBUG_LOCKS_WARN_ON(pc - uval > pc))
6090 		return;
6091 	/*
6092 	 * Is the spinlock portion underflowing?
6093 	 */
6094 	if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
6095 			!(preempt_count() & PREEMPT_MASK)))
6096 		return;
6097 #endif
6098 
6099 	preempt_latency_stop(val);
6100 	__preempt_count_sub(val);
6101 }
6102 EXPORT_SYMBOL(preempt_count_sub);
6103 NOKPROBE_SYMBOL(preempt_count_sub);
6104 
6105 #else
6106 static inline void preempt_latency_start(int val) { }
6107 static inline void preempt_latency_stop(int val) { }
6108 #endif
6109 
6110 static inline unsigned long get_preempt_disable_ip(struct task_struct *p)
6111 {
6112 #ifdef CONFIG_DEBUG_PREEMPT
6113 	return p->preempt_disable_ip;
6114 #else
6115 	return 0;
6116 #endif
6117 }
6118 
6119 /*
6120  * Print scheduling while atomic bug:
6121  */
6122 static noinline void __schedule_bug(struct task_struct *prev)
6123 {
6124 	/* Save this before calling printk(), since that will clobber it */
6125 	unsigned long preempt_disable_ip = get_preempt_disable_ip(current);
6126 
6127 	if (oops_in_progress)
6128 		return;
6129 
6130 	printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n",
6131 		prev->comm, prev->pid, preempt_count());
6132 
6133 	debug_show_held_locks(prev);
6134 	print_modules();
6135 	if (irqs_disabled())
6136 		print_irqtrace_events(prev);
6137 	if (IS_ENABLED(CONFIG_DEBUG_PREEMPT)) {
6138 		pr_err("Preemption disabled at:");
6139 		print_ip_sym(KERN_ERR, preempt_disable_ip);
6140 	}
6141 	check_panic_on_warn("scheduling while atomic");
6142 
6143 	dump_stack();
6144 	add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
6145 }
6146 
6147 /*
6148  * Various schedule()-time debugging checks and statistics:
6149  */
6150 static inline void schedule_debug(struct task_struct *prev, bool preempt)
6151 {
6152 #ifdef CONFIG_SCHED_STACK_END_CHECK
6153 	if (task_stack_end_corrupted(prev))
6154 		panic("corrupted stack end detected inside scheduler\n");
6155 
6156 	if (task_scs_end_corrupted(prev))
6157 		panic("corrupted shadow stack detected inside scheduler\n");
6158 #endif
6159 
6160 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
6161 	if (!preempt && READ_ONCE(prev->__state) && prev->non_block_count) {
6162 		printk(KERN_ERR "BUG: scheduling in a non-blocking section: %s/%d/%i\n",
6163 			prev->comm, prev->pid, prev->non_block_count);
6164 		dump_stack();
6165 		add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
6166 	}
6167 #endif
6168 
6169 	if (unlikely(in_atomic_preempt_off())) {
6170 		__schedule_bug(prev);
6171 		preempt_count_set(PREEMPT_DISABLED);
6172 	}
6173 	rcu_sleep_check();
6174 	WARN_ON_ONCE(ct_state() == CT_STATE_USER);
6175 
6176 	profile_hit(SCHED_PROFILING, __builtin_return_address(0));
6177 
6178 	schedstat_inc(this_rq()->sched_count);
6179 }
6180 
6181 static void prev_balance(struct rq *rq, struct rq_flags *rf)
6182 {
6183 	const struct sched_class *start_class = rq->donor->sched_class;
6184 	const struct sched_class *class;
6185 
6186 	/*
6187 	 * We must do the balancing pass before put_prev_task(), such
6188 	 * that when we release the rq->lock the task is in the same
6189 	 * state as before we took rq->lock.
6190 	 *
6191 	 * We can terminate the balance pass as soon as we know there is
6192 	 * a runnable task of @class priority or higher.
6193 	 */
6194 	for_active_class_range(class, start_class, &idle_sched_class) {
6195 		if (class->balance && class->balance(rq, rf))
6196 			break;
6197 	}
6198 }
6199 
6200 /*
6201  * Pick up the highest-prio task:
6202  */
6203 static inline struct task_struct *
6204 __pick_next_task(struct rq *rq, struct rq_flags *rf)
6205 	__must_hold(__rq_lockp(rq))
6206 {
6207 	const struct sched_class *class;
6208 	struct task_struct *p;
6209 
6210 	rq->dl_server = NULL;
6211 
6212 	if (scx_enabled())
6213 		goto restart;
6214 
6215 	/*
6216 	 * Optimization: we know that if all tasks are in the fair class we can
6217 	 * call that function directly, but only if the @prev task wasn't of a
6218 	 * higher scheduling class, because otherwise those lose the
6219 	 * opportunity to pull in more work from other CPUs.
6220 	 */
6221 	if (likely(!sched_class_above(rq->donor->sched_class, &fair_sched_class) &&
6222 		   rq->nr_running == rq->cfs.h_nr_queued)) {
6223 
6224 		p = pick_task_fair(rq, rf);
6225 		if (unlikely(p == RETRY_TASK))
6226 			goto restart;
6227 
6228 		/* Assume the next prioritized class is idle_sched_class */
6229 		if (!p)
6230 			p = pick_task_idle(rq, rf);
6231 
6232 		put_prev_set_next_task(rq, rq->donor, p);
6233 		return p;
6234 	}
6235 
6236 restart:
6237 	prev_balance(rq, rf);
6238 
6239 	for_each_active_class(class) {
6240 		p = class->pick_task(rq, rf);
6241 		if (unlikely(p == RETRY_TASK))
6242 			goto restart;
6243 		if (p) {
6244 			put_prev_set_next_task(rq, rq->donor, p);
6245 			return p;
6246 		}
6247 	}
6248 
6249 	BUG(); /* The idle class should always have a runnable task. */
6250 }
6251 
6252 #ifdef CONFIG_SCHED_CORE
6253 static inline bool is_task_rq_idle(struct task_struct *t)
6254 {
6255 	return (task_rq(t)->idle == t);
6256 }
6257 
6258 static inline bool cookie_equals(struct task_struct *a, unsigned long cookie)
6259 {
6260 	return is_task_rq_idle(a) || (a->core_cookie == cookie);
6261 }
6262 
6263 static inline bool cookie_match(struct task_struct *a, struct task_struct *b)
6264 {
6265 	if (is_task_rq_idle(a) || is_task_rq_idle(b))
6266 		return true;
6267 
6268 	return a->core_cookie == b->core_cookie;
6269 }
6270 
6271 /*
6272  * Careful; this can return RETRY_TASK, it does not include the retry-loop
6273  * itself due to the whole SMT pick retry thing below.
6274  */
6275 static inline struct task_struct *pick_task(struct rq *rq, struct rq_flags *rf)
6276 {
6277 	const struct sched_class *class;
6278 	struct task_struct *p;
6279 
6280 	rq->dl_server = NULL;
6281 
6282 	for_each_active_class(class) {
6283 		p = class->pick_task(rq, rf);
6284 		if (p)
6285 			return p;
6286 	}
6287 
6288 	BUG(); /* The idle class should always have a runnable task. */
6289 }
6290 
6291 extern void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_fi);
6292 
6293 static void queue_core_balance(struct rq *rq);
6294 
6295 static struct task_struct *
6296 pick_next_task(struct rq *rq, struct rq_flags *rf)
6297 	__must_hold(__rq_lockp(rq))
6298 {
6299 	struct task_struct *next, *p, *max;
6300 	bool fi_before = false;
6301 	bool core_clock_updated = (rq == rq->core);
6302 	unsigned long cookie;
6303 	int i, cpu, occ = 0;
6304 	struct rq *rq_i;
6305 	bool need_sync = false;
6306 
6307 	if (!sched_core_enabled(rq))
6308 		return __pick_next_task(rq, rf);
6309 
6310 	cpu = cpu_of(rq);
6311 
6312 	/* Stopper task is switching into idle, no need core-wide selection. */
6313 	if (cpu_is_offline(cpu)) {
6314 		/*
6315 		 * Reset core_pick so that we don't enter the fastpath when
6316 		 * coming online. core_pick would already be migrated to
6317 		 * another cpu during offline.
6318 		 */
6319 		rq->core_pick = NULL;
6320 		rq->core_dl_server = NULL;
6321 		return __pick_next_task(rq, rf);
6322 	}
6323 
6324 	rq->core->core_pick_in_flight++;
6325 
6326 	/*
6327 	 * If there were no {en,de}queues since we picked (IOW, the task
6328 	 * pointers are all still valid), and we haven't scheduled the last
6329 	 * pick yet, do so now.
6330 	 *
6331 	 * rq->core_pick can be NULL if no selection was made for a CPU because
6332 	 * it was either offline or went offline during a sibling's core-wide
6333 	 * selection. In this case, do a core-wide selection.
6334 	 */
6335 	if (rq->core->core_pick_seq == rq->core->core_task_seq &&
6336 	    rq->core_pick) {
6337 		next = rq->core_pick;
6338 		rq->dl_server = rq->core_dl_server;
6339 		rq->core_pick = NULL;
6340 		rq->core_dl_server = NULL;
6341 		goto out_set_next;
6342 	}
6343 
6344 	prev_balance(rq, rf);
6345 
6346 restart:
6347 	need_sync |= !!rq->core->core_cookie;
6348 
6349 	/* reset state */
6350 	rq->core->core_cookie = 0UL;
6351 	if (rq->core->core_forceidle_count) {
6352 		if (!core_clock_updated) {
6353 			update_rq_clock(rq->core);
6354 			core_clock_updated = true;
6355 		}
6356 		sched_core_account_forceidle(rq);
6357 		/* reset after accounting force idle */
6358 		rq->core->core_forceidle_start = 0;
6359 		rq->core->core_forceidle_count = 0;
6360 		rq->core->core_forceidle_occupation = 0;
6361 		need_sync = true;
6362 		fi_before = true;
6363 	}
6364 
6365 	/*
6366 	 * core->core_task_seq, core->core_pick_seq
6367 	 *
6368 	 * @task_seq guards the task state ({en,de}queues)
6369 	 * @pick_seq is the @task_seq we did a selection on
6370 	 *
6371 	 * Once a core-wide selection is committed, a non-NULL core_pick denotes
6372 	 * a pick which still needs to be consumed on this CPU.
6373 	 *
6374 	 * However, preemptions can cause multiple picks on the same task set.
6375 	 * 'Fix' this by also increasing @task_seq for every pick.
6376 	 */
6377 	rq->core->core_task_seq++;
6378 
6379 	/*
6380 	 * Optimize for common case where this CPU has no cookies
6381 	 * and there are no cookied tasks running on siblings.
6382 	 */
6383 	if (!need_sync) {
6384 		next = pick_task(rq, rf);
6385 		if (unlikely(next == RETRY_TASK)) {
6386 			/* rq lock may have been dropped, clocks invalidated */
6387 			core_clock_updated = false;
6388 			if (!(rq->clock_update_flags & RQCF_UPDATED))
6389 				update_rq_clock(rq);
6390 			goto restart;
6391 		}
6392 
6393 		if (!next->core_cookie) {
6394 			rq->core_pick = NULL;
6395 			rq->core_dl_server = NULL;
6396 			/*
6397 			 * For robustness, update the min_vruntime_fi for
6398 			 * unconstrained picks as well.
6399 			 */
6400 			WARN_ON_ONCE(fi_before);
6401 			task_vruntime_update(rq, next, false);
6402 			goto out_set_next;
6403 		}
6404 	}
6405 
6406 	/*
6407 	 * For each thread: do the regular task pick and find the max prio task
6408 	 * amongst them.
6409 	 *
6410 	 * Tie-break prio towards the current CPU
6411 	 */
6412 	max = NULL;
6413 	for_each_core_cpu_wrap(i, rq) {
6414 		rq_i = cpu_rq(i);
6415 
6416 		/*
6417 		 * Current cpu always has its clock updated on entrance to
6418 		 * pick_next_task(). If the current cpu is not the core,
6419 		 * the core may also have been updated above.
6420 		 */
6421 		if (i != cpu && (rq_i != rq->core || !core_clock_updated))
6422 			update_rq_clock(rq_i);
6423 
6424 		p = pick_task(rq_i, rf);
6425 		if (unlikely(p == RETRY_TASK)) {
6426 			/* rq lock may have been dropped, clocks invalidated */
6427 			core_clock_updated = false;
6428 			if (!(rq->clock_update_flags & RQCF_UPDATED))
6429 				update_rq_clock(rq);
6430 			goto restart;
6431 		}
6432 
6433 		rq_i->core_pick = p;
6434 		rq_i->core_dl_server = rq_i->dl_server;
6435 
6436 		if (!max || prio_less(max, p, fi_before))
6437 			max = p;
6438 	}
6439 
6440 	cookie = rq->core->core_cookie = max->core_cookie;
6441 
6442 	/*
6443 	 * For each thread: try and find a runnable task that matches @max or
6444 	 * force idle.
6445 	 */
6446 	for_each_core_cpu(i, rq) {
6447 		rq_i = cpu_rq(i);
6448 		p = rq_i->core_pick;
6449 
6450 		if (!cookie_equals(p, cookie)) {
6451 			p = NULL;
6452 			if (cookie)
6453 				p = sched_core_find(rq_i, cookie);
6454 			if (!p)
6455 				p = idle_sched_class.pick_task(rq_i, rf);
6456 		}
6457 
6458 		rq_i->core_pick = p;
6459 		rq_i->core_dl_server = NULL;
6460 
6461 		if (p == rq_i->idle) {
6462 			if (rq_i->nr_running) {
6463 				rq->core->core_forceidle_count++;
6464 				if (!fi_before)
6465 					rq->core->core_forceidle_seq++;
6466 			}
6467 		} else {
6468 			occ++;
6469 		}
6470 	}
6471 
6472 	if (schedstat_enabled() && rq->core->core_forceidle_count) {
6473 		rq->core->core_forceidle_start = rq_clock(rq->core);
6474 		rq->core->core_forceidle_occupation = occ;
6475 	}
6476 
6477 	rq->core->core_pick_seq = rq->core->core_task_seq;
6478 	next = rq->core_pick;
6479 
6480 	/* Something should have been selected for current CPU */
6481 	WARN_ON_ONCE(!next);
6482 
6483 	/*
6484 	 * Reschedule siblings
6485 	 *
6486 	 * NOTE: L1TF -- at this point we're no longer running the old task and
6487 	 * sending an IPI (below) ensures the sibling will no longer be running
6488 	 * their task. This ensures there is no inter-sibling overlap between
6489 	 * non-matching user state.
6490 	 */
6491 	for_each_core_cpu(i, rq) {
6492 		rq_i = cpu_rq(i);
6493 
6494 		/*
6495 		 * An online sibling might have gone offline before a task
6496 		 * could be picked for it, or it might be offline but later
6497 		 * happen to come online, but its too late and nothing was
6498 		 * picked for it.  That's Ok - it will pick tasks for itself,
6499 		 * so ignore it.
6500 		 */
6501 		if (!rq_i->core_pick)
6502 			continue;
6503 
6504 		/*
6505 		 * Update for new !FI->FI transitions, or if continuing to be in !FI:
6506 		 * fi_before     fi      update?
6507 		 *  0            0       1
6508 		 *  0            1       1
6509 		 *  1            0       1
6510 		 *  1            1       0
6511 		 */
6512 		if (!(fi_before && rq->core->core_forceidle_count))
6513 			task_vruntime_update(rq_i, rq_i->core_pick, !!rq->core->core_forceidle_count);
6514 
6515 		rq_i->core_pick->core_occupation = occ;
6516 
6517 		if (i == cpu) {
6518 			rq_i->core_pick = NULL;
6519 			rq_i->core_dl_server = NULL;
6520 			continue;
6521 		}
6522 
6523 		/* Did we break L1TF mitigation requirements? */
6524 		WARN_ON_ONCE(!cookie_match(next, rq_i->core_pick));
6525 
6526 		if (rq_i->curr == rq_i->core_pick) {
6527 			rq_i->core_pick = NULL;
6528 			rq_i->core_dl_server = NULL;
6529 			continue;
6530 		}
6531 
6532 		resched_curr(rq_i);
6533 	}
6534 
6535 out_set_next:
6536 	rq->core->core_pick_in_flight--;
6537 	put_prev_set_next_task(rq, rq->donor, next);
6538 	if (rq->core->core_forceidle_count && next == rq->idle)
6539 		queue_core_balance(rq);
6540 
6541 	return next;
6542 }
6543 
6544 static bool try_steal_cookie(int this, int that)
6545 {
6546 	struct rq *dst = cpu_rq(this), *src = cpu_rq(that);
6547 	struct task_struct *p;
6548 	unsigned long cookie;
6549 	bool success = false;
6550 
6551 	guard(irq)();
6552 	guard(double_rq_lock)(dst, src);
6553 
6554 	cookie = dst->core->core_cookie;
6555 	if (!cookie)
6556 		return false;
6557 
6558 	if (dst->curr != dst->idle)
6559 		return false;
6560 
6561 	p = sched_core_find(src, cookie);
6562 	if (!p)
6563 		return false;
6564 
6565 	do {
6566 		if (p == src->core_pick || p == src->curr || p == src->donor)
6567 			goto next;
6568 
6569 		if (task_is_blocked(p))
6570 			goto next;
6571 
6572 		if (!is_cpu_allowed(p, this))
6573 			goto next;
6574 
6575 		if (p->core_occupation > dst->idle->core_occupation)
6576 			goto next;
6577 		/*
6578 		 * sched_core_find() and sched_core_next() will ensure
6579 		 * that task @p is not throttled now, we also need to
6580 		 * check whether the runqueue of the destination CPU is
6581 		 * being throttled.
6582 		 */
6583 		if (sched_task_is_throttled(p, this))
6584 			goto next;
6585 
6586 		move_queued_task_locked(src, dst, p);
6587 		resched_curr(dst);
6588 
6589 		success = true;
6590 		break;
6591 
6592 next:
6593 		p = sched_core_next(p, cookie);
6594 	} while (p);
6595 
6596 	return success;
6597 }
6598 
6599 static bool steal_cookie_task(int cpu, struct sched_domain *sd)
6600 {
6601 	int i;
6602 
6603 	for_each_cpu_wrap(i, sched_domain_span(sd), cpu + 1) {
6604 		if (i == cpu)
6605 			continue;
6606 
6607 		if (need_resched())
6608 			break;
6609 
6610 		if (try_steal_cookie(cpu, i))
6611 			return true;
6612 	}
6613 
6614 	return false;
6615 }
6616 
6617 static void sched_core_balance(struct rq *rq)
6618 	__must_hold(__rq_lockp(rq))
6619 {
6620 	struct sched_domain *sd;
6621 	int cpu = cpu_of(rq);
6622 
6623 	guard(preempt)();
6624 	guard(rcu)();
6625 
6626 	raw_spin_rq_unlock_irq(rq);
6627 	for_each_domain(cpu, sd) {
6628 		if (need_resched())
6629 			break;
6630 
6631 		if (steal_cookie_task(cpu, sd))
6632 			break;
6633 	}
6634 	raw_spin_rq_lock_irq(rq);
6635 }
6636 
6637 static DEFINE_PER_CPU(struct balance_callback, core_balance_head);
6638 
6639 static void queue_core_balance(struct rq *rq)
6640 {
6641 	if (!sched_core_enabled(rq))
6642 		return;
6643 
6644 	if (!rq->core->core_cookie)
6645 		return;
6646 
6647 	if (!rq->nr_running) /* not forced idle */
6648 		return;
6649 
6650 	queue_balance_callback(rq, &per_cpu(core_balance_head, rq->cpu), sched_core_balance);
6651 }
6652 
6653 DEFINE_LOCK_GUARD_1(core_lock, int,
6654 		    sched_core_lock(*_T->lock, &_T->flags),
6655 		    sched_core_unlock(*_T->lock, &_T->flags),
6656 		    unsigned long flags)
6657 
6658 static void sched_core_cpu_starting(unsigned int cpu)
6659 {
6660 	const struct cpumask *smt_mask = cpu_smt_mask(cpu);
6661 	struct rq *rq = cpu_rq(cpu), *core_rq = NULL;
6662 	int t;
6663 
6664 	guard(core_lock)(&cpu);
6665 
6666 	WARN_ON_ONCE(rq->core != rq);
6667 
6668 	/* if we're the first, we'll be our own leader */
6669 	if (cpumask_weight(smt_mask) == 1)
6670 		return;
6671 
6672 	/* find the leader */
6673 	for_each_cpu(t, smt_mask) {
6674 		if (t == cpu)
6675 			continue;
6676 		rq = cpu_rq(t);
6677 		if (rq->core == rq) {
6678 			core_rq = rq;
6679 			break;
6680 		}
6681 	}
6682 
6683 	if (WARN_ON_ONCE(!core_rq)) /* whoopsie */
6684 		return;
6685 
6686 	/* install and validate core_rq */
6687 	for_each_cpu(t, smt_mask) {
6688 		rq = cpu_rq(t);
6689 
6690 		if (t == cpu)
6691 			rq->core = core_rq;
6692 
6693 		WARN_ON_ONCE(rq->core != core_rq);
6694 	}
6695 }
6696 
6697 static void sched_core_cpu_deactivate(unsigned int cpu)
6698 {
6699 	const struct cpumask *smt_mask = cpu_smt_mask(cpu);
6700 	struct rq *rq = cpu_rq(cpu), *core_rq = NULL;
6701 	int t;
6702 
6703 	guard(core_lock)(&cpu);
6704 
6705 	/* if we're the last man standing, nothing to do */
6706 	if (cpumask_weight(smt_mask) == 1) {
6707 		WARN_ON_ONCE(rq->core != rq);
6708 		return;
6709 	}
6710 
6711 	/* if we're not the leader, nothing to do */
6712 	if (rq->core != rq)
6713 		return;
6714 
6715 	/* find a new leader */
6716 	for_each_cpu(t, smt_mask) {
6717 		if (t == cpu)
6718 			continue;
6719 		core_rq = cpu_rq(t);
6720 		break;
6721 	}
6722 
6723 	if (WARN_ON_ONCE(!core_rq)) /* impossible */
6724 		return;
6725 
6726 	/* copy the shared state to the new leader */
6727 	core_rq->core_task_seq             = rq->core_task_seq;
6728 	core_rq->core_pick_seq             = rq->core_pick_seq;
6729 	core_rq->core_cookie               = rq->core_cookie;
6730 	core_rq->core_forceidle_count      = rq->core_forceidle_count;
6731 	core_rq->core_forceidle_seq        = rq->core_forceidle_seq;
6732 	core_rq->core_forceidle_occupation = rq->core_forceidle_occupation;
6733 
6734 	/*
6735 	 * A stale leftover would bias the count forever if this CPU later
6736 	 * returns as its own leader. Move, don't copy.
6737 	 */
6738 	core_rq->core_pick_in_flight       = rq->core_pick_in_flight;
6739 	rq->core_pick_in_flight            = 0;
6740 
6741 	/*
6742 	 * Accounting edge for forced idle is handled in pick_next_task().
6743 	 * Don't need another one here, since the hotplug thread shouldn't
6744 	 * have a cookie.
6745 	 */
6746 	core_rq->core_forceidle_start = 0;
6747 
6748 	/* install new leader */
6749 	for_each_cpu(t, smt_mask) {
6750 		rq = cpu_rq(t);
6751 		rq->core = core_rq;
6752 	}
6753 }
6754 
6755 static inline void sched_core_cpu_dying(unsigned int cpu)
6756 {
6757 	struct rq *rq = cpu_rq(cpu);
6758 
6759 	if (rq->core != rq)
6760 		rq->core = rq;
6761 }
6762 
6763 #else /* !CONFIG_SCHED_CORE: */
6764 
6765 static inline void sched_core_cpu_starting(unsigned int cpu) {}
6766 static inline void sched_core_cpu_deactivate(unsigned int cpu) {}
6767 static inline void sched_core_cpu_dying(unsigned int cpu) {}
6768 
6769 static struct task_struct *
6770 pick_next_task(struct rq *rq, struct rq_flags *rf)
6771 	__must_hold(__rq_lockp(rq))
6772 {
6773 	return __pick_next_task(rq, rf);
6774 }
6775 
6776 #endif /* !CONFIG_SCHED_CORE */
6777 
6778 /*
6779  * Constants for the sched_mode argument of __schedule().
6780  *
6781  * The mode argument allows RT enabled kernels to differentiate a
6782  * preemption from blocking on an 'sleeping' spin/rwlock.
6783  */
6784 #define SM_IDLE			(-1)
6785 #define SM_NONE			0
6786 #define SM_PREEMPT		1
6787 #define SM_RTLOCK_WAIT		2
6788 
6789 /*
6790  * Helper function for __schedule()
6791  *
6792  * Tries to deactivate the task, unless the should_block arg
6793  * is false or if a signal is pending. In the case a signal
6794  * is pending, marks the task's __state as RUNNING (and clear
6795  * blocked_on).
6796  */
6797 static bool try_to_block_task(struct rq *rq, struct task_struct *p,
6798 			      unsigned long *task_state_p, bool should_block)
6799 {
6800 	unsigned long task_state = *task_state_p;
6801 
6802 	WARN_ON_ONCE(p->is_blocked);
6803 
6804 	if (signal_pending_state(task_state, p)) {
6805 		WRITE_ONCE(p->__state, TASK_RUNNING);
6806 		*task_state_p = TASK_RUNNING;
6807 		clear_task_blocked_on(p, NULL);
6808 
6809 		return false;
6810 	}
6811 
6812 	p->is_blocked = 1;
6813 
6814 	if (p->in_iowait) {
6815 		p->iowait_cpu = cpu_of(rq);
6816 		atomic_inc(&rq->nr_iowait);
6817 		delayacct_blkio_start();
6818 	}
6819 
6820 	/*
6821 	 * We check should_block after signal_pending because we
6822 	 * will want to wake the task in that case. But if
6823 	 * should_block is false, its likely due to the task being
6824 	 * blocked on a mutex, and we want to keep it on the runqueue
6825 	 * to be selectable for proxy-execution.
6826 	 */
6827 	if (!should_block)
6828 		return false;
6829 
6830 	block_task(rq, p, task_state);
6831 	return true;
6832 }
6833 
6834 #ifdef CONFIG_SCHED_PROXY_EXEC
6835 static inline void proxy_set_task_cpu(struct task_struct *p, int cpu)
6836 {
6837 	unsigned int wake_cpu;
6838 
6839 	/*
6840 	 * Since we are enqueuing a blocked task on a cpu it may
6841 	 * not be able to run on, preserve wake_cpu when we
6842 	 * __set_task_cpu so we can return the task to where it
6843 	 * was previously runnable.
6844 	 */
6845 	wake_cpu = p->wake_cpu;
6846 	__set_task_cpu(p, cpu);
6847 	p->wake_cpu = wake_cpu;
6848 }
6849 
6850 static inline struct task_struct *proxy_resched_idle(struct rq *rq)
6851 {
6852 	put_prev_set_next_task(rq, rq->donor, rq->idle);
6853 	rq->next_class = &idle_sched_class;
6854 	rq_set_donor(rq, rq->idle);
6855 	set_tsk_need_resched(rq->idle);
6856 	return rq->idle;
6857 }
6858 
6859 static void proxy_deactivate(struct rq *rq, struct task_struct *donor)
6860 {
6861 	unsigned long state = READ_ONCE(donor->__state);
6862 
6863 	WARN_ON_ONCE(state == TASK_RUNNING);
6864 	WARN_ON_ONCE(donor->blocked_on);
6865 	/*
6866 	 * Because we got donor from pick_next_task(), it is *crucial*
6867 	 * that we call proxy_resched_idle() before we deactivate it.
6868 	 * As once we deactivate donor, donor->on_rq is set to zero,
6869 	 * which allows ttwu() to immediately try to wake the task on
6870 	 * another rq. So we cannot use *any* references to donor
6871 	 * after that point. So things like cfs_rq->curr or rq->donor
6872 	 * need to be changed from next *before* we deactivate.
6873 	 */
6874 	proxy_resched_idle(rq);
6875 	block_task(rq, donor, state);
6876 }
6877 
6878 /*
6879  * Remove a retained proxy donor before changing its scheduler ownership.
6880  * The caller holds p->pi_lock, so p cannot wake and migrate if block_task()
6881  * drops it from the runqueue. If DELAY_DEQUEUE keeps a blocked fair task
6882  * queued, switching_from_fair() completes the dequeue in the immediately
6883  * following sched_change_begin().
6884  */
6885 void sched_proxy_block_task(struct rq *rq, struct task_struct *p)
6886 {
6887 	unsigned long state = READ_ONCE(p->__state);
6888 
6889 	lockdep_assert_held(&p->pi_lock);
6890 	lockdep_assert_rq_held(rq);
6891 
6892 	if (!p->is_blocked || !task_on_rq_queued(p))
6893 		return;
6894 	if (WARN_ON_ONCE(state == TASK_RUNNING))
6895 		return;
6896 
6897 	if (task_current_donor(rq, p))
6898 		proxy_reset_donor(rq);
6899 
6900 	if (!p->se.sched_delayed)
6901 		block_task(rq, p, state);
6902 
6903 	WARN_ON_ONCE(task_on_rq_queued(p) && !p->se.sched_delayed);
6904 }
6905 
6906 static inline void proxy_release_rq_lock(struct rq *rq, struct rq_flags *rf)
6907 	__releases(__rq_lockp(rq))
6908 {
6909 	/*
6910 	 * The class scheduler may have queued a balance callback
6911 	 * from pick_next_task() called earlier.
6912 	 *
6913 	 * So here we have to zap callbacks before unlocking the rq
6914 	 * as another CPU may jump in and call sched_balance_rq
6915 	 * which can trip the warning in rq_pin_lock() if we
6916 	 * leave callbacks set.
6917 	 *
6918 	 * After we later reaquire the rq lock, we will force __schedule()
6919 	 * to pick_again, so the callbacks will get re-established.
6920 	 */
6921 	zap_balance_callbacks(rq);
6922 	rq_unpin_lock(rq, rf);
6923 	raw_spin_rq_unlock(rq);
6924 }
6925 
6926 static inline void proxy_reacquire_rq_lock(struct rq *rq, struct rq_flags *rf)
6927 	__acquires(__rq_lockp(rq))
6928 {
6929 	raw_spin_rq_lock(rq);
6930 	rq_repin_lock(rq, rf);
6931 	update_rq_clock(rq);
6932 }
6933 
6934 /*
6935  * If the blocked-on relationship crosses CPUs, migrate @p to the
6936  * owner's CPU.
6937  *
6938  * This is because we must respect the CPU affinity of execution
6939  * contexts (owner) but we can ignore affinity for scheduling
6940  * contexts (@p). So we have to move scheduling contexts towards
6941  * potential execution contexts.
6942  *
6943  * Note: The owner can disappear, but simply migrate to @target_cpu
6944  * and leave that CPU to sort things out.
6945  */
6946 static void proxy_migrate_task(struct rq *rq, struct rq_flags *rf,
6947 			       struct task_struct *p, int target_cpu)
6948 	__must_hold(__rq_lockp(rq))
6949 {
6950 	struct rq *target_rq = cpu_rq(target_cpu);
6951 	LIST_HEAD(migrate_list);
6952 
6953 	lockdep_assert_rq_held(rq);
6954 	/*
6955 	 * Since we are migrating a blocked donor, it could be rq->donor,
6956 	 * and we want to make sure there aren't any references from this
6957 	 * rq to it before we drop the lock. This avoids another cpu
6958 	 * jumping in and grabbing the rq lock and referencing rq->donor
6959 	 * or cfs_rq->curr, etc after we have migrated it to another cpu,
6960 	 * and before we pick_again in __schedule.
6961 	 *
6962 	 * So call proxy_resched_idle() to drop the rq->donor references
6963 	 * before we release the lock.
6964 	 */
6965 	proxy_resched_idle(rq);
6966 	for (; p; p = p->blocked_donor) {
6967 		WARN_ON(p == rq->curr);
6968 		deactivate_task(rq, p, DEQUEUE_NOCLOCK);
6969 		proxy_set_task_cpu(p, target_cpu);
6970 		/*
6971 		 * We can re-use se.group_node to migrate the thing,
6972 		 * because @p is deactivated (won't be balanced) and
6973 		 * we hold the rq_lock.
6974 		 */
6975 		list_add(&p->se.group_node, &migrate_list);
6976 	}
6977 	proxy_release_rq_lock(rq, rf);
6978 
6979 	__attach_tasks(target_rq, &migrate_list);
6980 
6981 	proxy_reacquire_rq_lock(rq, rf);
6982 }
6983 
6984 /*
6985  * Find runnable lock owner to proxy for mutex blocked donor
6986  *
6987  * Follow the blocked-on relation:
6988  *
6989  *                ,-> task
6990  *                |     | blocked-on
6991  *                |     v
6992  *  blocked_donor |   mutex
6993  *                |     | owner
6994  *                |     v
6995  *                `-- task
6996  *
6997  * and set the blocked_donor relation, this latter is used by the mutex
6998  * code to find which (blocked) task to hand-off to.
6999  *
7000  * Lock order:
7001  *
7002  *   p->pi_lock
7003  *     rq->lock
7004  *       mutex->wait_lock
7005  *         p->blocked_lock
7006  *
7007  * Returns the task that is going to be used as execution context (the one
7008  * that is actually going to be run on cpu_of(rq)).
7009  */
7010 static struct task_struct *
7011 find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
7012 	__must_hold(__rq_lockp(rq))
7013 {
7014 	struct task_struct *owner = NULL;
7015 	bool curr_in_chain = false;
7016 	int this_cpu = cpu_of(rq);
7017 	struct task_struct *p;
7018 	int owner_cpu;
7019 
7020 	/* Follow blocked_on chain. */
7021 	for (p = donor; p->is_blocked; p = owner) {
7022 		/* if its PROXY_WAKING, do return migration or run if current */
7023 		struct mutex *mutex = p->blocked_on;
7024 		if (!mutex) {
7025 			clear_task_blocked_on(p, mutex);
7026 			if (task_current(rq, p)) {
7027 				unblock_task(p);
7028 				return p;
7029 			}
7030 			goto deactivate;
7031 		}
7032 
7033 		/*
7034 		 * By taking mutex->wait_lock we hold off concurrent mutex_unlock()
7035 		 * and ensure @owner sticks around.
7036 		 */
7037 		guard(raw_spinlock)(&mutex->wait_lock);
7038 		guard(raw_spinlock)(&p->blocked_lock);
7039 
7040 		/* Check again that p is blocked with blocked_lock held */
7041 		if (mutex != __get_task_blocked_on(p)) {
7042 			/*
7043 			 * Something changed in the blocked_on chain and
7044 			 * we don't know if only at this level. So, let's
7045 			 * just bail out completely and let __schedule()
7046 			 * figure things out (pick_again loop).
7047 			 */
7048 			return NULL;
7049 		}
7050 
7051 		if (task_current(rq, p))
7052 			curr_in_chain = true;
7053 
7054 		owner = __mutex_owner(mutex);
7055 		if (!owner) {
7056 			/*
7057 			 * If there is no owner, either clear blocked_on
7058 			 * and return p (if it is current and safe to
7059 			 * just run on this rq), or return-migrate the task.
7060 			 */
7061 			__clear_task_blocked_on(p, NULL);
7062 			if (task_current(rq, p)) {
7063 				unblock_task(p);
7064 				return p;
7065 			}
7066 			goto deactivate;
7067 		}
7068 
7069 		if (!READ_ONCE(owner->on_rq) || owner->se.sched_delayed) {
7070 			/* XXX Don't handle blocked owners/delayed dequeue yet */
7071 			if (curr_in_chain)
7072 				goto resched_idle;
7073 			__clear_task_blocked_on(p, NULL);
7074 			goto deactivate;
7075 		}
7076 
7077 		owner_cpu = task_cpu(owner);
7078 		if (owner_cpu != this_cpu) {
7079 			/*
7080 			 * @owner can disappear, simply migrate to @owner_cpu
7081 			 * and leave that CPU to sort things out.
7082 			 */
7083 			if (curr_in_chain)
7084 				goto resched_idle;
7085 			goto migrate_task;
7086 		}
7087 
7088 		if (task_on_rq_migrating(owner)) {
7089 			/*
7090 			 * One of the chain of mutex owners is currently migrating to this
7091 			 * CPU, but has not yet been enqueued because we are holding the
7092 			 * rq lock. As a simple solution, just schedule rq->idle to give
7093 			 * the migration a chance to complete. Much like the migrate_task
7094 			 * case we should end up back in find_proxy_task(), this time
7095 			 * hopefully with all relevant tasks already enqueued.
7096 			 */
7097 			goto resched_idle;
7098 		}
7099 
7100 		/*
7101 		 * Its possible to race where after we check owner->on_rq
7102 		 * but before we check (owner_cpu != this_cpu) that the
7103 		 * task on another cpu was migrated back to this cpu. In
7104 		 * that case it could slip by our  checks. So double check
7105 		 * we are still on this cpu and not migrating. If we get
7106 		 * inconsistent results, try again.
7107 		 */
7108 		if (!task_on_rq_queued(owner) || task_cpu(owner) != this_cpu)
7109 			return NULL;
7110 
7111 		if (owner == p) {
7112 			/*
7113 			 * It's possible we interleave with mutex_unlock like:
7114 			 *
7115 			 *				lock(&rq->lock);
7116 			 *				  find_proxy_task()
7117 			 * mutex_unlock()
7118 			 *   lock(&wait_lock);
7119 			 *   donor(owner) = current->blocked_donor;
7120 			 *   unlock(&wait_lock);
7121 			 *
7122 			 *   wake_up_q();
7123 			 *     ...
7124 			 *       ttwu_runnable()
7125 			 *         __task_rq_lock()
7126 			 *				  lock(&wait_lock);
7127 			 *				  owner == p
7128 			 *
7129 			 * Which leaves us to finish the ttwu_runnable() and make it go.
7130 			 *
7131 			 * So schedule rq->idle so that ttwu_runnable() can get the rq
7132 			 * lock and mark owner as running.
7133 			 */
7134 			goto resched_idle;
7135 		}
7136 		/*
7137 		 * OK, now we're absolutely sure @owner is on this
7138 		 * rq, therefore holding @rq->lock is sufficient to
7139 		 * guarantee its existence, as per ttwu_remote().
7140 		 */
7141 		owner->blocked_donor = p;
7142 	}
7143 	WARN_ON_ONCE(owner && !owner->on_rq);
7144 
7145 	if (owner && !sched_cpu_cookie_match(rq, owner)) {
7146 		if (curr_in_chain)
7147 			return proxy_resched_idle(rq);
7148 		p = donor; /* Deactivate the donor, not the runnable owner */
7149 		clear_task_blocked_on(p, NULL);
7150 		goto deactivate;
7151 	}
7152 	return owner;
7153 
7154 resched_idle:
7155 	return proxy_resched_idle(rq);
7156 deactivate:
7157 	proxy_deactivate(rq, p);
7158 	return NULL;
7159 migrate_task:
7160 	proxy_migrate_task(rq, rf, p, owner_cpu);
7161 	return NULL;
7162 }
7163 #else /* SCHED_PROXY_EXEC */
7164 static struct task_struct *
7165 find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
7166 {
7167 	WARN_ONCE(1, "This should never be called in the !SCHED_PROXY_EXEC case\n");
7168 	return donor;
7169 }
7170 #endif /* SCHED_PROXY_EXEC */
7171 
7172 /*
7173  * __schedule() is the main scheduler function.
7174  *
7175  * The main means of driving the scheduler and thus entering this function are:
7176  *
7177  *   1. Explicit blocking: mutex, semaphore, waitqueue, etc.
7178  *
7179  *   2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return
7180  *      paths. For example, see arch/x86/entry_64.S.
7181  *
7182  *      To drive preemption between tasks, the scheduler sets the flag in timer
7183  *      interrupt handler sched_tick().
7184  *
7185  *   3. Wakeups don't really cause entry into schedule(). They add a
7186  *      task to the run-queue and that's it.
7187  *
7188  *      Now, if the new task added to the run-queue preempts the current
7189  *      task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets
7190  *      called on the nearest possible occasion:
7191  *
7192  *       - If the kernel is preemptible (CONFIG_PREEMPTION=y):
7193  *
7194  *         - in syscall or exception context, at the next outmost
7195  *           preempt_enable(). (this might be as soon as the wake_up()'s
7196  *           spin_unlock()!)
7197  *
7198  *         - in IRQ context, return from interrupt-handler to
7199  *           preemptible context
7200  *
7201  *       - If the kernel is not preemptible (CONFIG_PREEMPTION is not set)
7202  *         then at the next:
7203  *
7204  *          - cond_resched() call
7205  *          - explicit schedule() call
7206  *          - return from syscall or exception to user-space
7207  *          - return from interrupt-handler to user-space
7208  *
7209  * WARNING: must be called with preemption disabled!
7210  */
7211 static void __sched notrace __schedule(int sched_mode)
7212 {
7213 	struct task_struct *prev, *next;
7214 	/*
7215 	 * On PREEMPT_RT kernel, SM_RTLOCK_WAIT is noted
7216 	 * as a preemption by schedule_debug() and RCU.
7217 	 */
7218 	bool preempt = sched_mode > SM_NONE;
7219 	bool is_switch = false;
7220 	unsigned long *switch_count;
7221 	unsigned long prev_state;
7222 	struct rq_flags rf;
7223 	struct rq *rq;
7224 	int cpu;
7225 
7226 	/* Trace preemptions consistently with task switches */
7227 	trace_sched_entry_tp(sched_mode == SM_PREEMPT);
7228 
7229 	cpu = smp_processor_id();
7230 	rq = cpu_rq(cpu);
7231 	prev = rq->curr;
7232 
7233 	schedule_debug(prev, preempt);
7234 
7235 	klp_sched_try_switch(prev);
7236 
7237 	local_irq_disable();
7238 	rcu_note_context_switch(preempt);
7239 	migrate_disable_switch(rq, prev);
7240 
7241 	/*
7242 	 * Make sure that signal_pending_state()->signal_pending() below
7243 	 * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE)
7244 	 * done by the caller to avoid the race with signal_wake_up():
7245 	 *
7246 	 * __set_current_state(@state)		signal_wake_up()
7247 	 * schedule()				  set_tsk_thread_flag(p, TIF_SIGPENDING)
7248 	 *					  wake_up_state(p, state)
7249 	 *   LOCK rq->lock			    LOCK p->pi_state
7250 	 *   smp_mb__after_spinlock()		    smp_mb__after_spinlock()
7251 	 *     if (signal_pending_state())	    if (p->state & @state)
7252 	 *
7253 	 * Also, the membarrier system call requires a full memory barrier
7254 	 * after coming from user-space, before storing to rq->curr; this
7255 	 * barrier matches a full barrier in the proximity of the membarrier
7256 	 * system call exit.
7257 	 */
7258 	rq_lock(rq, &rf);
7259 	smp_mb__after_spinlock();
7260 
7261 	hrtick_schedule_enter(rq);
7262 
7263 	/* Promote REQ to ACT */
7264 	rq->clock_update_flags <<= 1;
7265 	update_rq_clock(rq);
7266 	rq->clock_update_flags = RQCF_UPDATED;
7267 
7268 	switch_count = &prev->nivcsw;
7269 
7270 	/* Task state changes only considers SM_PREEMPT as preemption */
7271 	preempt = sched_mode == SM_PREEMPT;
7272 
7273 	/*
7274 	 * We must load prev->state once (task_struct::state is volatile), such
7275 	 * that we form a control dependency vs deactivate_task() below.
7276 	 */
7277 	prev_state = READ_ONCE(prev->__state);
7278 	if (sched_mode == SM_IDLE) {
7279 		/* SCX must consult the BPF scheduler to tell if rq is empty */
7280 		if (!rq->nr_running && !scx_enabled()) {
7281 			next = prev;
7282 			rq->next_class = &idle_sched_class;
7283 			goto picked;
7284 		}
7285 	} else if (!preempt && prev_state) {
7286 		/*
7287 		 * Keep mutex-blocked tasks on the runqueue for proxy execution
7288 		 * only when their scheduling class allows it. Without proxy
7289 		 * execution, task_is_blocked() always returns false.
7290 		 */
7291 		try_to_block_task(rq, prev, &prev_state,
7292 				  !task_is_blocked(prev) || !scx_allow_proxy_exec(prev));
7293 		switch_count = &prev->nvcsw;
7294 	}
7295 
7296 pick_again:
7297 	assert_balance_callbacks_empty(rq);
7298 	next = pick_next_task(rq, &rf);
7299 	rq->next_class = next->sched_class;
7300 	if (sched_proxy_exec()) {
7301 		struct task_struct *prev_donor = rq->donor;
7302 
7303 		rq_set_donor(rq, next);
7304 		next->blocked_donor = NULL;
7305 		if (unlikely(next->is_blocked)) {
7306 			next = find_proxy_task(rq, next, &rf);
7307 			if (!next) {
7308 				zap_balance_callbacks(rq);
7309 				goto pick_again;
7310 			}
7311 			if (next == rq->idle) {
7312 				zap_balance_callbacks(rq);
7313 				scx_proxy_reenqueue_retry(rq, next);
7314 				goto keep_resched;
7315 			}
7316 		}
7317 		if (rq->donor == prev_donor && prev != next) {
7318 			struct task_struct *donor = rq->donor;
7319 			/*
7320 			 * When transitioning like:
7321 			 *
7322 			 *         prev         next
7323 			 * donor:    B            B
7324 			 * curr:     A          B or C
7325 			 *
7326 			 * then put_prev_set_next_task() will not have done
7327 			 * anything, since B == B. However, A might have
7328 			 * missed a RT/DL balance opportunity due to being
7329 			 * on_cpu.
7330 			 */
7331 			donor->sched_class->put_prev_task(rq, donor, donor);
7332 			donor->sched_class->set_next_task(rq, donor, SNT_PICK);
7333 		}
7334 		scx_proxy_donor_start(rq);
7335 		scx_proxy_reenqueue_retry(rq, next);
7336 	} else {
7337 		rq_set_donor(rq, next);
7338 	}
7339 
7340 picked:
7341 	clear_tsk_need_resched(prev);
7342 	clear_preempt_need_resched();
7343 keep_resched:
7344 	rq->last_seen_need_resched_ns = 0;
7345 
7346 	is_switch = prev != next;
7347 	if (likely(is_switch)) {
7348 		rq->nr_switches++;
7349 		/*
7350 		 * RCU users of rcu_dereference(rq->curr) may not see
7351 		 * changes to task_struct made by pick_next_task().
7352 		 */
7353 		RCU_INIT_POINTER(rq->curr, next);
7354 
7355 		/*
7356 		 * The membarrier system call requires each architecture
7357 		 * to have a full memory barrier after updating
7358 		 * rq->curr, before returning to user-space.
7359 		 *
7360 		 * Here are the schemes providing that barrier on the
7361 		 * various architectures:
7362 		 * - mm ? switch_mm() : mmdrop() for x86, s390, sparc, PowerPC,
7363 		 *   RISC-V.  switch_mm() relies on membarrier_arch_switch_mm()
7364 		 *   on PowerPC and on RISC-V.
7365 		 * - finish_lock_switch() for weakly-ordered
7366 		 *   architectures where spin_unlock is a full barrier,
7367 		 * - switch_to() for arm64 (weakly-ordered, spin_unlock
7368 		 *   is a RELEASE barrier),
7369 		 *
7370 		 * The barrier matches a full barrier in the proximity of
7371 		 * the membarrier system call entry.
7372 		 *
7373 		 * On RISC-V, this barrier pairing is also needed for the
7374 		 * SYNC_CORE command when switching between processes, cf.
7375 		 * the inline comments in membarrier_arch_switch_mm().
7376 		 */
7377 		++*switch_count;
7378 
7379 		psi_account_irqtime(rq, prev, next);
7380 		psi_sched_switch(prev, next, !task_on_rq_queued(prev) ||
7381 					     prev->se.sched_delayed);
7382 
7383 		trace_sched_switch(preempt, prev, next, prev_state);
7384 
7385 		/* Also unlocks the rq: */
7386 		rq = context_switch(rq, prev, next, &rf);
7387 	} else {
7388 		rq_unpin_lock(rq, &rf);
7389 		__balance_callbacks(rq, NULL);
7390 		hrtick_schedule_exit(rq);
7391 		raw_spin_rq_unlock_irq(rq);
7392 	}
7393 	trace_sched_exit_tp(is_switch);
7394 }
7395 
7396 void __noreturn do_task_dead(void)
7397 {
7398 	/* Causes final put_task_struct in finish_task_switch(): */
7399 	set_special_state(TASK_DEAD);
7400 
7401 	/* Tell freezer to ignore us: */
7402 	current->flags |= PF_NOFREEZE;
7403 
7404 	__schedule(SM_NONE);
7405 	BUG();
7406 
7407 	/* Avoid "noreturn function does return" - but don't continue if BUG() is a NOP: */
7408 	for (;;)
7409 		cpu_relax();
7410 }
7411 
7412 static inline void sched_submit_work(struct task_struct *tsk)
7413 {
7414 	static DEFINE_WAIT_OVERRIDE_MAP(sched_map, LD_WAIT_CONFIG);
7415 	unsigned int task_flags;
7416 
7417 	/*
7418 	 * Establish LD_WAIT_CONFIG context to ensure none of the code called
7419 	 * will use a blocking primitive -- which would lead to recursion.
7420 	 */
7421 	lock_map_acquire_try(&sched_map);
7422 
7423 	task_flags = tsk->flags;
7424 	/*
7425 	 * If a worker goes to sleep, notify and ask workqueue whether it
7426 	 * wants to wake up a task to maintain concurrency.
7427 	 */
7428 	if (task_flags & PF_WQ_WORKER)
7429 		wq_worker_sleeping(tsk);
7430 	else if (task_flags & PF_IO_WORKER)
7431 		io_wq_worker_sleeping(tsk);
7432 
7433 	/*
7434 	 * spinlock and rwlock must not flush block requests.  This will
7435 	 * deadlock if the callback attempts to acquire a lock which is
7436 	 * already acquired.
7437 	 */
7438 	WARN_ON_ONCE(current->__state & TASK_RTLOCK_WAIT);
7439 
7440 	/*
7441 	 * If we are going to sleep and we have plugged IO queued,
7442 	 * make sure to submit it to avoid deadlocks.
7443 	 */
7444 	blk_flush_plug(tsk->plug, true);
7445 
7446 	lock_map_release(&sched_map);
7447 }
7448 
7449 static void sched_update_worker(struct task_struct *tsk)
7450 {
7451 	if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER)) {
7452 		if (tsk->flags & PF_WQ_WORKER)
7453 			wq_worker_running(tsk);
7454 		else
7455 			io_wq_worker_running(tsk);
7456 	}
7457 }
7458 
7459 static __always_inline void __schedule_loop(int sched_mode)
7460 {
7461 	do {
7462 		preempt_disable();
7463 		__schedule(sched_mode);
7464 		sched_preempt_enable_no_resched();
7465 	} while (need_resched());
7466 }
7467 
7468 asmlinkage __visible void __sched schedule(void)
7469 {
7470 	struct task_struct *tsk = current;
7471 
7472 #ifdef CONFIG_RT_MUTEXES
7473 	lockdep_assert(!tsk->sched_rt_mutex);
7474 #endif
7475 
7476 	if (!task_is_running(tsk))
7477 		sched_submit_work(tsk);
7478 	__schedule_loop(SM_NONE);
7479 	sched_update_worker(tsk);
7480 }
7481 EXPORT_SYMBOL(schedule);
7482 
7483 /*
7484  * synchronize_rcu_tasks() makes sure that no task is stuck in preempted
7485  * state (have scheduled out non-voluntarily) by making sure that all
7486  * tasks have either left the run queue or have gone into user space.
7487  * As idle tasks do not do either, they must not ever be preempted
7488  * (schedule out non-voluntarily).
7489  *
7490  * schedule_idle() is similar to schedule_preempt_disable() except that it
7491  * never enables preemption because it does not call sched_submit_work().
7492  */
7493 void __sched schedule_idle(void)
7494 {
7495 	/*
7496 	 * As this skips calling sched_submit_work(), which the idle task does
7497 	 * regardless because that function is a NOP when the task is in a
7498 	 * TASK_RUNNING state, make sure this isn't used someplace that the
7499 	 * current task can be in any other state. Note, idle is always in the
7500 	 * TASK_RUNNING state.
7501 	 */
7502 	WARN_ON_ONCE(current->__state);
7503 	do {
7504 		__schedule(SM_IDLE);
7505 	} while (need_resched());
7506 }
7507 
7508 #if defined(CONFIG_CONTEXT_TRACKING_USER) && !defined(CONFIG_HAVE_CONTEXT_TRACKING_USER_OFFSTACK)
7509 asmlinkage __visible void __sched schedule_user(void)
7510 {
7511 	/*
7512 	 * If we come here after a random call to set_need_resched(),
7513 	 * or we have been woken up remotely but the IPI has not yet arrived,
7514 	 * we haven't yet exited the RCU idle mode. Do it here manually until
7515 	 * we find a better solution.
7516 	 *
7517 	 * NB: There are buggy callers of this function.  Ideally we
7518 	 * should warn if prev_state != CT_STATE_USER, but that will trigger
7519 	 * too frequently to make sense yet.
7520 	 */
7521 	enum ctx_state prev_state = exception_enter();
7522 	schedule();
7523 	exception_exit(prev_state);
7524 }
7525 #endif
7526 
7527 /**
7528  * schedule_preempt_disabled - called with preemption disabled
7529  *
7530  * Returns with preemption disabled. Note: preempt_count must be 1
7531  */
7532 void __sched schedule_preempt_disabled(void)
7533 {
7534 	sched_preempt_enable_no_resched();
7535 	schedule();
7536 	preempt_disable();
7537 }
7538 
7539 #ifdef CONFIG_PREEMPT_RT
7540 void __sched notrace schedule_rtlock(void)
7541 {
7542 	__schedule_loop(SM_RTLOCK_WAIT);
7543 }
7544 NOKPROBE_SYMBOL(schedule_rtlock);
7545 #endif
7546 
7547 static void __sched notrace preempt_schedule_common(void)
7548 {
7549 	do {
7550 		/*
7551 		 * Because the function tracer can trace preempt_count_sub()
7552 		 * and it also uses preempt_enable/disable_notrace(), if
7553 		 * NEED_RESCHED is set, the preempt_enable_notrace() called
7554 		 * by the function tracer will call this function again and
7555 		 * cause infinite recursion.
7556 		 *
7557 		 * Preemption must be disabled here before the function
7558 		 * tracer can trace. Break up preempt_disable() into two
7559 		 * calls. One to disable preemption without fear of being
7560 		 * traced. The other to still record the preemption latency,
7561 		 * which can also be traced by the function tracer.
7562 		 */
7563 		preempt_disable_notrace();
7564 		preempt_latency_start(1);
7565 		__schedule(SM_PREEMPT);
7566 		preempt_latency_stop(1);
7567 		preempt_enable_no_resched_notrace();
7568 
7569 		/*
7570 		 * Check again in case we missed a preemption opportunity
7571 		 * between schedule and now.
7572 		 */
7573 	} while (need_resched());
7574 }
7575 
7576 #ifdef CONFIG_PREEMPTION
7577 /*
7578  * This is the entry point to schedule() from in-kernel preemption
7579  * off of preempt_enable.
7580  */
7581 asmlinkage __visible void __sched notrace preempt_schedule(void)
7582 {
7583 	/*
7584 	 * If there is a non-zero preempt_count or interrupts are disabled,
7585 	 * we do not want to preempt the current task. Just return..
7586 	 */
7587 	if (likely(!preemptible()))
7588 		return;
7589 	preempt_schedule_common();
7590 }
7591 NOKPROBE_SYMBOL(preempt_schedule);
7592 EXPORT_SYMBOL(preempt_schedule);
7593 
7594 /**
7595  * preempt_schedule_notrace - preempt_schedule called by tracing
7596  *
7597  * The tracing infrastructure uses preempt_enable_notrace to prevent
7598  * recursion and tracing preempt enabling caused by the tracing
7599  * infrastructure itself. But as tracing can happen in areas coming
7600  * from userspace or just about to enter userspace, a preempt enable
7601  * can occur before user_exit() is called. This will cause the scheduler
7602  * to be called when the system is still in usermode.
7603  *
7604  * To prevent this, the preempt_enable_notrace will use this function
7605  * instead of preempt_schedule() to exit user context if needed before
7606  * calling the scheduler.
7607  */
7608 asmlinkage __visible void __sched notrace preempt_schedule_notrace(void)
7609 {
7610 	enum ctx_state prev_ctx;
7611 
7612 	if (likely(!preemptible()))
7613 		return;
7614 
7615 	do {
7616 		/*
7617 		 * Because the function tracer can trace preempt_count_sub()
7618 		 * and it also uses preempt_enable/disable_notrace(), if
7619 		 * NEED_RESCHED is set, the preempt_enable_notrace() called
7620 		 * by the function tracer will call this function again and
7621 		 * cause infinite recursion.
7622 		 *
7623 		 * Preemption must be disabled here before the function
7624 		 * tracer can trace. Break up preempt_disable() into two
7625 		 * calls. One to disable preemption without fear of being
7626 		 * traced. The other to still record the preemption latency,
7627 		 * which can also be traced by the function tracer.
7628 		 */
7629 		preempt_disable_notrace();
7630 		preempt_latency_start(1);
7631 		/*
7632 		 * Needs preempt disabled in case user_exit() is traced
7633 		 * and the tracer calls preempt_enable_notrace() causing
7634 		 * an infinite recursion.
7635 		 */
7636 		prev_ctx = exception_enter();
7637 		__schedule(SM_PREEMPT);
7638 		exception_exit(prev_ctx);
7639 
7640 		preempt_latency_stop(1);
7641 		preempt_enable_no_resched_notrace();
7642 	} while (need_resched());
7643 }
7644 EXPORT_SYMBOL_GPL(preempt_schedule_notrace);
7645 
7646 #endif /* CONFIG_PREEMPTION */
7647 
7648 /*
7649  * This is the entry point to schedule() from kernel preemption
7650  * off of IRQ context.
7651  * Note, that this is called and return with IRQs disabled. This will
7652  * protect us against recursive calling from IRQ contexts.
7653  */
7654 asmlinkage __visible void __sched preempt_schedule_irq(void)
7655 {
7656 	enum ctx_state prev_state;
7657 
7658 	/* Catch callers which need to be fixed */
7659 	BUG_ON(preempt_count() || !irqs_disabled());
7660 
7661 	prev_state = exception_enter();
7662 
7663 	do {
7664 		preempt_disable();
7665 		local_irq_enable();
7666 		__schedule(SM_PREEMPT);
7667 		local_irq_disable();
7668 		sched_preempt_enable_no_resched();
7669 	} while (need_resched());
7670 
7671 	exception_exit(prev_state);
7672 }
7673 
7674 int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flags,
7675 			  void *key)
7676 {
7677 	WARN_ON_ONCE(wake_flags & ~(WF_SYNC|WF_CURRENT_CPU));
7678 	return try_to_wake_up(curr->private, mode, wake_flags);
7679 }
7680 EXPORT_SYMBOL(default_wake_function);
7681 
7682 const struct sched_class *__setscheduler_class(int policy, int prio)
7683 {
7684 	if (dl_prio(prio))
7685 		return &dl_sched_class;
7686 
7687 	if (rt_prio(prio))
7688 		return &rt_sched_class;
7689 
7690 #ifdef CONFIG_SCHED_CLASS_EXT
7691 	if (task_should_scx(policy))
7692 		return &ext_sched_class;
7693 #endif
7694 
7695 	return &fair_sched_class;
7696 }
7697 
7698 #ifdef CONFIG_RT_MUTEXES
7699 
7700 /*
7701  * Would be more useful with typeof()/auto_type but they don't mix with
7702  * bit-fields. Since it's a local thing, use int. Keep the generic sounding
7703  * name such that if someone were to implement this function we get to compare
7704  * notes.
7705  */
7706 #define fetch_and_set(x, v) ({ int _x = (x); (x) = (v); _x; })
7707 
7708 void rt_mutex_pre_schedule(void)
7709 {
7710 	lockdep_assert(!fetch_and_set(current->sched_rt_mutex, 1));
7711 	sched_submit_work(current);
7712 }
7713 
7714 /*
7715  * Used within the futex syscall context, skips sched_submit_work() because none
7716  * its work will be done. Asserts ensure that it is indeed the case.
7717  */
7718 void rt_mutex_futex_pre_schedule(void)
7719 {
7720 	lockdep_assert(!(current->flags & (PF_WQ_WORKER | PF_IO_WORKER)));
7721 	lockdep_assert(!current->plug);
7722 	lockdep_assert(!fetch_and_set(current->sched_rt_mutex, 1));
7723 }
7724 
7725 void rt_mutex_schedule(void)
7726 {
7727 	lockdep_assert(current->sched_rt_mutex);
7728 	__schedule_loop(SM_NONE);
7729 }
7730 
7731 void rt_mutex_post_schedule(void)
7732 {
7733 	sched_update_worker(current);
7734 	lockdep_assert(fetch_and_set(current->sched_rt_mutex, 0));
7735 }
7736 
7737 void rt_mutex_futex_post_schedule(void)
7738 {
7739 	lockdep_assert(fetch_and_set(current->sched_rt_mutex, 0));
7740 }
7741 
7742 /*
7743  * rt_mutex_setprio - set the current priority of a task
7744  * @p: task to boost
7745  * @pi_task: donor task
7746  *
7747  * This function changes the 'effective' priority of a task. It does
7748  * not touch ->normal_prio like __setscheduler().
7749  *
7750  * Used by the rt_mutex code to implement priority inheritance
7751  * logic. Call site only calls if the priority of the task changed.
7752  */
7753 void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task)
7754 {
7755 	int prio, oldprio, queue_flag =
7756 		DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK;
7757 	const struct sched_class *prev_class, *next_class;
7758 	struct rq_flags rf;
7759 	struct rq *rq;
7760 
7761 	/* XXX used to be waiter->prio, not waiter->task->prio */
7762 	prio = __rt_effective_prio(pi_task, p->normal_prio);
7763 
7764 	/*
7765 	 * If nothing changed; bail early.
7766 	 */
7767 	if (p->pi_top_task == pi_task && prio == p->prio && !dl_prio(prio))
7768 		return;
7769 
7770 	rq = __task_rq_lock(p, &rf);
7771 	update_rq_clock(rq);
7772 	/*
7773 	 * Set under pi_lock && rq->lock, such that the value can be used under
7774 	 * either lock.
7775 	 *
7776 	 * Note that there is loads of tricky to make this pointer cache work
7777 	 * right. rt_mutex_slowunlock()+rt_mutex_postunlock() work together to
7778 	 * ensure a task is de-boosted (pi_task is set to NULL) before the
7779 	 * task is allowed to run again (and can exit). This ensures the pointer
7780 	 * points to a blocked task -- which guarantees the task is present.
7781 	 */
7782 	p->pi_top_task = pi_task;
7783 
7784 	/*
7785 	 * For FIFO/RR we only need to set prio, if that matches we're done.
7786 	 */
7787 	if (prio == p->prio && !dl_prio(prio))
7788 		goto out_unlock;
7789 
7790 	/*
7791 	 * Idle task boosting is a no-no in general. There is one
7792 	 * exception, when PREEMPT_RT and NOHZ is active:
7793 	 *
7794 	 * The idle task calls get_next_timer_interrupt() and holds
7795 	 * the timer wheel base->lock on the CPU and another CPU wants
7796 	 * to access the timer (probably to cancel it). We can safely
7797 	 * ignore the boosting request, as the idle CPU runs this code
7798 	 * with interrupts disabled and will complete the lock
7799 	 * protected section without being interrupted. So there is no
7800 	 * real need to boost.
7801 	 */
7802 	if (unlikely(p == rq->idle)) {
7803 		WARN_ON(p != rq->curr);
7804 		WARN_ON(p->pi_blocked_on);
7805 		goto out_unlock;
7806 	}
7807 
7808 	trace_sched_pi_setprio(p, pi_task);
7809 	oldprio = p->prio;
7810 
7811 	if (oldprio == prio && !dl_prio(prio))
7812 		queue_flag &= ~DEQUEUE_MOVE;
7813 
7814 	prev_class = p->sched_class;
7815 	next_class = __setscheduler_class(p->policy, prio);
7816 
7817 	if (prev_class != next_class)
7818 		queue_flag |= DEQUEUE_CLASS;
7819 
7820 	scoped_guard (sched_change, p, queue_flag) {
7821 		/*
7822 		 * Boosting condition are:
7823 		 * 1. -rt task is running and holds mutex A
7824 		 *      --> -dl task blocks on mutex A
7825 		 *
7826 		 * 2. -dl task is running and holds mutex A
7827 		 *      --> -dl task blocks on mutex A and could preempt the
7828 		 *          running task
7829 		 */
7830 		if (dl_prio(prio)) {
7831 			if (!dl_prio(p->normal_prio) ||
7832 			    (pi_task && dl_prio(pi_task->prio) &&
7833 			     dl_entity_preempt(&pi_task->dl, &p->dl))) {
7834 				p->dl.pi_se = pi_task->dl.pi_se;
7835 				scope->flags |= ENQUEUE_REPLENISH;
7836 			} else {
7837 				p->dl.pi_se = &p->dl;
7838 			}
7839 		} else if (rt_prio(prio)) {
7840 			if (dl_prio(oldprio))
7841 				p->dl.pi_se = &p->dl;
7842 			if (oldprio < prio)
7843 				scope->flags |= ENQUEUE_HEAD;
7844 		} else {
7845 			if (dl_prio(oldprio))
7846 				p->dl.pi_se = &p->dl;
7847 			if (rt_prio(oldprio))
7848 				p->rt.timeout = 0;
7849 		}
7850 
7851 		p->sched_class = next_class;
7852 		p->prio = prio;
7853 	}
7854 out_unlock:
7855 	/* Caller holds task_struct::pi_lock, IRQs are still disabled */
7856 
7857 	__balance_callbacks(rq, &rf);
7858 	__task_rq_unlock(rq, p, &rf);
7859 }
7860 #endif /* CONFIG_RT_MUTEXES */
7861 
7862 #if !defined(CONFIG_PREEMPTION)
7863 int __sched __cond_resched(void)
7864 {
7865 	if (should_resched(0) && !irqs_disabled()) {
7866 		preempt_schedule_common();
7867 		return 1;
7868 	}
7869 	/*
7870 	 * In PREEMPT_RCU kernels, ->rcu_read_lock_nesting tells the tick
7871 	 * whether the current CPU is in an RCU read-side critical section,
7872 	 * so the tick can report quiescent states even for CPUs looping
7873 	 * in kernel context.  In contrast, in non-preemptible kernels,
7874 	 * RCU readers leave no in-memory hints, which means that CPU-bound
7875 	 * processes executing in kernel context might never report an
7876 	 * RCU quiescent state.  Therefore, the following code causes
7877 	 * cond_resched() to report a quiescent state, but only when RCU
7878 	 * is in urgent need of one.
7879 	 * A third case, preemptible, but non-PREEMPT_RCU provides for
7880 	 * urgently needed quiescent states via rcu_flavor_sched_clock_irq().
7881 	 */
7882 #ifndef CONFIG_PREEMPT_RCU
7883 	rcu_all_qs();
7884 #endif
7885 	return 0;
7886 }
7887 EXPORT_SYMBOL(__cond_resched);
7888 #endif
7889 
7890 /*
7891  * __cond_resched_lock() - if a reschedule is pending, drop the given lock,
7892  * call schedule, and on return reacquire the lock.
7893  *
7894  * This works OK both with and without CONFIG_PREEMPTION. We do strange low-level
7895  * operations here to prevent schedule() from being called twice (once via
7896  * spin_unlock(), once by hand).
7897  */
7898 int __cond_resched_lock(spinlock_t *lock)
7899 {
7900 	int resched = should_resched(PREEMPT_LOCK_OFFSET);
7901 	int ret = 0;
7902 
7903 	lockdep_assert_held(lock);
7904 
7905 	if (spin_needbreak(lock) || resched) {
7906 		spin_unlock(lock);
7907 		if (!_cond_resched())
7908 			cpu_relax();
7909 		ret = 1;
7910 		spin_lock(lock);
7911 	}
7912 	return ret;
7913 }
7914 EXPORT_SYMBOL(__cond_resched_lock);
7915 
7916 int __cond_resched_rwlock_read(rwlock_t *lock)
7917 {
7918 	int resched = should_resched(PREEMPT_LOCK_OFFSET);
7919 	int ret = 0;
7920 
7921 	lockdep_assert_held_read(lock);
7922 
7923 	if (rwlock_needbreak(lock) || resched) {
7924 		read_unlock(lock);
7925 		if (!_cond_resched())
7926 			cpu_relax();
7927 		ret = 1;
7928 		read_lock(lock);
7929 	}
7930 	return ret;
7931 }
7932 EXPORT_SYMBOL(__cond_resched_rwlock_read);
7933 
7934 int __cond_resched_rwlock_write(rwlock_t *lock)
7935 {
7936 	int resched = should_resched(PREEMPT_LOCK_OFFSET);
7937 	int ret = 0;
7938 
7939 	lockdep_assert_held_write(lock);
7940 
7941 	if (rwlock_needbreak(lock) || resched) {
7942 		write_unlock(lock);
7943 		if (!_cond_resched())
7944 			cpu_relax();
7945 		ret = 1;
7946 		write_lock(lock);
7947 	}
7948 	return ret;
7949 }
7950 EXPORT_SYMBOL(__cond_resched_rwlock_write);
7951 
7952 #ifdef CONFIG_PREEMPT_DYNAMIC
7953 
7954 # ifdef CONFIG_GENERIC_IRQ_ENTRY
7955 #  include <linux/irq-entry-common.h>
7956 # endif
7957 
7958 /*
7959  * NONE:
7960  *   (unselectable)
7961  *
7962  * VOLUNTARY:
7963  *   (unselectable)
7964  *
7965  * FULL:
7966  *   dynamic_preempt_lazy       <- false
7967  *
7968  * LAZY:
7969  *   dynamic_preempt_lazy       <- true
7970  */
7971 
7972 enum {
7973 	preempt_dynamic_undefined = -1,
7974 	preempt_dynamic_full,
7975 	preempt_dynamic_lazy,
7976 };
7977 
7978 int preempt_dynamic_mode = preempt_dynamic_undefined;
7979 
7980 int sched_dynamic_mode(const char *str)
7981 {
7982 	if (!strcmp(str, "full"))
7983 		return preempt_dynamic_full;
7984 
7985 	if (!strcmp(str, "lazy"))
7986 		return preempt_dynamic_lazy;
7987 
7988 	return -EINVAL;
7989 }
7990 
7991 # define preempt_dynamic_key_enable(f)	static_key_enable(&sk_dynamic_##f.key)
7992 # define preempt_dynamic_key_disable(f)	static_key_disable(&sk_dynamic_##f.key)
7993 
7994 static DEFINE_MUTEX(sched_dynamic_mutex);
7995 
7996 static void __sched_dynamic_update(int mode)
7997 {
7998 	switch (mode) {
7999 	case preempt_dynamic_full:
8000 		preempt_dynamic_key_disable(preempt_lazy);
8001 		if (mode != preempt_dynamic_mode)
8002 			pr_info("Dynamic Preempt: full\n");
8003 		break;
8004 
8005 	case preempt_dynamic_lazy:
8006 		preempt_dynamic_key_enable(preempt_lazy);
8007 		if (mode != preempt_dynamic_mode)
8008 			pr_info("Dynamic Preempt: lazy\n");
8009 		break;
8010 	}
8011 
8012 	WRITE_ONCE(preempt_dynamic_mode, mode);
8013 }
8014 
8015 void sched_dynamic_update(int mode)
8016 {
8017 	mutex_lock(&sched_dynamic_mutex);
8018 	__sched_dynamic_update(mode);
8019 	mutex_unlock(&sched_dynamic_mutex);
8020 }
8021 
8022 static int __init setup_preempt_mode(char *str)
8023 {
8024 	int mode = sched_dynamic_mode(str);
8025 	if (mode < 0) {
8026 		pr_warn("Dynamic Preempt: unsupported mode: %s\n", str);
8027 		return 0;
8028 	}
8029 
8030 	sched_dynamic_update(mode);
8031 	return 1;
8032 }
8033 __setup("preempt=", setup_preempt_mode);
8034 
8035 static void __init preempt_dynamic_init(void)
8036 {
8037 	if (preempt_dynamic_mode == preempt_dynamic_undefined) {
8038 		if (IS_ENABLED(CONFIG_PREEMPT_LAZY)) {
8039 			sched_dynamic_update(preempt_dynamic_lazy);
8040 		} else {
8041 			/* Default static call setting, nothing to do */
8042 			WARN_ON_ONCE(!IS_ENABLED(CONFIG_PREEMPT));
8043 			preempt_dynamic_mode = preempt_dynamic_full;
8044 			pr_info("Dynamic Preempt: full\n");
8045 		}
8046 	}
8047 }
8048 
8049 # define PREEMPT_MODEL_ACCESSOR(mode)					\
8050 	bool preempt_model_##mode(void)					\
8051 	{								\
8052 		int mode = READ_ONCE(preempt_dynamic_mode);		\
8053 		WARN_ON_ONCE(mode == preempt_dynamic_undefined);	\
8054 		return mode == preempt_dynamic_##mode;			\
8055 	}								\
8056 	EXPORT_SYMBOL_GPL(preempt_model_##mode)
8057 
8058 PREEMPT_MODEL_ACCESSOR(full);
8059 PREEMPT_MODEL_ACCESSOR(lazy);
8060 
8061 #else /* !CONFIG_PREEMPT_DYNAMIC: */
8062 
8063 #define preempt_dynamic_mode -1
8064 
8065 static inline void preempt_dynamic_init(void) { }
8066 
8067 #endif /* CONFIG_PREEMPT_DYNAMIC */
8068 
8069 const char *preempt_modes[] = {
8070 	"full", "lazy", NULL,
8071 };
8072 
8073 const char *preempt_model_str(void)
8074 {
8075 	bool brace = IS_ENABLED(CONFIG_PREEMPT_RT) &&
8076 		(IS_ENABLED(CONFIG_PREEMPT_DYNAMIC) ||
8077 		 IS_ENABLED(CONFIG_PREEMPT_LAZY));
8078 	static char buf[128];
8079 
8080 	if (IS_ENABLED(CONFIG_PREEMPT_BUILD)) {
8081 		struct seq_buf s;
8082 
8083 		seq_buf_init(&s, buf, sizeof(buf));
8084 		seq_buf_puts(&s, "PREEMPT");
8085 
8086 		if (IS_ENABLED(CONFIG_PREEMPT_RT))
8087 			seq_buf_printf(&s, "%sRT%s",
8088 				       brace ? "_{" : "_",
8089 				       brace ? "," : "");
8090 
8091 		if (IS_ENABLED(CONFIG_PREEMPT_DYNAMIC)) {
8092 			seq_buf_printf(&s, "(%s)%s",
8093 				       preempt_dynamic_mode >= 0 ?
8094 				       preempt_modes[preempt_dynamic_mode] : "undef",
8095 				       brace ? "}" : "");
8096 			return seq_buf_str(&s);
8097 		}
8098 
8099 		if (IS_ENABLED(CONFIG_PREEMPT_LAZY)) {
8100 			seq_buf_printf(&s, "LAZY%s",
8101 				       brace ? "}" : "");
8102 			return seq_buf_str(&s);
8103 		}
8104 
8105 		return seq_buf_str(&s);
8106 	}
8107 
8108 	if (IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY_BUILD))
8109 		return "VOLUNTARY";
8110 
8111 	return "NONE";
8112 }
8113 
8114 int io_schedule_prepare(void)
8115 {
8116 	int old_iowait = current->in_iowait;
8117 
8118 	current->in_iowait = 1;
8119 	blk_flush_plug(current->plug, true);
8120 	return old_iowait;
8121 }
8122 
8123 void io_schedule_finish(int token)
8124 {
8125 	current->in_iowait = token;
8126 }
8127 
8128 /*
8129  * This task is about to go to sleep on IO. Increment rq->nr_iowait so
8130  * that process accounting knows that this is a task in IO wait state.
8131  */
8132 long __sched io_schedule_timeout(long timeout)
8133 {
8134 	int token;
8135 	long ret;
8136 
8137 	token = io_schedule_prepare();
8138 	ret = schedule_timeout(timeout);
8139 	io_schedule_finish(token);
8140 
8141 	return ret;
8142 }
8143 EXPORT_SYMBOL(io_schedule_timeout);
8144 
8145 void __sched io_schedule(void)
8146 {
8147 	int token;
8148 
8149 	token = io_schedule_prepare();
8150 	schedule();
8151 	io_schedule_finish(token);
8152 }
8153 EXPORT_SYMBOL(io_schedule);
8154 
8155 void sched_show_task(struct task_struct *p)
8156 {
8157 	unsigned long free;
8158 	int ppid;
8159 
8160 	if (!try_get_task_stack(p))
8161 		return;
8162 
8163 	pr_info("task:%-15.15s state:%c", p->comm, task_state_to_char(p));
8164 
8165 	if (task_is_running(p))
8166 		pr_cont("  running task    ");
8167 	free = stack_not_used(p);
8168 	ppid = 0;
8169 	rcu_read_lock();
8170 	if (pid_alive(p))
8171 		ppid = task_pid_nr(rcu_dereference(p->real_parent));
8172 	rcu_read_unlock();
8173 	pr_cont(" stack:%-5lu pid:%-5d tgid:%-5d ppid:%-6d task_flags:0x%04x flags:0x%08lx\n",
8174 		free, task_pid_nr(p), task_tgid_nr(p),
8175 		ppid, p->flags, read_task_thread_flags(p));
8176 
8177 	print_worker_info(KERN_INFO, p);
8178 	print_stop_info(KERN_INFO, p);
8179 	print_scx_info(KERN_INFO, p);
8180 	show_stack(p, NULL, KERN_INFO);
8181 	put_task_stack(p);
8182 }
8183 EXPORT_SYMBOL_GPL(sched_show_task);
8184 
8185 static inline bool
8186 state_filter_match(unsigned long state_filter, struct task_struct *p)
8187 {
8188 	unsigned int state = READ_ONCE(p->__state);
8189 
8190 	/* no filter, everything matches */
8191 	if (!state_filter)
8192 		return true;
8193 
8194 	/* filter, but doesn't match */
8195 	if (!(state & state_filter))
8196 		return false;
8197 
8198 	/*
8199 	 * When looking for TASK_UNINTERRUPTIBLE skip TASK_IDLE (allows
8200 	 * TASK_KILLABLE).
8201 	 */
8202 	if (state_filter == TASK_UNINTERRUPTIBLE && (state & TASK_NOLOAD))
8203 		return false;
8204 
8205 	return true;
8206 }
8207 
8208 
8209 void show_state_filter(unsigned int state_filter)
8210 {
8211 	struct task_struct *g, *p;
8212 
8213 	rcu_read_lock();
8214 	for_each_process_thread(g, p) {
8215 		/*
8216 		 * reset the NMI-timeout, listing all files on a slow
8217 		 * console might take a lot of time:
8218 		 * Also, reset softlockup watchdogs on all CPUs, because
8219 		 * another CPU might be blocked waiting for us to process
8220 		 * an IPI.
8221 		 */
8222 		touch_nmi_watchdog();
8223 		touch_all_softlockup_watchdogs();
8224 		if (state_filter_match(state_filter, p))
8225 			sched_show_task(p);
8226 	}
8227 
8228 	if (!state_filter)
8229 		sysrq_sched_debug_show();
8230 
8231 	rcu_read_unlock();
8232 	/*
8233 	 * Only show locks if all tasks are dumped:
8234 	 */
8235 	if (!state_filter)
8236 		debug_show_all_locks();
8237 }
8238 
8239 /**
8240  * init_idle - set up an idle thread for a given CPU
8241  * @idle: task in question
8242  * @cpu: CPU the idle task belongs to
8243  *
8244  * NOTE: this function does not set the idle thread's NEED_RESCHED
8245  * flag, to make booting more robust.
8246  */
8247 void __init init_idle(struct task_struct *idle, int cpu)
8248 {
8249 	struct affinity_context ac = (struct affinity_context) {
8250 		.new_mask  = cpumask_of(cpu),
8251 		.flags     = 0,
8252 	};
8253 	struct rq *rq = cpu_rq(cpu);
8254 	unsigned long flags;
8255 
8256 	raw_spin_lock_irqsave(&idle->pi_lock, flags);
8257 	raw_spin_rq_lock(rq);
8258 
8259 	idle->__state = TASK_RUNNING;
8260 	idle->se.exec_start = sched_clock();
8261 	/*
8262 	 * PF_KTHREAD should already be set at this point; regardless, make it
8263 	 * look like a proper per-CPU kthread.
8264 	 */
8265 	idle->flags |= PF_KTHREAD | PF_NO_SETAFFINITY;
8266 	kthread_set_per_cpu(idle, cpu);
8267 
8268 	/*
8269 	 * No validation and serialization required at boot time and for
8270 	 * setting up the idle tasks of not yet online CPUs.
8271 	 */
8272 	set_cpus_allowed_common(idle, &ac);
8273 	/*
8274 	 * We're having a chicken and egg problem, even though we are
8275 	 * holding rq->lock, the CPU isn't yet set to this CPU so the
8276 	 * lockdep check in task_group() will fail.
8277 	 *
8278 	 * Similar case to sched_fork(). / Alternatively we could
8279 	 * use task_rq_lock() here and obtain the other rq->lock.
8280 	 *
8281 	 * Silence PROVE_RCU
8282 	 */
8283 	rcu_read_lock();
8284 	__set_task_cpu(idle, cpu);
8285 	rcu_read_unlock();
8286 
8287 	rq->idle = idle;
8288 	rq_set_donor(rq, idle);
8289 	rcu_assign_pointer(rq->curr, idle);
8290 	idle->on_rq = TASK_ON_RQ_QUEUED;
8291 	idle->on_cpu = 1;
8292 	raw_spin_rq_unlock(rq);
8293 	raw_spin_unlock_irqrestore(&idle->pi_lock, flags);
8294 
8295 	/* Set the preempt count _outside_ the spinlocks! */
8296 	init_idle_preempt_count(idle, cpu);
8297 
8298 	/*
8299 	 * The idle tasks have their own, simple scheduling class:
8300 	 */
8301 	idle->sched_class = &idle_sched_class;
8302 	ftrace_graph_init_idle_task(idle, cpu);
8303 	vtime_init_idle(idle, cpu);
8304 	sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu);
8305 }
8306 
8307 int cpuset_cpumask_can_shrink(const struct cpumask *cur,
8308 			      const struct cpumask *trial)
8309 {
8310 	int ret = 1;
8311 
8312 	if (cpumask_empty(cur))
8313 		return ret;
8314 
8315 	ret = dl_cpuset_cpumask_can_shrink(cur, trial);
8316 
8317 	return ret;
8318 }
8319 
8320 int task_can_attach(struct task_struct *p)
8321 {
8322 	int ret = 0;
8323 
8324 	/*
8325 	 * Kthreads which disallow setaffinity shouldn't be moved
8326 	 * to a new cpuset; we don't want to change their CPU
8327 	 * affinity and isolating such threads by their set of
8328 	 * allowed nodes is unnecessary.  Thus, cpusets are not
8329 	 * applicable for such threads.  This prevents checking for
8330 	 * success of set_cpus_allowed_ptr() on all attached tasks
8331 	 * before cpus_mask may be changed.
8332 	 */
8333 	if (p->flags & PF_NO_SETAFFINITY)
8334 		ret = -EINVAL;
8335 
8336 	return ret;
8337 }
8338 
8339 bool sched_smp_initialized __read_mostly;
8340 
8341 #ifdef CONFIG_NUMA_BALANCING
8342 /* Migrate current task p to target_cpu */
8343 int migrate_task_to(struct task_struct *p, int target_cpu)
8344 {
8345 	struct migration_arg arg = { p, target_cpu };
8346 	int curr_cpu = task_cpu(p);
8347 
8348 	if (curr_cpu == target_cpu)
8349 		return 0;
8350 
8351 	if (!cpumask_test_cpu(target_cpu, p->cpus_ptr))
8352 		return -EINVAL;
8353 
8354 	/* TODO: This is not properly updating schedstats */
8355 
8356 	trace_sched_move_numa(p, curr_cpu, target_cpu);
8357 	return stop_one_cpu(curr_cpu, migration_cpu_stop, &arg);
8358 }
8359 
8360 /*
8361  * Requeue a task on a given node and accurately track the number of NUMA
8362  * tasks on the runqueues
8363  */
8364 void sched_setnuma(struct task_struct *p, int nid)
8365 {
8366 	guard(task_rq_lock)(p);
8367 	scoped_guard (sched_change, p, DEQUEUE_SAVE)
8368 		p->numa_preferred_nid = nid;
8369 }
8370 #endif /* CONFIG_NUMA_BALANCING */
8371 
8372 #ifdef CONFIG_HOTPLUG_CPU
8373 /*
8374  * Invoked on the outgoing CPU in context of the CPU hotplug thread
8375  * after ensuring that there are no user space tasks left on the CPU.
8376  *
8377  * If there is a lazy mm in use on the hotplug thread, drop it and
8378  * switch to init_mm.
8379  *
8380  * The reference count on init_mm is dropped in finish_cpu().
8381  */
8382 static void sched_force_init_mm(void)
8383 {
8384 	struct mm_struct *mm = current->active_mm;
8385 
8386 	if (mm != &init_mm) {
8387 		mmgrab_lazy_tlb(&init_mm);
8388 		local_irq_disable();
8389 		current->active_mm = &init_mm;
8390 		switch_mm_irqs_off(mm, &init_mm, current);
8391 		local_irq_enable();
8392 		finish_arch_post_lock_switch();
8393 		mmdrop_lazy_tlb(mm);
8394 	}
8395 
8396 	/* finish_cpu(), as ran on the BP, will clean up the active_mm state */
8397 }
8398 
8399 static int __balance_push_cpu_stop(void *arg)
8400 {
8401 	struct task_struct *p = arg;
8402 	struct rq *rq = this_rq();
8403 	struct rq_flags rf;
8404 	int cpu;
8405 
8406 	scoped_guard (raw_spinlock_irq, &p->pi_lock) {
8407 		/*
8408 		 * We may change the underlying rq, but the locks held will
8409 		 * appropriately be "transferred" when switching.
8410 		 */
8411 		context_unsafe_alias(rq);
8412 
8413 		cpu = select_fallback_rq(rq->cpu, p);
8414 
8415 		rq_lock(rq, &rf);
8416 		update_rq_clock(rq);
8417 		if (task_rq(p) == rq && task_on_rq_queued(p))
8418 			rq = __migrate_task(rq, &rf, p, cpu);
8419 		rq_unlock(rq, &rf);
8420 	}
8421 
8422 	put_task_struct(p);
8423 
8424 	return 0;
8425 }
8426 
8427 static DEFINE_PER_CPU(struct cpu_stop_work, push_work);
8428 
8429 /*
8430  * Ensure we only run per-cpu kthreads once the CPU goes !active.
8431  *
8432  * This is enabled below SCHED_AP_ACTIVE; when !cpu_active(), but only
8433  * effective when the hotplug motion is down.
8434  */
8435 static void balance_push(struct rq *rq)
8436 	__must_hold(__rq_lockp(rq))
8437 {
8438 	struct task_struct *push_task = rq->curr;
8439 
8440 	lockdep_assert_rq_held(rq);
8441 
8442 	/*
8443 	 * Ensure the thing is persistent until balance_push_set(.on = false);
8444 	 */
8445 	rq->balance_callback = &balance_push_callback;
8446 
8447 	/*
8448 	 * Only active while going offline and when invoked on the outgoing
8449 	 * CPU.
8450 	 */
8451 	if (!cpu_dying(rq->cpu) || rq != this_rq())
8452 		return;
8453 
8454 	/*
8455 	 * Both the cpu-hotplug and stop task are in this case and are
8456 	 * required to complete the hotplug process.
8457 	 */
8458 	if (kthread_is_per_cpu(push_task) ||
8459 	    is_migration_disabled(push_task)) {
8460 
8461 		/*
8462 		 * If this is the idle task on the outgoing CPU try to wake
8463 		 * up the hotplug control thread which might wait for the
8464 		 * last task to vanish. The rcuwait_active() check is
8465 		 * accurate here because the waiter is pinned on this CPU
8466 		 * and can't obviously be running in parallel.
8467 		 *
8468 		 * On RT kernels this also has to check whether there are
8469 		 * pinned and scheduled out tasks on the runqueue. They
8470 		 * need to leave the migrate disabled section first.
8471 		 */
8472 		if (!rq->nr_running && !rq_has_pinned_tasks(rq) &&
8473 		    rcuwait_active(&rq->hotplug_wait)) {
8474 			raw_spin_rq_unlock(rq);
8475 			rcuwait_wake_up(&rq->hotplug_wait);
8476 			raw_spin_rq_lock(rq);
8477 		}
8478 		return;
8479 	}
8480 
8481 	get_task_struct(push_task);
8482 	/*
8483 	 * Temporarily drop rq->lock such that we can wake-up the stop task.
8484 	 * Both preemption and IRQs are still disabled.
8485 	 */
8486 	preempt_disable();
8487 	raw_spin_rq_unlock(rq);
8488 	stop_one_cpu_nowait(rq->cpu, __balance_push_cpu_stop, push_task,
8489 			    this_cpu_ptr(&push_work));
8490 	preempt_enable();
8491 	/*
8492 	 * At this point need_resched() is true and we'll take the loop in
8493 	 * schedule(). The next pick is obviously going to be the stop task
8494 	 * which kthread_is_per_cpu() and will push this task away.
8495 	 */
8496 	raw_spin_rq_lock(rq);
8497 }
8498 
8499 static void balance_push_set(int cpu, bool on)
8500 {
8501 	struct rq *rq = cpu_rq(cpu);
8502 	struct rq_flags rf;
8503 
8504 	rq_lock_irqsave(rq, &rf);
8505 	if (on) {
8506 		WARN_ON_ONCE(rq->balance_callback);
8507 		rq->balance_callback = &balance_push_callback;
8508 	} else if (rq->balance_callback == &balance_push_callback) {
8509 		rq->balance_callback = NULL;
8510 	}
8511 	rq_unlock_irqrestore(rq, &rf);
8512 }
8513 
8514 /*
8515  * Invoked from a CPUs hotplug control thread after the CPU has been marked
8516  * inactive. All tasks which are not per CPU kernel threads are either
8517  * pushed off this CPU now via balance_push() or placed on a different CPU
8518  * during wakeup. Wait until the CPU is quiescent.
8519  */
8520 static void balance_hotplug_wait(void)
8521 {
8522 	struct rq *rq = this_rq();
8523 
8524 	rcuwait_wait_event(&rq->hotplug_wait,
8525 			   rq->nr_running == 1 && !rq_has_pinned_tasks(rq),
8526 			   TASK_UNINTERRUPTIBLE);
8527 }
8528 
8529 #else /* !CONFIG_HOTPLUG_CPU: */
8530 
8531 static inline void balance_push(struct rq *rq)
8532 {
8533 }
8534 
8535 static inline void balance_push_set(int cpu, bool on)
8536 {
8537 }
8538 
8539 static inline void balance_hotplug_wait(void)
8540 {
8541 }
8542 
8543 #endif /* !CONFIG_HOTPLUG_CPU */
8544 
8545 void set_rq_online(struct rq *rq)
8546 {
8547 	if (!rq->online) {
8548 		const struct sched_class *class;
8549 
8550 		cpumask_set_cpu(rq->cpu, rq->rd->online);
8551 		rq->online = 1;
8552 
8553 		for_each_class(class) {
8554 			if (class->rq_online)
8555 				class->rq_online(rq);
8556 		}
8557 	}
8558 }
8559 
8560 void set_rq_offline(struct rq *rq)
8561 {
8562 	if (rq->online) {
8563 		const struct sched_class *class;
8564 
8565 		update_rq_clock(rq);
8566 		for_each_class(class) {
8567 			if (class->rq_offline)
8568 				class->rq_offline(rq);
8569 		}
8570 
8571 		cpumask_clear_cpu(rq->cpu, rq->rd->online);
8572 		rq->online = 0;
8573 	}
8574 }
8575 
8576 static inline void sched_set_rq_online(struct rq *rq, int cpu)
8577 {
8578 	struct rq_flags rf;
8579 
8580 	rq_lock_irqsave(rq, &rf);
8581 	if (rq->rd) {
8582 		BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
8583 		set_rq_online(rq);
8584 	}
8585 	rq_unlock_irqrestore(rq, &rf);
8586 }
8587 
8588 static inline void sched_set_rq_offline(struct rq *rq, int cpu)
8589 {
8590 	struct rq_flags rf;
8591 
8592 	rq_lock_irqsave(rq, &rf);
8593 	if (rq->rd) {
8594 		BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
8595 		set_rq_offline(rq);
8596 	}
8597 	rq_unlock_irqrestore(rq, &rf);
8598 }
8599 
8600 /*
8601  * used to mark begin/end of suspend/resume:
8602  */
8603 static int num_cpus_frozen;
8604 
8605 /*
8606  * Update cpusets according to cpu_active mask.  If cpusets are
8607  * disabled, cpuset_update_active_cpus() becomes a simple wrapper
8608  * around partition_sched_domains().
8609  *
8610  * If we come here as part of a suspend/resume, don't touch cpusets because we
8611  * want to restore it back to its original state upon resume anyway.
8612  */
8613 static void cpuset_cpu_active(void)
8614 {
8615 	if (cpuhp_tasks_frozen) {
8616 		/*
8617 		 * num_cpus_frozen tracks how many CPUs are involved in suspend
8618 		 * resume sequence. As long as this is not the last online
8619 		 * operation in the resume sequence, just build a single sched
8620 		 * domain, ignoring cpusets.
8621 		 */
8622 		cpuset_reset_sched_domains();
8623 		if (--num_cpus_frozen)
8624 			return;
8625 		/*
8626 		 * This is the last CPU online operation. So fall through and
8627 		 * restore the original sched domains by considering the
8628 		 * cpuset configurations.
8629 		 */
8630 		cpuset_force_rebuild();
8631 	}
8632 	cpuset_update_active_cpus();
8633 }
8634 
8635 static void cpuset_cpu_inactive(unsigned int cpu)
8636 {
8637 	if (!cpuhp_tasks_frozen) {
8638 		cpuset_update_active_cpus();
8639 	} else {
8640 		num_cpus_frozen++;
8641 		cpuset_reset_sched_domains();
8642 	}
8643 }
8644 
8645 static inline void sched_smt_present_inc(int cpu)
8646 {
8647 	if (cpumask_weight(cpu_smt_mask(cpu)) == 2)
8648 		static_branch_inc_cpuslocked(&sched_smt_present);
8649 }
8650 
8651 static inline void sched_smt_present_dec(int cpu)
8652 {
8653 	if (cpumask_weight(cpu_smt_mask(cpu)) == 2)
8654 		static_branch_dec_cpuslocked(&sched_smt_present);
8655 }
8656 
8657 int sched_cpu_activate(unsigned int cpu)
8658 {
8659 	struct rq *rq = cpu_rq(cpu);
8660 
8661 	/*
8662 	 * Clear the balance_push callback and prepare to schedule
8663 	 * regular tasks.
8664 	 */
8665 	balance_push_set(cpu, false);
8666 
8667 	/*
8668 	 * When going up, increment the number of cores with SMT present.
8669 	 */
8670 	sched_smt_present_inc(cpu);
8671 	set_cpu_active(cpu, true);
8672 
8673 	if (sched_smp_initialized) {
8674 		sched_update_numa(cpu, true);
8675 		sched_domains_numa_masks_set(cpu);
8676 		cpuset_cpu_active();
8677 	}
8678 
8679 	scx_rq_activate(rq);
8680 
8681 	/*
8682 	 * Put the rq online, if not already. This happens:
8683 	 *
8684 	 * 1) In the early boot process, because we build the real domains
8685 	 *    after all CPUs have been brought up.
8686 	 *
8687 	 * 2) At runtime, if cpuset_cpu_active() fails to rebuild the
8688 	 *    domains.
8689 	 */
8690 	sched_set_rq_online(rq, cpu);
8691 
8692 	/* preferred is subset of active and follows its state */
8693 	set_cpu_preferred(cpu, true);
8694 
8695 	return 0;
8696 }
8697 
8698 int sched_cpu_deactivate(unsigned int cpu)
8699 {
8700 	struct rq *rq = cpu_rq(cpu);
8701 	int ret;
8702 
8703 	ret = dl_bw_deactivate(cpu);
8704 
8705 	if (ret)
8706 		return ret;
8707 
8708 	set_cpu_preferred(cpu, false);
8709 
8710 	/*
8711 	 * Remove CPU from nohz.idle_cpus_mask to prevent participating in
8712 	 * load balancing when not active
8713 	 */
8714 	scoped_guard (rcu)
8715 		nohz_balance_exit_idle(rq);
8716 
8717 	set_cpu_active(cpu, false);
8718 
8719 	/*
8720 	 * From this point forward, this CPU will refuse to run any task that
8721 	 * is not: migrate_disable() or KTHREAD_IS_PER_CPU, and will actively
8722 	 * push those tasks away until this gets cleared, see
8723 	 * sched_cpu_dying().
8724 	 */
8725 	balance_push_set(cpu, true);
8726 
8727 	/*
8728 	 * We've cleared cpu_active_mask / set balance_push, wait for all
8729 	 * preempt-disabled and RCU users of this state to go away such that
8730 	 * all new such users will observe it.
8731 	 *
8732 	 * Specifically, we rely on ttwu to no longer target this CPU, see
8733 	 * ttwu_queue_cond() and is_cpu_allowed().
8734 	 *
8735 	 * Do sync before park smpboot threads to take care the RCU boost case.
8736 	 */
8737 	synchronize_rcu();
8738 
8739 	sched_domains_free_llc_id(cpu);
8740 
8741 	sched_set_rq_offline(rq, cpu);
8742 
8743 	scx_rq_deactivate(rq);
8744 
8745 	/*
8746 	 * When going down, decrement the number of cores with SMT present.
8747 	 */
8748 	sched_smt_present_dec(cpu);
8749 
8750 	sched_core_cpu_deactivate(cpu);
8751 
8752 	if (!sched_smp_initialized)
8753 		return 0;
8754 
8755 	sched_update_numa(cpu, false);
8756 	cpuset_cpu_inactive(cpu);
8757 	sched_domains_numa_masks_clear(cpu);
8758 	return 0;
8759 }
8760 
8761 static void sched_rq_cpu_starting(unsigned int cpu)
8762 {
8763 	struct rq *rq = cpu_rq(cpu);
8764 
8765 	rq->calc_load_update = calc_load_update;
8766 	update_max_interval();
8767 }
8768 
8769 int sched_cpu_starting(unsigned int cpu)
8770 {
8771 	sched_core_cpu_starting(cpu);
8772 	sched_rq_cpu_starting(cpu);
8773 	sched_tick_start(cpu);
8774 	return 0;
8775 }
8776 
8777 #ifdef CONFIG_HOTPLUG_CPU
8778 
8779 /*
8780  * Invoked immediately before the stopper thread is invoked to bring the
8781  * CPU down completely. At this point all per CPU kthreads except the
8782  * hotplug thread (current) and the stopper thread (inactive) have been
8783  * either parked or have been unbound from the outgoing CPU. Ensure that
8784  * any of those which might be on the way out are gone.
8785  *
8786  * If after this point a bound task is being woken on this CPU then the
8787  * responsible hotplug callback has failed to do it's job.
8788  * sched_cpu_dying() will catch it with the appropriate fireworks.
8789  */
8790 int sched_cpu_wait_empty(unsigned int cpu)
8791 {
8792 	balance_hotplug_wait();
8793 	sched_force_init_mm();
8794 	return 0;
8795 }
8796 
8797 /*
8798  * Since this CPU is going 'away' for a while, fold any nr_active delta we
8799  * might have. Called from the CPU stopper task after ensuring that the
8800  * stopper is the last running task on the CPU, so nr_active count is
8801  * stable. We need to take the tear-down thread which is calling this into
8802  * account, so we hand in adjust = 1 to the load calculation.
8803  *
8804  * Also see the comment "Global load-average calculations".
8805  */
8806 static void calc_load_migrate(struct rq *rq)
8807 {
8808 	long delta = calc_load_fold_active(rq, 1);
8809 
8810 	if (delta)
8811 		atomic_long_add(delta, &calc_load_tasks);
8812 }
8813 
8814 static void dump_rq_tasks(struct rq *rq, const char *loglvl)
8815 {
8816 	struct task_struct *g, *p;
8817 	int cpu = cpu_of(rq);
8818 
8819 	lockdep_assert_rq_held(rq);
8820 
8821 	printk("%sCPU%d enqueued tasks (%u total):\n", loglvl, cpu, rq->nr_running);
8822 	for_each_process_thread(g, p) {
8823 		if (task_cpu(p) != cpu)
8824 			continue;
8825 
8826 		if (!task_on_rq_queued(p))
8827 			continue;
8828 
8829 		printk("%s\tpid: %d, name: %s\n", loglvl, p->pid, p->comm);
8830 	}
8831 }
8832 
8833 int sched_cpu_dying(unsigned int cpu)
8834 {
8835 	struct rq *rq = cpu_rq(cpu);
8836 	struct rq_flags rf;
8837 
8838 	/* Handle pending wakeups and then migrate everything off */
8839 	sched_tick_stop(cpu);
8840 
8841 	rq_lock_irqsave(rq, &rf);
8842 	update_rq_clock(rq);
8843 	if (rq->nr_running != 1 || rq_has_pinned_tasks(rq)) {
8844 		WARN(true, "Dying CPU not properly vacated!");
8845 		dump_rq_tasks(rq, KERN_WARNING);
8846 	}
8847 	dl_server_stop(&rq->fair_server);
8848 #ifdef CONFIG_SCHED_CLASS_EXT
8849 	dl_server_stop(&rq->ext_server);
8850 #endif
8851 	rq_unlock_irqrestore(rq, &rf);
8852 
8853 	calc_load_migrate(rq);
8854 	update_max_interval();
8855 	hrtick_clear(rq);
8856 	sched_core_cpu_dying(cpu);
8857 	return 0;
8858 }
8859 #endif /* CONFIG_HOTPLUG_CPU */
8860 
8861 void __init sched_init_smp(void)
8862 {
8863 	sched_init_numa(NUMA_NO_NODE);
8864 
8865 	prandom_init_once(&sched_rnd_state);
8866 
8867 	/*
8868 	 * There's no userspace yet to cause hotplug operations; hence all the
8869 	 * CPU masks are stable and all blatant races in the below code cannot
8870 	 * happen.
8871 	 */
8872 	sched_domains_mutex_lock();
8873 	sched_init_domains(cpu_active_mask);
8874 	sched_domains_mutex_unlock();
8875 
8876 	/* Move init over to a non-isolated CPU */
8877 	if (set_cpus_allowed_ptr(current, housekeeping_cpumask(HK_TYPE_DOMAIN)) < 0)
8878 		BUG();
8879 	current->flags &= ~PF_NO_SETAFFINITY;
8880 	sched_init_granularity();
8881 
8882 	init_sched_rt_class();
8883 	init_sched_dl_class();
8884 
8885 	sched_init_dl_servers();
8886 
8887 	sched_smp_initialized = true;
8888 }
8889 
8890 static int __init migration_init(void)
8891 {
8892 	sched_cpu_starting(smp_processor_id());
8893 	return 0;
8894 }
8895 early_initcall(migration_init);
8896 
8897 int in_sched_functions(unsigned long addr)
8898 {
8899 	return in_lock_functions(addr) ||
8900 		(addr >= (unsigned long)__sched_text_start
8901 		&& addr < (unsigned long)__sched_text_end);
8902 }
8903 
8904 #ifdef CONFIG_CGROUP_SCHED
8905 /*
8906  * Default task group.
8907  * Every task in system belongs to this group at bootup.
8908  */
8909 struct task_group root_task_group;
8910 LIST_HEAD(task_groups);
8911 
8912 /* Cacheline aligned slab cache for task_group */
8913 static struct kmem_cache *task_group_cache __ro_after_init;
8914 #endif
8915 
8916 void __init sched_init(void)
8917 {
8918 	unsigned long __maybe_unused ptr = 0;
8919 	int i;
8920 
8921 	/* Make sure the linker didn't screw up */
8922 	BUG_ON(!sched_class_above(&stop_sched_class, &dl_sched_class));
8923 	BUG_ON(!sched_class_above(&dl_sched_class, &rt_sched_class));
8924 	BUG_ON(!sched_class_above(&rt_sched_class, &fair_sched_class));
8925 	BUG_ON(!sched_class_above(&fair_sched_class, &idle_sched_class));
8926 #ifdef CONFIG_SCHED_CLASS_EXT
8927 	BUG_ON(!sched_class_above(&fair_sched_class, &ext_sched_class));
8928 	BUG_ON(!sched_class_above(&ext_sched_class, &idle_sched_class));
8929 #endif
8930 
8931 	wait_bit_init();
8932 
8933 #ifdef CONFIG_FAIR_GROUP_SCHED
8934 	root_task_group.cfs_rq = &runqueues.cfs;
8935 
8936 	root_task_group.shares = ROOT_TASK_GROUP_LOAD;
8937 	init_cfs_bandwidth(&root_task_group.cfs_bandwidth, NULL);
8938 #endif /* CONFIG_FAIR_GROUP_SCHED */
8939 #ifdef CONFIG_EXT_GROUP_SCHED
8940 	scx_tg_init(&root_task_group);
8941 #endif /* CONFIG_EXT_GROUP_SCHED */
8942 #ifdef CONFIG_RT_GROUP_SCHED
8943 	ptr += 2 * nr_cpu_ids * sizeof(void **);
8944 	ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT);
8945 	root_task_group.rt_se = (struct sched_rt_entity **)ptr;
8946 	ptr += nr_cpu_ids * sizeof(void **);
8947 
8948 	root_task_group.rt_rq = (struct rt_rq **)ptr;
8949 	ptr += nr_cpu_ids * sizeof(void **);
8950 
8951 #endif /* CONFIG_RT_GROUP_SCHED */
8952 
8953 	init_defrootdomain();
8954 
8955 #ifdef CONFIG_RT_GROUP_SCHED
8956 	init_rt_bandwidth(&root_task_group.rt_bandwidth,
8957 			global_rt_period(), global_rt_runtime());
8958 #endif /* CONFIG_RT_GROUP_SCHED */
8959 
8960 #ifdef CONFIG_CGROUP_SCHED
8961 	task_group_cache = KMEM_CACHE(task_group, 0);
8962 
8963 	list_add(&root_task_group.list, &task_groups);
8964 	INIT_LIST_HEAD(&root_task_group.children);
8965 	INIT_LIST_HEAD(&root_task_group.siblings);
8966 	autogroup_init(&init_task);
8967 #endif /* CONFIG_CGROUP_SCHED */
8968 
8969 	for_each_possible_cpu(i) {
8970 		struct rq *rq;
8971 
8972 		rq = cpu_rq(i);
8973 		raw_spin_lock_init(&rq->__lock);
8974 		rq->nr_running = 0;
8975 		rq->calc_load_active = 0;
8976 		rq->calc_load_update = jiffies + LOAD_FREQ;
8977 		init_cfs_rq(&rq->cfs);
8978 		init_rt_rq(&rq->rt);
8979 		init_dl_rq(&rq->dl);
8980 #ifdef CONFIG_FAIR_GROUP_SCHED
8981 		INIT_LIST_HEAD(&rq->leaf_cfs_rq_list);
8982 		rq->tmp_alone_branch = &rq->leaf_cfs_rq_list;
8983 		/*
8984 		 * How much CPU bandwidth does root_task_group get?
8985 		 *
8986 		 * In case of task-groups formed through the cgroup filesystem, it
8987 		 * gets 100% of the CPU resources in the system. This overall
8988 		 * system CPU resource is divided among the tasks of
8989 		 * root_task_group and its child task-groups in a fair manner,
8990 		 * based on each entity's (task or task-group's) weight
8991 		 * (se->load.weight).
8992 		 *
8993 		 * In other words, if root_task_group has 10 tasks of weight
8994 		 * 1024) and two child groups A0 and A1 (of weight 1024 each),
8995 		 * then A0's share of the CPU resource is:
8996 		 *
8997 		 *	A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33%
8998 		 *
8999 		 * We achieve this by letting root_task_group's tasks sit
9000 		 * directly in rq->cfs (i.e root_task_group->se[] = NULL).
9001 		 */
9002 		init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL);
9003 #endif /* CONFIG_FAIR_GROUP_SCHED */
9004 
9005 #ifdef CONFIG_RT_GROUP_SCHED
9006 		/*
9007 		 * This is required for init cpu because rt.c:__enable_runtime()
9008 		 * starts working after scheduler_running, which is not the case
9009 		 * yet.
9010 		 */
9011 		rq->rt.rt_runtime = global_rt_runtime();
9012 		init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL);
9013 #endif
9014 		rq->next_class = &idle_sched_class;
9015 
9016 		rq->sd = NULL;
9017 		rq->rd = NULL;
9018 		rq->cpu_capacity = SCHED_CAPACITY_SCALE;
9019 		rq->balance_callback = &balance_push_callback;
9020 		rq->active_balance = 0;
9021 		rq->next_balance = jiffies;
9022 		rq->push_cpu = 0;
9023 		rq->cpu = i;
9024 		rq->online = 0;
9025 		rq->idle_stamp = 0;
9026 		rq->avg_idle = 2*sysctl_sched_migration_cost;
9027 		rq->max_idle_balance_cost = sysctl_sched_migration_cost;
9028 
9029 		INIT_LIST_HEAD(&rq->cfs_tasks);
9030 
9031 		rq_attach_root(rq, &def_root_domain);
9032 #ifdef CONFIG_NO_HZ_COMMON
9033 		rq->last_blocked_load_update_tick = jiffies;
9034 		atomic_set(&rq->nohz_flags, 0);
9035 
9036 		INIT_CSD(&rq->nohz_csd, nohz_csd_func, rq);
9037 #endif
9038 #ifdef CONFIG_HOTPLUG_CPU
9039 		rcuwait_init(&rq->hotplug_wait);
9040 #endif
9041 		hrtick_rq_init(rq);
9042 		atomic_set(&rq->nr_iowait, 0);
9043 		fair_server_init(rq);
9044 #ifdef CONFIG_SCHED_CLASS_EXT
9045 		ext_server_init(rq);
9046 #endif
9047 
9048 #ifdef CONFIG_SCHED_CORE
9049 		rq->core = rq;
9050 		rq->core_pick = NULL;
9051 		rq->core_dl_server = NULL;
9052 		rq->core_enabled = 0;
9053 		rq->core_tree = RB_ROOT;
9054 		rq->core_forceidle_count = 0;
9055 		rq->core_forceidle_occupation = 0;
9056 		rq->core_forceidle_start = 0;
9057 		rq->core_pick_in_flight = 0;
9058 
9059 		rq->core_cookie = 0UL;
9060 #endif
9061 #ifdef CONFIG_SCHED_CACHE
9062 		raw_spin_lock_init(&rq->cpu_epoch_lock);
9063 		rq->cpu_epoch_next = jiffies;
9064 #endif
9065 
9066 		zalloc_cpumask_var_node(&rq->scratch_mask, GFP_KERNEL, cpu_to_node(i));
9067 	}
9068 
9069 	set_load_weight(&init_task, false);
9070 	init_task.se.slice = sysctl_sched_base_slice,
9071 
9072 	/*
9073 	 * The boot idle thread does lazy MMU switching as well:
9074 	 */
9075 	mmgrab_lazy_tlb(&init_mm);
9076 	enter_lazy_tlb(&init_mm, current);
9077 
9078 	/*
9079 	 * The idle task doesn't need the kthread struct to function, but it
9080 	 * is dressed up as a per-CPU kthread and thus needs to play the part
9081 	 * if we want to avoid special-casing it in code that deals with per-CPU
9082 	 * kthreads.
9083 	 */
9084 	WARN_ON(!set_kthread_struct(current));
9085 
9086 	/*
9087 	 * Make us the idle thread. Technically, schedule() should not be
9088 	 * called from this thread, however somewhere below it might be,
9089 	 * but because we are the idle thread, we just pick up running again
9090 	 * when this runqueue becomes "idle".
9091 	 */
9092 	__sched_fork(0, current);
9093 	init_idle(current, smp_processor_id());
9094 
9095 	calc_load_update = jiffies + LOAD_FREQ;
9096 
9097 	idle_thread_set_boot_cpu();
9098 
9099 	balance_push_set(smp_processor_id(), false);
9100 	init_sched_fair_class();
9101 	init_sched_ext_class();
9102 
9103 	psi_init();
9104 
9105 	init_uclamp();
9106 
9107 	preempt_dynamic_init();
9108 
9109 	scheduler_running = 1;
9110 }
9111 
9112 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
9113 
9114 void __might_sleep(const char *file, int line)
9115 {
9116 	unsigned int state = get_current_state();
9117 	/*
9118 	 * Blocking primitives will set (and therefore destroy) current->state,
9119 	 * since we will exit with TASK_RUNNING make sure we enter with it,
9120 	 * otherwise we will destroy state.
9121 	 */
9122 	WARN_ONCE(state != TASK_RUNNING && current->task_state_change,
9123 			"do not call blocking ops when !TASK_RUNNING; "
9124 			"state=%x set at [<%p>] %pS\n", state,
9125 			(void *)current->task_state_change,
9126 			(void *)current->task_state_change);
9127 
9128 	__might_resched(file, line, 0);
9129 }
9130 EXPORT_SYMBOL(__might_sleep);
9131 
9132 static void print_preempt_disable_ip(int preempt_offset, unsigned long ip)
9133 {
9134 	if (!IS_ENABLED(CONFIG_DEBUG_PREEMPT))
9135 		return;
9136 
9137 	if (preempt_count() == preempt_offset)
9138 		return;
9139 
9140 	pr_err("Preemption disabled at:");
9141 	print_ip_sym(KERN_ERR, ip);
9142 }
9143 
9144 static inline bool resched_offsets_ok(unsigned int offsets)
9145 {
9146 	unsigned int nested = preempt_count();
9147 
9148 	nested += rcu_preempt_depth() << MIGHT_RESCHED_RCU_SHIFT;
9149 
9150 	return nested == offsets;
9151 }
9152 
9153 void __might_resched(const char *file, int line, unsigned int offsets)
9154 {
9155 	/* Ratelimiting timestamp: */
9156 	static unsigned long prev_jiffy;
9157 
9158 	unsigned long preempt_disable_ip;
9159 
9160 	/* WARN_ON_ONCE() by default, no rate limit required: */
9161 	rcu_sleep_check();
9162 
9163 	if ((resched_offsets_ok(offsets) && !irqs_disabled() &&
9164 	     !is_idle_task(current) && !current->non_block_count) ||
9165 	    system_state == SYSTEM_BOOTING || system_state > SYSTEM_RUNNING ||
9166 	    oops_in_progress)
9167 		return;
9168 
9169 	if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
9170 		return;
9171 	prev_jiffy = jiffies;
9172 
9173 	/* Save this before calling printk(), since that will clobber it: */
9174 	preempt_disable_ip = get_preempt_disable_ip(current);
9175 
9176 	pr_err("BUG: sleeping function called from invalid context at %s:%d\n",
9177 	       file, line);
9178 	pr_err("in_atomic(): %d, irqs_disabled(): %d, non_block: %d, pid: %d, name: %s\n",
9179 	       in_atomic(), irqs_disabled(), current->non_block_count,
9180 	       current->pid, current->comm);
9181 	pr_err("preempt_count: %x, expected: %x\n", preempt_count(),
9182 	       offsets & MIGHT_RESCHED_PREEMPT_MASK);
9183 
9184 	if (IS_ENABLED(CONFIG_PREEMPT_RCU)) {
9185 		pr_err("RCU nest depth: %d, expected: %u\n",
9186 		       rcu_preempt_depth(), offsets >> MIGHT_RESCHED_RCU_SHIFT);
9187 	}
9188 
9189 	if (task_stack_end_corrupted(current))
9190 		pr_emerg("Thread overran stack, or stack corrupted\n");
9191 
9192 	debug_show_held_locks(current);
9193 	if (irqs_disabled())
9194 		print_irqtrace_events(current);
9195 
9196 	print_preempt_disable_ip(offsets & MIGHT_RESCHED_PREEMPT_MASK,
9197 				 preempt_disable_ip);
9198 
9199 	dump_stack();
9200 	add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
9201 }
9202 EXPORT_SYMBOL(__might_resched);
9203 
9204 void __cant_sleep(const char *file, int line)
9205 {
9206 	static unsigned long prev_jiffy;
9207 
9208 	if (irqs_disabled())
9209 		return;
9210 
9211 	if (!IS_ENABLED(CONFIG_PREEMPT_COUNT))
9212 		return;
9213 
9214 	if (preempt_count())
9215 		return;
9216 
9217 	if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
9218 		return;
9219 	prev_jiffy = jiffies;
9220 
9221 	printk(KERN_ERR "BUG: assuming atomic context at %s:%d\n", file, line);
9222 	printk(KERN_ERR "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n",
9223 			in_atomic(), irqs_disabled(),
9224 			current->pid, current->comm);
9225 
9226 	debug_show_held_locks(current);
9227 	dump_stack();
9228 	add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
9229 }
9230 EXPORT_SYMBOL_GPL(__cant_sleep);
9231 
9232 # ifdef CONFIG_SMP
9233 void __cant_migrate(const char *file, int line)
9234 {
9235 	static unsigned long prev_jiffy;
9236 
9237 	if (irqs_disabled())
9238 		return;
9239 
9240 	if (is_migration_disabled(current))
9241 		return;
9242 
9243 	if (!IS_ENABLED(CONFIG_PREEMPT_COUNT))
9244 		return;
9245 
9246 	if (preempt_count())
9247 		return;
9248 
9249 	if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
9250 		return;
9251 	prev_jiffy = jiffies;
9252 
9253 	pr_err("BUG: assuming non migratable context at %s:%d\n", file, line);
9254 	pr_err("in_atomic(): %d, irqs_disabled(): %d, migration_disabled() %u pid: %d, name: %s\n",
9255 	       in_atomic(), irqs_disabled(), is_migration_disabled(current),
9256 	       current->pid, current->comm);
9257 
9258 	debug_show_held_locks(current);
9259 	dump_stack();
9260 	add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
9261 }
9262 EXPORT_SYMBOL_GPL(__cant_migrate);
9263 # endif /* CONFIG_SMP */
9264 #endif /* CONFIG_DEBUG_ATOMIC_SLEEP */
9265 
9266 #ifdef CONFIG_MAGIC_SYSRQ
9267 void normalize_rt_tasks(void)
9268 {
9269 	struct task_struct *g, *p;
9270 	struct sched_attr attr = {
9271 		.sched_policy = SCHED_NORMAL,
9272 	};
9273 
9274 	read_lock(&tasklist_lock);
9275 	for_each_process_thread(g, p) {
9276 		/*
9277 		 * Only normalize user tasks:
9278 		 */
9279 		if (p->flags & PF_KTHREAD)
9280 			continue;
9281 
9282 		p->se.exec_start = 0;
9283 		schedstat_set(p->stats.wait_start,  0);
9284 		schedstat_set(p->stats.sleep_start, 0);
9285 		schedstat_set(p->stats.block_start, 0);
9286 
9287 		if (!rt_or_dl_task(p)) {
9288 			/*
9289 			 * Renice negative nice level userspace
9290 			 * tasks back to 0:
9291 			 */
9292 			if (task_nice(p) < 0)
9293 				set_user_nice(p, 0);
9294 			continue;
9295 		}
9296 
9297 		__sched_setscheduler(p, &attr, false, false);
9298 	}
9299 	read_unlock(&tasklist_lock);
9300 }
9301 
9302 #endif /* CONFIG_MAGIC_SYSRQ */
9303 
9304 #ifdef CONFIG_KGDB_KDB
9305 /*
9306  * These functions are only useful for KDB.
9307  *
9308  * They can only be called when the whole system has been
9309  * stopped - every CPU needs to be quiescent, and no scheduling
9310  * activity can take place. Using them for anything else would
9311  * be a serious bug, and as a result, they aren't even visible
9312  * under any other configuration.
9313  */
9314 
9315 /**
9316  * curr_task - return the current task for a given CPU.
9317  * @cpu: the processor in question.
9318  *
9319  * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
9320  *
9321  * Return: The current task for @cpu.
9322  */
9323 struct task_struct *curr_task(int cpu)
9324 {
9325 	return cpu_curr(cpu);
9326 }
9327 
9328 #endif /* CONFIG_KGDB_KDB */
9329 
9330 #ifdef CONFIG_CGROUP_SCHED
9331 /* task_group_lock serializes the addition/removal of task groups */
9332 static DEFINE_SPINLOCK(task_group_lock);
9333 
9334 static inline void alloc_uclamp_sched_group(struct task_group *tg,
9335 					    struct task_group *parent)
9336 {
9337 #ifdef CONFIG_UCLAMP_TASK_GROUP
9338 	enum uclamp_id clamp_id;
9339 
9340 	for_each_clamp_id(clamp_id) {
9341 		uclamp_se_set(&tg->uclamp_req[clamp_id],
9342 			      uclamp_none(clamp_id), false);
9343 		tg->uclamp[clamp_id] = parent->uclamp[clamp_id];
9344 	}
9345 #endif
9346 }
9347 
9348 static void sched_free_group(struct task_group *tg)
9349 {
9350 	free_fair_sched_group(tg);
9351 	free_rt_sched_group(tg);
9352 	autogroup_free(tg);
9353 	kmem_cache_free(task_group_cache, tg);
9354 }
9355 
9356 static void sched_free_group_rcu(struct rcu_head *rcu)
9357 {
9358 	sched_free_group(container_of(rcu, struct task_group, rcu));
9359 }
9360 
9361 static void sched_unregister_group(struct task_group *tg)
9362 {
9363 	unregister_fair_sched_group(tg);
9364 	unregister_rt_sched_group(tg);
9365 	/*
9366 	 * We have to wait for yet another RCU grace period to expire, as
9367 	 * print_cfs_stats() might run concurrently.
9368 	 */
9369 	call_rcu(&tg->rcu, sched_free_group_rcu);
9370 }
9371 
9372 /* allocate runqueue etc for a new task group */
9373 struct task_group *sched_create_group(struct task_group *parent)
9374 {
9375 	struct task_group *tg;
9376 
9377 	tg = kmem_cache_alloc(task_group_cache, GFP_KERNEL | __GFP_ZERO);
9378 	if (!tg)
9379 		return ERR_PTR(-ENOMEM);
9380 
9381 	if (!alloc_fair_sched_group(tg, parent))
9382 		goto err;
9383 
9384 	if (!alloc_rt_sched_group(tg, parent))
9385 		goto err;
9386 
9387 	scx_tg_init(tg);
9388 	alloc_uclamp_sched_group(tg, parent);
9389 
9390 	return tg;
9391 
9392 err:
9393 	sched_free_group(tg);
9394 	return ERR_PTR(-ENOMEM);
9395 }
9396 
9397 void sched_online_group(struct task_group *tg, struct task_group *parent)
9398 {
9399 	unsigned long flags;
9400 
9401 	spin_lock_irqsave(&task_group_lock, flags);
9402 	list_add_tail_rcu(&tg->list, &task_groups);
9403 
9404 	/* Root should already exist: */
9405 	WARN_ON(!parent);
9406 
9407 	tg->parent = parent;
9408 	INIT_LIST_HEAD(&tg->children);
9409 	list_add_rcu(&tg->siblings, &parent->children);
9410 	spin_unlock_irqrestore(&task_group_lock, flags);
9411 
9412 	online_fair_sched_group(tg);
9413 }
9414 
9415 /* RCU callback to free various structures associated with a task group */
9416 static void sched_unregister_group_rcu(struct rcu_head *rhp)
9417 {
9418 	/* Now it should be safe to free those cfs_rqs: */
9419 	sched_unregister_group(container_of(rhp, struct task_group, rcu));
9420 }
9421 
9422 void sched_destroy_group(struct task_group *tg)
9423 {
9424 	/* Wait for possible concurrent references to cfs_rqs complete: */
9425 	call_rcu(&tg->rcu, sched_unregister_group_rcu);
9426 }
9427 
9428 void sched_release_group(struct task_group *tg)
9429 {
9430 	unsigned long flags;
9431 
9432 	/*
9433 	 * Unlink first, to avoid walk_tg_tree_from() from finding us (via
9434 	 * sched_cfs_period_timer()).
9435 	 *
9436 	 * For this to be effective, we have to wait for all pending users of
9437 	 * this task group to leave their RCU critical section to ensure no new
9438 	 * user will see our dying task group any more. Specifically ensure
9439 	 * that tg_unthrottle_up() won't add decayed cfs_rq's to it.
9440 	 *
9441 	 * We therefore defer calling unregister_fair_sched_group() to
9442 	 * sched_unregister_group() which is guarantied to get called only after the
9443 	 * current RCU grace period has expired.
9444 	 */
9445 	spin_lock_irqsave(&task_group_lock, flags);
9446 	list_del_rcu(&tg->list);
9447 	list_del_rcu(&tg->siblings);
9448 	spin_unlock_irqrestore(&task_group_lock, flags);
9449 }
9450 
9451 static void sched_change_group(struct task_struct *tsk)
9452 {
9453 	struct task_group *tg;
9454 
9455 	/*
9456 	 * All callers are synchronized by task_rq_lock(); we do not use RCU
9457 	 * which is pointless here. Thus, we pass "true" to task_css_check()
9458 	 * to prevent lockdep warnings.
9459 	 */
9460 	tg = container_of(task_css_check(tsk, cpu_cgrp_id, true),
9461 			  struct task_group, css);
9462 	tg = autogroup_task_group(tsk, tg);
9463 	tsk->sched_task_group = tg;
9464 
9465 #ifdef CONFIG_FAIR_GROUP_SCHED
9466 	if (tsk->sched_class->task_change_group)
9467 		tsk->sched_class->task_change_group(tsk);
9468 	else
9469 #endif
9470 		set_task_rq(tsk, task_cpu(tsk));
9471 }
9472 
9473 /*
9474  * Change task's runqueue when it moves between groups.
9475  *
9476  * The caller of this function should have put the task in its new group by
9477  * now. This function just updates tsk->se.cfs_rq and tsk->se.parent to reflect
9478  * its new group.
9479  */
9480 void sched_move_task(struct task_struct *tsk, bool for_autogroup)
9481 {
9482 	unsigned int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE;
9483 	bool resched = false;
9484 	bool queued = false;
9485 	struct rq *rq;
9486 
9487 	CLASS(task_rq_lock, rq_guard)(tsk);
9488 	rq = rq_guard.rq;
9489 
9490 	scoped_guard (sched_change, tsk, queue_flags) {
9491 		sched_change_group(tsk);
9492 		if (!for_autogroup)
9493 			scx_cgroup_move_task(tsk);
9494 		if (scope->running)
9495 			resched = true;
9496 		queued = scope->queued;
9497 	}
9498 
9499 	if (resched)
9500 		resched_curr(rq);
9501 	else if (queued)
9502 		wakeup_preempt(rq, tsk, 0);
9503 
9504 	__balance_callbacks(rq, &rq_guard.rf);
9505 }
9506 
9507 static struct cgroup_subsys_state *
9508 cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
9509 {
9510 	struct task_group *parent = css_tg(parent_css);
9511 	struct task_group *tg;
9512 
9513 	if (!parent) {
9514 		/* This is early initialization for the top cgroup */
9515 		return &root_task_group.css;
9516 	}
9517 
9518 	tg = sched_create_group(parent);
9519 	if (IS_ERR(tg))
9520 		return ERR_PTR(-ENOMEM);
9521 
9522 	return &tg->css;
9523 }
9524 
9525 /* Expose task group only after completing cgroup initialization */
9526 static int cpu_cgroup_css_online(struct cgroup_subsys_state *css)
9527 {
9528 	struct task_group *tg = css_tg(css);
9529 	struct task_group *parent = css_tg(css->parent);
9530 	int ret;
9531 
9532 	ret = scx_tg_online(tg);
9533 	if (ret)
9534 		return ret;
9535 
9536 	if (parent)
9537 		sched_online_group(tg, parent);
9538 
9539 #ifdef CONFIG_UCLAMP_TASK_GROUP
9540 	/* Propagate the effective uclamp value for the new group */
9541 	guard(mutex)(&uclamp_mutex);
9542 	guard(rcu)();
9543 	cpu_util_update_eff(css);
9544 #endif
9545 
9546 	return 0;
9547 }
9548 
9549 static void cpu_cgroup_css_offline(struct cgroup_subsys_state *css)
9550 {
9551 	struct task_group *tg = css_tg(css);
9552 
9553 	scx_tg_offline(tg);
9554 }
9555 
9556 static void cpu_cgroup_css_released(struct cgroup_subsys_state *css)
9557 {
9558 	struct task_group *tg = css_tg(css);
9559 
9560 	sched_release_group(tg);
9561 }
9562 
9563 static void cpu_cgroup_css_free(struct cgroup_subsys_state *css)
9564 {
9565 	struct task_group *tg = css_tg(css);
9566 
9567 	/*
9568 	 * Relies on the RCU grace period between css_released() and this.
9569 	 */
9570 	sched_unregister_group(tg);
9571 }
9572 
9573 static int cpu_cgroup_can_attach(struct cgroup_taskset *tset)
9574 {
9575 #ifdef CONFIG_RT_GROUP_SCHED
9576 	struct task_struct *task;
9577 	struct cgroup_subsys_state *css;
9578 
9579 	if (!rt_group_sched_enabled())
9580 		goto scx_check;
9581 
9582 	cgroup_taskset_for_each(task, css, tset) {
9583 		if (!sched_rt_can_attach(css_tg(css), task))
9584 			return -EINVAL;
9585 	}
9586 scx_check:
9587 #endif /* CONFIG_RT_GROUP_SCHED */
9588 	return scx_cgroup_can_attach(tset);
9589 }
9590 
9591 static void cpu_cgroup_attach(struct cgroup_taskset *tset)
9592 {
9593 	struct task_struct *task;
9594 	struct cgroup_subsys_state *css;
9595 
9596 	cgroup_taskset_for_each(task, css, tset)
9597 		sched_move_task(task, false);
9598 }
9599 
9600 static void cpu_cgroup_cancel_attach(struct cgroup_taskset *tset)
9601 {
9602 	scx_cgroup_cancel_attach(tset);
9603 }
9604 
9605 #ifdef CONFIG_UCLAMP_TASK_GROUP
9606 static void cpu_util_update_eff(struct cgroup_subsys_state *css)
9607 {
9608 	struct cgroup_subsys_state *top_css = css;
9609 	struct uclamp_se *uc_parent = NULL;
9610 	struct uclamp_se *uc_se = NULL;
9611 	unsigned int eff[UCLAMP_CNT];
9612 	enum uclamp_id clamp_id;
9613 	unsigned int clamps;
9614 
9615 	lockdep_assert_held(&uclamp_mutex);
9616 	WARN_ON_ONCE(!rcu_read_lock_held());
9617 
9618 	css_for_each_descendant_pre(css, top_css) {
9619 		uc_parent = css_tg(css)->parent
9620 			? css_tg(css)->parent->uclamp : NULL;
9621 
9622 		for_each_clamp_id(clamp_id) {
9623 			/* Assume effective clamps matches requested clamps */
9624 			eff[clamp_id] = css_tg(css)->uclamp_req[clamp_id].value;
9625 			/* Cap effective clamps with parent's effective clamps */
9626 			if (uc_parent &&
9627 			    eff[clamp_id] > uc_parent[clamp_id].value) {
9628 				eff[clamp_id] = uc_parent[clamp_id].value;
9629 			}
9630 		}
9631 		/* Ensure protection is always capped by limit */
9632 		eff[UCLAMP_MIN] = min(eff[UCLAMP_MIN], eff[UCLAMP_MAX]);
9633 
9634 		/* Propagate most restrictive effective clamps */
9635 		clamps = 0x0;
9636 		uc_se = css_tg(css)->uclamp;
9637 		for_each_clamp_id(clamp_id) {
9638 			if (eff[clamp_id] == uc_se[clamp_id].value)
9639 				continue;
9640 			uc_se[clamp_id].value = eff[clamp_id];
9641 			uc_se[clamp_id].bucket_id = uclamp_bucket_id(eff[clamp_id]);
9642 			clamps |= (0x1 << clamp_id);
9643 		}
9644 		if (!clamps) {
9645 			css = css_rightmost_descendant(css);
9646 			continue;
9647 		}
9648 
9649 		/* Immediately update descendants RUNNABLE tasks */
9650 		uclamp_update_active_tasks(css);
9651 	}
9652 }
9653 
9654 /*
9655  * Integer 10^N with a given N exponent by casting to integer the literal "1eN"
9656  * C expression. Since there is no way to convert a macro argument (N) into a
9657  * character constant, use two levels of macros.
9658  */
9659 #define _POW10(exp) ((unsigned int)1e##exp)
9660 #define POW10(exp) _POW10(exp)
9661 
9662 struct uclamp_request {
9663 #define UCLAMP_PERCENT_SHIFT	2
9664 #define UCLAMP_PERCENT_SCALE	(100 * POW10(UCLAMP_PERCENT_SHIFT))
9665 	s64 percent;
9666 	u64 util;
9667 	int ret;
9668 };
9669 
9670 static inline struct uclamp_request
9671 capacity_from_percent(char *buf)
9672 {
9673 	struct uclamp_request req = {
9674 		.percent = UCLAMP_PERCENT_SCALE,
9675 		.util = SCHED_CAPACITY_SCALE,
9676 		.ret = 0,
9677 	};
9678 
9679 	buf = strim(buf);
9680 	if (strcmp(buf, "max")) {
9681 		req.ret = cgroup_parse_float(buf, UCLAMP_PERCENT_SHIFT,
9682 					     &req.percent);
9683 		if (req.ret)
9684 			return req;
9685 		if ((u64)req.percent > UCLAMP_PERCENT_SCALE) {
9686 			req.ret = -ERANGE;
9687 			return req;
9688 		}
9689 
9690 		req.util = req.percent << SCHED_CAPACITY_SHIFT;
9691 		req.util = DIV_ROUND_CLOSEST_ULL(req.util, UCLAMP_PERCENT_SCALE);
9692 	}
9693 
9694 	return req;
9695 }
9696 
9697 static ssize_t cpu_uclamp_write(struct kernfs_open_file *of, char *buf,
9698 				size_t nbytes, loff_t off,
9699 				enum uclamp_id clamp_id)
9700 {
9701 	struct uclamp_request req;
9702 	struct task_group *tg;
9703 
9704 	req = capacity_from_percent(buf);
9705 	if (req.ret)
9706 		return req.ret;
9707 
9708 	sched_uclamp_enable();
9709 
9710 	guard(mutex)(&uclamp_mutex);
9711 	guard(rcu)();
9712 
9713 	tg = css_tg(of_css(of));
9714 	if (tg->uclamp_req[clamp_id].value != req.util)
9715 		uclamp_se_set(&tg->uclamp_req[clamp_id], req.util, false);
9716 
9717 	/*
9718 	 * Because of not recoverable conversion rounding we keep track of the
9719 	 * exact requested value
9720 	 */
9721 	tg->uclamp_pct[clamp_id] = req.percent;
9722 
9723 	/* Update effective clamps to track the most restrictive value */
9724 	cpu_util_update_eff(of_css(of));
9725 
9726 	return nbytes;
9727 }
9728 
9729 static ssize_t cpu_uclamp_min_write(struct kernfs_open_file *of,
9730 				    char *buf, size_t nbytes,
9731 				    loff_t off)
9732 {
9733 	return cpu_uclamp_write(of, buf, nbytes, off, UCLAMP_MIN);
9734 }
9735 
9736 static ssize_t cpu_uclamp_max_write(struct kernfs_open_file *of,
9737 				    char *buf, size_t nbytes,
9738 				    loff_t off)
9739 {
9740 	return cpu_uclamp_write(of, buf, nbytes, off, UCLAMP_MAX);
9741 }
9742 
9743 static inline void cpu_uclamp_print(struct seq_file *sf,
9744 				    enum uclamp_id clamp_id)
9745 {
9746 	struct task_group *tg;
9747 	u64 util_clamp;
9748 	u64 percent;
9749 	u32 rem;
9750 
9751 	scoped_guard (rcu) {
9752 		tg = css_tg(seq_css(sf));
9753 		util_clamp = tg->uclamp_req[clamp_id].value;
9754 	}
9755 
9756 	if (util_clamp == SCHED_CAPACITY_SCALE) {
9757 		seq_puts(sf, "max\n");
9758 		return;
9759 	}
9760 
9761 	percent = tg->uclamp_pct[clamp_id];
9762 	percent = div_u64_rem(percent, POW10(UCLAMP_PERCENT_SHIFT), &rem);
9763 	seq_printf(sf, "%llu.%0*u\n", percent, UCLAMP_PERCENT_SHIFT, rem);
9764 }
9765 
9766 static int cpu_uclamp_min_show(struct seq_file *sf, void *v)
9767 {
9768 	cpu_uclamp_print(sf, UCLAMP_MIN);
9769 	return 0;
9770 }
9771 
9772 static int cpu_uclamp_max_show(struct seq_file *sf, void *v)
9773 {
9774 	cpu_uclamp_print(sf, UCLAMP_MAX);
9775 	return 0;
9776 }
9777 #endif /* CONFIG_UCLAMP_TASK_GROUP */
9778 
9779 #ifdef CONFIG_GROUP_SCHED_WEIGHT
9780 static unsigned long tg_weight(struct task_group *tg)
9781 {
9782 #ifdef CONFIG_FAIR_GROUP_SCHED
9783 	return scale_load_down(tg->shares);
9784 #else
9785 	return sched_weight_from_cgroup(tg->scx.weight);
9786 #endif
9787 }
9788 
9789 static int cpu_shares_write_u64(struct cgroup_subsys_state *css,
9790 				struct cftype *cftype, u64 shareval)
9791 {
9792 	int ret;
9793 
9794 	if (shareval > scale_load_down(ULONG_MAX))
9795 		shareval = MAX_SHARES;
9796 	ret = sched_group_set_shares(css_tg(css), scale_load(shareval));
9797 	if (!ret)
9798 		scx_group_set_weight(css_tg(css),
9799 				     sched_weight_to_cgroup(shareval));
9800 	return ret;
9801 }
9802 
9803 static u64 cpu_shares_read_u64(struct cgroup_subsys_state *css,
9804 			       struct cftype *cft)
9805 {
9806 	return tg_weight(css_tg(css));
9807 }
9808 #endif /* CONFIG_GROUP_SCHED_WEIGHT */
9809 
9810 #ifdef CONFIG_CFS_BANDWIDTH
9811 static DEFINE_MUTEX(cfs_constraints_mutex);
9812 
9813 static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime);
9814 
9815 static int tg_set_cfs_bandwidth(struct task_group *tg,
9816 				u64 period_us, u64 quota_us, u64 burst_us)
9817 {
9818 	int i, ret = 0, runtime_enabled, runtime_was_enabled;
9819 	struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
9820 	u64 period, quota, burst;
9821 
9822 	period = (u64)period_us * NSEC_PER_USEC;
9823 
9824 	if (quota_us == RUNTIME_INF)
9825 		quota = RUNTIME_INF;
9826 	else
9827 		quota = (u64)quota_us * NSEC_PER_USEC;
9828 
9829 	burst = (u64)burst_us * NSEC_PER_USEC;
9830 
9831 	/*
9832 	 * Prevent race between setting of cfs_rq->runtime_enabled and
9833 	 * unthrottle_offline_cfs_rqs().
9834 	 */
9835 	guard(cpus_read_lock)();
9836 	guard(mutex)(&cfs_constraints_mutex);
9837 
9838 	ret = __cfs_schedulable(tg, period, quota);
9839 	if (ret)
9840 		return ret;
9841 
9842 	runtime_enabled = quota != RUNTIME_INF;
9843 	runtime_was_enabled = cfs_b->quota != RUNTIME_INF;
9844 	/*
9845 	 * If we need to toggle cfs_bandwidth_used, off->on must occur
9846 	 * before making related changes, and on->off must occur afterwards
9847 	 */
9848 	if (runtime_enabled && !runtime_was_enabled)
9849 		cfs_bandwidth_usage_inc();
9850 
9851 	scoped_guard (raw_spinlock_irq, &cfs_b->lock) {
9852 		cfs_b->period = ns_to_ktime(period);
9853 		cfs_b->quota = quota;
9854 		cfs_b->burst = burst;
9855 
9856 		__refill_cfs_bandwidth_runtime(cfs_b);
9857 
9858 		/*
9859 		 * Restart the period timer (if active) to handle new
9860 		 * period expiry:
9861 		 */
9862 		if (runtime_enabled)
9863 			start_cfs_bandwidth(cfs_b);
9864 	}
9865 
9866 	for_each_online_cpu(i) {
9867 		struct cfs_rq *cfs_rq = tg_cfs_rq(tg, i);
9868 		struct rq *rq = cfs_rq->rq;
9869 
9870 		guard(rq_lock_irq)(rq);
9871 
9872 		cfs_rq->runtime_enabled = runtime_enabled;
9873 		cfs_rq->runtime_remaining = 1;
9874 
9875 		if (cfs_rq->throttled) {
9876 			update_rq_clock(rq);
9877 			unthrottle_cfs_rq(cfs_rq);
9878 		}
9879 	}
9880 
9881 	if (runtime_was_enabled && !runtime_enabled)
9882 		cfs_bandwidth_usage_dec();
9883 
9884 	return 0;
9885 }
9886 
9887 static u64 tg_get_cfs_period(struct task_group *tg)
9888 {
9889 	u64 cfs_period_us;
9890 
9891 	cfs_period_us = ktime_to_ns(tg->cfs_bandwidth.period);
9892 	do_div(cfs_period_us, NSEC_PER_USEC);
9893 
9894 	return cfs_period_us;
9895 }
9896 
9897 static u64 tg_get_cfs_quota(struct task_group *tg)
9898 {
9899 	u64 quota_us;
9900 
9901 	if (tg->cfs_bandwidth.quota == RUNTIME_INF)
9902 		return RUNTIME_INF;
9903 
9904 	quota_us = tg->cfs_bandwidth.quota;
9905 	do_div(quota_us, NSEC_PER_USEC);
9906 
9907 	return quota_us;
9908 }
9909 
9910 static u64 tg_get_cfs_burst(struct task_group *tg)
9911 {
9912 	u64 burst_us;
9913 
9914 	burst_us = tg->cfs_bandwidth.burst;
9915 	do_div(burst_us, NSEC_PER_USEC);
9916 
9917 	return burst_us;
9918 }
9919 
9920 struct cfs_schedulable_data {
9921 	struct task_group *tg;
9922 	u64 period, quota;
9923 };
9924 
9925 /*
9926  * normalize group quota/period to be quota/max_period
9927  * note: units are usecs
9928  */
9929 static u64 normalize_cfs_quota(struct task_group *tg,
9930 			       struct cfs_schedulable_data *d)
9931 {
9932 	u64 quota, period;
9933 
9934 	if (tg == d->tg) {
9935 		period = d->period;
9936 		quota = d->quota;
9937 	} else {
9938 		period = tg_get_cfs_period(tg);
9939 		quota = tg_get_cfs_quota(tg);
9940 	}
9941 
9942 	/* note: these should typically be equivalent */
9943 	if (quota == RUNTIME_INF || quota == -1)
9944 		return RUNTIME_INF;
9945 
9946 	return to_ratio(period, quota);
9947 }
9948 
9949 static int tg_cfs_schedulable_down(struct task_group *tg, void *data)
9950 {
9951 	struct cfs_schedulable_data *d = data;
9952 	struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
9953 	s64 quota = 0, parent_quota = -1;
9954 
9955 	if (!tg->parent) {
9956 		quota = RUNTIME_INF;
9957 	} else {
9958 		struct cfs_bandwidth *parent_b = &tg->parent->cfs_bandwidth;
9959 
9960 		quota = normalize_cfs_quota(tg, d);
9961 		parent_quota = parent_b->hierarchical_quota;
9962 
9963 		/*
9964 		 * Ensure max(child_quota) <= parent_quota.  On cgroup2,
9965 		 * always take the non-RUNTIME_INF min.  On cgroup1, only
9966 		 * inherit when no limit is set. In both cases this is used
9967 		 * by the scheduler to determine if a given CFS task has a
9968 		 * bandwidth constraint at some higher level.
9969 		 */
9970 		if (cgroup_subsys_on_dfl(cpu_cgrp_subsys)) {
9971 			if (quota == RUNTIME_INF)
9972 				quota = parent_quota;
9973 			else if (parent_quota != RUNTIME_INF)
9974 				quota = min(quota, parent_quota);
9975 		} else {
9976 			if (quota == RUNTIME_INF)
9977 				quota = parent_quota;
9978 			else if (parent_quota != RUNTIME_INF && quota > parent_quota)
9979 				return -EINVAL;
9980 		}
9981 	}
9982 	cfs_b->hierarchical_quota = quota;
9983 
9984 	return 0;
9985 }
9986 
9987 static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota)
9988 {
9989 	struct cfs_schedulable_data data = {
9990 		.tg = tg,
9991 		.period = period,
9992 		.quota = quota,
9993 	};
9994 
9995 	if (quota != RUNTIME_INF) {
9996 		do_div(data.period, NSEC_PER_USEC);
9997 		do_div(data.quota, NSEC_PER_USEC);
9998 	}
9999 
10000 	guard(rcu)();
10001 	return walk_tg_tree(tg_cfs_schedulable_down, tg_nop, &data);
10002 }
10003 
10004 static int cpu_cfs_stat_show(struct seq_file *sf, void *v)
10005 {
10006 	struct task_group *tg = css_tg(seq_css(sf));
10007 	struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
10008 
10009 	seq_printf(sf, "nr_periods %d\n", cfs_b->nr_periods);
10010 	seq_printf(sf, "nr_throttled %d\n", cfs_b->nr_throttled);
10011 	seq_printf(sf, "throttled_time %llu\n", cfs_b->throttled_time);
10012 
10013 	if (schedstat_enabled() && tg != &root_task_group) {
10014 		struct sched_statistics *stats;
10015 		u64 ws = 0;
10016 		int i;
10017 
10018 		for_each_possible_cpu(i) {
10019 			stats = __schedstats_from_se(tg_se(tg, i));
10020 			ws += schedstat_val(stats->wait_sum);
10021 		}
10022 
10023 		seq_printf(sf, "wait_sum %llu\n", ws);
10024 	}
10025 
10026 	seq_printf(sf, "nr_bursts %d\n", cfs_b->nr_burst);
10027 	seq_printf(sf, "burst_time %llu\n", cfs_b->burst_time);
10028 
10029 	return 0;
10030 }
10031 
10032 static u64 throttled_time_self(struct task_group *tg)
10033 {
10034 	int i;
10035 	u64 total = 0;
10036 
10037 	for_each_possible_cpu(i) {
10038 		total += READ_ONCE(tg_cfs_rq(tg, i)->throttled_clock_self_time);
10039 	}
10040 
10041 	return total;
10042 }
10043 
10044 static int cpu_cfs_local_stat_show(struct seq_file *sf, void *v)
10045 {
10046 	struct task_group *tg = css_tg(seq_css(sf));
10047 
10048 	seq_printf(sf, "throttled_time %llu\n", throttled_time_self(tg));
10049 
10050 	return 0;
10051 }
10052 #endif /* CONFIG_CFS_BANDWIDTH */
10053 
10054 #ifdef CONFIG_GROUP_SCHED_BANDWIDTH
10055 const u64 max_bw_quota_period_us = 1 * USEC_PER_SEC; /* 1s */
10056 static const u64 min_bw_quota_period_us = 1 * USEC_PER_MSEC; /* 1ms */
10057 /* More than 203 days if BW_SHIFT equals 20. */
10058 static const u64 max_bw_runtime_us = MAX_BW;
10059 
10060 static void tg_bandwidth(struct task_group *tg,
10061 			 u64 *period_us_p, u64 *quota_us_p, u64 *burst_us_p)
10062 {
10063 #ifdef CONFIG_CFS_BANDWIDTH
10064 	if (period_us_p)
10065 		*period_us_p = tg_get_cfs_period(tg);
10066 	if (quota_us_p)
10067 		*quota_us_p = tg_get_cfs_quota(tg);
10068 	if (burst_us_p)
10069 		*burst_us_p = tg_get_cfs_burst(tg);
10070 #else /* !CONFIG_CFS_BANDWIDTH */
10071 	if (period_us_p)
10072 		*period_us_p = tg->scx.bw_period_us;
10073 	if (quota_us_p)
10074 		*quota_us_p = tg->scx.bw_quota_us;
10075 	if (burst_us_p)
10076 		*burst_us_p = tg->scx.bw_burst_us;
10077 #endif /* CONFIG_CFS_BANDWIDTH */
10078 }
10079 
10080 static u64 cpu_period_read_u64(struct cgroup_subsys_state *css,
10081 			       struct cftype *cft)
10082 {
10083 	u64 period_us;
10084 
10085 	tg_bandwidth(css_tg(css), &period_us, NULL, NULL);
10086 	return period_us;
10087 }
10088 
10089 static int tg_set_bandwidth(struct task_group *tg,
10090 			    u64 period_us, u64 quota_us, u64 burst_us)
10091 {
10092 	const u64 max_usec = U64_MAX / NSEC_PER_USEC;
10093 	int ret = 0;
10094 
10095 	if (tg == &root_task_group)
10096 		return -EINVAL;
10097 
10098 	/* Values should survive translation to nsec */
10099 	if (period_us > max_usec ||
10100 	    (quota_us != RUNTIME_INF && quota_us > max_usec) ||
10101 	    burst_us > max_usec)
10102 		return -EINVAL;
10103 
10104 	/*
10105 	 * Ensure we have some amount of bandwidth every period. This is to
10106 	 * prevent reaching a state of large arrears when throttled via
10107 	 * entity_tick() resulting in prolonged exit starvation.
10108 	 */
10109 	if (quota_us < min_bw_quota_period_us ||
10110 	    period_us < min_bw_quota_period_us)
10111 		return -EINVAL;
10112 
10113 	/*
10114 	 * Likewise, bound things on the other side by preventing insane quota
10115 	 * periods.  This also allows us to normalize in computing quota
10116 	 * feasibility.
10117 	 */
10118 	if (period_us > max_bw_quota_period_us)
10119 		return -EINVAL;
10120 
10121 	/*
10122 	 * Bound quota to defend quota against overflow during bandwidth shift.
10123 	 */
10124 	if (quota_us != RUNTIME_INF && quota_us > max_bw_runtime_us)
10125 		return -EINVAL;
10126 
10127 	if (quota_us != RUNTIME_INF && (burst_us > quota_us ||
10128 					burst_us + quota_us > max_bw_runtime_us))
10129 		return -EINVAL;
10130 
10131 #ifdef CONFIG_CFS_BANDWIDTH
10132 	ret = tg_set_cfs_bandwidth(tg, period_us, quota_us, burst_us);
10133 #endif /* CONFIG_CFS_BANDWIDTH */
10134 	if (!ret)
10135 		scx_group_set_bandwidth(tg, period_us, quota_us, burst_us);
10136 	return ret;
10137 }
10138 
10139 static s64 cpu_quota_read_s64(struct cgroup_subsys_state *css,
10140 			      struct cftype *cft)
10141 {
10142 	u64 quota_us;
10143 
10144 	tg_bandwidth(css_tg(css), NULL, &quota_us, NULL);
10145 	return quota_us;	/* (s64)RUNTIME_INF becomes -1 */
10146 }
10147 
10148 static u64 cpu_burst_read_u64(struct cgroup_subsys_state *css,
10149 			      struct cftype *cft)
10150 {
10151 	u64 burst_us;
10152 
10153 	tg_bandwidth(css_tg(css), NULL, NULL, &burst_us);
10154 	return burst_us;
10155 }
10156 
10157 static int cpu_period_write_u64(struct cgroup_subsys_state *css,
10158 				struct cftype *cftype, u64 period_us)
10159 {
10160 	struct task_group *tg = css_tg(css);
10161 	u64 quota_us, burst_us;
10162 
10163 	tg_bandwidth(tg, NULL, &quota_us, &burst_us);
10164 	return tg_set_bandwidth(tg, period_us, quota_us, burst_us);
10165 }
10166 
10167 static int cpu_quota_write_s64(struct cgroup_subsys_state *css,
10168 			       struct cftype *cftype, s64 quota_us)
10169 {
10170 	struct task_group *tg = css_tg(css);
10171 	u64 period_us, burst_us;
10172 
10173 	if (quota_us < 0)
10174 		quota_us = RUNTIME_INF;
10175 
10176 	tg_bandwidth(tg, &period_us, NULL, &burst_us);
10177 	return tg_set_bandwidth(tg, period_us, quota_us, burst_us);
10178 }
10179 
10180 static int cpu_burst_write_u64(struct cgroup_subsys_state *css,
10181 			       struct cftype *cftype, u64 burst_us)
10182 {
10183 	struct task_group *tg = css_tg(css);
10184 	u64 period_us, quota_us;
10185 
10186 	tg_bandwidth(tg, &period_us, &quota_us, NULL);
10187 	return tg_set_bandwidth(tg, period_us, quota_us, burst_us);
10188 }
10189 #endif /* CONFIG_GROUP_SCHED_BANDWIDTH */
10190 
10191 #ifdef CONFIG_RT_GROUP_SCHED
10192 static int cpu_rt_runtime_write(struct cgroup_subsys_state *css,
10193 				struct cftype *cft, s64 val)
10194 {
10195 	return sched_group_set_rt_runtime(css_tg(css), val);
10196 }
10197 
10198 static s64 cpu_rt_runtime_read(struct cgroup_subsys_state *css,
10199 			       struct cftype *cft)
10200 {
10201 	return sched_group_rt_runtime(css_tg(css));
10202 }
10203 
10204 static int cpu_rt_period_write_uint(struct cgroup_subsys_state *css,
10205 				    struct cftype *cftype, u64 rt_period_us)
10206 {
10207 	return sched_group_set_rt_period(css_tg(css), rt_period_us);
10208 }
10209 
10210 static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css,
10211 				   struct cftype *cft)
10212 {
10213 	return sched_group_rt_period(css_tg(css));
10214 }
10215 #endif /* CONFIG_RT_GROUP_SCHED */
10216 
10217 #ifdef CONFIG_GROUP_SCHED_WEIGHT
10218 static s64 cpu_idle_read_s64(struct cgroup_subsys_state *css,
10219 			       struct cftype *cft)
10220 {
10221 	return css_tg(css)->idle;
10222 }
10223 
10224 static int cpu_idle_write_s64(struct cgroup_subsys_state *css,
10225 				struct cftype *cft, s64 idle)
10226 {
10227 	int ret;
10228 
10229 	ret = sched_group_set_idle(css_tg(css), idle);
10230 	if (!ret)
10231 		scx_group_set_idle(css_tg(css), idle);
10232 	return ret;
10233 }
10234 #endif /* CONFIG_GROUP_SCHED_WEIGHT */
10235 
10236 static struct cftype cpu_legacy_files[] = {
10237 #ifdef CONFIG_GROUP_SCHED_WEIGHT
10238 	{
10239 		.name = "shares",
10240 		.read_u64 = cpu_shares_read_u64,
10241 		.write_u64 = cpu_shares_write_u64,
10242 	},
10243 	{
10244 		.name = "idle",
10245 		.read_s64 = cpu_idle_read_s64,
10246 		.write_s64 = cpu_idle_write_s64,
10247 	},
10248 #endif
10249 #ifdef CONFIG_GROUP_SCHED_BANDWIDTH
10250 	{
10251 		.name = "cfs_period_us",
10252 		.read_u64 = cpu_period_read_u64,
10253 		.write_u64 = cpu_period_write_u64,
10254 	},
10255 	{
10256 		.name = "cfs_quota_us",
10257 		.read_s64 = cpu_quota_read_s64,
10258 		.write_s64 = cpu_quota_write_s64,
10259 	},
10260 	{
10261 		.name = "cfs_burst_us",
10262 		.read_u64 = cpu_burst_read_u64,
10263 		.write_u64 = cpu_burst_write_u64,
10264 	},
10265 #endif
10266 #ifdef CONFIG_CFS_BANDWIDTH
10267 	{
10268 		.name = "stat",
10269 		.seq_show = cpu_cfs_stat_show,
10270 	},
10271 	{
10272 		.name = "stat.local",
10273 		.seq_show = cpu_cfs_local_stat_show,
10274 	},
10275 #endif
10276 #ifdef CONFIG_UCLAMP_TASK_GROUP
10277 	{
10278 		.name = "uclamp.min",
10279 		.flags = CFTYPE_NOT_ON_ROOT,
10280 		.seq_show = cpu_uclamp_min_show,
10281 		.write = cpu_uclamp_min_write,
10282 	},
10283 	{
10284 		.name = "uclamp.max",
10285 		.flags = CFTYPE_NOT_ON_ROOT,
10286 		.seq_show = cpu_uclamp_max_show,
10287 		.write = cpu_uclamp_max_write,
10288 	},
10289 #endif
10290 	{ }	/* Terminate */
10291 };
10292 
10293 #ifdef CONFIG_RT_GROUP_SCHED
10294 static struct cftype rt_group_files[] = {
10295 	{
10296 		.name = "rt_runtime_us",
10297 		.read_s64 = cpu_rt_runtime_read,
10298 		.write_s64 = cpu_rt_runtime_write,
10299 	},
10300 	{
10301 		.name = "rt_period_us",
10302 		.read_u64 = cpu_rt_period_read_uint,
10303 		.write_u64 = cpu_rt_period_write_uint,
10304 	},
10305 	{ }	/* Terminate */
10306 };
10307 
10308 # ifdef CONFIG_RT_GROUP_SCHED_DEFAULT_DISABLED
10309 DEFINE_STATIC_KEY_FALSE(rt_group_sched);
10310 # else
10311 DEFINE_STATIC_KEY_TRUE(rt_group_sched);
10312 # endif
10313 
10314 static int __init setup_rt_group_sched(char *str)
10315 {
10316 	long val;
10317 
10318 	if (kstrtol(str, 0, &val) || val < 0 || val > 1) {
10319 		pr_warn("Unable to set rt_group_sched\n");
10320 		return 1;
10321 	}
10322 	if (val)
10323 		static_branch_enable(&rt_group_sched);
10324 	else
10325 		static_branch_disable(&rt_group_sched);
10326 
10327 	return 1;
10328 }
10329 __setup("rt_group_sched=", setup_rt_group_sched);
10330 
10331 static int __init cpu_rt_group_init(void)
10332 {
10333 	if (!rt_group_sched_enabled())
10334 		return 0;
10335 
10336 	WARN_ON(cgroup_add_legacy_cftypes(&cpu_cgrp_subsys, rt_group_files));
10337 	return 0;
10338 }
10339 subsys_initcall(cpu_rt_group_init);
10340 #endif /* CONFIG_RT_GROUP_SCHED */
10341 
10342 static int cpu_extra_stat_show(struct seq_file *sf,
10343 			       struct cgroup_subsys_state *css)
10344 {
10345 #ifdef CONFIG_CFS_BANDWIDTH
10346 	{
10347 		struct task_group *tg = css_tg(css);
10348 		struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
10349 		u64 throttled_usec, burst_usec;
10350 
10351 		throttled_usec = cfs_b->throttled_time;
10352 		do_div(throttled_usec, NSEC_PER_USEC);
10353 		burst_usec = cfs_b->burst_time;
10354 		do_div(burst_usec, NSEC_PER_USEC);
10355 
10356 		seq_printf(sf, "nr_periods %d\n"
10357 			   "nr_throttled %d\n"
10358 			   "throttled_usec %llu\n"
10359 			   "nr_bursts %d\n"
10360 			   "burst_usec %llu\n",
10361 			   cfs_b->nr_periods, cfs_b->nr_throttled,
10362 			   throttled_usec, cfs_b->nr_burst, burst_usec);
10363 	}
10364 #endif /* CONFIG_CFS_BANDWIDTH */
10365 	return 0;
10366 }
10367 
10368 static int cpu_local_stat_show(struct seq_file *sf,
10369 			       struct cgroup_subsys_state *css)
10370 {
10371 #ifdef CONFIG_CFS_BANDWIDTH
10372 	{
10373 		struct task_group *tg = css_tg(css);
10374 		u64 throttled_self_usec;
10375 
10376 		throttled_self_usec = throttled_time_self(tg);
10377 		do_div(throttled_self_usec, NSEC_PER_USEC);
10378 
10379 		seq_printf(sf, "throttled_usec %llu\n",
10380 			   throttled_self_usec);
10381 	}
10382 #endif
10383 	return 0;
10384 }
10385 
10386 #ifdef CONFIG_GROUP_SCHED_WEIGHT
10387 
10388 static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css,
10389 			       struct cftype *cft)
10390 {
10391 	return sched_weight_to_cgroup(tg_weight(css_tg(css)));
10392 }
10393 
10394 static int cpu_weight_write_u64(struct cgroup_subsys_state *css,
10395 				struct cftype *cft, u64 cgrp_weight)
10396 {
10397 	unsigned long weight;
10398 	int ret;
10399 
10400 	if (cgrp_weight < CGROUP_WEIGHT_MIN || cgrp_weight > CGROUP_WEIGHT_MAX)
10401 		return -ERANGE;
10402 
10403 	weight = sched_weight_from_cgroup(cgrp_weight);
10404 
10405 	ret = sched_group_set_shares(css_tg(css), scale_load(weight));
10406 	if (!ret)
10407 		scx_group_set_weight(css_tg(css), cgrp_weight);
10408 	return ret;
10409 }
10410 
10411 static s64 cpu_weight_nice_read_s64(struct cgroup_subsys_state *css,
10412 				    struct cftype *cft)
10413 {
10414 	unsigned long weight = tg_weight(css_tg(css));
10415 	int last_delta = INT_MAX;
10416 	int prio, delta;
10417 
10418 	/* find the closest nice value to the current weight */
10419 	for (prio = 0; prio < ARRAY_SIZE(sched_prio_to_weight); prio++) {
10420 		delta = abs(sched_prio_to_weight[prio] - weight);
10421 		if (delta >= last_delta)
10422 			break;
10423 		last_delta = delta;
10424 	}
10425 
10426 	return PRIO_TO_NICE(prio - 1 + MAX_RT_PRIO);
10427 }
10428 
10429 static int cpu_weight_nice_write_s64(struct cgroup_subsys_state *css,
10430 				     struct cftype *cft, s64 nice)
10431 {
10432 	unsigned long weight;
10433 	int idx, ret;
10434 
10435 	if (nice < MIN_NICE || nice > MAX_NICE)
10436 		return -ERANGE;
10437 
10438 	idx = NICE_TO_PRIO(nice) - MAX_RT_PRIO;
10439 	idx = array_index_nospec(idx, 40);
10440 	weight = sched_prio_to_weight[idx];
10441 
10442 	ret = sched_group_set_shares(css_tg(css), scale_load(weight));
10443 	if (!ret)
10444 		scx_group_set_weight(css_tg(css),
10445 				     sched_weight_to_cgroup(weight));
10446 	return ret;
10447 }
10448 #endif /* CONFIG_GROUP_SCHED_WEIGHT */
10449 
10450 static void __maybe_unused cpu_period_quota_print(struct seq_file *sf,
10451 						  long period, long quota)
10452 {
10453 	if (quota < 0)
10454 		seq_puts(sf, "max");
10455 	else
10456 		seq_printf(sf, "%ld", quota);
10457 
10458 	seq_printf(sf, " %ld\n", period);
10459 }
10460 
10461 /* caller should put the current value in *@periodp before calling */
10462 static int __maybe_unused cpu_period_quota_parse(char *buf, u64 *period_us_p,
10463 						 u64 *quota_us_p)
10464 {
10465 	char tok[21];	/* U64_MAX */
10466 
10467 	if (sscanf(buf, "%20s %llu", tok, period_us_p) < 1)
10468 		return -EINVAL;
10469 
10470 	if (sscanf(tok, "%llu", quota_us_p) < 1) {
10471 		if (!strcmp(tok, "max"))
10472 			*quota_us_p = RUNTIME_INF;
10473 		else
10474 			return -EINVAL;
10475 	}
10476 
10477 	return 0;
10478 }
10479 
10480 #ifdef CONFIG_GROUP_SCHED_BANDWIDTH
10481 static int cpu_max_show(struct seq_file *sf, void *v)
10482 {
10483 	struct task_group *tg = css_tg(seq_css(sf));
10484 	u64 period_us, quota_us;
10485 
10486 	tg_bandwidth(tg, &period_us, &quota_us, NULL);
10487 	cpu_period_quota_print(sf, period_us, quota_us);
10488 	return 0;
10489 }
10490 
10491 static ssize_t cpu_max_write(struct kernfs_open_file *of,
10492 			     char *buf, size_t nbytes, loff_t off)
10493 {
10494 	struct task_group *tg = css_tg(of_css(of));
10495 	u64 period_us, quota_us, burst_us;
10496 	int ret;
10497 
10498 	tg_bandwidth(tg, &period_us, NULL, &burst_us);
10499 	ret = cpu_period_quota_parse(buf, &period_us, &quota_us);
10500 	if (!ret)
10501 		ret = tg_set_bandwidth(tg, period_us, quota_us, burst_us);
10502 	return ret ?: nbytes;
10503 }
10504 #endif /* CONFIG_CFS_BANDWIDTH */
10505 
10506 static struct cftype cpu_files[] = {
10507 #ifdef CONFIG_GROUP_SCHED_WEIGHT
10508 	{
10509 		.name = "weight",
10510 		.flags = CFTYPE_NOT_ON_ROOT,
10511 		.read_u64 = cpu_weight_read_u64,
10512 		.write_u64 = cpu_weight_write_u64,
10513 	},
10514 	{
10515 		.name = "weight.nice",
10516 		.flags = CFTYPE_NOT_ON_ROOT,
10517 		.read_s64 = cpu_weight_nice_read_s64,
10518 		.write_s64 = cpu_weight_nice_write_s64,
10519 	},
10520 	{
10521 		.name = "idle",
10522 		.flags = CFTYPE_NOT_ON_ROOT,
10523 		.read_s64 = cpu_idle_read_s64,
10524 		.write_s64 = cpu_idle_write_s64,
10525 	},
10526 #endif
10527 #ifdef CONFIG_GROUP_SCHED_BANDWIDTH
10528 	{
10529 		.name = "max",
10530 		.flags = CFTYPE_NOT_ON_ROOT,
10531 		.seq_show = cpu_max_show,
10532 		.write = cpu_max_write,
10533 	},
10534 	{
10535 		.name = "max.burst",
10536 		.flags = CFTYPE_NOT_ON_ROOT,
10537 		.read_u64 = cpu_burst_read_u64,
10538 		.write_u64 = cpu_burst_write_u64,
10539 	},
10540 #endif /* CONFIG_CFS_BANDWIDTH */
10541 #ifdef CONFIG_UCLAMP_TASK_GROUP
10542 	{
10543 		.name = "uclamp.min",
10544 		.flags = CFTYPE_NOT_ON_ROOT,
10545 		.seq_show = cpu_uclamp_min_show,
10546 		.write = cpu_uclamp_min_write,
10547 	},
10548 	{
10549 		.name = "uclamp.max",
10550 		.flags = CFTYPE_NOT_ON_ROOT,
10551 		.seq_show = cpu_uclamp_max_show,
10552 		.write = cpu_uclamp_max_write,
10553 	},
10554 #endif /* CONFIG_UCLAMP_TASK_GROUP */
10555 	{ }	/* terminate */
10556 };
10557 
10558 struct cgroup_subsys cpu_cgrp_subsys = {
10559 	.css_alloc	= cpu_cgroup_css_alloc,
10560 	.css_online	= cpu_cgroup_css_online,
10561 	.css_offline	= cpu_cgroup_css_offline,
10562 	.css_released	= cpu_cgroup_css_released,
10563 	.css_free	= cpu_cgroup_css_free,
10564 	.css_extra_stat_show = cpu_extra_stat_show,
10565 	.css_local_stat_show = cpu_local_stat_show,
10566 	.can_attach	= cpu_cgroup_can_attach,
10567 	.attach		= cpu_cgroup_attach,
10568 	.cancel_attach	= cpu_cgroup_cancel_attach,
10569 	.legacy_cftypes	= cpu_legacy_files,
10570 	.dfl_cftypes	= cpu_files,
10571 	.early_init	= true,
10572 	.threaded	= true,
10573 };
10574 
10575 #endif /* CONFIG_CGROUP_SCHED */
10576 
10577 void dump_cpu_task(int cpu)
10578 {
10579 	if (in_hardirq() && cpu == smp_processor_id()) {
10580 		struct pt_regs *regs;
10581 
10582 		regs = get_irq_regs();
10583 		if (regs) {
10584 			show_regs(regs);
10585 			return;
10586 		}
10587 	}
10588 
10589 	if (trigger_single_cpu_backtrace(cpu))
10590 		return;
10591 
10592 	pr_info("Task dump for CPU %d:\n", cpu);
10593 	sched_show_task(cpu_curr(cpu));
10594 }
10595 
10596 /*
10597  * Nice levels are multiplicative, with a gentle 10% change for every
10598  * nice level changed. I.e. when a CPU-bound task goes from nice 0 to
10599  * nice 1, it will get ~10% less CPU time than another CPU-bound task
10600  * that remained on nice 0.
10601  *
10602  * The "10% effect" is relative and cumulative: from _any_ nice level,
10603  * if you go up 1 level, it's -10% CPU usage, if you go down 1 level
10604  * it's +10% CPU usage. (to achieve that we use a multiplier of 1.25.
10605  * If a task goes up by ~10% and another task goes down by ~10% then
10606  * the relative distance between them is ~25%.)
10607  */
10608 const int sched_prio_to_weight[40] = {
10609  /* -20 */     88761,     71755,     56483,     46273,     36291,
10610  /* -15 */     29154,     23254,     18705,     14949,     11916,
10611  /* -10 */      9548,      7620,      6100,      4904,      3906,
10612  /*  -5 */      3121,      2501,      1991,      1586,      1277,
10613  /*   0 */      1024,       820,       655,       526,       423,
10614  /*   5 */       335,       272,       215,       172,       137,
10615  /*  10 */       110,        87,        70,        56,        45,
10616  /*  15 */        36,        29,        23,        18,        15,
10617 };
10618 
10619 /*
10620  * Inverse (2^32/x) values of the sched_prio_to_weight[] array, pre-calculated.
10621  *
10622  * In cases where the weight does not change often, we can use the
10623  * pre-calculated inverse to speed up arithmetics by turning divisions
10624  * into multiplications:
10625  */
10626 const u32 sched_prio_to_wmult[40] = {
10627  /* -20 */     48388,     59856,     76040,     92818,    118348,
10628  /* -15 */    147320,    184698,    229616,    287308,    360437,
10629  /* -10 */    449829,    563644,    704093,    875809,   1099582,
10630  /*  -5 */   1376151,   1717300,   2157191,   2708050,   3363326,
10631  /*   0 */   4194304,   5237765,   6557202,   8165337,  10153587,
10632  /*   5 */  12820798,  15790321,  19976592,  24970740,  31350126,
10633  /*  10 */  39045157,  49367440,  61356676,  76695844,  95443717,
10634  /*  15 */ 119304647, 148102320, 186737708, 238609294, 286331153,
10635 };
10636 
10637 void call_trace_sched_update_nr_running(struct rq *rq, int count)
10638 {
10639         trace_sched_update_nr_running_tp(rq, count);
10640 }
10641 
10642 #ifdef CONFIG_SCHED_MM_CID
10643 /*
10644  * Concurrency IDentifier management
10645  *
10646  * Serialization rules:
10647  *
10648  * mm::mm_cid::mutex:	Serializes fork() and exit() and therefore
10649  *			protects mm::mm_cid::users and mode switch
10650  *			transitions
10651  *
10652  * mm::mm_cid::lock:	Serializes mm_update_max_cids() and
10653  *			mm_update_cpus_allowed(). Nests in mm_cid::mutex
10654  *			and runqueue lock.
10655  *
10656  * The mm_cidmask bitmap is not protected by any of the mm::mm_cid locks
10657  * and can only be modified with atomic operations.
10658  *
10659  * The mm::mm_cid:pcpu per CPU storage is protected by the CPUs runqueue
10660  * lock.
10661  *
10662  * CID ownership:
10663  *
10664  * A CID is either owned by a task (stored in task_struct::mm_cid.cid) or
10665  * by a CPU (stored in mm::mm_cid.pcpu::cid). CIDs owned by CPUs have the
10666  * MM_CID_ONCPU bit set.
10667  *
10668  * During the transition of ownership mode, the MM_CID_TRANSIT bit is set
10669  * on the CIDs. When this bit is set the tasks drop the CID back into the
10670  * pool when scheduling out.
10671  *
10672  * Both bits (ONCPU and TRANSIT) are filtered out by task_cid() when the
10673  * CID is actually handed over to user space in the RSEQ memory.
10674  *
10675  * Mode switching:
10676  *
10677  * The ownership mode is per process and stored in mm:mm_cid::mode with the
10678  * following possible states:
10679  *
10680  *	0:				Per task ownership
10681  *	0 | MM_CID_TRANSIT:		Transition from per CPU to per task
10682  *	MM_CID_ONCPU:			Per CPU ownership
10683  *	MM_CID_ONCPU | MM_CID_TRANSIT:	Transition from per task to per CPU
10684  *
10685  * All transitions of ownership mode happen in two phases:
10686  *
10687  *  1) mm:mm_cid::mode has the MM_CID_TRANSIT bit set. This is OR'ed on the
10688  *     CIDs and denotes that the CID is only temporarily owned by a
10689  *     task. When the task schedules out it drops the CID back into the
10690  *     pool if this bit is set.
10691  *
10692  *  2) The initiating context walks the per CPU space or the tasks to fixup
10693  *     or drop the CIDs and after completion it clears MM_CID_TRANSIT in
10694  *     mm:mm_cid::mode. After that point the CIDs are strictly task or CPU
10695  *     owned again.
10696  *
10697  * This two phase transition is required to prevent CID space exhaustion
10698  * during the transition as a direct transfer of ownership would fail:
10699  *
10700  *   - On task to CPU mode switch if a task is scheduled in on one CPU and
10701  *     then migrated to another CPU before the fixup freed enough per task
10702  *     CIDs.
10703  *
10704  *   - On CPU to task mode switch if two tasks are scheduled in on the same
10705  *     CPU before the fixup freed per CPU CIDs.
10706  *
10707  *   Both scenarios can result in a live lock because sched_in() is invoked
10708  *   with runqueue lock held and loops in search of a CID and the fixup
10709  *   thread can't make progress freeing them up because it is stuck on the
10710  *   same runqueue lock.
10711  *
10712  * While MM_CID_TRANSIT is active during the transition phase the MM_CID
10713  * bitmap can be contended, but that's a temporary contention bound to the
10714  * transition period. After that everything goes back into steady state and
10715  * nothing except fork() and exit() will touch the bitmap. This is an
10716  * acceptable tradeoff as it completely avoids complex serialization,
10717  * memory barriers and atomic operations for the common case.
10718  *
10719  * Aside of that this mechanism also ensures RT compability:
10720  *
10721  *   - The task which runs the fixup is fully preemptible except for the
10722  *     short runqueue lock held sections.
10723  *
10724  *   - The transient impact of the bitmap contention is only problematic
10725  *     when there is a thundering herd scenario of tasks scheduling in and
10726  *     out concurrently. There is not much which can be done about that
10727  *     except for avoiding mode switching by a proper overall system
10728  *     configuration.
10729  *
10730  * Switching to per CPU mode happens when the user count becomes greater
10731  * than the maximum number of CIDs, which is calculated by:
10732  *
10733  *	opt_cids = min(mm_cid::nr_cpus_allowed, mm_cid::users);
10734  *	max_cids = min(1.25 * opt_cids, num_possible_cpus());
10735  *
10736  * The +25% allowance is useful for tight CPU masks in scenarios where only
10737  * a few threads are created and destroyed to avoid frequent mode
10738  * switches. Though this allowance shrinks, the closer opt_cids becomes to
10739  * num_possible_cpus(), which is the (unfortunate) hard ABI limit.
10740  *
10741  * At the point of switching to per CPU mode the new user is not yet
10742  * visible in the system, so the task which initiated the fork() runs the
10743  * fixup function. mm_cid_fixup_tasks_to_cpu() walks the thread list and
10744  * either marks each task owned CID with MM_CID_TRANSIT if the task is
10745  * running on a CPU or drops it into the CID pool if a task is not on a
10746  * CPU. Tasks which schedule in before the task walk reaches them do the
10747  * handover in mm_cid_schedin(). When mm_cid_fixup_tasks_to_cpus()
10748  * completes it is guaranteed that no task related to that MM owns a CID
10749  * anymore.
10750  *
10751  * Switching back to task mode happens when the user count goes below the
10752  * threshold which was recorded on the per CPU mode switch:
10753  *
10754  *	pcpu_thrs = min(opt_cids - (opt_cids / 4), num_possible_cpus() / 2);
10755  *
10756  * This threshold is updated when a affinity change increases the number of
10757  * allowed CPUs for the MM, which might cause a switch back to per task
10758  * mode.
10759  *
10760  * If the switch back was initiated by a exiting task, then that task runs
10761  * the fixup function. If it was initiated by a affinity change, then it's
10762  * run either in the deferred update function in context of a workqueue or
10763  * by a task which forks a new one or by a task which exits. Whatever
10764  * happens first. mm_cid_fixup_cpus_to_task() walks through the possible
10765  * CPUs and either marks the CPU owned CIDs with MM_CID_TRANSIT if a
10766  * related task is running on the CPU or drops it into the pool. Tasks
10767  * which are scheduled in before the fixup covered them do the handover
10768  * themself. When mm_cid_fixup_cpus_to_tasks() completes it is guaranteed
10769  * that no CID related to that MM is owned by a CPU anymore.
10770  */
10771 
10772 /*
10773  * Update the CID range properties when the constraints change. Invoked via
10774  * fork(), exit() and affinity changes
10775  */
10776 static void __mm_update_max_cids(struct mm_mm_cid *mc)
10777 {
10778 	unsigned int opt_cids, max_cids;
10779 
10780 	/* Calculate the new optimal constraint */
10781 	opt_cids = min(mc->nr_cpus_allowed, mc->users);
10782 
10783 	/* Adjust the maximum CIDs to +25% limited by the number of possible CPUs */
10784 	max_cids = min(opt_cids + (opt_cids / 4), num_possible_cpus());
10785 	WRITE_ONCE(mc->max_cids, max_cids);
10786 }
10787 
10788 static inline unsigned int mm_cid_calc_pcpu_thrs(struct mm_mm_cid *mc)
10789 {
10790 	unsigned int nr_cpus = mc->nr_cpus_allowed;
10791 
10792 	/* Has to be at least 1 because 0 indicates PCPU mode off */
10793 	return max(min(nr_cpus - nr_cpus / 4, num_possible_cpus() / 2), 1);
10794 }
10795 
10796 static bool mm_update_max_cids(struct mm_struct *mm)
10797 {
10798 	struct mm_mm_cid *mc = &mm->mm_cid;
10799 	bool percpu = cid_on_cpu(mc->mode);
10800 
10801 	lockdep_assert_held(&mm->mm_cid.lock);
10802 
10803 	/* Clear deferred mode switch flag. A change is handled by the caller */
10804 	mc->update_deferred = false;
10805 	__mm_update_max_cids(mc);
10806 
10807 	/* Check whether owner mode must be changed */
10808 	if (!percpu) {
10809 		/* Enable per CPU mode when the number of users is above max_cids */
10810 		if (mc->users > mc->max_cids)
10811 			mc->pcpu_thrs = mm_cid_calc_pcpu_thrs(mc);
10812 	} else {
10813 		/* Switch back to per task if user count under threshold */
10814 		if (mc->users < mc->pcpu_thrs)
10815 			mc->pcpu_thrs = 0;
10816 	}
10817 
10818 	/* Mode change required? */
10819 	if (percpu == !!mc->pcpu_thrs)
10820 		return false;
10821 
10822 	/* Flip the mode and set the transition flag to bridge the transfer */
10823 	WRITE_ONCE(mc->mode, mc->mode ^ (MM_CID_TRANSIT | MM_CID_ONCPU));
10824 	/*
10825 	 * Order the store against the subsequent fixups so that
10826 	 * acquire(rq::lock) cannot be reordered by the CPU before the
10827 	 * store.
10828 	 */
10829 	smp_mb();
10830 	return true;
10831 }
10832 
10833 static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk)
10834 {
10835 	struct cpumask *mm_allowed;
10836 	struct mm_mm_cid *mc;
10837 	unsigned int weight;
10838 
10839 	if (!mm || !READ_ONCE(mm->mm_cid.users))
10840 		return;
10841 	/*
10842 	 * mm::mm_cid::mm_cpus_allowed is the superset of each threads
10843 	 * allowed CPUs mask which means it can only grow.
10844 	 */
10845 	mc = &mm->mm_cid;
10846 	guard(raw_spinlock)(&mc->lock);
10847 	mm_allowed = mm_cpus_allowed(mm);
10848 	weight = cpumask_weighted_or(mm_allowed, mm_allowed, affmsk);
10849 	if (weight == mc->nr_cpus_allowed)
10850 		return;
10851 
10852 	WRITE_ONCE(mc->nr_cpus_allowed, weight);
10853 	__mm_update_max_cids(mc);
10854 	if (!cid_on_cpu(mc->mode))
10855 		return;
10856 
10857 	/* Adjust the threshold to the wider set */
10858 	mc->pcpu_thrs = mm_cid_calc_pcpu_thrs(mc);
10859 	/* Switch back to per task mode? */
10860 	if (mc->users >= mc->pcpu_thrs)
10861 		return;
10862 
10863 	/* Don't queue twice */
10864 	if (mc->update_deferred)
10865 		return;
10866 
10867 	/* Queue the irq work, which schedules the real work */
10868 	mc->update_deferred = true;
10869 	irq_work_queue(&mc->irq_work);
10870 }
10871 
10872 static inline void mm_cid_complete_transit(struct mm_struct *mm, unsigned int mode)
10873 {
10874 	/*
10875 	 * Ensure that the store removing the TRANSIT bit cannot be
10876 	 * reordered by the CPU before the fixups have been completed.
10877 	 */
10878 	smp_mb();
10879 	WRITE_ONCE(mm->mm_cid.mode, mode);
10880 }
10881 
10882 static inline void mm_cid_transit_to_task(struct task_struct *t, struct mm_cid_pcpu *pcp)
10883 {
10884 	if (cid_on_cpu(t->mm_cid.cid)) {
10885 		unsigned int cid = cpu_cid_to_cid(t->mm_cid.cid);
10886 
10887 		t->mm_cid.cid = cid_to_transit_cid(cid);
10888 		pcp->cid = t->mm_cid.cid;
10889 	}
10890 }
10891 
10892 static void mm_cid_fixup_cpus_to_tasks(struct mm_struct *mm)
10893 {
10894 	unsigned int cpu;
10895 
10896 	/* Walk the CPUs and fixup all stale CIDs */
10897 	for_each_possible_cpu(cpu) {
10898 		struct mm_cid_pcpu *pcp = per_cpu_ptr(mm->mm_cid.pcpu, cpu);
10899 		struct rq *rq = cpu_rq(cpu);
10900 
10901 		/* Remote access to mm::mm_cid::pcpu requires rq_lock */
10902 		guard(rq_lock_irq)(rq);
10903 		/* Is the CID still owned by the CPU? */
10904 		if (cid_on_cpu(pcp->cid)) {
10905 			/*
10906 			 * If rq->curr has @mm, transfer it with the
10907 			 * transition bit set. Otherwise drop it.
10908 			 */
10909 			if (rq->curr->mm == mm && rq->curr->mm_cid.active)
10910 				mm_cid_transit_to_task(rq->curr, pcp);
10911 			else
10912 				mm_drop_cid_on_cpu(mm, pcp);
10913 
10914 		} else if (rq->curr->mm == mm && rq->curr->mm_cid.active) {
10915 			unsigned int cid = rq->curr->mm_cid.cid;
10916 
10917 			/*
10918 			 * Set the transition bit only on a genuine task-owned
10919 			 * CID. A running active task can legitimately have
10920 			 * MM_CID_UNSET here: in per-CPU mode CIDs are assigned
10921 			 * lazily on schedule-in, so the fork()/execve() window
10922 			 * leaves the task active with no owned CID. Setting the
10923 			 * transition bit on MM_CID_UNSET would later feed
10924 			 * clear_bit() an out-of-bounds bit number via
10925 			 * mm_cid_schedout(), so exclude it. A CPU-owned
10926 			 * (MM_CID_ONCPU) CID is handled by the cid_on_cpu()
10927 			 * branch above and never reaches here.
10928 			 */
10929 			if (cid != MM_CID_UNSET && !cid_in_transit(cid)) {
10930 				cid = cid_to_transit_cid(cid);
10931 				rq->curr->mm_cid.cid = cid;
10932 				pcp->cid = cid;
10933 			}
10934 		}
10935 	}
10936 	mm_cid_complete_transit(mm, 0);
10937 }
10938 
10939 static inline void mm_cid_transit_to_cpu(struct task_struct *t, struct mm_cid_pcpu *pcp)
10940 {
10941 	if (cid_on_task(t->mm_cid.cid)) {
10942 		t->mm_cid.cid = cid_to_transit_cid(t->mm_cid.cid);
10943 		pcp->cid = t->mm_cid.cid;
10944 	}
10945 }
10946 
10947 static void mm_cid_fixup_task_to_cpu(struct task_struct *t, struct mm_struct *mm)
10948 {
10949 	/* Remote access to mm::mm_cid::pcpu requires rq_lock */
10950 	guard(task_rq_lock)(t);
10951 	if (cid_on_task(t->mm_cid.cid)) {
10952 		/* If running on the CPU, put the CID in transit mode, otherwise drop it */
10953 		if (task_rq(t)->curr == t)
10954 			mm_cid_transit_to_cpu(t, per_cpu_ptr(mm->mm_cid.pcpu, task_cpu(t)));
10955 		else
10956 			mm_unset_cid_on_task(t);
10957 	}
10958 }
10959 
10960 static void mm_cid_fixup_tasks_to_cpus(void)
10961 {
10962 	struct mm_struct *mm = current->mm;
10963 	struct task_struct *t;
10964 
10965 	lockdep_assert_held(&mm->mm_cid.mutex);
10966 
10967 	hlist_for_each_entry(t, &mm->mm_cid.user_list, mm_cid.node) {
10968 		/* Current has already transferred before invoking the fixup. */
10969 		if (t != current)
10970 			mm_cid_fixup_task_to_cpu(t, mm);
10971 	}
10972 
10973 	mm_cid_complete_transit(mm, MM_CID_ONCPU);
10974 }
10975 
10976 static bool sched_mm_cid_add_user(struct task_struct *t, struct mm_struct *mm)
10977 {
10978 	lockdep_assert_held(&mm->mm_cid.lock);
10979 
10980 	t->mm_cid.active = 1;
10981 	hlist_add_head(&t->mm_cid.node, &mm->mm_cid.user_list);
10982 	mm->mm_cid.users++;
10983 	return mm_update_max_cids(mm);
10984 }
10985 
10986 static void sched_mm_cid_fork(struct task_struct *t)
10987 {
10988 	struct mm_struct *mm = t->mm;
10989 	bool percpu;
10990 
10991 	if (!mm)
10992 		return;
10993 
10994 	WARN_ON_ONCE(t->mm_cid.cid != MM_CID_UNSET);
10995 
10996 	guard(mutex)(&mm->mm_cid.mutex);
10997 	scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
10998 		struct mm_cid_pcpu *pcp = this_cpu_ptr(mm->mm_cid.pcpu);
10999 
11000 		/* First user ? */
11001 		if (!mm->mm_cid.users) {
11002 			sched_mm_cid_add_user(t, mm);
11003 			t->mm_cid.cid = mm_get_cid(mm);
11004 			/* Required for execve() */
11005 			pcp->cid = t->mm_cid.cid;
11006 			return;
11007 		}
11008 
11009 		if (!sched_mm_cid_add_user(t, mm)) {
11010 			if (!cid_on_cpu(mm->mm_cid.mode))
11011 				t->mm_cid.cid = mm_get_cid(mm);
11012 			return;
11013 		}
11014 
11015 		/* Handle the mode change and transfer current's CID */
11016 		percpu = cid_on_cpu(mm->mm_cid.mode);
11017 		if (!percpu)
11018 			mm_cid_transit_to_task(current, pcp);
11019 		else
11020 			mm_cid_transit_to_cpu(current, pcp);
11021 	}
11022 
11023 	if (percpu) {
11024 		mm_cid_fixup_tasks_to_cpus();
11025 	} else {
11026 		mm_cid_fixup_cpus_to_tasks(mm);
11027 		t->mm_cid.cid = mm_get_cid(mm);
11028 	}
11029 }
11030 
11031 static bool sched_mm_cid_remove_user(struct task_struct *t)
11032 {
11033 	lockdep_assert_held(&t->mm->mm_cid.lock);
11034 
11035 	t->mm_cid.active = 0;
11036 	/* Clear the transition bit */
11037 	t->mm_cid.cid = cid_from_transit_cid(t->mm_cid.cid);
11038 	mm_unset_cid_on_task(t);
11039 	hlist_del_init(&t->mm_cid.node);
11040 	t->mm->mm_cid.users--;
11041 	return mm_update_max_cids(t->mm);
11042 }
11043 
11044 static bool __sched_mm_cid_exit(struct task_struct *t)
11045 {
11046 	struct mm_struct *mm = t->mm;
11047 
11048 	if (!sched_mm_cid_remove_user(t))
11049 		return false;
11050 	/*
11051 	 * Contrary to fork() this only deals with a switch back to per
11052 	 * task mode either because the above decreased users or an
11053 	 * affinity change increased the number of allowed CPUs and the
11054 	 * deferred fixup did not run yet.
11055 	 */
11056 	if (WARN_ON_ONCE(cid_on_cpu(mm->mm_cid.mode)))
11057 		return false;
11058 	/*
11059 	 * A failed fork(2) cleanup never gets here, so @current must have
11060 	 * the same MM as @t. That's true for exit() and the failed
11061 	 * pthread_create() cleanup case.
11062 	 */
11063 	if (WARN_ON_ONCE(current->mm != mm))
11064 		return false;
11065 	return true;
11066 }
11067 
11068 /*
11069  * When a task exits, the MM CID held by the task is not longer required as
11070  * the task cannot return to user space.
11071  */
11072 void sched_mm_cid_exit(struct task_struct *t)
11073 {
11074 	struct mm_struct *mm = t->mm;
11075 
11076 	if (!mm || !t->mm_cid.active)
11077 		return;
11078 	/*
11079 	 * Ensure that only one instance is doing MM CID operations within
11080 	 * a MM. The common case is uncontended. The rare fixup case adds
11081 	 * some overhead.
11082 	 */
11083 	scoped_guard(mutex, &mm->mm_cid.mutex) {
11084 		/* mm_cid::mutex is sufficient to protect mm_cid::users */
11085 		if (likely(mm->mm_cid.users > 1)) {
11086 			scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
11087 				if (!__sched_mm_cid_exit(t))
11088 					return;
11089 				/*
11090 				 * Mode change. The task has the CID unset
11091 				 * already and dealt with an eventually set
11092 				 * TRANSIT bit. If the CID is owned by the CPU
11093 				 * then drop it.
11094 				 */
11095 				mm_drop_cid_on_cpu(mm, this_cpu_ptr(mm->mm_cid.pcpu));
11096 			}
11097 			mm_cid_fixup_cpus_to_tasks(mm);
11098 			return;
11099 		}
11100 		/* Last user */
11101 		scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
11102 			/* Required across execve() */
11103 			if (t == current)
11104 				mm_cid_transit_to_task(t, this_cpu_ptr(mm->mm_cid.pcpu));
11105 			/* Ignore mode change. There is nothing to do. */
11106 			sched_mm_cid_remove_user(t);
11107 		}
11108 	}
11109 
11110 	/*
11111 	 * As this is the last user (execve(), process exit or failed
11112 	 * fork(2)) there is no concurrency anymore.
11113 	 *
11114 	 * Synchronize eventually pending work to ensure that there are no
11115 	 * dangling references left. @t->mm_cid.users is zero so nothing
11116 	 * can queue this work anymore.
11117 	 */
11118 	irq_work_sync(&mm->mm_cid.irq_work);
11119 	cancel_work_sync(&mm->mm_cid.work);
11120 }
11121 
11122 /* Deactivate MM CID allocation across execve() */
11123 void sched_mm_cid_before_execve(struct task_struct *t)
11124 {
11125 	sched_mm_cid_exit(t);
11126 }
11127 
11128 /* Reactivate MM CID after execve() */
11129 void sched_mm_cid_after_execve(struct task_struct *t)
11130 {
11131 	if (t->mm)
11132 		sched_mm_cid_fork(t);
11133 }
11134 
11135 static void mm_cid_work_fn(struct work_struct *work)
11136 {
11137 	struct mm_struct *mm = container_of(work, struct mm_struct, mm_cid.work);
11138 
11139 	guard(mutex)(&mm->mm_cid.mutex);
11140 	/* Did the last user task exit already? */
11141 	if (!mm->mm_cid.users)
11142 		return;
11143 
11144 	scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
11145 		/* Have fork() or exit() handled it already? */
11146 		if (!mm->mm_cid.update_deferred)
11147 			return;
11148 		/* This clears mm_cid::update_deferred */
11149 		if (!mm_update_max_cids(mm))
11150 			return;
11151 		/* Affinity changes can only switch back to task mode */
11152 		if (WARN_ON_ONCE(cid_on_cpu(mm->mm_cid.mode)))
11153 			return;
11154 	}
11155 	mm_cid_fixup_cpus_to_tasks(mm);
11156 }
11157 
11158 static void mm_cid_irq_work(struct irq_work *work)
11159 {
11160 	struct mm_struct *mm = container_of(work, struct mm_struct, mm_cid.irq_work);
11161 
11162 	/*
11163 	 * Needs to be unconditional because mm_cid::lock cannot be held
11164 	 * when scheduling work as mm_update_cpus_allowed() nests inside
11165 	 * rq::lock and schedule_work() might end up in wakeup...
11166 	 */
11167 	schedule_work(&mm->mm_cid.work);
11168 }
11169 
11170 void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
11171 {
11172 	mm->mm_cid.max_cids = 0;
11173 	mm->mm_cid.mode = 0;
11174 	mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
11175 	mm->mm_cid.users = 0;
11176 	mm->mm_cid.pcpu_thrs = 0;
11177 	mm->mm_cid.update_deferred = 0;
11178 	raw_spin_lock_init(&mm->mm_cid.lock);
11179 	mutex_init(&mm->mm_cid.mutex);
11180 	mm->mm_cid.irq_work = IRQ_WORK_INIT_HARD(mm_cid_irq_work);
11181 	INIT_WORK(&mm->mm_cid.work, mm_cid_work_fn);
11182 	INIT_HLIST_HEAD(&mm->mm_cid.user_list);
11183 	cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
11184 	bitmap_zero(mm_cidmask(mm), num_possible_cpus());
11185 }
11186 #else /* CONFIG_SCHED_MM_CID */
11187 static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk) { }
11188 static inline void sched_mm_cid_fork(struct task_struct *t) { }
11189 #endif /* !CONFIG_SCHED_MM_CID */
11190 
11191 static DEFINE_PER_CPU(struct sched_change_ctx, sched_change_ctx);
11192 
11193 struct sched_change_ctx *sched_change_begin(struct task_struct *p, unsigned int flags)
11194 {
11195 	struct sched_change_ctx *ctx = this_cpu_ptr(&sched_change_ctx);
11196 	struct rq *rq = task_rq(p);
11197 
11198 	/*
11199 	 * Must exclusively use matched flags since this is both dequeue and
11200 	 * enqueue.
11201 	 */
11202 	WARN_ON_ONCE(flags & 0xFFFF0000);
11203 
11204 	lockdep_assert_rq_held(rq);
11205 
11206 	if (!(flags & DEQUEUE_NOCLOCK)) {
11207 		update_rq_clock(rq);
11208 		flags |= DEQUEUE_NOCLOCK;
11209 	}
11210 
11211 	if ((flags & DEQUEUE_CLASS) && p->sched_class->switching_from)
11212 		p->sched_class->switching_from(rq, p);
11213 
11214 	*ctx = (struct sched_change_ctx){
11215 		.p = p,
11216 		.class = p->sched_class,
11217 		.flags = flags,
11218 		.queued = task_on_rq_queued(p),
11219 		.running = task_current_donor(rq, p),
11220 	};
11221 
11222 	if (!(flags & DEQUEUE_CLASS)) {
11223 		if (p->sched_class->get_prio)
11224 			ctx->prio = p->sched_class->get_prio(rq, p);
11225 		else
11226 			ctx->prio = p->prio;
11227 	}
11228 
11229 	if (ctx->queued)
11230 		dequeue_task(rq, p, flags);
11231 	if (ctx->running)
11232 		put_prev_task(rq, p);
11233 
11234 	if ((flags & DEQUEUE_CLASS) && p->sched_class->switched_from)
11235 		p->sched_class->switched_from(rq, p);
11236 
11237 	return ctx;
11238 }
11239 
11240 void sched_change_end(struct sched_change_ctx *ctx)
11241 {
11242 	struct task_struct *p = ctx->p;
11243 	struct rq *rq = task_rq(p);
11244 
11245 	lockdep_assert_rq_held(rq);
11246 
11247 	/*
11248 	 * Changing class without *QUEUE_CLASS is bad.
11249 	 */
11250 	WARN_ON_ONCE(p->sched_class != ctx->class && !(ctx->flags & ENQUEUE_CLASS));
11251 
11252 	if ((ctx->flags & ENQUEUE_CLASS) && p->sched_class->switching_to)
11253 		p->sched_class->switching_to(rq, p);
11254 
11255 	if (ctx->queued)
11256 		enqueue_task(rq, p, ctx->flags);
11257 	if (ctx->running)
11258 		set_next_task(rq, p);
11259 
11260 	if (ctx->flags & ENQUEUE_CLASS) {
11261 		if (p->sched_class->switched_to)
11262 			p->sched_class->switched_to(rq, p);
11263 
11264 		/*
11265 		 * Check if the task can preempt the current in case the class
11266 		 * was upgraded. Note that none of the switch*_from() methods
11267 		 * know the new class and none of the switch*_to() methods know
11268 		 * the old class so we must resched here.
11269 		 */
11270 		if (ctx->queued)
11271 			wakeup_preempt(rq, p, 0);
11272 
11273 		if (ctx->running) {
11274 			/*
11275 			 * If this was a degradation in class; make sure to
11276 			 * reschedule.
11277 			 */
11278 			if (sched_class_above(ctx->class, p->sched_class))
11279 				resched_curr(rq);
11280 		}
11281 	} else {
11282 		p->sched_class->prio_changed(rq, p, ctx->prio);
11283 	}
11284 }
11285 
11286 #ifdef CONFIG_PREFERRED_CPU
11287 static DEFINE_PER_CPU(struct cpu_stop_work, npc_push_task_work);
11288 
11289 static int sched_non_preferred_cpu_push_stop(void *arg)
11290 {
11291 	struct task_struct *p = arg;
11292 	struct rq *rq = this_rq();
11293 	struct rq_flags rf;
11294 	int cpu;
11295 
11296 	if (cpu_preferred(rq->cpu)) {
11297 		scoped_guard(rq_lock_irqsave, rq)
11298 			rq->npc_push_work_pending = false;
11299 		put_task_struct(p);
11300 		return 0;
11301 	}
11302 
11303 	scoped_guard (raw_spinlock_irq, &p->pi_lock) {
11304 		/*
11305 		 * select_fallback_rq() may acquire the rq lock in case of
11306 		 * fallback. So call it before grabbing rq lock. If the task
11307 		 * migrates to another CPU before the rq lock is acquired,
11308 		 * subsequent validation of task's current rq will help to
11309 		 * safely bail out.
11310 		 */
11311 		cpu = select_fallback_rq(rq->cpu, p);
11312 		context_unsafe_alias(rq);
11313 		rq_lock(rq, &rf);
11314 		rq->npc_push_work_pending = false;
11315 		update_rq_clock(rq);
11316 
11317 		if (task_rq(p) == rq && task_on_rq_queued(p)) {
11318 			struct rq *dest_rq = __migrate_task(rq, &rf, p, cpu);
11319 
11320 			if (rq != dest_rq)
11321 				schedstat_inc(p->stats.nr_migrations_cpu_non_preferred);
11322 			rq = dest_rq;
11323 		}
11324 		rq_unlock(rq, &rf);
11325 	}
11326 
11327 	put_task_struct(p);
11328 	return 0;
11329 }
11330 
11331 /*
11332  * Push the current task running on non-preferred CPU(npc).
11333  * Using this non preferred CPU will lead to more contention
11334  * in the host. So it is better not to use this CPU.
11335  *
11336  * Since task is running, call a stopper to push the task out. This is
11337  * similar to how task moves during hotplug. In select_fallback_rq() a
11338  * preferred CPU will be chosen and henceforth task shouldn't come back to
11339  * this CPU again.
11340  *
11341  * Works for FAIR class only.
11342  *
11343  * If task is affined only on non-preferred CPUs, no point in moving it out.
11344  */
11345 void sched_push_current_non_preferred_cpu(struct rq *rq)
11346 {
11347 	struct task_struct *push_task = rq->curr;
11348 
11349 	scoped_guard(rq_lock, rq) {
11350 		/* Push the task if its explicit affinity allows */
11351 		if (!task_can_migrate_to_preferred(push_task, rq->cpu))
11352 			return;
11353 
11354 		/* There is already a stopper thread. Don't race with it. */
11355 		if (rq->npc_push_work_pending)
11356 			return;
11357 
11358 		if (is_migration_disabled(push_task))
11359 			return;
11360 
11361 		rq->npc_push_work_pending = true;
11362 	}
11363 
11364 	/* sched_tick runs with interrupts disabled. */
11365 	get_task_struct(push_task);
11366 	stop_one_cpu_nowait(rq->cpu, sched_non_preferred_cpu_push_stop,
11367 			    push_task, this_cpu_ptr(&npc_push_task_work));
11368 }
11369 #endif
11370