xref: /linux/kernel/sched/core.c (revision 26fbb4c8c7c3ee9a4c3b4de555a8587b5a19154e)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  *  kernel/sched/core.c
4  *
5  *  Core kernel scheduler code and related syscalls
6  *
7  *  Copyright (C) 1991-2002  Linus Torvalds
8  */
9 #define CREATE_TRACE_POINTS
10 #include <trace/events/sched.h>
11 #undef CREATE_TRACE_POINTS
12 
13 #include "sched.h"
14 
15 #include <linux/nospec.h>
16 
17 #include <linux/kcov.h>
18 #include <linux/scs.h>
19 
20 #include <asm/switch_to.h>
21 #include <asm/tlb.h>
22 
23 #include "../workqueue_internal.h"
24 #include "../../fs/io-wq.h"
25 #include "../smpboot.h"
26 
27 #include "pelt.h"
28 #include "smp.h"
29 
30 /*
31  * Export tracepoints that act as a bare tracehook (ie: have no trace event
32  * associated with them) to allow external modules to probe them.
33  */
34 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_cfs_tp);
35 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_rt_tp);
36 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_dl_tp);
37 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_irq_tp);
38 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_se_tp);
39 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_cpu_capacity_tp);
40 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_overutilized_tp);
41 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_cfs_tp);
42 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_se_tp);
43 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_update_nr_running_tp);
44 
45 DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
46 
47 #ifdef CONFIG_SCHED_DEBUG
48 /*
49  * Debugging: various feature bits
50  *
51  * If SCHED_DEBUG is disabled, each compilation unit has its own copy of
52  * sysctl_sched_features, defined in sched.h, to allow constants propagation
53  * at compile time and compiler optimization based on features default.
54  */
55 #define SCHED_FEAT(name, enabled)	\
56 	(1UL << __SCHED_FEAT_##name) * enabled |
57 const_debug unsigned int sysctl_sched_features =
58 #include "features.h"
59 	0;
60 #undef SCHED_FEAT
61 #endif
62 
63 /*
64  * Number of tasks to iterate in a single balance run.
65  * Limited because this is done with IRQs disabled.
66  */
67 const_debug unsigned int sysctl_sched_nr_migrate = 32;
68 
69 /*
70  * period over which we measure -rt task CPU usage in us.
71  * default: 1s
72  */
73 unsigned int sysctl_sched_rt_period = 1000000;
74 
75 __read_mostly int scheduler_running;
76 
77 /*
78  * part of the period that we allow rt tasks to run in us.
79  * default: 0.95s
80  */
81 int sysctl_sched_rt_runtime = 950000;
82 
83 
84 /*
85  * Serialization rules:
86  *
87  * Lock order:
88  *
89  *   p->pi_lock
90  *     rq->lock
91  *       hrtimer_cpu_base->lock (hrtimer_start() for bandwidth controls)
92  *
93  *  rq1->lock
94  *    rq2->lock  where: rq1 < rq2
95  *
96  * Regular state:
97  *
98  * Normal scheduling state is serialized by rq->lock. __schedule() takes the
99  * local CPU's rq->lock, it optionally removes the task from the runqueue and
100  * always looks at the local rq data structures to find the most eligible task
101  * to run next.
102  *
103  * Task enqueue is also under rq->lock, possibly taken from another CPU.
104  * Wakeups from another LLC domain might use an IPI to transfer the enqueue to
105  * the local CPU to avoid bouncing the runqueue state around [ see
106  * ttwu_queue_wakelist() ]
107  *
108  * Task wakeup, specifically wakeups that involve migration, are horribly
109  * complicated to avoid having to take two rq->locks.
110  *
111  * Special state:
112  *
113  * System-calls and anything external will use task_rq_lock() which acquires
114  * both p->pi_lock and rq->lock. As a consequence the state they change is
115  * stable while holding either lock:
116  *
117  *  - sched_setaffinity()/
118  *    set_cpus_allowed_ptr():	p->cpus_ptr, p->nr_cpus_allowed
119  *  - set_user_nice():		p->se.load, p->*prio
120  *  - __sched_setscheduler():	p->sched_class, p->policy, p->*prio,
121  *				p->se.load, p->rt_priority,
122  *				p->dl.dl_{runtime, deadline, period, flags, bw, density}
123  *  - sched_setnuma():		p->numa_preferred_nid
124  *  - sched_move_task()/
125  *    cpu_cgroup_fork():	p->sched_task_group
126  *  - uclamp_update_active()	p->uclamp*
127  *
128  * p->state <- TASK_*:
129  *
130  *   is changed locklessly using set_current_state(), __set_current_state() or
131  *   set_special_state(), see their respective comments, or by
132  *   try_to_wake_up(). This latter uses p->pi_lock to serialize against
133  *   concurrent self.
134  *
135  * p->on_rq <- { 0, 1 = TASK_ON_RQ_QUEUED, 2 = TASK_ON_RQ_MIGRATING }:
136  *
137  *   is set by activate_task() and cleared by deactivate_task(), under
138  *   rq->lock. Non-zero indicates the task is runnable, the special
139  *   ON_RQ_MIGRATING state is used for migration without holding both
140  *   rq->locks. It indicates task_cpu() is not stable, see task_rq_lock().
141  *
142  * p->on_cpu <- { 0, 1 }:
143  *
144  *   is set by prepare_task() and cleared by finish_task() such that it will be
145  *   set before p is scheduled-in and cleared after p is scheduled-out, both
146  *   under rq->lock. Non-zero indicates the task is running on its CPU.
147  *
148  *   [ The astute reader will observe that it is possible for two tasks on one
149  *     CPU to have ->on_cpu = 1 at the same time. ]
150  *
151  * task_cpu(p): is changed by set_task_cpu(), the rules are:
152  *
153  *  - Don't call set_task_cpu() on a blocked task:
154  *
155  *    We don't care what CPU we're not running on, this simplifies hotplug,
156  *    the CPU assignment of blocked tasks isn't required to be valid.
157  *
158  *  - for try_to_wake_up(), called under p->pi_lock:
159  *
160  *    This allows try_to_wake_up() to only take one rq->lock, see its comment.
161  *
162  *  - for migration called under rq->lock:
163  *    [ see task_on_rq_migrating() in task_rq_lock() ]
164  *
165  *    o move_queued_task()
166  *    o detach_task()
167  *
168  *  - for migration called under double_rq_lock():
169  *
170  *    o __migrate_swap_task()
171  *    o push_rt_task() / pull_rt_task()
172  *    o push_dl_task() / pull_dl_task()
173  *    o dl_task_offline_migration()
174  *
175  */
176 
177 /*
178  * __task_rq_lock - lock the rq @p resides on.
179  */
180 struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf)
181 	__acquires(rq->lock)
182 {
183 	struct rq *rq;
184 
185 	lockdep_assert_held(&p->pi_lock);
186 
187 	for (;;) {
188 		rq = task_rq(p);
189 		raw_spin_lock(&rq->lock);
190 		if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) {
191 			rq_pin_lock(rq, rf);
192 			return rq;
193 		}
194 		raw_spin_unlock(&rq->lock);
195 
196 		while (unlikely(task_on_rq_migrating(p)))
197 			cpu_relax();
198 	}
199 }
200 
201 /*
202  * task_rq_lock - lock p->pi_lock and lock the rq @p resides on.
203  */
204 struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf)
205 	__acquires(p->pi_lock)
206 	__acquires(rq->lock)
207 {
208 	struct rq *rq;
209 
210 	for (;;) {
211 		raw_spin_lock_irqsave(&p->pi_lock, rf->flags);
212 		rq = task_rq(p);
213 		raw_spin_lock(&rq->lock);
214 		/*
215 		 *	move_queued_task()		task_rq_lock()
216 		 *
217 		 *	ACQUIRE (rq->lock)
218 		 *	[S] ->on_rq = MIGRATING		[L] rq = task_rq()
219 		 *	WMB (__set_task_cpu())		ACQUIRE (rq->lock);
220 		 *	[S] ->cpu = new_cpu		[L] task_rq()
221 		 *					[L] ->on_rq
222 		 *	RELEASE (rq->lock)
223 		 *
224 		 * If we observe the old CPU in task_rq_lock(), the acquire of
225 		 * the old rq->lock will fully serialize against the stores.
226 		 *
227 		 * If we observe the new CPU in task_rq_lock(), the address
228 		 * dependency headed by '[L] rq = task_rq()' and the acquire
229 		 * will pair with the WMB to ensure we then also see migrating.
230 		 */
231 		if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) {
232 			rq_pin_lock(rq, rf);
233 			return rq;
234 		}
235 		raw_spin_unlock(&rq->lock);
236 		raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags);
237 
238 		while (unlikely(task_on_rq_migrating(p)))
239 			cpu_relax();
240 	}
241 }
242 
243 /*
244  * RQ-clock updating methods:
245  */
246 
247 static void update_rq_clock_task(struct rq *rq, s64 delta)
248 {
249 /*
250  * In theory, the compile should just see 0 here, and optimize out the call
251  * to sched_rt_avg_update. But I don't trust it...
252  */
253 	s64 __maybe_unused steal = 0, irq_delta = 0;
254 
255 #ifdef CONFIG_IRQ_TIME_ACCOUNTING
256 	irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time;
257 
258 	/*
259 	 * Since irq_time is only updated on {soft,}irq_exit, we might run into
260 	 * this case when a previous update_rq_clock() happened inside a
261 	 * {soft,}irq region.
262 	 *
263 	 * When this happens, we stop ->clock_task and only update the
264 	 * prev_irq_time stamp to account for the part that fit, so that a next
265 	 * update will consume the rest. This ensures ->clock_task is
266 	 * monotonic.
267 	 *
268 	 * It does however cause some slight miss-attribution of {soft,}irq
269 	 * time, a more accurate solution would be to update the irq_time using
270 	 * the current rq->clock timestamp, except that would require using
271 	 * atomic ops.
272 	 */
273 	if (irq_delta > delta)
274 		irq_delta = delta;
275 
276 	rq->prev_irq_time += irq_delta;
277 	delta -= irq_delta;
278 #endif
279 #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
280 	if (static_key_false((&paravirt_steal_rq_enabled))) {
281 		steal = paravirt_steal_clock(cpu_of(rq));
282 		steal -= rq->prev_steal_time_rq;
283 
284 		if (unlikely(steal > delta))
285 			steal = delta;
286 
287 		rq->prev_steal_time_rq += steal;
288 		delta -= steal;
289 	}
290 #endif
291 
292 	rq->clock_task += delta;
293 
294 #ifdef CONFIG_HAVE_SCHED_AVG_IRQ
295 	if ((irq_delta + steal) && sched_feat(NONTASK_CAPACITY))
296 		update_irq_load_avg(rq, irq_delta + steal);
297 #endif
298 	update_rq_clock_pelt(rq, delta);
299 }
300 
301 void update_rq_clock(struct rq *rq)
302 {
303 	s64 delta;
304 
305 	lockdep_assert_held(&rq->lock);
306 
307 	if (rq->clock_update_flags & RQCF_ACT_SKIP)
308 		return;
309 
310 #ifdef CONFIG_SCHED_DEBUG
311 	if (sched_feat(WARN_DOUBLE_CLOCK))
312 		SCHED_WARN_ON(rq->clock_update_flags & RQCF_UPDATED);
313 	rq->clock_update_flags |= RQCF_UPDATED;
314 #endif
315 
316 	delta = sched_clock_cpu(cpu_of(rq)) - rq->clock;
317 	if (delta < 0)
318 		return;
319 	rq->clock += delta;
320 	update_rq_clock_task(rq, delta);
321 }
322 
323 #ifdef CONFIG_SCHED_HRTICK
324 /*
325  * Use HR-timers to deliver accurate preemption points.
326  */
327 
328 static void hrtick_clear(struct rq *rq)
329 {
330 	if (hrtimer_active(&rq->hrtick_timer))
331 		hrtimer_cancel(&rq->hrtick_timer);
332 }
333 
334 /*
335  * High-resolution timer tick.
336  * Runs from hardirq context with interrupts disabled.
337  */
338 static enum hrtimer_restart hrtick(struct hrtimer *timer)
339 {
340 	struct rq *rq = container_of(timer, struct rq, hrtick_timer);
341 	struct rq_flags rf;
342 
343 	WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());
344 
345 	rq_lock(rq, &rf);
346 	update_rq_clock(rq);
347 	rq->curr->sched_class->task_tick(rq, rq->curr, 1);
348 	rq_unlock(rq, &rf);
349 
350 	return HRTIMER_NORESTART;
351 }
352 
353 #ifdef CONFIG_SMP
354 
355 static void __hrtick_restart(struct rq *rq)
356 {
357 	struct hrtimer *timer = &rq->hrtick_timer;
358 
359 	hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED_HARD);
360 }
361 
362 /*
363  * called from hardirq (IPI) context
364  */
365 static void __hrtick_start(void *arg)
366 {
367 	struct rq *rq = arg;
368 	struct rq_flags rf;
369 
370 	rq_lock(rq, &rf);
371 	__hrtick_restart(rq);
372 	rq_unlock(rq, &rf);
373 }
374 
375 /*
376  * Called to set the hrtick timer state.
377  *
378  * called with rq->lock held and irqs disabled
379  */
380 void hrtick_start(struct rq *rq, u64 delay)
381 {
382 	struct hrtimer *timer = &rq->hrtick_timer;
383 	ktime_t time;
384 	s64 delta;
385 
386 	/*
387 	 * Don't schedule slices shorter than 10000ns, that just
388 	 * doesn't make sense and can cause timer DoS.
389 	 */
390 	delta = max_t(s64, delay, 10000LL);
391 	time = ktime_add_ns(timer->base->get_time(), delta);
392 
393 	hrtimer_set_expires(timer, time);
394 
395 	if (rq == this_rq())
396 		__hrtick_restart(rq);
397 	else
398 		smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd);
399 }
400 
401 #else
402 /*
403  * Called to set the hrtick timer state.
404  *
405  * called with rq->lock held and irqs disabled
406  */
407 void hrtick_start(struct rq *rq, u64 delay)
408 {
409 	/*
410 	 * Don't schedule slices shorter than 10000ns, that just
411 	 * doesn't make sense. Rely on vruntime for fairness.
412 	 */
413 	delay = max_t(u64, delay, 10000LL);
414 	hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay),
415 		      HRTIMER_MODE_REL_PINNED_HARD);
416 }
417 
418 #endif /* CONFIG_SMP */
419 
420 static void hrtick_rq_init(struct rq *rq)
421 {
422 #ifdef CONFIG_SMP
423 	INIT_CSD(&rq->hrtick_csd, __hrtick_start, rq);
424 #endif
425 	hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
426 	rq->hrtick_timer.function = hrtick;
427 }
428 #else	/* CONFIG_SCHED_HRTICK */
429 static inline void hrtick_clear(struct rq *rq)
430 {
431 }
432 
433 static inline void hrtick_rq_init(struct rq *rq)
434 {
435 }
436 #endif	/* CONFIG_SCHED_HRTICK */
437 
438 /*
439  * cmpxchg based fetch_or, macro so it works for different integer types
440  */
441 #define fetch_or(ptr, mask)						\
442 	({								\
443 		typeof(ptr) _ptr = (ptr);				\
444 		typeof(mask) _mask = (mask);				\
445 		typeof(*_ptr) _old, _val = *_ptr;			\
446 									\
447 		for (;;) {						\
448 			_old = cmpxchg(_ptr, _val, _val | _mask);	\
449 			if (_old == _val)				\
450 				break;					\
451 			_val = _old;					\
452 		}							\
453 	_old;								\
454 })
455 
456 #if defined(CONFIG_SMP) && defined(TIF_POLLING_NRFLAG)
457 /*
458  * Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG,
459  * this avoids any races wrt polling state changes and thereby avoids
460  * spurious IPIs.
461  */
462 static bool set_nr_and_not_polling(struct task_struct *p)
463 {
464 	struct thread_info *ti = task_thread_info(p);
465 	return !(fetch_or(&ti->flags, _TIF_NEED_RESCHED) & _TIF_POLLING_NRFLAG);
466 }
467 
468 /*
469  * Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set.
470  *
471  * If this returns true, then the idle task promises to call
472  * sched_ttwu_pending() and reschedule soon.
473  */
474 static bool set_nr_if_polling(struct task_struct *p)
475 {
476 	struct thread_info *ti = task_thread_info(p);
477 	typeof(ti->flags) old, val = READ_ONCE(ti->flags);
478 
479 	for (;;) {
480 		if (!(val & _TIF_POLLING_NRFLAG))
481 			return false;
482 		if (val & _TIF_NEED_RESCHED)
483 			return true;
484 		old = cmpxchg(&ti->flags, val, val | _TIF_NEED_RESCHED);
485 		if (old == val)
486 			break;
487 		val = old;
488 	}
489 	return true;
490 }
491 
492 #else
493 static bool set_nr_and_not_polling(struct task_struct *p)
494 {
495 	set_tsk_need_resched(p);
496 	return true;
497 }
498 
499 #ifdef CONFIG_SMP
500 static bool set_nr_if_polling(struct task_struct *p)
501 {
502 	return false;
503 }
504 #endif
505 #endif
506 
507 static bool __wake_q_add(struct wake_q_head *head, struct task_struct *task)
508 {
509 	struct wake_q_node *node = &task->wake_q;
510 
511 	/*
512 	 * Atomically grab the task, if ->wake_q is !nil already it means
513 	 * it's already queued (either by us or someone else) and will get the
514 	 * wakeup due to that.
515 	 *
516 	 * In order to ensure that a pending wakeup will observe our pending
517 	 * state, even in the failed case, an explicit smp_mb() must be used.
518 	 */
519 	smp_mb__before_atomic();
520 	if (unlikely(cmpxchg_relaxed(&node->next, NULL, WAKE_Q_TAIL)))
521 		return false;
522 
523 	/*
524 	 * The head is context local, there can be no concurrency.
525 	 */
526 	*head->lastp = node;
527 	head->lastp = &node->next;
528 	return true;
529 }
530 
531 /**
532  * wake_q_add() - queue a wakeup for 'later' waking.
533  * @head: the wake_q_head to add @task to
534  * @task: the task to queue for 'later' wakeup
535  *
536  * Queue a task for later wakeup, most likely by the wake_up_q() call in the
537  * same context, _HOWEVER_ this is not guaranteed, the wakeup can come
538  * instantly.
539  *
540  * This function must be used as-if it were wake_up_process(); IOW the task
541  * must be ready to be woken at this location.
542  */
543 void wake_q_add(struct wake_q_head *head, struct task_struct *task)
544 {
545 	if (__wake_q_add(head, task))
546 		get_task_struct(task);
547 }
548 
549 /**
550  * wake_q_add_safe() - safely queue a wakeup for 'later' waking.
551  * @head: the wake_q_head to add @task to
552  * @task: the task to queue for 'later' wakeup
553  *
554  * Queue a task for later wakeup, most likely by the wake_up_q() call in the
555  * same context, _HOWEVER_ this is not guaranteed, the wakeup can come
556  * instantly.
557  *
558  * This function must be used as-if it were wake_up_process(); IOW the task
559  * must be ready to be woken at this location.
560  *
561  * This function is essentially a task-safe equivalent to wake_q_add(). Callers
562  * that already hold reference to @task can call the 'safe' version and trust
563  * wake_q to do the right thing depending whether or not the @task is already
564  * queued for wakeup.
565  */
566 void wake_q_add_safe(struct wake_q_head *head, struct task_struct *task)
567 {
568 	if (!__wake_q_add(head, task))
569 		put_task_struct(task);
570 }
571 
572 void wake_up_q(struct wake_q_head *head)
573 {
574 	struct wake_q_node *node = head->first;
575 
576 	while (node != WAKE_Q_TAIL) {
577 		struct task_struct *task;
578 
579 		task = container_of(node, struct task_struct, wake_q);
580 		BUG_ON(!task);
581 		/* Task can safely be re-inserted now: */
582 		node = node->next;
583 		task->wake_q.next = NULL;
584 
585 		/*
586 		 * wake_up_process() executes a full barrier, which pairs with
587 		 * the queueing in wake_q_add() so as not to miss wakeups.
588 		 */
589 		wake_up_process(task);
590 		put_task_struct(task);
591 	}
592 }
593 
594 /*
595  * resched_curr - mark rq's current task 'to be rescheduled now'.
596  *
597  * On UP this means the setting of the need_resched flag, on SMP it
598  * might also involve a cross-CPU call to trigger the scheduler on
599  * the target CPU.
600  */
601 void resched_curr(struct rq *rq)
602 {
603 	struct task_struct *curr = rq->curr;
604 	int cpu;
605 
606 	lockdep_assert_held(&rq->lock);
607 
608 	if (test_tsk_need_resched(curr))
609 		return;
610 
611 	cpu = cpu_of(rq);
612 
613 	if (cpu == smp_processor_id()) {
614 		set_tsk_need_resched(curr);
615 		set_preempt_need_resched();
616 		return;
617 	}
618 
619 	if (set_nr_and_not_polling(curr))
620 		smp_send_reschedule(cpu);
621 	else
622 		trace_sched_wake_idle_without_ipi(cpu);
623 }
624 
625 void resched_cpu(int cpu)
626 {
627 	struct rq *rq = cpu_rq(cpu);
628 	unsigned long flags;
629 
630 	raw_spin_lock_irqsave(&rq->lock, flags);
631 	if (cpu_online(cpu) || cpu == smp_processor_id())
632 		resched_curr(rq);
633 	raw_spin_unlock_irqrestore(&rq->lock, flags);
634 }
635 
636 #ifdef CONFIG_SMP
637 #ifdef CONFIG_NO_HZ_COMMON
638 /*
639  * In the semi idle case, use the nearest busy CPU for migrating timers
640  * from an idle CPU.  This is good for power-savings.
641  *
642  * We don't do similar optimization for completely idle system, as
643  * selecting an idle CPU will add more delays to the timers than intended
644  * (as that CPU's timer base may not be uptodate wrt jiffies etc).
645  */
646 int get_nohz_timer_target(void)
647 {
648 	int i, cpu = smp_processor_id(), default_cpu = -1;
649 	struct sched_domain *sd;
650 
651 	if (housekeeping_cpu(cpu, HK_FLAG_TIMER)) {
652 		if (!idle_cpu(cpu))
653 			return cpu;
654 		default_cpu = cpu;
655 	}
656 
657 	rcu_read_lock();
658 	for_each_domain(cpu, sd) {
659 		for_each_cpu_and(i, sched_domain_span(sd),
660 			housekeeping_cpumask(HK_FLAG_TIMER)) {
661 			if (cpu == i)
662 				continue;
663 
664 			if (!idle_cpu(i)) {
665 				cpu = i;
666 				goto unlock;
667 			}
668 		}
669 	}
670 
671 	if (default_cpu == -1)
672 		default_cpu = housekeeping_any_cpu(HK_FLAG_TIMER);
673 	cpu = default_cpu;
674 unlock:
675 	rcu_read_unlock();
676 	return cpu;
677 }
678 
679 /*
680  * When add_timer_on() enqueues a timer into the timer wheel of an
681  * idle CPU then this timer might expire before the next timer event
682  * which is scheduled to wake up that CPU. In case of a completely
683  * idle system the next event might even be infinite time into the
684  * future. wake_up_idle_cpu() ensures that the CPU is woken up and
685  * leaves the inner idle loop so the newly added timer is taken into
686  * account when the CPU goes back to idle and evaluates the timer
687  * wheel for the next timer event.
688  */
689 static void wake_up_idle_cpu(int cpu)
690 {
691 	struct rq *rq = cpu_rq(cpu);
692 
693 	if (cpu == smp_processor_id())
694 		return;
695 
696 	if (set_nr_and_not_polling(rq->idle))
697 		smp_send_reschedule(cpu);
698 	else
699 		trace_sched_wake_idle_without_ipi(cpu);
700 }
701 
702 static bool wake_up_full_nohz_cpu(int cpu)
703 {
704 	/*
705 	 * We just need the target to call irq_exit() and re-evaluate
706 	 * the next tick. The nohz full kick at least implies that.
707 	 * If needed we can still optimize that later with an
708 	 * empty IRQ.
709 	 */
710 	if (cpu_is_offline(cpu))
711 		return true;  /* Don't try to wake offline CPUs. */
712 	if (tick_nohz_full_cpu(cpu)) {
713 		if (cpu != smp_processor_id() ||
714 		    tick_nohz_tick_stopped())
715 			tick_nohz_full_kick_cpu(cpu);
716 		return true;
717 	}
718 
719 	return false;
720 }
721 
722 /*
723  * Wake up the specified CPU.  If the CPU is going offline, it is the
724  * caller's responsibility to deal with the lost wakeup, for example,
725  * by hooking into the CPU_DEAD notifier like timers and hrtimers do.
726  */
727 void wake_up_nohz_cpu(int cpu)
728 {
729 	if (!wake_up_full_nohz_cpu(cpu))
730 		wake_up_idle_cpu(cpu);
731 }
732 
733 static void nohz_csd_func(void *info)
734 {
735 	struct rq *rq = info;
736 	int cpu = cpu_of(rq);
737 	unsigned int flags;
738 
739 	/*
740 	 * Release the rq::nohz_csd.
741 	 */
742 	flags = atomic_fetch_andnot(NOHZ_KICK_MASK, nohz_flags(cpu));
743 	WARN_ON(!(flags & NOHZ_KICK_MASK));
744 
745 	rq->idle_balance = idle_cpu(cpu);
746 	if (rq->idle_balance && !need_resched()) {
747 		rq->nohz_idle_balance = flags;
748 		raise_softirq_irqoff(SCHED_SOFTIRQ);
749 	}
750 }
751 
752 #endif /* CONFIG_NO_HZ_COMMON */
753 
754 #ifdef CONFIG_NO_HZ_FULL
755 bool sched_can_stop_tick(struct rq *rq)
756 {
757 	int fifo_nr_running;
758 
759 	/* Deadline tasks, even if single, need the tick */
760 	if (rq->dl.dl_nr_running)
761 		return false;
762 
763 	/*
764 	 * If there are more than one RR tasks, we need the tick to affect the
765 	 * actual RR behaviour.
766 	 */
767 	if (rq->rt.rr_nr_running) {
768 		if (rq->rt.rr_nr_running == 1)
769 			return true;
770 		else
771 			return false;
772 	}
773 
774 	/*
775 	 * If there's no RR tasks, but FIFO tasks, we can skip the tick, no
776 	 * forced preemption between FIFO tasks.
777 	 */
778 	fifo_nr_running = rq->rt.rt_nr_running - rq->rt.rr_nr_running;
779 	if (fifo_nr_running)
780 		return true;
781 
782 	/*
783 	 * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left;
784 	 * if there's more than one we need the tick for involuntary
785 	 * preemption.
786 	 */
787 	if (rq->nr_running > 1)
788 		return false;
789 
790 	return true;
791 }
792 #endif /* CONFIG_NO_HZ_FULL */
793 #endif /* CONFIG_SMP */
794 
795 #if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \
796 			(defined(CONFIG_SMP) || defined(CONFIG_CFS_BANDWIDTH)))
797 /*
798  * Iterate task_group tree rooted at *from, calling @down when first entering a
799  * node and @up when leaving it for the final time.
800  *
801  * Caller must hold rcu_lock or sufficient equivalent.
802  */
803 int walk_tg_tree_from(struct task_group *from,
804 			     tg_visitor down, tg_visitor up, void *data)
805 {
806 	struct task_group *parent, *child;
807 	int ret;
808 
809 	parent = from;
810 
811 down:
812 	ret = (*down)(parent, data);
813 	if (ret)
814 		goto out;
815 	list_for_each_entry_rcu(child, &parent->children, siblings) {
816 		parent = child;
817 		goto down;
818 
819 up:
820 		continue;
821 	}
822 	ret = (*up)(parent, data);
823 	if (ret || parent == from)
824 		goto out;
825 
826 	child = parent;
827 	parent = parent->parent;
828 	if (parent)
829 		goto up;
830 out:
831 	return ret;
832 }
833 
834 int tg_nop(struct task_group *tg, void *data)
835 {
836 	return 0;
837 }
838 #endif
839 
840 static void set_load_weight(struct task_struct *p, bool update_load)
841 {
842 	int prio = p->static_prio - MAX_RT_PRIO;
843 	struct load_weight *load = &p->se.load;
844 
845 	/*
846 	 * SCHED_IDLE tasks get minimal weight:
847 	 */
848 	if (task_has_idle_policy(p)) {
849 		load->weight = scale_load(WEIGHT_IDLEPRIO);
850 		load->inv_weight = WMULT_IDLEPRIO;
851 		return;
852 	}
853 
854 	/*
855 	 * SCHED_OTHER tasks have to update their load when changing their
856 	 * weight
857 	 */
858 	if (update_load && p->sched_class == &fair_sched_class) {
859 		reweight_task(p, prio);
860 	} else {
861 		load->weight = scale_load(sched_prio_to_weight[prio]);
862 		load->inv_weight = sched_prio_to_wmult[prio];
863 	}
864 }
865 
866 #ifdef CONFIG_UCLAMP_TASK
867 /*
868  * Serializes updates of utilization clamp values
869  *
870  * The (slow-path) user-space triggers utilization clamp value updates which
871  * can require updates on (fast-path) scheduler's data structures used to
872  * support enqueue/dequeue operations.
873  * While the per-CPU rq lock protects fast-path update operations, user-space
874  * requests are serialized using a mutex to reduce the risk of conflicting
875  * updates or API abuses.
876  */
877 static DEFINE_MUTEX(uclamp_mutex);
878 
879 /* Max allowed minimum utilization */
880 unsigned int sysctl_sched_uclamp_util_min = SCHED_CAPACITY_SCALE;
881 
882 /* Max allowed maximum utilization */
883 unsigned int sysctl_sched_uclamp_util_max = SCHED_CAPACITY_SCALE;
884 
885 /*
886  * By default RT tasks run at the maximum performance point/capacity of the
887  * system. Uclamp enforces this by always setting UCLAMP_MIN of RT tasks to
888  * SCHED_CAPACITY_SCALE.
889  *
890  * This knob allows admins to change the default behavior when uclamp is being
891  * used. In battery powered devices, particularly, running at the maximum
892  * capacity and frequency will increase energy consumption and shorten the
893  * battery life.
894  *
895  * This knob only affects RT tasks that their uclamp_se->user_defined == false.
896  *
897  * This knob will not override the system default sched_util_clamp_min defined
898  * above.
899  */
900 unsigned int sysctl_sched_uclamp_util_min_rt_default = SCHED_CAPACITY_SCALE;
901 
902 /* All clamps are required to be less or equal than these values */
903 static struct uclamp_se uclamp_default[UCLAMP_CNT];
904 
905 /*
906  * This static key is used to reduce the uclamp overhead in the fast path. It
907  * primarily disables the call to uclamp_rq_{inc, dec}() in
908  * enqueue/dequeue_task().
909  *
910  * This allows users to continue to enable uclamp in their kernel config with
911  * minimum uclamp overhead in the fast path.
912  *
913  * As soon as userspace modifies any of the uclamp knobs, the static key is
914  * enabled, since we have an actual users that make use of uclamp
915  * functionality.
916  *
917  * The knobs that would enable this static key are:
918  *
919  *   * A task modifying its uclamp value with sched_setattr().
920  *   * An admin modifying the sysctl_sched_uclamp_{min, max} via procfs.
921  *   * An admin modifying the cgroup cpu.uclamp.{min, max}
922  */
923 DEFINE_STATIC_KEY_FALSE(sched_uclamp_used);
924 
925 /* Integer rounded range for each bucket */
926 #define UCLAMP_BUCKET_DELTA DIV_ROUND_CLOSEST(SCHED_CAPACITY_SCALE, UCLAMP_BUCKETS)
927 
928 #define for_each_clamp_id(clamp_id) \
929 	for ((clamp_id) = 0; (clamp_id) < UCLAMP_CNT; (clamp_id)++)
930 
931 static inline unsigned int uclamp_bucket_id(unsigned int clamp_value)
932 {
933 	return clamp_value / UCLAMP_BUCKET_DELTA;
934 }
935 
936 static inline unsigned int uclamp_none(enum uclamp_id clamp_id)
937 {
938 	if (clamp_id == UCLAMP_MIN)
939 		return 0;
940 	return SCHED_CAPACITY_SCALE;
941 }
942 
943 static inline void uclamp_se_set(struct uclamp_se *uc_se,
944 				 unsigned int value, bool user_defined)
945 {
946 	uc_se->value = value;
947 	uc_se->bucket_id = uclamp_bucket_id(value);
948 	uc_se->user_defined = user_defined;
949 }
950 
951 static inline unsigned int
952 uclamp_idle_value(struct rq *rq, enum uclamp_id clamp_id,
953 		  unsigned int clamp_value)
954 {
955 	/*
956 	 * Avoid blocked utilization pushing up the frequency when we go
957 	 * idle (which drops the max-clamp) by retaining the last known
958 	 * max-clamp.
959 	 */
960 	if (clamp_id == UCLAMP_MAX) {
961 		rq->uclamp_flags |= UCLAMP_FLAG_IDLE;
962 		return clamp_value;
963 	}
964 
965 	return uclamp_none(UCLAMP_MIN);
966 }
967 
968 static inline void uclamp_idle_reset(struct rq *rq, enum uclamp_id clamp_id,
969 				     unsigned int clamp_value)
970 {
971 	/* Reset max-clamp retention only on idle exit */
972 	if (!(rq->uclamp_flags & UCLAMP_FLAG_IDLE))
973 		return;
974 
975 	WRITE_ONCE(rq->uclamp[clamp_id].value, clamp_value);
976 }
977 
978 static inline
979 unsigned int uclamp_rq_max_value(struct rq *rq, enum uclamp_id clamp_id,
980 				   unsigned int clamp_value)
981 {
982 	struct uclamp_bucket *bucket = rq->uclamp[clamp_id].bucket;
983 	int bucket_id = UCLAMP_BUCKETS - 1;
984 
985 	/*
986 	 * Since both min and max clamps are max aggregated, find the
987 	 * top most bucket with tasks in.
988 	 */
989 	for ( ; bucket_id >= 0; bucket_id--) {
990 		if (!bucket[bucket_id].tasks)
991 			continue;
992 		return bucket[bucket_id].value;
993 	}
994 
995 	/* No tasks -- default clamp values */
996 	return uclamp_idle_value(rq, clamp_id, clamp_value);
997 }
998 
999 static void __uclamp_update_util_min_rt_default(struct task_struct *p)
1000 {
1001 	unsigned int default_util_min;
1002 	struct uclamp_se *uc_se;
1003 
1004 	lockdep_assert_held(&p->pi_lock);
1005 
1006 	uc_se = &p->uclamp_req[UCLAMP_MIN];
1007 
1008 	/* Only sync if user didn't override the default */
1009 	if (uc_se->user_defined)
1010 		return;
1011 
1012 	default_util_min = sysctl_sched_uclamp_util_min_rt_default;
1013 	uclamp_se_set(uc_se, default_util_min, false);
1014 }
1015 
1016 static void uclamp_update_util_min_rt_default(struct task_struct *p)
1017 {
1018 	struct rq_flags rf;
1019 	struct rq *rq;
1020 
1021 	if (!rt_task(p))
1022 		return;
1023 
1024 	/* Protect updates to p->uclamp_* */
1025 	rq = task_rq_lock(p, &rf);
1026 	__uclamp_update_util_min_rt_default(p);
1027 	task_rq_unlock(rq, p, &rf);
1028 }
1029 
1030 static void uclamp_sync_util_min_rt_default(void)
1031 {
1032 	struct task_struct *g, *p;
1033 
1034 	/*
1035 	 * copy_process()			sysctl_uclamp
1036 	 *					  uclamp_min_rt = X;
1037 	 *   write_lock(&tasklist_lock)		  read_lock(&tasklist_lock)
1038 	 *   // link thread			  smp_mb__after_spinlock()
1039 	 *   write_unlock(&tasklist_lock)	  read_unlock(&tasklist_lock);
1040 	 *   sched_post_fork()			  for_each_process_thread()
1041 	 *     __uclamp_sync_rt()		    __uclamp_sync_rt()
1042 	 *
1043 	 * Ensures that either sched_post_fork() will observe the new
1044 	 * uclamp_min_rt or for_each_process_thread() will observe the new
1045 	 * task.
1046 	 */
1047 	read_lock(&tasklist_lock);
1048 	smp_mb__after_spinlock();
1049 	read_unlock(&tasklist_lock);
1050 
1051 	rcu_read_lock();
1052 	for_each_process_thread(g, p)
1053 		uclamp_update_util_min_rt_default(p);
1054 	rcu_read_unlock();
1055 }
1056 
1057 static inline struct uclamp_se
1058 uclamp_tg_restrict(struct task_struct *p, enum uclamp_id clamp_id)
1059 {
1060 	struct uclamp_se uc_req = p->uclamp_req[clamp_id];
1061 #ifdef CONFIG_UCLAMP_TASK_GROUP
1062 	struct uclamp_se uc_max;
1063 
1064 	/*
1065 	 * Tasks in autogroups or root task group will be
1066 	 * restricted by system defaults.
1067 	 */
1068 	if (task_group_is_autogroup(task_group(p)))
1069 		return uc_req;
1070 	if (task_group(p) == &root_task_group)
1071 		return uc_req;
1072 
1073 	uc_max = task_group(p)->uclamp[clamp_id];
1074 	if (uc_req.value > uc_max.value || !uc_req.user_defined)
1075 		return uc_max;
1076 #endif
1077 
1078 	return uc_req;
1079 }
1080 
1081 /*
1082  * The effective clamp bucket index of a task depends on, by increasing
1083  * priority:
1084  * - the task specific clamp value, when explicitly requested from userspace
1085  * - the task group effective clamp value, for tasks not either in the root
1086  *   group or in an autogroup
1087  * - the system default clamp value, defined by the sysadmin
1088  */
1089 static inline struct uclamp_se
1090 uclamp_eff_get(struct task_struct *p, enum uclamp_id clamp_id)
1091 {
1092 	struct uclamp_se uc_req = uclamp_tg_restrict(p, clamp_id);
1093 	struct uclamp_se uc_max = uclamp_default[clamp_id];
1094 
1095 	/* System default restrictions always apply */
1096 	if (unlikely(uc_req.value > uc_max.value))
1097 		return uc_max;
1098 
1099 	return uc_req;
1100 }
1101 
1102 unsigned long uclamp_eff_value(struct task_struct *p, enum uclamp_id clamp_id)
1103 {
1104 	struct uclamp_se uc_eff;
1105 
1106 	/* Task currently refcounted: use back-annotated (effective) value */
1107 	if (p->uclamp[clamp_id].active)
1108 		return (unsigned long)p->uclamp[clamp_id].value;
1109 
1110 	uc_eff = uclamp_eff_get(p, clamp_id);
1111 
1112 	return (unsigned long)uc_eff.value;
1113 }
1114 
1115 /*
1116  * When a task is enqueued on a rq, the clamp bucket currently defined by the
1117  * task's uclamp::bucket_id is refcounted on that rq. This also immediately
1118  * updates the rq's clamp value if required.
1119  *
1120  * Tasks can have a task-specific value requested from user-space, track
1121  * within each bucket the maximum value for tasks refcounted in it.
1122  * This "local max aggregation" allows to track the exact "requested" value
1123  * for each bucket when all its RUNNABLE tasks require the same clamp.
1124  */
1125 static inline void uclamp_rq_inc_id(struct rq *rq, struct task_struct *p,
1126 				    enum uclamp_id clamp_id)
1127 {
1128 	struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id];
1129 	struct uclamp_se *uc_se = &p->uclamp[clamp_id];
1130 	struct uclamp_bucket *bucket;
1131 
1132 	lockdep_assert_held(&rq->lock);
1133 
1134 	/* Update task effective clamp */
1135 	p->uclamp[clamp_id] = uclamp_eff_get(p, clamp_id);
1136 
1137 	bucket = &uc_rq->bucket[uc_se->bucket_id];
1138 	bucket->tasks++;
1139 	uc_se->active = true;
1140 
1141 	uclamp_idle_reset(rq, clamp_id, uc_se->value);
1142 
1143 	/*
1144 	 * Local max aggregation: rq buckets always track the max
1145 	 * "requested" clamp value of its RUNNABLE tasks.
1146 	 */
1147 	if (bucket->tasks == 1 || uc_se->value > bucket->value)
1148 		bucket->value = uc_se->value;
1149 
1150 	if (uc_se->value > READ_ONCE(uc_rq->value))
1151 		WRITE_ONCE(uc_rq->value, uc_se->value);
1152 }
1153 
1154 /*
1155  * When a task is dequeued from a rq, the clamp bucket refcounted by the task
1156  * is released. If this is the last task reference counting the rq's max
1157  * active clamp value, then the rq's clamp value is updated.
1158  *
1159  * Both refcounted tasks and rq's cached clamp values are expected to be
1160  * always valid. If it's detected they are not, as defensive programming,
1161  * enforce the expected state and warn.
1162  */
1163 static inline void uclamp_rq_dec_id(struct rq *rq, struct task_struct *p,
1164 				    enum uclamp_id clamp_id)
1165 {
1166 	struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id];
1167 	struct uclamp_se *uc_se = &p->uclamp[clamp_id];
1168 	struct uclamp_bucket *bucket;
1169 	unsigned int bkt_clamp;
1170 	unsigned int rq_clamp;
1171 
1172 	lockdep_assert_held(&rq->lock);
1173 
1174 	/*
1175 	 * If sched_uclamp_used was enabled after task @p was enqueued,
1176 	 * we could end up with unbalanced call to uclamp_rq_dec_id().
1177 	 *
1178 	 * In this case the uc_se->active flag should be false since no uclamp
1179 	 * accounting was performed at enqueue time and we can just return
1180 	 * here.
1181 	 *
1182 	 * Need to be careful of the following enqueue/dequeue ordering
1183 	 * problem too
1184 	 *
1185 	 *	enqueue(taskA)
1186 	 *	// sched_uclamp_used gets enabled
1187 	 *	enqueue(taskB)
1188 	 *	dequeue(taskA)
1189 	 *	// Must not decrement bucket->tasks here
1190 	 *	dequeue(taskB)
1191 	 *
1192 	 * where we could end up with stale data in uc_se and
1193 	 * bucket[uc_se->bucket_id].
1194 	 *
1195 	 * The following check here eliminates the possibility of such race.
1196 	 */
1197 	if (unlikely(!uc_se->active))
1198 		return;
1199 
1200 	bucket = &uc_rq->bucket[uc_se->bucket_id];
1201 
1202 	SCHED_WARN_ON(!bucket->tasks);
1203 	if (likely(bucket->tasks))
1204 		bucket->tasks--;
1205 
1206 	uc_se->active = false;
1207 
1208 	/*
1209 	 * Keep "local max aggregation" simple and accept to (possibly)
1210 	 * overboost some RUNNABLE tasks in the same bucket.
1211 	 * The rq clamp bucket value is reset to its base value whenever
1212 	 * there are no more RUNNABLE tasks refcounting it.
1213 	 */
1214 	if (likely(bucket->tasks))
1215 		return;
1216 
1217 	rq_clamp = READ_ONCE(uc_rq->value);
1218 	/*
1219 	 * Defensive programming: this should never happen. If it happens,
1220 	 * e.g. due to future modification, warn and fixup the expected value.
1221 	 */
1222 	SCHED_WARN_ON(bucket->value > rq_clamp);
1223 	if (bucket->value >= rq_clamp) {
1224 		bkt_clamp = uclamp_rq_max_value(rq, clamp_id, uc_se->value);
1225 		WRITE_ONCE(uc_rq->value, bkt_clamp);
1226 	}
1227 }
1228 
1229 static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p)
1230 {
1231 	enum uclamp_id clamp_id;
1232 
1233 	/*
1234 	 * Avoid any overhead until uclamp is actually used by the userspace.
1235 	 *
1236 	 * The condition is constructed such that a NOP is generated when
1237 	 * sched_uclamp_used is disabled.
1238 	 */
1239 	if (!static_branch_unlikely(&sched_uclamp_used))
1240 		return;
1241 
1242 	if (unlikely(!p->sched_class->uclamp_enabled))
1243 		return;
1244 
1245 	for_each_clamp_id(clamp_id)
1246 		uclamp_rq_inc_id(rq, p, clamp_id);
1247 
1248 	/* Reset clamp idle holding when there is one RUNNABLE task */
1249 	if (rq->uclamp_flags & UCLAMP_FLAG_IDLE)
1250 		rq->uclamp_flags &= ~UCLAMP_FLAG_IDLE;
1251 }
1252 
1253 static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p)
1254 {
1255 	enum uclamp_id clamp_id;
1256 
1257 	/*
1258 	 * Avoid any overhead until uclamp is actually used by the userspace.
1259 	 *
1260 	 * The condition is constructed such that a NOP is generated when
1261 	 * sched_uclamp_used is disabled.
1262 	 */
1263 	if (!static_branch_unlikely(&sched_uclamp_used))
1264 		return;
1265 
1266 	if (unlikely(!p->sched_class->uclamp_enabled))
1267 		return;
1268 
1269 	for_each_clamp_id(clamp_id)
1270 		uclamp_rq_dec_id(rq, p, clamp_id);
1271 }
1272 
1273 static inline void
1274 uclamp_update_active(struct task_struct *p, enum uclamp_id clamp_id)
1275 {
1276 	struct rq_flags rf;
1277 	struct rq *rq;
1278 
1279 	/*
1280 	 * Lock the task and the rq where the task is (or was) queued.
1281 	 *
1282 	 * We might lock the (previous) rq of a !RUNNABLE task, but that's the
1283 	 * price to pay to safely serialize util_{min,max} updates with
1284 	 * enqueues, dequeues and migration operations.
1285 	 * This is the same locking schema used by __set_cpus_allowed_ptr().
1286 	 */
1287 	rq = task_rq_lock(p, &rf);
1288 
1289 	/*
1290 	 * Setting the clamp bucket is serialized by task_rq_lock().
1291 	 * If the task is not yet RUNNABLE and its task_struct is not
1292 	 * affecting a valid clamp bucket, the next time it's enqueued,
1293 	 * it will already see the updated clamp bucket value.
1294 	 */
1295 	if (p->uclamp[clamp_id].active) {
1296 		uclamp_rq_dec_id(rq, p, clamp_id);
1297 		uclamp_rq_inc_id(rq, p, clamp_id);
1298 	}
1299 
1300 	task_rq_unlock(rq, p, &rf);
1301 }
1302 
1303 #ifdef CONFIG_UCLAMP_TASK_GROUP
1304 static inline void
1305 uclamp_update_active_tasks(struct cgroup_subsys_state *css,
1306 			   unsigned int clamps)
1307 {
1308 	enum uclamp_id clamp_id;
1309 	struct css_task_iter it;
1310 	struct task_struct *p;
1311 
1312 	css_task_iter_start(css, 0, &it);
1313 	while ((p = css_task_iter_next(&it))) {
1314 		for_each_clamp_id(clamp_id) {
1315 			if ((0x1 << clamp_id) & clamps)
1316 				uclamp_update_active(p, clamp_id);
1317 		}
1318 	}
1319 	css_task_iter_end(&it);
1320 }
1321 
1322 static void cpu_util_update_eff(struct cgroup_subsys_state *css);
1323 static void uclamp_update_root_tg(void)
1324 {
1325 	struct task_group *tg = &root_task_group;
1326 
1327 	uclamp_se_set(&tg->uclamp_req[UCLAMP_MIN],
1328 		      sysctl_sched_uclamp_util_min, false);
1329 	uclamp_se_set(&tg->uclamp_req[UCLAMP_MAX],
1330 		      sysctl_sched_uclamp_util_max, false);
1331 
1332 	rcu_read_lock();
1333 	cpu_util_update_eff(&root_task_group.css);
1334 	rcu_read_unlock();
1335 }
1336 #else
1337 static void uclamp_update_root_tg(void) { }
1338 #endif
1339 
1340 int sysctl_sched_uclamp_handler(struct ctl_table *table, int write,
1341 				void *buffer, size_t *lenp, loff_t *ppos)
1342 {
1343 	bool update_root_tg = false;
1344 	int old_min, old_max, old_min_rt;
1345 	int result;
1346 
1347 	mutex_lock(&uclamp_mutex);
1348 	old_min = sysctl_sched_uclamp_util_min;
1349 	old_max = sysctl_sched_uclamp_util_max;
1350 	old_min_rt = sysctl_sched_uclamp_util_min_rt_default;
1351 
1352 	result = proc_dointvec(table, write, buffer, lenp, ppos);
1353 	if (result)
1354 		goto undo;
1355 	if (!write)
1356 		goto done;
1357 
1358 	if (sysctl_sched_uclamp_util_min > sysctl_sched_uclamp_util_max ||
1359 	    sysctl_sched_uclamp_util_max > SCHED_CAPACITY_SCALE	||
1360 	    sysctl_sched_uclamp_util_min_rt_default > SCHED_CAPACITY_SCALE) {
1361 
1362 		result = -EINVAL;
1363 		goto undo;
1364 	}
1365 
1366 	if (old_min != sysctl_sched_uclamp_util_min) {
1367 		uclamp_se_set(&uclamp_default[UCLAMP_MIN],
1368 			      sysctl_sched_uclamp_util_min, false);
1369 		update_root_tg = true;
1370 	}
1371 	if (old_max != sysctl_sched_uclamp_util_max) {
1372 		uclamp_se_set(&uclamp_default[UCLAMP_MAX],
1373 			      sysctl_sched_uclamp_util_max, false);
1374 		update_root_tg = true;
1375 	}
1376 
1377 	if (update_root_tg) {
1378 		static_branch_enable(&sched_uclamp_used);
1379 		uclamp_update_root_tg();
1380 	}
1381 
1382 	if (old_min_rt != sysctl_sched_uclamp_util_min_rt_default) {
1383 		static_branch_enable(&sched_uclamp_used);
1384 		uclamp_sync_util_min_rt_default();
1385 	}
1386 
1387 	/*
1388 	 * We update all RUNNABLE tasks only when task groups are in use.
1389 	 * Otherwise, keep it simple and do just a lazy update at each next
1390 	 * task enqueue time.
1391 	 */
1392 
1393 	goto done;
1394 
1395 undo:
1396 	sysctl_sched_uclamp_util_min = old_min;
1397 	sysctl_sched_uclamp_util_max = old_max;
1398 	sysctl_sched_uclamp_util_min_rt_default = old_min_rt;
1399 done:
1400 	mutex_unlock(&uclamp_mutex);
1401 
1402 	return result;
1403 }
1404 
1405 static int uclamp_validate(struct task_struct *p,
1406 			   const struct sched_attr *attr)
1407 {
1408 	int util_min = p->uclamp_req[UCLAMP_MIN].value;
1409 	int util_max = p->uclamp_req[UCLAMP_MAX].value;
1410 
1411 	if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MIN) {
1412 		util_min = attr->sched_util_min;
1413 
1414 		if (util_min + 1 > SCHED_CAPACITY_SCALE + 1)
1415 			return -EINVAL;
1416 	}
1417 
1418 	if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MAX) {
1419 		util_max = attr->sched_util_max;
1420 
1421 		if (util_max + 1 > SCHED_CAPACITY_SCALE + 1)
1422 			return -EINVAL;
1423 	}
1424 
1425 	if (util_min != -1 && util_max != -1 && util_min > util_max)
1426 		return -EINVAL;
1427 
1428 	/*
1429 	 * We have valid uclamp attributes; make sure uclamp is enabled.
1430 	 *
1431 	 * We need to do that here, because enabling static branches is a
1432 	 * blocking operation which obviously cannot be done while holding
1433 	 * scheduler locks.
1434 	 */
1435 	static_branch_enable(&sched_uclamp_used);
1436 
1437 	return 0;
1438 }
1439 
1440 static bool uclamp_reset(const struct sched_attr *attr,
1441 			 enum uclamp_id clamp_id,
1442 			 struct uclamp_se *uc_se)
1443 {
1444 	/* Reset on sched class change for a non user-defined clamp value. */
1445 	if (likely(!(attr->sched_flags & SCHED_FLAG_UTIL_CLAMP)) &&
1446 	    !uc_se->user_defined)
1447 		return true;
1448 
1449 	/* Reset on sched_util_{min,max} == -1. */
1450 	if (clamp_id == UCLAMP_MIN &&
1451 	    attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MIN &&
1452 	    attr->sched_util_min == -1) {
1453 		return true;
1454 	}
1455 
1456 	if (clamp_id == UCLAMP_MAX &&
1457 	    attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MAX &&
1458 	    attr->sched_util_max == -1) {
1459 		return true;
1460 	}
1461 
1462 	return false;
1463 }
1464 
1465 static void __setscheduler_uclamp(struct task_struct *p,
1466 				  const struct sched_attr *attr)
1467 {
1468 	enum uclamp_id clamp_id;
1469 
1470 	for_each_clamp_id(clamp_id) {
1471 		struct uclamp_se *uc_se = &p->uclamp_req[clamp_id];
1472 		unsigned int value;
1473 
1474 		if (!uclamp_reset(attr, clamp_id, uc_se))
1475 			continue;
1476 
1477 		/*
1478 		 * RT by default have a 100% boost value that could be modified
1479 		 * at runtime.
1480 		 */
1481 		if (unlikely(rt_task(p) && clamp_id == UCLAMP_MIN))
1482 			value = sysctl_sched_uclamp_util_min_rt_default;
1483 		else
1484 			value = uclamp_none(clamp_id);
1485 
1486 		uclamp_se_set(uc_se, value, false);
1487 
1488 	}
1489 
1490 	if (likely(!(attr->sched_flags & SCHED_FLAG_UTIL_CLAMP)))
1491 		return;
1492 
1493 	if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MIN &&
1494 	    attr->sched_util_min != -1) {
1495 		uclamp_se_set(&p->uclamp_req[UCLAMP_MIN],
1496 			      attr->sched_util_min, true);
1497 	}
1498 
1499 	if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MAX &&
1500 	    attr->sched_util_max != -1) {
1501 		uclamp_se_set(&p->uclamp_req[UCLAMP_MAX],
1502 			      attr->sched_util_max, true);
1503 	}
1504 }
1505 
1506 static void uclamp_fork(struct task_struct *p)
1507 {
1508 	enum uclamp_id clamp_id;
1509 
1510 	/*
1511 	 * We don't need to hold task_rq_lock() when updating p->uclamp_* here
1512 	 * as the task is still at its early fork stages.
1513 	 */
1514 	for_each_clamp_id(clamp_id)
1515 		p->uclamp[clamp_id].active = false;
1516 
1517 	if (likely(!p->sched_reset_on_fork))
1518 		return;
1519 
1520 	for_each_clamp_id(clamp_id) {
1521 		uclamp_se_set(&p->uclamp_req[clamp_id],
1522 			      uclamp_none(clamp_id), false);
1523 	}
1524 }
1525 
1526 static void uclamp_post_fork(struct task_struct *p)
1527 {
1528 	uclamp_update_util_min_rt_default(p);
1529 }
1530 
1531 static void __init init_uclamp_rq(struct rq *rq)
1532 {
1533 	enum uclamp_id clamp_id;
1534 	struct uclamp_rq *uc_rq = rq->uclamp;
1535 
1536 	for_each_clamp_id(clamp_id) {
1537 		uc_rq[clamp_id] = (struct uclamp_rq) {
1538 			.value = uclamp_none(clamp_id)
1539 		};
1540 	}
1541 
1542 	rq->uclamp_flags = 0;
1543 }
1544 
1545 static void __init init_uclamp(void)
1546 {
1547 	struct uclamp_se uc_max = {};
1548 	enum uclamp_id clamp_id;
1549 	int cpu;
1550 
1551 	for_each_possible_cpu(cpu)
1552 		init_uclamp_rq(cpu_rq(cpu));
1553 
1554 	for_each_clamp_id(clamp_id) {
1555 		uclamp_se_set(&init_task.uclamp_req[clamp_id],
1556 			      uclamp_none(clamp_id), false);
1557 	}
1558 
1559 	/* System defaults allow max clamp values for both indexes */
1560 	uclamp_se_set(&uc_max, uclamp_none(UCLAMP_MAX), false);
1561 	for_each_clamp_id(clamp_id) {
1562 		uclamp_default[clamp_id] = uc_max;
1563 #ifdef CONFIG_UCLAMP_TASK_GROUP
1564 		root_task_group.uclamp_req[clamp_id] = uc_max;
1565 		root_task_group.uclamp[clamp_id] = uc_max;
1566 #endif
1567 	}
1568 }
1569 
1570 #else /* CONFIG_UCLAMP_TASK */
1571 static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p) { }
1572 static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p) { }
1573 static inline int uclamp_validate(struct task_struct *p,
1574 				  const struct sched_attr *attr)
1575 {
1576 	return -EOPNOTSUPP;
1577 }
1578 static void __setscheduler_uclamp(struct task_struct *p,
1579 				  const struct sched_attr *attr) { }
1580 static inline void uclamp_fork(struct task_struct *p) { }
1581 static inline void uclamp_post_fork(struct task_struct *p) { }
1582 static inline void init_uclamp(void) { }
1583 #endif /* CONFIG_UCLAMP_TASK */
1584 
1585 static inline void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
1586 {
1587 	if (!(flags & ENQUEUE_NOCLOCK))
1588 		update_rq_clock(rq);
1589 
1590 	if (!(flags & ENQUEUE_RESTORE)) {
1591 		sched_info_queued(rq, p);
1592 		psi_enqueue(p, flags & ENQUEUE_WAKEUP);
1593 	}
1594 
1595 	uclamp_rq_inc(rq, p);
1596 	p->sched_class->enqueue_task(rq, p, flags);
1597 }
1598 
1599 static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
1600 {
1601 	if (!(flags & DEQUEUE_NOCLOCK))
1602 		update_rq_clock(rq);
1603 
1604 	if (!(flags & DEQUEUE_SAVE)) {
1605 		sched_info_dequeued(rq, p);
1606 		psi_dequeue(p, flags & DEQUEUE_SLEEP);
1607 	}
1608 
1609 	uclamp_rq_dec(rq, p);
1610 	p->sched_class->dequeue_task(rq, p, flags);
1611 }
1612 
1613 void activate_task(struct rq *rq, struct task_struct *p, int flags)
1614 {
1615 	enqueue_task(rq, p, flags);
1616 
1617 	p->on_rq = TASK_ON_RQ_QUEUED;
1618 }
1619 
1620 void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
1621 {
1622 	p->on_rq = (flags & DEQUEUE_SLEEP) ? 0 : TASK_ON_RQ_MIGRATING;
1623 
1624 	dequeue_task(rq, p, flags);
1625 }
1626 
1627 /*
1628  * __normal_prio - return the priority that is based on the static prio
1629  */
1630 static inline int __normal_prio(struct task_struct *p)
1631 {
1632 	return p->static_prio;
1633 }
1634 
1635 /*
1636  * Calculate the expected normal priority: i.e. priority
1637  * without taking RT-inheritance into account. Might be
1638  * boosted by interactivity modifiers. Changes upon fork,
1639  * setprio syscalls, and whenever the interactivity
1640  * estimator recalculates.
1641  */
1642 static inline int normal_prio(struct task_struct *p)
1643 {
1644 	int prio;
1645 
1646 	if (task_has_dl_policy(p))
1647 		prio = MAX_DL_PRIO-1;
1648 	else if (task_has_rt_policy(p))
1649 		prio = MAX_RT_PRIO-1 - p->rt_priority;
1650 	else
1651 		prio = __normal_prio(p);
1652 	return prio;
1653 }
1654 
1655 /*
1656  * Calculate the current priority, i.e. the priority
1657  * taken into account by the scheduler. This value might
1658  * be boosted by RT tasks, or might be boosted by
1659  * interactivity modifiers. Will be RT if the task got
1660  * RT-boosted. If not then it returns p->normal_prio.
1661  */
1662 static int effective_prio(struct task_struct *p)
1663 {
1664 	p->normal_prio = normal_prio(p);
1665 	/*
1666 	 * If we are RT tasks or we were boosted to RT priority,
1667 	 * keep the priority unchanged. Otherwise, update priority
1668 	 * to the normal priority:
1669 	 */
1670 	if (!rt_prio(p->prio))
1671 		return p->normal_prio;
1672 	return p->prio;
1673 }
1674 
1675 /**
1676  * task_curr - is this task currently executing on a CPU?
1677  * @p: the task in question.
1678  *
1679  * Return: 1 if the task is currently executing. 0 otherwise.
1680  */
1681 inline int task_curr(const struct task_struct *p)
1682 {
1683 	return cpu_curr(task_cpu(p)) == p;
1684 }
1685 
1686 /*
1687  * switched_from, switched_to and prio_changed must _NOT_ drop rq->lock,
1688  * use the balance_callback list if you want balancing.
1689  *
1690  * this means any call to check_class_changed() must be followed by a call to
1691  * balance_callback().
1692  */
1693 static inline void check_class_changed(struct rq *rq, struct task_struct *p,
1694 				       const struct sched_class *prev_class,
1695 				       int oldprio)
1696 {
1697 	if (prev_class != p->sched_class) {
1698 		if (prev_class->switched_from)
1699 			prev_class->switched_from(rq, p);
1700 
1701 		p->sched_class->switched_to(rq, p);
1702 	} else if (oldprio != p->prio || dl_task(p))
1703 		p->sched_class->prio_changed(rq, p, oldprio);
1704 }
1705 
1706 void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
1707 {
1708 	if (p->sched_class == rq->curr->sched_class)
1709 		rq->curr->sched_class->check_preempt_curr(rq, p, flags);
1710 	else if (p->sched_class > rq->curr->sched_class)
1711 		resched_curr(rq);
1712 
1713 	/*
1714 	 * A queue event has occurred, and we're going to schedule.  In
1715 	 * this case, we can save a useless back to back clock update.
1716 	 */
1717 	if (task_on_rq_queued(rq->curr) && test_tsk_need_resched(rq->curr))
1718 		rq_clock_skip_update(rq);
1719 }
1720 
1721 #ifdef CONFIG_SMP
1722 
1723 static void
1724 __do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask, u32 flags);
1725 
1726 static int __set_cpus_allowed_ptr(struct task_struct *p,
1727 				  const struct cpumask *new_mask,
1728 				  u32 flags);
1729 
1730 static void migrate_disable_switch(struct rq *rq, struct task_struct *p)
1731 {
1732 	if (likely(!p->migration_disabled))
1733 		return;
1734 
1735 	if (p->cpus_ptr != &p->cpus_mask)
1736 		return;
1737 
1738 	/*
1739 	 * Violates locking rules! see comment in __do_set_cpus_allowed().
1740 	 */
1741 	__do_set_cpus_allowed(p, cpumask_of(rq->cpu), SCA_MIGRATE_DISABLE);
1742 }
1743 
1744 void migrate_disable(void)
1745 {
1746 	struct task_struct *p = current;
1747 
1748 	if (p->migration_disabled) {
1749 		p->migration_disabled++;
1750 		return;
1751 	}
1752 
1753 	preempt_disable();
1754 	this_rq()->nr_pinned++;
1755 	p->migration_disabled = 1;
1756 	preempt_enable();
1757 }
1758 EXPORT_SYMBOL_GPL(migrate_disable);
1759 
1760 void migrate_enable(void)
1761 {
1762 	struct task_struct *p = current;
1763 
1764 	if (p->migration_disabled > 1) {
1765 		p->migration_disabled--;
1766 		return;
1767 	}
1768 
1769 	/*
1770 	 * Ensure stop_task runs either before or after this, and that
1771 	 * __set_cpus_allowed_ptr(SCA_MIGRATE_ENABLE) doesn't schedule().
1772 	 */
1773 	preempt_disable();
1774 	if (p->cpus_ptr != &p->cpus_mask)
1775 		__set_cpus_allowed_ptr(p, &p->cpus_mask, SCA_MIGRATE_ENABLE);
1776 	/*
1777 	 * Mustn't clear migration_disabled() until cpus_ptr points back at the
1778 	 * regular cpus_mask, otherwise things that race (eg.
1779 	 * select_fallback_rq) get confused.
1780 	 */
1781 	barrier();
1782 	p->migration_disabled = 0;
1783 	this_rq()->nr_pinned--;
1784 	preempt_enable();
1785 }
1786 EXPORT_SYMBOL_GPL(migrate_enable);
1787 
1788 static inline bool rq_has_pinned_tasks(struct rq *rq)
1789 {
1790 	return rq->nr_pinned;
1791 }
1792 
1793 /*
1794  * Per-CPU kthreads are allowed to run on !active && online CPUs, see
1795  * __set_cpus_allowed_ptr() and select_fallback_rq().
1796  */
1797 static inline bool is_cpu_allowed(struct task_struct *p, int cpu)
1798 {
1799 	/* When not in the task's cpumask, no point in looking further. */
1800 	if (!cpumask_test_cpu(cpu, p->cpus_ptr))
1801 		return false;
1802 
1803 	/* migrate_disabled() must be allowed to finish. */
1804 	if (is_migration_disabled(p))
1805 		return cpu_online(cpu);
1806 
1807 	/* Non kernel threads are not allowed during either online or offline. */
1808 	if (!(p->flags & PF_KTHREAD))
1809 		return cpu_active(cpu);
1810 
1811 	/* KTHREAD_IS_PER_CPU is always allowed. */
1812 	if (kthread_is_per_cpu(p))
1813 		return cpu_online(cpu);
1814 
1815 	/* Regular kernel threads don't get to stay during offline. */
1816 	if (cpu_rq(cpu)->balance_push)
1817 		return false;
1818 
1819 	/* But are allowed during online. */
1820 	return cpu_online(cpu);
1821 }
1822 
1823 /*
1824  * This is how migration works:
1825  *
1826  * 1) we invoke migration_cpu_stop() on the target CPU using
1827  *    stop_one_cpu().
1828  * 2) stopper starts to run (implicitly forcing the migrated thread
1829  *    off the CPU)
1830  * 3) it checks whether the migrated task is still in the wrong runqueue.
1831  * 4) if it's in the wrong runqueue then the migration thread removes
1832  *    it and puts it into the right queue.
1833  * 5) stopper completes and stop_one_cpu() returns and the migration
1834  *    is done.
1835  */
1836 
1837 /*
1838  * move_queued_task - move a queued task to new rq.
1839  *
1840  * Returns (locked) new rq. Old rq's lock is released.
1841  */
1842 static struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf,
1843 				   struct task_struct *p, int new_cpu)
1844 {
1845 	lockdep_assert_held(&rq->lock);
1846 
1847 	deactivate_task(rq, p, DEQUEUE_NOCLOCK);
1848 	set_task_cpu(p, new_cpu);
1849 	rq_unlock(rq, rf);
1850 
1851 	rq = cpu_rq(new_cpu);
1852 
1853 	rq_lock(rq, rf);
1854 	BUG_ON(task_cpu(p) != new_cpu);
1855 	activate_task(rq, p, 0);
1856 	check_preempt_curr(rq, p, 0);
1857 
1858 	return rq;
1859 }
1860 
1861 struct migration_arg {
1862 	struct task_struct		*task;
1863 	int				dest_cpu;
1864 	struct set_affinity_pending	*pending;
1865 };
1866 
1867 struct set_affinity_pending {
1868 	refcount_t		refs;
1869 	struct completion	done;
1870 	struct cpu_stop_work	stop_work;
1871 	struct migration_arg	arg;
1872 };
1873 
1874 /*
1875  * Move (not current) task off this CPU, onto the destination CPU. We're doing
1876  * this because either it can't run here any more (set_cpus_allowed()
1877  * away from this CPU, or CPU going down), or because we're
1878  * attempting to rebalance this task on exec (sched_exec).
1879  *
1880  * So we race with normal scheduler movements, but that's OK, as long
1881  * as the task is no longer on this CPU.
1882  */
1883 static struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf,
1884 				 struct task_struct *p, int dest_cpu)
1885 {
1886 	/* Affinity changed (again). */
1887 	if (!is_cpu_allowed(p, dest_cpu))
1888 		return rq;
1889 
1890 	update_rq_clock(rq);
1891 	rq = move_queued_task(rq, rf, p, dest_cpu);
1892 
1893 	return rq;
1894 }
1895 
1896 /*
1897  * migration_cpu_stop - this will be executed by a highprio stopper thread
1898  * and performs thread migration by bumping thread off CPU then
1899  * 'pushing' onto another runqueue.
1900  */
1901 static int migration_cpu_stop(void *data)
1902 {
1903 	struct set_affinity_pending *pending;
1904 	struct migration_arg *arg = data;
1905 	struct task_struct *p = arg->task;
1906 	int dest_cpu = arg->dest_cpu;
1907 	struct rq *rq = this_rq();
1908 	bool complete = false;
1909 	struct rq_flags rf;
1910 
1911 	/*
1912 	 * The original target CPU might have gone down and we might
1913 	 * be on another CPU but it doesn't matter.
1914 	 */
1915 	local_irq_save(rf.flags);
1916 	/*
1917 	 * We need to explicitly wake pending tasks before running
1918 	 * __migrate_task() such that we will not miss enforcing cpus_ptr
1919 	 * during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test.
1920 	 */
1921 	flush_smp_call_function_from_idle();
1922 
1923 	raw_spin_lock(&p->pi_lock);
1924 	rq_lock(rq, &rf);
1925 
1926 	pending = p->migration_pending;
1927 	/*
1928 	 * If task_rq(p) != rq, it cannot be migrated here, because we're
1929 	 * holding rq->lock, if p->on_rq == 0 it cannot get enqueued because
1930 	 * we're holding p->pi_lock.
1931 	 */
1932 	if (task_rq(p) == rq) {
1933 		if (is_migration_disabled(p))
1934 			goto out;
1935 
1936 		if (pending) {
1937 			p->migration_pending = NULL;
1938 			complete = true;
1939 		}
1940 
1941 		/* migrate_enable() --  we must not race against SCA */
1942 		if (dest_cpu < 0) {
1943 			/*
1944 			 * When this was migrate_enable() but we no longer
1945 			 * have a @pending, a concurrent SCA 'fixed' things
1946 			 * and we should be valid again. Nothing to do.
1947 			 */
1948 			if (!pending) {
1949 				WARN_ON_ONCE(!cpumask_test_cpu(task_cpu(p), &p->cpus_mask));
1950 				goto out;
1951 			}
1952 
1953 			dest_cpu = cpumask_any_distribute(&p->cpus_mask);
1954 		}
1955 
1956 		if (task_on_rq_queued(p))
1957 			rq = __migrate_task(rq, &rf, p, dest_cpu);
1958 		else
1959 			p->wake_cpu = dest_cpu;
1960 
1961 	} else if (dest_cpu < 0 || pending) {
1962 		/*
1963 		 * This happens when we get migrated between migrate_enable()'s
1964 		 * preempt_enable() and scheduling the stopper task. At that
1965 		 * point we're a regular task again and not current anymore.
1966 		 *
1967 		 * A !PREEMPT kernel has a giant hole here, which makes it far
1968 		 * more likely.
1969 		 */
1970 
1971 		/*
1972 		 * The task moved before the stopper got to run. We're holding
1973 		 * ->pi_lock, so the allowed mask is stable - if it got
1974 		 * somewhere allowed, we're done.
1975 		 */
1976 		if (pending && cpumask_test_cpu(task_cpu(p), p->cpus_ptr)) {
1977 			p->migration_pending = NULL;
1978 			complete = true;
1979 			goto out;
1980 		}
1981 
1982 		/*
1983 		 * When this was migrate_enable() but we no longer have an
1984 		 * @pending, a concurrent SCA 'fixed' things and we should be
1985 		 * valid again. Nothing to do.
1986 		 */
1987 		if (!pending) {
1988 			WARN_ON_ONCE(!cpumask_test_cpu(task_cpu(p), &p->cpus_mask));
1989 			goto out;
1990 		}
1991 
1992 		/*
1993 		 * When migrate_enable() hits a rq mis-match we can't reliably
1994 		 * determine is_migration_disabled() and so have to chase after
1995 		 * it.
1996 		 */
1997 		task_rq_unlock(rq, p, &rf);
1998 		stop_one_cpu_nowait(task_cpu(p), migration_cpu_stop,
1999 				    &pending->arg, &pending->stop_work);
2000 		return 0;
2001 	}
2002 out:
2003 	task_rq_unlock(rq, p, &rf);
2004 
2005 	if (complete)
2006 		complete_all(&pending->done);
2007 
2008 	/* For pending->{arg,stop_work} */
2009 	pending = arg->pending;
2010 	if (pending && refcount_dec_and_test(&pending->refs))
2011 		wake_up_var(&pending->refs);
2012 
2013 	return 0;
2014 }
2015 
2016 int push_cpu_stop(void *arg)
2017 {
2018 	struct rq *lowest_rq = NULL, *rq = this_rq();
2019 	struct task_struct *p = arg;
2020 
2021 	raw_spin_lock_irq(&p->pi_lock);
2022 	raw_spin_lock(&rq->lock);
2023 
2024 	if (task_rq(p) != rq)
2025 		goto out_unlock;
2026 
2027 	if (is_migration_disabled(p)) {
2028 		p->migration_flags |= MDF_PUSH;
2029 		goto out_unlock;
2030 	}
2031 
2032 	p->migration_flags &= ~MDF_PUSH;
2033 
2034 	if (p->sched_class->find_lock_rq)
2035 		lowest_rq = p->sched_class->find_lock_rq(p, rq);
2036 
2037 	if (!lowest_rq)
2038 		goto out_unlock;
2039 
2040 	// XXX validate p is still the highest prio task
2041 	if (task_rq(p) == rq) {
2042 		deactivate_task(rq, p, 0);
2043 		set_task_cpu(p, lowest_rq->cpu);
2044 		activate_task(lowest_rq, p, 0);
2045 		resched_curr(lowest_rq);
2046 	}
2047 
2048 	double_unlock_balance(rq, lowest_rq);
2049 
2050 out_unlock:
2051 	rq->push_busy = false;
2052 	raw_spin_unlock(&rq->lock);
2053 	raw_spin_unlock_irq(&p->pi_lock);
2054 
2055 	put_task_struct(p);
2056 	return 0;
2057 }
2058 
2059 /*
2060  * sched_class::set_cpus_allowed must do the below, but is not required to
2061  * actually call this function.
2062  */
2063 void set_cpus_allowed_common(struct task_struct *p, const struct cpumask *new_mask, u32 flags)
2064 {
2065 	if (flags & (SCA_MIGRATE_ENABLE | SCA_MIGRATE_DISABLE)) {
2066 		p->cpus_ptr = new_mask;
2067 		return;
2068 	}
2069 
2070 	cpumask_copy(&p->cpus_mask, new_mask);
2071 	p->nr_cpus_allowed = cpumask_weight(new_mask);
2072 }
2073 
2074 static void
2075 __do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask, u32 flags)
2076 {
2077 	struct rq *rq = task_rq(p);
2078 	bool queued, running;
2079 
2080 	/*
2081 	 * This here violates the locking rules for affinity, since we're only
2082 	 * supposed to change these variables while holding both rq->lock and
2083 	 * p->pi_lock.
2084 	 *
2085 	 * HOWEVER, it magically works, because ttwu() is the only code that
2086 	 * accesses these variables under p->pi_lock and only does so after
2087 	 * smp_cond_load_acquire(&p->on_cpu, !VAL), and we're in __schedule()
2088 	 * before finish_task().
2089 	 *
2090 	 * XXX do further audits, this smells like something putrid.
2091 	 */
2092 	if (flags & SCA_MIGRATE_DISABLE)
2093 		SCHED_WARN_ON(!p->on_cpu);
2094 	else
2095 		lockdep_assert_held(&p->pi_lock);
2096 
2097 	queued = task_on_rq_queued(p);
2098 	running = task_current(rq, p);
2099 
2100 	if (queued) {
2101 		/*
2102 		 * Because __kthread_bind() calls this on blocked tasks without
2103 		 * holding rq->lock.
2104 		 */
2105 		lockdep_assert_held(&rq->lock);
2106 		dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK);
2107 	}
2108 	if (running)
2109 		put_prev_task(rq, p);
2110 
2111 	p->sched_class->set_cpus_allowed(p, new_mask, flags);
2112 
2113 	if (queued)
2114 		enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK);
2115 	if (running)
2116 		set_next_task(rq, p);
2117 }
2118 
2119 void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
2120 {
2121 	__do_set_cpus_allowed(p, new_mask, 0);
2122 }
2123 
2124 /*
2125  * This function is wildly self concurrent; here be dragons.
2126  *
2127  *
2128  * When given a valid mask, __set_cpus_allowed_ptr() must block until the
2129  * designated task is enqueued on an allowed CPU. If that task is currently
2130  * running, we have to kick it out using the CPU stopper.
2131  *
2132  * Migrate-Disable comes along and tramples all over our nice sandcastle.
2133  * Consider:
2134  *
2135  *     Initial conditions: P0->cpus_mask = [0, 1]
2136  *
2137  *     P0@CPU0                  P1
2138  *
2139  *     migrate_disable();
2140  *     <preempted>
2141  *                              set_cpus_allowed_ptr(P0, [1]);
2142  *
2143  * P1 *cannot* return from this set_cpus_allowed_ptr() call until P0 executes
2144  * its outermost migrate_enable() (i.e. it exits its Migrate-Disable region).
2145  * This means we need the following scheme:
2146  *
2147  *     P0@CPU0                  P1
2148  *
2149  *     migrate_disable();
2150  *     <preempted>
2151  *                              set_cpus_allowed_ptr(P0, [1]);
2152  *                                <blocks>
2153  *     <resumes>
2154  *     migrate_enable();
2155  *       __set_cpus_allowed_ptr();
2156  *       <wakes local stopper>
2157  *                         `--> <woken on migration completion>
2158  *
2159  * Now the fun stuff: there may be several P1-like tasks, i.e. multiple
2160  * concurrent set_cpus_allowed_ptr(P0, [*]) calls. CPU affinity changes of any
2161  * task p are serialized by p->pi_lock, which we can leverage: the one that
2162  * should come into effect at the end of the Migrate-Disable region is the last
2163  * one. This means we only need to track a single cpumask (i.e. p->cpus_mask),
2164  * but we still need to properly signal those waiting tasks at the appropriate
2165  * moment.
2166  *
2167  * This is implemented using struct set_affinity_pending. The first
2168  * __set_cpus_allowed_ptr() caller within a given Migrate-Disable region will
2169  * setup an instance of that struct and install it on the targeted task_struct.
2170  * Any and all further callers will reuse that instance. Those then wait for
2171  * a completion signaled at the tail of the CPU stopper callback (1), triggered
2172  * on the end of the Migrate-Disable region (i.e. outermost migrate_enable()).
2173  *
2174  *
2175  * (1) In the cases covered above. There is one more where the completion is
2176  * signaled within affine_move_task() itself: when a subsequent affinity request
2177  * cancels the need for an active migration. Consider:
2178  *
2179  *     Initial conditions: P0->cpus_mask = [0, 1]
2180  *
2181  *     P0@CPU0            P1                             P2
2182  *
2183  *     migrate_disable();
2184  *     <preempted>
2185  *                        set_cpus_allowed_ptr(P0, [1]);
2186  *                          <blocks>
2187  *                                                       set_cpus_allowed_ptr(P0, [0, 1]);
2188  *                                                         <signal completion>
2189  *                          <awakes>
2190  *
2191  * Note that the above is safe vs a concurrent migrate_enable(), as any
2192  * pending affinity completion is preceded by an uninstallation of
2193  * p->migration_pending done with p->pi_lock held.
2194  */
2195 static int affine_move_task(struct rq *rq, struct task_struct *p, struct rq_flags *rf,
2196 			    int dest_cpu, unsigned int flags)
2197 {
2198 	struct set_affinity_pending my_pending = { }, *pending = NULL;
2199 	struct migration_arg arg = {
2200 		.task = p,
2201 		.dest_cpu = dest_cpu,
2202 	};
2203 	bool complete = false;
2204 
2205 	/* Can the task run on the task's current CPU? If so, we're done */
2206 	if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask)) {
2207 		struct task_struct *push_task = NULL;
2208 
2209 		if ((flags & SCA_MIGRATE_ENABLE) &&
2210 		    (p->migration_flags & MDF_PUSH) && !rq->push_busy) {
2211 			rq->push_busy = true;
2212 			push_task = get_task_struct(p);
2213 		}
2214 
2215 		pending = p->migration_pending;
2216 		if (pending) {
2217 			refcount_inc(&pending->refs);
2218 			p->migration_pending = NULL;
2219 			complete = true;
2220 		}
2221 		task_rq_unlock(rq, p, rf);
2222 
2223 		if (push_task) {
2224 			stop_one_cpu_nowait(rq->cpu, push_cpu_stop,
2225 					    p, &rq->push_work);
2226 		}
2227 
2228 		if (complete)
2229 			goto do_complete;
2230 
2231 		return 0;
2232 	}
2233 
2234 	if (!(flags & SCA_MIGRATE_ENABLE)) {
2235 		/* serialized by p->pi_lock */
2236 		if (!p->migration_pending) {
2237 			/* Install the request */
2238 			refcount_set(&my_pending.refs, 1);
2239 			init_completion(&my_pending.done);
2240 			p->migration_pending = &my_pending;
2241 		} else {
2242 			pending = p->migration_pending;
2243 			refcount_inc(&pending->refs);
2244 		}
2245 	}
2246 	pending = p->migration_pending;
2247 	/*
2248 	 * - !MIGRATE_ENABLE:
2249 	 *   we'll have installed a pending if there wasn't one already.
2250 	 *
2251 	 * - MIGRATE_ENABLE:
2252 	 *   we're here because the current CPU isn't matching anymore,
2253 	 *   the only way that can happen is because of a concurrent
2254 	 *   set_cpus_allowed_ptr() call, which should then still be
2255 	 *   pending completion.
2256 	 *
2257 	 * Either way, we really should have a @pending here.
2258 	 */
2259 	if (WARN_ON_ONCE(!pending)) {
2260 		task_rq_unlock(rq, p, rf);
2261 		return -EINVAL;
2262 	}
2263 
2264 	if (flags & SCA_MIGRATE_ENABLE) {
2265 
2266 		refcount_inc(&pending->refs); /* pending->{arg,stop_work} */
2267 		p->migration_flags &= ~MDF_PUSH;
2268 		task_rq_unlock(rq, p, rf);
2269 
2270 		pending->arg = (struct migration_arg) {
2271 			.task = p,
2272 			.dest_cpu = -1,
2273 			.pending = pending,
2274 		};
2275 
2276 		stop_one_cpu_nowait(cpu_of(rq), migration_cpu_stop,
2277 				    &pending->arg, &pending->stop_work);
2278 
2279 		return 0;
2280 	}
2281 
2282 	if (task_running(rq, p) || p->state == TASK_WAKING) {
2283 		/*
2284 		 * Lessen races (and headaches) by delegating
2285 		 * is_migration_disabled(p) checks to the stopper, which will
2286 		 * run on the same CPU as said p.
2287 		 */
2288 		task_rq_unlock(rq, p, rf);
2289 		stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
2290 
2291 	} else {
2292 
2293 		if (!is_migration_disabled(p)) {
2294 			if (task_on_rq_queued(p))
2295 				rq = move_queued_task(rq, rf, p, dest_cpu);
2296 
2297 			p->migration_pending = NULL;
2298 			complete = true;
2299 		}
2300 		task_rq_unlock(rq, p, rf);
2301 
2302 do_complete:
2303 		if (complete)
2304 			complete_all(&pending->done);
2305 	}
2306 
2307 	wait_for_completion(&pending->done);
2308 
2309 	if (refcount_dec_and_test(&pending->refs))
2310 		wake_up_var(&pending->refs);
2311 
2312 	/*
2313 	 * Block the original owner of &pending until all subsequent callers
2314 	 * have seen the completion and decremented the refcount
2315 	 */
2316 	wait_var_event(&my_pending.refs, !refcount_read(&my_pending.refs));
2317 
2318 	return 0;
2319 }
2320 
2321 /*
2322  * Change a given task's CPU affinity. Migrate the thread to a
2323  * proper CPU and schedule it away if the CPU it's executing on
2324  * is removed from the allowed bitmask.
2325  *
2326  * NOTE: the caller must have a valid reference to the task, the
2327  * task must not exit() & deallocate itself prematurely. The
2328  * call is not atomic; no spinlocks may be held.
2329  */
2330 static int __set_cpus_allowed_ptr(struct task_struct *p,
2331 				  const struct cpumask *new_mask,
2332 				  u32 flags)
2333 {
2334 	const struct cpumask *cpu_valid_mask = cpu_active_mask;
2335 	unsigned int dest_cpu;
2336 	struct rq_flags rf;
2337 	struct rq *rq;
2338 	int ret = 0;
2339 
2340 	rq = task_rq_lock(p, &rf);
2341 	update_rq_clock(rq);
2342 
2343 	if (p->flags & PF_KTHREAD || is_migration_disabled(p)) {
2344 		/*
2345 		 * Kernel threads are allowed on online && !active CPUs,
2346 		 * however, during cpu-hot-unplug, even these might get pushed
2347 		 * away if not KTHREAD_IS_PER_CPU.
2348 		 *
2349 		 * Specifically, migration_disabled() tasks must not fail the
2350 		 * cpumask_any_and_distribute() pick below, esp. so on
2351 		 * SCA_MIGRATE_ENABLE, otherwise we'll not call
2352 		 * set_cpus_allowed_common() and actually reset p->cpus_ptr.
2353 		 */
2354 		cpu_valid_mask = cpu_online_mask;
2355 	}
2356 
2357 	/*
2358 	 * Must re-check here, to close a race against __kthread_bind(),
2359 	 * sched_setaffinity() is not guaranteed to observe the flag.
2360 	 */
2361 	if ((flags & SCA_CHECK) && (p->flags & PF_NO_SETAFFINITY)) {
2362 		ret = -EINVAL;
2363 		goto out;
2364 	}
2365 
2366 	if (!(flags & SCA_MIGRATE_ENABLE)) {
2367 		if (cpumask_equal(&p->cpus_mask, new_mask))
2368 			goto out;
2369 
2370 		if (WARN_ON_ONCE(p == current &&
2371 				 is_migration_disabled(p) &&
2372 				 !cpumask_test_cpu(task_cpu(p), new_mask))) {
2373 			ret = -EBUSY;
2374 			goto out;
2375 		}
2376 	}
2377 
2378 	/*
2379 	 * Picking a ~random cpu helps in cases where we are changing affinity
2380 	 * for groups of tasks (ie. cpuset), so that load balancing is not
2381 	 * immediately required to distribute the tasks within their new mask.
2382 	 */
2383 	dest_cpu = cpumask_any_and_distribute(cpu_valid_mask, new_mask);
2384 	if (dest_cpu >= nr_cpu_ids) {
2385 		ret = -EINVAL;
2386 		goto out;
2387 	}
2388 
2389 	__do_set_cpus_allowed(p, new_mask, flags);
2390 
2391 	return affine_move_task(rq, p, &rf, dest_cpu, flags);
2392 
2393 out:
2394 	task_rq_unlock(rq, p, &rf);
2395 
2396 	return ret;
2397 }
2398 
2399 int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
2400 {
2401 	return __set_cpus_allowed_ptr(p, new_mask, 0);
2402 }
2403 EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
2404 
2405 void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
2406 {
2407 #ifdef CONFIG_SCHED_DEBUG
2408 	/*
2409 	 * We should never call set_task_cpu() on a blocked task,
2410 	 * ttwu() will sort out the placement.
2411 	 */
2412 	WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING &&
2413 			!p->on_rq);
2414 
2415 	/*
2416 	 * Migrating fair class task must have p->on_rq = TASK_ON_RQ_MIGRATING,
2417 	 * because schedstat_wait_{start,end} rebase migrating task's wait_start
2418 	 * time relying on p->on_rq.
2419 	 */
2420 	WARN_ON_ONCE(p->state == TASK_RUNNING &&
2421 		     p->sched_class == &fair_sched_class &&
2422 		     (p->on_rq && !task_on_rq_migrating(p)));
2423 
2424 #ifdef CONFIG_LOCKDEP
2425 	/*
2426 	 * The caller should hold either p->pi_lock or rq->lock, when changing
2427 	 * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks.
2428 	 *
2429 	 * sched_move_task() holds both and thus holding either pins the cgroup,
2430 	 * see task_group().
2431 	 *
2432 	 * Furthermore, all task_rq users should acquire both locks, see
2433 	 * task_rq_lock().
2434 	 */
2435 	WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) ||
2436 				      lockdep_is_held(&task_rq(p)->lock)));
2437 #endif
2438 	/*
2439 	 * Clearly, migrating tasks to offline CPUs is a fairly daft thing.
2440 	 */
2441 	WARN_ON_ONCE(!cpu_online(new_cpu));
2442 
2443 	WARN_ON_ONCE(is_migration_disabled(p));
2444 #endif
2445 
2446 	trace_sched_migrate_task(p, new_cpu);
2447 
2448 	if (task_cpu(p) != new_cpu) {
2449 		if (p->sched_class->migrate_task_rq)
2450 			p->sched_class->migrate_task_rq(p, new_cpu);
2451 		p->se.nr_migrations++;
2452 		rseq_migrate(p);
2453 		perf_event_task_migrate(p);
2454 	}
2455 
2456 	__set_task_cpu(p, new_cpu);
2457 }
2458 
2459 #ifdef CONFIG_NUMA_BALANCING
2460 static void __migrate_swap_task(struct task_struct *p, int cpu)
2461 {
2462 	if (task_on_rq_queued(p)) {
2463 		struct rq *src_rq, *dst_rq;
2464 		struct rq_flags srf, drf;
2465 
2466 		src_rq = task_rq(p);
2467 		dst_rq = cpu_rq(cpu);
2468 
2469 		rq_pin_lock(src_rq, &srf);
2470 		rq_pin_lock(dst_rq, &drf);
2471 
2472 		deactivate_task(src_rq, p, 0);
2473 		set_task_cpu(p, cpu);
2474 		activate_task(dst_rq, p, 0);
2475 		check_preempt_curr(dst_rq, p, 0);
2476 
2477 		rq_unpin_lock(dst_rq, &drf);
2478 		rq_unpin_lock(src_rq, &srf);
2479 
2480 	} else {
2481 		/*
2482 		 * Task isn't running anymore; make it appear like we migrated
2483 		 * it before it went to sleep. This means on wakeup we make the
2484 		 * previous CPU our target instead of where it really is.
2485 		 */
2486 		p->wake_cpu = cpu;
2487 	}
2488 }
2489 
2490 struct migration_swap_arg {
2491 	struct task_struct *src_task, *dst_task;
2492 	int src_cpu, dst_cpu;
2493 };
2494 
2495 static int migrate_swap_stop(void *data)
2496 {
2497 	struct migration_swap_arg *arg = data;
2498 	struct rq *src_rq, *dst_rq;
2499 	int ret = -EAGAIN;
2500 
2501 	if (!cpu_active(arg->src_cpu) || !cpu_active(arg->dst_cpu))
2502 		return -EAGAIN;
2503 
2504 	src_rq = cpu_rq(arg->src_cpu);
2505 	dst_rq = cpu_rq(arg->dst_cpu);
2506 
2507 	double_raw_lock(&arg->src_task->pi_lock,
2508 			&arg->dst_task->pi_lock);
2509 	double_rq_lock(src_rq, dst_rq);
2510 
2511 	if (task_cpu(arg->dst_task) != arg->dst_cpu)
2512 		goto unlock;
2513 
2514 	if (task_cpu(arg->src_task) != arg->src_cpu)
2515 		goto unlock;
2516 
2517 	if (!cpumask_test_cpu(arg->dst_cpu, arg->src_task->cpus_ptr))
2518 		goto unlock;
2519 
2520 	if (!cpumask_test_cpu(arg->src_cpu, arg->dst_task->cpus_ptr))
2521 		goto unlock;
2522 
2523 	__migrate_swap_task(arg->src_task, arg->dst_cpu);
2524 	__migrate_swap_task(arg->dst_task, arg->src_cpu);
2525 
2526 	ret = 0;
2527 
2528 unlock:
2529 	double_rq_unlock(src_rq, dst_rq);
2530 	raw_spin_unlock(&arg->dst_task->pi_lock);
2531 	raw_spin_unlock(&arg->src_task->pi_lock);
2532 
2533 	return ret;
2534 }
2535 
2536 /*
2537  * Cross migrate two tasks
2538  */
2539 int migrate_swap(struct task_struct *cur, struct task_struct *p,
2540 		int target_cpu, int curr_cpu)
2541 {
2542 	struct migration_swap_arg arg;
2543 	int ret = -EINVAL;
2544 
2545 	arg = (struct migration_swap_arg){
2546 		.src_task = cur,
2547 		.src_cpu = curr_cpu,
2548 		.dst_task = p,
2549 		.dst_cpu = target_cpu,
2550 	};
2551 
2552 	if (arg.src_cpu == arg.dst_cpu)
2553 		goto out;
2554 
2555 	/*
2556 	 * These three tests are all lockless; this is OK since all of them
2557 	 * will be re-checked with proper locks held further down the line.
2558 	 */
2559 	if (!cpu_active(arg.src_cpu) || !cpu_active(arg.dst_cpu))
2560 		goto out;
2561 
2562 	if (!cpumask_test_cpu(arg.dst_cpu, arg.src_task->cpus_ptr))
2563 		goto out;
2564 
2565 	if (!cpumask_test_cpu(arg.src_cpu, arg.dst_task->cpus_ptr))
2566 		goto out;
2567 
2568 	trace_sched_swap_numa(cur, arg.src_cpu, p, arg.dst_cpu);
2569 	ret = stop_two_cpus(arg.dst_cpu, arg.src_cpu, migrate_swap_stop, &arg);
2570 
2571 out:
2572 	return ret;
2573 }
2574 #endif /* CONFIG_NUMA_BALANCING */
2575 
2576 /*
2577  * wait_task_inactive - wait for a thread to unschedule.
2578  *
2579  * If @match_state is nonzero, it's the @p->state value just checked and
2580  * not expected to change.  If it changes, i.e. @p might have woken up,
2581  * then return zero.  When we succeed in waiting for @p to be off its CPU,
2582  * we return a positive number (its total switch count).  If a second call
2583  * a short while later returns the same number, the caller can be sure that
2584  * @p has remained unscheduled the whole time.
2585  *
2586  * The caller must ensure that the task *will* unschedule sometime soon,
2587  * else this function might spin for a *long* time. This function can't
2588  * be called with interrupts off, or it may introduce deadlock with
2589  * smp_call_function() if an IPI is sent by the same process we are
2590  * waiting to become inactive.
2591  */
2592 unsigned long wait_task_inactive(struct task_struct *p, long match_state)
2593 {
2594 	int running, queued;
2595 	struct rq_flags rf;
2596 	unsigned long ncsw;
2597 	struct rq *rq;
2598 
2599 	for (;;) {
2600 		/*
2601 		 * We do the initial early heuristics without holding
2602 		 * any task-queue locks at all. We'll only try to get
2603 		 * the runqueue lock when things look like they will
2604 		 * work out!
2605 		 */
2606 		rq = task_rq(p);
2607 
2608 		/*
2609 		 * If the task is actively running on another CPU
2610 		 * still, just relax and busy-wait without holding
2611 		 * any locks.
2612 		 *
2613 		 * NOTE! Since we don't hold any locks, it's not
2614 		 * even sure that "rq" stays as the right runqueue!
2615 		 * But we don't care, since "task_running()" will
2616 		 * return false if the runqueue has changed and p
2617 		 * is actually now running somewhere else!
2618 		 */
2619 		while (task_running(rq, p)) {
2620 			if (match_state && unlikely(p->state != match_state))
2621 				return 0;
2622 			cpu_relax();
2623 		}
2624 
2625 		/*
2626 		 * Ok, time to look more closely! We need the rq
2627 		 * lock now, to be *sure*. If we're wrong, we'll
2628 		 * just go back and repeat.
2629 		 */
2630 		rq = task_rq_lock(p, &rf);
2631 		trace_sched_wait_task(p);
2632 		running = task_running(rq, p);
2633 		queued = task_on_rq_queued(p);
2634 		ncsw = 0;
2635 		if (!match_state || p->state == match_state)
2636 			ncsw = p->nvcsw | LONG_MIN; /* sets MSB */
2637 		task_rq_unlock(rq, p, &rf);
2638 
2639 		/*
2640 		 * If it changed from the expected state, bail out now.
2641 		 */
2642 		if (unlikely(!ncsw))
2643 			break;
2644 
2645 		/*
2646 		 * Was it really running after all now that we
2647 		 * checked with the proper locks actually held?
2648 		 *
2649 		 * Oops. Go back and try again..
2650 		 */
2651 		if (unlikely(running)) {
2652 			cpu_relax();
2653 			continue;
2654 		}
2655 
2656 		/*
2657 		 * It's not enough that it's not actively running,
2658 		 * it must be off the runqueue _entirely_, and not
2659 		 * preempted!
2660 		 *
2661 		 * So if it was still runnable (but just not actively
2662 		 * running right now), it's preempted, and we should
2663 		 * yield - it could be a while.
2664 		 */
2665 		if (unlikely(queued)) {
2666 			ktime_t to = NSEC_PER_SEC / HZ;
2667 
2668 			set_current_state(TASK_UNINTERRUPTIBLE);
2669 			schedule_hrtimeout(&to, HRTIMER_MODE_REL);
2670 			continue;
2671 		}
2672 
2673 		/*
2674 		 * Ahh, all good. It wasn't running, and it wasn't
2675 		 * runnable, which means that it will never become
2676 		 * running in the future either. We're all done!
2677 		 */
2678 		break;
2679 	}
2680 
2681 	return ncsw;
2682 }
2683 
2684 /***
2685  * kick_process - kick a running thread to enter/exit the kernel
2686  * @p: the to-be-kicked thread
2687  *
2688  * Cause a process which is running on another CPU to enter
2689  * kernel-mode, without any delay. (to get signals handled.)
2690  *
2691  * NOTE: this function doesn't have to take the runqueue lock,
2692  * because all it wants to ensure is that the remote task enters
2693  * the kernel. If the IPI races and the task has been migrated
2694  * to another CPU then no harm is done and the purpose has been
2695  * achieved as well.
2696  */
2697 void kick_process(struct task_struct *p)
2698 {
2699 	int cpu;
2700 
2701 	preempt_disable();
2702 	cpu = task_cpu(p);
2703 	if ((cpu != smp_processor_id()) && task_curr(p))
2704 		smp_send_reschedule(cpu);
2705 	preempt_enable();
2706 }
2707 EXPORT_SYMBOL_GPL(kick_process);
2708 
2709 /*
2710  * ->cpus_ptr is protected by both rq->lock and p->pi_lock
2711  *
2712  * A few notes on cpu_active vs cpu_online:
2713  *
2714  *  - cpu_active must be a subset of cpu_online
2715  *
2716  *  - on CPU-up we allow per-CPU kthreads on the online && !active CPU,
2717  *    see __set_cpus_allowed_ptr(). At this point the newly online
2718  *    CPU isn't yet part of the sched domains, and balancing will not
2719  *    see it.
2720  *
2721  *  - on CPU-down we clear cpu_active() to mask the sched domains and
2722  *    avoid the load balancer to place new tasks on the to be removed
2723  *    CPU. Existing tasks will remain running there and will be taken
2724  *    off.
2725  *
2726  * This means that fallback selection must not select !active CPUs.
2727  * And can assume that any active CPU must be online. Conversely
2728  * select_task_rq() below may allow selection of !active CPUs in order
2729  * to satisfy the above rules.
2730  */
2731 static int select_fallback_rq(int cpu, struct task_struct *p)
2732 {
2733 	int nid = cpu_to_node(cpu);
2734 	const struct cpumask *nodemask = NULL;
2735 	enum { cpuset, possible, fail } state = cpuset;
2736 	int dest_cpu;
2737 
2738 	/*
2739 	 * If the node that the CPU is on has been offlined, cpu_to_node()
2740 	 * will return -1. There is no CPU on the node, and we should
2741 	 * select the CPU on the other node.
2742 	 */
2743 	if (nid != -1) {
2744 		nodemask = cpumask_of_node(nid);
2745 
2746 		/* Look for allowed, online CPU in same node. */
2747 		for_each_cpu(dest_cpu, nodemask) {
2748 			if (!cpu_active(dest_cpu))
2749 				continue;
2750 			if (cpumask_test_cpu(dest_cpu, p->cpus_ptr))
2751 				return dest_cpu;
2752 		}
2753 	}
2754 
2755 	for (;;) {
2756 		/* Any allowed, online CPU? */
2757 		for_each_cpu(dest_cpu, p->cpus_ptr) {
2758 			if (!is_cpu_allowed(p, dest_cpu))
2759 				continue;
2760 
2761 			goto out;
2762 		}
2763 
2764 		/* No more Mr. Nice Guy. */
2765 		switch (state) {
2766 		case cpuset:
2767 			if (IS_ENABLED(CONFIG_CPUSETS)) {
2768 				cpuset_cpus_allowed_fallback(p);
2769 				state = possible;
2770 				break;
2771 			}
2772 			fallthrough;
2773 		case possible:
2774 			/*
2775 			 * XXX When called from select_task_rq() we only
2776 			 * hold p->pi_lock and again violate locking order.
2777 			 *
2778 			 * More yuck to audit.
2779 			 */
2780 			do_set_cpus_allowed(p, cpu_possible_mask);
2781 			state = fail;
2782 			break;
2783 
2784 		case fail:
2785 			BUG();
2786 			break;
2787 		}
2788 	}
2789 
2790 out:
2791 	if (state != cpuset) {
2792 		/*
2793 		 * Don't tell them about moving exiting tasks or
2794 		 * kernel threads (both mm NULL), since they never
2795 		 * leave kernel.
2796 		 */
2797 		if (p->mm && printk_ratelimit()) {
2798 			printk_deferred("process %d (%s) no longer affine to cpu%d\n",
2799 					task_pid_nr(p), p->comm, cpu);
2800 		}
2801 	}
2802 
2803 	return dest_cpu;
2804 }
2805 
2806 /*
2807  * The caller (fork, wakeup) owns p->pi_lock, ->cpus_ptr is stable.
2808  */
2809 static inline
2810 int select_task_rq(struct task_struct *p, int cpu, int wake_flags)
2811 {
2812 	lockdep_assert_held(&p->pi_lock);
2813 
2814 	if (p->nr_cpus_allowed > 1 && !is_migration_disabled(p))
2815 		cpu = p->sched_class->select_task_rq(p, cpu, wake_flags);
2816 	else
2817 		cpu = cpumask_any(p->cpus_ptr);
2818 
2819 	/*
2820 	 * In order not to call set_task_cpu() on a blocking task we need
2821 	 * to rely on ttwu() to place the task on a valid ->cpus_ptr
2822 	 * CPU.
2823 	 *
2824 	 * Since this is common to all placement strategies, this lives here.
2825 	 *
2826 	 * [ this allows ->select_task() to simply return task_cpu(p) and
2827 	 *   not worry about this generic constraint ]
2828 	 */
2829 	if (unlikely(!is_cpu_allowed(p, cpu)))
2830 		cpu = select_fallback_rq(task_cpu(p), p);
2831 
2832 	return cpu;
2833 }
2834 
2835 void sched_set_stop_task(int cpu, struct task_struct *stop)
2836 {
2837 	static struct lock_class_key stop_pi_lock;
2838 	struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };
2839 	struct task_struct *old_stop = cpu_rq(cpu)->stop;
2840 
2841 	if (stop) {
2842 		/*
2843 		 * Make it appear like a SCHED_FIFO task, its something
2844 		 * userspace knows about and won't get confused about.
2845 		 *
2846 		 * Also, it will make PI more or less work without too
2847 		 * much confusion -- but then, stop work should not
2848 		 * rely on PI working anyway.
2849 		 */
2850 		sched_setscheduler_nocheck(stop, SCHED_FIFO, &param);
2851 
2852 		stop->sched_class = &stop_sched_class;
2853 
2854 		/*
2855 		 * The PI code calls rt_mutex_setprio() with ->pi_lock held to
2856 		 * adjust the effective priority of a task. As a result,
2857 		 * rt_mutex_setprio() can trigger (RT) balancing operations,
2858 		 * which can then trigger wakeups of the stop thread to push
2859 		 * around the current task.
2860 		 *
2861 		 * The stop task itself will never be part of the PI-chain, it
2862 		 * never blocks, therefore that ->pi_lock recursion is safe.
2863 		 * Tell lockdep about this by placing the stop->pi_lock in its
2864 		 * own class.
2865 		 */
2866 		lockdep_set_class(&stop->pi_lock, &stop_pi_lock);
2867 	}
2868 
2869 	cpu_rq(cpu)->stop = stop;
2870 
2871 	if (old_stop) {
2872 		/*
2873 		 * Reset it back to a normal scheduling class so that
2874 		 * it can die in pieces.
2875 		 */
2876 		old_stop->sched_class = &rt_sched_class;
2877 	}
2878 }
2879 
2880 #else /* CONFIG_SMP */
2881 
2882 static inline int __set_cpus_allowed_ptr(struct task_struct *p,
2883 					 const struct cpumask *new_mask,
2884 					 u32 flags)
2885 {
2886 	return set_cpus_allowed_ptr(p, new_mask);
2887 }
2888 
2889 static inline void migrate_disable_switch(struct rq *rq, struct task_struct *p) { }
2890 
2891 static inline bool rq_has_pinned_tasks(struct rq *rq)
2892 {
2893 	return false;
2894 }
2895 
2896 #endif /* !CONFIG_SMP */
2897 
2898 static void
2899 ttwu_stat(struct task_struct *p, int cpu, int wake_flags)
2900 {
2901 	struct rq *rq;
2902 
2903 	if (!schedstat_enabled())
2904 		return;
2905 
2906 	rq = this_rq();
2907 
2908 #ifdef CONFIG_SMP
2909 	if (cpu == rq->cpu) {
2910 		__schedstat_inc(rq->ttwu_local);
2911 		__schedstat_inc(p->se.statistics.nr_wakeups_local);
2912 	} else {
2913 		struct sched_domain *sd;
2914 
2915 		__schedstat_inc(p->se.statistics.nr_wakeups_remote);
2916 		rcu_read_lock();
2917 		for_each_domain(rq->cpu, sd) {
2918 			if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
2919 				__schedstat_inc(sd->ttwu_wake_remote);
2920 				break;
2921 			}
2922 		}
2923 		rcu_read_unlock();
2924 	}
2925 
2926 	if (wake_flags & WF_MIGRATED)
2927 		__schedstat_inc(p->se.statistics.nr_wakeups_migrate);
2928 #endif /* CONFIG_SMP */
2929 
2930 	__schedstat_inc(rq->ttwu_count);
2931 	__schedstat_inc(p->se.statistics.nr_wakeups);
2932 
2933 	if (wake_flags & WF_SYNC)
2934 		__schedstat_inc(p->se.statistics.nr_wakeups_sync);
2935 }
2936 
2937 /*
2938  * Mark the task runnable and perform wakeup-preemption.
2939  */
2940 static void ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags,
2941 			   struct rq_flags *rf)
2942 {
2943 	check_preempt_curr(rq, p, wake_flags);
2944 	p->state = TASK_RUNNING;
2945 	trace_sched_wakeup(p);
2946 
2947 #ifdef CONFIG_SMP
2948 	if (p->sched_class->task_woken) {
2949 		/*
2950 		 * Our task @p is fully woken up and running; so it's safe to
2951 		 * drop the rq->lock, hereafter rq is only used for statistics.
2952 		 */
2953 		rq_unpin_lock(rq, rf);
2954 		p->sched_class->task_woken(rq, p);
2955 		rq_repin_lock(rq, rf);
2956 	}
2957 
2958 	if (rq->idle_stamp) {
2959 		u64 delta = rq_clock(rq) - rq->idle_stamp;
2960 		u64 max = 2*rq->max_idle_balance_cost;
2961 
2962 		update_avg(&rq->avg_idle, delta);
2963 
2964 		if (rq->avg_idle > max)
2965 			rq->avg_idle = max;
2966 
2967 		rq->idle_stamp = 0;
2968 	}
2969 #endif
2970 }
2971 
2972 static void
2973 ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags,
2974 		 struct rq_flags *rf)
2975 {
2976 	int en_flags = ENQUEUE_WAKEUP | ENQUEUE_NOCLOCK;
2977 
2978 	lockdep_assert_held(&rq->lock);
2979 
2980 	if (p->sched_contributes_to_load)
2981 		rq->nr_uninterruptible--;
2982 
2983 #ifdef CONFIG_SMP
2984 	if (wake_flags & WF_MIGRATED)
2985 		en_flags |= ENQUEUE_MIGRATED;
2986 	else
2987 #endif
2988 	if (p->in_iowait) {
2989 		delayacct_blkio_end(p);
2990 		atomic_dec(&task_rq(p)->nr_iowait);
2991 	}
2992 
2993 	activate_task(rq, p, en_flags);
2994 	ttwu_do_wakeup(rq, p, wake_flags, rf);
2995 }
2996 
2997 /*
2998  * Consider @p being inside a wait loop:
2999  *
3000  *   for (;;) {
3001  *      set_current_state(TASK_UNINTERRUPTIBLE);
3002  *
3003  *      if (CONDITION)
3004  *         break;
3005  *
3006  *      schedule();
3007  *   }
3008  *   __set_current_state(TASK_RUNNING);
3009  *
3010  * between set_current_state() and schedule(). In this case @p is still
3011  * runnable, so all that needs doing is change p->state back to TASK_RUNNING in
3012  * an atomic manner.
3013  *
3014  * By taking task_rq(p)->lock we serialize against schedule(), if @p->on_rq
3015  * then schedule() must still happen and p->state can be changed to
3016  * TASK_RUNNING. Otherwise we lost the race, schedule() has happened, and we
3017  * need to do a full wakeup with enqueue.
3018  *
3019  * Returns: %true when the wakeup is done,
3020  *          %false otherwise.
3021  */
3022 static int ttwu_runnable(struct task_struct *p, int wake_flags)
3023 {
3024 	struct rq_flags rf;
3025 	struct rq *rq;
3026 	int ret = 0;
3027 
3028 	rq = __task_rq_lock(p, &rf);
3029 	if (task_on_rq_queued(p)) {
3030 		/* check_preempt_curr() may use rq clock */
3031 		update_rq_clock(rq);
3032 		ttwu_do_wakeup(rq, p, wake_flags, &rf);
3033 		ret = 1;
3034 	}
3035 	__task_rq_unlock(rq, &rf);
3036 
3037 	return ret;
3038 }
3039 
3040 #ifdef CONFIG_SMP
3041 void sched_ttwu_pending(void *arg)
3042 {
3043 	struct llist_node *llist = arg;
3044 	struct rq *rq = this_rq();
3045 	struct task_struct *p, *t;
3046 	struct rq_flags rf;
3047 
3048 	if (!llist)
3049 		return;
3050 
3051 	/*
3052 	 * rq::ttwu_pending racy indication of out-standing wakeups.
3053 	 * Races such that false-negatives are possible, since they
3054 	 * are shorter lived that false-positives would be.
3055 	 */
3056 	WRITE_ONCE(rq->ttwu_pending, 0);
3057 
3058 	rq_lock_irqsave(rq, &rf);
3059 	update_rq_clock(rq);
3060 
3061 	llist_for_each_entry_safe(p, t, llist, wake_entry.llist) {
3062 		if (WARN_ON_ONCE(p->on_cpu))
3063 			smp_cond_load_acquire(&p->on_cpu, !VAL);
3064 
3065 		if (WARN_ON_ONCE(task_cpu(p) != cpu_of(rq)))
3066 			set_task_cpu(p, cpu_of(rq));
3067 
3068 		ttwu_do_activate(rq, p, p->sched_remote_wakeup ? WF_MIGRATED : 0, &rf);
3069 	}
3070 
3071 	rq_unlock_irqrestore(rq, &rf);
3072 }
3073 
3074 void send_call_function_single_ipi(int cpu)
3075 {
3076 	struct rq *rq = cpu_rq(cpu);
3077 
3078 	if (!set_nr_if_polling(rq->idle))
3079 		arch_send_call_function_single_ipi(cpu);
3080 	else
3081 		trace_sched_wake_idle_without_ipi(cpu);
3082 }
3083 
3084 /*
3085  * Queue a task on the target CPUs wake_list and wake the CPU via IPI if
3086  * necessary. The wakee CPU on receipt of the IPI will queue the task
3087  * via sched_ttwu_wakeup() for activation so the wakee incurs the cost
3088  * of the wakeup instead of the waker.
3089  */
3090 static void __ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags)
3091 {
3092 	struct rq *rq = cpu_rq(cpu);
3093 
3094 	p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED);
3095 
3096 	WRITE_ONCE(rq->ttwu_pending, 1);
3097 	__smp_call_single_queue(cpu, &p->wake_entry.llist);
3098 }
3099 
3100 void wake_up_if_idle(int cpu)
3101 {
3102 	struct rq *rq = cpu_rq(cpu);
3103 	struct rq_flags rf;
3104 
3105 	rcu_read_lock();
3106 
3107 	if (!is_idle_task(rcu_dereference(rq->curr)))
3108 		goto out;
3109 
3110 	if (set_nr_if_polling(rq->idle)) {
3111 		trace_sched_wake_idle_without_ipi(cpu);
3112 	} else {
3113 		rq_lock_irqsave(rq, &rf);
3114 		if (is_idle_task(rq->curr))
3115 			smp_send_reschedule(cpu);
3116 		/* Else CPU is not idle, do nothing here: */
3117 		rq_unlock_irqrestore(rq, &rf);
3118 	}
3119 
3120 out:
3121 	rcu_read_unlock();
3122 }
3123 
3124 bool cpus_share_cache(int this_cpu, int that_cpu)
3125 {
3126 	return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu);
3127 }
3128 
3129 static inline bool ttwu_queue_cond(int cpu, int wake_flags)
3130 {
3131 	/*
3132 	 * Do not complicate things with the async wake_list while the CPU is
3133 	 * in hotplug state.
3134 	 */
3135 	if (!cpu_active(cpu))
3136 		return false;
3137 
3138 	/*
3139 	 * If the CPU does not share cache, then queue the task on the
3140 	 * remote rqs wakelist to avoid accessing remote data.
3141 	 */
3142 	if (!cpus_share_cache(smp_processor_id(), cpu))
3143 		return true;
3144 
3145 	/*
3146 	 * If the task is descheduling and the only running task on the
3147 	 * CPU then use the wakelist to offload the task activation to
3148 	 * the soon-to-be-idle CPU as the current CPU is likely busy.
3149 	 * nr_running is checked to avoid unnecessary task stacking.
3150 	 */
3151 	if ((wake_flags & WF_ON_CPU) && cpu_rq(cpu)->nr_running <= 1)
3152 		return true;
3153 
3154 	return false;
3155 }
3156 
3157 static bool ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags)
3158 {
3159 	if (sched_feat(TTWU_QUEUE) && ttwu_queue_cond(cpu, wake_flags)) {
3160 		if (WARN_ON_ONCE(cpu == smp_processor_id()))
3161 			return false;
3162 
3163 		sched_clock_cpu(cpu); /* Sync clocks across CPUs */
3164 		__ttwu_queue_wakelist(p, cpu, wake_flags);
3165 		return true;
3166 	}
3167 
3168 	return false;
3169 }
3170 
3171 #else /* !CONFIG_SMP */
3172 
3173 static inline bool ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags)
3174 {
3175 	return false;
3176 }
3177 
3178 #endif /* CONFIG_SMP */
3179 
3180 static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags)
3181 {
3182 	struct rq *rq = cpu_rq(cpu);
3183 	struct rq_flags rf;
3184 
3185 	if (ttwu_queue_wakelist(p, cpu, wake_flags))
3186 		return;
3187 
3188 	rq_lock(rq, &rf);
3189 	update_rq_clock(rq);
3190 	ttwu_do_activate(rq, p, wake_flags, &rf);
3191 	rq_unlock(rq, &rf);
3192 }
3193 
3194 /*
3195  * Notes on Program-Order guarantees on SMP systems.
3196  *
3197  *  MIGRATION
3198  *
3199  * The basic program-order guarantee on SMP systems is that when a task [t]
3200  * migrates, all its activity on its old CPU [c0] happens-before any subsequent
3201  * execution on its new CPU [c1].
3202  *
3203  * For migration (of runnable tasks) this is provided by the following means:
3204  *
3205  *  A) UNLOCK of the rq(c0)->lock scheduling out task t
3206  *  B) migration for t is required to synchronize *both* rq(c0)->lock and
3207  *     rq(c1)->lock (if not at the same time, then in that order).
3208  *  C) LOCK of the rq(c1)->lock scheduling in task
3209  *
3210  * Release/acquire chaining guarantees that B happens after A and C after B.
3211  * Note: the CPU doing B need not be c0 or c1
3212  *
3213  * Example:
3214  *
3215  *   CPU0            CPU1            CPU2
3216  *
3217  *   LOCK rq(0)->lock
3218  *   sched-out X
3219  *   sched-in Y
3220  *   UNLOCK rq(0)->lock
3221  *
3222  *                                   LOCK rq(0)->lock // orders against CPU0
3223  *                                   dequeue X
3224  *                                   UNLOCK rq(0)->lock
3225  *
3226  *                                   LOCK rq(1)->lock
3227  *                                   enqueue X
3228  *                                   UNLOCK rq(1)->lock
3229  *
3230  *                   LOCK rq(1)->lock // orders against CPU2
3231  *                   sched-out Z
3232  *                   sched-in X
3233  *                   UNLOCK rq(1)->lock
3234  *
3235  *
3236  *  BLOCKING -- aka. SLEEP + WAKEUP
3237  *
3238  * For blocking we (obviously) need to provide the same guarantee as for
3239  * migration. However the means are completely different as there is no lock
3240  * chain to provide order. Instead we do:
3241  *
3242  *   1) smp_store_release(X->on_cpu, 0)   -- finish_task()
3243  *   2) smp_cond_load_acquire(!X->on_cpu) -- try_to_wake_up()
3244  *
3245  * Example:
3246  *
3247  *   CPU0 (schedule)  CPU1 (try_to_wake_up) CPU2 (schedule)
3248  *
3249  *   LOCK rq(0)->lock LOCK X->pi_lock
3250  *   dequeue X
3251  *   sched-out X
3252  *   smp_store_release(X->on_cpu, 0);
3253  *
3254  *                    smp_cond_load_acquire(&X->on_cpu, !VAL);
3255  *                    X->state = WAKING
3256  *                    set_task_cpu(X,2)
3257  *
3258  *                    LOCK rq(2)->lock
3259  *                    enqueue X
3260  *                    X->state = RUNNING
3261  *                    UNLOCK rq(2)->lock
3262  *
3263  *                                          LOCK rq(2)->lock // orders against CPU1
3264  *                                          sched-out Z
3265  *                                          sched-in X
3266  *                                          UNLOCK rq(2)->lock
3267  *
3268  *                    UNLOCK X->pi_lock
3269  *   UNLOCK rq(0)->lock
3270  *
3271  *
3272  * However, for wakeups there is a second guarantee we must provide, namely we
3273  * must ensure that CONDITION=1 done by the caller can not be reordered with
3274  * accesses to the task state; see try_to_wake_up() and set_current_state().
3275  */
3276 
3277 /**
3278  * try_to_wake_up - wake up a thread
3279  * @p: the thread to be awakened
3280  * @state: the mask of task states that can be woken
3281  * @wake_flags: wake modifier flags (WF_*)
3282  *
3283  * Conceptually does:
3284  *
3285  *   If (@state & @p->state) @p->state = TASK_RUNNING.
3286  *
3287  * If the task was not queued/runnable, also place it back on a runqueue.
3288  *
3289  * This function is atomic against schedule() which would dequeue the task.
3290  *
3291  * It issues a full memory barrier before accessing @p->state, see the comment
3292  * with set_current_state().
3293  *
3294  * Uses p->pi_lock to serialize against concurrent wake-ups.
3295  *
3296  * Relies on p->pi_lock stabilizing:
3297  *  - p->sched_class
3298  *  - p->cpus_ptr
3299  *  - p->sched_task_group
3300  * in order to do migration, see its use of select_task_rq()/set_task_cpu().
3301  *
3302  * Tries really hard to only take one task_rq(p)->lock for performance.
3303  * Takes rq->lock in:
3304  *  - ttwu_runnable()    -- old rq, unavoidable, see comment there;
3305  *  - ttwu_queue()       -- new rq, for enqueue of the task;
3306  *  - psi_ttwu_dequeue() -- much sadness :-( accounting will kill us.
3307  *
3308  * As a consequence we race really badly with just about everything. See the
3309  * many memory barriers and their comments for details.
3310  *
3311  * Return: %true if @p->state changes (an actual wakeup was done),
3312  *	   %false otherwise.
3313  */
3314 static int
3315 try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
3316 {
3317 	unsigned long flags;
3318 	int cpu, success = 0;
3319 
3320 	preempt_disable();
3321 	if (p == current) {
3322 		/*
3323 		 * We're waking current, this means 'p->on_rq' and 'task_cpu(p)
3324 		 * == smp_processor_id()'. Together this means we can special
3325 		 * case the whole 'p->on_rq && ttwu_runnable()' case below
3326 		 * without taking any locks.
3327 		 *
3328 		 * In particular:
3329 		 *  - we rely on Program-Order guarantees for all the ordering,
3330 		 *  - we're serialized against set_special_state() by virtue of
3331 		 *    it disabling IRQs (this allows not taking ->pi_lock).
3332 		 */
3333 		if (!(p->state & state))
3334 			goto out;
3335 
3336 		success = 1;
3337 		trace_sched_waking(p);
3338 		p->state = TASK_RUNNING;
3339 		trace_sched_wakeup(p);
3340 		goto out;
3341 	}
3342 
3343 	/*
3344 	 * If we are going to wake up a thread waiting for CONDITION we
3345 	 * need to ensure that CONDITION=1 done by the caller can not be
3346 	 * reordered with p->state check below. This pairs with smp_store_mb()
3347 	 * in set_current_state() that the waiting thread does.
3348 	 */
3349 	raw_spin_lock_irqsave(&p->pi_lock, flags);
3350 	smp_mb__after_spinlock();
3351 	if (!(p->state & state))
3352 		goto unlock;
3353 
3354 	trace_sched_waking(p);
3355 
3356 	/* We're going to change ->state: */
3357 	success = 1;
3358 
3359 	/*
3360 	 * Ensure we load p->on_rq _after_ p->state, otherwise it would
3361 	 * be possible to, falsely, observe p->on_rq == 0 and get stuck
3362 	 * in smp_cond_load_acquire() below.
3363 	 *
3364 	 * sched_ttwu_pending()			try_to_wake_up()
3365 	 *   STORE p->on_rq = 1			  LOAD p->state
3366 	 *   UNLOCK rq->lock
3367 	 *
3368 	 * __schedule() (switch to task 'p')
3369 	 *   LOCK rq->lock			  smp_rmb();
3370 	 *   smp_mb__after_spinlock();
3371 	 *   UNLOCK rq->lock
3372 	 *
3373 	 * [task p]
3374 	 *   STORE p->state = UNINTERRUPTIBLE	  LOAD p->on_rq
3375 	 *
3376 	 * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in
3377 	 * __schedule().  See the comment for smp_mb__after_spinlock().
3378 	 *
3379 	 * A similar smb_rmb() lives in try_invoke_on_locked_down_task().
3380 	 */
3381 	smp_rmb();
3382 	if (READ_ONCE(p->on_rq) && ttwu_runnable(p, wake_flags))
3383 		goto unlock;
3384 
3385 #ifdef CONFIG_SMP
3386 	/*
3387 	 * Ensure we load p->on_cpu _after_ p->on_rq, otherwise it would be
3388 	 * possible to, falsely, observe p->on_cpu == 0.
3389 	 *
3390 	 * One must be running (->on_cpu == 1) in order to remove oneself
3391 	 * from the runqueue.
3392 	 *
3393 	 * __schedule() (switch to task 'p')	try_to_wake_up()
3394 	 *   STORE p->on_cpu = 1		  LOAD p->on_rq
3395 	 *   UNLOCK rq->lock
3396 	 *
3397 	 * __schedule() (put 'p' to sleep)
3398 	 *   LOCK rq->lock			  smp_rmb();
3399 	 *   smp_mb__after_spinlock();
3400 	 *   STORE p->on_rq = 0			  LOAD p->on_cpu
3401 	 *
3402 	 * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in
3403 	 * __schedule().  See the comment for smp_mb__after_spinlock().
3404 	 *
3405 	 * Form a control-dep-acquire with p->on_rq == 0 above, to ensure
3406 	 * schedule()'s deactivate_task() has 'happened' and p will no longer
3407 	 * care about it's own p->state. See the comment in __schedule().
3408 	 */
3409 	smp_acquire__after_ctrl_dep();
3410 
3411 	/*
3412 	 * We're doing the wakeup (@success == 1), they did a dequeue (p->on_rq
3413 	 * == 0), which means we need to do an enqueue, change p->state to
3414 	 * TASK_WAKING such that we can unlock p->pi_lock before doing the
3415 	 * enqueue, such as ttwu_queue_wakelist().
3416 	 */
3417 	p->state = TASK_WAKING;
3418 
3419 	/*
3420 	 * If the owning (remote) CPU is still in the middle of schedule() with
3421 	 * this task as prev, considering queueing p on the remote CPUs wake_list
3422 	 * which potentially sends an IPI instead of spinning on p->on_cpu to
3423 	 * let the waker make forward progress. This is safe because IRQs are
3424 	 * disabled and the IPI will deliver after on_cpu is cleared.
3425 	 *
3426 	 * Ensure we load task_cpu(p) after p->on_cpu:
3427 	 *
3428 	 * set_task_cpu(p, cpu);
3429 	 *   STORE p->cpu = @cpu
3430 	 * __schedule() (switch to task 'p')
3431 	 *   LOCK rq->lock
3432 	 *   smp_mb__after_spin_lock()		smp_cond_load_acquire(&p->on_cpu)
3433 	 *   STORE p->on_cpu = 1		LOAD p->cpu
3434 	 *
3435 	 * to ensure we observe the correct CPU on which the task is currently
3436 	 * scheduling.
3437 	 */
3438 	if (smp_load_acquire(&p->on_cpu) &&
3439 	    ttwu_queue_wakelist(p, task_cpu(p), wake_flags | WF_ON_CPU))
3440 		goto unlock;
3441 
3442 	/*
3443 	 * If the owning (remote) CPU is still in the middle of schedule() with
3444 	 * this task as prev, wait until it's done referencing the task.
3445 	 *
3446 	 * Pairs with the smp_store_release() in finish_task().
3447 	 *
3448 	 * This ensures that tasks getting woken will be fully ordered against
3449 	 * their previous state and preserve Program Order.
3450 	 */
3451 	smp_cond_load_acquire(&p->on_cpu, !VAL);
3452 
3453 	cpu = select_task_rq(p, p->wake_cpu, wake_flags | WF_TTWU);
3454 	if (task_cpu(p) != cpu) {
3455 		if (p->in_iowait) {
3456 			delayacct_blkio_end(p);
3457 			atomic_dec(&task_rq(p)->nr_iowait);
3458 		}
3459 
3460 		wake_flags |= WF_MIGRATED;
3461 		psi_ttwu_dequeue(p);
3462 		set_task_cpu(p, cpu);
3463 	}
3464 #else
3465 	cpu = task_cpu(p);
3466 #endif /* CONFIG_SMP */
3467 
3468 	ttwu_queue(p, cpu, wake_flags);
3469 unlock:
3470 	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
3471 out:
3472 	if (success)
3473 		ttwu_stat(p, task_cpu(p), wake_flags);
3474 	preempt_enable();
3475 
3476 	return success;
3477 }
3478 
3479 /**
3480  * try_invoke_on_locked_down_task - Invoke a function on task in fixed state
3481  * @p: Process for which the function is to be invoked.
3482  * @func: Function to invoke.
3483  * @arg: Argument to function.
3484  *
3485  * If the specified task can be quickly locked into a definite state
3486  * (either sleeping or on a given runqueue), arrange to keep it in that
3487  * state while invoking @func(@arg).  This function can use ->on_rq and
3488  * task_curr() to work out what the state is, if required.  Given that
3489  * @func can be invoked with a runqueue lock held, it had better be quite
3490  * lightweight.
3491  *
3492  * Returns:
3493  *	@false if the task slipped out from under the locks.
3494  *	@true if the task was locked onto a runqueue or is sleeping.
3495  *		However, @func can override this by returning @false.
3496  */
3497 bool try_invoke_on_locked_down_task(struct task_struct *p, bool (*func)(struct task_struct *t, void *arg), void *arg)
3498 {
3499 	bool ret = false;
3500 	struct rq_flags rf;
3501 	struct rq *rq;
3502 
3503 	lockdep_assert_irqs_enabled();
3504 	raw_spin_lock_irq(&p->pi_lock);
3505 	if (p->on_rq) {
3506 		rq = __task_rq_lock(p, &rf);
3507 		if (task_rq(p) == rq)
3508 			ret = func(p, arg);
3509 		rq_unlock(rq, &rf);
3510 	} else {
3511 		switch (p->state) {
3512 		case TASK_RUNNING:
3513 		case TASK_WAKING:
3514 			break;
3515 		default:
3516 			smp_rmb(); // See smp_rmb() comment in try_to_wake_up().
3517 			if (!p->on_rq)
3518 				ret = func(p, arg);
3519 		}
3520 	}
3521 	raw_spin_unlock_irq(&p->pi_lock);
3522 	return ret;
3523 }
3524 
3525 /**
3526  * wake_up_process - Wake up a specific process
3527  * @p: The process to be woken up.
3528  *
3529  * Attempt to wake up the nominated process and move it to the set of runnable
3530  * processes.
3531  *
3532  * Return: 1 if the process was woken up, 0 if it was already running.
3533  *
3534  * This function executes a full memory barrier before accessing the task state.
3535  */
3536 int wake_up_process(struct task_struct *p)
3537 {
3538 	return try_to_wake_up(p, TASK_NORMAL, 0);
3539 }
3540 EXPORT_SYMBOL(wake_up_process);
3541 
3542 int wake_up_state(struct task_struct *p, unsigned int state)
3543 {
3544 	return try_to_wake_up(p, state, 0);
3545 }
3546 
3547 /*
3548  * Perform scheduler related setup for a newly forked process p.
3549  * p is forked by current.
3550  *
3551  * __sched_fork() is basic setup used by init_idle() too:
3552  */
3553 static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
3554 {
3555 	p->on_rq			= 0;
3556 
3557 	p->se.on_rq			= 0;
3558 	p->se.exec_start		= 0;
3559 	p->se.sum_exec_runtime		= 0;
3560 	p->se.prev_sum_exec_runtime	= 0;
3561 	p->se.nr_migrations		= 0;
3562 	p->se.vruntime			= 0;
3563 	INIT_LIST_HEAD(&p->se.group_node);
3564 
3565 #ifdef CONFIG_FAIR_GROUP_SCHED
3566 	p->se.cfs_rq			= NULL;
3567 #endif
3568 
3569 #ifdef CONFIG_SCHEDSTATS
3570 	/* Even if schedstat is disabled, there should not be garbage */
3571 	memset(&p->se.statistics, 0, sizeof(p->se.statistics));
3572 #endif
3573 
3574 	RB_CLEAR_NODE(&p->dl.rb_node);
3575 	init_dl_task_timer(&p->dl);
3576 	init_dl_inactive_task_timer(&p->dl);
3577 	__dl_clear_params(p);
3578 
3579 	INIT_LIST_HEAD(&p->rt.run_list);
3580 	p->rt.timeout		= 0;
3581 	p->rt.time_slice	= sched_rr_timeslice;
3582 	p->rt.on_rq		= 0;
3583 	p->rt.on_list		= 0;
3584 
3585 #ifdef CONFIG_PREEMPT_NOTIFIERS
3586 	INIT_HLIST_HEAD(&p->preempt_notifiers);
3587 #endif
3588 
3589 #ifdef CONFIG_COMPACTION
3590 	p->capture_control = NULL;
3591 #endif
3592 	init_numa_balancing(clone_flags, p);
3593 #ifdef CONFIG_SMP
3594 	p->wake_entry.u_flags = CSD_TYPE_TTWU;
3595 	p->migration_pending = NULL;
3596 #endif
3597 }
3598 
3599 DEFINE_STATIC_KEY_FALSE(sched_numa_balancing);
3600 
3601 #ifdef CONFIG_NUMA_BALANCING
3602 
3603 void set_numabalancing_state(bool enabled)
3604 {
3605 	if (enabled)
3606 		static_branch_enable(&sched_numa_balancing);
3607 	else
3608 		static_branch_disable(&sched_numa_balancing);
3609 }
3610 
3611 #ifdef CONFIG_PROC_SYSCTL
3612 int sysctl_numa_balancing(struct ctl_table *table, int write,
3613 			  void *buffer, size_t *lenp, loff_t *ppos)
3614 {
3615 	struct ctl_table t;
3616 	int err;
3617 	int state = static_branch_likely(&sched_numa_balancing);
3618 
3619 	if (write && !capable(CAP_SYS_ADMIN))
3620 		return -EPERM;
3621 
3622 	t = *table;
3623 	t.data = &state;
3624 	err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
3625 	if (err < 0)
3626 		return err;
3627 	if (write)
3628 		set_numabalancing_state(state);
3629 	return err;
3630 }
3631 #endif
3632 #endif
3633 
3634 #ifdef CONFIG_SCHEDSTATS
3635 
3636 DEFINE_STATIC_KEY_FALSE(sched_schedstats);
3637 static bool __initdata __sched_schedstats = false;
3638 
3639 static void set_schedstats(bool enabled)
3640 {
3641 	if (enabled)
3642 		static_branch_enable(&sched_schedstats);
3643 	else
3644 		static_branch_disable(&sched_schedstats);
3645 }
3646 
3647 void force_schedstat_enabled(void)
3648 {
3649 	if (!schedstat_enabled()) {
3650 		pr_info("kernel profiling enabled schedstats, disable via kernel.sched_schedstats.\n");
3651 		static_branch_enable(&sched_schedstats);
3652 	}
3653 }
3654 
3655 static int __init setup_schedstats(char *str)
3656 {
3657 	int ret = 0;
3658 	if (!str)
3659 		goto out;
3660 
3661 	/*
3662 	 * This code is called before jump labels have been set up, so we can't
3663 	 * change the static branch directly just yet.  Instead set a temporary
3664 	 * variable so init_schedstats() can do it later.
3665 	 */
3666 	if (!strcmp(str, "enable")) {
3667 		__sched_schedstats = true;
3668 		ret = 1;
3669 	} else if (!strcmp(str, "disable")) {
3670 		__sched_schedstats = false;
3671 		ret = 1;
3672 	}
3673 out:
3674 	if (!ret)
3675 		pr_warn("Unable to parse schedstats=\n");
3676 
3677 	return ret;
3678 }
3679 __setup("schedstats=", setup_schedstats);
3680 
3681 static void __init init_schedstats(void)
3682 {
3683 	set_schedstats(__sched_schedstats);
3684 }
3685 
3686 #ifdef CONFIG_PROC_SYSCTL
3687 int sysctl_schedstats(struct ctl_table *table, int write, void *buffer,
3688 		size_t *lenp, loff_t *ppos)
3689 {
3690 	struct ctl_table t;
3691 	int err;
3692 	int state = static_branch_likely(&sched_schedstats);
3693 
3694 	if (write && !capable(CAP_SYS_ADMIN))
3695 		return -EPERM;
3696 
3697 	t = *table;
3698 	t.data = &state;
3699 	err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
3700 	if (err < 0)
3701 		return err;
3702 	if (write)
3703 		set_schedstats(state);
3704 	return err;
3705 }
3706 #endif /* CONFIG_PROC_SYSCTL */
3707 #else  /* !CONFIG_SCHEDSTATS */
3708 static inline void init_schedstats(void) {}
3709 #endif /* CONFIG_SCHEDSTATS */
3710 
3711 /*
3712  * fork()/clone()-time setup:
3713  */
3714 int sched_fork(unsigned long clone_flags, struct task_struct *p)
3715 {
3716 	unsigned long flags;
3717 
3718 	__sched_fork(clone_flags, p);
3719 	/*
3720 	 * We mark the process as NEW here. This guarantees that
3721 	 * nobody will actually run it, and a signal or other external
3722 	 * event cannot wake it up and insert it on the runqueue either.
3723 	 */
3724 	p->state = TASK_NEW;
3725 
3726 	/*
3727 	 * Make sure we do not leak PI boosting priority to the child.
3728 	 */
3729 	p->prio = current->normal_prio;
3730 
3731 	uclamp_fork(p);
3732 
3733 	/*
3734 	 * Revert to default priority/policy on fork if requested.
3735 	 */
3736 	if (unlikely(p->sched_reset_on_fork)) {
3737 		if (task_has_dl_policy(p) || task_has_rt_policy(p)) {
3738 			p->policy = SCHED_NORMAL;
3739 			p->static_prio = NICE_TO_PRIO(0);
3740 			p->rt_priority = 0;
3741 		} else if (PRIO_TO_NICE(p->static_prio) < 0)
3742 			p->static_prio = NICE_TO_PRIO(0);
3743 
3744 		p->prio = p->normal_prio = __normal_prio(p);
3745 		set_load_weight(p, false);
3746 
3747 		/*
3748 		 * We don't need the reset flag anymore after the fork. It has
3749 		 * fulfilled its duty:
3750 		 */
3751 		p->sched_reset_on_fork = 0;
3752 	}
3753 
3754 	if (dl_prio(p->prio))
3755 		return -EAGAIN;
3756 	else if (rt_prio(p->prio))
3757 		p->sched_class = &rt_sched_class;
3758 	else
3759 		p->sched_class = &fair_sched_class;
3760 
3761 	init_entity_runnable_average(&p->se);
3762 
3763 	/*
3764 	 * The child is not yet in the pid-hash so no cgroup attach races,
3765 	 * and the cgroup is pinned to this child due to cgroup_fork()
3766 	 * is ran before sched_fork().
3767 	 *
3768 	 * Silence PROVE_RCU.
3769 	 */
3770 	raw_spin_lock_irqsave(&p->pi_lock, flags);
3771 	rseq_migrate(p);
3772 	/*
3773 	 * We're setting the CPU for the first time, we don't migrate,
3774 	 * so use __set_task_cpu().
3775 	 */
3776 	__set_task_cpu(p, smp_processor_id());
3777 	if (p->sched_class->task_fork)
3778 		p->sched_class->task_fork(p);
3779 	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
3780 
3781 #ifdef CONFIG_SCHED_INFO
3782 	if (likely(sched_info_on()))
3783 		memset(&p->sched_info, 0, sizeof(p->sched_info));
3784 #endif
3785 #if defined(CONFIG_SMP)
3786 	p->on_cpu = 0;
3787 #endif
3788 	init_task_preempt_count(p);
3789 #ifdef CONFIG_SMP
3790 	plist_node_init(&p->pushable_tasks, MAX_PRIO);
3791 	RB_CLEAR_NODE(&p->pushable_dl_tasks);
3792 #endif
3793 	return 0;
3794 }
3795 
3796 void sched_post_fork(struct task_struct *p)
3797 {
3798 	uclamp_post_fork(p);
3799 }
3800 
3801 unsigned long to_ratio(u64 period, u64 runtime)
3802 {
3803 	if (runtime == RUNTIME_INF)
3804 		return BW_UNIT;
3805 
3806 	/*
3807 	 * Doing this here saves a lot of checks in all
3808 	 * the calling paths, and returning zero seems
3809 	 * safe for them anyway.
3810 	 */
3811 	if (period == 0)
3812 		return 0;
3813 
3814 	return div64_u64(runtime << BW_SHIFT, period);
3815 }
3816 
3817 /*
3818  * wake_up_new_task - wake up a newly created task for the first time.
3819  *
3820  * This function will do some initial scheduler statistics housekeeping
3821  * that must be done for every newly created context, then puts the task
3822  * on the runqueue and wakes it.
3823  */
3824 void wake_up_new_task(struct task_struct *p)
3825 {
3826 	struct rq_flags rf;
3827 	struct rq *rq;
3828 
3829 	raw_spin_lock_irqsave(&p->pi_lock, rf.flags);
3830 	p->state = TASK_RUNNING;
3831 #ifdef CONFIG_SMP
3832 	/*
3833 	 * Fork balancing, do it here and not earlier because:
3834 	 *  - cpus_ptr can change in the fork path
3835 	 *  - any previously selected CPU might disappear through hotplug
3836 	 *
3837 	 * Use __set_task_cpu() to avoid calling sched_class::migrate_task_rq,
3838 	 * as we're not fully set-up yet.
3839 	 */
3840 	p->recent_used_cpu = task_cpu(p);
3841 	rseq_migrate(p);
3842 	__set_task_cpu(p, select_task_rq(p, task_cpu(p), WF_FORK));
3843 #endif
3844 	rq = __task_rq_lock(p, &rf);
3845 	update_rq_clock(rq);
3846 	post_init_entity_util_avg(p);
3847 
3848 	activate_task(rq, p, ENQUEUE_NOCLOCK);
3849 	trace_sched_wakeup_new(p);
3850 	check_preempt_curr(rq, p, WF_FORK);
3851 #ifdef CONFIG_SMP
3852 	if (p->sched_class->task_woken) {
3853 		/*
3854 		 * Nothing relies on rq->lock after this, so it's fine to
3855 		 * drop it.
3856 		 */
3857 		rq_unpin_lock(rq, &rf);
3858 		p->sched_class->task_woken(rq, p);
3859 		rq_repin_lock(rq, &rf);
3860 	}
3861 #endif
3862 	task_rq_unlock(rq, p, &rf);
3863 }
3864 
3865 #ifdef CONFIG_PREEMPT_NOTIFIERS
3866 
3867 static DEFINE_STATIC_KEY_FALSE(preempt_notifier_key);
3868 
3869 void preempt_notifier_inc(void)
3870 {
3871 	static_branch_inc(&preempt_notifier_key);
3872 }
3873 EXPORT_SYMBOL_GPL(preempt_notifier_inc);
3874 
3875 void preempt_notifier_dec(void)
3876 {
3877 	static_branch_dec(&preempt_notifier_key);
3878 }
3879 EXPORT_SYMBOL_GPL(preempt_notifier_dec);
3880 
3881 /**
3882  * preempt_notifier_register - tell me when current is being preempted & rescheduled
3883  * @notifier: notifier struct to register
3884  */
3885 void preempt_notifier_register(struct preempt_notifier *notifier)
3886 {
3887 	if (!static_branch_unlikely(&preempt_notifier_key))
3888 		WARN(1, "registering preempt_notifier while notifiers disabled\n");
3889 
3890 	hlist_add_head(&notifier->link, &current->preempt_notifiers);
3891 }
3892 EXPORT_SYMBOL_GPL(preempt_notifier_register);
3893 
3894 /**
3895  * preempt_notifier_unregister - no longer interested in preemption notifications
3896  * @notifier: notifier struct to unregister
3897  *
3898  * This is *not* safe to call from within a preemption notifier.
3899  */
3900 void preempt_notifier_unregister(struct preempt_notifier *notifier)
3901 {
3902 	hlist_del(&notifier->link);
3903 }
3904 EXPORT_SYMBOL_GPL(preempt_notifier_unregister);
3905 
3906 static void __fire_sched_in_preempt_notifiers(struct task_struct *curr)
3907 {
3908 	struct preempt_notifier *notifier;
3909 
3910 	hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
3911 		notifier->ops->sched_in(notifier, raw_smp_processor_id());
3912 }
3913 
3914 static __always_inline void fire_sched_in_preempt_notifiers(struct task_struct *curr)
3915 {
3916 	if (static_branch_unlikely(&preempt_notifier_key))
3917 		__fire_sched_in_preempt_notifiers(curr);
3918 }
3919 
3920 static void
3921 __fire_sched_out_preempt_notifiers(struct task_struct *curr,
3922 				   struct task_struct *next)
3923 {
3924 	struct preempt_notifier *notifier;
3925 
3926 	hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
3927 		notifier->ops->sched_out(notifier, next);
3928 }
3929 
3930 static __always_inline void
3931 fire_sched_out_preempt_notifiers(struct task_struct *curr,
3932 				 struct task_struct *next)
3933 {
3934 	if (static_branch_unlikely(&preempt_notifier_key))
3935 		__fire_sched_out_preempt_notifiers(curr, next);
3936 }
3937 
3938 #else /* !CONFIG_PREEMPT_NOTIFIERS */
3939 
3940 static inline void fire_sched_in_preempt_notifiers(struct task_struct *curr)
3941 {
3942 }
3943 
3944 static inline void
3945 fire_sched_out_preempt_notifiers(struct task_struct *curr,
3946 				 struct task_struct *next)
3947 {
3948 }
3949 
3950 #endif /* CONFIG_PREEMPT_NOTIFIERS */
3951 
3952 static inline void prepare_task(struct task_struct *next)
3953 {
3954 #ifdef CONFIG_SMP
3955 	/*
3956 	 * Claim the task as running, we do this before switching to it
3957 	 * such that any running task will have this set.
3958 	 *
3959 	 * See the ttwu() WF_ON_CPU case and its ordering comment.
3960 	 */
3961 	WRITE_ONCE(next->on_cpu, 1);
3962 #endif
3963 }
3964 
3965 static inline void finish_task(struct task_struct *prev)
3966 {
3967 #ifdef CONFIG_SMP
3968 	/*
3969 	 * This must be the very last reference to @prev from this CPU. After
3970 	 * p->on_cpu is cleared, the task can be moved to a different CPU. We
3971 	 * must ensure this doesn't happen until the switch is completely
3972 	 * finished.
3973 	 *
3974 	 * In particular, the load of prev->state in finish_task_switch() must
3975 	 * happen before this.
3976 	 *
3977 	 * Pairs with the smp_cond_load_acquire() in try_to_wake_up().
3978 	 */
3979 	smp_store_release(&prev->on_cpu, 0);
3980 #endif
3981 }
3982 
3983 #ifdef CONFIG_SMP
3984 
3985 static void do_balance_callbacks(struct rq *rq, struct callback_head *head)
3986 {
3987 	void (*func)(struct rq *rq);
3988 	struct callback_head *next;
3989 
3990 	lockdep_assert_held(&rq->lock);
3991 
3992 	while (head) {
3993 		func = (void (*)(struct rq *))head->func;
3994 		next = head->next;
3995 		head->next = NULL;
3996 		head = next;
3997 
3998 		func(rq);
3999 	}
4000 }
4001 
4002 static void balance_push(struct rq *rq);
4003 
4004 struct callback_head balance_push_callback = {
4005 	.next = NULL,
4006 	.func = (void (*)(struct callback_head *))balance_push,
4007 };
4008 
4009 static inline struct callback_head *splice_balance_callbacks(struct rq *rq)
4010 {
4011 	struct callback_head *head = rq->balance_callback;
4012 
4013 	lockdep_assert_held(&rq->lock);
4014 	if (head)
4015 		rq->balance_callback = NULL;
4016 
4017 	return head;
4018 }
4019 
4020 static void __balance_callbacks(struct rq *rq)
4021 {
4022 	do_balance_callbacks(rq, splice_balance_callbacks(rq));
4023 }
4024 
4025 static inline void balance_callbacks(struct rq *rq, struct callback_head *head)
4026 {
4027 	unsigned long flags;
4028 
4029 	if (unlikely(head)) {
4030 		raw_spin_lock_irqsave(&rq->lock, flags);
4031 		do_balance_callbacks(rq, head);
4032 		raw_spin_unlock_irqrestore(&rq->lock, flags);
4033 	}
4034 }
4035 
4036 #else
4037 
4038 static inline void __balance_callbacks(struct rq *rq)
4039 {
4040 }
4041 
4042 static inline struct callback_head *splice_balance_callbacks(struct rq *rq)
4043 {
4044 	return NULL;
4045 }
4046 
4047 static inline void balance_callbacks(struct rq *rq, struct callback_head *head)
4048 {
4049 }
4050 
4051 #endif
4052 
4053 static inline void
4054 prepare_lock_switch(struct rq *rq, struct task_struct *next, struct rq_flags *rf)
4055 {
4056 	/*
4057 	 * Since the runqueue lock will be released by the next
4058 	 * task (which is an invalid locking op but in the case
4059 	 * of the scheduler it's an obvious special-case), so we
4060 	 * do an early lockdep release here:
4061 	 */
4062 	rq_unpin_lock(rq, rf);
4063 	spin_release(&rq->lock.dep_map, _THIS_IP_);
4064 #ifdef CONFIG_DEBUG_SPINLOCK
4065 	/* this is a valid case when another task releases the spinlock */
4066 	rq->lock.owner = next;
4067 #endif
4068 }
4069 
4070 static inline void finish_lock_switch(struct rq *rq)
4071 {
4072 	/*
4073 	 * If we are tracking spinlock dependencies then we have to
4074 	 * fix up the runqueue lock - which gets 'carried over' from
4075 	 * prev into current:
4076 	 */
4077 	spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_);
4078 	__balance_callbacks(rq);
4079 	raw_spin_unlock_irq(&rq->lock);
4080 }
4081 
4082 /*
4083  * NOP if the arch has not defined these:
4084  */
4085 
4086 #ifndef prepare_arch_switch
4087 # define prepare_arch_switch(next)	do { } while (0)
4088 #endif
4089 
4090 #ifndef finish_arch_post_lock_switch
4091 # define finish_arch_post_lock_switch()	do { } while (0)
4092 #endif
4093 
4094 static inline void kmap_local_sched_out(void)
4095 {
4096 #ifdef CONFIG_KMAP_LOCAL
4097 	if (unlikely(current->kmap_ctrl.idx))
4098 		__kmap_local_sched_out();
4099 #endif
4100 }
4101 
4102 static inline void kmap_local_sched_in(void)
4103 {
4104 #ifdef CONFIG_KMAP_LOCAL
4105 	if (unlikely(current->kmap_ctrl.idx))
4106 		__kmap_local_sched_in();
4107 #endif
4108 }
4109 
4110 /**
4111  * prepare_task_switch - prepare to switch tasks
4112  * @rq: the runqueue preparing to switch
4113  * @prev: the current task that is being switched out
4114  * @next: the task we are going to switch to.
4115  *
4116  * This is called with the rq lock held and interrupts off. It must
4117  * be paired with a subsequent finish_task_switch after the context
4118  * switch.
4119  *
4120  * prepare_task_switch sets up locking and calls architecture specific
4121  * hooks.
4122  */
4123 static inline void
4124 prepare_task_switch(struct rq *rq, struct task_struct *prev,
4125 		    struct task_struct *next)
4126 {
4127 	kcov_prepare_switch(prev);
4128 	sched_info_switch(rq, prev, next);
4129 	perf_event_task_sched_out(prev, next);
4130 	rseq_preempt(prev);
4131 	fire_sched_out_preempt_notifiers(prev, next);
4132 	kmap_local_sched_out();
4133 	prepare_task(next);
4134 	prepare_arch_switch(next);
4135 }
4136 
4137 /**
4138  * finish_task_switch - clean up after a task-switch
4139  * @prev: the thread we just switched away from.
4140  *
4141  * finish_task_switch must be called after the context switch, paired
4142  * with a prepare_task_switch call before the context switch.
4143  * finish_task_switch will reconcile locking set up by prepare_task_switch,
4144  * and do any other architecture-specific cleanup actions.
4145  *
4146  * Note that we may have delayed dropping an mm in context_switch(). If
4147  * so, we finish that here outside of the runqueue lock. (Doing it
4148  * with the lock held can cause deadlocks; see schedule() for
4149  * details.)
4150  *
4151  * The context switch have flipped the stack from under us and restored the
4152  * local variables which were saved when this task called schedule() in the
4153  * past. prev == current is still correct but we need to recalculate this_rq
4154  * because prev may have moved to another CPU.
4155  */
4156 static struct rq *finish_task_switch(struct task_struct *prev)
4157 	__releases(rq->lock)
4158 {
4159 	struct rq *rq = this_rq();
4160 	struct mm_struct *mm = rq->prev_mm;
4161 	long prev_state;
4162 
4163 	/*
4164 	 * The previous task will have left us with a preempt_count of 2
4165 	 * because it left us after:
4166 	 *
4167 	 *	schedule()
4168 	 *	  preempt_disable();			// 1
4169 	 *	  __schedule()
4170 	 *	    raw_spin_lock_irq(&rq->lock)	// 2
4171 	 *
4172 	 * Also, see FORK_PREEMPT_COUNT.
4173 	 */
4174 	if (WARN_ONCE(preempt_count() != 2*PREEMPT_DISABLE_OFFSET,
4175 		      "corrupted preempt_count: %s/%d/0x%x\n",
4176 		      current->comm, current->pid, preempt_count()))
4177 		preempt_count_set(FORK_PREEMPT_COUNT);
4178 
4179 	rq->prev_mm = NULL;
4180 
4181 	/*
4182 	 * A task struct has one reference for the use as "current".
4183 	 * If a task dies, then it sets TASK_DEAD in tsk->state and calls
4184 	 * schedule one last time. The schedule call will never return, and
4185 	 * the scheduled task must drop that reference.
4186 	 *
4187 	 * We must observe prev->state before clearing prev->on_cpu (in
4188 	 * finish_task), otherwise a concurrent wakeup can get prev
4189 	 * running on another CPU and we could rave with its RUNNING -> DEAD
4190 	 * transition, resulting in a double drop.
4191 	 */
4192 	prev_state = prev->state;
4193 	vtime_task_switch(prev);
4194 	perf_event_task_sched_in(prev, current);
4195 	finish_task(prev);
4196 	finish_lock_switch(rq);
4197 	finish_arch_post_lock_switch();
4198 	kcov_finish_switch(current);
4199 	/*
4200 	 * kmap_local_sched_out() is invoked with rq::lock held and
4201 	 * interrupts disabled. There is no requirement for that, but the
4202 	 * sched out code does not have an interrupt enabled section.
4203 	 * Restoring the maps on sched in does not require interrupts being
4204 	 * disabled either.
4205 	 */
4206 	kmap_local_sched_in();
4207 
4208 	fire_sched_in_preempt_notifiers(current);
4209 	/*
4210 	 * When switching through a kernel thread, the loop in
4211 	 * membarrier_{private,global}_expedited() may have observed that
4212 	 * kernel thread and not issued an IPI. It is therefore possible to
4213 	 * schedule between user->kernel->user threads without passing though
4214 	 * switch_mm(). Membarrier requires a barrier after storing to
4215 	 * rq->curr, before returning to userspace, so provide them here:
4216 	 *
4217 	 * - a full memory barrier for {PRIVATE,GLOBAL}_EXPEDITED, implicitly
4218 	 *   provided by mmdrop(),
4219 	 * - a sync_core for SYNC_CORE.
4220 	 */
4221 	if (mm) {
4222 		membarrier_mm_sync_core_before_usermode(mm);
4223 		mmdrop(mm);
4224 	}
4225 	if (unlikely(prev_state == TASK_DEAD)) {
4226 		if (prev->sched_class->task_dead)
4227 			prev->sched_class->task_dead(prev);
4228 
4229 		/*
4230 		 * Remove function-return probe instances associated with this
4231 		 * task and put them back on the free list.
4232 		 */
4233 		kprobe_flush_task(prev);
4234 
4235 		/* Task is done with its stack. */
4236 		put_task_stack(prev);
4237 
4238 		put_task_struct_rcu_user(prev);
4239 	}
4240 
4241 	tick_nohz_task_switch();
4242 	return rq;
4243 }
4244 
4245 /**
4246  * schedule_tail - first thing a freshly forked thread must call.
4247  * @prev: the thread we just switched away from.
4248  */
4249 asmlinkage __visible void schedule_tail(struct task_struct *prev)
4250 	__releases(rq->lock)
4251 {
4252 	struct rq *rq;
4253 
4254 	/*
4255 	 * New tasks start with FORK_PREEMPT_COUNT, see there and
4256 	 * finish_task_switch() for details.
4257 	 *
4258 	 * finish_task_switch() will drop rq->lock() and lower preempt_count
4259 	 * and the preempt_enable() will end up enabling preemption (on
4260 	 * PREEMPT_COUNT kernels).
4261 	 */
4262 
4263 	rq = finish_task_switch(prev);
4264 	preempt_enable();
4265 
4266 	if (current->set_child_tid)
4267 		put_user(task_pid_vnr(current), current->set_child_tid);
4268 
4269 	calculate_sigpending();
4270 }
4271 
4272 /*
4273  * context_switch - switch to the new MM and the new thread's register state.
4274  */
4275 static __always_inline struct rq *
4276 context_switch(struct rq *rq, struct task_struct *prev,
4277 	       struct task_struct *next, struct rq_flags *rf)
4278 {
4279 	prepare_task_switch(rq, prev, next);
4280 
4281 	/*
4282 	 * For paravirt, this is coupled with an exit in switch_to to
4283 	 * combine the page table reload and the switch backend into
4284 	 * one hypercall.
4285 	 */
4286 	arch_start_context_switch(prev);
4287 
4288 	/*
4289 	 * kernel -> kernel   lazy + transfer active
4290 	 *   user -> kernel   lazy + mmgrab() active
4291 	 *
4292 	 * kernel ->   user   switch + mmdrop() active
4293 	 *   user ->   user   switch
4294 	 */
4295 	if (!next->mm) {                                // to kernel
4296 		enter_lazy_tlb(prev->active_mm, next);
4297 
4298 		next->active_mm = prev->active_mm;
4299 		if (prev->mm)                           // from user
4300 			mmgrab(prev->active_mm);
4301 		else
4302 			prev->active_mm = NULL;
4303 	} else {                                        // to user
4304 		membarrier_switch_mm(rq, prev->active_mm, next->mm);
4305 		/*
4306 		 * sys_membarrier() requires an smp_mb() between setting
4307 		 * rq->curr / membarrier_switch_mm() and returning to userspace.
4308 		 *
4309 		 * The below provides this either through switch_mm(), or in
4310 		 * case 'prev->active_mm == next->mm' through
4311 		 * finish_task_switch()'s mmdrop().
4312 		 */
4313 		switch_mm_irqs_off(prev->active_mm, next->mm, next);
4314 
4315 		if (!prev->mm) {                        // from kernel
4316 			/* will mmdrop() in finish_task_switch(). */
4317 			rq->prev_mm = prev->active_mm;
4318 			prev->active_mm = NULL;
4319 		}
4320 	}
4321 
4322 	rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP);
4323 
4324 	prepare_lock_switch(rq, next, rf);
4325 
4326 	/* Here we just switch the register state and the stack. */
4327 	switch_to(prev, next, prev);
4328 	barrier();
4329 
4330 	return finish_task_switch(prev);
4331 }
4332 
4333 /*
4334  * nr_running and nr_context_switches:
4335  *
4336  * externally visible scheduler statistics: current number of runnable
4337  * threads, total number of context switches performed since bootup.
4338  */
4339 unsigned long nr_running(void)
4340 {
4341 	unsigned long i, sum = 0;
4342 
4343 	for_each_online_cpu(i)
4344 		sum += cpu_rq(i)->nr_running;
4345 
4346 	return sum;
4347 }
4348 
4349 /*
4350  * Check if only the current task is running on the CPU.
4351  *
4352  * Caution: this function does not check that the caller has disabled
4353  * preemption, thus the result might have a time-of-check-to-time-of-use
4354  * race.  The caller is responsible to use it correctly, for example:
4355  *
4356  * - from a non-preemptible section (of course)
4357  *
4358  * - from a thread that is bound to a single CPU
4359  *
4360  * - in a loop with very short iterations (e.g. a polling loop)
4361  */
4362 bool single_task_running(void)
4363 {
4364 	return raw_rq()->nr_running == 1;
4365 }
4366 EXPORT_SYMBOL(single_task_running);
4367 
4368 unsigned long long nr_context_switches(void)
4369 {
4370 	int i;
4371 	unsigned long long sum = 0;
4372 
4373 	for_each_possible_cpu(i)
4374 		sum += cpu_rq(i)->nr_switches;
4375 
4376 	return sum;
4377 }
4378 
4379 /*
4380  * Consumers of these two interfaces, like for example the cpuidle menu
4381  * governor, are using nonsensical data. Preferring shallow idle state selection
4382  * for a CPU that has IO-wait which might not even end up running the task when
4383  * it does become runnable.
4384  */
4385 
4386 unsigned long nr_iowait_cpu(int cpu)
4387 {
4388 	return atomic_read(&cpu_rq(cpu)->nr_iowait);
4389 }
4390 
4391 /*
4392  * IO-wait accounting, and how it's mostly bollocks (on SMP).
4393  *
4394  * The idea behind IO-wait account is to account the idle time that we could
4395  * have spend running if it were not for IO. That is, if we were to improve the
4396  * storage performance, we'd have a proportional reduction in IO-wait time.
4397  *
4398  * This all works nicely on UP, where, when a task blocks on IO, we account
4399  * idle time as IO-wait, because if the storage were faster, it could've been
4400  * running and we'd not be idle.
4401  *
4402  * This has been extended to SMP, by doing the same for each CPU. This however
4403  * is broken.
4404  *
4405  * Imagine for instance the case where two tasks block on one CPU, only the one
4406  * CPU will have IO-wait accounted, while the other has regular idle. Even
4407  * though, if the storage were faster, both could've ran at the same time,
4408  * utilising both CPUs.
4409  *
4410  * This means, that when looking globally, the current IO-wait accounting on
4411  * SMP is a lower bound, by reason of under accounting.
4412  *
4413  * Worse, since the numbers are provided per CPU, they are sometimes
4414  * interpreted per CPU, and that is nonsensical. A blocked task isn't strictly
4415  * associated with any one particular CPU, it can wake to another CPU than it
4416  * blocked on. This means the per CPU IO-wait number is meaningless.
4417  *
4418  * Task CPU affinities can make all that even more 'interesting'.
4419  */
4420 
4421 unsigned long nr_iowait(void)
4422 {
4423 	unsigned long i, sum = 0;
4424 
4425 	for_each_possible_cpu(i)
4426 		sum += nr_iowait_cpu(i);
4427 
4428 	return sum;
4429 }
4430 
4431 #ifdef CONFIG_SMP
4432 
4433 /*
4434  * sched_exec - execve() is a valuable balancing opportunity, because at
4435  * this point the task has the smallest effective memory and cache footprint.
4436  */
4437 void sched_exec(void)
4438 {
4439 	struct task_struct *p = current;
4440 	unsigned long flags;
4441 	int dest_cpu;
4442 
4443 	raw_spin_lock_irqsave(&p->pi_lock, flags);
4444 	dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), WF_EXEC);
4445 	if (dest_cpu == smp_processor_id())
4446 		goto unlock;
4447 
4448 	if (likely(cpu_active(dest_cpu))) {
4449 		struct migration_arg arg = { p, dest_cpu };
4450 
4451 		raw_spin_unlock_irqrestore(&p->pi_lock, flags);
4452 		stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg);
4453 		return;
4454 	}
4455 unlock:
4456 	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
4457 }
4458 
4459 #endif
4460 
4461 DEFINE_PER_CPU(struct kernel_stat, kstat);
4462 DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat);
4463 
4464 EXPORT_PER_CPU_SYMBOL(kstat);
4465 EXPORT_PER_CPU_SYMBOL(kernel_cpustat);
4466 
4467 /*
4468  * The function fair_sched_class.update_curr accesses the struct curr
4469  * and its field curr->exec_start; when called from task_sched_runtime(),
4470  * we observe a high rate of cache misses in practice.
4471  * Prefetching this data results in improved performance.
4472  */
4473 static inline void prefetch_curr_exec_start(struct task_struct *p)
4474 {
4475 #ifdef CONFIG_FAIR_GROUP_SCHED
4476 	struct sched_entity *curr = (&p->se)->cfs_rq->curr;
4477 #else
4478 	struct sched_entity *curr = (&task_rq(p)->cfs)->curr;
4479 #endif
4480 	prefetch(curr);
4481 	prefetch(&curr->exec_start);
4482 }
4483 
4484 /*
4485  * Return accounted runtime for the task.
4486  * In case the task is currently running, return the runtime plus current's
4487  * pending runtime that have not been accounted yet.
4488  */
4489 unsigned long long task_sched_runtime(struct task_struct *p)
4490 {
4491 	struct rq_flags rf;
4492 	struct rq *rq;
4493 	u64 ns;
4494 
4495 #if defined(CONFIG_64BIT) && defined(CONFIG_SMP)
4496 	/*
4497 	 * 64-bit doesn't need locks to atomically read a 64-bit value.
4498 	 * So we have a optimization chance when the task's delta_exec is 0.
4499 	 * Reading ->on_cpu is racy, but this is ok.
4500 	 *
4501 	 * If we race with it leaving CPU, we'll take a lock. So we're correct.
4502 	 * If we race with it entering CPU, unaccounted time is 0. This is
4503 	 * indistinguishable from the read occurring a few cycles earlier.
4504 	 * If we see ->on_cpu without ->on_rq, the task is leaving, and has
4505 	 * been accounted, so we're correct here as well.
4506 	 */
4507 	if (!p->on_cpu || !task_on_rq_queued(p))
4508 		return p->se.sum_exec_runtime;
4509 #endif
4510 
4511 	rq = task_rq_lock(p, &rf);
4512 	/*
4513 	 * Must be ->curr _and_ ->on_rq.  If dequeued, we would
4514 	 * project cycles that may never be accounted to this
4515 	 * thread, breaking clock_gettime().
4516 	 */
4517 	if (task_current(rq, p) && task_on_rq_queued(p)) {
4518 		prefetch_curr_exec_start(p);
4519 		update_rq_clock(rq);
4520 		p->sched_class->update_curr(rq);
4521 	}
4522 	ns = p->se.sum_exec_runtime;
4523 	task_rq_unlock(rq, p, &rf);
4524 
4525 	return ns;
4526 }
4527 
4528 /*
4529  * This function gets called by the timer code, with HZ frequency.
4530  * We call it with interrupts disabled.
4531  */
4532 void scheduler_tick(void)
4533 {
4534 	int cpu = smp_processor_id();
4535 	struct rq *rq = cpu_rq(cpu);
4536 	struct task_struct *curr = rq->curr;
4537 	struct rq_flags rf;
4538 	unsigned long thermal_pressure;
4539 
4540 	arch_scale_freq_tick();
4541 	sched_clock_tick();
4542 
4543 	rq_lock(rq, &rf);
4544 
4545 	update_rq_clock(rq);
4546 	thermal_pressure = arch_scale_thermal_pressure(cpu_of(rq));
4547 	update_thermal_load_avg(rq_clock_thermal(rq), rq, thermal_pressure);
4548 	curr->sched_class->task_tick(rq, curr, 0);
4549 	calc_global_load_tick(rq);
4550 	psi_task_tick(rq);
4551 
4552 	rq_unlock(rq, &rf);
4553 
4554 	perf_event_task_tick();
4555 
4556 #ifdef CONFIG_SMP
4557 	rq->idle_balance = idle_cpu(cpu);
4558 	trigger_load_balance(rq);
4559 #endif
4560 }
4561 
4562 #ifdef CONFIG_NO_HZ_FULL
4563 
4564 struct tick_work {
4565 	int			cpu;
4566 	atomic_t		state;
4567 	struct delayed_work	work;
4568 };
4569 /* Values for ->state, see diagram below. */
4570 #define TICK_SCHED_REMOTE_OFFLINE	0
4571 #define TICK_SCHED_REMOTE_OFFLINING	1
4572 #define TICK_SCHED_REMOTE_RUNNING	2
4573 
4574 /*
4575  * State diagram for ->state:
4576  *
4577  *
4578  *          TICK_SCHED_REMOTE_OFFLINE
4579  *                    |   ^
4580  *                    |   |
4581  *                    |   | sched_tick_remote()
4582  *                    |   |
4583  *                    |   |
4584  *                    +--TICK_SCHED_REMOTE_OFFLINING
4585  *                    |   ^
4586  *                    |   |
4587  * sched_tick_start() |   | sched_tick_stop()
4588  *                    |   |
4589  *                    V   |
4590  *          TICK_SCHED_REMOTE_RUNNING
4591  *
4592  *
4593  * Other transitions get WARN_ON_ONCE(), except that sched_tick_remote()
4594  * and sched_tick_start() are happy to leave the state in RUNNING.
4595  */
4596 
4597 static struct tick_work __percpu *tick_work_cpu;
4598 
4599 static void sched_tick_remote(struct work_struct *work)
4600 {
4601 	struct delayed_work *dwork = to_delayed_work(work);
4602 	struct tick_work *twork = container_of(dwork, struct tick_work, work);
4603 	int cpu = twork->cpu;
4604 	struct rq *rq = cpu_rq(cpu);
4605 	struct task_struct *curr;
4606 	struct rq_flags rf;
4607 	u64 delta;
4608 	int os;
4609 
4610 	/*
4611 	 * Handle the tick only if it appears the remote CPU is running in full
4612 	 * dynticks mode. The check is racy by nature, but missing a tick or
4613 	 * having one too much is no big deal because the scheduler tick updates
4614 	 * statistics and checks timeslices in a time-independent way, regardless
4615 	 * of when exactly it is running.
4616 	 */
4617 	if (!tick_nohz_tick_stopped_cpu(cpu))
4618 		goto out_requeue;
4619 
4620 	rq_lock_irq(rq, &rf);
4621 	curr = rq->curr;
4622 	if (cpu_is_offline(cpu))
4623 		goto out_unlock;
4624 
4625 	update_rq_clock(rq);
4626 
4627 	if (!is_idle_task(curr)) {
4628 		/*
4629 		 * Make sure the next tick runs within a reasonable
4630 		 * amount of time.
4631 		 */
4632 		delta = rq_clock_task(rq) - curr->se.exec_start;
4633 		WARN_ON_ONCE(delta > (u64)NSEC_PER_SEC * 3);
4634 	}
4635 	curr->sched_class->task_tick(rq, curr, 0);
4636 
4637 	calc_load_nohz_remote(rq);
4638 out_unlock:
4639 	rq_unlock_irq(rq, &rf);
4640 out_requeue:
4641 
4642 	/*
4643 	 * Run the remote tick once per second (1Hz). This arbitrary
4644 	 * frequency is large enough to avoid overload but short enough
4645 	 * to keep scheduler internal stats reasonably up to date.  But
4646 	 * first update state to reflect hotplug activity if required.
4647 	 */
4648 	os = atomic_fetch_add_unless(&twork->state, -1, TICK_SCHED_REMOTE_RUNNING);
4649 	WARN_ON_ONCE(os == TICK_SCHED_REMOTE_OFFLINE);
4650 	if (os == TICK_SCHED_REMOTE_RUNNING)
4651 		queue_delayed_work(system_unbound_wq, dwork, HZ);
4652 }
4653 
4654 static void sched_tick_start(int cpu)
4655 {
4656 	int os;
4657 	struct tick_work *twork;
4658 
4659 	if (housekeeping_cpu(cpu, HK_FLAG_TICK))
4660 		return;
4661 
4662 	WARN_ON_ONCE(!tick_work_cpu);
4663 
4664 	twork = per_cpu_ptr(tick_work_cpu, cpu);
4665 	os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_RUNNING);
4666 	WARN_ON_ONCE(os == TICK_SCHED_REMOTE_RUNNING);
4667 	if (os == TICK_SCHED_REMOTE_OFFLINE) {
4668 		twork->cpu = cpu;
4669 		INIT_DELAYED_WORK(&twork->work, sched_tick_remote);
4670 		queue_delayed_work(system_unbound_wq, &twork->work, HZ);
4671 	}
4672 }
4673 
4674 #ifdef CONFIG_HOTPLUG_CPU
4675 static void sched_tick_stop(int cpu)
4676 {
4677 	struct tick_work *twork;
4678 	int os;
4679 
4680 	if (housekeeping_cpu(cpu, HK_FLAG_TICK))
4681 		return;
4682 
4683 	WARN_ON_ONCE(!tick_work_cpu);
4684 
4685 	twork = per_cpu_ptr(tick_work_cpu, cpu);
4686 	/* There cannot be competing actions, but don't rely on stop-machine. */
4687 	os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_OFFLINING);
4688 	WARN_ON_ONCE(os != TICK_SCHED_REMOTE_RUNNING);
4689 	/* Don't cancel, as this would mess up the state machine. */
4690 }
4691 #endif /* CONFIG_HOTPLUG_CPU */
4692 
4693 int __init sched_tick_offload_init(void)
4694 {
4695 	tick_work_cpu = alloc_percpu(struct tick_work);
4696 	BUG_ON(!tick_work_cpu);
4697 	return 0;
4698 }
4699 
4700 #else /* !CONFIG_NO_HZ_FULL */
4701 static inline void sched_tick_start(int cpu) { }
4702 static inline void sched_tick_stop(int cpu) { }
4703 #endif
4704 
4705 #if defined(CONFIG_PREEMPTION) && (defined(CONFIG_DEBUG_PREEMPT) || \
4706 				defined(CONFIG_TRACE_PREEMPT_TOGGLE))
4707 /*
4708  * If the value passed in is equal to the current preempt count
4709  * then we just disabled preemption. Start timing the latency.
4710  */
4711 static inline void preempt_latency_start(int val)
4712 {
4713 	if (preempt_count() == val) {
4714 		unsigned long ip = get_lock_parent_ip();
4715 #ifdef CONFIG_DEBUG_PREEMPT
4716 		current->preempt_disable_ip = ip;
4717 #endif
4718 		trace_preempt_off(CALLER_ADDR0, ip);
4719 	}
4720 }
4721 
4722 void preempt_count_add(int val)
4723 {
4724 #ifdef CONFIG_DEBUG_PREEMPT
4725 	/*
4726 	 * Underflow?
4727 	 */
4728 	if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
4729 		return;
4730 #endif
4731 	__preempt_count_add(val);
4732 #ifdef CONFIG_DEBUG_PREEMPT
4733 	/*
4734 	 * Spinlock count overflowing soon?
4735 	 */
4736 	DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
4737 				PREEMPT_MASK - 10);
4738 #endif
4739 	preempt_latency_start(val);
4740 }
4741 EXPORT_SYMBOL(preempt_count_add);
4742 NOKPROBE_SYMBOL(preempt_count_add);
4743 
4744 /*
4745  * If the value passed in equals to the current preempt count
4746  * then we just enabled preemption. Stop timing the latency.
4747  */
4748 static inline void preempt_latency_stop(int val)
4749 {
4750 	if (preempt_count() == val)
4751 		trace_preempt_on(CALLER_ADDR0, get_lock_parent_ip());
4752 }
4753 
4754 void preempt_count_sub(int val)
4755 {
4756 #ifdef CONFIG_DEBUG_PREEMPT
4757 	/*
4758 	 * Underflow?
4759 	 */
4760 	if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
4761 		return;
4762 	/*
4763 	 * Is the spinlock portion underflowing?
4764 	 */
4765 	if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
4766 			!(preempt_count() & PREEMPT_MASK)))
4767 		return;
4768 #endif
4769 
4770 	preempt_latency_stop(val);
4771 	__preempt_count_sub(val);
4772 }
4773 EXPORT_SYMBOL(preempt_count_sub);
4774 NOKPROBE_SYMBOL(preempt_count_sub);
4775 
4776 #else
4777 static inline void preempt_latency_start(int val) { }
4778 static inline void preempt_latency_stop(int val) { }
4779 #endif
4780 
4781 static inline unsigned long get_preempt_disable_ip(struct task_struct *p)
4782 {
4783 #ifdef CONFIG_DEBUG_PREEMPT
4784 	return p->preempt_disable_ip;
4785 #else
4786 	return 0;
4787 #endif
4788 }
4789 
4790 /*
4791  * Print scheduling while atomic bug:
4792  */
4793 static noinline void __schedule_bug(struct task_struct *prev)
4794 {
4795 	/* Save this before calling printk(), since that will clobber it */
4796 	unsigned long preempt_disable_ip = get_preempt_disable_ip(current);
4797 
4798 	if (oops_in_progress)
4799 		return;
4800 
4801 	printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n",
4802 		prev->comm, prev->pid, preempt_count());
4803 
4804 	debug_show_held_locks(prev);
4805 	print_modules();
4806 	if (irqs_disabled())
4807 		print_irqtrace_events(prev);
4808 	if (IS_ENABLED(CONFIG_DEBUG_PREEMPT)
4809 	    && in_atomic_preempt_off()) {
4810 		pr_err("Preemption disabled at:");
4811 		print_ip_sym(KERN_ERR, preempt_disable_ip);
4812 	}
4813 	if (panic_on_warn)
4814 		panic("scheduling while atomic\n");
4815 
4816 	dump_stack();
4817 	add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
4818 }
4819 
4820 /*
4821  * Various schedule()-time debugging checks and statistics:
4822  */
4823 static inline void schedule_debug(struct task_struct *prev, bool preempt)
4824 {
4825 #ifdef CONFIG_SCHED_STACK_END_CHECK
4826 	if (task_stack_end_corrupted(prev))
4827 		panic("corrupted stack end detected inside scheduler\n");
4828 
4829 	if (task_scs_end_corrupted(prev))
4830 		panic("corrupted shadow stack detected inside scheduler\n");
4831 #endif
4832 
4833 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
4834 	if (!preempt && prev->state && prev->non_block_count) {
4835 		printk(KERN_ERR "BUG: scheduling in a non-blocking section: %s/%d/%i\n",
4836 			prev->comm, prev->pid, prev->non_block_count);
4837 		dump_stack();
4838 		add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
4839 	}
4840 #endif
4841 
4842 	if (unlikely(in_atomic_preempt_off())) {
4843 		__schedule_bug(prev);
4844 		preempt_count_set(PREEMPT_DISABLED);
4845 	}
4846 	rcu_sleep_check();
4847 	SCHED_WARN_ON(ct_state() == CONTEXT_USER);
4848 
4849 	profile_hit(SCHED_PROFILING, __builtin_return_address(0));
4850 
4851 	schedstat_inc(this_rq()->sched_count);
4852 }
4853 
4854 static void put_prev_task_balance(struct rq *rq, struct task_struct *prev,
4855 				  struct rq_flags *rf)
4856 {
4857 #ifdef CONFIG_SMP
4858 	const struct sched_class *class;
4859 	/*
4860 	 * We must do the balancing pass before put_prev_task(), such
4861 	 * that when we release the rq->lock the task is in the same
4862 	 * state as before we took rq->lock.
4863 	 *
4864 	 * We can terminate the balance pass as soon as we know there is
4865 	 * a runnable task of @class priority or higher.
4866 	 */
4867 	for_class_range(class, prev->sched_class, &idle_sched_class) {
4868 		if (class->balance(rq, prev, rf))
4869 			break;
4870 	}
4871 #endif
4872 
4873 	put_prev_task(rq, prev);
4874 }
4875 
4876 /*
4877  * Pick up the highest-prio task:
4878  */
4879 static inline struct task_struct *
4880 pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
4881 {
4882 	const struct sched_class *class;
4883 	struct task_struct *p;
4884 
4885 	/*
4886 	 * Optimization: we know that if all tasks are in the fair class we can
4887 	 * call that function directly, but only if the @prev task wasn't of a
4888 	 * higher scheduling class, because otherwise those lose the
4889 	 * opportunity to pull in more work from other CPUs.
4890 	 */
4891 	if (likely(prev->sched_class <= &fair_sched_class &&
4892 		   rq->nr_running == rq->cfs.h_nr_running)) {
4893 
4894 		p = pick_next_task_fair(rq, prev, rf);
4895 		if (unlikely(p == RETRY_TASK))
4896 			goto restart;
4897 
4898 		/* Assumes fair_sched_class->next == idle_sched_class */
4899 		if (!p) {
4900 			put_prev_task(rq, prev);
4901 			p = pick_next_task_idle(rq);
4902 		}
4903 
4904 		return p;
4905 	}
4906 
4907 restart:
4908 	put_prev_task_balance(rq, prev, rf);
4909 
4910 	for_each_class(class) {
4911 		p = class->pick_next_task(rq);
4912 		if (p)
4913 			return p;
4914 	}
4915 
4916 	/* The idle class should always have a runnable task: */
4917 	BUG();
4918 }
4919 
4920 /*
4921  * __schedule() is the main scheduler function.
4922  *
4923  * The main means of driving the scheduler and thus entering this function are:
4924  *
4925  *   1. Explicit blocking: mutex, semaphore, waitqueue, etc.
4926  *
4927  *   2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return
4928  *      paths. For example, see arch/x86/entry_64.S.
4929  *
4930  *      To drive preemption between tasks, the scheduler sets the flag in timer
4931  *      interrupt handler scheduler_tick().
4932  *
4933  *   3. Wakeups don't really cause entry into schedule(). They add a
4934  *      task to the run-queue and that's it.
4935  *
4936  *      Now, if the new task added to the run-queue preempts the current
4937  *      task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets
4938  *      called on the nearest possible occasion:
4939  *
4940  *       - If the kernel is preemptible (CONFIG_PREEMPTION=y):
4941  *
4942  *         - in syscall or exception context, at the next outmost
4943  *           preempt_enable(). (this might be as soon as the wake_up()'s
4944  *           spin_unlock()!)
4945  *
4946  *         - in IRQ context, return from interrupt-handler to
4947  *           preemptible context
4948  *
4949  *       - If the kernel is not preemptible (CONFIG_PREEMPTION is not set)
4950  *         then at the next:
4951  *
4952  *          - cond_resched() call
4953  *          - explicit schedule() call
4954  *          - return from syscall or exception to user-space
4955  *          - return from interrupt-handler to user-space
4956  *
4957  * WARNING: must be called with preemption disabled!
4958  */
4959 static void __sched notrace __schedule(bool preempt)
4960 {
4961 	struct task_struct *prev, *next;
4962 	unsigned long *switch_count;
4963 	unsigned long prev_state;
4964 	struct rq_flags rf;
4965 	struct rq *rq;
4966 	int cpu;
4967 
4968 	cpu = smp_processor_id();
4969 	rq = cpu_rq(cpu);
4970 	prev = rq->curr;
4971 
4972 	schedule_debug(prev, preempt);
4973 
4974 	if (sched_feat(HRTICK))
4975 		hrtick_clear(rq);
4976 
4977 	local_irq_disable();
4978 	rcu_note_context_switch(preempt);
4979 
4980 	/*
4981 	 * Make sure that signal_pending_state()->signal_pending() below
4982 	 * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE)
4983 	 * done by the caller to avoid the race with signal_wake_up():
4984 	 *
4985 	 * __set_current_state(@state)		signal_wake_up()
4986 	 * schedule()				  set_tsk_thread_flag(p, TIF_SIGPENDING)
4987 	 *					  wake_up_state(p, state)
4988 	 *   LOCK rq->lock			    LOCK p->pi_state
4989 	 *   smp_mb__after_spinlock()		    smp_mb__after_spinlock()
4990 	 *     if (signal_pending_state())	    if (p->state & @state)
4991 	 *
4992 	 * Also, the membarrier system call requires a full memory barrier
4993 	 * after coming from user-space, before storing to rq->curr.
4994 	 */
4995 	rq_lock(rq, &rf);
4996 	smp_mb__after_spinlock();
4997 
4998 	/* Promote REQ to ACT */
4999 	rq->clock_update_flags <<= 1;
5000 	update_rq_clock(rq);
5001 
5002 	switch_count = &prev->nivcsw;
5003 
5004 	/*
5005 	 * We must load prev->state once (task_struct::state is volatile), such
5006 	 * that:
5007 	 *
5008 	 *  - we form a control dependency vs deactivate_task() below.
5009 	 *  - ptrace_{,un}freeze_traced() can change ->state underneath us.
5010 	 */
5011 	prev_state = prev->state;
5012 	if (!preempt && prev_state) {
5013 		if (signal_pending_state(prev_state, prev)) {
5014 			prev->state = TASK_RUNNING;
5015 		} else {
5016 			prev->sched_contributes_to_load =
5017 				(prev_state & TASK_UNINTERRUPTIBLE) &&
5018 				!(prev_state & TASK_NOLOAD) &&
5019 				!(prev->flags & PF_FROZEN);
5020 
5021 			if (prev->sched_contributes_to_load)
5022 				rq->nr_uninterruptible++;
5023 
5024 			/*
5025 			 * __schedule()			ttwu()
5026 			 *   prev_state = prev->state;    if (p->on_rq && ...)
5027 			 *   if (prev_state)		    goto out;
5028 			 *     p->on_rq = 0;		  smp_acquire__after_ctrl_dep();
5029 			 *				  p->state = TASK_WAKING
5030 			 *
5031 			 * Where __schedule() and ttwu() have matching control dependencies.
5032 			 *
5033 			 * After this, schedule() must not care about p->state any more.
5034 			 */
5035 			deactivate_task(rq, prev, DEQUEUE_SLEEP | DEQUEUE_NOCLOCK);
5036 
5037 			if (prev->in_iowait) {
5038 				atomic_inc(&rq->nr_iowait);
5039 				delayacct_blkio_start();
5040 			}
5041 		}
5042 		switch_count = &prev->nvcsw;
5043 	}
5044 
5045 	next = pick_next_task(rq, prev, &rf);
5046 	clear_tsk_need_resched(prev);
5047 	clear_preempt_need_resched();
5048 
5049 	if (likely(prev != next)) {
5050 		rq->nr_switches++;
5051 		/*
5052 		 * RCU users of rcu_dereference(rq->curr) may not see
5053 		 * changes to task_struct made by pick_next_task().
5054 		 */
5055 		RCU_INIT_POINTER(rq->curr, next);
5056 		/*
5057 		 * The membarrier system call requires each architecture
5058 		 * to have a full memory barrier after updating
5059 		 * rq->curr, before returning to user-space.
5060 		 *
5061 		 * Here are the schemes providing that barrier on the
5062 		 * various architectures:
5063 		 * - mm ? switch_mm() : mmdrop() for x86, s390, sparc, PowerPC.
5064 		 *   switch_mm() rely on membarrier_arch_switch_mm() on PowerPC.
5065 		 * - finish_lock_switch() for weakly-ordered
5066 		 *   architectures where spin_unlock is a full barrier,
5067 		 * - switch_to() for arm64 (weakly-ordered, spin_unlock
5068 		 *   is a RELEASE barrier),
5069 		 */
5070 		++*switch_count;
5071 
5072 		migrate_disable_switch(rq, prev);
5073 		psi_sched_switch(prev, next, !task_on_rq_queued(prev));
5074 
5075 		trace_sched_switch(preempt, prev, next);
5076 
5077 		/* Also unlocks the rq: */
5078 		rq = context_switch(rq, prev, next, &rf);
5079 	} else {
5080 		rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP);
5081 
5082 		rq_unpin_lock(rq, &rf);
5083 		__balance_callbacks(rq);
5084 		raw_spin_unlock_irq(&rq->lock);
5085 	}
5086 }
5087 
5088 void __noreturn do_task_dead(void)
5089 {
5090 	/* Causes final put_task_struct in finish_task_switch(): */
5091 	set_special_state(TASK_DEAD);
5092 
5093 	/* Tell freezer to ignore us: */
5094 	current->flags |= PF_NOFREEZE;
5095 
5096 	__schedule(false);
5097 	BUG();
5098 
5099 	/* Avoid "noreturn function does return" - but don't continue if BUG() is a NOP: */
5100 	for (;;)
5101 		cpu_relax();
5102 }
5103 
5104 static inline void sched_submit_work(struct task_struct *tsk)
5105 {
5106 	unsigned int task_flags;
5107 
5108 	if (!tsk->state)
5109 		return;
5110 
5111 	task_flags = tsk->flags;
5112 	/*
5113 	 * If a worker went to sleep, notify and ask workqueue whether
5114 	 * it wants to wake up a task to maintain concurrency.
5115 	 * As this function is called inside the schedule() context,
5116 	 * we disable preemption to avoid it calling schedule() again
5117 	 * in the possible wakeup of a kworker and because wq_worker_sleeping()
5118 	 * requires it.
5119 	 */
5120 	if (task_flags & (PF_WQ_WORKER | PF_IO_WORKER)) {
5121 		preempt_disable();
5122 		if (task_flags & PF_WQ_WORKER)
5123 			wq_worker_sleeping(tsk);
5124 		else
5125 			io_wq_worker_sleeping(tsk);
5126 		preempt_enable_no_resched();
5127 	}
5128 
5129 	if (tsk_is_pi_blocked(tsk))
5130 		return;
5131 
5132 	/*
5133 	 * If we are going to sleep and we have plugged IO queued,
5134 	 * make sure to submit it to avoid deadlocks.
5135 	 */
5136 	if (blk_needs_flush_plug(tsk))
5137 		blk_schedule_flush_plug(tsk);
5138 }
5139 
5140 static void sched_update_worker(struct task_struct *tsk)
5141 {
5142 	if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER)) {
5143 		if (tsk->flags & PF_WQ_WORKER)
5144 			wq_worker_running(tsk);
5145 		else
5146 			io_wq_worker_running(tsk);
5147 	}
5148 }
5149 
5150 asmlinkage __visible void __sched schedule(void)
5151 {
5152 	struct task_struct *tsk = current;
5153 
5154 	sched_submit_work(tsk);
5155 	do {
5156 		preempt_disable();
5157 		__schedule(false);
5158 		sched_preempt_enable_no_resched();
5159 	} while (need_resched());
5160 	sched_update_worker(tsk);
5161 }
5162 EXPORT_SYMBOL(schedule);
5163 
5164 /*
5165  * synchronize_rcu_tasks() makes sure that no task is stuck in preempted
5166  * state (have scheduled out non-voluntarily) by making sure that all
5167  * tasks have either left the run queue or have gone into user space.
5168  * As idle tasks do not do either, they must not ever be preempted
5169  * (schedule out non-voluntarily).
5170  *
5171  * schedule_idle() is similar to schedule_preempt_disable() except that it
5172  * never enables preemption because it does not call sched_submit_work().
5173  */
5174 void __sched schedule_idle(void)
5175 {
5176 	/*
5177 	 * As this skips calling sched_submit_work(), which the idle task does
5178 	 * regardless because that function is a nop when the task is in a
5179 	 * TASK_RUNNING state, make sure this isn't used someplace that the
5180 	 * current task can be in any other state. Note, idle is always in the
5181 	 * TASK_RUNNING state.
5182 	 */
5183 	WARN_ON_ONCE(current->state);
5184 	do {
5185 		__schedule(false);
5186 	} while (need_resched());
5187 }
5188 
5189 #if defined(CONFIG_CONTEXT_TRACKING) && !defined(CONFIG_HAVE_CONTEXT_TRACKING_OFFSTACK)
5190 asmlinkage __visible void __sched schedule_user(void)
5191 {
5192 	/*
5193 	 * If we come here after a random call to set_need_resched(),
5194 	 * or we have been woken up remotely but the IPI has not yet arrived,
5195 	 * we haven't yet exited the RCU idle mode. Do it here manually until
5196 	 * we find a better solution.
5197 	 *
5198 	 * NB: There are buggy callers of this function.  Ideally we
5199 	 * should warn if prev_state != CONTEXT_USER, but that will trigger
5200 	 * too frequently to make sense yet.
5201 	 */
5202 	enum ctx_state prev_state = exception_enter();
5203 	schedule();
5204 	exception_exit(prev_state);
5205 }
5206 #endif
5207 
5208 /**
5209  * schedule_preempt_disabled - called with preemption disabled
5210  *
5211  * Returns with preemption disabled. Note: preempt_count must be 1
5212  */
5213 void __sched schedule_preempt_disabled(void)
5214 {
5215 	sched_preempt_enable_no_resched();
5216 	schedule();
5217 	preempt_disable();
5218 }
5219 
5220 static void __sched notrace preempt_schedule_common(void)
5221 {
5222 	do {
5223 		/*
5224 		 * Because the function tracer can trace preempt_count_sub()
5225 		 * and it also uses preempt_enable/disable_notrace(), if
5226 		 * NEED_RESCHED is set, the preempt_enable_notrace() called
5227 		 * by the function tracer will call this function again and
5228 		 * cause infinite recursion.
5229 		 *
5230 		 * Preemption must be disabled here before the function
5231 		 * tracer can trace. Break up preempt_disable() into two
5232 		 * calls. One to disable preemption without fear of being
5233 		 * traced. The other to still record the preemption latency,
5234 		 * which can also be traced by the function tracer.
5235 		 */
5236 		preempt_disable_notrace();
5237 		preempt_latency_start(1);
5238 		__schedule(true);
5239 		preempt_latency_stop(1);
5240 		preempt_enable_no_resched_notrace();
5241 
5242 		/*
5243 		 * Check again in case we missed a preemption opportunity
5244 		 * between schedule and now.
5245 		 */
5246 	} while (need_resched());
5247 }
5248 
5249 #ifdef CONFIG_PREEMPTION
5250 /*
5251  * This is the entry point to schedule() from in-kernel preemption
5252  * off of preempt_enable.
5253  */
5254 asmlinkage __visible void __sched notrace preempt_schedule(void)
5255 {
5256 	/*
5257 	 * If there is a non-zero preempt_count or interrupts are disabled,
5258 	 * we do not want to preempt the current task. Just return..
5259 	 */
5260 	if (likely(!preemptible()))
5261 		return;
5262 
5263 	preempt_schedule_common();
5264 }
5265 NOKPROBE_SYMBOL(preempt_schedule);
5266 EXPORT_SYMBOL(preempt_schedule);
5267 
5268 /**
5269  * preempt_schedule_notrace - preempt_schedule called by tracing
5270  *
5271  * The tracing infrastructure uses preempt_enable_notrace to prevent
5272  * recursion and tracing preempt enabling caused by the tracing
5273  * infrastructure itself. But as tracing can happen in areas coming
5274  * from userspace or just about to enter userspace, a preempt enable
5275  * can occur before user_exit() is called. This will cause the scheduler
5276  * to be called when the system is still in usermode.
5277  *
5278  * To prevent this, the preempt_enable_notrace will use this function
5279  * instead of preempt_schedule() to exit user context if needed before
5280  * calling the scheduler.
5281  */
5282 asmlinkage __visible void __sched notrace preempt_schedule_notrace(void)
5283 {
5284 	enum ctx_state prev_ctx;
5285 
5286 	if (likely(!preemptible()))
5287 		return;
5288 
5289 	do {
5290 		/*
5291 		 * Because the function tracer can trace preempt_count_sub()
5292 		 * and it also uses preempt_enable/disable_notrace(), if
5293 		 * NEED_RESCHED is set, the preempt_enable_notrace() called
5294 		 * by the function tracer will call this function again and
5295 		 * cause infinite recursion.
5296 		 *
5297 		 * Preemption must be disabled here before the function
5298 		 * tracer can trace. Break up preempt_disable() into two
5299 		 * calls. One to disable preemption without fear of being
5300 		 * traced. The other to still record the preemption latency,
5301 		 * which can also be traced by the function tracer.
5302 		 */
5303 		preempt_disable_notrace();
5304 		preempt_latency_start(1);
5305 		/*
5306 		 * Needs preempt disabled in case user_exit() is traced
5307 		 * and the tracer calls preempt_enable_notrace() causing
5308 		 * an infinite recursion.
5309 		 */
5310 		prev_ctx = exception_enter();
5311 		__schedule(true);
5312 		exception_exit(prev_ctx);
5313 
5314 		preempt_latency_stop(1);
5315 		preempt_enable_no_resched_notrace();
5316 	} while (need_resched());
5317 }
5318 EXPORT_SYMBOL_GPL(preempt_schedule_notrace);
5319 
5320 #endif /* CONFIG_PREEMPTION */
5321 
5322 /*
5323  * This is the entry point to schedule() from kernel preemption
5324  * off of irq context.
5325  * Note, that this is called and return with irqs disabled. This will
5326  * protect us against recursive calling from irq.
5327  */
5328 asmlinkage __visible void __sched preempt_schedule_irq(void)
5329 {
5330 	enum ctx_state prev_state;
5331 
5332 	/* Catch callers which need to be fixed */
5333 	BUG_ON(preempt_count() || !irqs_disabled());
5334 
5335 	prev_state = exception_enter();
5336 
5337 	do {
5338 		preempt_disable();
5339 		local_irq_enable();
5340 		__schedule(true);
5341 		local_irq_disable();
5342 		sched_preempt_enable_no_resched();
5343 	} while (need_resched());
5344 
5345 	exception_exit(prev_state);
5346 }
5347 
5348 int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flags,
5349 			  void *key)
5350 {
5351 	WARN_ON_ONCE(IS_ENABLED(CONFIG_SCHED_DEBUG) && wake_flags & ~WF_SYNC);
5352 	return try_to_wake_up(curr->private, mode, wake_flags);
5353 }
5354 EXPORT_SYMBOL(default_wake_function);
5355 
5356 #ifdef CONFIG_RT_MUTEXES
5357 
5358 static inline int __rt_effective_prio(struct task_struct *pi_task, int prio)
5359 {
5360 	if (pi_task)
5361 		prio = min(prio, pi_task->prio);
5362 
5363 	return prio;
5364 }
5365 
5366 static inline int rt_effective_prio(struct task_struct *p, int prio)
5367 {
5368 	struct task_struct *pi_task = rt_mutex_get_top_task(p);
5369 
5370 	return __rt_effective_prio(pi_task, prio);
5371 }
5372 
5373 /*
5374  * rt_mutex_setprio - set the current priority of a task
5375  * @p: task to boost
5376  * @pi_task: donor task
5377  *
5378  * This function changes the 'effective' priority of a task. It does
5379  * not touch ->normal_prio like __setscheduler().
5380  *
5381  * Used by the rt_mutex code to implement priority inheritance
5382  * logic. Call site only calls if the priority of the task changed.
5383  */
5384 void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task)
5385 {
5386 	int prio, oldprio, queued, running, queue_flag =
5387 		DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK;
5388 	const struct sched_class *prev_class;
5389 	struct rq_flags rf;
5390 	struct rq *rq;
5391 
5392 	/* XXX used to be waiter->prio, not waiter->task->prio */
5393 	prio = __rt_effective_prio(pi_task, p->normal_prio);
5394 
5395 	/*
5396 	 * If nothing changed; bail early.
5397 	 */
5398 	if (p->pi_top_task == pi_task && prio == p->prio && !dl_prio(prio))
5399 		return;
5400 
5401 	rq = __task_rq_lock(p, &rf);
5402 	update_rq_clock(rq);
5403 	/*
5404 	 * Set under pi_lock && rq->lock, such that the value can be used under
5405 	 * either lock.
5406 	 *
5407 	 * Note that there is loads of tricky to make this pointer cache work
5408 	 * right. rt_mutex_slowunlock()+rt_mutex_postunlock() work together to
5409 	 * ensure a task is de-boosted (pi_task is set to NULL) before the
5410 	 * task is allowed to run again (and can exit). This ensures the pointer
5411 	 * points to a blocked task -- which guarantees the task is present.
5412 	 */
5413 	p->pi_top_task = pi_task;
5414 
5415 	/*
5416 	 * For FIFO/RR we only need to set prio, if that matches we're done.
5417 	 */
5418 	if (prio == p->prio && !dl_prio(prio))
5419 		goto out_unlock;
5420 
5421 	/*
5422 	 * Idle task boosting is a nono in general. There is one
5423 	 * exception, when PREEMPT_RT and NOHZ is active:
5424 	 *
5425 	 * The idle task calls get_next_timer_interrupt() and holds
5426 	 * the timer wheel base->lock on the CPU and another CPU wants
5427 	 * to access the timer (probably to cancel it). We can safely
5428 	 * ignore the boosting request, as the idle CPU runs this code
5429 	 * with interrupts disabled and will complete the lock
5430 	 * protected section without being interrupted. So there is no
5431 	 * real need to boost.
5432 	 */
5433 	if (unlikely(p == rq->idle)) {
5434 		WARN_ON(p != rq->curr);
5435 		WARN_ON(p->pi_blocked_on);
5436 		goto out_unlock;
5437 	}
5438 
5439 	trace_sched_pi_setprio(p, pi_task);
5440 	oldprio = p->prio;
5441 
5442 	if (oldprio == prio)
5443 		queue_flag &= ~DEQUEUE_MOVE;
5444 
5445 	prev_class = p->sched_class;
5446 	queued = task_on_rq_queued(p);
5447 	running = task_current(rq, p);
5448 	if (queued)
5449 		dequeue_task(rq, p, queue_flag);
5450 	if (running)
5451 		put_prev_task(rq, p);
5452 
5453 	/*
5454 	 * Boosting condition are:
5455 	 * 1. -rt task is running and holds mutex A
5456 	 *      --> -dl task blocks on mutex A
5457 	 *
5458 	 * 2. -dl task is running and holds mutex A
5459 	 *      --> -dl task blocks on mutex A and could preempt the
5460 	 *          running task
5461 	 */
5462 	if (dl_prio(prio)) {
5463 		if (!dl_prio(p->normal_prio) ||
5464 		    (pi_task && dl_prio(pi_task->prio) &&
5465 		     dl_entity_preempt(&pi_task->dl, &p->dl))) {
5466 			p->dl.pi_se = pi_task->dl.pi_se;
5467 			queue_flag |= ENQUEUE_REPLENISH;
5468 		} else {
5469 			p->dl.pi_se = &p->dl;
5470 		}
5471 		p->sched_class = &dl_sched_class;
5472 	} else if (rt_prio(prio)) {
5473 		if (dl_prio(oldprio))
5474 			p->dl.pi_se = &p->dl;
5475 		if (oldprio < prio)
5476 			queue_flag |= ENQUEUE_HEAD;
5477 		p->sched_class = &rt_sched_class;
5478 	} else {
5479 		if (dl_prio(oldprio))
5480 			p->dl.pi_se = &p->dl;
5481 		if (rt_prio(oldprio))
5482 			p->rt.timeout = 0;
5483 		p->sched_class = &fair_sched_class;
5484 	}
5485 
5486 	p->prio = prio;
5487 
5488 	if (queued)
5489 		enqueue_task(rq, p, queue_flag);
5490 	if (running)
5491 		set_next_task(rq, p);
5492 
5493 	check_class_changed(rq, p, prev_class, oldprio);
5494 out_unlock:
5495 	/* Avoid rq from going away on us: */
5496 	preempt_disable();
5497 
5498 	rq_unpin_lock(rq, &rf);
5499 	__balance_callbacks(rq);
5500 	raw_spin_unlock(&rq->lock);
5501 
5502 	preempt_enable();
5503 }
5504 #else
5505 static inline int rt_effective_prio(struct task_struct *p, int prio)
5506 {
5507 	return prio;
5508 }
5509 #endif
5510 
5511 void set_user_nice(struct task_struct *p, long nice)
5512 {
5513 	bool queued, running;
5514 	int old_prio;
5515 	struct rq_flags rf;
5516 	struct rq *rq;
5517 
5518 	if (task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE)
5519 		return;
5520 	/*
5521 	 * We have to be careful, if called from sys_setpriority(),
5522 	 * the task might be in the middle of scheduling on another CPU.
5523 	 */
5524 	rq = task_rq_lock(p, &rf);
5525 	update_rq_clock(rq);
5526 
5527 	/*
5528 	 * The RT priorities are set via sched_setscheduler(), but we still
5529 	 * allow the 'normal' nice value to be set - but as expected
5530 	 * it won't have any effect on scheduling until the task is
5531 	 * SCHED_DEADLINE, SCHED_FIFO or SCHED_RR:
5532 	 */
5533 	if (task_has_dl_policy(p) || task_has_rt_policy(p)) {
5534 		p->static_prio = NICE_TO_PRIO(nice);
5535 		goto out_unlock;
5536 	}
5537 	queued = task_on_rq_queued(p);
5538 	running = task_current(rq, p);
5539 	if (queued)
5540 		dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK);
5541 	if (running)
5542 		put_prev_task(rq, p);
5543 
5544 	p->static_prio = NICE_TO_PRIO(nice);
5545 	set_load_weight(p, true);
5546 	old_prio = p->prio;
5547 	p->prio = effective_prio(p);
5548 
5549 	if (queued)
5550 		enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK);
5551 	if (running)
5552 		set_next_task(rq, p);
5553 
5554 	/*
5555 	 * If the task increased its priority or is running and
5556 	 * lowered its priority, then reschedule its CPU:
5557 	 */
5558 	p->sched_class->prio_changed(rq, p, old_prio);
5559 
5560 out_unlock:
5561 	task_rq_unlock(rq, p, &rf);
5562 }
5563 EXPORT_SYMBOL(set_user_nice);
5564 
5565 /*
5566  * can_nice - check if a task can reduce its nice value
5567  * @p: task
5568  * @nice: nice value
5569  */
5570 int can_nice(const struct task_struct *p, const int nice)
5571 {
5572 	/* Convert nice value [19,-20] to rlimit style value [1,40]: */
5573 	int nice_rlim = nice_to_rlimit(nice);
5574 
5575 	return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) ||
5576 		capable(CAP_SYS_NICE));
5577 }
5578 
5579 #ifdef __ARCH_WANT_SYS_NICE
5580 
5581 /*
5582  * sys_nice - change the priority of the current process.
5583  * @increment: priority increment
5584  *
5585  * sys_setpriority is a more generic, but much slower function that
5586  * does similar things.
5587  */
5588 SYSCALL_DEFINE1(nice, int, increment)
5589 {
5590 	long nice, retval;
5591 
5592 	/*
5593 	 * Setpriority might change our priority at the same moment.
5594 	 * We don't have to worry. Conceptually one call occurs first
5595 	 * and we have a single winner.
5596 	 */
5597 	increment = clamp(increment, -NICE_WIDTH, NICE_WIDTH);
5598 	nice = task_nice(current) + increment;
5599 
5600 	nice = clamp_val(nice, MIN_NICE, MAX_NICE);
5601 	if (increment < 0 && !can_nice(current, nice))
5602 		return -EPERM;
5603 
5604 	retval = security_task_setnice(current, nice);
5605 	if (retval)
5606 		return retval;
5607 
5608 	set_user_nice(current, nice);
5609 	return 0;
5610 }
5611 
5612 #endif
5613 
5614 /**
5615  * task_prio - return the priority value of a given task.
5616  * @p: the task in question.
5617  *
5618  * Return: The priority value as seen by users in /proc.
5619  * RT tasks are offset by -200. Normal tasks are centered
5620  * around 0, value goes from -16 to +15.
5621  */
5622 int task_prio(const struct task_struct *p)
5623 {
5624 	return p->prio - MAX_RT_PRIO;
5625 }
5626 
5627 /**
5628  * idle_cpu - is a given CPU idle currently?
5629  * @cpu: the processor in question.
5630  *
5631  * Return: 1 if the CPU is currently idle. 0 otherwise.
5632  */
5633 int idle_cpu(int cpu)
5634 {
5635 	struct rq *rq = cpu_rq(cpu);
5636 
5637 	if (rq->curr != rq->idle)
5638 		return 0;
5639 
5640 	if (rq->nr_running)
5641 		return 0;
5642 
5643 #ifdef CONFIG_SMP
5644 	if (rq->ttwu_pending)
5645 		return 0;
5646 #endif
5647 
5648 	return 1;
5649 }
5650 
5651 /**
5652  * available_idle_cpu - is a given CPU idle for enqueuing work.
5653  * @cpu: the CPU in question.
5654  *
5655  * Return: 1 if the CPU is currently idle. 0 otherwise.
5656  */
5657 int available_idle_cpu(int cpu)
5658 {
5659 	if (!idle_cpu(cpu))
5660 		return 0;
5661 
5662 	if (vcpu_is_preempted(cpu))
5663 		return 0;
5664 
5665 	return 1;
5666 }
5667 
5668 /**
5669  * idle_task - return the idle task for a given CPU.
5670  * @cpu: the processor in question.
5671  *
5672  * Return: The idle task for the CPU @cpu.
5673  */
5674 struct task_struct *idle_task(int cpu)
5675 {
5676 	return cpu_rq(cpu)->idle;
5677 }
5678 
5679 /**
5680  * find_process_by_pid - find a process with a matching PID value.
5681  * @pid: the pid in question.
5682  *
5683  * The task of @pid, if found. %NULL otherwise.
5684  */
5685 static struct task_struct *find_process_by_pid(pid_t pid)
5686 {
5687 	return pid ? find_task_by_vpid(pid) : current;
5688 }
5689 
5690 /*
5691  * sched_setparam() passes in -1 for its policy, to let the functions
5692  * it calls know not to change it.
5693  */
5694 #define SETPARAM_POLICY	-1
5695 
5696 static void __setscheduler_params(struct task_struct *p,
5697 		const struct sched_attr *attr)
5698 {
5699 	int policy = attr->sched_policy;
5700 
5701 	if (policy == SETPARAM_POLICY)
5702 		policy = p->policy;
5703 
5704 	p->policy = policy;
5705 
5706 	if (dl_policy(policy))
5707 		__setparam_dl(p, attr);
5708 	else if (fair_policy(policy))
5709 		p->static_prio = NICE_TO_PRIO(attr->sched_nice);
5710 
5711 	/*
5712 	 * __sched_setscheduler() ensures attr->sched_priority == 0 when
5713 	 * !rt_policy. Always setting this ensures that things like
5714 	 * getparam()/getattr() don't report silly values for !rt tasks.
5715 	 */
5716 	p->rt_priority = attr->sched_priority;
5717 	p->normal_prio = normal_prio(p);
5718 	set_load_weight(p, true);
5719 }
5720 
5721 /* Actually do priority change: must hold pi & rq lock. */
5722 static void __setscheduler(struct rq *rq, struct task_struct *p,
5723 			   const struct sched_attr *attr, bool keep_boost)
5724 {
5725 	/*
5726 	 * If params can't change scheduling class changes aren't allowed
5727 	 * either.
5728 	 */
5729 	if (attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)
5730 		return;
5731 
5732 	__setscheduler_params(p, attr);
5733 
5734 	/*
5735 	 * Keep a potential priority boosting if called from
5736 	 * sched_setscheduler().
5737 	 */
5738 	p->prio = normal_prio(p);
5739 	if (keep_boost)
5740 		p->prio = rt_effective_prio(p, p->prio);
5741 
5742 	if (dl_prio(p->prio))
5743 		p->sched_class = &dl_sched_class;
5744 	else if (rt_prio(p->prio))
5745 		p->sched_class = &rt_sched_class;
5746 	else
5747 		p->sched_class = &fair_sched_class;
5748 }
5749 
5750 /*
5751  * Check the target process has a UID that matches the current process's:
5752  */
5753 static bool check_same_owner(struct task_struct *p)
5754 {
5755 	const struct cred *cred = current_cred(), *pcred;
5756 	bool match;
5757 
5758 	rcu_read_lock();
5759 	pcred = __task_cred(p);
5760 	match = (uid_eq(cred->euid, pcred->euid) ||
5761 		 uid_eq(cred->euid, pcred->uid));
5762 	rcu_read_unlock();
5763 	return match;
5764 }
5765 
5766 static int __sched_setscheduler(struct task_struct *p,
5767 				const struct sched_attr *attr,
5768 				bool user, bool pi)
5769 {
5770 	int newprio = dl_policy(attr->sched_policy) ? MAX_DL_PRIO - 1 :
5771 		      MAX_RT_PRIO - 1 - attr->sched_priority;
5772 	int retval, oldprio, oldpolicy = -1, queued, running;
5773 	int new_effective_prio, policy = attr->sched_policy;
5774 	const struct sched_class *prev_class;
5775 	struct callback_head *head;
5776 	struct rq_flags rf;
5777 	int reset_on_fork;
5778 	int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK;
5779 	struct rq *rq;
5780 
5781 	/* The pi code expects interrupts enabled */
5782 	BUG_ON(pi && in_interrupt());
5783 recheck:
5784 	/* Double check policy once rq lock held: */
5785 	if (policy < 0) {
5786 		reset_on_fork = p->sched_reset_on_fork;
5787 		policy = oldpolicy = p->policy;
5788 	} else {
5789 		reset_on_fork = !!(attr->sched_flags & SCHED_FLAG_RESET_ON_FORK);
5790 
5791 		if (!valid_policy(policy))
5792 			return -EINVAL;
5793 	}
5794 
5795 	if (attr->sched_flags & ~(SCHED_FLAG_ALL | SCHED_FLAG_SUGOV))
5796 		return -EINVAL;
5797 
5798 	/*
5799 	 * Valid priorities for SCHED_FIFO and SCHED_RR are
5800 	 * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL,
5801 	 * SCHED_BATCH and SCHED_IDLE is 0.
5802 	 */
5803 	if ((p->mm && attr->sched_priority > MAX_USER_RT_PRIO-1) ||
5804 	    (!p->mm && attr->sched_priority > MAX_RT_PRIO-1))
5805 		return -EINVAL;
5806 	if ((dl_policy(policy) && !__checkparam_dl(attr)) ||
5807 	    (rt_policy(policy) != (attr->sched_priority != 0)))
5808 		return -EINVAL;
5809 
5810 	/*
5811 	 * Allow unprivileged RT tasks to decrease priority:
5812 	 */
5813 	if (user && !capable(CAP_SYS_NICE)) {
5814 		if (fair_policy(policy)) {
5815 			if (attr->sched_nice < task_nice(p) &&
5816 			    !can_nice(p, attr->sched_nice))
5817 				return -EPERM;
5818 		}
5819 
5820 		if (rt_policy(policy)) {
5821 			unsigned long rlim_rtprio =
5822 					task_rlimit(p, RLIMIT_RTPRIO);
5823 
5824 			/* Can't set/change the rt policy: */
5825 			if (policy != p->policy && !rlim_rtprio)
5826 				return -EPERM;
5827 
5828 			/* Can't increase priority: */
5829 			if (attr->sched_priority > p->rt_priority &&
5830 			    attr->sched_priority > rlim_rtprio)
5831 				return -EPERM;
5832 		}
5833 
5834 		 /*
5835 		  * Can't set/change SCHED_DEADLINE policy at all for now
5836 		  * (safest behavior); in the future we would like to allow
5837 		  * unprivileged DL tasks to increase their relative deadline
5838 		  * or reduce their runtime (both ways reducing utilization)
5839 		  */
5840 		if (dl_policy(policy))
5841 			return -EPERM;
5842 
5843 		/*
5844 		 * Treat SCHED_IDLE as nice 20. Only allow a switch to
5845 		 * SCHED_NORMAL if the RLIMIT_NICE would normally permit it.
5846 		 */
5847 		if (task_has_idle_policy(p) && !idle_policy(policy)) {
5848 			if (!can_nice(p, task_nice(p)))
5849 				return -EPERM;
5850 		}
5851 
5852 		/* Can't change other user's priorities: */
5853 		if (!check_same_owner(p))
5854 			return -EPERM;
5855 
5856 		/* Normal users shall not reset the sched_reset_on_fork flag: */
5857 		if (p->sched_reset_on_fork && !reset_on_fork)
5858 			return -EPERM;
5859 	}
5860 
5861 	if (user) {
5862 		if (attr->sched_flags & SCHED_FLAG_SUGOV)
5863 			return -EINVAL;
5864 
5865 		retval = security_task_setscheduler(p);
5866 		if (retval)
5867 			return retval;
5868 	}
5869 
5870 	/* Update task specific "requested" clamps */
5871 	if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP) {
5872 		retval = uclamp_validate(p, attr);
5873 		if (retval)
5874 			return retval;
5875 	}
5876 
5877 	if (pi)
5878 		cpuset_read_lock();
5879 
5880 	/*
5881 	 * Make sure no PI-waiters arrive (or leave) while we are
5882 	 * changing the priority of the task:
5883 	 *
5884 	 * To be able to change p->policy safely, the appropriate
5885 	 * runqueue lock must be held.
5886 	 */
5887 	rq = task_rq_lock(p, &rf);
5888 	update_rq_clock(rq);
5889 
5890 	/*
5891 	 * Changing the policy of the stop threads its a very bad idea:
5892 	 */
5893 	if (p == rq->stop) {
5894 		retval = -EINVAL;
5895 		goto unlock;
5896 	}
5897 
5898 	/*
5899 	 * If not changing anything there's no need to proceed further,
5900 	 * but store a possible modification of reset_on_fork.
5901 	 */
5902 	if (unlikely(policy == p->policy)) {
5903 		if (fair_policy(policy) && attr->sched_nice != task_nice(p))
5904 			goto change;
5905 		if (rt_policy(policy) && attr->sched_priority != p->rt_priority)
5906 			goto change;
5907 		if (dl_policy(policy) && dl_param_changed(p, attr))
5908 			goto change;
5909 		if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP)
5910 			goto change;
5911 
5912 		p->sched_reset_on_fork = reset_on_fork;
5913 		retval = 0;
5914 		goto unlock;
5915 	}
5916 change:
5917 
5918 	if (user) {
5919 #ifdef CONFIG_RT_GROUP_SCHED
5920 		/*
5921 		 * Do not allow realtime tasks into groups that have no runtime
5922 		 * assigned.
5923 		 */
5924 		if (rt_bandwidth_enabled() && rt_policy(policy) &&
5925 				task_group(p)->rt_bandwidth.rt_runtime == 0 &&
5926 				!task_group_is_autogroup(task_group(p))) {
5927 			retval = -EPERM;
5928 			goto unlock;
5929 		}
5930 #endif
5931 #ifdef CONFIG_SMP
5932 		if (dl_bandwidth_enabled() && dl_policy(policy) &&
5933 				!(attr->sched_flags & SCHED_FLAG_SUGOV)) {
5934 			cpumask_t *span = rq->rd->span;
5935 
5936 			/*
5937 			 * Don't allow tasks with an affinity mask smaller than
5938 			 * the entire root_domain to become SCHED_DEADLINE. We
5939 			 * will also fail if there's no bandwidth available.
5940 			 */
5941 			if (!cpumask_subset(span, p->cpus_ptr) ||
5942 			    rq->rd->dl_bw.bw == 0) {
5943 				retval = -EPERM;
5944 				goto unlock;
5945 			}
5946 		}
5947 #endif
5948 	}
5949 
5950 	/* Re-check policy now with rq lock held: */
5951 	if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
5952 		policy = oldpolicy = -1;
5953 		task_rq_unlock(rq, p, &rf);
5954 		if (pi)
5955 			cpuset_read_unlock();
5956 		goto recheck;
5957 	}
5958 
5959 	/*
5960 	 * If setscheduling to SCHED_DEADLINE (or changing the parameters
5961 	 * of a SCHED_DEADLINE task) we need to check if enough bandwidth
5962 	 * is available.
5963 	 */
5964 	if ((dl_policy(policy) || dl_task(p)) && sched_dl_overflow(p, policy, attr)) {
5965 		retval = -EBUSY;
5966 		goto unlock;
5967 	}
5968 
5969 	p->sched_reset_on_fork = reset_on_fork;
5970 	oldprio = p->prio;
5971 
5972 	if (pi) {
5973 		/*
5974 		 * Take priority boosted tasks into account. If the new
5975 		 * effective priority is unchanged, we just store the new
5976 		 * normal parameters and do not touch the scheduler class and
5977 		 * the runqueue. This will be done when the task deboost
5978 		 * itself.
5979 		 */
5980 		new_effective_prio = rt_effective_prio(p, newprio);
5981 		if (new_effective_prio == oldprio)
5982 			queue_flags &= ~DEQUEUE_MOVE;
5983 	}
5984 
5985 	queued = task_on_rq_queued(p);
5986 	running = task_current(rq, p);
5987 	if (queued)
5988 		dequeue_task(rq, p, queue_flags);
5989 	if (running)
5990 		put_prev_task(rq, p);
5991 
5992 	prev_class = p->sched_class;
5993 
5994 	__setscheduler(rq, p, attr, pi);
5995 	__setscheduler_uclamp(p, attr);
5996 
5997 	if (queued) {
5998 		/*
5999 		 * We enqueue to tail when the priority of a task is
6000 		 * increased (user space view).
6001 		 */
6002 		if (oldprio < p->prio)
6003 			queue_flags |= ENQUEUE_HEAD;
6004 
6005 		enqueue_task(rq, p, queue_flags);
6006 	}
6007 	if (running)
6008 		set_next_task(rq, p);
6009 
6010 	check_class_changed(rq, p, prev_class, oldprio);
6011 
6012 	/* Avoid rq from going away on us: */
6013 	preempt_disable();
6014 	head = splice_balance_callbacks(rq);
6015 	task_rq_unlock(rq, p, &rf);
6016 
6017 	if (pi) {
6018 		cpuset_read_unlock();
6019 		rt_mutex_adjust_pi(p);
6020 	}
6021 
6022 	/* Run balance callbacks after we've adjusted the PI chain: */
6023 	balance_callbacks(rq, head);
6024 	preempt_enable();
6025 
6026 	return 0;
6027 
6028 unlock:
6029 	task_rq_unlock(rq, p, &rf);
6030 	if (pi)
6031 		cpuset_read_unlock();
6032 	return retval;
6033 }
6034 
6035 static int _sched_setscheduler(struct task_struct *p, int policy,
6036 			       const struct sched_param *param, bool check)
6037 {
6038 	struct sched_attr attr = {
6039 		.sched_policy   = policy,
6040 		.sched_priority = param->sched_priority,
6041 		.sched_nice	= PRIO_TO_NICE(p->static_prio),
6042 	};
6043 
6044 	/* Fixup the legacy SCHED_RESET_ON_FORK hack. */
6045 	if ((policy != SETPARAM_POLICY) && (policy & SCHED_RESET_ON_FORK)) {
6046 		attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK;
6047 		policy &= ~SCHED_RESET_ON_FORK;
6048 		attr.sched_policy = policy;
6049 	}
6050 
6051 	return __sched_setscheduler(p, &attr, check, true);
6052 }
6053 /**
6054  * sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
6055  * @p: the task in question.
6056  * @policy: new policy.
6057  * @param: structure containing the new RT priority.
6058  *
6059  * Use sched_set_fifo(), read its comment.
6060  *
6061  * Return: 0 on success. An error code otherwise.
6062  *
6063  * NOTE that the task may be already dead.
6064  */
6065 int sched_setscheduler(struct task_struct *p, int policy,
6066 		       const struct sched_param *param)
6067 {
6068 	return _sched_setscheduler(p, policy, param, true);
6069 }
6070 
6071 int sched_setattr(struct task_struct *p, const struct sched_attr *attr)
6072 {
6073 	return __sched_setscheduler(p, attr, true, true);
6074 }
6075 
6076 int sched_setattr_nocheck(struct task_struct *p, const struct sched_attr *attr)
6077 {
6078 	return __sched_setscheduler(p, attr, false, true);
6079 }
6080 
6081 /**
6082  * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace.
6083  * @p: the task in question.
6084  * @policy: new policy.
6085  * @param: structure containing the new RT priority.
6086  *
6087  * Just like sched_setscheduler, only don't bother checking if the
6088  * current context has permission.  For example, this is needed in
6089  * stop_machine(): we create temporary high priority worker threads,
6090  * but our caller might not have that capability.
6091  *
6092  * Return: 0 on success. An error code otherwise.
6093  */
6094 int sched_setscheduler_nocheck(struct task_struct *p, int policy,
6095 			       const struct sched_param *param)
6096 {
6097 	return _sched_setscheduler(p, policy, param, false);
6098 }
6099 
6100 /*
6101  * SCHED_FIFO is a broken scheduler model; that is, it is fundamentally
6102  * incapable of resource management, which is the one thing an OS really should
6103  * be doing.
6104  *
6105  * This is of course the reason it is limited to privileged users only.
6106  *
6107  * Worse still; it is fundamentally impossible to compose static priority
6108  * workloads. You cannot take two correctly working static prio workloads
6109  * and smash them together and still expect them to work.
6110  *
6111  * For this reason 'all' FIFO tasks the kernel creates are basically at:
6112  *
6113  *   MAX_RT_PRIO / 2
6114  *
6115  * The administrator _MUST_ configure the system, the kernel simply doesn't
6116  * know enough information to make a sensible choice.
6117  */
6118 void sched_set_fifo(struct task_struct *p)
6119 {
6120 	struct sched_param sp = { .sched_priority = MAX_RT_PRIO / 2 };
6121 	WARN_ON_ONCE(sched_setscheduler_nocheck(p, SCHED_FIFO, &sp) != 0);
6122 }
6123 EXPORT_SYMBOL_GPL(sched_set_fifo);
6124 
6125 /*
6126  * For when you don't much care about FIFO, but want to be above SCHED_NORMAL.
6127  */
6128 void sched_set_fifo_low(struct task_struct *p)
6129 {
6130 	struct sched_param sp = { .sched_priority = 1 };
6131 	WARN_ON_ONCE(sched_setscheduler_nocheck(p, SCHED_FIFO, &sp) != 0);
6132 }
6133 EXPORT_SYMBOL_GPL(sched_set_fifo_low);
6134 
6135 void sched_set_normal(struct task_struct *p, int nice)
6136 {
6137 	struct sched_attr attr = {
6138 		.sched_policy = SCHED_NORMAL,
6139 		.sched_nice = nice,
6140 	};
6141 	WARN_ON_ONCE(sched_setattr_nocheck(p, &attr) != 0);
6142 }
6143 EXPORT_SYMBOL_GPL(sched_set_normal);
6144 
6145 static int
6146 do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
6147 {
6148 	struct sched_param lparam;
6149 	struct task_struct *p;
6150 	int retval;
6151 
6152 	if (!param || pid < 0)
6153 		return -EINVAL;
6154 	if (copy_from_user(&lparam, param, sizeof(struct sched_param)))
6155 		return -EFAULT;
6156 
6157 	rcu_read_lock();
6158 	retval = -ESRCH;
6159 	p = find_process_by_pid(pid);
6160 	if (likely(p))
6161 		get_task_struct(p);
6162 	rcu_read_unlock();
6163 
6164 	if (likely(p)) {
6165 		retval = sched_setscheduler(p, policy, &lparam);
6166 		put_task_struct(p);
6167 	}
6168 
6169 	return retval;
6170 }
6171 
6172 /*
6173  * Mimics kernel/events/core.c perf_copy_attr().
6174  */
6175 static int sched_copy_attr(struct sched_attr __user *uattr, struct sched_attr *attr)
6176 {
6177 	u32 size;
6178 	int ret;
6179 
6180 	/* Zero the full structure, so that a short copy will be nice: */
6181 	memset(attr, 0, sizeof(*attr));
6182 
6183 	ret = get_user(size, &uattr->size);
6184 	if (ret)
6185 		return ret;
6186 
6187 	/* ABI compatibility quirk: */
6188 	if (!size)
6189 		size = SCHED_ATTR_SIZE_VER0;
6190 	if (size < SCHED_ATTR_SIZE_VER0 || size > PAGE_SIZE)
6191 		goto err_size;
6192 
6193 	ret = copy_struct_from_user(attr, sizeof(*attr), uattr, size);
6194 	if (ret) {
6195 		if (ret == -E2BIG)
6196 			goto err_size;
6197 		return ret;
6198 	}
6199 
6200 	if ((attr->sched_flags & SCHED_FLAG_UTIL_CLAMP) &&
6201 	    size < SCHED_ATTR_SIZE_VER1)
6202 		return -EINVAL;
6203 
6204 	/*
6205 	 * XXX: Do we want to be lenient like existing syscalls; or do we want
6206 	 * to be strict and return an error on out-of-bounds values?
6207 	 */
6208 	attr->sched_nice = clamp(attr->sched_nice, MIN_NICE, MAX_NICE);
6209 
6210 	return 0;
6211 
6212 err_size:
6213 	put_user(sizeof(*attr), &uattr->size);
6214 	return -E2BIG;
6215 }
6216 
6217 /**
6218  * sys_sched_setscheduler - set/change the scheduler policy and RT priority
6219  * @pid: the pid in question.
6220  * @policy: new policy.
6221  * @param: structure containing the new RT priority.
6222  *
6223  * Return: 0 on success. An error code otherwise.
6224  */
6225 SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy, struct sched_param __user *, param)
6226 {
6227 	if (policy < 0)
6228 		return -EINVAL;
6229 
6230 	return do_sched_setscheduler(pid, policy, param);
6231 }
6232 
6233 /**
6234  * sys_sched_setparam - set/change the RT priority of a thread
6235  * @pid: the pid in question.
6236  * @param: structure containing the new RT priority.
6237  *
6238  * Return: 0 on success. An error code otherwise.
6239  */
6240 SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param)
6241 {
6242 	return do_sched_setscheduler(pid, SETPARAM_POLICY, param);
6243 }
6244 
6245 /**
6246  * sys_sched_setattr - same as above, but with extended sched_attr
6247  * @pid: the pid in question.
6248  * @uattr: structure containing the extended parameters.
6249  * @flags: for future extension.
6250  */
6251 SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr,
6252 			       unsigned int, flags)
6253 {
6254 	struct sched_attr attr;
6255 	struct task_struct *p;
6256 	int retval;
6257 
6258 	if (!uattr || pid < 0 || flags)
6259 		return -EINVAL;
6260 
6261 	retval = sched_copy_attr(uattr, &attr);
6262 	if (retval)
6263 		return retval;
6264 
6265 	if ((int)attr.sched_policy < 0)
6266 		return -EINVAL;
6267 	if (attr.sched_flags & SCHED_FLAG_KEEP_POLICY)
6268 		attr.sched_policy = SETPARAM_POLICY;
6269 
6270 	rcu_read_lock();
6271 	retval = -ESRCH;
6272 	p = find_process_by_pid(pid);
6273 	if (likely(p))
6274 		get_task_struct(p);
6275 	rcu_read_unlock();
6276 
6277 	if (likely(p)) {
6278 		retval = sched_setattr(p, &attr);
6279 		put_task_struct(p);
6280 	}
6281 
6282 	return retval;
6283 }
6284 
6285 /**
6286  * sys_sched_getscheduler - get the policy (scheduling class) of a thread
6287  * @pid: the pid in question.
6288  *
6289  * Return: On success, the policy of the thread. Otherwise, a negative error
6290  * code.
6291  */
6292 SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid)
6293 {
6294 	struct task_struct *p;
6295 	int retval;
6296 
6297 	if (pid < 0)
6298 		return -EINVAL;
6299 
6300 	retval = -ESRCH;
6301 	rcu_read_lock();
6302 	p = find_process_by_pid(pid);
6303 	if (p) {
6304 		retval = security_task_getscheduler(p);
6305 		if (!retval)
6306 			retval = p->policy
6307 				| (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0);
6308 	}
6309 	rcu_read_unlock();
6310 	return retval;
6311 }
6312 
6313 /**
6314  * sys_sched_getparam - get the RT priority of a thread
6315  * @pid: the pid in question.
6316  * @param: structure containing the RT priority.
6317  *
6318  * Return: On success, 0 and the RT priority is in @param. Otherwise, an error
6319  * code.
6320  */
6321 SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param)
6322 {
6323 	struct sched_param lp = { .sched_priority = 0 };
6324 	struct task_struct *p;
6325 	int retval;
6326 
6327 	if (!param || pid < 0)
6328 		return -EINVAL;
6329 
6330 	rcu_read_lock();
6331 	p = find_process_by_pid(pid);
6332 	retval = -ESRCH;
6333 	if (!p)
6334 		goto out_unlock;
6335 
6336 	retval = security_task_getscheduler(p);
6337 	if (retval)
6338 		goto out_unlock;
6339 
6340 	if (task_has_rt_policy(p))
6341 		lp.sched_priority = p->rt_priority;
6342 	rcu_read_unlock();
6343 
6344 	/*
6345 	 * This one might sleep, we cannot do it with a spinlock held ...
6346 	 */
6347 	retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0;
6348 
6349 	return retval;
6350 
6351 out_unlock:
6352 	rcu_read_unlock();
6353 	return retval;
6354 }
6355 
6356 /*
6357  * Copy the kernel size attribute structure (which might be larger
6358  * than what user-space knows about) to user-space.
6359  *
6360  * Note that all cases are valid: user-space buffer can be larger or
6361  * smaller than the kernel-space buffer. The usual case is that both
6362  * have the same size.
6363  */
6364 static int
6365 sched_attr_copy_to_user(struct sched_attr __user *uattr,
6366 			struct sched_attr *kattr,
6367 			unsigned int usize)
6368 {
6369 	unsigned int ksize = sizeof(*kattr);
6370 
6371 	if (!access_ok(uattr, usize))
6372 		return -EFAULT;
6373 
6374 	/*
6375 	 * sched_getattr() ABI forwards and backwards compatibility:
6376 	 *
6377 	 * If usize == ksize then we just copy everything to user-space and all is good.
6378 	 *
6379 	 * If usize < ksize then we only copy as much as user-space has space for,
6380 	 * this keeps ABI compatibility as well. We skip the rest.
6381 	 *
6382 	 * If usize > ksize then user-space is using a newer version of the ABI,
6383 	 * which part the kernel doesn't know about. Just ignore it - tooling can
6384 	 * detect the kernel's knowledge of attributes from the attr->size value
6385 	 * which is set to ksize in this case.
6386 	 */
6387 	kattr->size = min(usize, ksize);
6388 
6389 	if (copy_to_user(uattr, kattr, kattr->size))
6390 		return -EFAULT;
6391 
6392 	return 0;
6393 }
6394 
6395 /**
6396  * sys_sched_getattr - similar to sched_getparam, but with sched_attr
6397  * @pid: the pid in question.
6398  * @uattr: structure containing the extended parameters.
6399  * @usize: sizeof(attr) for fwd/bwd comp.
6400  * @flags: for future extension.
6401  */
6402 SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr,
6403 		unsigned int, usize, unsigned int, flags)
6404 {
6405 	struct sched_attr kattr = { };
6406 	struct task_struct *p;
6407 	int retval;
6408 
6409 	if (!uattr || pid < 0 || usize > PAGE_SIZE ||
6410 	    usize < SCHED_ATTR_SIZE_VER0 || flags)
6411 		return -EINVAL;
6412 
6413 	rcu_read_lock();
6414 	p = find_process_by_pid(pid);
6415 	retval = -ESRCH;
6416 	if (!p)
6417 		goto out_unlock;
6418 
6419 	retval = security_task_getscheduler(p);
6420 	if (retval)
6421 		goto out_unlock;
6422 
6423 	kattr.sched_policy = p->policy;
6424 	if (p->sched_reset_on_fork)
6425 		kattr.sched_flags |= SCHED_FLAG_RESET_ON_FORK;
6426 	if (task_has_dl_policy(p))
6427 		__getparam_dl(p, &kattr);
6428 	else if (task_has_rt_policy(p))
6429 		kattr.sched_priority = p->rt_priority;
6430 	else
6431 		kattr.sched_nice = task_nice(p);
6432 
6433 #ifdef CONFIG_UCLAMP_TASK
6434 	/*
6435 	 * This could race with another potential updater, but this is fine
6436 	 * because it'll correctly read the old or the new value. We don't need
6437 	 * to guarantee who wins the race as long as it doesn't return garbage.
6438 	 */
6439 	kattr.sched_util_min = p->uclamp_req[UCLAMP_MIN].value;
6440 	kattr.sched_util_max = p->uclamp_req[UCLAMP_MAX].value;
6441 #endif
6442 
6443 	rcu_read_unlock();
6444 
6445 	return sched_attr_copy_to_user(uattr, &kattr, usize);
6446 
6447 out_unlock:
6448 	rcu_read_unlock();
6449 	return retval;
6450 }
6451 
6452 long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
6453 {
6454 	cpumask_var_t cpus_allowed, new_mask;
6455 	struct task_struct *p;
6456 	int retval;
6457 
6458 	rcu_read_lock();
6459 
6460 	p = find_process_by_pid(pid);
6461 	if (!p) {
6462 		rcu_read_unlock();
6463 		return -ESRCH;
6464 	}
6465 
6466 	/* Prevent p going away */
6467 	get_task_struct(p);
6468 	rcu_read_unlock();
6469 
6470 	if (p->flags & PF_NO_SETAFFINITY) {
6471 		retval = -EINVAL;
6472 		goto out_put_task;
6473 	}
6474 	if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
6475 		retval = -ENOMEM;
6476 		goto out_put_task;
6477 	}
6478 	if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
6479 		retval = -ENOMEM;
6480 		goto out_free_cpus_allowed;
6481 	}
6482 	retval = -EPERM;
6483 	if (!check_same_owner(p)) {
6484 		rcu_read_lock();
6485 		if (!ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE)) {
6486 			rcu_read_unlock();
6487 			goto out_free_new_mask;
6488 		}
6489 		rcu_read_unlock();
6490 	}
6491 
6492 	retval = security_task_setscheduler(p);
6493 	if (retval)
6494 		goto out_free_new_mask;
6495 
6496 
6497 	cpuset_cpus_allowed(p, cpus_allowed);
6498 	cpumask_and(new_mask, in_mask, cpus_allowed);
6499 
6500 	/*
6501 	 * Since bandwidth control happens on root_domain basis,
6502 	 * if admission test is enabled, we only admit -deadline
6503 	 * tasks allowed to run on all the CPUs in the task's
6504 	 * root_domain.
6505 	 */
6506 #ifdef CONFIG_SMP
6507 	if (task_has_dl_policy(p) && dl_bandwidth_enabled()) {
6508 		rcu_read_lock();
6509 		if (!cpumask_subset(task_rq(p)->rd->span, new_mask)) {
6510 			retval = -EBUSY;
6511 			rcu_read_unlock();
6512 			goto out_free_new_mask;
6513 		}
6514 		rcu_read_unlock();
6515 	}
6516 #endif
6517 again:
6518 	retval = __set_cpus_allowed_ptr(p, new_mask, SCA_CHECK);
6519 
6520 	if (!retval) {
6521 		cpuset_cpus_allowed(p, cpus_allowed);
6522 		if (!cpumask_subset(new_mask, cpus_allowed)) {
6523 			/*
6524 			 * We must have raced with a concurrent cpuset
6525 			 * update. Just reset the cpus_allowed to the
6526 			 * cpuset's cpus_allowed
6527 			 */
6528 			cpumask_copy(new_mask, cpus_allowed);
6529 			goto again;
6530 		}
6531 	}
6532 out_free_new_mask:
6533 	free_cpumask_var(new_mask);
6534 out_free_cpus_allowed:
6535 	free_cpumask_var(cpus_allowed);
6536 out_put_task:
6537 	put_task_struct(p);
6538 	return retval;
6539 }
6540 
6541 static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
6542 			     struct cpumask *new_mask)
6543 {
6544 	if (len < cpumask_size())
6545 		cpumask_clear(new_mask);
6546 	else if (len > cpumask_size())
6547 		len = cpumask_size();
6548 
6549 	return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
6550 }
6551 
6552 /**
6553  * sys_sched_setaffinity - set the CPU affinity of a process
6554  * @pid: pid of the process
6555  * @len: length in bytes of the bitmask pointed to by user_mask_ptr
6556  * @user_mask_ptr: user-space pointer to the new CPU mask
6557  *
6558  * Return: 0 on success. An error code otherwise.
6559  */
6560 SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len,
6561 		unsigned long __user *, user_mask_ptr)
6562 {
6563 	cpumask_var_t new_mask;
6564 	int retval;
6565 
6566 	if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
6567 		return -ENOMEM;
6568 
6569 	retval = get_user_cpu_mask(user_mask_ptr, len, new_mask);
6570 	if (retval == 0)
6571 		retval = sched_setaffinity(pid, new_mask);
6572 	free_cpumask_var(new_mask);
6573 	return retval;
6574 }
6575 
6576 long sched_getaffinity(pid_t pid, struct cpumask *mask)
6577 {
6578 	struct task_struct *p;
6579 	unsigned long flags;
6580 	int retval;
6581 
6582 	rcu_read_lock();
6583 
6584 	retval = -ESRCH;
6585 	p = find_process_by_pid(pid);
6586 	if (!p)
6587 		goto out_unlock;
6588 
6589 	retval = security_task_getscheduler(p);
6590 	if (retval)
6591 		goto out_unlock;
6592 
6593 	raw_spin_lock_irqsave(&p->pi_lock, flags);
6594 	cpumask_and(mask, &p->cpus_mask, cpu_active_mask);
6595 	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
6596 
6597 out_unlock:
6598 	rcu_read_unlock();
6599 
6600 	return retval;
6601 }
6602 
6603 /**
6604  * sys_sched_getaffinity - get the CPU affinity of a process
6605  * @pid: pid of the process
6606  * @len: length in bytes of the bitmask pointed to by user_mask_ptr
6607  * @user_mask_ptr: user-space pointer to hold the current CPU mask
6608  *
6609  * Return: size of CPU mask copied to user_mask_ptr on success. An
6610  * error code otherwise.
6611  */
6612 SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len,
6613 		unsigned long __user *, user_mask_ptr)
6614 {
6615 	int ret;
6616 	cpumask_var_t mask;
6617 
6618 	if ((len * BITS_PER_BYTE) < nr_cpu_ids)
6619 		return -EINVAL;
6620 	if (len & (sizeof(unsigned long)-1))
6621 		return -EINVAL;
6622 
6623 	if (!alloc_cpumask_var(&mask, GFP_KERNEL))
6624 		return -ENOMEM;
6625 
6626 	ret = sched_getaffinity(pid, mask);
6627 	if (ret == 0) {
6628 		unsigned int retlen = min(len, cpumask_size());
6629 
6630 		if (copy_to_user(user_mask_ptr, mask, retlen))
6631 			ret = -EFAULT;
6632 		else
6633 			ret = retlen;
6634 	}
6635 	free_cpumask_var(mask);
6636 
6637 	return ret;
6638 }
6639 
6640 static void do_sched_yield(void)
6641 {
6642 	struct rq_flags rf;
6643 	struct rq *rq;
6644 
6645 	rq = this_rq_lock_irq(&rf);
6646 
6647 	schedstat_inc(rq->yld_count);
6648 	current->sched_class->yield_task(rq);
6649 
6650 	preempt_disable();
6651 	rq_unlock_irq(rq, &rf);
6652 	sched_preempt_enable_no_resched();
6653 
6654 	schedule();
6655 }
6656 
6657 /**
6658  * sys_sched_yield - yield the current processor to other threads.
6659  *
6660  * This function yields the current CPU to other tasks. If there are no
6661  * other threads running on this CPU then this function will return.
6662  *
6663  * Return: 0.
6664  */
6665 SYSCALL_DEFINE0(sched_yield)
6666 {
6667 	do_sched_yield();
6668 	return 0;
6669 }
6670 
6671 #ifndef CONFIG_PREEMPTION
6672 int __sched _cond_resched(void)
6673 {
6674 	if (should_resched(0)) {
6675 		preempt_schedule_common();
6676 		return 1;
6677 	}
6678 	rcu_all_qs();
6679 	return 0;
6680 }
6681 EXPORT_SYMBOL(_cond_resched);
6682 #endif
6683 
6684 /*
6685  * __cond_resched_lock() - if a reschedule is pending, drop the given lock,
6686  * call schedule, and on return reacquire the lock.
6687  *
6688  * This works OK both with and without CONFIG_PREEMPTION. We do strange low-level
6689  * operations here to prevent schedule() from being called twice (once via
6690  * spin_unlock(), once by hand).
6691  */
6692 int __cond_resched_lock(spinlock_t *lock)
6693 {
6694 	int resched = should_resched(PREEMPT_LOCK_OFFSET);
6695 	int ret = 0;
6696 
6697 	lockdep_assert_held(lock);
6698 
6699 	if (spin_needbreak(lock) || resched) {
6700 		spin_unlock(lock);
6701 		if (resched)
6702 			preempt_schedule_common();
6703 		else
6704 			cpu_relax();
6705 		ret = 1;
6706 		spin_lock(lock);
6707 	}
6708 	return ret;
6709 }
6710 EXPORT_SYMBOL(__cond_resched_lock);
6711 
6712 /**
6713  * yield - yield the current processor to other threads.
6714  *
6715  * Do not ever use this function, there's a 99% chance you're doing it wrong.
6716  *
6717  * The scheduler is at all times free to pick the calling task as the most
6718  * eligible task to run, if removing the yield() call from your code breaks
6719  * it, it's already broken.
6720  *
6721  * Typical broken usage is:
6722  *
6723  * while (!event)
6724  *	yield();
6725  *
6726  * where one assumes that yield() will let 'the other' process run that will
6727  * make event true. If the current task is a SCHED_FIFO task that will never
6728  * happen. Never use yield() as a progress guarantee!!
6729  *
6730  * If you want to use yield() to wait for something, use wait_event().
6731  * If you want to use yield() to be 'nice' for others, use cond_resched().
6732  * If you still want to use yield(), do not!
6733  */
6734 void __sched yield(void)
6735 {
6736 	set_current_state(TASK_RUNNING);
6737 	do_sched_yield();
6738 }
6739 EXPORT_SYMBOL(yield);
6740 
6741 /**
6742  * yield_to - yield the current processor to another thread in
6743  * your thread group, or accelerate that thread toward the
6744  * processor it's on.
6745  * @p: target task
6746  * @preempt: whether task preemption is allowed or not
6747  *
6748  * It's the caller's job to ensure that the target task struct
6749  * can't go away on us before we can do any checks.
6750  *
6751  * Return:
6752  *	true (>0) if we indeed boosted the target task.
6753  *	false (0) if we failed to boost the target.
6754  *	-ESRCH if there's no task to yield to.
6755  */
6756 int __sched yield_to(struct task_struct *p, bool preempt)
6757 {
6758 	struct task_struct *curr = current;
6759 	struct rq *rq, *p_rq;
6760 	unsigned long flags;
6761 	int yielded = 0;
6762 
6763 	local_irq_save(flags);
6764 	rq = this_rq();
6765 
6766 again:
6767 	p_rq = task_rq(p);
6768 	/*
6769 	 * If we're the only runnable task on the rq and target rq also
6770 	 * has only one task, there's absolutely no point in yielding.
6771 	 */
6772 	if (rq->nr_running == 1 && p_rq->nr_running == 1) {
6773 		yielded = -ESRCH;
6774 		goto out_irq;
6775 	}
6776 
6777 	double_rq_lock(rq, p_rq);
6778 	if (task_rq(p) != p_rq) {
6779 		double_rq_unlock(rq, p_rq);
6780 		goto again;
6781 	}
6782 
6783 	if (!curr->sched_class->yield_to_task)
6784 		goto out_unlock;
6785 
6786 	if (curr->sched_class != p->sched_class)
6787 		goto out_unlock;
6788 
6789 	if (task_running(p_rq, p) || p->state)
6790 		goto out_unlock;
6791 
6792 	yielded = curr->sched_class->yield_to_task(rq, p);
6793 	if (yielded) {
6794 		schedstat_inc(rq->yld_count);
6795 		/*
6796 		 * Make p's CPU reschedule; pick_next_entity takes care of
6797 		 * fairness.
6798 		 */
6799 		if (preempt && rq != p_rq)
6800 			resched_curr(p_rq);
6801 	}
6802 
6803 out_unlock:
6804 	double_rq_unlock(rq, p_rq);
6805 out_irq:
6806 	local_irq_restore(flags);
6807 
6808 	if (yielded > 0)
6809 		schedule();
6810 
6811 	return yielded;
6812 }
6813 EXPORT_SYMBOL_GPL(yield_to);
6814 
6815 int io_schedule_prepare(void)
6816 {
6817 	int old_iowait = current->in_iowait;
6818 
6819 	current->in_iowait = 1;
6820 	blk_schedule_flush_plug(current);
6821 
6822 	return old_iowait;
6823 }
6824 
6825 void io_schedule_finish(int token)
6826 {
6827 	current->in_iowait = token;
6828 }
6829 
6830 /*
6831  * This task is about to go to sleep on IO. Increment rq->nr_iowait so
6832  * that process accounting knows that this is a task in IO wait state.
6833  */
6834 long __sched io_schedule_timeout(long timeout)
6835 {
6836 	int token;
6837 	long ret;
6838 
6839 	token = io_schedule_prepare();
6840 	ret = schedule_timeout(timeout);
6841 	io_schedule_finish(token);
6842 
6843 	return ret;
6844 }
6845 EXPORT_SYMBOL(io_schedule_timeout);
6846 
6847 void __sched io_schedule(void)
6848 {
6849 	int token;
6850 
6851 	token = io_schedule_prepare();
6852 	schedule();
6853 	io_schedule_finish(token);
6854 }
6855 EXPORT_SYMBOL(io_schedule);
6856 
6857 /**
6858  * sys_sched_get_priority_max - return maximum RT priority.
6859  * @policy: scheduling class.
6860  *
6861  * Return: On success, this syscall returns the maximum
6862  * rt_priority that can be used by a given scheduling class.
6863  * On failure, a negative error code is returned.
6864  */
6865 SYSCALL_DEFINE1(sched_get_priority_max, int, policy)
6866 {
6867 	int ret = -EINVAL;
6868 
6869 	switch (policy) {
6870 	case SCHED_FIFO:
6871 	case SCHED_RR:
6872 		ret = MAX_USER_RT_PRIO-1;
6873 		break;
6874 	case SCHED_DEADLINE:
6875 	case SCHED_NORMAL:
6876 	case SCHED_BATCH:
6877 	case SCHED_IDLE:
6878 		ret = 0;
6879 		break;
6880 	}
6881 	return ret;
6882 }
6883 
6884 /**
6885  * sys_sched_get_priority_min - return minimum RT priority.
6886  * @policy: scheduling class.
6887  *
6888  * Return: On success, this syscall returns the minimum
6889  * rt_priority that can be used by a given scheduling class.
6890  * On failure, a negative error code is returned.
6891  */
6892 SYSCALL_DEFINE1(sched_get_priority_min, int, policy)
6893 {
6894 	int ret = -EINVAL;
6895 
6896 	switch (policy) {
6897 	case SCHED_FIFO:
6898 	case SCHED_RR:
6899 		ret = 1;
6900 		break;
6901 	case SCHED_DEADLINE:
6902 	case SCHED_NORMAL:
6903 	case SCHED_BATCH:
6904 	case SCHED_IDLE:
6905 		ret = 0;
6906 	}
6907 	return ret;
6908 }
6909 
6910 static int sched_rr_get_interval(pid_t pid, struct timespec64 *t)
6911 {
6912 	struct task_struct *p;
6913 	unsigned int time_slice;
6914 	struct rq_flags rf;
6915 	struct rq *rq;
6916 	int retval;
6917 
6918 	if (pid < 0)
6919 		return -EINVAL;
6920 
6921 	retval = -ESRCH;
6922 	rcu_read_lock();
6923 	p = find_process_by_pid(pid);
6924 	if (!p)
6925 		goto out_unlock;
6926 
6927 	retval = security_task_getscheduler(p);
6928 	if (retval)
6929 		goto out_unlock;
6930 
6931 	rq = task_rq_lock(p, &rf);
6932 	time_slice = 0;
6933 	if (p->sched_class->get_rr_interval)
6934 		time_slice = p->sched_class->get_rr_interval(rq, p);
6935 	task_rq_unlock(rq, p, &rf);
6936 
6937 	rcu_read_unlock();
6938 	jiffies_to_timespec64(time_slice, t);
6939 	return 0;
6940 
6941 out_unlock:
6942 	rcu_read_unlock();
6943 	return retval;
6944 }
6945 
6946 /**
6947  * sys_sched_rr_get_interval - return the default timeslice of a process.
6948  * @pid: pid of the process.
6949  * @interval: userspace pointer to the timeslice value.
6950  *
6951  * this syscall writes the default timeslice value of a given process
6952  * into the user-space timespec buffer. A value of '0' means infinity.
6953  *
6954  * Return: On success, 0 and the timeslice is in @interval. Otherwise,
6955  * an error code.
6956  */
6957 SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid,
6958 		struct __kernel_timespec __user *, interval)
6959 {
6960 	struct timespec64 t;
6961 	int retval = sched_rr_get_interval(pid, &t);
6962 
6963 	if (retval == 0)
6964 		retval = put_timespec64(&t, interval);
6965 
6966 	return retval;
6967 }
6968 
6969 #ifdef CONFIG_COMPAT_32BIT_TIME
6970 SYSCALL_DEFINE2(sched_rr_get_interval_time32, pid_t, pid,
6971 		struct old_timespec32 __user *, interval)
6972 {
6973 	struct timespec64 t;
6974 	int retval = sched_rr_get_interval(pid, &t);
6975 
6976 	if (retval == 0)
6977 		retval = put_old_timespec32(&t, interval);
6978 	return retval;
6979 }
6980 #endif
6981 
6982 void sched_show_task(struct task_struct *p)
6983 {
6984 	unsigned long free = 0;
6985 	int ppid;
6986 
6987 	if (!try_get_task_stack(p))
6988 		return;
6989 
6990 	pr_info("task:%-15.15s state:%c", p->comm, task_state_to_char(p));
6991 
6992 	if (p->state == TASK_RUNNING)
6993 		pr_cont("  running task    ");
6994 #ifdef CONFIG_DEBUG_STACK_USAGE
6995 	free = stack_not_used(p);
6996 #endif
6997 	ppid = 0;
6998 	rcu_read_lock();
6999 	if (pid_alive(p))
7000 		ppid = task_pid_nr(rcu_dereference(p->real_parent));
7001 	rcu_read_unlock();
7002 	pr_cont(" stack:%5lu pid:%5d ppid:%6d flags:0x%08lx\n",
7003 		free, task_pid_nr(p), ppid,
7004 		(unsigned long)task_thread_info(p)->flags);
7005 
7006 	print_worker_info(KERN_INFO, p);
7007 	print_stop_info(KERN_INFO, p);
7008 	show_stack(p, NULL, KERN_INFO);
7009 	put_task_stack(p);
7010 }
7011 EXPORT_SYMBOL_GPL(sched_show_task);
7012 
7013 static inline bool
7014 state_filter_match(unsigned long state_filter, struct task_struct *p)
7015 {
7016 	/* no filter, everything matches */
7017 	if (!state_filter)
7018 		return true;
7019 
7020 	/* filter, but doesn't match */
7021 	if (!(p->state & state_filter))
7022 		return false;
7023 
7024 	/*
7025 	 * When looking for TASK_UNINTERRUPTIBLE skip TASK_IDLE (allows
7026 	 * TASK_KILLABLE).
7027 	 */
7028 	if (state_filter == TASK_UNINTERRUPTIBLE && p->state == TASK_IDLE)
7029 		return false;
7030 
7031 	return true;
7032 }
7033 
7034 
7035 void show_state_filter(unsigned long state_filter)
7036 {
7037 	struct task_struct *g, *p;
7038 
7039 	rcu_read_lock();
7040 	for_each_process_thread(g, p) {
7041 		/*
7042 		 * reset the NMI-timeout, listing all files on a slow
7043 		 * console might take a lot of time:
7044 		 * Also, reset softlockup watchdogs on all CPUs, because
7045 		 * another CPU might be blocked waiting for us to process
7046 		 * an IPI.
7047 		 */
7048 		touch_nmi_watchdog();
7049 		touch_all_softlockup_watchdogs();
7050 		if (state_filter_match(state_filter, p))
7051 			sched_show_task(p);
7052 	}
7053 
7054 #ifdef CONFIG_SCHED_DEBUG
7055 	if (!state_filter)
7056 		sysrq_sched_debug_show();
7057 #endif
7058 	rcu_read_unlock();
7059 	/*
7060 	 * Only show locks if all tasks are dumped:
7061 	 */
7062 	if (!state_filter)
7063 		debug_show_all_locks();
7064 }
7065 
7066 /**
7067  * init_idle - set up an idle thread for a given CPU
7068  * @idle: task in question
7069  * @cpu: CPU the idle task belongs to
7070  *
7071  * NOTE: this function does not set the idle thread's NEED_RESCHED
7072  * flag, to make booting more robust.
7073  */
7074 void init_idle(struct task_struct *idle, int cpu)
7075 {
7076 	struct rq *rq = cpu_rq(cpu);
7077 	unsigned long flags;
7078 
7079 	__sched_fork(0, idle);
7080 
7081 	raw_spin_lock_irqsave(&idle->pi_lock, flags);
7082 	raw_spin_lock(&rq->lock);
7083 
7084 	idle->state = TASK_RUNNING;
7085 	idle->se.exec_start = sched_clock();
7086 	idle->flags |= PF_IDLE;
7087 
7088 	scs_task_reset(idle);
7089 	kasan_unpoison_task_stack(idle);
7090 
7091 #ifdef CONFIG_SMP
7092 	/*
7093 	 * It's possible that init_idle() gets called multiple times on a task,
7094 	 * in that case do_set_cpus_allowed() will not do the right thing.
7095 	 *
7096 	 * And since this is boot we can forgo the serialization.
7097 	 */
7098 	set_cpus_allowed_common(idle, cpumask_of(cpu), 0);
7099 #endif
7100 	/*
7101 	 * We're having a chicken and egg problem, even though we are
7102 	 * holding rq->lock, the CPU isn't yet set to this CPU so the
7103 	 * lockdep check in task_group() will fail.
7104 	 *
7105 	 * Similar case to sched_fork(). / Alternatively we could
7106 	 * use task_rq_lock() here and obtain the other rq->lock.
7107 	 *
7108 	 * Silence PROVE_RCU
7109 	 */
7110 	rcu_read_lock();
7111 	__set_task_cpu(idle, cpu);
7112 	rcu_read_unlock();
7113 
7114 	rq->idle = idle;
7115 	rcu_assign_pointer(rq->curr, idle);
7116 	idle->on_rq = TASK_ON_RQ_QUEUED;
7117 #ifdef CONFIG_SMP
7118 	idle->on_cpu = 1;
7119 #endif
7120 	raw_spin_unlock(&rq->lock);
7121 	raw_spin_unlock_irqrestore(&idle->pi_lock, flags);
7122 
7123 	/* Set the preempt count _outside_ the spinlocks! */
7124 	init_idle_preempt_count(idle, cpu);
7125 
7126 	/*
7127 	 * The idle tasks have their own, simple scheduling class:
7128 	 */
7129 	idle->sched_class = &idle_sched_class;
7130 	ftrace_graph_init_idle_task(idle, cpu);
7131 	vtime_init_idle(idle, cpu);
7132 #ifdef CONFIG_SMP
7133 	sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu);
7134 #endif
7135 }
7136 
7137 #ifdef CONFIG_SMP
7138 
7139 int cpuset_cpumask_can_shrink(const struct cpumask *cur,
7140 			      const struct cpumask *trial)
7141 {
7142 	int ret = 1;
7143 
7144 	if (!cpumask_weight(cur))
7145 		return ret;
7146 
7147 	ret = dl_cpuset_cpumask_can_shrink(cur, trial);
7148 
7149 	return ret;
7150 }
7151 
7152 int task_can_attach(struct task_struct *p,
7153 		    const struct cpumask *cs_cpus_allowed)
7154 {
7155 	int ret = 0;
7156 
7157 	/*
7158 	 * Kthreads which disallow setaffinity shouldn't be moved
7159 	 * to a new cpuset; we don't want to change their CPU
7160 	 * affinity and isolating such threads by their set of
7161 	 * allowed nodes is unnecessary.  Thus, cpusets are not
7162 	 * applicable for such threads.  This prevents checking for
7163 	 * success of set_cpus_allowed_ptr() on all attached tasks
7164 	 * before cpus_mask may be changed.
7165 	 */
7166 	if (p->flags & PF_NO_SETAFFINITY) {
7167 		ret = -EINVAL;
7168 		goto out;
7169 	}
7170 
7171 	if (dl_task(p) && !cpumask_intersects(task_rq(p)->rd->span,
7172 					      cs_cpus_allowed))
7173 		ret = dl_task_can_attach(p, cs_cpus_allowed);
7174 
7175 out:
7176 	return ret;
7177 }
7178 
7179 bool sched_smp_initialized __read_mostly;
7180 
7181 #ifdef CONFIG_NUMA_BALANCING
7182 /* Migrate current task p to target_cpu */
7183 int migrate_task_to(struct task_struct *p, int target_cpu)
7184 {
7185 	struct migration_arg arg = { p, target_cpu };
7186 	int curr_cpu = task_cpu(p);
7187 
7188 	if (curr_cpu == target_cpu)
7189 		return 0;
7190 
7191 	if (!cpumask_test_cpu(target_cpu, p->cpus_ptr))
7192 		return -EINVAL;
7193 
7194 	/* TODO: This is not properly updating schedstats */
7195 
7196 	trace_sched_move_numa(p, curr_cpu, target_cpu);
7197 	return stop_one_cpu(curr_cpu, migration_cpu_stop, &arg);
7198 }
7199 
7200 /*
7201  * Requeue a task on a given node and accurately track the number of NUMA
7202  * tasks on the runqueues
7203  */
7204 void sched_setnuma(struct task_struct *p, int nid)
7205 {
7206 	bool queued, running;
7207 	struct rq_flags rf;
7208 	struct rq *rq;
7209 
7210 	rq = task_rq_lock(p, &rf);
7211 	queued = task_on_rq_queued(p);
7212 	running = task_current(rq, p);
7213 
7214 	if (queued)
7215 		dequeue_task(rq, p, DEQUEUE_SAVE);
7216 	if (running)
7217 		put_prev_task(rq, p);
7218 
7219 	p->numa_preferred_nid = nid;
7220 
7221 	if (queued)
7222 		enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK);
7223 	if (running)
7224 		set_next_task(rq, p);
7225 	task_rq_unlock(rq, p, &rf);
7226 }
7227 #endif /* CONFIG_NUMA_BALANCING */
7228 
7229 #ifdef CONFIG_HOTPLUG_CPU
7230 /*
7231  * Ensure that the idle task is using init_mm right before its CPU goes
7232  * offline.
7233  */
7234 void idle_task_exit(void)
7235 {
7236 	struct mm_struct *mm = current->active_mm;
7237 
7238 	BUG_ON(cpu_online(smp_processor_id()));
7239 	BUG_ON(current != this_rq()->idle);
7240 
7241 	if (mm != &init_mm) {
7242 		switch_mm(mm, &init_mm, current);
7243 		finish_arch_post_lock_switch();
7244 	}
7245 
7246 	/* finish_cpu(), as ran on the BP, will clean up the active_mm state */
7247 }
7248 
7249 static int __balance_push_cpu_stop(void *arg)
7250 {
7251 	struct task_struct *p = arg;
7252 	struct rq *rq = this_rq();
7253 	struct rq_flags rf;
7254 	int cpu;
7255 
7256 	raw_spin_lock_irq(&p->pi_lock);
7257 	rq_lock(rq, &rf);
7258 
7259 	update_rq_clock(rq);
7260 
7261 	if (task_rq(p) == rq && task_on_rq_queued(p)) {
7262 		cpu = select_fallback_rq(rq->cpu, p);
7263 		rq = __migrate_task(rq, &rf, p, cpu);
7264 	}
7265 
7266 	rq_unlock(rq, &rf);
7267 	raw_spin_unlock_irq(&p->pi_lock);
7268 
7269 	put_task_struct(p);
7270 
7271 	return 0;
7272 }
7273 
7274 static DEFINE_PER_CPU(struct cpu_stop_work, push_work);
7275 
7276 /*
7277  * Ensure we only run per-cpu kthreads once the CPU goes !active.
7278  */
7279 static void balance_push(struct rq *rq)
7280 {
7281 	struct task_struct *push_task = rq->curr;
7282 
7283 	lockdep_assert_held(&rq->lock);
7284 	SCHED_WARN_ON(rq->cpu != smp_processor_id());
7285 	/*
7286 	 * Ensure the thing is persistent until balance_push_set(.on = false);
7287 	 */
7288 	rq->balance_callback = &balance_push_callback;
7289 
7290 	/*
7291 	 * Both the cpu-hotplug and stop task are in this case and are
7292 	 * required to complete the hotplug process.
7293 	 *
7294 	 * XXX: the idle task does not match kthread_is_per_cpu() due to
7295 	 * histerical raisins.
7296 	 */
7297 	if (rq->idle == push_task ||
7298 	    ((push_task->flags & PF_KTHREAD) && kthread_is_per_cpu(push_task)) ||
7299 	    is_migration_disabled(push_task)) {
7300 
7301 		/*
7302 		 * If this is the idle task on the outgoing CPU try to wake
7303 		 * up the hotplug control thread which might wait for the
7304 		 * last task to vanish. The rcuwait_active() check is
7305 		 * accurate here because the waiter is pinned on this CPU
7306 		 * and can't obviously be running in parallel.
7307 		 *
7308 		 * On RT kernels this also has to check whether there are
7309 		 * pinned and scheduled out tasks on the runqueue. They
7310 		 * need to leave the migrate disabled section first.
7311 		 */
7312 		if (!rq->nr_running && !rq_has_pinned_tasks(rq) &&
7313 		    rcuwait_active(&rq->hotplug_wait)) {
7314 			raw_spin_unlock(&rq->lock);
7315 			rcuwait_wake_up(&rq->hotplug_wait);
7316 			raw_spin_lock(&rq->lock);
7317 		}
7318 		return;
7319 	}
7320 
7321 	get_task_struct(push_task);
7322 	/*
7323 	 * Temporarily drop rq->lock such that we can wake-up the stop task.
7324 	 * Both preemption and IRQs are still disabled.
7325 	 */
7326 	raw_spin_unlock(&rq->lock);
7327 	stop_one_cpu_nowait(rq->cpu, __balance_push_cpu_stop, push_task,
7328 			    this_cpu_ptr(&push_work));
7329 	/*
7330 	 * At this point need_resched() is true and we'll take the loop in
7331 	 * schedule(). The next pick is obviously going to be the stop task
7332 	 * which kthread_is_per_cpu() and will push this task away.
7333 	 */
7334 	raw_spin_lock(&rq->lock);
7335 }
7336 
7337 static void balance_push_set(int cpu, bool on)
7338 {
7339 	struct rq *rq = cpu_rq(cpu);
7340 	struct rq_flags rf;
7341 
7342 	rq_lock_irqsave(rq, &rf);
7343 	rq->balance_push = on;
7344 	if (on) {
7345 		WARN_ON_ONCE(rq->balance_callback);
7346 		rq->balance_callback = &balance_push_callback;
7347 	} else if (rq->balance_callback == &balance_push_callback) {
7348 		rq->balance_callback = NULL;
7349 	}
7350 	rq_unlock_irqrestore(rq, &rf);
7351 }
7352 
7353 /*
7354  * Invoked from a CPUs hotplug control thread after the CPU has been marked
7355  * inactive. All tasks which are not per CPU kernel threads are either
7356  * pushed off this CPU now via balance_push() or placed on a different CPU
7357  * during wakeup. Wait until the CPU is quiescent.
7358  */
7359 static void balance_hotplug_wait(void)
7360 {
7361 	struct rq *rq = this_rq();
7362 
7363 	rcuwait_wait_event(&rq->hotplug_wait,
7364 			   rq->nr_running == 1 && !rq_has_pinned_tasks(rq),
7365 			   TASK_UNINTERRUPTIBLE);
7366 }
7367 
7368 #else
7369 
7370 static inline void balance_push(struct rq *rq)
7371 {
7372 }
7373 
7374 static inline void balance_push_set(int cpu, bool on)
7375 {
7376 }
7377 
7378 static inline void balance_hotplug_wait(void)
7379 {
7380 }
7381 
7382 #endif /* CONFIG_HOTPLUG_CPU */
7383 
7384 void set_rq_online(struct rq *rq)
7385 {
7386 	if (!rq->online) {
7387 		const struct sched_class *class;
7388 
7389 		cpumask_set_cpu(rq->cpu, rq->rd->online);
7390 		rq->online = 1;
7391 
7392 		for_each_class(class) {
7393 			if (class->rq_online)
7394 				class->rq_online(rq);
7395 		}
7396 	}
7397 }
7398 
7399 void set_rq_offline(struct rq *rq)
7400 {
7401 	if (rq->online) {
7402 		const struct sched_class *class;
7403 
7404 		for_each_class(class) {
7405 			if (class->rq_offline)
7406 				class->rq_offline(rq);
7407 		}
7408 
7409 		cpumask_clear_cpu(rq->cpu, rq->rd->online);
7410 		rq->online = 0;
7411 	}
7412 }
7413 
7414 /*
7415  * used to mark begin/end of suspend/resume:
7416  */
7417 static int num_cpus_frozen;
7418 
7419 /*
7420  * Update cpusets according to cpu_active mask.  If cpusets are
7421  * disabled, cpuset_update_active_cpus() becomes a simple wrapper
7422  * around partition_sched_domains().
7423  *
7424  * If we come here as part of a suspend/resume, don't touch cpusets because we
7425  * want to restore it back to its original state upon resume anyway.
7426  */
7427 static void cpuset_cpu_active(void)
7428 {
7429 	if (cpuhp_tasks_frozen) {
7430 		/*
7431 		 * num_cpus_frozen tracks how many CPUs are involved in suspend
7432 		 * resume sequence. As long as this is not the last online
7433 		 * operation in the resume sequence, just build a single sched
7434 		 * domain, ignoring cpusets.
7435 		 */
7436 		partition_sched_domains(1, NULL, NULL);
7437 		if (--num_cpus_frozen)
7438 			return;
7439 		/*
7440 		 * This is the last CPU online operation. So fall through and
7441 		 * restore the original sched domains by considering the
7442 		 * cpuset configurations.
7443 		 */
7444 		cpuset_force_rebuild();
7445 	}
7446 	cpuset_update_active_cpus();
7447 }
7448 
7449 static int cpuset_cpu_inactive(unsigned int cpu)
7450 {
7451 	if (!cpuhp_tasks_frozen) {
7452 		if (dl_cpu_busy(cpu))
7453 			return -EBUSY;
7454 		cpuset_update_active_cpus();
7455 	} else {
7456 		num_cpus_frozen++;
7457 		partition_sched_domains(1, NULL, NULL);
7458 	}
7459 	return 0;
7460 }
7461 
7462 int sched_cpu_activate(unsigned int cpu)
7463 {
7464 	struct rq *rq = cpu_rq(cpu);
7465 	struct rq_flags rf;
7466 
7467 	/*
7468 	 * Make sure that when the hotplug state machine does a roll-back
7469 	 * we clear balance_push. Ideally that would happen earlier...
7470 	 */
7471 	balance_push_set(cpu, false);
7472 
7473 #ifdef CONFIG_SCHED_SMT
7474 	/*
7475 	 * When going up, increment the number of cores with SMT present.
7476 	 */
7477 	if (cpumask_weight(cpu_smt_mask(cpu)) == 2)
7478 		static_branch_inc_cpuslocked(&sched_smt_present);
7479 #endif
7480 	set_cpu_active(cpu, true);
7481 
7482 	if (sched_smp_initialized) {
7483 		sched_domains_numa_masks_set(cpu);
7484 		cpuset_cpu_active();
7485 	}
7486 
7487 	/*
7488 	 * Put the rq online, if not already. This happens:
7489 	 *
7490 	 * 1) In the early boot process, because we build the real domains
7491 	 *    after all CPUs have been brought up.
7492 	 *
7493 	 * 2) At runtime, if cpuset_cpu_active() fails to rebuild the
7494 	 *    domains.
7495 	 */
7496 	rq_lock_irqsave(rq, &rf);
7497 	if (rq->rd) {
7498 		BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
7499 		set_rq_online(rq);
7500 	}
7501 	rq_unlock_irqrestore(rq, &rf);
7502 
7503 	return 0;
7504 }
7505 
7506 int sched_cpu_deactivate(unsigned int cpu)
7507 {
7508 	struct rq *rq = cpu_rq(cpu);
7509 	struct rq_flags rf;
7510 	int ret;
7511 
7512 	set_cpu_active(cpu, false);
7513 
7514 	/*
7515 	 * From this point forward, this CPU will refuse to run any task that
7516 	 * is not: migrate_disable() or KTHREAD_IS_PER_CPU, and will actively
7517 	 * push those tasks away until this gets cleared, see
7518 	 * sched_cpu_dying().
7519 	 */
7520 	balance_push_set(cpu, true);
7521 
7522 	/*
7523 	 * We've cleared cpu_active_mask / set balance_push, wait for all
7524 	 * preempt-disabled and RCU users of this state to go away such that
7525 	 * all new such users will observe it.
7526 	 *
7527 	 * Specifically, we rely on ttwu to no longer target this CPU, see
7528 	 * ttwu_queue_cond() and is_cpu_allowed().
7529 	 *
7530 	 * Do sync before park smpboot threads to take care the rcu boost case.
7531 	 */
7532 	synchronize_rcu();
7533 
7534 	rq_lock_irqsave(rq, &rf);
7535 	if (rq->rd) {
7536 		update_rq_clock(rq);
7537 		BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
7538 		set_rq_offline(rq);
7539 	}
7540 	rq_unlock_irqrestore(rq, &rf);
7541 
7542 #ifdef CONFIG_SCHED_SMT
7543 	/*
7544 	 * When going down, decrement the number of cores with SMT present.
7545 	 */
7546 	if (cpumask_weight(cpu_smt_mask(cpu)) == 2)
7547 		static_branch_dec_cpuslocked(&sched_smt_present);
7548 #endif
7549 
7550 	if (!sched_smp_initialized)
7551 		return 0;
7552 
7553 	ret = cpuset_cpu_inactive(cpu);
7554 	if (ret) {
7555 		balance_push_set(cpu, false);
7556 		set_cpu_active(cpu, true);
7557 		return ret;
7558 	}
7559 	sched_domains_numa_masks_clear(cpu);
7560 	return 0;
7561 }
7562 
7563 static void sched_rq_cpu_starting(unsigned int cpu)
7564 {
7565 	struct rq *rq = cpu_rq(cpu);
7566 
7567 	rq->calc_load_update = calc_load_update;
7568 	update_max_interval();
7569 }
7570 
7571 int sched_cpu_starting(unsigned int cpu)
7572 {
7573 	sched_rq_cpu_starting(cpu);
7574 	sched_tick_start(cpu);
7575 	return 0;
7576 }
7577 
7578 #ifdef CONFIG_HOTPLUG_CPU
7579 
7580 /*
7581  * Invoked immediately before the stopper thread is invoked to bring the
7582  * CPU down completely. At this point all per CPU kthreads except the
7583  * hotplug thread (current) and the stopper thread (inactive) have been
7584  * either parked or have been unbound from the outgoing CPU. Ensure that
7585  * any of those which might be on the way out are gone.
7586  *
7587  * If after this point a bound task is being woken on this CPU then the
7588  * responsible hotplug callback has failed to do it's job.
7589  * sched_cpu_dying() will catch it with the appropriate fireworks.
7590  */
7591 int sched_cpu_wait_empty(unsigned int cpu)
7592 {
7593 	balance_hotplug_wait();
7594 	return 0;
7595 }
7596 
7597 /*
7598  * Since this CPU is going 'away' for a while, fold any nr_active delta we
7599  * might have. Called from the CPU stopper task after ensuring that the
7600  * stopper is the last running task on the CPU, so nr_active count is
7601  * stable. We need to take the teardown thread which is calling this into
7602  * account, so we hand in adjust = 1 to the load calculation.
7603  *
7604  * Also see the comment "Global load-average calculations".
7605  */
7606 static void calc_load_migrate(struct rq *rq)
7607 {
7608 	long delta = calc_load_fold_active(rq, 1);
7609 
7610 	if (delta)
7611 		atomic_long_add(delta, &calc_load_tasks);
7612 }
7613 
7614 static void dump_rq_tasks(struct rq *rq, const char *loglvl)
7615 {
7616 	struct task_struct *g, *p;
7617 	int cpu = cpu_of(rq);
7618 
7619 	lockdep_assert_held(&rq->lock);
7620 
7621 	printk("%sCPU%d enqueued tasks (%u total):\n", loglvl, cpu, rq->nr_running);
7622 	for_each_process_thread(g, p) {
7623 		if (task_cpu(p) != cpu)
7624 			continue;
7625 
7626 		if (!task_on_rq_queued(p))
7627 			continue;
7628 
7629 		printk("%s\tpid: %d, name: %s\n", loglvl, p->pid, p->comm);
7630 	}
7631 }
7632 
7633 int sched_cpu_dying(unsigned int cpu)
7634 {
7635 	struct rq *rq = cpu_rq(cpu);
7636 	struct rq_flags rf;
7637 
7638 	/* Handle pending wakeups and then migrate everything off */
7639 	sched_tick_stop(cpu);
7640 
7641 	rq_lock_irqsave(rq, &rf);
7642 	if (rq->nr_running != 1 || rq_has_pinned_tasks(rq)) {
7643 		WARN(true, "Dying CPU not properly vacated!");
7644 		dump_rq_tasks(rq, KERN_WARNING);
7645 	}
7646 	rq_unlock_irqrestore(rq, &rf);
7647 
7648 	/*
7649 	 * Now that the CPU is offline, make sure we're welcome
7650 	 * to new tasks once we come back up.
7651 	 */
7652 	balance_push_set(cpu, false);
7653 
7654 	calc_load_migrate(rq);
7655 	update_max_interval();
7656 	nohz_balance_exit_idle(rq);
7657 	hrtick_clear(rq);
7658 	return 0;
7659 }
7660 #endif
7661 
7662 void __init sched_init_smp(void)
7663 {
7664 	sched_init_numa();
7665 
7666 	/*
7667 	 * There's no userspace yet to cause hotplug operations; hence all the
7668 	 * CPU masks are stable and all blatant races in the below code cannot
7669 	 * happen.
7670 	 */
7671 	mutex_lock(&sched_domains_mutex);
7672 	sched_init_domains(cpu_active_mask);
7673 	mutex_unlock(&sched_domains_mutex);
7674 
7675 	/* Move init over to a non-isolated CPU */
7676 	if (set_cpus_allowed_ptr(current, housekeeping_cpumask(HK_FLAG_DOMAIN)) < 0)
7677 		BUG();
7678 	sched_init_granularity();
7679 
7680 	init_sched_rt_class();
7681 	init_sched_dl_class();
7682 
7683 	sched_smp_initialized = true;
7684 }
7685 
7686 static int __init migration_init(void)
7687 {
7688 	sched_cpu_starting(smp_processor_id());
7689 	return 0;
7690 }
7691 early_initcall(migration_init);
7692 
7693 #else
7694 void __init sched_init_smp(void)
7695 {
7696 	sched_init_granularity();
7697 }
7698 #endif /* CONFIG_SMP */
7699 
7700 int in_sched_functions(unsigned long addr)
7701 {
7702 	return in_lock_functions(addr) ||
7703 		(addr >= (unsigned long)__sched_text_start
7704 		&& addr < (unsigned long)__sched_text_end);
7705 }
7706 
7707 #ifdef CONFIG_CGROUP_SCHED
7708 /*
7709  * Default task group.
7710  * Every task in system belongs to this group at bootup.
7711  */
7712 struct task_group root_task_group;
7713 LIST_HEAD(task_groups);
7714 
7715 /* Cacheline aligned slab cache for task_group */
7716 static struct kmem_cache *task_group_cache __read_mostly;
7717 #endif
7718 
7719 DECLARE_PER_CPU(cpumask_var_t, load_balance_mask);
7720 DECLARE_PER_CPU(cpumask_var_t, select_idle_mask);
7721 
7722 void __init sched_init(void)
7723 {
7724 	unsigned long ptr = 0;
7725 	int i;
7726 
7727 	/* Make sure the linker didn't screw up */
7728 	BUG_ON(&idle_sched_class + 1 != &fair_sched_class ||
7729 	       &fair_sched_class + 1 != &rt_sched_class ||
7730 	       &rt_sched_class + 1   != &dl_sched_class);
7731 #ifdef CONFIG_SMP
7732 	BUG_ON(&dl_sched_class + 1 != &stop_sched_class);
7733 #endif
7734 
7735 	wait_bit_init();
7736 
7737 #ifdef CONFIG_FAIR_GROUP_SCHED
7738 	ptr += 2 * nr_cpu_ids * sizeof(void **);
7739 #endif
7740 #ifdef CONFIG_RT_GROUP_SCHED
7741 	ptr += 2 * nr_cpu_ids * sizeof(void **);
7742 #endif
7743 	if (ptr) {
7744 		ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT);
7745 
7746 #ifdef CONFIG_FAIR_GROUP_SCHED
7747 		root_task_group.se = (struct sched_entity **)ptr;
7748 		ptr += nr_cpu_ids * sizeof(void **);
7749 
7750 		root_task_group.cfs_rq = (struct cfs_rq **)ptr;
7751 		ptr += nr_cpu_ids * sizeof(void **);
7752 
7753 		root_task_group.shares = ROOT_TASK_GROUP_LOAD;
7754 		init_cfs_bandwidth(&root_task_group.cfs_bandwidth);
7755 #endif /* CONFIG_FAIR_GROUP_SCHED */
7756 #ifdef CONFIG_RT_GROUP_SCHED
7757 		root_task_group.rt_se = (struct sched_rt_entity **)ptr;
7758 		ptr += nr_cpu_ids * sizeof(void **);
7759 
7760 		root_task_group.rt_rq = (struct rt_rq **)ptr;
7761 		ptr += nr_cpu_ids * sizeof(void **);
7762 
7763 #endif /* CONFIG_RT_GROUP_SCHED */
7764 	}
7765 #ifdef CONFIG_CPUMASK_OFFSTACK
7766 	for_each_possible_cpu(i) {
7767 		per_cpu(load_balance_mask, i) = (cpumask_var_t)kzalloc_node(
7768 			cpumask_size(), GFP_KERNEL, cpu_to_node(i));
7769 		per_cpu(select_idle_mask, i) = (cpumask_var_t)kzalloc_node(
7770 			cpumask_size(), GFP_KERNEL, cpu_to_node(i));
7771 	}
7772 #endif /* CONFIG_CPUMASK_OFFSTACK */
7773 
7774 	init_rt_bandwidth(&def_rt_bandwidth, global_rt_period(), global_rt_runtime());
7775 	init_dl_bandwidth(&def_dl_bandwidth, global_rt_period(), global_rt_runtime());
7776 
7777 #ifdef CONFIG_SMP
7778 	init_defrootdomain();
7779 #endif
7780 
7781 #ifdef CONFIG_RT_GROUP_SCHED
7782 	init_rt_bandwidth(&root_task_group.rt_bandwidth,
7783 			global_rt_period(), global_rt_runtime());
7784 #endif /* CONFIG_RT_GROUP_SCHED */
7785 
7786 #ifdef CONFIG_CGROUP_SCHED
7787 	task_group_cache = KMEM_CACHE(task_group, 0);
7788 
7789 	list_add(&root_task_group.list, &task_groups);
7790 	INIT_LIST_HEAD(&root_task_group.children);
7791 	INIT_LIST_HEAD(&root_task_group.siblings);
7792 	autogroup_init(&init_task);
7793 #endif /* CONFIG_CGROUP_SCHED */
7794 
7795 	for_each_possible_cpu(i) {
7796 		struct rq *rq;
7797 
7798 		rq = cpu_rq(i);
7799 		raw_spin_lock_init(&rq->lock);
7800 		rq->nr_running = 0;
7801 		rq->calc_load_active = 0;
7802 		rq->calc_load_update = jiffies + LOAD_FREQ;
7803 		init_cfs_rq(&rq->cfs);
7804 		init_rt_rq(&rq->rt);
7805 		init_dl_rq(&rq->dl);
7806 #ifdef CONFIG_FAIR_GROUP_SCHED
7807 		INIT_LIST_HEAD(&rq->leaf_cfs_rq_list);
7808 		rq->tmp_alone_branch = &rq->leaf_cfs_rq_list;
7809 		/*
7810 		 * How much CPU bandwidth does root_task_group get?
7811 		 *
7812 		 * In case of task-groups formed thr' the cgroup filesystem, it
7813 		 * gets 100% of the CPU resources in the system. This overall
7814 		 * system CPU resource is divided among the tasks of
7815 		 * root_task_group and its child task-groups in a fair manner,
7816 		 * based on each entity's (task or task-group's) weight
7817 		 * (se->load.weight).
7818 		 *
7819 		 * In other words, if root_task_group has 10 tasks of weight
7820 		 * 1024) and two child groups A0 and A1 (of weight 1024 each),
7821 		 * then A0's share of the CPU resource is:
7822 		 *
7823 		 *	A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33%
7824 		 *
7825 		 * We achieve this by letting root_task_group's tasks sit
7826 		 * directly in rq->cfs (i.e root_task_group->se[] = NULL).
7827 		 */
7828 		init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL);
7829 #endif /* CONFIG_FAIR_GROUP_SCHED */
7830 
7831 		rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime;
7832 #ifdef CONFIG_RT_GROUP_SCHED
7833 		init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL);
7834 #endif
7835 #ifdef CONFIG_SMP
7836 		rq->sd = NULL;
7837 		rq->rd = NULL;
7838 		rq->cpu_capacity = rq->cpu_capacity_orig = SCHED_CAPACITY_SCALE;
7839 		rq->balance_callback = NULL;
7840 		rq->active_balance = 0;
7841 		rq->next_balance = jiffies;
7842 		rq->push_cpu = 0;
7843 		rq->cpu = i;
7844 		rq->online = 0;
7845 		rq->idle_stamp = 0;
7846 		rq->avg_idle = 2*sysctl_sched_migration_cost;
7847 		rq->max_idle_balance_cost = sysctl_sched_migration_cost;
7848 
7849 		INIT_LIST_HEAD(&rq->cfs_tasks);
7850 
7851 		rq_attach_root(rq, &def_root_domain);
7852 #ifdef CONFIG_NO_HZ_COMMON
7853 		rq->last_blocked_load_update_tick = jiffies;
7854 		atomic_set(&rq->nohz_flags, 0);
7855 
7856 		INIT_CSD(&rq->nohz_csd, nohz_csd_func, rq);
7857 #endif
7858 #ifdef CONFIG_HOTPLUG_CPU
7859 		rcuwait_init(&rq->hotplug_wait);
7860 #endif
7861 #endif /* CONFIG_SMP */
7862 		hrtick_rq_init(rq);
7863 		atomic_set(&rq->nr_iowait, 0);
7864 	}
7865 
7866 	set_load_weight(&init_task, false);
7867 
7868 	/*
7869 	 * The boot idle thread does lazy MMU switching as well:
7870 	 */
7871 	mmgrab(&init_mm);
7872 	enter_lazy_tlb(&init_mm, current);
7873 
7874 	/*
7875 	 * Make us the idle thread. Technically, schedule() should not be
7876 	 * called from this thread, however somewhere below it might be,
7877 	 * but because we are the idle thread, we just pick up running again
7878 	 * when this runqueue becomes "idle".
7879 	 */
7880 	init_idle(current, smp_processor_id());
7881 
7882 	calc_load_update = jiffies + LOAD_FREQ;
7883 
7884 #ifdef CONFIG_SMP
7885 	idle_thread_set_boot_cpu();
7886 #endif
7887 	init_sched_fair_class();
7888 
7889 	init_schedstats();
7890 
7891 	psi_init();
7892 
7893 	init_uclamp();
7894 
7895 	scheduler_running = 1;
7896 }
7897 
7898 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
7899 static inline int preempt_count_equals(int preempt_offset)
7900 {
7901 	int nested = preempt_count() + rcu_preempt_depth();
7902 
7903 	return (nested == preempt_offset);
7904 }
7905 
7906 void __might_sleep(const char *file, int line, int preempt_offset)
7907 {
7908 	/*
7909 	 * Blocking primitives will set (and therefore destroy) current->state,
7910 	 * since we will exit with TASK_RUNNING make sure we enter with it,
7911 	 * otherwise we will destroy state.
7912 	 */
7913 	WARN_ONCE(current->state != TASK_RUNNING && current->task_state_change,
7914 			"do not call blocking ops when !TASK_RUNNING; "
7915 			"state=%lx set at [<%p>] %pS\n",
7916 			current->state,
7917 			(void *)current->task_state_change,
7918 			(void *)current->task_state_change);
7919 
7920 	___might_sleep(file, line, preempt_offset);
7921 }
7922 EXPORT_SYMBOL(__might_sleep);
7923 
7924 void ___might_sleep(const char *file, int line, int preempt_offset)
7925 {
7926 	/* Ratelimiting timestamp: */
7927 	static unsigned long prev_jiffy;
7928 
7929 	unsigned long preempt_disable_ip;
7930 
7931 	/* WARN_ON_ONCE() by default, no rate limit required: */
7932 	rcu_sleep_check();
7933 
7934 	if ((preempt_count_equals(preempt_offset) && !irqs_disabled() &&
7935 	     !is_idle_task(current) && !current->non_block_count) ||
7936 	    system_state == SYSTEM_BOOTING || system_state > SYSTEM_RUNNING ||
7937 	    oops_in_progress)
7938 		return;
7939 
7940 	if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
7941 		return;
7942 	prev_jiffy = jiffies;
7943 
7944 	/* Save this before calling printk(), since that will clobber it: */
7945 	preempt_disable_ip = get_preempt_disable_ip(current);
7946 
7947 	printk(KERN_ERR
7948 		"BUG: sleeping function called from invalid context at %s:%d\n",
7949 			file, line);
7950 	printk(KERN_ERR
7951 		"in_atomic(): %d, irqs_disabled(): %d, non_block: %d, pid: %d, name: %s\n",
7952 			in_atomic(), irqs_disabled(), current->non_block_count,
7953 			current->pid, current->comm);
7954 
7955 	if (task_stack_end_corrupted(current))
7956 		printk(KERN_EMERG "Thread overran stack, or stack corrupted\n");
7957 
7958 	debug_show_held_locks(current);
7959 	if (irqs_disabled())
7960 		print_irqtrace_events(current);
7961 	if (IS_ENABLED(CONFIG_DEBUG_PREEMPT)
7962 	    && !preempt_count_equals(preempt_offset)) {
7963 		pr_err("Preemption disabled at:");
7964 		print_ip_sym(KERN_ERR, preempt_disable_ip);
7965 	}
7966 	dump_stack();
7967 	add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
7968 }
7969 EXPORT_SYMBOL(___might_sleep);
7970 
7971 void __cant_sleep(const char *file, int line, int preempt_offset)
7972 {
7973 	static unsigned long prev_jiffy;
7974 
7975 	if (irqs_disabled())
7976 		return;
7977 
7978 	if (!IS_ENABLED(CONFIG_PREEMPT_COUNT))
7979 		return;
7980 
7981 	if (preempt_count() > preempt_offset)
7982 		return;
7983 
7984 	if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
7985 		return;
7986 	prev_jiffy = jiffies;
7987 
7988 	printk(KERN_ERR "BUG: assuming atomic context at %s:%d\n", file, line);
7989 	printk(KERN_ERR "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n",
7990 			in_atomic(), irqs_disabled(),
7991 			current->pid, current->comm);
7992 
7993 	debug_show_held_locks(current);
7994 	dump_stack();
7995 	add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
7996 }
7997 EXPORT_SYMBOL_GPL(__cant_sleep);
7998 
7999 #ifdef CONFIG_SMP
8000 void __cant_migrate(const char *file, int line)
8001 {
8002 	static unsigned long prev_jiffy;
8003 
8004 	if (irqs_disabled())
8005 		return;
8006 
8007 	if (is_migration_disabled(current))
8008 		return;
8009 
8010 	if (!IS_ENABLED(CONFIG_PREEMPT_COUNT))
8011 		return;
8012 
8013 	if (preempt_count() > 0)
8014 		return;
8015 
8016 	if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
8017 		return;
8018 	prev_jiffy = jiffies;
8019 
8020 	pr_err("BUG: assuming non migratable context at %s:%d\n", file, line);
8021 	pr_err("in_atomic(): %d, irqs_disabled(): %d, migration_disabled() %u pid: %d, name: %s\n",
8022 	       in_atomic(), irqs_disabled(), is_migration_disabled(current),
8023 	       current->pid, current->comm);
8024 
8025 	debug_show_held_locks(current);
8026 	dump_stack();
8027 	add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
8028 }
8029 EXPORT_SYMBOL_GPL(__cant_migrate);
8030 #endif
8031 #endif
8032 
8033 #ifdef CONFIG_MAGIC_SYSRQ
8034 void normalize_rt_tasks(void)
8035 {
8036 	struct task_struct *g, *p;
8037 	struct sched_attr attr = {
8038 		.sched_policy = SCHED_NORMAL,
8039 	};
8040 
8041 	read_lock(&tasklist_lock);
8042 	for_each_process_thread(g, p) {
8043 		/*
8044 		 * Only normalize user tasks:
8045 		 */
8046 		if (p->flags & PF_KTHREAD)
8047 			continue;
8048 
8049 		p->se.exec_start = 0;
8050 		schedstat_set(p->se.statistics.wait_start,  0);
8051 		schedstat_set(p->se.statistics.sleep_start, 0);
8052 		schedstat_set(p->se.statistics.block_start, 0);
8053 
8054 		if (!dl_task(p) && !rt_task(p)) {
8055 			/*
8056 			 * Renice negative nice level userspace
8057 			 * tasks back to 0:
8058 			 */
8059 			if (task_nice(p) < 0)
8060 				set_user_nice(p, 0);
8061 			continue;
8062 		}
8063 
8064 		__sched_setscheduler(p, &attr, false, false);
8065 	}
8066 	read_unlock(&tasklist_lock);
8067 }
8068 
8069 #endif /* CONFIG_MAGIC_SYSRQ */
8070 
8071 #if defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB)
8072 /*
8073  * These functions are only useful for the IA64 MCA handling, or kdb.
8074  *
8075  * They can only be called when the whole system has been
8076  * stopped - every CPU needs to be quiescent, and no scheduling
8077  * activity can take place. Using them for anything else would
8078  * be a serious bug, and as a result, they aren't even visible
8079  * under any other configuration.
8080  */
8081 
8082 /**
8083  * curr_task - return the current task for a given CPU.
8084  * @cpu: the processor in question.
8085  *
8086  * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
8087  *
8088  * Return: The current task for @cpu.
8089  */
8090 struct task_struct *curr_task(int cpu)
8091 {
8092 	return cpu_curr(cpu);
8093 }
8094 
8095 #endif /* defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) */
8096 
8097 #ifdef CONFIG_IA64
8098 /**
8099  * ia64_set_curr_task - set the current task for a given CPU.
8100  * @cpu: the processor in question.
8101  * @p: the task pointer to set.
8102  *
8103  * Description: This function must only be used when non-maskable interrupts
8104  * are serviced on a separate stack. It allows the architecture to switch the
8105  * notion of the current task on a CPU in a non-blocking manner. This function
8106  * must be called with all CPU's synchronized, and interrupts disabled, the
8107  * and caller must save the original value of the current task (see
8108  * curr_task() above) and restore that value before reenabling interrupts and
8109  * re-starting the system.
8110  *
8111  * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
8112  */
8113 void ia64_set_curr_task(int cpu, struct task_struct *p)
8114 {
8115 	cpu_curr(cpu) = p;
8116 }
8117 
8118 #endif
8119 
8120 #ifdef CONFIG_CGROUP_SCHED
8121 /* task_group_lock serializes the addition/removal of task groups */
8122 static DEFINE_SPINLOCK(task_group_lock);
8123 
8124 static inline void alloc_uclamp_sched_group(struct task_group *tg,
8125 					    struct task_group *parent)
8126 {
8127 #ifdef CONFIG_UCLAMP_TASK_GROUP
8128 	enum uclamp_id clamp_id;
8129 
8130 	for_each_clamp_id(clamp_id) {
8131 		uclamp_se_set(&tg->uclamp_req[clamp_id],
8132 			      uclamp_none(clamp_id), false);
8133 		tg->uclamp[clamp_id] = parent->uclamp[clamp_id];
8134 	}
8135 #endif
8136 }
8137 
8138 static void sched_free_group(struct task_group *tg)
8139 {
8140 	free_fair_sched_group(tg);
8141 	free_rt_sched_group(tg);
8142 	autogroup_free(tg);
8143 	kmem_cache_free(task_group_cache, tg);
8144 }
8145 
8146 /* allocate runqueue etc for a new task group */
8147 struct task_group *sched_create_group(struct task_group *parent)
8148 {
8149 	struct task_group *tg;
8150 
8151 	tg = kmem_cache_alloc(task_group_cache, GFP_KERNEL | __GFP_ZERO);
8152 	if (!tg)
8153 		return ERR_PTR(-ENOMEM);
8154 
8155 	if (!alloc_fair_sched_group(tg, parent))
8156 		goto err;
8157 
8158 	if (!alloc_rt_sched_group(tg, parent))
8159 		goto err;
8160 
8161 	alloc_uclamp_sched_group(tg, parent);
8162 
8163 	return tg;
8164 
8165 err:
8166 	sched_free_group(tg);
8167 	return ERR_PTR(-ENOMEM);
8168 }
8169 
8170 void sched_online_group(struct task_group *tg, struct task_group *parent)
8171 {
8172 	unsigned long flags;
8173 
8174 	spin_lock_irqsave(&task_group_lock, flags);
8175 	list_add_rcu(&tg->list, &task_groups);
8176 
8177 	/* Root should already exist: */
8178 	WARN_ON(!parent);
8179 
8180 	tg->parent = parent;
8181 	INIT_LIST_HEAD(&tg->children);
8182 	list_add_rcu(&tg->siblings, &parent->children);
8183 	spin_unlock_irqrestore(&task_group_lock, flags);
8184 
8185 	online_fair_sched_group(tg);
8186 }
8187 
8188 /* rcu callback to free various structures associated with a task group */
8189 static void sched_free_group_rcu(struct rcu_head *rhp)
8190 {
8191 	/* Now it should be safe to free those cfs_rqs: */
8192 	sched_free_group(container_of(rhp, struct task_group, rcu));
8193 }
8194 
8195 void sched_destroy_group(struct task_group *tg)
8196 {
8197 	/* Wait for possible concurrent references to cfs_rqs complete: */
8198 	call_rcu(&tg->rcu, sched_free_group_rcu);
8199 }
8200 
8201 void sched_offline_group(struct task_group *tg)
8202 {
8203 	unsigned long flags;
8204 
8205 	/* End participation in shares distribution: */
8206 	unregister_fair_sched_group(tg);
8207 
8208 	spin_lock_irqsave(&task_group_lock, flags);
8209 	list_del_rcu(&tg->list);
8210 	list_del_rcu(&tg->siblings);
8211 	spin_unlock_irqrestore(&task_group_lock, flags);
8212 }
8213 
8214 static void sched_change_group(struct task_struct *tsk, int type)
8215 {
8216 	struct task_group *tg;
8217 
8218 	/*
8219 	 * All callers are synchronized by task_rq_lock(); we do not use RCU
8220 	 * which is pointless here. Thus, we pass "true" to task_css_check()
8221 	 * to prevent lockdep warnings.
8222 	 */
8223 	tg = container_of(task_css_check(tsk, cpu_cgrp_id, true),
8224 			  struct task_group, css);
8225 	tg = autogroup_task_group(tsk, tg);
8226 	tsk->sched_task_group = tg;
8227 
8228 #ifdef CONFIG_FAIR_GROUP_SCHED
8229 	if (tsk->sched_class->task_change_group)
8230 		tsk->sched_class->task_change_group(tsk, type);
8231 	else
8232 #endif
8233 		set_task_rq(tsk, task_cpu(tsk));
8234 }
8235 
8236 /*
8237  * Change task's runqueue when it moves between groups.
8238  *
8239  * The caller of this function should have put the task in its new group by
8240  * now. This function just updates tsk->se.cfs_rq and tsk->se.parent to reflect
8241  * its new group.
8242  */
8243 void sched_move_task(struct task_struct *tsk)
8244 {
8245 	int queued, running, queue_flags =
8246 		DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK;
8247 	struct rq_flags rf;
8248 	struct rq *rq;
8249 
8250 	rq = task_rq_lock(tsk, &rf);
8251 	update_rq_clock(rq);
8252 
8253 	running = task_current(rq, tsk);
8254 	queued = task_on_rq_queued(tsk);
8255 
8256 	if (queued)
8257 		dequeue_task(rq, tsk, queue_flags);
8258 	if (running)
8259 		put_prev_task(rq, tsk);
8260 
8261 	sched_change_group(tsk, TASK_MOVE_GROUP);
8262 
8263 	if (queued)
8264 		enqueue_task(rq, tsk, queue_flags);
8265 	if (running) {
8266 		set_next_task(rq, tsk);
8267 		/*
8268 		 * After changing group, the running task may have joined a
8269 		 * throttled one but it's still the running task. Trigger a
8270 		 * resched to make sure that task can still run.
8271 		 */
8272 		resched_curr(rq);
8273 	}
8274 
8275 	task_rq_unlock(rq, tsk, &rf);
8276 }
8277 
8278 static inline struct task_group *css_tg(struct cgroup_subsys_state *css)
8279 {
8280 	return css ? container_of(css, struct task_group, css) : NULL;
8281 }
8282 
8283 static struct cgroup_subsys_state *
8284 cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
8285 {
8286 	struct task_group *parent = css_tg(parent_css);
8287 	struct task_group *tg;
8288 
8289 	if (!parent) {
8290 		/* This is early initialization for the top cgroup */
8291 		return &root_task_group.css;
8292 	}
8293 
8294 	tg = sched_create_group(parent);
8295 	if (IS_ERR(tg))
8296 		return ERR_PTR(-ENOMEM);
8297 
8298 	return &tg->css;
8299 }
8300 
8301 /* Expose task group only after completing cgroup initialization */
8302 static int cpu_cgroup_css_online(struct cgroup_subsys_state *css)
8303 {
8304 	struct task_group *tg = css_tg(css);
8305 	struct task_group *parent = css_tg(css->parent);
8306 
8307 	if (parent)
8308 		sched_online_group(tg, parent);
8309 
8310 #ifdef CONFIG_UCLAMP_TASK_GROUP
8311 	/* Propagate the effective uclamp value for the new group */
8312 	cpu_util_update_eff(css);
8313 #endif
8314 
8315 	return 0;
8316 }
8317 
8318 static void cpu_cgroup_css_released(struct cgroup_subsys_state *css)
8319 {
8320 	struct task_group *tg = css_tg(css);
8321 
8322 	sched_offline_group(tg);
8323 }
8324 
8325 static void cpu_cgroup_css_free(struct cgroup_subsys_state *css)
8326 {
8327 	struct task_group *tg = css_tg(css);
8328 
8329 	/*
8330 	 * Relies on the RCU grace period between css_released() and this.
8331 	 */
8332 	sched_free_group(tg);
8333 }
8334 
8335 /*
8336  * This is called before wake_up_new_task(), therefore we really only
8337  * have to set its group bits, all the other stuff does not apply.
8338  */
8339 static void cpu_cgroup_fork(struct task_struct *task)
8340 {
8341 	struct rq_flags rf;
8342 	struct rq *rq;
8343 
8344 	rq = task_rq_lock(task, &rf);
8345 
8346 	update_rq_clock(rq);
8347 	sched_change_group(task, TASK_SET_GROUP);
8348 
8349 	task_rq_unlock(rq, task, &rf);
8350 }
8351 
8352 static int cpu_cgroup_can_attach(struct cgroup_taskset *tset)
8353 {
8354 	struct task_struct *task;
8355 	struct cgroup_subsys_state *css;
8356 	int ret = 0;
8357 
8358 	cgroup_taskset_for_each(task, css, tset) {
8359 #ifdef CONFIG_RT_GROUP_SCHED
8360 		if (!sched_rt_can_attach(css_tg(css), task))
8361 			return -EINVAL;
8362 #endif
8363 		/*
8364 		 * Serialize against wake_up_new_task() such that if it's
8365 		 * running, we're sure to observe its full state.
8366 		 */
8367 		raw_spin_lock_irq(&task->pi_lock);
8368 		/*
8369 		 * Avoid calling sched_move_task() before wake_up_new_task()
8370 		 * has happened. This would lead to problems with PELT, due to
8371 		 * move wanting to detach+attach while we're not attached yet.
8372 		 */
8373 		if (task->state == TASK_NEW)
8374 			ret = -EINVAL;
8375 		raw_spin_unlock_irq(&task->pi_lock);
8376 
8377 		if (ret)
8378 			break;
8379 	}
8380 	return ret;
8381 }
8382 
8383 static void cpu_cgroup_attach(struct cgroup_taskset *tset)
8384 {
8385 	struct task_struct *task;
8386 	struct cgroup_subsys_state *css;
8387 
8388 	cgroup_taskset_for_each(task, css, tset)
8389 		sched_move_task(task);
8390 }
8391 
8392 #ifdef CONFIG_UCLAMP_TASK_GROUP
8393 static void cpu_util_update_eff(struct cgroup_subsys_state *css)
8394 {
8395 	struct cgroup_subsys_state *top_css = css;
8396 	struct uclamp_se *uc_parent = NULL;
8397 	struct uclamp_se *uc_se = NULL;
8398 	unsigned int eff[UCLAMP_CNT];
8399 	enum uclamp_id clamp_id;
8400 	unsigned int clamps;
8401 
8402 	css_for_each_descendant_pre(css, top_css) {
8403 		uc_parent = css_tg(css)->parent
8404 			? css_tg(css)->parent->uclamp : NULL;
8405 
8406 		for_each_clamp_id(clamp_id) {
8407 			/* Assume effective clamps matches requested clamps */
8408 			eff[clamp_id] = css_tg(css)->uclamp_req[clamp_id].value;
8409 			/* Cap effective clamps with parent's effective clamps */
8410 			if (uc_parent &&
8411 			    eff[clamp_id] > uc_parent[clamp_id].value) {
8412 				eff[clamp_id] = uc_parent[clamp_id].value;
8413 			}
8414 		}
8415 		/* Ensure protection is always capped by limit */
8416 		eff[UCLAMP_MIN] = min(eff[UCLAMP_MIN], eff[UCLAMP_MAX]);
8417 
8418 		/* Propagate most restrictive effective clamps */
8419 		clamps = 0x0;
8420 		uc_se = css_tg(css)->uclamp;
8421 		for_each_clamp_id(clamp_id) {
8422 			if (eff[clamp_id] == uc_se[clamp_id].value)
8423 				continue;
8424 			uc_se[clamp_id].value = eff[clamp_id];
8425 			uc_se[clamp_id].bucket_id = uclamp_bucket_id(eff[clamp_id]);
8426 			clamps |= (0x1 << clamp_id);
8427 		}
8428 		if (!clamps) {
8429 			css = css_rightmost_descendant(css);
8430 			continue;
8431 		}
8432 
8433 		/* Immediately update descendants RUNNABLE tasks */
8434 		uclamp_update_active_tasks(css, clamps);
8435 	}
8436 }
8437 
8438 /*
8439  * Integer 10^N with a given N exponent by casting to integer the literal "1eN"
8440  * C expression. Since there is no way to convert a macro argument (N) into a
8441  * character constant, use two levels of macros.
8442  */
8443 #define _POW10(exp) ((unsigned int)1e##exp)
8444 #define POW10(exp) _POW10(exp)
8445 
8446 struct uclamp_request {
8447 #define UCLAMP_PERCENT_SHIFT	2
8448 #define UCLAMP_PERCENT_SCALE	(100 * POW10(UCLAMP_PERCENT_SHIFT))
8449 	s64 percent;
8450 	u64 util;
8451 	int ret;
8452 };
8453 
8454 static inline struct uclamp_request
8455 capacity_from_percent(char *buf)
8456 {
8457 	struct uclamp_request req = {
8458 		.percent = UCLAMP_PERCENT_SCALE,
8459 		.util = SCHED_CAPACITY_SCALE,
8460 		.ret = 0,
8461 	};
8462 
8463 	buf = strim(buf);
8464 	if (strcmp(buf, "max")) {
8465 		req.ret = cgroup_parse_float(buf, UCLAMP_PERCENT_SHIFT,
8466 					     &req.percent);
8467 		if (req.ret)
8468 			return req;
8469 		if ((u64)req.percent > UCLAMP_PERCENT_SCALE) {
8470 			req.ret = -ERANGE;
8471 			return req;
8472 		}
8473 
8474 		req.util = req.percent << SCHED_CAPACITY_SHIFT;
8475 		req.util = DIV_ROUND_CLOSEST_ULL(req.util, UCLAMP_PERCENT_SCALE);
8476 	}
8477 
8478 	return req;
8479 }
8480 
8481 static ssize_t cpu_uclamp_write(struct kernfs_open_file *of, char *buf,
8482 				size_t nbytes, loff_t off,
8483 				enum uclamp_id clamp_id)
8484 {
8485 	struct uclamp_request req;
8486 	struct task_group *tg;
8487 
8488 	req = capacity_from_percent(buf);
8489 	if (req.ret)
8490 		return req.ret;
8491 
8492 	static_branch_enable(&sched_uclamp_used);
8493 
8494 	mutex_lock(&uclamp_mutex);
8495 	rcu_read_lock();
8496 
8497 	tg = css_tg(of_css(of));
8498 	if (tg->uclamp_req[clamp_id].value != req.util)
8499 		uclamp_se_set(&tg->uclamp_req[clamp_id], req.util, false);
8500 
8501 	/*
8502 	 * Because of not recoverable conversion rounding we keep track of the
8503 	 * exact requested value
8504 	 */
8505 	tg->uclamp_pct[clamp_id] = req.percent;
8506 
8507 	/* Update effective clamps to track the most restrictive value */
8508 	cpu_util_update_eff(of_css(of));
8509 
8510 	rcu_read_unlock();
8511 	mutex_unlock(&uclamp_mutex);
8512 
8513 	return nbytes;
8514 }
8515 
8516 static ssize_t cpu_uclamp_min_write(struct kernfs_open_file *of,
8517 				    char *buf, size_t nbytes,
8518 				    loff_t off)
8519 {
8520 	return cpu_uclamp_write(of, buf, nbytes, off, UCLAMP_MIN);
8521 }
8522 
8523 static ssize_t cpu_uclamp_max_write(struct kernfs_open_file *of,
8524 				    char *buf, size_t nbytes,
8525 				    loff_t off)
8526 {
8527 	return cpu_uclamp_write(of, buf, nbytes, off, UCLAMP_MAX);
8528 }
8529 
8530 static inline void cpu_uclamp_print(struct seq_file *sf,
8531 				    enum uclamp_id clamp_id)
8532 {
8533 	struct task_group *tg;
8534 	u64 util_clamp;
8535 	u64 percent;
8536 	u32 rem;
8537 
8538 	rcu_read_lock();
8539 	tg = css_tg(seq_css(sf));
8540 	util_clamp = tg->uclamp_req[clamp_id].value;
8541 	rcu_read_unlock();
8542 
8543 	if (util_clamp == SCHED_CAPACITY_SCALE) {
8544 		seq_puts(sf, "max\n");
8545 		return;
8546 	}
8547 
8548 	percent = tg->uclamp_pct[clamp_id];
8549 	percent = div_u64_rem(percent, POW10(UCLAMP_PERCENT_SHIFT), &rem);
8550 	seq_printf(sf, "%llu.%0*u\n", percent, UCLAMP_PERCENT_SHIFT, rem);
8551 }
8552 
8553 static int cpu_uclamp_min_show(struct seq_file *sf, void *v)
8554 {
8555 	cpu_uclamp_print(sf, UCLAMP_MIN);
8556 	return 0;
8557 }
8558 
8559 static int cpu_uclamp_max_show(struct seq_file *sf, void *v)
8560 {
8561 	cpu_uclamp_print(sf, UCLAMP_MAX);
8562 	return 0;
8563 }
8564 #endif /* CONFIG_UCLAMP_TASK_GROUP */
8565 
8566 #ifdef CONFIG_FAIR_GROUP_SCHED
8567 static int cpu_shares_write_u64(struct cgroup_subsys_state *css,
8568 				struct cftype *cftype, u64 shareval)
8569 {
8570 	if (shareval > scale_load_down(ULONG_MAX))
8571 		shareval = MAX_SHARES;
8572 	return sched_group_set_shares(css_tg(css), scale_load(shareval));
8573 }
8574 
8575 static u64 cpu_shares_read_u64(struct cgroup_subsys_state *css,
8576 			       struct cftype *cft)
8577 {
8578 	struct task_group *tg = css_tg(css);
8579 
8580 	return (u64) scale_load_down(tg->shares);
8581 }
8582 
8583 #ifdef CONFIG_CFS_BANDWIDTH
8584 static DEFINE_MUTEX(cfs_constraints_mutex);
8585 
8586 const u64 max_cfs_quota_period = 1 * NSEC_PER_SEC; /* 1s */
8587 static const u64 min_cfs_quota_period = 1 * NSEC_PER_MSEC; /* 1ms */
8588 /* More than 203 days if BW_SHIFT equals 20. */
8589 static const u64 max_cfs_runtime = MAX_BW * NSEC_PER_USEC;
8590 
8591 static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime);
8592 
8593 static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
8594 {
8595 	int i, ret = 0, runtime_enabled, runtime_was_enabled;
8596 	struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
8597 
8598 	if (tg == &root_task_group)
8599 		return -EINVAL;
8600 
8601 	/*
8602 	 * Ensure we have at some amount of bandwidth every period.  This is
8603 	 * to prevent reaching a state of large arrears when throttled via
8604 	 * entity_tick() resulting in prolonged exit starvation.
8605 	 */
8606 	if (quota < min_cfs_quota_period || period < min_cfs_quota_period)
8607 		return -EINVAL;
8608 
8609 	/*
8610 	 * Likewise, bound things on the otherside by preventing insane quota
8611 	 * periods.  This also allows us to normalize in computing quota
8612 	 * feasibility.
8613 	 */
8614 	if (period > max_cfs_quota_period)
8615 		return -EINVAL;
8616 
8617 	/*
8618 	 * Bound quota to defend quota against overflow during bandwidth shift.
8619 	 */
8620 	if (quota != RUNTIME_INF && quota > max_cfs_runtime)
8621 		return -EINVAL;
8622 
8623 	/*
8624 	 * Prevent race between setting of cfs_rq->runtime_enabled and
8625 	 * unthrottle_offline_cfs_rqs().
8626 	 */
8627 	get_online_cpus();
8628 	mutex_lock(&cfs_constraints_mutex);
8629 	ret = __cfs_schedulable(tg, period, quota);
8630 	if (ret)
8631 		goto out_unlock;
8632 
8633 	runtime_enabled = quota != RUNTIME_INF;
8634 	runtime_was_enabled = cfs_b->quota != RUNTIME_INF;
8635 	/*
8636 	 * If we need to toggle cfs_bandwidth_used, off->on must occur
8637 	 * before making related changes, and on->off must occur afterwards
8638 	 */
8639 	if (runtime_enabled && !runtime_was_enabled)
8640 		cfs_bandwidth_usage_inc();
8641 	raw_spin_lock_irq(&cfs_b->lock);
8642 	cfs_b->period = ns_to_ktime(period);
8643 	cfs_b->quota = quota;
8644 
8645 	__refill_cfs_bandwidth_runtime(cfs_b);
8646 
8647 	/* Restart the period timer (if active) to handle new period expiry: */
8648 	if (runtime_enabled)
8649 		start_cfs_bandwidth(cfs_b);
8650 
8651 	raw_spin_unlock_irq(&cfs_b->lock);
8652 
8653 	for_each_online_cpu(i) {
8654 		struct cfs_rq *cfs_rq = tg->cfs_rq[i];
8655 		struct rq *rq = cfs_rq->rq;
8656 		struct rq_flags rf;
8657 
8658 		rq_lock_irq(rq, &rf);
8659 		cfs_rq->runtime_enabled = runtime_enabled;
8660 		cfs_rq->runtime_remaining = 0;
8661 
8662 		if (cfs_rq->throttled)
8663 			unthrottle_cfs_rq(cfs_rq);
8664 		rq_unlock_irq(rq, &rf);
8665 	}
8666 	if (runtime_was_enabled && !runtime_enabled)
8667 		cfs_bandwidth_usage_dec();
8668 out_unlock:
8669 	mutex_unlock(&cfs_constraints_mutex);
8670 	put_online_cpus();
8671 
8672 	return ret;
8673 }
8674 
8675 static int tg_set_cfs_quota(struct task_group *tg, long cfs_quota_us)
8676 {
8677 	u64 quota, period;
8678 
8679 	period = ktime_to_ns(tg->cfs_bandwidth.period);
8680 	if (cfs_quota_us < 0)
8681 		quota = RUNTIME_INF;
8682 	else if ((u64)cfs_quota_us <= U64_MAX / NSEC_PER_USEC)
8683 		quota = (u64)cfs_quota_us * NSEC_PER_USEC;
8684 	else
8685 		return -EINVAL;
8686 
8687 	return tg_set_cfs_bandwidth(tg, period, quota);
8688 }
8689 
8690 static long tg_get_cfs_quota(struct task_group *tg)
8691 {
8692 	u64 quota_us;
8693 
8694 	if (tg->cfs_bandwidth.quota == RUNTIME_INF)
8695 		return -1;
8696 
8697 	quota_us = tg->cfs_bandwidth.quota;
8698 	do_div(quota_us, NSEC_PER_USEC);
8699 
8700 	return quota_us;
8701 }
8702 
8703 static int tg_set_cfs_period(struct task_group *tg, long cfs_period_us)
8704 {
8705 	u64 quota, period;
8706 
8707 	if ((u64)cfs_period_us > U64_MAX / NSEC_PER_USEC)
8708 		return -EINVAL;
8709 
8710 	period = (u64)cfs_period_us * NSEC_PER_USEC;
8711 	quota = tg->cfs_bandwidth.quota;
8712 
8713 	return tg_set_cfs_bandwidth(tg, period, quota);
8714 }
8715 
8716 static long tg_get_cfs_period(struct task_group *tg)
8717 {
8718 	u64 cfs_period_us;
8719 
8720 	cfs_period_us = ktime_to_ns(tg->cfs_bandwidth.period);
8721 	do_div(cfs_period_us, NSEC_PER_USEC);
8722 
8723 	return cfs_period_us;
8724 }
8725 
8726 static s64 cpu_cfs_quota_read_s64(struct cgroup_subsys_state *css,
8727 				  struct cftype *cft)
8728 {
8729 	return tg_get_cfs_quota(css_tg(css));
8730 }
8731 
8732 static int cpu_cfs_quota_write_s64(struct cgroup_subsys_state *css,
8733 				   struct cftype *cftype, s64 cfs_quota_us)
8734 {
8735 	return tg_set_cfs_quota(css_tg(css), cfs_quota_us);
8736 }
8737 
8738 static u64 cpu_cfs_period_read_u64(struct cgroup_subsys_state *css,
8739 				   struct cftype *cft)
8740 {
8741 	return tg_get_cfs_period(css_tg(css));
8742 }
8743 
8744 static int cpu_cfs_period_write_u64(struct cgroup_subsys_state *css,
8745 				    struct cftype *cftype, u64 cfs_period_us)
8746 {
8747 	return tg_set_cfs_period(css_tg(css), cfs_period_us);
8748 }
8749 
8750 struct cfs_schedulable_data {
8751 	struct task_group *tg;
8752 	u64 period, quota;
8753 };
8754 
8755 /*
8756  * normalize group quota/period to be quota/max_period
8757  * note: units are usecs
8758  */
8759 static u64 normalize_cfs_quota(struct task_group *tg,
8760 			       struct cfs_schedulable_data *d)
8761 {
8762 	u64 quota, period;
8763 
8764 	if (tg == d->tg) {
8765 		period = d->period;
8766 		quota = d->quota;
8767 	} else {
8768 		period = tg_get_cfs_period(tg);
8769 		quota = tg_get_cfs_quota(tg);
8770 	}
8771 
8772 	/* note: these should typically be equivalent */
8773 	if (quota == RUNTIME_INF || quota == -1)
8774 		return RUNTIME_INF;
8775 
8776 	return to_ratio(period, quota);
8777 }
8778 
8779 static int tg_cfs_schedulable_down(struct task_group *tg, void *data)
8780 {
8781 	struct cfs_schedulable_data *d = data;
8782 	struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
8783 	s64 quota = 0, parent_quota = -1;
8784 
8785 	if (!tg->parent) {
8786 		quota = RUNTIME_INF;
8787 	} else {
8788 		struct cfs_bandwidth *parent_b = &tg->parent->cfs_bandwidth;
8789 
8790 		quota = normalize_cfs_quota(tg, d);
8791 		parent_quota = parent_b->hierarchical_quota;
8792 
8793 		/*
8794 		 * Ensure max(child_quota) <= parent_quota.  On cgroup2,
8795 		 * always take the min.  On cgroup1, only inherit when no
8796 		 * limit is set:
8797 		 */
8798 		if (cgroup_subsys_on_dfl(cpu_cgrp_subsys)) {
8799 			quota = min(quota, parent_quota);
8800 		} else {
8801 			if (quota == RUNTIME_INF)
8802 				quota = parent_quota;
8803 			else if (parent_quota != RUNTIME_INF && quota > parent_quota)
8804 				return -EINVAL;
8805 		}
8806 	}
8807 	cfs_b->hierarchical_quota = quota;
8808 
8809 	return 0;
8810 }
8811 
8812 static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota)
8813 {
8814 	int ret;
8815 	struct cfs_schedulable_data data = {
8816 		.tg = tg,
8817 		.period = period,
8818 		.quota = quota,
8819 	};
8820 
8821 	if (quota != RUNTIME_INF) {
8822 		do_div(data.period, NSEC_PER_USEC);
8823 		do_div(data.quota, NSEC_PER_USEC);
8824 	}
8825 
8826 	rcu_read_lock();
8827 	ret = walk_tg_tree(tg_cfs_schedulable_down, tg_nop, &data);
8828 	rcu_read_unlock();
8829 
8830 	return ret;
8831 }
8832 
8833 static int cpu_cfs_stat_show(struct seq_file *sf, void *v)
8834 {
8835 	struct task_group *tg = css_tg(seq_css(sf));
8836 	struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
8837 
8838 	seq_printf(sf, "nr_periods %d\n", cfs_b->nr_periods);
8839 	seq_printf(sf, "nr_throttled %d\n", cfs_b->nr_throttled);
8840 	seq_printf(sf, "throttled_time %llu\n", cfs_b->throttled_time);
8841 
8842 	if (schedstat_enabled() && tg != &root_task_group) {
8843 		u64 ws = 0;
8844 		int i;
8845 
8846 		for_each_possible_cpu(i)
8847 			ws += schedstat_val(tg->se[i]->statistics.wait_sum);
8848 
8849 		seq_printf(sf, "wait_sum %llu\n", ws);
8850 	}
8851 
8852 	return 0;
8853 }
8854 #endif /* CONFIG_CFS_BANDWIDTH */
8855 #endif /* CONFIG_FAIR_GROUP_SCHED */
8856 
8857 #ifdef CONFIG_RT_GROUP_SCHED
8858 static int cpu_rt_runtime_write(struct cgroup_subsys_state *css,
8859 				struct cftype *cft, s64 val)
8860 {
8861 	return sched_group_set_rt_runtime(css_tg(css), val);
8862 }
8863 
8864 static s64 cpu_rt_runtime_read(struct cgroup_subsys_state *css,
8865 			       struct cftype *cft)
8866 {
8867 	return sched_group_rt_runtime(css_tg(css));
8868 }
8869 
8870 static int cpu_rt_period_write_uint(struct cgroup_subsys_state *css,
8871 				    struct cftype *cftype, u64 rt_period_us)
8872 {
8873 	return sched_group_set_rt_period(css_tg(css), rt_period_us);
8874 }
8875 
8876 static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css,
8877 				   struct cftype *cft)
8878 {
8879 	return sched_group_rt_period(css_tg(css));
8880 }
8881 #endif /* CONFIG_RT_GROUP_SCHED */
8882 
8883 static struct cftype cpu_legacy_files[] = {
8884 #ifdef CONFIG_FAIR_GROUP_SCHED
8885 	{
8886 		.name = "shares",
8887 		.read_u64 = cpu_shares_read_u64,
8888 		.write_u64 = cpu_shares_write_u64,
8889 	},
8890 #endif
8891 #ifdef CONFIG_CFS_BANDWIDTH
8892 	{
8893 		.name = "cfs_quota_us",
8894 		.read_s64 = cpu_cfs_quota_read_s64,
8895 		.write_s64 = cpu_cfs_quota_write_s64,
8896 	},
8897 	{
8898 		.name = "cfs_period_us",
8899 		.read_u64 = cpu_cfs_period_read_u64,
8900 		.write_u64 = cpu_cfs_period_write_u64,
8901 	},
8902 	{
8903 		.name = "stat",
8904 		.seq_show = cpu_cfs_stat_show,
8905 	},
8906 #endif
8907 #ifdef CONFIG_RT_GROUP_SCHED
8908 	{
8909 		.name = "rt_runtime_us",
8910 		.read_s64 = cpu_rt_runtime_read,
8911 		.write_s64 = cpu_rt_runtime_write,
8912 	},
8913 	{
8914 		.name = "rt_period_us",
8915 		.read_u64 = cpu_rt_period_read_uint,
8916 		.write_u64 = cpu_rt_period_write_uint,
8917 	},
8918 #endif
8919 #ifdef CONFIG_UCLAMP_TASK_GROUP
8920 	{
8921 		.name = "uclamp.min",
8922 		.flags = CFTYPE_NOT_ON_ROOT,
8923 		.seq_show = cpu_uclamp_min_show,
8924 		.write = cpu_uclamp_min_write,
8925 	},
8926 	{
8927 		.name = "uclamp.max",
8928 		.flags = CFTYPE_NOT_ON_ROOT,
8929 		.seq_show = cpu_uclamp_max_show,
8930 		.write = cpu_uclamp_max_write,
8931 	},
8932 #endif
8933 	{ }	/* Terminate */
8934 };
8935 
8936 static int cpu_extra_stat_show(struct seq_file *sf,
8937 			       struct cgroup_subsys_state *css)
8938 {
8939 #ifdef CONFIG_CFS_BANDWIDTH
8940 	{
8941 		struct task_group *tg = css_tg(css);
8942 		struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
8943 		u64 throttled_usec;
8944 
8945 		throttled_usec = cfs_b->throttled_time;
8946 		do_div(throttled_usec, NSEC_PER_USEC);
8947 
8948 		seq_printf(sf, "nr_periods %d\n"
8949 			   "nr_throttled %d\n"
8950 			   "throttled_usec %llu\n",
8951 			   cfs_b->nr_periods, cfs_b->nr_throttled,
8952 			   throttled_usec);
8953 	}
8954 #endif
8955 	return 0;
8956 }
8957 
8958 #ifdef CONFIG_FAIR_GROUP_SCHED
8959 static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css,
8960 			       struct cftype *cft)
8961 {
8962 	struct task_group *tg = css_tg(css);
8963 	u64 weight = scale_load_down(tg->shares);
8964 
8965 	return DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024);
8966 }
8967 
8968 static int cpu_weight_write_u64(struct cgroup_subsys_state *css,
8969 				struct cftype *cft, u64 weight)
8970 {
8971 	/*
8972 	 * cgroup weight knobs should use the common MIN, DFL and MAX
8973 	 * values which are 1, 100 and 10000 respectively.  While it loses
8974 	 * a bit of range on both ends, it maps pretty well onto the shares
8975 	 * value used by scheduler and the round-trip conversions preserve
8976 	 * the original value over the entire range.
8977 	 */
8978 	if (weight < CGROUP_WEIGHT_MIN || weight > CGROUP_WEIGHT_MAX)
8979 		return -ERANGE;
8980 
8981 	weight = DIV_ROUND_CLOSEST_ULL(weight * 1024, CGROUP_WEIGHT_DFL);
8982 
8983 	return sched_group_set_shares(css_tg(css), scale_load(weight));
8984 }
8985 
8986 static s64 cpu_weight_nice_read_s64(struct cgroup_subsys_state *css,
8987 				    struct cftype *cft)
8988 {
8989 	unsigned long weight = scale_load_down(css_tg(css)->shares);
8990 	int last_delta = INT_MAX;
8991 	int prio, delta;
8992 
8993 	/* find the closest nice value to the current weight */
8994 	for (prio = 0; prio < ARRAY_SIZE(sched_prio_to_weight); prio++) {
8995 		delta = abs(sched_prio_to_weight[prio] - weight);
8996 		if (delta >= last_delta)
8997 			break;
8998 		last_delta = delta;
8999 	}
9000 
9001 	return PRIO_TO_NICE(prio - 1 + MAX_RT_PRIO);
9002 }
9003 
9004 static int cpu_weight_nice_write_s64(struct cgroup_subsys_state *css,
9005 				     struct cftype *cft, s64 nice)
9006 {
9007 	unsigned long weight;
9008 	int idx;
9009 
9010 	if (nice < MIN_NICE || nice > MAX_NICE)
9011 		return -ERANGE;
9012 
9013 	idx = NICE_TO_PRIO(nice) - MAX_RT_PRIO;
9014 	idx = array_index_nospec(idx, 40);
9015 	weight = sched_prio_to_weight[idx];
9016 
9017 	return sched_group_set_shares(css_tg(css), scale_load(weight));
9018 }
9019 #endif
9020 
9021 static void __maybe_unused cpu_period_quota_print(struct seq_file *sf,
9022 						  long period, long quota)
9023 {
9024 	if (quota < 0)
9025 		seq_puts(sf, "max");
9026 	else
9027 		seq_printf(sf, "%ld", quota);
9028 
9029 	seq_printf(sf, " %ld\n", period);
9030 }
9031 
9032 /* caller should put the current value in *@periodp before calling */
9033 static int __maybe_unused cpu_period_quota_parse(char *buf,
9034 						 u64 *periodp, u64 *quotap)
9035 {
9036 	char tok[21];	/* U64_MAX */
9037 
9038 	if (sscanf(buf, "%20s %llu", tok, periodp) < 1)
9039 		return -EINVAL;
9040 
9041 	*periodp *= NSEC_PER_USEC;
9042 
9043 	if (sscanf(tok, "%llu", quotap))
9044 		*quotap *= NSEC_PER_USEC;
9045 	else if (!strcmp(tok, "max"))
9046 		*quotap = RUNTIME_INF;
9047 	else
9048 		return -EINVAL;
9049 
9050 	return 0;
9051 }
9052 
9053 #ifdef CONFIG_CFS_BANDWIDTH
9054 static int cpu_max_show(struct seq_file *sf, void *v)
9055 {
9056 	struct task_group *tg = css_tg(seq_css(sf));
9057 
9058 	cpu_period_quota_print(sf, tg_get_cfs_period(tg), tg_get_cfs_quota(tg));
9059 	return 0;
9060 }
9061 
9062 static ssize_t cpu_max_write(struct kernfs_open_file *of,
9063 			     char *buf, size_t nbytes, loff_t off)
9064 {
9065 	struct task_group *tg = css_tg(of_css(of));
9066 	u64 period = tg_get_cfs_period(tg);
9067 	u64 quota;
9068 	int ret;
9069 
9070 	ret = cpu_period_quota_parse(buf, &period, &quota);
9071 	if (!ret)
9072 		ret = tg_set_cfs_bandwidth(tg, period, quota);
9073 	return ret ?: nbytes;
9074 }
9075 #endif
9076 
9077 static struct cftype cpu_files[] = {
9078 #ifdef CONFIG_FAIR_GROUP_SCHED
9079 	{
9080 		.name = "weight",
9081 		.flags = CFTYPE_NOT_ON_ROOT,
9082 		.read_u64 = cpu_weight_read_u64,
9083 		.write_u64 = cpu_weight_write_u64,
9084 	},
9085 	{
9086 		.name = "weight.nice",
9087 		.flags = CFTYPE_NOT_ON_ROOT,
9088 		.read_s64 = cpu_weight_nice_read_s64,
9089 		.write_s64 = cpu_weight_nice_write_s64,
9090 	},
9091 #endif
9092 #ifdef CONFIG_CFS_BANDWIDTH
9093 	{
9094 		.name = "max",
9095 		.flags = CFTYPE_NOT_ON_ROOT,
9096 		.seq_show = cpu_max_show,
9097 		.write = cpu_max_write,
9098 	},
9099 #endif
9100 #ifdef CONFIG_UCLAMP_TASK_GROUP
9101 	{
9102 		.name = "uclamp.min",
9103 		.flags = CFTYPE_NOT_ON_ROOT,
9104 		.seq_show = cpu_uclamp_min_show,
9105 		.write = cpu_uclamp_min_write,
9106 	},
9107 	{
9108 		.name = "uclamp.max",
9109 		.flags = CFTYPE_NOT_ON_ROOT,
9110 		.seq_show = cpu_uclamp_max_show,
9111 		.write = cpu_uclamp_max_write,
9112 	},
9113 #endif
9114 	{ }	/* terminate */
9115 };
9116 
9117 struct cgroup_subsys cpu_cgrp_subsys = {
9118 	.css_alloc	= cpu_cgroup_css_alloc,
9119 	.css_online	= cpu_cgroup_css_online,
9120 	.css_released	= cpu_cgroup_css_released,
9121 	.css_free	= cpu_cgroup_css_free,
9122 	.css_extra_stat_show = cpu_extra_stat_show,
9123 	.fork		= cpu_cgroup_fork,
9124 	.can_attach	= cpu_cgroup_can_attach,
9125 	.attach		= cpu_cgroup_attach,
9126 	.legacy_cftypes	= cpu_legacy_files,
9127 	.dfl_cftypes	= cpu_files,
9128 	.early_init	= true,
9129 	.threaded	= true,
9130 };
9131 
9132 #endif	/* CONFIG_CGROUP_SCHED */
9133 
9134 void dump_cpu_task(int cpu)
9135 {
9136 	pr_info("Task dump for CPU %d:\n", cpu);
9137 	sched_show_task(cpu_curr(cpu));
9138 }
9139 
9140 /*
9141  * Nice levels are multiplicative, with a gentle 10% change for every
9142  * nice level changed. I.e. when a CPU-bound task goes from nice 0 to
9143  * nice 1, it will get ~10% less CPU time than another CPU-bound task
9144  * that remained on nice 0.
9145  *
9146  * The "10% effect" is relative and cumulative: from _any_ nice level,
9147  * if you go up 1 level, it's -10% CPU usage, if you go down 1 level
9148  * it's +10% CPU usage. (to achieve that we use a multiplier of 1.25.
9149  * If a task goes up by ~10% and another task goes down by ~10% then
9150  * the relative distance between them is ~25%.)
9151  */
9152 const int sched_prio_to_weight[40] = {
9153  /* -20 */     88761,     71755,     56483,     46273,     36291,
9154  /* -15 */     29154,     23254,     18705,     14949,     11916,
9155  /* -10 */      9548,      7620,      6100,      4904,      3906,
9156  /*  -5 */      3121,      2501,      1991,      1586,      1277,
9157  /*   0 */      1024,       820,       655,       526,       423,
9158  /*   5 */       335,       272,       215,       172,       137,
9159  /*  10 */       110,        87,        70,        56,        45,
9160  /*  15 */        36,        29,        23,        18,        15,
9161 };
9162 
9163 /*
9164  * Inverse (2^32/x) values of the sched_prio_to_weight[] array, precalculated.
9165  *
9166  * In cases where the weight does not change often, we can use the
9167  * precalculated inverse to speed up arithmetics by turning divisions
9168  * into multiplications:
9169  */
9170 const u32 sched_prio_to_wmult[40] = {
9171  /* -20 */     48388,     59856,     76040,     92818,    118348,
9172  /* -15 */    147320,    184698,    229616,    287308,    360437,
9173  /* -10 */    449829,    563644,    704093,    875809,   1099582,
9174  /*  -5 */   1376151,   1717300,   2157191,   2708050,   3363326,
9175  /*   0 */   4194304,   5237765,   6557202,   8165337,  10153587,
9176  /*   5 */  12820798,  15790321,  19976592,  24970740,  31350126,
9177  /*  10 */  39045157,  49367440,  61356676,  76695844,  95443717,
9178  /*  15 */ 119304647, 148102320, 186737708, 238609294, 286331153,
9179 };
9180 
9181 void call_trace_sched_update_nr_running(struct rq *rq, int count)
9182 {
9183         trace_sched_update_nr_running_tp(rq, count);
9184 }
9185