xref: /linux/include/linux/sched.h (revision 8c4890d1c3358fb8023d46e1e554c41d54f02878)
1b2441318SGreg Kroah-Hartman /* SPDX-License-Identifier: GPL-2.0 */
21da177e4SLinus Torvalds #ifndef _LINUX_SCHED_H
31da177e4SLinus Torvalds #define _LINUX_SCHED_H
41da177e4SLinus Torvalds 
55eca1c10SIngo Molnar /*
65eca1c10SIngo Molnar  * Define 'struct task_struct' and provide the main scheduler
75eca1c10SIngo Molnar  * APIs (schedule(), wakeup variants, etc.)
85eca1c10SIngo Molnar  */
95eca1c10SIngo Molnar 
10607ca46eSDavid Howells #include <uapi/linux/sched.h>
11b7b3c76aSDavid Woodhouse 
1270b8157eSIngo Molnar #include <asm/current.h>
1370b8157eSIngo Molnar 
145eca1c10SIngo Molnar #include <linux/pid.h>
155eca1c10SIngo Molnar #include <linux/sem.h>
165eca1c10SIngo Molnar #include <linux/shm.h>
175eca1c10SIngo Molnar #include <linux/kcov.h>
185eca1c10SIngo Molnar #include <linux/mutex.h>
195eca1c10SIngo Molnar #include <linux/plist.h>
205eca1c10SIngo Molnar #include <linux/hrtimer.h>
215eca1c10SIngo Molnar #include <linux/seccomp.h>
225eca1c10SIngo Molnar #include <linux/nodemask.h>
235eca1c10SIngo Molnar #include <linux/rcupdate.h>
24ec1d2819SElena Reshetova #include <linux/refcount.h>
255eca1c10SIngo Molnar #include <linux/resource.h>
265eca1c10SIngo Molnar #include <linux/latencytop.h>
275eca1c10SIngo Molnar #include <linux/sched/prio.h>
289eacb5c7SThomas Gleixner #include <linux/sched/types.h>
295eca1c10SIngo Molnar #include <linux/signal_types.h>
305eca1c10SIngo Molnar #include <linux/mm_types_task.h>
315eca1c10SIngo Molnar #include <linux/task_io_accounting.h>
322b69942fSThomas Gleixner #include <linux/posix-timers.h>
33d7822b1eSMathieu Desnoyers #include <linux/rseq.h>
34dfd402a4SMarco Elver #include <linux/kcsan.h>
355eca1c10SIngo Molnar 
365eca1c10SIngo Molnar /* task_struct member predeclarations (sorted alphabetically): */
37c7af7877SIngo Molnar struct audit_context;
38c7af7877SIngo Molnar struct backing_dev_info;
39c7af7877SIngo Molnar struct bio_list;
40c7af7877SIngo Molnar struct blk_plug;
413c93a0c0SQais Yousef struct capture_control;
42c7af7877SIngo Molnar struct cfs_rq;
43c7af7877SIngo Molnar struct fs_struct;
44c7af7877SIngo Molnar struct futex_pi_state;
45c7af7877SIngo Molnar struct io_context;
46c7af7877SIngo Molnar struct mempolicy;
47c7af7877SIngo Molnar struct nameidata;
48c7af7877SIngo Molnar struct nsproxy;
49c7af7877SIngo Molnar struct perf_event_context;
50c7af7877SIngo Molnar struct pid_namespace;
51c7af7877SIngo Molnar struct pipe_inode_info;
52c7af7877SIngo Molnar struct rcu_node;
53c7af7877SIngo Molnar struct reclaim_state;
54c7af7877SIngo Molnar struct robust_list_head;
553c93a0c0SQais Yousef struct root_domain;
563c93a0c0SQais Yousef struct rq;
57e2d1e2aeSIngo Molnar struct sched_attr;
58e2d1e2aeSIngo Molnar struct sched_param;
5943ae34cbSIngo Molnar struct seq_file;
60c7af7877SIngo Molnar struct sighand_struct;
61c7af7877SIngo Molnar struct signal_struct;
62c7af7877SIngo Molnar struct task_delay_info;
634cf86d77SIngo Molnar struct task_group;
641da177e4SLinus Torvalds 
654a8342d2SLinus Torvalds /*
664a8342d2SLinus Torvalds  * Task state bitmask. NOTE! These bits are also
674a8342d2SLinus Torvalds  * encoded in fs/proc/array.c: get_task_state().
684a8342d2SLinus Torvalds  *
694a8342d2SLinus Torvalds  * We have two separate sets of flags: task->state
704a8342d2SLinus Torvalds  * is about runnability, while task->exit_state are
714a8342d2SLinus Torvalds  * about the task exiting. Confusing, but this way
724a8342d2SLinus Torvalds  * modifying one set can't modify the other one by
734a8342d2SLinus Torvalds  * mistake.
744a8342d2SLinus Torvalds  */
755eca1c10SIngo Molnar 
765eca1c10SIngo Molnar /* Used in tsk->state: */
7792c4bc9fSPeter Zijlstra #define TASK_RUNNING			0x0000
7892c4bc9fSPeter Zijlstra #define TASK_INTERRUPTIBLE		0x0001
7992c4bc9fSPeter Zijlstra #define TASK_UNINTERRUPTIBLE		0x0002
8092c4bc9fSPeter Zijlstra #define __TASK_STOPPED			0x0004
8192c4bc9fSPeter Zijlstra #define __TASK_TRACED			0x0008
825eca1c10SIngo Molnar /* Used in tsk->exit_state: */
8392c4bc9fSPeter Zijlstra #define EXIT_DEAD			0x0010
8492c4bc9fSPeter Zijlstra #define EXIT_ZOMBIE			0x0020
85abd50b39SOleg Nesterov #define EXIT_TRACE			(EXIT_ZOMBIE | EXIT_DEAD)
865eca1c10SIngo Molnar /* Used in tsk->state again: */
878ef9925bSPeter Zijlstra #define TASK_PARKED			0x0040
888ef9925bSPeter Zijlstra #define TASK_DEAD			0x0080
898ef9925bSPeter Zijlstra #define TASK_WAKEKILL			0x0100
908ef9925bSPeter Zijlstra #define TASK_WAKING			0x0200
9192c4bc9fSPeter Zijlstra #define TASK_NOLOAD			0x0400
9292c4bc9fSPeter Zijlstra #define TASK_NEW			0x0800
9392c4bc9fSPeter Zijlstra #define TASK_STATE_MAX			0x1000
94f021a3c2SMatthew Wilcox 
955eca1c10SIngo Molnar /* Convenience macros for the sake of set_current_state: */
96f021a3c2SMatthew Wilcox #define TASK_KILLABLE			(TASK_WAKEKILL | TASK_UNINTERRUPTIBLE)
97f021a3c2SMatthew Wilcox #define TASK_STOPPED			(TASK_WAKEKILL | __TASK_STOPPED)
98f021a3c2SMatthew Wilcox #define TASK_TRACED			(TASK_WAKEKILL | __TASK_TRACED)
991da177e4SLinus Torvalds 
10080ed87c8SPeter Zijlstra #define TASK_IDLE			(TASK_UNINTERRUPTIBLE | TASK_NOLOAD)
10180ed87c8SPeter Zijlstra 
1025eca1c10SIngo Molnar /* Convenience macros for the sake of wake_up(): */
10392a1f4bcSMatthew Wilcox #define TASK_NORMAL			(TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE)
10492a1f4bcSMatthew Wilcox 
1055eca1c10SIngo Molnar /* get_task_state(): */
10692a1f4bcSMatthew Wilcox #define TASK_REPORT			(TASK_RUNNING | TASK_INTERRUPTIBLE | \
107f021a3c2SMatthew Wilcox 					 TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \
1088ef9925bSPeter Zijlstra 					 __TASK_TRACED | EXIT_DEAD | EXIT_ZOMBIE | \
1098ef9925bSPeter Zijlstra 					 TASK_PARKED)
11092a1f4bcSMatthew Wilcox 
111f021a3c2SMatthew Wilcox #define task_is_traced(task)		((task->state & __TASK_TRACED) != 0)
1125eca1c10SIngo Molnar 
113f021a3c2SMatthew Wilcox #define task_is_stopped(task)		((task->state & __TASK_STOPPED) != 0)
1145eca1c10SIngo Molnar 
1155eca1c10SIngo Molnar #define task_is_stopped_or_traced(task)	((task->state & (__TASK_STOPPED | __TASK_TRACED)) != 0)
1165eca1c10SIngo Molnar 
1175eca1c10SIngo Molnar #define task_contributes_to_load(task)	((task->state & TASK_UNINTERRUPTIBLE) != 0 && \
11880ed87c8SPeter Zijlstra 					 (task->flags & PF_FROZEN) == 0 && \
11980ed87c8SPeter Zijlstra 					 (task->state & TASK_NOLOAD) == 0)
1201da177e4SLinus Torvalds 
1218eb23b9fSPeter Zijlstra #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
1228eb23b9fSPeter Zijlstra 
123b5bf9a90SPeter Zijlstra /*
124b5bf9a90SPeter Zijlstra  * Special states are those that do not use the normal wait-loop pattern. See
125b5bf9a90SPeter Zijlstra  * the comment with set_special_state().
126b5bf9a90SPeter Zijlstra  */
127b5bf9a90SPeter Zijlstra #define is_special_task_state(state)				\
1281cef1150SPeter Zijlstra 	((state) & (__TASK_STOPPED | __TASK_TRACED | TASK_PARKED | TASK_DEAD))
129b5bf9a90SPeter Zijlstra 
1308eb23b9fSPeter Zijlstra #define __set_current_state(state_value)			\
1318eb23b9fSPeter Zijlstra 	do {							\
132b5bf9a90SPeter Zijlstra 		WARN_ON_ONCE(is_special_task_state(state_value));\
1338eb23b9fSPeter Zijlstra 		current->task_state_change = _THIS_IP_;		\
1348eb23b9fSPeter Zijlstra 		current->state = (state_value);			\
1358eb23b9fSPeter Zijlstra 	} while (0)
136b5bf9a90SPeter Zijlstra 
1378eb23b9fSPeter Zijlstra #define set_current_state(state_value)				\
1388eb23b9fSPeter Zijlstra 	do {							\
139b5bf9a90SPeter Zijlstra 		WARN_ON_ONCE(is_special_task_state(state_value));\
1408eb23b9fSPeter Zijlstra 		current->task_state_change = _THIS_IP_;		\
141b92b8b35SPeter Zijlstra 		smp_store_mb(current->state, (state_value));	\
1428eb23b9fSPeter Zijlstra 	} while (0)
1438eb23b9fSPeter Zijlstra 
144b5bf9a90SPeter Zijlstra #define set_special_state(state_value)					\
145b5bf9a90SPeter Zijlstra 	do {								\
146b5bf9a90SPeter Zijlstra 		unsigned long flags; /* may shadow */			\
147b5bf9a90SPeter Zijlstra 		WARN_ON_ONCE(!is_special_task_state(state_value));	\
148b5bf9a90SPeter Zijlstra 		raw_spin_lock_irqsave(&current->pi_lock, flags);	\
149b5bf9a90SPeter Zijlstra 		current->task_state_change = _THIS_IP_;			\
150b5bf9a90SPeter Zijlstra 		current->state = (state_value);				\
151b5bf9a90SPeter Zijlstra 		raw_spin_unlock_irqrestore(&current->pi_lock, flags);	\
152b5bf9a90SPeter Zijlstra 	} while (0)
1538eb23b9fSPeter Zijlstra #else
154498d0c57SAndrew Morton /*
155498d0c57SAndrew Morton  * set_current_state() includes a barrier so that the write of current->state
156498d0c57SAndrew Morton  * is correctly serialised wrt the caller's subsequent test of whether to
157498d0c57SAndrew Morton  * actually sleep:
158498d0c57SAndrew Morton  *
159a2250238SPeter Zijlstra  *   for (;;) {
160498d0c57SAndrew Morton  *	set_current_state(TASK_UNINTERRUPTIBLE);
161a2250238SPeter Zijlstra  *	if (!need_sleep)
162a2250238SPeter Zijlstra  *		break;
163498d0c57SAndrew Morton  *
164a2250238SPeter Zijlstra  *	schedule();
165a2250238SPeter Zijlstra  *   }
166a2250238SPeter Zijlstra  *   __set_current_state(TASK_RUNNING);
167a2250238SPeter Zijlstra  *
168a2250238SPeter Zijlstra  * If the caller does not need such serialisation (because, for instance, the
169a2250238SPeter Zijlstra  * condition test and condition change and wakeup are under the same lock) then
170a2250238SPeter Zijlstra  * use __set_current_state().
171a2250238SPeter Zijlstra  *
172a2250238SPeter Zijlstra  * The above is typically ordered against the wakeup, which does:
173a2250238SPeter Zijlstra  *
174a2250238SPeter Zijlstra  *   need_sleep = false;
175a2250238SPeter Zijlstra  *   wake_up_state(p, TASK_UNINTERRUPTIBLE);
176a2250238SPeter Zijlstra  *
1777696f991SAndrea Parri  * where wake_up_state() executes a full memory barrier before accessing the
1787696f991SAndrea Parri  * task state.
179a2250238SPeter Zijlstra  *
180a2250238SPeter Zijlstra  * Wakeup will do: if (@state & p->state) p->state = TASK_RUNNING, that is,
181a2250238SPeter Zijlstra  * once it observes the TASK_UNINTERRUPTIBLE store the waking CPU can issue a
182a2250238SPeter Zijlstra  * TASK_RUNNING store which can collide with __set_current_state(TASK_RUNNING).
183a2250238SPeter Zijlstra  *
184b5bf9a90SPeter Zijlstra  * However, with slightly different timing the wakeup TASK_RUNNING store can
185dfcb245eSIngo Molnar  * also collide with the TASK_UNINTERRUPTIBLE store. Losing that store is not
186b5bf9a90SPeter Zijlstra  * a problem either because that will result in one extra go around the loop
187b5bf9a90SPeter Zijlstra  * and our @cond test will save the day.
188a2250238SPeter Zijlstra  *
189a2250238SPeter Zijlstra  * Also see the comments of try_to_wake_up().
190498d0c57SAndrew Morton  */
191b5bf9a90SPeter Zijlstra #define __set_current_state(state_value)				\
192b5bf9a90SPeter Zijlstra 	current->state = (state_value)
193b5bf9a90SPeter Zijlstra 
194b5bf9a90SPeter Zijlstra #define set_current_state(state_value)					\
195b5bf9a90SPeter Zijlstra 	smp_store_mb(current->state, (state_value))
196b5bf9a90SPeter Zijlstra 
197b5bf9a90SPeter Zijlstra /*
198b5bf9a90SPeter Zijlstra  * set_special_state() should be used for those states when the blocking task
199b5bf9a90SPeter Zijlstra  * can not use the regular condition based wait-loop. In that case we must
200b5bf9a90SPeter Zijlstra  * serialize against wakeups such that any possible in-flight TASK_RUNNING stores
201b5bf9a90SPeter Zijlstra  * will not collide with our state change.
202b5bf9a90SPeter Zijlstra  */
203b5bf9a90SPeter Zijlstra #define set_special_state(state_value)					\
204b5bf9a90SPeter Zijlstra 	do {								\
205b5bf9a90SPeter Zijlstra 		unsigned long flags; /* may shadow */			\
206b5bf9a90SPeter Zijlstra 		raw_spin_lock_irqsave(&current->pi_lock, flags);	\
207b5bf9a90SPeter Zijlstra 		current->state = (state_value);				\
208b5bf9a90SPeter Zijlstra 		raw_spin_unlock_irqrestore(&current->pi_lock, flags);	\
209b5bf9a90SPeter Zijlstra 	} while (0)
210b5bf9a90SPeter Zijlstra 
2118eb23b9fSPeter Zijlstra #endif
2128eb23b9fSPeter Zijlstra 
2135eca1c10SIngo Molnar /* Task command name length: */
2141da177e4SLinus Torvalds #define TASK_COMM_LEN			16
2151da177e4SLinus Torvalds 
2161da177e4SLinus Torvalds extern void scheduler_tick(void);
2171da177e4SLinus Torvalds 
2181da177e4SLinus Torvalds #define	MAX_SCHEDULE_TIMEOUT		LONG_MAX
2195eca1c10SIngo Molnar 
2205eca1c10SIngo Molnar extern long schedule_timeout(long timeout);
2215eca1c10SIngo Molnar extern long schedule_timeout_interruptible(long timeout);
2225eca1c10SIngo Molnar extern long schedule_timeout_killable(long timeout);
2235eca1c10SIngo Molnar extern long schedule_timeout_uninterruptible(long timeout);
2245eca1c10SIngo Molnar extern long schedule_timeout_idle(long timeout);
2251da177e4SLinus Torvalds asmlinkage void schedule(void);
226c5491ea7SThomas Gleixner extern void schedule_preempt_disabled(void);
22719c95f26SJulien Thierry asmlinkage void preempt_schedule_irq(void);
2281da177e4SLinus Torvalds 
22910ab5643STejun Heo extern int __must_check io_schedule_prepare(void);
23010ab5643STejun Heo extern void io_schedule_finish(int token);
2319cff8adeSNeilBrown extern long io_schedule_timeout(long timeout);
23210ab5643STejun Heo extern void io_schedule(void);
2339cff8adeSNeilBrown 
234f06febc9SFrank Mayhar /**
2350ba42a59SMasanari Iida  * struct prev_cputime - snapshot of system and user cputime
236d37f761dSFrederic Weisbecker  * @utime: time spent in user mode
237d37f761dSFrederic Weisbecker  * @stime: time spent in system mode
2389d7fb042SPeter Zijlstra  * @lock: protects the above two fields
239d37f761dSFrederic Weisbecker  *
2409d7fb042SPeter Zijlstra  * Stores previous user/system time values such that we can guarantee
2419d7fb042SPeter Zijlstra  * monotonicity.
242d37f761dSFrederic Weisbecker  */
2439d7fb042SPeter Zijlstra struct prev_cputime {
2449d7fb042SPeter Zijlstra #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
2455613fda9SFrederic Weisbecker 	u64				utime;
2465613fda9SFrederic Weisbecker 	u64				stime;
2479d7fb042SPeter Zijlstra 	raw_spinlock_t			lock;
2489d7fb042SPeter Zijlstra #endif
249d37f761dSFrederic Weisbecker };
250d37f761dSFrederic Weisbecker 
251bac5b6b6SFrederic Weisbecker enum vtime_state {
252bac5b6b6SFrederic Weisbecker 	/* Task is sleeping or running in a CPU with VTIME inactive: */
253bac5b6b6SFrederic Weisbecker 	VTIME_INACTIVE = 0,
25414faf6fcSFrederic Weisbecker 	/* Task is idle */
25514faf6fcSFrederic Weisbecker 	VTIME_IDLE,
256bac5b6b6SFrederic Weisbecker 	/* Task runs in kernelspace in a CPU with VTIME active: */
257bac5b6b6SFrederic Weisbecker 	VTIME_SYS,
25814faf6fcSFrederic Weisbecker 	/* Task runs in userspace in a CPU with VTIME active: */
25914faf6fcSFrederic Weisbecker 	VTIME_USER,
260e6d5bf3eSFrederic Weisbecker 	/* Task runs as guests in a CPU with VTIME active: */
261e6d5bf3eSFrederic Weisbecker 	VTIME_GUEST,
262bac5b6b6SFrederic Weisbecker };
263bac5b6b6SFrederic Weisbecker 
264bac5b6b6SFrederic Weisbecker struct vtime {
265bac5b6b6SFrederic Weisbecker 	seqcount_t		seqcount;
266bac5b6b6SFrederic Weisbecker 	unsigned long long	starttime;
267bac5b6b6SFrederic Weisbecker 	enum vtime_state	state;
268802f4a82SFrederic Weisbecker 	unsigned int		cpu;
2692a42eb95SWanpeng Li 	u64			utime;
2702a42eb95SWanpeng Li 	u64			stime;
2712a42eb95SWanpeng Li 	u64			gtime;
272bac5b6b6SFrederic Weisbecker };
273bac5b6b6SFrederic Weisbecker 
27469842cbaSPatrick Bellasi /*
27569842cbaSPatrick Bellasi  * Utilization clamp constraints.
27669842cbaSPatrick Bellasi  * @UCLAMP_MIN:	Minimum utilization
27769842cbaSPatrick Bellasi  * @UCLAMP_MAX:	Maximum utilization
27869842cbaSPatrick Bellasi  * @UCLAMP_CNT:	Utilization clamp constraints count
27969842cbaSPatrick Bellasi  */
28069842cbaSPatrick Bellasi enum uclamp_id {
28169842cbaSPatrick Bellasi 	UCLAMP_MIN = 0,
28269842cbaSPatrick Bellasi 	UCLAMP_MAX,
28369842cbaSPatrick Bellasi 	UCLAMP_CNT
28469842cbaSPatrick Bellasi };
28569842cbaSPatrick Bellasi 
286f9a25f77SMathieu Poirier #ifdef CONFIG_SMP
287f9a25f77SMathieu Poirier extern struct root_domain def_root_domain;
288f9a25f77SMathieu Poirier extern struct mutex sched_domains_mutex;
289f9a25f77SMathieu Poirier #endif
290f9a25f77SMathieu Poirier 
2911da177e4SLinus Torvalds struct sched_info {
2927f5f8e8dSIngo Molnar #ifdef CONFIG_SCHED_INFO
2935eca1c10SIngo Molnar 	/* Cumulative counters: */
2941da177e4SLinus Torvalds 
2955eca1c10SIngo Molnar 	/* # of times we have run on this CPU: */
2965eca1c10SIngo Molnar 	unsigned long			pcount;
2975eca1c10SIngo Molnar 
2985eca1c10SIngo Molnar 	/* Time spent waiting on a runqueue: */
2995eca1c10SIngo Molnar 	unsigned long long		run_delay;
3005eca1c10SIngo Molnar 
3015eca1c10SIngo Molnar 	/* Timestamps: */
3025eca1c10SIngo Molnar 
3035eca1c10SIngo Molnar 	/* When did we last run on a CPU? */
3045eca1c10SIngo Molnar 	unsigned long long		last_arrival;
3055eca1c10SIngo Molnar 
3065eca1c10SIngo Molnar 	/* When were we last queued to run? */
3075eca1c10SIngo Molnar 	unsigned long long		last_queued;
3085eca1c10SIngo Molnar 
309f6db8347SNaveen N. Rao #endif /* CONFIG_SCHED_INFO */
3107f5f8e8dSIngo Molnar };
3111da177e4SLinus Torvalds 
3121da177e4SLinus Torvalds /*
3136ecdd749SYuyang Du  * Integer metrics need fixed point arithmetic, e.g., sched/fair
3146ecdd749SYuyang Du  * has a few: load, load_avg, util_avg, freq, and capacity.
3156ecdd749SYuyang Du  *
3166ecdd749SYuyang Du  * We define a basic fixed point arithmetic range, and then formalize
3176ecdd749SYuyang Du  * all these metrics based on that basic range.
3186ecdd749SYuyang Du  */
3196ecdd749SYuyang Du # define SCHED_FIXEDPOINT_SHIFT		10
3206ecdd749SYuyang Du # define SCHED_FIXEDPOINT_SCALE		(1L << SCHED_FIXEDPOINT_SHIFT)
3216ecdd749SYuyang Du 
32269842cbaSPatrick Bellasi /* Increase resolution of cpu_capacity calculations */
32369842cbaSPatrick Bellasi # define SCHED_CAPACITY_SHIFT		SCHED_FIXEDPOINT_SHIFT
32469842cbaSPatrick Bellasi # define SCHED_CAPACITY_SCALE		(1L << SCHED_CAPACITY_SHIFT)
32569842cbaSPatrick Bellasi 
32620b8a59fSIngo Molnar struct load_weight {
3279dbdb155SPeter Zijlstra 	unsigned long			weight;
3289dbdb155SPeter Zijlstra 	u32				inv_weight;
32920b8a59fSIngo Molnar };
33020b8a59fSIngo Molnar 
3317f65ea42SPatrick Bellasi /**
3327f65ea42SPatrick Bellasi  * struct util_est - Estimation utilization of FAIR tasks
3337f65ea42SPatrick Bellasi  * @enqueued: instantaneous estimated utilization of a task/cpu
3347f65ea42SPatrick Bellasi  * @ewma:     the Exponential Weighted Moving Average (EWMA)
3357f65ea42SPatrick Bellasi  *            utilization of a task
3367f65ea42SPatrick Bellasi  *
3377f65ea42SPatrick Bellasi  * Support data structure to track an Exponential Weighted Moving Average
3387f65ea42SPatrick Bellasi  * (EWMA) of a FAIR task's utilization. New samples are added to the moving
3397f65ea42SPatrick Bellasi  * average each time a task completes an activation. Sample's weight is chosen
3407f65ea42SPatrick Bellasi  * so that the EWMA will be relatively insensitive to transient changes to the
3417f65ea42SPatrick Bellasi  * task's workload.
3427f65ea42SPatrick Bellasi  *
3437f65ea42SPatrick Bellasi  * The enqueued attribute has a slightly different meaning for tasks and cpus:
3447f65ea42SPatrick Bellasi  * - task:   the task's util_avg at last task dequeue time
3457f65ea42SPatrick Bellasi  * - cfs_rq: the sum of util_est.enqueued for each RUNNABLE task on that CPU
3467f65ea42SPatrick Bellasi  * Thus, the util_est.enqueued of a task represents the contribution on the
3477f65ea42SPatrick Bellasi  * estimated utilization of the CPU where that task is currently enqueued.
3487f65ea42SPatrick Bellasi  *
3497f65ea42SPatrick Bellasi  * Only for tasks we track a moving average of the past instantaneous
3507f65ea42SPatrick Bellasi  * estimated utilization. This allows to absorb sporadic drops in utilization
3517f65ea42SPatrick Bellasi  * of an otherwise almost periodic task.
3527f65ea42SPatrick Bellasi  */
3537f65ea42SPatrick Bellasi struct util_est {
3547f65ea42SPatrick Bellasi 	unsigned int			enqueued;
3557f65ea42SPatrick Bellasi 	unsigned int			ewma;
3567f65ea42SPatrick Bellasi #define UTIL_EST_WEIGHT_SHIFT		2
357317d359dSPeter Zijlstra } __attribute__((__aligned__(sizeof(u64))));
3587f65ea42SPatrick Bellasi 
3599d89c257SYuyang Du /*
3609f683953SVincent Guittot  * The load/runnable/util_avg accumulates an infinite geometric series
3610dacee1bSVincent Guittot  * (see __update_load_avg_cfs_rq() in kernel/sched/pelt.c).
3627b595334SYuyang Du  *
3637b595334SYuyang Du  * [load_avg definition]
3647b595334SYuyang Du  *
3657b595334SYuyang Du  *   load_avg = runnable% * scale_load_down(load)
3667b595334SYuyang Du  *
3679f683953SVincent Guittot  * [runnable_avg definition]
3689f683953SVincent Guittot  *
3699f683953SVincent Guittot  *   runnable_avg = runnable% * SCHED_CAPACITY_SCALE
3707b595334SYuyang Du  *
3717b595334SYuyang Du  * [util_avg definition]
3727b595334SYuyang Du  *
3737b595334SYuyang Du  *   util_avg = running% * SCHED_CAPACITY_SCALE
3747b595334SYuyang Du  *
3759f683953SVincent Guittot  * where runnable% is the time ratio that a sched_entity is runnable and
3769f683953SVincent Guittot  * running% the time ratio that a sched_entity is running.
3777b595334SYuyang Du  *
3789f683953SVincent Guittot  * For cfs_rq, they are the aggregated values of all runnable and blocked
3799f683953SVincent Guittot  * sched_entities.
3809f683953SVincent Guittot  *
3819f683953SVincent Guittot  * The load/runnable/util_avg doesn't direcly factor frequency scaling and CPU
3829f683953SVincent Guittot  * capacity scaling. The scaling is done through the rq_clock_pelt that is used
3839f683953SVincent Guittot  * for computing those signals (see update_rq_clock_pelt())
3847b595334SYuyang Du  *
38523127296SVincent Guittot  * N.B., the above ratios (runnable% and running%) themselves are in the
38623127296SVincent Guittot  * range of [0, 1]. To do fixed point arithmetics, we therefore scale them
38723127296SVincent Guittot  * to as large a range as necessary. This is for example reflected by
38823127296SVincent Guittot  * util_avg's SCHED_CAPACITY_SCALE.
3897b595334SYuyang Du  *
3907b595334SYuyang Du  * [Overflow issue]
3917b595334SYuyang Du  *
3927b595334SYuyang Du  * The 64-bit load_sum can have 4353082796 (=2^64/47742/88761) entities
3937b595334SYuyang Du  * with the highest load (=88761), always runnable on a single cfs_rq,
3947b595334SYuyang Du  * and should not overflow as the number already hits PID_MAX_LIMIT.
3957b595334SYuyang Du  *
3967b595334SYuyang Du  * For all other cases (including 32-bit kernels), struct load_weight's
3977b595334SYuyang Du  * weight will overflow first before we do, because:
3987b595334SYuyang Du  *
3997b595334SYuyang Du  *    Max(load_avg) <= Max(load.weight)
4007b595334SYuyang Du  *
4017b595334SYuyang Du  * Then it is the load_weight's responsibility to consider overflow
4027b595334SYuyang Du  * issues.
4039d89c257SYuyang Du  */
4049d85f21cSPaul Turner struct sched_avg {
4055eca1c10SIngo Molnar 	u64				last_update_time;
4065eca1c10SIngo Molnar 	u64				load_sum;
4079f683953SVincent Guittot 	u64				runnable_sum;
4085eca1c10SIngo Molnar 	u32				util_sum;
4095eca1c10SIngo Molnar 	u32				period_contrib;
4105eca1c10SIngo Molnar 	unsigned long			load_avg;
4119f683953SVincent Guittot 	unsigned long			runnable_avg;
4125eca1c10SIngo Molnar 	unsigned long			util_avg;
4137f65ea42SPatrick Bellasi 	struct util_est			util_est;
414317d359dSPeter Zijlstra } ____cacheline_aligned;
4159d85f21cSPaul Turner 
41641acab88SLucas De Marchi struct sched_statistics {
4177f5f8e8dSIngo Molnar #ifdef CONFIG_SCHEDSTATS
41894c18227SIngo Molnar 	u64				wait_start;
41994c18227SIngo Molnar 	u64				wait_max;
4206d082592SArjan van de Ven 	u64				wait_count;
4216d082592SArjan van de Ven 	u64				wait_sum;
4228f0dfc34SArjan van de Ven 	u64				iowait_count;
4238f0dfc34SArjan van de Ven 	u64				iowait_sum;
42494c18227SIngo Molnar 
42594c18227SIngo Molnar 	u64				sleep_start;
42620b8a59fSIngo Molnar 	u64				sleep_max;
42794c18227SIngo Molnar 	s64				sum_sleep_runtime;
42894c18227SIngo Molnar 
42994c18227SIngo Molnar 	u64				block_start;
43020b8a59fSIngo Molnar 	u64				block_max;
43120b8a59fSIngo Molnar 	u64				exec_max;
432eba1ed4bSIngo Molnar 	u64				slice_max;
433cc367732SIngo Molnar 
434cc367732SIngo Molnar 	u64				nr_migrations_cold;
435cc367732SIngo Molnar 	u64				nr_failed_migrations_affine;
436cc367732SIngo Molnar 	u64				nr_failed_migrations_running;
437cc367732SIngo Molnar 	u64				nr_failed_migrations_hot;
438cc367732SIngo Molnar 	u64				nr_forced_migrations;
439cc367732SIngo Molnar 
440cc367732SIngo Molnar 	u64				nr_wakeups;
441cc367732SIngo Molnar 	u64				nr_wakeups_sync;
442cc367732SIngo Molnar 	u64				nr_wakeups_migrate;
443cc367732SIngo Molnar 	u64				nr_wakeups_local;
444cc367732SIngo Molnar 	u64				nr_wakeups_remote;
445cc367732SIngo Molnar 	u64				nr_wakeups_affine;
446cc367732SIngo Molnar 	u64				nr_wakeups_affine_attempts;
447cc367732SIngo Molnar 	u64				nr_wakeups_passive;
448cc367732SIngo Molnar 	u64				nr_wakeups_idle;
44941acab88SLucas De Marchi #endif
4507f5f8e8dSIngo Molnar };
45141acab88SLucas De Marchi 
45241acab88SLucas De Marchi struct sched_entity {
4535eca1c10SIngo Molnar 	/* For load-balancing: */
4545eca1c10SIngo Molnar 	struct load_weight		load;
45541acab88SLucas De Marchi 	struct rb_node			run_node;
45641acab88SLucas De Marchi 	struct list_head		group_node;
45741acab88SLucas De Marchi 	unsigned int			on_rq;
45841acab88SLucas De Marchi 
45941acab88SLucas De Marchi 	u64				exec_start;
46041acab88SLucas De Marchi 	u64				sum_exec_runtime;
46141acab88SLucas De Marchi 	u64				vruntime;
46241acab88SLucas De Marchi 	u64				prev_sum_exec_runtime;
46341acab88SLucas De Marchi 
46441acab88SLucas De Marchi 	u64				nr_migrations;
46541acab88SLucas De Marchi 
46641acab88SLucas De Marchi 	struct sched_statistics		statistics;
46794c18227SIngo Molnar 
46820b8a59fSIngo Molnar #ifdef CONFIG_FAIR_GROUP_SCHED
469fed14d45SPeter Zijlstra 	int				depth;
47020b8a59fSIngo Molnar 	struct sched_entity		*parent;
47120b8a59fSIngo Molnar 	/* rq on which this entity is (to be) queued: */
47220b8a59fSIngo Molnar 	struct cfs_rq			*cfs_rq;
47320b8a59fSIngo Molnar 	/* rq "owned" by this entity/group: */
47420b8a59fSIngo Molnar 	struct cfs_rq			*my_q;
4759f683953SVincent Guittot 	/* cached value of my_q->h_nr_running */
4769f683953SVincent Guittot 	unsigned long			runnable_weight;
47720b8a59fSIngo Molnar #endif
4788bd75c77SClark Williams 
479141965c7SAlex Shi #ifdef CONFIG_SMP
4805a107804SJiri Olsa 	/*
4815a107804SJiri Olsa 	 * Per entity load average tracking.
4825a107804SJiri Olsa 	 *
4835a107804SJiri Olsa 	 * Put into separate cache line so it does not
4845a107804SJiri Olsa 	 * collide with read-mostly values above.
4855a107804SJiri Olsa 	 */
486317d359dSPeter Zijlstra 	struct sched_avg		avg;
4879d85f21cSPaul Turner #endif
48820b8a59fSIngo Molnar };
48970b97a7fSIngo Molnar 
490fa717060SPeter Zijlstra struct sched_rt_entity {
491fa717060SPeter Zijlstra 	struct list_head		run_list;
49278f2c7dbSPeter Zijlstra 	unsigned long			timeout;
49357d2aa00SYing Xue 	unsigned long			watchdog_stamp;
494bee367edSRichard Kennedy 	unsigned int			time_slice;
495ff77e468SPeter Zijlstra 	unsigned short			on_rq;
496ff77e468SPeter Zijlstra 	unsigned short			on_list;
4976f505b16SPeter Zijlstra 
49858d6c2d7SPeter Zijlstra 	struct sched_rt_entity		*back;
499052f1dc7SPeter Zijlstra #ifdef CONFIG_RT_GROUP_SCHED
5006f505b16SPeter Zijlstra 	struct sched_rt_entity		*parent;
5016f505b16SPeter Zijlstra 	/* rq on which this entity is (to be) queued: */
5026f505b16SPeter Zijlstra 	struct rt_rq			*rt_rq;
5036f505b16SPeter Zijlstra 	/* rq "owned" by this entity/group: */
5046f505b16SPeter Zijlstra 	struct rt_rq			*my_q;
5056f505b16SPeter Zijlstra #endif
5063859a271SKees Cook } __randomize_layout;
507fa717060SPeter Zijlstra 
508aab03e05SDario Faggioli struct sched_dl_entity {
509aab03e05SDario Faggioli 	struct rb_node			rb_node;
510aab03e05SDario Faggioli 
511aab03e05SDario Faggioli 	/*
512aab03e05SDario Faggioli 	 * Original scheduling parameters. Copied here from sched_attr
5134027d080Sxiaofeng.yan 	 * during sched_setattr(), they will remain the same until
5144027d080Sxiaofeng.yan 	 * the next sched_setattr().
515aab03e05SDario Faggioli 	 */
5165eca1c10SIngo Molnar 	u64				dl_runtime;	/* Maximum runtime for each instance	*/
5175eca1c10SIngo Molnar 	u64				dl_deadline;	/* Relative deadline of each instance	*/
5185eca1c10SIngo Molnar 	u64				dl_period;	/* Separation of two instances (period) */
51954d6d303SDaniel Bristot de Oliveira 	u64				dl_bw;		/* dl_runtime / dl_period		*/
5203effcb42SDaniel Bristot de Oliveira 	u64				dl_density;	/* dl_runtime / dl_deadline		*/
521aab03e05SDario Faggioli 
522aab03e05SDario Faggioli 	/*
523aab03e05SDario Faggioli 	 * Actual scheduling parameters. Initialized with the values above,
524dfcb245eSIngo Molnar 	 * they are continuously updated during task execution. Note that
525aab03e05SDario Faggioli 	 * the remaining runtime could be < 0 in case we are in overrun.
526aab03e05SDario Faggioli 	 */
5275eca1c10SIngo Molnar 	s64				runtime;	/* Remaining runtime for this instance	*/
5285eca1c10SIngo Molnar 	u64				deadline;	/* Absolute deadline for this instance	*/
5295eca1c10SIngo Molnar 	unsigned int			flags;		/* Specifying the scheduler behaviour	*/
530aab03e05SDario Faggioli 
531aab03e05SDario Faggioli 	/*
532aab03e05SDario Faggioli 	 * Some bool flags:
533aab03e05SDario Faggioli 	 *
534aab03e05SDario Faggioli 	 * @dl_throttled tells if we exhausted the runtime. If so, the
535aab03e05SDario Faggioli 	 * task has to wait for a replenishment to be performed at the
536aab03e05SDario Faggioli 	 * next firing of dl_timer.
537aab03e05SDario Faggioli 	 *
5382d3d891dSDario Faggioli 	 * @dl_boosted tells if we are boosted due to DI. If so we are
5392d3d891dSDario Faggioli 	 * outside bandwidth enforcement mechanism (but only until we
5405bfd126eSJuri Lelli 	 * exit the critical section);
5415bfd126eSJuri Lelli 	 *
5425eca1c10SIngo Molnar 	 * @dl_yielded tells if task gave up the CPU before consuming
5435bfd126eSJuri Lelli 	 * all its available runtime during the last job.
544209a0cbdSLuca Abeni 	 *
545209a0cbdSLuca Abeni 	 * @dl_non_contending tells if the task is inactive while still
546209a0cbdSLuca Abeni 	 * contributing to the active utilization. In other words, it
547209a0cbdSLuca Abeni 	 * indicates if the inactive timer has been armed and its handler
548209a0cbdSLuca Abeni 	 * has not been executed yet. This flag is useful to avoid race
549209a0cbdSLuca Abeni 	 * conditions between the inactive timer handler and the wakeup
550209a0cbdSLuca Abeni 	 * code.
55134be3930SJuri Lelli 	 *
55234be3930SJuri Lelli 	 * @dl_overrun tells if the task asked to be informed about runtime
55334be3930SJuri Lelli 	 * overruns.
554aab03e05SDario Faggioli 	 */
555aa5222e9SDan Carpenter 	unsigned int			dl_throttled      : 1;
556aa5222e9SDan Carpenter 	unsigned int			dl_boosted        : 1;
557aa5222e9SDan Carpenter 	unsigned int			dl_yielded        : 1;
558aa5222e9SDan Carpenter 	unsigned int			dl_non_contending : 1;
55934be3930SJuri Lelli 	unsigned int			dl_overrun	  : 1;
560aab03e05SDario Faggioli 
561aab03e05SDario Faggioli 	/*
562aab03e05SDario Faggioli 	 * Bandwidth enforcement timer. Each -deadline task has its
563aab03e05SDario Faggioli 	 * own bandwidth to be enforced, thus we need one timer per task.
564aab03e05SDario Faggioli 	 */
565aab03e05SDario Faggioli 	struct hrtimer			dl_timer;
566209a0cbdSLuca Abeni 
567209a0cbdSLuca Abeni 	/*
568209a0cbdSLuca Abeni 	 * Inactive timer, responsible for decreasing the active utilization
569209a0cbdSLuca Abeni 	 * at the "0-lag time". When a -deadline task blocks, it contributes
570209a0cbdSLuca Abeni 	 * to GRUB's active utilization until the "0-lag time", hence a
571209a0cbdSLuca Abeni 	 * timer is needed to decrease the active utilization at the correct
572209a0cbdSLuca Abeni 	 * time.
573209a0cbdSLuca Abeni 	 */
574209a0cbdSLuca Abeni 	struct hrtimer inactive_timer;
575aab03e05SDario Faggioli };
5768bd75c77SClark Williams 
57769842cbaSPatrick Bellasi #ifdef CONFIG_UCLAMP_TASK
57869842cbaSPatrick Bellasi /* Number of utilization clamp buckets (shorter alias) */
57969842cbaSPatrick Bellasi #define UCLAMP_BUCKETS CONFIG_UCLAMP_BUCKETS_COUNT
58069842cbaSPatrick Bellasi 
58169842cbaSPatrick Bellasi /*
58269842cbaSPatrick Bellasi  * Utilization clamp for a scheduling entity
58369842cbaSPatrick Bellasi  * @value:		clamp value "assigned" to a se
58469842cbaSPatrick Bellasi  * @bucket_id:		bucket index corresponding to the "assigned" value
585e8f14172SPatrick Bellasi  * @active:		the se is currently refcounted in a rq's bucket
586a509a7cdSPatrick Bellasi  * @user_defined:	the requested clamp value comes from user-space
58769842cbaSPatrick Bellasi  *
58869842cbaSPatrick Bellasi  * The bucket_id is the index of the clamp bucket matching the clamp value
58969842cbaSPatrick Bellasi  * which is pre-computed and stored to avoid expensive integer divisions from
59069842cbaSPatrick Bellasi  * the fast path.
591e8f14172SPatrick Bellasi  *
592e8f14172SPatrick Bellasi  * The active bit is set whenever a task has got an "effective" value assigned,
593e8f14172SPatrick Bellasi  * which can be different from the clamp value "requested" from user-space.
594e8f14172SPatrick Bellasi  * This allows to know a task is refcounted in the rq's bucket corresponding
595e8f14172SPatrick Bellasi  * to the "effective" bucket_id.
596a509a7cdSPatrick Bellasi  *
597a509a7cdSPatrick Bellasi  * The user_defined bit is set whenever a task has got a task-specific clamp
598a509a7cdSPatrick Bellasi  * value requested from userspace, i.e. the system defaults apply to this task
599a509a7cdSPatrick Bellasi  * just as a restriction. This allows to relax default clamps when a less
600a509a7cdSPatrick Bellasi  * restrictive task-specific value has been requested, thus allowing to
601a509a7cdSPatrick Bellasi  * implement a "nice" semantic. For example, a task running with a 20%
602a509a7cdSPatrick Bellasi  * default boost can still drop its own boosting to 0%.
60369842cbaSPatrick Bellasi  */
60469842cbaSPatrick Bellasi struct uclamp_se {
60569842cbaSPatrick Bellasi 	unsigned int value		: bits_per(SCHED_CAPACITY_SCALE);
60669842cbaSPatrick Bellasi 	unsigned int bucket_id		: bits_per(UCLAMP_BUCKETS);
607e8f14172SPatrick Bellasi 	unsigned int active		: 1;
608a509a7cdSPatrick Bellasi 	unsigned int user_defined	: 1;
60969842cbaSPatrick Bellasi };
61069842cbaSPatrick Bellasi #endif /* CONFIG_UCLAMP_TASK */
61169842cbaSPatrick Bellasi 
6121d082fd0SPaul E. McKenney union rcu_special {
6131d082fd0SPaul E. McKenney 	struct {
6148203d6d0SPaul E. McKenney 		u8			blocked;
6158203d6d0SPaul E. McKenney 		u8			need_qs;
61605f41571SPaul E. McKenney 		u8			exp_hint; /* Hint for performance. */
617276c4104SPaul E. McKenney 		u8			need_mb; /* Readers need smp_mb(). */
6188203d6d0SPaul E. McKenney 	} b; /* Bits. */
61905f41571SPaul E. McKenney 	u32 s; /* Set of bits. */
6201d082fd0SPaul E. McKenney };
62186848966SPaul E. McKenney 
6228dc85d54SPeter Zijlstra enum perf_event_task_context {
6238dc85d54SPeter Zijlstra 	perf_invalid_context = -1,
6248dc85d54SPeter Zijlstra 	perf_hw_context = 0,
62589a1e187SPeter Zijlstra 	perf_sw_context,
6268dc85d54SPeter Zijlstra 	perf_nr_task_contexts,
6278dc85d54SPeter Zijlstra };
6288dc85d54SPeter Zijlstra 
629eb61baf6SIngo Molnar struct wake_q_node {
630eb61baf6SIngo Molnar 	struct wake_q_node *next;
631eb61baf6SIngo Molnar };
632eb61baf6SIngo Molnar 
6331da177e4SLinus Torvalds struct task_struct {
634c65eacbeSAndy Lutomirski #ifdef CONFIG_THREAD_INFO_IN_TASK
635c65eacbeSAndy Lutomirski 	/*
636c65eacbeSAndy Lutomirski 	 * For reasons of header soup (see current_thread_info()), this
637c65eacbeSAndy Lutomirski 	 * must be the first element of task_struct.
638c65eacbeSAndy Lutomirski 	 */
639c65eacbeSAndy Lutomirski 	struct thread_info		thread_info;
640c65eacbeSAndy Lutomirski #endif
6415eca1c10SIngo Molnar 	/* -1 unrunnable, 0 runnable, >0 stopped: */
6425eca1c10SIngo Molnar 	volatile long			state;
64329e48ce8SKees Cook 
64429e48ce8SKees Cook 	/*
64529e48ce8SKees Cook 	 * This begins the randomizable portion of task_struct. Only
64629e48ce8SKees Cook 	 * scheduling-critical items should be added above here.
64729e48ce8SKees Cook 	 */
64829e48ce8SKees Cook 	randomized_struct_fields_start
64929e48ce8SKees Cook 
650f7e4217bSRoman Zippel 	void				*stack;
651ec1d2819SElena Reshetova 	refcount_t			usage;
6525eca1c10SIngo Molnar 	/* Per task flags (PF_*), defined further below: */
6535eca1c10SIngo Molnar 	unsigned int			flags;
65497dc32cdSWilliam Cohen 	unsigned int			ptrace;
6551da177e4SLinus Torvalds 
6562dd73a4fSPeter Williams #ifdef CONFIG_SMP
6573ca7a440SPeter Zijlstra 	int				on_cpu;
658*8c4890d1SPeter Zijlstra 	struct __call_single_node	wake_entry;
659c65eacbeSAndy Lutomirski #ifdef CONFIG_THREAD_INFO_IN_TASK
6605eca1c10SIngo Molnar 	/* Current CPU: */
6615eca1c10SIngo Molnar 	unsigned int			cpu;
662c65eacbeSAndy Lutomirski #endif
66363b0e9edSMike Galbraith 	unsigned int			wakee_flips;
66462470419SMichael Wang 	unsigned long			wakee_flip_decay_ts;
66563b0e9edSMike Galbraith 	struct task_struct		*last_wakee;
666ac66f547SPeter Zijlstra 
66732e839ddSMel Gorman 	/*
66832e839ddSMel Gorman 	 * recent_used_cpu is initially set as the last CPU used by a task
66932e839ddSMel Gorman 	 * that wakes affine another task. Waker/wakee relationships can
67032e839ddSMel Gorman 	 * push tasks around a CPU where each wakeup moves to the next one.
67132e839ddSMel Gorman 	 * Tracking a recently used CPU allows a quick search for a recently
67232e839ddSMel Gorman 	 * used CPU that may be idle.
67332e839ddSMel Gorman 	 */
67432e839ddSMel Gorman 	int				recent_used_cpu;
675ac66f547SPeter Zijlstra 	int				wake_cpu;
6764866cde0SNick Piggin #endif
677fd2f4419SPeter Zijlstra 	int				on_rq;
67850e645a8SIngo Molnar 
6795eca1c10SIngo Molnar 	int				prio;
6805eca1c10SIngo Molnar 	int				static_prio;
6815eca1c10SIngo Molnar 	int				normal_prio;
682c7aceabaSRichard Kennedy 	unsigned int			rt_priority;
6835eca1c10SIngo Molnar 
6845522d5d5SIngo Molnar 	const struct sched_class	*sched_class;
68520b8a59fSIngo Molnar 	struct sched_entity		se;
686fa717060SPeter Zijlstra 	struct sched_rt_entity		rt;
6878323f26cSPeter Zijlstra #ifdef CONFIG_CGROUP_SCHED
6888323f26cSPeter Zijlstra 	struct task_group		*sched_task_group;
6898323f26cSPeter Zijlstra #endif
690aab03e05SDario Faggioli 	struct sched_dl_entity		dl;
6911da177e4SLinus Torvalds 
69269842cbaSPatrick Bellasi #ifdef CONFIG_UCLAMP_TASK
693e8f14172SPatrick Bellasi 	/* Clamp values requested for a scheduling entity */
694e8f14172SPatrick Bellasi 	struct uclamp_se		uclamp_req[UCLAMP_CNT];
695e8f14172SPatrick Bellasi 	/* Effective clamp values used for a scheduling entity */
69669842cbaSPatrick Bellasi 	struct uclamp_se		uclamp[UCLAMP_CNT];
69769842cbaSPatrick Bellasi #endif
69869842cbaSPatrick Bellasi 
699e107be36SAvi Kivity #ifdef CONFIG_PREEMPT_NOTIFIERS
7005eca1c10SIngo Molnar 	/* List of struct preempt_notifier: */
701e107be36SAvi Kivity 	struct hlist_head		preempt_notifiers;
702e107be36SAvi Kivity #endif
703e107be36SAvi Kivity 
7046c5c9341SAlexey Dobriyan #ifdef CONFIG_BLK_DEV_IO_TRACE
7052056a782SJens Axboe 	unsigned int			btrace_seq;
7066c5c9341SAlexey Dobriyan #endif
7071da177e4SLinus Torvalds 
70897dc32cdSWilliam Cohen 	unsigned int			policy;
70929baa747SPeter Zijlstra 	int				nr_cpus_allowed;
7103bd37062SSebastian Andrzej Siewior 	const cpumask_t			*cpus_ptr;
7113bd37062SSebastian Andrzej Siewior 	cpumask_t			cpus_mask;
7121da177e4SLinus Torvalds 
713a57eb940SPaul E. McKenney #ifdef CONFIG_PREEMPT_RCU
714e260be67SPaul E. McKenney 	int				rcu_read_lock_nesting;
7151d082fd0SPaul E. McKenney 	union rcu_special		rcu_read_unlock_special;
716f41d911fSPaul E. McKenney 	struct list_head		rcu_node_entry;
717a57eb940SPaul E. McKenney 	struct rcu_node			*rcu_blocked_node;
71828f6569aSPranith Kumar #endif /* #ifdef CONFIG_PREEMPT_RCU */
7195eca1c10SIngo Molnar 
7208315f422SPaul E. McKenney #ifdef CONFIG_TASKS_RCU
7218315f422SPaul E. McKenney 	unsigned long			rcu_tasks_nvcsw;
722ccdd29ffSPaul E. McKenney 	u8				rcu_tasks_holdout;
723ccdd29ffSPaul E. McKenney 	u8				rcu_tasks_idx;
724176f8f7aSPaul E. McKenney 	int				rcu_tasks_idle_cpu;
725ccdd29ffSPaul E. McKenney 	struct list_head		rcu_tasks_holdout_list;
7268315f422SPaul E. McKenney #endif /* #ifdef CONFIG_TASKS_RCU */
727e260be67SPaul E. McKenney 
728d5f177d3SPaul E. McKenney #ifdef CONFIG_TASKS_TRACE_RCU
729d5f177d3SPaul E. McKenney 	int				trc_reader_nesting;
730d5f177d3SPaul E. McKenney 	int				trc_ipi_to_cpu;
731276c4104SPaul E. McKenney 	union rcu_special		trc_reader_special;
732d5f177d3SPaul E. McKenney 	bool				trc_reader_checked;
733d5f177d3SPaul E. McKenney 	struct list_head		trc_holdout_list;
734d5f177d3SPaul E. McKenney #endif /* #ifdef CONFIG_TASKS_TRACE_RCU */
735d5f177d3SPaul E. McKenney 
7361da177e4SLinus Torvalds 	struct sched_info		sched_info;
7371da177e4SLinus Torvalds 
7381da177e4SLinus Torvalds 	struct list_head		tasks;
739806c09a7SDario Faggioli #ifdef CONFIG_SMP
740917b627dSGregory Haskins 	struct plist_node		pushable_tasks;
7411baca4ceSJuri Lelli 	struct rb_node			pushable_dl_tasks;
742806c09a7SDario Faggioli #endif
7431da177e4SLinus Torvalds 
7445eca1c10SIngo Molnar 	struct mm_struct		*mm;
7455eca1c10SIngo Molnar 	struct mm_struct		*active_mm;
746314ff785SIngo Molnar 
747314ff785SIngo Molnar 	/* Per-thread vma caching: */
748314ff785SIngo Molnar 	struct vmacache			vmacache;
749314ff785SIngo Molnar 
7505eca1c10SIngo Molnar #ifdef SPLIT_RSS_COUNTING
75134e55232SKAMEZAWA Hiroyuki 	struct task_rss_stat		rss_stat;
75234e55232SKAMEZAWA Hiroyuki #endif
75397dc32cdSWilliam Cohen 	int				exit_state;
7545eca1c10SIngo Molnar 	int				exit_code;
7555eca1c10SIngo Molnar 	int				exit_signal;
7565eca1c10SIngo Molnar 	/* The signal sent when the parent dies: */
7575eca1c10SIngo Molnar 	int				pdeath_signal;
7585eca1c10SIngo Molnar 	/* JOBCTL_*, siglock protected: */
7595eca1c10SIngo Molnar 	unsigned long			jobctl;
7609b89f6baSAndrei Epure 
7615eca1c10SIngo Molnar 	/* Used for emulating ABI behavior of previous Linux versions: */
76297dc32cdSWilliam Cohen 	unsigned int			personality;
7639b89f6baSAndrei Epure 
7645eca1c10SIngo Molnar 	/* Scheduler bits, serialized by scheduler locks: */
765ca94c442SLennart Poettering 	unsigned			sched_reset_on_fork:1;
766a8e4f2eaSPeter Zijlstra 	unsigned			sched_contributes_to_load:1;
767ff303e66SPeter Zijlstra 	unsigned			sched_migrated:1;
768b7e7ade3SPeter Zijlstra 	unsigned			sched_remote_wakeup:1;
769eb414681SJohannes Weiner #ifdef CONFIG_PSI
770eb414681SJohannes Weiner 	unsigned			sched_psi_wake_requeue:1;
771eb414681SJohannes Weiner #endif
772eb414681SJohannes Weiner 
7735eca1c10SIngo Molnar 	/* Force alignment to the next boundary: */
7745eca1c10SIngo Molnar 	unsigned			:0;
775be958bdcSPeter Zijlstra 
7765eca1c10SIngo Molnar 	/* Unserialized, strictly 'current' */
7775eca1c10SIngo Molnar 
7785eca1c10SIngo Molnar 	/* Bit to tell LSMs we're in execve(): */
7795eca1c10SIngo Molnar 	unsigned			in_execve:1;
780be958bdcSPeter Zijlstra 	unsigned			in_iowait:1;
7815eca1c10SIngo Molnar #ifndef TIF_RESTORE_SIGMASK
7827e781418SAndy Lutomirski 	unsigned			restore_sigmask:1;
7837e781418SAndy Lutomirski #endif
784626ebc41STejun Heo #ifdef CONFIG_MEMCG
78529ef680aSMichal Hocko 	unsigned			in_user_fault:1;
786127424c8SJohannes Weiner #endif
787ff303e66SPeter Zijlstra #ifdef CONFIG_COMPAT_BRK
788ff303e66SPeter Zijlstra 	unsigned			brk_randomized:1;
789ff303e66SPeter Zijlstra #endif
79077f88796STejun Heo #ifdef CONFIG_CGROUPS
79177f88796STejun Heo 	/* disallow userland-initiated cgroup migration */
79277f88796STejun Heo 	unsigned			no_cgroup_migration:1;
79376f969e8SRoman Gushchin 	/* task is frozen/stopped (used by the cgroup freezer) */
79476f969e8SRoman Gushchin 	unsigned			frozen:1;
79577f88796STejun Heo #endif
796d09d8df3SJosef Bacik #ifdef CONFIG_BLK_CGROUP
797d09d8df3SJosef Bacik 	unsigned			use_memdelay:1;
798d09d8df3SJosef Bacik #endif
7991066d1b6SYafang Shao #ifdef CONFIG_PSI
8001066d1b6SYafang Shao 	/* Stalled due to lack of memory */
8011066d1b6SYafang Shao 	unsigned			in_memstall:1;
8021066d1b6SYafang Shao #endif
8036f185c29SVladimir Davydov 
8045eca1c10SIngo Molnar 	unsigned long			atomic_flags; /* Flags requiring atomic access. */
8051d4457f9SKees Cook 
806f56141e3SAndy Lutomirski 	struct restart_block		restart_block;
807f56141e3SAndy Lutomirski 
8081da177e4SLinus Torvalds 	pid_t				pid;
8091da177e4SLinus Torvalds 	pid_t				tgid;
8100a425405SArjan van de Ven 
811050e9baaSLinus Torvalds #ifdef CONFIG_STACKPROTECTOR
8125eca1c10SIngo Molnar 	/* Canary value for the -fstack-protector GCC feature: */
8130a425405SArjan van de Ven 	unsigned long			stack_canary;
8141314562aSHiroshi Shimamoto #endif
8151da177e4SLinus Torvalds 	/*
8165eca1c10SIngo Molnar 	 * Pointers to the (original) parent process, youngest child, younger sibling,
8171da177e4SLinus Torvalds 	 * older sibling, respectively.  (p->father can be replaced with
818f470021aSRoland McGrath 	 * p->real_parent->pid)
8191da177e4SLinus Torvalds 	 */
8205eca1c10SIngo Molnar 
8215eca1c10SIngo Molnar 	/* Real parent process: */
8225eca1c10SIngo Molnar 	struct task_struct __rcu	*real_parent;
8235eca1c10SIngo Molnar 
8245eca1c10SIngo Molnar 	/* Recipient of SIGCHLD, wait4() reports: */
8255eca1c10SIngo Molnar 	struct task_struct __rcu	*parent;
8261da177e4SLinus Torvalds 
827f470021aSRoland McGrath 	/*
8285eca1c10SIngo Molnar 	 * Children/sibling form the list of natural children:
8295eca1c10SIngo Molnar 	 */
8305eca1c10SIngo Molnar 	struct list_head		children;
8315eca1c10SIngo Molnar 	struct list_head		sibling;
8325eca1c10SIngo Molnar 	struct task_struct		*group_leader;
8335eca1c10SIngo Molnar 
8345eca1c10SIngo Molnar 	/*
8355eca1c10SIngo Molnar 	 * 'ptraced' is the list of tasks this task is using ptrace() on.
8365eca1c10SIngo Molnar 	 *
837f470021aSRoland McGrath 	 * This includes both natural children and PTRACE_ATTACH targets.
8385eca1c10SIngo Molnar 	 * 'ptrace_entry' is this task's link on the p->parent->ptraced list.
839f470021aSRoland McGrath 	 */
840f470021aSRoland McGrath 	struct list_head		ptraced;
841f470021aSRoland McGrath 	struct list_head		ptrace_entry;
842f470021aSRoland McGrath 
8431da177e4SLinus Torvalds 	/* PID/PID hash table linkage. */
8442c470475SEric W. Biederman 	struct pid			*thread_pid;
8452c470475SEric W. Biederman 	struct hlist_node		pid_links[PIDTYPE_MAX];
84647e65328SOleg Nesterov 	struct list_head		thread_group;
8470c740d0aSOleg Nesterov 	struct list_head		thread_node;
8481da177e4SLinus Torvalds 
8495eca1c10SIngo Molnar 	struct completion		*vfork_done;
8501da177e4SLinus Torvalds 
8515eca1c10SIngo Molnar 	/* CLONE_CHILD_SETTID: */
8525eca1c10SIngo Molnar 	int __user			*set_child_tid;
8535eca1c10SIngo Molnar 
8545eca1c10SIngo Molnar 	/* CLONE_CHILD_CLEARTID: */
8555eca1c10SIngo Molnar 	int __user			*clear_child_tid;
8565eca1c10SIngo Molnar 
8575eca1c10SIngo Molnar 	u64				utime;
8585eca1c10SIngo Molnar 	u64				stime;
85940565b5aSStanislaw Gruszka #ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
8605eca1c10SIngo Molnar 	u64				utimescaled;
8615eca1c10SIngo Molnar 	u64				stimescaled;
86240565b5aSStanislaw Gruszka #endif
86316a6d9beSFrederic Weisbecker 	u64				gtime;
8649d7fb042SPeter Zijlstra 	struct prev_cputime		prev_cputime;
8656a61671bSFrederic Weisbecker #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
866bac5b6b6SFrederic Weisbecker 	struct vtime			vtime;
8676a61671bSFrederic Weisbecker #endif
868d027d45dSFrederic Weisbecker 
869d027d45dSFrederic Weisbecker #ifdef CONFIG_NO_HZ_FULL
870f009a7a7SFrederic Weisbecker 	atomic_t			tick_dep_mask;
871d027d45dSFrederic Weisbecker #endif
8725eca1c10SIngo Molnar 	/* Context switch counts: */
8735eca1c10SIngo Molnar 	unsigned long			nvcsw;
8745eca1c10SIngo Molnar 	unsigned long			nivcsw;
8755eca1c10SIngo Molnar 
8765eca1c10SIngo Molnar 	/* Monotonic time in nsecs: */
8775eca1c10SIngo Molnar 	u64				start_time;
8785eca1c10SIngo Molnar 
8795eca1c10SIngo Molnar 	/* Boot based time in nsecs: */
880cf25e24dSPeter Zijlstra 	u64				start_boottime;
8815eca1c10SIngo Molnar 
8825eca1c10SIngo Molnar 	/* MM fault and swap info: this can arguably be seen as either mm-specific or thread-specific: */
8835eca1c10SIngo Molnar 	unsigned long			min_flt;
8845eca1c10SIngo Molnar 	unsigned long			maj_flt;
8851da177e4SLinus Torvalds 
8862b69942fSThomas Gleixner 	/* Empty if CONFIG_POSIX_CPUTIMERS=n */
8872b69942fSThomas Gleixner 	struct posix_cputimers		posix_cputimers;
8881da177e4SLinus Torvalds 
8895eca1c10SIngo Molnar 	/* Process credentials: */
8905eca1c10SIngo Molnar 
8915eca1c10SIngo Molnar 	/* Tracer's credentials at attach: */
8925eca1c10SIngo Molnar 	const struct cred __rcu		*ptracer_cred;
8935eca1c10SIngo Molnar 
8945eca1c10SIngo Molnar 	/* Objective and real subjective task credentials (COW): */
8955eca1c10SIngo Molnar 	const struct cred __rcu		*real_cred;
8965eca1c10SIngo Molnar 
8975eca1c10SIngo Molnar 	/* Effective (overridable) subjective task credentials (COW): */
8985eca1c10SIngo Molnar 	const struct cred __rcu		*cred;
8995eca1c10SIngo Molnar 
9007743c48eSDavid Howells #ifdef CONFIG_KEYS
9017743c48eSDavid Howells 	/* Cached requested key. */
9027743c48eSDavid Howells 	struct key			*cached_requested_key;
9037743c48eSDavid Howells #endif
9047743c48eSDavid Howells 
9055eca1c10SIngo Molnar 	/*
9065eca1c10SIngo Molnar 	 * executable name, excluding path.
9075eca1c10SIngo Molnar 	 *
9085eca1c10SIngo Molnar 	 * - normally initialized setup_new_exec()
9095eca1c10SIngo Molnar 	 * - access it with [gs]et_task_comm()
9105eca1c10SIngo Molnar 	 * - lock it with task_lock()
9115eca1c10SIngo Molnar 	 */
9125eca1c10SIngo Molnar 	char				comm[TASK_COMM_LEN];
9135eca1c10SIngo Molnar 
914756daf26SNeilBrown 	struct nameidata		*nameidata;
9155eca1c10SIngo Molnar 
9163d5b6fccSAlexey Dobriyan #ifdef CONFIG_SYSVIPC
9171da177e4SLinus Torvalds 	struct sysv_sem			sysvsem;
918ab602f79SJack Miller 	struct sysv_shm			sysvshm;
9193d5b6fccSAlexey Dobriyan #endif
920e162b39aSMandeep Singh Baines #ifdef CONFIG_DETECT_HUNG_TASK
92182a1fcb9SIngo Molnar 	unsigned long			last_switch_count;
922a2e51445SDmitry Vyukov 	unsigned long			last_switch_time;
92382a1fcb9SIngo Molnar #endif
9245eca1c10SIngo Molnar 	/* Filesystem information: */
9251da177e4SLinus Torvalds 	struct fs_struct		*fs;
9265eca1c10SIngo Molnar 
9275eca1c10SIngo Molnar 	/* Open file information: */
9281da177e4SLinus Torvalds 	struct files_struct		*files;
9295eca1c10SIngo Molnar 
9305eca1c10SIngo Molnar 	/* Namespaces: */
931ab516013SSerge E. Hallyn 	struct nsproxy			*nsproxy;
9325eca1c10SIngo Molnar 
9335eca1c10SIngo Molnar 	/* Signal handlers: */
9341da177e4SLinus Torvalds 	struct signal_struct		*signal;
935913292c9SMadhuparna Bhowmik 	struct sighand_struct __rcu		*sighand;
9365eca1c10SIngo Molnar 	sigset_t			blocked;
9375eca1c10SIngo Molnar 	sigset_t			real_blocked;
9385eca1c10SIngo Molnar 	/* Restored if set_restore_sigmask() was used: */
9395eca1c10SIngo Molnar 	sigset_t			saved_sigmask;
9401da177e4SLinus Torvalds 	struct sigpending		pending;
9411da177e4SLinus Torvalds 	unsigned long			sas_ss_sp;
9421da177e4SLinus Torvalds 	size_t				sas_ss_size;
9435eca1c10SIngo Molnar 	unsigned int			sas_ss_flags;
9442e01fabeSOleg Nesterov 
94567d12145SAl Viro 	struct callback_head		*task_works;
946e73f8959SOleg Nesterov 
9474b7d248bSRichard Guy Briggs #ifdef CONFIG_AUDIT
948bfef93a5SAl Viro #ifdef CONFIG_AUDITSYSCALL
9495f3d544fSRichard Guy Briggs 	struct audit_context		*audit_context;
9505f3d544fSRichard Guy Briggs #endif
951e1760bd5SEric W. Biederman 	kuid_t				loginuid;
9524746ec5bSEric Paris 	unsigned int			sessionid;
953bfef93a5SAl Viro #endif
954932ecebbSWill Drewry 	struct seccomp			seccomp;
9551da177e4SLinus Torvalds 
9565eca1c10SIngo Molnar 	/* Thread group tracking: */
957d1e7fd64SEric W. Biederman 	u64				parent_exec_id;
958d1e7fd64SEric W. Biederman 	u64				self_exec_id;
9595eca1c10SIngo Molnar 
9605eca1c10SIngo Molnar 	/* Protection against (de-)allocation: mm, files, fs, tty, keyrings, mems_allowed, mempolicy: */
9611da177e4SLinus Torvalds 	spinlock_t			alloc_lock;
9621da177e4SLinus Torvalds 
963b29739f9SIngo Molnar 	/* Protection of the PI data structures: */
9641d615482SThomas Gleixner 	raw_spinlock_t			pi_lock;
965b29739f9SIngo Molnar 
96676751049SPeter Zijlstra 	struct wake_q_node		wake_q;
96776751049SPeter Zijlstra 
96823f78d4aSIngo Molnar #ifdef CONFIG_RT_MUTEXES
9695eca1c10SIngo Molnar 	/* PI waiters blocked on a rt_mutex held by this task: */
970a23ba907SDavidlohr Bueso 	struct rb_root_cached		pi_waiters;
971e96a7705SXunlei Pang 	/* Updated under owner's pi_lock and rq lock */
972e96a7705SXunlei Pang 	struct task_struct		*pi_top_task;
9735eca1c10SIngo Molnar 	/* Deadlock detection and priority inheritance handling: */
97423f78d4aSIngo Molnar 	struct rt_mutex_waiter		*pi_blocked_on;
97523f78d4aSIngo Molnar #endif
97623f78d4aSIngo Molnar 
977408894eeSIngo Molnar #ifdef CONFIG_DEBUG_MUTEXES
9785eca1c10SIngo Molnar 	/* Mutex deadlock detection: */
979408894eeSIngo Molnar 	struct mutex_waiter		*blocked_on;
980408894eeSIngo Molnar #endif
9815eca1c10SIngo Molnar 
982312364f3SDaniel Vetter #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
983312364f3SDaniel Vetter 	int				non_block_count;
984312364f3SDaniel Vetter #endif
985312364f3SDaniel Vetter 
986de30a2b3SIngo Molnar #ifdef CONFIG_TRACE_IRQFLAGS
987de30a2b3SIngo Molnar 	unsigned int			irq_events;
988de8f5e4fSPeter Zijlstra 	unsigned int			hardirq_threaded;
989de30a2b3SIngo Molnar 	unsigned long			hardirq_enable_ip;
990de30a2b3SIngo Molnar 	unsigned long			hardirq_disable_ip;
991fa1452e8SHiroshi Shimamoto 	unsigned int			hardirq_enable_event;
992de30a2b3SIngo Molnar 	unsigned int			hardirq_disable_event;
993fa1452e8SHiroshi Shimamoto 	int				hardirqs_enabled;
994de30a2b3SIngo Molnar 	int				hardirq_context;
995c86e9b98SPeter Zijlstra 	u64				hardirq_chain_key;
996fa1452e8SHiroshi Shimamoto 	unsigned long			softirq_disable_ip;
997fa1452e8SHiroshi Shimamoto 	unsigned long			softirq_enable_ip;
998fa1452e8SHiroshi Shimamoto 	unsigned int			softirq_disable_event;
999fa1452e8SHiroshi Shimamoto 	unsigned int			softirq_enable_event;
1000fa1452e8SHiroshi Shimamoto 	int				softirqs_enabled;
1001de30a2b3SIngo Molnar 	int				softirq_context;
100240db1739SSebastian Andrzej Siewior 	int				irq_config;
1003de30a2b3SIngo Molnar #endif
10045eca1c10SIngo Molnar 
1005fbb9ce95SIngo Molnar #ifdef CONFIG_LOCKDEP
1006bdb9441eSPeter Zijlstra # define MAX_LOCK_DEPTH			48UL
1007fbb9ce95SIngo Molnar 	u64				curr_chain_key;
1008fbb9ce95SIngo Molnar 	int				lockdep_depth;
1009fbb9ce95SIngo Molnar 	unsigned int			lockdep_recursion;
1010c7aceabaSRichard Kennedy 	struct held_lock		held_locks[MAX_LOCK_DEPTH];
1011fbb9ce95SIngo Molnar #endif
10125eca1c10SIngo Molnar 
1013c6d30853SAndrey Ryabinin #ifdef CONFIG_UBSAN
1014c6d30853SAndrey Ryabinin 	unsigned int			in_ubsan;
1015c6d30853SAndrey Ryabinin #endif
1016408894eeSIngo Molnar 
10175eca1c10SIngo Molnar 	/* Journalling filesystem info: */
10181da177e4SLinus Torvalds 	void				*journal_info;
10191da177e4SLinus Torvalds 
10205eca1c10SIngo Molnar 	/* Stacked block device info: */
1021bddd87c7SAkinobu Mita 	struct bio_list			*bio_list;
1022d89d8796SNeil Brown 
102373c10101SJens Axboe #ifdef CONFIG_BLOCK
10245eca1c10SIngo Molnar 	/* Stack plugging: */
102573c10101SJens Axboe 	struct blk_plug			*plug;
102673c10101SJens Axboe #endif
102773c10101SJens Axboe 
10285eca1c10SIngo Molnar 	/* VM state: */
10291da177e4SLinus Torvalds 	struct reclaim_state		*reclaim_state;
10301da177e4SLinus Torvalds 
10311da177e4SLinus Torvalds 	struct backing_dev_info		*backing_dev_info;
10321da177e4SLinus Torvalds 
10331da177e4SLinus Torvalds 	struct io_context		*io_context;
10341da177e4SLinus Torvalds 
10355e1f0f09SMel Gorman #ifdef CONFIG_COMPACTION
10365e1f0f09SMel Gorman 	struct capture_control		*capture_control;
10375e1f0f09SMel Gorman #endif
10385eca1c10SIngo Molnar 	/* Ptrace state: */
10391da177e4SLinus Torvalds 	unsigned long			ptrace_message;
1040ae7795bcSEric W. Biederman 	kernel_siginfo_t		*last_siginfo;
10415eca1c10SIngo Molnar 
10427c3ab738SAndrew Morton 	struct task_io_accounting	ioac;
1043eb414681SJohannes Weiner #ifdef CONFIG_PSI
1044eb414681SJohannes Weiner 	/* Pressure stall state */
1045eb414681SJohannes Weiner 	unsigned int			psi_flags;
1046eb414681SJohannes Weiner #endif
10475eca1c10SIngo Molnar #ifdef CONFIG_TASK_XACCT
10485eca1c10SIngo Molnar 	/* Accumulated RSS usage: */
10495eca1c10SIngo Molnar 	u64				acct_rss_mem1;
10505eca1c10SIngo Molnar 	/* Accumulated virtual memory usage: */
10515eca1c10SIngo Molnar 	u64				acct_vm_mem1;
10525eca1c10SIngo Molnar 	/* stime + utime since last update: */
10535eca1c10SIngo Molnar 	u64				acct_timexpd;
10541da177e4SLinus Torvalds #endif
10551da177e4SLinus Torvalds #ifdef CONFIG_CPUSETS
10565eca1c10SIngo Molnar 	/* Protected by ->alloc_lock: */
10575eca1c10SIngo Molnar 	nodemask_t			mems_allowed;
10585eca1c10SIngo Molnar 	/* Seqence number to catch updates: */
10595eca1c10SIngo Molnar 	seqcount_t			mems_allowed_seq;
1060825a46afSPaul Jackson 	int				cpuset_mem_spread_rotor;
10616adef3ebSJack Steiner 	int				cpuset_slab_spread_rotor;
10621da177e4SLinus Torvalds #endif
1063ddbcc7e8SPaul Menage #ifdef CONFIG_CGROUPS
10645eca1c10SIngo Molnar 	/* Control Group info protected by css_set_lock: */
10652c392b8cSArnd Bergmann 	struct css_set __rcu		*cgroups;
10665eca1c10SIngo Molnar 	/* cg_list protected by css_set_lock and tsk->alloc_lock: */
1067817929ecSPaul Menage 	struct list_head		cg_list;
1068ddbcc7e8SPaul Menage #endif
1069e6d42931SJohannes Weiner #ifdef CONFIG_X86_CPU_RESCTRL
10700734ded1SVikas Shivappa 	u32				closid;
1071d6aaba61SVikas Shivappa 	u32				rmid;
1072e02737d5SFenghua Yu #endif
107342b2dd0aSAlexey Dobriyan #ifdef CONFIG_FUTEX
10740771dfefSIngo Molnar 	struct robust_list_head __user	*robust_list;
107534f192c6SIngo Molnar #ifdef CONFIG_COMPAT
107634f192c6SIngo Molnar 	struct compat_robust_list_head __user *compat_robust_list;
107734f192c6SIngo Molnar #endif
1078c87e2837SIngo Molnar 	struct list_head		pi_state_list;
1079c87e2837SIngo Molnar 	struct futex_pi_state		*pi_state_cache;
10803f186d97SThomas Gleixner 	struct mutex			futex_exit_mutex;
10813d4775dfSThomas Gleixner 	unsigned int			futex_state;
108242b2dd0aSAlexey Dobriyan #endif
1083cdd6c482SIngo Molnar #ifdef CONFIG_PERF_EVENTS
10848dc85d54SPeter Zijlstra 	struct perf_event_context	*perf_event_ctxp[perf_nr_task_contexts];
1085cdd6c482SIngo Molnar 	struct mutex			perf_event_mutex;
1086cdd6c482SIngo Molnar 	struct list_head		perf_event_list;
1087a63eaf34SPaul Mackerras #endif
10888f47b187SThomas Gleixner #ifdef CONFIG_DEBUG_PREEMPT
10898f47b187SThomas Gleixner 	unsigned long			preempt_disable_ip;
10908f47b187SThomas Gleixner #endif
1091c7aceabaSRichard Kennedy #ifdef CONFIG_NUMA
10925eca1c10SIngo Molnar 	/* Protected by alloc_lock: */
10935eca1c10SIngo Molnar 	struct mempolicy		*mempolicy;
109445816682SVlastimil Babka 	short				il_prev;
1095207205a2SEric Dumazet 	short				pref_node_fork;
1096c7aceabaSRichard Kennedy #endif
1097cbee9f88SPeter Zijlstra #ifdef CONFIG_NUMA_BALANCING
1098cbee9f88SPeter Zijlstra 	int				numa_scan_seq;
1099cbee9f88SPeter Zijlstra 	unsigned int			numa_scan_period;
1100598f0ec0SMel Gorman 	unsigned int			numa_scan_period_max;
1101de1c9ce6SRik van Riel 	int				numa_preferred_nid;
11026b9a7460SMel Gorman 	unsigned long			numa_migrate_retry;
11035eca1c10SIngo Molnar 	/* Migration stamp: */
11045eca1c10SIngo Molnar 	u64				node_stamp;
11057e2703e6SRik van Riel 	u64				last_task_numa_placement;
11067e2703e6SRik van Riel 	u64				last_sum_exec_runtime;
1107cbee9f88SPeter Zijlstra 	struct callback_head		numa_work;
1108f809ca9aSMel Gorman 
1109cb361d8cSJann Horn 	/*
1110cb361d8cSJann Horn 	 * This pointer is only modified for current in syscall and
1111cb361d8cSJann Horn 	 * pagefault context (and for tasks being destroyed), so it can be read
1112cb361d8cSJann Horn 	 * from any of the following contexts:
1113cb361d8cSJann Horn 	 *  - RCU read-side critical section
1114cb361d8cSJann Horn 	 *  - current->numa_group from everywhere
1115cb361d8cSJann Horn 	 *  - task's runqueue locked, task not running
1116cb361d8cSJann Horn 	 */
1117cb361d8cSJann Horn 	struct numa_group __rcu		*numa_group;
11188c8a743cSPeter Zijlstra 
1119745d6147SMel Gorman 	/*
112044dba3d5SIulia Manda 	 * numa_faults is an array split into four regions:
112144dba3d5SIulia Manda 	 * faults_memory, faults_cpu, faults_memory_buffer, faults_cpu_buffer
112244dba3d5SIulia Manda 	 * in this precise order.
112344dba3d5SIulia Manda 	 *
112444dba3d5SIulia Manda 	 * faults_memory: Exponential decaying average of faults on a per-node
112544dba3d5SIulia Manda 	 * basis. Scheduling placement decisions are made based on these
112644dba3d5SIulia Manda 	 * counts. The values remain static for the duration of a PTE scan.
112744dba3d5SIulia Manda 	 * faults_cpu: Track the nodes the process was running on when a NUMA
112844dba3d5SIulia Manda 	 * hinting fault was incurred.
112944dba3d5SIulia Manda 	 * faults_memory_buffer and faults_cpu_buffer: Record faults per node
113044dba3d5SIulia Manda 	 * during the current scan window. When the scan completes, the counts
113144dba3d5SIulia Manda 	 * in faults_memory and faults_cpu decay and these values are copied.
1132745d6147SMel Gorman 	 */
113344dba3d5SIulia Manda 	unsigned long			*numa_faults;
113483e1d2cdSMel Gorman 	unsigned long			total_numa_faults;
1135745d6147SMel Gorman 
1136745d6147SMel Gorman 	/*
113704bb2f94SRik van Riel 	 * numa_faults_locality tracks if faults recorded during the last
1138074c2381SMel Gorman 	 * scan window were remote/local or failed to migrate. The task scan
1139074c2381SMel Gorman 	 * period is adapted based on the locality of the faults with different
1140074c2381SMel Gorman 	 * weights depending on whether they were shared or private faults
114104bb2f94SRik van Riel 	 */
1142074c2381SMel Gorman 	unsigned long			numa_faults_locality[3];
114304bb2f94SRik van Riel 
1144b32e86b4SIngo Molnar 	unsigned long			numa_pages_migrated;
1145cbee9f88SPeter Zijlstra #endif /* CONFIG_NUMA_BALANCING */
1146cbee9f88SPeter Zijlstra 
1147d7822b1eSMathieu Desnoyers #ifdef CONFIG_RSEQ
1148d7822b1eSMathieu Desnoyers 	struct rseq __user *rseq;
1149d7822b1eSMathieu Desnoyers 	u32 rseq_sig;
1150d7822b1eSMathieu Desnoyers 	/*
1151d7822b1eSMathieu Desnoyers 	 * RmW on rseq_event_mask must be performed atomically
1152d7822b1eSMathieu Desnoyers 	 * with respect to preemption.
1153d7822b1eSMathieu Desnoyers 	 */
1154d7822b1eSMathieu Desnoyers 	unsigned long rseq_event_mask;
1155d7822b1eSMathieu Desnoyers #endif
1156d7822b1eSMathieu Desnoyers 
115772b252aeSMel Gorman 	struct tlbflush_unmap_batch	tlb_ubc;
115872b252aeSMel Gorman 
11593fbd7ee2SEric W. Biederman 	union {
11603fbd7ee2SEric W. Biederman 		refcount_t		rcu_users;
1161e56d0903SIngo Molnar 		struct rcu_head		rcu;
11623fbd7ee2SEric W. Biederman 	};
1163b92ce558SJens Axboe 
11645eca1c10SIngo Molnar 	/* Cache last used pipe for splice(): */
1165b92ce558SJens Axboe 	struct pipe_inode_info		*splice_pipe;
11665640f768SEric Dumazet 
11675640f768SEric Dumazet 	struct page_frag		task_frag;
11685640f768SEric Dumazet 
1169ca74e92bSShailabh Nagar #ifdef CONFIG_TASK_DELAY_ACCT
1170ca74e92bSShailabh Nagar 	struct task_delay_info		*delays;
1171ca74e92bSShailabh Nagar #endif
117247913d4eSIngo Molnar 
1173f4f154fdSAkinobu Mita #ifdef CONFIG_FAULT_INJECTION
1174f4f154fdSAkinobu Mita 	int				make_it_fail;
11759049f2f6SAkinobu Mita 	unsigned int			fail_nth;
1176f4f154fdSAkinobu Mita #endif
11779d823e8fSWu Fengguang 	/*
11785eca1c10SIngo Molnar 	 * When (nr_dirtied >= nr_dirtied_pause), it's time to call
11795eca1c10SIngo Molnar 	 * balance_dirty_pages() for a dirty throttling pause:
11809d823e8fSWu Fengguang 	 */
11819d823e8fSWu Fengguang 	int				nr_dirtied;
11829d823e8fSWu Fengguang 	int				nr_dirtied_pause;
11835eca1c10SIngo Molnar 	/* Start of a write-and-pause period: */
11845eca1c10SIngo Molnar 	unsigned long			dirty_paused_when;
11859d823e8fSWu Fengguang 
11869745512cSArjan van de Ven #ifdef CONFIG_LATENCYTOP
11879745512cSArjan van de Ven 	int				latency_record_count;
11889745512cSArjan van de Ven 	struct latency_record		latency_record[LT_SAVECOUNT];
11899745512cSArjan van de Ven #endif
11906976675dSArjan van de Ven 	/*
11915eca1c10SIngo Molnar 	 * Time slack values; these are used to round up poll() and
11926976675dSArjan van de Ven 	 * select() etc timeout values. These are in nanoseconds.
11936976675dSArjan van de Ven 	 */
1194da8b44d5SJohn Stultz 	u64				timer_slack_ns;
1195da8b44d5SJohn Stultz 	u64				default_timer_slack_ns;
1196f8d570a4SDavid Miller 
11970b24beccSAndrey Ryabinin #ifdef CONFIG_KASAN
11980b24beccSAndrey Ryabinin 	unsigned int			kasan_depth;
11990b24beccSAndrey Ryabinin #endif
1200dfd402a4SMarco Elver #ifdef CONFIG_KCSAN
1201dfd402a4SMarco Elver 	struct kcsan_ctx		kcsan_ctx;
1202dfd402a4SMarco Elver #endif
12035eca1c10SIngo Molnar 
1204fb52607aSFrederic Weisbecker #ifdef CONFIG_FUNCTION_GRAPH_TRACER
12055eca1c10SIngo Molnar 	/* Index of current stored address in ret_stack: */
1206f201ae23SFrederic Weisbecker 	int				curr_ret_stack;
120739eb456dSSteven Rostedt (VMware) 	int				curr_ret_depth;
12085eca1c10SIngo Molnar 
12095eca1c10SIngo Molnar 	/* Stack of return addresses for return function tracing: */
1210f201ae23SFrederic Weisbecker 	struct ftrace_ret_stack		*ret_stack;
12115eca1c10SIngo Molnar 
12125eca1c10SIngo Molnar 	/* Timestamp for last schedule: */
12138aef2d28SSteven Rostedt 	unsigned long long		ftrace_timestamp;
12145eca1c10SIngo Molnar 
1215f201ae23SFrederic Weisbecker 	/*
1216f201ae23SFrederic Weisbecker 	 * Number of functions that haven't been traced
12175eca1c10SIngo Molnar 	 * because of depth overrun:
1218f201ae23SFrederic Weisbecker 	 */
1219f201ae23SFrederic Weisbecker 	atomic_t			trace_overrun;
12205eca1c10SIngo Molnar 
12215eca1c10SIngo Molnar 	/* Pause tracing: */
1222380c4b14SFrederic Weisbecker 	atomic_t			tracing_graph_pause;
1223f201ae23SFrederic Weisbecker #endif
12245eca1c10SIngo Molnar 
1225ea4e2bc4SSteven Rostedt #ifdef CONFIG_TRACING
12265eca1c10SIngo Molnar 	/* State flags for use by tracers: */
1227ea4e2bc4SSteven Rostedt 	unsigned long			trace;
12285eca1c10SIngo Molnar 
12295eca1c10SIngo Molnar 	/* Bitmask and counter of trace recursion: */
1230261842b7SSteven Rostedt 	unsigned long			trace_recursion;
1231261842b7SSteven Rostedt #endif /* CONFIG_TRACING */
12325eca1c10SIngo Molnar 
12335c9a8750SDmitry Vyukov #ifdef CONFIG_KCOV
1234eec028c9SAndrey Konovalov 	/* See kernel/kcov.c for more details. */
1235eec028c9SAndrey Konovalov 
12365eca1c10SIngo Molnar 	/* Coverage collection mode enabled for this task (0 if disabled): */
12370ed557aaSMark Rutland 	unsigned int			kcov_mode;
12385eca1c10SIngo Molnar 
12395eca1c10SIngo Molnar 	/* Size of the kcov_area: */
12405eca1c10SIngo Molnar 	unsigned int			kcov_size;
12415eca1c10SIngo Molnar 
12425eca1c10SIngo Molnar 	/* Buffer for coverage collection: */
12435c9a8750SDmitry Vyukov 	void				*kcov_area;
12445eca1c10SIngo Molnar 
12455eca1c10SIngo Molnar 	/* KCOV descriptor wired with this task or NULL: */
12465c9a8750SDmitry Vyukov 	struct kcov			*kcov;
1247eec028c9SAndrey Konovalov 
1248eec028c9SAndrey Konovalov 	/* KCOV common handle for remote coverage collection: */
1249eec028c9SAndrey Konovalov 	u64				kcov_handle;
1250eec028c9SAndrey Konovalov 
1251eec028c9SAndrey Konovalov 	/* KCOV sequence number: */
1252eec028c9SAndrey Konovalov 	int				kcov_sequence;
12535ff3b30aSAndrey Konovalov 
12545ff3b30aSAndrey Konovalov 	/* Collect coverage from softirq context: */
12555ff3b30aSAndrey Konovalov 	unsigned int			kcov_softirq;
12565c9a8750SDmitry Vyukov #endif
12575eca1c10SIngo Molnar 
12586f185c29SVladimir Davydov #ifdef CONFIG_MEMCG
1259626ebc41STejun Heo 	struct mem_cgroup		*memcg_in_oom;
1260626ebc41STejun Heo 	gfp_t				memcg_oom_gfp_mask;
1261626ebc41STejun Heo 	int				memcg_oom_order;
1262b23afb93STejun Heo 
12635eca1c10SIngo Molnar 	/* Number of pages to reclaim on returning to userland: */
1264b23afb93STejun Heo 	unsigned int			memcg_nr_pages_over_high;
1265d46eb14bSShakeel Butt 
1266d46eb14bSShakeel Butt 	/* Used by memcontrol for targeted memcg charge: */
1267d46eb14bSShakeel Butt 	struct mem_cgroup		*active_memcg;
1268569b846dSKAMEZAWA Hiroyuki #endif
12695eca1c10SIngo Molnar 
1270d09d8df3SJosef Bacik #ifdef CONFIG_BLK_CGROUP
1271d09d8df3SJosef Bacik 	struct request_queue		*throttle_queue;
1272d09d8df3SJosef Bacik #endif
1273d09d8df3SJosef Bacik 
12740326f5a9SSrikar Dronamraju #ifdef CONFIG_UPROBES
12750326f5a9SSrikar Dronamraju 	struct uprobe_task		*utask;
12760326f5a9SSrikar Dronamraju #endif
1277cafe5635SKent Overstreet #if defined(CONFIG_BCACHE) || defined(CONFIG_BCACHE_MODULE)
1278cafe5635SKent Overstreet 	unsigned int			sequential_io;
1279cafe5635SKent Overstreet 	unsigned int			sequential_io_avg;
1280cafe5635SKent Overstreet #endif
12818eb23b9fSPeter Zijlstra #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
12828eb23b9fSPeter Zijlstra 	unsigned long			task_state_change;
12838eb23b9fSPeter Zijlstra #endif
12848bcbde54SDavid Hildenbrand 	int				pagefault_disabled;
128503049269SMichal Hocko #ifdef CONFIG_MMU
128629c696e1SVladimir Davydov 	struct task_struct		*oom_reaper_list;
128703049269SMichal Hocko #endif
1288ba14a194SAndy Lutomirski #ifdef CONFIG_VMAP_STACK
1289ba14a194SAndy Lutomirski 	struct vm_struct		*stack_vm_area;
1290ba14a194SAndy Lutomirski #endif
129168f24b08SAndy Lutomirski #ifdef CONFIG_THREAD_INFO_IN_TASK
12925eca1c10SIngo Molnar 	/* A live task holds one reference: */
1293f0b89d39SElena Reshetova 	refcount_t			stack_refcount;
129468f24b08SAndy Lutomirski #endif
1295d83a7cb3SJosh Poimboeuf #ifdef CONFIG_LIVEPATCH
1296d83a7cb3SJosh Poimboeuf 	int patch_state;
1297d83a7cb3SJosh Poimboeuf #endif
1298e4e55b47STetsuo Handa #ifdef CONFIG_SECURITY
1299e4e55b47STetsuo Handa 	/* Used by LSM modules for access restriction: */
1300e4e55b47STetsuo Handa 	void				*security;
1301e4e55b47STetsuo Handa #endif
130229e48ce8SKees Cook 
1303afaef01cSAlexander Popov #ifdef CONFIG_GCC_PLUGIN_STACKLEAK
1304afaef01cSAlexander Popov 	unsigned long			lowest_stack;
1305c8d12627SAlexander Popov 	unsigned long			prev_lowest_stack;
1306afaef01cSAlexander Popov #endif
1307afaef01cSAlexander Popov 
13085567d11cSPeter Zijlstra #ifdef CONFIG_X86_MCE
13095567d11cSPeter Zijlstra 	u64				mce_addr;
131017fae129STony Luck 	__u64				mce_ripv : 1,
131117fae129STony Luck 					mce_whole_page : 1,
131217fae129STony Luck 					__mce_reserved : 62;
13135567d11cSPeter Zijlstra 	struct callback_head		mce_kill_me;
13145567d11cSPeter Zijlstra #endif
13155567d11cSPeter Zijlstra 
131629e48ce8SKees Cook 	/*
131729e48ce8SKees Cook 	 * New fields for task_struct should be added above here, so that
131829e48ce8SKees Cook 	 * they are included in the randomized portion of task_struct.
131929e48ce8SKees Cook 	 */
132029e48ce8SKees Cook 	randomized_struct_fields_end
132129e48ce8SKees Cook 
13225eca1c10SIngo Molnar 	/* CPU-specific state of this task: */
13230c8c0f03SDave Hansen 	struct thread_struct		thread;
13245eca1c10SIngo Molnar 
13250c8c0f03SDave Hansen 	/*
13260c8c0f03SDave Hansen 	 * WARNING: on x86, 'thread_struct' contains a variable-sized
13270c8c0f03SDave Hansen 	 * structure.  It *MUST* be at the end of 'task_struct'.
13280c8c0f03SDave Hansen 	 *
13290c8c0f03SDave Hansen 	 * Do not put anything below here!
13300c8c0f03SDave Hansen 	 */
13311da177e4SLinus Torvalds };
13321da177e4SLinus Torvalds 
1333e868171aSAlexey Dobriyan static inline struct pid *task_pid(struct task_struct *task)
133422c935f4SEric W. Biederman {
13352c470475SEric W. Biederman 	return task->thread_pid;
133622c935f4SEric W. Biederman }
133722c935f4SEric W. Biederman 
13387af57294SPavel Emelyanov /*
13397af57294SPavel Emelyanov  * the helpers to get the task's different pids as they are seen
13407af57294SPavel Emelyanov  * from various namespaces
13417af57294SPavel Emelyanov  *
13427af57294SPavel Emelyanov  * task_xid_nr()     : global id, i.e. the id seen from the init namespace;
134344c4e1b2SEric W. Biederman  * task_xid_vnr()    : virtual id, i.e. the id seen from the pid namespace of
134444c4e1b2SEric W. Biederman  *                     current.
13457af57294SPavel Emelyanov  * task_xid_nr_ns()  : id seen from the ns specified;
13467af57294SPavel Emelyanov  *
13477af57294SPavel Emelyanov  * see also pid_nr() etc in include/linux/pid.h
13487af57294SPavel Emelyanov  */
13495eca1c10SIngo Molnar pid_t __task_pid_nr_ns(struct task_struct *task, enum pid_type type, struct pid_namespace *ns);
13507af57294SPavel Emelyanov 
1351e868171aSAlexey Dobriyan static inline pid_t task_pid_nr(struct task_struct *tsk)
13527af57294SPavel Emelyanov {
13537af57294SPavel Emelyanov 	return tsk->pid;
13547af57294SPavel Emelyanov }
13557af57294SPavel Emelyanov 
13565eca1c10SIngo Molnar static inline pid_t task_pid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
135752ee2dfdSOleg Nesterov {
135852ee2dfdSOleg Nesterov 	return __task_pid_nr_ns(tsk, PIDTYPE_PID, ns);
135952ee2dfdSOleg Nesterov }
13607af57294SPavel Emelyanov 
13617af57294SPavel Emelyanov static inline pid_t task_pid_vnr(struct task_struct *tsk)
13627af57294SPavel Emelyanov {
136352ee2dfdSOleg Nesterov 	return __task_pid_nr_ns(tsk, PIDTYPE_PID, NULL);
13647af57294SPavel Emelyanov }
13657af57294SPavel Emelyanov 
13667af57294SPavel Emelyanov 
1367e868171aSAlexey Dobriyan static inline pid_t task_tgid_nr(struct task_struct *tsk)
13687af57294SPavel Emelyanov {
13697af57294SPavel Emelyanov 	return tsk->tgid;
13707af57294SPavel Emelyanov }
13717af57294SPavel Emelyanov 
13725eca1c10SIngo Molnar /**
13735eca1c10SIngo Molnar  * pid_alive - check that a task structure is not stale
13745eca1c10SIngo Molnar  * @p: Task structure to be checked.
13755eca1c10SIngo Molnar  *
13765eca1c10SIngo Molnar  * Test if a process is not yet dead (at most zombie state)
13775eca1c10SIngo Molnar  * If pid_alive fails, then pointers within the task structure
13785eca1c10SIngo Molnar  * can be stale and must not be dereferenced.
13795eca1c10SIngo Molnar  *
13805eca1c10SIngo Molnar  * Return: 1 if the process is alive. 0 otherwise.
13815eca1c10SIngo Molnar  */
13825eca1c10SIngo Molnar static inline int pid_alive(const struct task_struct *p)
13835eca1c10SIngo Molnar {
13842c470475SEric W. Biederman 	return p->thread_pid != NULL;
13855eca1c10SIngo Molnar }
13867af57294SPavel Emelyanov 
13875eca1c10SIngo Molnar static inline pid_t task_pgrp_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
13887af57294SPavel Emelyanov {
138952ee2dfdSOleg Nesterov 	return __task_pid_nr_ns(tsk, PIDTYPE_PGID, ns);
13907af57294SPavel Emelyanov }
13917af57294SPavel Emelyanov 
13927af57294SPavel Emelyanov static inline pid_t task_pgrp_vnr(struct task_struct *tsk)
13937af57294SPavel Emelyanov {
139452ee2dfdSOleg Nesterov 	return __task_pid_nr_ns(tsk, PIDTYPE_PGID, NULL);
13957af57294SPavel Emelyanov }
13967af57294SPavel Emelyanov 
13977af57294SPavel Emelyanov 
13985eca1c10SIngo Molnar static inline pid_t task_session_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
13997af57294SPavel Emelyanov {
140052ee2dfdSOleg Nesterov 	return __task_pid_nr_ns(tsk, PIDTYPE_SID, ns);
14017af57294SPavel Emelyanov }
14027af57294SPavel Emelyanov 
14037af57294SPavel Emelyanov static inline pid_t task_session_vnr(struct task_struct *tsk)
14047af57294SPavel Emelyanov {
140552ee2dfdSOleg Nesterov 	return __task_pid_nr_ns(tsk, PIDTYPE_SID, NULL);
14067af57294SPavel Emelyanov }
14077af57294SPavel Emelyanov 
1408dd1c1f2fSOleg Nesterov static inline pid_t task_tgid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
1409dd1c1f2fSOleg Nesterov {
14106883f81aSEric W. Biederman 	return __task_pid_nr_ns(tsk, PIDTYPE_TGID, ns);
1411dd1c1f2fSOleg Nesterov }
1412dd1c1f2fSOleg Nesterov 
1413dd1c1f2fSOleg Nesterov static inline pid_t task_tgid_vnr(struct task_struct *tsk)
1414dd1c1f2fSOleg Nesterov {
14156883f81aSEric W. Biederman 	return __task_pid_nr_ns(tsk, PIDTYPE_TGID, NULL);
1416dd1c1f2fSOleg Nesterov }
1417dd1c1f2fSOleg Nesterov 
1418dd1c1f2fSOleg Nesterov static inline pid_t task_ppid_nr_ns(const struct task_struct *tsk, struct pid_namespace *ns)
1419dd1c1f2fSOleg Nesterov {
1420dd1c1f2fSOleg Nesterov 	pid_t pid = 0;
1421dd1c1f2fSOleg Nesterov 
1422dd1c1f2fSOleg Nesterov 	rcu_read_lock();
1423dd1c1f2fSOleg Nesterov 	if (pid_alive(tsk))
1424dd1c1f2fSOleg Nesterov 		pid = task_tgid_nr_ns(rcu_dereference(tsk->real_parent), ns);
1425dd1c1f2fSOleg Nesterov 	rcu_read_unlock();
1426dd1c1f2fSOleg Nesterov 
1427dd1c1f2fSOleg Nesterov 	return pid;
1428dd1c1f2fSOleg Nesterov }
1429dd1c1f2fSOleg Nesterov 
1430dd1c1f2fSOleg Nesterov static inline pid_t task_ppid_nr(const struct task_struct *tsk)
1431dd1c1f2fSOleg Nesterov {
1432dd1c1f2fSOleg Nesterov 	return task_ppid_nr_ns(tsk, &init_pid_ns);
1433dd1c1f2fSOleg Nesterov }
1434dd1c1f2fSOleg Nesterov 
14355eca1c10SIngo Molnar /* Obsolete, do not use: */
14361b0f7ffdSOleg Nesterov static inline pid_t task_pgrp_nr(struct task_struct *tsk)
14371b0f7ffdSOleg Nesterov {
14381b0f7ffdSOleg Nesterov 	return task_pgrp_nr_ns(tsk, &init_pid_ns);
14391b0f7ffdSOleg Nesterov }
14407af57294SPavel Emelyanov 
144106eb6184SPeter Zijlstra #define TASK_REPORT_IDLE	(TASK_REPORT + 1)
144206eb6184SPeter Zijlstra #define TASK_REPORT_MAX		(TASK_REPORT_IDLE << 1)
144306eb6184SPeter Zijlstra 
14441d48b080SPeter Zijlstra static inline unsigned int task_state_index(struct task_struct *tsk)
144520435d84SXie XiuQi {
14461593baabSPeter Zijlstra 	unsigned int tsk_state = READ_ONCE(tsk->state);
14471593baabSPeter Zijlstra 	unsigned int state = (tsk_state | tsk->exit_state) & TASK_REPORT;
144820435d84SXie XiuQi 
144906eb6184SPeter Zijlstra 	BUILD_BUG_ON_NOT_POWER_OF_2(TASK_REPORT_MAX);
145006eb6184SPeter Zijlstra 
145106eb6184SPeter Zijlstra 	if (tsk_state == TASK_IDLE)
145206eb6184SPeter Zijlstra 		state = TASK_REPORT_IDLE;
145306eb6184SPeter Zijlstra 
14541593baabSPeter Zijlstra 	return fls(state);
14551593baabSPeter Zijlstra }
145620435d84SXie XiuQi 
14571d48b080SPeter Zijlstra static inline char task_index_to_char(unsigned int state)
14581593baabSPeter Zijlstra {
14598ef9925bSPeter Zijlstra 	static const char state_char[] = "RSDTtXZPI";
14601593baabSPeter Zijlstra 
146106eb6184SPeter Zijlstra 	BUILD_BUG_ON(1 + ilog2(TASK_REPORT_MAX) != sizeof(state_char) - 1);
14621593baabSPeter Zijlstra 
14631593baabSPeter Zijlstra 	return state_char[state];
14641593baabSPeter Zijlstra }
14651593baabSPeter Zijlstra 
14661593baabSPeter Zijlstra static inline char task_state_to_char(struct task_struct *tsk)
14671593baabSPeter Zijlstra {
14681d48b080SPeter Zijlstra 	return task_index_to_char(task_state_index(tsk));
146920435d84SXie XiuQi }
147020435d84SXie XiuQi 
14711da177e4SLinus Torvalds /**
1472570f5241SSergey Senozhatsky  * is_global_init - check if a task structure is init. Since init
1473570f5241SSergey Senozhatsky  * is free to have sub-threads we need to check tgid.
14743260259fSHenne  * @tsk: Task structure to be checked.
14753260259fSHenne  *
14763260259fSHenne  * Check if a task structure is the first user space task the kernel created.
1477e69f6186SYacine Belkadi  *
1478e69f6186SYacine Belkadi  * Return: 1 if the task structure is init. 0 otherwise.
1479f400e198SSukadev Bhattiprolu  */
1480e868171aSAlexey Dobriyan static inline int is_global_init(struct task_struct *tsk)
1481b461cc03SPavel Emelyanov {
1482570f5241SSergey Senozhatsky 	return task_tgid_nr(tsk) == 1;
1483b461cc03SPavel Emelyanov }
1484b460cbc5SSerge E. Hallyn 
14859ec52099SCedric Le Goater extern struct pid *cad_pid;
14869ec52099SCedric Le Goater 
14871da177e4SLinus Torvalds /*
14881da177e4SLinus Torvalds  * Per process flags
14891da177e4SLinus Torvalds  */
1490c1de45caSPeter Zijlstra #define PF_IDLE			0x00000002	/* I am an IDLE thread */
14915eca1c10SIngo Molnar #define PF_EXITING		0x00000004	/* Getting shut down */
149294886b84SLaurent Vivier #define PF_VCPU			0x00000010	/* I'm a virtual CPU */
149321aa9af0STejun Heo #define PF_WQ_WORKER		0x00000020	/* I'm a workqueue worker */
14945eca1c10SIngo Molnar #define PF_FORKNOEXEC		0x00000040	/* Forked but didn't exec */
14955eca1c10SIngo Molnar #define PF_MCE_PROCESS		0x00000080      /* Process policy on mce errors */
14965eca1c10SIngo Molnar #define PF_SUPERPRIV		0x00000100	/* Used super-user privileges */
14975eca1c10SIngo Molnar #define PF_DUMPCORE		0x00000200	/* Dumped core */
14985eca1c10SIngo Molnar #define PF_SIGNALED		0x00000400	/* Killed by a signal */
14991da177e4SLinus Torvalds #define PF_MEMALLOC		0x00000800	/* Allocating memory */
15005eca1c10SIngo Molnar #define PF_NPROC_EXCEEDED	0x00001000	/* set_user() noticed that RLIMIT_NPROC was exceeded */
15015eca1c10SIngo Molnar #define PF_USED_MATH		0x00002000	/* If unset the fpu must be initialized before use */
15025eca1c10SIngo Molnar #define PF_USED_ASYNC		0x00004000	/* Used async_schedule*(), used by module init */
15035eca1c10SIngo Molnar #define PF_NOFREEZE		0x00008000	/* This thread should not be frozen */
15045eca1c10SIngo Molnar #define PF_FROZEN		0x00010000	/* Frozen for system suspend */
15057dea19f9SMichal Hocko #define PF_KSWAPD		0x00020000	/* I am kswapd */
15067dea19f9SMichal Hocko #define PF_MEMALLOC_NOFS	0x00040000	/* All allocation requests will inherit GFP_NOFS */
15077dea19f9SMichal Hocko #define PF_MEMALLOC_NOIO	0x00080000	/* All allocation requests will inherit GFP_NOIO */
1508a37b0715SNeilBrown #define PF_LOCAL_THROTTLE	0x00100000	/* Throttle writes only against the bdi I write to,
1509a37b0715SNeilBrown 						 * I am cleaning dirty pages from some other bdi. */
1510246bb0b1SOleg Nesterov #define PF_KTHREAD		0x00200000	/* I am a kernel thread */
15115eca1c10SIngo Molnar #define PF_RANDOMIZE		0x00400000	/* Randomize virtual address space */
1512b31dc66aSJens Axboe #define PF_SWAPWRITE		0x00800000	/* Allowed to write to swap */
151373ab1cb2STaehee Yoo #define PF_UMH			0x02000000	/* I'm an Usermodehelper process */
15143bd37062SSebastian Andrzej Siewior #define PF_NO_SETAFFINITY	0x04000000	/* Userland is not allowed to meddle with cpus_mask */
15154db96cf0SAndi Kleen #define PF_MCE_EARLY		0x08000000      /* Early kill for mce process policy */
1516d7fefcc8SAneesh Kumar K.V #define PF_MEMALLOC_NOCMA	0x10000000	/* All allocation request will have _GFP_MOVABLE cleared */
1517771b53d0SJens Axboe #define PF_IO_WORKER		0x20000000	/* Task is an IO worker */
151858a69cb4STejun Heo #define PF_FREEZER_SKIP		0x40000000	/* Freezer should not count it as freezable */
15195eca1c10SIngo Molnar #define PF_SUSPEND_TASK		0x80000000      /* This thread called freeze_processes() and should not be frozen */
15201da177e4SLinus Torvalds 
15211da177e4SLinus Torvalds /*
15221da177e4SLinus Torvalds  * Only the _current_ task can read/write to tsk->flags, but other
15231da177e4SLinus Torvalds  * tasks can access tsk->flags in readonly mode for example
15241da177e4SLinus Torvalds  * with tsk_used_math (like during threaded core dumping).
15251da177e4SLinus Torvalds  * There is however an exception to this rule during ptrace
15261da177e4SLinus Torvalds  * or during fork: the ptracer task is allowed to write to the
15271da177e4SLinus Torvalds  * child->flags of its traced child (same goes for fork, the parent
15281da177e4SLinus Torvalds  * can write to the child->flags), because we're guaranteed the
15291da177e4SLinus Torvalds  * child is not running and in turn not changing child->flags
15301da177e4SLinus Torvalds  * at the same time the parent does it.
15311da177e4SLinus Torvalds  */
15321da177e4SLinus Torvalds #define clear_stopped_child_used_math(child)	do { (child)->flags &= ~PF_USED_MATH; } while (0)
15331da177e4SLinus Torvalds #define set_stopped_child_used_math(child)	do { (child)->flags |= PF_USED_MATH; } while (0)
15341da177e4SLinus Torvalds #define clear_used_math()			clear_stopped_child_used_math(current)
15351da177e4SLinus Torvalds #define set_used_math()				set_stopped_child_used_math(current)
15365eca1c10SIngo Molnar 
15371da177e4SLinus Torvalds #define conditional_stopped_child_used_math(condition, child) \
15381da177e4SLinus Torvalds 	do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= (condition) ? PF_USED_MATH : 0; } while (0)
15395eca1c10SIngo Molnar 
15405eca1c10SIngo Molnar #define conditional_used_math(condition)	conditional_stopped_child_used_math(condition, current)
15415eca1c10SIngo Molnar 
15421da177e4SLinus Torvalds #define copy_to_stopped_child_used_math(child) \
15431da177e4SLinus Torvalds 	do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= current->flags & PF_USED_MATH; } while (0)
15445eca1c10SIngo Molnar 
15451da177e4SLinus Torvalds /* NOTE: this will return 0 or PF_USED_MATH, it will never return 1 */
15461da177e4SLinus Torvalds #define tsk_used_math(p)			((p)->flags & PF_USED_MATH)
15471da177e4SLinus Torvalds #define used_math()				tsk_used_math(current)
15481da177e4SLinus Torvalds 
154962ec05ddSThomas Gleixner static inline bool is_percpu_thread(void)
155062ec05ddSThomas Gleixner {
155162ec05ddSThomas Gleixner #ifdef CONFIG_SMP
155262ec05ddSThomas Gleixner 	return (current->flags & PF_NO_SETAFFINITY) &&
155362ec05ddSThomas Gleixner 		(current->nr_cpus_allowed  == 1);
155462ec05ddSThomas Gleixner #else
155562ec05ddSThomas Gleixner 	return true;
155662ec05ddSThomas Gleixner #endif
155762ec05ddSThomas Gleixner }
155862ec05ddSThomas Gleixner 
15591d4457f9SKees Cook /* Per-process atomic flags. */
1560a2b86f77SZefan Li #define PFA_NO_NEW_PRIVS		0	/* May not gain new privileges. */
15612ad654bcSZefan Li #define PFA_SPREAD_PAGE			1	/* Spread page cache over cpuset */
15622ad654bcSZefan Li #define PFA_SPREAD_SLAB			2	/* Spread some slab caches over cpuset */
1563356e4bffSThomas Gleixner #define PFA_SPEC_SSB_DISABLE		3	/* Speculative Store Bypass disabled */
1564356e4bffSThomas Gleixner #define PFA_SPEC_SSB_FORCE_DISABLE	4	/* Speculative Store Bypass force disabled*/
15659137bb27SThomas Gleixner #define PFA_SPEC_IB_DISABLE		5	/* Indirect branch speculation restricted */
15669137bb27SThomas Gleixner #define PFA_SPEC_IB_FORCE_DISABLE	6	/* Indirect branch speculation permanently restricted */
156771368af9SWaiman Long #define PFA_SPEC_SSB_NOEXEC		7	/* Speculative Store Bypass clear on execve() */
15681d4457f9SKees Cook 
1569e0e5070bSZefan Li #define TASK_PFA_TEST(name, func)					\
1570e0e5070bSZefan Li 	static inline bool task_##func(struct task_struct *p)		\
1571e0e5070bSZefan Li 	{ return test_bit(PFA_##name, &p->atomic_flags); }
15725eca1c10SIngo Molnar 
1573e0e5070bSZefan Li #define TASK_PFA_SET(name, func)					\
1574e0e5070bSZefan Li 	static inline void task_set_##func(struct task_struct *p)	\
1575e0e5070bSZefan Li 	{ set_bit(PFA_##name, &p->atomic_flags); }
15765eca1c10SIngo Molnar 
1577e0e5070bSZefan Li #define TASK_PFA_CLEAR(name, func)					\
1578e0e5070bSZefan Li 	static inline void task_clear_##func(struct task_struct *p)	\
1579e0e5070bSZefan Li 	{ clear_bit(PFA_##name, &p->atomic_flags); }
15801d4457f9SKees Cook 
1581e0e5070bSZefan Li TASK_PFA_TEST(NO_NEW_PRIVS, no_new_privs)
1582e0e5070bSZefan Li TASK_PFA_SET(NO_NEW_PRIVS, no_new_privs)
15831d4457f9SKees Cook 
15842ad654bcSZefan Li TASK_PFA_TEST(SPREAD_PAGE, spread_page)
15852ad654bcSZefan Li TASK_PFA_SET(SPREAD_PAGE, spread_page)
15862ad654bcSZefan Li TASK_PFA_CLEAR(SPREAD_PAGE, spread_page)
15872ad654bcSZefan Li 
15882ad654bcSZefan Li TASK_PFA_TEST(SPREAD_SLAB, spread_slab)
15892ad654bcSZefan Li TASK_PFA_SET(SPREAD_SLAB, spread_slab)
15902ad654bcSZefan Li TASK_PFA_CLEAR(SPREAD_SLAB, spread_slab)
1591544b2c91STejun Heo 
1592356e4bffSThomas Gleixner TASK_PFA_TEST(SPEC_SSB_DISABLE, spec_ssb_disable)
1593356e4bffSThomas Gleixner TASK_PFA_SET(SPEC_SSB_DISABLE, spec_ssb_disable)
1594356e4bffSThomas Gleixner TASK_PFA_CLEAR(SPEC_SSB_DISABLE, spec_ssb_disable)
1595356e4bffSThomas Gleixner 
159671368af9SWaiman Long TASK_PFA_TEST(SPEC_SSB_NOEXEC, spec_ssb_noexec)
159771368af9SWaiman Long TASK_PFA_SET(SPEC_SSB_NOEXEC, spec_ssb_noexec)
159871368af9SWaiman Long TASK_PFA_CLEAR(SPEC_SSB_NOEXEC, spec_ssb_noexec)
159971368af9SWaiman Long 
1600356e4bffSThomas Gleixner TASK_PFA_TEST(SPEC_SSB_FORCE_DISABLE, spec_ssb_force_disable)
1601356e4bffSThomas Gleixner TASK_PFA_SET(SPEC_SSB_FORCE_DISABLE, spec_ssb_force_disable)
1602356e4bffSThomas Gleixner 
16039137bb27SThomas Gleixner TASK_PFA_TEST(SPEC_IB_DISABLE, spec_ib_disable)
16049137bb27SThomas Gleixner TASK_PFA_SET(SPEC_IB_DISABLE, spec_ib_disable)
16059137bb27SThomas Gleixner TASK_PFA_CLEAR(SPEC_IB_DISABLE, spec_ib_disable)
16069137bb27SThomas Gleixner 
16079137bb27SThomas Gleixner TASK_PFA_TEST(SPEC_IB_FORCE_DISABLE, spec_ib_force_disable)
16089137bb27SThomas Gleixner TASK_PFA_SET(SPEC_IB_FORCE_DISABLE, spec_ib_force_disable)
16099137bb27SThomas Gleixner 
16105eca1c10SIngo Molnar static inline void
1611717a94b5SNeilBrown current_restore_flags(unsigned long orig_flags, unsigned long flags)
1612907aed48SMel Gorman {
1613717a94b5SNeilBrown 	current->flags &= ~flags;
1614717a94b5SNeilBrown 	current->flags |= orig_flags & flags;
1615907aed48SMel Gorman }
1616907aed48SMel Gorman 
16175eca1c10SIngo Molnar extern int cpuset_cpumask_can_shrink(const struct cpumask *cur, const struct cpumask *trial);
16185eca1c10SIngo Molnar extern int task_can_attach(struct task_struct *p, const struct cpumask *cs_cpus_allowed);
16191da177e4SLinus Torvalds #ifdef CONFIG_SMP
16205eca1c10SIngo Molnar extern void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask);
16215eca1c10SIngo Molnar extern int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask);
16221da177e4SLinus Torvalds #else
16235eca1c10SIngo Molnar static inline void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
16241e1b6c51SKOSAKI Motohiro {
16251e1b6c51SKOSAKI Motohiro }
16265eca1c10SIngo Molnar static inline int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
16271da177e4SLinus Torvalds {
162896f874e2SRusty Russell 	if (!cpumask_test_cpu(0, new_mask))
16291da177e4SLinus Torvalds 		return -EINVAL;
16301da177e4SLinus Torvalds 	return 0;
16311da177e4SLinus Torvalds }
16321da177e4SLinus Torvalds #endif
1633e0ad9556SRusty Russell 
1634fa93384fSDan Carpenter extern int yield_to(struct task_struct *p, bool preempt);
163536c8b586SIngo Molnar extern void set_user_nice(struct task_struct *p, long nice);
163636c8b586SIngo Molnar extern int task_prio(const struct task_struct *p);
16375eca1c10SIngo Molnar 
1638d0ea0268SDongsheng Yang /**
1639d0ea0268SDongsheng Yang  * task_nice - return the nice value of a given task.
1640d0ea0268SDongsheng Yang  * @p: the task in question.
1641d0ea0268SDongsheng Yang  *
1642d0ea0268SDongsheng Yang  * Return: The nice value [ -20 ... 0 ... 19 ].
1643d0ea0268SDongsheng Yang  */
1644d0ea0268SDongsheng Yang static inline int task_nice(const struct task_struct *p)
1645d0ea0268SDongsheng Yang {
1646d0ea0268SDongsheng Yang 	return PRIO_TO_NICE((p)->static_prio);
1647d0ea0268SDongsheng Yang }
16485eca1c10SIngo Molnar 
164936c8b586SIngo Molnar extern int can_nice(const struct task_struct *p, const int nice);
165036c8b586SIngo Molnar extern int task_curr(const struct task_struct *p);
16511da177e4SLinus Torvalds extern int idle_cpu(int cpu);
1652943d355dSRohit Jain extern int available_idle_cpu(int cpu);
16535eca1c10SIngo Molnar extern int sched_setscheduler(struct task_struct *, int, const struct sched_param *);
16545eca1c10SIngo Molnar extern int sched_setscheduler_nocheck(struct task_struct *, int, const struct sched_param *);
16555eca1c10SIngo Molnar extern int sched_setattr(struct task_struct *, const struct sched_attr *);
1656794a56ebSJuri Lelli extern int sched_setattr_nocheck(struct task_struct *, const struct sched_attr *);
165736c8b586SIngo Molnar extern struct task_struct *idle_task(int cpu);
16585eca1c10SIngo Molnar 
1659c4f30608SPaul E. McKenney /**
1660c4f30608SPaul E. McKenney  * is_idle_task - is the specified task an idle task?
1661fa757281SRandy Dunlap  * @p: the task in question.
1662e69f6186SYacine Belkadi  *
1663e69f6186SYacine Belkadi  * Return: 1 if @p is an idle task. 0 otherwise.
1664c4f30608SPaul E. McKenney  */
16657061ca3bSPaul E. McKenney static inline bool is_idle_task(const struct task_struct *p)
1666c4f30608SPaul E. McKenney {
1667c1de45caSPeter Zijlstra 	return !!(p->flags & PF_IDLE);
1668c4f30608SPaul E. McKenney }
16695eca1c10SIngo Molnar 
167036c8b586SIngo Molnar extern struct task_struct *curr_task(int cpu);
1671a458ae2eSPeter Zijlstra extern void ia64_set_curr_task(int cpu, struct task_struct *p);
16721da177e4SLinus Torvalds 
16731da177e4SLinus Torvalds void yield(void);
16741da177e4SLinus Torvalds 
16751da177e4SLinus Torvalds union thread_union {
16760500871fSDavid Howells #ifndef CONFIG_ARCH_TASK_STRUCT_ON_STACK
16770500871fSDavid Howells 	struct task_struct task;
16780500871fSDavid Howells #endif
1679c65eacbeSAndy Lutomirski #ifndef CONFIG_THREAD_INFO_IN_TASK
16801da177e4SLinus Torvalds 	struct thread_info thread_info;
1681c65eacbeSAndy Lutomirski #endif
16821da177e4SLinus Torvalds 	unsigned long stack[THREAD_SIZE/sizeof(long)];
16831da177e4SLinus Torvalds };
16841da177e4SLinus Torvalds 
16850500871fSDavid Howells #ifndef CONFIG_THREAD_INFO_IN_TASK
16860500871fSDavid Howells extern struct thread_info init_thread_info;
16870500871fSDavid Howells #endif
16880500871fSDavid Howells 
16890500871fSDavid Howells extern unsigned long init_stack[THREAD_SIZE / sizeof(unsigned long)];
16900500871fSDavid Howells 
1691f3ac6067SIngo Molnar #ifdef CONFIG_THREAD_INFO_IN_TASK
1692f3ac6067SIngo Molnar static inline struct thread_info *task_thread_info(struct task_struct *task)
1693f3ac6067SIngo Molnar {
1694f3ac6067SIngo Molnar 	return &task->thread_info;
1695f3ac6067SIngo Molnar }
1696f3ac6067SIngo Molnar #elif !defined(__HAVE_THREAD_FUNCTIONS)
1697f3ac6067SIngo Molnar # define task_thread_info(task)	((struct thread_info *)(task)->stack)
1698f3ac6067SIngo Molnar #endif
1699f3ac6067SIngo Molnar 
1700198fe21bSPavel Emelyanov /*
1701198fe21bSPavel Emelyanov  * find a task by one of its numerical ids
1702198fe21bSPavel Emelyanov  *
1703198fe21bSPavel Emelyanov  * find_task_by_pid_ns():
1704198fe21bSPavel Emelyanov  *      finds a task by its pid in the specified namespace
1705228ebcbeSPavel Emelyanov  * find_task_by_vpid():
1706228ebcbeSPavel Emelyanov  *      finds a task by its virtual pid
1707198fe21bSPavel Emelyanov  *
1708e49859e7SPavel Emelyanov  * see also find_vpid() etc in include/linux/pid.h
1709198fe21bSPavel Emelyanov  */
1710198fe21bSPavel Emelyanov 
1711228ebcbeSPavel Emelyanov extern struct task_struct *find_task_by_vpid(pid_t nr);
17125eca1c10SIngo Molnar extern struct task_struct *find_task_by_pid_ns(pid_t nr, struct pid_namespace *ns);
1713198fe21bSPavel Emelyanov 
17142ee08260SMike Rapoport /*
17152ee08260SMike Rapoport  * find a task by its virtual pid and get the task struct
17162ee08260SMike Rapoport  */
17172ee08260SMike Rapoport extern struct task_struct *find_get_task_by_vpid(pid_t nr);
17182ee08260SMike Rapoport 
1719b3c97528SHarvey Harrison extern int wake_up_state(struct task_struct *tsk, unsigned int state);
1720b3c97528SHarvey Harrison extern int wake_up_process(struct task_struct *tsk);
17213e51e3edSSamir Bellabes extern void wake_up_new_task(struct task_struct *tsk);
17225eca1c10SIngo Molnar 
17231da177e4SLinus Torvalds #ifdef CONFIG_SMP
17241da177e4SLinus Torvalds extern void kick_process(struct task_struct *tsk);
17251da177e4SLinus Torvalds #else
17261da177e4SLinus Torvalds static inline void kick_process(struct task_struct *tsk) { }
17271da177e4SLinus Torvalds #endif
17281da177e4SLinus Torvalds 
172982b89778SAdrian Hunter extern void __set_task_comm(struct task_struct *tsk, const char *from, bool exec);
17305eca1c10SIngo Molnar 
173182b89778SAdrian Hunter static inline void set_task_comm(struct task_struct *tsk, const char *from)
173282b89778SAdrian Hunter {
173382b89778SAdrian Hunter 	__set_task_comm(tsk, from, false);
173482b89778SAdrian Hunter }
17355eca1c10SIngo Molnar 
17363756f640SArnd Bergmann extern char *__get_task_comm(char *to, size_t len, struct task_struct *tsk);
17373756f640SArnd Bergmann #define get_task_comm(buf, tsk) ({			\
17383756f640SArnd Bergmann 	BUILD_BUG_ON(sizeof(buf) != TASK_COMM_LEN);	\
17393756f640SArnd Bergmann 	__get_task_comm(buf, sizeof(buf), tsk);		\
17403756f640SArnd Bergmann })
17411da177e4SLinus Torvalds 
17421da177e4SLinus Torvalds #ifdef CONFIG_SMP
17432a0a24ebSThomas Gleixner static __always_inline void scheduler_ipi(void)
17442a0a24ebSThomas Gleixner {
17452a0a24ebSThomas Gleixner 	/*
17462a0a24ebSThomas Gleixner 	 * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting
17472a0a24ebSThomas Gleixner 	 * TIF_NEED_RESCHED remotely (for the first time) will also send
17482a0a24ebSThomas Gleixner 	 * this IPI.
17492a0a24ebSThomas Gleixner 	 */
17502a0a24ebSThomas Gleixner 	preempt_fold_need_resched();
17512a0a24ebSThomas Gleixner }
175285ba2d86SRoland McGrath extern unsigned long wait_task_inactive(struct task_struct *, long match_state);
17531da177e4SLinus Torvalds #else
1754184748ccSPeter Zijlstra static inline void scheduler_ipi(void) { }
17555eca1c10SIngo Molnar static inline unsigned long wait_task_inactive(struct task_struct *p, long match_state)
175685ba2d86SRoland McGrath {
175785ba2d86SRoland McGrath 	return 1;
175885ba2d86SRoland McGrath }
17591da177e4SLinus Torvalds #endif
17601da177e4SLinus Torvalds 
17615eca1c10SIngo Molnar /*
17625eca1c10SIngo Molnar  * Set thread flags in other task's structures.
17635eca1c10SIngo Molnar  * See asm/thread_info.h for TIF_xxxx flags available:
17641da177e4SLinus Torvalds  */
17651da177e4SLinus Torvalds static inline void set_tsk_thread_flag(struct task_struct *tsk, int flag)
17661da177e4SLinus Torvalds {
1767a1261f54SAl Viro 	set_ti_thread_flag(task_thread_info(tsk), flag);
17681da177e4SLinus Torvalds }
17691da177e4SLinus Torvalds 
17701da177e4SLinus Torvalds static inline void clear_tsk_thread_flag(struct task_struct *tsk, int flag)
17711da177e4SLinus Torvalds {
1772a1261f54SAl Viro 	clear_ti_thread_flag(task_thread_info(tsk), flag);
17731da177e4SLinus Torvalds }
17741da177e4SLinus Torvalds 
177593ee37c2SDave Martin static inline void update_tsk_thread_flag(struct task_struct *tsk, int flag,
177693ee37c2SDave Martin 					  bool value)
177793ee37c2SDave Martin {
177893ee37c2SDave Martin 	update_ti_thread_flag(task_thread_info(tsk), flag, value);
177993ee37c2SDave Martin }
178093ee37c2SDave Martin 
17811da177e4SLinus Torvalds static inline int test_and_set_tsk_thread_flag(struct task_struct *tsk, int flag)
17821da177e4SLinus Torvalds {
1783a1261f54SAl Viro 	return test_and_set_ti_thread_flag(task_thread_info(tsk), flag);
17841da177e4SLinus Torvalds }
17851da177e4SLinus Torvalds 
17861da177e4SLinus Torvalds static inline int test_and_clear_tsk_thread_flag(struct task_struct *tsk, int flag)
17871da177e4SLinus Torvalds {
1788a1261f54SAl Viro 	return test_and_clear_ti_thread_flag(task_thread_info(tsk), flag);
17891da177e4SLinus Torvalds }
17901da177e4SLinus Torvalds 
17911da177e4SLinus Torvalds static inline int test_tsk_thread_flag(struct task_struct *tsk, int flag)
17921da177e4SLinus Torvalds {
1793a1261f54SAl Viro 	return test_ti_thread_flag(task_thread_info(tsk), flag);
17941da177e4SLinus Torvalds }
17951da177e4SLinus Torvalds 
17961da177e4SLinus Torvalds static inline void set_tsk_need_resched(struct task_struct *tsk)
17971da177e4SLinus Torvalds {
17981da177e4SLinus Torvalds 	set_tsk_thread_flag(tsk,TIF_NEED_RESCHED);
17991da177e4SLinus Torvalds }
18001da177e4SLinus Torvalds 
18011da177e4SLinus Torvalds static inline void clear_tsk_need_resched(struct task_struct *tsk)
18021da177e4SLinus Torvalds {
18031da177e4SLinus Torvalds 	clear_tsk_thread_flag(tsk,TIF_NEED_RESCHED);
18041da177e4SLinus Torvalds }
18051da177e4SLinus Torvalds 
18068ae121acSGregory Haskins static inline int test_tsk_need_resched(struct task_struct *tsk)
18078ae121acSGregory Haskins {
18088ae121acSGregory Haskins 	return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED));
18098ae121acSGregory Haskins }
18108ae121acSGregory Haskins 
18111da177e4SLinus Torvalds /*
18121da177e4SLinus Torvalds  * cond_resched() and cond_resched_lock(): latency reduction via
18131da177e4SLinus Torvalds  * explicit rescheduling in places that are safe. The return
18141da177e4SLinus Torvalds  * value indicates whether a reschedule was done in fact.
18151da177e4SLinus Torvalds  * cond_resched_lock() will drop the spinlock before scheduling,
18161da177e4SLinus Torvalds  */
1817c1a280b6SThomas Gleixner #ifndef CONFIG_PREEMPTION
1818c3921ab7SLinus Torvalds extern int _cond_resched(void);
181935a773a0SPeter Zijlstra #else
182035a773a0SPeter Zijlstra static inline int _cond_resched(void) { return 0; }
182135a773a0SPeter Zijlstra #endif
18226f80bd98SFrederic Weisbecker 
1823613afbf8SFrederic Weisbecker #define cond_resched() ({			\
18243427445aSPeter Zijlstra 	___might_sleep(__FILE__, __LINE__, 0);	\
1825613afbf8SFrederic Weisbecker 	_cond_resched();			\
1826613afbf8SFrederic Weisbecker })
18276f80bd98SFrederic Weisbecker 
1828613afbf8SFrederic Weisbecker extern int __cond_resched_lock(spinlock_t *lock);
1829613afbf8SFrederic Weisbecker 
1830613afbf8SFrederic Weisbecker #define cond_resched_lock(lock) ({				\
18313427445aSPeter Zijlstra 	___might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET);\
1832613afbf8SFrederic Weisbecker 	__cond_resched_lock(lock);				\
1833613afbf8SFrederic Weisbecker })
1834613afbf8SFrederic Weisbecker 
1835f6f3c437SSimon Horman static inline void cond_resched_rcu(void)
1836f6f3c437SSimon Horman {
1837f6f3c437SSimon Horman #if defined(CONFIG_DEBUG_ATOMIC_SLEEP) || !defined(CONFIG_PREEMPT_RCU)
1838f6f3c437SSimon Horman 	rcu_read_unlock();
1839f6f3c437SSimon Horman 	cond_resched();
1840f6f3c437SSimon Horman 	rcu_read_lock();
1841f6f3c437SSimon Horman #endif
1842f6f3c437SSimon Horman }
1843f6f3c437SSimon Horman 
18441da177e4SLinus Torvalds /*
18451da177e4SLinus Torvalds  * Does a critical section need to be broken due to another
1846c1a280b6SThomas Gleixner  * task waiting?: (technically does not depend on CONFIG_PREEMPTION,
184795c354feSNick Piggin  * but a general need for low latency)
18481da177e4SLinus Torvalds  */
184995c354feSNick Piggin static inline int spin_needbreak(spinlock_t *lock)
18501da177e4SLinus Torvalds {
1851c1a280b6SThomas Gleixner #ifdef CONFIG_PREEMPTION
185295c354feSNick Piggin 	return spin_is_contended(lock);
185395c354feSNick Piggin #else
18541da177e4SLinus Torvalds 	return 0;
185595c354feSNick Piggin #endif
18561da177e4SLinus Torvalds }
18571da177e4SLinus Torvalds 
185875f93fedSPeter Zijlstra static __always_inline bool need_resched(void)
185975f93fedSPeter Zijlstra {
186075f93fedSPeter Zijlstra 	return unlikely(tif_need_resched());
186175f93fedSPeter Zijlstra }
186275f93fedSPeter Zijlstra 
1863ee761f62SThomas Gleixner /*
18641da177e4SLinus Torvalds  * Wrappers for p->thread_info->cpu access. No-op on UP.
18651da177e4SLinus Torvalds  */
18661da177e4SLinus Torvalds #ifdef CONFIG_SMP
18671da177e4SLinus Torvalds 
18681da177e4SLinus Torvalds static inline unsigned int task_cpu(const struct task_struct *p)
18691da177e4SLinus Torvalds {
1870c65eacbeSAndy Lutomirski #ifdef CONFIG_THREAD_INFO_IN_TASK
1871c546951dSAndrea Parri 	return READ_ONCE(p->cpu);
1872c65eacbeSAndy Lutomirski #else
1873c546951dSAndrea Parri 	return READ_ONCE(task_thread_info(p)->cpu);
1874c65eacbeSAndy Lutomirski #endif
18751da177e4SLinus Torvalds }
18761da177e4SLinus Torvalds 
1877c65cc870SIngo Molnar extern void set_task_cpu(struct task_struct *p, unsigned int cpu);
18781da177e4SLinus Torvalds 
18791da177e4SLinus Torvalds #else
18801da177e4SLinus Torvalds 
18811da177e4SLinus Torvalds static inline unsigned int task_cpu(const struct task_struct *p)
18821da177e4SLinus Torvalds {
18831da177e4SLinus Torvalds 	return 0;
18841da177e4SLinus Torvalds }
18851da177e4SLinus Torvalds 
18861da177e4SLinus Torvalds static inline void set_task_cpu(struct task_struct *p, unsigned int cpu)
18871da177e4SLinus Torvalds {
18881da177e4SLinus Torvalds }
18891da177e4SLinus Torvalds 
18901da177e4SLinus Torvalds #endif /* CONFIG_SMP */
18911da177e4SLinus Torvalds 
1892d9345c65SPan Xinhui /*
1893d9345c65SPan Xinhui  * In order to reduce various lock holder preemption latencies provide an
1894d9345c65SPan Xinhui  * interface to see if a vCPU is currently running or not.
1895d9345c65SPan Xinhui  *
1896d9345c65SPan Xinhui  * This allows us to terminate optimistic spin loops and block, analogous to
1897d9345c65SPan Xinhui  * the native optimistic spin heuristic of testing if the lock owner task is
1898d9345c65SPan Xinhui  * running or not.
1899d9345c65SPan Xinhui  */
1900d9345c65SPan Xinhui #ifndef vcpu_is_preempted
190142fd8baaSQian Cai static inline bool vcpu_is_preempted(int cpu)
190242fd8baaSQian Cai {
190342fd8baaSQian Cai 	return false;
190442fd8baaSQian Cai }
1905d9345c65SPan Xinhui #endif
1906d9345c65SPan Xinhui 
190796f874e2SRusty Russell extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask);
190896f874e2SRusty Russell extern long sched_getaffinity(pid_t pid, struct cpumask *mask);
19095c45bf27SSiddha, Suresh B 
191082455257SDave Hansen #ifndef TASK_SIZE_OF
191182455257SDave Hansen #define TASK_SIZE_OF(tsk)	TASK_SIZE
191282455257SDave Hansen #endif
191382455257SDave Hansen 
1914d7822b1eSMathieu Desnoyers #ifdef CONFIG_RSEQ
1915d7822b1eSMathieu Desnoyers 
1916d7822b1eSMathieu Desnoyers /*
1917d7822b1eSMathieu Desnoyers  * Map the event mask on the user-space ABI enum rseq_cs_flags
1918d7822b1eSMathieu Desnoyers  * for direct mask checks.
1919d7822b1eSMathieu Desnoyers  */
1920d7822b1eSMathieu Desnoyers enum rseq_event_mask_bits {
1921d7822b1eSMathieu Desnoyers 	RSEQ_EVENT_PREEMPT_BIT	= RSEQ_CS_FLAG_NO_RESTART_ON_PREEMPT_BIT,
1922d7822b1eSMathieu Desnoyers 	RSEQ_EVENT_SIGNAL_BIT	= RSEQ_CS_FLAG_NO_RESTART_ON_SIGNAL_BIT,
1923d7822b1eSMathieu Desnoyers 	RSEQ_EVENT_MIGRATE_BIT	= RSEQ_CS_FLAG_NO_RESTART_ON_MIGRATE_BIT,
1924d7822b1eSMathieu Desnoyers };
1925d7822b1eSMathieu Desnoyers 
1926d7822b1eSMathieu Desnoyers enum rseq_event_mask {
1927d7822b1eSMathieu Desnoyers 	RSEQ_EVENT_PREEMPT	= (1U << RSEQ_EVENT_PREEMPT_BIT),
1928d7822b1eSMathieu Desnoyers 	RSEQ_EVENT_SIGNAL	= (1U << RSEQ_EVENT_SIGNAL_BIT),
1929d7822b1eSMathieu Desnoyers 	RSEQ_EVENT_MIGRATE	= (1U << RSEQ_EVENT_MIGRATE_BIT),
1930d7822b1eSMathieu Desnoyers };
1931d7822b1eSMathieu Desnoyers 
1932d7822b1eSMathieu Desnoyers static inline void rseq_set_notify_resume(struct task_struct *t)
1933d7822b1eSMathieu Desnoyers {
1934d7822b1eSMathieu Desnoyers 	if (t->rseq)
1935d7822b1eSMathieu Desnoyers 		set_tsk_thread_flag(t, TIF_NOTIFY_RESUME);
1936d7822b1eSMathieu Desnoyers }
1937d7822b1eSMathieu Desnoyers 
1938784e0300SWill Deacon void __rseq_handle_notify_resume(struct ksignal *sig, struct pt_regs *regs);
1939d7822b1eSMathieu Desnoyers 
1940784e0300SWill Deacon static inline void rseq_handle_notify_resume(struct ksignal *ksig,
1941784e0300SWill Deacon 					     struct pt_regs *regs)
1942d7822b1eSMathieu Desnoyers {
1943d7822b1eSMathieu Desnoyers 	if (current->rseq)
1944784e0300SWill Deacon 		__rseq_handle_notify_resume(ksig, regs);
1945d7822b1eSMathieu Desnoyers }
1946d7822b1eSMathieu Desnoyers 
1947784e0300SWill Deacon static inline void rseq_signal_deliver(struct ksignal *ksig,
1948784e0300SWill Deacon 				       struct pt_regs *regs)
1949d7822b1eSMathieu Desnoyers {
1950d7822b1eSMathieu Desnoyers 	preempt_disable();
1951d7822b1eSMathieu Desnoyers 	__set_bit(RSEQ_EVENT_SIGNAL_BIT, &current->rseq_event_mask);
1952d7822b1eSMathieu Desnoyers 	preempt_enable();
1953784e0300SWill Deacon 	rseq_handle_notify_resume(ksig, regs);
1954d7822b1eSMathieu Desnoyers }
1955d7822b1eSMathieu Desnoyers 
1956d7822b1eSMathieu Desnoyers /* rseq_preempt() requires preemption to be disabled. */
1957d7822b1eSMathieu Desnoyers static inline void rseq_preempt(struct task_struct *t)
1958d7822b1eSMathieu Desnoyers {
1959d7822b1eSMathieu Desnoyers 	__set_bit(RSEQ_EVENT_PREEMPT_BIT, &t->rseq_event_mask);
1960d7822b1eSMathieu Desnoyers 	rseq_set_notify_resume(t);
1961d7822b1eSMathieu Desnoyers }
1962d7822b1eSMathieu Desnoyers 
1963d7822b1eSMathieu Desnoyers /* rseq_migrate() requires preemption to be disabled. */
1964d7822b1eSMathieu Desnoyers static inline void rseq_migrate(struct task_struct *t)
1965d7822b1eSMathieu Desnoyers {
1966d7822b1eSMathieu Desnoyers 	__set_bit(RSEQ_EVENT_MIGRATE_BIT, &t->rseq_event_mask);
1967d7822b1eSMathieu Desnoyers 	rseq_set_notify_resume(t);
1968d7822b1eSMathieu Desnoyers }
1969d7822b1eSMathieu Desnoyers 
1970d7822b1eSMathieu Desnoyers /*
1971d7822b1eSMathieu Desnoyers  * If parent process has a registered restartable sequences area, the
1972463f550fSMathieu Desnoyers  * child inherits. Unregister rseq for a clone with CLONE_VM set.
1973d7822b1eSMathieu Desnoyers  */
1974d7822b1eSMathieu Desnoyers static inline void rseq_fork(struct task_struct *t, unsigned long clone_flags)
1975d7822b1eSMathieu Desnoyers {
1976463f550fSMathieu Desnoyers 	if (clone_flags & CLONE_VM) {
1977d7822b1eSMathieu Desnoyers 		t->rseq = NULL;
1978d7822b1eSMathieu Desnoyers 		t->rseq_sig = 0;
1979d7822b1eSMathieu Desnoyers 		t->rseq_event_mask = 0;
1980d7822b1eSMathieu Desnoyers 	} else {
1981d7822b1eSMathieu Desnoyers 		t->rseq = current->rseq;
1982d7822b1eSMathieu Desnoyers 		t->rseq_sig = current->rseq_sig;
1983d7822b1eSMathieu Desnoyers 		t->rseq_event_mask = current->rseq_event_mask;
1984d7822b1eSMathieu Desnoyers 	}
1985d7822b1eSMathieu Desnoyers }
1986d7822b1eSMathieu Desnoyers 
1987d7822b1eSMathieu Desnoyers static inline void rseq_execve(struct task_struct *t)
1988d7822b1eSMathieu Desnoyers {
1989d7822b1eSMathieu Desnoyers 	t->rseq = NULL;
1990d7822b1eSMathieu Desnoyers 	t->rseq_sig = 0;
1991d7822b1eSMathieu Desnoyers 	t->rseq_event_mask = 0;
1992d7822b1eSMathieu Desnoyers }
1993d7822b1eSMathieu Desnoyers 
1994d7822b1eSMathieu Desnoyers #else
1995d7822b1eSMathieu Desnoyers 
1996d7822b1eSMathieu Desnoyers static inline void rseq_set_notify_resume(struct task_struct *t)
1997d7822b1eSMathieu Desnoyers {
1998d7822b1eSMathieu Desnoyers }
1999784e0300SWill Deacon static inline void rseq_handle_notify_resume(struct ksignal *ksig,
2000784e0300SWill Deacon 					     struct pt_regs *regs)
2001d7822b1eSMathieu Desnoyers {
2002d7822b1eSMathieu Desnoyers }
2003784e0300SWill Deacon static inline void rseq_signal_deliver(struct ksignal *ksig,
2004784e0300SWill Deacon 				       struct pt_regs *regs)
2005d7822b1eSMathieu Desnoyers {
2006d7822b1eSMathieu Desnoyers }
2007d7822b1eSMathieu Desnoyers static inline void rseq_preempt(struct task_struct *t)
2008d7822b1eSMathieu Desnoyers {
2009d7822b1eSMathieu Desnoyers }
2010d7822b1eSMathieu Desnoyers static inline void rseq_migrate(struct task_struct *t)
2011d7822b1eSMathieu Desnoyers {
2012d7822b1eSMathieu Desnoyers }
2013d7822b1eSMathieu Desnoyers static inline void rseq_fork(struct task_struct *t, unsigned long clone_flags)
2014d7822b1eSMathieu Desnoyers {
2015d7822b1eSMathieu Desnoyers }
2016d7822b1eSMathieu Desnoyers static inline void rseq_execve(struct task_struct *t)
2017d7822b1eSMathieu Desnoyers {
2018d7822b1eSMathieu Desnoyers }
2019d7822b1eSMathieu Desnoyers 
2020d7822b1eSMathieu Desnoyers #endif
2021d7822b1eSMathieu Desnoyers 
202273ab1cb2STaehee Yoo void __exit_umh(struct task_struct *tsk);
202373ab1cb2STaehee Yoo 
202473ab1cb2STaehee Yoo static inline void exit_umh(struct task_struct *tsk)
202573ab1cb2STaehee Yoo {
202673ab1cb2STaehee Yoo 	if (unlikely(tsk->flags & PF_UMH))
202773ab1cb2STaehee Yoo 		__exit_umh(tsk);
202873ab1cb2STaehee Yoo }
202973ab1cb2STaehee Yoo 
2030d7822b1eSMathieu Desnoyers #ifdef CONFIG_DEBUG_RSEQ
2031d7822b1eSMathieu Desnoyers 
2032d7822b1eSMathieu Desnoyers void rseq_syscall(struct pt_regs *regs);
2033d7822b1eSMathieu Desnoyers 
2034d7822b1eSMathieu Desnoyers #else
2035d7822b1eSMathieu Desnoyers 
2036d7822b1eSMathieu Desnoyers static inline void rseq_syscall(struct pt_regs *regs)
2037d7822b1eSMathieu Desnoyers {
2038d7822b1eSMathieu Desnoyers }
2039d7822b1eSMathieu Desnoyers 
2040d7822b1eSMathieu Desnoyers #endif
2041d7822b1eSMathieu Desnoyers 
20423c93a0c0SQais Yousef const struct sched_avg *sched_trace_cfs_rq_avg(struct cfs_rq *cfs_rq);
20433c93a0c0SQais Yousef char *sched_trace_cfs_rq_path(struct cfs_rq *cfs_rq, char *str, int len);
20443c93a0c0SQais Yousef int sched_trace_cfs_rq_cpu(struct cfs_rq *cfs_rq);
20453c93a0c0SQais Yousef 
20463c93a0c0SQais Yousef const struct sched_avg *sched_trace_rq_avg_rt(struct rq *rq);
20473c93a0c0SQais Yousef const struct sched_avg *sched_trace_rq_avg_dl(struct rq *rq);
20483c93a0c0SQais Yousef const struct sched_avg *sched_trace_rq_avg_irq(struct rq *rq);
20493c93a0c0SQais Yousef 
20503c93a0c0SQais Yousef int sched_trace_rq_cpu(struct rq *rq);
20513c93a0c0SQais Yousef 
20523c93a0c0SQais Yousef const struct cpumask *sched_trace_rd_span(struct root_domain *rd);
20533c93a0c0SQais Yousef 
20541da177e4SLinus Torvalds #endif
2055