xref: /linux/include/linux/sched.h (revision 69842cba9ace84849bb9b8edcdf2cefccd97901c)
1b2441318SGreg Kroah-Hartman /* SPDX-License-Identifier: GPL-2.0 */
21da177e4SLinus Torvalds #ifndef _LINUX_SCHED_H
31da177e4SLinus Torvalds #define _LINUX_SCHED_H
41da177e4SLinus Torvalds 
55eca1c10SIngo Molnar /*
65eca1c10SIngo Molnar  * Define 'struct task_struct' and provide the main scheduler
75eca1c10SIngo Molnar  * APIs (schedule(), wakeup variants, etc.)
85eca1c10SIngo Molnar  */
95eca1c10SIngo Molnar 
10607ca46eSDavid Howells #include <uapi/linux/sched.h>
11b7b3c76aSDavid Woodhouse 
1270b8157eSIngo Molnar #include <asm/current.h>
1370b8157eSIngo Molnar 
145eca1c10SIngo Molnar #include <linux/pid.h>
155eca1c10SIngo Molnar #include <linux/sem.h>
165eca1c10SIngo Molnar #include <linux/shm.h>
175eca1c10SIngo Molnar #include <linux/kcov.h>
185eca1c10SIngo Molnar #include <linux/mutex.h>
195eca1c10SIngo Molnar #include <linux/plist.h>
205eca1c10SIngo Molnar #include <linux/hrtimer.h>
215eca1c10SIngo Molnar #include <linux/seccomp.h>
225eca1c10SIngo Molnar #include <linux/nodemask.h>
235eca1c10SIngo Molnar #include <linux/rcupdate.h>
24ec1d2819SElena Reshetova #include <linux/refcount.h>
255eca1c10SIngo Molnar #include <linux/resource.h>
265eca1c10SIngo Molnar #include <linux/latencytop.h>
275eca1c10SIngo Molnar #include <linux/sched/prio.h>
285eca1c10SIngo Molnar #include <linux/signal_types.h>
295eca1c10SIngo Molnar #include <linux/mm_types_task.h>
305eca1c10SIngo Molnar #include <linux/task_io_accounting.h>
31d7822b1eSMathieu Desnoyers #include <linux/rseq.h>
325eca1c10SIngo Molnar 
335eca1c10SIngo Molnar /* task_struct member predeclarations (sorted alphabetically): */
34c7af7877SIngo Molnar struct audit_context;
35c7af7877SIngo Molnar struct backing_dev_info;
36c7af7877SIngo Molnar struct bio_list;
37c7af7877SIngo Molnar struct blk_plug;
383c93a0c0SQais Yousef struct capture_control;
39c7af7877SIngo Molnar struct cfs_rq;
40c7af7877SIngo Molnar struct fs_struct;
41c7af7877SIngo Molnar struct futex_pi_state;
42c7af7877SIngo Molnar struct io_context;
43c7af7877SIngo Molnar struct mempolicy;
44c7af7877SIngo Molnar struct nameidata;
45c7af7877SIngo Molnar struct nsproxy;
46c7af7877SIngo Molnar struct perf_event_context;
47c7af7877SIngo Molnar struct pid_namespace;
48c7af7877SIngo Molnar struct pipe_inode_info;
49c7af7877SIngo Molnar struct rcu_node;
50c7af7877SIngo Molnar struct reclaim_state;
51c7af7877SIngo Molnar struct robust_list_head;
523c93a0c0SQais Yousef struct root_domain;
533c93a0c0SQais Yousef struct rq;
54e2d1e2aeSIngo Molnar struct sched_attr;
55e2d1e2aeSIngo Molnar struct sched_param;
5643ae34cbSIngo Molnar struct seq_file;
57c7af7877SIngo Molnar struct sighand_struct;
58c7af7877SIngo Molnar struct signal_struct;
59c7af7877SIngo Molnar struct task_delay_info;
604cf86d77SIngo Molnar struct task_group;
611da177e4SLinus Torvalds 
624a8342d2SLinus Torvalds /*
634a8342d2SLinus Torvalds  * Task state bitmask. NOTE! These bits are also
644a8342d2SLinus Torvalds  * encoded in fs/proc/array.c: get_task_state().
654a8342d2SLinus Torvalds  *
664a8342d2SLinus Torvalds  * We have two separate sets of flags: task->state
674a8342d2SLinus Torvalds  * is about runnability, while task->exit_state are
684a8342d2SLinus Torvalds  * about the task exiting. Confusing, but this way
694a8342d2SLinus Torvalds  * modifying one set can't modify the other one by
704a8342d2SLinus Torvalds  * mistake.
714a8342d2SLinus Torvalds  */
725eca1c10SIngo Molnar 
735eca1c10SIngo Molnar /* Used in tsk->state: */
7492c4bc9fSPeter Zijlstra #define TASK_RUNNING			0x0000
7592c4bc9fSPeter Zijlstra #define TASK_INTERRUPTIBLE		0x0001
7692c4bc9fSPeter Zijlstra #define TASK_UNINTERRUPTIBLE		0x0002
7792c4bc9fSPeter Zijlstra #define __TASK_STOPPED			0x0004
7892c4bc9fSPeter Zijlstra #define __TASK_TRACED			0x0008
795eca1c10SIngo Molnar /* Used in tsk->exit_state: */
8092c4bc9fSPeter Zijlstra #define EXIT_DEAD			0x0010
8192c4bc9fSPeter Zijlstra #define EXIT_ZOMBIE			0x0020
82abd50b39SOleg Nesterov #define EXIT_TRACE			(EXIT_ZOMBIE | EXIT_DEAD)
835eca1c10SIngo Molnar /* Used in tsk->state again: */
848ef9925bSPeter Zijlstra #define TASK_PARKED			0x0040
858ef9925bSPeter Zijlstra #define TASK_DEAD			0x0080
868ef9925bSPeter Zijlstra #define TASK_WAKEKILL			0x0100
878ef9925bSPeter Zijlstra #define TASK_WAKING			0x0200
8892c4bc9fSPeter Zijlstra #define TASK_NOLOAD			0x0400
8992c4bc9fSPeter Zijlstra #define TASK_NEW			0x0800
9092c4bc9fSPeter Zijlstra #define TASK_STATE_MAX			0x1000
91f021a3c2SMatthew Wilcox 
925eca1c10SIngo Molnar /* Convenience macros for the sake of set_current_state: */
93f021a3c2SMatthew Wilcox #define TASK_KILLABLE			(TASK_WAKEKILL | TASK_UNINTERRUPTIBLE)
94f021a3c2SMatthew Wilcox #define TASK_STOPPED			(TASK_WAKEKILL | __TASK_STOPPED)
95f021a3c2SMatthew Wilcox #define TASK_TRACED			(TASK_WAKEKILL | __TASK_TRACED)
961da177e4SLinus Torvalds 
9780ed87c8SPeter Zijlstra #define TASK_IDLE			(TASK_UNINTERRUPTIBLE | TASK_NOLOAD)
9880ed87c8SPeter Zijlstra 
995eca1c10SIngo Molnar /* Convenience macros for the sake of wake_up(): */
10092a1f4bcSMatthew Wilcox #define TASK_NORMAL			(TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE)
10192a1f4bcSMatthew Wilcox 
1025eca1c10SIngo Molnar /* get_task_state(): */
10392a1f4bcSMatthew Wilcox #define TASK_REPORT			(TASK_RUNNING | TASK_INTERRUPTIBLE | \
104f021a3c2SMatthew Wilcox 					 TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \
1058ef9925bSPeter Zijlstra 					 __TASK_TRACED | EXIT_DEAD | EXIT_ZOMBIE | \
1068ef9925bSPeter Zijlstra 					 TASK_PARKED)
10792a1f4bcSMatthew Wilcox 
108f021a3c2SMatthew Wilcox #define task_is_traced(task)		((task->state & __TASK_TRACED) != 0)
1095eca1c10SIngo Molnar 
110f021a3c2SMatthew Wilcox #define task_is_stopped(task)		((task->state & __TASK_STOPPED) != 0)
1115eca1c10SIngo Molnar 
1125eca1c10SIngo Molnar #define task_is_stopped_or_traced(task)	((task->state & (__TASK_STOPPED | __TASK_TRACED)) != 0)
1135eca1c10SIngo Molnar 
1145eca1c10SIngo Molnar #define task_contributes_to_load(task)	((task->state & TASK_UNINTERRUPTIBLE) != 0 && \
11580ed87c8SPeter Zijlstra 					 (task->flags & PF_FROZEN) == 0 && \
11680ed87c8SPeter Zijlstra 					 (task->state & TASK_NOLOAD) == 0)
1171da177e4SLinus Torvalds 
1188eb23b9fSPeter Zijlstra #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
1198eb23b9fSPeter Zijlstra 
120b5bf9a90SPeter Zijlstra /*
121b5bf9a90SPeter Zijlstra  * Special states are those that do not use the normal wait-loop pattern. See
122b5bf9a90SPeter Zijlstra  * the comment with set_special_state().
123b5bf9a90SPeter Zijlstra  */
124b5bf9a90SPeter Zijlstra #define is_special_task_state(state)				\
1251cef1150SPeter Zijlstra 	((state) & (__TASK_STOPPED | __TASK_TRACED | TASK_PARKED | TASK_DEAD))
126b5bf9a90SPeter Zijlstra 
1278eb23b9fSPeter Zijlstra #define __set_current_state(state_value)			\
1288eb23b9fSPeter Zijlstra 	do {							\
129b5bf9a90SPeter Zijlstra 		WARN_ON_ONCE(is_special_task_state(state_value));\
1308eb23b9fSPeter Zijlstra 		current->task_state_change = _THIS_IP_;		\
1318eb23b9fSPeter Zijlstra 		current->state = (state_value);			\
1328eb23b9fSPeter Zijlstra 	} while (0)
133b5bf9a90SPeter Zijlstra 
1348eb23b9fSPeter Zijlstra #define set_current_state(state_value)				\
1358eb23b9fSPeter Zijlstra 	do {							\
136b5bf9a90SPeter Zijlstra 		WARN_ON_ONCE(is_special_task_state(state_value));\
1378eb23b9fSPeter Zijlstra 		current->task_state_change = _THIS_IP_;		\
138b92b8b35SPeter Zijlstra 		smp_store_mb(current->state, (state_value));	\
1398eb23b9fSPeter Zijlstra 	} while (0)
1408eb23b9fSPeter Zijlstra 
141b5bf9a90SPeter Zijlstra #define set_special_state(state_value)					\
142b5bf9a90SPeter Zijlstra 	do {								\
143b5bf9a90SPeter Zijlstra 		unsigned long flags; /* may shadow */			\
144b5bf9a90SPeter Zijlstra 		WARN_ON_ONCE(!is_special_task_state(state_value));	\
145b5bf9a90SPeter Zijlstra 		raw_spin_lock_irqsave(&current->pi_lock, flags);	\
146b5bf9a90SPeter Zijlstra 		current->task_state_change = _THIS_IP_;			\
147b5bf9a90SPeter Zijlstra 		current->state = (state_value);				\
148b5bf9a90SPeter Zijlstra 		raw_spin_unlock_irqrestore(&current->pi_lock, flags);	\
149b5bf9a90SPeter Zijlstra 	} while (0)
1508eb23b9fSPeter Zijlstra #else
151498d0c57SAndrew Morton /*
152498d0c57SAndrew Morton  * set_current_state() includes a barrier so that the write of current->state
153498d0c57SAndrew Morton  * is correctly serialised wrt the caller's subsequent test of whether to
154498d0c57SAndrew Morton  * actually sleep:
155498d0c57SAndrew Morton  *
156a2250238SPeter Zijlstra  *   for (;;) {
157498d0c57SAndrew Morton  *	set_current_state(TASK_UNINTERRUPTIBLE);
158a2250238SPeter Zijlstra  *	if (!need_sleep)
159a2250238SPeter Zijlstra  *		break;
160498d0c57SAndrew Morton  *
161a2250238SPeter Zijlstra  *	schedule();
162a2250238SPeter Zijlstra  *   }
163a2250238SPeter Zijlstra  *   __set_current_state(TASK_RUNNING);
164a2250238SPeter Zijlstra  *
165a2250238SPeter Zijlstra  * If the caller does not need such serialisation (because, for instance, the
166a2250238SPeter Zijlstra  * condition test and condition change and wakeup are under the same lock) then
167a2250238SPeter Zijlstra  * use __set_current_state().
168a2250238SPeter Zijlstra  *
169a2250238SPeter Zijlstra  * The above is typically ordered against the wakeup, which does:
170a2250238SPeter Zijlstra  *
171a2250238SPeter Zijlstra  *   need_sleep = false;
172a2250238SPeter Zijlstra  *   wake_up_state(p, TASK_UNINTERRUPTIBLE);
173a2250238SPeter Zijlstra  *
1747696f991SAndrea Parri  * where wake_up_state() executes a full memory barrier before accessing the
1757696f991SAndrea Parri  * task state.
176a2250238SPeter Zijlstra  *
177a2250238SPeter Zijlstra  * Wakeup will do: if (@state & p->state) p->state = TASK_RUNNING, that is,
178a2250238SPeter Zijlstra  * once it observes the TASK_UNINTERRUPTIBLE store the waking CPU can issue a
179a2250238SPeter Zijlstra  * TASK_RUNNING store which can collide with __set_current_state(TASK_RUNNING).
180a2250238SPeter Zijlstra  *
181b5bf9a90SPeter Zijlstra  * However, with slightly different timing the wakeup TASK_RUNNING store can
182dfcb245eSIngo Molnar  * also collide with the TASK_UNINTERRUPTIBLE store. Losing that store is not
183b5bf9a90SPeter Zijlstra  * a problem either because that will result in one extra go around the loop
184b5bf9a90SPeter Zijlstra  * and our @cond test will save the day.
185a2250238SPeter Zijlstra  *
186a2250238SPeter Zijlstra  * Also see the comments of try_to_wake_up().
187498d0c57SAndrew Morton  */
188b5bf9a90SPeter Zijlstra #define __set_current_state(state_value)				\
189b5bf9a90SPeter Zijlstra 	current->state = (state_value)
190b5bf9a90SPeter Zijlstra 
191b5bf9a90SPeter Zijlstra #define set_current_state(state_value)					\
192b5bf9a90SPeter Zijlstra 	smp_store_mb(current->state, (state_value))
193b5bf9a90SPeter Zijlstra 
194b5bf9a90SPeter Zijlstra /*
195b5bf9a90SPeter Zijlstra  * set_special_state() should be used for those states when the blocking task
196b5bf9a90SPeter Zijlstra  * can not use the regular condition based wait-loop. In that case we must
197b5bf9a90SPeter Zijlstra  * serialize against wakeups such that any possible in-flight TASK_RUNNING stores
198b5bf9a90SPeter Zijlstra  * will not collide with our state change.
199b5bf9a90SPeter Zijlstra  */
200b5bf9a90SPeter Zijlstra #define set_special_state(state_value)					\
201b5bf9a90SPeter Zijlstra 	do {								\
202b5bf9a90SPeter Zijlstra 		unsigned long flags; /* may shadow */			\
203b5bf9a90SPeter Zijlstra 		raw_spin_lock_irqsave(&current->pi_lock, flags);	\
204b5bf9a90SPeter Zijlstra 		current->state = (state_value);				\
205b5bf9a90SPeter Zijlstra 		raw_spin_unlock_irqrestore(&current->pi_lock, flags);	\
206b5bf9a90SPeter Zijlstra 	} while (0)
207b5bf9a90SPeter Zijlstra 
2088eb23b9fSPeter Zijlstra #endif
2098eb23b9fSPeter Zijlstra 
2105eca1c10SIngo Molnar /* Task command name length: */
2111da177e4SLinus Torvalds #define TASK_COMM_LEN			16
2121da177e4SLinus Torvalds 
2131da177e4SLinus Torvalds extern void scheduler_tick(void);
2141da177e4SLinus Torvalds 
2151da177e4SLinus Torvalds #define	MAX_SCHEDULE_TIMEOUT		LONG_MAX
2165eca1c10SIngo Molnar 
2175eca1c10SIngo Molnar extern long schedule_timeout(long timeout);
2185eca1c10SIngo Molnar extern long schedule_timeout_interruptible(long timeout);
2195eca1c10SIngo Molnar extern long schedule_timeout_killable(long timeout);
2205eca1c10SIngo Molnar extern long schedule_timeout_uninterruptible(long timeout);
2215eca1c10SIngo Molnar extern long schedule_timeout_idle(long timeout);
2221da177e4SLinus Torvalds asmlinkage void schedule(void);
223c5491ea7SThomas Gleixner extern void schedule_preempt_disabled(void);
2241da177e4SLinus Torvalds 
22510ab5643STejun Heo extern int __must_check io_schedule_prepare(void);
22610ab5643STejun Heo extern void io_schedule_finish(int token);
2279cff8adeSNeilBrown extern long io_schedule_timeout(long timeout);
22810ab5643STejun Heo extern void io_schedule(void);
2299cff8adeSNeilBrown 
230f06febc9SFrank Mayhar /**
2310ba42a59SMasanari Iida  * struct prev_cputime - snapshot of system and user cputime
232d37f761dSFrederic Weisbecker  * @utime: time spent in user mode
233d37f761dSFrederic Weisbecker  * @stime: time spent in system mode
2349d7fb042SPeter Zijlstra  * @lock: protects the above two fields
235d37f761dSFrederic Weisbecker  *
2369d7fb042SPeter Zijlstra  * Stores previous user/system time values such that we can guarantee
2379d7fb042SPeter Zijlstra  * monotonicity.
238d37f761dSFrederic Weisbecker  */
2399d7fb042SPeter Zijlstra struct prev_cputime {
2409d7fb042SPeter Zijlstra #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
2415613fda9SFrederic Weisbecker 	u64				utime;
2425613fda9SFrederic Weisbecker 	u64				stime;
2439d7fb042SPeter Zijlstra 	raw_spinlock_t			lock;
2449d7fb042SPeter Zijlstra #endif
245d37f761dSFrederic Weisbecker };
246d37f761dSFrederic Weisbecker 
247d37f761dSFrederic Weisbecker /**
248f06febc9SFrank Mayhar  * struct task_cputime - collected CPU time counts
2495613fda9SFrederic Weisbecker  * @utime:		time spent in user mode, in nanoseconds
2505613fda9SFrederic Weisbecker  * @stime:		time spent in kernel mode, in nanoseconds
251f06febc9SFrank Mayhar  * @sum_exec_runtime:	total time spent on the CPU, in nanoseconds
252f06febc9SFrank Mayhar  *
2539d7fb042SPeter Zijlstra  * This structure groups together three kinds of CPU time that are tracked for
2549d7fb042SPeter Zijlstra  * threads and thread groups.  Most things considering CPU time want to group
2559d7fb042SPeter Zijlstra  * these counts together and treat all three of them in parallel.
256f06febc9SFrank Mayhar  */
257f06febc9SFrank Mayhar struct task_cputime {
2585613fda9SFrederic Weisbecker 	u64				utime;
2595613fda9SFrederic Weisbecker 	u64				stime;
260f06febc9SFrank Mayhar 	unsigned long long		sum_exec_runtime;
261f06febc9SFrank Mayhar };
2629d7fb042SPeter Zijlstra 
2635eca1c10SIngo Molnar /* Alternate field names when used on cache expirations: */
264f06febc9SFrank Mayhar #define virt_exp			utime
2659d7fb042SPeter Zijlstra #define prof_exp			stime
266f06febc9SFrank Mayhar #define sched_exp			sum_exec_runtime
267f06febc9SFrank Mayhar 
268bac5b6b6SFrederic Weisbecker enum vtime_state {
269bac5b6b6SFrederic Weisbecker 	/* Task is sleeping or running in a CPU with VTIME inactive: */
270bac5b6b6SFrederic Weisbecker 	VTIME_INACTIVE = 0,
271bac5b6b6SFrederic Weisbecker 	/* Task runs in userspace in a CPU with VTIME active: */
272bac5b6b6SFrederic Weisbecker 	VTIME_USER,
273bac5b6b6SFrederic Weisbecker 	/* Task runs in kernelspace in a CPU with VTIME active: */
274bac5b6b6SFrederic Weisbecker 	VTIME_SYS,
275bac5b6b6SFrederic Weisbecker };
276bac5b6b6SFrederic Weisbecker 
277bac5b6b6SFrederic Weisbecker struct vtime {
278bac5b6b6SFrederic Weisbecker 	seqcount_t		seqcount;
279bac5b6b6SFrederic Weisbecker 	unsigned long long	starttime;
280bac5b6b6SFrederic Weisbecker 	enum vtime_state	state;
2812a42eb95SWanpeng Li 	u64			utime;
2822a42eb95SWanpeng Li 	u64			stime;
2832a42eb95SWanpeng Li 	u64			gtime;
284bac5b6b6SFrederic Weisbecker };
285bac5b6b6SFrederic Weisbecker 
286*69842cbaSPatrick Bellasi /*
287*69842cbaSPatrick Bellasi  * Utilization clamp constraints.
288*69842cbaSPatrick Bellasi  * @UCLAMP_MIN:	Minimum utilization
289*69842cbaSPatrick Bellasi  * @UCLAMP_MAX:	Maximum utilization
290*69842cbaSPatrick Bellasi  * @UCLAMP_CNT:	Utilization clamp constraints count
291*69842cbaSPatrick Bellasi  */
292*69842cbaSPatrick Bellasi enum uclamp_id {
293*69842cbaSPatrick Bellasi 	UCLAMP_MIN = 0,
294*69842cbaSPatrick Bellasi 	UCLAMP_MAX,
295*69842cbaSPatrick Bellasi 	UCLAMP_CNT
296*69842cbaSPatrick Bellasi };
297*69842cbaSPatrick Bellasi 
2981da177e4SLinus Torvalds struct sched_info {
2997f5f8e8dSIngo Molnar #ifdef CONFIG_SCHED_INFO
3005eca1c10SIngo Molnar 	/* Cumulative counters: */
3011da177e4SLinus Torvalds 
3025eca1c10SIngo Molnar 	/* # of times we have run on this CPU: */
3035eca1c10SIngo Molnar 	unsigned long			pcount;
3045eca1c10SIngo Molnar 
3055eca1c10SIngo Molnar 	/* Time spent waiting on a runqueue: */
3065eca1c10SIngo Molnar 	unsigned long long		run_delay;
3075eca1c10SIngo Molnar 
3085eca1c10SIngo Molnar 	/* Timestamps: */
3095eca1c10SIngo Molnar 
3105eca1c10SIngo Molnar 	/* When did we last run on a CPU? */
3115eca1c10SIngo Molnar 	unsigned long long		last_arrival;
3125eca1c10SIngo Molnar 
3135eca1c10SIngo Molnar 	/* When were we last queued to run? */
3145eca1c10SIngo Molnar 	unsigned long long		last_queued;
3155eca1c10SIngo Molnar 
316f6db8347SNaveen N. Rao #endif /* CONFIG_SCHED_INFO */
3177f5f8e8dSIngo Molnar };
3181da177e4SLinus Torvalds 
3191da177e4SLinus Torvalds /*
3206ecdd749SYuyang Du  * Integer metrics need fixed point arithmetic, e.g., sched/fair
3216ecdd749SYuyang Du  * has a few: load, load_avg, util_avg, freq, and capacity.
3226ecdd749SYuyang Du  *
3236ecdd749SYuyang Du  * We define a basic fixed point arithmetic range, and then formalize
3246ecdd749SYuyang Du  * all these metrics based on that basic range.
3256ecdd749SYuyang Du  */
3266ecdd749SYuyang Du # define SCHED_FIXEDPOINT_SHIFT		10
3276ecdd749SYuyang Du # define SCHED_FIXEDPOINT_SCALE		(1L << SCHED_FIXEDPOINT_SHIFT)
3286ecdd749SYuyang Du 
329*69842cbaSPatrick Bellasi /* Increase resolution of cpu_capacity calculations */
330*69842cbaSPatrick Bellasi # define SCHED_CAPACITY_SHIFT		SCHED_FIXEDPOINT_SHIFT
331*69842cbaSPatrick Bellasi # define SCHED_CAPACITY_SCALE		(1L << SCHED_CAPACITY_SHIFT)
332*69842cbaSPatrick Bellasi 
33320b8a59fSIngo Molnar struct load_weight {
3349dbdb155SPeter Zijlstra 	unsigned long			weight;
3359dbdb155SPeter Zijlstra 	u32				inv_weight;
33620b8a59fSIngo Molnar };
33720b8a59fSIngo Molnar 
3387f65ea42SPatrick Bellasi /**
3397f65ea42SPatrick Bellasi  * struct util_est - Estimation utilization of FAIR tasks
3407f65ea42SPatrick Bellasi  * @enqueued: instantaneous estimated utilization of a task/cpu
3417f65ea42SPatrick Bellasi  * @ewma:     the Exponential Weighted Moving Average (EWMA)
3427f65ea42SPatrick Bellasi  *            utilization of a task
3437f65ea42SPatrick Bellasi  *
3447f65ea42SPatrick Bellasi  * Support data structure to track an Exponential Weighted Moving Average
3457f65ea42SPatrick Bellasi  * (EWMA) of a FAIR task's utilization. New samples are added to the moving
3467f65ea42SPatrick Bellasi  * average each time a task completes an activation. Sample's weight is chosen
3477f65ea42SPatrick Bellasi  * so that the EWMA will be relatively insensitive to transient changes to the
3487f65ea42SPatrick Bellasi  * task's workload.
3497f65ea42SPatrick Bellasi  *
3507f65ea42SPatrick Bellasi  * The enqueued attribute has a slightly different meaning for tasks and cpus:
3517f65ea42SPatrick Bellasi  * - task:   the task's util_avg at last task dequeue time
3527f65ea42SPatrick Bellasi  * - cfs_rq: the sum of util_est.enqueued for each RUNNABLE task on that CPU
3537f65ea42SPatrick Bellasi  * Thus, the util_est.enqueued of a task represents the contribution on the
3547f65ea42SPatrick Bellasi  * estimated utilization of the CPU where that task is currently enqueued.
3557f65ea42SPatrick Bellasi  *
3567f65ea42SPatrick Bellasi  * Only for tasks we track a moving average of the past instantaneous
3577f65ea42SPatrick Bellasi  * estimated utilization. This allows to absorb sporadic drops in utilization
3587f65ea42SPatrick Bellasi  * of an otherwise almost periodic task.
3597f65ea42SPatrick Bellasi  */
3607f65ea42SPatrick Bellasi struct util_est {
3617f65ea42SPatrick Bellasi 	unsigned int			enqueued;
3627f65ea42SPatrick Bellasi 	unsigned int			ewma;
3637f65ea42SPatrick Bellasi #define UTIL_EST_WEIGHT_SHIFT		2
364317d359dSPeter Zijlstra } __attribute__((__aligned__(sizeof(u64))));
3657f65ea42SPatrick Bellasi 
3669d89c257SYuyang Du /*
3677b595334SYuyang Du  * The load_avg/util_avg accumulates an infinite geometric series
3687b595334SYuyang Du  * (see __update_load_avg() in kernel/sched/fair.c).
3697b595334SYuyang Du  *
3707b595334SYuyang Du  * [load_avg definition]
3717b595334SYuyang Du  *
3727b595334SYuyang Du  *   load_avg = runnable% * scale_load_down(load)
3737b595334SYuyang Du  *
3747b595334SYuyang Du  * where runnable% is the time ratio that a sched_entity is runnable.
3757b595334SYuyang Du  * For cfs_rq, it is the aggregated load_avg of all runnable and
3769d89c257SYuyang Du  * blocked sched_entities.
3777b595334SYuyang Du  *
3787b595334SYuyang Du  * [util_avg definition]
3797b595334SYuyang Du  *
3807b595334SYuyang Du  *   util_avg = running% * SCHED_CAPACITY_SCALE
3817b595334SYuyang Du  *
3827b595334SYuyang Du  * where running% is the time ratio that a sched_entity is running on
3837b595334SYuyang Du  * a CPU. For cfs_rq, it is the aggregated util_avg of all runnable
3847b595334SYuyang Du  * and blocked sched_entities.
3857b595334SYuyang Du  *
38623127296SVincent Guittot  * load_avg and util_avg don't direcly factor frequency scaling and CPU
38723127296SVincent Guittot  * capacity scaling. The scaling is done through the rq_clock_pelt that
38823127296SVincent Guittot  * is used for computing those signals (see update_rq_clock_pelt())
3897b595334SYuyang Du  *
39023127296SVincent Guittot  * N.B., the above ratios (runnable% and running%) themselves are in the
39123127296SVincent Guittot  * range of [0, 1]. To do fixed point arithmetics, we therefore scale them
39223127296SVincent Guittot  * to as large a range as necessary. This is for example reflected by
39323127296SVincent Guittot  * util_avg's SCHED_CAPACITY_SCALE.
3947b595334SYuyang Du  *
3957b595334SYuyang Du  * [Overflow issue]
3967b595334SYuyang Du  *
3977b595334SYuyang Du  * The 64-bit load_sum can have 4353082796 (=2^64/47742/88761) entities
3987b595334SYuyang Du  * with the highest load (=88761), always runnable on a single cfs_rq,
3997b595334SYuyang Du  * and should not overflow as the number already hits PID_MAX_LIMIT.
4007b595334SYuyang Du  *
4017b595334SYuyang Du  * For all other cases (including 32-bit kernels), struct load_weight's
4027b595334SYuyang Du  * weight will overflow first before we do, because:
4037b595334SYuyang Du  *
4047b595334SYuyang Du  *    Max(load_avg) <= Max(load.weight)
4057b595334SYuyang Du  *
4067b595334SYuyang Du  * Then it is the load_weight's responsibility to consider overflow
4077b595334SYuyang Du  * issues.
4089d89c257SYuyang Du  */
4099d85f21cSPaul Turner struct sched_avg {
4105eca1c10SIngo Molnar 	u64				last_update_time;
4115eca1c10SIngo Molnar 	u64				load_sum;
4121ea6c46aSPeter Zijlstra 	u64				runnable_load_sum;
4135eca1c10SIngo Molnar 	u32				util_sum;
4145eca1c10SIngo Molnar 	u32				period_contrib;
4155eca1c10SIngo Molnar 	unsigned long			load_avg;
4161ea6c46aSPeter Zijlstra 	unsigned long			runnable_load_avg;
4175eca1c10SIngo Molnar 	unsigned long			util_avg;
4187f65ea42SPatrick Bellasi 	struct util_est			util_est;
419317d359dSPeter Zijlstra } ____cacheline_aligned;
4209d85f21cSPaul Turner 
42141acab88SLucas De Marchi struct sched_statistics {
4227f5f8e8dSIngo Molnar #ifdef CONFIG_SCHEDSTATS
42394c18227SIngo Molnar 	u64				wait_start;
42494c18227SIngo Molnar 	u64				wait_max;
4256d082592SArjan van de Ven 	u64				wait_count;
4266d082592SArjan van de Ven 	u64				wait_sum;
4278f0dfc34SArjan van de Ven 	u64				iowait_count;
4288f0dfc34SArjan van de Ven 	u64				iowait_sum;
42994c18227SIngo Molnar 
43094c18227SIngo Molnar 	u64				sleep_start;
43120b8a59fSIngo Molnar 	u64				sleep_max;
43294c18227SIngo Molnar 	s64				sum_sleep_runtime;
43394c18227SIngo Molnar 
43494c18227SIngo Molnar 	u64				block_start;
43520b8a59fSIngo Molnar 	u64				block_max;
43620b8a59fSIngo Molnar 	u64				exec_max;
437eba1ed4bSIngo Molnar 	u64				slice_max;
438cc367732SIngo Molnar 
439cc367732SIngo Molnar 	u64				nr_migrations_cold;
440cc367732SIngo Molnar 	u64				nr_failed_migrations_affine;
441cc367732SIngo Molnar 	u64				nr_failed_migrations_running;
442cc367732SIngo Molnar 	u64				nr_failed_migrations_hot;
443cc367732SIngo Molnar 	u64				nr_forced_migrations;
444cc367732SIngo Molnar 
445cc367732SIngo Molnar 	u64				nr_wakeups;
446cc367732SIngo Molnar 	u64				nr_wakeups_sync;
447cc367732SIngo Molnar 	u64				nr_wakeups_migrate;
448cc367732SIngo Molnar 	u64				nr_wakeups_local;
449cc367732SIngo Molnar 	u64				nr_wakeups_remote;
450cc367732SIngo Molnar 	u64				nr_wakeups_affine;
451cc367732SIngo Molnar 	u64				nr_wakeups_affine_attempts;
452cc367732SIngo Molnar 	u64				nr_wakeups_passive;
453cc367732SIngo Molnar 	u64				nr_wakeups_idle;
45441acab88SLucas De Marchi #endif
4557f5f8e8dSIngo Molnar };
45641acab88SLucas De Marchi 
45741acab88SLucas De Marchi struct sched_entity {
4585eca1c10SIngo Molnar 	/* For load-balancing: */
4595eca1c10SIngo Molnar 	struct load_weight		load;
4601ea6c46aSPeter Zijlstra 	unsigned long			runnable_weight;
46141acab88SLucas De Marchi 	struct rb_node			run_node;
46241acab88SLucas De Marchi 	struct list_head		group_node;
46341acab88SLucas De Marchi 	unsigned int			on_rq;
46441acab88SLucas De Marchi 
46541acab88SLucas De Marchi 	u64				exec_start;
46641acab88SLucas De Marchi 	u64				sum_exec_runtime;
46741acab88SLucas De Marchi 	u64				vruntime;
46841acab88SLucas De Marchi 	u64				prev_sum_exec_runtime;
46941acab88SLucas De Marchi 
47041acab88SLucas De Marchi 	u64				nr_migrations;
47141acab88SLucas De Marchi 
47241acab88SLucas De Marchi 	struct sched_statistics		statistics;
47394c18227SIngo Molnar 
47420b8a59fSIngo Molnar #ifdef CONFIG_FAIR_GROUP_SCHED
475fed14d45SPeter Zijlstra 	int				depth;
47620b8a59fSIngo Molnar 	struct sched_entity		*parent;
47720b8a59fSIngo Molnar 	/* rq on which this entity is (to be) queued: */
47820b8a59fSIngo Molnar 	struct cfs_rq			*cfs_rq;
47920b8a59fSIngo Molnar 	/* rq "owned" by this entity/group: */
48020b8a59fSIngo Molnar 	struct cfs_rq			*my_q;
48120b8a59fSIngo Molnar #endif
4828bd75c77SClark Williams 
483141965c7SAlex Shi #ifdef CONFIG_SMP
4845a107804SJiri Olsa 	/*
4855a107804SJiri Olsa 	 * Per entity load average tracking.
4865a107804SJiri Olsa 	 *
4875a107804SJiri Olsa 	 * Put into separate cache line so it does not
4885a107804SJiri Olsa 	 * collide with read-mostly values above.
4895a107804SJiri Olsa 	 */
490317d359dSPeter Zijlstra 	struct sched_avg		avg;
4919d85f21cSPaul Turner #endif
49220b8a59fSIngo Molnar };
49370b97a7fSIngo Molnar 
494fa717060SPeter Zijlstra struct sched_rt_entity {
495fa717060SPeter Zijlstra 	struct list_head		run_list;
49678f2c7dbSPeter Zijlstra 	unsigned long			timeout;
49757d2aa00SYing Xue 	unsigned long			watchdog_stamp;
498bee367edSRichard Kennedy 	unsigned int			time_slice;
499ff77e468SPeter Zijlstra 	unsigned short			on_rq;
500ff77e468SPeter Zijlstra 	unsigned short			on_list;
5016f505b16SPeter Zijlstra 
50258d6c2d7SPeter Zijlstra 	struct sched_rt_entity		*back;
503052f1dc7SPeter Zijlstra #ifdef CONFIG_RT_GROUP_SCHED
5046f505b16SPeter Zijlstra 	struct sched_rt_entity		*parent;
5056f505b16SPeter Zijlstra 	/* rq on which this entity is (to be) queued: */
5066f505b16SPeter Zijlstra 	struct rt_rq			*rt_rq;
5076f505b16SPeter Zijlstra 	/* rq "owned" by this entity/group: */
5086f505b16SPeter Zijlstra 	struct rt_rq			*my_q;
5096f505b16SPeter Zijlstra #endif
5103859a271SKees Cook } __randomize_layout;
511fa717060SPeter Zijlstra 
512aab03e05SDario Faggioli struct sched_dl_entity {
513aab03e05SDario Faggioli 	struct rb_node			rb_node;
514aab03e05SDario Faggioli 
515aab03e05SDario Faggioli 	/*
516aab03e05SDario Faggioli 	 * Original scheduling parameters. Copied here from sched_attr
5174027d080Sxiaofeng.yan 	 * during sched_setattr(), they will remain the same until
5184027d080Sxiaofeng.yan 	 * the next sched_setattr().
519aab03e05SDario Faggioli 	 */
5205eca1c10SIngo Molnar 	u64				dl_runtime;	/* Maximum runtime for each instance	*/
5215eca1c10SIngo Molnar 	u64				dl_deadline;	/* Relative deadline of each instance	*/
5225eca1c10SIngo Molnar 	u64				dl_period;	/* Separation of two instances (period) */
52354d6d303SDaniel Bristot de Oliveira 	u64				dl_bw;		/* dl_runtime / dl_period		*/
5243effcb42SDaniel Bristot de Oliveira 	u64				dl_density;	/* dl_runtime / dl_deadline		*/
525aab03e05SDario Faggioli 
526aab03e05SDario Faggioli 	/*
527aab03e05SDario Faggioli 	 * Actual scheduling parameters. Initialized with the values above,
528dfcb245eSIngo Molnar 	 * they are continuously updated during task execution. Note that
529aab03e05SDario Faggioli 	 * the remaining runtime could be < 0 in case we are in overrun.
530aab03e05SDario Faggioli 	 */
5315eca1c10SIngo Molnar 	s64				runtime;	/* Remaining runtime for this instance	*/
5325eca1c10SIngo Molnar 	u64				deadline;	/* Absolute deadline for this instance	*/
5335eca1c10SIngo Molnar 	unsigned int			flags;		/* Specifying the scheduler behaviour	*/
534aab03e05SDario Faggioli 
535aab03e05SDario Faggioli 	/*
536aab03e05SDario Faggioli 	 * Some bool flags:
537aab03e05SDario Faggioli 	 *
538aab03e05SDario Faggioli 	 * @dl_throttled tells if we exhausted the runtime. If so, the
539aab03e05SDario Faggioli 	 * task has to wait for a replenishment to be performed at the
540aab03e05SDario Faggioli 	 * next firing of dl_timer.
541aab03e05SDario Faggioli 	 *
5422d3d891dSDario Faggioli 	 * @dl_boosted tells if we are boosted due to DI. If so we are
5432d3d891dSDario Faggioli 	 * outside bandwidth enforcement mechanism (but only until we
5445bfd126eSJuri Lelli 	 * exit the critical section);
5455bfd126eSJuri Lelli 	 *
5465eca1c10SIngo Molnar 	 * @dl_yielded tells if task gave up the CPU before consuming
5475bfd126eSJuri Lelli 	 * all its available runtime during the last job.
548209a0cbdSLuca Abeni 	 *
549209a0cbdSLuca Abeni 	 * @dl_non_contending tells if the task is inactive while still
550209a0cbdSLuca Abeni 	 * contributing to the active utilization. In other words, it
551209a0cbdSLuca Abeni 	 * indicates if the inactive timer has been armed and its handler
552209a0cbdSLuca Abeni 	 * has not been executed yet. This flag is useful to avoid race
553209a0cbdSLuca Abeni 	 * conditions between the inactive timer handler and the wakeup
554209a0cbdSLuca Abeni 	 * code.
55534be3930SJuri Lelli 	 *
55634be3930SJuri Lelli 	 * @dl_overrun tells if the task asked to be informed about runtime
55734be3930SJuri Lelli 	 * overruns.
558aab03e05SDario Faggioli 	 */
559aa5222e9SDan Carpenter 	unsigned int			dl_throttled      : 1;
560aa5222e9SDan Carpenter 	unsigned int			dl_boosted        : 1;
561aa5222e9SDan Carpenter 	unsigned int			dl_yielded        : 1;
562aa5222e9SDan Carpenter 	unsigned int			dl_non_contending : 1;
56334be3930SJuri Lelli 	unsigned int			dl_overrun	  : 1;
564aab03e05SDario Faggioli 
565aab03e05SDario Faggioli 	/*
566aab03e05SDario Faggioli 	 * Bandwidth enforcement timer. Each -deadline task has its
567aab03e05SDario Faggioli 	 * own bandwidth to be enforced, thus we need one timer per task.
568aab03e05SDario Faggioli 	 */
569aab03e05SDario Faggioli 	struct hrtimer			dl_timer;
570209a0cbdSLuca Abeni 
571209a0cbdSLuca Abeni 	/*
572209a0cbdSLuca Abeni 	 * Inactive timer, responsible for decreasing the active utilization
573209a0cbdSLuca Abeni 	 * at the "0-lag time". When a -deadline task blocks, it contributes
574209a0cbdSLuca Abeni 	 * to GRUB's active utilization until the "0-lag time", hence a
575209a0cbdSLuca Abeni 	 * timer is needed to decrease the active utilization at the correct
576209a0cbdSLuca Abeni 	 * time.
577209a0cbdSLuca Abeni 	 */
578209a0cbdSLuca Abeni 	struct hrtimer inactive_timer;
579aab03e05SDario Faggioli };
5808bd75c77SClark Williams 
581*69842cbaSPatrick Bellasi #ifdef CONFIG_UCLAMP_TASK
582*69842cbaSPatrick Bellasi /* Number of utilization clamp buckets (shorter alias) */
583*69842cbaSPatrick Bellasi #define UCLAMP_BUCKETS CONFIG_UCLAMP_BUCKETS_COUNT
584*69842cbaSPatrick Bellasi 
585*69842cbaSPatrick Bellasi /*
586*69842cbaSPatrick Bellasi  * Utilization clamp for a scheduling entity
587*69842cbaSPatrick Bellasi  * @value:		clamp value "assigned" to a se
588*69842cbaSPatrick Bellasi  * @bucket_id:		bucket index corresponding to the "assigned" value
589*69842cbaSPatrick Bellasi  *
590*69842cbaSPatrick Bellasi  * The bucket_id is the index of the clamp bucket matching the clamp value
591*69842cbaSPatrick Bellasi  * which is pre-computed and stored to avoid expensive integer divisions from
592*69842cbaSPatrick Bellasi  * the fast path.
593*69842cbaSPatrick Bellasi  */
594*69842cbaSPatrick Bellasi struct uclamp_se {
595*69842cbaSPatrick Bellasi 	unsigned int value		: bits_per(SCHED_CAPACITY_SCALE);
596*69842cbaSPatrick Bellasi 	unsigned int bucket_id		: bits_per(UCLAMP_BUCKETS);
597*69842cbaSPatrick Bellasi };
598*69842cbaSPatrick Bellasi #endif /* CONFIG_UCLAMP_TASK */
599*69842cbaSPatrick Bellasi 
6001d082fd0SPaul E. McKenney union rcu_special {
6011d082fd0SPaul E. McKenney 	struct {
6028203d6d0SPaul E. McKenney 		u8			blocked;
6038203d6d0SPaul E. McKenney 		u8			need_qs;
60405f41571SPaul E. McKenney 		u8			exp_hint; /* Hint for performance. */
60505f41571SPaul E. McKenney 		u8			pad; /* No garbage from compiler! */
6068203d6d0SPaul E. McKenney 	} b; /* Bits. */
60705f41571SPaul E. McKenney 	u32 s; /* Set of bits. */
6081d082fd0SPaul E. McKenney };
60986848966SPaul E. McKenney 
6108dc85d54SPeter Zijlstra enum perf_event_task_context {
6118dc85d54SPeter Zijlstra 	perf_invalid_context = -1,
6128dc85d54SPeter Zijlstra 	perf_hw_context = 0,
61389a1e187SPeter Zijlstra 	perf_sw_context,
6148dc85d54SPeter Zijlstra 	perf_nr_task_contexts,
6158dc85d54SPeter Zijlstra };
6168dc85d54SPeter Zijlstra 
617eb61baf6SIngo Molnar struct wake_q_node {
618eb61baf6SIngo Molnar 	struct wake_q_node *next;
619eb61baf6SIngo Molnar };
620eb61baf6SIngo Molnar 
6211da177e4SLinus Torvalds struct task_struct {
622c65eacbeSAndy Lutomirski #ifdef CONFIG_THREAD_INFO_IN_TASK
623c65eacbeSAndy Lutomirski 	/*
624c65eacbeSAndy Lutomirski 	 * For reasons of header soup (see current_thread_info()), this
625c65eacbeSAndy Lutomirski 	 * must be the first element of task_struct.
626c65eacbeSAndy Lutomirski 	 */
627c65eacbeSAndy Lutomirski 	struct thread_info		thread_info;
628c65eacbeSAndy Lutomirski #endif
6295eca1c10SIngo Molnar 	/* -1 unrunnable, 0 runnable, >0 stopped: */
6305eca1c10SIngo Molnar 	volatile long			state;
63129e48ce8SKees Cook 
63229e48ce8SKees Cook 	/*
63329e48ce8SKees Cook 	 * This begins the randomizable portion of task_struct. Only
63429e48ce8SKees Cook 	 * scheduling-critical items should be added above here.
63529e48ce8SKees Cook 	 */
63629e48ce8SKees Cook 	randomized_struct_fields_start
63729e48ce8SKees Cook 
638f7e4217bSRoman Zippel 	void				*stack;
639ec1d2819SElena Reshetova 	refcount_t			usage;
6405eca1c10SIngo Molnar 	/* Per task flags (PF_*), defined further below: */
6415eca1c10SIngo Molnar 	unsigned int			flags;
64297dc32cdSWilliam Cohen 	unsigned int			ptrace;
6431da177e4SLinus Torvalds 
6442dd73a4fSPeter Williams #ifdef CONFIG_SMP
645fa14ff4aSPeter Zijlstra 	struct llist_node		wake_entry;
6463ca7a440SPeter Zijlstra 	int				on_cpu;
647c65eacbeSAndy Lutomirski #ifdef CONFIG_THREAD_INFO_IN_TASK
6485eca1c10SIngo Molnar 	/* Current CPU: */
6495eca1c10SIngo Molnar 	unsigned int			cpu;
650c65eacbeSAndy Lutomirski #endif
65163b0e9edSMike Galbraith 	unsigned int			wakee_flips;
65262470419SMichael Wang 	unsigned long			wakee_flip_decay_ts;
65363b0e9edSMike Galbraith 	struct task_struct		*last_wakee;
654ac66f547SPeter Zijlstra 
65532e839ddSMel Gorman 	/*
65632e839ddSMel Gorman 	 * recent_used_cpu is initially set as the last CPU used by a task
65732e839ddSMel Gorman 	 * that wakes affine another task. Waker/wakee relationships can
65832e839ddSMel Gorman 	 * push tasks around a CPU where each wakeup moves to the next one.
65932e839ddSMel Gorman 	 * Tracking a recently used CPU allows a quick search for a recently
66032e839ddSMel Gorman 	 * used CPU that may be idle.
66132e839ddSMel Gorman 	 */
66232e839ddSMel Gorman 	int				recent_used_cpu;
663ac66f547SPeter Zijlstra 	int				wake_cpu;
6644866cde0SNick Piggin #endif
665fd2f4419SPeter Zijlstra 	int				on_rq;
66650e645a8SIngo Molnar 
6675eca1c10SIngo Molnar 	int				prio;
6685eca1c10SIngo Molnar 	int				static_prio;
6695eca1c10SIngo Molnar 	int				normal_prio;
670c7aceabaSRichard Kennedy 	unsigned int			rt_priority;
6715eca1c10SIngo Molnar 
6725522d5d5SIngo Molnar 	const struct sched_class	*sched_class;
67320b8a59fSIngo Molnar 	struct sched_entity		se;
674fa717060SPeter Zijlstra 	struct sched_rt_entity		rt;
6758323f26cSPeter Zijlstra #ifdef CONFIG_CGROUP_SCHED
6768323f26cSPeter Zijlstra 	struct task_group		*sched_task_group;
6778323f26cSPeter Zijlstra #endif
678aab03e05SDario Faggioli 	struct sched_dl_entity		dl;
6791da177e4SLinus Torvalds 
680*69842cbaSPatrick Bellasi #ifdef CONFIG_UCLAMP_TASK
681*69842cbaSPatrick Bellasi 	struct uclamp_se		uclamp[UCLAMP_CNT];
682*69842cbaSPatrick Bellasi #endif
683*69842cbaSPatrick Bellasi 
684e107be36SAvi Kivity #ifdef CONFIG_PREEMPT_NOTIFIERS
6855eca1c10SIngo Molnar 	/* List of struct preempt_notifier: */
686e107be36SAvi Kivity 	struct hlist_head		preempt_notifiers;
687e107be36SAvi Kivity #endif
688e107be36SAvi Kivity 
6896c5c9341SAlexey Dobriyan #ifdef CONFIG_BLK_DEV_IO_TRACE
6902056a782SJens Axboe 	unsigned int			btrace_seq;
6916c5c9341SAlexey Dobriyan #endif
6921da177e4SLinus Torvalds 
69397dc32cdSWilliam Cohen 	unsigned int			policy;
69429baa747SPeter Zijlstra 	int				nr_cpus_allowed;
6953bd37062SSebastian Andrzej Siewior 	const cpumask_t			*cpus_ptr;
6963bd37062SSebastian Andrzej Siewior 	cpumask_t			cpus_mask;
6971da177e4SLinus Torvalds 
698a57eb940SPaul E. McKenney #ifdef CONFIG_PREEMPT_RCU
699e260be67SPaul E. McKenney 	int				rcu_read_lock_nesting;
7001d082fd0SPaul E. McKenney 	union rcu_special		rcu_read_unlock_special;
701f41d911fSPaul E. McKenney 	struct list_head		rcu_node_entry;
702a57eb940SPaul E. McKenney 	struct rcu_node			*rcu_blocked_node;
70328f6569aSPranith Kumar #endif /* #ifdef CONFIG_PREEMPT_RCU */
7045eca1c10SIngo Molnar 
7058315f422SPaul E. McKenney #ifdef CONFIG_TASKS_RCU
7068315f422SPaul E. McKenney 	unsigned long			rcu_tasks_nvcsw;
707ccdd29ffSPaul E. McKenney 	u8				rcu_tasks_holdout;
708ccdd29ffSPaul E. McKenney 	u8				rcu_tasks_idx;
709176f8f7aSPaul E. McKenney 	int				rcu_tasks_idle_cpu;
710ccdd29ffSPaul E. McKenney 	struct list_head		rcu_tasks_holdout_list;
7118315f422SPaul E. McKenney #endif /* #ifdef CONFIG_TASKS_RCU */
712e260be67SPaul E. McKenney 
7131da177e4SLinus Torvalds 	struct sched_info		sched_info;
7141da177e4SLinus Torvalds 
7151da177e4SLinus Torvalds 	struct list_head		tasks;
716806c09a7SDario Faggioli #ifdef CONFIG_SMP
717917b627dSGregory Haskins 	struct plist_node		pushable_tasks;
7181baca4ceSJuri Lelli 	struct rb_node			pushable_dl_tasks;
719806c09a7SDario Faggioli #endif
7201da177e4SLinus Torvalds 
7215eca1c10SIngo Molnar 	struct mm_struct		*mm;
7225eca1c10SIngo Molnar 	struct mm_struct		*active_mm;
723314ff785SIngo Molnar 
724314ff785SIngo Molnar 	/* Per-thread vma caching: */
725314ff785SIngo Molnar 	struct vmacache			vmacache;
726314ff785SIngo Molnar 
7275eca1c10SIngo Molnar #ifdef SPLIT_RSS_COUNTING
72834e55232SKAMEZAWA Hiroyuki 	struct task_rss_stat		rss_stat;
72934e55232SKAMEZAWA Hiroyuki #endif
73097dc32cdSWilliam Cohen 	int				exit_state;
7315eca1c10SIngo Molnar 	int				exit_code;
7325eca1c10SIngo Molnar 	int				exit_signal;
7335eca1c10SIngo Molnar 	/* The signal sent when the parent dies: */
7345eca1c10SIngo Molnar 	int				pdeath_signal;
7355eca1c10SIngo Molnar 	/* JOBCTL_*, siglock protected: */
7365eca1c10SIngo Molnar 	unsigned long			jobctl;
7379b89f6baSAndrei Epure 
7385eca1c10SIngo Molnar 	/* Used for emulating ABI behavior of previous Linux versions: */
73997dc32cdSWilliam Cohen 	unsigned int			personality;
7409b89f6baSAndrei Epure 
7415eca1c10SIngo Molnar 	/* Scheduler bits, serialized by scheduler locks: */
742ca94c442SLennart Poettering 	unsigned			sched_reset_on_fork:1;
743a8e4f2eaSPeter Zijlstra 	unsigned			sched_contributes_to_load:1;
744ff303e66SPeter Zijlstra 	unsigned			sched_migrated:1;
745b7e7ade3SPeter Zijlstra 	unsigned			sched_remote_wakeup:1;
746eb414681SJohannes Weiner #ifdef CONFIG_PSI
747eb414681SJohannes Weiner 	unsigned			sched_psi_wake_requeue:1;
748eb414681SJohannes Weiner #endif
749eb414681SJohannes Weiner 
7505eca1c10SIngo Molnar 	/* Force alignment to the next boundary: */
7515eca1c10SIngo Molnar 	unsigned			:0;
752be958bdcSPeter Zijlstra 
7535eca1c10SIngo Molnar 	/* Unserialized, strictly 'current' */
7545eca1c10SIngo Molnar 
7555eca1c10SIngo Molnar 	/* Bit to tell LSMs we're in execve(): */
7565eca1c10SIngo Molnar 	unsigned			in_execve:1;
757be958bdcSPeter Zijlstra 	unsigned			in_iowait:1;
7585eca1c10SIngo Molnar #ifndef TIF_RESTORE_SIGMASK
7597e781418SAndy Lutomirski 	unsigned			restore_sigmask:1;
7607e781418SAndy Lutomirski #endif
761626ebc41STejun Heo #ifdef CONFIG_MEMCG
76229ef680aSMichal Hocko 	unsigned			in_user_fault:1;
763127424c8SJohannes Weiner #endif
764ff303e66SPeter Zijlstra #ifdef CONFIG_COMPAT_BRK
765ff303e66SPeter Zijlstra 	unsigned			brk_randomized:1;
766ff303e66SPeter Zijlstra #endif
76777f88796STejun Heo #ifdef CONFIG_CGROUPS
76877f88796STejun Heo 	/* disallow userland-initiated cgroup migration */
76977f88796STejun Heo 	unsigned			no_cgroup_migration:1;
77076f969e8SRoman Gushchin 	/* task is frozen/stopped (used by the cgroup freezer) */
77176f969e8SRoman Gushchin 	unsigned			frozen:1;
77277f88796STejun Heo #endif
773d09d8df3SJosef Bacik #ifdef CONFIG_BLK_CGROUP
774d09d8df3SJosef Bacik 	/* to be used once the psi infrastructure lands upstream. */
775d09d8df3SJosef Bacik 	unsigned			use_memdelay:1;
776d09d8df3SJosef Bacik #endif
7776f185c29SVladimir Davydov 
7785eca1c10SIngo Molnar 	unsigned long			atomic_flags; /* Flags requiring atomic access. */
7791d4457f9SKees Cook 
780f56141e3SAndy Lutomirski 	struct restart_block		restart_block;
781f56141e3SAndy Lutomirski 
7821da177e4SLinus Torvalds 	pid_t				pid;
7831da177e4SLinus Torvalds 	pid_t				tgid;
7840a425405SArjan van de Ven 
785050e9baaSLinus Torvalds #ifdef CONFIG_STACKPROTECTOR
7865eca1c10SIngo Molnar 	/* Canary value for the -fstack-protector GCC feature: */
7870a425405SArjan van de Ven 	unsigned long			stack_canary;
7881314562aSHiroshi Shimamoto #endif
7891da177e4SLinus Torvalds 	/*
7905eca1c10SIngo Molnar 	 * Pointers to the (original) parent process, youngest child, younger sibling,
7911da177e4SLinus Torvalds 	 * older sibling, respectively.  (p->father can be replaced with
792f470021aSRoland McGrath 	 * p->real_parent->pid)
7931da177e4SLinus Torvalds 	 */
7945eca1c10SIngo Molnar 
7955eca1c10SIngo Molnar 	/* Real parent process: */
7965eca1c10SIngo Molnar 	struct task_struct __rcu	*real_parent;
7975eca1c10SIngo Molnar 
7985eca1c10SIngo Molnar 	/* Recipient of SIGCHLD, wait4() reports: */
7995eca1c10SIngo Molnar 	struct task_struct __rcu	*parent;
8001da177e4SLinus Torvalds 
801f470021aSRoland McGrath 	/*
8025eca1c10SIngo Molnar 	 * Children/sibling form the list of natural children:
8035eca1c10SIngo Molnar 	 */
8045eca1c10SIngo Molnar 	struct list_head		children;
8055eca1c10SIngo Molnar 	struct list_head		sibling;
8065eca1c10SIngo Molnar 	struct task_struct		*group_leader;
8075eca1c10SIngo Molnar 
8085eca1c10SIngo Molnar 	/*
8095eca1c10SIngo Molnar 	 * 'ptraced' is the list of tasks this task is using ptrace() on.
8105eca1c10SIngo Molnar 	 *
811f470021aSRoland McGrath 	 * This includes both natural children and PTRACE_ATTACH targets.
8125eca1c10SIngo Molnar 	 * 'ptrace_entry' is this task's link on the p->parent->ptraced list.
813f470021aSRoland McGrath 	 */
814f470021aSRoland McGrath 	struct list_head		ptraced;
815f470021aSRoland McGrath 	struct list_head		ptrace_entry;
816f470021aSRoland McGrath 
8171da177e4SLinus Torvalds 	/* PID/PID hash table linkage. */
8182c470475SEric W. Biederman 	struct pid			*thread_pid;
8192c470475SEric W. Biederman 	struct hlist_node		pid_links[PIDTYPE_MAX];
82047e65328SOleg Nesterov 	struct list_head		thread_group;
8210c740d0aSOleg Nesterov 	struct list_head		thread_node;
8221da177e4SLinus Torvalds 
8235eca1c10SIngo Molnar 	struct completion		*vfork_done;
8241da177e4SLinus Torvalds 
8255eca1c10SIngo Molnar 	/* CLONE_CHILD_SETTID: */
8265eca1c10SIngo Molnar 	int __user			*set_child_tid;
8275eca1c10SIngo Molnar 
8285eca1c10SIngo Molnar 	/* CLONE_CHILD_CLEARTID: */
8295eca1c10SIngo Molnar 	int __user			*clear_child_tid;
8305eca1c10SIngo Molnar 
8315eca1c10SIngo Molnar 	u64				utime;
8325eca1c10SIngo Molnar 	u64				stime;
83340565b5aSStanislaw Gruszka #ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
8345eca1c10SIngo Molnar 	u64				utimescaled;
8355eca1c10SIngo Molnar 	u64				stimescaled;
83640565b5aSStanislaw Gruszka #endif
83716a6d9beSFrederic Weisbecker 	u64				gtime;
8389d7fb042SPeter Zijlstra 	struct prev_cputime		prev_cputime;
8396a61671bSFrederic Weisbecker #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
840bac5b6b6SFrederic Weisbecker 	struct vtime			vtime;
8416a61671bSFrederic Weisbecker #endif
842d027d45dSFrederic Weisbecker 
843d027d45dSFrederic Weisbecker #ifdef CONFIG_NO_HZ_FULL
844f009a7a7SFrederic Weisbecker 	atomic_t			tick_dep_mask;
845d027d45dSFrederic Weisbecker #endif
8465eca1c10SIngo Molnar 	/* Context switch counts: */
8475eca1c10SIngo Molnar 	unsigned long			nvcsw;
8485eca1c10SIngo Molnar 	unsigned long			nivcsw;
8495eca1c10SIngo Molnar 
8505eca1c10SIngo Molnar 	/* Monotonic time in nsecs: */
8515eca1c10SIngo Molnar 	u64				start_time;
8525eca1c10SIngo Molnar 
8535eca1c10SIngo Molnar 	/* Boot based time in nsecs: */
8545eca1c10SIngo Molnar 	u64				real_start_time;
8555eca1c10SIngo Molnar 
8565eca1c10SIngo Molnar 	/* MM fault and swap info: this can arguably be seen as either mm-specific or thread-specific: */
8575eca1c10SIngo Molnar 	unsigned long			min_flt;
8585eca1c10SIngo Molnar 	unsigned long			maj_flt;
8591da177e4SLinus Torvalds 
860b18b6a9cSNicolas Pitre #ifdef CONFIG_POSIX_TIMERS
861f06febc9SFrank Mayhar 	struct task_cputime		cputime_expires;
8621da177e4SLinus Torvalds 	struct list_head		cpu_timers[3];
863b18b6a9cSNicolas Pitre #endif
8641da177e4SLinus Torvalds 
8655eca1c10SIngo Molnar 	/* Process credentials: */
8665eca1c10SIngo Molnar 
8675eca1c10SIngo Molnar 	/* Tracer's credentials at attach: */
8685eca1c10SIngo Molnar 	const struct cred __rcu		*ptracer_cred;
8695eca1c10SIngo Molnar 
8705eca1c10SIngo Molnar 	/* Objective and real subjective task credentials (COW): */
8715eca1c10SIngo Molnar 	const struct cred __rcu		*real_cred;
8725eca1c10SIngo Molnar 
8735eca1c10SIngo Molnar 	/* Effective (overridable) subjective task credentials (COW): */
8745eca1c10SIngo Molnar 	const struct cred __rcu		*cred;
8755eca1c10SIngo Molnar 
8765eca1c10SIngo Molnar 	/*
8775eca1c10SIngo Molnar 	 * executable name, excluding path.
8785eca1c10SIngo Molnar 	 *
8795eca1c10SIngo Molnar 	 * - normally initialized setup_new_exec()
8805eca1c10SIngo Molnar 	 * - access it with [gs]et_task_comm()
8815eca1c10SIngo Molnar 	 * - lock it with task_lock()
8825eca1c10SIngo Molnar 	 */
8835eca1c10SIngo Molnar 	char				comm[TASK_COMM_LEN];
8845eca1c10SIngo Molnar 
885756daf26SNeilBrown 	struct nameidata		*nameidata;
8865eca1c10SIngo Molnar 
8873d5b6fccSAlexey Dobriyan #ifdef CONFIG_SYSVIPC
8881da177e4SLinus Torvalds 	struct sysv_sem			sysvsem;
889ab602f79SJack Miller 	struct sysv_shm			sysvshm;
8903d5b6fccSAlexey Dobriyan #endif
891e162b39aSMandeep Singh Baines #ifdef CONFIG_DETECT_HUNG_TASK
89282a1fcb9SIngo Molnar 	unsigned long			last_switch_count;
893a2e51445SDmitry Vyukov 	unsigned long			last_switch_time;
89482a1fcb9SIngo Molnar #endif
8955eca1c10SIngo Molnar 	/* Filesystem information: */
8961da177e4SLinus Torvalds 	struct fs_struct		*fs;
8975eca1c10SIngo Molnar 
8985eca1c10SIngo Molnar 	/* Open file information: */
8991da177e4SLinus Torvalds 	struct files_struct		*files;
9005eca1c10SIngo Molnar 
9015eca1c10SIngo Molnar 	/* Namespaces: */
902ab516013SSerge E. Hallyn 	struct nsproxy			*nsproxy;
9035eca1c10SIngo Molnar 
9045eca1c10SIngo Molnar 	/* Signal handlers: */
9051da177e4SLinus Torvalds 	struct signal_struct		*signal;
9061da177e4SLinus Torvalds 	struct sighand_struct		*sighand;
9075eca1c10SIngo Molnar 	sigset_t			blocked;
9085eca1c10SIngo Molnar 	sigset_t			real_blocked;
9095eca1c10SIngo Molnar 	/* Restored if set_restore_sigmask() was used: */
9105eca1c10SIngo Molnar 	sigset_t			saved_sigmask;
9111da177e4SLinus Torvalds 	struct sigpending		pending;
9121da177e4SLinus Torvalds 	unsigned long			sas_ss_sp;
9131da177e4SLinus Torvalds 	size_t				sas_ss_size;
9145eca1c10SIngo Molnar 	unsigned int			sas_ss_flags;
9152e01fabeSOleg Nesterov 
91667d12145SAl Viro 	struct callback_head		*task_works;
917e73f8959SOleg Nesterov 
9184b7d248bSRichard Guy Briggs #ifdef CONFIG_AUDIT
919bfef93a5SAl Viro #ifdef CONFIG_AUDITSYSCALL
9205f3d544fSRichard Guy Briggs 	struct audit_context		*audit_context;
9215f3d544fSRichard Guy Briggs #endif
922e1760bd5SEric W. Biederman 	kuid_t				loginuid;
9234746ec5bSEric Paris 	unsigned int			sessionid;
924bfef93a5SAl Viro #endif
925932ecebbSWill Drewry 	struct seccomp			seccomp;
9261da177e4SLinus Torvalds 
9275eca1c10SIngo Molnar 	/* Thread group tracking: */
9281da177e4SLinus Torvalds 	u32				parent_exec_id;
9291da177e4SLinus Torvalds 	u32				self_exec_id;
9305eca1c10SIngo Molnar 
9315eca1c10SIngo Molnar 	/* Protection against (de-)allocation: mm, files, fs, tty, keyrings, mems_allowed, mempolicy: */
9321da177e4SLinus Torvalds 	spinlock_t			alloc_lock;
9331da177e4SLinus Torvalds 
934b29739f9SIngo Molnar 	/* Protection of the PI data structures: */
9351d615482SThomas Gleixner 	raw_spinlock_t			pi_lock;
936b29739f9SIngo Molnar 
93776751049SPeter Zijlstra 	struct wake_q_node		wake_q;
93876751049SPeter Zijlstra 
93923f78d4aSIngo Molnar #ifdef CONFIG_RT_MUTEXES
9405eca1c10SIngo Molnar 	/* PI waiters blocked on a rt_mutex held by this task: */
941a23ba907SDavidlohr Bueso 	struct rb_root_cached		pi_waiters;
942e96a7705SXunlei Pang 	/* Updated under owner's pi_lock and rq lock */
943e96a7705SXunlei Pang 	struct task_struct		*pi_top_task;
9445eca1c10SIngo Molnar 	/* Deadlock detection and priority inheritance handling: */
94523f78d4aSIngo Molnar 	struct rt_mutex_waiter		*pi_blocked_on;
94623f78d4aSIngo Molnar #endif
94723f78d4aSIngo Molnar 
948408894eeSIngo Molnar #ifdef CONFIG_DEBUG_MUTEXES
9495eca1c10SIngo Molnar 	/* Mutex deadlock detection: */
950408894eeSIngo Molnar 	struct mutex_waiter		*blocked_on;
951408894eeSIngo Molnar #endif
9525eca1c10SIngo Molnar 
953de30a2b3SIngo Molnar #ifdef CONFIG_TRACE_IRQFLAGS
954de30a2b3SIngo Molnar 	unsigned int			irq_events;
955de30a2b3SIngo Molnar 	unsigned long			hardirq_enable_ip;
956de30a2b3SIngo Molnar 	unsigned long			hardirq_disable_ip;
957fa1452e8SHiroshi Shimamoto 	unsigned int			hardirq_enable_event;
958de30a2b3SIngo Molnar 	unsigned int			hardirq_disable_event;
959fa1452e8SHiroshi Shimamoto 	int				hardirqs_enabled;
960de30a2b3SIngo Molnar 	int				hardirq_context;
961fa1452e8SHiroshi Shimamoto 	unsigned long			softirq_disable_ip;
962fa1452e8SHiroshi Shimamoto 	unsigned long			softirq_enable_ip;
963fa1452e8SHiroshi Shimamoto 	unsigned int			softirq_disable_event;
964fa1452e8SHiroshi Shimamoto 	unsigned int			softirq_enable_event;
965fa1452e8SHiroshi Shimamoto 	int				softirqs_enabled;
966de30a2b3SIngo Molnar 	int				softirq_context;
967de30a2b3SIngo Molnar #endif
9685eca1c10SIngo Molnar 
969fbb9ce95SIngo Molnar #ifdef CONFIG_LOCKDEP
970bdb9441eSPeter Zijlstra # define MAX_LOCK_DEPTH			48UL
971fbb9ce95SIngo Molnar 	u64				curr_chain_key;
972fbb9ce95SIngo Molnar 	int				lockdep_depth;
973fbb9ce95SIngo Molnar 	unsigned int			lockdep_recursion;
974c7aceabaSRichard Kennedy 	struct held_lock		held_locks[MAX_LOCK_DEPTH];
975fbb9ce95SIngo Molnar #endif
9765eca1c10SIngo Molnar 
977c6d30853SAndrey Ryabinin #ifdef CONFIG_UBSAN
978c6d30853SAndrey Ryabinin 	unsigned int			in_ubsan;
979c6d30853SAndrey Ryabinin #endif
980408894eeSIngo Molnar 
9815eca1c10SIngo Molnar 	/* Journalling filesystem info: */
9821da177e4SLinus Torvalds 	void				*journal_info;
9831da177e4SLinus Torvalds 
9845eca1c10SIngo Molnar 	/* Stacked block device info: */
985bddd87c7SAkinobu Mita 	struct bio_list			*bio_list;
986d89d8796SNeil Brown 
98773c10101SJens Axboe #ifdef CONFIG_BLOCK
9885eca1c10SIngo Molnar 	/* Stack plugging: */
98973c10101SJens Axboe 	struct blk_plug			*plug;
99073c10101SJens Axboe #endif
99173c10101SJens Axboe 
9925eca1c10SIngo Molnar 	/* VM state: */
9931da177e4SLinus Torvalds 	struct reclaim_state		*reclaim_state;
9941da177e4SLinus Torvalds 
9951da177e4SLinus Torvalds 	struct backing_dev_info		*backing_dev_info;
9961da177e4SLinus Torvalds 
9971da177e4SLinus Torvalds 	struct io_context		*io_context;
9981da177e4SLinus Torvalds 
9995e1f0f09SMel Gorman #ifdef CONFIG_COMPACTION
10005e1f0f09SMel Gorman 	struct capture_control		*capture_control;
10015e1f0f09SMel Gorman #endif
10025eca1c10SIngo Molnar 	/* Ptrace state: */
10031da177e4SLinus Torvalds 	unsigned long			ptrace_message;
1004ae7795bcSEric W. Biederman 	kernel_siginfo_t		*last_siginfo;
10055eca1c10SIngo Molnar 
10067c3ab738SAndrew Morton 	struct task_io_accounting	ioac;
1007eb414681SJohannes Weiner #ifdef CONFIG_PSI
1008eb414681SJohannes Weiner 	/* Pressure stall state */
1009eb414681SJohannes Weiner 	unsigned int			psi_flags;
1010eb414681SJohannes Weiner #endif
10115eca1c10SIngo Molnar #ifdef CONFIG_TASK_XACCT
10125eca1c10SIngo Molnar 	/* Accumulated RSS usage: */
10135eca1c10SIngo Molnar 	u64				acct_rss_mem1;
10145eca1c10SIngo Molnar 	/* Accumulated virtual memory usage: */
10155eca1c10SIngo Molnar 	u64				acct_vm_mem1;
10165eca1c10SIngo Molnar 	/* stime + utime since last update: */
10175eca1c10SIngo Molnar 	u64				acct_timexpd;
10181da177e4SLinus Torvalds #endif
10191da177e4SLinus Torvalds #ifdef CONFIG_CPUSETS
10205eca1c10SIngo Molnar 	/* Protected by ->alloc_lock: */
10215eca1c10SIngo Molnar 	nodemask_t			mems_allowed;
10225eca1c10SIngo Molnar 	/* Seqence number to catch updates: */
10235eca1c10SIngo Molnar 	seqcount_t			mems_allowed_seq;
1024825a46afSPaul Jackson 	int				cpuset_mem_spread_rotor;
10256adef3ebSJack Steiner 	int				cpuset_slab_spread_rotor;
10261da177e4SLinus Torvalds #endif
1027ddbcc7e8SPaul Menage #ifdef CONFIG_CGROUPS
10285eca1c10SIngo Molnar 	/* Control Group info protected by css_set_lock: */
10292c392b8cSArnd Bergmann 	struct css_set __rcu		*cgroups;
10305eca1c10SIngo Molnar 	/* cg_list protected by css_set_lock and tsk->alloc_lock: */
1031817929ecSPaul Menage 	struct list_head		cg_list;
1032ddbcc7e8SPaul Menage #endif
1033e6d42931SJohannes Weiner #ifdef CONFIG_X86_CPU_RESCTRL
10340734ded1SVikas Shivappa 	u32				closid;
1035d6aaba61SVikas Shivappa 	u32				rmid;
1036e02737d5SFenghua Yu #endif
103742b2dd0aSAlexey Dobriyan #ifdef CONFIG_FUTEX
10380771dfefSIngo Molnar 	struct robust_list_head __user	*robust_list;
103934f192c6SIngo Molnar #ifdef CONFIG_COMPAT
104034f192c6SIngo Molnar 	struct compat_robust_list_head __user *compat_robust_list;
104134f192c6SIngo Molnar #endif
1042c87e2837SIngo Molnar 	struct list_head		pi_state_list;
1043c87e2837SIngo Molnar 	struct futex_pi_state		*pi_state_cache;
104442b2dd0aSAlexey Dobriyan #endif
1045cdd6c482SIngo Molnar #ifdef CONFIG_PERF_EVENTS
10468dc85d54SPeter Zijlstra 	struct perf_event_context	*perf_event_ctxp[perf_nr_task_contexts];
1047cdd6c482SIngo Molnar 	struct mutex			perf_event_mutex;
1048cdd6c482SIngo Molnar 	struct list_head		perf_event_list;
1049a63eaf34SPaul Mackerras #endif
10508f47b187SThomas Gleixner #ifdef CONFIG_DEBUG_PREEMPT
10518f47b187SThomas Gleixner 	unsigned long			preempt_disable_ip;
10528f47b187SThomas Gleixner #endif
1053c7aceabaSRichard Kennedy #ifdef CONFIG_NUMA
10545eca1c10SIngo Molnar 	/* Protected by alloc_lock: */
10555eca1c10SIngo Molnar 	struct mempolicy		*mempolicy;
105645816682SVlastimil Babka 	short				il_prev;
1057207205a2SEric Dumazet 	short				pref_node_fork;
1058c7aceabaSRichard Kennedy #endif
1059cbee9f88SPeter Zijlstra #ifdef CONFIG_NUMA_BALANCING
1060cbee9f88SPeter Zijlstra 	int				numa_scan_seq;
1061cbee9f88SPeter Zijlstra 	unsigned int			numa_scan_period;
1062598f0ec0SMel Gorman 	unsigned int			numa_scan_period_max;
1063de1c9ce6SRik van Riel 	int				numa_preferred_nid;
10646b9a7460SMel Gorman 	unsigned long			numa_migrate_retry;
10655eca1c10SIngo Molnar 	/* Migration stamp: */
10665eca1c10SIngo Molnar 	u64				node_stamp;
10677e2703e6SRik van Riel 	u64				last_task_numa_placement;
10687e2703e6SRik van Riel 	u64				last_sum_exec_runtime;
1069cbee9f88SPeter Zijlstra 	struct callback_head		numa_work;
1070f809ca9aSMel Gorman 
10718c8a743cSPeter Zijlstra 	struct numa_group		*numa_group;
10728c8a743cSPeter Zijlstra 
1073745d6147SMel Gorman 	/*
107444dba3d5SIulia Manda 	 * numa_faults is an array split into four regions:
107544dba3d5SIulia Manda 	 * faults_memory, faults_cpu, faults_memory_buffer, faults_cpu_buffer
107644dba3d5SIulia Manda 	 * in this precise order.
107744dba3d5SIulia Manda 	 *
107844dba3d5SIulia Manda 	 * faults_memory: Exponential decaying average of faults on a per-node
107944dba3d5SIulia Manda 	 * basis. Scheduling placement decisions are made based on these
108044dba3d5SIulia Manda 	 * counts. The values remain static for the duration of a PTE scan.
108144dba3d5SIulia Manda 	 * faults_cpu: Track the nodes the process was running on when a NUMA
108244dba3d5SIulia Manda 	 * hinting fault was incurred.
108344dba3d5SIulia Manda 	 * faults_memory_buffer and faults_cpu_buffer: Record faults per node
108444dba3d5SIulia Manda 	 * during the current scan window. When the scan completes, the counts
108544dba3d5SIulia Manda 	 * in faults_memory and faults_cpu decay and these values are copied.
1086745d6147SMel Gorman 	 */
108744dba3d5SIulia Manda 	unsigned long			*numa_faults;
108883e1d2cdSMel Gorman 	unsigned long			total_numa_faults;
1089745d6147SMel Gorman 
1090745d6147SMel Gorman 	/*
109104bb2f94SRik van Riel 	 * numa_faults_locality tracks if faults recorded during the last
1092074c2381SMel Gorman 	 * scan window were remote/local or failed to migrate. The task scan
1093074c2381SMel Gorman 	 * period is adapted based on the locality of the faults with different
1094074c2381SMel Gorman 	 * weights depending on whether they were shared or private faults
109504bb2f94SRik van Riel 	 */
1096074c2381SMel Gorman 	unsigned long			numa_faults_locality[3];
109704bb2f94SRik van Riel 
1098b32e86b4SIngo Molnar 	unsigned long			numa_pages_migrated;
1099cbee9f88SPeter Zijlstra #endif /* CONFIG_NUMA_BALANCING */
1100cbee9f88SPeter Zijlstra 
1101d7822b1eSMathieu Desnoyers #ifdef CONFIG_RSEQ
1102d7822b1eSMathieu Desnoyers 	struct rseq __user *rseq;
1103d7822b1eSMathieu Desnoyers 	u32 rseq_sig;
1104d7822b1eSMathieu Desnoyers 	/*
1105d7822b1eSMathieu Desnoyers 	 * RmW on rseq_event_mask must be performed atomically
1106d7822b1eSMathieu Desnoyers 	 * with respect to preemption.
1107d7822b1eSMathieu Desnoyers 	 */
1108d7822b1eSMathieu Desnoyers 	unsigned long rseq_event_mask;
1109d7822b1eSMathieu Desnoyers #endif
1110d7822b1eSMathieu Desnoyers 
111172b252aeSMel Gorman 	struct tlbflush_unmap_batch	tlb_ubc;
111272b252aeSMel Gorman 
1113e56d0903SIngo Molnar 	struct rcu_head			rcu;
1114b92ce558SJens Axboe 
11155eca1c10SIngo Molnar 	/* Cache last used pipe for splice(): */
1116b92ce558SJens Axboe 	struct pipe_inode_info		*splice_pipe;
11175640f768SEric Dumazet 
11185640f768SEric Dumazet 	struct page_frag		task_frag;
11195640f768SEric Dumazet 
1120ca74e92bSShailabh Nagar #ifdef CONFIG_TASK_DELAY_ACCT
1121ca74e92bSShailabh Nagar 	struct task_delay_info		*delays;
1122ca74e92bSShailabh Nagar #endif
112347913d4eSIngo Molnar 
1124f4f154fdSAkinobu Mita #ifdef CONFIG_FAULT_INJECTION
1125f4f154fdSAkinobu Mita 	int				make_it_fail;
11269049f2f6SAkinobu Mita 	unsigned int			fail_nth;
1127f4f154fdSAkinobu Mita #endif
11289d823e8fSWu Fengguang 	/*
11295eca1c10SIngo Molnar 	 * When (nr_dirtied >= nr_dirtied_pause), it's time to call
11305eca1c10SIngo Molnar 	 * balance_dirty_pages() for a dirty throttling pause:
11319d823e8fSWu Fengguang 	 */
11329d823e8fSWu Fengguang 	int				nr_dirtied;
11339d823e8fSWu Fengguang 	int				nr_dirtied_pause;
11345eca1c10SIngo Molnar 	/* Start of a write-and-pause period: */
11355eca1c10SIngo Molnar 	unsigned long			dirty_paused_when;
11369d823e8fSWu Fengguang 
11379745512cSArjan van de Ven #ifdef CONFIG_LATENCYTOP
11389745512cSArjan van de Ven 	int				latency_record_count;
11399745512cSArjan van de Ven 	struct latency_record		latency_record[LT_SAVECOUNT];
11409745512cSArjan van de Ven #endif
11416976675dSArjan van de Ven 	/*
11425eca1c10SIngo Molnar 	 * Time slack values; these are used to round up poll() and
11436976675dSArjan van de Ven 	 * select() etc timeout values. These are in nanoseconds.
11446976675dSArjan van de Ven 	 */
1145da8b44d5SJohn Stultz 	u64				timer_slack_ns;
1146da8b44d5SJohn Stultz 	u64				default_timer_slack_ns;
1147f8d570a4SDavid Miller 
11480b24beccSAndrey Ryabinin #ifdef CONFIG_KASAN
11490b24beccSAndrey Ryabinin 	unsigned int			kasan_depth;
11500b24beccSAndrey Ryabinin #endif
11515eca1c10SIngo Molnar 
1152fb52607aSFrederic Weisbecker #ifdef CONFIG_FUNCTION_GRAPH_TRACER
11535eca1c10SIngo Molnar 	/* Index of current stored address in ret_stack: */
1154f201ae23SFrederic Weisbecker 	int				curr_ret_stack;
115539eb456dSSteven Rostedt (VMware) 	int				curr_ret_depth;
11565eca1c10SIngo Molnar 
11575eca1c10SIngo Molnar 	/* Stack of return addresses for return function tracing: */
1158f201ae23SFrederic Weisbecker 	struct ftrace_ret_stack		*ret_stack;
11595eca1c10SIngo Molnar 
11605eca1c10SIngo Molnar 	/* Timestamp for last schedule: */
11618aef2d28SSteven Rostedt 	unsigned long long		ftrace_timestamp;
11625eca1c10SIngo Molnar 
1163f201ae23SFrederic Weisbecker 	/*
1164f201ae23SFrederic Weisbecker 	 * Number of functions that haven't been traced
11655eca1c10SIngo Molnar 	 * because of depth overrun:
1166f201ae23SFrederic Weisbecker 	 */
1167f201ae23SFrederic Weisbecker 	atomic_t			trace_overrun;
11685eca1c10SIngo Molnar 
11695eca1c10SIngo Molnar 	/* Pause tracing: */
1170380c4b14SFrederic Weisbecker 	atomic_t			tracing_graph_pause;
1171f201ae23SFrederic Weisbecker #endif
11725eca1c10SIngo Molnar 
1173ea4e2bc4SSteven Rostedt #ifdef CONFIG_TRACING
11745eca1c10SIngo Molnar 	/* State flags for use by tracers: */
1175ea4e2bc4SSteven Rostedt 	unsigned long			trace;
11765eca1c10SIngo Molnar 
11775eca1c10SIngo Molnar 	/* Bitmask and counter of trace recursion: */
1178261842b7SSteven Rostedt 	unsigned long			trace_recursion;
1179261842b7SSteven Rostedt #endif /* CONFIG_TRACING */
11805eca1c10SIngo Molnar 
11815c9a8750SDmitry Vyukov #ifdef CONFIG_KCOV
11825eca1c10SIngo Molnar 	/* Coverage collection mode enabled for this task (0 if disabled): */
11830ed557aaSMark Rutland 	unsigned int			kcov_mode;
11845eca1c10SIngo Molnar 
11855eca1c10SIngo Molnar 	/* Size of the kcov_area: */
11865eca1c10SIngo Molnar 	unsigned int			kcov_size;
11875eca1c10SIngo Molnar 
11885eca1c10SIngo Molnar 	/* Buffer for coverage collection: */
11895c9a8750SDmitry Vyukov 	void				*kcov_area;
11905eca1c10SIngo Molnar 
11915eca1c10SIngo Molnar 	/* KCOV descriptor wired with this task or NULL: */
11925c9a8750SDmitry Vyukov 	struct kcov			*kcov;
11935c9a8750SDmitry Vyukov #endif
11945eca1c10SIngo Molnar 
11956f185c29SVladimir Davydov #ifdef CONFIG_MEMCG
1196626ebc41STejun Heo 	struct mem_cgroup		*memcg_in_oom;
1197626ebc41STejun Heo 	gfp_t				memcg_oom_gfp_mask;
1198626ebc41STejun Heo 	int				memcg_oom_order;
1199b23afb93STejun Heo 
12005eca1c10SIngo Molnar 	/* Number of pages to reclaim on returning to userland: */
1201b23afb93STejun Heo 	unsigned int			memcg_nr_pages_over_high;
1202d46eb14bSShakeel Butt 
1203d46eb14bSShakeel Butt 	/* Used by memcontrol for targeted memcg charge: */
1204d46eb14bSShakeel Butt 	struct mem_cgroup		*active_memcg;
1205569b846dSKAMEZAWA Hiroyuki #endif
12065eca1c10SIngo Molnar 
1207d09d8df3SJosef Bacik #ifdef CONFIG_BLK_CGROUP
1208d09d8df3SJosef Bacik 	struct request_queue		*throttle_queue;
1209d09d8df3SJosef Bacik #endif
1210d09d8df3SJosef Bacik 
12110326f5a9SSrikar Dronamraju #ifdef CONFIG_UPROBES
12120326f5a9SSrikar Dronamraju 	struct uprobe_task		*utask;
12130326f5a9SSrikar Dronamraju #endif
1214cafe5635SKent Overstreet #if defined(CONFIG_BCACHE) || defined(CONFIG_BCACHE_MODULE)
1215cafe5635SKent Overstreet 	unsigned int			sequential_io;
1216cafe5635SKent Overstreet 	unsigned int			sequential_io_avg;
1217cafe5635SKent Overstreet #endif
12188eb23b9fSPeter Zijlstra #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
12198eb23b9fSPeter Zijlstra 	unsigned long			task_state_change;
12208eb23b9fSPeter Zijlstra #endif
12218bcbde54SDavid Hildenbrand 	int				pagefault_disabled;
122203049269SMichal Hocko #ifdef CONFIG_MMU
122329c696e1SVladimir Davydov 	struct task_struct		*oom_reaper_list;
122403049269SMichal Hocko #endif
1225ba14a194SAndy Lutomirski #ifdef CONFIG_VMAP_STACK
1226ba14a194SAndy Lutomirski 	struct vm_struct		*stack_vm_area;
1227ba14a194SAndy Lutomirski #endif
122868f24b08SAndy Lutomirski #ifdef CONFIG_THREAD_INFO_IN_TASK
12295eca1c10SIngo Molnar 	/* A live task holds one reference: */
1230f0b89d39SElena Reshetova 	refcount_t			stack_refcount;
123168f24b08SAndy Lutomirski #endif
1232d83a7cb3SJosh Poimboeuf #ifdef CONFIG_LIVEPATCH
1233d83a7cb3SJosh Poimboeuf 	int patch_state;
1234d83a7cb3SJosh Poimboeuf #endif
1235e4e55b47STetsuo Handa #ifdef CONFIG_SECURITY
1236e4e55b47STetsuo Handa 	/* Used by LSM modules for access restriction: */
1237e4e55b47STetsuo Handa 	void				*security;
1238e4e55b47STetsuo Handa #endif
123929e48ce8SKees Cook 
1240afaef01cSAlexander Popov #ifdef CONFIG_GCC_PLUGIN_STACKLEAK
1241afaef01cSAlexander Popov 	unsigned long			lowest_stack;
1242c8d12627SAlexander Popov 	unsigned long			prev_lowest_stack;
1243afaef01cSAlexander Popov #endif
1244afaef01cSAlexander Popov 
124529e48ce8SKees Cook 	/*
124629e48ce8SKees Cook 	 * New fields for task_struct should be added above here, so that
124729e48ce8SKees Cook 	 * they are included in the randomized portion of task_struct.
124829e48ce8SKees Cook 	 */
124929e48ce8SKees Cook 	randomized_struct_fields_end
125029e48ce8SKees Cook 
12515eca1c10SIngo Molnar 	/* CPU-specific state of this task: */
12520c8c0f03SDave Hansen 	struct thread_struct		thread;
12535eca1c10SIngo Molnar 
12540c8c0f03SDave Hansen 	/*
12550c8c0f03SDave Hansen 	 * WARNING: on x86, 'thread_struct' contains a variable-sized
12560c8c0f03SDave Hansen 	 * structure.  It *MUST* be at the end of 'task_struct'.
12570c8c0f03SDave Hansen 	 *
12580c8c0f03SDave Hansen 	 * Do not put anything below here!
12590c8c0f03SDave Hansen 	 */
12601da177e4SLinus Torvalds };
12611da177e4SLinus Torvalds 
1262e868171aSAlexey Dobriyan static inline struct pid *task_pid(struct task_struct *task)
126322c935f4SEric W. Biederman {
12642c470475SEric W. Biederman 	return task->thread_pid;
126522c935f4SEric W. Biederman }
126622c935f4SEric W. Biederman 
12677af57294SPavel Emelyanov /*
12687af57294SPavel Emelyanov  * the helpers to get the task's different pids as they are seen
12697af57294SPavel Emelyanov  * from various namespaces
12707af57294SPavel Emelyanov  *
12717af57294SPavel Emelyanov  * task_xid_nr()     : global id, i.e. the id seen from the init namespace;
127244c4e1b2SEric W. Biederman  * task_xid_vnr()    : virtual id, i.e. the id seen from the pid namespace of
127344c4e1b2SEric W. Biederman  *                     current.
12747af57294SPavel Emelyanov  * task_xid_nr_ns()  : id seen from the ns specified;
12757af57294SPavel Emelyanov  *
12767af57294SPavel Emelyanov  * see also pid_nr() etc in include/linux/pid.h
12777af57294SPavel Emelyanov  */
12785eca1c10SIngo Molnar pid_t __task_pid_nr_ns(struct task_struct *task, enum pid_type type, struct pid_namespace *ns);
12797af57294SPavel Emelyanov 
1280e868171aSAlexey Dobriyan static inline pid_t task_pid_nr(struct task_struct *tsk)
12817af57294SPavel Emelyanov {
12827af57294SPavel Emelyanov 	return tsk->pid;
12837af57294SPavel Emelyanov }
12847af57294SPavel Emelyanov 
12855eca1c10SIngo Molnar static inline pid_t task_pid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
128652ee2dfdSOleg Nesterov {
128752ee2dfdSOleg Nesterov 	return __task_pid_nr_ns(tsk, PIDTYPE_PID, ns);
128852ee2dfdSOleg Nesterov }
12897af57294SPavel Emelyanov 
12907af57294SPavel Emelyanov static inline pid_t task_pid_vnr(struct task_struct *tsk)
12917af57294SPavel Emelyanov {
129252ee2dfdSOleg Nesterov 	return __task_pid_nr_ns(tsk, PIDTYPE_PID, NULL);
12937af57294SPavel Emelyanov }
12947af57294SPavel Emelyanov 
12957af57294SPavel Emelyanov 
1296e868171aSAlexey Dobriyan static inline pid_t task_tgid_nr(struct task_struct *tsk)
12977af57294SPavel Emelyanov {
12987af57294SPavel Emelyanov 	return tsk->tgid;
12997af57294SPavel Emelyanov }
13007af57294SPavel Emelyanov 
13015eca1c10SIngo Molnar /**
13025eca1c10SIngo Molnar  * pid_alive - check that a task structure is not stale
13035eca1c10SIngo Molnar  * @p: Task structure to be checked.
13045eca1c10SIngo Molnar  *
13055eca1c10SIngo Molnar  * Test if a process is not yet dead (at most zombie state)
13065eca1c10SIngo Molnar  * If pid_alive fails, then pointers within the task structure
13075eca1c10SIngo Molnar  * can be stale and must not be dereferenced.
13085eca1c10SIngo Molnar  *
13095eca1c10SIngo Molnar  * Return: 1 if the process is alive. 0 otherwise.
13105eca1c10SIngo Molnar  */
13115eca1c10SIngo Molnar static inline int pid_alive(const struct task_struct *p)
13125eca1c10SIngo Molnar {
13132c470475SEric W. Biederman 	return p->thread_pid != NULL;
13145eca1c10SIngo Molnar }
13157af57294SPavel Emelyanov 
13165eca1c10SIngo Molnar static inline pid_t task_pgrp_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
13177af57294SPavel Emelyanov {
131852ee2dfdSOleg Nesterov 	return __task_pid_nr_ns(tsk, PIDTYPE_PGID, ns);
13197af57294SPavel Emelyanov }
13207af57294SPavel Emelyanov 
13217af57294SPavel Emelyanov static inline pid_t task_pgrp_vnr(struct task_struct *tsk)
13227af57294SPavel Emelyanov {
132352ee2dfdSOleg Nesterov 	return __task_pid_nr_ns(tsk, PIDTYPE_PGID, NULL);
13247af57294SPavel Emelyanov }
13257af57294SPavel Emelyanov 
13267af57294SPavel Emelyanov 
13275eca1c10SIngo Molnar static inline pid_t task_session_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
13287af57294SPavel Emelyanov {
132952ee2dfdSOleg Nesterov 	return __task_pid_nr_ns(tsk, PIDTYPE_SID, ns);
13307af57294SPavel Emelyanov }
13317af57294SPavel Emelyanov 
13327af57294SPavel Emelyanov static inline pid_t task_session_vnr(struct task_struct *tsk)
13337af57294SPavel Emelyanov {
133452ee2dfdSOleg Nesterov 	return __task_pid_nr_ns(tsk, PIDTYPE_SID, NULL);
13357af57294SPavel Emelyanov }
13367af57294SPavel Emelyanov 
1337dd1c1f2fSOleg Nesterov static inline pid_t task_tgid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
1338dd1c1f2fSOleg Nesterov {
13396883f81aSEric W. Biederman 	return __task_pid_nr_ns(tsk, PIDTYPE_TGID, ns);
1340dd1c1f2fSOleg Nesterov }
1341dd1c1f2fSOleg Nesterov 
1342dd1c1f2fSOleg Nesterov static inline pid_t task_tgid_vnr(struct task_struct *tsk)
1343dd1c1f2fSOleg Nesterov {
13446883f81aSEric W. Biederman 	return __task_pid_nr_ns(tsk, PIDTYPE_TGID, NULL);
1345dd1c1f2fSOleg Nesterov }
1346dd1c1f2fSOleg Nesterov 
1347dd1c1f2fSOleg Nesterov static inline pid_t task_ppid_nr_ns(const struct task_struct *tsk, struct pid_namespace *ns)
1348dd1c1f2fSOleg Nesterov {
1349dd1c1f2fSOleg Nesterov 	pid_t pid = 0;
1350dd1c1f2fSOleg Nesterov 
1351dd1c1f2fSOleg Nesterov 	rcu_read_lock();
1352dd1c1f2fSOleg Nesterov 	if (pid_alive(tsk))
1353dd1c1f2fSOleg Nesterov 		pid = task_tgid_nr_ns(rcu_dereference(tsk->real_parent), ns);
1354dd1c1f2fSOleg Nesterov 	rcu_read_unlock();
1355dd1c1f2fSOleg Nesterov 
1356dd1c1f2fSOleg Nesterov 	return pid;
1357dd1c1f2fSOleg Nesterov }
1358dd1c1f2fSOleg Nesterov 
1359dd1c1f2fSOleg Nesterov static inline pid_t task_ppid_nr(const struct task_struct *tsk)
1360dd1c1f2fSOleg Nesterov {
1361dd1c1f2fSOleg Nesterov 	return task_ppid_nr_ns(tsk, &init_pid_ns);
1362dd1c1f2fSOleg Nesterov }
1363dd1c1f2fSOleg Nesterov 
13645eca1c10SIngo Molnar /* Obsolete, do not use: */
13651b0f7ffdSOleg Nesterov static inline pid_t task_pgrp_nr(struct task_struct *tsk)
13661b0f7ffdSOleg Nesterov {
13671b0f7ffdSOleg Nesterov 	return task_pgrp_nr_ns(tsk, &init_pid_ns);
13681b0f7ffdSOleg Nesterov }
13697af57294SPavel Emelyanov 
137006eb6184SPeter Zijlstra #define TASK_REPORT_IDLE	(TASK_REPORT + 1)
137106eb6184SPeter Zijlstra #define TASK_REPORT_MAX		(TASK_REPORT_IDLE << 1)
137206eb6184SPeter Zijlstra 
13731d48b080SPeter Zijlstra static inline unsigned int task_state_index(struct task_struct *tsk)
137420435d84SXie XiuQi {
13751593baabSPeter Zijlstra 	unsigned int tsk_state = READ_ONCE(tsk->state);
13761593baabSPeter Zijlstra 	unsigned int state = (tsk_state | tsk->exit_state) & TASK_REPORT;
137720435d84SXie XiuQi 
137806eb6184SPeter Zijlstra 	BUILD_BUG_ON_NOT_POWER_OF_2(TASK_REPORT_MAX);
137906eb6184SPeter Zijlstra 
138006eb6184SPeter Zijlstra 	if (tsk_state == TASK_IDLE)
138106eb6184SPeter Zijlstra 		state = TASK_REPORT_IDLE;
138206eb6184SPeter Zijlstra 
13831593baabSPeter Zijlstra 	return fls(state);
13841593baabSPeter Zijlstra }
138520435d84SXie XiuQi 
13861d48b080SPeter Zijlstra static inline char task_index_to_char(unsigned int state)
13871593baabSPeter Zijlstra {
13888ef9925bSPeter Zijlstra 	static const char state_char[] = "RSDTtXZPI";
13891593baabSPeter Zijlstra 
139006eb6184SPeter Zijlstra 	BUILD_BUG_ON(1 + ilog2(TASK_REPORT_MAX) != sizeof(state_char) - 1);
13911593baabSPeter Zijlstra 
13921593baabSPeter Zijlstra 	return state_char[state];
13931593baabSPeter Zijlstra }
13941593baabSPeter Zijlstra 
13951593baabSPeter Zijlstra static inline char task_state_to_char(struct task_struct *tsk)
13961593baabSPeter Zijlstra {
13971d48b080SPeter Zijlstra 	return task_index_to_char(task_state_index(tsk));
139820435d84SXie XiuQi }
139920435d84SXie XiuQi 
14001da177e4SLinus Torvalds /**
1401570f5241SSergey Senozhatsky  * is_global_init - check if a task structure is init. Since init
1402570f5241SSergey Senozhatsky  * is free to have sub-threads we need to check tgid.
14033260259fSHenne  * @tsk: Task structure to be checked.
14043260259fSHenne  *
14053260259fSHenne  * Check if a task structure is the first user space task the kernel created.
1406e69f6186SYacine Belkadi  *
1407e69f6186SYacine Belkadi  * Return: 1 if the task structure is init. 0 otherwise.
1408f400e198SSukadev Bhattiprolu  */
1409e868171aSAlexey Dobriyan static inline int is_global_init(struct task_struct *tsk)
1410b461cc03SPavel Emelyanov {
1411570f5241SSergey Senozhatsky 	return task_tgid_nr(tsk) == 1;
1412b461cc03SPavel Emelyanov }
1413b460cbc5SSerge E. Hallyn 
14149ec52099SCedric Le Goater extern struct pid *cad_pid;
14159ec52099SCedric Le Goater 
14161da177e4SLinus Torvalds /*
14171da177e4SLinus Torvalds  * Per process flags
14181da177e4SLinus Torvalds  */
1419c1de45caSPeter Zijlstra #define PF_IDLE			0x00000002	/* I am an IDLE thread */
14205eca1c10SIngo Molnar #define PF_EXITING		0x00000004	/* Getting shut down */
14215eca1c10SIngo Molnar #define PF_EXITPIDONE		0x00000008	/* PI exit done on shut down */
142294886b84SLaurent Vivier #define PF_VCPU			0x00000010	/* I'm a virtual CPU */
142321aa9af0STejun Heo #define PF_WQ_WORKER		0x00000020	/* I'm a workqueue worker */
14245eca1c10SIngo Molnar #define PF_FORKNOEXEC		0x00000040	/* Forked but didn't exec */
14255eca1c10SIngo Molnar #define PF_MCE_PROCESS		0x00000080      /* Process policy on mce errors */
14265eca1c10SIngo Molnar #define PF_SUPERPRIV		0x00000100	/* Used super-user privileges */
14275eca1c10SIngo Molnar #define PF_DUMPCORE		0x00000200	/* Dumped core */
14285eca1c10SIngo Molnar #define PF_SIGNALED		0x00000400	/* Killed by a signal */
14291da177e4SLinus Torvalds #define PF_MEMALLOC		0x00000800	/* Allocating memory */
14305eca1c10SIngo Molnar #define PF_NPROC_EXCEEDED	0x00001000	/* set_user() noticed that RLIMIT_NPROC was exceeded */
14315eca1c10SIngo Molnar #define PF_USED_MATH		0x00002000	/* If unset the fpu must be initialized before use */
14325eca1c10SIngo Molnar #define PF_USED_ASYNC		0x00004000	/* Used async_schedule*(), used by module init */
14335eca1c10SIngo Molnar #define PF_NOFREEZE		0x00008000	/* This thread should not be frozen */
14345eca1c10SIngo Molnar #define PF_FROZEN		0x00010000	/* Frozen for system suspend */
14357dea19f9SMichal Hocko #define PF_KSWAPD		0x00020000	/* I am kswapd */
14367dea19f9SMichal Hocko #define PF_MEMALLOC_NOFS	0x00040000	/* All allocation requests will inherit GFP_NOFS */
14377dea19f9SMichal Hocko #define PF_MEMALLOC_NOIO	0x00080000	/* All allocation requests will inherit GFP_NOIO */
14381da177e4SLinus Torvalds #define PF_LESS_THROTTLE	0x00100000	/* Throttle me less: I clean memory */
1439246bb0b1SOleg Nesterov #define PF_KTHREAD		0x00200000	/* I am a kernel thread */
14405eca1c10SIngo Molnar #define PF_RANDOMIZE		0x00400000	/* Randomize virtual address space */
1441b31dc66aSJens Axboe #define PF_SWAPWRITE		0x00800000	/* Allowed to write to swap */
1442eb414681SJohannes Weiner #define PF_MEMSTALL		0x01000000	/* Stalled due to lack of memory */
144373ab1cb2STaehee Yoo #define PF_UMH			0x02000000	/* I'm an Usermodehelper process */
14443bd37062SSebastian Andrzej Siewior #define PF_NO_SETAFFINITY	0x04000000	/* Userland is not allowed to meddle with cpus_mask */
14454db96cf0SAndi Kleen #define PF_MCE_EARLY		0x08000000      /* Early kill for mce process policy */
1446d7fefcc8SAneesh Kumar K.V #define PF_MEMALLOC_NOCMA	0x10000000	/* All allocation request will have _GFP_MOVABLE cleared */
144758a69cb4STejun Heo #define PF_FREEZER_SKIP		0x40000000	/* Freezer should not count it as freezable */
14485eca1c10SIngo Molnar #define PF_SUSPEND_TASK		0x80000000      /* This thread called freeze_processes() and should not be frozen */
14491da177e4SLinus Torvalds 
14501da177e4SLinus Torvalds /*
14511da177e4SLinus Torvalds  * Only the _current_ task can read/write to tsk->flags, but other
14521da177e4SLinus Torvalds  * tasks can access tsk->flags in readonly mode for example
14531da177e4SLinus Torvalds  * with tsk_used_math (like during threaded core dumping).
14541da177e4SLinus Torvalds  * There is however an exception to this rule during ptrace
14551da177e4SLinus Torvalds  * or during fork: the ptracer task is allowed to write to the
14561da177e4SLinus Torvalds  * child->flags of its traced child (same goes for fork, the parent
14571da177e4SLinus Torvalds  * can write to the child->flags), because we're guaranteed the
14581da177e4SLinus Torvalds  * child is not running and in turn not changing child->flags
14591da177e4SLinus Torvalds  * at the same time the parent does it.
14601da177e4SLinus Torvalds  */
14611da177e4SLinus Torvalds #define clear_stopped_child_used_math(child)	do { (child)->flags &= ~PF_USED_MATH; } while (0)
14621da177e4SLinus Torvalds #define set_stopped_child_used_math(child)	do { (child)->flags |= PF_USED_MATH; } while (0)
14631da177e4SLinus Torvalds #define clear_used_math()			clear_stopped_child_used_math(current)
14641da177e4SLinus Torvalds #define set_used_math()				set_stopped_child_used_math(current)
14655eca1c10SIngo Molnar 
14661da177e4SLinus Torvalds #define conditional_stopped_child_used_math(condition, child) \
14671da177e4SLinus Torvalds 	do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= (condition) ? PF_USED_MATH : 0; } while (0)
14685eca1c10SIngo Molnar 
14695eca1c10SIngo Molnar #define conditional_used_math(condition)	conditional_stopped_child_used_math(condition, current)
14705eca1c10SIngo Molnar 
14711da177e4SLinus Torvalds #define copy_to_stopped_child_used_math(child) \
14721da177e4SLinus Torvalds 	do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= current->flags & PF_USED_MATH; } while (0)
14735eca1c10SIngo Molnar 
14741da177e4SLinus Torvalds /* NOTE: this will return 0 or PF_USED_MATH, it will never return 1 */
14751da177e4SLinus Torvalds #define tsk_used_math(p)			((p)->flags & PF_USED_MATH)
14761da177e4SLinus Torvalds #define used_math()				tsk_used_math(current)
14771da177e4SLinus Torvalds 
147862ec05ddSThomas Gleixner static inline bool is_percpu_thread(void)
147962ec05ddSThomas Gleixner {
148062ec05ddSThomas Gleixner #ifdef CONFIG_SMP
148162ec05ddSThomas Gleixner 	return (current->flags & PF_NO_SETAFFINITY) &&
148262ec05ddSThomas Gleixner 		(current->nr_cpus_allowed  == 1);
148362ec05ddSThomas Gleixner #else
148462ec05ddSThomas Gleixner 	return true;
148562ec05ddSThomas Gleixner #endif
148662ec05ddSThomas Gleixner }
148762ec05ddSThomas Gleixner 
14881d4457f9SKees Cook /* Per-process atomic flags. */
1489a2b86f77SZefan Li #define PFA_NO_NEW_PRIVS		0	/* May not gain new privileges. */
14902ad654bcSZefan Li #define PFA_SPREAD_PAGE			1	/* Spread page cache over cpuset */
14912ad654bcSZefan Li #define PFA_SPREAD_SLAB			2	/* Spread some slab caches over cpuset */
1492356e4bffSThomas Gleixner #define PFA_SPEC_SSB_DISABLE		3	/* Speculative Store Bypass disabled */
1493356e4bffSThomas Gleixner #define PFA_SPEC_SSB_FORCE_DISABLE	4	/* Speculative Store Bypass force disabled*/
14949137bb27SThomas Gleixner #define PFA_SPEC_IB_DISABLE		5	/* Indirect branch speculation restricted */
14959137bb27SThomas Gleixner #define PFA_SPEC_IB_FORCE_DISABLE	6	/* Indirect branch speculation permanently restricted */
149671368af9SWaiman Long #define PFA_SPEC_SSB_NOEXEC		7	/* Speculative Store Bypass clear on execve() */
14971d4457f9SKees Cook 
1498e0e5070bSZefan Li #define TASK_PFA_TEST(name, func)					\
1499e0e5070bSZefan Li 	static inline bool task_##func(struct task_struct *p)		\
1500e0e5070bSZefan Li 	{ return test_bit(PFA_##name, &p->atomic_flags); }
15015eca1c10SIngo Molnar 
1502e0e5070bSZefan Li #define TASK_PFA_SET(name, func)					\
1503e0e5070bSZefan Li 	static inline void task_set_##func(struct task_struct *p)	\
1504e0e5070bSZefan Li 	{ set_bit(PFA_##name, &p->atomic_flags); }
15055eca1c10SIngo Molnar 
1506e0e5070bSZefan Li #define TASK_PFA_CLEAR(name, func)					\
1507e0e5070bSZefan Li 	static inline void task_clear_##func(struct task_struct *p)	\
1508e0e5070bSZefan Li 	{ clear_bit(PFA_##name, &p->atomic_flags); }
15091d4457f9SKees Cook 
1510e0e5070bSZefan Li TASK_PFA_TEST(NO_NEW_PRIVS, no_new_privs)
1511e0e5070bSZefan Li TASK_PFA_SET(NO_NEW_PRIVS, no_new_privs)
15121d4457f9SKees Cook 
15132ad654bcSZefan Li TASK_PFA_TEST(SPREAD_PAGE, spread_page)
15142ad654bcSZefan Li TASK_PFA_SET(SPREAD_PAGE, spread_page)
15152ad654bcSZefan Li TASK_PFA_CLEAR(SPREAD_PAGE, spread_page)
15162ad654bcSZefan Li 
15172ad654bcSZefan Li TASK_PFA_TEST(SPREAD_SLAB, spread_slab)
15182ad654bcSZefan Li TASK_PFA_SET(SPREAD_SLAB, spread_slab)
15192ad654bcSZefan Li TASK_PFA_CLEAR(SPREAD_SLAB, spread_slab)
1520544b2c91STejun Heo 
1521356e4bffSThomas Gleixner TASK_PFA_TEST(SPEC_SSB_DISABLE, spec_ssb_disable)
1522356e4bffSThomas Gleixner TASK_PFA_SET(SPEC_SSB_DISABLE, spec_ssb_disable)
1523356e4bffSThomas Gleixner TASK_PFA_CLEAR(SPEC_SSB_DISABLE, spec_ssb_disable)
1524356e4bffSThomas Gleixner 
152571368af9SWaiman Long TASK_PFA_TEST(SPEC_SSB_NOEXEC, spec_ssb_noexec)
152671368af9SWaiman Long TASK_PFA_SET(SPEC_SSB_NOEXEC, spec_ssb_noexec)
152771368af9SWaiman Long TASK_PFA_CLEAR(SPEC_SSB_NOEXEC, spec_ssb_noexec)
152871368af9SWaiman Long 
1529356e4bffSThomas Gleixner TASK_PFA_TEST(SPEC_SSB_FORCE_DISABLE, spec_ssb_force_disable)
1530356e4bffSThomas Gleixner TASK_PFA_SET(SPEC_SSB_FORCE_DISABLE, spec_ssb_force_disable)
1531356e4bffSThomas Gleixner 
15329137bb27SThomas Gleixner TASK_PFA_TEST(SPEC_IB_DISABLE, spec_ib_disable)
15339137bb27SThomas Gleixner TASK_PFA_SET(SPEC_IB_DISABLE, spec_ib_disable)
15349137bb27SThomas Gleixner TASK_PFA_CLEAR(SPEC_IB_DISABLE, spec_ib_disable)
15359137bb27SThomas Gleixner 
15369137bb27SThomas Gleixner TASK_PFA_TEST(SPEC_IB_FORCE_DISABLE, spec_ib_force_disable)
15379137bb27SThomas Gleixner TASK_PFA_SET(SPEC_IB_FORCE_DISABLE, spec_ib_force_disable)
15389137bb27SThomas Gleixner 
15395eca1c10SIngo Molnar static inline void
1540717a94b5SNeilBrown current_restore_flags(unsigned long orig_flags, unsigned long flags)
1541907aed48SMel Gorman {
1542717a94b5SNeilBrown 	current->flags &= ~flags;
1543717a94b5SNeilBrown 	current->flags |= orig_flags & flags;
1544907aed48SMel Gorman }
1545907aed48SMel Gorman 
15465eca1c10SIngo Molnar extern int cpuset_cpumask_can_shrink(const struct cpumask *cur, const struct cpumask *trial);
15475eca1c10SIngo Molnar extern int task_can_attach(struct task_struct *p, const struct cpumask *cs_cpus_allowed);
15481da177e4SLinus Torvalds #ifdef CONFIG_SMP
15495eca1c10SIngo Molnar extern void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask);
15505eca1c10SIngo Molnar extern int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask);
15511da177e4SLinus Torvalds #else
15525eca1c10SIngo Molnar static inline void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
15531e1b6c51SKOSAKI Motohiro {
15541e1b6c51SKOSAKI Motohiro }
15555eca1c10SIngo Molnar static inline int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
15561da177e4SLinus Torvalds {
155796f874e2SRusty Russell 	if (!cpumask_test_cpu(0, new_mask))
15581da177e4SLinus Torvalds 		return -EINVAL;
15591da177e4SLinus Torvalds 	return 0;
15601da177e4SLinus Torvalds }
15611da177e4SLinus Torvalds #endif
1562e0ad9556SRusty Russell 
15636d0d2878SChristian Borntraeger #ifndef cpu_relax_yield
15646d0d2878SChristian Borntraeger #define cpu_relax_yield() cpu_relax()
15656d0d2878SChristian Borntraeger #endif
15666d0d2878SChristian Borntraeger 
1567fa93384fSDan Carpenter extern int yield_to(struct task_struct *p, bool preempt);
156836c8b586SIngo Molnar extern void set_user_nice(struct task_struct *p, long nice);
156936c8b586SIngo Molnar extern int task_prio(const struct task_struct *p);
15705eca1c10SIngo Molnar 
1571d0ea0268SDongsheng Yang /**
1572d0ea0268SDongsheng Yang  * task_nice - return the nice value of a given task.
1573d0ea0268SDongsheng Yang  * @p: the task in question.
1574d0ea0268SDongsheng Yang  *
1575d0ea0268SDongsheng Yang  * Return: The nice value [ -20 ... 0 ... 19 ].
1576d0ea0268SDongsheng Yang  */
1577d0ea0268SDongsheng Yang static inline int task_nice(const struct task_struct *p)
1578d0ea0268SDongsheng Yang {
1579d0ea0268SDongsheng Yang 	return PRIO_TO_NICE((p)->static_prio);
1580d0ea0268SDongsheng Yang }
15815eca1c10SIngo Molnar 
158236c8b586SIngo Molnar extern int can_nice(const struct task_struct *p, const int nice);
158336c8b586SIngo Molnar extern int task_curr(const struct task_struct *p);
15841da177e4SLinus Torvalds extern int idle_cpu(int cpu);
1585943d355dSRohit Jain extern int available_idle_cpu(int cpu);
15865eca1c10SIngo Molnar extern int sched_setscheduler(struct task_struct *, int, const struct sched_param *);
15875eca1c10SIngo Molnar extern int sched_setscheduler_nocheck(struct task_struct *, int, const struct sched_param *);
15885eca1c10SIngo Molnar extern int sched_setattr(struct task_struct *, const struct sched_attr *);
1589794a56ebSJuri Lelli extern int sched_setattr_nocheck(struct task_struct *, const struct sched_attr *);
159036c8b586SIngo Molnar extern struct task_struct *idle_task(int cpu);
15915eca1c10SIngo Molnar 
1592c4f30608SPaul E. McKenney /**
1593c4f30608SPaul E. McKenney  * is_idle_task - is the specified task an idle task?
1594fa757281SRandy Dunlap  * @p: the task in question.
1595e69f6186SYacine Belkadi  *
1596e69f6186SYacine Belkadi  * Return: 1 if @p is an idle task. 0 otherwise.
1597c4f30608SPaul E. McKenney  */
15987061ca3bSPaul E. McKenney static inline bool is_idle_task(const struct task_struct *p)
1599c4f30608SPaul E. McKenney {
1600c1de45caSPeter Zijlstra 	return !!(p->flags & PF_IDLE);
1601c4f30608SPaul E. McKenney }
16025eca1c10SIngo Molnar 
160336c8b586SIngo Molnar extern struct task_struct *curr_task(int cpu);
1604a458ae2eSPeter Zijlstra extern void ia64_set_curr_task(int cpu, struct task_struct *p);
16051da177e4SLinus Torvalds 
16061da177e4SLinus Torvalds void yield(void);
16071da177e4SLinus Torvalds 
16081da177e4SLinus Torvalds union thread_union {
16090500871fSDavid Howells #ifndef CONFIG_ARCH_TASK_STRUCT_ON_STACK
16100500871fSDavid Howells 	struct task_struct task;
16110500871fSDavid Howells #endif
1612c65eacbeSAndy Lutomirski #ifndef CONFIG_THREAD_INFO_IN_TASK
16131da177e4SLinus Torvalds 	struct thread_info thread_info;
1614c65eacbeSAndy Lutomirski #endif
16151da177e4SLinus Torvalds 	unsigned long stack[THREAD_SIZE/sizeof(long)];
16161da177e4SLinus Torvalds };
16171da177e4SLinus Torvalds 
16180500871fSDavid Howells #ifndef CONFIG_THREAD_INFO_IN_TASK
16190500871fSDavid Howells extern struct thread_info init_thread_info;
16200500871fSDavid Howells #endif
16210500871fSDavid Howells 
16220500871fSDavid Howells extern unsigned long init_stack[THREAD_SIZE / sizeof(unsigned long)];
16230500871fSDavid Howells 
1624f3ac6067SIngo Molnar #ifdef CONFIG_THREAD_INFO_IN_TASK
1625f3ac6067SIngo Molnar static inline struct thread_info *task_thread_info(struct task_struct *task)
1626f3ac6067SIngo Molnar {
1627f3ac6067SIngo Molnar 	return &task->thread_info;
1628f3ac6067SIngo Molnar }
1629f3ac6067SIngo Molnar #elif !defined(__HAVE_THREAD_FUNCTIONS)
1630f3ac6067SIngo Molnar # define task_thread_info(task)	((struct thread_info *)(task)->stack)
1631f3ac6067SIngo Molnar #endif
1632f3ac6067SIngo Molnar 
1633198fe21bSPavel Emelyanov /*
1634198fe21bSPavel Emelyanov  * find a task by one of its numerical ids
1635198fe21bSPavel Emelyanov  *
1636198fe21bSPavel Emelyanov  * find_task_by_pid_ns():
1637198fe21bSPavel Emelyanov  *      finds a task by its pid in the specified namespace
1638228ebcbeSPavel Emelyanov  * find_task_by_vpid():
1639228ebcbeSPavel Emelyanov  *      finds a task by its virtual pid
1640198fe21bSPavel Emelyanov  *
1641e49859e7SPavel Emelyanov  * see also find_vpid() etc in include/linux/pid.h
1642198fe21bSPavel Emelyanov  */
1643198fe21bSPavel Emelyanov 
1644228ebcbeSPavel Emelyanov extern struct task_struct *find_task_by_vpid(pid_t nr);
16455eca1c10SIngo Molnar extern struct task_struct *find_task_by_pid_ns(pid_t nr, struct pid_namespace *ns);
1646198fe21bSPavel Emelyanov 
16472ee08260SMike Rapoport /*
16482ee08260SMike Rapoport  * find a task by its virtual pid and get the task struct
16492ee08260SMike Rapoport  */
16502ee08260SMike Rapoport extern struct task_struct *find_get_task_by_vpid(pid_t nr);
16512ee08260SMike Rapoport 
1652b3c97528SHarvey Harrison extern int wake_up_state(struct task_struct *tsk, unsigned int state);
1653b3c97528SHarvey Harrison extern int wake_up_process(struct task_struct *tsk);
16543e51e3edSSamir Bellabes extern void wake_up_new_task(struct task_struct *tsk);
16555eca1c10SIngo Molnar 
16561da177e4SLinus Torvalds #ifdef CONFIG_SMP
16571da177e4SLinus Torvalds extern void kick_process(struct task_struct *tsk);
16581da177e4SLinus Torvalds #else
16591da177e4SLinus Torvalds static inline void kick_process(struct task_struct *tsk) { }
16601da177e4SLinus Torvalds #endif
16611da177e4SLinus Torvalds 
166282b89778SAdrian Hunter extern void __set_task_comm(struct task_struct *tsk, const char *from, bool exec);
16635eca1c10SIngo Molnar 
166482b89778SAdrian Hunter static inline void set_task_comm(struct task_struct *tsk, const char *from)
166582b89778SAdrian Hunter {
166682b89778SAdrian Hunter 	__set_task_comm(tsk, from, false);
166782b89778SAdrian Hunter }
16685eca1c10SIngo Molnar 
16693756f640SArnd Bergmann extern char *__get_task_comm(char *to, size_t len, struct task_struct *tsk);
16703756f640SArnd Bergmann #define get_task_comm(buf, tsk) ({			\
16713756f640SArnd Bergmann 	BUILD_BUG_ON(sizeof(buf) != TASK_COMM_LEN);	\
16723756f640SArnd Bergmann 	__get_task_comm(buf, sizeof(buf), tsk);		\
16733756f640SArnd Bergmann })
16741da177e4SLinus Torvalds 
16751da177e4SLinus Torvalds #ifdef CONFIG_SMP
1676317f3941SPeter Zijlstra void scheduler_ipi(void);
167785ba2d86SRoland McGrath extern unsigned long wait_task_inactive(struct task_struct *, long match_state);
16781da177e4SLinus Torvalds #else
1679184748ccSPeter Zijlstra static inline void scheduler_ipi(void) { }
16805eca1c10SIngo Molnar static inline unsigned long wait_task_inactive(struct task_struct *p, long match_state)
168185ba2d86SRoland McGrath {
168285ba2d86SRoland McGrath 	return 1;
168385ba2d86SRoland McGrath }
16841da177e4SLinus Torvalds #endif
16851da177e4SLinus Torvalds 
16865eca1c10SIngo Molnar /*
16875eca1c10SIngo Molnar  * Set thread flags in other task's structures.
16885eca1c10SIngo Molnar  * See asm/thread_info.h for TIF_xxxx flags available:
16891da177e4SLinus Torvalds  */
16901da177e4SLinus Torvalds static inline void set_tsk_thread_flag(struct task_struct *tsk, int flag)
16911da177e4SLinus Torvalds {
1692a1261f54SAl Viro 	set_ti_thread_flag(task_thread_info(tsk), flag);
16931da177e4SLinus Torvalds }
16941da177e4SLinus Torvalds 
16951da177e4SLinus Torvalds static inline void clear_tsk_thread_flag(struct task_struct *tsk, int flag)
16961da177e4SLinus Torvalds {
1697a1261f54SAl Viro 	clear_ti_thread_flag(task_thread_info(tsk), flag);
16981da177e4SLinus Torvalds }
16991da177e4SLinus Torvalds 
170093ee37c2SDave Martin static inline void update_tsk_thread_flag(struct task_struct *tsk, int flag,
170193ee37c2SDave Martin 					  bool value)
170293ee37c2SDave Martin {
170393ee37c2SDave Martin 	update_ti_thread_flag(task_thread_info(tsk), flag, value);
170493ee37c2SDave Martin }
170593ee37c2SDave Martin 
17061da177e4SLinus Torvalds static inline int test_and_set_tsk_thread_flag(struct task_struct *tsk, int flag)
17071da177e4SLinus Torvalds {
1708a1261f54SAl Viro 	return test_and_set_ti_thread_flag(task_thread_info(tsk), flag);
17091da177e4SLinus Torvalds }
17101da177e4SLinus Torvalds 
17111da177e4SLinus Torvalds static inline int test_and_clear_tsk_thread_flag(struct task_struct *tsk, int flag)
17121da177e4SLinus Torvalds {
1713a1261f54SAl Viro 	return test_and_clear_ti_thread_flag(task_thread_info(tsk), flag);
17141da177e4SLinus Torvalds }
17151da177e4SLinus Torvalds 
17161da177e4SLinus Torvalds static inline int test_tsk_thread_flag(struct task_struct *tsk, int flag)
17171da177e4SLinus Torvalds {
1718a1261f54SAl Viro 	return test_ti_thread_flag(task_thread_info(tsk), flag);
17191da177e4SLinus Torvalds }
17201da177e4SLinus Torvalds 
17211da177e4SLinus Torvalds static inline void set_tsk_need_resched(struct task_struct *tsk)
17221da177e4SLinus Torvalds {
17231da177e4SLinus Torvalds 	set_tsk_thread_flag(tsk,TIF_NEED_RESCHED);
17241da177e4SLinus Torvalds }
17251da177e4SLinus Torvalds 
17261da177e4SLinus Torvalds static inline void clear_tsk_need_resched(struct task_struct *tsk)
17271da177e4SLinus Torvalds {
17281da177e4SLinus Torvalds 	clear_tsk_thread_flag(tsk,TIF_NEED_RESCHED);
17291da177e4SLinus Torvalds }
17301da177e4SLinus Torvalds 
17318ae121acSGregory Haskins static inline int test_tsk_need_resched(struct task_struct *tsk)
17328ae121acSGregory Haskins {
17338ae121acSGregory Haskins 	return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED));
17348ae121acSGregory Haskins }
17358ae121acSGregory Haskins 
17361da177e4SLinus Torvalds /*
17371da177e4SLinus Torvalds  * cond_resched() and cond_resched_lock(): latency reduction via
17381da177e4SLinus Torvalds  * explicit rescheduling in places that are safe. The return
17391da177e4SLinus Torvalds  * value indicates whether a reschedule was done in fact.
17401da177e4SLinus Torvalds  * cond_resched_lock() will drop the spinlock before scheduling,
17411da177e4SLinus Torvalds  */
174235a773a0SPeter Zijlstra #ifndef CONFIG_PREEMPT
1743c3921ab7SLinus Torvalds extern int _cond_resched(void);
174435a773a0SPeter Zijlstra #else
174535a773a0SPeter Zijlstra static inline int _cond_resched(void) { return 0; }
174635a773a0SPeter Zijlstra #endif
17476f80bd98SFrederic Weisbecker 
1748613afbf8SFrederic Weisbecker #define cond_resched() ({			\
17493427445aSPeter Zijlstra 	___might_sleep(__FILE__, __LINE__, 0);	\
1750613afbf8SFrederic Weisbecker 	_cond_resched();			\
1751613afbf8SFrederic Weisbecker })
17526f80bd98SFrederic Weisbecker 
1753613afbf8SFrederic Weisbecker extern int __cond_resched_lock(spinlock_t *lock);
1754613afbf8SFrederic Weisbecker 
1755613afbf8SFrederic Weisbecker #define cond_resched_lock(lock) ({				\
17563427445aSPeter Zijlstra 	___might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET);\
1757613afbf8SFrederic Weisbecker 	__cond_resched_lock(lock);				\
1758613afbf8SFrederic Weisbecker })
1759613afbf8SFrederic Weisbecker 
1760f6f3c437SSimon Horman static inline void cond_resched_rcu(void)
1761f6f3c437SSimon Horman {
1762f6f3c437SSimon Horman #if defined(CONFIG_DEBUG_ATOMIC_SLEEP) || !defined(CONFIG_PREEMPT_RCU)
1763f6f3c437SSimon Horman 	rcu_read_unlock();
1764f6f3c437SSimon Horman 	cond_resched();
1765f6f3c437SSimon Horman 	rcu_read_lock();
1766f6f3c437SSimon Horman #endif
1767f6f3c437SSimon Horman }
1768f6f3c437SSimon Horman 
17691da177e4SLinus Torvalds /*
17701da177e4SLinus Torvalds  * Does a critical section need to be broken due to another
177195c354feSNick Piggin  * task waiting?: (technically does not depend on CONFIG_PREEMPT,
177295c354feSNick Piggin  * but a general need for low latency)
17731da177e4SLinus Torvalds  */
177495c354feSNick Piggin static inline int spin_needbreak(spinlock_t *lock)
17751da177e4SLinus Torvalds {
177695c354feSNick Piggin #ifdef CONFIG_PREEMPT
177795c354feSNick Piggin 	return spin_is_contended(lock);
177895c354feSNick Piggin #else
17791da177e4SLinus Torvalds 	return 0;
178095c354feSNick Piggin #endif
17811da177e4SLinus Torvalds }
17821da177e4SLinus Torvalds 
178375f93fedSPeter Zijlstra static __always_inline bool need_resched(void)
178475f93fedSPeter Zijlstra {
178575f93fedSPeter Zijlstra 	return unlikely(tif_need_resched());
178675f93fedSPeter Zijlstra }
178775f93fedSPeter Zijlstra 
1788ee761f62SThomas Gleixner /*
17891da177e4SLinus Torvalds  * Wrappers for p->thread_info->cpu access. No-op on UP.
17901da177e4SLinus Torvalds  */
17911da177e4SLinus Torvalds #ifdef CONFIG_SMP
17921da177e4SLinus Torvalds 
17931da177e4SLinus Torvalds static inline unsigned int task_cpu(const struct task_struct *p)
17941da177e4SLinus Torvalds {
1795c65eacbeSAndy Lutomirski #ifdef CONFIG_THREAD_INFO_IN_TASK
1796c546951dSAndrea Parri 	return READ_ONCE(p->cpu);
1797c65eacbeSAndy Lutomirski #else
1798c546951dSAndrea Parri 	return READ_ONCE(task_thread_info(p)->cpu);
1799c65eacbeSAndy Lutomirski #endif
18001da177e4SLinus Torvalds }
18011da177e4SLinus Torvalds 
1802c65cc870SIngo Molnar extern void set_task_cpu(struct task_struct *p, unsigned int cpu);
18031da177e4SLinus Torvalds 
18041da177e4SLinus Torvalds #else
18051da177e4SLinus Torvalds 
18061da177e4SLinus Torvalds static inline unsigned int task_cpu(const struct task_struct *p)
18071da177e4SLinus Torvalds {
18081da177e4SLinus Torvalds 	return 0;
18091da177e4SLinus Torvalds }
18101da177e4SLinus Torvalds 
18111da177e4SLinus Torvalds static inline void set_task_cpu(struct task_struct *p, unsigned int cpu)
18121da177e4SLinus Torvalds {
18131da177e4SLinus Torvalds }
18141da177e4SLinus Torvalds 
18151da177e4SLinus Torvalds #endif /* CONFIG_SMP */
18161da177e4SLinus Torvalds 
1817d9345c65SPan Xinhui /*
1818d9345c65SPan Xinhui  * In order to reduce various lock holder preemption latencies provide an
1819d9345c65SPan Xinhui  * interface to see if a vCPU is currently running or not.
1820d9345c65SPan Xinhui  *
1821d9345c65SPan Xinhui  * This allows us to terminate optimistic spin loops and block, analogous to
1822d9345c65SPan Xinhui  * the native optimistic spin heuristic of testing if the lock owner task is
1823d9345c65SPan Xinhui  * running or not.
1824d9345c65SPan Xinhui  */
1825d9345c65SPan Xinhui #ifndef vcpu_is_preempted
1826d9345c65SPan Xinhui # define vcpu_is_preempted(cpu)	false
1827d9345c65SPan Xinhui #endif
1828d9345c65SPan Xinhui 
182996f874e2SRusty Russell extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask);
183096f874e2SRusty Russell extern long sched_getaffinity(pid_t pid, struct cpumask *mask);
18315c45bf27SSiddha, Suresh B 
183282455257SDave Hansen #ifndef TASK_SIZE_OF
183382455257SDave Hansen #define TASK_SIZE_OF(tsk)	TASK_SIZE
183482455257SDave Hansen #endif
183582455257SDave Hansen 
1836d7822b1eSMathieu Desnoyers #ifdef CONFIG_RSEQ
1837d7822b1eSMathieu Desnoyers 
1838d7822b1eSMathieu Desnoyers /*
1839d7822b1eSMathieu Desnoyers  * Map the event mask on the user-space ABI enum rseq_cs_flags
1840d7822b1eSMathieu Desnoyers  * for direct mask checks.
1841d7822b1eSMathieu Desnoyers  */
1842d7822b1eSMathieu Desnoyers enum rseq_event_mask_bits {
1843d7822b1eSMathieu Desnoyers 	RSEQ_EVENT_PREEMPT_BIT	= RSEQ_CS_FLAG_NO_RESTART_ON_PREEMPT_BIT,
1844d7822b1eSMathieu Desnoyers 	RSEQ_EVENT_SIGNAL_BIT	= RSEQ_CS_FLAG_NO_RESTART_ON_SIGNAL_BIT,
1845d7822b1eSMathieu Desnoyers 	RSEQ_EVENT_MIGRATE_BIT	= RSEQ_CS_FLAG_NO_RESTART_ON_MIGRATE_BIT,
1846d7822b1eSMathieu Desnoyers };
1847d7822b1eSMathieu Desnoyers 
1848d7822b1eSMathieu Desnoyers enum rseq_event_mask {
1849d7822b1eSMathieu Desnoyers 	RSEQ_EVENT_PREEMPT	= (1U << RSEQ_EVENT_PREEMPT_BIT),
1850d7822b1eSMathieu Desnoyers 	RSEQ_EVENT_SIGNAL	= (1U << RSEQ_EVENT_SIGNAL_BIT),
1851d7822b1eSMathieu Desnoyers 	RSEQ_EVENT_MIGRATE	= (1U << RSEQ_EVENT_MIGRATE_BIT),
1852d7822b1eSMathieu Desnoyers };
1853d7822b1eSMathieu Desnoyers 
1854d7822b1eSMathieu Desnoyers static inline void rseq_set_notify_resume(struct task_struct *t)
1855d7822b1eSMathieu Desnoyers {
1856d7822b1eSMathieu Desnoyers 	if (t->rseq)
1857d7822b1eSMathieu Desnoyers 		set_tsk_thread_flag(t, TIF_NOTIFY_RESUME);
1858d7822b1eSMathieu Desnoyers }
1859d7822b1eSMathieu Desnoyers 
1860784e0300SWill Deacon void __rseq_handle_notify_resume(struct ksignal *sig, struct pt_regs *regs);
1861d7822b1eSMathieu Desnoyers 
1862784e0300SWill Deacon static inline void rseq_handle_notify_resume(struct ksignal *ksig,
1863784e0300SWill Deacon 					     struct pt_regs *regs)
1864d7822b1eSMathieu Desnoyers {
1865d7822b1eSMathieu Desnoyers 	if (current->rseq)
1866784e0300SWill Deacon 		__rseq_handle_notify_resume(ksig, regs);
1867d7822b1eSMathieu Desnoyers }
1868d7822b1eSMathieu Desnoyers 
1869784e0300SWill Deacon static inline void rseq_signal_deliver(struct ksignal *ksig,
1870784e0300SWill Deacon 				       struct pt_regs *regs)
1871d7822b1eSMathieu Desnoyers {
1872d7822b1eSMathieu Desnoyers 	preempt_disable();
1873d7822b1eSMathieu Desnoyers 	__set_bit(RSEQ_EVENT_SIGNAL_BIT, &current->rseq_event_mask);
1874d7822b1eSMathieu Desnoyers 	preempt_enable();
1875784e0300SWill Deacon 	rseq_handle_notify_resume(ksig, regs);
1876d7822b1eSMathieu Desnoyers }
1877d7822b1eSMathieu Desnoyers 
1878d7822b1eSMathieu Desnoyers /* rseq_preempt() requires preemption to be disabled. */
1879d7822b1eSMathieu Desnoyers static inline void rseq_preempt(struct task_struct *t)
1880d7822b1eSMathieu Desnoyers {
1881d7822b1eSMathieu Desnoyers 	__set_bit(RSEQ_EVENT_PREEMPT_BIT, &t->rseq_event_mask);
1882d7822b1eSMathieu Desnoyers 	rseq_set_notify_resume(t);
1883d7822b1eSMathieu Desnoyers }
1884d7822b1eSMathieu Desnoyers 
1885d7822b1eSMathieu Desnoyers /* rseq_migrate() requires preemption to be disabled. */
1886d7822b1eSMathieu Desnoyers static inline void rseq_migrate(struct task_struct *t)
1887d7822b1eSMathieu Desnoyers {
1888d7822b1eSMathieu Desnoyers 	__set_bit(RSEQ_EVENT_MIGRATE_BIT, &t->rseq_event_mask);
1889d7822b1eSMathieu Desnoyers 	rseq_set_notify_resume(t);
1890d7822b1eSMathieu Desnoyers }
1891d7822b1eSMathieu Desnoyers 
1892d7822b1eSMathieu Desnoyers /*
1893d7822b1eSMathieu Desnoyers  * If parent process has a registered restartable sequences area, the
18949a789fcfSMathieu Desnoyers  * child inherits. Only applies when forking a process, not a thread.
1895d7822b1eSMathieu Desnoyers  */
1896d7822b1eSMathieu Desnoyers static inline void rseq_fork(struct task_struct *t, unsigned long clone_flags)
1897d7822b1eSMathieu Desnoyers {
1898d7822b1eSMathieu Desnoyers 	if (clone_flags & CLONE_THREAD) {
1899d7822b1eSMathieu Desnoyers 		t->rseq = NULL;
1900d7822b1eSMathieu Desnoyers 		t->rseq_sig = 0;
1901d7822b1eSMathieu Desnoyers 		t->rseq_event_mask = 0;
1902d7822b1eSMathieu Desnoyers 	} else {
1903d7822b1eSMathieu Desnoyers 		t->rseq = current->rseq;
1904d7822b1eSMathieu Desnoyers 		t->rseq_sig = current->rseq_sig;
1905d7822b1eSMathieu Desnoyers 		t->rseq_event_mask = current->rseq_event_mask;
1906d7822b1eSMathieu Desnoyers 	}
1907d7822b1eSMathieu Desnoyers }
1908d7822b1eSMathieu Desnoyers 
1909d7822b1eSMathieu Desnoyers static inline void rseq_execve(struct task_struct *t)
1910d7822b1eSMathieu Desnoyers {
1911d7822b1eSMathieu Desnoyers 	t->rseq = NULL;
1912d7822b1eSMathieu Desnoyers 	t->rseq_sig = 0;
1913d7822b1eSMathieu Desnoyers 	t->rseq_event_mask = 0;
1914d7822b1eSMathieu Desnoyers }
1915d7822b1eSMathieu Desnoyers 
1916d7822b1eSMathieu Desnoyers #else
1917d7822b1eSMathieu Desnoyers 
1918d7822b1eSMathieu Desnoyers static inline void rseq_set_notify_resume(struct task_struct *t)
1919d7822b1eSMathieu Desnoyers {
1920d7822b1eSMathieu Desnoyers }
1921784e0300SWill Deacon static inline void rseq_handle_notify_resume(struct ksignal *ksig,
1922784e0300SWill Deacon 					     struct pt_regs *regs)
1923d7822b1eSMathieu Desnoyers {
1924d7822b1eSMathieu Desnoyers }
1925784e0300SWill Deacon static inline void rseq_signal_deliver(struct ksignal *ksig,
1926784e0300SWill Deacon 				       struct pt_regs *regs)
1927d7822b1eSMathieu Desnoyers {
1928d7822b1eSMathieu Desnoyers }
1929d7822b1eSMathieu Desnoyers static inline void rseq_preempt(struct task_struct *t)
1930d7822b1eSMathieu Desnoyers {
1931d7822b1eSMathieu Desnoyers }
1932d7822b1eSMathieu Desnoyers static inline void rseq_migrate(struct task_struct *t)
1933d7822b1eSMathieu Desnoyers {
1934d7822b1eSMathieu Desnoyers }
1935d7822b1eSMathieu Desnoyers static inline void rseq_fork(struct task_struct *t, unsigned long clone_flags)
1936d7822b1eSMathieu Desnoyers {
1937d7822b1eSMathieu Desnoyers }
1938d7822b1eSMathieu Desnoyers static inline void rseq_execve(struct task_struct *t)
1939d7822b1eSMathieu Desnoyers {
1940d7822b1eSMathieu Desnoyers }
1941d7822b1eSMathieu Desnoyers 
1942d7822b1eSMathieu Desnoyers #endif
1943d7822b1eSMathieu Desnoyers 
194473ab1cb2STaehee Yoo void __exit_umh(struct task_struct *tsk);
194573ab1cb2STaehee Yoo 
194673ab1cb2STaehee Yoo static inline void exit_umh(struct task_struct *tsk)
194773ab1cb2STaehee Yoo {
194873ab1cb2STaehee Yoo 	if (unlikely(tsk->flags & PF_UMH))
194973ab1cb2STaehee Yoo 		__exit_umh(tsk);
195073ab1cb2STaehee Yoo }
195173ab1cb2STaehee Yoo 
1952d7822b1eSMathieu Desnoyers #ifdef CONFIG_DEBUG_RSEQ
1953d7822b1eSMathieu Desnoyers 
1954d7822b1eSMathieu Desnoyers void rseq_syscall(struct pt_regs *regs);
1955d7822b1eSMathieu Desnoyers 
1956d7822b1eSMathieu Desnoyers #else
1957d7822b1eSMathieu Desnoyers 
1958d7822b1eSMathieu Desnoyers static inline void rseq_syscall(struct pt_regs *regs)
1959d7822b1eSMathieu Desnoyers {
1960d7822b1eSMathieu Desnoyers }
1961d7822b1eSMathieu Desnoyers 
1962d7822b1eSMathieu Desnoyers #endif
1963d7822b1eSMathieu Desnoyers 
19643c93a0c0SQais Yousef const struct sched_avg *sched_trace_cfs_rq_avg(struct cfs_rq *cfs_rq);
19653c93a0c0SQais Yousef char *sched_trace_cfs_rq_path(struct cfs_rq *cfs_rq, char *str, int len);
19663c93a0c0SQais Yousef int sched_trace_cfs_rq_cpu(struct cfs_rq *cfs_rq);
19673c93a0c0SQais Yousef 
19683c93a0c0SQais Yousef const struct sched_avg *sched_trace_rq_avg_rt(struct rq *rq);
19693c93a0c0SQais Yousef const struct sched_avg *sched_trace_rq_avg_dl(struct rq *rq);
19703c93a0c0SQais Yousef const struct sched_avg *sched_trace_rq_avg_irq(struct rq *rq);
19713c93a0c0SQais Yousef 
19723c93a0c0SQais Yousef int sched_trace_rq_cpu(struct rq *rq);
19733c93a0c0SQais Yousef 
19743c93a0c0SQais Yousef const struct cpumask *sched_trace_rd_span(struct root_domain *rd);
19753c93a0c0SQais Yousef 
19761da177e4SLinus Torvalds #endif
1977