xref: /linux/include/linux/sched.h (revision eb61baf69871b9836783a81bc451189edb0d9de2)
11da177e4SLinus Torvalds #ifndef _LINUX_SCHED_H
21da177e4SLinus Torvalds #define _LINUX_SCHED_H
31da177e4SLinus Torvalds 
4607ca46eSDavid Howells #include <uapi/linux/sched.h>
5b7b3c76aSDavid Woodhouse 
65c228079SDongsheng Yang #include <linux/sched/prio.h>
75c228079SDongsheng Yang 
8b7b3c76aSDavid Woodhouse 
9b7b3c76aSDavid Woodhouse struct sched_param {
10b7b3c76aSDavid Woodhouse 	int sched_priority;
11b7b3c76aSDavid Woodhouse };
12b7b3c76aSDavid Woodhouse 
131da177e4SLinus Torvalds #include <asm/param.h>	/* for HZ */
141da177e4SLinus Torvalds 
151da177e4SLinus Torvalds #include <linux/capability.h>
161da177e4SLinus Torvalds #include <linux/threads.h>
171da177e4SLinus Torvalds #include <linux/kernel.h>
181da177e4SLinus Torvalds #include <linux/types.h>
191da177e4SLinus Torvalds #include <linux/timex.h>
201da177e4SLinus Torvalds #include <linux/jiffies.h>
21b69339baSIngo Molnar #include <linux/mutex.h>
22fb00aca4SPeter Zijlstra #include <linux/plist.h>
231da177e4SLinus Torvalds #include <linux/rbtree.h>
241da177e4SLinus Torvalds #include <linux/thread_info.h>
251da177e4SLinus Torvalds #include <linux/cpumask.h>
261da177e4SLinus Torvalds #include <linux/errno.h>
271da177e4SLinus Torvalds #include <linux/nodemask.h>
28c92ff1bdSMartin Schwidefsky #include <linux/mm_types.h>
2992cf2118SFrederic Weisbecker #include <linux/preempt.h>
301da177e4SLinus Torvalds 
311da177e4SLinus Torvalds #include <asm/page.h>
321da177e4SLinus Torvalds #include <asm/ptrace.h>
331da177e4SLinus Torvalds 
341da177e4SLinus Torvalds #include <linux/smp.h>
351da177e4SLinus Torvalds #include <linux/sem.h>
36ab602f79SJack Miller #include <linux/shm.h>
371da177e4SLinus Torvalds #include <linux/signal.h>
381da177e4SLinus Torvalds #include <linux/compiler.h>
391da177e4SLinus Torvalds #include <linux/completion.h>
40f361bf4aSIngo Molnar #include <linux/signal_types.h>
411da177e4SLinus Torvalds #include <linux/pid.h>
421da177e4SLinus Torvalds #include <linux/percpu.h>
431da177e4SLinus Torvalds #include <linux/topology.h>
441da177e4SLinus Torvalds #include <linux/seccomp.h>
45e56d0903SIngo Molnar #include <linux/rcupdate.h>
4605725f7eSJiri Pirko #include <linux/rculist.h>
4723f78d4aSIngo Molnar #include <linux/rtmutex.h>
481da177e4SLinus Torvalds 
49a3b6714eSDavid Woodhouse #include <linux/time.h>
50a3b6714eSDavid Woodhouse #include <linux/param.h>
51a3b6714eSDavid Woodhouse #include <linux/resource.h>
52a3b6714eSDavid Woodhouse #include <linux/timer.h>
53a3b6714eSDavid Woodhouse #include <linux/hrtimer.h>
545c9a8750SDmitry Vyukov #include <linux/kcov.h>
557c3ab738SAndrew Morton #include <linux/task_io_accounting.h>
569745512cSArjan van de Ven #include <linux/latencytop.h>
579e2b2dc4SDavid Howells #include <linux/cred.h>
58fa14ff4aSPeter Zijlstra #include <linux/llist.h>
597b44ab97SEric W. Biederman #include <linux/uidgid.h>
6021caf2fcSMing Lei #include <linux/gfp.h>
61fd771233SIngo Molnar #include <linux/topology.h>
62d4311ff1SAaron Tomlin #include <linux/magic.h>
637d7efec3STejun Heo #include <linux/cgroup-defs.h>
64a3b6714eSDavid Woodhouse 
65a3b6714eSDavid Woodhouse #include <asm/processor.h>
6636d57ac4SH. J. Lu 
67d50dde5aSDario Faggioli #define SCHED_ATTR_SIZE_VER0	48	/* sizeof first published struct */
68d50dde5aSDario Faggioli 
69d50dde5aSDario Faggioli /*
70d50dde5aSDario Faggioli  * Extended scheduling parameters data structure.
71d50dde5aSDario Faggioli  *
72d50dde5aSDario Faggioli  * This is needed because the original struct sched_param can not be
73d50dde5aSDario Faggioli  * altered without introducing ABI issues with legacy applications
74d50dde5aSDario Faggioli  * (e.g., in sched_getparam()).
75d50dde5aSDario Faggioli  *
76d50dde5aSDario Faggioli  * However, the possibility of specifying more than just a priority for
77d50dde5aSDario Faggioli  * the tasks may be useful for a wide variety of application fields, e.g.,
78d50dde5aSDario Faggioli  * multimedia, streaming, automation and control, and many others.
79d50dde5aSDario Faggioli  *
80d50dde5aSDario Faggioli  * This variant (sched_attr) is meant at describing a so-called
81d50dde5aSDario Faggioli  * sporadic time-constrained task. In such model a task is specified by:
82d50dde5aSDario Faggioli  *  - the activation period or minimum instance inter-arrival time;
83d50dde5aSDario Faggioli  *  - the maximum (or average, depending on the actual scheduling
84d50dde5aSDario Faggioli  *    discipline) computation time of all instances, a.k.a. runtime;
85d50dde5aSDario Faggioli  *  - the deadline (relative to the actual activation time) of each
86d50dde5aSDario Faggioli  *    instance.
87d50dde5aSDario Faggioli  * Very briefly, a periodic (sporadic) task asks for the execution of
88d50dde5aSDario Faggioli  * some specific computation --which is typically called an instance--
89d50dde5aSDario Faggioli  * (at most) every period. Moreover, each instance typically lasts no more
90d50dde5aSDario Faggioli  * than the runtime and must be completed by time instant t equal to
91d50dde5aSDario Faggioli  * the instance activation time + the deadline.
92d50dde5aSDario Faggioli  *
93d50dde5aSDario Faggioli  * This is reflected by the actual fields of the sched_attr structure:
94d50dde5aSDario Faggioli  *
95d50dde5aSDario Faggioli  *  @size		size of the structure, for fwd/bwd compat.
96d50dde5aSDario Faggioli  *
97d50dde5aSDario Faggioli  *  @sched_policy	task's scheduling policy
98d50dde5aSDario Faggioli  *  @sched_flags	for customizing the scheduler behaviour
99d50dde5aSDario Faggioli  *  @sched_nice		task's nice value      (SCHED_NORMAL/BATCH)
100d50dde5aSDario Faggioli  *  @sched_priority	task's static priority (SCHED_FIFO/RR)
101d50dde5aSDario Faggioli  *  @sched_deadline	representative of the task's deadline
102d50dde5aSDario Faggioli  *  @sched_runtime	representative of the task's runtime
103d50dde5aSDario Faggioli  *  @sched_period	representative of the task's period
104d50dde5aSDario Faggioli  *
105d50dde5aSDario Faggioli  * Given this task model, there are a multiplicity of scheduling algorithms
106d50dde5aSDario Faggioli  * and policies, that can be used to ensure all the tasks will make their
107d50dde5aSDario Faggioli  * timing constraints.
108aab03e05SDario Faggioli  *
109aab03e05SDario Faggioli  * As of now, the SCHED_DEADLINE policy (sched_dl scheduling class) is the
110aab03e05SDario Faggioli  * only user of this new interface. More information about the algorithm
111aab03e05SDario Faggioli  * available in the scheduling class file or in Documentation/.
112d50dde5aSDario Faggioli  */
113d50dde5aSDario Faggioli struct sched_attr {
114d50dde5aSDario Faggioli 	u32 size;
115d50dde5aSDario Faggioli 
116d50dde5aSDario Faggioli 	u32 sched_policy;
117d50dde5aSDario Faggioli 	u64 sched_flags;
118d50dde5aSDario Faggioli 
119d50dde5aSDario Faggioli 	/* SCHED_NORMAL, SCHED_BATCH */
120d50dde5aSDario Faggioli 	s32 sched_nice;
121d50dde5aSDario Faggioli 
122d50dde5aSDario Faggioli 	/* SCHED_FIFO, SCHED_RR */
123d50dde5aSDario Faggioli 	u32 sched_priority;
124d50dde5aSDario Faggioli 
125d50dde5aSDario Faggioli 	/* SCHED_DEADLINE */
126d50dde5aSDario Faggioli 	u64 sched_runtime;
127d50dde5aSDario Faggioli 	u64 sched_deadline;
128d50dde5aSDario Faggioli 	u64 sched_period;
129d50dde5aSDario Faggioli };
130d50dde5aSDario Faggioli 
131c87e2837SIngo Molnar struct futex_pi_state;
132286100a6SAlexey Dobriyan struct robust_list_head;
133bddd87c7SAkinobu Mita struct bio_list;
1345ad4e53bSAl Viro struct fs_struct;
135cdd6c482SIngo Molnar struct perf_event_context;
13673c10101SJens Axboe struct blk_plug;
137c4ad8f98SLinus Torvalds struct filename;
13889076bc3SAl Viro struct nameidata;
1391da177e4SLinus Torvalds 
1401da177e4SLinus Torvalds /*
1411da177e4SLinus Torvalds  * These are the constant used to fake the fixed-point load-average
1421da177e4SLinus Torvalds  * counting. Some notes:
1431da177e4SLinus Torvalds  *  - 11 bit fractions expand to 22 bits by the multiplies: this gives
1441da177e4SLinus Torvalds  *    a load-average precision of 10 bits integer + 11 bits fractional
1451da177e4SLinus Torvalds  *  - if you want to count load-averages more often, you need more
1461da177e4SLinus Torvalds  *    precision, or rounding will get you. With 2-second counting freq,
1471da177e4SLinus Torvalds  *    the EXP_n values would be 1981, 2034 and 2043 if still using only
1481da177e4SLinus Torvalds  *    11 bit fractions.
1491da177e4SLinus Torvalds  */
1501da177e4SLinus Torvalds extern unsigned long avenrun[];		/* Load averages */
1512d02494fSThomas Gleixner extern void get_avenrun(unsigned long *loads, unsigned long offset, int shift);
1521da177e4SLinus Torvalds 
1531da177e4SLinus Torvalds #define FSHIFT		11		/* nr of bits of precision */
1541da177e4SLinus Torvalds #define FIXED_1		(1<<FSHIFT)	/* 1.0 as fixed-point */
1550c2043abSLinus Torvalds #define LOAD_FREQ	(5*HZ+1)	/* 5 sec intervals */
1561da177e4SLinus Torvalds #define EXP_1		1884		/* 1/exp(5sec/1min) as fixed-point */
1571da177e4SLinus Torvalds #define EXP_5		2014		/* 1/exp(5sec/5min) */
1581da177e4SLinus Torvalds #define EXP_15		2037		/* 1/exp(5sec/15min) */
1591da177e4SLinus Torvalds 
1601da177e4SLinus Torvalds #define CALC_LOAD(load,exp,n) \
1611da177e4SLinus Torvalds 	load *= exp; \
1621da177e4SLinus Torvalds 	load += n*(FIXED_1-exp); \
1631da177e4SLinus Torvalds 	load >>= FSHIFT;
1641da177e4SLinus Torvalds 
1651da177e4SLinus Torvalds extern unsigned long total_forks;
1661da177e4SLinus Torvalds extern int nr_threads;
1671da177e4SLinus Torvalds DECLARE_PER_CPU(unsigned long, process_counts);
1681da177e4SLinus Torvalds extern int nr_processes(void);
1691da177e4SLinus Torvalds extern unsigned long nr_running(void);
1702ee507c4STim Chen extern bool single_task_running(void);
1711da177e4SLinus Torvalds extern unsigned long nr_iowait(void);
1728c215bd3SPeter Zijlstra extern unsigned long nr_iowait_cpu(int cpu);
173372ba8cbSMel Gorman extern void get_iowait_load(unsigned long *nr_waiters, unsigned long *load);
17469d25870SArjan van de Ven 
1750f004f5aSPeter Zijlstra extern void calc_global_load(unsigned long ticks);
1763289bdb4SPeter Zijlstra 
1773289bdb4SPeter Zijlstra #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
1781f41906aSFrederic Weisbecker extern void cpu_load_update_nohz_start(void);
1791f41906aSFrederic Weisbecker extern void cpu_load_update_nohz_stop(void);
1803289bdb4SPeter Zijlstra #else
1811f41906aSFrederic Weisbecker static inline void cpu_load_update_nohz_start(void) { }
1821f41906aSFrederic Weisbecker static inline void cpu_load_update_nohz_stop(void) { }
1833289bdb4SPeter Zijlstra #endif
1841da177e4SLinus Torvalds 
185b637a328SPaul E. McKenney extern void dump_cpu_task(int cpu);
186b637a328SPaul E. McKenney 
18743ae34cbSIngo Molnar struct seq_file;
18843ae34cbSIngo Molnar struct cfs_rq;
1894cf86d77SIngo Molnar struct task_group;
19043ae34cbSIngo Molnar #ifdef CONFIG_SCHED_DEBUG
19143ae34cbSIngo Molnar extern void proc_sched_show_task(struct task_struct *p, struct seq_file *m);
19243ae34cbSIngo Molnar extern void proc_sched_set_task(struct task_struct *p);
19343ae34cbSIngo Molnar #endif
1941da177e4SLinus Torvalds 
1954a8342d2SLinus Torvalds /*
1964a8342d2SLinus Torvalds  * Task state bitmask. NOTE! These bits are also
1974a8342d2SLinus Torvalds  * encoded in fs/proc/array.c: get_task_state().
1984a8342d2SLinus Torvalds  *
1994a8342d2SLinus Torvalds  * We have two separate sets of flags: task->state
2004a8342d2SLinus Torvalds  * is about runnability, while task->exit_state are
2014a8342d2SLinus Torvalds  * about the task exiting. Confusing, but this way
2024a8342d2SLinus Torvalds  * modifying one set can't modify the other one by
2034a8342d2SLinus Torvalds  * mistake.
2044a8342d2SLinus Torvalds  */
2051da177e4SLinus Torvalds #define TASK_RUNNING		0
2061da177e4SLinus Torvalds #define TASK_INTERRUPTIBLE	1
2071da177e4SLinus Torvalds #define TASK_UNINTERRUPTIBLE	2
208f021a3c2SMatthew Wilcox #define __TASK_STOPPED		4
209f021a3c2SMatthew Wilcox #define __TASK_TRACED		8
2104a8342d2SLinus Torvalds /* in tsk->exit_state */
211ad86622bSOleg Nesterov #define EXIT_DEAD		16
212ad86622bSOleg Nesterov #define EXIT_ZOMBIE		32
213abd50b39SOleg Nesterov #define EXIT_TRACE		(EXIT_ZOMBIE | EXIT_DEAD)
2144a8342d2SLinus Torvalds /* in tsk->state again */
215af927232SMike Galbraith #define TASK_DEAD		64
216f021a3c2SMatthew Wilcox #define TASK_WAKEKILL		128
217e9c84311SPeter Zijlstra #define TASK_WAKING		256
218f2530dc7SThomas Gleixner #define TASK_PARKED		512
21980ed87c8SPeter Zijlstra #define TASK_NOLOAD		1024
2207dc603c9SPeter Zijlstra #define TASK_NEW		2048
2217dc603c9SPeter Zijlstra #define TASK_STATE_MAX		4096
222f021a3c2SMatthew Wilcox 
2237dc603c9SPeter Zijlstra #define TASK_STATE_TO_CHAR_STR "RSDTtXZxKWPNn"
22473342151SPeter Zijlstra 
225642fa448SDavidlohr Bueso /* Convenience macros for the sake of set_current_state */
226f021a3c2SMatthew Wilcox #define TASK_KILLABLE		(TASK_WAKEKILL | TASK_UNINTERRUPTIBLE)
227f021a3c2SMatthew Wilcox #define TASK_STOPPED		(TASK_WAKEKILL | __TASK_STOPPED)
228f021a3c2SMatthew Wilcox #define TASK_TRACED		(TASK_WAKEKILL | __TASK_TRACED)
2291da177e4SLinus Torvalds 
23080ed87c8SPeter Zijlstra #define TASK_IDLE		(TASK_UNINTERRUPTIBLE | TASK_NOLOAD)
23180ed87c8SPeter Zijlstra 
23292a1f4bcSMatthew Wilcox /* Convenience macros for the sake of wake_up */
23392a1f4bcSMatthew Wilcox #define TASK_NORMAL		(TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE)
234f021a3c2SMatthew Wilcox #define TASK_ALL		(TASK_NORMAL | __TASK_STOPPED | __TASK_TRACED)
23592a1f4bcSMatthew Wilcox 
23692a1f4bcSMatthew Wilcox /* get_task_state() */
23792a1f4bcSMatthew Wilcox #define TASK_REPORT		(TASK_RUNNING | TASK_INTERRUPTIBLE | \
238f021a3c2SMatthew Wilcox 				 TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \
23974e37200SOleg Nesterov 				 __TASK_TRACED | EXIT_ZOMBIE | EXIT_DEAD)
24092a1f4bcSMatthew Wilcox 
241f021a3c2SMatthew Wilcox #define task_is_traced(task)	((task->state & __TASK_TRACED) != 0)
242f021a3c2SMatthew Wilcox #define task_is_stopped(task)	((task->state & __TASK_STOPPED) != 0)
24392a1f4bcSMatthew Wilcox #define task_is_stopped_or_traced(task)	\
244f021a3c2SMatthew Wilcox 			((task->state & (__TASK_STOPPED | __TASK_TRACED)) != 0)
24592a1f4bcSMatthew Wilcox #define task_contributes_to_load(task)	\
246e3c8ca83SNathan Lynch 				((task->state & TASK_UNINTERRUPTIBLE) != 0 && \
24780ed87c8SPeter Zijlstra 				 (task->flags & PF_FROZEN) == 0 && \
24880ed87c8SPeter Zijlstra 				 (task->state & TASK_NOLOAD) == 0)
2491da177e4SLinus Torvalds 
2508eb23b9fSPeter Zijlstra #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
2518eb23b9fSPeter Zijlstra 
2528eb23b9fSPeter Zijlstra #define __set_current_state(state_value)			\
2538eb23b9fSPeter Zijlstra 	do {							\
2548eb23b9fSPeter Zijlstra 		current->task_state_change = _THIS_IP_;		\
2558eb23b9fSPeter Zijlstra 		current->state = (state_value);			\
2568eb23b9fSPeter Zijlstra 	} while (0)
2578eb23b9fSPeter Zijlstra #define set_current_state(state_value)				\
2588eb23b9fSPeter Zijlstra 	do {							\
2598eb23b9fSPeter Zijlstra 		current->task_state_change = _THIS_IP_;		\
260b92b8b35SPeter Zijlstra 		smp_store_mb(current->state, (state_value));	\
2618eb23b9fSPeter Zijlstra 	} while (0)
2628eb23b9fSPeter Zijlstra 
2638eb23b9fSPeter Zijlstra #else
264498d0c57SAndrew Morton /*
265498d0c57SAndrew Morton  * set_current_state() includes a barrier so that the write of current->state
266498d0c57SAndrew Morton  * is correctly serialised wrt the caller's subsequent test of whether to
267498d0c57SAndrew Morton  * actually sleep:
268498d0c57SAndrew Morton  *
269a2250238SPeter Zijlstra  *   for (;;) {
270498d0c57SAndrew Morton  *	set_current_state(TASK_UNINTERRUPTIBLE);
271a2250238SPeter Zijlstra  *	if (!need_sleep)
272a2250238SPeter Zijlstra  *		break;
273498d0c57SAndrew Morton  *
274a2250238SPeter Zijlstra  *	schedule();
275a2250238SPeter Zijlstra  *   }
276a2250238SPeter Zijlstra  *   __set_current_state(TASK_RUNNING);
277a2250238SPeter Zijlstra  *
278a2250238SPeter Zijlstra  * If the caller does not need such serialisation (because, for instance, the
279a2250238SPeter Zijlstra  * condition test and condition change and wakeup are under the same lock) then
280a2250238SPeter Zijlstra  * use __set_current_state().
281a2250238SPeter Zijlstra  *
282a2250238SPeter Zijlstra  * The above is typically ordered against the wakeup, which does:
283a2250238SPeter Zijlstra  *
284a2250238SPeter Zijlstra  *	need_sleep = false;
285a2250238SPeter Zijlstra  *	wake_up_state(p, TASK_UNINTERRUPTIBLE);
286a2250238SPeter Zijlstra  *
287a2250238SPeter Zijlstra  * Where wake_up_state() (and all other wakeup primitives) imply enough
288a2250238SPeter Zijlstra  * barriers to order the store of the variable against wakeup.
289a2250238SPeter Zijlstra  *
290a2250238SPeter Zijlstra  * Wakeup will do: if (@state & p->state) p->state = TASK_RUNNING, that is,
291a2250238SPeter Zijlstra  * once it observes the TASK_UNINTERRUPTIBLE store the waking CPU can issue a
292a2250238SPeter Zijlstra  * TASK_RUNNING store which can collide with __set_current_state(TASK_RUNNING).
293a2250238SPeter Zijlstra  *
294a2250238SPeter Zijlstra  * This is obviously fine, since they both store the exact same value.
295a2250238SPeter Zijlstra  *
296a2250238SPeter Zijlstra  * Also see the comments of try_to_wake_up().
297498d0c57SAndrew Morton  */
2981da177e4SLinus Torvalds #define __set_current_state(state_value)		\
2991da177e4SLinus Torvalds 	do { current->state = (state_value); } while (0)
3001da177e4SLinus Torvalds #define set_current_state(state_value)			\
301b92b8b35SPeter Zijlstra 	smp_store_mb(current->state, (state_value))
3021da177e4SLinus Torvalds 
3038eb23b9fSPeter Zijlstra #endif
3048eb23b9fSPeter Zijlstra 
3051da177e4SLinus Torvalds /* Task command name length */
3061da177e4SLinus Torvalds #define TASK_COMM_LEN 16
3071da177e4SLinus Torvalds 
3081da177e4SLinus Torvalds #include <linux/spinlock.h>
3091da177e4SLinus Torvalds 
3101da177e4SLinus Torvalds /*
3111da177e4SLinus Torvalds  * This serializes "schedule()" and also protects
3121da177e4SLinus Torvalds  * the run-queue from deletions/modifications (but
3131da177e4SLinus Torvalds  * _adding_ to the beginning of the run-queue has
3141da177e4SLinus Torvalds  * a separate lock).
3151da177e4SLinus Torvalds  */
3161da177e4SLinus Torvalds extern rwlock_t tasklist_lock;
3171da177e4SLinus Torvalds extern spinlock_t mmlist_lock;
3181da177e4SLinus Torvalds 
31936c8b586SIngo Molnar struct task_struct;
3201da177e4SLinus Torvalds 
321db1466b3SPaul E. McKenney #ifdef CONFIG_PROVE_RCU
322db1466b3SPaul E. McKenney extern int lockdep_tasklist_lock_is_held(void);
323db1466b3SPaul E. McKenney #endif /* #ifdef CONFIG_PROVE_RCU */
324db1466b3SPaul E. McKenney 
3251da177e4SLinus Torvalds extern void sched_init(void);
3261da177e4SLinus Torvalds extern void sched_init_smp(void);
3272d07b255SHarvey Harrison extern asmlinkage void schedule_tail(struct task_struct *prev);
32836c8b586SIngo Molnar extern void init_idle(struct task_struct *idle, int cpu);
3291df21055SIngo Molnar extern void init_idle_bootup_task(struct task_struct *idle);
3301da177e4SLinus Torvalds 
3313fa0818bSRik van Riel extern cpumask_var_t cpu_isolated_map;
3323fa0818bSRik van Riel 
33389f19f04SAndrew Morton extern int runqueue_is_locked(int cpu);
334017730c1SIngo Molnar 
3353451d024SFrederic Weisbecker #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
336c1cc017cSAlex Shi extern void nohz_balance_enter_idle(int cpu);
33769e1e811SSuresh Siddha extern void set_cpu_sd_state_idle(void);
338bc7a34b8SThomas Gleixner extern int get_nohz_timer_target(void);
33946cb4b7cSSiddha, Suresh B #else
340c1cc017cSAlex Shi static inline void nohz_balance_enter_idle(int cpu) { }
341fdaabd80SPeter Zijlstra static inline void set_cpu_sd_state_idle(void) { }
34246cb4b7cSSiddha, Suresh B #endif
3431da177e4SLinus Torvalds 
344e59e2ae2SIngo Molnar /*
34539bc89fdSIngo Molnar  * Only dump TASK_* tasks. (0 for all tasks)
346e59e2ae2SIngo Molnar  */
347e59e2ae2SIngo Molnar extern void show_state_filter(unsigned long state_filter);
348e59e2ae2SIngo Molnar 
349e59e2ae2SIngo Molnar static inline void show_state(void)
350e59e2ae2SIngo Molnar {
35139bc89fdSIngo Molnar 	show_state_filter(0);
352e59e2ae2SIngo Molnar }
353e59e2ae2SIngo Molnar 
3541da177e4SLinus Torvalds extern void show_regs(struct pt_regs *);
3551da177e4SLinus Torvalds 
3561da177e4SLinus Torvalds /*
3571da177e4SLinus Torvalds  * TASK is a pointer to the task whose backtrace we want to see (or NULL for current
3581da177e4SLinus Torvalds  * task), SP is the stack pointer of the first frame that should be shown in the back
3591da177e4SLinus Torvalds  * trace (or NULL if the entire call-chain of the task should be shown).
3601da177e4SLinus Torvalds  */
3611da177e4SLinus Torvalds extern void show_stack(struct task_struct *task, unsigned long *sp);
3621da177e4SLinus Torvalds 
3631da177e4SLinus Torvalds extern void cpu_init (void);
3641da177e4SLinus Torvalds extern void trap_init(void);
3651da177e4SLinus Torvalds extern void update_process_times(int user);
3661da177e4SLinus Torvalds extern void scheduler_tick(void);
3679cf7243dSThomas Gleixner extern int sched_cpu_starting(unsigned int cpu);
36840190a78SThomas Gleixner extern int sched_cpu_activate(unsigned int cpu);
36940190a78SThomas Gleixner extern int sched_cpu_deactivate(unsigned int cpu);
3701da177e4SLinus Torvalds 
371f2785ddbSThomas Gleixner #ifdef CONFIG_HOTPLUG_CPU
372f2785ddbSThomas Gleixner extern int sched_cpu_dying(unsigned int cpu);
373f2785ddbSThomas Gleixner #else
374f2785ddbSThomas Gleixner # define sched_cpu_dying	NULL
375f2785ddbSThomas Gleixner #endif
3761da177e4SLinus Torvalds 
37782a1fcb9SIngo Molnar extern void sched_show_task(struct task_struct *p);
37882a1fcb9SIngo Molnar 
37919cc36c0SFrederic Weisbecker #ifdef CONFIG_LOCKUP_DETECTOR
38003e0d461STejun Heo extern void touch_softlockup_watchdog_sched(void);
3818446f1d3SIngo Molnar extern void touch_softlockup_watchdog(void);
382d6ad3e28SJason Wessel extern void touch_softlockup_watchdog_sync(void);
38304c9167fSJeremy Fitzhardinge extern void touch_all_softlockup_watchdogs(void);
384332fbdbcSDon Zickus extern int proc_dowatchdog_thresh(struct ctl_table *table, int write,
3858d65af78SAlexey Dobriyan 				  void __user *buffer,
386baf48f65SMandeep Singh Baines 				  size_t *lenp, loff_t *ppos);
3879c44bc03SIngo Molnar extern unsigned int  softlockup_panic;
388ac1f5912SDon Zickus extern unsigned int  hardlockup_panic;
389004417a6SPeter Zijlstra void lockup_detector_init(void);
3908446f1d3SIngo Molnar #else
39103e0d461STejun Heo static inline void touch_softlockup_watchdog_sched(void)
39203e0d461STejun Heo {
39303e0d461STejun Heo }
3948446f1d3SIngo Molnar static inline void touch_softlockup_watchdog(void)
3958446f1d3SIngo Molnar {
3968446f1d3SIngo Molnar }
397d6ad3e28SJason Wessel static inline void touch_softlockup_watchdog_sync(void)
398d6ad3e28SJason Wessel {
399d6ad3e28SJason Wessel }
40004c9167fSJeremy Fitzhardinge static inline void touch_all_softlockup_watchdogs(void)
40104c9167fSJeremy Fitzhardinge {
40204c9167fSJeremy Fitzhardinge }
403004417a6SPeter Zijlstra static inline void lockup_detector_init(void)
404004417a6SPeter Zijlstra {
405004417a6SPeter Zijlstra }
4068446f1d3SIngo Molnar #endif
4078446f1d3SIngo Molnar 
4088b414521SMarcelo Tosatti #ifdef CONFIG_DETECT_HUNG_TASK
4098b414521SMarcelo Tosatti void reset_hung_task_detector(void);
4108b414521SMarcelo Tosatti #else
4118b414521SMarcelo Tosatti static inline void reset_hung_task_detector(void)
4128b414521SMarcelo Tosatti {
4138b414521SMarcelo Tosatti }
4148b414521SMarcelo Tosatti #endif
4158b414521SMarcelo Tosatti 
4161da177e4SLinus Torvalds /* Attach to any functions which should be ignored in wchan output. */
4171da177e4SLinus Torvalds #define __sched		__attribute__((__section__(".sched.text")))
418deaf2227SIngo Molnar 
419deaf2227SIngo Molnar /* Linker adds these: start and end of __sched functions */
420deaf2227SIngo Molnar extern char __sched_text_start[], __sched_text_end[];
421deaf2227SIngo Molnar 
4221da177e4SLinus Torvalds /* Is this address in the __sched functions? */
4231da177e4SLinus Torvalds extern int in_sched_functions(unsigned long addr);
4241da177e4SLinus Torvalds 
4251da177e4SLinus Torvalds #define	MAX_SCHEDULE_TIMEOUT	LONG_MAX
426b3c97528SHarvey Harrison extern signed long schedule_timeout(signed long timeout);
42764ed93a2SNishanth Aravamudan extern signed long schedule_timeout_interruptible(signed long timeout);
428294d5cc2SMatthew Wilcox extern signed long schedule_timeout_killable(signed long timeout);
42964ed93a2SNishanth Aravamudan extern signed long schedule_timeout_uninterruptible(signed long timeout);
43069b27bafSAndrew Morton extern signed long schedule_timeout_idle(signed long timeout);
4311da177e4SLinus Torvalds asmlinkage void schedule(void);
432c5491ea7SThomas Gleixner extern void schedule_preempt_disabled(void);
4331da177e4SLinus Torvalds 
43410ab5643STejun Heo extern int __must_check io_schedule_prepare(void);
43510ab5643STejun Heo extern void io_schedule_finish(int token);
4369cff8adeSNeilBrown extern long io_schedule_timeout(long timeout);
43710ab5643STejun Heo extern void io_schedule(void);
4389cff8adeSNeilBrown 
4399af6528eSPeter Zijlstra void __noreturn do_task_dead(void);
4409af6528eSPeter Zijlstra 
441ab516013SSerge E. Hallyn struct nsproxy;
442acce292cSCedric Le Goater struct user_namespace;
4431da177e4SLinus Torvalds 
444efc1a3b1SDavid Howells #ifdef CONFIG_MMU
445efc1a3b1SDavid Howells extern void arch_pick_mmap_layout(struct mm_struct *mm);
4461da177e4SLinus Torvalds extern unsigned long
4471da177e4SLinus Torvalds arch_get_unmapped_area(struct file *, unsigned long, unsigned long,
4481da177e4SLinus Torvalds 		       unsigned long, unsigned long);
4491da177e4SLinus Torvalds extern unsigned long
4501da177e4SLinus Torvalds arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr,
4511da177e4SLinus Torvalds 			  unsigned long len, unsigned long pgoff,
4521da177e4SLinus Torvalds 			  unsigned long flags);
453efc1a3b1SDavid Howells #else
454efc1a3b1SDavid Howells static inline void arch_pick_mmap_layout(struct mm_struct *mm) {}
455efc1a3b1SDavid Howells #endif
4561da177e4SLinus Torvalds 
457d049f74fSKees Cook #define SUID_DUMP_DISABLE	0	/* No setuid dumping */
458d049f74fSKees Cook #define SUID_DUMP_USER		1	/* Dump as user of process */
459d049f74fSKees Cook #define SUID_DUMP_ROOT		2	/* Dump as root */
460d049f74fSKees Cook 
4616c5d5238SKawai, Hidehiro /* mm flags */
462f8af4da3SHugh Dickins 
4637288e118SOleg Nesterov /* for SUID_DUMP_* above */
4643cb4a0bbSKawai, Hidehiro #define MMF_DUMPABLE_BITS 2
465f8af4da3SHugh Dickins #define MMF_DUMPABLE_MASK ((1 << MMF_DUMPABLE_BITS) - 1)
4663cb4a0bbSKawai, Hidehiro 
467942be387SOleg Nesterov extern void set_dumpable(struct mm_struct *mm, int value);
468942be387SOleg Nesterov /*
469942be387SOleg Nesterov  * This returns the actual value of the suid_dumpable flag. For things
470942be387SOleg Nesterov  * that are using this for checking for privilege transitions, it must
471942be387SOleg Nesterov  * test against SUID_DUMP_USER rather than treating it as a boolean
472942be387SOleg Nesterov  * value.
473942be387SOleg Nesterov  */
474942be387SOleg Nesterov static inline int __get_dumpable(unsigned long mm_flags)
475942be387SOleg Nesterov {
476942be387SOleg Nesterov 	return mm_flags & MMF_DUMPABLE_MASK;
477942be387SOleg Nesterov }
478942be387SOleg Nesterov 
479942be387SOleg Nesterov static inline int get_dumpable(struct mm_struct *mm)
480942be387SOleg Nesterov {
481942be387SOleg Nesterov 	return __get_dumpable(mm->flags);
482942be387SOleg Nesterov }
483942be387SOleg Nesterov 
4843cb4a0bbSKawai, Hidehiro /* coredump filter bits */
4853cb4a0bbSKawai, Hidehiro #define MMF_DUMP_ANON_PRIVATE	2
4863cb4a0bbSKawai, Hidehiro #define MMF_DUMP_ANON_SHARED	3
4873cb4a0bbSKawai, Hidehiro #define MMF_DUMP_MAPPED_PRIVATE	4
4883cb4a0bbSKawai, Hidehiro #define MMF_DUMP_MAPPED_SHARED	5
48982df3973SRoland McGrath #define MMF_DUMP_ELF_HEADERS	6
490e575f111SKOSAKI Motohiro #define MMF_DUMP_HUGETLB_PRIVATE 7
491e575f111SKOSAKI Motohiro #define MMF_DUMP_HUGETLB_SHARED  8
4925037835cSRoss Zwisler #define MMF_DUMP_DAX_PRIVATE	9
4935037835cSRoss Zwisler #define MMF_DUMP_DAX_SHARED	10
494f8af4da3SHugh Dickins 
4953cb4a0bbSKawai, Hidehiro #define MMF_DUMP_FILTER_SHIFT	MMF_DUMPABLE_BITS
4965037835cSRoss Zwisler #define MMF_DUMP_FILTER_BITS	9
4973cb4a0bbSKawai, Hidehiro #define MMF_DUMP_FILTER_MASK \
4983cb4a0bbSKawai, Hidehiro 	(((1 << MMF_DUMP_FILTER_BITS) - 1) << MMF_DUMP_FILTER_SHIFT)
4993cb4a0bbSKawai, Hidehiro #define MMF_DUMP_FILTER_DEFAULT \
500e575f111SKOSAKI Motohiro 	((1 << MMF_DUMP_ANON_PRIVATE) |	(1 << MMF_DUMP_ANON_SHARED) |\
501656eb2cdSRoland McGrath 	 (1 << MMF_DUMP_HUGETLB_PRIVATE) | MMF_DUMP_MASK_DEFAULT_ELF)
502656eb2cdSRoland McGrath 
503656eb2cdSRoland McGrath #ifdef CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS
504656eb2cdSRoland McGrath # define MMF_DUMP_MASK_DEFAULT_ELF	(1 << MMF_DUMP_ELF_HEADERS)
505656eb2cdSRoland McGrath #else
506656eb2cdSRoland McGrath # define MMF_DUMP_MASK_DEFAULT_ELF	0
507656eb2cdSRoland McGrath #endif
508f8af4da3SHugh Dickins 					/* leave room for more dump flags */
509f8af4da3SHugh Dickins #define MMF_VM_MERGEABLE	16	/* KSM may merge identical pages */
510ba76149fSAndrea Arcangeli #define MMF_VM_HUGEPAGE		17	/* set when VM_HUGEPAGE is set on vma */
5113fb4afd9SStanislav Kinsburskiy /*
5123fb4afd9SStanislav Kinsburskiy  * This one-shot flag is dropped due to necessity of changing exe once again
5133fb4afd9SStanislav Kinsburskiy  * on NFS restore
5143fb4afd9SStanislav Kinsburskiy  */
5153fb4afd9SStanislav Kinsburskiy //#define MMF_EXE_FILE_CHANGED	18	/* see prctl_set_mm_exe_file() */
516f8af4da3SHugh Dickins 
5179f68f672SOleg Nesterov #define MMF_HAS_UPROBES		19	/* has uprobes */
5189f68f672SOleg Nesterov #define MMF_RECALC_UPROBES	20	/* MMF_HAS_UPROBES can be wrong */
519862e3073SMichal Hocko #define MMF_OOM_SKIP		21	/* mm is of no interest for the OOM killer */
5203f70dc38SMichal Hocko #define MMF_UNSTABLE		22	/* mm is unstable for copy_from_user */
5216fcb52a5SAaron Lu #define MMF_HUGE_ZERO_PAGE	23      /* mm has ever used the global huge zero page */
522f8ac4ec9SOleg Nesterov 
523f8af4da3SHugh Dickins #define MMF_INIT_MASK		(MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK)
5246c5d5238SKawai, Hidehiro 
5251da177e4SLinus Torvalds struct sighand_struct {
5261da177e4SLinus Torvalds 	atomic_t		count;
5271da177e4SLinus Torvalds 	struct k_sigaction	action[_NSIG];
5281da177e4SLinus Torvalds 	spinlock_t		siglock;
529b8fceee1SDavide Libenzi 	wait_queue_head_t	signalfd_wqh;
5301da177e4SLinus Torvalds };
5311da177e4SLinus Torvalds 
5320e464814SKaiGai Kohei struct pacct_struct {
533f6ec29a4SKaiGai Kohei 	int			ac_flag;
534f6ec29a4SKaiGai Kohei 	long			ac_exitcode;
5350e464814SKaiGai Kohei 	unsigned long		ac_mem;
536d4bc42afSFrederic Weisbecker 	u64			ac_utime, ac_stime;
53777787bfbSKaiGai Kohei 	unsigned long		ac_minflt, ac_majflt;
5380e464814SKaiGai Kohei };
5390e464814SKaiGai Kohei 
54042c4ab41SStanislaw Gruszka struct cpu_itimer {
541858cf3a8SFrederic Weisbecker 	u64 expires;
542858cf3a8SFrederic Weisbecker 	u64 incr;
54342c4ab41SStanislaw Gruszka };
54442c4ab41SStanislaw Gruszka 
545f06febc9SFrank Mayhar /**
5469d7fb042SPeter Zijlstra  * struct prev_cputime - snaphsot of system and user cputime
547d37f761dSFrederic Weisbecker  * @utime: time spent in user mode
548d37f761dSFrederic Weisbecker  * @stime: time spent in system mode
5499d7fb042SPeter Zijlstra  * @lock: protects the above two fields
550d37f761dSFrederic Weisbecker  *
5519d7fb042SPeter Zijlstra  * Stores previous user/system time values such that we can guarantee
5529d7fb042SPeter Zijlstra  * monotonicity.
553d37f761dSFrederic Weisbecker  */
5549d7fb042SPeter Zijlstra struct prev_cputime {
5559d7fb042SPeter Zijlstra #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
5565613fda9SFrederic Weisbecker 	u64 utime;
5575613fda9SFrederic Weisbecker 	u64 stime;
5589d7fb042SPeter Zijlstra 	raw_spinlock_t lock;
5599d7fb042SPeter Zijlstra #endif
560d37f761dSFrederic Weisbecker };
561d37f761dSFrederic Weisbecker 
5629d7fb042SPeter Zijlstra static inline void prev_cputime_init(struct prev_cputime *prev)
5639d7fb042SPeter Zijlstra {
5649d7fb042SPeter Zijlstra #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
5659d7fb042SPeter Zijlstra 	prev->utime = prev->stime = 0;
5669d7fb042SPeter Zijlstra 	raw_spin_lock_init(&prev->lock);
5679d7fb042SPeter Zijlstra #endif
5689d7fb042SPeter Zijlstra }
5699d7fb042SPeter Zijlstra 
570d37f761dSFrederic Weisbecker /**
571f06febc9SFrank Mayhar  * struct task_cputime - collected CPU time counts
5725613fda9SFrederic Weisbecker  * @utime:		time spent in user mode, in nanoseconds
5735613fda9SFrederic Weisbecker  * @stime:		time spent in kernel mode, in nanoseconds
574f06febc9SFrank Mayhar  * @sum_exec_runtime:	total time spent on the CPU, in nanoseconds
575f06febc9SFrank Mayhar  *
5769d7fb042SPeter Zijlstra  * This structure groups together three kinds of CPU time that are tracked for
5779d7fb042SPeter Zijlstra  * threads and thread groups.  Most things considering CPU time want to group
5789d7fb042SPeter Zijlstra  * these counts together and treat all three of them in parallel.
579f06febc9SFrank Mayhar  */
580f06febc9SFrank Mayhar struct task_cputime {
5815613fda9SFrederic Weisbecker 	u64 utime;
5825613fda9SFrederic Weisbecker 	u64 stime;
583f06febc9SFrank Mayhar 	unsigned long long sum_exec_runtime;
584f06febc9SFrank Mayhar };
5859d7fb042SPeter Zijlstra 
586f06febc9SFrank Mayhar /* Alternate field names when used to cache expirations. */
587f06febc9SFrank Mayhar #define virt_exp	utime
5889d7fb042SPeter Zijlstra #define prof_exp	stime
589f06febc9SFrank Mayhar #define sched_exp	sum_exec_runtime
590f06febc9SFrank Mayhar 
591971e8a98SJason Low /*
592971e8a98SJason Low  * This is the atomic variant of task_cputime, which can be used for
593971e8a98SJason Low  * storing and updating task_cputime statistics without locking.
594971e8a98SJason Low  */
595971e8a98SJason Low struct task_cputime_atomic {
596971e8a98SJason Low 	atomic64_t utime;
597971e8a98SJason Low 	atomic64_t stime;
598971e8a98SJason Low 	atomic64_t sum_exec_runtime;
599971e8a98SJason Low };
600971e8a98SJason Low 
601971e8a98SJason Low #define INIT_CPUTIME_ATOMIC \
602971e8a98SJason Low 	(struct task_cputime_atomic) {				\
603971e8a98SJason Low 		.utime = ATOMIC64_INIT(0),			\
604971e8a98SJason Low 		.stime = ATOMIC64_INIT(0),			\
605971e8a98SJason Low 		.sum_exec_runtime = ATOMIC64_INIT(0),		\
606971e8a98SJason Low 	}
607971e8a98SJason Low 
608609ca066SPeter Zijlstra #define PREEMPT_DISABLED	(PREEMPT_DISABLE_OFFSET + PREEMPT_ENABLED)
609a233f112SPeter Zijlstra 
610c99e6efeSPeter Zijlstra /*
61187dcbc06SPeter Zijlstra  * Disable preemption until the scheduler is running -- use an unconditional
61287dcbc06SPeter Zijlstra  * value so that it also works on !PREEMPT_COUNT kernels.
613d86ee480SPeter Zijlstra  *
61487dcbc06SPeter Zijlstra  * Reset by start_kernel()->sched_init()->init_idle()->init_idle_preempt_count().
615c99e6efeSPeter Zijlstra  */
61687dcbc06SPeter Zijlstra #define INIT_PREEMPT_COUNT	PREEMPT_OFFSET
617c99e6efeSPeter Zijlstra 
618609ca066SPeter Zijlstra /*
619609ca066SPeter Zijlstra  * Initial preempt_count value; reflects the preempt_count schedule invariant
620609ca066SPeter Zijlstra  * which states that during context switches:
621609ca066SPeter Zijlstra  *
622609ca066SPeter Zijlstra  *    preempt_count() == 2*PREEMPT_DISABLE_OFFSET
623609ca066SPeter Zijlstra  *
624609ca066SPeter Zijlstra  * Note: PREEMPT_DISABLE_OFFSET is 0 for !PREEMPT_COUNT kernels.
625609ca066SPeter Zijlstra  * Note: See finish_task_switch().
626609ca066SPeter Zijlstra  */
627609ca066SPeter Zijlstra #define FORK_PREEMPT_COUNT	(2*PREEMPT_DISABLE_OFFSET + PREEMPT_ENABLED)
6284cd4c1b4SPeter Zijlstra 
629f06febc9SFrank Mayhar /**
630f06febc9SFrank Mayhar  * struct thread_group_cputimer - thread group interval timer counts
631920ce39fSJason Low  * @cputime_atomic:	atomic thread group interval timers.
632d5c373ebSJason Low  * @running:		true when there are timers running and
633d5c373ebSJason Low  *			@cputime_atomic receives updates.
634c8d75aa4SJason Low  * @checking_timer:	true when a thread in the group is in the
635c8d75aa4SJason Low  *			process of checking for thread group timers.
636f06febc9SFrank Mayhar  *
637f06febc9SFrank Mayhar  * This structure contains the version of task_cputime, above, that is
6384cd4c1b4SPeter Zijlstra  * used for thread group CPU timer calculations.
639f06febc9SFrank Mayhar  */
6404cd4c1b4SPeter Zijlstra struct thread_group_cputimer {
64171107445SJason Low 	struct task_cputime_atomic cputime_atomic;
642d5c373ebSJason Low 	bool running;
643c8d75aa4SJason Low 	bool checking_timer;
644f06febc9SFrank Mayhar };
645f06febc9SFrank Mayhar 
6464714d1d3SBen Blum #include <linux/rwsem.h>
6475091faa4SMike Galbraith struct autogroup;
6485091faa4SMike Galbraith 
6491da177e4SLinus Torvalds /*
650e815f0a8SJonathan Neuschäfer  * NOTE! "signal_struct" does not have its own
6511da177e4SLinus Torvalds  * locking, because a shared signal_struct always
6521da177e4SLinus Torvalds  * implies a shared sighand_struct, so locking
6531da177e4SLinus Torvalds  * sighand_struct is always a proper superset of
6541da177e4SLinus Torvalds  * the locking of signal_struct.
6551da177e4SLinus Torvalds  */
6561da177e4SLinus Torvalds struct signal_struct {
657ea6d290cSOleg Nesterov 	atomic_t		sigcnt;
6581da177e4SLinus Torvalds 	atomic_t		live;
659b3ac022cSOleg Nesterov 	int			nr_threads;
6600c740d0aSOleg Nesterov 	struct list_head	thread_head;
6611da177e4SLinus Torvalds 
6621da177e4SLinus Torvalds 	wait_queue_head_t	wait_chldexit;	/* for wait4() */
6631da177e4SLinus Torvalds 
6641da177e4SLinus Torvalds 	/* current thread group signal load-balancing target: */
66536c8b586SIngo Molnar 	struct task_struct	*curr_target;
6661da177e4SLinus Torvalds 
6671da177e4SLinus Torvalds 	/* shared signal handling: */
6681da177e4SLinus Torvalds 	struct sigpending	shared_pending;
6691da177e4SLinus Torvalds 
6701da177e4SLinus Torvalds 	/* thread group exit support */
6711da177e4SLinus Torvalds 	int			group_exit_code;
6721da177e4SLinus Torvalds 	/* overloaded:
6731da177e4SLinus Torvalds 	 * - notify group_exit_task when ->count is equal to notify_count
6741da177e4SLinus Torvalds 	 * - everyone except group_exit_task is stopped during signal delivery
6751da177e4SLinus Torvalds 	 *   of fatal signals, group_exit_task processes the signal.
6761da177e4SLinus Torvalds 	 */
6771da177e4SLinus Torvalds 	int			notify_count;
67807dd20e0SRichard Kennedy 	struct task_struct	*group_exit_task;
6791da177e4SLinus Torvalds 
6801da177e4SLinus Torvalds 	/* thread group stop support, overloads group_exit_code too */
6811da177e4SLinus Torvalds 	int			group_stop_count;
6821da177e4SLinus Torvalds 	unsigned int		flags; /* see SIGNAL_* flags below */
6831da177e4SLinus Torvalds 
684ebec18a6SLennart Poettering 	/*
685ebec18a6SLennart Poettering 	 * PR_SET_CHILD_SUBREAPER marks a process, like a service
686ebec18a6SLennart Poettering 	 * manager, to re-parent orphan (double-forking) child processes
687ebec18a6SLennart Poettering 	 * to this process instead of 'init'. The service manager is
688ebec18a6SLennart Poettering 	 * able to receive SIGCHLD signals and is able to investigate
689ebec18a6SLennart Poettering 	 * the process until it calls wait(). All children of this
690ebec18a6SLennart Poettering 	 * process will inherit a flag if they should look for a
691ebec18a6SLennart Poettering 	 * child_subreaper process at exit.
692ebec18a6SLennart Poettering 	 */
693ebec18a6SLennart Poettering 	unsigned int		is_child_subreaper:1;
694ebec18a6SLennart Poettering 	unsigned int		has_child_subreaper:1;
695ebec18a6SLennart Poettering 
696b18b6a9cSNicolas Pitre #ifdef CONFIG_POSIX_TIMERS
697b18b6a9cSNicolas Pitre 
6981da177e4SLinus Torvalds 	/* POSIX.1b Interval Timers */
6995ed67f05SPavel Emelyanov 	int			posix_timer_id;
7001da177e4SLinus Torvalds 	struct list_head	posix_timers;
7011da177e4SLinus Torvalds 
7021da177e4SLinus Torvalds 	/* ITIMER_REAL timer for the process */
7032ff678b8SThomas Gleixner 	struct hrtimer real_timer;
7042ff678b8SThomas Gleixner 	ktime_t it_real_incr;
7051da177e4SLinus Torvalds 
70642c4ab41SStanislaw Gruszka 	/*
70742c4ab41SStanislaw Gruszka 	 * ITIMER_PROF and ITIMER_VIRTUAL timers for the process, we use
70842c4ab41SStanislaw Gruszka 	 * CPUCLOCK_PROF and CPUCLOCK_VIRT for indexing array as these
70942c4ab41SStanislaw Gruszka 	 * values are defined to 0 and 1 respectively
71042c4ab41SStanislaw Gruszka 	 */
71142c4ab41SStanislaw Gruszka 	struct cpu_itimer it[2];
7121da177e4SLinus Torvalds 
713f06febc9SFrank Mayhar 	/*
7144cd4c1b4SPeter Zijlstra 	 * Thread group totals for process CPU timers.
7154cd4c1b4SPeter Zijlstra 	 * See thread_group_cputimer(), et al, for details.
716f06febc9SFrank Mayhar 	 */
7174cd4c1b4SPeter Zijlstra 	struct thread_group_cputimer cputimer;
718f06febc9SFrank Mayhar 
719f06febc9SFrank Mayhar 	/* Earliest-expiration cache. */
720f06febc9SFrank Mayhar 	struct task_cputime cputime_expires;
721f06febc9SFrank Mayhar 
722b18b6a9cSNicolas Pitre 	struct list_head cpu_timers[3];
723b18b6a9cSNicolas Pitre 
724b18b6a9cSNicolas Pitre #endif
725b18b6a9cSNicolas Pitre 
726b18b6a9cSNicolas Pitre 	struct pid *leader_pid;
727b18b6a9cSNicolas Pitre 
728d027d45dSFrederic Weisbecker #ifdef CONFIG_NO_HZ_FULL
729f009a7a7SFrederic Weisbecker 	atomic_t tick_dep_mask;
730d027d45dSFrederic Weisbecker #endif
731d027d45dSFrederic Weisbecker 
732ab521dc0SEric W. Biederman 	struct pid *tty_old_pgrp;
7331ec320afSCedric Le Goater 
7341da177e4SLinus Torvalds 	/* boolean value for session group leader */
7351da177e4SLinus Torvalds 	int leader;
7361da177e4SLinus Torvalds 
7371da177e4SLinus Torvalds 	struct tty_struct *tty; /* NULL if no tty */
7381da177e4SLinus Torvalds 
7395091faa4SMike Galbraith #ifdef CONFIG_SCHED_AUTOGROUP
7405091faa4SMike Galbraith 	struct autogroup *autogroup;
7415091faa4SMike Galbraith #endif
7421da177e4SLinus Torvalds 	/*
7431da177e4SLinus Torvalds 	 * Cumulative resource counters for dead threads in the group,
7441da177e4SLinus Torvalds 	 * and for reaped dead child processes forked by this group.
7451da177e4SLinus Torvalds 	 * Live threads maintain their own counters and add to these
7461da177e4SLinus Torvalds 	 * in __exit_signal, except for the group leader.
7471da177e4SLinus Torvalds 	 */
748e78c3496SRik van Riel 	seqlock_t stats_lock;
7495613fda9SFrederic Weisbecker 	u64 utime, stime, cutime, cstime;
75016a6d9beSFrederic Weisbecker 	u64 gtime;
75116a6d9beSFrederic Weisbecker 	u64 cgtime;
7529d7fb042SPeter Zijlstra 	struct prev_cputime prev_cputime;
7531da177e4SLinus Torvalds 	unsigned long nvcsw, nivcsw, cnvcsw, cnivcsw;
7541da177e4SLinus Torvalds 	unsigned long min_flt, maj_flt, cmin_flt, cmaj_flt;
7556eaeeabaSEric Dumazet 	unsigned long inblock, oublock, cinblock, coublock;
7561f10206cSJiri Pirko 	unsigned long maxrss, cmaxrss;
757940389b8SAndrea Righi 	struct task_io_accounting ioac;
7581da177e4SLinus Torvalds 
7591da177e4SLinus Torvalds 	/*
76032bd671dSPeter Zijlstra 	 * Cumulative ns of schedule CPU time fo dead threads in the
76132bd671dSPeter Zijlstra 	 * group, not including a zombie group leader, (This only differs
76232bd671dSPeter Zijlstra 	 * from jiffies_to_ns(utime + stime) if sched_clock uses something
76332bd671dSPeter Zijlstra 	 * other than jiffies.)
76432bd671dSPeter Zijlstra 	 */
76532bd671dSPeter Zijlstra 	unsigned long long sum_sched_runtime;
76632bd671dSPeter Zijlstra 
76732bd671dSPeter Zijlstra 	/*
7681da177e4SLinus Torvalds 	 * We don't bother to synchronize most readers of this at all,
7691da177e4SLinus Torvalds 	 * because there is no reader checking a limit that actually needs
7701da177e4SLinus Torvalds 	 * to get both rlim_cur and rlim_max atomically, and either one
7711da177e4SLinus Torvalds 	 * alone is a single word that can safely be read normally.
7721da177e4SLinus Torvalds 	 * getrlimit/setrlimit use task_lock(current->group_leader) to
7731da177e4SLinus Torvalds 	 * protect this instead of the siglock, because they really
7741da177e4SLinus Torvalds 	 * have no need to disable irqs.
7751da177e4SLinus Torvalds 	 */
7761da177e4SLinus Torvalds 	struct rlimit rlim[RLIM_NLIMITS];
7771da177e4SLinus Torvalds 
7780e464814SKaiGai Kohei #ifdef CONFIG_BSD_PROCESS_ACCT
7790e464814SKaiGai Kohei 	struct pacct_struct pacct;	/* per-process accounting information */
7800e464814SKaiGai Kohei #endif
781ad4ecbcbSShailabh Nagar #ifdef CONFIG_TASKSTATS
782ad4ecbcbSShailabh Nagar 	struct taskstats *stats;
783ad4ecbcbSShailabh Nagar #endif
784522ed776SMiloslav Trmac #ifdef CONFIG_AUDIT
785522ed776SMiloslav Trmac 	unsigned audit_tty;
786522ed776SMiloslav Trmac 	struct tty_audit_buf *tty_audit_buf;
787522ed776SMiloslav Trmac #endif
78828b83c51SKOSAKI Motohiro 
789c96fc2d8STetsuo Handa 	/*
790c96fc2d8STetsuo Handa 	 * Thread is the potential origin of an oom condition; kill first on
791c96fc2d8STetsuo Handa 	 * oom
792c96fc2d8STetsuo Handa 	 */
793c96fc2d8STetsuo Handa 	bool oom_flag_origin;
794a9c58b90SDavid Rientjes 	short oom_score_adj;		/* OOM kill score adjustment */
795a9c58b90SDavid Rientjes 	short oom_score_adj_min;	/* OOM kill score adjustment min value.
796dabb16f6SMandeep Singh Baines 					 * Only settable by CAP_SYS_RESOURCE. */
79726db62f1SMichal Hocko 	struct mm_struct *oom_mm;	/* recorded mm when the thread group got
79826db62f1SMichal Hocko 					 * killed by the oom killer */
7999b1bf12dSKOSAKI Motohiro 
8009b1bf12dSKOSAKI Motohiro 	struct mutex cred_guard_mutex;	/* guard against foreign influences on
8019b1bf12dSKOSAKI Motohiro 					 * credential calculations
8029b1bf12dSKOSAKI Motohiro 					 * (notably. ptrace) */
8031da177e4SLinus Torvalds };
8041da177e4SLinus Torvalds 
8051da177e4SLinus Torvalds /*
8061da177e4SLinus Torvalds  * Bits in flags field of signal_struct.
8071da177e4SLinus Torvalds  */
8081da177e4SLinus Torvalds #define SIGNAL_STOP_STOPPED	0x00000001 /* job control stop in effect */
809ee77f075SOleg Nesterov #define SIGNAL_STOP_CONTINUED	0x00000002 /* SIGCONT since WCONTINUED reap */
810ee77f075SOleg Nesterov #define SIGNAL_GROUP_EXIT	0x00000004 /* group exit in progress */
811403bad72SOleg Nesterov #define SIGNAL_GROUP_COREDUMP	0x00000008 /* coredump in progress */
812e4420551SOleg Nesterov /*
813e4420551SOleg Nesterov  * Pending notifications to parent.
814e4420551SOleg Nesterov  */
815e4420551SOleg Nesterov #define SIGNAL_CLD_STOPPED	0x00000010
816e4420551SOleg Nesterov #define SIGNAL_CLD_CONTINUED	0x00000020
817e4420551SOleg Nesterov #define SIGNAL_CLD_MASK		(SIGNAL_CLD_STOPPED|SIGNAL_CLD_CONTINUED)
8181da177e4SLinus Torvalds 
819fae5fa44SOleg Nesterov #define SIGNAL_UNKILLABLE	0x00000040 /* for init: ignore fatal signals */
820fae5fa44SOleg Nesterov 
8212d39b3cdSJamie Iles #define SIGNAL_STOP_MASK (SIGNAL_CLD_MASK | SIGNAL_STOP_STOPPED | \
8222d39b3cdSJamie Iles 			  SIGNAL_STOP_CONTINUED)
8232d39b3cdSJamie Iles 
8242d39b3cdSJamie Iles static inline void signal_set_stop_flags(struct signal_struct *sig,
8252d39b3cdSJamie Iles 					 unsigned int flags)
8262d39b3cdSJamie Iles {
8272d39b3cdSJamie Iles 	WARN_ON(sig->flags & (SIGNAL_GROUP_EXIT|SIGNAL_GROUP_COREDUMP));
8282d39b3cdSJamie Iles 	sig->flags = (sig->flags & ~SIGNAL_STOP_MASK) | flags;
8292d39b3cdSJamie Iles }
8302d39b3cdSJamie Iles 
831ed5d2cacSOleg Nesterov /* If true, all threads except ->group_exit_task have pending SIGKILL */
832ed5d2cacSOleg Nesterov static inline int signal_group_exit(const struct signal_struct *sig)
833ed5d2cacSOleg Nesterov {
834ed5d2cacSOleg Nesterov 	return	(sig->flags & SIGNAL_GROUP_EXIT) ||
835ed5d2cacSOleg Nesterov 		(sig->group_exit_task != NULL);
836ed5d2cacSOleg Nesterov }
837ed5d2cacSOleg Nesterov 
8381da177e4SLinus Torvalds /*
8391da177e4SLinus Torvalds  * Some day this will be a full-fledged user tracking system..
8401da177e4SLinus Torvalds  */
8411da177e4SLinus Torvalds struct user_struct {
8421da177e4SLinus Torvalds 	atomic_t __count;	/* reference count */
8431da177e4SLinus Torvalds 	atomic_t processes;	/* How many processes does this user have? */
8441da177e4SLinus Torvalds 	atomic_t sigpending;	/* How many pending signals does this user have? */
8454afeff85SEric Paris #ifdef CONFIG_FANOTIFY
8464afeff85SEric Paris 	atomic_t fanotify_listeners;
8474afeff85SEric Paris #endif
8487ef9964eSDavide Libenzi #ifdef CONFIG_EPOLL
84952bd19f7SRobin Holt 	atomic_long_t epoll_watches; /* The number of file descriptors currently watched */
8507ef9964eSDavide Libenzi #endif
851970a8645SAlexey Dobriyan #ifdef CONFIG_POSIX_MQUEUE
8521da177e4SLinus Torvalds 	/* protected by mq_lock	*/
8531da177e4SLinus Torvalds 	unsigned long mq_bytes;	/* How many bytes can be allocated to mqueue? */
854970a8645SAlexey Dobriyan #endif
8551da177e4SLinus Torvalds 	unsigned long locked_shm; /* How many pages of mlocked shm ? */
856712f4aadSwilly tarreau 	unsigned long unix_inflight;	/* How many files in flight in unix sockets */
857759c0114SWilly Tarreau 	atomic_long_t pipe_bufs;  /* how many pages are allocated in pipe buffers */
8581da177e4SLinus Torvalds 
8591da177e4SLinus Torvalds #ifdef CONFIG_KEYS
8601da177e4SLinus Torvalds 	struct key *uid_keyring;	/* UID specific keyring */
8611da177e4SLinus Torvalds 	struct key *session_keyring;	/* UID's default session keyring */
8621da177e4SLinus Torvalds #endif
8631da177e4SLinus Torvalds 
8641da177e4SLinus Torvalds 	/* Hash table maintenance information */
865735de223SPavel Emelyanov 	struct hlist_node uidhash_node;
8667b44ab97SEric W. Biederman 	kuid_t uid;
86724e377a8SSrivatsa Vaddagiri 
868aaac3ba9SAlexei Starovoitov #if defined(CONFIG_PERF_EVENTS) || defined(CONFIG_BPF_SYSCALL)
869789f90fcSPeter Zijlstra 	atomic_long_t locked_vm;
870789f90fcSPeter Zijlstra #endif
8711da177e4SLinus Torvalds };
8721da177e4SLinus Torvalds 
873eb41d946SKay Sievers extern int uids_sysfs_init(void);
8745cb350baSDhaval Giani 
8757b44ab97SEric W. Biederman extern struct user_struct *find_user(kuid_t);
8761da177e4SLinus Torvalds 
8771da177e4SLinus Torvalds extern struct user_struct root_user;
8781da177e4SLinus Torvalds #define INIT_USER (&root_user)
8791da177e4SLinus Torvalds 
880b6dff3ecSDavid Howells 
8811da177e4SLinus Torvalds struct backing_dev_info;
8821da177e4SLinus Torvalds struct reclaim_state;
8831da177e4SLinus Torvalds 
884f6db8347SNaveen N. Rao #ifdef CONFIG_SCHED_INFO
8851da177e4SLinus Torvalds struct sched_info {
8861da177e4SLinus Torvalds 	/* cumulative counters */
8872d72376bSIngo Molnar 	unsigned long pcount;	      /* # of times run on this cpu */
8889c2c4802SKen Chen 	unsigned long long run_delay; /* time spent waiting on a runqueue */
8891da177e4SLinus Torvalds 
8901da177e4SLinus Torvalds 	/* timestamps */
891172ba844SBalbir Singh 	unsigned long long last_arrival,/* when we last ran on a cpu */
8921da177e4SLinus Torvalds 			   last_queued;	/* when we were last queued to run */
8931da177e4SLinus Torvalds };
894f6db8347SNaveen N. Rao #endif /* CONFIG_SCHED_INFO */
8951da177e4SLinus Torvalds 
896ca74e92bSShailabh Nagar #ifdef CONFIG_TASK_DELAY_ACCT
897ca74e92bSShailabh Nagar struct task_delay_info {
898ca74e92bSShailabh Nagar 	spinlock_t	lock;
899ca74e92bSShailabh Nagar 	unsigned int	flags;	/* Private per-task flags */
900ca74e92bSShailabh Nagar 
901ca74e92bSShailabh Nagar 	/* For each stat XXX, add following, aligned appropriately
902ca74e92bSShailabh Nagar 	 *
903ca74e92bSShailabh Nagar 	 * struct timespec XXX_start, XXX_end;
904ca74e92bSShailabh Nagar 	 * u64 XXX_delay;
905ca74e92bSShailabh Nagar 	 * u32 XXX_count;
906ca74e92bSShailabh Nagar 	 *
907ca74e92bSShailabh Nagar 	 * Atomicity of updates to XXX_delay, XXX_count protected by
908ca74e92bSShailabh Nagar 	 * single lock above (split into XXX_lock if contention is an issue).
909ca74e92bSShailabh Nagar 	 */
9100ff92245SShailabh Nagar 
9110ff92245SShailabh Nagar 	/*
9120ff92245SShailabh Nagar 	 * XXX_count is incremented on every XXX operation, the delay
9130ff92245SShailabh Nagar 	 * associated with the operation is added to XXX_delay.
9140ff92245SShailabh Nagar 	 * XXX_delay contains the accumulated delay time in nanoseconds.
9150ff92245SShailabh Nagar 	 */
9169667a23dSThomas Gleixner 	u64 blkio_start;	/* Shared by blkio, swapin */
9170ff92245SShailabh Nagar 	u64 blkio_delay;	/* wait for sync block io completion */
9180ff92245SShailabh Nagar 	u64 swapin_delay;	/* wait for swapin block io completion */
9190ff92245SShailabh Nagar 	u32 blkio_count;	/* total count of the number of sync block */
9200ff92245SShailabh Nagar 				/* io operations performed */
9210ff92245SShailabh Nagar 	u32 swapin_count;	/* total count of the number of swapin block */
9220ff92245SShailabh Nagar 				/* io operations performed */
923873b4771SKeika Kobayashi 
9249667a23dSThomas Gleixner 	u64 freepages_start;
925873b4771SKeika Kobayashi 	u64 freepages_delay;	/* wait for memory reclaim */
926873b4771SKeika Kobayashi 	u32 freepages_count;	/* total count of memory reclaim */
927ca74e92bSShailabh Nagar };
92852f17b6cSChandra Seetharaman #endif	/* CONFIG_TASK_DELAY_ACCT */
92952f17b6cSChandra Seetharaman 
93052f17b6cSChandra Seetharaman static inline int sched_info_on(void)
93152f17b6cSChandra Seetharaman {
93252f17b6cSChandra Seetharaman #ifdef CONFIG_SCHEDSTATS
93352f17b6cSChandra Seetharaman 	return 1;
93452f17b6cSChandra Seetharaman #elif defined(CONFIG_TASK_DELAY_ACCT)
93552f17b6cSChandra Seetharaman 	extern int delayacct_on;
93652f17b6cSChandra Seetharaman 	return delayacct_on;
93752f17b6cSChandra Seetharaman #else
93852f17b6cSChandra Seetharaman 	return 0;
939ca74e92bSShailabh Nagar #endif
94052f17b6cSChandra Seetharaman }
941ca74e92bSShailabh Nagar 
942cb251765SMel Gorman #ifdef CONFIG_SCHEDSTATS
943cb251765SMel Gorman void force_schedstat_enabled(void);
944cb251765SMel Gorman #endif
945cb251765SMel Gorman 
9461da177e4SLinus Torvalds /*
9476ecdd749SYuyang Du  * Integer metrics need fixed point arithmetic, e.g., sched/fair
9486ecdd749SYuyang Du  * has a few: load, load_avg, util_avg, freq, and capacity.
9496ecdd749SYuyang Du  *
9506ecdd749SYuyang Du  * We define a basic fixed point arithmetic range, and then formalize
9516ecdd749SYuyang Du  * all these metrics based on that basic range.
9526ecdd749SYuyang Du  */
9536ecdd749SYuyang Du # define SCHED_FIXEDPOINT_SHIFT	10
9546ecdd749SYuyang Du # define SCHED_FIXEDPOINT_SCALE	(1L << SCHED_FIXEDPOINT_SHIFT)
9556ecdd749SYuyang Du 
9561da177e4SLinus Torvalds struct io_context;			/* See blkdev.h */
9571da177e4SLinus Torvalds 
9581da177e4SLinus Torvalds 
959383f2835SChen, Kenneth W #ifdef ARCH_HAS_PREFETCH_SWITCH_STACK
96036c8b586SIngo Molnar extern void prefetch_stack(struct task_struct *t);
961383f2835SChen, Kenneth W #else
962383f2835SChen, Kenneth W static inline void prefetch_stack(struct task_struct *t) { }
963383f2835SChen, Kenneth W #endif
9641da177e4SLinus Torvalds 
9651da177e4SLinus Torvalds struct audit_context;		/* See audit.c */
9661da177e4SLinus Torvalds struct mempolicy;
967b92ce558SJens Axboe struct pipe_inode_info;
9684865ecf1SSerge E. Hallyn struct uts_namespace;
9691da177e4SLinus Torvalds 
97020b8a59fSIngo Molnar struct load_weight {
9719dbdb155SPeter Zijlstra 	unsigned long weight;
9729dbdb155SPeter Zijlstra 	u32 inv_weight;
97320b8a59fSIngo Molnar };
97420b8a59fSIngo Molnar 
9759d89c257SYuyang Du /*
9767b595334SYuyang Du  * The load_avg/util_avg accumulates an infinite geometric series
9777b595334SYuyang Du  * (see __update_load_avg() in kernel/sched/fair.c).
9787b595334SYuyang Du  *
9797b595334SYuyang Du  * [load_avg definition]
9807b595334SYuyang Du  *
9817b595334SYuyang Du  *   load_avg = runnable% * scale_load_down(load)
9827b595334SYuyang Du  *
9837b595334SYuyang Du  * where runnable% is the time ratio that a sched_entity is runnable.
9847b595334SYuyang Du  * For cfs_rq, it is the aggregated load_avg of all runnable and
9859d89c257SYuyang Du  * blocked sched_entities.
9867b595334SYuyang Du  *
9877b595334SYuyang Du  * load_avg may also take frequency scaling into account:
9887b595334SYuyang Du  *
9897b595334SYuyang Du  *   load_avg = runnable% * scale_load_down(load) * freq%
9907b595334SYuyang Du  *
9917b595334SYuyang Du  * where freq% is the CPU frequency normalized to the highest frequency.
9927b595334SYuyang Du  *
9937b595334SYuyang Du  * [util_avg definition]
9947b595334SYuyang Du  *
9957b595334SYuyang Du  *   util_avg = running% * SCHED_CAPACITY_SCALE
9967b595334SYuyang Du  *
9977b595334SYuyang Du  * where running% is the time ratio that a sched_entity is running on
9987b595334SYuyang Du  * a CPU. For cfs_rq, it is the aggregated util_avg of all runnable
9997b595334SYuyang Du  * and blocked sched_entities.
10007b595334SYuyang Du  *
10017b595334SYuyang Du  * util_avg may also factor frequency scaling and CPU capacity scaling:
10027b595334SYuyang Du  *
10037b595334SYuyang Du  *   util_avg = running% * SCHED_CAPACITY_SCALE * freq% * capacity%
10047b595334SYuyang Du  *
10057b595334SYuyang Du  * where freq% is the same as above, and capacity% is the CPU capacity
10067b595334SYuyang Du  * normalized to the greatest capacity (due to uarch differences, etc).
10077b595334SYuyang Du  *
10087b595334SYuyang Du  * N.B., the above ratios (runnable%, running%, freq%, and capacity%)
10097b595334SYuyang Du  * themselves are in the range of [0, 1]. To do fixed point arithmetics,
10107b595334SYuyang Du  * we therefore scale them to as large a range as necessary. This is for
10117b595334SYuyang Du  * example reflected by util_avg's SCHED_CAPACITY_SCALE.
10127b595334SYuyang Du  *
10137b595334SYuyang Du  * [Overflow issue]
10147b595334SYuyang Du  *
10157b595334SYuyang Du  * The 64-bit load_sum can have 4353082796 (=2^64/47742/88761) entities
10167b595334SYuyang Du  * with the highest load (=88761), always runnable on a single cfs_rq,
10177b595334SYuyang Du  * and should not overflow as the number already hits PID_MAX_LIMIT.
10187b595334SYuyang Du  *
10197b595334SYuyang Du  * For all other cases (including 32-bit kernels), struct load_weight's
10207b595334SYuyang Du  * weight will overflow first before we do, because:
10217b595334SYuyang Du  *
10227b595334SYuyang Du  *    Max(load_avg) <= Max(load.weight)
10237b595334SYuyang Du  *
10247b595334SYuyang Du  * Then it is the load_weight's responsibility to consider overflow
10257b595334SYuyang Du  * issues.
10269d89c257SYuyang Du  */
10279d85f21cSPaul Turner struct sched_avg {
10289d89c257SYuyang Du 	u64 last_update_time, load_sum;
10299d89c257SYuyang Du 	u32 util_sum, period_contrib;
10309d89c257SYuyang Du 	unsigned long load_avg, util_avg;
10319d85f21cSPaul Turner };
10329d85f21cSPaul Turner 
103394c18227SIngo Molnar #ifdef CONFIG_SCHEDSTATS
103441acab88SLucas De Marchi struct sched_statistics {
103594c18227SIngo Molnar 	u64			wait_start;
103694c18227SIngo Molnar 	u64			wait_max;
10376d082592SArjan van de Ven 	u64			wait_count;
10386d082592SArjan van de Ven 	u64			wait_sum;
10398f0dfc34SArjan van de Ven 	u64			iowait_count;
10408f0dfc34SArjan van de Ven 	u64			iowait_sum;
104194c18227SIngo Molnar 
104294c18227SIngo Molnar 	u64			sleep_start;
104320b8a59fSIngo Molnar 	u64			sleep_max;
104494c18227SIngo Molnar 	s64			sum_sleep_runtime;
104594c18227SIngo Molnar 
104694c18227SIngo Molnar 	u64			block_start;
104720b8a59fSIngo Molnar 	u64			block_max;
104820b8a59fSIngo Molnar 	u64			exec_max;
1049eba1ed4bSIngo Molnar 	u64			slice_max;
1050cc367732SIngo Molnar 
1051cc367732SIngo Molnar 	u64			nr_migrations_cold;
1052cc367732SIngo Molnar 	u64			nr_failed_migrations_affine;
1053cc367732SIngo Molnar 	u64			nr_failed_migrations_running;
1054cc367732SIngo Molnar 	u64			nr_failed_migrations_hot;
1055cc367732SIngo Molnar 	u64			nr_forced_migrations;
1056cc367732SIngo Molnar 
1057cc367732SIngo Molnar 	u64			nr_wakeups;
1058cc367732SIngo Molnar 	u64			nr_wakeups_sync;
1059cc367732SIngo Molnar 	u64			nr_wakeups_migrate;
1060cc367732SIngo Molnar 	u64			nr_wakeups_local;
1061cc367732SIngo Molnar 	u64			nr_wakeups_remote;
1062cc367732SIngo Molnar 	u64			nr_wakeups_affine;
1063cc367732SIngo Molnar 	u64			nr_wakeups_affine_attempts;
1064cc367732SIngo Molnar 	u64			nr_wakeups_passive;
1065cc367732SIngo Molnar 	u64			nr_wakeups_idle;
106641acab88SLucas De Marchi };
106741acab88SLucas De Marchi #endif
106841acab88SLucas De Marchi 
106941acab88SLucas De Marchi struct sched_entity {
107041acab88SLucas De Marchi 	struct load_weight	load;		/* for load-balancing */
107141acab88SLucas De Marchi 	struct rb_node		run_node;
107241acab88SLucas De Marchi 	struct list_head	group_node;
107341acab88SLucas De Marchi 	unsigned int		on_rq;
107441acab88SLucas De Marchi 
107541acab88SLucas De Marchi 	u64			exec_start;
107641acab88SLucas De Marchi 	u64			sum_exec_runtime;
107741acab88SLucas De Marchi 	u64			vruntime;
107841acab88SLucas De Marchi 	u64			prev_sum_exec_runtime;
107941acab88SLucas De Marchi 
108041acab88SLucas De Marchi 	u64			nr_migrations;
108141acab88SLucas De Marchi 
108241acab88SLucas De Marchi #ifdef CONFIG_SCHEDSTATS
108341acab88SLucas De Marchi 	struct sched_statistics statistics;
108494c18227SIngo Molnar #endif
108594c18227SIngo Molnar 
108620b8a59fSIngo Molnar #ifdef CONFIG_FAIR_GROUP_SCHED
1087fed14d45SPeter Zijlstra 	int			depth;
108820b8a59fSIngo Molnar 	struct sched_entity	*parent;
108920b8a59fSIngo Molnar 	/* rq on which this entity is (to be) queued: */
109020b8a59fSIngo Molnar 	struct cfs_rq		*cfs_rq;
109120b8a59fSIngo Molnar 	/* rq "owned" by this entity/group: */
109220b8a59fSIngo Molnar 	struct cfs_rq		*my_q;
109320b8a59fSIngo Molnar #endif
10948bd75c77SClark Williams 
1095141965c7SAlex Shi #ifdef CONFIG_SMP
10965a107804SJiri Olsa 	/*
10975a107804SJiri Olsa 	 * Per entity load average tracking.
10985a107804SJiri Olsa 	 *
10995a107804SJiri Olsa 	 * Put into separate cache line so it does not
11005a107804SJiri Olsa 	 * collide with read-mostly values above.
11015a107804SJiri Olsa 	 */
11025a107804SJiri Olsa 	struct sched_avg	avg ____cacheline_aligned_in_smp;
11039d85f21cSPaul Turner #endif
110420b8a59fSIngo Molnar };
110570b97a7fSIngo Molnar 
1106fa717060SPeter Zijlstra struct sched_rt_entity {
1107fa717060SPeter Zijlstra 	struct list_head run_list;
110878f2c7dbSPeter Zijlstra 	unsigned long timeout;
110957d2aa00SYing Xue 	unsigned long watchdog_stamp;
1110bee367edSRichard Kennedy 	unsigned int time_slice;
1111ff77e468SPeter Zijlstra 	unsigned short on_rq;
1112ff77e468SPeter Zijlstra 	unsigned short on_list;
11136f505b16SPeter Zijlstra 
111458d6c2d7SPeter Zijlstra 	struct sched_rt_entity *back;
1115052f1dc7SPeter Zijlstra #ifdef CONFIG_RT_GROUP_SCHED
11166f505b16SPeter Zijlstra 	struct sched_rt_entity	*parent;
11176f505b16SPeter Zijlstra 	/* rq on which this entity is (to be) queued: */
11186f505b16SPeter Zijlstra 	struct rt_rq		*rt_rq;
11196f505b16SPeter Zijlstra 	/* rq "owned" by this entity/group: */
11206f505b16SPeter Zijlstra 	struct rt_rq		*my_q;
11216f505b16SPeter Zijlstra #endif
1122fa717060SPeter Zijlstra };
1123fa717060SPeter Zijlstra 
1124aab03e05SDario Faggioli struct sched_dl_entity {
1125aab03e05SDario Faggioli 	struct rb_node	rb_node;
1126aab03e05SDario Faggioli 
1127aab03e05SDario Faggioli 	/*
1128aab03e05SDario Faggioli 	 * Original scheduling parameters. Copied here from sched_attr
11294027d080Sxiaofeng.yan 	 * during sched_setattr(), they will remain the same until
11304027d080Sxiaofeng.yan 	 * the next sched_setattr().
1131aab03e05SDario Faggioli 	 */
1132aab03e05SDario Faggioli 	u64 dl_runtime;		/* maximum runtime for each instance	*/
1133aab03e05SDario Faggioli 	u64 dl_deadline;	/* relative deadline of each instance	*/
1134755378a4SHarald Gustafsson 	u64 dl_period;		/* separation of two instances (period) */
1135332ac17eSDario Faggioli 	u64 dl_bw;		/* dl_runtime / dl_deadline		*/
1136aab03e05SDario Faggioli 
1137aab03e05SDario Faggioli 	/*
1138aab03e05SDario Faggioli 	 * Actual scheduling parameters. Initialized with the values above,
1139aab03e05SDario Faggioli 	 * they are continously updated during task execution. Note that
1140aab03e05SDario Faggioli 	 * the remaining runtime could be < 0 in case we are in overrun.
1141aab03e05SDario Faggioli 	 */
1142aab03e05SDario Faggioli 	s64 runtime;		/* remaining runtime for this instance	*/
1143aab03e05SDario Faggioli 	u64 deadline;		/* absolute deadline for this instance	*/
1144aab03e05SDario Faggioli 	unsigned int flags;	/* specifying the scheduler behaviour	*/
1145aab03e05SDario Faggioli 
1146aab03e05SDario Faggioli 	/*
1147aab03e05SDario Faggioli 	 * Some bool flags:
1148aab03e05SDario Faggioli 	 *
1149aab03e05SDario Faggioli 	 * @dl_throttled tells if we exhausted the runtime. If so, the
1150aab03e05SDario Faggioli 	 * task has to wait for a replenishment to be performed at the
1151aab03e05SDario Faggioli 	 * next firing of dl_timer.
1152aab03e05SDario Faggioli 	 *
11532d3d891dSDario Faggioli 	 * @dl_boosted tells if we are boosted due to DI. If so we are
11542d3d891dSDario Faggioli 	 * outside bandwidth enforcement mechanism (but only until we
11555bfd126eSJuri Lelli 	 * exit the critical section);
11565bfd126eSJuri Lelli 	 *
11575bfd126eSJuri Lelli 	 * @dl_yielded tells if task gave up the cpu before consuming
11585bfd126eSJuri Lelli 	 * all its available runtime during the last job.
1159aab03e05SDario Faggioli 	 */
116072f9f3fdSLuca Abeni 	int dl_throttled, dl_boosted, dl_yielded;
1161aab03e05SDario Faggioli 
1162aab03e05SDario Faggioli 	/*
1163aab03e05SDario Faggioli 	 * Bandwidth enforcement timer. Each -deadline task has its
1164aab03e05SDario Faggioli 	 * own bandwidth to be enforced, thus we need one timer per task.
1165aab03e05SDario Faggioli 	 */
1166aab03e05SDario Faggioli 	struct hrtimer dl_timer;
1167aab03e05SDario Faggioli };
11688bd75c77SClark Williams 
11691d082fd0SPaul E. McKenney union rcu_special {
11701d082fd0SPaul E. McKenney 	struct {
11718203d6d0SPaul E. McKenney 		u8 blocked;
11728203d6d0SPaul E. McKenney 		u8 need_qs;
11738203d6d0SPaul E. McKenney 		u8 exp_need_qs;
11748203d6d0SPaul E. McKenney 		u8 pad;	/* Otherwise the compiler can store garbage here. */
11758203d6d0SPaul E. McKenney 	} b; /* Bits. */
11768203d6d0SPaul E. McKenney 	u32 s; /* Set of bits. */
11771d082fd0SPaul E. McKenney };
117886848966SPaul E. McKenney struct rcu_node;
117986848966SPaul E. McKenney 
11808dc85d54SPeter Zijlstra enum perf_event_task_context {
11818dc85d54SPeter Zijlstra 	perf_invalid_context = -1,
11828dc85d54SPeter Zijlstra 	perf_hw_context = 0,
118389a1e187SPeter Zijlstra 	perf_sw_context,
11848dc85d54SPeter Zijlstra 	perf_nr_task_contexts,
11858dc85d54SPeter Zijlstra };
11868dc85d54SPeter Zijlstra 
1187*eb61baf6SIngo Molnar struct wake_q_node {
1188*eb61baf6SIngo Molnar 	struct wake_q_node *next;
1189*eb61baf6SIngo Molnar };
1190*eb61baf6SIngo Molnar 
119172b252aeSMel Gorman /* Track pages that require TLB flushes */
119272b252aeSMel Gorman struct tlbflush_unmap_batch {
119372b252aeSMel Gorman 	/*
119472b252aeSMel Gorman 	 * Each bit set is a CPU that potentially has a TLB entry for one of
119572b252aeSMel Gorman 	 * the PFNs being flushed. See set_tlb_ubc_flush_pending().
119672b252aeSMel Gorman 	 */
119772b252aeSMel Gorman 	struct cpumask cpumask;
119872b252aeSMel Gorman 
119972b252aeSMel Gorman 	/* True if any bit in cpumask is set */
120072b252aeSMel Gorman 	bool flush_required;
1201d950c947SMel Gorman 
1202d950c947SMel Gorman 	/*
1203d950c947SMel Gorman 	 * If true then the PTE was dirty when unmapped. The entry must be
1204d950c947SMel Gorman 	 * flushed before IO is initiated or a stale TLB entry potentially
1205d950c947SMel Gorman 	 * allows an update without redirtying the page.
1206d950c947SMel Gorman 	 */
1207d950c947SMel Gorman 	bool writable;
120872b252aeSMel Gorman };
120972b252aeSMel Gorman 
12101da177e4SLinus Torvalds struct task_struct {
1211c65eacbeSAndy Lutomirski #ifdef CONFIG_THREAD_INFO_IN_TASK
1212c65eacbeSAndy Lutomirski 	/*
1213c65eacbeSAndy Lutomirski 	 * For reasons of header soup (see current_thread_info()), this
1214c65eacbeSAndy Lutomirski 	 * must be the first element of task_struct.
1215c65eacbeSAndy Lutomirski 	 */
1216c65eacbeSAndy Lutomirski 	struct thread_info thread_info;
1217c65eacbeSAndy Lutomirski #endif
12181da177e4SLinus Torvalds 	volatile long state;	/* -1 unrunnable, 0 runnable, >0 stopped */
1219f7e4217bSRoman Zippel 	void *stack;
12201da177e4SLinus Torvalds 	atomic_t usage;
122197dc32cdSWilliam Cohen 	unsigned int flags;	/* per process flags, defined below */
122297dc32cdSWilliam Cohen 	unsigned int ptrace;
12231da177e4SLinus Torvalds 
12242dd73a4fSPeter Williams #ifdef CONFIG_SMP
1225fa14ff4aSPeter Zijlstra 	struct llist_node wake_entry;
12263ca7a440SPeter Zijlstra 	int on_cpu;
1227c65eacbeSAndy Lutomirski #ifdef CONFIG_THREAD_INFO_IN_TASK
1228c65eacbeSAndy Lutomirski 	unsigned int cpu;	/* current CPU */
1229c65eacbeSAndy Lutomirski #endif
123063b0e9edSMike Galbraith 	unsigned int wakee_flips;
123162470419SMichael Wang 	unsigned long wakee_flip_decay_ts;
123263b0e9edSMike Galbraith 	struct task_struct *last_wakee;
1233ac66f547SPeter Zijlstra 
1234ac66f547SPeter Zijlstra 	int wake_cpu;
12354866cde0SNick Piggin #endif
1236fd2f4419SPeter Zijlstra 	int on_rq;
123750e645a8SIngo Molnar 
1238b29739f9SIngo Molnar 	int prio, static_prio, normal_prio;
1239c7aceabaSRichard Kennedy 	unsigned int rt_priority;
12405522d5d5SIngo Molnar 	const struct sched_class *sched_class;
124120b8a59fSIngo Molnar 	struct sched_entity se;
1242fa717060SPeter Zijlstra 	struct sched_rt_entity rt;
12438323f26cSPeter Zijlstra #ifdef CONFIG_CGROUP_SCHED
12448323f26cSPeter Zijlstra 	struct task_group *sched_task_group;
12458323f26cSPeter Zijlstra #endif
1246aab03e05SDario Faggioli 	struct sched_dl_entity dl;
12471da177e4SLinus Torvalds 
1248e107be36SAvi Kivity #ifdef CONFIG_PREEMPT_NOTIFIERS
1249e107be36SAvi Kivity 	/* list of struct preempt_notifier: */
1250e107be36SAvi Kivity 	struct hlist_head preempt_notifiers;
1251e107be36SAvi Kivity #endif
1252e107be36SAvi Kivity 
12536c5c9341SAlexey Dobriyan #ifdef CONFIG_BLK_DEV_IO_TRACE
12542056a782SJens Axboe 	unsigned int btrace_seq;
12556c5c9341SAlexey Dobriyan #endif
12561da177e4SLinus Torvalds 
125797dc32cdSWilliam Cohen 	unsigned int policy;
125829baa747SPeter Zijlstra 	int nr_cpus_allowed;
12591da177e4SLinus Torvalds 	cpumask_t cpus_allowed;
12601da177e4SLinus Torvalds 
1261a57eb940SPaul E. McKenney #ifdef CONFIG_PREEMPT_RCU
1262e260be67SPaul E. McKenney 	int rcu_read_lock_nesting;
12631d082fd0SPaul E. McKenney 	union rcu_special rcu_read_unlock_special;
1264f41d911fSPaul E. McKenney 	struct list_head rcu_node_entry;
1265a57eb940SPaul E. McKenney 	struct rcu_node *rcu_blocked_node;
126628f6569aSPranith Kumar #endif /* #ifdef CONFIG_PREEMPT_RCU */
12678315f422SPaul E. McKenney #ifdef CONFIG_TASKS_RCU
12688315f422SPaul E. McKenney 	unsigned long rcu_tasks_nvcsw;
12698315f422SPaul E. McKenney 	bool rcu_tasks_holdout;
12708315f422SPaul E. McKenney 	struct list_head rcu_tasks_holdout_list;
1271176f8f7aSPaul E. McKenney 	int rcu_tasks_idle_cpu;
12728315f422SPaul E. McKenney #endif /* #ifdef CONFIG_TASKS_RCU */
1273e260be67SPaul E. McKenney 
1274f6db8347SNaveen N. Rao #ifdef CONFIG_SCHED_INFO
12751da177e4SLinus Torvalds 	struct sched_info sched_info;
12761da177e4SLinus Torvalds #endif
12771da177e4SLinus Torvalds 
12781da177e4SLinus Torvalds 	struct list_head tasks;
1279806c09a7SDario Faggioli #ifdef CONFIG_SMP
1280917b627dSGregory Haskins 	struct plist_node pushable_tasks;
12811baca4ceSJuri Lelli 	struct rb_node pushable_dl_tasks;
1282806c09a7SDario Faggioli #endif
12831da177e4SLinus Torvalds 
12841da177e4SLinus Torvalds 	struct mm_struct *mm, *active_mm;
1285314ff785SIngo Molnar 
1286314ff785SIngo Molnar 	/* Per-thread vma caching: */
1287314ff785SIngo Molnar 	struct vmacache vmacache;
1288314ff785SIngo Molnar 
128934e55232SKAMEZAWA Hiroyuki #if defined(SPLIT_RSS_COUNTING)
129034e55232SKAMEZAWA Hiroyuki 	struct task_rss_stat	rss_stat;
129134e55232SKAMEZAWA Hiroyuki #endif
12921da177e4SLinus Torvalds /* task state */
129397dc32cdSWilliam Cohen 	int exit_state;
12941da177e4SLinus Torvalds 	int exit_code, exit_signal;
12951da177e4SLinus Torvalds 	int pdeath_signal;  /*  The signal sent when the parent dies  */
1296e7cc4173SPalmer Dabbelt 	unsigned long jobctl;	/* JOBCTL_*, siglock protected */
12979b89f6baSAndrei Epure 
12989b89f6baSAndrei Epure 	/* Used for emulating ABI behavior of previous Linux versions */
129997dc32cdSWilliam Cohen 	unsigned int personality;
13009b89f6baSAndrei Epure 
1301be958bdcSPeter Zijlstra 	/* scheduler bits, serialized by scheduler locks */
1302ca94c442SLennart Poettering 	unsigned sched_reset_on_fork:1;
1303a8e4f2eaSPeter Zijlstra 	unsigned sched_contributes_to_load:1;
1304ff303e66SPeter Zijlstra 	unsigned sched_migrated:1;
1305b7e7ade3SPeter Zijlstra 	unsigned sched_remote_wakeup:1;
1306be958bdcSPeter Zijlstra 	unsigned :0; /* force alignment to the next boundary */
1307be958bdcSPeter Zijlstra 
1308be958bdcSPeter Zijlstra 	/* unserialized, strictly 'current' */
1309be958bdcSPeter Zijlstra 	unsigned in_execve:1; /* bit to tell LSMs we're in execve */
1310be958bdcSPeter Zijlstra 	unsigned in_iowait:1;
13117e781418SAndy Lutomirski #if !defined(TIF_RESTORE_SIGMASK)
13127e781418SAndy Lutomirski 	unsigned restore_sigmask:1;
13137e781418SAndy Lutomirski #endif
1314626ebc41STejun Heo #ifdef CONFIG_MEMCG
1315626ebc41STejun Heo 	unsigned memcg_may_oom:1;
1316127424c8SJohannes Weiner #ifndef CONFIG_SLOB
13176f185c29SVladimir Davydov 	unsigned memcg_kmem_skip_account:1;
13186f185c29SVladimir Davydov #endif
1319127424c8SJohannes Weiner #endif
1320ff303e66SPeter Zijlstra #ifdef CONFIG_COMPAT_BRK
1321ff303e66SPeter Zijlstra 	unsigned brk_randomized:1;
1322ff303e66SPeter Zijlstra #endif
13236f185c29SVladimir Davydov 
13241d4457f9SKees Cook 	unsigned long atomic_flags; /* Flags needing atomic access. */
13251d4457f9SKees Cook 
1326f56141e3SAndy Lutomirski 	struct restart_block restart_block;
1327f56141e3SAndy Lutomirski 
13281da177e4SLinus Torvalds 	pid_t pid;
13291da177e4SLinus Torvalds 	pid_t tgid;
13300a425405SArjan van de Ven 
13311314562aSHiroshi Shimamoto #ifdef CONFIG_CC_STACKPROTECTOR
13320a425405SArjan van de Ven 	/* Canary value for the -fstack-protector gcc feature */
13330a425405SArjan van de Ven 	unsigned long stack_canary;
13341314562aSHiroshi Shimamoto #endif
13351da177e4SLinus Torvalds 	/*
13361da177e4SLinus Torvalds 	 * pointers to (original) parent process, youngest child, younger sibling,
13371da177e4SLinus Torvalds 	 * older sibling, respectively.  (p->father can be replaced with
1338f470021aSRoland McGrath 	 * p->real_parent->pid)
13391da177e4SLinus Torvalds 	 */
1340abd63bc3SKees Cook 	struct task_struct __rcu *real_parent; /* real parent process */
1341abd63bc3SKees Cook 	struct task_struct __rcu *parent; /* recipient of SIGCHLD, wait4() reports */
13421da177e4SLinus Torvalds 	/*
1343f470021aSRoland McGrath 	 * children/sibling forms the list of my natural children
13441da177e4SLinus Torvalds 	 */
13451da177e4SLinus Torvalds 	struct list_head children;	/* list of my children */
13461da177e4SLinus Torvalds 	struct list_head sibling;	/* linkage in my parent's children list */
13471da177e4SLinus Torvalds 	struct task_struct *group_leader;	/* threadgroup leader */
13481da177e4SLinus Torvalds 
1349f470021aSRoland McGrath 	/*
1350f470021aSRoland McGrath 	 * ptraced is the list of tasks this task is using ptrace on.
1351f470021aSRoland McGrath 	 * This includes both natural children and PTRACE_ATTACH targets.
1352f470021aSRoland McGrath 	 * p->ptrace_entry is p's link on the p->parent->ptraced list.
1353f470021aSRoland McGrath 	 */
1354f470021aSRoland McGrath 	struct list_head ptraced;
1355f470021aSRoland McGrath 	struct list_head ptrace_entry;
1356f470021aSRoland McGrath 
13571da177e4SLinus Torvalds 	/* PID/PID hash table linkage. */
135892476d7fSEric W. Biederman 	struct pid_link pids[PIDTYPE_MAX];
135947e65328SOleg Nesterov 	struct list_head thread_group;
13600c740d0aSOleg Nesterov 	struct list_head thread_node;
13611da177e4SLinus Torvalds 
13621da177e4SLinus Torvalds 	struct completion *vfork_done;		/* for vfork() */
13631da177e4SLinus Torvalds 	int __user *set_child_tid;		/* CLONE_CHILD_SETTID */
13641da177e4SLinus Torvalds 	int __user *clear_child_tid;		/* CLONE_CHILD_CLEARTID */
13651da177e4SLinus Torvalds 
13665613fda9SFrederic Weisbecker 	u64 utime, stime;
136740565b5aSStanislaw Gruszka #ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
13685613fda9SFrederic Weisbecker 	u64 utimescaled, stimescaled;
136940565b5aSStanislaw Gruszka #endif
137016a6d9beSFrederic Weisbecker 	u64 gtime;
13719d7fb042SPeter Zijlstra 	struct prev_cputime prev_cputime;
13726a61671bSFrederic Weisbecker #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
1373b7ce2277SFrederic Weisbecker 	seqcount_t vtime_seqcount;
13746a61671bSFrederic Weisbecker 	unsigned long long vtime_snap;
13756a61671bSFrederic Weisbecker 	enum {
13767098c1eaSFrederic Weisbecker 		/* Task is sleeping or running in a CPU with VTIME inactive */
13777098c1eaSFrederic Weisbecker 		VTIME_INACTIVE = 0,
13787098c1eaSFrederic Weisbecker 		/* Task runs in userspace in a CPU with VTIME active */
13796a61671bSFrederic Weisbecker 		VTIME_USER,
13807098c1eaSFrederic Weisbecker 		/* Task runs in kernelspace in a CPU with VTIME active */
13816a61671bSFrederic Weisbecker 		VTIME_SYS,
13826a61671bSFrederic Weisbecker 	} vtime_snap_whence;
13836a61671bSFrederic Weisbecker #endif
1384d027d45dSFrederic Weisbecker 
1385d027d45dSFrederic Weisbecker #ifdef CONFIG_NO_HZ_FULL
1386f009a7a7SFrederic Weisbecker 	atomic_t tick_dep_mask;
1387d027d45dSFrederic Weisbecker #endif
13881da177e4SLinus Torvalds 	unsigned long nvcsw, nivcsw; /* context switch counts */
1389ccbf62d8SThomas Gleixner 	u64 start_time;		/* monotonic time in nsec */
139057e0be04SThomas Gleixner 	u64 real_start_time;	/* boot based time in nsec */
13911da177e4SLinus Torvalds /* mm fault and swap info: this can arguably be seen as either mm-specific or thread-specific */
13921da177e4SLinus Torvalds 	unsigned long min_flt, maj_flt;
13931da177e4SLinus Torvalds 
1394b18b6a9cSNicolas Pitre #ifdef CONFIG_POSIX_TIMERS
1395f06febc9SFrank Mayhar 	struct task_cputime cputime_expires;
13961da177e4SLinus Torvalds 	struct list_head cpu_timers[3];
1397b18b6a9cSNicolas Pitre #endif
13981da177e4SLinus Torvalds 
13991da177e4SLinus Torvalds /* process credentials */
140064b875f7SEric W. Biederman 	const struct cred __rcu *ptracer_cred; /* Tracer's credentials at attach */
14011b0ba1c9SArnd Bergmann 	const struct cred __rcu *real_cred; /* objective and real subjective task
14023b11a1deSDavid Howells 					 * credentials (COW) */
14031b0ba1c9SArnd Bergmann 	const struct cred __rcu *cred;	/* effective (overridable) subjective task
14043b11a1deSDavid Howells 					 * credentials (COW) */
140536772092SPaolo 'Blaisorblade' Giarrusso 	char comm[TASK_COMM_LEN]; /* executable name excluding path
140636772092SPaolo 'Blaisorblade' Giarrusso 				     - access with [gs]et_task_comm (which lock
140736772092SPaolo 'Blaisorblade' Giarrusso 				       it with task_lock())
1408221af7f8SLinus Torvalds 				     - initialized normally by setup_new_exec */
14091da177e4SLinus Torvalds /* file system info */
1410756daf26SNeilBrown 	struct nameidata *nameidata;
14113d5b6fccSAlexey Dobriyan #ifdef CONFIG_SYSVIPC
14121da177e4SLinus Torvalds /* ipc stuff */
14131da177e4SLinus Torvalds 	struct sysv_sem sysvsem;
1414ab602f79SJack Miller 	struct sysv_shm sysvshm;
14153d5b6fccSAlexey Dobriyan #endif
1416e162b39aSMandeep Singh Baines #ifdef CONFIG_DETECT_HUNG_TASK
141782a1fcb9SIngo Molnar /* hung task detection */
141882a1fcb9SIngo Molnar 	unsigned long last_switch_count;
141982a1fcb9SIngo Molnar #endif
14201da177e4SLinus Torvalds /* filesystem information */
14211da177e4SLinus Torvalds 	struct fs_struct *fs;
14221da177e4SLinus Torvalds /* open file information */
14231da177e4SLinus Torvalds 	struct files_struct *files;
14241651e14eSSerge E. Hallyn /* namespaces */
1425ab516013SSerge E. Hallyn 	struct nsproxy *nsproxy;
14261da177e4SLinus Torvalds /* signal handlers */
14271da177e4SLinus Torvalds 	struct signal_struct *signal;
14281da177e4SLinus Torvalds 	struct sighand_struct *sighand;
14291da177e4SLinus Torvalds 
14301da177e4SLinus Torvalds 	sigset_t blocked, real_blocked;
1431f3de272bSRoland McGrath 	sigset_t saved_sigmask;	/* restored if set_restore_sigmask() was used */
14321da177e4SLinus Torvalds 	struct sigpending pending;
14331da177e4SLinus Torvalds 
14341da177e4SLinus Torvalds 	unsigned long sas_ss_sp;
14351da177e4SLinus Torvalds 	size_t sas_ss_size;
14362a742138SStas Sergeev 	unsigned sas_ss_flags;
14372e01fabeSOleg Nesterov 
143867d12145SAl Viro 	struct callback_head *task_works;
1439e73f8959SOleg Nesterov 
14401da177e4SLinus Torvalds 	struct audit_context *audit_context;
1441bfef93a5SAl Viro #ifdef CONFIG_AUDITSYSCALL
1442e1760bd5SEric W. Biederman 	kuid_t loginuid;
14434746ec5bSEric Paris 	unsigned int sessionid;
1444bfef93a5SAl Viro #endif
1445932ecebbSWill Drewry 	struct seccomp seccomp;
14461da177e4SLinus Torvalds 
14471da177e4SLinus Torvalds /* Thread group tracking */
14481da177e4SLinus Torvalds    	u32 parent_exec_id;
14491da177e4SLinus Torvalds    	u32 self_exec_id;
145058568d2aSMiao Xie /* Protection of (de-)allocation: mm, files, fs, tty, keyrings, mems_allowed,
145158568d2aSMiao Xie  * mempolicy */
14521da177e4SLinus Torvalds 	spinlock_t alloc_lock;
14531da177e4SLinus Torvalds 
1454b29739f9SIngo Molnar 	/* Protection of the PI data structures: */
14551d615482SThomas Gleixner 	raw_spinlock_t pi_lock;
1456b29739f9SIngo Molnar 
145776751049SPeter Zijlstra 	struct wake_q_node wake_q;
145876751049SPeter Zijlstra 
145923f78d4aSIngo Molnar #ifdef CONFIG_RT_MUTEXES
146023f78d4aSIngo Molnar 	/* PI waiters blocked on a rt_mutex held by this task */
1461fb00aca4SPeter Zijlstra 	struct rb_root pi_waiters;
1462fb00aca4SPeter Zijlstra 	struct rb_node *pi_waiters_leftmost;
146323f78d4aSIngo Molnar 	/* Deadlock detection and priority inheritance handling */
146423f78d4aSIngo Molnar 	struct rt_mutex_waiter *pi_blocked_on;
146523f78d4aSIngo Molnar #endif
146623f78d4aSIngo Molnar 
1467408894eeSIngo Molnar #ifdef CONFIG_DEBUG_MUTEXES
1468408894eeSIngo Molnar 	/* mutex deadlock detection */
1469408894eeSIngo Molnar 	struct mutex_waiter *blocked_on;
1470408894eeSIngo Molnar #endif
1471de30a2b3SIngo Molnar #ifdef CONFIG_TRACE_IRQFLAGS
1472de30a2b3SIngo Molnar 	unsigned int irq_events;
1473de30a2b3SIngo Molnar 	unsigned long hardirq_enable_ip;
1474de30a2b3SIngo Molnar 	unsigned long hardirq_disable_ip;
1475fa1452e8SHiroshi Shimamoto 	unsigned int hardirq_enable_event;
1476de30a2b3SIngo Molnar 	unsigned int hardirq_disable_event;
1477fa1452e8SHiroshi Shimamoto 	int hardirqs_enabled;
1478de30a2b3SIngo Molnar 	int hardirq_context;
1479fa1452e8SHiroshi Shimamoto 	unsigned long softirq_disable_ip;
1480fa1452e8SHiroshi Shimamoto 	unsigned long softirq_enable_ip;
1481fa1452e8SHiroshi Shimamoto 	unsigned int softirq_disable_event;
1482fa1452e8SHiroshi Shimamoto 	unsigned int softirq_enable_event;
1483fa1452e8SHiroshi Shimamoto 	int softirqs_enabled;
1484de30a2b3SIngo Molnar 	int softirq_context;
1485de30a2b3SIngo Molnar #endif
1486fbb9ce95SIngo Molnar #ifdef CONFIG_LOCKDEP
1487bdb9441eSPeter Zijlstra # define MAX_LOCK_DEPTH 48UL
1488fbb9ce95SIngo Molnar 	u64 curr_chain_key;
1489fbb9ce95SIngo Molnar 	int lockdep_depth;
1490fbb9ce95SIngo Molnar 	unsigned int lockdep_recursion;
1491c7aceabaSRichard Kennedy 	struct held_lock held_locks[MAX_LOCK_DEPTH];
1492cf40bd16SNick Piggin 	gfp_t lockdep_reclaim_gfp;
1493fbb9ce95SIngo Molnar #endif
1494c6d30853SAndrey Ryabinin #ifdef CONFIG_UBSAN
1495c6d30853SAndrey Ryabinin 	unsigned int in_ubsan;
1496c6d30853SAndrey Ryabinin #endif
1497408894eeSIngo Molnar 
14981da177e4SLinus Torvalds /* journalling filesystem info */
14991da177e4SLinus Torvalds 	void *journal_info;
15001da177e4SLinus Torvalds 
1501d89d8796SNeil Brown /* stacked block device info */
1502bddd87c7SAkinobu Mita 	struct bio_list *bio_list;
1503d89d8796SNeil Brown 
150473c10101SJens Axboe #ifdef CONFIG_BLOCK
150573c10101SJens Axboe /* stack plugging */
150673c10101SJens Axboe 	struct blk_plug *plug;
150773c10101SJens Axboe #endif
150873c10101SJens Axboe 
15091da177e4SLinus Torvalds /* VM state */
15101da177e4SLinus Torvalds 	struct reclaim_state *reclaim_state;
15111da177e4SLinus Torvalds 
15121da177e4SLinus Torvalds 	struct backing_dev_info *backing_dev_info;
15131da177e4SLinus Torvalds 
15141da177e4SLinus Torvalds 	struct io_context *io_context;
15151da177e4SLinus Torvalds 
15161da177e4SLinus Torvalds 	unsigned long ptrace_message;
15171da177e4SLinus Torvalds 	siginfo_t *last_siginfo; /* For ptrace use.  */
15187c3ab738SAndrew Morton 	struct task_io_accounting ioac;
15198f0ab514SJay Lan #if defined(CONFIG_TASK_XACCT)
15201da177e4SLinus Torvalds 	u64 acct_rss_mem1;	/* accumulated rss usage */
15211da177e4SLinus Torvalds 	u64 acct_vm_mem1;	/* accumulated virtual memory usage */
1522605dc2b3SFrederic Weisbecker 	u64 acct_timexpd;	/* stime + utime since last update */
15231da177e4SLinus Torvalds #endif
15241da177e4SLinus Torvalds #ifdef CONFIG_CPUSETS
152558568d2aSMiao Xie 	nodemask_t mems_allowed;	/* Protected by alloc_lock */
1526cc9a6c87SMel Gorman 	seqcount_t mems_allowed_seq;	/* Seqence no to catch updates */
1527825a46afSPaul Jackson 	int cpuset_mem_spread_rotor;
15286adef3ebSJack Steiner 	int cpuset_slab_spread_rotor;
15291da177e4SLinus Torvalds #endif
1530ddbcc7e8SPaul Menage #ifdef CONFIG_CGROUPS
1531817929ecSPaul Menage 	/* Control Group info protected by css_set_lock */
15322c392b8cSArnd Bergmann 	struct css_set __rcu *cgroups;
1533817929ecSPaul Menage 	/* cg_list protected by css_set_lock and tsk->alloc_lock */
1534817929ecSPaul Menage 	struct list_head cg_list;
1535ddbcc7e8SPaul Menage #endif
1536e02737d5SFenghua Yu #ifdef CONFIG_INTEL_RDT_A
1537e02737d5SFenghua Yu 	int closid;
1538e02737d5SFenghua Yu #endif
153942b2dd0aSAlexey Dobriyan #ifdef CONFIG_FUTEX
15400771dfefSIngo Molnar 	struct robust_list_head __user *robust_list;
154134f192c6SIngo Molnar #ifdef CONFIG_COMPAT
154234f192c6SIngo Molnar 	struct compat_robust_list_head __user *compat_robust_list;
154334f192c6SIngo Molnar #endif
1544c87e2837SIngo Molnar 	struct list_head pi_state_list;
1545c87e2837SIngo Molnar 	struct futex_pi_state *pi_state_cache;
154642b2dd0aSAlexey Dobriyan #endif
1547cdd6c482SIngo Molnar #ifdef CONFIG_PERF_EVENTS
15488dc85d54SPeter Zijlstra 	struct perf_event_context *perf_event_ctxp[perf_nr_task_contexts];
1549cdd6c482SIngo Molnar 	struct mutex perf_event_mutex;
1550cdd6c482SIngo Molnar 	struct list_head perf_event_list;
1551a63eaf34SPaul Mackerras #endif
15528f47b187SThomas Gleixner #ifdef CONFIG_DEBUG_PREEMPT
15538f47b187SThomas Gleixner 	unsigned long preempt_disable_ip;
15548f47b187SThomas Gleixner #endif
1555c7aceabaSRichard Kennedy #ifdef CONFIG_NUMA
155658568d2aSMiao Xie 	struct mempolicy *mempolicy;	/* Protected by alloc_lock */
1557c7aceabaSRichard Kennedy 	short il_next;
1558207205a2SEric Dumazet 	short pref_node_fork;
1559c7aceabaSRichard Kennedy #endif
1560cbee9f88SPeter Zijlstra #ifdef CONFIG_NUMA_BALANCING
1561cbee9f88SPeter Zijlstra 	int numa_scan_seq;
1562cbee9f88SPeter Zijlstra 	unsigned int numa_scan_period;
1563598f0ec0SMel Gorman 	unsigned int numa_scan_period_max;
1564de1c9ce6SRik van Riel 	int numa_preferred_nid;
15656b9a7460SMel Gorman 	unsigned long numa_migrate_retry;
1566cbee9f88SPeter Zijlstra 	u64 node_stamp;			/* migration stamp  */
15677e2703e6SRik van Riel 	u64 last_task_numa_placement;
15687e2703e6SRik van Riel 	u64 last_sum_exec_runtime;
1569cbee9f88SPeter Zijlstra 	struct callback_head numa_work;
1570f809ca9aSMel Gorman 
15718c8a743cSPeter Zijlstra 	struct list_head numa_entry;
15728c8a743cSPeter Zijlstra 	struct numa_group *numa_group;
15738c8a743cSPeter Zijlstra 
1574745d6147SMel Gorman 	/*
157544dba3d5SIulia Manda 	 * numa_faults is an array split into four regions:
157644dba3d5SIulia Manda 	 * faults_memory, faults_cpu, faults_memory_buffer, faults_cpu_buffer
157744dba3d5SIulia Manda 	 * in this precise order.
157844dba3d5SIulia Manda 	 *
157944dba3d5SIulia Manda 	 * faults_memory: Exponential decaying average of faults on a per-node
158044dba3d5SIulia Manda 	 * basis. Scheduling placement decisions are made based on these
158144dba3d5SIulia Manda 	 * counts. The values remain static for the duration of a PTE scan.
158244dba3d5SIulia Manda 	 * faults_cpu: Track the nodes the process was running on when a NUMA
158344dba3d5SIulia Manda 	 * hinting fault was incurred.
158444dba3d5SIulia Manda 	 * faults_memory_buffer and faults_cpu_buffer: Record faults per node
158544dba3d5SIulia Manda 	 * during the current scan window. When the scan completes, the counts
158644dba3d5SIulia Manda 	 * in faults_memory and faults_cpu decay and these values are copied.
1587745d6147SMel Gorman 	 */
158844dba3d5SIulia Manda 	unsigned long *numa_faults;
158983e1d2cdSMel Gorman 	unsigned long total_numa_faults;
1590745d6147SMel Gorman 
1591745d6147SMel Gorman 	/*
159204bb2f94SRik van Riel 	 * numa_faults_locality tracks if faults recorded during the last
1593074c2381SMel Gorman 	 * scan window were remote/local or failed to migrate. The task scan
1594074c2381SMel Gorman 	 * period is adapted based on the locality of the faults with different
1595074c2381SMel Gorman 	 * weights depending on whether they were shared or private faults
159604bb2f94SRik van Riel 	 */
1597074c2381SMel Gorman 	unsigned long numa_faults_locality[3];
159804bb2f94SRik van Riel 
1599b32e86b4SIngo Molnar 	unsigned long numa_pages_migrated;
1600cbee9f88SPeter Zijlstra #endif /* CONFIG_NUMA_BALANCING */
1601cbee9f88SPeter Zijlstra 
160272b252aeSMel Gorman #ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
160372b252aeSMel Gorman 	struct tlbflush_unmap_batch tlb_ubc;
160472b252aeSMel Gorman #endif
160572b252aeSMel Gorman 
1606e56d0903SIngo Molnar 	struct rcu_head rcu;
1607b92ce558SJens Axboe 
1608b92ce558SJens Axboe 	/*
1609b92ce558SJens Axboe 	 * cache last used pipe for splice
1610b92ce558SJens Axboe 	 */
1611b92ce558SJens Axboe 	struct pipe_inode_info *splice_pipe;
16125640f768SEric Dumazet 
16135640f768SEric Dumazet 	struct page_frag task_frag;
16145640f768SEric Dumazet 
1615ca74e92bSShailabh Nagar #ifdef	CONFIG_TASK_DELAY_ACCT
1616ca74e92bSShailabh Nagar 	struct task_delay_info *delays;
1617ca74e92bSShailabh Nagar #endif
1618f4f154fdSAkinobu Mita #ifdef CONFIG_FAULT_INJECTION
1619f4f154fdSAkinobu Mita 	int make_it_fail;
1620f4f154fdSAkinobu Mita #endif
16219d823e8fSWu Fengguang 	/*
16229d823e8fSWu Fengguang 	 * when (nr_dirtied >= nr_dirtied_pause), it's time to call
16239d823e8fSWu Fengguang 	 * balance_dirty_pages() for some dirty throttling pause
16249d823e8fSWu Fengguang 	 */
16259d823e8fSWu Fengguang 	int nr_dirtied;
16269d823e8fSWu Fengguang 	int nr_dirtied_pause;
162783712358SWu Fengguang 	unsigned long dirty_paused_when; /* start of a write-and-pause period */
16289d823e8fSWu Fengguang 
16299745512cSArjan van de Ven #ifdef CONFIG_LATENCYTOP
16309745512cSArjan van de Ven 	int latency_record_count;
16319745512cSArjan van de Ven 	struct latency_record latency_record[LT_SAVECOUNT];
16329745512cSArjan van de Ven #endif
16336976675dSArjan van de Ven 	/*
16346976675dSArjan van de Ven 	 * time slack values; these are used to round up poll() and
16356976675dSArjan van de Ven 	 * select() etc timeout values. These are in nanoseconds.
16366976675dSArjan van de Ven 	 */
1637da8b44d5SJohn Stultz 	u64 timer_slack_ns;
1638da8b44d5SJohn Stultz 	u64 default_timer_slack_ns;
1639f8d570a4SDavid Miller 
16400b24beccSAndrey Ryabinin #ifdef CONFIG_KASAN
16410b24beccSAndrey Ryabinin 	unsigned int kasan_depth;
16420b24beccSAndrey Ryabinin #endif
1643fb52607aSFrederic Weisbecker #ifdef CONFIG_FUNCTION_GRAPH_TRACER
16443ad2f3fbSDaniel Mack 	/* Index of current stored address in ret_stack */
1645f201ae23SFrederic Weisbecker 	int curr_ret_stack;
1646f201ae23SFrederic Weisbecker 	/* Stack of return addresses for return function tracing */
1647f201ae23SFrederic Weisbecker 	struct ftrace_ret_stack	*ret_stack;
16488aef2d28SSteven Rostedt 	/* time stamp for last schedule */
16498aef2d28SSteven Rostedt 	unsigned long long ftrace_timestamp;
1650f201ae23SFrederic Weisbecker 	/*
1651f201ae23SFrederic Weisbecker 	 * Number of functions that haven't been traced
1652f201ae23SFrederic Weisbecker 	 * because of depth overrun.
1653f201ae23SFrederic Weisbecker 	 */
1654f201ae23SFrederic Weisbecker 	atomic_t trace_overrun;
1655380c4b14SFrederic Weisbecker 	/* Pause for the tracing */
1656380c4b14SFrederic Weisbecker 	atomic_t tracing_graph_pause;
1657f201ae23SFrederic Weisbecker #endif
1658ea4e2bc4SSteven Rostedt #ifdef CONFIG_TRACING
1659ea4e2bc4SSteven Rostedt 	/* state flags for use by tracers */
1660ea4e2bc4SSteven Rostedt 	unsigned long trace;
1661b1cff0adSSteven Rostedt 	/* bitmask and counter of trace recursion */
1662261842b7SSteven Rostedt 	unsigned long trace_recursion;
1663261842b7SSteven Rostedt #endif /* CONFIG_TRACING */
16645c9a8750SDmitry Vyukov #ifdef CONFIG_KCOV
16655c9a8750SDmitry Vyukov 	/* Coverage collection mode enabled for this task (0 if disabled). */
16665c9a8750SDmitry Vyukov 	enum kcov_mode kcov_mode;
16675c9a8750SDmitry Vyukov 	/* Size of the kcov_area. */
16685c9a8750SDmitry Vyukov 	unsigned	kcov_size;
16695c9a8750SDmitry Vyukov 	/* Buffer for coverage collection. */
16705c9a8750SDmitry Vyukov 	void		*kcov_area;
16715c9a8750SDmitry Vyukov 	/* kcov desciptor wired with this task or NULL. */
16725c9a8750SDmitry Vyukov 	struct kcov	*kcov;
16735c9a8750SDmitry Vyukov #endif
16746f185c29SVladimir Davydov #ifdef CONFIG_MEMCG
1675626ebc41STejun Heo 	struct mem_cgroup *memcg_in_oom;
1676626ebc41STejun Heo 	gfp_t memcg_oom_gfp_mask;
1677626ebc41STejun Heo 	int memcg_oom_order;
1678b23afb93STejun Heo 
1679b23afb93STejun Heo 	/* number of pages to reclaim on returning to userland */
1680b23afb93STejun Heo 	unsigned int memcg_nr_pages_over_high;
1681569b846dSKAMEZAWA Hiroyuki #endif
16820326f5a9SSrikar Dronamraju #ifdef CONFIG_UPROBES
16830326f5a9SSrikar Dronamraju 	struct uprobe_task *utask;
16840326f5a9SSrikar Dronamraju #endif
1685cafe5635SKent Overstreet #if defined(CONFIG_BCACHE) || defined(CONFIG_BCACHE_MODULE)
1686cafe5635SKent Overstreet 	unsigned int	sequential_io;
1687cafe5635SKent Overstreet 	unsigned int	sequential_io_avg;
1688cafe5635SKent Overstreet #endif
16898eb23b9fSPeter Zijlstra #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
16908eb23b9fSPeter Zijlstra 	unsigned long	task_state_change;
16918eb23b9fSPeter Zijlstra #endif
16928bcbde54SDavid Hildenbrand 	int pagefault_disabled;
169303049269SMichal Hocko #ifdef CONFIG_MMU
169429c696e1SVladimir Davydov 	struct task_struct *oom_reaper_list;
169503049269SMichal Hocko #endif
1696ba14a194SAndy Lutomirski #ifdef CONFIG_VMAP_STACK
1697ba14a194SAndy Lutomirski 	struct vm_struct *stack_vm_area;
1698ba14a194SAndy Lutomirski #endif
169968f24b08SAndy Lutomirski #ifdef CONFIG_THREAD_INFO_IN_TASK
170068f24b08SAndy Lutomirski 	/* A live task holds one reference. */
170168f24b08SAndy Lutomirski 	atomic_t stack_refcount;
170268f24b08SAndy Lutomirski #endif
17030c8c0f03SDave Hansen /* CPU-specific state of this task */
17040c8c0f03SDave Hansen 	struct thread_struct thread;
17050c8c0f03SDave Hansen /*
17060c8c0f03SDave Hansen  * WARNING: on x86, 'thread_struct' contains a variable-sized
17070c8c0f03SDave Hansen  * structure.  It *MUST* be at the end of 'task_struct'.
17080c8c0f03SDave Hansen  *
17090c8c0f03SDave Hansen  * Do not put anything below here!
17100c8c0f03SDave Hansen  */
17111da177e4SLinus Torvalds };
17121da177e4SLinus Torvalds 
17135aaeb5c0SIngo Molnar #ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT
17145aaeb5c0SIngo Molnar extern int arch_task_struct_size __read_mostly;
17155aaeb5c0SIngo Molnar #else
17165aaeb5c0SIngo Molnar # define arch_task_struct_size (sizeof(struct task_struct))
17175aaeb5c0SIngo Molnar #endif
17180c8c0f03SDave Hansen 
1719ba14a194SAndy Lutomirski #ifdef CONFIG_VMAP_STACK
1720ba14a194SAndy Lutomirski static inline struct vm_struct *task_stack_vm_area(const struct task_struct *t)
1721ba14a194SAndy Lutomirski {
1722ba14a194SAndy Lutomirski 	return t->stack_vm_area;
1723ba14a194SAndy Lutomirski }
1724ba14a194SAndy Lutomirski #else
1725ba14a194SAndy Lutomirski static inline struct vm_struct *task_stack_vm_area(const struct task_struct *t)
1726ba14a194SAndy Lutomirski {
1727ba14a194SAndy Lutomirski 	return NULL;
1728ba14a194SAndy Lutomirski }
1729ba14a194SAndy Lutomirski #endif
1730ba14a194SAndy Lutomirski 
17316688cc05SPeter Zijlstra #define TNF_MIGRATED	0x01
17326688cc05SPeter Zijlstra #define TNF_NO_GROUP	0x02
1733dabe1d99SRik van Riel #define TNF_SHARED	0x04
173404bb2f94SRik van Riel #define TNF_FAULT_LOCAL	0x08
1735074c2381SMel Gorman #define TNF_MIGRATE_FAIL 0x10
17366688cc05SPeter Zijlstra 
1737b18dc5f2SMichal Hocko static inline bool in_vfork(struct task_struct *tsk)
1738b18dc5f2SMichal Hocko {
1739b18dc5f2SMichal Hocko 	bool ret;
1740b18dc5f2SMichal Hocko 
1741b18dc5f2SMichal Hocko 	/*
1742b18dc5f2SMichal Hocko 	 * need RCU to access ->real_parent if CLONE_VM was used along with
1743b18dc5f2SMichal Hocko 	 * CLONE_PARENT.
1744b18dc5f2SMichal Hocko 	 *
1745b18dc5f2SMichal Hocko 	 * We check real_parent->mm == tsk->mm because CLONE_VFORK does not
1746b18dc5f2SMichal Hocko 	 * imply CLONE_VM
1747b18dc5f2SMichal Hocko 	 *
1748b18dc5f2SMichal Hocko 	 * CLONE_VFORK can be used with CLONE_PARENT/CLONE_THREAD and thus
1749b18dc5f2SMichal Hocko 	 * ->real_parent is not necessarily the task doing vfork(), so in
1750b18dc5f2SMichal Hocko 	 * theory we can't rely on task_lock() if we want to dereference it.
1751b18dc5f2SMichal Hocko 	 *
1752b18dc5f2SMichal Hocko 	 * And in this case we can't trust the real_parent->mm == tsk->mm
1753b18dc5f2SMichal Hocko 	 * check, it can be false negative. But we do not care, if init or
1754b18dc5f2SMichal Hocko 	 * another oom-unkillable task does this it should blame itself.
1755b18dc5f2SMichal Hocko 	 */
1756b18dc5f2SMichal Hocko 	rcu_read_lock();
1757b18dc5f2SMichal Hocko 	ret = tsk->vfork_done && tsk->real_parent->mm == tsk->mm;
1758b18dc5f2SMichal Hocko 	rcu_read_unlock();
1759b18dc5f2SMichal Hocko 
1760b18dc5f2SMichal Hocko 	return ret;
1761b18dc5f2SMichal Hocko }
1762b18dc5f2SMichal Hocko 
1763cbee9f88SPeter Zijlstra #ifdef CONFIG_NUMA_BALANCING
17646688cc05SPeter Zijlstra extern void task_numa_fault(int last_node, int node, int pages, int flags);
1765e29cf08bSMel Gorman extern pid_t task_numa_group_id(struct task_struct *p);
17661a687c2eSMel Gorman extern void set_numabalancing_state(bool enabled);
176782727018SRik van Riel extern void task_numa_free(struct task_struct *p);
176810f39042SRik van Riel extern bool should_numa_migrate_memory(struct task_struct *p, struct page *page,
176910f39042SRik van Riel 					int src_nid, int dst_cpu);
1770cbee9f88SPeter Zijlstra #else
1771ac8e895bSMel Gorman static inline void task_numa_fault(int last_node, int node, int pages,
17726688cc05SPeter Zijlstra 				   int flags)
1773cbee9f88SPeter Zijlstra {
1774cbee9f88SPeter Zijlstra }
1775e29cf08bSMel Gorman static inline pid_t task_numa_group_id(struct task_struct *p)
1776e29cf08bSMel Gorman {
1777e29cf08bSMel Gorman 	return 0;
1778e29cf08bSMel Gorman }
17791a687c2eSMel Gorman static inline void set_numabalancing_state(bool enabled)
17801a687c2eSMel Gorman {
17811a687c2eSMel Gorman }
178282727018SRik van Riel static inline void task_numa_free(struct task_struct *p)
178382727018SRik van Riel {
178482727018SRik van Riel }
178510f39042SRik van Riel static inline bool should_numa_migrate_memory(struct task_struct *p,
178610f39042SRik van Riel 				struct page *page, int src_nid, int dst_cpu)
178710f39042SRik van Riel {
178810f39042SRik van Riel 	return true;
178910f39042SRik van Riel }
1790cbee9f88SPeter Zijlstra #endif
1791cbee9f88SPeter Zijlstra 
1792e868171aSAlexey Dobriyan static inline struct pid *task_pid(struct task_struct *task)
179322c935f4SEric W. Biederman {
179422c935f4SEric W. Biederman 	return task->pids[PIDTYPE_PID].pid;
179522c935f4SEric W. Biederman }
179622c935f4SEric W. Biederman 
1797e868171aSAlexey Dobriyan static inline struct pid *task_tgid(struct task_struct *task)
179822c935f4SEric W. Biederman {
179922c935f4SEric W. Biederman 	return task->group_leader->pids[PIDTYPE_PID].pid;
180022c935f4SEric W. Biederman }
180122c935f4SEric W. Biederman 
18026dda81f4SOleg Nesterov /*
18036dda81f4SOleg Nesterov  * Without tasklist or rcu lock it is not safe to dereference
18046dda81f4SOleg Nesterov  * the result of task_pgrp/task_session even if task == current,
18056dda81f4SOleg Nesterov  * we can race with another thread doing sys_setsid/sys_setpgid.
18066dda81f4SOleg Nesterov  */
1807e868171aSAlexey Dobriyan static inline struct pid *task_pgrp(struct task_struct *task)
180822c935f4SEric W. Biederman {
180922c935f4SEric W. Biederman 	return task->group_leader->pids[PIDTYPE_PGID].pid;
181022c935f4SEric W. Biederman }
181122c935f4SEric W. Biederman 
1812e868171aSAlexey Dobriyan static inline struct pid *task_session(struct task_struct *task)
181322c935f4SEric W. Biederman {
181422c935f4SEric W. Biederman 	return task->group_leader->pids[PIDTYPE_SID].pid;
181522c935f4SEric W. Biederman }
181622c935f4SEric W. Biederman 
18177af57294SPavel Emelyanov struct pid_namespace;
18187af57294SPavel Emelyanov 
18197af57294SPavel Emelyanov /*
18207af57294SPavel Emelyanov  * the helpers to get the task's different pids as they are seen
18217af57294SPavel Emelyanov  * from various namespaces
18227af57294SPavel Emelyanov  *
18237af57294SPavel Emelyanov  * task_xid_nr()     : global id, i.e. the id seen from the init namespace;
182444c4e1b2SEric W. Biederman  * task_xid_vnr()    : virtual id, i.e. the id seen from the pid namespace of
182544c4e1b2SEric W. Biederman  *                     current.
18267af57294SPavel Emelyanov  * task_xid_nr_ns()  : id seen from the ns specified;
18277af57294SPavel Emelyanov  *
18287af57294SPavel Emelyanov  * set_task_vxid()   : assigns a virtual id to a task;
18297af57294SPavel Emelyanov  *
18307af57294SPavel Emelyanov  * see also pid_nr() etc in include/linux/pid.h
18317af57294SPavel Emelyanov  */
183252ee2dfdSOleg Nesterov pid_t __task_pid_nr_ns(struct task_struct *task, enum pid_type type,
183352ee2dfdSOleg Nesterov 			struct pid_namespace *ns);
18347af57294SPavel Emelyanov 
1835e868171aSAlexey Dobriyan static inline pid_t task_pid_nr(struct task_struct *tsk)
18367af57294SPavel Emelyanov {
18377af57294SPavel Emelyanov 	return tsk->pid;
18387af57294SPavel Emelyanov }
18397af57294SPavel Emelyanov 
184052ee2dfdSOleg Nesterov static inline pid_t task_pid_nr_ns(struct task_struct *tsk,
184152ee2dfdSOleg Nesterov 					struct pid_namespace *ns)
184252ee2dfdSOleg Nesterov {
184352ee2dfdSOleg Nesterov 	return __task_pid_nr_ns(tsk, PIDTYPE_PID, ns);
184452ee2dfdSOleg Nesterov }
18457af57294SPavel Emelyanov 
18467af57294SPavel Emelyanov static inline pid_t task_pid_vnr(struct task_struct *tsk)
18477af57294SPavel Emelyanov {
184852ee2dfdSOleg Nesterov 	return __task_pid_nr_ns(tsk, PIDTYPE_PID, NULL);
18497af57294SPavel Emelyanov }
18507af57294SPavel Emelyanov 
18517af57294SPavel Emelyanov 
1852e868171aSAlexey Dobriyan static inline pid_t task_tgid_nr(struct task_struct *tsk)
18537af57294SPavel Emelyanov {
18547af57294SPavel Emelyanov 	return tsk->tgid;
18557af57294SPavel Emelyanov }
18567af57294SPavel Emelyanov 
18572f2a3a46SPavel Emelyanov pid_t task_tgid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns);
18587af57294SPavel Emelyanov 
18597af57294SPavel Emelyanov static inline pid_t task_tgid_vnr(struct task_struct *tsk)
18607af57294SPavel Emelyanov {
18617af57294SPavel Emelyanov 	return pid_vnr(task_tgid(tsk));
18627af57294SPavel Emelyanov }
18637af57294SPavel Emelyanov 
18647af57294SPavel Emelyanov 
186580e0b6e8SRichard Guy Briggs static inline int pid_alive(const struct task_struct *p);
1866ad36d282SRichard Guy Briggs static inline pid_t task_ppid_nr_ns(const struct task_struct *tsk, struct pid_namespace *ns)
1867ad36d282SRichard Guy Briggs {
1868ad36d282SRichard Guy Briggs 	pid_t pid = 0;
1869ad36d282SRichard Guy Briggs 
1870ad36d282SRichard Guy Briggs 	rcu_read_lock();
1871ad36d282SRichard Guy Briggs 	if (pid_alive(tsk))
1872ad36d282SRichard Guy Briggs 		pid = task_tgid_nr_ns(rcu_dereference(tsk->real_parent), ns);
1873ad36d282SRichard Guy Briggs 	rcu_read_unlock();
1874ad36d282SRichard Guy Briggs 
1875ad36d282SRichard Guy Briggs 	return pid;
1876ad36d282SRichard Guy Briggs }
1877ad36d282SRichard Guy Briggs 
1878ad36d282SRichard Guy Briggs static inline pid_t task_ppid_nr(const struct task_struct *tsk)
1879ad36d282SRichard Guy Briggs {
1880ad36d282SRichard Guy Briggs 	return task_ppid_nr_ns(tsk, &init_pid_ns);
1881ad36d282SRichard Guy Briggs }
1882ad36d282SRichard Guy Briggs 
188352ee2dfdSOleg Nesterov static inline pid_t task_pgrp_nr_ns(struct task_struct *tsk,
188452ee2dfdSOleg Nesterov 					struct pid_namespace *ns)
18857af57294SPavel Emelyanov {
188652ee2dfdSOleg Nesterov 	return __task_pid_nr_ns(tsk, PIDTYPE_PGID, ns);
18877af57294SPavel Emelyanov }
18887af57294SPavel Emelyanov 
18897af57294SPavel Emelyanov static inline pid_t task_pgrp_vnr(struct task_struct *tsk)
18907af57294SPavel Emelyanov {
189152ee2dfdSOleg Nesterov 	return __task_pid_nr_ns(tsk, PIDTYPE_PGID, NULL);
18927af57294SPavel Emelyanov }
18937af57294SPavel Emelyanov 
18947af57294SPavel Emelyanov 
189552ee2dfdSOleg Nesterov static inline pid_t task_session_nr_ns(struct task_struct *tsk,
189652ee2dfdSOleg Nesterov 					struct pid_namespace *ns)
18977af57294SPavel Emelyanov {
189852ee2dfdSOleg Nesterov 	return __task_pid_nr_ns(tsk, PIDTYPE_SID, ns);
18997af57294SPavel Emelyanov }
19007af57294SPavel Emelyanov 
19017af57294SPavel Emelyanov static inline pid_t task_session_vnr(struct task_struct *tsk)
19027af57294SPavel Emelyanov {
190352ee2dfdSOleg Nesterov 	return __task_pid_nr_ns(tsk, PIDTYPE_SID, NULL);
19047af57294SPavel Emelyanov }
19057af57294SPavel Emelyanov 
19061b0f7ffdSOleg Nesterov /* obsolete, do not use */
19071b0f7ffdSOleg Nesterov static inline pid_t task_pgrp_nr(struct task_struct *tsk)
19081b0f7ffdSOleg Nesterov {
19091b0f7ffdSOleg Nesterov 	return task_pgrp_nr_ns(tsk, &init_pid_ns);
19101b0f7ffdSOleg Nesterov }
19117af57294SPavel Emelyanov 
19121da177e4SLinus Torvalds /**
19131da177e4SLinus Torvalds  * pid_alive - check that a task structure is not stale
19141da177e4SLinus Torvalds  * @p: Task structure to be checked.
19151da177e4SLinus Torvalds  *
19161da177e4SLinus Torvalds  * Test if a process is not yet dead (at most zombie state)
19171da177e4SLinus Torvalds  * If pid_alive fails, then pointers within the task structure
19181da177e4SLinus Torvalds  * can be stale and must not be dereferenced.
1919e69f6186SYacine Belkadi  *
1920e69f6186SYacine Belkadi  * Return: 1 if the process is alive. 0 otherwise.
19211da177e4SLinus Torvalds  */
1922ad36d282SRichard Guy Briggs static inline int pid_alive(const struct task_struct *p)
19231da177e4SLinus Torvalds {
192492476d7fSEric W. Biederman 	return p->pids[PIDTYPE_PID].pid != NULL;
19251da177e4SLinus Torvalds }
19261da177e4SLinus Torvalds 
1927f400e198SSukadev Bhattiprolu /**
1928570f5241SSergey Senozhatsky  * is_global_init - check if a task structure is init. Since init
1929570f5241SSergey Senozhatsky  * is free to have sub-threads we need to check tgid.
19303260259fSHenne  * @tsk: Task structure to be checked.
19313260259fSHenne  *
19323260259fSHenne  * Check if a task structure is the first user space task the kernel created.
1933e69f6186SYacine Belkadi  *
1934e69f6186SYacine Belkadi  * Return: 1 if the task structure is init. 0 otherwise.
1935f400e198SSukadev Bhattiprolu  */
1936e868171aSAlexey Dobriyan static inline int is_global_init(struct task_struct *tsk)
1937b461cc03SPavel Emelyanov {
1938570f5241SSergey Senozhatsky 	return task_tgid_nr(tsk) == 1;
1939b461cc03SPavel Emelyanov }
1940b460cbc5SSerge E. Hallyn 
19419ec52099SCedric Le Goater extern struct pid *cad_pid;
19429ec52099SCedric Le Goater 
19431da177e4SLinus Torvalds extern void free_task(struct task_struct *tsk);
19441da177e4SLinus Torvalds #define get_task_struct(tsk) do { atomic_inc(&(tsk)->usage); } while(0)
1945e56d0903SIngo Molnar 
1946158d9ebdSAndrew Morton extern void __put_task_struct(struct task_struct *t);
1947e56d0903SIngo Molnar 
1948e56d0903SIngo Molnar static inline void put_task_struct(struct task_struct *t)
1949e56d0903SIngo Molnar {
1950e56d0903SIngo Molnar 	if (atomic_dec_and_test(&t->usage))
19518c7904a0SEric W. Biederman 		__put_task_struct(t);
1952e56d0903SIngo Molnar }
19531da177e4SLinus Torvalds 
1954150593bfSOleg Nesterov struct task_struct *task_rcu_dereference(struct task_struct **ptask);
1955150593bfSOleg Nesterov struct task_struct *try_get_task_struct(struct task_struct **ptask);
1956150593bfSOleg Nesterov 
19576a61671bSFrederic Weisbecker #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
19586a61671bSFrederic Weisbecker extern void task_cputime(struct task_struct *t,
19595613fda9SFrederic Weisbecker 			 u64 *utime, u64 *stime);
196016a6d9beSFrederic Weisbecker extern u64 task_gtime(struct task_struct *t);
19616a61671bSFrederic Weisbecker #else
19626fac4829SFrederic Weisbecker static inline void task_cputime(struct task_struct *t,
19635613fda9SFrederic Weisbecker 				u64 *utime, u64 *stime)
19646fac4829SFrederic Weisbecker {
19656fac4829SFrederic Weisbecker 	*utime = t->utime;
19666fac4829SFrederic Weisbecker 	*stime = t->stime;
19676fac4829SFrederic Weisbecker }
19686fac4829SFrederic Weisbecker 
196916a6d9beSFrederic Weisbecker static inline u64 task_gtime(struct task_struct *t)
19706a61671bSFrederic Weisbecker {
19716a61671bSFrederic Weisbecker 	return t->gtime;
19726a61671bSFrederic Weisbecker }
19736a61671bSFrederic Weisbecker #endif
197440565b5aSStanislaw Gruszka 
197540565b5aSStanislaw Gruszka #ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
1976b31dc66aSJens Axboe static inline void task_cputime_scaled(struct task_struct *t,
19775613fda9SFrederic Weisbecker 				       u64 *utimescaled,
19785613fda9SFrederic Weisbecker 				       u64 *stimescaled)
1979b31dc66aSJens Axboe {
19804db96cf0SAndi Kleen 	*utimescaled = t->utimescaled;
198161a87122SThomas Gleixner 	*stimescaled = t->stimescaled;
198258a69cb4STejun Heo }
198340565b5aSStanislaw Gruszka #else
198440565b5aSStanislaw Gruszka static inline void task_cputime_scaled(struct task_struct *t,
19855613fda9SFrederic Weisbecker 				       u64 *utimescaled,
19865613fda9SFrederic Weisbecker 				       u64 *stimescaled)
19871da177e4SLinus Torvalds {
198840565b5aSStanislaw Gruszka 	task_cputime(t, utimescaled, stimescaled);
19891da177e4SLinus Torvalds }
19901da177e4SLinus Torvalds #endif
199140565b5aSStanislaw Gruszka 
19925613fda9SFrederic Weisbecker extern void task_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st);
19935613fda9SFrederic Weisbecker extern void thread_group_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st);
19941da177e4SLinus Torvalds 
19951da177e4SLinus Torvalds /*
19961da177e4SLinus Torvalds  * Per process flags
19971da177e4SLinus Torvalds  */
1998c1de45caSPeter Zijlstra #define PF_IDLE		0x00000002	/* I am an IDLE thread */
19991da177e4SLinus Torvalds #define PF_EXITING	0x00000004	/* getting shut down */
20001da177e4SLinus Torvalds #define PF_EXITPIDONE	0x00000008	/* pi exit done on shut down */
20011da177e4SLinus Torvalds #define PF_VCPU		0x00000010	/* I'm a virtual CPU */
20021da177e4SLinus Torvalds #define PF_WQ_WORKER	0x00000020	/* I'm a workqueue worker */
20031da177e4SLinus Torvalds #define PF_FORKNOEXEC	0x00000040	/* forked but didn't exec */
20041da177e4SLinus Torvalds #define PF_MCE_PROCESS  0x00000080      /* process policy on mce errors */
20051da177e4SLinus Torvalds #define PF_SUPERPRIV	0x00000100	/* used super-user privileges */
20061da177e4SLinus Torvalds #define PF_DUMPCORE	0x00000200	/* dumped core */
20071da177e4SLinus Torvalds #define PF_SIGNALED	0x00000400	/* killed by a signal */
20081da177e4SLinus Torvalds #define PF_MEMALLOC	0x00000800	/* Allocating memory */
20091da177e4SLinus Torvalds #define PF_NPROC_EXCEEDED 0x00001000	/* set_user noticed that RLIMIT_NPROC was exceeded */
20101da177e4SLinus Torvalds #define PF_USED_MATH	0x00002000	/* if unset the fpu must be initialized before use */
20111da177e4SLinus Torvalds #define PF_USED_ASYNC	0x00004000	/* used async_schedule*(), used by module init */
20121da177e4SLinus Torvalds #define PF_NOFREEZE	0x00008000	/* this thread should not be frozen */
20131da177e4SLinus Torvalds #define PF_FROZEN	0x00010000	/* frozen for system suspend */
20141da177e4SLinus Torvalds #define PF_FSTRANS	0x00020000	/* inside a filesystem transaction */
20151da177e4SLinus Torvalds #define PF_KSWAPD	0x00040000	/* I am kswapd */
201621caf2fcSMing Lei #define PF_MEMALLOC_NOIO 0x00080000	/* Allocating memory without IO involved */
20171da177e4SLinus Torvalds #define PF_LESS_THROTTLE 0x00100000	/* Throttle me less: I clean memory */
20181da177e4SLinus Torvalds #define PF_KTHREAD	0x00200000	/* I am a kernel thread */
20191da177e4SLinus Torvalds #define PF_RANDOMIZE	0x00400000	/* randomize virtual address space */
20201da177e4SLinus Torvalds #define PF_SWAPWRITE	0x00800000	/* Allowed to write to swap */
202114a40ffcSTejun Heo #define PF_NO_SETAFFINITY 0x04000000	/* Userland is not allowed to meddle with cpus_allowed */
20221da177e4SLinus Torvalds #define PF_MCE_EARLY    0x08000000      /* Early kill for mce process policy */
20231da177e4SLinus Torvalds #define PF_MUTEX_TESTER	0x20000000	/* Thread belongs to the rt mutex tester */
20241da177e4SLinus Torvalds #define PF_FREEZER_SKIP	0x40000000	/* Freezer should not count it as freezable */
20252b44c4dbSColin Cross #define PF_SUSPEND_TASK 0x80000000      /* this thread called freeze_processes and should not be frozen */
20261da177e4SLinus Torvalds 
20271da177e4SLinus Torvalds /*
20281da177e4SLinus Torvalds  * Only the _current_ task can read/write to tsk->flags, but other
20291da177e4SLinus Torvalds  * tasks can access tsk->flags in readonly mode for example
20301da177e4SLinus Torvalds  * with tsk_used_math (like during threaded core dumping).
20311da177e4SLinus Torvalds  * There is however an exception to this rule during ptrace
20321da177e4SLinus Torvalds  * or during fork: the ptracer task is allowed to write to the
20331da177e4SLinus Torvalds  * child->flags of its traced child (same goes for fork, the parent
20341da177e4SLinus Torvalds  * can write to the child->flags), because we're guaranteed the
20351da177e4SLinus Torvalds  * child is not running and in turn not changing child->flags
20361da177e4SLinus Torvalds  * at the same time the parent does it.
20371da177e4SLinus Torvalds  */
20381da177e4SLinus Torvalds #define clear_stopped_child_used_math(child) do { (child)->flags &= ~PF_USED_MATH; } while (0)
20391da177e4SLinus Torvalds #define set_stopped_child_used_math(child) do { (child)->flags |= PF_USED_MATH; } while (0)
20401da177e4SLinus Torvalds #define clear_used_math() clear_stopped_child_used_math(current)
20411da177e4SLinus Torvalds #define set_used_math() set_stopped_child_used_math(current)
20421da177e4SLinus Torvalds #define conditional_stopped_child_used_math(condition, child) \
20431da177e4SLinus Torvalds 	do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= (condition) ? PF_USED_MATH : 0; } while (0)
20441da177e4SLinus Torvalds #define conditional_used_math(condition) \
20451da177e4SLinus Torvalds 	conditional_stopped_child_used_math(condition, current)
20461da177e4SLinus Torvalds #define copy_to_stopped_child_used_math(child) \
20471da177e4SLinus Torvalds 	do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= current->flags & PF_USED_MATH; } while (0)
20481da177e4SLinus Torvalds /* NOTE: this will return 0 or PF_USED_MATH, it will never return 1 */
20491da177e4SLinus Torvalds #define tsk_used_math(p) ((p)->flags & PF_USED_MATH)
20501da177e4SLinus Torvalds #define used_math() tsk_used_math(current)
20511da177e4SLinus Torvalds 
2052934f3072SJunxiao Bi /* __GFP_IO isn't allowed if PF_MEMALLOC_NOIO is set in current->flags
2053934f3072SJunxiao Bi  * __GFP_FS is also cleared as it implies __GFP_IO.
2054934f3072SJunxiao Bi  */
205521caf2fcSMing Lei static inline gfp_t memalloc_noio_flags(gfp_t flags)
205621caf2fcSMing Lei {
205721caf2fcSMing Lei 	if (unlikely(current->flags & PF_MEMALLOC_NOIO))
2058934f3072SJunxiao Bi 		flags &= ~(__GFP_IO | __GFP_FS);
205921caf2fcSMing Lei 	return flags;
206021caf2fcSMing Lei }
206121caf2fcSMing Lei 
206221caf2fcSMing Lei static inline unsigned int memalloc_noio_save(void)
206321caf2fcSMing Lei {
206421caf2fcSMing Lei 	unsigned int flags = current->flags & PF_MEMALLOC_NOIO;
206521caf2fcSMing Lei 	current->flags |= PF_MEMALLOC_NOIO;
206621caf2fcSMing Lei 	return flags;
206721caf2fcSMing Lei }
206821caf2fcSMing Lei 
206921caf2fcSMing Lei static inline void memalloc_noio_restore(unsigned int flags)
207021caf2fcSMing Lei {
207121caf2fcSMing Lei 	current->flags = (current->flags & ~PF_MEMALLOC_NOIO) | flags;
207221caf2fcSMing Lei }
207321caf2fcSMing Lei 
20741d4457f9SKees Cook /* Per-process atomic flags. */
2075a2b86f77SZefan Li #define PFA_NO_NEW_PRIVS 0	/* May not gain new privileges. */
20762ad654bcSZefan Li #define PFA_SPREAD_PAGE  1      /* Spread page cache over cpuset */
20772ad654bcSZefan Li #define PFA_SPREAD_SLAB  2      /* Spread some slab caches over cpuset */
207877ed2c57STetsuo Handa #define PFA_LMK_WAITING  3      /* Lowmemorykiller is waiting */
20791d4457f9SKees Cook 
20801d4457f9SKees Cook 
2081e0e5070bSZefan Li #define TASK_PFA_TEST(name, func)					\
2082e0e5070bSZefan Li 	static inline bool task_##func(struct task_struct *p)		\
2083e0e5070bSZefan Li 	{ return test_bit(PFA_##name, &p->atomic_flags); }
2084e0e5070bSZefan Li #define TASK_PFA_SET(name, func)					\
2085e0e5070bSZefan Li 	static inline void task_set_##func(struct task_struct *p)	\
2086e0e5070bSZefan Li 	{ set_bit(PFA_##name, &p->atomic_flags); }
2087e0e5070bSZefan Li #define TASK_PFA_CLEAR(name, func)					\
2088e0e5070bSZefan Li 	static inline void task_clear_##func(struct task_struct *p)	\
2089e0e5070bSZefan Li 	{ clear_bit(PFA_##name, &p->atomic_flags); }
20901d4457f9SKees Cook 
2091e0e5070bSZefan Li TASK_PFA_TEST(NO_NEW_PRIVS, no_new_privs)
2092e0e5070bSZefan Li TASK_PFA_SET(NO_NEW_PRIVS, no_new_privs)
20931d4457f9SKees Cook 
20942ad654bcSZefan Li TASK_PFA_TEST(SPREAD_PAGE, spread_page)
20952ad654bcSZefan Li TASK_PFA_SET(SPREAD_PAGE, spread_page)
20962ad654bcSZefan Li TASK_PFA_CLEAR(SPREAD_PAGE, spread_page)
20972ad654bcSZefan Li 
20982ad654bcSZefan Li TASK_PFA_TEST(SPREAD_SLAB, spread_slab)
20992ad654bcSZefan Li TASK_PFA_SET(SPREAD_SLAB, spread_slab)
21002ad654bcSZefan Li TASK_PFA_CLEAR(SPREAD_SLAB, spread_slab)
2101544b2c91STejun Heo 
210277ed2c57STetsuo Handa TASK_PFA_TEST(LMK_WAITING, lmk_waiting)
210377ed2c57STetsuo Handa TASK_PFA_SET(LMK_WAITING, lmk_waiting)
210477ed2c57STetsuo Handa 
210539efa3efSTejun Heo /*
2106e5c1902eSTejun Heo  * task->jobctl flags
2107d79fdd6dSTejun Heo  */
2108ee77f075SOleg Nesterov #define JOBCTL_STOP_SIGMASK	0xffff	/* signr of the last group stop */
2109e5c1902eSTejun Heo 
2110f41d911fSPaul E. McKenney #define JOBCTL_STOP_DEQUEUED_BIT 16	/* stop signal dequeued */
2111a8f072c1STejun Heo #define JOBCTL_STOP_PENDING_BIT	17	/* task should stop for group stop */
2112a8f072c1STejun Heo #define JOBCTL_STOP_CONSUME_BIT	18	/* consume group stop count */
2113a8f072c1STejun Heo #define JOBCTL_TRAP_STOP_BIT	19	/* trap for STOP */
211473ddff2bSTejun Heo #define JOBCTL_TRAP_NOTIFY_BIT	20	/* trap for NOTIFY */
2115fb1d910cSTejun Heo #define JOBCTL_TRAPPING_BIT	21	/* switching to TRACED */
2116a8f072c1STejun Heo #define JOBCTL_LISTENING_BIT	22	/* ptracer is listening for events */
2117a8f072c1STejun Heo 
2118b76808e6SPalmer Dabbelt #define JOBCTL_STOP_DEQUEUED	(1UL << JOBCTL_STOP_DEQUEUED_BIT)
2119b76808e6SPalmer Dabbelt #define JOBCTL_STOP_PENDING	(1UL << JOBCTL_STOP_PENDING_BIT)
2120b76808e6SPalmer Dabbelt #define JOBCTL_STOP_CONSUME	(1UL << JOBCTL_STOP_CONSUME_BIT)
2121b76808e6SPalmer Dabbelt #define JOBCTL_TRAP_STOP	(1UL << JOBCTL_TRAP_STOP_BIT)
2122b76808e6SPalmer Dabbelt #define JOBCTL_TRAP_NOTIFY	(1UL << JOBCTL_TRAP_NOTIFY_BIT)
2123b76808e6SPalmer Dabbelt #define JOBCTL_TRAPPING		(1UL << JOBCTL_TRAPPING_BIT)
2124b76808e6SPalmer Dabbelt #define JOBCTL_LISTENING	(1UL << JOBCTL_LISTENING_BIT)
2125a8f072c1STejun Heo 
2126fb1d910cSTejun Heo #define JOBCTL_TRAP_MASK	(JOBCTL_TRAP_STOP | JOBCTL_TRAP_NOTIFY)
212773ddff2bSTejun Heo #define JOBCTL_PENDING_MASK	(JOBCTL_STOP_PENDING | JOBCTL_TRAP_MASK)
21283759a0d9STejun Heo 
21297dd3db54STejun Heo extern bool task_set_jobctl_pending(struct task_struct *task,
2130b76808e6SPalmer Dabbelt 				    unsigned long mask);
213173ddff2bSTejun Heo extern void task_clear_jobctl_trapping(struct task_struct *task);
21323759a0d9STejun Heo extern void task_clear_jobctl_pending(struct task_struct *task,
2133b76808e6SPalmer Dabbelt 				      unsigned long mask);
213439efa3efSTejun Heo 
2135f41d911fSPaul E. McKenney static inline void rcu_copy_process(struct task_struct *p)
2136f41d911fSPaul E. McKenney {
21371da177e4SLinus Torvalds #ifdef CONFIG_PREEMPT_RCU
2138f41d911fSPaul E. McKenney 	p->rcu_read_lock_nesting = 0;
21391d082fd0SPaul E. McKenney 	p->rcu_read_unlock_special.s = 0;
2140dd5d19baSPaul E. McKenney 	p->rcu_blocked_node = NULL;
2141f41d911fSPaul E. McKenney 	INIT_LIST_HEAD(&p->rcu_node_entry);
21428315f422SPaul E. McKenney #endif /* #ifdef CONFIG_PREEMPT_RCU */
21438315f422SPaul E. McKenney #ifdef CONFIG_TASKS_RCU
21448315f422SPaul E. McKenney 	p->rcu_tasks_holdout = false;
21458315f422SPaul E. McKenney 	INIT_LIST_HEAD(&p->rcu_tasks_holdout_list);
2146176f8f7aSPaul E. McKenney 	p->rcu_tasks_idle_cpu = -1;
21478315f422SPaul E. McKenney #endif /* #ifdef CONFIG_TASKS_RCU */
2148f41d911fSPaul E. McKenney }
2149f41d911fSPaul E. McKenney 
2150907aed48SMel Gorman static inline void tsk_restore_flags(struct task_struct *task,
2151907aed48SMel Gorman 				unsigned long orig_flags, unsigned long flags)
2152907aed48SMel Gorman {
2153907aed48SMel Gorman 	task->flags &= ~flags;
2154907aed48SMel Gorman 	task->flags |= orig_flags & flags;
2155907aed48SMel Gorman }
2156907aed48SMel Gorman 
2157f82f8042SJuri Lelli extern int cpuset_cpumask_can_shrink(const struct cpumask *cur,
2158f82f8042SJuri Lelli 				     const struct cpumask *trial);
21597f51412aSJuri Lelli extern int task_can_attach(struct task_struct *p,
21607f51412aSJuri Lelli 			   const struct cpumask *cs_cpus_allowed);
21611da177e4SLinus Torvalds #ifdef CONFIG_SMP
21621e1b6c51SKOSAKI Motohiro extern void do_set_cpus_allowed(struct task_struct *p,
21631e1b6c51SKOSAKI Motohiro 			       const struct cpumask *new_mask);
21641e1b6c51SKOSAKI Motohiro 
2165cd8ba7cdSMike Travis extern int set_cpus_allowed_ptr(struct task_struct *p,
216696f874e2SRusty Russell 				const struct cpumask *new_mask);
21671da177e4SLinus Torvalds #else
21681e1b6c51SKOSAKI Motohiro static inline void do_set_cpus_allowed(struct task_struct *p,
21691e1b6c51SKOSAKI Motohiro 				      const struct cpumask *new_mask)
21701e1b6c51SKOSAKI Motohiro {
21711e1b6c51SKOSAKI Motohiro }
2172cd8ba7cdSMike Travis static inline int set_cpus_allowed_ptr(struct task_struct *p,
217396f874e2SRusty Russell 				       const struct cpumask *new_mask)
21741da177e4SLinus Torvalds {
217596f874e2SRusty Russell 	if (!cpumask_test_cpu(0, new_mask))
21761da177e4SLinus Torvalds 		return -EINVAL;
21771da177e4SLinus Torvalds 	return 0;
21781da177e4SLinus Torvalds }
21791da177e4SLinus Torvalds #endif
2180e0ad9556SRusty Russell 
21813451d024SFrederic Weisbecker #ifdef CONFIG_NO_HZ_COMMON
21825167e8d5SPeter Zijlstra void calc_load_enter_idle(void);
21835167e8d5SPeter Zijlstra void calc_load_exit_idle(void);
21845167e8d5SPeter Zijlstra #else
21855167e8d5SPeter Zijlstra static inline void calc_load_enter_idle(void) { }
21865167e8d5SPeter Zijlstra static inline void calc_load_exit_idle(void) { }
21873451d024SFrederic Weisbecker #endif /* CONFIG_NO_HZ_COMMON */
21885167e8d5SPeter Zijlstra 
21896d0d2878SChristian Borntraeger #ifndef cpu_relax_yield
21906d0d2878SChristian Borntraeger #define cpu_relax_yield() cpu_relax()
21916d0d2878SChristian Borntraeger #endif
21926d0d2878SChristian Borntraeger 
2193b342501cSIngo Molnar /*
2194c676329aSPeter Zijlstra  * Do not use outside of architecture code which knows its limitations.
2195c676329aSPeter Zijlstra  *
2196c676329aSPeter Zijlstra  * sched_clock() has no promise of monotonicity or bounded drift between
2197c676329aSPeter Zijlstra  * CPUs, use (which you should not) requires disabling IRQs.
2198c676329aSPeter Zijlstra  *
2199c676329aSPeter Zijlstra  * Please use one of the three interfaces below.
2200b342501cSIngo Molnar  */
22011bbfa6f2SMike Frysinger extern unsigned long long notrace sched_clock(void);
2202c676329aSPeter Zijlstra /*
2203489a71b0SHiroshi Shimamoto  * See the comment in kernel/sched/clock.c
2204c676329aSPeter Zijlstra  */
2205545a2bf7SCyril Bur extern u64 running_clock(void);
2206c676329aSPeter Zijlstra extern u64 sched_clock_cpu(int cpu);
2207c676329aSPeter Zijlstra 
2208e436d800SIngo Molnar 
2209c1955a3dSPeter Zijlstra extern void sched_clock_init(void);
2210c1955a3dSPeter Zijlstra 
22113e51f33fSPeter Zijlstra #ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
22129881b024SPeter Zijlstra static inline void sched_clock_init_late(void)
22139881b024SPeter Zijlstra {
22149881b024SPeter Zijlstra }
22159881b024SPeter Zijlstra 
22163e51f33fSPeter Zijlstra static inline void sched_clock_tick(void)
22173e51f33fSPeter Zijlstra {
22183e51f33fSPeter Zijlstra }
22193e51f33fSPeter Zijlstra 
2220733ce725SPaul Gortmaker static inline void clear_sched_clock_stable(void)
2221733ce725SPaul Gortmaker {
2222733ce725SPaul Gortmaker }
2223733ce725SPaul Gortmaker 
22243e51f33fSPeter Zijlstra static inline void sched_clock_idle_sleep_event(void)
22253e51f33fSPeter Zijlstra {
22263e51f33fSPeter Zijlstra }
22273e51f33fSPeter Zijlstra 
22283e51f33fSPeter Zijlstra static inline void sched_clock_idle_wakeup_event(u64 delta_ns)
22293e51f33fSPeter Zijlstra {
22303e51f33fSPeter Zijlstra }
22312c923e94SDaniel Lezcano 
22322c923e94SDaniel Lezcano static inline u64 cpu_clock(int cpu)
22332c923e94SDaniel Lezcano {
22342c923e94SDaniel Lezcano 	return sched_clock();
22352c923e94SDaniel Lezcano }
22362c923e94SDaniel Lezcano 
22372c923e94SDaniel Lezcano static inline u64 local_clock(void)
22382c923e94SDaniel Lezcano {
22392c923e94SDaniel Lezcano 	return sched_clock();
22402c923e94SDaniel Lezcano }
22413e51f33fSPeter Zijlstra #else
22429881b024SPeter Zijlstra extern void sched_clock_init_late(void);
2243c676329aSPeter Zijlstra /*
2244c676329aSPeter Zijlstra  * Architectures can set this to 1 if they have specified
2245c676329aSPeter Zijlstra  * CONFIG_HAVE_UNSTABLE_SCHED_CLOCK in their arch Kconfig,
2246c676329aSPeter Zijlstra  * but then during bootup it turns out that sched_clock()
2247c676329aSPeter Zijlstra  * is reliable after all:
2248c676329aSPeter Zijlstra  */
224935af99e6SPeter Zijlstra extern int sched_clock_stable(void);
225035af99e6SPeter Zijlstra extern void clear_sched_clock_stable(void);
2251c676329aSPeter Zijlstra 
22523e51f33fSPeter Zijlstra extern void sched_clock_tick(void);
22533e51f33fSPeter Zijlstra extern void sched_clock_idle_sleep_event(void);
22543e51f33fSPeter Zijlstra extern void sched_clock_idle_wakeup_event(u64 delta_ns);
22552c923e94SDaniel Lezcano 
22562c923e94SDaniel Lezcano /*
22572c923e94SDaniel Lezcano  * As outlined in clock.c, provides a fast, high resolution, nanosecond
22582c923e94SDaniel Lezcano  * time source that is monotonic per cpu argument and has bounded drift
22592c923e94SDaniel Lezcano  * between cpus.
22602c923e94SDaniel Lezcano  *
22612c923e94SDaniel Lezcano  * ######################### BIG FAT WARNING ##########################
22622c923e94SDaniel Lezcano  * # when comparing cpu_clock(i) to cpu_clock(j) for i != j, time can #
22632c923e94SDaniel Lezcano  * # go backwards !!                                                  #
22642c923e94SDaniel Lezcano  * ####################################################################
22652c923e94SDaniel Lezcano  */
22662c923e94SDaniel Lezcano static inline u64 cpu_clock(int cpu)
22672c923e94SDaniel Lezcano {
22682c923e94SDaniel Lezcano 	return sched_clock_cpu(cpu);
22692c923e94SDaniel Lezcano }
22702c923e94SDaniel Lezcano 
22712c923e94SDaniel Lezcano static inline u64 local_clock(void)
22722c923e94SDaniel Lezcano {
22732c923e94SDaniel Lezcano 	return sched_clock_cpu(raw_smp_processor_id());
22742c923e94SDaniel Lezcano }
22753e51f33fSPeter Zijlstra #endif
22763e51f33fSPeter Zijlstra 
2277b52bfee4SVenkatesh Pallipadi #ifdef CONFIG_IRQ_TIME_ACCOUNTING
2278b52bfee4SVenkatesh Pallipadi /*
2279b52bfee4SVenkatesh Pallipadi  * An i/f to runtime opt-in for irq time accounting based off of sched_clock.
2280b52bfee4SVenkatesh Pallipadi  * The reason for this explicit opt-in is not to have perf penalty with
2281b52bfee4SVenkatesh Pallipadi  * slow sched_clocks.
2282b52bfee4SVenkatesh Pallipadi  */
2283b52bfee4SVenkatesh Pallipadi extern void enable_sched_clock_irqtime(void);
2284b52bfee4SVenkatesh Pallipadi extern void disable_sched_clock_irqtime(void);
2285b52bfee4SVenkatesh Pallipadi #else
2286b52bfee4SVenkatesh Pallipadi static inline void enable_sched_clock_irqtime(void) {}
2287b52bfee4SVenkatesh Pallipadi static inline void disable_sched_clock_irqtime(void) {}
2288b52bfee4SVenkatesh Pallipadi #endif
2289b52bfee4SVenkatesh Pallipadi 
229036c8b586SIngo Molnar extern unsigned long long
229141b86e9cSIngo Molnar task_sched_runtime(struct task_struct *task);
22921da177e4SLinus Torvalds 
22931da177e4SLinus Torvalds /* sched_exec is called by processes performing an exec */
22941da177e4SLinus Torvalds #ifdef CONFIG_SMP
22951da177e4SLinus Torvalds extern void sched_exec(void);
22961da177e4SLinus Torvalds #else
22971da177e4SLinus Torvalds #define sched_exec()   {}
22981da177e4SLinus Torvalds #endif
22991da177e4SLinus Torvalds 
23002aa44d05SIngo Molnar extern void sched_clock_idle_sleep_event(void);
23012aa44d05SIngo Molnar extern void sched_clock_idle_wakeup_event(u64 delta_ns);
2302bb29ab26SIngo Molnar 
23031da177e4SLinus Torvalds #ifdef CONFIG_HOTPLUG_CPU
23041da177e4SLinus Torvalds extern void idle_task_exit(void);
23051da177e4SLinus Torvalds #else
23061da177e4SLinus Torvalds static inline void idle_task_exit(void) {}
23071da177e4SLinus Torvalds #endif
23081da177e4SLinus Torvalds 
23093451d024SFrederic Weisbecker #if defined(CONFIG_NO_HZ_COMMON) && defined(CONFIG_SMP)
23101c20091eSFrederic Weisbecker extern void wake_up_nohz_cpu(int cpu);
231106d8308cSThomas Gleixner #else
23121c20091eSFrederic Weisbecker static inline void wake_up_nohz_cpu(int cpu) { }
231306d8308cSThomas Gleixner #endif
231406d8308cSThomas Gleixner 
2315ce831b38SFrederic Weisbecker #ifdef CONFIG_NO_HZ_FULL
2316265f22a9SFrederic Weisbecker extern u64 scheduler_tick_max_deferment(void);
2317bf0f6f24SIngo Molnar #endif
2318bf0f6f24SIngo Molnar 
23195091faa4SMike Galbraith #ifdef CONFIG_SCHED_AUTOGROUP
23205091faa4SMike Galbraith extern void sched_autogroup_create_attach(struct task_struct *p);
23215091faa4SMike Galbraith extern void sched_autogroup_detach(struct task_struct *p);
23225091faa4SMike Galbraith extern void sched_autogroup_fork(struct signal_struct *sig);
23235091faa4SMike Galbraith extern void sched_autogroup_exit(struct signal_struct *sig);
23248e5bfa8cSOleg Nesterov extern void sched_autogroup_exit_task(struct task_struct *p);
23255091faa4SMike Galbraith #ifdef CONFIG_PROC_FS
23265091faa4SMike Galbraith extern void proc_sched_autogroup_show_task(struct task_struct *p, struct seq_file *m);
23272e5b5b3aSHiroshi Shimamoto extern int proc_sched_autogroup_set_nice(struct task_struct *p, int nice);
23285091faa4SMike Galbraith #endif
23295091faa4SMike Galbraith #else
23305091faa4SMike Galbraith static inline void sched_autogroup_create_attach(struct task_struct *p) { }
23315091faa4SMike Galbraith static inline void sched_autogroup_detach(struct task_struct *p) { }
23325091faa4SMike Galbraith static inline void sched_autogroup_fork(struct signal_struct *sig) { }
23335091faa4SMike Galbraith static inline void sched_autogroup_exit(struct signal_struct *sig) { }
23348e5bfa8cSOleg Nesterov static inline void sched_autogroup_exit_task(struct task_struct *p) { }
23355091faa4SMike Galbraith #endif
23365091faa4SMike Galbraith 
2337fa93384fSDan Carpenter extern int yield_to(struct task_struct *p, bool preempt);
233836c8b586SIngo Molnar extern void set_user_nice(struct task_struct *p, long nice);
233936c8b586SIngo Molnar extern int task_prio(const struct task_struct *p);
2340d0ea0268SDongsheng Yang /**
2341d0ea0268SDongsheng Yang  * task_nice - return the nice value of a given task.
2342d0ea0268SDongsheng Yang  * @p: the task in question.
2343d0ea0268SDongsheng Yang  *
2344d0ea0268SDongsheng Yang  * Return: The nice value [ -20 ... 0 ... 19 ].
2345d0ea0268SDongsheng Yang  */
2346d0ea0268SDongsheng Yang static inline int task_nice(const struct task_struct *p)
2347d0ea0268SDongsheng Yang {
2348d0ea0268SDongsheng Yang 	return PRIO_TO_NICE((p)->static_prio);
2349d0ea0268SDongsheng Yang }
235036c8b586SIngo Molnar extern int can_nice(const struct task_struct *p, const int nice);
235136c8b586SIngo Molnar extern int task_curr(const struct task_struct *p);
23521da177e4SLinus Torvalds extern int idle_cpu(int cpu);
2353fe7de49fSKOSAKI Motohiro extern int sched_setscheduler(struct task_struct *, int,
2354fe7de49fSKOSAKI Motohiro 			      const struct sched_param *);
2355961ccdddSRusty Russell extern int sched_setscheduler_nocheck(struct task_struct *, int,
2356fe7de49fSKOSAKI Motohiro 				      const struct sched_param *);
2357d50dde5aSDario Faggioli extern int sched_setattr(struct task_struct *,
2358d50dde5aSDario Faggioli 			 const struct sched_attr *);
235936c8b586SIngo Molnar extern struct task_struct *idle_task(int cpu);
2360c4f30608SPaul E. McKenney /**
2361c4f30608SPaul E. McKenney  * is_idle_task - is the specified task an idle task?
2362fa757281SRandy Dunlap  * @p: the task in question.
2363e69f6186SYacine Belkadi  *
2364e69f6186SYacine Belkadi  * Return: 1 if @p is an idle task. 0 otherwise.
2365c4f30608SPaul E. McKenney  */
23667061ca3bSPaul E. McKenney static inline bool is_idle_task(const struct task_struct *p)
2367c4f30608SPaul E. McKenney {
2368c1de45caSPeter Zijlstra 	return !!(p->flags & PF_IDLE);
2369c4f30608SPaul E. McKenney }
237036c8b586SIngo Molnar extern struct task_struct *curr_task(int cpu);
2371a458ae2eSPeter Zijlstra extern void ia64_set_curr_task(int cpu, struct task_struct *p);
23721da177e4SLinus Torvalds 
23731da177e4SLinus Torvalds void yield(void);
23741da177e4SLinus Torvalds 
23751da177e4SLinus Torvalds union thread_union {
2376c65eacbeSAndy Lutomirski #ifndef CONFIG_THREAD_INFO_IN_TASK
23771da177e4SLinus Torvalds 	struct thread_info thread_info;
2378c65eacbeSAndy Lutomirski #endif
23791da177e4SLinus Torvalds 	unsigned long stack[THREAD_SIZE/sizeof(long)];
23801da177e4SLinus Torvalds };
23811da177e4SLinus Torvalds 
23821da177e4SLinus Torvalds #ifndef __HAVE_ARCH_KSTACK_END
23831da177e4SLinus Torvalds static inline int kstack_end(void *addr)
23841da177e4SLinus Torvalds {
23851da177e4SLinus Torvalds 	/* Reliable end of stack detection:
23861da177e4SLinus Torvalds 	 * Some APM bios versions misalign the stack
23871da177e4SLinus Torvalds 	 */
23881da177e4SLinus Torvalds 	return !(((unsigned long)addr+sizeof(void*)-1) & (THREAD_SIZE-sizeof(void*)));
23891da177e4SLinus Torvalds }
23901da177e4SLinus Torvalds #endif
23911da177e4SLinus Torvalds 
23921da177e4SLinus Torvalds extern union thread_union init_thread_union;
23931da177e4SLinus Torvalds extern struct task_struct init_task;
23941da177e4SLinus Torvalds 
23951da177e4SLinus Torvalds extern struct   mm_struct init_mm;
23961da177e4SLinus Torvalds 
2397198fe21bSPavel Emelyanov extern struct pid_namespace init_pid_ns;
2398198fe21bSPavel Emelyanov 
2399198fe21bSPavel Emelyanov /*
2400198fe21bSPavel Emelyanov  * find a task by one of its numerical ids
2401198fe21bSPavel Emelyanov  *
2402198fe21bSPavel Emelyanov  * find_task_by_pid_ns():
2403198fe21bSPavel Emelyanov  *      finds a task by its pid in the specified namespace
2404228ebcbeSPavel Emelyanov  * find_task_by_vpid():
2405228ebcbeSPavel Emelyanov  *      finds a task by its virtual pid
2406198fe21bSPavel Emelyanov  *
2407e49859e7SPavel Emelyanov  * see also find_vpid() etc in include/linux/pid.h
2408198fe21bSPavel Emelyanov  */
2409198fe21bSPavel Emelyanov 
2410228ebcbeSPavel Emelyanov extern struct task_struct *find_task_by_vpid(pid_t nr);
2411228ebcbeSPavel Emelyanov extern struct task_struct *find_task_by_pid_ns(pid_t nr,
2412228ebcbeSPavel Emelyanov 		struct pid_namespace *ns);
2413198fe21bSPavel Emelyanov 
24141da177e4SLinus Torvalds /* per-UID process charging. */
24157b44ab97SEric W. Biederman extern struct user_struct * alloc_uid(kuid_t);
24161da177e4SLinus Torvalds static inline struct user_struct *get_uid(struct user_struct *u)
24171da177e4SLinus Torvalds {
24181da177e4SLinus Torvalds 	atomic_inc(&u->__count);
24191da177e4SLinus Torvalds 	return u;
24201da177e4SLinus Torvalds }
24211da177e4SLinus Torvalds extern void free_uid(struct user_struct *);
24221da177e4SLinus Torvalds 
24231da177e4SLinus Torvalds #include <asm/current.h>
24241da177e4SLinus Torvalds 
2425f0af911aSTorben Hohn extern void xtime_update(unsigned long ticks);
24261da177e4SLinus Torvalds 
2427b3c97528SHarvey Harrison extern int wake_up_state(struct task_struct *tsk, unsigned int state);
2428b3c97528SHarvey Harrison extern int wake_up_process(struct task_struct *tsk);
24293e51e3edSSamir Bellabes extern void wake_up_new_task(struct task_struct *tsk);
24301da177e4SLinus Torvalds #ifdef CONFIG_SMP
24311da177e4SLinus Torvalds  extern void kick_process(struct task_struct *tsk);
24321da177e4SLinus Torvalds #else
24331da177e4SLinus Torvalds  static inline void kick_process(struct task_struct *tsk) { }
24341da177e4SLinus Torvalds #endif
2435aab03e05SDario Faggioli extern int sched_fork(unsigned long clone_flags, struct task_struct *p);
2436ad46c2c4SIngo Molnar extern void sched_dead(struct task_struct *p);
24371da177e4SLinus Torvalds 
24381da177e4SLinus Torvalds extern void proc_caches_init(void);
24391da177e4SLinus Torvalds extern void flush_signals(struct task_struct *);
244010ab825bSOleg Nesterov extern void ignore_signals(struct task_struct *);
24411da177e4SLinus Torvalds extern void flush_signal_handlers(struct task_struct *, int force_default);
24421da177e4SLinus Torvalds extern int dequeue_signal(struct task_struct *tsk, sigset_t *mask, siginfo_t *info);
24431da177e4SLinus Torvalds 
2444be0e6f29SOleg Nesterov static inline int kernel_dequeue_signal(siginfo_t *info)
24451da177e4SLinus Torvalds {
2446be0e6f29SOleg Nesterov 	struct task_struct *tsk = current;
2447be0e6f29SOleg Nesterov 	siginfo_t __info;
24481da177e4SLinus Torvalds 	int ret;
24491da177e4SLinus Torvalds 
2450be0e6f29SOleg Nesterov 	spin_lock_irq(&tsk->sighand->siglock);
2451be0e6f29SOleg Nesterov 	ret = dequeue_signal(tsk, &tsk->blocked, info ?: &__info);
2452be0e6f29SOleg Nesterov 	spin_unlock_irq(&tsk->sighand->siglock);
24531da177e4SLinus Torvalds 
24541da177e4SLinus Torvalds 	return ret;
24551da177e4SLinus Torvalds }
24561da177e4SLinus Torvalds 
24579a13049eSOleg Nesterov static inline void kernel_signal_stop(void)
24589a13049eSOleg Nesterov {
24599a13049eSOleg Nesterov 	spin_lock_irq(&current->sighand->siglock);
24609a13049eSOleg Nesterov 	if (current->jobctl & JOBCTL_STOP_DEQUEUED)
24619a13049eSOleg Nesterov 		__set_current_state(TASK_STOPPED);
24629a13049eSOleg Nesterov 	spin_unlock_irq(&current->sighand->siglock);
24639a13049eSOleg Nesterov 
24649a13049eSOleg Nesterov 	schedule();
24659a13049eSOleg Nesterov }
24669a13049eSOleg Nesterov 
24671da177e4SLinus Torvalds extern void release_task(struct task_struct * p);
24681da177e4SLinus Torvalds extern int send_sig_info(int, struct siginfo *, struct task_struct *);
24691da177e4SLinus Torvalds extern int force_sigsegv(int, struct task_struct *);
24701da177e4SLinus Torvalds extern int force_sig_info(int, struct siginfo *, struct task_struct *);
2471c4b92fc1SEric W. Biederman extern int __kill_pgrp_info(int sig, struct siginfo *info, struct pid *pgrp);
2472c4b92fc1SEric W. Biederman extern int kill_pid_info(int sig, struct siginfo *info, struct pid *pid);
2473d178bc3aSSerge Hallyn extern int kill_pid_info_as_cred(int, struct siginfo *, struct pid *,
2474d178bc3aSSerge Hallyn 				const struct cred *, u32);
2475c4b92fc1SEric W. Biederman extern int kill_pgrp(struct pid *pid, int sig, int priv);
2476c4b92fc1SEric W. Biederman extern int kill_pid(struct pid *pid, int sig, int priv);
2477c3de4b38SMatthew Wilcox extern int kill_proc_info(int, struct siginfo *, pid_t);
247886773473SOleg Nesterov extern __must_check bool do_notify_parent(struct task_struct *, int);
2479a7f0765eSOleg Nesterov extern void __wake_up_parent(struct task_struct *p, struct task_struct *parent);
24801da177e4SLinus Torvalds extern void force_sig(int, struct task_struct *);
24811da177e4SLinus Torvalds extern int send_sig(int, struct task_struct *, int);
248209faef11SOleg Nesterov extern int zap_other_threads(struct task_struct *p);
24831da177e4SLinus Torvalds extern struct sigqueue *sigqueue_alloc(void);
24841da177e4SLinus Torvalds extern void sigqueue_free(struct sigqueue *);
2485ac5c2153SOleg Nesterov extern int send_sigqueue(struct sigqueue *,  struct task_struct *, int group);
24869ac95f2fSOleg Nesterov extern int do_sigaction(int, struct k_sigaction *, struct k_sigaction *);
24871da177e4SLinus Torvalds 
24887e781418SAndy Lutomirski #ifdef TIF_RESTORE_SIGMASK
24897e781418SAndy Lutomirski /*
24907e781418SAndy Lutomirski  * Legacy restore_sigmask accessors.  These are inefficient on
24917e781418SAndy Lutomirski  * SMP architectures because they require atomic operations.
24927e781418SAndy Lutomirski  */
24937e781418SAndy Lutomirski 
24947e781418SAndy Lutomirski /**
24957e781418SAndy Lutomirski  * set_restore_sigmask() - make sure saved_sigmask processing gets done
24967e781418SAndy Lutomirski  *
24977e781418SAndy Lutomirski  * This sets TIF_RESTORE_SIGMASK and ensures that the arch signal code
24987e781418SAndy Lutomirski  * will run before returning to user mode, to process the flag.  For
24997e781418SAndy Lutomirski  * all callers, TIF_SIGPENDING is already set or it's no harm to set
25007e781418SAndy Lutomirski  * it.  TIF_RESTORE_SIGMASK need not be in the set of bits that the
25017e781418SAndy Lutomirski  * arch code will notice on return to user mode, in case those bits
25027e781418SAndy Lutomirski  * are scarce.  We set TIF_SIGPENDING here to ensure that the arch
25037e781418SAndy Lutomirski  * signal code always gets run when TIF_RESTORE_SIGMASK is set.
25047e781418SAndy Lutomirski  */
25057e781418SAndy Lutomirski static inline void set_restore_sigmask(void)
25067e781418SAndy Lutomirski {
25077e781418SAndy Lutomirski 	set_thread_flag(TIF_RESTORE_SIGMASK);
25087e781418SAndy Lutomirski 	WARN_ON(!test_thread_flag(TIF_SIGPENDING));
25097e781418SAndy Lutomirski }
25107e781418SAndy Lutomirski static inline void clear_restore_sigmask(void)
25117e781418SAndy Lutomirski {
25127e781418SAndy Lutomirski 	clear_thread_flag(TIF_RESTORE_SIGMASK);
25137e781418SAndy Lutomirski }
25147e781418SAndy Lutomirski static inline bool test_restore_sigmask(void)
25157e781418SAndy Lutomirski {
25167e781418SAndy Lutomirski 	return test_thread_flag(TIF_RESTORE_SIGMASK);
25177e781418SAndy Lutomirski }
25187e781418SAndy Lutomirski static inline bool test_and_clear_restore_sigmask(void)
25197e781418SAndy Lutomirski {
25207e781418SAndy Lutomirski 	return test_and_clear_thread_flag(TIF_RESTORE_SIGMASK);
25217e781418SAndy Lutomirski }
25227e781418SAndy Lutomirski 
25237e781418SAndy Lutomirski #else	/* TIF_RESTORE_SIGMASK */
25247e781418SAndy Lutomirski 
25257e781418SAndy Lutomirski /* Higher-quality implementation, used if TIF_RESTORE_SIGMASK doesn't exist. */
25267e781418SAndy Lutomirski static inline void set_restore_sigmask(void)
25277e781418SAndy Lutomirski {
25287e781418SAndy Lutomirski 	current->restore_sigmask = true;
25297e781418SAndy Lutomirski 	WARN_ON(!test_thread_flag(TIF_SIGPENDING));
25307e781418SAndy Lutomirski }
25317e781418SAndy Lutomirski static inline void clear_restore_sigmask(void)
25327e781418SAndy Lutomirski {
25337e781418SAndy Lutomirski 	current->restore_sigmask = false;
25347e781418SAndy Lutomirski }
25357e781418SAndy Lutomirski static inline bool test_restore_sigmask(void)
25367e781418SAndy Lutomirski {
25377e781418SAndy Lutomirski 	return current->restore_sigmask;
25387e781418SAndy Lutomirski }
25397e781418SAndy Lutomirski static inline bool test_and_clear_restore_sigmask(void)
25407e781418SAndy Lutomirski {
25417e781418SAndy Lutomirski 	if (!current->restore_sigmask)
25427e781418SAndy Lutomirski 		return false;
25437e781418SAndy Lutomirski 	current->restore_sigmask = false;
25447e781418SAndy Lutomirski 	return true;
25457e781418SAndy Lutomirski }
25467e781418SAndy Lutomirski #endif
25477e781418SAndy Lutomirski 
254851a7b448SAl Viro static inline void restore_saved_sigmask(void)
254951a7b448SAl Viro {
255051a7b448SAl Viro 	if (test_and_clear_restore_sigmask())
255177097ae5SAl Viro 		__set_current_blocked(&current->saved_sigmask);
255251a7b448SAl Viro }
255351a7b448SAl Viro 
2554b7f9a11aSAl Viro static inline sigset_t *sigmask_to_save(void)
2555b7f9a11aSAl Viro {
2556b7f9a11aSAl Viro 	sigset_t *res = &current->blocked;
2557b7f9a11aSAl Viro 	if (unlikely(test_restore_sigmask()))
2558b7f9a11aSAl Viro 		res = &current->saved_sigmask;
2559b7f9a11aSAl Viro 	return res;
2560b7f9a11aSAl Viro }
2561b7f9a11aSAl Viro 
25629ec52099SCedric Le Goater static inline int kill_cad_pid(int sig, int priv)
25639ec52099SCedric Le Goater {
25649ec52099SCedric Le Goater 	return kill_pid(cad_pid, sig, priv);
25659ec52099SCedric Le Goater }
25669ec52099SCedric Le Goater 
25671da177e4SLinus Torvalds /* These can be the second arg to send_sig_info/send_group_sig_info.  */
25681da177e4SLinus Torvalds #define SEND_SIG_NOINFO ((struct siginfo *) 0)
25691da177e4SLinus Torvalds #define SEND_SIG_PRIV	((struct siginfo *) 1)
25701da177e4SLinus Torvalds #define SEND_SIG_FORCED	((struct siginfo *) 2)
25711da177e4SLinus Torvalds 
25722a855dd0SSebastian Andrzej Siewior /*
25732a855dd0SSebastian Andrzej Siewior  * True if we are on the alternate signal stack.
25742a855dd0SSebastian Andrzej Siewior  */
25751da177e4SLinus Torvalds static inline int on_sig_stack(unsigned long sp)
25761da177e4SLinus Torvalds {
2577c876eeabSAndy Lutomirski 	/*
2578c876eeabSAndy Lutomirski 	 * If the signal stack is SS_AUTODISARM then, by construction, we
2579c876eeabSAndy Lutomirski 	 * can't be on the signal stack unless user code deliberately set
2580c876eeabSAndy Lutomirski 	 * SS_AUTODISARM when we were already on it.
2581c876eeabSAndy Lutomirski 	 *
2582c876eeabSAndy Lutomirski 	 * This improves reliability: if user state gets corrupted such that
2583c876eeabSAndy Lutomirski 	 * the stack pointer points very close to the end of the signal stack,
2584c876eeabSAndy Lutomirski 	 * then this check will enable the signal to be handled anyway.
2585c876eeabSAndy Lutomirski 	 */
2586c876eeabSAndy Lutomirski 	if (current->sas_ss_flags & SS_AUTODISARM)
2587c876eeabSAndy Lutomirski 		return 0;
2588c876eeabSAndy Lutomirski 
25892a855dd0SSebastian Andrzej Siewior #ifdef CONFIG_STACK_GROWSUP
25902a855dd0SSebastian Andrzej Siewior 	return sp >= current->sas_ss_sp &&
25912a855dd0SSebastian Andrzej Siewior 		sp - current->sas_ss_sp < current->sas_ss_size;
25922a855dd0SSebastian Andrzej Siewior #else
25932a855dd0SSebastian Andrzej Siewior 	return sp > current->sas_ss_sp &&
25942a855dd0SSebastian Andrzej Siewior 		sp - current->sas_ss_sp <= current->sas_ss_size;
25952a855dd0SSebastian Andrzej Siewior #endif
25961da177e4SLinus Torvalds }
25971da177e4SLinus Torvalds 
25981da177e4SLinus Torvalds static inline int sas_ss_flags(unsigned long sp)
25991da177e4SLinus Torvalds {
260072f15c03SRichard Weinberger 	if (!current->sas_ss_size)
260172f15c03SRichard Weinberger 		return SS_DISABLE;
260272f15c03SRichard Weinberger 
260372f15c03SRichard Weinberger 	return on_sig_stack(sp) ? SS_ONSTACK : 0;
26041da177e4SLinus Torvalds }
26051da177e4SLinus Torvalds 
26062a742138SStas Sergeev static inline void sas_ss_reset(struct task_struct *p)
26072a742138SStas Sergeev {
26082a742138SStas Sergeev 	p->sas_ss_sp = 0;
26092a742138SStas Sergeev 	p->sas_ss_size = 0;
26102a742138SStas Sergeev 	p->sas_ss_flags = SS_DISABLE;
26112a742138SStas Sergeev }
26122a742138SStas Sergeev 
26135a1b98d3SAl Viro static inline unsigned long sigsp(unsigned long sp, struct ksignal *ksig)
26145a1b98d3SAl Viro {
26155a1b98d3SAl Viro 	if (unlikely((ksig->ka.sa.sa_flags & SA_ONSTACK)) && ! sas_ss_flags(sp))
26165a1b98d3SAl Viro #ifdef CONFIG_STACK_GROWSUP
26175a1b98d3SAl Viro 		return current->sas_ss_sp;
26185a1b98d3SAl Viro #else
26195a1b98d3SAl Viro 		return current->sas_ss_sp + current->sas_ss_size;
26205a1b98d3SAl Viro #endif
26215a1b98d3SAl Viro 	return sp;
26225a1b98d3SAl Viro }
26235a1b98d3SAl Viro 
26241da177e4SLinus Torvalds /*
26251da177e4SLinus Torvalds  * Routines for handling mm_structs
26261da177e4SLinus Torvalds  */
26271da177e4SLinus Torvalds extern struct mm_struct * mm_alloc(void);
26281da177e4SLinus Torvalds 
2629f1f10076SVegard Nossum /**
2630f1f10076SVegard Nossum  * mmgrab() - Pin a &struct mm_struct.
2631f1f10076SVegard Nossum  * @mm: The &struct mm_struct to pin.
2632f1f10076SVegard Nossum  *
2633f1f10076SVegard Nossum  * Make sure that @mm will not get freed even after the owning task
2634f1f10076SVegard Nossum  * exits. This doesn't guarantee that the associated address space
2635f1f10076SVegard Nossum  * will still exist later on and mmget_not_zero() has to be used before
2636f1f10076SVegard Nossum  * accessing it.
2637f1f10076SVegard Nossum  *
2638f1f10076SVegard Nossum  * This is a preferred way to to pin @mm for a longer/unbounded amount
2639f1f10076SVegard Nossum  * of time.
2640f1f10076SVegard Nossum  *
2641f1f10076SVegard Nossum  * Use mmdrop() to release the reference acquired by mmgrab().
2642f1f10076SVegard Nossum  *
2643f1f10076SVegard Nossum  * See also <Documentation/vm/active_mm.txt> for an in-depth explanation
2644f1f10076SVegard Nossum  * of &mm_struct.mm_count vs &mm_struct.mm_users.
2645f1f10076SVegard Nossum  */
2646f1f10076SVegard Nossum static inline void mmgrab(struct mm_struct *mm)
2647f1f10076SVegard Nossum {
2648f1f10076SVegard Nossum 	atomic_inc(&mm->mm_count);
2649f1f10076SVegard Nossum }
2650f1f10076SVegard Nossum 
26511da177e4SLinus Torvalds /* mmdrop drops the mm and the page tables */
2652b3c97528SHarvey Harrison extern void __mmdrop(struct mm_struct *);
26531da177e4SLinus Torvalds static inline void mmdrop(struct mm_struct *mm)
26541da177e4SLinus Torvalds {
26556fb43d7bSIngo Molnar 	if (unlikely(atomic_dec_and_test(&mm->mm_count)))
26561da177e4SLinus Torvalds 		__mmdrop(mm);
26571da177e4SLinus Torvalds }
26581da177e4SLinus Torvalds 
26597283094eSMichal Hocko static inline void mmdrop_async_fn(struct work_struct *work)
26607283094eSMichal Hocko {
26617283094eSMichal Hocko 	struct mm_struct *mm = container_of(work, struct mm_struct, async_put_work);
26627283094eSMichal Hocko 	__mmdrop(mm);
26637283094eSMichal Hocko }
26647283094eSMichal Hocko 
26657283094eSMichal Hocko static inline void mmdrop_async(struct mm_struct *mm)
26667283094eSMichal Hocko {
26677283094eSMichal Hocko 	if (unlikely(atomic_dec_and_test(&mm->mm_count))) {
26687283094eSMichal Hocko 		INIT_WORK(&mm->async_put_work, mmdrop_async_fn);
26697283094eSMichal Hocko 		schedule_work(&mm->async_put_work);
26707283094eSMichal Hocko 	}
26717283094eSMichal Hocko }
26727283094eSMichal Hocko 
26733fce371bSVegard Nossum /**
26743fce371bSVegard Nossum  * mmget() - Pin the address space associated with a &struct mm_struct.
26753fce371bSVegard Nossum  * @mm: The address space to pin.
26763fce371bSVegard Nossum  *
26773fce371bSVegard Nossum  * Make sure that the address space of the given &struct mm_struct doesn't
26783fce371bSVegard Nossum  * go away. This does not protect against parts of the address space being
26793fce371bSVegard Nossum  * modified or freed, however.
26803fce371bSVegard Nossum  *
26813fce371bSVegard Nossum  * Never use this function to pin this address space for an
26823fce371bSVegard Nossum  * unbounded/indefinite amount of time.
26833fce371bSVegard Nossum  *
26843fce371bSVegard Nossum  * Use mmput() to release the reference acquired by mmget().
26853fce371bSVegard Nossum  *
26863fce371bSVegard Nossum  * See also <Documentation/vm/active_mm.txt> for an in-depth explanation
26873fce371bSVegard Nossum  * of &mm_struct.mm_count vs &mm_struct.mm_users.
26883fce371bSVegard Nossum  */
26893fce371bSVegard Nossum static inline void mmget(struct mm_struct *mm)
26903fce371bSVegard Nossum {
26913fce371bSVegard Nossum 	atomic_inc(&mm->mm_users);
26923fce371bSVegard Nossum }
26933fce371bSVegard Nossum 
2694d2005e3fSOleg Nesterov static inline bool mmget_not_zero(struct mm_struct *mm)
2695d2005e3fSOleg Nesterov {
2696d2005e3fSOleg Nesterov 	return atomic_inc_not_zero(&mm->mm_users);
26971da177e4SLinus Torvalds }
26981da177e4SLinus Torvalds 
26991da177e4SLinus Torvalds /* mmput gets rid of the mappings and all user-space */
27001da177e4SLinus Torvalds extern void mmput(struct mm_struct *);
27017ef949d7SMichal Hocko #ifdef CONFIG_MMU
27027ef949d7SMichal Hocko /* same as above but performs the slow path from the async context. Can
2703ec8d7c14SMichal Hocko  * be called from the atomic context as well
2704ec8d7c14SMichal Hocko  */
2705ec8d7c14SMichal Hocko extern void mmput_async(struct mm_struct *);
27067ef949d7SMichal Hocko #endif
2707ec8d7c14SMichal Hocko 
27081da177e4SLinus Torvalds /* Grab a reference to a task's mm, if it is not already going away */
27091da177e4SLinus Torvalds extern struct mm_struct *get_task_mm(struct task_struct *task);
27108cdb878dSChristopher Yeoh /*
27118cdb878dSChristopher Yeoh  * Grab a reference to a task's mm, if it is not already going away
27128cdb878dSChristopher Yeoh  * and ptrace_may_access with the mode parameter passed to it
27138cdb878dSChristopher Yeoh  * succeeds.
27148cdb878dSChristopher Yeoh  */
27158cdb878dSChristopher Yeoh extern struct mm_struct *mm_access(struct task_struct *task, unsigned int mode);
27161da177e4SLinus Torvalds /* Remove the current tasks stale references to the old mm_struct */
27171da177e4SLinus Torvalds extern void mm_release(struct task_struct *, struct mm_struct *);
27181da177e4SLinus Torvalds 
27193033f14aSJosh Triplett #ifdef CONFIG_HAVE_COPY_THREAD_TLS
27203033f14aSJosh Triplett extern int copy_thread_tls(unsigned long, unsigned long, unsigned long,
27213033f14aSJosh Triplett 			struct task_struct *, unsigned long);
27223033f14aSJosh Triplett #else
27236f2c55b8SAlexey Dobriyan extern int copy_thread(unsigned long, unsigned long, unsigned long,
2724afa86fc4SAl Viro 			struct task_struct *);
27253033f14aSJosh Triplett 
27263033f14aSJosh Triplett /* Architectures that haven't opted into copy_thread_tls get the tls argument
27273033f14aSJosh Triplett  * via pt_regs, so ignore the tls argument passed via C. */
27283033f14aSJosh Triplett static inline int copy_thread_tls(
27293033f14aSJosh Triplett 		unsigned long clone_flags, unsigned long sp, unsigned long arg,
27303033f14aSJosh Triplett 		struct task_struct *p, unsigned long tls)
27313033f14aSJosh Triplett {
27323033f14aSJosh Triplett 	return copy_thread(clone_flags, sp, arg, p);
27333033f14aSJosh Triplett }
27343033f14aSJosh Triplett #endif
27351da177e4SLinus Torvalds extern void flush_thread(void);
27365f56a5dfSJiri Slaby 
27375f56a5dfSJiri Slaby #ifdef CONFIG_HAVE_EXIT_THREAD
2738e6464694SJiri Slaby extern void exit_thread(struct task_struct *tsk);
27395f56a5dfSJiri Slaby #else
2740e6464694SJiri Slaby static inline void exit_thread(struct task_struct *tsk)
27415f56a5dfSJiri Slaby {
27425f56a5dfSJiri Slaby }
27435f56a5dfSJiri Slaby #endif
27441da177e4SLinus Torvalds 
27451da177e4SLinus Torvalds extern void exit_files(struct task_struct *);
2746a7e5328aSOleg Nesterov extern void __cleanup_sighand(struct sighand_struct *);
2747cbaffba1SOleg Nesterov 
27481da177e4SLinus Torvalds extern void exit_itimers(struct signal_struct *);
2749cbaffba1SOleg Nesterov extern void flush_itimer_signals(void);
27501da177e4SLinus Torvalds 
27519402c95fSJoe Perches extern void do_group_exit(int);
27521da177e4SLinus Torvalds 
2753c4ad8f98SLinus Torvalds extern int do_execve(struct filename *,
2754d7627467SDavid Howells 		     const char __user * const __user *,
2755da3d4c5fSAl Viro 		     const char __user * const __user *);
275651f39a1fSDavid Drysdale extern int do_execveat(int, struct filename *,
275751f39a1fSDavid Drysdale 		       const char __user * const __user *,
275851f39a1fSDavid Drysdale 		       const char __user * const __user *,
275951f39a1fSDavid Drysdale 		       int);
27603033f14aSJosh Triplett extern long _do_fork(unsigned long, unsigned long, unsigned long, int __user *, int __user *, unsigned long);
2761e80d6661SAl Viro extern long do_fork(unsigned long, unsigned long, unsigned long, int __user *, int __user *);
276236c8b586SIngo Molnar struct task_struct *fork_idle(int);
27632aa3a7f8SAl Viro extern pid_t kernel_thread(int (*fn)(void *), void *arg, unsigned long flags);
27641da177e4SLinus Torvalds 
276582b89778SAdrian Hunter extern void __set_task_comm(struct task_struct *tsk, const char *from, bool exec);
276682b89778SAdrian Hunter static inline void set_task_comm(struct task_struct *tsk, const char *from)
276782b89778SAdrian Hunter {
276882b89778SAdrian Hunter 	__set_task_comm(tsk, from, false);
276982b89778SAdrian Hunter }
277059714d65SAndrew Morton extern char *get_task_comm(char *to, struct task_struct *tsk);
27711da177e4SLinus Torvalds 
27721da177e4SLinus Torvalds #ifdef CONFIG_SMP
2773317f3941SPeter Zijlstra void scheduler_ipi(void);
277485ba2d86SRoland McGrath extern unsigned long wait_task_inactive(struct task_struct *, long match_state);
27751da177e4SLinus Torvalds #else
2776184748ccSPeter Zijlstra static inline void scheduler_ipi(void) { }
277785ba2d86SRoland McGrath static inline unsigned long wait_task_inactive(struct task_struct *p,
277885ba2d86SRoland McGrath 					       long match_state)
277985ba2d86SRoland McGrath {
278085ba2d86SRoland McGrath 	return 1;
278185ba2d86SRoland McGrath }
27821da177e4SLinus Torvalds #endif
27831da177e4SLinus Torvalds 
2784fafe870fSFrederic Weisbecker #define tasklist_empty() \
2785fafe870fSFrederic Weisbecker 	list_empty(&init_task.tasks)
2786fafe870fSFrederic Weisbecker 
278705725f7eSJiri Pirko #define next_task(p) \
278805725f7eSJiri Pirko 	list_entry_rcu((p)->tasks.next, struct task_struct, tasks)
27891da177e4SLinus Torvalds 
27901da177e4SLinus Torvalds #define for_each_process(p) \
27911da177e4SLinus Torvalds 	for (p = &init_task ; (p = next_task(p)) != &init_task ; )
27921da177e4SLinus Torvalds 
27935bb459bbSOleg Nesterov extern bool current_is_single_threaded(void);
2794d84f4f99SDavid Howells 
27951da177e4SLinus Torvalds /*
27961da177e4SLinus Torvalds  * Careful: do_each_thread/while_each_thread is a double loop so
27971da177e4SLinus Torvalds  *          'break' will not work as expected - use goto instead.
27981da177e4SLinus Torvalds  */
27991da177e4SLinus Torvalds #define do_each_thread(g, t) \
28001da177e4SLinus Torvalds 	for (g = t = &init_task ; (g = t = next_task(g)) != &init_task ; ) do
28011da177e4SLinus Torvalds 
28021da177e4SLinus Torvalds #define while_each_thread(g, t) \
28031da177e4SLinus Torvalds 	while ((t = next_thread(t)) != g)
28041da177e4SLinus Torvalds 
28050c740d0aSOleg Nesterov #define __for_each_thread(signal, t)	\
28060c740d0aSOleg Nesterov 	list_for_each_entry_rcu(t, &(signal)->thread_head, thread_node)
28070c740d0aSOleg Nesterov 
28080c740d0aSOleg Nesterov #define for_each_thread(p, t)		\
28090c740d0aSOleg Nesterov 	__for_each_thread((p)->signal, t)
28100c740d0aSOleg Nesterov 
28110c740d0aSOleg Nesterov /* Careful: this is a double loop, 'break' won't work as expected. */
28120c740d0aSOleg Nesterov #define for_each_process_thread(p, t)	\
28130c740d0aSOleg Nesterov 	for_each_process(p) for_each_thread(p, t)
28140c740d0aSOleg Nesterov 
28150f1b92cbSOleg Nesterov typedef int (*proc_visitor)(struct task_struct *p, void *data);
28160f1b92cbSOleg Nesterov void walk_process_tree(struct task_struct *top, proc_visitor, void *);
28170f1b92cbSOleg Nesterov 
28187e49827cSOleg Nesterov static inline int get_nr_threads(struct task_struct *tsk)
28197e49827cSOleg Nesterov {
2820b3ac022cSOleg Nesterov 	return tsk->signal->nr_threads;
28217e49827cSOleg Nesterov }
28227e49827cSOleg Nesterov 
2823087806b1SOleg Nesterov static inline bool thread_group_leader(struct task_struct *p)
2824087806b1SOleg Nesterov {
2825087806b1SOleg Nesterov 	return p->exit_signal >= 0;
2826087806b1SOleg Nesterov }
28271da177e4SLinus Torvalds 
28280804ef4bSEric W. Biederman /* Do to the insanities of de_thread it is possible for a process
28290804ef4bSEric W. Biederman  * to have the pid of the thread group leader without actually being
28300804ef4bSEric W. Biederman  * the thread group leader.  For iteration through the pids in proc
28310804ef4bSEric W. Biederman  * all we care about is that we have a task with the appropriate
28320804ef4bSEric W. Biederman  * pid, we don't actually care if we have the right task.
28330804ef4bSEric W. Biederman  */
2834e1403b8eSOleg Nesterov static inline bool has_group_leader_pid(struct task_struct *p)
28350804ef4bSEric W. Biederman {
2836e1403b8eSOleg Nesterov 	return task_pid(p) == p->signal->leader_pid;
28370804ef4bSEric W. Biederman }
28380804ef4bSEric W. Biederman 
2839bac0abd6SPavel Emelyanov static inline
2840e1403b8eSOleg Nesterov bool same_thread_group(struct task_struct *p1, struct task_struct *p2)
2841bac0abd6SPavel Emelyanov {
2842e1403b8eSOleg Nesterov 	return p1->signal == p2->signal;
2843bac0abd6SPavel Emelyanov }
2844bac0abd6SPavel Emelyanov 
284536c8b586SIngo Molnar static inline struct task_struct *next_thread(const struct task_struct *p)
284647e65328SOleg Nesterov {
284705725f7eSJiri Pirko 	return list_entry_rcu(p->thread_group.next,
284836c8b586SIngo Molnar 			      struct task_struct, thread_group);
284947e65328SOleg Nesterov }
285047e65328SOleg Nesterov 
2851e868171aSAlexey Dobriyan static inline int thread_group_empty(struct task_struct *p)
28521da177e4SLinus Torvalds {
285347e65328SOleg Nesterov 	return list_empty(&p->thread_group);
28541da177e4SLinus Torvalds }
28551da177e4SLinus Torvalds 
28561da177e4SLinus Torvalds #define delay_group_leader(p) \
28571da177e4SLinus Torvalds 		(thread_group_leader(p) && !thread_group_empty(p))
28581da177e4SLinus Torvalds 
28591da177e4SLinus Torvalds /*
2860260ea101SEric W. Biederman  * Protects ->fs, ->files, ->mm, ->group_info, ->comm, keyring
286122e2c507SJens Axboe  * subscriptions and synchronises with wait4().  Also used in procfs.  Also
2862ddbcc7e8SPaul Menage  * pins the final release of task.io_context.  Also protects ->cpuset and
2863d68b46feSOleg Nesterov  * ->cgroup.subsys[]. And ->vfork_done.
28641da177e4SLinus Torvalds  *
28651da177e4SLinus Torvalds  * Nests both inside and outside of read_lock(&tasklist_lock).
28661da177e4SLinus Torvalds  * It must not be nested with write_lock_irq(&tasklist_lock),
28671da177e4SLinus Torvalds  * neither inside nor outside.
28681da177e4SLinus Torvalds  */
28691da177e4SLinus Torvalds static inline void task_lock(struct task_struct *p)
28701da177e4SLinus Torvalds {
28711da177e4SLinus Torvalds 	spin_lock(&p->alloc_lock);
28721da177e4SLinus Torvalds }
28731da177e4SLinus Torvalds 
28741da177e4SLinus Torvalds static inline void task_unlock(struct task_struct *p)
28751da177e4SLinus Torvalds {
28761da177e4SLinus Torvalds 	spin_unlock(&p->alloc_lock);
28771da177e4SLinus Torvalds }
28781da177e4SLinus Torvalds 
2879b8ed374eSNamhyung Kim extern struct sighand_struct *__lock_task_sighand(struct task_struct *tsk,
2880f63ee72eSOleg Nesterov 							unsigned long *flags);
2881f63ee72eSOleg Nesterov 
28829388dc30SAnton Vorontsov static inline struct sighand_struct *lock_task_sighand(struct task_struct *tsk,
28839388dc30SAnton Vorontsov 						       unsigned long *flags)
28849388dc30SAnton Vorontsov {
28859388dc30SAnton Vorontsov 	struct sighand_struct *ret;
28869388dc30SAnton Vorontsov 
28879388dc30SAnton Vorontsov 	ret = __lock_task_sighand(tsk, flags);
28889388dc30SAnton Vorontsov 	(void)__cond_lock(&tsk->sighand->siglock, ret);
28899388dc30SAnton Vorontsov 	return ret;
28909388dc30SAnton Vorontsov }
2891b8ed374eSNamhyung Kim 
2892f63ee72eSOleg Nesterov static inline void unlock_task_sighand(struct task_struct *tsk,
2893f63ee72eSOleg Nesterov 						unsigned long *flags)
2894f63ee72eSOleg Nesterov {
2895f63ee72eSOleg Nesterov 	spin_unlock_irqrestore(&tsk->sighand->siglock, *flags);
2896f63ee72eSOleg Nesterov }
2897f63ee72eSOleg Nesterov 
2898c65eacbeSAndy Lutomirski #ifdef CONFIG_THREAD_INFO_IN_TASK
2899c65eacbeSAndy Lutomirski 
2900c65eacbeSAndy Lutomirski static inline struct thread_info *task_thread_info(struct task_struct *task)
2901c65eacbeSAndy Lutomirski {
2902c65eacbeSAndy Lutomirski 	return &task->thread_info;
2903c65eacbeSAndy Lutomirski }
2904c6c314a6SAndy Lutomirski 
2905c6c314a6SAndy Lutomirski /*
2906c6c314a6SAndy Lutomirski  * When accessing the stack of a non-current task that might exit, use
2907c6c314a6SAndy Lutomirski  * try_get_task_stack() instead.  task_stack_page will return a pointer
2908c6c314a6SAndy Lutomirski  * that could get freed out from under you.
2909c6c314a6SAndy Lutomirski  */
2910c65eacbeSAndy Lutomirski static inline void *task_stack_page(const struct task_struct *task)
2911c65eacbeSAndy Lutomirski {
2912c65eacbeSAndy Lutomirski 	return task->stack;
2913c65eacbeSAndy Lutomirski }
2914c6c314a6SAndy Lutomirski 
2915c65eacbeSAndy Lutomirski #define setup_thread_stack(new,old)	do { } while(0)
2916c6c314a6SAndy Lutomirski 
2917c65eacbeSAndy Lutomirski static inline unsigned long *end_of_stack(const struct task_struct *task)
2918c65eacbeSAndy Lutomirski {
2919c65eacbeSAndy Lutomirski 	return task->stack;
2920c65eacbeSAndy Lutomirski }
2921c65eacbeSAndy Lutomirski 
2922c65eacbeSAndy Lutomirski #elif !defined(__HAVE_THREAD_FUNCTIONS)
2923f037360fSAl Viro 
2924f7e4217bSRoman Zippel #define task_thread_info(task)	((struct thread_info *)(task)->stack)
2925c65eacbeSAndy Lutomirski #define task_stack_page(task)	((void *)(task)->stack)
2926a1261f54SAl Viro 
292710ebffdeSAl Viro static inline void setup_thread_stack(struct task_struct *p, struct task_struct *org)
292810ebffdeSAl Viro {
292910ebffdeSAl Viro 	*task_thread_info(p) = *task_thread_info(org);
293010ebffdeSAl Viro 	task_thread_info(p)->task = p;
293110ebffdeSAl Viro }
293210ebffdeSAl Viro 
29336a40281aSChuck Ebbert /*
29346a40281aSChuck Ebbert  * Return the address of the last usable long on the stack.
29356a40281aSChuck Ebbert  *
29366a40281aSChuck Ebbert  * When the stack grows down, this is just above the thread
29376a40281aSChuck Ebbert  * info struct. Going any lower will corrupt the threadinfo.
29386a40281aSChuck Ebbert  *
29396a40281aSChuck Ebbert  * When the stack grows up, this is the highest address.
29406a40281aSChuck Ebbert  * Beyond that position, we corrupt data on the next page.
29416a40281aSChuck Ebbert  */
294210ebffdeSAl Viro static inline unsigned long *end_of_stack(struct task_struct *p)
294310ebffdeSAl Viro {
29446a40281aSChuck Ebbert #ifdef CONFIG_STACK_GROWSUP
29456a40281aSChuck Ebbert 	return (unsigned long *)((unsigned long)task_thread_info(p) + THREAD_SIZE) - 1;
29466a40281aSChuck Ebbert #else
2947f7e4217bSRoman Zippel 	return (unsigned long *)(task_thread_info(p) + 1);
29486a40281aSChuck Ebbert #endif
294910ebffdeSAl Viro }
295010ebffdeSAl Viro 
2951f037360fSAl Viro #endif
2952c6c314a6SAndy Lutomirski 
295368f24b08SAndy Lutomirski #ifdef CONFIG_THREAD_INFO_IN_TASK
295468f24b08SAndy Lutomirski static inline void *try_get_task_stack(struct task_struct *tsk)
295568f24b08SAndy Lutomirski {
295668f24b08SAndy Lutomirski 	return atomic_inc_not_zero(&tsk->stack_refcount) ?
295768f24b08SAndy Lutomirski 		task_stack_page(tsk) : NULL;
295868f24b08SAndy Lutomirski }
295968f24b08SAndy Lutomirski 
296068f24b08SAndy Lutomirski extern void put_task_stack(struct task_struct *tsk);
296168f24b08SAndy Lutomirski #else
2962c6c314a6SAndy Lutomirski static inline void *try_get_task_stack(struct task_struct *tsk)
2963c6c314a6SAndy Lutomirski {
2964c6c314a6SAndy Lutomirski 	return task_stack_page(tsk);
2965c6c314a6SAndy Lutomirski }
2966c6c314a6SAndy Lutomirski 
2967c6c314a6SAndy Lutomirski static inline void put_task_stack(struct task_struct *tsk) {}
296868f24b08SAndy Lutomirski #endif
2969c6c314a6SAndy Lutomirski 
2970a70857e4SAaron Tomlin #define task_stack_end_corrupted(task) \
2971a70857e4SAaron Tomlin 		(*(end_of_stack(task)) != STACK_END_MAGIC)
2972f037360fSAl Viro 
29738b05c7e6SFUJITA Tomonori static inline int object_is_on_stack(void *obj)
29748b05c7e6SFUJITA Tomonori {
29758b05c7e6SFUJITA Tomonori 	void *stack = task_stack_page(current);
29768b05c7e6SFUJITA Tomonori 
29778b05c7e6SFUJITA Tomonori 	return (obj >= stack) && (obj < (stack + THREAD_SIZE));
29788b05c7e6SFUJITA Tomonori }
29798b05c7e6SFUJITA Tomonori 
2980b235beeaSLinus Torvalds extern void thread_stack_cache_init(void);
29818c9843e5SBenjamin Herrenschmidt 
29827c9f8861SEric Sandeen #ifdef CONFIG_DEBUG_STACK_USAGE
29837c9f8861SEric Sandeen static inline unsigned long stack_not_used(struct task_struct *p)
29847c9f8861SEric Sandeen {
29857c9f8861SEric Sandeen 	unsigned long *n = end_of_stack(p);
29867c9f8861SEric Sandeen 
29877c9f8861SEric Sandeen 	do { 	/* Skip over canary */
29886c31da34SHelge Deller # ifdef CONFIG_STACK_GROWSUP
29896c31da34SHelge Deller 		n--;
29906c31da34SHelge Deller # else
29917c9f8861SEric Sandeen 		n++;
29926c31da34SHelge Deller # endif
29937c9f8861SEric Sandeen 	} while (!*n);
29947c9f8861SEric Sandeen 
29956c31da34SHelge Deller # ifdef CONFIG_STACK_GROWSUP
29966c31da34SHelge Deller 	return (unsigned long)end_of_stack(p) - (unsigned long)n;
29976c31da34SHelge Deller # else
29987c9f8861SEric Sandeen 	return (unsigned long)n - (unsigned long)end_of_stack(p);
29996c31da34SHelge Deller # endif
30007c9f8861SEric Sandeen }
30017c9f8861SEric Sandeen #endif
3002d4311ff1SAaron Tomlin extern void set_task_stack_end_magic(struct task_struct *tsk);
30037c9f8861SEric Sandeen 
30041da177e4SLinus Torvalds /* set thread flags in other task's structures
30051da177e4SLinus Torvalds  * - see asm/thread_info.h for TIF_xxxx flags available
30061da177e4SLinus Torvalds  */
30071da177e4SLinus Torvalds static inline void set_tsk_thread_flag(struct task_struct *tsk, int flag)
30081da177e4SLinus Torvalds {
3009a1261f54SAl Viro 	set_ti_thread_flag(task_thread_info(tsk), flag);
30101da177e4SLinus Torvalds }
30111da177e4SLinus Torvalds 
30121da177e4SLinus Torvalds static inline void clear_tsk_thread_flag(struct task_struct *tsk, int flag)
30131da177e4SLinus Torvalds {
3014a1261f54SAl Viro 	clear_ti_thread_flag(task_thread_info(tsk), flag);
30151da177e4SLinus Torvalds }
30161da177e4SLinus Torvalds 
30171da177e4SLinus Torvalds static inline int test_and_set_tsk_thread_flag(struct task_struct *tsk, int flag)
30181da177e4SLinus Torvalds {
3019a1261f54SAl Viro 	return test_and_set_ti_thread_flag(task_thread_info(tsk), flag);
30201da177e4SLinus Torvalds }
30211da177e4SLinus Torvalds 
30221da177e4SLinus Torvalds static inline int test_and_clear_tsk_thread_flag(struct task_struct *tsk, int flag)
30231da177e4SLinus Torvalds {
3024a1261f54SAl Viro 	return test_and_clear_ti_thread_flag(task_thread_info(tsk), flag);
30251da177e4SLinus Torvalds }
30261da177e4SLinus Torvalds 
30271da177e4SLinus Torvalds static inline int test_tsk_thread_flag(struct task_struct *tsk, int flag)
30281da177e4SLinus Torvalds {
3029a1261f54SAl Viro 	return test_ti_thread_flag(task_thread_info(tsk), flag);
30301da177e4SLinus Torvalds }
30311da177e4SLinus Torvalds 
30321da177e4SLinus Torvalds static inline void set_tsk_need_resched(struct task_struct *tsk)
30331da177e4SLinus Torvalds {
30341da177e4SLinus Torvalds 	set_tsk_thread_flag(tsk,TIF_NEED_RESCHED);
30351da177e4SLinus Torvalds }
30361da177e4SLinus Torvalds 
30371da177e4SLinus Torvalds static inline void clear_tsk_need_resched(struct task_struct *tsk)
30381da177e4SLinus Torvalds {
30391da177e4SLinus Torvalds 	clear_tsk_thread_flag(tsk,TIF_NEED_RESCHED);
30401da177e4SLinus Torvalds }
30411da177e4SLinus Torvalds 
30428ae121acSGregory Haskins static inline int test_tsk_need_resched(struct task_struct *tsk)
30438ae121acSGregory Haskins {
30448ae121acSGregory Haskins 	return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED));
30458ae121acSGregory Haskins }
30468ae121acSGregory Haskins 
3047690cc3ffSEric W. Biederman static inline int restart_syscall(void)
3048690cc3ffSEric W. Biederman {
3049690cc3ffSEric W. Biederman 	set_tsk_thread_flag(current, TIF_SIGPENDING);
3050690cc3ffSEric W. Biederman 	return -ERESTARTNOINTR;
3051690cc3ffSEric W. Biederman }
3052690cc3ffSEric W. Biederman 
30531da177e4SLinus Torvalds static inline int signal_pending(struct task_struct *p)
30541da177e4SLinus Torvalds {
30551da177e4SLinus Torvalds 	return unlikely(test_tsk_thread_flag(p,TIF_SIGPENDING));
30561da177e4SLinus Torvalds }
30571da177e4SLinus Torvalds 
3058d9588725SRoland McGrath static inline int __fatal_signal_pending(struct task_struct *p)
3059d9588725SRoland McGrath {
3060d9588725SRoland McGrath 	return unlikely(sigismember(&p->pending.signal, SIGKILL));
3061d9588725SRoland McGrath }
3062f776d12dSMatthew Wilcox 
3063f776d12dSMatthew Wilcox static inline int fatal_signal_pending(struct task_struct *p)
3064f776d12dSMatthew Wilcox {
3065f776d12dSMatthew Wilcox 	return signal_pending(p) && __fatal_signal_pending(p);
3066f776d12dSMatthew Wilcox }
3067f776d12dSMatthew Wilcox 
306816882c1eSOleg Nesterov static inline int signal_pending_state(long state, struct task_struct *p)
306916882c1eSOleg Nesterov {
307016882c1eSOleg Nesterov 	if (!(state & (TASK_INTERRUPTIBLE | TASK_WAKEKILL)))
307116882c1eSOleg Nesterov 		return 0;
307216882c1eSOleg Nesterov 	if (!signal_pending(p))
307316882c1eSOleg Nesterov 		return 0;
307416882c1eSOleg Nesterov 
307516882c1eSOleg Nesterov 	return (state & TASK_INTERRUPTIBLE) || __fatal_signal_pending(p);
307616882c1eSOleg Nesterov }
307716882c1eSOleg Nesterov 
30781da177e4SLinus Torvalds /*
30791da177e4SLinus Torvalds  * cond_resched() and cond_resched_lock(): latency reduction via
30801da177e4SLinus Torvalds  * explicit rescheduling in places that are safe. The return
30811da177e4SLinus Torvalds  * value indicates whether a reschedule was done in fact.
30821da177e4SLinus Torvalds  * cond_resched_lock() will drop the spinlock before scheduling,
30831da177e4SLinus Torvalds  * cond_resched_softirq() will enable bhs before scheduling.
30841da177e4SLinus Torvalds  */
308535a773a0SPeter Zijlstra #ifndef CONFIG_PREEMPT
3086c3921ab7SLinus Torvalds extern int _cond_resched(void);
308735a773a0SPeter Zijlstra #else
308835a773a0SPeter Zijlstra static inline int _cond_resched(void) { return 0; }
308935a773a0SPeter Zijlstra #endif
30906f80bd98SFrederic Weisbecker 
3091613afbf8SFrederic Weisbecker #define cond_resched() ({			\
30923427445aSPeter Zijlstra 	___might_sleep(__FILE__, __LINE__, 0);	\
3093613afbf8SFrederic Weisbecker 	_cond_resched();			\
3094613afbf8SFrederic Weisbecker })
30956f80bd98SFrederic Weisbecker 
3096613afbf8SFrederic Weisbecker extern int __cond_resched_lock(spinlock_t *lock);
3097613afbf8SFrederic Weisbecker 
3098613afbf8SFrederic Weisbecker #define cond_resched_lock(lock) ({				\
30993427445aSPeter Zijlstra 	___might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET);\
3100613afbf8SFrederic Weisbecker 	__cond_resched_lock(lock);				\
3101613afbf8SFrederic Weisbecker })
3102613afbf8SFrederic Weisbecker 
3103613afbf8SFrederic Weisbecker extern int __cond_resched_softirq(void);
3104613afbf8SFrederic Weisbecker 
3105613afbf8SFrederic Weisbecker #define cond_resched_softirq() ({					\
31063427445aSPeter Zijlstra 	___might_sleep(__FILE__, __LINE__, SOFTIRQ_DISABLE_OFFSET);	\
3107613afbf8SFrederic Weisbecker 	__cond_resched_softirq();					\
3108613afbf8SFrederic Weisbecker })
31091da177e4SLinus Torvalds 
3110f6f3c437SSimon Horman static inline void cond_resched_rcu(void)
3111f6f3c437SSimon Horman {
3112f6f3c437SSimon Horman #if defined(CONFIG_DEBUG_ATOMIC_SLEEP) || !defined(CONFIG_PREEMPT_RCU)
3113f6f3c437SSimon Horman 	rcu_read_unlock();
3114f6f3c437SSimon Horman 	cond_resched();
3115f6f3c437SSimon Horman 	rcu_read_lock();
3116f6f3c437SSimon Horman #endif
3117f6f3c437SSimon Horman }
3118f6f3c437SSimon Horman 
31191da177e4SLinus Torvalds /*
31201da177e4SLinus Torvalds  * Does a critical section need to be broken due to another
312195c354feSNick Piggin  * task waiting?: (technically does not depend on CONFIG_PREEMPT,
312295c354feSNick Piggin  * but a general need for low latency)
31231da177e4SLinus Torvalds  */
312495c354feSNick Piggin static inline int spin_needbreak(spinlock_t *lock)
31251da177e4SLinus Torvalds {
312695c354feSNick Piggin #ifdef CONFIG_PREEMPT
312795c354feSNick Piggin 	return spin_is_contended(lock);
312895c354feSNick Piggin #else
31291da177e4SLinus Torvalds 	return 0;
313095c354feSNick Piggin #endif
31311da177e4SLinus Torvalds }
31321da177e4SLinus Torvalds 
313375f93fedSPeter Zijlstra static __always_inline bool need_resched(void)
313475f93fedSPeter Zijlstra {
313575f93fedSPeter Zijlstra 	return unlikely(tif_need_resched());
313675f93fedSPeter Zijlstra }
313775f93fedSPeter Zijlstra 
3138ee761f62SThomas Gleixner /*
3139f06febc9SFrank Mayhar  * Thread group CPU time accounting.
3140f06febc9SFrank Mayhar  */
31414cd4c1b4SPeter Zijlstra void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times);
31424da94d49SPeter Zijlstra void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times);
3143f06febc9SFrank Mayhar 
3144f06febc9SFrank Mayhar /*
31457bb44adeSRoland McGrath  * Reevaluate whether the task has signals pending delivery.
31467bb44adeSRoland McGrath  * Wake the task if so.
31477bb44adeSRoland McGrath  * This is required every time the blocked sigset_t changes.
31487bb44adeSRoland McGrath  * callers must hold sighand->siglock.
31497bb44adeSRoland McGrath  */
31507bb44adeSRoland McGrath extern void recalc_sigpending_and_wake(struct task_struct *t);
31511da177e4SLinus Torvalds extern void recalc_sigpending(void);
31521da177e4SLinus Torvalds 
3153910ffdb1SOleg Nesterov extern void signal_wake_up_state(struct task_struct *t, unsigned int state);
3154910ffdb1SOleg Nesterov 
3155910ffdb1SOleg Nesterov static inline void signal_wake_up(struct task_struct *t, bool resume)
3156910ffdb1SOleg Nesterov {
3157910ffdb1SOleg Nesterov 	signal_wake_up_state(t, resume ? TASK_WAKEKILL : 0);
3158910ffdb1SOleg Nesterov }
3159910ffdb1SOleg Nesterov static inline void ptrace_signal_wake_up(struct task_struct *t, bool resume)
3160910ffdb1SOleg Nesterov {
3161910ffdb1SOleg Nesterov 	signal_wake_up_state(t, resume ? __TASK_TRACED : 0);
3162910ffdb1SOleg Nesterov }
31631da177e4SLinus Torvalds 
31641da177e4SLinus Torvalds /*
31651da177e4SLinus Torvalds  * Wrappers for p->thread_info->cpu access. No-op on UP.
31661da177e4SLinus Torvalds  */
31671da177e4SLinus Torvalds #ifdef CONFIG_SMP
31681da177e4SLinus Torvalds 
31691da177e4SLinus Torvalds static inline unsigned int task_cpu(const struct task_struct *p)
31701da177e4SLinus Torvalds {
3171c65eacbeSAndy Lutomirski #ifdef CONFIG_THREAD_INFO_IN_TASK
3172c65eacbeSAndy Lutomirski 	return p->cpu;
3173c65eacbeSAndy Lutomirski #else
3174a1261f54SAl Viro 	return task_thread_info(p)->cpu;
3175c65eacbeSAndy Lutomirski #endif
31761da177e4SLinus Torvalds }
31771da177e4SLinus Torvalds 
3178b32e86b4SIngo Molnar static inline int task_node(const struct task_struct *p)
3179b32e86b4SIngo Molnar {
3180b32e86b4SIngo Molnar 	return cpu_to_node(task_cpu(p));
3181b32e86b4SIngo Molnar }
3182b32e86b4SIngo Molnar 
3183c65cc870SIngo Molnar extern void set_task_cpu(struct task_struct *p, unsigned int cpu);
31841da177e4SLinus Torvalds 
31851da177e4SLinus Torvalds #else
31861da177e4SLinus Torvalds 
31871da177e4SLinus Torvalds static inline unsigned int task_cpu(const struct task_struct *p)
31881da177e4SLinus Torvalds {
31891da177e4SLinus Torvalds 	return 0;
31901da177e4SLinus Torvalds }
31911da177e4SLinus Torvalds 
31921da177e4SLinus Torvalds static inline void set_task_cpu(struct task_struct *p, unsigned int cpu)
31931da177e4SLinus Torvalds {
31941da177e4SLinus Torvalds }
31951da177e4SLinus Torvalds 
31961da177e4SLinus Torvalds #endif /* CONFIG_SMP */
31971da177e4SLinus Torvalds 
3198d9345c65SPan Xinhui /*
3199d9345c65SPan Xinhui  * In order to reduce various lock holder preemption latencies provide an
3200d9345c65SPan Xinhui  * interface to see if a vCPU is currently running or not.
3201d9345c65SPan Xinhui  *
3202d9345c65SPan Xinhui  * This allows us to terminate optimistic spin loops and block, analogous to
3203d9345c65SPan Xinhui  * the native optimistic spin heuristic of testing if the lock owner task is
3204d9345c65SPan Xinhui  * running or not.
3205d9345c65SPan Xinhui  */
3206d9345c65SPan Xinhui #ifndef vcpu_is_preempted
3207d9345c65SPan Xinhui # define vcpu_is_preempted(cpu)	false
3208d9345c65SPan Xinhui #endif
3209d9345c65SPan Xinhui 
321096f874e2SRusty Russell extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask);
321196f874e2SRusty Russell extern long sched_getaffinity(pid_t pid, struct cpumask *mask);
32125c45bf27SSiddha, Suresh B 
32137c941438SDhaval Giani #ifdef CONFIG_CGROUP_SCHED
321407e06b01SYong Zhang extern struct task_group root_task_group;
32158323f26cSPeter Zijlstra #endif /* CONFIG_CGROUP_SCHED */
32169b5b7751SSrivatsa Vaddagiri 
321754e99124SDhaval Giani extern int task_can_switch_user(struct user_struct *up,
321854e99124SDhaval Giani 					struct task_struct *tsk);
321954e99124SDhaval Giani 
32204b98d11bSAlexey Dobriyan #ifdef CONFIG_TASK_XACCT
32214b98d11bSAlexey Dobriyan static inline void add_rchar(struct task_struct *tsk, ssize_t amt)
32224b98d11bSAlexey Dobriyan {
3223940389b8SAndrea Righi 	tsk->ioac.rchar += amt;
32244b98d11bSAlexey Dobriyan }
32254b98d11bSAlexey Dobriyan 
32264b98d11bSAlexey Dobriyan static inline void add_wchar(struct task_struct *tsk, ssize_t amt)
32274b98d11bSAlexey Dobriyan {
3228940389b8SAndrea Righi 	tsk->ioac.wchar += amt;
32294b98d11bSAlexey Dobriyan }
32304b98d11bSAlexey Dobriyan 
32314b98d11bSAlexey Dobriyan static inline void inc_syscr(struct task_struct *tsk)
32324b98d11bSAlexey Dobriyan {
3233940389b8SAndrea Righi 	tsk->ioac.syscr++;
32344b98d11bSAlexey Dobriyan }
32354b98d11bSAlexey Dobriyan 
32364b98d11bSAlexey Dobriyan static inline void inc_syscw(struct task_struct *tsk)
32374b98d11bSAlexey Dobriyan {
3238940389b8SAndrea Righi 	tsk->ioac.syscw++;
32394b98d11bSAlexey Dobriyan }
32404b98d11bSAlexey Dobriyan #else
32414b98d11bSAlexey Dobriyan static inline void add_rchar(struct task_struct *tsk, ssize_t amt)
32424b98d11bSAlexey Dobriyan {
32434b98d11bSAlexey Dobriyan }
32444b98d11bSAlexey Dobriyan 
32454b98d11bSAlexey Dobriyan static inline void add_wchar(struct task_struct *tsk, ssize_t amt)
32464b98d11bSAlexey Dobriyan {
32474b98d11bSAlexey Dobriyan }
32484b98d11bSAlexey Dobriyan 
32494b98d11bSAlexey Dobriyan static inline void inc_syscr(struct task_struct *tsk)
32504b98d11bSAlexey Dobriyan {
32514b98d11bSAlexey Dobriyan }
32524b98d11bSAlexey Dobriyan 
32534b98d11bSAlexey Dobriyan static inline void inc_syscw(struct task_struct *tsk)
32544b98d11bSAlexey Dobriyan {
32554b98d11bSAlexey Dobriyan }
32564b98d11bSAlexey Dobriyan #endif
32574b98d11bSAlexey Dobriyan 
325882455257SDave Hansen #ifndef TASK_SIZE_OF
325982455257SDave Hansen #define TASK_SIZE_OF(tsk)	TASK_SIZE
326082455257SDave Hansen #endif
326182455257SDave Hansen 
3262f98bafa0SOleg Nesterov #ifdef CONFIG_MEMCG
3263cf475ad2SBalbir Singh extern void mm_update_next_owner(struct mm_struct *mm);
3264cf475ad2SBalbir Singh #else
3265cf475ad2SBalbir Singh static inline void mm_update_next_owner(struct mm_struct *mm)
3266cf475ad2SBalbir Singh {
3267cf475ad2SBalbir Singh }
3268f98bafa0SOleg Nesterov #endif /* CONFIG_MEMCG */
3269cf475ad2SBalbir Singh 
32703e10e716SJiri Slaby static inline unsigned long task_rlimit(const struct task_struct *tsk,
32713e10e716SJiri Slaby 		unsigned int limit)
32723e10e716SJiri Slaby {
3273316c1608SJason Low 	return READ_ONCE(tsk->signal->rlim[limit].rlim_cur);
32743e10e716SJiri Slaby }
32753e10e716SJiri Slaby 
32763e10e716SJiri Slaby static inline unsigned long task_rlimit_max(const struct task_struct *tsk,
32773e10e716SJiri Slaby 		unsigned int limit)
32783e10e716SJiri Slaby {
3279316c1608SJason Low 	return READ_ONCE(tsk->signal->rlim[limit].rlim_max);
32803e10e716SJiri Slaby }
32813e10e716SJiri Slaby 
32823e10e716SJiri Slaby static inline unsigned long rlimit(unsigned int limit)
32833e10e716SJiri Slaby {
32843e10e716SJiri Slaby 	return task_rlimit(current, limit);
32853e10e716SJiri Slaby }
32863e10e716SJiri Slaby 
32873e10e716SJiri Slaby static inline unsigned long rlimit_max(unsigned int limit)
32883e10e716SJiri Slaby {
32893e10e716SJiri Slaby 	return task_rlimit_max(current, limit);
32903e10e716SJiri Slaby }
32913e10e716SJiri Slaby 
329258919e83SRafael J. Wysocki #define SCHED_CPUFREQ_RT	(1U << 0)
329358919e83SRafael J. Wysocki #define SCHED_CPUFREQ_DL	(1U << 1)
32948c34ab19SRafael J. Wysocki #define SCHED_CPUFREQ_IOWAIT	(1U << 2)
329558919e83SRafael J. Wysocki 
329658919e83SRafael J. Wysocki #define SCHED_CPUFREQ_RT_DL	(SCHED_CPUFREQ_RT | SCHED_CPUFREQ_DL)
329758919e83SRafael J. Wysocki 
3298adaf9fcdSRafael J. Wysocki #ifdef CONFIG_CPU_FREQ
3299adaf9fcdSRafael J. Wysocki struct update_util_data {
330058919e83SRafael J. Wysocki        void (*func)(struct update_util_data *data, u64 time, unsigned int flags);
3301adaf9fcdSRafael J. Wysocki };
3302adaf9fcdSRafael J. Wysocki 
33030bed612bSRafael J. Wysocki void cpufreq_add_update_util_hook(int cpu, struct update_util_data *data,
33040bed612bSRafael J. Wysocki                        void (*func)(struct update_util_data *data, u64 time,
330558919e83SRafael J. Wysocki 				    unsigned int flags));
33060bed612bSRafael J. Wysocki void cpufreq_remove_update_util_hook(int cpu);
3307adaf9fcdSRafael J. Wysocki #endif /* CONFIG_CPU_FREQ */
3308adaf9fcdSRafael J. Wysocki 
33091da177e4SLinus Torvalds #endif
3310