xref: /linux/include/linux/sched.h (revision f009a7a767e792d5ab0b46c08d46236ea5271dd9)
11da177e4SLinus Torvalds #ifndef _LINUX_SCHED_H
21da177e4SLinus Torvalds #define _LINUX_SCHED_H
31da177e4SLinus Torvalds 
4607ca46eSDavid Howells #include <uapi/linux/sched.h>
5b7b3c76aSDavid Woodhouse 
65c228079SDongsheng Yang #include <linux/sched/prio.h>
75c228079SDongsheng Yang 
8b7b3c76aSDavid Woodhouse 
9b7b3c76aSDavid Woodhouse struct sched_param {
10b7b3c76aSDavid Woodhouse 	int sched_priority;
11b7b3c76aSDavid Woodhouse };
12b7b3c76aSDavid Woodhouse 
131da177e4SLinus Torvalds #include <asm/param.h>	/* for HZ */
141da177e4SLinus Torvalds 
151da177e4SLinus Torvalds #include <linux/capability.h>
161da177e4SLinus Torvalds #include <linux/threads.h>
171da177e4SLinus Torvalds #include <linux/kernel.h>
181da177e4SLinus Torvalds #include <linux/types.h>
191da177e4SLinus Torvalds #include <linux/timex.h>
201da177e4SLinus Torvalds #include <linux/jiffies.h>
21fb00aca4SPeter Zijlstra #include <linux/plist.h>
221da177e4SLinus Torvalds #include <linux/rbtree.h>
231da177e4SLinus Torvalds #include <linux/thread_info.h>
241da177e4SLinus Torvalds #include <linux/cpumask.h>
251da177e4SLinus Torvalds #include <linux/errno.h>
261da177e4SLinus Torvalds #include <linux/nodemask.h>
27c92ff1bdSMartin Schwidefsky #include <linux/mm_types.h>
2892cf2118SFrederic Weisbecker #include <linux/preempt.h>
291da177e4SLinus Torvalds 
301da177e4SLinus Torvalds #include <asm/page.h>
311da177e4SLinus Torvalds #include <asm/ptrace.h>
32bfc3f028SFrederic Weisbecker #include <linux/cputime.h>
331da177e4SLinus Torvalds 
341da177e4SLinus Torvalds #include <linux/smp.h>
351da177e4SLinus Torvalds #include <linux/sem.h>
36ab602f79SJack Miller #include <linux/shm.h>
371da177e4SLinus Torvalds #include <linux/signal.h>
381da177e4SLinus Torvalds #include <linux/compiler.h>
391da177e4SLinus Torvalds #include <linux/completion.h>
401da177e4SLinus Torvalds #include <linux/pid.h>
411da177e4SLinus Torvalds #include <linux/percpu.h>
421da177e4SLinus Torvalds #include <linux/topology.h>
433e26c149SPeter Zijlstra #include <linux/proportions.h>
441da177e4SLinus Torvalds #include <linux/seccomp.h>
45e56d0903SIngo Molnar #include <linux/rcupdate.h>
4605725f7eSJiri Pirko #include <linux/rculist.h>
4723f78d4aSIngo Molnar #include <linux/rtmutex.h>
481da177e4SLinus Torvalds 
49a3b6714eSDavid Woodhouse #include <linux/time.h>
50a3b6714eSDavid Woodhouse #include <linux/param.h>
51a3b6714eSDavid Woodhouse #include <linux/resource.h>
52a3b6714eSDavid Woodhouse #include <linux/timer.h>
53a3b6714eSDavid Woodhouse #include <linux/hrtimer.h>
545c9a8750SDmitry Vyukov #include <linux/kcov.h>
557c3ab738SAndrew Morton #include <linux/task_io_accounting.h>
569745512cSArjan van de Ven #include <linux/latencytop.h>
579e2b2dc4SDavid Howells #include <linux/cred.h>
58fa14ff4aSPeter Zijlstra #include <linux/llist.h>
597b44ab97SEric W. Biederman #include <linux/uidgid.h>
6021caf2fcSMing Lei #include <linux/gfp.h>
61d4311ff1SAaron Tomlin #include <linux/magic.h>
627d7efec3STejun Heo #include <linux/cgroup-defs.h>
63a3b6714eSDavid Woodhouse 
64a3b6714eSDavid Woodhouse #include <asm/processor.h>
6536d57ac4SH. J. Lu 
66d50dde5aSDario Faggioli #define SCHED_ATTR_SIZE_VER0	48	/* sizeof first published struct */
67d50dde5aSDario Faggioli 
68d50dde5aSDario Faggioli /*
69d50dde5aSDario Faggioli  * Extended scheduling parameters data structure.
70d50dde5aSDario Faggioli  *
71d50dde5aSDario Faggioli  * This is needed because the original struct sched_param can not be
72d50dde5aSDario Faggioli  * altered without introducing ABI issues with legacy applications
73d50dde5aSDario Faggioli  * (e.g., in sched_getparam()).
74d50dde5aSDario Faggioli  *
75d50dde5aSDario Faggioli  * However, the possibility of specifying more than just a priority for
76d50dde5aSDario Faggioli  * the tasks may be useful for a wide variety of application fields, e.g.,
77d50dde5aSDario Faggioli  * multimedia, streaming, automation and control, and many others.
78d50dde5aSDario Faggioli  *
79d50dde5aSDario Faggioli  * This variant (sched_attr) is meant at describing a so-called
80d50dde5aSDario Faggioli  * sporadic time-constrained task. In such model a task is specified by:
81d50dde5aSDario Faggioli  *  - the activation period or minimum instance inter-arrival time;
82d50dde5aSDario Faggioli  *  - the maximum (or average, depending on the actual scheduling
83d50dde5aSDario Faggioli  *    discipline) computation time of all instances, a.k.a. runtime;
84d50dde5aSDario Faggioli  *  - the deadline (relative to the actual activation time) of each
85d50dde5aSDario Faggioli  *    instance.
86d50dde5aSDario Faggioli  * Very briefly, a periodic (sporadic) task asks for the execution of
87d50dde5aSDario Faggioli  * some specific computation --which is typically called an instance--
88d50dde5aSDario Faggioli  * (at most) every period. Moreover, each instance typically lasts no more
89d50dde5aSDario Faggioli  * than the runtime and must be completed by time instant t equal to
90d50dde5aSDario Faggioli  * the instance activation time + the deadline.
91d50dde5aSDario Faggioli  *
92d50dde5aSDario Faggioli  * This is reflected by the actual fields of the sched_attr structure:
93d50dde5aSDario Faggioli  *
94d50dde5aSDario Faggioli  *  @size		size of the structure, for fwd/bwd compat.
95d50dde5aSDario Faggioli  *
96d50dde5aSDario Faggioli  *  @sched_policy	task's scheduling policy
97d50dde5aSDario Faggioli  *  @sched_flags	for customizing the scheduler behaviour
98d50dde5aSDario Faggioli  *  @sched_nice		task's nice value      (SCHED_NORMAL/BATCH)
99d50dde5aSDario Faggioli  *  @sched_priority	task's static priority (SCHED_FIFO/RR)
100d50dde5aSDario Faggioli  *  @sched_deadline	representative of the task's deadline
101d50dde5aSDario Faggioli  *  @sched_runtime	representative of the task's runtime
102d50dde5aSDario Faggioli  *  @sched_period	representative of the task's period
103d50dde5aSDario Faggioli  *
104d50dde5aSDario Faggioli  * Given this task model, there are a multiplicity of scheduling algorithms
105d50dde5aSDario Faggioli  * and policies, that can be used to ensure all the tasks will make their
106d50dde5aSDario Faggioli  * timing constraints.
107aab03e05SDario Faggioli  *
108aab03e05SDario Faggioli  * As of now, the SCHED_DEADLINE policy (sched_dl scheduling class) is the
109aab03e05SDario Faggioli  * only user of this new interface. More information about the algorithm
110aab03e05SDario Faggioli  * available in the scheduling class file or in Documentation/.
111d50dde5aSDario Faggioli  */
112d50dde5aSDario Faggioli struct sched_attr {
113d50dde5aSDario Faggioli 	u32 size;
114d50dde5aSDario Faggioli 
115d50dde5aSDario Faggioli 	u32 sched_policy;
116d50dde5aSDario Faggioli 	u64 sched_flags;
117d50dde5aSDario Faggioli 
118d50dde5aSDario Faggioli 	/* SCHED_NORMAL, SCHED_BATCH */
119d50dde5aSDario Faggioli 	s32 sched_nice;
120d50dde5aSDario Faggioli 
121d50dde5aSDario Faggioli 	/* SCHED_FIFO, SCHED_RR */
122d50dde5aSDario Faggioli 	u32 sched_priority;
123d50dde5aSDario Faggioli 
124d50dde5aSDario Faggioli 	/* SCHED_DEADLINE */
125d50dde5aSDario Faggioli 	u64 sched_runtime;
126d50dde5aSDario Faggioli 	u64 sched_deadline;
127d50dde5aSDario Faggioli 	u64 sched_period;
128d50dde5aSDario Faggioli };
129d50dde5aSDario Faggioli 
130c87e2837SIngo Molnar struct futex_pi_state;
131286100a6SAlexey Dobriyan struct robust_list_head;
132bddd87c7SAkinobu Mita struct bio_list;
1335ad4e53bSAl Viro struct fs_struct;
134cdd6c482SIngo Molnar struct perf_event_context;
13573c10101SJens Axboe struct blk_plug;
136c4ad8f98SLinus Torvalds struct filename;
13789076bc3SAl Viro struct nameidata;
1381da177e4SLinus Torvalds 
139615d6e87SDavidlohr Bueso #define VMACACHE_BITS 2
140615d6e87SDavidlohr Bueso #define VMACACHE_SIZE (1U << VMACACHE_BITS)
141615d6e87SDavidlohr Bueso #define VMACACHE_MASK (VMACACHE_SIZE - 1)
142615d6e87SDavidlohr Bueso 
1431da177e4SLinus Torvalds /*
1441da177e4SLinus Torvalds  * These are the constant used to fake the fixed-point load-average
1451da177e4SLinus Torvalds  * counting. Some notes:
1461da177e4SLinus Torvalds  *  - 11 bit fractions expand to 22 bits by the multiplies: this gives
1471da177e4SLinus Torvalds  *    a load-average precision of 10 bits integer + 11 bits fractional
1481da177e4SLinus Torvalds  *  - if you want to count load-averages more often, you need more
1491da177e4SLinus Torvalds  *    precision, or rounding will get you. With 2-second counting freq,
1501da177e4SLinus Torvalds  *    the EXP_n values would be 1981, 2034 and 2043 if still using only
1511da177e4SLinus Torvalds  *    11 bit fractions.
1521da177e4SLinus Torvalds  */
1531da177e4SLinus Torvalds extern unsigned long avenrun[];		/* Load averages */
1542d02494fSThomas Gleixner extern void get_avenrun(unsigned long *loads, unsigned long offset, int shift);
1551da177e4SLinus Torvalds 
1561da177e4SLinus Torvalds #define FSHIFT		11		/* nr of bits of precision */
1571da177e4SLinus Torvalds #define FIXED_1		(1<<FSHIFT)	/* 1.0 as fixed-point */
1580c2043abSLinus Torvalds #define LOAD_FREQ	(5*HZ+1)	/* 5 sec intervals */
1591da177e4SLinus Torvalds #define EXP_1		1884		/* 1/exp(5sec/1min) as fixed-point */
1601da177e4SLinus Torvalds #define EXP_5		2014		/* 1/exp(5sec/5min) */
1611da177e4SLinus Torvalds #define EXP_15		2037		/* 1/exp(5sec/15min) */
1621da177e4SLinus Torvalds 
1631da177e4SLinus Torvalds #define CALC_LOAD(load,exp,n) \
1641da177e4SLinus Torvalds 	load *= exp; \
1651da177e4SLinus Torvalds 	load += n*(FIXED_1-exp); \
1661da177e4SLinus Torvalds 	load >>= FSHIFT;
1671da177e4SLinus Torvalds 
1681da177e4SLinus Torvalds extern unsigned long total_forks;
1691da177e4SLinus Torvalds extern int nr_threads;
1701da177e4SLinus Torvalds DECLARE_PER_CPU(unsigned long, process_counts);
1711da177e4SLinus Torvalds extern int nr_processes(void);
1721da177e4SLinus Torvalds extern unsigned long nr_running(void);
1732ee507c4STim Chen extern bool single_task_running(void);
1741da177e4SLinus Torvalds extern unsigned long nr_iowait(void);
1758c215bd3SPeter Zijlstra extern unsigned long nr_iowait_cpu(int cpu);
176372ba8cbSMel Gorman extern void get_iowait_load(unsigned long *nr_waiters, unsigned long *load);
17769d25870SArjan van de Ven 
1780f004f5aSPeter Zijlstra extern void calc_global_load(unsigned long ticks);
1793289bdb4SPeter Zijlstra 
1803289bdb4SPeter Zijlstra #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
181525705d1SByungchul Park extern void update_cpu_load_nohz(int active);
1823289bdb4SPeter Zijlstra #else
183525705d1SByungchul Park static inline void update_cpu_load_nohz(int active) { }
1843289bdb4SPeter Zijlstra #endif
1851da177e4SLinus Torvalds 
186b637a328SPaul E. McKenney extern void dump_cpu_task(int cpu);
187b637a328SPaul E. McKenney 
18843ae34cbSIngo Molnar struct seq_file;
18943ae34cbSIngo Molnar struct cfs_rq;
1904cf86d77SIngo Molnar struct task_group;
19143ae34cbSIngo Molnar #ifdef CONFIG_SCHED_DEBUG
19243ae34cbSIngo Molnar extern void proc_sched_show_task(struct task_struct *p, struct seq_file *m);
19343ae34cbSIngo Molnar extern void proc_sched_set_task(struct task_struct *p);
19443ae34cbSIngo Molnar #endif
1951da177e4SLinus Torvalds 
1964a8342d2SLinus Torvalds /*
1974a8342d2SLinus Torvalds  * Task state bitmask. NOTE! These bits are also
1984a8342d2SLinus Torvalds  * encoded in fs/proc/array.c: get_task_state().
1994a8342d2SLinus Torvalds  *
2004a8342d2SLinus Torvalds  * We have two separate sets of flags: task->state
2014a8342d2SLinus Torvalds  * is about runnability, while task->exit_state are
2024a8342d2SLinus Torvalds  * about the task exiting. Confusing, but this way
2034a8342d2SLinus Torvalds  * modifying one set can't modify the other one by
2044a8342d2SLinus Torvalds  * mistake.
2054a8342d2SLinus Torvalds  */
2061da177e4SLinus Torvalds #define TASK_RUNNING		0
2071da177e4SLinus Torvalds #define TASK_INTERRUPTIBLE	1
2081da177e4SLinus Torvalds #define TASK_UNINTERRUPTIBLE	2
209f021a3c2SMatthew Wilcox #define __TASK_STOPPED		4
210f021a3c2SMatthew Wilcox #define __TASK_TRACED		8
2114a8342d2SLinus Torvalds /* in tsk->exit_state */
212ad86622bSOleg Nesterov #define EXIT_DEAD		16
213ad86622bSOleg Nesterov #define EXIT_ZOMBIE		32
214abd50b39SOleg Nesterov #define EXIT_TRACE		(EXIT_ZOMBIE | EXIT_DEAD)
2154a8342d2SLinus Torvalds /* in tsk->state again */
216af927232SMike Galbraith #define TASK_DEAD		64
217f021a3c2SMatthew Wilcox #define TASK_WAKEKILL		128
218e9c84311SPeter Zijlstra #define TASK_WAKING		256
219f2530dc7SThomas Gleixner #define TASK_PARKED		512
22080ed87c8SPeter Zijlstra #define TASK_NOLOAD		1024
22180ed87c8SPeter Zijlstra #define TASK_STATE_MAX		2048
222f021a3c2SMatthew Wilcox 
22380ed87c8SPeter Zijlstra #define TASK_STATE_TO_CHAR_STR "RSDTtXZxKWPN"
22473342151SPeter Zijlstra 
225e1781538SPeter Zijlstra extern char ___assert_task_state[1 - 2*!!(
226e1781538SPeter Zijlstra 		sizeof(TASK_STATE_TO_CHAR_STR)-1 != ilog2(TASK_STATE_MAX)+1)];
227f021a3c2SMatthew Wilcox 
228f021a3c2SMatthew Wilcox /* Convenience macros for the sake of set_task_state */
229f021a3c2SMatthew Wilcox #define TASK_KILLABLE		(TASK_WAKEKILL | TASK_UNINTERRUPTIBLE)
230f021a3c2SMatthew Wilcox #define TASK_STOPPED		(TASK_WAKEKILL | __TASK_STOPPED)
231f021a3c2SMatthew Wilcox #define TASK_TRACED		(TASK_WAKEKILL | __TASK_TRACED)
2321da177e4SLinus Torvalds 
23380ed87c8SPeter Zijlstra #define TASK_IDLE		(TASK_UNINTERRUPTIBLE | TASK_NOLOAD)
23480ed87c8SPeter Zijlstra 
23592a1f4bcSMatthew Wilcox /* Convenience macros for the sake of wake_up */
23692a1f4bcSMatthew Wilcox #define TASK_NORMAL		(TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE)
237f021a3c2SMatthew Wilcox #define TASK_ALL		(TASK_NORMAL | __TASK_STOPPED | __TASK_TRACED)
23892a1f4bcSMatthew Wilcox 
23992a1f4bcSMatthew Wilcox /* get_task_state() */
24092a1f4bcSMatthew Wilcox #define TASK_REPORT		(TASK_RUNNING | TASK_INTERRUPTIBLE | \
241f021a3c2SMatthew Wilcox 				 TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \
24274e37200SOleg Nesterov 				 __TASK_TRACED | EXIT_ZOMBIE | EXIT_DEAD)
24392a1f4bcSMatthew Wilcox 
244f021a3c2SMatthew Wilcox #define task_is_traced(task)	((task->state & __TASK_TRACED) != 0)
245f021a3c2SMatthew Wilcox #define task_is_stopped(task)	((task->state & __TASK_STOPPED) != 0)
24692a1f4bcSMatthew Wilcox #define task_is_stopped_or_traced(task)	\
247f021a3c2SMatthew Wilcox 			((task->state & (__TASK_STOPPED | __TASK_TRACED)) != 0)
24892a1f4bcSMatthew Wilcox #define task_contributes_to_load(task)	\
249e3c8ca83SNathan Lynch 				((task->state & TASK_UNINTERRUPTIBLE) != 0 && \
25080ed87c8SPeter Zijlstra 				 (task->flags & PF_FROZEN) == 0 && \
25180ed87c8SPeter Zijlstra 				 (task->state & TASK_NOLOAD) == 0)
2521da177e4SLinus Torvalds 
2538eb23b9fSPeter Zijlstra #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
2548eb23b9fSPeter Zijlstra 
2558eb23b9fSPeter Zijlstra #define __set_task_state(tsk, state_value)			\
2568eb23b9fSPeter Zijlstra 	do {							\
2578eb23b9fSPeter Zijlstra 		(tsk)->task_state_change = _THIS_IP_;		\
2588eb23b9fSPeter Zijlstra 		(tsk)->state = (state_value);			\
2598eb23b9fSPeter Zijlstra 	} while (0)
2608eb23b9fSPeter Zijlstra #define set_task_state(tsk, state_value)			\
2618eb23b9fSPeter Zijlstra 	do {							\
2628eb23b9fSPeter Zijlstra 		(tsk)->task_state_change = _THIS_IP_;		\
263b92b8b35SPeter Zijlstra 		smp_store_mb((tsk)->state, (state_value));		\
2648eb23b9fSPeter Zijlstra 	} while (0)
2658eb23b9fSPeter Zijlstra 
2668eb23b9fSPeter Zijlstra /*
2678eb23b9fSPeter Zijlstra  * set_current_state() includes a barrier so that the write of current->state
2688eb23b9fSPeter Zijlstra  * is correctly serialised wrt the caller's subsequent test of whether to
2698eb23b9fSPeter Zijlstra  * actually sleep:
2708eb23b9fSPeter Zijlstra  *
2718eb23b9fSPeter Zijlstra  *	set_current_state(TASK_UNINTERRUPTIBLE);
2728eb23b9fSPeter Zijlstra  *	if (do_i_need_to_sleep())
2738eb23b9fSPeter Zijlstra  *		schedule();
2748eb23b9fSPeter Zijlstra  *
2758eb23b9fSPeter Zijlstra  * If the caller does not need such serialisation then use __set_current_state()
2768eb23b9fSPeter Zijlstra  */
2778eb23b9fSPeter Zijlstra #define __set_current_state(state_value)			\
2788eb23b9fSPeter Zijlstra 	do {							\
2798eb23b9fSPeter Zijlstra 		current->task_state_change = _THIS_IP_;		\
2808eb23b9fSPeter Zijlstra 		current->state = (state_value);			\
2818eb23b9fSPeter Zijlstra 	} while (0)
2828eb23b9fSPeter Zijlstra #define set_current_state(state_value)				\
2838eb23b9fSPeter Zijlstra 	do {							\
2848eb23b9fSPeter Zijlstra 		current->task_state_change = _THIS_IP_;		\
285b92b8b35SPeter Zijlstra 		smp_store_mb(current->state, (state_value));		\
2868eb23b9fSPeter Zijlstra 	} while (0)
2878eb23b9fSPeter Zijlstra 
2888eb23b9fSPeter Zijlstra #else
2898eb23b9fSPeter Zijlstra 
2901da177e4SLinus Torvalds #define __set_task_state(tsk, state_value)		\
2911da177e4SLinus Torvalds 	do { (tsk)->state = (state_value); } while (0)
2921da177e4SLinus Torvalds #define set_task_state(tsk, state_value)		\
293b92b8b35SPeter Zijlstra 	smp_store_mb((tsk)->state, (state_value))
2941da177e4SLinus Torvalds 
295498d0c57SAndrew Morton /*
296498d0c57SAndrew Morton  * set_current_state() includes a barrier so that the write of current->state
297498d0c57SAndrew Morton  * is correctly serialised wrt the caller's subsequent test of whether to
298498d0c57SAndrew Morton  * actually sleep:
299498d0c57SAndrew Morton  *
300498d0c57SAndrew Morton  *	set_current_state(TASK_UNINTERRUPTIBLE);
301498d0c57SAndrew Morton  *	if (do_i_need_to_sleep())
302498d0c57SAndrew Morton  *		schedule();
303498d0c57SAndrew Morton  *
304498d0c57SAndrew Morton  * If the caller does not need such serialisation then use __set_current_state()
305498d0c57SAndrew Morton  */
3061da177e4SLinus Torvalds #define __set_current_state(state_value)		\
3071da177e4SLinus Torvalds 	do { current->state = (state_value); } while (0)
3081da177e4SLinus Torvalds #define set_current_state(state_value)			\
309b92b8b35SPeter Zijlstra 	smp_store_mb(current->state, (state_value))
3101da177e4SLinus Torvalds 
3118eb23b9fSPeter Zijlstra #endif
3128eb23b9fSPeter Zijlstra 
3131da177e4SLinus Torvalds /* Task command name length */
3141da177e4SLinus Torvalds #define TASK_COMM_LEN 16
3151da177e4SLinus Torvalds 
3161da177e4SLinus Torvalds #include <linux/spinlock.h>
3171da177e4SLinus Torvalds 
3181da177e4SLinus Torvalds /*
3191da177e4SLinus Torvalds  * This serializes "schedule()" and also protects
3201da177e4SLinus Torvalds  * the run-queue from deletions/modifications (but
3211da177e4SLinus Torvalds  * _adding_ to the beginning of the run-queue has
3221da177e4SLinus Torvalds  * a separate lock).
3231da177e4SLinus Torvalds  */
3241da177e4SLinus Torvalds extern rwlock_t tasklist_lock;
3251da177e4SLinus Torvalds extern spinlock_t mmlist_lock;
3261da177e4SLinus Torvalds 
32736c8b586SIngo Molnar struct task_struct;
3281da177e4SLinus Torvalds 
329db1466b3SPaul E. McKenney #ifdef CONFIG_PROVE_RCU
330db1466b3SPaul E. McKenney extern int lockdep_tasklist_lock_is_held(void);
331db1466b3SPaul E. McKenney #endif /* #ifdef CONFIG_PROVE_RCU */
332db1466b3SPaul E. McKenney 
3331da177e4SLinus Torvalds extern void sched_init(void);
3341da177e4SLinus Torvalds extern void sched_init_smp(void);
3352d07b255SHarvey Harrison extern asmlinkage void schedule_tail(struct task_struct *prev);
33636c8b586SIngo Molnar extern void init_idle(struct task_struct *idle, int cpu);
3371df21055SIngo Molnar extern void init_idle_bootup_task(struct task_struct *idle);
3381da177e4SLinus Torvalds 
3393fa0818bSRik van Riel extern cpumask_var_t cpu_isolated_map;
3403fa0818bSRik van Riel 
34189f19f04SAndrew Morton extern int runqueue_is_locked(int cpu);
342017730c1SIngo Molnar 
3433451d024SFrederic Weisbecker #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
344c1cc017cSAlex Shi extern void nohz_balance_enter_idle(int cpu);
34569e1e811SSuresh Siddha extern void set_cpu_sd_state_idle(void);
346bc7a34b8SThomas Gleixner extern int get_nohz_timer_target(void);
34746cb4b7cSSiddha, Suresh B #else
348c1cc017cSAlex Shi static inline void nohz_balance_enter_idle(int cpu) { }
349fdaabd80SPeter Zijlstra static inline void set_cpu_sd_state_idle(void) { }
35046cb4b7cSSiddha, Suresh B #endif
3511da177e4SLinus Torvalds 
352e59e2ae2SIngo Molnar /*
35339bc89fdSIngo Molnar  * Only dump TASK_* tasks. (0 for all tasks)
354e59e2ae2SIngo Molnar  */
355e59e2ae2SIngo Molnar extern void show_state_filter(unsigned long state_filter);
356e59e2ae2SIngo Molnar 
357e59e2ae2SIngo Molnar static inline void show_state(void)
358e59e2ae2SIngo Molnar {
35939bc89fdSIngo Molnar 	show_state_filter(0);
360e59e2ae2SIngo Molnar }
361e59e2ae2SIngo Molnar 
3621da177e4SLinus Torvalds extern void show_regs(struct pt_regs *);
3631da177e4SLinus Torvalds 
3641da177e4SLinus Torvalds /*
3651da177e4SLinus Torvalds  * TASK is a pointer to the task whose backtrace we want to see (or NULL for current
3661da177e4SLinus Torvalds  * task), SP is the stack pointer of the first frame that should be shown in the back
3671da177e4SLinus Torvalds  * trace (or NULL if the entire call-chain of the task should be shown).
3681da177e4SLinus Torvalds  */
3691da177e4SLinus Torvalds extern void show_stack(struct task_struct *task, unsigned long *sp);
3701da177e4SLinus Torvalds 
3711da177e4SLinus Torvalds extern void cpu_init (void);
3721da177e4SLinus Torvalds extern void trap_init(void);
3731da177e4SLinus Torvalds extern void update_process_times(int user);
3741da177e4SLinus Torvalds extern void scheduler_tick(void);
3751da177e4SLinus Torvalds 
37682a1fcb9SIngo Molnar extern void sched_show_task(struct task_struct *p);
37782a1fcb9SIngo Molnar 
37819cc36c0SFrederic Weisbecker #ifdef CONFIG_LOCKUP_DETECTOR
37903e0d461STejun Heo extern void touch_softlockup_watchdog_sched(void);
3808446f1d3SIngo Molnar extern void touch_softlockup_watchdog(void);
381d6ad3e28SJason Wessel extern void touch_softlockup_watchdog_sync(void);
38204c9167fSJeremy Fitzhardinge extern void touch_all_softlockup_watchdogs(void);
383332fbdbcSDon Zickus extern int proc_dowatchdog_thresh(struct ctl_table *table, int write,
3848d65af78SAlexey Dobriyan 				  void __user *buffer,
385baf48f65SMandeep Singh Baines 				  size_t *lenp, loff_t *ppos);
3869c44bc03SIngo Molnar extern unsigned int  softlockup_panic;
387ac1f5912SDon Zickus extern unsigned int  hardlockup_panic;
388004417a6SPeter Zijlstra void lockup_detector_init(void);
3898446f1d3SIngo Molnar #else
39003e0d461STejun Heo static inline void touch_softlockup_watchdog_sched(void)
39103e0d461STejun Heo {
39203e0d461STejun Heo }
3938446f1d3SIngo Molnar static inline void touch_softlockup_watchdog(void)
3948446f1d3SIngo Molnar {
3958446f1d3SIngo Molnar }
396d6ad3e28SJason Wessel static inline void touch_softlockup_watchdog_sync(void)
397d6ad3e28SJason Wessel {
398d6ad3e28SJason Wessel }
39904c9167fSJeremy Fitzhardinge static inline void touch_all_softlockup_watchdogs(void)
40004c9167fSJeremy Fitzhardinge {
40104c9167fSJeremy Fitzhardinge }
402004417a6SPeter Zijlstra static inline void lockup_detector_init(void)
403004417a6SPeter Zijlstra {
404004417a6SPeter Zijlstra }
4058446f1d3SIngo Molnar #endif
4068446f1d3SIngo Molnar 
4078b414521SMarcelo Tosatti #ifdef CONFIG_DETECT_HUNG_TASK
4088b414521SMarcelo Tosatti void reset_hung_task_detector(void);
4098b414521SMarcelo Tosatti #else
4108b414521SMarcelo Tosatti static inline void reset_hung_task_detector(void)
4118b414521SMarcelo Tosatti {
4128b414521SMarcelo Tosatti }
4138b414521SMarcelo Tosatti #endif
4148b414521SMarcelo Tosatti 
4151da177e4SLinus Torvalds /* Attach to any functions which should be ignored in wchan output. */
4161da177e4SLinus Torvalds #define __sched		__attribute__((__section__(".sched.text")))
417deaf2227SIngo Molnar 
418deaf2227SIngo Molnar /* Linker adds these: start and end of __sched functions */
419deaf2227SIngo Molnar extern char __sched_text_start[], __sched_text_end[];
420deaf2227SIngo Molnar 
4211da177e4SLinus Torvalds /* Is this address in the __sched functions? */
4221da177e4SLinus Torvalds extern int in_sched_functions(unsigned long addr);
4231da177e4SLinus Torvalds 
4241da177e4SLinus Torvalds #define	MAX_SCHEDULE_TIMEOUT	LONG_MAX
425b3c97528SHarvey Harrison extern signed long schedule_timeout(signed long timeout);
42664ed93a2SNishanth Aravamudan extern signed long schedule_timeout_interruptible(signed long timeout);
427294d5cc2SMatthew Wilcox extern signed long schedule_timeout_killable(signed long timeout);
42864ed93a2SNishanth Aravamudan extern signed long schedule_timeout_uninterruptible(signed long timeout);
42969b27bafSAndrew Morton extern signed long schedule_timeout_idle(signed long timeout);
4301da177e4SLinus Torvalds asmlinkage void schedule(void);
431c5491ea7SThomas Gleixner extern void schedule_preempt_disabled(void);
4321da177e4SLinus Torvalds 
4339cff8adeSNeilBrown extern long io_schedule_timeout(long timeout);
4349cff8adeSNeilBrown 
4359cff8adeSNeilBrown static inline void io_schedule(void)
4369cff8adeSNeilBrown {
4379cff8adeSNeilBrown 	io_schedule_timeout(MAX_SCHEDULE_TIMEOUT);
4389cff8adeSNeilBrown }
4399cff8adeSNeilBrown 
440ab516013SSerge E. Hallyn struct nsproxy;
441acce292cSCedric Le Goater struct user_namespace;
4421da177e4SLinus Torvalds 
443efc1a3b1SDavid Howells #ifdef CONFIG_MMU
444efc1a3b1SDavid Howells extern void arch_pick_mmap_layout(struct mm_struct *mm);
4451da177e4SLinus Torvalds extern unsigned long
4461da177e4SLinus Torvalds arch_get_unmapped_area(struct file *, unsigned long, unsigned long,
4471da177e4SLinus Torvalds 		       unsigned long, unsigned long);
4481da177e4SLinus Torvalds extern unsigned long
4491da177e4SLinus Torvalds arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr,
4501da177e4SLinus Torvalds 			  unsigned long len, unsigned long pgoff,
4511da177e4SLinus Torvalds 			  unsigned long flags);
452efc1a3b1SDavid Howells #else
453efc1a3b1SDavid Howells static inline void arch_pick_mmap_layout(struct mm_struct *mm) {}
454efc1a3b1SDavid Howells #endif
4551da177e4SLinus Torvalds 
456d049f74fSKees Cook #define SUID_DUMP_DISABLE	0	/* No setuid dumping */
457d049f74fSKees Cook #define SUID_DUMP_USER		1	/* Dump as user of process */
458d049f74fSKees Cook #define SUID_DUMP_ROOT		2	/* Dump as root */
459d049f74fSKees Cook 
4606c5d5238SKawai, Hidehiro /* mm flags */
461f8af4da3SHugh Dickins 
4627288e118SOleg Nesterov /* for SUID_DUMP_* above */
4633cb4a0bbSKawai, Hidehiro #define MMF_DUMPABLE_BITS 2
464f8af4da3SHugh Dickins #define MMF_DUMPABLE_MASK ((1 << MMF_DUMPABLE_BITS) - 1)
4653cb4a0bbSKawai, Hidehiro 
466942be387SOleg Nesterov extern void set_dumpable(struct mm_struct *mm, int value);
467942be387SOleg Nesterov /*
468942be387SOleg Nesterov  * This returns the actual value of the suid_dumpable flag. For things
469942be387SOleg Nesterov  * that are using this for checking for privilege transitions, it must
470942be387SOleg Nesterov  * test against SUID_DUMP_USER rather than treating it as a boolean
471942be387SOleg Nesterov  * value.
472942be387SOleg Nesterov  */
473942be387SOleg Nesterov static inline int __get_dumpable(unsigned long mm_flags)
474942be387SOleg Nesterov {
475942be387SOleg Nesterov 	return mm_flags & MMF_DUMPABLE_MASK;
476942be387SOleg Nesterov }
477942be387SOleg Nesterov 
478942be387SOleg Nesterov static inline int get_dumpable(struct mm_struct *mm)
479942be387SOleg Nesterov {
480942be387SOleg Nesterov 	return __get_dumpable(mm->flags);
481942be387SOleg Nesterov }
482942be387SOleg Nesterov 
4833cb4a0bbSKawai, Hidehiro /* coredump filter bits */
4843cb4a0bbSKawai, Hidehiro #define MMF_DUMP_ANON_PRIVATE	2
4853cb4a0bbSKawai, Hidehiro #define MMF_DUMP_ANON_SHARED	3
4863cb4a0bbSKawai, Hidehiro #define MMF_DUMP_MAPPED_PRIVATE	4
4873cb4a0bbSKawai, Hidehiro #define MMF_DUMP_MAPPED_SHARED	5
48882df3973SRoland McGrath #define MMF_DUMP_ELF_HEADERS	6
489e575f111SKOSAKI Motohiro #define MMF_DUMP_HUGETLB_PRIVATE 7
490e575f111SKOSAKI Motohiro #define MMF_DUMP_HUGETLB_SHARED  8
4915037835cSRoss Zwisler #define MMF_DUMP_DAX_PRIVATE	9
4925037835cSRoss Zwisler #define MMF_DUMP_DAX_SHARED	10
493f8af4da3SHugh Dickins 
4943cb4a0bbSKawai, Hidehiro #define MMF_DUMP_FILTER_SHIFT	MMF_DUMPABLE_BITS
4955037835cSRoss Zwisler #define MMF_DUMP_FILTER_BITS	9
4963cb4a0bbSKawai, Hidehiro #define MMF_DUMP_FILTER_MASK \
4973cb4a0bbSKawai, Hidehiro 	(((1 << MMF_DUMP_FILTER_BITS) - 1) << MMF_DUMP_FILTER_SHIFT)
4983cb4a0bbSKawai, Hidehiro #define MMF_DUMP_FILTER_DEFAULT \
499e575f111SKOSAKI Motohiro 	((1 << MMF_DUMP_ANON_PRIVATE) |	(1 << MMF_DUMP_ANON_SHARED) |\
500656eb2cdSRoland McGrath 	 (1 << MMF_DUMP_HUGETLB_PRIVATE) | MMF_DUMP_MASK_DEFAULT_ELF)
501656eb2cdSRoland McGrath 
502656eb2cdSRoland McGrath #ifdef CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS
503656eb2cdSRoland McGrath # define MMF_DUMP_MASK_DEFAULT_ELF	(1 << MMF_DUMP_ELF_HEADERS)
504656eb2cdSRoland McGrath #else
505656eb2cdSRoland McGrath # define MMF_DUMP_MASK_DEFAULT_ELF	0
506656eb2cdSRoland McGrath #endif
507f8af4da3SHugh Dickins 					/* leave room for more dump flags */
508f8af4da3SHugh Dickins #define MMF_VM_MERGEABLE	16	/* KSM may merge identical pages */
509ba76149fSAndrea Arcangeli #define MMF_VM_HUGEPAGE		17	/* set when VM_HUGEPAGE is set on vma */
510bafb282dSKonstantin Khlebnikov #define MMF_EXE_FILE_CHANGED	18	/* see prctl_set_mm_exe_file() */
511f8af4da3SHugh Dickins 
5129f68f672SOleg Nesterov #define MMF_HAS_UPROBES		19	/* has uprobes */
5139f68f672SOleg Nesterov #define MMF_RECALC_UPROBES	20	/* MMF_HAS_UPROBES can be wrong */
514f8ac4ec9SOleg Nesterov 
515f8af4da3SHugh Dickins #define MMF_INIT_MASK		(MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK)
5166c5d5238SKawai, Hidehiro 
5171da177e4SLinus Torvalds struct sighand_struct {
5181da177e4SLinus Torvalds 	atomic_t		count;
5191da177e4SLinus Torvalds 	struct k_sigaction	action[_NSIG];
5201da177e4SLinus Torvalds 	spinlock_t		siglock;
521b8fceee1SDavide Libenzi 	wait_queue_head_t	signalfd_wqh;
5221da177e4SLinus Torvalds };
5231da177e4SLinus Torvalds 
5240e464814SKaiGai Kohei struct pacct_struct {
525f6ec29a4SKaiGai Kohei 	int			ac_flag;
526f6ec29a4SKaiGai Kohei 	long			ac_exitcode;
5270e464814SKaiGai Kohei 	unsigned long		ac_mem;
52877787bfbSKaiGai Kohei 	cputime_t		ac_utime, ac_stime;
52977787bfbSKaiGai Kohei 	unsigned long		ac_minflt, ac_majflt;
5300e464814SKaiGai Kohei };
5310e464814SKaiGai Kohei 
53242c4ab41SStanislaw Gruszka struct cpu_itimer {
53342c4ab41SStanislaw Gruszka 	cputime_t expires;
53442c4ab41SStanislaw Gruszka 	cputime_t incr;
5358356b5f9SStanislaw Gruszka 	u32 error;
5368356b5f9SStanislaw Gruszka 	u32 incr_error;
53742c4ab41SStanislaw Gruszka };
53842c4ab41SStanislaw Gruszka 
539f06febc9SFrank Mayhar /**
5409d7fb042SPeter Zijlstra  * struct prev_cputime - snaphsot of system and user cputime
541d37f761dSFrederic Weisbecker  * @utime: time spent in user mode
542d37f761dSFrederic Weisbecker  * @stime: time spent in system mode
5439d7fb042SPeter Zijlstra  * @lock: protects the above two fields
544d37f761dSFrederic Weisbecker  *
5459d7fb042SPeter Zijlstra  * Stores previous user/system time values such that we can guarantee
5469d7fb042SPeter Zijlstra  * monotonicity.
547d37f761dSFrederic Weisbecker  */
5489d7fb042SPeter Zijlstra struct prev_cputime {
5499d7fb042SPeter Zijlstra #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
550d37f761dSFrederic Weisbecker 	cputime_t utime;
551d37f761dSFrederic Weisbecker 	cputime_t stime;
5529d7fb042SPeter Zijlstra 	raw_spinlock_t lock;
5539d7fb042SPeter Zijlstra #endif
554d37f761dSFrederic Weisbecker };
555d37f761dSFrederic Weisbecker 
5569d7fb042SPeter Zijlstra static inline void prev_cputime_init(struct prev_cputime *prev)
5579d7fb042SPeter Zijlstra {
5589d7fb042SPeter Zijlstra #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
5599d7fb042SPeter Zijlstra 	prev->utime = prev->stime = 0;
5609d7fb042SPeter Zijlstra 	raw_spin_lock_init(&prev->lock);
5619d7fb042SPeter Zijlstra #endif
5629d7fb042SPeter Zijlstra }
5639d7fb042SPeter Zijlstra 
564d37f761dSFrederic Weisbecker /**
565f06febc9SFrank Mayhar  * struct task_cputime - collected CPU time counts
566f06febc9SFrank Mayhar  * @utime:		time spent in user mode, in &cputime_t units
567f06febc9SFrank Mayhar  * @stime:		time spent in kernel mode, in &cputime_t units
568f06febc9SFrank Mayhar  * @sum_exec_runtime:	total time spent on the CPU, in nanoseconds
569f06febc9SFrank Mayhar  *
5709d7fb042SPeter Zijlstra  * This structure groups together three kinds of CPU time that are tracked for
5719d7fb042SPeter Zijlstra  * threads and thread groups.  Most things considering CPU time want to group
5729d7fb042SPeter Zijlstra  * these counts together and treat all three of them in parallel.
573f06febc9SFrank Mayhar  */
574f06febc9SFrank Mayhar struct task_cputime {
575f06febc9SFrank Mayhar 	cputime_t utime;
576f06febc9SFrank Mayhar 	cputime_t stime;
577f06febc9SFrank Mayhar 	unsigned long long sum_exec_runtime;
578f06febc9SFrank Mayhar };
5799d7fb042SPeter Zijlstra 
580f06febc9SFrank Mayhar /* Alternate field names when used to cache expirations. */
581f06febc9SFrank Mayhar #define virt_exp	utime
5829d7fb042SPeter Zijlstra #define prof_exp	stime
583f06febc9SFrank Mayhar #define sched_exp	sum_exec_runtime
584f06febc9SFrank Mayhar 
5854cd4c1b4SPeter Zijlstra #define INIT_CPUTIME	\
5864cd4c1b4SPeter Zijlstra 	(struct task_cputime) {					\
58764861634SMartin Schwidefsky 		.utime = 0,					\
58864861634SMartin Schwidefsky 		.stime = 0,					\
5894cd4c1b4SPeter Zijlstra 		.sum_exec_runtime = 0,				\
5904cd4c1b4SPeter Zijlstra 	}
5914cd4c1b4SPeter Zijlstra 
592971e8a98SJason Low /*
593971e8a98SJason Low  * This is the atomic variant of task_cputime, which can be used for
594971e8a98SJason Low  * storing and updating task_cputime statistics without locking.
595971e8a98SJason Low  */
596971e8a98SJason Low struct task_cputime_atomic {
597971e8a98SJason Low 	atomic64_t utime;
598971e8a98SJason Low 	atomic64_t stime;
599971e8a98SJason Low 	atomic64_t sum_exec_runtime;
600971e8a98SJason Low };
601971e8a98SJason Low 
602971e8a98SJason Low #define INIT_CPUTIME_ATOMIC \
603971e8a98SJason Low 	(struct task_cputime_atomic) {				\
604971e8a98SJason Low 		.utime = ATOMIC64_INIT(0),			\
605971e8a98SJason Low 		.stime = ATOMIC64_INIT(0),			\
606971e8a98SJason Low 		.sum_exec_runtime = ATOMIC64_INIT(0),		\
607971e8a98SJason Low 	}
608971e8a98SJason Low 
609609ca066SPeter Zijlstra #define PREEMPT_DISABLED	(PREEMPT_DISABLE_OFFSET + PREEMPT_ENABLED)
610a233f112SPeter Zijlstra 
611c99e6efeSPeter Zijlstra /*
61287dcbc06SPeter Zijlstra  * Disable preemption until the scheduler is running -- use an unconditional
61387dcbc06SPeter Zijlstra  * value so that it also works on !PREEMPT_COUNT kernels.
614d86ee480SPeter Zijlstra  *
61587dcbc06SPeter Zijlstra  * Reset by start_kernel()->sched_init()->init_idle()->init_idle_preempt_count().
616c99e6efeSPeter Zijlstra  */
61787dcbc06SPeter Zijlstra #define INIT_PREEMPT_COUNT	PREEMPT_OFFSET
618c99e6efeSPeter Zijlstra 
619609ca066SPeter Zijlstra /*
620609ca066SPeter Zijlstra  * Initial preempt_count value; reflects the preempt_count schedule invariant
621609ca066SPeter Zijlstra  * which states that during context switches:
622609ca066SPeter Zijlstra  *
623609ca066SPeter Zijlstra  *    preempt_count() == 2*PREEMPT_DISABLE_OFFSET
624609ca066SPeter Zijlstra  *
625609ca066SPeter Zijlstra  * Note: PREEMPT_DISABLE_OFFSET is 0 for !PREEMPT_COUNT kernels.
626609ca066SPeter Zijlstra  * Note: See finish_task_switch().
627609ca066SPeter Zijlstra  */
628609ca066SPeter Zijlstra #define FORK_PREEMPT_COUNT	(2*PREEMPT_DISABLE_OFFSET + PREEMPT_ENABLED)
6294cd4c1b4SPeter Zijlstra 
630f06febc9SFrank Mayhar /**
631f06febc9SFrank Mayhar  * struct thread_group_cputimer - thread group interval timer counts
632920ce39fSJason Low  * @cputime_atomic:	atomic thread group interval timers.
633d5c373ebSJason Low  * @running:		true when there are timers running and
634d5c373ebSJason Low  *			@cputime_atomic receives updates.
635c8d75aa4SJason Low  * @checking_timer:	true when a thread in the group is in the
636c8d75aa4SJason Low  *			process of checking for thread group timers.
637f06febc9SFrank Mayhar  *
638f06febc9SFrank Mayhar  * This structure contains the version of task_cputime, above, that is
6394cd4c1b4SPeter Zijlstra  * used for thread group CPU timer calculations.
640f06febc9SFrank Mayhar  */
6414cd4c1b4SPeter Zijlstra struct thread_group_cputimer {
64271107445SJason Low 	struct task_cputime_atomic cputime_atomic;
643d5c373ebSJason Low 	bool running;
644c8d75aa4SJason Low 	bool checking_timer;
645f06febc9SFrank Mayhar };
646f06febc9SFrank Mayhar 
6474714d1d3SBen Blum #include <linux/rwsem.h>
6485091faa4SMike Galbraith struct autogroup;
6495091faa4SMike Galbraith 
6501da177e4SLinus Torvalds /*
651e815f0a8SJonathan Neuschäfer  * NOTE! "signal_struct" does not have its own
6521da177e4SLinus Torvalds  * locking, because a shared signal_struct always
6531da177e4SLinus Torvalds  * implies a shared sighand_struct, so locking
6541da177e4SLinus Torvalds  * sighand_struct is always a proper superset of
6551da177e4SLinus Torvalds  * the locking of signal_struct.
6561da177e4SLinus Torvalds  */
6571da177e4SLinus Torvalds struct signal_struct {
658ea6d290cSOleg Nesterov 	atomic_t		sigcnt;
6591da177e4SLinus Torvalds 	atomic_t		live;
660b3ac022cSOleg Nesterov 	int			nr_threads;
6610c740d0aSOleg Nesterov 	struct list_head	thread_head;
6621da177e4SLinus Torvalds 
6631da177e4SLinus Torvalds 	wait_queue_head_t	wait_chldexit;	/* for wait4() */
6641da177e4SLinus Torvalds 
6651da177e4SLinus Torvalds 	/* current thread group signal load-balancing target: */
66636c8b586SIngo Molnar 	struct task_struct	*curr_target;
6671da177e4SLinus Torvalds 
6681da177e4SLinus Torvalds 	/* shared signal handling: */
6691da177e4SLinus Torvalds 	struct sigpending	shared_pending;
6701da177e4SLinus Torvalds 
6711da177e4SLinus Torvalds 	/* thread group exit support */
6721da177e4SLinus Torvalds 	int			group_exit_code;
6731da177e4SLinus Torvalds 	/* overloaded:
6741da177e4SLinus Torvalds 	 * - notify group_exit_task when ->count is equal to notify_count
6751da177e4SLinus Torvalds 	 * - everyone except group_exit_task is stopped during signal delivery
6761da177e4SLinus Torvalds 	 *   of fatal signals, group_exit_task processes the signal.
6771da177e4SLinus Torvalds 	 */
6781da177e4SLinus Torvalds 	int			notify_count;
67907dd20e0SRichard Kennedy 	struct task_struct	*group_exit_task;
6801da177e4SLinus Torvalds 
6811da177e4SLinus Torvalds 	/* thread group stop support, overloads group_exit_code too */
6821da177e4SLinus Torvalds 	int			group_stop_count;
6831da177e4SLinus Torvalds 	unsigned int		flags; /* see SIGNAL_* flags below */
6841da177e4SLinus Torvalds 
685ebec18a6SLennart Poettering 	/*
686ebec18a6SLennart Poettering 	 * PR_SET_CHILD_SUBREAPER marks a process, like a service
687ebec18a6SLennart Poettering 	 * manager, to re-parent orphan (double-forking) child processes
688ebec18a6SLennart Poettering 	 * to this process instead of 'init'. The service manager is
689ebec18a6SLennart Poettering 	 * able to receive SIGCHLD signals and is able to investigate
690ebec18a6SLennart Poettering 	 * the process until it calls wait(). All children of this
691ebec18a6SLennart Poettering 	 * process will inherit a flag if they should look for a
692ebec18a6SLennart Poettering 	 * child_subreaper process at exit.
693ebec18a6SLennart Poettering 	 */
694ebec18a6SLennart Poettering 	unsigned int		is_child_subreaper:1;
695ebec18a6SLennart Poettering 	unsigned int		has_child_subreaper:1;
696ebec18a6SLennart Poettering 
6971da177e4SLinus Torvalds 	/* POSIX.1b Interval Timers */
6985ed67f05SPavel Emelyanov 	int			posix_timer_id;
6991da177e4SLinus Torvalds 	struct list_head	posix_timers;
7001da177e4SLinus Torvalds 
7011da177e4SLinus Torvalds 	/* ITIMER_REAL timer for the process */
7022ff678b8SThomas Gleixner 	struct hrtimer real_timer;
703fea9d175SOleg Nesterov 	struct pid *leader_pid;
7042ff678b8SThomas Gleixner 	ktime_t it_real_incr;
7051da177e4SLinus Torvalds 
70642c4ab41SStanislaw Gruszka 	/*
70742c4ab41SStanislaw Gruszka 	 * ITIMER_PROF and ITIMER_VIRTUAL timers for the process, we use
70842c4ab41SStanislaw Gruszka 	 * CPUCLOCK_PROF and CPUCLOCK_VIRT for indexing array as these
70942c4ab41SStanislaw Gruszka 	 * values are defined to 0 and 1 respectively
71042c4ab41SStanislaw Gruszka 	 */
71142c4ab41SStanislaw Gruszka 	struct cpu_itimer it[2];
7121da177e4SLinus Torvalds 
713f06febc9SFrank Mayhar 	/*
7144cd4c1b4SPeter Zijlstra 	 * Thread group totals for process CPU timers.
7154cd4c1b4SPeter Zijlstra 	 * See thread_group_cputimer(), et al, for details.
716f06febc9SFrank Mayhar 	 */
7174cd4c1b4SPeter Zijlstra 	struct thread_group_cputimer cputimer;
718f06febc9SFrank Mayhar 
719f06febc9SFrank Mayhar 	/* Earliest-expiration cache. */
720f06febc9SFrank Mayhar 	struct task_cputime cputime_expires;
721f06febc9SFrank Mayhar 
722d027d45dSFrederic Weisbecker #ifdef CONFIG_NO_HZ_FULL
723*f009a7a7SFrederic Weisbecker 	atomic_t tick_dep_mask;
724d027d45dSFrederic Weisbecker #endif
725d027d45dSFrederic Weisbecker 
726f06febc9SFrank Mayhar 	struct list_head cpu_timers[3];
727f06febc9SFrank Mayhar 
728ab521dc0SEric W. Biederman 	struct pid *tty_old_pgrp;
7291ec320afSCedric Le Goater 
7301da177e4SLinus Torvalds 	/* boolean value for session group leader */
7311da177e4SLinus Torvalds 	int leader;
7321da177e4SLinus Torvalds 
7331da177e4SLinus Torvalds 	struct tty_struct *tty; /* NULL if no tty */
7341da177e4SLinus Torvalds 
7355091faa4SMike Galbraith #ifdef CONFIG_SCHED_AUTOGROUP
7365091faa4SMike Galbraith 	struct autogroup *autogroup;
7375091faa4SMike Galbraith #endif
7381da177e4SLinus Torvalds 	/*
7391da177e4SLinus Torvalds 	 * Cumulative resource counters for dead threads in the group,
7401da177e4SLinus Torvalds 	 * and for reaped dead child processes forked by this group.
7411da177e4SLinus Torvalds 	 * Live threads maintain their own counters and add to these
7421da177e4SLinus Torvalds 	 * in __exit_signal, except for the group leader.
7431da177e4SLinus Torvalds 	 */
744e78c3496SRik van Riel 	seqlock_t stats_lock;
74532bd671dSPeter Zijlstra 	cputime_t utime, stime, cutime, cstime;
7469ac52315SLaurent Vivier 	cputime_t gtime;
7479ac52315SLaurent Vivier 	cputime_t cgtime;
7489d7fb042SPeter Zijlstra 	struct prev_cputime prev_cputime;
7491da177e4SLinus Torvalds 	unsigned long nvcsw, nivcsw, cnvcsw, cnivcsw;
7501da177e4SLinus Torvalds 	unsigned long min_flt, maj_flt, cmin_flt, cmaj_flt;
7516eaeeabaSEric Dumazet 	unsigned long inblock, oublock, cinblock, coublock;
7521f10206cSJiri Pirko 	unsigned long maxrss, cmaxrss;
753940389b8SAndrea Righi 	struct task_io_accounting ioac;
7541da177e4SLinus Torvalds 
7551da177e4SLinus Torvalds 	/*
75632bd671dSPeter Zijlstra 	 * Cumulative ns of schedule CPU time fo dead threads in the
75732bd671dSPeter Zijlstra 	 * group, not including a zombie group leader, (This only differs
75832bd671dSPeter Zijlstra 	 * from jiffies_to_ns(utime + stime) if sched_clock uses something
75932bd671dSPeter Zijlstra 	 * other than jiffies.)
76032bd671dSPeter Zijlstra 	 */
76132bd671dSPeter Zijlstra 	unsigned long long sum_sched_runtime;
76232bd671dSPeter Zijlstra 
76332bd671dSPeter Zijlstra 	/*
7641da177e4SLinus Torvalds 	 * We don't bother to synchronize most readers of this at all,
7651da177e4SLinus Torvalds 	 * because there is no reader checking a limit that actually needs
7661da177e4SLinus Torvalds 	 * to get both rlim_cur and rlim_max atomically, and either one
7671da177e4SLinus Torvalds 	 * alone is a single word that can safely be read normally.
7681da177e4SLinus Torvalds 	 * getrlimit/setrlimit use task_lock(current->group_leader) to
7691da177e4SLinus Torvalds 	 * protect this instead of the siglock, because they really
7701da177e4SLinus Torvalds 	 * have no need to disable irqs.
7711da177e4SLinus Torvalds 	 */
7721da177e4SLinus Torvalds 	struct rlimit rlim[RLIM_NLIMITS];
7731da177e4SLinus Torvalds 
7740e464814SKaiGai Kohei #ifdef CONFIG_BSD_PROCESS_ACCT
7750e464814SKaiGai Kohei 	struct pacct_struct pacct;	/* per-process accounting information */
7760e464814SKaiGai Kohei #endif
777ad4ecbcbSShailabh Nagar #ifdef CONFIG_TASKSTATS
778ad4ecbcbSShailabh Nagar 	struct taskstats *stats;
779ad4ecbcbSShailabh Nagar #endif
780522ed776SMiloslav Trmac #ifdef CONFIG_AUDIT
781522ed776SMiloslav Trmac 	unsigned audit_tty;
782522ed776SMiloslav Trmac 	struct tty_audit_buf *tty_audit_buf;
783522ed776SMiloslav Trmac #endif
78428b83c51SKOSAKI Motohiro 
785e1e12d2fSDavid Rientjes 	oom_flags_t oom_flags;
786a9c58b90SDavid Rientjes 	short oom_score_adj;		/* OOM kill score adjustment */
787a9c58b90SDavid Rientjes 	short oom_score_adj_min;	/* OOM kill score adjustment min value.
788dabb16f6SMandeep Singh Baines 					 * Only settable by CAP_SYS_RESOURCE. */
7899b1bf12dSKOSAKI Motohiro 
7909b1bf12dSKOSAKI Motohiro 	struct mutex cred_guard_mutex;	/* guard against foreign influences on
7919b1bf12dSKOSAKI Motohiro 					 * credential calculations
7929b1bf12dSKOSAKI Motohiro 					 * (notably. ptrace) */
7931da177e4SLinus Torvalds };
7941da177e4SLinus Torvalds 
7951da177e4SLinus Torvalds /*
7961da177e4SLinus Torvalds  * Bits in flags field of signal_struct.
7971da177e4SLinus Torvalds  */
7981da177e4SLinus Torvalds #define SIGNAL_STOP_STOPPED	0x00000001 /* job control stop in effect */
799ee77f075SOleg Nesterov #define SIGNAL_STOP_CONTINUED	0x00000002 /* SIGCONT since WCONTINUED reap */
800ee77f075SOleg Nesterov #define SIGNAL_GROUP_EXIT	0x00000004 /* group exit in progress */
801403bad72SOleg Nesterov #define SIGNAL_GROUP_COREDUMP	0x00000008 /* coredump in progress */
802e4420551SOleg Nesterov /*
803e4420551SOleg Nesterov  * Pending notifications to parent.
804e4420551SOleg Nesterov  */
805e4420551SOleg Nesterov #define SIGNAL_CLD_STOPPED	0x00000010
806e4420551SOleg Nesterov #define SIGNAL_CLD_CONTINUED	0x00000020
807e4420551SOleg Nesterov #define SIGNAL_CLD_MASK		(SIGNAL_CLD_STOPPED|SIGNAL_CLD_CONTINUED)
8081da177e4SLinus Torvalds 
809fae5fa44SOleg Nesterov #define SIGNAL_UNKILLABLE	0x00000040 /* for init: ignore fatal signals */
810fae5fa44SOleg Nesterov 
811ed5d2cacSOleg Nesterov /* If true, all threads except ->group_exit_task have pending SIGKILL */
812ed5d2cacSOleg Nesterov static inline int signal_group_exit(const struct signal_struct *sig)
813ed5d2cacSOleg Nesterov {
814ed5d2cacSOleg Nesterov 	return	(sig->flags & SIGNAL_GROUP_EXIT) ||
815ed5d2cacSOleg Nesterov 		(sig->group_exit_task != NULL);
816ed5d2cacSOleg Nesterov }
817ed5d2cacSOleg Nesterov 
8181da177e4SLinus Torvalds /*
8191da177e4SLinus Torvalds  * Some day this will be a full-fledged user tracking system..
8201da177e4SLinus Torvalds  */
8211da177e4SLinus Torvalds struct user_struct {
8221da177e4SLinus Torvalds 	atomic_t __count;	/* reference count */
8231da177e4SLinus Torvalds 	atomic_t processes;	/* How many processes does this user have? */
8241da177e4SLinus Torvalds 	atomic_t sigpending;	/* How many pending signals does this user have? */
8252d9048e2SAmy Griffis #ifdef CONFIG_INOTIFY_USER
8260eeca283SRobert Love 	atomic_t inotify_watches; /* How many inotify watches does this user have? */
8270eeca283SRobert Love 	atomic_t inotify_devs;	/* How many inotify devs does this user have opened? */
8280eeca283SRobert Love #endif
8294afeff85SEric Paris #ifdef CONFIG_FANOTIFY
8304afeff85SEric Paris 	atomic_t fanotify_listeners;
8314afeff85SEric Paris #endif
8327ef9964eSDavide Libenzi #ifdef CONFIG_EPOLL
83352bd19f7SRobin Holt 	atomic_long_t epoll_watches; /* The number of file descriptors currently watched */
8347ef9964eSDavide Libenzi #endif
835970a8645SAlexey Dobriyan #ifdef CONFIG_POSIX_MQUEUE
8361da177e4SLinus Torvalds 	/* protected by mq_lock	*/
8371da177e4SLinus Torvalds 	unsigned long mq_bytes;	/* How many bytes can be allocated to mqueue? */
838970a8645SAlexey Dobriyan #endif
8391da177e4SLinus Torvalds 	unsigned long locked_shm; /* How many pages of mlocked shm ? */
840712f4aadSwilly tarreau 	unsigned long unix_inflight;	/* How many files in flight in unix sockets */
841759c0114SWilly Tarreau 	atomic_long_t pipe_bufs;  /* how many pages are allocated in pipe buffers */
8421da177e4SLinus Torvalds 
8431da177e4SLinus Torvalds #ifdef CONFIG_KEYS
8441da177e4SLinus Torvalds 	struct key *uid_keyring;	/* UID specific keyring */
8451da177e4SLinus Torvalds 	struct key *session_keyring;	/* UID's default session keyring */
8461da177e4SLinus Torvalds #endif
8471da177e4SLinus Torvalds 
8481da177e4SLinus Torvalds 	/* Hash table maintenance information */
849735de223SPavel Emelyanov 	struct hlist_node uidhash_node;
8507b44ab97SEric W. Biederman 	kuid_t uid;
85124e377a8SSrivatsa Vaddagiri 
852aaac3ba9SAlexei Starovoitov #if defined(CONFIG_PERF_EVENTS) || defined(CONFIG_BPF_SYSCALL)
853789f90fcSPeter Zijlstra 	atomic_long_t locked_vm;
854789f90fcSPeter Zijlstra #endif
8551da177e4SLinus Torvalds };
8561da177e4SLinus Torvalds 
857eb41d946SKay Sievers extern int uids_sysfs_init(void);
8585cb350baSDhaval Giani 
8597b44ab97SEric W. Biederman extern struct user_struct *find_user(kuid_t);
8601da177e4SLinus Torvalds 
8611da177e4SLinus Torvalds extern struct user_struct root_user;
8621da177e4SLinus Torvalds #define INIT_USER (&root_user)
8631da177e4SLinus Torvalds 
864b6dff3ecSDavid Howells 
8651da177e4SLinus Torvalds struct backing_dev_info;
8661da177e4SLinus Torvalds struct reclaim_state;
8671da177e4SLinus Torvalds 
868f6db8347SNaveen N. Rao #ifdef CONFIG_SCHED_INFO
8691da177e4SLinus Torvalds struct sched_info {
8701da177e4SLinus Torvalds 	/* cumulative counters */
8712d72376bSIngo Molnar 	unsigned long pcount;	      /* # of times run on this cpu */
8729c2c4802SKen Chen 	unsigned long long run_delay; /* time spent waiting on a runqueue */
8731da177e4SLinus Torvalds 
8741da177e4SLinus Torvalds 	/* timestamps */
875172ba844SBalbir Singh 	unsigned long long last_arrival,/* when we last ran on a cpu */
8761da177e4SLinus Torvalds 			   last_queued;	/* when we were last queued to run */
8771da177e4SLinus Torvalds };
878f6db8347SNaveen N. Rao #endif /* CONFIG_SCHED_INFO */
8791da177e4SLinus Torvalds 
880ca74e92bSShailabh Nagar #ifdef CONFIG_TASK_DELAY_ACCT
881ca74e92bSShailabh Nagar struct task_delay_info {
882ca74e92bSShailabh Nagar 	spinlock_t	lock;
883ca74e92bSShailabh Nagar 	unsigned int	flags;	/* Private per-task flags */
884ca74e92bSShailabh Nagar 
885ca74e92bSShailabh Nagar 	/* For each stat XXX, add following, aligned appropriately
886ca74e92bSShailabh Nagar 	 *
887ca74e92bSShailabh Nagar 	 * struct timespec XXX_start, XXX_end;
888ca74e92bSShailabh Nagar 	 * u64 XXX_delay;
889ca74e92bSShailabh Nagar 	 * u32 XXX_count;
890ca74e92bSShailabh Nagar 	 *
891ca74e92bSShailabh Nagar 	 * Atomicity of updates to XXX_delay, XXX_count protected by
892ca74e92bSShailabh Nagar 	 * single lock above (split into XXX_lock if contention is an issue).
893ca74e92bSShailabh Nagar 	 */
8940ff92245SShailabh Nagar 
8950ff92245SShailabh Nagar 	/*
8960ff92245SShailabh Nagar 	 * XXX_count is incremented on every XXX operation, the delay
8970ff92245SShailabh Nagar 	 * associated with the operation is added to XXX_delay.
8980ff92245SShailabh Nagar 	 * XXX_delay contains the accumulated delay time in nanoseconds.
8990ff92245SShailabh Nagar 	 */
9009667a23dSThomas Gleixner 	u64 blkio_start;	/* Shared by blkio, swapin */
9010ff92245SShailabh Nagar 	u64 blkio_delay;	/* wait for sync block io completion */
9020ff92245SShailabh Nagar 	u64 swapin_delay;	/* wait for swapin block io completion */
9030ff92245SShailabh Nagar 	u32 blkio_count;	/* total count of the number of sync block */
9040ff92245SShailabh Nagar 				/* io operations performed */
9050ff92245SShailabh Nagar 	u32 swapin_count;	/* total count of the number of swapin block */
9060ff92245SShailabh Nagar 				/* io operations performed */
907873b4771SKeika Kobayashi 
9089667a23dSThomas Gleixner 	u64 freepages_start;
909873b4771SKeika Kobayashi 	u64 freepages_delay;	/* wait for memory reclaim */
910873b4771SKeika Kobayashi 	u32 freepages_count;	/* total count of memory reclaim */
911ca74e92bSShailabh Nagar };
91252f17b6cSChandra Seetharaman #endif	/* CONFIG_TASK_DELAY_ACCT */
91352f17b6cSChandra Seetharaman 
91452f17b6cSChandra Seetharaman static inline int sched_info_on(void)
91552f17b6cSChandra Seetharaman {
91652f17b6cSChandra Seetharaman #ifdef CONFIG_SCHEDSTATS
91752f17b6cSChandra Seetharaman 	return 1;
91852f17b6cSChandra Seetharaman #elif defined(CONFIG_TASK_DELAY_ACCT)
91952f17b6cSChandra Seetharaman 	extern int delayacct_on;
92052f17b6cSChandra Seetharaman 	return delayacct_on;
92152f17b6cSChandra Seetharaman #else
92252f17b6cSChandra Seetharaman 	return 0;
923ca74e92bSShailabh Nagar #endif
92452f17b6cSChandra Seetharaman }
925ca74e92bSShailabh Nagar 
926cb251765SMel Gorman #ifdef CONFIG_SCHEDSTATS
927cb251765SMel Gorman void force_schedstat_enabled(void);
928cb251765SMel Gorman #endif
929cb251765SMel Gorman 
930d15bcfdbSIngo Molnar enum cpu_idle_type {
931d15bcfdbSIngo Molnar 	CPU_IDLE,
932d15bcfdbSIngo Molnar 	CPU_NOT_IDLE,
933d15bcfdbSIngo Molnar 	CPU_NEWLY_IDLE,
934d15bcfdbSIngo Molnar 	CPU_MAX_IDLE_TYPES
9351da177e4SLinus Torvalds };
9361da177e4SLinus Torvalds 
9371da177e4SLinus Torvalds /*
938ca8ce3d0SNicolas Pitre  * Increase resolution of cpu_capacity calculations
9391399fa78SNikhil Rao  */
940ca8ce3d0SNicolas Pitre #define SCHED_CAPACITY_SHIFT	10
941ca8ce3d0SNicolas Pitre #define SCHED_CAPACITY_SCALE	(1L << SCHED_CAPACITY_SHIFT)
9421da177e4SLinus Torvalds 
9431399fa78SNikhil Rao /*
94476751049SPeter Zijlstra  * Wake-queues are lists of tasks with a pending wakeup, whose
94576751049SPeter Zijlstra  * callers have already marked the task as woken internally,
94676751049SPeter Zijlstra  * and can thus carry on. A common use case is being able to
94776751049SPeter Zijlstra  * do the wakeups once the corresponding user lock as been
94876751049SPeter Zijlstra  * released.
94976751049SPeter Zijlstra  *
95076751049SPeter Zijlstra  * We hold reference to each task in the list across the wakeup,
95176751049SPeter Zijlstra  * thus guaranteeing that the memory is still valid by the time
95276751049SPeter Zijlstra  * the actual wakeups are performed in wake_up_q().
95376751049SPeter Zijlstra  *
95476751049SPeter Zijlstra  * One per task suffices, because there's never a need for a task to be
95576751049SPeter Zijlstra  * in two wake queues simultaneously; it is forbidden to abandon a task
95676751049SPeter Zijlstra  * in a wake queue (a call to wake_up_q() _must_ follow), so if a task is
95776751049SPeter Zijlstra  * already in a wake queue, the wakeup will happen soon and the second
95876751049SPeter Zijlstra  * waker can just skip it.
95976751049SPeter Zijlstra  *
96076751049SPeter Zijlstra  * The WAKE_Q macro declares and initializes the list head.
96176751049SPeter Zijlstra  * wake_up_q() does NOT reinitialize the list; it's expected to be
96276751049SPeter Zijlstra  * called near the end of a function, where the fact that the queue is
96376751049SPeter Zijlstra  * not used again will be easy to see by inspection.
96476751049SPeter Zijlstra  *
96576751049SPeter Zijlstra  * Note that this can cause spurious wakeups. schedule() callers
96676751049SPeter Zijlstra  * must ensure the call is done inside a loop, confirming that the
96776751049SPeter Zijlstra  * wakeup condition has in fact occurred.
96876751049SPeter Zijlstra  */
96976751049SPeter Zijlstra struct wake_q_node {
97076751049SPeter Zijlstra 	struct wake_q_node *next;
97176751049SPeter Zijlstra };
97276751049SPeter Zijlstra 
97376751049SPeter Zijlstra struct wake_q_head {
97476751049SPeter Zijlstra 	struct wake_q_node *first;
97576751049SPeter Zijlstra 	struct wake_q_node **lastp;
97676751049SPeter Zijlstra };
97776751049SPeter Zijlstra 
97876751049SPeter Zijlstra #define WAKE_Q_TAIL ((struct wake_q_node *) 0x01)
97976751049SPeter Zijlstra 
98076751049SPeter Zijlstra #define WAKE_Q(name)					\
98176751049SPeter Zijlstra 	struct wake_q_head name = { WAKE_Q_TAIL, &name.first }
98276751049SPeter Zijlstra 
98376751049SPeter Zijlstra extern void wake_q_add(struct wake_q_head *head,
98476751049SPeter Zijlstra 		       struct task_struct *task);
98576751049SPeter Zijlstra extern void wake_up_q(struct wake_q_head *head);
98676751049SPeter Zijlstra 
98776751049SPeter Zijlstra /*
9881399fa78SNikhil Rao  * sched-domains (multiprocessor balancing) declarations:
9891399fa78SNikhil Rao  */
9902dd73a4fSPeter Williams #ifdef CONFIG_SMP
991b5d978e0SPeter Zijlstra #define SD_LOAD_BALANCE		0x0001	/* Do load balancing on this domain. */
992b5d978e0SPeter Zijlstra #define SD_BALANCE_NEWIDLE	0x0002	/* Balance when about to become idle */
993b5d978e0SPeter Zijlstra #define SD_BALANCE_EXEC		0x0004	/* Balance on exec */
994b5d978e0SPeter Zijlstra #define SD_BALANCE_FORK		0x0008	/* Balance on fork, clone */
995c88d5910SPeter Zijlstra #define SD_BALANCE_WAKE		0x0010  /* Balance on wakeup */
996b5d978e0SPeter Zijlstra #define SD_WAKE_AFFINE		0x0020	/* Wake task to waking CPU */
9975d4dfdddSNicolas Pitre #define SD_SHARE_CPUCAPACITY	0x0080	/* Domain members share cpu power */
998d77b3ed5SVincent Guittot #define SD_SHARE_POWERDOMAIN	0x0100	/* Domain members share power domain */
999b5d978e0SPeter Zijlstra #define SD_SHARE_PKG_RESOURCES	0x0200	/* Domain members share cpu pkg resources */
1000b5d978e0SPeter Zijlstra #define SD_SERIALIZE		0x0400	/* Only a single load balancing instance */
1001532cb4c4SMichael Neuling #define SD_ASYM_PACKING		0x0800  /* Place busy groups earlier in the domain */
1002b5d978e0SPeter Zijlstra #define SD_PREFER_SIBLING	0x1000	/* Prefer to place tasks in a sibling domain */
1003e3589f6cSPeter Zijlstra #define SD_OVERLAP		0x2000	/* sched_domains of this level overlap */
10043a7053b3SMel Gorman #define SD_NUMA			0x4000	/* cross-node balancing */
10055c45bf27SSiddha, Suresh B 
1006143e1e28SVincent Guittot #ifdef CONFIG_SCHED_SMT
1007b6220ad6SGuenter Roeck static inline int cpu_smt_flags(void)
1008143e1e28SVincent Guittot {
10095d4dfdddSNicolas Pitre 	return SD_SHARE_CPUCAPACITY | SD_SHARE_PKG_RESOURCES;
1010143e1e28SVincent Guittot }
1011143e1e28SVincent Guittot #endif
1012143e1e28SVincent Guittot 
1013143e1e28SVincent Guittot #ifdef CONFIG_SCHED_MC
1014b6220ad6SGuenter Roeck static inline int cpu_core_flags(void)
1015143e1e28SVincent Guittot {
1016143e1e28SVincent Guittot 	return SD_SHARE_PKG_RESOURCES;
1017143e1e28SVincent Guittot }
1018143e1e28SVincent Guittot #endif
1019143e1e28SVincent Guittot 
1020143e1e28SVincent Guittot #ifdef CONFIG_NUMA
1021b6220ad6SGuenter Roeck static inline int cpu_numa_flags(void)
1022143e1e28SVincent Guittot {
1023143e1e28SVincent Guittot 	return SD_NUMA;
1024143e1e28SVincent Guittot }
1025143e1e28SVincent Guittot #endif
1026532cb4c4SMichael Neuling 
10271d3504fcSHidetoshi Seto struct sched_domain_attr {
10281d3504fcSHidetoshi Seto 	int relax_domain_level;
10291d3504fcSHidetoshi Seto };
10301d3504fcSHidetoshi Seto 
10311d3504fcSHidetoshi Seto #define SD_ATTR_INIT	(struct sched_domain_attr) {	\
10321d3504fcSHidetoshi Seto 	.relax_domain_level = -1,			\
10331d3504fcSHidetoshi Seto }
10341d3504fcSHidetoshi Seto 
103560495e77SPeter Zijlstra extern int sched_domain_level_max;
103660495e77SPeter Zijlstra 
10375e6521eaSLi Zefan struct sched_group;
10385e6521eaSLi Zefan 
10391da177e4SLinus Torvalds struct sched_domain {
10401da177e4SLinus Torvalds 	/* These fields must be setup */
10411da177e4SLinus Torvalds 	struct sched_domain *parent;	/* top domain must be null terminated */
10421a848870SSiddha, Suresh B 	struct sched_domain *child;	/* bottom domain must be null terminated */
10431da177e4SLinus Torvalds 	struct sched_group *groups;	/* the balancing groups of the domain */
10441da177e4SLinus Torvalds 	unsigned long min_interval;	/* Minimum balance interval ms */
10451da177e4SLinus Torvalds 	unsigned long max_interval;	/* Maximum balance interval ms */
10461da177e4SLinus Torvalds 	unsigned int busy_factor;	/* less balancing by factor if busy */
10471da177e4SLinus Torvalds 	unsigned int imbalance_pct;	/* No balance until over watermark */
10481da177e4SLinus Torvalds 	unsigned int cache_nice_tries;	/* Leave cache hot tasks for # tries */
10497897986bSNick Piggin 	unsigned int busy_idx;
10507897986bSNick Piggin 	unsigned int idle_idx;
10517897986bSNick Piggin 	unsigned int newidle_idx;
10527897986bSNick Piggin 	unsigned int wake_idx;
1053147cbb4bSNick Piggin 	unsigned int forkexec_idx;
1054a52bfd73SPeter Zijlstra 	unsigned int smt_gain;
105525f55d9dSVincent Guittot 
105625f55d9dSVincent Guittot 	int nohz_idle;			/* NOHZ IDLE status */
10571da177e4SLinus Torvalds 	int flags;			/* See SD_* */
105860495e77SPeter Zijlstra 	int level;
10591da177e4SLinus Torvalds 
10601da177e4SLinus Torvalds 	/* Runtime fields. */
10611da177e4SLinus Torvalds 	unsigned long last_balance;	/* init to jiffies. units in jiffies */
10621da177e4SLinus Torvalds 	unsigned int balance_interval;	/* initialise to 1. units in ms. */
10631da177e4SLinus Torvalds 	unsigned int nr_balance_failed; /* initialise to 0 */
10641da177e4SLinus Torvalds 
1065f48627e6SJason Low 	/* idle_balance() stats */
10669bd721c5SJason Low 	u64 max_newidle_lb_cost;
1067f48627e6SJason Low 	unsigned long next_decay_max_lb_cost;
10682398f2c6SPeter Zijlstra 
10691da177e4SLinus Torvalds #ifdef CONFIG_SCHEDSTATS
10701da177e4SLinus Torvalds 	/* load_balance() stats */
1071480b9434SKen Chen 	unsigned int lb_count[CPU_MAX_IDLE_TYPES];
1072480b9434SKen Chen 	unsigned int lb_failed[CPU_MAX_IDLE_TYPES];
1073480b9434SKen Chen 	unsigned int lb_balanced[CPU_MAX_IDLE_TYPES];
1074480b9434SKen Chen 	unsigned int lb_imbalance[CPU_MAX_IDLE_TYPES];
1075480b9434SKen Chen 	unsigned int lb_gained[CPU_MAX_IDLE_TYPES];
1076480b9434SKen Chen 	unsigned int lb_hot_gained[CPU_MAX_IDLE_TYPES];
1077480b9434SKen Chen 	unsigned int lb_nobusyg[CPU_MAX_IDLE_TYPES];
1078480b9434SKen Chen 	unsigned int lb_nobusyq[CPU_MAX_IDLE_TYPES];
10791da177e4SLinus Torvalds 
10801da177e4SLinus Torvalds 	/* Active load balancing */
1081480b9434SKen Chen 	unsigned int alb_count;
1082480b9434SKen Chen 	unsigned int alb_failed;
1083480b9434SKen Chen 	unsigned int alb_pushed;
10841da177e4SLinus Torvalds 
108568767a0aSNick Piggin 	/* SD_BALANCE_EXEC stats */
1086480b9434SKen Chen 	unsigned int sbe_count;
1087480b9434SKen Chen 	unsigned int sbe_balanced;
1088480b9434SKen Chen 	unsigned int sbe_pushed;
10891da177e4SLinus Torvalds 
109068767a0aSNick Piggin 	/* SD_BALANCE_FORK stats */
1091480b9434SKen Chen 	unsigned int sbf_count;
1092480b9434SKen Chen 	unsigned int sbf_balanced;
1093480b9434SKen Chen 	unsigned int sbf_pushed;
109468767a0aSNick Piggin 
10951da177e4SLinus Torvalds 	/* try_to_wake_up() stats */
1096480b9434SKen Chen 	unsigned int ttwu_wake_remote;
1097480b9434SKen Chen 	unsigned int ttwu_move_affine;
1098480b9434SKen Chen 	unsigned int ttwu_move_balance;
10991da177e4SLinus Torvalds #endif
1100a5d8c348SIngo Molnar #ifdef CONFIG_SCHED_DEBUG
1101a5d8c348SIngo Molnar 	char *name;
1102a5d8c348SIngo Molnar #endif
1103dce840a0SPeter Zijlstra 	union {
1104dce840a0SPeter Zijlstra 		void *private;		/* used during construction */
1105dce840a0SPeter Zijlstra 		struct rcu_head rcu;	/* used during destruction */
1106dce840a0SPeter Zijlstra 	};
11076c99e9adSRusty Russell 
1108669c55e9SPeter Zijlstra 	unsigned int span_weight;
11094200efd9SIngo Molnar 	/*
11104200efd9SIngo Molnar 	 * Span of all CPUs in this domain.
11114200efd9SIngo Molnar 	 *
11124200efd9SIngo Molnar 	 * NOTE: this field is variable length. (Allocated dynamically
11134200efd9SIngo Molnar 	 * by attaching extra space to the end of the structure,
11144200efd9SIngo Molnar 	 * depending on how many CPUs the kernel has booted up with)
11154200efd9SIngo Molnar 	 */
11164200efd9SIngo Molnar 	unsigned long span[0];
11171da177e4SLinus Torvalds };
11181da177e4SLinus Torvalds 
1119758b2cdcSRusty Russell static inline struct cpumask *sched_domain_span(struct sched_domain *sd)
1120758b2cdcSRusty Russell {
11216c99e9adSRusty Russell 	return to_cpumask(sd->span);
1122758b2cdcSRusty Russell }
1123758b2cdcSRusty Russell 
1124acc3f5d7SRusty Russell extern void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
11251d3504fcSHidetoshi Seto 				    struct sched_domain_attr *dattr_new);
1126029190c5SPaul Jackson 
1127acc3f5d7SRusty Russell /* Allocate an array of sched domains, for partition_sched_domains(). */
1128acc3f5d7SRusty Russell cpumask_var_t *alloc_sched_domains(unsigned int ndoms);
1129acc3f5d7SRusty Russell void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms);
1130acc3f5d7SRusty Russell 
113139be3501SPeter Zijlstra bool cpus_share_cache(int this_cpu, int that_cpu);
113239be3501SPeter Zijlstra 
1133143e1e28SVincent Guittot typedef const struct cpumask *(*sched_domain_mask_f)(int cpu);
1134b6220ad6SGuenter Roeck typedef int (*sched_domain_flags_f)(void);
1135143e1e28SVincent Guittot 
1136143e1e28SVincent Guittot #define SDTL_OVERLAP	0x01
1137143e1e28SVincent Guittot 
1138143e1e28SVincent Guittot struct sd_data {
1139143e1e28SVincent Guittot 	struct sched_domain **__percpu sd;
1140143e1e28SVincent Guittot 	struct sched_group **__percpu sg;
114163b2ca30SNicolas Pitre 	struct sched_group_capacity **__percpu sgc;
1142143e1e28SVincent Guittot };
1143143e1e28SVincent Guittot 
1144143e1e28SVincent Guittot struct sched_domain_topology_level {
1145143e1e28SVincent Guittot 	sched_domain_mask_f mask;
1146143e1e28SVincent Guittot 	sched_domain_flags_f sd_flags;
1147143e1e28SVincent Guittot 	int		    flags;
1148143e1e28SVincent Guittot 	int		    numa_level;
1149143e1e28SVincent Guittot 	struct sd_data      data;
1150143e1e28SVincent Guittot #ifdef CONFIG_SCHED_DEBUG
1151143e1e28SVincent Guittot 	char                *name;
1152143e1e28SVincent Guittot #endif
1153143e1e28SVincent Guittot };
1154143e1e28SVincent Guittot 
1155143e1e28SVincent Guittot extern void set_sched_topology(struct sched_domain_topology_level *tl);
1156f6be8af1SChuansheng Liu extern void wake_up_if_idle(int cpu);
1157143e1e28SVincent Guittot 
1158143e1e28SVincent Guittot #ifdef CONFIG_SCHED_DEBUG
1159143e1e28SVincent Guittot # define SD_INIT_NAME(type)		.name = #type
1160143e1e28SVincent Guittot #else
1161143e1e28SVincent Guittot # define SD_INIT_NAME(type)
1162143e1e28SVincent Guittot #endif
1163143e1e28SVincent Guittot 
11641b427c15SIngo Molnar #else /* CONFIG_SMP */
11651da177e4SLinus Torvalds 
11661b427c15SIngo Molnar struct sched_domain_attr;
11671b427c15SIngo Molnar 
11681b427c15SIngo Molnar static inline void
1169acc3f5d7SRusty Russell partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
11701b427c15SIngo Molnar 			struct sched_domain_attr *dattr_new)
1171d02c7a8cSCon Kolivas {
1172d02c7a8cSCon Kolivas }
117339be3501SPeter Zijlstra 
117439be3501SPeter Zijlstra static inline bool cpus_share_cache(int this_cpu, int that_cpu)
117539be3501SPeter Zijlstra {
117639be3501SPeter Zijlstra 	return true;
117739be3501SPeter Zijlstra }
117839be3501SPeter Zijlstra 
11791b427c15SIngo Molnar #endif	/* !CONFIG_SMP */
11801da177e4SLinus Torvalds 
118147fe38fcSPeter Zijlstra 
11821da177e4SLinus Torvalds struct io_context;			/* See blkdev.h */
11831da177e4SLinus Torvalds 
11841da177e4SLinus Torvalds 
1185383f2835SChen, Kenneth W #ifdef ARCH_HAS_PREFETCH_SWITCH_STACK
118636c8b586SIngo Molnar extern void prefetch_stack(struct task_struct *t);
1187383f2835SChen, Kenneth W #else
1188383f2835SChen, Kenneth W static inline void prefetch_stack(struct task_struct *t) { }
1189383f2835SChen, Kenneth W #endif
11901da177e4SLinus Torvalds 
11911da177e4SLinus Torvalds struct audit_context;		/* See audit.c */
11921da177e4SLinus Torvalds struct mempolicy;
1193b92ce558SJens Axboe struct pipe_inode_info;
11944865ecf1SSerge E. Hallyn struct uts_namespace;
11951da177e4SLinus Torvalds 
119620b8a59fSIngo Molnar struct load_weight {
11979dbdb155SPeter Zijlstra 	unsigned long weight;
11989dbdb155SPeter Zijlstra 	u32 inv_weight;
119920b8a59fSIngo Molnar };
120020b8a59fSIngo Molnar 
12019d89c257SYuyang Du /*
12029d89c257SYuyang Du  * The load_avg/util_avg accumulates an infinite geometric series.
1203e0f5f3afSDietmar Eggemann  * 1) load_avg factors frequency scaling into the amount of time that a
1204e0f5f3afSDietmar Eggemann  * sched_entity is runnable on a rq into its weight. For cfs_rq, it is the
1205e0f5f3afSDietmar Eggemann  * aggregated such weights of all runnable and blocked sched_entities.
1206e3279a2eSDietmar Eggemann  * 2) util_avg factors frequency and cpu scaling into the amount of time
12079d89c257SYuyang Du  * that a sched_entity is running on a CPU, in the range [0..SCHED_LOAD_SCALE].
12089d89c257SYuyang Du  * For cfs_rq, it is the aggregated such times of all runnable and
12099d89c257SYuyang Du  * blocked sched_entities.
12109d89c257SYuyang Du  * The 64 bit load_sum can:
12119d89c257SYuyang Du  * 1) for cfs_rq, afford 4353082796 (=2^64/47742/88761) entities with
12129d89c257SYuyang Du  * the highest weight (=88761) always runnable, we should not overflow
12139d89c257SYuyang Du  * 2) for entity, support any load.weight always runnable
12149d89c257SYuyang Du  */
12159d85f21cSPaul Turner struct sched_avg {
12169d89c257SYuyang Du 	u64 last_update_time, load_sum;
12179d89c257SYuyang Du 	u32 util_sum, period_contrib;
12189d89c257SYuyang Du 	unsigned long load_avg, util_avg;
12199d85f21cSPaul Turner };
12209d85f21cSPaul Turner 
122194c18227SIngo Molnar #ifdef CONFIG_SCHEDSTATS
122241acab88SLucas De Marchi struct sched_statistics {
122394c18227SIngo Molnar 	u64			wait_start;
122494c18227SIngo Molnar 	u64			wait_max;
12256d082592SArjan van de Ven 	u64			wait_count;
12266d082592SArjan van de Ven 	u64			wait_sum;
12278f0dfc34SArjan van de Ven 	u64			iowait_count;
12288f0dfc34SArjan van de Ven 	u64			iowait_sum;
122994c18227SIngo Molnar 
123094c18227SIngo Molnar 	u64			sleep_start;
123120b8a59fSIngo Molnar 	u64			sleep_max;
123294c18227SIngo Molnar 	s64			sum_sleep_runtime;
123394c18227SIngo Molnar 
123494c18227SIngo Molnar 	u64			block_start;
123520b8a59fSIngo Molnar 	u64			block_max;
123620b8a59fSIngo Molnar 	u64			exec_max;
1237eba1ed4bSIngo Molnar 	u64			slice_max;
1238cc367732SIngo Molnar 
1239cc367732SIngo Molnar 	u64			nr_migrations_cold;
1240cc367732SIngo Molnar 	u64			nr_failed_migrations_affine;
1241cc367732SIngo Molnar 	u64			nr_failed_migrations_running;
1242cc367732SIngo Molnar 	u64			nr_failed_migrations_hot;
1243cc367732SIngo Molnar 	u64			nr_forced_migrations;
1244cc367732SIngo Molnar 
1245cc367732SIngo Molnar 	u64			nr_wakeups;
1246cc367732SIngo Molnar 	u64			nr_wakeups_sync;
1247cc367732SIngo Molnar 	u64			nr_wakeups_migrate;
1248cc367732SIngo Molnar 	u64			nr_wakeups_local;
1249cc367732SIngo Molnar 	u64			nr_wakeups_remote;
1250cc367732SIngo Molnar 	u64			nr_wakeups_affine;
1251cc367732SIngo Molnar 	u64			nr_wakeups_affine_attempts;
1252cc367732SIngo Molnar 	u64			nr_wakeups_passive;
1253cc367732SIngo Molnar 	u64			nr_wakeups_idle;
125441acab88SLucas De Marchi };
125541acab88SLucas De Marchi #endif
125641acab88SLucas De Marchi 
125741acab88SLucas De Marchi struct sched_entity {
125841acab88SLucas De Marchi 	struct load_weight	load;		/* for load-balancing */
125941acab88SLucas De Marchi 	struct rb_node		run_node;
126041acab88SLucas De Marchi 	struct list_head	group_node;
126141acab88SLucas De Marchi 	unsigned int		on_rq;
126241acab88SLucas De Marchi 
126341acab88SLucas De Marchi 	u64			exec_start;
126441acab88SLucas De Marchi 	u64			sum_exec_runtime;
126541acab88SLucas De Marchi 	u64			vruntime;
126641acab88SLucas De Marchi 	u64			prev_sum_exec_runtime;
126741acab88SLucas De Marchi 
126841acab88SLucas De Marchi 	u64			nr_migrations;
126941acab88SLucas De Marchi 
127041acab88SLucas De Marchi #ifdef CONFIG_SCHEDSTATS
127141acab88SLucas De Marchi 	struct sched_statistics statistics;
127294c18227SIngo Molnar #endif
127394c18227SIngo Molnar 
127420b8a59fSIngo Molnar #ifdef CONFIG_FAIR_GROUP_SCHED
1275fed14d45SPeter Zijlstra 	int			depth;
127620b8a59fSIngo Molnar 	struct sched_entity	*parent;
127720b8a59fSIngo Molnar 	/* rq on which this entity is (to be) queued: */
127820b8a59fSIngo Molnar 	struct cfs_rq		*cfs_rq;
127920b8a59fSIngo Molnar 	/* rq "owned" by this entity/group: */
128020b8a59fSIngo Molnar 	struct cfs_rq		*my_q;
128120b8a59fSIngo Molnar #endif
12828bd75c77SClark Williams 
1283141965c7SAlex Shi #ifdef CONFIG_SMP
12845a107804SJiri Olsa 	/*
12855a107804SJiri Olsa 	 * Per entity load average tracking.
12865a107804SJiri Olsa 	 *
12875a107804SJiri Olsa 	 * Put into separate cache line so it does not
12885a107804SJiri Olsa 	 * collide with read-mostly values above.
12895a107804SJiri Olsa 	 */
12905a107804SJiri Olsa 	struct sched_avg	avg ____cacheline_aligned_in_smp;
12919d85f21cSPaul Turner #endif
129220b8a59fSIngo Molnar };
129370b97a7fSIngo Molnar 
1294fa717060SPeter Zijlstra struct sched_rt_entity {
1295fa717060SPeter Zijlstra 	struct list_head run_list;
129678f2c7dbSPeter Zijlstra 	unsigned long timeout;
129757d2aa00SYing Xue 	unsigned long watchdog_stamp;
1298bee367edSRichard Kennedy 	unsigned int time_slice;
1299ff77e468SPeter Zijlstra 	unsigned short on_rq;
1300ff77e468SPeter Zijlstra 	unsigned short on_list;
13016f505b16SPeter Zijlstra 
130258d6c2d7SPeter Zijlstra 	struct sched_rt_entity *back;
1303052f1dc7SPeter Zijlstra #ifdef CONFIG_RT_GROUP_SCHED
13046f505b16SPeter Zijlstra 	struct sched_rt_entity	*parent;
13056f505b16SPeter Zijlstra 	/* rq on which this entity is (to be) queued: */
13066f505b16SPeter Zijlstra 	struct rt_rq		*rt_rq;
13076f505b16SPeter Zijlstra 	/* rq "owned" by this entity/group: */
13086f505b16SPeter Zijlstra 	struct rt_rq		*my_q;
13096f505b16SPeter Zijlstra #endif
1310fa717060SPeter Zijlstra };
1311fa717060SPeter Zijlstra 
1312aab03e05SDario Faggioli struct sched_dl_entity {
1313aab03e05SDario Faggioli 	struct rb_node	rb_node;
1314aab03e05SDario Faggioli 
1315aab03e05SDario Faggioli 	/*
1316aab03e05SDario Faggioli 	 * Original scheduling parameters. Copied here from sched_attr
13174027d080Sxiaofeng.yan 	 * during sched_setattr(), they will remain the same until
13184027d080Sxiaofeng.yan 	 * the next sched_setattr().
1319aab03e05SDario Faggioli 	 */
1320aab03e05SDario Faggioli 	u64 dl_runtime;		/* maximum runtime for each instance	*/
1321aab03e05SDario Faggioli 	u64 dl_deadline;	/* relative deadline of each instance	*/
1322755378a4SHarald Gustafsson 	u64 dl_period;		/* separation of two instances (period) */
1323332ac17eSDario Faggioli 	u64 dl_bw;		/* dl_runtime / dl_deadline		*/
1324aab03e05SDario Faggioli 
1325aab03e05SDario Faggioli 	/*
1326aab03e05SDario Faggioli 	 * Actual scheduling parameters. Initialized with the values above,
1327aab03e05SDario Faggioli 	 * they are continously updated during task execution. Note that
1328aab03e05SDario Faggioli 	 * the remaining runtime could be < 0 in case we are in overrun.
1329aab03e05SDario Faggioli 	 */
1330aab03e05SDario Faggioli 	s64 runtime;		/* remaining runtime for this instance	*/
1331aab03e05SDario Faggioli 	u64 deadline;		/* absolute deadline for this instance	*/
1332aab03e05SDario Faggioli 	unsigned int flags;	/* specifying the scheduler behaviour	*/
1333aab03e05SDario Faggioli 
1334aab03e05SDario Faggioli 	/*
1335aab03e05SDario Faggioli 	 * Some bool flags:
1336aab03e05SDario Faggioli 	 *
1337aab03e05SDario Faggioli 	 * @dl_throttled tells if we exhausted the runtime. If so, the
1338aab03e05SDario Faggioli 	 * task has to wait for a replenishment to be performed at the
1339aab03e05SDario Faggioli 	 * next firing of dl_timer.
1340aab03e05SDario Faggioli 	 *
13412d3d891dSDario Faggioli 	 * @dl_boosted tells if we are boosted due to DI. If so we are
13422d3d891dSDario Faggioli 	 * outside bandwidth enforcement mechanism (but only until we
13435bfd126eSJuri Lelli 	 * exit the critical section);
13445bfd126eSJuri Lelli 	 *
13455bfd126eSJuri Lelli 	 * @dl_yielded tells if task gave up the cpu before consuming
13465bfd126eSJuri Lelli 	 * all its available runtime during the last job.
1347aab03e05SDario Faggioli 	 */
134872f9f3fdSLuca Abeni 	int dl_throttled, dl_boosted, dl_yielded;
1349aab03e05SDario Faggioli 
1350aab03e05SDario Faggioli 	/*
1351aab03e05SDario Faggioli 	 * Bandwidth enforcement timer. Each -deadline task has its
1352aab03e05SDario Faggioli 	 * own bandwidth to be enforced, thus we need one timer per task.
1353aab03e05SDario Faggioli 	 */
1354aab03e05SDario Faggioli 	struct hrtimer dl_timer;
1355aab03e05SDario Faggioli };
13568bd75c77SClark Williams 
13571d082fd0SPaul E. McKenney union rcu_special {
13581d082fd0SPaul E. McKenney 	struct {
13598203d6d0SPaul E. McKenney 		u8 blocked;
13608203d6d0SPaul E. McKenney 		u8 need_qs;
13618203d6d0SPaul E. McKenney 		u8 exp_need_qs;
13628203d6d0SPaul E. McKenney 		u8 pad;	/* Otherwise the compiler can store garbage here. */
13638203d6d0SPaul E. McKenney 	} b; /* Bits. */
13648203d6d0SPaul E. McKenney 	u32 s; /* Set of bits. */
13651d082fd0SPaul E. McKenney };
136686848966SPaul E. McKenney struct rcu_node;
136786848966SPaul E. McKenney 
13688dc85d54SPeter Zijlstra enum perf_event_task_context {
13698dc85d54SPeter Zijlstra 	perf_invalid_context = -1,
13708dc85d54SPeter Zijlstra 	perf_hw_context = 0,
137189a1e187SPeter Zijlstra 	perf_sw_context,
13728dc85d54SPeter Zijlstra 	perf_nr_task_contexts,
13738dc85d54SPeter Zijlstra };
13748dc85d54SPeter Zijlstra 
137572b252aeSMel Gorman /* Track pages that require TLB flushes */
137672b252aeSMel Gorman struct tlbflush_unmap_batch {
137772b252aeSMel Gorman 	/*
137872b252aeSMel Gorman 	 * Each bit set is a CPU that potentially has a TLB entry for one of
137972b252aeSMel Gorman 	 * the PFNs being flushed. See set_tlb_ubc_flush_pending().
138072b252aeSMel Gorman 	 */
138172b252aeSMel Gorman 	struct cpumask cpumask;
138272b252aeSMel Gorman 
138372b252aeSMel Gorman 	/* True if any bit in cpumask is set */
138472b252aeSMel Gorman 	bool flush_required;
1385d950c947SMel Gorman 
1386d950c947SMel Gorman 	/*
1387d950c947SMel Gorman 	 * If true then the PTE was dirty when unmapped. The entry must be
1388d950c947SMel Gorman 	 * flushed before IO is initiated or a stale TLB entry potentially
1389d950c947SMel Gorman 	 * allows an update without redirtying the page.
1390d950c947SMel Gorman 	 */
1391d950c947SMel Gorman 	bool writable;
139272b252aeSMel Gorman };
139372b252aeSMel Gorman 
13941da177e4SLinus Torvalds struct task_struct {
13951da177e4SLinus Torvalds 	volatile long state;	/* -1 unrunnable, 0 runnable, >0 stopped */
1396f7e4217bSRoman Zippel 	void *stack;
13971da177e4SLinus Torvalds 	atomic_t usage;
139897dc32cdSWilliam Cohen 	unsigned int flags;	/* per process flags, defined below */
139997dc32cdSWilliam Cohen 	unsigned int ptrace;
14001da177e4SLinus Torvalds 
14012dd73a4fSPeter Williams #ifdef CONFIG_SMP
1402fa14ff4aSPeter Zijlstra 	struct llist_node wake_entry;
14033ca7a440SPeter Zijlstra 	int on_cpu;
140463b0e9edSMike Galbraith 	unsigned int wakee_flips;
140562470419SMichael Wang 	unsigned long wakee_flip_decay_ts;
140663b0e9edSMike Galbraith 	struct task_struct *last_wakee;
1407ac66f547SPeter Zijlstra 
1408ac66f547SPeter Zijlstra 	int wake_cpu;
14094866cde0SNick Piggin #endif
1410fd2f4419SPeter Zijlstra 	int on_rq;
141150e645a8SIngo Molnar 
1412b29739f9SIngo Molnar 	int prio, static_prio, normal_prio;
1413c7aceabaSRichard Kennedy 	unsigned int rt_priority;
14145522d5d5SIngo Molnar 	const struct sched_class *sched_class;
141520b8a59fSIngo Molnar 	struct sched_entity se;
1416fa717060SPeter Zijlstra 	struct sched_rt_entity rt;
14178323f26cSPeter Zijlstra #ifdef CONFIG_CGROUP_SCHED
14188323f26cSPeter Zijlstra 	struct task_group *sched_task_group;
14198323f26cSPeter Zijlstra #endif
1420aab03e05SDario Faggioli 	struct sched_dl_entity dl;
14211da177e4SLinus Torvalds 
1422e107be36SAvi Kivity #ifdef CONFIG_PREEMPT_NOTIFIERS
1423e107be36SAvi Kivity 	/* list of struct preempt_notifier: */
1424e107be36SAvi Kivity 	struct hlist_head preempt_notifiers;
1425e107be36SAvi Kivity #endif
1426e107be36SAvi Kivity 
14276c5c9341SAlexey Dobriyan #ifdef CONFIG_BLK_DEV_IO_TRACE
14282056a782SJens Axboe 	unsigned int btrace_seq;
14296c5c9341SAlexey Dobriyan #endif
14301da177e4SLinus Torvalds 
143197dc32cdSWilliam Cohen 	unsigned int policy;
143229baa747SPeter Zijlstra 	int nr_cpus_allowed;
14331da177e4SLinus Torvalds 	cpumask_t cpus_allowed;
14341da177e4SLinus Torvalds 
1435a57eb940SPaul E. McKenney #ifdef CONFIG_PREEMPT_RCU
1436e260be67SPaul E. McKenney 	int rcu_read_lock_nesting;
14371d082fd0SPaul E. McKenney 	union rcu_special rcu_read_unlock_special;
1438f41d911fSPaul E. McKenney 	struct list_head rcu_node_entry;
1439a57eb940SPaul E. McKenney 	struct rcu_node *rcu_blocked_node;
144028f6569aSPranith Kumar #endif /* #ifdef CONFIG_PREEMPT_RCU */
14418315f422SPaul E. McKenney #ifdef CONFIG_TASKS_RCU
14428315f422SPaul E. McKenney 	unsigned long rcu_tasks_nvcsw;
14438315f422SPaul E. McKenney 	bool rcu_tasks_holdout;
14448315f422SPaul E. McKenney 	struct list_head rcu_tasks_holdout_list;
1445176f8f7aSPaul E. McKenney 	int rcu_tasks_idle_cpu;
14468315f422SPaul E. McKenney #endif /* #ifdef CONFIG_TASKS_RCU */
1447e260be67SPaul E. McKenney 
1448f6db8347SNaveen N. Rao #ifdef CONFIG_SCHED_INFO
14491da177e4SLinus Torvalds 	struct sched_info sched_info;
14501da177e4SLinus Torvalds #endif
14511da177e4SLinus Torvalds 
14521da177e4SLinus Torvalds 	struct list_head tasks;
1453806c09a7SDario Faggioli #ifdef CONFIG_SMP
1454917b627dSGregory Haskins 	struct plist_node pushable_tasks;
14551baca4ceSJuri Lelli 	struct rb_node pushable_dl_tasks;
1456806c09a7SDario Faggioli #endif
14571da177e4SLinus Torvalds 
14581da177e4SLinus Torvalds 	struct mm_struct *mm, *active_mm;
1459615d6e87SDavidlohr Bueso 	/* per-thread vma caching */
1460615d6e87SDavidlohr Bueso 	u32 vmacache_seqnum;
1461615d6e87SDavidlohr Bueso 	struct vm_area_struct *vmacache[VMACACHE_SIZE];
146234e55232SKAMEZAWA Hiroyuki #if defined(SPLIT_RSS_COUNTING)
146334e55232SKAMEZAWA Hiroyuki 	struct task_rss_stat	rss_stat;
146434e55232SKAMEZAWA Hiroyuki #endif
14651da177e4SLinus Torvalds /* task state */
146697dc32cdSWilliam Cohen 	int exit_state;
14671da177e4SLinus Torvalds 	int exit_code, exit_signal;
14681da177e4SLinus Torvalds 	int pdeath_signal;  /*  The signal sent when the parent dies  */
1469e7cc4173SPalmer Dabbelt 	unsigned long jobctl;	/* JOBCTL_*, siglock protected */
14709b89f6baSAndrei Epure 
14719b89f6baSAndrei Epure 	/* Used for emulating ABI behavior of previous Linux versions */
147297dc32cdSWilliam Cohen 	unsigned int personality;
14739b89f6baSAndrei Epure 
1474be958bdcSPeter Zijlstra 	/* scheduler bits, serialized by scheduler locks */
1475ca94c442SLennart Poettering 	unsigned sched_reset_on_fork:1;
1476a8e4f2eaSPeter Zijlstra 	unsigned sched_contributes_to_load:1;
1477ff303e66SPeter Zijlstra 	unsigned sched_migrated:1;
1478be958bdcSPeter Zijlstra 	unsigned :0; /* force alignment to the next boundary */
1479be958bdcSPeter Zijlstra 
1480be958bdcSPeter Zijlstra 	/* unserialized, strictly 'current' */
1481be958bdcSPeter Zijlstra 	unsigned in_execve:1; /* bit to tell LSMs we're in execve */
1482be958bdcSPeter Zijlstra 	unsigned in_iowait:1;
1483626ebc41STejun Heo #ifdef CONFIG_MEMCG
1484626ebc41STejun Heo 	unsigned memcg_may_oom:1;
1485127424c8SJohannes Weiner #ifndef CONFIG_SLOB
14866f185c29SVladimir Davydov 	unsigned memcg_kmem_skip_account:1;
14876f185c29SVladimir Davydov #endif
1488127424c8SJohannes Weiner #endif
1489ff303e66SPeter Zijlstra #ifdef CONFIG_COMPAT_BRK
1490ff303e66SPeter Zijlstra 	unsigned brk_randomized:1;
1491ff303e66SPeter Zijlstra #endif
14926f185c29SVladimir Davydov 
14931d4457f9SKees Cook 	unsigned long atomic_flags; /* Flags needing atomic access. */
14941d4457f9SKees Cook 
1495f56141e3SAndy Lutomirski 	struct restart_block restart_block;
1496f56141e3SAndy Lutomirski 
14971da177e4SLinus Torvalds 	pid_t pid;
14981da177e4SLinus Torvalds 	pid_t tgid;
14990a425405SArjan van de Ven 
15001314562aSHiroshi Shimamoto #ifdef CONFIG_CC_STACKPROTECTOR
15010a425405SArjan van de Ven 	/* Canary value for the -fstack-protector gcc feature */
15020a425405SArjan van de Ven 	unsigned long stack_canary;
15031314562aSHiroshi Shimamoto #endif
15041da177e4SLinus Torvalds 	/*
15051da177e4SLinus Torvalds 	 * pointers to (original) parent process, youngest child, younger sibling,
15061da177e4SLinus Torvalds 	 * older sibling, respectively.  (p->father can be replaced with
1507f470021aSRoland McGrath 	 * p->real_parent->pid)
15081da177e4SLinus Torvalds 	 */
1509abd63bc3SKees Cook 	struct task_struct __rcu *real_parent; /* real parent process */
1510abd63bc3SKees Cook 	struct task_struct __rcu *parent; /* recipient of SIGCHLD, wait4() reports */
15111da177e4SLinus Torvalds 	/*
1512f470021aSRoland McGrath 	 * children/sibling forms the list of my natural children
15131da177e4SLinus Torvalds 	 */
15141da177e4SLinus Torvalds 	struct list_head children;	/* list of my children */
15151da177e4SLinus Torvalds 	struct list_head sibling;	/* linkage in my parent's children list */
15161da177e4SLinus Torvalds 	struct task_struct *group_leader;	/* threadgroup leader */
15171da177e4SLinus Torvalds 
1518f470021aSRoland McGrath 	/*
1519f470021aSRoland McGrath 	 * ptraced is the list of tasks this task is using ptrace on.
1520f470021aSRoland McGrath 	 * This includes both natural children and PTRACE_ATTACH targets.
1521f470021aSRoland McGrath 	 * p->ptrace_entry is p's link on the p->parent->ptraced list.
1522f470021aSRoland McGrath 	 */
1523f470021aSRoland McGrath 	struct list_head ptraced;
1524f470021aSRoland McGrath 	struct list_head ptrace_entry;
1525f470021aSRoland McGrath 
15261da177e4SLinus Torvalds 	/* PID/PID hash table linkage. */
152792476d7fSEric W. Biederman 	struct pid_link pids[PIDTYPE_MAX];
152847e65328SOleg Nesterov 	struct list_head thread_group;
15290c740d0aSOleg Nesterov 	struct list_head thread_node;
15301da177e4SLinus Torvalds 
15311da177e4SLinus Torvalds 	struct completion *vfork_done;		/* for vfork() */
15321da177e4SLinus Torvalds 	int __user *set_child_tid;		/* CLONE_CHILD_SETTID */
15331da177e4SLinus Torvalds 	int __user *clear_child_tid;		/* CLONE_CHILD_CLEARTID */
15341da177e4SLinus Torvalds 
1535c66f08beSMichael Neuling 	cputime_t utime, stime, utimescaled, stimescaled;
15369ac52315SLaurent Vivier 	cputime_t gtime;
15379d7fb042SPeter Zijlstra 	struct prev_cputime prev_cputime;
15386a61671bSFrederic Weisbecker #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
1539b7ce2277SFrederic Weisbecker 	seqcount_t vtime_seqcount;
15406a61671bSFrederic Weisbecker 	unsigned long long vtime_snap;
15416a61671bSFrederic Weisbecker 	enum {
15427098c1eaSFrederic Weisbecker 		/* Task is sleeping or running in a CPU with VTIME inactive */
15437098c1eaSFrederic Weisbecker 		VTIME_INACTIVE = 0,
15447098c1eaSFrederic Weisbecker 		/* Task runs in userspace in a CPU with VTIME active */
15456a61671bSFrederic Weisbecker 		VTIME_USER,
15467098c1eaSFrederic Weisbecker 		/* Task runs in kernelspace in a CPU with VTIME active */
15476a61671bSFrederic Weisbecker 		VTIME_SYS,
15486a61671bSFrederic Weisbecker 	} vtime_snap_whence;
15496a61671bSFrederic Weisbecker #endif
1550d027d45dSFrederic Weisbecker 
1551d027d45dSFrederic Weisbecker #ifdef CONFIG_NO_HZ_FULL
1552*f009a7a7SFrederic Weisbecker 	atomic_t tick_dep_mask;
1553d027d45dSFrederic Weisbecker #endif
15541da177e4SLinus Torvalds 	unsigned long nvcsw, nivcsw; /* context switch counts */
1555ccbf62d8SThomas Gleixner 	u64 start_time;		/* monotonic time in nsec */
155657e0be04SThomas Gleixner 	u64 real_start_time;	/* boot based time in nsec */
15571da177e4SLinus Torvalds /* mm fault and swap info: this can arguably be seen as either mm-specific or thread-specific */
15581da177e4SLinus Torvalds 	unsigned long min_flt, maj_flt;
15591da177e4SLinus Torvalds 
1560f06febc9SFrank Mayhar 	struct task_cputime cputime_expires;
15611da177e4SLinus Torvalds 	struct list_head cpu_timers[3];
15621da177e4SLinus Torvalds 
15631da177e4SLinus Torvalds /* process credentials */
15641b0ba1c9SArnd Bergmann 	const struct cred __rcu *real_cred; /* objective and real subjective task
15653b11a1deSDavid Howells 					 * credentials (COW) */
15661b0ba1c9SArnd Bergmann 	const struct cred __rcu *cred;	/* effective (overridable) subjective task
15673b11a1deSDavid Howells 					 * credentials (COW) */
156836772092SPaolo 'Blaisorblade' Giarrusso 	char comm[TASK_COMM_LEN]; /* executable name excluding path
156936772092SPaolo 'Blaisorblade' Giarrusso 				     - access with [gs]et_task_comm (which lock
157036772092SPaolo 'Blaisorblade' Giarrusso 				       it with task_lock())
1571221af7f8SLinus Torvalds 				     - initialized normally by setup_new_exec */
15721da177e4SLinus Torvalds /* file system info */
1573756daf26SNeilBrown 	struct nameidata *nameidata;
15743d5b6fccSAlexey Dobriyan #ifdef CONFIG_SYSVIPC
15751da177e4SLinus Torvalds /* ipc stuff */
15761da177e4SLinus Torvalds 	struct sysv_sem sysvsem;
1577ab602f79SJack Miller 	struct sysv_shm sysvshm;
15783d5b6fccSAlexey Dobriyan #endif
1579e162b39aSMandeep Singh Baines #ifdef CONFIG_DETECT_HUNG_TASK
158082a1fcb9SIngo Molnar /* hung task detection */
158182a1fcb9SIngo Molnar 	unsigned long last_switch_count;
158282a1fcb9SIngo Molnar #endif
15831da177e4SLinus Torvalds /* filesystem information */
15841da177e4SLinus Torvalds 	struct fs_struct *fs;
15851da177e4SLinus Torvalds /* open file information */
15861da177e4SLinus Torvalds 	struct files_struct *files;
15871651e14eSSerge E. Hallyn /* namespaces */
1588ab516013SSerge E. Hallyn 	struct nsproxy *nsproxy;
15891da177e4SLinus Torvalds /* signal handlers */
15901da177e4SLinus Torvalds 	struct signal_struct *signal;
15911da177e4SLinus Torvalds 	struct sighand_struct *sighand;
15921da177e4SLinus Torvalds 
15931da177e4SLinus Torvalds 	sigset_t blocked, real_blocked;
1594f3de272bSRoland McGrath 	sigset_t saved_sigmask;	/* restored if set_restore_sigmask() was used */
15951da177e4SLinus Torvalds 	struct sigpending pending;
15961da177e4SLinus Torvalds 
15971da177e4SLinus Torvalds 	unsigned long sas_ss_sp;
15981da177e4SLinus Torvalds 	size_t sas_ss_size;
15992e01fabeSOleg Nesterov 
160067d12145SAl Viro 	struct callback_head *task_works;
1601e73f8959SOleg Nesterov 
16021da177e4SLinus Torvalds 	struct audit_context *audit_context;
1603bfef93a5SAl Viro #ifdef CONFIG_AUDITSYSCALL
1604e1760bd5SEric W. Biederman 	kuid_t loginuid;
16054746ec5bSEric Paris 	unsigned int sessionid;
1606bfef93a5SAl Viro #endif
1607932ecebbSWill Drewry 	struct seccomp seccomp;
16081da177e4SLinus Torvalds 
16091da177e4SLinus Torvalds /* Thread group tracking */
16101da177e4SLinus Torvalds    	u32 parent_exec_id;
16111da177e4SLinus Torvalds    	u32 self_exec_id;
161258568d2aSMiao Xie /* Protection of (de-)allocation: mm, files, fs, tty, keyrings, mems_allowed,
161358568d2aSMiao Xie  * mempolicy */
16141da177e4SLinus Torvalds 	spinlock_t alloc_lock;
16151da177e4SLinus Torvalds 
1616b29739f9SIngo Molnar 	/* Protection of the PI data structures: */
16171d615482SThomas Gleixner 	raw_spinlock_t pi_lock;
1618b29739f9SIngo Molnar 
161976751049SPeter Zijlstra 	struct wake_q_node wake_q;
162076751049SPeter Zijlstra 
162123f78d4aSIngo Molnar #ifdef CONFIG_RT_MUTEXES
162223f78d4aSIngo Molnar 	/* PI waiters blocked on a rt_mutex held by this task */
1623fb00aca4SPeter Zijlstra 	struct rb_root pi_waiters;
1624fb00aca4SPeter Zijlstra 	struct rb_node *pi_waiters_leftmost;
162523f78d4aSIngo Molnar 	/* Deadlock detection and priority inheritance handling */
162623f78d4aSIngo Molnar 	struct rt_mutex_waiter *pi_blocked_on;
162723f78d4aSIngo Molnar #endif
162823f78d4aSIngo Molnar 
1629408894eeSIngo Molnar #ifdef CONFIG_DEBUG_MUTEXES
1630408894eeSIngo Molnar 	/* mutex deadlock detection */
1631408894eeSIngo Molnar 	struct mutex_waiter *blocked_on;
1632408894eeSIngo Molnar #endif
1633de30a2b3SIngo Molnar #ifdef CONFIG_TRACE_IRQFLAGS
1634de30a2b3SIngo Molnar 	unsigned int irq_events;
1635de30a2b3SIngo Molnar 	unsigned long hardirq_enable_ip;
1636de30a2b3SIngo Molnar 	unsigned long hardirq_disable_ip;
1637fa1452e8SHiroshi Shimamoto 	unsigned int hardirq_enable_event;
1638de30a2b3SIngo Molnar 	unsigned int hardirq_disable_event;
1639fa1452e8SHiroshi Shimamoto 	int hardirqs_enabled;
1640de30a2b3SIngo Molnar 	int hardirq_context;
1641fa1452e8SHiroshi Shimamoto 	unsigned long softirq_disable_ip;
1642fa1452e8SHiroshi Shimamoto 	unsigned long softirq_enable_ip;
1643fa1452e8SHiroshi Shimamoto 	unsigned int softirq_disable_event;
1644fa1452e8SHiroshi Shimamoto 	unsigned int softirq_enable_event;
1645fa1452e8SHiroshi Shimamoto 	int softirqs_enabled;
1646de30a2b3SIngo Molnar 	int softirq_context;
1647de30a2b3SIngo Molnar #endif
1648fbb9ce95SIngo Molnar #ifdef CONFIG_LOCKDEP
1649bdb9441eSPeter Zijlstra # define MAX_LOCK_DEPTH 48UL
1650fbb9ce95SIngo Molnar 	u64 curr_chain_key;
1651fbb9ce95SIngo Molnar 	int lockdep_depth;
1652fbb9ce95SIngo Molnar 	unsigned int lockdep_recursion;
1653c7aceabaSRichard Kennedy 	struct held_lock held_locks[MAX_LOCK_DEPTH];
1654cf40bd16SNick Piggin 	gfp_t lockdep_reclaim_gfp;
1655fbb9ce95SIngo Molnar #endif
1656c6d30853SAndrey Ryabinin #ifdef CONFIG_UBSAN
1657c6d30853SAndrey Ryabinin 	unsigned int in_ubsan;
1658c6d30853SAndrey Ryabinin #endif
1659408894eeSIngo Molnar 
16601da177e4SLinus Torvalds /* journalling filesystem info */
16611da177e4SLinus Torvalds 	void *journal_info;
16621da177e4SLinus Torvalds 
1663d89d8796SNeil Brown /* stacked block device info */
1664bddd87c7SAkinobu Mita 	struct bio_list *bio_list;
1665d89d8796SNeil Brown 
166673c10101SJens Axboe #ifdef CONFIG_BLOCK
166773c10101SJens Axboe /* stack plugging */
166873c10101SJens Axboe 	struct blk_plug *plug;
166973c10101SJens Axboe #endif
167073c10101SJens Axboe 
16711da177e4SLinus Torvalds /* VM state */
16721da177e4SLinus Torvalds 	struct reclaim_state *reclaim_state;
16731da177e4SLinus Torvalds 
16741da177e4SLinus Torvalds 	struct backing_dev_info *backing_dev_info;
16751da177e4SLinus Torvalds 
16761da177e4SLinus Torvalds 	struct io_context *io_context;
16771da177e4SLinus Torvalds 
16781da177e4SLinus Torvalds 	unsigned long ptrace_message;
16791da177e4SLinus Torvalds 	siginfo_t *last_siginfo; /* For ptrace use.  */
16807c3ab738SAndrew Morton 	struct task_io_accounting ioac;
16818f0ab514SJay Lan #if defined(CONFIG_TASK_XACCT)
16821da177e4SLinus Torvalds 	u64 acct_rss_mem1;	/* accumulated rss usage */
16831da177e4SLinus Torvalds 	u64 acct_vm_mem1;	/* accumulated virtual memory usage */
168449b5cf34SJonathan Lim 	cputime_t acct_timexpd;	/* stime + utime since last update */
16851da177e4SLinus Torvalds #endif
16861da177e4SLinus Torvalds #ifdef CONFIG_CPUSETS
168758568d2aSMiao Xie 	nodemask_t mems_allowed;	/* Protected by alloc_lock */
1688cc9a6c87SMel Gorman 	seqcount_t mems_allowed_seq;	/* Seqence no to catch updates */
1689825a46afSPaul Jackson 	int cpuset_mem_spread_rotor;
16906adef3ebSJack Steiner 	int cpuset_slab_spread_rotor;
16911da177e4SLinus Torvalds #endif
1692ddbcc7e8SPaul Menage #ifdef CONFIG_CGROUPS
1693817929ecSPaul Menage 	/* Control Group info protected by css_set_lock */
16942c392b8cSArnd Bergmann 	struct css_set __rcu *cgroups;
1695817929ecSPaul Menage 	/* cg_list protected by css_set_lock and tsk->alloc_lock */
1696817929ecSPaul Menage 	struct list_head cg_list;
1697ddbcc7e8SPaul Menage #endif
169842b2dd0aSAlexey Dobriyan #ifdef CONFIG_FUTEX
16990771dfefSIngo Molnar 	struct robust_list_head __user *robust_list;
170034f192c6SIngo Molnar #ifdef CONFIG_COMPAT
170134f192c6SIngo Molnar 	struct compat_robust_list_head __user *compat_robust_list;
170234f192c6SIngo Molnar #endif
1703c87e2837SIngo Molnar 	struct list_head pi_state_list;
1704c87e2837SIngo Molnar 	struct futex_pi_state *pi_state_cache;
170542b2dd0aSAlexey Dobriyan #endif
1706cdd6c482SIngo Molnar #ifdef CONFIG_PERF_EVENTS
17078dc85d54SPeter Zijlstra 	struct perf_event_context *perf_event_ctxp[perf_nr_task_contexts];
1708cdd6c482SIngo Molnar 	struct mutex perf_event_mutex;
1709cdd6c482SIngo Molnar 	struct list_head perf_event_list;
1710a63eaf34SPaul Mackerras #endif
17118f47b187SThomas Gleixner #ifdef CONFIG_DEBUG_PREEMPT
17128f47b187SThomas Gleixner 	unsigned long preempt_disable_ip;
17138f47b187SThomas Gleixner #endif
1714c7aceabaSRichard Kennedy #ifdef CONFIG_NUMA
171558568d2aSMiao Xie 	struct mempolicy *mempolicy;	/* Protected by alloc_lock */
1716c7aceabaSRichard Kennedy 	short il_next;
1717207205a2SEric Dumazet 	short pref_node_fork;
1718c7aceabaSRichard Kennedy #endif
1719cbee9f88SPeter Zijlstra #ifdef CONFIG_NUMA_BALANCING
1720cbee9f88SPeter Zijlstra 	int numa_scan_seq;
1721cbee9f88SPeter Zijlstra 	unsigned int numa_scan_period;
1722598f0ec0SMel Gorman 	unsigned int numa_scan_period_max;
1723de1c9ce6SRik van Riel 	int numa_preferred_nid;
17246b9a7460SMel Gorman 	unsigned long numa_migrate_retry;
1725cbee9f88SPeter Zijlstra 	u64 node_stamp;			/* migration stamp  */
17267e2703e6SRik van Riel 	u64 last_task_numa_placement;
17277e2703e6SRik van Riel 	u64 last_sum_exec_runtime;
1728cbee9f88SPeter Zijlstra 	struct callback_head numa_work;
1729f809ca9aSMel Gorman 
17308c8a743cSPeter Zijlstra 	struct list_head numa_entry;
17318c8a743cSPeter Zijlstra 	struct numa_group *numa_group;
17328c8a743cSPeter Zijlstra 
1733745d6147SMel Gorman 	/*
173444dba3d5SIulia Manda 	 * numa_faults is an array split into four regions:
173544dba3d5SIulia Manda 	 * faults_memory, faults_cpu, faults_memory_buffer, faults_cpu_buffer
173644dba3d5SIulia Manda 	 * in this precise order.
173744dba3d5SIulia Manda 	 *
173844dba3d5SIulia Manda 	 * faults_memory: Exponential decaying average of faults on a per-node
173944dba3d5SIulia Manda 	 * basis. Scheduling placement decisions are made based on these
174044dba3d5SIulia Manda 	 * counts. The values remain static for the duration of a PTE scan.
174144dba3d5SIulia Manda 	 * faults_cpu: Track the nodes the process was running on when a NUMA
174244dba3d5SIulia Manda 	 * hinting fault was incurred.
174344dba3d5SIulia Manda 	 * faults_memory_buffer and faults_cpu_buffer: Record faults per node
174444dba3d5SIulia Manda 	 * during the current scan window. When the scan completes, the counts
174544dba3d5SIulia Manda 	 * in faults_memory and faults_cpu decay and these values are copied.
1746745d6147SMel Gorman 	 */
174744dba3d5SIulia Manda 	unsigned long *numa_faults;
174883e1d2cdSMel Gorman 	unsigned long total_numa_faults;
1749745d6147SMel Gorman 
1750745d6147SMel Gorman 	/*
175104bb2f94SRik van Riel 	 * numa_faults_locality tracks if faults recorded during the last
1752074c2381SMel Gorman 	 * scan window were remote/local or failed to migrate. The task scan
1753074c2381SMel Gorman 	 * period is adapted based on the locality of the faults with different
1754074c2381SMel Gorman 	 * weights depending on whether they were shared or private faults
175504bb2f94SRik van Riel 	 */
1756074c2381SMel Gorman 	unsigned long numa_faults_locality[3];
175704bb2f94SRik van Riel 
1758b32e86b4SIngo Molnar 	unsigned long numa_pages_migrated;
1759cbee9f88SPeter Zijlstra #endif /* CONFIG_NUMA_BALANCING */
1760cbee9f88SPeter Zijlstra 
176172b252aeSMel Gorman #ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
176272b252aeSMel Gorman 	struct tlbflush_unmap_batch tlb_ubc;
176372b252aeSMel Gorman #endif
176472b252aeSMel Gorman 
1765e56d0903SIngo Molnar 	struct rcu_head rcu;
1766b92ce558SJens Axboe 
1767b92ce558SJens Axboe 	/*
1768b92ce558SJens Axboe 	 * cache last used pipe for splice
1769b92ce558SJens Axboe 	 */
1770b92ce558SJens Axboe 	struct pipe_inode_info *splice_pipe;
17715640f768SEric Dumazet 
17725640f768SEric Dumazet 	struct page_frag task_frag;
17735640f768SEric Dumazet 
1774ca74e92bSShailabh Nagar #ifdef	CONFIG_TASK_DELAY_ACCT
1775ca74e92bSShailabh Nagar 	struct task_delay_info *delays;
1776ca74e92bSShailabh Nagar #endif
1777f4f154fdSAkinobu Mita #ifdef CONFIG_FAULT_INJECTION
1778f4f154fdSAkinobu Mita 	int make_it_fail;
1779f4f154fdSAkinobu Mita #endif
17809d823e8fSWu Fengguang 	/*
17819d823e8fSWu Fengguang 	 * when (nr_dirtied >= nr_dirtied_pause), it's time to call
17829d823e8fSWu Fengguang 	 * balance_dirty_pages() for some dirty throttling pause
17839d823e8fSWu Fengguang 	 */
17849d823e8fSWu Fengguang 	int nr_dirtied;
17859d823e8fSWu Fengguang 	int nr_dirtied_pause;
178683712358SWu Fengguang 	unsigned long dirty_paused_when; /* start of a write-and-pause period */
17879d823e8fSWu Fengguang 
17889745512cSArjan van de Ven #ifdef CONFIG_LATENCYTOP
17899745512cSArjan van de Ven 	int latency_record_count;
17909745512cSArjan van de Ven 	struct latency_record latency_record[LT_SAVECOUNT];
17919745512cSArjan van de Ven #endif
17926976675dSArjan van de Ven 	/*
17936976675dSArjan van de Ven 	 * time slack values; these are used to round up poll() and
17946976675dSArjan van de Ven 	 * select() etc timeout values. These are in nanoseconds.
17956976675dSArjan van de Ven 	 */
1796da8b44d5SJohn Stultz 	u64 timer_slack_ns;
1797da8b44d5SJohn Stultz 	u64 default_timer_slack_ns;
1798f8d570a4SDavid Miller 
17990b24beccSAndrey Ryabinin #ifdef CONFIG_KASAN
18000b24beccSAndrey Ryabinin 	unsigned int kasan_depth;
18010b24beccSAndrey Ryabinin #endif
1802fb52607aSFrederic Weisbecker #ifdef CONFIG_FUNCTION_GRAPH_TRACER
18033ad2f3fbSDaniel Mack 	/* Index of current stored address in ret_stack */
1804f201ae23SFrederic Weisbecker 	int curr_ret_stack;
1805f201ae23SFrederic Weisbecker 	/* Stack of return addresses for return function tracing */
1806f201ae23SFrederic Weisbecker 	struct ftrace_ret_stack	*ret_stack;
18078aef2d28SSteven Rostedt 	/* time stamp for last schedule */
18088aef2d28SSteven Rostedt 	unsigned long long ftrace_timestamp;
1809f201ae23SFrederic Weisbecker 	/*
1810f201ae23SFrederic Weisbecker 	 * Number of functions that haven't been traced
1811f201ae23SFrederic Weisbecker 	 * because of depth overrun.
1812f201ae23SFrederic Weisbecker 	 */
1813f201ae23SFrederic Weisbecker 	atomic_t trace_overrun;
1814380c4b14SFrederic Weisbecker 	/* Pause for the tracing */
1815380c4b14SFrederic Weisbecker 	atomic_t tracing_graph_pause;
1816f201ae23SFrederic Weisbecker #endif
1817ea4e2bc4SSteven Rostedt #ifdef CONFIG_TRACING
1818ea4e2bc4SSteven Rostedt 	/* state flags for use by tracers */
1819ea4e2bc4SSteven Rostedt 	unsigned long trace;
1820b1cff0adSSteven Rostedt 	/* bitmask and counter of trace recursion */
1821261842b7SSteven Rostedt 	unsigned long trace_recursion;
1822261842b7SSteven Rostedt #endif /* CONFIG_TRACING */
18235c9a8750SDmitry Vyukov #ifdef CONFIG_KCOV
18245c9a8750SDmitry Vyukov 	/* Coverage collection mode enabled for this task (0 if disabled). */
18255c9a8750SDmitry Vyukov 	enum kcov_mode kcov_mode;
18265c9a8750SDmitry Vyukov 	/* Size of the kcov_area. */
18275c9a8750SDmitry Vyukov 	unsigned	kcov_size;
18285c9a8750SDmitry Vyukov 	/* Buffer for coverage collection. */
18295c9a8750SDmitry Vyukov 	void		*kcov_area;
18305c9a8750SDmitry Vyukov 	/* kcov desciptor wired with this task or NULL. */
18315c9a8750SDmitry Vyukov 	struct kcov	*kcov;
18325c9a8750SDmitry Vyukov #endif
18336f185c29SVladimir Davydov #ifdef CONFIG_MEMCG
1834626ebc41STejun Heo 	struct mem_cgroup *memcg_in_oom;
1835626ebc41STejun Heo 	gfp_t memcg_oom_gfp_mask;
1836626ebc41STejun Heo 	int memcg_oom_order;
1837b23afb93STejun Heo 
1838b23afb93STejun Heo 	/* number of pages to reclaim on returning to userland */
1839b23afb93STejun Heo 	unsigned int memcg_nr_pages_over_high;
1840569b846dSKAMEZAWA Hiroyuki #endif
18410326f5a9SSrikar Dronamraju #ifdef CONFIG_UPROBES
18420326f5a9SSrikar Dronamraju 	struct uprobe_task *utask;
18430326f5a9SSrikar Dronamraju #endif
1844cafe5635SKent Overstreet #if defined(CONFIG_BCACHE) || defined(CONFIG_BCACHE_MODULE)
1845cafe5635SKent Overstreet 	unsigned int	sequential_io;
1846cafe5635SKent Overstreet 	unsigned int	sequential_io_avg;
1847cafe5635SKent Overstreet #endif
18488eb23b9fSPeter Zijlstra #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
18498eb23b9fSPeter Zijlstra 	unsigned long	task_state_change;
18508eb23b9fSPeter Zijlstra #endif
18518bcbde54SDavid Hildenbrand 	int pagefault_disabled;
185203049269SMichal Hocko #ifdef CONFIG_MMU
185329c696e1SVladimir Davydov 	struct task_struct *oom_reaper_list;
185403049269SMichal Hocko #endif
18550c8c0f03SDave Hansen /* CPU-specific state of this task */
18560c8c0f03SDave Hansen 	struct thread_struct thread;
18570c8c0f03SDave Hansen /*
18580c8c0f03SDave Hansen  * WARNING: on x86, 'thread_struct' contains a variable-sized
18590c8c0f03SDave Hansen  * structure.  It *MUST* be at the end of 'task_struct'.
18600c8c0f03SDave Hansen  *
18610c8c0f03SDave Hansen  * Do not put anything below here!
18620c8c0f03SDave Hansen  */
18631da177e4SLinus Torvalds };
18641da177e4SLinus Torvalds 
18655aaeb5c0SIngo Molnar #ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT
18665aaeb5c0SIngo Molnar extern int arch_task_struct_size __read_mostly;
18675aaeb5c0SIngo Molnar #else
18685aaeb5c0SIngo Molnar # define arch_task_struct_size (sizeof(struct task_struct))
18695aaeb5c0SIngo Molnar #endif
18700c8c0f03SDave Hansen 
187176e6eee0SRusty Russell /* Future-safe accessor for struct task_struct's cpus_allowed. */
1872a4636818SRusty Russell #define tsk_cpus_allowed(tsk) (&(tsk)->cpus_allowed)
187376e6eee0SRusty Russell 
18746688cc05SPeter Zijlstra #define TNF_MIGRATED	0x01
18756688cc05SPeter Zijlstra #define TNF_NO_GROUP	0x02
1876dabe1d99SRik van Riel #define TNF_SHARED	0x04
187704bb2f94SRik van Riel #define TNF_FAULT_LOCAL	0x08
1878074c2381SMel Gorman #define TNF_MIGRATE_FAIL 0x10
18796688cc05SPeter Zijlstra 
1880cbee9f88SPeter Zijlstra #ifdef CONFIG_NUMA_BALANCING
18816688cc05SPeter Zijlstra extern void task_numa_fault(int last_node, int node, int pages, int flags);
1882e29cf08bSMel Gorman extern pid_t task_numa_group_id(struct task_struct *p);
18831a687c2eSMel Gorman extern void set_numabalancing_state(bool enabled);
188482727018SRik van Riel extern void task_numa_free(struct task_struct *p);
188510f39042SRik van Riel extern bool should_numa_migrate_memory(struct task_struct *p, struct page *page,
188610f39042SRik van Riel 					int src_nid, int dst_cpu);
1887cbee9f88SPeter Zijlstra #else
1888ac8e895bSMel Gorman static inline void task_numa_fault(int last_node, int node, int pages,
18896688cc05SPeter Zijlstra 				   int flags)
1890cbee9f88SPeter Zijlstra {
1891cbee9f88SPeter Zijlstra }
1892e29cf08bSMel Gorman static inline pid_t task_numa_group_id(struct task_struct *p)
1893e29cf08bSMel Gorman {
1894e29cf08bSMel Gorman 	return 0;
1895e29cf08bSMel Gorman }
18961a687c2eSMel Gorman static inline void set_numabalancing_state(bool enabled)
18971a687c2eSMel Gorman {
18981a687c2eSMel Gorman }
189982727018SRik van Riel static inline void task_numa_free(struct task_struct *p)
190082727018SRik van Riel {
190182727018SRik van Riel }
190210f39042SRik van Riel static inline bool should_numa_migrate_memory(struct task_struct *p,
190310f39042SRik van Riel 				struct page *page, int src_nid, int dst_cpu)
190410f39042SRik van Riel {
190510f39042SRik van Riel 	return true;
190610f39042SRik van Riel }
1907cbee9f88SPeter Zijlstra #endif
1908cbee9f88SPeter Zijlstra 
1909e868171aSAlexey Dobriyan static inline struct pid *task_pid(struct task_struct *task)
191022c935f4SEric W. Biederman {
191122c935f4SEric W. Biederman 	return task->pids[PIDTYPE_PID].pid;
191222c935f4SEric W. Biederman }
191322c935f4SEric W. Biederman 
1914e868171aSAlexey Dobriyan static inline struct pid *task_tgid(struct task_struct *task)
191522c935f4SEric W. Biederman {
191622c935f4SEric W. Biederman 	return task->group_leader->pids[PIDTYPE_PID].pid;
191722c935f4SEric W. Biederman }
191822c935f4SEric W. Biederman 
19196dda81f4SOleg Nesterov /*
19206dda81f4SOleg Nesterov  * Without tasklist or rcu lock it is not safe to dereference
19216dda81f4SOleg Nesterov  * the result of task_pgrp/task_session even if task == current,
19226dda81f4SOleg Nesterov  * we can race with another thread doing sys_setsid/sys_setpgid.
19236dda81f4SOleg Nesterov  */
1924e868171aSAlexey Dobriyan static inline struct pid *task_pgrp(struct task_struct *task)
192522c935f4SEric W. Biederman {
192622c935f4SEric W. Biederman 	return task->group_leader->pids[PIDTYPE_PGID].pid;
192722c935f4SEric W. Biederman }
192822c935f4SEric W. Biederman 
1929e868171aSAlexey Dobriyan static inline struct pid *task_session(struct task_struct *task)
193022c935f4SEric W. Biederman {
193122c935f4SEric W. Biederman 	return task->group_leader->pids[PIDTYPE_SID].pid;
193222c935f4SEric W. Biederman }
193322c935f4SEric W. Biederman 
19347af57294SPavel Emelyanov struct pid_namespace;
19357af57294SPavel Emelyanov 
19367af57294SPavel Emelyanov /*
19377af57294SPavel Emelyanov  * the helpers to get the task's different pids as they are seen
19387af57294SPavel Emelyanov  * from various namespaces
19397af57294SPavel Emelyanov  *
19407af57294SPavel Emelyanov  * task_xid_nr()     : global id, i.e. the id seen from the init namespace;
194144c4e1b2SEric W. Biederman  * task_xid_vnr()    : virtual id, i.e. the id seen from the pid namespace of
194244c4e1b2SEric W. Biederman  *                     current.
19437af57294SPavel Emelyanov  * task_xid_nr_ns()  : id seen from the ns specified;
19447af57294SPavel Emelyanov  *
19457af57294SPavel Emelyanov  * set_task_vxid()   : assigns a virtual id to a task;
19467af57294SPavel Emelyanov  *
19477af57294SPavel Emelyanov  * see also pid_nr() etc in include/linux/pid.h
19487af57294SPavel Emelyanov  */
194952ee2dfdSOleg Nesterov pid_t __task_pid_nr_ns(struct task_struct *task, enum pid_type type,
195052ee2dfdSOleg Nesterov 			struct pid_namespace *ns);
19517af57294SPavel Emelyanov 
1952e868171aSAlexey Dobriyan static inline pid_t task_pid_nr(struct task_struct *tsk)
19537af57294SPavel Emelyanov {
19547af57294SPavel Emelyanov 	return tsk->pid;
19557af57294SPavel Emelyanov }
19567af57294SPavel Emelyanov 
195752ee2dfdSOleg Nesterov static inline pid_t task_pid_nr_ns(struct task_struct *tsk,
195852ee2dfdSOleg Nesterov 					struct pid_namespace *ns)
195952ee2dfdSOleg Nesterov {
196052ee2dfdSOleg Nesterov 	return __task_pid_nr_ns(tsk, PIDTYPE_PID, ns);
196152ee2dfdSOleg Nesterov }
19627af57294SPavel Emelyanov 
19637af57294SPavel Emelyanov static inline pid_t task_pid_vnr(struct task_struct *tsk)
19647af57294SPavel Emelyanov {
196552ee2dfdSOleg Nesterov 	return __task_pid_nr_ns(tsk, PIDTYPE_PID, NULL);
19667af57294SPavel Emelyanov }
19677af57294SPavel Emelyanov 
19687af57294SPavel Emelyanov 
1969e868171aSAlexey Dobriyan static inline pid_t task_tgid_nr(struct task_struct *tsk)
19707af57294SPavel Emelyanov {
19717af57294SPavel Emelyanov 	return tsk->tgid;
19727af57294SPavel Emelyanov }
19737af57294SPavel Emelyanov 
19742f2a3a46SPavel Emelyanov pid_t task_tgid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns);
19757af57294SPavel Emelyanov 
19767af57294SPavel Emelyanov static inline pid_t task_tgid_vnr(struct task_struct *tsk)
19777af57294SPavel Emelyanov {
19787af57294SPavel Emelyanov 	return pid_vnr(task_tgid(tsk));
19797af57294SPavel Emelyanov }
19807af57294SPavel Emelyanov 
19817af57294SPavel Emelyanov 
198280e0b6e8SRichard Guy Briggs static inline int pid_alive(const struct task_struct *p);
1983ad36d282SRichard Guy Briggs static inline pid_t task_ppid_nr_ns(const struct task_struct *tsk, struct pid_namespace *ns)
1984ad36d282SRichard Guy Briggs {
1985ad36d282SRichard Guy Briggs 	pid_t pid = 0;
1986ad36d282SRichard Guy Briggs 
1987ad36d282SRichard Guy Briggs 	rcu_read_lock();
1988ad36d282SRichard Guy Briggs 	if (pid_alive(tsk))
1989ad36d282SRichard Guy Briggs 		pid = task_tgid_nr_ns(rcu_dereference(tsk->real_parent), ns);
1990ad36d282SRichard Guy Briggs 	rcu_read_unlock();
1991ad36d282SRichard Guy Briggs 
1992ad36d282SRichard Guy Briggs 	return pid;
1993ad36d282SRichard Guy Briggs }
1994ad36d282SRichard Guy Briggs 
1995ad36d282SRichard Guy Briggs static inline pid_t task_ppid_nr(const struct task_struct *tsk)
1996ad36d282SRichard Guy Briggs {
1997ad36d282SRichard Guy Briggs 	return task_ppid_nr_ns(tsk, &init_pid_ns);
1998ad36d282SRichard Guy Briggs }
1999ad36d282SRichard Guy Briggs 
200052ee2dfdSOleg Nesterov static inline pid_t task_pgrp_nr_ns(struct task_struct *tsk,
200152ee2dfdSOleg Nesterov 					struct pid_namespace *ns)
20027af57294SPavel Emelyanov {
200352ee2dfdSOleg Nesterov 	return __task_pid_nr_ns(tsk, PIDTYPE_PGID, ns);
20047af57294SPavel Emelyanov }
20057af57294SPavel Emelyanov 
20067af57294SPavel Emelyanov static inline pid_t task_pgrp_vnr(struct task_struct *tsk)
20077af57294SPavel Emelyanov {
200852ee2dfdSOleg Nesterov 	return __task_pid_nr_ns(tsk, PIDTYPE_PGID, NULL);
20097af57294SPavel Emelyanov }
20107af57294SPavel Emelyanov 
20117af57294SPavel Emelyanov 
201252ee2dfdSOleg Nesterov static inline pid_t task_session_nr_ns(struct task_struct *tsk,
201352ee2dfdSOleg Nesterov 					struct pid_namespace *ns)
20147af57294SPavel Emelyanov {
201552ee2dfdSOleg Nesterov 	return __task_pid_nr_ns(tsk, PIDTYPE_SID, ns);
20167af57294SPavel Emelyanov }
20177af57294SPavel Emelyanov 
20187af57294SPavel Emelyanov static inline pid_t task_session_vnr(struct task_struct *tsk)
20197af57294SPavel Emelyanov {
202052ee2dfdSOleg Nesterov 	return __task_pid_nr_ns(tsk, PIDTYPE_SID, NULL);
20217af57294SPavel Emelyanov }
20227af57294SPavel Emelyanov 
20231b0f7ffdSOleg Nesterov /* obsolete, do not use */
20241b0f7ffdSOleg Nesterov static inline pid_t task_pgrp_nr(struct task_struct *tsk)
20251b0f7ffdSOleg Nesterov {
20261b0f7ffdSOleg Nesterov 	return task_pgrp_nr_ns(tsk, &init_pid_ns);
20271b0f7ffdSOleg Nesterov }
20287af57294SPavel Emelyanov 
20291da177e4SLinus Torvalds /**
20301da177e4SLinus Torvalds  * pid_alive - check that a task structure is not stale
20311da177e4SLinus Torvalds  * @p: Task structure to be checked.
20321da177e4SLinus Torvalds  *
20331da177e4SLinus Torvalds  * Test if a process is not yet dead (at most zombie state)
20341da177e4SLinus Torvalds  * If pid_alive fails, then pointers within the task structure
20351da177e4SLinus Torvalds  * can be stale and must not be dereferenced.
2036e69f6186SYacine Belkadi  *
2037e69f6186SYacine Belkadi  * Return: 1 if the process is alive. 0 otherwise.
20381da177e4SLinus Torvalds  */
2039ad36d282SRichard Guy Briggs static inline int pid_alive(const struct task_struct *p)
20401da177e4SLinus Torvalds {
204192476d7fSEric W. Biederman 	return p->pids[PIDTYPE_PID].pid != NULL;
20421da177e4SLinus Torvalds }
20431da177e4SLinus Torvalds 
2044f400e198SSukadev Bhattiprolu /**
2045570f5241SSergey Senozhatsky  * is_global_init - check if a task structure is init. Since init
2046570f5241SSergey Senozhatsky  * is free to have sub-threads we need to check tgid.
20473260259fSHenne  * @tsk: Task structure to be checked.
20483260259fSHenne  *
20493260259fSHenne  * Check if a task structure is the first user space task the kernel created.
2050e69f6186SYacine Belkadi  *
2051e69f6186SYacine Belkadi  * Return: 1 if the task structure is init. 0 otherwise.
2052f400e198SSukadev Bhattiprolu  */
2053e868171aSAlexey Dobriyan static inline int is_global_init(struct task_struct *tsk)
2054b461cc03SPavel Emelyanov {
2055570f5241SSergey Senozhatsky 	return task_tgid_nr(tsk) == 1;
2056b461cc03SPavel Emelyanov }
2057b460cbc5SSerge E. Hallyn 
20589ec52099SCedric Le Goater extern struct pid *cad_pid;
20599ec52099SCedric Le Goater 
20601da177e4SLinus Torvalds extern void free_task(struct task_struct *tsk);
20611da177e4SLinus Torvalds #define get_task_struct(tsk) do { atomic_inc(&(tsk)->usage); } while(0)
2062e56d0903SIngo Molnar 
2063158d9ebdSAndrew Morton extern void __put_task_struct(struct task_struct *t);
2064e56d0903SIngo Molnar 
2065e56d0903SIngo Molnar static inline void put_task_struct(struct task_struct *t)
2066e56d0903SIngo Molnar {
2067e56d0903SIngo Molnar 	if (atomic_dec_and_test(&t->usage))
20688c7904a0SEric W. Biederman 		__put_task_struct(t);
2069e56d0903SIngo Molnar }
20701da177e4SLinus Torvalds 
20716a61671bSFrederic Weisbecker #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
20726a61671bSFrederic Weisbecker extern void task_cputime(struct task_struct *t,
20736a61671bSFrederic Weisbecker 			 cputime_t *utime, cputime_t *stime);
20746a61671bSFrederic Weisbecker extern void task_cputime_scaled(struct task_struct *t,
20756a61671bSFrederic Weisbecker 				cputime_t *utimescaled, cputime_t *stimescaled);
20766a61671bSFrederic Weisbecker extern cputime_t task_gtime(struct task_struct *t);
20776a61671bSFrederic Weisbecker #else
20786fac4829SFrederic Weisbecker static inline void task_cputime(struct task_struct *t,
20796fac4829SFrederic Weisbecker 				cputime_t *utime, cputime_t *stime)
20806fac4829SFrederic Weisbecker {
20816fac4829SFrederic Weisbecker 	if (utime)
20826fac4829SFrederic Weisbecker 		*utime = t->utime;
20836fac4829SFrederic Weisbecker 	if (stime)
20846fac4829SFrederic Weisbecker 		*stime = t->stime;
20856fac4829SFrederic Weisbecker }
20866fac4829SFrederic Weisbecker 
20876fac4829SFrederic Weisbecker static inline void task_cputime_scaled(struct task_struct *t,
20886fac4829SFrederic Weisbecker 				       cputime_t *utimescaled,
20896fac4829SFrederic Weisbecker 				       cputime_t *stimescaled)
20906fac4829SFrederic Weisbecker {
20916fac4829SFrederic Weisbecker 	if (utimescaled)
20926fac4829SFrederic Weisbecker 		*utimescaled = t->utimescaled;
20936fac4829SFrederic Weisbecker 	if (stimescaled)
20946fac4829SFrederic Weisbecker 		*stimescaled = t->stimescaled;
20956fac4829SFrederic Weisbecker }
20966a61671bSFrederic Weisbecker 
20976a61671bSFrederic Weisbecker static inline cputime_t task_gtime(struct task_struct *t)
20986a61671bSFrederic Weisbecker {
20996a61671bSFrederic Weisbecker 	return t->gtime;
21006a61671bSFrederic Weisbecker }
21016a61671bSFrederic Weisbecker #endif
2102e80d0a1aSFrederic Weisbecker extern void task_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime_t *st);
2103e80d0a1aSFrederic Weisbecker extern void thread_group_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime_t *st);
210449048622SBalbir Singh 
21051da177e4SLinus Torvalds /*
21061da177e4SLinus Torvalds  * Per process flags
21071da177e4SLinus Torvalds  */
21081da177e4SLinus Torvalds #define PF_EXITING	0x00000004	/* getting shut down */
2109778e9a9cSAlexey Kuznetsov #define PF_EXITPIDONE	0x00000008	/* pi exit done on shut down */
211094886b84SLaurent Vivier #define PF_VCPU		0x00000010	/* I'm a virtual CPU */
211121aa9af0STejun Heo #define PF_WQ_WORKER	0x00000020	/* I'm a workqueue worker */
21121da177e4SLinus Torvalds #define PF_FORKNOEXEC	0x00000040	/* forked but didn't exec */
21134db96cf0SAndi Kleen #define PF_MCE_PROCESS  0x00000080      /* process policy on mce errors */
21141da177e4SLinus Torvalds #define PF_SUPERPRIV	0x00000100	/* used super-user privileges */
21151da177e4SLinus Torvalds #define PF_DUMPCORE	0x00000200	/* dumped core */
21161da177e4SLinus Torvalds #define PF_SIGNALED	0x00000400	/* killed by a signal */
21171da177e4SLinus Torvalds #define PF_MEMALLOC	0x00000800	/* Allocating memory */
211872fa5997SVasiliy Kulikov #define PF_NPROC_EXCEEDED 0x00001000	/* set_user noticed that RLIMIT_NPROC was exceeded */
21191da177e4SLinus Torvalds #define PF_USED_MATH	0x00002000	/* if unset the fpu must be initialized before use */
2120774a1221STejun Heo #define PF_USED_ASYNC	0x00004000	/* used async_schedule*(), used by module init */
21211da177e4SLinus Torvalds #define PF_NOFREEZE	0x00008000	/* this thread should not be frozen */
21221da177e4SLinus Torvalds #define PF_FROZEN	0x00010000	/* frozen for system suspend */
21231da177e4SLinus Torvalds #define PF_FSTRANS	0x00020000	/* inside a filesystem transaction */
21241da177e4SLinus Torvalds #define PF_KSWAPD	0x00040000	/* I am kswapd */
212521caf2fcSMing Lei #define PF_MEMALLOC_NOIO 0x00080000	/* Allocating memory without IO involved */
21261da177e4SLinus Torvalds #define PF_LESS_THROTTLE 0x00100000	/* Throttle me less: I clean memory */
2127246bb0b1SOleg Nesterov #define PF_KTHREAD	0x00200000	/* I am a kernel thread */
2128b31dc66aSJens Axboe #define PF_RANDOMIZE	0x00400000	/* randomize virtual address space */
2129b31dc66aSJens Axboe #define PF_SWAPWRITE	0x00800000	/* Allowed to write to swap */
213014a40ffcSTejun Heo #define PF_NO_SETAFFINITY 0x04000000	/* Userland is not allowed to meddle with cpus_allowed */
21314db96cf0SAndi Kleen #define PF_MCE_EARLY    0x08000000      /* Early kill for mce process policy */
213261a87122SThomas Gleixner #define PF_MUTEX_TESTER	0x20000000	/* Thread belongs to the rt mutex tester */
213358a69cb4STejun Heo #define PF_FREEZER_SKIP	0x40000000	/* Freezer should not count it as freezable */
21342b44c4dbSColin Cross #define PF_SUSPEND_TASK 0x80000000      /* this thread called freeze_processes and should not be frozen */
21351da177e4SLinus Torvalds 
21361da177e4SLinus Torvalds /*
21371da177e4SLinus Torvalds  * Only the _current_ task can read/write to tsk->flags, but other
21381da177e4SLinus Torvalds  * tasks can access tsk->flags in readonly mode for example
21391da177e4SLinus Torvalds  * with tsk_used_math (like during threaded core dumping).
21401da177e4SLinus Torvalds  * There is however an exception to this rule during ptrace
21411da177e4SLinus Torvalds  * or during fork: the ptracer task is allowed to write to the
21421da177e4SLinus Torvalds  * child->flags of its traced child (same goes for fork, the parent
21431da177e4SLinus Torvalds  * can write to the child->flags), because we're guaranteed the
21441da177e4SLinus Torvalds  * child is not running and in turn not changing child->flags
21451da177e4SLinus Torvalds  * at the same time the parent does it.
21461da177e4SLinus Torvalds  */
21471da177e4SLinus Torvalds #define clear_stopped_child_used_math(child) do { (child)->flags &= ~PF_USED_MATH; } while (0)
21481da177e4SLinus Torvalds #define set_stopped_child_used_math(child) do { (child)->flags |= PF_USED_MATH; } while (0)
21491da177e4SLinus Torvalds #define clear_used_math() clear_stopped_child_used_math(current)
21501da177e4SLinus Torvalds #define set_used_math() set_stopped_child_used_math(current)
21511da177e4SLinus Torvalds #define conditional_stopped_child_used_math(condition, child) \
21521da177e4SLinus Torvalds 	do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= (condition) ? PF_USED_MATH : 0; } while (0)
21531da177e4SLinus Torvalds #define conditional_used_math(condition) \
21541da177e4SLinus Torvalds 	conditional_stopped_child_used_math(condition, current)
21551da177e4SLinus Torvalds #define copy_to_stopped_child_used_math(child) \
21561da177e4SLinus Torvalds 	do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= current->flags & PF_USED_MATH; } while (0)
21571da177e4SLinus Torvalds /* NOTE: this will return 0 or PF_USED_MATH, it will never return 1 */
21581da177e4SLinus Torvalds #define tsk_used_math(p) ((p)->flags & PF_USED_MATH)
21591da177e4SLinus Torvalds #define used_math() tsk_used_math(current)
21601da177e4SLinus Torvalds 
2161934f3072SJunxiao Bi /* __GFP_IO isn't allowed if PF_MEMALLOC_NOIO is set in current->flags
2162934f3072SJunxiao Bi  * __GFP_FS is also cleared as it implies __GFP_IO.
2163934f3072SJunxiao Bi  */
216421caf2fcSMing Lei static inline gfp_t memalloc_noio_flags(gfp_t flags)
216521caf2fcSMing Lei {
216621caf2fcSMing Lei 	if (unlikely(current->flags & PF_MEMALLOC_NOIO))
2167934f3072SJunxiao Bi 		flags &= ~(__GFP_IO | __GFP_FS);
216821caf2fcSMing Lei 	return flags;
216921caf2fcSMing Lei }
217021caf2fcSMing Lei 
217121caf2fcSMing Lei static inline unsigned int memalloc_noio_save(void)
217221caf2fcSMing Lei {
217321caf2fcSMing Lei 	unsigned int flags = current->flags & PF_MEMALLOC_NOIO;
217421caf2fcSMing Lei 	current->flags |= PF_MEMALLOC_NOIO;
217521caf2fcSMing Lei 	return flags;
217621caf2fcSMing Lei }
217721caf2fcSMing Lei 
217821caf2fcSMing Lei static inline void memalloc_noio_restore(unsigned int flags)
217921caf2fcSMing Lei {
218021caf2fcSMing Lei 	current->flags = (current->flags & ~PF_MEMALLOC_NOIO) | flags;
218121caf2fcSMing Lei }
218221caf2fcSMing Lei 
21831d4457f9SKees Cook /* Per-process atomic flags. */
2184a2b86f77SZefan Li #define PFA_NO_NEW_PRIVS 0	/* May not gain new privileges. */
21852ad654bcSZefan Li #define PFA_SPREAD_PAGE  1      /* Spread page cache over cpuset */
21862ad654bcSZefan Li #define PFA_SPREAD_SLAB  2      /* Spread some slab caches over cpuset */
21871d4457f9SKees Cook 
21881d4457f9SKees Cook 
2189e0e5070bSZefan Li #define TASK_PFA_TEST(name, func)					\
2190e0e5070bSZefan Li 	static inline bool task_##func(struct task_struct *p)		\
2191e0e5070bSZefan Li 	{ return test_bit(PFA_##name, &p->atomic_flags); }
2192e0e5070bSZefan Li #define TASK_PFA_SET(name, func)					\
2193e0e5070bSZefan Li 	static inline void task_set_##func(struct task_struct *p)	\
2194e0e5070bSZefan Li 	{ set_bit(PFA_##name, &p->atomic_flags); }
2195e0e5070bSZefan Li #define TASK_PFA_CLEAR(name, func)					\
2196e0e5070bSZefan Li 	static inline void task_clear_##func(struct task_struct *p)	\
2197e0e5070bSZefan Li 	{ clear_bit(PFA_##name, &p->atomic_flags); }
21981d4457f9SKees Cook 
2199e0e5070bSZefan Li TASK_PFA_TEST(NO_NEW_PRIVS, no_new_privs)
2200e0e5070bSZefan Li TASK_PFA_SET(NO_NEW_PRIVS, no_new_privs)
22011d4457f9SKees Cook 
22022ad654bcSZefan Li TASK_PFA_TEST(SPREAD_PAGE, spread_page)
22032ad654bcSZefan Li TASK_PFA_SET(SPREAD_PAGE, spread_page)
22042ad654bcSZefan Li TASK_PFA_CLEAR(SPREAD_PAGE, spread_page)
22052ad654bcSZefan Li 
22062ad654bcSZefan Li TASK_PFA_TEST(SPREAD_SLAB, spread_slab)
22072ad654bcSZefan Li TASK_PFA_SET(SPREAD_SLAB, spread_slab)
22082ad654bcSZefan Li TASK_PFA_CLEAR(SPREAD_SLAB, spread_slab)
2209544b2c91STejun Heo 
2210a8f072c1STejun Heo /*
2211a8f072c1STejun Heo  * task->jobctl flags
2212a8f072c1STejun Heo  */
2213a8f072c1STejun Heo #define JOBCTL_STOP_SIGMASK	0xffff	/* signr of the last group stop */
221473ddff2bSTejun Heo 
2215fb1d910cSTejun Heo #define JOBCTL_STOP_DEQUEUED_BIT 16	/* stop signal dequeued */
2216a8f072c1STejun Heo #define JOBCTL_STOP_PENDING_BIT	17	/* task should stop for group stop */
2217544b2c91STejun Heo #define JOBCTL_STOP_CONSUME_BIT	18	/* consume group stop count */
2218a8f072c1STejun Heo #define JOBCTL_TRAP_STOP_BIT	19	/* trap for STOP */
2219fb1d910cSTejun Heo #define JOBCTL_TRAP_NOTIFY_BIT	20	/* trap for NOTIFY */
222073ddff2bSTejun Heo #define JOBCTL_TRAPPING_BIT	21	/* switching to TRACED */
22213759a0d9STejun Heo #define JOBCTL_LISTENING_BIT	22	/* ptracer is listening for events */
22227dd3db54STejun Heo 
2223b76808e6SPalmer Dabbelt #define JOBCTL_STOP_DEQUEUED	(1UL << JOBCTL_STOP_DEQUEUED_BIT)
2224b76808e6SPalmer Dabbelt #define JOBCTL_STOP_PENDING	(1UL << JOBCTL_STOP_PENDING_BIT)
2225b76808e6SPalmer Dabbelt #define JOBCTL_STOP_CONSUME	(1UL << JOBCTL_STOP_CONSUME_BIT)
2226b76808e6SPalmer Dabbelt #define JOBCTL_TRAP_STOP	(1UL << JOBCTL_TRAP_STOP_BIT)
2227b76808e6SPalmer Dabbelt #define JOBCTL_TRAP_NOTIFY	(1UL << JOBCTL_TRAP_NOTIFY_BIT)
2228b76808e6SPalmer Dabbelt #define JOBCTL_TRAPPING		(1UL << JOBCTL_TRAPPING_BIT)
2229b76808e6SPalmer Dabbelt #define JOBCTL_LISTENING	(1UL << JOBCTL_LISTENING_BIT)
22303759a0d9STejun Heo 
22313759a0d9STejun Heo #define JOBCTL_TRAP_MASK	(JOBCTL_TRAP_STOP | JOBCTL_TRAP_NOTIFY)
22327dd3db54STejun Heo #define JOBCTL_PENDING_MASK	(JOBCTL_STOP_PENDING | JOBCTL_TRAP_MASK)
22337dd3db54STejun Heo 
223473ddff2bSTejun Heo extern bool task_set_jobctl_pending(struct task_struct *task,
2235b76808e6SPalmer Dabbelt 				    unsigned long mask);
2236a57eb940SPaul E. McKenney extern void task_clear_jobctl_trapping(struct task_struct *task);
2237f41d911fSPaul E. McKenney extern void task_clear_jobctl_pending(struct task_struct *task,
2238b76808e6SPalmer Dabbelt 				      unsigned long mask);
2239a57eb940SPaul E. McKenney 
2240f41d911fSPaul E. McKenney static inline void rcu_copy_process(struct task_struct *p)
2241f41d911fSPaul E. McKenney {
22421da177e4SLinus Torvalds #ifdef CONFIG_PREEMPT_RCU
2243f41d911fSPaul E. McKenney 	p->rcu_read_lock_nesting = 0;
22441d082fd0SPaul E. McKenney 	p->rcu_read_unlock_special.s = 0;
2245dd5d19baSPaul E. McKenney 	p->rcu_blocked_node = NULL;
2246f41d911fSPaul E. McKenney 	INIT_LIST_HEAD(&p->rcu_node_entry);
22478315f422SPaul E. McKenney #endif /* #ifdef CONFIG_PREEMPT_RCU */
22488315f422SPaul E. McKenney #ifdef CONFIG_TASKS_RCU
22498315f422SPaul E. McKenney 	p->rcu_tasks_holdout = false;
22508315f422SPaul E. McKenney 	INIT_LIST_HEAD(&p->rcu_tasks_holdout_list);
2251176f8f7aSPaul E. McKenney 	p->rcu_tasks_idle_cpu = -1;
22528315f422SPaul E. McKenney #endif /* #ifdef CONFIG_TASKS_RCU */
2253f41d911fSPaul E. McKenney }
2254f41d911fSPaul E. McKenney 
2255907aed48SMel Gorman static inline void tsk_restore_flags(struct task_struct *task,
2256907aed48SMel Gorman 				unsigned long orig_flags, unsigned long flags)
2257907aed48SMel Gorman {
2258907aed48SMel Gorman 	task->flags &= ~flags;
2259907aed48SMel Gorman 	task->flags |= orig_flags & flags;
2260907aed48SMel Gorman }
2261907aed48SMel Gorman 
2262f82f8042SJuri Lelli extern int cpuset_cpumask_can_shrink(const struct cpumask *cur,
2263f82f8042SJuri Lelli 				     const struct cpumask *trial);
22647f51412aSJuri Lelli extern int task_can_attach(struct task_struct *p,
22657f51412aSJuri Lelli 			   const struct cpumask *cs_cpus_allowed);
22661da177e4SLinus Torvalds #ifdef CONFIG_SMP
22671e1b6c51SKOSAKI Motohiro extern void do_set_cpus_allowed(struct task_struct *p,
22681e1b6c51SKOSAKI Motohiro 			       const struct cpumask *new_mask);
22691e1b6c51SKOSAKI Motohiro 
2270cd8ba7cdSMike Travis extern int set_cpus_allowed_ptr(struct task_struct *p,
227196f874e2SRusty Russell 				const struct cpumask *new_mask);
22721da177e4SLinus Torvalds #else
22731e1b6c51SKOSAKI Motohiro static inline void do_set_cpus_allowed(struct task_struct *p,
22741e1b6c51SKOSAKI Motohiro 				      const struct cpumask *new_mask)
22751e1b6c51SKOSAKI Motohiro {
22761e1b6c51SKOSAKI Motohiro }
2277cd8ba7cdSMike Travis static inline int set_cpus_allowed_ptr(struct task_struct *p,
227896f874e2SRusty Russell 				       const struct cpumask *new_mask)
22791da177e4SLinus Torvalds {
228096f874e2SRusty Russell 	if (!cpumask_test_cpu(0, new_mask))
22811da177e4SLinus Torvalds 		return -EINVAL;
22821da177e4SLinus Torvalds 	return 0;
22831da177e4SLinus Torvalds }
22841da177e4SLinus Torvalds #endif
2285e0ad9556SRusty Russell 
22863451d024SFrederic Weisbecker #ifdef CONFIG_NO_HZ_COMMON
22875167e8d5SPeter Zijlstra void calc_load_enter_idle(void);
22885167e8d5SPeter Zijlstra void calc_load_exit_idle(void);
22895167e8d5SPeter Zijlstra #else
22905167e8d5SPeter Zijlstra static inline void calc_load_enter_idle(void) { }
22915167e8d5SPeter Zijlstra static inline void calc_load_exit_idle(void) { }
22923451d024SFrederic Weisbecker #endif /* CONFIG_NO_HZ_COMMON */
22935167e8d5SPeter Zijlstra 
2294b342501cSIngo Molnar /*
2295c676329aSPeter Zijlstra  * Do not use outside of architecture code which knows its limitations.
2296c676329aSPeter Zijlstra  *
2297c676329aSPeter Zijlstra  * sched_clock() has no promise of monotonicity or bounded drift between
2298c676329aSPeter Zijlstra  * CPUs, use (which you should not) requires disabling IRQs.
2299c676329aSPeter Zijlstra  *
2300c676329aSPeter Zijlstra  * Please use one of the three interfaces below.
2301b342501cSIngo Molnar  */
23021bbfa6f2SMike Frysinger extern unsigned long long notrace sched_clock(void);
2303c676329aSPeter Zijlstra /*
2304489a71b0SHiroshi Shimamoto  * See the comment in kernel/sched/clock.c
2305c676329aSPeter Zijlstra  */
2306c676329aSPeter Zijlstra extern u64 cpu_clock(int cpu);
2307c676329aSPeter Zijlstra extern u64 local_clock(void);
2308545a2bf7SCyril Bur extern u64 running_clock(void);
2309c676329aSPeter Zijlstra extern u64 sched_clock_cpu(int cpu);
2310c676329aSPeter Zijlstra 
2311e436d800SIngo Molnar 
2312c1955a3dSPeter Zijlstra extern void sched_clock_init(void);
2313c1955a3dSPeter Zijlstra 
23143e51f33fSPeter Zijlstra #ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
23153e51f33fSPeter Zijlstra static inline void sched_clock_tick(void)
23163e51f33fSPeter Zijlstra {
23173e51f33fSPeter Zijlstra }
23183e51f33fSPeter Zijlstra 
23193e51f33fSPeter Zijlstra static inline void sched_clock_idle_sleep_event(void)
23203e51f33fSPeter Zijlstra {
23213e51f33fSPeter Zijlstra }
23223e51f33fSPeter Zijlstra 
23233e51f33fSPeter Zijlstra static inline void sched_clock_idle_wakeup_event(u64 delta_ns)
23243e51f33fSPeter Zijlstra {
23253e51f33fSPeter Zijlstra }
23263e51f33fSPeter Zijlstra #else
2327c676329aSPeter Zijlstra /*
2328c676329aSPeter Zijlstra  * Architectures can set this to 1 if they have specified
2329c676329aSPeter Zijlstra  * CONFIG_HAVE_UNSTABLE_SCHED_CLOCK in their arch Kconfig,
2330c676329aSPeter Zijlstra  * but then during bootup it turns out that sched_clock()
2331c676329aSPeter Zijlstra  * is reliable after all:
2332c676329aSPeter Zijlstra  */
233335af99e6SPeter Zijlstra extern int sched_clock_stable(void);
233435af99e6SPeter Zijlstra extern void set_sched_clock_stable(void);
233535af99e6SPeter Zijlstra extern void clear_sched_clock_stable(void);
2336c676329aSPeter Zijlstra 
23373e51f33fSPeter Zijlstra extern void sched_clock_tick(void);
23383e51f33fSPeter Zijlstra extern void sched_clock_idle_sleep_event(void);
23393e51f33fSPeter Zijlstra extern void sched_clock_idle_wakeup_event(u64 delta_ns);
23403e51f33fSPeter Zijlstra #endif
23413e51f33fSPeter Zijlstra 
2342b52bfee4SVenkatesh Pallipadi #ifdef CONFIG_IRQ_TIME_ACCOUNTING
2343b52bfee4SVenkatesh Pallipadi /*
2344b52bfee4SVenkatesh Pallipadi  * An i/f to runtime opt-in for irq time accounting based off of sched_clock.
2345b52bfee4SVenkatesh Pallipadi  * The reason for this explicit opt-in is not to have perf penalty with
2346b52bfee4SVenkatesh Pallipadi  * slow sched_clocks.
2347b52bfee4SVenkatesh Pallipadi  */
2348b52bfee4SVenkatesh Pallipadi extern void enable_sched_clock_irqtime(void);
2349b52bfee4SVenkatesh Pallipadi extern void disable_sched_clock_irqtime(void);
2350b52bfee4SVenkatesh Pallipadi #else
2351b52bfee4SVenkatesh Pallipadi static inline void enable_sched_clock_irqtime(void) {}
2352b52bfee4SVenkatesh Pallipadi static inline void disable_sched_clock_irqtime(void) {}
2353b52bfee4SVenkatesh Pallipadi #endif
2354b52bfee4SVenkatesh Pallipadi 
235536c8b586SIngo Molnar extern unsigned long long
235641b86e9cSIngo Molnar task_sched_runtime(struct task_struct *task);
23571da177e4SLinus Torvalds 
23581da177e4SLinus Torvalds /* sched_exec is called by processes performing an exec */
23591da177e4SLinus Torvalds #ifdef CONFIG_SMP
23601da177e4SLinus Torvalds extern void sched_exec(void);
23611da177e4SLinus Torvalds #else
23621da177e4SLinus Torvalds #define sched_exec()   {}
23631da177e4SLinus Torvalds #endif
23641da177e4SLinus Torvalds 
23652aa44d05SIngo Molnar extern void sched_clock_idle_sleep_event(void);
23662aa44d05SIngo Molnar extern void sched_clock_idle_wakeup_event(u64 delta_ns);
2367bb29ab26SIngo Molnar 
23681da177e4SLinus Torvalds #ifdef CONFIG_HOTPLUG_CPU
23691da177e4SLinus Torvalds extern void idle_task_exit(void);
23701da177e4SLinus Torvalds #else
23711da177e4SLinus Torvalds static inline void idle_task_exit(void) {}
23721da177e4SLinus Torvalds #endif
23731da177e4SLinus Torvalds 
23743451d024SFrederic Weisbecker #if defined(CONFIG_NO_HZ_COMMON) && defined(CONFIG_SMP)
23751c20091eSFrederic Weisbecker extern void wake_up_nohz_cpu(int cpu);
237606d8308cSThomas Gleixner #else
23771c20091eSFrederic Weisbecker static inline void wake_up_nohz_cpu(int cpu) { }
237806d8308cSThomas Gleixner #endif
237906d8308cSThomas Gleixner 
2380ce831b38SFrederic Weisbecker #ifdef CONFIG_NO_HZ_FULL
2381265f22a9SFrederic Weisbecker extern u64 scheduler_tick_max_deferment(void);
2382bf0f6f24SIngo Molnar #endif
2383bf0f6f24SIngo Molnar 
23845091faa4SMike Galbraith #ifdef CONFIG_SCHED_AUTOGROUP
23855091faa4SMike Galbraith extern void sched_autogroup_create_attach(struct task_struct *p);
23865091faa4SMike Galbraith extern void sched_autogroup_detach(struct task_struct *p);
23875091faa4SMike Galbraith extern void sched_autogroup_fork(struct signal_struct *sig);
23885091faa4SMike Galbraith extern void sched_autogroup_exit(struct signal_struct *sig);
23895091faa4SMike Galbraith #ifdef CONFIG_PROC_FS
23905091faa4SMike Galbraith extern void proc_sched_autogroup_show_task(struct task_struct *p, struct seq_file *m);
23912e5b5b3aSHiroshi Shimamoto extern int proc_sched_autogroup_set_nice(struct task_struct *p, int nice);
23925091faa4SMike Galbraith #endif
23935091faa4SMike Galbraith #else
23945091faa4SMike Galbraith static inline void sched_autogroup_create_attach(struct task_struct *p) { }
23955091faa4SMike Galbraith static inline void sched_autogroup_detach(struct task_struct *p) { }
23965091faa4SMike Galbraith static inline void sched_autogroup_fork(struct signal_struct *sig) { }
23975091faa4SMike Galbraith static inline void sched_autogroup_exit(struct signal_struct *sig) { }
23985091faa4SMike Galbraith #endif
23995091faa4SMike Galbraith 
2400fa93384fSDan Carpenter extern int yield_to(struct task_struct *p, bool preempt);
240136c8b586SIngo Molnar extern void set_user_nice(struct task_struct *p, long nice);
240236c8b586SIngo Molnar extern int task_prio(const struct task_struct *p);
2403d0ea0268SDongsheng Yang /**
2404d0ea0268SDongsheng Yang  * task_nice - return the nice value of a given task.
2405d0ea0268SDongsheng Yang  * @p: the task in question.
2406d0ea0268SDongsheng Yang  *
2407d0ea0268SDongsheng Yang  * Return: The nice value [ -20 ... 0 ... 19 ].
2408d0ea0268SDongsheng Yang  */
2409d0ea0268SDongsheng Yang static inline int task_nice(const struct task_struct *p)
2410d0ea0268SDongsheng Yang {
2411d0ea0268SDongsheng Yang 	return PRIO_TO_NICE((p)->static_prio);
2412d0ea0268SDongsheng Yang }
241336c8b586SIngo Molnar extern int can_nice(const struct task_struct *p, const int nice);
241436c8b586SIngo Molnar extern int task_curr(const struct task_struct *p);
24151da177e4SLinus Torvalds extern int idle_cpu(int cpu);
2416fe7de49fSKOSAKI Motohiro extern int sched_setscheduler(struct task_struct *, int,
2417fe7de49fSKOSAKI Motohiro 			      const struct sched_param *);
2418961ccdddSRusty Russell extern int sched_setscheduler_nocheck(struct task_struct *, int,
2419fe7de49fSKOSAKI Motohiro 				      const struct sched_param *);
2420d50dde5aSDario Faggioli extern int sched_setattr(struct task_struct *,
2421d50dde5aSDario Faggioli 			 const struct sched_attr *);
242236c8b586SIngo Molnar extern struct task_struct *idle_task(int cpu);
2423c4f30608SPaul E. McKenney /**
2424c4f30608SPaul E. McKenney  * is_idle_task - is the specified task an idle task?
2425fa757281SRandy Dunlap  * @p: the task in question.
2426e69f6186SYacine Belkadi  *
2427e69f6186SYacine Belkadi  * Return: 1 if @p is an idle task. 0 otherwise.
2428c4f30608SPaul E. McKenney  */
24297061ca3bSPaul E. McKenney static inline bool is_idle_task(const struct task_struct *p)
2430c4f30608SPaul E. McKenney {
2431c4f30608SPaul E. McKenney 	return p->pid == 0;
2432c4f30608SPaul E. McKenney }
243336c8b586SIngo Molnar extern struct task_struct *curr_task(int cpu);
243436c8b586SIngo Molnar extern void set_curr_task(int cpu, struct task_struct *p);
24351da177e4SLinus Torvalds 
24361da177e4SLinus Torvalds void yield(void);
24371da177e4SLinus Torvalds 
24381da177e4SLinus Torvalds union thread_union {
24391da177e4SLinus Torvalds 	struct thread_info thread_info;
24401da177e4SLinus Torvalds 	unsigned long stack[THREAD_SIZE/sizeof(long)];
24411da177e4SLinus Torvalds };
24421da177e4SLinus Torvalds 
24431da177e4SLinus Torvalds #ifndef __HAVE_ARCH_KSTACK_END
24441da177e4SLinus Torvalds static inline int kstack_end(void *addr)
24451da177e4SLinus Torvalds {
24461da177e4SLinus Torvalds 	/* Reliable end of stack detection:
24471da177e4SLinus Torvalds 	 * Some APM bios versions misalign the stack
24481da177e4SLinus Torvalds 	 */
24491da177e4SLinus Torvalds 	return !(((unsigned long)addr+sizeof(void*)-1) & (THREAD_SIZE-sizeof(void*)));
24501da177e4SLinus Torvalds }
24511da177e4SLinus Torvalds #endif
24521da177e4SLinus Torvalds 
24531da177e4SLinus Torvalds extern union thread_union init_thread_union;
24541da177e4SLinus Torvalds extern struct task_struct init_task;
24551da177e4SLinus Torvalds 
24561da177e4SLinus Torvalds extern struct   mm_struct init_mm;
24571da177e4SLinus Torvalds 
2458198fe21bSPavel Emelyanov extern struct pid_namespace init_pid_ns;
2459198fe21bSPavel Emelyanov 
2460198fe21bSPavel Emelyanov /*
2461198fe21bSPavel Emelyanov  * find a task by one of its numerical ids
2462198fe21bSPavel Emelyanov  *
2463198fe21bSPavel Emelyanov  * find_task_by_pid_ns():
2464198fe21bSPavel Emelyanov  *      finds a task by its pid in the specified namespace
2465228ebcbeSPavel Emelyanov  * find_task_by_vpid():
2466228ebcbeSPavel Emelyanov  *      finds a task by its virtual pid
2467198fe21bSPavel Emelyanov  *
2468e49859e7SPavel Emelyanov  * see also find_vpid() etc in include/linux/pid.h
2469198fe21bSPavel Emelyanov  */
2470198fe21bSPavel Emelyanov 
2471228ebcbeSPavel Emelyanov extern struct task_struct *find_task_by_vpid(pid_t nr);
2472228ebcbeSPavel Emelyanov extern struct task_struct *find_task_by_pid_ns(pid_t nr,
2473228ebcbeSPavel Emelyanov 		struct pid_namespace *ns);
2474198fe21bSPavel Emelyanov 
24751da177e4SLinus Torvalds /* per-UID process charging. */
24767b44ab97SEric W. Biederman extern struct user_struct * alloc_uid(kuid_t);
24771da177e4SLinus Torvalds static inline struct user_struct *get_uid(struct user_struct *u)
24781da177e4SLinus Torvalds {
24791da177e4SLinus Torvalds 	atomic_inc(&u->__count);
24801da177e4SLinus Torvalds 	return u;
24811da177e4SLinus Torvalds }
24821da177e4SLinus Torvalds extern void free_uid(struct user_struct *);
24831da177e4SLinus Torvalds 
24841da177e4SLinus Torvalds #include <asm/current.h>
24851da177e4SLinus Torvalds 
2486f0af911aSTorben Hohn extern void xtime_update(unsigned long ticks);
24871da177e4SLinus Torvalds 
2488b3c97528SHarvey Harrison extern int wake_up_state(struct task_struct *tsk, unsigned int state);
2489b3c97528SHarvey Harrison extern int wake_up_process(struct task_struct *tsk);
24903e51e3edSSamir Bellabes extern void wake_up_new_task(struct task_struct *tsk);
24911da177e4SLinus Torvalds #ifdef CONFIG_SMP
24921da177e4SLinus Torvalds  extern void kick_process(struct task_struct *tsk);
24931da177e4SLinus Torvalds #else
24941da177e4SLinus Torvalds  static inline void kick_process(struct task_struct *tsk) { }
24951da177e4SLinus Torvalds #endif
2496aab03e05SDario Faggioli extern int sched_fork(unsigned long clone_flags, struct task_struct *p);
2497ad46c2c4SIngo Molnar extern void sched_dead(struct task_struct *p);
24981da177e4SLinus Torvalds 
24991da177e4SLinus Torvalds extern void proc_caches_init(void);
25001da177e4SLinus Torvalds extern void flush_signals(struct task_struct *);
250110ab825bSOleg Nesterov extern void ignore_signals(struct task_struct *);
25021da177e4SLinus Torvalds extern void flush_signal_handlers(struct task_struct *, int force_default);
25031da177e4SLinus Torvalds extern int dequeue_signal(struct task_struct *tsk, sigset_t *mask, siginfo_t *info);
25041da177e4SLinus Torvalds 
2505be0e6f29SOleg Nesterov static inline int kernel_dequeue_signal(siginfo_t *info)
25061da177e4SLinus Torvalds {
2507be0e6f29SOleg Nesterov 	struct task_struct *tsk = current;
2508be0e6f29SOleg Nesterov 	siginfo_t __info;
25091da177e4SLinus Torvalds 	int ret;
25101da177e4SLinus Torvalds 
2511be0e6f29SOleg Nesterov 	spin_lock_irq(&tsk->sighand->siglock);
2512be0e6f29SOleg Nesterov 	ret = dequeue_signal(tsk, &tsk->blocked, info ?: &__info);
2513be0e6f29SOleg Nesterov 	spin_unlock_irq(&tsk->sighand->siglock);
25141da177e4SLinus Torvalds 
25151da177e4SLinus Torvalds 	return ret;
25161da177e4SLinus Torvalds }
25171da177e4SLinus Torvalds 
25189a13049eSOleg Nesterov static inline void kernel_signal_stop(void)
25199a13049eSOleg Nesterov {
25209a13049eSOleg Nesterov 	spin_lock_irq(&current->sighand->siglock);
25219a13049eSOleg Nesterov 	if (current->jobctl & JOBCTL_STOP_DEQUEUED)
25229a13049eSOleg Nesterov 		__set_current_state(TASK_STOPPED);
25239a13049eSOleg Nesterov 	spin_unlock_irq(&current->sighand->siglock);
25249a13049eSOleg Nesterov 
25259a13049eSOleg Nesterov 	schedule();
25269a13049eSOleg Nesterov }
25279a13049eSOleg Nesterov 
25281da177e4SLinus Torvalds extern void release_task(struct task_struct * p);
25291da177e4SLinus Torvalds extern int send_sig_info(int, struct siginfo *, struct task_struct *);
25301da177e4SLinus Torvalds extern int force_sigsegv(int, struct task_struct *);
25311da177e4SLinus Torvalds extern int force_sig_info(int, struct siginfo *, struct task_struct *);
2532c4b92fc1SEric W. Biederman extern int __kill_pgrp_info(int sig, struct siginfo *info, struct pid *pgrp);
2533c4b92fc1SEric W. Biederman extern int kill_pid_info(int sig, struct siginfo *info, struct pid *pid);
2534d178bc3aSSerge Hallyn extern int kill_pid_info_as_cred(int, struct siginfo *, struct pid *,
2535d178bc3aSSerge Hallyn 				const struct cred *, u32);
2536c4b92fc1SEric W. Biederman extern int kill_pgrp(struct pid *pid, int sig, int priv);
2537c4b92fc1SEric W. Biederman extern int kill_pid(struct pid *pid, int sig, int priv);
2538c3de4b38SMatthew Wilcox extern int kill_proc_info(int, struct siginfo *, pid_t);
253986773473SOleg Nesterov extern __must_check bool do_notify_parent(struct task_struct *, int);
2540a7f0765eSOleg Nesterov extern void __wake_up_parent(struct task_struct *p, struct task_struct *parent);
25411da177e4SLinus Torvalds extern void force_sig(int, struct task_struct *);
25421da177e4SLinus Torvalds extern int send_sig(int, struct task_struct *, int);
254309faef11SOleg Nesterov extern int zap_other_threads(struct task_struct *p);
25441da177e4SLinus Torvalds extern struct sigqueue *sigqueue_alloc(void);
25451da177e4SLinus Torvalds extern void sigqueue_free(struct sigqueue *);
2546ac5c2153SOleg Nesterov extern int send_sigqueue(struct sigqueue *,  struct task_struct *, int group);
25479ac95f2fSOleg Nesterov extern int do_sigaction(int, struct k_sigaction *, struct k_sigaction *);
25481da177e4SLinus Torvalds 
254951a7b448SAl Viro static inline void restore_saved_sigmask(void)
255051a7b448SAl Viro {
255151a7b448SAl Viro 	if (test_and_clear_restore_sigmask())
255277097ae5SAl Viro 		__set_current_blocked(&current->saved_sigmask);
255351a7b448SAl Viro }
255451a7b448SAl Viro 
2555b7f9a11aSAl Viro static inline sigset_t *sigmask_to_save(void)
2556b7f9a11aSAl Viro {
2557b7f9a11aSAl Viro 	sigset_t *res = &current->blocked;
2558b7f9a11aSAl Viro 	if (unlikely(test_restore_sigmask()))
2559b7f9a11aSAl Viro 		res = &current->saved_sigmask;
2560b7f9a11aSAl Viro 	return res;
2561b7f9a11aSAl Viro }
2562b7f9a11aSAl Viro 
25639ec52099SCedric Le Goater static inline int kill_cad_pid(int sig, int priv)
25649ec52099SCedric Le Goater {
25659ec52099SCedric Le Goater 	return kill_pid(cad_pid, sig, priv);
25669ec52099SCedric Le Goater }
25679ec52099SCedric Le Goater 
25681da177e4SLinus Torvalds /* These can be the second arg to send_sig_info/send_group_sig_info.  */
25691da177e4SLinus Torvalds #define SEND_SIG_NOINFO ((struct siginfo *) 0)
25701da177e4SLinus Torvalds #define SEND_SIG_PRIV	((struct siginfo *) 1)
25711da177e4SLinus Torvalds #define SEND_SIG_FORCED	((struct siginfo *) 2)
25721da177e4SLinus Torvalds 
25732a855dd0SSebastian Andrzej Siewior /*
25742a855dd0SSebastian Andrzej Siewior  * True if we are on the alternate signal stack.
25752a855dd0SSebastian Andrzej Siewior  */
25761da177e4SLinus Torvalds static inline int on_sig_stack(unsigned long sp)
25771da177e4SLinus Torvalds {
25782a855dd0SSebastian Andrzej Siewior #ifdef CONFIG_STACK_GROWSUP
25792a855dd0SSebastian Andrzej Siewior 	return sp >= current->sas_ss_sp &&
25802a855dd0SSebastian Andrzej Siewior 		sp - current->sas_ss_sp < current->sas_ss_size;
25812a855dd0SSebastian Andrzej Siewior #else
25822a855dd0SSebastian Andrzej Siewior 	return sp > current->sas_ss_sp &&
25832a855dd0SSebastian Andrzej Siewior 		sp - current->sas_ss_sp <= current->sas_ss_size;
25842a855dd0SSebastian Andrzej Siewior #endif
25851da177e4SLinus Torvalds }
25861da177e4SLinus Torvalds 
25871da177e4SLinus Torvalds static inline int sas_ss_flags(unsigned long sp)
25881da177e4SLinus Torvalds {
258972f15c03SRichard Weinberger 	if (!current->sas_ss_size)
259072f15c03SRichard Weinberger 		return SS_DISABLE;
259172f15c03SRichard Weinberger 
259272f15c03SRichard Weinberger 	return on_sig_stack(sp) ? SS_ONSTACK : 0;
25931da177e4SLinus Torvalds }
25941da177e4SLinus Torvalds 
25955a1b98d3SAl Viro static inline unsigned long sigsp(unsigned long sp, struct ksignal *ksig)
25965a1b98d3SAl Viro {
25975a1b98d3SAl Viro 	if (unlikely((ksig->ka.sa.sa_flags & SA_ONSTACK)) && ! sas_ss_flags(sp))
25985a1b98d3SAl Viro #ifdef CONFIG_STACK_GROWSUP
25995a1b98d3SAl Viro 		return current->sas_ss_sp;
26005a1b98d3SAl Viro #else
26015a1b98d3SAl Viro 		return current->sas_ss_sp + current->sas_ss_size;
26025a1b98d3SAl Viro #endif
26035a1b98d3SAl Viro 	return sp;
26045a1b98d3SAl Viro }
26055a1b98d3SAl Viro 
26061da177e4SLinus Torvalds /*
26071da177e4SLinus Torvalds  * Routines for handling mm_structs
26081da177e4SLinus Torvalds  */
26091da177e4SLinus Torvalds extern struct mm_struct * mm_alloc(void);
26101da177e4SLinus Torvalds 
26111da177e4SLinus Torvalds /* mmdrop drops the mm and the page tables */
2612b3c97528SHarvey Harrison extern void __mmdrop(struct mm_struct *);
26131da177e4SLinus Torvalds static inline void mmdrop(struct mm_struct * mm)
26141da177e4SLinus Torvalds {
26156fb43d7bSIngo Molnar 	if (unlikely(atomic_dec_and_test(&mm->mm_count)))
26161da177e4SLinus Torvalds 		__mmdrop(mm);
26171da177e4SLinus Torvalds }
26181da177e4SLinus Torvalds 
26191da177e4SLinus Torvalds /* mmput gets rid of the mappings and all user-space */
26201da177e4SLinus Torvalds extern void mmput(struct mm_struct *);
26211da177e4SLinus Torvalds /* Grab a reference to a task's mm, if it is not already going away */
26221da177e4SLinus Torvalds extern struct mm_struct *get_task_mm(struct task_struct *task);
26238cdb878dSChristopher Yeoh /*
26248cdb878dSChristopher Yeoh  * Grab a reference to a task's mm, if it is not already going away
26258cdb878dSChristopher Yeoh  * and ptrace_may_access with the mode parameter passed to it
26268cdb878dSChristopher Yeoh  * succeeds.
26278cdb878dSChristopher Yeoh  */
26288cdb878dSChristopher Yeoh extern struct mm_struct *mm_access(struct task_struct *task, unsigned int mode);
26291da177e4SLinus Torvalds /* Remove the current tasks stale references to the old mm_struct */
26301da177e4SLinus Torvalds extern void mm_release(struct task_struct *, struct mm_struct *);
26311da177e4SLinus Torvalds 
26323033f14aSJosh Triplett #ifdef CONFIG_HAVE_COPY_THREAD_TLS
26333033f14aSJosh Triplett extern int copy_thread_tls(unsigned long, unsigned long, unsigned long,
26343033f14aSJosh Triplett 			struct task_struct *, unsigned long);
26353033f14aSJosh Triplett #else
26366f2c55b8SAlexey Dobriyan extern int copy_thread(unsigned long, unsigned long, unsigned long,
2637afa86fc4SAl Viro 			struct task_struct *);
26383033f14aSJosh Triplett 
26393033f14aSJosh Triplett /* Architectures that haven't opted into copy_thread_tls get the tls argument
26403033f14aSJosh Triplett  * via pt_regs, so ignore the tls argument passed via C. */
26413033f14aSJosh Triplett static inline int copy_thread_tls(
26423033f14aSJosh Triplett 		unsigned long clone_flags, unsigned long sp, unsigned long arg,
26433033f14aSJosh Triplett 		struct task_struct *p, unsigned long tls)
26443033f14aSJosh Triplett {
26453033f14aSJosh Triplett 	return copy_thread(clone_flags, sp, arg, p);
26463033f14aSJosh Triplett }
26473033f14aSJosh Triplett #endif
26481da177e4SLinus Torvalds extern void flush_thread(void);
26491da177e4SLinus Torvalds extern void exit_thread(void);
26501da177e4SLinus Torvalds 
26511da177e4SLinus Torvalds extern void exit_files(struct task_struct *);
2652a7e5328aSOleg Nesterov extern void __cleanup_sighand(struct sighand_struct *);
2653cbaffba1SOleg Nesterov 
26541da177e4SLinus Torvalds extern void exit_itimers(struct signal_struct *);
2655cbaffba1SOleg Nesterov extern void flush_itimer_signals(void);
26561da177e4SLinus Torvalds 
26579402c95fSJoe Perches extern void do_group_exit(int);
26581da177e4SLinus Torvalds 
2659c4ad8f98SLinus Torvalds extern int do_execve(struct filename *,
2660d7627467SDavid Howells 		     const char __user * const __user *,
2661da3d4c5fSAl Viro 		     const char __user * const __user *);
266251f39a1fSDavid Drysdale extern int do_execveat(int, struct filename *,
266351f39a1fSDavid Drysdale 		       const char __user * const __user *,
266451f39a1fSDavid Drysdale 		       const char __user * const __user *,
266551f39a1fSDavid Drysdale 		       int);
26663033f14aSJosh Triplett extern long _do_fork(unsigned long, unsigned long, unsigned long, int __user *, int __user *, unsigned long);
2667e80d6661SAl Viro extern long do_fork(unsigned long, unsigned long, unsigned long, int __user *, int __user *);
266836c8b586SIngo Molnar struct task_struct *fork_idle(int);
26692aa3a7f8SAl Viro extern pid_t kernel_thread(int (*fn)(void *), void *arg, unsigned long flags);
26701da177e4SLinus Torvalds 
267182b89778SAdrian Hunter extern void __set_task_comm(struct task_struct *tsk, const char *from, bool exec);
267282b89778SAdrian Hunter static inline void set_task_comm(struct task_struct *tsk, const char *from)
267382b89778SAdrian Hunter {
267482b89778SAdrian Hunter 	__set_task_comm(tsk, from, false);
267582b89778SAdrian Hunter }
267659714d65SAndrew Morton extern char *get_task_comm(char *to, struct task_struct *tsk);
26771da177e4SLinus Torvalds 
26781da177e4SLinus Torvalds #ifdef CONFIG_SMP
2679317f3941SPeter Zijlstra void scheduler_ipi(void);
268085ba2d86SRoland McGrath extern unsigned long wait_task_inactive(struct task_struct *, long match_state);
26811da177e4SLinus Torvalds #else
2682184748ccSPeter Zijlstra static inline void scheduler_ipi(void) { }
268385ba2d86SRoland McGrath static inline unsigned long wait_task_inactive(struct task_struct *p,
268485ba2d86SRoland McGrath 					       long match_state)
268585ba2d86SRoland McGrath {
268685ba2d86SRoland McGrath 	return 1;
268785ba2d86SRoland McGrath }
26881da177e4SLinus Torvalds #endif
26891da177e4SLinus Torvalds 
2690fafe870fSFrederic Weisbecker #define tasklist_empty() \
2691fafe870fSFrederic Weisbecker 	list_empty(&init_task.tasks)
2692fafe870fSFrederic Weisbecker 
269305725f7eSJiri Pirko #define next_task(p) \
269405725f7eSJiri Pirko 	list_entry_rcu((p)->tasks.next, struct task_struct, tasks)
26951da177e4SLinus Torvalds 
26961da177e4SLinus Torvalds #define for_each_process(p) \
26971da177e4SLinus Torvalds 	for (p = &init_task ; (p = next_task(p)) != &init_task ; )
26981da177e4SLinus Torvalds 
26995bb459bbSOleg Nesterov extern bool current_is_single_threaded(void);
2700d84f4f99SDavid Howells 
27011da177e4SLinus Torvalds /*
27021da177e4SLinus Torvalds  * Careful: do_each_thread/while_each_thread is a double loop so
27031da177e4SLinus Torvalds  *          'break' will not work as expected - use goto instead.
27041da177e4SLinus Torvalds  */
27051da177e4SLinus Torvalds #define do_each_thread(g, t) \
27061da177e4SLinus Torvalds 	for (g = t = &init_task ; (g = t = next_task(g)) != &init_task ; ) do
27071da177e4SLinus Torvalds 
27081da177e4SLinus Torvalds #define while_each_thread(g, t) \
27091da177e4SLinus Torvalds 	while ((t = next_thread(t)) != g)
27101da177e4SLinus Torvalds 
27110c740d0aSOleg Nesterov #define __for_each_thread(signal, t)	\
27120c740d0aSOleg Nesterov 	list_for_each_entry_rcu(t, &(signal)->thread_head, thread_node)
27130c740d0aSOleg Nesterov 
27140c740d0aSOleg Nesterov #define for_each_thread(p, t)		\
27150c740d0aSOleg Nesterov 	__for_each_thread((p)->signal, t)
27160c740d0aSOleg Nesterov 
27170c740d0aSOleg Nesterov /* Careful: this is a double loop, 'break' won't work as expected. */
27180c740d0aSOleg Nesterov #define for_each_process_thread(p, t)	\
27190c740d0aSOleg Nesterov 	for_each_process(p) for_each_thread(p, t)
27200c740d0aSOleg Nesterov 
27217e49827cSOleg Nesterov static inline int get_nr_threads(struct task_struct *tsk)
27227e49827cSOleg Nesterov {
2723b3ac022cSOleg Nesterov 	return tsk->signal->nr_threads;
27247e49827cSOleg Nesterov }
27257e49827cSOleg Nesterov 
2726087806b1SOleg Nesterov static inline bool thread_group_leader(struct task_struct *p)
2727087806b1SOleg Nesterov {
2728087806b1SOleg Nesterov 	return p->exit_signal >= 0;
2729087806b1SOleg Nesterov }
27301da177e4SLinus Torvalds 
27310804ef4bSEric W. Biederman /* Do to the insanities of de_thread it is possible for a process
27320804ef4bSEric W. Biederman  * to have the pid of the thread group leader without actually being
27330804ef4bSEric W. Biederman  * the thread group leader.  For iteration through the pids in proc
27340804ef4bSEric W. Biederman  * all we care about is that we have a task with the appropriate
27350804ef4bSEric W. Biederman  * pid, we don't actually care if we have the right task.
27360804ef4bSEric W. Biederman  */
2737e1403b8eSOleg Nesterov static inline bool has_group_leader_pid(struct task_struct *p)
27380804ef4bSEric W. Biederman {
2739e1403b8eSOleg Nesterov 	return task_pid(p) == p->signal->leader_pid;
27400804ef4bSEric W. Biederman }
27410804ef4bSEric W. Biederman 
2742bac0abd6SPavel Emelyanov static inline
2743e1403b8eSOleg Nesterov bool same_thread_group(struct task_struct *p1, struct task_struct *p2)
2744bac0abd6SPavel Emelyanov {
2745e1403b8eSOleg Nesterov 	return p1->signal == p2->signal;
2746bac0abd6SPavel Emelyanov }
2747bac0abd6SPavel Emelyanov 
274836c8b586SIngo Molnar static inline struct task_struct *next_thread(const struct task_struct *p)
274947e65328SOleg Nesterov {
275005725f7eSJiri Pirko 	return list_entry_rcu(p->thread_group.next,
275136c8b586SIngo Molnar 			      struct task_struct, thread_group);
275247e65328SOleg Nesterov }
275347e65328SOleg Nesterov 
2754e868171aSAlexey Dobriyan static inline int thread_group_empty(struct task_struct *p)
27551da177e4SLinus Torvalds {
275647e65328SOleg Nesterov 	return list_empty(&p->thread_group);
27571da177e4SLinus Torvalds }
27581da177e4SLinus Torvalds 
27591da177e4SLinus Torvalds #define delay_group_leader(p) \
27601da177e4SLinus Torvalds 		(thread_group_leader(p) && !thread_group_empty(p))
27611da177e4SLinus Torvalds 
27621da177e4SLinus Torvalds /*
2763260ea101SEric W. Biederman  * Protects ->fs, ->files, ->mm, ->group_info, ->comm, keyring
276422e2c507SJens Axboe  * subscriptions and synchronises with wait4().  Also used in procfs.  Also
2765ddbcc7e8SPaul Menage  * pins the final release of task.io_context.  Also protects ->cpuset and
2766d68b46feSOleg Nesterov  * ->cgroup.subsys[]. And ->vfork_done.
27671da177e4SLinus Torvalds  *
27681da177e4SLinus Torvalds  * Nests both inside and outside of read_lock(&tasklist_lock).
27691da177e4SLinus Torvalds  * It must not be nested with write_lock_irq(&tasklist_lock),
27701da177e4SLinus Torvalds  * neither inside nor outside.
27711da177e4SLinus Torvalds  */
27721da177e4SLinus Torvalds static inline void task_lock(struct task_struct *p)
27731da177e4SLinus Torvalds {
27741da177e4SLinus Torvalds 	spin_lock(&p->alloc_lock);
27751da177e4SLinus Torvalds }
27761da177e4SLinus Torvalds 
27771da177e4SLinus Torvalds static inline void task_unlock(struct task_struct *p)
27781da177e4SLinus Torvalds {
27791da177e4SLinus Torvalds 	spin_unlock(&p->alloc_lock);
27801da177e4SLinus Torvalds }
27811da177e4SLinus Torvalds 
2782b8ed374eSNamhyung Kim extern struct sighand_struct *__lock_task_sighand(struct task_struct *tsk,
2783f63ee72eSOleg Nesterov 							unsigned long *flags);
2784f63ee72eSOleg Nesterov 
27859388dc30SAnton Vorontsov static inline struct sighand_struct *lock_task_sighand(struct task_struct *tsk,
27869388dc30SAnton Vorontsov 						       unsigned long *flags)
27879388dc30SAnton Vorontsov {
27889388dc30SAnton Vorontsov 	struct sighand_struct *ret;
27899388dc30SAnton Vorontsov 
27909388dc30SAnton Vorontsov 	ret = __lock_task_sighand(tsk, flags);
27919388dc30SAnton Vorontsov 	(void)__cond_lock(&tsk->sighand->siglock, ret);
27929388dc30SAnton Vorontsov 	return ret;
27939388dc30SAnton Vorontsov }
2794b8ed374eSNamhyung Kim 
2795f63ee72eSOleg Nesterov static inline void unlock_task_sighand(struct task_struct *tsk,
2796f63ee72eSOleg Nesterov 						unsigned long *flags)
2797f63ee72eSOleg Nesterov {
2798f63ee72eSOleg Nesterov 	spin_unlock_irqrestore(&tsk->sighand->siglock, *flags);
2799f63ee72eSOleg Nesterov }
2800f63ee72eSOleg Nesterov 
28017d7efec3STejun Heo /**
28027d7efec3STejun Heo  * threadgroup_change_begin - mark the beginning of changes to a threadgroup
28037d7efec3STejun Heo  * @tsk: task causing the changes
28047d7efec3STejun Heo  *
28057d7efec3STejun Heo  * All operations which modify a threadgroup - a new thread joining the
28067d7efec3STejun Heo  * group, death of a member thread (the assertion of PF_EXITING) and
28077d7efec3STejun Heo  * exec(2) dethreading the process and replacing the leader - are wrapped
28087d7efec3STejun Heo  * by threadgroup_change_{begin|end}().  This is to provide a place which
28097d7efec3STejun Heo  * subsystems needing threadgroup stability can hook into for
28107d7efec3STejun Heo  * synchronization.
28117d7efec3STejun Heo  */
2812257058aeSTejun Heo static inline void threadgroup_change_begin(struct task_struct *tsk)
28134714d1d3SBen Blum {
28147d7efec3STejun Heo 	might_sleep();
28157d7efec3STejun Heo 	cgroup_threadgroup_change_begin(tsk);
28164714d1d3SBen Blum }
28177d7efec3STejun Heo 
28187d7efec3STejun Heo /**
28197d7efec3STejun Heo  * threadgroup_change_end - mark the end of changes to a threadgroup
28207d7efec3STejun Heo  * @tsk: task causing the changes
28217d7efec3STejun Heo  *
28227d7efec3STejun Heo  * See threadgroup_change_begin().
28237d7efec3STejun Heo  */
2824257058aeSTejun Heo static inline void threadgroup_change_end(struct task_struct *tsk)
28254714d1d3SBen Blum {
28267d7efec3STejun Heo 	cgroup_threadgroup_change_end(tsk);
28274714d1d3SBen Blum }
282877e4ef99STejun Heo 
2829f037360fSAl Viro #ifndef __HAVE_THREAD_FUNCTIONS
2830f037360fSAl Viro 
2831f7e4217bSRoman Zippel #define task_thread_info(task)	((struct thread_info *)(task)->stack)
2832f7e4217bSRoman Zippel #define task_stack_page(task)	((task)->stack)
2833a1261f54SAl Viro 
283410ebffdeSAl Viro static inline void setup_thread_stack(struct task_struct *p, struct task_struct *org)
283510ebffdeSAl Viro {
283610ebffdeSAl Viro 	*task_thread_info(p) = *task_thread_info(org);
283710ebffdeSAl Viro 	task_thread_info(p)->task = p;
283810ebffdeSAl Viro }
283910ebffdeSAl Viro 
28406a40281aSChuck Ebbert /*
28416a40281aSChuck Ebbert  * Return the address of the last usable long on the stack.
28426a40281aSChuck Ebbert  *
28436a40281aSChuck Ebbert  * When the stack grows down, this is just above the thread
28446a40281aSChuck Ebbert  * info struct. Going any lower will corrupt the threadinfo.
28456a40281aSChuck Ebbert  *
28466a40281aSChuck Ebbert  * When the stack grows up, this is the highest address.
28476a40281aSChuck Ebbert  * Beyond that position, we corrupt data on the next page.
28486a40281aSChuck Ebbert  */
284910ebffdeSAl Viro static inline unsigned long *end_of_stack(struct task_struct *p)
285010ebffdeSAl Viro {
28516a40281aSChuck Ebbert #ifdef CONFIG_STACK_GROWSUP
28526a40281aSChuck Ebbert 	return (unsigned long *)((unsigned long)task_thread_info(p) + THREAD_SIZE) - 1;
28536a40281aSChuck Ebbert #else
2854f7e4217bSRoman Zippel 	return (unsigned long *)(task_thread_info(p) + 1);
28556a40281aSChuck Ebbert #endif
285610ebffdeSAl Viro }
285710ebffdeSAl Viro 
2858f037360fSAl Viro #endif
2859a70857e4SAaron Tomlin #define task_stack_end_corrupted(task) \
2860a70857e4SAaron Tomlin 		(*(end_of_stack(task)) != STACK_END_MAGIC)
2861f037360fSAl Viro 
28628b05c7e6SFUJITA Tomonori static inline int object_is_on_stack(void *obj)
28638b05c7e6SFUJITA Tomonori {
28648b05c7e6SFUJITA Tomonori 	void *stack = task_stack_page(current);
28658b05c7e6SFUJITA Tomonori 
28668b05c7e6SFUJITA Tomonori 	return (obj >= stack) && (obj < (stack + THREAD_SIZE));
28678b05c7e6SFUJITA Tomonori }
28688b05c7e6SFUJITA Tomonori 
28698c9843e5SBenjamin Herrenschmidt extern void thread_info_cache_init(void);
28708c9843e5SBenjamin Herrenschmidt 
28717c9f8861SEric Sandeen #ifdef CONFIG_DEBUG_STACK_USAGE
28727c9f8861SEric Sandeen static inline unsigned long stack_not_used(struct task_struct *p)
28737c9f8861SEric Sandeen {
28747c9f8861SEric Sandeen 	unsigned long *n = end_of_stack(p);
28757c9f8861SEric Sandeen 
28767c9f8861SEric Sandeen 	do { 	/* Skip over canary */
28776c31da34SHelge Deller # ifdef CONFIG_STACK_GROWSUP
28786c31da34SHelge Deller 		n--;
28796c31da34SHelge Deller # else
28807c9f8861SEric Sandeen 		n++;
28816c31da34SHelge Deller # endif
28827c9f8861SEric Sandeen 	} while (!*n);
28837c9f8861SEric Sandeen 
28846c31da34SHelge Deller # ifdef CONFIG_STACK_GROWSUP
28856c31da34SHelge Deller 	return (unsigned long)end_of_stack(p) - (unsigned long)n;
28866c31da34SHelge Deller # else
28877c9f8861SEric Sandeen 	return (unsigned long)n - (unsigned long)end_of_stack(p);
28886c31da34SHelge Deller # endif
28897c9f8861SEric Sandeen }
28907c9f8861SEric Sandeen #endif
2891d4311ff1SAaron Tomlin extern void set_task_stack_end_magic(struct task_struct *tsk);
28927c9f8861SEric Sandeen 
28931da177e4SLinus Torvalds /* set thread flags in other task's structures
28941da177e4SLinus Torvalds  * - see asm/thread_info.h for TIF_xxxx flags available
28951da177e4SLinus Torvalds  */
28961da177e4SLinus Torvalds static inline void set_tsk_thread_flag(struct task_struct *tsk, int flag)
28971da177e4SLinus Torvalds {
2898a1261f54SAl Viro 	set_ti_thread_flag(task_thread_info(tsk), flag);
28991da177e4SLinus Torvalds }
29001da177e4SLinus Torvalds 
29011da177e4SLinus Torvalds static inline void clear_tsk_thread_flag(struct task_struct *tsk, int flag)
29021da177e4SLinus Torvalds {
2903a1261f54SAl Viro 	clear_ti_thread_flag(task_thread_info(tsk), flag);
29041da177e4SLinus Torvalds }
29051da177e4SLinus Torvalds 
29061da177e4SLinus Torvalds static inline int test_and_set_tsk_thread_flag(struct task_struct *tsk, int flag)
29071da177e4SLinus Torvalds {
2908a1261f54SAl Viro 	return test_and_set_ti_thread_flag(task_thread_info(tsk), flag);
29091da177e4SLinus Torvalds }
29101da177e4SLinus Torvalds 
29111da177e4SLinus Torvalds static inline int test_and_clear_tsk_thread_flag(struct task_struct *tsk, int flag)
29121da177e4SLinus Torvalds {
2913a1261f54SAl Viro 	return test_and_clear_ti_thread_flag(task_thread_info(tsk), flag);
29141da177e4SLinus Torvalds }
29151da177e4SLinus Torvalds 
29161da177e4SLinus Torvalds static inline int test_tsk_thread_flag(struct task_struct *tsk, int flag)
29171da177e4SLinus Torvalds {
2918a1261f54SAl Viro 	return test_ti_thread_flag(task_thread_info(tsk), flag);
29191da177e4SLinus Torvalds }
29201da177e4SLinus Torvalds 
29211da177e4SLinus Torvalds static inline void set_tsk_need_resched(struct task_struct *tsk)
29221da177e4SLinus Torvalds {
29231da177e4SLinus Torvalds 	set_tsk_thread_flag(tsk,TIF_NEED_RESCHED);
29241da177e4SLinus Torvalds }
29251da177e4SLinus Torvalds 
29261da177e4SLinus Torvalds static inline void clear_tsk_need_resched(struct task_struct *tsk)
29271da177e4SLinus Torvalds {
29281da177e4SLinus Torvalds 	clear_tsk_thread_flag(tsk,TIF_NEED_RESCHED);
29291da177e4SLinus Torvalds }
29301da177e4SLinus Torvalds 
29318ae121acSGregory Haskins static inline int test_tsk_need_resched(struct task_struct *tsk)
29328ae121acSGregory Haskins {
29338ae121acSGregory Haskins 	return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED));
29348ae121acSGregory Haskins }
29358ae121acSGregory Haskins 
2936690cc3ffSEric W. Biederman static inline int restart_syscall(void)
2937690cc3ffSEric W. Biederman {
2938690cc3ffSEric W. Biederman 	set_tsk_thread_flag(current, TIF_SIGPENDING);
2939690cc3ffSEric W. Biederman 	return -ERESTARTNOINTR;
2940690cc3ffSEric W. Biederman }
2941690cc3ffSEric W. Biederman 
29421da177e4SLinus Torvalds static inline int signal_pending(struct task_struct *p)
29431da177e4SLinus Torvalds {
29441da177e4SLinus Torvalds 	return unlikely(test_tsk_thread_flag(p,TIF_SIGPENDING));
29451da177e4SLinus Torvalds }
29461da177e4SLinus Torvalds 
2947d9588725SRoland McGrath static inline int __fatal_signal_pending(struct task_struct *p)
2948d9588725SRoland McGrath {
2949d9588725SRoland McGrath 	return unlikely(sigismember(&p->pending.signal, SIGKILL));
2950d9588725SRoland McGrath }
2951f776d12dSMatthew Wilcox 
2952f776d12dSMatthew Wilcox static inline int fatal_signal_pending(struct task_struct *p)
2953f776d12dSMatthew Wilcox {
2954f776d12dSMatthew Wilcox 	return signal_pending(p) && __fatal_signal_pending(p);
2955f776d12dSMatthew Wilcox }
2956f776d12dSMatthew Wilcox 
295716882c1eSOleg Nesterov static inline int signal_pending_state(long state, struct task_struct *p)
295816882c1eSOleg Nesterov {
295916882c1eSOleg Nesterov 	if (!(state & (TASK_INTERRUPTIBLE | TASK_WAKEKILL)))
296016882c1eSOleg Nesterov 		return 0;
296116882c1eSOleg Nesterov 	if (!signal_pending(p))
296216882c1eSOleg Nesterov 		return 0;
296316882c1eSOleg Nesterov 
296416882c1eSOleg Nesterov 	return (state & TASK_INTERRUPTIBLE) || __fatal_signal_pending(p);
296516882c1eSOleg Nesterov }
296616882c1eSOleg Nesterov 
29671da177e4SLinus Torvalds /*
29681da177e4SLinus Torvalds  * cond_resched() and cond_resched_lock(): latency reduction via
29691da177e4SLinus Torvalds  * explicit rescheduling in places that are safe. The return
29701da177e4SLinus Torvalds  * value indicates whether a reschedule was done in fact.
29711da177e4SLinus Torvalds  * cond_resched_lock() will drop the spinlock before scheduling,
29721da177e4SLinus Torvalds  * cond_resched_softirq() will enable bhs before scheduling.
29731da177e4SLinus Torvalds  */
2974c3921ab7SLinus Torvalds extern int _cond_resched(void);
29756f80bd98SFrederic Weisbecker 
2976613afbf8SFrederic Weisbecker #define cond_resched() ({			\
29773427445aSPeter Zijlstra 	___might_sleep(__FILE__, __LINE__, 0);	\
2978613afbf8SFrederic Weisbecker 	_cond_resched();			\
2979613afbf8SFrederic Weisbecker })
29806f80bd98SFrederic Weisbecker 
2981613afbf8SFrederic Weisbecker extern int __cond_resched_lock(spinlock_t *lock);
2982613afbf8SFrederic Weisbecker 
2983613afbf8SFrederic Weisbecker #define cond_resched_lock(lock) ({				\
29843427445aSPeter Zijlstra 	___might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET);\
2985613afbf8SFrederic Weisbecker 	__cond_resched_lock(lock);				\
2986613afbf8SFrederic Weisbecker })
2987613afbf8SFrederic Weisbecker 
2988613afbf8SFrederic Weisbecker extern int __cond_resched_softirq(void);
2989613afbf8SFrederic Weisbecker 
2990613afbf8SFrederic Weisbecker #define cond_resched_softirq() ({					\
29913427445aSPeter Zijlstra 	___might_sleep(__FILE__, __LINE__, SOFTIRQ_DISABLE_OFFSET);	\
2992613afbf8SFrederic Weisbecker 	__cond_resched_softirq();					\
2993613afbf8SFrederic Weisbecker })
29941da177e4SLinus Torvalds 
2995f6f3c437SSimon Horman static inline void cond_resched_rcu(void)
2996f6f3c437SSimon Horman {
2997f6f3c437SSimon Horman #if defined(CONFIG_DEBUG_ATOMIC_SLEEP) || !defined(CONFIG_PREEMPT_RCU)
2998f6f3c437SSimon Horman 	rcu_read_unlock();
2999f6f3c437SSimon Horman 	cond_resched();
3000f6f3c437SSimon Horman 	rcu_read_lock();
3001f6f3c437SSimon Horman #endif
3002f6f3c437SSimon Horman }
3003f6f3c437SSimon Horman 
30041da177e4SLinus Torvalds /*
30051da177e4SLinus Torvalds  * Does a critical section need to be broken due to another
300695c354feSNick Piggin  * task waiting?: (technically does not depend on CONFIG_PREEMPT,
300795c354feSNick Piggin  * but a general need for low latency)
30081da177e4SLinus Torvalds  */
300995c354feSNick Piggin static inline int spin_needbreak(spinlock_t *lock)
30101da177e4SLinus Torvalds {
301195c354feSNick Piggin #ifdef CONFIG_PREEMPT
301295c354feSNick Piggin 	return spin_is_contended(lock);
301395c354feSNick Piggin #else
30141da177e4SLinus Torvalds 	return 0;
301595c354feSNick Piggin #endif
30161da177e4SLinus Torvalds }
30171da177e4SLinus Torvalds 
30187bb44adeSRoland McGrath /*
3019ee761f62SThomas Gleixner  * Idle thread specific functions to determine the need_resched
302069dd0f84SPeter Zijlstra  * polling state.
3021ee761f62SThomas Gleixner  */
302269dd0f84SPeter Zijlstra #ifdef TIF_POLLING_NRFLAG
3023ee761f62SThomas Gleixner static inline int tsk_is_polling(struct task_struct *p)
3024ee761f62SThomas Gleixner {
3025ee761f62SThomas Gleixner 	return test_tsk_thread_flag(p, TIF_POLLING_NRFLAG);
3026ee761f62SThomas Gleixner }
3027ea811747SPeter Zijlstra 
3028ea811747SPeter Zijlstra static inline void __current_set_polling(void)
30293a98f871SThomas Gleixner {
30303a98f871SThomas Gleixner 	set_thread_flag(TIF_POLLING_NRFLAG);
30313a98f871SThomas Gleixner }
30323a98f871SThomas Gleixner 
3033ea811747SPeter Zijlstra static inline bool __must_check current_set_polling_and_test(void)
3034ea811747SPeter Zijlstra {
3035ea811747SPeter Zijlstra 	__current_set_polling();
3036ea811747SPeter Zijlstra 
3037ea811747SPeter Zijlstra 	/*
3038ea811747SPeter Zijlstra 	 * Polling state must be visible before we test NEED_RESCHED,
30398875125eSKirill Tkhai 	 * paired by resched_curr()
3040ea811747SPeter Zijlstra 	 */
30414e857c58SPeter Zijlstra 	smp_mb__after_atomic();
3042ea811747SPeter Zijlstra 
3043ea811747SPeter Zijlstra 	return unlikely(tif_need_resched());
3044ea811747SPeter Zijlstra }
3045ea811747SPeter Zijlstra 
3046ea811747SPeter Zijlstra static inline void __current_clr_polling(void)
30473a98f871SThomas Gleixner {
30483a98f871SThomas Gleixner 	clear_thread_flag(TIF_POLLING_NRFLAG);
30493a98f871SThomas Gleixner }
3050ea811747SPeter Zijlstra 
3051ea811747SPeter Zijlstra static inline bool __must_check current_clr_polling_and_test(void)
3052ea811747SPeter Zijlstra {
3053ea811747SPeter Zijlstra 	__current_clr_polling();
3054ea811747SPeter Zijlstra 
3055ea811747SPeter Zijlstra 	/*
3056ea811747SPeter Zijlstra 	 * Polling state must be visible before we test NEED_RESCHED,
30578875125eSKirill Tkhai 	 * paired by resched_curr()
3058ea811747SPeter Zijlstra 	 */
30594e857c58SPeter Zijlstra 	smp_mb__after_atomic();
3060ea811747SPeter Zijlstra 
3061ea811747SPeter Zijlstra 	return unlikely(tif_need_resched());
3062ea811747SPeter Zijlstra }
3063ea811747SPeter Zijlstra 
3064ee761f62SThomas Gleixner #else
3065ee761f62SThomas Gleixner static inline int tsk_is_polling(struct task_struct *p) { return 0; }
3066ea811747SPeter Zijlstra static inline void __current_set_polling(void) { }
3067ea811747SPeter Zijlstra static inline void __current_clr_polling(void) { }
3068ea811747SPeter Zijlstra 
3069ea811747SPeter Zijlstra static inline bool __must_check current_set_polling_and_test(void)
3070ea811747SPeter Zijlstra {
3071ea811747SPeter Zijlstra 	return unlikely(tif_need_resched());
3072ea811747SPeter Zijlstra }
3073ea811747SPeter Zijlstra static inline bool __must_check current_clr_polling_and_test(void)
3074ea811747SPeter Zijlstra {
3075ea811747SPeter Zijlstra 	return unlikely(tif_need_resched());
3076ea811747SPeter Zijlstra }
3077ee761f62SThomas Gleixner #endif
3078ee761f62SThomas Gleixner 
30798cb75e0cSPeter Zijlstra static inline void current_clr_polling(void)
30808cb75e0cSPeter Zijlstra {
30818cb75e0cSPeter Zijlstra 	__current_clr_polling();
30828cb75e0cSPeter Zijlstra 
30838cb75e0cSPeter Zijlstra 	/*
30848cb75e0cSPeter Zijlstra 	 * Ensure we check TIF_NEED_RESCHED after we clear the polling bit.
30858cb75e0cSPeter Zijlstra 	 * Once the bit is cleared, we'll get IPIs with every new
30868cb75e0cSPeter Zijlstra 	 * TIF_NEED_RESCHED and the IPI handler, scheduler_ipi(), will also
30878cb75e0cSPeter Zijlstra 	 * fold.
30888cb75e0cSPeter Zijlstra 	 */
30898875125eSKirill Tkhai 	smp_mb(); /* paired with resched_curr() */
30908cb75e0cSPeter Zijlstra 
30918cb75e0cSPeter Zijlstra 	preempt_fold_need_resched();
30928cb75e0cSPeter Zijlstra }
30938cb75e0cSPeter Zijlstra 
309475f93fedSPeter Zijlstra static __always_inline bool need_resched(void)
309575f93fedSPeter Zijlstra {
309675f93fedSPeter Zijlstra 	return unlikely(tif_need_resched());
309775f93fedSPeter Zijlstra }
309875f93fedSPeter Zijlstra 
3099ee761f62SThomas Gleixner /*
3100f06febc9SFrank Mayhar  * Thread group CPU time accounting.
3101f06febc9SFrank Mayhar  */
31024cd4c1b4SPeter Zijlstra void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times);
31034da94d49SPeter Zijlstra void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times);
3104f06febc9SFrank Mayhar 
3105f06febc9SFrank Mayhar /*
31067bb44adeSRoland McGrath  * Reevaluate whether the task has signals pending delivery.
31077bb44adeSRoland McGrath  * Wake the task if so.
31087bb44adeSRoland McGrath  * This is required every time the blocked sigset_t changes.
31097bb44adeSRoland McGrath  * callers must hold sighand->siglock.
31107bb44adeSRoland McGrath  */
31117bb44adeSRoland McGrath extern void recalc_sigpending_and_wake(struct task_struct *t);
31121da177e4SLinus Torvalds extern void recalc_sigpending(void);
31131da177e4SLinus Torvalds 
3114910ffdb1SOleg Nesterov extern void signal_wake_up_state(struct task_struct *t, unsigned int state);
3115910ffdb1SOleg Nesterov 
3116910ffdb1SOleg Nesterov static inline void signal_wake_up(struct task_struct *t, bool resume)
3117910ffdb1SOleg Nesterov {
3118910ffdb1SOleg Nesterov 	signal_wake_up_state(t, resume ? TASK_WAKEKILL : 0);
3119910ffdb1SOleg Nesterov }
3120910ffdb1SOleg Nesterov static inline void ptrace_signal_wake_up(struct task_struct *t, bool resume)
3121910ffdb1SOleg Nesterov {
3122910ffdb1SOleg Nesterov 	signal_wake_up_state(t, resume ? __TASK_TRACED : 0);
3123910ffdb1SOleg Nesterov }
31241da177e4SLinus Torvalds 
31251da177e4SLinus Torvalds /*
31261da177e4SLinus Torvalds  * Wrappers for p->thread_info->cpu access. No-op on UP.
31271da177e4SLinus Torvalds  */
31281da177e4SLinus Torvalds #ifdef CONFIG_SMP
31291da177e4SLinus Torvalds 
31301da177e4SLinus Torvalds static inline unsigned int task_cpu(const struct task_struct *p)
31311da177e4SLinus Torvalds {
3132a1261f54SAl Viro 	return task_thread_info(p)->cpu;
31331da177e4SLinus Torvalds }
31341da177e4SLinus Torvalds 
3135b32e86b4SIngo Molnar static inline int task_node(const struct task_struct *p)
3136b32e86b4SIngo Molnar {
3137b32e86b4SIngo Molnar 	return cpu_to_node(task_cpu(p));
3138b32e86b4SIngo Molnar }
3139b32e86b4SIngo Molnar 
3140c65cc870SIngo Molnar extern void set_task_cpu(struct task_struct *p, unsigned int cpu);
31411da177e4SLinus Torvalds 
31421da177e4SLinus Torvalds #else
31431da177e4SLinus Torvalds 
31441da177e4SLinus Torvalds static inline unsigned int task_cpu(const struct task_struct *p)
31451da177e4SLinus Torvalds {
31461da177e4SLinus Torvalds 	return 0;
31471da177e4SLinus Torvalds }
31481da177e4SLinus Torvalds 
31491da177e4SLinus Torvalds static inline void set_task_cpu(struct task_struct *p, unsigned int cpu)
31501da177e4SLinus Torvalds {
31511da177e4SLinus Torvalds }
31521da177e4SLinus Torvalds 
31531da177e4SLinus Torvalds #endif /* CONFIG_SMP */
31541da177e4SLinus Torvalds 
315596f874e2SRusty Russell extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask);
315696f874e2SRusty Russell extern long sched_getaffinity(pid_t pid, struct cpumask *mask);
31575c45bf27SSiddha, Suresh B 
31587c941438SDhaval Giani #ifdef CONFIG_CGROUP_SCHED
315907e06b01SYong Zhang extern struct task_group root_task_group;
31608323f26cSPeter Zijlstra #endif /* CONFIG_CGROUP_SCHED */
31619b5b7751SSrivatsa Vaddagiri 
316254e99124SDhaval Giani extern int task_can_switch_user(struct user_struct *up,
316354e99124SDhaval Giani 					struct task_struct *tsk);
316454e99124SDhaval Giani 
31654b98d11bSAlexey Dobriyan #ifdef CONFIG_TASK_XACCT
31664b98d11bSAlexey Dobriyan static inline void add_rchar(struct task_struct *tsk, ssize_t amt)
31674b98d11bSAlexey Dobriyan {
3168940389b8SAndrea Righi 	tsk->ioac.rchar += amt;
31694b98d11bSAlexey Dobriyan }
31704b98d11bSAlexey Dobriyan 
31714b98d11bSAlexey Dobriyan static inline void add_wchar(struct task_struct *tsk, ssize_t amt)
31724b98d11bSAlexey Dobriyan {
3173940389b8SAndrea Righi 	tsk->ioac.wchar += amt;
31744b98d11bSAlexey Dobriyan }
31754b98d11bSAlexey Dobriyan 
31764b98d11bSAlexey Dobriyan static inline void inc_syscr(struct task_struct *tsk)
31774b98d11bSAlexey Dobriyan {
3178940389b8SAndrea Righi 	tsk->ioac.syscr++;
31794b98d11bSAlexey Dobriyan }
31804b98d11bSAlexey Dobriyan 
31814b98d11bSAlexey Dobriyan static inline void inc_syscw(struct task_struct *tsk)
31824b98d11bSAlexey Dobriyan {
3183940389b8SAndrea Righi 	tsk->ioac.syscw++;
31844b98d11bSAlexey Dobriyan }
31854b98d11bSAlexey Dobriyan #else
31864b98d11bSAlexey Dobriyan static inline void add_rchar(struct task_struct *tsk, ssize_t amt)
31874b98d11bSAlexey Dobriyan {
31884b98d11bSAlexey Dobriyan }
31894b98d11bSAlexey Dobriyan 
31904b98d11bSAlexey Dobriyan static inline void add_wchar(struct task_struct *tsk, ssize_t amt)
31914b98d11bSAlexey Dobriyan {
31924b98d11bSAlexey Dobriyan }
31934b98d11bSAlexey Dobriyan 
31944b98d11bSAlexey Dobriyan static inline void inc_syscr(struct task_struct *tsk)
31954b98d11bSAlexey Dobriyan {
31964b98d11bSAlexey Dobriyan }
31974b98d11bSAlexey Dobriyan 
31984b98d11bSAlexey Dobriyan static inline void inc_syscw(struct task_struct *tsk)
31994b98d11bSAlexey Dobriyan {
32004b98d11bSAlexey Dobriyan }
32014b98d11bSAlexey Dobriyan #endif
32024b98d11bSAlexey Dobriyan 
320382455257SDave Hansen #ifndef TASK_SIZE_OF
320482455257SDave Hansen #define TASK_SIZE_OF(tsk)	TASK_SIZE
320582455257SDave Hansen #endif
320682455257SDave Hansen 
3207f98bafa0SOleg Nesterov #ifdef CONFIG_MEMCG
3208cf475ad2SBalbir Singh extern void mm_update_next_owner(struct mm_struct *mm);
3209cf475ad2SBalbir Singh #else
3210cf475ad2SBalbir Singh static inline void mm_update_next_owner(struct mm_struct *mm)
3211cf475ad2SBalbir Singh {
3212cf475ad2SBalbir Singh }
3213f98bafa0SOleg Nesterov #endif /* CONFIG_MEMCG */
3214cf475ad2SBalbir Singh 
32153e10e716SJiri Slaby static inline unsigned long task_rlimit(const struct task_struct *tsk,
32163e10e716SJiri Slaby 		unsigned int limit)
32173e10e716SJiri Slaby {
3218316c1608SJason Low 	return READ_ONCE(tsk->signal->rlim[limit].rlim_cur);
32193e10e716SJiri Slaby }
32203e10e716SJiri Slaby 
32213e10e716SJiri Slaby static inline unsigned long task_rlimit_max(const struct task_struct *tsk,
32223e10e716SJiri Slaby 		unsigned int limit)
32233e10e716SJiri Slaby {
3224316c1608SJason Low 	return READ_ONCE(tsk->signal->rlim[limit].rlim_max);
32253e10e716SJiri Slaby }
32263e10e716SJiri Slaby 
32273e10e716SJiri Slaby static inline unsigned long rlimit(unsigned int limit)
32283e10e716SJiri Slaby {
32293e10e716SJiri Slaby 	return task_rlimit(current, limit);
32303e10e716SJiri Slaby }
32313e10e716SJiri Slaby 
32323e10e716SJiri Slaby static inline unsigned long rlimit_max(unsigned int limit)
32333e10e716SJiri Slaby {
32343e10e716SJiri Slaby 	return task_rlimit_max(current, limit);
32353e10e716SJiri Slaby }
32363e10e716SJiri Slaby 
3237adaf9fcdSRafael J. Wysocki #ifdef CONFIG_CPU_FREQ
3238adaf9fcdSRafael J. Wysocki struct update_util_data {
3239adaf9fcdSRafael J. Wysocki 	void (*func)(struct update_util_data *data,
3240adaf9fcdSRafael J. Wysocki 		     u64 time, unsigned long util, unsigned long max);
3241adaf9fcdSRafael J. Wysocki };
3242adaf9fcdSRafael J. Wysocki 
3243adaf9fcdSRafael J. Wysocki void cpufreq_set_update_util_data(int cpu, struct update_util_data *data);
3244adaf9fcdSRafael J. Wysocki #endif /* CONFIG_CPU_FREQ */
3245adaf9fcdSRafael J. Wysocki 
32461da177e4SLinus Torvalds #endif
3247