11da177e4SLinus Torvalds #ifndef _LINUX_SCHED_H 21da177e4SLinus Torvalds #define _LINUX_SCHED_H 31da177e4SLinus Torvalds 4607ca46eSDavid Howells #include <uapi/linux/sched.h> 5b7b3c76aSDavid Woodhouse 65c228079SDongsheng Yang #include <linux/sched/prio.h> 75c228079SDongsheng Yang 8b7b3c76aSDavid Woodhouse 9b7b3c76aSDavid Woodhouse struct sched_param { 10b7b3c76aSDavid Woodhouse int sched_priority; 11b7b3c76aSDavid Woodhouse }; 12b7b3c76aSDavid Woodhouse 131da177e4SLinus Torvalds #include <asm/param.h> /* for HZ */ 141da177e4SLinus Torvalds 151da177e4SLinus Torvalds #include <linux/capability.h> 161da177e4SLinus Torvalds #include <linux/threads.h> 171da177e4SLinus Torvalds #include <linux/kernel.h> 181da177e4SLinus Torvalds #include <linux/types.h> 191da177e4SLinus Torvalds #include <linux/timex.h> 201da177e4SLinus Torvalds #include <linux/jiffies.h> 21fb00aca4SPeter Zijlstra #include <linux/plist.h> 221da177e4SLinus Torvalds #include <linux/rbtree.h> 231da177e4SLinus Torvalds #include <linux/thread_info.h> 241da177e4SLinus Torvalds #include <linux/cpumask.h> 251da177e4SLinus Torvalds #include <linux/errno.h> 261da177e4SLinus Torvalds #include <linux/nodemask.h> 27c92ff1bdSMartin Schwidefsky #include <linux/mm_types.h> 2892cf2118SFrederic Weisbecker #include <linux/preempt.h> 291da177e4SLinus Torvalds 301da177e4SLinus Torvalds #include <asm/page.h> 311da177e4SLinus Torvalds #include <asm/ptrace.h> 32bfc3f028SFrederic Weisbecker #include <linux/cputime.h> 331da177e4SLinus Torvalds 341da177e4SLinus Torvalds #include <linux/smp.h> 351da177e4SLinus Torvalds #include <linux/sem.h> 36ab602f79SJack Miller #include <linux/shm.h> 371da177e4SLinus Torvalds #include <linux/signal.h> 381da177e4SLinus Torvalds #include <linux/compiler.h> 391da177e4SLinus Torvalds #include <linux/completion.h> 401da177e4SLinus Torvalds #include <linux/pid.h> 411da177e4SLinus Torvalds #include <linux/percpu.h> 421da177e4SLinus Torvalds #include <linux/topology.h> 433e26c149SPeter Zijlstra #include <linux/proportions.h> 441da177e4SLinus Torvalds #include <linux/seccomp.h> 45e56d0903SIngo Molnar #include <linux/rcupdate.h> 4605725f7eSJiri Pirko #include <linux/rculist.h> 4723f78d4aSIngo Molnar #include <linux/rtmutex.h> 481da177e4SLinus Torvalds 49a3b6714eSDavid Woodhouse #include <linux/time.h> 50a3b6714eSDavid Woodhouse #include <linux/param.h> 51a3b6714eSDavid Woodhouse #include <linux/resource.h> 52a3b6714eSDavid Woodhouse #include <linux/timer.h> 53a3b6714eSDavid Woodhouse #include <linux/hrtimer.h> 545c9a8750SDmitry Vyukov #include <linux/kcov.h> 557c3ab738SAndrew Morton #include <linux/task_io_accounting.h> 569745512cSArjan van de Ven #include <linux/latencytop.h> 579e2b2dc4SDavid Howells #include <linux/cred.h> 58fa14ff4aSPeter Zijlstra #include <linux/llist.h> 597b44ab97SEric W. Biederman #include <linux/uidgid.h> 6021caf2fcSMing Lei #include <linux/gfp.h> 61d4311ff1SAaron Tomlin #include <linux/magic.h> 627d7efec3STejun Heo #include <linux/cgroup-defs.h> 63a3b6714eSDavid Woodhouse 64a3b6714eSDavid Woodhouse #include <asm/processor.h> 6536d57ac4SH. J. Lu 66d50dde5aSDario Faggioli #define SCHED_ATTR_SIZE_VER0 48 /* sizeof first published struct */ 67d50dde5aSDario Faggioli 68d50dde5aSDario Faggioli /* 69d50dde5aSDario Faggioli * Extended scheduling parameters data structure. 70d50dde5aSDario Faggioli * 71d50dde5aSDario Faggioli * This is needed because the original struct sched_param can not be 72d50dde5aSDario Faggioli * altered without introducing ABI issues with legacy applications 73d50dde5aSDario Faggioli * (e.g., in sched_getparam()). 74d50dde5aSDario Faggioli * 75d50dde5aSDario Faggioli * However, the possibility of specifying more than just a priority for 76d50dde5aSDario Faggioli * the tasks may be useful for a wide variety of application fields, e.g., 77d50dde5aSDario Faggioli * multimedia, streaming, automation and control, and many others. 78d50dde5aSDario Faggioli * 79d50dde5aSDario Faggioli * This variant (sched_attr) is meant at describing a so-called 80d50dde5aSDario Faggioli * sporadic time-constrained task. In such model a task is specified by: 81d50dde5aSDario Faggioli * - the activation period or minimum instance inter-arrival time; 82d50dde5aSDario Faggioli * - the maximum (or average, depending on the actual scheduling 83d50dde5aSDario Faggioli * discipline) computation time of all instances, a.k.a. runtime; 84d50dde5aSDario Faggioli * - the deadline (relative to the actual activation time) of each 85d50dde5aSDario Faggioli * instance. 86d50dde5aSDario Faggioli * Very briefly, a periodic (sporadic) task asks for the execution of 87d50dde5aSDario Faggioli * some specific computation --which is typically called an instance-- 88d50dde5aSDario Faggioli * (at most) every period. Moreover, each instance typically lasts no more 89d50dde5aSDario Faggioli * than the runtime and must be completed by time instant t equal to 90d50dde5aSDario Faggioli * the instance activation time + the deadline. 91d50dde5aSDario Faggioli * 92d50dde5aSDario Faggioli * This is reflected by the actual fields of the sched_attr structure: 93d50dde5aSDario Faggioli * 94d50dde5aSDario Faggioli * @size size of the structure, for fwd/bwd compat. 95d50dde5aSDario Faggioli * 96d50dde5aSDario Faggioli * @sched_policy task's scheduling policy 97d50dde5aSDario Faggioli * @sched_flags for customizing the scheduler behaviour 98d50dde5aSDario Faggioli * @sched_nice task's nice value (SCHED_NORMAL/BATCH) 99d50dde5aSDario Faggioli * @sched_priority task's static priority (SCHED_FIFO/RR) 100d50dde5aSDario Faggioli * @sched_deadline representative of the task's deadline 101d50dde5aSDario Faggioli * @sched_runtime representative of the task's runtime 102d50dde5aSDario Faggioli * @sched_period representative of the task's period 103d50dde5aSDario Faggioli * 104d50dde5aSDario Faggioli * Given this task model, there are a multiplicity of scheduling algorithms 105d50dde5aSDario Faggioli * and policies, that can be used to ensure all the tasks will make their 106d50dde5aSDario Faggioli * timing constraints. 107aab03e05SDario Faggioli * 108aab03e05SDario Faggioli * As of now, the SCHED_DEADLINE policy (sched_dl scheduling class) is the 109aab03e05SDario Faggioli * only user of this new interface. More information about the algorithm 110aab03e05SDario Faggioli * available in the scheduling class file or in Documentation/. 111d50dde5aSDario Faggioli */ 112d50dde5aSDario Faggioli struct sched_attr { 113d50dde5aSDario Faggioli u32 size; 114d50dde5aSDario Faggioli 115d50dde5aSDario Faggioli u32 sched_policy; 116d50dde5aSDario Faggioli u64 sched_flags; 117d50dde5aSDario Faggioli 118d50dde5aSDario Faggioli /* SCHED_NORMAL, SCHED_BATCH */ 119d50dde5aSDario Faggioli s32 sched_nice; 120d50dde5aSDario Faggioli 121d50dde5aSDario Faggioli /* SCHED_FIFO, SCHED_RR */ 122d50dde5aSDario Faggioli u32 sched_priority; 123d50dde5aSDario Faggioli 124d50dde5aSDario Faggioli /* SCHED_DEADLINE */ 125d50dde5aSDario Faggioli u64 sched_runtime; 126d50dde5aSDario Faggioli u64 sched_deadline; 127d50dde5aSDario Faggioli u64 sched_period; 128d50dde5aSDario Faggioli }; 129d50dde5aSDario Faggioli 130c87e2837SIngo Molnar struct futex_pi_state; 131286100a6SAlexey Dobriyan struct robust_list_head; 132bddd87c7SAkinobu Mita struct bio_list; 1335ad4e53bSAl Viro struct fs_struct; 134cdd6c482SIngo Molnar struct perf_event_context; 13573c10101SJens Axboe struct blk_plug; 136c4ad8f98SLinus Torvalds struct filename; 13789076bc3SAl Viro struct nameidata; 1381da177e4SLinus Torvalds 139615d6e87SDavidlohr Bueso #define VMACACHE_BITS 2 140615d6e87SDavidlohr Bueso #define VMACACHE_SIZE (1U << VMACACHE_BITS) 141615d6e87SDavidlohr Bueso #define VMACACHE_MASK (VMACACHE_SIZE - 1) 142615d6e87SDavidlohr Bueso 1431da177e4SLinus Torvalds /* 1441da177e4SLinus Torvalds * These are the constant used to fake the fixed-point load-average 1451da177e4SLinus Torvalds * counting. Some notes: 1461da177e4SLinus Torvalds * - 11 bit fractions expand to 22 bits by the multiplies: this gives 1471da177e4SLinus Torvalds * a load-average precision of 10 bits integer + 11 bits fractional 1481da177e4SLinus Torvalds * - if you want to count load-averages more often, you need more 1491da177e4SLinus Torvalds * precision, or rounding will get you. With 2-second counting freq, 1501da177e4SLinus Torvalds * the EXP_n values would be 1981, 2034 and 2043 if still using only 1511da177e4SLinus Torvalds * 11 bit fractions. 1521da177e4SLinus Torvalds */ 1531da177e4SLinus Torvalds extern unsigned long avenrun[]; /* Load averages */ 1542d02494fSThomas Gleixner extern void get_avenrun(unsigned long *loads, unsigned long offset, int shift); 1551da177e4SLinus Torvalds 1561da177e4SLinus Torvalds #define FSHIFT 11 /* nr of bits of precision */ 1571da177e4SLinus Torvalds #define FIXED_1 (1<<FSHIFT) /* 1.0 as fixed-point */ 1580c2043abSLinus Torvalds #define LOAD_FREQ (5*HZ+1) /* 5 sec intervals */ 1591da177e4SLinus Torvalds #define EXP_1 1884 /* 1/exp(5sec/1min) as fixed-point */ 1601da177e4SLinus Torvalds #define EXP_5 2014 /* 1/exp(5sec/5min) */ 1611da177e4SLinus Torvalds #define EXP_15 2037 /* 1/exp(5sec/15min) */ 1621da177e4SLinus Torvalds 1631da177e4SLinus Torvalds #define CALC_LOAD(load,exp,n) \ 1641da177e4SLinus Torvalds load *= exp; \ 1651da177e4SLinus Torvalds load += n*(FIXED_1-exp); \ 1661da177e4SLinus Torvalds load >>= FSHIFT; 1671da177e4SLinus Torvalds 1681da177e4SLinus Torvalds extern unsigned long total_forks; 1691da177e4SLinus Torvalds extern int nr_threads; 1701da177e4SLinus Torvalds DECLARE_PER_CPU(unsigned long, process_counts); 1711da177e4SLinus Torvalds extern int nr_processes(void); 1721da177e4SLinus Torvalds extern unsigned long nr_running(void); 1732ee507c4STim Chen extern bool single_task_running(void); 1741da177e4SLinus Torvalds extern unsigned long nr_iowait(void); 1758c215bd3SPeter Zijlstra extern unsigned long nr_iowait_cpu(int cpu); 176372ba8cbSMel Gorman extern void get_iowait_load(unsigned long *nr_waiters, unsigned long *load); 17769d25870SArjan van de Ven 1780f004f5aSPeter Zijlstra extern void calc_global_load(unsigned long ticks); 1793289bdb4SPeter Zijlstra 1803289bdb4SPeter Zijlstra #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON) 181525705d1SByungchul Park extern void update_cpu_load_nohz(int active); 1823289bdb4SPeter Zijlstra #else 183525705d1SByungchul Park static inline void update_cpu_load_nohz(int active) { } 1843289bdb4SPeter Zijlstra #endif 1851da177e4SLinus Torvalds 186b637a328SPaul E. McKenney extern void dump_cpu_task(int cpu); 187b637a328SPaul E. McKenney 18843ae34cbSIngo Molnar struct seq_file; 18943ae34cbSIngo Molnar struct cfs_rq; 1904cf86d77SIngo Molnar struct task_group; 19143ae34cbSIngo Molnar #ifdef CONFIG_SCHED_DEBUG 19243ae34cbSIngo Molnar extern void proc_sched_show_task(struct task_struct *p, struct seq_file *m); 19343ae34cbSIngo Molnar extern void proc_sched_set_task(struct task_struct *p); 19443ae34cbSIngo Molnar #endif 1951da177e4SLinus Torvalds 1964a8342d2SLinus Torvalds /* 1974a8342d2SLinus Torvalds * Task state bitmask. NOTE! These bits are also 1984a8342d2SLinus Torvalds * encoded in fs/proc/array.c: get_task_state(). 1994a8342d2SLinus Torvalds * 2004a8342d2SLinus Torvalds * We have two separate sets of flags: task->state 2014a8342d2SLinus Torvalds * is about runnability, while task->exit_state are 2024a8342d2SLinus Torvalds * about the task exiting. Confusing, but this way 2034a8342d2SLinus Torvalds * modifying one set can't modify the other one by 2044a8342d2SLinus Torvalds * mistake. 2054a8342d2SLinus Torvalds */ 2061da177e4SLinus Torvalds #define TASK_RUNNING 0 2071da177e4SLinus Torvalds #define TASK_INTERRUPTIBLE 1 2081da177e4SLinus Torvalds #define TASK_UNINTERRUPTIBLE 2 209f021a3c2SMatthew Wilcox #define __TASK_STOPPED 4 210f021a3c2SMatthew Wilcox #define __TASK_TRACED 8 2114a8342d2SLinus Torvalds /* in tsk->exit_state */ 212ad86622bSOleg Nesterov #define EXIT_DEAD 16 213ad86622bSOleg Nesterov #define EXIT_ZOMBIE 32 214abd50b39SOleg Nesterov #define EXIT_TRACE (EXIT_ZOMBIE | EXIT_DEAD) 2154a8342d2SLinus Torvalds /* in tsk->state again */ 216af927232SMike Galbraith #define TASK_DEAD 64 217f021a3c2SMatthew Wilcox #define TASK_WAKEKILL 128 218e9c84311SPeter Zijlstra #define TASK_WAKING 256 219f2530dc7SThomas Gleixner #define TASK_PARKED 512 22080ed87c8SPeter Zijlstra #define TASK_NOLOAD 1024 22180ed87c8SPeter Zijlstra #define TASK_STATE_MAX 2048 222f021a3c2SMatthew Wilcox 22380ed87c8SPeter Zijlstra #define TASK_STATE_TO_CHAR_STR "RSDTtXZxKWPN" 22473342151SPeter Zijlstra 225e1781538SPeter Zijlstra extern char ___assert_task_state[1 - 2*!!( 226e1781538SPeter Zijlstra sizeof(TASK_STATE_TO_CHAR_STR)-1 != ilog2(TASK_STATE_MAX)+1)]; 227f021a3c2SMatthew Wilcox 228f021a3c2SMatthew Wilcox /* Convenience macros for the sake of set_task_state */ 229f021a3c2SMatthew Wilcox #define TASK_KILLABLE (TASK_WAKEKILL | TASK_UNINTERRUPTIBLE) 230f021a3c2SMatthew Wilcox #define TASK_STOPPED (TASK_WAKEKILL | __TASK_STOPPED) 231f021a3c2SMatthew Wilcox #define TASK_TRACED (TASK_WAKEKILL | __TASK_TRACED) 2321da177e4SLinus Torvalds 23380ed87c8SPeter Zijlstra #define TASK_IDLE (TASK_UNINTERRUPTIBLE | TASK_NOLOAD) 23480ed87c8SPeter Zijlstra 23592a1f4bcSMatthew Wilcox /* Convenience macros for the sake of wake_up */ 23692a1f4bcSMatthew Wilcox #define TASK_NORMAL (TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE) 237f021a3c2SMatthew Wilcox #define TASK_ALL (TASK_NORMAL | __TASK_STOPPED | __TASK_TRACED) 23892a1f4bcSMatthew Wilcox 23992a1f4bcSMatthew Wilcox /* get_task_state() */ 24092a1f4bcSMatthew Wilcox #define TASK_REPORT (TASK_RUNNING | TASK_INTERRUPTIBLE | \ 241f021a3c2SMatthew Wilcox TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \ 24274e37200SOleg Nesterov __TASK_TRACED | EXIT_ZOMBIE | EXIT_DEAD) 24392a1f4bcSMatthew Wilcox 244f021a3c2SMatthew Wilcox #define task_is_traced(task) ((task->state & __TASK_TRACED) != 0) 245f021a3c2SMatthew Wilcox #define task_is_stopped(task) ((task->state & __TASK_STOPPED) != 0) 24692a1f4bcSMatthew Wilcox #define task_is_stopped_or_traced(task) \ 247f021a3c2SMatthew Wilcox ((task->state & (__TASK_STOPPED | __TASK_TRACED)) != 0) 24892a1f4bcSMatthew Wilcox #define task_contributes_to_load(task) \ 249e3c8ca83SNathan Lynch ((task->state & TASK_UNINTERRUPTIBLE) != 0 && \ 25080ed87c8SPeter Zijlstra (task->flags & PF_FROZEN) == 0 && \ 25180ed87c8SPeter Zijlstra (task->state & TASK_NOLOAD) == 0) 2521da177e4SLinus Torvalds 2538eb23b9fSPeter Zijlstra #ifdef CONFIG_DEBUG_ATOMIC_SLEEP 2548eb23b9fSPeter Zijlstra 2558eb23b9fSPeter Zijlstra #define __set_task_state(tsk, state_value) \ 2568eb23b9fSPeter Zijlstra do { \ 2578eb23b9fSPeter Zijlstra (tsk)->task_state_change = _THIS_IP_; \ 2588eb23b9fSPeter Zijlstra (tsk)->state = (state_value); \ 2598eb23b9fSPeter Zijlstra } while (0) 2608eb23b9fSPeter Zijlstra #define set_task_state(tsk, state_value) \ 2618eb23b9fSPeter Zijlstra do { \ 2628eb23b9fSPeter Zijlstra (tsk)->task_state_change = _THIS_IP_; \ 263b92b8b35SPeter Zijlstra smp_store_mb((tsk)->state, (state_value)); \ 2648eb23b9fSPeter Zijlstra } while (0) 2658eb23b9fSPeter Zijlstra 2668eb23b9fSPeter Zijlstra /* 2678eb23b9fSPeter Zijlstra * set_current_state() includes a barrier so that the write of current->state 2688eb23b9fSPeter Zijlstra * is correctly serialised wrt the caller's subsequent test of whether to 2698eb23b9fSPeter Zijlstra * actually sleep: 2708eb23b9fSPeter Zijlstra * 2718eb23b9fSPeter Zijlstra * set_current_state(TASK_UNINTERRUPTIBLE); 2728eb23b9fSPeter Zijlstra * if (do_i_need_to_sleep()) 2738eb23b9fSPeter Zijlstra * schedule(); 2748eb23b9fSPeter Zijlstra * 2758eb23b9fSPeter Zijlstra * If the caller does not need such serialisation then use __set_current_state() 2768eb23b9fSPeter Zijlstra */ 2778eb23b9fSPeter Zijlstra #define __set_current_state(state_value) \ 2788eb23b9fSPeter Zijlstra do { \ 2798eb23b9fSPeter Zijlstra current->task_state_change = _THIS_IP_; \ 2808eb23b9fSPeter Zijlstra current->state = (state_value); \ 2818eb23b9fSPeter Zijlstra } while (0) 2828eb23b9fSPeter Zijlstra #define set_current_state(state_value) \ 2838eb23b9fSPeter Zijlstra do { \ 2848eb23b9fSPeter Zijlstra current->task_state_change = _THIS_IP_; \ 285b92b8b35SPeter Zijlstra smp_store_mb(current->state, (state_value)); \ 2868eb23b9fSPeter Zijlstra } while (0) 2878eb23b9fSPeter Zijlstra 2888eb23b9fSPeter Zijlstra #else 2898eb23b9fSPeter Zijlstra 2901da177e4SLinus Torvalds #define __set_task_state(tsk, state_value) \ 2911da177e4SLinus Torvalds do { (tsk)->state = (state_value); } while (0) 2921da177e4SLinus Torvalds #define set_task_state(tsk, state_value) \ 293b92b8b35SPeter Zijlstra smp_store_mb((tsk)->state, (state_value)) 2941da177e4SLinus Torvalds 295498d0c57SAndrew Morton /* 296498d0c57SAndrew Morton * set_current_state() includes a barrier so that the write of current->state 297498d0c57SAndrew Morton * is correctly serialised wrt the caller's subsequent test of whether to 298498d0c57SAndrew Morton * actually sleep: 299498d0c57SAndrew Morton * 300498d0c57SAndrew Morton * set_current_state(TASK_UNINTERRUPTIBLE); 301498d0c57SAndrew Morton * if (do_i_need_to_sleep()) 302498d0c57SAndrew Morton * schedule(); 303498d0c57SAndrew Morton * 304498d0c57SAndrew Morton * If the caller does not need such serialisation then use __set_current_state() 305498d0c57SAndrew Morton */ 3061da177e4SLinus Torvalds #define __set_current_state(state_value) \ 3071da177e4SLinus Torvalds do { current->state = (state_value); } while (0) 3081da177e4SLinus Torvalds #define set_current_state(state_value) \ 309b92b8b35SPeter Zijlstra smp_store_mb(current->state, (state_value)) 3101da177e4SLinus Torvalds 3118eb23b9fSPeter Zijlstra #endif 3128eb23b9fSPeter Zijlstra 3131da177e4SLinus Torvalds /* Task command name length */ 3141da177e4SLinus Torvalds #define TASK_COMM_LEN 16 3151da177e4SLinus Torvalds 3161da177e4SLinus Torvalds #include <linux/spinlock.h> 3171da177e4SLinus Torvalds 3181da177e4SLinus Torvalds /* 3191da177e4SLinus Torvalds * This serializes "schedule()" and also protects 3201da177e4SLinus Torvalds * the run-queue from deletions/modifications (but 3211da177e4SLinus Torvalds * _adding_ to the beginning of the run-queue has 3221da177e4SLinus Torvalds * a separate lock). 3231da177e4SLinus Torvalds */ 3241da177e4SLinus Torvalds extern rwlock_t tasklist_lock; 3251da177e4SLinus Torvalds extern spinlock_t mmlist_lock; 3261da177e4SLinus Torvalds 32736c8b586SIngo Molnar struct task_struct; 3281da177e4SLinus Torvalds 329db1466b3SPaul E. McKenney #ifdef CONFIG_PROVE_RCU 330db1466b3SPaul E. McKenney extern int lockdep_tasklist_lock_is_held(void); 331db1466b3SPaul E. McKenney #endif /* #ifdef CONFIG_PROVE_RCU */ 332db1466b3SPaul E. McKenney 3331da177e4SLinus Torvalds extern void sched_init(void); 3341da177e4SLinus Torvalds extern void sched_init_smp(void); 3352d07b255SHarvey Harrison extern asmlinkage void schedule_tail(struct task_struct *prev); 33636c8b586SIngo Molnar extern void init_idle(struct task_struct *idle, int cpu); 3371df21055SIngo Molnar extern void init_idle_bootup_task(struct task_struct *idle); 3381da177e4SLinus Torvalds 3393fa0818bSRik van Riel extern cpumask_var_t cpu_isolated_map; 3403fa0818bSRik van Riel 34189f19f04SAndrew Morton extern int runqueue_is_locked(int cpu); 342017730c1SIngo Molnar 3433451d024SFrederic Weisbecker #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON) 344c1cc017cSAlex Shi extern void nohz_balance_enter_idle(int cpu); 34569e1e811SSuresh Siddha extern void set_cpu_sd_state_idle(void); 346bc7a34b8SThomas Gleixner extern int get_nohz_timer_target(void); 34746cb4b7cSSiddha, Suresh B #else 348c1cc017cSAlex Shi static inline void nohz_balance_enter_idle(int cpu) { } 349fdaabd80SPeter Zijlstra static inline void set_cpu_sd_state_idle(void) { } 35046cb4b7cSSiddha, Suresh B #endif 3511da177e4SLinus Torvalds 352e59e2ae2SIngo Molnar /* 35339bc89fdSIngo Molnar * Only dump TASK_* tasks. (0 for all tasks) 354e59e2ae2SIngo Molnar */ 355e59e2ae2SIngo Molnar extern void show_state_filter(unsigned long state_filter); 356e59e2ae2SIngo Molnar 357e59e2ae2SIngo Molnar static inline void show_state(void) 358e59e2ae2SIngo Molnar { 35939bc89fdSIngo Molnar show_state_filter(0); 360e59e2ae2SIngo Molnar } 361e59e2ae2SIngo Molnar 3621da177e4SLinus Torvalds extern void show_regs(struct pt_regs *); 3631da177e4SLinus Torvalds 3641da177e4SLinus Torvalds /* 3651da177e4SLinus Torvalds * TASK is a pointer to the task whose backtrace we want to see (or NULL for current 3661da177e4SLinus Torvalds * task), SP is the stack pointer of the first frame that should be shown in the back 3671da177e4SLinus Torvalds * trace (or NULL if the entire call-chain of the task should be shown). 3681da177e4SLinus Torvalds */ 3691da177e4SLinus Torvalds extern void show_stack(struct task_struct *task, unsigned long *sp); 3701da177e4SLinus Torvalds 3711da177e4SLinus Torvalds extern void cpu_init (void); 3721da177e4SLinus Torvalds extern void trap_init(void); 3731da177e4SLinus Torvalds extern void update_process_times(int user); 3741da177e4SLinus Torvalds extern void scheduler_tick(void); 3751da177e4SLinus Torvalds 37682a1fcb9SIngo Molnar extern void sched_show_task(struct task_struct *p); 37782a1fcb9SIngo Molnar 37819cc36c0SFrederic Weisbecker #ifdef CONFIG_LOCKUP_DETECTOR 37903e0d461STejun Heo extern void touch_softlockup_watchdog_sched(void); 3808446f1d3SIngo Molnar extern void touch_softlockup_watchdog(void); 381d6ad3e28SJason Wessel extern void touch_softlockup_watchdog_sync(void); 38204c9167fSJeremy Fitzhardinge extern void touch_all_softlockup_watchdogs(void); 383332fbdbcSDon Zickus extern int proc_dowatchdog_thresh(struct ctl_table *table, int write, 3848d65af78SAlexey Dobriyan void __user *buffer, 385baf48f65SMandeep Singh Baines size_t *lenp, loff_t *ppos); 3869c44bc03SIngo Molnar extern unsigned int softlockup_panic; 387ac1f5912SDon Zickus extern unsigned int hardlockup_panic; 388004417a6SPeter Zijlstra void lockup_detector_init(void); 3898446f1d3SIngo Molnar #else 39003e0d461STejun Heo static inline void touch_softlockup_watchdog_sched(void) 39103e0d461STejun Heo { 39203e0d461STejun Heo } 3938446f1d3SIngo Molnar static inline void touch_softlockup_watchdog(void) 3948446f1d3SIngo Molnar { 3958446f1d3SIngo Molnar } 396d6ad3e28SJason Wessel static inline void touch_softlockup_watchdog_sync(void) 397d6ad3e28SJason Wessel { 398d6ad3e28SJason Wessel } 39904c9167fSJeremy Fitzhardinge static inline void touch_all_softlockup_watchdogs(void) 40004c9167fSJeremy Fitzhardinge { 40104c9167fSJeremy Fitzhardinge } 402004417a6SPeter Zijlstra static inline void lockup_detector_init(void) 403004417a6SPeter Zijlstra { 404004417a6SPeter Zijlstra } 4058446f1d3SIngo Molnar #endif 4068446f1d3SIngo Molnar 4078b414521SMarcelo Tosatti #ifdef CONFIG_DETECT_HUNG_TASK 4088b414521SMarcelo Tosatti void reset_hung_task_detector(void); 4098b414521SMarcelo Tosatti #else 4108b414521SMarcelo Tosatti static inline void reset_hung_task_detector(void) 4118b414521SMarcelo Tosatti { 4128b414521SMarcelo Tosatti } 4138b414521SMarcelo Tosatti #endif 4148b414521SMarcelo Tosatti 4151da177e4SLinus Torvalds /* Attach to any functions which should be ignored in wchan output. */ 4161da177e4SLinus Torvalds #define __sched __attribute__((__section__(".sched.text"))) 417deaf2227SIngo Molnar 418deaf2227SIngo Molnar /* Linker adds these: start and end of __sched functions */ 419deaf2227SIngo Molnar extern char __sched_text_start[], __sched_text_end[]; 420deaf2227SIngo Molnar 4211da177e4SLinus Torvalds /* Is this address in the __sched functions? */ 4221da177e4SLinus Torvalds extern int in_sched_functions(unsigned long addr); 4231da177e4SLinus Torvalds 4241da177e4SLinus Torvalds #define MAX_SCHEDULE_TIMEOUT LONG_MAX 425b3c97528SHarvey Harrison extern signed long schedule_timeout(signed long timeout); 42664ed93a2SNishanth Aravamudan extern signed long schedule_timeout_interruptible(signed long timeout); 427294d5cc2SMatthew Wilcox extern signed long schedule_timeout_killable(signed long timeout); 42864ed93a2SNishanth Aravamudan extern signed long schedule_timeout_uninterruptible(signed long timeout); 42969b27bafSAndrew Morton extern signed long schedule_timeout_idle(signed long timeout); 4301da177e4SLinus Torvalds asmlinkage void schedule(void); 431c5491ea7SThomas Gleixner extern void schedule_preempt_disabled(void); 4321da177e4SLinus Torvalds 4339cff8adeSNeilBrown extern long io_schedule_timeout(long timeout); 4349cff8adeSNeilBrown 4359cff8adeSNeilBrown static inline void io_schedule(void) 4369cff8adeSNeilBrown { 4379cff8adeSNeilBrown io_schedule_timeout(MAX_SCHEDULE_TIMEOUT); 4389cff8adeSNeilBrown } 4399cff8adeSNeilBrown 440ab516013SSerge E. Hallyn struct nsproxy; 441acce292cSCedric Le Goater struct user_namespace; 4421da177e4SLinus Torvalds 443efc1a3b1SDavid Howells #ifdef CONFIG_MMU 444efc1a3b1SDavid Howells extern void arch_pick_mmap_layout(struct mm_struct *mm); 4451da177e4SLinus Torvalds extern unsigned long 4461da177e4SLinus Torvalds arch_get_unmapped_area(struct file *, unsigned long, unsigned long, 4471da177e4SLinus Torvalds unsigned long, unsigned long); 4481da177e4SLinus Torvalds extern unsigned long 4491da177e4SLinus Torvalds arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr, 4501da177e4SLinus Torvalds unsigned long len, unsigned long pgoff, 4511da177e4SLinus Torvalds unsigned long flags); 452efc1a3b1SDavid Howells #else 453efc1a3b1SDavid Howells static inline void arch_pick_mmap_layout(struct mm_struct *mm) {} 454efc1a3b1SDavid Howells #endif 4551da177e4SLinus Torvalds 456d049f74fSKees Cook #define SUID_DUMP_DISABLE 0 /* No setuid dumping */ 457d049f74fSKees Cook #define SUID_DUMP_USER 1 /* Dump as user of process */ 458d049f74fSKees Cook #define SUID_DUMP_ROOT 2 /* Dump as root */ 459d049f74fSKees Cook 4606c5d5238SKawai, Hidehiro /* mm flags */ 461f8af4da3SHugh Dickins 4627288e118SOleg Nesterov /* for SUID_DUMP_* above */ 4633cb4a0bbSKawai, Hidehiro #define MMF_DUMPABLE_BITS 2 464f8af4da3SHugh Dickins #define MMF_DUMPABLE_MASK ((1 << MMF_DUMPABLE_BITS) - 1) 4653cb4a0bbSKawai, Hidehiro 466942be387SOleg Nesterov extern void set_dumpable(struct mm_struct *mm, int value); 467942be387SOleg Nesterov /* 468942be387SOleg Nesterov * This returns the actual value of the suid_dumpable flag. For things 469942be387SOleg Nesterov * that are using this for checking for privilege transitions, it must 470942be387SOleg Nesterov * test against SUID_DUMP_USER rather than treating it as a boolean 471942be387SOleg Nesterov * value. 472942be387SOleg Nesterov */ 473942be387SOleg Nesterov static inline int __get_dumpable(unsigned long mm_flags) 474942be387SOleg Nesterov { 475942be387SOleg Nesterov return mm_flags & MMF_DUMPABLE_MASK; 476942be387SOleg Nesterov } 477942be387SOleg Nesterov 478942be387SOleg Nesterov static inline int get_dumpable(struct mm_struct *mm) 479942be387SOleg Nesterov { 480942be387SOleg Nesterov return __get_dumpable(mm->flags); 481942be387SOleg Nesterov } 482942be387SOleg Nesterov 4833cb4a0bbSKawai, Hidehiro /* coredump filter bits */ 4843cb4a0bbSKawai, Hidehiro #define MMF_DUMP_ANON_PRIVATE 2 4853cb4a0bbSKawai, Hidehiro #define MMF_DUMP_ANON_SHARED 3 4863cb4a0bbSKawai, Hidehiro #define MMF_DUMP_MAPPED_PRIVATE 4 4873cb4a0bbSKawai, Hidehiro #define MMF_DUMP_MAPPED_SHARED 5 48882df3973SRoland McGrath #define MMF_DUMP_ELF_HEADERS 6 489e575f111SKOSAKI Motohiro #define MMF_DUMP_HUGETLB_PRIVATE 7 490e575f111SKOSAKI Motohiro #define MMF_DUMP_HUGETLB_SHARED 8 4915037835cSRoss Zwisler #define MMF_DUMP_DAX_PRIVATE 9 4925037835cSRoss Zwisler #define MMF_DUMP_DAX_SHARED 10 493f8af4da3SHugh Dickins 4943cb4a0bbSKawai, Hidehiro #define MMF_DUMP_FILTER_SHIFT MMF_DUMPABLE_BITS 4955037835cSRoss Zwisler #define MMF_DUMP_FILTER_BITS 9 4963cb4a0bbSKawai, Hidehiro #define MMF_DUMP_FILTER_MASK \ 4973cb4a0bbSKawai, Hidehiro (((1 << MMF_DUMP_FILTER_BITS) - 1) << MMF_DUMP_FILTER_SHIFT) 4983cb4a0bbSKawai, Hidehiro #define MMF_DUMP_FILTER_DEFAULT \ 499e575f111SKOSAKI Motohiro ((1 << MMF_DUMP_ANON_PRIVATE) | (1 << MMF_DUMP_ANON_SHARED) |\ 500656eb2cdSRoland McGrath (1 << MMF_DUMP_HUGETLB_PRIVATE) | MMF_DUMP_MASK_DEFAULT_ELF) 501656eb2cdSRoland McGrath 502656eb2cdSRoland McGrath #ifdef CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS 503656eb2cdSRoland McGrath # define MMF_DUMP_MASK_DEFAULT_ELF (1 << MMF_DUMP_ELF_HEADERS) 504656eb2cdSRoland McGrath #else 505656eb2cdSRoland McGrath # define MMF_DUMP_MASK_DEFAULT_ELF 0 506656eb2cdSRoland McGrath #endif 507f8af4da3SHugh Dickins /* leave room for more dump flags */ 508f8af4da3SHugh Dickins #define MMF_VM_MERGEABLE 16 /* KSM may merge identical pages */ 509ba76149fSAndrea Arcangeli #define MMF_VM_HUGEPAGE 17 /* set when VM_HUGEPAGE is set on vma */ 510bafb282dSKonstantin Khlebnikov #define MMF_EXE_FILE_CHANGED 18 /* see prctl_set_mm_exe_file() */ 511f8af4da3SHugh Dickins 5129f68f672SOleg Nesterov #define MMF_HAS_UPROBES 19 /* has uprobes */ 5139f68f672SOleg Nesterov #define MMF_RECALC_UPROBES 20 /* MMF_HAS_UPROBES can be wrong */ 514f8ac4ec9SOleg Nesterov 515f8af4da3SHugh Dickins #define MMF_INIT_MASK (MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK) 5166c5d5238SKawai, Hidehiro 5171da177e4SLinus Torvalds struct sighand_struct { 5181da177e4SLinus Torvalds atomic_t count; 5191da177e4SLinus Torvalds struct k_sigaction action[_NSIG]; 5201da177e4SLinus Torvalds spinlock_t siglock; 521b8fceee1SDavide Libenzi wait_queue_head_t signalfd_wqh; 5221da177e4SLinus Torvalds }; 5231da177e4SLinus Torvalds 5240e464814SKaiGai Kohei struct pacct_struct { 525f6ec29a4SKaiGai Kohei int ac_flag; 526f6ec29a4SKaiGai Kohei long ac_exitcode; 5270e464814SKaiGai Kohei unsigned long ac_mem; 52877787bfbSKaiGai Kohei cputime_t ac_utime, ac_stime; 52977787bfbSKaiGai Kohei unsigned long ac_minflt, ac_majflt; 5300e464814SKaiGai Kohei }; 5310e464814SKaiGai Kohei 53242c4ab41SStanislaw Gruszka struct cpu_itimer { 53342c4ab41SStanislaw Gruszka cputime_t expires; 53442c4ab41SStanislaw Gruszka cputime_t incr; 5358356b5f9SStanislaw Gruszka u32 error; 5368356b5f9SStanislaw Gruszka u32 incr_error; 53742c4ab41SStanislaw Gruszka }; 53842c4ab41SStanislaw Gruszka 539f06febc9SFrank Mayhar /** 5409d7fb042SPeter Zijlstra * struct prev_cputime - snaphsot of system and user cputime 541d37f761dSFrederic Weisbecker * @utime: time spent in user mode 542d37f761dSFrederic Weisbecker * @stime: time spent in system mode 5439d7fb042SPeter Zijlstra * @lock: protects the above two fields 544d37f761dSFrederic Weisbecker * 5459d7fb042SPeter Zijlstra * Stores previous user/system time values such that we can guarantee 5469d7fb042SPeter Zijlstra * monotonicity. 547d37f761dSFrederic Weisbecker */ 5489d7fb042SPeter Zijlstra struct prev_cputime { 5499d7fb042SPeter Zijlstra #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE 550d37f761dSFrederic Weisbecker cputime_t utime; 551d37f761dSFrederic Weisbecker cputime_t stime; 5529d7fb042SPeter Zijlstra raw_spinlock_t lock; 5539d7fb042SPeter Zijlstra #endif 554d37f761dSFrederic Weisbecker }; 555d37f761dSFrederic Weisbecker 5569d7fb042SPeter Zijlstra static inline void prev_cputime_init(struct prev_cputime *prev) 5579d7fb042SPeter Zijlstra { 5589d7fb042SPeter Zijlstra #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE 5599d7fb042SPeter Zijlstra prev->utime = prev->stime = 0; 5609d7fb042SPeter Zijlstra raw_spin_lock_init(&prev->lock); 5619d7fb042SPeter Zijlstra #endif 5629d7fb042SPeter Zijlstra } 5639d7fb042SPeter Zijlstra 564d37f761dSFrederic Weisbecker /** 565f06febc9SFrank Mayhar * struct task_cputime - collected CPU time counts 566f06febc9SFrank Mayhar * @utime: time spent in user mode, in &cputime_t units 567f06febc9SFrank Mayhar * @stime: time spent in kernel mode, in &cputime_t units 568f06febc9SFrank Mayhar * @sum_exec_runtime: total time spent on the CPU, in nanoseconds 569f06febc9SFrank Mayhar * 5709d7fb042SPeter Zijlstra * This structure groups together three kinds of CPU time that are tracked for 5719d7fb042SPeter Zijlstra * threads and thread groups. Most things considering CPU time want to group 5729d7fb042SPeter Zijlstra * these counts together and treat all three of them in parallel. 573f06febc9SFrank Mayhar */ 574f06febc9SFrank Mayhar struct task_cputime { 575f06febc9SFrank Mayhar cputime_t utime; 576f06febc9SFrank Mayhar cputime_t stime; 577f06febc9SFrank Mayhar unsigned long long sum_exec_runtime; 578f06febc9SFrank Mayhar }; 5799d7fb042SPeter Zijlstra 580f06febc9SFrank Mayhar /* Alternate field names when used to cache expirations. */ 581f06febc9SFrank Mayhar #define virt_exp utime 5829d7fb042SPeter Zijlstra #define prof_exp stime 583f06febc9SFrank Mayhar #define sched_exp sum_exec_runtime 584f06febc9SFrank Mayhar 5854cd4c1b4SPeter Zijlstra #define INIT_CPUTIME \ 5864cd4c1b4SPeter Zijlstra (struct task_cputime) { \ 58764861634SMartin Schwidefsky .utime = 0, \ 58864861634SMartin Schwidefsky .stime = 0, \ 5894cd4c1b4SPeter Zijlstra .sum_exec_runtime = 0, \ 5904cd4c1b4SPeter Zijlstra } 5914cd4c1b4SPeter Zijlstra 592971e8a98SJason Low /* 593971e8a98SJason Low * This is the atomic variant of task_cputime, which can be used for 594971e8a98SJason Low * storing and updating task_cputime statistics without locking. 595971e8a98SJason Low */ 596971e8a98SJason Low struct task_cputime_atomic { 597971e8a98SJason Low atomic64_t utime; 598971e8a98SJason Low atomic64_t stime; 599971e8a98SJason Low atomic64_t sum_exec_runtime; 600971e8a98SJason Low }; 601971e8a98SJason Low 602971e8a98SJason Low #define INIT_CPUTIME_ATOMIC \ 603971e8a98SJason Low (struct task_cputime_atomic) { \ 604971e8a98SJason Low .utime = ATOMIC64_INIT(0), \ 605971e8a98SJason Low .stime = ATOMIC64_INIT(0), \ 606971e8a98SJason Low .sum_exec_runtime = ATOMIC64_INIT(0), \ 607971e8a98SJason Low } 608971e8a98SJason Low 609609ca066SPeter Zijlstra #define PREEMPT_DISABLED (PREEMPT_DISABLE_OFFSET + PREEMPT_ENABLED) 610a233f112SPeter Zijlstra 611c99e6efeSPeter Zijlstra /* 61287dcbc06SPeter Zijlstra * Disable preemption until the scheduler is running -- use an unconditional 61387dcbc06SPeter Zijlstra * value so that it also works on !PREEMPT_COUNT kernels. 614d86ee480SPeter Zijlstra * 61587dcbc06SPeter Zijlstra * Reset by start_kernel()->sched_init()->init_idle()->init_idle_preempt_count(). 616c99e6efeSPeter Zijlstra */ 61787dcbc06SPeter Zijlstra #define INIT_PREEMPT_COUNT PREEMPT_OFFSET 618c99e6efeSPeter Zijlstra 619609ca066SPeter Zijlstra /* 620609ca066SPeter Zijlstra * Initial preempt_count value; reflects the preempt_count schedule invariant 621609ca066SPeter Zijlstra * which states that during context switches: 622609ca066SPeter Zijlstra * 623609ca066SPeter Zijlstra * preempt_count() == 2*PREEMPT_DISABLE_OFFSET 624609ca066SPeter Zijlstra * 625609ca066SPeter Zijlstra * Note: PREEMPT_DISABLE_OFFSET is 0 for !PREEMPT_COUNT kernels. 626609ca066SPeter Zijlstra * Note: See finish_task_switch(). 627609ca066SPeter Zijlstra */ 628609ca066SPeter Zijlstra #define FORK_PREEMPT_COUNT (2*PREEMPT_DISABLE_OFFSET + PREEMPT_ENABLED) 6294cd4c1b4SPeter Zijlstra 630f06febc9SFrank Mayhar /** 631f06febc9SFrank Mayhar * struct thread_group_cputimer - thread group interval timer counts 632920ce39fSJason Low * @cputime_atomic: atomic thread group interval timers. 633d5c373ebSJason Low * @running: true when there are timers running and 634d5c373ebSJason Low * @cputime_atomic receives updates. 635c8d75aa4SJason Low * @checking_timer: true when a thread in the group is in the 636c8d75aa4SJason Low * process of checking for thread group timers. 637f06febc9SFrank Mayhar * 638f06febc9SFrank Mayhar * This structure contains the version of task_cputime, above, that is 6394cd4c1b4SPeter Zijlstra * used for thread group CPU timer calculations. 640f06febc9SFrank Mayhar */ 6414cd4c1b4SPeter Zijlstra struct thread_group_cputimer { 64271107445SJason Low struct task_cputime_atomic cputime_atomic; 643d5c373ebSJason Low bool running; 644c8d75aa4SJason Low bool checking_timer; 645f06febc9SFrank Mayhar }; 646f06febc9SFrank Mayhar 6474714d1d3SBen Blum #include <linux/rwsem.h> 6485091faa4SMike Galbraith struct autogroup; 6495091faa4SMike Galbraith 6501da177e4SLinus Torvalds /* 651e815f0a8SJonathan Neuschäfer * NOTE! "signal_struct" does not have its own 6521da177e4SLinus Torvalds * locking, because a shared signal_struct always 6531da177e4SLinus Torvalds * implies a shared sighand_struct, so locking 6541da177e4SLinus Torvalds * sighand_struct is always a proper superset of 6551da177e4SLinus Torvalds * the locking of signal_struct. 6561da177e4SLinus Torvalds */ 6571da177e4SLinus Torvalds struct signal_struct { 658ea6d290cSOleg Nesterov atomic_t sigcnt; 6591da177e4SLinus Torvalds atomic_t live; 660b3ac022cSOleg Nesterov int nr_threads; 6610c740d0aSOleg Nesterov struct list_head thread_head; 6621da177e4SLinus Torvalds 6631da177e4SLinus Torvalds wait_queue_head_t wait_chldexit; /* for wait4() */ 6641da177e4SLinus Torvalds 6651da177e4SLinus Torvalds /* current thread group signal load-balancing target: */ 66636c8b586SIngo Molnar struct task_struct *curr_target; 6671da177e4SLinus Torvalds 6681da177e4SLinus Torvalds /* shared signal handling: */ 6691da177e4SLinus Torvalds struct sigpending shared_pending; 6701da177e4SLinus Torvalds 6711da177e4SLinus Torvalds /* thread group exit support */ 6721da177e4SLinus Torvalds int group_exit_code; 6731da177e4SLinus Torvalds /* overloaded: 6741da177e4SLinus Torvalds * - notify group_exit_task when ->count is equal to notify_count 6751da177e4SLinus Torvalds * - everyone except group_exit_task is stopped during signal delivery 6761da177e4SLinus Torvalds * of fatal signals, group_exit_task processes the signal. 6771da177e4SLinus Torvalds */ 6781da177e4SLinus Torvalds int notify_count; 67907dd20e0SRichard Kennedy struct task_struct *group_exit_task; 6801da177e4SLinus Torvalds 6811da177e4SLinus Torvalds /* thread group stop support, overloads group_exit_code too */ 6821da177e4SLinus Torvalds int group_stop_count; 6831da177e4SLinus Torvalds unsigned int flags; /* see SIGNAL_* flags below */ 6841da177e4SLinus Torvalds 685ebec18a6SLennart Poettering /* 686ebec18a6SLennart Poettering * PR_SET_CHILD_SUBREAPER marks a process, like a service 687ebec18a6SLennart Poettering * manager, to re-parent orphan (double-forking) child processes 688ebec18a6SLennart Poettering * to this process instead of 'init'. The service manager is 689ebec18a6SLennart Poettering * able to receive SIGCHLD signals and is able to investigate 690ebec18a6SLennart Poettering * the process until it calls wait(). All children of this 691ebec18a6SLennart Poettering * process will inherit a flag if they should look for a 692ebec18a6SLennart Poettering * child_subreaper process at exit. 693ebec18a6SLennart Poettering */ 694ebec18a6SLennart Poettering unsigned int is_child_subreaper:1; 695ebec18a6SLennart Poettering unsigned int has_child_subreaper:1; 696ebec18a6SLennart Poettering 6971da177e4SLinus Torvalds /* POSIX.1b Interval Timers */ 6985ed67f05SPavel Emelyanov int posix_timer_id; 6991da177e4SLinus Torvalds struct list_head posix_timers; 7001da177e4SLinus Torvalds 7011da177e4SLinus Torvalds /* ITIMER_REAL timer for the process */ 7022ff678b8SThomas Gleixner struct hrtimer real_timer; 703fea9d175SOleg Nesterov struct pid *leader_pid; 7042ff678b8SThomas Gleixner ktime_t it_real_incr; 7051da177e4SLinus Torvalds 70642c4ab41SStanislaw Gruszka /* 70742c4ab41SStanislaw Gruszka * ITIMER_PROF and ITIMER_VIRTUAL timers for the process, we use 70842c4ab41SStanislaw Gruszka * CPUCLOCK_PROF and CPUCLOCK_VIRT for indexing array as these 70942c4ab41SStanislaw Gruszka * values are defined to 0 and 1 respectively 71042c4ab41SStanislaw Gruszka */ 71142c4ab41SStanislaw Gruszka struct cpu_itimer it[2]; 7121da177e4SLinus Torvalds 713f06febc9SFrank Mayhar /* 7144cd4c1b4SPeter Zijlstra * Thread group totals for process CPU timers. 7154cd4c1b4SPeter Zijlstra * See thread_group_cputimer(), et al, for details. 716f06febc9SFrank Mayhar */ 7174cd4c1b4SPeter Zijlstra struct thread_group_cputimer cputimer; 718f06febc9SFrank Mayhar 719f06febc9SFrank Mayhar /* Earliest-expiration cache. */ 720f06febc9SFrank Mayhar struct task_cputime cputime_expires; 721f06febc9SFrank Mayhar 722d027d45dSFrederic Weisbecker #ifdef CONFIG_NO_HZ_FULL 723*f009a7a7SFrederic Weisbecker atomic_t tick_dep_mask; 724d027d45dSFrederic Weisbecker #endif 725d027d45dSFrederic Weisbecker 726f06febc9SFrank Mayhar struct list_head cpu_timers[3]; 727f06febc9SFrank Mayhar 728ab521dc0SEric W. Biederman struct pid *tty_old_pgrp; 7291ec320afSCedric Le Goater 7301da177e4SLinus Torvalds /* boolean value for session group leader */ 7311da177e4SLinus Torvalds int leader; 7321da177e4SLinus Torvalds 7331da177e4SLinus Torvalds struct tty_struct *tty; /* NULL if no tty */ 7341da177e4SLinus Torvalds 7355091faa4SMike Galbraith #ifdef CONFIG_SCHED_AUTOGROUP 7365091faa4SMike Galbraith struct autogroup *autogroup; 7375091faa4SMike Galbraith #endif 7381da177e4SLinus Torvalds /* 7391da177e4SLinus Torvalds * Cumulative resource counters for dead threads in the group, 7401da177e4SLinus Torvalds * and for reaped dead child processes forked by this group. 7411da177e4SLinus Torvalds * Live threads maintain their own counters and add to these 7421da177e4SLinus Torvalds * in __exit_signal, except for the group leader. 7431da177e4SLinus Torvalds */ 744e78c3496SRik van Riel seqlock_t stats_lock; 74532bd671dSPeter Zijlstra cputime_t utime, stime, cutime, cstime; 7469ac52315SLaurent Vivier cputime_t gtime; 7479ac52315SLaurent Vivier cputime_t cgtime; 7489d7fb042SPeter Zijlstra struct prev_cputime prev_cputime; 7491da177e4SLinus Torvalds unsigned long nvcsw, nivcsw, cnvcsw, cnivcsw; 7501da177e4SLinus Torvalds unsigned long min_flt, maj_flt, cmin_flt, cmaj_flt; 7516eaeeabaSEric Dumazet unsigned long inblock, oublock, cinblock, coublock; 7521f10206cSJiri Pirko unsigned long maxrss, cmaxrss; 753940389b8SAndrea Righi struct task_io_accounting ioac; 7541da177e4SLinus Torvalds 7551da177e4SLinus Torvalds /* 75632bd671dSPeter Zijlstra * Cumulative ns of schedule CPU time fo dead threads in the 75732bd671dSPeter Zijlstra * group, not including a zombie group leader, (This only differs 75832bd671dSPeter Zijlstra * from jiffies_to_ns(utime + stime) if sched_clock uses something 75932bd671dSPeter Zijlstra * other than jiffies.) 76032bd671dSPeter Zijlstra */ 76132bd671dSPeter Zijlstra unsigned long long sum_sched_runtime; 76232bd671dSPeter Zijlstra 76332bd671dSPeter Zijlstra /* 7641da177e4SLinus Torvalds * We don't bother to synchronize most readers of this at all, 7651da177e4SLinus Torvalds * because there is no reader checking a limit that actually needs 7661da177e4SLinus Torvalds * to get both rlim_cur and rlim_max atomically, and either one 7671da177e4SLinus Torvalds * alone is a single word that can safely be read normally. 7681da177e4SLinus Torvalds * getrlimit/setrlimit use task_lock(current->group_leader) to 7691da177e4SLinus Torvalds * protect this instead of the siglock, because they really 7701da177e4SLinus Torvalds * have no need to disable irqs. 7711da177e4SLinus Torvalds */ 7721da177e4SLinus Torvalds struct rlimit rlim[RLIM_NLIMITS]; 7731da177e4SLinus Torvalds 7740e464814SKaiGai Kohei #ifdef CONFIG_BSD_PROCESS_ACCT 7750e464814SKaiGai Kohei struct pacct_struct pacct; /* per-process accounting information */ 7760e464814SKaiGai Kohei #endif 777ad4ecbcbSShailabh Nagar #ifdef CONFIG_TASKSTATS 778ad4ecbcbSShailabh Nagar struct taskstats *stats; 779ad4ecbcbSShailabh Nagar #endif 780522ed776SMiloslav Trmac #ifdef CONFIG_AUDIT 781522ed776SMiloslav Trmac unsigned audit_tty; 782522ed776SMiloslav Trmac struct tty_audit_buf *tty_audit_buf; 783522ed776SMiloslav Trmac #endif 78428b83c51SKOSAKI Motohiro 785e1e12d2fSDavid Rientjes oom_flags_t oom_flags; 786a9c58b90SDavid Rientjes short oom_score_adj; /* OOM kill score adjustment */ 787a9c58b90SDavid Rientjes short oom_score_adj_min; /* OOM kill score adjustment min value. 788dabb16f6SMandeep Singh Baines * Only settable by CAP_SYS_RESOURCE. */ 7899b1bf12dSKOSAKI Motohiro 7909b1bf12dSKOSAKI Motohiro struct mutex cred_guard_mutex; /* guard against foreign influences on 7919b1bf12dSKOSAKI Motohiro * credential calculations 7929b1bf12dSKOSAKI Motohiro * (notably. ptrace) */ 7931da177e4SLinus Torvalds }; 7941da177e4SLinus Torvalds 7951da177e4SLinus Torvalds /* 7961da177e4SLinus Torvalds * Bits in flags field of signal_struct. 7971da177e4SLinus Torvalds */ 7981da177e4SLinus Torvalds #define SIGNAL_STOP_STOPPED 0x00000001 /* job control stop in effect */ 799ee77f075SOleg Nesterov #define SIGNAL_STOP_CONTINUED 0x00000002 /* SIGCONT since WCONTINUED reap */ 800ee77f075SOleg Nesterov #define SIGNAL_GROUP_EXIT 0x00000004 /* group exit in progress */ 801403bad72SOleg Nesterov #define SIGNAL_GROUP_COREDUMP 0x00000008 /* coredump in progress */ 802e4420551SOleg Nesterov /* 803e4420551SOleg Nesterov * Pending notifications to parent. 804e4420551SOleg Nesterov */ 805e4420551SOleg Nesterov #define SIGNAL_CLD_STOPPED 0x00000010 806e4420551SOleg Nesterov #define SIGNAL_CLD_CONTINUED 0x00000020 807e4420551SOleg Nesterov #define SIGNAL_CLD_MASK (SIGNAL_CLD_STOPPED|SIGNAL_CLD_CONTINUED) 8081da177e4SLinus Torvalds 809fae5fa44SOleg Nesterov #define SIGNAL_UNKILLABLE 0x00000040 /* for init: ignore fatal signals */ 810fae5fa44SOleg Nesterov 811ed5d2cacSOleg Nesterov /* If true, all threads except ->group_exit_task have pending SIGKILL */ 812ed5d2cacSOleg Nesterov static inline int signal_group_exit(const struct signal_struct *sig) 813ed5d2cacSOleg Nesterov { 814ed5d2cacSOleg Nesterov return (sig->flags & SIGNAL_GROUP_EXIT) || 815ed5d2cacSOleg Nesterov (sig->group_exit_task != NULL); 816ed5d2cacSOleg Nesterov } 817ed5d2cacSOleg Nesterov 8181da177e4SLinus Torvalds /* 8191da177e4SLinus Torvalds * Some day this will be a full-fledged user tracking system.. 8201da177e4SLinus Torvalds */ 8211da177e4SLinus Torvalds struct user_struct { 8221da177e4SLinus Torvalds atomic_t __count; /* reference count */ 8231da177e4SLinus Torvalds atomic_t processes; /* How many processes does this user have? */ 8241da177e4SLinus Torvalds atomic_t sigpending; /* How many pending signals does this user have? */ 8252d9048e2SAmy Griffis #ifdef CONFIG_INOTIFY_USER 8260eeca283SRobert Love atomic_t inotify_watches; /* How many inotify watches does this user have? */ 8270eeca283SRobert Love atomic_t inotify_devs; /* How many inotify devs does this user have opened? */ 8280eeca283SRobert Love #endif 8294afeff85SEric Paris #ifdef CONFIG_FANOTIFY 8304afeff85SEric Paris atomic_t fanotify_listeners; 8314afeff85SEric Paris #endif 8327ef9964eSDavide Libenzi #ifdef CONFIG_EPOLL 83352bd19f7SRobin Holt atomic_long_t epoll_watches; /* The number of file descriptors currently watched */ 8347ef9964eSDavide Libenzi #endif 835970a8645SAlexey Dobriyan #ifdef CONFIG_POSIX_MQUEUE 8361da177e4SLinus Torvalds /* protected by mq_lock */ 8371da177e4SLinus Torvalds unsigned long mq_bytes; /* How many bytes can be allocated to mqueue? */ 838970a8645SAlexey Dobriyan #endif 8391da177e4SLinus Torvalds unsigned long locked_shm; /* How many pages of mlocked shm ? */ 840712f4aadSwilly tarreau unsigned long unix_inflight; /* How many files in flight in unix sockets */ 841759c0114SWilly Tarreau atomic_long_t pipe_bufs; /* how many pages are allocated in pipe buffers */ 8421da177e4SLinus Torvalds 8431da177e4SLinus Torvalds #ifdef CONFIG_KEYS 8441da177e4SLinus Torvalds struct key *uid_keyring; /* UID specific keyring */ 8451da177e4SLinus Torvalds struct key *session_keyring; /* UID's default session keyring */ 8461da177e4SLinus Torvalds #endif 8471da177e4SLinus Torvalds 8481da177e4SLinus Torvalds /* Hash table maintenance information */ 849735de223SPavel Emelyanov struct hlist_node uidhash_node; 8507b44ab97SEric W. Biederman kuid_t uid; 85124e377a8SSrivatsa Vaddagiri 852aaac3ba9SAlexei Starovoitov #if defined(CONFIG_PERF_EVENTS) || defined(CONFIG_BPF_SYSCALL) 853789f90fcSPeter Zijlstra atomic_long_t locked_vm; 854789f90fcSPeter Zijlstra #endif 8551da177e4SLinus Torvalds }; 8561da177e4SLinus Torvalds 857eb41d946SKay Sievers extern int uids_sysfs_init(void); 8585cb350baSDhaval Giani 8597b44ab97SEric W. Biederman extern struct user_struct *find_user(kuid_t); 8601da177e4SLinus Torvalds 8611da177e4SLinus Torvalds extern struct user_struct root_user; 8621da177e4SLinus Torvalds #define INIT_USER (&root_user) 8631da177e4SLinus Torvalds 864b6dff3ecSDavid Howells 8651da177e4SLinus Torvalds struct backing_dev_info; 8661da177e4SLinus Torvalds struct reclaim_state; 8671da177e4SLinus Torvalds 868f6db8347SNaveen N. Rao #ifdef CONFIG_SCHED_INFO 8691da177e4SLinus Torvalds struct sched_info { 8701da177e4SLinus Torvalds /* cumulative counters */ 8712d72376bSIngo Molnar unsigned long pcount; /* # of times run on this cpu */ 8729c2c4802SKen Chen unsigned long long run_delay; /* time spent waiting on a runqueue */ 8731da177e4SLinus Torvalds 8741da177e4SLinus Torvalds /* timestamps */ 875172ba844SBalbir Singh unsigned long long last_arrival,/* when we last ran on a cpu */ 8761da177e4SLinus Torvalds last_queued; /* when we were last queued to run */ 8771da177e4SLinus Torvalds }; 878f6db8347SNaveen N. Rao #endif /* CONFIG_SCHED_INFO */ 8791da177e4SLinus Torvalds 880ca74e92bSShailabh Nagar #ifdef CONFIG_TASK_DELAY_ACCT 881ca74e92bSShailabh Nagar struct task_delay_info { 882ca74e92bSShailabh Nagar spinlock_t lock; 883ca74e92bSShailabh Nagar unsigned int flags; /* Private per-task flags */ 884ca74e92bSShailabh Nagar 885ca74e92bSShailabh Nagar /* For each stat XXX, add following, aligned appropriately 886ca74e92bSShailabh Nagar * 887ca74e92bSShailabh Nagar * struct timespec XXX_start, XXX_end; 888ca74e92bSShailabh Nagar * u64 XXX_delay; 889ca74e92bSShailabh Nagar * u32 XXX_count; 890ca74e92bSShailabh Nagar * 891ca74e92bSShailabh Nagar * Atomicity of updates to XXX_delay, XXX_count protected by 892ca74e92bSShailabh Nagar * single lock above (split into XXX_lock if contention is an issue). 893ca74e92bSShailabh Nagar */ 8940ff92245SShailabh Nagar 8950ff92245SShailabh Nagar /* 8960ff92245SShailabh Nagar * XXX_count is incremented on every XXX operation, the delay 8970ff92245SShailabh Nagar * associated with the operation is added to XXX_delay. 8980ff92245SShailabh Nagar * XXX_delay contains the accumulated delay time in nanoseconds. 8990ff92245SShailabh Nagar */ 9009667a23dSThomas Gleixner u64 blkio_start; /* Shared by blkio, swapin */ 9010ff92245SShailabh Nagar u64 blkio_delay; /* wait for sync block io completion */ 9020ff92245SShailabh Nagar u64 swapin_delay; /* wait for swapin block io completion */ 9030ff92245SShailabh Nagar u32 blkio_count; /* total count of the number of sync block */ 9040ff92245SShailabh Nagar /* io operations performed */ 9050ff92245SShailabh Nagar u32 swapin_count; /* total count of the number of swapin block */ 9060ff92245SShailabh Nagar /* io operations performed */ 907873b4771SKeika Kobayashi 9089667a23dSThomas Gleixner u64 freepages_start; 909873b4771SKeika Kobayashi u64 freepages_delay; /* wait for memory reclaim */ 910873b4771SKeika Kobayashi u32 freepages_count; /* total count of memory reclaim */ 911ca74e92bSShailabh Nagar }; 91252f17b6cSChandra Seetharaman #endif /* CONFIG_TASK_DELAY_ACCT */ 91352f17b6cSChandra Seetharaman 91452f17b6cSChandra Seetharaman static inline int sched_info_on(void) 91552f17b6cSChandra Seetharaman { 91652f17b6cSChandra Seetharaman #ifdef CONFIG_SCHEDSTATS 91752f17b6cSChandra Seetharaman return 1; 91852f17b6cSChandra Seetharaman #elif defined(CONFIG_TASK_DELAY_ACCT) 91952f17b6cSChandra Seetharaman extern int delayacct_on; 92052f17b6cSChandra Seetharaman return delayacct_on; 92152f17b6cSChandra Seetharaman #else 92252f17b6cSChandra Seetharaman return 0; 923ca74e92bSShailabh Nagar #endif 92452f17b6cSChandra Seetharaman } 925ca74e92bSShailabh Nagar 926cb251765SMel Gorman #ifdef CONFIG_SCHEDSTATS 927cb251765SMel Gorman void force_schedstat_enabled(void); 928cb251765SMel Gorman #endif 929cb251765SMel Gorman 930d15bcfdbSIngo Molnar enum cpu_idle_type { 931d15bcfdbSIngo Molnar CPU_IDLE, 932d15bcfdbSIngo Molnar CPU_NOT_IDLE, 933d15bcfdbSIngo Molnar CPU_NEWLY_IDLE, 934d15bcfdbSIngo Molnar CPU_MAX_IDLE_TYPES 9351da177e4SLinus Torvalds }; 9361da177e4SLinus Torvalds 9371da177e4SLinus Torvalds /* 938ca8ce3d0SNicolas Pitre * Increase resolution of cpu_capacity calculations 9391399fa78SNikhil Rao */ 940ca8ce3d0SNicolas Pitre #define SCHED_CAPACITY_SHIFT 10 941ca8ce3d0SNicolas Pitre #define SCHED_CAPACITY_SCALE (1L << SCHED_CAPACITY_SHIFT) 9421da177e4SLinus Torvalds 9431399fa78SNikhil Rao /* 94476751049SPeter Zijlstra * Wake-queues are lists of tasks with a pending wakeup, whose 94576751049SPeter Zijlstra * callers have already marked the task as woken internally, 94676751049SPeter Zijlstra * and can thus carry on. A common use case is being able to 94776751049SPeter Zijlstra * do the wakeups once the corresponding user lock as been 94876751049SPeter Zijlstra * released. 94976751049SPeter Zijlstra * 95076751049SPeter Zijlstra * We hold reference to each task in the list across the wakeup, 95176751049SPeter Zijlstra * thus guaranteeing that the memory is still valid by the time 95276751049SPeter Zijlstra * the actual wakeups are performed in wake_up_q(). 95376751049SPeter Zijlstra * 95476751049SPeter Zijlstra * One per task suffices, because there's never a need for a task to be 95576751049SPeter Zijlstra * in two wake queues simultaneously; it is forbidden to abandon a task 95676751049SPeter Zijlstra * in a wake queue (a call to wake_up_q() _must_ follow), so if a task is 95776751049SPeter Zijlstra * already in a wake queue, the wakeup will happen soon and the second 95876751049SPeter Zijlstra * waker can just skip it. 95976751049SPeter Zijlstra * 96076751049SPeter Zijlstra * The WAKE_Q macro declares and initializes the list head. 96176751049SPeter Zijlstra * wake_up_q() does NOT reinitialize the list; it's expected to be 96276751049SPeter Zijlstra * called near the end of a function, where the fact that the queue is 96376751049SPeter Zijlstra * not used again will be easy to see by inspection. 96476751049SPeter Zijlstra * 96576751049SPeter Zijlstra * Note that this can cause spurious wakeups. schedule() callers 96676751049SPeter Zijlstra * must ensure the call is done inside a loop, confirming that the 96776751049SPeter Zijlstra * wakeup condition has in fact occurred. 96876751049SPeter Zijlstra */ 96976751049SPeter Zijlstra struct wake_q_node { 97076751049SPeter Zijlstra struct wake_q_node *next; 97176751049SPeter Zijlstra }; 97276751049SPeter Zijlstra 97376751049SPeter Zijlstra struct wake_q_head { 97476751049SPeter Zijlstra struct wake_q_node *first; 97576751049SPeter Zijlstra struct wake_q_node **lastp; 97676751049SPeter Zijlstra }; 97776751049SPeter Zijlstra 97876751049SPeter Zijlstra #define WAKE_Q_TAIL ((struct wake_q_node *) 0x01) 97976751049SPeter Zijlstra 98076751049SPeter Zijlstra #define WAKE_Q(name) \ 98176751049SPeter Zijlstra struct wake_q_head name = { WAKE_Q_TAIL, &name.first } 98276751049SPeter Zijlstra 98376751049SPeter Zijlstra extern void wake_q_add(struct wake_q_head *head, 98476751049SPeter Zijlstra struct task_struct *task); 98576751049SPeter Zijlstra extern void wake_up_q(struct wake_q_head *head); 98676751049SPeter Zijlstra 98776751049SPeter Zijlstra /* 9881399fa78SNikhil Rao * sched-domains (multiprocessor balancing) declarations: 9891399fa78SNikhil Rao */ 9902dd73a4fSPeter Williams #ifdef CONFIG_SMP 991b5d978e0SPeter Zijlstra #define SD_LOAD_BALANCE 0x0001 /* Do load balancing on this domain. */ 992b5d978e0SPeter Zijlstra #define SD_BALANCE_NEWIDLE 0x0002 /* Balance when about to become idle */ 993b5d978e0SPeter Zijlstra #define SD_BALANCE_EXEC 0x0004 /* Balance on exec */ 994b5d978e0SPeter Zijlstra #define SD_BALANCE_FORK 0x0008 /* Balance on fork, clone */ 995c88d5910SPeter Zijlstra #define SD_BALANCE_WAKE 0x0010 /* Balance on wakeup */ 996b5d978e0SPeter Zijlstra #define SD_WAKE_AFFINE 0x0020 /* Wake task to waking CPU */ 9975d4dfdddSNicolas Pitre #define SD_SHARE_CPUCAPACITY 0x0080 /* Domain members share cpu power */ 998d77b3ed5SVincent Guittot #define SD_SHARE_POWERDOMAIN 0x0100 /* Domain members share power domain */ 999b5d978e0SPeter Zijlstra #define SD_SHARE_PKG_RESOURCES 0x0200 /* Domain members share cpu pkg resources */ 1000b5d978e0SPeter Zijlstra #define SD_SERIALIZE 0x0400 /* Only a single load balancing instance */ 1001532cb4c4SMichael Neuling #define SD_ASYM_PACKING 0x0800 /* Place busy groups earlier in the domain */ 1002b5d978e0SPeter Zijlstra #define SD_PREFER_SIBLING 0x1000 /* Prefer to place tasks in a sibling domain */ 1003e3589f6cSPeter Zijlstra #define SD_OVERLAP 0x2000 /* sched_domains of this level overlap */ 10043a7053b3SMel Gorman #define SD_NUMA 0x4000 /* cross-node balancing */ 10055c45bf27SSiddha, Suresh B 1006143e1e28SVincent Guittot #ifdef CONFIG_SCHED_SMT 1007b6220ad6SGuenter Roeck static inline int cpu_smt_flags(void) 1008143e1e28SVincent Guittot { 10095d4dfdddSNicolas Pitre return SD_SHARE_CPUCAPACITY | SD_SHARE_PKG_RESOURCES; 1010143e1e28SVincent Guittot } 1011143e1e28SVincent Guittot #endif 1012143e1e28SVincent Guittot 1013143e1e28SVincent Guittot #ifdef CONFIG_SCHED_MC 1014b6220ad6SGuenter Roeck static inline int cpu_core_flags(void) 1015143e1e28SVincent Guittot { 1016143e1e28SVincent Guittot return SD_SHARE_PKG_RESOURCES; 1017143e1e28SVincent Guittot } 1018143e1e28SVincent Guittot #endif 1019143e1e28SVincent Guittot 1020143e1e28SVincent Guittot #ifdef CONFIG_NUMA 1021b6220ad6SGuenter Roeck static inline int cpu_numa_flags(void) 1022143e1e28SVincent Guittot { 1023143e1e28SVincent Guittot return SD_NUMA; 1024143e1e28SVincent Guittot } 1025143e1e28SVincent Guittot #endif 1026532cb4c4SMichael Neuling 10271d3504fcSHidetoshi Seto struct sched_domain_attr { 10281d3504fcSHidetoshi Seto int relax_domain_level; 10291d3504fcSHidetoshi Seto }; 10301d3504fcSHidetoshi Seto 10311d3504fcSHidetoshi Seto #define SD_ATTR_INIT (struct sched_domain_attr) { \ 10321d3504fcSHidetoshi Seto .relax_domain_level = -1, \ 10331d3504fcSHidetoshi Seto } 10341d3504fcSHidetoshi Seto 103560495e77SPeter Zijlstra extern int sched_domain_level_max; 103660495e77SPeter Zijlstra 10375e6521eaSLi Zefan struct sched_group; 10385e6521eaSLi Zefan 10391da177e4SLinus Torvalds struct sched_domain { 10401da177e4SLinus Torvalds /* These fields must be setup */ 10411da177e4SLinus Torvalds struct sched_domain *parent; /* top domain must be null terminated */ 10421a848870SSiddha, Suresh B struct sched_domain *child; /* bottom domain must be null terminated */ 10431da177e4SLinus Torvalds struct sched_group *groups; /* the balancing groups of the domain */ 10441da177e4SLinus Torvalds unsigned long min_interval; /* Minimum balance interval ms */ 10451da177e4SLinus Torvalds unsigned long max_interval; /* Maximum balance interval ms */ 10461da177e4SLinus Torvalds unsigned int busy_factor; /* less balancing by factor if busy */ 10471da177e4SLinus Torvalds unsigned int imbalance_pct; /* No balance until over watermark */ 10481da177e4SLinus Torvalds unsigned int cache_nice_tries; /* Leave cache hot tasks for # tries */ 10497897986bSNick Piggin unsigned int busy_idx; 10507897986bSNick Piggin unsigned int idle_idx; 10517897986bSNick Piggin unsigned int newidle_idx; 10527897986bSNick Piggin unsigned int wake_idx; 1053147cbb4bSNick Piggin unsigned int forkexec_idx; 1054a52bfd73SPeter Zijlstra unsigned int smt_gain; 105525f55d9dSVincent Guittot 105625f55d9dSVincent Guittot int nohz_idle; /* NOHZ IDLE status */ 10571da177e4SLinus Torvalds int flags; /* See SD_* */ 105860495e77SPeter Zijlstra int level; 10591da177e4SLinus Torvalds 10601da177e4SLinus Torvalds /* Runtime fields. */ 10611da177e4SLinus Torvalds unsigned long last_balance; /* init to jiffies. units in jiffies */ 10621da177e4SLinus Torvalds unsigned int balance_interval; /* initialise to 1. units in ms. */ 10631da177e4SLinus Torvalds unsigned int nr_balance_failed; /* initialise to 0 */ 10641da177e4SLinus Torvalds 1065f48627e6SJason Low /* idle_balance() stats */ 10669bd721c5SJason Low u64 max_newidle_lb_cost; 1067f48627e6SJason Low unsigned long next_decay_max_lb_cost; 10682398f2c6SPeter Zijlstra 10691da177e4SLinus Torvalds #ifdef CONFIG_SCHEDSTATS 10701da177e4SLinus Torvalds /* load_balance() stats */ 1071480b9434SKen Chen unsigned int lb_count[CPU_MAX_IDLE_TYPES]; 1072480b9434SKen Chen unsigned int lb_failed[CPU_MAX_IDLE_TYPES]; 1073480b9434SKen Chen unsigned int lb_balanced[CPU_MAX_IDLE_TYPES]; 1074480b9434SKen Chen unsigned int lb_imbalance[CPU_MAX_IDLE_TYPES]; 1075480b9434SKen Chen unsigned int lb_gained[CPU_MAX_IDLE_TYPES]; 1076480b9434SKen Chen unsigned int lb_hot_gained[CPU_MAX_IDLE_TYPES]; 1077480b9434SKen Chen unsigned int lb_nobusyg[CPU_MAX_IDLE_TYPES]; 1078480b9434SKen Chen unsigned int lb_nobusyq[CPU_MAX_IDLE_TYPES]; 10791da177e4SLinus Torvalds 10801da177e4SLinus Torvalds /* Active load balancing */ 1081480b9434SKen Chen unsigned int alb_count; 1082480b9434SKen Chen unsigned int alb_failed; 1083480b9434SKen Chen unsigned int alb_pushed; 10841da177e4SLinus Torvalds 108568767a0aSNick Piggin /* SD_BALANCE_EXEC stats */ 1086480b9434SKen Chen unsigned int sbe_count; 1087480b9434SKen Chen unsigned int sbe_balanced; 1088480b9434SKen Chen unsigned int sbe_pushed; 10891da177e4SLinus Torvalds 109068767a0aSNick Piggin /* SD_BALANCE_FORK stats */ 1091480b9434SKen Chen unsigned int sbf_count; 1092480b9434SKen Chen unsigned int sbf_balanced; 1093480b9434SKen Chen unsigned int sbf_pushed; 109468767a0aSNick Piggin 10951da177e4SLinus Torvalds /* try_to_wake_up() stats */ 1096480b9434SKen Chen unsigned int ttwu_wake_remote; 1097480b9434SKen Chen unsigned int ttwu_move_affine; 1098480b9434SKen Chen unsigned int ttwu_move_balance; 10991da177e4SLinus Torvalds #endif 1100a5d8c348SIngo Molnar #ifdef CONFIG_SCHED_DEBUG 1101a5d8c348SIngo Molnar char *name; 1102a5d8c348SIngo Molnar #endif 1103dce840a0SPeter Zijlstra union { 1104dce840a0SPeter Zijlstra void *private; /* used during construction */ 1105dce840a0SPeter Zijlstra struct rcu_head rcu; /* used during destruction */ 1106dce840a0SPeter Zijlstra }; 11076c99e9adSRusty Russell 1108669c55e9SPeter Zijlstra unsigned int span_weight; 11094200efd9SIngo Molnar /* 11104200efd9SIngo Molnar * Span of all CPUs in this domain. 11114200efd9SIngo Molnar * 11124200efd9SIngo Molnar * NOTE: this field is variable length. (Allocated dynamically 11134200efd9SIngo Molnar * by attaching extra space to the end of the structure, 11144200efd9SIngo Molnar * depending on how many CPUs the kernel has booted up with) 11154200efd9SIngo Molnar */ 11164200efd9SIngo Molnar unsigned long span[0]; 11171da177e4SLinus Torvalds }; 11181da177e4SLinus Torvalds 1119758b2cdcSRusty Russell static inline struct cpumask *sched_domain_span(struct sched_domain *sd) 1120758b2cdcSRusty Russell { 11216c99e9adSRusty Russell return to_cpumask(sd->span); 1122758b2cdcSRusty Russell } 1123758b2cdcSRusty Russell 1124acc3f5d7SRusty Russell extern void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[], 11251d3504fcSHidetoshi Seto struct sched_domain_attr *dattr_new); 1126029190c5SPaul Jackson 1127acc3f5d7SRusty Russell /* Allocate an array of sched domains, for partition_sched_domains(). */ 1128acc3f5d7SRusty Russell cpumask_var_t *alloc_sched_domains(unsigned int ndoms); 1129acc3f5d7SRusty Russell void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms); 1130acc3f5d7SRusty Russell 113139be3501SPeter Zijlstra bool cpus_share_cache(int this_cpu, int that_cpu); 113239be3501SPeter Zijlstra 1133143e1e28SVincent Guittot typedef const struct cpumask *(*sched_domain_mask_f)(int cpu); 1134b6220ad6SGuenter Roeck typedef int (*sched_domain_flags_f)(void); 1135143e1e28SVincent Guittot 1136143e1e28SVincent Guittot #define SDTL_OVERLAP 0x01 1137143e1e28SVincent Guittot 1138143e1e28SVincent Guittot struct sd_data { 1139143e1e28SVincent Guittot struct sched_domain **__percpu sd; 1140143e1e28SVincent Guittot struct sched_group **__percpu sg; 114163b2ca30SNicolas Pitre struct sched_group_capacity **__percpu sgc; 1142143e1e28SVincent Guittot }; 1143143e1e28SVincent Guittot 1144143e1e28SVincent Guittot struct sched_domain_topology_level { 1145143e1e28SVincent Guittot sched_domain_mask_f mask; 1146143e1e28SVincent Guittot sched_domain_flags_f sd_flags; 1147143e1e28SVincent Guittot int flags; 1148143e1e28SVincent Guittot int numa_level; 1149143e1e28SVincent Guittot struct sd_data data; 1150143e1e28SVincent Guittot #ifdef CONFIG_SCHED_DEBUG 1151143e1e28SVincent Guittot char *name; 1152143e1e28SVincent Guittot #endif 1153143e1e28SVincent Guittot }; 1154143e1e28SVincent Guittot 1155143e1e28SVincent Guittot extern void set_sched_topology(struct sched_domain_topology_level *tl); 1156f6be8af1SChuansheng Liu extern void wake_up_if_idle(int cpu); 1157143e1e28SVincent Guittot 1158143e1e28SVincent Guittot #ifdef CONFIG_SCHED_DEBUG 1159143e1e28SVincent Guittot # define SD_INIT_NAME(type) .name = #type 1160143e1e28SVincent Guittot #else 1161143e1e28SVincent Guittot # define SD_INIT_NAME(type) 1162143e1e28SVincent Guittot #endif 1163143e1e28SVincent Guittot 11641b427c15SIngo Molnar #else /* CONFIG_SMP */ 11651da177e4SLinus Torvalds 11661b427c15SIngo Molnar struct sched_domain_attr; 11671b427c15SIngo Molnar 11681b427c15SIngo Molnar static inline void 1169acc3f5d7SRusty Russell partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[], 11701b427c15SIngo Molnar struct sched_domain_attr *dattr_new) 1171d02c7a8cSCon Kolivas { 1172d02c7a8cSCon Kolivas } 117339be3501SPeter Zijlstra 117439be3501SPeter Zijlstra static inline bool cpus_share_cache(int this_cpu, int that_cpu) 117539be3501SPeter Zijlstra { 117639be3501SPeter Zijlstra return true; 117739be3501SPeter Zijlstra } 117839be3501SPeter Zijlstra 11791b427c15SIngo Molnar #endif /* !CONFIG_SMP */ 11801da177e4SLinus Torvalds 118147fe38fcSPeter Zijlstra 11821da177e4SLinus Torvalds struct io_context; /* See blkdev.h */ 11831da177e4SLinus Torvalds 11841da177e4SLinus Torvalds 1185383f2835SChen, Kenneth W #ifdef ARCH_HAS_PREFETCH_SWITCH_STACK 118636c8b586SIngo Molnar extern void prefetch_stack(struct task_struct *t); 1187383f2835SChen, Kenneth W #else 1188383f2835SChen, Kenneth W static inline void prefetch_stack(struct task_struct *t) { } 1189383f2835SChen, Kenneth W #endif 11901da177e4SLinus Torvalds 11911da177e4SLinus Torvalds struct audit_context; /* See audit.c */ 11921da177e4SLinus Torvalds struct mempolicy; 1193b92ce558SJens Axboe struct pipe_inode_info; 11944865ecf1SSerge E. Hallyn struct uts_namespace; 11951da177e4SLinus Torvalds 119620b8a59fSIngo Molnar struct load_weight { 11979dbdb155SPeter Zijlstra unsigned long weight; 11989dbdb155SPeter Zijlstra u32 inv_weight; 119920b8a59fSIngo Molnar }; 120020b8a59fSIngo Molnar 12019d89c257SYuyang Du /* 12029d89c257SYuyang Du * The load_avg/util_avg accumulates an infinite geometric series. 1203e0f5f3afSDietmar Eggemann * 1) load_avg factors frequency scaling into the amount of time that a 1204e0f5f3afSDietmar Eggemann * sched_entity is runnable on a rq into its weight. For cfs_rq, it is the 1205e0f5f3afSDietmar Eggemann * aggregated such weights of all runnable and blocked sched_entities. 1206e3279a2eSDietmar Eggemann * 2) util_avg factors frequency and cpu scaling into the amount of time 12079d89c257SYuyang Du * that a sched_entity is running on a CPU, in the range [0..SCHED_LOAD_SCALE]. 12089d89c257SYuyang Du * For cfs_rq, it is the aggregated such times of all runnable and 12099d89c257SYuyang Du * blocked sched_entities. 12109d89c257SYuyang Du * The 64 bit load_sum can: 12119d89c257SYuyang Du * 1) for cfs_rq, afford 4353082796 (=2^64/47742/88761) entities with 12129d89c257SYuyang Du * the highest weight (=88761) always runnable, we should not overflow 12139d89c257SYuyang Du * 2) for entity, support any load.weight always runnable 12149d89c257SYuyang Du */ 12159d85f21cSPaul Turner struct sched_avg { 12169d89c257SYuyang Du u64 last_update_time, load_sum; 12179d89c257SYuyang Du u32 util_sum, period_contrib; 12189d89c257SYuyang Du unsigned long load_avg, util_avg; 12199d85f21cSPaul Turner }; 12209d85f21cSPaul Turner 122194c18227SIngo Molnar #ifdef CONFIG_SCHEDSTATS 122241acab88SLucas De Marchi struct sched_statistics { 122394c18227SIngo Molnar u64 wait_start; 122494c18227SIngo Molnar u64 wait_max; 12256d082592SArjan van de Ven u64 wait_count; 12266d082592SArjan van de Ven u64 wait_sum; 12278f0dfc34SArjan van de Ven u64 iowait_count; 12288f0dfc34SArjan van de Ven u64 iowait_sum; 122994c18227SIngo Molnar 123094c18227SIngo Molnar u64 sleep_start; 123120b8a59fSIngo Molnar u64 sleep_max; 123294c18227SIngo Molnar s64 sum_sleep_runtime; 123394c18227SIngo Molnar 123494c18227SIngo Molnar u64 block_start; 123520b8a59fSIngo Molnar u64 block_max; 123620b8a59fSIngo Molnar u64 exec_max; 1237eba1ed4bSIngo Molnar u64 slice_max; 1238cc367732SIngo Molnar 1239cc367732SIngo Molnar u64 nr_migrations_cold; 1240cc367732SIngo Molnar u64 nr_failed_migrations_affine; 1241cc367732SIngo Molnar u64 nr_failed_migrations_running; 1242cc367732SIngo Molnar u64 nr_failed_migrations_hot; 1243cc367732SIngo Molnar u64 nr_forced_migrations; 1244cc367732SIngo Molnar 1245cc367732SIngo Molnar u64 nr_wakeups; 1246cc367732SIngo Molnar u64 nr_wakeups_sync; 1247cc367732SIngo Molnar u64 nr_wakeups_migrate; 1248cc367732SIngo Molnar u64 nr_wakeups_local; 1249cc367732SIngo Molnar u64 nr_wakeups_remote; 1250cc367732SIngo Molnar u64 nr_wakeups_affine; 1251cc367732SIngo Molnar u64 nr_wakeups_affine_attempts; 1252cc367732SIngo Molnar u64 nr_wakeups_passive; 1253cc367732SIngo Molnar u64 nr_wakeups_idle; 125441acab88SLucas De Marchi }; 125541acab88SLucas De Marchi #endif 125641acab88SLucas De Marchi 125741acab88SLucas De Marchi struct sched_entity { 125841acab88SLucas De Marchi struct load_weight load; /* for load-balancing */ 125941acab88SLucas De Marchi struct rb_node run_node; 126041acab88SLucas De Marchi struct list_head group_node; 126141acab88SLucas De Marchi unsigned int on_rq; 126241acab88SLucas De Marchi 126341acab88SLucas De Marchi u64 exec_start; 126441acab88SLucas De Marchi u64 sum_exec_runtime; 126541acab88SLucas De Marchi u64 vruntime; 126641acab88SLucas De Marchi u64 prev_sum_exec_runtime; 126741acab88SLucas De Marchi 126841acab88SLucas De Marchi u64 nr_migrations; 126941acab88SLucas De Marchi 127041acab88SLucas De Marchi #ifdef CONFIG_SCHEDSTATS 127141acab88SLucas De Marchi struct sched_statistics statistics; 127294c18227SIngo Molnar #endif 127394c18227SIngo Molnar 127420b8a59fSIngo Molnar #ifdef CONFIG_FAIR_GROUP_SCHED 1275fed14d45SPeter Zijlstra int depth; 127620b8a59fSIngo Molnar struct sched_entity *parent; 127720b8a59fSIngo Molnar /* rq on which this entity is (to be) queued: */ 127820b8a59fSIngo Molnar struct cfs_rq *cfs_rq; 127920b8a59fSIngo Molnar /* rq "owned" by this entity/group: */ 128020b8a59fSIngo Molnar struct cfs_rq *my_q; 128120b8a59fSIngo Molnar #endif 12828bd75c77SClark Williams 1283141965c7SAlex Shi #ifdef CONFIG_SMP 12845a107804SJiri Olsa /* 12855a107804SJiri Olsa * Per entity load average tracking. 12865a107804SJiri Olsa * 12875a107804SJiri Olsa * Put into separate cache line so it does not 12885a107804SJiri Olsa * collide with read-mostly values above. 12895a107804SJiri Olsa */ 12905a107804SJiri Olsa struct sched_avg avg ____cacheline_aligned_in_smp; 12919d85f21cSPaul Turner #endif 129220b8a59fSIngo Molnar }; 129370b97a7fSIngo Molnar 1294fa717060SPeter Zijlstra struct sched_rt_entity { 1295fa717060SPeter Zijlstra struct list_head run_list; 129678f2c7dbSPeter Zijlstra unsigned long timeout; 129757d2aa00SYing Xue unsigned long watchdog_stamp; 1298bee367edSRichard Kennedy unsigned int time_slice; 1299ff77e468SPeter Zijlstra unsigned short on_rq; 1300ff77e468SPeter Zijlstra unsigned short on_list; 13016f505b16SPeter Zijlstra 130258d6c2d7SPeter Zijlstra struct sched_rt_entity *back; 1303052f1dc7SPeter Zijlstra #ifdef CONFIG_RT_GROUP_SCHED 13046f505b16SPeter Zijlstra struct sched_rt_entity *parent; 13056f505b16SPeter Zijlstra /* rq on which this entity is (to be) queued: */ 13066f505b16SPeter Zijlstra struct rt_rq *rt_rq; 13076f505b16SPeter Zijlstra /* rq "owned" by this entity/group: */ 13086f505b16SPeter Zijlstra struct rt_rq *my_q; 13096f505b16SPeter Zijlstra #endif 1310fa717060SPeter Zijlstra }; 1311fa717060SPeter Zijlstra 1312aab03e05SDario Faggioli struct sched_dl_entity { 1313aab03e05SDario Faggioli struct rb_node rb_node; 1314aab03e05SDario Faggioli 1315aab03e05SDario Faggioli /* 1316aab03e05SDario Faggioli * Original scheduling parameters. Copied here from sched_attr 13174027d080Sxiaofeng.yan * during sched_setattr(), they will remain the same until 13184027d080Sxiaofeng.yan * the next sched_setattr(). 1319aab03e05SDario Faggioli */ 1320aab03e05SDario Faggioli u64 dl_runtime; /* maximum runtime for each instance */ 1321aab03e05SDario Faggioli u64 dl_deadline; /* relative deadline of each instance */ 1322755378a4SHarald Gustafsson u64 dl_period; /* separation of two instances (period) */ 1323332ac17eSDario Faggioli u64 dl_bw; /* dl_runtime / dl_deadline */ 1324aab03e05SDario Faggioli 1325aab03e05SDario Faggioli /* 1326aab03e05SDario Faggioli * Actual scheduling parameters. Initialized with the values above, 1327aab03e05SDario Faggioli * they are continously updated during task execution. Note that 1328aab03e05SDario Faggioli * the remaining runtime could be < 0 in case we are in overrun. 1329aab03e05SDario Faggioli */ 1330aab03e05SDario Faggioli s64 runtime; /* remaining runtime for this instance */ 1331aab03e05SDario Faggioli u64 deadline; /* absolute deadline for this instance */ 1332aab03e05SDario Faggioli unsigned int flags; /* specifying the scheduler behaviour */ 1333aab03e05SDario Faggioli 1334aab03e05SDario Faggioli /* 1335aab03e05SDario Faggioli * Some bool flags: 1336aab03e05SDario Faggioli * 1337aab03e05SDario Faggioli * @dl_throttled tells if we exhausted the runtime. If so, the 1338aab03e05SDario Faggioli * task has to wait for a replenishment to be performed at the 1339aab03e05SDario Faggioli * next firing of dl_timer. 1340aab03e05SDario Faggioli * 13412d3d891dSDario Faggioli * @dl_boosted tells if we are boosted due to DI. If so we are 13422d3d891dSDario Faggioli * outside bandwidth enforcement mechanism (but only until we 13435bfd126eSJuri Lelli * exit the critical section); 13445bfd126eSJuri Lelli * 13455bfd126eSJuri Lelli * @dl_yielded tells if task gave up the cpu before consuming 13465bfd126eSJuri Lelli * all its available runtime during the last job. 1347aab03e05SDario Faggioli */ 134872f9f3fdSLuca Abeni int dl_throttled, dl_boosted, dl_yielded; 1349aab03e05SDario Faggioli 1350aab03e05SDario Faggioli /* 1351aab03e05SDario Faggioli * Bandwidth enforcement timer. Each -deadline task has its 1352aab03e05SDario Faggioli * own bandwidth to be enforced, thus we need one timer per task. 1353aab03e05SDario Faggioli */ 1354aab03e05SDario Faggioli struct hrtimer dl_timer; 1355aab03e05SDario Faggioli }; 13568bd75c77SClark Williams 13571d082fd0SPaul E. McKenney union rcu_special { 13581d082fd0SPaul E. McKenney struct { 13598203d6d0SPaul E. McKenney u8 blocked; 13608203d6d0SPaul E. McKenney u8 need_qs; 13618203d6d0SPaul E. McKenney u8 exp_need_qs; 13628203d6d0SPaul E. McKenney u8 pad; /* Otherwise the compiler can store garbage here. */ 13638203d6d0SPaul E. McKenney } b; /* Bits. */ 13648203d6d0SPaul E. McKenney u32 s; /* Set of bits. */ 13651d082fd0SPaul E. McKenney }; 136686848966SPaul E. McKenney struct rcu_node; 136786848966SPaul E. McKenney 13688dc85d54SPeter Zijlstra enum perf_event_task_context { 13698dc85d54SPeter Zijlstra perf_invalid_context = -1, 13708dc85d54SPeter Zijlstra perf_hw_context = 0, 137189a1e187SPeter Zijlstra perf_sw_context, 13728dc85d54SPeter Zijlstra perf_nr_task_contexts, 13738dc85d54SPeter Zijlstra }; 13748dc85d54SPeter Zijlstra 137572b252aeSMel Gorman /* Track pages that require TLB flushes */ 137672b252aeSMel Gorman struct tlbflush_unmap_batch { 137772b252aeSMel Gorman /* 137872b252aeSMel Gorman * Each bit set is a CPU that potentially has a TLB entry for one of 137972b252aeSMel Gorman * the PFNs being flushed. See set_tlb_ubc_flush_pending(). 138072b252aeSMel Gorman */ 138172b252aeSMel Gorman struct cpumask cpumask; 138272b252aeSMel Gorman 138372b252aeSMel Gorman /* True if any bit in cpumask is set */ 138472b252aeSMel Gorman bool flush_required; 1385d950c947SMel Gorman 1386d950c947SMel Gorman /* 1387d950c947SMel Gorman * If true then the PTE was dirty when unmapped. The entry must be 1388d950c947SMel Gorman * flushed before IO is initiated or a stale TLB entry potentially 1389d950c947SMel Gorman * allows an update without redirtying the page. 1390d950c947SMel Gorman */ 1391d950c947SMel Gorman bool writable; 139272b252aeSMel Gorman }; 139372b252aeSMel Gorman 13941da177e4SLinus Torvalds struct task_struct { 13951da177e4SLinus Torvalds volatile long state; /* -1 unrunnable, 0 runnable, >0 stopped */ 1396f7e4217bSRoman Zippel void *stack; 13971da177e4SLinus Torvalds atomic_t usage; 139897dc32cdSWilliam Cohen unsigned int flags; /* per process flags, defined below */ 139997dc32cdSWilliam Cohen unsigned int ptrace; 14001da177e4SLinus Torvalds 14012dd73a4fSPeter Williams #ifdef CONFIG_SMP 1402fa14ff4aSPeter Zijlstra struct llist_node wake_entry; 14033ca7a440SPeter Zijlstra int on_cpu; 140463b0e9edSMike Galbraith unsigned int wakee_flips; 140562470419SMichael Wang unsigned long wakee_flip_decay_ts; 140663b0e9edSMike Galbraith struct task_struct *last_wakee; 1407ac66f547SPeter Zijlstra 1408ac66f547SPeter Zijlstra int wake_cpu; 14094866cde0SNick Piggin #endif 1410fd2f4419SPeter Zijlstra int on_rq; 141150e645a8SIngo Molnar 1412b29739f9SIngo Molnar int prio, static_prio, normal_prio; 1413c7aceabaSRichard Kennedy unsigned int rt_priority; 14145522d5d5SIngo Molnar const struct sched_class *sched_class; 141520b8a59fSIngo Molnar struct sched_entity se; 1416fa717060SPeter Zijlstra struct sched_rt_entity rt; 14178323f26cSPeter Zijlstra #ifdef CONFIG_CGROUP_SCHED 14188323f26cSPeter Zijlstra struct task_group *sched_task_group; 14198323f26cSPeter Zijlstra #endif 1420aab03e05SDario Faggioli struct sched_dl_entity dl; 14211da177e4SLinus Torvalds 1422e107be36SAvi Kivity #ifdef CONFIG_PREEMPT_NOTIFIERS 1423e107be36SAvi Kivity /* list of struct preempt_notifier: */ 1424e107be36SAvi Kivity struct hlist_head preempt_notifiers; 1425e107be36SAvi Kivity #endif 1426e107be36SAvi Kivity 14276c5c9341SAlexey Dobriyan #ifdef CONFIG_BLK_DEV_IO_TRACE 14282056a782SJens Axboe unsigned int btrace_seq; 14296c5c9341SAlexey Dobriyan #endif 14301da177e4SLinus Torvalds 143197dc32cdSWilliam Cohen unsigned int policy; 143229baa747SPeter Zijlstra int nr_cpus_allowed; 14331da177e4SLinus Torvalds cpumask_t cpus_allowed; 14341da177e4SLinus Torvalds 1435a57eb940SPaul E. McKenney #ifdef CONFIG_PREEMPT_RCU 1436e260be67SPaul E. McKenney int rcu_read_lock_nesting; 14371d082fd0SPaul E. McKenney union rcu_special rcu_read_unlock_special; 1438f41d911fSPaul E. McKenney struct list_head rcu_node_entry; 1439a57eb940SPaul E. McKenney struct rcu_node *rcu_blocked_node; 144028f6569aSPranith Kumar #endif /* #ifdef CONFIG_PREEMPT_RCU */ 14418315f422SPaul E. McKenney #ifdef CONFIG_TASKS_RCU 14428315f422SPaul E. McKenney unsigned long rcu_tasks_nvcsw; 14438315f422SPaul E. McKenney bool rcu_tasks_holdout; 14448315f422SPaul E. McKenney struct list_head rcu_tasks_holdout_list; 1445176f8f7aSPaul E. McKenney int rcu_tasks_idle_cpu; 14468315f422SPaul E. McKenney #endif /* #ifdef CONFIG_TASKS_RCU */ 1447e260be67SPaul E. McKenney 1448f6db8347SNaveen N. Rao #ifdef CONFIG_SCHED_INFO 14491da177e4SLinus Torvalds struct sched_info sched_info; 14501da177e4SLinus Torvalds #endif 14511da177e4SLinus Torvalds 14521da177e4SLinus Torvalds struct list_head tasks; 1453806c09a7SDario Faggioli #ifdef CONFIG_SMP 1454917b627dSGregory Haskins struct plist_node pushable_tasks; 14551baca4ceSJuri Lelli struct rb_node pushable_dl_tasks; 1456806c09a7SDario Faggioli #endif 14571da177e4SLinus Torvalds 14581da177e4SLinus Torvalds struct mm_struct *mm, *active_mm; 1459615d6e87SDavidlohr Bueso /* per-thread vma caching */ 1460615d6e87SDavidlohr Bueso u32 vmacache_seqnum; 1461615d6e87SDavidlohr Bueso struct vm_area_struct *vmacache[VMACACHE_SIZE]; 146234e55232SKAMEZAWA Hiroyuki #if defined(SPLIT_RSS_COUNTING) 146334e55232SKAMEZAWA Hiroyuki struct task_rss_stat rss_stat; 146434e55232SKAMEZAWA Hiroyuki #endif 14651da177e4SLinus Torvalds /* task state */ 146697dc32cdSWilliam Cohen int exit_state; 14671da177e4SLinus Torvalds int exit_code, exit_signal; 14681da177e4SLinus Torvalds int pdeath_signal; /* The signal sent when the parent dies */ 1469e7cc4173SPalmer Dabbelt unsigned long jobctl; /* JOBCTL_*, siglock protected */ 14709b89f6baSAndrei Epure 14719b89f6baSAndrei Epure /* Used for emulating ABI behavior of previous Linux versions */ 147297dc32cdSWilliam Cohen unsigned int personality; 14739b89f6baSAndrei Epure 1474be958bdcSPeter Zijlstra /* scheduler bits, serialized by scheduler locks */ 1475ca94c442SLennart Poettering unsigned sched_reset_on_fork:1; 1476a8e4f2eaSPeter Zijlstra unsigned sched_contributes_to_load:1; 1477ff303e66SPeter Zijlstra unsigned sched_migrated:1; 1478be958bdcSPeter Zijlstra unsigned :0; /* force alignment to the next boundary */ 1479be958bdcSPeter Zijlstra 1480be958bdcSPeter Zijlstra /* unserialized, strictly 'current' */ 1481be958bdcSPeter Zijlstra unsigned in_execve:1; /* bit to tell LSMs we're in execve */ 1482be958bdcSPeter Zijlstra unsigned in_iowait:1; 1483626ebc41STejun Heo #ifdef CONFIG_MEMCG 1484626ebc41STejun Heo unsigned memcg_may_oom:1; 1485127424c8SJohannes Weiner #ifndef CONFIG_SLOB 14866f185c29SVladimir Davydov unsigned memcg_kmem_skip_account:1; 14876f185c29SVladimir Davydov #endif 1488127424c8SJohannes Weiner #endif 1489ff303e66SPeter Zijlstra #ifdef CONFIG_COMPAT_BRK 1490ff303e66SPeter Zijlstra unsigned brk_randomized:1; 1491ff303e66SPeter Zijlstra #endif 14926f185c29SVladimir Davydov 14931d4457f9SKees Cook unsigned long atomic_flags; /* Flags needing atomic access. */ 14941d4457f9SKees Cook 1495f56141e3SAndy Lutomirski struct restart_block restart_block; 1496f56141e3SAndy Lutomirski 14971da177e4SLinus Torvalds pid_t pid; 14981da177e4SLinus Torvalds pid_t tgid; 14990a425405SArjan van de Ven 15001314562aSHiroshi Shimamoto #ifdef CONFIG_CC_STACKPROTECTOR 15010a425405SArjan van de Ven /* Canary value for the -fstack-protector gcc feature */ 15020a425405SArjan van de Ven unsigned long stack_canary; 15031314562aSHiroshi Shimamoto #endif 15041da177e4SLinus Torvalds /* 15051da177e4SLinus Torvalds * pointers to (original) parent process, youngest child, younger sibling, 15061da177e4SLinus Torvalds * older sibling, respectively. (p->father can be replaced with 1507f470021aSRoland McGrath * p->real_parent->pid) 15081da177e4SLinus Torvalds */ 1509abd63bc3SKees Cook struct task_struct __rcu *real_parent; /* real parent process */ 1510abd63bc3SKees Cook struct task_struct __rcu *parent; /* recipient of SIGCHLD, wait4() reports */ 15111da177e4SLinus Torvalds /* 1512f470021aSRoland McGrath * children/sibling forms the list of my natural children 15131da177e4SLinus Torvalds */ 15141da177e4SLinus Torvalds struct list_head children; /* list of my children */ 15151da177e4SLinus Torvalds struct list_head sibling; /* linkage in my parent's children list */ 15161da177e4SLinus Torvalds struct task_struct *group_leader; /* threadgroup leader */ 15171da177e4SLinus Torvalds 1518f470021aSRoland McGrath /* 1519f470021aSRoland McGrath * ptraced is the list of tasks this task is using ptrace on. 1520f470021aSRoland McGrath * This includes both natural children and PTRACE_ATTACH targets. 1521f470021aSRoland McGrath * p->ptrace_entry is p's link on the p->parent->ptraced list. 1522f470021aSRoland McGrath */ 1523f470021aSRoland McGrath struct list_head ptraced; 1524f470021aSRoland McGrath struct list_head ptrace_entry; 1525f470021aSRoland McGrath 15261da177e4SLinus Torvalds /* PID/PID hash table linkage. */ 152792476d7fSEric W. Biederman struct pid_link pids[PIDTYPE_MAX]; 152847e65328SOleg Nesterov struct list_head thread_group; 15290c740d0aSOleg Nesterov struct list_head thread_node; 15301da177e4SLinus Torvalds 15311da177e4SLinus Torvalds struct completion *vfork_done; /* for vfork() */ 15321da177e4SLinus Torvalds int __user *set_child_tid; /* CLONE_CHILD_SETTID */ 15331da177e4SLinus Torvalds int __user *clear_child_tid; /* CLONE_CHILD_CLEARTID */ 15341da177e4SLinus Torvalds 1535c66f08beSMichael Neuling cputime_t utime, stime, utimescaled, stimescaled; 15369ac52315SLaurent Vivier cputime_t gtime; 15379d7fb042SPeter Zijlstra struct prev_cputime prev_cputime; 15386a61671bSFrederic Weisbecker #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN 1539b7ce2277SFrederic Weisbecker seqcount_t vtime_seqcount; 15406a61671bSFrederic Weisbecker unsigned long long vtime_snap; 15416a61671bSFrederic Weisbecker enum { 15427098c1eaSFrederic Weisbecker /* Task is sleeping or running in a CPU with VTIME inactive */ 15437098c1eaSFrederic Weisbecker VTIME_INACTIVE = 0, 15447098c1eaSFrederic Weisbecker /* Task runs in userspace in a CPU with VTIME active */ 15456a61671bSFrederic Weisbecker VTIME_USER, 15467098c1eaSFrederic Weisbecker /* Task runs in kernelspace in a CPU with VTIME active */ 15476a61671bSFrederic Weisbecker VTIME_SYS, 15486a61671bSFrederic Weisbecker } vtime_snap_whence; 15496a61671bSFrederic Weisbecker #endif 1550d027d45dSFrederic Weisbecker 1551d027d45dSFrederic Weisbecker #ifdef CONFIG_NO_HZ_FULL 1552*f009a7a7SFrederic Weisbecker atomic_t tick_dep_mask; 1553d027d45dSFrederic Weisbecker #endif 15541da177e4SLinus Torvalds unsigned long nvcsw, nivcsw; /* context switch counts */ 1555ccbf62d8SThomas Gleixner u64 start_time; /* monotonic time in nsec */ 155657e0be04SThomas Gleixner u64 real_start_time; /* boot based time in nsec */ 15571da177e4SLinus Torvalds /* mm fault and swap info: this can arguably be seen as either mm-specific or thread-specific */ 15581da177e4SLinus Torvalds unsigned long min_flt, maj_flt; 15591da177e4SLinus Torvalds 1560f06febc9SFrank Mayhar struct task_cputime cputime_expires; 15611da177e4SLinus Torvalds struct list_head cpu_timers[3]; 15621da177e4SLinus Torvalds 15631da177e4SLinus Torvalds /* process credentials */ 15641b0ba1c9SArnd Bergmann const struct cred __rcu *real_cred; /* objective and real subjective task 15653b11a1deSDavid Howells * credentials (COW) */ 15661b0ba1c9SArnd Bergmann const struct cred __rcu *cred; /* effective (overridable) subjective task 15673b11a1deSDavid Howells * credentials (COW) */ 156836772092SPaolo 'Blaisorblade' Giarrusso char comm[TASK_COMM_LEN]; /* executable name excluding path 156936772092SPaolo 'Blaisorblade' Giarrusso - access with [gs]et_task_comm (which lock 157036772092SPaolo 'Blaisorblade' Giarrusso it with task_lock()) 1571221af7f8SLinus Torvalds - initialized normally by setup_new_exec */ 15721da177e4SLinus Torvalds /* file system info */ 1573756daf26SNeilBrown struct nameidata *nameidata; 15743d5b6fccSAlexey Dobriyan #ifdef CONFIG_SYSVIPC 15751da177e4SLinus Torvalds /* ipc stuff */ 15761da177e4SLinus Torvalds struct sysv_sem sysvsem; 1577ab602f79SJack Miller struct sysv_shm sysvshm; 15783d5b6fccSAlexey Dobriyan #endif 1579e162b39aSMandeep Singh Baines #ifdef CONFIG_DETECT_HUNG_TASK 158082a1fcb9SIngo Molnar /* hung task detection */ 158182a1fcb9SIngo Molnar unsigned long last_switch_count; 158282a1fcb9SIngo Molnar #endif 15831da177e4SLinus Torvalds /* filesystem information */ 15841da177e4SLinus Torvalds struct fs_struct *fs; 15851da177e4SLinus Torvalds /* open file information */ 15861da177e4SLinus Torvalds struct files_struct *files; 15871651e14eSSerge E. Hallyn /* namespaces */ 1588ab516013SSerge E. Hallyn struct nsproxy *nsproxy; 15891da177e4SLinus Torvalds /* signal handlers */ 15901da177e4SLinus Torvalds struct signal_struct *signal; 15911da177e4SLinus Torvalds struct sighand_struct *sighand; 15921da177e4SLinus Torvalds 15931da177e4SLinus Torvalds sigset_t blocked, real_blocked; 1594f3de272bSRoland McGrath sigset_t saved_sigmask; /* restored if set_restore_sigmask() was used */ 15951da177e4SLinus Torvalds struct sigpending pending; 15961da177e4SLinus Torvalds 15971da177e4SLinus Torvalds unsigned long sas_ss_sp; 15981da177e4SLinus Torvalds size_t sas_ss_size; 15992e01fabeSOleg Nesterov 160067d12145SAl Viro struct callback_head *task_works; 1601e73f8959SOleg Nesterov 16021da177e4SLinus Torvalds struct audit_context *audit_context; 1603bfef93a5SAl Viro #ifdef CONFIG_AUDITSYSCALL 1604e1760bd5SEric W. Biederman kuid_t loginuid; 16054746ec5bSEric Paris unsigned int sessionid; 1606bfef93a5SAl Viro #endif 1607932ecebbSWill Drewry struct seccomp seccomp; 16081da177e4SLinus Torvalds 16091da177e4SLinus Torvalds /* Thread group tracking */ 16101da177e4SLinus Torvalds u32 parent_exec_id; 16111da177e4SLinus Torvalds u32 self_exec_id; 161258568d2aSMiao Xie /* Protection of (de-)allocation: mm, files, fs, tty, keyrings, mems_allowed, 161358568d2aSMiao Xie * mempolicy */ 16141da177e4SLinus Torvalds spinlock_t alloc_lock; 16151da177e4SLinus Torvalds 1616b29739f9SIngo Molnar /* Protection of the PI data structures: */ 16171d615482SThomas Gleixner raw_spinlock_t pi_lock; 1618b29739f9SIngo Molnar 161976751049SPeter Zijlstra struct wake_q_node wake_q; 162076751049SPeter Zijlstra 162123f78d4aSIngo Molnar #ifdef CONFIG_RT_MUTEXES 162223f78d4aSIngo Molnar /* PI waiters blocked on a rt_mutex held by this task */ 1623fb00aca4SPeter Zijlstra struct rb_root pi_waiters; 1624fb00aca4SPeter Zijlstra struct rb_node *pi_waiters_leftmost; 162523f78d4aSIngo Molnar /* Deadlock detection and priority inheritance handling */ 162623f78d4aSIngo Molnar struct rt_mutex_waiter *pi_blocked_on; 162723f78d4aSIngo Molnar #endif 162823f78d4aSIngo Molnar 1629408894eeSIngo Molnar #ifdef CONFIG_DEBUG_MUTEXES 1630408894eeSIngo Molnar /* mutex deadlock detection */ 1631408894eeSIngo Molnar struct mutex_waiter *blocked_on; 1632408894eeSIngo Molnar #endif 1633de30a2b3SIngo Molnar #ifdef CONFIG_TRACE_IRQFLAGS 1634de30a2b3SIngo Molnar unsigned int irq_events; 1635de30a2b3SIngo Molnar unsigned long hardirq_enable_ip; 1636de30a2b3SIngo Molnar unsigned long hardirq_disable_ip; 1637fa1452e8SHiroshi Shimamoto unsigned int hardirq_enable_event; 1638de30a2b3SIngo Molnar unsigned int hardirq_disable_event; 1639fa1452e8SHiroshi Shimamoto int hardirqs_enabled; 1640de30a2b3SIngo Molnar int hardirq_context; 1641fa1452e8SHiroshi Shimamoto unsigned long softirq_disable_ip; 1642fa1452e8SHiroshi Shimamoto unsigned long softirq_enable_ip; 1643fa1452e8SHiroshi Shimamoto unsigned int softirq_disable_event; 1644fa1452e8SHiroshi Shimamoto unsigned int softirq_enable_event; 1645fa1452e8SHiroshi Shimamoto int softirqs_enabled; 1646de30a2b3SIngo Molnar int softirq_context; 1647de30a2b3SIngo Molnar #endif 1648fbb9ce95SIngo Molnar #ifdef CONFIG_LOCKDEP 1649bdb9441eSPeter Zijlstra # define MAX_LOCK_DEPTH 48UL 1650fbb9ce95SIngo Molnar u64 curr_chain_key; 1651fbb9ce95SIngo Molnar int lockdep_depth; 1652fbb9ce95SIngo Molnar unsigned int lockdep_recursion; 1653c7aceabaSRichard Kennedy struct held_lock held_locks[MAX_LOCK_DEPTH]; 1654cf40bd16SNick Piggin gfp_t lockdep_reclaim_gfp; 1655fbb9ce95SIngo Molnar #endif 1656c6d30853SAndrey Ryabinin #ifdef CONFIG_UBSAN 1657c6d30853SAndrey Ryabinin unsigned int in_ubsan; 1658c6d30853SAndrey Ryabinin #endif 1659408894eeSIngo Molnar 16601da177e4SLinus Torvalds /* journalling filesystem info */ 16611da177e4SLinus Torvalds void *journal_info; 16621da177e4SLinus Torvalds 1663d89d8796SNeil Brown /* stacked block device info */ 1664bddd87c7SAkinobu Mita struct bio_list *bio_list; 1665d89d8796SNeil Brown 166673c10101SJens Axboe #ifdef CONFIG_BLOCK 166773c10101SJens Axboe /* stack plugging */ 166873c10101SJens Axboe struct blk_plug *plug; 166973c10101SJens Axboe #endif 167073c10101SJens Axboe 16711da177e4SLinus Torvalds /* VM state */ 16721da177e4SLinus Torvalds struct reclaim_state *reclaim_state; 16731da177e4SLinus Torvalds 16741da177e4SLinus Torvalds struct backing_dev_info *backing_dev_info; 16751da177e4SLinus Torvalds 16761da177e4SLinus Torvalds struct io_context *io_context; 16771da177e4SLinus Torvalds 16781da177e4SLinus Torvalds unsigned long ptrace_message; 16791da177e4SLinus Torvalds siginfo_t *last_siginfo; /* For ptrace use. */ 16807c3ab738SAndrew Morton struct task_io_accounting ioac; 16818f0ab514SJay Lan #if defined(CONFIG_TASK_XACCT) 16821da177e4SLinus Torvalds u64 acct_rss_mem1; /* accumulated rss usage */ 16831da177e4SLinus Torvalds u64 acct_vm_mem1; /* accumulated virtual memory usage */ 168449b5cf34SJonathan Lim cputime_t acct_timexpd; /* stime + utime since last update */ 16851da177e4SLinus Torvalds #endif 16861da177e4SLinus Torvalds #ifdef CONFIG_CPUSETS 168758568d2aSMiao Xie nodemask_t mems_allowed; /* Protected by alloc_lock */ 1688cc9a6c87SMel Gorman seqcount_t mems_allowed_seq; /* Seqence no to catch updates */ 1689825a46afSPaul Jackson int cpuset_mem_spread_rotor; 16906adef3ebSJack Steiner int cpuset_slab_spread_rotor; 16911da177e4SLinus Torvalds #endif 1692ddbcc7e8SPaul Menage #ifdef CONFIG_CGROUPS 1693817929ecSPaul Menage /* Control Group info protected by css_set_lock */ 16942c392b8cSArnd Bergmann struct css_set __rcu *cgroups; 1695817929ecSPaul Menage /* cg_list protected by css_set_lock and tsk->alloc_lock */ 1696817929ecSPaul Menage struct list_head cg_list; 1697ddbcc7e8SPaul Menage #endif 169842b2dd0aSAlexey Dobriyan #ifdef CONFIG_FUTEX 16990771dfefSIngo Molnar struct robust_list_head __user *robust_list; 170034f192c6SIngo Molnar #ifdef CONFIG_COMPAT 170134f192c6SIngo Molnar struct compat_robust_list_head __user *compat_robust_list; 170234f192c6SIngo Molnar #endif 1703c87e2837SIngo Molnar struct list_head pi_state_list; 1704c87e2837SIngo Molnar struct futex_pi_state *pi_state_cache; 170542b2dd0aSAlexey Dobriyan #endif 1706cdd6c482SIngo Molnar #ifdef CONFIG_PERF_EVENTS 17078dc85d54SPeter Zijlstra struct perf_event_context *perf_event_ctxp[perf_nr_task_contexts]; 1708cdd6c482SIngo Molnar struct mutex perf_event_mutex; 1709cdd6c482SIngo Molnar struct list_head perf_event_list; 1710a63eaf34SPaul Mackerras #endif 17118f47b187SThomas Gleixner #ifdef CONFIG_DEBUG_PREEMPT 17128f47b187SThomas Gleixner unsigned long preempt_disable_ip; 17138f47b187SThomas Gleixner #endif 1714c7aceabaSRichard Kennedy #ifdef CONFIG_NUMA 171558568d2aSMiao Xie struct mempolicy *mempolicy; /* Protected by alloc_lock */ 1716c7aceabaSRichard Kennedy short il_next; 1717207205a2SEric Dumazet short pref_node_fork; 1718c7aceabaSRichard Kennedy #endif 1719cbee9f88SPeter Zijlstra #ifdef CONFIG_NUMA_BALANCING 1720cbee9f88SPeter Zijlstra int numa_scan_seq; 1721cbee9f88SPeter Zijlstra unsigned int numa_scan_period; 1722598f0ec0SMel Gorman unsigned int numa_scan_period_max; 1723de1c9ce6SRik van Riel int numa_preferred_nid; 17246b9a7460SMel Gorman unsigned long numa_migrate_retry; 1725cbee9f88SPeter Zijlstra u64 node_stamp; /* migration stamp */ 17267e2703e6SRik van Riel u64 last_task_numa_placement; 17277e2703e6SRik van Riel u64 last_sum_exec_runtime; 1728cbee9f88SPeter Zijlstra struct callback_head numa_work; 1729f809ca9aSMel Gorman 17308c8a743cSPeter Zijlstra struct list_head numa_entry; 17318c8a743cSPeter Zijlstra struct numa_group *numa_group; 17328c8a743cSPeter Zijlstra 1733745d6147SMel Gorman /* 173444dba3d5SIulia Manda * numa_faults is an array split into four regions: 173544dba3d5SIulia Manda * faults_memory, faults_cpu, faults_memory_buffer, faults_cpu_buffer 173644dba3d5SIulia Manda * in this precise order. 173744dba3d5SIulia Manda * 173844dba3d5SIulia Manda * faults_memory: Exponential decaying average of faults on a per-node 173944dba3d5SIulia Manda * basis. Scheduling placement decisions are made based on these 174044dba3d5SIulia Manda * counts. The values remain static for the duration of a PTE scan. 174144dba3d5SIulia Manda * faults_cpu: Track the nodes the process was running on when a NUMA 174244dba3d5SIulia Manda * hinting fault was incurred. 174344dba3d5SIulia Manda * faults_memory_buffer and faults_cpu_buffer: Record faults per node 174444dba3d5SIulia Manda * during the current scan window. When the scan completes, the counts 174544dba3d5SIulia Manda * in faults_memory and faults_cpu decay and these values are copied. 1746745d6147SMel Gorman */ 174744dba3d5SIulia Manda unsigned long *numa_faults; 174883e1d2cdSMel Gorman unsigned long total_numa_faults; 1749745d6147SMel Gorman 1750745d6147SMel Gorman /* 175104bb2f94SRik van Riel * numa_faults_locality tracks if faults recorded during the last 1752074c2381SMel Gorman * scan window were remote/local or failed to migrate. The task scan 1753074c2381SMel Gorman * period is adapted based on the locality of the faults with different 1754074c2381SMel Gorman * weights depending on whether they were shared or private faults 175504bb2f94SRik van Riel */ 1756074c2381SMel Gorman unsigned long numa_faults_locality[3]; 175704bb2f94SRik van Riel 1758b32e86b4SIngo Molnar unsigned long numa_pages_migrated; 1759cbee9f88SPeter Zijlstra #endif /* CONFIG_NUMA_BALANCING */ 1760cbee9f88SPeter Zijlstra 176172b252aeSMel Gorman #ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH 176272b252aeSMel Gorman struct tlbflush_unmap_batch tlb_ubc; 176372b252aeSMel Gorman #endif 176472b252aeSMel Gorman 1765e56d0903SIngo Molnar struct rcu_head rcu; 1766b92ce558SJens Axboe 1767b92ce558SJens Axboe /* 1768b92ce558SJens Axboe * cache last used pipe for splice 1769b92ce558SJens Axboe */ 1770b92ce558SJens Axboe struct pipe_inode_info *splice_pipe; 17715640f768SEric Dumazet 17725640f768SEric Dumazet struct page_frag task_frag; 17735640f768SEric Dumazet 1774ca74e92bSShailabh Nagar #ifdef CONFIG_TASK_DELAY_ACCT 1775ca74e92bSShailabh Nagar struct task_delay_info *delays; 1776ca74e92bSShailabh Nagar #endif 1777f4f154fdSAkinobu Mita #ifdef CONFIG_FAULT_INJECTION 1778f4f154fdSAkinobu Mita int make_it_fail; 1779f4f154fdSAkinobu Mita #endif 17809d823e8fSWu Fengguang /* 17819d823e8fSWu Fengguang * when (nr_dirtied >= nr_dirtied_pause), it's time to call 17829d823e8fSWu Fengguang * balance_dirty_pages() for some dirty throttling pause 17839d823e8fSWu Fengguang */ 17849d823e8fSWu Fengguang int nr_dirtied; 17859d823e8fSWu Fengguang int nr_dirtied_pause; 178683712358SWu Fengguang unsigned long dirty_paused_when; /* start of a write-and-pause period */ 17879d823e8fSWu Fengguang 17889745512cSArjan van de Ven #ifdef CONFIG_LATENCYTOP 17899745512cSArjan van de Ven int latency_record_count; 17909745512cSArjan van de Ven struct latency_record latency_record[LT_SAVECOUNT]; 17919745512cSArjan van de Ven #endif 17926976675dSArjan van de Ven /* 17936976675dSArjan van de Ven * time slack values; these are used to round up poll() and 17946976675dSArjan van de Ven * select() etc timeout values. These are in nanoseconds. 17956976675dSArjan van de Ven */ 1796da8b44d5SJohn Stultz u64 timer_slack_ns; 1797da8b44d5SJohn Stultz u64 default_timer_slack_ns; 1798f8d570a4SDavid Miller 17990b24beccSAndrey Ryabinin #ifdef CONFIG_KASAN 18000b24beccSAndrey Ryabinin unsigned int kasan_depth; 18010b24beccSAndrey Ryabinin #endif 1802fb52607aSFrederic Weisbecker #ifdef CONFIG_FUNCTION_GRAPH_TRACER 18033ad2f3fbSDaniel Mack /* Index of current stored address in ret_stack */ 1804f201ae23SFrederic Weisbecker int curr_ret_stack; 1805f201ae23SFrederic Weisbecker /* Stack of return addresses for return function tracing */ 1806f201ae23SFrederic Weisbecker struct ftrace_ret_stack *ret_stack; 18078aef2d28SSteven Rostedt /* time stamp for last schedule */ 18088aef2d28SSteven Rostedt unsigned long long ftrace_timestamp; 1809f201ae23SFrederic Weisbecker /* 1810f201ae23SFrederic Weisbecker * Number of functions that haven't been traced 1811f201ae23SFrederic Weisbecker * because of depth overrun. 1812f201ae23SFrederic Weisbecker */ 1813f201ae23SFrederic Weisbecker atomic_t trace_overrun; 1814380c4b14SFrederic Weisbecker /* Pause for the tracing */ 1815380c4b14SFrederic Weisbecker atomic_t tracing_graph_pause; 1816f201ae23SFrederic Weisbecker #endif 1817ea4e2bc4SSteven Rostedt #ifdef CONFIG_TRACING 1818ea4e2bc4SSteven Rostedt /* state flags for use by tracers */ 1819ea4e2bc4SSteven Rostedt unsigned long trace; 1820b1cff0adSSteven Rostedt /* bitmask and counter of trace recursion */ 1821261842b7SSteven Rostedt unsigned long trace_recursion; 1822261842b7SSteven Rostedt #endif /* CONFIG_TRACING */ 18235c9a8750SDmitry Vyukov #ifdef CONFIG_KCOV 18245c9a8750SDmitry Vyukov /* Coverage collection mode enabled for this task (0 if disabled). */ 18255c9a8750SDmitry Vyukov enum kcov_mode kcov_mode; 18265c9a8750SDmitry Vyukov /* Size of the kcov_area. */ 18275c9a8750SDmitry Vyukov unsigned kcov_size; 18285c9a8750SDmitry Vyukov /* Buffer for coverage collection. */ 18295c9a8750SDmitry Vyukov void *kcov_area; 18305c9a8750SDmitry Vyukov /* kcov desciptor wired with this task or NULL. */ 18315c9a8750SDmitry Vyukov struct kcov *kcov; 18325c9a8750SDmitry Vyukov #endif 18336f185c29SVladimir Davydov #ifdef CONFIG_MEMCG 1834626ebc41STejun Heo struct mem_cgroup *memcg_in_oom; 1835626ebc41STejun Heo gfp_t memcg_oom_gfp_mask; 1836626ebc41STejun Heo int memcg_oom_order; 1837b23afb93STejun Heo 1838b23afb93STejun Heo /* number of pages to reclaim on returning to userland */ 1839b23afb93STejun Heo unsigned int memcg_nr_pages_over_high; 1840569b846dSKAMEZAWA Hiroyuki #endif 18410326f5a9SSrikar Dronamraju #ifdef CONFIG_UPROBES 18420326f5a9SSrikar Dronamraju struct uprobe_task *utask; 18430326f5a9SSrikar Dronamraju #endif 1844cafe5635SKent Overstreet #if defined(CONFIG_BCACHE) || defined(CONFIG_BCACHE_MODULE) 1845cafe5635SKent Overstreet unsigned int sequential_io; 1846cafe5635SKent Overstreet unsigned int sequential_io_avg; 1847cafe5635SKent Overstreet #endif 18488eb23b9fSPeter Zijlstra #ifdef CONFIG_DEBUG_ATOMIC_SLEEP 18498eb23b9fSPeter Zijlstra unsigned long task_state_change; 18508eb23b9fSPeter Zijlstra #endif 18518bcbde54SDavid Hildenbrand int pagefault_disabled; 185203049269SMichal Hocko #ifdef CONFIG_MMU 185329c696e1SVladimir Davydov struct task_struct *oom_reaper_list; 185403049269SMichal Hocko #endif 18550c8c0f03SDave Hansen /* CPU-specific state of this task */ 18560c8c0f03SDave Hansen struct thread_struct thread; 18570c8c0f03SDave Hansen /* 18580c8c0f03SDave Hansen * WARNING: on x86, 'thread_struct' contains a variable-sized 18590c8c0f03SDave Hansen * structure. It *MUST* be at the end of 'task_struct'. 18600c8c0f03SDave Hansen * 18610c8c0f03SDave Hansen * Do not put anything below here! 18620c8c0f03SDave Hansen */ 18631da177e4SLinus Torvalds }; 18641da177e4SLinus Torvalds 18655aaeb5c0SIngo Molnar #ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT 18665aaeb5c0SIngo Molnar extern int arch_task_struct_size __read_mostly; 18675aaeb5c0SIngo Molnar #else 18685aaeb5c0SIngo Molnar # define arch_task_struct_size (sizeof(struct task_struct)) 18695aaeb5c0SIngo Molnar #endif 18700c8c0f03SDave Hansen 187176e6eee0SRusty Russell /* Future-safe accessor for struct task_struct's cpus_allowed. */ 1872a4636818SRusty Russell #define tsk_cpus_allowed(tsk) (&(tsk)->cpus_allowed) 187376e6eee0SRusty Russell 18746688cc05SPeter Zijlstra #define TNF_MIGRATED 0x01 18756688cc05SPeter Zijlstra #define TNF_NO_GROUP 0x02 1876dabe1d99SRik van Riel #define TNF_SHARED 0x04 187704bb2f94SRik van Riel #define TNF_FAULT_LOCAL 0x08 1878074c2381SMel Gorman #define TNF_MIGRATE_FAIL 0x10 18796688cc05SPeter Zijlstra 1880cbee9f88SPeter Zijlstra #ifdef CONFIG_NUMA_BALANCING 18816688cc05SPeter Zijlstra extern void task_numa_fault(int last_node, int node, int pages, int flags); 1882e29cf08bSMel Gorman extern pid_t task_numa_group_id(struct task_struct *p); 18831a687c2eSMel Gorman extern void set_numabalancing_state(bool enabled); 188482727018SRik van Riel extern void task_numa_free(struct task_struct *p); 188510f39042SRik van Riel extern bool should_numa_migrate_memory(struct task_struct *p, struct page *page, 188610f39042SRik van Riel int src_nid, int dst_cpu); 1887cbee9f88SPeter Zijlstra #else 1888ac8e895bSMel Gorman static inline void task_numa_fault(int last_node, int node, int pages, 18896688cc05SPeter Zijlstra int flags) 1890cbee9f88SPeter Zijlstra { 1891cbee9f88SPeter Zijlstra } 1892e29cf08bSMel Gorman static inline pid_t task_numa_group_id(struct task_struct *p) 1893e29cf08bSMel Gorman { 1894e29cf08bSMel Gorman return 0; 1895e29cf08bSMel Gorman } 18961a687c2eSMel Gorman static inline void set_numabalancing_state(bool enabled) 18971a687c2eSMel Gorman { 18981a687c2eSMel Gorman } 189982727018SRik van Riel static inline void task_numa_free(struct task_struct *p) 190082727018SRik van Riel { 190182727018SRik van Riel } 190210f39042SRik van Riel static inline bool should_numa_migrate_memory(struct task_struct *p, 190310f39042SRik van Riel struct page *page, int src_nid, int dst_cpu) 190410f39042SRik van Riel { 190510f39042SRik van Riel return true; 190610f39042SRik van Riel } 1907cbee9f88SPeter Zijlstra #endif 1908cbee9f88SPeter Zijlstra 1909e868171aSAlexey Dobriyan static inline struct pid *task_pid(struct task_struct *task) 191022c935f4SEric W. Biederman { 191122c935f4SEric W. Biederman return task->pids[PIDTYPE_PID].pid; 191222c935f4SEric W. Biederman } 191322c935f4SEric W. Biederman 1914e868171aSAlexey Dobriyan static inline struct pid *task_tgid(struct task_struct *task) 191522c935f4SEric W. Biederman { 191622c935f4SEric W. Biederman return task->group_leader->pids[PIDTYPE_PID].pid; 191722c935f4SEric W. Biederman } 191822c935f4SEric W. Biederman 19196dda81f4SOleg Nesterov /* 19206dda81f4SOleg Nesterov * Without tasklist or rcu lock it is not safe to dereference 19216dda81f4SOleg Nesterov * the result of task_pgrp/task_session even if task == current, 19226dda81f4SOleg Nesterov * we can race with another thread doing sys_setsid/sys_setpgid. 19236dda81f4SOleg Nesterov */ 1924e868171aSAlexey Dobriyan static inline struct pid *task_pgrp(struct task_struct *task) 192522c935f4SEric W. Biederman { 192622c935f4SEric W. Biederman return task->group_leader->pids[PIDTYPE_PGID].pid; 192722c935f4SEric W. Biederman } 192822c935f4SEric W. Biederman 1929e868171aSAlexey Dobriyan static inline struct pid *task_session(struct task_struct *task) 193022c935f4SEric W. Biederman { 193122c935f4SEric W. Biederman return task->group_leader->pids[PIDTYPE_SID].pid; 193222c935f4SEric W. Biederman } 193322c935f4SEric W. Biederman 19347af57294SPavel Emelyanov struct pid_namespace; 19357af57294SPavel Emelyanov 19367af57294SPavel Emelyanov /* 19377af57294SPavel Emelyanov * the helpers to get the task's different pids as they are seen 19387af57294SPavel Emelyanov * from various namespaces 19397af57294SPavel Emelyanov * 19407af57294SPavel Emelyanov * task_xid_nr() : global id, i.e. the id seen from the init namespace; 194144c4e1b2SEric W. Biederman * task_xid_vnr() : virtual id, i.e. the id seen from the pid namespace of 194244c4e1b2SEric W. Biederman * current. 19437af57294SPavel Emelyanov * task_xid_nr_ns() : id seen from the ns specified; 19447af57294SPavel Emelyanov * 19457af57294SPavel Emelyanov * set_task_vxid() : assigns a virtual id to a task; 19467af57294SPavel Emelyanov * 19477af57294SPavel Emelyanov * see also pid_nr() etc in include/linux/pid.h 19487af57294SPavel Emelyanov */ 194952ee2dfdSOleg Nesterov pid_t __task_pid_nr_ns(struct task_struct *task, enum pid_type type, 195052ee2dfdSOleg Nesterov struct pid_namespace *ns); 19517af57294SPavel Emelyanov 1952e868171aSAlexey Dobriyan static inline pid_t task_pid_nr(struct task_struct *tsk) 19537af57294SPavel Emelyanov { 19547af57294SPavel Emelyanov return tsk->pid; 19557af57294SPavel Emelyanov } 19567af57294SPavel Emelyanov 195752ee2dfdSOleg Nesterov static inline pid_t task_pid_nr_ns(struct task_struct *tsk, 195852ee2dfdSOleg Nesterov struct pid_namespace *ns) 195952ee2dfdSOleg Nesterov { 196052ee2dfdSOleg Nesterov return __task_pid_nr_ns(tsk, PIDTYPE_PID, ns); 196152ee2dfdSOleg Nesterov } 19627af57294SPavel Emelyanov 19637af57294SPavel Emelyanov static inline pid_t task_pid_vnr(struct task_struct *tsk) 19647af57294SPavel Emelyanov { 196552ee2dfdSOleg Nesterov return __task_pid_nr_ns(tsk, PIDTYPE_PID, NULL); 19667af57294SPavel Emelyanov } 19677af57294SPavel Emelyanov 19687af57294SPavel Emelyanov 1969e868171aSAlexey Dobriyan static inline pid_t task_tgid_nr(struct task_struct *tsk) 19707af57294SPavel Emelyanov { 19717af57294SPavel Emelyanov return tsk->tgid; 19727af57294SPavel Emelyanov } 19737af57294SPavel Emelyanov 19742f2a3a46SPavel Emelyanov pid_t task_tgid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns); 19757af57294SPavel Emelyanov 19767af57294SPavel Emelyanov static inline pid_t task_tgid_vnr(struct task_struct *tsk) 19777af57294SPavel Emelyanov { 19787af57294SPavel Emelyanov return pid_vnr(task_tgid(tsk)); 19797af57294SPavel Emelyanov } 19807af57294SPavel Emelyanov 19817af57294SPavel Emelyanov 198280e0b6e8SRichard Guy Briggs static inline int pid_alive(const struct task_struct *p); 1983ad36d282SRichard Guy Briggs static inline pid_t task_ppid_nr_ns(const struct task_struct *tsk, struct pid_namespace *ns) 1984ad36d282SRichard Guy Briggs { 1985ad36d282SRichard Guy Briggs pid_t pid = 0; 1986ad36d282SRichard Guy Briggs 1987ad36d282SRichard Guy Briggs rcu_read_lock(); 1988ad36d282SRichard Guy Briggs if (pid_alive(tsk)) 1989ad36d282SRichard Guy Briggs pid = task_tgid_nr_ns(rcu_dereference(tsk->real_parent), ns); 1990ad36d282SRichard Guy Briggs rcu_read_unlock(); 1991ad36d282SRichard Guy Briggs 1992ad36d282SRichard Guy Briggs return pid; 1993ad36d282SRichard Guy Briggs } 1994ad36d282SRichard Guy Briggs 1995ad36d282SRichard Guy Briggs static inline pid_t task_ppid_nr(const struct task_struct *tsk) 1996ad36d282SRichard Guy Briggs { 1997ad36d282SRichard Guy Briggs return task_ppid_nr_ns(tsk, &init_pid_ns); 1998ad36d282SRichard Guy Briggs } 1999ad36d282SRichard Guy Briggs 200052ee2dfdSOleg Nesterov static inline pid_t task_pgrp_nr_ns(struct task_struct *tsk, 200152ee2dfdSOleg Nesterov struct pid_namespace *ns) 20027af57294SPavel Emelyanov { 200352ee2dfdSOleg Nesterov return __task_pid_nr_ns(tsk, PIDTYPE_PGID, ns); 20047af57294SPavel Emelyanov } 20057af57294SPavel Emelyanov 20067af57294SPavel Emelyanov static inline pid_t task_pgrp_vnr(struct task_struct *tsk) 20077af57294SPavel Emelyanov { 200852ee2dfdSOleg Nesterov return __task_pid_nr_ns(tsk, PIDTYPE_PGID, NULL); 20097af57294SPavel Emelyanov } 20107af57294SPavel Emelyanov 20117af57294SPavel Emelyanov 201252ee2dfdSOleg Nesterov static inline pid_t task_session_nr_ns(struct task_struct *tsk, 201352ee2dfdSOleg Nesterov struct pid_namespace *ns) 20147af57294SPavel Emelyanov { 201552ee2dfdSOleg Nesterov return __task_pid_nr_ns(tsk, PIDTYPE_SID, ns); 20167af57294SPavel Emelyanov } 20177af57294SPavel Emelyanov 20187af57294SPavel Emelyanov static inline pid_t task_session_vnr(struct task_struct *tsk) 20197af57294SPavel Emelyanov { 202052ee2dfdSOleg Nesterov return __task_pid_nr_ns(tsk, PIDTYPE_SID, NULL); 20217af57294SPavel Emelyanov } 20227af57294SPavel Emelyanov 20231b0f7ffdSOleg Nesterov /* obsolete, do not use */ 20241b0f7ffdSOleg Nesterov static inline pid_t task_pgrp_nr(struct task_struct *tsk) 20251b0f7ffdSOleg Nesterov { 20261b0f7ffdSOleg Nesterov return task_pgrp_nr_ns(tsk, &init_pid_ns); 20271b0f7ffdSOleg Nesterov } 20287af57294SPavel Emelyanov 20291da177e4SLinus Torvalds /** 20301da177e4SLinus Torvalds * pid_alive - check that a task structure is not stale 20311da177e4SLinus Torvalds * @p: Task structure to be checked. 20321da177e4SLinus Torvalds * 20331da177e4SLinus Torvalds * Test if a process is not yet dead (at most zombie state) 20341da177e4SLinus Torvalds * If pid_alive fails, then pointers within the task structure 20351da177e4SLinus Torvalds * can be stale and must not be dereferenced. 2036e69f6186SYacine Belkadi * 2037e69f6186SYacine Belkadi * Return: 1 if the process is alive. 0 otherwise. 20381da177e4SLinus Torvalds */ 2039ad36d282SRichard Guy Briggs static inline int pid_alive(const struct task_struct *p) 20401da177e4SLinus Torvalds { 204192476d7fSEric W. Biederman return p->pids[PIDTYPE_PID].pid != NULL; 20421da177e4SLinus Torvalds } 20431da177e4SLinus Torvalds 2044f400e198SSukadev Bhattiprolu /** 2045570f5241SSergey Senozhatsky * is_global_init - check if a task structure is init. Since init 2046570f5241SSergey Senozhatsky * is free to have sub-threads we need to check tgid. 20473260259fSHenne * @tsk: Task structure to be checked. 20483260259fSHenne * 20493260259fSHenne * Check if a task structure is the first user space task the kernel created. 2050e69f6186SYacine Belkadi * 2051e69f6186SYacine Belkadi * Return: 1 if the task structure is init. 0 otherwise. 2052f400e198SSukadev Bhattiprolu */ 2053e868171aSAlexey Dobriyan static inline int is_global_init(struct task_struct *tsk) 2054b461cc03SPavel Emelyanov { 2055570f5241SSergey Senozhatsky return task_tgid_nr(tsk) == 1; 2056b461cc03SPavel Emelyanov } 2057b460cbc5SSerge E. Hallyn 20589ec52099SCedric Le Goater extern struct pid *cad_pid; 20599ec52099SCedric Le Goater 20601da177e4SLinus Torvalds extern void free_task(struct task_struct *tsk); 20611da177e4SLinus Torvalds #define get_task_struct(tsk) do { atomic_inc(&(tsk)->usage); } while(0) 2062e56d0903SIngo Molnar 2063158d9ebdSAndrew Morton extern void __put_task_struct(struct task_struct *t); 2064e56d0903SIngo Molnar 2065e56d0903SIngo Molnar static inline void put_task_struct(struct task_struct *t) 2066e56d0903SIngo Molnar { 2067e56d0903SIngo Molnar if (atomic_dec_and_test(&t->usage)) 20688c7904a0SEric W. Biederman __put_task_struct(t); 2069e56d0903SIngo Molnar } 20701da177e4SLinus Torvalds 20716a61671bSFrederic Weisbecker #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN 20726a61671bSFrederic Weisbecker extern void task_cputime(struct task_struct *t, 20736a61671bSFrederic Weisbecker cputime_t *utime, cputime_t *stime); 20746a61671bSFrederic Weisbecker extern void task_cputime_scaled(struct task_struct *t, 20756a61671bSFrederic Weisbecker cputime_t *utimescaled, cputime_t *stimescaled); 20766a61671bSFrederic Weisbecker extern cputime_t task_gtime(struct task_struct *t); 20776a61671bSFrederic Weisbecker #else 20786fac4829SFrederic Weisbecker static inline void task_cputime(struct task_struct *t, 20796fac4829SFrederic Weisbecker cputime_t *utime, cputime_t *stime) 20806fac4829SFrederic Weisbecker { 20816fac4829SFrederic Weisbecker if (utime) 20826fac4829SFrederic Weisbecker *utime = t->utime; 20836fac4829SFrederic Weisbecker if (stime) 20846fac4829SFrederic Weisbecker *stime = t->stime; 20856fac4829SFrederic Weisbecker } 20866fac4829SFrederic Weisbecker 20876fac4829SFrederic Weisbecker static inline void task_cputime_scaled(struct task_struct *t, 20886fac4829SFrederic Weisbecker cputime_t *utimescaled, 20896fac4829SFrederic Weisbecker cputime_t *stimescaled) 20906fac4829SFrederic Weisbecker { 20916fac4829SFrederic Weisbecker if (utimescaled) 20926fac4829SFrederic Weisbecker *utimescaled = t->utimescaled; 20936fac4829SFrederic Weisbecker if (stimescaled) 20946fac4829SFrederic Weisbecker *stimescaled = t->stimescaled; 20956fac4829SFrederic Weisbecker } 20966a61671bSFrederic Weisbecker 20976a61671bSFrederic Weisbecker static inline cputime_t task_gtime(struct task_struct *t) 20986a61671bSFrederic Weisbecker { 20996a61671bSFrederic Weisbecker return t->gtime; 21006a61671bSFrederic Weisbecker } 21016a61671bSFrederic Weisbecker #endif 2102e80d0a1aSFrederic Weisbecker extern void task_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime_t *st); 2103e80d0a1aSFrederic Weisbecker extern void thread_group_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime_t *st); 210449048622SBalbir Singh 21051da177e4SLinus Torvalds /* 21061da177e4SLinus Torvalds * Per process flags 21071da177e4SLinus Torvalds */ 21081da177e4SLinus Torvalds #define PF_EXITING 0x00000004 /* getting shut down */ 2109778e9a9cSAlexey Kuznetsov #define PF_EXITPIDONE 0x00000008 /* pi exit done on shut down */ 211094886b84SLaurent Vivier #define PF_VCPU 0x00000010 /* I'm a virtual CPU */ 211121aa9af0STejun Heo #define PF_WQ_WORKER 0x00000020 /* I'm a workqueue worker */ 21121da177e4SLinus Torvalds #define PF_FORKNOEXEC 0x00000040 /* forked but didn't exec */ 21134db96cf0SAndi Kleen #define PF_MCE_PROCESS 0x00000080 /* process policy on mce errors */ 21141da177e4SLinus Torvalds #define PF_SUPERPRIV 0x00000100 /* used super-user privileges */ 21151da177e4SLinus Torvalds #define PF_DUMPCORE 0x00000200 /* dumped core */ 21161da177e4SLinus Torvalds #define PF_SIGNALED 0x00000400 /* killed by a signal */ 21171da177e4SLinus Torvalds #define PF_MEMALLOC 0x00000800 /* Allocating memory */ 211872fa5997SVasiliy Kulikov #define PF_NPROC_EXCEEDED 0x00001000 /* set_user noticed that RLIMIT_NPROC was exceeded */ 21191da177e4SLinus Torvalds #define PF_USED_MATH 0x00002000 /* if unset the fpu must be initialized before use */ 2120774a1221STejun Heo #define PF_USED_ASYNC 0x00004000 /* used async_schedule*(), used by module init */ 21211da177e4SLinus Torvalds #define PF_NOFREEZE 0x00008000 /* this thread should not be frozen */ 21221da177e4SLinus Torvalds #define PF_FROZEN 0x00010000 /* frozen for system suspend */ 21231da177e4SLinus Torvalds #define PF_FSTRANS 0x00020000 /* inside a filesystem transaction */ 21241da177e4SLinus Torvalds #define PF_KSWAPD 0x00040000 /* I am kswapd */ 212521caf2fcSMing Lei #define PF_MEMALLOC_NOIO 0x00080000 /* Allocating memory without IO involved */ 21261da177e4SLinus Torvalds #define PF_LESS_THROTTLE 0x00100000 /* Throttle me less: I clean memory */ 2127246bb0b1SOleg Nesterov #define PF_KTHREAD 0x00200000 /* I am a kernel thread */ 2128b31dc66aSJens Axboe #define PF_RANDOMIZE 0x00400000 /* randomize virtual address space */ 2129b31dc66aSJens Axboe #define PF_SWAPWRITE 0x00800000 /* Allowed to write to swap */ 213014a40ffcSTejun Heo #define PF_NO_SETAFFINITY 0x04000000 /* Userland is not allowed to meddle with cpus_allowed */ 21314db96cf0SAndi Kleen #define PF_MCE_EARLY 0x08000000 /* Early kill for mce process policy */ 213261a87122SThomas Gleixner #define PF_MUTEX_TESTER 0x20000000 /* Thread belongs to the rt mutex tester */ 213358a69cb4STejun Heo #define PF_FREEZER_SKIP 0x40000000 /* Freezer should not count it as freezable */ 21342b44c4dbSColin Cross #define PF_SUSPEND_TASK 0x80000000 /* this thread called freeze_processes and should not be frozen */ 21351da177e4SLinus Torvalds 21361da177e4SLinus Torvalds /* 21371da177e4SLinus Torvalds * Only the _current_ task can read/write to tsk->flags, but other 21381da177e4SLinus Torvalds * tasks can access tsk->flags in readonly mode for example 21391da177e4SLinus Torvalds * with tsk_used_math (like during threaded core dumping). 21401da177e4SLinus Torvalds * There is however an exception to this rule during ptrace 21411da177e4SLinus Torvalds * or during fork: the ptracer task is allowed to write to the 21421da177e4SLinus Torvalds * child->flags of its traced child (same goes for fork, the parent 21431da177e4SLinus Torvalds * can write to the child->flags), because we're guaranteed the 21441da177e4SLinus Torvalds * child is not running and in turn not changing child->flags 21451da177e4SLinus Torvalds * at the same time the parent does it. 21461da177e4SLinus Torvalds */ 21471da177e4SLinus Torvalds #define clear_stopped_child_used_math(child) do { (child)->flags &= ~PF_USED_MATH; } while (0) 21481da177e4SLinus Torvalds #define set_stopped_child_used_math(child) do { (child)->flags |= PF_USED_MATH; } while (0) 21491da177e4SLinus Torvalds #define clear_used_math() clear_stopped_child_used_math(current) 21501da177e4SLinus Torvalds #define set_used_math() set_stopped_child_used_math(current) 21511da177e4SLinus Torvalds #define conditional_stopped_child_used_math(condition, child) \ 21521da177e4SLinus Torvalds do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= (condition) ? PF_USED_MATH : 0; } while (0) 21531da177e4SLinus Torvalds #define conditional_used_math(condition) \ 21541da177e4SLinus Torvalds conditional_stopped_child_used_math(condition, current) 21551da177e4SLinus Torvalds #define copy_to_stopped_child_used_math(child) \ 21561da177e4SLinus Torvalds do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= current->flags & PF_USED_MATH; } while (0) 21571da177e4SLinus Torvalds /* NOTE: this will return 0 or PF_USED_MATH, it will never return 1 */ 21581da177e4SLinus Torvalds #define tsk_used_math(p) ((p)->flags & PF_USED_MATH) 21591da177e4SLinus Torvalds #define used_math() tsk_used_math(current) 21601da177e4SLinus Torvalds 2161934f3072SJunxiao Bi /* __GFP_IO isn't allowed if PF_MEMALLOC_NOIO is set in current->flags 2162934f3072SJunxiao Bi * __GFP_FS is also cleared as it implies __GFP_IO. 2163934f3072SJunxiao Bi */ 216421caf2fcSMing Lei static inline gfp_t memalloc_noio_flags(gfp_t flags) 216521caf2fcSMing Lei { 216621caf2fcSMing Lei if (unlikely(current->flags & PF_MEMALLOC_NOIO)) 2167934f3072SJunxiao Bi flags &= ~(__GFP_IO | __GFP_FS); 216821caf2fcSMing Lei return flags; 216921caf2fcSMing Lei } 217021caf2fcSMing Lei 217121caf2fcSMing Lei static inline unsigned int memalloc_noio_save(void) 217221caf2fcSMing Lei { 217321caf2fcSMing Lei unsigned int flags = current->flags & PF_MEMALLOC_NOIO; 217421caf2fcSMing Lei current->flags |= PF_MEMALLOC_NOIO; 217521caf2fcSMing Lei return flags; 217621caf2fcSMing Lei } 217721caf2fcSMing Lei 217821caf2fcSMing Lei static inline void memalloc_noio_restore(unsigned int flags) 217921caf2fcSMing Lei { 218021caf2fcSMing Lei current->flags = (current->flags & ~PF_MEMALLOC_NOIO) | flags; 218121caf2fcSMing Lei } 218221caf2fcSMing Lei 21831d4457f9SKees Cook /* Per-process atomic flags. */ 2184a2b86f77SZefan Li #define PFA_NO_NEW_PRIVS 0 /* May not gain new privileges. */ 21852ad654bcSZefan Li #define PFA_SPREAD_PAGE 1 /* Spread page cache over cpuset */ 21862ad654bcSZefan Li #define PFA_SPREAD_SLAB 2 /* Spread some slab caches over cpuset */ 21871d4457f9SKees Cook 21881d4457f9SKees Cook 2189e0e5070bSZefan Li #define TASK_PFA_TEST(name, func) \ 2190e0e5070bSZefan Li static inline bool task_##func(struct task_struct *p) \ 2191e0e5070bSZefan Li { return test_bit(PFA_##name, &p->atomic_flags); } 2192e0e5070bSZefan Li #define TASK_PFA_SET(name, func) \ 2193e0e5070bSZefan Li static inline void task_set_##func(struct task_struct *p) \ 2194e0e5070bSZefan Li { set_bit(PFA_##name, &p->atomic_flags); } 2195e0e5070bSZefan Li #define TASK_PFA_CLEAR(name, func) \ 2196e0e5070bSZefan Li static inline void task_clear_##func(struct task_struct *p) \ 2197e0e5070bSZefan Li { clear_bit(PFA_##name, &p->atomic_flags); } 21981d4457f9SKees Cook 2199e0e5070bSZefan Li TASK_PFA_TEST(NO_NEW_PRIVS, no_new_privs) 2200e0e5070bSZefan Li TASK_PFA_SET(NO_NEW_PRIVS, no_new_privs) 22011d4457f9SKees Cook 22022ad654bcSZefan Li TASK_PFA_TEST(SPREAD_PAGE, spread_page) 22032ad654bcSZefan Li TASK_PFA_SET(SPREAD_PAGE, spread_page) 22042ad654bcSZefan Li TASK_PFA_CLEAR(SPREAD_PAGE, spread_page) 22052ad654bcSZefan Li 22062ad654bcSZefan Li TASK_PFA_TEST(SPREAD_SLAB, spread_slab) 22072ad654bcSZefan Li TASK_PFA_SET(SPREAD_SLAB, spread_slab) 22082ad654bcSZefan Li TASK_PFA_CLEAR(SPREAD_SLAB, spread_slab) 2209544b2c91STejun Heo 2210a8f072c1STejun Heo /* 2211a8f072c1STejun Heo * task->jobctl flags 2212a8f072c1STejun Heo */ 2213a8f072c1STejun Heo #define JOBCTL_STOP_SIGMASK 0xffff /* signr of the last group stop */ 221473ddff2bSTejun Heo 2215fb1d910cSTejun Heo #define JOBCTL_STOP_DEQUEUED_BIT 16 /* stop signal dequeued */ 2216a8f072c1STejun Heo #define JOBCTL_STOP_PENDING_BIT 17 /* task should stop for group stop */ 2217544b2c91STejun Heo #define JOBCTL_STOP_CONSUME_BIT 18 /* consume group stop count */ 2218a8f072c1STejun Heo #define JOBCTL_TRAP_STOP_BIT 19 /* trap for STOP */ 2219fb1d910cSTejun Heo #define JOBCTL_TRAP_NOTIFY_BIT 20 /* trap for NOTIFY */ 222073ddff2bSTejun Heo #define JOBCTL_TRAPPING_BIT 21 /* switching to TRACED */ 22213759a0d9STejun Heo #define JOBCTL_LISTENING_BIT 22 /* ptracer is listening for events */ 22227dd3db54STejun Heo 2223b76808e6SPalmer Dabbelt #define JOBCTL_STOP_DEQUEUED (1UL << JOBCTL_STOP_DEQUEUED_BIT) 2224b76808e6SPalmer Dabbelt #define JOBCTL_STOP_PENDING (1UL << JOBCTL_STOP_PENDING_BIT) 2225b76808e6SPalmer Dabbelt #define JOBCTL_STOP_CONSUME (1UL << JOBCTL_STOP_CONSUME_BIT) 2226b76808e6SPalmer Dabbelt #define JOBCTL_TRAP_STOP (1UL << JOBCTL_TRAP_STOP_BIT) 2227b76808e6SPalmer Dabbelt #define JOBCTL_TRAP_NOTIFY (1UL << JOBCTL_TRAP_NOTIFY_BIT) 2228b76808e6SPalmer Dabbelt #define JOBCTL_TRAPPING (1UL << JOBCTL_TRAPPING_BIT) 2229b76808e6SPalmer Dabbelt #define JOBCTL_LISTENING (1UL << JOBCTL_LISTENING_BIT) 22303759a0d9STejun Heo 22313759a0d9STejun Heo #define JOBCTL_TRAP_MASK (JOBCTL_TRAP_STOP | JOBCTL_TRAP_NOTIFY) 22327dd3db54STejun Heo #define JOBCTL_PENDING_MASK (JOBCTL_STOP_PENDING | JOBCTL_TRAP_MASK) 22337dd3db54STejun Heo 223473ddff2bSTejun Heo extern bool task_set_jobctl_pending(struct task_struct *task, 2235b76808e6SPalmer Dabbelt unsigned long mask); 2236a57eb940SPaul E. McKenney extern void task_clear_jobctl_trapping(struct task_struct *task); 2237f41d911fSPaul E. McKenney extern void task_clear_jobctl_pending(struct task_struct *task, 2238b76808e6SPalmer Dabbelt unsigned long mask); 2239a57eb940SPaul E. McKenney 2240f41d911fSPaul E. McKenney static inline void rcu_copy_process(struct task_struct *p) 2241f41d911fSPaul E. McKenney { 22421da177e4SLinus Torvalds #ifdef CONFIG_PREEMPT_RCU 2243f41d911fSPaul E. McKenney p->rcu_read_lock_nesting = 0; 22441d082fd0SPaul E. McKenney p->rcu_read_unlock_special.s = 0; 2245dd5d19baSPaul E. McKenney p->rcu_blocked_node = NULL; 2246f41d911fSPaul E. McKenney INIT_LIST_HEAD(&p->rcu_node_entry); 22478315f422SPaul E. McKenney #endif /* #ifdef CONFIG_PREEMPT_RCU */ 22488315f422SPaul E. McKenney #ifdef CONFIG_TASKS_RCU 22498315f422SPaul E. McKenney p->rcu_tasks_holdout = false; 22508315f422SPaul E. McKenney INIT_LIST_HEAD(&p->rcu_tasks_holdout_list); 2251176f8f7aSPaul E. McKenney p->rcu_tasks_idle_cpu = -1; 22528315f422SPaul E. McKenney #endif /* #ifdef CONFIG_TASKS_RCU */ 2253f41d911fSPaul E. McKenney } 2254f41d911fSPaul E. McKenney 2255907aed48SMel Gorman static inline void tsk_restore_flags(struct task_struct *task, 2256907aed48SMel Gorman unsigned long orig_flags, unsigned long flags) 2257907aed48SMel Gorman { 2258907aed48SMel Gorman task->flags &= ~flags; 2259907aed48SMel Gorman task->flags |= orig_flags & flags; 2260907aed48SMel Gorman } 2261907aed48SMel Gorman 2262f82f8042SJuri Lelli extern int cpuset_cpumask_can_shrink(const struct cpumask *cur, 2263f82f8042SJuri Lelli const struct cpumask *trial); 22647f51412aSJuri Lelli extern int task_can_attach(struct task_struct *p, 22657f51412aSJuri Lelli const struct cpumask *cs_cpus_allowed); 22661da177e4SLinus Torvalds #ifdef CONFIG_SMP 22671e1b6c51SKOSAKI Motohiro extern void do_set_cpus_allowed(struct task_struct *p, 22681e1b6c51SKOSAKI Motohiro const struct cpumask *new_mask); 22691e1b6c51SKOSAKI Motohiro 2270cd8ba7cdSMike Travis extern int set_cpus_allowed_ptr(struct task_struct *p, 227196f874e2SRusty Russell const struct cpumask *new_mask); 22721da177e4SLinus Torvalds #else 22731e1b6c51SKOSAKI Motohiro static inline void do_set_cpus_allowed(struct task_struct *p, 22741e1b6c51SKOSAKI Motohiro const struct cpumask *new_mask) 22751e1b6c51SKOSAKI Motohiro { 22761e1b6c51SKOSAKI Motohiro } 2277cd8ba7cdSMike Travis static inline int set_cpus_allowed_ptr(struct task_struct *p, 227896f874e2SRusty Russell const struct cpumask *new_mask) 22791da177e4SLinus Torvalds { 228096f874e2SRusty Russell if (!cpumask_test_cpu(0, new_mask)) 22811da177e4SLinus Torvalds return -EINVAL; 22821da177e4SLinus Torvalds return 0; 22831da177e4SLinus Torvalds } 22841da177e4SLinus Torvalds #endif 2285e0ad9556SRusty Russell 22863451d024SFrederic Weisbecker #ifdef CONFIG_NO_HZ_COMMON 22875167e8d5SPeter Zijlstra void calc_load_enter_idle(void); 22885167e8d5SPeter Zijlstra void calc_load_exit_idle(void); 22895167e8d5SPeter Zijlstra #else 22905167e8d5SPeter Zijlstra static inline void calc_load_enter_idle(void) { } 22915167e8d5SPeter Zijlstra static inline void calc_load_exit_idle(void) { } 22923451d024SFrederic Weisbecker #endif /* CONFIG_NO_HZ_COMMON */ 22935167e8d5SPeter Zijlstra 2294b342501cSIngo Molnar /* 2295c676329aSPeter Zijlstra * Do not use outside of architecture code which knows its limitations. 2296c676329aSPeter Zijlstra * 2297c676329aSPeter Zijlstra * sched_clock() has no promise of monotonicity or bounded drift between 2298c676329aSPeter Zijlstra * CPUs, use (which you should not) requires disabling IRQs. 2299c676329aSPeter Zijlstra * 2300c676329aSPeter Zijlstra * Please use one of the three interfaces below. 2301b342501cSIngo Molnar */ 23021bbfa6f2SMike Frysinger extern unsigned long long notrace sched_clock(void); 2303c676329aSPeter Zijlstra /* 2304489a71b0SHiroshi Shimamoto * See the comment in kernel/sched/clock.c 2305c676329aSPeter Zijlstra */ 2306c676329aSPeter Zijlstra extern u64 cpu_clock(int cpu); 2307c676329aSPeter Zijlstra extern u64 local_clock(void); 2308545a2bf7SCyril Bur extern u64 running_clock(void); 2309c676329aSPeter Zijlstra extern u64 sched_clock_cpu(int cpu); 2310c676329aSPeter Zijlstra 2311e436d800SIngo Molnar 2312c1955a3dSPeter Zijlstra extern void sched_clock_init(void); 2313c1955a3dSPeter Zijlstra 23143e51f33fSPeter Zijlstra #ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK 23153e51f33fSPeter Zijlstra static inline void sched_clock_tick(void) 23163e51f33fSPeter Zijlstra { 23173e51f33fSPeter Zijlstra } 23183e51f33fSPeter Zijlstra 23193e51f33fSPeter Zijlstra static inline void sched_clock_idle_sleep_event(void) 23203e51f33fSPeter Zijlstra { 23213e51f33fSPeter Zijlstra } 23223e51f33fSPeter Zijlstra 23233e51f33fSPeter Zijlstra static inline void sched_clock_idle_wakeup_event(u64 delta_ns) 23243e51f33fSPeter Zijlstra { 23253e51f33fSPeter Zijlstra } 23263e51f33fSPeter Zijlstra #else 2327c676329aSPeter Zijlstra /* 2328c676329aSPeter Zijlstra * Architectures can set this to 1 if they have specified 2329c676329aSPeter Zijlstra * CONFIG_HAVE_UNSTABLE_SCHED_CLOCK in their arch Kconfig, 2330c676329aSPeter Zijlstra * but then during bootup it turns out that sched_clock() 2331c676329aSPeter Zijlstra * is reliable after all: 2332c676329aSPeter Zijlstra */ 233335af99e6SPeter Zijlstra extern int sched_clock_stable(void); 233435af99e6SPeter Zijlstra extern void set_sched_clock_stable(void); 233535af99e6SPeter Zijlstra extern void clear_sched_clock_stable(void); 2336c676329aSPeter Zijlstra 23373e51f33fSPeter Zijlstra extern void sched_clock_tick(void); 23383e51f33fSPeter Zijlstra extern void sched_clock_idle_sleep_event(void); 23393e51f33fSPeter Zijlstra extern void sched_clock_idle_wakeup_event(u64 delta_ns); 23403e51f33fSPeter Zijlstra #endif 23413e51f33fSPeter Zijlstra 2342b52bfee4SVenkatesh Pallipadi #ifdef CONFIG_IRQ_TIME_ACCOUNTING 2343b52bfee4SVenkatesh Pallipadi /* 2344b52bfee4SVenkatesh Pallipadi * An i/f to runtime opt-in for irq time accounting based off of sched_clock. 2345b52bfee4SVenkatesh Pallipadi * The reason for this explicit opt-in is not to have perf penalty with 2346b52bfee4SVenkatesh Pallipadi * slow sched_clocks. 2347b52bfee4SVenkatesh Pallipadi */ 2348b52bfee4SVenkatesh Pallipadi extern void enable_sched_clock_irqtime(void); 2349b52bfee4SVenkatesh Pallipadi extern void disable_sched_clock_irqtime(void); 2350b52bfee4SVenkatesh Pallipadi #else 2351b52bfee4SVenkatesh Pallipadi static inline void enable_sched_clock_irqtime(void) {} 2352b52bfee4SVenkatesh Pallipadi static inline void disable_sched_clock_irqtime(void) {} 2353b52bfee4SVenkatesh Pallipadi #endif 2354b52bfee4SVenkatesh Pallipadi 235536c8b586SIngo Molnar extern unsigned long long 235641b86e9cSIngo Molnar task_sched_runtime(struct task_struct *task); 23571da177e4SLinus Torvalds 23581da177e4SLinus Torvalds /* sched_exec is called by processes performing an exec */ 23591da177e4SLinus Torvalds #ifdef CONFIG_SMP 23601da177e4SLinus Torvalds extern void sched_exec(void); 23611da177e4SLinus Torvalds #else 23621da177e4SLinus Torvalds #define sched_exec() {} 23631da177e4SLinus Torvalds #endif 23641da177e4SLinus Torvalds 23652aa44d05SIngo Molnar extern void sched_clock_idle_sleep_event(void); 23662aa44d05SIngo Molnar extern void sched_clock_idle_wakeup_event(u64 delta_ns); 2367bb29ab26SIngo Molnar 23681da177e4SLinus Torvalds #ifdef CONFIG_HOTPLUG_CPU 23691da177e4SLinus Torvalds extern void idle_task_exit(void); 23701da177e4SLinus Torvalds #else 23711da177e4SLinus Torvalds static inline void idle_task_exit(void) {} 23721da177e4SLinus Torvalds #endif 23731da177e4SLinus Torvalds 23743451d024SFrederic Weisbecker #if defined(CONFIG_NO_HZ_COMMON) && defined(CONFIG_SMP) 23751c20091eSFrederic Weisbecker extern void wake_up_nohz_cpu(int cpu); 237606d8308cSThomas Gleixner #else 23771c20091eSFrederic Weisbecker static inline void wake_up_nohz_cpu(int cpu) { } 237806d8308cSThomas Gleixner #endif 237906d8308cSThomas Gleixner 2380ce831b38SFrederic Weisbecker #ifdef CONFIG_NO_HZ_FULL 2381265f22a9SFrederic Weisbecker extern u64 scheduler_tick_max_deferment(void); 2382bf0f6f24SIngo Molnar #endif 2383bf0f6f24SIngo Molnar 23845091faa4SMike Galbraith #ifdef CONFIG_SCHED_AUTOGROUP 23855091faa4SMike Galbraith extern void sched_autogroup_create_attach(struct task_struct *p); 23865091faa4SMike Galbraith extern void sched_autogroup_detach(struct task_struct *p); 23875091faa4SMike Galbraith extern void sched_autogroup_fork(struct signal_struct *sig); 23885091faa4SMike Galbraith extern void sched_autogroup_exit(struct signal_struct *sig); 23895091faa4SMike Galbraith #ifdef CONFIG_PROC_FS 23905091faa4SMike Galbraith extern void proc_sched_autogroup_show_task(struct task_struct *p, struct seq_file *m); 23912e5b5b3aSHiroshi Shimamoto extern int proc_sched_autogroup_set_nice(struct task_struct *p, int nice); 23925091faa4SMike Galbraith #endif 23935091faa4SMike Galbraith #else 23945091faa4SMike Galbraith static inline void sched_autogroup_create_attach(struct task_struct *p) { } 23955091faa4SMike Galbraith static inline void sched_autogroup_detach(struct task_struct *p) { } 23965091faa4SMike Galbraith static inline void sched_autogroup_fork(struct signal_struct *sig) { } 23975091faa4SMike Galbraith static inline void sched_autogroup_exit(struct signal_struct *sig) { } 23985091faa4SMike Galbraith #endif 23995091faa4SMike Galbraith 2400fa93384fSDan Carpenter extern int yield_to(struct task_struct *p, bool preempt); 240136c8b586SIngo Molnar extern void set_user_nice(struct task_struct *p, long nice); 240236c8b586SIngo Molnar extern int task_prio(const struct task_struct *p); 2403d0ea0268SDongsheng Yang /** 2404d0ea0268SDongsheng Yang * task_nice - return the nice value of a given task. 2405d0ea0268SDongsheng Yang * @p: the task in question. 2406d0ea0268SDongsheng Yang * 2407d0ea0268SDongsheng Yang * Return: The nice value [ -20 ... 0 ... 19 ]. 2408d0ea0268SDongsheng Yang */ 2409d0ea0268SDongsheng Yang static inline int task_nice(const struct task_struct *p) 2410d0ea0268SDongsheng Yang { 2411d0ea0268SDongsheng Yang return PRIO_TO_NICE((p)->static_prio); 2412d0ea0268SDongsheng Yang } 241336c8b586SIngo Molnar extern int can_nice(const struct task_struct *p, const int nice); 241436c8b586SIngo Molnar extern int task_curr(const struct task_struct *p); 24151da177e4SLinus Torvalds extern int idle_cpu(int cpu); 2416fe7de49fSKOSAKI Motohiro extern int sched_setscheduler(struct task_struct *, int, 2417fe7de49fSKOSAKI Motohiro const struct sched_param *); 2418961ccdddSRusty Russell extern int sched_setscheduler_nocheck(struct task_struct *, int, 2419fe7de49fSKOSAKI Motohiro const struct sched_param *); 2420d50dde5aSDario Faggioli extern int sched_setattr(struct task_struct *, 2421d50dde5aSDario Faggioli const struct sched_attr *); 242236c8b586SIngo Molnar extern struct task_struct *idle_task(int cpu); 2423c4f30608SPaul E. McKenney /** 2424c4f30608SPaul E. McKenney * is_idle_task - is the specified task an idle task? 2425fa757281SRandy Dunlap * @p: the task in question. 2426e69f6186SYacine Belkadi * 2427e69f6186SYacine Belkadi * Return: 1 if @p is an idle task. 0 otherwise. 2428c4f30608SPaul E. McKenney */ 24297061ca3bSPaul E. McKenney static inline bool is_idle_task(const struct task_struct *p) 2430c4f30608SPaul E. McKenney { 2431c4f30608SPaul E. McKenney return p->pid == 0; 2432c4f30608SPaul E. McKenney } 243336c8b586SIngo Molnar extern struct task_struct *curr_task(int cpu); 243436c8b586SIngo Molnar extern void set_curr_task(int cpu, struct task_struct *p); 24351da177e4SLinus Torvalds 24361da177e4SLinus Torvalds void yield(void); 24371da177e4SLinus Torvalds 24381da177e4SLinus Torvalds union thread_union { 24391da177e4SLinus Torvalds struct thread_info thread_info; 24401da177e4SLinus Torvalds unsigned long stack[THREAD_SIZE/sizeof(long)]; 24411da177e4SLinus Torvalds }; 24421da177e4SLinus Torvalds 24431da177e4SLinus Torvalds #ifndef __HAVE_ARCH_KSTACK_END 24441da177e4SLinus Torvalds static inline int kstack_end(void *addr) 24451da177e4SLinus Torvalds { 24461da177e4SLinus Torvalds /* Reliable end of stack detection: 24471da177e4SLinus Torvalds * Some APM bios versions misalign the stack 24481da177e4SLinus Torvalds */ 24491da177e4SLinus Torvalds return !(((unsigned long)addr+sizeof(void*)-1) & (THREAD_SIZE-sizeof(void*))); 24501da177e4SLinus Torvalds } 24511da177e4SLinus Torvalds #endif 24521da177e4SLinus Torvalds 24531da177e4SLinus Torvalds extern union thread_union init_thread_union; 24541da177e4SLinus Torvalds extern struct task_struct init_task; 24551da177e4SLinus Torvalds 24561da177e4SLinus Torvalds extern struct mm_struct init_mm; 24571da177e4SLinus Torvalds 2458198fe21bSPavel Emelyanov extern struct pid_namespace init_pid_ns; 2459198fe21bSPavel Emelyanov 2460198fe21bSPavel Emelyanov /* 2461198fe21bSPavel Emelyanov * find a task by one of its numerical ids 2462198fe21bSPavel Emelyanov * 2463198fe21bSPavel Emelyanov * find_task_by_pid_ns(): 2464198fe21bSPavel Emelyanov * finds a task by its pid in the specified namespace 2465228ebcbeSPavel Emelyanov * find_task_by_vpid(): 2466228ebcbeSPavel Emelyanov * finds a task by its virtual pid 2467198fe21bSPavel Emelyanov * 2468e49859e7SPavel Emelyanov * see also find_vpid() etc in include/linux/pid.h 2469198fe21bSPavel Emelyanov */ 2470198fe21bSPavel Emelyanov 2471228ebcbeSPavel Emelyanov extern struct task_struct *find_task_by_vpid(pid_t nr); 2472228ebcbeSPavel Emelyanov extern struct task_struct *find_task_by_pid_ns(pid_t nr, 2473228ebcbeSPavel Emelyanov struct pid_namespace *ns); 2474198fe21bSPavel Emelyanov 24751da177e4SLinus Torvalds /* per-UID process charging. */ 24767b44ab97SEric W. Biederman extern struct user_struct * alloc_uid(kuid_t); 24771da177e4SLinus Torvalds static inline struct user_struct *get_uid(struct user_struct *u) 24781da177e4SLinus Torvalds { 24791da177e4SLinus Torvalds atomic_inc(&u->__count); 24801da177e4SLinus Torvalds return u; 24811da177e4SLinus Torvalds } 24821da177e4SLinus Torvalds extern void free_uid(struct user_struct *); 24831da177e4SLinus Torvalds 24841da177e4SLinus Torvalds #include <asm/current.h> 24851da177e4SLinus Torvalds 2486f0af911aSTorben Hohn extern void xtime_update(unsigned long ticks); 24871da177e4SLinus Torvalds 2488b3c97528SHarvey Harrison extern int wake_up_state(struct task_struct *tsk, unsigned int state); 2489b3c97528SHarvey Harrison extern int wake_up_process(struct task_struct *tsk); 24903e51e3edSSamir Bellabes extern void wake_up_new_task(struct task_struct *tsk); 24911da177e4SLinus Torvalds #ifdef CONFIG_SMP 24921da177e4SLinus Torvalds extern void kick_process(struct task_struct *tsk); 24931da177e4SLinus Torvalds #else 24941da177e4SLinus Torvalds static inline void kick_process(struct task_struct *tsk) { } 24951da177e4SLinus Torvalds #endif 2496aab03e05SDario Faggioli extern int sched_fork(unsigned long clone_flags, struct task_struct *p); 2497ad46c2c4SIngo Molnar extern void sched_dead(struct task_struct *p); 24981da177e4SLinus Torvalds 24991da177e4SLinus Torvalds extern void proc_caches_init(void); 25001da177e4SLinus Torvalds extern void flush_signals(struct task_struct *); 250110ab825bSOleg Nesterov extern void ignore_signals(struct task_struct *); 25021da177e4SLinus Torvalds extern void flush_signal_handlers(struct task_struct *, int force_default); 25031da177e4SLinus Torvalds extern int dequeue_signal(struct task_struct *tsk, sigset_t *mask, siginfo_t *info); 25041da177e4SLinus Torvalds 2505be0e6f29SOleg Nesterov static inline int kernel_dequeue_signal(siginfo_t *info) 25061da177e4SLinus Torvalds { 2507be0e6f29SOleg Nesterov struct task_struct *tsk = current; 2508be0e6f29SOleg Nesterov siginfo_t __info; 25091da177e4SLinus Torvalds int ret; 25101da177e4SLinus Torvalds 2511be0e6f29SOleg Nesterov spin_lock_irq(&tsk->sighand->siglock); 2512be0e6f29SOleg Nesterov ret = dequeue_signal(tsk, &tsk->blocked, info ?: &__info); 2513be0e6f29SOleg Nesterov spin_unlock_irq(&tsk->sighand->siglock); 25141da177e4SLinus Torvalds 25151da177e4SLinus Torvalds return ret; 25161da177e4SLinus Torvalds } 25171da177e4SLinus Torvalds 25189a13049eSOleg Nesterov static inline void kernel_signal_stop(void) 25199a13049eSOleg Nesterov { 25209a13049eSOleg Nesterov spin_lock_irq(¤t->sighand->siglock); 25219a13049eSOleg Nesterov if (current->jobctl & JOBCTL_STOP_DEQUEUED) 25229a13049eSOleg Nesterov __set_current_state(TASK_STOPPED); 25239a13049eSOleg Nesterov spin_unlock_irq(¤t->sighand->siglock); 25249a13049eSOleg Nesterov 25259a13049eSOleg Nesterov schedule(); 25269a13049eSOleg Nesterov } 25279a13049eSOleg Nesterov 25281da177e4SLinus Torvalds extern void release_task(struct task_struct * p); 25291da177e4SLinus Torvalds extern int send_sig_info(int, struct siginfo *, struct task_struct *); 25301da177e4SLinus Torvalds extern int force_sigsegv(int, struct task_struct *); 25311da177e4SLinus Torvalds extern int force_sig_info(int, struct siginfo *, struct task_struct *); 2532c4b92fc1SEric W. Biederman extern int __kill_pgrp_info(int sig, struct siginfo *info, struct pid *pgrp); 2533c4b92fc1SEric W. Biederman extern int kill_pid_info(int sig, struct siginfo *info, struct pid *pid); 2534d178bc3aSSerge Hallyn extern int kill_pid_info_as_cred(int, struct siginfo *, struct pid *, 2535d178bc3aSSerge Hallyn const struct cred *, u32); 2536c4b92fc1SEric W. Biederman extern int kill_pgrp(struct pid *pid, int sig, int priv); 2537c4b92fc1SEric W. Biederman extern int kill_pid(struct pid *pid, int sig, int priv); 2538c3de4b38SMatthew Wilcox extern int kill_proc_info(int, struct siginfo *, pid_t); 253986773473SOleg Nesterov extern __must_check bool do_notify_parent(struct task_struct *, int); 2540a7f0765eSOleg Nesterov extern void __wake_up_parent(struct task_struct *p, struct task_struct *parent); 25411da177e4SLinus Torvalds extern void force_sig(int, struct task_struct *); 25421da177e4SLinus Torvalds extern int send_sig(int, struct task_struct *, int); 254309faef11SOleg Nesterov extern int zap_other_threads(struct task_struct *p); 25441da177e4SLinus Torvalds extern struct sigqueue *sigqueue_alloc(void); 25451da177e4SLinus Torvalds extern void sigqueue_free(struct sigqueue *); 2546ac5c2153SOleg Nesterov extern int send_sigqueue(struct sigqueue *, struct task_struct *, int group); 25479ac95f2fSOleg Nesterov extern int do_sigaction(int, struct k_sigaction *, struct k_sigaction *); 25481da177e4SLinus Torvalds 254951a7b448SAl Viro static inline void restore_saved_sigmask(void) 255051a7b448SAl Viro { 255151a7b448SAl Viro if (test_and_clear_restore_sigmask()) 255277097ae5SAl Viro __set_current_blocked(¤t->saved_sigmask); 255351a7b448SAl Viro } 255451a7b448SAl Viro 2555b7f9a11aSAl Viro static inline sigset_t *sigmask_to_save(void) 2556b7f9a11aSAl Viro { 2557b7f9a11aSAl Viro sigset_t *res = ¤t->blocked; 2558b7f9a11aSAl Viro if (unlikely(test_restore_sigmask())) 2559b7f9a11aSAl Viro res = ¤t->saved_sigmask; 2560b7f9a11aSAl Viro return res; 2561b7f9a11aSAl Viro } 2562b7f9a11aSAl Viro 25639ec52099SCedric Le Goater static inline int kill_cad_pid(int sig, int priv) 25649ec52099SCedric Le Goater { 25659ec52099SCedric Le Goater return kill_pid(cad_pid, sig, priv); 25669ec52099SCedric Le Goater } 25679ec52099SCedric Le Goater 25681da177e4SLinus Torvalds /* These can be the second arg to send_sig_info/send_group_sig_info. */ 25691da177e4SLinus Torvalds #define SEND_SIG_NOINFO ((struct siginfo *) 0) 25701da177e4SLinus Torvalds #define SEND_SIG_PRIV ((struct siginfo *) 1) 25711da177e4SLinus Torvalds #define SEND_SIG_FORCED ((struct siginfo *) 2) 25721da177e4SLinus Torvalds 25732a855dd0SSebastian Andrzej Siewior /* 25742a855dd0SSebastian Andrzej Siewior * True if we are on the alternate signal stack. 25752a855dd0SSebastian Andrzej Siewior */ 25761da177e4SLinus Torvalds static inline int on_sig_stack(unsigned long sp) 25771da177e4SLinus Torvalds { 25782a855dd0SSebastian Andrzej Siewior #ifdef CONFIG_STACK_GROWSUP 25792a855dd0SSebastian Andrzej Siewior return sp >= current->sas_ss_sp && 25802a855dd0SSebastian Andrzej Siewior sp - current->sas_ss_sp < current->sas_ss_size; 25812a855dd0SSebastian Andrzej Siewior #else 25822a855dd0SSebastian Andrzej Siewior return sp > current->sas_ss_sp && 25832a855dd0SSebastian Andrzej Siewior sp - current->sas_ss_sp <= current->sas_ss_size; 25842a855dd0SSebastian Andrzej Siewior #endif 25851da177e4SLinus Torvalds } 25861da177e4SLinus Torvalds 25871da177e4SLinus Torvalds static inline int sas_ss_flags(unsigned long sp) 25881da177e4SLinus Torvalds { 258972f15c03SRichard Weinberger if (!current->sas_ss_size) 259072f15c03SRichard Weinberger return SS_DISABLE; 259172f15c03SRichard Weinberger 259272f15c03SRichard Weinberger return on_sig_stack(sp) ? SS_ONSTACK : 0; 25931da177e4SLinus Torvalds } 25941da177e4SLinus Torvalds 25955a1b98d3SAl Viro static inline unsigned long sigsp(unsigned long sp, struct ksignal *ksig) 25965a1b98d3SAl Viro { 25975a1b98d3SAl Viro if (unlikely((ksig->ka.sa.sa_flags & SA_ONSTACK)) && ! sas_ss_flags(sp)) 25985a1b98d3SAl Viro #ifdef CONFIG_STACK_GROWSUP 25995a1b98d3SAl Viro return current->sas_ss_sp; 26005a1b98d3SAl Viro #else 26015a1b98d3SAl Viro return current->sas_ss_sp + current->sas_ss_size; 26025a1b98d3SAl Viro #endif 26035a1b98d3SAl Viro return sp; 26045a1b98d3SAl Viro } 26055a1b98d3SAl Viro 26061da177e4SLinus Torvalds /* 26071da177e4SLinus Torvalds * Routines for handling mm_structs 26081da177e4SLinus Torvalds */ 26091da177e4SLinus Torvalds extern struct mm_struct * mm_alloc(void); 26101da177e4SLinus Torvalds 26111da177e4SLinus Torvalds /* mmdrop drops the mm and the page tables */ 2612b3c97528SHarvey Harrison extern void __mmdrop(struct mm_struct *); 26131da177e4SLinus Torvalds static inline void mmdrop(struct mm_struct * mm) 26141da177e4SLinus Torvalds { 26156fb43d7bSIngo Molnar if (unlikely(atomic_dec_and_test(&mm->mm_count))) 26161da177e4SLinus Torvalds __mmdrop(mm); 26171da177e4SLinus Torvalds } 26181da177e4SLinus Torvalds 26191da177e4SLinus Torvalds /* mmput gets rid of the mappings and all user-space */ 26201da177e4SLinus Torvalds extern void mmput(struct mm_struct *); 26211da177e4SLinus Torvalds /* Grab a reference to a task's mm, if it is not already going away */ 26221da177e4SLinus Torvalds extern struct mm_struct *get_task_mm(struct task_struct *task); 26238cdb878dSChristopher Yeoh /* 26248cdb878dSChristopher Yeoh * Grab a reference to a task's mm, if it is not already going away 26258cdb878dSChristopher Yeoh * and ptrace_may_access with the mode parameter passed to it 26268cdb878dSChristopher Yeoh * succeeds. 26278cdb878dSChristopher Yeoh */ 26288cdb878dSChristopher Yeoh extern struct mm_struct *mm_access(struct task_struct *task, unsigned int mode); 26291da177e4SLinus Torvalds /* Remove the current tasks stale references to the old mm_struct */ 26301da177e4SLinus Torvalds extern void mm_release(struct task_struct *, struct mm_struct *); 26311da177e4SLinus Torvalds 26323033f14aSJosh Triplett #ifdef CONFIG_HAVE_COPY_THREAD_TLS 26333033f14aSJosh Triplett extern int copy_thread_tls(unsigned long, unsigned long, unsigned long, 26343033f14aSJosh Triplett struct task_struct *, unsigned long); 26353033f14aSJosh Triplett #else 26366f2c55b8SAlexey Dobriyan extern int copy_thread(unsigned long, unsigned long, unsigned long, 2637afa86fc4SAl Viro struct task_struct *); 26383033f14aSJosh Triplett 26393033f14aSJosh Triplett /* Architectures that haven't opted into copy_thread_tls get the tls argument 26403033f14aSJosh Triplett * via pt_regs, so ignore the tls argument passed via C. */ 26413033f14aSJosh Triplett static inline int copy_thread_tls( 26423033f14aSJosh Triplett unsigned long clone_flags, unsigned long sp, unsigned long arg, 26433033f14aSJosh Triplett struct task_struct *p, unsigned long tls) 26443033f14aSJosh Triplett { 26453033f14aSJosh Triplett return copy_thread(clone_flags, sp, arg, p); 26463033f14aSJosh Triplett } 26473033f14aSJosh Triplett #endif 26481da177e4SLinus Torvalds extern void flush_thread(void); 26491da177e4SLinus Torvalds extern void exit_thread(void); 26501da177e4SLinus Torvalds 26511da177e4SLinus Torvalds extern void exit_files(struct task_struct *); 2652a7e5328aSOleg Nesterov extern void __cleanup_sighand(struct sighand_struct *); 2653cbaffba1SOleg Nesterov 26541da177e4SLinus Torvalds extern void exit_itimers(struct signal_struct *); 2655cbaffba1SOleg Nesterov extern void flush_itimer_signals(void); 26561da177e4SLinus Torvalds 26579402c95fSJoe Perches extern void do_group_exit(int); 26581da177e4SLinus Torvalds 2659c4ad8f98SLinus Torvalds extern int do_execve(struct filename *, 2660d7627467SDavid Howells const char __user * const __user *, 2661da3d4c5fSAl Viro const char __user * const __user *); 266251f39a1fSDavid Drysdale extern int do_execveat(int, struct filename *, 266351f39a1fSDavid Drysdale const char __user * const __user *, 266451f39a1fSDavid Drysdale const char __user * const __user *, 266551f39a1fSDavid Drysdale int); 26663033f14aSJosh Triplett extern long _do_fork(unsigned long, unsigned long, unsigned long, int __user *, int __user *, unsigned long); 2667e80d6661SAl Viro extern long do_fork(unsigned long, unsigned long, unsigned long, int __user *, int __user *); 266836c8b586SIngo Molnar struct task_struct *fork_idle(int); 26692aa3a7f8SAl Viro extern pid_t kernel_thread(int (*fn)(void *), void *arg, unsigned long flags); 26701da177e4SLinus Torvalds 267182b89778SAdrian Hunter extern void __set_task_comm(struct task_struct *tsk, const char *from, bool exec); 267282b89778SAdrian Hunter static inline void set_task_comm(struct task_struct *tsk, const char *from) 267382b89778SAdrian Hunter { 267482b89778SAdrian Hunter __set_task_comm(tsk, from, false); 267582b89778SAdrian Hunter } 267659714d65SAndrew Morton extern char *get_task_comm(char *to, struct task_struct *tsk); 26771da177e4SLinus Torvalds 26781da177e4SLinus Torvalds #ifdef CONFIG_SMP 2679317f3941SPeter Zijlstra void scheduler_ipi(void); 268085ba2d86SRoland McGrath extern unsigned long wait_task_inactive(struct task_struct *, long match_state); 26811da177e4SLinus Torvalds #else 2682184748ccSPeter Zijlstra static inline void scheduler_ipi(void) { } 268385ba2d86SRoland McGrath static inline unsigned long wait_task_inactive(struct task_struct *p, 268485ba2d86SRoland McGrath long match_state) 268585ba2d86SRoland McGrath { 268685ba2d86SRoland McGrath return 1; 268785ba2d86SRoland McGrath } 26881da177e4SLinus Torvalds #endif 26891da177e4SLinus Torvalds 2690fafe870fSFrederic Weisbecker #define tasklist_empty() \ 2691fafe870fSFrederic Weisbecker list_empty(&init_task.tasks) 2692fafe870fSFrederic Weisbecker 269305725f7eSJiri Pirko #define next_task(p) \ 269405725f7eSJiri Pirko list_entry_rcu((p)->tasks.next, struct task_struct, tasks) 26951da177e4SLinus Torvalds 26961da177e4SLinus Torvalds #define for_each_process(p) \ 26971da177e4SLinus Torvalds for (p = &init_task ; (p = next_task(p)) != &init_task ; ) 26981da177e4SLinus Torvalds 26995bb459bbSOleg Nesterov extern bool current_is_single_threaded(void); 2700d84f4f99SDavid Howells 27011da177e4SLinus Torvalds /* 27021da177e4SLinus Torvalds * Careful: do_each_thread/while_each_thread is a double loop so 27031da177e4SLinus Torvalds * 'break' will not work as expected - use goto instead. 27041da177e4SLinus Torvalds */ 27051da177e4SLinus Torvalds #define do_each_thread(g, t) \ 27061da177e4SLinus Torvalds for (g = t = &init_task ; (g = t = next_task(g)) != &init_task ; ) do 27071da177e4SLinus Torvalds 27081da177e4SLinus Torvalds #define while_each_thread(g, t) \ 27091da177e4SLinus Torvalds while ((t = next_thread(t)) != g) 27101da177e4SLinus Torvalds 27110c740d0aSOleg Nesterov #define __for_each_thread(signal, t) \ 27120c740d0aSOleg Nesterov list_for_each_entry_rcu(t, &(signal)->thread_head, thread_node) 27130c740d0aSOleg Nesterov 27140c740d0aSOleg Nesterov #define for_each_thread(p, t) \ 27150c740d0aSOleg Nesterov __for_each_thread((p)->signal, t) 27160c740d0aSOleg Nesterov 27170c740d0aSOleg Nesterov /* Careful: this is a double loop, 'break' won't work as expected. */ 27180c740d0aSOleg Nesterov #define for_each_process_thread(p, t) \ 27190c740d0aSOleg Nesterov for_each_process(p) for_each_thread(p, t) 27200c740d0aSOleg Nesterov 27217e49827cSOleg Nesterov static inline int get_nr_threads(struct task_struct *tsk) 27227e49827cSOleg Nesterov { 2723b3ac022cSOleg Nesterov return tsk->signal->nr_threads; 27247e49827cSOleg Nesterov } 27257e49827cSOleg Nesterov 2726087806b1SOleg Nesterov static inline bool thread_group_leader(struct task_struct *p) 2727087806b1SOleg Nesterov { 2728087806b1SOleg Nesterov return p->exit_signal >= 0; 2729087806b1SOleg Nesterov } 27301da177e4SLinus Torvalds 27310804ef4bSEric W. Biederman /* Do to the insanities of de_thread it is possible for a process 27320804ef4bSEric W. Biederman * to have the pid of the thread group leader without actually being 27330804ef4bSEric W. Biederman * the thread group leader. For iteration through the pids in proc 27340804ef4bSEric W. Biederman * all we care about is that we have a task with the appropriate 27350804ef4bSEric W. Biederman * pid, we don't actually care if we have the right task. 27360804ef4bSEric W. Biederman */ 2737e1403b8eSOleg Nesterov static inline bool has_group_leader_pid(struct task_struct *p) 27380804ef4bSEric W. Biederman { 2739e1403b8eSOleg Nesterov return task_pid(p) == p->signal->leader_pid; 27400804ef4bSEric W. Biederman } 27410804ef4bSEric W. Biederman 2742bac0abd6SPavel Emelyanov static inline 2743e1403b8eSOleg Nesterov bool same_thread_group(struct task_struct *p1, struct task_struct *p2) 2744bac0abd6SPavel Emelyanov { 2745e1403b8eSOleg Nesterov return p1->signal == p2->signal; 2746bac0abd6SPavel Emelyanov } 2747bac0abd6SPavel Emelyanov 274836c8b586SIngo Molnar static inline struct task_struct *next_thread(const struct task_struct *p) 274947e65328SOleg Nesterov { 275005725f7eSJiri Pirko return list_entry_rcu(p->thread_group.next, 275136c8b586SIngo Molnar struct task_struct, thread_group); 275247e65328SOleg Nesterov } 275347e65328SOleg Nesterov 2754e868171aSAlexey Dobriyan static inline int thread_group_empty(struct task_struct *p) 27551da177e4SLinus Torvalds { 275647e65328SOleg Nesterov return list_empty(&p->thread_group); 27571da177e4SLinus Torvalds } 27581da177e4SLinus Torvalds 27591da177e4SLinus Torvalds #define delay_group_leader(p) \ 27601da177e4SLinus Torvalds (thread_group_leader(p) && !thread_group_empty(p)) 27611da177e4SLinus Torvalds 27621da177e4SLinus Torvalds /* 2763260ea101SEric W. Biederman * Protects ->fs, ->files, ->mm, ->group_info, ->comm, keyring 276422e2c507SJens Axboe * subscriptions and synchronises with wait4(). Also used in procfs. Also 2765ddbcc7e8SPaul Menage * pins the final release of task.io_context. Also protects ->cpuset and 2766d68b46feSOleg Nesterov * ->cgroup.subsys[]. And ->vfork_done. 27671da177e4SLinus Torvalds * 27681da177e4SLinus Torvalds * Nests both inside and outside of read_lock(&tasklist_lock). 27691da177e4SLinus Torvalds * It must not be nested with write_lock_irq(&tasklist_lock), 27701da177e4SLinus Torvalds * neither inside nor outside. 27711da177e4SLinus Torvalds */ 27721da177e4SLinus Torvalds static inline void task_lock(struct task_struct *p) 27731da177e4SLinus Torvalds { 27741da177e4SLinus Torvalds spin_lock(&p->alloc_lock); 27751da177e4SLinus Torvalds } 27761da177e4SLinus Torvalds 27771da177e4SLinus Torvalds static inline void task_unlock(struct task_struct *p) 27781da177e4SLinus Torvalds { 27791da177e4SLinus Torvalds spin_unlock(&p->alloc_lock); 27801da177e4SLinus Torvalds } 27811da177e4SLinus Torvalds 2782b8ed374eSNamhyung Kim extern struct sighand_struct *__lock_task_sighand(struct task_struct *tsk, 2783f63ee72eSOleg Nesterov unsigned long *flags); 2784f63ee72eSOleg Nesterov 27859388dc30SAnton Vorontsov static inline struct sighand_struct *lock_task_sighand(struct task_struct *tsk, 27869388dc30SAnton Vorontsov unsigned long *flags) 27879388dc30SAnton Vorontsov { 27889388dc30SAnton Vorontsov struct sighand_struct *ret; 27899388dc30SAnton Vorontsov 27909388dc30SAnton Vorontsov ret = __lock_task_sighand(tsk, flags); 27919388dc30SAnton Vorontsov (void)__cond_lock(&tsk->sighand->siglock, ret); 27929388dc30SAnton Vorontsov return ret; 27939388dc30SAnton Vorontsov } 2794b8ed374eSNamhyung Kim 2795f63ee72eSOleg Nesterov static inline void unlock_task_sighand(struct task_struct *tsk, 2796f63ee72eSOleg Nesterov unsigned long *flags) 2797f63ee72eSOleg Nesterov { 2798f63ee72eSOleg Nesterov spin_unlock_irqrestore(&tsk->sighand->siglock, *flags); 2799f63ee72eSOleg Nesterov } 2800f63ee72eSOleg Nesterov 28017d7efec3STejun Heo /** 28027d7efec3STejun Heo * threadgroup_change_begin - mark the beginning of changes to a threadgroup 28037d7efec3STejun Heo * @tsk: task causing the changes 28047d7efec3STejun Heo * 28057d7efec3STejun Heo * All operations which modify a threadgroup - a new thread joining the 28067d7efec3STejun Heo * group, death of a member thread (the assertion of PF_EXITING) and 28077d7efec3STejun Heo * exec(2) dethreading the process and replacing the leader - are wrapped 28087d7efec3STejun Heo * by threadgroup_change_{begin|end}(). This is to provide a place which 28097d7efec3STejun Heo * subsystems needing threadgroup stability can hook into for 28107d7efec3STejun Heo * synchronization. 28117d7efec3STejun Heo */ 2812257058aeSTejun Heo static inline void threadgroup_change_begin(struct task_struct *tsk) 28134714d1d3SBen Blum { 28147d7efec3STejun Heo might_sleep(); 28157d7efec3STejun Heo cgroup_threadgroup_change_begin(tsk); 28164714d1d3SBen Blum } 28177d7efec3STejun Heo 28187d7efec3STejun Heo /** 28197d7efec3STejun Heo * threadgroup_change_end - mark the end of changes to a threadgroup 28207d7efec3STejun Heo * @tsk: task causing the changes 28217d7efec3STejun Heo * 28227d7efec3STejun Heo * See threadgroup_change_begin(). 28237d7efec3STejun Heo */ 2824257058aeSTejun Heo static inline void threadgroup_change_end(struct task_struct *tsk) 28254714d1d3SBen Blum { 28267d7efec3STejun Heo cgroup_threadgroup_change_end(tsk); 28274714d1d3SBen Blum } 282877e4ef99STejun Heo 2829f037360fSAl Viro #ifndef __HAVE_THREAD_FUNCTIONS 2830f037360fSAl Viro 2831f7e4217bSRoman Zippel #define task_thread_info(task) ((struct thread_info *)(task)->stack) 2832f7e4217bSRoman Zippel #define task_stack_page(task) ((task)->stack) 2833a1261f54SAl Viro 283410ebffdeSAl Viro static inline void setup_thread_stack(struct task_struct *p, struct task_struct *org) 283510ebffdeSAl Viro { 283610ebffdeSAl Viro *task_thread_info(p) = *task_thread_info(org); 283710ebffdeSAl Viro task_thread_info(p)->task = p; 283810ebffdeSAl Viro } 283910ebffdeSAl Viro 28406a40281aSChuck Ebbert /* 28416a40281aSChuck Ebbert * Return the address of the last usable long on the stack. 28426a40281aSChuck Ebbert * 28436a40281aSChuck Ebbert * When the stack grows down, this is just above the thread 28446a40281aSChuck Ebbert * info struct. Going any lower will corrupt the threadinfo. 28456a40281aSChuck Ebbert * 28466a40281aSChuck Ebbert * When the stack grows up, this is the highest address. 28476a40281aSChuck Ebbert * Beyond that position, we corrupt data on the next page. 28486a40281aSChuck Ebbert */ 284910ebffdeSAl Viro static inline unsigned long *end_of_stack(struct task_struct *p) 285010ebffdeSAl Viro { 28516a40281aSChuck Ebbert #ifdef CONFIG_STACK_GROWSUP 28526a40281aSChuck Ebbert return (unsigned long *)((unsigned long)task_thread_info(p) + THREAD_SIZE) - 1; 28536a40281aSChuck Ebbert #else 2854f7e4217bSRoman Zippel return (unsigned long *)(task_thread_info(p) + 1); 28556a40281aSChuck Ebbert #endif 285610ebffdeSAl Viro } 285710ebffdeSAl Viro 2858f037360fSAl Viro #endif 2859a70857e4SAaron Tomlin #define task_stack_end_corrupted(task) \ 2860a70857e4SAaron Tomlin (*(end_of_stack(task)) != STACK_END_MAGIC) 2861f037360fSAl Viro 28628b05c7e6SFUJITA Tomonori static inline int object_is_on_stack(void *obj) 28638b05c7e6SFUJITA Tomonori { 28648b05c7e6SFUJITA Tomonori void *stack = task_stack_page(current); 28658b05c7e6SFUJITA Tomonori 28668b05c7e6SFUJITA Tomonori return (obj >= stack) && (obj < (stack + THREAD_SIZE)); 28678b05c7e6SFUJITA Tomonori } 28688b05c7e6SFUJITA Tomonori 28698c9843e5SBenjamin Herrenschmidt extern void thread_info_cache_init(void); 28708c9843e5SBenjamin Herrenschmidt 28717c9f8861SEric Sandeen #ifdef CONFIG_DEBUG_STACK_USAGE 28727c9f8861SEric Sandeen static inline unsigned long stack_not_used(struct task_struct *p) 28737c9f8861SEric Sandeen { 28747c9f8861SEric Sandeen unsigned long *n = end_of_stack(p); 28757c9f8861SEric Sandeen 28767c9f8861SEric Sandeen do { /* Skip over canary */ 28776c31da34SHelge Deller # ifdef CONFIG_STACK_GROWSUP 28786c31da34SHelge Deller n--; 28796c31da34SHelge Deller # else 28807c9f8861SEric Sandeen n++; 28816c31da34SHelge Deller # endif 28827c9f8861SEric Sandeen } while (!*n); 28837c9f8861SEric Sandeen 28846c31da34SHelge Deller # ifdef CONFIG_STACK_GROWSUP 28856c31da34SHelge Deller return (unsigned long)end_of_stack(p) - (unsigned long)n; 28866c31da34SHelge Deller # else 28877c9f8861SEric Sandeen return (unsigned long)n - (unsigned long)end_of_stack(p); 28886c31da34SHelge Deller # endif 28897c9f8861SEric Sandeen } 28907c9f8861SEric Sandeen #endif 2891d4311ff1SAaron Tomlin extern void set_task_stack_end_magic(struct task_struct *tsk); 28927c9f8861SEric Sandeen 28931da177e4SLinus Torvalds /* set thread flags in other task's structures 28941da177e4SLinus Torvalds * - see asm/thread_info.h for TIF_xxxx flags available 28951da177e4SLinus Torvalds */ 28961da177e4SLinus Torvalds static inline void set_tsk_thread_flag(struct task_struct *tsk, int flag) 28971da177e4SLinus Torvalds { 2898a1261f54SAl Viro set_ti_thread_flag(task_thread_info(tsk), flag); 28991da177e4SLinus Torvalds } 29001da177e4SLinus Torvalds 29011da177e4SLinus Torvalds static inline void clear_tsk_thread_flag(struct task_struct *tsk, int flag) 29021da177e4SLinus Torvalds { 2903a1261f54SAl Viro clear_ti_thread_flag(task_thread_info(tsk), flag); 29041da177e4SLinus Torvalds } 29051da177e4SLinus Torvalds 29061da177e4SLinus Torvalds static inline int test_and_set_tsk_thread_flag(struct task_struct *tsk, int flag) 29071da177e4SLinus Torvalds { 2908a1261f54SAl Viro return test_and_set_ti_thread_flag(task_thread_info(tsk), flag); 29091da177e4SLinus Torvalds } 29101da177e4SLinus Torvalds 29111da177e4SLinus Torvalds static inline int test_and_clear_tsk_thread_flag(struct task_struct *tsk, int flag) 29121da177e4SLinus Torvalds { 2913a1261f54SAl Viro return test_and_clear_ti_thread_flag(task_thread_info(tsk), flag); 29141da177e4SLinus Torvalds } 29151da177e4SLinus Torvalds 29161da177e4SLinus Torvalds static inline int test_tsk_thread_flag(struct task_struct *tsk, int flag) 29171da177e4SLinus Torvalds { 2918a1261f54SAl Viro return test_ti_thread_flag(task_thread_info(tsk), flag); 29191da177e4SLinus Torvalds } 29201da177e4SLinus Torvalds 29211da177e4SLinus Torvalds static inline void set_tsk_need_resched(struct task_struct *tsk) 29221da177e4SLinus Torvalds { 29231da177e4SLinus Torvalds set_tsk_thread_flag(tsk,TIF_NEED_RESCHED); 29241da177e4SLinus Torvalds } 29251da177e4SLinus Torvalds 29261da177e4SLinus Torvalds static inline void clear_tsk_need_resched(struct task_struct *tsk) 29271da177e4SLinus Torvalds { 29281da177e4SLinus Torvalds clear_tsk_thread_flag(tsk,TIF_NEED_RESCHED); 29291da177e4SLinus Torvalds } 29301da177e4SLinus Torvalds 29318ae121acSGregory Haskins static inline int test_tsk_need_resched(struct task_struct *tsk) 29328ae121acSGregory Haskins { 29338ae121acSGregory Haskins return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED)); 29348ae121acSGregory Haskins } 29358ae121acSGregory Haskins 2936690cc3ffSEric W. Biederman static inline int restart_syscall(void) 2937690cc3ffSEric W. Biederman { 2938690cc3ffSEric W. Biederman set_tsk_thread_flag(current, TIF_SIGPENDING); 2939690cc3ffSEric W. Biederman return -ERESTARTNOINTR; 2940690cc3ffSEric W. Biederman } 2941690cc3ffSEric W. Biederman 29421da177e4SLinus Torvalds static inline int signal_pending(struct task_struct *p) 29431da177e4SLinus Torvalds { 29441da177e4SLinus Torvalds return unlikely(test_tsk_thread_flag(p,TIF_SIGPENDING)); 29451da177e4SLinus Torvalds } 29461da177e4SLinus Torvalds 2947d9588725SRoland McGrath static inline int __fatal_signal_pending(struct task_struct *p) 2948d9588725SRoland McGrath { 2949d9588725SRoland McGrath return unlikely(sigismember(&p->pending.signal, SIGKILL)); 2950d9588725SRoland McGrath } 2951f776d12dSMatthew Wilcox 2952f776d12dSMatthew Wilcox static inline int fatal_signal_pending(struct task_struct *p) 2953f776d12dSMatthew Wilcox { 2954f776d12dSMatthew Wilcox return signal_pending(p) && __fatal_signal_pending(p); 2955f776d12dSMatthew Wilcox } 2956f776d12dSMatthew Wilcox 295716882c1eSOleg Nesterov static inline int signal_pending_state(long state, struct task_struct *p) 295816882c1eSOleg Nesterov { 295916882c1eSOleg Nesterov if (!(state & (TASK_INTERRUPTIBLE | TASK_WAKEKILL))) 296016882c1eSOleg Nesterov return 0; 296116882c1eSOleg Nesterov if (!signal_pending(p)) 296216882c1eSOleg Nesterov return 0; 296316882c1eSOleg Nesterov 296416882c1eSOleg Nesterov return (state & TASK_INTERRUPTIBLE) || __fatal_signal_pending(p); 296516882c1eSOleg Nesterov } 296616882c1eSOleg Nesterov 29671da177e4SLinus Torvalds /* 29681da177e4SLinus Torvalds * cond_resched() and cond_resched_lock(): latency reduction via 29691da177e4SLinus Torvalds * explicit rescheduling in places that are safe. The return 29701da177e4SLinus Torvalds * value indicates whether a reschedule was done in fact. 29711da177e4SLinus Torvalds * cond_resched_lock() will drop the spinlock before scheduling, 29721da177e4SLinus Torvalds * cond_resched_softirq() will enable bhs before scheduling. 29731da177e4SLinus Torvalds */ 2974c3921ab7SLinus Torvalds extern int _cond_resched(void); 29756f80bd98SFrederic Weisbecker 2976613afbf8SFrederic Weisbecker #define cond_resched() ({ \ 29773427445aSPeter Zijlstra ___might_sleep(__FILE__, __LINE__, 0); \ 2978613afbf8SFrederic Weisbecker _cond_resched(); \ 2979613afbf8SFrederic Weisbecker }) 29806f80bd98SFrederic Weisbecker 2981613afbf8SFrederic Weisbecker extern int __cond_resched_lock(spinlock_t *lock); 2982613afbf8SFrederic Weisbecker 2983613afbf8SFrederic Weisbecker #define cond_resched_lock(lock) ({ \ 29843427445aSPeter Zijlstra ___might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET);\ 2985613afbf8SFrederic Weisbecker __cond_resched_lock(lock); \ 2986613afbf8SFrederic Weisbecker }) 2987613afbf8SFrederic Weisbecker 2988613afbf8SFrederic Weisbecker extern int __cond_resched_softirq(void); 2989613afbf8SFrederic Weisbecker 2990613afbf8SFrederic Weisbecker #define cond_resched_softirq() ({ \ 29913427445aSPeter Zijlstra ___might_sleep(__FILE__, __LINE__, SOFTIRQ_DISABLE_OFFSET); \ 2992613afbf8SFrederic Weisbecker __cond_resched_softirq(); \ 2993613afbf8SFrederic Weisbecker }) 29941da177e4SLinus Torvalds 2995f6f3c437SSimon Horman static inline void cond_resched_rcu(void) 2996f6f3c437SSimon Horman { 2997f6f3c437SSimon Horman #if defined(CONFIG_DEBUG_ATOMIC_SLEEP) || !defined(CONFIG_PREEMPT_RCU) 2998f6f3c437SSimon Horman rcu_read_unlock(); 2999f6f3c437SSimon Horman cond_resched(); 3000f6f3c437SSimon Horman rcu_read_lock(); 3001f6f3c437SSimon Horman #endif 3002f6f3c437SSimon Horman } 3003f6f3c437SSimon Horman 30041da177e4SLinus Torvalds /* 30051da177e4SLinus Torvalds * Does a critical section need to be broken due to another 300695c354feSNick Piggin * task waiting?: (technically does not depend on CONFIG_PREEMPT, 300795c354feSNick Piggin * but a general need for low latency) 30081da177e4SLinus Torvalds */ 300995c354feSNick Piggin static inline int spin_needbreak(spinlock_t *lock) 30101da177e4SLinus Torvalds { 301195c354feSNick Piggin #ifdef CONFIG_PREEMPT 301295c354feSNick Piggin return spin_is_contended(lock); 301395c354feSNick Piggin #else 30141da177e4SLinus Torvalds return 0; 301595c354feSNick Piggin #endif 30161da177e4SLinus Torvalds } 30171da177e4SLinus Torvalds 30187bb44adeSRoland McGrath /* 3019ee761f62SThomas Gleixner * Idle thread specific functions to determine the need_resched 302069dd0f84SPeter Zijlstra * polling state. 3021ee761f62SThomas Gleixner */ 302269dd0f84SPeter Zijlstra #ifdef TIF_POLLING_NRFLAG 3023ee761f62SThomas Gleixner static inline int tsk_is_polling(struct task_struct *p) 3024ee761f62SThomas Gleixner { 3025ee761f62SThomas Gleixner return test_tsk_thread_flag(p, TIF_POLLING_NRFLAG); 3026ee761f62SThomas Gleixner } 3027ea811747SPeter Zijlstra 3028ea811747SPeter Zijlstra static inline void __current_set_polling(void) 30293a98f871SThomas Gleixner { 30303a98f871SThomas Gleixner set_thread_flag(TIF_POLLING_NRFLAG); 30313a98f871SThomas Gleixner } 30323a98f871SThomas Gleixner 3033ea811747SPeter Zijlstra static inline bool __must_check current_set_polling_and_test(void) 3034ea811747SPeter Zijlstra { 3035ea811747SPeter Zijlstra __current_set_polling(); 3036ea811747SPeter Zijlstra 3037ea811747SPeter Zijlstra /* 3038ea811747SPeter Zijlstra * Polling state must be visible before we test NEED_RESCHED, 30398875125eSKirill Tkhai * paired by resched_curr() 3040ea811747SPeter Zijlstra */ 30414e857c58SPeter Zijlstra smp_mb__after_atomic(); 3042ea811747SPeter Zijlstra 3043ea811747SPeter Zijlstra return unlikely(tif_need_resched()); 3044ea811747SPeter Zijlstra } 3045ea811747SPeter Zijlstra 3046ea811747SPeter Zijlstra static inline void __current_clr_polling(void) 30473a98f871SThomas Gleixner { 30483a98f871SThomas Gleixner clear_thread_flag(TIF_POLLING_NRFLAG); 30493a98f871SThomas Gleixner } 3050ea811747SPeter Zijlstra 3051ea811747SPeter Zijlstra static inline bool __must_check current_clr_polling_and_test(void) 3052ea811747SPeter Zijlstra { 3053ea811747SPeter Zijlstra __current_clr_polling(); 3054ea811747SPeter Zijlstra 3055ea811747SPeter Zijlstra /* 3056ea811747SPeter Zijlstra * Polling state must be visible before we test NEED_RESCHED, 30578875125eSKirill Tkhai * paired by resched_curr() 3058ea811747SPeter Zijlstra */ 30594e857c58SPeter Zijlstra smp_mb__after_atomic(); 3060ea811747SPeter Zijlstra 3061ea811747SPeter Zijlstra return unlikely(tif_need_resched()); 3062ea811747SPeter Zijlstra } 3063ea811747SPeter Zijlstra 3064ee761f62SThomas Gleixner #else 3065ee761f62SThomas Gleixner static inline int tsk_is_polling(struct task_struct *p) { return 0; } 3066ea811747SPeter Zijlstra static inline void __current_set_polling(void) { } 3067ea811747SPeter Zijlstra static inline void __current_clr_polling(void) { } 3068ea811747SPeter Zijlstra 3069ea811747SPeter Zijlstra static inline bool __must_check current_set_polling_and_test(void) 3070ea811747SPeter Zijlstra { 3071ea811747SPeter Zijlstra return unlikely(tif_need_resched()); 3072ea811747SPeter Zijlstra } 3073ea811747SPeter Zijlstra static inline bool __must_check current_clr_polling_and_test(void) 3074ea811747SPeter Zijlstra { 3075ea811747SPeter Zijlstra return unlikely(tif_need_resched()); 3076ea811747SPeter Zijlstra } 3077ee761f62SThomas Gleixner #endif 3078ee761f62SThomas Gleixner 30798cb75e0cSPeter Zijlstra static inline void current_clr_polling(void) 30808cb75e0cSPeter Zijlstra { 30818cb75e0cSPeter Zijlstra __current_clr_polling(); 30828cb75e0cSPeter Zijlstra 30838cb75e0cSPeter Zijlstra /* 30848cb75e0cSPeter Zijlstra * Ensure we check TIF_NEED_RESCHED after we clear the polling bit. 30858cb75e0cSPeter Zijlstra * Once the bit is cleared, we'll get IPIs with every new 30868cb75e0cSPeter Zijlstra * TIF_NEED_RESCHED and the IPI handler, scheduler_ipi(), will also 30878cb75e0cSPeter Zijlstra * fold. 30888cb75e0cSPeter Zijlstra */ 30898875125eSKirill Tkhai smp_mb(); /* paired with resched_curr() */ 30908cb75e0cSPeter Zijlstra 30918cb75e0cSPeter Zijlstra preempt_fold_need_resched(); 30928cb75e0cSPeter Zijlstra } 30938cb75e0cSPeter Zijlstra 309475f93fedSPeter Zijlstra static __always_inline bool need_resched(void) 309575f93fedSPeter Zijlstra { 309675f93fedSPeter Zijlstra return unlikely(tif_need_resched()); 309775f93fedSPeter Zijlstra } 309875f93fedSPeter Zijlstra 3099ee761f62SThomas Gleixner /* 3100f06febc9SFrank Mayhar * Thread group CPU time accounting. 3101f06febc9SFrank Mayhar */ 31024cd4c1b4SPeter Zijlstra void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times); 31034da94d49SPeter Zijlstra void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times); 3104f06febc9SFrank Mayhar 3105f06febc9SFrank Mayhar /* 31067bb44adeSRoland McGrath * Reevaluate whether the task has signals pending delivery. 31077bb44adeSRoland McGrath * Wake the task if so. 31087bb44adeSRoland McGrath * This is required every time the blocked sigset_t changes. 31097bb44adeSRoland McGrath * callers must hold sighand->siglock. 31107bb44adeSRoland McGrath */ 31117bb44adeSRoland McGrath extern void recalc_sigpending_and_wake(struct task_struct *t); 31121da177e4SLinus Torvalds extern void recalc_sigpending(void); 31131da177e4SLinus Torvalds 3114910ffdb1SOleg Nesterov extern void signal_wake_up_state(struct task_struct *t, unsigned int state); 3115910ffdb1SOleg Nesterov 3116910ffdb1SOleg Nesterov static inline void signal_wake_up(struct task_struct *t, bool resume) 3117910ffdb1SOleg Nesterov { 3118910ffdb1SOleg Nesterov signal_wake_up_state(t, resume ? TASK_WAKEKILL : 0); 3119910ffdb1SOleg Nesterov } 3120910ffdb1SOleg Nesterov static inline void ptrace_signal_wake_up(struct task_struct *t, bool resume) 3121910ffdb1SOleg Nesterov { 3122910ffdb1SOleg Nesterov signal_wake_up_state(t, resume ? __TASK_TRACED : 0); 3123910ffdb1SOleg Nesterov } 31241da177e4SLinus Torvalds 31251da177e4SLinus Torvalds /* 31261da177e4SLinus Torvalds * Wrappers for p->thread_info->cpu access. No-op on UP. 31271da177e4SLinus Torvalds */ 31281da177e4SLinus Torvalds #ifdef CONFIG_SMP 31291da177e4SLinus Torvalds 31301da177e4SLinus Torvalds static inline unsigned int task_cpu(const struct task_struct *p) 31311da177e4SLinus Torvalds { 3132a1261f54SAl Viro return task_thread_info(p)->cpu; 31331da177e4SLinus Torvalds } 31341da177e4SLinus Torvalds 3135b32e86b4SIngo Molnar static inline int task_node(const struct task_struct *p) 3136b32e86b4SIngo Molnar { 3137b32e86b4SIngo Molnar return cpu_to_node(task_cpu(p)); 3138b32e86b4SIngo Molnar } 3139b32e86b4SIngo Molnar 3140c65cc870SIngo Molnar extern void set_task_cpu(struct task_struct *p, unsigned int cpu); 31411da177e4SLinus Torvalds 31421da177e4SLinus Torvalds #else 31431da177e4SLinus Torvalds 31441da177e4SLinus Torvalds static inline unsigned int task_cpu(const struct task_struct *p) 31451da177e4SLinus Torvalds { 31461da177e4SLinus Torvalds return 0; 31471da177e4SLinus Torvalds } 31481da177e4SLinus Torvalds 31491da177e4SLinus Torvalds static inline void set_task_cpu(struct task_struct *p, unsigned int cpu) 31501da177e4SLinus Torvalds { 31511da177e4SLinus Torvalds } 31521da177e4SLinus Torvalds 31531da177e4SLinus Torvalds #endif /* CONFIG_SMP */ 31541da177e4SLinus Torvalds 315596f874e2SRusty Russell extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask); 315696f874e2SRusty Russell extern long sched_getaffinity(pid_t pid, struct cpumask *mask); 31575c45bf27SSiddha, Suresh B 31587c941438SDhaval Giani #ifdef CONFIG_CGROUP_SCHED 315907e06b01SYong Zhang extern struct task_group root_task_group; 31608323f26cSPeter Zijlstra #endif /* CONFIG_CGROUP_SCHED */ 31619b5b7751SSrivatsa Vaddagiri 316254e99124SDhaval Giani extern int task_can_switch_user(struct user_struct *up, 316354e99124SDhaval Giani struct task_struct *tsk); 316454e99124SDhaval Giani 31654b98d11bSAlexey Dobriyan #ifdef CONFIG_TASK_XACCT 31664b98d11bSAlexey Dobriyan static inline void add_rchar(struct task_struct *tsk, ssize_t amt) 31674b98d11bSAlexey Dobriyan { 3168940389b8SAndrea Righi tsk->ioac.rchar += amt; 31694b98d11bSAlexey Dobriyan } 31704b98d11bSAlexey Dobriyan 31714b98d11bSAlexey Dobriyan static inline void add_wchar(struct task_struct *tsk, ssize_t amt) 31724b98d11bSAlexey Dobriyan { 3173940389b8SAndrea Righi tsk->ioac.wchar += amt; 31744b98d11bSAlexey Dobriyan } 31754b98d11bSAlexey Dobriyan 31764b98d11bSAlexey Dobriyan static inline void inc_syscr(struct task_struct *tsk) 31774b98d11bSAlexey Dobriyan { 3178940389b8SAndrea Righi tsk->ioac.syscr++; 31794b98d11bSAlexey Dobriyan } 31804b98d11bSAlexey Dobriyan 31814b98d11bSAlexey Dobriyan static inline void inc_syscw(struct task_struct *tsk) 31824b98d11bSAlexey Dobriyan { 3183940389b8SAndrea Righi tsk->ioac.syscw++; 31844b98d11bSAlexey Dobriyan } 31854b98d11bSAlexey Dobriyan #else 31864b98d11bSAlexey Dobriyan static inline void add_rchar(struct task_struct *tsk, ssize_t amt) 31874b98d11bSAlexey Dobriyan { 31884b98d11bSAlexey Dobriyan } 31894b98d11bSAlexey Dobriyan 31904b98d11bSAlexey Dobriyan static inline void add_wchar(struct task_struct *tsk, ssize_t amt) 31914b98d11bSAlexey Dobriyan { 31924b98d11bSAlexey Dobriyan } 31934b98d11bSAlexey Dobriyan 31944b98d11bSAlexey Dobriyan static inline void inc_syscr(struct task_struct *tsk) 31954b98d11bSAlexey Dobriyan { 31964b98d11bSAlexey Dobriyan } 31974b98d11bSAlexey Dobriyan 31984b98d11bSAlexey Dobriyan static inline void inc_syscw(struct task_struct *tsk) 31994b98d11bSAlexey Dobriyan { 32004b98d11bSAlexey Dobriyan } 32014b98d11bSAlexey Dobriyan #endif 32024b98d11bSAlexey Dobriyan 320382455257SDave Hansen #ifndef TASK_SIZE_OF 320482455257SDave Hansen #define TASK_SIZE_OF(tsk) TASK_SIZE 320582455257SDave Hansen #endif 320682455257SDave Hansen 3207f98bafa0SOleg Nesterov #ifdef CONFIG_MEMCG 3208cf475ad2SBalbir Singh extern void mm_update_next_owner(struct mm_struct *mm); 3209cf475ad2SBalbir Singh #else 3210cf475ad2SBalbir Singh static inline void mm_update_next_owner(struct mm_struct *mm) 3211cf475ad2SBalbir Singh { 3212cf475ad2SBalbir Singh } 3213f98bafa0SOleg Nesterov #endif /* CONFIG_MEMCG */ 3214cf475ad2SBalbir Singh 32153e10e716SJiri Slaby static inline unsigned long task_rlimit(const struct task_struct *tsk, 32163e10e716SJiri Slaby unsigned int limit) 32173e10e716SJiri Slaby { 3218316c1608SJason Low return READ_ONCE(tsk->signal->rlim[limit].rlim_cur); 32193e10e716SJiri Slaby } 32203e10e716SJiri Slaby 32213e10e716SJiri Slaby static inline unsigned long task_rlimit_max(const struct task_struct *tsk, 32223e10e716SJiri Slaby unsigned int limit) 32233e10e716SJiri Slaby { 3224316c1608SJason Low return READ_ONCE(tsk->signal->rlim[limit].rlim_max); 32253e10e716SJiri Slaby } 32263e10e716SJiri Slaby 32273e10e716SJiri Slaby static inline unsigned long rlimit(unsigned int limit) 32283e10e716SJiri Slaby { 32293e10e716SJiri Slaby return task_rlimit(current, limit); 32303e10e716SJiri Slaby } 32313e10e716SJiri Slaby 32323e10e716SJiri Slaby static inline unsigned long rlimit_max(unsigned int limit) 32333e10e716SJiri Slaby { 32343e10e716SJiri Slaby return task_rlimit_max(current, limit); 32353e10e716SJiri Slaby } 32363e10e716SJiri Slaby 3237adaf9fcdSRafael J. Wysocki #ifdef CONFIG_CPU_FREQ 3238adaf9fcdSRafael J. Wysocki struct update_util_data { 3239adaf9fcdSRafael J. Wysocki void (*func)(struct update_util_data *data, 3240adaf9fcdSRafael J. Wysocki u64 time, unsigned long util, unsigned long max); 3241adaf9fcdSRafael J. Wysocki }; 3242adaf9fcdSRafael J. Wysocki 3243adaf9fcdSRafael J. Wysocki void cpufreq_set_update_util_data(int cpu, struct update_util_data *data); 3244adaf9fcdSRafael J. Wysocki #endif /* CONFIG_CPU_FREQ */ 3245adaf9fcdSRafael J. Wysocki 32461da177e4SLinus Torvalds #endif 3247