1b2441318SGreg Kroah-Hartman /* SPDX-License-Identifier: GPL-2.0 */ 21da177e4SLinus Torvalds #ifndef _LINUX_SCHED_H 31da177e4SLinus Torvalds #define _LINUX_SCHED_H 41da177e4SLinus Torvalds 55eca1c10SIngo Molnar /* 65eca1c10SIngo Molnar * Define 'struct task_struct' and provide the main scheduler 75eca1c10SIngo Molnar * APIs (schedule(), wakeup variants, etc.) 85eca1c10SIngo Molnar */ 95eca1c10SIngo Molnar 10607ca46eSDavid Howells #include <uapi/linux/sched.h> 11b7b3c76aSDavid Woodhouse 1270b8157eSIngo Molnar #include <asm/current.h> 1370b8157eSIngo Molnar 145eca1c10SIngo Molnar #include <linux/pid.h> 155eca1c10SIngo Molnar #include <linux/sem.h> 165eca1c10SIngo Molnar #include <linux/shm.h> 175eca1c10SIngo Molnar #include <linux/kcov.h> 185eca1c10SIngo Molnar #include <linux/mutex.h> 195eca1c10SIngo Molnar #include <linux/plist.h> 205eca1c10SIngo Molnar #include <linux/hrtimer.h> 215eca1c10SIngo Molnar #include <linux/seccomp.h> 225eca1c10SIngo Molnar #include <linux/nodemask.h> 235eca1c10SIngo Molnar #include <linux/rcupdate.h> 24ec1d2819SElena Reshetova #include <linux/refcount.h> 255eca1c10SIngo Molnar #include <linux/resource.h> 265eca1c10SIngo Molnar #include <linux/latencytop.h> 275eca1c10SIngo Molnar #include <linux/sched/prio.h> 285eca1c10SIngo Molnar #include <linux/signal_types.h> 295eca1c10SIngo Molnar #include <linux/mm_types_task.h> 305eca1c10SIngo Molnar #include <linux/task_io_accounting.h> 31d7822b1eSMathieu Desnoyers #include <linux/rseq.h> 325eca1c10SIngo Molnar 335eca1c10SIngo Molnar /* task_struct member predeclarations (sorted alphabetically): */ 34c7af7877SIngo Molnar struct audit_context; 35c7af7877SIngo Molnar struct backing_dev_info; 36c7af7877SIngo Molnar struct bio_list; 37c7af7877SIngo Molnar struct blk_plug; 383c93a0c0SQais Yousef struct capture_control; 39c7af7877SIngo Molnar struct cfs_rq; 40c7af7877SIngo Molnar struct fs_struct; 41c7af7877SIngo Molnar struct futex_pi_state; 42c7af7877SIngo Molnar struct io_context; 43c7af7877SIngo Molnar struct mempolicy; 44c7af7877SIngo Molnar struct nameidata; 45c7af7877SIngo Molnar struct nsproxy; 46c7af7877SIngo Molnar struct perf_event_context; 47c7af7877SIngo Molnar struct pid_namespace; 48c7af7877SIngo Molnar struct pipe_inode_info; 49c7af7877SIngo Molnar struct rcu_node; 50c7af7877SIngo Molnar struct reclaim_state; 51c7af7877SIngo Molnar struct robust_list_head; 523c93a0c0SQais Yousef struct root_domain; 533c93a0c0SQais Yousef struct rq; 54e2d1e2aeSIngo Molnar struct sched_attr; 55e2d1e2aeSIngo Molnar struct sched_param; 5643ae34cbSIngo Molnar struct seq_file; 57c7af7877SIngo Molnar struct sighand_struct; 58c7af7877SIngo Molnar struct signal_struct; 59c7af7877SIngo Molnar struct task_delay_info; 604cf86d77SIngo Molnar struct task_group; 611da177e4SLinus Torvalds 624a8342d2SLinus Torvalds /* 634a8342d2SLinus Torvalds * Task state bitmask. NOTE! These bits are also 644a8342d2SLinus Torvalds * encoded in fs/proc/array.c: get_task_state(). 654a8342d2SLinus Torvalds * 664a8342d2SLinus Torvalds * We have two separate sets of flags: task->state 674a8342d2SLinus Torvalds * is about runnability, while task->exit_state are 684a8342d2SLinus Torvalds * about the task exiting. Confusing, but this way 694a8342d2SLinus Torvalds * modifying one set can't modify the other one by 704a8342d2SLinus Torvalds * mistake. 714a8342d2SLinus Torvalds */ 725eca1c10SIngo Molnar 735eca1c10SIngo Molnar /* Used in tsk->state: */ 7492c4bc9fSPeter Zijlstra #define TASK_RUNNING 0x0000 7592c4bc9fSPeter Zijlstra #define TASK_INTERRUPTIBLE 0x0001 7692c4bc9fSPeter Zijlstra #define TASK_UNINTERRUPTIBLE 0x0002 7792c4bc9fSPeter Zijlstra #define __TASK_STOPPED 0x0004 7892c4bc9fSPeter Zijlstra #define __TASK_TRACED 0x0008 795eca1c10SIngo Molnar /* Used in tsk->exit_state: */ 8092c4bc9fSPeter Zijlstra #define EXIT_DEAD 0x0010 8192c4bc9fSPeter Zijlstra #define EXIT_ZOMBIE 0x0020 82abd50b39SOleg Nesterov #define EXIT_TRACE (EXIT_ZOMBIE | EXIT_DEAD) 835eca1c10SIngo Molnar /* Used in tsk->state again: */ 848ef9925bSPeter Zijlstra #define TASK_PARKED 0x0040 858ef9925bSPeter Zijlstra #define TASK_DEAD 0x0080 868ef9925bSPeter Zijlstra #define TASK_WAKEKILL 0x0100 878ef9925bSPeter Zijlstra #define TASK_WAKING 0x0200 8892c4bc9fSPeter Zijlstra #define TASK_NOLOAD 0x0400 8992c4bc9fSPeter Zijlstra #define TASK_NEW 0x0800 9092c4bc9fSPeter Zijlstra #define TASK_STATE_MAX 0x1000 91f021a3c2SMatthew Wilcox 925eca1c10SIngo Molnar /* Convenience macros for the sake of set_current_state: */ 93f021a3c2SMatthew Wilcox #define TASK_KILLABLE (TASK_WAKEKILL | TASK_UNINTERRUPTIBLE) 94f021a3c2SMatthew Wilcox #define TASK_STOPPED (TASK_WAKEKILL | __TASK_STOPPED) 95f021a3c2SMatthew Wilcox #define TASK_TRACED (TASK_WAKEKILL | __TASK_TRACED) 961da177e4SLinus Torvalds 9780ed87c8SPeter Zijlstra #define TASK_IDLE (TASK_UNINTERRUPTIBLE | TASK_NOLOAD) 9880ed87c8SPeter Zijlstra 995eca1c10SIngo Molnar /* Convenience macros for the sake of wake_up(): */ 10092a1f4bcSMatthew Wilcox #define TASK_NORMAL (TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE) 10192a1f4bcSMatthew Wilcox 1025eca1c10SIngo Molnar /* get_task_state(): */ 10392a1f4bcSMatthew Wilcox #define TASK_REPORT (TASK_RUNNING | TASK_INTERRUPTIBLE | \ 104f021a3c2SMatthew Wilcox TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \ 1058ef9925bSPeter Zijlstra __TASK_TRACED | EXIT_DEAD | EXIT_ZOMBIE | \ 1068ef9925bSPeter Zijlstra TASK_PARKED) 10792a1f4bcSMatthew Wilcox 108f021a3c2SMatthew Wilcox #define task_is_traced(task) ((task->state & __TASK_TRACED) != 0) 1095eca1c10SIngo Molnar 110f021a3c2SMatthew Wilcox #define task_is_stopped(task) ((task->state & __TASK_STOPPED) != 0) 1115eca1c10SIngo Molnar 1125eca1c10SIngo Molnar #define task_is_stopped_or_traced(task) ((task->state & (__TASK_STOPPED | __TASK_TRACED)) != 0) 1135eca1c10SIngo Molnar 1145eca1c10SIngo Molnar #define task_contributes_to_load(task) ((task->state & TASK_UNINTERRUPTIBLE) != 0 && \ 11580ed87c8SPeter Zijlstra (task->flags & PF_FROZEN) == 0 && \ 11680ed87c8SPeter Zijlstra (task->state & TASK_NOLOAD) == 0) 1171da177e4SLinus Torvalds 1188eb23b9fSPeter Zijlstra #ifdef CONFIG_DEBUG_ATOMIC_SLEEP 1198eb23b9fSPeter Zijlstra 120b5bf9a90SPeter Zijlstra /* 121b5bf9a90SPeter Zijlstra * Special states are those that do not use the normal wait-loop pattern. See 122b5bf9a90SPeter Zijlstra * the comment with set_special_state(). 123b5bf9a90SPeter Zijlstra */ 124b5bf9a90SPeter Zijlstra #define is_special_task_state(state) \ 1251cef1150SPeter Zijlstra ((state) & (__TASK_STOPPED | __TASK_TRACED | TASK_PARKED | TASK_DEAD)) 126b5bf9a90SPeter Zijlstra 1278eb23b9fSPeter Zijlstra #define __set_current_state(state_value) \ 1288eb23b9fSPeter Zijlstra do { \ 129b5bf9a90SPeter Zijlstra WARN_ON_ONCE(is_special_task_state(state_value));\ 1308eb23b9fSPeter Zijlstra current->task_state_change = _THIS_IP_; \ 1318eb23b9fSPeter Zijlstra current->state = (state_value); \ 1328eb23b9fSPeter Zijlstra } while (0) 133b5bf9a90SPeter Zijlstra 1348eb23b9fSPeter Zijlstra #define set_current_state(state_value) \ 1358eb23b9fSPeter Zijlstra do { \ 136b5bf9a90SPeter Zijlstra WARN_ON_ONCE(is_special_task_state(state_value));\ 1378eb23b9fSPeter Zijlstra current->task_state_change = _THIS_IP_; \ 138b92b8b35SPeter Zijlstra smp_store_mb(current->state, (state_value)); \ 1398eb23b9fSPeter Zijlstra } while (0) 1408eb23b9fSPeter Zijlstra 141b5bf9a90SPeter Zijlstra #define set_special_state(state_value) \ 142b5bf9a90SPeter Zijlstra do { \ 143b5bf9a90SPeter Zijlstra unsigned long flags; /* may shadow */ \ 144b5bf9a90SPeter Zijlstra WARN_ON_ONCE(!is_special_task_state(state_value)); \ 145b5bf9a90SPeter Zijlstra raw_spin_lock_irqsave(¤t->pi_lock, flags); \ 146b5bf9a90SPeter Zijlstra current->task_state_change = _THIS_IP_; \ 147b5bf9a90SPeter Zijlstra current->state = (state_value); \ 148b5bf9a90SPeter Zijlstra raw_spin_unlock_irqrestore(¤t->pi_lock, flags); \ 149b5bf9a90SPeter Zijlstra } while (0) 1508eb23b9fSPeter Zijlstra #else 151498d0c57SAndrew Morton /* 152498d0c57SAndrew Morton * set_current_state() includes a barrier so that the write of current->state 153498d0c57SAndrew Morton * is correctly serialised wrt the caller's subsequent test of whether to 154498d0c57SAndrew Morton * actually sleep: 155498d0c57SAndrew Morton * 156a2250238SPeter Zijlstra * for (;;) { 157498d0c57SAndrew Morton * set_current_state(TASK_UNINTERRUPTIBLE); 158a2250238SPeter Zijlstra * if (!need_sleep) 159a2250238SPeter Zijlstra * break; 160498d0c57SAndrew Morton * 161a2250238SPeter Zijlstra * schedule(); 162a2250238SPeter Zijlstra * } 163a2250238SPeter Zijlstra * __set_current_state(TASK_RUNNING); 164a2250238SPeter Zijlstra * 165a2250238SPeter Zijlstra * If the caller does not need such serialisation (because, for instance, the 166a2250238SPeter Zijlstra * condition test and condition change and wakeup are under the same lock) then 167a2250238SPeter Zijlstra * use __set_current_state(). 168a2250238SPeter Zijlstra * 169a2250238SPeter Zijlstra * The above is typically ordered against the wakeup, which does: 170a2250238SPeter Zijlstra * 171a2250238SPeter Zijlstra * need_sleep = false; 172a2250238SPeter Zijlstra * wake_up_state(p, TASK_UNINTERRUPTIBLE); 173a2250238SPeter Zijlstra * 1747696f991SAndrea Parri * where wake_up_state() executes a full memory barrier before accessing the 1757696f991SAndrea Parri * task state. 176a2250238SPeter Zijlstra * 177a2250238SPeter Zijlstra * Wakeup will do: if (@state & p->state) p->state = TASK_RUNNING, that is, 178a2250238SPeter Zijlstra * once it observes the TASK_UNINTERRUPTIBLE store the waking CPU can issue a 179a2250238SPeter Zijlstra * TASK_RUNNING store which can collide with __set_current_state(TASK_RUNNING). 180a2250238SPeter Zijlstra * 181b5bf9a90SPeter Zijlstra * However, with slightly different timing the wakeup TASK_RUNNING store can 182dfcb245eSIngo Molnar * also collide with the TASK_UNINTERRUPTIBLE store. Losing that store is not 183b5bf9a90SPeter Zijlstra * a problem either because that will result in one extra go around the loop 184b5bf9a90SPeter Zijlstra * and our @cond test will save the day. 185a2250238SPeter Zijlstra * 186a2250238SPeter Zijlstra * Also see the comments of try_to_wake_up(). 187498d0c57SAndrew Morton */ 188b5bf9a90SPeter Zijlstra #define __set_current_state(state_value) \ 189b5bf9a90SPeter Zijlstra current->state = (state_value) 190b5bf9a90SPeter Zijlstra 191b5bf9a90SPeter Zijlstra #define set_current_state(state_value) \ 192b5bf9a90SPeter Zijlstra smp_store_mb(current->state, (state_value)) 193b5bf9a90SPeter Zijlstra 194b5bf9a90SPeter Zijlstra /* 195b5bf9a90SPeter Zijlstra * set_special_state() should be used for those states when the blocking task 196b5bf9a90SPeter Zijlstra * can not use the regular condition based wait-loop. In that case we must 197b5bf9a90SPeter Zijlstra * serialize against wakeups such that any possible in-flight TASK_RUNNING stores 198b5bf9a90SPeter Zijlstra * will not collide with our state change. 199b5bf9a90SPeter Zijlstra */ 200b5bf9a90SPeter Zijlstra #define set_special_state(state_value) \ 201b5bf9a90SPeter Zijlstra do { \ 202b5bf9a90SPeter Zijlstra unsigned long flags; /* may shadow */ \ 203b5bf9a90SPeter Zijlstra raw_spin_lock_irqsave(¤t->pi_lock, flags); \ 204b5bf9a90SPeter Zijlstra current->state = (state_value); \ 205b5bf9a90SPeter Zijlstra raw_spin_unlock_irqrestore(¤t->pi_lock, flags); \ 206b5bf9a90SPeter Zijlstra } while (0) 207b5bf9a90SPeter Zijlstra 2088eb23b9fSPeter Zijlstra #endif 2098eb23b9fSPeter Zijlstra 2105eca1c10SIngo Molnar /* Task command name length: */ 2111da177e4SLinus Torvalds #define TASK_COMM_LEN 16 2121da177e4SLinus Torvalds 2131da177e4SLinus Torvalds extern void scheduler_tick(void); 2141da177e4SLinus Torvalds 2151da177e4SLinus Torvalds #define MAX_SCHEDULE_TIMEOUT LONG_MAX 2165eca1c10SIngo Molnar 2175eca1c10SIngo Molnar extern long schedule_timeout(long timeout); 2185eca1c10SIngo Molnar extern long schedule_timeout_interruptible(long timeout); 2195eca1c10SIngo Molnar extern long schedule_timeout_killable(long timeout); 2205eca1c10SIngo Molnar extern long schedule_timeout_uninterruptible(long timeout); 2215eca1c10SIngo Molnar extern long schedule_timeout_idle(long timeout); 2221da177e4SLinus Torvalds asmlinkage void schedule(void); 223c5491ea7SThomas Gleixner extern void schedule_preempt_disabled(void); 2241da177e4SLinus Torvalds 22510ab5643STejun Heo extern int __must_check io_schedule_prepare(void); 22610ab5643STejun Heo extern void io_schedule_finish(int token); 2279cff8adeSNeilBrown extern long io_schedule_timeout(long timeout); 22810ab5643STejun Heo extern void io_schedule(void); 2299cff8adeSNeilBrown 230f06febc9SFrank Mayhar /** 2310ba42a59SMasanari Iida * struct prev_cputime - snapshot of system and user cputime 232d37f761dSFrederic Weisbecker * @utime: time spent in user mode 233d37f761dSFrederic Weisbecker * @stime: time spent in system mode 2349d7fb042SPeter Zijlstra * @lock: protects the above two fields 235d37f761dSFrederic Weisbecker * 2369d7fb042SPeter Zijlstra * Stores previous user/system time values such that we can guarantee 2379d7fb042SPeter Zijlstra * monotonicity. 238d37f761dSFrederic Weisbecker */ 2399d7fb042SPeter Zijlstra struct prev_cputime { 2409d7fb042SPeter Zijlstra #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE 2415613fda9SFrederic Weisbecker u64 utime; 2425613fda9SFrederic Weisbecker u64 stime; 2439d7fb042SPeter Zijlstra raw_spinlock_t lock; 2449d7fb042SPeter Zijlstra #endif 245d37f761dSFrederic Weisbecker }; 246d37f761dSFrederic Weisbecker 247d37f761dSFrederic Weisbecker /** 248f06febc9SFrank Mayhar * struct task_cputime - collected CPU time counts 2495613fda9SFrederic Weisbecker * @utime: time spent in user mode, in nanoseconds 2505613fda9SFrederic Weisbecker * @stime: time spent in kernel mode, in nanoseconds 251f06febc9SFrank Mayhar * @sum_exec_runtime: total time spent on the CPU, in nanoseconds 252f06febc9SFrank Mayhar * 2539d7fb042SPeter Zijlstra * This structure groups together three kinds of CPU time that are tracked for 2549d7fb042SPeter Zijlstra * threads and thread groups. Most things considering CPU time want to group 2559d7fb042SPeter Zijlstra * these counts together and treat all three of them in parallel. 256f06febc9SFrank Mayhar */ 257f06febc9SFrank Mayhar struct task_cputime { 2585613fda9SFrederic Weisbecker u64 utime; 2595613fda9SFrederic Weisbecker u64 stime; 260f06febc9SFrank Mayhar unsigned long long sum_exec_runtime; 261f06febc9SFrank Mayhar }; 2629d7fb042SPeter Zijlstra 2635eca1c10SIngo Molnar /* Alternate field names when used on cache expirations: */ 264f06febc9SFrank Mayhar #define virt_exp utime 2659d7fb042SPeter Zijlstra #define prof_exp stime 266f06febc9SFrank Mayhar #define sched_exp sum_exec_runtime 267f06febc9SFrank Mayhar 268bac5b6b6SFrederic Weisbecker enum vtime_state { 269bac5b6b6SFrederic Weisbecker /* Task is sleeping or running in a CPU with VTIME inactive: */ 270bac5b6b6SFrederic Weisbecker VTIME_INACTIVE = 0, 271bac5b6b6SFrederic Weisbecker /* Task runs in userspace in a CPU with VTIME active: */ 272bac5b6b6SFrederic Weisbecker VTIME_USER, 273bac5b6b6SFrederic Weisbecker /* Task runs in kernelspace in a CPU with VTIME active: */ 274bac5b6b6SFrederic Weisbecker VTIME_SYS, 275bac5b6b6SFrederic Weisbecker }; 276bac5b6b6SFrederic Weisbecker 277bac5b6b6SFrederic Weisbecker struct vtime { 278bac5b6b6SFrederic Weisbecker seqcount_t seqcount; 279bac5b6b6SFrederic Weisbecker unsigned long long starttime; 280bac5b6b6SFrederic Weisbecker enum vtime_state state; 2812a42eb95SWanpeng Li u64 utime; 2822a42eb95SWanpeng Li u64 stime; 2832a42eb95SWanpeng Li u64 gtime; 284bac5b6b6SFrederic Weisbecker }; 285bac5b6b6SFrederic Weisbecker 286*69842cbaSPatrick Bellasi /* 287*69842cbaSPatrick Bellasi * Utilization clamp constraints. 288*69842cbaSPatrick Bellasi * @UCLAMP_MIN: Minimum utilization 289*69842cbaSPatrick Bellasi * @UCLAMP_MAX: Maximum utilization 290*69842cbaSPatrick Bellasi * @UCLAMP_CNT: Utilization clamp constraints count 291*69842cbaSPatrick Bellasi */ 292*69842cbaSPatrick Bellasi enum uclamp_id { 293*69842cbaSPatrick Bellasi UCLAMP_MIN = 0, 294*69842cbaSPatrick Bellasi UCLAMP_MAX, 295*69842cbaSPatrick Bellasi UCLAMP_CNT 296*69842cbaSPatrick Bellasi }; 297*69842cbaSPatrick Bellasi 2981da177e4SLinus Torvalds struct sched_info { 2997f5f8e8dSIngo Molnar #ifdef CONFIG_SCHED_INFO 3005eca1c10SIngo Molnar /* Cumulative counters: */ 3011da177e4SLinus Torvalds 3025eca1c10SIngo Molnar /* # of times we have run on this CPU: */ 3035eca1c10SIngo Molnar unsigned long pcount; 3045eca1c10SIngo Molnar 3055eca1c10SIngo Molnar /* Time spent waiting on a runqueue: */ 3065eca1c10SIngo Molnar unsigned long long run_delay; 3075eca1c10SIngo Molnar 3085eca1c10SIngo Molnar /* Timestamps: */ 3095eca1c10SIngo Molnar 3105eca1c10SIngo Molnar /* When did we last run on a CPU? */ 3115eca1c10SIngo Molnar unsigned long long last_arrival; 3125eca1c10SIngo Molnar 3135eca1c10SIngo Molnar /* When were we last queued to run? */ 3145eca1c10SIngo Molnar unsigned long long last_queued; 3155eca1c10SIngo Molnar 316f6db8347SNaveen N. Rao #endif /* CONFIG_SCHED_INFO */ 3177f5f8e8dSIngo Molnar }; 3181da177e4SLinus Torvalds 3191da177e4SLinus Torvalds /* 3206ecdd749SYuyang Du * Integer metrics need fixed point arithmetic, e.g., sched/fair 3216ecdd749SYuyang Du * has a few: load, load_avg, util_avg, freq, and capacity. 3226ecdd749SYuyang Du * 3236ecdd749SYuyang Du * We define a basic fixed point arithmetic range, and then formalize 3246ecdd749SYuyang Du * all these metrics based on that basic range. 3256ecdd749SYuyang Du */ 3266ecdd749SYuyang Du # define SCHED_FIXEDPOINT_SHIFT 10 3276ecdd749SYuyang Du # define SCHED_FIXEDPOINT_SCALE (1L << SCHED_FIXEDPOINT_SHIFT) 3286ecdd749SYuyang Du 329*69842cbaSPatrick Bellasi /* Increase resolution of cpu_capacity calculations */ 330*69842cbaSPatrick Bellasi # define SCHED_CAPACITY_SHIFT SCHED_FIXEDPOINT_SHIFT 331*69842cbaSPatrick Bellasi # define SCHED_CAPACITY_SCALE (1L << SCHED_CAPACITY_SHIFT) 332*69842cbaSPatrick Bellasi 33320b8a59fSIngo Molnar struct load_weight { 3349dbdb155SPeter Zijlstra unsigned long weight; 3359dbdb155SPeter Zijlstra u32 inv_weight; 33620b8a59fSIngo Molnar }; 33720b8a59fSIngo Molnar 3387f65ea42SPatrick Bellasi /** 3397f65ea42SPatrick Bellasi * struct util_est - Estimation utilization of FAIR tasks 3407f65ea42SPatrick Bellasi * @enqueued: instantaneous estimated utilization of a task/cpu 3417f65ea42SPatrick Bellasi * @ewma: the Exponential Weighted Moving Average (EWMA) 3427f65ea42SPatrick Bellasi * utilization of a task 3437f65ea42SPatrick Bellasi * 3447f65ea42SPatrick Bellasi * Support data structure to track an Exponential Weighted Moving Average 3457f65ea42SPatrick Bellasi * (EWMA) of a FAIR task's utilization. New samples are added to the moving 3467f65ea42SPatrick Bellasi * average each time a task completes an activation. Sample's weight is chosen 3477f65ea42SPatrick Bellasi * so that the EWMA will be relatively insensitive to transient changes to the 3487f65ea42SPatrick Bellasi * task's workload. 3497f65ea42SPatrick Bellasi * 3507f65ea42SPatrick Bellasi * The enqueued attribute has a slightly different meaning for tasks and cpus: 3517f65ea42SPatrick Bellasi * - task: the task's util_avg at last task dequeue time 3527f65ea42SPatrick Bellasi * - cfs_rq: the sum of util_est.enqueued for each RUNNABLE task on that CPU 3537f65ea42SPatrick Bellasi * Thus, the util_est.enqueued of a task represents the contribution on the 3547f65ea42SPatrick Bellasi * estimated utilization of the CPU where that task is currently enqueued. 3557f65ea42SPatrick Bellasi * 3567f65ea42SPatrick Bellasi * Only for tasks we track a moving average of the past instantaneous 3577f65ea42SPatrick Bellasi * estimated utilization. This allows to absorb sporadic drops in utilization 3587f65ea42SPatrick Bellasi * of an otherwise almost periodic task. 3597f65ea42SPatrick Bellasi */ 3607f65ea42SPatrick Bellasi struct util_est { 3617f65ea42SPatrick Bellasi unsigned int enqueued; 3627f65ea42SPatrick Bellasi unsigned int ewma; 3637f65ea42SPatrick Bellasi #define UTIL_EST_WEIGHT_SHIFT 2 364317d359dSPeter Zijlstra } __attribute__((__aligned__(sizeof(u64)))); 3657f65ea42SPatrick Bellasi 3669d89c257SYuyang Du /* 3677b595334SYuyang Du * The load_avg/util_avg accumulates an infinite geometric series 3687b595334SYuyang Du * (see __update_load_avg() in kernel/sched/fair.c). 3697b595334SYuyang Du * 3707b595334SYuyang Du * [load_avg definition] 3717b595334SYuyang Du * 3727b595334SYuyang Du * load_avg = runnable% * scale_load_down(load) 3737b595334SYuyang Du * 3747b595334SYuyang Du * where runnable% is the time ratio that a sched_entity is runnable. 3757b595334SYuyang Du * For cfs_rq, it is the aggregated load_avg of all runnable and 3769d89c257SYuyang Du * blocked sched_entities. 3777b595334SYuyang Du * 3787b595334SYuyang Du * [util_avg definition] 3797b595334SYuyang Du * 3807b595334SYuyang Du * util_avg = running% * SCHED_CAPACITY_SCALE 3817b595334SYuyang Du * 3827b595334SYuyang Du * where running% is the time ratio that a sched_entity is running on 3837b595334SYuyang Du * a CPU. For cfs_rq, it is the aggregated util_avg of all runnable 3847b595334SYuyang Du * and blocked sched_entities. 3857b595334SYuyang Du * 38623127296SVincent Guittot * load_avg and util_avg don't direcly factor frequency scaling and CPU 38723127296SVincent Guittot * capacity scaling. The scaling is done through the rq_clock_pelt that 38823127296SVincent Guittot * is used for computing those signals (see update_rq_clock_pelt()) 3897b595334SYuyang Du * 39023127296SVincent Guittot * N.B., the above ratios (runnable% and running%) themselves are in the 39123127296SVincent Guittot * range of [0, 1]. To do fixed point arithmetics, we therefore scale them 39223127296SVincent Guittot * to as large a range as necessary. This is for example reflected by 39323127296SVincent Guittot * util_avg's SCHED_CAPACITY_SCALE. 3947b595334SYuyang Du * 3957b595334SYuyang Du * [Overflow issue] 3967b595334SYuyang Du * 3977b595334SYuyang Du * The 64-bit load_sum can have 4353082796 (=2^64/47742/88761) entities 3987b595334SYuyang Du * with the highest load (=88761), always runnable on a single cfs_rq, 3997b595334SYuyang Du * and should not overflow as the number already hits PID_MAX_LIMIT. 4007b595334SYuyang Du * 4017b595334SYuyang Du * For all other cases (including 32-bit kernels), struct load_weight's 4027b595334SYuyang Du * weight will overflow first before we do, because: 4037b595334SYuyang Du * 4047b595334SYuyang Du * Max(load_avg) <= Max(load.weight) 4057b595334SYuyang Du * 4067b595334SYuyang Du * Then it is the load_weight's responsibility to consider overflow 4077b595334SYuyang Du * issues. 4089d89c257SYuyang Du */ 4099d85f21cSPaul Turner struct sched_avg { 4105eca1c10SIngo Molnar u64 last_update_time; 4115eca1c10SIngo Molnar u64 load_sum; 4121ea6c46aSPeter Zijlstra u64 runnable_load_sum; 4135eca1c10SIngo Molnar u32 util_sum; 4145eca1c10SIngo Molnar u32 period_contrib; 4155eca1c10SIngo Molnar unsigned long load_avg; 4161ea6c46aSPeter Zijlstra unsigned long runnable_load_avg; 4175eca1c10SIngo Molnar unsigned long util_avg; 4187f65ea42SPatrick Bellasi struct util_est util_est; 419317d359dSPeter Zijlstra } ____cacheline_aligned; 4209d85f21cSPaul Turner 42141acab88SLucas De Marchi struct sched_statistics { 4227f5f8e8dSIngo Molnar #ifdef CONFIG_SCHEDSTATS 42394c18227SIngo Molnar u64 wait_start; 42494c18227SIngo Molnar u64 wait_max; 4256d082592SArjan van de Ven u64 wait_count; 4266d082592SArjan van de Ven u64 wait_sum; 4278f0dfc34SArjan van de Ven u64 iowait_count; 4288f0dfc34SArjan van de Ven u64 iowait_sum; 42994c18227SIngo Molnar 43094c18227SIngo Molnar u64 sleep_start; 43120b8a59fSIngo Molnar u64 sleep_max; 43294c18227SIngo Molnar s64 sum_sleep_runtime; 43394c18227SIngo Molnar 43494c18227SIngo Molnar u64 block_start; 43520b8a59fSIngo Molnar u64 block_max; 43620b8a59fSIngo Molnar u64 exec_max; 437eba1ed4bSIngo Molnar u64 slice_max; 438cc367732SIngo Molnar 439cc367732SIngo Molnar u64 nr_migrations_cold; 440cc367732SIngo Molnar u64 nr_failed_migrations_affine; 441cc367732SIngo Molnar u64 nr_failed_migrations_running; 442cc367732SIngo Molnar u64 nr_failed_migrations_hot; 443cc367732SIngo Molnar u64 nr_forced_migrations; 444cc367732SIngo Molnar 445cc367732SIngo Molnar u64 nr_wakeups; 446cc367732SIngo Molnar u64 nr_wakeups_sync; 447cc367732SIngo Molnar u64 nr_wakeups_migrate; 448cc367732SIngo Molnar u64 nr_wakeups_local; 449cc367732SIngo Molnar u64 nr_wakeups_remote; 450cc367732SIngo Molnar u64 nr_wakeups_affine; 451cc367732SIngo Molnar u64 nr_wakeups_affine_attempts; 452cc367732SIngo Molnar u64 nr_wakeups_passive; 453cc367732SIngo Molnar u64 nr_wakeups_idle; 45441acab88SLucas De Marchi #endif 4557f5f8e8dSIngo Molnar }; 45641acab88SLucas De Marchi 45741acab88SLucas De Marchi struct sched_entity { 4585eca1c10SIngo Molnar /* For load-balancing: */ 4595eca1c10SIngo Molnar struct load_weight load; 4601ea6c46aSPeter Zijlstra unsigned long runnable_weight; 46141acab88SLucas De Marchi struct rb_node run_node; 46241acab88SLucas De Marchi struct list_head group_node; 46341acab88SLucas De Marchi unsigned int on_rq; 46441acab88SLucas De Marchi 46541acab88SLucas De Marchi u64 exec_start; 46641acab88SLucas De Marchi u64 sum_exec_runtime; 46741acab88SLucas De Marchi u64 vruntime; 46841acab88SLucas De Marchi u64 prev_sum_exec_runtime; 46941acab88SLucas De Marchi 47041acab88SLucas De Marchi u64 nr_migrations; 47141acab88SLucas De Marchi 47241acab88SLucas De Marchi struct sched_statistics statistics; 47394c18227SIngo Molnar 47420b8a59fSIngo Molnar #ifdef CONFIG_FAIR_GROUP_SCHED 475fed14d45SPeter Zijlstra int depth; 47620b8a59fSIngo Molnar struct sched_entity *parent; 47720b8a59fSIngo Molnar /* rq on which this entity is (to be) queued: */ 47820b8a59fSIngo Molnar struct cfs_rq *cfs_rq; 47920b8a59fSIngo Molnar /* rq "owned" by this entity/group: */ 48020b8a59fSIngo Molnar struct cfs_rq *my_q; 48120b8a59fSIngo Molnar #endif 4828bd75c77SClark Williams 483141965c7SAlex Shi #ifdef CONFIG_SMP 4845a107804SJiri Olsa /* 4855a107804SJiri Olsa * Per entity load average tracking. 4865a107804SJiri Olsa * 4875a107804SJiri Olsa * Put into separate cache line so it does not 4885a107804SJiri Olsa * collide with read-mostly values above. 4895a107804SJiri Olsa */ 490317d359dSPeter Zijlstra struct sched_avg avg; 4919d85f21cSPaul Turner #endif 49220b8a59fSIngo Molnar }; 49370b97a7fSIngo Molnar 494fa717060SPeter Zijlstra struct sched_rt_entity { 495fa717060SPeter Zijlstra struct list_head run_list; 49678f2c7dbSPeter Zijlstra unsigned long timeout; 49757d2aa00SYing Xue unsigned long watchdog_stamp; 498bee367edSRichard Kennedy unsigned int time_slice; 499ff77e468SPeter Zijlstra unsigned short on_rq; 500ff77e468SPeter Zijlstra unsigned short on_list; 5016f505b16SPeter Zijlstra 50258d6c2d7SPeter Zijlstra struct sched_rt_entity *back; 503052f1dc7SPeter Zijlstra #ifdef CONFIG_RT_GROUP_SCHED 5046f505b16SPeter Zijlstra struct sched_rt_entity *parent; 5056f505b16SPeter Zijlstra /* rq on which this entity is (to be) queued: */ 5066f505b16SPeter Zijlstra struct rt_rq *rt_rq; 5076f505b16SPeter Zijlstra /* rq "owned" by this entity/group: */ 5086f505b16SPeter Zijlstra struct rt_rq *my_q; 5096f505b16SPeter Zijlstra #endif 5103859a271SKees Cook } __randomize_layout; 511fa717060SPeter Zijlstra 512aab03e05SDario Faggioli struct sched_dl_entity { 513aab03e05SDario Faggioli struct rb_node rb_node; 514aab03e05SDario Faggioli 515aab03e05SDario Faggioli /* 516aab03e05SDario Faggioli * Original scheduling parameters. Copied here from sched_attr 5174027d080Sxiaofeng.yan * during sched_setattr(), they will remain the same until 5184027d080Sxiaofeng.yan * the next sched_setattr(). 519aab03e05SDario Faggioli */ 5205eca1c10SIngo Molnar u64 dl_runtime; /* Maximum runtime for each instance */ 5215eca1c10SIngo Molnar u64 dl_deadline; /* Relative deadline of each instance */ 5225eca1c10SIngo Molnar u64 dl_period; /* Separation of two instances (period) */ 52354d6d303SDaniel Bristot de Oliveira u64 dl_bw; /* dl_runtime / dl_period */ 5243effcb42SDaniel Bristot de Oliveira u64 dl_density; /* dl_runtime / dl_deadline */ 525aab03e05SDario Faggioli 526aab03e05SDario Faggioli /* 527aab03e05SDario Faggioli * Actual scheduling parameters. Initialized with the values above, 528dfcb245eSIngo Molnar * they are continuously updated during task execution. Note that 529aab03e05SDario Faggioli * the remaining runtime could be < 0 in case we are in overrun. 530aab03e05SDario Faggioli */ 5315eca1c10SIngo Molnar s64 runtime; /* Remaining runtime for this instance */ 5325eca1c10SIngo Molnar u64 deadline; /* Absolute deadline for this instance */ 5335eca1c10SIngo Molnar unsigned int flags; /* Specifying the scheduler behaviour */ 534aab03e05SDario Faggioli 535aab03e05SDario Faggioli /* 536aab03e05SDario Faggioli * Some bool flags: 537aab03e05SDario Faggioli * 538aab03e05SDario Faggioli * @dl_throttled tells if we exhausted the runtime. If so, the 539aab03e05SDario Faggioli * task has to wait for a replenishment to be performed at the 540aab03e05SDario Faggioli * next firing of dl_timer. 541aab03e05SDario Faggioli * 5422d3d891dSDario Faggioli * @dl_boosted tells if we are boosted due to DI. If so we are 5432d3d891dSDario Faggioli * outside bandwidth enforcement mechanism (but only until we 5445bfd126eSJuri Lelli * exit the critical section); 5455bfd126eSJuri Lelli * 5465eca1c10SIngo Molnar * @dl_yielded tells if task gave up the CPU before consuming 5475bfd126eSJuri Lelli * all its available runtime during the last job. 548209a0cbdSLuca Abeni * 549209a0cbdSLuca Abeni * @dl_non_contending tells if the task is inactive while still 550209a0cbdSLuca Abeni * contributing to the active utilization. In other words, it 551209a0cbdSLuca Abeni * indicates if the inactive timer has been armed and its handler 552209a0cbdSLuca Abeni * has not been executed yet. This flag is useful to avoid race 553209a0cbdSLuca Abeni * conditions between the inactive timer handler and the wakeup 554209a0cbdSLuca Abeni * code. 55534be3930SJuri Lelli * 55634be3930SJuri Lelli * @dl_overrun tells if the task asked to be informed about runtime 55734be3930SJuri Lelli * overruns. 558aab03e05SDario Faggioli */ 559aa5222e9SDan Carpenter unsigned int dl_throttled : 1; 560aa5222e9SDan Carpenter unsigned int dl_boosted : 1; 561aa5222e9SDan Carpenter unsigned int dl_yielded : 1; 562aa5222e9SDan Carpenter unsigned int dl_non_contending : 1; 56334be3930SJuri Lelli unsigned int dl_overrun : 1; 564aab03e05SDario Faggioli 565aab03e05SDario Faggioli /* 566aab03e05SDario Faggioli * Bandwidth enforcement timer. Each -deadline task has its 567aab03e05SDario Faggioli * own bandwidth to be enforced, thus we need one timer per task. 568aab03e05SDario Faggioli */ 569aab03e05SDario Faggioli struct hrtimer dl_timer; 570209a0cbdSLuca Abeni 571209a0cbdSLuca Abeni /* 572209a0cbdSLuca Abeni * Inactive timer, responsible for decreasing the active utilization 573209a0cbdSLuca Abeni * at the "0-lag time". When a -deadline task blocks, it contributes 574209a0cbdSLuca Abeni * to GRUB's active utilization until the "0-lag time", hence a 575209a0cbdSLuca Abeni * timer is needed to decrease the active utilization at the correct 576209a0cbdSLuca Abeni * time. 577209a0cbdSLuca Abeni */ 578209a0cbdSLuca Abeni struct hrtimer inactive_timer; 579aab03e05SDario Faggioli }; 5808bd75c77SClark Williams 581*69842cbaSPatrick Bellasi #ifdef CONFIG_UCLAMP_TASK 582*69842cbaSPatrick Bellasi /* Number of utilization clamp buckets (shorter alias) */ 583*69842cbaSPatrick Bellasi #define UCLAMP_BUCKETS CONFIG_UCLAMP_BUCKETS_COUNT 584*69842cbaSPatrick Bellasi 585*69842cbaSPatrick Bellasi /* 586*69842cbaSPatrick Bellasi * Utilization clamp for a scheduling entity 587*69842cbaSPatrick Bellasi * @value: clamp value "assigned" to a se 588*69842cbaSPatrick Bellasi * @bucket_id: bucket index corresponding to the "assigned" value 589*69842cbaSPatrick Bellasi * 590*69842cbaSPatrick Bellasi * The bucket_id is the index of the clamp bucket matching the clamp value 591*69842cbaSPatrick Bellasi * which is pre-computed and stored to avoid expensive integer divisions from 592*69842cbaSPatrick Bellasi * the fast path. 593*69842cbaSPatrick Bellasi */ 594*69842cbaSPatrick Bellasi struct uclamp_se { 595*69842cbaSPatrick Bellasi unsigned int value : bits_per(SCHED_CAPACITY_SCALE); 596*69842cbaSPatrick Bellasi unsigned int bucket_id : bits_per(UCLAMP_BUCKETS); 597*69842cbaSPatrick Bellasi }; 598*69842cbaSPatrick Bellasi #endif /* CONFIG_UCLAMP_TASK */ 599*69842cbaSPatrick Bellasi 6001d082fd0SPaul E. McKenney union rcu_special { 6011d082fd0SPaul E. McKenney struct { 6028203d6d0SPaul E. McKenney u8 blocked; 6038203d6d0SPaul E. McKenney u8 need_qs; 60405f41571SPaul E. McKenney u8 exp_hint; /* Hint for performance. */ 60505f41571SPaul E. McKenney u8 pad; /* No garbage from compiler! */ 6068203d6d0SPaul E. McKenney } b; /* Bits. */ 60705f41571SPaul E. McKenney u32 s; /* Set of bits. */ 6081d082fd0SPaul E. McKenney }; 60986848966SPaul E. McKenney 6108dc85d54SPeter Zijlstra enum perf_event_task_context { 6118dc85d54SPeter Zijlstra perf_invalid_context = -1, 6128dc85d54SPeter Zijlstra perf_hw_context = 0, 61389a1e187SPeter Zijlstra perf_sw_context, 6148dc85d54SPeter Zijlstra perf_nr_task_contexts, 6158dc85d54SPeter Zijlstra }; 6168dc85d54SPeter Zijlstra 617eb61baf6SIngo Molnar struct wake_q_node { 618eb61baf6SIngo Molnar struct wake_q_node *next; 619eb61baf6SIngo Molnar }; 620eb61baf6SIngo Molnar 6211da177e4SLinus Torvalds struct task_struct { 622c65eacbeSAndy Lutomirski #ifdef CONFIG_THREAD_INFO_IN_TASK 623c65eacbeSAndy Lutomirski /* 624c65eacbeSAndy Lutomirski * For reasons of header soup (see current_thread_info()), this 625c65eacbeSAndy Lutomirski * must be the first element of task_struct. 626c65eacbeSAndy Lutomirski */ 627c65eacbeSAndy Lutomirski struct thread_info thread_info; 628c65eacbeSAndy Lutomirski #endif 6295eca1c10SIngo Molnar /* -1 unrunnable, 0 runnable, >0 stopped: */ 6305eca1c10SIngo Molnar volatile long state; 63129e48ce8SKees Cook 63229e48ce8SKees Cook /* 63329e48ce8SKees Cook * This begins the randomizable portion of task_struct. Only 63429e48ce8SKees Cook * scheduling-critical items should be added above here. 63529e48ce8SKees Cook */ 63629e48ce8SKees Cook randomized_struct_fields_start 63729e48ce8SKees Cook 638f7e4217bSRoman Zippel void *stack; 639ec1d2819SElena Reshetova refcount_t usage; 6405eca1c10SIngo Molnar /* Per task flags (PF_*), defined further below: */ 6415eca1c10SIngo Molnar unsigned int flags; 64297dc32cdSWilliam Cohen unsigned int ptrace; 6431da177e4SLinus Torvalds 6442dd73a4fSPeter Williams #ifdef CONFIG_SMP 645fa14ff4aSPeter Zijlstra struct llist_node wake_entry; 6463ca7a440SPeter Zijlstra int on_cpu; 647c65eacbeSAndy Lutomirski #ifdef CONFIG_THREAD_INFO_IN_TASK 6485eca1c10SIngo Molnar /* Current CPU: */ 6495eca1c10SIngo Molnar unsigned int cpu; 650c65eacbeSAndy Lutomirski #endif 65163b0e9edSMike Galbraith unsigned int wakee_flips; 65262470419SMichael Wang unsigned long wakee_flip_decay_ts; 65363b0e9edSMike Galbraith struct task_struct *last_wakee; 654ac66f547SPeter Zijlstra 65532e839ddSMel Gorman /* 65632e839ddSMel Gorman * recent_used_cpu is initially set as the last CPU used by a task 65732e839ddSMel Gorman * that wakes affine another task. Waker/wakee relationships can 65832e839ddSMel Gorman * push tasks around a CPU where each wakeup moves to the next one. 65932e839ddSMel Gorman * Tracking a recently used CPU allows a quick search for a recently 66032e839ddSMel Gorman * used CPU that may be idle. 66132e839ddSMel Gorman */ 66232e839ddSMel Gorman int recent_used_cpu; 663ac66f547SPeter Zijlstra int wake_cpu; 6644866cde0SNick Piggin #endif 665fd2f4419SPeter Zijlstra int on_rq; 66650e645a8SIngo Molnar 6675eca1c10SIngo Molnar int prio; 6685eca1c10SIngo Molnar int static_prio; 6695eca1c10SIngo Molnar int normal_prio; 670c7aceabaSRichard Kennedy unsigned int rt_priority; 6715eca1c10SIngo Molnar 6725522d5d5SIngo Molnar const struct sched_class *sched_class; 67320b8a59fSIngo Molnar struct sched_entity se; 674fa717060SPeter Zijlstra struct sched_rt_entity rt; 6758323f26cSPeter Zijlstra #ifdef CONFIG_CGROUP_SCHED 6768323f26cSPeter Zijlstra struct task_group *sched_task_group; 6778323f26cSPeter Zijlstra #endif 678aab03e05SDario Faggioli struct sched_dl_entity dl; 6791da177e4SLinus Torvalds 680*69842cbaSPatrick Bellasi #ifdef CONFIG_UCLAMP_TASK 681*69842cbaSPatrick Bellasi struct uclamp_se uclamp[UCLAMP_CNT]; 682*69842cbaSPatrick Bellasi #endif 683*69842cbaSPatrick Bellasi 684e107be36SAvi Kivity #ifdef CONFIG_PREEMPT_NOTIFIERS 6855eca1c10SIngo Molnar /* List of struct preempt_notifier: */ 686e107be36SAvi Kivity struct hlist_head preempt_notifiers; 687e107be36SAvi Kivity #endif 688e107be36SAvi Kivity 6896c5c9341SAlexey Dobriyan #ifdef CONFIG_BLK_DEV_IO_TRACE 6902056a782SJens Axboe unsigned int btrace_seq; 6916c5c9341SAlexey Dobriyan #endif 6921da177e4SLinus Torvalds 69397dc32cdSWilliam Cohen unsigned int policy; 69429baa747SPeter Zijlstra int nr_cpus_allowed; 6953bd37062SSebastian Andrzej Siewior const cpumask_t *cpus_ptr; 6963bd37062SSebastian Andrzej Siewior cpumask_t cpus_mask; 6971da177e4SLinus Torvalds 698a57eb940SPaul E. McKenney #ifdef CONFIG_PREEMPT_RCU 699e260be67SPaul E. McKenney int rcu_read_lock_nesting; 7001d082fd0SPaul E. McKenney union rcu_special rcu_read_unlock_special; 701f41d911fSPaul E. McKenney struct list_head rcu_node_entry; 702a57eb940SPaul E. McKenney struct rcu_node *rcu_blocked_node; 70328f6569aSPranith Kumar #endif /* #ifdef CONFIG_PREEMPT_RCU */ 7045eca1c10SIngo Molnar 7058315f422SPaul E. McKenney #ifdef CONFIG_TASKS_RCU 7068315f422SPaul E. McKenney unsigned long rcu_tasks_nvcsw; 707ccdd29ffSPaul E. McKenney u8 rcu_tasks_holdout; 708ccdd29ffSPaul E. McKenney u8 rcu_tasks_idx; 709176f8f7aSPaul E. McKenney int rcu_tasks_idle_cpu; 710ccdd29ffSPaul E. McKenney struct list_head rcu_tasks_holdout_list; 7118315f422SPaul E. McKenney #endif /* #ifdef CONFIG_TASKS_RCU */ 712e260be67SPaul E. McKenney 7131da177e4SLinus Torvalds struct sched_info sched_info; 7141da177e4SLinus Torvalds 7151da177e4SLinus Torvalds struct list_head tasks; 716806c09a7SDario Faggioli #ifdef CONFIG_SMP 717917b627dSGregory Haskins struct plist_node pushable_tasks; 7181baca4ceSJuri Lelli struct rb_node pushable_dl_tasks; 719806c09a7SDario Faggioli #endif 7201da177e4SLinus Torvalds 7215eca1c10SIngo Molnar struct mm_struct *mm; 7225eca1c10SIngo Molnar struct mm_struct *active_mm; 723314ff785SIngo Molnar 724314ff785SIngo Molnar /* Per-thread vma caching: */ 725314ff785SIngo Molnar struct vmacache vmacache; 726314ff785SIngo Molnar 7275eca1c10SIngo Molnar #ifdef SPLIT_RSS_COUNTING 72834e55232SKAMEZAWA Hiroyuki struct task_rss_stat rss_stat; 72934e55232SKAMEZAWA Hiroyuki #endif 73097dc32cdSWilliam Cohen int exit_state; 7315eca1c10SIngo Molnar int exit_code; 7325eca1c10SIngo Molnar int exit_signal; 7335eca1c10SIngo Molnar /* The signal sent when the parent dies: */ 7345eca1c10SIngo Molnar int pdeath_signal; 7355eca1c10SIngo Molnar /* JOBCTL_*, siglock protected: */ 7365eca1c10SIngo Molnar unsigned long jobctl; 7379b89f6baSAndrei Epure 7385eca1c10SIngo Molnar /* Used for emulating ABI behavior of previous Linux versions: */ 73997dc32cdSWilliam Cohen unsigned int personality; 7409b89f6baSAndrei Epure 7415eca1c10SIngo Molnar /* Scheduler bits, serialized by scheduler locks: */ 742ca94c442SLennart Poettering unsigned sched_reset_on_fork:1; 743a8e4f2eaSPeter Zijlstra unsigned sched_contributes_to_load:1; 744ff303e66SPeter Zijlstra unsigned sched_migrated:1; 745b7e7ade3SPeter Zijlstra unsigned sched_remote_wakeup:1; 746eb414681SJohannes Weiner #ifdef CONFIG_PSI 747eb414681SJohannes Weiner unsigned sched_psi_wake_requeue:1; 748eb414681SJohannes Weiner #endif 749eb414681SJohannes Weiner 7505eca1c10SIngo Molnar /* Force alignment to the next boundary: */ 7515eca1c10SIngo Molnar unsigned :0; 752be958bdcSPeter Zijlstra 7535eca1c10SIngo Molnar /* Unserialized, strictly 'current' */ 7545eca1c10SIngo Molnar 7555eca1c10SIngo Molnar /* Bit to tell LSMs we're in execve(): */ 7565eca1c10SIngo Molnar unsigned in_execve:1; 757be958bdcSPeter Zijlstra unsigned in_iowait:1; 7585eca1c10SIngo Molnar #ifndef TIF_RESTORE_SIGMASK 7597e781418SAndy Lutomirski unsigned restore_sigmask:1; 7607e781418SAndy Lutomirski #endif 761626ebc41STejun Heo #ifdef CONFIG_MEMCG 76229ef680aSMichal Hocko unsigned in_user_fault:1; 763127424c8SJohannes Weiner #endif 764ff303e66SPeter Zijlstra #ifdef CONFIG_COMPAT_BRK 765ff303e66SPeter Zijlstra unsigned brk_randomized:1; 766ff303e66SPeter Zijlstra #endif 76777f88796STejun Heo #ifdef CONFIG_CGROUPS 76877f88796STejun Heo /* disallow userland-initiated cgroup migration */ 76977f88796STejun Heo unsigned no_cgroup_migration:1; 77076f969e8SRoman Gushchin /* task is frozen/stopped (used by the cgroup freezer) */ 77176f969e8SRoman Gushchin unsigned frozen:1; 77277f88796STejun Heo #endif 773d09d8df3SJosef Bacik #ifdef CONFIG_BLK_CGROUP 774d09d8df3SJosef Bacik /* to be used once the psi infrastructure lands upstream. */ 775d09d8df3SJosef Bacik unsigned use_memdelay:1; 776d09d8df3SJosef Bacik #endif 7776f185c29SVladimir Davydov 7785eca1c10SIngo Molnar unsigned long atomic_flags; /* Flags requiring atomic access. */ 7791d4457f9SKees Cook 780f56141e3SAndy Lutomirski struct restart_block restart_block; 781f56141e3SAndy Lutomirski 7821da177e4SLinus Torvalds pid_t pid; 7831da177e4SLinus Torvalds pid_t tgid; 7840a425405SArjan van de Ven 785050e9baaSLinus Torvalds #ifdef CONFIG_STACKPROTECTOR 7865eca1c10SIngo Molnar /* Canary value for the -fstack-protector GCC feature: */ 7870a425405SArjan van de Ven unsigned long stack_canary; 7881314562aSHiroshi Shimamoto #endif 7891da177e4SLinus Torvalds /* 7905eca1c10SIngo Molnar * Pointers to the (original) parent process, youngest child, younger sibling, 7911da177e4SLinus Torvalds * older sibling, respectively. (p->father can be replaced with 792f470021aSRoland McGrath * p->real_parent->pid) 7931da177e4SLinus Torvalds */ 7945eca1c10SIngo Molnar 7955eca1c10SIngo Molnar /* Real parent process: */ 7965eca1c10SIngo Molnar struct task_struct __rcu *real_parent; 7975eca1c10SIngo Molnar 7985eca1c10SIngo Molnar /* Recipient of SIGCHLD, wait4() reports: */ 7995eca1c10SIngo Molnar struct task_struct __rcu *parent; 8001da177e4SLinus Torvalds 801f470021aSRoland McGrath /* 8025eca1c10SIngo Molnar * Children/sibling form the list of natural children: 8035eca1c10SIngo Molnar */ 8045eca1c10SIngo Molnar struct list_head children; 8055eca1c10SIngo Molnar struct list_head sibling; 8065eca1c10SIngo Molnar struct task_struct *group_leader; 8075eca1c10SIngo Molnar 8085eca1c10SIngo Molnar /* 8095eca1c10SIngo Molnar * 'ptraced' is the list of tasks this task is using ptrace() on. 8105eca1c10SIngo Molnar * 811f470021aSRoland McGrath * This includes both natural children and PTRACE_ATTACH targets. 8125eca1c10SIngo Molnar * 'ptrace_entry' is this task's link on the p->parent->ptraced list. 813f470021aSRoland McGrath */ 814f470021aSRoland McGrath struct list_head ptraced; 815f470021aSRoland McGrath struct list_head ptrace_entry; 816f470021aSRoland McGrath 8171da177e4SLinus Torvalds /* PID/PID hash table linkage. */ 8182c470475SEric W. Biederman struct pid *thread_pid; 8192c470475SEric W. Biederman struct hlist_node pid_links[PIDTYPE_MAX]; 82047e65328SOleg Nesterov struct list_head thread_group; 8210c740d0aSOleg Nesterov struct list_head thread_node; 8221da177e4SLinus Torvalds 8235eca1c10SIngo Molnar struct completion *vfork_done; 8241da177e4SLinus Torvalds 8255eca1c10SIngo Molnar /* CLONE_CHILD_SETTID: */ 8265eca1c10SIngo Molnar int __user *set_child_tid; 8275eca1c10SIngo Molnar 8285eca1c10SIngo Molnar /* CLONE_CHILD_CLEARTID: */ 8295eca1c10SIngo Molnar int __user *clear_child_tid; 8305eca1c10SIngo Molnar 8315eca1c10SIngo Molnar u64 utime; 8325eca1c10SIngo Molnar u64 stime; 83340565b5aSStanislaw Gruszka #ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME 8345eca1c10SIngo Molnar u64 utimescaled; 8355eca1c10SIngo Molnar u64 stimescaled; 83640565b5aSStanislaw Gruszka #endif 83716a6d9beSFrederic Weisbecker u64 gtime; 8389d7fb042SPeter Zijlstra struct prev_cputime prev_cputime; 8396a61671bSFrederic Weisbecker #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN 840bac5b6b6SFrederic Weisbecker struct vtime vtime; 8416a61671bSFrederic Weisbecker #endif 842d027d45dSFrederic Weisbecker 843d027d45dSFrederic Weisbecker #ifdef CONFIG_NO_HZ_FULL 844f009a7a7SFrederic Weisbecker atomic_t tick_dep_mask; 845d027d45dSFrederic Weisbecker #endif 8465eca1c10SIngo Molnar /* Context switch counts: */ 8475eca1c10SIngo Molnar unsigned long nvcsw; 8485eca1c10SIngo Molnar unsigned long nivcsw; 8495eca1c10SIngo Molnar 8505eca1c10SIngo Molnar /* Monotonic time in nsecs: */ 8515eca1c10SIngo Molnar u64 start_time; 8525eca1c10SIngo Molnar 8535eca1c10SIngo Molnar /* Boot based time in nsecs: */ 8545eca1c10SIngo Molnar u64 real_start_time; 8555eca1c10SIngo Molnar 8565eca1c10SIngo Molnar /* MM fault and swap info: this can arguably be seen as either mm-specific or thread-specific: */ 8575eca1c10SIngo Molnar unsigned long min_flt; 8585eca1c10SIngo Molnar unsigned long maj_flt; 8591da177e4SLinus Torvalds 860b18b6a9cSNicolas Pitre #ifdef CONFIG_POSIX_TIMERS 861f06febc9SFrank Mayhar struct task_cputime cputime_expires; 8621da177e4SLinus Torvalds struct list_head cpu_timers[3]; 863b18b6a9cSNicolas Pitre #endif 8641da177e4SLinus Torvalds 8655eca1c10SIngo Molnar /* Process credentials: */ 8665eca1c10SIngo Molnar 8675eca1c10SIngo Molnar /* Tracer's credentials at attach: */ 8685eca1c10SIngo Molnar const struct cred __rcu *ptracer_cred; 8695eca1c10SIngo Molnar 8705eca1c10SIngo Molnar /* Objective and real subjective task credentials (COW): */ 8715eca1c10SIngo Molnar const struct cred __rcu *real_cred; 8725eca1c10SIngo Molnar 8735eca1c10SIngo Molnar /* Effective (overridable) subjective task credentials (COW): */ 8745eca1c10SIngo Molnar const struct cred __rcu *cred; 8755eca1c10SIngo Molnar 8765eca1c10SIngo Molnar /* 8775eca1c10SIngo Molnar * executable name, excluding path. 8785eca1c10SIngo Molnar * 8795eca1c10SIngo Molnar * - normally initialized setup_new_exec() 8805eca1c10SIngo Molnar * - access it with [gs]et_task_comm() 8815eca1c10SIngo Molnar * - lock it with task_lock() 8825eca1c10SIngo Molnar */ 8835eca1c10SIngo Molnar char comm[TASK_COMM_LEN]; 8845eca1c10SIngo Molnar 885756daf26SNeilBrown struct nameidata *nameidata; 8865eca1c10SIngo Molnar 8873d5b6fccSAlexey Dobriyan #ifdef CONFIG_SYSVIPC 8881da177e4SLinus Torvalds struct sysv_sem sysvsem; 889ab602f79SJack Miller struct sysv_shm sysvshm; 8903d5b6fccSAlexey Dobriyan #endif 891e162b39aSMandeep Singh Baines #ifdef CONFIG_DETECT_HUNG_TASK 89282a1fcb9SIngo Molnar unsigned long last_switch_count; 893a2e51445SDmitry Vyukov unsigned long last_switch_time; 89482a1fcb9SIngo Molnar #endif 8955eca1c10SIngo Molnar /* Filesystem information: */ 8961da177e4SLinus Torvalds struct fs_struct *fs; 8975eca1c10SIngo Molnar 8985eca1c10SIngo Molnar /* Open file information: */ 8991da177e4SLinus Torvalds struct files_struct *files; 9005eca1c10SIngo Molnar 9015eca1c10SIngo Molnar /* Namespaces: */ 902ab516013SSerge E. Hallyn struct nsproxy *nsproxy; 9035eca1c10SIngo Molnar 9045eca1c10SIngo Molnar /* Signal handlers: */ 9051da177e4SLinus Torvalds struct signal_struct *signal; 9061da177e4SLinus Torvalds struct sighand_struct *sighand; 9075eca1c10SIngo Molnar sigset_t blocked; 9085eca1c10SIngo Molnar sigset_t real_blocked; 9095eca1c10SIngo Molnar /* Restored if set_restore_sigmask() was used: */ 9105eca1c10SIngo Molnar sigset_t saved_sigmask; 9111da177e4SLinus Torvalds struct sigpending pending; 9121da177e4SLinus Torvalds unsigned long sas_ss_sp; 9131da177e4SLinus Torvalds size_t sas_ss_size; 9145eca1c10SIngo Molnar unsigned int sas_ss_flags; 9152e01fabeSOleg Nesterov 91667d12145SAl Viro struct callback_head *task_works; 917e73f8959SOleg Nesterov 9184b7d248bSRichard Guy Briggs #ifdef CONFIG_AUDIT 919bfef93a5SAl Viro #ifdef CONFIG_AUDITSYSCALL 9205f3d544fSRichard Guy Briggs struct audit_context *audit_context; 9215f3d544fSRichard Guy Briggs #endif 922e1760bd5SEric W. Biederman kuid_t loginuid; 9234746ec5bSEric Paris unsigned int sessionid; 924bfef93a5SAl Viro #endif 925932ecebbSWill Drewry struct seccomp seccomp; 9261da177e4SLinus Torvalds 9275eca1c10SIngo Molnar /* Thread group tracking: */ 9281da177e4SLinus Torvalds u32 parent_exec_id; 9291da177e4SLinus Torvalds u32 self_exec_id; 9305eca1c10SIngo Molnar 9315eca1c10SIngo Molnar /* Protection against (de-)allocation: mm, files, fs, tty, keyrings, mems_allowed, mempolicy: */ 9321da177e4SLinus Torvalds spinlock_t alloc_lock; 9331da177e4SLinus Torvalds 934b29739f9SIngo Molnar /* Protection of the PI data structures: */ 9351d615482SThomas Gleixner raw_spinlock_t pi_lock; 936b29739f9SIngo Molnar 93776751049SPeter Zijlstra struct wake_q_node wake_q; 93876751049SPeter Zijlstra 93923f78d4aSIngo Molnar #ifdef CONFIG_RT_MUTEXES 9405eca1c10SIngo Molnar /* PI waiters blocked on a rt_mutex held by this task: */ 941a23ba907SDavidlohr Bueso struct rb_root_cached pi_waiters; 942e96a7705SXunlei Pang /* Updated under owner's pi_lock and rq lock */ 943e96a7705SXunlei Pang struct task_struct *pi_top_task; 9445eca1c10SIngo Molnar /* Deadlock detection and priority inheritance handling: */ 94523f78d4aSIngo Molnar struct rt_mutex_waiter *pi_blocked_on; 94623f78d4aSIngo Molnar #endif 94723f78d4aSIngo Molnar 948408894eeSIngo Molnar #ifdef CONFIG_DEBUG_MUTEXES 9495eca1c10SIngo Molnar /* Mutex deadlock detection: */ 950408894eeSIngo Molnar struct mutex_waiter *blocked_on; 951408894eeSIngo Molnar #endif 9525eca1c10SIngo Molnar 953de30a2b3SIngo Molnar #ifdef CONFIG_TRACE_IRQFLAGS 954de30a2b3SIngo Molnar unsigned int irq_events; 955de30a2b3SIngo Molnar unsigned long hardirq_enable_ip; 956de30a2b3SIngo Molnar unsigned long hardirq_disable_ip; 957fa1452e8SHiroshi Shimamoto unsigned int hardirq_enable_event; 958de30a2b3SIngo Molnar unsigned int hardirq_disable_event; 959fa1452e8SHiroshi Shimamoto int hardirqs_enabled; 960de30a2b3SIngo Molnar int hardirq_context; 961fa1452e8SHiroshi Shimamoto unsigned long softirq_disable_ip; 962fa1452e8SHiroshi Shimamoto unsigned long softirq_enable_ip; 963fa1452e8SHiroshi Shimamoto unsigned int softirq_disable_event; 964fa1452e8SHiroshi Shimamoto unsigned int softirq_enable_event; 965fa1452e8SHiroshi Shimamoto int softirqs_enabled; 966de30a2b3SIngo Molnar int softirq_context; 967de30a2b3SIngo Molnar #endif 9685eca1c10SIngo Molnar 969fbb9ce95SIngo Molnar #ifdef CONFIG_LOCKDEP 970bdb9441eSPeter Zijlstra # define MAX_LOCK_DEPTH 48UL 971fbb9ce95SIngo Molnar u64 curr_chain_key; 972fbb9ce95SIngo Molnar int lockdep_depth; 973fbb9ce95SIngo Molnar unsigned int lockdep_recursion; 974c7aceabaSRichard Kennedy struct held_lock held_locks[MAX_LOCK_DEPTH]; 975fbb9ce95SIngo Molnar #endif 9765eca1c10SIngo Molnar 977c6d30853SAndrey Ryabinin #ifdef CONFIG_UBSAN 978c6d30853SAndrey Ryabinin unsigned int in_ubsan; 979c6d30853SAndrey Ryabinin #endif 980408894eeSIngo Molnar 9815eca1c10SIngo Molnar /* Journalling filesystem info: */ 9821da177e4SLinus Torvalds void *journal_info; 9831da177e4SLinus Torvalds 9845eca1c10SIngo Molnar /* Stacked block device info: */ 985bddd87c7SAkinobu Mita struct bio_list *bio_list; 986d89d8796SNeil Brown 98773c10101SJens Axboe #ifdef CONFIG_BLOCK 9885eca1c10SIngo Molnar /* Stack plugging: */ 98973c10101SJens Axboe struct blk_plug *plug; 99073c10101SJens Axboe #endif 99173c10101SJens Axboe 9925eca1c10SIngo Molnar /* VM state: */ 9931da177e4SLinus Torvalds struct reclaim_state *reclaim_state; 9941da177e4SLinus Torvalds 9951da177e4SLinus Torvalds struct backing_dev_info *backing_dev_info; 9961da177e4SLinus Torvalds 9971da177e4SLinus Torvalds struct io_context *io_context; 9981da177e4SLinus Torvalds 9995e1f0f09SMel Gorman #ifdef CONFIG_COMPACTION 10005e1f0f09SMel Gorman struct capture_control *capture_control; 10015e1f0f09SMel Gorman #endif 10025eca1c10SIngo Molnar /* Ptrace state: */ 10031da177e4SLinus Torvalds unsigned long ptrace_message; 1004ae7795bcSEric W. Biederman kernel_siginfo_t *last_siginfo; 10055eca1c10SIngo Molnar 10067c3ab738SAndrew Morton struct task_io_accounting ioac; 1007eb414681SJohannes Weiner #ifdef CONFIG_PSI 1008eb414681SJohannes Weiner /* Pressure stall state */ 1009eb414681SJohannes Weiner unsigned int psi_flags; 1010eb414681SJohannes Weiner #endif 10115eca1c10SIngo Molnar #ifdef CONFIG_TASK_XACCT 10125eca1c10SIngo Molnar /* Accumulated RSS usage: */ 10135eca1c10SIngo Molnar u64 acct_rss_mem1; 10145eca1c10SIngo Molnar /* Accumulated virtual memory usage: */ 10155eca1c10SIngo Molnar u64 acct_vm_mem1; 10165eca1c10SIngo Molnar /* stime + utime since last update: */ 10175eca1c10SIngo Molnar u64 acct_timexpd; 10181da177e4SLinus Torvalds #endif 10191da177e4SLinus Torvalds #ifdef CONFIG_CPUSETS 10205eca1c10SIngo Molnar /* Protected by ->alloc_lock: */ 10215eca1c10SIngo Molnar nodemask_t mems_allowed; 10225eca1c10SIngo Molnar /* Seqence number to catch updates: */ 10235eca1c10SIngo Molnar seqcount_t mems_allowed_seq; 1024825a46afSPaul Jackson int cpuset_mem_spread_rotor; 10256adef3ebSJack Steiner int cpuset_slab_spread_rotor; 10261da177e4SLinus Torvalds #endif 1027ddbcc7e8SPaul Menage #ifdef CONFIG_CGROUPS 10285eca1c10SIngo Molnar /* Control Group info protected by css_set_lock: */ 10292c392b8cSArnd Bergmann struct css_set __rcu *cgroups; 10305eca1c10SIngo Molnar /* cg_list protected by css_set_lock and tsk->alloc_lock: */ 1031817929ecSPaul Menage struct list_head cg_list; 1032ddbcc7e8SPaul Menage #endif 1033e6d42931SJohannes Weiner #ifdef CONFIG_X86_CPU_RESCTRL 10340734ded1SVikas Shivappa u32 closid; 1035d6aaba61SVikas Shivappa u32 rmid; 1036e02737d5SFenghua Yu #endif 103742b2dd0aSAlexey Dobriyan #ifdef CONFIG_FUTEX 10380771dfefSIngo Molnar struct robust_list_head __user *robust_list; 103934f192c6SIngo Molnar #ifdef CONFIG_COMPAT 104034f192c6SIngo Molnar struct compat_robust_list_head __user *compat_robust_list; 104134f192c6SIngo Molnar #endif 1042c87e2837SIngo Molnar struct list_head pi_state_list; 1043c87e2837SIngo Molnar struct futex_pi_state *pi_state_cache; 104442b2dd0aSAlexey Dobriyan #endif 1045cdd6c482SIngo Molnar #ifdef CONFIG_PERF_EVENTS 10468dc85d54SPeter Zijlstra struct perf_event_context *perf_event_ctxp[perf_nr_task_contexts]; 1047cdd6c482SIngo Molnar struct mutex perf_event_mutex; 1048cdd6c482SIngo Molnar struct list_head perf_event_list; 1049a63eaf34SPaul Mackerras #endif 10508f47b187SThomas Gleixner #ifdef CONFIG_DEBUG_PREEMPT 10518f47b187SThomas Gleixner unsigned long preempt_disable_ip; 10528f47b187SThomas Gleixner #endif 1053c7aceabaSRichard Kennedy #ifdef CONFIG_NUMA 10545eca1c10SIngo Molnar /* Protected by alloc_lock: */ 10555eca1c10SIngo Molnar struct mempolicy *mempolicy; 105645816682SVlastimil Babka short il_prev; 1057207205a2SEric Dumazet short pref_node_fork; 1058c7aceabaSRichard Kennedy #endif 1059cbee9f88SPeter Zijlstra #ifdef CONFIG_NUMA_BALANCING 1060cbee9f88SPeter Zijlstra int numa_scan_seq; 1061cbee9f88SPeter Zijlstra unsigned int numa_scan_period; 1062598f0ec0SMel Gorman unsigned int numa_scan_period_max; 1063de1c9ce6SRik van Riel int numa_preferred_nid; 10646b9a7460SMel Gorman unsigned long numa_migrate_retry; 10655eca1c10SIngo Molnar /* Migration stamp: */ 10665eca1c10SIngo Molnar u64 node_stamp; 10677e2703e6SRik van Riel u64 last_task_numa_placement; 10687e2703e6SRik van Riel u64 last_sum_exec_runtime; 1069cbee9f88SPeter Zijlstra struct callback_head numa_work; 1070f809ca9aSMel Gorman 10718c8a743cSPeter Zijlstra struct numa_group *numa_group; 10728c8a743cSPeter Zijlstra 1073745d6147SMel Gorman /* 107444dba3d5SIulia Manda * numa_faults is an array split into four regions: 107544dba3d5SIulia Manda * faults_memory, faults_cpu, faults_memory_buffer, faults_cpu_buffer 107644dba3d5SIulia Manda * in this precise order. 107744dba3d5SIulia Manda * 107844dba3d5SIulia Manda * faults_memory: Exponential decaying average of faults on a per-node 107944dba3d5SIulia Manda * basis. Scheduling placement decisions are made based on these 108044dba3d5SIulia Manda * counts. The values remain static for the duration of a PTE scan. 108144dba3d5SIulia Manda * faults_cpu: Track the nodes the process was running on when a NUMA 108244dba3d5SIulia Manda * hinting fault was incurred. 108344dba3d5SIulia Manda * faults_memory_buffer and faults_cpu_buffer: Record faults per node 108444dba3d5SIulia Manda * during the current scan window. When the scan completes, the counts 108544dba3d5SIulia Manda * in faults_memory and faults_cpu decay and these values are copied. 1086745d6147SMel Gorman */ 108744dba3d5SIulia Manda unsigned long *numa_faults; 108883e1d2cdSMel Gorman unsigned long total_numa_faults; 1089745d6147SMel Gorman 1090745d6147SMel Gorman /* 109104bb2f94SRik van Riel * numa_faults_locality tracks if faults recorded during the last 1092074c2381SMel Gorman * scan window were remote/local or failed to migrate. The task scan 1093074c2381SMel Gorman * period is adapted based on the locality of the faults with different 1094074c2381SMel Gorman * weights depending on whether they were shared or private faults 109504bb2f94SRik van Riel */ 1096074c2381SMel Gorman unsigned long numa_faults_locality[3]; 109704bb2f94SRik van Riel 1098b32e86b4SIngo Molnar unsigned long numa_pages_migrated; 1099cbee9f88SPeter Zijlstra #endif /* CONFIG_NUMA_BALANCING */ 1100cbee9f88SPeter Zijlstra 1101d7822b1eSMathieu Desnoyers #ifdef CONFIG_RSEQ 1102d7822b1eSMathieu Desnoyers struct rseq __user *rseq; 1103d7822b1eSMathieu Desnoyers u32 rseq_sig; 1104d7822b1eSMathieu Desnoyers /* 1105d7822b1eSMathieu Desnoyers * RmW on rseq_event_mask must be performed atomically 1106d7822b1eSMathieu Desnoyers * with respect to preemption. 1107d7822b1eSMathieu Desnoyers */ 1108d7822b1eSMathieu Desnoyers unsigned long rseq_event_mask; 1109d7822b1eSMathieu Desnoyers #endif 1110d7822b1eSMathieu Desnoyers 111172b252aeSMel Gorman struct tlbflush_unmap_batch tlb_ubc; 111272b252aeSMel Gorman 1113e56d0903SIngo Molnar struct rcu_head rcu; 1114b92ce558SJens Axboe 11155eca1c10SIngo Molnar /* Cache last used pipe for splice(): */ 1116b92ce558SJens Axboe struct pipe_inode_info *splice_pipe; 11175640f768SEric Dumazet 11185640f768SEric Dumazet struct page_frag task_frag; 11195640f768SEric Dumazet 1120ca74e92bSShailabh Nagar #ifdef CONFIG_TASK_DELAY_ACCT 1121ca74e92bSShailabh Nagar struct task_delay_info *delays; 1122ca74e92bSShailabh Nagar #endif 112347913d4eSIngo Molnar 1124f4f154fdSAkinobu Mita #ifdef CONFIG_FAULT_INJECTION 1125f4f154fdSAkinobu Mita int make_it_fail; 11269049f2f6SAkinobu Mita unsigned int fail_nth; 1127f4f154fdSAkinobu Mita #endif 11289d823e8fSWu Fengguang /* 11295eca1c10SIngo Molnar * When (nr_dirtied >= nr_dirtied_pause), it's time to call 11305eca1c10SIngo Molnar * balance_dirty_pages() for a dirty throttling pause: 11319d823e8fSWu Fengguang */ 11329d823e8fSWu Fengguang int nr_dirtied; 11339d823e8fSWu Fengguang int nr_dirtied_pause; 11345eca1c10SIngo Molnar /* Start of a write-and-pause period: */ 11355eca1c10SIngo Molnar unsigned long dirty_paused_when; 11369d823e8fSWu Fengguang 11379745512cSArjan van de Ven #ifdef CONFIG_LATENCYTOP 11389745512cSArjan van de Ven int latency_record_count; 11399745512cSArjan van de Ven struct latency_record latency_record[LT_SAVECOUNT]; 11409745512cSArjan van de Ven #endif 11416976675dSArjan van de Ven /* 11425eca1c10SIngo Molnar * Time slack values; these are used to round up poll() and 11436976675dSArjan van de Ven * select() etc timeout values. These are in nanoseconds. 11446976675dSArjan van de Ven */ 1145da8b44d5SJohn Stultz u64 timer_slack_ns; 1146da8b44d5SJohn Stultz u64 default_timer_slack_ns; 1147f8d570a4SDavid Miller 11480b24beccSAndrey Ryabinin #ifdef CONFIG_KASAN 11490b24beccSAndrey Ryabinin unsigned int kasan_depth; 11500b24beccSAndrey Ryabinin #endif 11515eca1c10SIngo Molnar 1152fb52607aSFrederic Weisbecker #ifdef CONFIG_FUNCTION_GRAPH_TRACER 11535eca1c10SIngo Molnar /* Index of current stored address in ret_stack: */ 1154f201ae23SFrederic Weisbecker int curr_ret_stack; 115539eb456dSSteven Rostedt (VMware) int curr_ret_depth; 11565eca1c10SIngo Molnar 11575eca1c10SIngo Molnar /* Stack of return addresses for return function tracing: */ 1158f201ae23SFrederic Weisbecker struct ftrace_ret_stack *ret_stack; 11595eca1c10SIngo Molnar 11605eca1c10SIngo Molnar /* Timestamp for last schedule: */ 11618aef2d28SSteven Rostedt unsigned long long ftrace_timestamp; 11625eca1c10SIngo Molnar 1163f201ae23SFrederic Weisbecker /* 1164f201ae23SFrederic Weisbecker * Number of functions that haven't been traced 11655eca1c10SIngo Molnar * because of depth overrun: 1166f201ae23SFrederic Weisbecker */ 1167f201ae23SFrederic Weisbecker atomic_t trace_overrun; 11685eca1c10SIngo Molnar 11695eca1c10SIngo Molnar /* Pause tracing: */ 1170380c4b14SFrederic Weisbecker atomic_t tracing_graph_pause; 1171f201ae23SFrederic Weisbecker #endif 11725eca1c10SIngo Molnar 1173ea4e2bc4SSteven Rostedt #ifdef CONFIG_TRACING 11745eca1c10SIngo Molnar /* State flags for use by tracers: */ 1175ea4e2bc4SSteven Rostedt unsigned long trace; 11765eca1c10SIngo Molnar 11775eca1c10SIngo Molnar /* Bitmask and counter of trace recursion: */ 1178261842b7SSteven Rostedt unsigned long trace_recursion; 1179261842b7SSteven Rostedt #endif /* CONFIG_TRACING */ 11805eca1c10SIngo Molnar 11815c9a8750SDmitry Vyukov #ifdef CONFIG_KCOV 11825eca1c10SIngo Molnar /* Coverage collection mode enabled for this task (0 if disabled): */ 11830ed557aaSMark Rutland unsigned int kcov_mode; 11845eca1c10SIngo Molnar 11855eca1c10SIngo Molnar /* Size of the kcov_area: */ 11865eca1c10SIngo Molnar unsigned int kcov_size; 11875eca1c10SIngo Molnar 11885eca1c10SIngo Molnar /* Buffer for coverage collection: */ 11895c9a8750SDmitry Vyukov void *kcov_area; 11905eca1c10SIngo Molnar 11915eca1c10SIngo Molnar /* KCOV descriptor wired with this task or NULL: */ 11925c9a8750SDmitry Vyukov struct kcov *kcov; 11935c9a8750SDmitry Vyukov #endif 11945eca1c10SIngo Molnar 11956f185c29SVladimir Davydov #ifdef CONFIG_MEMCG 1196626ebc41STejun Heo struct mem_cgroup *memcg_in_oom; 1197626ebc41STejun Heo gfp_t memcg_oom_gfp_mask; 1198626ebc41STejun Heo int memcg_oom_order; 1199b23afb93STejun Heo 12005eca1c10SIngo Molnar /* Number of pages to reclaim on returning to userland: */ 1201b23afb93STejun Heo unsigned int memcg_nr_pages_over_high; 1202d46eb14bSShakeel Butt 1203d46eb14bSShakeel Butt /* Used by memcontrol for targeted memcg charge: */ 1204d46eb14bSShakeel Butt struct mem_cgroup *active_memcg; 1205569b846dSKAMEZAWA Hiroyuki #endif 12065eca1c10SIngo Molnar 1207d09d8df3SJosef Bacik #ifdef CONFIG_BLK_CGROUP 1208d09d8df3SJosef Bacik struct request_queue *throttle_queue; 1209d09d8df3SJosef Bacik #endif 1210d09d8df3SJosef Bacik 12110326f5a9SSrikar Dronamraju #ifdef CONFIG_UPROBES 12120326f5a9SSrikar Dronamraju struct uprobe_task *utask; 12130326f5a9SSrikar Dronamraju #endif 1214cafe5635SKent Overstreet #if defined(CONFIG_BCACHE) || defined(CONFIG_BCACHE_MODULE) 1215cafe5635SKent Overstreet unsigned int sequential_io; 1216cafe5635SKent Overstreet unsigned int sequential_io_avg; 1217cafe5635SKent Overstreet #endif 12188eb23b9fSPeter Zijlstra #ifdef CONFIG_DEBUG_ATOMIC_SLEEP 12198eb23b9fSPeter Zijlstra unsigned long task_state_change; 12208eb23b9fSPeter Zijlstra #endif 12218bcbde54SDavid Hildenbrand int pagefault_disabled; 122203049269SMichal Hocko #ifdef CONFIG_MMU 122329c696e1SVladimir Davydov struct task_struct *oom_reaper_list; 122403049269SMichal Hocko #endif 1225ba14a194SAndy Lutomirski #ifdef CONFIG_VMAP_STACK 1226ba14a194SAndy Lutomirski struct vm_struct *stack_vm_area; 1227ba14a194SAndy Lutomirski #endif 122868f24b08SAndy Lutomirski #ifdef CONFIG_THREAD_INFO_IN_TASK 12295eca1c10SIngo Molnar /* A live task holds one reference: */ 1230f0b89d39SElena Reshetova refcount_t stack_refcount; 123168f24b08SAndy Lutomirski #endif 1232d83a7cb3SJosh Poimboeuf #ifdef CONFIG_LIVEPATCH 1233d83a7cb3SJosh Poimboeuf int patch_state; 1234d83a7cb3SJosh Poimboeuf #endif 1235e4e55b47STetsuo Handa #ifdef CONFIG_SECURITY 1236e4e55b47STetsuo Handa /* Used by LSM modules for access restriction: */ 1237e4e55b47STetsuo Handa void *security; 1238e4e55b47STetsuo Handa #endif 123929e48ce8SKees Cook 1240afaef01cSAlexander Popov #ifdef CONFIG_GCC_PLUGIN_STACKLEAK 1241afaef01cSAlexander Popov unsigned long lowest_stack; 1242c8d12627SAlexander Popov unsigned long prev_lowest_stack; 1243afaef01cSAlexander Popov #endif 1244afaef01cSAlexander Popov 124529e48ce8SKees Cook /* 124629e48ce8SKees Cook * New fields for task_struct should be added above here, so that 124729e48ce8SKees Cook * they are included in the randomized portion of task_struct. 124829e48ce8SKees Cook */ 124929e48ce8SKees Cook randomized_struct_fields_end 125029e48ce8SKees Cook 12515eca1c10SIngo Molnar /* CPU-specific state of this task: */ 12520c8c0f03SDave Hansen struct thread_struct thread; 12535eca1c10SIngo Molnar 12540c8c0f03SDave Hansen /* 12550c8c0f03SDave Hansen * WARNING: on x86, 'thread_struct' contains a variable-sized 12560c8c0f03SDave Hansen * structure. It *MUST* be at the end of 'task_struct'. 12570c8c0f03SDave Hansen * 12580c8c0f03SDave Hansen * Do not put anything below here! 12590c8c0f03SDave Hansen */ 12601da177e4SLinus Torvalds }; 12611da177e4SLinus Torvalds 1262e868171aSAlexey Dobriyan static inline struct pid *task_pid(struct task_struct *task) 126322c935f4SEric W. Biederman { 12642c470475SEric W. Biederman return task->thread_pid; 126522c935f4SEric W. Biederman } 126622c935f4SEric W. Biederman 12677af57294SPavel Emelyanov /* 12687af57294SPavel Emelyanov * the helpers to get the task's different pids as they are seen 12697af57294SPavel Emelyanov * from various namespaces 12707af57294SPavel Emelyanov * 12717af57294SPavel Emelyanov * task_xid_nr() : global id, i.e. the id seen from the init namespace; 127244c4e1b2SEric W. Biederman * task_xid_vnr() : virtual id, i.e. the id seen from the pid namespace of 127344c4e1b2SEric W. Biederman * current. 12747af57294SPavel Emelyanov * task_xid_nr_ns() : id seen from the ns specified; 12757af57294SPavel Emelyanov * 12767af57294SPavel Emelyanov * see also pid_nr() etc in include/linux/pid.h 12777af57294SPavel Emelyanov */ 12785eca1c10SIngo Molnar pid_t __task_pid_nr_ns(struct task_struct *task, enum pid_type type, struct pid_namespace *ns); 12797af57294SPavel Emelyanov 1280e868171aSAlexey Dobriyan static inline pid_t task_pid_nr(struct task_struct *tsk) 12817af57294SPavel Emelyanov { 12827af57294SPavel Emelyanov return tsk->pid; 12837af57294SPavel Emelyanov } 12847af57294SPavel Emelyanov 12855eca1c10SIngo Molnar static inline pid_t task_pid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns) 128652ee2dfdSOleg Nesterov { 128752ee2dfdSOleg Nesterov return __task_pid_nr_ns(tsk, PIDTYPE_PID, ns); 128852ee2dfdSOleg Nesterov } 12897af57294SPavel Emelyanov 12907af57294SPavel Emelyanov static inline pid_t task_pid_vnr(struct task_struct *tsk) 12917af57294SPavel Emelyanov { 129252ee2dfdSOleg Nesterov return __task_pid_nr_ns(tsk, PIDTYPE_PID, NULL); 12937af57294SPavel Emelyanov } 12947af57294SPavel Emelyanov 12957af57294SPavel Emelyanov 1296e868171aSAlexey Dobriyan static inline pid_t task_tgid_nr(struct task_struct *tsk) 12977af57294SPavel Emelyanov { 12987af57294SPavel Emelyanov return tsk->tgid; 12997af57294SPavel Emelyanov } 13007af57294SPavel Emelyanov 13015eca1c10SIngo Molnar /** 13025eca1c10SIngo Molnar * pid_alive - check that a task structure is not stale 13035eca1c10SIngo Molnar * @p: Task structure to be checked. 13045eca1c10SIngo Molnar * 13055eca1c10SIngo Molnar * Test if a process is not yet dead (at most zombie state) 13065eca1c10SIngo Molnar * If pid_alive fails, then pointers within the task structure 13075eca1c10SIngo Molnar * can be stale and must not be dereferenced. 13085eca1c10SIngo Molnar * 13095eca1c10SIngo Molnar * Return: 1 if the process is alive. 0 otherwise. 13105eca1c10SIngo Molnar */ 13115eca1c10SIngo Molnar static inline int pid_alive(const struct task_struct *p) 13125eca1c10SIngo Molnar { 13132c470475SEric W. Biederman return p->thread_pid != NULL; 13145eca1c10SIngo Molnar } 13157af57294SPavel Emelyanov 13165eca1c10SIngo Molnar static inline pid_t task_pgrp_nr_ns(struct task_struct *tsk, struct pid_namespace *ns) 13177af57294SPavel Emelyanov { 131852ee2dfdSOleg Nesterov return __task_pid_nr_ns(tsk, PIDTYPE_PGID, ns); 13197af57294SPavel Emelyanov } 13207af57294SPavel Emelyanov 13217af57294SPavel Emelyanov static inline pid_t task_pgrp_vnr(struct task_struct *tsk) 13227af57294SPavel Emelyanov { 132352ee2dfdSOleg Nesterov return __task_pid_nr_ns(tsk, PIDTYPE_PGID, NULL); 13247af57294SPavel Emelyanov } 13257af57294SPavel Emelyanov 13267af57294SPavel Emelyanov 13275eca1c10SIngo Molnar static inline pid_t task_session_nr_ns(struct task_struct *tsk, struct pid_namespace *ns) 13287af57294SPavel Emelyanov { 132952ee2dfdSOleg Nesterov return __task_pid_nr_ns(tsk, PIDTYPE_SID, ns); 13307af57294SPavel Emelyanov } 13317af57294SPavel Emelyanov 13327af57294SPavel Emelyanov static inline pid_t task_session_vnr(struct task_struct *tsk) 13337af57294SPavel Emelyanov { 133452ee2dfdSOleg Nesterov return __task_pid_nr_ns(tsk, PIDTYPE_SID, NULL); 13357af57294SPavel Emelyanov } 13367af57294SPavel Emelyanov 1337dd1c1f2fSOleg Nesterov static inline pid_t task_tgid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns) 1338dd1c1f2fSOleg Nesterov { 13396883f81aSEric W. Biederman return __task_pid_nr_ns(tsk, PIDTYPE_TGID, ns); 1340dd1c1f2fSOleg Nesterov } 1341dd1c1f2fSOleg Nesterov 1342dd1c1f2fSOleg Nesterov static inline pid_t task_tgid_vnr(struct task_struct *tsk) 1343dd1c1f2fSOleg Nesterov { 13446883f81aSEric W. Biederman return __task_pid_nr_ns(tsk, PIDTYPE_TGID, NULL); 1345dd1c1f2fSOleg Nesterov } 1346dd1c1f2fSOleg Nesterov 1347dd1c1f2fSOleg Nesterov static inline pid_t task_ppid_nr_ns(const struct task_struct *tsk, struct pid_namespace *ns) 1348dd1c1f2fSOleg Nesterov { 1349dd1c1f2fSOleg Nesterov pid_t pid = 0; 1350dd1c1f2fSOleg Nesterov 1351dd1c1f2fSOleg Nesterov rcu_read_lock(); 1352dd1c1f2fSOleg Nesterov if (pid_alive(tsk)) 1353dd1c1f2fSOleg Nesterov pid = task_tgid_nr_ns(rcu_dereference(tsk->real_parent), ns); 1354dd1c1f2fSOleg Nesterov rcu_read_unlock(); 1355dd1c1f2fSOleg Nesterov 1356dd1c1f2fSOleg Nesterov return pid; 1357dd1c1f2fSOleg Nesterov } 1358dd1c1f2fSOleg Nesterov 1359dd1c1f2fSOleg Nesterov static inline pid_t task_ppid_nr(const struct task_struct *tsk) 1360dd1c1f2fSOleg Nesterov { 1361dd1c1f2fSOleg Nesterov return task_ppid_nr_ns(tsk, &init_pid_ns); 1362dd1c1f2fSOleg Nesterov } 1363dd1c1f2fSOleg Nesterov 13645eca1c10SIngo Molnar /* Obsolete, do not use: */ 13651b0f7ffdSOleg Nesterov static inline pid_t task_pgrp_nr(struct task_struct *tsk) 13661b0f7ffdSOleg Nesterov { 13671b0f7ffdSOleg Nesterov return task_pgrp_nr_ns(tsk, &init_pid_ns); 13681b0f7ffdSOleg Nesterov } 13697af57294SPavel Emelyanov 137006eb6184SPeter Zijlstra #define TASK_REPORT_IDLE (TASK_REPORT + 1) 137106eb6184SPeter Zijlstra #define TASK_REPORT_MAX (TASK_REPORT_IDLE << 1) 137206eb6184SPeter Zijlstra 13731d48b080SPeter Zijlstra static inline unsigned int task_state_index(struct task_struct *tsk) 137420435d84SXie XiuQi { 13751593baabSPeter Zijlstra unsigned int tsk_state = READ_ONCE(tsk->state); 13761593baabSPeter Zijlstra unsigned int state = (tsk_state | tsk->exit_state) & TASK_REPORT; 137720435d84SXie XiuQi 137806eb6184SPeter Zijlstra BUILD_BUG_ON_NOT_POWER_OF_2(TASK_REPORT_MAX); 137906eb6184SPeter Zijlstra 138006eb6184SPeter Zijlstra if (tsk_state == TASK_IDLE) 138106eb6184SPeter Zijlstra state = TASK_REPORT_IDLE; 138206eb6184SPeter Zijlstra 13831593baabSPeter Zijlstra return fls(state); 13841593baabSPeter Zijlstra } 138520435d84SXie XiuQi 13861d48b080SPeter Zijlstra static inline char task_index_to_char(unsigned int state) 13871593baabSPeter Zijlstra { 13888ef9925bSPeter Zijlstra static const char state_char[] = "RSDTtXZPI"; 13891593baabSPeter Zijlstra 139006eb6184SPeter Zijlstra BUILD_BUG_ON(1 + ilog2(TASK_REPORT_MAX) != sizeof(state_char) - 1); 13911593baabSPeter Zijlstra 13921593baabSPeter Zijlstra return state_char[state]; 13931593baabSPeter Zijlstra } 13941593baabSPeter Zijlstra 13951593baabSPeter Zijlstra static inline char task_state_to_char(struct task_struct *tsk) 13961593baabSPeter Zijlstra { 13971d48b080SPeter Zijlstra return task_index_to_char(task_state_index(tsk)); 139820435d84SXie XiuQi } 139920435d84SXie XiuQi 14001da177e4SLinus Torvalds /** 1401570f5241SSergey Senozhatsky * is_global_init - check if a task structure is init. Since init 1402570f5241SSergey Senozhatsky * is free to have sub-threads we need to check tgid. 14033260259fSHenne * @tsk: Task structure to be checked. 14043260259fSHenne * 14053260259fSHenne * Check if a task structure is the first user space task the kernel created. 1406e69f6186SYacine Belkadi * 1407e69f6186SYacine Belkadi * Return: 1 if the task structure is init. 0 otherwise. 1408f400e198SSukadev Bhattiprolu */ 1409e868171aSAlexey Dobriyan static inline int is_global_init(struct task_struct *tsk) 1410b461cc03SPavel Emelyanov { 1411570f5241SSergey Senozhatsky return task_tgid_nr(tsk) == 1; 1412b461cc03SPavel Emelyanov } 1413b460cbc5SSerge E. Hallyn 14149ec52099SCedric Le Goater extern struct pid *cad_pid; 14159ec52099SCedric Le Goater 14161da177e4SLinus Torvalds /* 14171da177e4SLinus Torvalds * Per process flags 14181da177e4SLinus Torvalds */ 1419c1de45caSPeter Zijlstra #define PF_IDLE 0x00000002 /* I am an IDLE thread */ 14205eca1c10SIngo Molnar #define PF_EXITING 0x00000004 /* Getting shut down */ 14215eca1c10SIngo Molnar #define PF_EXITPIDONE 0x00000008 /* PI exit done on shut down */ 142294886b84SLaurent Vivier #define PF_VCPU 0x00000010 /* I'm a virtual CPU */ 142321aa9af0STejun Heo #define PF_WQ_WORKER 0x00000020 /* I'm a workqueue worker */ 14245eca1c10SIngo Molnar #define PF_FORKNOEXEC 0x00000040 /* Forked but didn't exec */ 14255eca1c10SIngo Molnar #define PF_MCE_PROCESS 0x00000080 /* Process policy on mce errors */ 14265eca1c10SIngo Molnar #define PF_SUPERPRIV 0x00000100 /* Used super-user privileges */ 14275eca1c10SIngo Molnar #define PF_DUMPCORE 0x00000200 /* Dumped core */ 14285eca1c10SIngo Molnar #define PF_SIGNALED 0x00000400 /* Killed by a signal */ 14291da177e4SLinus Torvalds #define PF_MEMALLOC 0x00000800 /* Allocating memory */ 14305eca1c10SIngo Molnar #define PF_NPROC_EXCEEDED 0x00001000 /* set_user() noticed that RLIMIT_NPROC was exceeded */ 14315eca1c10SIngo Molnar #define PF_USED_MATH 0x00002000 /* If unset the fpu must be initialized before use */ 14325eca1c10SIngo Molnar #define PF_USED_ASYNC 0x00004000 /* Used async_schedule*(), used by module init */ 14335eca1c10SIngo Molnar #define PF_NOFREEZE 0x00008000 /* This thread should not be frozen */ 14345eca1c10SIngo Molnar #define PF_FROZEN 0x00010000 /* Frozen for system suspend */ 14357dea19f9SMichal Hocko #define PF_KSWAPD 0x00020000 /* I am kswapd */ 14367dea19f9SMichal Hocko #define PF_MEMALLOC_NOFS 0x00040000 /* All allocation requests will inherit GFP_NOFS */ 14377dea19f9SMichal Hocko #define PF_MEMALLOC_NOIO 0x00080000 /* All allocation requests will inherit GFP_NOIO */ 14381da177e4SLinus Torvalds #define PF_LESS_THROTTLE 0x00100000 /* Throttle me less: I clean memory */ 1439246bb0b1SOleg Nesterov #define PF_KTHREAD 0x00200000 /* I am a kernel thread */ 14405eca1c10SIngo Molnar #define PF_RANDOMIZE 0x00400000 /* Randomize virtual address space */ 1441b31dc66aSJens Axboe #define PF_SWAPWRITE 0x00800000 /* Allowed to write to swap */ 1442eb414681SJohannes Weiner #define PF_MEMSTALL 0x01000000 /* Stalled due to lack of memory */ 144373ab1cb2STaehee Yoo #define PF_UMH 0x02000000 /* I'm an Usermodehelper process */ 14443bd37062SSebastian Andrzej Siewior #define PF_NO_SETAFFINITY 0x04000000 /* Userland is not allowed to meddle with cpus_mask */ 14454db96cf0SAndi Kleen #define PF_MCE_EARLY 0x08000000 /* Early kill for mce process policy */ 1446d7fefcc8SAneesh Kumar K.V #define PF_MEMALLOC_NOCMA 0x10000000 /* All allocation request will have _GFP_MOVABLE cleared */ 144758a69cb4STejun Heo #define PF_FREEZER_SKIP 0x40000000 /* Freezer should not count it as freezable */ 14485eca1c10SIngo Molnar #define PF_SUSPEND_TASK 0x80000000 /* This thread called freeze_processes() and should not be frozen */ 14491da177e4SLinus Torvalds 14501da177e4SLinus Torvalds /* 14511da177e4SLinus Torvalds * Only the _current_ task can read/write to tsk->flags, but other 14521da177e4SLinus Torvalds * tasks can access tsk->flags in readonly mode for example 14531da177e4SLinus Torvalds * with tsk_used_math (like during threaded core dumping). 14541da177e4SLinus Torvalds * There is however an exception to this rule during ptrace 14551da177e4SLinus Torvalds * or during fork: the ptracer task is allowed to write to the 14561da177e4SLinus Torvalds * child->flags of its traced child (same goes for fork, the parent 14571da177e4SLinus Torvalds * can write to the child->flags), because we're guaranteed the 14581da177e4SLinus Torvalds * child is not running and in turn not changing child->flags 14591da177e4SLinus Torvalds * at the same time the parent does it. 14601da177e4SLinus Torvalds */ 14611da177e4SLinus Torvalds #define clear_stopped_child_used_math(child) do { (child)->flags &= ~PF_USED_MATH; } while (0) 14621da177e4SLinus Torvalds #define set_stopped_child_used_math(child) do { (child)->flags |= PF_USED_MATH; } while (0) 14631da177e4SLinus Torvalds #define clear_used_math() clear_stopped_child_used_math(current) 14641da177e4SLinus Torvalds #define set_used_math() set_stopped_child_used_math(current) 14655eca1c10SIngo Molnar 14661da177e4SLinus Torvalds #define conditional_stopped_child_used_math(condition, child) \ 14671da177e4SLinus Torvalds do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= (condition) ? PF_USED_MATH : 0; } while (0) 14685eca1c10SIngo Molnar 14695eca1c10SIngo Molnar #define conditional_used_math(condition) conditional_stopped_child_used_math(condition, current) 14705eca1c10SIngo Molnar 14711da177e4SLinus Torvalds #define copy_to_stopped_child_used_math(child) \ 14721da177e4SLinus Torvalds do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= current->flags & PF_USED_MATH; } while (0) 14735eca1c10SIngo Molnar 14741da177e4SLinus Torvalds /* NOTE: this will return 0 or PF_USED_MATH, it will never return 1 */ 14751da177e4SLinus Torvalds #define tsk_used_math(p) ((p)->flags & PF_USED_MATH) 14761da177e4SLinus Torvalds #define used_math() tsk_used_math(current) 14771da177e4SLinus Torvalds 147862ec05ddSThomas Gleixner static inline bool is_percpu_thread(void) 147962ec05ddSThomas Gleixner { 148062ec05ddSThomas Gleixner #ifdef CONFIG_SMP 148162ec05ddSThomas Gleixner return (current->flags & PF_NO_SETAFFINITY) && 148262ec05ddSThomas Gleixner (current->nr_cpus_allowed == 1); 148362ec05ddSThomas Gleixner #else 148462ec05ddSThomas Gleixner return true; 148562ec05ddSThomas Gleixner #endif 148662ec05ddSThomas Gleixner } 148762ec05ddSThomas Gleixner 14881d4457f9SKees Cook /* Per-process atomic flags. */ 1489a2b86f77SZefan Li #define PFA_NO_NEW_PRIVS 0 /* May not gain new privileges. */ 14902ad654bcSZefan Li #define PFA_SPREAD_PAGE 1 /* Spread page cache over cpuset */ 14912ad654bcSZefan Li #define PFA_SPREAD_SLAB 2 /* Spread some slab caches over cpuset */ 1492356e4bffSThomas Gleixner #define PFA_SPEC_SSB_DISABLE 3 /* Speculative Store Bypass disabled */ 1493356e4bffSThomas Gleixner #define PFA_SPEC_SSB_FORCE_DISABLE 4 /* Speculative Store Bypass force disabled*/ 14949137bb27SThomas Gleixner #define PFA_SPEC_IB_DISABLE 5 /* Indirect branch speculation restricted */ 14959137bb27SThomas Gleixner #define PFA_SPEC_IB_FORCE_DISABLE 6 /* Indirect branch speculation permanently restricted */ 149671368af9SWaiman Long #define PFA_SPEC_SSB_NOEXEC 7 /* Speculative Store Bypass clear on execve() */ 14971d4457f9SKees Cook 1498e0e5070bSZefan Li #define TASK_PFA_TEST(name, func) \ 1499e0e5070bSZefan Li static inline bool task_##func(struct task_struct *p) \ 1500e0e5070bSZefan Li { return test_bit(PFA_##name, &p->atomic_flags); } 15015eca1c10SIngo Molnar 1502e0e5070bSZefan Li #define TASK_PFA_SET(name, func) \ 1503e0e5070bSZefan Li static inline void task_set_##func(struct task_struct *p) \ 1504e0e5070bSZefan Li { set_bit(PFA_##name, &p->atomic_flags); } 15055eca1c10SIngo Molnar 1506e0e5070bSZefan Li #define TASK_PFA_CLEAR(name, func) \ 1507e0e5070bSZefan Li static inline void task_clear_##func(struct task_struct *p) \ 1508e0e5070bSZefan Li { clear_bit(PFA_##name, &p->atomic_flags); } 15091d4457f9SKees Cook 1510e0e5070bSZefan Li TASK_PFA_TEST(NO_NEW_PRIVS, no_new_privs) 1511e0e5070bSZefan Li TASK_PFA_SET(NO_NEW_PRIVS, no_new_privs) 15121d4457f9SKees Cook 15132ad654bcSZefan Li TASK_PFA_TEST(SPREAD_PAGE, spread_page) 15142ad654bcSZefan Li TASK_PFA_SET(SPREAD_PAGE, spread_page) 15152ad654bcSZefan Li TASK_PFA_CLEAR(SPREAD_PAGE, spread_page) 15162ad654bcSZefan Li 15172ad654bcSZefan Li TASK_PFA_TEST(SPREAD_SLAB, spread_slab) 15182ad654bcSZefan Li TASK_PFA_SET(SPREAD_SLAB, spread_slab) 15192ad654bcSZefan Li TASK_PFA_CLEAR(SPREAD_SLAB, spread_slab) 1520544b2c91STejun Heo 1521356e4bffSThomas Gleixner TASK_PFA_TEST(SPEC_SSB_DISABLE, spec_ssb_disable) 1522356e4bffSThomas Gleixner TASK_PFA_SET(SPEC_SSB_DISABLE, spec_ssb_disable) 1523356e4bffSThomas Gleixner TASK_PFA_CLEAR(SPEC_SSB_DISABLE, spec_ssb_disable) 1524356e4bffSThomas Gleixner 152571368af9SWaiman Long TASK_PFA_TEST(SPEC_SSB_NOEXEC, spec_ssb_noexec) 152671368af9SWaiman Long TASK_PFA_SET(SPEC_SSB_NOEXEC, spec_ssb_noexec) 152771368af9SWaiman Long TASK_PFA_CLEAR(SPEC_SSB_NOEXEC, spec_ssb_noexec) 152871368af9SWaiman Long 1529356e4bffSThomas Gleixner TASK_PFA_TEST(SPEC_SSB_FORCE_DISABLE, spec_ssb_force_disable) 1530356e4bffSThomas Gleixner TASK_PFA_SET(SPEC_SSB_FORCE_DISABLE, spec_ssb_force_disable) 1531356e4bffSThomas Gleixner 15329137bb27SThomas Gleixner TASK_PFA_TEST(SPEC_IB_DISABLE, spec_ib_disable) 15339137bb27SThomas Gleixner TASK_PFA_SET(SPEC_IB_DISABLE, spec_ib_disable) 15349137bb27SThomas Gleixner TASK_PFA_CLEAR(SPEC_IB_DISABLE, spec_ib_disable) 15359137bb27SThomas Gleixner 15369137bb27SThomas Gleixner TASK_PFA_TEST(SPEC_IB_FORCE_DISABLE, spec_ib_force_disable) 15379137bb27SThomas Gleixner TASK_PFA_SET(SPEC_IB_FORCE_DISABLE, spec_ib_force_disable) 15389137bb27SThomas Gleixner 15395eca1c10SIngo Molnar static inline void 1540717a94b5SNeilBrown current_restore_flags(unsigned long orig_flags, unsigned long flags) 1541907aed48SMel Gorman { 1542717a94b5SNeilBrown current->flags &= ~flags; 1543717a94b5SNeilBrown current->flags |= orig_flags & flags; 1544907aed48SMel Gorman } 1545907aed48SMel Gorman 15465eca1c10SIngo Molnar extern int cpuset_cpumask_can_shrink(const struct cpumask *cur, const struct cpumask *trial); 15475eca1c10SIngo Molnar extern int task_can_attach(struct task_struct *p, const struct cpumask *cs_cpus_allowed); 15481da177e4SLinus Torvalds #ifdef CONFIG_SMP 15495eca1c10SIngo Molnar extern void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask); 15505eca1c10SIngo Molnar extern int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask); 15511da177e4SLinus Torvalds #else 15525eca1c10SIngo Molnar static inline void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) 15531e1b6c51SKOSAKI Motohiro { 15541e1b6c51SKOSAKI Motohiro } 15555eca1c10SIngo Molnar static inline int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask) 15561da177e4SLinus Torvalds { 155796f874e2SRusty Russell if (!cpumask_test_cpu(0, new_mask)) 15581da177e4SLinus Torvalds return -EINVAL; 15591da177e4SLinus Torvalds return 0; 15601da177e4SLinus Torvalds } 15611da177e4SLinus Torvalds #endif 1562e0ad9556SRusty Russell 15636d0d2878SChristian Borntraeger #ifndef cpu_relax_yield 15646d0d2878SChristian Borntraeger #define cpu_relax_yield() cpu_relax() 15656d0d2878SChristian Borntraeger #endif 15666d0d2878SChristian Borntraeger 1567fa93384fSDan Carpenter extern int yield_to(struct task_struct *p, bool preempt); 156836c8b586SIngo Molnar extern void set_user_nice(struct task_struct *p, long nice); 156936c8b586SIngo Molnar extern int task_prio(const struct task_struct *p); 15705eca1c10SIngo Molnar 1571d0ea0268SDongsheng Yang /** 1572d0ea0268SDongsheng Yang * task_nice - return the nice value of a given task. 1573d0ea0268SDongsheng Yang * @p: the task in question. 1574d0ea0268SDongsheng Yang * 1575d0ea0268SDongsheng Yang * Return: The nice value [ -20 ... 0 ... 19 ]. 1576d0ea0268SDongsheng Yang */ 1577d0ea0268SDongsheng Yang static inline int task_nice(const struct task_struct *p) 1578d0ea0268SDongsheng Yang { 1579d0ea0268SDongsheng Yang return PRIO_TO_NICE((p)->static_prio); 1580d0ea0268SDongsheng Yang } 15815eca1c10SIngo Molnar 158236c8b586SIngo Molnar extern int can_nice(const struct task_struct *p, const int nice); 158336c8b586SIngo Molnar extern int task_curr(const struct task_struct *p); 15841da177e4SLinus Torvalds extern int idle_cpu(int cpu); 1585943d355dSRohit Jain extern int available_idle_cpu(int cpu); 15865eca1c10SIngo Molnar extern int sched_setscheduler(struct task_struct *, int, const struct sched_param *); 15875eca1c10SIngo Molnar extern int sched_setscheduler_nocheck(struct task_struct *, int, const struct sched_param *); 15885eca1c10SIngo Molnar extern int sched_setattr(struct task_struct *, const struct sched_attr *); 1589794a56ebSJuri Lelli extern int sched_setattr_nocheck(struct task_struct *, const struct sched_attr *); 159036c8b586SIngo Molnar extern struct task_struct *idle_task(int cpu); 15915eca1c10SIngo Molnar 1592c4f30608SPaul E. McKenney /** 1593c4f30608SPaul E. McKenney * is_idle_task - is the specified task an idle task? 1594fa757281SRandy Dunlap * @p: the task in question. 1595e69f6186SYacine Belkadi * 1596e69f6186SYacine Belkadi * Return: 1 if @p is an idle task. 0 otherwise. 1597c4f30608SPaul E. McKenney */ 15987061ca3bSPaul E. McKenney static inline bool is_idle_task(const struct task_struct *p) 1599c4f30608SPaul E. McKenney { 1600c1de45caSPeter Zijlstra return !!(p->flags & PF_IDLE); 1601c4f30608SPaul E. McKenney } 16025eca1c10SIngo Molnar 160336c8b586SIngo Molnar extern struct task_struct *curr_task(int cpu); 1604a458ae2eSPeter Zijlstra extern void ia64_set_curr_task(int cpu, struct task_struct *p); 16051da177e4SLinus Torvalds 16061da177e4SLinus Torvalds void yield(void); 16071da177e4SLinus Torvalds 16081da177e4SLinus Torvalds union thread_union { 16090500871fSDavid Howells #ifndef CONFIG_ARCH_TASK_STRUCT_ON_STACK 16100500871fSDavid Howells struct task_struct task; 16110500871fSDavid Howells #endif 1612c65eacbeSAndy Lutomirski #ifndef CONFIG_THREAD_INFO_IN_TASK 16131da177e4SLinus Torvalds struct thread_info thread_info; 1614c65eacbeSAndy Lutomirski #endif 16151da177e4SLinus Torvalds unsigned long stack[THREAD_SIZE/sizeof(long)]; 16161da177e4SLinus Torvalds }; 16171da177e4SLinus Torvalds 16180500871fSDavid Howells #ifndef CONFIG_THREAD_INFO_IN_TASK 16190500871fSDavid Howells extern struct thread_info init_thread_info; 16200500871fSDavid Howells #endif 16210500871fSDavid Howells 16220500871fSDavid Howells extern unsigned long init_stack[THREAD_SIZE / sizeof(unsigned long)]; 16230500871fSDavid Howells 1624f3ac6067SIngo Molnar #ifdef CONFIG_THREAD_INFO_IN_TASK 1625f3ac6067SIngo Molnar static inline struct thread_info *task_thread_info(struct task_struct *task) 1626f3ac6067SIngo Molnar { 1627f3ac6067SIngo Molnar return &task->thread_info; 1628f3ac6067SIngo Molnar } 1629f3ac6067SIngo Molnar #elif !defined(__HAVE_THREAD_FUNCTIONS) 1630f3ac6067SIngo Molnar # define task_thread_info(task) ((struct thread_info *)(task)->stack) 1631f3ac6067SIngo Molnar #endif 1632f3ac6067SIngo Molnar 1633198fe21bSPavel Emelyanov /* 1634198fe21bSPavel Emelyanov * find a task by one of its numerical ids 1635198fe21bSPavel Emelyanov * 1636198fe21bSPavel Emelyanov * find_task_by_pid_ns(): 1637198fe21bSPavel Emelyanov * finds a task by its pid in the specified namespace 1638228ebcbeSPavel Emelyanov * find_task_by_vpid(): 1639228ebcbeSPavel Emelyanov * finds a task by its virtual pid 1640198fe21bSPavel Emelyanov * 1641e49859e7SPavel Emelyanov * see also find_vpid() etc in include/linux/pid.h 1642198fe21bSPavel Emelyanov */ 1643198fe21bSPavel Emelyanov 1644228ebcbeSPavel Emelyanov extern struct task_struct *find_task_by_vpid(pid_t nr); 16455eca1c10SIngo Molnar extern struct task_struct *find_task_by_pid_ns(pid_t nr, struct pid_namespace *ns); 1646198fe21bSPavel Emelyanov 16472ee08260SMike Rapoport /* 16482ee08260SMike Rapoport * find a task by its virtual pid and get the task struct 16492ee08260SMike Rapoport */ 16502ee08260SMike Rapoport extern struct task_struct *find_get_task_by_vpid(pid_t nr); 16512ee08260SMike Rapoport 1652b3c97528SHarvey Harrison extern int wake_up_state(struct task_struct *tsk, unsigned int state); 1653b3c97528SHarvey Harrison extern int wake_up_process(struct task_struct *tsk); 16543e51e3edSSamir Bellabes extern void wake_up_new_task(struct task_struct *tsk); 16555eca1c10SIngo Molnar 16561da177e4SLinus Torvalds #ifdef CONFIG_SMP 16571da177e4SLinus Torvalds extern void kick_process(struct task_struct *tsk); 16581da177e4SLinus Torvalds #else 16591da177e4SLinus Torvalds static inline void kick_process(struct task_struct *tsk) { } 16601da177e4SLinus Torvalds #endif 16611da177e4SLinus Torvalds 166282b89778SAdrian Hunter extern void __set_task_comm(struct task_struct *tsk, const char *from, bool exec); 16635eca1c10SIngo Molnar 166482b89778SAdrian Hunter static inline void set_task_comm(struct task_struct *tsk, const char *from) 166582b89778SAdrian Hunter { 166682b89778SAdrian Hunter __set_task_comm(tsk, from, false); 166782b89778SAdrian Hunter } 16685eca1c10SIngo Molnar 16693756f640SArnd Bergmann extern char *__get_task_comm(char *to, size_t len, struct task_struct *tsk); 16703756f640SArnd Bergmann #define get_task_comm(buf, tsk) ({ \ 16713756f640SArnd Bergmann BUILD_BUG_ON(sizeof(buf) != TASK_COMM_LEN); \ 16723756f640SArnd Bergmann __get_task_comm(buf, sizeof(buf), tsk); \ 16733756f640SArnd Bergmann }) 16741da177e4SLinus Torvalds 16751da177e4SLinus Torvalds #ifdef CONFIG_SMP 1676317f3941SPeter Zijlstra void scheduler_ipi(void); 167785ba2d86SRoland McGrath extern unsigned long wait_task_inactive(struct task_struct *, long match_state); 16781da177e4SLinus Torvalds #else 1679184748ccSPeter Zijlstra static inline void scheduler_ipi(void) { } 16805eca1c10SIngo Molnar static inline unsigned long wait_task_inactive(struct task_struct *p, long match_state) 168185ba2d86SRoland McGrath { 168285ba2d86SRoland McGrath return 1; 168385ba2d86SRoland McGrath } 16841da177e4SLinus Torvalds #endif 16851da177e4SLinus Torvalds 16865eca1c10SIngo Molnar /* 16875eca1c10SIngo Molnar * Set thread flags in other task's structures. 16885eca1c10SIngo Molnar * See asm/thread_info.h for TIF_xxxx flags available: 16891da177e4SLinus Torvalds */ 16901da177e4SLinus Torvalds static inline void set_tsk_thread_flag(struct task_struct *tsk, int flag) 16911da177e4SLinus Torvalds { 1692a1261f54SAl Viro set_ti_thread_flag(task_thread_info(tsk), flag); 16931da177e4SLinus Torvalds } 16941da177e4SLinus Torvalds 16951da177e4SLinus Torvalds static inline void clear_tsk_thread_flag(struct task_struct *tsk, int flag) 16961da177e4SLinus Torvalds { 1697a1261f54SAl Viro clear_ti_thread_flag(task_thread_info(tsk), flag); 16981da177e4SLinus Torvalds } 16991da177e4SLinus Torvalds 170093ee37c2SDave Martin static inline void update_tsk_thread_flag(struct task_struct *tsk, int flag, 170193ee37c2SDave Martin bool value) 170293ee37c2SDave Martin { 170393ee37c2SDave Martin update_ti_thread_flag(task_thread_info(tsk), flag, value); 170493ee37c2SDave Martin } 170593ee37c2SDave Martin 17061da177e4SLinus Torvalds static inline int test_and_set_tsk_thread_flag(struct task_struct *tsk, int flag) 17071da177e4SLinus Torvalds { 1708a1261f54SAl Viro return test_and_set_ti_thread_flag(task_thread_info(tsk), flag); 17091da177e4SLinus Torvalds } 17101da177e4SLinus Torvalds 17111da177e4SLinus Torvalds static inline int test_and_clear_tsk_thread_flag(struct task_struct *tsk, int flag) 17121da177e4SLinus Torvalds { 1713a1261f54SAl Viro return test_and_clear_ti_thread_flag(task_thread_info(tsk), flag); 17141da177e4SLinus Torvalds } 17151da177e4SLinus Torvalds 17161da177e4SLinus Torvalds static inline int test_tsk_thread_flag(struct task_struct *tsk, int flag) 17171da177e4SLinus Torvalds { 1718a1261f54SAl Viro return test_ti_thread_flag(task_thread_info(tsk), flag); 17191da177e4SLinus Torvalds } 17201da177e4SLinus Torvalds 17211da177e4SLinus Torvalds static inline void set_tsk_need_resched(struct task_struct *tsk) 17221da177e4SLinus Torvalds { 17231da177e4SLinus Torvalds set_tsk_thread_flag(tsk,TIF_NEED_RESCHED); 17241da177e4SLinus Torvalds } 17251da177e4SLinus Torvalds 17261da177e4SLinus Torvalds static inline void clear_tsk_need_resched(struct task_struct *tsk) 17271da177e4SLinus Torvalds { 17281da177e4SLinus Torvalds clear_tsk_thread_flag(tsk,TIF_NEED_RESCHED); 17291da177e4SLinus Torvalds } 17301da177e4SLinus Torvalds 17318ae121acSGregory Haskins static inline int test_tsk_need_resched(struct task_struct *tsk) 17328ae121acSGregory Haskins { 17338ae121acSGregory Haskins return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED)); 17348ae121acSGregory Haskins } 17358ae121acSGregory Haskins 17361da177e4SLinus Torvalds /* 17371da177e4SLinus Torvalds * cond_resched() and cond_resched_lock(): latency reduction via 17381da177e4SLinus Torvalds * explicit rescheduling in places that are safe. The return 17391da177e4SLinus Torvalds * value indicates whether a reschedule was done in fact. 17401da177e4SLinus Torvalds * cond_resched_lock() will drop the spinlock before scheduling, 17411da177e4SLinus Torvalds */ 174235a773a0SPeter Zijlstra #ifndef CONFIG_PREEMPT 1743c3921ab7SLinus Torvalds extern int _cond_resched(void); 174435a773a0SPeter Zijlstra #else 174535a773a0SPeter Zijlstra static inline int _cond_resched(void) { return 0; } 174635a773a0SPeter Zijlstra #endif 17476f80bd98SFrederic Weisbecker 1748613afbf8SFrederic Weisbecker #define cond_resched() ({ \ 17493427445aSPeter Zijlstra ___might_sleep(__FILE__, __LINE__, 0); \ 1750613afbf8SFrederic Weisbecker _cond_resched(); \ 1751613afbf8SFrederic Weisbecker }) 17526f80bd98SFrederic Weisbecker 1753613afbf8SFrederic Weisbecker extern int __cond_resched_lock(spinlock_t *lock); 1754613afbf8SFrederic Weisbecker 1755613afbf8SFrederic Weisbecker #define cond_resched_lock(lock) ({ \ 17563427445aSPeter Zijlstra ___might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET);\ 1757613afbf8SFrederic Weisbecker __cond_resched_lock(lock); \ 1758613afbf8SFrederic Weisbecker }) 1759613afbf8SFrederic Weisbecker 1760f6f3c437SSimon Horman static inline void cond_resched_rcu(void) 1761f6f3c437SSimon Horman { 1762f6f3c437SSimon Horman #if defined(CONFIG_DEBUG_ATOMIC_SLEEP) || !defined(CONFIG_PREEMPT_RCU) 1763f6f3c437SSimon Horman rcu_read_unlock(); 1764f6f3c437SSimon Horman cond_resched(); 1765f6f3c437SSimon Horman rcu_read_lock(); 1766f6f3c437SSimon Horman #endif 1767f6f3c437SSimon Horman } 1768f6f3c437SSimon Horman 17691da177e4SLinus Torvalds /* 17701da177e4SLinus Torvalds * Does a critical section need to be broken due to another 177195c354feSNick Piggin * task waiting?: (technically does not depend on CONFIG_PREEMPT, 177295c354feSNick Piggin * but a general need for low latency) 17731da177e4SLinus Torvalds */ 177495c354feSNick Piggin static inline int spin_needbreak(spinlock_t *lock) 17751da177e4SLinus Torvalds { 177695c354feSNick Piggin #ifdef CONFIG_PREEMPT 177795c354feSNick Piggin return spin_is_contended(lock); 177895c354feSNick Piggin #else 17791da177e4SLinus Torvalds return 0; 178095c354feSNick Piggin #endif 17811da177e4SLinus Torvalds } 17821da177e4SLinus Torvalds 178375f93fedSPeter Zijlstra static __always_inline bool need_resched(void) 178475f93fedSPeter Zijlstra { 178575f93fedSPeter Zijlstra return unlikely(tif_need_resched()); 178675f93fedSPeter Zijlstra } 178775f93fedSPeter Zijlstra 1788ee761f62SThomas Gleixner /* 17891da177e4SLinus Torvalds * Wrappers for p->thread_info->cpu access. No-op on UP. 17901da177e4SLinus Torvalds */ 17911da177e4SLinus Torvalds #ifdef CONFIG_SMP 17921da177e4SLinus Torvalds 17931da177e4SLinus Torvalds static inline unsigned int task_cpu(const struct task_struct *p) 17941da177e4SLinus Torvalds { 1795c65eacbeSAndy Lutomirski #ifdef CONFIG_THREAD_INFO_IN_TASK 1796c546951dSAndrea Parri return READ_ONCE(p->cpu); 1797c65eacbeSAndy Lutomirski #else 1798c546951dSAndrea Parri return READ_ONCE(task_thread_info(p)->cpu); 1799c65eacbeSAndy Lutomirski #endif 18001da177e4SLinus Torvalds } 18011da177e4SLinus Torvalds 1802c65cc870SIngo Molnar extern void set_task_cpu(struct task_struct *p, unsigned int cpu); 18031da177e4SLinus Torvalds 18041da177e4SLinus Torvalds #else 18051da177e4SLinus Torvalds 18061da177e4SLinus Torvalds static inline unsigned int task_cpu(const struct task_struct *p) 18071da177e4SLinus Torvalds { 18081da177e4SLinus Torvalds return 0; 18091da177e4SLinus Torvalds } 18101da177e4SLinus Torvalds 18111da177e4SLinus Torvalds static inline void set_task_cpu(struct task_struct *p, unsigned int cpu) 18121da177e4SLinus Torvalds { 18131da177e4SLinus Torvalds } 18141da177e4SLinus Torvalds 18151da177e4SLinus Torvalds #endif /* CONFIG_SMP */ 18161da177e4SLinus Torvalds 1817d9345c65SPan Xinhui /* 1818d9345c65SPan Xinhui * In order to reduce various lock holder preemption latencies provide an 1819d9345c65SPan Xinhui * interface to see if a vCPU is currently running or not. 1820d9345c65SPan Xinhui * 1821d9345c65SPan Xinhui * This allows us to terminate optimistic spin loops and block, analogous to 1822d9345c65SPan Xinhui * the native optimistic spin heuristic of testing if the lock owner task is 1823d9345c65SPan Xinhui * running or not. 1824d9345c65SPan Xinhui */ 1825d9345c65SPan Xinhui #ifndef vcpu_is_preempted 1826d9345c65SPan Xinhui # define vcpu_is_preempted(cpu) false 1827d9345c65SPan Xinhui #endif 1828d9345c65SPan Xinhui 182996f874e2SRusty Russell extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask); 183096f874e2SRusty Russell extern long sched_getaffinity(pid_t pid, struct cpumask *mask); 18315c45bf27SSiddha, Suresh B 183282455257SDave Hansen #ifndef TASK_SIZE_OF 183382455257SDave Hansen #define TASK_SIZE_OF(tsk) TASK_SIZE 183482455257SDave Hansen #endif 183582455257SDave Hansen 1836d7822b1eSMathieu Desnoyers #ifdef CONFIG_RSEQ 1837d7822b1eSMathieu Desnoyers 1838d7822b1eSMathieu Desnoyers /* 1839d7822b1eSMathieu Desnoyers * Map the event mask on the user-space ABI enum rseq_cs_flags 1840d7822b1eSMathieu Desnoyers * for direct mask checks. 1841d7822b1eSMathieu Desnoyers */ 1842d7822b1eSMathieu Desnoyers enum rseq_event_mask_bits { 1843d7822b1eSMathieu Desnoyers RSEQ_EVENT_PREEMPT_BIT = RSEQ_CS_FLAG_NO_RESTART_ON_PREEMPT_BIT, 1844d7822b1eSMathieu Desnoyers RSEQ_EVENT_SIGNAL_BIT = RSEQ_CS_FLAG_NO_RESTART_ON_SIGNAL_BIT, 1845d7822b1eSMathieu Desnoyers RSEQ_EVENT_MIGRATE_BIT = RSEQ_CS_FLAG_NO_RESTART_ON_MIGRATE_BIT, 1846d7822b1eSMathieu Desnoyers }; 1847d7822b1eSMathieu Desnoyers 1848d7822b1eSMathieu Desnoyers enum rseq_event_mask { 1849d7822b1eSMathieu Desnoyers RSEQ_EVENT_PREEMPT = (1U << RSEQ_EVENT_PREEMPT_BIT), 1850d7822b1eSMathieu Desnoyers RSEQ_EVENT_SIGNAL = (1U << RSEQ_EVENT_SIGNAL_BIT), 1851d7822b1eSMathieu Desnoyers RSEQ_EVENT_MIGRATE = (1U << RSEQ_EVENT_MIGRATE_BIT), 1852d7822b1eSMathieu Desnoyers }; 1853d7822b1eSMathieu Desnoyers 1854d7822b1eSMathieu Desnoyers static inline void rseq_set_notify_resume(struct task_struct *t) 1855d7822b1eSMathieu Desnoyers { 1856d7822b1eSMathieu Desnoyers if (t->rseq) 1857d7822b1eSMathieu Desnoyers set_tsk_thread_flag(t, TIF_NOTIFY_RESUME); 1858d7822b1eSMathieu Desnoyers } 1859d7822b1eSMathieu Desnoyers 1860784e0300SWill Deacon void __rseq_handle_notify_resume(struct ksignal *sig, struct pt_regs *regs); 1861d7822b1eSMathieu Desnoyers 1862784e0300SWill Deacon static inline void rseq_handle_notify_resume(struct ksignal *ksig, 1863784e0300SWill Deacon struct pt_regs *regs) 1864d7822b1eSMathieu Desnoyers { 1865d7822b1eSMathieu Desnoyers if (current->rseq) 1866784e0300SWill Deacon __rseq_handle_notify_resume(ksig, regs); 1867d7822b1eSMathieu Desnoyers } 1868d7822b1eSMathieu Desnoyers 1869784e0300SWill Deacon static inline void rseq_signal_deliver(struct ksignal *ksig, 1870784e0300SWill Deacon struct pt_regs *regs) 1871d7822b1eSMathieu Desnoyers { 1872d7822b1eSMathieu Desnoyers preempt_disable(); 1873d7822b1eSMathieu Desnoyers __set_bit(RSEQ_EVENT_SIGNAL_BIT, ¤t->rseq_event_mask); 1874d7822b1eSMathieu Desnoyers preempt_enable(); 1875784e0300SWill Deacon rseq_handle_notify_resume(ksig, regs); 1876d7822b1eSMathieu Desnoyers } 1877d7822b1eSMathieu Desnoyers 1878d7822b1eSMathieu Desnoyers /* rseq_preempt() requires preemption to be disabled. */ 1879d7822b1eSMathieu Desnoyers static inline void rseq_preempt(struct task_struct *t) 1880d7822b1eSMathieu Desnoyers { 1881d7822b1eSMathieu Desnoyers __set_bit(RSEQ_EVENT_PREEMPT_BIT, &t->rseq_event_mask); 1882d7822b1eSMathieu Desnoyers rseq_set_notify_resume(t); 1883d7822b1eSMathieu Desnoyers } 1884d7822b1eSMathieu Desnoyers 1885d7822b1eSMathieu Desnoyers /* rseq_migrate() requires preemption to be disabled. */ 1886d7822b1eSMathieu Desnoyers static inline void rseq_migrate(struct task_struct *t) 1887d7822b1eSMathieu Desnoyers { 1888d7822b1eSMathieu Desnoyers __set_bit(RSEQ_EVENT_MIGRATE_BIT, &t->rseq_event_mask); 1889d7822b1eSMathieu Desnoyers rseq_set_notify_resume(t); 1890d7822b1eSMathieu Desnoyers } 1891d7822b1eSMathieu Desnoyers 1892d7822b1eSMathieu Desnoyers /* 1893d7822b1eSMathieu Desnoyers * If parent process has a registered restartable sequences area, the 18949a789fcfSMathieu Desnoyers * child inherits. Only applies when forking a process, not a thread. 1895d7822b1eSMathieu Desnoyers */ 1896d7822b1eSMathieu Desnoyers static inline void rseq_fork(struct task_struct *t, unsigned long clone_flags) 1897d7822b1eSMathieu Desnoyers { 1898d7822b1eSMathieu Desnoyers if (clone_flags & CLONE_THREAD) { 1899d7822b1eSMathieu Desnoyers t->rseq = NULL; 1900d7822b1eSMathieu Desnoyers t->rseq_sig = 0; 1901d7822b1eSMathieu Desnoyers t->rseq_event_mask = 0; 1902d7822b1eSMathieu Desnoyers } else { 1903d7822b1eSMathieu Desnoyers t->rseq = current->rseq; 1904d7822b1eSMathieu Desnoyers t->rseq_sig = current->rseq_sig; 1905d7822b1eSMathieu Desnoyers t->rseq_event_mask = current->rseq_event_mask; 1906d7822b1eSMathieu Desnoyers } 1907d7822b1eSMathieu Desnoyers } 1908d7822b1eSMathieu Desnoyers 1909d7822b1eSMathieu Desnoyers static inline void rseq_execve(struct task_struct *t) 1910d7822b1eSMathieu Desnoyers { 1911d7822b1eSMathieu Desnoyers t->rseq = NULL; 1912d7822b1eSMathieu Desnoyers t->rseq_sig = 0; 1913d7822b1eSMathieu Desnoyers t->rseq_event_mask = 0; 1914d7822b1eSMathieu Desnoyers } 1915d7822b1eSMathieu Desnoyers 1916d7822b1eSMathieu Desnoyers #else 1917d7822b1eSMathieu Desnoyers 1918d7822b1eSMathieu Desnoyers static inline void rseq_set_notify_resume(struct task_struct *t) 1919d7822b1eSMathieu Desnoyers { 1920d7822b1eSMathieu Desnoyers } 1921784e0300SWill Deacon static inline void rseq_handle_notify_resume(struct ksignal *ksig, 1922784e0300SWill Deacon struct pt_regs *regs) 1923d7822b1eSMathieu Desnoyers { 1924d7822b1eSMathieu Desnoyers } 1925784e0300SWill Deacon static inline void rseq_signal_deliver(struct ksignal *ksig, 1926784e0300SWill Deacon struct pt_regs *regs) 1927d7822b1eSMathieu Desnoyers { 1928d7822b1eSMathieu Desnoyers } 1929d7822b1eSMathieu Desnoyers static inline void rseq_preempt(struct task_struct *t) 1930d7822b1eSMathieu Desnoyers { 1931d7822b1eSMathieu Desnoyers } 1932d7822b1eSMathieu Desnoyers static inline void rseq_migrate(struct task_struct *t) 1933d7822b1eSMathieu Desnoyers { 1934d7822b1eSMathieu Desnoyers } 1935d7822b1eSMathieu Desnoyers static inline void rseq_fork(struct task_struct *t, unsigned long clone_flags) 1936d7822b1eSMathieu Desnoyers { 1937d7822b1eSMathieu Desnoyers } 1938d7822b1eSMathieu Desnoyers static inline void rseq_execve(struct task_struct *t) 1939d7822b1eSMathieu Desnoyers { 1940d7822b1eSMathieu Desnoyers } 1941d7822b1eSMathieu Desnoyers 1942d7822b1eSMathieu Desnoyers #endif 1943d7822b1eSMathieu Desnoyers 194473ab1cb2STaehee Yoo void __exit_umh(struct task_struct *tsk); 194573ab1cb2STaehee Yoo 194673ab1cb2STaehee Yoo static inline void exit_umh(struct task_struct *tsk) 194773ab1cb2STaehee Yoo { 194873ab1cb2STaehee Yoo if (unlikely(tsk->flags & PF_UMH)) 194973ab1cb2STaehee Yoo __exit_umh(tsk); 195073ab1cb2STaehee Yoo } 195173ab1cb2STaehee Yoo 1952d7822b1eSMathieu Desnoyers #ifdef CONFIG_DEBUG_RSEQ 1953d7822b1eSMathieu Desnoyers 1954d7822b1eSMathieu Desnoyers void rseq_syscall(struct pt_regs *regs); 1955d7822b1eSMathieu Desnoyers 1956d7822b1eSMathieu Desnoyers #else 1957d7822b1eSMathieu Desnoyers 1958d7822b1eSMathieu Desnoyers static inline void rseq_syscall(struct pt_regs *regs) 1959d7822b1eSMathieu Desnoyers { 1960d7822b1eSMathieu Desnoyers } 1961d7822b1eSMathieu Desnoyers 1962d7822b1eSMathieu Desnoyers #endif 1963d7822b1eSMathieu Desnoyers 19643c93a0c0SQais Yousef const struct sched_avg *sched_trace_cfs_rq_avg(struct cfs_rq *cfs_rq); 19653c93a0c0SQais Yousef char *sched_trace_cfs_rq_path(struct cfs_rq *cfs_rq, char *str, int len); 19663c93a0c0SQais Yousef int sched_trace_cfs_rq_cpu(struct cfs_rq *cfs_rq); 19673c93a0c0SQais Yousef 19683c93a0c0SQais Yousef const struct sched_avg *sched_trace_rq_avg_rt(struct rq *rq); 19693c93a0c0SQais Yousef const struct sched_avg *sched_trace_rq_avg_dl(struct rq *rq); 19703c93a0c0SQais Yousef const struct sched_avg *sched_trace_rq_avg_irq(struct rq *rq); 19713c93a0c0SQais Yousef 19723c93a0c0SQais Yousef int sched_trace_rq_cpu(struct rq *rq); 19733c93a0c0SQais Yousef 19743c93a0c0SQais Yousef const struct cpumask *sched_trace_rd_span(struct root_domain *rd); 19753c93a0c0SQais Yousef 19761da177e4SLinus Torvalds #endif 1977