1 #ifndef _LINUX_SCHED_H 2 #define _LINUX_SCHED_H 3 4 #include <uapi/linux/sched.h> 5 6 #include <linux/sched/prio.h> 7 8 9 struct sched_param { 10 int sched_priority; 11 }; 12 13 #include <asm/param.h> /* for HZ */ 14 15 #include <linux/capability.h> 16 #include <linux/threads.h> 17 #include <linux/kernel.h> 18 #include <linux/types.h> 19 #include <linux/timex.h> 20 #include <linux/jiffies.h> 21 #include <linux/plist.h> 22 #include <linux/rbtree.h> 23 #include <linux/thread_info.h> 24 #include <linux/cpumask.h> 25 #include <linux/errno.h> 26 #include <linux/nodemask.h> 27 #include <linux/mm_types.h> 28 #include <linux/preempt.h> 29 30 #include <asm/page.h> 31 #include <asm/ptrace.h> 32 #include <linux/cputime.h> 33 34 #include <linux/smp.h> 35 #include <linux/sem.h> 36 #include <linux/shm.h> 37 #include <linux/signal.h> 38 #include <linux/compiler.h> 39 #include <linux/completion.h> 40 #include <linux/pid.h> 41 #include <linux/percpu.h> 42 #include <linux/topology.h> 43 #include <linux/seccomp.h> 44 #include <linux/rcupdate.h> 45 #include <linux/rculist.h> 46 #include <linux/rtmutex.h> 47 48 #include <linux/time.h> 49 #include <linux/param.h> 50 #include <linux/resource.h> 51 #include <linux/timer.h> 52 #include <linux/hrtimer.h> 53 #include <linux/kcov.h> 54 #include <linux/task_io_accounting.h> 55 #include <linux/latencytop.h> 56 #include <linux/cred.h> 57 #include <linux/llist.h> 58 #include <linux/uidgid.h> 59 #include <linux/gfp.h> 60 #include <linux/magic.h> 61 #include <linux/cgroup-defs.h> 62 63 #include <asm/processor.h> 64 65 #define SCHED_ATTR_SIZE_VER0 48 /* sizeof first published struct */ 66 67 /* 68 * Extended scheduling parameters data structure. 69 * 70 * This is needed because the original struct sched_param can not be 71 * altered without introducing ABI issues with legacy applications 72 * (e.g., in sched_getparam()). 73 * 74 * However, the possibility of specifying more than just a priority for 75 * the tasks may be useful for a wide variety of application fields, e.g., 76 * multimedia, streaming, automation and control, and many others. 77 * 78 * This variant (sched_attr) is meant at describing a so-called 79 * sporadic time-constrained task. In such model a task is specified by: 80 * - the activation period or minimum instance inter-arrival time; 81 * - the maximum (or average, depending on the actual scheduling 82 * discipline) computation time of all instances, a.k.a. runtime; 83 * - the deadline (relative to the actual activation time) of each 84 * instance. 85 * Very briefly, a periodic (sporadic) task asks for the execution of 86 * some specific computation --which is typically called an instance-- 87 * (at most) every period. Moreover, each instance typically lasts no more 88 * than the runtime and must be completed by time instant t equal to 89 * the instance activation time + the deadline. 90 * 91 * This is reflected by the actual fields of the sched_attr structure: 92 * 93 * @size size of the structure, for fwd/bwd compat. 94 * 95 * @sched_policy task's scheduling policy 96 * @sched_flags for customizing the scheduler behaviour 97 * @sched_nice task's nice value (SCHED_NORMAL/BATCH) 98 * @sched_priority task's static priority (SCHED_FIFO/RR) 99 * @sched_deadline representative of the task's deadline 100 * @sched_runtime representative of the task's runtime 101 * @sched_period representative of the task's period 102 * 103 * Given this task model, there are a multiplicity of scheduling algorithms 104 * and policies, that can be used to ensure all the tasks will make their 105 * timing constraints. 106 * 107 * As of now, the SCHED_DEADLINE policy (sched_dl scheduling class) is the 108 * only user of this new interface. More information about the algorithm 109 * available in the scheduling class file or in Documentation/. 110 */ 111 struct sched_attr { 112 u32 size; 113 114 u32 sched_policy; 115 u64 sched_flags; 116 117 /* SCHED_NORMAL, SCHED_BATCH */ 118 s32 sched_nice; 119 120 /* SCHED_FIFO, SCHED_RR */ 121 u32 sched_priority; 122 123 /* SCHED_DEADLINE */ 124 u64 sched_runtime; 125 u64 sched_deadline; 126 u64 sched_period; 127 }; 128 129 struct futex_pi_state; 130 struct robust_list_head; 131 struct bio_list; 132 struct fs_struct; 133 struct perf_event_context; 134 struct blk_plug; 135 struct filename; 136 struct nameidata; 137 138 #define VMACACHE_BITS 2 139 #define VMACACHE_SIZE (1U << VMACACHE_BITS) 140 #define VMACACHE_MASK (VMACACHE_SIZE - 1) 141 142 /* 143 * These are the constant used to fake the fixed-point load-average 144 * counting. Some notes: 145 * - 11 bit fractions expand to 22 bits by the multiplies: this gives 146 * a load-average precision of 10 bits integer + 11 bits fractional 147 * - if you want to count load-averages more often, you need more 148 * precision, or rounding will get you. With 2-second counting freq, 149 * the EXP_n values would be 1981, 2034 and 2043 if still using only 150 * 11 bit fractions. 151 */ 152 extern unsigned long avenrun[]; /* Load averages */ 153 extern void get_avenrun(unsigned long *loads, unsigned long offset, int shift); 154 155 #define FSHIFT 11 /* nr of bits of precision */ 156 #define FIXED_1 (1<<FSHIFT) /* 1.0 as fixed-point */ 157 #define LOAD_FREQ (5*HZ+1) /* 5 sec intervals */ 158 #define EXP_1 1884 /* 1/exp(5sec/1min) as fixed-point */ 159 #define EXP_5 2014 /* 1/exp(5sec/5min) */ 160 #define EXP_15 2037 /* 1/exp(5sec/15min) */ 161 162 #define CALC_LOAD(load,exp,n) \ 163 load *= exp; \ 164 load += n*(FIXED_1-exp); \ 165 load >>= FSHIFT; 166 167 extern unsigned long total_forks; 168 extern int nr_threads; 169 DECLARE_PER_CPU(unsigned long, process_counts); 170 extern int nr_processes(void); 171 extern unsigned long nr_running(void); 172 extern bool single_task_running(void); 173 extern unsigned long nr_iowait(void); 174 extern unsigned long nr_iowait_cpu(int cpu); 175 extern void get_iowait_load(unsigned long *nr_waiters, unsigned long *load); 176 177 extern void calc_global_load(unsigned long ticks); 178 179 #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON) 180 extern void cpu_load_update_nohz_start(void); 181 extern void cpu_load_update_nohz_stop(void); 182 #else 183 static inline void cpu_load_update_nohz_start(void) { } 184 static inline void cpu_load_update_nohz_stop(void) { } 185 #endif 186 187 extern void dump_cpu_task(int cpu); 188 189 struct seq_file; 190 struct cfs_rq; 191 struct task_group; 192 #ifdef CONFIG_SCHED_DEBUG 193 extern void proc_sched_show_task(struct task_struct *p, struct seq_file *m); 194 extern void proc_sched_set_task(struct task_struct *p); 195 #endif 196 197 /* 198 * Task state bitmask. NOTE! These bits are also 199 * encoded in fs/proc/array.c: get_task_state(). 200 * 201 * We have two separate sets of flags: task->state 202 * is about runnability, while task->exit_state are 203 * about the task exiting. Confusing, but this way 204 * modifying one set can't modify the other one by 205 * mistake. 206 */ 207 #define TASK_RUNNING 0 208 #define TASK_INTERRUPTIBLE 1 209 #define TASK_UNINTERRUPTIBLE 2 210 #define __TASK_STOPPED 4 211 #define __TASK_TRACED 8 212 /* in tsk->exit_state */ 213 #define EXIT_DEAD 16 214 #define EXIT_ZOMBIE 32 215 #define EXIT_TRACE (EXIT_ZOMBIE | EXIT_DEAD) 216 /* in tsk->state again */ 217 #define TASK_DEAD 64 218 #define TASK_WAKEKILL 128 219 #define TASK_WAKING 256 220 #define TASK_PARKED 512 221 #define TASK_NOLOAD 1024 222 #define TASK_NEW 2048 223 #define TASK_STATE_MAX 4096 224 225 #define TASK_STATE_TO_CHAR_STR "RSDTtXZxKWPNn" 226 227 extern char ___assert_task_state[1 - 2*!!( 228 sizeof(TASK_STATE_TO_CHAR_STR)-1 != ilog2(TASK_STATE_MAX)+1)]; 229 230 /* Convenience macros for the sake of set_task_state */ 231 #define TASK_KILLABLE (TASK_WAKEKILL | TASK_UNINTERRUPTIBLE) 232 #define TASK_STOPPED (TASK_WAKEKILL | __TASK_STOPPED) 233 #define TASK_TRACED (TASK_WAKEKILL | __TASK_TRACED) 234 235 #define TASK_IDLE (TASK_UNINTERRUPTIBLE | TASK_NOLOAD) 236 237 /* Convenience macros for the sake of wake_up */ 238 #define TASK_NORMAL (TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE) 239 #define TASK_ALL (TASK_NORMAL | __TASK_STOPPED | __TASK_TRACED) 240 241 /* get_task_state() */ 242 #define TASK_REPORT (TASK_RUNNING | TASK_INTERRUPTIBLE | \ 243 TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \ 244 __TASK_TRACED | EXIT_ZOMBIE | EXIT_DEAD) 245 246 #define task_is_traced(task) ((task->state & __TASK_TRACED) != 0) 247 #define task_is_stopped(task) ((task->state & __TASK_STOPPED) != 0) 248 #define task_is_stopped_or_traced(task) \ 249 ((task->state & (__TASK_STOPPED | __TASK_TRACED)) != 0) 250 #define task_contributes_to_load(task) \ 251 ((task->state & TASK_UNINTERRUPTIBLE) != 0 && \ 252 (task->flags & PF_FROZEN) == 0 && \ 253 (task->state & TASK_NOLOAD) == 0) 254 255 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP 256 257 #define __set_task_state(tsk, state_value) \ 258 do { \ 259 (tsk)->task_state_change = _THIS_IP_; \ 260 (tsk)->state = (state_value); \ 261 } while (0) 262 #define set_task_state(tsk, state_value) \ 263 do { \ 264 (tsk)->task_state_change = _THIS_IP_; \ 265 smp_store_mb((tsk)->state, (state_value)); \ 266 } while (0) 267 268 #define __set_current_state(state_value) \ 269 do { \ 270 current->task_state_change = _THIS_IP_; \ 271 current->state = (state_value); \ 272 } while (0) 273 #define set_current_state(state_value) \ 274 do { \ 275 current->task_state_change = _THIS_IP_; \ 276 smp_store_mb(current->state, (state_value)); \ 277 } while (0) 278 279 #else 280 281 /* 282 * @tsk had better be current, or you get to keep the pieces. 283 * 284 * The only reason is that computing current can be more expensive than 285 * using a pointer that's already available. 286 * 287 * Therefore, see set_current_state(). 288 */ 289 #define __set_task_state(tsk, state_value) \ 290 do { (tsk)->state = (state_value); } while (0) 291 #define set_task_state(tsk, state_value) \ 292 smp_store_mb((tsk)->state, (state_value)) 293 294 /* 295 * set_current_state() includes a barrier so that the write of current->state 296 * is correctly serialised wrt the caller's subsequent test of whether to 297 * actually sleep: 298 * 299 * for (;;) { 300 * set_current_state(TASK_UNINTERRUPTIBLE); 301 * if (!need_sleep) 302 * break; 303 * 304 * schedule(); 305 * } 306 * __set_current_state(TASK_RUNNING); 307 * 308 * If the caller does not need such serialisation (because, for instance, the 309 * condition test and condition change and wakeup are under the same lock) then 310 * use __set_current_state(). 311 * 312 * The above is typically ordered against the wakeup, which does: 313 * 314 * need_sleep = false; 315 * wake_up_state(p, TASK_UNINTERRUPTIBLE); 316 * 317 * Where wake_up_state() (and all other wakeup primitives) imply enough 318 * barriers to order the store of the variable against wakeup. 319 * 320 * Wakeup will do: if (@state & p->state) p->state = TASK_RUNNING, that is, 321 * once it observes the TASK_UNINTERRUPTIBLE store the waking CPU can issue a 322 * TASK_RUNNING store which can collide with __set_current_state(TASK_RUNNING). 323 * 324 * This is obviously fine, since they both store the exact same value. 325 * 326 * Also see the comments of try_to_wake_up(). 327 */ 328 #define __set_current_state(state_value) \ 329 do { current->state = (state_value); } while (0) 330 #define set_current_state(state_value) \ 331 smp_store_mb(current->state, (state_value)) 332 333 #endif 334 335 /* Task command name length */ 336 #define TASK_COMM_LEN 16 337 338 #include <linux/spinlock.h> 339 340 /* 341 * This serializes "schedule()" and also protects 342 * the run-queue from deletions/modifications (but 343 * _adding_ to the beginning of the run-queue has 344 * a separate lock). 345 */ 346 extern rwlock_t tasklist_lock; 347 extern spinlock_t mmlist_lock; 348 349 struct task_struct; 350 351 #ifdef CONFIG_PROVE_RCU 352 extern int lockdep_tasklist_lock_is_held(void); 353 #endif /* #ifdef CONFIG_PROVE_RCU */ 354 355 extern void sched_init(void); 356 extern void sched_init_smp(void); 357 extern asmlinkage void schedule_tail(struct task_struct *prev); 358 extern void init_idle(struct task_struct *idle, int cpu); 359 extern void init_idle_bootup_task(struct task_struct *idle); 360 361 extern cpumask_var_t cpu_isolated_map; 362 363 extern int runqueue_is_locked(int cpu); 364 365 #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON) 366 extern void nohz_balance_enter_idle(int cpu); 367 extern void set_cpu_sd_state_idle(void); 368 extern int get_nohz_timer_target(void); 369 #else 370 static inline void nohz_balance_enter_idle(int cpu) { } 371 static inline void set_cpu_sd_state_idle(void) { } 372 #endif 373 374 /* 375 * Only dump TASK_* tasks. (0 for all tasks) 376 */ 377 extern void show_state_filter(unsigned long state_filter); 378 379 static inline void show_state(void) 380 { 381 show_state_filter(0); 382 } 383 384 extern void show_regs(struct pt_regs *); 385 386 /* 387 * TASK is a pointer to the task whose backtrace we want to see (or NULL for current 388 * task), SP is the stack pointer of the first frame that should be shown in the back 389 * trace (or NULL if the entire call-chain of the task should be shown). 390 */ 391 extern void show_stack(struct task_struct *task, unsigned long *sp); 392 393 extern void cpu_init (void); 394 extern void trap_init(void); 395 extern void update_process_times(int user); 396 extern void scheduler_tick(void); 397 extern int sched_cpu_starting(unsigned int cpu); 398 extern int sched_cpu_activate(unsigned int cpu); 399 extern int sched_cpu_deactivate(unsigned int cpu); 400 401 #ifdef CONFIG_HOTPLUG_CPU 402 extern int sched_cpu_dying(unsigned int cpu); 403 #else 404 # define sched_cpu_dying NULL 405 #endif 406 407 extern void sched_show_task(struct task_struct *p); 408 409 #ifdef CONFIG_LOCKUP_DETECTOR 410 extern void touch_softlockup_watchdog_sched(void); 411 extern void touch_softlockup_watchdog(void); 412 extern void touch_softlockup_watchdog_sync(void); 413 extern void touch_all_softlockup_watchdogs(void); 414 extern int proc_dowatchdog_thresh(struct ctl_table *table, int write, 415 void __user *buffer, 416 size_t *lenp, loff_t *ppos); 417 extern unsigned int softlockup_panic; 418 extern unsigned int hardlockup_panic; 419 void lockup_detector_init(void); 420 #else 421 static inline void touch_softlockup_watchdog_sched(void) 422 { 423 } 424 static inline void touch_softlockup_watchdog(void) 425 { 426 } 427 static inline void touch_softlockup_watchdog_sync(void) 428 { 429 } 430 static inline void touch_all_softlockup_watchdogs(void) 431 { 432 } 433 static inline void lockup_detector_init(void) 434 { 435 } 436 #endif 437 438 #ifdef CONFIG_DETECT_HUNG_TASK 439 void reset_hung_task_detector(void); 440 #else 441 static inline void reset_hung_task_detector(void) 442 { 443 } 444 #endif 445 446 /* Attach to any functions which should be ignored in wchan output. */ 447 #define __sched __attribute__((__section__(".sched.text"))) 448 449 /* Linker adds these: start and end of __sched functions */ 450 extern char __sched_text_start[], __sched_text_end[]; 451 452 /* Is this address in the __sched functions? */ 453 extern int in_sched_functions(unsigned long addr); 454 455 #define MAX_SCHEDULE_TIMEOUT LONG_MAX 456 extern signed long schedule_timeout(signed long timeout); 457 extern signed long schedule_timeout_interruptible(signed long timeout); 458 extern signed long schedule_timeout_killable(signed long timeout); 459 extern signed long schedule_timeout_uninterruptible(signed long timeout); 460 extern signed long schedule_timeout_idle(signed long timeout); 461 asmlinkage void schedule(void); 462 extern void schedule_preempt_disabled(void); 463 464 extern long io_schedule_timeout(long timeout); 465 466 static inline void io_schedule(void) 467 { 468 io_schedule_timeout(MAX_SCHEDULE_TIMEOUT); 469 } 470 471 void __noreturn do_task_dead(void); 472 473 struct nsproxy; 474 struct user_namespace; 475 476 #ifdef CONFIG_MMU 477 extern void arch_pick_mmap_layout(struct mm_struct *mm); 478 extern unsigned long 479 arch_get_unmapped_area(struct file *, unsigned long, unsigned long, 480 unsigned long, unsigned long); 481 extern unsigned long 482 arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr, 483 unsigned long len, unsigned long pgoff, 484 unsigned long flags); 485 #else 486 static inline void arch_pick_mmap_layout(struct mm_struct *mm) {} 487 #endif 488 489 #define SUID_DUMP_DISABLE 0 /* No setuid dumping */ 490 #define SUID_DUMP_USER 1 /* Dump as user of process */ 491 #define SUID_DUMP_ROOT 2 /* Dump as root */ 492 493 /* mm flags */ 494 495 /* for SUID_DUMP_* above */ 496 #define MMF_DUMPABLE_BITS 2 497 #define MMF_DUMPABLE_MASK ((1 << MMF_DUMPABLE_BITS) - 1) 498 499 extern void set_dumpable(struct mm_struct *mm, int value); 500 /* 501 * This returns the actual value of the suid_dumpable flag. For things 502 * that are using this for checking for privilege transitions, it must 503 * test against SUID_DUMP_USER rather than treating it as a boolean 504 * value. 505 */ 506 static inline int __get_dumpable(unsigned long mm_flags) 507 { 508 return mm_flags & MMF_DUMPABLE_MASK; 509 } 510 511 static inline int get_dumpable(struct mm_struct *mm) 512 { 513 return __get_dumpable(mm->flags); 514 } 515 516 /* coredump filter bits */ 517 #define MMF_DUMP_ANON_PRIVATE 2 518 #define MMF_DUMP_ANON_SHARED 3 519 #define MMF_DUMP_MAPPED_PRIVATE 4 520 #define MMF_DUMP_MAPPED_SHARED 5 521 #define MMF_DUMP_ELF_HEADERS 6 522 #define MMF_DUMP_HUGETLB_PRIVATE 7 523 #define MMF_DUMP_HUGETLB_SHARED 8 524 #define MMF_DUMP_DAX_PRIVATE 9 525 #define MMF_DUMP_DAX_SHARED 10 526 527 #define MMF_DUMP_FILTER_SHIFT MMF_DUMPABLE_BITS 528 #define MMF_DUMP_FILTER_BITS 9 529 #define MMF_DUMP_FILTER_MASK \ 530 (((1 << MMF_DUMP_FILTER_BITS) - 1) << MMF_DUMP_FILTER_SHIFT) 531 #define MMF_DUMP_FILTER_DEFAULT \ 532 ((1 << MMF_DUMP_ANON_PRIVATE) | (1 << MMF_DUMP_ANON_SHARED) |\ 533 (1 << MMF_DUMP_HUGETLB_PRIVATE) | MMF_DUMP_MASK_DEFAULT_ELF) 534 535 #ifdef CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS 536 # define MMF_DUMP_MASK_DEFAULT_ELF (1 << MMF_DUMP_ELF_HEADERS) 537 #else 538 # define MMF_DUMP_MASK_DEFAULT_ELF 0 539 #endif 540 /* leave room for more dump flags */ 541 #define MMF_VM_MERGEABLE 16 /* KSM may merge identical pages */ 542 #define MMF_VM_HUGEPAGE 17 /* set when VM_HUGEPAGE is set on vma */ 543 /* 544 * This one-shot flag is dropped due to necessity of changing exe once again 545 * on NFS restore 546 */ 547 //#define MMF_EXE_FILE_CHANGED 18 /* see prctl_set_mm_exe_file() */ 548 549 #define MMF_HAS_UPROBES 19 /* has uprobes */ 550 #define MMF_RECALC_UPROBES 20 /* MMF_HAS_UPROBES can be wrong */ 551 #define MMF_OOM_SKIP 21 /* mm is of no interest for the OOM killer */ 552 #define MMF_UNSTABLE 22 /* mm is unstable for copy_from_user */ 553 #define MMF_HUGE_ZERO_PAGE 23 /* mm has ever used the global huge zero page */ 554 555 #define MMF_INIT_MASK (MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK) 556 557 struct sighand_struct { 558 atomic_t count; 559 struct k_sigaction action[_NSIG]; 560 spinlock_t siglock; 561 wait_queue_head_t signalfd_wqh; 562 }; 563 564 struct pacct_struct { 565 int ac_flag; 566 long ac_exitcode; 567 unsigned long ac_mem; 568 cputime_t ac_utime, ac_stime; 569 unsigned long ac_minflt, ac_majflt; 570 }; 571 572 struct cpu_itimer { 573 cputime_t expires; 574 cputime_t incr; 575 u32 error; 576 u32 incr_error; 577 }; 578 579 /** 580 * struct prev_cputime - snaphsot of system and user cputime 581 * @utime: time spent in user mode 582 * @stime: time spent in system mode 583 * @lock: protects the above two fields 584 * 585 * Stores previous user/system time values such that we can guarantee 586 * monotonicity. 587 */ 588 struct prev_cputime { 589 #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE 590 cputime_t utime; 591 cputime_t stime; 592 raw_spinlock_t lock; 593 #endif 594 }; 595 596 static inline void prev_cputime_init(struct prev_cputime *prev) 597 { 598 #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE 599 prev->utime = prev->stime = 0; 600 raw_spin_lock_init(&prev->lock); 601 #endif 602 } 603 604 /** 605 * struct task_cputime - collected CPU time counts 606 * @utime: time spent in user mode, in &cputime_t units 607 * @stime: time spent in kernel mode, in &cputime_t units 608 * @sum_exec_runtime: total time spent on the CPU, in nanoseconds 609 * 610 * This structure groups together three kinds of CPU time that are tracked for 611 * threads and thread groups. Most things considering CPU time want to group 612 * these counts together and treat all three of them in parallel. 613 */ 614 struct task_cputime { 615 cputime_t utime; 616 cputime_t stime; 617 unsigned long long sum_exec_runtime; 618 }; 619 620 /* Alternate field names when used to cache expirations. */ 621 #define virt_exp utime 622 #define prof_exp stime 623 #define sched_exp sum_exec_runtime 624 625 #define INIT_CPUTIME \ 626 (struct task_cputime) { \ 627 .utime = 0, \ 628 .stime = 0, \ 629 .sum_exec_runtime = 0, \ 630 } 631 632 /* 633 * This is the atomic variant of task_cputime, which can be used for 634 * storing and updating task_cputime statistics without locking. 635 */ 636 struct task_cputime_atomic { 637 atomic64_t utime; 638 atomic64_t stime; 639 atomic64_t sum_exec_runtime; 640 }; 641 642 #define INIT_CPUTIME_ATOMIC \ 643 (struct task_cputime_atomic) { \ 644 .utime = ATOMIC64_INIT(0), \ 645 .stime = ATOMIC64_INIT(0), \ 646 .sum_exec_runtime = ATOMIC64_INIT(0), \ 647 } 648 649 #define PREEMPT_DISABLED (PREEMPT_DISABLE_OFFSET + PREEMPT_ENABLED) 650 651 /* 652 * Disable preemption until the scheduler is running -- use an unconditional 653 * value so that it also works on !PREEMPT_COUNT kernels. 654 * 655 * Reset by start_kernel()->sched_init()->init_idle()->init_idle_preempt_count(). 656 */ 657 #define INIT_PREEMPT_COUNT PREEMPT_OFFSET 658 659 /* 660 * Initial preempt_count value; reflects the preempt_count schedule invariant 661 * which states that during context switches: 662 * 663 * preempt_count() == 2*PREEMPT_DISABLE_OFFSET 664 * 665 * Note: PREEMPT_DISABLE_OFFSET is 0 for !PREEMPT_COUNT kernels. 666 * Note: See finish_task_switch(). 667 */ 668 #define FORK_PREEMPT_COUNT (2*PREEMPT_DISABLE_OFFSET + PREEMPT_ENABLED) 669 670 /** 671 * struct thread_group_cputimer - thread group interval timer counts 672 * @cputime_atomic: atomic thread group interval timers. 673 * @running: true when there are timers running and 674 * @cputime_atomic receives updates. 675 * @checking_timer: true when a thread in the group is in the 676 * process of checking for thread group timers. 677 * 678 * This structure contains the version of task_cputime, above, that is 679 * used for thread group CPU timer calculations. 680 */ 681 struct thread_group_cputimer { 682 struct task_cputime_atomic cputime_atomic; 683 bool running; 684 bool checking_timer; 685 }; 686 687 #include <linux/rwsem.h> 688 struct autogroup; 689 690 /* 691 * NOTE! "signal_struct" does not have its own 692 * locking, because a shared signal_struct always 693 * implies a shared sighand_struct, so locking 694 * sighand_struct is always a proper superset of 695 * the locking of signal_struct. 696 */ 697 struct signal_struct { 698 atomic_t sigcnt; 699 atomic_t live; 700 int nr_threads; 701 struct list_head thread_head; 702 703 wait_queue_head_t wait_chldexit; /* for wait4() */ 704 705 /* current thread group signal load-balancing target: */ 706 struct task_struct *curr_target; 707 708 /* shared signal handling: */ 709 struct sigpending shared_pending; 710 711 /* thread group exit support */ 712 int group_exit_code; 713 /* overloaded: 714 * - notify group_exit_task when ->count is equal to notify_count 715 * - everyone except group_exit_task is stopped during signal delivery 716 * of fatal signals, group_exit_task processes the signal. 717 */ 718 int notify_count; 719 struct task_struct *group_exit_task; 720 721 /* thread group stop support, overloads group_exit_code too */ 722 int group_stop_count; 723 unsigned int flags; /* see SIGNAL_* flags below */ 724 725 /* 726 * PR_SET_CHILD_SUBREAPER marks a process, like a service 727 * manager, to re-parent orphan (double-forking) child processes 728 * to this process instead of 'init'. The service manager is 729 * able to receive SIGCHLD signals and is able to investigate 730 * the process until it calls wait(). All children of this 731 * process will inherit a flag if they should look for a 732 * child_subreaper process at exit. 733 */ 734 unsigned int is_child_subreaper:1; 735 unsigned int has_child_subreaper:1; 736 737 /* POSIX.1b Interval Timers */ 738 int posix_timer_id; 739 struct list_head posix_timers; 740 741 /* ITIMER_REAL timer for the process */ 742 struct hrtimer real_timer; 743 struct pid *leader_pid; 744 ktime_t it_real_incr; 745 746 /* 747 * ITIMER_PROF and ITIMER_VIRTUAL timers for the process, we use 748 * CPUCLOCK_PROF and CPUCLOCK_VIRT for indexing array as these 749 * values are defined to 0 and 1 respectively 750 */ 751 struct cpu_itimer it[2]; 752 753 /* 754 * Thread group totals for process CPU timers. 755 * See thread_group_cputimer(), et al, for details. 756 */ 757 struct thread_group_cputimer cputimer; 758 759 /* Earliest-expiration cache. */ 760 struct task_cputime cputime_expires; 761 762 #ifdef CONFIG_NO_HZ_FULL 763 atomic_t tick_dep_mask; 764 #endif 765 766 struct list_head cpu_timers[3]; 767 768 struct pid *tty_old_pgrp; 769 770 /* boolean value for session group leader */ 771 int leader; 772 773 struct tty_struct *tty; /* NULL if no tty */ 774 775 #ifdef CONFIG_SCHED_AUTOGROUP 776 struct autogroup *autogroup; 777 #endif 778 /* 779 * Cumulative resource counters for dead threads in the group, 780 * and for reaped dead child processes forked by this group. 781 * Live threads maintain their own counters and add to these 782 * in __exit_signal, except for the group leader. 783 */ 784 seqlock_t stats_lock; 785 cputime_t utime, stime, cutime, cstime; 786 cputime_t gtime; 787 cputime_t cgtime; 788 struct prev_cputime prev_cputime; 789 unsigned long nvcsw, nivcsw, cnvcsw, cnivcsw; 790 unsigned long min_flt, maj_flt, cmin_flt, cmaj_flt; 791 unsigned long inblock, oublock, cinblock, coublock; 792 unsigned long maxrss, cmaxrss; 793 struct task_io_accounting ioac; 794 795 /* 796 * Cumulative ns of schedule CPU time fo dead threads in the 797 * group, not including a zombie group leader, (This only differs 798 * from jiffies_to_ns(utime + stime) if sched_clock uses something 799 * other than jiffies.) 800 */ 801 unsigned long long sum_sched_runtime; 802 803 /* 804 * We don't bother to synchronize most readers of this at all, 805 * because there is no reader checking a limit that actually needs 806 * to get both rlim_cur and rlim_max atomically, and either one 807 * alone is a single word that can safely be read normally. 808 * getrlimit/setrlimit use task_lock(current->group_leader) to 809 * protect this instead of the siglock, because they really 810 * have no need to disable irqs. 811 */ 812 struct rlimit rlim[RLIM_NLIMITS]; 813 814 #ifdef CONFIG_BSD_PROCESS_ACCT 815 struct pacct_struct pacct; /* per-process accounting information */ 816 #endif 817 #ifdef CONFIG_TASKSTATS 818 struct taskstats *stats; 819 #endif 820 #ifdef CONFIG_AUDIT 821 unsigned audit_tty; 822 struct tty_audit_buf *tty_audit_buf; 823 #endif 824 825 /* 826 * Thread is the potential origin of an oom condition; kill first on 827 * oom 828 */ 829 bool oom_flag_origin; 830 short oom_score_adj; /* OOM kill score adjustment */ 831 short oom_score_adj_min; /* OOM kill score adjustment min value. 832 * Only settable by CAP_SYS_RESOURCE. */ 833 struct mm_struct *oom_mm; /* recorded mm when the thread group got 834 * killed by the oom killer */ 835 836 struct mutex cred_guard_mutex; /* guard against foreign influences on 837 * credential calculations 838 * (notably. ptrace) */ 839 }; 840 841 /* 842 * Bits in flags field of signal_struct. 843 */ 844 #define SIGNAL_STOP_STOPPED 0x00000001 /* job control stop in effect */ 845 #define SIGNAL_STOP_CONTINUED 0x00000002 /* SIGCONT since WCONTINUED reap */ 846 #define SIGNAL_GROUP_EXIT 0x00000004 /* group exit in progress */ 847 #define SIGNAL_GROUP_COREDUMP 0x00000008 /* coredump in progress */ 848 /* 849 * Pending notifications to parent. 850 */ 851 #define SIGNAL_CLD_STOPPED 0x00000010 852 #define SIGNAL_CLD_CONTINUED 0x00000020 853 #define SIGNAL_CLD_MASK (SIGNAL_CLD_STOPPED|SIGNAL_CLD_CONTINUED) 854 855 #define SIGNAL_UNKILLABLE 0x00000040 /* for init: ignore fatal signals */ 856 857 /* If true, all threads except ->group_exit_task have pending SIGKILL */ 858 static inline int signal_group_exit(const struct signal_struct *sig) 859 { 860 return (sig->flags & SIGNAL_GROUP_EXIT) || 861 (sig->group_exit_task != NULL); 862 } 863 864 /* 865 * Some day this will be a full-fledged user tracking system.. 866 */ 867 struct user_struct { 868 atomic_t __count; /* reference count */ 869 atomic_t processes; /* How many processes does this user have? */ 870 atomic_t sigpending; /* How many pending signals does this user have? */ 871 #ifdef CONFIG_INOTIFY_USER 872 atomic_t inotify_watches; /* How many inotify watches does this user have? */ 873 atomic_t inotify_devs; /* How many inotify devs does this user have opened? */ 874 #endif 875 #ifdef CONFIG_FANOTIFY 876 atomic_t fanotify_listeners; 877 #endif 878 #ifdef CONFIG_EPOLL 879 atomic_long_t epoll_watches; /* The number of file descriptors currently watched */ 880 #endif 881 #ifdef CONFIG_POSIX_MQUEUE 882 /* protected by mq_lock */ 883 unsigned long mq_bytes; /* How many bytes can be allocated to mqueue? */ 884 #endif 885 unsigned long locked_shm; /* How many pages of mlocked shm ? */ 886 unsigned long unix_inflight; /* How many files in flight in unix sockets */ 887 atomic_long_t pipe_bufs; /* how many pages are allocated in pipe buffers */ 888 889 #ifdef CONFIG_KEYS 890 struct key *uid_keyring; /* UID specific keyring */ 891 struct key *session_keyring; /* UID's default session keyring */ 892 #endif 893 894 /* Hash table maintenance information */ 895 struct hlist_node uidhash_node; 896 kuid_t uid; 897 898 #if defined(CONFIG_PERF_EVENTS) || defined(CONFIG_BPF_SYSCALL) 899 atomic_long_t locked_vm; 900 #endif 901 }; 902 903 extern int uids_sysfs_init(void); 904 905 extern struct user_struct *find_user(kuid_t); 906 907 extern struct user_struct root_user; 908 #define INIT_USER (&root_user) 909 910 911 struct backing_dev_info; 912 struct reclaim_state; 913 914 #ifdef CONFIG_SCHED_INFO 915 struct sched_info { 916 /* cumulative counters */ 917 unsigned long pcount; /* # of times run on this cpu */ 918 unsigned long long run_delay; /* time spent waiting on a runqueue */ 919 920 /* timestamps */ 921 unsigned long long last_arrival,/* when we last ran on a cpu */ 922 last_queued; /* when we were last queued to run */ 923 }; 924 #endif /* CONFIG_SCHED_INFO */ 925 926 #ifdef CONFIG_TASK_DELAY_ACCT 927 struct task_delay_info { 928 spinlock_t lock; 929 unsigned int flags; /* Private per-task flags */ 930 931 /* For each stat XXX, add following, aligned appropriately 932 * 933 * struct timespec XXX_start, XXX_end; 934 * u64 XXX_delay; 935 * u32 XXX_count; 936 * 937 * Atomicity of updates to XXX_delay, XXX_count protected by 938 * single lock above (split into XXX_lock if contention is an issue). 939 */ 940 941 /* 942 * XXX_count is incremented on every XXX operation, the delay 943 * associated with the operation is added to XXX_delay. 944 * XXX_delay contains the accumulated delay time in nanoseconds. 945 */ 946 u64 blkio_start; /* Shared by blkio, swapin */ 947 u64 blkio_delay; /* wait for sync block io completion */ 948 u64 swapin_delay; /* wait for swapin block io completion */ 949 u32 blkio_count; /* total count of the number of sync block */ 950 /* io operations performed */ 951 u32 swapin_count; /* total count of the number of swapin block */ 952 /* io operations performed */ 953 954 u64 freepages_start; 955 u64 freepages_delay; /* wait for memory reclaim */ 956 u32 freepages_count; /* total count of memory reclaim */ 957 }; 958 #endif /* CONFIG_TASK_DELAY_ACCT */ 959 960 static inline int sched_info_on(void) 961 { 962 #ifdef CONFIG_SCHEDSTATS 963 return 1; 964 #elif defined(CONFIG_TASK_DELAY_ACCT) 965 extern int delayacct_on; 966 return delayacct_on; 967 #else 968 return 0; 969 #endif 970 } 971 972 #ifdef CONFIG_SCHEDSTATS 973 void force_schedstat_enabled(void); 974 #endif 975 976 enum cpu_idle_type { 977 CPU_IDLE, 978 CPU_NOT_IDLE, 979 CPU_NEWLY_IDLE, 980 CPU_MAX_IDLE_TYPES 981 }; 982 983 /* 984 * Integer metrics need fixed point arithmetic, e.g., sched/fair 985 * has a few: load, load_avg, util_avg, freq, and capacity. 986 * 987 * We define a basic fixed point arithmetic range, and then formalize 988 * all these metrics based on that basic range. 989 */ 990 # define SCHED_FIXEDPOINT_SHIFT 10 991 # define SCHED_FIXEDPOINT_SCALE (1L << SCHED_FIXEDPOINT_SHIFT) 992 993 /* 994 * Increase resolution of cpu_capacity calculations 995 */ 996 #define SCHED_CAPACITY_SHIFT SCHED_FIXEDPOINT_SHIFT 997 #define SCHED_CAPACITY_SCALE (1L << SCHED_CAPACITY_SHIFT) 998 999 /* 1000 * Wake-queues are lists of tasks with a pending wakeup, whose 1001 * callers have already marked the task as woken internally, 1002 * and can thus carry on. A common use case is being able to 1003 * do the wakeups once the corresponding user lock as been 1004 * released. 1005 * 1006 * We hold reference to each task in the list across the wakeup, 1007 * thus guaranteeing that the memory is still valid by the time 1008 * the actual wakeups are performed in wake_up_q(). 1009 * 1010 * One per task suffices, because there's never a need for a task to be 1011 * in two wake queues simultaneously; it is forbidden to abandon a task 1012 * in a wake queue (a call to wake_up_q() _must_ follow), so if a task is 1013 * already in a wake queue, the wakeup will happen soon and the second 1014 * waker can just skip it. 1015 * 1016 * The DEFINE_WAKE_Q macro declares and initializes the list head. 1017 * wake_up_q() does NOT reinitialize the list; it's expected to be 1018 * called near the end of a function, where the fact that the queue is 1019 * not used again will be easy to see by inspection. 1020 * 1021 * Note that this can cause spurious wakeups. schedule() callers 1022 * must ensure the call is done inside a loop, confirming that the 1023 * wakeup condition has in fact occurred. 1024 */ 1025 struct wake_q_node { 1026 struct wake_q_node *next; 1027 }; 1028 1029 struct wake_q_head { 1030 struct wake_q_node *first; 1031 struct wake_q_node **lastp; 1032 }; 1033 1034 #define WAKE_Q_TAIL ((struct wake_q_node *) 0x01) 1035 1036 #define DEFINE_WAKE_Q(name) \ 1037 struct wake_q_head name = { WAKE_Q_TAIL, &name.first } 1038 1039 extern void wake_q_add(struct wake_q_head *head, 1040 struct task_struct *task); 1041 extern void wake_up_q(struct wake_q_head *head); 1042 1043 /* 1044 * sched-domains (multiprocessor balancing) declarations: 1045 */ 1046 #ifdef CONFIG_SMP 1047 #define SD_LOAD_BALANCE 0x0001 /* Do load balancing on this domain. */ 1048 #define SD_BALANCE_NEWIDLE 0x0002 /* Balance when about to become idle */ 1049 #define SD_BALANCE_EXEC 0x0004 /* Balance on exec */ 1050 #define SD_BALANCE_FORK 0x0008 /* Balance on fork, clone */ 1051 #define SD_BALANCE_WAKE 0x0010 /* Balance on wakeup */ 1052 #define SD_WAKE_AFFINE 0x0020 /* Wake task to waking CPU */ 1053 #define SD_ASYM_CPUCAPACITY 0x0040 /* Groups have different max cpu capacities */ 1054 #define SD_SHARE_CPUCAPACITY 0x0080 /* Domain members share cpu capacity */ 1055 #define SD_SHARE_POWERDOMAIN 0x0100 /* Domain members share power domain */ 1056 #define SD_SHARE_PKG_RESOURCES 0x0200 /* Domain members share cpu pkg resources */ 1057 #define SD_SERIALIZE 0x0400 /* Only a single load balancing instance */ 1058 #define SD_ASYM_PACKING 0x0800 /* Place busy groups earlier in the domain */ 1059 #define SD_PREFER_SIBLING 0x1000 /* Prefer to place tasks in a sibling domain */ 1060 #define SD_OVERLAP 0x2000 /* sched_domains of this level overlap */ 1061 #define SD_NUMA 0x4000 /* cross-node balancing */ 1062 1063 #ifdef CONFIG_SCHED_SMT 1064 static inline int cpu_smt_flags(void) 1065 { 1066 return SD_SHARE_CPUCAPACITY | SD_SHARE_PKG_RESOURCES; 1067 } 1068 #endif 1069 1070 #ifdef CONFIG_SCHED_MC 1071 static inline int cpu_core_flags(void) 1072 { 1073 return SD_SHARE_PKG_RESOURCES; 1074 } 1075 #endif 1076 1077 #ifdef CONFIG_NUMA 1078 static inline int cpu_numa_flags(void) 1079 { 1080 return SD_NUMA; 1081 } 1082 #endif 1083 1084 extern int arch_asym_cpu_priority(int cpu); 1085 1086 struct sched_domain_attr { 1087 int relax_domain_level; 1088 }; 1089 1090 #define SD_ATTR_INIT (struct sched_domain_attr) { \ 1091 .relax_domain_level = -1, \ 1092 } 1093 1094 extern int sched_domain_level_max; 1095 1096 struct sched_group; 1097 1098 struct sched_domain_shared { 1099 atomic_t ref; 1100 atomic_t nr_busy_cpus; 1101 int has_idle_cores; 1102 }; 1103 1104 struct sched_domain { 1105 /* These fields must be setup */ 1106 struct sched_domain *parent; /* top domain must be null terminated */ 1107 struct sched_domain *child; /* bottom domain must be null terminated */ 1108 struct sched_group *groups; /* the balancing groups of the domain */ 1109 unsigned long min_interval; /* Minimum balance interval ms */ 1110 unsigned long max_interval; /* Maximum balance interval ms */ 1111 unsigned int busy_factor; /* less balancing by factor if busy */ 1112 unsigned int imbalance_pct; /* No balance until over watermark */ 1113 unsigned int cache_nice_tries; /* Leave cache hot tasks for # tries */ 1114 unsigned int busy_idx; 1115 unsigned int idle_idx; 1116 unsigned int newidle_idx; 1117 unsigned int wake_idx; 1118 unsigned int forkexec_idx; 1119 unsigned int smt_gain; 1120 1121 int nohz_idle; /* NOHZ IDLE status */ 1122 int flags; /* See SD_* */ 1123 int level; 1124 1125 /* Runtime fields. */ 1126 unsigned long last_balance; /* init to jiffies. units in jiffies */ 1127 unsigned int balance_interval; /* initialise to 1. units in ms. */ 1128 unsigned int nr_balance_failed; /* initialise to 0 */ 1129 1130 /* idle_balance() stats */ 1131 u64 max_newidle_lb_cost; 1132 unsigned long next_decay_max_lb_cost; 1133 1134 u64 avg_scan_cost; /* select_idle_sibling */ 1135 1136 #ifdef CONFIG_SCHEDSTATS 1137 /* load_balance() stats */ 1138 unsigned int lb_count[CPU_MAX_IDLE_TYPES]; 1139 unsigned int lb_failed[CPU_MAX_IDLE_TYPES]; 1140 unsigned int lb_balanced[CPU_MAX_IDLE_TYPES]; 1141 unsigned int lb_imbalance[CPU_MAX_IDLE_TYPES]; 1142 unsigned int lb_gained[CPU_MAX_IDLE_TYPES]; 1143 unsigned int lb_hot_gained[CPU_MAX_IDLE_TYPES]; 1144 unsigned int lb_nobusyg[CPU_MAX_IDLE_TYPES]; 1145 unsigned int lb_nobusyq[CPU_MAX_IDLE_TYPES]; 1146 1147 /* Active load balancing */ 1148 unsigned int alb_count; 1149 unsigned int alb_failed; 1150 unsigned int alb_pushed; 1151 1152 /* SD_BALANCE_EXEC stats */ 1153 unsigned int sbe_count; 1154 unsigned int sbe_balanced; 1155 unsigned int sbe_pushed; 1156 1157 /* SD_BALANCE_FORK stats */ 1158 unsigned int sbf_count; 1159 unsigned int sbf_balanced; 1160 unsigned int sbf_pushed; 1161 1162 /* try_to_wake_up() stats */ 1163 unsigned int ttwu_wake_remote; 1164 unsigned int ttwu_move_affine; 1165 unsigned int ttwu_move_balance; 1166 #endif 1167 #ifdef CONFIG_SCHED_DEBUG 1168 char *name; 1169 #endif 1170 union { 1171 void *private; /* used during construction */ 1172 struct rcu_head rcu; /* used during destruction */ 1173 }; 1174 struct sched_domain_shared *shared; 1175 1176 unsigned int span_weight; 1177 /* 1178 * Span of all CPUs in this domain. 1179 * 1180 * NOTE: this field is variable length. (Allocated dynamically 1181 * by attaching extra space to the end of the structure, 1182 * depending on how many CPUs the kernel has booted up with) 1183 */ 1184 unsigned long span[0]; 1185 }; 1186 1187 static inline struct cpumask *sched_domain_span(struct sched_domain *sd) 1188 { 1189 return to_cpumask(sd->span); 1190 } 1191 1192 extern void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[], 1193 struct sched_domain_attr *dattr_new); 1194 1195 /* Allocate an array of sched domains, for partition_sched_domains(). */ 1196 cpumask_var_t *alloc_sched_domains(unsigned int ndoms); 1197 void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms); 1198 1199 bool cpus_share_cache(int this_cpu, int that_cpu); 1200 1201 typedef const struct cpumask *(*sched_domain_mask_f)(int cpu); 1202 typedef int (*sched_domain_flags_f)(void); 1203 1204 #define SDTL_OVERLAP 0x01 1205 1206 struct sd_data { 1207 struct sched_domain **__percpu sd; 1208 struct sched_domain_shared **__percpu sds; 1209 struct sched_group **__percpu sg; 1210 struct sched_group_capacity **__percpu sgc; 1211 }; 1212 1213 struct sched_domain_topology_level { 1214 sched_domain_mask_f mask; 1215 sched_domain_flags_f sd_flags; 1216 int flags; 1217 int numa_level; 1218 struct sd_data data; 1219 #ifdef CONFIG_SCHED_DEBUG 1220 char *name; 1221 #endif 1222 }; 1223 1224 extern void set_sched_topology(struct sched_domain_topology_level *tl); 1225 extern void wake_up_if_idle(int cpu); 1226 1227 #ifdef CONFIG_SCHED_DEBUG 1228 # define SD_INIT_NAME(type) .name = #type 1229 #else 1230 # define SD_INIT_NAME(type) 1231 #endif 1232 1233 #else /* CONFIG_SMP */ 1234 1235 struct sched_domain_attr; 1236 1237 static inline void 1238 partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[], 1239 struct sched_domain_attr *dattr_new) 1240 { 1241 } 1242 1243 static inline bool cpus_share_cache(int this_cpu, int that_cpu) 1244 { 1245 return true; 1246 } 1247 1248 #endif /* !CONFIG_SMP */ 1249 1250 1251 struct io_context; /* See blkdev.h */ 1252 1253 1254 #ifdef ARCH_HAS_PREFETCH_SWITCH_STACK 1255 extern void prefetch_stack(struct task_struct *t); 1256 #else 1257 static inline void prefetch_stack(struct task_struct *t) { } 1258 #endif 1259 1260 struct audit_context; /* See audit.c */ 1261 struct mempolicy; 1262 struct pipe_inode_info; 1263 struct uts_namespace; 1264 1265 struct load_weight { 1266 unsigned long weight; 1267 u32 inv_weight; 1268 }; 1269 1270 /* 1271 * The load_avg/util_avg accumulates an infinite geometric series 1272 * (see __update_load_avg() in kernel/sched/fair.c). 1273 * 1274 * [load_avg definition] 1275 * 1276 * load_avg = runnable% * scale_load_down(load) 1277 * 1278 * where runnable% is the time ratio that a sched_entity is runnable. 1279 * For cfs_rq, it is the aggregated load_avg of all runnable and 1280 * blocked sched_entities. 1281 * 1282 * load_avg may also take frequency scaling into account: 1283 * 1284 * load_avg = runnable% * scale_load_down(load) * freq% 1285 * 1286 * where freq% is the CPU frequency normalized to the highest frequency. 1287 * 1288 * [util_avg definition] 1289 * 1290 * util_avg = running% * SCHED_CAPACITY_SCALE 1291 * 1292 * where running% is the time ratio that a sched_entity is running on 1293 * a CPU. For cfs_rq, it is the aggregated util_avg of all runnable 1294 * and blocked sched_entities. 1295 * 1296 * util_avg may also factor frequency scaling and CPU capacity scaling: 1297 * 1298 * util_avg = running% * SCHED_CAPACITY_SCALE * freq% * capacity% 1299 * 1300 * where freq% is the same as above, and capacity% is the CPU capacity 1301 * normalized to the greatest capacity (due to uarch differences, etc). 1302 * 1303 * N.B., the above ratios (runnable%, running%, freq%, and capacity%) 1304 * themselves are in the range of [0, 1]. To do fixed point arithmetics, 1305 * we therefore scale them to as large a range as necessary. This is for 1306 * example reflected by util_avg's SCHED_CAPACITY_SCALE. 1307 * 1308 * [Overflow issue] 1309 * 1310 * The 64-bit load_sum can have 4353082796 (=2^64/47742/88761) entities 1311 * with the highest load (=88761), always runnable on a single cfs_rq, 1312 * and should not overflow as the number already hits PID_MAX_LIMIT. 1313 * 1314 * For all other cases (including 32-bit kernels), struct load_weight's 1315 * weight will overflow first before we do, because: 1316 * 1317 * Max(load_avg) <= Max(load.weight) 1318 * 1319 * Then it is the load_weight's responsibility to consider overflow 1320 * issues. 1321 */ 1322 struct sched_avg { 1323 u64 last_update_time, load_sum; 1324 u32 util_sum, period_contrib; 1325 unsigned long load_avg, util_avg; 1326 }; 1327 1328 #ifdef CONFIG_SCHEDSTATS 1329 struct sched_statistics { 1330 u64 wait_start; 1331 u64 wait_max; 1332 u64 wait_count; 1333 u64 wait_sum; 1334 u64 iowait_count; 1335 u64 iowait_sum; 1336 1337 u64 sleep_start; 1338 u64 sleep_max; 1339 s64 sum_sleep_runtime; 1340 1341 u64 block_start; 1342 u64 block_max; 1343 u64 exec_max; 1344 u64 slice_max; 1345 1346 u64 nr_migrations_cold; 1347 u64 nr_failed_migrations_affine; 1348 u64 nr_failed_migrations_running; 1349 u64 nr_failed_migrations_hot; 1350 u64 nr_forced_migrations; 1351 1352 u64 nr_wakeups; 1353 u64 nr_wakeups_sync; 1354 u64 nr_wakeups_migrate; 1355 u64 nr_wakeups_local; 1356 u64 nr_wakeups_remote; 1357 u64 nr_wakeups_affine; 1358 u64 nr_wakeups_affine_attempts; 1359 u64 nr_wakeups_passive; 1360 u64 nr_wakeups_idle; 1361 }; 1362 #endif 1363 1364 struct sched_entity { 1365 struct load_weight load; /* for load-balancing */ 1366 struct rb_node run_node; 1367 struct list_head group_node; 1368 unsigned int on_rq; 1369 1370 u64 exec_start; 1371 u64 sum_exec_runtime; 1372 u64 vruntime; 1373 u64 prev_sum_exec_runtime; 1374 1375 u64 nr_migrations; 1376 1377 #ifdef CONFIG_SCHEDSTATS 1378 struct sched_statistics statistics; 1379 #endif 1380 1381 #ifdef CONFIG_FAIR_GROUP_SCHED 1382 int depth; 1383 struct sched_entity *parent; 1384 /* rq on which this entity is (to be) queued: */ 1385 struct cfs_rq *cfs_rq; 1386 /* rq "owned" by this entity/group: */ 1387 struct cfs_rq *my_q; 1388 #endif 1389 1390 #ifdef CONFIG_SMP 1391 /* 1392 * Per entity load average tracking. 1393 * 1394 * Put into separate cache line so it does not 1395 * collide with read-mostly values above. 1396 */ 1397 struct sched_avg avg ____cacheline_aligned_in_smp; 1398 #endif 1399 }; 1400 1401 struct sched_rt_entity { 1402 struct list_head run_list; 1403 unsigned long timeout; 1404 unsigned long watchdog_stamp; 1405 unsigned int time_slice; 1406 unsigned short on_rq; 1407 unsigned short on_list; 1408 1409 struct sched_rt_entity *back; 1410 #ifdef CONFIG_RT_GROUP_SCHED 1411 struct sched_rt_entity *parent; 1412 /* rq on which this entity is (to be) queued: */ 1413 struct rt_rq *rt_rq; 1414 /* rq "owned" by this entity/group: */ 1415 struct rt_rq *my_q; 1416 #endif 1417 }; 1418 1419 struct sched_dl_entity { 1420 struct rb_node rb_node; 1421 1422 /* 1423 * Original scheduling parameters. Copied here from sched_attr 1424 * during sched_setattr(), they will remain the same until 1425 * the next sched_setattr(). 1426 */ 1427 u64 dl_runtime; /* maximum runtime for each instance */ 1428 u64 dl_deadline; /* relative deadline of each instance */ 1429 u64 dl_period; /* separation of two instances (period) */ 1430 u64 dl_bw; /* dl_runtime / dl_deadline */ 1431 1432 /* 1433 * Actual scheduling parameters. Initialized with the values above, 1434 * they are continously updated during task execution. Note that 1435 * the remaining runtime could be < 0 in case we are in overrun. 1436 */ 1437 s64 runtime; /* remaining runtime for this instance */ 1438 u64 deadline; /* absolute deadline for this instance */ 1439 unsigned int flags; /* specifying the scheduler behaviour */ 1440 1441 /* 1442 * Some bool flags: 1443 * 1444 * @dl_throttled tells if we exhausted the runtime. If so, the 1445 * task has to wait for a replenishment to be performed at the 1446 * next firing of dl_timer. 1447 * 1448 * @dl_boosted tells if we are boosted due to DI. If so we are 1449 * outside bandwidth enforcement mechanism (but only until we 1450 * exit the critical section); 1451 * 1452 * @dl_yielded tells if task gave up the cpu before consuming 1453 * all its available runtime during the last job. 1454 */ 1455 int dl_throttled, dl_boosted, dl_yielded; 1456 1457 /* 1458 * Bandwidth enforcement timer. Each -deadline task has its 1459 * own bandwidth to be enforced, thus we need one timer per task. 1460 */ 1461 struct hrtimer dl_timer; 1462 }; 1463 1464 union rcu_special { 1465 struct { 1466 u8 blocked; 1467 u8 need_qs; 1468 u8 exp_need_qs; 1469 u8 pad; /* Otherwise the compiler can store garbage here. */ 1470 } b; /* Bits. */ 1471 u32 s; /* Set of bits. */ 1472 }; 1473 struct rcu_node; 1474 1475 enum perf_event_task_context { 1476 perf_invalid_context = -1, 1477 perf_hw_context = 0, 1478 perf_sw_context, 1479 perf_nr_task_contexts, 1480 }; 1481 1482 /* Track pages that require TLB flushes */ 1483 struct tlbflush_unmap_batch { 1484 /* 1485 * Each bit set is a CPU that potentially has a TLB entry for one of 1486 * the PFNs being flushed. See set_tlb_ubc_flush_pending(). 1487 */ 1488 struct cpumask cpumask; 1489 1490 /* True if any bit in cpumask is set */ 1491 bool flush_required; 1492 1493 /* 1494 * If true then the PTE was dirty when unmapped. The entry must be 1495 * flushed before IO is initiated or a stale TLB entry potentially 1496 * allows an update without redirtying the page. 1497 */ 1498 bool writable; 1499 }; 1500 1501 struct task_struct { 1502 #ifdef CONFIG_THREAD_INFO_IN_TASK 1503 /* 1504 * For reasons of header soup (see current_thread_info()), this 1505 * must be the first element of task_struct. 1506 */ 1507 struct thread_info thread_info; 1508 #endif 1509 volatile long state; /* -1 unrunnable, 0 runnable, >0 stopped */ 1510 void *stack; 1511 atomic_t usage; 1512 unsigned int flags; /* per process flags, defined below */ 1513 unsigned int ptrace; 1514 1515 #ifdef CONFIG_SMP 1516 struct llist_node wake_entry; 1517 int on_cpu; 1518 #ifdef CONFIG_THREAD_INFO_IN_TASK 1519 unsigned int cpu; /* current CPU */ 1520 #endif 1521 unsigned int wakee_flips; 1522 unsigned long wakee_flip_decay_ts; 1523 struct task_struct *last_wakee; 1524 1525 int wake_cpu; 1526 #endif 1527 int on_rq; 1528 1529 int prio, static_prio, normal_prio; 1530 unsigned int rt_priority; 1531 const struct sched_class *sched_class; 1532 struct sched_entity se; 1533 struct sched_rt_entity rt; 1534 #ifdef CONFIG_CGROUP_SCHED 1535 struct task_group *sched_task_group; 1536 #endif 1537 struct sched_dl_entity dl; 1538 1539 #ifdef CONFIG_PREEMPT_NOTIFIERS 1540 /* list of struct preempt_notifier: */ 1541 struct hlist_head preempt_notifiers; 1542 #endif 1543 1544 #ifdef CONFIG_BLK_DEV_IO_TRACE 1545 unsigned int btrace_seq; 1546 #endif 1547 1548 unsigned int policy; 1549 int nr_cpus_allowed; 1550 cpumask_t cpus_allowed; 1551 1552 #ifdef CONFIG_PREEMPT_RCU 1553 int rcu_read_lock_nesting; 1554 union rcu_special rcu_read_unlock_special; 1555 struct list_head rcu_node_entry; 1556 struct rcu_node *rcu_blocked_node; 1557 #endif /* #ifdef CONFIG_PREEMPT_RCU */ 1558 #ifdef CONFIG_TASKS_RCU 1559 unsigned long rcu_tasks_nvcsw; 1560 bool rcu_tasks_holdout; 1561 struct list_head rcu_tasks_holdout_list; 1562 int rcu_tasks_idle_cpu; 1563 #endif /* #ifdef CONFIG_TASKS_RCU */ 1564 1565 #ifdef CONFIG_SCHED_INFO 1566 struct sched_info sched_info; 1567 #endif 1568 1569 struct list_head tasks; 1570 #ifdef CONFIG_SMP 1571 struct plist_node pushable_tasks; 1572 struct rb_node pushable_dl_tasks; 1573 #endif 1574 1575 struct mm_struct *mm, *active_mm; 1576 /* per-thread vma caching */ 1577 u32 vmacache_seqnum; 1578 struct vm_area_struct *vmacache[VMACACHE_SIZE]; 1579 #if defined(SPLIT_RSS_COUNTING) 1580 struct task_rss_stat rss_stat; 1581 #endif 1582 /* task state */ 1583 int exit_state; 1584 int exit_code, exit_signal; 1585 int pdeath_signal; /* The signal sent when the parent dies */ 1586 unsigned long jobctl; /* JOBCTL_*, siglock protected */ 1587 1588 /* Used for emulating ABI behavior of previous Linux versions */ 1589 unsigned int personality; 1590 1591 /* scheduler bits, serialized by scheduler locks */ 1592 unsigned sched_reset_on_fork:1; 1593 unsigned sched_contributes_to_load:1; 1594 unsigned sched_migrated:1; 1595 unsigned sched_remote_wakeup:1; 1596 unsigned :0; /* force alignment to the next boundary */ 1597 1598 /* unserialized, strictly 'current' */ 1599 unsigned in_execve:1; /* bit to tell LSMs we're in execve */ 1600 unsigned in_iowait:1; 1601 #if !defined(TIF_RESTORE_SIGMASK) 1602 unsigned restore_sigmask:1; 1603 #endif 1604 #ifdef CONFIG_MEMCG 1605 unsigned memcg_may_oom:1; 1606 #ifndef CONFIG_SLOB 1607 unsigned memcg_kmem_skip_account:1; 1608 #endif 1609 #endif 1610 #ifdef CONFIG_COMPAT_BRK 1611 unsigned brk_randomized:1; 1612 #endif 1613 1614 unsigned long atomic_flags; /* Flags needing atomic access. */ 1615 1616 struct restart_block restart_block; 1617 1618 pid_t pid; 1619 pid_t tgid; 1620 1621 #ifdef CONFIG_CC_STACKPROTECTOR 1622 /* Canary value for the -fstack-protector gcc feature */ 1623 unsigned long stack_canary; 1624 #endif 1625 /* 1626 * pointers to (original) parent process, youngest child, younger sibling, 1627 * older sibling, respectively. (p->father can be replaced with 1628 * p->real_parent->pid) 1629 */ 1630 struct task_struct __rcu *real_parent; /* real parent process */ 1631 struct task_struct __rcu *parent; /* recipient of SIGCHLD, wait4() reports */ 1632 /* 1633 * children/sibling forms the list of my natural children 1634 */ 1635 struct list_head children; /* list of my children */ 1636 struct list_head sibling; /* linkage in my parent's children list */ 1637 struct task_struct *group_leader; /* threadgroup leader */ 1638 1639 /* 1640 * ptraced is the list of tasks this task is using ptrace on. 1641 * This includes both natural children and PTRACE_ATTACH targets. 1642 * p->ptrace_entry is p's link on the p->parent->ptraced list. 1643 */ 1644 struct list_head ptraced; 1645 struct list_head ptrace_entry; 1646 1647 /* PID/PID hash table linkage. */ 1648 struct pid_link pids[PIDTYPE_MAX]; 1649 struct list_head thread_group; 1650 struct list_head thread_node; 1651 1652 struct completion *vfork_done; /* for vfork() */ 1653 int __user *set_child_tid; /* CLONE_CHILD_SETTID */ 1654 int __user *clear_child_tid; /* CLONE_CHILD_CLEARTID */ 1655 1656 cputime_t utime, stime; 1657 #ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME 1658 cputime_t utimescaled, stimescaled; 1659 #endif 1660 cputime_t gtime; 1661 struct prev_cputime prev_cputime; 1662 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN 1663 seqcount_t vtime_seqcount; 1664 unsigned long long vtime_snap; 1665 enum { 1666 /* Task is sleeping or running in a CPU with VTIME inactive */ 1667 VTIME_INACTIVE = 0, 1668 /* Task runs in userspace in a CPU with VTIME active */ 1669 VTIME_USER, 1670 /* Task runs in kernelspace in a CPU with VTIME active */ 1671 VTIME_SYS, 1672 } vtime_snap_whence; 1673 #endif 1674 1675 #ifdef CONFIG_NO_HZ_FULL 1676 atomic_t tick_dep_mask; 1677 #endif 1678 unsigned long nvcsw, nivcsw; /* context switch counts */ 1679 u64 start_time; /* monotonic time in nsec */ 1680 u64 real_start_time; /* boot based time in nsec */ 1681 /* mm fault and swap info: this can arguably be seen as either mm-specific or thread-specific */ 1682 unsigned long min_flt, maj_flt; 1683 1684 struct task_cputime cputime_expires; 1685 struct list_head cpu_timers[3]; 1686 1687 /* process credentials */ 1688 const struct cred __rcu *ptracer_cred; /* Tracer's credentials at attach */ 1689 const struct cred __rcu *real_cred; /* objective and real subjective task 1690 * credentials (COW) */ 1691 const struct cred __rcu *cred; /* effective (overridable) subjective task 1692 * credentials (COW) */ 1693 char comm[TASK_COMM_LEN]; /* executable name excluding path 1694 - access with [gs]et_task_comm (which lock 1695 it with task_lock()) 1696 - initialized normally by setup_new_exec */ 1697 /* file system info */ 1698 struct nameidata *nameidata; 1699 #ifdef CONFIG_SYSVIPC 1700 /* ipc stuff */ 1701 struct sysv_sem sysvsem; 1702 struct sysv_shm sysvshm; 1703 #endif 1704 #ifdef CONFIG_DETECT_HUNG_TASK 1705 /* hung task detection */ 1706 unsigned long last_switch_count; 1707 #endif 1708 /* filesystem information */ 1709 struct fs_struct *fs; 1710 /* open file information */ 1711 struct files_struct *files; 1712 /* namespaces */ 1713 struct nsproxy *nsproxy; 1714 /* signal handlers */ 1715 struct signal_struct *signal; 1716 struct sighand_struct *sighand; 1717 1718 sigset_t blocked, real_blocked; 1719 sigset_t saved_sigmask; /* restored if set_restore_sigmask() was used */ 1720 struct sigpending pending; 1721 1722 unsigned long sas_ss_sp; 1723 size_t sas_ss_size; 1724 unsigned sas_ss_flags; 1725 1726 struct callback_head *task_works; 1727 1728 struct audit_context *audit_context; 1729 #ifdef CONFIG_AUDITSYSCALL 1730 kuid_t loginuid; 1731 unsigned int sessionid; 1732 #endif 1733 struct seccomp seccomp; 1734 1735 /* Thread group tracking */ 1736 u32 parent_exec_id; 1737 u32 self_exec_id; 1738 /* Protection of (de-)allocation: mm, files, fs, tty, keyrings, mems_allowed, 1739 * mempolicy */ 1740 spinlock_t alloc_lock; 1741 1742 /* Protection of the PI data structures: */ 1743 raw_spinlock_t pi_lock; 1744 1745 struct wake_q_node wake_q; 1746 1747 #ifdef CONFIG_RT_MUTEXES 1748 /* PI waiters blocked on a rt_mutex held by this task */ 1749 struct rb_root pi_waiters; 1750 struct rb_node *pi_waiters_leftmost; 1751 /* Deadlock detection and priority inheritance handling */ 1752 struct rt_mutex_waiter *pi_blocked_on; 1753 #endif 1754 1755 #ifdef CONFIG_DEBUG_MUTEXES 1756 /* mutex deadlock detection */ 1757 struct mutex_waiter *blocked_on; 1758 #endif 1759 #ifdef CONFIG_TRACE_IRQFLAGS 1760 unsigned int irq_events; 1761 unsigned long hardirq_enable_ip; 1762 unsigned long hardirq_disable_ip; 1763 unsigned int hardirq_enable_event; 1764 unsigned int hardirq_disable_event; 1765 int hardirqs_enabled; 1766 int hardirq_context; 1767 unsigned long softirq_disable_ip; 1768 unsigned long softirq_enable_ip; 1769 unsigned int softirq_disable_event; 1770 unsigned int softirq_enable_event; 1771 int softirqs_enabled; 1772 int softirq_context; 1773 #endif 1774 #ifdef CONFIG_LOCKDEP 1775 # define MAX_LOCK_DEPTH 48UL 1776 u64 curr_chain_key; 1777 int lockdep_depth; 1778 unsigned int lockdep_recursion; 1779 struct held_lock held_locks[MAX_LOCK_DEPTH]; 1780 gfp_t lockdep_reclaim_gfp; 1781 #endif 1782 #ifdef CONFIG_UBSAN 1783 unsigned int in_ubsan; 1784 #endif 1785 1786 /* journalling filesystem info */ 1787 void *journal_info; 1788 1789 /* stacked block device info */ 1790 struct bio_list *bio_list; 1791 1792 #ifdef CONFIG_BLOCK 1793 /* stack plugging */ 1794 struct blk_plug *plug; 1795 #endif 1796 1797 /* VM state */ 1798 struct reclaim_state *reclaim_state; 1799 1800 struct backing_dev_info *backing_dev_info; 1801 1802 struct io_context *io_context; 1803 1804 unsigned long ptrace_message; 1805 siginfo_t *last_siginfo; /* For ptrace use. */ 1806 struct task_io_accounting ioac; 1807 #if defined(CONFIG_TASK_XACCT) 1808 u64 acct_rss_mem1; /* accumulated rss usage */ 1809 u64 acct_vm_mem1; /* accumulated virtual memory usage */ 1810 cputime_t acct_timexpd; /* stime + utime since last update */ 1811 #endif 1812 #ifdef CONFIG_CPUSETS 1813 nodemask_t mems_allowed; /* Protected by alloc_lock */ 1814 seqcount_t mems_allowed_seq; /* Seqence no to catch updates */ 1815 int cpuset_mem_spread_rotor; 1816 int cpuset_slab_spread_rotor; 1817 #endif 1818 #ifdef CONFIG_CGROUPS 1819 /* Control Group info protected by css_set_lock */ 1820 struct css_set __rcu *cgroups; 1821 /* cg_list protected by css_set_lock and tsk->alloc_lock */ 1822 struct list_head cg_list; 1823 #endif 1824 #ifdef CONFIG_FUTEX 1825 struct robust_list_head __user *robust_list; 1826 #ifdef CONFIG_COMPAT 1827 struct compat_robust_list_head __user *compat_robust_list; 1828 #endif 1829 struct list_head pi_state_list; 1830 struct futex_pi_state *pi_state_cache; 1831 #endif 1832 #ifdef CONFIG_PERF_EVENTS 1833 struct perf_event_context *perf_event_ctxp[perf_nr_task_contexts]; 1834 struct mutex perf_event_mutex; 1835 struct list_head perf_event_list; 1836 #endif 1837 #ifdef CONFIG_DEBUG_PREEMPT 1838 unsigned long preempt_disable_ip; 1839 #endif 1840 #ifdef CONFIG_NUMA 1841 struct mempolicy *mempolicy; /* Protected by alloc_lock */ 1842 short il_next; 1843 short pref_node_fork; 1844 #endif 1845 #ifdef CONFIG_NUMA_BALANCING 1846 int numa_scan_seq; 1847 unsigned int numa_scan_period; 1848 unsigned int numa_scan_period_max; 1849 int numa_preferred_nid; 1850 unsigned long numa_migrate_retry; 1851 u64 node_stamp; /* migration stamp */ 1852 u64 last_task_numa_placement; 1853 u64 last_sum_exec_runtime; 1854 struct callback_head numa_work; 1855 1856 struct list_head numa_entry; 1857 struct numa_group *numa_group; 1858 1859 /* 1860 * numa_faults is an array split into four regions: 1861 * faults_memory, faults_cpu, faults_memory_buffer, faults_cpu_buffer 1862 * in this precise order. 1863 * 1864 * faults_memory: Exponential decaying average of faults on a per-node 1865 * basis. Scheduling placement decisions are made based on these 1866 * counts. The values remain static for the duration of a PTE scan. 1867 * faults_cpu: Track the nodes the process was running on when a NUMA 1868 * hinting fault was incurred. 1869 * faults_memory_buffer and faults_cpu_buffer: Record faults per node 1870 * during the current scan window. When the scan completes, the counts 1871 * in faults_memory and faults_cpu decay and these values are copied. 1872 */ 1873 unsigned long *numa_faults; 1874 unsigned long total_numa_faults; 1875 1876 /* 1877 * numa_faults_locality tracks if faults recorded during the last 1878 * scan window were remote/local or failed to migrate. The task scan 1879 * period is adapted based on the locality of the faults with different 1880 * weights depending on whether they were shared or private faults 1881 */ 1882 unsigned long numa_faults_locality[3]; 1883 1884 unsigned long numa_pages_migrated; 1885 #endif /* CONFIG_NUMA_BALANCING */ 1886 1887 #ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH 1888 struct tlbflush_unmap_batch tlb_ubc; 1889 #endif 1890 1891 struct rcu_head rcu; 1892 1893 /* 1894 * cache last used pipe for splice 1895 */ 1896 struct pipe_inode_info *splice_pipe; 1897 1898 struct page_frag task_frag; 1899 1900 #ifdef CONFIG_TASK_DELAY_ACCT 1901 struct task_delay_info *delays; 1902 #endif 1903 #ifdef CONFIG_FAULT_INJECTION 1904 int make_it_fail; 1905 #endif 1906 /* 1907 * when (nr_dirtied >= nr_dirtied_pause), it's time to call 1908 * balance_dirty_pages() for some dirty throttling pause 1909 */ 1910 int nr_dirtied; 1911 int nr_dirtied_pause; 1912 unsigned long dirty_paused_when; /* start of a write-and-pause period */ 1913 1914 #ifdef CONFIG_LATENCYTOP 1915 int latency_record_count; 1916 struct latency_record latency_record[LT_SAVECOUNT]; 1917 #endif 1918 /* 1919 * time slack values; these are used to round up poll() and 1920 * select() etc timeout values. These are in nanoseconds. 1921 */ 1922 u64 timer_slack_ns; 1923 u64 default_timer_slack_ns; 1924 1925 #ifdef CONFIG_KASAN 1926 unsigned int kasan_depth; 1927 #endif 1928 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 1929 /* Index of current stored address in ret_stack */ 1930 int curr_ret_stack; 1931 /* Stack of return addresses for return function tracing */ 1932 struct ftrace_ret_stack *ret_stack; 1933 /* time stamp for last schedule */ 1934 unsigned long long ftrace_timestamp; 1935 /* 1936 * Number of functions that haven't been traced 1937 * because of depth overrun. 1938 */ 1939 atomic_t trace_overrun; 1940 /* Pause for the tracing */ 1941 atomic_t tracing_graph_pause; 1942 #endif 1943 #ifdef CONFIG_TRACING 1944 /* state flags for use by tracers */ 1945 unsigned long trace; 1946 /* bitmask and counter of trace recursion */ 1947 unsigned long trace_recursion; 1948 #endif /* CONFIG_TRACING */ 1949 #ifdef CONFIG_KCOV 1950 /* Coverage collection mode enabled for this task (0 if disabled). */ 1951 enum kcov_mode kcov_mode; 1952 /* Size of the kcov_area. */ 1953 unsigned kcov_size; 1954 /* Buffer for coverage collection. */ 1955 void *kcov_area; 1956 /* kcov desciptor wired with this task or NULL. */ 1957 struct kcov *kcov; 1958 #endif 1959 #ifdef CONFIG_MEMCG 1960 struct mem_cgroup *memcg_in_oom; 1961 gfp_t memcg_oom_gfp_mask; 1962 int memcg_oom_order; 1963 1964 /* number of pages to reclaim on returning to userland */ 1965 unsigned int memcg_nr_pages_over_high; 1966 #endif 1967 #ifdef CONFIG_UPROBES 1968 struct uprobe_task *utask; 1969 #endif 1970 #if defined(CONFIG_BCACHE) || defined(CONFIG_BCACHE_MODULE) 1971 unsigned int sequential_io; 1972 unsigned int sequential_io_avg; 1973 #endif 1974 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP 1975 unsigned long task_state_change; 1976 #endif 1977 int pagefault_disabled; 1978 #ifdef CONFIG_MMU 1979 struct task_struct *oom_reaper_list; 1980 #endif 1981 #ifdef CONFIG_VMAP_STACK 1982 struct vm_struct *stack_vm_area; 1983 #endif 1984 #ifdef CONFIG_THREAD_INFO_IN_TASK 1985 /* A live task holds one reference. */ 1986 atomic_t stack_refcount; 1987 #endif 1988 /* CPU-specific state of this task */ 1989 struct thread_struct thread; 1990 /* 1991 * WARNING: on x86, 'thread_struct' contains a variable-sized 1992 * structure. It *MUST* be at the end of 'task_struct'. 1993 * 1994 * Do not put anything below here! 1995 */ 1996 }; 1997 1998 #ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT 1999 extern int arch_task_struct_size __read_mostly; 2000 #else 2001 # define arch_task_struct_size (sizeof(struct task_struct)) 2002 #endif 2003 2004 #ifdef CONFIG_VMAP_STACK 2005 static inline struct vm_struct *task_stack_vm_area(const struct task_struct *t) 2006 { 2007 return t->stack_vm_area; 2008 } 2009 #else 2010 static inline struct vm_struct *task_stack_vm_area(const struct task_struct *t) 2011 { 2012 return NULL; 2013 } 2014 #endif 2015 2016 /* Future-safe accessor for struct task_struct's cpus_allowed. */ 2017 #define tsk_cpus_allowed(tsk) (&(tsk)->cpus_allowed) 2018 2019 static inline int tsk_nr_cpus_allowed(struct task_struct *p) 2020 { 2021 return p->nr_cpus_allowed; 2022 } 2023 2024 #define TNF_MIGRATED 0x01 2025 #define TNF_NO_GROUP 0x02 2026 #define TNF_SHARED 0x04 2027 #define TNF_FAULT_LOCAL 0x08 2028 #define TNF_MIGRATE_FAIL 0x10 2029 2030 static inline bool in_vfork(struct task_struct *tsk) 2031 { 2032 bool ret; 2033 2034 /* 2035 * need RCU to access ->real_parent if CLONE_VM was used along with 2036 * CLONE_PARENT. 2037 * 2038 * We check real_parent->mm == tsk->mm because CLONE_VFORK does not 2039 * imply CLONE_VM 2040 * 2041 * CLONE_VFORK can be used with CLONE_PARENT/CLONE_THREAD and thus 2042 * ->real_parent is not necessarily the task doing vfork(), so in 2043 * theory we can't rely on task_lock() if we want to dereference it. 2044 * 2045 * And in this case we can't trust the real_parent->mm == tsk->mm 2046 * check, it can be false negative. But we do not care, if init or 2047 * another oom-unkillable task does this it should blame itself. 2048 */ 2049 rcu_read_lock(); 2050 ret = tsk->vfork_done && tsk->real_parent->mm == tsk->mm; 2051 rcu_read_unlock(); 2052 2053 return ret; 2054 } 2055 2056 #ifdef CONFIG_NUMA_BALANCING 2057 extern void task_numa_fault(int last_node, int node, int pages, int flags); 2058 extern pid_t task_numa_group_id(struct task_struct *p); 2059 extern void set_numabalancing_state(bool enabled); 2060 extern void task_numa_free(struct task_struct *p); 2061 extern bool should_numa_migrate_memory(struct task_struct *p, struct page *page, 2062 int src_nid, int dst_cpu); 2063 #else 2064 static inline void task_numa_fault(int last_node, int node, int pages, 2065 int flags) 2066 { 2067 } 2068 static inline pid_t task_numa_group_id(struct task_struct *p) 2069 { 2070 return 0; 2071 } 2072 static inline void set_numabalancing_state(bool enabled) 2073 { 2074 } 2075 static inline void task_numa_free(struct task_struct *p) 2076 { 2077 } 2078 static inline bool should_numa_migrate_memory(struct task_struct *p, 2079 struct page *page, int src_nid, int dst_cpu) 2080 { 2081 return true; 2082 } 2083 #endif 2084 2085 static inline struct pid *task_pid(struct task_struct *task) 2086 { 2087 return task->pids[PIDTYPE_PID].pid; 2088 } 2089 2090 static inline struct pid *task_tgid(struct task_struct *task) 2091 { 2092 return task->group_leader->pids[PIDTYPE_PID].pid; 2093 } 2094 2095 /* 2096 * Without tasklist or rcu lock it is not safe to dereference 2097 * the result of task_pgrp/task_session even if task == current, 2098 * we can race with another thread doing sys_setsid/sys_setpgid. 2099 */ 2100 static inline struct pid *task_pgrp(struct task_struct *task) 2101 { 2102 return task->group_leader->pids[PIDTYPE_PGID].pid; 2103 } 2104 2105 static inline struct pid *task_session(struct task_struct *task) 2106 { 2107 return task->group_leader->pids[PIDTYPE_SID].pid; 2108 } 2109 2110 struct pid_namespace; 2111 2112 /* 2113 * the helpers to get the task's different pids as they are seen 2114 * from various namespaces 2115 * 2116 * task_xid_nr() : global id, i.e. the id seen from the init namespace; 2117 * task_xid_vnr() : virtual id, i.e. the id seen from the pid namespace of 2118 * current. 2119 * task_xid_nr_ns() : id seen from the ns specified; 2120 * 2121 * set_task_vxid() : assigns a virtual id to a task; 2122 * 2123 * see also pid_nr() etc in include/linux/pid.h 2124 */ 2125 pid_t __task_pid_nr_ns(struct task_struct *task, enum pid_type type, 2126 struct pid_namespace *ns); 2127 2128 static inline pid_t task_pid_nr(struct task_struct *tsk) 2129 { 2130 return tsk->pid; 2131 } 2132 2133 static inline pid_t task_pid_nr_ns(struct task_struct *tsk, 2134 struct pid_namespace *ns) 2135 { 2136 return __task_pid_nr_ns(tsk, PIDTYPE_PID, ns); 2137 } 2138 2139 static inline pid_t task_pid_vnr(struct task_struct *tsk) 2140 { 2141 return __task_pid_nr_ns(tsk, PIDTYPE_PID, NULL); 2142 } 2143 2144 2145 static inline pid_t task_tgid_nr(struct task_struct *tsk) 2146 { 2147 return tsk->tgid; 2148 } 2149 2150 pid_t task_tgid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns); 2151 2152 static inline pid_t task_tgid_vnr(struct task_struct *tsk) 2153 { 2154 return pid_vnr(task_tgid(tsk)); 2155 } 2156 2157 2158 static inline int pid_alive(const struct task_struct *p); 2159 static inline pid_t task_ppid_nr_ns(const struct task_struct *tsk, struct pid_namespace *ns) 2160 { 2161 pid_t pid = 0; 2162 2163 rcu_read_lock(); 2164 if (pid_alive(tsk)) 2165 pid = task_tgid_nr_ns(rcu_dereference(tsk->real_parent), ns); 2166 rcu_read_unlock(); 2167 2168 return pid; 2169 } 2170 2171 static inline pid_t task_ppid_nr(const struct task_struct *tsk) 2172 { 2173 return task_ppid_nr_ns(tsk, &init_pid_ns); 2174 } 2175 2176 static inline pid_t task_pgrp_nr_ns(struct task_struct *tsk, 2177 struct pid_namespace *ns) 2178 { 2179 return __task_pid_nr_ns(tsk, PIDTYPE_PGID, ns); 2180 } 2181 2182 static inline pid_t task_pgrp_vnr(struct task_struct *tsk) 2183 { 2184 return __task_pid_nr_ns(tsk, PIDTYPE_PGID, NULL); 2185 } 2186 2187 2188 static inline pid_t task_session_nr_ns(struct task_struct *tsk, 2189 struct pid_namespace *ns) 2190 { 2191 return __task_pid_nr_ns(tsk, PIDTYPE_SID, ns); 2192 } 2193 2194 static inline pid_t task_session_vnr(struct task_struct *tsk) 2195 { 2196 return __task_pid_nr_ns(tsk, PIDTYPE_SID, NULL); 2197 } 2198 2199 /* obsolete, do not use */ 2200 static inline pid_t task_pgrp_nr(struct task_struct *tsk) 2201 { 2202 return task_pgrp_nr_ns(tsk, &init_pid_ns); 2203 } 2204 2205 /** 2206 * pid_alive - check that a task structure is not stale 2207 * @p: Task structure to be checked. 2208 * 2209 * Test if a process is not yet dead (at most zombie state) 2210 * If pid_alive fails, then pointers within the task structure 2211 * can be stale and must not be dereferenced. 2212 * 2213 * Return: 1 if the process is alive. 0 otherwise. 2214 */ 2215 static inline int pid_alive(const struct task_struct *p) 2216 { 2217 return p->pids[PIDTYPE_PID].pid != NULL; 2218 } 2219 2220 /** 2221 * is_global_init - check if a task structure is init. Since init 2222 * is free to have sub-threads we need to check tgid. 2223 * @tsk: Task structure to be checked. 2224 * 2225 * Check if a task structure is the first user space task the kernel created. 2226 * 2227 * Return: 1 if the task structure is init. 0 otherwise. 2228 */ 2229 static inline int is_global_init(struct task_struct *tsk) 2230 { 2231 return task_tgid_nr(tsk) == 1; 2232 } 2233 2234 extern struct pid *cad_pid; 2235 2236 extern void free_task(struct task_struct *tsk); 2237 #define get_task_struct(tsk) do { atomic_inc(&(tsk)->usage); } while(0) 2238 2239 extern void __put_task_struct(struct task_struct *t); 2240 2241 static inline void put_task_struct(struct task_struct *t) 2242 { 2243 if (atomic_dec_and_test(&t->usage)) 2244 __put_task_struct(t); 2245 } 2246 2247 struct task_struct *task_rcu_dereference(struct task_struct **ptask); 2248 struct task_struct *try_get_task_struct(struct task_struct **ptask); 2249 2250 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN 2251 extern void task_cputime(struct task_struct *t, 2252 cputime_t *utime, cputime_t *stime); 2253 extern cputime_t task_gtime(struct task_struct *t); 2254 #else 2255 static inline void task_cputime(struct task_struct *t, 2256 cputime_t *utime, cputime_t *stime) 2257 { 2258 *utime = t->utime; 2259 *stime = t->stime; 2260 } 2261 2262 static inline cputime_t task_gtime(struct task_struct *t) 2263 { 2264 return t->gtime; 2265 } 2266 #endif 2267 2268 #ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME 2269 static inline void task_cputime_scaled(struct task_struct *t, 2270 cputime_t *utimescaled, 2271 cputime_t *stimescaled) 2272 { 2273 *utimescaled = t->utimescaled; 2274 *stimescaled = t->stimescaled; 2275 } 2276 #else 2277 static inline void task_cputime_scaled(struct task_struct *t, 2278 cputime_t *utimescaled, 2279 cputime_t *stimescaled) 2280 { 2281 task_cputime(t, utimescaled, stimescaled); 2282 } 2283 #endif 2284 2285 extern void task_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime_t *st); 2286 extern void thread_group_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime_t *st); 2287 2288 /* 2289 * Per process flags 2290 */ 2291 #define PF_IDLE 0x00000002 /* I am an IDLE thread */ 2292 #define PF_EXITING 0x00000004 /* getting shut down */ 2293 #define PF_EXITPIDONE 0x00000008 /* pi exit done on shut down */ 2294 #define PF_VCPU 0x00000010 /* I'm a virtual CPU */ 2295 #define PF_WQ_WORKER 0x00000020 /* I'm a workqueue worker */ 2296 #define PF_FORKNOEXEC 0x00000040 /* forked but didn't exec */ 2297 #define PF_MCE_PROCESS 0x00000080 /* process policy on mce errors */ 2298 #define PF_SUPERPRIV 0x00000100 /* used super-user privileges */ 2299 #define PF_DUMPCORE 0x00000200 /* dumped core */ 2300 #define PF_SIGNALED 0x00000400 /* killed by a signal */ 2301 #define PF_MEMALLOC 0x00000800 /* Allocating memory */ 2302 #define PF_NPROC_EXCEEDED 0x00001000 /* set_user noticed that RLIMIT_NPROC was exceeded */ 2303 #define PF_USED_MATH 0x00002000 /* if unset the fpu must be initialized before use */ 2304 #define PF_USED_ASYNC 0x00004000 /* used async_schedule*(), used by module init */ 2305 #define PF_NOFREEZE 0x00008000 /* this thread should not be frozen */ 2306 #define PF_FROZEN 0x00010000 /* frozen for system suspend */ 2307 #define PF_FSTRANS 0x00020000 /* inside a filesystem transaction */ 2308 #define PF_KSWAPD 0x00040000 /* I am kswapd */ 2309 #define PF_MEMALLOC_NOIO 0x00080000 /* Allocating memory without IO involved */ 2310 #define PF_LESS_THROTTLE 0x00100000 /* Throttle me less: I clean memory */ 2311 #define PF_KTHREAD 0x00200000 /* I am a kernel thread */ 2312 #define PF_RANDOMIZE 0x00400000 /* randomize virtual address space */ 2313 #define PF_SWAPWRITE 0x00800000 /* Allowed to write to swap */ 2314 #define PF_NO_SETAFFINITY 0x04000000 /* Userland is not allowed to meddle with cpus_allowed */ 2315 #define PF_MCE_EARLY 0x08000000 /* Early kill for mce process policy */ 2316 #define PF_MUTEX_TESTER 0x20000000 /* Thread belongs to the rt mutex tester */ 2317 #define PF_FREEZER_SKIP 0x40000000 /* Freezer should not count it as freezable */ 2318 #define PF_SUSPEND_TASK 0x80000000 /* this thread called freeze_processes and should not be frozen */ 2319 2320 /* 2321 * Only the _current_ task can read/write to tsk->flags, but other 2322 * tasks can access tsk->flags in readonly mode for example 2323 * with tsk_used_math (like during threaded core dumping). 2324 * There is however an exception to this rule during ptrace 2325 * or during fork: the ptracer task is allowed to write to the 2326 * child->flags of its traced child (same goes for fork, the parent 2327 * can write to the child->flags), because we're guaranteed the 2328 * child is not running and in turn not changing child->flags 2329 * at the same time the parent does it. 2330 */ 2331 #define clear_stopped_child_used_math(child) do { (child)->flags &= ~PF_USED_MATH; } while (0) 2332 #define set_stopped_child_used_math(child) do { (child)->flags |= PF_USED_MATH; } while (0) 2333 #define clear_used_math() clear_stopped_child_used_math(current) 2334 #define set_used_math() set_stopped_child_used_math(current) 2335 #define conditional_stopped_child_used_math(condition, child) \ 2336 do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= (condition) ? PF_USED_MATH : 0; } while (0) 2337 #define conditional_used_math(condition) \ 2338 conditional_stopped_child_used_math(condition, current) 2339 #define copy_to_stopped_child_used_math(child) \ 2340 do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= current->flags & PF_USED_MATH; } while (0) 2341 /* NOTE: this will return 0 or PF_USED_MATH, it will never return 1 */ 2342 #define tsk_used_math(p) ((p)->flags & PF_USED_MATH) 2343 #define used_math() tsk_used_math(current) 2344 2345 /* __GFP_IO isn't allowed if PF_MEMALLOC_NOIO is set in current->flags 2346 * __GFP_FS is also cleared as it implies __GFP_IO. 2347 */ 2348 static inline gfp_t memalloc_noio_flags(gfp_t flags) 2349 { 2350 if (unlikely(current->flags & PF_MEMALLOC_NOIO)) 2351 flags &= ~(__GFP_IO | __GFP_FS); 2352 return flags; 2353 } 2354 2355 static inline unsigned int memalloc_noio_save(void) 2356 { 2357 unsigned int flags = current->flags & PF_MEMALLOC_NOIO; 2358 current->flags |= PF_MEMALLOC_NOIO; 2359 return flags; 2360 } 2361 2362 static inline void memalloc_noio_restore(unsigned int flags) 2363 { 2364 current->flags = (current->flags & ~PF_MEMALLOC_NOIO) | flags; 2365 } 2366 2367 /* Per-process atomic flags. */ 2368 #define PFA_NO_NEW_PRIVS 0 /* May not gain new privileges. */ 2369 #define PFA_SPREAD_PAGE 1 /* Spread page cache over cpuset */ 2370 #define PFA_SPREAD_SLAB 2 /* Spread some slab caches over cpuset */ 2371 #define PFA_LMK_WAITING 3 /* Lowmemorykiller is waiting */ 2372 2373 2374 #define TASK_PFA_TEST(name, func) \ 2375 static inline bool task_##func(struct task_struct *p) \ 2376 { return test_bit(PFA_##name, &p->atomic_flags); } 2377 #define TASK_PFA_SET(name, func) \ 2378 static inline void task_set_##func(struct task_struct *p) \ 2379 { set_bit(PFA_##name, &p->atomic_flags); } 2380 #define TASK_PFA_CLEAR(name, func) \ 2381 static inline void task_clear_##func(struct task_struct *p) \ 2382 { clear_bit(PFA_##name, &p->atomic_flags); } 2383 2384 TASK_PFA_TEST(NO_NEW_PRIVS, no_new_privs) 2385 TASK_PFA_SET(NO_NEW_PRIVS, no_new_privs) 2386 2387 TASK_PFA_TEST(SPREAD_PAGE, spread_page) 2388 TASK_PFA_SET(SPREAD_PAGE, spread_page) 2389 TASK_PFA_CLEAR(SPREAD_PAGE, spread_page) 2390 2391 TASK_PFA_TEST(SPREAD_SLAB, spread_slab) 2392 TASK_PFA_SET(SPREAD_SLAB, spread_slab) 2393 TASK_PFA_CLEAR(SPREAD_SLAB, spread_slab) 2394 2395 TASK_PFA_TEST(LMK_WAITING, lmk_waiting) 2396 TASK_PFA_SET(LMK_WAITING, lmk_waiting) 2397 2398 /* 2399 * task->jobctl flags 2400 */ 2401 #define JOBCTL_STOP_SIGMASK 0xffff /* signr of the last group stop */ 2402 2403 #define JOBCTL_STOP_DEQUEUED_BIT 16 /* stop signal dequeued */ 2404 #define JOBCTL_STOP_PENDING_BIT 17 /* task should stop for group stop */ 2405 #define JOBCTL_STOP_CONSUME_BIT 18 /* consume group stop count */ 2406 #define JOBCTL_TRAP_STOP_BIT 19 /* trap for STOP */ 2407 #define JOBCTL_TRAP_NOTIFY_BIT 20 /* trap for NOTIFY */ 2408 #define JOBCTL_TRAPPING_BIT 21 /* switching to TRACED */ 2409 #define JOBCTL_LISTENING_BIT 22 /* ptracer is listening for events */ 2410 2411 #define JOBCTL_STOP_DEQUEUED (1UL << JOBCTL_STOP_DEQUEUED_BIT) 2412 #define JOBCTL_STOP_PENDING (1UL << JOBCTL_STOP_PENDING_BIT) 2413 #define JOBCTL_STOP_CONSUME (1UL << JOBCTL_STOP_CONSUME_BIT) 2414 #define JOBCTL_TRAP_STOP (1UL << JOBCTL_TRAP_STOP_BIT) 2415 #define JOBCTL_TRAP_NOTIFY (1UL << JOBCTL_TRAP_NOTIFY_BIT) 2416 #define JOBCTL_TRAPPING (1UL << JOBCTL_TRAPPING_BIT) 2417 #define JOBCTL_LISTENING (1UL << JOBCTL_LISTENING_BIT) 2418 2419 #define JOBCTL_TRAP_MASK (JOBCTL_TRAP_STOP | JOBCTL_TRAP_NOTIFY) 2420 #define JOBCTL_PENDING_MASK (JOBCTL_STOP_PENDING | JOBCTL_TRAP_MASK) 2421 2422 extern bool task_set_jobctl_pending(struct task_struct *task, 2423 unsigned long mask); 2424 extern void task_clear_jobctl_trapping(struct task_struct *task); 2425 extern void task_clear_jobctl_pending(struct task_struct *task, 2426 unsigned long mask); 2427 2428 static inline void rcu_copy_process(struct task_struct *p) 2429 { 2430 #ifdef CONFIG_PREEMPT_RCU 2431 p->rcu_read_lock_nesting = 0; 2432 p->rcu_read_unlock_special.s = 0; 2433 p->rcu_blocked_node = NULL; 2434 INIT_LIST_HEAD(&p->rcu_node_entry); 2435 #endif /* #ifdef CONFIG_PREEMPT_RCU */ 2436 #ifdef CONFIG_TASKS_RCU 2437 p->rcu_tasks_holdout = false; 2438 INIT_LIST_HEAD(&p->rcu_tasks_holdout_list); 2439 p->rcu_tasks_idle_cpu = -1; 2440 #endif /* #ifdef CONFIG_TASKS_RCU */ 2441 } 2442 2443 static inline void tsk_restore_flags(struct task_struct *task, 2444 unsigned long orig_flags, unsigned long flags) 2445 { 2446 task->flags &= ~flags; 2447 task->flags |= orig_flags & flags; 2448 } 2449 2450 extern int cpuset_cpumask_can_shrink(const struct cpumask *cur, 2451 const struct cpumask *trial); 2452 extern int task_can_attach(struct task_struct *p, 2453 const struct cpumask *cs_cpus_allowed); 2454 #ifdef CONFIG_SMP 2455 extern void do_set_cpus_allowed(struct task_struct *p, 2456 const struct cpumask *new_mask); 2457 2458 extern int set_cpus_allowed_ptr(struct task_struct *p, 2459 const struct cpumask *new_mask); 2460 #else 2461 static inline void do_set_cpus_allowed(struct task_struct *p, 2462 const struct cpumask *new_mask) 2463 { 2464 } 2465 static inline int set_cpus_allowed_ptr(struct task_struct *p, 2466 const struct cpumask *new_mask) 2467 { 2468 if (!cpumask_test_cpu(0, new_mask)) 2469 return -EINVAL; 2470 return 0; 2471 } 2472 #endif 2473 2474 #ifdef CONFIG_NO_HZ_COMMON 2475 void calc_load_enter_idle(void); 2476 void calc_load_exit_idle(void); 2477 #else 2478 static inline void calc_load_enter_idle(void) { } 2479 static inline void calc_load_exit_idle(void) { } 2480 #endif /* CONFIG_NO_HZ_COMMON */ 2481 2482 #ifndef cpu_relax_yield 2483 #define cpu_relax_yield() cpu_relax() 2484 #endif 2485 2486 /* 2487 * Do not use outside of architecture code which knows its limitations. 2488 * 2489 * sched_clock() has no promise of monotonicity or bounded drift between 2490 * CPUs, use (which you should not) requires disabling IRQs. 2491 * 2492 * Please use one of the three interfaces below. 2493 */ 2494 extern unsigned long long notrace sched_clock(void); 2495 /* 2496 * See the comment in kernel/sched/clock.c 2497 */ 2498 extern u64 running_clock(void); 2499 extern u64 sched_clock_cpu(int cpu); 2500 2501 2502 extern void sched_clock_init(void); 2503 2504 #ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK 2505 static inline void sched_clock_tick(void) 2506 { 2507 } 2508 2509 static inline void sched_clock_idle_sleep_event(void) 2510 { 2511 } 2512 2513 static inline void sched_clock_idle_wakeup_event(u64 delta_ns) 2514 { 2515 } 2516 2517 static inline u64 cpu_clock(int cpu) 2518 { 2519 return sched_clock(); 2520 } 2521 2522 static inline u64 local_clock(void) 2523 { 2524 return sched_clock(); 2525 } 2526 #else 2527 /* 2528 * Architectures can set this to 1 if they have specified 2529 * CONFIG_HAVE_UNSTABLE_SCHED_CLOCK in their arch Kconfig, 2530 * but then during bootup it turns out that sched_clock() 2531 * is reliable after all: 2532 */ 2533 extern int sched_clock_stable(void); 2534 extern void set_sched_clock_stable(void); 2535 extern void clear_sched_clock_stable(void); 2536 2537 extern void sched_clock_tick(void); 2538 extern void sched_clock_idle_sleep_event(void); 2539 extern void sched_clock_idle_wakeup_event(u64 delta_ns); 2540 2541 /* 2542 * As outlined in clock.c, provides a fast, high resolution, nanosecond 2543 * time source that is monotonic per cpu argument and has bounded drift 2544 * between cpus. 2545 * 2546 * ######################### BIG FAT WARNING ########################## 2547 * # when comparing cpu_clock(i) to cpu_clock(j) for i != j, time can # 2548 * # go backwards !! # 2549 * #################################################################### 2550 */ 2551 static inline u64 cpu_clock(int cpu) 2552 { 2553 return sched_clock_cpu(cpu); 2554 } 2555 2556 static inline u64 local_clock(void) 2557 { 2558 return sched_clock_cpu(raw_smp_processor_id()); 2559 } 2560 #endif 2561 2562 #ifdef CONFIG_IRQ_TIME_ACCOUNTING 2563 /* 2564 * An i/f to runtime opt-in for irq time accounting based off of sched_clock. 2565 * The reason for this explicit opt-in is not to have perf penalty with 2566 * slow sched_clocks. 2567 */ 2568 extern void enable_sched_clock_irqtime(void); 2569 extern void disable_sched_clock_irqtime(void); 2570 #else 2571 static inline void enable_sched_clock_irqtime(void) {} 2572 static inline void disable_sched_clock_irqtime(void) {} 2573 #endif 2574 2575 extern unsigned long long 2576 task_sched_runtime(struct task_struct *task); 2577 2578 /* sched_exec is called by processes performing an exec */ 2579 #ifdef CONFIG_SMP 2580 extern void sched_exec(void); 2581 #else 2582 #define sched_exec() {} 2583 #endif 2584 2585 extern void sched_clock_idle_sleep_event(void); 2586 extern void sched_clock_idle_wakeup_event(u64 delta_ns); 2587 2588 #ifdef CONFIG_HOTPLUG_CPU 2589 extern void idle_task_exit(void); 2590 #else 2591 static inline void idle_task_exit(void) {} 2592 #endif 2593 2594 #if defined(CONFIG_NO_HZ_COMMON) && defined(CONFIG_SMP) 2595 extern void wake_up_nohz_cpu(int cpu); 2596 #else 2597 static inline void wake_up_nohz_cpu(int cpu) { } 2598 #endif 2599 2600 #ifdef CONFIG_NO_HZ_FULL 2601 extern u64 scheduler_tick_max_deferment(void); 2602 #endif 2603 2604 #ifdef CONFIG_SCHED_AUTOGROUP 2605 extern void sched_autogroup_create_attach(struct task_struct *p); 2606 extern void sched_autogroup_detach(struct task_struct *p); 2607 extern void sched_autogroup_fork(struct signal_struct *sig); 2608 extern void sched_autogroup_exit(struct signal_struct *sig); 2609 extern void sched_autogroup_exit_task(struct task_struct *p); 2610 #ifdef CONFIG_PROC_FS 2611 extern void proc_sched_autogroup_show_task(struct task_struct *p, struct seq_file *m); 2612 extern int proc_sched_autogroup_set_nice(struct task_struct *p, int nice); 2613 #endif 2614 #else 2615 static inline void sched_autogroup_create_attach(struct task_struct *p) { } 2616 static inline void sched_autogroup_detach(struct task_struct *p) { } 2617 static inline void sched_autogroup_fork(struct signal_struct *sig) { } 2618 static inline void sched_autogroup_exit(struct signal_struct *sig) { } 2619 static inline void sched_autogroup_exit_task(struct task_struct *p) { } 2620 #endif 2621 2622 extern int yield_to(struct task_struct *p, bool preempt); 2623 extern void set_user_nice(struct task_struct *p, long nice); 2624 extern int task_prio(const struct task_struct *p); 2625 /** 2626 * task_nice - return the nice value of a given task. 2627 * @p: the task in question. 2628 * 2629 * Return: The nice value [ -20 ... 0 ... 19 ]. 2630 */ 2631 static inline int task_nice(const struct task_struct *p) 2632 { 2633 return PRIO_TO_NICE((p)->static_prio); 2634 } 2635 extern int can_nice(const struct task_struct *p, const int nice); 2636 extern int task_curr(const struct task_struct *p); 2637 extern int idle_cpu(int cpu); 2638 extern int sched_setscheduler(struct task_struct *, int, 2639 const struct sched_param *); 2640 extern int sched_setscheduler_nocheck(struct task_struct *, int, 2641 const struct sched_param *); 2642 extern int sched_setattr(struct task_struct *, 2643 const struct sched_attr *); 2644 extern struct task_struct *idle_task(int cpu); 2645 /** 2646 * is_idle_task - is the specified task an idle task? 2647 * @p: the task in question. 2648 * 2649 * Return: 1 if @p is an idle task. 0 otherwise. 2650 */ 2651 static inline bool is_idle_task(const struct task_struct *p) 2652 { 2653 return !!(p->flags & PF_IDLE); 2654 } 2655 extern struct task_struct *curr_task(int cpu); 2656 extern void ia64_set_curr_task(int cpu, struct task_struct *p); 2657 2658 void yield(void); 2659 2660 union thread_union { 2661 #ifndef CONFIG_THREAD_INFO_IN_TASK 2662 struct thread_info thread_info; 2663 #endif 2664 unsigned long stack[THREAD_SIZE/sizeof(long)]; 2665 }; 2666 2667 #ifndef __HAVE_ARCH_KSTACK_END 2668 static inline int kstack_end(void *addr) 2669 { 2670 /* Reliable end of stack detection: 2671 * Some APM bios versions misalign the stack 2672 */ 2673 return !(((unsigned long)addr+sizeof(void*)-1) & (THREAD_SIZE-sizeof(void*))); 2674 } 2675 #endif 2676 2677 extern union thread_union init_thread_union; 2678 extern struct task_struct init_task; 2679 2680 extern struct mm_struct init_mm; 2681 2682 extern struct pid_namespace init_pid_ns; 2683 2684 /* 2685 * find a task by one of its numerical ids 2686 * 2687 * find_task_by_pid_ns(): 2688 * finds a task by its pid in the specified namespace 2689 * find_task_by_vpid(): 2690 * finds a task by its virtual pid 2691 * 2692 * see also find_vpid() etc in include/linux/pid.h 2693 */ 2694 2695 extern struct task_struct *find_task_by_vpid(pid_t nr); 2696 extern struct task_struct *find_task_by_pid_ns(pid_t nr, 2697 struct pid_namespace *ns); 2698 2699 /* per-UID process charging. */ 2700 extern struct user_struct * alloc_uid(kuid_t); 2701 static inline struct user_struct *get_uid(struct user_struct *u) 2702 { 2703 atomic_inc(&u->__count); 2704 return u; 2705 } 2706 extern void free_uid(struct user_struct *); 2707 2708 #include <asm/current.h> 2709 2710 extern void xtime_update(unsigned long ticks); 2711 2712 extern int wake_up_state(struct task_struct *tsk, unsigned int state); 2713 extern int wake_up_process(struct task_struct *tsk); 2714 extern void wake_up_new_task(struct task_struct *tsk); 2715 #ifdef CONFIG_SMP 2716 extern void kick_process(struct task_struct *tsk); 2717 #else 2718 static inline void kick_process(struct task_struct *tsk) { } 2719 #endif 2720 extern int sched_fork(unsigned long clone_flags, struct task_struct *p); 2721 extern void sched_dead(struct task_struct *p); 2722 2723 extern void proc_caches_init(void); 2724 extern void flush_signals(struct task_struct *); 2725 extern void ignore_signals(struct task_struct *); 2726 extern void flush_signal_handlers(struct task_struct *, int force_default); 2727 extern int dequeue_signal(struct task_struct *tsk, sigset_t *mask, siginfo_t *info); 2728 2729 static inline int kernel_dequeue_signal(siginfo_t *info) 2730 { 2731 struct task_struct *tsk = current; 2732 siginfo_t __info; 2733 int ret; 2734 2735 spin_lock_irq(&tsk->sighand->siglock); 2736 ret = dequeue_signal(tsk, &tsk->blocked, info ?: &__info); 2737 spin_unlock_irq(&tsk->sighand->siglock); 2738 2739 return ret; 2740 } 2741 2742 static inline void kernel_signal_stop(void) 2743 { 2744 spin_lock_irq(¤t->sighand->siglock); 2745 if (current->jobctl & JOBCTL_STOP_DEQUEUED) 2746 __set_current_state(TASK_STOPPED); 2747 spin_unlock_irq(¤t->sighand->siglock); 2748 2749 schedule(); 2750 } 2751 2752 extern void release_task(struct task_struct * p); 2753 extern int send_sig_info(int, struct siginfo *, struct task_struct *); 2754 extern int force_sigsegv(int, struct task_struct *); 2755 extern int force_sig_info(int, struct siginfo *, struct task_struct *); 2756 extern int __kill_pgrp_info(int sig, struct siginfo *info, struct pid *pgrp); 2757 extern int kill_pid_info(int sig, struct siginfo *info, struct pid *pid); 2758 extern int kill_pid_info_as_cred(int, struct siginfo *, struct pid *, 2759 const struct cred *, u32); 2760 extern int kill_pgrp(struct pid *pid, int sig, int priv); 2761 extern int kill_pid(struct pid *pid, int sig, int priv); 2762 extern int kill_proc_info(int, struct siginfo *, pid_t); 2763 extern __must_check bool do_notify_parent(struct task_struct *, int); 2764 extern void __wake_up_parent(struct task_struct *p, struct task_struct *parent); 2765 extern void force_sig(int, struct task_struct *); 2766 extern int send_sig(int, struct task_struct *, int); 2767 extern int zap_other_threads(struct task_struct *p); 2768 extern struct sigqueue *sigqueue_alloc(void); 2769 extern void sigqueue_free(struct sigqueue *); 2770 extern int send_sigqueue(struct sigqueue *, struct task_struct *, int group); 2771 extern int do_sigaction(int, struct k_sigaction *, struct k_sigaction *); 2772 2773 #ifdef TIF_RESTORE_SIGMASK 2774 /* 2775 * Legacy restore_sigmask accessors. These are inefficient on 2776 * SMP architectures because they require atomic operations. 2777 */ 2778 2779 /** 2780 * set_restore_sigmask() - make sure saved_sigmask processing gets done 2781 * 2782 * This sets TIF_RESTORE_SIGMASK and ensures that the arch signal code 2783 * will run before returning to user mode, to process the flag. For 2784 * all callers, TIF_SIGPENDING is already set or it's no harm to set 2785 * it. TIF_RESTORE_SIGMASK need not be in the set of bits that the 2786 * arch code will notice on return to user mode, in case those bits 2787 * are scarce. We set TIF_SIGPENDING here to ensure that the arch 2788 * signal code always gets run when TIF_RESTORE_SIGMASK is set. 2789 */ 2790 static inline void set_restore_sigmask(void) 2791 { 2792 set_thread_flag(TIF_RESTORE_SIGMASK); 2793 WARN_ON(!test_thread_flag(TIF_SIGPENDING)); 2794 } 2795 static inline void clear_restore_sigmask(void) 2796 { 2797 clear_thread_flag(TIF_RESTORE_SIGMASK); 2798 } 2799 static inline bool test_restore_sigmask(void) 2800 { 2801 return test_thread_flag(TIF_RESTORE_SIGMASK); 2802 } 2803 static inline bool test_and_clear_restore_sigmask(void) 2804 { 2805 return test_and_clear_thread_flag(TIF_RESTORE_SIGMASK); 2806 } 2807 2808 #else /* TIF_RESTORE_SIGMASK */ 2809 2810 /* Higher-quality implementation, used if TIF_RESTORE_SIGMASK doesn't exist. */ 2811 static inline void set_restore_sigmask(void) 2812 { 2813 current->restore_sigmask = true; 2814 WARN_ON(!test_thread_flag(TIF_SIGPENDING)); 2815 } 2816 static inline void clear_restore_sigmask(void) 2817 { 2818 current->restore_sigmask = false; 2819 } 2820 static inline bool test_restore_sigmask(void) 2821 { 2822 return current->restore_sigmask; 2823 } 2824 static inline bool test_and_clear_restore_sigmask(void) 2825 { 2826 if (!current->restore_sigmask) 2827 return false; 2828 current->restore_sigmask = false; 2829 return true; 2830 } 2831 #endif 2832 2833 static inline void restore_saved_sigmask(void) 2834 { 2835 if (test_and_clear_restore_sigmask()) 2836 __set_current_blocked(¤t->saved_sigmask); 2837 } 2838 2839 static inline sigset_t *sigmask_to_save(void) 2840 { 2841 sigset_t *res = ¤t->blocked; 2842 if (unlikely(test_restore_sigmask())) 2843 res = ¤t->saved_sigmask; 2844 return res; 2845 } 2846 2847 static inline int kill_cad_pid(int sig, int priv) 2848 { 2849 return kill_pid(cad_pid, sig, priv); 2850 } 2851 2852 /* These can be the second arg to send_sig_info/send_group_sig_info. */ 2853 #define SEND_SIG_NOINFO ((struct siginfo *) 0) 2854 #define SEND_SIG_PRIV ((struct siginfo *) 1) 2855 #define SEND_SIG_FORCED ((struct siginfo *) 2) 2856 2857 /* 2858 * True if we are on the alternate signal stack. 2859 */ 2860 static inline int on_sig_stack(unsigned long sp) 2861 { 2862 /* 2863 * If the signal stack is SS_AUTODISARM then, by construction, we 2864 * can't be on the signal stack unless user code deliberately set 2865 * SS_AUTODISARM when we were already on it. 2866 * 2867 * This improves reliability: if user state gets corrupted such that 2868 * the stack pointer points very close to the end of the signal stack, 2869 * then this check will enable the signal to be handled anyway. 2870 */ 2871 if (current->sas_ss_flags & SS_AUTODISARM) 2872 return 0; 2873 2874 #ifdef CONFIG_STACK_GROWSUP 2875 return sp >= current->sas_ss_sp && 2876 sp - current->sas_ss_sp < current->sas_ss_size; 2877 #else 2878 return sp > current->sas_ss_sp && 2879 sp - current->sas_ss_sp <= current->sas_ss_size; 2880 #endif 2881 } 2882 2883 static inline int sas_ss_flags(unsigned long sp) 2884 { 2885 if (!current->sas_ss_size) 2886 return SS_DISABLE; 2887 2888 return on_sig_stack(sp) ? SS_ONSTACK : 0; 2889 } 2890 2891 static inline void sas_ss_reset(struct task_struct *p) 2892 { 2893 p->sas_ss_sp = 0; 2894 p->sas_ss_size = 0; 2895 p->sas_ss_flags = SS_DISABLE; 2896 } 2897 2898 static inline unsigned long sigsp(unsigned long sp, struct ksignal *ksig) 2899 { 2900 if (unlikely((ksig->ka.sa.sa_flags & SA_ONSTACK)) && ! sas_ss_flags(sp)) 2901 #ifdef CONFIG_STACK_GROWSUP 2902 return current->sas_ss_sp; 2903 #else 2904 return current->sas_ss_sp + current->sas_ss_size; 2905 #endif 2906 return sp; 2907 } 2908 2909 /* 2910 * Routines for handling mm_structs 2911 */ 2912 extern struct mm_struct * mm_alloc(void); 2913 2914 /* mmdrop drops the mm and the page tables */ 2915 extern void __mmdrop(struct mm_struct *); 2916 static inline void mmdrop(struct mm_struct *mm) 2917 { 2918 if (unlikely(atomic_dec_and_test(&mm->mm_count))) 2919 __mmdrop(mm); 2920 } 2921 2922 static inline void mmdrop_async_fn(struct work_struct *work) 2923 { 2924 struct mm_struct *mm = container_of(work, struct mm_struct, async_put_work); 2925 __mmdrop(mm); 2926 } 2927 2928 static inline void mmdrop_async(struct mm_struct *mm) 2929 { 2930 if (unlikely(atomic_dec_and_test(&mm->mm_count))) { 2931 INIT_WORK(&mm->async_put_work, mmdrop_async_fn); 2932 schedule_work(&mm->async_put_work); 2933 } 2934 } 2935 2936 static inline bool mmget_not_zero(struct mm_struct *mm) 2937 { 2938 return atomic_inc_not_zero(&mm->mm_users); 2939 } 2940 2941 /* mmput gets rid of the mappings and all user-space */ 2942 extern void mmput(struct mm_struct *); 2943 #ifdef CONFIG_MMU 2944 /* same as above but performs the slow path from the async context. Can 2945 * be called from the atomic context as well 2946 */ 2947 extern void mmput_async(struct mm_struct *); 2948 #endif 2949 2950 /* Grab a reference to a task's mm, if it is not already going away */ 2951 extern struct mm_struct *get_task_mm(struct task_struct *task); 2952 /* 2953 * Grab a reference to a task's mm, if it is not already going away 2954 * and ptrace_may_access with the mode parameter passed to it 2955 * succeeds. 2956 */ 2957 extern struct mm_struct *mm_access(struct task_struct *task, unsigned int mode); 2958 /* Remove the current tasks stale references to the old mm_struct */ 2959 extern void mm_release(struct task_struct *, struct mm_struct *); 2960 2961 #ifdef CONFIG_HAVE_COPY_THREAD_TLS 2962 extern int copy_thread_tls(unsigned long, unsigned long, unsigned long, 2963 struct task_struct *, unsigned long); 2964 #else 2965 extern int copy_thread(unsigned long, unsigned long, unsigned long, 2966 struct task_struct *); 2967 2968 /* Architectures that haven't opted into copy_thread_tls get the tls argument 2969 * via pt_regs, so ignore the tls argument passed via C. */ 2970 static inline int copy_thread_tls( 2971 unsigned long clone_flags, unsigned long sp, unsigned long arg, 2972 struct task_struct *p, unsigned long tls) 2973 { 2974 return copy_thread(clone_flags, sp, arg, p); 2975 } 2976 #endif 2977 extern void flush_thread(void); 2978 2979 #ifdef CONFIG_HAVE_EXIT_THREAD 2980 extern void exit_thread(struct task_struct *tsk); 2981 #else 2982 static inline void exit_thread(struct task_struct *tsk) 2983 { 2984 } 2985 #endif 2986 2987 extern void exit_files(struct task_struct *); 2988 extern void __cleanup_sighand(struct sighand_struct *); 2989 2990 extern void exit_itimers(struct signal_struct *); 2991 extern void flush_itimer_signals(void); 2992 2993 extern void do_group_exit(int); 2994 2995 extern int do_execve(struct filename *, 2996 const char __user * const __user *, 2997 const char __user * const __user *); 2998 extern int do_execveat(int, struct filename *, 2999 const char __user * const __user *, 3000 const char __user * const __user *, 3001 int); 3002 extern long _do_fork(unsigned long, unsigned long, unsigned long, int __user *, int __user *, unsigned long); 3003 extern long do_fork(unsigned long, unsigned long, unsigned long, int __user *, int __user *); 3004 struct task_struct *fork_idle(int); 3005 extern pid_t kernel_thread(int (*fn)(void *), void *arg, unsigned long flags); 3006 3007 extern void __set_task_comm(struct task_struct *tsk, const char *from, bool exec); 3008 static inline void set_task_comm(struct task_struct *tsk, const char *from) 3009 { 3010 __set_task_comm(tsk, from, false); 3011 } 3012 extern char *get_task_comm(char *to, struct task_struct *tsk); 3013 3014 #ifdef CONFIG_SMP 3015 void scheduler_ipi(void); 3016 extern unsigned long wait_task_inactive(struct task_struct *, long match_state); 3017 #else 3018 static inline void scheduler_ipi(void) { } 3019 static inline unsigned long wait_task_inactive(struct task_struct *p, 3020 long match_state) 3021 { 3022 return 1; 3023 } 3024 #endif 3025 3026 #define tasklist_empty() \ 3027 list_empty(&init_task.tasks) 3028 3029 #define next_task(p) \ 3030 list_entry_rcu((p)->tasks.next, struct task_struct, tasks) 3031 3032 #define for_each_process(p) \ 3033 for (p = &init_task ; (p = next_task(p)) != &init_task ; ) 3034 3035 extern bool current_is_single_threaded(void); 3036 3037 /* 3038 * Careful: do_each_thread/while_each_thread is a double loop so 3039 * 'break' will not work as expected - use goto instead. 3040 */ 3041 #define do_each_thread(g, t) \ 3042 for (g = t = &init_task ; (g = t = next_task(g)) != &init_task ; ) do 3043 3044 #define while_each_thread(g, t) \ 3045 while ((t = next_thread(t)) != g) 3046 3047 #define __for_each_thread(signal, t) \ 3048 list_for_each_entry_rcu(t, &(signal)->thread_head, thread_node) 3049 3050 #define for_each_thread(p, t) \ 3051 __for_each_thread((p)->signal, t) 3052 3053 /* Careful: this is a double loop, 'break' won't work as expected. */ 3054 #define for_each_process_thread(p, t) \ 3055 for_each_process(p) for_each_thread(p, t) 3056 3057 static inline int get_nr_threads(struct task_struct *tsk) 3058 { 3059 return tsk->signal->nr_threads; 3060 } 3061 3062 static inline bool thread_group_leader(struct task_struct *p) 3063 { 3064 return p->exit_signal >= 0; 3065 } 3066 3067 /* Do to the insanities of de_thread it is possible for a process 3068 * to have the pid of the thread group leader without actually being 3069 * the thread group leader. For iteration through the pids in proc 3070 * all we care about is that we have a task with the appropriate 3071 * pid, we don't actually care if we have the right task. 3072 */ 3073 static inline bool has_group_leader_pid(struct task_struct *p) 3074 { 3075 return task_pid(p) == p->signal->leader_pid; 3076 } 3077 3078 static inline 3079 bool same_thread_group(struct task_struct *p1, struct task_struct *p2) 3080 { 3081 return p1->signal == p2->signal; 3082 } 3083 3084 static inline struct task_struct *next_thread(const struct task_struct *p) 3085 { 3086 return list_entry_rcu(p->thread_group.next, 3087 struct task_struct, thread_group); 3088 } 3089 3090 static inline int thread_group_empty(struct task_struct *p) 3091 { 3092 return list_empty(&p->thread_group); 3093 } 3094 3095 #define delay_group_leader(p) \ 3096 (thread_group_leader(p) && !thread_group_empty(p)) 3097 3098 /* 3099 * Protects ->fs, ->files, ->mm, ->group_info, ->comm, keyring 3100 * subscriptions and synchronises with wait4(). Also used in procfs. Also 3101 * pins the final release of task.io_context. Also protects ->cpuset and 3102 * ->cgroup.subsys[]. And ->vfork_done. 3103 * 3104 * Nests both inside and outside of read_lock(&tasklist_lock). 3105 * It must not be nested with write_lock_irq(&tasklist_lock), 3106 * neither inside nor outside. 3107 */ 3108 static inline void task_lock(struct task_struct *p) 3109 { 3110 spin_lock(&p->alloc_lock); 3111 } 3112 3113 static inline void task_unlock(struct task_struct *p) 3114 { 3115 spin_unlock(&p->alloc_lock); 3116 } 3117 3118 extern struct sighand_struct *__lock_task_sighand(struct task_struct *tsk, 3119 unsigned long *flags); 3120 3121 static inline struct sighand_struct *lock_task_sighand(struct task_struct *tsk, 3122 unsigned long *flags) 3123 { 3124 struct sighand_struct *ret; 3125 3126 ret = __lock_task_sighand(tsk, flags); 3127 (void)__cond_lock(&tsk->sighand->siglock, ret); 3128 return ret; 3129 } 3130 3131 static inline void unlock_task_sighand(struct task_struct *tsk, 3132 unsigned long *flags) 3133 { 3134 spin_unlock_irqrestore(&tsk->sighand->siglock, *flags); 3135 } 3136 3137 /** 3138 * threadgroup_change_begin - mark the beginning of changes to a threadgroup 3139 * @tsk: task causing the changes 3140 * 3141 * All operations which modify a threadgroup - a new thread joining the 3142 * group, death of a member thread (the assertion of PF_EXITING) and 3143 * exec(2) dethreading the process and replacing the leader - are wrapped 3144 * by threadgroup_change_{begin|end}(). This is to provide a place which 3145 * subsystems needing threadgroup stability can hook into for 3146 * synchronization. 3147 */ 3148 static inline void threadgroup_change_begin(struct task_struct *tsk) 3149 { 3150 might_sleep(); 3151 cgroup_threadgroup_change_begin(tsk); 3152 } 3153 3154 /** 3155 * threadgroup_change_end - mark the end of changes to a threadgroup 3156 * @tsk: task causing the changes 3157 * 3158 * See threadgroup_change_begin(). 3159 */ 3160 static inline void threadgroup_change_end(struct task_struct *tsk) 3161 { 3162 cgroup_threadgroup_change_end(tsk); 3163 } 3164 3165 #ifdef CONFIG_THREAD_INFO_IN_TASK 3166 3167 static inline struct thread_info *task_thread_info(struct task_struct *task) 3168 { 3169 return &task->thread_info; 3170 } 3171 3172 /* 3173 * When accessing the stack of a non-current task that might exit, use 3174 * try_get_task_stack() instead. task_stack_page will return a pointer 3175 * that could get freed out from under you. 3176 */ 3177 static inline void *task_stack_page(const struct task_struct *task) 3178 { 3179 return task->stack; 3180 } 3181 3182 #define setup_thread_stack(new,old) do { } while(0) 3183 3184 static inline unsigned long *end_of_stack(const struct task_struct *task) 3185 { 3186 return task->stack; 3187 } 3188 3189 #elif !defined(__HAVE_THREAD_FUNCTIONS) 3190 3191 #define task_thread_info(task) ((struct thread_info *)(task)->stack) 3192 #define task_stack_page(task) ((void *)(task)->stack) 3193 3194 static inline void setup_thread_stack(struct task_struct *p, struct task_struct *org) 3195 { 3196 *task_thread_info(p) = *task_thread_info(org); 3197 task_thread_info(p)->task = p; 3198 } 3199 3200 /* 3201 * Return the address of the last usable long on the stack. 3202 * 3203 * When the stack grows down, this is just above the thread 3204 * info struct. Going any lower will corrupt the threadinfo. 3205 * 3206 * When the stack grows up, this is the highest address. 3207 * Beyond that position, we corrupt data on the next page. 3208 */ 3209 static inline unsigned long *end_of_stack(struct task_struct *p) 3210 { 3211 #ifdef CONFIG_STACK_GROWSUP 3212 return (unsigned long *)((unsigned long)task_thread_info(p) + THREAD_SIZE) - 1; 3213 #else 3214 return (unsigned long *)(task_thread_info(p) + 1); 3215 #endif 3216 } 3217 3218 #endif 3219 3220 #ifdef CONFIG_THREAD_INFO_IN_TASK 3221 static inline void *try_get_task_stack(struct task_struct *tsk) 3222 { 3223 return atomic_inc_not_zero(&tsk->stack_refcount) ? 3224 task_stack_page(tsk) : NULL; 3225 } 3226 3227 extern void put_task_stack(struct task_struct *tsk); 3228 #else 3229 static inline void *try_get_task_stack(struct task_struct *tsk) 3230 { 3231 return task_stack_page(tsk); 3232 } 3233 3234 static inline void put_task_stack(struct task_struct *tsk) {} 3235 #endif 3236 3237 #define task_stack_end_corrupted(task) \ 3238 (*(end_of_stack(task)) != STACK_END_MAGIC) 3239 3240 static inline int object_is_on_stack(void *obj) 3241 { 3242 void *stack = task_stack_page(current); 3243 3244 return (obj >= stack) && (obj < (stack + THREAD_SIZE)); 3245 } 3246 3247 extern void thread_stack_cache_init(void); 3248 3249 #ifdef CONFIG_DEBUG_STACK_USAGE 3250 static inline unsigned long stack_not_used(struct task_struct *p) 3251 { 3252 unsigned long *n = end_of_stack(p); 3253 3254 do { /* Skip over canary */ 3255 # ifdef CONFIG_STACK_GROWSUP 3256 n--; 3257 # else 3258 n++; 3259 # endif 3260 } while (!*n); 3261 3262 # ifdef CONFIG_STACK_GROWSUP 3263 return (unsigned long)end_of_stack(p) - (unsigned long)n; 3264 # else 3265 return (unsigned long)n - (unsigned long)end_of_stack(p); 3266 # endif 3267 } 3268 #endif 3269 extern void set_task_stack_end_magic(struct task_struct *tsk); 3270 3271 /* set thread flags in other task's structures 3272 * - see asm/thread_info.h for TIF_xxxx flags available 3273 */ 3274 static inline void set_tsk_thread_flag(struct task_struct *tsk, int flag) 3275 { 3276 set_ti_thread_flag(task_thread_info(tsk), flag); 3277 } 3278 3279 static inline void clear_tsk_thread_flag(struct task_struct *tsk, int flag) 3280 { 3281 clear_ti_thread_flag(task_thread_info(tsk), flag); 3282 } 3283 3284 static inline int test_and_set_tsk_thread_flag(struct task_struct *tsk, int flag) 3285 { 3286 return test_and_set_ti_thread_flag(task_thread_info(tsk), flag); 3287 } 3288 3289 static inline int test_and_clear_tsk_thread_flag(struct task_struct *tsk, int flag) 3290 { 3291 return test_and_clear_ti_thread_flag(task_thread_info(tsk), flag); 3292 } 3293 3294 static inline int test_tsk_thread_flag(struct task_struct *tsk, int flag) 3295 { 3296 return test_ti_thread_flag(task_thread_info(tsk), flag); 3297 } 3298 3299 static inline void set_tsk_need_resched(struct task_struct *tsk) 3300 { 3301 set_tsk_thread_flag(tsk,TIF_NEED_RESCHED); 3302 } 3303 3304 static inline void clear_tsk_need_resched(struct task_struct *tsk) 3305 { 3306 clear_tsk_thread_flag(tsk,TIF_NEED_RESCHED); 3307 } 3308 3309 static inline int test_tsk_need_resched(struct task_struct *tsk) 3310 { 3311 return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED)); 3312 } 3313 3314 static inline int restart_syscall(void) 3315 { 3316 set_tsk_thread_flag(current, TIF_SIGPENDING); 3317 return -ERESTARTNOINTR; 3318 } 3319 3320 static inline int signal_pending(struct task_struct *p) 3321 { 3322 return unlikely(test_tsk_thread_flag(p,TIF_SIGPENDING)); 3323 } 3324 3325 static inline int __fatal_signal_pending(struct task_struct *p) 3326 { 3327 return unlikely(sigismember(&p->pending.signal, SIGKILL)); 3328 } 3329 3330 static inline int fatal_signal_pending(struct task_struct *p) 3331 { 3332 return signal_pending(p) && __fatal_signal_pending(p); 3333 } 3334 3335 static inline int signal_pending_state(long state, struct task_struct *p) 3336 { 3337 if (!(state & (TASK_INTERRUPTIBLE | TASK_WAKEKILL))) 3338 return 0; 3339 if (!signal_pending(p)) 3340 return 0; 3341 3342 return (state & TASK_INTERRUPTIBLE) || __fatal_signal_pending(p); 3343 } 3344 3345 /* 3346 * cond_resched() and cond_resched_lock(): latency reduction via 3347 * explicit rescheduling in places that are safe. The return 3348 * value indicates whether a reschedule was done in fact. 3349 * cond_resched_lock() will drop the spinlock before scheduling, 3350 * cond_resched_softirq() will enable bhs before scheduling. 3351 */ 3352 #ifndef CONFIG_PREEMPT 3353 extern int _cond_resched(void); 3354 #else 3355 static inline int _cond_resched(void) { return 0; } 3356 #endif 3357 3358 #define cond_resched() ({ \ 3359 ___might_sleep(__FILE__, __LINE__, 0); \ 3360 _cond_resched(); \ 3361 }) 3362 3363 extern int __cond_resched_lock(spinlock_t *lock); 3364 3365 #define cond_resched_lock(lock) ({ \ 3366 ___might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET);\ 3367 __cond_resched_lock(lock); \ 3368 }) 3369 3370 extern int __cond_resched_softirq(void); 3371 3372 #define cond_resched_softirq() ({ \ 3373 ___might_sleep(__FILE__, __LINE__, SOFTIRQ_DISABLE_OFFSET); \ 3374 __cond_resched_softirq(); \ 3375 }) 3376 3377 static inline void cond_resched_rcu(void) 3378 { 3379 #if defined(CONFIG_DEBUG_ATOMIC_SLEEP) || !defined(CONFIG_PREEMPT_RCU) 3380 rcu_read_unlock(); 3381 cond_resched(); 3382 rcu_read_lock(); 3383 #endif 3384 } 3385 3386 static inline unsigned long get_preempt_disable_ip(struct task_struct *p) 3387 { 3388 #ifdef CONFIG_DEBUG_PREEMPT 3389 return p->preempt_disable_ip; 3390 #else 3391 return 0; 3392 #endif 3393 } 3394 3395 /* 3396 * Does a critical section need to be broken due to another 3397 * task waiting?: (technically does not depend on CONFIG_PREEMPT, 3398 * but a general need for low latency) 3399 */ 3400 static inline int spin_needbreak(spinlock_t *lock) 3401 { 3402 #ifdef CONFIG_PREEMPT 3403 return spin_is_contended(lock); 3404 #else 3405 return 0; 3406 #endif 3407 } 3408 3409 /* 3410 * Idle thread specific functions to determine the need_resched 3411 * polling state. 3412 */ 3413 #ifdef TIF_POLLING_NRFLAG 3414 static inline int tsk_is_polling(struct task_struct *p) 3415 { 3416 return test_tsk_thread_flag(p, TIF_POLLING_NRFLAG); 3417 } 3418 3419 static inline void __current_set_polling(void) 3420 { 3421 set_thread_flag(TIF_POLLING_NRFLAG); 3422 } 3423 3424 static inline bool __must_check current_set_polling_and_test(void) 3425 { 3426 __current_set_polling(); 3427 3428 /* 3429 * Polling state must be visible before we test NEED_RESCHED, 3430 * paired by resched_curr() 3431 */ 3432 smp_mb__after_atomic(); 3433 3434 return unlikely(tif_need_resched()); 3435 } 3436 3437 static inline void __current_clr_polling(void) 3438 { 3439 clear_thread_flag(TIF_POLLING_NRFLAG); 3440 } 3441 3442 static inline bool __must_check current_clr_polling_and_test(void) 3443 { 3444 __current_clr_polling(); 3445 3446 /* 3447 * Polling state must be visible before we test NEED_RESCHED, 3448 * paired by resched_curr() 3449 */ 3450 smp_mb__after_atomic(); 3451 3452 return unlikely(tif_need_resched()); 3453 } 3454 3455 #else 3456 static inline int tsk_is_polling(struct task_struct *p) { return 0; } 3457 static inline void __current_set_polling(void) { } 3458 static inline void __current_clr_polling(void) { } 3459 3460 static inline bool __must_check current_set_polling_and_test(void) 3461 { 3462 return unlikely(tif_need_resched()); 3463 } 3464 static inline bool __must_check current_clr_polling_and_test(void) 3465 { 3466 return unlikely(tif_need_resched()); 3467 } 3468 #endif 3469 3470 static inline void current_clr_polling(void) 3471 { 3472 __current_clr_polling(); 3473 3474 /* 3475 * Ensure we check TIF_NEED_RESCHED after we clear the polling bit. 3476 * Once the bit is cleared, we'll get IPIs with every new 3477 * TIF_NEED_RESCHED and the IPI handler, scheduler_ipi(), will also 3478 * fold. 3479 */ 3480 smp_mb(); /* paired with resched_curr() */ 3481 3482 preempt_fold_need_resched(); 3483 } 3484 3485 static __always_inline bool need_resched(void) 3486 { 3487 return unlikely(tif_need_resched()); 3488 } 3489 3490 /* 3491 * Thread group CPU time accounting. 3492 */ 3493 void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times); 3494 void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times); 3495 3496 /* 3497 * Reevaluate whether the task has signals pending delivery. 3498 * Wake the task if so. 3499 * This is required every time the blocked sigset_t changes. 3500 * callers must hold sighand->siglock. 3501 */ 3502 extern void recalc_sigpending_and_wake(struct task_struct *t); 3503 extern void recalc_sigpending(void); 3504 3505 extern void signal_wake_up_state(struct task_struct *t, unsigned int state); 3506 3507 static inline void signal_wake_up(struct task_struct *t, bool resume) 3508 { 3509 signal_wake_up_state(t, resume ? TASK_WAKEKILL : 0); 3510 } 3511 static inline void ptrace_signal_wake_up(struct task_struct *t, bool resume) 3512 { 3513 signal_wake_up_state(t, resume ? __TASK_TRACED : 0); 3514 } 3515 3516 /* 3517 * Wrappers for p->thread_info->cpu access. No-op on UP. 3518 */ 3519 #ifdef CONFIG_SMP 3520 3521 static inline unsigned int task_cpu(const struct task_struct *p) 3522 { 3523 #ifdef CONFIG_THREAD_INFO_IN_TASK 3524 return p->cpu; 3525 #else 3526 return task_thread_info(p)->cpu; 3527 #endif 3528 } 3529 3530 static inline int task_node(const struct task_struct *p) 3531 { 3532 return cpu_to_node(task_cpu(p)); 3533 } 3534 3535 extern void set_task_cpu(struct task_struct *p, unsigned int cpu); 3536 3537 #else 3538 3539 static inline unsigned int task_cpu(const struct task_struct *p) 3540 { 3541 return 0; 3542 } 3543 3544 static inline void set_task_cpu(struct task_struct *p, unsigned int cpu) 3545 { 3546 } 3547 3548 #endif /* CONFIG_SMP */ 3549 3550 /* 3551 * In order to reduce various lock holder preemption latencies provide an 3552 * interface to see if a vCPU is currently running or not. 3553 * 3554 * This allows us to terminate optimistic spin loops and block, analogous to 3555 * the native optimistic spin heuristic of testing if the lock owner task is 3556 * running or not. 3557 */ 3558 #ifndef vcpu_is_preempted 3559 # define vcpu_is_preempted(cpu) false 3560 #endif 3561 3562 extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask); 3563 extern long sched_getaffinity(pid_t pid, struct cpumask *mask); 3564 3565 #ifdef CONFIG_CGROUP_SCHED 3566 extern struct task_group root_task_group; 3567 #endif /* CONFIG_CGROUP_SCHED */ 3568 3569 extern int task_can_switch_user(struct user_struct *up, 3570 struct task_struct *tsk); 3571 3572 #ifdef CONFIG_TASK_XACCT 3573 static inline void add_rchar(struct task_struct *tsk, ssize_t amt) 3574 { 3575 tsk->ioac.rchar += amt; 3576 } 3577 3578 static inline void add_wchar(struct task_struct *tsk, ssize_t amt) 3579 { 3580 tsk->ioac.wchar += amt; 3581 } 3582 3583 static inline void inc_syscr(struct task_struct *tsk) 3584 { 3585 tsk->ioac.syscr++; 3586 } 3587 3588 static inline void inc_syscw(struct task_struct *tsk) 3589 { 3590 tsk->ioac.syscw++; 3591 } 3592 #else 3593 static inline void add_rchar(struct task_struct *tsk, ssize_t amt) 3594 { 3595 } 3596 3597 static inline void add_wchar(struct task_struct *tsk, ssize_t amt) 3598 { 3599 } 3600 3601 static inline void inc_syscr(struct task_struct *tsk) 3602 { 3603 } 3604 3605 static inline void inc_syscw(struct task_struct *tsk) 3606 { 3607 } 3608 #endif 3609 3610 #ifndef TASK_SIZE_OF 3611 #define TASK_SIZE_OF(tsk) TASK_SIZE 3612 #endif 3613 3614 #ifdef CONFIG_MEMCG 3615 extern void mm_update_next_owner(struct mm_struct *mm); 3616 #else 3617 static inline void mm_update_next_owner(struct mm_struct *mm) 3618 { 3619 } 3620 #endif /* CONFIG_MEMCG */ 3621 3622 static inline unsigned long task_rlimit(const struct task_struct *tsk, 3623 unsigned int limit) 3624 { 3625 return READ_ONCE(tsk->signal->rlim[limit].rlim_cur); 3626 } 3627 3628 static inline unsigned long task_rlimit_max(const struct task_struct *tsk, 3629 unsigned int limit) 3630 { 3631 return READ_ONCE(tsk->signal->rlim[limit].rlim_max); 3632 } 3633 3634 static inline unsigned long rlimit(unsigned int limit) 3635 { 3636 return task_rlimit(current, limit); 3637 } 3638 3639 static inline unsigned long rlimit_max(unsigned int limit) 3640 { 3641 return task_rlimit_max(current, limit); 3642 } 3643 3644 #define SCHED_CPUFREQ_RT (1U << 0) 3645 #define SCHED_CPUFREQ_DL (1U << 1) 3646 #define SCHED_CPUFREQ_IOWAIT (1U << 2) 3647 3648 #define SCHED_CPUFREQ_RT_DL (SCHED_CPUFREQ_RT | SCHED_CPUFREQ_DL) 3649 3650 #ifdef CONFIG_CPU_FREQ 3651 struct update_util_data { 3652 void (*func)(struct update_util_data *data, u64 time, unsigned int flags); 3653 }; 3654 3655 void cpufreq_add_update_util_hook(int cpu, struct update_util_data *data, 3656 void (*func)(struct update_util_data *data, u64 time, 3657 unsigned int flags)); 3658 void cpufreq_remove_update_util_hook(int cpu); 3659 #endif /* CONFIG_CPU_FREQ */ 3660 3661 #endif 3662