1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/kernel/exit.c 4 * 5 * Copyright (C) 1991, 1992 Linus Torvalds 6 */ 7 8 #include <linux/mm.h> 9 #include <linux/slab.h> 10 #include <linux/sched/autogroup.h> 11 #include <linux/sched/mm.h> 12 #include <linux/sched/stat.h> 13 #include <linux/sched/task.h> 14 #include <linux/sched/task_stack.h> 15 #include <linux/sched/cputime.h> 16 #include <linux/interrupt.h> 17 #include <linux/module.h> 18 #include <linux/capability.h> 19 #include <linux/completion.h> 20 #include <linux/personality.h> 21 #include <linux/tty.h> 22 #include <linux/iocontext.h> 23 #include <linux/key.h> 24 #include <linux/cpu.h> 25 #include <linux/acct.h> 26 #include <linux/tsacct_kern.h> 27 #include <linux/file.h> 28 #include <linux/freezer.h> 29 #include <linux/binfmts.h> 30 #include <linux/nsproxy.h> 31 #include <linux/pid_namespace.h> 32 #include <linux/ptrace.h> 33 #include <linux/profile.h> 34 #include <linux/mount.h> 35 #include <linux/proc_fs.h> 36 #include <linux/kthread.h> 37 #include <linux/mempolicy.h> 38 #include <linux/taskstats_kern.h> 39 #include <linux/delayacct.h> 40 #include <linux/cgroup.h> 41 #include <linux/syscalls.h> 42 #include <linux/signal.h> 43 #include <linux/posix-timers.h> 44 #include <linux/cn_proc.h> 45 #include <linux/mutex.h> 46 #include <linux/futex.h> 47 #include <linux/pipe_fs_i.h> 48 #include <linux/audit.h> /* for audit_free() */ 49 #include <linux/resource.h> 50 #include <linux/task_io_accounting_ops.h> 51 #include <linux/task_work.h> 52 #include <linux/fs_struct.h> 53 #include <linux/init_task.h> 54 #include <linux/perf_event.h> 55 #include <trace/events/sched.h> 56 #include <linux/hw_breakpoint.h> 57 #include <linux/oom.h> 58 #include <linux/writeback.h> 59 #include <linux/shm.h> 60 #include <linux/kcov.h> 61 #include <linux/kmsan.h> 62 #include <linux/random.h> 63 #include <linux/rcuwait.h> 64 #include <linux/compat.h> 65 #include <linux/io_uring.h> 66 #include <linux/kprobes.h> 67 #include <linux/rethook.h> 68 #include <linux/sysfs.h> 69 #include <linux/user_events.h> 70 #include <linux/unwind_deferred.h> 71 #include <linux/uaccess.h> 72 #include <linux/pidfs.h> 73 74 #include <uapi/linux/wait.h> 75 76 #include <asm/unistd.h> 77 #include <asm/mmu_context.h> 78 79 #include "exit.h" 80 81 /* 82 * The default value should be high enough to not crash a system that randomly 83 * crashes its kernel from time to time, but low enough to at least not permit 84 * overflowing 32-bit refcounts or the ldsem writer count. 85 */ 86 static unsigned int oops_limit = 10000; 87 88 #ifdef CONFIG_SYSCTL 89 static const struct ctl_table kern_exit_table[] = { 90 { 91 .procname = "oops_limit", 92 .data = &oops_limit, 93 .maxlen = sizeof(oops_limit), 94 .mode = 0644, 95 .proc_handler = proc_douintvec, 96 }, 97 }; 98 99 static __init int kernel_exit_sysctls_init(void) 100 { 101 register_sysctl_init("kernel", kern_exit_table); 102 return 0; 103 } 104 late_initcall(kernel_exit_sysctls_init); 105 #endif 106 107 static atomic_t oops_count = ATOMIC_INIT(0); 108 109 #ifdef CONFIG_SYSFS 110 static ssize_t oops_count_show(struct kobject *kobj, struct kobj_attribute *attr, 111 char *page) 112 { 113 return sysfs_emit(page, "%d\n", atomic_read(&oops_count)); 114 } 115 116 static struct kobj_attribute oops_count_attr = __ATTR_RO(oops_count); 117 118 static __init int kernel_exit_sysfs_init(void) 119 { 120 sysfs_add_file_to_group(kernel_kobj, &oops_count_attr.attr, NULL); 121 return 0; 122 } 123 late_initcall(kernel_exit_sysfs_init); 124 #endif 125 126 /* 127 * For things release_task() would like to do *after* tasklist_lock is released. 128 */ 129 struct release_task_post { 130 struct pid *pids[PIDTYPE_MAX]; 131 }; 132 133 static void __unhash_process(struct release_task_post *post, struct task_struct *p, 134 bool group_dead) 135 { 136 struct pid *pid = task_pid(p); 137 138 nr_threads--; 139 140 detach_pid(post->pids, p, PIDTYPE_PID); 141 wake_up_all(&pid->wait_pidfd); 142 143 if (group_dead) { 144 detach_pid(post->pids, p, PIDTYPE_TGID); 145 detach_pid(post->pids, p, PIDTYPE_PGID); 146 detach_pid(post->pids, p, PIDTYPE_SID); 147 148 list_del_rcu(&p->tasks); 149 list_del_init(&p->sibling); 150 __this_cpu_dec(process_counts); 151 } 152 list_del_rcu(&p->thread_node); 153 } 154 155 /* 156 * This function expects the tasklist_lock write-locked. 157 */ 158 static void __exit_signal(struct release_task_post *post, struct task_struct *tsk) 159 { 160 struct signal_struct *sig = tsk->signal; 161 bool group_dead = thread_group_leader(tsk); 162 struct sighand_struct *sighand; 163 struct tty_struct *tty; 164 u64 utime, stime; 165 166 sighand = rcu_dereference_check(tsk->sighand, 167 lockdep_tasklist_lock_is_held()); 168 spin_lock(&sighand->siglock); 169 170 #ifdef CONFIG_POSIX_TIMERS 171 posix_cpu_timers_exit(tsk); 172 if (group_dead) 173 posix_cpu_timers_exit_group(tsk); 174 #endif 175 176 if (group_dead) { 177 tty = sig->tty; 178 sig->tty = NULL; 179 } else { 180 /* 181 * If there is any task waiting for the group exit 182 * then notify it: 183 */ 184 if (sig->notify_count > 0 && !--sig->notify_count) 185 wake_up_process(sig->group_exec_task); 186 187 if (tsk == sig->curr_target) 188 sig->curr_target = next_thread(tsk); 189 } 190 191 /* 192 * Accumulate here the counters for all threads as they die. We could 193 * skip the group leader because it is the last user of signal_struct, 194 * but we want to avoid the race with thread_group_cputime() which can 195 * see the empty ->thread_head list. 196 */ 197 task_cputime(tsk, &utime, &stime); 198 write_seqlock(&sig->stats_lock); 199 sig->utime += utime; 200 sig->stime += stime; 201 sig->gtime += task_gtime(tsk); 202 sig->min_flt += tsk->min_flt; 203 sig->maj_flt += tsk->maj_flt; 204 sig->nvcsw += tsk->nvcsw; 205 sig->nivcsw += tsk->nivcsw; 206 sig->inblock += task_io_get_inblock(tsk); 207 sig->oublock += task_io_get_oublock(tsk); 208 task_io_accounting_add(&sig->ioac, &tsk->ioac); 209 sig->sum_sched_runtime += tsk->se.sum_exec_runtime; 210 sig->nr_threads--; 211 __unhash_process(post, tsk, group_dead); 212 write_sequnlock(&sig->stats_lock); 213 214 /* 215 * Ensure that all preceeding state is visible. Pairs with 216 * the smp_acquire__after_ctrl_dep() in the sighand == NULL 217 * path of lock_task_sighand(). 218 */ 219 smp_store_release(&tsk->sighand, NULL); 220 spin_unlock(&sighand->siglock); 221 222 __cleanup_sighand(sighand); 223 if (group_dead) 224 tty_kref_put(tty); 225 } 226 227 static void delayed_put_task_struct(struct rcu_head *rhp) 228 { 229 struct task_struct *tsk = container_of(rhp, struct task_struct, rcu); 230 231 kprobe_flush_task(tsk); 232 rethook_flush_task(tsk); 233 perf_event_delayed_put(tsk); 234 trace_sched_process_free(tsk); 235 put_task_struct(tsk); 236 } 237 238 void put_task_struct_rcu_user(struct task_struct *task) 239 { 240 if (refcount_dec_and_test(&task->rcu_users)) 241 call_rcu(&task->rcu, delayed_put_task_struct); 242 } 243 244 void __weak release_thread(struct task_struct *dead_task) 245 { 246 } 247 248 void release_task(struct task_struct *p) 249 { 250 struct release_task_post post; 251 struct task_struct *leader; 252 struct pid *thread_pid; 253 int zap_leader; 254 repeat: 255 memset(&post, 0, sizeof(post)); 256 257 /* don't need to get the RCU readlock here - the process is dead and 258 * can't be modifying its own credentials. */ 259 dec_rlimit_ucounts(task_ucounts(p), UCOUNT_RLIMIT_NPROC, 1); 260 261 pidfs_exit(p); 262 cgroup_task_release(p); 263 264 /* 265 * Pin @thread_pid before __unhash_process() clears it. The last 266 * PIDTYPE detach can otherwise free it before proc_flush_pid(). 267 */ 268 thread_pid = get_pid(task_pid(p)); 269 270 write_lock_irq(&tasklist_lock); 271 ptrace_release_task(p); 272 __exit_signal(&post, p); 273 274 /* 275 * If we are the last non-leader member of the thread 276 * group, and the leader is zombie, then notify the 277 * group leader's parent process. (if it wants notification.) 278 */ 279 zap_leader = 0; 280 leader = p->group_leader; 281 if (leader != p && thread_group_empty(leader) 282 && leader->exit_state == EXIT_ZOMBIE) { 283 /* for pidfs_exit() and do_notify_parent() */ 284 if (leader->signal->flags & SIGNAL_GROUP_EXIT) 285 leader->exit_code = leader->signal->group_exit_code; 286 /* 287 * If we were the last child thread and the leader has 288 * exited already, and the leader's parent ignores SIGCHLD, 289 * then we are the one who should release the leader. 290 */ 291 zap_leader = do_notify_parent(leader, leader->exit_signal); 292 if (zap_leader) 293 leader->exit_state = EXIT_DEAD; 294 } 295 296 write_unlock_irq(&tasklist_lock); 297 proc_flush_pid(thread_pid); 298 put_pid(thread_pid); 299 exit_cred_namespaces(p); 300 add_device_randomness(&p->se.sum_exec_runtime, 301 sizeof(p->se.sum_exec_runtime)); 302 free_pids(post.pids); 303 release_thread(p); 304 /* 305 * This task was already removed from the process/thread/pid lists and 306 * lock_task_sighand(p) can't succeed. If it's the group leader then 307 * flush tsk->signal->shared_pending. tsk->pending has been flushed 308 * already in exit_signals(). Nothing else can touch 309 * signal->shared_pending anymore, so flush_sigqueue() can be invoked 310 * lockless. 311 */ 312 if (thread_group_leader(p)) 313 flush_sigqueue(&p->signal->shared_pending); 314 315 put_task_struct_rcu_user(p); 316 317 p = leader; 318 if (unlikely(zap_leader)) 319 goto repeat; 320 } 321 322 int rcuwait_wake_up(struct rcuwait *w) 323 { 324 int ret = 0; 325 struct task_struct *task; 326 327 rcu_read_lock(); 328 329 /* 330 * Order condition vs @task, such that everything prior to the load 331 * of @task is visible. This is the condition as to why the user called 332 * rcuwait_wake() in the first place. Pairs with set_current_state() 333 * barrier (A) in rcuwait_wait_event(). 334 * 335 * WAIT WAKE 336 * [S] tsk = current [S] cond = true 337 * MB (A) MB (B) 338 * [L] cond [L] tsk 339 */ 340 smp_mb(); /* (B) */ 341 342 task = rcu_dereference(w->task); 343 if (task) 344 ret = wake_up_process(task); 345 rcu_read_unlock(); 346 347 return ret; 348 } 349 EXPORT_SYMBOL_GPL(rcuwait_wake_up); 350 351 /* 352 * Determine if a process group is "orphaned", according to the POSIX 353 * definition in 2.2.2.52. Orphaned process groups are not to be affected 354 * by terminal-generated stop signals. Newly orphaned process groups are 355 * to receive a SIGHUP and a SIGCONT. 356 * 357 * "I ask you, have you ever known what it is to be an orphan?" 358 */ 359 static int will_become_orphaned_pgrp(struct pid *pgrp, 360 struct task_struct *ignored_task) 361 { 362 struct task_struct *p; 363 364 do_each_pid_task(pgrp, PIDTYPE_PGID, p) { 365 if ((p == ignored_task) || 366 (p->exit_state && thread_group_empty(p)) || 367 is_global_init(p->real_parent)) 368 continue; 369 370 if (task_pgrp(p->real_parent) != pgrp && 371 task_session(p->real_parent) == task_session(p)) 372 return 0; 373 } while_each_pid_task(pgrp, PIDTYPE_PGID, p); 374 375 return 1; 376 } 377 378 int is_current_pgrp_orphaned(void) 379 { 380 int retval; 381 382 read_lock(&tasklist_lock); 383 retval = will_become_orphaned_pgrp(task_pgrp(current), NULL); 384 read_unlock(&tasklist_lock); 385 386 return retval; 387 } 388 389 static bool has_stopped_jobs(struct pid *pgrp) 390 { 391 struct task_struct *p; 392 393 do_each_pid_task(pgrp, PIDTYPE_PGID, p) { 394 if (p->signal->flags & SIGNAL_STOP_STOPPED) 395 return true; 396 } while_each_pid_task(pgrp, PIDTYPE_PGID, p); 397 398 return false; 399 } 400 401 /* 402 * Check to see if any process groups have become orphaned as 403 * a result of our exiting, and if they have any stopped jobs, 404 * send them a SIGHUP and then a SIGCONT. (POSIX 3.2.2.2) 405 */ 406 static void 407 kill_orphaned_pgrp(struct task_struct *tsk, struct task_struct *parent) 408 { 409 struct pid *pgrp = task_pgrp(tsk); 410 struct task_struct *ignored_task = tsk; 411 412 if (!parent) 413 /* exit: our father is in a different pgrp than 414 * we are and we were the only connection outside. 415 */ 416 parent = tsk->real_parent; 417 else 418 /* reparent: our child is in a different pgrp than 419 * we are, and it was the only connection outside. 420 */ 421 ignored_task = NULL; 422 423 if (task_pgrp(parent) != pgrp && 424 task_session(parent) == task_session(tsk) && 425 will_become_orphaned_pgrp(pgrp, ignored_task) && 426 has_stopped_jobs(pgrp)) { 427 __kill_pgrp_info(SIGHUP, SEND_SIG_PRIV, pgrp); 428 __kill_pgrp_info(SIGCONT, SEND_SIG_PRIV, pgrp); 429 } 430 } 431 432 static void coredump_task_exit(struct task_struct *tsk, 433 struct core_state *core_state) 434 { 435 struct core_thread self; 436 437 self.task = tsk; 438 if (self.task->flags & PF_SIGNALED) 439 self.next = xchg(&core_state->dumper.next, &self); 440 else 441 self.task = NULL; 442 /* 443 * Implies mb(), the result of xchg() must be visible 444 * to core_state->dumper. 445 */ 446 if (atomic_dec_and_test(&core_state->nr_threads)) 447 complete(&core_state->startup); 448 449 for (;;) { 450 set_current_state(TASK_IDLE|TASK_FREEZABLE); 451 if (!self.task) /* see coredump_finish() */ 452 break; 453 schedule(); 454 } 455 __set_current_state(TASK_RUNNING); 456 } 457 458 #ifdef CONFIG_MEMCG 459 /* drops tasklist_lock if succeeds */ 460 static bool __try_to_set_owner(struct task_struct *tsk, struct mm_struct *mm) 461 { 462 bool ret = false; 463 464 task_lock(tsk); 465 if (likely(tsk->mm == mm)) { 466 /* tsk can't pass exit_mm/exec_mmap and exit */ 467 read_unlock(&tasklist_lock); 468 WRITE_ONCE(mm->owner, tsk); 469 lru_gen_migrate_mm(mm); 470 ret = true; 471 } 472 task_unlock(tsk); 473 return ret; 474 } 475 476 static bool try_to_set_owner(struct task_struct *g, struct mm_struct *mm) 477 { 478 struct task_struct *t; 479 480 for_each_thread(g, t) { 481 struct mm_struct *t_mm = READ_ONCE(t->mm); 482 if (t_mm == mm) { 483 if (__try_to_set_owner(t, mm)) 484 return true; 485 } else if (t_mm) 486 break; 487 } 488 489 return false; 490 } 491 492 /* 493 * A task is exiting. If it owned this mm, find a new owner for the mm. 494 */ 495 void mm_update_next_owner(struct mm_struct *mm) 496 { 497 struct task_struct *g, *p = current; 498 499 /* 500 * If the exiting or execing task is not the owner, it's 501 * someone else's problem. 502 */ 503 if (mm->owner != p) 504 return; 505 /* 506 * The current owner is exiting/execing and there are no other 507 * candidates. Do not leave the mm pointing to a possibly 508 * freed task structure. 509 */ 510 if (atomic_read(&mm->mm_users) <= 1) { 511 WRITE_ONCE(mm->owner, NULL); 512 return; 513 } 514 515 read_lock(&tasklist_lock); 516 /* 517 * Search in the children 518 */ 519 list_for_each_entry(g, &p->children, sibling) { 520 if (try_to_set_owner(g, mm)) 521 goto ret; 522 } 523 /* 524 * Search in the siblings 525 */ 526 list_for_each_entry(g, &p->real_parent->children, sibling) { 527 if (try_to_set_owner(g, mm)) 528 goto ret; 529 } 530 /* 531 * Search through everything else, we should not get here often. 532 */ 533 for_each_process(g) { 534 if (atomic_read(&mm->mm_users) <= 1) 535 break; 536 if (g->flags & PF_KTHREAD) 537 continue; 538 if (try_to_set_owner(g, mm)) 539 goto ret; 540 } 541 read_unlock(&tasklist_lock); 542 /* 543 * We found no owner yet mm_users > 1: this implies that we are 544 * most likely racing with swapoff (try_to_unuse()) or /proc or 545 * ptrace or page migration (get_task_mm()). Mark owner as NULL. 546 */ 547 WRITE_ONCE(mm->owner, NULL); 548 ret: 549 return; 550 551 } 552 #endif /* CONFIG_MEMCG */ 553 554 #if defined(CONFIG_SCHED_CACHE) && defined(CONFIG_NUMA_BALANCING) 555 /* 556 * Subtract the memory footprint of the current task from 557 * mm. 558 */ 559 static void exit_mm_sched_cache(struct mm_struct *mm) 560 { 561 unsigned long fp, sub; 562 563 if (!current->total_numa_faults) 564 return; 565 /* 566 * No lock protection due to performance considerations. 567 * Make sure mm->sc_stat.footprint does not become 568 * negative. 569 */ 570 fp = READ_ONCE(mm->sc_stat.footprint); 571 sub = min(fp, current->total_numa_faults); 572 WRITE_ONCE(mm->sc_stat.footprint, fp - sub); 573 } 574 #else 575 static inline void exit_mm_sched_cache(struct mm_struct *mm) 576 { 577 } 578 #endif /* CONFIG_SCHED_CACHE CONFIG_NUMA_BALANCING */ 579 580 /* 581 * Turn us into a lazy TLB process if we 582 * aren't already.. 583 */ 584 static void exit_mm(void) 585 { 586 struct mm_struct *mm = current->mm; 587 588 mm_exit_exec_release(current, mm); 589 if (!mm) 590 return; 591 592 exit_mm_sched_cache(mm); 593 594 mmap_read_lock(mm); 595 mmgrab_lazy_tlb(mm); 596 BUG_ON(mm != current->active_mm); 597 /* more a memory barrier than a real lock */ 598 task_lock(current); 599 /* 600 * When a thread stops operating on an address space, the loop 601 * in membarrier_private_expedited() may not observe that 602 * tsk->mm, and the loop in membarrier_global_expedited() may 603 * not observe a MEMBARRIER_STATE_GLOBAL_EXPEDITED 604 * rq->membarrier_state, so those would not issue an IPI. 605 * Membarrier requires a memory barrier after accessing 606 * user-space memory, before clearing tsk->mm or the 607 * rq->membarrier_state. 608 */ 609 smp_mb__after_spinlock(); 610 local_irq_disable(); 611 current->mm = NULL; 612 membarrier_update_current_mm(NULL); 613 enter_lazy_tlb(mm, current); 614 local_irq_enable(); 615 task_unlock(current); 616 mmap_read_unlock(mm); 617 mm_update_next_owner(mm); 618 mmput(mm); 619 if (test_thread_flag(TIF_MEMDIE)) 620 exit_oom_victim(); 621 } 622 623 static struct task_struct *find_alive_thread(struct task_struct *p) 624 { 625 struct task_struct *t; 626 627 for_each_thread(p, t) { 628 if (!(t->flags & PF_EXITING)) 629 return t; 630 } 631 return NULL; 632 } 633 634 static struct task_struct *find_child_reaper(struct task_struct *father, 635 struct list_head *dead) 636 __releases(&tasklist_lock) 637 __acquires(&tasklist_lock) 638 { 639 struct pid_namespace *pid_ns = task_active_pid_ns(father); 640 struct task_struct *reaper = pid_ns->child_reaper; 641 struct task_struct *p, *n; 642 643 if (likely(reaper != father)) 644 return reaper; 645 646 reaper = find_alive_thread(father); 647 if (reaper) { 648 ASSERT_EXCLUSIVE_WRITER(pid_ns->child_reaper); 649 WRITE_ONCE(pid_ns->child_reaper, reaper); 650 return reaper; 651 } 652 653 write_unlock_irq(&tasklist_lock); 654 655 list_for_each_entry_safe(p, n, dead, ptrace_entry) { 656 list_del_init(&p->ptrace_entry); 657 release_task(p); 658 } 659 660 zap_pid_ns_processes(pid_ns); 661 write_lock_irq(&tasklist_lock); 662 663 return father; 664 } 665 666 /* 667 * When we die, we re-parent all our children, and try to: 668 * 1. give them to another thread in our thread group, if such a member exists 669 * 2. give it to the first ancestor process which prctl'd itself as a 670 * child_subreaper for its children (like a service manager) 671 * 3. give it to the init process (PID 1) in our pid namespace 672 */ 673 static struct task_struct *find_new_reaper(struct task_struct *father, 674 struct task_struct *child_reaper) 675 { 676 struct task_struct *thread, *reaper; 677 678 thread = find_alive_thread(father); 679 if (thread) 680 return thread; 681 682 if (father->signal->has_child_subreaper) { 683 unsigned int ns_level = task_pid(father)->level; 684 /* 685 * Find the first ->is_child_subreaper ancestor in our pid_ns. 686 * We can't check reaper != child_reaper to ensure we do not 687 * cross the namespaces, the exiting parent could be injected 688 * by setns() + fork(). 689 * We check pid->level, this is slightly more efficient than 690 * task_active_pid_ns(reaper) != task_active_pid_ns(father). 691 */ 692 for (reaper = father->real_parent; 693 task_pid(reaper)->level == ns_level; 694 reaper = reaper->real_parent) { 695 if (reaper == &init_task) 696 break; 697 if (!reaper->signal->is_child_subreaper) 698 continue; 699 thread = find_alive_thread(reaper); 700 if (thread) 701 return thread; 702 } 703 } 704 705 return child_reaper; 706 } 707 708 /* 709 * Any that need to be release_task'd are put on the @dead list. 710 */ 711 static void reparent_leader(struct task_struct *father, struct task_struct *p, 712 struct list_head *dead) 713 { 714 if (unlikely(p->exit_state == EXIT_DEAD)) 715 return; 716 717 /* We don't want people slaying init. */ 718 p->exit_signal = SIGCHLD; 719 720 /* If it has exited notify the new parent about this child's death. */ 721 if (!p->ptrace && 722 p->exit_state == EXIT_ZOMBIE && thread_group_empty(p)) { 723 if (do_notify_parent(p, p->exit_signal)) { 724 p->exit_state = EXIT_DEAD; 725 list_add(&p->ptrace_entry, dead); 726 } 727 } 728 729 kill_orphaned_pgrp(p, father); 730 } 731 732 /* 733 * Make init inherit all the child processes 734 */ 735 static void forget_original_parent(struct task_struct *father, 736 struct list_head *dead) 737 { 738 struct task_struct *p, *t, *reaper; 739 740 if (unlikely(!list_empty(&father->ptraced))) 741 exit_ptrace(father, dead); 742 743 /* Can drop and reacquire tasklist_lock */ 744 reaper = find_child_reaper(father, dead); 745 if (list_empty(&father->children)) 746 return; 747 748 reaper = find_new_reaper(father, reaper); 749 list_for_each_entry(p, &father->children, sibling) { 750 for_each_thread(p, t) { 751 RCU_INIT_POINTER(t->real_parent, reaper); 752 BUG_ON((!t->ptrace) != (rcu_access_pointer(t->parent) == father)); 753 if (likely(!t->ptrace)) 754 t->parent = t->real_parent; 755 if (t->pdeath_signal) 756 group_send_sig_info(t->pdeath_signal, 757 SEND_SIG_NOINFO, t, 758 PIDTYPE_TGID); 759 } 760 /* 761 * If this is a threaded reparent there is no need to 762 * notify anyone anything has happened. 763 */ 764 if (!same_thread_group(reaper, father)) 765 reparent_leader(father, p, dead); 766 } 767 list_splice_tail_init(&father->children, &reaper->children); 768 } 769 770 /* 771 * Send signals to all our closest relatives so that they know 772 * to properly mourn us.. 773 */ 774 static void exit_notify(struct task_struct *tsk, int group_dead) 775 { 776 bool autoreap; 777 struct task_struct *p, *n; 778 LIST_HEAD(dead); 779 780 write_lock_irq(&tasklist_lock); 781 forget_original_parent(tsk, &dead); 782 783 if (group_dead) 784 kill_orphaned_pgrp(tsk->group_leader, NULL); 785 786 tsk->exit_state = EXIT_ZOMBIE; 787 788 if (unlikely(tsk->ptrace)) { 789 int sig = thread_group_empty(tsk) && !ptrace_reparented(tsk) 790 ? tsk->exit_signal : SIGCHLD; 791 autoreap = do_notify_parent(tsk, sig); 792 } else if (thread_group_leader(tsk)) { 793 autoreap = thread_group_empty(tsk) && 794 do_notify_parent(tsk, tsk->exit_signal); 795 } else { 796 autoreap = true; 797 /* untraced sub-thread */ 798 do_notify_pidfd(tsk); 799 } 800 801 if (autoreap) { 802 tsk->exit_state = EXIT_DEAD; 803 list_add(&tsk->ptrace_entry, &dead); 804 } 805 806 /* mt-exec, de_thread() is waiting for group leader */ 807 if (unlikely(tsk->signal->notify_count < 0)) 808 wake_up_process(tsk->signal->group_exec_task); 809 write_unlock_irq(&tasklist_lock); 810 811 list_for_each_entry_safe(p, n, &dead, ptrace_entry) { 812 list_del_init(&p->ptrace_entry); 813 release_task(p); 814 } 815 } 816 817 #ifdef CONFIG_DEBUG_STACK_USAGE 818 #ifdef CONFIG_STACK_GROWSUP 819 unsigned long stack_not_used(struct task_struct *p) 820 { 821 unsigned long *n = end_of_stack(p); 822 823 do { /* Skip over canary */ 824 n--; 825 } while (!*n); 826 827 return (unsigned long)end_of_stack(p) - (unsigned long)n; 828 } 829 #else /* !CONFIG_STACK_GROWSUP */ 830 unsigned long stack_not_used(struct task_struct *p) 831 { 832 unsigned long *n = end_of_stack(p); 833 834 do { /* Skip over canary */ 835 n++; 836 } while (!*n); 837 838 return (unsigned long)n - (unsigned long)end_of_stack(p); 839 } 840 #endif /* CONFIG_STACK_GROWSUP */ 841 842 /* Count the maximum pages reached in kernel stacks */ 843 static inline void kstack_histogram(unsigned long used_stack) 844 { 845 #ifdef CONFIG_VM_EVENT_COUNTERS 846 if (used_stack <= 1024) 847 count_vm_event(KSTACK_1K); 848 #if THREAD_SIZE > 1024 849 else if (used_stack <= 2048) 850 count_vm_event(KSTACK_2K); 851 #endif 852 #if THREAD_SIZE > 2048 853 else if (used_stack <= 4096) 854 count_vm_event(KSTACK_4K); 855 #endif 856 #if THREAD_SIZE > 4096 857 else if (used_stack <= 8192) 858 count_vm_event(KSTACK_8K); 859 #endif 860 #if THREAD_SIZE > 8192 861 else if (used_stack <= 16384) 862 count_vm_event(KSTACK_16K); 863 #endif 864 #if THREAD_SIZE > 16384 865 else if (used_stack <= 32768) 866 count_vm_event(KSTACK_32K); 867 #endif 868 #if THREAD_SIZE > 32768 869 else if (used_stack <= 65536) 870 count_vm_event(KSTACK_64K); 871 #endif 872 #if THREAD_SIZE > 65536 873 else 874 count_vm_event(KSTACK_REST); 875 #endif 876 #endif /* CONFIG_VM_EVENT_COUNTERS */ 877 } 878 879 static void check_stack_usage(void) 880 { 881 static DEFINE_SPINLOCK(low_water_lock); 882 static int lowest_to_date = THREAD_SIZE; 883 unsigned long free; 884 885 free = stack_not_used(current); 886 kstack_histogram(THREAD_SIZE - free); 887 888 if (free >= lowest_to_date) 889 return; 890 891 spin_lock(&low_water_lock); 892 if (free < lowest_to_date) { 893 pr_info("%s (%d) used greatest stack depth: %lu bytes left\n", 894 current->comm, task_pid_nr(current), free); 895 lowest_to_date = free; 896 } 897 spin_unlock(&low_water_lock); 898 } 899 #else /* !CONFIG_DEBUG_STACK_USAGE */ 900 static inline void check_stack_usage(void) {} 901 #endif /* CONFIG_DEBUG_STACK_USAGE */ 902 903 static void synchronize_group_exit(struct task_struct *tsk, long code) 904 { 905 struct sighand_struct *sighand = tsk->sighand; 906 struct signal_struct *signal = tsk->signal; 907 struct core_state *core_state; 908 909 spin_lock_irq(&sighand->siglock); 910 signal->quick_threads--; 911 if ((signal->quick_threads == 0) && 912 !(signal->flags & SIGNAL_GROUP_EXIT)) { 913 signal->flags = SIGNAL_GROUP_EXIT; 914 signal->group_exit_code = code; 915 signal->group_stop_count = 0; 916 } 917 /* 918 * Serialize with any possible pending coredump. 919 * We must hold siglock around checking core_state 920 * and setting PF_POSTCOREDUMP. The core-inducing thread 921 * will increment ->nr_threads for each thread in the 922 * group without PF_POSTCOREDUMP set. 923 */ 924 tsk->flags |= PF_POSTCOREDUMP; 925 core_state = signal->core_state; 926 spin_unlock_irq(&sighand->siglock); 927 928 if (unlikely(core_state)) 929 coredump_task_exit(tsk, core_state); 930 } 931 932 void __noreturn do_exit(long code) 933 { 934 struct task_struct *tsk = current; 935 struct kthread *kthread; 936 int group_dead; 937 938 WARN_ON(irqs_disabled()); 939 WARN_ON(tsk->plug); 940 941 kthread = tsk_is_kthread(tsk); 942 if (unlikely(kthread)) 943 kthread_do_exit(kthread, code); 944 945 kcov_task_exit(tsk); 946 kmsan_task_exit(tsk); 947 948 synchronize_group_exit(tsk, code); 949 ptrace_event(PTRACE_EVENT_EXIT, code); 950 user_events_exit(tsk); 951 952 io_uring_files_cancel(); 953 sched_mm_cid_exit(tsk); 954 exit_signals(tsk); /* sets PF_EXITING */ 955 956 seccomp_filter_release(tsk); 957 958 acct_update_integrals(tsk); 959 group_dead = atomic_dec_and_test(&tsk->signal->live); 960 if (group_dead) { 961 /* 962 * If the last thread of global init has exited, panic 963 * immediately to get a useable coredump. 964 */ 965 if (unlikely(is_global_init(tsk))) 966 panic("Attempted to kill init! exitcode=0x%08x\n", 967 tsk->signal->group_exit_code ?: (int)code); 968 969 #ifdef CONFIG_POSIX_TIMERS 970 hrtimer_cancel(&tsk->signal->real_timer); 971 exit_itimers(tsk); 972 #endif 973 if (tsk->mm) 974 setmax_mm_hiwater_rss(&tsk->signal->maxrss, tsk->mm); 975 } 976 acct_collect(code, group_dead); 977 if (group_dead) 978 tty_audit_exit(); 979 audit_free(tsk); 980 981 tsk->exit_code = code; 982 taskstats_exit(tsk, group_dead); 983 trace_sched_process_exit(tsk, group_dead); 984 985 /* 986 * Since sampling can touch ->mm, make sure to stop everything before we 987 * tear it down. 988 * 989 * Also flushes inherited counters to the parent - before the parent 990 * gets woken up by child-exit notifications. 991 */ 992 perf_event_exit_task(tsk); 993 /* 994 * PF_EXITING (above) ensures unwind_deferred_request() will no 995 * longer add new unwinds. While exit_mm() (below) will destroy the 996 * abaility to do unwinds. So flush any pending unwinds here. 997 */ 998 unwind_deferred_task_exit(tsk); 999 1000 exit_mm(); 1001 1002 if (group_dead) 1003 acct_process(); 1004 1005 exit_sem(tsk); 1006 exit_shm(tsk); 1007 exit_files(tsk); 1008 exit_fs(tsk); 1009 if (group_dead) 1010 disassociate_ctty(1); 1011 exit_nsproxy_namespaces(tsk); 1012 exit_task_work(tsk); 1013 exit_thread(tsk); 1014 1015 sched_autogroup_exit_task(tsk); 1016 cgroup_task_exit(tsk); 1017 1018 /* 1019 * FIXME: do that only when needed, using sched_exit tracepoint 1020 */ 1021 flush_ptrace_hw_breakpoint(tsk); 1022 1023 exit_tasks_rcu_start(); 1024 exit_notify(tsk, group_dead); 1025 proc_exit_connector(tsk); 1026 mpol_put_task_policy(tsk); 1027 #ifdef CONFIG_FUTEX 1028 if (unlikely(current->futex.pi_state_cache)) 1029 kfree(current->futex.pi_state_cache); 1030 #endif 1031 /* 1032 * Make sure we are holding no locks: 1033 */ 1034 debug_check_no_locks_held(); 1035 1036 if (tsk->io_context) 1037 exit_io_context(tsk); 1038 1039 if (tsk->splice_pipe) 1040 free_pipe_info(tsk->splice_pipe); 1041 1042 if (tsk->task_frag.page) 1043 put_page(tsk->task_frag.page); 1044 1045 exit_task_stack_account(tsk); 1046 1047 check_stack_usage(); 1048 preempt_disable(); 1049 if (tsk->nr_dirtied) 1050 __this_cpu_add(dirty_throttle_leaks, tsk->nr_dirtied); 1051 exit_rcu(); 1052 exit_tasks_rcu_finish(); 1053 1054 lockdep_free_task(tsk); 1055 do_task_dead(); 1056 } 1057 EXPORT_SYMBOL(do_exit); 1058 1059 void __noreturn make_task_dead(int signr) 1060 { 1061 /* 1062 * Take the task off the cpu after something catastrophic has 1063 * happened. 1064 * 1065 * We can get here from a kernel oops, sometimes with preemption off. 1066 * Start by checking for critical errors. 1067 * Then fix up important state like USER_DS and preemption. 1068 * Then do everything else. 1069 */ 1070 struct task_struct *tsk = current; 1071 unsigned int limit; 1072 1073 if (unlikely(in_interrupt())) 1074 panic("Aiee, killing interrupt handler!"); 1075 if (unlikely(!tsk->pid)) 1076 panic("Attempted to kill the idle task!"); 1077 1078 if (unlikely(irqs_disabled())) { 1079 pr_info("note: %s[%d] exited with irqs disabled\n", 1080 current->comm, task_pid_nr(current)); 1081 local_irq_enable(); 1082 } 1083 if (unlikely(in_atomic())) { 1084 pr_info("note: %s[%d] exited with preempt_count %d\n", 1085 current->comm, task_pid_nr(current), 1086 preempt_count()); 1087 preempt_count_set(PREEMPT_ENABLED); 1088 } 1089 1090 /* 1091 * Every time the system oopses, if the oops happens while a reference 1092 * to an object was held, the reference leaks. 1093 * If the oops doesn't also leak memory, repeated oopsing can cause 1094 * reference counters to wrap around (if they're not using refcount_t). 1095 * This means that repeated oopsing can make unexploitable-looking bugs 1096 * exploitable through repeated oopsing. 1097 * To make sure this can't happen, place an upper bound on how often the 1098 * kernel may oops without panic(). 1099 */ 1100 limit = READ_ONCE(oops_limit); 1101 if (atomic_inc_return(&oops_count) >= limit && limit) 1102 panic("Oopsed too often (kernel.oops_limit is %d)", limit); 1103 1104 /* 1105 * We're taking recursive faults here in make_task_dead. Safest is to just 1106 * leave this task alone and wait for reboot. 1107 */ 1108 if (unlikely(tsk->flags & PF_EXITING)) { 1109 pr_alert("Fixing recursive fault but reboot is needed!\n"); 1110 futex_exit_recursive(tsk); 1111 tsk->exit_state = EXIT_DEAD; 1112 refcount_inc(&tsk->rcu_users); 1113 preempt_disable(); 1114 do_task_dead(); 1115 } 1116 1117 do_exit(signr); 1118 } 1119 1120 SYSCALL_DEFINE1(exit, int, error_code) 1121 { 1122 do_exit((error_code & 0xff) << 8); 1123 } 1124 1125 /* 1126 * Take down every thread in the group. This is called by fatal signals 1127 * as well as by sys_exit_group (below). 1128 */ 1129 void __noreturn 1130 do_group_exit(int exit_code) 1131 { 1132 struct signal_struct *sig = current->signal; 1133 1134 if (sig->flags & SIGNAL_GROUP_EXIT) 1135 exit_code = sig->group_exit_code; 1136 else if (sig->group_exec_task) 1137 exit_code = 0; 1138 else { 1139 struct sighand_struct *const sighand = current->sighand; 1140 1141 spin_lock_irq(&sighand->siglock); 1142 if (sig->flags & SIGNAL_GROUP_EXIT) 1143 /* Another thread got here before we took the lock. */ 1144 exit_code = sig->group_exit_code; 1145 else if (sig->group_exec_task) 1146 exit_code = 0; 1147 else { 1148 sig->group_exit_code = exit_code; 1149 sig->flags = SIGNAL_GROUP_EXIT; 1150 zap_other_threads(current); 1151 } 1152 spin_unlock_irq(&sighand->siglock); 1153 } 1154 1155 do_exit(exit_code); 1156 /* NOTREACHED */ 1157 } 1158 1159 /* 1160 * this kills every thread in the thread group. Note that any externally 1161 * wait4()-ing process will get the correct exit code - even if this 1162 * thread is not the thread group leader. 1163 */ 1164 SYSCALL_DEFINE1(exit_group, int, error_code) 1165 { 1166 do_group_exit((error_code & 0xff) << 8); 1167 /* NOTREACHED */ 1168 return 0; 1169 } 1170 1171 static int eligible_pid(struct wait_opts *wo, struct task_struct *p) 1172 { 1173 return wo->wo_type == PIDTYPE_MAX || 1174 task_pid_type(p, wo->wo_type) == wo->wo_pid; 1175 } 1176 1177 static int 1178 eligible_child(struct wait_opts *wo, bool ptrace, struct task_struct *p) 1179 { 1180 if (!eligible_pid(wo, p)) 1181 return 0; 1182 1183 /* 1184 * Wait for all children (clone and not) if __WALL is set or 1185 * if it is traced by us. 1186 */ 1187 if (ptrace || (wo->wo_flags & __WALL)) 1188 return 1; 1189 1190 /* 1191 * Otherwise, wait for clone children *only* if __WCLONE is set; 1192 * otherwise, wait for non-clone children *only*. 1193 * 1194 * Note: a "clone" child here is one that reports to its parent 1195 * using a signal other than SIGCHLD, or a non-leader thread which 1196 * we can only see if it is traced by us. 1197 */ 1198 if ((p->exit_signal != SIGCHLD) ^ !!(wo->wo_flags & __WCLONE)) 1199 return 0; 1200 1201 return 1; 1202 } 1203 1204 /* 1205 * Handle sys_wait4 work for one task in state EXIT_ZOMBIE. We hold 1206 * read_lock(&tasklist_lock) on entry. If we return zero, we still hold 1207 * the lock and this task is uninteresting. If we return nonzero, we have 1208 * released the lock and the system call should return. 1209 */ 1210 static int wait_task_zombie(struct wait_opts *wo, struct task_struct *p) 1211 { 1212 int state, status; 1213 pid_t pid = task_pid_vnr(p); 1214 uid_t uid = from_kuid_munged(current_user_ns(), task_uid(p)); 1215 struct waitid_info *infop; 1216 1217 if (!likely(wo->wo_flags & WEXITED)) 1218 return 0; 1219 1220 if (unlikely(wo->wo_flags & WNOWAIT)) { 1221 status = (p->signal->flags & SIGNAL_GROUP_EXIT) 1222 ? p->signal->group_exit_code : p->exit_code; 1223 get_task_struct(p); 1224 read_unlock(&tasklist_lock); 1225 sched_annotate_sleep(); 1226 if (wo->wo_rusage) 1227 getrusage(p, RUSAGE_BOTH, wo->wo_rusage); 1228 put_task_struct(p); 1229 goto out_info; 1230 } 1231 /* 1232 * Move the task's state to DEAD/TRACE, only one thread can do this. 1233 */ 1234 state = (ptrace_reparented(p) && thread_group_leader(p)) ? 1235 EXIT_TRACE : EXIT_DEAD; 1236 if (cmpxchg(&p->exit_state, EXIT_ZOMBIE, state) != EXIT_ZOMBIE) 1237 return 0; 1238 /* 1239 * We own this thread, nobody else can reap it. 1240 */ 1241 read_unlock(&tasklist_lock); 1242 sched_annotate_sleep(); 1243 1244 /* 1245 * Check thread_group_leader() to exclude the traced sub-threads. 1246 */ 1247 if (state == EXIT_DEAD && thread_group_leader(p)) { 1248 struct signal_struct *sig = p->signal; 1249 struct signal_struct *psig = current->signal; 1250 unsigned long maxrss; 1251 u64 tgutime, tgstime; 1252 1253 /* 1254 * The resource counters for the group leader are in its 1255 * own task_struct. Those for dead threads in the group 1256 * are in its signal_struct, as are those for the child 1257 * processes it has previously reaped. All these 1258 * accumulate in the parent's signal_struct c* fields. 1259 * 1260 * We don't bother to take a lock here to protect these 1261 * p->signal fields because the whole thread group is dead 1262 * and nobody can change them. 1263 * 1264 * psig->stats_lock also protects us from our sub-threads 1265 * which can reap other children at the same time. 1266 * 1267 * We use thread_group_cputime_adjusted() to get times for 1268 * the thread group, which consolidates times for all threads 1269 * in the group including the group leader. 1270 */ 1271 thread_group_cputime_adjusted(p, &tgutime, &tgstime); 1272 write_seqlock_irq(&psig->stats_lock); 1273 psig->cutime += tgutime + sig->cutime; 1274 psig->cstime += tgstime + sig->cstime; 1275 psig->cgtime += task_gtime(p) + sig->gtime + sig->cgtime; 1276 psig->cmin_flt += 1277 p->min_flt + sig->min_flt + sig->cmin_flt; 1278 psig->cmaj_flt += 1279 p->maj_flt + sig->maj_flt + sig->cmaj_flt; 1280 psig->cnvcsw += 1281 p->nvcsw + sig->nvcsw + sig->cnvcsw; 1282 psig->cnivcsw += 1283 p->nivcsw + sig->nivcsw + sig->cnivcsw; 1284 psig->cinblock += 1285 task_io_get_inblock(p) + 1286 sig->inblock + sig->cinblock; 1287 psig->coublock += 1288 task_io_get_oublock(p) + 1289 sig->oublock + sig->coublock; 1290 maxrss = max(sig->maxrss, sig->cmaxrss); 1291 if (psig->cmaxrss < maxrss) 1292 psig->cmaxrss = maxrss; 1293 task_io_accounting_add(&psig->ioac, &p->ioac); 1294 task_io_accounting_add(&psig->ioac, &sig->ioac); 1295 write_sequnlock_irq(&psig->stats_lock); 1296 } 1297 1298 if (wo->wo_rusage) 1299 getrusage(p, RUSAGE_BOTH, wo->wo_rusage); 1300 status = (p->signal->flags & SIGNAL_GROUP_EXIT) 1301 ? p->signal->group_exit_code : p->exit_code; 1302 wo->wo_stat = status; 1303 1304 if (state == EXIT_TRACE) { 1305 write_lock_irq(&tasklist_lock); 1306 /* We dropped tasklist, ptracer could die and untrace */ 1307 ptrace_unlink(p); 1308 1309 /* If parent wants a zombie, don't release it now */ 1310 state = EXIT_ZOMBIE; 1311 if (do_notify_parent(p, p->exit_signal)) 1312 state = EXIT_DEAD; 1313 p->exit_state = state; 1314 write_unlock_irq(&tasklist_lock); 1315 } 1316 if (state == EXIT_DEAD) 1317 release_task(p); 1318 1319 out_info: 1320 infop = wo->wo_info; 1321 if (infop) { 1322 if ((status & 0x7f) == 0) { 1323 infop->cause = CLD_EXITED; 1324 infop->status = status >> 8; 1325 } else { 1326 infop->cause = (status & 0x80) ? CLD_DUMPED : CLD_KILLED; 1327 infop->status = status & 0x7f; 1328 } 1329 infop->pid = pid; 1330 infop->uid = uid; 1331 } 1332 1333 return pid; 1334 } 1335 1336 static int *task_stopped_code(struct task_struct *p, bool ptrace) 1337 { 1338 if (ptrace) { 1339 if (task_is_traced(p) && !(p->jobctl & JOBCTL_LISTENING)) 1340 return &p->exit_code; 1341 } else { 1342 if (p->signal->flags & SIGNAL_STOP_STOPPED) 1343 return &p->signal->group_exit_code; 1344 } 1345 return NULL; 1346 } 1347 1348 /** 1349 * wait_task_stopped - Wait for %TASK_STOPPED or %TASK_TRACED 1350 * @wo: wait options 1351 * @ptrace: is the wait for ptrace 1352 * @p: task to wait for 1353 * 1354 * Handle sys_wait4() work for %p in state %TASK_STOPPED or %TASK_TRACED. 1355 * 1356 * CONTEXT: 1357 * read_lock(&tasklist_lock), which is released if return value is 1358 * non-zero. Also, grabs and releases @p->sighand->siglock. 1359 * 1360 * RETURNS: 1361 * 0 if wait condition didn't exist and search for other wait conditions 1362 * should continue. Non-zero return, -errno on failure and @p's pid on 1363 * success, implies that tasklist_lock is released and wait condition 1364 * search should terminate. 1365 */ 1366 static int wait_task_stopped(struct wait_opts *wo, 1367 int ptrace, struct task_struct *p) 1368 { 1369 struct waitid_info *infop; 1370 int exit_code, *p_code, why; 1371 uid_t uid = 0; /* unneeded, required by compiler */ 1372 pid_t pid; 1373 1374 /* 1375 * Traditionally we see ptrace'd stopped tasks regardless of options. 1376 */ 1377 if (!ptrace && !(wo->wo_flags & WUNTRACED)) 1378 return 0; 1379 1380 if (!task_stopped_code(p, ptrace)) 1381 return 0; 1382 1383 exit_code = 0; 1384 spin_lock_irq(&p->sighand->siglock); 1385 1386 p_code = task_stopped_code(p, ptrace); 1387 if (unlikely(!p_code)) 1388 goto unlock_sig; 1389 1390 exit_code = *p_code; 1391 if (!exit_code) 1392 goto unlock_sig; 1393 1394 if (!unlikely(wo->wo_flags & WNOWAIT)) 1395 *p_code = 0; 1396 1397 uid = from_kuid_munged(current_user_ns(), task_uid(p)); 1398 unlock_sig: 1399 spin_unlock_irq(&p->sighand->siglock); 1400 if (!exit_code) 1401 return 0; 1402 1403 /* 1404 * Now we are pretty sure this task is interesting. 1405 * Make sure it doesn't get reaped out from under us while we 1406 * give up the lock and then examine it below. We don't want to 1407 * keep holding onto the tasklist_lock while we call getrusage and 1408 * possibly take page faults for user memory. 1409 */ 1410 get_task_struct(p); 1411 pid = task_pid_vnr(p); 1412 why = ptrace ? CLD_TRAPPED : CLD_STOPPED; 1413 read_unlock(&tasklist_lock); 1414 sched_annotate_sleep(); 1415 if (wo->wo_rusage) 1416 getrusage(p, RUSAGE_BOTH, wo->wo_rusage); 1417 put_task_struct(p); 1418 1419 if (likely(!(wo->wo_flags & WNOWAIT))) 1420 wo->wo_stat = (exit_code << 8) | 0x7f; 1421 1422 infop = wo->wo_info; 1423 if (infop) { 1424 infop->cause = why; 1425 infop->status = exit_code; 1426 infop->pid = pid; 1427 infop->uid = uid; 1428 } 1429 return pid; 1430 } 1431 1432 /* 1433 * Handle do_wait work for one task in a live, non-stopped state. 1434 * read_lock(&tasklist_lock) on entry. If we return zero, we still hold 1435 * the lock and this task is uninteresting. If we return nonzero, we have 1436 * released the lock and the system call should return. 1437 */ 1438 static int wait_task_continued(struct wait_opts *wo, struct task_struct *p) 1439 { 1440 struct waitid_info *infop; 1441 pid_t pid; 1442 uid_t uid; 1443 1444 if (!unlikely(wo->wo_flags & WCONTINUED)) 1445 return 0; 1446 1447 if (!(p->signal->flags & SIGNAL_STOP_CONTINUED)) 1448 return 0; 1449 1450 spin_lock_irq(&p->sighand->siglock); 1451 /* Re-check with the lock held. */ 1452 if (!(p->signal->flags & SIGNAL_STOP_CONTINUED)) { 1453 spin_unlock_irq(&p->sighand->siglock); 1454 return 0; 1455 } 1456 if (!unlikely(wo->wo_flags & WNOWAIT)) 1457 p->signal->flags &= ~SIGNAL_STOP_CONTINUED; 1458 uid = from_kuid_munged(current_user_ns(), task_uid(p)); 1459 spin_unlock_irq(&p->sighand->siglock); 1460 1461 pid = task_pid_vnr(p); 1462 get_task_struct(p); 1463 read_unlock(&tasklist_lock); 1464 sched_annotate_sleep(); 1465 if (wo->wo_rusage) 1466 getrusage(p, RUSAGE_BOTH, wo->wo_rusage); 1467 put_task_struct(p); 1468 1469 infop = wo->wo_info; 1470 if (!infop) { 1471 wo->wo_stat = 0xffff; 1472 } else { 1473 infop->cause = CLD_CONTINUED; 1474 infop->pid = pid; 1475 infop->uid = uid; 1476 infop->status = SIGCONT; 1477 } 1478 return pid; 1479 } 1480 1481 /* 1482 * Consider @p for a wait by @parent. 1483 * 1484 * -ECHILD should be in ->notask_error before the first call. 1485 * Returns nonzero for a final return, when we have unlocked tasklist_lock. 1486 * Returns zero if the search for a child should continue; 1487 * then ->notask_error is 0 if @p is an eligible child, 1488 * or still -ECHILD. 1489 */ 1490 static int wait_consider_task(struct wait_opts *wo, int ptrace, 1491 struct task_struct *p) 1492 { 1493 /* 1494 * We can race with wait_task_zombie() from another thread. 1495 * Ensure that EXIT_ZOMBIE -> EXIT_DEAD/EXIT_TRACE transition 1496 * can't confuse the checks below. 1497 */ 1498 int exit_state = READ_ONCE(p->exit_state); 1499 int ret; 1500 1501 if (unlikely(exit_state == EXIT_DEAD)) 1502 return 0; 1503 1504 ret = eligible_child(wo, ptrace, p); 1505 if (!ret) 1506 return ret; 1507 1508 if (unlikely(exit_state == EXIT_TRACE)) { 1509 /* 1510 * ptrace == 0 means we are the natural parent. In this case 1511 * we should clear notask_error, debugger will notify us. 1512 */ 1513 if (likely(!ptrace)) 1514 wo->notask_error = 0; 1515 return 0; 1516 } 1517 1518 if (likely(!ptrace) && unlikely(p->ptrace)) { 1519 /* 1520 * If it is traced by its real parent's group, just pretend 1521 * the caller is ptrace_do_wait() and reap this child if it 1522 * is zombie. 1523 * 1524 * This also hides group stop state from real parent; otherwise 1525 * a single stop can be reported twice as group and ptrace stop. 1526 * If a ptracer wants to distinguish these two events for its 1527 * own children it should create a separate process which takes 1528 * the role of real parent. 1529 */ 1530 if (!ptrace_reparented(p)) 1531 ptrace = 1; 1532 } 1533 1534 /* slay zombie? */ 1535 if (exit_state == EXIT_ZOMBIE) { 1536 /* we don't reap group leaders with subthreads */ 1537 if (!delay_group_leader(p)) { 1538 /* 1539 * A zombie ptracee is only visible to its ptracer. 1540 * Notification and reaping will be cascaded to the 1541 * real parent when the ptracer detaches. 1542 */ 1543 if (unlikely(ptrace) || likely(!p->ptrace)) 1544 return wait_task_zombie(wo, p); 1545 } 1546 1547 /* 1548 * Allow access to stopped/continued state via zombie by 1549 * falling through. Clearing of notask_error is complex. 1550 * 1551 * When !@ptrace: 1552 * 1553 * If WEXITED is set, notask_error should naturally be 1554 * cleared. If not, subset of WSTOPPED|WCONTINUED is set, 1555 * so, if there are live subthreads, there are events to 1556 * wait for. If all subthreads are dead, it's still safe 1557 * to clear - this function will be called again in finite 1558 * amount time once all the subthreads are released and 1559 * will then return without clearing. 1560 * 1561 * When @ptrace: 1562 * 1563 * Stopped state is per-task and thus can't change once the 1564 * target task dies. Only continued and exited can happen. 1565 * Clear notask_error if WCONTINUED | WEXITED. 1566 */ 1567 if (likely(!ptrace) || (wo->wo_flags & (WCONTINUED | WEXITED))) 1568 wo->notask_error = 0; 1569 } else { 1570 /* 1571 * @p is alive and it's gonna stop, continue or exit, so 1572 * there always is something to wait for. 1573 */ 1574 wo->notask_error = 0; 1575 } 1576 1577 /* 1578 * Wait for stopped. Depending on @ptrace, different stopped state 1579 * is used and the two don't interact with each other. 1580 */ 1581 ret = wait_task_stopped(wo, ptrace, p); 1582 if (ret) 1583 return ret; 1584 1585 /* 1586 * Wait for continued. There's only one continued state and the 1587 * ptracer can consume it which can confuse the real parent. Don't 1588 * use WCONTINUED from ptracer. You don't need or want it. 1589 */ 1590 return wait_task_continued(wo, p); 1591 } 1592 1593 /* 1594 * Do the work of do_wait() for one thread in the group, @tsk. 1595 * 1596 * -ECHILD should be in ->notask_error before the first call. 1597 * Returns nonzero for a final return, when we have unlocked tasklist_lock. 1598 * Returns zero if the search for a child should continue; then 1599 * ->notask_error is 0 if there were any eligible children, 1600 * or still -ECHILD. 1601 */ 1602 static int do_wait_thread(struct wait_opts *wo, struct task_struct *tsk) 1603 { 1604 struct task_struct *p; 1605 1606 list_for_each_entry(p, &tsk->children, sibling) { 1607 int ret = wait_consider_task(wo, 0, p); 1608 1609 if (ret) 1610 return ret; 1611 } 1612 1613 return 0; 1614 } 1615 1616 static int ptrace_do_wait(struct wait_opts *wo, struct task_struct *tsk) 1617 { 1618 struct task_struct *p; 1619 1620 list_for_each_entry(p, &tsk->ptraced, ptrace_entry) { 1621 int ret = wait_consider_task(wo, 1, p); 1622 1623 if (ret) 1624 return ret; 1625 } 1626 1627 return 0; 1628 } 1629 1630 bool pid_child_should_wake(struct wait_opts *wo, struct task_struct *p) 1631 { 1632 if (!eligible_pid(wo, p)) 1633 return false; 1634 1635 if ((wo->wo_flags & __WNOTHREAD) && wo->child_wait.private != p->parent) 1636 return false; 1637 1638 return true; 1639 } 1640 1641 static int child_wait_callback(wait_queue_entry_t *wait, unsigned mode, 1642 int sync, void *key) 1643 { 1644 struct wait_opts *wo = container_of(wait, struct wait_opts, 1645 child_wait); 1646 struct task_struct *p = key; 1647 1648 if (pid_child_should_wake(wo, p)) 1649 return default_wake_function(wait, mode, sync, key); 1650 1651 return 0; 1652 } 1653 1654 void __wake_up_parent(struct task_struct *p, struct task_struct *parent) 1655 { 1656 __wake_up_sync_key(&parent->signal->wait_chldexit, 1657 TASK_INTERRUPTIBLE, p); 1658 } 1659 1660 static bool is_effectively_child(struct wait_opts *wo, bool ptrace, 1661 struct task_struct *target) 1662 { 1663 struct task_struct *parent = 1664 !ptrace ? target->real_parent : target->parent; 1665 1666 return current == parent || (!(wo->wo_flags & __WNOTHREAD) && 1667 same_thread_group(current, parent)); 1668 } 1669 1670 /* 1671 * Optimization for waiting on PIDTYPE_PID. No need to iterate through child 1672 * and tracee lists to find the target task. 1673 */ 1674 static int do_wait_pid(struct wait_opts *wo) 1675 { 1676 bool ptrace; 1677 struct task_struct *target; 1678 int retval; 1679 1680 ptrace = false; 1681 target = pid_task(wo->wo_pid, PIDTYPE_TGID); 1682 if (target && is_effectively_child(wo, ptrace, target)) { 1683 retval = wait_consider_task(wo, ptrace, target); 1684 if (retval) 1685 return retval; 1686 } 1687 1688 ptrace = true; 1689 target = pid_task(wo->wo_pid, PIDTYPE_PID); 1690 if (target && target->ptrace && 1691 is_effectively_child(wo, ptrace, target)) { 1692 retval = wait_consider_task(wo, ptrace, target); 1693 if (retval) 1694 return retval; 1695 } 1696 1697 return 0; 1698 } 1699 1700 long __do_wait(struct wait_opts *wo) 1701 { 1702 long retval; 1703 1704 /* 1705 * If there is nothing that can match our criteria, just get out. 1706 * We will clear ->notask_error to zero if we see any child that 1707 * might later match our criteria, even if we are not able to reap 1708 * it yet. 1709 */ 1710 wo->notask_error = -ECHILD; 1711 if ((wo->wo_type < PIDTYPE_MAX) && 1712 (!wo->wo_pid || !pid_has_task(wo->wo_pid, wo->wo_type))) 1713 goto notask; 1714 1715 read_lock(&tasklist_lock); 1716 1717 if (wo->wo_type == PIDTYPE_PID) { 1718 retval = do_wait_pid(wo); 1719 if (retval) 1720 return retval; 1721 } else { 1722 struct task_struct *tsk = current; 1723 1724 do { 1725 retval = do_wait_thread(wo, tsk); 1726 if (retval) 1727 return retval; 1728 1729 retval = ptrace_do_wait(wo, tsk); 1730 if (retval) 1731 return retval; 1732 1733 if (wo->wo_flags & __WNOTHREAD) 1734 break; 1735 } while_each_thread(current, tsk); 1736 } 1737 read_unlock(&tasklist_lock); 1738 1739 notask: 1740 retval = wo->notask_error; 1741 if (!retval && !(wo->wo_flags & WNOHANG)) 1742 return -ERESTARTSYS; 1743 1744 return retval; 1745 } 1746 1747 static long do_wait(struct wait_opts *wo) 1748 { 1749 int retval; 1750 1751 trace_sched_process_wait(wo->wo_pid); 1752 1753 init_waitqueue_func_entry(&wo->child_wait, child_wait_callback); 1754 wo->child_wait.private = current; 1755 add_wait_queue(¤t->signal->wait_chldexit, &wo->child_wait); 1756 1757 do { 1758 set_current_state(TASK_INTERRUPTIBLE); 1759 retval = __do_wait(wo); 1760 if (retval != -ERESTARTSYS) 1761 break; 1762 if (signal_pending(current)) 1763 break; 1764 schedule(); 1765 } while (1); 1766 1767 __set_current_state(TASK_RUNNING); 1768 remove_wait_queue(¤t->signal->wait_chldexit, &wo->child_wait); 1769 return retval; 1770 } 1771 1772 int kernel_waitid_prepare(struct wait_opts *wo, int which, pid_t upid, 1773 struct waitid_info *infop, int options, 1774 struct rusage *ru) 1775 { 1776 unsigned int f_flags = 0; 1777 struct pid *pid = NULL; 1778 enum pid_type type; 1779 1780 if (options & ~(WNOHANG|WNOWAIT|WEXITED|WSTOPPED|WCONTINUED| 1781 __WNOTHREAD|__WCLONE|__WALL)) 1782 return -EINVAL; 1783 if (!(options & (WEXITED|WSTOPPED|WCONTINUED))) 1784 return -EINVAL; 1785 1786 switch (which) { 1787 case P_ALL: 1788 type = PIDTYPE_MAX; 1789 break; 1790 case P_PID: 1791 type = PIDTYPE_PID; 1792 if (upid <= 0) 1793 return -EINVAL; 1794 1795 pid = find_get_pid(upid); 1796 break; 1797 case P_PGID: 1798 type = PIDTYPE_PGID; 1799 if (upid < 0) 1800 return -EINVAL; 1801 1802 if (upid) 1803 pid = find_get_pid(upid); 1804 else 1805 pid = get_task_pid(current, PIDTYPE_PGID); 1806 break; 1807 case P_PIDFD: 1808 type = PIDTYPE_PID; 1809 if (upid < 0) 1810 return -EINVAL; 1811 1812 pid = pidfd_get_pid(upid, &f_flags); 1813 if (IS_ERR(pid)) 1814 return PTR_ERR(pid); 1815 1816 break; 1817 default: 1818 return -EINVAL; 1819 } 1820 1821 wo->wo_type = type; 1822 wo->wo_pid = pid; 1823 wo->wo_flags = options; 1824 wo->wo_info = infop; 1825 wo->wo_rusage = ru; 1826 if (f_flags & O_NONBLOCK) 1827 wo->wo_flags |= WNOHANG; 1828 1829 return 0; 1830 } 1831 1832 static long kernel_waitid(int which, pid_t upid, struct waitid_info *infop, 1833 int options, struct rusage *ru) 1834 { 1835 struct wait_opts wo; 1836 long ret; 1837 1838 ret = kernel_waitid_prepare(&wo, which, upid, infop, options, ru); 1839 if (ret) 1840 return ret; 1841 1842 ret = do_wait(&wo); 1843 if (!ret && !(options & WNOHANG) && (wo.wo_flags & WNOHANG)) 1844 ret = -EAGAIN; 1845 1846 put_pid(wo.wo_pid); 1847 return ret; 1848 } 1849 1850 SYSCALL_DEFINE5(waitid, int, which, pid_t, upid, struct siginfo __user *, 1851 infop, int, options, struct rusage __user *, ru) 1852 { 1853 struct rusage r; 1854 struct waitid_info info = {.status = 0}; 1855 long err = kernel_waitid(which, upid, &info, options, ru ? &r : NULL); 1856 int signo = 0; 1857 1858 if (err > 0) { 1859 signo = SIGCHLD; 1860 err = 0; 1861 if (ru && copy_to_user(ru, &r, sizeof(struct rusage))) 1862 return -EFAULT; 1863 } 1864 if (!infop) 1865 return err; 1866 1867 if (!user_write_access_begin(infop, sizeof(*infop))) 1868 return -EFAULT; 1869 1870 unsafe_put_user(signo, &infop->si_signo, Efault); 1871 unsafe_put_user(0, &infop->si_errno, Efault); 1872 unsafe_put_user(info.cause, &infop->si_code, Efault); 1873 unsafe_put_user(info.pid, &infop->si_pid, Efault); 1874 unsafe_put_user(info.uid, &infop->si_uid, Efault); 1875 unsafe_put_user(info.status, &infop->si_status, Efault); 1876 user_write_access_end(); 1877 return err; 1878 Efault: 1879 user_write_access_end(); 1880 return -EFAULT; 1881 } 1882 1883 long kernel_wait4(pid_t upid, int __user *stat_addr, int options, 1884 struct rusage *ru) 1885 { 1886 struct wait_opts wo; 1887 struct pid *pid = NULL; 1888 enum pid_type type; 1889 long ret; 1890 1891 if (options & ~(WNOHANG|WUNTRACED|WCONTINUED| 1892 __WNOTHREAD|__WCLONE|__WALL)) 1893 return -EINVAL; 1894 1895 /* -INT_MIN is not defined */ 1896 if (upid == INT_MIN) 1897 return -ESRCH; 1898 1899 if (upid == -1) 1900 type = PIDTYPE_MAX; 1901 else if (upid < 0) { 1902 type = PIDTYPE_PGID; 1903 pid = find_get_pid(-upid); 1904 } else if (upid == 0) { 1905 type = PIDTYPE_PGID; 1906 pid = get_task_pid(current, PIDTYPE_PGID); 1907 } else /* upid > 0 */ { 1908 type = PIDTYPE_PID; 1909 pid = find_get_pid(upid); 1910 } 1911 1912 wo.wo_type = type; 1913 wo.wo_pid = pid; 1914 wo.wo_flags = options | WEXITED; 1915 wo.wo_info = NULL; 1916 wo.wo_stat = 0; 1917 wo.wo_rusage = ru; 1918 ret = do_wait(&wo); 1919 put_pid(pid); 1920 if (ret > 0 && stat_addr && put_user(wo.wo_stat, stat_addr)) 1921 ret = -EFAULT; 1922 1923 return ret; 1924 } 1925 1926 int kernel_wait(pid_t pid, int *stat) 1927 { 1928 struct wait_opts wo = { 1929 .wo_type = PIDTYPE_PID, 1930 .wo_pid = find_get_pid(pid), 1931 .wo_flags = WEXITED, 1932 }; 1933 int ret; 1934 1935 ret = do_wait(&wo); 1936 if (ret > 0 && wo.wo_stat) 1937 *stat = wo.wo_stat; 1938 put_pid(wo.wo_pid); 1939 return ret; 1940 } 1941 1942 SYSCALL_DEFINE4(wait4, pid_t, upid, int __user *, stat_addr, 1943 int, options, struct rusage __user *, ru) 1944 { 1945 struct rusage r; 1946 long err = kernel_wait4(upid, stat_addr, options, ru ? &r : NULL); 1947 1948 if (err > 0) { 1949 if (ru && copy_to_user(ru, &r, sizeof(struct rusage))) 1950 return -EFAULT; 1951 } 1952 return err; 1953 } 1954 1955 #ifdef __ARCH_WANT_SYS_WAITPID 1956 1957 /* 1958 * sys_waitpid() remains for compatibility. waitpid() should be 1959 * implemented by calling sys_wait4() from libc.a. 1960 */ 1961 SYSCALL_DEFINE3(waitpid, pid_t, pid, int __user *, stat_addr, int, options) 1962 { 1963 return kernel_wait4(pid, stat_addr, options, NULL); 1964 } 1965 1966 #endif 1967 1968 #ifdef CONFIG_COMPAT 1969 COMPAT_SYSCALL_DEFINE4(wait4, 1970 compat_pid_t, pid, 1971 compat_uint_t __user *, stat_addr, 1972 int, options, 1973 struct compat_rusage __user *, ru) 1974 { 1975 struct rusage r; 1976 long err = kernel_wait4(pid, stat_addr, options, ru ? &r : NULL); 1977 if (err > 0) { 1978 if (ru && put_compat_rusage(&r, ru)) 1979 return -EFAULT; 1980 } 1981 return err; 1982 } 1983 1984 COMPAT_SYSCALL_DEFINE5(waitid, 1985 int, which, compat_pid_t, pid, 1986 struct compat_siginfo __user *, infop, int, options, 1987 struct compat_rusage __user *, uru) 1988 { 1989 struct rusage ru; 1990 struct waitid_info info = {.status = 0}; 1991 long err = kernel_waitid(which, pid, &info, options, uru ? &ru : NULL); 1992 int signo = 0; 1993 if (err > 0) { 1994 signo = SIGCHLD; 1995 err = 0; 1996 if (uru) { 1997 /* kernel_waitid() overwrites everything in ru */ 1998 if (COMPAT_USE_64BIT_TIME) 1999 err = copy_to_user(uru, &ru, sizeof(ru)); 2000 else 2001 err = put_compat_rusage(&ru, uru); 2002 if (err) 2003 return -EFAULT; 2004 } 2005 } 2006 2007 if (!infop) 2008 return err; 2009 2010 if (!user_write_access_begin(infop, sizeof(*infop))) 2011 return -EFAULT; 2012 2013 unsafe_put_user(signo, &infop->si_signo, Efault); 2014 unsafe_put_user(0, &infop->si_errno, Efault); 2015 unsafe_put_user(info.cause, &infop->si_code, Efault); 2016 unsafe_put_user(info.pid, &infop->si_pid, Efault); 2017 unsafe_put_user(info.uid, &infop->si_uid, Efault); 2018 unsafe_put_user(info.status, &infop->si_status, Efault); 2019 user_write_access_end(); 2020 return err; 2021 Efault: 2022 user_write_access_end(); 2023 return -EFAULT; 2024 } 2025 #endif 2026 2027 /* 2028 * This needs to be __function_aligned as GCC implicitly makes any 2029 * implementation of abort() cold and drops alignment specified by 2030 * -falign-functions=N. 2031 * 2032 * See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=88345#c11 2033 */ 2034 __weak __function_aligned void abort(void) 2035 { 2036 BUG(); 2037 2038 /* if that doesn't kill us, halt */ 2039 panic("Oops failed to kill thread"); 2040 } 2041 EXPORT_SYMBOL(abort); 2042