1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/kernel/exit.c 4 * 5 * Copyright (C) 1991, 1992 Linus Torvalds 6 */ 7 8 #include <linux/mm.h> 9 #include <linux/slab.h> 10 #include <linux/sched/autogroup.h> 11 #include <linux/sched/mm.h> 12 #include <linux/sched/stat.h> 13 #include <linux/sched/task.h> 14 #include <linux/sched/task_stack.h> 15 #include <linux/sched/cputime.h> 16 #include <linux/interrupt.h> 17 #include <linux/module.h> 18 #include <linux/capability.h> 19 #include <linux/completion.h> 20 #include <linux/personality.h> 21 #include <linux/tty.h> 22 #include <linux/iocontext.h> 23 #include <linux/key.h> 24 #include <linux/cpu.h> 25 #include <linux/acct.h> 26 #include <linux/tsacct_kern.h> 27 #include <linux/file.h> 28 #include <linux/freezer.h> 29 #include <linux/binfmts.h> 30 #include <linux/nsproxy.h> 31 #include <linux/pid_namespace.h> 32 #include <linux/ptrace.h> 33 #include <linux/profile.h> 34 #include <linux/mount.h> 35 #include <linux/proc_fs.h> 36 #include <linux/kthread.h> 37 #include <linux/mempolicy.h> 38 #include <linux/taskstats_kern.h> 39 #include <linux/delayacct.h> 40 #include <linux/cgroup.h> 41 #include <linux/syscalls.h> 42 #include <linux/signal.h> 43 #include <linux/posix-timers.h> 44 #include <linux/cn_proc.h> 45 #include <linux/mutex.h> 46 #include <linux/futex.h> 47 #include <linux/pipe_fs_i.h> 48 #include <linux/audit.h> /* for audit_free() */ 49 #include <linux/resource.h> 50 #include <linux/task_io_accounting_ops.h> 51 #include <linux/blkdev.h> 52 #include <linux/task_work.h> 53 #include <linux/fs_struct.h> 54 #include <linux/init_task.h> 55 #include <linux/perf_event.h> 56 #include <trace/events/sched.h> 57 #include <linux/hw_breakpoint.h> 58 #include <linux/oom.h> 59 #include <linux/writeback.h> 60 #include <linux/shm.h> 61 #include <linux/kcov.h> 62 #include <linux/kmsan.h> 63 #include <linux/random.h> 64 #include <linux/rcuwait.h> 65 #include <linux/compat.h> 66 #include <linux/io_uring.h> 67 #include <linux/kprobes.h> 68 #include <linux/rethook.h> 69 #include <linux/sysfs.h> 70 #include <linux/user_events.h> 71 #include <linux/uaccess.h> 72 #include <linux/pidfs.h> 73 74 #include <uapi/linux/wait.h> 75 76 #include <asm/unistd.h> 77 #include <asm/mmu_context.h> 78 79 #include "exit.h" 80 81 /* 82 * The default value should be high enough to not crash a system that randomly 83 * crashes its kernel from time to time, but low enough to at least not permit 84 * overflowing 32-bit refcounts or the ldsem writer count. 85 */ 86 static unsigned int oops_limit = 10000; 87 88 #ifdef CONFIG_SYSCTL 89 static const struct ctl_table kern_exit_table[] = { 90 { 91 .procname = "oops_limit", 92 .data = &oops_limit, 93 .maxlen = sizeof(oops_limit), 94 .mode = 0644, 95 .proc_handler = proc_douintvec, 96 }, 97 }; 98 99 static __init int kernel_exit_sysctls_init(void) 100 { 101 register_sysctl_init("kernel", kern_exit_table); 102 return 0; 103 } 104 late_initcall(kernel_exit_sysctls_init); 105 #endif 106 107 static atomic_t oops_count = ATOMIC_INIT(0); 108 109 #ifdef CONFIG_SYSFS 110 static ssize_t oops_count_show(struct kobject *kobj, struct kobj_attribute *attr, 111 char *page) 112 { 113 return sysfs_emit(page, "%d\n", atomic_read(&oops_count)); 114 } 115 116 static struct kobj_attribute oops_count_attr = __ATTR_RO(oops_count); 117 118 static __init int kernel_exit_sysfs_init(void) 119 { 120 sysfs_add_file_to_group(kernel_kobj, &oops_count_attr.attr, NULL); 121 return 0; 122 } 123 late_initcall(kernel_exit_sysfs_init); 124 #endif 125 126 /* 127 * For things release_task() would like to do *after* tasklist_lock is released. 128 */ 129 struct release_task_post { 130 struct pid *pids[PIDTYPE_MAX]; 131 }; 132 133 static void __unhash_process(struct release_task_post *post, struct task_struct *p, 134 bool group_dead) 135 { 136 nr_threads--; 137 detach_pid(post->pids, p, PIDTYPE_PID); 138 if (group_dead) { 139 detach_pid(post->pids, p, PIDTYPE_TGID); 140 detach_pid(post->pids, p, PIDTYPE_PGID); 141 detach_pid(post->pids, p, PIDTYPE_SID); 142 143 list_del_rcu(&p->tasks); 144 list_del_init(&p->sibling); 145 __this_cpu_dec(process_counts); 146 } 147 list_del_rcu(&p->thread_node); 148 } 149 150 /* 151 * This function expects the tasklist_lock write-locked. 152 */ 153 static void __exit_signal(struct release_task_post *post, struct task_struct *tsk) 154 { 155 struct signal_struct *sig = tsk->signal; 156 bool group_dead = thread_group_leader(tsk); 157 struct sighand_struct *sighand; 158 struct tty_struct *tty; 159 u64 utime, stime; 160 161 sighand = rcu_dereference_check(tsk->sighand, 162 lockdep_tasklist_lock_is_held()); 163 spin_lock(&sighand->siglock); 164 165 #ifdef CONFIG_POSIX_TIMERS 166 posix_cpu_timers_exit(tsk); 167 if (group_dead) 168 posix_cpu_timers_exit_group(tsk); 169 #endif 170 171 if (group_dead) { 172 tty = sig->tty; 173 sig->tty = NULL; 174 } else { 175 /* 176 * If there is any task waiting for the group exit 177 * then notify it: 178 */ 179 if (sig->notify_count > 0 && !--sig->notify_count) 180 wake_up_process(sig->group_exec_task); 181 182 if (tsk == sig->curr_target) 183 sig->curr_target = next_thread(tsk); 184 } 185 186 /* 187 * Accumulate here the counters for all threads as they die. We could 188 * skip the group leader because it is the last user of signal_struct, 189 * but we want to avoid the race with thread_group_cputime() which can 190 * see the empty ->thread_head list. 191 */ 192 task_cputime(tsk, &utime, &stime); 193 write_seqlock(&sig->stats_lock); 194 sig->utime += utime; 195 sig->stime += stime; 196 sig->gtime += task_gtime(tsk); 197 sig->min_flt += tsk->min_flt; 198 sig->maj_flt += tsk->maj_flt; 199 sig->nvcsw += tsk->nvcsw; 200 sig->nivcsw += tsk->nivcsw; 201 sig->inblock += task_io_get_inblock(tsk); 202 sig->oublock += task_io_get_oublock(tsk); 203 task_io_accounting_add(&sig->ioac, &tsk->ioac); 204 sig->sum_sched_runtime += tsk->se.sum_exec_runtime; 205 sig->nr_threads--; 206 __unhash_process(post, tsk, group_dead); 207 write_sequnlock(&sig->stats_lock); 208 209 tsk->sighand = NULL; 210 spin_unlock(&sighand->siglock); 211 212 __cleanup_sighand(sighand); 213 if (group_dead) 214 tty_kref_put(tty); 215 } 216 217 static void delayed_put_task_struct(struct rcu_head *rhp) 218 { 219 struct task_struct *tsk = container_of(rhp, struct task_struct, rcu); 220 221 kprobe_flush_task(tsk); 222 rethook_flush_task(tsk); 223 perf_event_delayed_put(tsk); 224 trace_sched_process_free(tsk); 225 put_task_struct(tsk); 226 } 227 228 void put_task_struct_rcu_user(struct task_struct *task) 229 { 230 if (refcount_dec_and_test(&task->rcu_users)) 231 call_rcu(&task->rcu, delayed_put_task_struct); 232 } 233 234 void __weak release_thread(struct task_struct *dead_task) 235 { 236 } 237 238 void release_task(struct task_struct *p) 239 { 240 struct release_task_post post; 241 struct task_struct *leader; 242 struct pid *thread_pid; 243 int zap_leader; 244 repeat: 245 memset(&post, 0, sizeof(post)); 246 247 /* don't need to get the RCU readlock here - the process is dead and 248 * can't be modifying its own credentials. But shut RCU-lockdep up */ 249 rcu_read_lock(); 250 dec_rlimit_ucounts(task_ucounts(p), UCOUNT_RLIMIT_NPROC, 1); 251 rcu_read_unlock(); 252 253 pidfs_exit(p); 254 cgroup_release(p); 255 256 thread_pid = get_pid(p->thread_pid); 257 258 write_lock_irq(&tasklist_lock); 259 ptrace_release_task(p); 260 __exit_signal(&post, p); 261 262 /* 263 * If we are the last non-leader member of the thread 264 * group, and the leader is zombie, then notify the 265 * group leader's parent process. (if it wants notification.) 266 */ 267 zap_leader = 0; 268 leader = p->group_leader; 269 if (leader != p && thread_group_empty(leader) 270 && leader->exit_state == EXIT_ZOMBIE) { 271 /* for pidfs_exit() and do_notify_parent() */ 272 if (leader->signal->flags & SIGNAL_GROUP_EXIT) 273 leader->exit_code = leader->signal->group_exit_code; 274 /* 275 * If we were the last child thread and the leader has 276 * exited already, and the leader's parent ignores SIGCHLD, 277 * then we are the one who should release the leader. 278 */ 279 zap_leader = do_notify_parent(leader, leader->exit_signal); 280 if (zap_leader) 281 leader->exit_state = EXIT_DEAD; 282 } 283 284 write_unlock_irq(&tasklist_lock); 285 proc_flush_pid(thread_pid); 286 put_pid(thread_pid); 287 add_device_randomness(&p->se.sum_exec_runtime, 288 sizeof(p->se.sum_exec_runtime)); 289 free_pids(post.pids); 290 release_thread(p); 291 /* 292 * This task was already removed from the process/thread/pid lists 293 * and lock_task_sighand(p) can't succeed. Nobody else can touch 294 * ->pending or, if group dead, signal->shared_pending. We can call 295 * flush_sigqueue() lockless. 296 */ 297 flush_sigqueue(&p->pending); 298 if (thread_group_leader(p)) 299 flush_sigqueue(&p->signal->shared_pending); 300 301 put_task_struct_rcu_user(p); 302 303 p = leader; 304 if (unlikely(zap_leader)) 305 goto repeat; 306 } 307 308 int rcuwait_wake_up(struct rcuwait *w) 309 { 310 int ret = 0; 311 struct task_struct *task; 312 313 rcu_read_lock(); 314 315 /* 316 * Order condition vs @task, such that everything prior to the load 317 * of @task is visible. This is the condition as to why the user called 318 * rcuwait_wake() in the first place. Pairs with set_current_state() 319 * barrier (A) in rcuwait_wait_event(). 320 * 321 * WAIT WAKE 322 * [S] tsk = current [S] cond = true 323 * MB (A) MB (B) 324 * [L] cond [L] tsk 325 */ 326 smp_mb(); /* (B) */ 327 328 task = rcu_dereference(w->task); 329 if (task) 330 ret = wake_up_process(task); 331 rcu_read_unlock(); 332 333 return ret; 334 } 335 EXPORT_SYMBOL_GPL(rcuwait_wake_up); 336 337 /* 338 * Determine if a process group is "orphaned", according to the POSIX 339 * definition in 2.2.2.52. Orphaned process groups are not to be affected 340 * by terminal-generated stop signals. Newly orphaned process groups are 341 * to receive a SIGHUP and a SIGCONT. 342 * 343 * "I ask you, have you ever known what it is to be an orphan?" 344 */ 345 static int will_become_orphaned_pgrp(struct pid *pgrp, 346 struct task_struct *ignored_task) 347 { 348 struct task_struct *p; 349 350 do_each_pid_task(pgrp, PIDTYPE_PGID, p) { 351 if ((p == ignored_task) || 352 (p->exit_state && thread_group_empty(p)) || 353 is_global_init(p->real_parent)) 354 continue; 355 356 if (task_pgrp(p->real_parent) != pgrp && 357 task_session(p->real_parent) == task_session(p)) 358 return 0; 359 } while_each_pid_task(pgrp, PIDTYPE_PGID, p); 360 361 return 1; 362 } 363 364 int is_current_pgrp_orphaned(void) 365 { 366 int retval; 367 368 read_lock(&tasklist_lock); 369 retval = will_become_orphaned_pgrp(task_pgrp(current), NULL); 370 read_unlock(&tasklist_lock); 371 372 return retval; 373 } 374 375 static bool has_stopped_jobs(struct pid *pgrp) 376 { 377 struct task_struct *p; 378 379 do_each_pid_task(pgrp, PIDTYPE_PGID, p) { 380 if (p->signal->flags & SIGNAL_STOP_STOPPED) 381 return true; 382 } while_each_pid_task(pgrp, PIDTYPE_PGID, p); 383 384 return false; 385 } 386 387 /* 388 * Check to see if any process groups have become orphaned as 389 * a result of our exiting, and if they have any stopped jobs, 390 * send them a SIGHUP and then a SIGCONT. (POSIX 3.2.2.2) 391 */ 392 static void 393 kill_orphaned_pgrp(struct task_struct *tsk, struct task_struct *parent) 394 { 395 struct pid *pgrp = task_pgrp(tsk); 396 struct task_struct *ignored_task = tsk; 397 398 if (!parent) 399 /* exit: our father is in a different pgrp than 400 * we are and we were the only connection outside. 401 */ 402 parent = tsk->real_parent; 403 else 404 /* reparent: our child is in a different pgrp than 405 * we are, and it was the only connection outside. 406 */ 407 ignored_task = NULL; 408 409 if (task_pgrp(parent) != pgrp && 410 task_session(parent) == task_session(tsk) && 411 will_become_orphaned_pgrp(pgrp, ignored_task) && 412 has_stopped_jobs(pgrp)) { 413 __kill_pgrp_info(SIGHUP, SEND_SIG_PRIV, pgrp); 414 __kill_pgrp_info(SIGCONT, SEND_SIG_PRIV, pgrp); 415 } 416 } 417 418 static void coredump_task_exit(struct task_struct *tsk, 419 struct core_state *core_state) 420 { 421 struct core_thread self; 422 423 self.task = tsk; 424 if (self.task->flags & PF_SIGNALED) 425 self.next = xchg(&core_state->dumper.next, &self); 426 else 427 self.task = NULL; 428 /* 429 * Implies mb(), the result of xchg() must be visible 430 * to core_state->dumper. 431 */ 432 if (atomic_dec_and_test(&core_state->nr_threads)) 433 complete(&core_state->startup); 434 435 for (;;) { 436 set_current_state(TASK_IDLE|TASK_FREEZABLE); 437 if (!self.task) /* see coredump_finish() */ 438 break; 439 schedule(); 440 } 441 __set_current_state(TASK_RUNNING); 442 } 443 444 #ifdef CONFIG_MEMCG 445 /* drops tasklist_lock if succeeds */ 446 static bool __try_to_set_owner(struct task_struct *tsk, struct mm_struct *mm) 447 { 448 bool ret = false; 449 450 task_lock(tsk); 451 if (likely(tsk->mm == mm)) { 452 /* tsk can't pass exit_mm/exec_mmap and exit */ 453 read_unlock(&tasklist_lock); 454 WRITE_ONCE(mm->owner, tsk); 455 lru_gen_migrate_mm(mm); 456 ret = true; 457 } 458 task_unlock(tsk); 459 return ret; 460 } 461 462 static bool try_to_set_owner(struct task_struct *g, struct mm_struct *mm) 463 { 464 struct task_struct *t; 465 466 for_each_thread(g, t) { 467 struct mm_struct *t_mm = READ_ONCE(t->mm); 468 if (t_mm == mm) { 469 if (__try_to_set_owner(t, mm)) 470 return true; 471 } else if (t_mm) 472 break; 473 } 474 475 return false; 476 } 477 478 /* 479 * A task is exiting. If it owned this mm, find a new owner for the mm. 480 */ 481 void mm_update_next_owner(struct mm_struct *mm) 482 { 483 struct task_struct *g, *p = current; 484 485 /* 486 * If the exiting or execing task is not the owner, it's 487 * someone else's problem. 488 */ 489 if (mm->owner != p) 490 return; 491 /* 492 * The current owner is exiting/execing and there are no other 493 * candidates. Do not leave the mm pointing to a possibly 494 * freed task structure. 495 */ 496 if (atomic_read(&mm->mm_users) <= 1) { 497 WRITE_ONCE(mm->owner, NULL); 498 return; 499 } 500 501 read_lock(&tasklist_lock); 502 /* 503 * Search in the children 504 */ 505 list_for_each_entry(g, &p->children, sibling) { 506 if (try_to_set_owner(g, mm)) 507 goto ret; 508 } 509 /* 510 * Search in the siblings 511 */ 512 list_for_each_entry(g, &p->real_parent->children, sibling) { 513 if (try_to_set_owner(g, mm)) 514 goto ret; 515 } 516 /* 517 * Search through everything else, we should not get here often. 518 */ 519 for_each_process(g) { 520 if (atomic_read(&mm->mm_users) <= 1) 521 break; 522 if (g->flags & PF_KTHREAD) 523 continue; 524 if (try_to_set_owner(g, mm)) 525 goto ret; 526 } 527 read_unlock(&tasklist_lock); 528 /* 529 * We found no owner yet mm_users > 1: this implies that we are 530 * most likely racing with swapoff (try_to_unuse()) or /proc or 531 * ptrace or page migration (get_task_mm()). Mark owner as NULL. 532 */ 533 WRITE_ONCE(mm->owner, NULL); 534 ret: 535 return; 536 537 } 538 #endif /* CONFIG_MEMCG */ 539 540 /* 541 * Turn us into a lazy TLB process if we 542 * aren't already.. 543 */ 544 static void exit_mm(void) 545 { 546 struct mm_struct *mm = current->mm; 547 548 exit_mm_release(current, mm); 549 if (!mm) 550 return; 551 mmap_read_lock(mm); 552 mmgrab_lazy_tlb(mm); 553 BUG_ON(mm != current->active_mm); 554 /* more a memory barrier than a real lock */ 555 task_lock(current); 556 /* 557 * When a thread stops operating on an address space, the loop 558 * in membarrier_private_expedited() may not observe that 559 * tsk->mm, and the loop in membarrier_global_expedited() may 560 * not observe a MEMBARRIER_STATE_GLOBAL_EXPEDITED 561 * rq->membarrier_state, so those would not issue an IPI. 562 * Membarrier requires a memory barrier after accessing 563 * user-space memory, before clearing tsk->mm or the 564 * rq->membarrier_state. 565 */ 566 smp_mb__after_spinlock(); 567 local_irq_disable(); 568 current->mm = NULL; 569 membarrier_update_current_mm(NULL); 570 enter_lazy_tlb(mm, current); 571 local_irq_enable(); 572 task_unlock(current); 573 mmap_read_unlock(mm); 574 mm_update_next_owner(mm); 575 mmput(mm); 576 if (test_thread_flag(TIF_MEMDIE)) 577 exit_oom_victim(); 578 } 579 580 static struct task_struct *find_alive_thread(struct task_struct *p) 581 { 582 struct task_struct *t; 583 584 for_each_thread(p, t) { 585 if (!(t->flags & PF_EXITING)) 586 return t; 587 } 588 return NULL; 589 } 590 591 static struct task_struct *find_child_reaper(struct task_struct *father, 592 struct list_head *dead) 593 __releases(&tasklist_lock) 594 __acquires(&tasklist_lock) 595 { 596 struct pid_namespace *pid_ns = task_active_pid_ns(father); 597 struct task_struct *reaper = pid_ns->child_reaper; 598 struct task_struct *p, *n; 599 600 if (likely(reaper != father)) 601 return reaper; 602 603 reaper = find_alive_thread(father); 604 if (reaper) { 605 pid_ns->child_reaper = reaper; 606 return reaper; 607 } 608 609 write_unlock_irq(&tasklist_lock); 610 611 list_for_each_entry_safe(p, n, dead, ptrace_entry) { 612 list_del_init(&p->ptrace_entry); 613 release_task(p); 614 } 615 616 zap_pid_ns_processes(pid_ns); 617 write_lock_irq(&tasklist_lock); 618 619 return father; 620 } 621 622 /* 623 * When we die, we re-parent all our children, and try to: 624 * 1. give them to another thread in our thread group, if such a member exists 625 * 2. give it to the first ancestor process which prctl'd itself as a 626 * child_subreaper for its children (like a service manager) 627 * 3. give it to the init process (PID 1) in our pid namespace 628 */ 629 static struct task_struct *find_new_reaper(struct task_struct *father, 630 struct task_struct *child_reaper) 631 { 632 struct task_struct *thread, *reaper; 633 634 thread = find_alive_thread(father); 635 if (thread) 636 return thread; 637 638 if (father->signal->has_child_subreaper) { 639 unsigned int ns_level = task_pid(father)->level; 640 /* 641 * Find the first ->is_child_subreaper ancestor in our pid_ns. 642 * We can't check reaper != child_reaper to ensure we do not 643 * cross the namespaces, the exiting parent could be injected 644 * by setns() + fork(). 645 * We check pid->level, this is slightly more efficient than 646 * task_active_pid_ns(reaper) != task_active_pid_ns(father). 647 */ 648 for (reaper = father->real_parent; 649 task_pid(reaper)->level == ns_level; 650 reaper = reaper->real_parent) { 651 if (reaper == &init_task) 652 break; 653 if (!reaper->signal->is_child_subreaper) 654 continue; 655 thread = find_alive_thread(reaper); 656 if (thread) 657 return thread; 658 } 659 } 660 661 return child_reaper; 662 } 663 664 /* 665 * Any that need to be release_task'd are put on the @dead list. 666 */ 667 static void reparent_leader(struct task_struct *father, struct task_struct *p, 668 struct list_head *dead) 669 { 670 if (unlikely(p->exit_state == EXIT_DEAD)) 671 return; 672 673 /* We don't want people slaying init. */ 674 p->exit_signal = SIGCHLD; 675 676 /* If it has exited notify the new parent about this child's death. */ 677 if (!p->ptrace && 678 p->exit_state == EXIT_ZOMBIE && thread_group_empty(p)) { 679 if (do_notify_parent(p, p->exit_signal)) { 680 p->exit_state = EXIT_DEAD; 681 list_add(&p->ptrace_entry, dead); 682 } 683 } 684 685 kill_orphaned_pgrp(p, father); 686 } 687 688 /* 689 * This does two things: 690 * 691 * A. Make init inherit all the child processes 692 * B. Check to see if any process groups have become orphaned 693 * as a result of our exiting, and if they have any stopped 694 * jobs, send them a SIGHUP and then a SIGCONT. (POSIX 3.2.2.2) 695 */ 696 static void forget_original_parent(struct task_struct *father, 697 struct list_head *dead) 698 { 699 struct task_struct *p, *t, *reaper; 700 701 if (unlikely(!list_empty(&father->ptraced))) 702 exit_ptrace(father, dead); 703 704 /* Can drop and reacquire tasklist_lock */ 705 reaper = find_child_reaper(father, dead); 706 if (list_empty(&father->children)) 707 return; 708 709 reaper = find_new_reaper(father, reaper); 710 list_for_each_entry(p, &father->children, sibling) { 711 for_each_thread(p, t) { 712 RCU_INIT_POINTER(t->real_parent, reaper); 713 BUG_ON((!t->ptrace) != (rcu_access_pointer(t->parent) == father)); 714 if (likely(!t->ptrace)) 715 t->parent = t->real_parent; 716 if (t->pdeath_signal) 717 group_send_sig_info(t->pdeath_signal, 718 SEND_SIG_NOINFO, t, 719 PIDTYPE_TGID); 720 } 721 /* 722 * If this is a threaded reparent there is no need to 723 * notify anyone anything has happened. 724 */ 725 if (!same_thread_group(reaper, father)) 726 reparent_leader(father, p, dead); 727 } 728 list_splice_tail_init(&father->children, &reaper->children); 729 } 730 731 /* 732 * Send signals to all our closest relatives so that they know 733 * to properly mourn us.. 734 */ 735 static void exit_notify(struct task_struct *tsk, int group_dead) 736 { 737 bool autoreap; 738 struct task_struct *p, *n; 739 LIST_HEAD(dead); 740 741 write_lock_irq(&tasklist_lock); 742 forget_original_parent(tsk, &dead); 743 744 if (group_dead) 745 kill_orphaned_pgrp(tsk->group_leader, NULL); 746 747 tsk->exit_state = EXIT_ZOMBIE; 748 749 if (unlikely(tsk->ptrace)) { 750 int sig = thread_group_leader(tsk) && 751 thread_group_empty(tsk) && 752 !ptrace_reparented(tsk) ? 753 tsk->exit_signal : SIGCHLD; 754 autoreap = do_notify_parent(tsk, sig); 755 } else if (thread_group_leader(tsk)) { 756 autoreap = thread_group_empty(tsk) && 757 do_notify_parent(tsk, tsk->exit_signal); 758 } else { 759 autoreap = true; 760 /* untraced sub-thread */ 761 do_notify_pidfd(tsk); 762 } 763 764 if (autoreap) { 765 tsk->exit_state = EXIT_DEAD; 766 list_add(&tsk->ptrace_entry, &dead); 767 } 768 769 /* mt-exec, de_thread() is waiting for group leader */ 770 if (unlikely(tsk->signal->notify_count < 0)) 771 wake_up_process(tsk->signal->group_exec_task); 772 write_unlock_irq(&tasklist_lock); 773 774 list_for_each_entry_safe(p, n, &dead, ptrace_entry) { 775 list_del_init(&p->ptrace_entry); 776 release_task(p); 777 } 778 } 779 780 #ifdef CONFIG_DEBUG_STACK_USAGE 781 unsigned long stack_not_used(struct task_struct *p) 782 { 783 unsigned long *n = end_of_stack(p); 784 785 do { /* Skip over canary */ 786 # ifdef CONFIG_STACK_GROWSUP 787 n--; 788 # else 789 n++; 790 # endif 791 } while (!*n); 792 793 # ifdef CONFIG_STACK_GROWSUP 794 return (unsigned long)end_of_stack(p) - (unsigned long)n; 795 # else 796 return (unsigned long)n - (unsigned long)end_of_stack(p); 797 # endif 798 } 799 800 /* Count the maximum pages reached in kernel stacks */ 801 static inline void kstack_histogram(unsigned long used_stack) 802 { 803 #ifdef CONFIG_VM_EVENT_COUNTERS 804 if (used_stack <= 1024) 805 count_vm_event(KSTACK_1K); 806 #if THREAD_SIZE > 1024 807 else if (used_stack <= 2048) 808 count_vm_event(KSTACK_2K); 809 #endif 810 #if THREAD_SIZE > 2048 811 else if (used_stack <= 4096) 812 count_vm_event(KSTACK_4K); 813 #endif 814 #if THREAD_SIZE > 4096 815 else if (used_stack <= 8192) 816 count_vm_event(KSTACK_8K); 817 #endif 818 #if THREAD_SIZE > 8192 819 else if (used_stack <= 16384) 820 count_vm_event(KSTACK_16K); 821 #endif 822 #if THREAD_SIZE > 16384 823 else if (used_stack <= 32768) 824 count_vm_event(KSTACK_32K); 825 #endif 826 #if THREAD_SIZE > 32768 827 else if (used_stack <= 65536) 828 count_vm_event(KSTACK_64K); 829 #endif 830 #if THREAD_SIZE > 65536 831 else 832 count_vm_event(KSTACK_REST); 833 #endif 834 #endif /* CONFIG_VM_EVENT_COUNTERS */ 835 } 836 837 static void check_stack_usage(void) 838 { 839 static DEFINE_SPINLOCK(low_water_lock); 840 static int lowest_to_date = THREAD_SIZE; 841 unsigned long free; 842 843 free = stack_not_used(current); 844 kstack_histogram(THREAD_SIZE - free); 845 846 if (free >= lowest_to_date) 847 return; 848 849 spin_lock(&low_water_lock); 850 if (free < lowest_to_date) { 851 pr_info("%s (%d) used greatest stack depth: %lu bytes left\n", 852 current->comm, task_pid_nr(current), free); 853 lowest_to_date = free; 854 } 855 spin_unlock(&low_water_lock); 856 } 857 #else 858 static inline void check_stack_usage(void) {} 859 #endif 860 861 static void synchronize_group_exit(struct task_struct *tsk, long code) 862 { 863 struct sighand_struct *sighand = tsk->sighand; 864 struct signal_struct *signal = tsk->signal; 865 struct core_state *core_state; 866 867 spin_lock_irq(&sighand->siglock); 868 signal->quick_threads--; 869 if ((signal->quick_threads == 0) && 870 !(signal->flags & SIGNAL_GROUP_EXIT)) { 871 signal->flags = SIGNAL_GROUP_EXIT; 872 signal->group_exit_code = code; 873 signal->group_stop_count = 0; 874 } 875 /* 876 * Serialize with any possible pending coredump. 877 * We must hold siglock around checking core_state 878 * and setting PF_POSTCOREDUMP. The core-inducing thread 879 * will increment ->nr_threads for each thread in the 880 * group without PF_POSTCOREDUMP set. 881 */ 882 tsk->flags |= PF_POSTCOREDUMP; 883 core_state = signal->core_state; 884 spin_unlock_irq(&sighand->siglock); 885 886 if (unlikely(core_state)) 887 coredump_task_exit(tsk, core_state); 888 } 889 890 void __noreturn do_exit(long code) 891 { 892 struct task_struct *tsk = current; 893 int group_dead; 894 895 WARN_ON(irqs_disabled()); 896 WARN_ON(tsk->plug); 897 898 kcov_task_exit(tsk); 899 kmsan_task_exit(tsk); 900 901 synchronize_group_exit(tsk, code); 902 ptrace_event(PTRACE_EVENT_EXIT, code); 903 user_events_exit(tsk); 904 905 io_uring_files_cancel(); 906 exit_signals(tsk); /* sets PF_EXITING */ 907 908 seccomp_filter_release(tsk); 909 910 acct_update_integrals(tsk); 911 group_dead = atomic_dec_and_test(&tsk->signal->live); 912 if (group_dead) { 913 /* 914 * If the last thread of global init has exited, panic 915 * immediately to get a useable coredump. 916 */ 917 if (unlikely(is_global_init(tsk))) 918 panic("Attempted to kill init! exitcode=0x%08x\n", 919 tsk->signal->group_exit_code ?: (int)code); 920 921 #ifdef CONFIG_POSIX_TIMERS 922 hrtimer_cancel(&tsk->signal->real_timer); 923 exit_itimers(tsk); 924 #endif 925 if (tsk->mm) 926 setmax_mm_hiwater_rss(&tsk->signal->maxrss, tsk->mm); 927 } 928 acct_collect(code, group_dead); 929 if (group_dead) 930 tty_audit_exit(); 931 audit_free(tsk); 932 933 tsk->exit_code = code; 934 taskstats_exit(tsk, group_dead); 935 trace_sched_process_exit(tsk, group_dead); 936 937 exit_mm(); 938 939 if (group_dead) 940 acct_process(); 941 942 exit_sem(tsk); 943 exit_shm(tsk); 944 exit_files(tsk); 945 exit_fs(tsk); 946 if (group_dead) 947 disassociate_ctty(1); 948 exit_task_namespaces(tsk); 949 exit_task_work(tsk); 950 exit_thread(tsk); 951 952 /* 953 * Flush inherited counters to the parent - before the parent 954 * gets woken up by child-exit notifications. 955 * 956 * because of cgroup mode, must be called before cgroup_exit() 957 */ 958 perf_event_exit_task(tsk); 959 960 sched_autogroup_exit_task(tsk); 961 cgroup_exit(tsk); 962 963 /* 964 * FIXME: do that only when needed, using sched_exit tracepoint 965 */ 966 flush_ptrace_hw_breakpoint(tsk); 967 968 exit_tasks_rcu_start(); 969 exit_notify(tsk, group_dead); 970 proc_exit_connector(tsk); 971 mpol_put_task_policy(tsk); 972 #ifdef CONFIG_FUTEX 973 if (unlikely(current->pi_state_cache)) 974 kfree(current->pi_state_cache); 975 #endif 976 /* 977 * Make sure we are holding no locks: 978 */ 979 debug_check_no_locks_held(); 980 981 if (tsk->io_context) 982 exit_io_context(tsk); 983 984 if (tsk->splice_pipe) 985 free_pipe_info(tsk->splice_pipe); 986 987 if (tsk->task_frag.page) 988 put_page(tsk->task_frag.page); 989 990 exit_task_stack_account(tsk); 991 992 check_stack_usage(); 993 preempt_disable(); 994 if (tsk->nr_dirtied) 995 __this_cpu_add(dirty_throttle_leaks, tsk->nr_dirtied); 996 exit_rcu(); 997 exit_tasks_rcu_finish(); 998 999 lockdep_free_task(tsk); 1000 do_task_dead(); 1001 } 1002 1003 void __noreturn make_task_dead(int signr) 1004 { 1005 /* 1006 * Take the task off the cpu after something catastrophic has 1007 * happened. 1008 * 1009 * We can get here from a kernel oops, sometimes with preemption off. 1010 * Start by checking for critical errors. 1011 * Then fix up important state like USER_DS and preemption. 1012 * Then do everything else. 1013 */ 1014 struct task_struct *tsk = current; 1015 unsigned int limit; 1016 1017 if (unlikely(in_interrupt())) 1018 panic("Aiee, killing interrupt handler!"); 1019 if (unlikely(!tsk->pid)) 1020 panic("Attempted to kill the idle task!"); 1021 1022 if (unlikely(irqs_disabled())) { 1023 pr_info("note: %s[%d] exited with irqs disabled\n", 1024 current->comm, task_pid_nr(current)); 1025 local_irq_enable(); 1026 } 1027 if (unlikely(in_atomic())) { 1028 pr_info("note: %s[%d] exited with preempt_count %d\n", 1029 current->comm, task_pid_nr(current), 1030 preempt_count()); 1031 preempt_count_set(PREEMPT_ENABLED); 1032 } 1033 1034 /* 1035 * Every time the system oopses, if the oops happens while a reference 1036 * to an object was held, the reference leaks. 1037 * If the oops doesn't also leak memory, repeated oopsing can cause 1038 * reference counters to wrap around (if they're not using refcount_t). 1039 * This means that repeated oopsing can make unexploitable-looking bugs 1040 * exploitable through repeated oopsing. 1041 * To make sure this can't happen, place an upper bound on how often the 1042 * kernel may oops without panic(). 1043 */ 1044 limit = READ_ONCE(oops_limit); 1045 if (atomic_inc_return(&oops_count) >= limit && limit) 1046 panic("Oopsed too often (kernel.oops_limit is %d)", limit); 1047 1048 /* 1049 * We're taking recursive faults here in make_task_dead. Safest is to just 1050 * leave this task alone and wait for reboot. 1051 */ 1052 if (unlikely(tsk->flags & PF_EXITING)) { 1053 pr_alert("Fixing recursive fault but reboot is needed!\n"); 1054 futex_exit_recursive(tsk); 1055 tsk->exit_state = EXIT_DEAD; 1056 refcount_inc(&tsk->rcu_users); 1057 do_task_dead(); 1058 } 1059 1060 do_exit(signr); 1061 } 1062 1063 SYSCALL_DEFINE1(exit, int, error_code) 1064 { 1065 do_exit((error_code&0xff)<<8); 1066 } 1067 1068 /* 1069 * Take down every thread in the group. This is called by fatal signals 1070 * as well as by sys_exit_group (below). 1071 */ 1072 void __noreturn 1073 do_group_exit(int exit_code) 1074 { 1075 struct signal_struct *sig = current->signal; 1076 1077 if (sig->flags & SIGNAL_GROUP_EXIT) 1078 exit_code = sig->group_exit_code; 1079 else if (sig->group_exec_task) 1080 exit_code = 0; 1081 else { 1082 struct sighand_struct *const sighand = current->sighand; 1083 1084 spin_lock_irq(&sighand->siglock); 1085 if (sig->flags & SIGNAL_GROUP_EXIT) 1086 /* Another thread got here before we took the lock. */ 1087 exit_code = sig->group_exit_code; 1088 else if (sig->group_exec_task) 1089 exit_code = 0; 1090 else { 1091 sig->group_exit_code = exit_code; 1092 sig->flags = SIGNAL_GROUP_EXIT; 1093 zap_other_threads(current); 1094 } 1095 spin_unlock_irq(&sighand->siglock); 1096 } 1097 1098 do_exit(exit_code); 1099 /* NOTREACHED */ 1100 } 1101 1102 /* 1103 * this kills every thread in the thread group. Note that any externally 1104 * wait4()-ing process will get the correct exit code - even if this 1105 * thread is not the thread group leader. 1106 */ 1107 SYSCALL_DEFINE1(exit_group, int, error_code) 1108 { 1109 do_group_exit((error_code & 0xff) << 8); 1110 /* NOTREACHED */ 1111 return 0; 1112 } 1113 1114 static int eligible_pid(struct wait_opts *wo, struct task_struct *p) 1115 { 1116 return wo->wo_type == PIDTYPE_MAX || 1117 task_pid_type(p, wo->wo_type) == wo->wo_pid; 1118 } 1119 1120 static int 1121 eligible_child(struct wait_opts *wo, bool ptrace, struct task_struct *p) 1122 { 1123 if (!eligible_pid(wo, p)) 1124 return 0; 1125 1126 /* 1127 * Wait for all children (clone and not) if __WALL is set or 1128 * if it is traced by us. 1129 */ 1130 if (ptrace || (wo->wo_flags & __WALL)) 1131 return 1; 1132 1133 /* 1134 * Otherwise, wait for clone children *only* if __WCLONE is set; 1135 * otherwise, wait for non-clone children *only*. 1136 * 1137 * Note: a "clone" child here is one that reports to its parent 1138 * using a signal other than SIGCHLD, or a non-leader thread which 1139 * we can only see if it is traced by us. 1140 */ 1141 if ((p->exit_signal != SIGCHLD) ^ !!(wo->wo_flags & __WCLONE)) 1142 return 0; 1143 1144 return 1; 1145 } 1146 1147 /* 1148 * Handle sys_wait4 work for one task in state EXIT_ZOMBIE. We hold 1149 * read_lock(&tasklist_lock) on entry. If we return zero, we still hold 1150 * the lock and this task is uninteresting. If we return nonzero, we have 1151 * released the lock and the system call should return. 1152 */ 1153 static int wait_task_zombie(struct wait_opts *wo, struct task_struct *p) 1154 { 1155 int state, status; 1156 pid_t pid = task_pid_vnr(p); 1157 uid_t uid = from_kuid_munged(current_user_ns(), task_uid(p)); 1158 struct waitid_info *infop; 1159 1160 if (!likely(wo->wo_flags & WEXITED)) 1161 return 0; 1162 1163 if (unlikely(wo->wo_flags & WNOWAIT)) { 1164 status = (p->signal->flags & SIGNAL_GROUP_EXIT) 1165 ? p->signal->group_exit_code : p->exit_code; 1166 get_task_struct(p); 1167 read_unlock(&tasklist_lock); 1168 sched_annotate_sleep(); 1169 if (wo->wo_rusage) 1170 getrusage(p, RUSAGE_BOTH, wo->wo_rusage); 1171 put_task_struct(p); 1172 goto out_info; 1173 } 1174 /* 1175 * Move the task's state to DEAD/TRACE, only one thread can do this. 1176 */ 1177 state = (ptrace_reparented(p) && thread_group_leader(p)) ? 1178 EXIT_TRACE : EXIT_DEAD; 1179 if (cmpxchg(&p->exit_state, EXIT_ZOMBIE, state) != EXIT_ZOMBIE) 1180 return 0; 1181 /* 1182 * We own this thread, nobody else can reap it. 1183 */ 1184 read_unlock(&tasklist_lock); 1185 sched_annotate_sleep(); 1186 1187 /* 1188 * Check thread_group_leader() to exclude the traced sub-threads. 1189 */ 1190 if (state == EXIT_DEAD && thread_group_leader(p)) { 1191 struct signal_struct *sig = p->signal; 1192 struct signal_struct *psig = current->signal; 1193 unsigned long maxrss; 1194 u64 tgutime, tgstime; 1195 1196 /* 1197 * The resource counters for the group leader are in its 1198 * own task_struct. Those for dead threads in the group 1199 * are in its signal_struct, as are those for the child 1200 * processes it has previously reaped. All these 1201 * accumulate in the parent's signal_struct c* fields. 1202 * 1203 * We don't bother to take a lock here to protect these 1204 * p->signal fields because the whole thread group is dead 1205 * and nobody can change them. 1206 * 1207 * psig->stats_lock also protects us from our sub-threads 1208 * which can reap other children at the same time. 1209 * 1210 * We use thread_group_cputime_adjusted() to get times for 1211 * the thread group, which consolidates times for all threads 1212 * in the group including the group leader. 1213 */ 1214 thread_group_cputime_adjusted(p, &tgutime, &tgstime); 1215 write_seqlock_irq(&psig->stats_lock); 1216 psig->cutime += tgutime + sig->cutime; 1217 psig->cstime += tgstime + sig->cstime; 1218 psig->cgtime += task_gtime(p) + sig->gtime + sig->cgtime; 1219 psig->cmin_flt += 1220 p->min_flt + sig->min_flt + sig->cmin_flt; 1221 psig->cmaj_flt += 1222 p->maj_flt + sig->maj_flt + sig->cmaj_flt; 1223 psig->cnvcsw += 1224 p->nvcsw + sig->nvcsw + sig->cnvcsw; 1225 psig->cnivcsw += 1226 p->nivcsw + sig->nivcsw + sig->cnivcsw; 1227 psig->cinblock += 1228 task_io_get_inblock(p) + 1229 sig->inblock + sig->cinblock; 1230 psig->coublock += 1231 task_io_get_oublock(p) + 1232 sig->oublock + sig->coublock; 1233 maxrss = max(sig->maxrss, sig->cmaxrss); 1234 if (psig->cmaxrss < maxrss) 1235 psig->cmaxrss = maxrss; 1236 task_io_accounting_add(&psig->ioac, &p->ioac); 1237 task_io_accounting_add(&psig->ioac, &sig->ioac); 1238 write_sequnlock_irq(&psig->stats_lock); 1239 } 1240 1241 if (wo->wo_rusage) 1242 getrusage(p, RUSAGE_BOTH, wo->wo_rusage); 1243 status = (p->signal->flags & SIGNAL_GROUP_EXIT) 1244 ? p->signal->group_exit_code : p->exit_code; 1245 wo->wo_stat = status; 1246 1247 if (state == EXIT_TRACE) { 1248 write_lock_irq(&tasklist_lock); 1249 /* We dropped tasklist, ptracer could die and untrace */ 1250 ptrace_unlink(p); 1251 1252 /* If parent wants a zombie, don't release it now */ 1253 state = EXIT_ZOMBIE; 1254 if (do_notify_parent(p, p->exit_signal)) 1255 state = EXIT_DEAD; 1256 p->exit_state = state; 1257 write_unlock_irq(&tasklist_lock); 1258 } 1259 if (state == EXIT_DEAD) 1260 release_task(p); 1261 1262 out_info: 1263 infop = wo->wo_info; 1264 if (infop) { 1265 if ((status & 0x7f) == 0) { 1266 infop->cause = CLD_EXITED; 1267 infop->status = status >> 8; 1268 } else { 1269 infop->cause = (status & 0x80) ? CLD_DUMPED : CLD_KILLED; 1270 infop->status = status & 0x7f; 1271 } 1272 infop->pid = pid; 1273 infop->uid = uid; 1274 } 1275 1276 return pid; 1277 } 1278 1279 static int *task_stopped_code(struct task_struct *p, bool ptrace) 1280 { 1281 if (ptrace) { 1282 if (task_is_traced(p) && !(p->jobctl & JOBCTL_LISTENING)) 1283 return &p->exit_code; 1284 } else { 1285 if (p->signal->flags & SIGNAL_STOP_STOPPED) 1286 return &p->signal->group_exit_code; 1287 } 1288 return NULL; 1289 } 1290 1291 /** 1292 * wait_task_stopped - Wait for %TASK_STOPPED or %TASK_TRACED 1293 * @wo: wait options 1294 * @ptrace: is the wait for ptrace 1295 * @p: task to wait for 1296 * 1297 * Handle sys_wait4() work for %p in state %TASK_STOPPED or %TASK_TRACED. 1298 * 1299 * CONTEXT: 1300 * read_lock(&tasklist_lock), which is released if return value is 1301 * non-zero. Also, grabs and releases @p->sighand->siglock. 1302 * 1303 * RETURNS: 1304 * 0 if wait condition didn't exist and search for other wait conditions 1305 * should continue. Non-zero return, -errno on failure and @p's pid on 1306 * success, implies that tasklist_lock is released and wait condition 1307 * search should terminate. 1308 */ 1309 static int wait_task_stopped(struct wait_opts *wo, 1310 int ptrace, struct task_struct *p) 1311 { 1312 struct waitid_info *infop; 1313 int exit_code, *p_code, why; 1314 uid_t uid = 0; /* unneeded, required by compiler */ 1315 pid_t pid; 1316 1317 /* 1318 * Traditionally we see ptrace'd stopped tasks regardless of options. 1319 */ 1320 if (!ptrace && !(wo->wo_flags & WUNTRACED)) 1321 return 0; 1322 1323 if (!task_stopped_code(p, ptrace)) 1324 return 0; 1325 1326 exit_code = 0; 1327 spin_lock_irq(&p->sighand->siglock); 1328 1329 p_code = task_stopped_code(p, ptrace); 1330 if (unlikely(!p_code)) 1331 goto unlock_sig; 1332 1333 exit_code = *p_code; 1334 if (!exit_code) 1335 goto unlock_sig; 1336 1337 if (!unlikely(wo->wo_flags & WNOWAIT)) 1338 *p_code = 0; 1339 1340 uid = from_kuid_munged(current_user_ns(), task_uid(p)); 1341 unlock_sig: 1342 spin_unlock_irq(&p->sighand->siglock); 1343 if (!exit_code) 1344 return 0; 1345 1346 /* 1347 * Now we are pretty sure this task is interesting. 1348 * Make sure it doesn't get reaped out from under us while we 1349 * give up the lock and then examine it below. We don't want to 1350 * keep holding onto the tasklist_lock while we call getrusage and 1351 * possibly take page faults for user memory. 1352 */ 1353 get_task_struct(p); 1354 pid = task_pid_vnr(p); 1355 why = ptrace ? CLD_TRAPPED : CLD_STOPPED; 1356 read_unlock(&tasklist_lock); 1357 sched_annotate_sleep(); 1358 if (wo->wo_rusage) 1359 getrusage(p, RUSAGE_BOTH, wo->wo_rusage); 1360 put_task_struct(p); 1361 1362 if (likely(!(wo->wo_flags & WNOWAIT))) 1363 wo->wo_stat = (exit_code << 8) | 0x7f; 1364 1365 infop = wo->wo_info; 1366 if (infop) { 1367 infop->cause = why; 1368 infop->status = exit_code; 1369 infop->pid = pid; 1370 infop->uid = uid; 1371 } 1372 return pid; 1373 } 1374 1375 /* 1376 * Handle do_wait work for one task in a live, non-stopped state. 1377 * read_lock(&tasklist_lock) on entry. If we return zero, we still hold 1378 * the lock and this task is uninteresting. If we return nonzero, we have 1379 * released the lock and the system call should return. 1380 */ 1381 static int wait_task_continued(struct wait_opts *wo, struct task_struct *p) 1382 { 1383 struct waitid_info *infop; 1384 pid_t pid; 1385 uid_t uid; 1386 1387 if (!unlikely(wo->wo_flags & WCONTINUED)) 1388 return 0; 1389 1390 if (!(p->signal->flags & SIGNAL_STOP_CONTINUED)) 1391 return 0; 1392 1393 spin_lock_irq(&p->sighand->siglock); 1394 /* Re-check with the lock held. */ 1395 if (!(p->signal->flags & SIGNAL_STOP_CONTINUED)) { 1396 spin_unlock_irq(&p->sighand->siglock); 1397 return 0; 1398 } 1399 if (!unlikely(wo->wo_flags & WNOWAIT)) 1400 p->signal->flags &= ~SIGNAL_STOP_CONTINUED; 1401 uid = from_kuid_munged(current_user_ns(), task_uid(p)); 1402 spin_unlock_irq(&p->sighand->siglock); 1403 1404 pid = task_pid_vnr(p); 1405 get_task_struct(p); 1406 read_unlock(&tasklist_lock); 1407 sched_annotate_sleep(); 1408 if (wo->wo_rusage) 1409 getrusage(p, RUSAGE_BOTH, wo->wo_rusage); 1410 put_task_struct(p); 1411 1412 infop = wo->wo_info; 1413 if (!infop) { 1414 wo->wo_stat = 0xffff; 1415 } else { 1416 infop->cause = CLD_CONTINUED; 1417 infop->pid = pid; 1418 infop->uid = uid; 1419 infop->status = SIGCONT; 1420 } 1421 return pid; 1422 } 1423 1424 /* 1425 * Consider @p for a wait by @parent. 1426 * 1427 * -ECHILD should be in ->notask_error before the first call. 1428 * Returns nonzero for a final return, when we have unlocked tasklist_lock. 1429 * Returns zero if the search for a child should continue; 1430 * then ->notask_error is 0 if @p is an eligible child, 1431 * or still -ECHILD. 1432 */ 1433 static int wait_consider_task(struct wait_opts *wo, int ptrace, 1434 struct task_struct *p) 1435 { 1436 /* 1437 * We can race with wait_task_zombie() from another thread. 1438 * Ensure that EXIT_ZOMBIE -> EXIT_DEAD/EXIT_TRACE transition 1439 * can't confuse the checks below. 1440 */ 1441 int exit_state = READ_ONCE(p->exit_state); 1442 int ret; 1443 1444 if (unlikely(exit_state == EXIT_DEAD)) 1445 return 0; 1446 1447 ret = eligible_child(wo, ptrace, p); 1448 if (!ret) 1449 return ret; 1450 1451 if (unlikely(exit_state == EXIT_TRACE)) { 1452 /* 1453 * ptrace == 0 means we are the natural parent. In this case 1454 * we should clear notask_error, debugger will notify us. 1455 */ 1456 if (likely(!ptrace)) 1457 wo->notask_error = 0; 1458 return 0; 1459 } 1460 1461 if (likely(!ptrace) && unlikely(p->ptrace)) { 1462 /* 1463 * If it is traced by its real parent's group, just pretend 1464 * the caller is ptrace_do_wait() and reap this child if it 1465 * is zombie. 1466 * 1467 * This also hides group stop state from real parent; otherwise 1468 * a single stop can be reported twice as group and ptrace stop. 1469 * If a ptracer wants to distinguish these two events for its 1470 * own children it should create a separate process which takes 1471 * the role of real parent. 1472 */ 1473 if (!ptrace_reparented(p)) 1474 ptrace = 1; 1475 } 1476 1477 /* slay zombie? */ 1478 if (exit_state == EXIT_ZOMBIE) { 1479 /* we don't reap group leaders with subthreads */ 1480 if (!delay_group_leader(p)) { 1481 /* 1482 * A zombie ptracee is only visible to its ptracer. 1483 * Notification and reaping will be cascaded to the 1484 * real parent when the ptracer detaches. 1485 */ 1486 if (unlikely(ptrace) || likely(!p->ptrace)) 1487 return wait_task_zombie(wo, p); 1488 } 1489 1490 /* 1491 * Allow access to stopped/continued state via zombie by 1492 * falling through. Clearing of notask_error is complex. 1493 * 1494 * When !@ptrace: 1495 * 1496 * If WEXITED is set, notask_error should naturally be 1497 * cleared. If not, subset of WSTOPPED|WCONTINUED is set, 1498 * so, if there are live subthreads, there are events to 1499 * wait for. If all subthreads are dead, it's still safe 1500 * to clear - this function will be called again in finite 1501 * amount time once all the subthreads are released and 1502 * will then return without clearing. 1503 * 1504 * When @ptrace: 1505 * 1506 * Stopped state is per-task and thus can't change once the 1507 * target task dies. Only continued and exited can happen. 1508 * Clear notask_error if WCONTINUED | WEXITED. 1509 */ 1510 if (likely(!ptrace) || (wo->wo_flags & (WCONTINUED | WEXITED))) 1511 wo->notask_error = 0; 1512 } else { 1513 /* 1514 * @p is alive and it's gonna stop, continue or exit, so 1515 * there always is something to wait for. 1516 */ 1517 wo->notask_error = 0; 1518 } 1519 1520 /* 1521 * Wait for stopped. Depending on @ptrace, different stopped state 1522 * is used and the two don't interact with each other. 1523 */ 1524 ret = wait_task_stopped(wo, ptrace, p); 1525 if (ret) 1526 return ret; 1527 1528 /* 1529 * Wait for continued. There's only one continued state and the 1530 * ptracer can consume it which can confuse the real parent. Don't 1531 * use WCONTINUED from ptracer. You don't need or want it. 1532 */ 1533 return wait_task_continued(wo, p); 1534 } 1535 1536 /* 1537 * Do the work of do_wait() for one thread in the group, @tsk. 1538 * 1539 * -ECHILD should be in ->notask_error before the first call. 1540 * Returns nonzero for a final return, when we have unlocked tasklist_lock. 1541 * Returns zero if the search for a child should continue; then 1542 * ->notask_error is 0 if there were any eligible children, 1543 * or still -ECHILD. 1544 */ 1545 static int do_wait_thread(struct wait_opts *wo, struct task_struct *tsk) 1546 { 1547 struct task_struct *p; 1548 1549 list_for_each_entry(p, &tsk->children, sibling) { 1550 int ret = wait_consider_task(wo, 0, p); 1551 1552 if (ret) 1553 return ret; 1554 } 1555 1556 return 0; 1557 } 1558 1559 static int ptrace_do_wait(struct wait_opts *wo, struct task_struct *tsk) 1560 { 1561 struct task_struct *p; 1562 1563 list_for_each_entry(p, &tsk->ptraced, ptrace_entry) { 1564 int ret = wait_consider_task(wo, 1, p); 1565 1566 if (ret) 1567 return ret; 1568 } 1569 1570 return 0; 1571 } 1572 1573 bool pid_child_should_wake(struct wait_opts *wo, struct task_struct *p) 1574 { 1575 if (!eligible_pid(wo, p)) 1576 return false; 1577 1578 if ((wo->wo_flags & __WNOTHREAD) && wo->child_wait.private != p->parent) 1579 return false; 1580 1581 return true; 1582 } 1583 1584 static int child_wait_callback(wait_queue_entry_t *wait, unsigned mode, 1585 int sync, void *key) 1586 { 1587 struct wait_opts *wo = container_of(wait, struct wait_opts, 1588 child_wait); 1589 struct task_struct *p = key; 1590 1591 if (pid_child_should_wake(wo, p)) 1592 return default_wake_function(wait, mode, sync, key); 1593 1594 return 0; 1595 } 1596 1597 void __wake_up_parent(struct task_struct *p, struct task_struct *parent) 1598 { 1599 __wake_up_sync_key(&parent->signal->wait_chldexit, 1600 TASK_INTERRUPTIBLE, p); 1601 } 1602 1603 static bool is_effectively_child(struct wait_opts *wo, bool ptrace, 1604 struct task_struct *target) 1605 { 1606 struct task_struct *parent = 1607 !ptrace ? target->real_parent : target->parent; 1608 1609 return current == parent || (!(wo->wo_flags & __WNOTHREAD) && 1610 same_thread_group(current, parent)); 1611 } 1612 1613 /* 1614 * Optimization for waiting on PIDTYPE_PID. No need to iterate through child 1615 * and tracee lists to find the target task. 1616 */ 1617 static int do_wait_pid(struct wait_opts *wo) 1618 { 1619 bool ptrace; 1620 struct task_struct *target; 1621 int retval; 1622 1623 ptrace = false; 1624 target = pid_task(wo->wo_pid, PIDTYPE_TGID); 1625 if (target && is_effectively_child(wo, ptrace, target)) { 1626 retval = wait_consider_task(wo, ptrace, target); 1627 if (retval) 1628 return retval; 1629 } 1630 1631 ptrace = true; 1632 target = pid_task(wo->wo_pid, PIDTYPE_PID); 1633 if (target && target->ptrace && 1634 is_effectively_child(wo, ptrace, target)) { 1635 retval = wait_consider_task(wo, ptrace, target); 1636 if (retval) 1637 return retval; 1638 } 1639 1640 return 0; 1641 } 1642 1643 long __do_wait(struct wait_opts *wo) 1644 { 1645 long retval; 1646 1647 /* 1648 * If there is nothing that can match our criteria, just get out. 1649 * We will clear ->notask_error to zero if we see any child that 1650 * might later match our criteria, even if we are not able to reap 1651 * it yet. 1652 */ 1653 wo->notask_error = -ECHILD; 1654 if ((wo->wo_type < PIDTYPE_MAX) && 1655 (!wo->wo_pid || !pid_has_task(wo->wo_pid, wo->wo_type))) 1656 goto notask; 1657 1658 read_lock(&tasklist_lock); 1659 1660 if (wo->wo_type == PIDTYPE_PID) { 1661 retval = do_wait_pid(wo); 1662 if (retval) 1663 return retval; 1664 } else { 1665 struct task_struct *tsk = current; 1666 1667 do { 1668 retval = do_wait_thread(wo, tsk); 1669 if (retval) 1670 return retval; 1671 1672 retval = ptrace_do_wait(wo, tsk); 1673 if (retval) 1674 return retval; 1675 1676 if (wo->wo_flags & __WNOTHREAD) 1677 break; 1678 } while_each_thread(current, tsk); 1679 } 1680 read_unlock(&tasklist_lock); 1681 1682 notask: 1683 retval = wo->notask_error; 1684 if (!retval && !(wo->wo_flags & WNOHANG)) 1685 return -ERESTARTSYS; 1686 1687 return retval; 1688 } 1689 1690 static long do_wait(struct wait_opts *wo) 1691 { 1692 int retval; 1693 1694 trace_sched_process_wait(wo->wo_pid); 1695 1696 init_waitqueue_func_entry(&wo->child_wait, child_wait_callback); 1697 wo->child_wait.private = current; 1698 add_wait_queue(¤t->signal->wait_chldexit, &wo->child_wait); 1699 1700 do { 1701 set_current_state(TASK_INTERRUPTIBLE); 1702 retval = __do_wait(wo); 1703 if (retval != -ERESTARTSYS) 1704 break; 1705 if (signal_pending(current)) 1706 break; 1707 schedule(); 1708 } while (1); 1709 1710 __set_current_state(TASK_RUNNING); 1711 remove_wait_queue(¤t->signal->wait_chldexit, &wo->child_wait); 1712 return retval; 1713 } 1714 1715 int kernel_waitid_prepare(struct wait_opts *wo, int which, pid_t upid, 1716 struct waitid_info *infop, int options, 1717 struct rusage *ru) 1718 { 1719 unsigned int f_flags = 0; 1720 struct pid *pid = NULL; 1721 enum pid_type type; 1722 1723 if (options & ~(WNOHANG|WNOWAIT|WEXITED|WSTOPPED|WCONTINUED| 1724 __WNOTHREAD|__WCLONE|__WALL)) 1725 return -EINVAL; 1726 if (!(options & (WEXITED|WSTOPPED|WCONTINUED))) 1727 return -EINVAL; 1728 1729 switch (which) { 1730 case P_ALL: 1731 type = PIDTYPE_MAX; 1732 break; 1733 case P_PID: 1734 type = PIDTYPE_PID; 1735 if (upid <= 0) 1736 return -EINVAL; 1737 1738 pid = find_get_pid(upid); 1739 break; 1740 case P_PGID: 1741 type = PIDTYPE_PGID; 1742 if (upid < 0) 1743 return -EINVAL; 1744 1745 if (upid) 1746 pid = find_get_pid(upid); 1747 else 1748 pid = get_task_pid(current, PIDTYPE_PGID); 1749 break; 1750 case P_PIDFD: 1751 type = PIDTYPE_PID; 1752 if (upid < 0) 1753 return -EINVAL; 1754 1755 pid = pidfd_get_pid(upid, &f_flags); 1756 if (IS_ERR(pid)) 1757 return PTR_ERR(pid); 1758 1759 break; 1760 default: 1761 return -EINVAL; 1762 } 1763 1764 wo->wo_type = type; 1765 wo->wo_pid = pid; 1766 wo->wo_flags = options; 1767 wo->wo_info = infop; 1768 wo->wo_rusage = ru; 1769 if (f_flags & O_NONBLOCK) 1770 wo->wo_flags |= WNOHANG; 1771 1772 return 0; 1773 } 1774 1775 static long kernel_waitid(int which, pid_t upid, struct waitid_info *infop, 1776 int options, struct rusage *ru) 1777 { 1778 struct wait_opts wo; 1779 long ret; 1780 1781 ret = kernel_waitid_prepare(&wo, which, upid, infop, options, ru); 1782 if (ret) 1783 return ret; 1784 1785 ret = do_wait(&wo); 1786 if (!ret && !(options & WNOHANG) && (wo.wo_flags & WNOHANG)) 1787 ret = -EAGAIN; 1788 1789 put_pid(wo.wo_pid); 1790 return ret; 1791 } 1792 1793 SYSCALL_DEFINE5(waitid, int, which, pid_t, upid, struct siginfo __user *, 1794 infop, int, options, struct rusage __user *, ru) 1795 { 1796 struct rusage r; 1797 struct waitid_info info = {.status = 0}; 1798 long err = kernel_waitid(which, upid, &info, options, ru ? &r : NULL); 1799 int signo = 0; 1800 1801 if (err > 0) { 1802 signo = SIGCHLD; 1803 err = 0; 1804 if (ru && copy_to_user(ru, &r, sizeof(struct rusage))) 1805 return -EFAULT; 1806 } 1807 if (!infop) 1808 return err; 1809 1810 if (!user_write_access_begin(infop, sizeof(*infop))) 1811 return -EFAULT; 1812 1813 unsafe_put_user(signo, &infop->si_signo, Efault); 1814 unsafe_put_user(0, &infop->si_errno, Efault); 1815 unsafe_put_user(info.cause, &infop->si_code, Efault); 1816 unsafe_put_user(info.pid, &infop->si_pid, Efault); 1817 unsafe_put_user(info.uid, &infop->si_uid, Efault); 1818 unsafe_put_user(info.status, &infop->si_status, Efault); 1819 user_write_access_end(); 1820 return err; 1821 Efault: 1822 user_write_access_end(); 1823 return -EFAULT; 1824 } 1825 1826 long kernel_wait4(pid_t upid, int __user *stat_addr, int options, 1827 struct rusage *ru) 1828 { 1829 struct wait_opts wo; 1830 struct pid *pid = NULL; 1831 enum pid_type type; 1832 long ret; 1833 1834 if (options & ~(WNOHANG|WUNTRACED|WCONTINUED| 1835 __WNOTHREAD|__WCLONE|__WALL)) 1836 return -EINVAL; 1837 1838 /* -INT_MIN is not defined */ 1839 if (upid == INT_MIN) 1840 return -ESRCH; 1841 1842 if (upid == -1) 1843 type = PIDTYPE_MAX; 1844 else if (upid < 0) { 1845 type = PIDTYPE_PGID; 1846 pid = find_get_pid(-upid); 1847 } else if (upid == 0) { 1848 type = PIDTYPE_PGID; 1849 pid = get_task_pid(current, PIDTYPE_PGID); 1850 } else /* upid > 0 */ { 1851 type = PIDTYPE_PID; 1852 pid = find_get_pid(upid); 1853 } 1854 1855 wo.wo_type = type; 1856 wo.wo_pid = pid; 1857 wo.wo_flags = options | WEXITED; 1858 wo.wo_info = NULL; 1859 wo.wo_stat = 0; 1860 wo.wo_rusage = ru; 1861 ret = do_wait(&wo); 1862 put_pid(pid); 1863 if (ret > 0 && stat_addr && put_user(wo.wo_stat, stat_addr)) 1864 ret = -EFAULT; 1865 1866 return ret; 1867 } 1868 1869 int kernel_wait(pid_t pid, int *stat) 1870 { 1871 struct wait_opts wo = { 1872 .wo_type = PIDTYPE_PID, 1873 .wo_pid = find_get_pid(pid), 1874 .wo_flags = WEXITED, 1875 }; 1876 int ret; 1877 1878 ret = do_wait(&wo); 1879 if (ret > 0 && wo.wo_stat) 1880 *stat = wo.wo_stat; 1881 put_pid(wo.wo_pid); 1882 return ret; 1883 } 1884 1885 SYSCALL_DEFINE4(wait4, pid_t, upid, int __user *, stat_addr, 1886 int, options, struct rusage __user *, ru) 1887 { 1888 struct rusage r; 1889 long err = kernel_wait4(upid, stat_addr, options, ru ? &r : NULL); 1890 1891 if (err > 0) { 1892 if (ru && copy_to_user(ru, &r, sizeof(struct rusage))) 1893 return -EFAULT; 1894 } 1895 return err; 1896 } 1897 1898 #ifdef __ARCH_WANT_SYS_WAITPID 1899 1900 /* 1901 * sys_waitpid() remains for compatibility. waitpid() should be 1902 * implemented by calling sys_wait4() from libc.a. 1903 */ 1904 SYSCALL_DEFINE3(waitpid, pid_t, pid, int __user *, stat_addr, int, options) 1905 { 1906 return kernel_wait4(pid, stat_addr, options, NULL); 1907 } 1908 1909 #endif 1910 1911 #ifdef CONFIG_COMPAT 1912 COMPAT_SYSCALL_DEFINE4(wait4, 1913 compat_pid_t, pid, 1914 compat_uint_t __user *, stat_addr, 1915 int, options, 1916 struct compat_rusage __user *, ru) 1917 { 1918 struct rusage r; 1919 long err = kernel_wait4(pid, stat_addr, options, ru ? &r : NULL); 1920 if (err > 0) { 1921 if (ru && put_compat_rusage(&r, ru)) 1922 return -EFAULT; 1923 } 1924 return err; 1925 } 1926 1927 COMPAT_SYSCALL_DEFINE5(waitid, 1928 int, which, compat_pid_t, pid, 1929 struct compat_siginfo __user *, infop, int, options, 1930 struct compat_rusage __user *, uru) 1931 { 1932 struct rusage ru; 1933 struct waitid_info info = {.status = 0}; 1934 long err = kernel_waitid(which, pid, &info, options, uru ? &ru : NULL); 1935 int signo = 0; 1936 if (err > 0) { 1937 signo = SIGCHLD; 1938 err = 0; 1939 if (uru) { 1940 /* kernel_waitid() overwrites everything in ru */ 1941 if (COMPAT_USE_64BIT_TIME) 1942 err = copy_to_user(uru, &ru, sizeof(ru)); 1943 else 1944 err = put_compat_rusage(&ru, uru); 1945 if (err) 1946 return -EFAULT; 1947 } 1948 } 1949 1950 if (!infop) 1951 return err; 1952 1953 if (!user_write_access_begin(infop, sizeof(*infop))) 1954 return -EFAULT; 1955 1956 unsafe_put_user(signo, &infop->si_signo, Efault); 1957 unsafe_put_user(0, &infop->si_errno, Efault); 1958 unsafe_put_user(info.cause, &infop->si_code, Efault); 1959 unsafe_put_user(info.pid, &infop->si_pid, Efault); 1960 unsafe_put_user(info.uid, &infop->si_uid, Efault); 1961 unsafe_put_user(info.status, &infop->si_status, Efault); 1962 user_write_access_end(); 1963 return err; 1964 Efault: 1965 user_write_access_end(); 1966 return -EFAULT; 1967 } 1968 #endif 1969 1970 /* 1971 * This needs to be __function_aligned as GCC implicitly makes any 1972 * implementation of abort() cold and drops alignment specified by 1973 * -falign-functions=N. 1974 * 1975 * See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=88345#c11 1976 */ 1977 __weak __function_aligned void abort(void) 1978 { 1979 BUG(); 1980 1981 /* if that doesn't kill us, halt */ 1982 panic("Oops failed to kill thread"); 1983 } 1984 EXPORT_SYMBOL(abort); 1985