1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/kernel/fork.c 4 * 5 * Copyright (C) 1991, 1992 Linus Torvalds 6 */ 7 8 /* 9 * 'fork.c' contains the help-routines for the 'fork' system call 10 * (see also entry.S and others). 11 * Fork is rather simple, once you get the hang of it, but the memory 12 * management can be a bitch. See 'mm/memory.c': 'copy_page_range()' 13 */ 14 15 #include <linux/anon_inodes.h> 16 #include <linux/slab.h> 17 #include <linux/sched/autogroup.h> 18 #include <linux/sched/mm.h> 19 #include <linux/sched/user.h> 20 #include <linux/sched/numa_balancing.h> 21 #include <linux/sched/stat.h> 22 #include <linux/sched/task.h> 23 #include <linux/sched/task_stack.h> 24 #include <linux/sched/cputime.h> 25 #include <linux/sched/ext.h> 26 #include <linux/sched/exec_state.h> 27 #include <linux/seq_file.h> 28 #include <linux/rtmutex.h> 29 #include <linux/init.h> 30 #include <linux/unistd.h> 31 #include <linux/module.h> 32 #include <linux/vmalloc.h> 33 #include <linux/completion.h> 34 #include <linux/personality.h> 35 #include <linux/mempolicy.h> 36 #include <linux/sem.h> 37 #include <linux/file.h> 38 #include <linux/fdtable.h> 39 #include <linux/iocontext.h> 40 #include <linux/key.h> 41 #include <linux/kmsan.h> 42 #include <linux/binfmts.h> 43 #include <linux/mman.h> 44 #include <linux/mmu_notifier.h> 45 #include <linux/fs.h> 46 #include <linux/mm.h> 47 #include <linux/mm_inline.h> 48 #include <linux/memblock.h> 49 #include <linux/nsproxy.h> 50 #include <linux/ns/ns_common_types.h> 51 #include <linux/capability.h> 52 #include <linux/cpu.h> 53 #include <linux/cgroup.h> 54 #include <linux/security.h> 55 #include <linux/hugetlb.h> 56 #include <linux/seccomp.h> 57 #include <linux/swap.h> 58 #include <linux/syscalls.h> 59 #include <linux/syscall_user_dispatch.h> 60 #include <linux/jiffies.h> 61 #include <linux/futex.h> 62 #include <linux/compat.h> 63 #include <linux/kthread.h> 64 #include <linux/task_io_accounting_ops.h> 65 #include <linux/rcupdate.h> 66 #include <linux/ptrace.h> 67 #include <linux/mount.h> 68 #include <linux/audit.h> 69 #include <linux/memcontrol.h> 70 #include <linux/ftrace.h> 71 #include <linux/proc_fs.h> 72 #include <linux/profile.h> 73 #include <linux/rmap.h> 74 #include <linux/ksm.h> 75 #include <linux/acct.h> 76 #include <linux/userfaultfd_k.h> 77 #include <linux/tsacct_kern.h> 78 #include <linux/cn_proc.h> 79 #include <linux/freezer.h> 80 #include <linux/delayacct.h> 81 #include <linux/taskstats_kern.h> 82 #include <linux/tty.h> 83 #include <linux/fs_struct.h> 84 #include <linux/magic.h> 85 #include <linux/perf_event.h> 86 #include <linux/posix-timers.h> 87 #include <linux/user-return-notifier.h> 88 #include <linux/oom.h> 89 #include <linux/khugepaged.h> 90 #include <linux/signalfd.h> 91 #include <linux/uprobes.h> 92 #include <linux/aio.h> 93 #include <linux/compiler.h> 94 #include <linux/sysctl.h> 95 #include <linux/kcov.h> 96 #include <linux/livepatch.h> 97 #include <linux/thread_info.h> 98 #include <linux/kstack_erase.h> 99 #include <linux/kasan.h> 100 #include <linux/randomize_kstack.h> 101 #include <linux/scs.h> 102 #include <linux/io_uring.h> 103 #include <linux/io_uring_types.h> 104 #include <linux/bpf.h> 105 #include <linux/stackprotector.h> 106 #include <linux/user_events.h> 107 #include <linux/iommu.h> 108 #include <linux/rseq.h> 109 #include <uapi/linux/pidfd.h> 110 #include <linux/pidfs.h> 111 #include <linux/tick.h> 112 #include <linux/unwind_deferred.h> 113 #include <linux/pgalloc.h> 114 #include <linux/uaccess.h> 115 116 #include <asm/mmu_context.h> 117 #include <asm/cacheflush.h> 118 #include <asm/tlbflush.h> 119 120 /* For dup_mmap(). */ 121 #include "../mm/internal.h" 122 123 #include <trace/events/sched.h> 124 125 #define CREATE_TRACE_POINTS 126 #include <trace/events/task.h> 127 128 #include <kunit/visibility.h> 129 130 /* 131 * Minimum number of threads to boot the kernel 132 */ 133 #define MIN_THREADS 20 134 135 /* 136 * Maximum number of threads 137 */ 138 #define MAX_THREADS FUTEX_TID_MASK 139 140 /* 141 * Protected counters by write_lock_irq(&tasklist_lock) 142 */ 143 unsigned long total_forks; /* Handle normal Linux uptimes. */ 144 int nr_threads; /* The idle threads do not count.. */ 145 146 static int max_threads __read_mostly; /* tunable limit on nr_threads */ 147 148 #define NAMED_ARRAY_INDEX(x) [x] = __stringify(x) 149 150 static const char * const resident_page_types[] = { 151 NAMED_ARRAY_INDEX(MM_FILEPAGES), 152 NAMED_ARRAY_INDEX(MM_ANONPAGES), 153 NAMED_ARRAY_INDEX(MM_SWAPENTS), 154 NAMED_ARRAY_INDEX(MM_SHMEMPAGES), 155 }; 156 157 DEFINE_PER_CPU(unsigned long, process_counts) = 0; 158 159 __cacheline_aligned DEFINE_RWLOCK(tasklist_lock); /* outer */ 160 161 #ifdef CONFIG_PROVE_RCU 162 int lockdep_tasklist_lock_is_held(void) 163 { 164 return lockdep_is_held(&tasklist_lock); 165 } 166 EXPORT_SYMBOL_GPL(lockdep_tasklist_lock_is_held); 167 #endif /* #ifdef CONFIG_PROVE_RCU */ 168 169 int nr_processes(void) 170 { 171 int cpu; 172 int total = 0; 173 174 for_each_possible_cpu(cpu) 175 total += per_cpu(process_counts, cpu); 176 177 return total; 178 } 179 180 void __weak arch_release_task_struct(struct task_struct *tsk) 181 { 182 } 183 184 static struct kmem_cache *task_struct_cachep; 185 186 static inline struct task_struct *alloc_task_struct_node(int node) 187 { 188 return kmem_cache_alloc_node(task_struct_cachep, GFP_KERNEL, node); 189 } 190 191 static inline void free_task_struct(struct task_struct *tsk) 192 { 193 kmem_cache_free(task_struct_cachep, tsk); 194 } 195 196 #ifdef CONFIG_VMAP_STACK 197 /* 198 * vmalloc() is a bit slow, and calling vfree() enough times will force a TLB 199 * flush. Try to minimize the number of calls by caching stacks. 200 */ 201 #define NR_CACHED_STACKS 2 202 static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]); 203 /* 204 * Allocated stacks are cached and later reused by new threads, so memcg 205 * accounting is performed by the code assigning/releasing stacks to tasks. 206 * We need a zeroed memory without __GFP_ACCOUNT. 207 */ 208 #define GFP_VMAP_STACK (GFP_KERNEL | __GFP_ZERO | __GFP_SKIP_KASAN) 209 210 struct vm_stack { 211 struct rcu_head rcu; 212 struct vm_struct *stack_vm_area; 213 }; 214 215 static struct vm_struct *alloc_thread_stack_node_from_cache(struct task_struct *tsk, int node) 216 { 217 struct vm_struct *vm_area; 218 unsigned int i; 219 220 /* 221 * If the node has memory, we are guaranteed the stacks are backed by local pages. 222 * Otherwise the pages are arbitrary. 223 * 224 * Note that depending on cpuset it is possible we will get migrated to a different 225 * node immediately after allocating here, so this does *not* guarantee locality for 226 * arbitrary callers. 227 */ 228 scoped_guard(preempt) { 229 if (node != NUMA_NO_NODE && numa_node_id() != node) 230 return NULL; 231 232 for (i = 0; i < NR_CACHED_STACKS; i++) { 233 vm_area = this_cpu_xchg(cached_stacks[i], NULL); 234 if (vm_area) 235 return vm_area; 236 } 237 } 238 239 return NULL; 240 } 241 242 static bool try_release_thread_stack_to_cache(struct vm_struct *vm_area) 243 { 244 unsigned int i; 245 int nid; 246 247 /* 248 * Don't cache stacks if any of the pages don't match the local domain, unless 249 * there is no local memory to begin with. 250 * 251 * Note that lack of local memory does not automatically mean it makes no difference 252 * performance-wise which other domain backs the stack. In this case we are merely 253 * trying to avoid constantly going to vmalloc. 254 */ 255 scoped_guard(preempt) { 256 nid = numa_node_id(); 257 if (node_state(nid, N_MEMORY)) { 258 for (i = 0; i < vm_area->nr_pages; i++) { 259 struct page *page = vm_area->pages[i]; 260 if (page_to_nid(page) != nid) 261 return false; 262 } 263 } 264 265 for (i = 0; i < NR_CACHED_STACKS; i++) { 266 struct vm_struct *tmp = NULL; 267 268 if (this_cpu_try_cmpxchg(cached_stacks[i], &tmp, vm_area)) 269 return true; 270 } 271 } 272 return false; 273 } 274 275 static void thread_stack_free_rcu(struct rcu_head *rh) 276 { 277 struct vm_stack *vm_stack = container_of(rh, struct vm_stack, rcu); 278 struct vm_struct *vm_area = vm_stack->stack_vm_area; 279 280 if (try_release_thread_stack_to_cache(vm_stack->stack_vm_area)) 281 return; 282 283 vfree(vm_area->addr); 284 } 285 286 static void thread_stack_delayed_free(struct task_struct *tsk) 287 { 288 struct vm_stack *vm_stack = tsk->stack; 289 290 vm_stack->stack_vm_area = tsk->stack_vm_area; 291 call_rcu(&vm_stack->rcu, thread_stack_free_rcu); 292 } 293 294 static int free_vm_stack_cache(unsigned int cpu) 295 { 296 struct vm_struct **cached_vm_stack_areas = per_cpu_ptr(cached_stacks, cpu); 297 int i; 298 299 for (i = 0; i < NR_CACHED_STACKS; i++) { 300 struct vm_struct *vm_area = cached_vm_stack_areas[i]; 301 302 if (!vm_area) 303 continue; 304 305 vfree(vm_area->addr); 306 cached_vm_stack_areas[i] = NULL; 307 } 308 309 return 0; 310 } 311 312 static int memcg_charge_kernel_stack(struct vm_struct *vm_area) 313 { 314 int i; 315 int ret; 316 int nr_charged = 0; 317 318 BUG_ON(vm_area->nr_pages != THREAD_SIZE / PAGE_SIZE); 319 320 for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) { 321 ret = memcg_kmem_charge_page(vm_area->pages[i], GFP_KERNEL, 0); 322 if (ret) 323 goto err; 324 nr_charged++; 325 } 326 return 0; 327 err: 328 for (i = 0; i < nr_charged; i++) 329 memcg_kmem_uncharge_page(vm_area->pages[i], 0); 330 return ret; 331 } 332 333 static int alloc_thread_stack_node(struct task_struct *tsk, int node) 334 { 335 struct vm_struct *vm_area; 336 void *stack; 337 338 vm_area = alloc_thread_stack_node_from_cache(tsk, node); 339 if (vm_area) { 340 if (memcg_charge_kernel_stack(vm_area)) { 341 vfree(vm_area->addr); 342 return -ENOMEM; 343 } 344 345 /* Reset stack metadata. */ 346 if (!kasan_hw_tags_enabled()) 347 kasan_unpoison_range(vm_area->addr, THREAD_SIZE); 348 349 stack = kasan_reset_tag(vm_area->addr); 350 351 /* Clear stale pointers from reused stack. */ 352 clear_pages(vm_area->addr, vm_area->nr_pages); 353 354 tsk->stack_vm_area = vm_area; 355 tsk->stack = stack; 356 return 0; 357 } 358 359 stack = __vmalloc_node(THREAD_SIZE, THREAD_ALIGN, 360 GFP_VMAP_STACK, 361 node, __builtin_return_address(0)); 362 if (!stack) 363 return -ENOMEM; 364 365 vm_area = find_vm_area(stack); 366 if (memcg_charge_kernel_stack(vm_area)) { 367 vfree(stack); 368 return -ENOMEM; 369 } 370 /* 371 * We can't call find_vm_area() in interrupt context, and 372 * free_thread_stack() can be called in interrupt context, 373 * so cache the vm_struct. 374 */ 375 tsk->stack_vm_area = vm_area; 376 stack = kasan_reset_tag(stack); 377 tsk->stack = stack; 378 return 0; 379 } 380 381 static void free_thread_stack(struct task_struct *tsk) 382 { 383 if (!try_release_thread_stack_to_cache(tsk->stack_vm_area)) 384 thread_stack_delayed_free(tsk); 385 386 tsk->stack = NULL; 387 tsk->stack_vm_area = NULL; 388 } 389 390 #else /* !CONFIG_VMAP_STACK */ 391 392 /* 393 * Allocate pages if THREAD_SIZE is >= PAGE_SIZE, otherwise use a 394 * kmemcache based allocator. 395 */ 396 #if THREAD_SIZE >= PAGE_SIZE 397 398 static void thread_stack_free_rcu(struct rcu_head *rh) 399 { 400 __free_pages(virt_to_page(rh), THREAD_SIZE_ORDER); 401 } 402 403 static void thread_stack_delayed_free(struct task_struct *tsk) 404 { 405 struct rcu_head *rh = tsk->stack; 406 407 call_rcu(rh, thread_stack_free_rcu); 408 } 409 410 static int alloc_thread_stack_node(struct task_struct *tsk, int node) 411 { 412 struct page *page = alloc_pages_node(node, THREADINFO_GFP, 413 THREAD_SIZE_ORDER); 414 415 if (likely(page)) { 416 tsk->stack = kasan_reset_tag(page_address(page)); 417 return 0; 418 } 419 return -ENOMEM; 420 } 421 422 static void free_thread_stack(struct task_struct *tsk) 423 { 424 thread_stack_delayed_free(tsk); 425 tsk->stack = NULL; 426 } 427 428 #else /* !(THREAD_SIZE >= PAGE_SIZE) */ 429 430 static struct kmem_cache *thread_stack_cache; 431 432 static void thread_stack_free_rcu(struct rcu_head *rh) 433 { 434 kmem_cache_free(thread_stack_cache, rh); 435 } 436 437 static void thread_stack_delayed_free(struct task_struct *tsk) 438 { 439 struct rcu_head *rh = tsk->stack; 440 441 call_rcu(rh, thread_stack_free_rcu); 442 } 443 444 static int alloc_thread_stack_node(struct task_struct *tsk, int node) 445 { 446 unsigned long *stack; 447 stack = kmem_cache_alloc_node(thread_stack_cache, THREADINFO_GFP, node); 448 stack = kasan_reset_tag(stack); 449 tsk->stack = stack; 450 return stack ? 0 : -ENOMEM; 451 } 452 453 static void free_thread_stack(struct task_struct *tsk) 454 { 455 thread_stack_delayed_free(tsk); 456 tsk->stack = NULL; 457 } 458 459 void thread_stack_cache_init(void) 460 { 461 thread_stack_cache = kmem_cache_create_usercopy("thread_stack", 462 THREAD_SIZE, THREAD_SIZE, 0, 0, 463 THREAD_SIZE, NULL); 464 BUG_ON(thread_stack_cache == NULL); 465 } 466 467 #endif /* THREAD_SIZE >= PAGE_SIZE */ 468 #endif /* CONFIG_VMAP_STACK */ 469 470 /* SLAB cache for signal_struct structures (tsk->signal) */ 471 static struct kmem_cache *signal_cachep; 472 473 /* SLAB cache for sighand_struct structures (tsk->sighand) */ 474 struct kmem_cache *sighand_cachep; 475 476 /* SLAB cache for files_struct structures (tsk->files) */ 477 struct kmem_cache *files_cachep; 478 479 /* SLAB cache for fs_struct structures (tsk->fs) */ 480 struct kmem_cache *fs_cachep; 481 482 /* SLAB cache for mm_struct structures (tsk->mm) */ 483 static struct kmem_cache *mm_cachep; 484 485 static void account_kernel_stack(struct task_struct *tsk, int account) 486 { 487 if (IS_ENABLED(CONFIG_VMAP_STACK)) { 488 struct vm_struct *vm_area = task_stack_vm_area(tsk); 489 int i; 490 491 for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) 492 mod_lruvec_page_state(vm_area->pages[i], NR_KERNEL_STACK_KB, 493 account * (PAGE_SIZE / 1024)); 494 } else { 495 void *stack = task_stack_page(tsk); 496 497 /* All stack pages are in the same node. */ 498 mod_lruvec_kmem_state(stack, NR_KERNEL_STACK_KB, 499 account * (THREAD_SIZE / 1024)); 500 } 501 } 502 503 void exit_task_stack_account(struct task_struct *tsk) 504 { 505 account_kernel_stack(tsk, -1); 506 507 if (IS_ENABLED(CONFIG_VMAP_STACK)) { 508 struct vm_struct *vm_area; 509 int i; 510 511 vm_area = task_stack_vm_area(tsk); 512 for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) 513 memcg_kmem_uncharge_page(vm_area->pages[i], 0); 514 } 515 } 516 517 static void release_task_stack(struct task_struct *tsk) 518 { 519 if (WARN_ON(READ_ONCE(tsk->__state) != TASK_DEAD)) 520 return; /* Better to leak the stack than to free prematurely */ 521 522 free_thread_stack(tsk); 523 } 524 525 #ifdef CONFIG_THREAD_INFO_IN_TASK 526 void put_task_stack(struct task_struct *tsk) 527 { 528 if (refcount_dec_and_test(&tsk->stack_refcount)) 529 release_task_stack(tsk); 530 } 531 #endif 532 533 void free_task(struct task_struct *tsk) 534 { 535 #ifdef CONFIG_SECCOMP 536 WARN_ON_ONCE(tsk->seccomp.filter); 537 #endif 538 release_user_cpus_ptr(tsk); 539 scs_release(tsk); 540 smp_task_ipi_mask_free(tsk); 541 542 #ifndef CONFIG_THREAD_INFO_IN_TASK 543 /* 544 * The task is finally done with both the stack and thread_info, 545 * so free both. 546 */ 547 release_task_stack(tsk); 548 #else 549 /* 550 * If the task had a separate stack allocation, it should be gone 551 * by now. 552 */ 553 WARN_ON_ONCE(refcount_read(&tsk->stack_refcount) != 0); 554 #endif 555 rt_mutex_debug_task_free(tsk); 556 ftrace_graph_exit_task(tsk); 557 arch_release_task_struct(tsk); 558 if (tsk->flags & PF_KTHREAD) 559 free_kthread_struct(tsk); 560 bpf_task_storage_free(tsk); 561 put_task_exec_state(rcu_access_pointer(tsk->exec_state)); 562 free_task_struct(tsk); 563 } 564 EXPORT_SYMBOL(free_task); 565 566 void dup_mm_exe_file(struct mm_struct *mm, struct mm_struct *oldmm) 567 { 568 struct file *exe_file; 569 570 exe_file = get_mm_exe_file(oldmm); 571 RCU_INIT_POINTER(mm->exe_file, exe_file); 572 /* 573 * We depend on the oldmm having properly denied write access to the 574 * exe_file already. 575 */ 576 if (exe_file && exe_file_deny_write_access(exe_file)) 577 pr_warn_once("exe_file_deny_write_access() failed in %s\n", __func__); 578 } 579 580 #ifdef CONFIG_MMU 581 static inline int mm_alloc_pgd(struct mm_struct *mm) 582 { 583 mm->pgd = pgd_alloc(mm); 584 if (unlikely(!mm->pgd)) 585 return -ENOMEM; 586 return 0; 587 } 588 589 static inline void mm_free_pgd(struct mm_struct *mm) 590 { 591 pgd_free(mm, mm->pgd); 592 } 593 #else 594 #define mm_alloc_pgd(mm) (0) 595 #define mm_free_pgd(mm) 596 #endif /* CONFIG_MMU */ 597 598 #ifdef CONFIG_MM_ID 599 static DEFINE_IDA(mm_ida); 600 601 static inline int mm_alloc_id(struct mm_struct *mm) 602 { 603 int ret; 604 605 ret = ida_alloc_range(&mm_ida, MM_ID_MIN, MM_ID_MAX, GFP_KERNEL); 606 if (ret < 0) 607 return ret; 608 mm->mm_id = ret; 609 return 0; 610 } 611 612 static inline void mm_free_id(struct mm_struct *mm) 613 { 614 const mm_id_t id = mm->mm_id; 615 616 mm->mm_id = MM_ID_DUMMY; 617 if (id == MM_ID_DUMMY) 618 return; 619 if (WARN_ON_ONCE(id < MM_ID_MIN || id > MM_ID_MAX)) 620 return; 621 ida_free(&mm_ida, id); 622 } 623 #else /* !CONFIG_MM_ID */ 624 static inline int mm_alloc_id(struct mm_struct *mm) { return 0; } 625 static inline void mm_free_id(struct mm_struct *mm) {} 626 #endif /* CONFIG_MM_ID */ 627 628 static void check_mm(struct mm_struct *mm) 629 { 630 int i; 631 632 BUILD_BUG_ON_MSG(ARRAY_SIZE(resident_page_types) != NR_MM_COUNTERS, 633 "Please make sure 'struct resident_page_types[]' is updated as well"); 634 635 for (i = 0; i < NR_MM_COUNTERS; i++) { 636 long x = percpu_counter_sum(&mm->rss_stat[i]); 637 638 if (unlikely(x)) { 639 pr_alert("BUG: Bad rss-counter state mm:%p type:%s val:%ld Comm:%s Pid:%d\n", 640 mm, resident_page_types[i], x, 641 current->comm, 642 task_pid_nr(current)); 643 } 644 } 645 646 if (mm_pgtables_bytes(mm)) 647 pr_alert("BUG: non-zero pgtables_bytes on freeing mm: %ld\n", 648 mm_pgtables_bytes(mm)); 649 650 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !defined(CONFIG_SPLIT_PMD_PTLOCKS) 651 VM_BUG_ON_MM(mm->pmd_huge_pte, mm); 652 #endif 653 } 654 655 #define allocate_mm() (kmem_cache_alloc(mm_cachep, GFP_KERNEL)) 656 #define free_mm(mm) (kmem_cache_free(mm_cachep, (mm))) 657 658 static void do_check_lazy_tlb(void *arg) 659 { 660 struct mm_struct *mm = arg; 661 662 WARN_ON_ONCE(current->active_mm == mm); 663 } 664 665 static void do_shoot_lazy_tlb(void *arg) 666 { 667 struct mm_struct *mm = arg; 668 669 if (current->active_mm == mm) { 670 WARN_ON_ONCE(current->mm); 671 current->active_mm = &init_mm; 672 switch_mm(mm, &init_mm, current); 673 } 674 } 675 676 static void cleanup_lazy_tlbs(struct mm_struct *mm) 677 { 678 if (!IS_ENABLED(CONFIG_MMU_LAZY_TLB_SHOOTDOWN)) { 679 /* 680 * In this case, lazy tlb mms are refounted and would not reach 681 * __mmdrop until all CPUs have switched away and mmdrop()ed. 682 */ 683 return; 684 } 685 686 /* 687 * Lazy mm shootdown does not refcount "lazy tlb mm" usage, rather it 688 * requires lazy mm users to switch to another mm when the refcount 689 * drops to zero, before the mm is freed. This requires IPIs here to 690 * switch kernel threads to init_mm. 691 * 692 * archs that use IPIs to flush TLBs can piggy-back that lazy tlb mm 693 * switch with the final userspace teardown TLB flush which leaves the 694 * mm lazy on this CPU but no others, reducing the need for additional 695 * IPIs here. There are cases where a final IPI is still required here, 696 * such as the final mmdrop being performed on a different CPU than the 697 * one exiting, or kernel threads using the mm when userspace exits. 698 * 699 * IPI overheads have not found to be expensive, but they could be 700 * reduced in a number of possible ways, for example (roughly 701 * increasing order of complexity): 702 * - The last lazy reference created by exit_mm() could instead switch 703 * to init_mm, however it's probable this will run on the same CPU 704 * immediately afterwards, so this may not reduce IPIs much. 705 * - A batch of mms requiring IPIs could be gathered and freed at once. 706 * - CPUs store active_mm where it can be remotely checked without a 707 * lock, to filter out false-positives in the cpumask. 708 * - After mm_users or mm_count reaches zero, switching away from the 709 * mm could clear mm_cpumask to reduce some IPIs, perhaps together 710 * with some batching or delaying of the final IPIs. 711 * - A delayed freeing and RCU-like quiescing sequence based on mm 712 * switching to avoid IPIs completely. 713 */ 714 on_each_cpu_mask(mm_cpumask(mm), do_shoot_lazy_tlb, (void *)mm, 1); 715 if (IS_ENABLED(CONFIG_DEBUG_VM_SHOOT_LAZIES)) 716 on_each_cpu(do_check_lazy_tlb, (void *)mm, 1); 717 } 718 719 /* 720 * Called when the last reference to the mm 721 * is dropped: either by a lazy thread or by 722 * mmput. Free the page directory and the mm. 723 */ 724 void __mmdrop(struct mm_struct *mm) 725 { 726 BUG_ON(mm == &init_mm); 727 WARN_ON_ONCE(mm == current->mm); 728 729 /* Ensure no CPUs are using this as their lazy tlb mm */ 730 cleanup_lazy_tlbs(mm); 731 732 WARN_ON_ONCE(mm == current->active_mm); 733 mm_destroy_sched(mm); 734 mm_free_pgd(mm); 735 mm_free_id(mm); 736 destroy_context(mm); 737 mmu_notifier_subscriptions_destroy(mm); 738 check_mm(mm); 739 mm_pasid_drop(mm); 740 mm_destroy_cid(mm); 741 percpu_counter_destroy_many(mm->rss_stat, NR_MM_COUNTERS); 742 743 free_mm(mm); 744 } 745 EXPORT_SYMBOL_GPL(__mmdrop); 746 747 static void mmdrop_async_fn(struct work_struct *work) 748 { 749 struct mm_struct *mm; 750 751 mm = container_of(work, struct mm_struct, async_put_work); 752 __mmdrop(mm); 753 } 754 755 static void mmdrop_async(struct mm_struct *mm) 756 { 757 if (unlikely(atomic_dec_and_test(&mm->mm_count))) { 758 INIT_WORK(&mm->async_put_work, mmdrop_async_fn); 759 schedule_work(&mm->async_put_work); 760 } 761 } 762 763 static inline void free_signal_struct(struct signal_struct *sig) 764 { 765 taskstats_tgid_free(sig); 766 sched_autogroup_exit(sig); 767 /* 768 * __mmdrop is not safe to call from softirq context on x86 due to 769 * pgd_dtor so postpone it to the async context 770 */ 771 if (sig->oom_mm) 772 mmdrop_async(sig->oom_mm); 773 kmem_cache_free(signal_cachep, sig); 774 } 775 776 static inline void put_signal_struct(struct signal_struct *sig) 777 { 778 if (refcount_dec_and_test(&sig->sigcnt)) 779 free_signal_struct(sig); 780 } 781 782 void __put_task_struct(struct task_struct *tsk) 783 { 784 WARN_ON(!tsk->exit_state); 785 WARN_ON(refcount_read(&tsk->usage)); 786 WARN_ON(tsk == current); 787 788 unwind_task_free(tsk); 789 io_uring_free(tsk); 790 cgroup_task_free(tsk); 791 task_numa_free(tsk, true); 792 security_task_free(tsk); 793 exit_creds(tsk); 794 delayacct_tsk_free(tsk); 795 put_signal_struct(tsk->signal); 796 sched_core_free(tsk); 797 free_task(tsk); 798 } 799 EXPORT_SYMBOL_GPL(__put_task_struct); 800 801 void __put_task_struct_rcu_cb(struct rcu_head *rhp) 802 { 803 struct task_struct *task = container_of(rhp, struct task_struct, rcu); 804 805 __put_task_struct(task); 806 } 807 EXPORT_SYMBOL_GPL(__put_task_struct_rcu_cb); 808 809 void __init __weak arch_task_cache_init(void) { } 810 811 /* 812 * set_max_threads 813 */ 814 static void __init set_max_threads(unsigned int max_threads_suggested) 815 { 816 u64 threads; 817 unsigned long nr_pages = memblock_estimated_nr_free_pages(); 818 819 /* 820 * The number of threads shall be limited such that the thread 821 * structures may only consume a small part of the available memory. 822 */ 823 if (fls64(nr_pages) + fls64(PAGE_SIZE) > 64) 824 threads = MAX_THREADS; 825 else 826 threads = div64_u64((u64) nr_pages * (u64) PAGE_SIZE, 827 (u64) THREAD_SIZE * 8UL); 828 829 if (threads > max_threads_suggested) 830 threads = max_threads_suggested; 831 832 max_threads = clamp_t(u64, threads, MIN_THREADS, MAX_THREADS); 833 } 834 835 #ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT 836 /* Initialized by the architecture: */ 837 int arch_task_struct_size __read_mostly; 838 #endif 839 840 static void __init task_struct_whitelist(unsigned long *offset, unsigned long *size) 841 { 842 /* Fetch thread_struct whitelist for the architecture. */ 843 arch_thread_struct_whitelist(offset, size); 844 845 /* 846 * Handle zero-sized whitelist or empty thread_struct, otherwise 847 * adjust offset to position of thread_struct in task_struct. 848 */ 849 if (unlikely(*size == 0)) 850 *offset = 0; 851 else 852 *offset += offsetof(struct task_struct, thread); 853 } 854 855 void __init fork_init(void) 856 { 857 int i; 858 #ifndef ARCH_MIN_TASKALIGN 859 #define ARCH_MIN_TASKALIGN 0 860 #endif 861 int align = max3(L1_CACHE_BYTES, ARCH_MIN_TASKALIGN, 862 __alignof__(struct task_struct)); 863 unsigned long useroffset, usersize; 864 865 /* create a slab on which task_structs can be allocated */ 866 task_struct_whitelist(&useroffset, &usersize); 867 task_struct_cachep = kmem_cache_create_usercopy("task_struct", 868 arch_task_struct_size, align, 869 SLAB_PANIC|SLAB_ACCOUNT, 870 useroffset, usersize, NULL); 871 872 /* do the arch specific task caches init */ 873 arch_task_cache_init(); 874 875 set_max_threads(MAX_THREADS); 876 877 init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2; 878 init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2; 879 init_task.signal->rlim[RLIMIT_SIGPENDING] = 880 init_task.signal->rlim[RLIMIT_NPROC]; 881 882 for (i = 0; i < UCOUNT_COUNTS; i++) 883 init_user_ns.ucount_max[i] = max_threads/2; 884 885 set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_NPROC, RLIM_INFINITY); 886 set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_MSGQUEUE, RLIM_INFINITY); 887 set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_SIGPENDING, RLIM_INFINITY); 888 set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_MEMLOCK, RLIM_INFINITY); 889 890 #ifdef CONFIG_VMAP_STACK 891 cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "fork:vm_stack_cache", 892 NULL, free_vm_stack_cache); 893 #endif 894 895 scs_init(); 896 897 lockdep_init_task(&init_task); 898 uprobes_init(); 899 } 900 901 int __weak arch_dup_task_struct(struct task_struct *dst, 902 struct task_struct *src) 903 { 904 *dst = *src; 905 return 0; 906 } 907 908 void set_task_stack_end_magic(struct task_struct *tsk) 909 { 910 unsigned long *stackend; 911 912 stackend = end_of_stack(tsk); 913 *stackend = STACK_END_MAGIC; /* for overflow detection */ 914 } 915 916 static struct task_struct *dup_task_struct(struct task_struct *orig, int node) 917 { 918 struct task_struct *tsk; 919 int err; 920 921 if (node == NUMA_NO_NODE) 922 node = tsk_fork_get_node(orig); 923 tsk = alloc_task_struct_node(node); 924 if (!tsk) 925 return NULL; 926 927 err = arch_dup_task_struct(tsk, orig); 928 if (err) 929 goto free_tsk; 930 931 err = alloc_thread_stack_node(tsk, node); 932 if (err) 933 goto free_tsk; 934 935 #ifdef CONFIG_THREAD_INFO_IN_TASK 936 refcount_set(&tsk->stack_refcount, 1); 937 #endif 938 account_kernel_stack(tsk, 1); 939 940 err = smp_task_ipi_mask_alloc(tsk); 941 if (err) 942 goto free_stack; 943 944 err = scs_prepare(tsk, node); 945 if (err) 946 goto free_ipi_mask; 947 948 #ifdef CONFIG_SECCOMP 949 /* 950 * We must handle setting up seccomp filters once we're under 951 * the sighand lock in case orig has changed between now and 952 * then. Until then, filter must be NULL to avoid messing up 953 * the usage counts on the error path calling free_task. 954 */ 955 tsk->seccomp.filter = NULL; 956 #endif 957 958 RCU_INIT_POINTER(tsk->exec_state, NULL); 959 960 setup_thread_stack(tsk, orig); 961 clear_user_return_notifier(tsk); 962 clear_tsk_need_resched(tsk); 963 set_task_stack_end_magic(tsk); 964 clear_syscall_work_syscall_user_dispatch(tsk); 965 966 #ifdef CONFIG_STACKPROTECTOR 967 tsk->stack_canary = get_random_canary(); 968 #endif 969 if (orig->cpus_ptr == &orig->cpus_mask) 970 tsk->cpus_ptr = &tsk->cpus_mask; 971 dup_user_cpus_ptr(tsk, orig, node); 972 973 /* 974 * Nobody can keep the task while copy_process() may still fail, which 975 * frees it no matter what. 976 */ 977 refcount_set(&tsk->rcu_users, 0); 978 /* One for the rcu users */ 979 refcount_set(&tsk->usage, 1); 980 #ifdef CONFIG_BLK_DEV_IO_TRACE 981 tsk->btrace_seq = 0; 982 #endif 983 tsk->splice_pipe = NULL; 984 tsk->task_frag.page = NULL; 985 tsk->wake_q.next = NULL; 986 tsk->worker_private = NULL; 987 988 kcov_task_init(tsk); 989 kmsan_task_create(tsk); 990 kmap_local_fork(tsk); 991 992 #ifdef CONFIG_FAULT_INJECTION 993 tsk->fail_nth = 0; 994 #endif 995 996 #ifdef CONFIG_BLK_CGROUP 997 tsk->throttle_disk = NULL; 998 tsk->use_memdelay = 0; 999 #endif 1000 1001 #ifdef CONFIG_ARCH_HAS_CPU_PASID 1002 tsk->pasid_activated = 0; 1003 #endif 1004 1005 #ifdef CONFIG_MEMCG 1006 tsk->active_memcg = NULL; 1007 #endif 1008 1009 #ifdef CONFIG_X86_BUS_LOCK_DETECT 1010 tsk->reported_split_lock = 0; 1011 #endif 1012 1013 #ifdef CONFIG_SCHED_MM_CID 1014 tsk->mm_cid.cid = MM_CID_UNSET; 1015 tsk->mm_cid.active = 0; 1016 INIT_HLIST_NODE(&tsk->mm_cid.node); 1017 #endif 1018 1019 #ifdef CONFIG_BPF_SYSCALL 1020 RCU_INIT_POINTER(tsk->bpf_storage, NULL); 1021 tsk->bpf_ctx = NULL; 1022 #endif 1023 return tsk; 1024 1025 free_ipi_mask: 1026 smp_task_ipi_mask_free(tsk); 1027 free_stack: 1028 exit_task_stack_account(tsk); 1029 free_thread_stack(tsk); 1030 free_tsk: 1031 free_task_struct(tsk); 1032 return NULL; 1033 } 1034 1035 __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock); 1036 1037 static unsigned long coredump_filter = MMF_DUMP_FILTER_DEFAULT; 1038 1039 static int __init coredump_filter_setup(char *s) 1040 { 1041 if (kstrtoul(s, 0, &coredump_filter)) 1042 return 0; 1043 coredump_filter <<= MMF_DUMP_FILTER_SHIFT; 1044 coredump_filter &= MMF_DUMP_FILTER_MASK; 1045 return 1; 1046 } 1047 1048 __setup("coredump_filter=", coredump_filter_setup); 1049 1050 #include <linux/init_task.h> 1051 1052 static void mm_init_aio(struct mm_struct *mm) 1053 { 1054 #ifdef CONFIG_AIO 1055 spin_lock_init(&mm->ioctx_lock); 1056 mm->ioctx_table = NULL; 1057 #endif 1058 } 1059 1060 static __always_inline void mm_clear_owner(struct mm_struct *mm, 1061 struct task_struct *p) 1062 { 1063 #ifdef CONFIG_MEMCG 1064 if (mm->owner == p) 1065 WRITE_ONCE(mm->owner, NULL); 1066 #endif 1067 } 1068 1069 static void mm_init_owner(struct mm_struct *mm, struct task_struct *p) 1070 { 1071 #ifdef CONFIG_MEMCG 1072 mm->owner = p; 1073 #endif 1074 } 1075 1076 static void mm_init_uprobes_state(struct mm_struct *mm) 1077 { 1078 #ifdef CONFIG_UPROBES 1079 mm->uprobes_state.xol_area = NULL; 1080 #endif 1081 } 1082 1083 static void mmap_init_lock(struct mm_struct *mm) 1084 { 1085 init_rwsem(&mm->mmap_lock); 1086 mm_lock_seqcount_init(mm); 1087 rcuwait_init(&mm->vma_writer_wait); 1088 } 1089 1090 static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p) 1091 { 1092 mt_init_flags(&mm->mm_mt, MM_MT_FLAGS); 1093 mt_set_external_lock(&mm->mm_mt, &mm->mmap_lock); 1094 atomic_set(&mm->mm_users, 1); 1095 atomic_set(&mm->mm_count, 1); 1096 seqcount_init(&mm->write_protect_seq); 1097 mmap_init_lock(mm); 1098 INIT_LIST_HEAD(&mm->mmlist); 1099 mm_pgtables_bytes_init(mm); 1100 mm->map_count = 0; 1101 mm->locked_vm = 0; 1102 atomic64_set(&mm->pinned_vm, 0); 1103 memset(&mm->rss_stat, 0, sizeof(mm->rss_stat)); 1104 spin_lock_init(&mm->page_table_lock); 1105 spin_lock_init(&mm->arg_lock); 1106 mm_init_cpumask(mm); 1107 mm_init_aio(mm); 1108 mm_init_owner(mm, p); 1109 mm_pasid_init(mm); 1110 RCU_INIT_POINTER(mm->exe_file, NULL); 1111 mmu_notifier_subscriptions_init(mm); 1112 init_tlb_flush_pending(mm); 1113 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !defined(CONFIG_SPLIT_PMD_PTLOCKS) 1114 mm->pmd_huge_pte = NULL; 1115 #endif 1116 mm_init_uprobes_state(mm); 1117 hugetlb_count_init(mm); 1118 futex_mm_init(mm); 1119 1120 mm_flags_clear_all(mm); 1121 if (current->mm) { 1122 unsigned long flags = __mm_flags_get_word(current->mm); 1123 1124 __mm_flags_overwrite_word(mm, mmf_init_legacy_flags(flags)); 1125 mm->def_flags = current->mm->def_flags & VM_INIT_DEF_MASK; 1126 } else { 1127 __mm_flags_overwrite_word(mm, coredump_filter); 1128 mm->def_flags = 0; 1129 } 1130 1131 if (mm_alloc_pgd(mm)) 1132 goto fail_mm_init; 1133 1134 if (mm_alloc_id(mm)) 1135 goto fail_noid; 1136 1137 if (init_new_context(p, mm)) 1138 goto fail_nocontext; 1139 1140 if (mm_alloc_cid(mm, p)) 1141 goto fail_cid; 1142 1143 if (mm_alloc_sched(mm)) 1144 goto fail_sched; 1145 1146 if (percpu_counter_init_many(mm->rss_stat, 0, GFP_KERNEL_ACCOUNT, 1147 NR_MM_COUNTERS)) 1148 goto fail_pcpu; 1149 1150 lru_gen_init_mm(mm); 1151 return mm; 1152 1153 fail_pcpu: 1154 mm_destroy_sched(mm); 1155 fail_sched: 1156 mm_destroy_cid(mm); 1157 fail_cid: 1158 destroy_context(mm); 1159 fail_nocontext: 1160 mm_free_id(mm); 1161 fail_noid: 1162 mm_free_pgd(mm); 1163 fail_mm_init: 1164 free_mm(mm); 1165 return NULL; 1166 } 1167 1168 /* 1169 * Allocate and initialize an mm_struct. 1170 */ 1171 struct mm_struct *mm_alloc(void) 1172 { 1173 struct mm_struct *mm; 1174 1175 mm = allocate_mm(); 1176 if (!mm) 1177 return NULL; 1178 1179 memset(mm, 0, sizeof(*mm)); 1180 return mm_init(mm, current); 1181 } 1182 EXPORT_SYMBOL_IF_KUNIT(mm_alloc); 1183 1184 static inline void __mmput(struct mm_struct *mm) 1185 { 1186 VM_BUG_ON(atomic_read(&mm->mm_users)); 1187 1188 uprobe_clear_state(mm); 1189 exit_aio(mm); 1190 ksm_exit(mm); 1191 khugepaged_exit(mm); /* must run before exit_mmap */ 1192 exit_mmap(mm); 1193 mm_put_huge_zero_folio(mm); 1194 set_mm_exe_file(mm, NULL); 1195 if (!list_empty(&mm->mmlist)) { 1196 spin_lock(&mmlist_lock); 1197 list_del(&mm->mmlist); 1198 spin_unlock(&mmlist_lock); 1199 } 1200 if (mm->binfmt) 1201 module_put(mm->binfmt->module); 1202 lru_gen_del_mm(mm); 1203 futex_hash_free(mm); 1204 mmdrop(mm); 1205 } 1206 1207 /* 1208 * Decrement the use count and release all resources for an mm. 1209 */ 1210 void mmput(struct mm_struct *mm) 1211 { 1212 might_sleep(); 1213 1214 if (atomic_dec_and_test(&mm->mm_users)) 1215 __mmput(mm); 1216 } 1217 EXPORT_SYMBOL_GPL(mmput); 1218 1219 #if defined(CONFIG_MMU) || defined(CONFIG_FUTEX_PRIVATE_HASH) 1220 static void mmput_async_fn(struct work_struct *work) 1221 { 1222 struct mm_struct *mm = container_of(work, struct mm_struct, 1223 async_put_work); 1224 1225 __mmput(mm); 1226 } 1227 1228 void mmput_async(struct mm_struct *mm) 1229 { 1230 if (atomic_dec_and_test(&mm->mm_users)) { 1231 INIT_WORK(&mm->async_put_work, mmput_async_fn); 1232 schedule_work(&mm->async_put_work); 1233 } 1234 } 1235 EXPORT_SYMBOL_GPL(mmput_async); 1236 #endif 1237 1238 /** 1239 * set_mm_exe_file - change a reference to the mm's executable file 1240 * @mm: The mm to change. 1241 * @new_exe_file: The new file to use. 1242 * 1243 * This changes mm's executable file (shown as symlink /proc/[pid]/exe). 1244 * 1245 * Main users are mmput() and sys_execve(). Callers prevent concurrent 1246 * invocations: in mmput() nobody alive left, in execve it happens before 1247 * the new mm is made visible to anyone. 1248 * 1249 * Can only fail if new_exe_file != NULL. 1250 */ 1251 int set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file) 1252 { 1253 struct file *old_exe_file; 1254 1255 /* 1256 * It is safe to dereference the exe_file without RCU as 1257 * this function is only called if nobody else can access 1258 * this mm -- see comment above for justification. 1259 */ 1260 old_exe_file = rcu_dereference_raw(mm->exe_file); 1261 1262 if (new_exe_file) { 1263 /* 1264 * We expect the caller (i.e., sys_execve) to already denied 1265 * write access, so this is unlikely to fail. 1266 */ 1267 if (unlikely(exe_file_deny_write_access(new_exe_file))) 1268 return -EACCES; 1269 get_file(new_exe_file); 1270 } 1271 rcu_assign_pointer(mm->exe_file, new_exe_file); 1272 if (old_exe_file) { 1273 exe_file_allow_write_access(old_exe_file); 1274 fput(old_exe_file); 1275 } 1276 return 0; 1277 } 1278 1279 /** 1280 * replace_mm_exe_file - replace a reference to the mm's executable file 1281 * @mm: The mm to change. 1282 * @new_exe_file: The new file to use. 1283 * 1284 * This changes mm's executable file (shown as symlink /proc/[pid]/exe). 1285 * 1286 * Main user is sys_prctl(PR_SET_MM_MAP/EXE_FILE). 1287 */ 1288 int replace_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file) 1289 { 1290 struct vm_area_struct *vma; 1291 struct file *old_exe_file; 1292 int ret = 0; 1293 1294 /* Forbid mm->exe_file change if old file still mapped. */ 1295 old_exe_file = get_mm_exe_file(mm); 1296 if (old_exe_file) { 1297 VMA_ITERATOR(vmi, mm, 0); 1298 mmap_read_lock(mm); 1299 for_each_vma(vmi, vma) { 1300 if (!vma->vm_file) 1301 continue; 1302 if (path_equal(&vma->vm_file->f_path, 1303 &old_exe_file->f_path)) { 1304 ret = -EBUSY; 1305 break; 1306 } 1307 } 1308 mmap_read_unlock(mm); 1309 fput(old_exe_file); 1310 if (ret) 1311 return ret; 1312 } 1313 1314 ret = exe_file_deny_write_access(new_exe_file); 1315 if (ret) 1316 return -EACCES; 1317 get_file(new_exe_file); 1318 1319 /* set the new file */ 1320 mmap_write_lock(mm); 1321 old_exe_file = rcu_dereference_raw(mm->exe_file); 1322 rcu_assign_pointer(mm->exe_file, new_exe_file); 1323 mmap_write_unlock(mm); 1324 1325 if (old_exe_file) { 1326 exe_file_allow_write_access(old_exe_file); 1327 fput(old_exe_file); 1328 } 1329 return 0; 1330 } 1331 1332 /** 1333 * get_mm_exe_file - acquire a reference to the mm's executable file 1334 * @mm: The mm of interest. 1335 * 1336 * Returns %NULL if mm has no associated executable file. 1337 * User must release file via fput(). 1338 */ 1339 struct file *get_mm_exe_file(struct mm_struct *mm) 1340 { 1341 struct file *exe_file; 1342 1343 rcu_read_lock(); 1344 exe_file = get_file_rcu(&mm->exe_file); 1345 rcu_read_unlock(); 1346 return exe_file; 1347 } 1348 1349 /** 1350 * get_task_exe_file - acquire a reference to the task's executable file 1351 * @task: The task. 1352 * 1353 * Returns %NULL if task's mm (if any) has no associated executable file or 1354 * this is a kernel thread with borrowed mm (see the comment above get_task_mm). 1355 * User must release file via fput(). 1356 */ 1357 struct file *get_task_exe_file(struct task_struct *task) 1358 { 1359 struct file *exe_file = NULL; 1360 struct mm_struct *mm; 1361 1362 if (task->flags & PF_KTHREAD) 1363 return NULL; 1364 1365 task_lock(task); 1366 mm = task->mm; 1367 if (mm) 1368 exe_file = get_mm_exe_file(mm); 1369 task_unlock(task); 1370 return exe_file; 1371 } 1372 1373 /** 1374 * get_task_mm - acquire a reference to the task's mm 1375 * @task: The task. 1376 * 1377 * Returns %NULL if the task has no mm. Checks PF_KTHREAD (meaning 1378 * this kernel workthread has transiently adopted a user mm with kthread_use_mm, 1379 * to do its AIO) is not set and if so returns a reference to it, after 1380 * bumping up the use count. User must release the mm via mmput() 1381 * after use. Typically used by /proc and ptrace. 1382 */ 1383 struct mm_struct *get_task_mm(struct task_struct *task) 1384 { 1385 struct mm_struct *mm; 1386 1387 if (task->flags & PF_KTHREAD) 1388 return NULL; 1389 1390 task_lock(task); 1391 mm = task->mm; 1392 if (mm) 1393 mmget(mm); 1394 task_unlock(task); 1395 return mm; 1396 } 1397 EXPORT_SYMBOL_GPL(get_task_mm); 1398 1399 static bool may_access_mm(struct mm_struct *mm, struct task_struct *task, unsigned int mode) 1400 { 1401 if (mm == current->mm) 1402 return true; 1403 if (ptrace_may_access(task, mode)) 1404 return true; 1405 if ((mode & PTRACE_MODE_READ) && perfmon_capable()) 1406 return true; 1407 return false; 1408 } 1409 1410 struct mm_struct *mm_access(struct task_struct *task, unsigned int mode) 1411 { 1412 struct mm_struct *mm; 1413 int err; 1414 1415 err = down_read_killable(&task->signal->exec_update_lock); 1416 if (err) 1417 return ERR_PTR(err); 1418 1419 mm = get_task_mm(task); 1420 if (!mm) { 1421 mm = ERR_PTR(-ESRCH); 1422 } else if (!may_access_mm(mm, task, mode)) { 1423 mmput(mm); 1424 mm = ERR_PTR(-EACCES); 1425 } 1426 up_read(&task->signal->exec_update_lock); 1427 1428 return mm; 1429 } 1430 1431 static void complete_vfork_done(struct task_struct *tsk) 1432 { 1433 struct completion *vfork; 1434 1435 task_lock(tsk); 1436 vfork = tsk->vfork_done; 1437 if (likely(vfork)) { 1438 tsk->vfork_done = NULL; 1439 complete(vfork); 1440 } 1441 task_unlock(tsk); 1442 } 1443 1444 static int wait_for_vfork_done(struct task_struct *child, 1445 struct completion *vfork) 1446 { 1447 unsigned int state = TASK_KILLABLE|TASK_FREEZABLE; 1448 int killed; 1449 1450 cgroup_enter_frozen(); 1451 killed = wait_for_completion_state(vfork, state); 1452 cgroup_leave_frozen(false); 1453 1454 if (killed) { 1455 task_lock(child); 1456 child->vfork_done = NULL; 1457 task_unlock(child); 1458 } 1459 1460 put_task_struct(child); 1461 return killed; 1462 } 1463 1464 /* Please note the differences between mmput and mm_release. 1465 * mmput is called whenever we stop holding onto a mm_struct, 1466 * error success whatever. 1467 * 1468 * mm_release is called after a mm_struct has been removed 1469 * from the current process. 1470 * 1471 * This difference is important for error handling, when we 1472 * only half set up a mm_struct for a new process and need to restore 1473 * the old one. Because we mmput the new mm_struct before 1474 * restoring the old one. . . 1475 * Eric Biederman 10 January 1998 1476 */ 1477 static void mm_release(struct task_struct *tsk, struct mm_struct *mm) 1478 { 1479 uprobe_free_utask(tsk); 1480 1481 /* Get rid of any cached register state */ 1482 deactivate_mm(tsk, mm); 1483 1484 /* 1485 * Signal userspace if we're not exiting with a core dump 1486 * because we want to leave the value intact for debugging 1487 * purposes. 1488 */ 1489 if (tsk->clear_child_tid) { 1490 if (atomic_read(&mm->mm_users) > 1) { 1491 /* 1492 * We don't check the error code - if userspace has 1493 * not set up a proper pointer then tough luck. 1494 */ 1495 put_user(0, tsk->clear_child_tid); 1496 do_futex(tsk->clear_child_tid, FUTEX_WAKE, 1497 1, NULL, NULL, 0, 0); 1498 } 1499 tsk->clear_child_tid = NULL; 1500 } 1501 1502 /* 1503 * All done, finally we can wake up parent and return this mm to him. 1504 * Also kthread_stop() uses this completion for synchronization. 1505 */ 1506 if (tsk->vfork_done) 1507 complete_vfork_done(tsk); 1508 } 1509 1510 void mm_exit_exec_release(struct task_struct *tsk, struct mm_struct *mm) 1511 { 1512 futex_exit_exec_release(tsk); 1513 mm_release(tsk, mm); 1514 } 1515 1516 /** 1517 * dup_mm() - duplicates an existing mm structure 1518 * @tsk: the task_struct with which the new mm will be associated. 1519 * @oldmm: the mm to duplicate. 1520 * 1521 * Allocates a new mm structure and duplicates the provided @oldmm structure 1522 * content into it. 1523 * 1524 * Return: the duplicated mm or NULL on failure. 1525 */ 1526 static struct mm_struct *dup_mm(struct task_struct *tsk, 1527 struct mm_struct *oldmm) 1528 { 1529 struct mm_struct *mm; 1530 int err; 1531 1532 mm = allocate_mm(); 1533 if (!mm) 1534 goto fail_nomem; 1535 1536 uprobe_start_dup_mmap(); 1537 if (mmap_write_lock_killable(oldmm)) { 1538 free_mm(mm); 1539 goto fail_uprobe; 1540 } 1541 1542 memcpy(mm, oldmm, sizeof(*mm)); 1543 1544 if (!mm_init(mm, tsk)) { 1545 mmap_write_unlock(oldmm); 1546 goto fail_uprobe; 1547 } 1548 1549 err = dup_mmap(mm, oldmm); 1550 if (err) 1551 goto free_pt; 1552 uprobe_end_dup_mmap(); 1553 1554 mm->hiwater_rss = get_mm_rss(mm); 1555 mm->hiwater_vm = mm->total_vm; 1556 1557 if (mm->binfmt && !try_module_get(mm->binfmt->module)) 1558 goto free_pt; 1559 1560 return mm; 1561 1562 free_pt: 1563 /* don't put binfmt in mmput, we haven't got module yet */ 1564 mm->binfmt = NULL; 1565 mm_init_owner(mm, NULL); 1566 mmput(mm); 1567 if (!err) 1568 return NULL; 1569 fail_uprobe: 1570 uprobe_end_dup_mmap(); 1571 fail_nomem: 1572 return NULL; 1573 } 1574 1575 static int copy_mm(u64 clone_flags, struct task_struct *tsk) 1576 { 1577 struct mm_struct *mm, *oldmm; 1578 1579 tsk->min_flt = tsk->maj_flt = 0; 1580 tsk->nvcsw = tsk->nivcsw = 0; 1581 #ifdef CONFIG_DETECT_HUNG_TASK 1582 tsk->last_switch_count = tsk->nvcsw + tsk->nivcsw; 1583 tsk->last_switch_time = 0; 1584 #endif 1585 1586 tsk->mm = NULL; 1587 tsk->active_mm = NULL; 1588 1589 /* 1590 * Are we cloning a kernel thread? 1591 * 1592 * We need to steal a active VM for that.. 1593 */ 1594 oldmm = current->mm; 1595 if (!oldmm) 1596 return 0; 1597 1598 if (clone_flags & CLONE_VM) { 1599 mmget(oldmm); 1600 mm = oldmm; 1601 } else { 1602 mm = dup_mm(tsk, current->mm); 1603 if (!mm) 1604 return -ENOMEM; 1605 } 1606 1607 tsk->mm = mm; 1608 tsk->active_mm = mm; 1609 sched_cache_fork(tsk); 1610 return 0; 1611 } 1612 1613 static int copy_exec_state(u64 clone_flags, struct task_struct *tsk) 1614 { 1615 struct task_exec_state *exec_state; 1616 1617 /* CLONE_VM siblings refcount-share the parent's exec_state. */ 1618 if (clone_flags & CLONE_VM) { 1619 exec_state = rcu_dereference_protected(current->exec_state, true); 1620 refcount_inc(&exec_state->count); 1621 rcu_assign_pointer(tsk->exec_state, exec_state); 1622 return 0; 1623 } 1624 1625 /* Everyone else inherits a fresh copy. */ 1626 return task_exec_state_copy(tsk); 1627 } 1628 1629 enum copy_fs_flags_t { 1630 COPY_FS_UMH = (1 << 0), 1631 COPY_FS_IDLE = (1 << 1), 1632 }; 1633 1634 static int copy_fs(u64 clone_flags, struct task_struct *tsk, 1635 enum copy_fs_flags_t flags) 1636 { 1637 struct fs_struct *fs; 1638 1639 /* 1640 * Usermodehelper may copy userspace_init_fs filesystem state but 1641 * they don't get to create mount namespaces, share the 1642 * filesystem state, or be started from a non-initial mount 1643 * namespace. 1644 */ 1645 if (flags & COPY_FS_UMH) { 1646 if (clone_flags & (CLONE_NEWNS | CLONE_FS)) 1647 return -EINVAL; 1648 if (current->nsproxy->mnt_ns != &init_mnt_ns) 1649 return -EINVAL; 1650 fs = userspace_init_fs; 1651 } else if (flags & COPY_FS_IDLE) { 1652 fs = &init_fs; 1653 } else { 1654 fs = current->fs; 1655 VFS_WARN_ON_ONCE(current->fs != current->real_fs); 1656 } 1657 1658 if (clone_flags & CLONE_FS) { 1659 read_seqlock_excl(&fs->seq); 1660 /* "users" and "in_exec" locked for check_unsafe_exec() */ 1661 if (fs->in_exec) { 1662 read_sequnlock_excl(&fs->seq); 1663 return -EAGAIN; 1664 } 1665 fs->users++; 1666 read_sequnlock_excl(&fs->seq); 1667 tsk->real_fs = tsk->fs = fs; 1668 return 0; 1669 } 1670 tsk->real_fs = tsk->fs = copy_fs_struct(fs); 1671 if (!tsk->fs) 1672 return -ENOMEM; 1673 return 0; 1674 } 1675 1676 static int copy_files(u64 clone_flags, struct task_struct *tsk, 1677 int no_files) 1678 { 1679 struct files_struct *oldf, *newf; 1680 1681 /* 1682 * A background process may not have any files ... 1683 */ 1684 oldf = current->files; 1685 if (!oldf) 1686 return 0; 1687 1688 if (no_files) { 1689 tsk->files = NULL; 1690 return 0; 1691 } 1692 1693 if (clone_flags & CLONE_FILES) { 1694 atomic_inc(&oldf->count); 1695 tsk->files = oldf; 1696 return 0; 1697 } 1698 1699 newf = dup_fd(oldf, NULL); 1700 if (IS_ERR(newf)) 1701 return PTR_ERR(newf); 1702 1703 tsk->files = newf; 1704 return 0; 1705 } 1706 1707 static int copy_sighand(u64 clone_flags, struct task_struct *tsk) 1708 { 1709 struct sighand_struct *sig; 1710 1711 if (clone_flags & CLONE_SIGHAND) { 1712 refcount_inc(¤t->sighand->count); 1713 return 0; 1714 } 1715 sig = kmem_cache_alloc(sighand_cachep, GFP_KERNEL); 1716 RCU_INIT_POINTER(tsk->sighand, sig); 1717 if (!sig) 1718 return -ENOMEM; 1719 1720 refcount_set(&sig->count, 1); 1721 spin_lock_irq(¤t->sighand->siglock); 1722 memcpy(sig->action, current->sighand->action, sizeof(sig->action)); 1723 spin_unlock_irq(¤t->sighand->siglock); 1724 1725 /* Reset all signal handler not set to SIG_IGN to SIG_DFL. */ 1726 if (clone_flags & CLONE_CLEAR_SIGHAND) 1727 flush_signal_handlers(tsk, 0); 1728 1729 return 0; 1730 } 1731 1732 void __cleanup_sighand(struct sighand_struct *sighand) 1733 { 1734 if (refcount_dec_and_test(&sighand->count)) { 1735 signalfd_cleanup(sighand); 1736 /* 1737 * sighand_cachep is SLAB_TYPESAFE_BY_RCU so we can free it 1738 * without an RCU grace period, see __lock_task_sighand(). 1739 */ 1740 kmem_cache_free(sighand_cachep, sighand); 1741 } 1742 } 1743 1744 /* 1745 * Initialize POSIX timer handling for a thread group. 1746 */ 1747 static void posix_cpu_timers_init_group(struct signal_struct *sig) 1748 { 1749 struct posix_cputimers *pct = &sig->posix_cputimers; 1750 unsigned long cpu_limit; 1751 1752 cpu_limit = READ_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur); 1753 posix_cputimers_group_init(pct, cpu_limit); 1754 } 1755 1756 static int copy_signal(u64 clone_flags, struct task_struct *tsk) 1757 { 1758 struct signal_struct *sig; 1759 1760 if (clone_flags & CLONE_THREAD) 1761 return 0; 1762 1763 sig = kmem_cache_zalloc(signal_cachep, GFP_KERNEL); 1764 tsk->signal = sig; 1765 if (!sig) 1766 return -ENOMEM; 1767 1768 sig->nr_threads = 1; 1769 sig->quick_threads = 1; 1770 atomic_set(&sig->live, 1); 1771 refcount_set(&sig->sigcnt, 1); 1772 1773 /* list_add(thread_node, thread_head) without INIT_LIST_HEAD() */ 1774 sig->thread_head = (struct list_head)LIST_HEAD_INIT(tsk->thread_node); 1775 tsk->thread_node = (struct list_head)LIST_HEAD_INIT(sig->thread_head); 1776 1777 init_waitqueue_head(&sig->wait_chldexit); 1778 sig->curr_target = tsk; 1779 init_sigpending(&sig->shared_pending); 1780 INIT_HLIST_HEAD(&sig->multiprocess); 1781 seqlock_init(&sig->stats_lock); 1782 prev_cputime_init(&sig->prev_cputime); 1783 1784 #ifdef CONFIG_POSIX_TIMERS 1785 INIT_HLIST_HEAD(&sig->posix_timers); 1786 INIT_HLIST_HEAD(&sig->ignored_posix_timers); 1787 hrtimer_setup(&sig->real_timer, it_real_fn, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 1788 #endif 1789 1790 task_lock(current->group_leader); 1791 memcpy(sig->rlim, current->signal->rlim, sizeof sig->rlim); 1792 task_unlock(current->group_leader); 1793 1794 posix_cpu_timers_init_group(sig); 1795 1796 tty_audit_fork(sig); 1797 sched_autogroup_fork(sig); 1798 1799 #ifdef CONFIG_CGROUPS 1800 init_rwsem(&sig->cgroup_threadgroup_rwsem); 1801 #endif 1802 1803 sig->oom_score_adj = current->signal->oom_score_adj; 1804 sig->oom_score_adj_min = current->signal->oom_score_adj_min; 1805 1806 mutex_init(&sig->cred_guard_mutex); 1807 init_rwsem(&sig->exec_update_lock); 1808 1809 return 0; 1810 } 1811 1812 static void copy_seccomp(struct task_struct *p) 1813 { 1814 #ifdef CONFIG_SECCOMP 1815 /* 1816 * Must be called with sighand->lock held, which is common to 1817 * all threads in the group. Holding cred_guard_mutex is not 1818 * needed because this new task is not yet running and cannot 1819 * be racing exec. 1820 */ 1821 assert_spin_locked(¤t->sighand->siglock); 1822 1823 /* Ref-count the new filter user, and assign it. */ 1824 get_seccomp_filter(current); 1825 p->seccomp = current->seccomp; 1826 1827 /* 1828 * Explicitly enable no_new_privs here in case it got set 1829 * between the task_struct being duplicated and holding the 1830 * sighand lock. The seccomp state and nnp must be in sync. 1831 */ 1832 if (task_no_new_privs(current)) 1833 task_set_no_new_privs(p); 1834 1835 /* 1836 * If the parent gained a seccomp mode after copying thread 1837 * flags and between before we held the sighand lock, we have 1838 * to manually enable the seccomp thread flag here. 1839 */ 1840 if (p->seccomp.mode != SECCOMP_MODE_DISABLED) 1841 set_task_syscall_work(p, SECCOMP); 1842 #endif 1843 } 1844 1845 SYSCALL_DEFINE1(set_tid_address, int __user *, tidptr) 1846 { 1847 current->clear_child_tid = tidptr; 1848 1849 return task_pid_vnr(current); 1850 } 1851 1852 static void rt_mutex_init_task(struct task_struct *p) 1853 { 1854 raw_spin_lock_init(&p->pi_lock); 1855 #ifdef CONFIG_RT_MUTEXES 1856 p->pi_waiters = RB_ROOT_CACHED; 1857 p->pi_top_task = NULL; 1858 p->pi_blocked_on = NULL; 1859 #endif 1860 } 1861 1862 static inline void init_task_pid_links(struct task_struct *task) 1863 { 1864 enum pid_type type; 1865 1866 for (type = PIDTYPE_PID; type < PIDTYPE_MAX; ++type) 1867 INIT_HLIST_NODE(&task->pid_links[type]); 1868 } 1869 1870 static inline void 1871 init_task_pid(struct task_struct *task, enum pid_type type, struct pid *pid) 1872 { 1873 if (type == PIDTYPE_PID) 1874 task->thread_pid = pid; 1875 else 1876 task->signal->pids[type] = pid; 1877 } 1878 1879 static inline void rcu_copy_process(struct task_struct *p) 1880 { 1881 #ifdef CONFIG_PREEMPT_RCU 1882 p->rcu_read_lock_nesting = 0; 1883 p->rcu_read_unlock_special.s = 0; 1884 p->rcu_blocked_node = NULL; 1885 INIT_LIST_HEAD(&p->rcu_node_entry); 1886 #endif /* #ifdef CONFIG_PREEMPT_RCU */ 1887 #ifdef CONFIG_TASKS_RCU 1888 p->rcu_tasks_holdout = false; 1889 INIT_LIST_HEAD(&p->rcu_tasks_holdout_list); 1890 p->rcu_tasks_idle_cpu = -1; 1891 INIT_LIST_HEAD(&p->rcu_tasks_exit_list); 1892 #endif /* #ifdef CONFIG_TASKS_RCU */ 1893 #ifdef CONFIG_TASKS_TRACE_RCU 1894 p->trc_reader_nesting = 0; 1895 #endif /* #ifdef CONFIG_TASKS_TRACE_RCU */ 1896 } 1897 1898 /** 1899 * pidfd_prepare - allocate a new pidfd_file and reserve a pidfd 1900 * @pid: the struct pid for which to create a pidfd 1901 * @flags: flags of the new @pidfd 1902 * @ret_file: return the new pidfs file 1903 * 1904 * Allocate a new file that stashes @pid and reserve a new pidfd number in the 1905 * caller's file descriptor table. The pidfd is reserved but not installed yet. 1906 * 1907 * The helper verifies that @pid is still in use, without PIDFD_THREAD the 1908 * task identified by @pid must be a thread-group leader. 1909 * 1910 * If this function returns successfully the caller is responsible to either 1911 * call fd_install() passing the returned pidfd and pidfd file as arguments in 1912 * order to install the pidfd into its file descriptor table or they must use 1913 * put_unused_fd() and fput() on the returned pidfd and pidfd file 1914 * respectively. 1915 * 1916 * This function is useful when a pidfd must already be reserved but there 1917 * might still be points of failure afterwards and the caller wants to ensure 1918 * that no pidfd is leaked into its file descriptor table. 1919 * 1920 * Return: On success, a reserved pidfd is returned from the function and a new 1921 * pidfd file is returned in the last argument to the function. On 1922 * error, a negative error code is returned from the function and the 1923 * last argument remains unchanged. 1924 */ 1925 int pidfd_prepare(struct pid *pid, unsigned int flags, struct file **ret_file) 1926 { 1927 struct file *pidfs_file; 1928 1929 /* 1930 * PIDFD_STALE is only allowed to be passed if the caller knows 1931 * that @pid is already registered in pidfs and thus 1932 * PIDFD_INFO_EXIT information is guaranteed to be available. 1933 */ 1934 if (!(flags & PIDFD_STALE)) { 1935 /* 1936 * While holding the pidfd waitqueue lock removing the 1937 * task linkage for the thread-group leader pid 1938 * (PIDTYPE_TGID) isn't possible. Thus, if there's still 1939 * task linkage for PIDTYPE_PID not having thread-group 1940 * leader linkage for the pid means it wasn't a 1941 * thread-group leader in the first place. 1942 */ 1943 guard(spinlock_irq)(&pid->wait_pidfd.lock); 1944 1945 /* Task has already been reaped. */ 1946 if (!pid_has_task(pid, PIDTYPE_PID)) 1947 return -ESRCH; 1948 /* 1949 * If this struct pid isn't used as a thread-group 1950 * leader but the caller requested to create a 1951 * thread-group leader pidfd then report ENOENT. 1952 */ 1953 if (!(flags & PIDFD_THREAD) && !pid_has_task(pid, PIDTYPE_TGID)) 1954 return -ENOENT; 1955 } 1956 1957 CLASS(get_unused_fd, pidfd)(O_CLOEXEC); 1958 if (pidfd < 0) 1959 return pidfd; 1960 1961 pidfs_file = pidfs_alloc_file(pid, flags | O_RDWR); 1962 if (IS_ERR(pidfs_file)) 1963 return PTR_ERR(pidfs_file); 1964 1965 *ret_file = pidfs_file; 1966 return take_fd(pidfd); 1967 } 1968 1969 static void __delayed_free_task(struct rcu_head *rhp) 1970 { 1971 struct task_struct *tsk = container_of(rhp, struct task_struct, rcu); 1972 1973 free_task(tsk); 1974 } 1975 1976 static __always_inline void delayed_free_task(struct task_struct *tsk) 1977 { 1978 if (IS_ENABLED(CONFIG_MEMCG)) 1979 call_rcu(&tsk->rcu, __delayed_free_task); 1980 else 1981 free_task(tsk); 1982 } 1983 1984 static void copy_oom_score_adj(u64 clone_flags, struct task_struct *tsk) 1985 { 1986 /* Skip if kernel thread */ 1987 if (!tsk->mm) 1988 return; 1989 1990 /* Skip if spawning a thread or using vfork */ 1991 if ((clone_flags & (CLONE_VM | CLONE_THREAD | CLONE_VFORK)) != CLONE_VM) 1992 return; 1993 1994 /* We need to synchronize with __set_oom_adj */ 1995 mutex_lock(&oom_adj_mutex); 1996 mm_flags_set(MMF_MULTIPROCESS, tsk->mm); 1997 /* Update the values in case they were changed after copy_signal */ 1998 tsk->signal->oom_score_adj = current->signal->oom_score_adj; 1999 tsk->signal->oom_score_adj_min = current->signal->oom_score_adj_min; 2000 mutex_unlock(&oom_adj_mutex); 2001 } 2002 2003 #ifdef CONFIG_RV 2004 static void rv_task_fork(struct task_struct *p) 2005 { 2006 memset(&p->rv, 0, sizeof(p->rv)); 2007 } 2008 #else 2009 #define rv_task_fork(p) do {} while (0) 2010 #endif 2011 2012 static bool need_futex_hash_allocate_default(u64 clone_flags) 2013 { 2014 /* 2015 * Allocate a default futex hash for any sibling that will 2016 * share the parent's mm. 2017 */ 2018 return clone_flags & CLONE_VM; 2019 } 2020 2021 /* 2022 * This creates a new process as a copy of the old one, 2023 * but does not actually start it yet. 2024 * 2025 * It copies the registers, and all the appropriate 2026 * parts of the process environment (as per the clone 2027 * flags). The actual kick-off is left to the caller. 2028 */ 2029 __latent_entropy struct task_struct *copy_process( 2030 struct pid *pid, 2031 int trace, 2032 int node, 2033 struct kernel_clone_args *args) 2034 { 2035 int pidfd = -1, retval; 2036 struct task_struct *p; 2037 struct multiprocess_signals delayed; 2038 struct file *pidfile = NULL; 2039 const u64 clone_flags = args->flags; 2040 struct nsproxy *nsp = current->nsproxy; 2041 2042 /* 2043 * Don't allow sharing the root directory with processes in a different 2044 * namespace 2045 */ 2046 if ((clone_flags & (CLONE_NEWNS|CLONE_FS)) == (CLONE_NEWNS|CLONE_FS)) 2047 return ERR_PTR(-EINVAL); 2048 2049 if ((clone_flags & (CLONE_NEWUSER|CLONE_FS)) == (CLONE_NEWUSER|CLONE_FS)) 2050 return ERR_PTR(-EINVAL); 2051 2052 /* 2053 * Thread groups must share signals as well, and detached threads 2054 * can only be started up within the thread group. 2055 */ 2056 if ((clone_flags & CLONE_THREAD) && !(clone_flags & CLONE_SIGHAND)) 2057 return ERR_PTR(-EINVAL); 2058 2059 /* 2060 * Shared signal handlers imply shared VM. By way of the above, 2061 * thread groups also imply shared VM. Blocking this case allows 2062 * for various simplifications in other code. 2063 */ 2064 if ((clone_flags & CLONE_SIGHAND) && !(clone_flags & CLONE_VM)) 2065 return ERR_PTR(-EINVAL); 2066 2067 /* 2068 * Siblings of global init remain as zombies on exit since they are 2069 * not reaped by their parent (swapper). To solve this and to avoid 2070 * multi-rooted process trees, prevent global and container-inits 2071 * from creating siblings. 2072 */ 2073 if ((clone_flags & CLONE_PARENT) && 2074 current->signal->flags & SIGNAL_UNKILLABLE) 2075 return ERR_PTR(-EINVAL); 2076 2077 /* 2078 * If the new process will be in a different pid or user namespace 2079 * do not allow it to share a thread group with the forking task. 2080 */ 2081 if (clone_flags & CLONE_THREAD) { 2082 if ((clone_flags & (CLONE_NEWUSER | CLONE_NEWPID)) || 2083 (task_active_pid_ns(current) != nsp->pid_ns_for_children)) 2084 return ERR_PTR(-EINVAL); 2085 } 2086 2087 if (clone_flags & CLONE_PIDFD) { 2088 /* 2089 * - CLONE_DETACHED is blocked so that we can potentially 2090 * reuse it later for CLONE_PIDFD. 2091 */ 2092 if (clone_flags & CLONE_DETACHED) 2093 return ERR_PTR(-EINVAL); 2094 } 2095 2096 if (clone_flags & CLONE_AUTOREAP) { 2097 if (clone_flags & CLONE_THREAD) 2098 return ERR_PTR(-EINVAL); 2099 if (clone_flags & CLONE_PARENT) 2100 return ERR_PTR(-EINVAL); 2101 if (args->exit_signal) 2102 return ERR_PTR(-EINVAL); 2103 } 2104 2105 if ((clone_flags & CLONE_PARENT) && current->signal->autoreap) 2106 return ERR_PTR(-EINVAL); 2107 2108 if (clone_flags & CLONE_NNP) { 2109 if (clone_flags & CLONE_THREAD) 2110 return ERR_PTR(-EINVAL); 2111 } 2112 2113 if (clone_flags & CLONE_PIDFD_AUTOKILL) { 2114 if (!(clone_flags & CLONE_PIDFD)) 2115 return ERR_PTR(-EINVAL); 2116 if (!(clone_flags & CLONE_AUTOREAP)) 2117 return ERR_PTR(-EINVAL); 2118 if (clone_flags & CLONE_THREAD) 2119 return ERR_PTR(-EINVAL); 2120 /* 2121 * Without CLONE_NNP the child could escalate privileges 2122 * after being spawned, so require CAP_SYS_ADMIN. 2123 * With CLONE_NNP the child can't gain new privileges, 2124 * so allow unprivileged usage. 2125 */ 2126 if (!(clone_flags & CLONE_NNP) && 2127 !ns_capable(current_user_ns(), CAP_SYS_ADMIN)) 2128 return ERR_PTR(-EPERM); 2129 } 2130 2131 /* 2132 * Force any signals received before this point to be delivered 2133 * before the fork happens. Collect up signals sent to multiple 2134 * processes that happen during the fork and delay them so that 2135 * they appear to happen after the fork. 2136 */ 2137 sigemptyset(&delayed.signal); 2138 INIT_HLIST_NODE(&delayed.node); 2139 2140 spin_lock_irq(¤t->sighand->siglock); 2141 if (!(clone_flags & CLONE_THREAD)) 2142 hlist_add_head(&delayed.node, ¤t->signal->multiprocess); 2143 recalc_sigpending(); 2144 spin_unlock_irq(¤t->sighand->siglock); 2145 retval = -ERESTARTNOINTR; 2146 if (task_sigpending(current)) 2147 goto fork_out; 2148 2149 retval = -ENOMEM; 2150 p = dup_task_struct(current, node); 2151 if (!p) 2152 goto fork_out; 2153 /* 2154 * Must run before the first fallible op, so error paths never 2155 * free the parent's ret_stack. 2156 */ 2157 ftrace_graph_init_task(p); 2158 retval = copy_exec_state(clone_flags, p); 2159 if (retval) 2160 goto bad_fork_free; 2161 p->flags &= ~PF_KTHREAD; 2162 if (args->kthread) 2163 p->flags |= PF_KTHREAD; 2164 if (args->user_worker) { 2165 /* A user worker takes only the signals nobody can block. */ 2166 p->flags |= PF_USER_WORKER; 2167 siginitsetinv(&p->blocked, SIG_KERNEL_ONLY_MASK); 2168 } 2169 if (args->io_thread) 2170 p->flags |= PF_IO_WORKER; 2171 2172 if (args->name) 2173 strscpy_pad(p->comm, args->name, sizeof(p->comm)); 2174 2175 p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? args->child_tid : NULL; 2176 /* 2177 * TID is cleared in mm_release() when the task exits 2178 */ 2179 p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? args->child_tid : NULL; 2180 2181 rt_mutex_init_task(p); 2182 raw_spin_lock_init(&p->blocked_lock); 2183 2184 lockdep_assert_irqs_enabled(); 2185 #ifdef CONFIG_PROVE_LOCKING 2186 DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled); 2187 #endif 2188 retval = copy_creds(p, clone_flags); 2189 if (retval < 0) 2190 goto bad_fork_free; 2191 2192 retval = -EAGAIN; 2193 if (is_rlimit_overlimit(task_ucounts(p), UCOUNT_RLIMIT_NPROC, rlimit(RLIMIT_NPROC))) { 2194 if (p->real_cred->user != INIT_USER && 2195 !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN)) 2196 goto bad_fork_cleanup_count; 2197 } 2198 current->flags &= ~PF_NPROC_EXCEEDED; 2199 2200 /* 2201 * If multiple threads are within copy_process(), then this check 2202 * triggers too late. This doesn't hurt, the check is only there 2203 * to stop root fork bombs. 2204 */ 2205 retval = -EAGAIN; 2206 if (data_race(nr_threads >= max_threads)) 2207 goto bad_fork_cleanup_count; 2208 2209 delayacct_tsk_init(p); /* Must remain after dup_task_struct() */ 2210 p->flags &= ~(PF_SUPERPRIV | PF_WQ_WORKER | PF_IDLE | PF_NO_SETAFFINITY); 2211 p->flags |= PF_FORKNOEXEC; 2212 INIT_LIST_HEAD(&p->children); 2213 INIT_LIST_HEAD(&p->sibling); 2214 rcu_copy_process(p); 2215 p->vfork_done = NULL; 2216 /* Set by copy_files(), exit_files() on the error path skips NULL. */ 2217 p->files = NULL; 2218 spin_lock_init(&p->alloc_lock); 2219 2220 init_sigpending(&p->pending); 2221 2222 p->utime = p->stime = p->gtime = 0; 2223 #ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME 2224 p->utimescaled = p->stimescaled = 0; 2225 #endif 2226 prev_cputime_init(&p->prev_cputime); 2227 2228 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN 2229 seqcount_init(&p->vtime.seqcount); 2230 p->vtime.starttime = 0; 2231 p->vtime.state = VTIME_INACTIVE; 2232 #endif 2233 2234 #ifdef CONFIG_IO_URING 2235 p->io_uring = NULL; 2236 retval = io_uring_fork(p); 2237 if (unlikely(retval)) 2238 goto bad_fork_cleanup_delayacct; 2239 retval = -EAGAIN; 2240 #endif 2241 2242 p->default_timer_slack_ns = current->timer_slack_ns; 2243 2244 #ifdef CONFIG_PSI 2245 p->psi_flags = 0; 2246 #endif 2247 2248 task_io_accounting_init(&p->ioac); 2249 acct_clear_integrals(p); 2250 2251 posix_cputimers_init(&p->posix_cputimers); 2252 tick_dep_init_task(p); 2253 2254 p->io_context = NULL; 2255 audit_set_context(p, NULL); 2256 cgroup_fork(p); 2257 if (args->kthread) { 2258 if (!set_kthread_struct(p)) 2259 goto bad_fork_cleanup_delayacct; 2260 } 2261 #ifdef CONFIG_NUMA 2262 p->mempolicy = mpol_dup(p->mempolicy); 2263 if (IS_ERR(p->mempolicy)) { 2264 retval = PTR_ERR(p->mempolicy); 2265 p->mempolicy = NULL; 2266 goto bad_fork_cleanup_delayacct; 2267 } 2268 #endif 2269 #ifdef CONFIG_CPUSETS 2270 p->cpuset_mem_spread_rotor = NUMA_NO_NODE; 2271 seqcount_spinlock_init(&p->mems_allowed_seq, &p->alloc_lock); 2272 #endif 2273 #ifdef CONFIG_TRACE_IRQFLAGS 2274 memset(&p->irqtrace, 0, sizeof(p->irqtrace)); 2275 p->irqtrace.hardirq_disable_ip = _THIS_IP_; 2276 p->irqtrace.softirq_enable_ip = _THIS_IP_; 2277 p->softirqs_enabled = 1; 2278 p->softirq_context = 0; 2279 #endif 2280 2281 p->pagefault_disabled = 0; 2282 2283 lockdep_init_task(p); 2284 2285 p->blocked_on = NULL; /* not blocked yet */ 2286 p->blocked_donor = NULL; /* nobody is boosting p yet */ 2287 2288 #ifdef CONFIG_BCACHE 2289 p->sequential_io = 0; 2290 p->sequential_io_avg = 0; 2291 #endif 2292 2293 unwind_task_init(p); 2294 2295 /* Perform scheduler related setup. Assign this task to a CPU. */ 2296 retval = sched_fork(clone_flags, p); 2297 if (retval) 2298 goto bad_fork_cleanup_policy; 2299 2300 retval = perf_event_init_task(p, clone_flags); 2301 if (retval) 2302 goto bad_fork_sched_cancel_fork; 2303 retval = audit_alloc(p); 2304 if (retval) 2305 goto bad_fork_cleanup_perf; 2306 /* copy all the process information */ 2307 shm_init_task(p); 2308 retval = security_task_alloc(p, clone_flags); 2309 if (retval) 2310 goto bad_fork_cleanup_audit; 2311 retval = copy_semundo(clone_flags, p); 2312 if (retval) 2313 goto bad_fork_cleanup_security; 2314 retval = copy_files(clone_flags, p, args->no_files); 2315 if (retval) 2316 goto bad_fork_cleanup_semundo; 2317 retval = copy_fs(clone_flags, p, 2318 (args->umh ? COPY_FS_UMH : 0) | 2319 (args->idle ? COPY_FS_IDLE : 0)); 2320 if (retval) 2321 goto bad_fork_cleanup_semundo; 2322 retval = copy_sighand(clone_flags, p); 2323 if (retval) 2324 goto bad_fork_cleanup_fs; 2325 retval = copy_signal(clone_flags, p); 2326 if (retval) 2327 goto bad_fork_cleanup_sighand; 2328 retval = copy_mm(clone_flags, p); 2329 if (retval) 2330 goto bad_fork_cleanup_signal; 2331 retval = copy_namespaces(clone_flags, p); 2332 if (retval) 2333 goto bad_fork_cleanup_mm; 2334 retval = copy_io(clone_flags, p); 2335 if (retval) 2336 goto bad_fork_cleanup_namespaces; 2337 retval = copy_thread(p, args); 2338 if (retval) 2339 goto bad_fork_cleanup_io; 2340 2341 stackleak_task_init(p); 2342 2343 if (pid != &init_struct_pid) { 2344 pid = alloc_pid(p->nsproxy->pid_ns_for_children, args->set_tid, 2345 args->set_tid_size); 2346 if (IS_ERR(pid)) { 2347 retval = PTR_ERR(pid); 2348 goto bad_fork_cleanup_thread; 2349 } 2350 } 2351 2352 /* 2353 * This has to happen after we've potentially unshared the file 2354 * descriptor table (so that the pidfd doesn't leak into the child 2355 * if the fd table isn't shared). 2356 */ 2357 if (clone_flags & CLONE_PIDFD) { 2358 unsigned flags = PIDFD_STALE; 2359 2360 if (clone_flags & CLONE_THREAD) 2361 flags |= PIDFD_THREAD; 2362 if (clone_flags & CLONE_PIDFD_AUTOKILL) 2363 flags |= PIDFD_AUTOKILL; 2364 2365 /* 2366 * Note that no task has been attached to @pid yet indicate 2367 * that via CLONE_PIDFD. 2368 */ 2369 retval = pidfd_prepare(pid, flags, &pidfile); 2370 if (retval < 0) 2371 goto bad_fork_free_pid; 2372 pidfd = retval; 2373 2374 retval = put_user(pidfd, args->pidfd); 2375 if (retval) 2376 goto bad_fork_put_pidfd; 2377 } 2378 2379 #ifdef CONFIG_BLOCK 2380 p->plug = NULL; 2381 p->flags &= ~PF_BLOCK_TS; 2382 #endif 2383 futex_init_task(p); 2384 2385 /* 2386 * sigaltstack should be cleared when sharing the same VM 2387 */ 2388 if ((clone_flags & (CLONE_VM|CLONE_VFORK)) == CLONE_VM) 2389 sas_ss_reset(p); 2390 2391 /* 2392 * Syscall tracing and stepping should be turned off in the 2393 * child regardless of CLONE_PTRACE. 2394 */ 2395 user_disable_single_step(p); 2396 clear_task_syscall_work(p, SYSCALL_TRACE); 2397 #if defined(CONFIG_GENERIC_ENTRY) || defined(TIF_SYSCALL_EMU) 2398 clear_task_syscall_work(p, SYSCALL_EMU); 2399 #endif 2400 clear_tsk_latency_tracing(p); 2401 2402 /* ok, now we should be set up.. */ 2403 p->pid = pid_nr(pid); 2404 if (clone_flags & CLONE_THREAD) { 2405 p->group_leader = current->group_leader; 2406 p->tgid = current->tgid; 2407 } else { 2408 p->group_leader = p; 2409 p->tgid = p->pid; 2410 } 2411 2412 p->nr_dirtied = 0; 2413 p->nr_dirtied_pause = 128 >> (PAGE_SHIFT - 10); 2414 p->dirty_paused_when = 0; 2415 2416 p->pdeath_signal = 0; 2417 p->task_works = NULL; 2418 clear_posix_cputimers_work(p); 2419 2420 #ifdef CONFIG_KRETPROBES 2421 p->kretprobe_instances.first = NULL; 2422 #endif 2423 #ifdef CONFIG_RETHOOK 2424 p->rethooks.first = NULL; 2425 #endif 2426 2427 /* 2428 * Ensure that the cgroup subsystem policies allow the new process to be 2429 * forked. It should be noted that the new process's css_set can be changed 2430 * between here and cgroup_post_fork() if an organisation operation is in 2431 * progress. 2432 */ 2433 retval = cgroup_can_fork(p, args); 2434 if (retval) 2435 goto bad_fork_put_pidfd; 2436 2437 /* 2438 * Now that the cgroups are pinned, re-clone the parent cgroup and put 2439 * the new task on the correct runqueue. All this *before* the task 2440 * becomes visible. 2441 * 2442 * This isn't part of ->can_fork() because while the re-cloning is 2443 * cgroup specific, it unconditionally needs to place the task on a 2444 * runqueue. 2445 */ 2446 retval = sched_cgroup_fork(p, args); 2447 if (retval) 2448 goto bad_fork_cancel_cgroup; 2449 2450 if (need_futex_hash_allocate_default(clone_flags)) { 2451 retval = futex_hash_allocate_default(); 2452 if (retval) 2453 goto bad_fork_cancel_cgroup; 2454 /* 2455 * If we fail beyond this point we don't free the allocated 2456 * futex hash map. We assume that another thread will be created 2457 * and makes use of it. The hash map will be freed once the main 2458 * thread terminates. 2459 */ 2460 } 2461 /* 2462 * From this point on we must avoid any synchronous user-space 2463 * communication until we take the tasklist-lock. In particular, we do 2464 * not want user-space to be able to predict the process start-time by 2465 * stalling fork(2) after we recorded the start_time but before it is 2466 * visible to the system. 2467 */ 2468 2469 p->start_time = ktime_get_ns(); 2470 p->start_boottime = ktime_get_boottime_ns(); 2471 2472 /* 2473 * Make it visible to the rest of the system, but dont wake it up yet. 2474 * Need tasklist lock for parent etc handling! 2475 */ 2476 write_lock_irq(&tasklist_lock); 2477 2478 /* CLONE_PARENT re-uses the old parent */ 2479 if (clone_flags & (CLONE_PARENT|CLONE_THREAD)) { 2480 p->real_parent = current->real_parent; 2481 p->parent_exec_id = current->parent_exec_id; 2482 if (clone_flags & CLONE_THREAD) 2483 p->exit_signal = -1; 2484 else 2485 p->exit_signal = current->group_leader->exit_signal; 2486 } else { 2487 p->real_parent = current; 2488 p->parent_exec_id = current->self_exec_id; 2489 p->exit_signal = args->exit_signal; 2490 } 2491 2492 klp_copy_process(p); 2493 2494 sched_core_fork(p); 2495 2496 spin_lock(¤t->sighand->siglock); 2497 2498 rv_task_fork(p); 2499 2500 rseq_fork(p, clone_flags); 2501 2502 /* 2503 * If zap_pid_ns_processes() was called after alloc_pid(), the new 2504 * child missed SIGKILL. If current is not in the same namespace, 2505 * we can't rely on fatal_signal_pending() below. 2506 */ 2507 if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) { 2508 retval = -ENOMEM; 2509 goto bad_fork_core_free; 2510 } 2511 2512 /* Let kill terminate clone/fork in the middle */ 2513 if (fatal_signal_pending(current)) { 2514 retval = -EINTR; 2515 goto bad_fork_core_free; 2516 } 2517 2518 /* No more failure paths after this point. */ 2519 2520 /* 2521 * One for the user space visible state that goes away when reaped. 2522 * One for the scheduler. 2523 */ 2524 refcount_set(&p->rcu_users, 2); 2525 2526 /* 2527 * Copy seccomp details explicitly here, in case they were changed 2528 * before holding sighand lock. 2529 */ 2530 copy_seccomp(p); 2531 2532 if (clone_flags & CLONE_NNP) 2533 task_set_no_new_privs(p); 2534 2535 init_task_pid_links(p); 2536 if (likely(p->pid)) { 2537 ptrace_init_task(p, (clone_flags & CLONE_PTRACE) || trace); 2538 2539 init_task_pid(p, PIDTYPE_PID, pid); 2540 if (thread_group_leader(p)) { 2541 init_task_pid(p, PIDTYPE_TGID, pid); 2542 init_task_pid(p, PIDTYPE_PGID, task_pgrp(current)); 2543 init_task_pid(p, PIDTYPE_SID, task_session(current)); 2544 2545 if (is_child_reaper(pid)) { 2546 struct pid_namespace *ns = ns_of_pid(pid); 2547 2548 ASSERT_EXCLUSIVE_WRITER(ns->child_reaper); 2549 WRITE_ONCE(ns->child_reaper, p); 2550 p->signal->flags |= SIGNAL_UNKILLABLE; 2551 } 2552 p->signal->shared_pending.signal = delayed.signal; 2553 p->signal->tty = tty_kref_get(current->signal->tty); 2554 /* 2555 * Inherit has_child_subreaper flag under the same 2556 * tasklist_lock with adding child to the process tree 2557 * for propagate_has_child_subreaper optimization. 2558 */ 2559 p->signal->has_child_subreaper = p->real_parent->signal->has_child_subreaper || 2560 p->real_parent->signal->is_child_subreaper; 2561 if (clone_flags & CLONE_AUTOREAP) 2562 p->signal->autoreap = 1; 2563 list_add_tail(&p->sibling, &p->real_parent->children); 2564 list_add_tail_rcu(&p->tasks, &init_task.tasks); 2565 attach_pid(p, PIDTYPE_TGID); 2566 attach_pid(p, PIDTYPE_PGID); 2567 attach_pid(p, PIDTYPE_SID); 2568 __this_cpu_inc(process_counts); 2569 } else { 2570 current->signal->nr_threads++; 2571 current->signal->quick_threads++; 2572 atomic_inc(¤t->signal->live); 2573 refcount_inc(¤t->signal->sigcnt); 2574 task_join_group_stop(p); 2575 list_add_tail_rcu(&p->thread_node, 2576 &p->signal->thread_head); 2577 } 2578 attach_pid(p, PIDTYPE_PID); 2579 nr_threads++; 2580 } 2581 total_forks++; 2582 hlist_del_init(&delayed.node); 2583 spin_unlock(¤t->sighand->siglock); 2584 syscall_tracepoint_update(p); 2585 write_unlock_irq(&tasklist_lock); 2586 2587 if (pidfile) 2588 fd_install(pidfd, pidfile); 2589 2590 proc_fork_connector(p); 2591 /* 2592 * sched_ext needs @p to be associated with its cgroup in its post_fork 2593 * hook. cgroup_post_fork() should come before sched_post_fork(). 2594 */ 2595 cgroup_post_fork(p, args); 2596 sched_post_fork(p); 2597 perf_event_fork(p); 2598 2599 trace_task_newtask(p, clone_flags); 2600 uprobe_copy_process(p, clone_flags); 2601 user_events_fork(p, clone_flags); 2602 2603 copy_oom_score_adj(clone_flags, p); 2604 2605 return p; 2606 2607 bad_fork_core_free: 2608 sched_core_free(p); 2609 spin_unlock(¤t->sighand->siglock); 2610 write_unlock_irq(&tasklist_lock); 2611 bad_fork_cancel_cgroup: 2612 cgroup_cancel_fork(p, args); 2613 bad_fork_put_pidfd: 2614 if (clone_flags & CLONE_PIDFD) { 2615 fput(pidfile); 2616 put_unused_fd(pidfd); 2617 } 2618 bad_fork_free_pid: 2619 if (pid != &init_struct_pid) 2620 free_pid(pid); 2621 bad_fork_cleanup_thread: 2622 exit_thread(p); 2623 bad_fork_cleanup_io: 2624 if (p->io_context) 2625 exit_io_context(p); 2626 bad_fork_cleanup_namespaces: 2627 exit_nsproxy_namespaces(p); 2628 bad_fork_cleanup_mm: 2629 sched_cache_fork_cleanup(p); 2630 if (p->mm) { 2631 mm_clear_owner(p->mm, p); 2632 mmput(p->mm); 2633 } 2634 bad_fork_cleanup_signal: 2635 if (!(clone_flags & CLONE_THREAD)) 2636 free_signal_struct(p->signal); 2637 bad_fork_cleanup_sighand: 2638 __cleanup_sighand(p->sighand); 2639 bad_fork_cleanup_fs: 2640 exit_fs(p); /* blocking */ 2641 bad_fork_cleanup_semundo: 2642 exit_sem(p); 2643 bad_fork_cleanup_security: 2644 security_task_free(p); 2645 bad_fork_cleanup_audit: 2646 audit_free(p); 2647 bad_fork_cleanup_perf: 2648 perf_event_free_task(p); 2649 bad_fork_sched_cancel_fork: 2650 sched_cancel_fork(p); 2651 /* ->release() of a file may need scx_fork_rwsem for write. */ 2652 exit_files(p); /* blocking */ 2653 bad_fork_cleanup_policy: 2654 lockdep_free_task(p); 2655 #ifdef CONFIG_NUMA 2656 mpol_put(p->mempolicy); 2657 #endif 2658 bad_fork_cleanup_delayacct: 2659 io_uring_free(p); 2660 delayacct_tsk_free(p); 2661 bad_fork_cleanup_count: 2662 dec_rlimit_ucounts(task_ucounts(p), UCOUNT_RLIMIT_NPROC, 1); 2663 exit_cred_namespaces(p); 2664 exit_creds(p); 2665 bad_fork_free: 2666 WRITE_ONCE(p->__state, TASK_DEAD); 2667 exit_task_stack_account(p); 2668 put_task_stack(p); 2669 delayed_free_task(p); 2670 fork_out: 2671 spin_lock_irq(¤t->sighand->siglock); 2672 hlist_del_init(&delayed.node); 2673 spin_unlock_irq(¤t->sighand->siglock); 2674 return ERR_PTR(retval); 2675 } 2676 2677 static inline void init_idle_pids(struct task_struct *idle) 2678 { 2679 enum pid_type type; 2680 2681 for (type = PIDTYPE_PID; type < PIDTYPE_MAX; ++type) { 2682 INIT_HLIST_NODE(&idle->pid_links[type]); /* not really needed */ 2683 init_task_pid(idle, type, &init_struct_pid); 2684 } 2685 } 2686 2687 static int idle_dummy(void *dummy) 2688 { 2689 /* This function is never called */ 2690 return 0; 2691 } 2692 2693 struct task_struct * __init fork_idle(int cpu) 2694 { 2695 struct task_struct *task; 2696 struct kernel_clone_args args = { 2697 .flags = CLONE_VM | CLONE_FS, 2698 .fn = &idle_dummy, 2699 .fn_arg = NULL, 2700 .kthread = 1, 2701 .idle = 1, 2702 }; 2703 2704 task = copy_process(&init_struct_pid, 0, cpu_to_node(cpu), &args); 2705 if (!IS_ERR(task)) { 2706 init_idle_pids(task); 2707 init_idle(task, cpu); 2708 } 2709 2710 return task; 2711 } 2712 2713 /* 2714 * This is like kernel_clone(), but shaved down and tailored to just 2715 * creating io_uring workers. It returns a created task, or an error pointer. 2716 * The returned task is inactive, and the caller must fire it up through 2717 * wake_up_new_task(p). All signals are blocked in the created task. 2718 */ 2719 struct task_struct *create_io_thread(int (*fn)(void *), void *arg, int node) 2720 { 2721 unsigned long flags = CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD| 2722 CLONE_IO|CLONE_VM|CLONE_UNTRACED; 2723 struct kernel_clone_args args = { 2724 .flags = flags, 2725 .fn = fn, 2726 .fn_arg = arg, 2727 .io_thread = 1, 2728 .user_worker = 1, 2729 }; 2730 2731 /* A creator past its fatal signal or its coredump point gets no thread. */ 2732 if (current->flags & (PF_SIGNALED | PF_POSTCOREDUMP)) 2733 return ERR_PTR(-EINTR); 2734 2735 return copy_process(NULL, 0, node, &args); 2736 } 2737 2738 /* 2739 * Ok, this is the main fork-routine. 2740 * 2741 * It copies the process, and if successful kick-starts 2742 * it and waits for it to finish using the VM if required. 2743 */ 2744 pid_t kernel_clone(struct kernel_clone_args *args) 2745 { 2746 u64 clone_flags = args->flags; 2747 struct completion vfork; 2748 struct pid *pid; 2749 struct task_struct *p; 2750 int trace = 0; 2751 pid_t nr; 2752 2753 /* 2754 * Creating an empty mount namespace implies creating a new mount 2755 * namespace. Set this before copy_process() so that the 2756 * CLONE_NEWNS|CLONE_FS mutual exclusion check works correctly. 2757 */ 2758 if (clone_flags & CLONE_EMPTY_MNTNS) { 2759 clone_flags |= CLONE_NEWNS; 2760 args->flags = clone_flags; 2761 } 2762 2763 /* 2764 * For legacy clone() calls, CLONE_PIDFD uses the parent_tid argument 2765 * to return the pidfd. Hence, CLONE_PIDFD and CLONE_PARENT_SETTID are 2766 * mutually exclusive. With clone3() CLONE_PIDFD has grown a separate 2767 * field in struct clone_args and it still doesn't make sense to have 2768 * them both point at the same memory location. Performing this check 2769 * here has the advantage that we don't need to have a separate helper 2770 * to check for legacy clone(). 2771 */ 2772 if ((clone_flags & CLONE_PIDFD) && 2773 (clone_flags & CLONE_PARENT_SETTID) && 2774 (args->pidfd == args->parent_tid)) 2775 return -EINVAL; 2776 2777 if (!valid_signal(args->exit_signal)) 2778 return -EINVAL; 2779 2780 /* 2781 * Determine whether and which event to report to ptracer. When 2782 * called from kernel_thread or CLONE_UNTRACED is explicitly 2783 * requested, no event is reported; otherwise, report if the event 2784 * for the type of forking is enabled. 2785 */ 2786 if (!(clone_flags & CLONE_UNTRACED)) { 2787 if (clone_flags & CLONE_VFORK) 2788 trace = PTRACE_EVENT_VFORK; 2789 else if (args->exit_signal != SIGCHLD) 2790 trace = PTRACE_EVENT_CLONE; 2791 else 2792 trace = PTRACE_EVENT_FORK; 2793 2794 if (likely(!ptrace_event_enabled(current, trace))) 2795 trace = 0; 2796 } 2797 2798 p = copy_process(NULL, trace, NUMA_NO_NODE, args); 2799 add_latent_entropy(); 2800 2801 if (IS_ERR(p)) 2802 return PTR_ERR(p); 2803 2804 /* 2805 * Do this prior waking up the new thread - the thread pointer 2806 * might get invalid after that point, if the thread exits quickly. 2807 */ 2808 trace_sched_process_fork(current, p); 2809 2810 pid = get_task_pid(p, PIDTYPE_PID); 2811 nr = pid_vnr(pid); 2812 2813 if (clone_flags & CLONE_PARENT_SETTID) 2814 put_user(nr, args->parent_tid); 2815 2816 if (clone_flags & CLONE_VFORK) { 2817 p->vfork_done = &vfork; 2818 init_completion(&vfork); 2819 get_task_struct(p); 2820 } 2821 2822 if (IS_ENABLED(CONFIG_LRU_GEN_WALKS_MMU) && !(clone_flags & CLONE_VM)) { 2823 /* lock the task to synchronize with memcg migration */ 2824 task_lock(p); 2825 lru_gen_add_mm(p->mm); 2826 task_unlock(p); 2827 } 2828 2829 wake_up_new_task(p); 2830 2831 /* forking complete and child started to run, tell ptracer */ 2832 if (unlikely(trace)) 2833 ptrace_event_pid(trace, pid); 2834 2835 if (clone_flags & CLONE_VFORK) { 2836 if (!wait_for_vfork_done(p, &vfork)) 2837 ptrace_event_pid(PTRACE_EVENT_VFORK_DONE, pid); 2838 } 2839 2840 put_pid(pid); 2841 return nr; 2842 } 2843 2844 /* 2845 * Create a kernel thread. 2846 */ 2847 pid_t kernel_thread(int (*fn)(void *), void *arg, const char *name, 2848 unsigned long flags) 2849 { 2850 struct kernel_clone_args args = { 2851 .flags = ((flags | CLONE_VM | CLONE_UNTRACED) & ~CSIGNAL), 2852 .exit_signal = (flags & CSIGNAL), 2853 .fn = fn, 2854 .fn_arg = arg, 2855 .name = name, 2856 .kthread = 1, 2857 }; 2858 2859 return kernel_clone(&args); 2860 } 2861 2862 /* 2863 * Create a user mode thread. 2864 */ 2865 pid_t user_mode_thread(int (*fn)(void *), void *arg, unsigned long flags) 2866 { 2867 struct kernel_clone_args args = { 2868 .flags = ((flags | CLONE_VM | CLONE_UNTRACED) & ~CSIGNAL), 2869 .exit_signal = (flags & CSIGNAL), 2870 .fn = fn, 2871 .fn_arg = arg, 2872 .umh = 1, 2873 }; 2874 2875 return kernel_clone(&args); 2876 } 2877 2878 #ifdef __ARCH_WANT_SYS_FORK 2879 SYSCALL_DEFINE0(fork) 2880 { 2881 #ifdef CONFIG_MMU 2882 struct kernel_clone_args args = { 2883 .exit_signal = SIGCHLD, 2884 }; 2885 2886 return kernel_clone(&args); 2887 #else 2888 /* can not support in nommu mode */ 2889 return -EINVAL; 2890 #endif 2891 } 2892 #endif 2893 2894 #ifdef __ARCH_WANT_SYS_VFORK 2895 SYSCALL_DEFINE0(vfork) 2896 { 2897 struct kernel_clone_args args = { 2898 .flags = CLONE_VFORK | CLONE_VM, 2899 .exit_signal = SIGCHLD, 2900 }; 2901 2902 return kernel_clone(&args); 2903 } 2904 #endif 2905 2906 #ifdef __ARCH_WANT_SYS_CLONE 2907 #ifdef CONFIG_CLONE_BACKWARDS 2908 SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp, 2909 int __user *, parent_tidptr, 2910 unsigned long, tls, 2911 int __user *, child_tidptr) 2912 #elif defined(CONFIG_CLONE_BACKWARDS2) 2913 SYSCALL_DEFINE5(clone, unsigned long, newsp, unsigned long, clone_flags, 2914 int __user *, parent_tidptr, 2915 int __user *, child_tidptr, 2916 unsigned long, tls) 2917 #elif defined(CONFIG_CLONE_BACKWARDS3) 2918 SYSCALL_DEFINE6(clone, unsigned long, clone_flags, unsigned long, newsp, 2919 int, stack_size, 2920 int __user *, parent_tidptr, 2921 int __user *, child_tidptr, 2922 unsigned long, tls) 2923 #else 2924 SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp, 2925 int __user *, parent_tidptr, 2926 int __user *, child_tidptr, 2927 unsigned long, tls) 2928 #endif 2929 { 2930 struct kernel_clone_args args = { 2931 .flags = (lower_32_bits(clone_flags) & ~CSIGNAL), 2932 .pidfd = parent_tidptr, 2933 .child_tid = child_tidptr, 2934 .parent_tid = parent_tidptr, 2935 .exit_signal = (lower_32_bits(clone_flags) & CSIGNAL), 2936 .stack = newsp, 2937 .tls = tls, 2938 }; 2939 2940 return kernel_clone(&args); 2941 } 2942 #endif 2943 2944 static noinline int copy_clone_args_from_user(struct kernel_clone_args *kargs, 2945 struct clone_args __user *uargs, 2946 size_t usize) 2947 { 2948 int err; 2949 struct clone_args args; 2950 pid_t *kset_tid = kargs->set_tid; 2951 2952 BUILD_BUG_ON(offsetofend(struct clone_args, tls) != 2953 CLONE_ARGS_SIZE_VER0); 2954 BUILD_BUG_ON(offsetofend(struct clone_args, set_tid_size) != 2955 CLONE_ARGS_SIZE_VER1); 2956 BUILD_BUG_ON(offsetofend(struct clone_args, cgroup) != 2957 CLONE_ARGS_SIZE_VER2); 2958 BUILD_BUG_ON(sizeof(struct clone_args) != CLONE_ARGS_SIZE_VER2); 2959 2960 if (unlikely(usize > PAGE_SIZE)) 2961 return -E2BIG; 2962 if (unlikely(usize < CLONE_ARGS_SIZE_VER0)) 2963 return -EINVAL; 2964 2965 err = copy_struct_from_user(&args, sizeof(args), uargs, usize); 2966 if (err) 2967 return err; 2968 2969 if (unlikely(args.set_tid_size > MAX_PID_NS_LEVEL)) 2970 return -EINVAL; 2971 2972 if (unlikely(!args.set_tid && args.set_tid_size > 0)) 2973 return -EINVAL; 2974 2975 if (unlikely(args.set_tid && args.set_tid_size == 0)) 2976 return -EINVAL; 2977 2978 /* 2979 * Verify that higher 32bits of exit_signal are unset 2980 */ 2981 if (unlikely(args.exit_signal & ~((u64)CSIGNAL))) 2982 return -EINVAL; 2983 2984 if ((args.flags & CLONE_INTO_CGROUP) && 2985 (args.cgroup > INT_MAX || usize < CLONE_ARGS_SIZE_VER2)) 2986 return -EINVAL; 2987 2988 *kargs = (struct kernel_clone_args){ 2989 .flags = args.flags, 2990 .pidfd = u64_to_user_ptr(args.pidfd), 2991 .child_tid = u64_to_user_ptr(args.child_tid), 2992 .parent_tid = u64_to_user_ptr(args.parent_tid), 2993 .exit_signal = args.exit_signal, 2994 .stack = args.stack, 2995 .stack_size = args.stack_size, 2996 .tls = args.tls, 2997 .set_tid_size = args.set_tid_size, 2998 .cgroup = args.cgroup, 2999 }; 3000 3001 if (args.set_tid && 3002 copy_from_user(kset_tid, u64_to_user_ptr(args.set_tid), 3003 (kargs->set_tid_size * sizeof(pid_t)))) 3004 return -EFAULT; 3005 3006 kargs->set_tid = kset_tid; 3007 3008 return 0; 3009 } 3010 3011 /** 3012 * clone3_stack_valid - check and prepare stack 3013 * @kargs: kernel clone args 3014 * 3015 * Verify that the stack arguments userspace gave us are sane. 3016 * In addition, set the stack direction for userspace since it's easy for us to 3017 * determine. 3018 */ 3019 static inline bool clone3_stack_valid(struct kernel_clone_args *kargs) 3020 { 3021 if (kargs->stack == 0) { 3022 if (kargs->stack_size > 0) 3023 return false; 3024 } else { 3025 if (kargs->stack_size == 0) 3026 return false; 3027 3028 if (!access_ok((void __user *)kargs->stack, kargs->stack_size)) 3029 return false; 3030 3031 #if !defined(CONFIG_STACK_GROWSUP) 3032 kargs->stack += kargs->stack_size; 3033 #endif 3034 } 3035 3036 return true; 3037 } 3038 3039 static bool clone3_args_valid(struct kernel_clone_args *kargs) 3040 { 3041 /* Verify that no unknown flags are passed along. */ 3042 if (kargs->flags & 3043 ~(CLONE_LEGACY_FLAGS | CLONE_CLEAR_SIGHAND | 3044 CLONE_INTO_CGROUP | CLONE_AUTOREAP | CLONE_NNP | 3045 CLONE_PIDFD_AUTOKILL | CLONE_EMPTY_MNTNS)) 3046 return false; 3047 3048 /* 3049 * - make the CLONE_DETACHED bit reusable for clone3 3050 * - make the CSIGNAL bits reusable for clone3 3051 */ 3052 if (kargs->flags & (CLONE_DETACHED | (CSIGNAL & (~CLONE_NEWTIME)))) 3053 return false; 3054 3055 if ((kargs->flags & (CLONE_SIGHAND | CLONE_CLEAR_SIGHAND)) == 3056 (CLONE_SIGHAND | CLONE_CLEAR_SIGHAND)) 3057 return false; 3058 3059 if ((kargs->flags & (CLONE_THREAD | CLONE_PARENT)) && 3060 kargs->exit_signal) 3061 return false; 3062 3063 if (!clone3_stack_valid(kargs)) 3064 return false; 3065 3066 return true; 3067 } 3068 3069 /** 3070 * sys_clone3 - create a new process with specific properties 3071 * @uargs: argument structure 3072 * @size: size of @uargs 3073 * 3074 * clone3() is the extensible successor to clone()/clone2(). 3075 * It takes a struct as argument that is versioned by its size. 3076 * 3077 * Return: On success, a positive PID for the child process. 3078 * On error, a negative errno number. 3079 */ 3080 SYSCALL_DEFINE2(clone3, struct clone_args __user *, uargs, size_t, size) 3081 { 3082 int err; 3083 3084 struct kernel_clone_args kargs; 3085 pid_t set_tid[MAX_PID_NS_LEVEL]; 3086 3087 #ifdef __ARCH_BROKEN_SYS_CLONE3 3088 #warning clone3() entry point is missing, please fix 3089 return -ENOSYS; 3090 #endif 3091 3092 kargs.set_tid = set_tid; 3093 3094 err = copy_clone_args_from_user(&kargs, uargs, size); 3095 if (err) 3096 return err; 3097 3098 if (!clone3_args_valid(&kargs)) 3099 return -EINVAL; 3100 3101 return kernel_clone(&kargs); 3102 } 3103 3104 void walk_process_tree(struct task_struct *top, proc_visitor visitor, void *data) 3105 { 3106 struct task_struct *leader, *parent, *child; 3107 int res; 3108 3109 read_lock(&tasklist_lock); 3110 leader = top = top->group_leader; 3111 down: 3112 for_each_thread(leader, parent) { 3113 list_for_each_entry(child, &parent->children, sibling) { 3114 res = visitor(child, data); 3115 if (res) { 3116 if (res < 0) 3117 goto out; 3118 leader = child; 3119 goto down; 3120 } 3121 up: 3122 ; 3123 } 3124 } 3125 3126 if (leader != top) { 3127 child = leader; 3128 parent = child->real_parent; 3129 leader = parent->group_leader; 3130 goto up; 3131 } 3132 out: 3133 read_unlock(&tasklist_lock); 3134 } 3135 3136 #ifndef ARCH_MIN_MMSTRUCT_ALIGN 3137 #define ARCH_MIN_MMSTRUCT_ALIGN 0 3138 #endif 3139 3140 static void sighand_ctor(void *data) 3141 { 3142 struct sighand_struct *sighand = data; 3143 3144 spin_lock_init(&sighand->siglock); 3145 init_waitqueue_head(&sighand->signalfd_wqh); 3146 } 3147 3148 void __init mm_cache_init(void) 3149 { 3150 unsigned int mm_size; 3151 3152 /* 3153 * The mm_cpumask is located at the end of mm_struct, and is 3154 * dynamically sized based on the maximum CPU number this system 3155 * can have, taking hotplug into account (nr_cpu_ids). 3156 */ 3157 mm_size = sizeof(struct mm_struct) + cpumask_size() + mm_cid_size(); 3158 3159 mm_cachep = kmem_cache_create_usercopy("mm_struct", 3160 mm_size, ARCH_MIN_MMSTRUCT_ALIGN, 3161 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, 3162 offsetof(struct mm_struct, saved_auxv), 3163 sizeof_field(struct mm_struct, saved_auxv), 3164 NULL); 3165 } 3166 3167 void __init proc_caches_init(void) 3168 { 3169 sighand_cachep = kmem_cache_create("sighand_cache", 3170 sizeof(struct sighand_struct), 0, 3171 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_TYPESAFE_BY_RCU| 3172 SLAB_ACCOUNT, sighand_ctor); 3173 signal_cachep = kmem_cache_create("signal_cache", 3174 sizeof(struct signal_struct), 0, 3175 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, 3176 NULL); 3177 exec_state_init(); 3178 files_cachep = kmem_cache_create("files_cache", 3179 sizeof(struct files_struct), 0, 3180 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, 3181 NULL); 3182 fs_cachep = kmem_cache_create("fs_cache", 3183 sizeof(struct fs_struct), 0, 3184 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, 3185 NULL); 3186 mmap_init(); 3187 nsproxy_cache_init(); 3188 } 3189 3190 /* 3191 * Check constraints on flags passed to the unshare system call. 3192 */ 3193 static int check_unshare_flags(unsigned long unshare_flags) 3194 { 3195 if (unshare_flags & ~(CLONE_THREAD|CLONE_FS|CLONE_SIGHAND| 3196 CLONE_VM|CLONE_FILES|CLONE_SYSVSEM| 3197 CLONE_NS_ALL | UNSHARE_EMPTY_MNTNS)) 3198 return -EINVAL; 3199 /* 3200 * Not implemented, but pretend it works if there is nothing 3201 * to unshare. Note that unsharing the address space or the 3202 * signal handlers also need to unshare the signal queues (aka 3203 * CLONE_THREAD). 3204 */ 3205 if (unshare_flags & (CLONE_THREAD | CLONE_SIGHAND | CLONE_VM)) { 3206 if (!thread_group_empty(current)) 3207 return -EINVAL; 3208 } 3209 if (unshare_flags & (CLONE_SIGHAND | CLONE_VM)) { 3210 if (refcount_read(¤t->sighand->count) > 1) 3211 return -EINVAL; 3212 } 3213 if (unshare_flags & CLONE_VM) { 3214 if (!current_is_single_threaded()) 3215 return -EINVAL; 3216 } 3217 3218 return 0; 3219 } 3220 3221 /* 3222 * Unshare the filesystem structure if it is being shared 3223 */ 3224 static int unshare_fs(unsigned long unshare_flags, struct fs_struct **new_fsp) 3225 { 3226 struct fs_struct *fs = current->fs; 3227 3228 if (!(unshare_flags & CLONE_FS) || !fs) 3229 return 0; 3230 3231 /* don't need lock here; in the worst case we'll do useless copy */ 3232 if (!(unshare_flags & CLONE_NEWNS) && fs->users == 1) 3233 return 0; 3234 3235 *new_fsp = copy_fs_struct(fs); 3236 if (!*new_fsp) 3237 return -ENOMEM; 3238 3239 return 0; 3240 } 3241 3242 /* 3243 * unshare allows a process to 'unshare' part of the process 3244 * context which was originally shared using clone. copy_* 3245 * functions used by kernel_clone() cannot be used here directly 3246 * because they modify an inactive task_struct that is being 3247 * constructed. Here we are modifying the current, active, 3248 * task_struct. 3249 */ 3250 int ksys_unshare(unsigned long unshare_flags) 3251 { 3252 struct fs_struct *new_fs = NULL; 3253 struct files_struct *new_fd = NULL; 3254 struct cred *new_cred = NULL; 3255 struct nsproxy *new_nsproxy = NULL; 3256 int do_sysvsem = 0; 3257 int err; 3258 3259 /* 3260 * If unsharing a user namespace must also unshare the thread group 3261 * and unshare the filesystem root and working directories. 3262 */ 3263 if (unshare_flags & CLONE_NEWUSER) 3264 unshare_flags |= CLONE_THREAD | CLONE_FS; 3265 /* 3266 * If unsharing vm, must also unshare signal handlers. 3267 */ 3268 if (unshare_flags & CLONE_VM) 3269 unshare_flags |= CLONE_SIGHAND; 3270 /* 3271 * If unsharing a signal handlers, must also unshare the signal queues. 3272 */ 3273 if (unshare_flags & CLONE_SIGHAND) 3274 unshare_flags |= CLONE_THREAD; 3275 /* 3276 * If unsharing namespace, must also unshare filesystem information. 3277 */ 3278 if (unshare_flags & UNSHARE_EMPTY_MNTNS) 3279 unshare_flags |= CLONE_NEWNS; 3280 if (unshare_flags & CLONE_NEWNS) 3281 unshare_flags |= CLONE_FS; 3282 3283 /* No unsharing with overriden fs state */ 3284 VFS_WARN_ON_ONCE(unshare_flags & (CLONE_NEWNS | CLONE_FS) && 3285 current->fs != current->real_fs); 3286 3287 err = check_unshare_flags(unshare_flags); 3288 if (err) 3289 goto bad_unshare_out; 3290 /* 3291 * CLONE_NEWIPC must also detach from the undolist: after switching 3292 * to a new ipc namespace, the semaphore arrays from the old 3293 * namespace are unreachable. 3294 */ 3295 if (unshare_flags & (CLONE_NEWIPC|CLONE_SYSVSEM)) 3296 do_sysvsem = 1; 3297 err = unshare_fs(unshare_flags, &new_fs); 3298 if (err) 3299 goto bad_unshare_out; 3300 err = unshare_fd(unshare_flags, &new_fd); 3301 if (err) 3302 goto bad_unshare_cleanup_fs; 3303 err = unshare_userns(unshare_flags, &new_cred); 3304 if (err) 3305 goto bad_unshare_cleanup_fd; 3306 err = unshare_nsproxy_namespaces(unshare_flags, &new_nsproxy, 3307 new_cred, new_fs); 3308 if (err) 3309 goto bad_unshare_cleanup_cred; 3310 if (new_cred) { 3311 err = set_cred_ucounts(new_cred); 3312 if (err) 3313 goto bad_unshare_cleanup_nsproxy; 3314 } 3315 3316 if (new_fs || new_fd || do_sysvsem || new_cred || new_nsproxy) { 3317 if (do_sysvsem) { 3318 /* 3319 * CLONE_SYSVSEM is equivalent to sys_exit(). 3320 */ 3321 exit_sem(current); 3322 } 3323 if (unshare_flags & CLONE_NEWIPC) { 3324 /* Orphan segments in old ns (see sem above). */ 3325 exit_shm(current); 3326 shm_init_task(current); 3327 } 3328 3329 if (new_fs) { 3330 new_fs = switch_fs_struct(new_fs); 3331 if (new_fs) 3332 free_fs_struct(no_free_ptr(new_fs)); 3333 } 3334 3335 if (new_nsproxy) { 3336 switch_task_namespaces(current, new_nsproxy); 3337 new_nsproxy = NULL; 3338 } 3339 3340 if (new_fd) 3341 switch_files_struct(current, no_free_ptr(new_fd)); 3342 3343 if (new_cred) { 3344 /* Install the new user namespace */ 3345 commit_creds(new_cred); 3346 new_cred = NULL; 3347 } 3348 } 3349 3350 perf_event_namespaces(current); 3351 3352 bad_unshare_cleanup_nsproxy: 3353 /* never installed, so no active references to drop */ 3354 if (new_nsproxy) 3355 nsproxy_free(new_nsproxy); 3356 bad_unshare_cleanup_cred: 3357 if (new_cred) 3358 put_cred(new_cred); 3359 bad_unshare_cleanup_fd: 3360 if (new_fd) 3361 put_files_struct(new_fd); 3362 bad_unshare_cleanup_fs: 3363 if (new_fs) 3364 free_fs_struct(new_fs); 3365 3366 bad_unshare_out: 3367 return err; 3368 } 3369 3370 SYSCALL_DEFINE1(unshare, unsigned long, unshare_flags) 3371 { 3372 return ksys_unshare(unshare_flags); 3373 } 3374 3375 static int sysctl_max_threads(const struct ctl_table *table, int write, 3376 void *buffer, size_t *lenp, loff_t *ppos) 3377 { 3378 struct ctl_table t; 3379 int ret; 3380 int threads = max_threads; 3381 int min = 1; 3382 int max = MAX_THREADS; 3383 3384 t = *table; 3385 t.data = &threads; 3386 t.extra1 = &min; 3387 t.extra2 = &max; 3388 3389 ret = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); 3390 if (ret || !write) 3391 return ret; 3392 3393 max_threads = threads; 3394 3395 return 0; 3396 } 3397 3398 static const struct ctl_table fork_sysctl_table[] = { 3399 { 3400 .procname = "threads-max", 3401 .data = NULL, 3402 .maxlen = sizeof(int), 3403 .mode = 0644, 3404 .proc_handler = sysctl_max_threads, 3405 }, 3406 }; 3407 3408 static int __init init_fork_sysctl(void) 3409 { 3410 register_sysctl_init("kernel", fork_sysctl_table); 3411 return 0; 3412 } 3413 3414 subsys_initcall(init_fork_sysctl); 3415