1 /* 2 * linux/kernel/fork.c 3 * 4 * Copyright (C) 1991, 1992 Linus Torvalds 5 */ 6 7 /* 8 * 'fork.c' contains the help-routines for the 'fork' system call 9 * (see also entry.S and others). 10 * Fork is rather simple, once you get the hang of it, but the memory 11 * management can be a bitch. See 'mm/memory.c': 'copy_page_range()' 12 */ 13 14 #include <linux/slab.h> 15 #include <linux/init.h> 16 #include <linux/unistd.h> 17 #include <linux/module.h> 18 #include <linux/vmalloc.h> 19 #include <linux/completion.h> 20 #include <linux/personality.h> 21 #include <linux/mempolicy.h> 22 #include <linux/sem.h> 23 #include <linux/file.h> 24 #include <linux/fdtable.h> 25 #include <linux/iocontext.h> 26 #include <linux/key.h> 27 #include <linux/binfmts.h> 28 #include <linux/mman.h> 29 #include <linux/mmu_notifier.h> 30 #include <linux/fs.h> 31 #include <linux/mm.h> 32 #include <linux/vmacache.h> 33 #include <linux/nsproxy.h> 34 #include <linux/capability.h> 35 #include <linux/cpu.h> 36 #include <linux/cgroup.h> 37 #include <linux/security.h> 38 #include <linux/hugetlb.h> 39 #include <linux/seccomp.h> 40 #include <linux/swap.h> 41 #include <linux/syscalls.h> 42 #include <linux/jiffies.h> 43 #include <linux/futex.h> 44 #include <linux/compat.h> 45 #include <linux/kthread.h> 46 #include <linux/task_io_accounting_ops.h> 47 #include <linux/rcupdate.h> 48 #include <linux/ptrace.h> 49 #include <linux/mount.h> 50 #include <linux/audit.h> 51 #include <linux/memcontrol.h> 52 #include <linux/ftrace.h> 53 #include <linux/proc_fs.h> 54 #include <linux/profile.h> 55 #include <linux/rmap.h> 56 #include <linux/ksm.h> 57 #include <linux/acct.h> 58 #include <linux/tsacct_kern.h> 59 #include <linux/cn_proc.h> 60 #include <linux/freezer.h> 61 #include <linux/delayacct.h> 62 #include <linux/taskstats_kern.h> 63 #include <linux/random.h> 64 #include <linux/tty.h> 65 #include <linux/blkdev.h> 66 #include <linux/fs_struct.h> 67 #include <linux/magic.h> 68 #include <linux/perf_event.h> 69 #include <linux/posix-timers.h> 70 #include <linux/user-return-notifier.h> 71 #include <linux/oom.h> 72 #include <linux/khugepaged.h> 73 #include <linux/signalfd.h> 74 #include <linux/uprobes.h> 75 #include <linux/aio.h> 76 #include <linux/compiler.h> 77 78 #include <asm/pgtable.h> 79 #include <asm/pgalloc.h> 80 #include <asm/uaccess.h> 81 #include <asm/mmu_context.h> 82 #include <asm/cacheflush.h> 83 #include <asm/tlbflush.h> 84 85 #include <trace/events/sched.h> 86 87 #define CREATE_TRACE_POINTS 88 #include <trace/events/task.h> 89 90 /* 91 * Protected counters by write_lock_irq(&tasklist_lock) 92 */ 93 unsigned long total_forks; /* Handle normal Linux uptimes. */ 94 int nr_threads; /* The idle threads do not count.. */ 95 96 int max_threads; /* tunable limit on nr_threads */ 97 98 DEFINE_PER_CPU(unsigned long, process_counts) = 0; 99 100 __cacheline_aligned DEFINE_RWLOCK(tasklist_lock); /* outer */ 101 102 #ifdef CONFIG_PROVE_RCU 103 int lockdep_tasklist_lock_is_held(void) 104 { 105 return lockdep_is_held(&tasklist_lock); 106 } 107 EXPORT_SYMBOL_GPL(lockdep_tasklist_lock_is_held); 108 #endif /* #ifdef CONFIG_PROVE_RCU */ 109 110 int nr_processes(void) 111 { 112 int cpu; 113 int total = 0; 114 115 for_each_possible_cpu(cpu) 116 total += per_cpu(process_counts, cpu); 117 118 return total; 119 } 120 121 void __weak arch_release_task_struct(struct task_struct *tsk) 122 { 123 } 124 125 #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR 126 static struct kmem_cache *task_struct_cachep; 127 128 static inline struct task_struct *alloc_task_struct_node(int node) 129 { 130 return kmem_cache_alloc_node(task_struct_cachep, GFP_KERNEL, node); 131 } 132 133 static inline void free_task_struct(struct task_struct *tsk) 134 { 135 kmem_cache_free(task_struct_cachep, tsk); 136 } 137 #endif 138 139 void __weak arch_release_thread_info(struct thread_info *ti) 140 { 141 } 142 143 #ifndef CONFIG_ARCH_THREAD_INFO_ALLOCATOR 144 145 /* 146 * Allocate pages if THREAD_SIZE is >= PAGE_SIZE, otherwise use a 147 * kmemcache based allocator. 148 */ 149 # if THREAD_SIZE >= PAGE_SIZE 150 static struct thread_info *alloc_thread_info_node(struct task_struct *tsk, 151 int node) 152 { 153 struct page *page = alloc_kmem_pages_node(node, THREADINFO_GFP, 154 THREAD_SIZE_ORDER); 155 156 return page ? page_address(page) : NULL; 157 } 158 159 static inline void free_thread_info(struct thread_info *ti) 160 { 161 free_kmem_pages((unsigned long)ti, THREAD_SIZE_ORDER); 162 } 163 # else 164 static struct kmem_cache *thread_info_cache; 165 166 static struct thread_info *alloc_thread_info_node(struct task_struct *tsk, 167 int node) 168 { 169 return kmem_cache_alloc_node(thread_info_cache, THREADINFO_GFP, node); 170 } 171 172 static void free_thread_info(struct thread_info *ti) 173 { 174 kmem_cache_free(thread_info_cache, ti); 175 } 176 177 void thread_info_cache_init(void) 178 { 179 thread_info_cache = kmem_cache_create("thread_info", THREAD_SIZE, 180 THREAD_SIZE, 0, NULL); 181 BUG_ON(thread_info_cache == NULL); 182 } 183 # endif 184 #endif 185 186 /* SLAB cache for signal_struct structures (tsk->signal) */ 187 static struct kmem_cache *signal_cachep; 188 189 /* SLAB cache for sighand_struct structures (tsk->sighand) */ 190 struct kmem_cache *sighand_cachep; 191 192 /* SLAB cache for files_struct structures (tsk->files) */ 193 struct kmem_cache *files_cachep; 194 195 /* SLAB cache for fs_struct structures (tsk->fs) */ 196 struct kmem_cache *fs_cachep; 197 198 /* SLAB cache for vm_area_struct structures */ 199 struct kmem_cache *vm_area_cachep; 200 201 /* SLAB cache for mm_struct structures (tsk->mm) */ 202 static struct kmem_cache *mm_cachep; 203 204 static void account_kernel_stack(struct thread_info *ti, int account) 205 { 206 struct zone *zone = page_zone(virt_to_page(ti)); 207 208 mod_zone_page_state(zone, NR_KERNEL_STACK, account); 209 } 210 211 void free_task(struct task_struct *tsk) 212 { 213 account_kernel_stack(tsk->stack, -1); 214 arch_release_thread_info(tsk->stack); 215 free_thread_info(tsk->stack); 216 rt_mutex_debug_task_free(tsk); 217 ftrace_graph_exit_task(tsk); 218 put_seccomp_filter(tsk); 219 arch_release_task_struct(tsk); 220 free_task_struct(tsk); 221 } 222 EXPORT_SYMBOL(free_task); 223 224 static inline void free_signal_struct(struct signal_struct *sig) 225 { 226 taskstats_tgid_free(sig); 227 sched_autogroup_exit(sig); 228 kmem_cache_free(signal_cachep, sig); 229 } 230 231 static inline void put_signal_struct(struct signal_struct *sig) 232 { 233 if (atomic_dec_and_test(&sig->sigcnt)) 234 free_signal_struct(sig); 235 } 236 237 void __put_task_struct(struct task_struct *tsk) 238 { 239 WARN_ON(!tsk->exit_state); 240 WARN_ON(atomic_read(&tsk->usage)); 241 WARN_ON(tsk == current); 242 243 task_numa_free(tsk); 244 security_task_free(tsk); 245 exit_creds(tsk); 246 delayacct_tsk_free(tsk); 247 put_signal_struct(tsk->signal); 248 249 if (!profile_handoff_task(tsk)) 250 free_task(tsk); 251 } 252 EXPORT_SYMBOL_GPL(__put_task_struct); 253 254 void __init __weak arch_task_cache_init(void) { } 255 256 void __init fork_init(unsigned long mempages) 257 { 258 #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR 259 #ifndef ARCH_MIN_TASKALIGN 260 #define ARCH_MIN_TASKALIGN L1_CACHE_BYTES 261 #endif 262 /* create a slab on which task_structs can be allocated */ 263 task_struct_cachep = 264 kmem_cache_create("task_struct", sizeof(struct task_struct), 265 ARCH_MIN_TASKALIGN, SLAB_PANIC | SLAB_NOTRACK, NULL); 266 #endif 267 268 /* do the arch specific task caches init */ 269 arch_task_cache_init(); 270 271 /* 272 * The default maximum number of threads is set to a safe 273 * value: the thread structures can take up at most half 274 * of memory. 275 */ 276 max_threads = mempages / (8 * THREAD_SIZE / PAGE_SIZE); 277 278 /* 279 * we need to allow at least 20 threads to boot a system 280 */ 281 if (max_threads < 20) 282 max_threads = 20; 283 284 init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2; 285 init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2; 286 init_task.signal->rlim[RLIMIT_SIGPENDING] = 287 init_task.signal->rlim[RLIMIT_NPROC]; 288 } 289 290 int __weak arch_dup_task_struct(struct task_struct *dst, 291 struct task_struct *src) 292 { 293 *dst = *src; 294 return 0; 295 } 296 297 static struct task_struct *dup_task_struct(struct task_struct *orig) 298 { 299 struct task_struct *tsk; 300 struct thread_info *ti; 301 unsigned long *stackend; 302 int node = tsk_fork_get_node(orig); 303 int err; 304 305 tsk = alloc_task_struct_node(node); 306 if (!tsk) 307 return NULL; 308 309 ti = alloc_thread_info_node(tsk, node); 310 if (!ti) 311 goto free_tsk; 312 313 err = arch_dup_task_struct(tsk, orig); 314 if (err) 315 goto free_ti; 316 317 tsk->stack = ti; 318 #ifdef CONFIG_SECCOMP 319 /* 320 * We must handle setting up seccomp filters once we're under 321 * the sighand lock in case orig has changed between now and 322 * then. Until then, filter must be NULL to avoid messing up 323 * the usage counts on the error path calling free_task. 324 */ 325 tsk->seccomp.filter = NULL; 326 #endif 327 328 setup_thread_stack(tsk, orig); 329 clear_user_return_notifier(tsk); 330 clear_tsk_need_resched(tsk); 331 stackend = end_of_stack(tsk); 332 *stackend = STACK_END_MAGIC; /* for overflow detection */ 333 334 #ifdef CONFIG_CC_STACKPROTECTOR 335 tsk->stack_canary = get_random_int(); 336 #endif 337 338 /* 339 * One for us, one for whoever does the "release_task()" (usually 340 * parent) 341 */ 342 atomic_set(&tsk->usage, 2); 343 #ifdef CONFIG_BLK_DEV_IO_TRACE 344 tsk->btrace_seq = 0; 345 #endif 346 tsk->splice_pipe = NULL; 347 tsk->task_frag.page = NULL; 348 349 account_kernel_stack(ti, 1); 350 351 return tsk; 352 353 free_ti: 354 free_thread_info(ti); 355 free_tsk: 356 free_task_struct(tsk); 357 return NULL; 358 } 359 360 #ifdef CONFIG_MMU 361 static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) 362 { 363 struct vm_area_struct *mpnt, *tmp, *prev, **pprev; 364 struct rb_node **rb_link, *rb_parent; 365 int retval; 366 unsigned long charge; 367 368 uprobe_start_dup_mmap(); 369 down_write(&oldmm->mmap_sem); 370 flush_cache_dup_mm(oldmm); 371 uprobe_dup_mmap(oldmm, mm); 372 /* 373 * Not linked in yet - no deadlock potential: 374 */ 375 down_write_nested(&mm->mmap_sem, SINGLE_DEPTH_NESTING); 376 377 mm->total_vm = oldmm->total_vm; 378 mm->shared_vm = oldmm->shared_vm; 379 mm->exec_vm = oldmm->exec_vm; 380 mm->stack_vm = oldmm->stack_vm; 381 382 rb_link = &mm->mm_rb.rb_node; 383 rb_parent = NULL; 384 pprev = &mm->mmap; 385 retval = ksm_fork(mm, oldmm); 386 if (retval) 387 goto out; 388 retval = khugepaged_fork(mm, oldmm); 389 if (retval) 390 goto out; 391 392 prev = NULL; 393 for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) { 394 struct file *file; 395 396 if (mpnt->vm_flags & VM_DONTCOPY) { 397 vm_stat_account(mm, mpnt->vm_flags, mpnt->vm_file, 398 -vma_pages(mpnt)); 399 continue; 400 } 401 charge = 0; 402 if (mpnt->vm_flags & VM_ACCOUNT) { 403 unsigned long len = vma_pages(mpnt); 404 405 if (security_vm_enough_memory_mm(oldmm, len)) /* sic */ 406 goto fail_nomem; 407 charge = len; 408 } 409 tmp = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL); 410 if (!tmp) 411 goto fail_nomem; 412 *tmp = *mpnt; 413 INIT_LIST_HEAD(&tmp->anon_vma_chain); 414 retval = vma_dup_policy(mpnt, tmp); 415 if (retval) 416 goto fail_nomem_policy; 417 tmp->vm_mm = mm; 418 if (anon_vma_fork(tmp, mpnt)) 419 goto fail_nomem_anon_vma_fork; 420 tmp->vm_flags &= ~VM_LOCKED; 421 tmp->vm_next = tmp->vm_prev = NULL; 422 file = tmp->vm_file; 423 if (file) { 424 struct inode *inode = file_inode(file); 425 struct address_space *mapping = file->f_mapping; 426 427 get_file(file); 428 if (tmp->vm_flags & VM_DENYWRITE) 429 atomic_dec(&inode->i_writecount); 430 mutex_lock(&mapping->i_mmap_mutex); 431 if (tmp->vm_flags & VM_SHARED) 432 atomic_inc(&mapping->i_mmap_writable); 433 flush_dcache_mmap_lock(mapping); 434 /* insert tmp into the share list, just after mpnt */ 435 if (unlikely(tmp->vm_flags & VM_NONLINEAR)) 436 vma_nonlinear_insert(tmp, 437 &mapping->i_mmap_nonlinear); 438 else 439 vma_interval_tree_insert_after(tmp, mpnt, 440 &mapping->i_mmap); 441 flush_dcache_mmap_unlock(mapping); 442 mutex_unlock(&mapping->i_mmap_mutex); 443 } 444 445 /* 446 * Clear hugetlb-related page reserves for children. This only 447 * affects MAP_PRIVATE mappings. Faults generated by the child 448 * are not guaranteed to succeed, even if read-only 449 */ 450 if (is_vm_hugetlb_page(tmp)) 451 reset_vma_resv_huge_pages(tmp); 452 453 /* 454 * Link in the new vma and copy the page table entries. 455 */ 456 *pprev = tmp; 457 pprev = &tmp->vm_next; 458 tmp->vm_prev = prev; 459 prev = tmp; 460 461 __vma_link_rb(mm, tmp, rb_link, rb_parent); 462 rb_link = &tmp->vm_rb.rb_right; 463 rb_parent = &tmp->vm_rb; 464 465 mm->map_count++; 466 retval = copy_page_range(mm, oldmm, mpnt); 467 468 if (tmp->vm_ops && tmp->vm_ops->open) 469 tmp->vm_ops->open(tmp); 470 471 if (retval) 472 goto out; 473 } 474 /* a new mm has just been created */ 475 arch_dup_mmap(oldmm, mm); 476 retval = 0; 477 out: 478 up_write(&mm->mmap_sem); 479 flush_tlb_mm(oldmm); 480 up_write(&oldmm->mmap_sem); 481 uprobe_end_dup_mmap(); 482 return retval; 483 fail_nomem_anon_vma_fork: 484 mpol_put(vma_policy(tmp)); 485 fail_nomem_policy: 486 kmem_cache_free(vm_area_cachep, tmp); 487 fail_nomem: 488 retval = -ENOMEM; 489 vm_unacct_memory(charge); 490 goto out; 491 } 492 493 static inline int mm_alloc_pgd(struct mm_struct *mm) 494 { 495 mm->pgd = pgd_alloc(mm); 496 if (unlikely(!mm->pgd)) 497 return -ENOMEM; 498 return 0; 499 } 500 501 static inline void mm_free_pgd(struct mm_struct *mm) 502 { 503 pgd_free(mm, mm->pgd); 504 } 505 #else 506 #define dup_mmap(mm, oldmm) (0) 507 #define mm_alloc_pgd(mm) (0) 508 #define mm_free_pgd(mm) 509 #endif /* CONFIG_MMU */ 510 511 __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock); 512 513 #define allocate_mm() (kmem_cache_alloc(mm_cachep, GFP_KERNEL)) 514 #define free_mm(mm) (kmem_cache_free(mm_cachep, (mm))) 515 516 static unsigned long default_dump_filter = MMF_DUMP_FILTER_DEFAULT; 517 518 static int __init coredump_filter_setup(char *s) 519 { 520 default_dump_filter = 521 (simple_strtoul(s, NULL, 0) << MMF_DUMP_FILTER_SHIFT) & 522 MMF_DUMP_FILTER_MASK; 523 return 1; 524 } 525 526 __setup("coredump_filter=", coredump_filter_setup); 527 528 #include <linux/init_task.h> 529 530 static void mm_init_aio(struct mm_struct *mm) 531 { 532 #ifdef CONFIG_AIO 533 spin_lock_init(&mm->ioctx_lock); 534 mm->ioctx_table = NULL; 535 #endif 536 } 537 538 static void mm_init_owner(struct mm_struct *mm, struct task_struct *p) 539 { 540 #ifdef CONFIG_MEMCG 541 mm->owner = p; 542 #endif 543 } 544 545 static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p) 546 { 547 mm->mmap = NULL; 548 mm->mm_rb = RB_ROOT; 549 mm->vmacache_seqnum = 0; 550 atomic_set(&mm->mm_users, 1); 551 atomic_set(&mm->mm_count, 1); 552 init_rwsem(&mm->mmap_sem); 553 INIT_LIST_HEAD(&mm->mmlist); 554 mm->core_state = NULL; 555 atomic_long_set(&mm->nr_ptes, 0); 556 mm->map_count = 0; 557 mm->locked_vm = 0; 558 mm->pinned_vm = 0; 559 memset(&mm->rss_stat, 0, sizeof(mm->rss_stat)); 560 spin_lock_init(&mm->page_table_lock); 561 mm_init_cpumask(mm); 562 mm_init_aio(mm); 563 mm_init_owner(mm, p); 564 mmu_notifier_mm_init(mm); 565 clear_tlb_flush_pending(mm); 566 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS 567 mm->pmd_huge_pte = NULL; 568 #endif 569 570 if (current->mm) { 571 mm->flags = current->mm->flags & MMF_INIT_MASK; 572 mm->def_flags = current->mm->def_flags & VM_INIT_DEF_MASK; 573 } else { 574 mm->flags = default_dump_filter; 575 mm->def_flags = 0; 576 } 577 578 if (mm_alloc_pgd(mm)) 579 goto fail_nopgd; 580 581 if (init_new_context(p, mm)) 582 goto fail_nocontext; 583 584 return mm; 585 586 fail_nocontext: 587 mm_free_pgd(mm); 588 fail_nopgd: 589 free_mm(mm); 590 return NULL; 591 } 592 593 static void check_mm(struct mm_struct *mm) 594 { 595 int i; 596 597 for (i = 0; i < NR_MM_COUNTERS; i++) { 598 long x = atomic_long_read(&mm->rss_stat.count[i]); 599 600 if (unlikely(x)) 601 printk(KERN_ALERT "BUG: Bad rss-counter state " 602 "mm:%p idx:%d val:%ld\n", mm, i, x); 603 } 604 605 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS 606 VM_BUG_ON(mm->pmd_huge_pte); 607 #endif 608 } 609 610 /* 611 * Allocate and initialize an mm_struct. 612 */ 613 struct mm_struct *mm_alloc(void) 614 { 615 struct mm_struct *mm; 616 617 mm = allocate_mm(); 618 if (!mm) 619 return NULL; 620 621 memset(mm, 0, sizeof(*mm)); 622 return mm_init(mm, current); 623 } 624 625 /* 626 * Called when the last reference to the mm 627 * is dropped: either by a lazy thread or by 628 * mmput. Free the page directory and the mm. 629 */ 630 void __mmdrop(struct mm_struct *mm) 631 { 632 BUG_ON(mm == &init_mm); 633 mm_free_pgd(mm); 634 destroy_context(mm); 635 mmu_notifier_mm_destroy(mm); 636 check_mm(mm); 637 free_mm(mm); 638 } 639 EXPORT_SYMBOL_GPL(__mmdrop); 640 641 /* 642 * Decrement the use count and release all resources for an mm. 643 */ 644 void mmput(struct mm_struct *mm) 645 { 646 might_sleep(); 647 648 if (atomic_dec_and_test(&mm->mm_users)) { 649 uprobe_clear_state(mm); 650 exit_aio(mm); 651 ksm_exit(mm); 652 khugepaged_exit(mm); /* must run before exit_mmap */ 653 exit_mmap(mm); 654 set_mm_exe_file(mm, NULL); 655 if (!list_empty(&mm->mmlist)) { 656 spin_lock(&mmlist_lock); 657 list_del(&mm->mmlist); 658 spin_unlock(&mmlist_lock); 659 } 660 if (mm->binfmt) 661 module_put(mm->binfmt->module); 662 mmdrop(mm); 663 } 664 } 665 EXPORT_SYMBOL_GPL(mmput); 666 667 void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file) 668 { 669 if (new_exe_file) 670 get_file(new_exe_file); 671 if (mm->exe_file) 672 fput(mm->exe_file); 673 mm->exe_file = new_exe_file; 674 } 675 676 struct file *get_mm_exe_file(struct mm_struct *mm) 677 { 678 struct file *exe_file; 679 680 /* We need mmap_sem to protect against races with removal of exe_file */ 681 down_read(&mm->mmap_sem); 682 exe_file = mm->exe_file; 683 if (exe_file) 684 get_file(exe_file); 685 up_read(&mm->mmap_sem); 686 return exe_file; 687 } 688 689 static void dup_mm_exe_file(struct mm_struct *oldmm, struct mm_struct *newmm) 690 { 691 /* It's safe to write the exe_file pointer without exe_file_lock because 692 * this is called during fork when the task is not yet in /proc */ 693 newmm->exe_file = get_mm_exe_file(oldmm); 694 } 695 696 /** 697 * get_task_mm - acquire a reference to the task's mm 698 * 699 * Returns %NULL if the task has no mm. Checks PF_KTHREAD (meaning 700 * this kernel workthread has transiently adopted a user mm with use_mm, 701 * to do its AIO) is not set and if so returns a reference to it, after 702 * bumping up the use count. User must release the mm via mmput() 703 * after use. Typically used by /proc and ptrace. 704 */ 705 struct mm_struct *get_task_mm(struct task_struct *task) 706 { 707 struct mm_struct *mm; 708 709 task_lock(task); 710 mm = task->mm; 711 if (mm) { 712 if (task->flags & PF_KTHREAD) 713 mm = NULL; 714 else 715 atomic_inc(&mm->mm_users); 716 } 717 task_unlock(task); 718 return mm; 719 } 720 EXPORT_SYMBOL_GPL(get_task_mm); 721 722 struct mm_struct *mm_access(struct task_struct *task, unsigned int mode) 723 { 724 struct mm_struct *mm; 725 int err; 726 727 err = mutex_lock_killable(&task->signal->cred_guard_mutex); 728 if (err) 729 return ERR_PTR(err); 730 731 mm = get_task_mm(task); 732 if (mm && mm != current->mm && 733 !ptrace_may_access(task, mode)) { 734 mmput(mm); 735 mm = ERR_PTR(-EACCES); 736 } 737 mutex_unlock(&task->signal->cred_guard_mutex); 738 739 return mm; 740 } 741 742 static void complete_vfork_done(struct task_struct *tsk) 743 { 744 struct completion *vfork; 745 746 task_lock(tsk); 747 vfork = tsk->vfork_done; 748 if (likely(vfork)) { 749 tsk->vfork_done = NULL; 750 complete(vfork); 751 } 752 task_unlock(tsk); 753 } 754 755 static int wait_for_vfork_done(struct task_struct *child, 756 struct completion *vfork) 757 { 758 int killed; 759 760 freezer_do_not_count(); 761 killed = wait_for_completion_killable(vfork); 762 freezer_count(); 763 764 if (killed) { 765 task_lock(child); 766 child->vfork_done = NULL; 767 task_unlock(child); 768 } 769 770 put_task_struct(child); 771 return killed; 772 } 773 774 /* Please note the differences between mmput and mm_release. 775 * mmput is called whenever we stop holding onto a mm_struct, 776 * error success whatever. 777 * 778 * mm_release is called after a mm_struct has been removed 779 * from the current process. 780 * 781 * This difference is important for error handling, when we 782 * only half set up a mm_struct for a new process and need to restore 783 * the old one. Because we mmput the new mm_struct before 784 * restoring the old one. . . 785 * Eric Biederman 10 January 1998 786 */ 787 void mm_release(struct task_struct *tsk, struct mm_struct *mm) 788 { 789 /* Get rid of any futexes when releasing the mm */ 790 #ifdef CONFIG_FUTEX 791 if (unlikely(tsk->robust_list)) { 792 exit_robust_list(tsk); 793 tsk->robust_list = NULL; 794 } 795 #ifdef CONFIG_COMPAT 796 if (unlikely(tsk->compat_robust_list)) { 797 compat_exit_robust_list(tsk); 798 tsk->compat_robust_list = NULL; 799 } 800 #endif 801 if (unlikely(!list_empty(&tsk->pi_state_list))) 802 exit_pi_state_list(tsk); 803 #endif 804 805 uprobe_free_utask(tsk); 806 807 /* Get rid of any cached register state */ 808 deactivate_mm(tsk, mm); 809 810 /* 811 * If we're exiting normally, clear a user-space tid field if 812 * requested. We leave this alone when dying by signal, to leave 813 * the value intact in a core dump, and to save the unnecessary 814 * trouble, say, a killed vfork parent shouldn't touch this mm. 815 * Userland only wants this done for a sys_exit. 816 */ 817 if (tsk->clear_child_tid) { 818 if (!(tsk->flags & PF_SIGNALED) && 819 atomic_read(&mm->mm_users) > 1) { 820 /* 821 * We don't check the error code - if userspace has 822 * not set up a proper pointer then tough luck. 823 */ 824 put_user(0, tsk->clear_child_tid); 825 sys_futex(tsk->clear_child_tid, FUTEX_WAKE, 826 1, NULL, NULL, 0); 827 } 828 tsk->clear_child_tid = NULL; 829 } 830 831 /* 832 * All done, finally we can wake up parent and return this mm to him. 833 * Also kthread_stop() uses this completion for synchronization. 834 */ 835 if (tsk->vfork_done) 836 complete_vfork_done(tsk); 837 } 838 839 /* 840 * Allocate a new mm structure and copy contents from the 841 * mm structure of the passed in task structure. 842 */ 843 static struct mm_struct *dup_mm(struct task_struct *tsk) 844 { 845 struct mm_struct *mm, *oldmm = current->mm; 846 int err; 847 848 mm = allocate_mm(); 849 if (!mm) 850 goto fail_nomem; 851 852 memcpy(mm, oldmm, sizeof(*mm)); 853 854 if (!mm_init(mm, tsk)) 855 goto fail_nomem; 856 857 dup_mm_exe_file(oldmm, mm); 858 859 err = dup_mmap(mm, oldmm); 860 if (err) 861 goto free_pt; 862 863 mm->hiwater_rss = get_mm_rss(mm); 864 mm->hiwater_vm = mm->total_vm; 865 866 if (mm->binfmt && !try_module_get(mm->binfmt->module)) 867 goto free_pt; 868 869 return mm; 870 871 free_pt: 872 /* don't put binfmt in mmput, we haven't got module yet */ 873 mm->binfmt = NULL; 874 mmput(mm); 875 876 fail_nomem: 877 return NULL; 878 } 879 880 static int copy_mm(unsigned long clone_flags, struct task_struct *tsk) 881 { 882 struct mm_struct *mm, *oldmm; 883 int retval; 884 885 tsk->min_flt = tsk->maj_flt = 0; 886 tsk->nvcsw = tsk->nivcsw = 0; 887 #ifdef CONFIG_DETECT_HUNG_TASK 888 tsk->last_switch_count = tsk->nvcsw + tsk->nivcsw; 889 #endif 890 891 tsk->mm = NULL; 892 tsk->active_mm = NULL; 893 894 /* 895 * Are we cloning a kernel thread? 896 * 897 * We need to steal a active VM for that.. 898 */ 899 oldmm = current->mm; 900 if (!oldmm) 901 return 0; 902 903 /* initialize the new vmacache entries */ 904 vmacache_flush(tsk); 905 906 if (clone_flags & CLONE_VM) { 907 atomic_inc(&oldmm->mm_users); 908 mm = oldmm; 909 goto good_mm; 910 } 911 912 retval = -ENOMEM; 913 mm = dup_mm(tsk); 914 if (!mm) 915 goto fail_nomem; 916 917 good_mm: 918 tsk->mm = mm; 919 tsk->active_mm = mm; 920 return 0; 921 922 fail_nomem: 923 return retval; 924 } 925 926 static int copy_fs(unsigned long clone_flags, struct task_struct *tsk) 927 { 928 struct fs_struct *fs = current->fs; 929 if (clone_flags & CLONE_FS) { 930 /* tsk->fs is already what we want */ 931 spin_lock(&fs->lock); 932 if (fs->in_exec) { 933 spin_unlock(&fs->lock); 934 return -EAGAIN; 935 } 936 fs->users++; 937 spin_unlock(&fs->lock); 938 return 0; 939 } 940 tsk->fs = copy_fs_struct(fs); 941 if (!tsk->fs) 942 return -ENOMEM; 943 return 0; 944 } 945 946 static int copy_files(unsigned long clone_flags, struct task_struct *tsk) 947 { 948 struct files_struct *oldf, *newf; 949 int error = 0; 950 951 /* 952 * A background process may not have any files ... 953 */ 954 oldf = current->files; 955 if (!oldf) 956 goto out; 957 958 if (clone_flags & CLONE_FILES) { 959 atomic_inc(&oldf->count); 960 goto out; 961 } 962 963 newf = dup_fd(oldf, &error); 964 if (!newf) 965 goto out; 966 967 tsk->files = newf; 968 error = 0; 969 out: 970 return error; 971 } 972 973 static int copy_io(unsigned long clone_flags, struct task_struct *tsk) 974 { 975 #ifdef CONFIG_BLOCK 976 struct io_context *ioc = current->io_context; 977 struct io_context *new_ioc; 978 979 if (!ioc) 980 return 0; 981 /* 982 * Share io context with parent, if CLONE_IO is set 983 */ 984 if (clone_flags & CLONE_IO) { 985 ioc_task_link(ioc); 986 tsk->io_context = ioc; 987 } else if (ioprio_valid(ioc->ioprio)) { 988 new_ioc = get_task_io_context(tsk, GFP_KERNEL, NUMA_NO_NODE); 989 if (unlikely(!new_ioc)) 990 return -ENOMEM; 991 992 new_ioc->ioprio = ioc->ioprio; 993 put_io_context(new_ioc); 994 } 995 #endif 996 return 0; 997 } 998 999 static int copy_sighand(unsigned long clone_flags, struct task_struct *tsk) 1000 { 1001 struct sighand_struct *sig; 1002 1003 if (clone_flags & CLONE_SIGHAND) { 1004 atomic_inc(¤t->sighand->count); 1005 return 0; 1006 } 1007 sig = kmem_cache_alloc(sighand_cachep, GFP_KERNEL); 1008 rcu_assign_pointer(tsk->sighand, sig); 1009 if (!sig) 1010 return -ENOMEM; 1011 atomic_set(&sig->count, 1); 1012 memcpy(sig->action, current->sighand->action, sizeof(sig->action)); 1013 return 0; 1014 } 1015 1016 void __cleanup_sighand(struct sighand_struct *sighand) 1017 { 1018 if (atomic_dec_and_test(&sighand->count)) { 1019 signalfd_cleanup(sighand); 1020 kmem_cache_free(sighand_cachep, sighand); 1021 } 1022 } 1023 1024 1025 /* 1026 * Initialize POSIX timer handling for a thread group. 1027 */ 1028 static void posix_cpu_timers_init_group(struct signal_struct *sig) 1029 { 1030 unsigned long cpu_limit; 1031 1032 /* Thread group counters. */ 1033 thread_group_cputime_init(sig); 1034 1035 cpu_limit = ACCESS_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur); 1036 if (cpu_limit != RLIM_INFINITY) { 1037 sig->cputime_expires.prof_exp = secs_to_cputime(cpu_limit); 1038 sig->cputimer.running = 1; 1039 } 1040 1041 /* The timer lists. */ 1042 INIT_LIST_HEAD(&sig->cpu_timers[0]); 1043 INIT_LIST_HEAD(&sig->cpu_timers[1]); 1044 INIT_LIST_HEAD(&sig->cpu_timers[2]); 1045 } 1046 1047 static int copy_signal(unsigned long clone_flags, struct task_struct *tsk) 1048 { 1049 struct signal_struct *sig; 1050 1051 if (clone_flags & CLONE_THREAD) 1052 return 0; 1053 1054 sig = kmem_cache_zalloc(signal_cachep, GFP_KERNEL); 1055 tsk->signal = sig; 1056 if (!sig) 1057 return -ENOMEM; 1058 1059 sig->nr_threads = 1; 1060 atomic_set(&sig->live, 1); 1061 atomic_set(&sig->sigcnt, 1); 1062 1063 /* list_add(thread_node, thread_head) without INIT_LIST_HEAD() */ 1064 sig->thread_head = (struct list_head)LIST_HEAD_INIT(tsk->thread_node); 1065 tsk->thread_node = (struct list_head)LIST_HEAD_INIT(sig->thread_head); 1066 1067 init_waitqueue_head(&sig->wait_chldexit); 1068 sig->curr_target = tsk; 1069 init_sigpending(&sig->shared_pending); 1070 INIT_LIST_HEAD(&sig->posix_timers); 1071 1072 hrtimer_init(&sig->real_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 1073 sig->real_timer.function = it_real_fn; 1074 1075 task_lock(current->group_leader); 1076 memcpy(sig->rlim, current->signal->rlim, sizeof sig->rlim); 1077 task_unlock(current->group_leader); 1078 1079 posix_cpu_timers_init_group(sig); 1080 1081 tty_audit_fork(sig); 1082 sched_autogroup_fork(sig); 1083 1084 #ifdef CONFIG_CGROUPS 1085 init_rwsem(&sig->group_rwsem); 1086 #endif 1087 1088 sig->oom_score_adj = current->signal->oom_score_adj; 1089 sig->oom_score_adj_min = current->signal->oom_score_adj_min; 1090 1091 sig->has_child_subreaper = current->signal->has_child_subreaper || 1092 current->signal->is_child_subreaper; 1093 1094 mutex_init(&sig->cred_guard_mutex); 1095 1096 return 0; 1097 } 1098 1099 static void copy_seccomp(struct task_struct *p) 1100 { 1101 #ifdef CONFIG_SECCOMP 1102 /* 1103 * Must be called with sighand->lock held, which is common to 1104 * all threads in the group. Holding cred_guard_mutex is not 1105 * needed because this new task is not yet running and cannot 1106 * be racing exec. 1107 */ 1108 assert_spin_locked(¤t->sighand->siglock); 1109 1110 /* Ref-count the new filter user, and assign it. */ 1111 get_seccomp_filter(current); 1112 p->seccomp = current->seccomp; 1113 1114 /* 1115 * Explicitly enable no_new_privs here in case it got set 1116 * between the task_struct being duplicated and holding the 1117 * sighand lock. The seccomp state and nnp must be in sync. 1118 */ 1119 if (task_no_new_privs(current)) 1120 task_set_no_new_privs(p); 1121 1122 /* 1123 * If the parent gained a seccomp mode after copying thread 1124 * flags and between before we held the sighand lock, we have 1125 * to manually enable the seccomp thread flag here. 1126 */ 1127 if (p->seccomp.mode != SECCOMP_MODE_DISABLED) 1128 set_tsk_thread_flag(p, TIF_SECCOMP); 1129 #endif 1130 } 1131 1132 SYSCALL_DEFINE1(set_tid_address, int __user *, tidptr) 1133 { 1134 current->clear_child_tid = tidptr; 1135 1136 return task_pid_vnr(current); 1137 } 1138 1139 static void rt_mutex_init_task(struct task_struct *p) 1140 { 1141 raw_spin_lock_init(&p->pi_lock); 1142 #ifdef CONFIG_RT_MUTEXES 1143 p->pi_waiters = RB_ROOT; 1144 p->pi_waiters_leftmost = NULL; 1145 p->pi_blocked_on = NULL; 1146 #endif 1147 } 1148 1149 /* 1150 * Initialize POSIX timer handling for a single task. 1151 */ 1152 static void posix_cpu_timers_init(struct task_struct *tsk) 1153 { 1154 tsk->cputime_expires.prof_exp = 0; 1155 tsk->cputime_expires.virt_exp = 0; 1156 tsk->cputime_expires.sched_exp = 0; 1157 INIT_LIST_HEAD(&tsk->cpu_timers[0]); 1158 INIT_LIST_HEAD(&tsk->cpu_timers[1]); 1159 INIT_LIST_HEAD(&tsk->cpu_timers[2]); 1160 } 1161 1162 static inline void 1163 init_task_pid(struct task_struct *task, enum pid_type type, struct pid *pid) 1164 { 1165 task->pids[type].pid = pid; 1166 } 1167 1168 /* 1169 * This creates a new process as a copy of the old one, 1170 * but does not actually start it yet. 1171 * 1172 * It copies the registers, and all the appropriate 1173 * parts of the process environment (as per the clone 1174 * flags). The actual kick-off is left to the caller. 1175 */ 1176 static struct task_struct *copy_process(unsigned long clone_flags, 1177 unsigned long stack_start, 1178 unsigned long stack_size, 1179 int __user *child_tidptr, 1180 struct pid *pid, 1181 int trace) 1182 { 1183 int retval; 1184 struct task_struct *p; 1185 1186 if ((clone_flags & (CLONE_NEWNS|CLONE_FS)) == (CLONE_NEWNS|CLONE_FS)) 1187 return ERR_PTR(-EINVAL); 1188 1189 if ((clone_flags & (CLONE_NEWUSER|CLONE_FS)) == (CLONE_NEWUSER|CLONE_FS)) 1190 return ERR_PTR(-EINVAL); 1191 1192 /* 1193 * Thread groups must share signals as well, and detached threads 1194 * can only be started up within the thread group. 1195 */ 1196 if ((clone_flags & CLONE_THREAD) && !(clone_flags & CLONE_SIGHAND)) 1197 return ERR_PTR(-EINVAL); 1198 1199 /* 1200 * Shared signal handlers imply shared VM. By way of the above, 1201 * thread groups also imply shared VM. Blocking this case allows 1202 * for various simplifications in other code. 1203 */ 1204 if ((clone_flags & CLONE_SIGHAND) && !(clone_flags & CLONE_VM)) 1205 return ERR_PTR(-EINVAL); 1206 1207 /* 1208 * Siblings of global init remain as zombies on exit since they are 1209 * not reaped by their parent (swapper). To solve this and to avoid 1210 * multi-rooted process trees, prevent global and container-inits 1211 * from creating siblings. 1212 */ 1213 if ((clone_flags & CLONE_PARENT) && 1214 current->signal->flags & SIGNAL_UNKILLABLE) 1215 return ERR_PTR(-EINVAL); 1216 1217 /* 1218 * If the new process will be in a different pid or user namespace 1219 * do not allow it to share a thread group or signal handlers or 1220 * parent with the forking task. 1221 */ 1222 if (clone_flags & CLONE_SIGHAND) { 1223 if ((clone_flags & (CLONE_NEWUSER | CLONE_NEWPID)) || 1224 (task_active_pid_ns(current) != 1225 current->nsproxy->pid_ns_for_children)) 1226 return ERR_PTR(-EINVAL); 1227 } 1228 1229 retval = security_task_create(clone_flags); 1230 if (retval) 1231 goto fork_out; 1232 1233 retval = -ENOMEM; 1234 p = dup_task_struct(current); 1235 if (!p) 1236 goto fork_out; 1237 1238 ftrace_graph_init_task(p); 1239 1240 rt_mutex_init_task(p); 1241 1242 #ifdef CONFIG_PROVE_LOCKING 1243 DEBUG_LOCKS_WARN_ON(!p->hardirqs_enabled); 1244 DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled); 1245 #endif 1246 retval = -EAGAIN; 1247 if (atomic_read(&p->real_cred->user->processes) >= 1248 task_rlimit(p, RLIMIT_NPROC)) { 1249 if (p->real_cred->user != INIT_USER && 1250 !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN)) 1251 goto bad_fork_free; 1252 } 1253 current->flags &= ~PF_NPROC_EXCEEDED; 1254 1255 retval = copy_creds(p, clone_flags); 1256 if (retval < 0) 1257 goto bad_fork_free; 1258 1259 /* 1260 * If multiple threads are within copy_process(), then this check 1261 * triggers too late. This doesn't hurt, the check is only there 1262 * to stop root fork bombs. 1263 */ 1264 retval = -EAGAIN; 1265 if (nr_threads >= max_threads) 1266 goto bad_fork_cleanup_count; 1267 1268 if (!try_module_get(task_thread_info(p)->exec_domain->module)) 1269 goto bad_fork_cleanup_count; 1270 1271 delayacct_tsk_init(p); /* Must remain after dup_task_struct() */ 1272 p->flags &= ~(PF_SUPERPRIV | PF_WQ_WORKER); 1273 p->flags |= PF_FORKNOEXEC; 1274 INIT_LIST_HEAD(&p->children); 1275 INIT_LIST_HEAD(&p->sibling); 1276 rcu_copy_process(p); 1277 p->vfork_done = NULL; 1278 spin_lock_init(&p->alloc_lock); 1279 1280 init_sigpending(&p->pending); 1281 1282 p->utime = p->stime = p->gtime = 0; 1283 p->utimescaled = p->stimescaled = 0; 1284 #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE 1285 p->prev_cputime.utime = p->prev_cputime.stime = 0; 1286 #endif 1287 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN 1288 seqlock_init(&p->vtime_seqlock); 1289 p->vtime_snap = 0; 1290 p->vtime_snap_whence = VTIME_SLEEPING; 1291 #endif 1292 1293 #if defined(SPLIT_RSS_COUNTING) 1294 memset(&p->rss_stat, 0, sizeof(p->rss_stat)); 1295 #endif 1296 1297 p->default_timer_slack_ns = current->timer_slack_ns; 1298 1299 task_io_accounting_init(&p->ioac); 1300 acct_clear_integrals(p); 1301 1302 posix_cpu_timers_init(p); 1303 1304 p->start_time = ktime_get_ns(); 1305 p->real_start_time = ktime_get_boot_ns(); 1306 p->io_context = NULL; 1307 p->audit_context = NULL; 1308 if (clone_flags & CLONE_THREAD) 1309 threadgroup_change_begin(current); 1310 cgroup_fork(p); 1311 #ifdef CONFIG_NUMA 1312 p->mempolicy = mpol_dup(p->mempolicy); 1313 if (IS_ERR(p->mempolicy)) { 1314 retval = PTR_ERR(p->mempolicy); 1315 p->mempolicy = NULL; 1316 goto bad_fork_cleanup_threadgroup_lock; 1317 } 1318 #endif 1319 #ifdef CONFIG_CPUSETS 1320 p->cpuset_mem_spread_rotor = NUMA_NO_NODE; 1321 p->cpuset_slab_spread_rotor = NUMA_NO_NODE; 1322 seqcount_init(&p->mems_allowed_seq); 1323 #endif 1324 #ifdef CONFIG_TRACE_IRQFLAGS 1325 p->irq_events = 0; 1326 p->hardirqs_enabled = 0; 1327 p->hardirq_enable_ip = 0; 1328 p->hardirq_enable_event = 0; 1329 p->hardirq_disable_ip = _THIS_IP_; 1330 p->hardirq_disable_event = 0; 1331 p->softirqs_enabled = 1; 1332 p->softirq_enable_ip = _THIS_IP_; 1333 p->softirq_enable_event = 0; 1334 p->softirq_disable_ip = 0; 1335 p->softirq_disable_event = 0; 1336 p->hardirq_context = 0; 1337 p->softirq_context = 0; 1338 #endif 1339 #ifdef CONFIG_LOCKDEP 1340 p->lockdep_depth = 0; /* no locks held yet */ 1341 p->curr_chain_key = 0; 1342 p->lockdep_recursion = 0; 1343 #endif 1344 1345 #ifdef CONFIG_DEBUG_MUTEXES 1346 p->blocked_on = NULL; /* not blocked yet */ 1347 #endif 1348 #ifdef CONFIG_BCACHE 1349 p->sequential_io = 0; 1350 p->sequential_io_avg = 0; 1351 #endif 1352 1353 /* Perform scheduler related setup. Assign this task to a CPU. */ 1354 retval = sched_fork(clone_flags, p); 1355 if (retval) 1356 goto bad_fork_cleanup_policy; 1357 1358 retval = perf_event_init_task(p); 1359 if (retval) 1360 goto bad_fork_cleanup_policy; 1361 retval = audit_alloc(p); 1362 if (retval) 1363 goto bad_fork_cleanup_perf; 1364 /* copy all the process information */ 1365 shm_init_task(p); 1366 retval = copy_semundo(clone_flags, p); 1367 if (retval) 1368 goto bad_fork_cleanup_audit; 1369 retval = copy_files(clone_flags, p); 1370 if (retval) 1371 goto bad_fork_cleanup_semundo; 1372 retval = copy_fs(clone_flags, p); 1373 if (retval) 1374 goto bad_fork_cleanup_files; 1375 retval = copy_sighand(clone_flags, p); 1376 if (retval) 1377 goto bad_fork_cleanup_fs; 1378 retval = copy_signal(clone_flags, p); 1379 if (retval) 1380 goto bad_fork_cleanup_sighand; 1381 retval = copy_mm(clone_flags, p); 1382 if (retval) 1383 goto bad_fork_cleanup_signal; 1384 retval = copy_namespaces(clone_flags, p); 1385 if (retval) 1386 goto bad_fork_cleanup_mm; 1387 retval = copy_io(clone_flags, p); 1388 if (retval) 1389 goto bad_fork_cleanup_namespaces; 1390 retval = copy_thread(clone_flags, stack_start, stack_size, p); 1391 if (retval) 1392 goto bad_fork_cleanup_io; 1393 1394 if (pid != &init_struct_pid) { 1395 retval = -ENOMEM; 1396 pid = alloc_pid(p->nsproxy->pid_ns_for_children); 1397 if (!pid) 1398 goto bad_fork_cleanup_io; 1399 } 1400 1401 p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? child_tidptr : NULL; 1402 /* 1403 * Clear TID on mm_release()? 1404 */ 1405 p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? child_tidptr : NULL; 1406 #ifdef CONFIG_BLOCK 1407 p->plug = NULL; 1408 #endif 1409 #ifdef CONFIG_FUTEX 1410 p->robust_list = NULL; 1411 #ifdef CONFIG_COMPAT 1412 p->compat_robust_list = NULL; 1413 #endif 1414 INIT_LIST_HEAD(&p->pi_state_list); 1415 p->pi_state_cache = NULL; 1416 #endif 1417 /* 1418 * sigaltstack should be cleared when sharing the same VM 1419 */ 1420 if ((clone_flags & (CLONE_VM|CLONE_VFORK)) == CLONE_VM) 1421 p->sas_ss_sp = p->sas_ss_size = 0; 1422 1423 /* 1424 * Syscall tracing and stepping should be turned off in the 1425 * child regardless of CLONE_PTRACE. 1426 */ 1427 user_disable_single_step(p); 1428 clear_tsk_thread_flag(p, TIF_SYSCALL_TRACE); 1429 #ifdef TIF_SYSCALL_EMU 1430 clear_tsk_thread_flag(p, TIF_SYSCALL_EMU); 1431 #endif 1432 clear_all_latency_tracing(p); 1433 1434 /* ok, now we should be set up.. */ 1435 p->pid = pid_nr(pid); 1436 if (clone_flags & CLONE_THREAD) { 1437 p->exit_signal = -1; 1438 p->group_leader = current->group_leader; 1439 p->tgid = current->tgid; 1440 } else { 1441 if (clone_flags & CLONE_PARENT) 1442 p->exit_signal = current->group_leader->exit_signal; 1443 else 1444 p->exit_signal = (clone_flags & CSIGNAL); 1445 p->group_leader = p; 1446 p->tgid = p->pid; 1447 } 1448 1449 p->nr_dirtied = 0; 1450 p->nr_dirtied_pause = 128 >> (PAGE_SHIFT - 10); 1451 p->dirty_paused_when = 0; 1452 1453 p->pdeath_signal = 0; 1454 INIT_LIST_HEAD(&p->thread_group); 1455 p->task_works = NULL; 1456 1457 /* 1458 * Make it visible to the rest of the system, but dont wake it up yet. 1459 * Need tasklist lock for parent etc handling! 1460 */ 1461 write_lock_irq(&tasklist_lock); 1462 1463 /* CLONE_PARENT re-uses the old parent */ 1464 if (clone_flags & (CLONE_PARENT|CLONE_THREAD)) { 1465 p->real_parent = current->real_parent; 1466 p->parent_exec_id = current->parent_exec_id; 1467 } else { 1468 p->real_parent = current; 1469 p->parent_exec_id = current->self_exec_id; 1470 } 1471 1472 spin_lock(¤t->sighand->siglock); 1473 1474 /* 1475 * Copy seccomp details explicitly here, in case they were changed 1476 * before holding sighand lock. 1477 */ 1478 copy_seccomp(p); 1479 1480 /* 1481 * Process group and session signals need to be delivered to just the 1482 * parent before the fork or both the parent and the child after the 1483 * fork. Restart if a signal comes in before we add the new process to 1484 * it's process group. 1485 * A fatal signal pending means that current will exit, so the new 1486 * thread can't slip out of an OOM kill (or normal SIGKILL). 1487 */ 1488 recalc_sigpending(); 1489 if (signal_pending(current)) { 1490 spin_unlock(¤t->sighand->siglock); 1491 write_unlock_irq(&tasklist_lock); 1492 retval = -ERESTARTNOINTR; 1493 goto bad_fork_free_pid; 1494 } 1495 1496 if (likely(p->pid)) { 1497 ptrace_init_task(p, (clone_flags & CLONE_PTRACE) || trace); 1498 1499 init_task_pid(p, PIDTYPE_PID, pid); 1500 if (thread_group_leader(p)) { 1501 init_task_pid(p, PIDTYPE_PGID, task_pgrp(current)); 1502 init_task_pid(p, PIDTYPE_SID, task_session(current)); 1503 1504 if (is_child_reaper(pid)) { 1505 ns_of_pid(pid)->child_reaper = p; 1506 p->signal->flags |= SIGNAL_UNKILLABLE; 1507 } 1508 1509 p->signal->leader_pid = pid; 1510 p->signal->tty = tty_kref_get(current->signal->tty); 1511 list_add_tail(&p->sibling, &p->real_parent->children); 1512 list_add_tail_rcu(&p->tasks, &init_task.tasks); 1513 attach_pid(p, PIDTYPE_PGID); 1514 attach_pid(p, PIDTYPE_SID); 1515 __this_cpu_inc(process_counts); 1516 } else { 1517 current->signal->nr_threads++; 1518 atomic_inc(¤t->signal->live); 1519 atomic_inc(¤t->signal->sigcnt); 1520 list_add_tail_rcu(&p->thread_group, 1521 &p->group_leader->thread_group); 1522 list_add_tail_rcu(&p->thread_node, 1523 &p->signal->thread_head); 1524 } 1525 attach_pid(p, PIDTYPE_PID); 1526 nr_threads++; 1527 } 1528 1529 total_forks++; 1530 spin_unlock(¤t->sighand->siglock); 1531 syscall_tracepoint_update(p); 1532 write_unlock_irq(&tasklist_lock); 1533 1534 proc_fork_connector(p); 1535 cgroup_post_fork(p); 1536 if (clone_flags & CLONE_THREAD) 1537 threadgroup_change_end(current); 1538 perf_event_fork(p); 1539 1540 trace_task_newtask(p, clone_flags); 1541 uprobe_copy_process(p, clone_flags); 1542 1543 return p; 1544 1545 bad_fork_free_pid: 1546 if (pid != &init_struct_pid) 1547 free_pid(pid); 1548 bad_fork_cleanup_io: 1549 if (p->io_context) 1550 exit_io_context(p); 1551 bad_fork_cleanup_namespaces: 1552 exit_task_namespaces(p); 1553 bad_fork_cleanup_mm: 1554 if (p->mm) 1555 mmput(p->mm); 1556 bad_fork_cleanup_signal: 1557 if (!(clone_flags & CLONE_THREAD)) 1558 free_signal_struct(p->signal); 1559 bad_fork_cleanup_sighand: 1560 __cleanup_sighand(p->sighand); 1561 bad_fork_cleanup_fs: 1562 exit_fs(p); /* blocking */ 1563 bad_fork_cleanup_files: 1564 exit_files(p); /* blocking */ 1565 bad_fork_cleanup_semundo: 1566 exit_sem(p); 1567 bad_fork_cleanup_audit: 1568 audit_free(p); 1569 bad_fork_cleanup_perf: 1570 perf_event_free_task(p); 1571 bad_fork_cleanup_policy: 1572 #ifdef CONFIG_NUMA 1573 mpol_put(p->mempolicy); 1574 bad_fork_cleanup_threadgroup_lock: 1575 #endif 1576 if (clone_flags & CLONE_THREAD) 1577 threadgroup_change_end(current); 1578 delayacct_tsk_free(p); 1579 module_put(task_thread_info(p)->exec_domain->module); 1580 bad_fork_cleanup_count: 1581 atomic_dec(&p->cred->user->processes); 1582 exit_creds(p); 1583 bad_fork_free: 1584 free_task(p); 1585 fork_out: 1586 return ERR_PTR(retval); 1587 } 1588 1589 static inline void init_idle_pids(struct pid_link *links) 1590 { 1591 enum pid_type type; 1592 1593 for (type = PIDTYPE_PID; type < PIDTYPE_MAX; ++type) { 1594 INIT_HLIST_NODE(&links[type].node); /* not really needed */ 1595 links[type].pid = &init_struct_pid; 1596 } 1597 } 1598 1599 struct task_struct *fork_idle(int cpu) 1600 { 1601 struct task_struct *task; 1602 task = copy_process(CLONE_VM, 0, 0, NULL, &init_struct_pid, 0); 1603 if (!IS_ERR(task)) { 1604 init_idle_pids(task->pids); 1605 init_idle(task, cpu); 1606 } 1607 1608 return task; 1609 } 1610 1611 /* 1612 * Ok, this is the main fork-routine. 1613 * 1614 * It copies the process, and if successful kick-starts 1615 * it and waits for it to finish using the VM if required. 1616 */ 1617 long do_fork(unsigned long clone_flags, 1618 unsigned long stack_start, 1619 unsigned long stack_size, 1620 int __user *parent_tidptr, 1621 int __user *child_tidptr) 1622 { 1623 struct task_struct *p; 1624 int trace = 0; 1625 long nr; 1626 1627 /* 1628 * Determine whether and which event to report to ptracer. When 1629 * called from kernel_thread or CLONE_UNTRACED is explicitly 1630 * requested, no event is reported; otherwise, report if the event 1631 * for the type of forking is enabled. 1632 */ 1633 if (!(clone_flags & CLONE_UNTRACED)) { 1634 if (clone_flags & CLONE_VFORK) 1635 trace = PTRACE_EVENT_VFORK; 1636 else if ((clone_flags & CSIGNAL) != SIGCHLD) 1637 trace = PTRACE_EVENT_CLONE; 1638 else 1639 trace = PTRACE_EVENT_FORK; 1640 1641 if (likely(!ptrace_event_enabled(current, trace))) 1642 trace = 0; 1643 } 1644 1645 p = copy_process(clone_flags, stack_start, stack_size, 1646 child_tidptr, NULL, trace); 1647 /* 1648 * Do this prior waking up the new thread - the thread pointer 1649 * might get invalid after that point, if the thread exits quickly. 1650 */ 1651 if (!IS_ERR(p)) { 1652 struct completion vfork; 1653 struct pid *pid; 1654 1655 trace_sched_process_fork(current, p); 1656 1657 pid = get_task_pid(p, PIDTYPE_PID); 1658 nr = pid_vnr(pid); 1659 1660 if (clone_flags & CLONE_PARENT_SETTID) 1661 put_user(nr, parent_tidptr); 1662 1663 if (clone_flags & CLONE_VFORK) { 1664 p->vfork_done = &vfork; 1665 init_completion(&vfork); 1666 get_task_struct(p); 1667 } 1668 1669 wake_up_new_task(p); 1670 1671 /* forking complete and child started to run, tell ptracer */ 1672 if (unlikely(trace)) 1673 ptrace_event_pid(trace, pid); 1674 1675 if (clone_flags & CLONE_VFORK) { 1676 if (!wait_for_vfork_done(p, &vfork)) 1677 ptrace_event_pid(PTRACE_EVENT_VFORK_DONE, pid); 1678 } 1679 1680 put_pid(pid); 1681 } else { 1682 nr = PTR_ERR(p); 1683 } 1684 return nr; 1685 } 1686 1687 /* 1688 * Create a kernel thread. 1689 */ 1690 pid_t kernel_thread(int (*fn)(void *), void *arg, unsigned long flags) 1691 { 1692 return do_fork(flags|CLONE_VM|CLONE_UNTRACED, (unsigned long)fn, 1693 (unsigned long)arg, NULL, NULL); 1694 } 1695 1696 #ifdef __ARCH_WANT_SYS_FORK 1697 SYSCALL_DEFINE0(fork) 1698 { 1699 #ifdef CONFIG_MMU 1700 return do_fork(SIGCHLD, 0, 0, NULL, NULL); 1701 #else 1702 /* can not support in nommu mode */ 1703 return -EINVAL; 1704 #endif 1705 } 1706 #endif 1707 1708 #ifdef __ARCH_WANT_SYS_VFORK 1709 SYSCALL_DEFINE0(vfork) 1710 { 1711 return do_fork(CLONE_VFORK | CLONE_VM | SIGCHLD, 0, 1712 0, NULL, NULL); 1713 } 1714 #endif 1715 1716 #ifdef __ARCH_WANT_SYS_CLONE 1717 #ifdef CONFIG_CLONE_BACKWARDS 1718 SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp, 1719 int __user *, parent_tidptr, 1720 int, tls_val, 1721 int __user *, child_tidptr) 1722 #elif defined(CONFIG_CLONE_BACKWARDS2) 1723 SYSCALL_DEFINE5(clone, unsigned long, newsp, unsigned long, clone_flags, 1724 int __user *, parent_tidptr, 1725 int __user *, child_tidptr, 1726 int, tls_val) 1727 #elif defined(CONFIG_CLONE_BACKWARDS3) 1728 SYSCALL_DEFINE6(clone, unsigned long, clone_flags, unsigned long, newsp, 1729 int, stack_size, 1730 int __user *, parent_tidptr, 1731 int __user *, child_tidptr, 1732 int, tls_val) 1733 #else 1734 SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp, 1735 int __user *, parent_tidptr, 1736 int __user *, child_tidptr, 1737 int, tls_val) 1738 #endif 1739 { 1740 return do_fork(clone_flags, newsp, 0, parent_tidptr, child_tidptr); 1741 } 1742 #endif 1743 1744 #ifndef ARCH_MIN_MMSTRUCT_ALIGN 1745 #define ARCH_MIN_MMSTRUCT_ALIGN 0 1746 #endif 1747 1748 static void sighand_ctor(void *data) 1749 { 1750 struct sighand_struct *sighand = data; 1751 1752 spin_lock_init(&sighand->siglock); 1753 init_waitqueue_head(&sighand->signalfd_wqh); 1754 } 1755 1756 void __init proc_caches_init(void) 1757 { 1758 sighand_cachep = kmem_cache_create("sighand_cache", 1759 sizeof(struct sighand_struct), 0, 1760 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_DESTROY_BY_RCU| 1761 SLAB_NOTRACK, sighand_ctor); 1762 signal_cachep = kmem_cache_create("signal_cache", 1763 sizeof(struct signal_struct), 0, 1764 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL); 1765 files_cachep = kmem_cache_create("files_cache", 1766 sizeof(struct files_struct), 0, 1767 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL); 1768 fs_cachep = kmem_cache_create("fs_cache", 1769 sizeof(struct fs_struct), 0, 1770 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL); 1771 /* 1772 * FIXME! The "sizeof(struct mm_struct)" currently includes the 1773 * whole struct cpumask for the OFFSTACK case. We could change 1774 * this to *only* allocate as much of it as required by the 1775 * maximum number of CPU's we can ever have. The cpumask_allocation 1776 * is at the end of the structure, exactly for that reason. 1777 */ 1778 mm_cachep = kmem_cache_create("mm_struct", 1779 sizeof(struct mm_struct), ARCH_MIN_MMSTRUCT_ALIGN, 1780 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL); 1781 vm_area_cachep = KMEM_CACHE(vm_area_struct, SLAB_PANIC); 1782 mmap_init(); 1783 nsproxy_cache_init(); 1784 } 1785 1786 /* 1787 * Check constraints on flags passed to the unshare system call. 1788 */ 1789 static int check_unshare_flags(unsigned long unshare_flags) 1790 { 1791 if (unshare_flags & ~(CLONE_THREAD|CLONE_FS|CLONE_NEWNS|CLONE_SIGHAND| 1792 CLONE_VM|CLONE_FILES|CLONE_SYSVSEM| 1793 CLONE_NEWUTS|CLONE_NEWIPC|CLONE_NEWNET| 1794 CLONE_NEWUSER|CLONE_NEWPID)) 1795 return -EINVAL; 1796 /* 1797 * Not implemented, but pretend it works if there is nothing to 1798 * unshare. Note that unsharing CLONE_THREAD or CLONE_SIGHAND 1799 * needs to unshare vm. 1800 */ 1801 if (unshare_flags & (CLONE_THREAD | CLONE_SIGHAND | CLONE_VM)) { 1802 /* FIXME: get_task_mm() increments ->mm_users */ 1803 if (atomic_read(¤t->mm->mm_users) > 1) 1804 return -EINVAL; 1805 } 1806 1807 return 0; 1808 } 1809 1810 /* 1811 * Unshare the filesystem structure if it is being shared 1812 */ 1813 static int unshare_fs(unsigned long unshare_flags, struct fs_struct **new_fsp) 1814 { 1815 struct fs_struct *fs = current->fs; 1816 1817 if (!(unshare_flags & CLONE_FS) || !fs) 1818 return 0; 1819 1820 /* don't need lock here; in the worst case we'll do useless copy */ 1821 if (fs->users == 1) 1822 return 0; 1823 1824 *new_fsp = copy_fs_struct(fs); 1825 if (!*new_fsp) 1826 return -ENOMEM; 1827 1828 return 0; 1829 } 1830 1831 /* 1832 * Unshare file descriptor table if it is being shared 1833 */ 1834 static int unshare_fd(unsigned long unshare_flags, struct files_struct **new_fdp) 1835 { 1836 struct files_struct *fd = current->files; 1837 int error = 0; 1838 1839 if ((unshare_flags & CLONE_FILES) && 1840 (fd && atomic_read(&fd->count) > 1)) { 1841 *new_fdp = dup_fd(fd, &error); 1842 if (!*new_fdp) 1843 return error; 1844 } 1845 1846 return 0; 1847 } 1848 1849 /* 1850 * unshare allows a process to 'unshare' part of the process 1851 * context which was originally shared using clone. copy_* 1852 * functions used by do_fork() cannot be used here directly 1853 * because they modify an inactive task_struct that is being 1854 * constructed. Here we are modifying the current, active, 1855 * task_struct. 1856 */ 1857 SYSCALL_DEFINE1(unshare, unsigned long, unshare_flags) 1858 { 1859 struct fs_struct *fs, *new_fs = NULL; 1860 struct files_struct *fd, *new_fd = NULL; 1861 struct cred *new_cred = NULL; 1862 struct nsproxy *new_nsproxy = NULL; 1863 int do_sysvsem = 0; 1864 int err; 1865 1866 /* 1867 * If unsharing a user namespace must also unshare the thread. 1868 */ 1869 if (unshare_flags & CLONE_NEWUSER) 1870 unshare_flags |= CLONE_THREAD | CLONE_FS; 1871 /* 1872 * If unsharing a thread from a thread group, must also unshare vm. 1873 */ 1874 if (unshare_flags & CLONE_THREAD) 1875 unshare_flags |= CLONE_VM; 1876 /* 1877 * If unsharing vm, must also unshare signal handlers. 1878 */ 1879 if (unshare_flags & CLONE_VM) 1880 unshare_flags |= CLONE_SIGHAND; 1881 /* 1882 * If unsharing namespace, must also unshare filesystem information. 1883 */ 1884 if (unshare_flags & CLONE_NEWNS) 1885 unshare_flags |= CLONE_FS; 1886 1887 err = check_unshare_flags(unshare_flags); 1888 if (err) 1889 goto bad_unshare_out; 1890 /* 1891 * CLONE_NEWIPC must also detach from the undolist: after switching 1892 * to a new ipc namespace, the semaphore arrays from the old 1893 * namespace are unreachable. 1894 */ 1895 if (unshare_flags & (CLONE_NEWIPC|CLONE_SYSVSEM)) 1896 do_sysvsem = 1; 1897 err = unshare_fs(unshare_flags, &new_fs); 1898 if (err) 1899 goto bad_unshare_out; 1900 err = unshare_fd(unshare_flags, &new_fd); 1901 if (err) 1902 goto bad_unshare_cleanup_fs; 1903 err = unshare_userns(unshare_flags, &new_cred); 1904 if (err) 1905 goto bad_unshare_cleanup_fd; 1906 err = unshare_nsproxy_namespaces(unshare_flags, &new_nsproxy, 1907 new_cred, new_fs); 1908 if (err) 1909 goto bad_unshare_cleanup_cred; 1910 1911 if (new_fs || new_fd || do_sysvsem || new_cred || new_nsproxy) { 1912 if (do_sysvsem) { 1913 /* 1914 * CLONE_SYSVSEM is equivalent to sys_exit(). 1915 */ 1916 exit_sem(current); 1917 } 1918 if (unshare_flags & CLONE_NEWIPC) { 1919 /* Orphan segments in old ns (see sem above). */ 1920 exit_shm(current); 1921 shm_init_task(current); 1922 } 1923 1924 if (new_nsproxy) 1925 switch_task_namespaces(current, new_nsproxy); 1926 1927 task_lock(current); 1928 1929 if (new_fs) { 1930 fs = current->fs; 1931 spin_lock(&fs->lock); 1932 current->fs = new_fs; 1933 if (--fs->users) 1934 new_fs = NULL; 1935 else 1936 new_fs = fs; 1937 spin_unlock(&fs->lock); 1938 } 1939 1940 if (new_fd) { 1941 fd = current->files; 1942 current->files = new_fd; 1943 new_fd = fd; 1944 } 1945 1946 task_unlock(current); 1947 1948 if (new_cred) { 1949 /* Install the new user namespace */ 1950 commit_creds(new_cred); 1951 new_cred = NULL; 1952 } 1953 } 1954 1955 bad_unshare_cleanup_cred: 1956 if (new_cred) 1957 put_cred(new_cred); 1958 bad_unshare_cleanup_fd: 1959 if (new_fd) 1960 put_files_struct(new_fd); 1961 1962 bad_unshare_cleanup_fs: 1963 if (new_fs) 1964 free_fs_struct(new_fs); 1965 1966 bad_unshare_out: 1967 return err; 1968 } 1969 1970 /* 1971 * Helper to unshare the files of the current task. 1972 * We don't want to expose copy_files internals to 1973 * the exec layer of the kernel. 1974 */ 1975 1976 int unshare_files(struct files_struct **displaced) 1977 { 1978 struct task_struct *task = current; 1979 struct files_struct *copy = NULL; 1980 int error; 1981 1982 error = unshare_fd(CLONE_FILES, ©); 1983 if (error || !copy) { 1984 *displaced = NULL; 1985 return error; 1986 } 1987 *displaced = task->files; 1988 task_lock(task); 1989 task->files = copy; 1990 task_unlock(task); 1991 return 0; 1992 } 1993