1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/mm/oom_kill.c 4 * 5 * Copyright (C) 1998,2000 Rik van Riel 6 * Thanks go out to Claus Fischer for some serious inspiration and 7 * for goading me into coding this file... 8 * Copyright (C) 2010 Google, Inc. 9 * Rewritten by David Rientjes 10 * 11 * The routines in this file are used to kill a process when 12 * we're seriously out of memory. This gets called from __alloc_pages() 13 * in mm/page_alloc.c when we really run out of memory. 14 * 15 * Since we won't call these routines often (on a well-configured 16 * machine) this file will double as a 'coding guide' and a signpost 17 * for newbie kernel hackers. It features several pointers to major 18 * kernel subsystems and hints as to where to find out what things do. 19 */ 20 21 #include <linux/oom.h> 22 #include <linux/mm.h> 23 #include <linux/err.h> 24 #include <linux/gfp.h> 25 #include <linux/sched.h> 26 #include <linux/sched/mm.h> 27 #include <linux/sched/coredump.h> 28 #include <linux/sched/task.h> 29 #include <linux/sched/debug.h> 30 #include <linux/swap.h> 31 #include <linux/timex.h> 32 #include <linux/jiffies.h> 33 #include <linux/cpuset.h> 34 #include <linux/export.h> 35 #include <linux/notifier.h> 36 #include <linux/memcontrol.h> 37 #include <linux/mempolicy.h> 38 #include <linux/security.h> 39 #include <linux/ptrace.h> 40 #include <linux/freezer.h> 41 #include <linux/ftrace.h> 42 #include <linux/ratelimit.h> 43 #include <linux/kthread.h> 44 #include <linux/init.h> 45 #include <linux/mmu_notifier.h> 46 47 #include <asm/tlb.h> 48 #include "internal.h" 49 #include "slab.h" 50 51 #define CREATE_TRACE_POINTS 52 #include <trace/events/oom.h> 53 54 int sysctl_panic_on_oom; 55 int sysctl_oom_kill_allocating_task; 56 int sysctl_oom_dump_tasks = 1; 57 58 /* 59 * Serializes oom killer invocations (out_of_memory()) from all contexts to 60 * prevent from over eager oom killing (e.g. when the oom killer is invoked 61 * from different domains). 62 * 63 * oom_killer_disable() relies on this lock to stabilize oom_killer_disabled 64 * and mark_oom_victim 65 */ 66 DEFINE_MUTEX(oom_lock); 67 68 static inline bool is_memcg_oom(struct oom_control *oc) 69 { 70 return oc->memcg != NULL; 71 } 72 73 #ifdef CONFIG_NUMA 74 /** 75 * oom_cpuset_eligible() - check task eligiblity for kill 76 * @start: task struct of which task to consider 77 * @oc: pointer to struct oom_control 78 * 79 * Task eligibility is determined by whether or not a candidate task, @tsk, 80 * shares the same mempolicy nodes as current if it is bound by such a policy 81 * and whether or not it has the same set of allowed cpuset nodes. 82 * 83 * This function is assuming oom-killer context and 'current' has triggered 84 * the oom-killer. 85 */ 86 static bool oom_cpuset_eligible(struct task_struct *start, 87 struct oom_control *oc) 88 { 89 struct task_struct *tsk; 90 bool ret = false; 91 const nodemask_t *mask = oc->nodemask; 92 93 if (is_memcg_oom(oc)) 94 return true; 95 96 rcu_read_lock(); 97 for_each_thread(start, tsk) { 98 if (mask) { 99 /* 100 * If this is a mempolicy constrained oom, tsk's 101 * cpuset is irrelevant. Only return true if its 102 * mempolicy intersects current, otherwise it may be 103 * needlessly killed. 104 */ 105 ret = mempolicy_nodemask_intersects(tsk, mask); 106 } else { 107 /* 108 * This is not a mempolicy constrained oom, so only 109 * check the mems of tsk's cpuset. 110 */ 111 ret = cpuset_mems_allowed_intersects(current, tsk); 112 } 113 if (ret) 114 break; 115 } 116 rcu_read_unlock(); 117 118 return ret; 119 } 120 #else 121 static bool oom_cpuset_eligible(struct task_struct *tsk, struct oom_control *oc) 122 { 123 return true; 124 } 125 #endif /* CONFIG_NUMA */ 126 127 /* 128 * The process p may have detached its own ->mm while exiting or through 129 * use_mm(), but one or more of its subthreads may still have a valid 130 * pointer. Return p, or any of its subthreads with a valid ->mm, with 131 * task_lock() held. 132 */ 133 struct task_struct *find_lock_task_mm(struct task_struct *p) 134 { 135 struct task_struct *t; 136 137 rcu_read_lock(); 138 139 for_each_thread(p, t) { 140 task_lock(t); 141 if (likely(t->mm)) 142 goto found; 143 task_unlock(t); 144 } 145 t = NULL; 146 found: 147 rcu_read_unlock(); 148 149 return t; 150 } 151 152 /* 153 * order == -1 means the oom kill is required by sysrq, otherwise only 154 * for display purposes. 155 */ 156 static inline bool is_sysrq_oom(struct oom_control *oc) 157 { 158 return oc->order == -1; 159 } 160 161 /* return true if the task is not adequate as candidate victim task. */ 162 static bool oom_unkillable_task(struct task_struct *p) 163 { 164 if (is_global_init(p)) 165 return true; 166 if (p->flags & PF_KTHREAD) 167 return true; 168 return false; 169 } 170 171 /* 172 * Print out unreclaimble slabs info when unreclaimable slabs amount is greater 173 * than all user memory (LRU pages) 174 */ 175 static bool is_dump_unreclaim_slabs(void) 176 { 177 unsigned long nr_lru; 178 179 nr_lru = global_node_page_state(NR_ACTIVE_ANON) + 180 global_node_page_state(NR_INACTIVE_ANON) + 181 global_node_page_state(NR_ACTIVE_FILE) + 182 global_node_page_state(NR_INACTIVE_FILE) + 183 global_node_page_state(NR_ISOLATED_ANON) + 184 global_node_page_state(NR_ISOLATED_FILE) + 185 global_node_page_state(NR_UNEVICTABLE); 186 187 return (global_node_page_state(NR_SLAB_UNRECLAIMABLE) > nr_lru); 188 } 189 190 /** 191 * oom_badness - heuristic function to determine which candidate task to kill 192 * @p: task struct of which task we should calculate 193 * @totalpages: total present RAM allowed for page allocation 194 * 195 * The heuristic for determining which task to kill is made to be as simple and 196 * predictable as possible. The goal is to return the highest value for the 197 * task consuming the most memory to avoid subsequent oom failures. 198 */ 199 unsigned long oom_badness(struct task_struct *p, unsigned long totalpages) 200 { 201 long points; 202 long adj; 203 204 if (oom_unkillable_task(p)) 205 return 0; 206 207 p = find_lock_task_mm(p); 208 if (!p) 209 return 0; 210 211 /* 212 * Do not even consider tasks which are explicitly marked oom 213 * unkillable or have been already oom reaped or the are in 214 * the middle of vfork 215 */ 216 adj = (long)p->signal->oom_score_adj; 217 if (adj == OOM_SCORE_ADJ_MIN || 218 test_bit(MMF_OOM_SKIP, &p->mm->flags) || 219 in_vfork(p)) { 220 task_unlock(p); 221 return 0; 222 } 223 224 /* 225 * The baseline for the badness score is the proportion of RAM that each 226 * task's rss, pagetable and swap space use. 227 */ 228 points = get_mm_rss(p->mm) + get_mm_counter(p->mm, MM_SWAPENTS) + 229 mm_pgtables_bytes(p->mm) / PAGE_SIZE; 230 task_unlock(p); 231 232 /* Normalize to oom_score_adj units */ 233 adj *= totalpages / 1000; 234 points += adj; 235 236 /* 237 * Never return 0 for an eligible task regardless of the root bonus and 238 * oom_score_adj (oom_score_adj can't be OOM_SCORE_ADJ_MIN here). 239 */ 240 return points > 0 ? points : 1; 241 } 242 243 static const char * const oom_constraint_text[] = { 244 [CONSTRAINT_NONE] = "CONSTRAINT_NONE", 245 [CONSTRAINT_CPUSET] = "CONSTRAINT_CPUSET", 246 [CONSTRAINT_MEMORY_POLICY] = "CONSTRAINT_MEMORY_POLICY", 247 [CONSTRAINT_MEMCG] = "CONSTRAINT_MEMCG", 248 }; 249 250 /* 251 * Determine the type of allocation constraint. 252 */ 253 static enum oom_constraint constrained_alloc(struct oom_control *oc) 254 { 255 struct zone *zone; 256 struct zoneref *z; 257 enum zone_type high_zoneidx = gfp_zone(oc->gfp_mask); 258 bool cpuset_limited = false; 259 int nid; 260 261 if (is_memcg_oom(oc)) { 262 oc->totalpages = mem_cgroup_get_max(oc->memcg) ?: 1; 263 return CONSTRAINT_MEMCG; 264 } 265 266 /* Default to all available memory */ 267 oc->totalpages = totalram_pages() + total_swap_pages; 268 269 if (!IS_ENABLED(CONFIG_NUMA)) 270 return CONSTRAINT_NONE; 271 272 if (!oc->zonelist) 273 return CONSTRAINT_NONE; 274 /* 275 * Reach here only when __GFP_NOFAIL is used. So, we should avoid 276 * to kill current.We have to random task kill in this case. 277 * Hopefully, CONSTRAINT_THISNODE...but no way to handle it, now. 278 */ 279 if (oc->gfp_mask & __GFP_THISNODE) 280 return CONSTRAINT_NONE; 281 282 /* 283 * This is not a __GFP_THISNODE allocation, so a truncated nodemask in 284 * the page allocator means a mempolicy is in effect. Cpuset policy 285 * is enforced in get_page_from_freelist(). 286 */ 287 if (oc->nodemask && 288 !nodes_subset(node_states[N_MEMORY], *oc->nodemask)) { 289 oc->totalpages = total_swap_pages; 290 for_each_node_mask(nid, *oc->nodemask) 291 oc->totalpages += node_present_pages(nid); 292 return CONSTRAINT_MEMORY_POLICY; 293 } 294 295 /* Check this allocation failure is caused by cpuset's wall function */ 296 for_each_zone_zonelist_nodemask(zone, z, oc->zonelist, 297 high_zoneidx, oc->nodemask) 298 if (!cpuset_zone_allowed(zone, oc->gfp_mask)) 299 cpuset_limited = true; 300 301 if (cpuset_limited) { 302 oc->totalpages = total_swap_pages; 303 for_each_node_mask(nid, cpuset_current_mems_allowed) 304 oc->totalpages += node_present_pages(nid); 305 return CONSTRAINT_CPUSET; 306 } 307 return CONSTRAINT_NONE; 308 } 309 310 static int oom_evaluate_task(struct task_struct *task, void *arg) 311 { 312 struct oom_control *oc = arg; 313 unsigned long points; 314 315 if (oom_unkillable_task(task)) 316 goto next; 317 318 /* p may not have freeable memory in nodemask */ 319 if (!is_memcg_oom(oc) && !oom_cpuset_eligible(task, oc)) 320 goto next; 321 322 /* 323 * This task already has access to memory reserves and is being killed. 324 * Don't allow any other task to have access to the reserves unless 325 * the task has MMF_OOM_SKIP because chances that it would release 326 * any memory is quite low. 327 */ 328 if (!is_sysrq_oom(oc) && tsk_is_oom_victim(task)) { 329 if (test_bit(MMF_OOM_SKIP, &task->signal->oom_mm->flags)) 330 goto next; 331 goto abort; 332 } 333 334 /* 335 * If task is allocating a lot of memory and has been marked to be 336 * killed first if it triggers an oom, then select it. 337 */ 338 if (oom_task_origin(task)) { 339 points = ULONG_MAX; 340 goto select; 341 } 342 343 points = oom_badness(task, oc->totalpages); 344 if (!points || points < oc->chosen_points) 345 goto next; 346 347 select: 348 if (oc->chosen) 349 put_task_struct(oc->chosen); 350 get_task_struct(task); 351 oc->chosen = task; 352 oc->chosen_points = points; 353 next: 354 return 0; 355 abort: 356 if (oc->chosen) 357 put_task_struct(oc->chosen); 358 oc->chosen = (void *)-1UL; 359 return 1; 360 } 361 362 /* 363 * Simple selection loop. We choose the process with the highest number of 364 * 'points'. In case scan was aborted, oc->chosen is set to -1. 365 */ 366 static void select_bad_process(struct oom_control *oc) 367 { 368 if (is_memcg_oom(oc)) 369 mem_cgroup_scan_tasks(oc->memcg, oom_evaluate_task, oc); 370 else { 371 struct task_struct *p; 372 373 rcu_read_lock(); 374 for_each_process(p) 375 if (oom_evaluate_task(p, oc)) 376 break; 377 rcu_read_unlock(); 378 } 379 } 380 381 static int dump_task(struct task_struct *p, void *arg) 382 { 383 struct oom_control *oc = arg; 384 struct task_struct *task; 385 386 if (oom_unkillable_task(p)) 387 return 0; 388 389 /* p may not have freeable memory in nodemask */ 390 if (!is_memcg_oom(oc) && !oom_cpuset_eligible(p, oc)) 391 return 0; 392 393 task = find_lock_task_mm(p); 394 if (!task) { 395 /* 396 * This is a kthread or all of p's threads have already 397 * detached their mm's. There's no need to report 398 * them; they can't be oom killed anyway. 399 */ 400 return 0; 401 } 402 403 pr_info("[%7d] %5d %5d %8lu %8lu %8ld %8lu %5hd %s\n", 404 task->pid, from_kuid(&init_user_ns, task_uid(task)), 405 task->tgid, task->mm->total_vm, get_mm_rss(task->mm), 406 mm_pgtables_bytes(task->mm), 407 get_mm_counter(task->mm, MM_SWAPENTS), 408 task->signal->oom_score_adj, task->comm); 409 task_unlock(task); 410 411 return 0; 412 } 413 414 /** 415 * dump_tasks - dump current memory state of all system tasks 416 * @oc: pointer to struct oom_control 417 * 418 * Dumps the current memory state of all eligible tasks. Tasks not in the same 419 * memcg, not in the same cpuset, or bound to a disjoint set of mempolicy nodes 420 * are not shown. 421 * State information includes task's pid, uid, tgid, vm size, rss, 422 * pgtables_bytes, swapents, oom_score_adj value, and name. 423 */ 424 static void dump_tasks(struct oom_control *oc) 425 { 426 pr_info("Tasks state (memory values in pages):\n"); 427 pr_info("[ pid ] uid tgid total_vm rss pgtables_bytes swapents oom_score_adj name\n"); 428 429 if (is_memcg_oom(oc)) 430 mem_cgroup_scan_tasks(oc->memcg, dump_task, oc); 431 else { 432 struct task_struct *p; 433 434 rcu_read_lock(); 435 for_each_process(p) 436 dump_task(p, oc); 437 rcu_read_unlock(); 438 } 439 } 440 441 static void dump_oom_summary(struct oom_control *oc, struct task_struct *victim) 442 { 443 /* one line summary of the oom killer context. */ 444 pr_info("oom-kill:constraint=%s,nodemask=%*pbl", 445 oom_constraint_text[oc->constraint], 446 nodemask_pr_args(oc->nodemask)); 447 cpuset_print_current_mems_allowed(); 448 mem_cgroup_print_oom_context(oc->memcg, victim); 449 pr_cont(",task=%s,pid=%d,uid=%d\n", victim->comm, victim->pid, 450 from_kuid(&init_user_ns, task_uid(victim))); 451 } 452 453 static void dump_header(struct oom_control *oc, struct task_struct *p) 454 { 455 pr_warn("%s invoked oom-killer: gfp_mask=%#x(%pGg), order=%d, oom_score_adj=%hd\n", 456 current->comm, oc->gfp_mask, &oc->gfp_mask, oc->order, 457 current->signal->oom_score_adj); 458 if (!IS_ENABLED(CONFIG_COMPACTION) && oc->order) 459 pr_warn("COMPACTION is disabled!!!\n"); 460 461 dump_stack(); 462 if (is_memcg_oom(oc)) 463 mem_cgroup_print_oom_meminfo(oc->memcg); 464 else { 465 show_mem(SHOW_MEM_FILTER_NODES, oc->nodemask); 466 if (is_dump_unreclaim_slabs()) 467 dump_unreclaimable_slab(); 468 } 469 if (sysctl_oom_dump_tasks) 470 dump_tasks(oc); 471 if (p) 472 dump_oom_summary(oc, p); 473 } 474 475 /* 476 * Number of OOM victims in flight 477 */ 478 static atomic_t oom_victims = ATOMIC_INIT(0); 479 static DECLARE_WAIT_QUEUE_HEAD(oom_victims_wait); 480 481 static bool oom_killer_disabled __read_mostly; 482 483 #define K(x) ((x) << (PAGE_SHIFT-10)) 484 485 /* 486 * task->mm can be NULL if the task is the exited group leader. So to 487 * determine whether the task is using a particular mm, we examine all the 488 * task's threads: if one of those is using this mm then this task was also 489 * using it. 490 */ 491 bool process_shares_mm(struct task_struct *p, struct mm_struct *mm) 492 { 493 struct task_struct *t; 494 495 for_each_thread(p, t) { 496 struct mm_struct *t_mm = READ_ONCE(t->mm); 497 if (t_mm) 498 return t_mm == mm; 499 } 500 return false; 501 } 502 503 #ifdef CONFIG_MMU 504 /* 505 * OOM Reaper kernel thread which tries to reap the memory used by the OOM 506 * victim (if that is possible) to help the OOM killer to move on. 507 */ 508 static struct task_struct *oom_reaper_th; 509 static DECLARE_WAIT_QUEUE_HEAD(oom_reaper_wait); 510 static struct task_struct *oom_reaper_list; 511 static DEFINE_SPINLOCK(oom_reaper_lock); 512 513 bool __oom_reap_task_mm(struct mm_struct *mm) 514 { 515 struct vm_area_struct *vma; 516 bool ret = true; 517 518 /* 519 * Tell all users of get_user/copy_from_user etc... that the content 520 * is no longer stable. No barriers really needed because unmapping 521 * should imply barriers already and the reader would hit a page fault 522 * if it stumbled over a reaped memory. 523 */ 524 set_bit(MMF_UNSTABLE, &mm->flags); 525 526 for (vma = mm->mmap ; vma; vma = vma->vm_next) { 527 if (!can_madv_lru_vma(vma)) 528 continue; 529 530 /* 531 * Only anonymous pages have a good chance to be dropped 532 * without additional steps which we cannot afford as we 533 * are OOM already. 534 * 535 * We do not even care about fs backed pages because all 536 * which are reclaimable have already been reclaimed and 537 * we do not want to block exit_mmap by keeping mm ref 538 * count elevated without a good reason. 539 */ 540 if (vma_is_anonymous(vma) || !(vma->vm_flags & VM_SHARED)) { 541 struct mmu_notifier_range range; 542 struct mmu_gather tlb; 543 544 mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, 545 vma, mm, vma->vm_start, 546 vma->vm_end); 547 tlb_gather_mmu(&tlb, mm, range.start, range.end); 548 if (mmu_notifier_invalidate_range_start_nonblock(&range)) { 549 tlb_finish_mmu(&tlb, range.start, range.end); 550 ret = false; 551 continue; 552 } 553 unmap_page_range(&tlb, vma, range.start, range.end, NULL); 554 mmu_notifier_invalidate_range_end(&range); 555 tlb_finish_mmu(&tlb, range.start, range.end); 556 } 557 } 558 559 return ret; 560 } 561 562 /* 563 * Reaps the address space of the give task. 564 * 565 * Returns true on success and false if none or part of the address space 566 * has been reclaimed and the caller should retry later. 567 */ 568 static bool oom_reap_task_mm(struct task_struct *tsk, struct mm_struct *mm) 569 { 570 bool ret = true; 571 572 if (!down_read_trylock(&mm->mmap_sem)) { 573 trace_skip_task_reaping(tsk->pid); 574 return false; 575 } 576 577 /* 578 * MMF_OOM_SKIP is set by exit_mmap when the OOM reaper can't 579 * work on the mm anymore. The check for MMF_OOM_SKIP must run 580 * under mmap_sem for reading because it serializes against the 581 * down_write();up_write() cycle in exit_mmap(). 582 */ 583 if (test_bit(MMF_OOM_SKIP, &mm->flags)) { 584 trace_skip_task_reaping(tsk->pid); 585 goto out_unlock; 586 } 587 588 trace_start_task_reaping(tsk->pid); 589 590 /* failed to reap part of the address space. Try again later */ 591 ret = __oom_reap_task_mm(mm); 592 if (!ret) 593 goto out_finish; 594 595 pr_info("oom_reaper: reaped process %d (%s), now anon-rss:%lukB, file-rss:%lukB, shmem-rss:%lukB\n", 596 task_pid_nr(tsk), tsk->comm, 597 K(get_mm_counter(mm, MM_ANONPAGES)), 598 K(get_mm_counter(mm, MM_FILEPAGES)), 599 K(get_mm_counter(mm, MM_SHMEMPAGES))); 600 out_finish: 601 trace_finish_task_reaping(tsk->pid); 602 out_unlock: 603 up_read(&mm->mmap_sem); 604 605 return ret; 606 } 607 608 #define MAX_OOM_REAP_RETRIES 10 609 static void oom_reap_task(struct task_struct *tsk) 610 { 611 int attempts = 0; 612 struct mm_struct *mm = tsk->signal->oom_mm; 613 614 /* Retry the down_read_trylock(mmap_sem) a few times */ 615 while (attempts++ < MAX_OOM_REAP_RETRIES && !oom_reap_task_mm(tsk, mm)) 616 schedule_timeout_idle(HZ/10); 617 618 if (attempts <= MAX_OOM_REAP_RETRIES || 619 test_bit(MMF_OOM_SKIP, &mm->flags)) 620 goto done; 621 622 pr_info("oom_reaper: unable to reap pid:%d (%s)\n", 623 task_pid_nr(tsk), tsk->comm); 624 sched_show_task(tsk); 625 debug_show_all_locks(); 626 627 done: 628 tsk->oom_reaper_list = NULL; 629 630 /* 631 * Hide this mm from OOM killer because it has been either reaped or 632 * somebody can't call up_write(mmap_sem). 633 */ 634 set_bit(MMF_OOM_SKIP, &mm->flags); 635 636 /* Drop a reference taken by wake_oom_reaper */ 637 put_task_struct(tsk); 638 } 639 640 static int oom_reaper(void *unused) 641 { 642 while (true) { 643 struct task_struct *tsk = NULL; 644 645 wait_event_freezable(oom_reaper_wait, oom_reaper_list != NULL); 646 spin_lock(&oom_reaper_lock); 647 if (oom_reaper_list != NULL) { 648 tsk = oom_reaper_list; 649 oom_reaper_list = tsk->oom_reaper_list; 650 } 651 spin_unlock(&oom_reaper_lock); 652 653 if (tsk) 654 oom_reap_task(tsk); 655 } 656 657 return 0; 658 } 659 660 static void wake_oom_reaper(struct task_struct *tsk) 661 { 662 /* mm is already queued? */ 663 if (test_and_set_bit(MMF_OOM_REAP_QUEUED, &tsk->signal->oom_mm->flags)) 664 return; 665 666 get_task_struct(tsk); 667 668 spin_lock(&oom_reaper_lock); 669 tsk->oom_reaper_list = oom_reaper_list; 670 oom_reaper_list = tsk; 671 spin_unlock(&oom_reaper_lock); 672 trace_wake_reaper(tsk->pid); 673 wake_up(&oom_reaper_wait); 674 } 675 676 static int __init oom_init(void) 677 { 678 oom_reaper_th = kthread_run(oom_reaper, NULL, "oom_reaper"); 679 return 0; 680 } 681 subsys_initcall(oom_init) 682 #else 683 static inline void wake_oom_reaper(struct task_struct *tsk) 684 { 685 } 686 #endif /* CONFIG_MMU */ 687 688 /** 689 * mark_oom_victim - mark the given task as OOM victim 690 * @tsk: task to mark 691 * 692 * Has to be called with oom_lock held and never after 693 * oom has been disabled already. 694 * 695 * tsk->mm has to be non NULL and caller has to guarantee it is stable (either 696 * under task_lock or operate on the current). 697 */ 698 static void mark_oom_victim(struct task_struct *tsk) 699 { 700 struct mm_struct *mm = tsk->mm; 701 702 WARN_ON(oom_killer_disabled); 703 /* OOM killer might race with memcg OOM */ 704 if (test_and_set_tsk_thread_flag(tsk, TIF_MEMDIE)) 705 return; 706 707 /* oom_mm is bound to the signal struct life time. */ 708 if (!cmpxchg(&tsk->signal->oom_mm, NULL, mm)) { 709 mmgrab(tsk->signal->oom_mm); 710 set_bit(MMF_OOM_VICTIM, &mm->flags); 711 } 712 713 /* 714 * Make sure that the task is woken up from uninterruptible sleep 715 * if it is frozen because OOM killer wouldn't be able to free 716 * any memory and livelock. freezing_slow_path will tell the freezer 717 * that TIF_MEMDIE tasks should be ignored. 718 */ 719 __thaw_task(tsk); 720 atomic_inc(&oom_victims); 721 trace_mark_victim(tsk->pid); 722 } 723 724 /** 725 * exit_oom_victim - note the exit of an OOM victim 726 */ 727 void exit_oom_victim(void) 728 { 729 clear_thread_flag(TIF_MEMDIE); 730 731 if (!atomic_dec_return(&oom_victims)) 732 wake_up_all(&oom_victims_wait); 733 } 734 735 /** 736 * oom_killer_enable - enable OOM killer 737 */ 738 void oom_killer_enable(void) 739 { 740 oom_killer_disabled = false; 741 pr_info("OOM killer enabled.\n"); 742 } 743 744 /** 745 * oom_killer_disable - disable OOM killer 746 * @timeout: maximum timeout to wait for oom victims in jiffies 747 * 748 * Forces all page allocations to fail rather than trigger OOM killer. 749 * Will block and wait until all OOM victims are killed or the given 750 * timeout expires. 751 * 752 * The function cannot be called when there are runnable user tasks because 753 * the userspace would see unexpected allocation failures as a result. Any 754 * new usage of this function should be consulted with MM people. 755 * 756 * Returns true if successful and false if the OOM killer cannot be 757 * disabled. 758 */ 759 bool oom_killer_disable(signed long timeout) 760 { 761 signed long ret; 762 763 /* 764 * Make sure to not race with an ongoing OOM killer. Check that the 765 * current is not killed (possibly due to sharing the victim's memory). 766 */ 767 if (mutex_lock_killable(&oom_lock)) 768 return false; 769 oom_killer_disabled = true; 770 mutex_unlock(&oom_lock); 771 772 ret = wait_event_interruptible_timeout(oom_victims_wait, 773 !atomic_read(&oom_victims), timeout); 774 if (ret <= 0) { 775 oom_killer_enable(); 776 return false; 777 } 778 pr_info("OOM killer disabled.\n"); 779 780 return true; 781 } 782 783 static inline bool __task_will_free_mem(struct task_struct *task) 784 { 785 struct signal_struct *sig = task->signal; 786 787 /* 788 * A coredumping process may sleep for an extended period in exit_mm(), 789 * so the oom killer cannot assume that the process will promptly exit 790 * and release memory. 791 */ 792 if (sig->flags & SIGNAL_GROUP_COREDUMP) 793 return false; 794 795 if (sig->flags & SIGNAL_GROUP_EXIT) 796 return true; 797 798 if (thread_group_empty(task) && (task->flags & PF_EXITING)) 799 return true; 800 801 return false; 802 } 803 804 /* 805 * Checks whether the given task is dying or exiting and likely to 806 * release its address space. This means that all threads and processes 807 * sharing the same mm have to be killed or exiting. 808 * Caller has to make sure that task->mm is stable (hold task_lock or 809 * it operates on the current). 810 */ 811 static bool task_will_free_mem(struct task_struct *task) 812 { 813 struct mm_struct *mm = task->mm; 814 struct task_struct *p; 815 bool ret = true; 816 817 /* 818 * Skip tasks without mm because it might have passed its exit_mm and 819 * exit_oom_victim. oom_reaper could have rescued that but do not rely 820 * on that for now. We can consider find_lock_task_mm in future. 821 */ 822 if (!mm) 823 return false; 824 825 if (!__task_will_free_mem(task)) 826 return false; 827 828 /* 829 * This task has already been drained by the oom reaper so there are 830 * only small chances it will free some more 831 */ 832 if (test_bit(MMF_OOM_SKIP, &mm->flags)) 833 return false; 834 835 if (atomic_read(&mm->mm_users) <= 1) 836 return true; 837 838 /* 839 * Make sure that all tasks which share the mm with the given tasks 840 * are dying as well to make sure that a) nobody pins its mm and 841 * b) the task is also reapable by the oom reaper. 842 */ 843 rcu_read_lock(); 844 for_each_process(p) { 845 if (!process_shares_mm(p, mm)) 846 continue; 847 if (same_thread_group(task, p)) 848 continue; 849 ret = __task_will_free_mem(p); 850 if (!ret) 851 break; 852 } 853 rcu_read_unlock(); 854 855 return ret; 856 } 857 858 static void __oom_kill_process(struct task_struct *victim, const char *message) 859 { 860 struct task_struct *p; 861 struct mm_struct *mm; 862 bool can_oom_reap = true; 863 864 p = find_lock_task_mm(victim); 865 if (!p) { 866 put_task_struct(victim); 867 return; 868 } else if (victim != p) { 869 get_task_struct(p); 870 put_task_struct(victim); 871 victim = p; 872 } 873 874 /* Get a reference to safely compare mm after task_unlock(victim) */ 875 mm = victim->mm; 876 mmgrab(mm); 877 878 /* Raise event before sending signal: task reaper must see this */ 879 count_vm_event(OOM_KILL); 880 memcg_memory_event_mm(mm, MEMCG_OOM_KILL); 881 882 /* 883 * We should send SIGKILL before granting access to memory reserves 884 * in order to prevent the OOM victim from depleting the memory 885 * reserves from the user space under its control. 886 */ 887 do_send_sig_info(SIGKILL, SEND_SIG_PRIV, victim, PIDTYPE_TGID); 888 mark_oom_victim(victim); 889 pr_err("%s: Killed process %d (%s) total-vm:%lukB, anon-rss:%lukB, file-rss:%lukB, shmem-rss:%lukB, UID:%u pgtables:%lukB oom_score_adj:%hd\n", 890 message, task_pid_nr(victim), victim->comm, K(mm->total_vm), 891 K(get_mm_counter(mm, MM_ANONPAGES)), 892 K(get_mm_counter(mm, MM_FILEPAGES)), 893 K(get_mm_counter(mm, MM_SHMEMPAGES)), 894 from_kuid(&init_user_ns, task_uid(victim)), 895 mm_pgtables_bytes(mm) >> 10, victim->signal->oom_score_adj); 896 task_unlock(victim); 897 898 /* 899 * Kill all user processes sharing victim->mm in other thread groups, if 900 * any. They don't get access to memory reserves, though, to avoid 901 * depletion of all memory. This prevents mm->mmap_sem livelock when an 902 * oom killed thread cannot exit because it requires the semaphore and 903 * its contended by another thread trying to allocate memory itself. 904 * That thread will now get access to memory reserves since it has a 905 * pending fatal signal. 906 */ 907 rcu_read_lock(); 908 for_each_process(p) { 909 if (!process_shares_mm(p, mm)) 910 continue; 911 if (same_thread_group(p, victim)) 912 continue; 913 if (is_global_init(p)) { 914 can_oom_reap = false; 915 set_bit(MMF_OOM_SKIP, &mm->flags); 916 pr_info("oom killer %d (%s) has mm pinned by %d (%s)\n", 917 task_pid_nr(victim), victim->comm, 918 task_pid_nr(p), p->comm); 919 continue; 920 } 921 /* 922 * No use_mm() user needs to read from the userspace so we are 923 * ok to reap it. 924 */ 925 if (unlikely(p->flags & PF_KTHREAD)) 926 continue; 927 do_send_sig_info(SIGKILL, SEND_SIG_PRIV, p, PIDTYPE_TGID); 928 } 929 rcu_read_unlock(); 930 931 if (can_oom_reap) 932 wake_oom_reaper(victim); 933 934 mmdrop(mm); 935 put_task_struct(victim); 936 } 937 #undef K 938 939 /* 940 * Kill provided task unless it's secured by setting 941 * oom_score_adj to OOM_SCORE_ADJ_MIN. 942 */ 943 static int oom_kill_memcg_member(struct task_struct *task, void *message) 944 { 945 if (task->signal->oom_score_adj != OOM_SCORE_ADJ_MIN && 946 !is_global_init(task)) { 947 get_task_struct(task); 948 __oom_kill_process(task, message); 949 } 950 return 0; 951 } 952 953 static void oom_kill_process(struct oom_control *oc, const char *message) 954 { 955 struct task_struct *victim = oc->chosen; 956 struct mem_cgroup *oom_group; 957 static DEFINE_RATELIMIT_STATE(oom_rs, DEFAULT_RATELIMIT_INTERVAL, 958 DEFAULT_RATELIMIT_BURST); 959 960 /* 961 * If the task is already exiting, don't alarm the sysadmin or kill 962 * its children or threads, just give it access to memory reserves 963 * so it can die quickly 964 */ 965 task_lock(victim); 966 if (task_will_free_mem(victim)) { 967 mark_oom_victim(victim); 968 wake_oom_reaper(victim); 969 task_unlock(victim); 970 put_task_struct(victim); 971 return; 972 } 973 task_unlock(victim); 974 975 if (__ratelimit(&oom_rs)) 976 dump_header(oc, victim); 977 978 /* 979 * Do we need to kill the entire memory cgroup? 980 * Or even one of the ancestor memory cgroups? 981 * Check this out before killing the victim task. 982 */ 983 oom_group = mem_cgroup_get_oom_group(victim, oc->memcg); 984 985 __oom_kill_process(victim, message); 986 987 /* 988 * If necessary, kill all tasks in the selected memory cgroup. 989 */ 990 if (oom_group) { 991 mem_cgroup_print_oom_group(oom_group); 992 mem_cgroup_scan_tasks(oom_group, oom_kill_memcg_member, 993 (void*)message); 994 mem_cgroup_put(oom_group); 995 } 996 } 997 998 /* 999 * Determines whether the kernel must panic because of the panic_on_oom sysctl. 1000 */ 1001 static void check_panic_on_oom(struct oom_control *oc) 1002 { 1003 if (likely(!sysctl_panic_on_oom)) 1004 return; 1005 if (sysctl_panic_on_oom != 2) { 1006 /* 1007 * panic_on_oom == 1 only affects CONSTRAINT_NONE, the kernel 1008 * does not panic for cpuset, mempolicy, or memcg allocation 1009 * failures. 1010 */ 1011 if (oc->constraint != CONSTRAINT_NONE) 1012 return; 1013 } 1014 /* Do not panic for oom kills triggered by sysrq */ 1015 if (is_sysrq_oom(oc)) 1016 return; 1017 dump_header(oc, NULL); 1018 panic("Out of memory: %s panic_on_oom is enabled\n", 1019 sysctl_panic_on_oom == 2 ? "compulsory" : "system-wide"); 1020 } 1021 1022 static BLOCKING_NOTIFIER_HEAD(oom_notify_list); 1023 1024 int register_oom_notifier(struct notifier_block *nb) 1025 { 1026 return blocking_notifier_chain_register(&oom_notify_list, nb); 1027 } 1028 EXPORT_SYMBOL_GPL(register_oom_notifier); 1029 1030 int unregister_oom_notifier(struct notifier_block *nb) 1031 { 1032 return blocking_notifier_chain_unregister(&oom_notify_list, nb); 1033 } 1034 EXPORT_SYMBOL_GPL(unregister_oom_notifier); 1035 1036 /** 1037 * out_of_memory - kill the "best" process when we run out of memory 1038 * @oc: pointer to struct oom_control 1039 * 1040 * If we run out of memory, we have the choice between either 1041 * killing a random task (bad), letting the system crash (worse) 1042 * OR try to be smart about which process to kill. Note that we 1043 * don't have to be perfect here, we just have to be good. 1044 */ 1045 bool out_of_memory(struct oom_control *oc) 1046 { 1047 unsigned long freed = 0; 1048 1049 if (oom_killer_disabled) 1050 return false; 1051 1052 if (!is_memcg_oom(oc)) { 1053 blocking_notifier_call_chain(&oom_notify_list, 0, &freed); 1054 if (freed > 0) 1055 /* Got some memory back in the last second. */ 1056 return true; 1057 } 1058 1059 /* 1060 * If current has a pending SIGKILL or is exiting, then automatically 1061 * select it. The goal is to allow it to allocate so that it may 1062 * quickly exit and free its memory. 1063 */ 1064 if (task_will_free_mem(current)) { 1065 mark_oom_victim(current); 1066 wake_oom_reaper(current); 1067 return true; 1068 } 1069 1070 /* 1071 * The OOM killer does not compensate for IO-less reclaim. 1072 * pagefault_out_of_memory lost its gfp context so we have to 1073 * make sure exclude 0 mask - all other users should have at least 1074 * ___GFP_DIRECT_RECLAIM to get here. But mem_cgroup_oom() has to 1075 * invoke the OOM killer even if it is a GFP_NOFS allocation. 1076 */ 1077 if (oc->gfp_mask && !(oc->gfp_mask & __GFP_FS) && !is_memcg_oom(oc)) 1078 return true; 1079 1080 /* 1081 * Check if there were limitations on the allocation (only relevant for 1082 * NUMA and memcg) that may require different handling. 1083 */ 1084 oc->constraint = constrained_alloc(oc); 1085 if (oc->constraint != CONSTRAINT_MEMORY_POLICY) 1086 oc->nodemask = NULL; 1087 check_panic_on_oom(oc); 1088 1089 if (!is_memcg_oom(oc) && sysctl_oom_kill_allocating_task && 1090 current->mm && !oom_unkillable_task(current) && 1091 oom_cpuset_eligible(current, oc) && 1092 current->signal->oom_score_adj != OOM_SCORE_ADJ_MIN) { 1093 get_task_struct(current); 1094 oc->chosen = current; 1095 oom_kill_process(oc, "Out of memory (oom_kill_allocating_task)"); 1096 return true; 1097 } 1098 1099 select_bad_process(oc); 1100 /* Found nothing?!?! */ 1101 if (!oc->chosen) { 1102 dump_header(oc, NULL); 1103 pr_warn("Out of memory and no killable processes...\n"); 1104 /* 1105 * If we got here due to an actual allocation at the 1106 * system level, we cannot survive this and will enter 1107 * an endless loop in the allocator. Bail out now. 1108 */ 1109 if (!is_sysrq_oom(oc) && !is_memcg_oom(oc)) 1110 panic("System is deadlocked on memory\n"); 1111 } 1112 if (oc->chosen && oc->chosen != (void *)-1UL) 1113 oom_kill_process(oc, !is_memcg_oom(oc) ? "Out of memory" : 1114 "Memory cgroup out of memory"); 1115 return !!oc->chosen; 1116 } 1117 1118 /* 1119 * The pagefault handler calls here because it is out of memory, so kill a 1120 * memory-hogging task. If oom_lock is held by somebody else, a parallel oom 1121 * killing is already in progress so do nothing. 1122 */ 1123 void pagefault_out_of_memory(void) 1124 { 1125 struct oom_control oc = { 1126 .zonelist = NULL, 1127 .nodemask = NULL, 1128 .memcg = NULL, 1129 .gfp_mask = 0, 1130 .order = 0, 1131 }; 1132 1133 if (mem_cgroup_oom_synchronize(true)) 1134 return; 1135 1136 if (!mutex_trylock(&oom_lock)) 1137 return; 1138 out_of_memory(&oc); 1139 mutex_unlock(&oom_lock); 1140 } 1141