Lines Matching +full:scrubber +full:- +full:done
1 // SPDX-License-Identifier: GPL-2.0-only
7 * hardware as being corrupted usually due to a multi-bit ECC memory or cache
11 * not-yet-corrupted-by-suspicious pages without killing anything.
23 * - You know how to test it.
24 * - You have a test that can be added to mce-test
25 * https://git.kernel.org/cgit/utils/cpu/mce/mce-test.git/
26 * - The case actually shows up as a frequent (top 10) page state in
27 * tools/mm/page-types when running a real workload.
41 #include <linux/page-flags.h>
50 #include <linux/backing-dev.h>
88 if (pfn != -1UL) in num_poisoned_pages_sub()
93 * MF_ATTR_RO - Create sysfs entry for each memory failure statistics.
102 &NODE_DATA(dev->id)->mf_stats; \
103 return sysfs_emit(buf, "%lu\n", mf_stats->_name); \
193 * dissolve_free_hugetlb_folio() returns 0 for non-hugetlb folios as well. in page_handle_poison()
199 * acceptable because soft-offlined page is not broken in page_handle_poison()
255 * from the VMAs. So do a brute-force search over all
280 struct task_struct *t = tk->tsk; in kill_proc()
281 short addr_lsb = tk->size_shift; in kill_proc()
285 pfn, t->comm, task_pid_nr(t)); in kill_proc()
289 (void __user *)tk->addr, addr_lsb); in kill_proc()
297 ret = send_sig_mceerr(BUS_MCEERR_AO, (void __user *)tk->addr, in kill_proc()
301 t->comm, task_pid_nr(t), ret); in kill_proc()
314 * TODO: Could shrink slab caches here if a lightweight range-based in shake_folio()
340 VM_BUG_ON_VMA(address == -EFAULT, vma); in dev_pagemap_mapping_shift()
341 pgd = pgd_offset(vma->vm_mm, address); in dev_pagemap_mapping_shift()
388 tk->addr = addr; in __add_to_kill()
390 tk->size_shift = dev_pagemap_mapping_shift(vma, tk->addr); in __add_to_kill()
392 tk->size_shift = folio_shift(page_folio(p)); in __add_to_kill()
395 * Send SIGKILL if "tk->addr == -EFAULT". Also, as in __add_to_kill()
396 * "tk->size_shift" is always non-zero for !is_zone_device_page(), in __add_to_kill()
397 * so "tk->size_shift == 0" effectively checks no mapping on in __add_to_kill()
404 if (tk->addr == -EFAULT) { in __add_to_kill()
406 page_to_pfn(p), tsk->comm); in __add_to_kill()
407 } else if (tk->size_shift == 0) { in __add_to_kill()
413 tk->tsk = tsk; in __add_to_kill()
414 list_add_tail(&tk->nd, to_kill); in __add_to_kill()
421 if (addr == -EFAULT) in add_to_kill_anon_file()
433 if (tk->tsk == tsk) in task_in_to_kill_list()
461 if (tk->addr == -EFAULT) { in kill_procs()
463 pfn, tk->tsk->comm, task_pid_nr(tk->tsk)); in kill_procs()
465 tk->tsk, PIDTYPE_PID); in kill_procs()
470 * something else on the address in-between. We could in kill_procs()
476 pfn, tk->tsk->comm, task_pid_nr(tk->tsk)); in kill_procs()
478 list_del(&tk->nd); in kill_procs()
479 put_task_struct(tk->tsk); in kill_procs()
497 if (t->flags & PF_MCE_PROCESS) { in find_early_kill_thread()
498 if (t->flags & PF_MCE_EARLY) in find_early_kill_thread()
522 if (!tsk->mm) in task_early_kill()
525 * Comparing ->mm here because current task might represent in task_early_kill()
528 if (force_early && tsk->mm == current->mm) in task_early_kill()
559 anon_vma_interval_tree_foreach(vmac, &av->rb_root, in collect_procs_anon()
561 vma = vmac->vma; in collect_procs_anon()
562 if (vma->vm_mm != t->mm) in collect_procs_anon()
581 struct address_space *mapping = folio->mapping; in collect_procs_file()
593 vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, in collect_procs_file()
602 if (vma->vm_mm != t->mm) in collect_procs_file()
645 vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { in collect_procs_fsdax()
646 if (vma->vm_mm == t->mm) in collect_procs_fsdax()
661 if (!folio->mapping) in collect_procs()
679 tk->addr = addr; in set_to_kill()
680 tk->size_shift = shift; in set_to_kill()
715 if (pfn <= hwp->pfn && hwp->pfn < pfn + HPAGE_PMD_NR) { in check_hwpoisoned_pmd_entry()
716 hwpoison_vaddr = addr + ((hwp->pfn - pfn) << PAGE_SHIFT); in check_hwpoisoned_pmd_entry()
717 set_to_kill(&hwp->tk, hwpoison_vaddr, PAGE_SHIFT); in check_hwpoisoned_pmd_entry()
733 struct hwpoison_walk *hwp = walk->private; in hwpoison_pte_range()
738 ptl = pmd_trans_huge_lock(pmdp, walk->vma); in hwpoison_pte_range()
745 mapped_pte = ptep = pte_offset_map_lock(walk->vma->vm_mm, pmdp, in hwpoison_pte_range()
752 hwp->pfn, &hwp->tk); in hwpoison_pte_range()
767 struct hwpoison_walk *hwp = walk->private; in hwpoison_hugetlb_range()
768 struct hstate *h = hstate_vma(walk->vma); in hwpoison_hugetlb_range()
773 ptl = huge_pte_lock(h, walk->mm, ptep); in hwpoison_hugetlb_range()
774 pte = huge_ptep_get(walk->mm, addr, ptep); in hwpoison_hugetlb_range()
776 hwp->pfn, &hwp->tk); in hwpoison_hugetlb_range()
820 if (!p->mm) in kill_accessing_process()
821 return -EFAULT; in kill_accessing_process()
823 mmap_read_lock(p->mm); in kill_accessing_process()
824 ret = walk_page_range(p->mm, 0, TASK_SIZE, &hwpoison_walk_ops, in kill_accessing_process()
834 mmap_read_unlock(p->mm); in kill_accessing_process()
836 return ret > 0 ? -EHWPOISON : 0; in kill_accessing_process()
840 * MF_IGNORED - The m-f() handler marks the page as PG_hwpoisoned'ed.
846 * MF_FAILED - The m-f() handler marks the page as PG_hwpoisoned'ed.
850 * by the m-f() handler immediately.
852 * MF_DELAYED - The m-f() handler marks the page as PG_hwpoisoned'ed.
857 * MF_RECOVERED - The m-f() handler marks the page as PG_hwpoisoned'ed.
859 * the buddy system, or hole-punnched out of the file mapping.
870 [MF_MSG_KERNEL_HIGH_ORDER] = "high-order kernel page",
902 * complain when the folio is unpoison-and-freed. in delete_from_lru_cache()
919 return -EIO; in delete_from_lru_cache()
927 if (mapping->a_ops->error_remove_folio) { in truncate_error_folio()
928 int err = mapping->a_ops->error_remove_folio(mapping, folio); in truncate_error_folio()
955 /* Callback ->action() has to unlock the relevant page inside it. */
968 int count = page_count(p) - 1; in has_extra_refcount()
971 count -= folio_nr_pages(page_folio(p)); in has_extra_refcount()
975 page_to_pfn(p), action_page_types[ps->type], count); in has_extra_refcount()
995 * This is a catch-all in case we fail to make sense of the page state.
1041 * so is expected to have an extra refcount after error-handling. in me_pagecache_clean()
1078 mapping_set_error(mapping, -EIO); in me_pagecache_dirty()
1093 * - clear dirty bit to prevent IO
1094 * - remove from LRU
1095 * - but keep in the swap cache, so that when we return to it on
1144 * - Error on hugepage is contained in hugepage unit (not in raw page unit.)
1185 * A page state is defined by its current page->flags bits.
1250 mf_stats = &NODE_DATA(nid)->mf_stats; in update_per_node_mf_stats()
1253 ++mf_stats->ignored; in update_per_node_mf_stats()
1256 ++mf_stats->failed; in update_per_node_mf_stats()
1259 ++mf_stats->delayed; in update_per_node_mf_stats()
1262 ++mf_stats->recovered; in update_per_node_mf_stats()
1268 ++mf_stats->total; in update_per_node_mf_stats()
1288 return (result == MF_RECOVERED || result == MF_DELAYED) ? 0 : -EBUSY; in action_result()
1296 /* page p should be unlocked after returning from ps->action(). */ in page_action()
1297 result = ps->action(ps, p); in page_action()
1304 return action_result(pfn, ps->type, result); in page_action()
1363 if (!HWPoisonHandlable(&folio->page, flags)) in __get_hwpoison_page()
1364 return -EBUSY; in __get_hwpoison_page()
1395 ret = -EBUSY; in get_any_page()
1400 ret = -EIO; in get_any_page()
1403 } else if (ret == -EBUSY) { in get_any_page()
1412 ret = -EIO; in get_any_page()
1431 ret = -EIO; in get_any_page()
1434 if (ret == -EIO) in get_any_page()
1461 return -EHWPOISON; in __get_unpoison_page()
1467 * get_hwpoison_page() - Get refcount for memory error handling
1472 * error on it, after checking that the error page is in a well-defined state
1473 * (defined as a page-type we can successfully handle the memory error on it,
1479 * extra care for the error page's state (as done in __get_hwpoison_page()),
1487 * 1 on success for in-use pages in a well-defined state,
1488 * -EIO for pages on which we can not handle memory errors,
1489 * -EBUSY when get_hwpoison_page() has raced with page lifecycle
1491 * -EHWPOISON when the page is hwpoisoned and taken off from buddy.
1551 return -EBUSY; in unmap_poisoned_folio()
1560 return folio_mapped(folio) ? -EBUSY : 0; in unmap_poisoned_folio()
1576 * Here we are interested only in user-mapped pages, so skip any in hwpoison_user_mappings()
1594 * mapped in dirty form. This has to be done before try_to_unmap, in hwpoison_user_mappings()
1613 * struct page and all unmaps done we can decide if in hwpoison_user_mappings()
1618 * use a more force-full uncatchable kill to prevent in hwpoison_user_mappings()
1639 if ((p->flags.f & ps->mask) == ps->res) in identify_page_state()
1642 page_flags |= (p->flags.f & (1UL << PG_dirty)); in identify_page_state()
1644 if (!ps->mask) in identify_page_state()
1646 if ((page_flags & ps->mask) == ps->res) in identify_page_state()
1677 if (tk->size_shift) in unmap_and_kill()
1678 size = max(size, 1UL << tk->size_shift); in unmap_and_kill()
1682 * Unmap the largest mapping to avoid breaking up device-dax in unmap_and_kill()
1687 loff_t start = ((loff_t)index << PAGE_SHIFT) & ~(size - 1); in unmap_and_kill()
1721 return -EBUSY; in mf_generic_kill_procs()
1723 if (hwpoison_filter(&folio->page)) { in mf_generic_kill_procs()
1724 rc = -EOPNOTSUPP; in mf_generic_kill_procs()
1728 switch (pgmap->type) { in mf_generic_kill_procs()
1733 * with device-side memory. in mf_generic_kill_procs()
1735 rc = -ENXIO; in mf_generic_kill_procs()
1745 SetPageHWPoison(&folio->page); in mf_generic_kill_procs()
1748 * Unlike System-RAM there is no possibility to swap in a in mf_generic_kill_procs()
1754 collect_procs(folio, &folio->page, &to_kill, true); in mf_generic_kill_procs()
1756 unmap_and_kill(&to_kill, pfn, folio->mapping, folio->index, flags); in mf_generic_kill_procs()
1764 * mf_dax_kill_procs - Collect and kill processes who are using this file range
1785 return -EBUSY; in mf_dax_kill_procs()
1811 * constructing singly linked list from ->_hugetlb_hwpoison field of folio.
1820 return (struct llist_head *)&folio->_hugetlb_hwpoison; in raw_hwp_list_head()
1846 llist_for_each_entry(p, raw_hwp_head->first, node) { in is_raw_hwpoison_page_in_hugepage()
1847 if (page == p->page) { in is_raw_hwpoison_page_in_hugepage()
1867 SetPageHWPoison(p->page); in __folio_free_raw_hwp()
1869 num_poisoned_pages_sub(page_to_pfn(p->page), 1); in __folio_free_raw_hwp()
1881 int ret = folio_test_set_hwpoison(folio) ? -EHWPOISON : 0; in folio_set_hugetlb_hwpoison()
1889 return -EHWPOISON; in folio_set_hugetlb_hwpoison()
1891 llist_for_each_entry(p, head->first, node) { in folio_set_hugetlb_hwpoison()
1892 if (p->page == page) in folio_set_hugetlb_hwpoison()
1893 return -EHWPOISON; in folio_set_hugetlb_hwpoison()
1898 raw_hwp->page = page; in folio_set_hugetlb_hwpoison()
1899 llist_add(&raw_hwp->node, head); in folio_set_hugetlb_hwpoison()
1952 * 0 - free hugepage
1953 * 1 - in-use hugepage
1954 * 2 - not a hugepage
1955 * -EBUSY - the hugepage is busy (try to retry)
1956 * -EHWPOISON - the hugepage is already hwpoisoned
1979 ret = -EBUSY; in __get_huge_page_for_hwpoison()
1985 ret = -EHWPOISON; in __get_huge_page_for_hwpoison()
2009 * PageHWPoison) should be done in single hugetlb_lock range.
2025 } else if (res == -EHWPOISON) { in try_memory_failure_hugetlb()
2032 } else if (res == -EBUSY) { in try_memory_failure_hugetlb()
2050 return -EOPNOTSUPP; in try_memory_failure_hugetlb()
2068 page_flags = folio->flags.f; in try_memory_failure_hugetlb()
2102 int rc = -ENXIO; in memory_failure_dev_pagemap()
2113 rc = pgmap->ops->memory_failure(pgmap, pfn, 1, flags); in memory_failure_dev_pagemap()
2118 if (rc != -EOPNOTSUPP) in memory_failure_dev_pagemap()
2126 if (rc != -EOPNOTSUPP) in memory_failure_dev_pagemap()
2133 * been RDMA pinned, not much can be done for recovery.
2148 * memory_failure - Handle memory failure of a page.
2159 * detected by a background scrubber)
2165 * 0 - success,
2166 * -ENXIO - memory not managed by the kernel
2167 * -EOPNOTSUPP - hwpoison_filter() filtered the error event,
2168 * -EHWPOISON - the page was already poisoned, potentially
2170 * other negative values - failure.
2206 res = -ENXIO; in memory_failure()
2216 res = -EHWPOISON; in memory_failure()
2229 * 2) it's part of a non-compound high order page. in memory_failure()
2271 res = -EOPNOTSUPP; in memory_failure()
2292 res = -EHWPOISON; in memory_failure()
2303 * We ignore non-LRU pages for good reasons. in memory_failure()
2304 * - PG_locked is only well defined for LRU pages and a few others in memory_failure()
2305 * - to avoid races with __SetPageLocked() in memory_failure()
2306 * - to avoid races with __SetPageSlab*() (and more non-atomic ops) in memory_failure()
2315 * We're only intended to deal with the non-Compound page here. in memory_failure()
2328 page_flags = folio->flags.f; in memory_failure()
2357 folio->mapping == NULL) { in memory_failure()
2392 * memory_failure_queue - Schedule handling memory failure of a page.
2403 * detected by a background scrubber)
2418 raw_spin_lock_irqsave(&mf_cpu->lock, proc_flags); in memory_failure_queue()
2419 buffer_overflow = !kfifo_put(&mf_cpu->fifo, entry); in memory_failure_queue()
2421 schedule_work_on(smp_processor_id(), &mf_cpu->work); in memory_failure_queue()
2422 raw_spin_unlock_irqrestore(&mf_cpu->lock, proc_flags); in memory_failure_queue()
2439 raw_spin_lock_irqsave(&mf_cpu->lock, proc_flags); in memory_failure_work_func()
2440 gotten = kfifo_get(&mf_cpu->fifo, &entry); in memory_failure_work_func()
2441 raw_spin_unlock_irqrestore(&mf_cpu->lock, proc_flags); in memory_failure_work_func()
2458 raw_spin_lock_init(&mf_cpu->lock); in memory_failure_init()
2459 INIT_KFIFO(mf_cpu->fifo); in memory_failure_init()
2460 INIT_WORK(&mf_cpu->work, memory_failure_work_func); in memory_failure_init()
2478 * unpoison_memory - Unpoison a previously poisoned page
2481 * Software-unpoison a page that has been poisoned by
2484 * This is only done on the software-level, so it only works
2487 * Returns 0 for success, otherwise -errno.
2493 int ret = -EBUSY, ghp; in unpoison_memory()
2501 return -EIO; in unpoison_memory()
2509 ret = -EOPNOTSUPP; in unpoison_memory()
2516 ret = -EOPNOTSUPP; in unpoison_memory()
2543 unpoison_pr_info("%#lx: the hwpoison page has non-NULL mapping\n", in unpoison_memory()
2556 ret = folio_test_clear_hwpoison(folio) ? 0 : -EBUSY; in unpoison_memory()
2558 if (ghp == -EHWPOISON) { in unpoison_memory()
2559 ret = put_page_back_buddy(p) ? 0 : -EBUSY; in unpoison_memory()
2587 unpoison_pr_info("%#lx: software-unpoisoned page\n", in unpoison_memory()
2598 * soft_offline_in_use_page handles hugetlb-pages and non-hugetlb pages.
2599 * If the page is a non-dirty unmapped page-cache page, it simply invalidates.
2620 return -EBUSY; in soft_offline_in_use_page()
2667 ret = -EBUSY; in soft_offline_in_use_page()
2673 pfn, msg_page[huge], ret, &page->flags.f); in soft_offline_in_use_page()
2675 ret = -EBUSY; in soft_offline_in_use_page()
2679 pfn, msg_page[huge], page_count(page), &page->flags.f); in soft_offline_in_use_page()
2680 ret = -EBUSY; in soft_offline_in_use_page()
2686 * soft_offline_page - Soft offline a page.
2687 * @pfn: pfn to soft-offline
2691 * -EOPNOTSUPP for hwpoison_filter() filtered the error event, or
2718 return -ENXIO; in soft_offline_page()
2721 /* Only online pages can be soft-offlined (esp., not ZONE_DEVICE). */ in soft_offline_page()
2725 return -EIO; in soft_offline_page()
2731 return -EOPNOTSUPP; in soft_offline_page()
2753 return -EOPNOTSUPP; in soft_offline_page()
2765 ret = -EBUSY; in soft_offline_page()