Lines Matching full:page
13 * Handles page cache pages in various states. The tricky part
14 * here is that we can access any page asynchronously in respect to
26 * - The case actually shows up as a frequent (top 10) page state in
27 * tools/mm/page-types when running a real workload.
42 #include <linux/page-flags.h>
179 * 1: the page is dissolved (if needed) and taken off from buddy,
180 * 0: the page is dissolved (if needed) and not taken off from buddy,
183 static int __page_handle_poison(struct page *page) in __page_handle_poison() argument
187 zone_pcp_disable(page_zone(page)); in __page_handle_poison()
188 ret = dissolve_free_hugetlb_folio(page_folio(page)); in __page_handle_poison()
190 ret = take_page_off_buddy(page); in __page_handle_poison()
191 zone_pcp_enable(page_zone(page)); in __page_handle_poison()
196 static bool page_handle_poison(struct page *page, bool hugepage_or_freepage, bool release) in page_handle_poison() argument
203 if (__page_handle_poison(page) <= 0) in page_handle_poison()
205 * We could fail to take off the target page from buddy in page_handle_poison()
206 * for example due to racy page allocation, but that's in page_handle_poison()
207 * acceptable because soft-offlined page is not broken in page_handle_poison()
214 SetPageHWPoison(page); in page_handle_poison()
216 put_page(page); in page_handle_poison()
217 page_ref_inc(page); in page_handle_poison()
218 num_poisoned_pages_inc(page_to_pfn(page)); in page_handle_poison()
238 static int hwpoison_filter(struct page *p) in hwpoison_filter()
253 * Kill all processes that have a poisoned page mapped and then isolate
254 * the page.
257 * Find all processes having the page mapped and kill them.
258 * But we keep a page reference around so that the page is not
260 * Then stash the page away
282 * Send all the processes who have the page mapped a signal.
300 * Signal other processes sharing the page if they have in kill_proc()
314 * Unknown page type encountered. Try to check whether it can turn PageLRU by
332 static void shake_page(struct page *page) in shake_page() argument
334 shake_folio(page_folio(page)); in shake_page()
384 static void __add_to_kill(struct task_struct *tsk, const struct page *p, in __add_to_kill()
406 * ZONE_DEVICE. Indeed, when a devdax page is mmapped N times in __add_to_kill()
408 * contain mappings for the page, but at least one VMA does. in __add_to_kill()
410 * has a mapping for the page. in __add_to_kill()
425 static void add_to_kill_anon_file(struct task_struct *tsk, const struct page *p, in add_to_kill_anon_file()
448 void add_to_kill_ksm(struct task_struct *tsk, const struct page *p, in add_to_kill_ksm()
470 pr_err("%#lx: forcibly killing %s:%d because of failure to unmap corrupted page\n", in kill_procs()
518 * to be signaled when some page under the process is hwpoisoned.
525 * processes sharing the same error page,if the process is "early kill", the
543 * Collect processes when the error hit an anonymous page.
546 const struct page *page, struct list_head *to_kill, in collect_procs_anon() argument
557 pgoff = page_pgoff(folio, page); in collect_procs_anon()
571 addr = page_mapped_in_vma(page, vma); in collect_procs_anon()
572 add_to_kill_anon_file(t, page, vma, to_kill, addr); in collect_procs_anon()
580 * Collect processes when the error hit a file mapped page.
583 const struct page *page, struct list_head *to_kill, in collect_procs_file() argument
593 pgoff = page_pgoff(folio, page); in collect_procs_file()
603 * the page but the corrupted page is not necessarily in collect_procs_file()
610 addr = page_address_in_vma(folio, page, vma); in collect_procs_file()
611 add_to_kill_anon_file(t, page, vma, to_kill, addr); in collect_procs_file()
619 static void add_to_kill_fsdax(struct task_struct *tsk, const struct page *p, in add_to_kill_fsdax()
628 * Collect processes when the error hit a fsdax page.
630 static void collect_procs_fsdax(const struct page *page, in collect_procs_fsdax() argument
644 * the current may not be the one accessing the fsdax page. in collect_procs_fsdax()
653 add_to_kill_fsdax(t, page, vma, to_kill, pgoff); in collect_procs_fsdax()
662 * Collect the processes who have the corrupted page mapped to kill.
664 static void collect_procs(const struct folio *folio, const struct page *page, in collect_procs() argument
670 collect_procs_ksm(folio, page, tokill, force_early); in collect_procs()
672 collect_procs_anon(folio, page, tokill, force_early); in collect_procs()
674 collect_procs_file(folio, page, tokill, force_early); in collect_procs()
813 * memory_failure() failed to unmap the error page at the first call, or
817 * so this function walks page table to find it. The returned virtual address
819 * process has multiple entries mapping the error page.
839 * ret = 0 when poison page is a clean page and it's dropped, no in kill_accessing_process()
850 * MF_IGNORED - The m-f() handler marks the page as PG_hwpoisoned'ed.
851 * But it could not do more to isolate the page from being accessed again,
853 * potential causes is that the page state has been changed due to
856 * MF_FAILED - The m-f() handler marks the page as PG_hwpoisoned'ed.
857 * It should have killed the process, but it can't isolate the page,
859 * the page again may trigger another MCE and the process will be killed
862 * MF_DELAYED - The m-f() handler marks the page as PG_hwpoisoned'ed.
863 * The page is unmapped, and is removed from the LRU or file mapping.
864 * An attempt to access the page again will trigger page fault and the
867 * MF_RECOVERED - The m-f() handler marks the page as PG_hwpoisoned'ed.
868 * The page has been completely isolated, that is, unmapped, taken out of
879 [MF_MSG_KERNEL] = "reserved kernel page",
880 [MF_MSG_KERNEL_HIGH_ORDER] = "high-order kernel page",
881 [MF_MSG_HUGE] = "huge page",
882 [MF_MSG_FREE_HUGE] = "free huge page",
883 [MF_MSG_GET_HWPOISON] = "get hwpoison page",
884 [MF_MSG_UNMAP_FAILED] = "unmapping failed page",
885 [MF_MSG_DIRTY_SWAPCACHE] = "dirty swapcache page",
886 [MF_MSG_CLEAN_SWAPCACHE] = "clean swapcache page",
887 [MF_MSG_DIRTY_MLOCKED_LRU] = "dirty mlocked LRU page",
888 [MF_MSG_CLEAN_MLOCKED_LRU] = "clean mlocked LRU page",
889 [MF_MSG_DIRTY_UNEVICTABLE_LRU] = "dirty unevictable LRU page",
890 [MF_MSG_CLEAN_UNEVICTABLE_LRU] = "clean unevictable LRU page",
891 [MF_MSG_DIRTY_LRU] = "dirty LRU page",
892 [MF_MSG_CLEAN_LRU] = "clean LRU page",
893 [MF_MSG_TRUNCATED_LRU] = "already truncated LRU page",
894 [MF_MSG_BUDDY] = "free buddy page",
895 [MF_MSG_DAX] = "dax page",
897 [MF_MSG_ALREADY_POISONED] = "already poisoned page",
898 [MF_MSG_PFN_MAP] = "non struct page pfn",
899 [MF_MSG_UNKNOWN] = "unknown page",
903 * XXX: It is possible that a page is isolated from LRU cache,
904 * and then kept in swap cache or failed to remove from page cache.
905 * The page count will stop it from being freed by unpoison.
912 * Clear sensible page flags, so that the buddy system won't in delete_from_lru_cache()
919 * Poisoned page might never drop its ref count to 0 so we have in delete_from_lru_cache()
942 pr_info("%#lx: Failed to punch page: %d\n", pfn, err); in truncate_error_folio()
966 /* Callback ->action() has to unlock the relevant page inside it. */
967 int (*action)(struct page_state *ps, struct page *p);
971 * Return true if page is still referenced by others, otherwise return
976 static bool has_extra_refcount(struct page_state *ps, struct page *p, in has_extra_refcount()
994 * Page in unknown state. Do nothing.
995 * This is a catch-all in case we fail to make sense of the page state.
997 static int me_unknown(struct page_state *ps, struct page *p) in me_unknown()
999 pr_err("%#lx: Unknown page state\n", page_to_pfn(p)); in me_unknown()
1005 * Clean (or cleaned) page cache page.
1007 static int me_pagecache_clean(struct page_state *ps, struct page *p) in me_pagecache_clean()
1026 * Now truncate the page in the page cache. This is really in me_pagecache_clean()
1040 * The shmem page is kept in page cache instead of truncating in me_pagecache_clean()
1061 * Dirty pagecache page
1062 * Issues: when the error hit a hole page the error is not properly
1065 static int me_pagecache_dirty(struct page_state *ps, struct page *p) in me_pagecache_dirty()
1087 * Dirty swap cache page is tricky to handle. The page could live both in page
1088 * table and swap cache(ie. page is freshly swapped in). So it could be
1096 * a later page fault, we know the application is accessing
1100 * Clean swap cache pages can be directly isolated. A later page fault will
1103 static int me_swapcache_dirty(struct page_state *ps, struct page *p) in me_swapcache_dirty()
1125 static int me_swapcache_clean(struct page_state *ps, struct page *p) in me_swapcache_clean()
1144 * - Error on hugepage is contained in hugepage unit (not in raw page unit.)
1145 * To narrow down kill region to one page, we need to break up pmd.
1147 static int me_huge_page(struct page_state *ps, struct page *p) in me_huge_page()
1157 /* The page is kept in page cache. */ in me_huge_page()
1183 * Various page states we can handle.
1185 * A page state is defined by its current page->flags bits.
1188 * This is quite tricky because we can access page at any time
1279 * setting PG_dirty outside page lock. See also comment above set_page_dirty().
1295 panic("Memory failure: %#lx: unrecoverable page", pfn); in action_result()
1300 static int page_action(struct page_state *ps, struct page *p, in page_action()
1305 /* page p should be unlocked after returning from ps->action(). */ in page_action()
1308 /* Could do more checks here if page looks ok */ in page_action()
1310 * Could adjust zone counters here to correct for the missing page. in page_action()
1316 static inline bool PageHWPoisonTakenOff(struct page *page) in PageHWPoisonTakenOff() argument
1318 return PageHWPoison(page) && page_private(page) == MAGIC_HWPOISON; in PageHWPoisonTakenOff()
1321 void SetPageHWPoisonTakenOff(struct page *page) in SetPageHWPoisonTakenOff() argument
1323 set_page_private(page, MAGIC_HWPOISON); in SetPageHWPoisonTakenOff()
1326 void ClearPageHWPoisonTakenOff(struct page *page) in ClearPageHWPoisonTakenOff() argument
1328 if (PageHWPoison(page)) in ClearPageHWPoisonTakenOff()
1329 set_page_private(page, 0); in ClearPageHWPoisonTakenOff()
1333 * Return true if a page type of a given page is supported by hwpoison
1338 static inline bool HWPoisonHandlable(struct page *page, unsigned long flags) in HWPoisonHandlable() argument
1340 if (PageSlab(page)) in HWPoisonHandlable()
1344 if ((flags & MF_SOFT_OFFLINE) && page_has_movable_ops(page)) in HWPoisonHandlable()
1347 return PageLRU(page) || is_free_buddy_page(page); in HWPoisonHandlable()
1355 static inline bool is_kernel_owned_page(struct page *page) in is_kernel_owned_page() argument
1357 struct page *head; in is_kernel_owned_page()
1360 /* PG_reserved is a per-page flag, never set on a compound page. */ in is_kernel_owned_page()
1361 if (PageReserved(page)) in is_kernel_owned_page()
1365 * Page-type bits live only on the head page, so resolve any tail in is_kernel_owned_page()
1371 * already set PageHWPoison, which bars the page from the allocator. in is_kernel_owned_page()
1374 head = compound_head(page); in is_kernel_owned_page()
1377 if (head != compound_head(page)) in is_kernel_owned_page()
1382 static int __get_hwpoison_page(struct page *page, unsigned long flags) in __get_hwpoison_page() argument
1384 struct folio *folio = page_folio(page); in __get_hwpoison_page()
1391 if (folio == page_folio(page)) in __get_hwpoison_page()
1395 folio = page_folio(page); in __get_hwpoison_page()
1404 if (!HWPoisonHandlable(&folio->page, flags)) in __get_hwpoison_page()
1408 if (folio == page_folio(page)) in __get_hwpoison_page()
1411 pr_info("%#lx cannot catch tail\n", page_to_pfn(page)); in __get_hwpoison_page()
1420 static int get_any_page(struct page *p, unsigned long flags) in get_any_page()
1429 * Page types we know are kernel-owned and cannot be recovered. in get_any_page()
1460 * page, retry. in get_any_page()
1475 * A page we cannot handle. Check whether we can turn in get_any_page()
1489 pr_err("%#lx: unhandlable page.\n", page_to_pfn(p)); in get_any_page()
1494 static int __get_unpoison_page(struct page *page) in __get_unpoison_page() argument
1496 struct folio *folio = page_folio(page); in __get_unpoison_page()
1503 if (folio == page_folio(page)) in __get_unpoison_page()
1514 if (PageHWPoisonTakenOff(page)) in __get_unpoison_page()
1517 return get_page_unless_zero(page) ? 1 : 0; in __get_unpoison_page()
1522 * @p: Raw error page (hit by memory error)
1525 * get_hwpoison_page() takes a page refcount of an error page to handle memory
1526 * error on it, after checking that the error page is in a well-defined state
1527 * (defined as a page-type we can successfully handle the memory error on it,
1528 * such as LRU page and hugetlb page).
1530 * Memory error handling could be triggered at any time on any type of page,
1533 * extra care for the error page's state (as done in __get_hwpoison_page()),
1537 * the given page has PG_hwpoison. So it's never reused for other page
1540 * Return: 0 on failure or free buddy (hugetlb) page,
1543 * -EBUSY when get_hwpoison_page() has raced with page lifecycle
1545 * -EHWPOISON when the page is hwpoisoned and taken off from buddy,
1548 * page-table, large-kmalloc) that the handler cannot recover.
1550 static int get_hwpoison_page(struct page *p, unsigned long flags) in get_hwpoison_page()
1574 pr_err("%#lx: keeping poisoned page in swap cache\n", pfn); in unmap_poisoned_folio()
1579 * Propagate the dirty bit from PTEs to struct page first, because we in unmap_poisoned_folio()
1580 * need this to decide if we should kill or just drop the page. in unmap_poisoned_folio()
1582 * be called inside page lock (it's recommended but not enforced). in unmap_poisoned_folio()
1591 pr_info("%#lx: corrupted page was clean: dropped without side effects\n", in unmap_poisoned_folio()
1624 static bool hwpoison_user_mappings(struct folio *folio, struct page *p, in hwpoison_user_mappings()
1650 * First collect all the processes that have the page in hwpoison_user_mappings()
1658 pr_err("%#lx: failed to unmap page (folio mapcount=%d)\n", in hwpoison_user_mappings()
1662 * try_to_unmap() might put mlocked page in lru cache, so call in hwpoison_user_mappings()
1670 * struct page and all unmaps done we can decide if in hwpoison_user_mappings()
1671 * killing is needed or not. Only kill when the page in hwpoison_user_mappings()
1685 static int identify_page_state(unsigned long pfn, struct page *p, in identify_page_state()
1691 * The first check uses the current page flags which may not have any in identify_page_state()
1692 * relevant information. The second check with the saved page flags is in identify_page_state()
1693 * carried out only if the first check can't determine the page status. in identify_page_state()
1710 * there is still more to do, hence the page refcount we took earlier
1713 static int try_to_split_thp_page(struct page *page, unsigned int new_order, in try_to_split_thp_page() argument
1718 lock_page(page); in try_to_split_thp_page()
1719 ret = split_huge_page_to_order(page, new_order); in try_to_split_thp_page()
1720 unlock_page(page); in try_to_split_thp_page()
1723 put_page(page); in try_to_split_thp_page()
1756 * The fsdax pages are initialized per base page, and the devdax pages
1759 * hwpoison, such that, if a subpage of a compound page is poisoned,
1760 * simply mark the compound head page is by far sufficient.
1773 * lock_page(), but dax pages do not use the page lock. This in mf_generic_kill_procs()
1781 if (hwpoison_filter(&folio->page)) { in mf_generic_kill_procs()
1800 * Use this flag as an indication that the dax page has been in mf_generic_kill_procs()
1803 SetPageHWPoison(&folio->page); in mf_generic_kill_procs()
1807 * different physical page at a given virtual address, so all in mf_generic_kill_procs()
1812 collect_procs(folio, &folio->page, &to_kill, true); in mf_generic_kill_procs()
1833 struct page *page; in mf_dax_kill_procs() local
1840 page = NULL; in mf_dax_kill_procs()
1841 cookie = dax_lock_mapping_entry(mapping, index, &page); in mf_dax_kill_procs()
1844 if (!page) in mf_dax_kill_procs()
1848 SetPageHWPoison(page); in mf_dax_kill_procs()
1854 collect_procs_fsdax(page, mapping, index, &to_kill, pre_remove); in mf_dax_kill_procs()
1855 unmap_and_kill(&to_kill, page_to_pfn(page), mapping, in mf_dax_kill_procs()
1868 * Struct raw_hwp_page represents information about "raw error page",
1873 struct page *page; member
1881 bool is_raw_hwpoison_page_in_hugepage(struct page *page) in is_raw_hwpoison_page_in_hugepage() argument
1885 struct folio *folio = page_folio(page); in is_raw_hwpoison_page_in_hugepage()
1892 return PageHWPoison(page); in is_raw_hwpoison_page_in_hugepage()
1905 if (page == p->page) { in is_raw_hwpoison_page_in_hugepage()
1925 SetPageHWPoison(p->page); in __folio_free_raw_hwp()
1927 num_poisoned_pages_sub(page_to_pfn(p->page), 1); in __folio_free_raw_hwp()
1938 #define MF_HUGETLB_PAGE_PRE_POISONED 4 /* exact page already poisoned */
1944 static int hugetlb_update_hwpoison(struct folio *folio, struct page *page) in hugetlb_update_hwpoison() argument
1960 if (p->page == page) in hugetlb_update_hwpoison()
1966 raw_hwp->page = page; in hugetlb_update_hwpoison()
2016 struct page *page = pfn_to_page(pfn); in get_huge_page_for_hwpoison() local
2022 folio = page_folio(page); in get_huge_page_for_hwpoison()
2044 rc = hugetlb_update_hwpoison(folio, page); in get_huge_page_for_hwpoison()
2075 * -ENOENT - no hugetlb page
2078 * -EHWPOISON - folio or exact page already poisoned
2084 struct page *p = pfn_to_page(pfn); in try_memory_failure_hugetlb()
2092 case MF_HUGETLB_NON_HUGEPAGE: /* fallback to normal page handling */ in try_memory_failure_hugetlb()
2207 * The calling condition is as such: thp split failed, page might have
2213 static void kill_procs_now(struct page *p, unsigned long pfn, int flags, in kill_procs_now()
2267 /* Check for pgoff not backed by struct page */ in add_to_kill_pgoff()
2281 * Collect processes when the error hit a PFN not backed by struct page.
2311 * memory_failure_pfn - Handle memory failure on a page not backed by
2312 * struct page.
2313 * @pfn: Page Number of the corrupted page
2318 * -EBUSY - Page PFN does not belong to any address space mapping.
2350 * physical page at a given virtual address, so all userspace in memory_failure_pfn()
2362 * memory_failure - Handle memory failure of a page.
2363 * @pfn: Page Number of the corrupted page
2368 * of a page. It tries its best to recover, which includes
2382 * -EHWPOISON - the page was already poisoned, potentially
2388 struct page *p; in memory_failure()
2396 panic("Memory failure on page %lx", pfn); in memory_failure()
2411 * The PFN is not backed by struct page. in memory_failure()
2434 * -ENOENT means the page we found is not hugetlb, so proceed with normal page handling in memory_failure()
2451 * 1) it's a free page, and therefore in safe hand: in memory_failure()
2453 * 2) it's part of a non-compound high order page. in memory_failure()
2455 * R/W the page; let's pray that the page has been in memory_failure()
2457 * In fact it's dangerous to directly bump up page count from 0, in memory_failure()
2482 /* Got a refcount on a handlable page. */ in memory_failure()
2486 * Stable unhandlable kernel-owned page (PG_reserved, in memory_failure()
2487 * slab, page tables, large-kmalloc). in memory_failure()
2493 /* Transient lifecycle race with the page allocator. */ in memory_failure()
2524 * get_hwpoison_page() since they handle either free page in memory_failure()
2525 * or unhandlable page. The refcount is bumped iff the in memory_failure()
2526 * page is a valid handlable page. in memory_failure()
2555 * walked by the page reclaim code, however that's not a big loss. in memory_failure()
2562 * We're only intended to deal with the non-Compound page here. in memory_failure()
2563 * The page cannot become compound pages again as folio has been in memory_failure()
2569 * We use page flags to determine what action should be taken, but in memory_failure()
2571 * example is an mlocked page, where PG_mlocked is cleared by in memory_failure()
2572 * folio_remove_rmap_*() in try_to_unmap_one(). So to determine page in memory_failure()
2573 * status correctly, we save a copy of the page flags at this time. in memory_failure()
2593 * Abort on fail: __filemap_remove_folio() assumes unmapped page. in memory_failure()
2639 * memory_failure_queue - Schedule handling memory failure of a page.
2640 * @pfn: Page Number of the corrupted page
2644 * when it detects hardware memory corruption of a page. It schedules
2645 * the recovering of error page, including dropping pages, killing
2725 * unpoison_memory - Unpoison a previously poisoned page
2726 * @pfn: Page number of the to be unpoisoned page
2728 * Software-unpoison a page that has been poisoned by
2739 struct page *p; in unpoison_memory()
2761 unpoison_pr_info("%#lx: huge zero page is not supported\n", in unpoison_memory()
2768 unpoison_pr_info("%#lx: page was already unpoisoned\n", in unpoison_memory()
2774 unpoison_pr_info("%#lx: someone grabs the hwpoison page\n", in unpoison_memory()
2784 unpoison_pr_info("%#lx: someone maps the hwpoison page\n", in unpoison_memory()
2790 unpoison_pr_info("%#lx: the hwpoison page has non-NULL mapping\n", in unpoison_memory()
2809 unpoison_pr_info("%#lx: failed to grab page\n", in unpoison_memory()
2834 unpoison_pr_info("%#lx: software-unpoisoned page\n", in unpoison_memory()
2846 * If the page is a non-dirty unmapped page-cache page, it simply invalidates.
2847 * If the page is mapped, it migrates the contents over.
2849 static int soft_offline_in_use_page(struct page *page) in soft_offline_in_use_page() argument
2852 unsigned long pfn = page_to_pfn(page); in soft_offline_in_use_page()
2853 struct folio *folio = page_folio(page); in soft_offline_in_use_page()
2854 char const *msg_page[] = {"page", "hugepage"}; in soft_offline_in_use_page()
2874 if (new_order || try_to_split_thp_page(page, /* new_order= */ 0, in soft_offline_in_use_page()
2879 folio = page_folio(page); in soft_offline_in_use_page()
2885 if (PageHWPoison(page)) { in soft_offline_in_use_page()
2888 pr_info("%#lx: page already poisoned\n", pfn); in soft_offline_in_use_page()
2895 * non dirty unmapped page cache pages. in soft_offline_in_use_page()
2902 page_handle_poison(page, false, true); in soft_offline_in_use_page()
2923 if (!page_handle_poison(page, huge, release)) in soft_offline_in_use_page()
2930 pfn, msg_page[huge], ret, &page->flags.f); in soft_offline_in_use_page()
2935 pr_info("%#lx: %s isolation failed, page count %d, type %pGp\n", in soft_offline_in_use_page()
2936 pfn, msg_page[huge], page_count(page), &page->flags.f); in soft_offline_in_use_page()
2943 * soft_offline_page - Soft offline a page.
2952 * Soft offline a page, by migration or invalidation,
2954 * a page is not corrupted yet (so it's still valid to access),
2971 struct page *page; in soft_offline_page() local
2979 page = pfn_to_online_page(pfn); in soft_offline_page()
2980 if (!page) { in soft_offline_page()
2993 if (PageHWPoison(page)) { in soft_offline_page()
2994 pr_info("%#lx: page already poisoned\n", pfn); in soft_offline_page()
3002 ret = get_hwpoison_page(page, flags | MF_SOFT_OFFLINE); in soft_offline_page()
3005 if (hwpoison_filter(page)) { in soft_offline_page()
3007 put_page(page); in soft_offline_page()
3014 ret = soft_offline_in_use_page(page); in soft_offline_page()
3016 if (!page_handle_poison(page, true, false)) { in soft_offline_page()