1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 257c8a661SMike Rapoport #include <linux/memblock.h> 36d80e53fSAlexey Dobriyan #include <linux/compiler.h> 46d80e53fSAlexey Dobriyan #include <linux/fs.h> 56d80e53fSAlexey Dobriyan #include <linux/init.h> 69a840895SHugh Dickins #include <linux/ksm.h> 76d80e53fSAlexey Dobriyan #include <linux/mm.h> 86d80e53fSAlexey Dobriyan #include <linux/mmzone.h> 956873f43SWang, Yalin #include <linux/huge_mm.h> 106d80e53fSAlexey Dobriyan #include <linux/proc_fs.h> 116d80e53fSAlexey Dobriyan #include <linux/seq_file.h> 1220a0307cSWu Fengguang #include <linux/hugetlb.h> 13dc90f084SChristoph Hellwig #include <linux/memremap.h> 1480ae2fdcSVladimir Davydov #include <linux/memcontrol.h> 1533c3fc71SVladimir Davydov #include <linux/mmu_notifier.h> 1633c3fc71SVladimir Davydov #include <linux/page_idle.h> 171a9b5b7fSWu Fengguang #include <linux/kernel-page-flags.h> 187c0f6ba6SLinus Torvalds #include <linux/uaccess.h> 196d80e53fSAlexey Dobriyan #include "internal.h" 206d80e53fSAlexey Dobriyan 216d80e53fSAlexey Dobriyan #define KPMSIZE sizeof(u64) 226d80e53fSAlexey Dobriyan #define KPMMASK (KPMSIZE - 1) 2333c3fc71SVladimir Davydov #define KPMBITS (KPMSIZE * BITS_PER_BYTE) 24ed7ce0f1SWu Fengguang 25abec749fSDavid Hildenbrand static inline unsigned long get_max_dump_pfn(void) 26abec749fSDavid Hildenbrand { 27abec749fSDavid Hildenbrand #ifdef CONFIG_SPARSEMEM 28abec749fSDavid Hildenbrand /* 29abec749fSDavid Hildenbrand * The memmap of early sections is completely populated and marked 30abec749fSDavid Hildenbrand * online even if max_pfn does not fall on a section boundary - 31abec749fSDavid Hildenbrand * pfn_to_online_page() will succeed on all pages. Allow inspecting 32abec749fSDavid Hildenbrand * these memmaps. 33abec749fSDavid Hildenbrand */ 34abec749fSDavid Hildenbrand return round_up(max_pfn, PAGES_PER_SECTION); 35abec749fSDavid Hildenbrand #else 36abec749fSDavid Hildenbrand return max_pfn; 37abec749fSDavid Hildenbrand #endif 38abec749fSDavid Hildenbrand } 39abec749fSDavid Hildenbrand 40cdd9a571SDavid Hildenbrand /* /proc/kpagecount - an array exposing page mapcounts 416d80e53fSAlexey Dobriyan * 426d80e53fSAlexey Dobriyan * Each entry is a u64 representing the corresponding 43cdd9a571SDavid Hildenbrand * physical page mapcount. 446d80e53fSAlexey Dobriyan */ 456d80e53fSAlexey Dobriyan static ssize_t kpagecount_read(struct file *file, char __user *buf, 466d80e53fSAlexey Dobriyan size_t count, loff_t *ppos) 476d80e53fSAlexey Dobriyan { 48abec749fSDavid Hildenbrand const unsigned long max_dump_pfn = get_max_dump_pfn(); 496d80e53fSAlexey Dobriyan u64 __user *out = (u64 __user *)buf; 506d80e53fSAlexey Dobriyan unsigned long src = *ppos; 516d80e53fSAlexey Dobriyan unsigned long pfn; 526d80e53fSAlexey Dobriyan ssize_t ret = 0; 536d80e53fSAlexey Dobriyan 546d80e53fSAlexey Dobriyan pfn = src / KPMSIZE; 556d80e53fSAlexey Dobriyan if (src & KPMMASK || count & KPMMASK) 566d80e53fSAlexey Dobriyan return -EINVAL; 57abec749fSDavid Hildenbrand if (src >= max_dump_pfn * KPMSIZE) 58abec749fSDavid Hildenbrand return 0; 59abec749fSDavid Hildenbrand count = min_t(unsigned long, count, (max_dump_pfn * KPMSIZE) - src); 606d80e53fSAlexey Dobriyan 616d80e53fSAlexey Dobriyan while (count > 0) { 62cdd9a571SDavid Hildenbrand struct page *page; 63cdd9a571SDavid Hildenbrand u64 mapcount = 0; 64cdd9a571SDavid Hildenbrand 65aad5f69bSDavid Hildenbrand /* 66aad5f69bSDavid Hildenbrand * TODO: ZONE_DEVICE support requires to identify 67aad5f69bSDavid Hildenbrand * memmaps that were actually initialized. 68aad5f69bSDavid Hildenbrand */ 69cdd9a571SDavid Hildenbrand page = pfn_to_online_page(pfn); 70cdd9a571SDavid Hildenbrand if (page) 71cdd9a571SDavid Hildenbrand mapcount = folio_precise_page_mapcount(page_folio(page), 72cdd9a571SDavid Hildenbrand page); 73aad5f69bSDavid Hildenbrand 74cdd9a571SDavid Hildenbrand if (put_user(mapcount, out)) { 756d80e53fSAlexey Dobriyan ret = -EFAULT; 766d80e53fSAlexey Dobriyan break; 776d80e53fSAlexey Dobriyan } 786d80e53fSAlexey Dobriyan 79ed7ce0f1SWu Fengguang pfn++; 80ed7ce0f1SWu Fengguang out++; 816d80e53fSAlexey Dobriyan count -= KPMSIZE; 82d3691d2cSVladimir Davydov 83d3691d2cSVladimir Davydov cond_resched(); 846d80e53fSAlexey Dobriyan } 856d80e53fSAlexey Dobriyan 866d80e53fSAlexey Dobriyan *ppos += (char __user *)out - buf; 876d80e53fSAlexey Dobriyan if (!ret) 886d80e53fSAlexey Dobriyan ret = (char __user *)out - buf; 896d80e53fSAlexey Dobriyan return ret; 906d80e53fSAlexey Dobriyan } 916d80e53fSAlexey Dobriyan 9297a32539SAlexey Dobriyan static const struct proc_ops kpagecount_proc_ops = { 93ef1d6178SAlexey Dobriyan .proc_flags = PROC_ENTRY_PERMANENT, 9497a32539SAlexey Dobriyan .proc_lseek = mem_lseek, 9597a32539SAlexey Dobriyan .proc_read = kpagecount_read, 966d80e53fSAlexey Dobriyan }; 976d80e53fSAlexey Dobriyan 986d80e53fSAlexey Dobriyan /* /proc/kpageflags - an array exposing page flags 996d80e53fSAlexey Dobriyan * 1006d80e53fSAlexey Dobriyan * Each entry is a u64 representing the corresponding 1016d80e53fSAlexey Dobriyan * physical page flags. 1026d80e53fSAlexey Dobriyan */ 1036d80e53fSAlexey Dobriyan 10417797549SWu Fengguang static inline u64 kpf_copy_bit(u64 kflags, int ubit, int kbit) 10517797549SWu Fengguang { 10617797549SWu Fengguang return ((kflags >> kbit) & 1) << ubit; 10717797549SWu Fengguang } 10817797549SWu Fengguang 109dee3d0beSMatthew Wilcox (Oracle) u64 stable_page_flags(const struct page *page) 11017797549SWu Fengguang { 111dee3d0beSMatthew Wilcox (Oracle) const struct folio *folio; 112dee3d0beSMatthew Wilcox (Oracle) unsigned long k; 113dee3d0beSMatthew Wilcox (Oracle) unsigned long mapping; 114dee3d0beSMatthew Wilcox (Oracle) bool is_anon; 115dee3d0beSMatthew Wilcox (Oracle) u64 u = 0; 11617797549SWu Fengguang 11717797549SWu Fengguang /* 11817797549SWu Fengguang * pseudo flag: KPF_NOPAGE 11917797549SWu Fengguang * it differentiates a memory hole from a page with no flags 12017797549SWu Fengguang */ 12117797549SWu Fengguang if (!page) 12217797549SWu Fengguang return 1 << KPF_NOPAGE; 123dee3d0beSMatthew Wilcox (Oracle) folio = page_folio(page); 12417797549SWu Fengguang 125dee3d0beSMatthew Wilcox (Oracle) k = folio->flags; 126dee3d0beSMatthew Wilcox (Oracle) mapping = (unsigned long)folio->mapping; 127dee3d0beSMatthew Wilcox (Oracle) is_anon = mapping & PAGE_MAPPING_ANON; 12817797549SWu Fengguang 12917797549SWu Fengguang /* 13017797549SWu Fengguang * pseudo flags for the well known (anonymous) memory mapped pages 13117797549SWu Fengguang */ 132fd1a745cSMatthew Wilcox (Oracle) if (page_mapped(page)) 13317797549SWu Fengguang u |= 1 << KPF_MMAP; 134dee3d0beSMatthew Wilcox (Oracle) if (is_anon) { 13517797549SWu Fengguang u |= 1 << KPF_ANON; 136dee3d0beSMatthew Wilcox (Oracle) if (mapping & PAGE_MAPPING_KSM) 1379a840895SHugh Dickins u |= 1 << KPF_KSM; 138dee3d0beSMatthew Wilcox (Oracle) } 13917797549SWu Fengguang 14017797549SWu Fengguang /* 14117797549SWu Fengguang * compound pages: export both head/tail info 14217797549SWu Fengguang * they together define a compound page's start/end pos and order 14317797549SWu Fengguang */ 144dee3d0beSMatthew Wilcox (Oracle) if (page == &folio->page) 145dee3d0beSMatthew Wilcox (Oracle) u |= kpf_copy_bit(k, KPF_COMPOUND_HEAD, PG_head); 146dee3d0beSMatthew Wilcox (Oracle) else 14717797549SWu Fengguang u |= 1 << KPF_COMPOUND_TAIL; 148dee3d0beSMatthew Wilcox (Oracle) if (folio_test_hugetlb(folio)) 14917797549SWu Fengguang u |= 1 << KPF_HUGE; 150*4c8763e8SRan Xiaokai else if (folio_test_large(folio) && 151*4c8763e8SRan Xiaokai folio_test_large_rmappable(folio)) { 152*4c8763e8SRan Xiaokai /* Note: we indicate any THPs here, not just PMD-sized ones */ 153e873c49fSNaoya Horiguchi u |= 1 << KPF_THP; 154*4c8763e8SRan Xiaokai } else if (is_huge_zero_folio(folio)) { 15556873f43SWang, Yalin u |= 1 << KPF_ZERO_PAGE; 15656873f43SWang, Yalin u |= 1 << KPF_THP; 157*4c8763e8SRan Xiaokai } else if (is_zero_folio(folio)) { 158*4c8763e8SRan Xiaokai u |= 1 << KPF_ZERO_PAGE; 15956873f43SWang, Yalin } 16056873f43SWang, Yalin 1615f24ce5fSAndrea Arcangeli /* 162d88e2a2bSVlastimil Babka * Caveats on high order pages: PG_buddy and PG_slab will only be set 163d88e2a2bSVlastimil Babka * on the head page. 1645f24ce5fSAndrea Arcangeli */ 1655f24ce5fSAndrea Arcangeli if (PageBuddy(page)) 1665f24ce5fSAndrea Arcangeli u |= 1 << KPF_BUDDY; 167832fc1deSNaoya Horiguchi else if (page_count(page) == 0 && is_free_buddy_page(page)) 168832fc1deSNaoya Horiguchi u |= 1 << KPF_BUDDY; 1695f24ce5fSAndrea Arcangeli 170ca215086SDavid Hildenbrand if (PageOffline(page)) 171ca215086SDavid Hildenbrand u |= 1 << KPF_OFFLINE; 1721d40a5eaSMatthew Wilcox if (PageTable(page)) 1731d40a5eaSMatthew Wilcox u |= 1 << KPF_PGTABLE; 174dee3d0beSMatthew Wilcox (Oracle) if (folio_test_slab(folio)) 1750a71649cSNaoya Horiguchi u |= 1 << KPF_SLAB; 17617797549SWu Fengguang 177dee3d0beSMatthew Wilcox (Oracle) #if defined(CONFIG_PAGE_IDLE_FLAG) && defined(CONFIG_64BIT) 178dee3d0beSMatthew Wilcox (Oracle) u |= kpf_copy_bit(k, KPF_IDLE, PG_idle); 179dee3d0beSMatthew Wilcox (Oracle) #else 180dee3d0beSMatthew Wilcox (Oracle) if (folio_test_idle(folio)) 181dee3d0beSMatthew Wilcox (Oracle) u |= 1 << KPF_IDLE; 182dee3d0beSMatthew Wilcox (Oracle) #endif 183dee3d0beSMatthew Wilcox (Oracle) 184dee3d0beSMatthew Wilcox (Oracle) u |= kpf_copy_bit(k, KPF_LOCKED, PG_locked); 18517797549SWu Fengguang u |= kpf_copy_bit(k, KPF_ERROR, PG_error); 18617797549SWu Fengguang u |= kpf_copy_bit(k, KPF_DIRTY, PG_dirty); 18717797549SWu Fengguang u |= kpf_copy_bit(k, KPF_UPTODATE, PG_uptodate); 18817797549SWu Fengguang u |= kpf_copy_bit(k, KPF_WRITEBACK, PG_writeback); 18917797549SWu Fengguang 19017797549SWu Fengguang u |= kpf_copy_bit(k, KPF_LRU, PG_lru); 19117797549SWu Fengguang u |= kpf_copy_bit(k, KPF_REFERENCED, PG_referenced); 19217797549SWu Fengguang u |= kpf_copy_bit(k, KPF_ACTIVE, PG_active); 19317797549SWu Fengguang u |= kpf_copy_bit(k, KPF_RECLAIM, PG_reclaim); 19417797549SWu Fengguang 195dee3d0beSMatthew Wilcox (Oracle) #define SWAPCACHE ((1 << PG_swapbacked) | (1 << PG_swapcache)) 196dee3d0beSMatthew Wilcox (Oracle) if ((k & SWAPCACHE) == SWAPCACHE) 197b6789123SHugh Dickins u |= 1 << KPF_SWAPCACHE; 19817797549SWu Fengguang u |= kpf_copy_bit(k, KPF_SWAPBACKED, PG_swapbacked); 19917797549SWu Fengguang 20017797549SWu Fengguang u |= kpf_copy_bit(k, KPF_UNEVICTABLE, PG_unevictable); 20117797549SWu Fengguang u |= kpf_copy_bit(k, KPF_MLOCKED, PG_mlocked); 20217797549SWu Fengguang 203253fb02dSWu Fengguang #ifdef CONFIG_MEMORY_FAILURE 204dee3d0beSMatthew Wilcox (Oracle) if (u & (1 << KPF_HUGE)) 205253fb02dSWu Fengguang u |= kpf_copy_bit(k, KPF_HWPOISON, PG_hwpoison); 206dee3d0beSMatthew Wilcox (Oracle) else 207dee3d0beSMatthew Wilcox (Oracle) u |= kpf_copy_bit(page->flags, KPF_HWPOISON, PG_hwpoison); 208253fb02dSWu Fengguang #endif 209253fb02dSWu Fengguang 210ed430fecSTakashi Iwai #ifdef CONFIG_ARCH_USES_PG_UNCACHED 21117797549SWu Fengguang u |= kpf_copy_bit(k, KPF_UNCACHED, PG_uncached); 21217797549SWu Fengguang #endif 21317797549SWu Fengguang 21417797549SWu Fengguang u |= kpf_copy_bit(k, KPF_RESERVED, PG_reserved); 21517797549SWu Fengguang u |= kpf_copy_bit(k, KPF_MAPPEDTODISK, PG_mappedtodisk); 21617797549SWu Fengguang u |= kpf_copy_bit(k, KPF_PRIVATE, PG_private); 21717797549SWu Fengguang u |= kpf_copy_bit(k, KPF_PRIVATE_2, PG_private_2); 21817797549SWu Fengguang u |= kpf_copy_bit(k, KPF_OWNER_PRIVATE, PG_owner_priv_1); 21917797549SWu Fengguang u |= kpf_copy_bit(k, KPF_ARCH, PG_arch_1); 220b0284cd2SCatalin Marinas #ifdef CONFIG_ARCH_USES_PG_ARCH_X 2214beba948SSteven Price u |= kpf_copy_bit(k, KPF_ARCH_2, PG_arch_2); 222ef6458b1SPeter Collingbourne u |= kpf_copy_bit(k, KPF_ARCH_3, PG_arch_3); 2234beba948SSteven Price #endif 22417797549SWu Fengguang 22517797549SWu Fengguang return u; 22617797549SWu Fengguang }; 2276d80e53fSAlexey Dobriyan 2286d80e53fSAlexey Dobriyan static ssize_t kpageflags_read(struct file *file, char __user *buf, 2296d80e53fSAlexey Dobriyan size_t count, loff_t *ppos) 2306d80e53fSAlexey Dobriyan { 231abec749fSDavid Hildenbrand const unsigned long max_dump_pfn = get_max_dump_pfn(); 2326d80e53fSAlexey Dobriyan u64 __user *out = (u64 __user *)buf; 2336d80e53fSAlexey Dobriyan unsigned long src = *ppos; 2346d80e53fSAlexey Dobriyan unsigned long pfn; 2356d80e53fSAlexey Dobriyan ssize_t ret = 0; 2366d80e53fSAlexey Dobriyan 2376d80e53fSAlexey Dobriyan pfn = src / KPMSIZE; 2386d80e53fSAlexey Dobriyan if (src & KPMMASK || count & KPMMASK) 2396d80e53fSAlexey Dobriyan return -EINVAL; 240abec749fSDavid Hildenbrand if (src >= max_dump_pfn * KPMSIZE) 241abec749fSDavid Hildenbrand return 0; 242abec749fSDavid Hildenbrand count = min_t(unsigned long, count, (max_dump_pfn * KPMSIZE) - src); 2436d80e53fSAlexey Dobriyan 2446d80e53fSAlexey Dobriyan while (count > 0) { 245aad5f69bSDavid Hildenbrand /* 246aad5f69bSDavid Hildenbrand * TODO: ZONE_DEVICE support requires to identify 247aad5f69bSDavid Hildenbrand * memmaps that were actually initialized. 248aad5f69bSDavid Hildenbrand */ 249dee3d0beSMatthew Wilcox (Oracle) struct page *page = pfn_to_online_page(pfn); 2506d80e53fSAlexey Dobriyan 251dee3d0beSMatthew Wilcox (Oracle) if (put_user(stable_page_flags(page), out)) { 2526d80e53fSAlexey Dobriyan ret = -EFAULT; 2536d80e53fSAlexey Dobriyan break; 2546d80e53fSAlexey Dobriyan } 2556d80e53fSAlexey Dobriyan 256ed7ce0f1SWu Fengguang pfn++; 257ed7ce0f1SWu Fengguang out++; 2586d80e53fSAlexey Dobriyan count -= KPMSIZE; 259d3691d2cSVladimir Davydov 260d3691d2cSVladimir Davydov cond_resched(); 2616d80e53fSAlexey Dobriyan } 2626d80e53fSAlexey Dobriyan 2636d80e53fSAlexey Dobriyan *ppos += (char __user *)out - buf; 2646d80e53fSAlexey Dobriyan if (!ret) 2656d80e53fSAlexey Dobriyan ret = (char __user *)out - buf; 2666d80e53fSAlexey Dobriyan return ret; 2676d80e53fSAlexey Dobriyan } 2686d80e53fSAlexey Dobriyan 26997a32539SAlexey Dobriyan static const struct proc_ops kpageflags_proc_ops = { 270ef1d6178SAlexey Dobriyan .proc_flags = PROC_ENTRY_PERMANENT, 27197a32539SAlexey Dobriyan .proc_lseek = mem_lseek, 27297a32539SAlexey Dobriyan .proc_read = kpageflags_read, 2736d80e53fSAlexey Dobriyan }; 2746d80e53fSAlexey Dobriyan 27580ae2fdcSVladimir Davydov #ifdef CONFIG_MEMCG 27680ae2fdcSVladimir Davydov static ssize_t kpagecgroup_read(struct file *file, char __user *buf, 27780ae2fdcSVladimir Davydov size_t count, loff_t *ppos) 27880ae2fdcSVladimir Davydov { 279abec749fSDavid Hildenbrand const unsigned long max_dump_pfn = get_max_dump_pfn(); 28080ae2fdcSVladimir Davydov u64 __user *out = (u64 __user *)buf; 28180ae2fdcSVladimir Davydov struct page *ppage; 28280ae2fdcSVladimir Davydov unsigned long src = *ppos; 28380ae2fdcSVladimir Davydov unsigned long pfn; 28480ae2fdcSVladimir Davydov ssize_t ret = 0; 28580ae2fdcSVladimir Davydov u64 ino; 28680ae2fdcSVladimir Davydov 28780ae2fdcSVladimir Davydov pfn = src / KPMSIZE; 28880ae2fdcSVladimir Davydov if (src & KPMMASK || count & KPMMASK) 28980ae2fdcSVladimir Davydov return -EINVAL; 290abec749fSDavid Hildenbrand if (src >= max_dump_pfn * KPMSIZE) 291abec749fSDavid Hildenbrand return 0; 292abec749fSDavid Hildenbrand count = min_t(unsigned long, count, (max_dump_pfn * KPMSIZE) - src); 29380ae2fdcSVladimir Davydov 29480ae2fdcSVladimir Davydov while (count > 0) { 295aad5f69bSDavid Hildenbrand /* 296aad5f69bSDavid Hildenbrand * TODO: ZONE_DEVICE support requires to identify 297aad5f69bSDavid Hildenbrand * memmaps that were actually initialized. 298aad5f69bSDavid Hildenbrand */ 299aad5f69bSDavid Hildenbrand ppage = pfn_to_online_page(pfn); 30080ae2fdcSVladimir Davydov 30180ae2fdcSVladimir Davydov if (ppage) 30280ae2fdcSVladimir Davydov ino = page_cgroup_ino(ppage); 30380ae2fdcSVladimir Davydov else 30480ae2fdcSVladimir Davydov ino = 0; 30580ae2fdcSVladimir Davydov 30680ae2fdcSVladimir Davydov if (put_user(ino, out)) { 30780ae2fdcSVladimir Davydov ret = -EFAULT; 30880ae2fdcSVladimir Davydov break; 30980ae2fdcSVladimir Davydov } 31080ae2fdcSVladimir Davydov 31180ae2fdcSVladimir Davydov pfn++; 31280ae2fdcSVladimir Davydov out++; 31380ae2fdcSVladimir Davydov count -= KPMSIZE; 314d3691d2cSVladimir Davydov 315d3691d2cSVladimir Davydov cond_resched(); 31680ae2fdcSVladimir Davydov } 31780ae2fdcSVladimir Davydov 31880ae2fdcSVladimir Davydov *ppos += (char __user *)out - buf; 31980ae2fdcSVladimir Davydov if (!ret) 32080ae2fdcSVladimir Davydov ret = (char __user *)out - buf; 32180ae2fdcSVladimir Davydov return ret; 32280ae2fdcSVladimir Davydov } 32380ae2fdcSVladimir Davydov 32497a32539SAlexey Dobriyan static const struct proc_ops kpagecgroup_proc_ops = { 325ef1d6178SAlexey Dobriyan .proc_flags = PROC_ENTRY_PERMANENT, 32697a32539SAlexey Dobriyan .proc_lseek = mem_lseek, 32797a32539SAlexey Dobriyan .proc_read = kpagecgroup_read, 32880ae2fdcSVladimir Davydov }; 32980ae2fdcSVladimir Davydov #endif /* CONFIG_MEMCG */ 33080ae2fdcSVladimir Davydov 3316d80e53fSAlexey Dobriyan static int __init proc_page_init(void) 3326d80e53fSAlexey Dobriyan { 33397a32539SAlexey Dobriyan proc_create("kpagecount", S_IRUSR, NULL, &kpagecount_proc_ops); 33497a32539SAlexey Dobriyan proc_create("kpageflags", S_IRUSR, NULL, &kpageflags_proc_ops); 33580ae2fdcSVladimir Davydov #ifdef CONFIG_MEMCG 33697a32539SAlexey Dobriyan proc_create("kpagecgroup", S_IRUSR, NULL, &kpagecgroup_proc_ops); 33780ae2fdcSVladimir Davydov #endif 3386d80e53fSAlexey Dobriyan return 0; 3396d80e53fSAlexey Dobriyan } 340abaf3787SPaul Gortmaker fs_initcall(proc_page_init); 341