1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 233c3fc71SVladimir Davydov #include <linux/init.h> 3*57c8a661SMike Rapoport #include <linux/memblock.h> 433c3fc71SVladimir Davydov #include <linux/fs.h> 533c3fc71SVladimir Davydov #include <linux/sysfs.h> 633c3fc71SVladimir Davydov #include <linux/kobject.h> 733c3fc71SVladimir Davydov #include <linux/mm.h> 833c3fc71SVladimir Davydov #include <linux/mmzone.h> 933c3fc71SVladimir Davydov #include <linux/pagemap.h> 1033c3fc71SVladimir Davydov #include <linux/rmap.h> 1133c3fc71SVladimir Davydov #include <linux/mmu_notifier.h> 1233c3fc71SVladimir Davydov #include <linux/page_ext.h> 1333c3fc71SVladimir Davydov #include <linux/page_idle.h> 1433c3fc71SVladimir Davydov 1533c3fc71SVladimir Davydov #define BITMAP_CHUNK_SIZE sizeof(u64) 1633c3fc71SVladimir Davydov #define BITMAP_CHUNK_BITS (BITMAP_CHUNK_SIZE * BITS_PER_BYTE) 1733c3fc71SVladimir Davydov 1833c3fc71SVladimir Davydov /* 1933c3fc71SVladimir Davydov * Idle page tracking only considers user memory pages, for other types of 2033c3fc71SVladimir Davydov * pages the idle flag is always unset and an attempt to set it is silently 2133c3fc71SVladimir Davydov * ignored. 2233c3fc71SVladimir Davydov * 2333c3fc71SVladimir Davydov * We treat a page as a user memory page if it is on an LRU list, because it is 2433c3fc71SVladimir Davydov * always safe to pass such a page to rmap_walk(), which is essential for idle 2533c3fc71SVladimir Davydov * page tracking. With such an indicator of user pages we can skip isolated 2633c3fc71SVladimir Davydov * pages, but since there are not usually many of them, it will hardly affect 2733c3fc71SVladimir Davydov * the overall result. 2833c3fc71SVladimir Davydov * 2933c3fc71SVladimir Davydov * This function tries to get a user memory page by pfn as described above. 3033c3fc71SVladimir Davydov */ 3133c3fc71SVladimir Davydov static struct page *page_idle_get_page(unsigned long pfn) 3233c3fc71SVladimir Davydov { 3333c3fc71SVladimir Davydov struct page *page; 3433c3fc71SVladimir Davydov struct zone *zone; 3533c3fc71SVladimir Davydov 3633c3fc71SVladimir Davydov if (!pfn_valid(pfn)) 3733c3fc71SVladimir Davydov return NULL; 3833c3fc71SVladimir Davydov 3933c3fc71SVladimir Davydov page = pfn_to_page(pfn); 4033c3fc71SVladimir Davydov if (!page || !PageLRU(page) || 4133c3fc71SVladimir Davydov !get_page_unless_zero(page)) 4233c3fc71SVladimir Davydov return NULL; 4333c3fc71SVladimir Davydov 4433c3fc71SVladimir Davydov zone = page_zone(page); 45a52633d8SMel Gorman spin_lock_irq(zone_lru_lock(zone)); 4633c3fc71SVladimir Davydov if (unlikely(!PageLRU(page))) { 4733c3fc71SVladimir Davydov put_page(page); 4833c3fc71SVladimir Davydov page = NULL; 4933c3fc71SVladimir Davydov } 50a52633d8SMel Gorman spin_unlock_irq(zone_lru_lock(zone)); 5133c3fc71SVladimir Davydov return page; 5233c3fc71SVladimir Davydov } 5333c3fc71SVladimir Davydov 54e4b82222SMinchan Kim static bool page_idle_clear_pte_refs_one(struct page *page, 5533c3fc71SVladimir Davydov struct vm_area_struct *vma, 5633c3fc71SVladimir Davydov unsigned long addr, void *arg) 5733c3fc71SVladimir Davydov { 58699fa216SKirill A. Shutemov struct page_vma_mapped_walk pvmw = { 59699fa216SKirill A. Shutemov .page = page, 60699fa216SKirill A. Shutemov .vma = vma, 61699fa216SKirill A. Shutemov .address = addr, 62699fa216SKirill A. Shutemov }; 6333c3fc71SVladimir Davydov bool referenced = false; 6433c3fc71SVladimir Davydov 65699fa216SKirill A. Shutemov while (page_vma_mapped_walk(&pvmw)) { 66699fa216SKirill A. Shutemov addr = pvmw.address; 67699fa216SKirill A. Shutemov if (pvmw.pte) { 68f0849ac0SYang Shi /* 69f0849ac0SYang Shi * For PTE-mapped THP, one sub page is referenced, 70f0849ac0SYang Shi * the whole THP is referenced. 71f0849ac0SYang Shi */ 72f0849ac0SYang Shi if (ptep_clear_young_notify(vma, addr, pvmw.pte)) 73f0849ac0SYang Shi referenced = true; 748749cfeaSVladimir Davydov } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { 75f0849ac0SYang Shi if (pmdp_clear_young_notify(vma, addr, pvmw.pmd)) 76f0849ac0SYang Shi referenced = true; 778749cfeaSVladimir Davydov } else { 788749cfeaSVladimir Davydov /* unexpected pmd-mapped page? */ 798749cfeaSVladimir Davydov WARN_ON_ONCE(1); 808749cfeaSVladimir Davydov } 81699fa216SKirill A. Shutemov } 828749cfeaSVladimir Davydov 8333c3fc71SVladimir Davydov if (referenced) { 8433c3fc71SVladimir Davydov clear_page_idle(page); 8533c3fc71SVladimir Davydov /* 8633c3fc71SVladimir Davydov * We cleared the referenced bit in a mapping to this page. To 8733c3fc71SVladimir Davydov * avoid interference with page reclaim, mark it young so that 8833c3fc71SVladimir Davydov * page_referenced() will return > 0. 8933c3fc71SVladimir Davydov */ 9033c3fc71SVladimir Davydov set_page_young(page); 9133c3fc71SVladimir Davydov } 92e4b82222SMinchan Kim return true; 9333c3fc71SVladimir Davydov } 9433c3fc71SVladimir Davydov 9533c3fc71SVladimir Davydov static void page_idle_clear_pte_refs(struct page *page) 9633c3fc71SVladimir Davydov { 9733c3fc71SVladimir Davydov /* 9833c3fc71SVladimir Davydov * Since rwc.arg is unused, rwc is effectively immutable, so we 9933c3fc71SVladimir Davydov * can make it static const to save some cycles and stack. 10033c3fc71SVladimir Davydov */ 10133c3fc71SVladimir Davydov static const struct rmap_walk_control rwc = { 10233c3fc71SVladimir Davydov .rmap_one = page_idle_clear_pte_refs_one, 10333c3fc71SVladimir Davydov .anon_lock = page_lock_anon_vma_read, 10433c3fc71SVladimir Davydov }; 10533c3fc71SVladimir Davydov bool need_lock; 10633c3fc71SVladimir Davydov 10733c3fc71SVladimir Davydov if (!page_mapped(page) || 10833c3fc71SVladimir Davydov !page_rmapping(page)) 10933c3fc71SVladimir Davydov return; 11033c3fc71SVladimir Davydov 11133c3fc71SVladimir Davydov need_lock = !PageAnon(page) || PageKsm(page); 11233c3fc71SVladimir Davydov if (need_lock && !trylock_page(page)) 11333c3fc71SVladimir Davydov return; 11433c3fc71SVladimir Davydov 11533c3fc71SVladimir Davydov rmap_walk(page, (struct rmap_walk_control *)&rwc); 11633c3fc71SVladimir Davydov 11733c3fc71SVladimir Davydov if (need_lock) 11833c3fc71SVladimir Davydov unlock_page(page); 11933c3fc71SVladimir Davydov } 12033c3fc71SVladimir Davydov 12133c3fc71SVladimir Davydov static ssize_t page_idle_bitmap_read(struct file *file, struct kobject *kobj, 12233c3fc71SVladimir Davydov struct bin_attribute *attr, char *buf, 12333c3fc71SVladimir Davydov loff_t pos, size_t count) 12433c3fc71SVladimir Davydov { 12533c3fc71SVladimir Davydov u64 *out = (u64 *)buf; 12633c3fc71SVladimir Davydov struct page *page; 12733c3fc71SVladimir Davydov unsigned long pfn, end_pfn; 12833c3fc71SVladimir Davydov int bit; 12933c3fc71SVladimir Davydov 13033c3fc71SVladimir Davydov if (pos % BITMAP_CHUNK_SIZE || count % BITMAP_CHUNK_SIZE) 13133c3fc71SVladimir Davydov return -EINVAL; 13233c3fc71SVladimir Davydov 13333c3fc71SVladimir Davydov pfn = pos * BITS_PER_BYTE; 13433c3fc71SVladimir Davydov if (pfn >= max_pfn) 13533c3fc71SVladimir Davydov return 0; 13633c3fc71SVladimir Davydov 13733c3fc71SVladimir Davydov end_pfn = pfn + count * BITS_PER_BYTE; 13833c3fc71SVladimir Davydov if (end_pfn > max_pfn) 13933c3fc71SVladimir Davydov end_pfn = ALIGN(max_pfn, BITMAP_CHUNK_BITS); 14033c3fc71SVladimir Davydov 14133c3fc71SVladimir Davydov for (; pfn < end_pfn; pfn++) { 14233c3fc71SVladimir Davydov bit = pfn % BITMAP_CHUNK_BITS; 14333c3fc71SVladimir Davydov if (!bit) 14433c3fc71SVladimir Davydov *out = 0ULL; 14533c3fc71SVladimir Davydov page = page_idle_get_page(pfn); 14633c3fc71SVladimir Davydov if (page) { 14733c3fc71SVladimir Davydov if (page_is_idle(page)) { 14833c3fc71SVladimir Davydov /* 14933c3fc71SVladimir Davydov * The page might have been referenced via a 15033c3fc71SVladimir Davydov * pte, in which case it is not idle. Clear 15133c3fc71SVladimir Davydov * refs and recheck. 15233c3fc71SVladimir Davydov */ 15333c3fc71SVladimir Davydov page_idle_clear_pte_refs(page); 15433c3fc71SVladimir Davydov if (page_is_idle(page)) 15533c3fc71SVladimir Davydov *out |= 1ULL << bit; 15633c3fc71SVladimir Davydov } 15733c3fc71SVladimir Davydov put_page(page); 15833c3fc71SVladimir Davydov } 15933c3fc71SVladimir Davydov if (bit == BITMAP_CHUNK_BITS - 1) 16033c3fc71SVladimir Davydov out++; 16133c3fc71SVladimir Davydov cond_resched(); 16233c3fc71SVladimir Davydov } 16333c3fc71SVladimir Davydov return (char *)out - buf; 16433c3fc71SVladimir Davydov } 16533c3fc71SVladimir Davydov 16633c3fc71SVladimir Davydov static ssize_t page_idle_bitmap_write(struct file *file, struct kobject *kobj, 16733c3fc71SVladimir Davydov struct bin_attribute *attr, char *buf, 16833c3fc71SVladimir Davydov loff_t pos, size_t count) 16933c3fc71SVladimir Davydov { 17033c3fc71SVladimir Davydov const u64 *in = (u64 *)buf; 17133c3fc71SVladimir Davydov struct page *page; 17233c3fc71SVladimir Davydov unsigned long pfn, end_pfn; 17333c3fc71SVladimir Davydov int bit; 17433c3fc71SVladimir Davydov 17533c3fc71SVladimir Davydov if (pos % BITMAP_CHUNK_SIZE || count % BITMAP_CHUNK_SIZE) 17633c3fc71SVladimir Davydov return -EINVAL; 17733c3fc71SVladimir Davydov 17833c3fc71SVladimir Davydov pfn = pos * BITS_PER_BYTE; 17933c3fc71SVladimir Davydov if (pfn >= max_pfn) 18033c3fc71SVladimir Davydov return -ENXIO; 18133c3fc71SVladimir Davydov 18233c3fc71SVladimir Davydov end_pfn = pfn + count * BITS_PER_BYTE; 18333c3fc71SVladimir Davydov if (end_pfn > max_pfn) 18433c3fc71SVladimir Davydov end_pfn = ALIGN(max_pfn, BITMAP_CHUNK_BITS); 18533c3fc71SVladimir Davydov 18633c3fc71SVladimir Davydov for (; pfn < end_pfn; pfn++) { 18733c3fc71SVladimir Davydov bit = pfn % BITMAP_CHUNK_BITS; 18833c3fc71SVladimir Davydov if ((*in >> bit) & 1) { 18933c3fc71SVladimir Davydov page = page_idle_get_page(pfn); 19033c3fc71SVladimir Davydov if (page) { 19133c3fc71SVladimir Davydov page_idle_clear_pte_refs(page); 19233c3fc71SVladimir Davydov set_page_idle(page); 19333c3fc71SVladimir Davydov put_page(page); 19433c3fc71SVladimir Davydov } 19533c3fc71SVladimir Davydov } 19633c3fc71SVladimir Davydov if (bit == BITMAP_CHUNK_BITS - 1) 19733c3fc71SVladimir Davydov in++; 19833c3fc71SVladimir Davydov cond_resched(); 19933c3fc71SVladimir Davydov } 20033c3fc71SVladimir Davydov return (char *)in - buf; 20133c3fc71SVladimir Davydov } 20233c3fc71SVladimir Davydov 20333c3fc71SVladimir Davydov static struct bin_attribute page_idle_bitmap_attr = 2040825a6f9SJoe Perches __BIN_ATTR(bitmap, 0600, 20533c3fc71SVladimir Davydov page_idle_bitmap_read, page_idle_bitmap_write, 0); 20633c3fc71SVladimir Davydov 20733c3fc71SVladimir Davydov static struct bin_attribute *page_idle_bin_attrs[] = { 20833c3fc71SVladimir Davydov &page_idle_bitmap_attr, 20933c3fc71SVladimir Davydov NULL, 21033c3fc71SVladimir Davydov }; 21133c3fc71SVladimir Davydov 212fd147cbbSArvind Yadav static const struct attribute_group page_idle_attr_group = { 21333c3fc71SVladimir Davydov .bin_attrs = page_idle_bin_attrs, 21433c3fc71SVladimir Davydov .name = "page_idle", 21533c3fc71SVladimir Davydov }; 21633c3fc71SVladimir Davydov 21733c3fc71SVladimir Davydov #ifndef CONFIG_64BIT 21833c3fc71SVladimir Davydov static bool need_page_idle(void) 21933c3fc71SVladimir Davydov { 22033c3fc71SVladimir Davydov return true; 22133c3fc71SVladimir Davydov } 22233c3fc71SVladimir Davydov struct page_ext_operations page_idle_ops = { 22333c3fc71SVladimir Davydov .need = need_page_idle, 22433c3fc71SVladimir Davydov }; 22533c3fc71SVladimir Davydov #endif 22633c3fc71SVladimir Davydov 22733c3fc71SVladimir Davydov static int __init page_idle_init(void) 22833c3fc71SVladimir Davydov { 22933c3fc71SVladimir Davydov int err; 23033c3fc71SVladimir Davydov 23133c3fc71SVladimir Davydov err = sysfs_create_group(mm_kobj, &page_idle_attr_group); 23233c3fc71SVladimir Davydov if (err) { 23333c3fc71SVladimir Davydov pr_err("page_idle: register sysfs failed\n"); 23433c3fc71SVladimir Davydov return err; 23533c3fc71SVladimir Davydov } 23633c3fc71SVladimir Davydov return 0; 23733c3fc71SVladimir Davydov } 23833c3fc71SVladimir Davydov subsys_initcall(page_idle_init); 239