171e3aac0SAndrea Arcangeli /* 271e3aac0SAndrea Arcangeli * Copyright (C) 2009 Red Hat, Inc. 371e3aac0SAndrea Arcangeli * 471e3aac0SAndrea Arcangeli * This work is licensed under the terms of the GNU GPL, version 2. See 571e3aac0SAndrea Arcangeli * the COPYING file in the top-level directory. 671e3aac0SAndrea Arcangeli */ 771e3aac0SAndrea Arcangeli 8ae3a8c1cSAndrew Morton #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 9ae3a8c1cSAndrew Morton 1071e3aac0SAndrea Arcangeli #include <linux/mm.h> 1171e3aac0SAndrea Arcangeli #include <linux/sched.h> 1271e3aac0SAndrea Arcangeli #include <linux/highmem.h> 1371e3aac0SAndrea Arcangeli #include <linux/hugetlb.h> 1471e3aac0SAndrea Arcangeli #include <linux/mmu_notifier.h> 1571e3aac0SAndrea Arcangeli #include <linux/rmap.h> 1671e3aac0SAndrea Arcangeli #include <linux/swap.h> 1797ae1749SKirill A. Shutemov #include <linux/shrinker.h> 18ba76149fSAndrea Arcangeli #include <linux/mm_inline.h> 19e9b61f19SKirill A. Shutemov #include <linux/swapops.h> 204897c765SMatthew Wilcox #include <linux/dax.h> 21ba76149fSAndrea Arcangeli #include <linux/kthread.h> 22ba76149fSAndrea Arcangeli #include <linux/khugepaged.h> 23878aee7dSAndrea Arcangeli #include <linux/freezer.h> 24f25748e3SDan Williams #include <linux/pfn_t.h> 25a664b2d8SAndrea Arcangeli #include <linux/mman.h> 263565fce3SDan Williams #include <linux/memremap.h> 27325adeb5SRalf Baechle #include <linux/pagemap.h> 2849071d43SKirill A. Shutemov #include <linux/debugfs.h> 294daae3b4SMel Gorman #include <linux/migrate.h> 3043b5fbbdSSasha Levin #include <linux/hashtable.h> 316b251fc9SAndrea Arcangeli #include <linux/userfaultfd_k.h> 3233c3fc71SVladimir Davydov #include <linux/page_idle.h> 3397ae1749SKirill A. Shutemov 3471e3aac0SAndrea Arcangeli #include <asm/tlb.h> 3571e3aac0SAndrea Arcangeli #include <asm/pgalloc.h> 3671e3aac0SAndrea Arcangeli #include "internal.h" 3771e3aac0SAndrea Arcangeli 387d2eba05SEbru Akagunduz enum scan_result { 397d2eba05SEbru Akagunduz SCAN_FAIL, 407d2eba05SEbru Akagunduz SCAN_SUCCEED, 417d2eba05SEbru Akagunduz SCAN_PMD_NULL, 427d2eba05SEbru Akagunduz SCAN_EXCEED_NONE_PTE, 437d2eba05SEbru Akagunduz SCAN_PTE_NON_PRESENT, 447d2eba05SEbru Akagunduz SCAN_PAGE_RO, 457d2eba05SEbru Akagunduz SCAN_NO_REFERENCED_PAGE, 467d2eba05SEbru Akagunduz SCAN_PAGE_NULL, 477d2eba05SEbru Akagunduz SCAN_SCAN_ABORT, 487d2eba05SEbru Akagunduz SCAN_PAGE_COUNT, 497d2eba05SEbru Akagunduz SCAN_PAGE_LRU, 507d2eba05SEbru Akagunduz SCAN_PAGE_LOCK, 517d2eba05SEbru Akagunduz SCAN_PAGE_ANON, 52b1caa957SKirill A. Shutemov SCAN_PAGE_COMPOUND, 537d2eba05SEbru Akagunduz SCAN_ANY_PROCESS, 547d2eba05SEbru Akagunduz SCAN_VMA_NULL, 557d2eba05SEbru Akagunduz SCAN_VMA_CHECK, 567d2eba05SEbru Akagunduz SCAN_ADDRESS_RANGE, 577d2eba05SEbru Akagunduz SCAN_SWAP_CACHE_PAGE, 587d2eba05SEbru Akagunduz SCAN_DEL_PAGE_LRU, 597d2eba05SEbru Akagunduz SCAN_ALLOC_HUGE_PAGE_FAIL, 607d2eba05SEbru Akagunduz SCAN_CGROUP_CHARGE_FAIL 617d2eba05SEbru Akagunduz }; 627d2eba05SEbru Akagunduz 637d2eba05SEbru Akagunduz #define CREATE_TRACE_POINTS 647d2eba05SEbru Akagunduz #include <trace/events/huge_memory.h> 657d2eba05SEbru Akagunduz 66ba76149fSAndrea Arcangeli /* 678bfa3f9aSJianguo Wu * By default transparent hugepage support is disabled in order that avoid 688bfa3f9aSJianguo Wu * to risk increase the memory footprint of applications without a guaranteed 698bfa3f9aSJianguo Wu * benefit. When transparent hugepage support is enabled, is for all mappings, 708bfa3f9aSJianguo Wu * and khugepaged scans all mappings. 718bfa3f9aSJianguo Wu * Defrag is invoked by khugepaged hugepage allocations and by page faults 728bfa3f9aSJianguo Wu * for all hugepage allocations. 73ba76149fSAndrea Arcangeli */ 7471e3aac0SAndrea Arcangeli unsigned long transparent_hugepage_flags __read_mostly = 7513ece886SAndrea Arcangeli #ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS 76ba76149fSAndrea Arcangeli (1<<TRANSPARENT_HUGEPAGE_FLAG)| 7713ece886SAndrea Arcangeli #endif 7813ece886SAndrea Arcangeli #ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE 7913ece886SAndrea Arcangeli (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)| 8013ece886SAndrea Arcangeli #endif 81d39d33c3SAndrea Arcangeli (1<<TRANSPARENT_HUGEPAGE_DEFRAG_FLAG)| 8279da5407SKirill A. Shutemov (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)| 8379da5407SKirill A. Shutemov (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 84ba76149fSAndrea Arcangeli 85ba76149fSAndrea Arcangeli /* default scan 8*512 pte (or vmas) every 30 second */ 86ba76149fSAndrea Arcangeli static unsigned int khugepaged_pages_to_scan __read_mostly = HPAGE_PMD_NR*8; 87ba76149fSAndrea Arcangeli static unsigned int khugepaged_pages_collapsed; 88ba76149fSAndrea Arcangeli static unsigned int khugepaged_full_scans; 89ba76149fSAndrea Arcangeli static unsigned int khugepaged_scan_sleep_millisecs __read_mostly = 10000; 90ba76149fSAndrea Arcangeli /* during fragmentation poll the hugepage allocator once every minute */ 91ba76149fSAndrea Arcangeli static unsigned int khugepaged_alloc_sleep_millisecs __read_mostly = 60000; 92ba76149fSAndrea Arcangeli static struct task_struct *khugepaged_thread __read_mostly; 93ba76149fSAndrea Arcangeli static DEFINE_MUTEX(khugepaged_mutex); 94ba76149fSAndrea Arcangeli static DEFINE_SPINLOCK(khugepaged_mm_lock); 95ba76149fSAndrea Arcangeli static DECLARE_WAIT_QUEUE_HEAD(khugepaged_wait); 96ba76149fSAndrea Arcangeli /* 97ba76149fSAndrea Arcangeli * default collapse hugepages if there is at least one pte mapped like 98ba76149fSAndrea Arcangeli * it would have happened if the vma was large enough during page 99ba76149fSAndrea Arcangeli * fault. 100ba76149fSAndrea Arcangeli */ 101ba76149fSAndrea Arcangeli static unsigned int khugepaged_max_ptes_none __read_mostly = HPAGE_PMD_NR-1; 102ba76149fSAndrea Arcangeli 103ba76149fSAndrea Arcangeli static int khugepaged(void *none); 104ba76149fSAndrea Arcangeli static int khugepaged_slab_init(void); 10565ebb64fSKirill A. Shutemov static void khugepaged_slab_exit(void); 106ba76149fSAndrea Arcangeli 10743b5fbbdSSasha Levin #define MM_SLOTS_HASH_BITS 10 10843b5fbbdSSasha Levin static __read_mostly DEFINE_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS); 10943b5fbbdSSasha Levin 110ba76149fSAndrea Arcangeli static struct kmem_cache *mm_slot_cache __read_mostly; 111ba76149fSAndrea Arcangeli 112ba76149fSAndrea Arcangeli /** 113ba76149fSAndrea Arcangeli * struct mm_slot - hash lookup from mm to mm_slot 114ba76149fSAndrea Arcangeli * @hash: hash collision list 115ba76149fSAndrea Arcangeli * @mm_node: khugepaged scan list headed in khugepaged_scan.mm_head 116ba76149fSAndrea Arcangeli * @mm: the mm that this information is valid for 117ba76149fSAndrea Arcangeli */ 118ba76149fSAndrea Arcangeli struct mm_slot { 119ba76149fSAndrea Arcangeli struct hlist_node hash; 120ba76149fSAndrea Arcangeli struct list_head mm_node; 121ba76149fSAndrea Arcangeli struct mm_struct *mm; 122ba76149fSAndrea Arcangeli }; 123ba76149fSAndrea Arcangeli 124ba76149fSAndrea Arcangeli /** 125ba76149fSAndrea Arcangeli * struct khugepaged_scan - cursor for scanning 126ba76149fSAndrea Arcangeli * @mm_head: the head of the mm list to scan 127ba76149fSAndrea Arcangeli * @mm_slot: the current mm_slot we are scanning 128ba76149fSAndrea Arcangeli * @address: the next address inside that to be scanned 129ba76149fSAndrea Arcangeli * 130ba76149fSAndrea Arcangeli * There is only the one khugepaged_scan instance of this cursor structure. 131ba76149fSAndrea Arcangeli */ 132ba76149fSAndrea Arcangeli struct khugepaged_scan { 133ba76149fSAndrea Arcangeli struct list_head mm_head; 134ba76149fSAndrea Arcangeli struct mm_slot *mm_slot; 135ba76149fSAndrea Arcangeli unsigned long address; 1362f1da642SH Hartley Sweeten }; 1372f1da642SH Hartley Sweeten static struct khugepaged_scan khugepaged_scan = { 138ba76149fSAndrea Arcangeli .mm_head = LIST_HEAD_INIT(khugepaged_scan.mm_head), 139ba76149fSAndrea Arcangeli }; 140ba76149fSAndrea Arcangeli 1419a982250SKirill A. Shutemov static DEFINE_SPINLOCK(split_queue_lock); 1429a982250SKirill A. Shutemov static LIST_HEAD(split_queue); 1439a982250SKirill A. Shutemov static unsigned long split_queue_len; 1449a982250SKirill A. Shutemov static struct shrinker deferred_split_shrinker; 145f000565aSAndrea Arcangeli 1462c0b80d4SNicholas Krause static void set_recommended_min_free_kbytes(void) 147f000565aSAndrea Arcangeli { 148f000565aSAndrea Arcangeli struct zone *zone; 149f000565aSAndrea Arcangeli int nr_zones = 0; 150f000565aSAndrea Arcangeli unsigned long recommended_min; 151f000565aSAndrea Arcangeli 152f000565aSAndrea Arcangeli for_each_populated_zone(zone) 153f000565aSAndrea Arcangeli nr_zones++; 154f000565aSAndrea Arcangeli 155974a786eSMel Gorman /* Ensure 2 pageblocks are free to assist fragmentation avoidance */ 156f000565aSAndrea Arcangeli recommended_min = pageblock_nr_pages * nr_zones * 2; 157f000565aSAndrea Arcangeli 158f000565aSAndrea Arcangeli /* 159f000565aSAndrea Arcangeli * Make sure that on average at least two pageblocks are almost free 160f000565aSAndrea Arcangeli * of another type, one for a migratetype to fall back to and a 161f000565aSAndrea Arcangeli * second to avoid subsequent fallbacks of other types There are 3 162f000565aSAndrea Arcangeli * MIGRATE_TYPES we care about. 163f000565aSAndrea Arcangeli */ 164f000565aSAndrea Arcangeli recommended_min += pageblock_nr_pages * nr_zones * 165f000565aSAndrea Arcangeli MIGRATE_PCPTYPES * MIGRATE_PCPTYPES; 166f000565aSAndrea Arcangeli 167f000565aSAndrea Arcangeli /* don't ever allow to reserve more than 5% of the lowmem */ 168f000565aSAndrea Arcangeli recommended_min = min(recommended_min, 169f000565aSAndrea Arcangeli (unsigned long) nr_free_buffer_pages() / 20); 170f000565aSAndrea Arcangeli recommended_min <<= (PAGE_SHIFT-10); 171f000565aSAndrea Arcangeli 17242aa83cbSHan Pingtian if (recommended_min > min_free_kbytes) { 17342aa83cbSHan Pingtian if (user_min_free_kbytes >= 0) 17442aa83cbSHan Pingtian pr_info("raising min_free_kbytes from %d to %lu " 17542aa83cbSHan Pingtian "to help transparent hugepage allocations\n", 17642aa83cbSHan Pingtian min_free_kbytes, recommended_min); 17742aa83cbSHan Pingtian 178f000565aSAndrea Arcangeli min_free_kbytes = recommended_min; 17942aa83cbSHan Pingtian } 180f000565aSAndrea Arcangeli setup_per_zone_wmarks(); 181f000565aSAndrea Arcangeli } 182f000565aSAndrea Arcangeli 18379553da2SKirill A. Shutemov static int start_stop_khugepaged(void) 184ba76149fSAndrea Arcangeli { 185ba76149fSAndrea Arcangeli int err = 0; 186ba76149fSAndrea Arcangeli if (khugepaged_enabled()) { 187ba76149fSAndrea Arcangeli if (!khugepaged_thread) 188ba76149fSAndrea Arcangeli khugepaged_thread = kthread_run(khugepaged, NULL, 189ba76149fSAndrea Arcangeli "khugepaged"); 19018e8e5c7SViresh Kumar if (IS_ERR(khugepaged_thread)) { 191ae3a8c1cSAndrew Morton pr_err("khugepaged: kthread_run(khugepaged) failed\n"); 192ba76149fSAndrea Arcangeli err = PTR_ERR(khugepaged_thread); 193ba76149fSAndrea Arcangeli khugepaged_thread = NULL; 19479553da2SKirill A. Shutemov goto fail; 195ba76149fSAndrea Arcangeli } 196911891afSXiao Guangrong 197911891afSXiao Guangrong if (!list_empty(&khugepaged_scan.mm_head)) 198ba76149fSAndrea Arcangeli wake_up_interruptible(&khugepaged_wait); 199f000565aSAndrea Arcangeli 200f000565aSAndrea Arcangeli set_recommended_min_free_kbytes(); 201911891afSXiao Guangrong } else if (khugepaged_thread) { 202911891afSXiao Guangrong kthread_stop(khugepaged_thread); 203911891afSXiao Guangrong khugepaged_thread = NULL; 204911891afSXiao Guangrong } 20579553da2SKirill A. Shutemov fail: 206ba76149fSAndrea Arcangeli return err; 207ba76149fSAndrea Arcangeli } 20871e3aac0SAndrea Arcangeli 20997ae1749SKirill A. Shutemov static atomic_t huge_zero_refcount; 21056873f43SWang, Yalin struct page *huge_zero_page __read_mostly; 2114a6c1297SKirill A. Shutemov 212fc437044SMatthew Wilcox struct page *get_huge_zero_page(void) 21397ae1749SKirill A. Shutemov { 21497ae1749SKirill A. Shutemov struct page *zero_page; 21597ae1749SKirill A. Shutemov retry: 21697ae1749SKirill A. Shutemov if (likely(atomic_inc_not_zero(&huge_zero_refcount))) 2174db0c3c2SJason Low return READ_ONCE(huge_zero_page); 21897ae1749SKirill A. Shutemov 21997ae1749SKirill A. Shutemov zero_page = alloc_pages((GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE, 22097ae1749SKirill A. Shutemov HPAGE_PMD_ORDER); 221d8a8e1f0SKirill A. Shutemov if (!zero_page) { 222d8a8e1f0SKirill A. Shutemov count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED); 2235918d10aSKirill A. Shutemov return NULL; 224d8a8e1f0SKirill A. Shutemov } 225d8a8e1f0SKirill A. Shutemov count_vm_event(THP_ZERO_PAGE_ALLOC); 22697ae1749SKirill A. Shutemov preempt_disable(); 2275918d10aSKirill A. Shutemov if (cmpxchg(&huge_zero_page, NULL, zero_page)) { 22897ae1749SKirill A. Shutemov preempt_enable(); 2295ddacbe9SYu Zhao __free_pages(zero_page, compound_order(zero_page)); 23097ae1749SKirill A. Shutemov goto retry; 23197ae1749SKirill A. Shutemov } 23297ae1749SKirill A. Shutemov 23397ae1749SKirill A. Shutemov /* We take additional reference here. It will be put back by shrinker */ 23497ae1749SKirill A. Shutemov atomic_set(&huge_zero_refcount, 2); 23597ae1749SKirill A. Shutemov preempt_enable(); 2364db0c3c2SJason Low return READ_ONCE(huge_zero_page); 23797ae1749SKirill A. Shutemov } 23897ae1749SKirill A. Shutemov 23997ae1749SKirill A. Shutemov static void put_huge_zero_page(void) 24097ae1749SKirill A. Shutemov { 24197ae1749SKirill A. Shutemov /* 24297ae1749SKirill A. Shutemov * Counter should never go to zero here. Only shrinker can put 24397ae1749SKirill A. Shutemov * last reference. 24497ae1749SKirill A. Shutemov */ 24597ae1749SKirill A. Shutemov BUG_ON(atomic_dec_and_test(&huge_zero_refcount)); 24697ae1749SKirill A. Shutemov } 24797ae1749SKirill A. Shutemov 24848896466SGlauber Costa static unsigned long shrink_huge_zero_page_count(struct shrinker *shrink, 24997ae1749SKirill A. Shutemov struct shrink_control *sc) 25097ae1749SKirill A. Shutemov { 25197ae1749SKirill A. Shutemov /* we can free zero page only if last reference remains */ 25297ae1749SKirill A. Shutemov return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0; 25348896466SGlauber Costa } 25497ae1749SKirill A. Shutemov 25548896466SGlauber Costa static unsigned long shrink_huge_zero_page_scan(struct shrinker *shrink, 25648896466SGlauber Costa struct shrink_control *sc) 25748896466SGlauber Costa { 25897ae1749SKirill A. Shutemov if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) { 2595918d10aSKirill A. Shutemov struct page *zero_page = xchg(&huge_zero_page, NULL); 2605918d10aSKirill A. Shutemov BUG_ON(zero_page == NULL); 2615ddacbe9SYu Zhao __free_pages(zero_page, compound_order(zero_page)); 26248896466SGlauber Costa return HPAGE_PMD_NR; 26397ae1749SKirill A. Shutemov } 26497ae1749SKirill A. Shutemov 26597ae1749SKirill A. Shutemov return 0; 26697ae1749SKirill A. Shutemov } 26797ae1749SKirill A. Shutemov 26897ae1749SKirill A. Shutemov static struct shrinker huge_zero_page_shrinker = { 26948896466SGlauber Costa .count_objects = shrink_huge_zero_page_count, 27048896466SGlauber Costa .scan_objects = shrink_huge_zero_page_scan, 27197ae1749SKirill A. Shutemov .seeks = DEFAULT_SEEKS, 27297ae1749SKirill A. Shutemov }; 27397ae1749SKirill A. Shutemov 27471e3aac0SAndrea Arcangeli #ifdef CONFIG_SYSFS 275ba76149fSAndrea Arcangeli 27671e3aac0SAndrea Arcangeli static ssize_t double_flag_show(struct kobject *kobj, 27771e3aac0SAndrea Arcangeli struct kobj_attribute *attr, char *buf, 27871e3aac0SAndrea Arcangeli enum transparent_hugepage_flag enabled, 27971e3aac0SAndrea Arcangeli enum transparent_hugepage_flag req_madv) 28071e3aac0SAndrea Arcangeli { 28171e3aac0SAndrea Arcangeli if (test_bit(enabled, &transparent_hugepage_flags)) { 28271e3aac0SAndrea Arcangeli VM_BUG_ON(test_bit(req_madv, &transparent_hugepage_flags)); 28371e3aac0SAndrea Arcangeli return sprintf(buf, "[always] madvise never\n"); 28471e3aac0SAndrea Arcangeli } else if (test_bit(req_madv, &transparent_hugepage_flags)) 28571e3aac0SAndrea Arcangeli return sprintf(buf, "always [madvise] never\n"); 28671e3aac0SAndrea Arcangeli else 28771e3aac0SAndrea Arcangeli return sprintf(buf, "always madvise [never]\n"); 28871e3aac0SAndrea Arcangeli } 28971e3aac0SAndrea Arcangeli static ssize_t double_flag_store(struct kobject *kobj, 29071e3aac0SAndrea Arcangeli struct kobj_attribute *attr, 29171e3aac0SAndrea Arcangeli const char *buf, size_t count, 29271e3aac0SAndrea Arcangeli enum transparent_hugepage_flag enabled, 29371e3aac0SAndrea Arcangeli enum transparent_hugepage_flag req_madv) 29471e3aac0SAndrea Arcangeli { 29571e3aac0SAndrea Arcangeli if (!memcmp("always", buf, 29671e3aac0SAndrea Arcangeli min(sizeof("always")-1, count))) { 29771e3aac0SAndrea Arcangeli set_bit(enabled, &transparent_hugepage_flags); 29871e3aac0SAndrea Arcangeli clear_bit(req_madv, &transparent_hugepage_flags); 29971e3aac0SAndrea Arcangeli } else if (!memcmp("madvise", buf, 30071e3aac0SAndrea Arcangeli min(sizeof("madvise")-1, count))) { 30171e3aac0SAndrea Arcangeli clear_bit(enabled, &transparent_hugepage_flags); 30271e3aac0SAndrea Arcangeli set_bit(req_madv, &transparent_hugepage_flags); 30371e3aac0SAndrea Arcangeli } else if (!memcmp("never", buf, 30471e3aac0SAndrea Arcangeli min(sizeof("never")-1, count))) { 30571e3aac0SAndrea Arcangeli clear_bit(enabled, &transparent_hugepage_flags); 30671e3aac0SAndrea Arcangeli clear_bit(req_madv, &transparent_hugepage_flags); 30771e3aac0SAndrea Arcangeli } else 30871e3aac0SAndrea Arcangeli return -EINVAL; 30971e3aac0SAndrea Arcangeli 31071e3aac0SAndrea Arcangeli return count; 31171e3aac0SAndrea Arcangeli } 31271e3aac0SAndrea Arcangeli 31371e3aac0SAndrea Arcangeli static ssize_t enabled_show(struct kobject *kobj, 31471e3aac0SAndrea Arcangeli struct kobj_attribute *attr, char *buf) 31571e3aac0SAndrea Arcangeli { 31671e3aac0SAndrea Arcangeli return double_flag_show(kobj, attr, buf, 31771e3aac0SAndrea Arcangeli TRANSPARENT_HUGEPAGE_FLAG, 31871e3aac0SAndrea Arcangeli TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG); 31971e3aac0SAndrea Arcangeli } 32071e3aac0SAndrea Arcangeli static ssize_t enabled_store(struct kobject *kobj, 32171e3aac0SAndrea Arcangeli struct kobj_attribute *attr, 32271e3aac0SAndrea Arcangeli const char *buf, size_t count) 32371e3aac0SAndrea Arcangeli { 324ba76149fSAndrea Arcangeli ssize_t ret; 325ba76149fSAndrea Arcangeli 326ba76149fSAndrea Arcangeli ret = double_flag_store(kobj, attr, buf, count, 32771e3aac0SAndrea Arcangeli TRANSPARENT_HUGEPAGE_FLAG, 32871e3aac0SAndrea Arcangeli TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG); 329ba76149fSAndrea Arcangeli 330ba76149fSAndrea Arcangeli if (ret > 0) { 331911891afSXiao Guangrong int err; 332911891afSXiao Guangrong 333911891afSXiao Guangrong mutex_lock(&khugepaged_mutex); 33479553da2SKirill A. Shutemov err = start_stop_khugepaged(); 335911891afSXiao Guangrong mutex_unlock(&khugepaged_mutex); 336911891afSXiao Guangrong 337ba76149fSAndrea Arcangeli if (err) 338ba76149fSAndrea Arcangeli ret = err; 339ba76149fSAndrea Arcangeli } 340ba76149fSAndrea Arcangeli 341ba76149fSAndrea Arcangeli return ret; 34271e3aac0SAndrea Arcangeli } 34371e3aac0SAndrea Arcangeli static struct kobj_attribute enabled_attr = 34471e3aac0SAndrea Arcangeli __ATTR(enabled, 0644, enabled_show, enabled_store); 34571e3aac0SAndrea Arcangeli 34671e3aac0SAndrea Arcangeli static ssize_t single_flag_show(struct kobject *kobj, 34771e3aac0SAndrea Arcangeli struct kobj_attribute *attr, char *buf, 34871e3aac0SAndrea Arcangeli enum transparent_hugepage_flag flag) 34971e3aac0SAndrea Arcangeli { 350e27e6151SBen Hutchings return sprintf(buf, "%d\n", 351e27e6151SBen Hutchings !!test_bit(flag, &transparent_hugepage_flags)); 35271e3aac0SAndrea Arcangeli } 353e27e6151SBen Hutchings 35471e3aac0SAndrea Arcangeli static ssize_t single_flag_store(struct kobject *kobj, 35571e3aac0SAndrea Arcangeli struct kobj_attribute *attr, 35671e3aac0SAndrea Arcangeli const char *buf, size_t count, 35771e3aac0SAndrea Arcangeli enum transparent_hugepage_flag flag) 35871e3aac0SAndrea Arcangeli { 359e27e6151SBen Hutchings unsigned long value; 360e27e6151SBen Hutchings int ret; 361e27e6151SBen Hutchings 362e27e6151SBen Hutchings ret = kstrtoul(buf, 10, &value); 363e27e6151SBen Hutchings if (ret < 0) 364e27e6151SBen Hutchings return ret; 365e27e6151SBen Hutchings if (value > 1) 36671e3aac0SAndrea Arcangeli return -EINVAL; 36771e3aac0SAndrea Arcangeli 368e27e6151SBen Hutchings if (value) 369e27e6151SBen Hutchings set_bit(flag, &transparent_hugepage_flags); 370e27e6151SBen Hutchings else 371e27e6151SBen Hutchings clear_bit(flag, &transparent_hugepage_flags); 372e27e6151SBen Hutchings 37371e3aac0SAndrea Arcangeli return count; 37471e3aac0SAndrea Arcangeli } 37571e3aac0SAndrea Arcangeli 37671e3aac0SAndrea Arcangeli /* 37771e3aac0SAndrea Arcangeli * Currently defrag only disables __GFP_NOWAIT for allocation. A blind 37871e3aac0SAndrea Arcangeli * __GFP_REPEAT is too aggressive, it's never worth swapping tons of 37971e3aac0SAndrea Arcangeli * memory just to allocate one more hugepage. 38071e3aac0SAndrea Arcangeli */ 38171e3aac0SAndrea Arcangeli static ssize_t defrag_show(struct kobject *kobj, 38271e3aac0SAndrea Arcangeli struct kobj_attribute *attr, char *buf) 38371e3aac0SAndrea Arcangeli { 38471e3aac0SAndrea Arcangeli return double_flag_show(kobj, attr, buf, 38571e3aac0SAndrea Arcangeli TRANSPARENT_HUGEPAGE_DEFRAG_FLAG, 38671e3aac0SAndrea Arcangeli TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG); 38771e3aac0SAndrea Arcangeli } 38871e3aac0SAndrea Arcangeli static ssize_t defrag_store(struct kobject *kobj, 38971e3aac0SAndrea Arcangeli struct kobj_attribute *attr, 39071e3aac0SAndrea Arcangeli const char *buf, size_t count) 39171e3aac0SAndrea Arcangeli { 39271e3aac0SAndrea Arcangeli return double_flag_store(kobj, attr, buf, count, 39371e3aac0SAndrea Arcangeli TRANSPARENT_HUGEPAGE_DEFRAG_FLAG, 39471e3aac0SAndrea Arcangeli TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG); 39571e3aac0SAndrea Arcangeli } 39671e3aac0SAndrea Arcangeli static struct kobj_attribute defrag_attr = 39771e3aac0SAndrea Arcangeli __ATTR(defrag, 0644, defrag_show, defrag_store); 39871e3aac0SAndrea Arcangeli 39979da5407SKirill A. Shutemov static ssize_t use_zero_page_show(struct kobject *kobj, 40079da5407SKirill A. Shutemov struct kobj_attribute *attr, char *buf) 40179da5407SKirill A. Shutemov { 40279da5407SKirill A. Shutemov return single_flag_show(kobj, attr, buf, 40379da5407SKirill A. Shutemov TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 40479da5407SKirill A. Shutemov } 40579da5407SKirill A. Shutemov static ssize_t use_zero_page_store(struct kobject *kobj, 40679da5407SKirill A. Shutemov struct kobj_attribute *attr, const char *buf, size_t count) 40779da5407SKirill A. Shutemov { 40879da5407SKirill A. Shutemov return single_flag_store(kobj, attr, buf, count, 40979da5407SKirill A. Shutemov TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 41079da5407SKirill A. Shutemov } 41179da5407SKirill A. Shutemov static struct kobj_attribute use_zero_page_attr = 41279da5407SKirill A. Shutemov __ATTR(use_zero_page, 0644, use_zero_page_show, use_zero_page_store); 41371e3aac0SAndrea Arcangeli #ifdef CONFIG_DEBUG_VM 41471e3aac0SAndrea Arcangeli static ssize_t debug_cow_show(struct kobject *kobj, 41571e3aac0SAndrea Arcangeli struct kobj_attribute *attr, char *buf) 41671e3aac0SAndrea Arcangeli { 41771e3aac0SAndrea Arcangeli return single_flag_show(kobj, attr, buf, 41871e3aac0SAndrea Arcangeli TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG); 41971e3aac0SAndrea Arcangeli } 42071e3aac0SAndrea Arcangeli static ssize_t debug_cow_store(struct kobject *kobj, 42171e3aac0SAndrea Arcangeli struct kobj_attribute *attr, 42271e3aac0SAndrea Arcangeli const char *buf, size_t count) 42371e3aac0SAndrea Arcangeli { 42471e3aac0SAndrea Arcangeli return single_flag_store(kobj, attr, buf, count, 42571e3aac0SAndrea Arcangeli TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG); 42671e3aac0SAndrea Arcangeli } 42771e3aac0SAndrea Arcangeli static struct kobj_attribute debug_cow_attr = 42871e3aac0SAndrea Arcangeli __ATTR(debug_cow, 0644, debug_cow_show, debug_cow_store); 42971e3aac0SAndrea Arcangeli #endif /* CONFIG_DEBUG_VM */ 43071e3aac0SAndrea Arcangeli 43171e3aac0SAndrea Arcangeli static struct attribute *hugepage_attr[] = { 43271e3aac0SAndrea Arcangeli &enabled_attr.attr, 43371e3aac0SAndrea Arcangeli &defrag_attr.attr, 43479da5407SKirill A. Shutemov &use_zero_page_attr.attr, 43571e3aac0SAndrea Arcangeli #ifdef CONFIG_DEBUG_VM 43671e3aac0SAndrea Arcangeli &debug_cow_attr.attr, 43771e3aac0SAndrea Arcangeli #endif 43871e3aac0SAndrea Arcangeli NULL, 43971e3aac0SAndrea Arcangeli }; 44071e3aac0SAndrea Arcangeli 44171e3aac0SAndrea Arcangeli static struct attribute_group hugepage_attr_group = { 44271e3aac0SAndrea Arcangeli .attrs = hugepage_attr, 443ba76149fSAndrea Arcangeli }; 444ba76149fSAndrea Arcangeli 445ba76149fSAndrea Arcangeli static ssize_t scan_sleep_millisecs_show(struct kobject *kobj, 446ba76149fSAndrea Arcangeli struct kobj_attribute *attr, 447ba76149fSAndrea Arcangeli char *buf) 448ba76149fSAndrea Arcangeli { 449ba76149fSAndrea Arcangeli return sprintf(buf, "%u\n", khugepaged_scan_sleep_millisecs); 450ba76149fSAndrea Arcangeli } 451ba76149fSAndrea Arcangeli 452ba76149fSAndrea Arcangeli static ssize_t scan_sleep_millisecs_store(struct kobject *kobj, 453ba76149fSAndrea Arcangeli struct kobj_attribute *attr, 454ba76149fSAndrea Arcangeli const char *buf, size_t count) 455ba76149fSAndrea Arcangeli { 456ba76149fSAndrea Arcangeli unsigned long msecs; 457ba76149fSAndrea Arcangeli int err; 458ba76149fSAndrea Arcangeli 4593dbb95f7SJingoo Han err = kstrtoul(buf, 10, &msecs); 460ba76149fSAndrea Arcangeli if (err || msecs > UINT_MAX) 461ba76149fSAndrea Arcangeli return -EINVAL; 462ba76149fSAndrea Arcangeli 463ba76149fSAndrea Arcangeli khugepaged_scan_sleep_millisecs = msecs; 464ba76149fSAndrea Arcangeli wake_up_interruptible(&khugepaged_wait); 465ba76149fSAndrea Arcangeli 466ba76149fSAndrea Arcangeli return count; 467ba76149fSAndrea Arcangeli } 468ba76149fSAndrea Arcangeli static struct kobj_attribute scan_sleep_millisecs_attr = 469ba76149fSAndrea Arcangeli __ATTR(scan_sleep_millisecs, 0644, scan_sleep_millisecs_show, 470ba76149fSAndrea Arcangeli scan_sleep_millisecs_store); 471ba76149fSAndrea Arcangeli 472ba76149fSAndrea Arcangeli static ssize_t alloc_sleep_millisecs_show(struct kobject *kobj, 473ba76149fSAndrea Arcangeli struct kobj_attribute *attr, 474ba76149fSAndrea Arcangeli char *buf) 475ba76149fSAndrea Arcangeli { 476ba76149fSAndrea Arcangeli return sprintf(buf, "%u\n", khugepaged_alloc_sleep_millisecs); 477ba76149fSAndrea Arcangeli } 478ba76149fSAndrea Arcangeli 479ba76149fSAndrea Arcangeli static ssize_t alloc_sleep_millisecs_store(struct kobject *kobj, 480ba76149fSAndrea Arcangeli struct kobj_attribute *attr, 481ba76149fSAndrea Arcangeli const char *buf, size_t count) 482ba76149fSAndrea Arcangeli { 483ba76149fSAndrea Arcangeli unsigned long msecs; 484ba76149fSAndrea Arcangeli int err; 485ba76149fSAndrea Arcangeli 4863dbb95f7SJingoo Han err = kstrtoul(buf, 10, &msecs); 487ba76149fSAndrea Arcangeli if (err || msecs > UINT_MAX) 488ba76149fSAndrea Arcangeli return -EINVAL; 489ba76149fSAndrea Arcangeli 490ba76149fSAndrea Arcangeli khugepaged_alloc_sleep_millisecs = msecs; 491ba76149fSAndrea Arcangeli wake_up_interruptible(&khugepaged_wait); 492ba76149fSAndrea Arcangeli 493ba76149fSAndrea Arcangeli return count; 494ba76149fSAndrea Arcangeli } 495ba76149fSAndrea Arcangeli static struct kobj_attribute alloc_sleep_millisecs_attr = 496ba76149fSAndrea Arcangeli __ATTR(alloc_sleep_millisecs, 0644, alloc_sleep_millisecs_show, 497ba76149fSAndrea Arcangeli alloc_sleep_millisecs_store); 498ba76149fSAndrea Arcangeli 499ba76149fSAndrea Arcangeli static ssize_t pages_to_scan_show(struct kobject *kobj, 500ba76149fSAndrea Arcangeli struct kobj_attribute *attr, 501ba76149fSAndrea Arcangeli char *buf) 502ba76149fSAndrea Arcangeli { 503ba76149fSAndrea Arcangeli return sprintf(buf, "%u\n", khugepaged_pages_to_scan); 504ba76149fSAndrea Arcangeli } 505ba76149fSAndrea Arcangeli static ssize_t pages_to_scan_store(struct kobject *kobj, 506ba76149fSAndrea Arcangeli struct kobj_attribute *attr, 507ba76149fSAndrea Arcangeli const char *buf, size_t count) 508ba76149fSAndrea Arcangeli { 509ba76149fSAndrea Arcangeli int err; 510ba76149fSAndrea Arcangeli unsigned long pages; 511ba76149fSAndrea Arcangeli 5123dbb95f7SJingoo Han err = kstrtoul(buf, 10, &pages); 513ba76149fSAndrea Arcangeli if (err || !pages || pages > UINT_MAX) 514ba76149fSAndrea Arcangeli return -EINVAL; 515ba76149fSAndrea Arcangeli 516ba76149fSAndrea Arcangeli khugepaged_pages_to_scan = pages; 517ba76149fSAndrea Arcangeli 518ba76149fSAndrea Arcangeli return count; 519ba76149fSAndrea Arcangeli } 520ba76149fSAndrea Arcangeli static struct kobj_attribute pages_to_scan_attr = 521ba76149fSAndrea Arcangeli __ATTR(pages_to_scan, 0644, pages_to_scan_show, 522ba76149fSAndrea Arcangeli pages_to_scan_store); 523ba76149fSAndrea Arcangeli 524ba76149fSAndrea Arcangeli static ssize_t pages_collapsed_show(struct kobject *kobj, 525ba76149fSAndrea Arcangeli struct kobj_attribute *attr, 526ba76149fSAndrea Arcangeli char *buf) 527ba76149fSAndrea Arcangeli { 528ba76149fSAndrea Arcangeli return sprintf(buf, "%u\n", khugepaged_pages_collapsed); 529ba76149fSAndrea Arcangeli } 530ba76149fSAndrea Arcangeli static struct kobj_attribute pages_collapsed_attr = 531ba76149fSAndrea Arcangeli __ATTR_RO(pages_collapsed); 532ba76149fSAndrea Arcangeli 533ba76149fSAndrea Arcangeli static ssize_t full_scans_show(struct kobject *kobj, 534ba76149fSAndrea Arcangeli struct kobj_attribute *attr, 535ba76149fSAndrea Arcangeli char *buf) 536ba76149fSAndrea Arcangeli { 537ba76149fSAndrea Arcangeli return sprintf(buf, "%u\n", khugepaged_full_scans); 538ba76149fSAndrea Arcangeli } 539ba76149fSAndrea Arcangeli static struct kobj_attribute full_scans_attr = 540ba76149fSAndrea Arcangeli __ATTR_RO(full_scans); 541ba76149fSAndrea Arcangeli 542ba76149fSAndrea Arcangeli static ssize_t khugepaged_defrag_show(struct kobject *kobj, 543ba76149fSAndrea Arcangeli struct kobj_attribute *attr, char *buf) 544ba76149fSAndrea Arcangeli { 545ba76149fSAndrea Arcangeli return single_flag_show(kobj, attr, buf, 546ba76149fSAndrea Arcangeli TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG); 547ba76149fSAndrea Arcangeli } 548ba76149fSAndrea Arcangeli static ssize_t khugepaged_defrag_store(struct kobject *kobj, 549ba76149fSAndrea Arcangeli struct kobj_attribute *attr, 550ba76149fSAndrea Arcangeli const char *buf, size_t count) 551ba76149fSAndrea Arcangeli { 552ba76149fSAndrea Arcangeli return single_flag_store(kobj, attr, buf, count, 553ba76149fSAndrea Arcangeli TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG); 554ba76149fSAndrea Arcangeli } 555ba76149fSAndrea Arcangeli static struct kobj_attribute khugepaged_defrag_attr = 556ba76149fSAndrea Arcangeli __ATTR(defrag, 0644, khugepaged_defrag_show, 557ba76149fSAndrea Arcangeli khugepaged_defrag_store); 558ba76149fSAndrea Arcangeli 559ba76149fSAndrea Arcangeli /* 560ba76149fSAndrea Arcangeli * max_ptes_none controls if khugepaged should collapse hugepages over 561ba76149fSAndrea Arcangeli * any unmapped ptes in turn potentially increasing the memory 562ba76149fSAndrea Arcangeli * footprint of the vmas. When max_ptes_none is 0 khugepaged will not 563ba76149fSAndrea Arcangeli * reduce the available free memory in the system as it 564ba76149fSAndrea Arcangeli * runs. Increasing max_ptes_none will instead potentially reduce the 565ba76149fSAndrea Arcangeli * free memory in the system during the khugepaged scan. 566ba76149fSAndrea Arcangeli */ 567ba76149fSAndrea Arcangeli static ssize_t khugepaged_max_ptes_none_show(struct kobject *kobj, 568ba76149fSAndrea Arcangeli struct kobj_attribute *attr, 569ba76149fSAndrea Arcangeli char *buf) 570ba76149fSAndrea Arcangeli { 571ba76149fSAndrea Arcangeli return sprintf(buf, "%u\n", khugepaged_max_ptes_none); 572ba76149fSAndrea Arcangeli } 573ba76149fSAndrea Arcangeli static ssize_t khugepaged_max_ptes_none_store(struct kobject *kobj, 574ba76149fSAndrea Arcangeli struct kobj_attribute *attr, 575ba76149fSAndrea Arcangeli const char *buf, size_t count) 576ba76149fSAndrea Arcangeli { 577ba76149fSAndrea Arcangeli int err; 578ba76149fSAndrea Arcangeli unsigned long max_ptes_none; 579ba76149fSAndrea Arcangeli 5803dbb95f7SJingoo Han err = kstrtoul(buf, 10, &max_ptes_none); 581ba76149fSAndrea Arcangeli if (err || max_ptes_none > HPAGE_PMD_NR-1) 582ba76149fSAndrea Arcangeli return -EINVAL; 583ba76149fSAndrea Arcangeli 584ba76149fSAndrea Arcangeli khugepaged_max_ptes_none = max_ptes_none; 585ba76149fSAndrea Arcangeli 586ba76149fSAndrea Arcangeli return count; 587ba76149fSAndrea Arcangeli } 588ba76149fSAndrea Arcangeli static struct kobj_attribute khugepaged_max_ptes_none_attr = 589ba76149fSAndrea Arcangeli __ATTR(max_ptes_none, 0644, khugepaged_max_ptes_none_show, 590ba76149fSAndrea Arcangeli khugepaged_max_ptes_none_store); 591ba76149fSAndrea Arcangeli 592ba76149fSAndrea Arcangeli static struct attribute *khugepaged_attr[] = { 593ba76149fSAndrea Arcangeli &khugepaged_defrag_attr.attr, 594ba76149fSAndrea Arcangeli &khugepaged_max_ptes_none_attr.attr, 595ba76149fSAndrea Arcangeli &pages_to_scan_attr.attr, 596ba76149fSAndrea Arcangeli &pages_collapsed_attr.attr, 597ba76149fSAndrea Arcangeli &full_scans_attr.attr, 598ba76149fSAndrea Arcangeli &scan_sleep_millisecs_attr.attr, 599ba76149fSAndrea Arcangeli &alloc_sleep_millisecs_attr.attr, 600ba76149fSAndrea Arcangeli NULL, 601ba76149fSAndrea Arcangeli }; 602ba76149fSAndrea Arcangeli 603ba76149fSAndrea Arcangeli static struct attribute_group khugepaged_attr_group = { 604ba76149fSAndrea Arcangeli .attrs = khugepaged_attr, 605ba76149fSAndrea Arcangeli .name = "khugepaged", 60671e3aac0SAndrea Arcangeli }; 607569e5590SShaohua Li 608569e5590SShaohua Li static int __init hugepage_init_sysfs(struct kobject **hugepage_kobj) 609569e5590SShaohua Li { 610569e5590SShaohua Li int err; 611569e5590SShaohua Li 612569e5590SShaohua Li *hugepage_kobj = kobject_create_and_add("transparent_hugepage", mm_kobj); 613569e5590SShaohua Li if (unlikely(!*hugepage_kobj)) { 614ae3a8c1cSAndrew Morton pr_err("failed to create transparent hugepage kobject\n"); 615569e5590SShaohua Li return -ENOMEM; 616569e5590SShaohua Li } 617569e5590SShaohua Li 618569e5590SShaohua Li err = sysfs_create_group(*hugepage_kobj, &hugepage_attr_group); 619569e5590SShaohua Li if (err) { 620ae3a8c1cSAndrew Morton pr_err("failed to register transparent hugepage group\n"); 621569e5590SShaohua Li goto delete_obj; 622569e5590SShaohua Li } 623569e5590SShaohua Li 624569e5590SShaohua Li err = sysfs_create_group(*hugepage_kobj, &khugepaged_attr_group); 625569e5590SShaohua Li if (err) { 626ae3a8c1cSAndrew Morton pr_err("failed to register transparent hugepage group\n"); 627569e5590SShaohua Li goto remove_hp_group; 628569e5590SShaohua Li } 629569e5590SShaohua Li 630569e5590SShaohua Li return 0; 631569e5590SShaohua Li 632569e5590SShaohua Li remove_hp_group: 633569e5590SShaohua Li sysfs_remove_group(*hugepage_kobj, &hugepage_attr_group); 634569e5590SShaohua Li delete_obj: 635569e5590SShaohua Li kobject_put(*hugepage_kobj); 636569e5590SShaohua Li return err; 637569e5590SShaohua Li } 638569e5590SShaohua Li 639569e5590SShaohua Li static void __init hugepage_exit_sysfs(struct kobject *hugepage_kobj) 640569e5590SShaohua Li { 641569e5590SShaohua Li sysfs_remove_group(hugepage_kobj, &khugepaged_attr_group); 642569e5590SShaohua Li sysfs_remove_group(hugepage_kobj, &hugepage_attr_group); 643569e5590SShaohua Li kobject_put(hugepage_kobj); 644569e5590SShaohua Li } 645569e5590SShaohua Li #else 646569e5590SShaohua Li static inline int hugepage_init_sysfs(struct kobject **hugepage_kobj) 647569e5590SShaohua Li { 648569e5590SShaohua Li return 0; 649569e5590SShaohua Li } 650569e5590SShaohua Li 651569e5590SShaohua Li static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj) 652569e5590SShaohua Li { 653569e5590SShaohua Li } 65471e3aac0SAndrea Arcangeli #endif /* CONFIG_SYSFS */ 65571e3aac0SAndrea Arcangeli 65671e3aac0SAndrea Arcangeli static int __init hugepage_init(void) 65771e3aac0SAndrea Arcangeli { 65871e3aac0SAndrea Arcangeli int err; 659569e5590SShaohua Li struct kobject *hugepage_kobj; 66071e3aac0SAndrea Arcangeli 6614b7167b9SAndrea Arcangeli if (!has_transparent_hugepage()) { 6624b7167b9SAndrea Arcangeli transparent_hugepage_flags = 0; 663569e5590SShaohua Li return -EINVAL; 6644b7167b9SAndrea Arcangeli } 6654b7167b9SAndrea Arcangeli 666569e5590SShaohua Li err = hugepage_init_sysfs(&hugepage_kobj); 667569e5590SShaohua Li if (err) 66865ebb64fSKirill A. Shutemov goto err_sysfs; 669ba76149fSAndrea Arcangeli 670ba76149fSAndrea Arcangeli err = khugepaged_slab_init(); 671ba76149fSAndrea Arcangeli if (err) 67265ebb64fSKirill A. Shutemov goto err_slab; 673ba76149fSAndrea Arcangeli 67465ebb64fSKirill A. Shutemov err = register_shrinker(&huge_zero_page_shrinker); 67565ebb64fSKirill A. Shutemov if (err) 67665ebb64fSKirill A. Shutemov goto err_hzp_shrinker; 6779a982250SKirill A. Shutemov err = register_shrinker(&deferred_split_shrinker); 6789a982250SKirill A. Shutemov if (err) 6799a982250SKirill A. Shutemov goto err_split_shrinker; 68097ae1749SKirill A. Shutemov 68197562cd2SRik van Riel /* 68297562cd2SRik van Riel * By default disable transparent hugepages on smaller systems, 68397562cd2SRik van Riel * where the extra memory used could hurt more than TLB overhead 68497562cd2SRik van Riel * is likely to save. The admin can still enable it through /sys. 68597562cd2SRik van Riel */ 68679553da2SKirill A. Shutemov if (totalram_pages < (512 << (20 - PAGE_SHIFT))) { 68797562cd2SRik van Riel transparent_hugepage_flags = 0; 68879553da2SKirill A. Shutemov return 0; 68979553da2SKirill A. Shutemov } 69097562cd2SRik van Riel 69179553da2SKirill A. Shutemov err = start_stop_khugepaged(); 69265ebb64fSKirill A. Shutemov if (err) 69365ebb64fSKirill A. Shutemov goto err_khugepaged; 694ba76149fSAndrea Arcangeli 695569e5590SShaohua Li return 0; 69665ebb64fSKirill A. Shutemov err_khugepaged: 6979a982250SKirill A. Shutemov unregister_shrinker(&deferred_split_shrinker); 6989a982250SKirill A. Shutemov err_split_shrinker: 69965ebb64fSKirill A. Shutemov unregister_shrinker(&huge_zero_page_shrinker); 70065ebb64fSKirill A. Shutemov err_hzp_shrinker: 70165ebb64fSKirill A. Shutemov khugepaged_slab_exit(); 70265ebb64fSKirill A. Shutemov err_slab: 703569e5590SShaohua Li hugepage_exit_sysfs(hugepage_kobj); 70465ebb64fSKirill A. Shutemov err_sysfs: 705ba76149fSAndrea Arcangeli return err; 70671e3aac0SAndrea Arcangeli } 707a64fb3cdSPaul Gortmaker subsys_initcall(hugepage_init); 70871e3aac0SAndrea Arcangeli 70971e3aac0SAndrea Arcangeli static int __init setup_transparent_hugepage(char *str) 71071e3aac0SAndrea Arcangeli { 71171e3aac0SAndrea Arcangeli int ret = 0; 71271e3aac0SAndrea Arcangeli if (!str) 71371e3aac0SAndrea Arcangeli goto out; 71471e3aac0SAndrea Arcangeli if (!strcmp(str, "always")) { 71571e3aac0SAndrea Arcangeli set_bit(TRANSPARENT_HUGEPAGE_FLAG, 71671e3aac0SAndrea Arcangeli &transparent_hugepage_flags); 71771e3aac0SAndrea Arcangeli clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 71871e3aac0SAndrea Arcangeli &transparent_hugepage_flags); 71971e3aac0SAndrea Arcangeli ret = 1; 72071e3aac0SAndrea Arcangeli } else if (!strcmp(str, "madvise")) { 72171e3aac0SAndrea Arcangeli clear_bit(TRANSPARENT_HUGEPAGE_FLAG, 72271e3aac0SAndrea Arcangeli &transparent_hugepage_flags); 72371e3aac0SAndrea Arcangeli set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 72471e3aac0SAndrea Arcangeli &transparent_hugepage_flags); 72571e3aac0SAndrea Arcangeli ret = 1; 72671e3aac0SAndrea Arcangeli } else if (!strcmp(str, "never")) { 72771e3aac0SAndrea Arcangeli clear_bit(TRANSPARENT_HUGEPAGE_FLAG, 72871e3aac0SAndrea Arcangeli &transparent_hugepage_flags); 72971e3aac0SAndrea Arcangeli clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 73071e3aac0SAndrea Arcangeli &transparent_hugepage_flags); 73171e3aac0SAndrea Arcangeli ret = 1; 73271e3aac0SAndrea Arcangeli } 73371e3aac0SAndrea Arcangeli out: 73471e3aac0SAndrea Arcangeli if (!ret) 735ae3a8c1cSAndrew Morton pr_warn("transparent_hugepage= cannot parse, ignored\n"); 73671e3aac0SAndrea Arcangeli return ret; 73771e3aac0SAndrea Arcangeli } 73871e3aac0SAndrea Arcangeli __setup("transparent_hugepage=", setup_transparent_hugepage); 73971e3aac0SAndrea Arcangeli 740b32967ffSMel Gorman pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma) 74171e3aac0SAndrea Arcangeli { 74271e3aac0SAndrea Arcangeli if (likely(vma->vm_flags & VM_WRITE)) 74371e3aac0SAndrea Arcangeli pmd = pmd_mkwrite(pmd); 74471e3aac0SAndrea Arcangeli return pmd; 74571e3aac0SAndrea Arcangeli } 74671e3aac0SAndrea Arcangeli 7473122359aSKirill A. Shutemov static inline pmd_t mk_huge_pmd(struct page *page, pgprot_t prot) 748b3092b3bSBob Liu { 749b3092b3bSBob Liu pmd_t entry; 7503122359aSKirill A. Shutemov entry = mk_pmd(page, prot); 751b3092b3bSBob Liu entry = pmd_mkhuge(entry); 752b3092b3bSBob Liu return entry; 753b3092b3bSBob Liu } 754b3092b3bSBob Liu 7559a982250SKirill A. Shutemov static inline struct list_head *page_deferred_list(struct page *page) 7569a982250SKirill A. Shutemov { 7579a982250SKirill A. Shutemov /* 7589a982250SKirill A. Shutemov * ->lru in the tail pages is occupied by compound_head. 7599a982250SKirill A. Shutemov * Let's use ->mapping + ->index in the second tail page as list_head. 7609a982250SKirill A. Shutemov */ 7619a982250SKirill A. Shutemov return (struct list_head *)&page[2].mapping; 7629a982250SKirill A. Shutemov } 7639a982250SKirill A. Shutemov 7649a982250SKirill A. Shutemov void prep_transhuge_page(struct page *page) 7659a982250SKirill A. Shutemov { 7669a982250SKirill A. Shutemov /* 7679a982250SKirill A. Shutemov * we use page->mapping and page->indexlru in second tail page 7689a982250SKirill A. Shutemov * as list_head: assuming THP order >= 2 7699a982250SKirill A. Shutemov */ 7709a982250SKirill A. Shutemov BUILD_BUG_ON(HPAGE_PMD_ORDER < 2); 7719a982250SKirill A. Shutemov 7729a982250SKirill A. Shutemov INIT_LIST_HEAD(page_deferred_list(page)); 7739a982250SKirill A. Shutemov set_compound_page_dtor(page, TRANSHUGE_PAGE_DTOR); 7749a982250SKirill A. Shutemov } 7759a982250SKirill A. Shutemov 77671e3aac0SAndrea Arcangeli static int __do_huge_pmd_anonymous_page(struct mm_struct *mm, 77771e3aac0SAndrea Arcangeli struct vm_area_struct *vma, 778230c92a8SAndrea Arcangeli unsigned long address, pmd_t *pmd, 7796b251fc9SAndrea Arcangeli struct page *page, gfp_t gfp, 7806b251fc9SAndrea Arcangeli unsigned int flags) 78171e3aac0SAndrea Arcangeli { 78200501b53SJohannes Weiner struct mem_cgroup *memcg; 78371e3aac0SAndrea Arcangeli pgtable_t pgtable; 784c4088ebdSKirill A. Shutemov spinlock_t *ptl; 785230c92a8SAndrea Arcangeli unsigned long haddr = address & HPAGE_PMD_MASK; 78671e3aac0SAndrea Arcangeli 787309381feSSasha Levin VM_BUG_ON_PAGE(!PageCompound(page), page); 78800501b53SJohannes Weiner 789f627c2f5SKirill A. Shutemov if (mem_cgroup_try_charge(page, mm, gfp, &memcg, true)) { 7906b251fc9SAndrea Arcangeli put_page(page); 7916b251fc9SAndrea Arcangeli count_vm_event(THP_FAULT_FALLBACK); 7926b251fc9SAndrea Arcangeli return VM_FAULT_FALLBACK; 7936b251fc9SAndrea Arcangeli } 79471e3aac0SAndrea Arcangeli 79500501b53SJohannes Weiner pgtable = pte_alloc_one(mm, haddr); 79600501b53SJohannes Weiner if (unlikely(!pgtable)) { 797f627c2f5SKirill A. Shutemov mem_cgroup_cancel_charge(page, memcg, true); 7986b251fc9SAndrea Arcangeli put_page(page); 79900501b53SJohannes Weiner return VM_FAULT_OOM; 80000501b53SJohannes Weiner } 80100501b53SJohannes Weiner 80271e3aac0SAndrea Arcangeli clear_huge_page(page, haddr, HPAGE_PMD_NR); 80352f37629SMinchan Kim /* 80452f37629SMinchan Kim * The memory barrier inside __SetPageUptodate makes sure that 80552f37629SMinchan Kim * clear_huge_page writes become visible before the set_pmd_at() 80652f37629SMinchan Kim * write. 80752f37629SMinchan Kim */ 80871e3aac0SAndrea Arcangeli __SetPageUptodate(page); 80971e3aac0SAndrea Arcangeli 810c4088ebdSKirill A. Shutemov ptl = pmd_lock(mm, pmd); 81171e3aac0SAndrea Arcangeli if (unlikely(!pmd_none(*pmd))) { 812c4088ebdSKirill A. Shutemov spin_unlock(ptl); 813f627c2f5SKirill A. Shutemov mem_cgroup_cancel_charge(page, memcg, true); 81471e3aac0SAndrea Arcangeli put_page(page); 81571e3aac0SAndrea Arcangeli pte_free(mm, pgtable); 81671e3aac0SAndrea Arcangeli } else { 81771e3aac0SAndrea Arcangeli pmd_t entry; 8186b251fc9SAndrea Arcangeli 8196b251fc9SAndrea Arcangeli /* Deliver the page fault to userland */ 8206b251fc9SAndrea Arcangeli if (userfaultfd_missing(vma)) { 8216b251fc9SAndrea Arcangeli int ret; 8226b251fc9SAndrea Arcangeli 8236b251fc9SAndrea Arcangeli spin_unlock(ptl); 824f627c2f5SKirill A. Shutemov mem_cgroup_cancel_charge(page, memcg, true); 8256b251fc9SAndrea Arcangeli put_page(page); 8266b251fc9SAndrea Arcangeli pte_free(mm, pgtable); 827230c92a8SAndrea Arcangeli ret = handle_userfault(vma, address, flags, 8286b251fc9SAndrea Arcangeli VM_UFFD_MISSING); 8296b251fc9SAndrea Arcangeli VM_BUG_ON(ret & VM_FAULT_FALLBACK); 8306b251fc9SAndrea Arcangeli return ret; 8316b251fc9SAndrea Arcangeli } 8326b251fc9SAndrea Arcangeli 8333122359aSKirill A. Shutemov entry = mk_huge_pmd(page, vma->vm_page_prot); 8343122359aSKirill A. Shutemov entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 835d281ee61SKirill A. Shutemov page_add_new_anon_rmap(page, vma, haddr, true); 836f627c2f5SKirill A. Shutemov mem_cgroup_commit_charge(page, memcg, false, true); 83700501b53SJohannes Weiner lru_cache_add_active_or_unevictable(page, vma); 8386b0b50b0SAneesh Kumar K.V pgtable_trans_huge_deposit(mm, pmd, pgtable); 83971e3aac0SAndrea Arcangeli set_pmd_at(mm, haddr, pmd, entry); 84071e3aac0SAndrea Arcangeli add_mm_counter(mm, MM_ANONPAGES, HPAGE_PMD_NR); 841e1f56c89SKirill A. Shutemov atomic_long_inc(&mm->nr_ptes); 842c4088ebdSKirill A. Shutemov spin_unlock(ptl); 8436b251fc9SAndrea Arcangeli count_vm_event(THP_FAULT_ALLOC); 84471e3aac0SAndrea Arcangeli } 84571e3aac0SAndrea Arcangeli 846aa2e878eSDavid Rientjes return 0; 84771e3aac0SAndrea Arcangeli } 84871e3aac0SAndrea Arcangeli 849cc5d462fSAndi Kleen static inline gfp_t alloc_hugepage_gfpmask(int defrag, gfp_t extra_gfp) 8500bbbc0b3SAndrea Arcangeli { 85171baba4bSMel Gorman return (GFP_TRANSHUGE & ~(defrag ? 0 : __GFP_RECLAIM)) | extra_gfp; 8520bbbc0b3SAndrea Arcangeli } 8530bbbc0b3SAndrea Arcangeli 854c4088ebdSKirill A. Shutemov /* Caller must hold page table lock. */ 855d295e341SKirill A. Shutemov static bool set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm, 85697ae1749SKirill A. Shutemov struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd, 8575918d10aSKirill A. Shutemov struct page *zero_page) 858fc9fe822SKirill A. Shutemov { 859fc9fe822SKirill A. Shutemov pmd_t entry; 8607c414164SAndrew Morton if (!pmd_none(*pmd)) 8617c414164SAndrew Morton return false; 8625918d10aSKirill A. Shutemov entry = mk_pmd(zero_page, vma->vm_page_prot); 863fc9fe822SKirill A. Shutemov entry = pmd_mkhuge(entry); 8646b0b50b0SAneesh Kumar K.V pgtable_trans_huge_deposit(mm, pmd, pgtable); 865fc9fe822SKirill A. Shutemov set_pmd_at(mm, haddr, pmd, entry); 866e1f56c89SKirill A. Shutemov atomic_long_inc(&mm->nr_ptes); 8677c414164SAndrew Morton return true; 868fc9fe822SKirill A. Shutemov } 869fc9fe822SKirill A. Shutemov 87071e3aac0SAndrea Arcangeli int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma, 87171e3aac0SAndrea Arcangeli unsigned long address, pmd_t *pmd, 87271e3aac0SAndrea Arcangeli unsigned int flags) 87371e3aac0SAndrea Arcangeli { 874077fcf11SAneesh Kumar K.V gfp_t gfp; 87571e3aac0SAndrea Arcangeli struct page *page; 87671e3aac0SAndrea Arcangeli unsigned long haddr = address & HPAGE_PMD_MASK; 87771e3aac0SAndrea Arcangeli 878128ec037SKirill A. Shutemov if (haddr < vma->vm_start || haddr + HPAGE_PMD_SIZE > vma->vm_end) 879c0292554SKirill A. Shutemov return VM_FAULT_FALLBACK; 88071e3aac0SAndrea Arcangeli if (unlikely(anon_vma_prepare(vma))) 88171e3aac0SAndrea Arcangeli return VM_FAULT_OOM; 8826d50e60cSDavid Rientjes if (unlikely(khugepaged_enter(vma, vma->vm_flags))) 883ba76149fSAndrea Arcangeli return VM_FAULT_OOM; 884593befa6SDominik Dingel if (!(flags & FAULT_FLAG_WRITE) && !mm_forbids_zeropage(mm) && 88579da5407SKirill A. Shutemov transparent_hugepage_use_zero_page()) { 886c4088ebdSKirill A. Shutemov spinlock_t *ptl; 88780371957SKirill A. Shutemov pgtable_t pgtable; 8885918d10aSKirill A. Shutemov struct page *zero_page; 8893ea41e62SKirill A. Shutemov bool set; 8906b251fc9SAndrea Arcangeli int ret; 89180371957SKirill A. Shutemov pgtable = pte_alloc_one(mm, haddr); 89280371957SKirill A. Shutemov if (unlikely(!pgtable)) 89380371957SKirill A. Shutemov return VM_FAULT_OOM; 8945918d10aSKirill A. Shutemov zero_page = get_huge_zero_page(); 8955918d10aSKirill A. Shutemov if (unlikely(!zero_page)) { 89697ae1749SKirill A. Shutemov pte_free(mm, pgtable); 89797ae1749SKirill A. Shutemov count_vm_event(THP_FAULT_FALLBACK); 898c0292554SKirill A. Shutemov return VM_FAULT_FALLBACK; 89997ae1749SKirill A. Shutemov } 900c4088ebdSKirill A. Shutemov ptl = pmd_lock(mm, pmd); 9016b251fc9SAndrea Arcangeli ret = 0; 9026b251fc9SAndrea Arcangeli set = false; 9036b251fc9SAndrea Arcangeli if (pmd_none(*pmd)) { 9046b251fc9SAndrea Arcangeli if (userfaultfd_missing(vma)) { 9056b251fc9SAndrea Arcangeli spin_unlock(ptl); 906230c92a8SAndrea Arcangeli ret = handle_userfault(vma, address, flags, 9076b251fc9SAndrea Arcangeli VM_UFFD_MISSING); 9086b251fc9SAndrea Arcangeli VM_BUG_ON(ret & VM_FAULT_FALLBACK); 9096b251fc9SAndrea Arcangeli } else { 9106b251fc9SAndrea Arcangeli set_huge_zero_page(pgtable, mm, vma, 9116b251fc9SAndrea Arcangeli haddr, pmd, 9125918d10aSKirill A. Shutemov zero_page); 913c4088ebdSKirill A. Shutemov spin_unlock(ptl); 9146b251fc9SAndrea Arcangeli set = true; 9156b251fc9SAndrea Arcangeli } 9166b251fc9SAndrea Arcangeli } else 9176b251fc9SAndrea Arcangeli spin_unlock(ptl); 9183ea41e62SKirill A. Shutemov if (!set) { 9193ea41e62SKirill A. Shutemov pte_free(mm, pgtable); 9203ea41e62SKirill A. Shutemov put_huge_zero_page(); 9213ea41e62SKirill A. Shutemov } 9226b251fc9SAndrea Arcangeli return ret; 92380371957SKirill A. Shutemov } 924077fcf11SAneesh Kumar K.V gfp = alloc_hugepage_gfpmask(transparent_hugepage_defrag(vma), 0); 925077fcf11SAneesh Kumar K.V page = alloc_hugepage_vma(gfp, vma, haddr, HPAGE_PMD_ORDER); 92681ab4201SAndi Kleen if (unlikely(!page)) { 92781ab4201SAndi Kleen count_vm_event(THP_FAULT_FALLBACK); 928c0292554SKirill A. Shutemov return VM_FAULT_FALLBACK; 92981ab4201SAndi Kleen } 9309a982250SKirill A. Shutemov prep_transhuge_page(page); 931230c92a8SAndrea Arcangeli return __do_huge_pmd_anonymous_page(mm, vma, address, pmd, page, gfp, 932230c92a8SAndrea Arcangeli flags); 93371e3aac0SAndrea Arcangeli } 93471e3aac0SAndrea Arcangeli 935ae18d6dcSMatthew Wilcox static void insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr, 936f25748e3SDan Williams pmd_t *pmd, pfn_t pfn, pgprot_t prot, bool write) 9375cad465dSMatthew Wilcox { 9385cad465dSMatthew Wilcox struct mm_struct *mm = vma->vm_mm; 9395cad465dSMatthew Wilcox pmd_t entry; 9405cad465dSMatthew Wilcox spinlock_t *ptl; 9415cad465dSMatthew Wilcox 9425cad465dSMatthew Wilcox ptl = pmd_lock(mm, pmd); 943f25748e3SDan Williams entry = pmd_mkhuge(pfn_t_pmd(pfn, prot)); 944f25748e3SDan Williams if (pfn_t_devmap(pfn)) 945f25748e3SDan Williams entry = pmd_mkdevmap(entry); 9465cad465dSMatthew Wilcox if (write) { 9475cad465dSMatthew Wilcox entry = pmd_mkyoung(pmd_mkdirty(entry)); 9485cad465dSMatthew Wilcox entry = maybe_pmd_mkwrite(entry, vma); 9495cad465dSMatthew Wilcox } 9505cad465dSMatthew Wilcox set_pmd_at(mm, addr, pmd, entry); 9515cad465dSMatthew Wilcox update_mmu_cache_pmd(vma, addr, pmd); 9525cad465dSMatthew Wilcox spin_unlock(ptl); 9535cad465dSMatthew Wilcox } 9545cad465dSMatthew Wilcox 9555cad465dSMatthew Wilcox int vmf_insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr, 956f25748e3SDan Williams pmd_t *pmd, pfn_t pfn, bool write) 9575cad465dSMatthew Wilcox { 9585cad465dSMatthew Wilcox pgprot_t pgprot = vma->vm_page_prot; 9595cad465dSMatthew Wilcox /* 9605cad465dSMatthew Wilcox * If we had pmd_special, we could avoid all these restrictions, 9615cad465dSMatthew Wilcox * but we need to be consistent with PTEs and architectures that 9625cad465dSMatthew Wilcox * can't support a 'special' bit. 9635cad465dSMatthew Wilcox */ 9645cad465dSMatthew Wilcox BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))); 9655cad465dSMatthew Wilcox BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 9665cad465dSMatthew Wilcox (VM_PFNMAP|VM_MIXEDMAP)); 9675cad465dSMatthew Wilcox BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 968f25748e3SDan Williams BUG_ON(!pfn_t_devmap(pfn)); 9695cad465dSMatthew Wilcox 9705cad465dSMatthew Wilcox if (addr < vma->vm_start || addr >= vma->vm_end) 9715cad465dSMatthew Wilcox return VM_FAULT_SIGBUS; 9725cad465dSMatthew Wilcox if (track_pfn_insert(vma, &pgprot, pfn)) 9735cad465dSMatthew Wilcox return VM_FAULT_SIGBUS; 974ae18d6dcSMatthew Wilcox insert_pfn_pmd(vma, addr, pmd, pfn, pgprot, write); 975ae18d6dcSMatthew Wilcox return VM_FAULT_NOPAGE; 9765cad465dSMatthew Wilcox } 9775cad465dSMatthew Wilcox 9783565fce3SDan Williams static void touch_pmd(struct vm_area_struct *vma, unsigned long addr, 9793565fce3SDan Williams pmd_t *pmd) 9803565fce3SDan Williams { 9813565fce3SDan Williams pmd_t _pmd; 9823565fce3SDan Williams 9833565fce3SDan Williams /* 9843565fce3SDan Williams * We should set the dirty bit only for FOLL_WRITE but for now 9853565fce3SDan Williams * the dirty bit in the pmd is meaningless. And if the dirty 9863565fce3SDan Williams * bit will become meaningful and we'll only set it with 9873565fce3SDan Williams * FOLL_WRITE, an atomic set_bit will be required on the pmd to 9883565fce3SDan Williams * set the young bit, instead of the current set_pmd_at. 9893565fce3SDan Williams */ 9903565fce3SDan Williams _pmd = pmd_mkyoung(pmd_mkdirty(*pmd)); 9913565fce3SDan Williams if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK, 9923565fce3SDan Williams pmd, _pmd, 1)) 9933565fce3SDan Williams update_mmu_cache_pmd(vma, addr, pmd); 9943565fce3SDan Williams } 9953565fce3SDan Williams 9963565fce3SDan Williams struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr, 9973565fce3SDan Williams pmd_t *pmd, int flags) 9983565fce3SDan Williams { 9993565fce3SDan Williams unsigned long pfn = pmd_pfn(*pmd); 10003565fce3SDan Williams struct mm_struct *mm = vma->vm_mm; 10013565fce3SDan Williams struct dev_pagemap *pgmap; 10023565fce3SDan Williams struct page *page; 10033565fce3SDan Williams 10043565fce3SDan Williams assert_spin_locked(pmd_lockptr(mm, pmd)); 10053565fce3SDan Williams 10063565fce3SDan Williams if (flags & FOLL_WRITE && !pmd_write(*pmd)) 10073565fce3SDan Williams return NULL; 10083565fce3SDan Williams 10093565fce3SDan Williams if (pmd_present(*pmd) && pmd_devmap(*pmd)) 10103565fce3SDan Williams /* pass */; 10113565fce3SDan Williams else 10123565fce3SDan Williams return NULL; 10133565fce3SDan Williams 10143565fce3SDan Williams if (flags & FOLL_TOUCH) 10153565fce3SDan Williams touch_pmd(vma, addr, pmd); 10163565fce3SDan Williams 10173565fce3SDan Williams /* 10183565fce3SDan Williams * device mapped pages can only be returned if the 10193565fce3SDan Williams * caller will manage the page reference count. 10203565fce3SDan Williams */ 10213565fce3SDan Williams if (!(flags & FOLL_GET)) 10223565fce3SDan Williams return ERR_PTR(-EEXIST); 10233565fce3SDan Williams 10243565fce3SDan Williams pfn += (addr & ~PMD_MASK) >> PAGE_SHIFT; 10253565fce3SDan Williams pgmap = get_dev_pagemap(pfn, NULL); 10263565fce3SDan Williams if (!pgmap) 10273565fce3SDan Williams return ERR_PTR(-EFAULT); 10283565fce3SDan Williams page = pfn_to_page(pfn); 10293565fce3SDan Williams get_page(page); 10303565fce3SDan Williams put_dev_pagemap(pgmap); 10313565fce3SDan Williams 10323565fce3SDan Williams return page; 10333565fce3SDan Williams } 10343565fce3SDan Williams 103571e3aac0SAndrea Arcangeli int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, 103671e3aac0SAndrea Arcangeli pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr, 103771e3aac0SAndrea Arcangeli struct vm_area_struct *vma) 103871e3aac0SAndrea Arcangeli { 1039c4088ebdSKirill A. Shutemov spinlock_t *dst_ptl, *src_ptl; 104071e3aac0SAndrea Arcangeli struct page *src_page; 104171e3aac0SAndrea Arcangeli pmd_t pmd; 104271e3aac0SAndrea Arcangeli pgtable_t pgtable; 104371e3aac0SAndrea Arcangeli int ret; 104471e3aac0SAndrea Arcangeli 104571e3aac0SAndrea Arcangeli ret = -ENOMEM; 104671e3aac0SAndrea Arcangeli pgtable = pte_alloc_one(dst_mm, addr); 104771e3aac0SAndrea Arcangeli if (unlikely(!pgtable)) 104871e3aac0SAndrea Arcangeli goto out; 104971e3aac0SAndrea Arcangeli 1050c4088ebdSKirill A. Shutemov dst_ptl = pmd_lock(dst_mm, dst_pmd); 1051c4088ebdSKirill A. Shutemov src_ptl = pmd_lockptr(src_mm, src_pmd); 1052c4088ebdSKirill A. Shutemov spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 105371e3aac0SAndrea Arcangeli 105471e3aac0SAndrea Arcangeli ret = -EAGAIN; 105571e3aac0SAndrea Arcangeli pmd = *src_pmd; 10565c7fb56eSDan Williams if (unlikely(!pmd_trans_huge(pmd) && !pmd_devmap(pmd))) { 105771e3aac0SAndrea Arcangeli pte_free(dst_mm, pgtable); 105871e3aac0SAndrea Arcangeli goto out_unlock; 105971e3aac0SAndrea Arcangeli } 1060fc9fe822SKirill A. Shutemov /* 1061c4088ebdSKirill A. Shutemov * When page table lock is held, the huge zero pmd should not be 1062fc9fe822SKirill A. Shutemov * under splitting since we don't split the page itself, only pmd to 1063fc9fe822SKirill A. Shutemov * a page table. 1064fc9fe822SKirill A. Shutemov */ 1065fc9fe822SKirill A. Shutemov if (is_huge_zero_pmd(pmd)) { 10665918d10aSKirill A. Shutemov struct page *zero_page; 106797ae1749SKirill A. Shutemov /* 106897ae1749SKirill A. Shutemov * get_huge_zero_page() will never allocate a new page here, 106997ae1749SKirill A. Shutemov * since we already have a zero page to copy. It just takes a 107097ae1749SKirill A. Shutemov * reference. 107197ae1749SKirill A. Shutemov */ 10725918d10aSKirill A. Shutemov zero_page = get_huge_zero_page(); 10736b251fc9SAndrea Arcangeli set_huge_zero_page(pgtable, dst_mm, vma, addr, dst_pmd, 10745918d10aSKirill A. Shutemov zero_page); 1075fc9fe822SKirill A. Shutemov ret = 0; 1076fc9fe822SKirill A. Shutemov goto out_unlock; 1077fc9fe822SKirill A. Shutemov } 1078de466bd6SMel Gorman 10795c7fb56eSDan Williams if (pmd_trans_huge(pmd)) { 10805c7fb56eSDan Williams /* thp accounting separate from pmd_devmap accounting */ 108171e3aac0SAndrea Arcangeli src_page = pmd_page(pmd); 1082309381feSSasha Levin VM_BUG_ON_PAGE(!PageHead(src_page), src_page); 108371e3aac0SAndrea Arcangeli get_page(src_page); 108453f9263bSKirill A. Shutemov page_dup_rmap(src_page, true); 108571e3aac0SAndrea Arcangeli add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); 10865c7fb56eSDan Williams atomic_long_inc(&dst_mm->nr_ptes); 10875c7fb56eSDan Williams pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); 10885c7fb56eSDan Williams } 108971e3aac0SAndrea Arcangeli 109071e3aac0SAndrea Arcangeli pmdp_set_wrprotect(src_mm, addr, src_pmd); 109171e3aac0SAndrea Arcangeli pmd = pmd_mkold(pmd_wrprotect(pmd)); 109271e3aac0SAndrea Arcangeli set_pmd_at(dst_mm, addr, dst_pmd, pmd); 109371e3aac0SAndrea Arcangeli 109471e3aac0SAndrea Arcangeli ret = 0; 109571e3aac0SAndrea Arcangeli out_unlock: 1096c4088ebdSKirill A. Shutemov spin_unlock(src_ptl); 1097c4088ebdSKirill A. Shutemov spin_unlock(dst_ptl); 109871e3aac0SAndrea Arcangeli out: 109971e3aac0SAndrea Arcangeli return ret; 110071e3aac0SAndrea Arcangeli } 110171e3aac0SAndrea Arcangeli 1102a1dd450bSWill Deacon void huge_pmd_set_accessed(struct mm_struct *mm, 1103a1dd450bSWill Deacon struct vm_area_struct *vma, 1104a1dd450bSWill Deacon unsigned long address, 1105a1dd450bSWill Deacon pmd_t *pmd, pmd_t orig_pmd, 1106a1dd450bSWill Deacon int dirty) 1107a1dd450bSWill Deacon { 1108c4088ebdSKirill A. Shutemov spinlock_t *ptl; 1109a1dd450bSWill Deacon pmd_t entry; 1110a1dd450bSWill Deacon unsigned long haddr; 1111a1dd450bSWill Deacon 1112c4088ebdSKirill A. Shutemov ptl = pmd_lock(mm, pmd); 1113a1dd450bSWill Deacon if (unlikely(!pmd_same(*pmd, orig_pmd))) 1114a1dd450bSWill Deacon goto unlock; 1115a1dd450bSWill Deacon 1116a1dd450bSWill Deacon entry = pmd_mkyoung(orig_pmd); 1117a1dd450bSWill Deacon haddr = address & HPAGE_PMD_MASK; 1118a1dd450bSWill Deacon if (pmdp_set_access_flags(vma, haddr, pmd, entry, dirty)) 1119a1dd450bSWill Deacon update_mmu_cache_pmd(vma, address, pmd); 1120a1dd450bSWill Deacon 1121a1dd450bSWill Deacon unlock: 1122c4088ebdSKirill A. Shutemov spin_unlock(ptl); 1123a1dd450bSWill Deacon } 1124a1dd450bSWill Deacon 112571e3aac0SAndrea Arcangeli static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm, 112671e3aac0SAndrea Arcangeli struct vm_area_struct *vma, 112771e3aac0SAndrea Arcangeli unsigned long address, 112871e3aac0SAndrea Arcangeli pmd_t *pmd, pmd_t orig_pmd, 112971e3aac0SAndrea Arcangeli struct page *page, 113071e3aac0SAndrea Arcangeli unsigned long haddr) 113171e3aac0SAndrea Arcangeli { 113200501b53SJohannes Weiner struct mem_cgroup *memcg; 1133c4088ebdSKirill A. Shutemov spinlock_t *ptl; 113471e3aac0SAndrea Arcangeli pgtable_t pgtable; 113571e3aac0SAndrea Arcangeli pmd_t _pmd; 113671e3aac0SAndrea Arcangeli int ret = 0, i; 113771e3aac0SAndrea Arcangeli struct page **pages; 11382ec74c3eSSagi Grimberg unsigned long mmun_start; /* For mmu_notifiers */ 11392ec74c3eSSagi Grimberg unsigned long mmun_end; /* For mmu_notifiers */ 114071e3aac0SAndrea Arcangeli 114171e3aac0SAndrea Arcangeli pages = kmalloc(sizeof(struct page *) * HPAGE_PMD_NR, 114271e3aac0SAndrea Arcangeli GFP_KERNEL); 114371e3aac0SAndrea Arcangeli if (unlikely(!pages)) { 114471e3aac0SAndrea Arcangeli ret |= VM_FAULT_OOM; 114571e3aac0SAndrea Arcangeli goto out; 114671e3aac0SAndrea Arcangeli } 114771e3aac0SAndrea Arcangeli 114871e3aac0SAndrea Arcangeli for (i = 0; i < HPAGE_PMD_NR; i++) { 1149cc5d462fSAndi Kleen pages[i] = alloc_page_vma_node(GFP_HIGHUSER_MOVABLE | 1150cc5d462fSAndi Kleen __GFP_OTHER_NODE, 115119ee151eSAndi Kleen vma, address, page_to_nid(page)); 1152b9bbfbe3SAndrea Arcangeli if (unlikely(!pages[i] || 115300501b53SJohannes Weiner mem_cgroup_try_charge(pages[i], mm, GFP_KERNEL, 1154f627c2f5SKirill A. Shutemov &memcg, false))) { 1155b9bbfbe3SAndrea Arcangeli if (pages[i]) 115671e3aac0SAndrea Arcangeli put_page(pages[i]); 1157b9bbfbe3SAndrea Arcangeli while (--i >= 0) { 115800501b53SJohannes Weiner memcg = (void *)page_private(pages[i]); 115900501b53SJohannes Weiner set_page_private(pages[i], 0); 1160f627c2f5SKirill A. Shutemov mem_cgroup_cancel_charge(pages[i], memcg, 1161f627c2f5SKirill A. Shutemov false); 1162b9bbfbe3SAndrea Arcangeli put_page(pages[i]); 1163b9bbfbe3SAndrea Arcangeli } 116471e3aac0SAndrea Arcangeli kfree(pages); 116571e3aac0SAndrea Arcangeli ret |= VM_FAULT_OOM; 116671e3aac0SAndrea Arcangeli goto out; 116771e3aac0SAndrea Arcangeli } 116800501b53SJohannes Weiner set_page_private(pages[i], (unsigned long)memcg); 116971e3aac0SAndrea Arcangeli } 117071e3aac0SAndrea Arcangeli 117171e3aac0SAndrea Arcangeli for (i = 0; i < HPAGE_PMD_NR; i++) { 117271e3aac0SAndrea Arcangeli copy_user_highpage(pages[i], page + i, 11730089e485SHillf Danton haddr + PAGE_SIZE * i, vma); 117471e3aac0SAndrea Arcangeli __SetPageUptodate(pages[i]); 117571e3aac0SAndrea Arcangeli cond_resched(); 117671e3aac0SAndrea Arcangeli } 117771e3aac0SAndrea Arcangeli 11782ec74c3eSSagi Grimberg mmun_start = haddr; 11792ec74c3eSSagi Grimberg mmun_end = haddr + HPAGE_PMD_SIZE; 11802ec74c3eSSagi Grimberg mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 11812ec74c3eSSagi Grimberg 1182c4088ebdSKirill A. Shutemov ptl = pmd_lock(mm, pmd); 118371e3aac0SAndrea Arcangeli if (unlikely(!pmd_same(*pmd, orig_pmd))) 118471e3aac0SAndrea Arcangeli goto out_free_pages; 1185309381feSSasha Levin VM_BUG_ON_PAGE(!PageHead(page), page); 118671e3aac0SAndrea Arcangeli 11878809aa2dSAneesh Kumar K.V pmdp_huge_clear_flush_notify(vma, haddr, pmd); 118871e3aac0SAndrea Arcangeli /* leave pmd empty until pte is filled */ 118971e3aac0SAndrea Arcangeli 11906b0b50b0SAneesh Kumar K.V pgtable = pgtable_trans_huge_withdraw(mm, pmd); 119171e3aac0SAndrea Arcangeli pmd_populate(mm, &_pmd, pgtable); 119271e3aac0SAndrea Arcangeli 119371e3aac0SAndrea Arcangeli for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) { 119471e3aac0SAndrea Arcangeli pte_t *pte, entry; 119571e3aac0SAndrea Arcangeli entry = mk_pte(pages[i], vma->vm_page_prot); 119671e3aac0SAndrea Arcangeli entry = maybe_mkwrite(pte_mkdirty(entry), vma); 119700501b53SJohannes Weiner memcg = (void *)page_private(pages[i]); 119800501b53SJohannes Weiner set_page_private(pages[i], 0); 1199d281ee61SKirill A. Shutemov page_add_new_anon_rmap(pages[i], vma, haddr, false); 1200f627c2f5SKirill A. Shutemov mem_cgroup_commit_charge(pages[i], memcg, false, false); 120100501b53SJohannes Weiner lru_cache_add_active_or_unevictable(pages[i], vma); 120271e3aac0SAndrea Arcangeli pte = pte_offset_map(&_pmd, haddr); 120371e3aac0SAndrea Arcangeli VM_BUG_ON(!pte_none(*pte)); 120471e3aac0SAndrea Arcangeli set_pte_at(mm, haddr, pte, entry); 120571e3aac0SAndrea Arcangeli pte_unmap(pte); 120671e3aac0SAndrea Arcangeli } 120771e3aac0SAndrea Arcangeli kfree(pages); 120871e3aac0SAndrea Arcangeli 120971e3aac0SAndrea Arcangeli smp_wmb(); /* make pte visible before pmd */ 121071e3aac0SAndrea Arcangeli pmd_populate(mm, pmd, pgtable); 1211d281ee61SKirill A. Shutemov page_remove_rmap(page, true); 1212c4088ebdSKirill A. Shutemov spin_unlock(ptl); 121371e3aac0SAndrea Arcangeli 12142ec74c3eSSagi Grimberg mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 12152ec74c3eSSagi Grimberg 121671e3aac0SAndrea Arcangeli ret |= VM_FAULT_WRITE; 121771e3aac0SAndrea Arcangeli put_page(page); 121871e3aac0SAndrea Arcangeli 121971e3aac0SAndrea Arcangeli out: 122071e3aac0SAndrea Arcangeli return ret; 122171e3aac0SAndrea Arcangeli 122271e3aac0SAndrea Arcangeli out_free_pages: 1223c4088ebdSKirill A. Shutemov spin_unlock(ptl); 12242ec74c3eSSagi Grimberg mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 1225b9bbfbe3SAndrea Arcangeli for (i = 0; i < HPAGE_PMD_NR; i++) { 122600501b53SJohannes Weiner memcg = (void *)page_private(pages[i]); 122700501b53SJohannes Weiner set_page_private(pages[i], 0); 1228f627c2f5SKirill A. Shutemov mem_cgroup_cancel_charge(pages[i], memcg, false); 122971e3aac0SAndrea Arcangeli put_page(pages[i]); 1230b9bbfbe3SAndrea Arcangeli } 123171e3aac0SAndrea Arcangeli kfree(pages); 123271e3aac0SAndrea Arcangeli goto out; 123371e3aac0SAndrea Arcangeli } 123471e3aac0SAndrea Arcangeli 123571e3aac0SAndrea Arcangeli int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma, 123671e3aac0SAndrea Arcangeli unsigned long address, pmd_t *pmd, pmd_t orig_pmd) 123771e3aac0SAndrea Arcangeli { 1238c4088ebdSKirill A. Shutemov spinlock_t *ptl; 123971e3aac0SAndrea Arcangeli int ret = 0; 124093b4796dSKirill A. Shutemov struct page *page = NULL, *new_page; 124100501b53SJohannes Weiner struct mem_cgroup *memcg; 124271e3aac0SAndrea Arcangeli unsigned long haddr; 12432ec74c3eSSagi Grimberg unsigned long mmun_start; /* For mmu_notifiers */ 12442ec74c3eSSagi Grimberg unsigned long mmun_end; /* For mmu_notifiers */ 12453b363692SMichal Hocko gfp_t huge_gfp; /* for allocation and charge */ 124671e3aac0SAndrea Arcangeli 1247c4088ebdSKirill A. Shutemov ptl = pmd_lockptr(mm, pmd); 124881d1b09cSSasha Levin VM_BUG_ON_VMA(!vma->anon_vma, vma); 124993b4796dSKirill A. Shutemov haddr = address & HPAGE_PMD_MASK; 125093b4796dSKirill A. Shutemov if (is_huge_zero_pmd(orig_pmd)) 125193b4796dSKirill A. Shutemov goto alloc; 1252c4088ebdSKirill A. Shutemov spin_lock(ptl); 125371e3aac0SAndrea Arcangeli if (unlikely(!pmd_same(*pmd, orig_pmd))) 125471e3aac0SAndrea Arcangeli goto out_unlock; 125571e3aac0SAndrea Arcangeli 125671e3aac0SAndrea Arcangeli page = pmd_page(orig_pmd); 1257309381feSSasha Levin VM_BUG_ON_PAGE(!PageCompound(page) || !PageHead(page), page); 12581f25fe20SKirill A. Shutemov /* 12591f25fe20SKirill A. Shutemov * We can only reuse the page if nobody else maps the huge page or it's 12601f25fe20SKirill A. Shutemov * part. We can do it by checking page_mapcount() on each sub-page, but 12611f25fe20SKirill A. Shutemov * it's expensive. 12621f25fe20SKirill A. Shutemov * The cheaper way is to check page_count() to be equal 1: every 12631f25fe20SKirill A. Shutemov * mapcount takes page reference reference, so this way we can 12641f25fe20SKirill A. Shutemov * guarantee, that the PMD is the only mapping. 12651f25fe20SKirill A. Shutemov * This can give false negative if somebody pinned the page, but that's 12661f25fe20SKirill A. Shutemov * fine. 12671f25fe20SKirill A. Shutemov */ 12681f25fe20SKirill A. Shutemov if (page_mapcount(page) == 1 && page_count(page) == 1) { 126971e3aac0SAndrea Arcangeli pmd_t entry; 127071e3aac0SAndrea Arcangeli entry = pmd_mkyoung(orig_pmd); 127171e3aac0SAndrea Arcangeli entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 127271e3aac0SAndrea Arcangeli if (pmdp_set_access_flags(vma, haddr, pmd, entry, 1)) 1273b113da65SDavid Miller update_mmu_cache_pmd(vma, address, pmd); 127471e3aac0SAndrea Arcangeli ret |= VM_FAULT_WRITE; 127571e3aac0SAndrea Arcangeli goto out_unlock; 127671e3aac0SAndrea Arcangeli } 1277ddc58f27SKirill A. Shutemov get_page(page); 1278c4088ebdSKirill A. Shutemov spin_unlock(ptl); 127993b4796dSKirill A. Shutemov alloc: 128071e3aac0SAndrea Arcangeli if (transparent_hugepage_enabled(vma) && 1281077fcf11SAneesh Kumar K.V !transparent_hugepage_debug_cow()) { 12823b363692SMichal Hocko huge_gfp = alloc_hugepage_gfpmask(transparent_hugepage_defrag(vma), 0); 12833b363692SMichal Hocko new_page = alloc_hugepage_vma(huge_gfp, vma, haddr, HPAGE_PMD_ORDER); 1284077fcf11SAneesh Kumar K.V } else 128571e3aac0SAndrea Arcangeli new_page = NULL; 128671e3aac0SAndrea Arcangeli 12879a982250SKirill A. Shutemov if (likely(new_page)) { 12889a982250SKirill A. Shutemov prep_transhuge_page(new_page); 12899a982250SKirill A. Shutemov } else { 1290eecc1e42SHugh Dickins if (!page) { 129178ddc534SKirill A. Shutemov split_huge_pmd(vma, pmd, address); 1292e9b71ca9SKirill A. Shutemov ret |= VM_FAULT_FALLBACK; 129393b4796dSKirill A. Shutemov } else { 129471e3aac0SAndrea Arcangeli ret = do_huge_pmd_wp_page_fallback(mm, vma, address, 129571e3aac0SAndrea Arcangeli pmd, orig_pmd, page, haddr); 12969845cbbdSKirill A. Shutemov if (ret & VM_FAULT_OOM) { 129778ddc534SKirill A. Shutemov split_huge_pmd(vma, pmd, address); 12989845cbbdSKirill A. Shutemov ret |= VM_FAULT_FALLBACK; 12999845cbbdSKirill A. Shutemov } 1300ddc58f27SKirill A. Shutemov put_page(page); 130193b4796dSKirill A. Shutemov } 130217766ddeSDavid Rientjes count_vm_event(THP_FAULT_FALLBACK); 130371e3aac0SAndrea Arcangeli goto out; 130471e3aac0SAndrea Arcangeli } 130571e3aac0SAndrea Arcangeli 1306f627c2f5SKirill A. Shutemov if (unlikely(mem_cgroup_try_charge(new_page, mm, huge_gfp, &memcg, 1307f627c2f5SKirill A. Shutemov true))) { 1308b9bbfbe3SAndrea Arcangeli put_page(new_page); 130993b4796dSKirill A. Shutemov if (page) { 131078ddc534SKirill A. Shutemov split_huge_pmd(vma, pmd, address); 1311ddc58f27SKirill A. Shutemov put_page(page); 13129845cbbdSKirill A. Shutemov } else 131378ddc534SKirill A. Shutemov split_huge_pmd(vma, pmd, address); 13149845cbbdSKirill A. Shutemov ret |= VM_FAULT_FALLBACK; 131517766ddeSDavid Rientjes count_vm_event(THP_FAULT_FALLBACK); 1316b9bbfbe3SAndrea Arcangeli goto out; 1317b9bbfbe3SAndrea Arcangeli } 1318b9bbfbe3SAndrea Arcangeli 131917766ddeSDavid Rientjes count_vm_event(THP_FAULT_ALLOC); 132017766ddeSDavid Rientjes 1321eecc1e42SHugh Dickins if (!page) 132293b4796dSKirill A. Shutemov clear_huge_page(new_page, haddr, HPAGE_PMD_NR); 132393b4796dSKirill A. Shutemov else 132471e3aac0SAndrea Arcangeli copy_user_huge_page(new_page, page, haddr, vma, HPAGE_PMD_NR); 132571e3aac0SAndrea Arcangeli __SetPageUptodate(new_page); 132671e3aac0SAndrea Arcangeli 13272ec74c3eSSagi Grimberg mmun_start = haddr; 13282ec74c3eSSagi Grimberg mmun_end = haddr + HPAGE_PMD_SIZE; 13292ec74c3eSSagi Grimberg mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 13302ec74c3eSSagi Grimberg 1331c4088ebdSKirill A. Shutemov spin_lock(ptl); 133293b4796dSKirill A. Shutemov if (page) 1333ddc58f27SKirill A. Shutemov put_page(page); 1334b9bbfbe3SAndrea Arcangeli if (unlikely(!pmd_same(*pmd, orig_pmd))) { 1335c4088ebdSKirill A. Shutemov spin_unlock(ptl); 1336f627c2f5SKirill A. Shutemov mem_cgroup_cancel_charge(new_page, memcg, true); 133771e3aac0SAndrea Arcangeli put_page(new_page); 13382ec74c3eSSagi Grimberg goto out_mn; 1339b9bbfbe3SAndrea Arcangeli } else { 134071e3aac0SAndrea Arcangeli pmd_t entry; 13413122359aSKirill A. Shutemov entry = mk_huge_pmd(new_page, vma->vm_page_prot); 13423122359aSKirill A. Shutemov entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 13438809aa2dSAneesh Kumar K.V pmdp_huge_clear_flush_notify(vma, haddr, pmd); 1344d281ee61SKirill A. Shutemov page_add_new_anon_rmap(new_page, vma, haddr, true); 1345f627c2f5SKirill A. Shutemov mem_cgroup_commit_charge(new_page, memcg, false, true); 134600501b53SJohannes Weiner lru_cache_add_active_or_unevictable(new_page, vma); 134771e3aac0SAndrea Arcangeli set_pmd_at(mm, haddr, pmd, entry); 1348b113da65SDavid Miller update_mmu_cache_pmd(vma, address, pmd); 1349eecc1e42SHugh Dickins if (!page) { 135093b4796dSKirill A. Shutemov add_mm_counter(mm, MM_ANONPAGES, HPAGE_PMD_NR); 135197ae1749SKirill A. Shutemov put_huge_zero_page(); 135297ae1749SKirill A. Shutemov } else { 1353309381feSSasha Levin VM_BUG_ON_PAGE(!PageHead(page), page); 1354d281ee61SKirill A. Shutemov page_remove_rmap(page, true); 135571e3aac0SAndrea Arcangeli put_page(page); 135693b4796dSKirill A. Shutemov } 135771e3aac0SAndrea Arcangeli ret |= VM_FAULT_WRITE; 135871e3aac0SAndrea Arcangeli } 1359c4088ebdSKirill A. Shutemov spin_unlock(ptl); 13602ec74c3eSSagi Grimberg out_mn: 13612ec74c3eSSagi Grimberg mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 13622ec74c3eSSagi Grimberg out: 13632ec74c3eSSagi Grimberg return ret; 136471e3aac0SAndrea Arcangeli out_unlock: 1365c4088ebdSKirill A. Shutemov spin_unlock(ptl); 136671e3aac0SAndrea Arcangeli return ret; 136771e3aac0SAndrea Arcangeli } 136871e3aac0SAndrea Arcangeli 1369b676b293SDavid Rientjes struct page *follow_trans_huge_pmd(struct vm_area_struct *vma, 137071e3aac0SAndrea Arcangeli unsigned long addr, 137171e3aac0SAndrea Arcangeli pmd_t *pmd, 137271e3aac0SAndrea Arcangeli unsigned int flags) 137371e3aac0SAndrea Arcangeli { 1374b676b293SDavid Rientjes struct mm_struct *mm = vma->vm_mm; 137571e3aac0SAndrea Arcangeli struct page *page = NULL; 137671e3aac0SAndrea Arcangeli 1377c4088ebdSKirill A. Shutemov assert_spin_locked(pmd_lockptr(mm, pmd)); 137871e3aac0SAndrea Arcangeli 137971e3aac0SAndrea Arcangeli if (flags & FOLL_WRITE && !pmd_write(*pmd)) 138071e3aac0SAndrea Arcangeli goto out; 138171e3aac0SAndrea Arcangeli 138285facf25SKirill A. Shutemov /* Avoid dumping huge zero page */ 138385facf25SKirill A. Shutemov if ((flags & FOLL_DUMP) && is_huge_zero_pmd(*pmd)) 138485facf25SKirill A. Shutemov return ERR_PTR(-EFAULT); 138585facf25SKirill A. Shutemov 13862b4847e7SMel Gorman /* Full NUMA hinting faults to serialise migration in fault paths */ 13878a0516edSMel Gorman if ((flags & FOLL_NUMA) && pmd_protnone(*pmd)) 13882b4847e7SMel Gorman goto out; 13892b4847e7SMel Gorman 139071e3aac0SAndrea Arcangeli page = pmd_page(*pmd); 1391309381feSSasha Levin VM_BUG_ON_PAGE(!PageHead(page), page); 13923565fce3SDan Williams if (flags & FOLL_TOUCH) 13933565fce3SDan Williams touch_pmd(vma, addr, pmd); 1394de60f5f1SEric B Munson if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) { 1395e90309c9SKirill A. Shutemov /* 1396e90309c9SKirill A. Shutemov * We don't mlock() pte-mapped THPs. This way we can avoid 1397e90309c9SKirill A. Shutemov * leaking mlocked pages into non-VM_LOCKED VMAs. 1398e90309c9SKirill A. Shutemov * 1399e90309c9SKirill A. Shutemov * In most cases the pmd is the only mapping of the page as we 1400e90309c9SKirill A. Shutemov * break COW for the mlock() -- see gup_flags |= FOLL_WRITE for 1401e90309c9SKirill A. Shutemov * writable private mappings in populate_vma_page_range(). 1402e90309c9SKirill A. Shutemov * 1403e90309c9SKirill A. Shutemov * The only scenario when we have the page shared here is if we 1404e90309c9SKirill A. Shutemov * mlocking read-only mapping shared over fork(). We skip 1405e90309c9SKirill A. Shutemov * mlocking such pages. 1406e90309c9SKirill A. Shutemov */ 1407e90309c9SKirill A. Shutemov if (compound_mapcount(page) == 1 && !PageDoubleMap(page) && 1408e90309c9SKirill A. Shutemov page->mapping && trylock_page(page)) { 1409b676b293SDavid Rientjes lru_add_drain(); 1410b676b293SDavid Rientjes if (page->mapping) 1411b676b293SDavid Rientjes mlock_vma_page(page); 1412b676b293SDavid Rientjes unlock_page(page); 1413b676b293SDavid Rientjes } 1414b676b293SDavid Rientjes } 141571e3aac0SAndrea Arcangeli page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT; 1416309381feSSasha Levin VM_BUG_ON_PAGE(!PageCompound(page), page); 141771e3aac0SAndrea Arcangeli if (flags & FOLL_GET) 1418ddc58f27SKirill A. Shutemov get_page(page); 141971e3aac0SAndrea Arcangeli 142071e3aac0SAndrea Arcangeli out: 142171e3aac0SAndrea Arcangeli return page; 142271e3aac0SAndrea Arcangeli } 142371e3aac0SAndrea Arcangeli 1424d10e63f2SMel Gorman /* NUMA hinting page fault entry point for trans huge pmds */ 14254daae3b4SMel Gorman int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma, 14264daae3b4SMel Gorman unsigned long addr, pmd_t pmd, pmd_t *pmdp) 1427d10e63f2SMel Gorman { 1428c4088ebdSKirill A. Shutemov spinlock_t *ptl; 1429b8916634SMel Gorman struct anon_vma *anon_vma = NULL; 1430b32967ffSMel Gorman struct page *page; 1431d10e63f2SMel Gorman unsigned long haddr = addr & HPAGE_PMD_MASK; 14328191acbdSMel Gorman int page_nid = -1, this_nid = numa_node_id(); 143390572890SPeter Zijlstra int target_nid, last_cpupid = -1; 14348191acbdSMel Gorman bool page_locked; 14358191acbdSMel Gorman bool migrated = false; 1436b191f9b1SMel Gorman bool was_writable; 14376688cc05SPeter Zijlstra int flags = 0; 1438d10e63f2SMel Gorman 1439c0e7cad9SMel Gorman /* A PROT_NONE fault should not end up here */ 1440c0e7cad9SMel Gorman BUG_ON(!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))); 1441c0e7cad9SMel Gorman 1442c4088ebdSKirill A. Shutemov ptl = pmd_lock(mm, pmdp); 1443d10e63f2SMel Gorman if (unlikely(!pmd_same(pmd, *pmdp))) 1444d10e63f2SMel Gorman goto out_unlock; 1445d10e63f2SMel Gorman 1446de466bd6SMel Gorman /* 1447de466bd6SMel Gorman * If there are potential migrations, wait for completion and retry 1448de466bd6SMel Gorman * without disrupting NUMA hinting information. Do not relock and 1449de466bd6SMel Gorman * check_same as the page may no longer be mapped. 1450de466bd6SMel Gorman */ 1451de466bd6SMel Gorman if (unlikely(pmd_trans_migrating(*pmdp))) { 14525d833062SMel Gorman page = pmd_page(*pmdp); 1453de466bd6SMel Gorman spin_unlock(ptl); 14545d833062SMel Gorman wait_on_page_locked(page); 1455de466bd6SMel Gorman goto out; 1456de466bd6SMel Gorman } 1457de466bd6SMel Gorman 1458d10e63f2SMel Gorman page = pmd_page(pmd); 1459a1a46184SMel Gorman BUG_ON(is_huge_zero_page(page)); 14608191acbdSMel Gorman page_nid = page_to_nid(page); 146190572890SPeter Zijlstra last_cpupid = page_cpupid_last(page); 146203c5a6e1SMel Gorman count_vm_numa_event(NUMA_HINT_FAULTS); 146304bb2f94SRik van Riel if (page_nid == this_nid) { 146403c5a6e1SMel Gorman count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL); 146504bb2f94SRik van Riel flags |= TNF_FAULT_LOCAL; 146604bb2f94SRik van Riel } 14674daae3b4SMel Gorman 1468bea66fbdSMel Gorman /* See similar comment in do_numa_page for explanation */ 1469bea66fbdSMel Gorman if (!(vma->vm_flags & VM_WRITE)) 14706688cc05SPeter Zijlstra flags |= TNF_NO_GROUP; 14716688cc05SPeter Zijlstra 14726688cc05SPeter Zijlstra /* 1473ff9042b1SMel Gorman * Acquire the page lock to serialise THP migrations but avoid dropping 1474ff9042b1SMel Gorman * page_table_lock if at all possible 1475ff9042b1SMel Gorman */ 1476b8916634SMel Gorman page_locked = trylock_page(page); 1477b8916634SMel Gorman target_nid = mpol_misplaced(page, vma, haddr); 1478b8916634SMel Gorman if (target_nid == -1) { 1479b8916634SMel Gorman /* If the page was locked, there are no parallel migrations */ 1480a54a407fSMel Gorman if (page_locked) 1481b8916634SMel Gorman goto clear_pmdnuma; 14822b4847e7SMel Gorman } 1483cbee9f88SPeter Zijlstra 1484de466bd6SMel Gorman /* Migration could have started since the pmd_trans_migrating check */ 14852b4847e7SMel Gorman if (!page_locked) { 1486c4088ebdSKirill A. Shutemov spin_unlock(ptl); 1487b8916634SMel Gorman wait_on_page_locked(page); 1488a54a407fSMel Gorman page_nid = -1; 1489b8916634SMel Gorman goto out; 1490b8916634SMel Gorman } 1491b8916634SMel Gorman 14922b4847e7SMel Gorman /* 14932b4847e7SMel Gorman * Page is misplaced. Page lock serialises migrations. Acquire anon_vma 14942b4847e7SMel Gorman * to serialises splits 14952b4847e7SMel Gorman */ 1496b8916634SMel Gorman get_page(page); 1497c4088ebdSKirill A. Shutemov spin_unlock(ptl); 1498b8916634SMel Gorman anon_vma = page_lock_anon_vma_read(page); 1499b32967ffSMel Gorman 1500c69307d5SPeter Zijlstra /* Confirm the PMD did not change while page_table_lock was released */ 1501c4088ebdSKirill A. Shutemov spin_lock(ptl); 1502b32967ffSMel Gorman if (unlikely(!pmd_same(pmd, *pmdp))) { 1503b32967ffSMel Gorman unlock_page(page); 1504b32967ffSMel Gorman put_page(page); 1505a54a407fSMel Gorman page_nid = -1; 1506b32967ffSMel Gorman goto out_unlock; 1507b32967ffSMel Gorman } 1508ff9042b1SMel Gorman 1509c3a489caSMel Gorman /* Bail if we fail to protect against THP splits for any reason */ 1510c3a489caSMel Gorman if (unlikely(!anon_vma)) { 1511c3a489caSMel Gorman put_page(page); 1512c3a489caSMel Gorman page_nid = -1; 1513c3a489caSMel Gorman goto clear_pmdnuma; 1514c3a489caSMel Gorman } 1515c3a489caSMel Gorman 1516a54a407fSMel Gorman /* 1517a54a407fSMel Gorman * Migrate the THP to the requested node, returns with page unlocked 15188a0516edSMel Gorman * and access rights restored. 1519a54a407fSMel Gorman */ 1520c4088ebdSKirill A. Shutemov spin_unlock(ptl); 1521b32967ffSMel Gorman migrated = migrate_misplaced_transhuge_page(mm, vma, 1522340ef390SHugh Dickins pmdp, pmd, addr, page, target_nid); 15236688cc05SPeter Zijlstra if (migrated) { 15246688cc05SPeter Zijlstra flags |= TNF_MIGRATED; 15258191acbdSMel Gorman page_nid = target_nid; 1526074c2381SMel Gorman } else 1527074c2381SMel Gorman flags |= TNF_MIGRATE_FAIL; 1528b32967ffSMel Gorman 15298191acbdSMel Gorman goto out; 15304daae3b4SMel Gorman clear_pmdnuma: 1531a54a407fSMel Gorman BUG_ON(!PageLocked(page)); 1532b191f9b1SMel Gorman was_writable = pmd_write(pmd); 15334d942466SMel Gorman pmd = pmd_modify(pmd, vma->vm_page_prot); 1534b7b04004SMel Gorman pmd = pmd_mkyoung(pmd); 1535b191f9b1SMel Gorman if (was_writable) 1536b191f9b1SMel Gorman pmd = pmd_mkwrite(pmd); 1537d10e63f2SMel Gorman set_pmd_at(mm, haddr, pmdp, pmd); 1538d10e63f2SMel Gorman update_mmu_cache_pmd(vma, addr, pmdp); 1539a54a407fSMel Gorman unlock_page(page); 1540d10e63f2SMel Gorman out_unlock: 1541c4088ebdSKirill A. Shutemov spin_unlock(ptl); 1542b8916634SMel Gorman 1543b8916634SMel Gorman out: 1544b8916634SMel Gorman if (anon_vma) 1545b8916634SMel Gorman page_unlock_anon_vma_read(anon_vma); 1546b8916634SMel Gorman 15478191acbdSMel Gorman if (page_nid != -1) 15486688cc05SPeter Zijlstra task_numa_fault(last_cpupid, page_nid, HPAGE_PMD_NR, flags); 15498191acbdSMel Gorman 1550d10e63f2SMel Gorman return 0; 1551d10e63f2SMel Gorman } 1552d10e63f2SMel Gorman 1553b8d3c4c3SMinchan Kim int madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 1554b8d3c4c3SMinchan Kim pmd_t *pmd, unsigned long addr, unsigned long next) 1555b8d3c4c3SMinchan Kim 1556b8d3c4c3SMinchan Kim { 1557b8d3c4c3SMinchan Kim spinlock_t *ptl; 1558b8d3c4c3SMinchan Kim pmd_t orig_pmd; 1559b8d3c4c3SMinchan Kim struct page *page; 1560b8d3c4c3SMinchan Kim struct mm_struct *mm = tlb->mm; 1561b8d3c4c3SMinchan Kim int ret = 0; 1562b8d3c4c3SMinchan Kim 1563*b6ec57f4SKirill A. Shutemov ptl = pmd_trans_huge_lock(pmd, vma); 1564*b6ec57f4SKirill A. Shutemov if (!ptl) 156525eedabeSLinus Torvalds goto out_unlocked; 1566b8d3c4c3SMinchan Kim 1567b8d3c4c3SMinchan Kim orig_pmd = *pmd; 1568b8d3c4c3SMinchan Kim if (is_huge_zero_pmd(orig_pmd)) { 1569b8d3c4c3SMinchan Kim ret = 1; 1570b8d3c4c3SMinchan Kim goto out; 1571b8d3c4c3SMinchan Kim } 1572b8d3c4c3SMinchan Kim 1573b8d3c4c3SMinchan Kim page = pmd_page(orig_pmd); 1574b8d3c4c3SMinchan Kim /* 1575b8d3c4c3SMinchan Kim * If other processes are mapping this page, we couldn't discard 1576b8d3c4c3SMinchan Kim * the page unless they all do MADV_FREE so let's skip the page. 1577b8d3c4c3SMinchan Kim */ 1578b8d3c4c3SMinchan Kim if (page_mapcount(page) != 1) 1579b8d3c4c3SMinchan Kim goto out; 1580b8d3c4c3SMinchan Kim 1581b8d3c4c3SMinchan Kim if (!trylock_page(page)) 1582b8d3c4c3SMinchan Kim goto out; 1583b8d3c4c3SMinchan Kim 1584b8d3c4c3SMinchan Kim /* 1585b8d3c4c3SMinchan Kim * If user want to discard part-pages of THP, split it so MADV_FREE 1586b8d3c4c3SMinchan Kim * will deactivate only them. 1587b8d3c4c3SMinchan Kim */ 1588b8d3c4c3SMinchan Kim if (next - addr != HPAGE_PMD_SIZE) { 1589b8d3c4c3SMinchan Kim get_page(page); 1590b8d3c4c3SMinchan Kim spin_unlock(ptl); 1591b8d3c4c3SMinchan Kim if (split_huge_page(page)) { 1592b8d3c4c3SMinchan Kim put_page(page); 1593b8d3c4c3SMinchan Kim unlock_page(page); 1594b8d3c4c3SMinchan Kim goto out_unlocked; 1595b8d3c4c3SMinchan Kim } 1596b8d3c4c3SMinchan Kim put_page(page); 1597b8d3c4c3SMinchan Kim unlock_page(page); 1598b8d3c4c3SMinchan Kim ret = 1; 1599b8d3c4c3SMinchan Kim goto out_unlocked; 1600b8d3c4c3SMinchan Kim } 1601b8d3c4c3SMinchan Kim 1602b8d3c4c3SMinchan Kim if (PageDirty(page)) 1603b8d3c4c3SMinchan Kim ClearPageDirty(page); 1604b8d3c4c3SMinchan Kim unlock_page(page); 1605b8d3c4c3SMinchan Kim 1606b8d3c4c3SMinchan Kim if (PageActive(page)) 1607b8d3c4c3SMinchan Kim deactivate_page(page); 1608b8d3c4c3SMinchan Kim 1609b8d3c4c3SMinchan Kim if (pmd_young(orig_pmd) || pmd_dirty(orig_pmd)) { 1610b8d3c4c3SMinchan Kim orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd, 1611b8d3c4c3SMinchan Kim tlb->fullmm); 1612b8d3c4c3SMinchan Kim orig_pmd = pmd_mkold(orig_pmd); 1613b8d3c4c3SMinchan Kim orig_pmd = pmd_mkclean(orig_pmd); 1614b8d3c4c3SMinchan Kim 1615b8d3c4c3SMinchan Kim set_pmd_at(mm, addr, pmd, orig_pmd); 1616b8d3c4c3SMinchan Kim tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 1617b8d3c4c3SMinchan Kim } 1618b8d3c4c3SMinchan Kim ret = 1; 1619b8d3c4c3SMinchan Kim out: 1620b8d3c4c3SMinchan Kim spin_unlock(ptl); 1621b8d3c4c3SMinchan Kim out_unlocked: 1622b8d3c4c3SMinchan Kim return ret; 1623b8d3c4c3SMinchan Kim } 1624b8d3c4c3SMinchan Kim 162571e3aac0SAndrea Arcangeli int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 1626f21760b1SShaohua Li pmd_t *pmd, unsigned long addr) 162771e3aac0SAndrea Arcangeli { 1628f5c8ad47SDavid Miller pmd_t orig_pmd; 1629da146769SKirill A. Shutemov spinlock_t *ptl; 1630da146769SKirill A. Shutemov 1631*b6ec57f4SKirill A. Shutemov ptl = __pmd_trans_huge_lock(pmd, vma); 1632*b6ec57f4SKirill A. Shutemov if (!ptl) 1633da146769SKirill A. Shutemov return 0; 1634a6bf2bb0SAneesh Kumar K.V /* 1635a6bf2bb0SAneesh Kumar K.V * For architectures like ppc64 we look at deposited pgtable 16368809aa2dSAneesh Kumar K.V * when calling pmdp_huge_get_and_clear. So do the 1637a6bf2bb0SAneesh Kumar K.V * pgtable_trans_huge_withdraw after finishing pmdp related 1638a6bf2bb0SAneesh Kumar K.V * operations. 1639a6bf2bb0SAneesh Kumar K.V */ 16408809aa2dSAneesh Kumar K.V orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd, 1641fcbe08d6SMartin Schwidefsky tlb->fullmm); 1642f21760b1SShaohua Li tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 16434897c765SMatthew Wilcox if (vma_is_dax(vma)) { 16444897c765SMatthew Wilcox spin_unlock(ptl); 1645da146769SKirill A. Shutemov if (is_huge_zero_pmd(orig_pmd)) 1646da146769SKirill A. Shutemov put_huge_zero_page(); 1647da146769SKirill A. Shutemov } else if (is_huge_zero_pmd(orig_pmd)) { 1648da146769SKirill A. Shutemov pte_free(tlb->mm, pgtable_trans_huge_withdraw(tlb->mm, pmd)); 1649e1f56c89SKirill A. Shutemov atomic_long_dec(&tlb->mm->nr_ptes); 1650bf929152SKirill A. Shutemov spin_unlock(ptl); 165197ae1749SKirill A. Shutemov put_huge_zero_page(); 1652479f0abbSKirill A. Shutemov } else { 16534897c765SMatthew Wilcox struct page *page = pmd_page(orig_pmd); 1654d281ee61SKirill A. Shutemov page_remove_rmap(page, true); 1655309381feSSasha Levin VM_BUG_ON_PAGE(page_mapcount(page) < 0, page); 165671e3aac0SAndrea Arcangeli add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR); 1657309381feSSasha Levin VM_BUG_ON_PAGE(!PageHead(page), page); 1658da146769SKirill A. Shutemov pte_free(tlb->mm, pgtable_trans_huge_withdraw(tlb->mm, pmd)); 1659e1f56c89SKirill A. Shutemov atomic_long_dec(&tlb->mm->nr_ptes); 1660bf929152SKirill A. Shutemov spin_unlock(ptl); 166171e3aac0SAndrea Arcangeli tlb_remove_page(tlb, page); 1662479f0abbSKirill A. Shutemov } 1663da146769SKirill A. Shutemov return 1; 166471e3aac0SAndrea Arcangeli } 166571e3aac0SAndrea Arcangeli 16664b471e88SKirill A. Shutemov bool move_huge_pmd(struct vm_area_struct *vma, struct vm_area_struct *new_vma, 166737a1c49aSAndrea Arcangeli unsigned long old_addr, 166837a1c49aSAndrea Arcangeli unsigned long new_addr, unsigned long old_end, 166937a1c49aSAndrea Arcangeli pmd_t *old_pmd, pmd_t *new_pmd) 167037a1c49aSAndrea Arcangeli { 1671bf929152SKirill A. Shutemov spinlock_t *old_ptl, *new_ptl; 167237a1c49aSAndrea Arcangeli pmd_t pmd; 167337a1c49aSAndrea Arcangeli 167437a1c49aSAndrea Arcangeli struct mm_struct *mm = vma->vm_mm; 167537a1c49aSAndrea Arcangeli 167637a1c49aSAndrea Arcangeli if ((old_addr & ~HPAGE_PMD_MASK) || 167737a1c49aSAndrea Arcangeli (new_addr & ~HPAGE_PMD_MASK) || 167837a1c49aSAndrea Arcangeli old_end - old_addr < HPAGE_PMD_SIZE || 167937a1c49aSAndrea Arcangeli (new_vma->vm_flags & VM_NOHUGEPAGE)) 16804b471e88SKirill A. Shutemov return false; 168137a1c49aSAndrea Arcangeli 168237a1c49aSAndrea Arcangeli /* 168337a1c49aSAndrea Arcangeli * The destination pmd shouldn't be established, free_pgtables() 168437a1c49aSAndrea Arcangeli * should have release it. 168537a1c49aSAndrea Arcangeli */ 168637a1c49aSAndrea Arcangeli if (WARN_ON(!pmd_none(*new_pmd))) { 168737a1c49aSAndrea Arcangeli VM_BUG_ON(pmd_trans_huge(*new_pmd)); 16884b471e88SKirill A. Shutemov return false; 168937a1c49aSAndrea Arcangeli } 169037a1c49aSAndrea Arcangeli 1691bf929152SKirill A. Shutemov /* 1692bf929152SKirill A. Shutemov * We don't have to worry about the ordering of src and dst 1693bf929152SKirill A. Shutemov * ptlocks because exclusive mmap_sem prevents deadlock. 1694bf929152SKirill A. Shutemov */ 1695*b6ec57f4SKirill A. Shutemov old_ptl = __pmd_trans_huge_lock(old_pmd, vma); 1696*b6ec57f4SKirill A. Shutemov if (old_ptl) { 1697bf929152SKirill A. Shutemov new_ptl = pmd_lockptr(mm, new_pmd); 1698bf929152SKirill A. Shutemov if (new_ptl != old_ptl) 1699bf929152SKirill A. Shutemov spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING); 17008809aa2dSAneesh Kumar K.V pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd); 170137a1c49aSAndrea Arcangeli VM_BUG_ON(!pmd_none(*new_pmd)); 17023592806cSKirill A. Shutemov 1703b3084f4dSAneesh Kumar K.V if (pmd_move_must_withdraw(new_ptl, old_ptl)) { 1704b3084f4dSAneesh Kumar K.V pgtable_t pgtable; 17053592806cSKirill A. Shutemov pgtable = pgtable_trans_huge_withdraw(mm, old_pmd); 17063592806cSKirill A. Shutemov pgtable_trans_huge_deposit(mm, new_pmd, pgtable); 17073592806cSKirill A. Shutemov } 1708b3084f4dSAneesh Kumar K.V set_pmd_at(mm, new_addr, new_pmd, pmd_mksoft_dirty(pmd)); 1709b3084f4dSAneesh Kumar K.V if (new_ptl != old_ptl) 1710b3084f4dSAneesh Kumar K.V spin_unlock(new_ptl); 1711bf929152SKirill A. Shutemov spin_unlock(old_ptl); 17124b471e88SKirill A. Shutemov return true; 171337a1c49aSAndrea Arcangeli } 17144b471e88SKirill A. Shutemov return false; 171537a1c49aSAndrea Arcangeli } 171637a1c49aSAndrea Arcangeli 1717f123d74aSMel Gorman /* 1718f123d74aSMel Gorman * Returns 1719f123d74aSMel Gorman * - 0 if PMD could not be locked 1720f123d74aSMel Gorman * - 1 if PMD was locked but protections unchange and TLB flush unnecessary 1721f123d74aSMel Gorman * - HPAGE_PMD_NR is protections changed and TLB flush necessary 1722f123d74aSMel Gorman */ 1723cd7548abSJohannes Weiner int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, 1724e944fd67SMel Gorman unsigned long addr, pgprot_t newprot, int prot_numa) 1725cd7548abSJohannes Weiner { 1726cd7548abSJohannes Weiner struct mm_struct *mm = vma->vm_mm; 1727bf929152SKirill A. Shutemov spinlock_t *ptl; 1728cd7548abSJohannes Weiner int ret = 0; 1729cd7548abSJohannes Weiner 1730*b6ec57f4SKirill A. Shutemov ptl = __pmd_trans_huge_lock(pmd, vma); 1731*b6ec57f4SKirill A. Shutemov if (ptl) { 1732cd7548abSJohannes Weiner pmd_t entry; 1733b191f9b1SMel Gorman bool preserve_write = prot_numa && pmd_write(*pmd); 1734ba68bc01SMel Gorman ret = 1; 1735e944fd67SMel Gorman 1736e944fd67SMel Gorman /* 1737e944fd67SMel Gorman * Avoid trapping faults against the zero page. The read-only 1738e944fd67SMel Gorman * data is likely to be read-cached on the local CPU and 1739e944fd67SMel Gorman * local/remote hits to the zero page are not interesting. 1740e944fd67SMel Gorman */ 1741e944fd67SMel Gorman if (prot_numa && is_huge_zero_pmd(*pmd)) { 1742e944fd67SMel Gorman spin_unlock(ptl); 1743ba68bc01SMel Gorman return ret; 1744e944fd67SMel Gorman } 1745e944fd67SMel Gorman 174610c1045fSMel Gorman if (!prot_numa || !pmd_protnone(*pmd)) { 17478809aa2dSAneesh Kumar K.V entry = pmdp_huge_get_and_clear_notify(mm, addr, pmd); 1748cd7548abSJohannes Weiner entry = pmd_modify(entry, newprot); 1749b191f9b1SMel Gorman if (preserve_write) 1750b191f9b1SMel Gorman entry = pmd_mkwrite(entry); 1751f123d74aSMel Gorman ret = HPAGE_PMD_NR; 175256eecdb9SAneesh Kumar K.V set_pmd_at(mm, addr, pmd, entry); 1753b191f9b1SMel Gorman BUG_ON(!preserve_write && pmd_write(entry)); 175410c1045fSMel Gorman } 1755bf929152SKirill A. Shutemov spin_unlock(ptl); 1756cd7548abSJohannes Weiner } 1757cd7548abSJohannes Weiner 1758cd7548abSJohannes Weiner return ret; 1759cd7548abSJohannes Weiner } 1760cd7548abSJohannes Weiner 1761025c5b24SNaoya Horiguchi /* 17624b471e88SKirill A. Shutemov * Returns true if a given pmd maps a thp, false otherwise. 1763025c5b24SNaoya Horiguchi * 17644b471e88SKirill A. Shutemov * Note that if it returns true, this routine returns without unlocking page 17654b471e88SKirill A. Shutemov * table lock. So callers must unlock it. 1766025c5b24SNaoya Horiguchi */ 1767*b6ec57f4SKirill A. Shutemov spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma) 1768025c5b24SNaoya Horiguchi { 1769*b6ec57f4SKirill A. Shutemov spinlock_t *ptl; 1770*b6ec57f4SKirill A. Shutemov ptl = pmd_lock(vma->vm_mm, pmd); 17715c7fb56eSDan Williams if (likely(pmd_trans_huge(*pmd) || pmd_devmap(*pmd))) 1772*b6ec57f4SKirill A. Shutemov return ptl; 1773*b6ec57f4SKirill A. Shutemov spin_unlock(ptl); 1774*b6ec57f4SKirill A. Shutemov return NULL; 1775025c5b24SNaoya Horiguchi } 1776025c5b24SNaoya Horiguchi 17779050d7ebSVlastimil Babka #define VM_NO_THP (VM_SPECIAL | VM_HUGETLB | VM_SHARED | VM_MAYSHARE) 177878f11a25SAndrea Arcangeli 177960ab3244SAndrea Arcangeli int hugepage_madvise(struct vm_area_struct *vma, 178060ab3244SAndrea Arcangeli unsigned long *vm_flags, int advice) 17810af4e98bSAndrea Arcangeli { 1782a664b2d8SAndrea Arcangeli switch (advice) { 1783a664b2d8SAndrea Arcangeli case MADV_HUGEPAGE: 17841e1836e8SAlex Thorlton #ifdef CONFIG_S390 17851e1836e8SAlex Thorlton /* 17861e1836e8SAlex Thorlton * qemu blindly sets MADV_HUGEPAGE on all allocations, but s390 17871e1836e8SAlex Thorlton * can't handle this properly after s390_enable_sie, so we simply 17881e1836e8SAlex Thorlton * ignore the madvise to prevent qemu from causing a SIGSEGV. 17891e1836e8SAlex Thorlton */ 17901e1836e8SAlex Thorlton if (mm_has_pgste(vma->vm_mm)) 17911e1836e8SAlex Thorlton return 0; 17921e1836e8SAlex Thorlton #endif 17930af4e98bSAndrea Arcangeli /* 17940af4e98bSAndrea Arcangeli * Be somewhat over-protective like KSM for now! 17950af4e98bSAndrea Arcangeli */ 17961a763615SJason J. Herne if (*vm_flags & VM_NO_THP) 17970af4e98bSAndrea Arcangeli return -EINVAL; 1798a664b2d8SAndrea Arcangeli *vm_flags &= ~VM_NOHUGEPAGE; 17990af4e98bSAndrea Arcangeli *vm_flags |= VM_HUGEPAGE; 180060ab3244SAndrea Arcangeli /* 180160ab3244SAndrea Arcangeli * If the vma become good for khugepaged to scan, 180260ab3244SAndrea Arcangeli * register it here without waiting a page fault that 180360ab3244SAndrea Arcangeli * may not happen any time soon. 180460ab3244SAndrea Arcangeli */ 18056d50e60cSDavid Rientjes if (unlikely(khugepaged_enter_vma_merge(vma, *vm_flags))) 180660ab3244SAndrea Arcangeli return -ENOMEM; 1807a664b2d8SAndrea Arcangeli break; 1808a664b2d8SAndrea Arcangeli case MADV_NOHUGEPAGE: 1809a664b2d8SAndrea Arcangeli /* 1810a664b2d8SAndrea Arcangeli * Be somewhat over-protective like KSM for now! 1811a664b2d8SAndrea Arcangeli */ 18121a763615SJason J. Herne if (*vm_flags & VM_NO_THP) 1813a664b2d8SAndrea Arcangeli return -EINVAL; 1814a664b2d8SAndrea Arcangeli *vm_flags &= ~VM_HUGEPAGE; 1815a664b2d8SAndrea Arcangeli *vm_flags |= VM_NOHUGEPAGE; 181660ab3244SAndrea Arcangeli /* 181760ab3244SAndrea Arcangeli * Setting VM_NOHUGEPAGE will prevent khugepaged from scanning 181860ab3244SAndrea Arcangeli * this vma even if we leave the mm registered in khugepaged if 181960ab3244SAndrea Arcangeli * it got registered before VM_NOHUGEPAGE was set. 182060ab3244SAndrea Arcangeli */ 1821a664b2d8SAndrea Arcangeli break; 1822a664b2d8SAndrea Arcangeli } 18230af4e98bSAndrea Arcangeli 18240af4e98bSAndrea Arcangeli return 0; 18250af4e98bSAndrea Arcangeli } 18260af4e98bSAndrea Arcangeli 1827ba76149fSAndrea Arcangeli static int __init khugepaged_slab_init(void) 1828ba76149fSAndrea Arcangeli { 1829ba76149fSAndrea Arcangeli mm_slot_cache = kmem_cache_create("khugepaged_mm_slot", 1830ba76149fSAndrea Arcangeli sizeof(struct mm_slot), 1831ba76149fSAndrea Arcangeli __alignof__(struct mm_slot), 0, NULL); 1832ba76149fSAndrea Arcangeli if (!mm_slot_cache) 1833ba76149fSAndrea Arcangeli return -ENOMEM; 1834ba76149fSAndrea Arcangeli 1835ba76149fSAndrea Arcangeli return 0; 1836ba76149fSAndrea Arcangeli } 1837ba76149fSAndrea Arcangeli 183865ebb64fSKirill A. Shutemov static void __init khugepaged_slab_exit(void) 183965ebb64fSKirill A. Shutemov { 184065ebb64fSKirill A. Shutemov kmem_cache_destroy(mm_slot_cache); 184165ebb64fSKirill A. Shutemov } 184265ebb64fSKirill A. Shutemov 1843ba76149fSAndrea Arcangeli static inline struct mm_slot *alloc_mm_slot(void) 1844ba76149fSAndrea Arcangeli { 1845ba76149fSAndrea Arcangeli if (!mm_slot_cache) /* initialization failed */ 1846ba76149fSAndrea Arcangeli return NULL; 1847ba76149fSAndrea Arcangeli return kmem_cache_zalloc(mm_slot_cache, GFP_KERNEL); 1848ba76149fSAndrea Arcangeli } 1849ba76149fSAndrea Arcangeli 1850ba76149fSAndrea Arcangeli static inline void free_mm_slot(struct mm_slot *mm_slot) 1851ba76149fSAndrea Arcangeli { 1852ba76149fSAndrea Arcangeli kmem_cache_free(mm_slot_cache, mm_slot); 1853ba76149fSAndrea Arcangeli } 1854ba76149fSAndrea Arcangeli 1855ba76149fSAndrea Arcangeli static struct mm_slot *get_mm_slot(struct mm_struct *mm) 1856ba76149fSAndrea Arcangeli { 1857ba76149fSAndrea Arcangeli struct mm_slot *mm_slot; 1858ba76149fSAndrea Arcangeli 1859b67bfe0dSSasha Levin hash_for_each_possible(mm_slots_hash, mm_slot, hash, (unsigned long)mm) 1860ba76149fSAndrea Arcangeli if (mm == mm_slot->mm) 1861ba76149fSAndrea Arcangeli return mm_slot; 186243b5fbbdSSasha Levin 1863ba76149fSAndrea Arcangeli return NULL; 1864ba76149fSAndrea Arcangeli } 1865ba76149fSAndrea Arcangeli 1866ba76149fSAndrea Arcangeli static void insert_to_mm_slots_hash(struct mm_struct *mm, 1867ba76149fSAndrea Arcangeli struct mm_slot *mm_slot) 1868ba76149fSAndrea Arcangeli { 1869ba76149fSAndrea Arcangeli mm_slot->mm = mm; 187043b5fbbdSSasha Levin hash_add(mm_slots_hash, &mm_slot->hash, (long)mm); 1871ba76149fSAndrea Arcangeli } 1872ba76149fSAndrea Arcangeli 1873ba76149fSAndrea Arcangeli static inline int khugepaged_test_exit(struct mm_struct *mm) 1874ba76149fSAndrea Arcangeli { 1875ba76149fSAndrea Arcangeli return atomic_read(&mm->mm_users) == 0; 1876ba76149fSAndrea Arcangeli } 1877ba76149fSAndrea Arcangeli 1878ba76149fSAndrea Arcangeli int __khugepaged_enter(struct mm_struct *mm) 1879ba76149fSAndrea Arcangeli { 1880ba76149fSAndrea Arcangeli struct mm_slot *mm_slot; 1881ba76149fSAndrea Arcangeli int wakeup; 1882ba76149fSAndrea Arcangeli 1883ba76149fSAndrea Arcangeli mm_slot = alloc_mm_slot(); 1884ba76149fSAndrea Arcangeli if (!mm_slot) 1885ba76149fSAndrea Arcangeli return -ENOMEM; 1886ba76149fSAndrea Arcangeli 1887ba76149fSAndrea Arcangeli /* __khugepaged_exit() must not run from under us */ 188896dad67fSSasha Levin VM_BUG_ON_MM(khugepaged_test_exit(mm), mm); 1889ba76149fSAndrea Arcangeli if (unlikely(test_and_set_bit(MMF_VM_HUGEPAGE, &mm->flags))) { 1890ba76149fSAndrea Arcangeli free_mm_slot(mm_slot); 1891ba76149fSAndrea Arcangeli return 0; 1892ba76149fSAndrea Arcangeli } 1893ba76149fSAndrea Arcangeli 1894ba76149fSAndrea Arcangeli spin_lock(&khugepaged_mm_lock); 1895ba76149fSAndrea Arcangeli insert_to_mm_slots_hash(mm, mm_slot); 1896ba76149fSAndrea Arcangeli /* 1897ba76149fSAndrea Arcangeli * Insert just behind the scanning cursor, to let the area settle 1898ba76149fSAndrea Arcangeli * down a little. 1899ba76149fSAndrea Arcangeli */ 1900ba76149fSAndrea Arcangeli wakeup = list_empty(&khugepaged_scan.mm_head); 1901ba76149fSAndrea Arcangeli list_add_tail(&mm_slot->mm_node, &khugepaged_scan.mm_head); 1902ba76149fSAndrea Arcangeli spin_unlock(&khugepaged_mm_lock); 1903ba76149fSAndrea Arcangeli 1904ba76149fSAndrea Arcangeli atomic_inc(&mm->mm_count); 1905ba76149fSAndrea Arcangeli if (wakeup) 1906ba76149fSAndrea Arcangeli wake_up_interruptible(&khugepaged_wait); 1907ba76149fSAndrea Arcangeli 1908ba76149fSAndrea Arcangeli return 0; 1909ba76149fSAndrea Arcangeli } 1910ba76149fSAndrea Arcangeli 19116d50e60cSDavid Rientjes int khugepaged_enter_vma_merge(struct vm_area_struct *vma, 19126d50e60cSDavid Rientjes unsigned long vm_flags) 1913ba76149fSAndrea Arcangeli { 1914ba76149fSAndrea Arcangeli unsigned long hstart, hend; 1915ba76149fSAndrea Arcangeli if (!vma->anon_vma) 1916ba76149fSAndrea Arcangeli /* 1917ba76149fSAndrea Arcangeli * Not yet faulted in so we will register later in the 1918ba76149fSAndrea Arcangeli * page fault if needed. 1919ba76149fSAndrea Arcangeli */ 1920ba76149fSAndrea Arcangeli return 0; 192178f11a25SAndrea Arcangeli if (vma->vm_ops) 1922ba76149fSAndrea Arcangeli /* khugepaged not yet working on file or special mappings */ 1923ba76149fSAndrea Arcangeli return 0; 19246d50e60cSDavid Rientjes VM_BUG_ON_VMA(vm_flags & VM_NO_THP, vma); 1925ba76149fSAndrea Arcangeli hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK; 1926ba76149fSAndrea Arcangeli hend = vma->vm_end & HPAGE_PMD_MASK; 1927ba76149fSAndrea Arcangeli if (hstart < hend) 19286d50e60cSDavid Rientjes return khugepaged_enter(vma, vm_flags); 1929ba76149fSAndrea Arcangeli return 0; 1930ba76149fSAndrea Arcangeli } 1931ba76149fSAndrea Arcangeli 1932ba76149fSAndrea Arcangeli void __khugepaged_exit(struct mm_struct *mm) 1933ba76149fSAndrea Arcangeli { 1934ba76149fSAndrea Arcangeli struct mm_slot *mm_slot; 1935ba76149fSAndrea Arcangeli int free = 0; 1936ba76149fSAndrea Arcangeli 1937ba76149fSAndrea Arcangeli spin_lock(&khugepaged_mm_lock); 1938ba76149fSAndrea Arcangeli mm_slot = get_mm_slot(mm); 1939ba76149fSAndrea Arcangeli if (mm_slot && khugepaged_scan.mm_slot != mm_slot) { 194043b5fbbdSSasha Levin hash_del(&mm_slot->hash); 1941ba76149fSAndrea Arcangeli list_del(&mm_slot->mm_node); 1942ba76149fSAndrea Arcangeli free = 1; 1943ba76149fSAndrea Arcangeli } 1944d788e80aSChris Wright spin_unlock(&khugepaged_mm_lock); 1945ba76149fSAndrea Arcangeli 1946ba76149fSAndrea Arcangeli if (free) { 1947ba76149fSAndrea Arcangeli clear_bit(MMF_VM_HUGEPAGE, &mm->flags); 1948ba76149fSAndrea Arcangeli free_mm_slot(mm_slot); 1949ba76149fSAndrea Arcangeli mmdrop(mm); 1950ba76149fSAndrea Arcangeli } else if (mm_slot) { 1951ba76149fSAndrea Arcangeli /* 1952ba76149fSAndrea Arcangeli * This is required to serialize against 1953ba76149fSAndrea Arcangeli * khugepaged_test_exit() (which is guaranteed to run 1954ba76149fSAndrea Arcangeli * under mmap sem read mode). Stop here (after we 1955ba76149fSAndrea Arcangeli * return all pagetables will be destroyed) until 1956ba76149fSAndrea Arcangeli * khugepaged has finished working on the pagetables 1957ba76149fSAndrea Arcangeli * under the mmap_sem. 1958ba76149fSAndrea Arcangeli */ 1959ba76149fSAndrea Arcangeli down_write(&mm->mmap_sem); 1960ba76149fSAndrea Arcangeli up_write(&mm->mmap_sem); 1961d788e80aSChris Wright } 1962ba76149fSAndrea Arcangeli } 1963ba76149fSAndrea Arcangeli 1964ba76149fSAndrea Arcangeli static void release_pte_page(struct page *page) 1965ba76149fSAndrea Arcangeli { 1966ba76149fSAndrea Arcangeli /* 0 stands for page_is_file_cache(page) == false */ 1967ba76149fSAndrea Arcangeli dec_zone_page_state(page, NR_ISOLATED_ANON + 0); 1968ba76149fSAndrea Arcangeli unlock_page(page); 1969ba76149fSAndrea Arcangeli putback_lru_page(page); 1970ba76149fSAndrea Arcangeli } 1971ba76149fSAndrea Arcangeli 1972ba76149fSAndrea Arcangeli static void release_pte_pages(pte_t *pte, pte_t *_pte) 1973ba76149fSAndrea Arcangeli { 1974ba76149fSAndrea Arcangeli while (--_pte >= pte) { 1975ba76149fSAndrea Arcangeli pte_t pteval = *_pte; 1976ca0984caSEbru Akagunduz if (!pte_none(pteval) && !is_zero_pfn(pte_pfn(pteval))) 1977ba76149fSAndrea Arcangeli release_pte_page(pte_page(pteval)); 1978ba76149fSAndrea Arcangeli } 1979ba76149fSAndrea Arcangeli } 1980ba76149fSAndrea Arcangeli 1981ba76149fSAndrea Arcangeli static int __collapse_huge_page_isolate(struct vm_area_struct *vma, 1982ba76149fSAndrea Arcangeli unsigned long address, 1983ba76149fSAndrea Arcangeli pte_t *pte) 1984ba76149fSAndrea Arcangeli { 19857d2eba05SEbru Akagunduz struct page *page = NULL; 1986ba76149fSAndrea Arcangeli pte_t *_pte; 19877d2eba05SEbru Akagunduz int none_or_zero = 0, result = 0; 198810359213SEbru Akagunduz bool referenced = false, writable = false; 19897d2eba05SEbru Akagunduz 1990ba76149fSAndrea Arcangeli for (_pte = pte; _pte < pte+HPAGE_PMD_NR; 1991ba76149fSAndrea Arcangeli _pte++, address += PAGE_SIZE) { 1992ba76149fSAndrea Arcangeli pte_t pteval = *_pte; 199347aee4d8SMinchan Kim if (pte_none(pteval) || (pte_present(pteval) && 199447aee4d8SMinchan Kim is_zero_pfn(pte_pfn(pteval)))) { 1995c1294d05SAndrea Arcangeli if (!userfaultfd_armed(vma) && 19967d2eba05SEbru Akagunduz ++none_or_zero <= khugepaged_max_ptes_none) { 1997ba76149fSAndrea Arcangeli continue; 19987d2eba05SEbru Akagunduz } else { 19997d2eba05SEbru Akagunduz result = SCAN_EXCEED_NONE_PTE; 2000ba76149fSAndrea Arcangeli goto out; 2001ba76149fSAndrea Arcangeli } 20027d2eba05SEbru Akagunduz } 20037d2eba05SEbru Akagunduz if (!pte_present(pteval)) { 20047d2eba05SEbru Akagunduz result = SCAN_PTE_NON_PRESENT; 2005ba76149fSAndrea Arcangeli goto out; 20067d2eba05SEbru Akagunduz } 2007ba76149fSAndrea Arcangeli page = vm_normal_page(vma, address, pteval); 20087d2eba05SEbru Akagunduz if (unlikely(!page)) { 20097d2eba05SEbru Akagunduz result = SCAN_PAGE_NULL; 2010ba76149fSAndrea Arcangeli goto out; 20117d2eba05SEbru Akagunduz } 2012344aa35cSBob Liu 2013309381feSSasha Levin VM_BUG_ON_PAGE(PageCompound(page), page); 2014309381feSSasha Levin VM_BUG_ON_PAGE(!PageAnon(page), page); 2015309381feSSasha Levin VM_BUG_ON_PAGE(!PageSwapBacked(page), page); 2016ba76149fSAndrea Arcangeli 2017ba76149fSAndrea Arcangeli /* 2018ba76149fSAndrea Arcangeli * We can do it before isolate_lru_page because the 2019ba76149fSAndrea Arcangeli * page can't be freed from under us. NOTE: PG_lock 2020ba76149fSAndrea Arcangeli * is needed to serialize against split_huge_page 2021ba76149fSAndrea Arcangeli * when invoked from the VM. 2022ba76149fSAndrea Arcangeli */ 20237d2eba05SEbru Akagunduz if (!trylock_page(page)) { 20247d2eba05SEbru Akagunduz result = SCAN_PAGE_LOCK; 2025ba76149fSAndrea Arcangeli goto out; 20267d2eba05SEbru Akagunduz } 202710359213SEbru Akagunduz 202810359213SEbru Akagunduz /* 202910359213SEbru Akagunduz * cannot use mapcount: can't collapse if there's a gup pin. 203010359213SEbru Akagunduz * The page must only be referenced by the scanned process 203110359213SEbru Akagunduz * and page swap cache. 203210359213SEbru Akagunduz */ 203310359213SEbru Akagunduz if (page_count(page) != 1 + !!PageSwapCache(page)) { 203410359213SEbru Akagunduz unlock_page(page); 20357d2eba05SEbru Akagunduz result = SCAN_PAGE_COUNT; 203610359213SEbru Akagunduz goto out; 203710359213SEbru Akagunduz } 203810359213SEbru Akagunduz if (pte_write(pteval)) { 203910359213SEbru Akagunduz writable = true; 204010359213SEbru Akagunduz } else { 204110359213SEbru Akagunduz if (PageSwapCache(page) && !reuse_swap_page(page)) { 204210359213SEbru Akagunduz unlock_page(page); 20437d2eba05SEbru Akagunduz result = SCAN_SWAP_CACHE_PAGE; 204410359213SEbru Akagunduz goto out; 204510359213SEbru Akagunduz } 204610359213SEbru Akagunduz /* 204710359213SEbru Akagunduz * Page is not in the swap cache. It can be collapsed 204810359213SEbru Akagunduz * into a THP. 204910359213SEbru Akagunduz */ 205010359213SEbru Akagunduz } 205110359213SEbru Akagunduz 2052ba76149fSAndrea Arcangeli /* 2053ba76149fSAndrea Arcangeli * Isolate the page to avoid collapsing an hugepage 2054ba76149fSAndrea Arcangeli * currently in use by the VM. 2055ba76149fSAndrea Arcangeli */ 2056ba76149fSAndrea Arcangeli if (isolate_lru_page(page)) { 2057ba76149fSAndrea Arcangeli unlock_page(page); 20587d2eba05SEbru Akagunduz result = SCAN_DEL_PAGE_LRU; 2059ba76149fSAndrea Arcangeli goto out; 2060ba76149fSAndrea Arcangeli } 2061ba76149fSAndrea Arcangeli /* 0 stands for page_is_file_cache(page) == false */ 2062ba76149fSAndrea Arcangeli inc_zone_page_state(page, NR_ISOLATED_ANON + 0); 2063309381feSSasha Levin VM_BUG_ON_PAGE(!PageLocked(page), page); 2064309381feSSasha Levin VM_BUG_ON_PAGE(PageLRU(page), page); 2065ba76149fSAndrea Arcangeli 2066ba76149fSAndrea Arcangeli /* If there is no mapped pte young don't collapse the page */ 206733c3fc71SVladimir Davydov if (pte_young(pteval) || 206833c3fc71SVladimir Davydov page_is_young(page) || PageReferenced(page) || 20698ee53820SAndrea Arcangeli mmu_notifier_test_young(vma->vm_mm, address)) 207010359213SEbru Akagunduz referenced = true; 2071ba76149fSAndrea Arcangeli } 20727d2eba05SEbru Akagunduz if (likely(writable)) { 20737d2eba05SEbru Akagunduz if (likely(referenced)) { 20747d2eba05SEbru Akagunduz result = SCAN_SUCCEED; 20757d2eba05SEbru Akagunduz trace_mm_collapse_huge_page_isolate(page_to_pfn(page), none_or_zero, 20767d2eba05SEbru Akagunduz referenced, writable, result); 2077344aa35cSBob Liu return 1; 20787d2eba05SEbru Akagunduz } 20797d2eba05SEbru Akagunduz } else { 20807d2eba05SEbru Akagunduz result = SCAN_PAGE_RO; 20817d2eba05SEbru Akagunduz } 20827d2eba05SEbru Akagunduz 2083ba76149fSAndrea Arcangeli out: 2084344aa35cSBob Liu release_pte_pages(pte, _pte); 20857d2eba05SEbru Akagunduz trace_mm_collapse_huge_page_isolate(page_to_pfn(page), none_or_zero, 20867d2eba05SEbru Akagunduz referenced, writable, result); 2087344aa35cSBob Liu return 0; 2088ba76149fSAndrea Arcangeli } 2089ba76149fSAndrea Arcangeli 2090ba76149fSAndrea Arcangeli static void __collapse_huge_page_copy(pte_t *pte, struct page *page, 2091ba76149fSAndrea Arcangeli struct vm_area_struct *vma, 2092ba76149fSAndrea Arcangeli unsigned long address, 2093ba76149fSAndrea Arcangeli spinlock_t *ptl) 2094ba76149fSAndrea Arcangeli { 2095ba76149fSAndrea Arcangeli pte_t *_pte; 2096ba76149fSAndrea Arcangeli for (_pte = pte; _pte < pte+HPAGE_PMD_NR; _pte++) { 2097ba76149fSAndrea Arcangeli pte_t pteval = *_pte; 2098ba76149fSAndrea Arcangeli struct page *src_page; 2099ba76149fSAndrea Arcangeli 2100ca0984caSEbru Akagunduz if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) { 2101ba76149fSAndrea Arcangeli clear_user_highpage(page, address); 2102ba76149fSAndrea Arcangeli add_mm_counter(vma->vm_mm, MM_ANONPAGES, 1); 2103ca0984caSEbru Akagunduz if (is_zero_pfn(pte_pfn(pteval))) { 2104ca0984caSEbru Akagunduz /* 2105ca0984caSEbru Akagunduz * ptl mostly unnecessary. 2106ca0984caSEbru Akagunduz */ 2107ca0984caSEbru Akagunduz spin_lock(ptl); 2108ca0984caSEbru Akagunduz /* 2109ca0984caSEbru Akagunduz * paravirt calls inside pte_clear here are 2110ca0984caSEbru Akagunduz * superfluous. 2111ca0984caSEbru Akagunduz */ 2112ca0984caSEbru Akagunduz pte_clear(vma->vm_mm, address, _pte); 2113ca0984caSEbru Akagunduz spin_unlock(ptl); 2114ca0984caSEbru Akagunduz } 2115ba76149fSAndrea Arcangeli } else { 2116ba76149fSAndrea Arcangeli src_page = pte_page(pteval); 2117ba76149fSAndrea Arcangeli copy_user_highpage(page, src_page, address, vma); 2118309381feSSasha Levin VM_BUG_ON_PAGE(page_mapcount(src_page) != 1, src_page); 2119ba76149fSAndrea Arcangeli release_pte_page(src_page); 2120ba76149fSAndrea Arcangeli /* 2121ba76149fSAndrea Arcangeli * ptl mostly unnecessary, but preempt has to 2122ba76149fSAndrea Arcangeli * be disabled to update the per-cpu stats 2123ba76149fSAndrea Arcangeli * inside page_remove_rmap(). 2124ba76149fSAndrea Arcangeli */ 2125ba76149fSAndrea Arcangeli spin_lock(ptl); 2126ba76149fSAndrea Arcangeli /* 2127ba76149fSAndrea Arcangeli * paravirt calls inside pte_clear here are 2128ba76149fSAndrea Arcangeli * superfluous. 2129ba76149fSAndrea Arcangeli */ 2130ba76149fSAndrea Arcangeli pte_clear(vma->vm_mm, address, _pte); 2131d281ee61SKirill A. Shutemov page_remove_rmap(src_page, false); 2132ba76149fSAndrea Arcangeli spin_unlock(ptl); 2133ba76149fSAndrea Arcangeli free_page_and_swap_cache(src_page); 2134ba76149fSAndrea Arcangeli } 2135ba76149fSAndrea Arcangeli 2136ba76149fSAndrea Arcangeli address += PAGE_SIZE; 2137ba76149fSAndrea Arcangeli page++; 2138ba76149fSAndrea Arcangeli } 2139ba76149fSAndrea Arcangeli } 2140ba76149fSAndrea Arcangeli 214126234f36SXiao Guangrong static void khugepaged_alloc_sleep(void) 214226234f36SXiao Guangrong { 2143bde43c6cSPetr Mladek DEFINE_WAIT(wait); 2144bde43c6cSPetr Mladek 2145bde43c6cSPetr Mladek add_wait_queue(&khugepaged_wait, &wait); 2146bde43c6cSPetr Mladek freezable_schedule_timeout_interruptible( 214726234f36SXiao Guangrong msecs_to_jiffies(khugepaged_alloc_sleep_millisecs)); 2148bde43c6cSPetr Mladek remove_wait_queue(&khugepaged_wait, &wait); 214926234f36SXiao Guangrong } 215026234f36SXiao Guangrong 21519f1b868aSBob Liu static int khugepaged_node_load[MAX_NUMNODES]; 21529f1b868aSBob Liu 215314a4e214SDavid Rientjes static bool khugepaged_scan_abort(int nid) 215414a4e214SDavid Rientjes { 215514a4e214SDavid Rientjes int i; 215614a4e214SDavid Rientjes 215714a4e214SDavid Rientjes /* 215814a4e214SDavid Rientjes * If zone_reclaim_mode is disabled, then no extra effort is made to 215914a4e214SDavid Rientjes * allocate memory locally. 216014a4e214SDavid Rientjes */ 216114a4e214SDavid Rientjes if (!zone_reclaim_mode) 216214a4e214SDavid Rientjes return false; 216314a4e214SDavid Rientjes 216414a4e214SDavid Rientjes /* If there is a count for this node already, it must be acceptable */ 216514a4e214SDavid Rientjes if (khugepaged_node_load[nid]) 216614a4e214SDavid Rientjes return false; 216714a4e214SDavid Rientjes 216814a4e214SDavid Rientjes for (i = 0; i < MAX_NUMNODES; i++) { 216914a4e214SDavid Rientjes if (!khugepaged_node_load[i]) 217014a4e214SDavid Rientjes continue; 217114a4e214SDavid Rientjes if (node_distance(nid, i) > RECLAIM_DISTANCE) 217214a4e214SDavid Rientjes return true; 217314a4e214SDavid Rientjes } 217414a4e214SDavid Rientjes return false; 217514a4e214SDavid Rientjes } 217614a4e214SDavid Rientjes 217726234f36SXiao Guangrong #ifdef CONFIG_NUMA 21789f1b868aSBob Liu static int khugepaged_find_target_node(void) 21799f1b868aSBob Liu { 21809f1b868aSBob Liu static int last_khugepaged_target_node = NUMA_NO_NODE; 21819f1b868aSBob Liu int nid, target_node = 0, max_value = 0; 21829f1b868aSBob Liu 21839f1b868aSBob Liu /* find first node with max normal pages hit */ 21849f1b868aSBob Liu for (nid = 0; nid < MAX_NUMNODES; nid++) 21859f1b868aSBob Liu if (khugepaged_node_load[nid] > max_value) { 21869f1b868aSBob Liu max_value = khugepaged_node_load[nid]; 21879f1b868aSBob Liu target_node = nid; 21889f1b868aSBob Liu } 21899f1b868aSBob Liu 21909f1b868aSBob Liu /* do some balance if several nodes have the same hit record */ 21919f1b868aSBob Liu if (target_node <= last_khugepaged_target_node) 21929f1b868aSBob Liu for (nid = last_khugepaged_target_node + 1; nid < MAX_NUMNODES; 21939f1b868aSBob Liu nid++) 21949f1b868aSBob Liu if (max_value == khugepaged_node_load[nid]) { 21959f1b868aSBob Liu target_node = nid; 21969f1b868aSBob Liu break; 21979f1b868aSBob Liu } 21989f1b868aSBob Liu 21999f1b868aSBob Liu last_khugepaged_target_node = target_node; 22009f1b868aSBob Liu return target_node; 22019f1b868aSBob Liu } 22029f1b868aSBob Liu 220326234f36SXiao Guangrong static bool khugepaged_prealloc_page(struct page **hpage, bool *wait) 220426234f36SXiao Guangrong { 220526234f36SXiao Guangrong if (IS_ERR(*hpage)) { 220626234f36SXiao Guangrong if (!*wait) 220726234f36SXiao Guangrong return false; 220826234f36SXiao Guangrong 220926234f36SXiao Guangrong *wait = false; 2210e3b4126cSXiao Guangrong *hpage = NULL; 221126234f36SXiao Guangrong khugepaged_alloc_sleep(); 221226234f36SXiao Guangrong } else if (*hpage) { 221326234f36SXiao Guangrong put_page(*hpage); 221426234f36SXiao Guangrong *hpage = NULL; 221526234f36SXiao Guangrong } 221626234f36SXiao Guangrong 221726234f36SXiao Guangrong return true; 221826234f36SXiao Guangrong } 221926234f36SXiao Guangrong 22203b363692SMichal Hocko static struct page * 22213b363692SMichal Hocko khugepaged_alloc_page(struct page **hpage, gfp_t gfp, struct mm_struct *mm, 2222d6669d68SAaron Tomlin unsigned long address, int node) 222326234f36SXiao Guangrong { 2224309381feSSasha Levin VM_BUG_ON_PAGE(*hpage, *hpage); 22258b164568SVlastimil Babka 222626234f36SXiao Guangrong /* 22278b164568SVlastimil Babka * Before allocating the hugepage, release the mmap_sem read lock. 22288b164568SVlastimil Babka * The allocation can take potentially a long time if it involves 22298b164568SVlastimil Babka * sync compaction, and we do not need to hold the mmap_sem during 22308b164568SVlastimil Babka * that. We will recheck the vma after taking it again in write mode. 223126234f36SXiao Guangrong */ 223226234f36SXiao Guangrong up_read(&mm->mmap_sem); 22338b164568SVlastimil Babka 223496db800fSVlastimil Babka *hpage = __alloc_pages_node(node, gfp, HPAGE_PMD_ORDER); 223526234f36SXiao Guangrong if (unlikely(!*hpage)) { 223626234f36SXiao Guangrong count_vm_event(THP_COLLAPSE_ALLOC_FAILED); 223726234f36SXiao Guangrong *hpage = ERR_PTR(-ENOMEM); 223826234f36SXiao Guangrong return NULL; 223926234f36SXiao Guangrong } 224026234f36SXiao Guangrong 22419a982250SKirill A. Shutemov prep_transhuge_page(*hpage); 224226234f36SXiao Guangrong count_vm_event(THP_COLLAPSE_ALLOC); 224326234f36SXiao Guangrong return *hpage; 224426234f36SXiao Guangrong } 224526234f36SXiao Guangrong #else 22469f1b868aSBob Liu static int khugepaged_find_target_node(void) 22479f1b868aSBob Liu { 22489f1b868aSBob Liu return 0; 22499f1b868aSBob Liu } 22509f1b868aSBob Liu 225110dc4155SBob Liu static inline struct page *alloc_hugepage(int defrag) 225210dc4155SBob Liu { 22539a982250SKirill A. Shutemov struct page *page; 22549a982250SKirill A. Shutemov 22559a982250SKirill A. Shutemov page = alloc_pages(alloc_hugepage_gfpmask(defrag, 0), HPAGE_PMD_ORDER); 22569a982250SKirill A. Shutemov if (page) 22579a982250SKirill A. Shutemov prep_transhuge_page(page); 22589a982250SKirill A. Shutemov return page; 225910dc4155SBob Liu } 226010dc4155SBob Liu 226126234f36SXiao Guangrong static struct page *khugepaged_alloc_hugepage(bool *wait) 226226234f36SXiao Guangrong { 226326234f36SXiao Guangrong struct page *hpage; 226426234f36SXiao Guangrong 226526234f36SXiao Guangrong do { 226626234f36SXiao Guangrong hpage = alloc_hugepage(khugepaged_defrag()); 226726234f36SXiao Guangrong if (!hpage) { 226826234f36SXiao Guangrong count_vm_event(THP_COLLAPSE_ALLOC_FAILED); 226926234f36SXiao Guangrong if (!*wait) 227026234f36SXiao Guangrong return NULL; 227126234f36SXiao Guangrong 227226234f36SXiao Guangrong *wait = false; 227326234f36SXiao Guangrong khugepaged_alloc_sleep(); 227426234f36SXiao Guangrong } else 227526234f36SXiao Guangrong count_vm_event(THP_COLLAPSE_ALLOC); 227626234f36SXiao Guangrong } while (unlikely(!hpage) && likely(khugepaged_enabled())); 227726234f36SXiao Guangrong 227826234f36SXiao Guangrong return hpage; 227926234f36SXiao Guangrong } 228026234f36SXiao Guangrong 228126234f36SXiao Guangrong static bool khugepaged_prealloc_page(struct page **hpage, bool *wait) 228226234f36SXiao Guangrong { 228326234f36SXiao Guangrong if (!*hpage) 228426234f36SXiao Guangrong *hpage = khugepaged_alloc_hugepage(wait); 228526234f36SXiao Guangrong 228626234f36SXiao Guangrong if (unlikely(!*hpage)) 228726234f36SXiao Guangrong return false; 228826234f36SXiao Guangrong 228926234f36SXiao Guangrong return true; 229026234f36SXiao Guangrong } 229126234f36SXiao Guangrong 22923b363692SMichal Hocko static struct page * 22933b363692SMichal Hocko khugepaged_alloc_page(struct page **hpage, gfp_t gfp, struct mm_struct *mm, 2294d6669d68SAaron Tomlin unsigned long address, int node) 229526234f36SXiao Guangrong { 229626234f36SXiao Guangrong up_read(&mm->mmap_sem); 229726234f36SXiao Guangrong VM_BUG_ON(!*hpage); 22983b363692SMichal Hocko 229926234f36SXiao Guangrong return *hpage; 230026234f36SXiao Guangrong } 230126234f36SXiao Guangrong #endif 230226234f36SXiao Guangrong 2303fa475e51SBob Liu static bool hugepage_vma_check(struct vm_area_struct *vma) 2304fa475e51SBob Liu { 2305fa475e51SBob Liu if ((!(vma->vm_flags & VM_HUGEPAGE) && !khugepaged_always()) || 2306fa475e51SBob Liu (vma->vm_flags & VM_NOHUGEPAGE)) 2307fa475e51SBob Liu return false; 2308fa475e51SBob Liu if (!vma->anon_vma || vma->vm_ops) 2309fa475e51SBob Liu return false; 2310fa475e51SBob Liu if (is_vma_temporary_stack(vma)) 2311fa475e51SBob Liu return false; 231281d1b09cSSasha Levin VM_BUG_ON_VMA(vma->vm_flags & VM_NO_THP, vma); 2313fa475e51SBob Liu return true; 2314fa475e51SBob Liu } 2315fa475e51SBob Liu 2316ba76149fSAndrea Arcangeli static void collapse_huge_page(struct mm_struct *mm, 2317ba76149fSAndrea Arcangeli unsigned long address, 2318ce83d217SAndrea Arcangeli struct page **hpage, 23195c4b4be3SAndi Kleen struct vm_area_struct *vma, 23205c4b4be3SAndi Kleen int node) 2321ba76149fSAndrea Arcangeli { 2322ba76149fSAndrea Arcangeli pmd_t *pmd, _pmd; 2323ba76149fSAndrea Arcangeli pte_t *pte; 2324ba76149fSAndrea Arcangeli pgtable_t pgtable; 2325ba76149fSAndrea Arcangeli struct page *new_page; 2326c4088ebdSKirill A. Shutemov spinlock_t *pmd_ptl, *pte_ptl; 2327629d9d1cSArnd Bergmann int isolated = 0, result = 0; 2328ba76149fSAndrea Arcangeli unsigned long hstart, hend; 232900501b53SJohannes Weiner struct mem_cgroup *memcg; 23302ec74c3eSSagi Grimberg unsigned long mmun_start; /* For mmu_notifiers */ 23312ec74c3eSSagi Grimberg unsigned long mmun_end; /* For mmu_notifiers */ 23323b363692SMichal Hocko gfp_t gfp; 2333ba76149fSAndrea Arcangeli 2334ba76149fSAndrea Arcangeli VM_BUG_ON(address & ~HPAGE_PMD_MASK); 2335692e0b35SAndrea Arcangeli 23363b363692SMichal Hocko /* Only allocate from the target node */ 23373b363692SMichal Hocko gfp = alloc_hugepage_gfpmask(khugepaged_defrag(), __GFP_OTHER_NODE) | 23383b363692SMichal Hocko __GFP_THISNODE; 23393b363692SMichal Hocko 234026234f36SXiao Guangrong /* release the mmap_sem read lock. */ 2341d6669d68SAaron Tomlin new_page = khugepaged_alloc_page(hpage, gfp, mm, address, node); 23427d2eba05SEbru Akagunduz if (!new_page) { 23437d2eba05SEbru Akagunduz result = SCAN_ALLOC_HUGE_PAGE_FAIL; 23447d2eba05SEbru Akagunduz goto out_nolock; 23457d2eba05SEbru Akagunduz } 2346ce83d217SAndrea Arcangeli 2347f627c2f5SKirill A. Shutemov if (unlikely(mem_cgroup_try_charge(new_page, mm, gfp, &memcg, true))) { 23487d2eba05SEbru Akagunduz result = SCAN_CGROUP_CHARGE_FAIL; 23497d2eba05SEbru Akagunduz goto out_nolock; 23507d2eba05SEbru Akagunduz } 2351ba76149fSAndrea Arcangeli 2352ba76149fSAndrea Arcangeli /* 2353ba76149fSAndrea Arcangeli * Prevent all access to pagetables with the exception of 2354ba76149fSAndrea Arcangeli * gup_fast later hanlded by the ptep_clear_flush and the VM 2355ba76149fSAndrea Arcangeli * handled by the anon_vma lock + PG_lock. 2356ba76149fSAndrea Arcangeli */ 2357ba76149fSAndrea Arcangeli down_write(&mm->mmap_sem); 23587d2eba05SEbru Akagunduz if (unlikely(khugepaged_test_exit(mm))) { 23597d2eba05SEbru Akagunduz result = SCAN_ANY_PROCESS; 2360ba76149fSAndrea Arcangeli goto out; 23617d2eba05SEbru Akagunduz } 2362ba76149fSAndrea Arcangeli 2363ba76149fSAndrea Arcangeli vma = find_vma(mm, address); 23647d2eba05SEbru Akagunduz if (!vma) { 23657d2eba05SEbru Akagunduz result = SCAN_VMA_NULL; 2366a8f531ebSLibin goto out; 23677d2eba05SEbru Akagunduz } 2368ba76149fSAndrea Arcangeli hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK; 2369ba76149fSAndrea Arcangeli hend = vma->vm_end & HPAGE_PMD_MASK; 23707d2eba05SEbru Akagunduz if (address < hstart || address + HPAGE_PMD_SIZE > hend) { 23717d2eba05SEbru Akagunduz result = SCAN_ADDRESS_RANGE; 2372ba76149fSAndrea Arcangeli goto out; 23737d2eba05SEbru Akagunduz } 23747d2eba05SEbru Akagunduz if (!hugepage_vma_check(vma)) { 23757d2eba05SEbru Akagunduz result = SCAN_VMA_CHECK; 2376ba76149fSAndrea Arcangeli goto out; 23777d2eba05SEbru Akagunduz } 23786219049aSBob Liu pmd = mm_find_pmd(mm, address); 23797d2eba05SEbru Akagunduz if (!pmd) { 23807d2eba05SEbru Akagunduz result = SCAN_PMD_NULL; 2381ba76149fSAndrea Arcangeli goto out; 23827d2eba05SEbru Akagunduz } 2383ba76149fSAndrea Arcangeli 23844fc3f1d6SIngo Molnar anon_vma_lock_write(vma->anon_vma); 2385ba76149fSAndrea Arcangeli 2386ba76149fSAndrea Arcangeli pte = pte_offset_map(pmd, address); 2387c4088ebdSKirill A. Shutemov pte_ptl = pte_lockptr(mm, pmd); 2388ba76149fSAndrea Arcangeli 23892ec74c3eSSagi Grimberg mmun_start = address; 23902ec74c3eSSagi Grimberg mmun_end = address + HPAGE_PMD_SIZE; 23912ec74c3eSSagi Grimberg mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 2392c4088ebdSKirill A. Shutemov pmd_ptl = pmd_lock(mm, pmd); /* probably unnecessary */ 2393ba76149fSAndrea Arcangeli /* 2394ba76149fSAndrea Arcangeli * After this gup_fast can't run anymore. This also removes 2395ba76149fSAndrea Arcangeli * any huge TLB entry from the CPU so we won't allow 2396ba76149fSAndrea Arcangeli * huge and small TLB entries for the same virtual address 2397ba76149fSAndrea Arcangeli * to avoid the risk of CPU bugs in that area. 2398ba76149fSAndrea Arcangeli */ 239915a25b2eSAneesh Kumar K.V _pmd = pmdp_collapse_flush(vma, address, pmd); 2400c4088ebdSKirill A. Shutemov spin_unlock(pmd_ptl); 24012ec74c3eSSagi Grimberg mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 2402ba76149fSAndrea Arcangeli 2403c4088ebdSKirill A. Shutemov spin_lock(pte_ptl); 2404ba76149fSAndrea Arcangeli isolated = __collapse_huge_page_isolate(vma, address, pte); 2405c4088ebdSKirill A. Shutemov spin_unlock(pte_ptl); 2406ba76149fSAndrea Arcangeli 2407ba76149fSAndrea Arcangeli if (unlikely(!isolated)) { 2408453c7192SJohannes Weiner pte_unmap(pte); 2409c4088ebdSKirill A. Shutemov spin_lock(pmd_ptl); 2410ba76149fSAndrea Arcangeli BUG_ON(!pmd_none(*pmd)); 24117c342512SAneesh Kumar K.V /* 24127c342512SAneesh Kumar K.V * We can only use set_pmd_at when establishing 24137c342512SAneesh Kumar K.V * hugepmds and never for establishing regular pmds that 24147c342512SAneesh Kumar K.V * points to regular pagetables. Use pmd_populate for that 24157c342512SAneesh Kumar K.V */ 24167c342512SAneesh Kumar K.V pmd_populate(mm, pmd, pmd_pgtable(_pmd)); 2417c4088ebdSKirill A. Shutemov spin_unlock(pmd_ptl); 241808b52706SKonstantin Khlebnikov anon_vma_unlock_write(vma->anon_vma); 24197d2eba05SEbru Akagunduz result = SCAN_FAIL; 2420ce83d217SAndrea Arcangeli goto out; 2421ba76149fSAndrea Arcangeli } 2422ba76149fSAndrea Arcangeli 2423ba76149fSAndrea Arcangeli /* 2424ba76149fSAndrea Arcangeli * All pages are isolated and locked so anon_vma rmap 2425ba76149fSAndrea Arcangeli * can't run anymore. 2426ba76149fSAndrea Arcangeli */ 242708b52706SKonstantin Khlebnikov anon_vma_unlock_write(vma->anon_vma); 2428ba76149fSAndrea Arcangeli 2429c4088ebdSKirill A. Shutemov __collapse_huge_page_copy(pte, new_page, vma, address, pte_ptl); 2430453c7192SJohannes Weiner pte_unmap(pte); 2431ba76149fSAndrea Arcangeli __SetPageUptodate(new_page); 2432ba76149fSAndrea Arcangeli pgtable = pmd_pgtable(_pmd); 2433ba76149fSAndrea Arcangeli 24343122359aSKirill A. Shutemov _pmd = mk_huge_pmd(new_page, vma->vm_page_prot); 24353122359aSKirill A. Shutemov _pmd = maybe_pmd_mkwrite(pmd_mkdirty(_pmd), vma); 2436ba76149fSAndrea Arcangeli 2437ba76149fSAndrea Arcangeli /* 2438ba76149fSAndrea Arcangeli * spin_lock() below is not the equivalent of smp_wmb(), so 2439ba76149fSAndrea Arcangeli * this is needed to avoid the copy_huge_page writes to become 2440ba76149fSAndrea Arcangeli * visible after the set_pmd_at() write. 2441ba76149fSAndrea Arcangeli */ 2442ba76149fSAndrea Arcangeli smp_wmb(); 2443ba76149fSAndrea Arcangeli 2444c4088ebdSKirill A. Shutemov spin_lock(pmd_ptl); 2445ba76149fSAndrea Arcangeli BUG_ON(!pmd_none(*pmd)); 2446d281ee61SKirill A. Shutemov page_add_new_anon_rmap(new_page, vma, address, true); 2447f627c2f5SKirill A. Shutemov mem_cgroup_commit_charge(new_page, memcg, false, true); 244800501b53SJohannes Weiner lru_cache_add_active_or_unevictable(new_page, vma); 2449fce144b4SAneesh Kumar K.V pgtable_trans_huge_deposit(mm, pmd, pgtable); 2450ba76149fSAndrea Arcangeli set_pmd_at(mm, address, pmd, _pmd); 2451b113da65SDavid Miller update_mmu_cache_pmd(vma, address, pmd); 2452c4088ebdSKirill A. Shutemov spin_unlock(pmd_ptl); 2453ba76149fSAndrea Arcangeli 2454ba76149fSAndrea Arcangeli *hpage = NULL; 2455420256efSXiao Guangrong 2456ba76149fSAndrea Arcangeli khugepaged_pages_collapsed++; 24577d2eba05SEbru Akagunduz result = SCAN_SUCCEED; 2458ce83d217SAndrea Arcangeli out_up_write: 2459ba76149fSAndrea Arcangeli up_write(&mm->mmap_sem); 24607d2eba05SEbru Akagunduz trace_mm_collapse_huge_page(mm, isolated, result); 24610bbbc0b3SAndrea Arcangeli return; 24620bbbc0b3SAndrea Arcangeli 24637d2eba05SEbru Akagunduz out_nolock: 24647d2eba05SEbru Akagunduz trace_mm_collapse_huge_page(mm, isolated, result); 24657d2eba05SEbru Akagunduz return; 2466ce83d217SAndrea Arcangeli out: 2467f627c2f5SKirill A. Shutemov mem_cgroup_cancel_charge(new_page, memcg, true); 2468ce83d217SAndrea Arcangeli goto out_up_write; 2469ba76149fSAndrea Arcangeli } 2470ba76149fSAndrea Arcangeli 2471ba76149fSAndrea Arcangeli static int khugepaged_scan_pmd(struct mm_struct *mm, 2472ba76149fSAndrea Arcangeli struct vm_area_struct *vma, 2473ba76149fSAndrea Arcangeli unsigned long address, 2474ba76149fSAndrea Arcangeli struct page **hpage) 2475ba76149fSAndrea Arcangeli { 2476ba76149fSAndrea Arcangeli pmd_t *pmd; 2477ba76149fSAndrea Arcangeli pte_t *pte, *_pte; 24787d2eba05SEbru Akagunduz int ret = 0, none_or_zero = 0, result = 0; 24797d2eba05SEbru Akagunduz struct page *page = NULL; 2480ba76149fSAndrea Arcangeli unsigned long _address; 2481ba76149fSAndrea Arcangeli spinlock_t *ptl; 248200ef2d2fSDavid Rientjes int node = NUMA_NO_NODE; 248310359213SEbru Akagunduz bool writable = false, referenced = false; 2484ba76149fSAndrea Arcangeli 2485ba76149fSAndrea Arcangeli VM_BUG_ON(address & ~HPAGE_PMD_MASK); 2486ba76149fSAndrea Arcangeli 24876219049aSBob Liu pmd = mm_find_pmd(mm, address); 24887d2eba05SEbru Akagunduz if (!pmd) { 24897d2eba05SEbru Akagunduz result = SCAN_PMD_NULL; 2490ba76149fSAndrea Arcangeli goto out; 24917d2eba05SEbru Akagunduz } 2492ba76149fSAndrea Arcangeli 24939f1b868aSBob Liu memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load)); 2494ba76149fSAndrea Arcangeli pte = pte_offset_map_lock(mm, pmd, address, &ptl); 2495ba76149fSAndrea Arcangeli for (_address = address, _pte = pte; _pte < pte+HPAGE_PMD_NR; 2496ba76149fSAndrea Arcangeli _pte++, _address += PAGE_SIZE) { 2497ba76149fSAndrea Arcangeli pte_t pteval = *_pte; 2498ca0984caSEbru Akagunduz if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) { 2499c1294d05SAndrea Arcangeli if (!userfaultfd_armed(vma) && 25007d2eba05SEbru Akagunduz ++none_or_zero <= khugepaged_max_ptes_none) { 2501ba76149fSAndrea Arcangeli continue; 25027d2eba05SEbru Akagunduz } else { 25037d2eba05SEbru Akagunduz result = SCAN_EXCEED_NONE_PTE; 2504ba76149fSAndrea Arcangeli goto out_unmap; 2505ba76149fSAndrea Arcangeli } 25067d2eba05SEbru Akagunduz } 25077d2eba05SEbru Akagunduz if (!pte_present(pteval)) { 25087d2eba05SEbru Akagunduz result = SCAN_PTE_NON_PRESENT; 2509ba76149fSAndrea Arcangeli goto out_unmap; 25107d2eba05SEbru Akagunduz } 251110359213SEbru Akagunduz if (pte_write(pteval)) 251210359213SEbru Akagunduz writable = true; 251310359213SEbru Akagunduz 2514ba76149fSAndrea Arcangeli page = vm_normal_page(vma, _address, pteval); 25157d2eba05SEbru Akagunduz if (unlikely(!page)) { 25167d2eba05SEbru Akagunduz result = SCAN_PAGE_NULL; 2517ba76149fSAndrea Arcangeli goto out_unmap; 25187d2eba05SEbru Akagunduz } 2519b1caa957SKirill A. Shutemov 2520b1caa957SKirill A. Shutemov /* TODO: teach khugepaged to collapse THP mapped with pte */ 2521b1caa957SKirill A. Shutemov if (PageCompound(page)) { 2522b1caa957SKirill A. Shutemov result = SCAN_PAGE_COMPOUND; 2523b1caa957SKirill A. Shutemov goto out_unmap; 2524b1caa957SKirill A. Shutemov } 2525b1caa957SKirill A. Shutemov 25265c4b4be3SAndi Kleen /* 25279f1b868aSBob Liu * Record which node the original page is from and save this 25289f1b868aSBob Liu * information to khugepaged_node_load[]. 25299f1b868aSBob Liu * Khupaged will allocate hugepage from the node has the max 25309f1b868aSBob Liu * hit record. 25315c4b4be3SAndi Kleen */ 25325c4b4be3SAndi Kleen node = page_to_nid(page); 25337d2eba05SEbru Akagunduz if (khugepaged_scan_abort(node)) { 25347d2eba05SEbru Akagunduz result = SCAN_SCAN_ABORT; 253514a4e214SDavid Rientjes goto out_unmap; 25367d2eba05SEbru Akagunduz } 25379f1b868aSBob Liu khugepaged_node_load[node]++; 25387d2eba05SEbru Akagunduz if (!PageLRU(page)) { 25397d2eba05SEbru Akagunduz result = SCAN_SCAN_ABORT; 2540ba76149fSAndrea Arcangeli goto out_unmap; 25417d2eba05SEbru Akagunduz } 25427d2eba05SEbru Akagunduz if (PageLocked(page)) { 25437d2eba05SEbru Akagunduz result = SCAN_PAGE_LOCK; 25447d2eba05SEbru Akagunduz goto out_unmap; 25457d2eba05SEbru Akagunduz } 25467d2eba05SEbru Akagunduz if (!PageAnon(page)) { 25477d2eba05SEbru Akagunduz result = SCAN_PAGE_ANON; 25487d2eba05SEbru Akagunduz goto out_unmap; 25497d2eba05SEbru Akagunduz } 25507d2eba05SEbru Akagunduz 255110359213SEbru Akagunduz /* 255210359213SEbru Akagunduz * cannot use mapcount: can't collapse if there's a gup pin. 255310359213SEbru Akagunduz * The page must only be referenced by the scanned process 255410359213SEbru Akagunduz * and page swap cache. 255510359213SEbru Akagunduz */ 25567d2eba05SEbru Akagunduz if (page_count(page) != 1 + !!PageSwapCache(page)) { 25577d2eba05SEbru Akagunduz result = SCAN_PAGE_COUNT; 2558ba76149fSAndrea Arcangeli goto out_unmap; 25597d2eba05SEbru Akagunduz } 256033c3fc71SVladimir Davydov if (pte_young(pteval) || 256133c3fc71SVladimir Davydov page_is_young(page) || PageReferenced(page) || 25628ee53820SAndrea Arcangeli mmu_notifier_test_young(vma->vm_mm, address)) 256310359213SEbru Akagunduz referenced = true; 2564ba76149fSAndrea Arcangeli } 25657d2eba05SEbru Akagunduz if (writable) { 25667d2eba05SEbru Akagunduz if (referenced) { 25677d2eba05SEbru Akagunduz result = SCAN_SUCCEED; 2568ba76149fSAndrea Arcangeli ret = 1; 25697d2eba05SEbru Akagunduz } else { 25707d2eba05SEbru Akagunduz result = SCAN_NO_REFERENCED_PAGE; 25717d2eba05SEbru Akagunduz } 25727d2eba05SEbru Akagunduz } else { 25737d2eba05SEbru Akagunduz result = SCAN_PAGE_RO; 25747d2eba05SEbru Akagunduz } 2575ba76149fSAndrea Arcangeli out_unmap: 2576ba76149fSAndrea Arcangeli pte_unmap_unlock(pte, ptl); 25779f1b868aSBob Liu if (ret) { 25789f1b868aSBob Liu node = khugepaged_find_target_node(); 2579ce83d217SAndrea Arcangeli /* collapse_huge_page will return with the mmap_sem released */ 25805c4b4be3SAndi Kleen collapse_huge_page(mm, address, hpage, vma, node); 25819f1b868aSBob Liu } 2582ba76149fSAndrea Arcangeli out: 25837d2eba05SEbru Akagunduz trace_mm_khugepaged_scan_pmd(mm, page_to_pfn(page), writable, referenced, 25847d2eba05SEbru Akagunduz none_or_zero, result); 2585ba76149fSAndrea Arcangeli return ret; 2586ba76149fSAndrea Arcangeli } 2587ba76149fSAndrea Arcangeli 2588ba76149fSAndrea Arcangeli static void collect_mm_slot(struct mm_slot *mm_slot) 2589ba76149fSAndrea Arcangeli { 2590ba76149fSAndrea Arcangeli struct mm_struct *mm = mm_slot->mm; 2591ba76149fSAndrea Arcangeli 2592b9980cdcSHugh Dickins VM_BUG_ON(NR_CPUS != 1 && !spin_is_locked(&khugepaged_mm_lock)); 2593ba76149fSAndrea Arcangeli 2594ba76149fSAndrea Arcangeli if (khugepaged_test_exit(mm)) { 2595ba76149fSAndrea Arcangeli /* free mm_slot */ 259643b5fbbdSSasha Levin hash_del(&mm_slot->hash); 2597ba76149fSAndrea Arcangeli list_del(&mm_slot->mm_node); 2598ba76149fSAndrea Arcangeli 2599ba76149fSAndrea Arcangeli /* 2600ba76149fSAndrea Arcangeli * Not strictly needed because the mm exited already. 2601ba76149fSAndrea Arcangeli * 2602ba76149fSAndrea Arcangeli * clear_bit(MMF_VM_HUGEPAGE, &mm->flags); 2603ba76149fSAndrea Arcangeli */ 2604ba76149fSAndrea Arcangeli 2605ba76149fSAndrea Arcangeli /* khugepaged_mm_lock actually not necessary for the below */ 2606ba76149fSAndrea Arcangeli free_mm_slot(mm_slot); 2607ba76149fSAndrea Arcangeli mmdrop(mm); 2608ba76149fSAndrea Arcangeli } 2609ba76149fSAndrea Arcangeli } 2610ba76149fSAndrea Arcangeli 2611ba76149fSAndrea Arcangeli static unsigned int khugepaged_scan_mm_slot(unsigned int pages, 2612ba76149fSAndrea Arcangeli struct page **hpage) 26132f1da642SH Hartley Sweeten __releases(&khugepaged_mm_lock) 26142f1da642SH Hartley Sweeten __acquires(&khugepaged_mm_lock) 2615ba76149fSAndrea Arcangeli { 2616ba76149fSAndrea Arcangeli struct mm_slot *mm_slot; 2617ba76149fSAndrea Arcangeli struct mm_struct *mm; 2618ba76149fSAndrea Arcangeli struct vm_area_struct *vma; 2619ba76149fSAndrea Arcangeli int progress = 0; 2620ba76149fSAndrea Arcangeli 2621ba76149fSAndrea Arcangeli VM_BUG_ON(!pages); 2622b9980cdcSHugh Dickins VM_BUG_ON(NR_CPUS != 1 && !spin_is_locked(&khugepaged_mm_lock)); 2623ba76149fSAndrea Arcangeli 2624ba76149fSAndrea Arcangeli if (khugepaged_scan.mm_slot) 2625ba76149fSAndrea Arcangeli mm_slot = khugepaged_scan.mm_slot; 2626ba76149fSAndrea Arcangeli else { 2627ba76149fSAndrea Arcangeli mm_slot = list_entry(khugepaged_scan.mm_head.next, 2628ba76149fSAndrea Arcangeli struct mm_slot, mm_node); 2629ba76149fSAndrea Arcangeli khugepaged_scan.address = 0; 2630ba76149fSAndrea Arcangeli khugepaged_scan.mm_slot = mm_slot; 2631ba76149fSAndrea Arcangeli } 2632ba76149fSAndrea Arcangeli spin_unlock(&khugepaged_mm_lock); 2633ba76149fSAndrea Arcangeli 2634ba76149fSAndrea Arcangeli mm = mm_slot->mm; 2635ba76149fSAndrea Arcangeli down_read(&mm->mmap_sem); 2636ba76149fSAndrea Arcangeli if (unlikely(khugepaged_test_exit(mm))) 2637ba76149fSAndrea Arcangeli vma = NULL; 2638ba76149fSAndrea Arcangeli else 2639ba76149fSAndrea Arcangeli vma = find_vma(mm, khugepaged_scan.address); 2640ba76149fSAndrea Arcangeli 2641ba76149fSAndrea Arcangeli progress++; 2642ba76149fSAndrea Arcangeli for (; vma; vma = vma->vm_next) { 2643ba76149fSAndrea Arcangeli unsigned long hstart, hend; 2644ba76149fSAndrea Arcangeli 2645ba76149fSAndrea Arcangeli cond_resched(); 2646ba76149fSAndrea Arcangeli if (unlikely(khugepaged_test_exit(mm))) { 2647ba76149fSAndrea Arcangeli progress++; 2648ba76149fSAndrea Arcangeli break; 2649ba76149fSAndrea Arcangeli } 2650fa475e51SBob Liu if (!hugepage_vma_check(vma)) { 2651a7d6e4ecSAndrea Arcangeli skip: 2652ba76149fSAndrea Arcangeli progress++; 2653ba76149fSAndrea Arcangeli continue; 2654ba76149fSAndrea Arcangeli } 2655ba76149fSAndrea Arcangeli hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK; 2656ba76149fSAndrea Arcangeli hend = vma->vm_end & HPAGE_PMD_MASK; 2657a7d6e4ecSAndrea Arcangeli if (hstart >= hend) 2658a7d6e4ecSAndrea Arcangeli goto skip; 2659a7d6e4ecSAndrea Arcangeli if (khugepaged_scan.address > hend) 2660a7d6e4ecSAndrea Arcangeli goto skip; 2661ba76149fSAndrea Arcangeli if (khugepaged_scan.address < hstart) 2662ba76149fSAndrea Arcangeli khugepaged_scan.address = hstart; 2663a7d6e4ecSAndrea Arcangeli VM_BUG_ON(khugepaged_scan.address & ~HPAGE_PMD_MASK); 2664ba76149fSAndrea Arcangeli 2665ba76149fSAndrea Arcangeli while (khugepaged_scan.address < hend) { 2666ba76149fSAndrea Arcangeli int ret; 2667ba76149fSAndrea Arcangeli cond_resched(); 2668ba76149fSAndrea Arcangeli if (unlikely(khugepaged_test_exit(mm))) 2669ba76149fSAndrea Arcangeli goto breakouterloop; 2670ba76149fSAndrea Arcangeli 2671ba76149fSAndrea Arcangeli VM_BUG_ON(khugepaged_scan.address < hstart || 2672ba76149fSAndrea Arcangeli khugepaged_scan.address + HPAGE_PMD_SIZE > 2673ba76149fSAndrea Arcangeli hend); 2674ba76149fSAndrea Arcangeli ret = khugepaged_scan_pmd(mm, vma, 2675ba76149fSAndrea Arcangeli khugepaged_scan.address, 2676ba76149fSAndrea Arcangeli hpage); 2677ba76149fSAndrea Arcangeli /* move to next address */ 2678ba76149fSAndrea Arcangeli khugepaged_scan.address += HPAGE_PMD_SIZE; 2679ba76149fSAndrea Arcangeli progress += HPAGE_PMD_NR; 2680ba76149fSAndrea Arcangeli if (ret) 2681ba76149fSAndrea Arcangeli /* we released mmap_sem so break loop */ 2682ba76149fSAndrea Arcangeli goto breakouterloop_mmap_sem; 2683ba76149fSAndrea Arcangeli if (progress >= pages) 2684ba76149fSAndrea Arcangeli goto breakouterloop; 2685ba76149fSAndrea Arcangeli } 2686ba76149fSAndrea Arcangeli } 2687ba76149fSAndrea Arcangeli breakouterloop: 2688ba76149fSAndrea Arcangeli up_read(&mm->mmap_sem); /* exit_mmap will destroy ptes after this */ 2689ba76149fSAndrea Arcangeli breakouterloop_mmap_sem: 2690ba76149fSAndrea Arcangeli 2691ba76149fSAndrea Arcangeli spin_lock(&khugepaged_mm_lock); 2692a7d6e4ecSAndrea Arcangeli VM_BUG_ON(khugepaged_scan.mm_slot != mm_slot); 2693ba76149fSAndrea Arcangeli /* 2694ba76149fSAndrea Arcangeli * Release the current mm_slot if this mm is about to die, or 2695ba76149fSAndrea Arcangeli * if we scanned all vmas of this mm. 2696ba76149fSAndrea Arcangeli */ 2697ba76149fSAndrea Arcangeli if (khugepaged_test_exit(mm) || !vma) { 2698ba76149fSAndrea Arcangeli /* 2699ba76149fSAndrea Arcangeli * Make sure that if mm_users is reaching zero while 2700ba76149fSAndrea Arcangeli * khugepaged runs here, khugepaged_exit will find 2701ba76149fSAndrea Arcangeli * mm_slot not pointing to the exiting mm. 2702ba76149fSAndrea Arcangeli */ 2703ba76149fSAndrea Arcangeli if (mm_slot->mm_node.next != &khugepaged_scan.mm_head) { 2704ba76149fSAndrea Arcangeli khugepaged_scan.mm_slot = list_entry( 2705ba76149fSAndrea Arcangeli mm_slot->mm_node.next, 2706ba76149fSAndrea Arcangeli struct mm_slot, mm_node); 2707ba76149fSAndrea Arcangeli khugepaged_scan.address = 0; 2708ba76149fSAndrea Arcangeli } else { 2709ba76149fSAndrea Arcangeli khugepaged_scan.mm_slot = NULL; 2710ba76149fSAndrea Arcangeli khugepaged_full_scans++; 2711ba76149fSAndrea Arcangeli } 2712ba76149fSAndrea Arcangeli 2713ba76149fSAndrea Arcangeli collect_mm_slot(mm_slot); 2714ba76149fSAndrea Arcangeli } 2715ba76149fSAndrea Arcangeli 2716ba76149fSAndrea Arcangeli return progress; 2717ba76149fSAndrea Arcangeli } 2718ba76149fSAndrea Arcangeli 2719ba76149fSAndrea Arcangeli static int khugepaged_has_work(void) 2720ba76149fSAndrea Arcangeli { 2721ba76149fSAndrea Arcangeli return !list_empty(&khugepaged_scan.mm_head) && 2722ba76149fSAndrea Arcangeli khugepaged_enabled(); 2723ba76149fSAndrea Arcangeli } 2724ba76149fSAndrea Arcangeli 2725ba76149fSAndrea Arcangeli static int khugepaged_wait_event(void) 2726ba76149fSAndrea Arcangeli { 2727ba76149fSAndrea Arcangeli return !list_empty(&khugepaged_scan.mm_head) || 27282017c0bfSXiao Guangrong kthread_should_stop(); 2729ba76149fSAndrea Arcangeli } 2730ba76149fSAndrea Arcangeli 2731d516904bSXiao Guangrong static void khugepaged_do_scan(void) 2732d516904bSXiao Guangrong { 2733d516904bSXiao Guangrong struct page *hpage = NULL; 2734ba76149fSAndrea Arcangeli unsigned int progress = 0, pass_through_head = 0; 2735ba76149fSAndrea Arcangeli unsigned int pages = khugepaged_pages_to_scan; 2736d516904bSXiao Guangrong bool wait = true; 2737ba76149fSAndrea Arcangeli 2738ba76149fSAndrea Arcangeli barrier(); /* write khugepaged_pages_to_scan to local stack */ 2739ba76149fSAndrea Arcangeli 2740ba76149fSAndrea Arcangeli while (progress < pages) { 274126234f36SXiao Guangrong if (!khugepaged_prealloc_page(&hpage, &wait)) 274226234f36SXiao Guangrong break; 2743d516904bSXiao Guangrong 2744420256efSXiao Guangrong cond_resched(); 2745ba76149fSAndrea Arcangeli 2746cd092411SJiri Kosina if (unlikely(kthread_should_stop() || try_to_freeze())) 2747878aee7dSAndrea Arcangeli break; 2748878aee7dSAndrea Arcangeli 2749ba76149fSAndrea Arcangeli spin_lock(&khugepaged_mm_lock); 2750ba76149fSAndrea Arcangeli if (!khugepaged_scan.mm_slot) 2751ba76149fSAndrea Arcangeli pass_through_head++; 2752ba76149fSAndrea Arcangeli if (khugepaged_has_work() && 2753ba76149fSAndrea Arcangeli pass_through_head < 2) 2754ba76149fSAndrea Arcangeli progress += khugepaged_scan_mm_slot(pages - progress, 2755d516904bSXiao Guangrong &hpage); 2756ba76149fSAndrea Arcangeli else 2757ba76149fSAndrea Arcangeli progress = pages; 2758ba76149fSAndrea Arcangeli spin_unlock(&khugepaged_mm_lock); 2759ba76149fSAndrea Arcangeli } 2760ba76149fSAndrea Arcangeli 2761d516904bSXiao Guangrong if (!IS_ERR_OR_NULL(hpage)) 2762d516904bSXiao Guangrong put_page(hpage); 2763ba76149fSAndrea Arcangeli } 27640bbbc0b3SAndrea Arcangeli 27652017c0bfSXiao Guangrong static void khugepaged_wait_work(void) 27662017c0bfSXiao Guangrong { 27672017c0bfSXiao Guangrong if (khugepaged_has_work()) { 27682017c0bfSXiao Guangrong if (!khugepaged_scan_sleep_millisecs) 27692017c0bfSXiao Guangrong return; 27702017c0bfSXiao Guangrong 27712017c0bfSXiao Guangrong wait_event_freezable_timeout(khugepaged_wait, 27722017c0bfSXiao Guangrong kthread_should_stop(), 27732017c0bfSXiao Guangrong msecs_to_jiffies(khugepaged_scan_sleep_millisecs)); 27742017c0bfSXiao Guangrong return; 27752017c0bfSXiao Guangrong } 27762017c0bfSXiao Guangrong 27772017c0bfSXiao Guangrong if (khugepaged_enabled()) 27782017c0bfSXiao Guangrong wait_event_freezable(khugepaged_wait, khugepaged_wait_event()); 27792017c0bfSXiao Guangrong } 27802017c0bfSXiao Guangrong 2781ba76149fSAndrea Arcangeli static int khugepaged(void *none) 2782ba76149fSAndrea Arcangeli { 2783ba76149fSAndrea Arcangeli struct mm_slot *mm_slot; 2784ba76149fSAndrea Arcangeli 2785878aee7dSAndrea Arcangeli set_freezable(); 27868698a745SDongsheng Yang set_user_nice(current, MAX_NICE); 2787ba76149fSAndrea Arcangeli 2788b7231789SXiao Guangrong while (!kthread_should_stop()) { 2789b7231789SXiao Guangrong khugepaged_do_scan(); 2790b7231789SXiao Guangrong khugepaged_wait_work(); 2791b7231789SXiao Guangrong } 2792ba76149fSAndrea Arcangeli 2793ba76149fSAndrea Arcangeli spin_lock(&khugepaged_mm_lock); 2794ba76149fSAndrea Arcangeli mm_slot = khugepaged_scan.mm_slot; 2795ba76149fSAndrea Arcangeli khugepaged_scan.mm_slot = NULL; 2796ba76149fSAndrea Arcangeli if (mm_slot) 2797ba76149fSAndrea Arcangeli collect_mm_slot(mm_slot); 2798ba76149fSAndrea Arcangeli spin_unlock(&khugepaged_mm_lock); 2799ba76149fSAndrea Arcangeli return 0; 2800ba76149fSAndrea Arcangeli } 2801ba76149fSAndrea Arcangeli 2802eef1b3baSKirill A. Shutemov static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, 2803eef1b3baSKirill A. Shutemov unsigned long haddr, pmd_t *pmd) 2804eef1b3baSKirill A. Shutemov { 2805eef1b3baSKirill A. Shutemov struct mm_struct *mm = vma->vm_mm; 2806eef1b3baSKirill A. Shutemov pgtable_t pgtable; 2807eef1b3baSKirill A. Shutemov pmd_t _pmd; 2808eef1b3baSKirill A. Shutemov int i; 2809eef1b3baSKirill A. Shutemov 2810eef1b3baSKirill A. Shutemov /* leave pmd empty until pte is filled */ 2811eef1b3baSKirill A. Shutemov pmdp_huge_clear_flush_notify(vma, haddr, pmd); 2812eef1b3baSKirill A. Shutemov 2813eef1b3baSKirill A. Shutemov pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2814eef1b3baSKirill A. Shutemov pmd_populate(mm, &_pmd, pgtable); 2815eef1b3baSKirill A. Shutemov 2816eef1b3baSKirill A. Shutemov for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) { 2817eef1b3baSKirill A. Shutemov pte_t *pte, entry; 2818eef1b3baSKirill A. Shutemov entry = pfn_pte(my_zero_pfn(haddr), vma->vm_page_prot); 2819eef1b3baSKirill A. Shutemov entry = pte_mkspecial(entry); 2820eef1b3baSKirill A. Shutemov pte = pte_offset_map(&_pmd, haddr); 2821eef1b3baSKirill A. Shutemov VM_BUG_ON(!pte_none(*pte)); 2822eef1b3baSKirill A. Shutemov set_pte_at(mm, haddr, pte, entry); 2823eef1b3baSKirill A. Shutemov pte_unmap(pte); 2824eef1b3baSKirill A. Shutemov } 2825eef1b3baSKirill A. Shutemov smp_wmb(); /* make pte visible before pmd */ 2826eef1b3baSKirill A. Shutemov pmd_populate(mm, pmd, pgtable); 2827eef1b3baSKirill A. Shutemov put_huge_zero_page(); 2828eef1b3baSKirill A. Shutemov } 2829eef1b3baSKirill A. Shutemov 2830eef1b3baSKirill A. Shutemov static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, 2831ba988280SKirill A. Shutemov unsigned long haddr, bool freeze) 2832eef1b3baSKirill A. Shutemov { 2833eef1b3baSKirill A. Shutemov struct mm_struct *mm = vma->vm_mm; 2834eef1b3baSKirill A. Shutemov struct page *page; 2835eef1b3baSKirill A. Shutemov pgtable_t pgtable; 2836eef1b3baSKirill A. Shutemov pmd_t _pmd; 2837b8d3c4c3SMinchan Kim bool young, write, dirty; 2838eef1b3baSKirill A. Shutemov int i; 2839eef1b3baSKirill A. Shutemov 2840eef1b3baSKirill A. Shutemov VM_BUG_ON(haddr & ~HPAGE_PMD_MASK); 2841eef1b3baSKirill A. Shutemov VM_BUG_ON_VMA(vma->vm_start > haddr, vma); 2842eef1b3baSKirill A. Shutemov VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma); 28435c7fb56eSDan Williams VM_BUG_ON(!pmd_trans_huge(*pmd) && !pmd_devmap(*pmd)); 2844eef1b3baSKirill A. Shutemov 2845eef1b3baSKirill A. Shutemov count_vm_event(THP_SPLIT_PMD); 2846eef1b3baSKirill A. Shutemov 2847eef1b3baSKirill A. Shutemov if (vma_is_dax(vma)) { 2848eef1b3baSKirill A. Shutemov pmd_t _pmd = pmdp_huge_clear_flush_notify(vma, haddr, pmd); 2849eef1b3baSKirill A. Shutemov if (is_huge_zero_pmd(_pmd)) 2850eef1b3baSKirill A. Shutemov put_huge_zero_page(); 2851eef1b3baSKirill A. Shutemov return; 2852eef1b3baSKirill A. Shutemov } else if (is_huge_zero_pmd(*pmd)) { 2853eef1b3baSKirill A. Shutemov return __split_huge_zero_page_pmd(vma, haddr, pmd); 2854eef1b3baSKirill A. Shutemov } 2855eef1b3baSKirill A. Shutemov 2856eef1b3baSKirill A. Shutemov page = pmd_page(*pmd); 2857eef1b3baSKirill A. Shutemov VM_BUG_ON_PAGE(!page_count(page), page); 2858eef1b3baSKirill A. Shutemov atomic_add(HPAGE_PMD_NR - 1, &page->_count); 2859eef1b3baSKirill A. Shutemov write = pmd_write(*pmd); 2860eef1b3baSKirill A. Shutemov young = pmd_young(*pmd); 2861b8d3c4c3SMinchan Kim dirty = pmd_dirty(*pmd); 2862eef1b3baSKirill A. Shutemov 2863eef1b3baSKirill A. Shutemov pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2864eef1b3baSKirill A. Shutemov pmd_populate(mm, &_pmd, pgtable); 2865eef1b3baSKirill A. Shutemov 2866eef1b3baSKirill A. Shutemov for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) { 2867eef1b3baSKirill A. Shutemov pte_t entry, *pte; 2868eef1b3baSKirill A. Shutemov /* 2869eef1b3baSKirill A. Shutemov * Note that NUMA hinting access restrictions are not 2870eef1b3baSKirill A. Shutemov * transferred to avoid any possibility of altering 2871eef1b3baSKirill A. Shutemov * permissions across VMAs. 2872eef1b3baSKirill A. Shutemov */ 2873ba988280SKirill A. Shutemov if (freeze) { 2874ba988280SKirill A. Shutemov swp_entry_t swp_entry; 2875ba988280SKirill A. Shutemov swp_entry = make_migration_entry(page + i, write); 2876ba988280SKirill A. Shutemov entry = swp_entry_to_pte(swp_entry); 2877ba988280SKirill A. Shutemov } else { 2878eef1b3baSKirill A. Shutemov entry = mk_pte(page + i, vma->vm_page_prot); 2879b8d3c4c3SMinchan Kim entry = maybe_mkwrite(entry, vma); 2880eef1b3baSKirill A. Shutemov if (!write) 2881eef1b3baSKirill A. Shutemov entry = pte_wrprotect(entry); 2882eef1b3baSKirill A. Shutemov if (!young) 2883eef1b3baSKirill A. Shutemov entry = pte_mkold(entry); 2884ba988280SKirill A. Shutemov } 2885b8d3c4c3SMinchan Kim if (dirty) 2886b8d3c4c3SMinchan Kim SetPageDirty(page + i); 2887eef1b3baSKirill A. Shutemov pte = pte_offset_map(&_pmd, haddr); 2888eef1b3baSKirill A. Shutemov BUG_ON(!pte_none(*pte)); 2889eef1b3baSKirill A. Shutemov set_pte_at(mm, haddr, pte, entry); 2890eef1b3baSKirill A. Shutemov atomic_inc(&page[i]._mapcount); 2891eef1b3baSKirill A. Shutemov pte_unmap(pte); 2892eef1b3baSKirill A. Shutemov } 2893eef1b3baSKirill A. Shutemov 2894eef1b3baSKirill A. Shutemov /* 2895eef1b3baSKirill A. Shutemov * Set PG_double_map before dropping compound_mapcount to avoid 2896eef1b3baSKirill A. Shutemov * false-negative page_mapped(). 2897eef1b3baSKirill A. Shutemov */ 2898eef1b3baSKirill A. Shutemov if (compound_mapcount(page) > 1 && !TestSetPageDoubleMap(page)) { 2899eef1b3baSKirill A. Shutemov for (i = 0; i < HPAGE_PMD_NR; i++) 2900eef1b3baSKirill A. Shutemov atomic_inc(&page[i]._mapcount); 2901eef1b3baSKirill A. Shutemov } 2902eef1b3baSKirill A. Shutemov 2903eef1b3baSKirill A. Shutemov if (atomic_add_negative(-1, compound_mapcount_ptr(page))) { 2904eef1b3baSKirill A. Shutemov /* Last compound_mapcount is gone. */ 2905eef1b3baSKirill A. Shutemov __dec_zone_page_state(page, NR_ANON_TRANSPARENT_HUGEPAGES); 2906eef1b3baSKirill A. Shutemov if (TestClearPageDoubleMap(page)) { 2907eef1b3baSKirill A. Shutemov /* No need in mapcount reference anymore */ 2908eef1b3baSKirill A. Shutemov for (i = 0; i < HPAGE_PMD_NR; i++) 2909eef1b3baSKirill A. Shutemov atomic_dec(&page[i]._mapcount); 2910eef1b3baSKirill A. Shutemov } 2911eef1b3baSKirill A. Shutemov } 2912eef1b3baSKirill A. Shutemov 2913eef1b3baSKirill A. Shutemov smp_wmb(); /* make pte visible before pmd */ 2914e9b61f19SKirill A. Shutemov /* 2915e9b61f19SKirill A. Shutemov * Up to this point the pmd is present and huge and userland has the 2916e9b61f19SKirill A. Shutemov * whole access to the hugepage during the split (which happens in 2917e9b61f19SKirill A. Shutemov * place). If we overwrite the pmd with the not-huge version pointing 2918e9b61f19SKirill A. Shutemov * to the pte here (which of course we could if all CPUs were bug 2919e9b61f19SKirill A. Shutemov * free), userland could trigger a small page size TLB miss on the 2920e9b61f19SKirill A. Shutemov * small sized TLB while the hugepage TLB entry is still established in 2921e9b61f19SKirill A. Shutemov * the huge TLB. Some CPU doesn't like that. 2922e9b61f19SKirill A. Shutemov * See http://support.amd.com/us/Processor_TechDocs/41322.pdf, Erratum 2923e9b61f19SKirill A. Shutemov * 383 on page 93. Intel should be safe but is also warns that it's 2924e9b61f19SKirill A. Shutemov * only safe if the permission and cache attributes of the two entries 2925e9b61f19SKirill A. Shutemov * loaded in the two TLB is identical (which should be the case here). 2926e9b61f19SKirill A. Shutemov * But it is generally safer to never allow small and huge TLB entries 2927e9b61f19SKirill A. Shutemov * for the same virtual address to be loaded simultaneously. So instead 2928e9b61f19SKirill A. Shutemov * of doing "pmd_populate(); flush_pmd_tlb_range();" we first mark the 2929e9b61f19SKirill A. Shutemov * current pmd notpresent (atomically because here the pmd_trans_huge 2930e9b61f19SKirill A. Shutemov * and pmd_trans_splitting must remain set at all times on the pmd 2931e9b61f19SKirill A. Shutemov * until the split is complete for this pmd), then we flush the SMP TLB 2932e9b61f19SKirill A. Shutemov * and finally we write the non-huge version of the pmd entry with 2933e9b61f19SKirill A. Shutemov * pmd_populate. 2934e9b61f19SKirill A. Shutemov */ 2935e9b61f19SKirill A. Shutemov pmdp_invalidate(vma, haddr, pmd); 2936eef1b3baSKirill A. Shutemov pmd_populate(mm, pmd, pgtable); 2937e9b61f19SKirill A. Shutemov 2938e9b61f19SKirill A. Shutemov if (freeze) { 2939e9b61f19SKirill A. Shutemov for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) { 2940e9b61f19SKirill A. Shutemov page_remove_rmap(page + i, false); 2941e9b61f19SKirill A. Shutemov put_page(page + i); 2942e9b61f19SKirill A. Shutemov } 2943e9b61f19SKirill A. Shutemov } 2944eef1b3baSKirill A. Shutemov } 2945eef1b3baSKirill A. Shutemov 2946eef1b3baSKirill A. Shutemov void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, 2947eef1b3baSKirill A. Shutemov unsigned long address) 2948eef1b3baSKirill A. Shutemov { 2949eef1b3baSKirill A. Shutemov spinlock_t *ptl; 2950eef1b3baSKirill A. Shutemov struct mm_struct *mm = vma->vm_mm; 2951e90309c9SKirill A. Shutemov struct page *page = NULL; 2952eef1b3baSKirill A. Shutemov unsigned long haddr = address & HPAGE_PMD_MASK; 2953eef1b3baSKirill A. Shutemov 2954eef1b3baSKirill A. Shutemov mmu_notifier_invalidate_range_start(mm, haddr, haddr + HPAGE_PMD_SIZE); 2955eef1b3baSKirill A. Shutemov ptl = pmd_lock(mm, pmd); 29565c7fb56eSDan Williams if (pmd_trans_huge(*pmd)) { 2957e90309c9SKirill A. Shutemov page = pmd_page(*pmd); 2958e90309c9SKirill A. Shutemov if (PageMlocked(page)) 2959e90309c9SKirill A. Shutemov get_page(page); 2960e90309c9SKirill A. Shutemov else 2961e90309c9SKirill A. Shutemov page = NULL; 29625c7fb56eSDan Williams } else if (!pmd_devmap(*pmd)) 29635c7fb56eSDan Williams goto out; 29645c7fb56eSDan Williams __split_huge_pmd_locked(vma, pmd, haddr, false); 2965e90309c9SKirill A. Shutemov out: 2966eef1b3baSKirill A. Shutemov spin_unlock(ptl); 2967eef1b3baSKirill A. Shutemov mmu_notifier_invalidate_range_end(mm, haddr, haddr + HPAGE_PMD_SIZE); 2968e90309c9SKirill A. Shutemov if (page) { 2969e90309c9SKirill A. Shutemov lock_page(page); 2970e90309c9SKirill A. Shutemov munlock_vma_page(page); 2971e90309c9SKirill A. Shutemov unlock_page(page); 2972e90309c9SKirill A. Shutemov put_page(page); 2973e90309c9SKirill A. Shutemov } 2974eef1b3baSKirill A. Shutemov } 2975eef1b3baSKirill A. Shutemov 297678ddc534SKirill A. Shutemov static void split_huge_pmd_address(struct vm_area_struct *vma, 297794fcc585SAndrea Arcangeli unsigned long address) 297894fcc585SAndrea Arcangeli { 2979f72e7dcdSHugh Dickins pgd_t *pgd; 2980f72e7dcdSHugh Dickins pud_t *pud; 298194fcc585SAndrea Arcangeli pmd_t *pmd; 298294fcc585SAndrea Arcangeli 298394fcc585SAndrea Arcangeli VM_BUG_ON(!(address & ~HPAGE_PMD_MASK)); 298494fcc585SAndrea Arcangeli 298578ddc534SKirill A. Shutemov pgd = pgd_offset(vma->vm_mm, address); 2986f72e7dcdSHugh Dickins if (!pgd_present(*pgd)) 2987f72e7dcdSHugh Dickins return; 2988f72e7dcdSHugh Dickins 2989f72e7dcdSHugh Dickins pud = pud_offset(pgd, address); 2990f72e7dcdSHugh Dickins if (!pud_present(*pud)) 2991f72e7dcdSHugh Dickins return; 2992f72e7dcdSHugh Dickins 2993f72e7dcdSHugh Dickins pmd = pmd_offset(pud, address); 29945c7fb56eSDan Williams if (!pmd_present(*pmd) || (!pmd_trans_huge(*pmd) && !pmd_devmap(*pmd))) 299594fcc585SAndrea Arcangeli return; 299694fcc585SAndrea Arcangeli /* 299794fcc585SAndrea Arcangeli * Caller holds the mmap_sem write mode, so a huge pmd cannot 299894fcc585SAndrea Arcangeli * materialize from under us. 299994fcc585SAndrea Arcangeli */ 3000ad0bed24SKirill A. Shutemov split_huge_pmd(vma, pmd, address); 300194fcc585SAndrea Arcangeli } 300294fcc585SAndrea Arcangeli 3003e1b9996bSKirill A. Shutemov void vma_adjust_trans_huge(struct vm_area_struct *vma, 300494fcc585SAndrea Arcangeli unsigned long start, 300594fcc585SAndrea Arcangeli unsigned long end, 300694fcc585SAndrea Arcangeli long adjust_next) 300794fcc585SAndrea Arcangeli { 300894fcc585SAndrea Arcangeli /* 300994fcc585SAndrea Arcangeli * If the new start address isn't hpage aligned and it could 301094fcc585SAndrea Arcangeli * previously contain an hugepage: check if we need to split 301194fcc585SAndrea Arcangeli * an huge pmd. 301294fcc585SAndrea Arcangeli */ 301394fcc585SAndrea Arcangeli if (start & ~HPAGE_PMD_MASK && 301494fcc585SAndrea Arcangeli (start & HPAGE_PMD_MASK) >= vma->vm_start && 301594fcc585SAndrea Arcangeli (start & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end) 301678ddc534SKirill A. Shutemov split_huge_pmd_address(vma, start); 301794fcc585SAndrea Arcangeli 301894fcc585SAndrea Arcangeli /* 301994fcc585SAndrea Arcangeli * If the new end address isn't hpage aligned and it could 302094fcc585SAndrea Arcangeli * previously contain an hugepage: check if we need to split 302194fcc585SAndrea Arcangeli * an huge pmd. 302294fcc585SAndrea Arcangeli */ 302394fcc585SAndrea Arcangeli if (end & ~HPAGE_PMD_MASK && 302494fcc585SAndrea Arcangeli (end & HPAGE_PMD_MASK) >= vma->vm_start && 302594fcc585SAndrea Arcangeli (end & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end) 302678ddc534SKirill A. Shutemov split_huge_pmd_address(vma, end); 302794fcc585SAndrea Arcangeli 302894fcc585SAndrea Arcangeli /* 302994fcc585SAndrea Arcangeli * If we're also updating the vma->vm_next->vm_start, if the new 303094fcc585SAndrea Arcangeli * vm_next->vm_start isn't page aligned and it could previously 303194fcc585SAndrea Arcangeli * contain an hugepage: check if we need to split an huge pmd. 303294fcc585SAndrea Arcangeli */ 303394fcc585SAndrea Arcangeli if (adjust_next > 0) { 303494fcc585SAndrea Arcangeli struct vm_area_struct *next = vma->vm_next; 303594fcc585SAndrea Arcangeli unsigned long nstart = next->vm_start; 303694fcc585SAndrea Arcangeli nstart += adjust_next << PAGE_SHIFT; 303794fcc585SAndrea Arcangeli if (nstart & ~HPAGE_PMD_MASK && 303894fcc585SAndrea Arcangeli (nstart & HPAGE_PMD_MASK) >= next->vm_start && 303994fcc585SAndrea Arcangeli (nstart & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= next->vm_end) 304078ddc534SKirill A. Shutemov split_huge_pmd_address(next, nstart); 304194fcc585SAndrea Arcangeli } 304294fcc585SAndrea Arcangeli } 3043e9b61f19SKirill A. Shutemov 3044e9b61f19SKirill A. Shutemov static void freeze_page_vma(struct vm_area_struct *vma, struct page *page, 3045e9b61f19SKirill A. Shutemov unsigned long address) 3046e9b61f19SKirill A. Shutemov { 3047bd56086fSKirill A. Shutemov unsigned long haddr = address & HPAGE_PMD_MASK; 3048e9b61f19SKirill A. Shutemov spinlock_t *ptl; 3049e9b61f19SKirill A. Shutemov pgd_t *pgd; 3050e9b61f19SKirill A. Shutemov pud_t *pud; 3051e9b61f19SKirill A. Shutemov pmd_t *pmd; 3052e9b61f19SKirill A. Shutemov pte_t *pte; 3053e9b61f19SKirill A. Shutemov int i, nr = HPAGE_PMD_NR; 3054e9b61f19SKirill A. Shutemov 3055e9b61f19SKirill A. Shutemov /* Skip pages which doesn't belong to the VMA */ 3056e9b61f19SKirill A. Shutemov if (address < vma->vm_start) { 3057e9b61f19SKirill A. Shutemov int off = (vma->vm_start - address) >> PAGE_SHIFT; 3058e9b61f19SKirill A. Shutemov page += off; 3059e9b61f19SKirill A. Shutemov nr -= off; 3060e9b61f19SKirill A. Shutemov address = vma->vm_start; 3061e9b61f19SKirill A. Shutemov } 3062e9b61f19SKirill A. Shutemov 3063e9b61f19SKirill A. Shutemov pgd = pgd_offset(vma->vm_mm, address); 3064e9b61f19SKirill A. Shutemov if (!pgd_present(*pgd)) 3065e9b61f19SKirill A. Shutemov return; 3066e9b61f19SKirill A. Shutemov pud = pud_offset(pgd, address); 3067e9b61f19SKirill A. Shutemov if (!pud_present(*pud)) 3068e9b61f19SKirill A. Shutemov return; 3069e9b61f19SKirill A. Shutemov pmd = pmd_offset(pud, address); 3070e9b61f19SKirill A. Shutemov ptl = pmd_lock(vma->vm_mm, pmd); 3071e9b61f19SKirill A. Shutemov if (!pmd_present(*pmd)) { 3072e9b61f19SKirill A. Shutemov spin_unlock(ptl); 3073e9b61f19SKirill A. Shutemov return; 3074e9b61f19SKirill A. Shutemov } 3075e9b61f19SKirill A. Shutemov if (pmd_trans_huge(*pmd)) { 3076e9b61f19SKirill A. Shutemov if (page == pmd_page(*pmd)) 3077bd56086fSKirill A. Shutemov __split_huge_pmd_locked(vma, pmd, haddr, true); 3078e9b61f19SKirill A. Shutemov spin_unlock(ptl); 3079e9b61f19SKirill A. Shutemov return; 3080e9b61f19SKirill A. Shutemov } 3081e9b61f19SKirill A. Shutemov spin_unlock(ptl); 3082e9b61f19SKirill A. Shutemov 3083e9b61f19SKirill A. Shutemov pte = pte_offset_map_lock(vma->vm_mm, pmd, address, &ptl); 3084bd56086fSKirill A. Shutemov for (i = 0; i < nr; i++, address += PAGE_SIZE, page++, pte++) { 3085e9b61f19SKirill A. Shutemov pte_t entry, swp_pte; 3086e9b61f19SKirill A. Shutemov swp_entry_t swp_entry; 3087e9b61f19SKirill A. Shutemov 3088bd56086fSKirill A. Shutemov /* 3089bd56086fSKirill A. Shutemov * We've just crossed page table boundary: need to map next one. 3090bd56086fSKirill A. Shutemov * It can happen if THP was mremaped to non PMD-aligned address. 3091bd56086fSKirill A. Shutemov */ 3092bd56086fSKirill A. Shutemov if (unlikely(address == haddr + HPAGE_PMD_SIZE)) { 3093bd56086fSKirill A. Shutemov pte_unmap_unlock(pte - 1, ptl); 3094bd56086fSKirill A. Shutemov pmd = mm_find_pmd(vma->vm_mm, address); 3095bd56086fSKirill A. Shutemov if (!pmd) 3096bd56086fSKirill A. Shutemov return; 3097bd56086fSKirill A. Shutemov pte = pte_offset_map_lock(vma->vm_mm, pmd, 3098bd56086fSKirill A. Shutemov address, &ptl); 3099bd56086fSKirill A. Shutemov } 3100bd56086fSKirill A. Shutemov 3101bd56086fSKirill A. Shutemov if (!pte_present(*pte)) 3102e9b61f19SKirill A. Shutemov continue; 3103bd56086fSKirill A. Shutemov if (page_to_pfn(page) != pte_pfn(*pte)) 3104e9b61f19SKirill A. Shutemov continue; 3105e9b61f19SKirill A. Shutemov flush_cache_page(vma, address, page_to_pfn(page)); 3106bd56086fSKirill A. Shutemov entry = ptep_clear_flush(vma, address, pte); 3107b8d3c4c3SMinchan Kim if (pte_dirty(entry)) 3108b8d3c4c3SMinchan Kim SetPageDirty(page); 3109e9b61f19SKirill A. Shutemov swp_entry = make_migration_entry(page, pte_write(entry)); 3110e9b61f19SKirill A. Shutemov swp_pte = swp_entry_to_pte(swp_entry); 3111e9b61f19SKirill A. Shutemov if (pte_soft_dirty(entry)) 3112e9b61f19SKirill A. Shutemov swp_pte = pte_swp_mksoft_dirty(swp_pte); 3113bd56086fSKirill A. Shutemov set_pte_at(vma->vm_mm, address, pte, swp_pte); 3114e9b61f19SKirill A. Shutemov page_remove_rmap(page, false); 3115e9b61f19SKirill A. Shutemov put_page(page); 3116e9b61f19SKirill A. Shutemov } 3117bd56086fSKirill A. Shutemov pte_unmap_unlock(pte - 1, ptl); 3118e9b61f19SKirill A. Shutemov } 3119e9b61f19SKirill A. Shutemov 3120e9b61f19SKirill A. Shutemov static void freeze_page(struct anon_vma *anon_vma, struct page *page) 3121e9b61f19SKirill A. Shutemov { 3122e9b61f19SKirill A. Shutemov struct anon_vma_chain *avc; 3123e9b61f19SKirill A. Shutemov pgoff_t pgoff = page_to_pgoff(page); 3124e9b61f19SKirill A. Shutemov 3125e9b61f19SKirill A. Shutemov VM_BUG_ON_PAGE(!PageHead(page), page); 3126e9b61f19SKirill A. Shutemov 3127e9b61f19SKirill A. Shutemov anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, pgoff, 3128e9b61f19SKirill A. Shutemov pgoff + HPAGE_PMD_NR - 1) { 3129bd56086fSKirill A. Shutemov unsigned long address = __vma_address(page, avc->vma); 3130e9b61f19SKirill A. Shutemov 3131e9b61f19SKirill A. Shutemov mmu_notifier_invalidate_range_start(avc->vma->vm_mm, 3132bd56086fSKirill A. Shutemov address, address + HPAGE_PMD_SIZE); 3133bd56086fSKirill A. Shutemov freeze_page_vma(avc->vma, page, address); 3134e9b61f19SKirill A. Shutemov mmu_notifier_invalidate_range_end(avc->vma->vm_mm, 3135bd56086fSKirill A. Shutemov address, address + HPAGE_PMD_SIZE); 3136e9b61f19SKirill A. Shutemov } 3137e9b61f19SKirill A. Shutemov } 3138e9b61f19SKirill A. Shutemov 3139e9b61f19SKirill A. Shutemov static void unfreeze_page_vma(struct vm_area_struct *vma, struct page *page, 3140e9b61f19SKirill A. Shutemov unsigned long address) 3141e9b61f19SKirill A. Shutemov { 3142e9b61f19SKirill A. Shutemov spinlock_t *ptl; 3143e9b61f19SKirill A. Shutemov pmd_t *pmd; 3144e9b61f19SKirill A. Shutemov pte_t *pte, entry; 3145e9b61f19SKirill A. Shutemov swp_entry_t swp_entry; 3146bd56086fSKirill A. Shutemov unsigned long haddr = address & HPAGE_PMD_MASK; 3147e9b61f19SKirill A. Shutemov int i, nr = HPAGE_PMD_NR; 3148e9b61f19SKirill A. Shutemov 3149e9b61f19SKirill A. Shutemov /* Skip pages which doesn't belong to the VMA */ 3150e9b61f19SKirill A. Shutemov if (address < vma->vm_start) { 3151e9b61f19SKirill A. Shutemov int off = (vma->vm_start - address) >> PAGE_SHIFT; 3152e9b61f19SKirill A. Shutemov page += off; 3153e9b61f19SKirill A. Shutemov nr -= off; 3154e9b61f19SKirill A. Shutemov address = vma->vm_start; 3155e9b61f19SKirill A. Shutemov } 3156e9b61f19SKirill A. Shutemov 3157e9b61f19SKirill A. Shutemov pmd = mm_find_pmd(vma->vm_mm, address); 3158e9b61f19SKirill A. Shutemov if (!pmd) 3159e9b61f19SKirill A. Shutemov return; 3160bd56086fSKirill A. Shutemov 3161e9b61f19SKirill A. Shutemov pte = pte_offset_map_lock(vma->vm_mm, pmd, address, &ptl); 3162bd56086fSKirill A. Shutemov for (i = 0; i < nr; i++, address += PAGE_SIZE, page++, pte++) { 3163bd56086fSKirill A. Shutemov /* 3164bd56086fSKirill A. Shutemov * We've just crossed page table boundary: need to map next one. 3165bd56086fSKirill A. Shutemov * It can happen if THP was mremaped to non-PMD aligned address. 3166bd56086fSKirill A. Shutemov */ 3167bd56086fSKirill A. Shutemov if (unlikely(address == haddr + HPAGE_PMD_SIZE)) { 3168bd56086fSKirill A. Shutemov pte_unmap_unlock(pte - 1, ptl); 3169bd56086fSKirill A. Shutemov pmd = mm_find_pmd(vma->vm_mm, address); 3170bd56086fSKirill A. Shutemov if (!pmd) 3171bd56086fSKirill A. Shutemov return; 3172bd56086fSKirill A. Shutemov pte = pte_offset_map_lock(vma->vm_mm, pmd, 3173bd56086fSKirill A. Shutemov address, &ptl); 3174bd56086fSKirill A. Shutemov } 3175bd56086fSKirill A. Shutemov 3176bd56086fSKirill A. Shutemov if (!is_swap_pte(*pte)) 3177e9b61f19SKirill A. Shutemov continue; 3178e9b61f19SKirill A. Shutemov 3179bd56086fSKirill A. Shutemov swp_entry = pte_to_swp_entry(*pte); 3180e9b61f19SKirill A. Shutemov if (!is_migration_entry(swp_entry)) 3181e9b61f19SKirill A. Shutemov continue; 3182e9b61f19SKirill A. Shutemov if (migration_entry_to_page(swp_entry) != page) 3183e9b61f19SKirill A. Shutemov continue; 3184e9b61f19SKirill A. Shutemov 3185e9b61f19SKirill A. Shutemov get_page(page); 3186e9b61f19SKirill A. Shutemov page_add_anon_rmap(page, vma, address, false); 3187e9b61f19SKirill A. Shutemov 3188e9b61f19SKirill A. Shutemov entry = pte_mkold(mk_pte(page, vma->vm_page_prot)); 3189b8d3c4c3SMinchan Kim if (PageDirty(page)) 3190e9b61f19SKirill A. Shutemov entry = pte_mkdirty(entry); 3191e9b61f19SKirill A. Shutemov if (is_write_migration_entry(swp_entry)) 3192e9b61f19SKirill A. Shutemov entry = maybe_mkwrite(entry, vma); 3193e9b61f19SKirill A. Shutemov 3194e9b61f19SKirill A. Shutemov flush_dcache_page(page); 3195bd56086fSKirill A. Shutemov set_pte_at(vma->vm_mm, address, pte, entry); 3196e9b61f19SKirill A. Shutemov 3197e9b61f19SKirill A. Shutemov /* No need to invalidate - it was non-present before */ 3198bd56086fSKirill A. Shutemov update_mmu_cache(vma, address, pte); 3199e9b61f19SKirill A. Shutemov } 3200bd56086fSKirill A. Shutemov pte_unmap_unlock(pte - 1, ptl); 3201e9b61f19SKirill A. Shutemov } 3202e9b61f19SKirill A. Shutemov 3203e9b61f19SKirill A. Shutemov static void unfreeze_page(struct anon_vma *anon_vma, struct page *page) 3204e9b61f19SKirill A. Shutemov { 3205e9b61f19SKirill A. Shutemov struct anon_vma_chain *avc; 3206e9b61f19SKirill A. Shutemov pgoff_t pgoff = page_to_pgoff(page); 3207e9b61f19SKirill A. Shutemov 3208e9b61f19SKirill A. Shutemov anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, 3209e9b61f19SKirill A. Shutemov pgoff, pgoff + HPAGE_PMD_NR - 1) { 3210e9b61f19SKirill A. Shutemov unsigned long address = __vma_address(page, avc->vma); 3211e9b61f19SKirill A. Shutemov 3212e9b61f19SKirill A. Shutemov mmu_notifier_invalidate_range_start(avc->vma->vm_mm, 3213e9b61f19SKirill A. Shutemov address, address + HPAGE_PMD_SIZE); 3214e9b61f19SKirill A. Shutemov unfreeze_page_vma(avc->vma, page, address); 3215e9b61f19SKirill A. Shutemov mmu_notifier_invalidate_range_end(avc->vma->vm_mm, 3216e9b61f19SKirill A. Shutemov address, address + HPAGE_PMD_SIZE); 3217e9b61f19SKirill A. Shutemov } 3218e9b61f19SKirill A. Shutemov } 3219e9b61f19SKirill A. Shutemov 3220e9b61f19SKirill A. Shutemov static int __split_huge_page_tail(struct page *head, int tail, 3221e9b61f19SKirill A. Shutemov struct lruvec *lruvec, struct list_head *list) 3222e9b61f19SKirill A. Shutemov { 3223e9b61f19SKirill A. Shutemov int mapcount; 3224e9b61f19SKirill A. Shutemov struct page *page_tail = head + tail; 3225e9b61f19SKirill A. Shutemov 3226e9b61f19SKirill A. Shutemov mapcount = atomic_read(&page_tail->_mapcount) + 1; 3227e9b61f19SKirill A. Shutemov VM_BUG_ON_PAGE(atomic_read(&page_tail->_count) != 0, page_tail); 3228e9b61f19SKirill A. Shutemov 3229e9b61f19SKirill A. Shutemov /* 3230e9b61f19SKirill A. Shutemov * tail_page->_count is zero and not changing from under us. But 3231e9b61f19SKirill A. Shutemov * get_page_unless_zero() may be running from under us on the 3232e9b61f19SKirill A. Shutemov * tail_page. If we used atomic_set() below instead of atomic_add(), we 3233e9b61f19SKirill A. Shutemov * would then run atomic_set() concurrently with 3234e9b61f19SKirill A. Shutemov * get_page_unless_zero(), and atomic_set() is implemented in C not 3235e9b61f19SKirill A. Shutemov * using locked ops. spin_unlock on x86 sometime uses locked ops 3236e9b61f19SKirill A. Shutemov * because of PPro errata 66, 92, so unless somebody can guarantee 3237e9b61f19SKirill A. Shutemov * atomic_set() here would be safe on all archs (and not only on x86), 3238e9b61f19SKirill A. Shutemov * it's safer to use atomic_add(). 3239e9b61f19SKirill A. Shutemov */ 3240e9b61f19SKirill A. Shutemov atomic_add(mapcount + 1, &page_tail->_count); 3241e9b61f19SKirill A. Shutemov 3242e9b61f19SKirill A. Shutemov 3243e9b61f19SKirill A. Shutemov page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 3244e9b61f19SKirill A. Shutemov page_tail->flags |= (head->flags & 3245e9b61f19SKirill A. Shutemov ((1L << PG_referenced) | 3246e9b61f19SKirill A. Shutemov (1L << PG_swapbacked) | 3247e9b61f19SKirill A. Shutemov (1L << PG_mlocked) | 3248e9b61f19SKirill A. Shutemov (1L << PG_uptodate) | 3249e9b61f19SKirill A. Shutemov (1L << PG_active) | 3250e9b61f19SKirill A. Shutemov (1L << PG_locked) | 3251b8d3c4c3SMinchan Kim (1L << PG_unevictable) | 3252b8d3c4c3SMinchan Kim (1L << PG_dirty))); 3253e9b61f19SKirill A. Shutemov 3254e9b61f19SKirill A. Shutemov /* 3255e9b61f19SKirill A. Shutemov * After clearing PageTail the gup refcount can be released. 3256e9b61f19SKirill A. Shutemov * Page flags also must be visible before we make the page non-compound. 3257e9b61f19SKirill A. Shutemov */ 3258e9b61f19SKirill A. Shutemov smp_wmb(); 3259e9b61f19SKirill A. Shutemov 3260e9b61f19SKirill A. Shutemov clear_compound_head(page_tail); 3261e9b61f19SKirill A. Shutemov 3262e9b61f19SKirill A. Shutemov if (page_is_young(head)) 3263e9b61f19SKirill A. Shutemov set_page_young(page_tail); 3264e9b61f19SKirill A. Shutemov if (page_is_idle(head)) 3265e9b61f19SKirill A. Shutemov set_page_idle(page_tail); 3266e9b61f19SKirill A. Shutemov 3267e9b61f19SKirill A. Shutemov /* ->mapping in first tail page is compound_mapcount */ 32689a982250SKirill A. Shutemov VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING, 3269e9b61f19SKirill A. Shutemov page_tail); 3270e9b61f19SKirill A. Shutemov page_tail->mapping = head->mapping; 3271e9b61f19SKirill A. Shutemov 3272e9b61f19SKirill A. Shutemov page_tail->index = head->index + tail; 3273e9b61f19SKirill A. Shutemov page_cpupid_xchg_last(page_tail, page_cpupid_last(head)); 3274e9b61f19SKirill A. Shutemov lru_add_page_tail(head, page_tail, lruvec, list); 3275e9b61f19SKirill A. Shutemov 3276e9b61f19SKirill A. Shutemov return mapcount; 3277e9b61f19SKirill A. Shutemov } 3278e9b61f19SKirill A. Shutemov 3279e9b61f19SKirill A. Shutemov static void __split_huge_page(struct page *page, struct list_head *list) 3280e9b61f19SKirill A. Shutemov { 3281e9b61f19SKirill A. Shutemov struct page *head = compound_head(page); 3282e9b61f19SKirill A. Shutemov struct zone *zone = page_zone(head); 3283e9b61f19SKirill A. Shutemov struct lruvec *lruvec; 3284e9b61f19SKirill A. Shutemov int i, tail_mapcount; 3285e9b61f19SKirill A. Shutemov 3286e9b61f19SKirill A. Shutemov /* prevent PageLRU to go away from under us, and freeze lru stats */ 3287e9b61f19SKirill A. Shutemov spin_lock_irq(&zone->lru_lock); 3288e9b61f19SKirill A. Shutemov lruvec = mem_cgroup_page_lruvec(head, zone); 3289e9b61f19SKirill A. Shutemov 3290e9b61f19SKirill A. Shutemov /* complete memcg works before add pages to LRU */ 3291e9b61f19SKirill A. Shutemov mem_cgroup_split_huge_fixup(head); 3292e9b61f19SKirill A. Shutemov 3293e9b61f19SKirill A. Shutemov tail_mapcount = 0; 3294e9b61f19SKirill A. Shutemov for (i = HPAGE_PMD_NR - 1; i >= 1; i--) 3295e9b61f19SKirill A. Shutemov tail_mapcount += __split_huge_page_tail(head, i, lruvec, list); 3296e9b61f19SKirill A. Shutemov atomic_sub(tail_mapcount, &head->_count); 3297e9b61f19SKirill A. Shutemov 3298e9b61f19SKirill A. Shutemov ClearPageCompound(head); 3299e9b61f19SKirill A. Shutemov spin_unlock_irq(&zone->lru_lock); 3300e9b61f19SKirill A. Shutemov 3301e9b61f19SKirill A. Shutemov unfreeze_page(page_anon_vma(head), head); 3302e9b61f19SKirill A. Shutemov 3303e9b61f19SKirill A. Shutemov for (i = 0; i < HPAGE_PMD_NR; i++) { 3304e9b61f19SKirill A. Shutemov struct page *subpage = head + i; 3305e9b61f19SKirill A. Shutemov if (subpage == page) 3306e9b61f19SKirill A. Shutemov continue; 3307e9b61f19SKirill A. Shutemov unlock_page(subpage); 3308e9b61f19SKirill A. Shutemov 3309e9b61f19SKirill A. Shutemov /* 3310e9b61f19SKirill A. Shutemov * Subpages may be freed if there wasn't any mapping 3311e9b61f19SKirill A. Shutemov * like if add_to_swap() is running on a lru page that 3312e9b61f19SKirill A. Shutemov * had its mapping zapped. And freeing these pages 3313e9b61f19SKirill A. Shutemov * requires taking the lru_lock so we do the put_page 3314e9b61f19SKirill A. Shutemov * of the tail pages after the split is complete. 3315e9b61f19SKirill A. Shutemov */ 3316e9b61f19SKirill A. Shutemov put_page(subpage); 3317e9b61f19SKirill A. Shutemov } 3318e9b61f19SKirill A. Shutemov } 3319e9b61f19SKirill A. Shutemov 3320b20ce5e0SKirill A. Shutemov int total_mapcount(struct page *page) 3321b20ce5e0SKirill A. Shutemov { 3322b20ce5e0SKirill A. Shutemov int i, ret; 3323b20ce5e0SKirill A. Shutemov 3324b20ce5e0SKirill A. Shutemov VM_BUG_ON_PAGE(PageTail(page), page); 3325b20ce5e0SKirill A. Shutemov 3326b20ce5e0SKirill A. Shutemov if (likely(!PageCompound(page))) 3327b20ce5e0SKirill A. Shutemov return atomic_read(&page->_mapcount) + 1; 3328b20ce5e0SKirill A. Shutemov 3329b20ce5e0SKirill A. Shutemov ret = compound_mapcount(page); 3330b20ce5e0SKirill A. Shutemov if (PageHuge(page)) 3331b20ce5e0SKirill A. Shutemov return ret; 3332b20ce5e0SKirill A. Shutemov for (i = 0; i < HPAGE_PMD_NR; i++) 3333b20ce5e0SKirill A. Shutemov ret += atomic_read(&page[i]._mapcount) + 1; 3334b20ce5e0SKirill A. Shutemov if (PageDoubleMap(page)) 3335b20ce5e0SKirill A. Shutemov ret -= HPAGE_PMD_NR; 3336b20ce5e0SKirill A. Shutemov return ret; 3337b20ce5e0SKirill A. Shutemov } 3338b20ce5e0SKirill A. Shutemov 3339e9b61f19SKirill A. Shutemov /* 3340e9b61f19SKirill A. Shutemov * This function splits huge page into normal pages. @page can point to any 3341e9b61f19SKirill A. Shutemov * subpage of huge page to split. Split doesn't change the position of @page. 3342e9b61f19SKirill A. Shutemov * 3343e9b61f19SKirill A. Shutemov * Only caller must hold pin on the @page, otherwise split fails with -EBUSY. 3344e9b61f19SKirill A. Shutemov * The huge page must be locked. 3345e9b61f19SKirill A. Shutemov * 3346e9b61f19SKirill A. Shutemov * If @list is null, tail pages will be added to LRU list, otherwise, to @list. 3347e9b61f19SKirill A. Shutemov * 3348e9b61f19SKirill A. Shutemov * Both head page and tail pages will inherit mapping, flags, and so on from 3349e9b61f19SKirill A. Shutemov * the hugepage. 3350e9b61f19SKirill A. Shutemov * 3351e9b61f19SKirill A. Shutemov * GUP pin and PG_locked transferred to @page. Rest subpages can be freed if 3352e9b61f19SKirill A. Shutemov * they are not mapped. 3353e9b61f19SKirill A. Shutemov * 3354e9b61f19SKirill A. Shutemov * Returns 0 if the hugepage is split successfully. 3355e9b61f19SKirill A. Shutemov * Returns -EBUSY if the page is pinned or if anon_vma disappeared from under 3356e9b61f19SKirill A. Shutemov * us. 3357e9b61f19SKirill A. Shutemov */ 3358e9b61f19SKirill A. Shutemov int split_huge_page_to_list(struct page *page, struct list_head *list) 3359e9b61f19SKirill A. Shutemov { 3360e9b61f19SKirill A. Shutemov struct page *head = compound_head(page); 3361e9b61f19SKirill A. Shutemov struct anon_vma *anon_vma; 3362e9b61f19SKirill A. Shutemov int count, mapcount, ret; 3363d9654322SKirill A. Shutemov bool mlocked; 33640b9b6fffSKirill A. Shutemov unsigned long flags; 3365e9b61f19SKirill A. Shutemov 3366e9b61f19SKirill A. Shutemov VM_BUG_ON_PAGE(is_huge_zero_page(page), page); 3367e9b61f19SKirill A. Shutemov VM_BUG_ON_PAGE(!PageAnon(page), page); 3368e9b61f19SKirill A. Shutemov VM_BUG_ON_PAGE(!PageLocked(page), page); 3369e9b61f19SKirill A. Shutemov VM_BUG_ON_PAGE(!PageSwapBacked(page), page); 3370e9b61f19SKirill A. Shutemov VM_BUG_ON_PAGE(!PageCompound(page), page); 3371e9b61f19SKirill A. Shutemov 3372e9b61f19SKirill A. Shutemov /* 3373e9b61f19SKirill A. Shutemov * The caller does not necessarily hold an mmap_sem that would prevent 3374e9b61f19SKirill A. Shutemov * the anon_vma disappearing so we first we take a reference to it 3375e9b61f19SKirill A. Shutemov * and then lock the anon_vma for write. This is similar to 3376e9b61f19SKirill A. Shutemov * page_lock_anon_vma_read except the write lock is taken to serialise 3377e9b61f19SKirill A. Shutemov * against parallel split or collapse operations. 3378e9b61f19SKirill A. Shutemov */ 3379e9b61f19SKirill A. Shutemov anon_vma = page_get_anon_vma(head); 3380e9b61f19SKirill A. Shutemov if (!anon_vma) { 3381e9b61f19SKirill A. Shutemov ret = -EBUSY; 3382e9b61f19SKirill A. Shutemov goto out; 3383e9b61f19SKirill A. Shutemov } 3384e9b61f19SKirill A. Shutemov anon_vma_lock_write(anon_vma); 3385e9b61f19SKirill A. Shutemov 3386e9b61f19SKirill A. Shutemov /* 3387e9b61f19SKirill A. Shutemov * Racy check if we can split the page, before freeze_page() will 3388e9b61f19SKirill A. Shutemov * split PMDs 3389e9b61f19SKirill A. Shutemov */ 3390e9b61f19SKirill A. Shutemov if (total_mapcount(head) != page_count(head) - 1) { 3391e9b61f19SKirill A. Shutemov ret = -EBUSY; 3392e9b61f19SKirill A. Shutemov goto out_unlock; 3393e9b61f19SKirill A. Shutemov } 3394e9b61f19SKirill A. Shutemov 3395d9654322SKirill A. Shutemov mlocked = PageMlocked(page); 3396e9b61f19SKirill A. Shutemov freeze_page(anon_vma, head); 3397e9b61f19SKirill A. Shutemov VM_BUG_ON_PAGE(compound_mapcount(head), head); 3398e9b61f19SKirill A. Shutemov 3399d9654322SKirill A. Shutemov /* Make sure the page is not on per-CPU pagevec as it takes pin */ 3400d9654322SKirill A. Shutemov if (mlocked) 3401d9654322SKirill A. Shutemov lru_add_drain(); 3402d9654322SKirill A. Shutemov 34039a982250SKirill A. Shutemov /* Prevent deferred_split_scan() touching ->_count */ 34040b9b6fffSKirill A. Shutemov spin_lock_irqsave(&split_queue_lock, flags); 3405e9b61f19SKirill A. Shutemov count = page_count(head); 3406e9b61f19SKirill A. Shutemov mapcount = total_mapcount(head); 3407bd56086fSKirill A. Shutemov if (!mapcount && count == 1) { 34089a982250SKirill A. Shutemov if (!list_empty(page_deferred_list(head))) { 34099a982250SKirill A. Shutemov split_queue_len--; 34109a982250SKirill A. Shutemov list_del(page_deferred_list(head)); 34119a982250SKirill A. Shutemov } 34120b9b6fffSKirill A. Shutemov spin_unlock_irqrestore(&split_queue_lock, flags); 3413e9b61f19SKirill A. Shutemov __split_huge_page(page, list); 3414e9b61f19SKirill A. Shutemov ret = 0; 3415bd56086fSKirill A. Shutemov } else if (IS_ENABLED(CONFIG_DEBUG_VM) && mapcount) { 34160b9b6fffSKirill A. Shutemov spin_unlock_irqrestore(&split_queue_lock, flags); 3417e9b61f19SKirill A. Shutemov pr_alert("total_mapcount: %u, page_count(): %u\n", 3418e9b61f19SKirill A. Shutemov mapcount, count); 3419e9b61f19SKirill A. Shutemov if (PageTail(page)) 3420e9b61f19SKirill A. Shutemov dump_page(head, NULL); 3421bd56086fSKirill A. Shutemov dump_page(page, "total_mapcount(head) > 0"); 3422e9b61f19SKirill A. Shutemov BUG(); 3423e9b61f19SKirill A. Shutemov } else { 34240b9b6fffSKirill A. Shutemov spin_unlock_irqrestore(&split_queue_lock, flags); 3425e9b61f19SKirill A. Shutemov unfreeze_page(anon_vma, head); 3426e9b61f19SKirill A. Shutemov ret = -EBUSY; 3427e9b61f19SKirill A. Shutemov } 3428e9b61f19SKirill A. Shutemov 3429e9b61f19SKirill A. Shutemov out_unlock: 3430e9b61f19SKirill A. Shutemov anon_vma_unlock_write(anon_vma); 3431e9b61f19SKirill A. Shutemov put_anon_vma(anon_vma); 3432e9b61f19SKirill A. Shutemov out: 3433e9b61f19SKirill A. Shutemov count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED); 3434e9b61f19SKirill A. Shutemov return ret; 3435e9b61f19SKirill A. Shutemov } 34369a982250SKirill A. Shutemov 34379a982250SKirill A. Shutemov void free_transhuge_page(struct page *page) 34389a982250SKirill A. Shutemov { 34399a982250SKirill A. Shutemov unsigned long flags; 34409a982250SKirill A. Shutemov 34419a982250SKirill A. Shutemov spin_lock_irqsave(&split_queue_lock, flags); 34429a982250SKirill A. Shutemov if (!list_empty(page_deferred_list(page))) { 34439a982250SKirill A. Shutemov split_queue_len--; 34449a982250SKirill A. Shutemov list_del(page_deferred_list(page)); 34459a982250SKirill A. Shutemov } 34469a982250SKirill A. Shutemov spin_unlock_irqrestore(&split_queue_lock, flags); 34479a982250SKirill A. Shutemov free_compound_page(page); 34489a982250SKirill A. Shutemov } 34499a982250SKirill A. Shutemov 34509a982250SKirill A. Shutemov void deferred_split_huge_page(struct page *page) 34519a982250SKirill A. Shutemov { 34529a982250SKirill A. Shutemov unsigned long flags; 34539a982250SKirill A. Shutemov 34549a982250SKirill A. Shutemov VM_BUG_ON_PAGE(!PageTransHuge(page), page); 34559a982250SKirill A. Shutemov 34569a982250SKirill A. Shutemov spin_lock_irqsave(&split_queue_lock, flags); 34579a982250SKirill A. Shutemov if (list_empty(page_deferred_list(page))) { 34589a982250SKirill A. Shutemov list_add_tail(page_deferred_list(page), &split_queue); 34599a982250SKirill A. Shutemov split_queue_len++; 34609a982250SKirill A. Shutemov } 34619a982250SKirill A. Shutemov spin_unlock_irqrestore(&split_queue_lock, flags); 34629a982250SKirill A. Shutemov } 34639a982250SKirill A. Shutemov 34649a982250SKirill A. Shutemov static unsigned long deferred_split_count(struct shrinker *shrink, 34659a982250SKirill A. Shutemov struct shrink_control *sc) 34669a982250SKirill A. Shutemov { 34679a982250SKirill A. Shutemov /* 34689a982250SKirill A. Shutemov * Split a page from split_queue will free up at least one page, 34699a982250SKirill A. Shutemov * at most HPAGE_PMD_NR - 1. We don't track exact number. 34709a982250SKirill A. Shutemov * Let's use HPAGE_PMD_NR / 2 as ballpark. 34719a982250SKirill A. Shutemov */ 34729a982250SKirill A. Shutemov return ACCESS_ONCE(split_queue_len) * HPAGE_PMD_NR / 2; 34739a982250SKirill A. Shutemov } 34749a982250SKirill A. Shutemov 34759a982250SKirill A. Shutemov static unsigned long deferred_split_scan(struct shrinker *shrink, 34769a982250SKirill A. Shutemov struct shrink_control *sc) 34779a982250SKirill A. Shutemov { 34789a982250SKirill A. Shutemov unsigned long flags; 34799a982250SKirill A. Shutemov LIST_HEAD(list), *pos, *next; 34809a982250SKirill A. Shutemov struct page *page; 34819a982250SKirill A. Shutemov int split = 0; 34829a982250SKirill A. Shutemov 34839a982250SKirill A. Shutemov spin_lock_irqsave(&split_queue_lock, flags); 34849a982250SKirill A. Shutemov list_splice_init(&split_queue, &list); 34859a982250SKirill A. Shutemov 34869a982250SKirill A. Shutemov /* Take pin on all head pages to avoid freeing them under us */ 34879a982250SKirill A. Shutemov list_for_each_safe(pos, next, &list) { 34889a982250SKirill A. Shutemov page = list_entry((void *)pos, struct page, mapping); 34899a982250SKirill A. Shutemov page = compound_head(page); 34909a982250SKirill A. Shutemov /* race with put_compound_page() */ 34919a982250SKirill A. Shutemov if (!get_page_unless_zero(page)) { 34929a982250SKirill A. Shutemov list_del_init(page_deferred_list(page)); 34939a982250SKirill A. Shutemov split_queue_len--; 34949a982250SKirill A. Shutemov } 34959a982250SKirill A. Shutemov } 34969a982250SKirill A. Shutemov spin_unlock_irqrestore(&split_queue_lock, flags); 34979a982250SKirill A. Shutemov 34989a982250SKirill A. Shutemov list_for_each_safe(pos, next, &list) { 34999a982250SKirill A. Shutemov page = list_entry((void *)pos, struct page, mapping); 35009a982250SKirill A. Shutemov lock_page(page); 35019a982250SKirill A. Shutemov /* split_huge_page() removes page from list on success */ 35029a982250SKirill A. Shutemov if (!split_huge_page(page)) 35039a982250SKirill A. Shutemov split++; 35049a982250SKirill A. Shutemov unlock_page(page); 35059a982250SKirill A. Shutemov put_page(page); 35069a982250SKirill A. Shutemov } 35079a982250SKirill A. Shutemov 35089a982250SKirill A. Shutemov spin_lock_irqsave(&split_queue_lock, flags); 35099a982250SKirill A. Shutemov list_splice_tail(&list, &split_queue); 35109a982250SKirill A. Shutemov spin_unlock_irqrestore(&split_queue_lock, flags); 35119a982250SKirill A. Shutemov 35129a982250SKirill A. Shutemov return split * HPAGE_PMD_NR / 2; 35139a982250SKirill A. Shutemov } 35149a982250SKirill A. Shutemov 35159a982250SKirill A. Shutemov static struct shrinker deferred_split_shrinker = { 35169a982250SKirill A. Shutemov .count_objects = deferred_split_count, 35179a982250SKirill A. Shutemov .scan_objects = deferred_split_scan, 35189a982250SKirill A. Shutemov .seeks = DEFAULT_SEEKS, 35199a982250SKirill A. Shutemov }; 352049071d43SKirill A. Shutemov 352149071d43SKirill A. Shutemov #ifdef CONFIG_DEBUG_FS 352249071d43SKirill A. Shutemov static int split_huge_pages_set(void *data, u64 val) 352349071d43SKirill A. Shutemov { 352449071d43SKirill A. Shutemov struct zone *zone; 352549071d43SKirill A. Shutemov struct page *page; 352649071d43SKirill A. Shutemov unsigned long pfn, max_zone_pfn; 352749071d43SKirill A. Shutemov unsigned long total = 0, split = 0; 352849071d43SKirill A. Shutemov 352949071d43SKirill A. Shutemov if (val != 1) 353049071d43SKirill A. Shutemov return -EINVAL; 353149071d43SKirill A. Shutemov 353249071d43SKirill A. Shutemov for_each_populated_zone(zone) { 353349071d43SKirill A. Shutemov max_zone_pfn = zone_end_pfn(zone); 353449071d43SKirill A. Shutemov for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) { 353549071d43SKirill A. Shutemov if (!pfn_valid(pfn)) 353649071d43SKirill A. Shutemov continue; 353749071d43SKirill A. Shutemov 353849071d43SKirill A. Shutemov page = pfn_to_page(pfn); 353949071d43SKirill A. Shutemov if (!get_page_unless_zero(page)) 354049071d43SKirill A. Shutemov continue; 354149071d43SKirill A. Shutemov 354249071d43SKirill A. Shutemov if (zone != page_zone(page)) 354349071d43SKirill A. Shutemov goto next; 354449071d43SKirill A. Shutemov 354549071d43SKirill A. Shutemov if (!PageHead(page) || !PageAnon(page) || 354649071d43SKirill A. Shutemov PageHuge(page)) 354749071d43SKirill A. Shutemov goto next; 354849071d43SKirill A. Shutemov 354949071d43SKirill A. Shutemov total++; 355049071d43SKirill A. Shutemov lock_page(page); 355149071d43SKirill A. Shutemov if (!split_huge_page(page)) 355249071d43SKirill A. Shutemov split++; 355349071d43SKirill A. Shutemov unlock_page(page); 355449071d43SKirill A. Shutemov next: 355549071d43SKirill A. Shutemov put_page(page); 355649071d43SKirill A. Shutemov } 355749071d43SKirill A. Shutemov } 355849071d43SKirill A. Shutemov 355949071d43SKirill A. Shutemov pr_info("%lu of %lu THP split", split, total); 356049071d43SKirill A. Shutemov 356149071d43SKirill A. Shutemov return 0; 356249071d43SKirill A. Shutemov } 356349071d43SKirill A. Shutemov DEFINE_SIMPLE_ATTRIBUTE(split_huge_pages_fops, NULL, split_huge_pages_set, 356449071d43SKirill A. Shutemov "%llu\n"); 356549071d43SKirill A. Shutemov 356649071d43SKirill A. Shutemov static int __init split_huge_pages_debugfs(void) 356749071d43SKirill A. Shutemov { 356849071d43SKirill A. Shutemov void *ret; 356949071d43SKirill A. Shutemov 357049071d43SKirill A. Shutemov ret = debugfs_create_file("split_huge_pages", 0644, NULL, NULL, 357149071d43SKirill A. Shutemov &split_huge_pages_fops); 357249071d43SKirill A. Shutemov if (!ret) 357349071d43SKirill A. Shutemov pr_warn("Failed to create split_huge_pages in debugfs"); 357449071d43SKirill A. Shutemov return 0; 357549071d43SKirill A. Shutemov } 357649071d43SKirill A. Shutemov late_initcall(split_huge_pages_debugfs); 357749071d43SKirill A. Shutemov #endif 3578