171e3aac0SAndrea Arcangeli /* 271e3aac0SAndrea Arcangeli * Copyright (C) 2009 Red Hat, Inc. 371e3aac0SAndrea Arcangeli * 471e3aac0SAndrea Arcangeli * This work is licensed under the terms of the GNU GPL, version 2. See 571e3aac0SAndrea Arcangeli * the COPYING file in the top-level directory. 671e3aac0SAndrea Arcangeli */ 771e3aac0SAndrea Arcangeli 8ae3a8c1cSAndrew Morton #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 9ae3a8c1cSAndrew Morton 1071e3aac0SAndrea Arcangeli #include <linux/mm.h> 1171e3aac0SAndrea Arcangeli #include <linux/sched.h> 1271e3aac0SAndrea Arcangeli #include <linux/highmem.h> 1371e3aac0SAndrea Arcangeli #include <linux/hugetlb.h> 1471e3aac0SAndrea Arcangeli #include <linux/mmu_notifier.h> 1571e3aac0SAndrea Arcangeli #include <linux/rmap.h> 1671e3aac0SAndrea Arcangeli #include <linux/swap.h> 1797ae1749SKirill A. Shutemov #include <linux/shrinker.h> 18ba76149fSAndrea Arcangeli #include <linux/mm_inline.h> 19e9b61f19SKirill A. Shutemov #include <linux/swapops.h> 204897c765SMatthew Wilcox #include <linux/dax.h> 21ba76149fSAndrea Arcangeli #include <linux/kthread.h> 22ba76149fSAndrea Arcangeli #include <linux/khugepaged.h> 23878aee7dSAndrea Arcangeli #include <linux/freezer.h> 24f25748e3SDan Williams #include <linux/pfn_t.h> 25a664b2d8SAndrea Arcangeli #include <linux/mman.h> 26325adeb5SRalf Baechle #include <linux/pagemap.h> 2749071d43SKirill A. Shutemov #include <linux/debugfs.h> 284daae3b4SMel Gorman #include <linux/migrate.h> 2943b5fbbdSSasha Levin #include <linux/hashtable.h> 306b251fc9SAndrea Arcangeli #include <linux/userfaultfd_k.h> 3133c3fc71SVladimir Davydov #include <linux/page_idle.h> 3297ae1749SKirill A. Shutemov 3371e3aac0SAndrea Arcangeli #include <asm/tlb.h> 3471e3aac0SAndrea Arcangeli #include <asm/pgalloc.h> 3571e3aac0SAndrea Arcangeli #include "internal.h" 3671e3aac0SAndrea Arcangeli 377d2eba05SEbru Akagunduz enum scan_result { 387d2eba05SEbru Akagunduz SCAN_FAIL, 397d2eba05SEbru Akagunduz SCAN_SUCCEED, 407d2eba05SEbru Akagunduz SCAN_PMD_NULL, 417d2eba05SEbru Akagunduz SCAN_EXCEED_NONE_PTE, 427d2eba05SEbru Akagunduz SCAN_PTE_NON_PRESENT, 437d2eba05SEbru Akagunduz SCAN_PAGE_RO, 447d2eba05SEbru Akagunduz SCAN_NO_REFERENCED_PAGE, 457d2eba05SEbru Akagunduz SCAN_PAGE_NULL, 467d2eba05SEbru Akagunduz SCAN_SCAN_ABORT, 477d2eba05SEbru Akagunduz SCAN_PAGE_COUNT, 487d2eba05SEbru Akagunduz SCAN_PAGE_LRU, 497d2eba05SEbru Akagunduz SCAN_PAGE_LOCK, 507d2eba05SEbru Akagunduz SCAN_PAGE_ANON, 51b1caa957SKirill A. Shutemov SCAN_PAGE_COMPOUND, 527d2eba05SEbru Akagunduz SCAN_ANY_PROCESS, 537d2eba05SEbru Akagunduz SCAN_VMA_NULL, 547d2eba05SEbru Akagunduz SCAN_VMA_CHECK, 557d2eba05SEbru Akagunduz SCAN_ADDRESS_RANGE, 567d2eba05SEbru Akagunduz SCAN_SWAP_CACHE_PAGE, 577d2eba05SEbru Akagunduz SCAN_DEL_PAGE_LRU, 587d2eba05SEbru Akagunduz SCAN_ALLOC_HUGE_PAGE_FAIL, 597d2eba05SEbru Akagunduz SCAN_CGROUP_CHARGE_FAIL 607d2eba05SEbru Akagunduz }; 617d2eba05SEbru Akagunduz 627d2eba05SEbru Akagunduz #define CREATE_TRACE_POINTS 637d2eba05SEbru Akagunduz #include <trace/events/huge_memory.h> 647d2eba05SEbru Akagunduz 65ba76149fSAndrea Arcangeli /* 668bfa3f9aSJianguo Wu * By default transparent hugepage support is disabled in order that avoid 678bfa3f9aSJianguo Wu * to risk increase the memory footprint of applications without a guaranteed 688bfa3f9aSJianguo Wu * benefit. When transparent hugepage support is enabled, is for all mappings, 698bfa3f9aSJianguo Wu * and khugepaged scans all mappings. 708bfa3f9aSJianguo Wu * Defrag is invoked by khugepaged hugepage allocations and by page faults 718bfa3f9aSJianguo Wu * for all hugepage allocations. 72ba76149fSAndrea Arcangeli */ 7371e3aac0SAndrea Arcangeli unsigned long transparent_hugepage_flags __read_mostly = 7413ece886SAndrea Arcangeli #ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS 75ba76149fSAndrea Arcangeli (1<<TRANSPARENT_HUGEPAGE_FLAG)| 7613ece886SAndrea Arcangeli #endif 7713ece886SAndrea Arcangeli #ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE 7813ece886SAndrea Arcangeli (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)| 7913ece886SAndrea Arcangeli #endif 80d39d33c3SAndrea Arcangeli (1<<TRANSPARENT_HUGEPAGE_DEFRAG_FLAG)| 8179da5407SKirill A. Shutemov (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)| 8279da5407SKirill A. Shutemov (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 83ba76149fSAndrea Arcangeli 84ba76149fSAndrea Arcangeli /* default scan 8*512 pte (or vmas) every 30 second */ 85ba76149fSAndrea Arcangeli static unsigned int khugepaged_pages_to_scan __read_mostly = HPAGE_PMD_NR*8; 86ba76149fSAndrea Arcangeli static unsigned int khugepaged_pages_collapsed; 87ba76149fSAndrea Arcangeli static unsigned int khugepaged_full_scans; 88ba76149fSAndrea Arcangeli static unsigned int khugepaged_scan_sleep_millisecs __read_mostly = 10000; 89ba76149fSAndrea Arcangeli /* during fragmentation poll the hugepage allocator once every minute */ 90ba76149fSAndrea Arcangeli static unsigned int khugepaged_alloc_sleep_millisecs __read_mostly = 60000; 91ba76149fSAndrea Arcangeli static struct task_struct *khugepaged_thread __read_mostly; 92ba76149fSAndrea Arcangeli static DEFINE_MUTEX(khugepaged_mutex); 93ba76149fSAndrea Arcangeli static DEFINE_SPINLOCK(khugepaged_mm_lock); 94ba76149fSAndrea Arcangeli static DECLARE_WAIT_QUEUE_HEAD(khugepaged_wait); 95ba76149fSAndrea Arcangeli /* 96ba76149fSAndrea Arcangeli * default collapse hugepages if there is at least one pte mapped like 97ba76149fSAndrea Arcangeli * it would have happened if the vma was large enough during page 98ba76149fSAndrea Arcangeli * fault. 99ba76149fSAndrea Arcangeli */ 100ba76149fSAndrea Arcangeli static unsigned int khugepaged_max_ptes_none __read_mostly = HPAGE_PMD_NR-1; 101ba76149fSAndrea Arcangeli 102ba76149fSAndrea Arcangeli static int khugepaged(void *none); 103ba76149fSAndrea Arcangeli static int khugepaged_slab_init(void); 10465ebb64fSKirill A. Shutemov static void khugepaged_slab_exit(void); 105ba76149fSAndrea Arcangeli 10643b5fbbdSSasha Levin #define MM_SLOTS_HASH_BITS 10 10743b5fbbdSSasha Levin static __read_mostly DEFINE_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS); 10843b5fbbdSSasha Levin 109ba76149fSAndrea Arcangeli static struct kmem_cache *mm_slot_cache __read_mostly; 110ba76149fSAndrea Arcangeli 111ba76149fSAndrea Arcangeli /** 112ba76149fSAndrea Arcangeli * struct mm_slot - hash lookup from mm to mm_slot 113ba76149fSAndrea Arcangeli * @hash: hash collision list 114ba76149fSAndrea Arcangeli * @mm_node: khugepaged scan list headed in khugepaged_scan.mm_head 115ba76149fSAndrea Arcangeli * @mm: the mm that this information is valid for 116ba76149fSAndrea Arcangeli */ 117ba76149fSAndrea Arcangeli struct mm_slot { 118ba76149fSAndrea Arcangeli struct hlist_node hash; 119ba76149fSAndrea Arcangeli struct list_head mm_node; 120ba76149fSAndrea Arcangeli struct mm_struct *mm; 121ba76149fSAndrea Arcangeli }; 122ba76149fSAndrea Arcangeli 123ba76149fSAndrea Arcangeli /** 124ba76149fSAndrea Arcangeli * struct khugepaged_scan - cursor for scanning 125ba76149fSAndrea Arcangeli * @mm_head: the head of the mm list to scan 126ba76149fSAndrea Arcangeli * @mm_slot: the current mm_slot we are scanning 127ba76149fSAndrea Arcangeli * @address: the next address inside that to be scanned 128ba76149fSAndrea Arcangeli * 129ba76149fSAndrea Arcangeli * There is only the one khugepaged_scan instance of this cursor structure. 130ba76149fSAndrea Arcangeli */ 131ba76149fSAndrea Arcangeli struct khugepaged_scan { 132ba76149fSAndrea Arcangeli struct list_head mm_head; 133ba76149fSAndrea Arcangeli struct mm_slot *mm_slot; 134ba76149fSAndrea Arcangeli unsigned long address; 1352f1da642SH Hartley Sweeten }; 1362f1da642SH Hartley Sweeten static struct khugepaged_scan khugepaged_scan = { 137ba76149fSAndrea Arcangeli .mm_head = LIST_HEAD_INIT(khugepaged_scan.mm_head), 138ba76149fSAndrea Arcangeli }; 139ba76149fSAndrea Arcangeli 1409a982250SKirill A. Shutemov static DEFINE_SPINLOCK(split_queue_lock); 1419a982250SKirill A. Shutemov static LIST_HEAD(split_queue); 1429a982250SKirill A. Shutemov static unsigned long split_queue_len; 1439a982250SKirill A. Shutemov static struct shrinker deferred_split_shrinker; 144f000565aSAndrea Arcangeli 1452c0b80d4SNicholas Krause static void set_recommended_min_free_kbytes(void) 146f000565aSAndrea Arcangeli { 147f000565aSAndrea Arcangeli struct zone *zone; 148f000565aSAndrea Arcangeli int nr_zones = 0; 149f000565aSAndrea Arcangeli unsigned long recommended_min; 150f000565aSAndrea Arcangeli 151f000565aSAndrea Arcangeli for_each_populated_zone(zone) 152f000565aSAndrea Arcangeli nr_zones++; 153f000565aSAndrea Arcangeli 154974a786eSMel Gorman /* Ensure 2 pageblocks are free to assist fragmentation avoidance */ 155f000565aSAndrea Arcangeli recommended_min = pageblock_nr_pages * nr_zones * 2; 156f000565aSAndrea Arcangeli 157f000565aSAndrea Arcangeli /* 158f000565aSAndrea Arcangeli * Make sure that on average at least two pageblocks are almost free 159f000565aSAndrea Arcangeli * of another type, one for a migratetype to fall back to and a 160f000565aSAndrea Arcangeli * second to avoid subsequent fallbacks of other types There are 3 161f000565aSAndrea Arcangeli * MIGRATE_TYPES we care about. 162f000565aSAndrea Arcangeli */ 163f000565aSAndrea Arcangeli recommended_min += pageblock_nr_pages * nr_zones * 164f000565aSAndrea Arcangeli MIGRATE_PCPTYPES * MIGRATE_PCPTYPES; 165f000565aSAndrea Arcangeli 166f000565aSAndrea Arcangeli /* don't ever allow to reserve more than 5% of the lowmem */ 167f000565aSAndrea Arcangeli recommended_min = min(recommended_min, 168f000565aSAndrea Arcangeli (unsigned long) nr_free_buffer_pages() / 20); 169f000565aSAndrea Arcangeli recommended_min <<= (PAGE_SHIFT-10); 170f000565aSAndrea Arcangeli 17142aa83cbSHan Pingtian if (recommended_min > min_free_kbytes) { 17242aa83cbSHan Pingtian if (user_min_free_kbytes >= 0) 17342aa83cbSHan Pingtian pr_info("raising min_free_kbytes from %d to %lu " 17442aa83cbSHan Pingtian "to help transparent hugepage allocations\n", 17542aa83cbSHan Pingtian min_free_kbytes, recommended_min); 17642aa83cbSHan Pingtian 177f000565aSAndrea Arcangeli min_free_kbytes = recommended_min; 17842aa83cbSHan Pingtian } 179f000565aSAndrea Arcangeli setup_per_zone_wmarks(); 180f000565aSAndrea Arcangeli } 181f000565aSAndrea Arcangeli 18279553da2SKirill A. Shutemov static int start_stop_khugepaged(void) 183ba76149fSAndrea Arcangeli { 184ba76149fSAndrea Arcangeli int err = 0; 185ba76149fSAndrea Arcangeli if (khugepaged_enabled()) { 186ba76149fSAndrea Arcangeli if (!khugepaged_thread) 187ba76149fSAndrea Arcangeli khugepaged_thread = kthread_run(khugepaged, NULL, 188ba76149fSAndrea Arcangeli "khugepaged"); 18918e8e5c7SViresh Kumar if (IS_ERR(khugepaged_thread)) { 190ae3a8c1cSAndrew Morton pr_err("khugepaged: kthread_run(khugepaged) failed\n"); 191ba76149fSAndrea Arcangeli err = PTR_ERR(khugepaged_thread); 192ba76149fSAndrea Arcangeli khugepaged_thread = NULL; 19379553da2SKirill A. Shutemov goto fail; 194ba76149fSAndrea Arcangeli } 195911891afSXiao Guangrong 196911891afSXiao Guangrong if (!list_empty(&khugepaged_scan.mm_head)) 197ba76149fSAndrea Arcangeli wake_up_interruptible(&khugepaged_wait); 198f000565aSAndrea Arcangeli 199f000565aSAndrea Arcangeli set_recommended_min_free_kbytes(); 200911891afSXiao Guangrong } else if (khugepaged_thread) { 201911891afSXiao Guangrong kthread_stop(khugepaged_thread); 202911891afSXiao Guangrong khugepaged_thread = NULL; 203911891afSXiao Guangrong } 20479553da2SKirill A. Shutemov fail: 205ba76149fSAndrea Arcangeli return err; 206ba76149fSAndrea Arcangeli } 20771e3aac0SAndrea Arcangeli 20897ae1749SKirill A. Shutemov static atomic_t huge_zero_refcount; 20956873f43SWang, Yalin struct page *huge_zero_page __read_mostly; 2104a6c1297SKirill A. Shutemov 211fc437044SMatthew Wilcox struct page *get_huge_zero_page(void) 21297ae1749SKirill A. Shutemov { 21397ae1749SKirill A. Shutemov struct page *zero_page; 21497ae1749SKirill A. Shutemov retry: 21597ae1749SKirill A. Shutemov if (likely(atomic_inc_not_zero(&huge_zero_refcount))) 2164db0c3c2SJason Low return READ_ONCE(huge_zero_page); 21797ae1749SKirill A. Shutemov 21897ae1749SKirill A. Shutemov zero_page = alloc_pages((GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE, 21997ae1749SKirill A. Shutemov HPAGE_PMD_ORDER); 220d8a8e1f0SKirill A. Shutemov if (!zero_page) { 221d8a8e1f0SKirill A. Shutemov count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED); 2225918d10aSKirill A. Shutemov return NULL; 223d8a8e1f0SKirill A. Shutemov } 224d8a8e1f0SKirill A. Shutemov count_vm_event(THP_ZERO_PAGE_ALLOC); 22597ae1749SKirill A. Shutemov preempt_disable(); 2265918d10aSKirill A. Shutemov if (cmpxchg(&huge_zero_page, NULL, zero_page)) { 22797ae1749SKirill A. Shutemov preempt_enable(); 2285ddacbe9SYu Zhao __free_pages(zero_page, compound_order(zero_page)); 22997ae1749SKirill A. Shutemov goto retry; 23097ae1749SKirill A. Shutemov } 23197ae1749SKirill A. Shutemov 23297ae1749SKirill A. Shutemov /* We take additional reference here. It will be put back by shrinker */ 23397ae1749SKirill A. Shutemov atomic_set(&huge_zero_refcount, 2); 23497ae1749SKirill A. Shutemov preempt_enable(); 2354db0c3c2SJason Low return READ_ONCE(huge_zero_page); 23697ae1749SKirill A. Shutemov } 23797ae1749SKirill A. Shutemov 23897ae1749SKirill A. Shutemov static void put_huge_zero_page(void) 23997ae1749SKirill A. Shutemov { 24097ae1749SKirill A. Shutemov /* 24197ae1749SKirill A. Shutemov * Counter should never go to zero here. Only shrinker can put 24297ae1749SKirill A. Shutemov * last reference. 24397ae1749SKirill A. Shutemov */ 24497ae1749SKirill A. Shutemov BUG_ON(atomic_dec_and_test(&huge_zero_refcount)); 24597ae1749SKirill A. Shutemov } 24697ae1749SKirill A. Shutemov 24748896466SGlauber Costa static unsigned long shrink_huge_zero_page_count(struct shrinker *shrink, 24897ae1749SKirill A. Shutemov struct shrink_control *sc) 24997ae1749SKirill A. Shutemov { 25097ae1749SKirill A. Shutemov /* we can free zero page only if last reference remains */ 25197ae1749SKirill A. Shutemov return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0; 25248896466SGlauber Costa } 25397ae1749SKirill A. Shutemov 25448896466SGlauber Costa static unsigned long shrink_huge_zero_page_scan(struct shrinker *shrink, 25548896466SGlauber Costa struct shrink_control *sc) 25648896466SGlauber Costa { 25797ae1749SKirill A. Shutemov if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) { 2585918d10aSKirill A. Shutemov struct page *zero_page = xchg(&huge_zero_page, NULL); 2595918d10aSKirill A. Shutemov BUG_ON(zero_page == NULL); 2605ddacbe9SYu Zhao __free_pages(zero_page, compound_order(zero_page)); 26148896466SGlauber Costa return HPAGE_PMD_NR; 26297ae1749SKirill A. Shutemov } 26397ae1749SKirill A. Shutemov 26497ae1749SKirill A. Shutemov return 0; 26597ae1749SKirill A. Shutemov } 26697ae1749SKirill A. Shutemov 26797ae1749SKirill A. Shutemov static struct shrinker huge_zero_page_shrinker = { 26848896466SGlauber Costa .count_objects = shrink_huge_zero_page_count, 26948896466SGlauber Costa .scan_objects = shrink_huge_zero_page_scan, 27097ae1749SKirill A. Shutemov .seeks = DEFAULT_SEEKS, 27197ae1749SKirill A. Shutemov }; 27297ae1749SKirill A. Shutemov 27371e3aac0SAndrea Arcangeli #ifdef CONFIG_SYSFS 274ba76149fSAndrea Arcangeli 27571e3aac0SAndrea Arcangeli static ssize_t double_flag_show(struct kobject *kobj, 27671e3aac0SAndrea Arcangeli struct kobj_attribute *attr, char *buf, 27771e3aac0SAndrea Arcangeli enum transparent_hugepage_flag enabled, 27871e3aac0SAndrea Arcangeli enum transparent_hugepage_flag req_madv) 27971e3aac0SAndrea Arcangeli { 28071e3aac0SAndrea Arcangeli if (test_bit(enabled, &transparent_hugepage_flags)) { 28171e3aac0SAndrea Arcangeli VM_BUG_ON(test_bit(req_madv, &transparent_hugepage_flags)); 28271e3aac0SAndrea Arcangeli return sprintf(buf, "[always] madvise never\n"); 28371e3aac0SAndrea Arcangeli } else if (test_bit(req_madv, &transparent_hugepage_flags)) 28471e3aac0SAndrea Arcangeli return sprintf(buf, "always [madvise] never\n"); 28571e3aac0SAndrea Arcangeli else 28671e3aac0SAndrea Arcangeli return sprintf(buf, "always madvise [never]\n"); 28771e3aac0SAndrea Arcangeli } 28871e3aac0SAndrea Arcangeli static ssize_t double_flag_store(struct kobject *kobj, 28971e3aac0SAndrea Arcangeli struct kobj_attribute *attr, 29071e3aac0SAndrea Arcangeli const char *buf, size_t count, 29171e3aac0SAndrea Arcangeli enum transparent_hugepage_flag enabled, 29271e3aac0SAndrea Arcangeli enum transparent_hugepage_flag req_madv) 29371e3aac0SAndrea Arcangeli { 29471e3aac0SAndrea Arcangeli if (!memcmp("always", buf, 29571e3aac0SAndrea Arcangeli min(sizeof("always")-1, count))) { 29671e3aac0SAndrea Arcangeli set_bit(enabled, &transparent_hugepage_flags); 29771e3aac0SAndrea Arcangeli clear_bit(req_madv, &transparent_hugepage_flags); 29871e3aac0SAndrea Arcangeli } else if (!memcmp("madvise", buf, 29971e3aac0SAndrea Arcangeli min(sizeof("madvise")-1, count))) { 30071e3aac0SAndrea Arcangeli clear_bit(enabled, &transparent_hugepage_flags); 30171e3aac0SAndrea Arcangeli set_bit(req_madv, &transparent_hugepage_flags); 30271e3aac0SAndrea Arcangeli } else if (!memcmp("never", buf, 30371e3aac0SAndrea Arcangeli min(sizeof("never")-1, count))) { 30471e3aac0SAndrea Arcangeli clear_bit(enabled, &transparent_hugepage_flags); 30571e3aac0SAndrea Arcangeli clear_bit(req_madv, &transparent_hugepage_flags); 30671e3aac0SAndrea Arcangeli } else 30771e3aac0SAndrea Arcangeli return -EINVAL; 30871e3aac0SAndrea Arcangeli 30971e3aac0SAndrea Arcangeli return count; 31071e3aac0SAndrea Arcangeli } 31171e3aac0SAndrea Arcangeli 31271e3aac0SAndrea Arcangeli static ssize_t enabled_show(struct kobject *kobj, 31371e3aac0SAndrea Arcangeli struct kobj_attribute *attr, char *buf) 31471e3aac0SAndrea Arcangeli { 31571e3aac0SAndrea Arcangeli return double_flag_show(kobj, attr, buf, 31671e3aac0SAndrea Arcangeli TRANSPARENT_HUGEPAGE_FLAG, 31771e3aac0SAndrea Arcangeli TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG); 31871e3aac0SAndrea Arcangeli } 31971e3aac0SAndrea Arcangeli static ssize_t enabled_store(struct kobject *kobj, 32071e3aac0SAndrea Arcangeli struct kobj_attribute *attr, 32171e3aac0SAndrea Arcangeli const char *buf, size_t count) 32271e3aac0SAndrea Arcangeli { 323ba76149fSAndrea Arcangeli ssize_t ret; 324ba76149fSAndrea Arcangeli 325ba76149fSAndrea Arcangeli ret = double_flag_store(kobj, attr, buf, count, 32671e3aac0SAndrea Arcangeli TRANSPARENT_HUGEPAGE_FLAG, 32771e3aac0SAndrea Arcangeli TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG); 328ba76149fSAndrea Arcangeli 329ba76149fSAndrea Arcangeli if (ret > 0) { 330911891afSXiao Guangrong int err; 331911891afSXiao Guangrong 332911891afSXiao Guangrong mutex_lock(&khugepaged_mutex); 33379553da2SKirill A. Shutemov err = start_stop_khugepaged(); 334911891afSXiao Guangrong mutex_unlock(&khugepaged_mutex); 335911891afSXiao Guangrong 336ba76149fSAndrea Arcangeli if (err) 337ba76149fSAndrea Arcangeli ret = err; 338ba76149fSAndrea Arcangeli } 339ba76149fSAndrea Arcangeli 340ba76149fSAndrea Arcangeli return ret; 34171e3aac0SAndrea Arcangeli } 34271e3aac0SAndrea Arcangeli static struct kobj_attribute enabled_attr = 34371e3aac0SAndrea Arcangeli __ATTR(enabled, 0644, enabled_show, enabled_store); 34471e3aac0SAndrea Arcangeli 34571e3aac0SAndrea Arcangeli static ssize_t single_flag_show(struct kobject *kobj, 34671e3aac0SAndrea Arcangeli struct kobj_attribute *attr, char *buf, 34771e3aac0SAndrea Arcangeli enum transparent_hugepage_flag flag) 34871e3aac0SAndrea Arcangeli { 349e27e6151SBen Hutchings return sprintf(buf, "%d\n", 350e27e6151SBen Hutchings !!test_bit(flag, &transparent_hugepage_flags)); 35171e3aac0SAndrea Arcangeli } 352e27e6151SBen Hutchings 35371e3aac0SAndrea Arcangeli static ssize_t single_flag_store(struct kobject *kobj, 35471e3aac0SAndrea Arcangeli struct kobj_attribute *attr, 35571e3aac0SAndrea Arcangeli const char *buf, size_t count, 35671e3aac0SAndrea Arcangeli enum transparent_hugepage_flag flag) 35771e3aac0SAndrea Arcangeli { 358e27e6151SBen Hutchings unsigned long value; 359e27e6151SBen Hutchings int ret; 360e27e6151SBen Hutchings 361e27e6151SBen Hutchings ret = kstrtoul(buf, 10, &value); 362e27e6151SBen Hutchings if (ret < 0) 363e27e6151SBen Hutchings return ret; 364e27e6151SBen Hutchings if (value > 1) 36571e3aac0SAndrea Arcangeli return -EINVAL; 36671e3aac0SAndrea Arcangeli 367e27e6151SBen Hutchings if (value) 368e27e6151SBen Hutchings set_bit(flag, &transparent_hugepage_flags); 369e27e6151SBen Hutchings else 370e27e6151SBen Hutchings clear_bit(flag, &transparent_hugepage_flags); 371e27e6151SBen Hutchings 37271e3aac0SAndrea Arcangeli return count; 37371e3aac0SAndrea Arcangeli } 37471e3aac0SAndrea Arcangeli 37571e3aac0SAndrea Arcangeli /* 37671e3aac0SAndrea Arcangeli * Currently defrag only disables __GFP_NOWAIT for allocation. A blind 37771e3aac0SAndrea Arcangeli * __GFP_REPEAT is too aggressive, it's never worth swapping tons of 37871e3aac0SAndrea Arcangeli * memory just to allocate one more hugepage. 37971e3aac0SAndrea Arcangeli */ 38071e3aac0SAndrea Arcangeli static ssize_t defrag_show(struct kobject *kobj, 38171e3aac0SAndrea Arcangeli struct kobj_attribute *attr, char *buf) 38271e3aac0SAndrea Arcangeli { 38371e3aac0SAndrea Arcangeli return double_flag_show(kobj, attr, buf, 38471e3aac0SAndrea Arcangeli TRANSPARENT_HUGEPAGE_DEFRAG_FLAG, 38571e3aac0SAndrea Arcangeli TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG); 38671e3aac0SAndrea Arcangeli } 38771e3aac0SAndrea Arcangeli static ssize_t defrag_store(struct kobject *kobj, 38871e3aac0SAndrea Arcangeli struct kobj_attribute *attr, 38971e3aac0SAndrea Arcangeli const char *buf, size_t count) 39071e3aac0SAndrea Arcangeli { 39171e3aac0SAndrea Arcangeli return double_flag_store(kobj, attr, buf, count, 39271e3aac0SAndrea Arcangeli TRANSPARENT_HUGEPAGE_DEFRAG_FLAG, 39371e3aac0SAndrea Arcangeli TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG); 39471e3aac0SAndrea Arcangeli } 39571e3aac0SAndrea Arcangeli static struct kobj_attribute defrag_attr = 39671e3aac0SAndrea Arcangeli __ATTR(defrag, 0644, defrag_show, defrag_store); 39771e3aac0SAndrea Arcangeli 39879da5407SKirill A. Shutemov static ssize_t use_zero_page_show(struct kobject *kobj, 39979da5407SKirill A. Shutemov struct kobj_attribute *attr, char *buf) 40079da5407SKirill A. Shutemov { 40179da5407SKirill A. Shutemov return single_flag_show(kobj, attr, buf, 40279da5407SKirill A. Shutemov TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 40379da5407SKirill A. Shutemov } 40479da5407SKirill A. Shutemov static ssize_t use_zero_page_store(struct kobject *kobj, 40579da5407SKirill A. Shutemov struct kobj_attribute *attr, const char *buf, size_t count) 40679da5407SKirill A. Shutemov { 40779da5407SKirill A. Shutemov return single_flag_store(kobj, attr, buf, count, 40879da5407SKirill A. Shutemov TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 40979da5407SKirill A. Shutemov } 41079da5407SKirill A. Shutemov static struct kobj_attribute use_zero_page_attr = 41179da5407SKirill A. Shutemov __ATTR(use_zero_page, 0644, use_zero_page_show, use_zero_page_store); 41271e3aac0SAndrea Arcangeli #ifdef CONFIG_DEBUG_VM 41371e3aac0SAndrea Arcangeli static ssize_t debug_cow_show(struct kobject *kobj, 41471e3aac0SAndrea Arcangeli struct kobj_attribute *attr, char *buf) 41571e3aac0SAndrea Arcangeli { 41671e3aac0SAndrea Arcangeli return single_flag_show(kobj, attr, buf, 41771e3aac0SAndrea Arcangeli TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG); 41871e3aac0SAndrea Arcangeli } 41971e3aac0SAndrea Arcangeli static ssize_t debug_cow_store(struct kobject *kobj, 42071e3aac0SAndrea Arcangeli struct kobj_attribute *attr, 42171e3aac0SAndrea Arcangeli const char *buf, size_t count) 42271e3aac0SAndrea Arcangeli { 42371e3aac0SAndrea Arcangeli return single_flag_store(kobj, attr, buf, count, 42471e3aac0SAndrea Arcangeli TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG); 42571e3aac0SAndrea Arcangeli } 42671e3aac0SAndrea Arcangeli static struct kobj_attribute debug_cow_attr = 42771e3aac0SAndrea Arcangeli __ATTR(debug_cow, 0644, debug_cow_show, debug_cow_store); 42871e3aac0SAndrea Arcangeli #endif /* CONFIG_DEBUG_VM */ 42971e3aac0SAndrea Arcangeli 43071e3aac0SAndrea Arcangeli static struct attribute *hugepage_attr[] = { 43171e3aac0SAndrea Arcangeli &enabled_attr.attr, 43271e3aac0SAndrea Arcangeli &defrag_attr.attr, 43379da5407SKirill A. Shutemov &use_zero_page_attr.attr, 43471e3aac0SAndrea Arcangeli #ifdef CONFIG_DEBUG_VM 43571e3aac0SAndrea Arcangeli &debug_cow_attr.attr, 43671e3aac0SAndrea Arcangeli #endif 43771e3aac0SAndrea Arcangeli NULL, 43871e3aac0SAndrea Arcangeli }; 43971e3aac0SAndrea Arcangeli 44071e3aac0SAndrea Arcangeli static struct attribute_group hugepage_attr_group = { 44171e3aac0SAndrea Arcangeli .attrs = hugepage_attr, 442ba76149fSAndrea Arcangeli }; 443ba76149fSAndrea Arcangeli 444ba76149fSAndrea Arcangeli static ssize_t scan_sleep_millisecs_show(struct kobject *kobj, 445ba76149fSAndrea Arcangeli struct kobj_attribute *attr, 446ba76149fSAndrea Arcangeli char *buf) 447ba76149fSAndrea Arcangeli { 448ba76149fSAndrea Arcangeli return sprintf(buf, "%u\n", khugepaged_scan_sleep_millisecs); 449ba76149fSAndrea Arcangeli } 450ba76149fSAndrea Arcangeli 451ba76149fSAndrea Arcangeli static ssize_t scan_sleep_millisecs_store(struct kobject *kobj, 452ba76149fSAndrea Arcangeli struct kobj_attribute *attr, 453ba76149fSAndrea Arcangeli const char *buf, size_t count) 454ba76149fSAndrea Arcangeli { 455ba76149fSAndrea Arcangeli unsigned long msecs; 456ba76149fSAndrea Arcangeli int err; 457ba76149fSAndrea Arcangeli 4583dbb95f7SJingoo Han err = kstrtoul(buf, 10, &msecs); 459ba76149fSAndrea Arcangeli if (err || msecs > UINT_MAX) 460ba76149fSAndrea Arcangeli return -EINVAL; 461ba76149fSAndrea Arcangeli 462ba76149fSAndrea Arcangeli khugepaged_scan_sleep_millisecs = msecs; 463ba76149fSAndrea Arcangeli wake_up_interruptible(&khugepaged_wait); 464ba76149fSAndrea Arcangeli 465ba76149fSAndrea Arcangeli return count; 466ba76149fSAndrea Arcangeli } 467ba76149fSAndrea Arcangeli static struct kobj_attribute scan_sleep_millisecs_attr = 468ba76149fSAndrea Arcangeli __ATTR(scan_sleep_millisecs, 0644, scan_sleep_millisecs_show, 469ba76149fSAndrea Arcangeli scan_sleep_millisecs_store); 470ba76149fSAndrea Arcangeli 471ba76149fSAndrea Arcangeli static ssize_t alloc_sleep_millisecs_show(struct kobject *kobj, 472ba76149fSAndrea Arcangeli struct kobj_attribute *attr, 473ba76149fSAndrea Arcangeli char *buf) 474ba76149fSAndrea Arcangeli { 475ba76149fSAndrea Arcangeli return sprintf(buf, "%u\n", khugepaged_alloc_sleep_millisecs); 476ba76149fSAndrea Arcangeli } 477ba76149fSAndrea Arcangeli 478ba76149fSAndrea Arcangeli static ssize_t alloc_sleep_millisecs_store(struct kobject *kobj, 479ba76149fSAndrea Arcangeli struct kobj_attribute *attr, 480ba76149fSAndrea Arcangeli const char *buf, size_t count) 481ba76149fSAndrea Arcangeli { 482ba76149fSAndrea Arcangeli unsigned long msecs; 483ba76149fSAndrea Arcangeli int err; 484ba76149fSAndrea Arcangeli 4853dbb95f7SJingoo Han err = kstrtoul(buf, 10, &msecs); 486ba76149fSAndrea Arcangeli if (err || msecs > UINT_MAX) 487ba76149fSAndrea Arcangeli return -EINVAL; 488ba76149fSAndrea Arcangeli 489ba76149fSAndrea Arcangeli khugepaged_alloc_sleep_millisecs = msecs; 490ba76149fSAndrea Arcangeli wake_up_interruptible(&khugepaged_wait); 491ba76149fSAndrea Arcangeli 492ba76149fSAndrea Arcangeli return count; 493ba76149fSAndrea Arcangeli } 494ba76149fSAndrea Arcangeli static struct kobj_attribute alloc_sleep_millisecs_attr = 495ba76149fSAndrea Arcangeli __ATTR(alloc_sleep_millisecs, 0644, alloc_sleep_millisecs_show, 496ba76149fSAndrea Arcangeli alloc_sleep_millisecs_store); 497ba76149fSAndrea Arcangeli 498ba76149fSAndrea Arcangeli static ssize_t pages_to_scan_show(struct kobject *kobj, 499ba76149fSAndrea Arcangeli struct kobj_attribute *attr, 500ba76149fSAndrea Arcangeli char *buf) 501ba76149fSAndrea Arcangeli { 502ba76149fSAndrea Arcangeli return sprintf(buf, "%u\n", khugepaged_pages_to_scan); 503ba76149fSAndrea Arcangeli } 504ba76149fSAndrea Arcangeli static ssize_t pages_to_scan_store(struct kobject *kobj, 505ba76149fSAndrea Arcangeli struct kobj_attribute *attr, 506ba76149fSAndrea Arcangeli const char *buf, size_t count) 507ba76149fSAndrea Arcangeli { 508ba76149fSAndrea Arcangeli int err; 509ba76149fSAndrea Arcangeli unsigned long pages; 510ba76149fSAndrea Arcangeli 5113dbb95f7SJingoo Han err = kstrtoul(buf, 10, &pages); 512ba76149fSAndrea Arcangeli if (err || !pages || pages > UINT_MAX) 513ba76149fSAndrea Arcangeli return -EINVAL; 514ba76149fSAndrea Arcangeli 515ba76149fSAndrea Arcangeli khugepaged_pages_to_scan = pages; 516ba76149fSAndrea Arcangeli 517ba76149fSAndrea Arcangeli return count; 518ba76149fSAndrea Arcangeli } 519ba76149fSAndrea Arcangeli static struct kobj_attribute pages_to_scan_attr = 520ba76149fSAndrea Arcangeli __ATTR(pages_to_scan, 0644, pages_to_scan_show, 521ba76149fSAndrea Arcangeli pages_to_scan_store); 522ba76149fSAndrea Arcangeli 523ba76149fSAndrea Arcangeli static ssize_t pages_collapsed_show(struct kobject *kobj, 524ba76149fSAndrea Arcangeli struct kobj_attribute *attr, 525ba76149fSAndrea Arcangeli char *buf) 526ba76149fSAndrea Arcangeli { 527ba76149fSAndrea Arcangeli return sprintf(buf, "%u\n", khugepaged_pages_collapsed); 528ba76149fSAndrea Arcangeli } 529ba76149fSAndrea Arcangeli static struct kobj_attribute pages_collapsed_attr = 530ba76149fSAndrea Arcangeli __ATTR_RO(pages_collapsed); 531ba76149fSAndrea Arcangeli 532ba76149fSAndrea Arcangeli static ssize_t full_scans_show(struct kobject *kobj, 533ba76149fSAndrea Arcangeli struct kobj_attribute *attr, 534ba76149fSAndrea Arcangeli char *buf) 535ba76149fSAndrea Arcangeli { 536ba76149fSAndrea Arcangeli return sprintf(buf, "%u\n", khugepaged_full_scans); 537ba76149fSAndrea Arcangeli } 538ba76149fSAndrea Arcangeli static struct kobj_attribute full_scans_attr = 539ba76149fSAndrea Arcangeli __ATTR_RO(full_scans); 540ba76149fSAndrea Arcangeli 541ba76149fSAndrea Arcangeli static ssize_t khugepaged_defrag_show(struct kobject *kobj, 542ba76149fSAndrea Arcangeli struct kobj_attribute *attr, char *buf) 543ba76149fSAndrea Arcangeli { 544ba76149fSAndrea Arcangeli return single_flag_show(kobj, attr, buf, 545ba76149fSAndrea Arcangeli TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG); 546ba76149fSAndrea Arcangeli } 547ba76149fSAndrea Arcangeli static ssize_t khugepaged_defrag_store(struct kobject *kobj, 548ba76149fSAndrea Arcangeli struct kobj_attribute *attr, 549ba76149fSAndrea Arcangeli const char *buf, size_t count) 550ba76149fSAndrea Arcangeli { 551ba76149fSAndrea Arcangeli return single_flag_store(kobj, attr, buf, count, 552ba76149fSAndrea Arcangeli TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG); 553ba76149fSAndrea Arcangeli } 554ba76149fSAndrea Arcangeli static struct kobj_attribute khugepaged_defrag_attr = 555ba76149fSAndrea Arcangeli __ATTR(defrag, 0644, khugepaged_defrag_show, 556ba76149fSAndrea Arcangeli khugepaged_defrag_store); 557ba76149fSAndrea Arcangeli 558ba76149fSAndrea Arcangeli /* 559ba76149fSAndrea Arcangeli * max_ptes_none controls if khugepaged should collapse hugepages over 560ba76149fSAndrea Arcangeli * any unmapped ptes in turn potentially increasing the memory 561ba76149fSAndrea Arcangeli * footprint of the vmas. When max_ptes_none is 0 khugepaged will not 562ba76149fSAndrea Arcangeli * reduce the available free memory in the system as it 563ba76149fSAndrea Arcangeli * runs. Increasing max_ptes_none will instead potentially reduce the 564ba76149fSAndrea Arcangeli * free memory in the system during the khugepaged scan. 565ba76149fSAndrea Arcangeli */ 566ba76149fSAndrea Arcangeli static ssize_t khugepaged_max_ptes_none_show(struct kobject *kobj, 567ba76149fSAndrea Arcangeli struct kobj_attribute *attr, 568ba76149fSAndrea Arcangeli char *buf) 569ba76149fSAndrea Arcangeli { 570ba76149fSAndrea Arcangeli return sprintf(buf, "%u\n", khugepaged_max_ptes_none); 571ba76149fSAndrea Arcangeli } 572ba76149fSAndrea Arcangeli static ssize_t khugepaged_max_ptes_none_store(struct kobject *kobj, 573ba76149fSAndrea Arcangeli struct kobj_attribute *attr, 574ba76149fSAndrea Arcangeli const char *buf, size_t count) 575ba76149fSAndrea Arcangeli { 576ba76149fSAndrea Arcangeli int err; 577ba76149fSAndrea Arcangeli unsigned long max_ptes_none; 578ba76149fSAndrea Arcangeli 5793dbb95f7SJingoo Han err = kstrtoul(buf, 10, &max_ptes_none); 580ba76149fSAndrea Arcangeli if (err || max_ptes_none > HPAGE_PMD_NR-1) 581ba76149fSAndrea Arcangeli return -EINVAL; 582ba76149fSAndrea Arcangeli 583ba76149fSAndrea Arcangeli khugepaged_max_ptes_none = max_ptes_none; 584ba76149fSAndrea Arcangeli 585ba76149fSAndrea Arcangeli return count; 586ba76149fSAndrea Arcangeli } 587ba76149fSAndrea Arcangeli static struct kobj_attribute khugepaged_max_ptes_none_attr = 588ba76149fSAndrea Arcangeli __ATTR(max_ptes_none, 0644, khugepaged_max_ptes_none_show, 589ba76149fSAndrea Arcangeli khugepaged_max_ptes_none_store); 590ba76149fSAndrea Arcangeli 591ba76149fSAndrea Arcangeli static struct attribute *khugepaged_attr[] = { 592ba76149fSAndrea Arcangeli &khugepaged_defrag_attr.attr, 593ba76149fSAndrea Arcangeli &khugepaged_max_ptes_none_attr.attr, 594ba76149fSAndrea Arcangeli &pages_to_scan_attr.attr, 595ba76149fSAndrea Arcangeli &pages_collapsed_attr.attr, 596ba76149fSAndrea Arcangeli &full_scans_attr.attr, 597ba76149fSAndrea Arcangeli &scan_sleep_millisecs_attr.attr, 598ba76149fSAndrea Arcangeli &alloc_sleep_millisecs_attr.attr, 599ba76149fSAndrea Arcangeli NULL, 600ba76149fSAndrea Arcangeli }; 601ba76149fSAndrea Arcangeli 602ba76149fSAndrea Arcangeli static struct attribute_group khugepaged_attr_group = { 603ba76149fSAndrea Arcangeli .attrs = khugepaged_attr, 604ba76149fSAndrea Arcangeli .name = "khugepaged", 60571e3aac0SAndrea Arcangeli }; 606569e5590SShaohua Li 607569e5590SShaohua Li static int __init hugepage_init_sysfs(struct kobject **hugepage_kobj) 608569e5590SShaohua Li { 609569e5590SShaohua Li int err; 610569e5590SShaohua Li 611569e5590SShaohua Li *hugepage_kobj = kobject_create_and_add("transparent_hugepage", mm_kobj); 612569e5590SShaohua Li if (unlikely(!*hugepage_kobj)) { 613ae3a8c1cSAndrew Morton pr_err("failed to create transparent hugepage kobject\n"); 614569e5590SShaohua Li return -ENOMEM; 615569e5590SShaohua Li } 616569e5590SShaohua Li 617569e5590SShaohua Li err = sysfs_create_group(*hugepage_kobj, &hugepage_attr_group); 618569e5590SShaohua Li if (err) { 619ae3a8c1cSAndrew Morton pr_err("failed to register transparent hugepage group\n"); 620569e5590SShaohua Li goto delete_obj; 621569e5590SShaohua Li } 622569e5590SShaohua Li 623569e5590SShaohua Li err = sysfs_create_group(*hugepage_kobj, &khugepaged_attr_group); 624569e5590SShaohua Li if (err) { 625ae3a8c1cSAndrew Morton pr_err("failed to register transparent hugepage group\n"); 626569e5590SShaohua Li goto remove_hp_group; 627569e5590SShaohua Li } 628569e5590SShaohua Li 629569e5590SShaohua Li return 0; 630569e5590SShaohua Li 631569e5590SShaohua Li remove_hp_group: 632569e5590SShaohua Li sysfs_remove_group(*hugepage_kobj, &hugepage_attr_group); 633569e5590SShaohua Li delete_obj: 634569e5590SShaohua Li kobject_put(*hugepage_kobj); 635569e5590SShaohua Li return err; 636569e5590SShaohua Li } 637569e5590SShaohua Li 638569e5590SShaohua Li static void __init hugepage_exit_sysfs(struct kobject *hugepage_kobj) 639569e5590SShaohua Li { 640569e5590SShaohua Li sysfs_remove_group(hugepage_kobj, &khugepaged_attr_group); 641569e5590SShaohua Li sysfs_remove_group(hugepage_kobj, &hugepage_attr_group); 642569e5590SShaohua Li kobject_put(hugepage_kobj); 643569e5590SShaohua Li } 644569e5590SShaohua Li #else 645569e5590SShaohua Li static inline int hugepage_init_sysfs(struct kobject **hugepage_kobj) 646569e5590SShaohua Li { 647569e5590SShaohua Li return 0; 648569e5590SShaohua Li } 649569e5590SShaohua Li 650569e5590SShaohua Li static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj) 651569e5590SShaohua Li { 652569e5590SShaohua Li } 65371e3aac0SAndrea Arcangeli #endif /* CONFIG_SYSFS */ 65471e3aac0SAndrea Arcangeli 65571e3aac0SAndrea Arcangeli static int __init hugepage_init(void) 65671e3aac0SAndrea Arcangeli { 65771e3aac0SAndrea Arcangeli int err; 658569e5590SShaohua Li struct kobject *hugepage_kobj; 65971e3aac0SAndrea Arcangeli 6604b7167b9SAndrea Arcangeli if (!has_transparent_hugepage()) { 6614b7167b9SAndrea Arcangeli transparent_hugepage_flags = 0; 662569e5590SShaohua Li return -EINVAL; 6634b7167b9SAndrea Arcangeli } 6644b7167b9SAndrea Arcangeli 665569e5590SShaohua Li err = hugepage_init_sysfs(&hugepage_kobj); 666569e5590SShaohua Li if (err) 66765ebb64fSKirill A. Shutemov goto err_sysfs; 668ba76149fSAndrea Arcangeli 669ba76149fSAndrea Arcangeli err = khugepaged_slab_init(); 670ba76149fSAndrea Arcangeli if (err) 67165ebb64fSKirill A. Shutemov goto err_slab; 672ba76149fSAndrea Arcangeli 67365ebb64fSKirill A. Shutemov err = register_shrinker(&huge_zero_page_shrinker); 67465ebb64fSKirill A. Shutemov if (err) 67565ebb64fSKirill A. Shutemov goto err_hzp_shrinker; 6769a982250SKirill A. Shutemov err = register_shrinker(&deferred_split_shrinker); 6779a982250SKirill A. Shutemov if (err) 6789a982250SKirill A. Shutemov goto err_split_shrinker; 67997ae1749SKirill A. Shutemov 68097562cd2SRik van Riel /* 68197562cd2SRik van Riel * By default disable transparent hugepages on smaller systems, 68297562cd2SRik van Riel * where the extra memory used could hurt more than TLB overhead 68397562cd2SRik van Riel * is likely to save. The admin can still enable it through /sys. 68497562cd2SRik van Riel */ 68579553da2SKirill A. Shutemov if (totalram_pages < (512 << (20 - PAGE_SHIFT))) { 68697562cd2SRik van Riel transparent_hugepage_flags = 0; 68779553da2SKirill A. Shutemov return 0; 68879553da2SKirill A. Shutemov } 68997562cd2SRik van Riel 69079553da2SKirill A. Shutemov err = start_stop_khugepaged(); 69165ebb64fSKirill A. Shutemov if (err) 69265ebb64fSKirill A. Shutemov goto err_khugepaged; 693ba76149fSAndrea Arcangeli 694569e5590SShaohua Li return 0; 69565ebb64fSKirill A. Shutemov err_khugepaged: 6969a982250SKirill A. Shutemov unregister_shrinker(&deferred_split_shrinker); 6979a982250SKirill A. Shutemov err_split_shrinker: 69865ebb64fSKirill A. Shutemov unregister_shrinker(&huge_zero_page_shrinker); 69965ebb64fSKirill A. Shutemov err_hzp_shrinker: 70065ebb64fSKirill A. Shutemov khugepaged_slab_exit(); 70165ebb64fSKirill A. Shutemov err_slab: 702569e5590SShaohua Li hugepage_exit_sysfs(hugepage_kobj); 70365ebb64fSKirill A. Shutemov err_sysfs: 704ba76149fSAndrea Arcangeli return err; 70571e3aac0SAndrea Arcangeli } 706a64fb3cdSPaul Gortmaker subsys_initcall(hugepage_init); 70771e3aac0SAndrea Arcangeli 70871e3aac0SAndrea Arcangeli static int __init setup_transparent_hugepage(char *str) 70971e3aac0SAndrea Arcangeli { 71071e3aac0SAndrea Arcangeli int ret = 0; 71171e3aac0SAndrea Arcangeli if (!str) 71271e3aac0SAndrea Arcangeli goto out; 71371e3aac0SAndrea Arcangeli if (!strcmp(str, "always")) { 71471e3aac0SAndrea Arcangeli set_bit(TRANSPARENT_HUGEPAGE_FLAG, 71571e3aac0SAndrea Arcangeli &transparent_hugepage_flags); 71671e3aac0SAndrea Arcangeli clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 71771e3aac0SAndrea Arcangeli &transparent_hugepage_flags); 71871e3aac0SAndrea Arcangeli ret = 1; 71971e3aac0SAndrea Arcangeli } else if (!strcmp(str, "madvise")) { 72071e3aac0SAndrea Arcangeli clear_bit(TRANSPARENT_HUGEPAGE_FLAG, 72171e3aac0SAndrea Arcangeli &transparent_hugepage_flags); 72271e3aac0SAndrea Arcangeli set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 72371e3aac0SAndrea Arcangeli &transparent_hugepage_flags); 72471e3aac0SAndrea Arcangeli ret = 1; 72571e3aac0SAndrea Arcangeli } else if (!strcmp(str, "never")) { 72671e3aac0SAndrea Arcangeli clear_bit(TRANSPARENT_HUGEPAGE_FLAG, 72771e3aac0SAndrea Arcangeli &transparent_hugepage_flags); 72871e3aac0SAndrea Arcangeli clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 72971e3aac0SAndrea Arcangeli &transparent_hugepage_flags); 73071e3aac0SAndrea Arcangeli ret = 1; 73171e3aac0SAndrea Arcangeli } 73271e3aac0SAndrea Arcangeli out: 73371e3aac0SAndrea Arcangeli if (!ret) 734ae3a8c1cSAndrew Morton pr_warn("transparent_hugepage= cannot parse, ignored\n"); 73571e3aac0SAndrea Arcangeli return ret; 73671e3aac0SAndrea Arcangeli } 73771e3aac0SAndrea Arcangeli __setup("transparent_hugepage=", setup_transparent_hugepage); 73871e3aac0SAndrea Arcangeli 739b32967ffSMel Gorman pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma) 74071e3aac0SAndrea Arcangeli { 74171e3aac0SAndrea Arcangeli if (likely(vma->vm_flags & VM_WRITE)) 74271e3aac0SAndrea Arcangeli pmd = pmd_mkwrite(pmd); 74371e3aac0SAndrea Arcangeli return pmd; 74471e3aac0SAndrea Arcangeli } 74571e3aac0SAndrea Arcangeli 7463122359aSKirill A. Shutemov static inline pmd_t mk_huge_pmd(struct page *page, pgprot_t prot) 747b3092b3bSBob Liu { 748b3092b3bSBob Liu pmd_t entry; 7493122359aSKirill A. Shutemov entry = mk_pmd(page, prot); 750b3092b3bSBob Liu entry = pmd_mkhuge(entry); 751b3092b3bSBob Liu return entry; 752b3092b3bSBob Liu } 753b3092b3bSBob Liu 7549a982250SKirill A. Shutemov static inline struct list_head *page_deferred_list(struct page *page) 7559a982250SKirill A. Shutemov { 7569a982250SKirill A. Shutemov /* 7579a982250SKirill A. Shutemov * ->lru in the tail pages is occupied by compound_head. 7589a982250SKirill A. Shutemov * Let's use ->mapping + ->index in the second tail page as list_head. 7599a982250SKirill A. Shutemov */ 7609a982250SKirill A. Shutemov return (struct list_head *)&page[2].mapping; 7619a982250SKirill A. Shutemov } 7629a982250SKirill A. Shutemov 7639a982250SKirill A. Shutemov void prep_transhuge_page(struct page *page) 7649a982250SKirill A. Shutemov { 7659a982250SKirill A. Shutemov /* 7669a982250SKirill A. Shutemov * we use page->mapping and page->indexlru in second tail page 7679a982250SKirill A. Shutemov * as list_head: assuming THP order >= 2 7689a982250SKirill A. Shutemov */ 7699a982250SKirill A. Shutemov BUILD_BUG_ON(HPAGE_PMD_ORDER < 2); 7709a982250SKirill A. Shutemov 7719a982250SKirill A. Shutemov INIT_LIST_HEAD(page_deferred_list(page)); 7729a982250SKirill A. Shutemov set_compound_page_dtor(page, TRANSHUGE_PAGE_DTOR); 7739a982250SKirill A. Shutemov } 7749a982250SKirill A. Shutemov 77571e3aac0SAndrea Arcangeli static int __do_huge_pmd_anonymous_page(struct mm_struct *mm, 77671e3aac0SAndrea Arcangeli struct vm_area_struct *vma, 777230c92a8SAndrea Arcangeli unsigned long address, pmd_t *pmd, 7786b251fc9SAndrea Arcangeli struct page *page, gfp_t gfp, 7796b251fc9SAndrea Arcangeli unsigned int flags) 78071e3aac0SAndrea Arcangeli { 78100501b53SJohannes Weiner struct mem_cgroup *memcg; 78271e3aac0SAndrea Arcangeli pgtable_t pgtable; 783c4088ebdSKirill A. Shutemov spinlock_t *ptl; 784230c92a8SAndrea Arcangeli unsigned long haddr = address & HPAGE_PMD_MASK; 78571e3aac0SAndrea Arcangeli 786309381feSSasha Levin VM_BUG_ON_PAGE(!PageCompound(page), page); 78700501b53SJohannes Weiner 788f627c2f5SKirill A. Shutemov if (mem_cgroup_try_charge(page, mm, gfp, &memcg, true)) { 7896b251fc9SAndrea Arcangeli put_page(page); 7906b251fc9SAndrea Arcangeli count_vm_event(THP_FAULT_FALLBACK); 7916b251fc9SAndrea Arcangeli return VM_FAULT_FALLBACK; 7926b251fc9SAndrea Arcangeli } 79371e3aac0SAndrea Arcangeli 79400501b53SJohannes Weiner pgtable = pte_alloc_one(mm, haddr); 79500501b53SJohannes Weiner if (unlikely(!pgtable)) { 796f627c2f5SKirill A. Shutemov mem_cgroup_cancel_charge(page, memcg, true); 7976b251fc9SAndrea Arcangeli put_page(page); 79800501b53SJohannes Weiner return VM_FAULT_OOM; 79900501b53SJohannes Weiner } 80000501b53SJohannes Weiner 80171e3aac0SAndrea Arcangeli clear_huge_page(page, haddr, HPAGE_PMD_NR); 80252f37629SMinchan Kim /* 80352f37629SMinchan Kim * The memory barrier inside __SetPageUptodate makes sure that 80452f37629SMinchan Kim * clear_huge_page writes become visible before the set_pmd_at() 80552f37629SMinchan Kim * write. 80652f37629SMinchan Kim */ 80771e3aac0SAndrea Arcangeli __SetPageUptodate(page); 80871e3aac0SAndrea Arcangeli 809c4088ebdSKirill A. Shutemov ptl = pmd_lock(mm, pmd); 81071e3aac0SAndrea Arcangeli if (unlikely(!pmd_none(*pmd))) { 811c4088ebdSKirill A. Shutemov spin_unlock(ptl); 812f627c2f5SKirill A. Shutemov mem_cgroup_cancel_charge(page, memcg, true); 81371e3aac0SAndrea Arcangeli put_page(page); 81471e3aac0SAndrea Arcangeli pte_free(mm, pgtable); 81571e3aac0SAndrea Arcangeli } else { 81671e3aac0SAndrea Arcangeli pmd_t entry; 8176b251fc9SAndrea Arcangeli 8186b251fc9SAndrea Arcangeli /* Deliver the page fault to userland */ 8196b251fc9SAndrea Arcangeli if (userfaultfd_missing(vma)) { 8206b251fc9SAndrea Arcangeli int ret; 8216b251fc9SAndrea Arcangeli 8226b251fc9SAndrea Arcangeli spin_unlock(ptl); 823f627c2f5SKirill A. Shutemov mem_cgroup_cancel_charge(page, memcg, true); 8246b251fc9SAndrea Arcangeli put_page(page); 8256b251fc9SAndrea Arcangeli pte_free(mm, pgtable); 826230c92a8SAndrea Arcangeli ret = handle_userfault(vma, address, flags, 8276b251fc9SAndrea Arcangeli VM_UFFD_MISSING); 8286b251fc9SAndrea Arcangeli VM_BUG_ON(ret & VM_FAULT_FALLBACK); 8296b251fc9SAndrea Arcangeli return ret; 8306b251fc9SAndrea Arcangeli } 8316b251fc9SAndrea Arcangeli 8323122359aSKirill A. Shutemov entry = mk_huge_pmd(page, vma->vm_page_prot); 8333122359aSKirill A. Shutemov entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 834d281ee61SKirill A. Shutemov page_add_new_anon_rmap(page, vma, haddr, true); 835f627c2f5SKirill A. Shutemov mem_cgroup_commit_charge(page, memcg, false, true); 83600501b53SJohannes Weiner lru_cache_add_active_or_unevictable(page, vma); 8376b0b50b0SAneesh Kumar K.V pgtable_trans_huge_deposit(mm, pmd, pgtable); 83871e3aac0SAndrea Arcangeli set_pmd_at(mm, haddr, pmd, entry); 83971e3aac0SAndrea Arcangeli add_mm_counter(mm, MM_ANONPAGES, HPAGE_PMD_NR); 840e1f56c89SKirill A. Shutemov atomic_long_inc(&mm->nr_ptes); 841c4088ebdSKirill A. Shutemov spin_unlock(ptl); 8426b251fc9SAndrea Arcangeli count_vm_event(THP_FAULT_ALLOC); 84371e3aac0SAndrea Arcangeli } 84471e3aac0SAndrea Arcangeli 845aa2e878eSDavid Rientjes return 0; 84671e3aac0SAndrea Arcangeli } 84771e3aac0SAndrea Arcangeli 848cc5d462fSAndi Kleen static inline gfp_t alloc_hugepage_gfpmask(int defrag, gfp_t extra_gfp) 8490bbbc0b3SAndrea Arcangeli { 85071baba4bSMel Gorman return (GFP_TRANSHUGE & ~(defrag ? 0 : __GFP_RECLAIM)) | extra_gfp; 8510bbbc0b3SAndrea Arcangeli } 8520bbbc0b3SAndrea Arcangeli 853c4088ebdSKirill A. Shutemov /* Caller must hold page table lock. */ 854d295e341SKirill A. Shutemov static bool set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm, 85597ae1749SKirill A. Shutemov struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd, 8565918d10aSKirill A. Shutemov struct page *zero_page) 857fc9fe822SKirill A. Shutemov { 858fc9fe822SKirill A. Shutemov pmd_t entry; 8597c414164SAndrew Morton if (!pmd_none(*pmd)) 8607c414164SAndrew Morton return false; 8615918d10aSKirill A. Shutemov entry = mk_pmd(zero_page, vma->vm_page_prot); 862fc9fe822SKirill A. Shutemov entry = pmd_mkhuge(entry); 8636b0b50b0SAneesh Kumar K.V pgtable_trans_huge_deposit(mm, pmd, pgtable); 864fc9fe822SKirill A. Shutemov set_pmd_at(mm, haddr, pmd, entry); 865e1f56c89SKirill A. Shutemov atomic_long_inc(&mm->nr_ptes); 8667c414164SAndrew Morton return true; 867fc9fe822SKirill A. Shutemov } 868fc9fe822SKirill A. Shutemov 86971e3aac0SAndrea Arcangeli int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma, 87071e3aac0SAndrea Arcangeli unsigned long address, pmd_t *pmd, 87171e3aac0SAndrea Arcangeli unsigned int flags) 87271e3aac0SAndrea Arcangeli { 873077fcf11SAneesh Kumar K.V gfp_t gfp; 87471e3aac0SAndrea Arcangeli struct page *page; 87571e3aac0SAndrea Arcangeli unsigned long haddr = address & HPAGE_PMD_MASK; 87671e3aac0SAndrea Arcangeli 877128ec037SKirill A. Shutemov if (haddr < vma->vm_start || haddr + HPAGE_PMD_SIZE > vma->vm_end) 878c0292554SKirill A. Shutemov return VM_FAULT_FALLBACK; 87971e3aac0SAndrea Arcangeli if (unlikely(anon_vma_prepare(vma))) 88071e3aac0SAndrea Arcangeli return VM_FAULT_OOM; 8816d50e60cSDavid Rientjes if (unlikely(khugepaged_enter(vma, vma->vm_flags))) 882ba76149fSAndrea Arcangeli return VM_FAULT_OOM; 883593befa6SDominik Dingel if (!(flags & FAULT_FLAG_WRITE) && !mm_forbids_zeropage(mm) && 88479da5407SKirill A. Shutemov transparent_hugepage_use_zero_page()) { 885c4088ebdSKirill A. Shutemov spinlock_t *ptl; 88680371957SKirill A. Shutemov pgtable_t pgtable; 8875918d10aSKirill A. Shutemov struct page *zero_page; 8883ea41e62SKirill A. Shutemov bool set; 8896b251fc9SAndrea Arcangeli int ret; 89080371957SKirill A. Shutemov pgtable = pte_alloc_one(mm, haddr); 89180371957SKirill A. Shutemov if (unlikely(!pgtable)) 89280371957SKirill A. Shutemov return VM_FAULT_OOM; 8935918d10aSKirill A. Shutemov zero_page = get_huge_zero_page(); 8945918d10aSKirill A. Shutemov if (unlikely(!zero_page)) { 89597ae1749SKirill A. Shutemov pte_free(mm, pgtable); 89697ae1749SKirill A. Shutemov count_vm_event(THP_FAULT_FALLBACK); 897c0292554SKirill A. Shutemov return VM_FAULT_FALLBACK; 89897ae1749SKirill A. Shutemov } 899c4088ebdSKirill A. Shutemov ptl = pmd_lock(mm, pmd); 9006b251fc9SAndrea Arcangeli ret = 0; 9016b251fc9SAndrea Arcangeli set = false; 9026b251fc9SAndrea Arcangeli if (pmd_none(*pmd)) { 9036b251fc9SAndrea Arcangeli if (userfaultfd_missing(vma)) { 9046b251fc9SAndrea Arcangeli spin_unlock(ptl); 905230c92a8SAndrea Arcangeli ret = handle_userfault(vma, address, flags, 9066b251fc9SAndrea Arcangeli VM_UFFD_MISSING); 9076b251fc9SAndrea Arcangeli VM_BUG_ON(ret & VM_FAULT_FALLBACK); 9086b251fc9SAndrea Arcangeli } else { 9096b251fc9SAndrea Arcangeli set_huge_zero_page(pgtable, mm, vma, 9106b251fc9SAndrea Arcangeli haddr, pmd, 9115918d10aSKirill A. Shutemov zero_page); 912c4088ebdSKirill A. Shutemov spin_unlock(ptl); 9136b251fc9SAndrea Arcangeli set = true; 9146b251fc9SAndrea Arcangeli } 9156b251fc9SAndrea Arcangeli } else 9166b251fc9SAndrea Arcangeli spin_unlock(ptl); 9173ea41e62SKirill A. Shutemov if (!set) { 9183ea41e62SKirill A. Shutemov pte_free(mm, pgtable); 9193ea41e62SKirill A. Shutemov put_huge_zero_page(); 9203ea41e62SKirill A. Shutemov } 9216b251fc9SAndrea Arcangeli return ret; 92280371957SKirill A. Shutemov } 923077fcf11SAneesh Kumar K.V gfp = alloc_hugepage_gfpmask(transparent_hugepage_defrag(vma), 0); 924077fcf11SAneesh Kumar K.V page = alloc_hugepage_vma(gfp, vma, haddr, HPAGE_PMD_ORDER); 92581ab4201SAndi Kleen if (unlikely(!page)) { 92681ab4201SAndi Kleen count_vm_event(THP_FAULT_FALLBACK); 927c0292554SKirill A. Shutemov return VM_FAULT_FALLBACK; 92881ab4201SAndi Kleen } 9299a982250SKirill A. Shutemov prep_transhuge_page(page); 930230c92a8SAndrea Arcangeli return __do_huge_pmd_anonymous_page(mm, vma, address, pmd, page, gfp, 931230c92a8SAndrea Arcangeli flags); 93271e3aac0SAndrea Arcangeli } 93371e3aac0SAndrea Arcangeli 934ae18d6dcSMatthew Wilcox static void insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr, 935f25748e3SDan Williams pmd_t *pmd, pfn_t pfn, pgprot_t prot, bool write) 9365cad465dSMatthew Wilcox { 9375cad465dSMatthew Wilcox struct mm_struct *mm = vma->vm_mm; 9385cad465dSMatthew Wilcox pmd_t entry; 9395cad465dSMatthew Wilcox spinlock_t *ptl; 9405cad465dSMatthew Wilcox 9415cad465dSMatthew Wilcox ptl = pmd_lock(mm, pmd); 942f25748e3SDan Williams entry = pmd_mkhuge(pfn_t_pmd(pfn, prot)); 943f25748e3SDan Williams if (pfn_t_devmap(pfn)) 944f25748e3SDan Williams entry = pmd_mkdevmap(entry); 9455cad465dSMatthew Wilcox if (write) { 9465cad465dSMatthew Wilcox entry = pmd_mkyoung(pmd_mkdirty(entry)); 9475cad465dSMatthew Wilcox entry = maybe_pmd_mkwrite(entry, vma); 9485cad465dSMatthew Wilcox } 9495cad465dSMatthew Wilcox set_pmd_at(mm, addr, pmd, entry); 9505cad465dSMatthew Wilcox update_mmu_cache_pmd(vma, addr, pmd); 9515cad465dSMatthew Wilcox spin_unlock(ptl); 9525cad465dSMatthew Wilcox } 9535cad465dSMatthew Wilcox 9545cad465dSMatthew Wilcox int vmf_insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr, 955f25748e3SDan Williams pmd_t *pmd, pfn_t pfn, bool write) 9565cad465dSMatthew Wilcox { 9575cad465dSMatthew Wilcox pgprot_t pgprot = vma->vm_page_prot; 9585cad465dSMatthew Wilcox /* 9595cad465dSMatthew Wilcox * If we had pmd_special, we could avoid all these restrictions, 9605cad465dSMatthew Wilcox * but we need to be consistent with PTEs and architectures that 9615cad465dSMatthew Wilcox * can't support a 'special' bit. 9625cad465dSMatthew Wilcox */ 9635cad465dSMatthew Wilcox BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))); 9645cad465dSMatthew Wilcox BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 9655cad465dSMatthew Wilcox (VM_PFNMAP|VM_MIXEDMAP)); 9665cad465dSMatthew Wilcox BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 967f25748e3SDan Williams BUG_ON(!pfn_t_devmap(pfn)); 9685cad465dSMatthew Wilcox 9695cad465dSMatthew Wilcox if (addr < vma->vm_start || addr >= vma->vm_end) 9705cad465dSMatthew Wilcox return VM_FAULT_SIGBUS; 9715cad465dSMatthew Wilcox if (track_pfn_insert(vma, &pgprot, pfn)) 9725cad465dSMatthew Wilcox return VM_FAULT_SIGBUS; 973ae18d6dcSMatthew Wilcox insert_pfn_pmd(vma, addr, pmd, pfn, pgprot, write); 974ae18d6dcSMatthew Wilcox return VM_FAULT_NOPAGE; 9755cad465dSMatthew Wilcox } 9765cad465dSMatthew Wilcox 97771e3aac0SAndrea Arcangeli int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, 97871e3aac0SAndrea Arcangeli pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr, 97971e3aac0SAndrea Arcangeli struct vm_area_struct *vma) 98071e3aac0SAndrea Arcangeli { 981c4088ebdSKirill A. Shutemov spinlock_t *dst_ptl, *src_ptl; 98271e3aac0SAndrea Arcangeli struct page *src_page; 98371e3aac0SAndrea Arcangeli pmd_t pmd; 98471e3aac0SAndrea Arcangeli pgtable_t pgtable; 98571e3aac0SAndrea Arcangeli int ret; 98671e3aac0SAndrea Arcangeli 98771e3aac0SAndrea Arcangeli ret = -ENOMEM; 98871e3aac0SAndrea Arcangeli pgtable = pte_alloc_one(dst_mm, addr); 98971e3aac0SAndrea Arcangeli if (unlikely(!pgtable)) 99071e3aac0SAndrea Arcangeli goto out; 99171e3aac0SAndrea Arcangeli 992c4088ebdSKirill A. Shutemov dst_ptl = pmd_lock(dst_mm, dst_pmd); 993c4088ebdSKirill A. Shutemov src_ptl = pmd_lockptr(src_mm, src_pmd); 994c4088ebdSKirill A. Shutemov spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 99571e3aac0SAndrea Arcangeli 99671e3aac0SAndrea Arcangeli ret = -EAGAIN; 99771e3aac0SAndrea Arcangeli pmd = *src_pmd; 998*5c7fb56eSDan Williams if (unlikely(!pmd_trans_huge(pmd) && !pmd_devmap(pmd))) { 99971e3aac0SAndrea Arcangeli pte_free(dst_mm, pgtable); 100071e3aac0SAndrea Arcangeli goto out_unlock; 100171e3aac0SAndrea Arcangeli } 1002fc9fe822SKirill A. Shutemov /* 1003c4088ebdSKirill A. Shutemov * When page table lock is held, the huge zero pmd should not be 1004fc9fe822SKirill A. Shutemov * under splitting since we don't split the page itself, only pmd to 1005fc9fe822SKirill A. Shutemov * a page table. 1006fc9fe822SKirill A. Shutemov */ 1007fc9fe822SKirill A. Shutemov if (is_huge_zero_pmd(pmd)) { 10085918d10aSKirill A. Shutemov struct page *zero_page; 100997ae1749SKirill A. Shutemov /* 101097ae1749SKirill A. Shutemov * get_huge_zero_page() will never allocate a new page here, 101197ae1749SKirill A. Shutemov * since we already have a zero page to copy. It just takes a 101297ae1749SKirill A. Shutemov * reference. 101397ae1749SKirill A. Shutemov */ 10145918d10aSKirill A. Shutemov zero_page = get_huge_zero_page(); 10156b251fc9SAndrea Arcangeli set_huge_zero_page(pgtable, dst_mm, vma, addr, dst_pmd, 10165918d10aSKirill A. Shutemov zero_page); 1017fc9fe822SKirill A. Shutemov ret = 0; 1018fc9fe822SKirill A. Shutemov goto out_unlock; 1019fc9fe822SKirill A. Shutemov } 1020de466bd6SMel Gorman 1021*5c7fb56eSDan Williams if (pmd_trans_huge(pmd)) { 1022*5c7fb56eSDan Williams /* thp accounting separate from pmd_devmap accounting */ 102371e3aac0SAndrea Arcangeli src_page = pmd_page(pmd); 1024309381feSSasha Levin VM_BUG_ON_PAGE(!PageHead(src_page), src_page); 102571e3aac0SAndrea Arcangeli get_page(src_page); 102653f9263bSKirill A. Shutemov page_dup_rmap(src_page, true); 102771e3aac0SAndrea Arcangeli add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); 1028*5c7fb56eSDan Williams atomic_long_inc(&dst_mm->nr_ptes); 1029*5c7fb56eSDan Williams pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); 1030*5c7fb56eSDan Williams } 103171e3aac0SAndrea Arcangeli 103271e3aac0SAndrea Arcangeli pmdp_set_wrprotect(src_mm, addr, src_pmd); 103371e3aac0SAndrea Arcangeli pmd = pmd_mkold(pmd_wrprotect(pmd)); 103471e3aac0SAndrea Arcangeli set_pmd_at(dst_mm, addr, dst_pmd, pmd); 103571e3aac0SAndrea Arcangeli 103671e3aac0SAndrea Arcangeli ret = 0; 103771e3aac0SAndrea Arcangeli out_unlock: 1038c4088ebdSKirill A. Shutemov spin_unlock(src_ptl); 1039c4088ebdSKirill A. Shutemov spin_unlock(dst_ptl); 104071e3aac0SAndrea Arcangeli out: 104171e3aac0SAndrea Arcangeli return ret; 104271e3aac0SAndrea Arcangeli } 104371e3aac0SAndrea Arcangeli 1044a1dd450bSWill Deacon void huge_pmd_set_accessed(struct mm_struct *mm, 1045a1dd450bSWill Deacon struct vm_area_struct *vma, 1046a1dd450bSWill Deacon unsigned long address, 1047a1dd450bSWill Deacon pmd_t *pmd, pmd_t orig_pmd, 1048a1dd450bSWill Deacon int dirty) 1049a1dd450bSWill Deacon { 1050c4088ebdSKirill A. Shutemov spinlock_t *ptl; 1051a1dd450bSWill Deacon pmd_t entry; 1052a1dd450bSWill Deacon unsigned long haddr; 1053a1dd450bSWill Deacon 1054c4088ebdSKirill A. Shutemov ptl = pmd_lock(mm, pmd); 1055a1dd450bSWill Deacon if (unlikely(!pmd_same(*pmd, orig_pmd))) 1056a1dd450bSWill Deacon goto unlock; 1057a1dd450bSWill Deacon 1058a1dd450bSWill Deacon entry = pmd_mkyoung(orig_pmd); 1059a1dd450bSWill Deacon haddr = address & HPAGE_PMD_MASK; 1060a1dd450bSWill Deacon if (pmdp_set_access_flags(vma, haddr, pmd, entry, dirty)) 1061a1dd450bSWill Deacon update_mmu_cache_pmd(vma, address, pmd); 1062a1dd450bSWill Deacon 1063a1dd450bSWill Deacon unlock: 1064c4088ebdSKirill A. Shutemov spin_unlock(ptl); 1065a1dd450bSWill Deacon } 1066a1dd450bSWill Deacon 106771e3aac0SAndrea Arcangeli static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm, 106871e3aac0SAndrea Arcangeli struct vm_area_struct *vma, 106971e3aac0SAndrea Arcangeli unsigned long address, 107071e3aac0SAndrea Arcangeli pmd_t *pmd, pmd_t orig_pmd, 107171e3aac0SAndrea Arcangeli struct page *page, 107271e3aac0SAndrea Arcangeli unsigned long haddr) 107371e3aac0SAndrea Arcangeli { 107400501b53SJohannes Weiner struct mem_cgroup *memcg; 1075c4088ebdSKirill A. Shutemov spinlock_t *ptl; 107671e3aac0SAndrea Arcangeli pgtable_t pgtable; 107771e3aac0SAndrea Arcangeli pmd_t _pmd; 107871e3aac0SAndrea Arcangeli int ret = 0, i; 107971e3aac0SAndrea Arcangeli struct page **pages; 10802ec74c3eSSagi Grimberg unsigned long mmun_start; /* For mmu_notifiers */ 10812ec74c3eSSagi Grimberg unsigned long mmun_end; /* For mmu_notifiers */ 108271e3aac0SAndrea Arcangeli 108371e3aac0SAndrea Arcangeli pages = kmalloc(sizeof(struct page *) * HPAGE_PMD_NR, 108471e3aac0SAndrea Arcangeli GFP_KERNEL); 108571e3aac0SAndrea Arcangeli if (unlikely(!pages)) { 108671e3aac0SAndrea Arcangeli ret |= VM_FAULT_OOM; 108771e3aac0SAndrea Arcangeli goto out; 108871e3aac0SAndrea Arcangeli } 108971e3aac0SAndrea Arcangeli 109071e3aac0SAndrea Arcangeli for (i = 0; i < HPAGE_PMD_NR; i++) { 1091cc5d462fSAndi Kleen pages[i] = alloc_page_vma_node(GFP_HIGHUSER_MOVABLE | 1092cc5d462fSAndi Kleen __GFP_OTHER_NODE, 109319ee151eSAndi Kleen vma, address, page_to_nid(page)); 1094b9bbfbe3SAndrea Arcangeli if (unlikely(!pages[i] || 109500501b53SJohannes Weiner mem_cgroup_try_charge(pages[i], mm, GFP_KERNEL, 1096f627c2f5SKirill A. Shutemov &memcg, false))) { 1097b9bbfbe3SAndrea Arcangeli if (pages[i]) 109871e3aac0SAndrea Arcangeli put_page(pages[i]); 1099b9bbfbe3SAndrea Arcangeli while (--i >= 0) { 110000501b53SJohannes Weiner memcg = (void *)page_private(pages[i]); 110100501b53SJohannes Weiner set_page_private(pages[i], 0); 1102f627c2f5SKirill A. Shutemov mem_cgroup_cancel_charge(pages[i], memcg, 1103f627c2f5SKirill A. Shutemov false); 1104b9bbfbe3SAndrea Arcangeli put_page(pages[i]); 1105b9bbfbe3SAndrea Arcangeli } 110671e3aac0SAndrea Arcangeli kfree(pages); 110771e3aac0SAndrea Arcangeli ret |= VM_FAULT_OOM; 110871e3aac0SAndrea Arcangeli goto out; 110971e3aac0SAndrea Arcangeli } 111000501b53SJohannes Weiner set_page_private(pages[i], (unsigned long)memcg); 111171e3aac0SAndrea Arcangeli } 111271e3aac0SAndrea Arcangeli 111371e3aac0SAndrea Arcangeli for (i = 0; i < HPAGE_PMD_NR; i++) { 111471e3aac0SAndrea Arcangeli copy_user_highpage(pages[i], page + i, 11150089e485SHillf Danton haddr + PAGE_SIZE * i, vma); 111671e3aac0SAndrea Arcangeli __SetPageUptodate(pages[i]); 111771e3aac0SAndrea Arcangeli cond_resched(); 111871e3aac0SAndrea Arcangeli } 111971e3aac0SAndrea Arcangeli 11202ec74c3eSSagi Grimberg mmun_start = haddr; 11212ec74c3eSSagi Grimberg mmun_end = haddr + HPAGE_PMD_SIZE; 11222ec74c3eSSagi Grimberg mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 11232ec74c3eSSagi Grimberg 1124c4088ebdSKirill A. Shutemov ptl = pmd_lock(mm, pmd); 112571e3aac0SAndrea Arcangeli if (unlikely(!pmd_same(*pmd, orig_pmd))) 112671e3aac0SAndrea Arcangeli goto out_free_pages; 1127309381feSSasha Levin VM_BUG_ON_PAGE(!PageHead(page), page); 112871e3aac0SAndrea Arcangeli 11298809aa2dSAneesh Kumar K.V pmdp_huge_clear_flush_notify(vma, haddr, pmd); 113071e3aac0SAndrea Arcangeli /* leave pmd empty until pte is filled */ 113171e3aac0SAndrea Arcangeli 11326b0b50b0SAneesh Kumar K.V pgtable = pgtable_trans_huge_withdraw(mm, pmd); 113371e3aac0SAndrea Arcangeli pmd_populate(mm, &_pmd, pgtable); 113471e3aac0SAndrea Arcangeli 113571e3aac0SAndrea Arcangeli for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) { 113671e3aac0SAndrea Arcangeli pte_t *pte, entry; 113771e3aac0SAndrea Arcangeli entry = mk_pte(pages[i], vma->vm_page_prot); 113871e3aac0SAndrea Arcangeli entry = maybe_mkwrite(pte_mkdirty(entry), vma); 113900501b53SJohannes Weiner memcg = (void *)page_private(pages[i]); 114000501b53SJohannes Weiner set_page_private(pages[i], 0); 1141d281ee61SKirill A. Shutemov page_add_new_anon_rmap(pages[i], vma, haddr, false); 1142f627c2f5SKirill A. Shutemov mem_cgroup_commit_charge(pages[i], memcg, false, false); 114300501b53SJohannes Weiner lru_cache_add_active_or_unevictable(pages[i], vma); 114471e3aac0SAndrea Arcangeli pte = pte_offset_map(&_pmd, haddr); 114571e3aac0SAndrea Arcangeli VM_BUG_ON(!pte_none(*pte)); 114671e3aac0SAndrea Arcangeli set_pte_at(mm, haddr, pte, entry); 114771e3aac0SAndrea Arcangeli pte_unmap(pte); 114871e3aac0SAndrea Arcangeli } 114971e3aac0SAndrea Arcangeli kfree(pages); 115071e3aac0SAndrea Arcangeli 115171e3aac0SAndrea Arcangeli smp_wmb(); /* make pte visible before pmd */ 115271e3aac0SAndrea Arcangeli pmd_populate(mm, pmd, pgtable); 1153d281ee61SKirill A. Shutemov page_remove_rmap(page, true); 1154c4088ebdSKirill A. Shutemov spin_unlock(ptl); 115571e3aac0SAndrea Arcangeli 11562ec74c3eSSagi Grimberg mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 11572ec74c3eSSagi Grimberg 115871e3aac0SAndrea Arcangeli ret |= VM_FAULT_WRITE; 115971e3aac0SAndrea Arcangeli put_page(page); 116071e3aac0SAndrea Arcangeli 116171e3aac0SAndrea Arcangeli out: 116271e3aac0SAndrea Arcangeli return ret; 116371e3aac0SAndrea Arcangeli 116471e3aac0SAndrea Arcangeli out_free_pages: 1165c4088ebdSKirill A. Shutemov spin_unlock(ptl); 11662ec74c3eSSagi Grimberg mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 1167b9bbfbe3SAndrea Arcangeli for (i = 0; i < HPAGE_PMD_NR; i++) { 116800501b53SJohannes Weiner memcg = (void *)page_private(pages[i]); 116900501b53SJohannes Weiner set_page_private(pages[i], 0); 1170f627c2f5SKirill A. Shutemov mem_cgroup_cancel_charge(pages[i], memcg, false); 117171e3aac0SAndrea Arcangeli put_page(pages[i]); 1172b9bbfbe3SAndrea Arcangeli } 117371e3aac0SAndrea Arcangeli kfree(pages); 117471e3aac0SAndrea Arcangeli goto out; 117571e3aac0SAndrea Arcangeli } 117671e3aac0SAndrea Arcangeli 117771e3aac0SAndrea Arcangeli int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma, 117871e3aac0SAndrea Arcangeli unsigned long address, pmd_t *pmd, pmd_t orig_pmd) 117971e3aac0SAndrea Arcangeli { 1180c4088ebdSKirill A. Shutemov spinlock_t *ptl; 118171e3aac0SAndrea Arcangeli int ret = 0; 118293b4796dSKirill A. Shutemov struct page *page = NULL, *new_page; 118300501b53SJohannes Weiner struct mem_cgroup *memcg; 118471e3aac0SAndrea Arcangeli unsigned long haddr; 11852ec74c3eSSagi Grimberg unsigned long mmun_start; /* For mmu_notifiers */ 11862ec74c3eSSagi Grimberg unsigned long mmun_end; /* For mmu_notifiers */ 11873b363692SMichal Hocko gfp_t huge_gfp; /* for allocation and charge */ 118871e3aac0SAndrea Arcangeli 1189c4088ebdSKirill A. Shutemov ptl = pmd_lockptr(mm, pmd); 119081d1b09cSSasha Levin VM_BUG_ON_VMA(!vma->anon_vma, vma); 119193b4796dSKirill A. Shutemov haddr = address & HPAGE_PMD_MASK; 119293b4796dSKirill A. Shutemov if (is_huge_zero_pmd(orig_pmd)) 119393b4796dSKirill A. Shutemov goto alloc; 1194c4088ebdSKirill A. Shutemov spin_lock(ptl); 119571e3aac0SAndrea Arcangeli if (unlikely(!pmd_same(*pmd, orig_pmd))) 119671e3aac0SAndrea Arcangeli goto out_unlock; 119771e3aac0SAndrea Arcangeli 119871e3aac0SAndrea Arcangeli page = pmd_page(orig_pmd); 1199309381feSSasha Levin VM_BUG_ON_PAGE(!PageCompound(page) || !PageHead(page), page); 12001f25fe20SKirill A. Shutemov /* 12011f25fe20SKirill A. Shutemov * We can only reuse the page if nobody else maps the huge page or it's 12021f25fe20SKirill A. Shutemov * part. We can do it by checking page_mapcount() on each sub-page, but 12031f25fe20SKirill A. Shutemov * it's expensive. 12041f25fe20SKirill A. Shutemov * The cheaper way is to check page_count() to be equal 1: every 12051f25fe20SKirill A. Shutemov * mapcount takes page reference reference, so this way we can 12061f25fe20SKirill A. Shutemov * guarantee, that the PMD is the only mapping. 12071f25fe20SKirill A. Shutemov * This can give false negative if somebody pinned the page, but that's 12081f25fe20SKirill A. Shutemov * fine. 12091f25fe20SKirill A. Shutemov */ 12101f25fe20SKirill A. Shutemov if (page_mapcount(page) == 1 && page_count(page) == 1) { 121171e3aac0SAndrea Arcangeli pmd_t entry; 121271e3aac0SAndrea Arcangeli entry = pmd_mkyoung(orig_pmd); 121371e3aac0SAndrea Arcangeli entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 121471e3aac0SAndrea Arcangeli if (pmdp_set_access_flags(vma, haddr, pmd, entry, 1)) 1215b113da65SDavid Miller update_mmu_cache_pmd(vma, address, pmd); 121671e3aac0SAndrea Arcangeli ret |= VM_FAULT_WRITE; 121771e3aac0SAndrea Arcangeli goto out_unlock; 121871e3aac0SAndrea Arcangeli } 1219ddc58f27SKirill A. Shutemov get_page(page); 1220c4088ebdSKirill A. Shutemov spin_unlock(ptl); 122193b4796dSKirill A. Shutemov alloc: 122271e3aac0SAndrea Arcangeli if (transparent_hugepage_enabled(vma) && 1223077fcf11SAneesh Kumar K.V !transparent_hugepage_debug_cow()) { 12243b363692SMichal Hocko huge_gfp = alloc_hugepage_gfpmask(transparent_hugepage_defrag(vma), 0); 12253b363692SMichal Hocko new_page = alloc_hugepage_vma(huge_gfp, vma, haddr, HPAGE_PMD_ORDER); 1226077fcf11SAneesh Kumar K.V } else 122771e3aac0SAndrea Arcangeli new_page = NULL; 122871e3aac0SAndrea Arcangeli 12299a982250SKirill A. Shutemov if (likely(new_page)) { 12309a982250SKirill A. Shutemov prep_transhuge_page(new_page); 12319a982250SKirill A. Shutemov } else { 1232eecc1e42SHugh Dickins if (!page) { 123378ddc534SKirill A. Shutemov split_huge_pmd(vma, pmd, address); 1234e9b71ca9SKirill A. Shutemov ret |= VM_FAULT_FALLBACK; 123593b4796dSKirill A. Shutemov } else { 123671e3aac0SAndrea Arcangeli ret = do_huge_pmd_wp_page_fallback(mm, vma, address, 123771e3aac0SAndrea Arcangeli pmd, orig_pmd, page, haddr); 12389845cbbdSKirill A. Shutemov if (ret & VM_FAULT_OOM) { 123978ddc534SKirill A. Shutemov split_huge_pmd(vma, pmd, address); 12409845cbbdSKirill A. Shutemov ret |= VM_FAULT_FALLBACK; 12419845cbbdSKirill A. Shutemov } 1242ddc58f27SKirill A. Shutemov put_page(page); 124393b4796dSKirill A. Shutemov } 124417766ddeSDavid Rientjes count_vm_event(THP_FAULT_FALLBACK); 124571e3aac0SAndrea Arcangeli goto out; 124671e3aac0SAndrea Arcangeli } 124771e3aac0SAndrea Arcangeli 1248f627c2f5SKirill A. Shutemov if (unlikely(mem_cgroup_try_charge(new_page, mm, huge_gfp, &memcg, 1249f627c2f5SKirill A. Shutemov true))) { 1250b9bbfbe3SAndrea Arcangeli put_page(new_page); 125193b4796dSKirill A. Shutemov if (page) { 125278ddc534SKirill A. Shutemov split_huge_pmd(vma, pmd, address); 1253ddc58f27SKirill A. Shutemov put_page(page); 12549845cbbdSKirill A. Shutemov } else 125578ddc534SKirill A. Shutemov split_huge_pmd(vma, pmd, address); 12569845cbbdSKirill A. Shutemov ret |= VM_FAULT_FALLBACK; 125717766ddeSDavid Rientjes count_vm_event(THP_FAULT_FALLBACK); 1258b9bbfbe3SAndrea Arcangeli goto out; 1259b9bbfbe3SAndrea Arcangeli } 1260b9bbfbe3SAndrea Arcangeli 126117766ddeSDavid Rientjes count_vm_event(THP_FAULT_ALLOC); 126217766ddeSDavid Rientjes 1263eecc1e42SHugh Dickins if (!page) 126493b4796dSKirill A. Shutemov clear_huge_page(new_page, haddr, HPAGE_PMD_NR); 126593b4796dSKirill A. Shutemov else 126671e3aac0SAndrea Arcangeli copy_user_huge_page(new_page, page, haddr, vma, HPAGE_PMD_NR); 126771e3aac0SAndrea Arcangeli __SetPageUptodate(new_page); 126871e3aac0SAndrea Arcangeli 12692ec74c3eSSagi Grimberg mmun_start = haddr; 12702ec74c3eSSagi Grimberg mmun_end = haddr + HPAGE_PMD_SIZE; 12712ec74c3eSSagi Grimberg mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 12722ec74c3eSSagi Grimberg 1273c4088ebdSKirill A. Shutemov spin_lock(ptl); 127493b4796dSKirill A. Shutemov if (page) 1275ddc58f27SKirill A. Shutemov put_page(page); 1276b9bbfbe3SAndrea Arcangeli if (unlikely(!pmd_same(*pmd, orig_pmd))) { 1277c4088ebdSKirill A. Shutemov spin_unlock(ptl); 1278f627c2f5SKirill A. Shutemov mem_cgroup_cancel_charge(new_page, memcg, true); 127971e3aac0SAndrea Arcangeli put_page(new_page); 12802ec74c3eSSagi Grimberg goto out_mn; 1281b9bbfbe3SAndrea Arcangeli } else { 128271e3aac0SAndrea Arcangeli pmd_t entry; 12833122359aSKirill A. Shutemov entry = mk_huge_pmd(new_page, vma->vm_page_prot); 12843122359aSKirill A. Shutemov entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 12858809aa2dSAneesh Kumar K.V pmdp_huge_clear_flush_notify(vma, haddr, pmd); 1286d281ee61SKirill A. Shutemov page_add_new_anon_rmap(new_page, vma, haddr, true); 1287f627c2f5SKirill A. Shutemov mem_cgroup_commit_charge(new_page, memcg, false, true); 128800501b53SJohannes Weiner lru_cache_add_active_or_unevictable(new_page, vma); 128971e3aac0SAndrea Arcangeli set_pmd_at(mm, haddr, pmd, entry); 1290b113da65SDavid Miller update_mmu_cache_pmd(vma, address, pmd); 1291eecc1e42SHugh Dickins if (!page) { 129293b4796dSKirill A. Shutemov add_mm_counter(mm, MM_ANONPAGES, HPAGE_PMD_NR); 129397ae1749SKirill A. Shutemov put_huge_zero_page(); 129497ae1749SKirill A. Shutemov } else { 1295309381feSSasha Levin VM_BUG_ON_PAGE(!PageHead(page), page); 1296d281ee61SKirill A. Shutemov page_remove_rmap(page, true); 129771e3aac0SAndrea Arcangeli put_page(page); 129893b4796dSKirill A. Shutemov } 129971e3aac0SAndrea Arcangeli ret |= VM_FAULT_WRITE; 130071e3aac0SAndrea Arcangeli } 1301c4088ebdSKirill A. Shutemov spin_unlock(ptl); 13022ec74c3eSSagi Grimberg out_mn: 13032ec74c3eSSagi Grimberg mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 13042ec74c3eSSagi Grimberg out: 13052ec74c3eSSagi Grimberg return ret; 130671e3aac0SAndrea Arcangeli out_unlock: 1307c4088ebdSKirill A. Shutemov spin_unlock(ptl); 130871e3aac0SAndrea Arcangeli return ret; 130971e3aac0SAndrea Arcangeli } 131071e3aac0SAndrea Arcangeli 1311b676b293SDavid Rientjes struct page *follow_trans_huge_pmd(struct vm_area_struct *vma, 131271e3aac0SAndrea Arcangeli unsigned long addr, 131371e3aac0SAndrea Arcangeli pmd_t *pmd, 131471e3aac0SAndrea Arcangeli unsigned int flags) 131571e3aac0SAndrea Arcangeli { 1316b676b293SDavid Rientjes struct mm_struct *mm = vma->vm_mm; 131771e3aac0SAndrea Arcangeli struct page *page = NULL; 131871e3aac0SAndrea Arcangeli 1319c4088ebdSKirill A. Shutemov assert_spin_locked(pmd_lockptr(mm, pmd)); 132071e3aac0SAndrea Arcangeli 132171e3aac0SAndrea Arcangeli if (flags & FOLL_WRITE && !pmd_write(*pmd)) 132271e3aac0SAndrea Arcangeli goto out; 132371e3aac0SAndrea Arcangeli 132485facf25SKirill A. Shutemov /* Avoid dumping huge zero page */ 132585facf25SKirill A. Shutemov if ((flags & FOLL_DUMP) && is_huge_zero_pmd(*pmd)) 132685facf25SKirill A. Shutemov return ERR_PTR(-EFAULT); 132785facf25SKirill A. Shutemov 13282b4847e7SMel Gorman /* Full NUMA hinting faults to serialise migration in fault paths */ 13298a0516edSMel Gorman if ((flags & FOLL_NUMA) && pmd_protnone(*pmd)) 13302b4847e7SMel Gorman goto out; 13312b4847e7SMel Gorman 133271e3aac0SAndrea Arcangeli page = pmd_page(*pmd); 1333309381feSSasha Levin VM_BUG_ON_PAGE(!PageHead(page), page); 133471e3aac0SAndrea Arcangeli if (flags & FOLL_TOUCH) { 133571e3aac0SAndrea Arcangeli pmd_t _pmd; 133671e3aac0SAndrea Arcangeli /* 133771e3aac0SAndrea Arcangeli * We should set the dirty bit only for FOLL_WRITE but 133871e3aac0SAndrea Arcangeli * for now the dirty bit in the pmd is meaningless. 133971e3aac0SAndrea Arcangeli * And if the dirty bit will become meaningful and 134071e3aac0SAndrea Arcangeli * we'll only set it with FOLL_WRITE, an atomic 134171e3aac0SAndrea Arcangeli * set_bit will be required on the pmd to set the 134271e3aac0SAndrea Arcangeli * young bit, instead of the current set_pmd_at. 134371e3aac0SAndrea Arcangeli */ 134471e3aac0SAndrea Arcangeli _pmd = pmd_mkyoung(pmd_mkdirty(*pmd)); 13458663890aSAneesh Kumar K.V if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK, 13468663890aSAneesh Kumar K.V pmd, _pmd, 1)) 13478663890aSAneesh Kumar K.V update_mmu_cache_pmd(vma, addr, pmd); 134871e3aac0SAndrea Arcangeli } 1349de60f5f1SEric B Munson if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) { 1350e90309c9SKirill A. Shutemov /* 1351e90309c9SKirill A. Shutemov * We don't mlock() pte-mapped THPs. This way we can avoid 1352e90309c9SKirill A. Shutemov * leaking mlocked pages into non-VM_LOCKED VMAs. 1353e90309c9SKirill A. Shutemov * 1354e90309c9SKirill A. Shutemov * In most cases the pmd is the only mapping of the page as we 1355e90309c9SKirill A. Shutemov * break COW for the mlock() -- see gup_flags |= FOLL_WRITE for 1356e90309c9SKirill A. Shutemov * writable private mappings in populate_vma_page_range(). 1357e90309c9SKirill A. Shutemov * 1358e90309c9SKirill A. Shutemov * The only scenario when we have the page shared here is if we 1359e90309c9SKirill A. Shutemov * mlocking read-only mapping shared over fork(). We skip 1360e90309c9SKirill A. Shutemov * mlocking such pages. 1361e90309c9SKirill A. Shutemov */ 1362e90309c9SKirill A. Shutemov if (compound_mapcount(page) == 1 && !PageDoubleMap(page) && 1363e90309c9SKirill A. Shutemov page->mapping && trylock_page(page)) { 1364b676b293SDavid Rientjes lru_add_drain(); 1365b676b293SDavid Rientjes if (page->mapping) 1366b676b293SDavid Rientjes mlock_vma_page(page); 1367b676b293SDavid Rientjes unlock_page(page); 1368b676b293SDavid Rientjes } 1369b676b293SDavid Rientjes } 137071e3aac0SAndrea Arcangeli page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT; 1371309381feSSasha Levin VM_BUG_ON_PAGE(!PageCompound(page), page); 137271e3aac0SAndrea Arcangeli if (flags & FOLL_GET) 1373ddc58f27SKirill A. Shutemov get_page(page); 137471e3aac0SAndrea Arcangeli 137571e3aac0SAndrea Arcangeli out: 137671e3aac0SAndrea Arcangeli return page; 137771e3aac0SAndrea Arcangeli } 137871e3aac0SAndrea Arcangeli 1379d10e63f2SMel Gorman /* NUMA hinting page fault entry point for trans huge pmds */ 13804daae3b4SMel Gorman int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma, 13814daae3b4SMel Gorman unsigned long addr, pmd_t pmd, pmd_t *pmdp) 1382d10e63f2SMel Gorman { 1383c4088ebdSKirill A. Shutemov spinlock_t *ptl; 1384b8916634SMel Gorman struct anon_vma *anon_vma = NULL; 1385b32967ffSMel Gorman struct page *page; 1386d10e63f2SMel Gorman unsigned long haddr = addr & HPAGE_PMD_MASK; 13878191acbdSMel Gorman int page_nid = -1, this_nid = numa_node_id(); 138890572890SPeter Zijlstra int target_nid, last_cpupid = -1; 13898191acbdSMel Gorman bool page_locked; 13908191acbdSMel Gorman bool migrated = false; 1391b191f9b1SMel Gorman bool was_writable; 13926688cc05SPeter Zijlstra int flags = 0; 1393d10e63f2SMel Gorman 1394c0e7cad9SMel Gorman /* A PROT_NONE fault should not end up here */ 1395c0e7cad9SMel Gorman BUG_ON(!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))); 1396c0e7cad9SMel Gorman 1397c4088ebdSKirill A. Shutemov ptl = pmd_lock(mm, pmdp); 1398d10e63f2SMel Gorman if (unlikely(!pmd_same(pmd, *pmdp))) 1399d10e63f2SMel Gorman goto out_unlock; 1400d10e63f2SMel Gorman 1401de466bd6SMel Gorman /* 1402de466bd6SMel Gorman * If there are potential migrations, wait for completion and retry 1403de466bd6SMel Gorman * without disrupting NUMA hinting information. Do not relock and 1404de466bd6SMel Gorman * check_same as the page may no longer be mapped. 1405de466bd6SMel Gorman */ 1406de466bd6SMel Gorman if (unlikely(pmd_trans_migrating(*pmdp))) { 14075d833062SMel Gorman page = pmd_page(*pmdp); 1408de466bd6SMel Gorman spin_unlock(ptl); 14095d833062SMel Gorman wait_on_page_locked(page); 1410de466bd6SMel Gorman goto out; 1411de466bd6SMel Gorman } 1412de466bd6SMel Gorman 1413d10e63f2SMel Gorman page = pmd_page(pmd); 1414a1a46184SMel Gorman BUG_ON(is_huge_zero_page(page)); 14158191acbdSMel Gorman page_nid = page_to_nid(page); 141690572890SPeter Zijlstra last_cpupid = page_cpupid_last(page); 141703c5a6e1SMel Gorman count_vm_numa_event(NUMA_HINT_FAULTS); 141804bb2f94SRik van Riel if (page_nid == this_nid) { 141903c5a6e1SMel Gorman count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL); 142004bb2f94SRik van Riel flags |= TNF_FAULT_LOCAL; 142104bb2f94SRik van Riel } 14224daae3b4SMel Gorman 1423bea66fbdSMel Gorman /* See similar comment in do_numa_page for explanation */ 1424bea66fbdSMel Gorman if (!(vma->vm_flags & VM_WRITE)) 14256688cc05SPeter Zijlstra flags |= TNF_NO_GROUP; 14266688cc05SPeter Zijlstra 14276688cc05SPeter Zijlstra /* 1428ff9042b1SMel Gorman * Acquire the page lock to serialise THP migrations but avoid dropping 1429ff9042b1SMel Gorman * page_table_lock if at all possible 1430ff9042b1SMel Gorman */ 1431b8916634SMel Gorman page_locked = trylock_page(page); 1432b8916634SMel Gorman target_nid = mpol_misplaced(page, vma, haddr); 1433b8916634SMel Gorman if (target_nid == -1) { 1434b8916634SMel Gorman /* If the page was locked, there are no parallel migrations */ 1435a54a407fSMel Gorman if (page_locked) 1436b8916634SMel Gorman goto clear_pmdnuma; 14372b4847e7SMel Gorman } 1438cbee9f88SPeter Zijlstra 1439de466bd6SMel Gorman /* Migration could have started since the pmd_trans_migrating check */ 14402b4847e7SMel Gorman if (!page_locked) { 1441c4088ebdSKirill A. Shutemov spin_unlock(ptl); 1442b8916634SMel Gorman wait_on_page_locked(page); 1443a54a407fSMel Gorman page_nid = -1; 1444b8916634SMel Gorman goto out; 1445b8916634SMel Gorman } 1446b8916634SMel Gorman 14472b4847e7SMel Gorman /* 14482b4847e7SMel Gorman * Page is misplaced. Page lock serialises migrations. Acquire anon_vma 14492b4847e7SMel Gorman * to serialises splits 14502b4847e7SMel Gorman */ 1451b8916634SMel Gorman get_page(page); 1452c4088ebdSKirill A. Shutemov spin_unlock(ptl); 1453b8916634SMel Gorman anon_vma = page_lock_anon_vma_read(page); 1454b32967ffSMel Gorman 1455c69307d5SPeter Zijlstra /* Confirm the PMD did not change while page_table_lock was released */ 1456c4088ebdSKirill A. Shutemov spin_lock(ptl); 1457b32967ffSMel Gorman if (unlikely(!pmd_same(pmd, *pmdp))) { 1458b32967ffSMel Gorman unlock_page(page); 1459b32967ffSMel Gorman put_page(page); 1460a54a407fSMel Gorman page_nid = -1; 1461b32967ffSMel Gorman goto out_unlock; 1462b32967ffSMel Gorman } 1463ff9042b1SMel Gorman 1464c3a489caSMel Gorman /* Bail if we fail to protect against THP splits for any reason */ 1465c3a489caSMel Gorman if (unlikely(!anon_vma)) { 1466c3a489caSMel Gorman put_page(page); 1467c3a489caSMel Gorman page_nid = -1; 1468c3a489caSMel Gorman goto clear_pmdnuma; 1469c3a489caSMel Gorman } 1470c3a489caSMel Gorman 1471a54a407fSMel Gorman /* 1472a54a407fSMel Gorman * Migrate the THP to the requested node, returns with page unlocked 14738a0516edSMel Gorman * and access rights restored. 1474a54a407fSMel Gorman */ 1475c4088ebdSKirill A. Shutemov spin_unlock(ptl); 1476b32967ffSMel Gorman migrated = migrate_misplaced_transhuge_page(mm, vma, 1477340ef390SHugh Dickins pmdp, pmd, addr, page, target_nid); 14786688cc05SPeter Zijlstra if (migrated) { 14796688cc05SPeter Zijlstra flags |= TNF_MIGRATED; 14808191acbdSMel Gorman page_nid = target_nid; 1481074c2381SMel Gorman } else 1482074c2381SMel Gorman flags |= TNF_MIGRATE_FAIL; 1483b32967ffSMel Gorman 14848191acbdSMel Gorman goto out; 14854daae3b4SMel Gorman clear_pmdnuma: 1486a54a407fSMel Gorman BUG_ON(!PageLocked(page)); 1487b191f9b1SMel Gorman was_writable = pmd_write(pmd); 14884d942466SMel Gorman pmd = pmd_modify(pmd, vma->vm_page_prot); 1489b7b04004SMel Gorman pmd = pmd_mkyoung(pmd); 1490b191f9b1SMel Gorman if (was_writable) 1491b191f9b1SMel Gorman pmd = pmd_mkwrite(pmd); 1492d10e63f2SMel Gorman set_pmd_at(mm, haddr, pmdp, pmd); 1493d10e63f2SMel Gorman update_mmu_cache_pmd(vma, addr, pmdp); 1494a54a407fSMel Gorman unlock_page(page); 1495d10e63f2SMel Gorman out_unlock: 1496c4088ebdSKirill A. Shutemov spin_unlock(ptl); 1497b8916634SMel Gorman 1498b8916634SMel Gorman out: 1499b8916634SMel Gorman if (anon_vma) 1500b8916634SMel Gorman page_unlock_anon_vma_read(anon_vma); 1501b8916634SMel Gorman 15028191acbdSMel Gorman if (page_nid != -1) 15036688cc05SPeter Zijlstra task_numa_fault(last_cpupid, page_nid, HPAGE_PMD_NR, flags); 15048191acbdSMel Gorman 1505d10e63f2SMel Gorman return 0; 1506d10e63f2SMel Gorman } 1507d10e63f2SMel Gorman 1508b8d3c4c3SMinchan Kim int madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 1509b8d3c4c3SMinchan Kim pmd_t *pmd, unsigned long addr, unsigned long next) 1510b8d3c4c3SMinchan Kim 1511b8d3c4c3SMinchan Kim { 1512b8d3c4c3SMinchan Kim spinlock_t *ptl; 1513b8d3c4c3SMinchan Kim pmd_t orig_pmd; 1514b8d3c4c3SMinchan Kim struct page *page; 1515b8d3c4c3SMinchan Kim struct mm_struct *mm = tlb->mm; 1516b8d3c4c3SMinchan Kim int ret = 0; 1517b8d3c4c3SMinchan Kim 1518b8d3c4c3SMinchan Kim if (!pmd_trans_huge_lock(pmd, vma, &ptl)) 1519b8d3c4c3SMinchan Kim goto out; 1520b8d3c4c3SMinchan Kim 1521b8d3c4c3SMinchan Kim orig_pmd = *pmd; 1522b8d3c4c3SMinchan Kim if (is_huge_zero_pmd(orig_pmd)) { 1523b8d3c4c3SMinchan Kim ret = 1; 1524b8d3c4c3SMinchan Kim goto out; 1525b8d3c4c3SMinchan Kim } 1526b8d3c4c3SMinchan Kim 1527b8d3c4c3SMinchan Kim page = pmd_page(orig_pmd); 1528b8d3c4c3SMinchan Kim /* 1529b8d3c4c3SMinchan Kim * If other processes are mapping this page, we couldn't discard 1530b8d3c4c3SMinchan Kim * the page unless they all do MADV_FREE so let's skip the page. 1531b8d3c4c3SMinchan Kim */ 1532b8d3c4c3SMinchan Kim if (page_mapcount(page) != 1) 1533b8d3c4c3SMinchan Kim goto out; 1534b8d3c4c3SMinchan Kim 1535b8d3c4c3SMinchan Kim if (!trylock_page(page)) 1536b8d3c4c3SMinchan Kim goto out; 1537b8d3c4c3SMinchan Kim 1538b8d3c4c3SMinchan Kim /* 1539b8d3c4c3SMinchan Kim * If user want to discard part-pages of THP, split it so MADV_FREE 1540b8d3c4c3SMinchan Kim * will deactivate only them. 1541b8d3c4c3SMinchan Kim */ 1542b8d3c4c3SMinchan Kim if (next - addr != HPAGE_PMD_SIZE) { 1543b8d3c4c3SMinchan Kim get_page(page); 1544b8d3c4c3SMinchan Kim spin_unlock(ptl); 1545b8d3c4c3SMinchan Kim if (split_huge_page(page)) { 1546b8d3c4c3SMinchan Kim put_page(page); 1547b8d3c4c3SMinchan Kim unlock_page(page); 1548b8d3c4c3SMinchan Kim goto out_unlocked; 1549b8d3c4c3SMinchan Kim } 1550b8d3c4c3SMinchan Kim put_page(page); 1551b8d3c4c3SMinchan Kim unlock_page(page); 1552b8d3c4c3SMinchan Kim ret = 1; 1553b8d3c4c3SMinchan Kim goto out_unlocked; 1554b8d3c4c3SMinchan Kim } 1555b8d3c4c3SMinchan Kim 1556b8d3c4c3SMinchan Kim if (PageDirty(page)) 1557b8d3c4c3SMinchan Kim ClearPageDirty(page); 1558b8d3c4c3SMinchan Kim unlock_page(page); 1559b8d3c4c3SMinchan Kim 1560b8d3c4c3SMinchan Kim if (PageActive(page)) 1561b8d3c4c3SMinchan Kim deactivate_page(page); 1562b8d3c4c3SMinchan Kim 1563b8d3c4c3SMinchan Kim if (pmd_young(orig_pmd) || pmd_dirty(orig_pmd)) { 1564b8d3c4c3SMinchan Kim orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd, 1565b8d3c4c3SMinchan Kim tlb->fullmm); 1566b8d3c4c3SMinchan Kim orig_pmd = pmd_mkold(orig_pmd); 1567b8d3c4c3SMinchan Kim orig_pmd = pmd_mkclean(orig_pmd); 1568b8d3c4c3SMinchan Kim 1569b8d3c4c3SMinchan Kim set_pmd_at(mm, addr, pmd, orig_pmd); 1570b8d3c4c3SMinchan Kim tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 1571b8d3c4c3SMinchan Kim } 1572b8d3c4c3SMinchan Kim ret = 1; 1573b8d3c4c3SMinchan Kim out: 1574b8d3c4c3SMinchan Kim spin_unlock(ptl); 1575b8d3c4c3SMinchan Kim out_unlocked: 1576b8d3c4c3SMinchan Kim return ret; 1577b8d3c4c3SMinchan Kim } 1578b8d3c4c3SMinchan Kim 157971e3aac0SAndrea Arcangeli int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 1580f21760b1SShaohua Li pmd_t *pmd, unsigned long addr) 158171e3aac0SAndrea Arcangeli { 1582f5c8ad47SDavid Miller pmd_t orig_pmd; 1583da146769SKirill A. Shutemov spinlock_t *ptl; 1584da146769SKirill A. Shutemov 15854b471e88SKirill A. Shutemov if (!__pmd_trans_huge_lock(pmd, vma, &ptl)) 1586da146769SKirill A. Shutemov return 0; 1587a6bf2bb0SAneesh Kumar K.V /* 1588a6bf2bb0SAneesh Kumar K.V * For architectures like ppc64 we look at deposited pgtable 15898809aa2dSAneesh Kumar K.V * when calling pmdp_huge_get_and_clear. So do the 1590a6bf2bb0SAneesh Kumar K.V * pgtable_trans_huge_withdraw after finishing pmdp related 1591a6bf2bb0SAneesh Kumar K.V * operations. 1592a6bf2bb0SAneesh Kumar K.V */ 15938809aa2dSAneesh Kumar K.V orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd, 1594fcbe08d6SMartin Schwidefsky tlb->fullmm); 1595f21760b1SShaohua Li tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 15964897c765SMatthew Wilcox if (vma_is_dax(vma)) { 15974897c765SMatthew Wilcox spin_unlock(ptl); 1598da146769SKirill A. Shutemov if (is_huge_zero_pmd(orig_pmd)) 1599da146769SKirill A. Shutemov put_huge_zero_page(); 1600da146769SKirill A. Shutemov } else if (is_huge_zero_pmd(orig_pmd)) { 1601da146769SKirill A. Shutemov pte_free(tlb->mm, pgtable_trans_huge_withdraw(tlb->mm, pmd)); 1602e1f56c89SKirill A. Shutemov atomic_long_dec(&tlb->mm->nr_ptes); 1603bf929152SKirill A. Shutemov spin_unlock(ptl); 160497ae1749SKirill A. Shutemov put_huge_zero_page(); 1605479f0abbSKirill A. Shutemov } else { 16064897c765SMatthew Wilcox struct page *page = pmd_page(orig_pmd); 1607d281ee61SKirill A. Shutemov page_remove_rmap(page, true); 1608309381feSSasha Levin VM_BUG_ON_PAGE(page_mapcount(page) < 0, page); 160971e3aac0SAndrea Arcangeli add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR); 1610309381feSSasha Levin VM_BUG_ON_PAGE(!PageHead(page), page); 1611da146769SKirill A. Shutemov pte_free(tlb->mm, pgtable_trans_huge_withdraw(tlb->mm, pmd)); 1612e1f56c89SKirill A. Shutemov atomic_long_dec(&tlb->mm->nr_ptes); 1613bf929152SKirill A. Shutemov spin_unlock(ptl); 161471e3aac0SAndrea Arcangeli tlb_remove_page(tlb, page); 1615479f0abbSKirill A. Shutemov } 1616da146769SKirill A. Shutemov return 1; 161771e3aac0SAndrea Arcangeli } 161871e3aac0SAndrea Arcangeli 16194b471e88SKirill A. Shutemov bool move_huge_pmd(struct vm_area_struct *vma, struct vm_area_struct *new_vma, 162037a1c49aSAndrea Arcangeli unsigned long old_addr, 162137a1c49aSAndrea Arcangeli unsigned long new_addr, unsigned long old_end, 162237a1c49aSAndrea Arcangeli pmd_t *old_pmd, pmd_t *new_pmd) 162337a1c49aSAndrea Arcangeli { 1624bf929152SKirill A. Shutemov spinlock_t *old_ptl, *new_ptl; 162537a1c49aSAndrea Arcangeli pmd_t pmd; 162637a1c49aSAndrea Arcangeli 162737a1c49aSAndrea Arcangeli struct mm_struct *mm = vma->vm_mm; 162837a1c49aSAndrea Arcangeli 162937a1c49aSAndrea Arcangeli if ((old_addr & ~HPAGE_PMD_MASK) || 163037a1c49aSAndrea Arcangeli (new_addr & ~HPAGE_PMD_MASK) || 163137a1c49aSAndrea Arcangeli old_end - old_addr < HPAGE_PMD_SIZE || 163237a1c49aSAndrea Arcangeli (new_vma->vm_flags & VM_NOHUGEPAGE)) 16334b471e88SKirill A. Shutemov return false; 163437a1c49aSAndrea Arcangeli 163537a1c49aSAndrea Arcangeli /* 163637a1c49aSAndrea Arcangeli * The destination pmd shouldn't be established, free_pgtables() 163737a1c49aSAndrea Arcangeli * should have release it. 163837a1c49aSAndrea Arcangeli */ 163937a1c49aSAndrea Arcangeli if (WARN_ON(!pmd_none(*new_pmd))) { 164037a1c49aSAndrea Arcangeli VM_BUG_ON(pmd_trans_huge(*new_pmd)); 16414b471e88SKirill A. Shutemov return false; 164237a1c49aSAndrea Arcangeli } 164337a1c49aSAndrea Arcangeli 1644bf929152SKirill A. Shutemov /* 1645bf929152SKirill A. Shutemov * We don't have to worry about the ordering of src and dst 1646bf929152SKirill A. Shutemov * ptlocks because exclusive mmap_sem prevents deadlock. 1647bf929152SKirill A. Shutemov */ 16484b471e88SKirill A. Shutemov if (__pmd_trans_huge_lock(old_pmd, vma, &old_ptl)) { 1649bf929152SKirill A. Shutemov new_ptl = pmd_lockptr(mm, new_pmd); 1650bf929152SKirill A. Shutemov if (new_ptl != old_ptl) 1651bf929152SKirill A. Shutemov spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING); 16528809aa2dSAneesh Kumar K.V pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd); 165337a1c49aSAndrea Arcangeli VM_BUG_ON(!pmd_none(*new_pmd)); 16543592806cSKirill A. Shutemov 1655b3084f4dSAneesh Kumar K.V if (pmd_move_must_withdraw(new_ptl, old_ptl)) { 1656b3084f4dSAneesh Kumar K.V pgtable_t pgtable; 16573592806cSKirill A. Shutemov pgtable = pgtable_trans_huge_withdraw(mm, old_pmd); 16583592806cSKirill A. Shutemov pgtable_trans_huge_deposit(mm, new_pmd, pgtable); 16593592806cSKirill A. Shutemov } 1660b3084f4dSAneesh Kumar K.V set_pmd_at(mm, new_addr, new_pmd, pmd_mksoft_dirty(pmd)); 1661b3084f4dSAneesh Kumar K.V if (new_ptl != old_ptl) 1662b3084f4dSAneesh Kumar K.V spin_unlock(new_ptl); 1663bf929152SKirill A. Shutemov spin_unlock(old_ptl); 16644b471e88SKirill A. Shutemov return true; 166537a1c49aSAndrea Arcangeli } 16664b471e88SKirill A. Shutemov return false; 166737a1c49aSAndrea Arcangeli } 166837a1c49aSAndrea Arcangeli 1669f123d74aSMel Gorman /* 1670f123d74aSMel Gorman * Returns 1671f123d74aSMel Gorman * - 0 if PMD could not be locked 1672f123d74aSMel Gorman * - 1 if PMD was locked but protections unchange and TLB flush unnecessary 1673f123d74aSMel Gorman * - HPAGE_PMD_NR is protections changed and TLB flush necessary 1674f123d74aSMel Gorman */ 1675cd7548abSJohannes Weiner int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, 1676e944fd67SMel Gorman unsigned long addr, pgprot_t newprot, int prot_numa) 1677cd7548abSJohannes Weiner { 1678cd7548abSJohannes Weiner struct mm_struct *mm = vma->vm_mm; 1679bf929152SKirill A. Shutemov spinlock_t *ptl; 1680cd7548abSJohannes Weiner int ret = 0; 1681cd7548abSJohannes Weiner 16824b471e88SKirill A. Shutemov if (__pmd_trans_huge_lock(pmd, vma, &ptl)) { 1683cd7548abSJohannes Weiner pmd_t entry; 1684b191f9b1SMel Gorman bool preserve_write = prot_numa && pmd_write(*pmd); 1685ba68bc01SMel Gorman ret = 1; 1686e944fd67SMel Gorman 1687e944fd67SMel Gorman /* 1688e944fd67SMel Gorman * Avoid trapping faults against the zero page. The read-only 1689e944fd67SMel Gorman * data is likely to be read-cached on the local CPU and 1690e944fd67SMel Gorman * local/remote hits to the zero page are not interesting. 1691e944fd67SMel Gorman */ 1692e944fd67SMel Gorman if (prot_numa && is_huge_zero_pmd(*pmd)) { 1693e944fd67SMel Gorman spin_unlock(ptl); 1694ba68bc01SMel Gorman return ret; 1695e944fd67SMel Gorman } 1696e944fd67SMel Gorman 169710c1045fSMel Gorman if (!prot_numa || !pmd_protnone(*pmd)) { 16988809aa2dSAneesh Kumar K.V entry = pmdp_huge_get_and_clear_notify(mm, addr, pmd); 1699cd7548abSJohannes Weiner entry = pmd_modify(entry, newprot); 1700b191f9b1SMel Gorman if (preserve_write) 1701b191f9b1SMel Gorman entry = pmd_mkwrite(entry); 1702f123d74aSMel Gorman ret = HPAGE_PMD_NR; 170356eecdb9SAneesh Kumar K.V set_pmd_at(mm, addr, pmd, entry); 1704b191f9b1SMel Gorman BUG_ON(!preserve_write && pmd_write(entry)); 170510c1045fSMel Gorman } 1706bf929152SKirill A. Shutemov spin_unlock(ptl); 1707cd7548abSJohannes Weiner } 1708cd7548abSJohannes Weiner 1709cd7548abSJohannes Weiner return ret; 1710cd7548abSJohannes Weiner } 1711cd7548abSJohannes Weiner 1712025c5b24SNaoya Horiguchi /* 17134b471e88SKirill A. Shutemov * Returns true if a given pmd maps a thp, false otherwise. 1714025c5b24SNaoya Horiguchi * 17154b471e88SKirill A. Shutemov * Note that if it returns true, this routine returns without unlocking page 17164b471e88SKirill A. Shutemov * table lock. So callers must unlock it. 1717025c5b24SNaoya Horiguchi */ 17184b471e88SKirill A. Shutemov bool __pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma, 1719bf929152SKirill A. Shutemov spinlock_t **ptl) 1720025c5b24SNaoya Horiguchi { 1721bf929152SKirill A. Shutemov *ptl = pmd_lock(vma->vm_mm, pmd); 1722*5c7fb56eSDan Williams if (likely(pmd_trans_huge(*pmd) || pmd_devmap(*pmd))) 17234b471e88SKirill A. Shutemov return true; 1724bf929152SKirill A. Shutemov spin_unlock(*ptl); 17254b471e88SKirill A. Shutemov return false; 1726025c5b24SNaoya Horiguchi } 1727025c5b24SNaoya Horiguchi 17289050d7ebSVlastimil Babka #define VM_NO_THP (VM_SPECIAL | VM_HUGETLB | VM_SHARED | VM_MAYSHARE) 172978f11a25SAndrea Arcangeli 173060ab3244SAndrea Arcangeli int hugepage_madvise(struct vm_area_struct *vma, 173160ab3244SAndrea Arcangeli unsigned long *vm_flags, int advice) 17320af4e98bSAndrea Arcangeli { 1733a664b2d8SAndrea Arcangeli switch (advice) { 1734a664b2d8SAndrea Arcangeli case MADV_HUGEPAGE: 17351e1836e8SAlex Thorlton #ifdef CONFIG_S390 17361e1836e8SAlex Thorlton /* 17371e1836e8SAlex Thorlton * qemu blindly sets MADV_HUGEPAGE on all allocations, but s390 17381e1836e8SAlex Thorlton * can't handle this properly after s390_enable_sie, so we simply 17391e1836e8SAlex Thorlton * ignore the madvise to prevent qemu from causing a SIGSEGV. 17401e1836e8SAlex Thorlton */ 17411e1836e8SAlex Thorlton if (mm_has_pgste(vma->vm_mm)) 17421e1836e8SAlex Thorlton return 0; 17431e1836e8SAlex Thorlton #endif 17440af4e98bSAndrea Arcangeli /* 17450af4e98bSAndrea Arcangeli * Be somewhat over-protective like KSM for now! 17460af4e98bSAndrea Arcangeli */ 17471a763615SJason J. Herne if (*vm_flags & VM_NO_THP) 17480af4e98bSAndrea Arcangeli return -EINVAL; 1749a664b2d8SAndrea Arcangeli *vm_flags &= ~VM_NOHUGEPAGE; 17500af4e98bSAndrea Arcangeli *vm_flags |= VM_HUGEPAGE; 175160ab3244SAndrea Arcangeli /* 175260ab3244SAndrea Arcangeli * If the vma become good for khugepaged to scan, 175360ab3244SAndrea Arcangeli * register it here without waiting a page fault that 175460ab3244SAndrea Arcangeli * may not happen any time soon. 175560ab3244SAndrea Arcangeli */ 17566d50e60cSDavid Rientjes if (unlikely(khugepaged_enter_vma_merge(vma, *vm_flags))) 175760ab3244SAndrea Arcangeli return -ENOMEM; 1758a664b2d8SAndrea Arcangeli break; 1759a664b2d8SAndrea Arcangeli case MADV_NOHUGEPAGE: 1760a664b2d8SAndrea Arcangeli /* 1761a664b2d8SAndrea Arcangeli * Be somewhat over-protective like KSM for now! 1762a664b2d8SAndrea Arcangeli */ 17631a763615SJason J. Herne if (*vm_flags & VM_NO_THP) 1764a664b2d8SAndrea Arcangeli return -EINVAL; 1765a664b2d8SAndrea Arcangeli *vm_flags &= ~VM_HUGEPAGE; 1766a664b2d8SAndrea Arcangeli *vm_flags |= VM_NOHUGEPAGE; 176760ab3244SAndrea Arcangeli /* 176860ab3244SAndrea Arcangeli * Setting VM_NOHUGEPAGE will prevent khugepaged from scanning 176960ab3244SAndrea Arcangeli * this vma even if we leave the mm registered in khugepaged if 177060ab3244SAndrea Arcangeli * it got registered before VM_NOHUGEPAGE was set. 177160ab3244SAndrea Arcangeli */ 1772a664b2d8SAndrea Arcangeli break; 1773a664b2d8SAndrea Arcangeli } 17740af4e98bSAndrea Arcangeli 17750af4e98bSAndrea Arcangeli return 0; 17760af4e98bSAndrea Arcangeli } 17770af4e98bSAndrea Arcangeli 1778ba76149fSAndrea Arcangeli static int __init khugepaged_slab_init(void) 1779ba76149fSAndrea Arcangeli { 1780ba76149fSAndrea Arcangeli mm_slot_cache = kmem_cache_create("khugepaged_mm_slot", 1781ba76149fSAndrea Arcangeli sizeof(struct mm_slot), 1782ba76149fSAndrea Arcangeli __alignof__(struct mm_slot), 0, NULL); 1783ba76149fSAndrea Arcangeli if (!mm_slot_cache) 1784ba76149fSAndrea Arcangeli return -ENOMEM; 1785ba76149fSAndrea Arcangeli 1786ba76149fSAndrea Arcangeli return 0; 1787ba76149fSAndrea Arcangeli } 1788ba76149fSAndrea Arcangeli 178965ebb64fSKirill A. Shutemov static void __init khugepaged_slab_exit(void) 179065ebb64fSKirill A. Shutemov { 179165ebb64fSKirill A. Shutemov kmem_cache_destroy(mm_slot_cache); 179265ebb64fSKirill A. Shutemov } 179365ebb64fSKirill A. Shutemov 1794ba76149fSAndrea Arcangeli static inline struct mm_slot *alloc_mm_slot(void) 1795ba76149fSAndrea Arcangeli { 1796ba76149fSAndrea Arcangeli if (!mm_slot_cache) /* initialization failed */ 1797ba76149fSAndrea Arcangeli return NULL; 1798ba76149fSAndrea Arcangeli return kmem_cache_zalloc(mm_slot_cache, GFP_KERNEL); 1799ba76149fSAndrea Arcangeli } 1800ba76149fSAndrea Arcangeli 1801ba76149fSAndrea Arcangeli static inline void free_mm_slot(struct mm_slot *mm_slot) 1802ba76149fSAndrea Arcangeli { 1803ba76149fSAndrea Arcangeli kmem_cache_free(mm_slot_cache, mm_slot); 1804ba76149fSAndrea Arcangeli } 1805ba76149fSAndrea Arcangeli 1806ba76149fSAndrea Arcangeli static struct mm_slot *get_mm_slot(struct mm_struct *mm) 1807ba76149fSAndrea Arcangeli { 1808ba76149fSAndrea Arcangeli struct mm_slot *mm_slot; 1809ba76149fSAndrea Arcangeli 1810b67bfe0dSSasha Levin hash_for_each_possible(mm_slots_hash, mm_slot, hash, (unsigned long)mm) 1811ba76149fSAndrea Arcangeli if (mm == mm_slot->mm) 1812ba76149fSAndrea Arcangeli return mm_slot; 181343b5fbbdSSasha Levin 1814ba76149fSAndrea Arcangeli return NULL; 1815ba76149fSAndrea Arcangeli } 1816ba76149fSAndrea Arcangeli 1817ba76149fSAndrea Arcangeli static void insert_to_mm_slots_hash(struct mm_struct *mm, 1818ba76149fSAndrea Arcangeli struct mm_slot *mm_slot) 1819ba76149fSAndrea Arcangeli { 1820ba76149fSAndrea Arcangeli mm_slot->mm = mm; 182143b5fbbdSSasha Levin hash_add(mm_slots_hash, &mm_slot->hash, (long)mm); 1822ba76149fSAndrea Arcangeli } 1823ba76149fSAndrea Arcangeli 1824ba76149fSAndrea Arcangeli static inline int khugepaged_test_exit(struct mm_struct *mm) 1825ba76149fSAndrea Arcangeli { 1826ba76149fSAndrea Arcangeli return atomic_read(&mm->mm_users) == 0; 1827ba76149fSAndrea Arcangeli } 1828ba76149fSAndrea Arcangeli 1829ba76149fSAndrea Arcangeli int __khugepaged_enter(struct mm_struct *mm) 1830ba76149fSAndrea Arcangeli { 1831ba76149fSAndrea Arcangeli struct mm_slot *mm_slot; 1832ba76149fSAndrea Arcangeli int wakeup; 1833ba76149fSAndrea Arcangeli 1834ba76149fSAndrea Arcangeli mm_slot = alloc_mm_slot(); 1835ba76149fSAndrea Arcangeli if (!mm_slot) 1836ba76149fSAndrea Arcangeli return -ENOMEM; 1837ba76149fSAndrea Arcangeli 1838ba76149fSAndrea Arcangeli /* __khugepaged_exit() must not run from under us */ 183996dad67fSSasha Levin VM_BUG_ON_MM(khugepaged_test_exit(mm), mm); 1840ba76149fSAndrea Arcangeli if (unlikely(test_and_set_bit(MMF_VM_HUGEPAGE, &mm->flags))) { 1841ba76149fSAndrea Arcangeli free_mm_slot(mm_slot); 1842ba76149fSAndrea Arcangeli return 0; 1843ba76149fSAndrea Arcangeli } 1844ba76149fSAndrea Arcangeli 1845ba76149fSAndrea Arcangeli spin_lock(&khugepaged_mm_lock); 1846ba76149fSAndrea Arcangeli insert_to_mm_slots_hash(mm, mm_slot); 1847ba76149fSAndrea Arcangeli /* 1848ba76149fSAndrea Arcangeli * Insert just behind the scanning cursor, to let the area settle 1849ba76149fSAndrea Arcangeli * down a little. 1850ba76149fSAndrea Arcangeli */ 1851ba76149fSAndrea Arcangeli wakeup = list_empty(&khugepaged_scan.mm_head); 1852ba76149fSAndrea Arcangeli list_add_tail(&mm_slot->mm_node, &khugepaged_scan.mm_head); 1853ba76149fSAndrea Arcangeli spin_unlock(&khugepaged_mm_lock); 1854ba76149fSAndrea Arcangeli 1855ba76149fSAndrea Arcangeli atomic_inc(&mm->mm_count); 1856ba76149fSAndrea Arcangeli if (wakeup) 1857ba76149fSAndrea Arcangeli wake_up_interruptible(&khugepaged_wait); 1858ba76149fSAndrea Arcangeli 1859ba76149fSAndrea Arcangeli return 0; 1860ba76149fSAndrea Arcangeli } 1861ba76149fSAndrea Arcangeli 18626d50e60cSDavid Rientjes int khugepaged_enter_vma_merge(struct vm_area_struct *vma, 18636d50e60cSDavid Rientjes unsigned long vm_flags) 1864ba76149fSAndrea Arcangeli { 1865ba76149fSAndrea Arcangeli unsigned long hstart, hend; 1866ba76149fSAndrea Arcangeli if (!vma->anon_vma) 1867ba76149fSAndrea Arcangeli /* 1868ba76149fSAndrea Arcangeli * Not yet faulted in so we will register later in the 1869ba76149fSAndrea Arcangeli * page fault if needed. 1870ba76149fSAndrea Arcangeli */ 1871ba76149fSAndrea Arcangeli return 0; 187278f11a25SAndrea Arcangeli if (vma->vm_ops) 1873ba76149fSAndrea Arcangeli /* khugepaged not yet working on file or special mappings */ 1874ba76149fSAndrea Arcangeli return 0; 18756d50e60cSDavid Rientjes VM_BUG_ON_VMA(vm_flags & VM_NO_THP, vma); 1876ba76149fSAndrea Arcangeli hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK; 1877ba76149fSAndrea Arcangeli hend = vma->vm_end & HPAGE_PMD_MASK; 1878ba76149fSAndrea Arcangeli if (hstart < hend) 18796d50e60cSDavid Rientjes return khugepaged_enter(vma, vm_flags); 1880ba76149fSAndrea Arcangeli return 0; 1881ba76149fSAndrea Arcangeli } 1882ba76149fSAndrea Arcangeli 1883ba76149fSAndrea Arcangeli void __khugepaged_exit(struct mm_struct *mm) 1884ba76149fSAndrea Arcangeli { 1885ba76149fSAndrea Arcangeli struct mm_slot *mm_slot; 1886ba76149fSAndrea Arcangeli int free = 0; 1887ba76149fSAndrea Arcangeli 1888ba76149fSAndrea Arcangeli spin_lock(&khugepaged_mm_lock); 1889ba76149fSAndrea Arcangeli mm_slot = get_mm_slot(mm); 1890ba76149fSAndrea Arcangeli if (mm_slot && khugepaged_scan.mm_slot != mm_slot) { 189143b5fbbdSSasha Levin hash_del(&mm_slot->hash); 1892ba76149fSAndrea Arcangeli list_del(&mm_slot->mm_node); 1893ba76149fSAndrea Arcangeli free = 1; 1894ba76149fSAndrea Arcangeli } 1895d788e80aSChris Wright spin_unlock(&khugepaged_mm_lock); 1896ba76149fSAndrea Arcangeli 1897ba76149fSAndrea Arcangeli if (free) { 1898ba76149fSAndrea Arcangeli clear_bit(MMF_VM_HUGEPAGE, &mm->flags); 1899ba76149fSAndrea Arcangeli free_mm_slot(mm_slot); 1900ba76149fSAndrea Arcangeli mmdrop(mm); 1901ba76149fSAndrea Arcangeli } else if (mm_slot) { 1902ba76149fSAndrea Arcangeli /* 1903ba76149fSAndrea Arcangeli * This is required to serialize against 1904ba76149fSAndrea Arcangeli * khugepaged_test_exit() (which is guaranteed to run 1905ba76149fSAndrea Arcangeli * under mmap sem read mode). Stop here (after we 1906ba76149fSAndrea Arcangeli * return all pagetables will be destroyed) until 1907ba76149fSAndrea Arcangeli * khugepaged has finished working on the pagetables 1908ba76149fSAndrea Arcangeli * under the mmap_sem. 1909ba76149fSAndrea Arcangeli */ 1910ba76149fSAndrea Arcangeli down_write(&mm->mmap_sem); 1911ba76149fSAndrea Arcangeli up_write(&mm->mmap_sem); 1912d788e80aSChris Wright } 1913ba76149fSAndrea Arcangeli } 1914ba76149fSAndrea Arcangeli 1915ba76149fSAndrea Arcangeli static void release_pte_page(struct page *page) 1916ba76149fSAndrea Arcangeli { 1917ba76149fSAndrea Arcangeli /* 0 stands for page_is_file_cache(page) == false */ 1918ba76149fSAndrea Arcangeli dec_zone_page_state(page, NR_ISOLATED_ANON + 0); 1919ba76149fSAndrea Arcangeli unlock_page(page); 1920ba76149fSAndrea Arcangeli putback_lru_page(page); 1921ba76149fSAndrea Arcangeli } 1922ba76149fSAndrea Arcangeli 1923ba76149fSAndrea Arcangeli static void release_pte_pages(pte_t *pte, pte_t *_pte) 1924ba76149fSAndrea Arcangeli { 1925ba76149fSAndrea Arcangeli while (--_pte >= pte) { 1926ba76149fSAndrea Arcangeli pte_t pteval = *_pte; 1927ca0984caSEbru Akagunduz if (!pte_none(pteval) && !is_zero_pfn(pte_pfn(pteval))) 1928ba76149fSAndrea Arcangeli release_pte_page(pte_page(pteval)); 1929ba76149fSAndrea Arcangeli } 1930ba76149fSAndrea Arcangeli } 1931ba76149fSAndrea Arcangeli 1932ba76149fSAndrea Arcangeli static int __collapse_huge_page_isolate(struct vm_area_struct *vma, 1933ba76149fSAndrea Arcangeli unsigned long address, 1934ba76149fSAndrea Arcangeli pte_t *pte) 1935ba76149fSAndrea Arcangeli { 19367d2eba05SEbru Akagunduz struct page *page = NULL; 1937ba76149fSAndrea Arcangeli pte_t *_pte; 19387d2eba05SEbru Akagunduz int none_or_zero = 0, result = 0; 193910359213SEbru Akagunduz bool referenced = false, writable = false; 19407d2eba05SEbru Akagunduz 1941ba76149fSAndrea Arcangeli for (_pte = pte; _pte < pte+HPAGE_PMD_NR; 1942ba76149fSAndrea Arcangeli _pte++, address += PAGE_SIZE) { 1943ba76149fSAndrea Arcangeli pte_t pteval = *_pte; 194447aee4d8SMinchan Kim if (pte_none(pteval) || (pte_present(pteval) && 194547aee4d8SMinchan Kim is_zero_pfn(pte_pfn(pteval)))) { 1946c1294d05SAndrea Arcangeli if (!userfaultfd_armed(vma) && 19477d2eba05SEbru Akagunduz ++none_or_zero <= khugepaged_max_ptes_none) { 1948ba76149fSAndrea Arcangeli continue; 19497d2eba05SEbru Akagunduz } else { 19507d2eba05SEbru Akagunduz result = SCAN_EXCEED_NONE_PTE; 1951ba76149fSAndrea Arcangeli goto out; 1952ba76149fSAndrea Arcangeli } 19537d2eba05SEbru Akagunduz } 19547d2eba05SEbru Akagunduz if (!pte_present(pteval)) { 19557d2eba05SEbru Akagunduz result = SCAN_PTE_NON_PRESENT; 1956ba76149fSAndrea Arcangeli goto out; 19577d2eba05SEbru Akagunduz } 1958ba76149fSAndrea Arcangeli page = vm_normal_page(vma, address, pteval); 19597d2eba05SEbru Akagunduz if (unlikely(!page)) { 19607d2eba05SEbru Akagunduz result = SCAN_PAGE_NULL; 1961ba76149fSAndrea Arcangeli goto out; 19627d2eba05SEbru Akagunduz } 1963344aa35cSBob Liu 1964309381feSSasha Levin VM_BUG_ON_PAGE(PageCompound(page), page); 1965309381feSSasha Levin VM_BUG_ON_PAGE(!PageAnon(page), page); 1966309381feSSasha Levin VM_BUG_ON_PAGE(!PageSwapBacked(page), page); 1967ba76149fSAndrea Arcangeli 1968ba76149fSAndrea Arcangeli /* 1969ba76149fSAndrea Arcangeli * We can do it before isolate_lru_page because the 1970ba76149fSAndrea Arcangeli * page can't be freed from under us. NOTE: PG_lock 1971ba76149fSAndrea Arcangeli * is needed to serialize against split_huge_page 1972ba76149fSAndrea Arcangeli * when invoked from the VM. 1973ba76149fSAndrea Arcangeli */ 19747d2eba05SEbru Akagunduz if (!trylock_page(page)) { 19757d2eba05SEbru Akagunduz result = SCAN_PAGE_LOCK; 1976ba76149fSAndrea Arcangeli goto out; 19777d2eba05SEbru Akagunduz } 197810359213SEbru Akagunduz 197910359213SEbru Akagunduz /* 198010359213SEbru Akagunduz * cannot use mapcount: can't collapse if there's a gup pin. 198110359213SEbru Akagunduz * The page must only be referenced by the scanned process 198210359213SEbru Akagunduz * and page swap cache. 198310359213SEbru Akagunduz */ 198410359213SEbru Akagunduz if (page_count(page) != 1 + !!PageSwapCache(page)) { 198510359213SEbru Akagunduz unlock_page(page); 19867d2eba05SEbru Akagunduz result = SCAN_PAGE_COUNT; 198710359213SEbru Akagunduz goto out; 198810359213SEbru Akagunduz } 198910359213SEbru Akagunduz if (pte_write(pteval)) { 199010359213SEbru Akagunduz writable = true; 199110359213SEbru Akagunduz } else { 199210359213SEbru Akagunduz if (PageSwapCache(page) && !reuse_swap_page(page)) { 199310359213SEbru Akagunduz unlock_page(page); 19947d2eba05SEbru Akagunduz result = SCAN_SWAP_CACHE_PAGE; 199510359213SEbru Akagunduz goto out; 199610359213SEbru Akagunduz } 199710359213SEbru Akagunduz /* 199810359213SEbru Akagunduz * Page is not in the swap cache. It can be collapsed 199910359213SEbru Akagunduz * into a THP. 200010359213SEbru Akagunduz */ 200110359213SEbru Akagunduz } 200210359213SEbru Akagunduz 2003ba76149fSAndrea Arcangeli /* 2004ba76149fSAndrea Arcangeli * Isolate the page to avoid collapsing an hugepage 2005ba76149fSAndrea Arcangeli * currently in use by the VM. 2006ba76149fSAndrea Arcangeli */ 2007ba76149fSAndrea Arcangeli if (isolate_lru_page(page)) { 2008ba76149fSAndrea Arcangeli unlock_page(page); 20097d2eba05SEbru Akagunduz result = SCAN_DEL_PAGE_LRU; 2010ba76149fSAndrea Arcangeli goto out; 2011ba76149fSAndrea Arcangeli } 2012ba76149fSAndrea Arcangeli /* 0 stands for page_is_file_cache(page) == false */ 2013ba76149fSAndrea Arcangeli inc_zone_page_state(page, NR_ISOLATED_ANON + 0); 2014309381feSSasha Levin VM_BUG_ON_PAGE(!PageLocked(page), page); 2015309381feSSasha Levin VM_BUG_ON_PAGE(PageLRU(page), page); 2016ba76149fSAndrea Arcangeli 2017ba76149fSAndrea Arcangeli /* If there is no mapped pte young don't collapse the page */ 201833c3fc71SVladimir Davydov if (pte_young(pteval) || 201933c3fc71SVladimir Davydov page_is_young(page) || PageReferenced(page) || 20208ee53820SAndrea Arcangeli mmu_notifier_test_young(vma->vm_mm, address)) 202110359213SEbru Akagunduz referenced = true; 2022ba76149fSAndrea Arcangeli } 20237d2eba05SEbru Akagunduz if (likely(writable)) { 20247d2eba05SEbru Akagunduz if (likely(referenced)) { 20257d2eba05SEbru Akagunduz result = SCAN_SUCCEED; 20267d2eba05SEbru Akagunduz trace_mm_collapse_huge_page_isolate(page_to_pfn(page), none_or_zero, 20277d2eba05SEbru Akagunduz referenced, writable, result); 2028344aa35cSBob Liu return 1; 20297d2eba05SEbru Akagunduz } 20307d2eba05SEbru Akagunduz } else { 20317d2eba05SEbru Akagunduz result = SCAN_PAGE_RO; 20327d2eba05SEbru Akagunduz } 20337d2eba05SEbru Akagunduz 2034ba76149fSAndrea Arcangeli out: 2035344aa35cSBob Liu release_pte_pages(pte, _pte); 20367d2eba05SEbru Akagunduz trace_mm_collapse_huge_page_isolate(page_to_pfn(page), none_or_zero, 20377d2eba05SEbru Akagunduz referenced, writable, result); 2038344aa35cSBob Liu return 0; 2039ba76149fSAndrea Arcangeli } 2040ba76149fSAndrea Arcangeli 2041ba76149fSAndrea Arcangeli static void __collapse_huge_page_copy(pte_t *pte, struct page *page, 2042ba76149fSAndrea Arcangeli struct vm_area_struct *vma, 2043ba76149fSAndrea Arcangeli unsigned long address, 2044ba76149fSAndrea Arcangeli spinlock_t *ptl) 2045ba76149fSAndrea Arcangeli { 2046ba76149fSAndrea Arcangeli pte_t *_pte; 2047ba76149fSAndrea Arcangeli for (_pte = pte; _pte < pte+HPAGE_PMD_NR; _pte++) { 2048ba76149fSAndrea Arcangeli pte_t pteval = *_pte; 2049ba76149fSAndrea Arcangeli struct page *src_page; 2050ba76149fSAndrea Arcangeli 2051ca0984caSEbru Akagunduz if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) { 2052ba76149fSAndrea Arcangeli clear_user_highpage(page, address); 2053ba76149fSAndrea Arcangeli add_mm_counter(vma->vm_mm, MM_ANONPAGES, 1); 2054ca0984caSEbru Akagunduz if (is_zero_pfn(pte_pfn(pteval))) { 2055ca0984caSEbru Akagunduz /* 2056ca0984caSEbru Akagunduz * ptl mostly unnecessary. 2057ca0984caSEbru Akagunduz */ 2058ca0984caSEbru Akagunduz spin_lock(ptl); 2059ca0984caSEbru Akagunduz /* 2060ca0984caSEbru Akagunduz * paravirt calls inside pte_clear here are 2061ca0984caSEbru Akagunduz * superfluous. 2062ca0984caSEbru Akagunduz */ 2063ca0984caSEbru Akagunduz pte_clear(vma->vm_mm, address, _pte); 2064ca0984caSEbru Akagunduz spin_unlock(ptl); 2065ca0984caSEbru Akagunduz } 2066ba76149fSAndrea Arcangeli } else { 2067ba76149fSAndrea Arcangeli src_page = pte_page(pteval); 2068ba76149fSAndrea Arcangeli copy_user_highpage(page, src_page, address, vma); 2069309381feSSasha Levin VM_BUG_ON_PAGE(page_mapcount(src_page) != 1, src_page); 2070ba76149fSAndrea Arcangeli release_pte_page(src_page); 2071ba76149fSAndrea Arcangeli /* 2072ba76149fSAndrea Arcangeli * ptl mostly unnecessary, but preempt has to 2073ba76149fSAndrea Arcangeli * be disabled to update the per-cpu stats 2074ba76149fSAndrea Arcangeli * inside page_remove_rmap(). 2075ba76149fSAndrea Arcangeli */ 2076ba76149fSAndrea Arcangeli spin_lock(ptl); 2077ba76149fSAndrea Arcangeli /* 2078ba76149fSAndrea Arcangeli * paravirt calls inside pte_clear here are 2079ba76149fSAndrea Arcangeli * superfluous. 2080ba76149fSAndrea Arcangeli */ 2081ba76149fSAndrea Arcangeli pte_clear(vma->vm_mm, address, _pte); 2082d281ee61SKirill A. Shutemov page_remove_rmap(src_page, false); 2083ba76149fSAndrea Arcangeli spin_unlock(ptl); 2084ba76149fSAndrea Arcangeli free_page_and_swap_cache(src_page); 2085ba76149fSAndrea Arcangeli } 2086ba76149fSAndrea Arcangeli 2087ba76149fSAndrea Arcangeli address += PAGE_SIZE; 2088ba76149fSAndrea Arcangeli page++; 2089ba76149fSAndrea Arcangeli } 2090ba76149fSAndrea Arcangeli } 2091ba76149fSAndrea Arcangeli 209226234f36SXiao Guangrong static void khugepaged_alloc_sleep(void) 209326234f36SXiao Guangrong { 2094bde43c6cSPetr Mladek DEFINE_WAIT(wait); 2095bde43c6cSPetr Mladek 2096bde43c6cSPetr Mladek add_wait_queue(&khugepaged_wait, &wait); 2097bde43c6cSPetr Mladek freezable_schedule_timeout_interruptible( 209826234f36SXiao Guangrong msecs_to_jiffies(khugepaged_alloc_sleep_millisecs)); 2099bde43c6cSPetr Mladek remove_wait_queue(&khugepaged_wait, &wait); 210026234f36SXiao Guangrong } 210126234f36SXiao Guangrong 21029f1b868aSBob Liu static int khugepaged_node_load[MAX_NUMNODES]; 21039f1b868aSBob Liu 210414a4e214SDavid Rientjes static bool khugepaged_scan_abort(int nid) 210514a4e214SDavid Rientjes { 210614a4e214SDavid Rientjes int i; 210714a4e214SDavid Rientjes 210814a4e214SDavid Rientjes /* 210914a4e214SDavid Rientjes * If zone_reclaim_mode is disabled, then no extra effort is made to 211014a4e214SDavid Rientjes * allocate memory locally. 211114a4e214SDavid Rientjes */ 211214a4e214SDavid Rientjes if (!zone_reclaim_mode) 211314a4e214SDavid Rientjes return false; 211414a4e214SDavid Rientjes 211514a4e214SDavid Rientjes /* If there is a count for this node already, it must be acceptable */ 211614a4e214SDavid Rientjes if (khugepaged_node_load[nid]) 211714a4e214SDavid Rientjes return false; 211814a4e214SDavid Rientjes 211914a4e214SDavid Rientjes for (i = 0; i < MAX_NUMNODES; i++) { 212014a4e214SDavid Rientjes if (!khugepaged_node_load[i]) 212114a4e214SDavid Rientjes continue; 212214a4e214SDavid Rientjes if (node_distance(nid, i) > RECLAIM_DISTANCE) 212314a4e214SDavid Rientjes return true; 212414a4e214SDavid Rientjes } 212514a4e214SDavid Rientjes return false; 212614a4e214SDavid Rientjes } 212714a4e214SDavid Rientjes 212826234f36SXiao Guangrong #ifdef CONFIG_NUMA 21299f1b868aSBob Liu static int khugepaged_find_target_node(void) 21309f1b868aSBob Liu { 21319f1b868aSBob Liu static int last_khugepaged_target_node = NUMA_NO_NODE; 21329f1b868aSBob Liu int nid, target_node = 0, max_value = 0; 21339f1b868aSBob Liu 21349f1b868aSBob Liu /* find first node with max normal pages hit */ 21359f1b868aSBob Liu for (nid = 0; nid < MAX_NUMNODES; nid++) 21369f1b868aSBob Liu if (khugepaged_node_load[nid] > max_value) { 21379f1b868aSBob Liu max_value = khugepaged_node_load[nid]; 21389f1b868aSBob Liu target_node = nid; 21399f1b868aSBob Liu } 21409f1b868aSBob Liu 21419f1b868aSBob Liu /* do some balance if several nodes have the same hit record */ 21429f1b868aSBob Liu if (target_node <= last_khugepaged_target_node) 21439f1b868aSBob Liu for (nid = last_khugepaged_target_node + 1; nid < MAX_NUMNODES; 21449f1b868aSBob Liu nid++) 21459f1b868aSBob Liu if (max_value == khugepaged_node_load[nid]) { 21469f1b868aSBob Liu target_node = nid; 21479f1b868aSBob Liu break; 21489f1b868aSBob Liu } 21499f1b868aSBob Liu 21509f1b868aSBob Liu last_khugepaged_target_node = target_node; 21519f1b868aSBob Liu return target_node; 21529f1b868aSBob Liu } 21539f1b868aSBob Liu 215426234f36SXiao Guangrong static bool khugepaged_prealloc_page(struct page **hpage, bool *wait) 215526234f36SXiao Guangrong { 215626234f36SXiao Guangrong if (IS_ERR(*hpage)) { 215726234f36SXiao Guangrong if (!*wait) 215826234f36SXiao Guangrong return false; 215926234f36SXiao Guangrong 216026234f36SXiao Guangrong *wait = false; 2161e3b4126cSXiao Guangrong *hpage = NULL; 216226234f36SXiao Guangrong khugepaged_alloc_sleep(); 216326234f36SXiao Guangrong } else if (*hpage) { 216426234f36SXiao Guangrong put_page(*hpage); 216526234f36SXiao Guangrong *hpage = NULL; 216626234f36SXiao Guangrong } 216726234f36SXiao Guangrong 216826234f36SXiao Guangrong return true; 216926234f36SXiao Guangrong } 217026234f36SXiao Guangrong 21713b363692SMichal Hocko static struct page * 21723b363692SMichal Hocko khugepaged_alloc_page(struct page **hpage, gfp_t gfp, struct mm_struct *mm, 2173d6669d68SAaron Tomlin unsigned long address, int node) 217426234f36SXiao Guangrong { 2175309381feSSasha Levin VM_BUG_ON_PAGE(*hpage, *hpage); 21768b164568SVlastimil Babka 217726234f36SXiao Guangrong /* 21788b164568SVlastimil Babka * Before allocating the hugepage, release the mmap_sem read lock. 21798b164568SVlastimil Babka * The allocation can take potentially a long time if it involves 21808b164568SVlastimil Babka * sync compaction, and we do not need to hold the mmap_sem during 21818b164568SVlastimil Babka * that. We will recheck the vma after taking it again in write mode. 218226234f36SXiao Guangrong */ 218326234f36SXiao Guangrong up_read(&mm->mmap_sem); 21848b164568SVlastimil Babka 218596db800fSVlastimil Babka *hpage = __alloc_pages_node(node, gfp, HPAGE_PMD_ORDER); 218626234f36SXiao Guangrong if (unlikely(!*hpage)) { 218726234f36SXiao Guangrong count_vm_event(THP_COLLAPSE_ALLOC_FAILED); 218826234f36SXiao Guangrong *hpage = ERR_PTR(-ENOMEM); 218926234f36SXiao Guangrong return NULL; 219026234f36SXiao Guangrong } 219126234f36SXiao Guangrong 21929a982250SKirill A. Shutemov prep_transhuge_page(*hpage); 219326234f36SXiao Guangrong count_vm_event(THP_COLLAPSE_ALLOC); 219426234f36SXiao Guangrong return *hpage; 219526234f36SXiao Guangrong } 219626234f36SXiao Guangrong #else 21979f1b868aSBob Liu static int khugepaged_find_target_node(void) 21989f1b868aSBob Liu { 21999f1b868aSBob Liu return 0; 22009f1b868aSBob Liu } 22019f1b868aSBob Liu 220210dc4155SBob Liu static inline struct page *alloc_hugepage(int defrag) 220310dc4155SBob Liu { 22049a982250SKirill A. Shutemov struct page *page; 22059a982250SKirill A. Shutemov 22069a982250SKirill A. Shutemov page = alloc_pages(alloc_hugepage_gfpmask(defrag, 0), HPAGE_PMD_ORDER); 22079a982250SKirill A. Shutemov if (page) 22089a982250SKirill A. Shutemov prep_transhuge_page(page); 22099a982250SKirill A. Shutemov return page; 221010dc4155SBob Liu } 221110dc4155SBob Liu 221226234f36SXiao Guangrong static struct page *khugepaged_alloc_hugepage(bool *wait) 221326234f36SXiao Guangrong { 221426234f36SXiao Guangrong struct page *hpage; 221526234f36SXiao Guangrong 221626234f36SXiao Guangrong do { 221726234f36SXiao Guangrong hpage = alloc_hugepage(khugepaged_defrag()); 221826234f36SXiao Guangrong if (!hpage) { 221926234f36SXiao Guangrong count_vm_event(THP_COLLAPSE_ALLOC_FAILED); 222026234f36SXiao Guangrong if (!*wait) 222126234f36SXiao Guangrong return NULL; 222226234f36SXiao Guangrong 222326234f36SXiao Guangrong *wait = false; 222426234f36SXiao Guangrong khugepaged_alloc_sleep(); 222526234f36SXiao Guangrong } else 222626234f36SXiao Guangrong count_vm_event(THP_COLLAPSE_ALLOC); 222726234f36SXiao Guangrong } while (unlikely(!hpage) && likely(khugepaged_enabled())); 222826234f36SXiao Guangrong 222926234f36SXiao Guangrong return hpage; 223026234f36SXiao Guangrong } 223126234f36SXiao Guangrong 223226234f36SXiao Guangrong static bool khugepaged_prealloc_page(struct page **hpage, bool *wait) 223326234f36SXiao Guangrong { 223426234f36SXiao Guangrong if (!*hpage) 223526234f36SXiao Guangrong *hpage = khugepaged_alloc_hugepage(wait); 223626234f36SXiao Guangrong 223726234f36SXiao Guangrong if (unlikely(!*hpage)) 223826234f36SXiao Guangrong return false; 223926234f36SXiao Guangrong 224026234f36SXiao Guangrong return true; 224126234f36SXiao Guangrong } 224226234f36SXiao Guangrong 22433b363692SMichal Hocko static struct page * 22443b363692SMichal Hocko khugepaged_alloc_page(struct page **hpage, gfp_t gfp, struct mm_struct *mm, 2245d6669d68SAaron Tomlin unsigned long address, int node) 224626234f36SXiao Guangrong { 224726234f36SXiao Guangrong up_read(&mm->mmap_sem); 224826234f36SXiao Guangrong VM_BUG_ON(!*hpage); 22493b363692SMichal Hocko 225026234f36SXiao Guangrong return *hpage; 225126234f36SXiao Guangrong } 225226234f36SXiao Guangrong #endif 225326234f36SXiao Guangrong 2254fa475e51SBob Liu static bool hugepage_vma_check(struct vm_area_struct *vma) 2255fa475e51SBob Liu { 2256fa475e51SBob Liu if ((!(vma->vm_flags & VM_HUGEPAGE) && !khugepaged_always()) || 2257fa475e51SBob Liu (vma->vm_flags & VM_NOHUGEPAGE)) 2258fa475e51SBob Liu return false; 2259fa475e51SBob Liu if (!vma->anon_vma || vma->vm_ops) 2260fa475e51SBob Liu return false; 2261fa475e51SBob Liu if (is_vma_temporary_stack(vma)) 2262fa475e51SBob Liu return false; 226381d1b09cSSasha Levin VM_BUG_ON_VMA(vma->vm_flags & VM_NO_THP, vma); 2264fa475e51SBob Liu return true; 2265fa475e51SBob Liu } 2266fa475e51SBob Liu 2267ba76149fSAndrea Arcangeli static void collapse_huge_page(struct mm_struct *mm, 2268ba76149fSAndrea Arcangeli unsigned long address, 2269ce83d217SAndrea Arcangeli struct page **hpage, 22705c4b4be3SAndi Kleen struct vm_area_struct *vma, 22715c4b4be3SAndi Kleen int node) 2272ba76149fSAndrea Arcangeli { 2273ba76149fSAndrea Arcangeli pmd_t *pmd, _pmd; 2274ba76149fSAndrea Arcangeli pte_t *pte; 2275ba76149fSAndrea Arcangeli pgtable_t pgtable; 2276ba76149fSAndrea Arcangeli struct page *new_page; 2277c4088ebdSKirill A. Shutemov spinlock_t *pmd_ptl, *pte_ptl; 22787d2eba05SEbru Akagunduz int isolated, result = 0; 2279ba76149fSAndrea Arcangeli unsigned long hstart, hend; 228000501b53SJohannes Weiner struct mem_cgroup *memcg; 22812ec74c3eSSagi Grimberg unsigned long mmun_start; /* For mmu_notifiers */ 22822ec74c3eSSagi Grimberg unsigned long mmun_end; /* For mmu_notifiers */ 22833b363692SMichal Hocko gfp_t gfp; 2284ba76149fSAndrea Arcangeli 2285ba76149fSAndrea Arcangeli VM_BUG_ON(address & ~HPAGE_PMD_MASK); 2286692e0b35SAndrea Arcangeli 22873b363692SMichal Hocko /* Only allocate from the target node */ 22883b363692SMichal Hocko gfp = alloc_hugepage_gfpmask(khugepaged_defrag(), __GFP_OTHER_NODE) | 22893b363692SMichal Hocko __GFP_THISNODE; 22903b363692SMichal Hocko 229126234f36SXiao Guangrong /* release the mmap_sem read lock. */ 2292d6669d68SAaron Tomlin new_page = khugepaged_alloc_page(hpage, gfp, mm, address, node); 22937d2eba05SEbru Akagunduz if (!new_page) { 22947d2eba05SEbru Akagunduz result = SCAN_ALLOC_HUGE_PAGE_FAIL; 22957d2eba05SEbru Akagunduz goto out_nolock; 22967d2eba05SEbru Akagunduz } 2297ce83d217SAndrea Arcangeli 2298f627c2f5SKirill A. Shutemov if (unlikely(mem_cgroup_try_charge(new_page, mm, gfp, &memcg, true))) { 22997d2eba05SEbru Akagunduz result = SCAN_CGROUP_CHARGE_FAIL; 23007d2eba05SEbru Akagunduz goto out_nolock; 23017d2eba05SEbru Akagunduz } 2302ba76149fSAndrea Arcangeli 2303ba76149fSAndrea Arcangeli /* 2304ba76149fSAndrea Arcangeli * Prevent all access to pagetables with the exception of 2305ba76149fSAndrea Arcangeli * gup_fast later hanlded by the ptep_clear_flush and the VM 2306ba76149fSAndrea Arcangeli * handled by the anon_vma lock + PG_lock. 2307ba76149fSAndrea Arcangeli */ 2308ba76149fSAndrea Arcangeli down_write(&mm->mmap_sem); 23097d2eba05SEbru Akagunduz if (unlikely(khugepaged_test_exit(mm))) { 23107d2eba05SEbru Akagunduz result = SCAN_ANY_PROCESS; 2311ba76149fSAndrea Arcangeli goto out; 23127d2eba05SEbru Akagunduz } 2313ba76149fSAndrea Arcangeli 2314ba76149fSAndrea Arcangeli vma = find_vma(mm, address); 23157d2eba05SEbru Akagunduz if (!vma) { 23167d2eba05SEbru Akagunduz result = SCAN_VMA_NULL; 2317a8f531ebSLibin goto out; 23187d2eba05SEbru Akagunduz } 2319ba76149fSAndrea Arcangeli hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK; 2320ba76149fSAndrea Arcangeli hend = vma->vm_end & HPAGE_PMD_MASK; 23217d2eba05SEbru Akagunduz if (address < hstart || address + HPAGE_PMD_SIZE > hend) { 23227d2eba05SEbru Akagunduz result = SCAN_ADDRESS_RANGE; 2323ba76149fSAndrea Arcangeli goto out; 23247d2eba05SEbru Akagunduz } 23257d2eba05SEbru Akagunduz if (!hugepage_vma_check(vma)) { 23267d2eba05SEbru Akagunduz result = SCAN_VMA_CHECK; 2327ba76149fSAndrea Arcangeli goto out; 23287d2eba05SEbru Akagunduz } 23296219049aSBob Liu pmd = mm_find_pmd(mm, address); 23307d2eba05SEbru Akagunduz if (!pmd) { 23317d2eba05SEbru Akagunduz result = SCAN_PMD_NULL; 2332ba76149fSAndrea Arcangeli goto out; 23337d2eba05SEbru Akagunduz } 2334ba76149fSAndrea Arcangeli 23354fc3f1d6SIngo Molnar anon_vma_lock_write(vma->anon_vma); 2336ba76149fSAndrea Arcangeli 2337ba76149fSAndrea Arcangeli pte = pte_offset_map(pmd, address); 2338c4088ebdSKirill A. Shutemov pte_ptl = pte_lockptr(mm, pmd); 2339ba76149fSAndrea Arcangeli 23402ec74c3eSSagi Grimberg mmun_start = address; 23412ec74c3eSSagi Grimberg mmun_end = address + HPAGE_PMD_SIZE; 23422ec74c3eSSagi Grimberg mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 2343c4088ebdSKirill A. Shutemov pmd_ptl = pmd_lock(mm, pmd); /* probably unnecessary */ 2344ba76149fSAndrea Arcangeli /* 2345ba76149fSAndrea Arcangeli * After this gup_fast can't run anymore. This also removes 2346ba76149fSAndrea Arcangeli * any huge TLB entry from the CPU so we won't allow 2347ba76149fSAndrea Arcangeli * huge and small TLB entries for the same virtual address 2348ba76149fSAndrea Arcangeli * to avoid the risk of CPU bugs in that area. 2349ba76149fSAndrea Arcangeli */ 235015a25b2eSAneesh Kumar K.V _pmd = pmdp_collapse_flush(vma, address, pmd); 2351c4088ebdSKirill A. Shutemov spin_unlock(pmd_ptl); 23522ec74c3eSSagi Grimberg mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 2353ba76149fSAndrea Arcangeli 2354c4088ebdSKirill A. Shutemov spin_lock(pte_ptl); 2355ba76149fSAndrea Arcangeli isolated = __collapse_huge_page_isolate(vma, address, pte); 2356c4088ebdSKirill A. Shutemov spin_unlock(pte_ptl); 2357ba76149fSAndrea Arcangeli 2358ba76149fSAndrea Arcangeli if (unlikely(!isolated)) { 2359453c7192SJohannes Weiner pte_unmap(pte); 2360c4088ebdSKirill A. Shutemov spin_lock(pmd_ptl); 2361ba76149fSAndrea Arcangeli BUG_ON(!pmd_none(*pmd)); 23627c342512SAneesh Kumar K.V /* 23637c342512SAneesh Kumar K.V * We can only use set_pmd_at when establishing 23647c342512SAneesh Kumar K.V * hugepmds and never for establishing regular pmds that 23657c342512SAneesh Kumar K.V * points to regular pagetables. Use pmd_populate for that 23667c342512SAneesh Kumar K.V */ 23677c342512SAneesh Kumar K.V pmd_populate(mm, pmd, pmd_pgtable(_pmd)); 2368c4088ebdSKirill A. Shutemov spin_unlock(pmd_ptl); 236908b52706SKonstantin Khlebnikov anon_vma_unlock_write(vma->anon_vma); 23707d2eba05SEbru Akagunduz result = SCAN_FAIL; 2371ce83d217SAndrea Arcangeli goto out; 2372ba76149fSAndrea Arcangeli } 2373ba76149fSAndrea Arcangeli 2374ba76149fSAndrea Arcangeli /* 2375ba76149fSAndrea Arcangeli * All pages are isolated and locked so anon_vma rmap 2376ba76149fSAndrea Arcangeli * can't run anymore. 2377ba76149fSAndrea Arcangeli */ 237808b52706SKonstantin Khlebnikov anon_vma_unlock_write(vma->anon_vma); 2379ba76149fSAndrea Arcangeli 2380c4088ebdSKirill A. Shutemov __collapse_huge_page_copy(pte, new_page, vma, address, pte_ptl); 2381453c7192SJohannes Weiner pte_unmap(pte); 2382ba76149fSAndrea Arcangeli __SetPageUptodate(new_page); 2383ba76149fSAndrea Arcangeli pgtable = pmd_pgtable(_pmd); 2384ba76149fSAndrea Arcangeli 23853122359aSKirill A. Shutemov _pmd = mk_huge_pmd(new_page, vma->vm_page_prot); 23863122359aSKirill A. Shutemov _pmd = maybe_pmd_mkwrite(pmd_mkdirty(_pmd), vma); 2387ba76149fSAndrea Arcangeli 2388ba76149fSAndrea Arcangeli /* 2389ba76149fSAndrea Arcangeli * spin_lock() below is not the equivalent of smp_wmb(), so 2390ba76149fSAndrea Arcangeli * this is needed to avoid the copy_huge_page writes to become 2391ba76149fSAndrea Arcangeli * visible after the set_pmd_at() write. 2392ba76149fSAndrea Arcangeli */ 2393ba76149fSAndrea Arcangeli smp_wmb(); 2394ba76149fSAndrea Arcangeli 2395c4088ebdSKirill A. Shutemov spin_lock(pmd_ptl); 2396ba76149fSAndrea Arcangeli BUG_ON(!pmd_none(*pmd)); 2397d281ee61SKirill A. Shutemov page_add_new_anon_rmap(new_page, vma, address, true); 2398f627c2f5SKirill A. Shutemov mem_cgroup_commit_charge(new_page, memcg, false, true); 239900501b53SJohannes Weiner lru_cache_add_active_or_unevictable(new_page, vma); 2400fce144b4SAneesh Kumar K.V pgtable_trans_huge_deposit(mm, pmd, pgtable); 2401ba76149fSAndrea Arcangeli set_pmd_at(mm, address, pmd, _pmd); 2402b113da65SDavid Miller update_mmu_cache_pmd(vma, address, pmd); 2403c4088ebdSKirill A. Shutemov spin_unlock(pmd_ptl); 2404ba76149fSAndrea Arcangeli 2405ba76149fSAndrea Arcangeli *hpage = NULL; 2406420256efSXiao Guangrong 2407ba76149fSAndrea Arcangeli khugepaged_pages_collapsed++; 24087d2eba05SEbru Akagunduz result = SCAN_SUCCEED; 2409ce83d217SAndrea Arcangeli out_up_write: 2410ba76149fSAndrea Arcangeli up_write(&mm->mmap_sem); 24117d2eba05SEbru Akagunduz trace_mm_collapse_huge_page(mm, isolated, result); 24120bbbc0b3SAndrea Arcangeli return; 24130bbbc0b3SAndrea Arcangeli 24147d2eba05SEbru Akagunduz out_nolock: 24157d2eba05SEbru Akagunduz trace_mm_collapse_huge_page(mm, isolated, result); 24167d2eba05SEbru Akagunduz return; 2417ce83d217SAndrea Arcangeli out: 2418f627c2f5SKirill A. Shutemov mem_cgroup_cancel_charge(new_page, memcg, true); 2419ce83d217SAndrea Arcangeli goto out_up_write; 2420ba76149fSAndrea Arcangeli } 2421ba76149fSAndrea Arcangeli 2422ba76149fSAndrea Arcangeli static int khugepaged_scan_pmd(struct mm_struct *mm, 2423ba76149fSAndrea Arcangeli struct vm_area_struct *vma, 2424ba76149fSAndrea Arcangeli unsigned long address, 2425ba76149fSAndrea Arcangeli struct page **hpage) 2426ba76149fSAndrea Arcangeli { 2427ba76149fSAndrea Arcangeli pmd_t *pmd; 2428ba76149fSAndrea Arcangeli pte_t *pte, *_pte; 24297d2eba05SEbru Akagunduz int ret = 0, none_or_zero = 0, result = 0; 24307d2eba05SEbru Akagunduz struct page *page = NULL; 2431ba76149fSAndrea Arcangeli unsigned long _address; 2432ba76149fSAndrea Arcangeli spinlock_t *ptl; 243300ef2d2fSDavid Rientjes int node = NUMA_NO_NODE; 243410359213SEbru Akagunduz bool writable = false, referenced = false; 2435ba76149fSAndrea Arcangeli 2436ba76149fSAndrea Arcangeli VM_BUG_ON(address & ~HPAGE_PMD_MASK); 2437ba76149fSAndrea Arcangeli 24386219049aSBob Liu pmd = mm_find_pmd(mm, address); 24397d2eba05SEbru Akagunduz if (!pmd) { 24407d2eba05SEbru Akagunduz result = SCAN_PMD_NULL; 2441ba76149fSAndrea Arcangeli goto out; 24427d2eba05SEbru Akagunduz } 2443ba76149fSAndrea Arcangeli 24449f1b868aSBob Liu memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load)); 2445ba76149fSAndrea Arcangeli pte = pte_offset_map_lock(mm, pmd, address, &ptl); 2446ba76149fSAndrea Arcangeli for (_address = address, _pte = pte; _pte < pte+HPAGE_PMD_NR; 2447ba76149fSAndrea Arcangeli _pte++, _address += PAGE_SIZE) { 2448ba76149fSAndrea Arcangeli pte_t pteval = *_pte; 2449ca0984caSEbru Akagunduz if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) { 2450c1294d05SAndrea Arcangeli if (!userfaultfd_armed(vma) && 24517d2eba05SEbru Akagunduz ++none_or_zero <= khugepaged_max_ptes_none) { 2452ba76149fSAndrea Arcangeli continue; 24537d2eba05SEbru Akagunduz } else { 24547d2eba05SEbru Akagunduz result = SCAN_EXCEED_NONE_PTE; 2455ba76149fSAndrea Arcangeli goto out_unmap; 2456ba76149fSAndrea Arcangeli } 24577d2eba05SEbru Akagunduz } 24587d2eba05SEbru Akagunduz if (!pte_present(pteval)) { 24597d2eba05SEbru Akagunduz result = SCAN_PTE_NON_PRESENT; 2460ba76149fSAndrea Arcangeli goto out_unmap; 24617d2eba05SEbru Akagunduz } 246210359213SEbru Akagunduz if (pte_write(pteval)) 246310359213SEbru Akagunduz writable = true; 246410359213SEbru Akagunduz 2465ba76149fSAndrea Arcangeli page = vm_normal_page(vma, _address, pteval); 24667d2eba05SEbru Akagunduz if (unlikely(!page)) { 24677d2eba05SEbru Akagunduz result = SCAN_PAGE_NULL; 2468ba76149fSAndrea Arcangeli goto out_unmap; 24697d2eba05SEbru Akagunduz } 2470b1caa957SKirill A. Shutemov 2471b1caa957SKirill A. Shutemov /* TODO: teach khugepaged to collapse THP mapped with pte */ 2472b1caa957SKirill A. Shutemov if (PageCompound(page)) { 2473b1caa957SKirill A. Shutemov result = SCAN_PAGE_COMPOUND; 2474b1caa957SKirill A. Shutemov goto out_unmap; 2475b1caa957SKirill A. Shutemov } 2476b1caa957SKirill A. Shutemov 24775c4b4be3SAndi Kleen /* 24789f1b868aSBob Liu * Record which node the original page is from and save this 24799f1b868aSBob Liu * information to khugepaged_node_load[]. 24809f1b868aSBob Liu * Khupaged will allocate hugepage from the node has the max 24819f1b868aSBob Liu * hit record. 24825c4b4be3SAndi Kleen */ 24835c4b4be3SAndi Kleen node = page_to_nid(page); 24847d2eba05SEbru Akagunduz if (khugepaged_scan_abort(node)) { 24857d2eba05SEbru Akagunduz result = SCAN_SCAN_ABORT; 248614a4e214SDavid Rientjes goto out_unmap; 24877d2eba05SEbru Akagunduz } 24889f1b868aSBob Liu khugepaged_node_load[node]++; 24897d2eba05SEbru Akagunduz if (!PageLRU(page)) { 24907d2eba05SEbru Akagunduz result = SCAN_SCAN_ABORT; 2491ba76149fSAndrea Arcangeli goto out_unmap; 24927d2eba05SEbru Akagunduz } 24937d2eba05SEbru Akagunduz if (PageLocked(page)) { 24947d2eba05SEbru Akagunduz result = SCAN_PAGE_LOCK; 24957d2eba05SEbru Akagunduz goto out_unmap; 24967d2eba05SEbru Akagunduz } 24977d2eba05SEbru Akagunduz if (!PageAnon(page)) { 24987d2eba05SEbru Akagunduz result = SCAN_PAGE_ANON; 24997d2eba05SEbru Akagunduz goto out_unmap; 25007d2eba05SEbru Akagunduz } 25017d2eba05SEbru Akagunduz 250210359213SEbru Akagunduz /* 250310359213SEbru Akagunduz * cannot use mapcount: can't collapse if there's a gup pin. 250410359213SEbru Akagunduz * The page must only be referenced by the scanned process 250510359213SEbru Akagunduz * and page swap cache. 250610359213SEbru Akagunduz */ 25077d2eba05SEbru Akagunduz if (page_count(page) != 1 + !!PageSwapCache(page)) { 25087d2eba05SEbru Akagunduz result = SCAN_PAGE_COUNT; 2509ba76149fSAndrea Arcangeli goto out_unmap; 25107d2eba05SEbru Akagunduz } 251133c3fc71SVladimir Davydov if (pte_young(pteval) || 251233c3fc71SVladimir Davydov page_is_young(page) || PageReferenced(page) || 25138ee53820SAndrea Arcangeli mmu_notifier_test_young(vma->vm_mm, address)) 251410359213SEbru Akagunduz referenced = true; 2515ba76149fSAndrea Arcangeli } 25167d2eba05SEbru Akagunduz if (writable) { 25177d2eba05SEbru Akagunduz if (referenced) { 25187d2eba05SEbru Akagunduz result = SCAN_SUCCEED; 2519ba76149fSAndrea Arcangeli ret = 1; 25207d2eba05SEbru Akagunduz } else { 25217d2eba05SEbru Akagunduz result = SCAN_NO_REFERENCED_PAGE; 25227d2eba05SEbru Akagunduz } 25237d2eba05SEbru Akagunduz } else { 25247d2eba05SEbru Akagunduz result = SCAN_PAGE_RO; 25257d2eba05SEbru Akagunduz } 2526ba76149fSAndrea Arcangeli out_unmap: 2527ba76149fSAndrea Arcangeli pte_unmap_unlock(pte, ptl); 25289f1b868aSBob Liu if (ret) { 25299f1b868aSBob Liu node = khugepaged_find_target_node(); 2530ce83d217SAndrea Arcangeli /* collapse_huge_page will return with the mmap_sem released */ 25315c4b4be3SAndi Kleen collapse_huge_page(mm, address, hpage, vma, node); 25329f1b868aSBob Liu } 2533ba76149fSAndrea Arcangeli out: 25347d2eba05SEbru Akagunduz trace_mm_khugepaged_scan_pmd(mm, page_to_pfn(page), writable, referenced, 25357d2eba05SEbru Akagunduz none_or_zero, result); 2536ba76149fSAndrea Arcangeli return ret; 2537ba76149fSAndrea Arcangeli } 2538ba76149fSAndrea Arcangeli 2539ba76149fSAndrea Arcangeli static void collect_mm_slot(struct mm_slot *mm_slot) 2540ba76149fSAndrea Arcangeli { 2541ba76149fSAndrea Arcangeli struct mm_struct *mm = mm_slot->mm; 2542ba76149fSAndrea Arcangeli 2543b9980cdcSHugh Dickins VM_BUG_ON(NR_CPUS != 1 && !spin_is_locked(&khugepaged_mm_lock)); 2544ba76149fSAndrea Arcangeli 2545ba76149fSAndrea Arcangeli if (khugepaged_test_exit(mm)) { 2546ba76149fSAndrea Arcangeli /* free mm_slot */ 254743b5fbbdSSasha Levin hash_del(&mm_slot->hash); 2548ba76149fSAndrea Arcangeli list_del(&mm_slot->mm_node); 2549ba76149fSAndrea Arcangeli 2550ba76149fSAndrea Arcangeli /* 2551ba76149fSAndrea Arcangeli * Not strictly needed because the mm exited already. 2552ba76149fSAndrea Arcangeli * 2553ba76149fSAndrea Arcangeli * clear_bit(MMF_VM_HUGEPAGE, &mm->flags); 2554ba76149fSAndrea Arcangeli */ 2555ba76149fSAndrea Arcangeli 2556ba76149fSAndrea Arcangeli /* khugepaged_mm_lock actually not necessary for the below */ 2557ba76149fSAndrea Arcangeli free_mm_slot(mm_slot); 2558ba76149fSAndrea Arcangeli mmdrop(mm); 2559ba76149fSAndrea Arcangeli } 2560ba76149fSAndrea Arcangeli } 2561ba76149fSAndrea Arcangeli 2562ba76149fSAndrea Arcangeli static unsigned int khugepaged_scan_mm_slot(unsigned int pages, 2563ba76149fSAndrea Arcangeli struct page **hpage) 25642f1da642SH Hartley Sweeten __releases(&khugepaged_mm_lock) 25652f1da642SH Hartley Sweeten __acquires(&khugepaged_mm_lock) 2566ba76149fSAndrea Arcangeli { 2567ba76149fSAndrea Arcangeli struct mm_slot *mm_slot; 2568ba76149fSAndrea Arcangeli struct mm_struct *mm; 2569ba76149fSAndrea Arcangeli struct vm_area_struct *vma; 2570ba76149fSAndrea Arcangeli int progress = 0; 2571ba76149fSAndrea Arcangeli 2572ba76149fSAndrea Arcangeli VM_BUG_ON(!pages); 2573b9980cdcSHugh Dickins VM_BUG_ON(NR_CPUS != 1 && !spin_is_locked(&khugepaged_mm_lock)); 2574ba76149fSAndrea Arcangeli 2575ba76149fSAndrea Arcangeli if (khugepaged_scan.mm_slot) 2576ba76149fSAndrea Arcangeli mm_slot = khugepaged_scan.mm_slot; 2577ba76149fSAndrea Arcangeli else { 2578ba76149fSAndrea Arcangeli mm_slot = list_entry(khugepaged_scan.mm_head.next, 2579ba76149fSAndrea Arcangeli struct mm_slot, mm_node); 2580ba76149fSAndrea Arcangeli khugepaged_scan.address = 0; 2581ba76149fSAndrea Arcangeli khugepaged_scan.mm_slot = mm_slot; 2582ba76149fSAndrea Arcangeli } 2583ba76149fSAndrea Arcangeli spin_unlock(&khugepaged_mm_lock); 2584ba76149fSAndrea Arcangeli 2585ba76149fSAndrea Arcangeli mm = mm_slot->mm; 2586ba76149fSAndrea Arcangeli down_read(&mm->mmap_sem); 2587ba76149fSAndrea Arcangeli if (unlikely(khugepaged_test_exit(mm))) 2588ba76149fSAndrea Arcangeli vma = NULL; 2589ba76149fSAndrea Arcangeli else 2590ba76149fSAndrea Arcangeli vma = find_vma(mm, khugepaged_scan.address); 2591ba76149fSAndrea Arcangeli 2592ba76149fSAndrea Arcangeli progress++; 2593ba76149fSAndrea Arcangeli for (; vma; vma = vma->vm_next) { 2594ba76149fSAndrea Arcangeli unsigned long hstart, hend; 2595ba76149fSAndrea Arcangeli 2596ba76149fSAndrea Arcangeli cond_resched(); 2597ba76149fSAndrea Arcangeli if (unlikely(khugepaged_test_exit(mm))) { 2598ba76149fSAndrea Arcangeli progress++; 2599ba76149fSAndrea Arcangeli break; 2600ba76149fSAndrea Arcangeli } 2601fa475e51SBob Liu if (!hugepage_vma_check(vma)) { 2602a7d6e4ecSAndrea Arcangeli skip: 2603ba76149fSAndrea Arcangeli progress++; 2604ba76149fSAndrea Arcangeli continue; 2605ba76149fSAndrea Arcangeli } 2606ba76149fSAndrea Arcangeli hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK; 2607ba76149fSAndrea Arcangeli hend = vma->vm_end & HPAGE_PMD_MASK; 2608a7d6e4ecSAndrea Arcangeli if (hstart >= hend) 2609a7d6e4ecSAndrea Arcangeli goto skip; 2610a7d6e4ecSAndrea Arcangeli if (khugepaged_scan.address > hend) 2611a7d6e4ecSAndrea Arcangeli goto skip; 2612ba76149fSAndrea Arcangeli if (khugepaged_scan.address < hstart) 2613ba76149fSAndrea Arcangeli khugepaged_scan.address = hstart; 2614a7d6e4ecSAndrea Arcangeli VM_BUG_ON(khugepaged_scan.address & ~HPAGE_PMD_MASK); 2615ba76149fSAndrea Arcangeli 2616ba76149fSAndrea Arcangeli while (khugepaged_scan.address < hend) { 2617ba76149fSAndrea Arcangeli int ret; 2618ba76149fSAndrea Arcangeli cond_resched(); 2619ba76149fSAndrea Arcangeli if (unlikely(khugepaged_test_exit(mm))) 2620ba76149fSAndrea Arcangeli goto breakouterloop; 2621ba76149fSAndrea Arcangeli 2622ba76149fSAndrea Arcangeli VM_BUG_ON(khugepaged_scan.address < hstart || 2623ba76149fSAndrea Arcangeli khugepaged_scan.address + HPAGE_PMD_SIZE > 2624ba76149fSAndrea Arcangeli hend); 2625ba76149fSAndrea Arcangeli ret = khugepaged_scan_pmd(mm, vma, 2626ba76149fSAndrea Arcangeli khugepaged_scan.address, 2627ba76149fSAndrea Arcangeli hpage); 2628ba76149fSAndrea Arcangeli /* move to next address */ 2629ba76149fSAndrea Arcangeli khugepaged_scan.address += HPAGE_PMD_SIZE; 2630ba76149fSAndrea Arcangeli progress += HPAGE_PMD_NR; 2631ba76149fSAndrea Arcangeli if (ret) 2632ba76149fSAndrea Arcangeli /* we released mmap_sem so break loop */ 2633ba76149fSAndrea Arcangeli goto breakouterloop_mmap_sem; 2634ba76149fSAndrea Arcangeli if (progress >= pages) 2635ba76149fSAndrea Arcangeli goto breakouterloop; 2636ba76149fSAndrea Arcangeli } 2637ba76149fSAndrea Arcangeli } 2638ba76149fSAndrea Arcangeli breakouterloop: 2639ba76149fSAndrea Arcangeli up_read(&mm->mmap_sem); /* exit_mmap will destroy ptes after this */ 2640ba76149fSAndrea Arcangeli breakouterloop_mmap_sem: 2641ba76149fSAndrea Arcangeli 2642ba76149fSAndrea Arcangeli spin_lock(&khugepaged_mm_lock); 2643a7d6e4ecSAndrea Arcangeli VM_BUG_ON(khugepaged_scan.mm_slot != mm_slot); 2644ba76149fSAndrea Arcangeli /* 2645ba76149fSAndrea Arcangeli * Release the current mm_slot if this mm is about to die, or 2646ba76149fSAndrea Arcangeli * if we scanned all vmas of this mm. 2647ba76149fSAndrea Arcangeli */ 2648ba76149fSAndrea Arcangeli if (khugepaged_test_exit(mm) || !vma) { 2649ba76149fSAndrea Arcangeli /* 2650ba76149fSAndrea Arcangeli * Make sure that if mm_users is reaching zero while 2651ba76149fSAndrea Arcangeli * khugepaged runs here, khugepaged_exit will find 2652ba76149fSAndrea Arcangeli * mm_slot not pointing to the exiting mm. 2653ba76149fSAndrea Arcangeli */ 2654ba76149fSAndrea Arcangeli if (mm_slot->mm_node.next != &khugepaged_scan.mm_head) { 2655ba76149fSAndrea Arcangeli khugepaged_scan.mm_slot = list_entry( 2656ba76149fSAndrea Arcangeli mm_slot->mm_node.next, 2657ba76149fSAndrea Arcangeli struct mm_slot, mm_node); 2658ba76149fSAndrea Arcangeli khugepaged_scan.address = 0; 2659ba76149fSAndrea Arcangeli } else { 2660ba76149fSAndrea Arcangeli khugepaged_scan.mm_slot = NULL; 2661ba76149fSAndrea Arcangeli khugepaged_full_scans++; 2662ba76149fSAndrea Arcangeli } 2663ba76149fSAndrea Arcangeli 2664ba76149fSAndrea Arcangeli collect_mm_slot(mm_slot); 2665ba76149fSAndrea Arcangeli } 2666ba76149fSAndrea Arcangeli 2667ba76149fSAndrea Arcangeli return progress; 2668ba76149fSAndrea Arcangeli } 2669ba76149fSAndrea Arcangeli 2670ba76149fSAndrea Arcangeli static int khugepaged_has_work(void) 2671ba76149fSAndrea Arcangeli { 2672ba76149fSAndrea Arcangeli return !list_empty(&khugepaged_scan.mm_head) && 2673ba76149fSAndrea Arcangeli khugepaged_enabled(); 2674ba76149fSAndrea Arcangeli } 2675ba76149fSAndrea Arcangeli 2676ba76149fSAndrea Arcangeli static int khugepaged_wait_event(void) 2677ba76149fSAndrea Arcangeli { 2678ba76149fSAndrea Arcangeli return !list_empty(&khugepaged_scan.mm_head) || 26792017c0bfSXiao Guangrong kthread_should_stop(); 2680ba76149fSAndrea Arcangeli } 2681ba76149fSAndrea Arcangeli 2682d516904bSXiao Guangrong static void khugepaged_do_scan(void) 2683d516904bSXiao Guangrong { 2684d516904bSXiao Guangrong struct page *hpage = NULL; 2685ba76149fSAndrea Arcangeli unsigned int progress = 0, pass_through_head = 0; 2686ba76149fSAndrea Arcangeli unsigned int pages = khugepaged_pages_to_scan; 2687d516904bSXiao Guangrong bool wait = true; 2688ba76149fSAndrea Arcangeli 2689ba76149fSAndrea Arcangeli barrier(); /* write khugepaged_pages_to_scan to local stack */ 2690ba76149fSAndrea Arcangeli 2691ba76149fSAndrea Arcangeli while (progress < pages) { 269226234f36SXiao Guangrong if (!khugepaged_prealloc_page(&hpage, &wait)) 269326234f36SXiao Guangrong break; 2694d516904bSXiao Guangrong 2695420256efSXiao Guangrong cond_resched(); 2696ba76149fSAndrea Arcangeli 2697cd092411SJiri Kosina if (unlikely(kthread_should_stop() || try_to_freeze())) 2698878aee7dSAndrea Arcangeli break; 2699878aee7dSAndrea Arcangeli 2700ba76149fSAndrea Arcangeli spin_lock(&khugepaged_mm_lock); 2701ba76149fSAndrea Arcangeli if (!khugepaged_scan.mm_slot) 2702ba76149fSAndrea Arcangeli pass_through_head++; 2703ba76149fSAndrea Arcangeli if (khugepaged_has_work() && 2704ba76149fSAndrea Arcangeli pass_through_head < 2) 2705ba76149fSAndrea Arcangeli progress += khugepaged_scan_mm_slot(pages - progress, 2706d516904bSXiao Guangrong &hpage); 2707ba76149fSAndrea Arcangeli else 2708ba76149fSAndrea Arcangeli progress = pages; 2709ba76149fSAndrea Arcangeli spin_unlock(&khugepaged_mm_lock); 2710ba76149fSAndrea Arcangeli } 2711ba76149fSAndrea Arcangeli 2712d516904bSXiao Guangrong if (!IS_ERR_OR_NULL(hpage)) 2713d516904bSXiao Guangrong put_page(hpage); 2714ba76149fSAndrea Arcangeli } 27150bbbc0b3SAndrea Arcangeli 27162017c0bfSXiao Guangrong static void khugepaged_wait_work(void) 27172017c0bfSXiao Guangrong { 27182017c0bfSXiao Guangrong if (khugepaged_has_work()) { 27192017c0bfSXiao Guangrong if (!khugepaged_scan_sleep_millisecs) 27202017c0bfSXiao Guangrong return; 27212017c0bfSXiao Guangrong 27222017c0bfSXiao Guangrong wait_event_freezable_timeout(khugepaged_wait, 27232017c0bfSXiao Guangrong kthread_should_stop(), 27242017c0bfSXiao Guangrong msecs_to_jiffies(khugepaged_scan_sleep_millisecs)); 27252017c0bfSXiao Guangrong return; 27262017c0bfSXiao Guangrong } 27272017c0bfSXiao Guangrong 27282017c0bfSXiao Guangrong if (khugepaged_enabled()) 27292017c0bfSXiao Guangrong wait_event_freezable(khugepaged_wait, khugepaged_wait_event()); 27302017c0bfSXiao Guangrong } 27312017c0bfSXiao Guangrong 2732ba76149fSAndrea Arcangeli static int khugepaged(void *none) 2733ba76149fSAndrea Arcangeli { 2734ba76149fSAndrea Arcangeli struct mm_slot *mm_slot; 2735ba76149fSAndrea Arcangeli 2736878aee7dSAndrea Arcangeli set_freezable(); 27378698a745SDongsheng Yang set_user_nice(current, MAX_NICE); 2738ba76149fSAndrea Arcangeli 2739b7231789SXiao Guangrong while (!kthread_should_stop()) { 2740b7231789SXiao Guangrong khugepaged_do_scan(); 2741b7231789SXiao Guangrong khugepaged_wait_work(); 2742b7231789SXiao Guangrong } 2743ba76149fSAndrea Arcangeli 2744ba76149fSAndrea Arcangeli spin_lock(&khugepaged_mm_lock); 2745ba76149fSAndrea Arcangeli mm_slot = khugepaged_scan.mm_slot; 2746ba76149fSAndrea Arcangeli khugepaged_scan.mm_slot = NULL; 2747ba76149fSAndrea Arcangeli if (mm_slot) 2748ba76149fSAndrea Arcangeli collect_mm_slot(mm_slot); 2749ba76149fSAndrea Arcangeli spin_unlock(&khugepaged_mm_lock); 2750ba76149fSAndrea Arcangeli return 0; 2751ba76149fSAndrea Arcangeli } 2752ba76149fSAndrea Arcangeli 2753eef1b3baSKirill A. Shutemov static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, 2754eef1b3baSKirill A. Shutemov unsigned long haddr, pmd_t *pmd) 2755eef1b3baSKirill A. Shutemov { 2756eef1b3baSKirill A. Shutemov struct mm_struct *mm = vma->vm_mm; 2757eef1b3baSKirill A. Shutemov pgtable_t pgtable; 2758eef1b3baSKirill A. Shutemov pmd_t _pmd; 2759eef1b3baSKirill A. Shutemov int i; 2760eef1b3baSKirill A. Shutemov 2761eef1b3baSKirill A. Shutemov /* leave pmd empty until pte is filled */ 2762eef1b3baSKirill A. Shutemov pmdp_huge_clear_flush_notify(vma, haddr, pmd); 2763eef1b3baSKirill A. Shutemov 2764eef1b3baSKirill A. Shutemov pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2765eef1b3baSKirill A. Shutemov pmd_populate(mm, &_pmd, pgtable); 2766eef1b3baSKirill A. Shutemov 2767eef1b3baSKirill A. Shutemov for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) { 2768eef1b3baSKirill A. Shutemov pte_t *pte, entry; 2769eef1b3baSKirill A. Shutemov entry = pfn_pte(my_zero_pfn(haddr), vma->vm_page_prot); 2770eef1b3baSKirill A. Shutemov entry = pte_mkspecial(entry); 2771eef1b3baSKirill A. Shutemov pte = pte_offset_map(&_pmd, haddr); 2772eef1b3baSKirill A. Shutemov VM_BUG_ON(!pte_none(*pte)); 2773eef1b3baSKirill A. Shutemov set_pte_at(mm, haddr, pte, entry); 2774eef1b3baSKirill A. Shutemov pte_unmap(pte); 2775eef1b3baSKirill A. Shutemov } 2776eef1b3baSKirill A. Shutemov smp_wmb(); /* make pte visible before pmd */ 2777eef1b3baSKirill A. Shutemov pmd_populate(mm, pmd, pgtable); 2778eef1b3baSKirill A. Shutemov put_huge_zero_page(); 2779eef1b3baSKirill A. Shutemov } 2780eef1b3baSKirill A. Shutemov 2781eef1b3baSKirill A. Shutemov static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, 2782ba988280SKirill A. Shutemov unsigned long haddr, bool freeze) 2783eef1b3baSKirill A. Shutemov { 2784eef1b3baSKirill A. Shutemov struct mm_struct *mm = vma->vm_mm; 2785eef1b3baSKirill A. Shutemov struct page *page; 2786eef1b3baSKirill A. Shutemov pgtable_t pgtable; 2787eef1b3baSKirill A. Shutemov pmd_t _pmd; 2788b8d3c4c3SMinchan Kim bool young, write, dirty; 2789eef1b3baSKirill A. Shutemov int i; 2790eef1b3baSKirill A. Shutemov 2791eef1b3baSKirill A. Shutemov VM_BUG_ON(haddr & ~HPAGE_PMD_MASK); 2792eef1b3baSKirill A. Shutemov VM_BUG_ON_VMA(vma->vm_start > haddr, vma); 2793eef1b3baSKirill A. Shutemov VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma); 2794*5c7fb56eSDan Williams VM_BUG_ON(!pmd_trans_huge(*pmd) && !pmd_devmap(*pmd)); 2795eef1b3baSKirill A. Shutemov 2796eef1b3baSKirill A. Shutemov count_vm_event(THP_SPLIT_PMD); 2797eef1b3baSKirill A. Shutemov 2798eef1b3baSKirill A. Shutemov if (vma_is_dax(vma)) { 2799eef1b3baSKirill A. Shutemov pmd_t _pmd = pmdp_huge_clear_flush_notify(vma, haddr, pmd); 2800eef1b3baSKirill A. Shutemov if (is_huge_zero_pmd(_pmd)) 2801eef1b3baSKirill A. Shutemov put_huge_zero_page(); 2802eef1b3baSKirill A. Shutemov return; 2803eef1b3baSKirill A. Shutemov } else if (is_huge_zero_pmd(*pmd)) { 2804eef1b3baSKirill A. Shutemov return __split_huge_zero_page_pmd(vma, haddr, pmd); 2805eef1b3baSKirill A. Shutemov } 2806eef1b3baSKirill A. Shutemov 2807eef1b3baSKirill A. Shutemov page = pmd_page(*pmd); 2808eef1b3baSKirill A. Shutemov VM_BUG_ON_PAGE(!page_count(page), page); 2809eef1b3baSKirill A. Shutemov atomic_add(HPAGE_PMD_NR - 1, &page->_count); 2810eef1b3baSKirill A. Shutemov write = pmd_write(*pmd); 2811eef1b3baSKirill A. Shutemov young = pmd_young(*pmd); 2812b8d3c4c3SMinchan Kim dirty = pmd_dirty(*pmd); 2813eef1b3baSKirill A. Shutemov 2814eef1b3baSKirill A. Shutemov pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2815eef1b3baSKirill A. Shutemov pmd_populate(mm, &_pmd, pgtable); 2816eef1b3baSKirill A. Shutemov 2817eef1b3baSKirill A. Shutemov for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) { 2818eef1b3baSKirill A. Shutemov pte_t entry, *pte; 2819eef1b3baSKirill A. Shutemov /* 2820eef1b3baSKirill A. Shutemov * Note that NUMA hinting access restrictions are not 2821eef1b3baSKirill A. Shutemov * transferred to avoid any possibility of altering 2822eef1b3baSKirill A. Shutemov * permissions across VMAs. 2823eef1b3baSKirill A. Shutemov */ 2824ba988280SKirill A. Shutemov if (freeze) { 2825ba988280SKirill A. Shutemov swp_entry_t swp_entry; 2826ba988280SKirill A. Shutemov swp_entry = make_migration_entry(page + i, write); 2827ba988280SKirill A. Shutemov entry = swp_entry_to_pte(swp_entry); 2828ba988280SKirill A. Shutemov } else { 2829eef1b3baSKirill A. Shutemov entry = mk_pte(page + i, vma->vm_page_prot); 2830b8d3c4c3SMinchan Kim entry = maybe_mkwrite(entry, vma); 2831eef1b3baSKirill A. Shutemov if (!write) 2832eef1b3baSKirill A. Shutemov entry = pte_wrprotect(entry); 2833eef1b3baSKirill A. Shutemov if (!young) 2834eef1b3baSKirill A. Shutemov entry = pte_mkold(entry); 2835ba988280SKirill A. Shutemov } 2836b8d3c4c3SMinchan Kim if (dirty) 2837b8d3c4c3SMinchan Kim SetPageDirty(page + i); 2838eef1b3baSKirill A. Shutemov pte = pte_offset_map(&_pmd, haddr); 2839eef1b3baSKirill A. Shutemov BUG_ON(!pte_none(*pte)); 2840eef1b3baSKirill A. Shutemov set_pte_at(mm, haddr, pte, entry); 2841eef1b3baSKirill A. Shutemov atomic_inc(&page[i]._mapcount); 2842eef1b3baSKirill A. Shutemov pte_unmap(pte); 2843eef1b3baSKirill A. Shutemov } 2844eef1b3baSKirill A. Shutemov 2845eef1b3baSKirill A. Shutemov /* 2846eef1b3baSKirill A. Shutemov * Set PG_double_map before dropping compound_mapcount to avoid 2847eef1b3baSKirill A. Shutemov * false-negative page_mapped(). 2848eef1b3baSKirill A. Shutemov */ 2849eef1b3baSKirill A. Shutemov if (compound_mapcount(page) > 1 && !TestSetPageDoubleMap(page)) { 2850eef1b3baSKirill A. Shutemov for (i = 0; i < HPAGE_PMD_NR; i++) 2851eef1b3baSKirill A. Shutemov atomic_inc(&page[i]._mapcount); 2852eef1b3baSKirill A. Shutemov } 2853eef1b3baSKirill A. Shutemov 2854eef1b3baSKirill A. Shutemov if (atomic_add_negative(-1, compound_mapcount_ptr(page))) { 2855eef1b3baSKirill A. Shutemov /* Last compound_mapcount is gone. */ 2856eef1b3baSKirill A. Shutemov __dec_zone_page_state(page, NR_ANON_TRANSPARENT_HUGEPAGES); 2857eef1b3baSKirill A. Shutemov if (TestClearPageDoubleMap(page)) { 2858eef1b3baSKirill A. Shutemov /* No need in mapcount reference anymore */ 2859eef1b3baSKirill A. Shutemov for (i = 0; i < HPAGE_PMD_NR; i++) 2860eef1b3baSKirill A. Shutemov atomic_dec(&page[i]._mapcount); 2861eef1b3baSKirill A. Shutemov } 2862eef1b3baSKirill A. Shutemov } 2863eef1b3baSKirill A. Shutemov 2864eef1b3baSKirill A. Shutemov smp_wmb(); /* make pte visible before pmd */ 2865e9b61f19SKirill A. Shutemov /* 2866e9b61f19SKirill A. Shutemov * Up to this point the pmd is present and huge and userland has the 2867e9b61f19SKirill A. Shutemov * whole access to the hugepage during the split (which happens in 2868e9b61f19SKirill A. Shutemov * place). If we overwrite the pmd with the not-huge version pointing 2869e9b61f19SKirill A. Shutemov * to the pte here (which of course we could if all CPUs were bug 2870e9b61f19SKirill A. Shutemov * free), userland could trigger a small page size TLB miss on the 2871e9b61f19SKirill A. Shutemov * small sized TLB while the hugepage TLB entry is still established in 2872e9b61f19SKirill A. Shutemov * the huge TLB. Some CPU doesn't like that. 2873e9b61f19SKirill A. Shutemov * See http://support.amd.com/us/Processor_TechDocs/41322.pdf, Erratum 2874e9b61f19SKirill A. Shutemov * 383 on page 93. Intel should be safe but is also warns that it's 2875e9b61f19SKirill A. Shutemov * only safe if the permission and cache attributes of the two entries 2876e9b61f19SKirill A. Shutemov * loaded in the two TLB is identical (which should be the case here). 2877e9b61f19SKirill A. Shutemov * But it is generally safer to never allow small and huge TLB entries 2878e9b61f19SKirill A. Shutemov * for the same virtual address to be loaded simultaneously. So instead 2879e9b61f19SKirill A. Shutemov * of doing "pmd_populate(); flush_pmd_tlb_range();" we first mark the 2880e9b61f19SKirill A. Shutemov * current pmd notpresent (atomically because here the pmd_trans_huge 2881e9b61f19SKirill A. Shutemov * and pmd_trans_splitting must remain set at all times on the pmd 2882e9b61f19SKirill A. Shutemov * until the split is complete for this pmd), then we flush the SMP TLB 2883e9b61f19SKirill A. Shutemov * and finally we write the non-huge version of the pmd entry with 2884e9b61f19SKirill A. Shutemov * pmd_populate. 2885e9b61f19SKirill A. Shutemov */ 2886e9b61f19SKirill A. Shutemov pmdp_invalidate(vma, haddr, pmd); 2887eef1b3baSKirill A. Shutemov pmd_populate(mm, pmd, pgtable); 2888e9b61f19SKirill A. Shutemov 2889e9b61f19SKirill A. Shutemov if (freeze) { 2890e9b61f19SKirill A. Shutemov for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) { 2891e9b61f19SKirill A. Shutemov page_remove_rmap(page + i, false); 2892e9b61f19SKirill A. Shutemov put_page(page + i); 2893e9b61f19SKirill A. Shutemov } 2894e9b61f19SKirill A. Shutemov } 2895eef1b3baSKirill A. Shutemov } 2896eef1b3baSKirill A. Shutemov 2897eef1b3baSKirill A. Shutemov void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, 2898eef1b3baSKirill A. Shutemov unsigned long address) 2899eef1b3baSKirill A. Shutemov { 2900eef1b3baSKirill A. Shutemov spinlock_t *ptl; 2901eef1b3baSKirill A. Shutemov struct mm_struct *mm = vma->vm_mm; 2902e90309c9SKirill A. Shutemov struct page *page = NULL; 2903eef1b3baSKirill A. Shutemov unsigned long haddr = address & HPAGE_PMD_MASK; 2904eef1b3baSKirill A. Shutemov 2905eef1b3baSKirill A. Shutemov mmu_notifier_invalidate_range_start(mm, haddr, haddr + HPAGE_PMD_SIZE); 2906eef1b3baSKirill A. Shutemov ptl = pmd_lock(mm, pmd); 2907*5c7fb56eSDan Williams if (pmd_trans_huge(*pmd)) { 2908e90309c9SKirill A. Shutemov page = pmd_page(*pmd); 2909e90309c9SKirill A. Shutemov if (PageMlocked(page)) 2910e90309c9SKirill A. Shutemov get_page(page); 2911e90309c9SKirill A. Shutemov else 2912e90309c9SKirill A. Shutemov page = NULL; 2913*5c7fb56eSDan Williams } else if (!pmd_devmap(*pmd)) 2914*5c7fb56eSDan Williams goto out; 2915*5c7fb56eSDan Williams __split_huge_pmd_locked(vma, pmd, haddr, false); 2916e90309c9SKirill A. Shutemov out: 2917eef1b3baSKirill A. Shutemov spin_unlock(ptl); 2918eef1b3baSKirill A. Shutemov mmu_notifier_invalidate_range_end(mm, haddr, haddr + HPAGE_PMD_SIZE); 2919e90309c9SKirill A. Shutemov if (page) { 2920e90309c9SKirill A. Shutemov lock_page(page); 2921e90309c9SKirill A. Shutemov munlock_vma_page(page); 2922e90309c9SKirill A. Shutemov unlock_page(page); 2923e90309c9SKirill A. Shutemov put_page(page); 2924e90309c9SKirill A. Shutemov } 2925eef1b3baSKirill A. Shutemov } 2926eef1b3baSKirill A. Shutemov 292778ddc534SKirill A. Shutemov static void split_huge_pmd_address(struct vm_area_struct *vma, 292894fcc585SAndrea Arcangeli unsigned long address) 292994fcc585SAndrea Arcangeli { 2930f72e7dcdSHugh Dickins pgd_t *pgd; 2931f72e7dcdSHugh Dickins pud_t *pud; 293294fcc585SAndrea Arcangeli pmd_t *pmd; 293394fcc585SAndrea Arcangeli 293494fcc585SAndrea Arcangeli VM_BUG_ON(!(address & ~HPAGE_PMD_MASK)); 293594fcc585SAndrea Arcangeli 293678ddc534SKirill A. Shutemov pgd = pgd_offset(vma->vm_mm, address); 2937f72e7dcdSHugh Dickins if (!pgd_present(*pgd)) 2938f72e7dcdSHugh Dickins return; 2939f72e7dcdSHugh Dickins 2940f72e7dcdSHugh Dickins pud = pud_offset(pgd, address); 2941f72e7dcdSHugh Dickins if (!pud_present(*pud)) 2942f72e7dcdSHugh Dickins return; 2943f72e7dcdSHugh Dickins 2944f72e7dcdSHugh Dickins pmd = pmd_offset(pud, address); 2945*5c7fb56eSDan Williams if (!pmd_present(*pmd) || (!pmd_trans_huge(*pmd) && !pmd_devmap(*pmd))) 294694fcc585SAndrea Arcangeli return; 294794fcc585SAndrea Arcangeli /* 294894fcc585SAndrea Arcangeli * Caller holds the mmap_sem write mode, so a huge pmd cannot 294994fcc585SAndrea Arcangeli * materialize from under us. 295094fcc585SAndrea Arcangeli */ 2951ad0bed24SKirill A. Shutemov split_huge_pmd(vma, pmd, address); 295294fcc585SAndrea Arcangeli } 295394fcc585SAndrea Arcangeli 2954e1b9996bSKirill A. Shutemov void vma_adjust_trans_huge(struct vm_area_struct *vma, 295594fcc585SAndrea Arcangeli unsigned long start, 295694fcc585SAndrea Arcangeli unsigned long end, 295794fcc585SAndrea Arcangeli long adjust_next) 295894fcc585SAndrea Arcangeli { 295994fcc585SAndrea Arcangeli /* 296094fcc585SAndrea Arcangeli * If the new start address isn't hpage aligned and it could 296194fcc585SAndrea Arcangeli * previously contain an hugepage: check if we need to split 296294fcc585SAndrea Arcangeli * an huge pmd. 296394fcc585SAndrea Arcangeli */ 296494fcc585SAndrea Arcangeli if (start & ~HPAGE_PMD_MASK && 296594fcc585SAndrea Arcangeli (start & HPAGE_PMD_MASK) >= vma->vm_start && 296694fcc585SAndrea Arcangeli (start & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end) 296778ddc534SKirill A. Shutemov split_huge_pmd_address(vma, start); 296894fcc585SAndrea Arcangeli 296994fcc585SAndrea Arcangeli /* 297094fcc585SAndrea Arcangeli * If the new end address isn't hpage aligned and it could 297194fcc585SAndrea Arcangeli * previously contain an hugepage: check if we need to split 297294fcc585SAndrea Arcangeli * an huge pmd. 297394fcc585SAndrea Arcangeli */ 297494fcc585SAndrea Arcangeli if (end & ~HPAGE_PMD_MASK && 297594fcc585SAndrea Arcangeli (end & HPAGE_PMD_MASK) >= vma->vm_start && 297694fcc585SAndrea Arcangeli (end & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end) 297778ddc534SKirill A. Shutemov split_huge_pmd_address(vma, end); 297894fcc585SAndrea Arcangeli 297994fcc585SAndrea Arcangeli /* 298094fcc585SAndrea Arcangeli * If we're also updating the vma->vm_next->vm_start, if the new 298194fcc585SAndrea Arcangeli * vm_next->vm_start isn't page aligned and it could previously 298294fcc585SAndrea Arcangeli * contain an hugepage: check if we need to split an huge pmd. 298394fcc585SAndrea Arcangeli */ 298494fcc585SAndrea Arcangeli if (adjust_next > 0) { 298594fcc585SAndrea Arcangeli struct vm_area_struct *next = vma->vm_next; 298694fcc585SAndrea Arcangeli unsigned long nstart = next->vm_start; 298794fcc585SAndrea Arcangeli nstart += adjust_next << PAGE_SHIFT; 298894fcc585SAndrea Arcangeli if (nstart & ~HPAGE_PMD_MASK && 298994fcc585SAndrea Arcangeli (nstart & HPAGE_PMD_MASK) >= next->vm_start && 299094fcc585SAndrea Arcangeli (nstart & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= next->vm_end) 299178ddc534SKirill A. Shutemov split_huge_pmd_address(next, nstart); 299294fcc585SAndrea Arcangeli } 299394fcc585SAndrea Arcangeli } 2994e9b61f19SKirill A. Shutemov 2995e9b61f19SKirill A. Shutemov static void freeze_page_vma(struct vm_area_struct *vma, struct page *page, 2996e9b61f19SKirill A. Shutemov unsigned long address) 2997e9b61f19SKirill A. Shutemov { 2998bd56086fSKirill A. Shutemov unsigned long haddr = address & HPAGE_PMD_MASK; 2999e9b61f19SKirill A. Shutemov spinlock_t *ptl; 3000e9b61f19SKirill A. Shutemov pgd_t *pgd; 3001e9b61f19SKirill A. Shutemov pud_t *pud; 3002e9b61f19SKirill A. Shutemov pmd_t *pmd; 3003e9b61f19SKirill A. Shutemov pte_t *pte; 3004e9b61f19SKirill A. Shutemov int i, nr = HPAGE_PMD_NR; 3005e9b61f19SKirill A. Shutemov 3006e9b61f19SKirill A. Shutemov /* Skip pages which doesn't belong to the VMA */ 3007e9b61f19SKirill A. Shutemov if (address < vma->vm_start) { 3008e9b61f19SKirill A. Shutemov int off = (vma->vm_start - address) >> PAGE_SHIFT; 3009e9b61f19SKirill A. Shutemov page += off; 3010e9b61f19SKirill A. Shutemov nr -= off; 3011e9b61f19SKirill A. Shutemov address = vma->vm_start; 3012e9b61f19SKirill A. Shutemov } 3013e9b61f19SKirill A. Shutemov 3014e9b61f19SKirill A. Shutemov pgd = pgd_offset(vma->vm_mm, address); 3015e9b61f19SKirill A. Shutemov if (!pgd_present(*pgd)) 3016e9b61f19SKirill A. Shutemov return; 3017e9b61f19SKirill A. Shutemov pud = pud_offset(pgd, address); 3018e9b61f19SKirill A. Shutemov if (!pud_present(*pud)) 3019e9b61f19SKirill A. Shutemov return; 3020e9b61f19SKirill A. Shutemov pmd = pmd_offset(pud, address); 3021e9b61f19SKirill A. Shutemov ptl = pmd_lock(vma->vm_mm, pmd); 3022e9b61f19SKirill A. Shutemov if (!pmd_present(*pmd)) { 3023e9b61f19SKirill A. Shutemov spin_unlock(ptl); 3024e9b61f19SKirill A. Shutemov return; 3025e9b61f19SKirill A. Shutemov } 3026e9b61f19SKirill A. Shutemov if (pmd_trans_huge(*pmd)) { 3027e9b61f19SKirill A. Shutemov if (page == pmd_page(*pmd)) 3028bd56086fSKirill A. Shutemov __split_huge_pmd_locked(vma, pmd, haddr, true); 3029e9b61f19SKirill A. Shutemov spin_unlock(ptl); 3030e9b61f19SKirill A. Shutemov return; 3031e9b61f19SKirill A. Shutemov } 3032e9b61f19SKirill A. Shutemov spin_unlock(ptl); 3033e9b61f19SKirill A. Shutemov 3034e9b61f19SKirill A. Shutemov pte = pte_offset_map_lock(vma->vm_mm, pmd, address, &ptl); 3035bd56086fSKirill A. Shutemov for (i = 0; i < nr; i++, address += PAGE_SIZE, page++, pte++) { 3036e9b61f19SKirill A. Shutemov pte_t entry, swp_pte; 3037e9b61f19SKirill A. Shutemov swp_entry_t swp_entry; 3038e9b61f19SKirill A. Shutemov 3039bd56086fSKirill A. Shutemov /* 3040bd56086fSKirill A. Shutemov * We've just crossed page table boundary: need to map next one. 3041bd56086fSKirill A. Shutemov * It can happen if THP was mremaped to non PMD-aligned address. 3042bd56086fSKirill A. Shutemov */ 3043bd56086fSKirill A. Shutemov if (unlikely(address == haddr + HPAGE_PMD_SIZE)) { 3044bd56086fSKirill A. Shutemov pte_unmap_unlock(pte - 1, ptl); 3045bd56086fSKirill A. Shutemov pmd = mm_find_pmd(vma->vm_mm, address); 3046bd56086fSKirill A. Shutemov if (!pmd) 3047bd56086fSKirill A. Shutemov return; 3048bd56086fSKirill A. Shutemov pte = pte_offset_map_lock(vma->vm_mm, pmd, 3049bd56086fSKirill A. Shutemov address, &ptl); 3050bd56086fSKirill A. Shutemov } 3051bd56086fSKirill A. Shutemov 3052bd56086fSKirill A. Shutemov if (!pte_present(*pte)) 3053e9b61f19SKirill A. Shutemov continue; 3054bd56086fSKirill A. Shutemov if (page_to_pfn(page) != pte_pfn(*pte)) 3055e9b61f19SKirill A. Shutemov continue; 3056e9b61f19SKirill A. Shutemov flush_cache_page(vma, address, page_to_pfn(page)); 3057bd56086fSKirill A. Shutemov entry = ptep_clear_flush(vma, address, pte); 3058b8d3c4c3SMinchan Kim if (pte_dirty(entry)) 3059b8d3c4c3SMinchan Kim SetPageDirty(page); 3060e9b61f19SKirill A. Shutemov swp_entry = make_migration_entry(page, pte_write(entry)); 3061e9b61f19SKirill A. Shutemov swp_pte = swp_entry_to_pte(swp_entry); 3062e9b61f19SKirill A. Shutemov if (pte_soft_dirty(entry)) 3063e9b61f19SKirill A. Shutemov swp_pte = pte_swp_mksoft_dirty(swp_pte); 3064bd56086fSKirill A. Shutemov set_pte_at(vma->vm_mm, address, pte, swp_pte); 3065e9b61f19SKirill A. Shutemov page_remove_rmap(page, false); 3066e9b61f19SKirill A. Shutemov put_page(page); 3067e9b61f19SKirill A. Shutemov } 3068bd56086fSKirill A. Shutemov pte_unmap_unlock(pte - 1, ptl); 3069e9b61f19SKirill A. Shutemov } 3070e9b61f19SKirill A. Shutemov 3071e9b61f19SKirill A. Shutemov static void freeze_page(struct anon_vma *anon_vma, struct page *page) 3072e9b61f19SKirill A. Shutemov { 3073e9b61f19SKirill A. Shutemov struct anon_vma_chain *avc; 3074e9b61f19SKirill A. Shutemov pgoff_t pgoff = page_to_pgoff(page); 3075e9b61f19SKirill A. Shutemov 3076e9b61f19SKirill A. Shutemov VM_BUG_ON_PAGE(!PageHead(page), page); 3077e9b61f19SKirill A. Shutemov 3078e9b61f19SKirill A. Shutemov anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, pgoff, 3079e9b61f19SKirill A. Shutemov pgoff + HPAGE_PMD_NR - 1) { 3080bd56086fSKirill A. Shutemov unsigned long address = __vma_address(page, avc->vma); 3081e9b61f19SKirill A. Shutemov 3082e9b61f19SKirill A. Shutemov mmu_notifier_invalidate_range_start(avc->vma->vm_mm, 3083bd56086fSKirill A. Shutemov address, address + HPAGE_PMD_SIZE); 3084bd56086fSKirill A. Shutemov freeze_page_vma(avc->vma, page, address); 3085e9b61f19SKirill A. Shutemov mmu_notifier_invalidate_range_end(avc->vma->vm_mm, 3086bd56086fSKirill A. Shutemov address, address + HPAGE_PMD_SIZE); 3087e9b61f19SKirill A. Shutemov } 3088e9b61f19SKirill A. Shutemov } 3089e9b61f19SKirill A. Shutemov 3090e9b61f19SKirill A. Shutemov static void unfreeze_page_vma(struct vm_area_struct *vma, struct page *page, 3091e9b61f19SKirill A. Shutemov unsigned long address) 3092e9b61f19SKirill A. Shutemov { 3093e9b61f19SKirill A. Shutemov spinlock_t *ptl; 3094e9b61f19SKirill A. Shutemov pmd_t *pmd; 3095e9b61f19SKirill A. Shutemov pte_t *pte, entry; 3096e9b61f19SKirill A. Shutemov swp_entry_t swp_entry; 3097bd56086fSKirill A. Shutemov unsigned long haddr = address & HPAGE_PMD_MASK; 3098e9b61f19SKirill A. Shutemov int i, nr = HPAGE_PMD_NR; 3099e9b61f19SKirill A. Shutemov 3100e9b61f19SKirill A. Shutemov /* Skip pages which doesn't belong to the VMA */ 3101e9b61f19SKirill A. Shutemov if (address < vma->vm_start) { 3102e9b61f19SKirill A. Shutemov int off = (vma->vm_start - address) >> PAGE_SHIFT; 3103e9b61f19SKirill A. Shutemov page += off; 3104e9b61f19SKirill A. Shutemov nr -= off; 3105e9b61f19SKirill A. Shutemov address = vma->vm_start; 3106e9b61f19SKirill A. Shutemov } 3107e9b61f19SKirill A. Shutemov 3108e9b61f19SKirill A. Shutemov pmd = mm_find_pmd(vma->vm_mm, address); 3109e9b61f19SKirill A. Shutemov if (!pmd) 3110e9b61f19SKirill A. Shutemov return; 3111bd56086fSKirill A. Shutemov 3112e9b61f19SKirill A. Shutemov pte = pte_offset_map_lock(vma->vm_mm, pmd, address, &ptl); 3113bd56086fSKirill A. Shutemov for (i = 0; i < nr; i++, address += PAGE_SIZE, page++, pte++) { 3114bd56086fSKirill A. Shutemov /* 3115bd56086fSKirill A. Shutemov * We've just crossed page table boundary: need to map next one. 3116bd56086fSKirill A. Shutemov * It can happen if THP was mremaped to non-PMD aligned address. 3117bd56086fSKirill A. Shutemov */ 3118bd56086fSKirill A. Shutemov if (unlikely(address == haddr + HPAGE_PMD_SIZE)) { 3119bd56086fSKirill A. Shutemov pte_unmap_unlock(pte - 1, ptl); 3120bd56086fSKirill A. Shutemov pmd = mm_find_pmd(vma->vm_mm, address); 3121bd56086fSKirill A. Shutemov if (!pmd) 3122bd56086fSKirill A. Shutemov return; 3123bd56086fSKirill A. Shutemov pte = pte_offset_map_lock(vma->vm_mm, pmd, 3124bd56086fSKirill A. Shutemov address, &ptl); 3125bd56086fSKirill A. Shutemov } 3126bd56086fSKirill A. Shutemov 3127bd56086fSKirill A. Shutemov if (!is_swap_pte(*pte)) 3128e9b61f19SKirill A. Shutemov continue; 3129e9b61f19SKirill A. Shutemov 3130bd56086fSKirill A. Shutemov swp_entry = pte_to_swp_entry(*pte); 3131e9b61f19SKirill A. Shutemov if (!is_migration_entry(swp_entry)) 3132e9b61f19SKirill A. Shutemov continue; 3133e9b61f19SKirill A. Shutemov if (migration_entry_to_page(swp_entry) != page) 3134e9b61f19SKirill A. Shutemov continue; 3135e9b61f19SKirill A. Shutemov 3136e9b61f19SKirill A. Shutemov get_page(page); 3137e9b61f19SKirill A. Shutemov page_add_anon_rmap(page, vma, address, false); 3138e9b61f19SKirill A. Shutemov 3139e9b61f19SKirill A. Shutemov entry = pte_mkold(mk_pte(page, vma->vm_page_prot)); 3140b8d3c4c3SMinchan Kim if (PageDirty(page)) 3141e9b61f19SKirill A. Shutemov entry = pte_mkdirty(entry); 3142e9b61f19SKirill A. Shutemov if (is_write_migration_entry(swp_entry)) 3143e9b61f19SKirill A. Shutemov entry = maybe_mkwrite(entry, vma); 3144e9b61f19SKirill A. Shutemov 3145e9b61f19SKirill A. Shutemov flush_dcache_page(page); 3146bd56086fSKirill A. Shutemov set_pte_at(vma->vm_mm, address, pte, entry); 3147e9b61f19SKirill A. Shutemov 3148e9b61f19SKirill A. Shutemov /* No need to invalidate - it was non-present before */ 3149bd56086fSKirill A. Shutemov update_mmu_cache(vma, address, pte); 3150e9b61f19SKirill A. Shutemov } 3151bd56086fSKirill A. Shutemov pte_unmap_unlock(pte - 1, ptl); 3152e9b61f19SKirill A. Shutemov } 3153e9b61f19SKirill A. Shutemov 3154e9b61f19SKirill A. Shutemov static void unfreeze_page(struct anon_vma *anon_vma, struct page *page) 3155e9b61f19SKirill A. Shutemov { 3156e9b61f19SKirill A. Shutemov struct anon_vma_chain *avc; 3157e9b61f19SKirill A. Shutemov pgoff_t pgoff = page_to_pgoff(page); 3158e9b61f19SKirill A. Shutemov 3159e9b61f19SKirill A. Shutemov anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, 3160e9b61f19SKirill A. Shutemov pgoff, pgoff + HPAGE_PMD_NR - 1) { 3161e9b61f19SKirill A. Shutemov unsigned long address = __vma_address(page, avc->vma); 3162e9b61f19SKirill A. Shutemov 3163e9b61f19SKirill A. Shutemov mmu_notifier_invalidate_range_start(avc->vma->vm_mm, 3164e9b61f19SKirill A. Shutemov address, address + HPAGE_PMD_SIZE); 3165e9b61f19SKirill A. Shutemov unfreeze_page_vma(avc->vma, page, address); 3166e9b61f19SKirill A. Shutemov mmu_notifier_invalidate_range_end(avc->vma->vm_mm, 3167e9b61f19SKirill A. Shutemov address, address + HPAGE_PMD_SIZE); 3168e9b61f19SKirill A. Shutemov } 3169e9b61f19SKirill A. Shutemov } 3170e9b61f19SKirill A. Shutemov 3171e9b61f19SKirill A. Shutemov static int __split_huge_page_tail(struct page *head, int tail, 3172e9b61f19SKirill A. Shutemov struct lruvec *lruvec, struct list_head *list) 3173e9b61f19SKirill A. Shutemov { 3174e9b61f19SKirill A. Shutemov int mapcount; 3175e9b61f19SKirill A. Shutemov struct page *page_tail = head + tail; 3176e9b61f19SKirill A. Shutemov 3177e9b61f19SKirill A. Shutemov mapcount = atomic_read(&page_tail->_mapcount) + 1; 3178e9b61f19SKirill A. Shutemov VM_BUG_ON_PAGE(atomic_read(&page_tail->_count) != 0, page_tail); 3179e9b61f19SKirill A. Shutemov 3180e9b61f19SKirill A. Shutemov /* 3181e9b61f19SKirill A. Shutemov * tail_page->_count is zero and not changing from under us. But 3182e9b61f19SKirill A. Shutemov * get_page_unless_zero() may be running from under us on the 3183e9b61f19SKirill A. Shutemov * tail_page. If we used atomic_set() below instead of atomic_add(), we 3184e9b61f19SKirill A. Shutemov * would then run atomic_set() concurrently with 3185e9b61f19SKirill A. Shutemov * get_page_unless_zero(), and atomic_set() is implemented in C not 3186e9b61f19SKirill A. Shutemov * using locked ops. spin_unlock on x86 sometime uses locked ops 3187e9b61f19SKirill A. Shutemov * because of PPro errata 66, 92, so unless somebody can guarantee 3188e9b61f19SKirill A. Shutemov * atomic_set() here would be safe on all archs (and not only on x86), 3189e9b61f19SKirill A. Shutemov * it's safer to use atomic_add(). 3190e9b61f19SKirill A. Shutemov */ 3191e9b61f19SKirill A. Shutemov atomic_add(mapcount + 1, &page_tail->_count); 3192e9b61f19SKirill A. Shutemov 3193e9b61f19SKirill A. Shutemov 3194e9b61f19SKirill A. Shutemov page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 3195e9b61f19SKirill A. Shutemov page_tail->flags |= (head->flags & 3196e9b61f19SKirill A. Shutemov ((1L << PG_referenced) | 3197e9b61f19SKirill A. Shutemov (1L << PG_swapbacked) | 3198e9b61f19SKirill A. Shutemov (1L << PG_mlocked) | 3199e9b61f19SKirill A. Shutemov (1L << PG_uptodate) | 3200e9b61f19SKirill A. Shutemov (1L << PG_active) | 3201e9b61f19SKirill A. Shutemov (1L << PG_locked) | 3202b8d3c4c3SMinchan Kim (1L << PG_unevictable) | 3203b8d3c4c3SMinchan Kim (1L << PG_dirty))); 3204e9b61f19SKirill A. Shutemov 3205e9b61f19SKirill A. Shutemov /* 3206e9b61f19SKirill A. Shutemov * After clearing PageTail the gup refcount can be released. 3207e9b61f19SKirill A. Shutemov * Page flags also must be visible before we make the page non-compound. 3208e9b61f19SKirill A. Shutemov */ 3209e9b61f19SKirill A. Shutemov smp_wmb(); 3210e9b61f19SKirill A. Shutemov 3211e9b61f19SKirill A. Shutemov clear_compound_head(page_tail); 3212e9b61f19SKirill A. Shutemov 3213e9b61f19SKirill A. Shutemov if (page_is_young(head)) 3214e9b61f19SKirill A. Shutemov set_page_young(page_tail); 3215e9b61f19SKirill A. Shutemov if (page_is_idle(head)) 3216e9b61f19SKirill A. Shutemov set_page_idle(page_tail); 3217e9b61f19SKirill A. Shutemov 3218e9b61f19SKirill A. Shutemov /* ->mapping in first tail page is compound_mapcount */ 32199a982250SKirill A. Shutemov VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING, 3220e9b61f19SKirill A. Shutemov page_tail); 3221e9b61f19SKirill A. Shutemov page_tail->mapping = head->mapping; 3222e9b61f19SKirill A. Shutemov 3223e9b61f19SKirill A. Shutemov page_tail->index = head->index + tail; 3224e9b61f19SKirill A. Shutemov page_cpupid_xchg_last(page_tail, page_cpupid_last(head)); 3225e9b61f19SKirill A. Shutemov lru_add_page_tail(head, page_tail, lruvec, list); 3226e9b61f19SKirill A. Shutemov 3227e9b61f19SKirill A. Shutemov return mapcount; 3228e9b61f19SKirill A. Shutemov } 3229e9b61f19SKirill A. Shutemov 3230e9b61f19SKirill A. Shutemov static void __split_huge_page(struct page *page, struct list_head *list) 3231e9b61f19SKirill A. Shutemov { 3232e9b61f19SKirill A. Shutemov struct page *head = compound_head(page); 3233e9b61f19SKirill A. Shutemov struct zone *zone = page_zone(head); 3234e9b61f19SKirill A. Shutemov struct lruvec *lruvec; 3235e9b61f19SKirill A. Shutemov int i, tail_mapcount; 3236e9b61f19SKirill A. Shutemov 3237e9b61f19SKirill A. Shutemov /* prevent PageLRU to go away from under us, and freeze lru stats */ 3238e9b61f19SKirill A. Shutemov spin_lock_irq(&zone->lru_lock); 3239e9b61f19SKirill A. Shutemov lruvec = mem_cgroup_page_lruvec(head, zone); 3240e9b61f19SKirill A. Shutemov 3241e9b61f19SKirill A. Shutemov /* complete memcg works before add pages to LRU */ 3242e9b61f19SKirill A. Shutemov mem_cgroup_split_huge_fixup(head); 3243e9b61f19SKirill A. Shutemov 3244e9b61f19SKirill A. Shutemov tail_mapcount = 0; 3245e9b61f19SKirill A. Shutemov for (i = HPAGE_PMD_NR - 1; i >= 1; i--) 3246e9b61f19SKirill A. Shutemov tail_mapcount += __split_huge_page_tail(head, i, lruvec, list); 3247e9b61f19SKirill A. Shutemov atomic_sub(tail_mapcount, &head->_count); 3248e9b61f19SKirill A. Shutemov 3249e9b61f19SKirill A. Shutemov ClearPageCompound(head); 3250e9b61f19SKirill A. Shutemov spin_unlock_irq(&zone->lru_lock); 3251e9b61f19SKirill A. Shutemov 3252e9b61f19SKirill A. Shutemov unfreeze_page(page_anon_vma(head), head); 3253e9b61f19SKirill A. Shutemov 3254e9b61f19SKirill A. Shutemov for (i = 0; i < HPAGE_PMD_NR; i++) { 3255e9b61f19SKirill A. Shutemov struct page *subpage = head + i; 3256e9b61f19SKirill A. Shutemov if (subpage == page) 3257e9b61f19SKirill A. Shutemov continue; 3258e9b61f19SKirill A. Shutemov unlock_page(subpage); 3259e9b61f19SKirill A. Shutemov 3260e9b61f19SKirill A. Shutemov /* 3261e9b61f19SKirill A. Shutemov * Subpages may be freed if there wasn't any mapping 3262e9b61f19SKirill A. Shutemov * like if add_to_swap() is running on a lru page that 3263e9b61f19SKirill A. Shutemov * had its mapping zapped. And freeing these pages 3264e9b61f19SKirill A. Shutemov * requires taking the lru_lock so we do the put_page 3265e9b61f19SKirill A. Shutemov * of the tail pages after the split is complete. 3266e9b61f19SKirill A. Shutemov */ 3267e9b61f19SKirill A. Shutemov put_page(subpage); 3268e9b61f19SKirill A. Shutemov } 3269e9b61f19SKirill A. Shutemov } 3270e9b61f19SKirill A. Shutemov 3271b20ce5e0SKirill A. Shutemov int total_mapcount(struct page *page) 3272b20ce5e0SKirill A. Shutemov { 3273b20ce5e0SKirill A. Shutemov int i, ret; 3274b20ce5e0SKirill A. Shutemov 3275b20ce5e0SKirill A. Shutemov VM_BUG_ON_PAGE(PageTail(page), page); 3276b20ce5e0SKirill A. Shutemov 3277b20ce5e0SKirill A. Shutemov if (likely(!PageCompound(page))) 3278b20ce5e0SKirill A. Shutemov return atomic_read(&page->_mapcount) + 1; 3279b20ce5e0SKirill A. Shutemov 3280b20ce5e0SKirill A. Shutemov ret = compound_mapcount(page); 3281b20ce5e0SKirill A. Shutemov if (PageHuge(page)) 3282b20ce5e0SKirill A. Shutemov return ret; 3283b20ce5e0SKirill A. Shutemov for (i = 0; i < HPAGE_PMD_NR; i++) 3284b20ce5e0SKirill A. Shutemov ret += atomic_read(&page[i]._mapcount) + 1; 3285b20ce5e0SKirill A. Shutemov if (PageDoubleMap(page)) 3286b20ce5e0SKirill A. Shutemov ret -= HPAGE_PMD_NR; 3287b20ce5e0SKirill A. Shutemov return ret; 3288b20ce5e0SKirill A. Shutemov } 3289b20ce5e0SKirill A. Shutemov 3290e9b61f19SKirill A. Shutemov /* 3291e9b61f19SKirill A. Shutemov * This function splits huge page into normal pages. @page can point to any 3292e9b61f19SKirill A. Shutemov * subpage of huge page to split. Split doesn't change the position of @page. 3293e9b61f19SKirill A. Shutemov * 3294e9b61f19SKirill A. Shutemov * Only caller must hold pin on the @page, otherwise split fails with -EBUSY. 3295e9b61f19SKirill A. Shutemov * The huge page must be locked. 3296e9b61f19SKirill A. Shutemov * 3297e9b61f19SKirill A. Shutemov * If @list is null, tail pages will be added to LRU list, otherwise, to @list. 3298e9b61f19SKirill A. Shutemov * 3299e9b61f19SKirill A. Shutemov * Both head page and tail pages will inherit mapping, flags, and so on from 3300e9b61f19SKirill A. Shutemov * the hugepage. 3301e9b61f19SKirill A. Shutemov * 3302e9b61f19SKirill A. Shutemov * GUP pin and PG_locked transferred to @page. Rest subpages can be freed if 3303e9b61f19SKirill A. Shutemov * they are not mapped. 3304e9b61f19SKirill A. Shutemov * 3305e9b61f19SKirill A. Shutemov * Returns 0 if the hugepage is split successfully. 3306e9b61f19SKirill A. Shutemov * Returns -EBUSY if the page is pinned or if anon_vma disappeared from under 3307e9b61f19SKirill A. Shutemov * us. 3308e9b61f19SKirill A. Shutemov */ 3309e9b61f19SKirill A. Shutemov int split_huge_page_to_list(struct page *page, struct list_head *list) 3310e9b61f19SKirill A. Shutemov { 3311e9b61f19SKirill A. Shutemov struct page *head = compound_head(page); 3312e9b61f19SKirill A. Shutemov struct anon_vma *anon_vma; 3313e9b61f19SKirill A. Shutemov int count, mapcount, ret; 3314d9654322SKirill A. Shutemov bool mlocked; 3315e9b61f19SKirill A. Shutemov 3316e9b61f19SKirill A. Shutemov VM_BUG_ON_PAGE(is_huge_zero_page(page), page); 3317e9b61f19SKirill A. Shutemov VM_BUG_ON_PAGE(!PageAnon(page), page); 3318e9b61f19SKirill A. Shutemov VM_BUG_ON_PAGE(!PageLocked(page), page); 3319e9b61f19SKirill A. Shutemov VM_BUG_ON_PAGE(!PageSwapBacked(page), page); 3320e9b61f19SKirill A. Shutemov VM_BUG_ON_PAGE(!PageCompound(page), page); 3321e9b61f19SKirill A. Shutemov 3322e9b61f19SKirill A. Shutemov /* 3323e9b61f19SKirill A. Shutemov * The caller does not necessarily hold an mmap_sem that would prevent 3324e9b61f19SKirill A. Shutemov * the anon_vma disappearing so we first we take a reference to it 3325e9b61f19SKirill A. Shutemov * and then lock the anon_vma for write. This is similar to 3326e9b61f19SKirill A. Shutemov * page_lock_anon_vma_read except the write lock is taken to serialise 3327e9b61f19SKirill A. Shutemov * against parallel split or collapse operations. 3328e9b61f19SKirill A. Shutemov */ 3329e9b61f19SKirill A. Shutemov anon_vma = page_get_anon_vma(head); 3330e9b61f19SKirill A. Shutemov if (!anon_vma) { 3331e9b61f19SKirill A. Shutemov ret = -EBUSY; 3332e9b61f19SKirill A. Shutemov goto out; 3333e9b61f19SKirill A. Shutemov } 3334e9b61f19SKirill A. Shutemov anon_vma_lock_write(anon_vma); 3335e9b61f19SKirill A. Shutemov 3336e9b61f19SKirill A. Shutemov /* 3337e9b61f19SKirill A. Shutemov * Racy check if we can split the page, before freeze_page() will 3338e9b61f19SKirill A. Shutemov * split PMDs 3339e9b61f19SKirill A. Shutemov */ 3340e9b61f19SKirill A. Shutemov if (total_mapcount(head) != page_count(head) - 1) { 3341e9b61f19SKirill A. Shutemov ret = -EBUSY; 3342e9b61f19SKirill A. Shutemov goto out_unlock; 3343e9b61f19SKirill A. Shutemov } 3344e9b61f19SKirill A. Shutemov 3345d9654322SKirill A. Shutemov mlocked = PageMlocked(page); 3346e9b61f19SKirill A. Shutemov freeze_page(anon_vma, head); 3347e9b61f19SKirill A. Shutemov VM_BUG_ON_PAGE(compound_mapcount(head), head); 3348e9b61f19SKirill A. Shutemov 3349d9654322SKirill A. Shutemov /* Make sure the page is not on per-CPU pagevec as it takes pin */ 3350d9654322SKirill A. Shutemov if (mlocked) 3351d9654322SKirill A. Shutemov lru_add_drain(); 3352d9654322SKirill A. Shutemov 33539a982250SKirill A. Shutemov /* Prevent deferred_split_scan() touching ->_count */ 33549a982250SKirill A. Shutemov spin_lock(&split_queue_lock); 3355e9b61f19SKirill A. Shutemov count = page_count(head); 3356e9b61f19SKirill A. Shutemov mapcount = total_mapcount(head); 3357bd56086fSKirill A. Shutemov if (!mapcount && count == 1) { 33589a982250SKirill A. Shutemov if (!list_empty(page_deferred_list(head))) { 33599a982250SKirill A. Shutemov split_queue_len--; 33609a982250SKirill A. Shutemov list_del(page_deferred_list(head)); 33619a982250SKirill A. Shutemov } 33629a982250SKirill A. Shutemov spin_unlock(&split_queue_lock); 3363e9b61f19SKirill A. Shutemov __split_huge_page(page, list); 3364e9b61f19SKirill A. Shutemov ret = 0; 3365bd56086fSKirill A. Shutemov } else if (IS_ENABLED(CONFIG_DEBUG_VM) && mapcount) { 33669a982250SKirill A. Shutemov spin_unlock(&split_queue_lock); 3367e9b61f19SKirill A. Shutemov pr_alert("total_mapcount: %u, page_count(): %u\n", 3368e9b61f19SKirill A. Shutemov mapcount, count); 3369e9b61f19SKirill A. Shutemov if (PageTail(page)) 3370e9b61f19SKirill A. Shutemov dump_page(head, NULL); 3371bd56086fSKirill A. Shutemov dump_page(page, "total_mapcount(head) > 0"); 3372e9b61f19SKirill A. Shutemov BUG(); 3373e9b61f19SKirill A. Shutemov } else { 33749a982250SKirill A. Shutemov spin_unlock(&split_queue_lock); 3375e9b61f19SKirill A. Shutemov unfreeze_page(anon_vma, head); 3376e9b61f19SKirill A. Shutemov ret = -EBUSY; 3377e9b61f19SKirill A. Shutemov } 3378e9b61f19SKirill A. Shutemov 3379e9b61f19SKirill A. Shutemov out_unlock: 3380e9b61f19SKirill A. Shutemov anon_vma_unlock_write(anon_vma); 3381e9b61f19SKirill A. Shutemov put_anon_vma(anon_vma); 3382e9b61f19SKirill A. Shutemov out: 3383e9b61f19SKirill A. Shutemov count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED); 3384e9b61f19SKirill A. Shutemov return ret; 3385e9b61f19SKirill A. Shutemov } 33869a982250SKirill A. Shutemov 33879a982250SKirill A. Shutemov void free_transhuge_page(struct page *page) 33889a982250SKirill A. Shutemov { 33899a982250SKirill A. Shutemov unsigned long flags; 33909a982250SKirill A. Shutemov 33919a982250SKirill A. Shutemov spin_lock_irqsave(&split_queue_lock, flags); 33929a982250SKirill A. Shutemov if (!list_empty(page_deferred_list(page))) { 33939a982250SKirill A. Shutemov split_queue_len--; 33949a982250SKirill A. Shutemov list_del(page_deferred_list(page)); 33959a982250SKirill A. Shutemov } 33969a982250SKirill A. Shutemov spin_unlock_irqrestore(&split_queue_lock, flags); 33979a982250SKirill A. Shutemov free_compound_page(page); 33989a982250SKirill A. Shutemov } 33999a982250SKirill A. Shutemov 34009a982250SKirill A. Shutemov void deferred_split_huge_page(struct page *page) 34019a982250SKirill A. Shutemov { 34029a982250SKirill A. Shutemov unsigned long flags; 34039a982250SKirill A. Shutemov 34049a982250SKirill A. Shutemov VM_BUG_ON_PAGE(!PageTransHuge(page), page); 34059a982250SKirill A. Shutemov 34069a982250SKirill A. Shutemov spin_lock_irqsave(&split_queue_lock, flags); 34079a982250SKirill A. Shutemov if (list_empty(page_deferred_list(page))) { 34089a982250SKirill A. Shutemov list_add_tail(page_deferred_list(page), &split_queue); 34099a982250SKirill A. Shutemov split_queue_len++; 34109a982250SKirill A. Shutemov } 34119a982250SKirill A. Shutemov spin_unlock_irqrestore(&split_queue_lock, flags); 34129a982250SKirill A. Shutemov } 34139a982250SKirill A. Shutemov 34149a982250SKirill A. Shutemov static unsigned long deferred_split_count(struct shrinker *shrink, 34159a982250SKirill A. Shutemov struct shrink_control *sc) 34169a982250SKirill A. Shutemov { 34179a982250SKirill A. Shutemov /* 34189a982250SKirill A. Shutemov * Split a page from split_queue will free up at least one page, 34199a982250SKirill A. Shutemov * at most HPAGE_PMD_NR - 1. We don't track exact number. 34209a982250SKirill A. Shutemov * Let's use HPAGE_PMD_NR / 2 as ballpark. 34219a982250SKirill A. Shutemov */ 34229a982250SKirill A. Shutemov return ACCESS_ONCE(split_queue_len) * HPAGE_PMD_NR / 2; 34239a982250SKirill A. Shutemov } 34249a982250SKirill A. Shutemov 34259a982250SKirill A. Shutemov static unsigned long deferred_split_scan(struct shrinker *shrink, 34269a982250SKirill A. Shutemov struct shrink_control *sc) 34279a982250SKirill A. Shutemov { 34289a982250SKirill A. Shutemov unsigned long flags; 34299a982250SKirill A. Shutemov LIST_HEAD(list), *pos, *next; 34309a982250SKirill A. Shutemov struct page *page; 34319a982250SKirill A. Shutemov int split = 0; 34329a982250SKirill A. Shutemov 34339a982250SKirill A. Shutemov spin_lock_irqsave(&split_queue_lock, flags); 34349a982250SKirill A. Shutemov list_splice_init(&split_queue, &list); 34359a982250SKirill A. Shutemov 34369a982250SKirill A. Shutemov /* Take pin on all head pages to avoid freeing them under us */ 34379a982250SKirill A. Shutemov list_for_each_safe(pos, next, &list) { 34389a982250SKirill A. Shutemov page = list_entry((void *)pos, struct page, mapping); 34399a982250SKirill A. Shutemov page = compound_head(page); 34409a982250SKirill A. Shutemov /* race with put_compound_page() */ 34419a982250SKirill A. Shutemov if (!get_page_unless_zero(page)) { 34429a982250SKirill A. Shutemov list_del_init(page_deferred_list(page)); 34439a982250SKirill A. Shutemov split_queue_len--; 34449a982250SKirill A. Shutemov } 34459a982250SKirill A. Shutemov } 34469a982250SKirill A. Shutemov spin_unlock_irqrestore(&split_queue_lock, flags); 34479a982250SKirill A. Shutemov 34489a982250SKirill A. Shutemov list_for_each_safe(pos, next, &list) { 34499a982250SKirill A. Shutemov page = list_entry((void *)pos, struct page, mapping); 34509a982250SKirill A. Shutemov lock_page(page); 34519a982250SKirill A. Shutemov /* split_huge_page() removes page from list on success */ 34529a982250SKirill A. Shutemov if (!split_huge_page(page)) 34539a982250SKirill A. Shutemov split++; 34549a982250SKirill A. Shutemov unlock_page(page); 34559a982250SKirill A. Shutemov put_page(page); 34569a982250SKirill A. Shutemov } 34579a982250SKirill A. Shutemov 34589a982250SKirill A. Shutemov spin_lock_irqsave(&split_queue_lock, flags); 34599a982250SKirill A. Shutemov list_splice_tail(&list, &split_queue); 34609a982250SKirill A. Shutemov spin_unlock_irqrestore(&split_queue_lock, flags); 34619a982250SKirill A. Shutemov 34629a982250SKirill A. Shutemov return split * HPAGE_PMD_NR / 2; 34639a982250SKirill A. Shutemov } 34649a982250SKirill A. Shutemov 34659a982250SKirill A. Shutemov static struct shrinker deferred_split_shrinker = { 34669a982250SKirill A. Shutemov .count_objects = deferred_split_count, 34679a982250SKirill A. Shutemov .scan_objects = deferred_split_scan, 34689a982250SKirill A. Shutemov .seeks = DEFAULT_SEEKS, 34699a982250SKirill A. Shutemov }; 347049071d43SKirill A. Shutemov 347149071d43SKirill A. Shutemov #ifdef CONFIG_DEBUG_FS 347249071d43SKirill A. Shutemov static int split_huge_pages_set(void *data, u64 val) 347349071d43SKirill A. Shutemov { 347449071d43SKirill A. Shutemov struct zone *zone; 347549071d43SKirill A. Shutemov struct page *page; 347649071d43SKirill A. Shutemov unsigned long pfn, max_zone_pfn; 347749071d43SKirill A. Shutemov unsigned long total = 0, split = 0; 347849071d43SKirill A. Shutemov 347949071d43SKirill A. Shutemov if (val != 1) 348049071d43SKirill A. Shutemov return -EINVAL; 348149071d43SKirill A. Shutemov 348249071d43SKirill A. Shutemov for_each_populated_zone(zone) { 348349071d43SKirill A. Shutemov max_zone_pfn = zone_end_pfn(zone); 348449071d43SKirill A. Shutemov for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) { 348549071d43SKirill A. Shutemov if (!pfn_valid(pfn)) 348649071d43SKirill A. Shutemov continue; 348749071d43SKirill A. Shutemov 348849071d43SKirill A. Shutemov page = pfn_to_page(pfn); 348949071d43SKirill A. Shutemov if (!get_page_unless_zero(page)) 349049071d43SKirill A. Shutemov continue; 349149071d43SKirill A. Shutemov 349249071d43SKirill A. Shutemov if (zone != page_zone(page)) 349349071d43SKirill A. Shutemov goto next; 349449071d43SKirill A. Shutemov 349549071d43SKirill A. Shutemov if (!PageHead(page) || !PageAnon(page) || 349649071d43SKirill A. Shutemov PageHuge(page)) 349749071d43SKirill A. Shutemov goto next; 349849071d43SKirill A. Shutemov 349949071d43SKirill A. Shutemov total++; 350049071d43SKirill A. Shutemov lock_page(page); 350149071d43SKirill A. Shutemov if (!split_huge_page(page)) 350249071d43SKirill A. Shutemov split++; 350349071d43SKirill A. Shutemov unlock_page(page); 350449071d43SKirill A. Shutemov next: 350549071d43SKirill A. Shutemov put_page(page); 350649071d43SKirill A. Shutemov } 350749071d43SKirill A. Shutemov } 350849071d43SKirill A. Shutemov 350949071d43SKirill A. Shutemov pr_info("%lu of %lu THP split", split, total); 351049071d43SKirill A. Shutemov 351149071d43SKirill A. Shutemov return 0; 351249071d43SKirill A. Shutemov } 351349071d43SKirill A. Shutemov DEFINE_SIMPLE_ATTRIBUTE(split_huge_pages_fops, NULL, split_huge_pages_set, 351449071d43SKirill A. Shutemov "%llu\n"); 351549071d43SKirill A. Shutemov 351649071d43SKirill A. Shutemov static int __init split_huge_pages_debugfs(void) 351749071d43SKirill A. Shutemov { 351849071d43SKirill A. Shutemov void *ret; 351949071d43SKirill A. Shutemov 352049071d43SKirill A. Shutemov ret = debugfs_create_file("split_huge_pages", 0644, NULL, NULL, 352149071d43SKirill A. Shutemov &split_huge_pages_fops); 352249071d43SKirill A. Shutemov if (!ret) 352349071d43SKirill A. Shutemov pr_warn("Failed to create split_huge_pages in debugfs"); 352449071d43SKirill A. Shutemov return 0; 352549071d43SKirill A. Shutemov } 352649071d43SKirill A. Shutemov late_initcall(split_huge_pages_debugfs); 352749071d43SKirill A. Shutemov #endif 3528