171e3aac0SAndrea Arcangeli /* 271e3aac0SAndrea Arcangeli * Copyright (C) 2009 Red Hat, Inc. 371e3aac0SAndrea Arcangeli * 471e3aac0SAndrea Arcangeli * This work is licensed under the terms of the GNU GPL, version 2. See 571e3aac0SAndrea Arcangeli * the COPYING file in the top-level directory. 671e3aac0SAndrea Arcangeli */ 771e3aac0SAndrea Arcangeli 8ae3a8c1cSAndrew Morton #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 9ae3a8c1cSAndrew Morton 1071e3aac0SAndrea Arcangeli #include <linux/mm.h> 1171e3aac0SAndrea Arcangeli #include <linux/sched.h> 1271e3aac0SAndrea Arcangeli #include <linux/highmem.h> 1371e3aac0SAndrea Arcangeli #include <linux/hugetlb.h> 1471e3aac0SAndrea Arcangeli #include <linux/mmu_notifier.h> 1571e3aac0SAndrea Arcangeli #include <linux/rmap.h> 1671e3aac0SAndrea Arcangeli #include <linux/swap.h> 1797ae1749SKirill A. Shutemov #include <linux/shrinker.h> 18ba76149fSAndrea Arcangeli #include <linux/mm_inline.h> 19e9b61f19SKirill A. Shutemov #include <linux/swapops.h> 204897c765SMatthew Wilcox #include <linux/dax.h> 21ba76149fSAndrea Arcangeli #include <linux/kthread.h> 22ba76149fSAndrea Arcangeli #include <linux/khugepaged.h> 23878aee7dSAndrea Arcangeli #include <linux/freezer.h> 24f25748e3SDan Williams #include <linux/pfn_t.h> 25a664b2d8SAndrea Arcangeli #include <linux/mman.h> 263565fce3SDan Williams #include <linux/memremap.h> 27325adeb5SRalf Baechle #include <linux/pagemap.h> 2849071d43SKirill A. Shutemov #include <linux/debugfs.h> 294daae3b4SMel Gorman #include <linux/migrate.h> 3043b5fbbdSSasha Levin #include <linux/hashtable.h> 316b251fc9SAndrea Arcangeli #include <linux/userfaultfd_k.h> 3233c3fc71SVladimir Davydov #include <linux/page_idle.h> 3397ae1749SKirill A. Shutemov 3471e3aac0SAndrea Arcangeli #include <asm/tlb.h> 3571e3aac0SAndrea Arcangeli #include <asm/pgalloc.h> 3671e3aac0SAndrea Arcangeli #include "internal.h" 3771e3aac0SAndrea Arcangeli 387d2eba05SEbru Akagunduz enum scan_result { 397d2eba05SEbru Akagunduz SCAN_FAIL, 407d2eba05SEbru Akagunduz SCAN_SUCCEED, 417d2eba05SEbru Akagunduz SCAN_PMD_NULL, 427d2eba05SEbru Akagunduz SCAN_EXCEED_NONE_PTE, 437d2eba05SEbru Akagunduz SCAN_PTE_NON_PRESENT, 447d2eba05SEbru Akagunduz SCAN_PAGE_RO, 457d2eba05SEbru Akagunduz SCAN_NO_REFERENCED_PAGE, 467d2eba05SEbru Akagunduz SCAN_PAGE_NULL, 477d2eba05SEbru Akagunduz SCAN_SCAN_ABORT, 487d2eba05SEbru Akagunduz SCAN_PAGE_COUNT, 497d2eba05SEbru Akagunduz SCAN_PAGE_LRU, 507d2eba05SEbru Akagunduz SCAN_PAGE_LOCK, 517d2eba05SEbru Akagunduz SCAN_PAGE_ANON, 52b1caa957SKirill A. Shutemov SCAN_PAGE_COMPOUND, 537d2eba05SEbru Akagunduz SCAN_ANY_PROCESS, 547d2eba05SEbru Akagunduz SCAN_VMA_NULL, 557d2eba05SEbru Akagunduz SCAN_VMA_CHECK, 567d2eba05SEbru Akagunduz SCAN_ADDRESS_RANGE, 577d2eba05SEbru Akagunduz SCAN_SWAP_CACHE_PAGE, 587d2eba05SEbru Akagunduz SCAN_DEL_PAGE_LRU, 597d2eba05SEbru Akagunduz SCAN_ALLOC_HUGE_PAGE_FAIL, 607d2eba05SEbru Akagunduz SCAN_CGROUP_CHARGE_FAIL 617d2eba05SEbru Akagunduz }; 627d2eba05SEbru Akagunduz 637d2eba05SEbru Akagunduz #define CREATE_TRACE_POINTS 647d2eba05SEbru Akagunduz #include <trace/events/huge_memory.h> 657d2eba05SEbru Akagunduz 66ba76149fSAndrea Arcangeli /* 678bfa3f9aSJianguo Wu * By default transparent hugepage support is disabled in order that avoid 688bfa3f9aSJianguo Wu * to risk increase the memory footprint of applications without a guaranteed 698bfa3f9aSJianguo Wu * benefit. When transparent hugepage support is enabled, is for all mappings, 708bfa3f9aSJianguo Wu * and khugepaged scans all mappings. 718bfa3f9aSJianguo Wu * Defrag is invoked by khugepaged hugepage allocations and by page faults 728bfa3f9aSJianguo Wu * for all hugepage allocations. 73ba76149fSAndrea Arcangeli */ 7471e3aac0SAndrea Arcangeli unsigned long transparent_hugepage_flags __read_mostly = 7513ece886SAndrea Arcangeli #ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS 76ba76149fSAndrea Arcangeli (1<<TRANSPARENT_HUGEPAGE_FLAG)| 7713ece886SAndrea Arcangeli #endif 7813ece886SAndrea Arcangeli #ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE 7913ece886SAndrea Arcangeli (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)| 8013ece886SAndrea Arcangeli #endif 81444eb2a4SMel Gorman (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG)| 8279da5407SKirill A. Shutemov (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)| 8379da5407SKirill A. Shutemov (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 84ba76149fSAndrea Arcangeli 85ba76149fSAndrea Arcangeli /* default scan 8*512 pte (or vmas) every 30 second */ 86ff20c2e0SKirill A. Shutemov static unsigned int khugepaged_pages_to_scan __read_mostly; 87ba76149fSAndrea Arcangeli static unsigned int khugepaged_pages_collapsed; 88ba76149fSAndrea Arcangeli static unsigned int khugepaged_full_scans; 89ba76149fSAndrea Arcangeli static unsigned int khugepaged_scan_sleep_millisecs __read_mostly = 10000; 90ba76149fSAndrea Arcangeli /* during fragmentation poll the hugepage allocator once every minute */ 91ba76149fSAndrea Arcangeli static unsigned int khugepaged_alloc_sleep_millisecs __read_mostly = 60000; 92f0508977SDavid Rientjes static unsigned long khugepaged_sleep_expire; 93ba76149fSAndrea Arcangeli static struct task_struct *khugepaged_thread __read_mostly; 94ba76149fSAndrea Arcangeli static DEFINE_MUTEX(khugepaged_mutex); 95ba76149fSAndrea Arcangeli static DEFINE_SPINLOCK(khugepaged_mm_lock); 96ba76149fSAndrea Arcangeli static DECLARE_WAIT_QUEUE_HEAD(khugepaged_wait); 97ba76149fSAndrea Arcangeli /* 98ba76149fSAndrea Arcangeli * default collapse hugepages if there is at least one pte mapped like 99ba76149fSAndrea Arcangeli * it would have happened if the vma was large enough during page 100ba76149fSAndrea Arcangeli * fault. 101ba76149fSAndrea Arcangeli */ 102ff20c2e0SKirill A. Shutemov static unsigned int khugepaged_max_ptes_none __read_mostly; 103ba76149fSAndrea Arcangeli 104ba76149fSAndrea Arcangeli static int khugepaged(void *none); 105ba76149fSAndrea Arcangeli static int khugepaged_slab_init(void); 10665ebb64fSKirill A. Shutemov static void khugepaged_slab_exit(void); 107ba76149fSAndrea Arcangeli 10843b5fbbdSSasha Levin #define MM_SLOTS_HASH_BITS 10 10943b5fbbdSSasha Levin static __read_mostly DEFINE_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS); 11043b5fbbdSSasha Levin 111ba76149fSAndrea Arcangeli static struct kmem_cache *mm_slot_cache __read_mostly; 112ba76149fSAndrea Arcangeli 113ba76149fSAndrea Arcangeli /** 114ba76149fSAndrea Arcangeli * struct mm_slot - hash lookup from mm to mm_slot 115ba76149fSAndrea Arcangeli * @hash: hash collision list 116ba76149fSAndrea Arcangeli * @mm_node: khugepaged scan list headed in khugepaged_scan.mm_head 117ba76149fSAndrea Arcangeli * @mm: the mm that this information is valid for 118ba76149fSAndrea Arcangeli */ 119ba76149fSAndrea Arcangeli struct mm_slot { 120ba76149fSAndrea Arcangeli struct hlist_node hash; 121ba76149fSAndrea Arcangeli struct list_head mm_node; 122ba76149fSAndrea Arcangeli struct mm_struct *mm; 123ba76149fSAndrea Arcangeli }; 124ba76149fSAndrea Arcangeli 125ba76149fSAndrea Arcangeli /** 126ba76149fSAndrea Arcangeli * struct khugepaged_scan - cursor for scanning 127ba76149fSAndrea Arcangeli * @mm_head: the head of the mm list to scan 128ba76149fSAndrea Arcangeli * @mm_slot: the current mm_slot we are scanning 129ba76149fSAndrea Arcangeli * @address: the next address inside that to be scanned 130ba76149fSAndrea Arcangeli * 131ba76149fSAndrea Arcangeli * There is only the one khugepaged_scan instance of this cursor structure. 132ba76149fSAndrea Arcangeli */ 133ba76149fSAndrea Arcangeli struct khugepaged_scan { 134ba76149fSAndrea Arcangeli struct list_head mm_head; 135ba76149fSAndrea Arcangeli struct mm_slot *mm_slot; 136ba76149fSAndrea Arcangeli unsigned long address; 1372f1da642SH Hartley Sweeten }; 1382f1da642SH Hartley Sweeten static struct khugepaged_scan khugepaged_scan = { 139ba76149fSAndrea Arcangeli .mm_head = LIST_HEAD_INIT(khugepaged_scan.mm_head), 140ba76149fSAndrea Arcangeli }; 141ba76149fSAndrea Arcangeli 1429a982250SKirill A. Shutemov static struct shrinker deferred_split_shrinker; 143f000565aSAndrea Arcangeli 1442c0b80d4SNicholas Krause static void set_recommended_min_free_kbytes(void) 145f000565aSAndrea Arcangeli { 146f000565aSAndrea Arcangeli struct zone *zone; 147f000565aSAndrea Arcangeli int nr_zones = 0; 148f000565aSAndrea Arcangeli unsigned long recommended_min; 149f000565aSAndrea Arcangeli 150f000565aSAndrea Arcangeli for_each_populated_zone(zone) 151f000565aSAndrea Arcangeli nr_zones++; 152f000565aSAndrea Arcangeli 153974a786eSMel Gorman /* Ensure 2 pageblocks are free to assist fragmentation avoidance */ 154f000565aSAndrea Arcangeli recommended_min = pageblock_nr_pages * nr_zones * 2; 155f000565aSAndrea Arcangeli 156f000565aSAndrea Arcangeli /* 157f000565aSAndrea Arcangeli * Make sure that on average at least two pageblocks are almost free 158f000565aSAndrea Arcangeli * of another type, one for a migratetype to fall back to and a 159f000565aSAndrea Arcangeli * second to avoid subsequent fallbacks of other types There are 3 160f000565aSAndrea Arcangeli * MIGRATE_TYPES we care about. 161f000565aSAndrea Arcangeli */ 162f000565aSAndrea Arcangeli recommended_min += pageblock_nr_pages * nr_zones * 163f000565aSAndrea Arcangeli MIGRATE_PCPTYPES * MIGRATE_PCPTYPES; 164f000565aSAndrea Arcangeli 165f000565aSAndrea Arcangeli /* don't ever allow to reserve more than 5% of the lowmem */ 166f000565aSAndrea Arcangeli recommended_min = min(recommended_min, 167f000565aSAndrea Arcangeli (unsigned long) nr_free_buffer_pages() / 20); 168f000565aSAndrea Arcangeli recommended_min <<= (PAGE_SHIFT-10); 169f000565aSAndrea Arcangeli 17042aa83cbSHan Pingtian if (recommended_min > min_free_kbytes) { 17142aa83cbSHan Pingtian if (user_min_free_kbytes >= 0) 172756a025fSJoe Perches pr_info("raising min_free_kbytes from %d to %lu to help transparent hugepage allocations\n", 17342aa83cbSHan Pingtian min_free_kbytes, recommended_min); 17442aa83cbSHan Pingtian 175f000565aSAndrea Arcangeli min_free_kbytes = recommended_min; 17642aa83cbSHan Pingtian } 177f000565aSAndrea Arcangeli setup_per_zone_wmarks(); 178f000565aSAndrea Arcangeli } 179f000565aSAndrea Arcangeli 18079553da2SKirill A. Shutemov static int start_stop_khugepaged(void) 181ba76149fSAndrea Arcangeli { 182ba76149fSAndrea Arcangeli int err = 0; 183ba76149fSAndrea Arcangeli if (khugepaged_enabled()) { 184ba76149fSAndrea Arcangeli if (!khugepaged_thread) 185ba76149fSAndrea Arcangeli khugepaged_thread = kthread_run(khugepaged, NULL, 186ba76149fSAndrea Arcangeli "khugepaged"); 18718e8e5c7SViresh Kumar if (IS_ERR(khugepaged_thread)) { 188ae3a8c1cSAndrew Morton pr_err("khugepaged: kthread_run(khugepaged) failed\n"); 189ba76149fSAndrea Arcangeli err = PTR_ERR(khugepaged_thread); 190ba76149fSAndrea Arcangeli khugepaged_thread = NULL; 19179553da2SKirill A. Shutemov goto fail; 192ba76149fSAndrea Arcangeli } 193911891afSXiao Guangrong 194911891afSXiao Guangrong if (!list_empty(&khugepaged_scan.mm_head)) 195ba76149fSAndrea Arcangeli wake_up_interruptible(&khugepaged_wait); 196f000565aSAndrea Arcangeli 197f000565aSAndrea Arcangeli set_recommended_min_free_kbytes(); 198911891afSXiao Guangrong } else if (khugepaged_thread) { 199911891afSXiao Guangrong kthread_stop(khugepaged_thread); 200911891afSXiao Guangrong khugepaged_thread = NULL; 201911891afSXiao Guangrong } 20279553da2SKirill A. Shutemov fail: 203ba76149fSAndrea Arcangeli return err; 204ba76149fSAndrea Arcangeli } 20571e3aac0SAndrea Arcangeli 20697ae1749SKirill A. Shutemov static atomic_t huge_zero_refcount; 20756873f43SWang, Yalin struct page *huge_zero_page __read_mostly; 2084a6c1297SKirill A. Shutemov 209fc437044SMatthew Wilcox struct page *get_huge_zero_page(void) 21097ae1749SKirill A. Shutemov { 21197ae1749SKirill A. Shutemov struct page *zero_page; 21297ae1749SKirill A. Shutemov retry: 21397ae1749SKirill A. Shutemov if (likely(atomic_inc_not_zero(&huge_zero_refcount))) 2144db0c3c2SJason Low return READ_ONCE(huge_zero_page); 21597ae1749SKirill A. Shutemov 21697ae1749SKirill A. Shutemov zero_page = alloc_pages((GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE, 21797ae1749SKirill A. Shutemov HPAGE_PMD_ORDER); 218d8a8e1f0SKirill A. Shutemov if (!zero_page) { 219d8a8e1f0SKirill A. Shutemov count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED); 2205918d10aSKirill A. Shutemov return NULL; 221d8a8e1f0SKirill A. Shutemov } 222d8a8e1f0SKirill A. Shutemov count_vm_event(THP_ZERO_PAGE_ALLOC); 22397ae1749SKirill A. Shutemov preempt_disable(); 2245918d10aSKirill A. Shutemov if (cmpxchg(&huge_zero_page, NULL, zero_page)) { 22597ae1749SKirill A. Shutemov preempt_enable(); 2265ddacbe9SYu Zhao __free_pages(zero_page, compound_order(zero_page)); 22797ae1749SKirill A. Shutemov goto retry; 22897ae1749SKirill A. Shutemov } 22997ae1749SKirill A. Shutemov 23097ae1749SKirill A. Shutemov /* We take additional reference here. It will be put back by shrinker */ 23197ae1749SKirill A. Shutemov atomic_set(&huge_zero_refcount, 2); 23297ae1749SKirill A. Shutemov preempt_enable(); 2334db0c3c2SJason Low return READ_ONCE(huge_zero_page); 23497ae1749SKirill A. Shutemov } 23597ae1749SKirill A. Shutemov 236aa88b68cSKirill A. Shutemov void put_huge_zero_page(void) 23797ae1749SKirill A. Shutemov { 23897ae1749SKirill A. Shutemov /* 23997ae1749SKirill A. Shutemov * Counter should never go to zero here. Only shrinker can put 24097ae1749SKirill A. Shutemov * last reference. 24197ae1749SKirill A. Shutemov */ 24297ae1749SKirill A. Shutemov BUG_ON(atomic_dec_and_test(&huge_zero_refcount)); 24397ae1749SKirill A. Shutemov } 24497ae1749SKirill A. Shutemov 24548896466SGlauber Costa static unsigned long shrink_huge_zero_page_count(struct shrinker *shrink, 24697ae1749SKirill A. Shutemov struct shrink_control *sc) 24797ae1749SKirill A. Shutemov { 24897ae1749SKirill A. Shutemov /* we can free zero page only if last reference remains */ 24997ae1749SKirill A. Shutemov return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0; 25048896466SGlauber Costa } 25197ae1749SKirill A. Shutemov 25248896466SGlauber Costa static unsigned long shrink_huge_zero_page_scan(struct shrinker *shrink, 25348896466SGlauber Costa struct shrink_control *sc) 25448896466SGlauber Costa { 25597ae1749SKirill A. Shutemov if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) { 2565918d10aSKirill A. Shutemov struct page *zero_page = xchg(&huge_zero_page, NULL); 2575918d10aSKirill A. Shutemov BUG_ON(zero_page == NULL); 2585ddacbe9SYu Zhao __free_pages(zero_page, compound_order(zero_page)); 25948896466SGlauber Costa return HPAGE_PMD_NR; 26097ae1749SKirill A. Shutemov } 26197ae1749SKirill A. Shutemov 26297ae1749SKirill A. Shutemov return 0; 26397ae1749SKirill A. Shutemov } 26497ae1749SKirill A. Shutemov 26597ae1749SKirill A. Shutemov static struct shrinker huge_zero_page_shrinker = { 26648896466SGlauber Costa .count_objects = shrink_huge_zero_page_count, 26748896466SGlauber Costa .scan_objects = shrink_huge_zero_page_scan, 26897ae1749SKirill A. Shutemov .seeks = DEFAULT_SEEKS, 26997ae1749SKirill A. Shutemov }; 27097ae1749SKirill A. Shutemov 27171e3aac0SAndrea Arcangeli #ifdef CONFIG_SYSFS 272ba76149fSAndrea Arcangeli 273444eb2a4SMel Gorman static ssize_t triple_flag_store(struct kobject *kobj, 27471e3aac0SAndrea Arcangeli struct kobj_attribute *attr, 27571e3aac0SAndrea Arcangeli const char *buf, size_t count, 27671e3aac0SAndrea Arcangeli enum transparent_hugepage_flag enabled, 277444eb2a4SMel Gorman enum transparent_hugepage_flag deferred, 27871e3aac0SAndrea Arcangeli enum transparent_hugepage_flag req_madv) 27971e3aac0SAndrea Arcangeli { 280444eb2a4SMel Gorman if (!memcmp("defer", buf, 281444eb2a4SMel Gorman min(sizeof("defer")-1, count))) { 282444eb2a4SMel Gorman if (enabled == deferred) 283444eb2a4SMel Gorman return -EINVAL; 284444eb2a4SMel Gorman clear_bit(enabled, &transparent_hugepage_flags); 28571e3aac0SAndrea Arcangeli clear_bit(req_madv, &transparent_hugepage_flags); 286444eb2a4SMel Gorman set_bit(deferred, &transparent_hugepage_flags); 287444eb2a4SMel Gorman } else if (!memcmp("always", buf, 288444eb2a4SMel Gorman min(sizeof("always")-1, count))) { 289444eb2a4SMel Gorman clear_bit(deferred, &transparent_hugepage_flags); 290444eb2a4SMel Gorman clear_bit(req_madv, &transparent_hugepage_flags); 291444eb2a4SMel Gorman set_bit(enabled, &transparent_hugepage_flags); 29271e3aac0SAndrea Arcangeli } else if (!memcmp("madvise", buf, 29371e3aac0SAndrea Arcangeli min(sizeof("madvise")-1, count))) { 29471e3aac0SAndrea Arcangeli clear_bit(enabled, &transparent_hugepage_flags); 295444eb2a4SMel Gorman clear_bit(deferred, &transparent_hugepage_flags); 29671e3aac0SAndrea Arcangeli set_bit(req_madv, &transparent_hugepage_flags); 29771e3aac0SAndrea Arcangeli } else if (!memcmp("never", buf, 29871e3aac0SAndrea Arcangeli min(sizeof("never")-1, count))) { 29971e3aac0SAndrea Arcangeli clear_bit(enabled, &transparent_hugepage_flags); 30071e3aac0SAndrea Arcangeli clear_bit(req_madv, &transparent_hugepage_flags); 301444eb2a4SMel Gorman clear_bit(deferred, &transparent_hugepage_flags); 30271e3aac0SAndrea Arcangeli } else 30371e3aac0SAndrea Arcangeli return -EINVAL; 30471e3aac0SAndrea Arcangeli 30571e3aac0SAndrea Arcangeli return count; 30671e3aac0SAndrea Arcangeli } 30771e3aac0SAndrea Arcangeli 30871e3aac0SAndrea Arcangeli static ssize_t enabled_show(struct kobject *kobj, 30971e3aac0SAndrea Arcangeli struct kobj_attribute *attr, char *buf) 31071e3aac0SAndrea Arcangeli { 311444eb2a4SMel Gorman if (test_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags)) 312444eb2a4SMel Gorman return sprintf(buf, "[always] madvise never\n"); 313444eb2a4SMel Gorman else if (test_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags)) 314444eb2a4SMel Gorman return sprintf(buf, "always [madvise] never\n"); 315444eb2a4SMel Gorman else 316444eb2a4SMel Gorman return sprintf(buf, "always madvise [never]\n"); 31771e3aac0SAndrea Arcangeli } 318444eb2a4SMel Gorman 31971e3aac0SAndrea Arcangeli static ssize_t enabled_store(struct kobject *kobj, 32071e3aac0SAndrea Arcangeli struct kobj_attribute *attr, 32171e3aac0SAndrea Arcangeli const char *buf, size_t count) 32271e3aac0SAndrea Arcangeli { 323ba76149fSAndrea Arcangeli ssize_t ret; 324ba76149fSAndrea Arcangeli 325444eb2a4SMel Gorman ret = triple_flag_store(kobj, attr, buf, count, 326444eb2a4SMel Gorman TRANSPARENT_HUGEPAGE_FLAG, 32771e3aac0SAndrea Arcangeli TRANSPARENT_HUGEPAGE_FLAG, 32871e3aac0SAndrea Arcangeli TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG); 329ba76149fSAndrea Arcangeli 330ba76149fSAndrea Arcangeli if (ret > 0) { 331911891afSXiao Guangrong int err; 332911891afSXiao Guangrong 333911891afSXiao Guangrong mutex_lock(&khugepaged_mutex); 33479553da2SKirill A. Shutemov err = start_stop_khugepaged(); 335911891afSXiao Guangrong mutex_unlock(&khugepaged_mutex); 336911891afSXiao Guangrong 337ba76149fSAndrea Arcangeli if (err) 338ba76149fSAndrea Arcangeli ret = err; 339ba76149fSAndrea Arcangeli } 340ba76149fSAndrea Arcangeli 341ba76149fSAndrea Arcangeli return ret; 34271e3aac0SAndrea Arcangeli } 34371e3aac0SAndrea Arcangeli static struct kobj_attribute enabled_attr = 34471e3aac0SAndrea Arcangeli __ATTR(enabled, 0644, enabled_show, enabled_store); 34571e3aac0SAndrea Arcangeli 34671e3aac0SAndrea Arcangeli static ssize_t single_flag_show(struct kobject *kobj, 34771e3aac0SAndrea Arcangeli struct kobj_attribute *attr, char *buf, 34871e3aac0SAndrea Arcangeli enum transparent_hugepage_flag flag) 34971e3aac0SAndrea Arcangeli { 350e27e6151SBen Hutchings return sprintf(buf, "%d\n", 351e27e6151SBen Hutchings !!test_bit(flag, &transparent_hugepage_flags)); 35271e3aac0SAndrea Arcangeli } 353e27e6151SBen Hutchings 35471e3aac0SAndrea Arcangeli static ssize_t single_flag_store(struct kobject *kobj, 35571e3aac0SAndrea Arcangeli struct kobj_attribute *attr, 35671e3aac0SAndrea Arcangeli const char *buf, size_t count, 35771e3aac0SAndrea Arcangeli enum transparent_hugepage_flag flag) 35871e3aac0SAndrea Arcangeli { 359e27e6151SBen Hutchings unsigned long value; 360e27e6151SBen Hutchings int ret; 361e27e6151SBen Hutchings 362e27e6151SBen Hutchings ret = kstrtoul(buf, 10, &value); 363e27e6151SBen Hutchings if (ret < 0) 364e27e6151SBen Hutchings return ret; 365e27e6151SBen Hutchings if (value > 1) 36671e3aac0SAndrea Arcangeli return -EINVAL; 36771e3aac0SAndrea Arcangeli 368e27e6151SBen Hutchings if (value) 369e27e6151SBen Hutchings set_bit(flag, &transparent_hugepage_flags); 370e27e6151SBen Hutchings else 371e27e6151SBen Hutchings clear_bit(flag, &transparent_hugepage_flags); 372e27e6151SBen Hutchings 37371e3aac0SAndrea Arcangeli return count; 37471e3aac0SAndrea Arcangeli } 37571e3aac0SAndrea Arcangeli 37671e3aac0SAndrea Arcangeli /* 37771e3aac0SAndrea Arcangeli * Currently defrag only disables __GFP_NOWAIT for allocation. A blind 37871e3aac0SAndrea Arcangeli * __GFP_REPEAT is too aggressive, it's never worth swapping tons of 37971e3aac0SAndrea Arcangeli * memory just to allocate one more hugepage. 38071e3aac0SAndrea Arcangeli */ 38171e3aac0SAndrea Arcangeli static ssize_t defrag_show(struct kobject *kobj, 38271e3aac0SAndrea Arcangeli struct kobj_attribute *attr, char *buf) 38371e3aac0SAndrea Arcangeli { 384444eb2a4SMel Gorman if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags)) 385444eb2a4SMel Gorman return sprintf(buf, "[always] defer madvise never\n"); 386444eb2a4SMel Gorman if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags)) 387444eb2a4SMel Gorman return sprintf(buf, "always [defer] madvise never\n"); 388444eb2a4SMel Gorman else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags)) 389444eb2a4SMel Gorman return sprintf(buf, "always defer [madvise] never\n"); 390444eb2a4SMel Gorman else 391444eb2a4SMel Gorman return sprintf(buf, "always defer madvise [never]\n"); 392444eb2a4SMel Gorman 39371e3aac0SAndrea Arcangeli } 39471e3aac0SAndrea Arcangeli static ssize_t defrag_store(struct kobject *kobj, 39571e3aac0SAndrea Arcangeli struct kobj_attribute *attr, 39671e3aac0SAndrea Arcangeli const char *buf, size_t count) 39771e3aac0SAndrea Arcangeli { 398444eb2a4SMel Gorman return triple_flag_store(kobj, attr, buf, count, 399444eb2a4SMel Gorman TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, 400444eb2a4SMel Gorman TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, 40171e3aac0SAndrea Arcangeli TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG); 40271e3aac0SAndrea Arcangeli } 40371e3aac0SAndrea Arcangeli static struct kobj_attribute defrag_attr = 40471e3aac0SAndrea Arcangeli __ATTR(defrag, 0644, defrag_show, defrag_store); 40571e3aac0SAndrea Arcangeli 40679da5407SKirill A. Shutemov static ssize_t use_zero_page_show(struct kobject *kobj, 40779da5407SKirill A. Shutemov struct kobj_attribute *attr, char *buf) 40879da5407SKirill A. Shutemov { 40979da5407SKirill A. Shutemov return single_flag_show(kobj, attr, buf, 41079da5407SKirill A. Shutemov TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 41179da5407SKirill A. Shutemov } 41279da5407SKirill A. Shutemov static ssize_t use_zero_page_store(struct kobject *kobj, 41379da5407SKirill A. Shutemov struct kobj_attribute *attr, const char *buf, size_t count) 41479da5407SKirill A. Shutemov { 41579da5407SKirill A. Shutemov return single_flag_store(kobj, attr, buf, count, 41679da5407SKirill A. Shutemov TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 41779da5407SKirill A. Shutemov } 41879da5407SKirill A. Shutemov static struct kobj_attribute use_zero_page_attr = 41979da5407SKirill A. Shutemov __ATTR(use_zero_page, 0644, use_zero_page_show, use_zero_page_store); 42071e3aac0SAndrea Arcangeli #ifdef CONFIG_DEBUG_VM 42171e3aac0SAndrea Arcangeli static ssize_t debug_cow_show(struct kobject *kobj, 42271e3aac0SAndrea Arcangeli struct kobj_attribute *attr, char *buf) 42371e3aac0SAndrea Arcangeli { 42471e3aac0SAndrea Arcangeli return single_flag_show(kobj, attr, buf, 42571e3aac0SAndrea Arcangeli TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG); 42671e3aac0SAndrea Arcangeli } 42771e3aac0SAndrea Arcangeli static ssize_t debug_cow_store(struct kobject *kobj, 42871e3aac0SAndrea Arcangeli struct kobj_attribute *attr, 42971e3aac0SAndrea Arcangeli const char *buf, size_t count) 43071e3aac0SAndrea Arcangeli { 43171e3aac0SAndrea Arcangeli return single_flag_store(kobj, attr, buf, count, 43271e3aac0SAndrea Arcangeli TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG); 43371e3aac0SAndrea Arcangeli } 43471e3aac0SAndrea Arcangeli static struct kobj_attribute debug_cow_attr = 43571e3aac0SAndrea Arcangeli __ATTR(debug_cow, 0644, debug_cow_show, debug_cow_store); 43671e3aac0SAndrea Arcangeli #endif /* CONFIG_DEBUG_VM */ 43771e3aac0SAndrea Arcangeli 43871e3aac0SAndrea Arcangeli static struct attribute *hugepage_attr[] = { 43971e3aac0SAndrea Arcangeli &enabled_attr.attr, 44071e3aac0SAndrea Arcangeli &defrag_attr.attr, 44179da5407SKirill A. Shutemov &use_zero_page_attr.attr, 44271e3aac0SAndrea Arcangeli #ifdef CONFIG_DEBUG_VM 44371e3aac0SAndrea Arcangeli &debug_cow_attr.attr, 44471e3aac0SAndrea Arcangeli #endif 44571e3aac0SAndrea Arcangeli NULL, 44671e3aac0SAndrea Arcangeli }; 44771e3aac0SAndrea Arcangeli 44871e3aac0SAndrea Arcangeli static struct attribute_group hugepage_attr_group = { 44971e3aac0SAndrea Arcangeli .attrs = hugepage_attr, 450ba76149fSAndrea Arcangeli }; 451ba76149fSAndrea Arcangeli 452ba76149fSAndrea Arcangeli static ssize_t scan_sleep_millisecs_show(struct kobject *kobj, 453ba76149fSAndrea Arcangeli struct kobj_attribute *attr, 454ba76149fSAndrea Arcangeli char *buf) 455ba76149fSAndrea Arcangeli { 456ba76149fSAndrea Arcangeli return sprintf(buf, "%u\n", khugepaged_scan_sleep_millisecs); 457ba76149fSAndrea Arcangeli } 458ba76149fSAndrea Arcangeli 459ba76149fSAndrea Arcangeli static ssize_t scan_sleep_millisecs_store(struct kobject *kobj, 460ba76149fSAndrea Arcangeli struct kobj_attribute *attr, 461ba76149fSAndrea Arcangeli const char *buf, size_t count) 462ba76149fSAndrea Arcangeli { 463ba76149fSAndrea Arcangeli unsigned long msecs; 464ba76149fSAndrea Arcangeli int err; 465ba76149fSAndrea Arcangeli 4663dbb95f7SJingoo Han err = kstrtoul(buf, 10, &msecs); 467ba76149fSAndrea Arcangeli if (err || msecs > UINT_MAX) 468ba76149fSAndrea Arcangeli return -EINVAL; 469ba76149fSAndrea Arcangeli 470ba76149fSAndrea Arcangeli khugepaged_scan_sleep_millisecs = msecs; 471f0508977SDavid Rientjes khugepaged_sleep_expire = 0; 472ba76149fSAndrea Arcangeli wake_up_interruptible(&khugepaged_wait); 473ba76149fSAndrea Arcangeli 474ba76149fSAndrea Arcangeli return count; 475ba76149fSAndrea Arcangeli } 476ba76149fSAndrea Arcangeli static struct kobj_attribute scan_sleep_millisecs_attr = 477ba76149fSAndrea Arcangeli __ATTR(scan_sleep_millisecs, 0644, scan_sleep_millisecs_show, 478ba76149fSAndrea Arcangeli scan_sleep_millisecs_store); 479ba76149fSAndrea Arcangeli 480ba76149fSAndrea Arcangeli static ssize_t alloc_sleep_millisecs_show(struct kobject *kobj, 481ba76149fSAndrea Arcangeli struct kobj_attribute *attr, 482ba76149fSAndrea Arcangeli char *buf) 483ba76149fSAndrea Arcangeli { 484ba76149fSAndrea Arcangeli return sprintf(buf, "%u\n", khugepaged_alloc_sleep_millisecs); 485ba76149fSAndrea Arcangeli } 486ba76149fSAndrea Arcangeli 487ba76149fSAndrea Arcangeli static ssize_t alloc_sleep_millisecs_store(struct kobject *kobj, 488ba76149fSAndrea Arcangeli struct kobj_attribute *attr, 489ba76149fSAndrea Arcangeli const char *buf, size_t count) 490ba76149fSAndrea Arcangeli { 491ba76149fSAndrea Arcangeli unsigned long msecs; 492ba76149fSAndrea Arcangeli int err; 493ba76149fSAndrea Arcangeli 4943dbb95f7SJingoo Han err = kstrtoul(buf, 10, &msecs); 495ba76149fSAndrea Arcangeli if (err || msecs > UINT_MAX) 496ba76149fSAndrea Arcangeli return -EINVAL; 497ba76149fSAndrea Arcangeli 498ba76149fSAndrea Arcangeli khugepaged_alloc_sleep_millisecs = msecs; 499f0508977SDavid Rientjes khugepaged_sleep_expire = 0; 500ba76149fSAndrea Arcangeli wake_up_interruptible(&khugepaged_wait); 501ba76149fSAndrea Arcangeli 502ba76149fSAndrea Arcangeli return count; 503ba76149fSAndrea Arcangeli } 504ba76149fSAndrea Arcangeli static struct kobj_attribute alloc_sleep_millisecs_attr = 505ba76149fSAndrea Arcangeli __ATTR(alloc_sleep_millisecs, 0644, alloc_sleep_millisecs_show, 506ba76149fSAndrea Arcangeli alloc_sleep_millisecs_store); 507ba76149fSAndrea Arcangeli 508ba76149fSAndrea Arcangeli static ssize_t pages_to_scan_show(struct kobject *kobj, 509ba76149fSAndrea Arcangeli struct kobj_attribute *attr, 510ba76149fSAndrea Arcangeli char *buf) 511ba76149fSAndrea Arcangeli { 512ba76149fSAndrea Arcangeli return sprintf(buf, "%u\n", khugepaged_pages_to_scan); 513ba76149fSAndrea Arcangeli } 514ba76149fSAndrea Arcangeli static ssize_t pages_to_scan_store(struct kobject *kobj, 515ba76149fSAndrea Arcangeli struct kobj_attribute *attr, 516ba76149fSAndrea Arcangeli const char *buf, size_t count) 517ba76149fSAndrea Arcangeli { 518ba76149fSAndrea Arcangeli int err; 519ba76149fSAndrea Arcangeli unsigned long pages; 520ba76149fSAndrea Arcangeli 5213dbb95f7SJingoo Han err = kstrtoul(buf, 10, &pages); 522ba76149fSAndrea Arcangeli if (err || !pages || pages > UINT_MAX) 523ba76149fSAndrea Arcangeli return -EINVAL; 524ba76149fSAndrea Arcangeli 525ba76149fSAndrea Arcangeli khugepaged_pages_to_scan = pages; 526ba76149fSAndrea Arcangeli 527ba76149fSAndrea Arcangeli return count; 528ba76149fSAndrea Arcangeli } 529ba76149fSAndrea Arcangeli static struct kobj_attribute pages_to_scan_attr = 530ba76149fSAndrea Arcangeli __ATTR(pages_to_scan, 0644, pages_to_scan_show, 531ba76149fSAndrea Arcangeli pages_to_scan_store); 532ba76149fSAndrea Arcangeli 533ba76149fSAndrea Arcangeli static ssize_t pages_collapsed_show(struct kobject *kobj, 534ba76149fSAndrea Arcangeli struct kobj_attribute *attr, 535ba76149fSAndrea Arcangeli char *buf) 536ba76149fSAndrea Arcangeli { 537ba76149fSAndrea Arcangeli return sprintf(buf, "%u\n", khugepaged_pages_collapsed); 538ba76149fSAndrea Arcangeli } 539ba76149fSAndrea Arcangeli static struct kobj_attribute pages_collapsed_attr = 540ba76149fSAndrea Arcangeli __ATTR_RO(pages_collapsed); 541ba76149fSAndrea Arcangeli 542ba76149fSAndrea Arcangeli static ssize_t full_scans_show(struct kobject *kobj, 543ba76149fSAndrea Arcangeli struct kobj_attribute *attr, 544ba76149fSAndrea Arcangeli char *buf) 545ba76149fSAndrea Arcangeli { 546ba76149fSAndrea Arcangeli return sprintf(buf, "%u\n", khugepaged_full_scans); 547ba76149fSAndrea Arcangeli } 548ba76149fSAndrea Arcangeli static struct kobj_attribute full_scans_attr = 549ba76149fSAndrea Arcangeli __ATTR_RO(full_scans); 550ba76149fSAndrea Arcangeli 551ba76149fSAndrea Arcangeli static ssize_t khugepaged_defrag_show(struct kobject *kobj, 552ba76149fSAndrea Arcangeli struct kobj_attribute *attr, char *buf) 553ba76149fSAndrea Arcangeli { 554ba76149fSAndrea Arcangeli return single_flag_show(kobj, attr, buf, 555ba76149fSAndrea Arcangeli TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG); 556ba76149fSAndrea Arcangeli } 557ba76149fSAndrea Arcangeli static ssize_t khugepaged_defrag_store(struct kobject *kobj, 558ba76149fSAndrea Arcangeli struct kobj_attribute *attr, 559ba76149fSAndrea Arcangeli const char *buf, size_t count) 560ba76149fSAndrea Arcangeli { 561ba76149fSAndrea Arcangeli return single_flag_store(kobj, attr, buf, count, 562ba76149fSAndrea Arcangeli TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG); 563ba76149fSAndrea Arcangeli } 564ba76149fSAndrea Arcangeli static struct kobj_attribute khugepaged_defrag_attr = 565ba76149fSAndrea Arcangeli __ATTR(defrag, 0644, khugepaged_defrag_show, 566ba76149fSAndrea Arcangeli khugepaged_defrag_store); 567ba76149fSAndrea Arcangeli 568ba76149fSAndrea Arcangeli /* 569ba76149fSAndrea Arcangeli * max_ptes_none controls if khugepaged should collapse hugepages over 570ba76149fSAndrea Arcangeli * any unmapped ptes in turn potentially increasing the memory 571ba76149fSAndrea Arcangeli * footprint of the vmas. When max_ptes_none is 0 khugepaged will not 572ba76149fSAndrea Arcangeli * reduce the available free memory in the system as it 573ba76149fSAndrea Arcangeli * runs. Increasing max_ptes_none will instead potentially reduce the 574ba76149fSAndrea Arcangeli * free memory in the system during the khugepaged scan. 575ba76149fSAndrea Arcangeli */ 576ba76149fSAndrea Arcangeli static ssize_t khugepaged_max_ptes_none_show(struct kobject *kobj, 577ba76149fSAndrea Arcangeli struct kobj_attribute *attr, 578ba76149fSAndrea Arcangeli char *buf) 579ba76149fSAndrea Arcangeli { 580ba76149fSAndrea Arcangeli return sprintf(buf, "%u\n", khugepaged_max_ptes_none); 581ba76149fSAndrea Arcangeli } 582ba76149fSAndrea Arcangeli static ssize_t khugepaged_max_ptes_none_store(struct kobject *kobj, 583ba76149fSAndrea Arcangeli struct kobj_attribute *attr, 584ba76149fSAndrea Arcangeli const char *buf, size_t count) 585ba76149fSAndrea Arcangeli { 586ba76149fSAndrea Arcangeli int err; 587ba76149fSAndrea Arcangeli unsigned long max_ptes_none; 588ba76149fSAndrea Arcangeli 5893dbb95f7SJingoo Han err = kstrtoul(buf, 10, &max_ptes_none); 590ba76149fSAndrea Arcangeli if (err || max_ptes_none > HPAGE_PMD_NR-1) 591ba76149fSAndrea Arcangeli return -EINVAL; 592ba76149fSAndrea Arcangeli 593ba76149fSAndrea Arcangeli khugepaged_max_ptes_none = max_ptes_none; 594ba76149fSAndrea Arcangeli 595ba76149fSAndrea Arcangeli return count; 596ba76149fSAndrea Arcangeli } 597ba76149fSAndrea Arcangeli static struct kobj_attribute khugepaged_max_ptes_none_attr = 598ba76149fSAndrea Arcangeli __ATTR(max_ptes_none, 0644, khugepaged_max_ptes_none_show, 599ba76149fSAndrea Arcangeli khugepaged_max_ptes_none_store); 600ba76149fSAndrea Arcangeli 601ba76149fSAndrea Arcangeli static struct attribute *khugepaged_attr[] = { 602ba76149fSAndrea Arcangeli &khugepaged_defrag_attr.attr, 603ba76149fSAndrea Arcangeli &khugepaged_max_ptes_none_attr.attr, 604ba76149fSAndrea Arcangeli &pages_to_scan_attr.attr, 605ba76149fSAndrea Arcangeli &pages_collapsed_attr.attr, 606ba76149fSAndrea Arcangeli &full_scans_attr.attr, 607ba76149fSAndrea Arcangeli &scan_sleep_millisecs_attr.attr, 608ba76149fSAndrea Arcangeli &alloc_sleep_millisecs_attr.attr, 609ba76149fSAndrea Arcangeli NULL, 610ba76149fSAndrea Arcangeli }; 611ba76149fSAndrea Arcangeli 612ba76149fSAndrea Arcangeli static struct attribute_group khugepaged_attr_group = { 613ba76149fSAndrea Arcangeli .attrs = khugepaged_attr, 614ba76149fSAndrea Arcangeli .name = "khugepaged", 61571e3aac0SAndrea Arcangeli }; 616569e5590SShaohua Li 617569e5590SShaohua Li static int __init hugepage_init_sysfs(struct kobject **hugepage_kobj) 618569e5590SShaohua Li { 619569e5590SShaohua Li int err; 620569e5590SShaohua Li 621569e5590SShaohua Li *hugepage_kobj = kobject_create_and_add("transparent_hugepage", mm_kobj); 622569e5590SShaohua Li if (unlikely(!*hugepage_kobj)) { 623ae3a8c1cSAndrew Morton pr_err("failed to create transparent hugepage kobject\n"); 624569e5590SShaohua Li return -ENOMEM; 625569e5590SShaohua Li } 626569e5590SShaohua Li 627569e5590SShaohua Li err = sysfs_create_group(*hugepage_kobj, &hugepage_attr_group); 628569e5590SShaohua Li if (err) { 629ae3a8c1cSAndrew Morton pr_err("failed to register transparent hugepage group\n"); 630569e5590SShaohua Li goto delete_obj; 631569e5590SShaohua Li } 632569e5590SShaohua Li 633569e5590SShaohua Li err = sysfs_create_group(*hugepage_kobj, &khugepaged_attr_group); 634569e5590SShaohua Li if (err) { 635ae3a8c1cSAndrew Morton pr_err("failed to register transparent hugepage group\n"); 636569e5590SShaohua Li goto remove_hp_group; 637569e5590SShaohua Li } 638569e5590SShaohua Li 639569e5590SShaohua Li return 0; 640569e5590SShaohua Li 641569e5590SShaohua Li remove_hp_group: 642569e5590SShaohua Li sysfs_remove_group(*hugepage_kobj, &hugepage_attr_group); 643569e5590SShaohua Li delete_obj: 644569e5590SShaohua Li kobject_put(*hugepage_kobj); 645569e5590SShaohua Li return err; 646569e5590SShaohua Li } 647569e5590SShaohua Li 648569e5590SShaohua Li static void __init hugepage_exit_sysfs(struct kobject *hugepage_kobj) 649569e5590SShaohua Li { 650569e5590SShaohua Li sysfs_remove_group(hugepage_kobj, &khugepaged_attr_group); 651569e5590SShaohua Li sysfs_remove_group(hugepage_kobj, &hugepage_attr_group); 652569e5590SShaohua Li kobject_put(hugepage_kobj); 653569e5590SShaohua Li } 654569e5590SShaohua Li #else 655569e5590SShaohua Li static inline int hugepage_init_sysfs(struct kobject **hugepage_kobj) 656569e5590SShaohua Li { 657569e5590SShaohua Li return 0; 658569e5590SShaohua Li } 659569e5590SShaohua Li 660569e5590SShaohua Li static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj) 661569e5590SShaohua Li { 662569e5590SShaohua Li } 66371e3aac0SAndrea Arcangeli #endif /* CONFIG_SYSFS */ 66471e3aac0SAndrea Arcangeli 66571e3aac0SAndrea Arcangeli static int __init hugepage_init(void) 66671e3aac0SAndrea Arcangeli { 66771e3aac0SAndrea Arcangeli int err; 668569e5590SShaohua Li struct kobject *hugepage_kobj; 66971e3aac0SAndrea Arcangeli 6704b7167b9SAndrea Arcangeli if (!has_transparent_hugepage()) { 6714b7167b9SAndrea Arcangeli transparent_hugepage_flags = 0; 672569e5590SShaohua Li return -EINVAL; 6734b7167b9SAndrea Arcangeli } 6744b7167b9SAndrea Arcangeli 675ff20c2e0SKirill A. Shutemov khugepaged_pages_to_scan = HPAGE_PMD_NR * 8; 676ff20c2e0SKirill A. Shutemov khugepaged_max_ptes_none = HPAGE_PMD_NR - 1; 677ff20c2e0SKirill A. Shutemov /* 678ff20c2e0SKirill A. Shutemov * hugepages can't be allocated by the buddy allocator 679ff20c2e0SKirill A. Shutemov */ 680ff20c2e0SKirill A. Shutemov MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER >= MAX_ORDER); 681ff20c2e0SKirill A. Shutemov /* 682ff20c2e0SKirill A. Shutemov * we use page->mapping and page->index in second tail page 683ff20c2e0SKirill A. Shutemov * as list_head: assuming THP order >= 2 684ff20c2e0SKirill A. Shutemov */ 685ff20c2e0SKirill A. Shutemov MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER < 2); 686ff20c2e0SKirill A. Shutemov 687569e5590SShaohua Li err = hugepage_init_sysfs(&hugepage_kobj); 688569e5590SShaohua Li if (err) 68965ebb64fSKirill A. Shutemov goto err_sysfs; 690ba76149fSAndrea Arcangeli 691ba76149fSAndrea Arcangeli err = khugepaged_slab_init(); 692ba76149fSAndrea Arcangeli if (err) 69365ebb64fSKirill A. Shutemov goto err_slab; 694ba76149fSAndrea Arcangeli 69565ebb64fSKirill A. Shutemov err = register_shrinker(&huge_zero_page_shrinker); 69665ebb64fSKirill A. Shutemov if (err) 69765ebb64fSKirill A. Shutemov goto err_hzp_shrinker; 6989a982250SKirill A. Shutemov err = register_shrinker(&deferred_split_shrinker); 6999a982250SKirill A. Shutemov if (err) 7009a982250SKirill A. Shutemov goto err_split_shrinker; 70197ae1749SKirill A. Shutemov 70297562cd2SRik van Riel /* 70397562cd2SRik van Riel * By default disable transparent hugepages on smaller systems, 70497562cd2SRik van Riel * where the extra memory used could hurt more than TLB overhead 70597562cd2SRik van Riel * is likely to save. The admin can still enable it through /sys. 70697562cd2SRik van Riel */ 70779553da2SKirill A. Shutemov if (totalram_pages < (512 << (20 - PAGE_SHIFT))) { 70897562cd2SRik van Riel transparent_hugepage_flags = 0; 70979553da2SKirill A. Shutemov return 0; 71079553da2SKirill A. Shutemov } 71197562cd2SRik van Riel 71279553da2SKirill A. Shutemov err = start_stop_khugepaged(); 71365ebb64fSKirill A. Shutemov if (err) 71465ebb64fSKirill A. Shutemov goto err_khugepaged; 715ba76149fSAndrea Arcangeli 716569e5590SShaohua Li return 0; 71765ebb64fSKirill A. Shutemov err_khugepaged: 7189a982250SKirill A. Shutemov unregister_shrinker(&deferred_split_shrinker); 7199a982250SKirill A. Shutemov err_split_shrinker: 72065ebb64fSKirill A. Shutemov unregister_shrinker(&huge_zero_page_shrinker); 72165ebb64fSKirill A. Shutemov err_hzp_shrinker: 72265ebb64fSKirill A. Shutemov khugepaged_slab_exit(); 72365ebb64fSKirill A. Shutemov err_slab: 724569e5590SShaohua Li hugepage_exit_sysfs(hugepage_kobj); 72565ebb64fSKirill A. Shutemov err_sysfs: 726ba76149fSAndrea Arcangeli return err; 72771e3aac0SAndrea Arcangeli } 728a64fb3cdSPaul Gortmaker subsys_initcall(hugepage_init); 72971e3aac0SAndrea Arcangeli 73071e3aac0SAndrea Arcangeli static int __init setup_transparent_hugepage(char *str) 73171e3aac0SAndrea Arcangeli { 73271e3aac0SAndrea Arcangeli int ret = 0; 73371e3aac0SAndrea Arcangeli if (!str) 73471e3aac0SAndrea Arcangeli goto out; 73571e3aac0SAndrea Arcangeli if (!strcmp(str, "always")) { 73671e3aac0SAndrea Arcangeli set_bit(TRANSPARENT_HUGEPAGE_FLAG, 73771e3aac0SAndrea Arcangeli &transparent_hugepage_flags); 73871e3aac0SAndrea Arcangeli clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 73971e3aac0SAndrea Arcangeli &transparent_hugepage_flags); 74071e3aac0SAndrea Arcangeli ret = 1; 74171e3aac0SAndrea Arcangeli } else if (!strcmp(str, "madvise")) { 74271e3aac0SAndrea Arcangeli clear_bit(TRANSPARENT_HUGEPAGE_FLAG, 74371e3aac0SAndrea Arcangeli &transparent_hugepage_flags); 74471e3aac0SAndrea Arcangeli set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 74571e3aac0SAndrea Arcangeli &transparent_hugepage_flags); 74671e3aac0SAndrea Arcangeli ret = 1; 74771e3aac0SAndrea Arcangeli } else if (!strcmp(str, "never")) { 74871e3aac0SAndrea Arcangeli clear_bit(TRANSPARENT_HUGEPAGE_FLAG, 74971e3aac0SAndrea Arcangeli &transparent_hugepage_flags); 75071e3aac0SAndrea Arcangeli clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 75171e3aac0SAndrea Arcangeli &transparent_hugepage_flags); 75271e3aac0SAndrea Arcangeli ret = 1; 75371e3aac0SAndrea Arcangeli } 75471e3aac0SAndrea Arcangeli out: 75571e3aac0SAndrea Arcangeli if (!ret) 756ae3a8c1cSAndrew Morton pr_warn("transparent_hugepage= cannot parse, ignored\n"); 75771e3aac0SAndrea Arcangeli return ret; 75871e3aac0SAndrea Arcangeli } 75971e3aac0SAndrea Arcangeli __setup("transparent_hugepage=", setup_transparent_hugepage); 76071e3aac0SAndrea Arcangeli 761b32967ffSMel Gorman pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma) 76271e3aac0SAndrea Arcangeli { 76371e3aac0SAndrea Arcangeli if (likely(vma->vm_flags & VM_WRITE)) 76471e3aac0SAndrea Arcangeli pmd = pmd_mkwrite(pmd); 76571e3aac0SAndrea Arcangeli return pmd; 76671e3aac0SAndrea Arcangeli } 76771e3aac0SAndrea Arcangeli 7683122359aSKirill A. Shutemov static inline pmd_t mk_huge_pmd(struct page *page, pgprot_t prot) 769b3092b3bSBob Liu { 770340a43beSYang Shi return pmd_mkhuge(mk_pmd(page, prot)); 771b3092b3bSBob Liu } 772b3092b3bSBob Liu 7739a982250SKirill A. Shutemov static inline struct list_head *page_deferred_list(struct page *page) 7749a982250SKirill A. Shutemov { 7759a982250SKirill A. Shutemov /* 7769a982250SKirill A. Shutemov * ->lru in the tail pages is occupied by compound_head. 7779a982250SKirill A. Shutemov * Let's use ->mapping + ->index in the second tail page as list_head. 7789a982250SKirill A. Shutemov */ 7799a982250SKirill A. Shutemov return (struct list_head *)&page[2].mapping; 7809a982250SKirill A. Shutemov } 7819a982250SKirill A. Shutemov 7829a982250SKirill A. Shutemov void prep_transhuge_page(struct page *page) 7839a982250SKirill A. Shutemov { 7849a982250SKirill A. Shutemov /* 7859a982250SKirill A. Shutemov * we use page->mapping and page->indexlru in second tail page 7869a982250SKirill A. Shutemov * as list_head: assuming THP order >= 2 7879a982250SKirill A. Shutemov */ 7889a982250SKirill A. Shutemov 7899a982250SKirill A. Shutemov INIT_LIST_HEAD(page_deferred_list(page)); 7909a982250SKirill A. Shutemov set_compound_page_dtor(page, TRANSHUGE_PAGE_DTOR); 7919a982250SKirill A. Shutemov } 7929a982250SKirill A. Shutemov 79371e3aac0SAndrea Arcangeli static int __do_huge_pmd_anonymous_page(struct mm_struct *mm, 79471e3aac0SAndrea Arcangeli struct vm_area_struct *vma, 795230c92a8SAndrea Arcangeli unsigned long address, pmd_t *pmd, 7966b251fc9SAndrea Arcangeli struct page *page, gfp_t gfp, 7976b251fc9SAndrea Arcangeli unsigned int flags) 79871e3aac0SAndrea Arcangeli { 79900501b53SJohannes Weiner struct mem_cgroup *memcg; 80071e3aac0SAndrea Arcangeli pgtable_t pgtable; 801c4088ebdSKirill A. Shutemov spinlock_t *ptl; 802230c92a8SAndrea Arcangeli unsigned long haddr = address & HPAGE_PMD_MASK; 80371e3aac0SAndrea Arcangeli 804309381feSSasha Levin VM_BUG_ON_PAGE(!PageCompound(page), page); 80500501b53SJohannes Weiner 806f627c2f5SKirill A. Shutemov if (mem_cgroup_try_charge(page, mm, gfp, &memcg, true)) { 8076b251fc9SAndrea Arcangeli put_page(page); 8086b251fc9SAndrea Arcangeli count_vm_event(THP_FAULT_FALLBACK); 8096b251fc9SAndrea Arcangeli return VM_FAULT_FALLBACK; 8106b251fc9SAndrea Arcangeli } 81171e3aac0SAndrea Arcangeli 81200501b53SJohannes Weiner pgtable = pte_alloc_one(mm, haddr); 81300501b53SJohannes Weiner if (unlikely(!pgtable)) { 814f627c2f5SKirill A. Shutemov mem_cgroup_cancel_charge(page, memcg, true); 8156b251fc9SAndrea Arcangeli put_page(page); 81600501b53SJohannes Weiner return VM_FAULT_OOM; 81700501b53SJohannes Weiner } 81800501b53SJohannes Weiner 81971e3aac0SAndrea Arcangeli clear_huge_page(page, haddr, HPAGE_PMD_NR); 82052f37629SMinchan Kim /* 82152f37629SMinchan Kim * The memory barrier inside __SetPageUptodate makes sure that 82252f37629SMinchan Kim * clear_huge_page writes become visible before the set_pmd_at() 82352f37629SMinchan Kim * write. 82452f37629SMinchan Kim */ 82571e3aac0SAndrea Arcangeli __SetPageUptodate(page); 82671e3aac0SAndrea Arcangeli 827c4088ebdSKirill A. Shutemov ptl = pmd_lock(mm, pmd); 82871e3aac0SAndrea Arcangeli if (unlikely(!pmd_none(*pmd))) { 829c4088ebdSKirill A. Shutemov spin_unlock(ptl); 830f627c2f5SKirill A. Shutemov mem_cgroup_cancel_charge(page, memcg, true); 83171e3aac0SAndrea Arcangeli put_page(page); 83271e3aac0SAndrea Arcangeli pte_free(mm, pgtable); 83371e3aac0SAndrea Arcangeli } else { 83471e3aac0SAndrea Arcangeli pmd_t entry; 8356b251fc9SAndrea Arcangeli 8366b251fc9SAndrea Arcangeli /* Deliver the page fault to userland */ 8376b251fc9SAndrea Arcangeli if (userfaultfd_missing(vma)) { 8386b251fc9SAndrea Arcangeli int ret; 8396b251fc9SAndrea Arcangeli 8406b251fc9SAndrea Arcangeli spin_unlock(ptl); 841f627c2f5SKirill A. Shutemov mem_cgroup_cancel_charge(page, memcg, true); 8426b251fc9SAndrea Arcangeli put_page(page); 8436b251fc9SAndrea Arcangeli pte_free(mm, pgtable); 844230c92a8SAndrea Arcangeli ret = handle_userfault(vma, address, flags, 8456b251fc9SAndrea Arcangeli VM_UFFD_MISSING); 8466b251fc9SAndrea Arcangeli VM_BUG_ON(ret & VM_FAULT_FALLBACK); 8476b251fc9SAndrea Arcangeli return ret; 8486b251fc9SAndrea Arcangeli } 8496b251fc9SAndrea Arcangeli 8503122359aSKirill A. Shutemov entry = mk_huge_pmd(page, vma->vm_page_prot); 8513122359aSKirill A. Shutemov entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 852d281ee61SKirill A. Shutemov page_add_new_anon_rmap(page, vma, haddr, true); 853f627c2f5SKirill A. Shutemov mem_cgroup_commit_charge(page, memcg, false, true); 85400501b53SJohannes Weiner lru_cache_add_active_or_unevictable(page, vma); 8556b0b50b0SAneesh Kumar K.V pgtable_trans_huge_deposit(mm, pmd, pgtable); 85671e3aac0SAndrea Arcangeli set_pmd_at(mm, haddr, pmd, entry); 85771e3aac0SAndrea Arcangeli add_mm_counter(mm, MM_ANONPAGES, HPAGE_PMD_NR); 858e1f56c89SKirill A. Shutemov atomic_long_inc(&mm->nr_ptes); 859c4088ebdSKirill A. Shutemov spin_unlock(ptl); 8606b251fc9SAndrea Arcangeli count_vm_event(THP_FAULT_ALLOC); 86171e3aac0SAndrea Arcangeli } 86271e3aac0SAndrea Arcangeli 863aa2e878eSDavid Rientjes return 0; 86471e3aac0SAndrea Arcangeli } 86571e3aac0SAndrea Arcangeli 866444eb2a4SMel Gorman /* 867444eb2a4SMel Gorman * If THP is set to always then directly reclaim/compact as necessary 868444eb2a4SMel Gorman * If set to defer then do no reclaim and defer to khugepaged 869444eb2a4SMel Gorman * If set to madvise and the VMA is flagged then directly reclaim/compact 870444eb2a4SMel Gorman */ 871444eb2a4SMel Gorman static inline gfp_t alloc_hugepage_direct_gfpmask(struct vm_area_struct *vma) 8720bbbc0b3SAndrea Arcangeli { 873444eb2a4SMel Gorman gfp_t reclaim_flags = 0; 874444eb2a4SMel Gorman 875444eb2a4SMel Gorman if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags) && 876444eb2a4SMel Gorman (vma->vm_flags & VM_HUGEPAGE)) 877444eb2a4SMel Gorman reclaim_flags = __GFP_DIRECT_RECLAIM; 878444eb2a4SMel Gorman else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags)) 879444eb2a4SMel Gorman reclaim_flags = __GFP_KSWAPD_RECLAIM; 880444eb2a4SMel Gorman else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags)) 881444eb2a4SMel Gorman reclaim_flags = __GFP_DIRECT_RECLAIM; 882444eb2a4SMel Gorman 883444eb2a4SMel Gorman return GFP_TRANSHUGE | reclaim_flags; 884444eb2a4SMel Gorman } 885444eb2a4SMel Gorman 886444eb2a4SMel Gorman /* Defrag for khugepaged will enter direct reclaim/compaction if necessary */ 887444eb2a4SMel Gorman static inline gfp_t alloc_hugepage_khugepaged_gfpmask(void) 888444eb2a4SMel Gorman { 889444eb2a4SMel Gorman return GFP_TRANSHUGE | (khugepaged_defrag() ? __GFP_DIRECT_RECLAIM : 0); 8900bbbc0b3SAndrea Arcangeli } 8910bbbc0b3SAndrea Arcangeli 892c4088ebdSKirill A. Shutemov /* Caller must hold page table lock. */ 893d295e341SKirill A. Shutemov static bool set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm, 89497ae1749SKirill A. Shutemov struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd, 8955918d10aSKirill A. Shutemov struct page *zero_page) 896fc9fe822SKirill A. Shutemov { 897fc9fe822SKirill A. Shutemov pmd_t entry; 8987c414164SAndrew Morton if (!pmd_none(*pmd)) 8997c414164SAndrew Morton return false; 9005918d10aSKirill A. Shutemov entry = mk_pmd(zero_page, vma->vm_page_prot); 901fc9fe822SKirill A. Shutemov entry = pmd_mkhuge(entry); 90212c9d70bSMatthew Wilcox if (pgtable) 9036b0b50b0SAneesh Kumar K.V pgtable_trans_huge_deposit(mm, pmd, pgtable); 904fc9fe822SKirill A. Shutemov set_pmd_at(mm, haddr, pmd, entry); 905e1f56c89SKirill A. Shutemov atomic_long_inc(&mm->nr_ptes); 9067c414164SAndrew Morton return true; 907fc9fe822SKirill A. Shutemov } 908fc9fe822SKirill A. Shutemov 90971e3aac0SAndrea Arcangeli int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma, 91071e3aac0SAndrea Arcangeli unsigned long address, pmd_t *pmd, 91171e3aac0SAndrea Arcangeli unsigned int flags) 91271e3aac0SAndrea Arcangeli { 913077fcf11SAneesh Kumar K.V gfp_t gfp; 91471e3aac0SAndrea Arcangeli struct page *page; 91571e3aac0SAndrea Arcangeli unsigned long haddr = address & HPAGE_PMD_MASK; 91671e3aac0SAndrea Arcangeli 917128ec037SKirill A. Shutemov if (haddr < vma->vm_start || haddr + HPAGE_PMD_SIZE > vma->vm_end) 918c0292554SKirill A. Shutemov return VM_FAULT_FALLBACK; 91971e3aac0SAndrea Arcangeli if (unlikely(anon_vma_prepare(vma))) 92071e3aac0SAndrea Arcangeli return VM_FAULT_OOM; 9216d50e60cSDavid Rientjes if (unlikely(khugepaged_enter(vma, vma->vm_flags))) 922ba76149fSAndrea Arcangeli return VM_FAULT_OOM; 923593befa6SDominik Dingel if (!(flags & FAULT_FLAG_WRITE) && !mm_forbids_zeropage(mm) && 92479da5407SKirill A. Shutemov transparent_hugepage_use_zero_page()) { 925c4088ebdSKirill A. Shutemov spinlock_t *ptl; 92680371957SKirill A. Shutemov pgtable_t pgtable; 9275918d10aSKirill A. Shutemov struct page *zero_page; 9283ea41e62SKirill A. Shutemov bool set; 9296b251fc9SAndrea Arcangeli int ret; 93080371957SKirill A. Shutemov pgtable = pte_alloc_one(mm, haddr); 93180371957SKirill A. Shutemov if (unlikely(!pgtable)) 93280371957SKirill A. Shutemov return VM_FAULT_OOM; 9335918d10aSKirill A. Shutemov zero_page = get_huge_zero_page(); 9345918d10aSKirill A. Shutemov if (unlikely(!zero_page)) { 93597ae1749SKirill A. Shutemov pte_free(mm, pgtable); 93697ae1749SKirill A. Shutemov count_vm_event(THP_FAULT_FALLBACK); 937c0292554SKirill A. Shutemov return VM_FAULT_FALLBACK; 93897ae1749SKirill A. Shutemov } 939c4088ebdSKirill A. Shutemov ptl = pmd_lock(mm, pmd); 9406b251fc9SAndrea Arcangeli ret = 0; 9416b251fc9SAndrea Arcangeli set = false; 9426b251fc9SAndrea Arcangeli if (pmd_none(*pmd)) { 9436b251fc9SAndrea Arcangeli if (userfaultfd_missing(vma)) { 9446b251fc9SAndrea Arcangeli spin_unlock(ptl); 945230c92a8SAndrea Arcangeli ret = handle_userfault(vma, address, flags, 9466b251fc9SAndrea Arcangeli VM_UFFD_MISSING); 9476b251fc9SAndrea Arcangeli VM_BUG_ON(ret & VM_FAULT_FALLBACK); 9486b251fc9SAndrea Arcangeli } else { 9496b251fc9SAndrea Arcangeli set_huge_zero_page(pgtable, mm, vma, 9506b251fc9SAndrea Arcangeli haddr, pmd, 9515918d10aSKirill A. Shutemov zero_page); 952c4088ebdSKirill A. Shutemov spin_unlock(ptl); 9536b251fc9SAndrea Arcangeli set = true; 9546b251fc9SAndrea Arcangeli } 9556b251fc9SAndrea Arcangeli } else 9566b251fc9SAndrea Arcangeli spin_unlock(ptl); 9573ea41e62SKirill A. Shutemov if (!set) { 9583ea41e62SKirill A. Shutemov pte_free(mm, pgtable); 9593ea41e62SKirill A. Shutemov put_huge_zero_page(); 9603ea41e62SKirill A. Shutemov } 9616b251fc9SAndrea Arcangeli return ret; 96280371957SKirill A. Shutemov } 963444eb2a4SMel Gorman gfp = alloc_hugepage_direct_gfpmask(vma); 964077fcf11SAneesh Kumar K.V page = alloc_hugepage_vma(gfp, vma, haddr, HPAGE_PMD_ORDER); 96581ab4201SAndi Kleen if (unlikely(!page)) { 96681ab4201SAndi Kleen count_vm_event(THP_FAULT_FALLBACK); 967c0292554SKirill A. Shutemov return VM_FAULT_FALLBACK; 96881ab4201SAndi Kleen } 9699a982250SKirill A. Shutemov prep_transhuge_page(page); 970230c92a8SAndrea Arcangeli return __do_huge_pmd_anonymous_page(mm, vma, address, pmd, page, gfp, 971230c92a8SAndrea Arcangeli flags); 97271e3aac0SAndrea Arcangeli } 97371e3aac0SAndrea Arcangeli 974ae18d6dcSMatthew Wilcox static void insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr, 975f25748e3SDan Williams pmd_t *pmd, pfn_t pfn, pgprot_t prot, bool write) 9765cad465dSMatthew Wilcox { 9775cad465dSMatthew Wilcox struct mm_struct *mm = vma->vm_mm; 9785cad465dSMatthew Wilcox pmd_t entry; 9795cad465dSMatthew Wilcox spinlock_t *ptl; 9805cad465dSMatthew Wilcox 9815cad465dSMatthew Wilcox ptl = pmd_lock(mm, pmd); 982f25748e3SDan Williams entry = pmd_mkhuge(pfn_t_pmd(pfn, prot)); 983f25748e3SDan Williams if (pfn_t_devmap(pfn)) 984f25748e3SDan Williams entry = pmd_mkdevmap(entry); 9855cad465dSMatthew Wilcox if (write) { 9865cad465dSMatthew Wilcox entry = pmd_mkyoung(pmd_mkdirty(entry)); 9875cad465dSMatthew Wilcox entry = maybe_pmd_mkwrite(entry, vma); 9885cad465dSMatthew Wilcox } 9895cad465dSMatthew Wilcox set_pmd_at(mm, addr, pmd, entry); 9905cad465dSMatthew Wilcox update_mmu_cache_pmd(vma, addr, pmd); 9915cad465dSMatthew Wilcox spin_unlock(ptl); 9925cad465dSMatthew Wilcox } 9935cad465dSMatthew Wilcox 9945cad465dSMatthew Wilcox int vmf_insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr, 995f25748e3SDan Williams pmd_t *pmd, pfn_t pfn, bool write) 9965cad465dSMatthew Wilcox { 9975cad465dSMatthew Wilcox pgprot_t pgprot = vma->vm_page_prot; 9985cad465dSMatthew Wilcox /* 9995cad465dSMatthew Wilcox * If we had pmd_special, we could avoid all these restrictions, 10005cad465dSMatthew Wilcox * but we need to be consistent with PTEs and architectures that 10015cad465dSMatthew Wilcox * can't support a 'special' bit. 10025cad465dSMatthew Wilcox */ 10035cad465dSMatthew Wilcox BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))); 10045cad465dSMatthew Wilcox BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 10055cad465dSMatthew Wilcox (VM_PFNMAP|VM_MIXEDMAP)); 10065cad465dSMatthew Wilcox BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 1007f25748e3SDan Williams BUG_ON(!pfn_t_devmap(pfn)); 10085cad465dSMatthew Wilcox 10095cad465dSMatthew Wilcox if (addr < vma->vm_start || addr >= vma->vm_end) 10105cad465dSMatthew Wilcox return VM_FAULT_SIGBUS; 10115cad465dSMatthew Wilcox if (track_pfn_insert(vma, &pgprot, pfn)) 10125cad465dSMatthew Wilcox return VM_FAULT_SIGBUS; 1013ae18d6dcSMatthew Wilcox insert_pfn_pmd(vma, addr, pmd, pfn, pgprot, write); 1014ae18d6dcSMatthew Wilcox return VM_FAULT_NOPAGE; 10155cad465dSMatthew Wilcox } 1016dee41079SDan Williams EXPORT_SYMBOL_GPL(vmf_insert_pfn_pmd); 10175cad465dSMatthew Wilcox 10183565fce3SDan Williams static void touch_pmd(struct vm_area_struct *vma, unsigned long addr, 10193565fce3SDan Williams pmd_t *pmd) 10203565fce3SDan Williams { 10213565fce3SDan Williams pmd_t _pmd; 10223565fce3SDan Williams 10233565fce3SDan Williams /* 10243565fce3SDan Williams * We should set the dirty bit only for FOLL_WRITE but for now 10253565fce3SDan Williams * the dirty bit in the pmd is meaningless. And if the dirty 10263565fce3SDan Williams * bit will become meaningful and we'll only set it with 10273565fce3SDan Williams * FOLL_WRITE, an atomic set_bit will be required on the pmd to 10283565fce3SDan Williams * set the young bit, instead of the current set_pmd_at. 10293565fce3SDan Williams */ 10303565fce3SDan Williams _pmd = pmd_mkyoung(pmd_mkdirty(*pmd)); 10313565fce3SDan Williams if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK, 10323565fce3SDan Williams pmd, _pmd, 1)) 10333565fce3SDan Williams update_mmu_cache_pmd(vma, addr, pmd); 10343565fce3SDan Williams } 10353565fce3SDan Williams 10363565fce3SDan Williams struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr, 10373565fce3SDan Williams pmd_t *pmd, int flags) 10383565fce3SDan Williams { 10393565fce3SDan Williams unsigned long pfn = pmd_pfn(*pmd); 10403565fce3SDan Williams struct mm_struct *mm = vma->vm_mm; 10413565fce3SDan Williams struct dev_pagemap *pgmap; 10423565fce3SDan Williams struct page *page; 10433565fce3SDan Williams 10443565fce3SDan Williams assert_spin_locked(pmd_lockptr(mm, pmd)); 10453565fce3SDan Williams 10463565fce3SDan Williams if (flags & FOLL_WRITE && !pmd_write(*pmd)) 10473565fce3SDan Williams return NULL; 10483565fce3SDan Williams 10493565fce3SDan Williams if (pmd_present(*pmd) && pmd_devmap(*pmd)) 10503565fce3SDan Williams /* pass */; 10513565fce3SDan Williams else 10523565fce3SDan Williams return NULL; 10533565fce3SDan Williams 10543565fce3SDan Williams if (flags & FOLL_TOUCH) 10553565fce3SDan Williams touch_pmd(vma, addr, pmd); 10563565fce3SDan Williams 10573565fce3SDan Williams /* 10583565fce3SDan Williams * device mapped pages can only be returned if the 10593565fce3SDan Williams * caller will manage the page reference count. 10603565fce3SDan Williams */ 10613565fce3SDan Williams if (!(flags & FOLL_GET)) 10623565fce3SDan Williams return ERR_PTR(-EEXIST); 10633565fce3SDan Williams 10643565fce3SDan Williams pfn += (addr & ~PMD_MASK) >> PAGE_SHIFT; 10653565fce3SDan Williams pgmap = get_dev_pagemap(pfn, NULL); 10663565fce3SDan Williams if (!pgmap) 10673565fce3SDan Williams return ERR_PTR(-EFAULT); 10683565fce3SDan Williams page = pfn_to_page(pfn); 10693565fce3SDan Williams get_page(page); 10703565fce3SDan Williams put_dev_pagemap(pgmap); 10713565fce3SDan Williams 10723565fce3SDan Williams return page; 10733565fce3SDan Williams } 10743565fce3SDan Williams 107571e3aac0SAndrea Arcangeli int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, 107671e3aac0SAndrea Arcangeli pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr, 107771e3aac0SAndrea Arcangeli struct vm_area_struct *vma) 107871e3aac0SAndrea Arcangeli { 1079c4088ebdSKirill A. Shutemov spinlock_t *dst_ptl, *src_ptl; 108071e3aac0SAndrea Arcangeli struct page *src_page; 108171e3aac0SAndrea Arcangeli pmd_t pmd; 108212c9d70bSMatthew Wilcox pgtable_t pgtable = NULL; 108371e3aac0SAndrea Arcangeli int ret; 108471e3aac0SAndrea Arcangeli 108512c9d70bSMatthew Wilcox if (!vma_is_dax(vma)) { 108671e3aac0SAndrea Arcangeli ret = -ENOMEM; 108771e3aac0SAndrea Arcangeli pgtable = pte_alloc_one(dst_mm, addr); 108871e3aac0SAndrea Arcangeli if (unlikely(!pgtable)) 108971e3aac0SAndrea Arcangeli goto out; 109012c9d70bSMatthew Wilcox } 109171e3aac0SAndrea Arcangeli 1092c4088ebdSKirill A. Shutemov dst_ptl = pmd_lock(dst_mm, dst_pmd); 1093c4088ebdSKirill A. Shutemov src_ptl = pmd_lockptr(src_mm, src_pmd); 1094c4088ebdSKirill A. Shutemov spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 109571e3aac0SAndrea Arcangeli 109671e3aac0SAndrea Arcangeli ret = -EAGAIN; 109771e3aac0SAndrea Arcangeli pmd = *src_pmd; 10985c7fb56eSDan Williams if (unlikely(!pmd_trans_huge(pmd) && !pmd_devmap(pmd))) { 109971e3aac0SAndrea Arcangeli pte_free(dst_mm, pgtable); 110071e3aac0SAndrea Arcangeli goto out_unlock; 110171e3aac0SAndrea Arcangeli } 1102fc9fe822SKirill A. Shutemov /* 1103c4088ebdSKirill A. Shutemov * When page table lock is held, the huge zero pmd should not be 1104fc9fe822SKirill A. Shutemov * under splitting since we don't split the page itself, only pmd to 1105fc9fe822SKirill A. Shutemov * a page table. 1106fc9fe822SKirill A. Shutemov */ 1107fc9fe822SKirill A. Shutemov if (is_huge_zero_pmd(pmd)) { 11085918d10aSKirill A. Shutemov struct page *zero_page; 110997ae1749SKirill A. Shutemov /* 111097ae1749SKirill A. Shutemov * get_huge_zero_page() will never allocate a new page here, 111197ae1749SKirill A. Shutemov * since we already have a zero page to copy. It just takes a 111297ae1749SKirill A. Shutemov * reference. 111397ae1749SKirill A. Shutemov */ 11145918d10aSKirill A. Shutemov zero_page = get_huge_zero_page(); 11156b251fc9SAndrea Arcangeli set_huge_zero_page(pgtable, dst_mm, vma, addr, dst_pmd, 11165918d10aSKirill A. Shutemov zero_page); 1117fc9fe822SKirill A. Shutemov ret = 0; 1118fc9fe822SKirill A. Shutemov goto out_unlock; 1119fc9fe822SKirill A. Shutemov } 1120de466bd6SMel Gorman 112112c9d70bSMatthew Wilcox if (!vma_is_dax(vma)) { 11225c7fb56eSDan Williams /* thp accounting separate from pmd_devmap accounting */ 112371e3aac0SAndrea Arcangeli src_page = pmd_page(pmd); 1124309381feSSasha Levin VM_BUG_ON_PAGE(!PageHead(src_page), src_page); 112571e3aac0SAndrea Arcangeli get_page(src_page); 112653f9263bSKirill A. Shutemov page_dup_rmap(src_page, true); 112771e3aac0SAndrea Arcangeli add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); 11285c7fb56eSDan Williams atomic_long_inc(&dst_mm->nr_ptes); 11295c7fb56eSDan Williams pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); 11305c7fb56eSDan Williams } 113171e3aac0SAndrea Arcangeli 113271e3aac0SAndrea Arcangeli pmdp_set_wrprotect(src_mm, addr, src_pmd); 113371e3aac0SAndrea Arcangeli pmd = pmd_mkold(pmd_wrprotect(pmd)); 113471e3aac0SAndrea Arcangeli set_pmd_at(dst_mm, addr, dst_pmd, pmd); 113571e3aac0SAndrea Arcangeli 113671e3aac0SAndrea Arcangeli ret = 0; 113771e3aac0SAndrea Arcangeli out_unlock: 1138c4088ebdSKirill A. Shutemov spin_unlock(src_ptl); 1139c4088ebdSKirill A. Shutemov spin_unlock(dst_ptl); 114071e3aac0SAndrea Arcangeli out: 114171e3aac0SAndrea Arcangeli return ret; 114271e3aac0SAndrea Arcangeli } 114371e3aac0SAndrea Arcangeli 1144a1dd450bSWill Deacon void huge_pmd_set_accessed(struct mm_struct *mm, 1145a1dd450bSWill Deacon struct vm_area_struct *vma, 1146a1dd450bSWill Deacon unsigned long address, 1147a1dd450bSWill Deacon pmd_t *pmd, pmd_t orig_pmd, 1148a1dd450bSWill Deacon int dirty) 1149a1dd450bSWill Deacon { 1150c4088ebdSKirill A. Shutemov spinlock_t *ptl; 1151a1dd450bSWill Deacon pmd_t entry; 1152a1dd450bSWill Deacon unsigned long haddr; 1153a1dd450bSWill Deacon 1154c4088ebdSKirill A. Shutemov ptl = pmd_lock(mm, pmd); 1155a1dd450bSWill Deacon if (unlikely(!pmd_same(*pmd, orig_pmd))) 1156a1dd450bSWill Deacon goto unlock; 1157a1dd450bSWill Deacon 1158a1dd450bSWill Deacon entry = pmd_mkyoung(orig_pmd); 1159a1dd450bSWill Deacon haddr = address & HPAGE_PMD_MASK; 1160a1dd450bSWill Deacon if (pmdp_set_access_flags(vma, haddr, pmd, entry, dirty)) 1161a1dd450bSWill Deacon update_mmu_cache_pmd(vma, address, pmd); 1162a1dd450bSWill Deacon 1163a1dd450bSWill Deacon unlock: 1164c4088ebdSKirill A. Shutemov spin_unlock(ptl); 1165a1dd450bSWill Deacon } 1166a1dd450bSWill Deacon 116771e3aac0SAndrea Arcangeli static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm, 116871e3aac0SAndrea Arcangeli struct vm_area_struct *vma, 116971e3aac0SAndrea Arcangeli unsigned long address, 117071e3aac0SAndrea Arcangeli pmd_t *pmd, pmd_t orig_pmd, 117171e3aac0SAndrea Arcangeli struct page *page, 117271e3aac0SAndrea Arcangeli unsigned long haddr) 117371e3aac0SAndrea Arcangeli { 117400501b53SJohannes Weiner struct mem_cgroup *memcg; 1175c4088ebdSKirill A. Shutemov spinlock_t *ptl; 117671e3aac0SAndrea Arcangeli pgtable_t pgtable; 117771e3aac0SAndrea Arcangeli pmd_t _pmd; 117871e3aac0SAndrea Arcangeli int ret = 0, i; 117971e3aac0SAndrea Arcangeli struct page **pages; 11802ec74c3eSSagi Grimberg unsigned long mmun_start; /* For mmu_notifiers */ 11812ec74c3eSSagi Grimberg unsigned long mmun_end; /* For mmu_notifiers */ 118271e3aac0SAndrea Arcangeli 118371e3aac0SAndrea Arcangeli pages = kmalloc(sizeof(struct page *) * HPAGE_PMD_NR, 118471e3aac0SAndrea Arcangeli GFP_KERNEL); 118571e3aac0SAndrea Arcangeli if (unlikely(!pages)) { 118671e3aac0SAndrea Arcangeli ret |= VM_FAULT_OOM; 118771e3aac0SAndrea Arcangeli goto out; 118871e3aac0SAndrea Arcangeli } 118971e3aac0SAndrea Arcangeli 119071e3aac0SAndrea Arcangeli for (i = 0; i < HPAGE_PMD_NR; i++) { 1191cc5d462fSAndi Kleen pages[i] = alloc_page_vma_node(GFP_HIGHUSER_MOVABLE | 1192cc5d462fSAndi Kleen __GFP_OTHER_NODE, 119319ee151eSAndi Kleen vma, address, page_to_nid(page)); 1194b9bbfbe3SAndrea Arcangeli if (unlikely(!pages[i] || 119500501b53SJohannes Weiner mem_cgroup_try_charge(pages[i], mm, GFP_KERNEL, 1196f627c2f5SKirill A. Shutemov &memcg, false))) { 1197b9bbfbe3SAndrea Arcangeli if (pages[i]) 119871e3aac0SAndrea Arcangeli put_page(pages[i]); 1199b9bbfbe3SAndrea Arcangeli while (--i >= 0) { 120000501b53SJohannes Weiner memcg = (void *)page_private(pages[i]); 120100501b53SJohannes Weiner set_page_private(pages[i], 0); 1202f627c2f5SKirill A. Shutemov mem_cgroup_cancel_charge(pages[i], memcg, 1203f627c2f5SKirill A. Shutemov false); 1204b9bbfbe3SAndrea Arcangeli put_page(pages[i]); 1205b9bbfbe3SAndrea Arcangeli } 120671e3aac0SAndrea Arcangeli kfree(pages); 120771e3aac0SAndrea Arcangeli ret |= VM_FAULT_OOM; 120871e3aac0SAndrea Arcangeli goto out; 120971e3aac0SAndrea Arcangeli } 121000501b53SJohannes Weiner set_page_private(pages[i], (unsigned long)memcg); 121171e3aac0SAndrea Arcangeli } 121271e3aac0SAndrea Arcangeli 121371e3aac0SAndrea Arcangeli for (i = 0; i < HPAGE_PMD_NR; i++) { 121471e3aac0SAndrea Arcangeli copy_user_highpage(pages[i], page + i, 12150089e485SHillf Danton haddr + PAGE_SIZE * i, vma); 121671e3aac0SAndrea Arcangeli __SetPageUptodate(pages[i]); 121771e3aac0SAndrea Arcangeli cond_resched(); 121871e3aac0SAndrea Arcangeli } 121971e3aac0SAndrea Arcangeli 12202ec74c3eSSagi Grimberg mmun_start = haddr; 12212ec74c3eSSagi Grimberg mmun_end = haddr + HPAGE_PMD_SIZE; 12222ec74c3eSSagi Grimberg mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 12232ec74c3eSSagi Grimberg 1224c4088ebdSKirill A. Shutemov ptl = pmd_lock(mm, pmd); 122571e3aac0SAndrea Arcangeli if (unlikely(!pmd_same(*pmd, orig_pmd))) 122671e3aac0SAndrea Arcangeli goto out_free_pages; 1227309381feSSasha Levin VM_BUG_ON_PAGE(!PageHead(page), page); 122871e3aac0SAndrea Arcangeli 12298809aa2dSAneesh Kumar K.V pmdp_huge_clear_flush_notify(vma, haddr, pmd); 123071e3aac0SAndrea Arcangeli /* leave pmd empty until pte is filled */ 123171e3aac0SAndrea Arcangeli 12326b0b50b0SAneesh Kumar K.V pgtable = pgtable_trans_huge_withdraw(mm, pmd); 123371e3aac0SAndrea Arcangeli pmd_populate(mm, &_pmd, pgtable); 123471e3aac0SAndrea Arcangeli 123571e3aac0SAndrea Arcangeli for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) { 123671e3aac0SAndrea Arcangeli pte_t *pte, entry; 123771e3aac0SAndrea Arcangeli entry = mk_pte(pages[i], vma->vm_page_prot); 123871e3aac0SAndrea Arcangeli entry = maybe_mkwrite(pte_mkdirty(entry), vma); 123900501b53SJohannes Weiner memcg = (void *)page_private(pages[i]); 124000501b53SJohannes Weiner set_page_private(pages[i], 0); 1241d281ee61SKirill A. Shutemov page_add_new_anon_rmap(pages[i], vma, haddr, false); 1242f627c2f5SKirill A. Shutemov mem_cgroup_commit_charge(pages[i], memcg, false, false); 124300501b53SJohannes Weiner lru_cache_add_active_or_unevictable(pages[i], vma); 124471e3aac0SAndrea Arcangeli pte = pte_offset_map(&_pmd, haddr); 124571e3aac0SAndrea Arcangeli VM_BUG_ON(!pte_none(*pte)); 124671e3aac0SAndrea Arcangeli set_pte_at(mm, haddr, pte, entry); 124771e3aac0SAndrea Arcangeli pte_unmap(pte); 124871e3aac0SAndrea Arcangeli } 124971e3aac0SAndrea Arcangeli kfree(pages); 125071e3aac0SAndrea Arcangeli 125171e3aac0SAndrea Arcangeli smp_wmb(); /* make pte visible before pmd */ 125271e3aac0SAndrea Arcangeli pmd_populate(mm, pmd, pgtable); 1253d281ee61SKirill A. Shutemov page_remove_rmap(page, true); 1254c4088ebdSKirill A. Shutemov spin_unlock(ptl); 125571e3aac0SAndrea Arcangeli 12562ec74c3eSSagi Grimberg mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 12572ec74c3eSSagi Grimberg 125871e3aac0SAndrea Arcangeli ret |= VM_FAULT_WRITE; 125971e3aac0SAndrea Arcangeli put_page(page); 126071e3aac0SAndrea Arcangeli 126171e3aac0SAndrea Arcangeli out: 126271e3aac0SAndrea Arcangeli return ret; 126371e3aac0SAndrea Arcangeli 126471e3aac0SAndrea Arcangeli out_free_pages: 1265c4088ebdSKirill A. Shutemov spin_unlock(ptl); 12662ec74c3eSSagi Grimberg mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 1267b9bbfbe3SAndrea Arcangeli for (i = 0; i < HPAGE_PMD_NR; i++) { 126800501b53SJohannes Weiner memcg = (void *)page_private(pages[i]); 126900501b53SJohannes Weiner set_page_private(pages[i], 0); 1270f627c2f5SKirill A. Shutemov mem_cgroup_cancel_charge(pages[i], memcg, false); 127171e3aac0SAndrea Arcangeli put_page(pages[i]); 1272b9bbfbe3SAndrea Arcangeli } 127371e3aac0SAndrea Arcangeli kfree(pages); 127471e3aac0SAndrea Arcangeli goto out; 127571e3aac0SAndrea Arcangeli } 127671e3aac0SAndrea Arcangeli 127771e3aac0SAndrea Arcangeli int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma, 127871e3aac0SAndrea Arcangeli unsigned long address, pmd_t *pmd, pmd_t orig_pmd) 127971e3aac0SAndrea Arcangeli { 1280c4088ebdSKirill A. Shutemov spinlock_t *ptl; 128171e3aac0SAndrea Arcangeli int ret = 0; 128293b4796dSKirill A. Shutemov struct page *page = NULL, *new_page; 128300501b53SJohannes Weiner struct mem_cgroup *memcg; 128471e3aac0SAndrea Arcangeli unsigned long haddr; 12852ec74c3eSSagi Grimberg unsigned long mmun_start; /* For mmu_notifiers */ 12862ec74c3eSSagi Grimberg unsigned long mmun_end; /* For mmu_notifiers */ 12873b363692SMichal Hocko gfp_t huge_gfp; /* for allocation and charge */ 128871e3aac0SAndrea Arcangeli 1289c4088ebdSKirill A. Shutemov ptl = pmd_lockptr(mm, pmd); 129081d1b09cSSasha Levin VM_BUG_ON_VMA(!vma->anon_vma, vma); 129193b4796dSKirill A. Shutemov haddr = address & HPAGE_PMD_MASK; 129293b4796dSKirill A. Shutemov if (is_huge_zero_pmd(orig_pmd)) 129393b4796dSKirill A. Shutemov goto alloc; 1294c4088ebdSKirill A. Shutemov spin_lock(ptl); 129571e3aac0SAndrea Arcangeli if (unlikely(!pmd_same(*pmd, orig_pmd))) 129671e3aac0SAndrea Arcangeli goto out_unlock; 129771e3aac0SAndrea Arcangeli 129871e3aac0SAndrea Arcangeli page = pmd_page(orig_pmd); 1299309381feSSasha Levin VM_BUG_ON_PAGE(!PageCompound(page) || !PageHead(page), page); 13001f25fe20SKirill A. Shutemov /* 13011f25fe20SKirill A. Shutemov * We can only reuse the page if nobody else maps the huge page or it's 13026d0a07edSAndrea Arcangeli * part. 13031f25fe20SKirill A. Shutemov */ 13046d0a07edSAndrea Arcangeli if (page_trans_huge_mapcount(page, NULL) == 1) { 130571e3aac0SAndrea Arcangeli pmd_t entry; 130671e3aac0SAndrea Arcangeli entry = pmd_mkyoung(orig_pmd); 130771e3aac0SAndrea Arcangeli entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 130871e3aac0SAndrea Arcangeli if (pmdp_set_access_flags(vma, haddr, pmd, entry, 1)) 1309b113da65SDavid Miller update_mmu_cache_pmd(vma, address, pmd); 131071e3aac0SAndrea Arcangeli ret |= VM_FAULT_WRITE; 131171e3aac0SAndrea Arcangeli goto out_unlock; 131271e3aac0SAndrea Arcangeli } 1313ddc58f27SKirill A. Shutemov get_page(page); 1314c4088ebdSKirill A. Shutemov spin_unlock(ptl); 131593b4796dSKirill A. Shutemov alloc: 131671e3aac0SAndrea Arcangeli if (transparent_hugepage_enabled(vma) && 1317077fcf11SAneesh Kumar K.V !transparent_hugepage_debug_cow()) { 1318444eb2a4SMel Gorman huge_gfp = alloc_hugepage_direct_gfpmask(vma); 13193b363692SMichal Hocko new_page = alloc_hugepage_vma(huge_gfp, vma, haddr, HPAGE_PMD_ORDER); 1320077fcf11SAneesh Kumar K.V } else 132171e3aac0SAndrea Arcangeli new_page = NULL; 132271e3aac0SAndrea Arcangeli 13239a982250SKirill A. Shutemov if (likely(new_page)) { 13249a982250SKirill A. Shutemov prep_transhuge_page(new_page); 13259a982250SKirill A. Shutemov } else { 1326eecc1e42SHugh Dickins if (!page) { 132778ddc534SKirill A. Shutemov split_huge_pmd(vma, pmd, address); 1328e9b71ca9SKirill A. Shutemov ret |= VM_FAULT_FALLBACK; 132993b4796dSKirill A. Shutemov } else { 133071e3aac0SAndrea Arcangeli ret = do_huge_pmd_wp_page_fallback(mm, vma, address, 133171e3aac0SAndrea Arcangeli pmd, orig_pmd, page, haddr); 13329845cbbdSKirill A. Shutemov if (ret & VM_FAULT_OOM) { 133378ddc534SKirill A. Shutemov split_huge_pmd(vma, pmd, address); 13349845cbbdSKirill A. Shutemov ret |= VM_FAULT_FALLBACK; 13359845cbbdSKirill A. Shutemov } 1336ddc58f27SKirill A. Shutemov put_page(page); 133793b4796dSKirill A. Shutemov } 133817766ddeSDavid Rientjes count_vm_event(THP_FAULT_FALLBACK); 133971e3aac0SAndrea Arcangeli goto out; 134071e3aac0SAndrea Arcangeli } 134171e3aac0SAndrea Arcangeli 1342f627c2f5SKirill A. Shutemov if (unlikely(mem_cgroup_try_charge(new_page, mm, huge_gfp, &memcg, 1343f627c2f5SKirill A. Shutemov true))) { 1344b9bbfbe3SAndrea Arcangeli put_page(new_page); 134593b4796dSKirill A. Shutemov if (page) { 134678ddc534SKirill A. Shutemov split_huge_pmd(vma, pmd, address); 1347ddc58f27SKirill A. Shutemov put_page(page); 13489845cbbdSKirill A. Shutemov } else 134978ddc534SKirill A. Shutemov split_huge_pmd(vma, pmd, address); 13509845cbbdSKirill A. Shutemov ret |= VM_FAULT_FALLBACK; 135117766ddeSDavid Rientjes count_vm_event(THP_FAULT_FALLBACK); 1352b9bbfbe3SAndrea Arcangeli goto out; 1353b9bbfbe3SAndrea Arcangeli } 1354b9bbfbe3SAndrea Arcangeli 135517766ddeSDavid Rientjes count_vm_event(THP_FAULT_ALLOC); 135617766ddeSDavid Rientjes 1357eecc1e42SHugh Dickins if (!page) 135893b4796dSKirill A. Shutemov clear_huge_page(new_page, haddr, HPAGE_PMD_NR); 135993b4796dSKirill A. Shutemov else 136071e3aac0SAndrea Arcangeli copy_user_huge_page(new_page, page, haddr, vma, HPAGE_PMD_NR); 136171e3aac0SAndrea Arcangeli __SetPageUptodate(new_page); 136271e3aac0SAndrea Arcangeli 13632ec74c3eSSagi Grimberg mmun_start = haddr; 13642ec74c3eSSagi Grimberg mmun_end = haddr + HPAGE_PMD_SIZE; 13652ec74c3eSSagi Grimberg mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 13662ec74c3eSSagi Grimberg 1367c4088ebdSKirill A. Shutemov spin_lock(ptl); 136893b4796dSKirill A. Shutemov if (page) 1369ddc58f27SKirill A. Shutemov put_page(page); 1370b9bbfbe3SAndrea Arcangeli if (unlikely(!pmd_same(*pmd, orig_pmd))) { 1371c4088ebdSKirill A. Shutemov spin_unlock(ptl); 1372f627c2f5SKirill A. Shutemov mem_cgroup_cancel_charge(new_page, memcg, true); 137371e3aac0SAndrea Arcangeli put_page(new_page); 13742ec74c3eSSagi Grimberg goto out_mn; 1375b9bbfbe3SAndrea Arcangeli } else { 137671e3aac0SAndrea Arcangeli pmd_t entry; 13773122359aSKirill A. Shutemov entry = mk_huge_pmd(new_page, vma->vm_page_prot); 13783122359aSKirill A. Shutemov entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 13798809aa2dSAneesh Kumar K.V pmdp_huge_clear_flush_notify(vma, haddr, pmd); 1380d281ee61SKirill A. Shutemov page_add_new_anon_rmap(new_page, vma, haddr, true); 1381f627c2f5SKirill A. Shutemov mem_cgroup_commit_charge(new_page, memcg, false, true); 138200501b53SJohannes Weiner lru_cache_add_active_or_unevictable(new_page, vma); 138371e3aac0SAndrea Arcangeli set_pmd_at(mm, haddr, pmd, entry); 1384b113da65SDavid Miller update_mmu_cache_pmd(vma, address, pmd); 1385eecc1e42SHugh Dickins if (!page) { 138693b4796dSKirill A. Shutemov add_mm_counter(mm, MM_ANONPAGES, HPAGE_PMD_NR); 138797ae1749SKirill A. Shutemov put_huge_zero_page(); 138897ae1749SKirill A. Shutemov } else { 1389309381feSSasha Levin VM_BUG_ON_PAGE(!PageHead(page), page); 1390d281ee61SKirill A. Shutemov page_remove_rmap(page, true); 139171e3aac0SAndrea Arcangeli put_page(page); 139293b4796dSKirill A. Shutemov } 139371e3aac0SAndrea Arcangeli ret |= VM_FAULT_WRITE; 139471e3aac0SAndrea Arcangeli } 1395c4088ebdSKirill A. Shutemov spin_unlock(ptl); 13962ec74c3eSSagi Grimberg out_mn: 13972ec74c3eSSagi Grimberg mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 13982ec74c3eSSagi Grimberg out: 13992ec74c3eSSagi Grimberg return ret; 140071e3aac0SAndrea Arcangeli out_unlock: 1401c4088ebdSKirill A. Shutemov spin_unlock(ptl); 140271e3aac0SAndrea Arcangeli return ret; 140371e3aac0SAndrea Arcangeli } 140471e3aac0SAndrea Arcangeli 1405b676b293SDavid Rientjes struct page *follow_trans_huge_pmd(struct vm_area_struct *vma, 140671e3aac0SAndrea Arcangeli unsigned long addr, 140771e3aac0SAndrea Arcangeli pmd_t *pmd, 140871e3aac0SAndrea Arcangeli unsigned int flags) 140971e3aac0SAndrea Arcangeli { 1410b676b293SDavid Rientjes struct mm_struct *mm = vma->vm_mm; 141171e3aac0SAndrea Arcangeli struct page *page = NULL; 141271e3aac0SAndrea Arcangeli 1413c4088ebdSKirill A. Shutemov assert_spin_locked(pmd_lockptr(mm, pmd)); 141471e3aac0SAndrea Arcangeli 141571e3aac0SAndrea Arcangeli if (flags & FOLL_WRITE && !pmd_write(*pmd)) 141671e3aac0SAndrea Arcangeli goto out; 141771e3aac0SAndrea Arcangeli 141885facf25SKirill A. Shutemov /* Avoid dumping huge zero page */ 141985facf25SKirill A. Shutemov if ((flags & FOLL_DUMP) && is_huge_zero_pmd(*pmd)) 142085facf25SKirill A. Shutemov return ERR_PTR(-EFAULT); 142185facf25SKirill A. Shutemov 14222b4847e7SMel Gorman /* Full NUMA hinting faults to serialise migration in fault paths */ 14238a0516edSMel Gorman if ((flags & FOLL_NUMA) && pmd_protnone(*pmd)) 14242b4847e7SMel Gorman goto out; 14252b4847e7SMel Gorman 142671e3aac0SAndrea Arcangeli page = pmd_page(*pmd); 1427309381feSSasha Levin VM_BUG_ON_PAGE(!PageHead(page), page); 14283565fce3SDan Williams if (flags & FOLL_TOUCH) 14293565fce3SDan Williams touch_pmd(vma, addr, pmd); 1430de60f5f1SEric B Munson if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) { 1431e90309c9SKirill A. Shutemov /* 1432e90309c9SKirill A. Shutemov * We don't mlock() pte-mapped THPs. This way we can avoid 1433e90309c9SKirill A. Shutemov * leaking mlocked pages into non-VM_LOCKED VMAs. 1434e90309c9SKirill A. Shutemov * 1435e90309c9SKirill A. Shutemov * In most cases the pmd is the only mapping of the page as we 1436e90309c9SKirill A. Shutemov * break COW for the mlock() -- see gup_flags |= FOLL_WRITE for 1437e90309c9SKirill A. Shutemov * writable private mappings in populate_vma_page_range(). 1438e90309c9SKirill A. Shutemov * 1439e90309c9SKirill A. Shutemov * The only scenario when we have the page shared here is if we 1440e90309c9SKirill A. Shutemov * mlocking read-only mapping shared over fork(). We skip 1441e90309c9SKirill A. Shutemov * mlocking such pages. 1442e90309c9SKirill A. Shutemov */ 1443e90309c9SKirill A. Shutemov if (compound_mapcount(page) == 1 && !PageDoubleMap(page) && 1444e90309c9SKirill A. Shutemov page->mapping && trylock_page(page)) { 1445b676b293SDavid Rientjes lru_add_drain(); 1446b676b293SDavid Rientjes if (page->mapping) 1447b676b293SDavid Rientjes mlock_vma_page(page); 1448b676b293SDavid Rientjes unlock_page(page); 1449b676b293SDavid Rientjes } 1450b676b293SDavid Rientjes } 145171e3aac0SAndrea Arcangeli page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT; 1452309381feSSasha Levin VM_BUG_ON_PAGE(!PageCompound(page), page); 145371e3aac0SAndrea Arcangeli if (flags & FOLL_GET) 1454ddc58f27SKirill A. Shutemov get_page(page); 145571e3aac0SAndrea Arcangeli 145671e3aac0SAndrea Arcangeli out: 145771e3aac0SAndrea Arcangeli return page; 145871e3aac0SAndrea Arcangeli } 145971e3aac0SAndrea Arcangeli 1460d10e63f2SMel Gorman /* NUMA hinting page fault entry point for trans huge pmds */ 14614daae3b4SMel Gorman int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma, 14624daae3b4SMel Gorman unsigned long addr, pmd_t pmd, pmd_t *pmdp) 1463d10e63f2SMel Gorman { 1464c4088ebdSKirill A. Shutemov spinlock_t *ptl; 1465b8916634SMel Gorman struct anon_vma *anon_vma = NULL; 1466b32967ffSMel Gorman struct page *page; 1467d10e63f2SMel Gorman unsigned long haddr = addr & HPAGE_PMD_MASK; 14688191acbdSMel Gorman int page_nid = -1, this_nid = numa_node_id(); 146990572890SPeter Zijlstra int target_nid, last_cpupid = -1; 14708191acbdSMel Gorman bool page_locked; 14718191acbdSMel Gorman bool migrated = false; 1472b191f9b1SMel Gorman bool was_writable; 14736688cc05SPeter Zijlstra int flags = 0; 1474d10e63f2SMel Gorman 1475c0e7cad9SMel Gorman /* A PROT_NONE fault should not end up here */ 1476c0e7cad9SMel Gorman BUG_ON(!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))); 1477c0e7cad9SMel Gorman 1478c4088ebdSKirill A. Shutemov ptl = pmd_lock(mm, pmdp); 1479d10e63f2SMel Gorman if (unlikely(!pmd_same(pmd, *pmdp))) 1480d10e63f2SMel Gorman goto out_unlock; 1481d10e63f2SMel Gorman 1482de466bd6SMel Gorman /* 1483de466bd6SMel Gorman * If there are potential migrations, wait for completion and retry 1484de466bd6SMel Gorman * without disrupting NUMA hinting information. Do not relock and 1485de466bd6SMel Gorman * check_same as the page may no longer be mapped. 1486de466bd6SMel Gorman */ 1487de466bd6SMel Gorman if (unlikely(pmd_trans_migrating(*pmdp))) { 14885d833062SMel Gorman page = pmd_page(*pmdp); 1489de466bd6SMel Gorman spin_unlock(ptl); 14905d833062SMel Gorman wait_on_page_locked(page); 1491de466bd6SMel Gorman goto out; 1492de466bd6SMel Gorman } 1493de466bd6SMel Gorman 1494d10e63f2SMel Gorman page = pmd_page(pmd); 1495a1a46184SMel Gorman BUG_ON(is_huge_zero_page(page)); 14968191acbdSMel Gorman page_nid = page_to_nid(page); 149790572890SPeter Zijlstra last_cpupid = page_cpupid_last(page); 149803c5a6e1SMel Gorman count_vm_numa_event(NUMA_HINT_FAULTS); 149904bb2f94SRik van Riel if (page_nid == this_nid) { 150003c5a6e1SMel Gorman count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL); 150104bb2f94SRik van Riel flags |= TNF_FAULT_LOCAL; 150204bb2f94SRik van Riel } 15034daae3b4SMel Gorman 1504bea66fbdSMel Gorman /* See similar comment in do_numa_page for explanation */ 1505bea66fbdSMel Gorman if (!(vma->vm_flags & VM_WRITE)) 15066688cc05SPeter Zijlstra flags |= TNF_NO_GROUP; 15076688cc05SPeter Zijlstra 15086688cc05SPeter Zijlstra /* 1509ff9042b1SMel Gorman * Acquire the page lock to serialise THP migrations but avoid dropping 1510ff9042b1SMel Gorman * page_table_lock if at all possible 1511ff9042b1SMel Gorman */ 1512b8916634SMel Gorman page_locked = trylock_page(page); 1513b8916634SMel Gorman target_nid = mpol_misplaced(page, vma, haddr); 1514b8916634SMel Gorman if (target_nid == -1) { 1515b8916634SMel Gorman /* If the page was locked, there are no parallel migrations */ 1516a54a407fSMel Gorman if (page_locked) 1517b8916634SMel Gorman goto clear_pmdnuma; 15182b4847e7SMel Gorman } 1519cbee9f88SPeter Zijlstra 1520de466bd6SMel Gorman /* Migration could have started since the pmd_trans_migrating check */ 15212b4847e7SMel Gorman if (!page_locked) { 1522c4088ebdSKirill A. Shutemov spin_unlock(ptl); 1523b8916634SMel Gorman wait_on_page_locked(page); 1524a54a407fSMel Gorman page_nid = -1; 1525b8916634SMel Gorman goto out; 1526b8916634SMel Gorman } 1527b8916634SMel Gorman 15282b4847e7SMel Gorman /* 15292b4847e7SMel Gorman * Page is misplaced. Page lock serialises migrations. Acquire anon_vma 15302b4847e7SMel Gorman * to serialises splits 15312b4847e7SMel Gorman */ 1532b8916634SMel Gorman get_page(page); 1533c4088ebdSKirill A. Shutemov spin_unlock(ptl); 1534b8916634SMel Gorman anon_vma = page_lock_anon_vma_read(page); 1535b32967ffSMel Gorman 1536c69307d5SPeter Zijlstra /* Confirm the PMD did not change while page_table_lock was released */ 1537c4088ebdSKirill A. Shutemov spin_lock(ptl); 1538b32967ffSMel Gorman if (unlikely(!pmd_same(pmd, *pmdp))) { 1539b32967ffSMel Gorman unlock_page(page); 1540b32967ffSMel Gorman put_page(page); 1541a54a407fSMel Gorman page_nid = -1; 1542b32967ffSMel Gorman goto out_unlock; 1543b32967ffSMel Gorman } 1544ff9042b1SMel Gorman 1545c3a489caSMel Gorman /* Bail if we fail to protect against THP splits for any reason */ 1546c3a489caSMel Gorman if (unlikely(!anon_vma)) { 1547c3a489caSMel Gorman put_page(page); 1548c3a489caSMel Gorman page_nid = -1; 1549c3a489caSMel Gorman goto clear_pmdnuma; 1550c3a489caSMel Gorman } 1551c3a489caSMel Gorman 1552a54a407fSMel Gorman /* 1553a54a407fSMel Gorman * Migrate the THP to the requested node, returns with page unlocked 15548a0516edSMel Gorman * and access rights restored. 1555a54a407fSMel Gorman */ 1556c4088ebdSKirill A. Shutemov spin_unlock(ptl); 1557b32967ffSMel Gorman migrated = migrate_misplaced_transhuge_page(mm, vma, 1558340ef390SHugh Dickins pmdp, pmd, addr, page, target_nid); 15596688cc05SPeter Zijlstra if (migrated) { 15606688cc05SPeter Zijlstra flags |= TNF_MIGRATED; 15618191acbdSMel Gorman page_nid = target_nid; 1562074c2381SMel Gorman } else 1563074c2381SMel Gorman flags |= TNF_MIGRATE_FAIL; 1564b32967ffSMel Gorman 15658191acbdSMel Gorman goto out; 15664daae3b4SMel Gorman clear_pmdnuma: 1567a54a407fSMel Gorman BUG_ON(!PageLocked(page)); 1568b191f9b1SMel Gorman was_writable = pmd_write(pmd); 15694d942466SMel Gorman pmd = pmd_modify(pmd, vma->vm_page_prot); 1570b7b04004SMel Gorman pmd = pmd_mkyoung(pmd); 1571b191f9b1SMel Gorman if (was_writable) 1572b191f9b1SMel Gorman pmd = pmd_mkwrite(pmd); 1573d10e63f2SMel Gorman set_pmd_at(mm, haddr, pmdp, pmd); 1574d10e63f2SMel Gorman update_mmu_cache_pmd(vma, addr, pmdp); 1575a54a407fSMel Gorman unlock_page(page); 1576d10e63f2SMel Gorman out_unlock: 1577c4088ebdSKirill A. Shutemov spin_unlock(ptl); 1578b8916634SMel Gorman 1579b8916634SMel Gorman out: 1580b8916634SMel Gorman if (anon_vma) 1581b8916634SMel Gorman page_unlock_anon_vma_read(anon_vma); 1582b8916634SMel Gorman 15838191acbdSMel Gorman if (page_nid != -1) 15846688cc05SPeter Zijlstra task_numa_fault(last_cpupid, page_nid, HPAGE_PMD_NR, flags); 15858191acbdSMel Gorman 1586d10e63f2SMel Gorman return 0; 1587d10e63f2SMel Gorman } 1588d10e63f2SMel Gorman 1589b8d3c4c3SMinchan Kim int madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 1590b8d3c4c3SMinchan Kim pmd_t *pmd, unsigned long addr, unsigned long next) 1591b8d3c4c3SMinchan Kim 1592b8d3c4c3SMinchan Kim { 1593b8d3c4c3SMinchan Kim spinlock_t *ptl; 1594b8d3c4c3SMinchan Kim pmd_t orig_pmd; 1595b8d3c4c3SMinchan Kim struct page *page; 1596b8d3c4c3SMinchan Kim struct mm_struct *mm = tlb->mm; 1597b8d3c4c3SMinchan Kim int ret = 0; 1598b8d3c4c3SMinchan Kim 1599b6ec57f4SKirill A. Shutemov ptl = pmd_trans_huge_lock(pmd, vma); 1600b6ec57f4SKirill A. Shutemov if (!ptl) 160125eedabeSLinus Torvalds goto out_unlocked; 1602b8d3c4c3SMinchan Kim 1603b8d3c4c3SMinchan Kim orig_pmd = *pmd; 1604b8d3c4c3SMinchan Kim if (is_huge_zero_pmd(orig_pmd)) { 1605b8d3c4c3SMinchan Kim ret = 1; 1606b8d3c4c3SMinchan Kim goto out; 1607b8d3c4c3SMinchan Kim } 1608b8d3c4c3SMinchan Kim 1609b8d3c4c3SMinchan Kim page = pmd_page(orig_pmd); 1610b8d3c4c3SMinchan Kim /* 1611b8d3c4c3SMinchan Kim * If other processes are mapping this page, we couldn't discard 1612b8d3c4c3SMinchan Kim * the page unless they all do MADV_FREE so let's skip the page. 1613b8d3c4c3SMinchan Kim */ 1614b8d3c4c3SMinchan Kim if (page_mapcount(page) != 1) 1615b8d3c4c3SMinchan Kim goto out; 1616b8d3c4c3SMinchan Kim 1617b8d3c4c3SMinchan Kim if (!trylock_page(page)) 1618b8d3c4c3SMinchan Kim goto out; 1619b8d3c4c3SMinchan Kim 1620b8d3c4c3SMinchan Kim /* 1621b8d3c4c3SMinchan Kim * If user want to discard part-pages of THP, split it so MADV_FREE 1622b8d3c4c3SMinchan Kim * will deactivate only them. 1623b8d3c4c3SMinchan Kim */ 1624b8d3c4c3SMinchan Kim if (next - addr != HPAGE_PMD_SIZE) { 1625b8d3c4c3SMinchan Kim get_page(page); 1626b8d3c4c3SMinchan Kim spin_unlock(ptl); 16279818b8cdSHuang Ying split_huge_page(page); 1628b8d3c4c3SMinchan Kim put_page(page); 1629b8d3c4c3SMinchan Kim unlock_page(page); 1630b8d3c4c3SMinchan Kim goto out_unlocked; 1631b8d3c4c3SMinchan Kim } 1632b8d3c4c3SMinchan Kim 1633b8d3c4c3SMinchan Kim if (PageDirty(page)) 1634b8d3c4c3SMinchan Kim ClearPageDirty(page); 1635b8d3c4c3SMinchan Kim unlock_page(page); 1636b8d3c4c3SMinchan Kim 1637b8d3c4c3SMinchan Kim if (PageActive(page)) 1638b8d3c4c3SMinchan Kim deactivate_page(page); 1639b8d3c4c3SMinchan Kim 1640b8d3c4c3SMinchan Kim if (pmd_young(orig_pmd) || pmd_dirty(orig_pmd)) { 1641b8d3c4c3SMinchan Kim orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd, 1642b8d3c4c3SMinchan Kim tlb->fullmm); 1643b8d3c4c3SMinchan Kim orig_pmd = pmd_mkold(orig_pmd); 1644b8d3c4c3SMinchan Kim orig_pmd = pmd_mkclean(orig_pmd); 1645b8d3c4c3SMinchan Kim 1646b8d3c4c3SMinchan Kim set_pmd_at(mm, addr, pmd, orig_pmd); 1647b8d3c4c3SMinchan Kim tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 1648b8d3c4c3SMinchan Kim } 1649b8d3c4c3SMinchan Kim ret = 1; 1650b8d3c4c3SMinchan Kim out: 1651b8d3c4c3SMinchan Kim spin_unlock(ptl); 1652b8d3c4c3SMinchan Kim out_unlocked: 1653b8d3c4c3SMinchan Kim return ret; 1654b8d3c4c3SMinchan Kim } 1655b8d3c4c3SMinchan Kim 165671e3aac0SAndrea Arcangeli int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 1657f21760b1SShaohua Li pmd_t *pmd, unsigned long addr) 165871e3aac0SAndrea Arcangeli { 1659f5c8ad47SDavid Miller pmd_t orig_pmd; 1660da146769SKirill A. Shutemov spinlock_t *ptl; 1661da146769SKirill A. Shutemov 1662b6ec57f4SKirill A. Shutemov ptl = __pmd_trans_huge_lock(pmd, vma); 1663b6ec57f4SKirill A. Shutemov if (!ptl) 1664da146769SKirill A. Shutemov return 0; 1665a6bf2bb0SAneesh Kumar K.V /* 1666a6bf2bb0SAneesh Kumar K.V * For architectures like ppc64 we look at deposited pgtable 16678809aa2dSAneesh Kumar K.V * when calling pmdp_huge_get_and_clear. So do the 1668a6bf2bb0SAneesh Kumar K.V * pgtable_trans_huge_withdraw after finishing pmdp related 1669a6bf2bb0SAneesh Kumar K.V * operations. 1670a6bf2bb0SAneesh Kumar K.V */ 16718809aa2dSAneesh Kumar K.V orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd, 1672fcbe08d6SMartin Schwidefsky tlb->fullmm); 1673f21760b1SShaohua Li tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 16744897c765SMatthew Wilcox if (vma_is_dax(vma)) { 16754897c765SMatthew Wilcox spin_unlock(ptl); 1676da146769SKirill A. Shutemov if (is_huge_zero_pmd(orig_pmd)) 1677aa88b68cSKirill A. Shutemov tlb_remove_page(tlb, pmd_page(orig_pmd)); 1678da146769SKirill A. Shutemov } else if (is_huge_zero_pmd(orig_pmd)) { 1679da146769SKirill A. Shutemov pte_free(tlb->mm, pgtable_trans_huge_withdraw(tlb->mm, pmd)); 1680e1f56c89SKirill A. Shutemov atomic_long_dec(&tlb->mm->nr_ptes); 1681bf929152SKirill A. Shutemov spin_unlock(ptl); 1682aa88b68cSKirill A. Shutemov tlb_remove_page(tlb, pmd_page(orig_pmd)); 1683479f0abbSKirill A. Shutemov } else { 16844897c765SMatthew Wilcox struct page *page = pmd_page(orig_pmd); 1685d281ee61SKirill A. Shutemov page_remove_rmap(page, true); 1686309381feSSasha Levin VM_BUG_ON_PAGE(page_mapcount(page) < 0, page); 168771e3aac0SAndrea Arcangeli add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR); 1688309381feSSasha Levin VM_BUG_ON_PAGE(!PageHead(page), page); 1689da146769SKirill A. Shutemov pte_free(tlb->mm, pgtable_trans_huge_withdraw(tlb->mm, pmd)); 1690e1f56c89SKirill A. Shutemov atomic_long_dec(&tlb->mm->nr_ptes); 1691bf929152SKirill A. Shutemov spin_unlock(ptl); 169271e3aac0SAndrea Arcangeli tlb_remove_page(tlb, page); 1693479f0abbSKirill A. Shutemov } 1694da146769SKirill A. Shutemov return 1; 169571e3aac0SAndrea Arcangeli } 169671e3aac0SAndrea Arcangeli 1697bf8616d5SHugh Dickins bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, 169837a1c49aSAndrea Arcangeli unsigned long new_addr, unsigned long old_end, 169937a1c49aSAndrea Arcangeli pmd_t *old_pmd, pmd_t *new_pmd) 170037a1c49aSAndrea Arcangeli { 1701bf929152SKirill A. Shutemov spinlock_t *old_ptl, *new_ptl; 170237a1c49aSAndrea Arcangeli pmd_t pmd; 170337a1c49aSAndrea Arcangeli struct mm_struct *mm = vma->vm_mm; 170437a1c49aSAndrea Arcangeli 170537a1c49aSAndrea Arcangeli if ((old_addr & ~HPAGE_PMD_MASK) || 170637a1c49aSAndrea Arcangeli (new_addr & ~HPAGE_PMD_MASK) || 1707bf8616d5SHugh Dickins old_end - old_addr < HPAGE_PMD_SIZE) 17084b471e88SKirill A. Shutemov return false; 170937a1c49aSAndrea Arcangeli 171037a1c49aSAndrea Arcangeli /* 171137a1c49aSAndrea Arcangeli * The destination pmd shouldn't be established, free_pgtables() 171237a1c49aSAndrea Arcangeli * should have release it. 171337a1c49aSAndrea Arcangeli */ 171437a1c49aSAndrea Arcangeli if (WARN_ON(!pmd_none(*new_pmd))) { 171537a1c49aSAndrea Arcangeli VM_BUG_ON(pmd_trans_huge(*new_pmd)); 17164b471e88SKirill A. Shutemov return false; 171737a1c49aSAndrea Arcangeli } 171837a1c49aSAndrea Arcangeli 1719bf929152SKirill A. Shutemov /* 1720bf929152SKirill A. Shutemov * We don't have to worry about the ordering of src and dst 1721bf929152SKirill A. Shutemov * ptlocks because exclusive mmap_sem prevents deadlock. 1722bf929152SKirill A. Shutemov */ 1723b6ec57f4SKirill A. Shutemov old_ptl = __pmd_trans_huge_lock(old_pmd, vma); 1724b6ec57f4SKirill A. Shutemov if (old_ptl) { 1725bf929152SKirill A. Shutemov new_ptl = pmd_lockptr(mm, new_pmd); 1726bf929152SKirill A. Shutemov if (new_ptl != old_ptl) 1727bf929152SKirill A. Shutemov spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING); 17288809aa2dSAneesh Kumar K.V pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd); 172937a1c49aSAndrea Arcangeli VM_BUG_ON(!pmd_none(*new_pmd)); 17303592806cSKirill A. Shutemov 173169a8ec2dSKirill A. Shutemov if (pmd_move_must_withdraw(new_ptl, old_ptl) && 173269a8ec2dSKirill A. Shutemov vma_is_anonymous(vma)) { 1733b3084f4dSAneesh Kumar K.V pgtable_t pgtable; 17343592806cSKirill A. Shutemov pgtable = pgtable_trans_huge_withdraw(mm, old_pmd); 17353592806cSKirill A. Shutemov pgtable_trans_huge_deposit(mm, new_pmd, pgtable); 17363592806cSKirill A. Shutemov } 1737b3084f4dSAneesh Kumar K.V set_pmd_at(mm, new_addr, new_pmd, pmd_mksoft_dirty(pmd)); 1738b3084f4dSAneesh Kumar K.V if (new_ptl != old_ptl) 1739b3084f4dSAneesh Kumar K.V spin_unlock(new_ptl); 1740bf929152SKirill A. Shutemov spin_unlock(old_ptl); 17414b471e88SKirill A. Shutemov return true; 174237a1c49aSAndrea Arcangeli } 17434b471e88SKirill A. Shutemov return false; 174437a1c49aSAndrea Arcangeli } 174537a1c49aSAndrea Arcangeli 1746f123d74aSMel Gorman /* 1747f123d74aSMel Gorman * Returns 1748f123d74aSMel Gorman * - 0 if PMD could not be locked 1749f123d74aSMel Gorman * - 1 if PMD was locked but protections unchange and TLB flush unnecessary 1750f123d74aSMel Gorman * - HPAGE_PMD_NR is protections changed and TLB flush necessary 1751f123d74aSMel Gorman */ 1752cd7548abSJohannes Weiner int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, 1753e944fd67SMel Gorman unsigned long addr, pgprot_t newprot, int prot_numa) 1754cd7548abSJohannes Weiner { 1755cd7548abSJohannes Weiner struct mm_struct *mm = vma->vm_mm; 1756bf929152SKirill A. Shutemov spinlock_t *ptl; 1757cd7548abSJohannes Weiner int ret = 0; 1758cd7548abSJohannes Weiner 1759b6ec57f4SKirill A. Shutemov ptl = __pmd_trans_huge_lock(pmd, vma); 1760b6ec57f4SKirill A. Shutemov if (ptl) { 1761cd7548abSJohannes Weiner pmd_t entry; 1762b191f9b1SMel Gorman bool preserve_write = prot_numa && pmd_write(*pmd); 1763ba68bc01SMel Gorman ret = 1; 1764e944fd67SMel Gorman 1765e944fd67SMel Gorman /* 1766e944fd67SMel Gorman * Avoid trapping faults against the zero page. The read-only 1767e944fd67SMel Gorman * data is likely to be read-cached on the local CPU and 1768e944fd67SMel Gorman * local/remote hits to the zero page are not interesting. 1769e944fd67SMel Gorman */ 1770e944fd67SMel Gorman if (prot_numa && is_huge_zero_pmd(*pmd)) { 1771e944fd67SMel Gorman spin_unlock(ptl); 1772ba68bc01SMel Gorman return ret; 1773e944fd67SMel Gorman } 1774e944fd67SMel Gorman 177510c1045fSMel Gorman if (!prot_numa || !pmd_protnone(*pmd)) { 17768809aa2dSAneesh Kumar K.V entry = pmdp_huge_get_and_clear_notify(mm, addr, pmd); 1777cd7548abSJohannes Weiner entry = pmd_modify(entry, newprot); 1778b191f9b1SMel Gorman if (preserve_write) 1779b191f9b1SMel Gorman entry = pmd_mkwrite(entry); 1780f123d74aSMel Gorman ret = HPAGE_PMD_NR; 178156eecdb9SAneesh Kumar K.V set_pmd_at(mm, addr, pmd, entry); 1782b191f9b1SMel Gorman BUG_ON(!preserve_write && pmd_write(entry)); 178310c1045fSMel Gorman } 1784bf929152SKirill A. Shutemov spin_unlock(ptl); 1785cd7548abSJohannes Weiner } 1786cd7548abSJohannes Weiner 1787cd7548abSJohannes Weiner return ret; 1788cd7548abSJohannes Weiner } 1789cd7548abSJohannes Weiner 1790025c5b24SNaoya Horiguchi /* 17914b471e88SKirill A. Shutemov * Returns true if a given pmd maps a thp, false otherwise. 1792025c5b24SNaoya Horiguchi * 17934b471e88SKirill A. Shutemov * Note that if it returns true, this routine returns without unlocking page 17944b471e88SKirill A. Shutemov * table lock. So callers must unlock it. 1795025c5b24SNaoya Horiguchi */ 1796b6ec57f4SKirill A. Shutemov spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma) 1797025c5b24SNaoya Horiguchi { 1798b6ec57f4SKirill A. Shutemov spinlock_t *ptl; 1799b6ec57f4SKirill A. Shutemov ptl = pmd_lock(vma->vm_mm, pmd); 18005c7fb56eSDan Williams if (likely(pmd_trans_huge(*pmd) || pmd_devmap(*pmd))) 1801b6ec57f4SKirill A. Shutemov return ptl; 1802b6ec57f4SKirill A. Shutemov spin_unlock(ptl); 1803b6ec57f4SKirill A. Shutemov return NULL; 1804025c5b24SNaoya Horiguchi } 1805025c5b24SNaoya Horiguchi 18069050d7ebSVlastimil Babka #define VM_NO_THP (VM_SPECIAL | VM_HUGETLB | VM_SHARED | VM_MAYSHARE) 180778f11a25SAndrea Arcangeli 180860ab3244SAndrea Arcangeli int hugepage_madvise(struct vm_area_struct *vma, 180960ab3244SAndrea Arcangeli unsigned long *vm_flags, int advice) 18100af4e98bSAndrea Arcangeli { 1811a664b2d8SAndrea Arcangeli switch (advice) { 1812a664b2d8SAndrea Arcangeli case MADV_HUGEPAGE: 18131e1836e8SAlex Thorlton #ifdef CONFIG_S390 18141e1836e8SAlex Thorlton /* 18151e1836e8SAlex Thorlton * qemu blindly sets MADV_HUGEPAGE on all allocations, but s390 18161e1836e8SAlex Thorlton * can't handle this properly after s390_enable_sie, so we simply 18171e1836e8SAlex Thorlton * ignore the madvise to prevent qemu from causing a SIGSEGV. 18181e1836e8SAlex Thorlton */ 18191e1836e8SAlex Thorlton if (mm_has_pgste(vma->vm_mm)) 18201e1836e8SAlex Thorlton return 0; 18211e1836e8SAlex Thorlton #endif 18220af4e98bSAndrea Arcangeli /* 18230af4e98bSAndrea Arcangeli * Be somewhat over-protective like KSM for now! 18240af4e98bSAndrea Arcangeli */ 18251a763615SJason J. Herne if (*vm_flags & VM_NO_THP) 18260af4e98bSAndrea Arcangeli return -EINVAL; 1827a664b2d8SAndrea Arcangeli *vm_flags &= ~VM_NOHUGEPAGE; 18280af4e98bSAndrea Arcangeli *vm_flags |= VM_HUGEPAGE; 182960ab3244SAndrea Arcangeli /* 183060ab3244SAndrea Arcangeli * If the vma become good for khugepaged to scan, 183160ab3244SAndrea Arcangeli * register it here without waiting a page fault that 183260ab3244SAndrea Arcangeli * may not happen any time soon. 183360ab3244SAndrea Arcangeli */ 18346d50e60cSDavid Rientjes if (unlikely(khugepaged_enter_vma_merge(vma, *vm_flags))) 183560ab3244SAndrea Arcangeli return -ENOMEM; 1836a664b2d8SAndrea Arcangeli break; 1837a664b2d8SAndrea Arcangeli case MADV_NOHUGEPAGE: 1838a664b2d8SAndrea Arcangeli /* 1839a664b2d8SAndrea Arcangeli * Be somewhat over-protective like KSM for now! 1840a664b2d8SAndrea Arcangeli */ 18411a763615SJason J. Herne if (*vm_flags & VM_NO_THP) 1842a664b2d8SAndrea Arcangeli return -EINVAL; 1843a664b2d8SAndrea Arcangeli *vm_flags &= ~VM_HUGEPAGE; 1844a664b2d8SAndrea Arcangeli *vm_flags |= VM_NOHUGEPAGE; 184560ab3244SAndrea Arcangeli /* 184660ab3244SAndrea Arcangeli * Setting VM_NOHUGEPAGE will prevent khugepaged from scanning 184760ab3244SAndrea Arcangeli * this vma even if we leave the mm registered in khugepaged if 184860ab3244SAndrea Arcangeli * it got registered before VM_NOHUGEPAGE was set. 184960ab3244SAndrea Arcangeli */ 1850a664b2d8SAndrea Arcangeli break; 1851a664b2d8SAndrea Arcangeli } 18520af4e98bSAndrea Arcangeli 18530af4e98bSAndrea Arcangeli return 0; 18540af4e98bSAndrea Arcangeli } 18550af4e98bSAndrea Arcangeli 1856ba76149fSAndrea Arcangeli static int __init khugepaged_slab_init(void) 1857ba76149fSAndrea Arcangeli { 1858ba76149fSAndrea Arcangeli mm_slot_cache = kmem_cache_create("khugepaged_mm_slot", 1859ba76149fSAndrea Arcangeli sizeof(struct mm_slot), 1860ba76149fSAndrea Arcangeli __alignof__(struct mm_slot), 0, NULL); 1861ba76149fSAndrea Arcangeli if (!mm_slot_cache) 1862ba76149fSAndrea Arcangeli return -ENOMEM; 1863ba76149fSAndrea Arcangeli 1864ba76149fSAndrea Arcangeli return 0; 1865ba76149fSAndrea Arcangeli } 1866ba76149fSAndrea Arcangeli 186765ebb64fSKirill A. Shutemov static void __init khugepaged_slab_exit(void) 186865ebb64fSKirill A. Shutemov { 186965ebb64fSKirill A. Shutemov kmem_cache_destroy(mm_slot_cache); 187065ebb64fSKirill A. Shutemov } 187165ebb64fSKirill A. Shutemov 1872ba76149fSAndrea Arcangeli static inline struct mm_slot *alloc_mm_slot(void) 1873ba76149fSAndrea Arcangeli { 1874ba76149fSAndrea Arcangeli if (!mm_slot_cache) /* initialization failed */ 1875ba76149fSAndrea Arcangeli return NULL; 1876ba76149fSAndrea Arcangeli return kmem_cache_zalloc(mm_slot_cache, GFP_KERNEL); 1877ba76149fSAndrea Arcangeli } 1878ba76149fSAndrea Arcangeli 1879ba76149fSAndrea Arcangeli static inline void free_mm_slot(struct mm_slot *mm_slot) 1880ba76149fSAndrea Arcangeli { 1881ba76149fSAndrea Arcangeli kmem_cache_free(mm_slot_cache, mm_slot); 1882ba76149fSAndrea Arcangeli } 1883ba76149fSAndrea Arcangeli 1884ba76149fSAndrea Arcangeli static struct mm_slot *get_mm_slot(struct mm_struct *mm) 1885ba76149fSAndrea Arcangeli { 1886ba76149fSAndrea Arcangeli struct mm_slot *mm_slot; 1887ba76149fSAndrea Arcangeli 1888b67bfe0dSSasha Levin hash_for_each_possible(mm_slots_hash, mm_slot, hash, (unsigned long)mm) 1889ba76149fSAndrea Arcangeli if (mm == mm_slot->mm) 1890ba76149fSAndrea Arcangeli return mm_slot; 189143b5fbbdSSasha Levin 1892ba76149fSAndrea Arcangeli return NULL; 1893ba76149fSAndrea Arcangeli } 1894ba76149fSAndrea Arcangeli 1895ba76149fSAndrea Arcangeli static void insert_to_mm_slots_hash(struct mm_struct *mm, 1896ba76149fSAndrea Arcangeli struct mm_slot *mm_slot) 1897ba76149fSAndrea Arcangeli { 1898ba76149fSAndrea Arcangeli mm_slot->mm = mm; 189943b5fbbdSSasha Levin hash_add(mm_slots_hash, &mm_slot->hash, (long)mm); 1900ba76149fSAndrea Arcangeli } 1901ba76149fSAndrea Arcangeli 1902ba76149fSAndrea Arcangeli static inline int khugepaged_test_exit(struct mm_struct *mm) 1903ba76149fSAndrea Arcangeli { 1904ba76149fSAndrea Arcangeli return atomic_read(&mm->mm_users) == 0; 1905ba76149fSAndrea Arcangeli } 1906ba76149fSAndrea Arcangeli 1907ba76149fSAndrea Arcangeli int __khugepaged_enter(struct mm_struct *mm) 1908ba76149fSAndrea Arcangeli { 1909ba76149fSAndrea Arcangeli struct mm_slot *mm_slot; 1910ba76149fSAndrea Arcangeli int wakeup; 1911ba76149fSAndrea Arcangeli 1912ba76149fSAndrea Arcangeli mm_slot = alloc_mm_slot(); 1913ba76149fSAndrea Arcangeli if (!mm_slot) 1914ba76149fSAndrea Arcangeli return -ENOMEM; 1915ba76149fSAndrea Arcangeli 1916ba76149fSAndrea Arcangeli /* __khugepaged_exit() must not run from under us */ 191796dad67fSSasha Levin VM_BUG_ON_MM(khugepaged_test_exit(mm), mm); 1918ba76149fSAndrea Arcangeli if (unlikely(test_and_set_bit(MMF_VM_HUGEPAGE, &mm->flags))) { 1919ba76149fSAndrea Arcangeli free_mm_slot(mm_slot); 1920ba76149fSAndrea Arcangeli return 0; 1921ba76149fSAndrea Arcangeli } 1922ba76149fSAndrea Arcangeli 1923ba76149fSAndrea Arcangeli spin_lock(&khugepaged_mm_lock); 1924ba76149fSAndrea Arcangeli insert_to_mm_slots_hash(mm, mm_slot); 1925ba76149fSAndrea Arcangeli /* 1926ba76149fSAndrea Arcangeli * Insert just behind the scanning cursor, to let the area settle 1927ba76149fSAndrea Arcangeli * down a little. 1928ba76149fSAndrea Arcangeli */ 1929ba76149fSAndrea Arcangeli wakeup = list_empty(&khugepaged_scan.mm_head); 1930ba76149fSAndrea Arcangeli list_add_tail(&mm_slot->mm_node, &khugepaged_scan.mm_head); 1931ba76149fSAndrea Arcangeli spin_unlock(&khugepaged_mm_lock); 1932ba76149fSAndrea Arcangeli 1933ba76149fSAndrea Arcangeli atomic_inc(&mm->mm_count); 1934ba76149fSAndrea Arcangeli if (wakeup) 1935ba76149fSAndrea Arcangeli wake_up_interruptible(&khugepaged_wait); 1936ba76149fSAndrea Arcangeli 1937ba76149fSAndrea Arcangeli return 0; 1938ba76149fSAndrea Arcangeli } 1939ba76149fSAndrea Arcangeli 19406d50e60cSDavid Rientjes int khugepaged_enter_vma_merge(struct vm_area_struct *vma, 19416d50e60cSDavid Rientjes unsigned long vm_flags) 1942ba76149fSAndrea Arcangeli { 1943ba76149fSAndrea Arcangeli unsigned long hstart, hend; 1944ba76149fSAndrea Arcangeli if (!vma->anon_vma) 1945ba76149fSAndrea Arcangeli /* 1946ba76149fSAndrea Arcangeli * Not yet faulted in so we will register later in the 1947ba76149fSAndrea Arcangeli * page fault if needed. 1948ba76149fSAndrea Arcangeli */ 1949ba76149fSAndrea Arcangeli return 0; 19503486b85aSKonstantin Khlebnikov if (vma->vm_ops || (vm_flags & VM_NO_THP)) 1951ba76149fSAndrea Arcangeli /* khugepaged not yet working on file or special mappings */ 1952ba76149fSAndrea Arcangeli return 0; 1953ba76149fSAndrea Arcangeli hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK; 1954ba76149fSAndrea Arcangeli hend = vma->vm_end & HPAGE_PMD_MASK; 1955ba76149fSAndrea Arcangeli if (hstart < hend) 19566d50e60cSDavid Rientjes return khugepaged_enter(vma, vm_flags); 1957ba76149fSAndrea Arcangeli return 0; 1958ba76149fSAndrea Arcangeli } 1959ba76149fSAndrea Arcangeli 1960ba76149fSAndrea Arcangeli void __khugepaged_exit(struct mm_struct *mm) 1961ba76149fSAndrea Arcangeli { 1962ba76149fSAndrea Arcangeli struct mm_slot *mm_slot; 1963ba76149fSAndrea Arcangeli int free = 0; 1964ba76149fSAndrea Arcangeli 1965ba76149fSAndrea Arcangeli spin_lock(&khugepaged_mm_lock); 1966ba76149fSAndrea Arcangeli mm_slot = get_mm_slot(mm); 1967ba76149fSAndrea Arcangeli if (mm_slot && khugepaged_scan.mm_slot != mm_slot) { 196843b5fbbdSSasha Levin hash_del(&mm_slot->hash); 1969ba76149fSAndrea Arcangeli list_del(&mm_slot->mm_node); 1970ba76149fSAndrea Arcangeli free = 1; 1971ba76149fSAndrea Arcangeli } 1972d788e80aSChris Wright spin_unlock(&khugepaged_mm_lock); 1973ba76149fSAndrea Arcangeli 1974ba76149fSAndrea Arcangeli if (free) { 1975ba76149fSAndrea Arcangeli clear_bit(MMF_VM_HUGEPAGE, &mm->flags); 1976ba76149fSAndrea Arcangeli free_mm_slot(mm_slot); 1977ba76149fSAndrea Arcangeli mmdrop(mm); 1978ba76149fSAndrea Arcangeli } else if (mm_slot) { 1979ba76149fSAndrea Arcangeli /* 1980ba76149fSAndrea Arcangeli * This is required to serialize against 1981ba76149fSAndrea Arcangeli * khugepaged_test_exit() (which is guaranteed to run 1982ba76149fSAndrea Arcangeli * under mmap sem read mode). Stop here (after we 1983ba76149fSAndrea Arcangeli * return all pagetables will be destroyed) until 1984ba76149fSAndrea Arcangeli * khugepaged has finished working on the pagetables 1985ba76149fSAndrea Arcangeli * under the mmap_sem. 1986ba76149fSAndrea Arcangeli */ 1987ba76149fSAndrea Arcangeli down_write(&mm->mmap_sem); 1988ba76149fSAndrea Arcangeli up_write(&mm->mmap_sem); 1989d788e80aSChris Wright } 1990ba76149fSAndrea Arcangeli } 1991ba76149fSAndrea Arcangeli 1992ba76149fSAndrea Arcangeli static void release_pte_page(struct page *page) 1993ba76149fSAndrea Arcangeli { 1994ba76149fSAndrea Arcangeli /* 0 stands for page_is_file_cache(page) == false */ 1995ba76149fSAndrea Arcangeli dec_zone_page_state(page, NR_ISOLATED_ANON + 0); 1996ba76149fSAndrea Arcangeli unlock_page(page); 1997ba76149fSAndrea Arcangeli putback_lru_page(page); 1998ba76149fSAndrea Arcangeli } 1999ba76149fSAndrea Arcangeli 2000ba76149fSAndrea Arcangeli static void release_pte_pages(pte_t *pte, pte_t *_pte) 2001ba76149fSAndrea Arcangeli { 2002ba76149fSAndrea Arcangeli while (--_pte >= pte) { 2003ba76149fSAndrea Arcangeli pte_t pteval = *_pte; 2004ca0984caSEbru Akagunduz if (!pte_none(pteval) && !is_zero_pfn(pte_pfn(pteval))) 2005ba76149fSAndrea Arcangeli release_pte_page(pte_page(pteval)); 2006ba76149fSAndrea Arcangeli } 2007ba76149fSAndrea Arcangeli } 2008ba76149fSAndrea Arcangeli 2009ba76149fSAndrea Arcangeli static int __collapse_huge_page_isolate(struct vm_area_struct *vma, 2010ba76149fSAndrea Arcangeli unsigned long address, 2011ba76149fSAndrea Arcangeli pte_t *pte) 2012ba76149fSAndrea Arcangeli { 20137d2eba05SEbru Akagunduz struct page *page = NULL; 2014ba76149fSAndrea Arcangeli pte_t *_pte; 20157d2eba05SEbru Akagunduz int none_or_zero = 0, result = 0; 201610359213SEbru Akagunduz bool referenced = false, writable = false; 20177d2eba05SEbru Akagunduz 2018ba76149fSAndrea Arcangeli for (_pte = pte; _pte < pte+HPAGE_PMD_NR; 2019ba76149fSAndrea Arcangeli _pte++, address += PAGE_SIZE) { 2020ba76149fSAndrea Arcangeli pte_t pteval = *_pte; 202147aee4d8SMinchan Kim if (pte_none(pteval) || (pte_present(pteval) && 202247aee4d8SMinchan Kim is_zero_pfn(pte_pfn(pteval)))) { 2023c1294d05SAndrea Arcangeli if (!userfaultfd_armed(vma) && 20247d2eba05SEbru Akagunduz ++none_or_zero <= khugepaged_max_ptes_none) { 2025ba76149fSAndrea Arcangeli continue; 20267d2eba05SEbru Akagunduz } else { 20277d2eba05SEbru Akagunduz result = SCAN_EXCEED_NONE_PTE; 2028ba76149fSAndrea Arcangeli goto out; 2029ba76149fSAndrea Arcangeli } 20307d2eba05SEbru Akagunduz } 20317d2eba05SEbru Akagunduz if (!pte_present(pteval)) { 20327d2eba05SEbru Akagunduz result = SCAN_PTE_NON_PRESENT; 2033ba76149fSAndrea Arcangeli goto out; 20347d2eba05SEbru Akagunduz } 2035ba76149fSAndrea Arcangeli page = vm_normal_page(vma, address, pteval); 20367d2eba05SEbru Akagunduz if (unlikely(!page)) { 20377d2eba05SEbru Akagunduz result = SCAN_PAGE_NULL; 2038ba76149fSAndrea Arcangeli goto out; 20397d2eba05SEbru Akagunduz } 2040344aa35cSBob Liu 2041309381feSSasha Levin VM_BUG_ON_PAGE(PageCompound(page), page); 2042309381feSSasha Levin VM_BUG_ON_PAGE(!PageAnon(page), page); 2043309381feSSasha Levin VM_BUG_ON_PAGE(!PageSwapBacked(page), page); 2044ba76149fSAndrea Arcangeli 2045ba76149fSAndrea Arcangeli /* 2046ba76149fSAndrea Arcangeli * We can do it before isolate_lru_page because the 2047ba76149fSAndrea Arcangeli * page can't be freed from under us. NOTE: PG_lock 2048ba76149fSAndrea Arcangeli * is needed to serialize against split_huge_page 2049ba76149fSAndrea Arcangeli * when invoked from the VM. 2050ba76149fSAndrea Arcangeli */ 20517d2eba05SEbru Akagunduz if (!trylock_page(page)) { 20527d2eba05SEbru Akagunduz result = SCAN_PAGE_LOCK; 2053ba76149fSAndrea Arcangeli goto out; 20547d2eba05SEbru Akagunduz } 205510359213SEbru Akagunduz 205610359213SEbru Akagunduz /* 205710359213SEbru Akagunduz * cannot use mapcount: can't collapse if there's a gup pin. 205810359213SEbru Akagunduz * The page must only be referenced by the scanned process 205910359213SEbru Akagunduz * and page swap cache. 206010359213SEbru Akagunduz */ 206110359213SEbru Akagunduz if (page_count(page) != 1 + !!PageSwapCache(page)) { 206210359213SEbru Akagunduz unlock_page(page); 20637d2eba05SEbru Akagunduz result = SCAN_PAGE_COUNT; 206410359213SEbru Akagunduz goto out; 206510359213SEbru Akagunduz } 206610359213SEbru Akagunduz if (pte_write(pteval)) { 206710359213SEbru Akagunduz writable = true; 206810359213SEbru Akagunduz } else { 20696d0a07edSAndrea Arcangeli if (PageSwapCache(page) && 20706d0a07edSAndrea Arcangeli !reuse_swap_page(page, NULL)) { 207110359213SEbru Akagunduz unlock_page(page); 20727d2eba05SEbru Akagunduz result = SCAN_SWAP_CACHE_PAGE; 207310359213SEbru Akagunduz goto out; 207410359213SEbru Akagunduz } 207510359213SEbru Akagunduz /* 207610359213SEbru Akagunduz * Page is not in the swap cache. It can be collapsed 207710359213SEbru Akagunduz * into a THP. 207810359213SEbru Akagunduz */ 207910359213SEbru Akagunduz } 208010359213SEbru Akagunduz 2081ba76149fSAndrea Arcangeli /* 2082ba76149fSAndrea Arcangeli * Isolate the page to avoid collapsing an hugepage 2083ba76149fSAndrea Arcangeli * currently in use by the VM. 2084ba76149fSAndrea Arcangeli */ 2085ba76149fSAndrea Arcangeli if (isolate_lru_page(page)) { 2086ba76149fSAndrea Arcangeli unlock_page(page); 20877d2eba05SEbru Akagunduz result = SCAN_DEL_PAGE_LRU; 2088ba76149fSAndrea Arcangeli goto out; 2089ba76149fSAndrea Arcangeli } 2090ba76149fSAndrea Arcangeli /* 0 stands for page_is_file_cache(page) == false */ 2091ba76149fSAndrea Arcangeli inc_zone_page_state(page, NR_ISOLATED_ANON + 0); 2092309381feSSasha Levin VM_BUG_ON_PAGE(!PageLocked(page), page); 2093309381feSSasha Levin VM_BUG_ON_PAGE(PageLRU(page), page); 2094ba76149fSAndrea Arcangeli 2095ba76149fSAndrea Arcangeli /* If there is no mapped pte young don't collapse the page */ 209633c3fc71SVladimir Davydov if (pte_young(pteval) || 209733c3fc71SVladimir Davydov page_is_young(page) || PageReferenced(page) || 20988ee53820SAndrea Arcangeli mmu_notifier_test_young(vma->vm_mm, address)) 209910359213SEbru Akagunduz referenced = true; 2100ba76149fSAndrea Arcangeli } 21017d2eba05SEbru Akagunduz if (likely(writable)) { 21027d2eba05SEbru Akagunduz if (likely(referenced)) { 21037d2eba05SEbru Akagunduz result = SCAN_SUCCEED; 210416fd0fe4Syalin wang trace_mm_collapse_huge_page_isolate(page, none_or_zero, 21057d2eba05SEbru Akagunduz referenced, writable, result); 2106344aa35cSBob Liu return 1; 21077d2eba05SEbru Akagunduz } 21087d2eba05SEbru Akagunduz } else { 21097d2eba05SEbru Akagunduz result = SCAN_PAGE_RO; 21107d2eba05SEbru Akagunduz } 21117d2eba05SEbru Akagunduz 2112ba76149fSAndrea Arcangeli out: 2113344aa35cSBob Liu release_pte_pages(pte, _pte); 211416fd0fe4Syalin wang trace_mm_collapse_huge_page_isolate(page, none_or_zero, 21157d2eba05SEbru Akagunduz referenced, writable, result); 2116344aa35cSBob Liu return 0; 2117ba76149fSAndrea Arcangeli } 2118ba76149fSAndrea Arcangeli 2119ba76149fSAndrea Arcangeli static void __collapse_huge_page_copy(pte_t *pte, struct page *page, 2120ba76149fSAndrea Arcangeli struct vm_area_struct *vma, 2121ba76149fSAndrea Arcangeli unsigned long address, 2122ba76149fSAndrea Arcangeli spinlock_t *ptl) 2123ba76149fSAndrea Arcangeli { 2124ba76149fSAndrea Arcangeli pte_t *_pte; 2125ba76149fSAndrea Arcangeli for (_pte = pte; _pte < pte+HPAGE_PMD_NR; _pte++) { 2126ba76149fSAndrea Arcangeli pte_t pteval = *_pte; 2127ba76149fSAndrea Arcangeli struct page *src_page; 2128ba76149fSAndrea Arcangeli 2129ca0984caSEbru Akagunduz if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) { 2130ba76149fSAndrea Arcangeli clear_user_highpage(page, address); 2131ba76149fSAndrea Arcangeli add_mm_counter(vma->vm_mm, MM_ANONPAGES, 1); 2132ca0984caSEbru Akagunduz if (is_zero_pfn(pte_pfn(pteval))) { 2133ca0984caSEbru Akagunduz /* 2134ca0984caSEbru Akagunduz * ptl mostly unnecessary. 2135ca0984caSEbru Akagunduz */ 2136ca0984caSEbru Akagunduz spin_lock(ptl); 2137ca0984caSEbru Akagunduz /* 2138ca0984caSEbru Akagunduz * paravirt calls inside pte_clear here are 2139ca0984caSEbru Akagunduz * superfluous. 2140ca0984caSEbru Akagunduz */ 2141ca0984caSEbru Akagunduz pte_clear(vma->vm_mm, address, _pte); 2142ca0984caSEbru Akagunduz spin_unlock(ptl); 2143ca0984caSEbru Akagunduz } 2144ba76149fSAndrea Arcangeli } else { 2145ba76149fSAndrea Arcangeli src_page = pte_page(pteval); 2146ba76149fSAndrea Arcangeli copy_user_highpage(page, src_page, address, vma); 2147309381feSSasha Levin VM_BUG_ON_PAGE(page_mapcount(src_page) != 1, src_page); 2148ba76149fSAndrea Arcangeli release_pte_page(src_page); 2149ba76149fSAndrea Arcangeli /* 2150ba76149fSAndrea Arcangeli * ptl mostly unnecessary, but preempt has to 2151ba76149fSAndrea Arcangeli * be disabled to update the per-cpu stats 2152ba76149fSAndrea Arcangeli * inside page_remove_rmap(). 2153ba76149fSAndrea Arcangeli */ 2154ba76149fSAndrea Arcangeli spin_lock(ptl); 2155ba76149fSAndrea Arcangeli /* 2156ba76149fSAndrea Arcangeli * paravirt calls inside pte_clear here are 2157ba76149fSAndrea Arcangeli * superfluous. 2158ba76149fSAndrea Arcangeli */ 2159ba76149fSAndrea Arcangeli pte_clear(vma->vm_mm, address, _pte); 2160d281ee61SKirill A. Shutemov page_remove_rmap(src_page, false); 2161ba76149fSAndrea Arcangeli spin_unlock(ptl); 2162ba76149fSAndrea Arcangeli free_page_and_swap_cache(src_page); 2163ba76149fSAndrea Arcangeli } 2164ba76149fSAndrea Arcangeli 2165ba76149fSAndrea Arcangeli address += PAGE_SIZE; 2166ba76149fSAndrea Arcangeli page++; 2167ba76149fSAndrea Arcangeli } 2168ba76149fSAndrea Arcangeli } 2169ba76149fSAndrea Arcangeli 217026234f36SXiao Guangrong static void khugepaged_alloc_sleep(void) 217126234f36SXiao Guangrong { 2172bde43c6cSPetr Mladek DEFINE_WAIT(wait); 2173bde43c6cSPetr Mladek 2174bde43c6cSPetr Mladek add_wait_queue(&khugepaged_wait, &wait); 2175bde43c6cSPetr Mladek freezable_schedule_timeout_interruptible( 217626234f36SXiao Guangrong msecs_to_jiffies(khugepaged_alloc_sleep_millisecs)); 2177bde43c6cSPetr Mladek remove_wait_queue(&khugepaged_wait, &wait); 217826234f36SXiao Guangrong } 217926234f36SXiao Guangrong 21809f1b868aSBob Liu static int khugepaged_node_load[MAX_NUMNODES]; 21819f1b868aSBob Liu 218214a4e214SDavid Rientjes static bool khugepaged_scan_abort(int nid) 218314a4e214SDavid Rientjes { 218414a4e214SDavid Rientjes int i; 218514a4e214SDavid Rientjes 218614a4e214SDavid Rientjes /* 218714a4e214SDavid Rientjes * If zone_reclaim_mode is disabled, then no extra effort is made to 218814a4e214SDavid Rientjes * allocate memory locally. 218914a4e214SDavid Rientjes */ 219014a4e214SDavid Rientjes if (!zone_reclaim_mode) 219114a4e214SDavid Rientjes return false; 219214a4e214SDavid Rientjes 219314a4e214SDavid Rientjes /* If there is a count for this node already, it must be acceptable */ 219414a4e214SDavid Rientjes if (khugepaged_node_load[nid]) 219514a4e214SDavid Rientjes return false; 219614a4e214SDavid Rientjes 219714a4e214SDavid Rientjes for (i = 0; i < MAX_NUMNODES; i++) { 219814a4e214SDavid Rientjes if (!khugepaged_node_load[i]) 219914a4e214SDavid Rientjes continue; 220014a4e214SDavid Rientjes if (node_distance(nid, i) > RECLAIM_DISTANCE) 220114a4e214SDavid Rientjes return true; 220214a4e214SDavid Rientjes } 220314a4e214SDavid Rientjes return false; 220414a4e214SDavid Rientjes } 220514a4e214SDavid Rientjes 220626234f36SXiao Guangrong #ifdef CONFIG_NUMA 22079f1b868aSBob Liu static int khugepaged_find_target_node(void) 22089f1b868aSBob Liu { 22099f1b868aSBob Liu static int last_khugepaged_target_node = NUMA_NO_NODE; 22109f1b868aSBob Liu int nid, target_node = 0, max_value = 0; 22119f1b868aSBob Liu 22129f1b868aSBob Liu /* find first node with max normal pages hit */ 22139f1b868aSBob Liu for (nid = 0; nid < MAX_NUMNODES; nid++) 22149f1b868aSBob Liu if (khugepaged_node_load[nid] > max_value) { 22159f1b868aSBob Liu max_value = khugepaged_node_load[nid]; 22169f1b868aSBob Liu target_node = nid; 22179f1b868aSBob Liu } 22189f1b868aSBob Liu 22199f1b868aSBob Liu /* do some balance if several nodes have the same hit record */ 22209f1b868aSBob Liu if (target_node <= last_khugepaged_target_node) 22219f1b868aSBob Liu for (nid = last_khugepaged_target_node + 1; nid < MAX_NUMNODES; 22229f1b868aSBob Liu nid++) 22239f1b868aSBob Liu if (max_value == khugepaged_node_load[nid]) { 22249f1b868aSBob Liu target_node = nid; 22259f1b868aSBob Liu break; 22269f1b868aSBob Liu } 22279f1b868aSBob Liu 22289f1b868aSBob Liu last_khugepaged_target_node = target_node; 22299f1b868aSBob Liu return target_node; 22309f1b868aSBob Liu } 22319f1b868aSBob Liu 223226234f36SXiao Guangrong static bool khugepaged_prealloc_page(struct page **hpage, bool *wait) 223326234f36SXiao Guangrong { 223426234f36SXiao Guangrong if (IS_ERR(*hpage)) { 223526234f36SXiao Guangrong if (!*wait) 223626234f36SXiao Guangrong return false; 223726234f36SXiao Guangrong 223826234f36SXiao Guangrong *wait = false; 2239e3b4126cSXiao Guangrong *hpage = NULL; 224026234f36SXiao Guangrong khugepaged_alloc_sleep(); 224126234f36SXiao Guangrong } else if (*hpage) { 224226234f36SXiao Guangrong put_page(*hpage); 224326234f36SXiao Guangrong *hpage = NULL; 224426234f36SXiao Guangrong } 224526234f36SXiao Guangrong 224626234f36SXiao Guangrong return true; 224726234f36SXiao Guangrong } 224826234f36SXiao Guangrong 22493b363692SMichal Hocko static struct page * 22503b363692SMichal Hocko khugepaged_alloc_page(struct page **hpage, gfp_t gfp, struct mm_struct *mm, 2251d6669d68SAaron Tomlin unsigned long address, int node) 225226234f36SXiao Guangrong { 2253309381feSSasha Levin VM_BUG_ON_PAGE(*hpage, *hpage); 22548b164568SVlastimil Babka 225526234f36SXiao Guangrong /* 22568b164568SVlastimil Babka * Before allocating the hugepage, release the mmap_sem read lock. 22578b164568SVlastimil Babka * The allocation can take potentially a long time if it involves 22588b164568SVlastimil Babka * sync compaction, and we do not need to hold the mmap_sem during 22598b164568SVlastimil Babka * that. We will recheck the vma after taking it again in write mode. 226026234f36SXiao Guangrong */ 226126234f36SXiao Guangrong up_read(&mm->mmap_sem); 22628b164568SVlastimil Babka 226396db800fSVlastimil Babka *hpage = __alloc_pages_node(node, gfp, HPAGE_PMD_ORDER); 226426234f36SXiao Guangrong if (unlikely(!*hpage)) { 226526234f36SXiao Guangrong count_vm_event(THP_COLLAPSE_ALLOC_FAILED); 226626234f36SXiao Guangrong *hpage = ERR_PTR(-ENOMEM); 226726234f36SXiao Guangrong return NULL; 226826234f36SXiao Guangrong } 226926234f36SXiao Guangrong 22709a982250SKirill A. Shutemov prep_transhuge_page(*hpage); 227126234f36SXiao Guangrong count_vm_event(THP_COLLAPSE_ALLOC); 227226234f36SXiao Guangrong return *hpage; 227326234f36SXiao Guangrong } 227426234f36SXiao Guangrong #else 22759f1b868aSBob Liu static int khugepaged_find_target_node(void) 22769f1b868aSBob Liu { 22779f1b868aSBob Liu return 0; 22789f1b868aSBob Liu } 22799f1b868aSBob Liu 2280444eb2a4SMel Gorman static inline struct page *alloc_khugepaged_hugepage(void) 228110dc4155SBob Liu { 22829a982250SKirill A. Shutemov struct page *page; 22839a982250SKirill A. Shutemov 2284444eb2a4SMel Gorman page = alloc_pages(alloc_hugepage_khugepaged_gfpmask(), 2285444eb2a4SMel Gorman HPAGE_PMD_ORDER); 22869a982250SKirill A. Shutemov if (page) 22879a982250SKirill A. Shutemov prep_transhuge_page(page); 22889a982250SKirill A. Shutemov return page; 228910dc4155SBob Liu } 229010dc4155SBob Liu 229126234f36SXiao Guangrong static struct page *khugepaged_alloc_hugepage(bool *wait) 229226234f36SXiao Guangrong { 229326234f36SXiao Guangrong struct page *hpage; 229426234f36SXiao Guangrong 229526234f36SXiao Guangrong do { 2296444eb2a4SMel Gorman hpage = alloc_khugepaged_hugepage(); 229726234f36SXiao Guangrong if (!hpage) { 229826234f36SXiao Guangrong count_vm_event(THP_COLLAPSE_ALLOC_FAILED); 229926234f36SXiao Guangrong if (!*wait) 230026234f36SXiao Guangrong return NULL; 230126234f36SXiao Guangrong 230226234f36SXiao Guangrong *wait = false; 230326234f36SXiao Guangrong khugepaged_alloc_sleep(); 230426234f36SXiao Guangrong } else 230526234f36SXiao Guangrong count_vm_event(THP_COLLAPSE_ALLOC); 230626234f36SXiao Guangrong } while (unlikely(!hpage) && likely(khugepaged_enabled())); 230726234f36SXiao Guangrong 230826234f36SXiao Guangrong return hpage; 230926234f36SXiao Guangrong } 231026234f36SXiao Guangrong 231126234f36SXiao Guangrong static bool khugepaged_prealloc_page(struct page **hpage, bool *wait) 231226234f36SXiao Guangrong { 231326234f36SXiao Guangrong if (!*hpage) 231426234f36SXiao Guangrong *hpage = khugepaged_alloc_hugepage(wait); 231526234f36SXiao Guangrong 231626234f36SXiao Guangrong if (unlikely(!*hpage)) 231726234f36SXiao Guangrong return false; 231826234f36SXiao Guangrong 231926234f36SXiao Guangrong return true; 232026234f36SXiao Guangrong } 232126234f36SXiao Guangrong 23223b363692SMichal Hocko static struct page * 23233b363692SMichal Hocko khugepaged_alloc_page(struct page **hpage, gfp_t gfp, struct mm_struct *mm, 2324d6669d68SAaron Tomlin unsigned long address, int node) 232526234f36SXiao Guangrong { 232626234f36SXiao Guangrong up_read(&mm->mmap_sem); 232726234f36SXiao Guangrong VM_BUG_ON(!*hpage); 23283b363692SMichal Hocko 232926234f36SXiao Guangrong return *hpage; 233026234f36SXiao Guangrong } 233126234f36SXiao Guangrong #endif 233226234f36SXiao Guangrong 2333fa475e51SBob Liu static bool hugepage_vma_check(struct vm_area_struct *vma) 2334fa475e51SBob Liu { 2335fa475e51SBob Liu if ((!(vma->vm_flags & VM_HUGEPAGE) && !khugepaged_always()) || 2336fa475e51SBob Liu (vma->vm_flags & VM_NOHUGEPAGE)) 2337fa475e51SBob Liu return false; 2338fa475e51SBob Liu if (!vma->anon_vma || vma->vm_ops) 2339fa475e51SBob Liu return false; 2340fa475e51SBob Liu if (is_vma_temporary_stack(vma)) 2341fa475e51SBob Liu return false; 23423486b85aSKonstantin Khlebnikov return !(vma->vm_flags & VM_NO_THP); 2343fa475e51SBob Liu } 2344fa475e51SBob Liu 2345ba76149fSAndrea Arcangeli static void collapse_huge_page(struct mm_struct *mm, 2346ba76149fSAndrea Arcangeli unsigned long address, 2347ce83d217SAndrea Arcangeli struct page **hpage, 23485c4b4be3SAndi Kleen struct vm_area_struct *vma, 23495c4b4be3SAndi Kleen int node) 2350ba76149fSAndrea Arcangeli { 2351ba76149fSAndrea Arcangeli pmd_t *pmd, _pmd; 2352ba76149fSAndrea Arcangeli pte_t *pte; 2353ba76149fSAndrea Arcangeli pgtable_t pgtable; 2354ba76149fSAndrea Arcangeli struct page *new_page; 2355c4088ebdSKirill A. Shutemov spinlock_t *pmd_ptl, *pte_ptl; 2356629d9d1cSArnd Bergmann int isolated = 0, result = 0; 2357ba76149fSAndrea Arcangeli unsigned long hstart, hend; 235800501b53SJohannes Weiner struct mem_cgroup *memcg; 23592ec74c3eSSagi Grimberg unsigned long mmun_start; /* For mmu_notifiers */ 23602ec74c3eSSagi Grimberg unsigned long mmun_end; /* For mmu_notifiers */ 23613b363692SMichal Hocko gfp_t gfp; 2362ba76149fSAndrea Arcangeli 2363ba76149fSAndrea Arcangeli VM_BUG_ON(address & ~HPAGE_PMD_MASK); 2364692e0b35SAndrea Arcangeli 23653b363692SMichal Hocko /* Only allocate from the target node */ 2366444eb2a4SMel Gorman gfp = alloc_hugepage_khugepaged_gfpmask() | __GFP_OTHER_NODE | __GFP_THISNODE; 23673b363692SMichal Hocko 236826234f36SXiao Guangrong /* release the mmap_sem read lock. */ 2369d6669d68SAaron Tomlin new_page = khugepaged_alloc_page(hpage, gfp, mm, address, node); 23707d2eba05SEbru Akagunduz if (!new_page) { 23717d2eba05SEbru Akagunduz result = SCAN_ALLOC_HUGE_PAGE_FAIL; 23727d2eba05SEbru Akagunduz goto out_nolock; 23737d2eba05SEbru Akagunduz } 2374ce83d217SAndrea Arcangeli 2375f627c2f5SKirill A. Shutemov if (unlikely(mem_cgroup_try_charge(new_page, mm, gfp, &memcg, true))) { 23767d2eba05SEbru Akagunduz result = SCAN_CGROUP_CHARGE_FAIL; 23777d2eba05SEbru Akagunduz goto out_nolock; 23787d2eba05SEbru Akagunduz } 2379ba76149fSAndrea Arcangeli 2380ba76149fSAndrea Arcangeli /* 2381ba76149fSAndrea Arcangeli * Prevent all access to pagetables with the exception of 2382ba76149fSAndrea Arcangeli * gup_fast later hanlded by the ptep_clear_flush and the VM 2383ba76149fSAndrea Arcangeli * handled by the anon_vma lock + PG_lock. 2384ba76149fSAndrea Arcangeli */ 2385ba76149fSAndrea Arcangeli down_write(&mm->mmap_sem); 23867d2eba05SEbru Akagunduz if (unlikely(khugepaged_test_exit(mm))) { 23877d2eba05SEbru Akagunduz result = SCAN_ANY_PROCESS; 2388ba76149fSAndrea Arcangeli goto out; 23897d2eba05SEbru Akagunduz } 2390ba76149fSAndrea Arcangeli 2391ba76149fSAndrea Arcangeli vma = find_vma(mm, address); 23927d2eba05SEbru Akagunduz if (!vma) { 23937d2eba05SEbru Akagunduz result = SCAN_VMA_NULL; 2394a8f531ebSLibin goto out; 23957d2eba05SEbru Akagunduz } 2396ba76149fSAndrea Arcangeli hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK; 2397ba76149fSAndrea Arcangeli hend = vma->vm_end & HPAGE_PMD_MASK; 23987d2eba05SEbru Akagunduz if (address < hstart || address + HPAGE_PMD_SIZE > hend) { 23997d2eba05SEbru Akagunduz result = SCAN_ADDRESS_RANGE; 2400ba76149fSAndrea Arcangeli goto out; 24017d2eba05SEbru Akagunduz } 24027d2eba05SEbru Akagunduz if (!hugepage_vma_check(vma)) { 24037d2eba05SEbru Akagunduz result = SCAN_VMA_CHECK; 2404ba76149fSAndrea Arcangeli goto out; 24057d2eba05SEbru Akagunduz } 24066219049aSBob Liu pmd = mm_find_pmd(mm, address); 24077d2eba05SEbru Akagunduz if (!pmd) { 24087d2eba05SEbru Akagunduz result = SCAN_PMD_NULL; 2409ba76149fSAndrea Arcangeli goto out; 24107d2eba05SEbru Akagunduz } 2411ba76149fSAndrea Arcangeli 24124fc3f1d6SIngo Molnar anon_vma_lock_write(vma->anon_vma); 2413ba76149fSAndrea Arcangeli 2414ba76149fSAndrea Arcangeli pte = pte_offset_map(pmd, address); 2415c4088ebdSKirill A. Shutemov pte_ptl = pte_lockptr(mm, pmd); 2416ba76149fSAndrea Arcangeli 24172ec74c3eSSagi Grimberg mmun_start = address; 24182ec74c3eSSagi Grimberg mmun_end = address + HPAGE_PMD_SIZE; 24192ec74c3eSSagi Grimberg mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 2420c4088ebdSKirill A. Shutemov pmd_ptl = pmd_lock(mm, pmd); /* probably unnecessary */ 2421ba76149fSAndrea Arcangeli /* 2422ba76149fSAndrea Arcangeli * After this gup_fast can't run anymore. This also removes 2423ba76149fSAndrea Arcangeli * any huge TLB entry from the CPU so we won't allow 2424ba76149fSAndrea Arcangeli * huge and small TLB entries for the same virtual address 2425ba76149fSAndrea Arcangeli * to avoid the risk of CPU bugs in that area. 2426ba76149fSAndrea Arcangeli */ 242715a25b2eSAneesh Kumar K.V _pmd = pmdp_collapse_flush(vma, address, pmd); 2428c4088ebdSKirill A. Shutemov spin_unlock(pmd_ptl); 24292ec74c3eSSagi Grimberg mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 2430ba76149fSAndrea Arcangeli 2431c4088ebdSKirill A. Shutemov spin_lock(pte_ptl); 2432ba76149fSAndrea Arcangeli isolated = __collapse_huge_page_isolate(vma, address, pte); 2433c4088ebdSKirill A. Shutemov spin_unlock(pte_ptl); 2434ba76149fSAndrea Arcangeli 2435ba76149fSAndrea Arcangeli if (unlikely(!isolated)) { 2436453c7192SJohannes Weiner pte_unmap(pte); 2437c4088ebdSKirill A. Shutemov spin_lock(pmd_ptl); 2438ba76149fSAndrea Arcangeli BUG_ON(!pmd_none(*pmd)); 24397c342512SAneesh Kumar K.V /* 24407c342512SAneesh Kumar K.V * We can only use set_pmd_at when establishing 24417c342512SAneesh Kumar K.V * hugepmds and never for establishing regular pmds that 24427c342512SAneesh Kumar K.V * points to regular pagetables. Use pmd_populate for that 24437c342512SAneesh Kumar K.V */ 24447c342512SAneesh Kumar K.V pmd_populate(mm, pmd, pmd_pgtable(_pmd)); 2445c4088ebdSKirill A. Shutemov spin_unlock(pmd_ptl); 244608b52706SKonstantin Khlebnikov anon_vma_unlock_write(vma->anon_vma); 24477d2eba05SEbru Akagunduz result = SCAN_FAIL; 2448ce83d217SAndrea Arcangeli goto out; 2449ba76149fSAndrea Arcangeli } 2450ba76149fSAndrea Arcangeli 2451ba76149fSAndrea Arcangeli /* 2452ba76149fSAndrea Arcangeli * All pages are isolated and locked so anon_vma rmap 2453ba76149fSAndrea Arcangeli * can't run anymore. 2454ba76149fSAndrea Arcangeli */ 245508b52706SKonstantin Khlebnikov anon_vma_unlock_write(vma->anon_vma); 2456ba76149fSAndrea Arcangeli 2457c4088ebdSKirill A. Shutemov __collapse_huge_page_copy(pte, new_page, vma, address, pte_ptl); 2458453c7192SJohannes Weiner pte_unmap(pte); 2459ba76149fSAndrea Arcangeli __SetPageUptodate(new_page); 2460ba76149fSAndrea Arcangeli pgtable = pmd_pgtable(_pmd); 2461ba76149fSAndrea Arcangeli 24623122359aSKirill A. Shutemov _pmd = mk_huge_pmd(new_page, vma->vm_page_prot); 24633122359aSKirill A. Shutemov _pmd = maybe_pmd_mkwrite(pmd_mkdirty(_pmd), vma); 2464ba76149fSAndrea Arcangeli 2465ba76149fSAndrea Arcangeli /* 2466ba76149fSAndrea Arcangeli * spin_lock() below is not the equivalent of smp_wmb(), so 2467ba76149fSAndrea Arcangeli * this is needed to avoid the copy_huge_page writes to become 2468ba76149fSAndrea Arcangeli * visible after the set_pmd_at() write. 2469ba76149fSAndrea Arcangeli */ 2470ba76149fSAndrea Arcangeli smp_wmb(); 2471ba76149fSAndrea Arcangeli 2472c4088ebdSKirill A. Shutemov spin_lock(pmd_ptl); 2473ba76149fSAndrea Arcangeli BUG_ON(!pmd_none(*pmd)); 2474d281ee61SKirill A. Shutemov page_add_new_anon_rmap(new_page, vma, address, true); 2475f627c2f5SKirill A. Shutemov mem_cgroup_commit_charge(new_page, memcg, false, true); 247600501b53SJohannes Weiner lru_cache_add_active_or_unevictable(new_page, vma); 2477fce144b4SAneesh Kumar K.V pgtable_trans_huge_deposit(mm, pmd, pgtable); 2478ba76149fSAndrea Arcangeli set_pmd_at(mm, address, pmd, _pmd); 2479b113da65SDavid Miller update_mmu_cache_pmd(vma, address, pmd); 2480c4088ebdSKirill A. Shutemov spin_unlock(pmd_ptl); 2481ba76149fSAndrea Arcangeli 2482ba76149fSAndrea Arcangeli *hpage = NULL; 2483420256efSXiao Guangrong 2484ba76149fSAndrea Arcangeli khugepaged_pages_collapsed++; 24857d2eba05SEbru Akagunduz result = SCAN_SUCCEED; 2486ce83d217SAndrea Arcangeli out_up_write: 2487ba76149fSAndrea Arcangeli up_write(&mm->mmap_sem); 24887d2eba05SEbru Akagunduz trace_mm_collapse_huge_page(mm, isolated, result); 24890bbbc0b3SAndrea Arcangeli return; 24900bbbc0b3SAndrea Arcangeli 24917d2eba05SEbru Akagunduz out_nolock: 24927d2eba05SEbru Akagunduz trace_mm_collapse_huge_page(mm, isolated, result); 24937d2eba05SEbru Akagunduz return; 2494ce83d217SAndrea Arcangeli out: 2495f627c2f5SKirill A. Shutemov mem_cgroup_cancel_charge(new_page, memcg, true); 2496ce83d217SAndrea Arcangeli goto out_up_write; 2497ba76149fSAndrea Arcangeli } 2498ba76149fSAndrea Arcangeli 2499ba76149fSAndrea Arcangeli static int khugepaged_scan_pmd(struct mm_struct *mm, 2500ba76149fSAndrea Arcangeli struct vm_area_struct *vma, 2501ba76149fSAndrea Arcangeli unsigned long address, 2502ba76149fSAndrea Arcangeli struct page **hpage) 2503ba76149fSAndrea Arcangeli { 2504ba76149fSAndrea Arcangeli pmd_t *pmd; 2505ba76149fSAndrea Arcangeli pte_t *pte, *_pte; 25067d2eba05SEbru Akagunduz int ret = 0, none_or_zero = 0, result = 0; 25077d2eba05SEbru Akagunduz struct page *page = NULL; 2508ba76149fSAndrea Arcangeli unsigned long _address; 2509ba76149fSAndrea Arcangeli spinlock_t *ptl; 251000ef2d2fSDavid Rientjes int node = NUMA_NO_NODE; 251110359213SEbru Akagunduz bool writable = false, referenced = false; 2512ba76149fSAndrea Arcangeli 2513ba76149fSAndrea Arcangeli VM_BUG_ON(address & ~HPAGE_PMD_MASK); 2514ba76149fSAndrea Arcangeli 25156219049aSBob Liu pmd = mm_find_pmd(mm, address); 25167d2eba05SEbru Akagunduz if (!pmd) { 25177d2eba05SEbru Akagunduz result = SCAN_PMD_NULL; 2518ba76149fSAndrea Arcangeli goto out; 25197d2eba05SEbru Akagunduz } 2520ba76149fSAndrea Arcangeli 25219f1b868aSBob Liu memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load)); 2522ba76149fSAndrea Arcangeli pte = pte_offset_map_lock(mm, pmd, address, &ptl); 2523ba76149fSAndrea Arcangeli for (_address = address, _pte = pte; _pte < pte+HPAGE_PMD_NR; 2524ba76149fSAndrea Arcangeli _pte++, _address += PAGE_SIZE) { 2525ba76149fSAndrea Arcangeli pte_t pteval = *_pte; 2526ca0984caSEbru Akagunduz if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) { 2527c1294d05SAndrea Arcangeli if (!userfaultfd_armed(vma) && 25287d2eba05SEbru Akagunduz ++none_or_zero <= khugepaged_max_ptes_none) { 2529ba76149fSAndrea Arcangeli continue; 25307d2eba05SEbru Akagunduz } else { 25317d2eba05SEbru Akagunduz result = SCAN_EXCEED_NONE_PTE; 2532ba76149fSAndrea Arcangeli goto out_unmap; 2533ba76149fSAndrea Arcangeli } 25347d2eba05SEbru Akagunduz } 25357d2eba05SEbru Akagunduz if (!pte_present(pteval)) { 25367d2eba05SEbru Akagunduz result = SCAN_PTE_NON_PRESENT; 2537ba76149fSAndrea Arcangeli goto out_unmap; 25387d2eba05SEbru Akagunduz } 253910359213SEbru Akagunduz if (pte_write(pteval)) 254010359213SEbru Akagunduz writable = true; 254110359213SEbru Akagunduz 2542ba76149fSAndrea Arcangeli page = vm_normal_page(vma, _address, pteval); 25437d2eba05SEbru Akagunduz if (unlikely(!page)) { 25447d2eba05SEbru Akagunduz result = SCAN_PAGE_NULL; 2545ba76149fSAndrea Arcangeli goto out_unmap; 25467d2eba05SEbru Akagunduz } 2547b1caa957SKirill A. Shutemov 2548b1caa957SKirill A. Shutemov /* TODO: teach khugepaged to collapse THP mapped with pte */ 2549b1caa957SKirill A. Shutemov if (PageCompound(page)) { 2550b1caa957SKirill A. Shutemov result = SCAN_PAGE_COMPOUND; 2551b1caa957SKirill A. Shutemov goto out_unmap; 2552b1caa957SKirill A. Shutemov } 2553b1caa957SKirill A. Shutemov 25545c4b4be3SAndi Kleen /* 25559f1b868aSBob Liu * Record which node the original page is from and save this 25569f1b868aSBob Liu * information to khugepaged_node_load[]. 25579f1b868aSBob Liu * Khupaged will allocate hugepage from the node has the max 25589f1b868aSBob Liu * hit record. 25595c4b4be3SAndi Kleen */ 25605c4b4be3SAndi Kleen node = page_to_nid(page); 25617d2eba05SEbru Akagunduz if (khugepaged_scan_abort(node)) { 25627d2eba05SEbru Akagunduz result = SCAN_SCAN_ABORT; 256314a4e214SDavid Rientjes goto out_unmap; 25647d2eba05SEbru Akagunduz } 25659f1b868aSBob Liu khugepaged_node_load[node]++; 25667d2eba05SEbru Akagunduz if (!PageLRU(page)) { 25670fda2788SKirill A. Shutemov result = SCAN_PAGE_LRU; 2568ba76149fSAndrea Arcangeli goto out_unmap; 25697d2eba05SEbru Akagunduz } 25707d2eba05SEbru Akagunduz if (PageLocked(page)) { 25717d2eba05SEbru Akagunduz result = SCAN_PAGE_LOCK; 25727d2eba05SEbru Akagunduz goto out_unmap; 25737d2eba05SEbru Akagunduz } 25747d2eba05SEbru Akagunduz if (!PageAnon(page)) { 25757d2eba05SEbru Akagunduz result = SCAN_PAGE_ANON; 25767d2eba05SEbru Akagunduz goto out_unmap; 25777d2eba05SEbru Akagunduz } 25787d2eba05SEbru Akagunduz 257910359213SEbru Akagunduz /* 258010359213SEbru Akagunduz * cannot use mapcount: can't collapse if there's a gup pin. 258110359213SEbru Akagunduz * The page must only be referenced by the scanned process 258210359213SEbru Akagunduz * and page swap cache. 258310359213SEbru Akagunduz */ 25847d2eba05SEbru Akagunduz if (page_count(page) != 1 + !!PageSwapCache(page)) { 25857d2eba05SEbru Akagunduz result = SCAN_PAGE_COUNT; 2586ba76149fSAndrea Arcangeli goto out_unmap; 25877d2eba05SEbru Akagunduz } 258833c3fc71SVladimir Davydov if (pte_young(pteval) || 258933c3fc71SVladimir Davydov page_is_young(page) || PageReferenced(page) || 25908ee53820SAndrea Arcangeli mmu_notifier_test_young(vma->vm_mm, address)) 259110359213SEbru Akagunduz referenced = true; 2592ba76149fSAndrea Arcangeli } 25937d2eba05SEbru Akagunduz if (writable) { 25947d2eba05SEbru Akagunduz if (referenced) { 25957d2eba05SEbru Akagunduz result = SCAN_SUCCEED; 2596ba76149fSAndrea Arcangeli ret = 1; 25977d2eba05SEbru Akagunduz } else { 25987d2eba05SEbru Akagunduz result = SCAN_NO_REFERENCED_PAGE; 25997d2eba05SEbru Akagunduz } 26007d2eba05SEbru Akagunduz } else { 26017d2eba05SEbru Akagunduz result = SCAN_PAGE_RO; 26027d2eba05SEbru Akagunduz } 2603ba76149fSAndrea Arcangeli out_unmap: 2604ba76149fSAndrea Arcangeli pte_unmap_unlock(pte, ptl); 26059f1b868aSBob Liu if (ret) { 26069f1b868aSBob Liu node = khugepaged_find_target_node(); 2607ce83d217SAndrea Arcangeli /* collapse_huge_page will return with the mmap_sem released */ 26085c4b4be3SAndi Kleen collapse_huge_page(mm, address, hpage, vma, node); 26099f1b868aSBob Liu } 2610ba76149fSAndrea Arcangeli out: 261116fd0fe4Syalin wang trace_mm_khugepaged_scan_pmd(mm, page, writable, referenced, 26127d2eba05SEbru Akagunduz none_or_zero, result); 2613ba76149fSAndrea Arcangeli return ret; 2614ba76149fSAndrea Arcangeli } 2615ba76149fSAndrea Arcangeli 2616ba76149fSAndrea Arcangeli static void collect_mm_slot(struct mm_slot *mm_slot) 2617ba76149fSAndrea Arcangeli { 2618ba76149fSAndrea Arcangeli struct mm_struct *mm = mm_slot->mm; 2619ba76149fSAndrea Arcangeli 2620b9980cdcSHugh Dickins VM_BUG_ON(NR_CPUS != 1 && !spin_is_locked(&khugepaged_mm_lock)); 2621ba76149fSAndrea Arcangeli 2622ba76149fSAndrea Arcangeli if (khugepaged_test_exit(mm)) { 2623ba76149fSAndrea Arcangeli /* free mm_slot */ 262443b5fbbdSSasha Levin hash_del(&mm_slot->hash); 2625ba76149fSAndrea Arcangeli list_del(&mm_slot->mm_node); 2626ba76149fSAndrea Arcangeli 2627ba76149fSAndrea Arcangeli /* 2628ba76149fSAndrea Arcangeli * Not strictly needed because the mm exited already. 2629ba76149fSAndrea Arcangeli * 2630ba76149fSAndrea Arcangeli * clear_bit(MMF_VM_HUGEPAGE, &mm->flags); 2631ba76149fSAndrea Arcangeli */ 2632ba76149fSAndrea Arcangeli 2633ba76149fSAndrea Arcangeli /* khugepaged_mm_lock actually not necessary for the below */ 2634ba76149fSAndrea Arcangeli free_mm_slot(mm_slot); 2635ba76149fSAndrea Arcangeli mmdrop(mm); 2636ba76149fSAndrea Arcangeli } 2637ba76149fSAndrea Arcangeli } 2638ba76149fSAndrea Arcangeli 2639ba76149fSAndrea Arcangeli static unsigned int khugepaged_scan_mm_slot(unsigned int pages, 2640ba76149fSAndrea Arcangeli struct page **hpage) 26412f1da642SH Hartley Sweeten __releases(&khugepaged_mm_lock) 26422f1da642SH Hartley Sweeten __acquires(&khugepaged_mm_lock) 2643ba76149fSAndrea Arcangeli { 2644ba76149fSAndrea Arcangeli struct mm_slot *mm_slot; 2645ba76149fSAndrea Arcangeli struct mm_struct *mm; 2646ba76149fSAndrea Arcangeli struct vm_area_struct *vma; 2647ba76149fSAndrea Arcangeli int progress = 0; 2648ba76149fSAndrea Arcangeli 2649ba76149fSAndrea Arcangeli VM_BUG_ON(!pages); 2650b9980cdcSHugh Dickins VM_BUG_ON(NR_CPUS != 1 && !spin_is_locked(&khugepaged_mm_lock)); 2651ba76149fSAndrea Arcangeli 2652ba76149fSAndrea Arcangeli if (khugepaged_scan.mm_slot) 2653ba76149fSAndrea Arcangeli mm_slot = khugepaged_scan.mm_slot; 2654ba76149fSAndrea Arcangeli else { 2655ba76149fSAndrea Arcangeli mm_slot = list_entry(khugepaged_scan.mm_head.next, 2656ba76149fSAndrea Arcangeli struct mm_slot, mm_node); 2657ba76149fSAndrea Arcangeli khugepaged_scan.address = 0; 2658ba76149fSAndrea Arcangeli khugepaged_scan.mm_slot = mm_slot; 2659ba76149fSAndrea Arcangeli } 2660ba76149fSAndrea Arcangeli spin_unlock(&khugepaged_mm_lock); 2661ba76149fSAndrea Arcangeli 2662ba76149fSAndrea Arcangeli mm = mm_slot->mm; 2663ba76149fSAndrea Arcangeli down_read(&mm->mmap_sem); 2664ba76149fSAndrea Arcangeli if (unlikely(khugepaged_test_exit(mm))) 2665ba76149fSAndrea Arcangeli vma = NULL; 2666ba76149fSAndrea Arcangeli else 2667ba76149fSAndrea Arcangeli vma = find_vma(mm, khugepaged_scan.address); 2668ba76149fSAndrea Arcangeli 2669ba76149fSAndrea Arcangeli progress++; 2670ba76149fSAndrea Arcangeli for (; vma; vma = vma->vm_next) { 2671ba76149fSAndrea Arcangeli unsigned long hstart, hend; 2672ba76149fSAndrea Arcangeli 2673ba76149fSAndrea Arcangeli cond_resched(); 2674ba76149fSAndrea Arcangeli if (unlikely(khugepaged_test_exit(mm))) { 2675ba76149fSAndrea Arcangeli progress++; 2676ba76149fSAndrea Arcangeli break; 2677ba76149fSAndrea Arcangeli } 2678fa475e51SBob Liu if (!hugepage_vma_check(vma)) { 2679a7d6e4ecSAndrea Arcangeli skip: 2680ba76149fSAndrea Arcangeli progress++; 2681ba76149fSAndrea Arcangeli continue; 2682ba76149fSAndrea Arcangeli } 2683ba76149fSAndrea Arcangeli hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK; 2684ba76149fSAndrea Arcangeli hend = vma->vm_end & HPAGE_PMD_MASK; 2685a7d6e4ecSAndrea Arcangeli if (hstart >= hend) 2686a7d6e4ecSAndrea Arcangeli goto skip; 2687a7d6e4ecSAndrea Arcangeli if (khugepaged_scan.address > hend) 2688a7d6e4ecSAndrea Arcangeli goto skip; 2689ba76149fSAndrea Arcangeli if (khugepaged_scan.address < hstart) 2690ba76149fSAndrea Arcangeli khugepaged_scan.address = hstart; 2691a7d6e4ecSAndrea Arcangeli VM_BUG_ON(khugepaged_scan.address & ~HPAGE_PMD_MASK); 2692ba76149fSAndrea Arcangeli 2693ba76149fSAndrea Arcangeli while (khugepaged_scan.address < hend) { 2694ba76149fSAndrea Arcangeli int ret; 2695ba76149fSAndrea Arcangeli cond_resched(); 2696ba76149fSAndrea Arcangeli if (unlikely(khugepaged_test_exit(mm))) 2697ba76149fSAndrea Arcangeli goto breakouterloop; 2698ba76149fSAndrea Arcangeli 2699ba76149fSAndrea Arcangeli VM_BUG_ON(khugepaged_scan.address < hstart || 2700ba76149fSAndrea Arcangeli khugepaged_scan.address + HPAGE_PMD_SIZE > 2701ba76149fSAndrea Arcangeli hend); 2702ba76149fSAndrea Arcangeli ret = khugepaged_scan_pmd(mm, vma, 2703ba76149fSAndrea Arcangeli khugepaged_scan.address, 2704ba76149fSAndrea Arcangeli hpage); 2705ba76149fSAndrea Arcangeli /* move to next address */ 2706ba76149fSAndrea Arcangeli khugepaged_scan.address += HPAGE_PMD_SIZE; 2707ba76149fSAndrea Arcangeli progress += HPAGE_PMD_NR; 2708ba76149fSAndrea Arcangeli if (ret) 2709ba76149fSAndrea Arcangeli /* we released mmap_sem so break loop */ 2710ba76149fSAndrea Arcangeli goto breakouterloop_mmap_sem; 2711ba76149fSAndrea Arcangeli if (progress >= pages) 2712ba76149fSAndrea Arcangeli goto breakouterloop; 2713ba76149fSAndrea Arcangeli } 2714ba76149fSAndrea Arcangeli } 2715ba76149fSAndrea Arcangeli breakouterloop: 2716ba76149fSAndrea Arcangeli up_read(&mm->mmap_sem); /* exit_mmap will destroy ptes after this */ 2717ba76149fSAndrea Arcangeli breakouterloop_mmap_sem: 2718ba76149fSAndrea Arcangeli 2719ba76149fSAndrea Arcangeli spin_lock(&khugepaged_mm_lock); 2720a7d6e4ecSAndrea Arcangeli VM_BUG_ON(khugepaged_scan.mm_slot != mm_slot); 2721ba76149fSAndrea Arcangeli /* 2722ba76149fSAndrea Arcangeli * Release the current mm_slot if this mm is about to die, or 2723ba76149fSAndrea Arcangeli * if we scanned all vmas of this mm. 2724ba76149fSAndrea Arcangeli */ 2725ba76149fSAndrea Arcangeli if (khugepaged_test_exit(mm) || !vma) { 2726ba76149fSAndrea Arcangeli /* 2727ba76149fSAndrea Arcangeli * Make sure that if mm_users is reaching zero while 2728ba76149fSAndrea Arcangeli * khugepaged runs here, khugepaged_exit will find 2729ba76149fSAndrea Arcangeli * mm_slot not pointing to the exiting mm. 2730ba76149fSAndrea Arcangeli */ 2731ba76149fSAndrea Arcangeli if (mm_slot->mm_node.next != &khugepaged_scan.mm_head) { 2732ba76149fSAndrea Arcangeli khugepaged_scan.mm_slot = list_entry( 2733ba76149fSAndrea Arcangeli mm_slot->mm_node.next, 2734ba76149fSAndrea Arcangeli struct mm_slot, mm_node); 2735ba76149fSAndrea Arcangeli khugepaged_scan.address = 0; 2736ba76149fSAndrea Arcangeli } else { 2737ba76149fSAndrea Arcangeli khugepaged_scan.mm_slot = NULL; 2738ba76149fSAndrea Arcangeli khugepaged_full_scans++; 2739ba76149fSAndrea Arcangeli } 2740ba76149fSAndrea Arcangeli 2741ba76149fSAndrea Arcangeli collect_mm_slot(mm_slot); 2742ba76149fSAndrea Arcangeli } 2743ba76149fSAndrea Arcangeli 2744ba76149fSAndrea Arcangeli return progress; 2745ba76149fSAndrea Arcangeli } 2746ba76149fSAndrea Arcangeli 2747ba76149fSAndrea Arcangeli static int khugepaged_has_work(void) 2748ba76149fSAndrea Arcangeli { 2749ba76149fSAndrea Arcangeli return !list_empty(&khugepaged_scan.mm_head) && 2750ba76149fSAndrea Arcangeli khugepaged_enabled(); 2751ba76149fSAndrea Arcangeli } 2752ba76149fSAndrea Arcangeli 2753ba76149fSAndrea Arcangeli static int khugepaged_wait_event(void) 2754ba76149fSAndrea Arcangeli { 2755ba76149fSAndrea Arcangeli return !list_empty(&khugepaged_scan.mm_head) || 27562017c0bfSXiao Guangrong kthread_should_stop(); 2757ba76149fSAndrea Arcangeli } 2758ba76149fSAndrea Arcangeli 2759d516904bSXiao Guangrong static void khugepaged_do_scan(void) 2760d516904bSXiao Guangrong { 2761d516904bSXiao Guangrong struct page *hpage = NULL; 2762ba76149fSAndrea Arcangeli unsigned int progress = 0, pass_through_head = 0; 2763ba76149fSAndrea Arcangeli unsigned int pages = khugepaged_pages_to_scan; 2764d516904bSXiao Guangrong bool wait = true; 2765ba76149fSAndrea Arcangeli 2766ba76149fSAndrea Arcangeli barrier(); /* write khugepaged_pages_to_scan to local stack */ 2767ba76149fSAndrea Arcangeli 2768ba76149fSAndrea Arcangeli while (progress < pages) { 276926234f36SXiao Guangrong if (!khugepaged_prealloc_page(&hpage, &wait)) 277026234f36SXiao Guangrong break; 2771d516904bSXiao Guangrong 2772420256efSXiao Guangrong cond_resched(); 2773ba76149fSAndrea Arcangeli 2774cd092411SJiri Kosina if (unlikely(kthread_should_stop() || try_to_freeze())) 2775878aee7dSAndrea Arcangeli break; 2776878aee7dSAndrea Arcangeli 2777ba76149fSAndrea Arcangeli spin_lock(&khugepaged_mm_lock); 2778ba76149fSAndrea Arcangeli if (!khugepaged_scan.mm_slot) 2779ba76149fSAndrea Arcangeli pass_through_head++; 2780ba76149fSAndrea Arcangeli if (khugepaged_has_work() && 2781ba76149fSAndrea Arcangeli pass_through_head < 2) 2782ba76149fSAndrea Arcangeli progress += khugepaged_scan_mm_slot(pages - progress, 2783d516904bSXiao Guangrong &hpage); 2784ba76149fSAndrea Arcangeli else 2785ba76149fSAndrea Arcangeli progress = pages; 2786ba76149fSAndrea Arcangeli spin_unlock(&khugepaged_mm_lock); 2787ba76149fSAndrea Arcangeli } 2788ba76149fSAndrea Arcangeli 2789d516904bSXiao Guangrong if (!IS_ERR_OR_NULL(hpage)) 2790d516904bSXiao Guangrong put_page(hpage); 2791ba76149fSAndrea Arcangeli } 27920bbbc0b3SAndrea Arcangeli 2793f0508977SDavid Rientjes static bool khugepaged_should_wakeup(void) 2794f0508977SDavid Rientjes { 2795f0508977SDavid Rientjes return kthread_should_stop() || 2796f0508977SDavid Rientjes time_after_eq(jiffies, khugepaged_sleep_expire); 2797f0508977SDavid Rientjes } 2798f0508977SDavid Rientjes 27992017c0bfSXiao Guangrong static void khugepaged_wait_work(void) 28002017c0bfSXiao Guangrong { 28012017c0bfSXiao Guangrong if (khugepaged_has_work()) { 2802f0508977SDavid Rientjes const unsigned long scan_sleep_jiffies = 2803f0508977SDavid Rientjes msecs_to_jiffies(khugepaged_scan_sleep_millisecs); 2804f0508977SDavid Rientjes 2805f0508977SDavid Rientjes if (!scan_sleep_jiffies) 28062017c0bfSXiao Guangrong return; 28072017c0bfSXiao Guangrong 2808f0508977SDavid Rientjes khugepaged_sleep_expire = jiffies + scan_sleep_jiffies; 28092017c0bfSXiao Guangrong wait_event_freezable_timeout(khugepaged_wait, 2810f0508977SDavid Rientjes khugepaged_should_wakeup(), 2811f0508977SDavid Rientjes scan_sleep_jiffies); 28122017c0bfSXiao Guangrong return; 28132017c0bfSXiao Guangrong } 28142017c0bfSXiao Guangrong 28152017c0bfSXiao Guangrong if (khugepaged_enabled()) 28162017c0bfSXiao Guangrong wait_event_freezable(khugepaged_wait, khugepaged_wait_event()); 28172017c0bfSXiao Guangrong } 28182017c0bfSXiao Guangrong 2819ba76149fSAndrea Arcangeli static int khugepaged(void *none) 2820ba76149fSAndrea Arcangeli { 2821ba76149fSAndrea Arcangeli struct mm_slot *mm_slot; 2822ba76149fSAndrea Arcangeli 2823878aee7dSAndrea Arcangeli set_freezable(); 28248698a745SDongsheng Yang set_user_nice(current, MAX_NICE); 2825ba76149fSAndrea Arcangeli 2826b7231789SXiao Guangrong while (!kthread_should_stop()) { 2827b7231789SXiao Guangrong khugepaged_do_scan(); 2828b7231789SXiao Guangrong khugepaged_wait_work(); 2829b7231789SXiao Guangrong } 2830ba76149fSAndrea Arcangeli 2831ba76149fSAndrea Arcangeli spin_lock(&khugepaged_mm_lock); 2832ba76149fSAndrea Arcangeli mm_slot = khugepaged_scan.mm_slot; 2833ba76149fSAndrea Arcangeli khugepaged_scan.mm_slot = NULL; 2834ba76149fSAndrea Arcangeli if (mm_slot) 2835ba76149fSAndrea Arcangeli collect_mm_slot(mm_slot); 2836ba76149fSAndrea Arcangeli spin_unlock(&khugepaged_mm_lock); 2837ba76149fSAndrea Arcangeli return 0; 2838ba76149fSAndrea Arcangeli } 2839ba76149fSAndrea Arcangeli 2840eef1b3baSKirill A. Shutemov static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, 2841eef1b3baSKirill A. Shutemov unsigned long haddr, pmd_t *pmd) 2842eef1b3baSKirill A. Shutemov { 2843eef1b3baSKirill A. Shutemov struct mm_struct *mm = vma->vm_mm; 2844eef1b3baSKirill A. Shutemov pgtable_t pgtable; 2845eef1b3baSKirill A. Shutemov pmd_t _pmd; 2846eef1b3baSKirill A. Shutemov int i; 2847eef1b3baSKirill A. Shutemov 2848eef1b3baSKirill A. Shutemov /* leave pmd empty until pte is filled */ 2849eef1b3baSKirill A. Shutemov pmdp_huge_clear_flush_notify(vma, haddr, pmd); 2850eef1b3baSKirill A. Shutemov 2851eef1b3baSKirill A. Shutemov pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2852eef1b3baSKirill A. Shutemov pmd_populate(mm, &_pmd, pgtable); 2853eef1b3baSKirill A. Shutemov 2854eef1b3baSKirill A. Shutemov for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) { 2855eef1b3baSKirill A. Shutemov pte_t *pte, entry; 2856eef1b3baSKirill A. Shutemov entry = pfn_pte(my_zero_pfn(haddr), vma->vm_page_prot); 2857eef1b3baSKirill A. Shutemov entry = pte_mkspecial(entry); 2858eef1b3baSKirill A. Shutemov pte = pte_offset_map(&_pmd, haddr); 2859eef1b3baSKirill A. Shutemov VM_BUG_ON(!pte_none(*pte)); 2860eef1b3baSKirill A. Shutemov set_pte_at(mm, haddr, pte, entry); 2861eef1b3baSKirill A. Shutemov pte_unmap(pte); 2862eef1b3baSKirill A. Shutemov } 2863eef1b3baSKirill A. Shutemov smp_wmb(); /* make pte visible before pmd */ 2864eef1b3baSKirill A. Shutemov pmd_populate(mm, pmd, pgtable); 2865eef1b3baSKirill A. Shutemov put_huge_zero_page(); 2866eef1b3baSKirill A. Shutemov } 2867eef1b3baSKirill A. Shutemov 2868eef1b3baSKirill A. Shutemov static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, 2869ba988280SKirill A. Shutemov unsigned long haddr, bool freeze) 2870eef1b3baSKirill A. Shutemov { 2871eef1b3baSKirill A. Shutemov struct mm_struct *mm = vma->vm_mm; 2872eef1b3baSKirill A. Shutemov struct page *page; 2873eef1b3baSKirill A. Shutemov pgtable_t pgtable; 2874eef1b3baSKirill A. Shutemov pmd_t _pmd; 2875b8d3c4c3SMinchan Kim bool young, write, dirty; 28762ac015e2SKirill A. Shutemov unsigned long addr; 2877eef1b3baSKirill A. Shutemov int i; 2878eef1b3baSKirill A. Shutemov 2879eef1b3baSKirill A. Shutemov VM_BUG_ON(haddr & ~HPAGE_PMD_MASK); 2880eef1b3baSKirill A. Shutemov VM_BUG_ON_VMA(vma->vm_start > haddr, vma); 2881eef1b3baSKirill A. Shutemov VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma); 28825c7fb56eSDan Williams VM_BUG_ON(!pmd_trans_huge(*pmd) && !pmd_devmap(*pmd)); 2883eef1b3baSKirill A. Shutemov 2884eef1b3baSKirill A. Shutemov count_vm_event(THP_SPLIT_PMD); 2885eef1b3baSKirill A. Shutemov 2886eef1b3baSKirill A. Shutemov if (vma_is_dax(vma)) { 2887eef1b3baSKirill A. Shutemov pmd_t _pmd = pmdp_huge_clear_flush_notify(vma, haddr, pmd); 2888eef1b3baSKirill A. Shutemov if (is_huge_zero_pmd(_pmd)) 2889eef1b3baSKirill A. Shutemov put_huge_zero_page(); 2890eef1b3baSKirill A. Shutemov return; 2891eef1b3baSKirill A. Shutemov } else if (is_huge_zero_pmd(*pmd)) { 2892eef1b3baSKirill A. Shutemov return __split_huge_zero_page_pmd(vma, haddr, pmd); 2893eef1b3baSKirill A. Shutemov } 2894eef1b3baSKirill A. Shutemov 2895eef1b3baSKirill A. Shutemov page = pmd_page(*pmd); 2896eef1b3baSKirill A. Shutemov VM_BUG_ON_PAGE(!page_count(page), page); 2897fe896d18SJoonsoo Kim page_ref_add(page, HPAGE_PMD_NR - 1); 2898eef1b3baSKirill A. Shutemov write = pmd_write(*pmd); 2899eef1b3baSKirill A. Shutemov young = pmd_young(*pmd); 2900b8d3c4c3SMinchan Kim dirty = pmd_dirty(*pmd); 2901eef1b3baSKirill A. Shutemov 2902c777e2a8SAneesh Kumar K.V pmdp_huge_split_prepare(vma, haddr, pmd); 2903eef1b3baSKirill A. Shutemov pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2904eef1b3baSKirill A. Shutemov pmd_populate(mm, &_pmd, pgtable); 2905eef1b3baSKirill A. Shutemov 29062ac015e2SKirill A. Shutemov for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) { 2907eef1b3baSKirill A. Shutemov pte_t entry, *pte; 2908eef1b3baSKirill A. Shutemov /* 2909eef1b3baSKirill A. Shutemov * Note that NUMA hinting access restrictions are not 2910eef1b3baSKirill A. Shutemov * transferred to avoid any possibility of altering 2911eef1b3baSKirill A. Shutemov * permissions across VMAs. 2912eef1b3baSKirill A. Shutemov */ 2913ba988280SKirill A. Shutemov if (freeze) { 2914ba988280SKirill A. Shutemov swp_entry_t swp_entry; 2915ba988280SKirill A. Shutemov swp_entry = make_migration_entry(page + i, write); 2916ba988280SKirill A. Shutemov entry = swp_entry_to_pte(swp_entry); 2917ba988280SKirill A. Shutemov } else { 2918eef1b3baSKirill A. Shutemov entry = mk_pte(page + i, vma->vm_page_prot); 2919b8d3c4c3SMinchan Kim entry = maybe_mkwrite(entry, vma); 2920eef1b3baSKirill A. Shutemov if (!write) 2921eef1b3baSKirill A. Shutemov entry = pte_wrprotect(entry); 2922eef1b3baSKirill A. Shutemov if (!young) 2923eef1b3baSKirill A. Shutemov entry = pte_mkold(entry); 2924ba988280SKirill A. Shutemov } 2925b8d3c4c3SMinchan Kim if (dirty) 2926b8d3c4c3SMinchan Kim SetPageDirty(page + i); 29272ac015e2SKirill A. Shutemov pte = pte_offset_map(&_pmd, addr); 2928eef1b3baSKirill A. Shutemov BUG_ON(!pte_none(*pte)); 29292ac015e2SKirill A. Shutemov set_pte_at(mm, addr, pte, entry); 2930eef1b3baSKirill A. Shutemov atomic_inc(&page[i]._mapcount); 2931eef1b3baSKirill A. Shutemov pte_unmap(pte); 2932eef1b3baSKirill A. Shutemov } 2933eef1b3baSKirill A. Shutemov 2934eef1b3baSKirill A. Shutemov /* 2935eef1b3baSKirill A. Shutemov * Set PG_double_map before dropping compound_mapcount to avoid 2936eef1b3baSKirill A. Shutemov * false-negative page_mapped(). 2937eef1b3baSKirill A. Shutemov */ 2938eef1b3baSKirill A. Shutemov if (compound_mapcount(page) > 1 && !TestSetPageDoubleMap(page)) { 2939eef1b3baSKirill A. Shutemov for (i = 0; i < HPAGE_PMD_NR; i++) 2940eef1b3baSKirill A. Shutemov atomic_inc(&page[i]._mapcount); 2941eef1b3baSKirill A. Shutemov } 2942eef1b3baSKirill A. Shutemov 2943eef1b3baSKirill A. Shutemov if (atomic_add_negative(-1, compound_mapcount_ptr(page))) { 2944eef1b3baSKirill A. Shutemov /* Last compound_mapcount is gone. */ 2945eef1b3baSKirill A. Shutemov __dec_zone_page_state(page, NR_ANON_TRANSPARENT_HUGEPAGES); 2946eef1b3baSKirill A. Shutemov if (TestClearPageDoubleMap(page)) { 2947eef1b3baSKirill A. Shutemov /* No need in mapcount reference anymore */ 2948eef1b3baSKirill A. Shutemov for (i = 0; i < HPAGE_PMD_NR; i++) 2949eef1b3baSKirill A. Shutemov atomic_dec(&page[i]._mapcount); 2950eef1b3baSKirill A. Shutemov } 2951eef1b3baSKirill A. Shutemov } 2952eef1b3baSKirill A. Shutemov 2953eef1b3baSKirill A. Shutemov smp_wmb(); /* make pte visible before pmd */ 2954e9b61f19SKirill A. Shutemov /* 2955e9b61f19SKirill A. Shutemov * Up to this point the pmd is present and huge and userland has the 2956e9b61f19SKirill A. Shutemov * whole access to the hugepage during the split (which happens in 2957e9b61f19SKirill A. Shutemov * place). If we overwrite the pmd with the not-huge version pointing 2958e9b61f19SKirill A. Shutemov * to the pte here (which of course we could if all CPUs were bug 2959e9b61f19SKirill A. Shutemov * free), userland could trigger a small page size TLB miss on the 2960e9b61f19SKirill A. Shutemov * small sized TLB while the hugepage TLB entry is still established in 2961e9b61f19SKirill A. Shutemov * the huge TLB. Some CPU doesn't like that. 2962e9b61f19SKirill A. Shutemov * See http://support.amd.com/us/Processor_TechDocs/41322.pdf, Erratum 2963e9b61f19SKirill A. Shutemov * 383 on page 93. Intel should be safe but is also warns that it's 2964e9b61f19SKirill A. Shutemov * only safe if the permission and cache attributes of the two entries 2965e9b61f19SKirill A. Shutemov * loaded in the two TLB is identical (which should be the case here). 2966e9b61f19SKirill A. Shutemov * But it is generally safer to never allow small and huge TLB entries 2967e9b61f19SKirill A. Shutemov * for the same virtual address to be loaded simultaneously. So instead 2968e9b61f19SKirill A. Shutemov * of doing "pmd_populate(); flush_pmd_tlb_range();" we first mark the 2969e9b61f19SKirill A. Shutemov * current pmd notpresent (atomically because here the pmd_trans_huge 2970e9b61f19SKirill A. Shutemov * and pmd_trans_splitting must remain set at all times on the pmd 2971e9b61f19SKirill A. Shutemov * until the split is complete for this pmd), then we flush the SMP TLB 2972e9b61f19SKirill A. Shutemov * and finally we write the non-huge version of the pmd entry with 2973e9b61f19SKirill A. Shutemov * pmd_populate. 2974e9b61f19SKirill A. Shutemov */ 2975e9b61f19SKirill A. Shutemov pmdp_invalidate(vma, haddr, pmd); 2976eef1b3baSKirill A. Shutemov pmd_populate(mm, pmd, pgtable); 2977e9b61f19SKirill A. Shutemov 2978e9b61f19SKirill A. Shutemov if (freeze) { 29792ac015e2SKirill A. Shutemov for (i = 0; i < HPAGE_PMD_NR; i++) { 2980e9b61f19SKirill A. Shutemov page_remove_rmap(page + i, false); 2981e9b61f19SKirill A. Shutemov put_page(page + i); 2982e9b61f19SKirill A. Shutemov } 2983e9b61f19SKirill A. Shutemov } 2984eef1b3baSKirill A. Shutemov } 2985eef1b3baSKirill A. Shutemov 2986eef1b3baSKirill A. Shutemov void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, 2987*33f4751eSNaoya Horiguchi unsigned long address, bool freeze, struct page *page) 2988eef1b3baSKirill A. Shutemov { 2989eef1b3baSKirill A. Shutemov spinlock_t *ptl; 2990eef1b3baSKirill A. Shutemov struct mm_struct *mm = vma->vm_mm; 2991eef1b3baSKirill A. Shutemov unsigned long haddr = address & HPAGE_PMD_MASK; 2992eef1b3baSKirill A. Shutemov 2993eef1b3baSKirill A. Shutemov mmu_notifier_invalidate_range_start(mm, haddr, haddr + HPAGE_PMD_SIZE); 2994eef1b3baSKirill A. Shutemov ptl = pmd_lock(mm, pmd); 2995*33f4751eSNaoya Horiguchi 2996*33f4751eSNaoya Horiguchi /* 2997*33f4751eSNaoya Horiguchi * If caller asks to setup a migration entries, we need a page to check 2998*33f4751eSNaoya Horiguchi * pmd against. Otherwise we can end up replacing wrong page. 2999*33f4751eSNaoya Horiguchi */ 3000*33f4751eSNaoya Horiguchi VM_BUG_ON(freeze && !page); 3001*33f4751eSNaoya Horiguchi if (page && page != pmd_page(*pmd)) 3002*33f4751eSNaoya Horiguchi goto out; 3003*33f4751eSNaoya Horiguchi 30045c7fb56eSDan Williams if (pmd_trans_huge(*pmd)) { 3005*33f4751eSNaoya Horiguchi page = pmd_page(*pmd); 3006e90309c9SKirill A. Shutemov if (PageMlocked(page)) 30075f737714SKirill A. Shutemov clear_page_mlock(page); 30085c7fb56eSDan Williams } else if (!pmd_devmap(*pmd)) 30095c7fb56eSDan Williams goto out; 3010fec89c10SKirill A. Shutemov __split_huge_pmd_locked(vma, pmd, haddr, freeze); 3011e90309c9SKirill A. Shutemov out: 3012eef1b3baSKirill A. Shutemov spin_unlock(ptl); 3013eef1b3baSKirill A. Shutemov mmu_notifier_invalidate_range_end(mm, haddr, haddr + HPAGE_PMD_SIZE); 3014eef1b3baSKirill A. Shutemov } 3015eef1b3baSKirill A. Shutemov 3016fec89c10SKirill A. Shutemov void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address, 3017fec89c10SKirill A. Shutemov bool freeze, struct page *page) 301894fcc585SAndrea Arcangeli { 3019f72e7dcdSHugh Dickins pgd_t *pgd; 3020f72e7dcdSHugh Dickins pud_t *pud; 302194fcc585SAndrea Arcangeli pmd_t *pmd; 302294fcc585SAndrea Arcangeli 302378ddc534SKirill A. Shutemov pgd = pgd_offset(vma->vm_mm, address); 3024f72e7dcdSHugh Dickins if (!pgd_present(*pgd)) 3025f72e7dcdSHugh Dickins return; 3026f72e7dcdSHugh Dickins 3027f72e7dcdSHugh Dickins pud = pud_offset(pgd, address); 3028f72e7dcdSHugh Dickins if (!pud_present(*pud)) 3029f72e7dcdSHugh Dickins return; 3030f72e7dcdSHugh Dickins 3031f72e7dcdSHugh Dickins pmd = pmd_offset(pud, address); 3032fec89c10SKirill A. Shutemov 3033*33f4751eSNaoya Horiguchi __split_huge_pmd(vma, pmd, address, freeze, page); 303494fcc585SAndrea Arcangeli } 303594fcc585SAndrea Arcangeli 3036e1b9996bSKirill A. Shutemov void vma_adjust_trans_huge(struct vm_area_struct *vma, 303794fcc585SAndrea Arcangeli unsigned long start, 303894fcc585SAndrea Arcangeli unsigned long end, 303994fcc585SAndrea Arcangeli long adjust_next) 304094fcc585SAndrea Arcangeli { 304194fcc585SAndrea Arcangeli /* 304294fcc585SAndrea Arcangeli * If the new start address isn't hpage aligned and it could 304394fcc585SAndrea Arcangeli * previously contain an hugepage: check if we need to split 304494fcc585SAndrea Arcangeli * an huge pmd. 304594fcc585SAndrea Arcangeli */ 304694fcc585SAndrea Arcangeli if (start & ~HPAGE_PMD_MASK && 304794fcc585SAndrea Arcangeli (start & HPAGE_PMD_MASK) >= vma->vm_start && 304894fcc585SAndrea Arcangeli (start & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end) 3049fec89c10SKirill A. Shutemov split_huge_pmd_address(vma, start, false, NULL); 305094fcc585SAndrea Arcangeli 305194fcc585SAndrea Arcangeli /* 305294fcc585SAndrea Arcangeli * If the new end address isn't hpage aligned and it could 305394fcc585SAndrea Arcangeli * previously contain an hugepage: check if we need to split 305494fcc585SAndrea Arcangeli * an huge pmd. 305594fcc585SAndrea Arcangeli */ 305694fcc585SAndrea Arcangeli if (end & ~HPAGE_PMD_MASK && 305794fcc585SAndrea Arcangeli (end & HPAGE_PMD_MASK) >= vma->vm_start && 305894fcc585SAndrea Arcangeli (end & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end) 3059fec89c10SKirill A. Shutemov split_huge_pmd_address(vma, end, false, NULL); 306094fcc585SAndrea Arcangeli 306194fcc585SAndrea Arcangeli /* 306294fcc585SAndrea Arcangeli * If we're also updating the vma->vm_next->vm_start, if the new 306394fcc585SAndrea Arcangeli * vm_next->vm_start isn't page aligned and it could previously 306494fcc585SAndrea Arcangeli * contain an hugepage: check if we need to split an huge pmd. 306594fcc585SAndrea Arcangeli */ 306694fcc585SAndrea Arcangeli if (adjust_next > 0) { 306794fcc585SAndrea Arcangeli struct vm_area_struct *next = vma->vm_next; 306894fcc585SAndrea Arcangeli unsigned long nstart = next->vm_start; 306994fcc585SAndrea Arcangeli nstart += adjust_next << PAGE_SHIFT; 307094fcc585SAndrea Arcangeli if (nstart & ~HPAGE_PMD_MASK && 307194fcc585SAndrea Arcangeli (nstart & HPAGE_PMD_MASK) >= next->vm_start && 307294fcc585SAndrea Arcangeli (nstart & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= next->vm_end) 3073fec89c10SKirill A. Shutemov split_huge_pmd_address(next, nstart, false, NULL); 307494fcc585SAndrea Arcangeli } 307594fcc585SAndrea Arcangeli } 3076e9b61f19SKirill A. Shutemov 3077fec89c10SKirill A. Shutemov static void freeze_page(struct page *page) 3078e9b61f19SKirill A. Shutemov { 3079fec89c10SKirill A. Shutemov enum ttu_flags ttu_flags = TTU_MIGRATION | TTU_IGNORE_MLOCK | 3080fec89c10SKirill A. Shutemov TTU_IGNORE_ACCESS | TTU_RMAP_LOCKED; 3081fec89c10SKirill A. Shutemov int i, ret; 3082e9b61f19SKirill A. Shutemov 3083e9b61f19SKirill A. Shutemov VM_BUG_ON_PAGE(!PageHead(page), page); 3084e9b61f19SKirill A. Shutemov 3085fec89c10SKirill A. Shutemov /* We only need TTU_SPLIT_HUGE_PMD once */ 3086fec89c10SKirill A. Shutemov ret = try_to_unmap(page, ttu_flags | TTU_SPLIT_HUGE_PMD); 3087fec89c10SKirill A. Shutemov for (i = 1; !ret && i < HPAGE_PMD_NR; i++) { 3088fec89c10SKirill A. Shutemov /* Cut short if the page is unmapped */ 3089fec89c10SKirill A. Shutemov if (page_count(page) == 1) 3090e9b61f19SKirill A. Shutemov return; 3091bd56086fSKirill A. Shutemov 3092fec89c10SKirill A. Shutemov ret = try_to_unmap(page + i, ttu_flags); 3093fec89c10SKirill A. Shutemov } 3094fec89c10SKirill A. Shutemov VM_BUG_ON(ret); 3095bd56086fSKirill A. Shutemov } 3096bd56086fSKirill A. Shutemov 3097fec89c10SKirill A. Shutemov static void unfreeze_page(struct page *page) 3098e9b61f19SKirill A. Shutemov { 3099fec89c10SKirill A. Shutemov int i; 3100e9b61f19SKirill A. Shutemov 3101fec89c10SKirill A. Shutemov for (i = 0; i < HPAGE_PMD_NR; i++) 3102fec89c10SKirill A. Shutemov remove_migration_ptes(page + i, page + i, true); 3103e9b61f19SKirill A. Shutemov } 3104e9b61f19SKirill A. Shutemov 31058df651c7SKirill A. Shutemov static void __split_huge_page_tail(struct page *head, int tail, 3106e9b61f19SKirill A. Shutemov struct lruvec *lruvec, struct list_head *list) 3107e9b61f19SKirill A. Shutemov { 3108e9b61f19SKirill A. Shutemov struct page *page_tail = head + tail; 3109e9b61f19SKirill A. Shutemov 31108df651c7SKirill A. Shutemov VM_BUG_ON_PAGE(atomic_read(&page_tail->_mapcount) != -1, page_tail); 3111fe896d18SJoonsoo Kim VM_BUG_ON_PAGE(page_ref_count(page_tail) != 0, page_tail); 3112e9b61f19SKirill A. Shutemov 3113e9b61f19SKirill A. Shutemov /* 31140139aa7bSJoonsoo Kim * tail_page->_refcount is zero and not changing from under us. But 3115e9b61f19SKirill A. Shutemov * get_page_unless_zero() may be running from under us on the 31168df651c7SKirill A. Shutemov * tail_page. If we used atomic_set() below instead of atomic_inc(), we 3117e9b61f19SKirill A. Shutemov * would then run atomic_set() concurrently with 3118e9b61f19SKirill A. Shutemov * get_page_unless_zero(), and atomic_set() is implemented in C not 3119e9b61f19SKirill A. Shutemov * using locked ops. spin_unlock on x86 sometime uses locked ops 3120e9b61f19SKirill A. Shutemov * because of PPro errata 66, 92, so unless somebody can guarantee 3121e9b61f19SKirill A. Shutemov * atomic_set() here would be safe on all archs (and not only on x86), 31228df651c7SKirill A. Shutemov * it's safer to use atomic_inc(). 3123e9b61f19SKirill A. Shutemov */ 3124fe896d18SJoonsoo Kim page_ref_inc(page_tail); 3125e9b61f19SKirill A. Shutemov 3126e9b61f19SKirill A. Shutemov page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 3127e9b61f19SKirill A. Shutemov page_tail->flags |= (head->flags & 3128e9b61f19SKirill A. Shutemov ((1L << PG_referenced) | 3129e9b61f19SKirill A. Shutemov (1L << PG_swapbacked) | 3130e9b61f19SKirill A. Shutemov (1L << PG_mlocked) | 3131e9b61f19SKirill A. Shutemov (1L << PG_uptodate) | 3132e9b61f19SKirill A. Shutemov (1L << PG_active) | 3133e9b61f19SKirill A. Shutemov (1L << PG_locked) | 3134b8d3c4c3SMinchan Kim (1L << PG_unevictable) | 3135b8d3c4c3SMinchan Kim (1L << PG_dirty))); 3136e9b61f19SKirill A. Shutemov 3137e9b61f19SKirill A. Shutemov /* 3138e9b61f19SKirill A. Shutemov * After clearing PageTail the gup refcount can be released. 3139e9b61f19SKirill A. Shutemov * Page flags also must be visible before we make the page non-compound. 3140e9b61f19SKirill A. Shutemov */ 3141e9b61f19SKirill A. Shutemov smp_wmb(); 3142e9b61f19SKirill A. Shutemov 3143e9b61f19SKirill A. Shutemov clear_compound_head(page_tail); 3144e9b61f19SKirill A. Shutemov 3145e9b61f19SKirill A. Shutemov if (page_is_young(head)) 3146e9b61f19SKirill A. Shutemov set_page_young(page_tail); 3147e9b61f19SKirill A. Shutemov if (page_is_idle(head)) 3148e9b61f19SKirill A. Shutemov set_page_idle(page_tail); 3149e9b61f19SKirill A. Shutemov 3150e9b61f19SKirill A. Shutemov /* ->mapping in first tail page is compound_mapcount */ 31519a982250SKirill A. Shutemov VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING, 3152e9b61f19SKirill A. Shutemov page_tail); 3153e9b61f19SKirill A. Shutemov page_tail->mapping = head->mapping; 3154e9b61f19SKirill A. Shutemov 3155e9b61f19SKirill A. Shutemov page_tail->index = head->index + tail; 3156e9b61f19SKirill A. Shutemov page_cpupid_xchg_last(page_tail, page_cpupid_last(head)); 3157e9b61f19SKirill A. Shutemov lru_add_page_tail(head, page_tail, lruvec, list); 3158e9b61f19SKirill A. Shutemov } 3159e9b61f19SKirill A. Shutemov 3160e9b61f19SKirill A. Shutemov static void __split_huge_page(struct page *page, struct list_head *list) 3161e9b61f19SKirill A. Shutemov { 3162e9b61f19SKirill A. Shutemov struct page *head = compound_head(page); 3163e9b61f19SKirill A. Shutemov struct zone *zone = page_zone(head); 3164e9b61f19SKirill A. Shutemov struct lruvec *lruvec; 31658df651c7SKirill A. Shutemov int i; 3166e9b61f19SKirill A. Shutemov 3167e9b61f19SKirill A. Shutemov /* prevent PageLRU to go away from under us, and freeze lru stats */ 3168e9b61f19SKirill A. Shutemov spin_lock_irq(&zone->lru_lock); 3169e9b61f19SKirill A. Shutemov lruvec = mem_cgroup_page_lruvec(head, zone); 3170e9b61f19SKirill A. Shutemov 3171e9b61f19SKirill A. Shutemov /* complete memcg works before add pages to LRU */ 3172e9b61f19SKirill A. Shutemov mem_cgroup_split_huge_fixup(head); 3173e9b61f19SKirill A. Shutemov 3174e9b61f19SKirill A. Shutemov for (i = HPAGE_PMD_NR - 1; i >= 1; i--) 31758df651c7SKirill A. Shutemov __split_huge_page_tail(head, i, lruvec, list); 3176e9b61f19SKirill A. Shutemov 3177e9b61f19SKirill A. Shutemov ClearPageCompound(head); 3178e9b61f19SKirill A. Shutemov spin_unlock_irq(&zone->lru_lock); 3179e9b61f19SKirill A. Shutemov 3180fec89c10SKirill A. Shutemov unfreeze_page(head); 3181e9b61f19SKirill A. Shutemov 3182e9b61f19SKirill A. Shutemov for (i = 0; i < HPAGE_PMD_NR; i++) { 3183e9b61f19SKirill A. Shutemov struct page *subpage = head + i; 3184e9b61f19SKirill A. Shutemov if (subpage == page) 3185e9b61f19SKirill A. Shutemov continue; 3186e9b61f19SKirill A. Shutemov unlock_page(subpage); 3187e9b61f19SKirill A. Shutemov 3188e9b61f19SKirill A. Shutemov /* 3189e9b61f19SKirill A. Shutemov * Subpages may be freed if there wasn't any mapping 3190e9b61f19SKirill A. Shutemov * like if add_to_swap() is running on a lru page that 3191e9b61f19SKirill A. Shutemov * had its mapping zapped. And freeing these pages 3192e9b61f19SKirill A. Shutemov * requires taking the lru_lock so we do the put_page 3193e9b61f19SKirill A. Shutemov * of the tail pages after the split is complete. 3194e9b61f19SKirill A. Shutemov */ 3195e9b61f19SKirill A. Shutemov put_page(subpage); 3196e9b61f19SKirill A. Shutemov } 3197e9b61f19SKirill A. Shutemov } 3198e9b61f19SKirill A. Shutemov 3199b20ce5e0SKirill A. Shutemov int total_mapcount(struct page *page) 3200b20ce5e0SKirill A. Shutemov { 3201b20ce5e0SKirill A. Shutemov int i, ret; 3202b20ce5e0SKirill A. Shutemov 3203b20ce5e0SKirill A. Shutemov VM_BUG_ON_PAGE(PageTail(page), page); 3204b20ce5e0SKirill A. Shutemov 3205b20ce5e0SKirill A. Shutemov if (likely(!PageCompound(page))) 3206b20ce5e0SKirill A. Shutemov return atomic_read(&page->_mapcount) + 1; 3207b20ce5e0SKirill A. Shutemov 3208b20ce5e0SKirill A. Shutemov ret = compound_mapcount(page); 3209b20ce5e0SKirill A. Shutemov if (PageHuge(page)) 3210b20ce5e0SKirill A. Shutemov return ret; 3211b20ce5e0SKirill A. Shutemov for (i = 0; i < HPAGE_PMD_NR; i++) 3212b20ce5e0SKirill A. Shutemov ret += atomic_read(&page[i]._mapcount) + 1; 3213b20ce5e0SKirill A. Shutemov if (PageDoubleMap(page)) 3214b20ce5e0SKirill A. Shutemov ret -= HPAGE_PMD_NR; 3215b20ce5e0SKirill A. Shutemov return ret; 3216b20ce5e0SKirill A. Shutemov } 3217b20ce5e0SKirill A. Shutemov 3218e9b61f19SKirill A. Shutemov /* 32196d0a07edSAndrea Arcangeli * This calculates accurately how many mappings a transparent hugepage 32206d0a07edSAndrea Arcangeli * has (unlike page_mapcount() which isn't fully accurate). This full 32216d0a07edSAndrea Arcangeli * accuracy is primarily needed to know if copy-on-write faults can 32226d0a07edSAndrea Arcangeli * reuse the page and change the mapping to read-write instead of 32236d0a07edSAndrea Arcangeli * copying them. At the same time this returns the total_mapcount too. 32246d0a07edSAndrea Arcangeli * 32256d0a07edSAndrea Arcangeli * The function returns the highest mapcount any one of the subpages 32266d0a07edSAndrea Arcangeli * has. If the return value is one, even if different processes are 32276d0a07edSAndrea Arcangeli * mapping different subpages of the transparent hugepage, they can 32286d0a07edSAndrea Arcangeli * all reuse it, because each process is reusing a different subpage. 32296d0a07edSAndrea Arcangeli * 32306d0a07edSAndrea Arcangeli * The total_mapcount is instead counting all virtual mappings of the 32316d0a07edSAndrea Arcangeli * subpages. If the total_mapcount is equal to "one", it tells the 32326d0a07edSAndrea Arcangeli * caller all mappings belong to the same "mm" and in turn the 32336d0a07edSAndrea Arcangeli * anon_vma of the transparent hugepage can become the vma->anon_vma 32346d0a07edSAndrea Arcangeli * local one as no other process may be mapping any of the subpages. 32356d0a07edSAndrea Arcangeli * 32366d0a07edSAndrea Arcangeli * It would be more accurate to replace page_mapcount() with 32376d0a07edSAndrea Arcangeli * page_trans_huge_mapcount(), however we only use 32386d0a07edSAndrea Arcangeli * page_trans_huge_mapcount() in the copy-on-write faults where we 32396d0a07edSAndrea Arcangeli * need full accuracy to avoid breaking page pinning, because 32406d0a07edSAndrea Arcangeli * page_trans_huge_mapcount() is slower than page_mapcount(). 32416d0a07edSAndrea Arcangeli */ 32426d0a07edSAndrea Arcangeli int page_trans_huge_mapcount(struct page *page, int *total_mapcount) 32436d0a07edSAndrea Arcangeli { 32446d0a07edSAndrea Arcangeli int i, ret, _total_mapcount, mapcount; 32456d0a07edSAndrea Arcangeli 32466d0a07edSAndrea Arcangeli /* hugetlbfs shouldn't call it */ 32476d0a07edSAndrea Arcangeli VM_BUG_ON_PAGE(PageHuge(page), page); 32486d0a07edSAndrea Arcangeli 32496d0a07edSAndrea Arcangeli if (likely(!PageTransCompound(page))) { 32506d0a07edSAndrea Arcangeli mapcount = atomic_read(&page->_mapcount) + 1; 32516d0a07edSAndrea Arcangeli if (total_mapcount) 32526d0a07edSAndrea Arcangeli *total_mapcount = mapcount; 32536d0a07edSAndrea Arcangeli return mapcount; 32546d0a07edSAndrea Arcangeli } 32556d0a07edSAndrea Arcangeli 32566d0a07edSAndrea Arcangeli page = compound_head(page); 32576d0a07edSAndrea Arcangeli 32586d0a07edSAndrea Arcangeli _total_mapcount = ret = 0; 32596d0a07edSAndrea Arcangeli for (i = 0; i < HPAGE_PMD_NR; i++) { 32606d0a07edSAndrea Arcangeli mapcount = atomic_read(&page[i]._mapcount) + 1; 32616d0a07edSAndrea Arcangeli ret = max(ret, mapcount); 32626d0a07edSAndrea Arcangeli _total_mapcount += mapcount; 32636d0a07edSAndrea Arcangeli } 32646d0a07edSAndrea Arcangeli if (PageDoubleMap(page)) { 32656d0a07edSAndrea Arcangeli ret -= 1; 32666d0a07edSAndrea Arcangeli _total_mapcount -= HPAGE_PMD_NR; 32676d0a07edSAndrea Arcangeli } 32686d0a07edSAndrea Arcangeli mapcount = compound_mapcount(page); 32696d0a07edSAndrea Arcangeli ret += mapcount; 32706d0a07edSAndrea Arcangeli _total_mapcount += mapcount; 32716d0a07edSAndrea Arcangeli if (total_mapcount) 32726d0a07edSAndrea Arcangeli *total_mapcount = _total_mapcount; 32736d0a07edSAndrea Arcangeli return ret; 32746d0a07edSAndrea Arcangeli } 32756d0a07edSAndrea Arcangeli 32766d0a07edSAndrea Arcangeli /* 3277e9b61f19SKirill A. Shutemov * This function splits huge page into normal pages. @page can point to any 3278e9b61f19SKirill A. Shutemov * subpage of huge page to split. Split doesn't change the position of @page. 3279e9b61f19SKirill A. Shutemov * 3280e9b61f19SKirill A. Shutemov * Only caller must hold pin on the @page, otherwise split fails with -EBUSY. 3281e9b61f19SKirill A. Shutemov * The huge page must be locked. 3282e9b61f19SKirill A. Shutemov * 3283e9b61f19SKirill A. Shutemov * If @list is null, tail pages will be added to LRU list, otherwise, to @list. 3284e9b61f19SKirill A. Shutemov * 3285e9b61f19SKirill A. Shutemov * Both head page and tail pages will inherit mapping, flags, and so on from 3286e9b61f19SKirill A. Shutemov * the hugepage. 3287e9b61f19SKirill A. Shutemov * 3288e9b61f19SKirill A. Shutemov * GUP pin and PG_locked transferred to @page. Rest subpages can be freed if 3289e9b61f19SKirill A. Shutemov * they are not mapped. 3290e9b61f19SKirill A. Shutemov * 3291e9b61f19SKirill A. Shutemov * Returns 0 if the hugepage is split successfully. 3292e9b61f19SKirill A. Shutemov * Returns -EBUSY if the page is pinned or if anon_vma disappeared from under 3293e9b61f19SKirill A. Shutemov * us. 3294e9b61f19SKirill A. Shutemov */ 3295e9b61f19SKirill A. Shutemov int split_huge_page_to_list(struct page *page, struct list_head *list) 3296e9b61f19SKirill A. Shutemov { 3297e9b61f19SKirill A. Shutemov struct page *head = compound_head(page); 3298a3d0a918SKirill A. Shutemov struct pglist_data *pgdata = NODE_DATA(page_to_nid(head)); 3299e9b61f19SKirill A. Shutemov struct anon_vma *anon_vma; 3300e9b61f19SKirill A. Shutemov int count, mapcount, ret; 3301d9654322SKirill A. Shutemov bool mlocked; 33020b9b6fffSKirill A. Shutemov unsigned long flags; 3303e9b61f19SKirill A. Shutemov 3304e9b61f19SKirill A. Shutemov VM_BUG_ON_PAGE(is_huge_zero_page(page), page); 3305e9b61f19SKirill A. Shutemov VM_BUG_ON_PAGE(!PageAnon(page), page); 3306e9b61f19SKirill A. Shutemov VM_BUG_ON_PAGE(!PageLocked(page), page); 3307e9b61f19SKirill A. Shutemov VM_BUG_ON_PAGE(!PageSwapBacked(page), page); 3308e9b61f19SKirill A. Shutemov VM_BUG_ON_PAGE(!PageCompound(page), page); 3309e9b61f19SKirill A. Shutemov 3310e9b61f19SKirill A. Shutemov /* 3311e9b61f19SKirill A. Shutemov * The caller does not necessarily hold an mmap_sem that would prevent 3312e9b61f19SKirill A. Shutemov * the anon_vma disappearing so we first we take a reference to it 3313e9b61f19SKirill A. Shutemov * and then lock the anon_vma for write. This is similar to 3314e9b61f19SKirill A. Shutemov * page_lock_anon_vma_read except the write lock is taken to serialise 3315e9b61f19SKirill A. Shutemov * against parallel split or collapse operations. 3316e9b61f19SKirill A. Shutemov */ 3317e9b61f19SKirill A. Shutemov anon_vma = page_get_anon_vma(head); 3318e9b61f19SKirill A. Shutemov if (!anon_vma) { 3319e9b61f19SKirill A. Shutemov ret = -EBUSY; 3320e9b61f19SKirill A. Shutemov goto out; 3321e9b61f19SKirill A. Shutemov } 3322e9b61f19SKirill A. Shutemov anon_vma_lock_write(anon_vma); 3323e9b61f19SKirill A. Shutemov 3324e9b61f19SKirill A. Shutemov /* 3325e9b61f19SKirill A. Shutemov * Racy check if we can split the page, before freeze_page() will 3326e9b61f19SKirill A. Shutemov * split PMDs 3327e9b61f19SKirill A. Shutemov */ 3328e9b61f19SKirill A. Shutemov if (total_mapcount(head) != page_count(head) - 1) { 3329e9b61f19SKirill A. Shutemov ret = -EBUSY; 3330e9b61f19SKirill A. Shutemov goto out_unlock; 3331e9b61f19SKirill A. Shutemov } 3332e9b61f19SKirill A. Shutemov 3333d9654322SKirill A. Shutemov mlocked = PageMlocked(page); 3334fec89c10SKirill A. Shutemov freeze_page(head); 3335e9b61f19SKirill A. Shutemov VM_BUG_ON_PAGE(compound_mapcount(head), head); 3336e9b61f19SKirill A. Shutemov 3337d9654322SKirill A. Shutemov /* Make sure the page is not on per-CPU pagevec as it takes pin */ 3338d9654322SKirill A. Shutemov if (mlocked) 3339d9654322SKirill A. Shutemov lru_add_drain(); 3340d9654322SKirill A. Shutemov 33410139aa7bSJoonsoo Kim /* Prevent deferred_split_scan() touching ->_refcount */ 3342a3d0a918SKirill A. Shutemov spin_lock_irqsave(&pgdata->split_queue_lock, flags); 3343e9b61f19SKirill A. Shutemov count = page_count(head); 3344e9b61f19SKirill A. Shutemov mapcount = total_mapcount(head); 3345bd56086fSKirill A. Shutemov if (!mapcount && count == 1) { 33469a982250SKirill A. Shutemov if (!list_empty(page_deferred_list(head))) { 3347a3d0a918SKirill A. Shutemov pgdata->split_queue_len--; 33489a982250SKirill A. Shutemov list_del(page_deferred_list(head)); 33499a982250SKirill A. Shutemov } 3350a3d0a918SKirill A. Shutemov spin_unlock_irqrestore(&pgdata->split_queue_lock, flags); 3351e9b61f19SKirill A. Shutemov __split_huge_page(page, list); 3352e9b61f19SKirill A. Shutemov ret = 0; 3353bd56086fSKirill A. Shutemov } else if (IS_ENABLED(CONFIG_DEBUG_VM) && mapcount) { 3354a3d0a918SKirill A. Shutemov spin_unlock_irqrestore(&pgdata->split_queue_lock, flags); 3355e9b61f19SKirill A. Shutemov pr_alert("total_mapcount: %u, page_count(): %u\n", 3356e9b61f19SKirill A. Shutemov mapcount, count); 3357e9b61f19SKirill A. Shutemov if (PageTail(page)) 3358e9b61f19SKirill A. Shutemov dump_page(head, NULL); 3359bd56086fSKirill A. Shutemov dump_page(page, "total_mapcount(head) > 0"); 3360e9b61f19SKirill A. Shutemov BUG(); 3361e9b61f19SKirill A. Shutemov } else { 3362a3d0a918SKirill A. Shutemov spin_unlock_irqrestore(&pgdata->split_queue_lock, flags); 3363fec89c10SKirill A. Shutemov unfreeze_page(head); 3364e9b61f19SKirill A. Shutemov ret = -EBUSY; 3365e9b61f19SKirill A. Shutemov } 3366e9b61f19SKirill A. Shutemov 3367e9b61f19SKirill A. Shutemov out_unlock: 3368e9b61f19SKirill A. Shutemov anon_vma_unlock_write(anon_vma); 3369e9b61f19SKirill A. Shutemov put_anon_vma(anon_vma); 3370e9b61f19SKirill A. Shutemov out: 3371e9b61f19SKirill A. Shutemov count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED); 3372e9b61f19SKirill A. Shutemov return ret; 3373e9b61f19SKirill A. Shutemov } 33749a982250SKirill A. Shutemov 33759a982250SKirill A. Shutemov void free_transhuge_page(struct page *page) 33769a982250SKirill A. Shutemov { 3377a3d0a918SKirill A. Shutemov struct pglist_data *pgdata = NODE_DATA(page_to_nid(page)); 33789a982250SKirill A. Shutemov unsigned long flags; 33799a982250SKirill A. Shutemov 3380a3d0a918SKirill A. Shutemov spin_lock_irqsave(&pgdata->split_queue_lock, flags); 33819a982250SKirill A. Shutemov if (!list_empty(page_deferred_list(page))) { 3382a3d0a918SKirill A. Shutemov pgdata->split_queue_len--; 33839a982250SKirill A. Shutemov list_del(page_deferred_list(page)); 33849a982250SKirill A. Shutemov } 3385a3d0a918SKirill A. Shutemov spin_unlock_irqrestore(&pgdata->split_queue_lock, flags); 33869a982250SKirill A. Shutemov free_compound_page(page); 33879a982250SKirill A. Shutemov } 33889a982250SKirill A. Shutemov 33899a982250SKirill A. Shutemov void deferred_split_huge_page(struct page *page) 33909a982250SKirill A. Shutemov { 3391a3d0a918SKirill A. Shutemov struct pglist_data *pgdata = NODE_DATA(page_to_nid(page)); 33929a982250SKirill A. Shutemov unsigned long flags; 33939a982250SKirill A. Shutemov 33949a982250SKirill A. Shutemov VM_BUG_ON_PAGE(!PageTransHuge(page), page); 33959a982250SKirill A. Shutemov 3396a3d0a918SKirill A. Shutemov spin_lock_irqsave(&pgdata->split_queue_lock, flags); 33979a982250SKirill A. Shutemov if (list_empty(page_deferred_list(page))) { 3398f9719a03SKirill A. Shutemov count_vm_event(THP_DEFERRED_SPLIT_PAGE); 3399a3d0a918SKirill A. Shutemov list_add_tail(page_deferred_list(page), &pgdata->split_queue); 3400a3d0a918SKirill A. Shutemov pgdata->split_queue_len++; 34019a982250SKirill A. Shutemov } 3402a3d0a918SKirill A. Shutemov spin_unlock_irqrestore(&pgdata->split_queue_lock, flags); 34039a982250SKirill A. Shutemov } 34049a982250SKirill A. Shutemov 34059a982250SKirill A. Shutemov static unsigned long deferred_split_count(struct shrinker *shrink, 34069a982250SKirill A. Shutemov struct shrink_control *sc) 34079a982250SKirill A. Shutemov { 3408a3d0a918SKirill A. Shutemov struct pglist_data *pgdata = NODE_DATA(sc->nid); 3409cb8d68ecSKirill A. Shutemov return ACCESS_ONCE(pgdata->split_queue_len); 34109a982250SKirill A. Shutemov } 34119a982250SKirill A. Shutemov 34129a982250SKirill A. Shutemov static unsigned long deferred_split_scan(struct shrinker *shrink, 34139a982250SKirill A. Shutemov struct shrink_control *sc) 34149a982250SKirill A. Shutemov { 3415a3d0a918SKirill A. Shutemov struct pglist_data *pgdata = NODE_DATA(sc->nid); 34169a982250SKirill A. Shutemov unsigned long flags; 34179a982250SKirill A. Shutemov LIST_HEAD(list), *pos, *next; 34189a982250SKirill A. Shutemov struct page *page; 34199a982250SKirill A. Shutemov int split = 0; 34209a982250SKirill A. Shutemov 3421a3d0a918SKirill A. Shutemov spin_lock_irqsave(&pgdata->split_queue_lock, flags); 34229a982250SKirill A. Shutemov /* Take pin on all head pages to avoid freeing them under us */ 3423ae026204SKirill A. Shutemov list_for_each_safe(pos, next, &pgdata->split_queue) { 34249a982250SKirill A. Shutemov page = list_entry((void *)pos, struct page, mapping); 34259a982250SKirill A. Shutemov page = compound_head(page); 3426e3ae1953SKirill A. Shutemov if (get_page_unless_zero(page)) { 3427e3ae1953SKirill A. Shutemov list_move(page_deferred_list(page), &list); 3428e3ae1953SKirill A. Shutemov } else { 3429e3ae1953SKirill A. Shutemov /* We lost race with put_compound_page() */ 34309a982250SKirill A. Shutemov list_del_init(page_deferred_list(page)); 3431a3d0a918SKirill A. Shutemov pgdata->split_queue_len--; 34329a982250SKirill A. Shutemov } 3433e3ae1953SKirill A. Shutemov if (!--sc->nr_to_scan) 3434e3ae1953SKirill A. Shutemov break; 34359a982250SKirill A. Shutemov } 3436a3d0a918SKirill A. Shutemov spin_unlock_irqrestore(&pgdata->split_queue_lock, flags); 34379a982250SKirill A. Shutemov 34389a982250SKirill A. Shutemov list_for_each_safe(pos, next, &list) { 34399a982250SKirill A. Shutemov page = list_entry((void *)pos, struct page, mapping); 34409a982250SKirill A. Shutemov lock_page(page); 34419a982250SKirill A. Shutemov /* split_huge_page() removes page from list on success */ 34429a982250SKirill A. Shutemov if (!split_huge_page(page)) 34439a982250SKirill A. Shutemov split++; 34449a982250SKirill A. Shutemov unlock_page(page); 34459a982250SKirill A. Shutemov put_page(page); 34469a982250SKirill A. Shutemov } 34479a982250SKirill A. Shutemov 3448a3d0a918SKirill A. Shutemov spin_lock_irqsave(&pgdata->split_queue_lock, flags); 3449a3d0a918SKirill A. Shutemov list_splice_tail(&list, &pgdata->split_queue); 3450a3d0a918SKirill A. Shutemov spin_unlock_irqrestore(&pgdata->split_queue_lock, flags); 34519a982250SKirill A. Shutemov 3452cb8d68ecSKirill A. Shutemov /* 3453cb8d68ecSKirill A. Shutemov * Stop shrinker if we didn't split any page, but the queue is empty. 3454cb8d68ecSKirill A. Shutemov * This can happen if pages were freed under us. 3455cb8d68ecSKirill A. Shutemov */ 3456cb8d68ecSKirill A. Shutemov if (!split && list_empty(&pgdata->split_queue)) 3457cb8d68ecSKirill A. Shutemov return SHRINK_STOP; 3458cb8d68ecSKirill A. Shutemov return split; 34599a982250SKirill A. Shutemov } 34609a982250SKirill A. Shutemov 34619a982250SKirill A. Shutemov static struct shrinker deferred_split_shrinker = { 34629a982250SKirill A. Shutemov .count_objects = deferred_split_count, 34639a982250SKirill A. Shutemov .scan_objects = deferred_split_scan, 34649a982250SKirill A. Shutemov .seeks = DEFAULT_SEEKS, 3465a3d0a918SKirill A. Shutemov .flags = SHRINKER_NUMA_AWARE, 34669a982250SKirill A. Shutemov }; 346749071d43SKirill A. Shutemov 346849071d43SKirill A. Shutemov #ifdef CONFIG_DEBUG_FS 346949071d43SKirill A. Shutemov static int split_huge_pages_set(void *data, u64 val) 347049071d43SKirill A. Shutemov { 347149071d43SKirill A. Shutemov struct zone *zone; 347249071d43SKirill A. Shutemov struct page *page; 347349071d43SKirill A. Shutemov unsigned long pfn, max_zone_pfn; 347449071d43SKirill A. Shutemov unsigned long total = 0, split = 0; 347549071d43SKirill A. Shutemov 347649071d43SKirill A. Shutemov if (val != 1) 347749071d43SKirill A. Shutemov return -EINVAL; 347849071d43SKirill A. Shutemov 347949071d43SKirill A. Shutemov for_each_populated_zone(zone) { 348049071d43SKirill A. Shutemov max_zone_pfn = zone_end_pfn(zone); 348149071d43SKirill A. Shutemov for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) { 348249071d43SKirill A. Shutemov if (!pfn_valid(pfn)) 348349071d43SKirill A. Shutemov continue; 348449071d43SKirill A. Shutemov 348549071d43SKirill A. Shutemov page = pfn_to_page(pfn); 348649071d43SKirill A. Shutemov if (!get_page_unless_zero(page)) 348749071d43SKirill A. Shutemov continue; 348849071d43SKirill A. Shutemov 348949071d43SKirill A. Shutemov if (zone != page_zone(page)) 349049071d43SKirill A. Shutemov goto next; 349149071d43SKirill A. Shutemov 349249071d43SKirill A. Shutemov if (!PageHead(page) || !PageAnon(page) || 349349071d43SKirill A. Shutemov PageHuge(page)) 349449071d43SKirill A. Shutemov goto next; 349549071d43SKirill A. Shutemov 349649071d43SKirill A. Shutemov total++; 349749071d43SKirill A. Shutemov lock_page(page); 349849071d43SKirill A. Shutemov if (!split_huge_page(page)) 349949071d43SKirill A. Shutemov split++; 350049071d43SKirill A. Shutemov unlock_page(page); 350149071d43SKirill A. Shutemov next: 350249071d43SKirill A. Shutemov put_page(page); 350349071d43SKirill A. Shutemov } 350449071d43SKirill A. Shutemov } 350549071d43SKirill A. Shutemov 3506145bdaa1SYang Shi pr_info("%lu of %lu THP split\n", split, total); 350749071d43SKirill A. Shutemov 350849071d43SKirill A. Shutemov return 0; 350949071d43SKirill A. Shutemov } 351049071d43SKirill A. Shutemov DEFINE_SIMPLE_ATTRIBUTE(split_huge_pages_fops, NULL, split_huge_pages_set, 351149071d43SKirill A. Shutemov "%llu\n"); 351249071d43SKirill A. Shutemov 351349071d43SKirill A. Shutemov static int __init split_huge_pages_debugfs(void) 351449071d43SKirill A. Shutemov { 351549071d43SKirill A. Shutemov void *ret; 351649071d43SKirill A. Shutemov 3517145bdaa1SYang Shi ret = debugfs_create_file("split_huge_pages", 0200, NULL, NULL, 351849071d43SKirill A. Shutemov &split_huge_pages_fops); 351949071d43SKirill A. Shutemov if (!ret) 352049071d43SKirill A. Shutemov pr_warn("Failed to create split_huge_pages in debugfs"); 352149071d43SKirill A. Shutemov return 0; 352249071d43SKirill A. Shutemov } 352349071d43SKirill A. Shutemov late_initcall(split_huge_pages_debugfs); 352449071d43SKirill A. Shutemov #endif 3525