1b20a3503SChristoph Lameter /* 214e0f9bcSHugh Dickins * Memory Migration functionality - linux/mm/migrate.c 3b20a3503SChristoph Lameter * 4b20a3503SChristoph Lameter * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter 5b20a3503SChristoph Lameter * 6b20a3503SChristoph Lameter * Page migration was first developed in the context of the memory hotplug 7b20a3503SChristoph Lameter * project. The main authors of the migration code are: 8b20a3503SChristoph Lameter * 9b20a3503SChristoph Lameter * IWAMOTO Toshihiro <iwamoto@valinux.co.jp> 10b20a3503SChristoph Lameter * Hirokazu Takahashi <taka@valinux.co.jp> 11b20a3503SChristoph Lameter * Dave Hansen <haveblue@us.ibm.com> 12cde53535SChristoph Lameter * Christoph Lameter 13b20a3503SChristoph Lameter */ 14b20a3503SChristoph Lameter 15b20a3503SChristoph Lameter #include <linux/migrate.h> 16b95f1b31SPaul Gortmaker #include <linux/export.h> 17b20a3503SChristoph Lameter #include <linux/swap.h> 180697212aSChristoph Lameter #include <linux/swapops.h> 19b20a3503SChristoph Lameter #include <linux/pagemap.h> 20e23ca00bSChristoph Lameter #include <linux/buffer_head.h> 21b20a3503SChristoph Lameter #include <linux/mm_inline.h> 22b488893aSPavel Emelyanov #include <linux/nsproxy.h> 23b20a3503SChristoph Lameter #include <linux/pagevec.h> 24e9995ef9SHugh Dickins #include <linux/ksm.h> 25b20a3503SChristoph Lameter #include <linux/rmap.h> 26b20a3503SChristoph Lameter #include <linux/topology.h> 27b20a3503SChristoph Lameter #include <linux/cpu.h> 28b20a3503SChristoph Lameter #include <linux/cpuset.h> 2904e62a29SChristoph Lameter #include <linux/writeback.h> 30742755a1SChristoph Lameter #include <linux/mempolicy.h> 31742755a1SChristoph Lameter #include <linux/vmalloc.h> 3286c3a764SDavid Quigley #include <linux/security.h> 3342cb14b1SHugh Dickins #include <linux/backing-dev.h> 34bda807d4SMinchan Kim #include <linux/compaction.h> 354f5ca265SAdrian Bunk #include <linux/syscalls.h> 36290408d4SNaoya Horiguchi #include <linux/hugetlb.h> 378e6ac7faSAneesh Kumar K.V #include <linux/hugetlb_cgroup.h> 385a0e3ad6STejun Heo #include <linux/gfp.h> 39*df6ad698SJérôme Glisse #include <linux/pfn_t.h> 40a5430ddaSJérôme Glisse #include <linux/memremap.h> 418315ada7SJérôme Glisse #include <linux/userfaultfd_k.h> 42bf6bddf1SRafael Aquini #include <linux/balloon_compaction.h> 43f714f4f2SMel Gorman #include <linux/mmu_notifier.h> 4433c3fc71SVladimir Davydov #include <linux/page_idle.h> 45d435edcaSVlastimil Babka #include <linux/page_owner.h> 466e84f315SIngo Molnar #include <linux/sched/mm.h> 47197e7e52SLinus Torvalds #include <linux/ptrace.h> 48b20a3503SChristoph Lameter 490d1836c3SMichal Nazarewicz #include <asm/tlbflush.h> 500d1836c3SMichal Nazarewicz 517b2a2d4aSMel Gorman #define CREATE_TRACE_POINTS 527b2a2d4aSMel Gorman #include <trace/events/migrate.h> 537b2a2d4aSMel Gorman 54b20a3503SChristoph Lameter #include "internal.h" 55b20a3503SChristoph Lameter 56b20a3503SChristoph Lameter /* 57742755a1SChristoph Lameter * migrate_prep() needs to be called before we start compiling a list of pages 58748446bbSMel Gorman * to be migrated using isolate_lru_page(). If scheduling work on other CPUs is 59748446bbSMel Gorman * undesirable, use migrate_prep_local() 60b20a3503SChristoph Lameter */ 61b20a3503SChristoph Lameter int migrate_prep(void) 62b20a3503SChristoph Lameter { 63b20a3503SChristoph Lameter /* 64b20a3503SChristoph Lameter * Clear the LRU lists so pages can be isolated. 65b20a3503SChristoph Lameter * Note that pages may be moved off the LRU after we have 66b20a3503SChristoph Lameter * drained them. Those pages will fail to migrate like other 67b20a3503SChristoph Lameter * pages that may be busy. 68b20a3503SChristoph Lameter */ 69b20a3503SChristoph Lameter lru_add_drain_all(); 70b20a3503SChristoph Lameter 71b20a3503SChristoph Lameter return 0; 72b20a3503SChristoph Lameter } 73b20a3503SChristoph Lameter 74748446bbSMel Gorman /* Do the necessary work of migrate_prep but not if it involves other CPUs */ 75748446bbSMel Gorman int migrate_prep_local(void) 76748446bbSMel Gorman { 77748446bbSMel Gorman lru_add_drain(); 78748446bbSMel Gorman 79748446bbSMel Gorman return 0; 80748446bbSMel Gorman } 81748446bbSMel Gorman 829e5bcd61SYisheng Xie int isolate_movable_page(struct page *page, isolate_mode_t mode) 83bda807d4SMinchan Kim { 84bda807d4SMinchan Kim struct address_space *mapping; 85bda807d4SMinchan Kim 86bda807d4SMinchan Kim /* 87bda807d4SMinchan Kim * Avoid burning cycles with pages that are yet under __free_pages(), 88bda807d4SMinchan Kim * or just got freed under us. 89bda807d4SMinchan Kim * 90bda807d4SMinchan Kim * In case we 'win' a race for a movable page being freed under us and 91bda807d4SMinchan Kim * raise its refcount preventing __free_pages() from doing its job 92bda807d4SMinchan Kim * the put_page() at the end of this block will take care of 93bda807d4SMinchan Kim * release this page, thus avoiding a nasty leakage. 94bda807d4SMinchan Kim */ 95bda807d4SMinchan Kim if (unlikely(!get_page_unless_zero(page))) 96bda807d4SMinchan Kim goto out; 97bda807d4SMinchan Kim 98bda807d4SMinchan Kim /* 99bda807d4SMinchan Kim * Check PageMovable before holding a PG_lock because page's owner 100bda807d4SMinchan Kim * assumes anybody doesn't touch PG_lock of newly allocated page 101bda807d4SMinchan Kim * so unconditionally grapping the lock ruins page's owner side. 102bda807d4SMinchan Kim */ 103bda807d4SMinchan Kim if (unlikely(!__PageMovable(page))) 104bda807d4SMinchan Kim goto out_putpage; 105bda807d4SMinchan Kim /* 106bda807d4SMinchan Kim * As movable pages are not isolated from LRU lists, concurrent 107bda807d4SMinchan Kim * compaction threads can race against page migration functions 108bda807d4SMinchan Kim * as well as race against the releasing a page. 109bda807d4SMinchan Kim * 110bda807d4SMinchan Kim * In order to avoid having an already isolated movable page 111bda807d4SMinchan Kim * being (wrongly) re-isolated while it is under migration, 112bda807d4SMinchan Kim * or to avoid attempting to isolate pages being released, 113bda807d4SMinchan Kim * lets be sure we have the page lock 114bda807d4SMinchan Kim * before proceeding with the movable page isolation steps. 115bda807d4SMinchan Kim */ 116bda807d4SMinchan Kim if (unlikely(!trylock_page(page))) 117bda807d4SMinchan Kim goto out_putpage; 118bda807d4SMinchan Kim 119bda807d4SMinchan Kim if (!PageMovable(page) || PageIsolated(page)) 120bda807d4SMinchan Kim goto out_no_isolated; 121bda807d4SMinchan Kim 122bda807d4SMinchan Kim mapping = page_mapping(page); 123bda807d4SMinchan Kim VM_BUG_ON_PAGE(!mapping, page); 124bda807d4SMinchan Kim 125bda807d4SMinchan Kim if (!mapping->a_ops->isolate_page(page, mode)) 126bda807d4SMinchan Kim goto out_no_isolated; 127bda807d4SMinchan Kim 128bda807d4SMinchan Kim /* Driver shouldn't use PG_isolated bit of page->flags */ 129bda807d4SMinchan Kim WARN_ON_ONCE(PageIsolated(page)); 130bda807d4SMinchan Kim __SetPageIsolated(page); 131bda807d4SMinchan Kim unlock_page(page); 132bda807d4SMinchan Kim 1339e5bcd61SYisheng Xie return 0; 134bda807d4SMinchan Kim 135bda807d4SMinchan Kim out_no_isolated: 136bda807d4SMinchan Kim unlock_page(page); 137bda807d4SMinchan Kim out_putpage: 138bda807d4SMinchan Kim put_page(page); 139bda807d4SMinchan Kim out: 1409e5bcd61SYisheng Xie return -EBUSY; 141bda807d4SMinchan Kim } 142bda807d4SMinchan Kim 143bda807d4SMinchan Kim /* It should be called on page which is PG_movable */ 144bda807d4SMinchan Kim void putback_movable_page(struct page *page) 145bda807d4SMinchan Kim { 146bda807d4SMinchan Kim struct address_space *mapping; 147bda807d4SMinchan Kim 148bda807d4SMinchan Kim VM_BUG_ON_PAGE(!PageLocked(page), page); 149bda807d4SMinchan Kim VM_BUG_ON_PAGE(!PageMovable(page), page); 150bda807d4SMinchan Kim VM_BUG_ON_PAGE(!PageIsolated(page), page); 151bda807d4SMinchan Kim 152bda807d4SMinchan Kim mapping = page_mapping(page); 153bda807d4SMinchan Kim mapping->a_ops->putback_page(page); 154bda807d4SMinchan Kim __ClearPageIsolated(page); 155bda807d4SMinchan Kim } 156bda807d4SMinchan Kim 157b20a3503SChristoph Lameter /* 1585733c7d1SRafael Aquini * Put previously isolated pages back onto the appropriate lists 1595733c7d1SRafael Aquini * from where they were once taken off for compaction/migration. 1605733c7d1SRafael Aquini * 16159c82b70SJoonsoo Kim * This function shall be used whenever the isolated pageset has been 16259c82b70SJoonsoo Kim * built from lru, balloon, hugetlbfs page. See isolate_migratepages_range() 16359c82b70SJoonsoo Kim * and isolate_huge_page(). 1645733c7d1SRafael Aquini */ 1655733c7d1SRafael Aquini void putback_movable_pages(struct list_head *l) 1665733c7d1SRafael Aquini { 1675733c7d1SRafael Aquini struct page *page; 1685733c7d1SRafael Aquini struct page *page2; 1695733c7d1SRafael Aquini 1705733c7d1SRafael Aquini list_for_each_entry_safe(page, page2, l, lru) { 17131caf665SNaoya Horiguchi if (unlikely(PageHuge(page))) { 17231caf665SNaoya Horiguchi putback_active_hugepage(page); 17331caf665SNaoya Horiguchi continue; 17431caf665SNaoya Horiguchi } 1755733c7d1SRafael Aquini list_del(&page->lru); 176bda807d4SMinchan Kim /* 177bda807d4SMinchan Kim * We isolated non-lru movable page so here we can use 178bda807d4SMinchan Kim * __PageMovable because LRU page's mapping cannot have 179bda807d4SMinchan Kim * PAGE_MAPPING_MOVABLE. 180bda807d4SMinchan Kim */ 181b1123ea6SMinchan Kim if (unlikely(__PageMovable(page))) { 182bda807d4SMinchan Kim VM_BUG_ON_PAGE(!PageIsolated(page), page); 183bda807d4SMinchan Kim lock_page(page); 184bda807d4SMinchan Kim if (PageMovable(page)) 185bda807d4SMinchan Kim putback_movable_page(page); 186bf6bddf1SRafael Aquini else 187bda807d4SMinchan Kim __ClearPageIsolated(page); 188bda807d4SMinchan Kim unlock_page(page); 189bda807d4SMinchan Kim put_page(page); 190bda807d4SMinchan Kim } else { 191e8db67ebSNaoya Horiguchi mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON + 192e8db67ebSNaoya Horiguchi page_is_file_cache(page), -hpage_nr_pages(page)); 193fc280fe8SRabin Vincent putback_lru_page(page); 194b20a3503SChristoph Lameter } 195b20a3503SChristoph Lameter } 196bda807d4SMinchan Kim } 197b20a3503SChristoph Lameter 1980697212aSChristoph Lameter /* 1990697212aSChristoph Lameter * Restore a potential migration pte to a working pte entry 2000697212aSChristoph Lameter */ 201e4b82222SMinchan Kim static bool remove_migration_pte(struct page *page, struct vm_area_struct *vma, 202e9995ef9SHugh Dickins unsigned long addr, void *old) 2030697212aSChristoph Lameter { 2043fe87967SKirill A. Shutemov struct page_vma_mapped_walk pvmw = { 2053fe87967SKirill A. Shutemov .page = old, 2063fe87967SKirill A. Shutemov .vma = vma, 2073fe87967SKirill A. Shutemov .address = addr, 2083fe87967SKirill A. Shutemov .flags = PVMW_SYNC | PVMW_MIGRATION, 2093fe87967SKirill A. Shutemov }; 2103fe87967SKirill A. Shutemov struct page *new; 2113fe87967SKirill A. Shutemov pte_t pte; 2120697212aSChristoph Lameter swp_entry_t entry; 2130697212aSChristoph Lameter 2143fe87967SKirill A. Shutemov VM_BUG_ON_PAGE(PageTail(page), page); 2153fe87967SKirill A. Shutemov while (page_vma_mapped_walk(&pvmw)) { 2164b0ece6fSNaoya Horiguchi if (PageKsm(page)) 2174b0ece6fSNaoya Horiguchi new = page; 2184b0ece6fSNaoya Horiguchi else 2193fe87967SKirill A. Shutemov new = page - pvmw.page->index + 2203fe87967SKirill A. Shutemov linear_page_index(vma, pvmw.address); 2210697212aSChristoph Lameter 222616b8371SZi Yan #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 223616b8371SZi Yan /* PMD-mapped THP migration entry */ 224616b8371SZi Yan if (!pvmw.pte) { 225616b8371SZi Yan VM_BUG_ON_PAGE(PageHuge(page) || !PageTransCompound(page), page); 226616b8371SZi Yan remove_migration_pmd(&pvmw, new); 227616b8371SZi Yan continue; 228616b8371SZi Yan } 229616b8371SZi Yan #endif 230616b8371SZi Yan 2310697212aSChristoph Lameter get_page(new); 2326d2329f8SAndrea Arcangeli pte = pte_mkold(mk_pte(new, READ_ONCE(vma->vm_page_prot))); 2333fe87967SKirill A. Shutemov if (pte_swp_soft_dirty(*pvmw.pte)) 234c3d16e16SCyrill Gorcunov pte = pte_mksoft_dirty(pte); 235d3cb8bf6SMel Gorman 2363fe87967SKirill A. Shutemov /* 2373fe87967SKirill A. Shutemov * Recheck VMA as permissions can change since migration started 2383fe87967SKirill A. Shutemov */ 2393fe87967SKirill A. Shutemov entry = pte_to_swp_entry(*pvmw.pte); 2400697212aSChristoph Lameter if (is_write_migration_entry(entry)) 241d3cb8bf6SMel Gorman pte = maybe_mkwrite(pte, vma); 242d3cb8bf6SMel Gorman 243*df6ad698SJérôme Glisse if (unlikely(is_zone_device_page(new))) { 244*df6ad698SJérôme Glisse if (is_device_private_page(new)) { 245a5430ddaSJérôme Glisse entry = make_device_private_entry(new, pte_write(pte)); 246a5430ddaSJérôme Glisse pte = swp_entry_to_pte(entry); 247*df6ad698SJérôme Glisse } else if (is_device_public_page(new)) { 248*df6ad698SJérôme Glisse pte = pte_mkdevmap(pte); 249*df6ad698SJérôme Glisse flush_dcache_page(new); 250*df6ad698SJérôme Glisse } 251a5430ddaSJérôme Glisse } else 252383321abSAneesh Kumar K.V flush_dcache_page(new); 253a5430ddaSJérôme Glisse 2543ef8fd7fSAndi Kleen #ifdef CONFIG_HUGETLB_PAGE 255be7517d6STony Lu if (PageHuge(new)) { 256290408d4SNaoya Horiguchi pte = pte_mkhuge(pte); 257be7517d6STony Lu pte = arch_make_huge_pte(pte, vma, new, 0); 258383321abSAneesh Kumar K.V set_huge_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte); 25904e62a29SChristoph Lameter if (PageAnon(new)) 2603fe87967SKirill A. Shutemov hugepage_add_anon_rmap(new, vma, pvmw.address); 261290408d4SNaoya Horiguchi else 26253f9263bSKirill A. Shutemov page_dup_rmap(new, true); 263383321abSAneesh Kumar K.V } else 264383321abSAneesh Kumar K.V #endif 265383321abSAneesh Kumar K.V { 266383321abSAneesh Kumar K.V set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte); 267383321abSAneesh Kumar K.V 268383321abSAneesh Kumar K.V if (PageAnon(new)) 2693fe87967SKirill A. Shutemov page_add_anon_rmap(new, vma, pvmw.address, false); 27004e62a29SChristoph Lameter else 271dd78feddSKirill A. Shutemov page_add_file_rmap(new, false); 272383321abSAneesh Kumar K.V } 273e388466dSKirill A. Shutemov if (vma->vm_flags & VM_LOCKED && !PageTransCompound(new)) 27451afb12bSHugh Dickins mlock_vma_page(new); 27551afb12bSHugh Dickins 27604e62a29SChristoph Lameter /* No need to invalidate - it was non-present before */ 2773fe87967SKirill A. Shutemov update_mmu_cache(vma, pvmw.address, pvmw.pte); 2783fe87967SKirill A. Shutemov } 2793fe87967SKirill A. Shutemov 280e4b82222SMinchan Kim return true; 2810697212aSChristoph Lameter } 2820697212aSChristoph Lameter 2830697212aSChristoph Lameter /* 28404e62a29SChristoph Lameter * Get rid of all migration entries and replace them by 28504e62a29SChristoph Lameter * references to the indicated page. 28604e62a29SChristoph Lameter */ 287e388466dSKirill A. Shutemov void remove_migration_ptes(struct page *old, struct page *new, bool locked) 28804e62a29SChristoph Lameter { 289051ac83aSJoonsoo Kim struct rmap_walk_control rwc = { 290051ac83aSJoonsoo Kim .rmap_one = remove_migration_pte, 291051ac83aSJoonsoo Kim .arg = old, 292051ac83aSJoonsoo Kim }; 293051ac83aSJoonsoo Kim 294e388466dSKirill A. Shutemov if (locked) 295e388466dSKirill A. Shutemov rmap_walk_locked(new, &rwc); 296e388466dSKirill A. Shutemov else 297051ac83aSJoonsoo Kim rmap_walk(new, &rwc); 29804e62a29SChristoph Lameter } 29904e62a29SChristoph Lameter 30004e62a29SChristoph Lameter /* 3010697212aSChristoph Lameter * Something used the pte of a page under migration. We need to 3020697212aSChristoph Lameter * get to the page and wait until migration is finished. 3030697212aSChristoph Lameter * When we return from this function the fault will be retried. 3040697212aSChristoph Lameter */ 305e66f17ffSNaoya Horiguchi void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep, 30630dad309SNaoya Horiguchi spinlock_t *ptl) 3070697212aSChristoph Lameter { 30830dad309SNaoya Horiguchi pte_t pte; 3090697212aSChristoph Lameter swp_entry_t entry; 3100697212aSChristoph Lameter struct page *page; 3110697212aSChristoph Lameter 31230dad309SNaoya Horiguchi spin_lock(ptl); 3130697212aSChristoph Lameter pte = *ptep; 3140697212aSChristoph Lameter if (!is_swap_pte(pte)) 3150697212aSChristoph Lameter goto out; 3160697212aSChristoph Lameter 3170697212aSChristoph Lameter entry = pte_to_swp_entry(pte); 3180697212aSChristoph Lameter if (!is_migration_entry(entry)) 3190697212aSChristoph Lameter goto out; 3200697212aSChristoph Lameter 3210697212aSChristoph Lameter page = migration_entry_to_page(entry); 3220697212aSChristoph Lameter 323e286781dSNick Piggin /* 324e286781dSNick Piggin * Once radix-tree replacement of page migration started, page_count 325e286781dSNick Piggin * *must* be zero. And, we don't want to call wait_on_page_locked() 326e286781dSNick Piggin * against a page without get_page(). 327e286781dSNick Piggin * So, we use get_page_unless_zero(), here. Even failed, page fault 328e286781dSNick Piggin * will occur again. 329e286781dSNick Piggin */ 330e286781dSNick Piggin if (!get_page_unless_zero(page)) 331e286781dSNick Piggin goto out; 3320697212aSChristoph Lameter pte_unmap_unlock(ptep, ptl); 3330697212aSChristoph Lameter wait_on_page_locked(page); 3340697212aSChristoph Lameter put_page(page); 3350697212aSChristoph Lameter return; 3360697212aSChristoph Lameter out: 3370697212aSChristoph Lameter pte_unmap_unlock(ptep, ptl); 3380697212aSChristoph Lameter } 3390697212aSChristoph Lameter 34030dad309SNaoya Horiguchi void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, 34130dad309SNaoya Horiguchi unsigned long address) 34230dad309SNaoya Horiguchi { 34330dad309SNaoya Horiguchi spinlock_t *ptl = pte_lockptr(mm, pmd); 34430dad309SNaoya Horiguchi pte_t *ptep = pte_offset_map(pmd, address); 34530dad309SNaoya Horiguchi __migration_entry_wait(mm, ptep, ptl); 34630dad309SNaoya Horiguchi } 34730dad309SNaoya Horiguchi 348cb900f41SKirill A. Shutemov void migration_entry_wait_huge(struct vm_area_struct *vma, 349cb900f41SKirill A. Shutemov struct mm_struct *mm, pte_t *pte) 35030dad309SNaoya Horiguchi { 351cb900f41SKirill A. Shutemov spinlock_t *ptl = huge_pte_lockptr(hstate_vma(vma), mm, pte); 35230dad309SNaoya Horiguchi __migration_entry_wait(mm, pte, ptl); 35330dad309SNaoya Horiguchi } 35430dad309SNaoya Horiguchi 355616b8371SZi Yan #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 356616b8371SZi Yan void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd) 357616b8371SZi Yan { 358616b8371SZi Yan spinlock_t *ptl; 359616b8371SZi Yan struct page *page; 360616b8371SZi Yan 361616b8371SZi Yan ptl = pmd_lock(mm, pmd); 362616b8371SZi Yan if (!is_pmd_migration_entry(*pmd)) 363616b8371SZi Yan goto unlock; 364616b8371SZi Yan page = migration_entry_to_page(pmd_to_swp_entry(*pmd)); 365616b8371SZi Yan if (!get_page_unless_zero(page)) 366616b8371SZi Yan goto unlock; 367616b8371SZi Yan spin_unlock(ptl); 368616b8371SZi Yan wait_on_page_locked(page); 369616b8371SZi Yan put_page(page); 370616b8371SZi Yan return; 371616b8371SZi Yan unlock: 372616b8371SZi Yan spin_unlock(ptl); 373616b8371SZi Yan } 374616b8371SZi Yan #endif 375616b8371SZi Yan 376b969c4abSMel Gorman #ifdef CONFIG_BLOCK 377b969c4abSMel Gorman /* Returns true if all buffers are successfully locked */ 378a6bc32b8SMel Gorman static bool buffer_migrate_lock_buffers(struct buffer_head *head, 379a6bc32b8SMel Gorman enum migrate_mode mode) 380b969c4abSMel Gorman { 381b969c4abSMel Gorman struct buffer_head *bh = head; 382b969c4abSMel Gorman 383b969c4abSMel Gorman /* Simple case, sync compaction */ 384a6bc32b8SMel Gorman if (mode != MIGRATE_ASYNC) { 385b969c4abSMel Gorman do { 386b969c4abSMel Gorman get_bh(bh); 387b969c4abSMel Gorman lock_buffer(bh); 388b969c4abSMel Gorman bh = bh->b_this_page; 389b969c4abSMel Gorman 390b969c4abSMel Gorman } while (bh != head); 391b969c4abSMel Gorman 392b969c4abSMel Gorman return true; 393b969c4abSMel Gorman } 394b969c4abSMel Gorman 395b969c4abSMel Gorman /* async case, we cannot block on lock_buffer so use trylock_buffer */ 396b969c4abSMel Gorman do { 397b969c4abSMel Gorman get_bh(bh); 398b969c4abSMel Gorman if (!trylock_buffer(bh)) { 399b969c4abSMel Gorman /* 400b969c4abSMel Gorman * We failed to lock the buffer and cannot stall in 401b969c4abSMel Gorman * async migration. Release the taken locks 402b969c4abSMel Gorman */ 403b969c4abSMel Gorman struct buffer_head *failed_bh = bh; 404b969c4abSMel Gorman put_bh(failed_bh); 405b969c4abSMel Gorman bh = head; 406b969c4abSMel Gorman while (bh != failed_bh) { 407b969c4abSMel Gorman unlock_buffer(bh); 408b969c4abSMel Gorman put_bh(bh); 409b969c4abSMel Gorman bh = bh->b_this_page; 410b969c4abSMel Gorman } 411b969c4abSMel Gorman return false; 412b969c4abSMel Gorman } 413b969c4abSMel Gorman 414b969c4abSMel Gorman bh = bh->b_this_page; 415b969c4abSMel Gorman } while (bh != head); 416b969c4abSMel Gorman return true; 417b969c4abSMel Gorman } 418b969c4abSMel Gorman #else 419b969c4abSMel Gorman static inline bool buffer_migrate_lock_buffers(struct buffer_head *head, 420a6bc32b8SMel Gorman enum migrate_mode mode) 421b969c4abSMel Gorman { 422b969c4abSMel Gorman return true; 423b969c4abSMel Gorman } 424b969c4abSMel Gorman #endif /* CONFIG_BLOCK */ 425b969c4abSMel Gorman 426b20a3503SChristoph Lameter /* 427c3fcf8a5SChristoph Lameter * Replace the page in the mapping. 4285b5c7120SChristoph Lameter * 4295b5c7120SChristoph Lameter * The number of remaining references must be: 4305b5c7120SChristoph Lameter * 1 for anonymous pages without a mapping 4315b5c7120SChristoph Lameter * 2 for pages with a mapping 432266cf658SDavid Howells * 3 for pages with a mapping and PagePrivate/PagePrivate2 set. 433b20a3503SChristoph Lameter */ 43436bc08ccSGu Zheng int migrate_page_move_mapping(struct address_space *mapping, 435b969c4abSMel Gorman struct page *newpage, struct page *page, 4368e321fefSBenjamin LaHaise struct buffer_head *head, enum migrate_mode mode, 4378e321fefSBenjamin LaHaise int extra_count) 438b20a3503SChristoph Lameter { 43942cb14b1SHugh Dickins struct zone *oldzone, *newzone; 44042cb14b1SHugh Dickins int dirty; 4418e321fefSBenjamin LaHaise int expected_count = 1 + extra_count; 4427cf9c2c7SNick Piggin void **pslot; 443b20a3503SChristoph Lameter 4448763cb45SJérôme Glisse /* 445*df6ad698SJérôme Glisse * Device public or private pages have an extra refcount as they are 446*df6ad698SJérôme Glisse * ZONE_DEVICE pages. 4478763cb45SJérôme Glisse */ 448*df6ad698SJérôme Glisse expected_count += is_device_private_page(page); 449*df6ad698SJérôme Glisse expected_count += is_device_public_page(page); 4508763cb45SJérôme Glisse 4516c5240aeSChristoph Lameter if (!mapping) { 4520e8c7d0fSChristoph Lameter /* Anonymous page without mapping */ 4538e321fefSBenjamin LaHaise if (page_count(page) != expected_count) 4546c5240aeSChristoph Lameter return -EAGAIN; 455cf4b769aSHugh Dickins 456cf4b769aSHugh Dickins /* No turning back from here */ 457cf4b769aSHugh Dickins newpage->index = page->index; 458cf4b769aSHugh Dickins newpage->mapping = page->mapping; 459cf4b769aSHugh Dickins if (PageSwapBacked(page)) 460fa9949daSHugh Dickins __SetPageSwapBacked(newpage); 461cf4b769aSHugh Dickins 46278bd5209SRafael Aquini return MIGRATEPAGE_SUCCESS; 4636c5240aeSChristoph Lameter } 4646c5240aeSChristoph Lameter 46542cb14b1SHugh Dickins oldzone = page_zone(page); 46642cb14b1SHugh Dickins newzone = page_zone(newpage); 46742cb14b1SHugh Dickins 46819fd6231SNick Piggin spin_lock_irq(&mapping->tree_lock); 469b20a3503SChristoph Lameter 4707cf9c2c7SNick Piggin pslot = radix_tree_lookup_slot(&mapping->page_tree, 471b20a3503SChristoph Lameter page_index(page)); 472b20a3503SChristoph Lameter 4738e321fefSBenjamin LaHaise expected_count += 1 + page_has_private(page); 474e286781dSNick Piggin if (page_count(page) != expected_count || 47529c1f677SMel Gorman radix_tree_deref_slot_protected(pslot, &mapping->tree_lock) != page) { 47619fd6231SNick Piggin spin_unlock_irq(&mapping->tree_lock); 477e23ca00bSChristoph Lameter return -EAGAIN; 478b20a3503SChristoph Lameter } 479b20a3503SChristoph Lameter 480fe896d18SJoonsoo Kim if (!page_ref_freeze(page, expected_count)) { 48119fd6231SNick Piggin spin_unlock_irq(&mapping->tree_lock); 482e286781dSNick Piggin return -EAGAIN; 483e286781dSNick Piggin } 484e286781dSNick Piggin 485b20a3503SChristoph Lameter /* 486b969c4abSMel Gorman * In the async migration case of moving a page with buffers, lock the 487b969c4abSMel Gorman * buffers using trylock before the mapping is moved. If the mapping 488b969c4abSMel Gorman * was moved, we later failed to lock the buffers and could not move 489b969c4abSMel Gorman * the mapping back due to an elevated page count, we would have to 490b969c4abSMel Gorman * block waiting on other references to be dropped. 491b969c4abSMel Gorman */ 492a6bc32b8SMel Gorman if (mode == MIGRATE_ASYNC && head && 493a6bc32b8SMel Gorman !buffer_migrate_lock_buffers(head, mode)) { 494fe896d18SJoonsoo Kim page_ref_unfreeze(page, expected_count); 495b969c4abSMel Gorman spin_unlock_irq(&mapping->tree_lock); 496b969c4abSMel Gorman return -EAGAIN; 497b969c4abSMel Gorman } 498b969c4abSMel Gorman 499b969c4abSMel Gorman /* 500cf4b769aSHugh Dickins * Now we know that no one else is looking at the page: 501cf4b769aSHugh Dickins * no turning back from here. 502b20a3503SChristoph Lameter */ 503cf4b769aSHugh Dickins newpage->index = page->index; 504cf4b769aSHugh Dickins newpage->mapping = page->mapping; 5057cf9c2c7SNick Piggin get_page(newpage); /* add cache reference */ 5066326fec1SNicholas Piggin if (PageSwapBacked(page)) { 5076326fec1SNicholas Piggin __SetPageSwapBacked(newpage); 508b20a3503SChristoph Lameter if (PageSwapCache(page)) { 509b20a3503SChristoph Lameter SetPageSwapCache(newpage); 510b20a3503SChristoph Lameter set_page_private(newpage, page_private(page)); 511b20a3503SChristoph Lameter } 5126326fec1SNicholas Piggin } else { 5136326fec1SNicholas Piggin VM_BUG_ON_PAGE(PageSwapCache(page), page); 5146326fec1SNicholas Piggin } 515b20a3503SChristoph Lameter 51642cb14b1SHugh Dickins /* Move dirty while page refs frozen and newpage not yet exposed */ 51742cb14b1SHugh Dickins dirty = PageDirty(page); 51842cb14b1SHugh Dickins if (dirty) { 51942cb14b1SHugh Dickins ClearPageDirty(page); 52042cb14b1SHugh Dickins SetPageDirty(newpage); 52142cb14b1SHugh Dickins } 52242cb14b1SHugh Dickins 5236d75f366SJohannes Weiner radix_tree_replace_slot(&mapping->page_tree, pslot, newpage); 5247cf9c2c7SNick Piggin 5257cf9c2c7SNick Piggin /* 526937a94c9SJacobo Giralt * Drop cache reference from old page by unfreezing 527937a94c9SJacobo Giralt * to one less reference. 5287cf9c2c7SNick Piggin * We know this isn't the last reference. 5297cf9c2c7SNick Piggin */ 530fe896d18SJoonsoo Kim page_ref_unfreeze(page, expected_count - 1); 5317cf9c2c7SNick Piggin 53242cb14b1SHugh Dickins spin_unlock(&mapping->tree_lock); 53342cb14b1SHugh Dickins /* Leave irq disabled to prevent preemption while updating stats */ 53442cb14b1SHugh Dickins 5350e8c7d0fSChristoph Lameter /* 5360e8c7d0fSChristoph Lameter * If moved to a different zone then also account 5370e8c7d0fSChristoph Lameter * the page for that zone. Other VM counters will be 5380e8c7d0fSChristoph Lameter * taken care of when we establish references to the 5390e8c7d0fSChristoph Lameter * new page and drop references to the old page. 5400e8c7d0fSChristoph Lameter * 5410e8c7d0fSChristoph Lameter * Note that anonymous pages are accounted for 5424b9d0fabSMel Gorman * via NR_FILE_PAGES and NR_ANON_MAPPED if they 5430e8c7d0fSChristoph Lameter * are mapped to swap space. 5440e8c7d0fSChristoph Lameter */ 54542cb14b1SHugh Dickins if (newzone != oldzone) { 54611fb9989SMel Gorman __dec_node_state(oldzone->zone_pgdat, NR_FILE_PAGES); 54711fb9989SMel Gorman __inc_node_state(newzone->zone_pgdat, NR_FILE_PAGES); 54842cb14b1SHugh Dickins if (PageSwapBacked(page) && !PageSwapCache(page)) { 54911fb9989SMel Gorman __dec_node_state(oldzone->zone_pgdat, NR_SHMEM); 55011fb9989SMel Gorman __inc_node_state(newzone->zone_pgdat, NR_SHMEM); 5514b02108aSKOSAKI Motohiro } 55242cb14b1SHugh Dickins if (dirty && mapping_cap_account_dirty(mapping)) { 55311fb9989SMel Gorman __dec_node_state(oldzone->zone_pgdat, NR_FILE_DIRTY); 5545a1c84b4SMel Gorman __dec_zone_state(oldzone, NR_ZONE_WRITE_PENDING); 55511fb9989SMel Gorman __inc_node_state(newzone->zone_pgdat, NR_FILE_DIRTY); 5565a1c84b4SMel Gorman __inc_zone_state(newzone, NR_ZONE_WRITE_PENDING); 55742cb14b1SHugh Dickins } 55842cb14b1SHugh Dickins } 55942cb14b1SHugh Dickins local_irq_enable(); 560b20a3503SChristoph Lameter 56178bd5209SRafael Aquini return MIGRATEPAGE_SUCCESS; 562b20a3503SChristoph Lameter } 5631118dce7SRichard Weinberger EXPORT_SYMBOL(migrate_page_move_mapping); 564b20a3503SChristoph Lameter 565b20a3503SChristoph Lameter /* 566290408d4SNaoya Horiguchi * The expected number of remaining references is the same as that 567290408d4SNaoya Horiguchi * of migrate_page_move_mapping(). 568290408d4SNaoya Horiguchi */ 569290408d4SNaoya Horiguchi int migrate_huge_page_move_mapping(struct address_space *mapping, 570290408d4SNaoya Horiguchi struct page *newpage, struct page *page) 571290408d4SNaoya Horiguchi { 572290408d4SNaoya Horiguchi int expected_count; 573290408d4SNaoya Horiguchi void **pslot; 574290408d4SNaoya Horiguchi 575290408d4SNaoya Horiguchi spin_lock_irq(&mapping->tree_lock); 576290408d4SNaoya Horiguchi 577290408d4SNaoya Horiguchi pslot = radix_tree_lookup_slot(&mapping->page_tree, 578290408d4SNaoya Horiguchi page_index(page)); 579290408d4SNaoya Horiguchi 580290408d4SNaoya Horiguchi expected_count = 2 + page_has_private(page); 581290408d4SNaoya Horiguchi if (page_count(page) != expected_count || 58229c1f677SMel Gorman radix_tree_deref_slot_protected(pslot, &mapping->tree_lock) != page) { 583290408d4SNaoya Horiguchi spin_unlock_irq(&mapping->tree_lock); 584290408d4SNaoya Horiguchi return -EAGAIN; 585290408d4SNaoya Horiguchi } 586290408d4SNaoya Horiguchi 587fe896d18SJoonsoo Kim if (!page_ref_freeze(page, expected_count)) { 588290408d4SNaoya Horiguchi spin_unlock_irq(&mapping->tree_lock); 589290408d4SNaoya Horiguchi return -EAGAIN; 590290408d4SNaoya Horiguchi } 591290408d4SNaoya Horiguchi 592cf4b769aSHugh Dickins newpage->index = page->index; 593cf4b769aSHugh Dickins newpage->mapping = page->mapping; 5946a93ca8fSJohannes Weiner 595290408d4SNaoya Horiguchi get_page(newpage); 596290408d4SNaoya Horiguchi 5976d75f366SJohannes Weiner radix_tree_replace_slot(&mapping->page_tree, pslot, newpage); 598290408d4SNaoya Horiguchi 599fe896d18SJoonsoo Kim page_ref_unfreeze(page, expected_count - 1); 600290408d4SNaoya Horiguchi 601290408d4SNaoya Horiguchi spin_unlock_irq(&mapping->tree_lock); 6026a93ca8fSJohannes Weiner 60378bd5209SRafael Aquini return MIGRATEPAGE_SUCCESS; 604290408d4SNaoya Horiguchi } 605290408d4SNaoya Horiguchi 606290408d4SNaoya Horiguchi /* 60730b0a105SDave Hansen * Gigantic pages are so large that we do not guarantee that page++ pointer 60830b0a105SDave Hansen * arithmetic will work across the entire page. We need something more 60930b0a105SDave Hansen * specialized. 61030b0a105SDave Hansen */ 61130b0a105SDave Hansen static void __copy_gigantic_page(struct page *dst, struct page *src, 61230b0a105SDave Hansen int nr_pages) 61330b0a105SDave Hansen { 61430b0a105SDave Hansen int i; 61530b0a105SDave Hansen struct page *dst_base = dst; 61630b0a105SDave Hansen struct page *src_base = src; 61730b0a105SDave Hansen 61830b0a105SDave Hansen for (i = 0; i < nr_pages; ) { 61930b0a105SDave Hansen cond_resched(); 62030b0a105SDave Hansen copy_highpage(dst, src); 62130b0a105SDave Hansen 62230b0a105SDave Hansen i++; 62330b0a105SDave Hansen dst = mem_map_next(dst, dst_base, i); 62430b0a105SDave Hansen src = mem_map_next(src, src_base, i); 62530b0a105SDave Hansen } 62630b0a105SDave Hansen } 62730b0a105SDave Hansen 62830b0a105SDave Hansen static void copy_huge_page(struct page *dst, struct page *src) 62930b0a105SDave Hansen { 63030b0a105SDave Hansen int i; 63130b0a105SDave Hansen int nr_pages; 63230b0a105SDave Hansen 63330b0a105SDave Hansen if (PageHuge(src)) { 63430b0a105SDave Hansen /* hugetlbfs page */ 63530b0a105SDave Hansen struct hstate *h = page_hstate(src); 63630b0a105SDave Hansen nr_pages = pages_per_huge_page(h); 63730b0a105SDave Hansen 63830b0a105SDave Hansen if (unlikely(nr_pages > MAX_ORDER_NR_PAGES)) { 63930b0a105SDave Hansen __copy_gigantic_page(dst, src, nr_pages); 64030b0a105SDave Hansen return; 64130b0a105SDave Hansen } 64230b0a105SDave Hansen } else { 64330b0a105SDave Hansen /* thp page */ 64430b0a105SDave Hansen BUG_ON(!PageTransHuge(src)); 64530b0a105SDave Hansen nr_pages = hpage_nr_pages(src); 64630b0a105SDave Hansen } 64730b0a105SDave Hansen 64830b0a105SDave Hansen for (i = 0; i < nr_pages; i++) { 64930b0a105SDave Hansen cond_resched(); 65030b0a105SDave Hansen copy_highpage(dst + i, src + i); 65130b0a105SDave Hansen } 65230b0a105SDave Hansen } 65330b0a105SDave Hansen 65430b0a105SDave Hansen /* 655b20a3503SChristoph Lameter * Copy the page to its new location 656b20a3503SChristoph Lameter */ 6572916ecc0SJérôme Glisse void migrate_page_states(struct page *newpage, struct page *page) 658b20a3503SChristoph Lameter { 6597851a45cSRik van Riel int cpupid; 6607851a45cSRik van Riel 661b20a3503SChristoph Lameter if (PageError(page)) 662b20a3503SChristoph Lameter SetPageError(newpage); 663b20a3503SChristoph Lameter if (PageReferenced(page)) 664b20a3503SChristoph Lameter SetPageReferenced(newpage); 665b20a3503SChristoph Lameter if (PageUptodate(page)) 666b20a3503SChristoph Lameter SetPageUptodate(newpage); 667894bc310SLee Schermerhorn if (TestClearPageActive(page)) { 668309381feSSasha Levin VM_BUG_ON_PAGE(PageUnevictable(page), page); 669b20a3503SChristoph Lameter SetPageActive(newpage); 670418b27efSLee Schermerhorn } else if (TestClearPageUnevictable(page)) 671418b27efSLee Schermerhorn SetPageUnevictable(newpage); 672b20a3503SChristoph Lameter if (PageChecked(page)) 673b20a3503SChristoph Lameter SetPageChecked(newpage); 674b20a3503SChristoph Lameter if (PageMappedToDisk(page)) 675b20a3503SChristoph Lameter SetPageMappedToDisk(newpage); 676b20a3503SChristoph Lameter 67742cb14b1SHugh Dickins /* Move dirty on pages not done by migrate_page_move_mapping() */ 67842cb14b1SHugh Dickins if (PageDirty(page)) 679752dc185SHugh Dickins SetPageDirty(newpage); 680b20a3503SChristoph Lameter 68133c3fc71SVladimir Davydov if (page_is_young(page)) 68233c3fc71SVladimir Davydov set_page_young(newpage); 68333c3fc71SVladimir Davydov if (page_is_idle(page)) 68433c3fc71SVladimir Davydov set_page_idle(newpage); 68533c3fc71SVladimir Davydov 6867851a45cSRik van Riel /* 6877851a45cSRik van Riel * Copy NUMA information to the new page, to prevent over-eager 6887851a45cSRik van Riel * future migrations of this same page. 6897851a45cSRik van Riel */ 6907851a45cSRik van Riel cpupid = page_cpupid_xchg_last(page, -1); 6917851a45cSRik van Riel page_cpupid_xchg_last(newpage, cpupid); 6927851a45cSRik van Riel 693e9995ef9SHugh Dickins ksm_migrate_page(newpage, page); 694c8d6553bSHugh Dickins /* 695c8d6553bSHugh Dickins * Please do not reorder this without considering how mm/ksm.c's 696c8d6553bSHugh Dickins * get_ksm_page() depends upon ksm_migrate_page() and PageSwapCache(). 697c8d6553bSHugh Dickins */ 698b3b3a99cSNaoya Horiguchi if (PageSwapCache(page)) 699b20a3503SChristoph Lameter ClearPageSwapCache(page); 700b20a3503SChristoph Lameter ClearPagePrivate(page); 701b20a3503SChristoph Lameter set_page_private(page, 0); 702b20a3503SChristoph Lameter 703b20a3503SChristoph Lameter /* 704b20a3503SChristoph Lameter * If any waiters have accumulated on the new page then 705b20a3503SChristoph Lameter * wake them up. 706b20a3503SChristoph Lameter */ 707b20a3503SChristoph Lameter if (PageWriteback(newpage)) 708b20a3503SChristoph Lameter end_page_writeback(newpage); 709d435edcaSVlastimil Babka 710d435edcaSVlastimil Babka copy_page_owner(page, newpage); 71174485cf2SJohannes Weiner 71274485cf2SJohannes Weiner mem_cgroup_migrate(page, newpage); 713b20a3503SChristoph Lameter } 7142916ecc0SJérôme Glisse EXPORT_SYMBOL(migrate_page_states); 7152916ecc0SJérôme Glisse 7162916ecc0SJérôme Glisse void migrate_page_copy(struct page *newpage, struct page *page) 7172916ecc0SJérôme Glisse { 7182916ecc0SJérôme Glisse if (PageHuge(page) || PageTransHuge(page)) 7192916ecc0SJérôme Glisse copy_huge_page(newpage, page); 7202916ecc0SJérôme Glisse else 7212916ecc0SJérôme Glisse copy_highpage(newpage, page); 7222916ecc0SJérôme Glisse 7232916ecc0SJérôme Glisse migrate_page_states(newpage, page); 7242916ecc0SJérôme Glisse } 7251118dce7SRichard Weinberger EXPORT_SYMBOL(migrate_page_copy); 726b20a3503SChristoph Lameter 7271d8b85ccSChristoph Lameter /************************************************************ 7281d8b85ccSChristoph Lameter * Migration functions 7291d8b85ccSChristoph Lameter ***********************************************************/ 7301d8b85ccSChristoph Lameter 731b20a3503SChristoph Lameter /* 732bda807d4SMinchan Kim * Common logic to directly migrate a single LRU page suitable for 733266cf658SDavid Howells * pages that do not use PagePrivate/PagePrivate2. 734b20a3503SChristoph Lameter * 735b20a3503SChristoph Lameter * Pages are locked upon entry and exit. 736b20a3503SChristoph Lameter */ 7372d1db3b1SChristoph Lameter int migrate_page(struct address_space *mapping, 738a6bc32b8SMel Gorman struct page *newpage, struct page *page, 739a6bc32b8SMel Gorman enum migrate_mode mode) 740b20a3503SChristoph Lameter { 741b20a3503SChristoph Lameter int rc; 742b20a3503SChristoph Lameter 743b20a3503SChristoph Lameter BUG_ON(PageWriteback(page)); /* Writeback must be complete */ 744b20a3503SChristoph Lameter 7458e321fefSBenjamin LaHaise rc = migrate_page_move_mapping(mapping, newpage, page, NULL, mode, 0); 746b20a3503SChristoph Lameter 74778bd5209SRafael Aquini if (rc != MIGRATEPAGE_SUCCESS) 748b20a3503SChristoph Lameter return rc; 749b20a3503SChristoph Lameter 7502916ecc0SJérôme Glisse if (mode != MIGRATE_SYNC_NO_COPY) 751b20a3503SChristoph Lameter migrate_page_copy(newpage, page); 7522916ecc0SJérôme Glisse else 7532916ecc0SJérôme Glisse migrate_page_states(newpage, page); 75478bd5209SRafael Aquini return MIGRATEPAGE_SUCCESS; 755b20a3503SChristoph Lameter } 756b20a3503SChristoph Lameter EXPORT_SYMBOL(migrate_page); 757b20a3503SChristoph Lameter 7589361401eSDavid Howells #ifdef CONFIG_BLOCK 759b20a3503SChristoph Lameter /* 7601d8b85ccSChristoph Lameter * Migration function for pages with buffers. This function can only be used 7611d8b85ccSChristoph Lameter * if the underlying filesystem guarantees that no other references to "page" 7621d8b85ccSChristoph Lameter * exist. 7631d8b85ccSChristoph Lameter */ 7642d1db3b1SChristoph Lameter int buffer_migrate_page(struct address_space *mapping, 765a6bc32b8SMel Gorman struct page *newpage, struct page *page, enum migrate_mode mode) 7661d8b85ccSChristoph Lameter { 7671d8b85ccSChristoph Lameter struct buffer_head *bh, *head; 7681d8b85ccSChristoph Lameter int rc; 7691d8b85ccSChristoph Lameter 7701d8b85ccSChristoph Lameter if (!page_has_buffers(page)) 771a6bc32b8SMel Gorman return migrate_page(mapping, newpage, page, mode); 7721d8b85ccSChristoph Lameter 7731d8b85ccSChristoph Lameter head = page_buffers(page); 7741d8b85ccSChristoph Lameter 7758e321fefSBenjamin LaHaise rc = migrate_page_move_mapping(mapping, newpage, page, head, mode, 0); 7761d8b85ccSChristoph Lameter 77778bd5209SRafael Aquini if (rc != MIGRATEPAGE_SUCCESS) 7781d8b85ccSChristoph Lameter return rc; 7791d8b85ccSChristoph Lameter 780b969c4abSMel Gorman /* 781b969c4abSMel Gorman * In the async case, migrate_page_move_mapping locked the buffers 782b969c4abSMel Gorman * with an IRQ-safe spinlock held. In the sync case, the buffers 783b969c4abSMel Gorman * need to be locked now 784b969c4abSMel Gorman */ 785a6bc32b8SMel Gorman if (mode != MIGRATE_ASYNC) 786a6bc32b8SMel Gorman BUG_ON(!buffer_migrate_lock_buffers(head, mode)); 7871d8b85ccSChristoph Lameter 7881d8b85ccSChristoph Lameter ClearPagePrivate(page); 7891d8b85ccSChristoph Lameter set_page_private(newpage, page_private(page)); 7901d8b85ccSChristoph Lameter set_page_private(page, 0); 7911d8b85ccSChristoph Lameter put_page(page); 7921d8b85ccSChristoph Lameter get_page(newpage); 7931d8b85ccSChristoph Lameter 7941d8b85ccSChristoph Lameter bh = head; 7951d8b85ccSChristoph Lameter do { 7961d8b85ccSChristoph Lameter set_bh_page(bh, newpage, bh_offset(bh)); 7971d8b85ccSChristoph Lameter bh = bh->b_this_page; 7981d8b85ccSChristoph Lameter 7991d8b85ccSChristoph Lameter } while (bh != head); 8001d8b85ccSChristoph Lameter 8011d8b85ccSChristoph Lameter SetPagePrivate(newpage); 8021d8b85ccSChristoph Lameter 8032916ecc0SJérôme Glisse if (mode != MIGRATE_SYNC_NO_COPY) 8041d8b85ccSChristoph Lameter migrate_page_copy(newpage, page); 8052916ecc0SJérôme Glisse else 8062916ecc0SJérôme Glisse migrate_page_states(newpage, page); 8071d8b85ccSChristoph Lameter 8081d8b85ccSChristoph Lameter bh = head; 8091d8b85ccSChristoph Lameter do { 8101d8b85ccSChristoph Lameter unlock_buffer(bh); 8111d8b85ccSChristoph Lameter put_bh(bh); 8121d8b85ccSChristoph Lameter bh = bh->b_this_page; 8131d8b85ccSChristoph Lameter 8141d8b85ccSChristoph Lameter } while (bh != head); 8151d8b85ccSChristoph Lameter 81678bd5209SRafael Aquini return MIGRATEPAGE_SUCCESS; 8171d8b85ccSChristoph Lameter } 8181d8b85ccSChristoph Lameter EXPORT_SYMBOL(buffer_migrate_page); 8199361401eSDavid Howells #endif 8201d8b85ccSChristoph Lameter 82104e62a29SChristoph Lameter /* 82204e62a29SChristoph Lameter * Writeback a page to clean the dirty state 82304e62a29SChristoph Lameter */ 82404e62a29SChristoph Lameter static int writeout(struct address_space *mapping, struct page *page) 82504e62a29SChristoph Lameter { 82604e62a29SChristoph Lameter struct writeback_control wbc = { 82704e62a29SChristoph Lameter .sync_mode = WB_SYNC_NONE, 82804e62a29SChristoph Lameter .nr_to_write = 1, 82904e62a29SChristoph Lameter .range_start = 0, 83004e62a29SChristoph Lameter .range_end = LLONG_MAX, 83104e62a29SChristoph Lameter .for_reclaim = 1 83204e62a29SChristoph Lameter }; 83304e62a29SChristoph Lameter int rc; 83404e62a29SChristoph Lameter 83504e62a29SChristoph Lameter if (!mapping->a_ops->writepage) 83604e62a29SChristoph Lameter /* No write method for the address space */ 83704e62a29SChristoph Lameter return -EINVAL; 83804e62a29SChristoph Lameter 83904e62a29SChristoph Lameter if (!clear_page_dirty_for_io(page)) 84004e62a29SChristoph Lameter /* Someone else already triggered a write */ 84104e62a29SChristoph Lameter return -EAGAIN; 84204e62a29SChristoph Lameter 84304e62a29SChristoph Lameter /* 84404e62a29SChristoph Lameter * A dirty page may imply that the underlying filesystem has 84504e62a29SChristoph Lameter * the page on some queue. So the page must be clean for 84604e62a29SChristoph Lameter * migration. Writeout may mean we loose the lock and the 84704e62a29SChristoph Lameter * page state is no longer what we checked for earlier. 84804e62a29SChristoph Lameter * At this point we know that the migration attempt cannot 84904e62a29SChristoph Lameter * be successful. 85004e62a29SChristoph Lameter */ 851e388466dSKirill A. Shutemov remove_migration_ptes(page, page, false); 85204e62a29SChristoph Lameter 85304e62a29SChristoph Lameter rc = mapping->a_ops->writepage(page, &wbc); 85404e62a29SChristoph Lameter 85504e62a29SChristoph Lameter if (rc != AOP_WRITEPAGE_ACTIVATE) 85604e62a29SChristoph Lameter /* unlocked. Relock */ 85704e62a29SChristoph Lameter lock_page(page); 85804e62a29SChristoph Lameter 859bda8550dSHugh Dickins return (rc < 0) ? -EIO : -EAGAIN; 86004e62a29SChristoph Lameter } 86104e62a29SChristoph Lameter 86204e62a29SChristoph Lameter /* 86304e62a29SChristoph Lameter * Default handling if a filesystem does not provide a migration function. 86404e62a29SChristoph Lameter */ 8658351a6e4SChristoph Lameter static int fallback_migrate_page(struct address_space *mapping, 866a6bc32b8SMel Gorman struct page *newpage, struct page *page, enum migrate_mode mode) 8678351a6e4SChristoph Lameter { 868b969c4abSMel Gorman if (PageDirty(page)) { 869a6bc32b8SMel Gorman /* Only writeback pages in full synchronous migration */ 8702916ecc0SJérôme Glisse switch (mode) { 8712916ecc0SJérôme Glisse case MIGRATE_SYNC: 8722916ecc0SJérôme Glisse case MIGRATE_SYNC_NO_COPY: 8732916ecc0SJérôme Glisse break; 8742916ecc0SJérôme Glisse default: 875b969c4abSMel Gorman return -EBUSY; 8762916ecc0SJérôme Glisse } 87704e62a29SChristoph Lameter return writeout(mapping, page); 878b969c4abSMel Gorman } 8798351a6e4SChristoph Lameter 8808351a6e4SChristoph Lameter /* 8818351a6e4SChristoph Lameter * Buffers may be managed in a filesystem specific way. 8828351a6e4SChristoph Lameter * We must have no buffers or drop them. 8838351a6e4SChristoph Lameter */ 884266cf658SDavid Howells if (page_has_private(page) && 8858351a6e4SChristoph Lameter !try_to_release_page(page, GFP_KERNEL)) 8868351a6e4SChristoph Lameter return -EAGAIN; 8878351a6e4SChristoph Lameter 888a6bc32b8SMel Gorman return migrate_page(mapping, newpage, page, mode); 8898351a6e4SChristoph Lameter } 8908351a6e4SChristoph Lameter 8911d8b85ccSChristoph Lameter /* 892e24f0b8fSChristoph Lameter * Move a page to a newly allocated page 893e24f0b8fSChristoph Lameter * The page is locked and all ptes have been successfully removed. 894b20a3503SChristoph Lameter * 895e24f0b8fSChristoph Lameter * The new page will have replaced the old page if this function 896e24f0b8fSChristoph Lameter * is successful. 897894bc310SLee Schermerhorn * 898894bc310SLee Schermerhorn * Return value: 899894bc310SLee Schermerhorn * < 0 - error code 90078bd5209SRafael Aquini * MIGRATEPAGE_SUCCESS - success 901b20a3503SChristoph Lameter */ 9023fe2011fSMel Gorman static int move_to_new_page(struct page *newpage, struct page *page, 9035c3f9a67SHugh Dickins enum migrate_mode mode) 904b20a3503SChristoph Lameter { 905e24f0b8fSChristoph Lameter struct address_space *mapping; 906bda807d4SMinchan Kim int rc = -EAGAIN; 907bda807d4SMinchan Kim bool is_lru = !__PageMovable(page); 908b20a3503SChristoph Lameter 9097db7671fSHugh Dickins VM_BUG_ON_PAGE(!PageLocked(page), page); 9107db7671fSHugh Dickins VM_BUG_ON_PAGE(!PageLocked(newpage), newpage); 911b20a3503SChristoph Lameter 912b20a3503SChristoph Lameter mapping = page_mapping(page); 913bda807d4SMinchan Kim 914bda807d4SMinchan Kim if (likely(is_lru)) { 915b20a3503SChristoph Lameter if (!mapping) 916a6bc32b8SMel Gorman rc = migrate_page(mapping, newpage, page, mode); 9176c5240aeSChristoph Lameter else if (mapping->a_ops->migratepage) 918b20a3503SChristoph Lameter /* 919bda807d4SMinchan Kim * Most pages have a mapping and most filesystems 920bda807d4SMinchan Kim * provide a migratepage callback. Anonymous pages 921bda807d4SMinchan Kim * are part of swap space which also has its own 922bda807d4SMinchan Kim * migratepage callback. This is the most common path 923bda807d4SMinchan Kim * for page migration. 924b20a3503SChristoph Lameter */ 925bda807d4SMinchan Kim rc = mapping->a_ops->migratepage(mapping, newpage, 926bda807d4SMinchan Kim page, mode); 9278351a6e4SChristoph Lameter else 928bda807d4SMinchan Kim rc = fallback_migrate_page(mapping, newpage, 929bda807d4SMinchan Kim page, mode); 930bda807d4SMinchan Kim } else { 931bda807d4SMinchan Kim /* 932bda807d4SMinchan Kim * In case of non-lru page, it could be released after 933bda807d4SMinchan Kim * isolation step. In that case, we shouldn't try migration. 934bda807d4SMinchan Kim */ 935bda807d4SMinchan Kim VM_BUG_ON_PAGE(!PageIsolated(page), page); 936bda807d4SMinchan Kim if (!PageMovable(page)) { 937bda807d4SMinchan Kim rc = MIGRATEPAGE_SUCCESS; 938bda807d4SMinchan Kim __ClearPageIsolated(page); 939bda807d4SMinchan Kim goto out; 940bda807d4SMinchan Kim } 941bda807d4SMinchan Kim 942bda807d4SMinchan Kim rc = mapping->a_ops->migratepage(mapping, newpage, 943bda807d4SMinchan Kim page, mode); 944bda807d4SMinchan Kim WARN_ON_ONCE(rc == MIGRATEPAGE_SUCCESS && 945bda807d4SMinchan Kim !PageIsolated(page)); 946bda807d4SMinchan Kim } 947b20a3503SChristoph Lameter 9485c3f9a67SHugh Dickins /* 9495c3f9a67SHugh Dickins * When successful, old pagecache page->mapping must be cleared before 9505c3f9a67SHugh Dickins * page is freed; but stats require that PageAnon be left as PageAnon. 9515c3f9a67SHugh Dickins */ 9525c3f9a67SHugh Dickins if (rc == MIGRATEPAGE_SUCCESS) { 953bda807d4SMinchan Kim if (__PageMovable(page)) { 954bda807d4SMinchan Kim VM_BUG_ON_PAGE(!PageIsolated(page), page); 955bda807d4SMinchan Kim 956bda807d4SMinchan Kim /* 957bda807d4SMinchan Kim * We clear PG_movable under page_lock so any compactor 958bda807d4SMinchan Kim * cannot try to migrate this page. 959bda807d4SMinchan Kim */ 960bda807d4SMinchan Kim __ClearPageIsolated(page); 961bda807d4SMinchan Kim } 962bda807d4SMinchan Kim 963bda807d4SMinchan Kim /* 964bda807d4SMinchan Kim * Anonymous and movable page->mapping will be cleard by 965bda807d4SMinchan Kim * free_pages_prepare so don't reset it here for keeping 966bda807d4SMinchan Kim * the type to work PageAnon, for example. 967bda807d4SMinchan Kim */ 968bda807d4SMinchan Kim if (!PageMappingFlags(page)) 9695c3f9a67SHugh Dickins page->mapping = NULL; 9703fe2011fSMel Gorman } 971bda807d4SMinchan Kim out: 972e24f0b8fSChristoph Lameter return rc; 973e24f0b8fSChristoph Lameter } 974e24f0b8fSChristoph Lameter 9750dabec93SMinchan Kim static int __unmap_and_move(struct page *page, struct page *newpage, 9769c620e2bSHugh Dickins int force, enum migrate_mode mode) 977e24f0b8fSChristoph Lameter { 9780dabec93SMinchan Kim int rc = -EAGAIN; 9792ebba6b7SHugh Dickins int page_was_mapped = 0; 9803f6c8272SMel Gorman struct anon_vma *anon_vma = NULL; 981bda807d4SMinchan Kim bool is_lru = !__PageMovable(page); 98295a402c3SChristoph Lameter 983529ae9aaSNick Piggin if (!trylock_page(page)) { 984a6bc32b8SMel Gorman if (!force || mode == MIGRATE_ASYNC) 9850dabec93SMinchan Kim goto out; 9863e7d3449SMel Gorman 9873e7d3449SMel Gorman /* 9883e7d3449SMel Gorman * It's not safe for direct compaction to call lock_page. 9893e7d3449SMel Gorman * For example, during page readahead pages are added locked 9903e7d3449SMel Gorman * to the LRU. Later, when the IO completes the pages are 9913e7d3449SMel Gorman * marked uptodate and unlocked. However, the queueing 9923e7d3449SMel Gorman * could be merging multiple pages for one bio (e.g. 9933e7d3449SMel Gorman * mpage_readpages). If an allocation happens for the 9943e7d3449SMel Gorman * second or third page, the process can end up locking 9953e7d3449SMel Gorman * the same page twice and deadlocking. Rather than 9963e7d3449SMel Gorman * trying to be clever about what pages can be locked, 9973e7d3449SMel Gorman * avoid the use of lock_page for direct compaction 9983e7d3449SMel Gorman * altogether. 9993e7d3449SMel Gorman */ 10003e7d3449SMel Gorman if (current->flags & PF_MEMALLOC) 10010dabec93SMinchan Kim goto out; 10023e7d3449SMel Gorman 1003e24f0b8fSChristoph Lameter lock_page(page); 1004e24f0b8fSChristoph Lameter } 1005e24f0b8fSChristoph Lameter 1006e24f0b8fSChristoph Lameter if (PageWriteback(page)) { 100711bc82d6SAndrea Arcangeli /* 1008fed5b64aSJianguo Wu * Only in the case of a full synchronous migration is it 1009a6bc32b8SMel Gorman * necessary to wait for PageWriteback. In the async case, 1010a6bc32b8SMel Gorman * the retry loop is too short and in the sync-light case, 1011a6bc32b8SMel Gorman * the overhead of stalling is too much 101211bc82d6SAndrea Arcangeli */ 10132916ecc0SJérôme Glisse switch (mode) { 10142916ecc0SJérôme Glisse case MIGRATE_SYNC: 10152916ecc0SJérôme Glisse case MIGRATE_SYNC_NO_COPY: 10162916ecc0SJérôme Glisse break; 10172916ecc0SJérôme Glisse default: 101811bc82d6SAndrea Arcangeli rc = -EBUSY; 10190a31bc97SJohannes Weiner goto out_unlock; 102011bc82d6SAndrea Arcangeli } 102111bc82d6SAndrea Arcangeli if (!force) 10220a31bc97SJohannes Weiner goto out_unlock; 1023e24f0b8fSChristoph Lameter wait_on_page_writeback(page); 1024e24f0b8fSChristoph Lameter } 102503f15c86SHugh Dickins 1026e24f0b8fSChristoph Lameter /* 1027dc386d4dSKAMEZAWA Hiroyuki * By try_to_unmap(), page->mapcount goes down to 0 here. In this case, 1028dc386d4dSKAMEZAWA Hiroyuki * we cannot notice that anon_vma is freed while we migrates a page. 10291ce82b69SHugh Dickins * This get_anon_vma() delays freeing anon_vma pointer until the end 1030dc386d4dSKAMEZAWA Hiroyuki * of migration. File cache pages are no problem because of page_lock() 1031989f89c5SKAMEZAWA Hiroyuki * File Caches may use write_page() or lock_page() in migration, then, 1032989f89c5SKAMEZAWA Hiroyuki * just care Anon page here. 10333fe2011fSMel Gorman * 103403f15c86SHugh Dickins * Only page_get_anon_vma() understands the subtleties of 103503f15c86SHugh Dickins * getting a hold on an anon_vma from outside one of its mms. 103603f15c86SHugh Dickins * But if we cannot get anon_vma, then we won't need it anyway, 103703f15c86SHugh Dickins * because that implies that the anon page is no longer mapped 103803f15c86SHugh Dickins * (and cannot be remapped so long as we hold the page lock). 10393fe2011fSMel Gorman */ 104003f15c86SHugh Dickins if (PageAnon(page) && !PageKsm(page)) 104103f15c86SHugh Dickins anon_vma = page_get_anon_vma(page); 104262e1c553SShaohua Li 10437db7671fSHugh Dickins /* 10447db7671fSHugh Dickins * Block others from accessing the new page when we get around to 10457db7671fSHugh Dickins * establishing additional references. We are usually the only one 10467db7671fSHugh Dickins * holding a reference to newpage at this point. We used to have a BUG 10477db7671fSHugh Dickins * here if trylock_page(newpage) fails, but would like to allow for 10487db7671fSHugh Dickins * cases where there might be a race with the previous use of newpage. 10497db7671fSHugh Dickins * This is much like races on refcount of oldpage: just don't BUG(). 10507db7671fSHugh Dickins */ 10517db7671fSHugh Dickins if (unlikely(!trylock_page(newpage))) 10527db7671fSHugh Dickins goto out_unlock; 10537db7671fSHugh Dickins 1054bda807d4SMinchan Kim if (unlikely(!is_lru)) { 1055bda807d4SMinchan Kim rc = move_to_new_page(newpage, page, mode); 1056bda807d4SMinchan Kim goto out_unlock_both; 1057bda807d4SMinchan Kim } 1058bda807d4SMinchan Kim 1059dc386d4dSKAMEZAWA Hiroyuki /* 106062e1c553SShaohua Li * Corner case handling: 106162e1c553SShaohua Li * 1. When a new swap-cache page is read into, it is added to the LRU 106262e1c553SShaohua Li * and treated as swapcache but it has no rmap yet. 106362e1c553SShaohua Li * Calling try_to_unmap() against a page->mapping==NULL page will 106462e1c553SShaohua Li * trigger a BUG. So handle it here. 106562e1c553SShaohua Li * 2. An orphaned page (see truncate_complete_page) might have 106662e1c553SShaohua Li * fs-private metadata. The page can be picked up due to memory 106762e1c553SShaohua Li * offlining. Everywhere else except page reclaim, the page is 106862e1c553SShaohua Li * invisible to the vm, so the page can not be migrated. So try to 106962e1c553SShaohua Li * free the metadata, so the page can be freed. 1070dc386d4dSKAMEZAWA Hiroyuki */ 107162e1c553SShaohua Li if (!page->mapping) { 1072309381feSSasha Levin VM_BUG_ON_PAGE(PageAnon(page), page); 10731ce82b69SHugh Dickins if (page_has_private(page)) { 107462e1c553SShaohua Li try_to_free_buffers(page); 10757db7671fSHugh Dickins goto out_unlock_both; 107662e1c553SShaohua Li } 10777db7671fSHugh Dickins } else if (page_mapped(page)) { 10787db7671fSHugh Dickins /* Establish migration ptes */ 107903f15c86SHugh Dickins VM_BUG_ON_PAGE(PageAnon(page) && !PageKsm(page) && !anon_vma, 108003f15c86SHugh Dickins page); 10812ebba6b7SHugh Dickins try_to_unmap(page, 1082da1b13ccSWanpeng Li TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS); 10832ebba6b7SHugh Dickins page_was_mapped = 1; 10842ebba6b7SHugh Dickins } 1085dc386d4dSKAMEZAWA Hiroyuki 1086e24f0b8fSChristoph Lameter if (!page_mapped(page)) 10875c3f9a67SHugh Dickins rc = move_to_new_page(newpage, page, mode); 1088e24f0b8fSChristoph Lameter 10895c3f9a67SHugh Dickins if (page_was_mapped) 10905c3f9a67SHugh Dickins remove_migration_ptes(page, 1091e388466dSKirill A. Shutemov rc == MIGRATEPAGE_SUCCESS ? newpage : page, false); 10923f6c8272SMel Gorman 10937db7671fSHugh Dickins out_unlock_both: 10947db7671fSHugh Dickins unlock_page(newpage); 10957db7671fSHugh Dickins out_unlock: 10963f6c8272SMel Gorman /* Drop an anon_vma reference if we took one */ 109776545066SRik van Riel if (anon_vma) 10989e60109fSPeter Zijlstra put_anon_vma(anon_vma); 1099b20a3503SChristoph Lameter unlock_page(page); 11000dabec93SMinchan Kim out: 1101c6c919ebSMinchan Kim /* 1102c6c919ebSMinchan Kim * If migration is successful, decrease refcount of the newpage 1103c6c919ebSMinchan Kim * which will not free the page because new page owner increased 1104c6c919ebSMinchan Kim * refcounter. As well, if it is LRU page, add the page to LRU 1105c6c919ebSMinchan Kim * list in here. 1106c6c919ebSMinchan Kim */ 1107c6c919ebSMinchan Kim if (rc == MIGRATEPAGE_SUCCESS) { 1108b1123ea6SMinchan Kim if (unlikely(__PageMovable(newpage))) 1109c6c919ebSMinchan Kim put_page(newpage); 1110c6c919ebSMinchan Kim else 1111c6c919ebSMinchan Kim putback_lru_page(newpage); 1112c6c919ebSMinchan Kim } 1113c6c919ebSMinchan Kim 11140dabec93SMinchan Kim return rc; 11150dabec93SMinchan Kim } 111695a402c3SChristoph Lameter 11170dabec93SMinchan Kim /* 1118ef2a5153SGeert Uytterhoeven * gcc 4.7 and 4.8 on arm get an ICEs when inlining unmap_and_move(). Work 1119ef2a5153SGeert Uytterhoeven * around it. 1120ef2a5153SGeert Uytterhoeven */ 1121ef2a5153SGeert Uytterhoeven #if (GCC_VERSION >= 40700 && GCC_VERSION < 40900) && defined(CONFIG_ARM) 1122ef2a5153SGeert Uytterhoeven #define ICE_noinline noinline 1123ef2a5153SGeert Uytterhoeven #else 1124ef2a5153SGeert Uytterhoeven #define ICE_noinline 1125ef2a5153SGeert Uytterhoeven #endif 1126ef2a5153SGeert Uytterhoeven 1127ef2a5153SGeert Uytterhoeven /* 11280dabec93SMinchan Kim * Obtain the lock on page, remove all ptes and migrate the page 11290dabec93SMinchan Kim * to the newly allocated page in newpage. 11300dabec93SMinchan Kim */ 1131ef2a5153SGeert Uytterhoeven static ICE_noinline int unmap_and_move(new_page_t get_new_page, 1132ef2a5153SGeert Uytterhoeven free_page_t put_new_page, 1133ef2a5153SGeert Uytterhoeven unsigned long private, struct page *page, 1134add05cecSNaoya Horiguchi int force, enum migrate_mode mode, 1135add05cecSNaoya Horiguchi enum migrate_reason reason) 11360dabec93SMinchan Kim { 11372def7424SHugh Dickins int rc = MIGRATEPAGE_SUCCESS; 11380dabec93SMinchan Kim int *result = NULL; 11392def7424SHugh Dickins struct page *newpage; 11400dabec93SMinchan Kim 11412def7424SHugh Dickins newpage = get_new_page(page, private, &result); 11420dabec93SMinchan Kim if (!newpage) 11430dabec93SMinchan Kim return -ENOMEM; 11440dabec93SMinchan Kim 11450dabec93SMinchan Kim if (page_count(page) == 1) { 11460dabec93SMinchan Kim /* page was freed from under us. So we are done. */ 1147c6c919ebSMinchan Kim ClearPageActive(page); 1148c6c919ebSMinchan Kim ClearPageUnevictable(page); 1149bda807d4SMinchan Kim if (unlikely(__PageMovable(page))) { 1150bda807d4SMinchan Kim lock_page(page); 1151bda807d4SMinchan Kim if (!PageMovable(page)) 1152bda807d4SMinchan Kim __ClearPageIsolated(page); 1153bda807d4SMinchan Kim unlock_page(page); 1154bda807d4SMinchan Kim } 1155c6c919ebSMinchan Kim if (put_new_page) 1156c6c919ebSMinchan Kim put_new_page(newpage, private); 1157c6c919ebSMinchan Kim else 1158c6c919ebSMinchan Kim put_page(newpage); 11590dabec93SMinchan Kim goto out; 11600dabec93SMinchan Kim } 11610dabec93SMinchan Kim 1162616b8371SZi Yan if (unlikely(PageTransHuge(page) && !PageTransHuge(newpage))) { 11634d2fa965SKirill A. Shutemov lock_page(page); 11644d2fa965SKirill A. Shutemov rc = split_huge_page(page); 11654d2fa965SKirill A. Shutemov unlock_page(page); 11664d2fa965SKirill A. Shutemov if (rc) 11670dabec93SMinchan Kim goto out; 11684d2fa965SKirill A. Shutemov } 11690dabec93SMinchan Kim 11709c620e2bSHugh Dickins rc = __unmap_and_move(page, newpage, force, mode); 1171c6c919ebSMinchan Kim if (rc == MIGRATEPAGE_SUCCESS) 11727cd12b4aSVlastimil Babka set_page_owner_migrate_reason(newpage, reason); 1173bf6bddf1SRafael Aquini 11740dabec93SMinchan Kim out: 1175e24f0b8fSChristoph Lameter if (rc != -EAGAIN) { 1176aaa994b3SChristoph Lameter /* 1177aaa994b3SChristoph Lameter * A page that has been migrated has all references 1178aaa994b3SChristoph Lameter * removed and will be freed. A page that has not been 1179aaa994b3SChristoph Lameter * migrated will have kepts its references and be 1180aaa994b3SChristoph Lameter * restored. 1181aaa994b3SChristoph Lameter */ 1182aaa994b3SChristoph Lameter list_del(&page->lru); 11836afcf8efSMing Ling 11846afcf8efSMing Ling /* 11856afcf8efSMing Ling * Compaction can migrate also non-LRU pages which are 11866afcf8efSMing Ling * not accounted to NR_ISOLATED_*. They can be recognized 11876afcf8efSMing Ling * as __PageMovable 11886afcf8efSMing Ling */ 11896afcf8efSMing Ling if (likely(!__PageMovable(page))) 1190e8db67ebSNaoya Horiguchi mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON + 1191e8db67ebSNaoya Horiguchi page_is_file_cache(page), -hpage_nr_pages(page)); 1192e24f0b8fSChristoph Lameter } 119368711a74SDavid Rientjes 119495a402c3SChristoph Lameter /* 1195c6c919ebSMinchan Kim * If migration is successful, releases reference grabbed during 1196c6c919ebSMinchan Kim * isolation. Otherwise, restore the page to right list unless 1197c6c919ebSMinchan Kim * we want to retry. 119895a402c3SChristoph Lameter */ 1199c6c919ebSMinchan Kim if (rc == MIGRATEPAGE_SUCCESS) { 1200c6c919ebSMinchan Kim put_page(page); 1201c6c919ebSMinchan Kim if (reason == MR_MEMORY_FAILURE) { 1202c6c919ebSMinchan Kim /* 1203c6c919ebSMinchan Kim * Set PG_HWPoison on just freed page 1204c6c919ebSMinchan Kim * intentionally. Although it's rather weird, 1205c6c919ebSMinchan Kim * it's how HWPoison flag works at the moment. 1206c6c919ebSMinchan Kim */ 1207c6c919ebSMinchan Kim if (!test_set_page_hwpoison(page)) 1208c6c919ebSMinchan Kim num_poisoned_pages_inc(); 1209c6c919ebSMinchan Kim } 1210c6c919ebSMinchan Kim } else { 1211bda807d4SMinchan Kim if (rc != -EAGAIN) { 1212bda807d4SMinchan Kim if (likely(!__PageMovable(page))) { 1213c6c919ebSMinchan Kim putback_lru_page(page); 1214bda807d4SMinchan Kim goto put_new; 1215bda807d4SMinchan Kim } 1216bda807d4SMinchan Kim 1217bda807d4SMinchan Kim lock_page(page); 1218bda807d4SMinchan Kim if (PageMovable(page)) 1219bda807d4SMinchan Kim putback_movable_page(page); 1220bda807d4SMinchan Kim else 1221bda807d4SMinchan Kim __ClearPageIsolated(page); 1222bda807d4SMinchan Kim unlock_page(page); 1223bda807d4SMinchan Kim put_page(page); 1224bda807d4SMinchan Kim } 1225bda807d4SMinchan Kim put_new: 1226cf4b769aSHugh Dickins if (put_new_page) 122768711a74SDavid Rientjes put_new_page(newpage, private); 1228c6c919ebSMinchan Kim else 1229d6d86c0aSKonstantin Khlebnikov put_page(newpage); 1230c6c919ebSMinchan Kim } 123168711a74SDavid Rientjes 1232742755a1SChristoph Lameter if (result) { 1233742755a1SChristoph Lameter if (rc) 1234742755a1SChristoph Lameter *result = rc; 1235742755a1SChristoph Lameter else 1236742755a1SChristoph Lameter *result = page_to_nid(newpage); 1237742755a1SChristoph Lameter } 1238e24f0b8fSChristoph Lameter return rc; 1239e24f0b8fSChristoph Lameter } 1240b20a3503SChristoph Lameter 1241e24f0b8fSChristoph Lameter /* 1242290408d4SNaoya Horiguchi * Counterpart of unmap_and_move_page() for hugepage migration. 1243290408d4SNaoya Horiguchi * 1244290408d4SNaoya Horiguchi * This function doesn't wait the completion of hugepage I/O 1245290408d4SNaoya Horiguchi * because there is no race between I/O and migration for hugepage. 1246290408d4SNaoya Horiguchi * Note that currently hugepage I/O occurs only in direct I/O 1247290408d4SNaoya Horiguchi * where no lock is held and PG_writeback is irrelevant, 1248290408d4SNaoya Horiguchi * and writeback status of all subpages are counted in the reference 1249290408d4SNaoya Horiguchi * count of the head page (i.e. if all subpages of a 2MB hugepage are 1250290408d4SNaoya Horiguchi * under direct I/O, the reference of the head page is 512 and a bit more.) 1251290408d4SNaoya Horiguchi * This means that when we try to migrate hugepage whose subpages are 1252290408d4SNaoya Horiguchi * doing direct I/O, some references remain after try_to_unmap() and 1253290408d4SNaoya Horiguchi * hugepage migration fails without data corruption. 1254290408d4SNaoya Horiguchi * 1255290408d4SNaoya Horiguchi * There is also no race when direct I/O is issued on the page under migration, 1256290408d4SNaoya Horiguchi * because then pte is replaced with migration swap entry and direct I/O code 1257290408d4SNaoya Horiguchi * will wait in the page fault for migration to complete. 1258290408d4SNaoya Horiguchi */ 1259290408d4SNaoya Horiguchi static int unmap_and_move_huge_page(new_page_t get_new_page, 126068711a74SDavid Rientjes free_page_t put_new_page, unsigned long private, 126168711a74SDavid Rientjes struct page *hpage, int force, 12627cd12b4aSVlastimil Babka enum migrate_mode mode, int reason) 1263290408d4SNaoya Horiguchi { 12642def7424SHugh Dickins int rc = -EAGAIN; 1265290408d4SNaoya Horiguchi int *result = NULL; 12662ebba6b7SHugh Dickins int page_was_mapped = 0; 126732665f2bSJoonsoo Kim struct page *new_hpage; 1268290408d4SNaoya Horiguchi struct anon_vma *anon_vma = NULL; 1269290408d4SNaoya Horiguchi 127083467efbSNaoya Horiguchi /* 127183467efbSNaoya Horiguchi * Movability of hugepages depends on architectures and hugepage size. 127283467efbSNaoya Horiguchi * This check is necessary because some callers of hugepage migration 127383467efbSNaoya Horiguchi * like soft offline and memory hotremove don't walk through page 127483467efbSNaoya Horiguchi * tables or check whether the hugepage is pmd-based or not before 127583467efbSNaoya Horiguchi * kicking migration. 127683467efbSNaoya Horiguchi */ 1277100873d7SNaoya Horiguchi if (!hugepage_migration_supported(page_hstate(hpage))) { 127832665f2bSJoonsoo Kim putback_active_hugepage(hpage); 127983467efbSNaoya Horiguchi return -ENOSYS; 128032665f2bSJoonsoo Kim } 128183467efbSNaoya Horiguchi 128232665f2bSJoonsoo Kim new_hpage = get_new_page(hpage, private, &result); 1283290408d4SNaoya Horiguchi if (!new_hpage) 1284290408d4SNaoya Horiguchi return -ENOMEM; 1285290408d4SNaoya Horiguchi 1286290408d4SNaoya Horiguchi if (!trylock_page(hpage)) { 12872916ecc0SJérôme Glisse if (!force) 1288290408d4SNaoya Horiguchi goto out; 12892916ecc0SJérôme Glisse switch (mode) { 12902916ecc0SJérôme Glisse case MIGRATE_SYNC: 12912916ecc0SJérôme Glisse case MIGRATE_SYNC_NO_COPY: 12922916ecc0SJérôme Glisse break; 12932916ecc0SJérôme Glisse default: 12942916ecc0SJérôme Glisse goto out; 12952916ecc0SJérôme Glisse } 1296290408d4SNaoya Horiguchi lock_page(hpage); 1297290408d4SNaoya Horiguchi } 1298290408d4SNaoya Horiguchi 1299746b18d4SPeter Zijlstra if (PageAnon(hpage)) 1300746b18d4SPeter Zijlstra anon_vma = page_get_anon_vma(hpage); 1301290408d4SNaoya Horiguchi 13027db7671fSHugh Dickins if (unlikely(!trylock_page(new_hpage))) 13037db7671fSHugh Dickins goto put_anon; 13047db7671fSHugh Dickins 13052ebba6b7SHugh Dickins if (page_mapped(hpage)) { 13062ebba6b7SHugh Dickins try_to_unmap(hpage, 13072ebba6b7SHugh Dickins TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS); 13082ebba6b7SHugh Dickins page_was_mapped = 1; 13092ebba6b7SHugh Dickins } 1310290408d4SNaoya Horiguchi 1311290408d4SNaoya Horiguchi if (!page_mapped(hpage)) 13125c3f9a67SHugh Dickins rc = move_to_new_page(new_hpage, hpage, mode); 1313290408d4SNaoya Horiguchi 13145c3f9a67SHugh Dickins if (page_was_mapped) 13155c3f9a67SHugh Dickins remove_migration_ptes(hpage, 1316e388466dSKirill A. Shutemov rc == MIGRATEPAGE_SUCCESS ? new_hpage : hpage, false); 1317290408d4SNaoya Horiguchi 13187db7671fSHugh Dickins unlock_page(new_hpage); 13197db7671fSHugh Dickins 13207db7671fSHugh Dickins put_anon: 1321fd4a4663SHugh Dickins if (anon_vma) 13229e60109fSPeter Zijlstra put_anon_vma(anon_vma); 13238e6ac7faSAneesh Kumar K.V 13242def7424SHugh Dickins if (rc == MIGRATEPAGE_SUCCESS) { 13258e6ac7faSAneesh Kumar K.V hugetlb_cgroup_migrate(hpage, new_hpage); 13262def7424SHugh Dickins put_new_page = NULL; 13277cd12b4aSVlastimil Babka set_page_owner_migrate_reason(new_hpage, reason); 13282def7424SHugh Dickins } 13298e6ac7faSAneesh Kumar K.V 1330290408d4SNaoya Horiguchi unlock_page(hpage); 133109761333SHillf Danton out: 1332b8ec1ceeSNaoya Horiguchi if (rc != -EAGAIN) 1333b8ec1ceeSNaoya Horiguchi putback_active_hugepage(hpage); 1334c3114a84SAnshuman Khandual if (reason == MR_MEMORY_FAILURE && !test_set_page_hwpoison(hpage)) 1335c3114a84SAnshuman Khandual num_poisoned_pages_inc(); 133668711a74SDavid Rientjes 133768711a74SDavid Rientjes /* 133868711a74SDavid Rientjes * If migration was not successful and there's a freeing callback, use 133968711a74SDavid Rientjes * it. Otherwise, put_page() will drop the reference grabbed during 134068711a74SDavid Rientjes * isolation. 134168711a74SDavid Rientjes */ 13422def7424SHugh Dickins if (put_new_page) 134368711a74SDavid Rientjes put_new_page(new_hpage, private); 134468711a74SDavid Rientjes else 13453aaa76e1SNaoya Horiguchi putback_active_hugepage(new_hpage); 134668711a74SDavid Rientjes 1347290408d4SNaoya Horiguchi if (result) { 1348290408d4SNaoya Horiguchi if (rc) 1349290408d4SNaoya Horiguchi *result = rc; 1350290408d4SNaoya Horiguchi else 1351290408d4SNaoya Horiguchi *result = page_to_nid(new_hpage); 1352290408d4SNaoya Horiguchi } 1353290408d4SNaoya Horiguchi return rc; 1354290408d4SNaoya Horiguchi } 1355290408d4SNaoya Horiguchi 1356290408d4SNaoya Horiguchi /* 1357c73e5c9cSSrivatsa S. Bhat * migrate_pages - migrate the pages specified in a list, to the free pages 1358c73e5c9cSSrivatsa S. Bhat * supplied as the target for the page migration 1359e24f0b8fSChristoph Lameter * 1360c73e5c9cSSrivatsa S. Bhat * @from: The list of pages to be migrated. 1361c73e5c9cSSrivatsa S. Bhat * @get_new_page: The function used to allocate free pages to be used 1362c73e5c9cSSrivatsa S. Bhat * as the target of the page migration. 136368711a74SDavid Rientjes * @put_new_page: The function used to free target pages if migration 136468711a74SDavid Rientjes * fails, or NULL if no special handling is necessary. 1365c73e5c9cSSrivatsa S. Bhat * @private: Private data to be passed on to get_new_page() 1366c73e5c9cSSrivatsa S. Bhat * @mode: The migration mode that specifies the constraints for 1367c73e5c9cSSrivatsa S. Bhat * page migration, if any. 1368c73e5c9cSSrivatsa S. Bhat * @reason: The reason for page migration. 1369e24f0b8fSChristoph Lameter * 1370c73e5c9cSSrivatsa S. Bhat * The function returns after 10 attempts or if no pages are movable any more 1371c73e5c9cSSrivatsa S. Bhat * because the list has become empty or no retryable pages exist any more. 137214e0f9bcSHugh Dickins * The caller should call putback_movable_pages() to return pages to the LRU 137328bd6578SMinchan Kim * or free list only if ret != 0. 1374e24f0b8fSChristoph Lameter * 1375c73e5c9cSSrivatsa S. Bhat * Returns the number of pages that were not migrated, or an error code. 1376e24f0b8fSChristoph Lameter */ 13779c620e2bSHugh Dickins int migrate_pages(struct list_head *from, new_page_t get_new_page, 137868711a74SDavid Rientjes free_page_t put_new_page, unsigned long private, 137968711a74SDavid Rientjes enum migrate_mode mode, int reason) 1380e24f0b8fSChristoph Lameter { 1381e24f0b8fSChristoph Lameter int retry = 1; 1382e24f0b8fSChristoph Lameter int nr_failed = 0; 13835647bc29SMel Gorman int nr_succeeded = 0; 1384e24f0b8fSChristoph Lameter int pass = 0; 1385e24f0b8fSChristoph Lameter struct page *page; 1386e24f0b8fSChristoph Lameter struct page *page2; 1387e24f0b8fSChristoph Lameter int swapwrite = current->flags & PF_SWAPWRITE; 1388e24f0b8fSChristoph Lameter int rc; 13892d1db3b1SChristoph Lameter 1390e24f0b8fSChristoph Lameter if (!swapwrite) 1391e24f0b8fSChristoph Lameter current->flags |= PF_SWAPWRITE; 1392e24f0b8fSChristoph Lameter 1393e24f0b8fSChristoph Lameter for(pass = 0; pass < 10 && retry; pass++) { 1394e24f0b8fSChristoph Lameter retry = 0; 1395e24f0b8fSChristoph Lameter 1396e24f0b8fSChristoph Lameter list_for_each_entry_safe(page, page2, from, lru) { 1397e24f0b8fSChristoph Lameter cond_resched(); 1398e24f0b8fSChristoph Lameter 139931caf665SNaoya Horiguchi if (PageHuge(page)) 140031caf665SNaoya Horiguchi rc = unmap_and_move_huge_page(get_new_page, 140168711a74SDavid Rientjes put_new_page, private, page, 14027cd12b4aSVlastimil Babka pass > 2, mode, reason); 140331caf665SNaoya Horiguchi else 140468711a74SDavid Rientjes rc = unmap_and_move(get_new_page, put_new_page, 1405add05cecSNaoya Horiguchi private, page, pass > 2, mode, 1406add05cecSNaoya Horiguchi reason); 1407e24f0b8fSChristoph Lameter 1408e24f0b8fSChristoph Lameter switch(rc) { 140995a402c3SChristoph Lameter case -ENOMEM: 1410dfef2ef4SDavid Rientjes nr_failed++; 141195a402c3SChristoph Lameter goto out; 1412e24f0b8fSChristoph Lameter case -EAGAIN: 1413b20a3503SChristoph Lameter retry++; 1414e24f0b8fSChristoph Lameter break; 141578bd5209SRafael Aquini case MIGRATEPAGE_SUCCESS: 14165647bc29SMel Gorman nr_succeeded++; 1417e24f0b8fSChristoph Lameter break; 1418e24f0b8fSChristoph Lameter default: 1419354a3363SNaoya Horiguchi /* 1420354a3363SNaoya Horiguchi * Permanent failure (-EBUSY, -ENOSYS, etc.): 1421354a3363SNaoya Horiguchi * unlike -EAGAIN case, the failed page is 1422354a3363SNaoya Horiguchi * removed from migration page list and not 1423354a3363SNaoya Horiguchi * retried in the next outer loop. 1424354a3363SNaoya Horiguchi */ 1425b20a3503SChristoph Lameter nr_failed++; 1426e24f0b8fSChristoph Lameter break; 1427b20a3503SChristoph Lameter } 1428b20a3503SChristoph Lameter } 1429e24f0b8fSChristoph Lameter } 1430f2f81fb2SVlastimil Babka nr_failed += retry; 1431f2f81fb2SVlastimil Babka rc = nr_failed; 143295a402c3SChristoph Lameter out: 14335647bc29SMel Gorman if (nr_succeeded) 14345647bc29SMel Gorman count_vm_events(PGMIGRATE_SUCCESS, nr_succeeded); 14355647bc29SMel Gorman if (nr_failed) 14365647bc29SMel Gorman count_vm_events(PGMIGRATE_FAIL, nr_failed); 14377b2a2d4aSMel Gorman trace_mm_migrate_pages(nr_succeeded, nr_failed, mode, reason); 14387b2a2d4aSMel Gorman 1439b20a3503SChristoph Lameter if (!swapwrite) 1440b20a3503SChristoph Lameter current->flags &= ~PF_SWAPWRITE; 1441b20a3503SChristoph Lameter 144295a402c3SChristoph Lameter return rc; 1443b20a3503SChristoph Lameter } 1444b20a3503SChristoph Lameter 1445742755a1SChristoph Lameter #ifdef CONFIG_NUMA 1446742755a1SChristoph Lameter /* 1447742755a1SChristoph Lameter * Move a list of individual pages 1448742755a1SChristoph Lameter */ 1449742755a1SChristoph Lameter struct page_to_node { 1450742755a1SChristoph Lameter unsigned long addr; 1451742755a1SChristoph Lameter struct page *page; 1452742755a1SChristoph Lameter int node; 1453742755a1SChristoph Lameter int status; 1454742755a1SChristoph Lameter }; 1455742755a1SChristoph Lameter 1456742755a1SChristoph Lameter static struct page *new_page_node(struct page *p, unsigned long private, 1457742755a1SChristoph Lameter int **result) 1458742755a1SChristoph Lameter { 1459742755a1SChristoph Lameter struct page_to_node *pm = (struct page_to_node *)private; 1460742755a1SChristoph Lameter 1461742755a1SChristoph Lameter while (pm->node != MAX_NUMNODES && pm->page != p) 1462742755a1SChristoph Lameter pm++; 1463742755a1SChristoph Lameter 1464742755a1SChristoph Lameter if (pm->node == MAX_NUMNODES) 1465742755a1SChristoph Lameter return NULL; 1466742755a1SChristoph Lameter 1467742755a1SChristoph Lameter *result = &pm->status; 1468742755a1SChristoph Lameter 1469e632a938SNaoya Horiguchi if (PageHuge(p)) 1470e632a938SNaoya Horiguchi return alloc_huge_page_node(page_hstate(compound_head(p)), 1471e632a938SNaoya Horiguchi pm->node); 1472e8db67ebSNaoya Horiguchi else if (thp_migration_supported() && PageTransHuge(p)) { 1473e8db67ebSNaoya Horiguchi struct page *thp; 1474e8db67ebSNaoya Horiguchi 1475e8db67ebSNaoya Horiguchi thp = alloc_pages_node(pm->node, 1476e8db67ebSNaoya Horiguchi (GFP_TRANSHUGE | __GFP_THISNODE) & ~__GFP_RECLAIM, 1477e8db67ebSNaoya Horiguchi HPAGE_PMD_ORDER); 1478e8db67ebSNaoya Horiguchi if (!thp) 1479e8db67ebSNaoya Horiguchi return NULL; 1480e8db67ebSNaoya Horiguchi prep_transhuge_page(thp); 1481e8db67ebSNaoya Horiguchi return thp; 1482e8db67ebSNaoya Horiguchi } else 148396db800fSVlastimil Babka return __alloc_pages_node(pm->node, 1484e97ca8e5SJohannes Weiner GFP_HIGHUSER_MOVABLE | __GFP_THISNODE, 0); 1485742755a1SChristoph Lameter } 1486742755a1SChristoph Lameter 1487742755a1SChristoph Lameter /* 1488742755a1SChristoph Lameter * Move a set of pages as indicated in the pm array. The addr 1489742755a1SChristoph Lameter * field must be set to the virtual address of the page to be moved 1490742755a1SChristoph Lameter * and the node number must contain a valid target node. 14915e9a0f02SBrice Goglin * The pm array ends with node = MAX_NUMNODES. 1492742755a1SChristoph Lameter */ 14935e9a0f02SBrice Goglin static int do_move_page_to_node_array(struct mm_struct *mm, 14945e9a0f02SBrice Goglin struct page_to_node *pm, 1495742755a1SChristoph Lameter int migrate_all) 1496742755a1SChristoph Lameter { 1497742755a1SChristoph Lameter int err; 1498742755a1SChristoph Lameter struct page_to_node *pp; 1499742755a1SChristoph Lameter LIST_HEAD(pagelist); 1500742755a1SChristoph Lameter 1501742755a1SChristoph Lameter down_read(&mm->mmap_sem); 1502742755a1SChristoph Lameter 1503742755a1SChristoph Lameter /* 1504742755a1SChristoph Lameter * Build a list of pages to migrate 1505742755a1SChristoph Lameter */ 1506742755a1SChristoph Lameter for (pp = pm; pp->node != MAX_NUMNODES; pp++) { 1507742755a1SChristoph Lameter struct vm_area_struct *vma; 1508742755a1SChristoph Lameter struct page *page; 1509e8db67ebSNaoya Horiguchi struct page *head; 1510e8db67ebSNaoya Horiguchi unsigned int follflags; 1511742755a1SChristoph Lameter 1512742755a1SChristoph Lameter err = -EFAULT; 1513742755a1SChristoph Lameter vma = find_vma(mm, pp->addr); 151470384dc6SGleb Natapov if (!vma || pp->addr < vma->vm_start || !vma_migratable(vma)) 1515742755a1SChristoph Lameter goto set_status; 1516742755a1SChristoph Lameter 1517d899844eSKirill A. Shutemov /* FOLL_DUMP to ignore special (like zero) pages */ 1518e8db67ebSNaoya Horiguchi follflags = FOLL_GET | FOLL_DUMP; 1519e8db67ebSNaoya Horiguchi if (!thp_migration_supported()) 1520e8db67ebSNaoya Horiguchi follflags |= FOLL_SPLIT; 1521e8db67ebSNaoya Horiguchi page = follow_page(vma, pp->addr, follflags); 152289f5b7daSLinus Torvalds 152389f5b7daSLinus Torvalds err = PTR_ERR(page); 152489f5b7daSLinus Torvalds if (IS_ERR(page)) 152589f5b7daSLinus Torvalds goto set_status; 152689f5b7daSLinus Torvalds 1527742755a1SChristoph Lameter err = -ENOENT; 1528742755a1SChristoph Lameter if (!page) 1529742755a1SChristoph Lameter goto set_status; 1530742755a1SChristoph Lameter 1531742755a1SChristoph Lameter err = page_to_nid(page); 1532742755a1SChristoph Lameter 1533742755a1SChristoph Lameter if (err == pp->node) 1534742755a1SChristoph Lameter /* 1535742755a1SChristoph Lameter * Node already in the right place 1536742755a1SChristoph Lameter */ 1537742755a1SChristoph Lameter goto put_and_set; 1538742755a1SChristoph Lameter 1539742755a1SChristoph Lameter err = -EACCES; 1540742755a1SChristoph Lameter if (page_mapcount(page) > 1 && 1541742755a1SChristoph Lameter !migrate_all) 1542742755a1SChristoph Lameter goto put_and_set; 1543742755a1SChristoph Lameter 1544e632a938SNaoya Horiguchi if (PageHuge(page)) { 1545e8db67ebSNaoya Horiguchi if (PageHead(page)) { 1546e632a938SNaoya Horiguchi isolate_huge_page(page, &pagelist); 1547e8db67ebSNaoya Horiguchi err = 0; 1548e8db67ebSNaoya Horiguchi pp->page = page; 1549e8db67ebSNaoya Horiguchi } 1550e632a938SNaoya Horiguchi goto put_and_set; 1551e632a938SNaoya Horiguchi } 1552e632a938SNaoya Horiguchi 1553e8db67ebSNaoya Horiguchi pp->page = compound_head(page); 1554e8db67ebSNaoya Horiguchi head = compound_head(page); 1555e8db67ebSNaoya Horiguchi err = isolate_lru_page(head); 15566d9c285aSKOSAKI Motohiro if (!err) { 1557e8db67ebSNaoya Horiguchi list_add_tail(&head->lru, &pagelist); 1558e8db67ebSNaoya Horiguchi mod_node_page_state(page_pgdat(head), 1559e8db67ebSNaoya Horiguchi NR_ISOLATED_ANON + page_is_file_cache(head), 1560e8db67ebSNaoya Horiguchi hpage_nr_pages(head)); 15616d9c285aSKOSAKI Motohiro } 1562742755a1SChristoph Lameter put_and_set: 1563742755a1SChristoph Lameter /* 1564742755a1SChristoph Lameter * Either remove the duplicate refcount from 1565742755a1SChristoph Lameter * isolate_lru_page() or drop the page ref if it was 1566742755a1SChristoph Lameter * not isolated. 1567742755a1SChristoph Lameter */ 1568742755a1SChristoph Lameter put_page(page); 1569742755a1SChristoph Lameter set_status: 1570742755a1SChristoph Lameter pp->status = err; 1571742755a1SChristoph Lameter } 1572742755a1SChristoph Lameter 1573e78bbfa8SBrice Goglin err = 0; 1574cf608ac1SMinchan Kim if (!list_empty(&pagelist)) { 157568711a74SDavid Rientjes err = migrate_pages(&pagelist, new_page_node, NULL, 15769c620e2bSHugh Dickins (unsigned long)pm, MIGRATE_SYNC, MR_SYSCALL); 1577cf608ac1SMinchan Kim if (err) 1578e632a938SNaoya Horiguchi putback_movable_pages(&pagelist); 1579cf608ac1SMinchan Kim } 1580742755a1SChristoph Lameter 1581742755a1SChristoph Lameter up_read(&mm->mmap_sem); 1582742755a1SChristoph Lameter return err; 1583742755a1SChristoph Lameter } 1584742755a1SChristoph Lameter 1585742755a1SChristoph Lameter /* 15865e9a0f02SBrice Goglin * Migrate an array of page address onto an array of nodes and fill 15875e9a0f02SBrice Goglin * the corresponding array of status. 15885e9a0f02SBrice Goglin */ 15893268c63eSChristoph Lameter static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes, 15905e9a0f02SBrice Goglin unsigned long nr_pages, 15915e9a0f02SBrice Goglin const void __user * __user *pages, 15925e9a0f02SBrice Goglin const int __user *nodes, 15935e9a0f02SBrice Goglin int __user *status, int flags) 15945e9a0f02SBrice Goglin { 15953140a227SBrice Goglin struct page_to_node *pm; 15963140a227SBrice Goglin unsigned long chunk_nr_pages; 15973140a227SBrice Goglin unsigned long chunk_start; 15983140a227SBrice Goglin int err; 15995e9a0f02SBrice Goglin 16005e9a0f02SBrice Goglin err = -ENOMEM; 16013140a227SBrice Goglin pm = (struct page_to_node *)__get_free_page(GFP_KERNEL); 16023140a227SBrice Goglin if (!pm) 16035e9a0f02SBrice Goglin goto out; 160435282a2dSBrice Goglin 160535282a2dSBrice Goglin migrate_prep(); 160635282a2dSBrice Goglin 16075e9a0f02SBrice Goglin /* 16083140a227SBrice Goglin * Store a chunk of page_to_node array in a page, 16093140a227SBrice Goglin * but keep the last one as a marker 16105e9a0f02SBrice Goglin */ 16113140a227SBrice Goglin chunk_nr_pages = (PAGE_SIZE / sizeof(struct page_to_node)) - 1; 16123140a227SBrice Goglin 16133140a227SBrice Goglin for (chunk_start = 0; 16143140a227SBrice Goglin chunk_start < nr_pages; 16153140a227SBrice Goglin chunk_start += chunk_nr_pages) { 16163140a227SBrice Goglin int j; 16173140a227SBrice Goglin 16183140a227SBrice Goglin if (chunk_start + chunk_nr_pages > nr_pages) 16193140a227SBrice Goglin chunk_nr_pages = nr_pages - chunk_start; 16203140a227SBrice Goglin 16213140a227SBrice Goglin /* fill the chunk pm with addrs and nodes from user-space */ 16223140a227SBrice Goglin for (j = 0; j < chunk_nr_pages; j++) { 16235e9a0f02SBrice Goglin const void __user *p; 16245e9a0f02SBrice Goglin int node; 16255e9a0f02SBrice Goglin 16263140a227SBrice Goglin err = -EFAULT; 16273140a227SBrice Goglin if (get_user(p, pages + j + chunk_start)) 16283140a227SBrice Goglin goto out_pm; 16293140a227SBrice Goglin pm[j].addr = (unsigned long) p; 16303140a227SBrice Goglin 16313140a227SBrice Goglin if (get_user(node, nodes + j + chunk_start)) 16325e9a0f02SBrice Goglin goto out_pm; 16335e9a0f02SBrice Goglin 16345e9a0f02SBrice Goglin err = -ENODEV; 16356f5a55f1SLinus Torvalds if (node < 0 || node >= MAX_NUMNODES) 16366f5a55f1SLinus Torvalds goto out_pm; 16376f5a55f1SLinus Torvalds 1638389162c2SLai Jiangshan if (!node_state(node, N_MEMORY)) 16395e9a0f02SBrice Goglin goto out_pm; 16405e9a0f02SBrice Goglin 16415e9a0f02SBrice Goglin err = -EACCES; 16425e9a0f02SBrice Goglin if (!node_isset(node, task_nodes)) 16435e9a0f02SBrice Goglin goto out_pm; 16445e9a0f02SBrice Goglin 16453140a227SBrice Goglin pm[j].node = node; 16465e9a0f02SBrice Goglin } 16475e9a0f02SBrice Goglin 16483140a227SBrice Goglin /* End marker for this chunk */ 16493140a227SBrice Goglin pm[chunk_nr_pages].node = MAX_NUMNODES; 16503140a227SBrice Goglin 16513140a227SBrice Goglin /* Migrate this chunk */ 16523140a227SBrice Goglin err = do_move_page_to_node_array(mm, pm, 16533140a227SBrice Goglin flags & MPOL_MF_MOVE_ALL); 16543140a227SBrice Goglin if (err < 0) 16553140a227SBrice Goglin goto out_pm; 16563140a227SBrice Goglin 16575e9a0f02SBrice Goglin /* Return status information */ 16583140a227SBrice Goglin for (j = 0; j < chunk_nr_pages; j++) 16593140a227SBrice Goglin if (put_user(pm[j].status, status + j + chunk_start)) { 16605e9a0f02SBrice Goglin err = -EFAULT; 16613140a227SBrice Goglin goto out_pm; 16623140a227SBrice Goglin } 16633140a227SBrice Goglin } 16643140a227SBrice Goglin err = 0; 16655e9a0f02SBrice Goglin 16665e9a0f02SBrice Goglin out_pm: 16673140a227SBrice Goglin free_page((unsigned long)pm); 16685e9a0f02SBrice Goglin out: 16695e9a0f02SBrice Goglin return err; 16705e9a0f02SBrice Goglin } 16715e9a0f02SBrice Goglin 16725e9a0f02SBrice Goglin /* 16732f007e74SBrice Goglin * Determine the nodes of an array of pages and store it in an array of status. 1674742755a1SChristoph Lameter */ 167580bba129SBrice Goglin static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages, 167680bba129SBrice Goglin const void __user **pages, int *status) 1677742755a1SChristoph Lameter { 16782f007e74SBrice Goglin unsigned long i; 1679742755a1SChristoph Lameter 16802f007e74SBrice Goglin down_read(&mm->mmap_sem); 16812f007e74SBrice Goglin 16822f007e74SBrice Goglin for (i = 0; i < nr_pages; i++) { 168380bba129SBrice Goglin unsigned long addr = (unsigned long)(*pages); 16842f007e74SBrice Goglin struct vm_area_struct *vma; 16852f007e74SBrice Goglin struct page *page; 1686c095adbcSKOSAKI Motohiro int err = -EFAULT; 16872f007e74SBrice Goglin 16882f007e74SBrice Goglin vma = find_vma(mm, addr); 168970384dc6SGleb Natapov if (!vma || addr < vma->vm_start) 1690742755a1SChristoph Lameter goto set_status; 1691742755a1SChristoph Lameter 1692d899844eSKirill A. Shutemov /* FOLL_DUMP to ignore special (like zero) pages */ 1693d899844eSKirill A. Shutemov page = follow_page(vma, addr, FOLL_DUMP); 169489f5b7daSLinus Torvalds 169589f5b7daSLinus Torvalds err = PTR_ERR(page); 169689f5b7daSLinus Torvalds if (IS_ERR(page)) 169789f5b7daSLinus Torvalds goto set_status; 169889f5b7daSLinus Torvalds 1699d899844eSKirill A. Shutemov err = page ? page_to_nid(page) : -ENOENT; 1700742755a1SChristoph Lameter set_status: 170180bba129SBrice Goglin *status = err; 170280bba129SBrice Goglin 170380bba129SBrice Goglin pages++; 170480bba129SBrice Goglin status++; 170580bba129SBrice Goglin } 170680bba129SBrice Goglin 170780bba129SBrice Goglin up_read(&mm->mmap_sem); 170880bba129SBrice Goglin } 170980bba129SBrice Goglin 171080bba129SBrice Goglin /* 171180bba129SBrice Goglin * Determine the nodes of a user array of pages and store it in 171280bba129SBrice Goglin * a user array of status. 171380bba129SBrice Goglin */ 171480bba129SBrice Goglin static int do_pages_stat(struct mm_struct *mm, unsigned long nr_pages, 171580bba129SBrice Goglin const void __user * __user *pages, 171680bba129SBrice Goglin int __user *status) 171780bba129SBrice Goglin { 171880bba129SBrice Goglin #define DO_PAGES_STAT_CHUNK_NR 16 171980bba129SBrice Goglin const void __user *chunk_pages[DO_PAGES_STAT_CHUNK_NR]; 172080bba129SBrice Goglin int chunk_status[DO_PAGES_STAT_CHUNK_NR]; 172180bba129SBrice Goglin 172287b8d1adSH. Peter Anvin while (nr_pages) { 172387b8d1adSH. Peter Anvin unsigned long chunk_nr; 172480bba129SBrice Goglin 172587b8d1adSH. Peter Anvin chunk_nr = nr_pages; 172687b8d1adSH. Peter Anvin if (chunk_nr > DO_PAGES_STAT_CHUNK_NR) 172787b8d1adSH. Peter Anvin chunk_nr = DO_PAGES_STAT_CHUNK_NR; 172887b8d1adSH. Peter Anvin 172987b8d1adSH. Peter Anvin if (copy_from_user(chunk_pages, pages, chunk_nr * sizeof(*chunk_pages))) 173087b8d1adSH. Peter Anvin break; 173180bba129SBrice Goglin 173280bba129SBrice Goglin do_pages_stat_array(mm, chunk_nr, chunk_pages, chunk_status); 173380bba129SBrice Goglin 173487b8d1adSH. Peter Anvin if (copy_to_user(status, chunk_status, chunk_nr * sizeof(*status))) 173587b8d1adSH. Peter Anvin break; 1736742755a1SChristoph Lameter 173787b8d1adSH. Peter Anvin pages += chunk_nr; 173887b8d1adSH. Peter Anvin status += chunk_nr; 173987b8d1adSH. Peter Anvin nr_pages -= chunk_nr; 174087b8d1adSH. Peter Anvin } 174187b8d1adSH. Peter Anvin return nr_pages ? -EFAULT : 0; 1742742755a1SChristoph Lameter } 1743742755a1SChristoph Lameter 1744742755a1SChristoph Lameter /* 1745742755a1SChristoph Lameter * Move a list of pages in the address space of the currently executing 1746742755a1SChristoph Lameter * process. 1747742755a1SChristoph Lameter */ 1748938bb9f5SHeiko Carstens SYSCALL_DEFINE6(move_pages, pid_t, pid, unsigned long, nr_pages, 1749938bb9f5SHeiko Carstens const void __user * __user *, pages, 1750938bb9f5SHeiko Carstens const int __user *, nodes, 1751938bb9f5SHeiko Carstens int __user *, status, int, flags) 1752742755a1SChristoph Lameter { 1753742755a1SChristoph Lameter struct task_struct *task; 1754742755a1SChristoph Lameter struct mm_struct *mm; 17555e9a0f02SBrice Goglin int err; 17563268c63eSChristoph Lameter nodemask_t task_nodes; 1757742755a1SChristoph Lameter 1758742755a1SChristoph Lameter /* Check flags */ 1759742755a1SChristoph Lameter if (flags & ~(MPOL_MF_MOVE|MPOL_MF_MOVE_ALL)) 1760742755a1SChristoph Lameter return -EINVAL; 1761742755a1SChristoph Lameter 1762742755a1SChristoph Lameter if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE)) 1763742755a1SChristoph Lameter return -EPERM; 1764742755a1SChristoph Lameter 1765742755a1SChristoph Lameter /* Find the mm_struct */ 1766a879bf58SGreg Thelen rcu_read_lock(); 1767228ebcbeSPavel Emelyanov task = pid ? find_task_by_vpid(pid) : current; 1768742755a1SChristoph Lameter if (!task) { 1769a879bf58SGreg Thelen rcu_read_unlock(); 1770742755a1SChristoph Lameter return -ESRCH; 1771742755a1SChristoph Lameter } 17723268c63eSChristoph Lameter get_task_struct(task); 1773742755a1SChristoph Lameter 1774742755a1SChristoph Lameter /* 1775742755a1SChristoph Lameter * Check if this process has the right to modify the specified 1776197e7e52SLinus Torvalds * process. Use the regular "ptrace_may_access()" checks. 1777742755a1SChristoph Lameter */ 1778197e7e52SLinus Torvalds if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) { 1779c69e8d9cSDavid Howells rcu_read_unlock(); 1780742755a1SChristoph Lameter err = -EPERM; 17815e9a0f02SBrice Goglin goto out; 1782742755a1SChristoph Lameter } 1783c69e8d9cSDavid Howells rcu_read_unlock(); 1784742755a1SChristoph Lameter 178586c3a764SDavid Quigley err = security_task_movememory(task); 178686c3a764SDavid Quigley if (err) 1787742755a1SChristoph Lameter goto out; 1788742755a1SChristoph Lameter 17893268c63eSChristoph Lameter task_nodes = cpuset_mems_allowed(task); 17903268c63eSChristoph Lameter mm = get_task_mm(task); 17913268c63eSChristoph Lameter put_task_struct(task); 17923268c63eSChristoph Lameter 17936e8b09eaSSasha Levin if (!mm) 17946e8b09eaSSasha Levin return -EINVAL; 17956e8b09eaSSasha Levin 17963268c63eSChristoph Lameter if (nodes) 17973268c63eSChristoph Lameter err = do_pages_move(mm, task_nodes, nr_pages, pages, 17983268c63eSChristoph Lameter nodes, status, flags); 17993268c63eSChristoph Lameter else 18005e9a0f02SBrice Goglin err = do_pages_stat(mm, nr_pages, pages, status); 18013268c63eSChristoph Lameter 18023268c63eSChristoph Lameter mmput(mm); 18033268c63eSChristoph Lameter return err; 1804742755a1SChristoph Lameter 1805742755a1SChristoph Lameter out: 18063268c63eSChristoph Lameter put_task_struct(task); 1807742755a1SChristoph Lameter return err; 1808742755a1SChristoph Lameter } 1809742755a1SChristoph Lameter 18107039e1dbSPeter Zijlstra #ifdef CONFIG_NUMA_BALANCING 18117039e1dbSPeter Zijlstra /* 18127039e1dbSPeter Zijlstra * Returns true if this is a safe migration target node for misplaced NUMA 18137039e1dbSPeter Zijlstra * pages. Currently it only checks the watermarks which crude 18147039e1dbSPeter Zijlstra */ 18157039e1dbSPeter Zijlstra static bool migrate_balanced_pgdat(struct pglist_data *pgdat, 18163abef4e6SMel Gorman unsigned long nr_migrate_pages) 18177039e1dbSPeter Zijlstra { 18187039e1dbSPeter Zijlstra int z; 1819599d0c95SMel Gorman 18207039e1dbSPeter Zijlstra for (z = pgdat->nr_zones - 1; z >= 0; z--) { 18217039e1dbSPeter Zijlstra struct zone *zone = pgdat->node_zones + z; 18227039e1dbSPeter Zijlstra 18237039e1dbSPeter Zijlstra if (!populated_zone(zone)) 18247039e1dbSPeter Zijlstra continue; 18257039e1dbSPeter Zijlstra 18267039e1dbSPeter Zijlstra /* Avoid waking kswapd by allocating pages_to_migrate pages. */ 18277039e1dbSPeter Zijlstra if (!zone_watermark_ok(zone, 0, 18287039e1dbSPeter Zijlstra high_wmark_pages(zone) + 18297039e1dbSPeter Zijlstra nr_migrate_pages, 18307039e1dbSPeter Zijlstra 0, 0)) 18317039e1dbSPeter Zijlstra continue; 18327039e1dbSPeter Zijlstra return true; 18337039e1dbSPeter Zijlstra } 18347039e1dbSPeter Zijlstra return false; 18357039e1dbSPeter Zijlstra } 18367039e1dbSPeter Zijlstra 18377039e1dbSPeter Zijlstra static struct page *alloc_misplaced_dst_page(struct page *page, 18387039e1dbSPeter Zijlstra unsigned long data, 18397039e1dbSPeter Zijlstra int **result) 18407039e1dbSPeter Zijlstra { 18417039e1dbSPeter Zijlstra int nid = (int) data; 18427039e1dbSPeter Zijlstra struct page *newpage; 18437039e1dbSPeter Zijlstra 184496db800fSVlastimil Babka newpage = __alloc_pages_node(nid, 1845e97ca8e5SJohannes Weiner (GFP_HIGHUSER_MOVABLE | 1846e97ca8e5SJohannes Weiner __GFP_THISNODE | __GFP_NOMEMALLOC | 1847e97ca8e5SJohannes Weiner __GFP_NORETRY | __GFP_NOWARN) & 18488479eba7SMel Gorman ~__GFP_RECLAIM, 0); 1849bac0382cSHillf Danton 18507039e1dbSPeter Zijlstra return newpage; 18517039e1dbSPeter Zijlstra } 18527039e1dbSPeter Zijlstra 18537039e1dbSPeter Zijlstra /* 1854a8f60772SMel Gorman * page migration rate limiting control. 1855a8f60772SMel Gorman * Do not migrate more than @pages_to_migrate in a @migrate_interval_millisecs 1856a8f60772SMel Gorman * window of time. Default here says do not migrate more than 1280M per second. 1857a8f60772SMel Gorman */ 1858a8f60772SMel Gorman static unsigned int migrate_interval_millisecs __read_mostly = 100; 1859a8f60772SMel Gorman static unsigned int ratelimit_pages __read_mostly = 128 << (20 - PAGE_SHIFT); 1860a8f60772SMel Gorman 1861b32967ffSMel Gorman /* Returns true if the node is migrate rate-limited after the update */ 18621c30e017SMel Gorman static bool numamigrate_update_ratelimit(pg_data_t *pgdat, 18631c30e017SMel Gorman unsigned long nr_pages) 1864b32967ffSMel Gorman { 1865b32967ffSMel Gorman /* 1866b32967ffSMel Gorman * Rate-limit the amount of data that is being migrated to a node. 1867b32967ffSMel Gorman * Optimal placement is no good if the memory bus is saturated and 1868b32967ffSMel Gorman * all the time is being spent migrating! 1869b32967ffSMel Gorman */ 1870b32967ffSMel Gorman if (time_after(jiffies, pgdat->numabalancing_migrate_next_window)) { 18711c5e9c27SMel Gorman spin_lock(&pgdat->numabalancing_migrate_lock); 1872b32967ffSMel Gorman pgdat->numabalancing_migrate_nr_pages = 0; 1873b32967ffSMel Gorman pgdat->numabalancing_migrate_next_window = jiffies + 1874b32967ffSMel Gorman msecs_to_jiffies(migrate_interval_millisecs); 18751c5e9c27SMel Gorman spin_unlock(&pgdat->numabalancing_migrate_lock); 1876b32967ffSMel Gorman } 1877af1839d7SMel Gorman if (pgdat->numabalancing_migrate_nr_pages > ratelimit_pages) { 1878af1839d7SMel Gorman trace_mm_numa_migrate_ratelimit(current, pgdat->node_id, 1879af1839d7SMel Gorman nr_pages); 18801c5e9c27SMel Gorman return true; 1881af1839d7SMel Gorman } 1882b32967ffSMel Gorman 18831c5e9c27SMel Gorman /* 18841c5e9c27SMel Gorman * This is an unlocked non-atomic update so errors are possible. 18851c5e9c27SMel Gorman * The consequences are failing to migrate when we potentiall should 18861c5e9c27SMel Gorman * have which is not severe enough to warrant locking. If it is ever 18871c5e9c27SMel Gorman * a problem, it can be converted to a per-cpu counter. 18881c5e9c27SMel Gorman */ 18891c5e9c27SMel Gorman pgdat->numabalancing_migrate_nr_pages += nr_pages; 18901c5e9c27SMel Gorman return false; 1891b32967ffSMel Gorman } 1892b32967ffSMel Gorman 18931c30e017SMel Gorman static int numamigrate_isolate_page(pg_data_t *pgdat, struct page *page) 1894b32967ffSMel Gorman { 1895340ef390SHugh Dickins int page_lru; 1896b32967ffSMel Gorman 1897309381feSSasha Levin VM_BUG_ON_PAGE(compound_order(page) && !PageTransHuge(page), page); 18983abef4e6SMel Gorman 1899b32967ffSMel Gorman /* Avoid migrating to a node that is nearly full */ 1900340ef390SHugh Dickins if (!migrate_balanced_pgdat(pgdat, 1UL << compound_order(page))) 1901340ef390SHugh Dickins return 0; 1902b32967ffSMel Gorman 1903340ef390SHugh Dickins if (isolate_lru_page(page)) 1904340ef390SHugh Dickins return 0; 1905340ef390SHugh Dickins 1906340ef390SHugh Dickins /* 1907340ef390SHugh Dickins * migrate_misplaced_transhuge_page() skips page migration's usual 1908340ef390SHugh Dickins * check on page_count(), so we must do it here, now that the page 1909340ef390SHugh Dickins * has been isolated: a GUP pin, or any other pin, prevents migration. 1910340ef390SHugh Dickins * The expected page count is 3: 1 for page's mapcount and 1 for the 1911340ef390SHugh Dickins * caller's pin and 1 for the reference taken by isolate_lru_page(). 1912340ef390SHugh Dickins */ 1913340ef390SHugh Dickins if (PageTransHuge(page) && page_count(page) != 3) { 1914340ef390SHugh Dickins putback_lru_page(page); 1915b32967ffSMel Gorman return 0; 1916b32967ffSMel Gorman } 1917b32967ffSMel Gorman 1918b32967ffSMel Gorman page_lru = page_is_file_cache(page); 1919599d0c95SMel Gorman mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON + page_lru, 1920340ef390SHugh Dickins hpage_nr_pages(page)); 1921b32967ffSMel Gorman 1922b32967ffSMel Gorman /* 1923340ef390SHugh Dickins * Isolating the page has taken another reference, so the 1924340ef390SHugh Dickins * caller's reference can be safely dropped without the page 1925340ef390SHugh Dickins * disappearing underneath us during migration. 1926b32967ffSMel Gorman */ 1927b32967ffSMel Gorman put_page(page); 1928340ef390SHugh Dickins return 1; 1929b32967ffSMel Gorman } 1930b32967ffSMel Gorman 1931de466bd6SMel Gorman bool pmd_trans_migrating(pmd_t pmd) 1932de466bd6SMel Gorman { 1933de466bd6SMel Gorman struct page *page = pmd_page(pmd); 1934de466bd6SMel Gorman return PageLocked(page); 1935de466bd6SMel Gorman } 1936de466bd6SMel Gorman 1937a8f60772SMel Gorman /* 19387039e1dbSPeter Zijlstra * Attempt to migrate a misplaced page to the specified destination 19397039e1dbSPeter Zijlstra * node. Caller is expected to have an elevated reference count on 19407039e1dbSPeter Zijlstra * the page that will be dropped by this function before returning. 19417039e1dbSPeter Zijlstra */ 19421bc115d8SMel Gorman int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma, 19431bc115d8SMel Gorman int node) 19447039e1dbSPeter Zijlstra { 1945a8f60772SMel Gorman pg_data_t *pgdat = NODE_DATA(node); 1946340ef390SHugh Dickins int isolated; 1947b32967ffSMel Gorman int nr_remaining; 19487039e1dbSPeter Zijlstra LIST_HEAD(migratepages); 19497039e1dbSPeter Zijlstra 19507039e1dbSPeter Zijlstra /* 19511bc115d8SMel Gorman * Don't migrate file pages that are mapped in multiple processes 19521bc115d8SMel Gorman * with execute permissions as they are probably shared libraries. 19537039e1dbSPeter Zijlstra */ 19541bc115d8SMel Gorman if (page_mapcount(page) != 1 && page_is_file_cache(page) && 19551bc115d8SMel Gorman (vma->vm_flags & VM_EXEC)) 19567039e1dbSPeter Zijlstra goto out; 19577039e1dbSPeter Zijlstra 1958a8f60772SMel Gorman /* 1959a8f60772SMel Gorman * Rate-limit the amount of data that is being migrated to a node. 1960a8f60772SMel Gorman * Optimal placement is no good if the memory bus is saturated and 1961a8f60772SMel Gorman * all the time is being spent migrating! 1962a8f60772SMel Gorman */ 1963340ef390SHugh Dickins if (numamigrate_update_ratelimit(pgdat, 1)) 1964a8f60772SMel Gorman goto out; 1965a8f60772SMel Gorman 1966b32967ffSMel Gorman isolated = numamigrate_isolate_page(pgdat, page); 1967b32967ffSMel Gorman if (!isolated) 19687039e1dbSPeter Zijlstra goto out; 19697039e1dbSPeter Zijlstra 19707039e1dbSPeter Zijlstra list_add(&page->lru, &migratepages); 19719c620e2bSHugh Dickins nr_remaining = migrate_pages(&migratepages, alloc_misplaced_dst_page, 197268711a74SDavid Rientjes NULL, node, MIGRATE_ASYNC, 197368711a74SDavid Rientjes MR_NUMA_MISPLACED); 19747039e1dbSPeter Zijlstra if (nr_remaining) { 197559c82b70SJoonsoo Kim if (!list_empty(&migratepages)) { 197659c82b70SJoonsoo Kim list_del(&page->lru); 1977599d0c95SMel Gorman dec_node_page_state(page, NR_ISOLATED_ANON + 197859c82b70SJoonsoo Kim page_is_file_cache(page)); 197959c82b70SJoonsoo Kim putback_lru_page(page); 198059c82b70SJoonsoo Kim } 19817039e1dbSPeter Zijlstra isolated = 0; 198203c5a6e1SMel Gorman } else 198303c5a6e1SMel Gorman count_vm_numa_event(NUMA_PAGE_MIGRATE); 19847039e1dbSPeter Zijlstra BUG_ON(!list_empty(&migratepages)); 19857039e1dbSPeter Zijlstra return isolated; 1986340ef390SHugh Dickins 1987340ef390SHugh Dickins out: 1988340ef390SHugh Dickins put_page(page); 1989340ef390SHugh Dickins return 0; 19907039e1dbSPeter Zijlstra } 1991220018d3SMel Gorman #endif /* CONFIG_NUMA_BALANCING */ 1992b32967ffSMel Gorman 1993220018d3SMel Gorman #if defined(CONFIG_NUMA_BALANCING) && defined(CONFIG_TRANSPARENT_HUGEPAGE) 1994340ef390SHugh Dickins /* 1995340ef390SHugh Dickins * Migrates a THP to a given target node. page must be locked and is unlocked 1996340ef390SHugh Dickins * before returning. 1997340ef390SHugh Dickins */ 1998b32967ffSMel Gorman int migrate_misplaced_transhuge_page(struct mm_struct *mm, 1999b32967ffSMel Gorman struct vm_area_struct *vma, 2000b32967ffSMel Gorman pmd_t *pmd, pmd_t entry, 2001b32967ffSMel Gorman unsigned long address, 2002b32967ffSMel Gorman struct page *page, int node) 2003b32967ffSMel Gorman { 2004c4088ebdSKirill A. Shutemov spinlock_t *ptl; 2005b32967ffSMel Gorman pg_data_t *pgdat = NODE_DATA(node); 2006b32967ffSMel Gorman int isolated = 0; 2007b32967ffSMel Gorman struct page *new_page = NULL; 2008b32967ffSMel Gorman int page_lru = page_is_file_cache(page); 2009f714f4f2SMel Gorman unsigned long mmun_start = address & HPAGE_PMD_MASK; 2010f714f4f2SMel Gorman unsigned long mmun_end = mmun_start + HPAGE_PMD_SIZE; 2011b32967ffSMel Gorman 2012b32967ffSMel Gorman /* 2013b32967ffSMel Gorman * Rate-limit the amount of data that is being migrated to a node. 2014b32967ffSMel Gorman * Optimal placement is no good if the memory bus is saturated and 2015b32967ffSMel Gorman * all the time is being spent migrating! 2016b32967ffSMel Gorman */ 2017d28d4335SMel Gorman if (numamigrate_update_ratelimit(pgdat, HPAGE_PMD_NR)) 2018b32967ffSMel Gorman goto out_dropref; 2019b32967ffSMel Gorman 2020b32967ffSMel Gorman new_page = alloc_pages_node(node, 202125160354SVlastimil Babka (GFP_TRANSHUGE_LIGHT | __GFP_THISNODE), 2022e97ca8e5SJohannes Weiner HPAGE_PMD_ORDER); 2023340ef390SHugh Dickins if (!new_page) 2024340ef390SHugh Dickins goto out_fail; 20259a982250SKirill A. Shutemov prep_transhuge_page(new_page); 2026340ef390SHugh Dickins 2027b32967ffSMel Gorman isolated = numamigrate_isolate_page(pgdat, page); 2028340ef390SHugh Dickins if (!isolated) { 2029b32967ffSMel Gorman put_page(new_page); 2030340ef390SHugh Dickins goto out_fail; 2031b32967ffSMel Gorman } 2032b0943d61SMel Gorman 2033b32967ffSMel Gorman /* Prepare a page as a migration target */ 203448c935adSKirill A. Shutemov __SetPageLocked(new_page); 2035d44d363fSShaohua Li if (PageSwapBacked(page)) 2036fa9949daSHugh Dickins __SetPageSwapBacked(new_page); 2037b32967ffSMel Gorman 2038b32967ffSMel Gorman /* anon mapping, we can simply copy page->mapping to the new page: */ 2039b32967ffSMel Gorman new_page->mapping = page->mapping; 2040b32967ffSMel Gorman new_page->index = page->index; 2041b32967ffSMel Gorman migrate_page_copy(new_page, page); 2042b32967ffSMel Gorman WARN_ON(PageLRU(new_page)); 2043b32967ffSMel Gorman 2044b32967ffSMel Gorman /* Recheck the target PMD */ 2045f714f4f2SMel Gorman mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 2046c4088ebdSKirill A. Shutemov ptl = pmd_lock(mm, pmd); 2047f4e177d1SWill Deacon if (unlikely(!pmd_same(*pmd, entry) || !page_ref_freeze(page, 2))) { 2048c4088ebdSKirill A. Shutemov spin_unlock(ptl); 2049f714f4f2SMel Gorman mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 2050b32967ffSMel Gorman 2051b32967ffSMel Gorman /* Reverse changes made by migrate_page_copy() */ 2052b32967ffSMel Gorman if (TestClearPageActive(new_page)) 2053b32967ffSMel Gorman SetPageActive(page); 2054b32967ffSMel Gorman if (TestClearPageUnevictable(new_page)) 2055b32967ffSMel Gorman SetPageUnevictable(page); 2056b32967ffSMel Gorman 2057b32967ffSMel Gorman unlock_page(new_page); 2058b32967ffSMel Gorman put_page(new_page); /* Free it */ 2059b32967ffSMel Gorman 2060a54a407fSMel Gorman /* Retake the callers reference and putback on LRU */ 2061a54a407fSMel Gorman get_page(page); 2062b32967ffSMel Gorman putback_lru_page(page); 2063599d0c95SMel Gorman mod_node_page_state(page_pgdat(page), 2064a54a407fSMel Gorman NR_ISOLATED_ANON + page_lru, -HPAGE_PMD_NR); 2065eb4489f6SMel Gorman 2066eb4489f6SMel Gorman goto out_unlock; 2067b32967ffSMel Gorman } 2068b32967ffSMel Gorman 206910102459SKirill A. Shutemov entry = mk_huge_pmd(new_page, vma->vm_page_prot); 20702b4847e7SMel Gorman entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 2071b32967ffSMel Gorman 20722b4847e7SMel Gorman /* 20732b4847e7SMel Gorman * Clear the old entry under pagetable lock and establish the new PTE. 20742b4847e7SMel Gorman * Any parallel GUP will either observe the old page blocking on the 20752b4847e7SMel Gorman * page lock, block on the page table lock or observe the new page. 20762b4847e7SMel Gorman * The SetPageUptodate on the new page and page_add_new_anon_rmap 20772b4847e7SMel Gorman * guarantee the copy is visible before the pagetable update. 20782b4847e7SMel Gorman */ 2079f714f4f2SMel Gorman flush_cache_range(vma, mmun_start, mmun_end); 2080d281ee61SKirill A. Shutemov page_add_anon_rmap(new_page, vma, mmun_start, true); 20818809aa2dSAneesh Kumar K.V pmdp_huge_clear_flush_notify(vma, mmun_start, pmd); 2082f714f4f2SMel Gorman set_pmd_at(mm, mmun_start, pmd, entry); 2083ce4a9cc5SStephen Rothwell update_mmu_cache_pmd(vma, address, &entry); 20842b4847e7SMel Gorman 2085f4e177d1SWill Deacon page_ref_unfreeze(page, 2); 208651afb12bSHugh Dickins mlock_migrate_page(new_page, page); 2087d281ee61SKirill A. Shutemov page_remove_rmap(page, true); 20887cd12b4aSVlastimil Babka set_page_owner_migrate_reason(new_page, MR_NUMA_MISPLACED); 20892b4847e7SMel Gorman 2090c4088ebdSKirill A. Shutemov spin_unlock(ptl); 2091f714f4f2SMel Gorman mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 2092b32967ffSMel Gorman 209311de9927SMel Gorman /* Take an "isolate" reference and put new page on the LRU. */ 209411de9927SMel Gorman get_page(new_page); 209511de9927SMel Gorman putback_lru_page(new_page); 209611de9927SMel Gorman 2097b32967ffSMel Gorman unlock_page(new_page); 2098b32967ffSMel Gorman unlock_page(page); 2099b32967ffSMel Gorman put_page(page); /* Drop the rmap reference */ 2100b32967ffSMel Gorman put_page(page); /* Drop the LRU isolation reference */ 2101b32967ffSMel Gorman 2102b32967ffSMel Gorman count_vm_events(PGMIGRATE_SUCCESS, HPAGE_PMD_NR); 2103b32967ffSMel Gorman count_vm_numa_events(NUMA_PAGE_MIGRATE, HPAGE_PMD_NR); 2104b32967ffSMel Gorman 2105599d0c95SMel Gorman mod_node_page_state(page_pgdat(page), 2106b32967ffSMel Gorman NR_ISOLATED_ANON + page_lru, 2107b32967ffSMel Gorman -HPAGE_PMD_NR); 2108b32967ffSMel Gorman return isolated; 2109b32967ffSMel Gorman 2110340ef390SHugh Dickins out_fail: 2111340ef390SHugh Dickins count_vm_events(PGMIGRATE_FAIL, HPAGE_PMD_NR); 2112b32967ffSMel Gorman out_dropref: 21132b4847e7SMel Gorman ptl = pmd_lock(mm, pmd); 21142b4847e7SMel Gorman if (pmd_same(*pmd, entry)) { 21154d942466SMel Gorman entry = pmd_modify(entry, vma->vm_page_prot); 2116f714f4f2SMel Gorman set_pmd_at(mm, mmun_start, pmd, entry); 2117a54a407fSMel Gorman update_mmu_cache_pmd(vma, address, &entry); 21182b4847e7SMel Gorman } 21192b4847e7SMel Gorman spin_unlock(ptl); 2120a54a407fSMel Gorman 2121eb4489f6SMel Gorman out_unlock: 2122340ef390SHugh Dickins unlock_page(page); 2123b32967ffSMel Gorman put_page(page); 2124b32967ffSMel Gorman return 0; 2125b32967ffSMel Gorman } 21267039e1dbSPeter Zijlstra #endif /* CONFIG_NUMA_BALANCING */ 21277039e1dbSPeter Zijlstra 21287039e1dbSPeter Zijlstra #endif /* CONFIG_NUMA */ 21298763cb45SJérôme Glisse 21308763cb45SJérôme Glisse struct migrate_vma { 21318763cb45SJérôme Glisse struct vm_area_struct *vma; 21328763cb45SJérôme Glisse unsigned long *dst; 21338763cb45SJérôme Glisse unsigned long *src; 21348763cb45SJérôme Glisse unsigned long cpages; 21358763cb45SJérôme Glisse unsigned long npages; 21368763cb45SJérôme Glisse unsigned long start; 21378763cb45SJérôme Glisse unsigned long end; 21388763cb45SJérôme Glisse }; 21398763cb45SJérôme Glisse 21408763cb45SJérôme Glisse static int migrate_vma_collect_hole(unsigned long start, 21418763cb45SJérôme Glisse unsigned long end, 21428763cb45SJérôme Glisse struct mm_walk *walk) 21438763cb45SJérôme Glisse { 21448763cb45SJérôme Glisse struct migrate_vma *migrate = walk->private; 21458763cb45SJérôme Glisse unsigned long addr; 21468763cb45SJérôme Glisse 21478763cb45SJérôme Glisse for (addr = start & PAGE_MASK; addr < end; addr += PAGE_SIZE) { 21488315ada7SJérôme Glisse migrate->src[migrate->npages++] = MIGRATE_PFN_MIGRATE; 21498315ada7SJérôme Glisse migrate->dst[migrate->npages] = 0; 21508315ada7SJérôme Glisse migrate->cpages++; 21518315ada7SJérôme Glisse } 21528315ada7SJérôme Glisse 21538315ada7SJérôme Glisse return 0; 21548315ada7SJérôme Glisse } 21558315ada7SJérôme Glisse 21568315ada7SJérôme Glisse static int migrate_vma_collect_skip(unsigned long start, 21578315ada7SJérôme Glisse unsigned long end, 21588315ada7SJérôme Glisse struct mm_walk *walk) 21598315ada7SJérôme Glisse { 21608315ada7SJérôme Glisse struct migrate_vma *migrate = walk->private; 21618315ada7SJérôme Glisse unsigned long addr; 21628315ada7SJérôme Glisse 21638315ada7SJérôme Glisse for (addr = start & PAGE_MASK; addr < end; addr += PAGE_SIZE) { 21648763cb45SJérôme Glisse migrate->dst[migrate->npages] = 0; 21658763cb45SJérôme Glisse migrate->src[migrate->npages++] = 0; 21668763cb45SJérôme Glisse } 21678763cb45SJérôme Glisse 21688763cb45SJérôme Glisse return 0; 21698763cb45SJérôme Glisse } 21708763cb45SJérôme Glisse 21718763cb45SJérôme Glisse static int migrate_vma_collect_pmd(pmd_t *pmdp, 21728763cb45SJérôme Glisse unsigned long start, 21738763cb45SJérôme Glisse unsigned long end, 21748763cb45SJérôme Glisse struct mm_walk *walk) 21758763cb45SJérôme Glisse { 21768763cb45SJérôme Glisse struct migrate_vma *migrate = walk->private; 21778763cb45SJérôme Glisse struct vm_area_struct *vma = walk->vma; 21788763cb45SJérôme Glisse struct mm_struct *mm = vma->vm_mm; 21798c3328f1SJérôme Glisse unsigned long addr = start, unmapped = 0; 21808763cb45SJérôme Glisse spinlock_t *ptl; 21818763cb45SJérôme Glisse pte_t *ptep; 21828763cb45SJérôme Glisse 21838763cb45SJérôme Glisse again: 21848763cb45SJérôme Glisse if (pmd_none(*pmdp)) 21858763cb45SJérôme Glisse return migrate_vma_collect_hole(start, end, walk); 21868763cb45SJérôme Glisse 21878763cb45SJérôme Glisse if (pmd_trans_huge(*pmdp)) { 21888763cb45SJérôme Glisse struct page *page; 21898763cb45SJérôme Glisse 21908763cb45SJérôme Glisse ptl = pmd_lock(mm, pmdp); 21918763cb45SJérôme Glisse if (unlikely(!pmd_trans_huge(*pmdp))) { 21928763cb45SJérôme Glisse spin_unlock(ptl); 21938763cb45SJérôme Glisse goto again; 21948763cb45SJérôme Glisse } 21958763cb45SJérôme Glisse 21968763cb45SJérôme Glisse page = pmd_page(*pmdp); 21978763cb45SJérôme Glisse if (is_huge_zero_page(page)) { 21988763cb45SJérôme Glisse spin_unlock(ptl); 21998763cb45SJérôme Glisse split_huge_pmd(vma, pmdp, addr); 22008763cb45SJérôme Glisse if (pmd_trans_unstable(pmdp)) 22018315ada7SJérôme Glisse return migrate_vma_collect_skip(start, end, 22028763cb45SJérôme Glisse walk); 22038763cb45SJérôme Glisse } else { 22048763cb45SJérôme Glisse int ret; 22058763cb45SJérôme Glisse 22068763cb45SJérôme Glisse get_page(page); 22078763cb45SJérôme Glisse spin_unlock(ptl); 22088763cb45SJérôme Glisse if (unlikely(!trylock_page(page))) 22098315ada7SJérôme Glisse return migrate_vma_collect_skip(start, end, 22108763cb45SJérôme Glisse walk); 22118763cb45SJérôme Glisse ret = split_huge_page(page); 22128763cb45SJérôme Glisse unlock_page(page); 22138763cb45SJérôme Glisse put_page(page); 22148315ada7SJérôme Glisse if (ret) 22158315ada7SJérôme Glisse return migrate_vma_collect_skip(start, end, 22168315ada7SJérôme Glisse walk); 22178315ada7SJérôme Glisse if (pmd_none(*pmdp)) 22188763cb45SJérôme Glisse return migrate_vma_collect_hole(start, end, 22198763cb45SJérôme Glisse walk); 22208763cb45SJérôme Glisse } 22218763cb45SJérôme Glisse } 22228763cb45SJérôme Glisse 22238763cb45SJérôme Glisse if (unlikely(pmd_bad(*pmdp))) 22248315ada7SJérôme Glisse return migrate_vma_collect_skip(start, end, walk); 22258763cb45SJérôme Glisse 22268763cb45SJérôme Glisse ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl); 22278c3328f1SJérôme Glisse arch_enter_lazy_mmu_mode(); 22288c3328f1SJérôme Glisse 22298763cb45SJérôme Glisse for (; addr < end; addr += PAGE_SIZE, ptep++) { 22308763cb45SJérôme Glisse unsigned long mpfn, pfn; 22318763cb45SJérôme Glisse struct page *page; 22328c3328f1SJérôme Glisse swp_entry_t entry; 22338763cb45SJérôme Glisse pte_t pte; 22348763cb45SJérôme Glisse 22358763cb45SJérôme Glisse pte = *ptep; 22368763cb45SJérôme Glisse pfn = pte_pfn(pte); 22378763cb45SJérôme Glisse 2238a5430ddaSJérôme Glisse if (pte_none(pte)) { 22398315ada7SJérôme Glisse mpfn = MIGRATE_PFN_MIGRATE; 22408315ada7SJérôme Glisse migrate->cpages++; 22418315ada7SJérôme Glisse pfn = 0; 22428763cb45SJérôme Glisse goto next; 22438763cb45SJérôme Glisse } 22448763cb45SJérôme Glisse 2245a5430ddaSJérôme Glisse if (!pte_present(pte)) { 2246a5430ddaSJérôme Glisse mpfn = pfn = 0; 2247a5430ddaSJérôme Glisse 2248a5430ddaSJérôme Glisse /* 2249a5430ddaSJérôme Glisse * Only care about unaddressable device page special 2250a5430ddaSJérôme Glisse * page table entry. Other special swap entries are not 2251a5430ddaSJérôme Glisse * migratable, and we ignore regular swapped page. 2252a5430ddaSJérôme Glisse */ 2253a5430ddaSJérôme Glisse entry = pte_to_swp_entry(pte); 2254a5430ddaSJérôme Glisse if (!is_device_private_entry(entry)) 2255a5430ddaSJérôme Glisse goto next; 2256a5430ddaSJérôme Glisse 2257a5430ddaSJérôme Glisse page = device_private_entry_to_page(entry); 2258a5430ddaSJérôme Glisse mpfn = migrate_pfn(page_to_pfn(page))| 2259a5430ddaSJérôme Glisse MIGRATE_PFN_DEVICE | MIGRATE_PFN_MIGRATE; 2260a5430ddaSJérôme Glisse if (is_write_device_private_entry(entry)) 2261a5430ddaSJérôme Glisse mpfn |= MIGRATE_PFN_WRITE; 2262a5430ddaSJérôme Glisse } else { 22638315ada7SJérôme Glisse if (is_zero_pfn(pfn)) { 22648315ada7SJérôme Glisse mpfn = MIGRATE_PFN_MIGRATE; 22658315ada7SJérôme Glisse migrate->cpages++; 22668315ada7SJérôme Glisse pfn = 0; 22678315ada7SJérôme Glisse goto next; 22688315ada7SJérôme Glisse } 2269*df6ad698SJérôme Glisse page = _vm_normal_page(migrate->vma, addr, pte, true); 2270a5430ddaSJérôme Glisse mpfn = migrate_pfn(pfn) | MIGRATE_PFN_MIGRATE; 2271a5430ddaSJérôme Glisse mpfn |= pte_write(pte) ? MIGRATE_PFN_WRITE : 0; 2272a5430ddaSJérôme Glisse } 2273a5430ddaSJérôme Glisse 2274a5430ddaSJérôme Glisse /* FIXME support THP */ 22758763cb45SJérôme Glisse if (!page || !page->mapping || PageTransCompound(page)) { 22768763cb45SJérôme Glisse mpfn = pfn = 0; 22778763cb45SJérôme Glisse goto next; 22788763cb45SJérôme Glisse } 2279a5430ddaSJérôme Glisse pfn = page_to_pfn(page); 22808763cb45SJérôme Glisse 22818763cb45SJérôme Glisse /* 22828763cb45SJérôme Glisse * By getting a reference on the page we pin it and that blocks 22838763cb45SJérôme Glisse * any kind of migration. Side effect is that it "freezes" the 22848763cb45SJérôme Glisse * pte. 22858763cb45SJérôme Glisse * 22868763cb45SJérôme Glisse * We drop this reference after isolating the page from the lru 22878763cb45SJérôme Glisse * for non device page (device page are not on the lru and thus 22888763cb45SJérôme Glisse * can't be dropped from it). 22898763cb45SJérôme Glisse */ 22908763cb45SJérôme Glisse get_page(page); 22918763cb45SJérôme Glisse migrate->cpages++; 22928763cb45SJérôme Glisse 22938c3328f1SJérôme Glisse /* 22948c3328f1SJérôme Glisse * Optimize for the common case where page is only mapped once 22958c3328f1SJérôme Glisse * in one process. If we can lock the page, then we can safely 22968c3328f1SJérôme Glisse * set up a special migration page table entry now. 22978c3328f1SJérôme Glisse */ 22988c3328f1SJérôme Glisse if (trylock_page(page)) { 22998c3328f1SJérôme Glisse pte_t swp_pte; 23008c3328f1SJérôme Glisse 23018c3328f1SJérôme Glisse mpfn |= MIGRATE_PFN_LOCKED; 23028c3328f1SJérôme Glisse ptep_get_and_clear(mm, addr, ptep); 23038c3328f1SJérôme Glisse 23048c3328f1SJérôme Glisse /* Setup special migration page table entry */ 23058c3328f1SJérôme Glisse entry = make_migration_entry(page, pte_write(pte)); 23068c3328f1SJérôme Glisse swp_pte = swp_entry_to_pte(entry); 23078c3328f1SJérôme Glisse if (pte_soft_dirty(pte)) 23088c3328f1SJérôme Glisse swp_pte = pte_swp_mksoft_dirty(swp_pte); 23098c3328f1SJérôme Glisse set_pte_at(mm, addr, ptep, swp_pte); 23108c3328f1SJérôme Glisse 23118c3328f1SJérôme Glisse /* 23128c3328f1SJérôme Glisse * This is like regular unmap: we remove the rmap and 23138c3328f1SJérôme Glisse * drop page refcount. Page won't be freed, as we took 23148c3328f1SJérôme Glisse * a reference just above. 23158c3328f1SJérôme Glisse */ 23168c3328f1SJérôme Glisse page_remove_rmap(page, false); 23178c3328f1SJérôme Glisse put_page(page); 2318a5430ddaSJérôme Glisse 2319a5430ddaSJérôme Glisse if (pte_present(pte)) 23208c3328f1SJérôme Glisse unmapped++; 23218c3328f1SJérôme Glisse } 23228c3328f1SJérôme Glisse 23238763cb45SJérôme Glisse next: 2324a5430ddaSJérôme Glisse migrate->dst[migrate->npages] = 0; 23258763cb45SJérôme Glisse migrate->src[migrate->npages++] = mpfn; 23268763cb45SJérôme Glisse } 23278c3328f1SJérôme Glisse arch_leave_lazy_mmu_mode(); 23288763cb45SJérôme Glisse pte_unmap_unlock(ptep - 1, ptl); 23298763cb45SJérôme Glisse 23308c3328f1SJérôme Glisse /* Only flush the TLB if we actually modified any entries */ 23318c3328f1SJérôme Glisse if (unmapped) 23328c3328f1SJérôme Glisse flush_tlb_range(walk->vma, start, end); 23338c3328f1SJérôme Glisse 23348763cb45SJérôme Glisse return 0; 23358763cb45SJérôme Glisse } 23368763cb45SJérôme Glisse 23378763cb45SJérôme Glisse /* 23388763cb45SJérôme Glisse * migrate_vma_collect() - collect pages over a range of virtual addresses 23398763cb45SJérôme Glisse * @migrate: migrate struct containing all migration information 23408763cb45SJérôme Glisse * 23418763cb45SJérôme Glisse * This will walk the CPU page table. For each virtual address backed by a 23428763cb45SJérôme Glisse * valid page, it updates the src array and takes a reference on the page, in 23438763cb45SJérôme Glisse * order to pin the page until we lock it and unmap it. 23448763cb45SJérôme Glisse */ 23458763cb45SJérôme Glisse static void migrate_vma_collect(struct migrate_vma *migrate) 23468763cb45SJérôme Glisse { 23478763cb45SJérôme Glisse struct mm_walk mm_walk; 23488763cb45SJérôme Glisse 23498763cb45SJérôme Glisse mm_walk.pmd_entry = migrate_vma_collect_pmd; 23508763cb45SJérôme Glisse mm_walk.pte_entry = NULL; 23518763cb45SJérôme Glisse mm_walk.pte_hole = migrate_vma_collect_hole; 23528763cb45SJérôme Glisse mm_walk.hugetlb_entry = NULL; 23538763cb45SJérôme Glisse mm_walk.test_walk = NULL; 23548763cb45SJérôme Glisse mm_walk.vma = migrate->vma; 23558763cb45SJérôme Glisse mm_walk.mm = migrate->vma->vm_mm; 23568763cb45SJérôme Glisse mm_walk.private = migrate; 23578763cb45SJérôme Glisse 23588c3328f1SJérôme Glisse mmu_notifier_invalidate_range_start(mm_walk.mm, 23598c3328f1SJérôme Glisse migrate->start, 23608c3328f1SJérôme Glisse migrate->end); 23618763cb45SJérôme Glisse walk_page_range(migrate->start, migrate->end, &mm_walk); 23628c3328f1SJérôme Glisse mmu_notifier_invalidate_range_end(mm_walk.mm, 23638c3328f1SJérôme Glisse migrate->start, 23648c3328f1SJérôme Glisse migrate->end); 23658763cb45SJérôme Glisse 23668763cb45SJérôme Glisse migrate->end = migrate->start + (migrate->npages << PAGE_SHIFT); 23678763cb45SJérôme Glisse } 23688763cb45SJérôme Glisse 23698763cb45SJérôme Glisse /* 23708763cb45SJérôme Glisse * migrate_vma_check_page() - check if page is pinned or not 23718763cb45SJérôme Glisse * @page: struct page to check 23728763cb45SJérôme Glisse * 23738763cb45SJérôme Glisse * Pinned pages cannot be migrated. This is the same test as in 23748763cb45SJérôme Glisse * migrate_page_move_mapping(), except that here we allow migration of a 23758763cb45SJérôme Glisse * ZONE_DEVICE page. 23768763cb45SJérôme Glisse */ 23778763cb45SJérôme Glisse static bool migrate_vma_check_page(struct page *page) 23788763cb45SJérôme Glisse { 23798763cb45SJérôme Glisse /* 23808763cb45SJérôme Glisse * One extra ref because caller holds an extra reference, either from 23818763cb45SJérôme Glisse * isolate_lru_page() for a regular page, or migrate_vma_collect() for 23828763cb45SJérôme Glisse * a device page. 23838763cb45SJérôme Glisse */ 23848763cb45SJérôme Glisse int extra = 1; 23858763cb45SJérôme Glisse 23868763cb45SJérôme Glisse /* 23878763cb45SJérôme Glisse * FIXME support THP (transparent huge page), it is bit more complex to 23888763cb45SJérôme Glisse * check them than regular pages, because they can be mapped with a pmd 23898763cb45SJérôme Glisse * or with a pte (split pte mapping). 23908763cb45SJérôme Glisse */ 23918763cb45SJérôme Glisse if (PageCompound(page)) 23928763cb45SJérôme Glisse return false; 23938763cb45SJérôme Glisse 2394a5430ddaSJérôme Glisse /* Page from ZONE_DEVICE have one extra reference */ 2395a5430ddaSJérôme Glisse if (is_zone_device_page(page)) { 2396a5430ddaSJérôme Glisse /* 2397a5430ddaSJérôme Glisse * Private page can never be pin as they have no valid pte and 2398a5430ddaSJérôme Glisse * GUP will fail for those. Yet if there is a pending migration 2399a5430ddaSJérôme Glisse * a thread might try to wait on the pte migration entry and 2400a5430ddaSJérôme Glisse * will bump the page reference count. Sadly there is no way to 2401a5430ddaSJérôme Glisse * differentiate a regular pin from migration wait. Hence to 2402a5430ddaSJérôme Glisse * avoid 2 racing thread trying to migrate back to CPU to enter 2403a5430ddaSJérôme Glisse * infinite loop (one stoping migration because the other is 2404a5430ddaSJérôme Glisse * waiting on pte migration entry). We always return true here. 2405a5430ddaSJérôme Glisse * 2406a5430ddaSJérôme Glisse * FIXME proper solution is to rework migration_entry_wait() so 2407a5430ddaSJérôme Glisse * it does not need to take a reference on page. 2408a5430ddaSJérôme Glisse */ 2409a5430ddaSJérôme Glisse if (is_device_private_page(page)) 2410a5430ddaSJérôme Glisse return true; 2411a5430ddaSJérôme Glisse 2412*df6ad698SJérôme Glisse /* 2413*df6ad698SJérôme Glisse * Only allow device public page to be migrated and account for 2414*df6ad698SJérôme Glisse * the extra reference count imply by ZONE_DEVICE pages. 2415*df6ad698SJérôme Glisse */ 2416*df6ad698SJérôme Glisse if (!is_device_public_page(page)) 2417a5430ddaSJérôme Glisse return false; 2418*df6ad698SJérôme Glisse extra++; 2419a5430ddaSJérôme Glisse } 2420a5430ddaSJérôme Glisse 2421*df6ad698SJérôme Glisse /* For file back page */ 2422*df6ad698SJérôme Glisse if (page_mapping(page)) 2423*df6ad698SJérôme Glisse extra += 1 + page_has_private(page); 2424*df6ad698SJérôme Glisse 24258763cb45SJérôme Glisse if ((page_count(page) - extra) > page_mapcount(page)) 24268763cb45SJérôme Glisse return false; 24278763cb45SJérôme Glisse 24288763cb45SJérôme Glisse return true; 24298763cb45SJérôme Glisse } 24308763cb45SJérôme Glisse 24318763cb45SJérôme Glisse /* 24328763cb45SJérôme Glisse * migrate_vma_prepare() - lock pages and isolate them from the lru 24338763cb45SJérôme Glisse * @migrate: migrate struct containing all migration information 24348763cb45SJérôme Glisse * 24358763cb45SJérôme Glisse * This locks pages that have been collected by migrate_vma_collect(). Once each 24368763cb45SJérôme Glisse * page is locked it is isolated from the lru (for non-device pages). Finally, 24378763cb45SJérôme Glisse * the ref taken by migrate_vma_collect() is dropped, as locked pages cannot be 24388763cb45SJérôme Glisse * migrated by concurrent kernel threads. 24398763cb45SJérôme Glisse */ 24408763cb45SJérôme Glisse static void migrate_vma_prepare(struct migrate_vma *migrate) 24418763cb45SJérôme Glisse { 24428763cb45SJérôme Glisse const unsigned long npages = migrate->npages; 24438c3328f1SJérôme Glisse const unsigned long start = migrate->start; 24448c3328f1SJérôme Glisse unsigned long addr, i, restore = 0; 24458763cb45SJérôme Glisse bool allow_drain = true; 24468763cb45SJérôme Glisse 24478763cb45SJérôme Glisse lru_add_drain(); 24488763cb45SJérôme Glisse 24498763cb45SJérôme Glisse for (i = 0; (i < npages) && migrate->cpages; i++) { 24508763cb45SJérôme Glisse struct page *page = migrate_pfn_to_page(migrate->src[i]); 24518c3328f1SJérôme Glisse bool remap = true; 24528763cb45SJérôme Glisse 24538763cb45SJérôme Glisse if (!page) 24548763cb45SJérôme Glisse continue; 24558763cb45SJérôme Glisse 24568c3328f1SJérôme Glisse if (!(migrate->src[i] & MIGRATE_PFN_LOCKED)) { 24578763cb45SJérôme Glisse /* 24588763cb45SJérôme Glisse * Because we are migrating several pages there can be 24598763cb45SJérôme Glisse * a deadlock between 2 concurrent migration where each 24608763cb45SJérôme Glisse * are waiting on each other page lock. 24618763cb45SJérôme Glisse * 24628763cb45SJérôme Glisse * Make migrate_vma() a best effort thing and backoff 24638763cb45SJérôme Glisse * for any page we can not lock right away. 24648763cb45SJérôme Glisse */ 24658763cb45SJérôme Glisse if (!trylock_page(page)) { 24668763cb45SJérôme Glisse migrate->src[i] = 0; 24678763cb45SJérôme Glisse migrate->cpages--; 24688763cb45SJérôme Glisse put_page(page); 24698763cb45SJérôme Glisse continue; 24708763cb45SJérôme Glisse } 24718c3328f1SJérôme Glisse remap = false; 24728763cb45SJérôme Glisse migrate->src[i] |= MIGRATE_PFN_LOCKED; 24738c3328f1SJérôme Glisse } 24748763cb45SJérôme Glisse 2475a5430ddaSJérôme Glisse /* ZONE_DEVICE pages are not on LRU */ 2476a5430ddaSJérôme Glisse if (!is_zone_device_page(page)) { 24778763cb45SJérôme Glisse if (!PageLRU(page) && allow_drain) { 24788763cb45SJérôme Glisse /* Drain CPU's pagevec */ 24798763cb45SJérôme Glisse lru_add_drain_all(); 24808763cb45SJérôme Glisse allow_drain = false; 24818763cb45SJérôme Glisse } 24828763cb45SJérôme Glisse 24838763cb45SJérôme Glisse if (isolate_lru_page(page)) { 24848c3328f1SJérôme Glisse if (remap) { 24858c3328f1SJérôme Glisse migrate->src[i] &= ~MIGRATE_PFN_MIGRATE; 24868c3328f1SJérôme Glisse migrate->cpages--; 24878c3328f1SJérôme Glisse restore++; 24888c3328f1SJérôme Glisse } else { 24898763cb45SJérôme Glisse migrate->src[i] = 0; 24908763cb45SJérôme Glisse unlock_page(page); 24918763cb45SJérôme Glisse migrate->cpages--; 24928763cb45SJérôme Glisse put_page(page); 24938c3328f1SJérôme Glisse } 24948763cb45SJérôme Glisse continue; 24958763cb45SJérôme Glisse } 24968763cb45SJérôme Glisse 2497a5430ddaSJérôme Glisse /* Drop the reference we took in collect */ 2498a5430ddaSJérôme Glisse put_page(page); 2499a5430ddaSJérôme Glisse } 2500a5430ddaSJérôme Glisse 25018763cb45SJérôme Glisse if (!migrate_vma_check_page(page)) { 25028c3328f1SJérôme Glisse if (remap) { 25038c3328f1SJérôme Glisse migrate->src[i] &= ~MIGRATE_PFN_MIGRATE; 25048c3328f1SJérôme Glisse migrate->cpages--; 25058c3328f1SJérôme Glisse restore++; 25068c3328f1SJérôme Glisse 2507a5430ddaSJérôme Glisse if (!is_zone_device_page(page)) { 25088c3328f1SJérôme Glisse get_page(page); 25098c3328f1SJérôme Glisse putback_lru_page(page); 2510a5430ddaSJérôme Glisse } 25118c3328f1SJérôme Glisse } else { 25128763cb45SJérôme Glisse migrate->src[i] = 0; 25138763cb45SJérôme Glisse unlock_page(page); 25148763cb45SJérôme Glisse migrate->cpages--; 25158763cb45SJérôme Glisse 2516a5430ddaSJérôme Glisse if (!is_zone_device_page(page)) 25178763cb45SJérôme Glisse putback_lru_page(page); 2518a5430ddaSJérôme Glisse else 2519a5430ddaSJérôme Glisse put_page(page); 25208763cb45SJérôme Glisse } 25218763cb45SJérôme Glisse } 25228763cb45SJérôme Glisse } 25238763cb45SJérôme Glisse 25248c3328f1SJérôme Glisse for (i = 0, addr = start; i < npages && restore; i++, addr += PAGE_SIZE) { 25258c3328f1SJérôme Glisse struct page *page = migrate_pfn_to_page(migrate->src[i]); 25268c3328f1SJérôme Glisse 25278c3328f1SJérôme Glisse if (!page || (migrate->src[i] & MIGRATE_PFN_MIGRATE)) 25288c3328f1SJérôme Glisse continue; 25298c3328f1SJérôme Glisse 25308c3328f1SJérôme Glisse remove_migration_pte(page, migrate->vma, addr, page); 25318c3328f1SJérôme Glisse 25328c3328f1SJérôme Glisse migrate->src[i] = 0; 25338c3328f1SJérôme Glisse unlock_page(page); 25348c3328f1SJérôme Glisse put_page(page); 25358c3328f1SJérôme Glisse restore--; 25368c3328f1SJérôme Glisse } 25378c3328f1SJérôme Glisse } 25388c3328f1SJérôme Glisse 25398763cb45SJérôme Glisse /* 25408763cb45SJérôme Glisse * migrate_vma_unmap() - replace page mapping with special migration pte entry 25418763cb45SJérôme Glisse * @migrate: migrate struct containing all migration information 25428763cb45SJérôme Glisse * 25438763cb45SJérôme Glisse * Replace page mapping (CPU page table pte) with a special migration pte entry 25448763cb45SJérôme Glisse * and check again if it has been pinned. Pinned pages are restored because we 25458763cb45SJérôme Glisse * cannot migrate them. 25468763cb45SJérôme Glisse * 25478763cb45SJérôme Glisse * This is the last step before we call the device driver callback to allocate 25488763cb45SJérôme Glisse * destination memory and copy contents of original page over to new page. 25498763cb45SJérôme Glisse */ 25508763cb45SJérôme Glisse static void migrate_vma_unmap(struct migrate_vma *migrate) 25518763cb45SJérôme Glisse { 25528763cb45SJérôme Glisse int flags = TTU_MIGRATION | TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS; 25538763cb45SJérôme Glisse const unsigned long npages = migrate->npages; 25548763cb45SJérôme Glisse const unsigned long start = migrate->start; 25558763cb45SJérôme Glisse unsigned long addr, i, restore = 0; 25568763cb45SJérôme Glisse 25578763cb45SJérôme Glisse for (i = 0; i < npages; i++) { 25588763cb45SJérôme Glisse struct page *page = migrate_pfn_to_page(migrate->src[i]); 25598763cb45SJérôme Glisse 25608763cb45SJérôme Glisse if (!page || !(migrate->src[i] & MIGRATE_PFN_MIGRATE)) 25618763cb45SJérôme Glisse continue; 25628763cb45SJérôme Glisse 25638c3328f1SJérôme Glisse if (page_mapped(page)) { 25648763cb45SJérôme Glisse try_to_unmap(page, flags); 25658c3328f1SJérôme Glisse if (page_mapped(page)) 25668c3328f1SJérôme Glisse goto restore; 25678c3328f1SJérôme Glisse } 25688c3328f1SJérôme Glisse 25698c3328f1SJérôme Glisse if (migrate_vma_check_page(page)) 25708c3328f1SJérôme Glisse continue; 25718c3328f1SJérôme Glisse 25728c3328f1SJérôme Glisse restore: 25738763cb45SJérôme Glisse migrate->src[i] &= ~MIGRATE_PFN_MIGRATE; 25748763cb45SJérôme Glisse migrate->cpages--; 25758763cb45SJérôme Glisse restore++; 25768763cb45SJérôme Glisse } 25778763cb45SJérôme Glisse 25788763cb45SJérôme Glisse for (addr = start, i = 0; i < npages && restore; addr += PAGE_SIZE, i++) { 25798763cb45SJérôme Glisse struct page *page = migrate_pfn_to_page(migrate->src[i]); 25808763cb45SJérôme Glisse 25818763cb45SJérôme Glisse if (!page || (migrate->src[i] & MIGRATE_PFN_MIGRATE)) 25828763cb45SJérôme Glisse continue; 25838763cb45SJérôme Glisse 25848763cb45SJérôme Glisse remove_migration_ptes(page, page, false); 25858763cb45SJérôme Glisse 25868763cb45SJérôme Glisse migrate->src[i] = 0; 25878763cb45SJérôme Glisse unlock_page(page); 25888763cb45SJérôme Glisse restore--; 25898763cb45SJérôme Glisse 2590a5430ddaSJérôme Glisse if (is_zone_device_page(page)) 2591a5430ddaSJérôme Glisse put_page(page); 2592a5430ddaSJérôme Glisse else 25938763cb45SJérôme Glisse putback_lru_page(page); 25948763cb45SJérôme Glisse } 25958763cb45SJérôme Glisse } 25968763cb45SJérôme Glisse 25978315ada7SJérôme Glisse static void migrate_vma_insert_page(struct migrate_vma *migrate, 25988315ada7SJérôme Glisse unsigned long addr, 25998315ada7SJérôme Glisse struct page *page, 26008315ada7SJérôme Glisse unsigned long *src, 26018315ada7SJérôme Glisse unsigned long *dst) 26028315ada7SJérôme Glisse { 26038315ada7SJérôme Glisse struct vm_area_struct *vma = migrate->vma; 26048315ada7SJérôme Glisse struct mm_struct *mm = vma->vm_mm; 26058315ada7SJérôme Glisse struct mem_cgroup *memcg; 26068315ada7SJérôme Glisse bool flush = false; 26078315ada7SJérôme Glisse spinlock_t *ptl; 26088315ada7SJérôme Glisse pte_t entry; 26098315ada7SJérôme Glisse pgd_t *pgdp; 26108315ada7SJérôme Glisse p4d_t *p4dp; 26118315ada7SJérôme Glisse pud_t *pudp; 26128315ada7SJérôme Glisse pmd_t *pmdp; 26138315ada7SJérôme Glisse pte_t *ptep; 26148315ada7SJérôme Glisse 26158315ada7SJérôme Glisse /* Only allow populating anonymous memory */ 26168315ada7SJérôme Glisse if (!vma_is_anonymous(vma)) 26178315ada7SJérôme Glisse goto abort; 26188315ada7SJérôme Glisse 26198315ada7SJérôme Glisse pgdp = pgd_offset(mm, addr); 26208315ada7SJérôme Glisse p4dp = p4d_alloc(mm, pgdp, addr); 26218315ada7SJérôme Glisse if (!p4dp) 26228315ada7SJérôme Glisse goto abort; 26238315ada7SJérôme Glisse pudp = pud_alloc(mm, p4dp, addr); 26248315ada7SJérôme Glisse if (!pudp) 26258315ada7SJérôme Glisse goto abort; 26268315ada7SJérôme Glisse pmdp = pmd_alloc(mm, pudp, addr); 26278315ada7SJérôme Glisse if (!pmdp) 26288315ada7SJérôme Glisse goto abort; 26298315ada7SJérôme Glisse 26308315ada7SJérôme Glisse if (pmd_trans_huge(*pmdp) || pmd_devmap(*pmdp)) 26318315ada7SJérôme Glisse goto abort; 26328315ada7SJérôme Glisse 26338315ada7SJérôme Glisse /* 26348315ada7SJérôme Glisse * Use pte_alloc() instead of pte_alloc_map(). We can't run 26358315ada7SJérôme Glisse * pte_offset_map() on pmds where a huge pmd might be created 26368315ada7SJérôme Glisse * from a different thread. 26378315ada7SJérôme Glisse * 26388315ada7SJérôme Glisse * pte_alloc_map() is safe to use under down_write(mmap_sem) or when 26398315ada7SJérôme Glisse * parallel threads are excluded by other means. 26408315ada7SJérôme Glisse * 26418315ada7SJérôme Glisse * Here we only have down_read(mmap_sem). 26428315ada7SJérôme Glisse */ 26438315ada7SJérôme Glisse if (pte_alloc(mm, pmdp, addr)) 26448315ada7SJérôme Glisse goto abort; 26458315ada7SJérôme Glisse 26468315ada7SJérôme Glisse /* See the comment in pte_alloc_one_map() */ 26478315ada7SJérôme Glisse if (unlikely(pmd_trans_unstable(pmdp))) 26488315ada7SJérôme Glisse goto abort; 26498315ada7SJérôme Glisse 26508315ada7SJérôme Glisse if (unlikely(anon_vma_prepare(vma))) 26518315ada7SJérôme Glisse goto abort; 26528315ada7SJérôme Glisse if (mem_cgroup_try_charge(page, vma->vm_mm, GFP_KERNEL, &memcg, false)) 26538315ada7SJérôme Glisse goto abort; 26548315ada7SJérôme Glisse 26558315ada7SJérôme Glisse /* 26568315ada7SJérôme Glisse * The memory barrier inside __SetPageUptodate makes sure that 26578315ada7SJérôme Glisse * preceding stores to the page contents become visible before 26588315ada7SJérôme Glisse * the set_pte_at() write. 26598315ada7SJérôme Glisse */ 26608315ada7SJérôme Glisse __SetPageUptodate(page); 26618315ada7SJérôme Glisse 2662*df6ad698SJérôme Glisse if (is_zone_device_page(page)) { 2663*df6ad698SJérôme Glisse if (is_device_private_page(page)) { 26648315ada7SJérôme Glisse swp_entry_t swp_entry; 26658315ada7SJérôme Glisse 26668315ada7SJérôme Glisse swp_entry = make_device_private_entry(page, vma->vm_flags & VM_WRITE); 26678315ada7SJérôme Glisse entry = swp_entry_to_pte(swp_entry); 2668*df6ad698SJérôme Glisse } else if (is_device_public_page(page)) { 2669*df6ad698SJérôme Glisse entry = pte_mkold(mk_pte(page, READ_ONCE(vma->vm_page_prot))); 2670*df6ad698SJérôme Glisse if (vma->vm_flags & VM_WRITE) 2671*df6ad698SJérôme Glisse entry = pte_mkwrite(pte_mkdirty(entry)); 2672*df6ad698SJérôme Glisse entry = pte_mkdevmap(entry); 2673*df6ad698SJérôme Glisse } 26748315ada7SJérôme Glisse } else { 26758315ada7SJérôme Glisse entry = mk_pte(page, vma->vm_page_prot); 26768315ada7SJérôme Glisse if (vma->vm_flags & VM_WRITE) 26778315ada7SJérôme Glisse entry = pte_mkwrite(pte_mkdirty(entry)); 26788315ada7SJérôme Glisse } 26798315ada7SJérôme Glisse 26808315ada7SJérôme Glisse ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl); 26818315ada7SJérôme Glisse 26828315ada7SJérôme Glisse if (pte_present(*ptep)) { 26838315ada7SJérôme Glisse unsigned long pfn = pte_pfn(*ptep); 26848315ada7SJérôme Glisse 26858315ada7SJérôme Glisse if (!is_zero_pfn(pfn)) { 26868315ada7SJérôme Glisse pte_unmap_unlock(ptep, ptl); 26878315ada7SJérôme Glisse mem_cgroup_cancel_charge(page, memcg, false); 26888315ada7SJérôme Glisse goto abort; 26898315ada7SJérôme Glisse } 26908315ada7SJérôme Glisse flush = true; 26918315ada7SJérôme Glisse } else if (!pte_none(*ptep)) { 26928315ada7SJérôme Glisse pte_unmap_unlock(ptep, ptl); 26938315ada7SJérôme Glisse mem_cgroup_cancel_charge(page, memcg, false); 26948315ada7SJérôme Glisse goto abort; 26958315ada7SJérôme Glisse } 26968315ada7SJérôme Glisse 26978315ada7SJérôme Glisse /* 26988315ada7SJérôme Glisse * Check for usefaultfd but do not deliver the fault. Instead, 26998315ada7SJérôme Glisse * just back off. 27008315ada7SJérôme Glisse */ 27018315ada7SJérôme Glisse if (userfaultfd_missing(vma)) { 27028315ada7SJérôme Glisse pte_unmap_unlock(ptep, ptl); 27038315ada7SJérôme Glisse mem_cgroup_cancel_charge(page, memcg, false); 27048315ada7SJérôme Glisse goto abort; 27058315ada7SJérôme Glisse } 27068315ada7SJérôme Glisse 27078315ada7SJérôme Glisse inc_mm_counter(mm, MM_ANONPAGES); 27088315ada7SJérôme Glisse page_add_new_anon_rmap(page, vma, addr, false); 27098315ada7SJérôme Glisse mem_cgroup_commit_charge(page, memcg, false, false); 27108315ada7SJérôme Glisse if (!is_zone_device_page(page)) 27118315ada7SJérôme Glisse lru_cache_add_active_or_unevictable(page, vma); 27128315ada7SJérôme Glisse get_page(page); 27138315ada7SJérôme Glisse 27148315ada7SJérôme Glisse if (flush) { 27158315ada7SJérôme Glisse flush_cache_page(vma, addr, pte_pfn(*ptep)); 27168315ada7SJérôme Glisse ptep_clear_flush_notify(vma, addr, ptep); 27178315ada7SJérôme Glisse set_pte_at_notify(mm, addr, ptep, entry); 27188315ada7SJérôme Glisse update_mmu_cache(vma, addr, ptep); 27198315ada7SJérôme Glisse } else { 27208315ada7SJérôme Glisse /* No need to invalidate - it was non-present before */ 27218315ada7SJérôme Glisse set_pte_at(mm, addr, ptep, entry); 27228315ada7SJérôme Glisse update_mmu_cache(vma, addr, ptep); 27238315ada7SJérôme Glisse } 27248315ada7SJérôme Glisse 27258315ada7SJérôme Glisse pte_unmap_unlock(ptep, ptl); 27268315ada7SJérôme Glisse *src = MIGRATE_PFN_MIGRATE; 27278315ada7SJérôme Glisse return; 27288315ada7SJérôme Glisse 27298315ada7SJérôme Glisse abort: 27308315ada7SJérôme Glisse *src &= ~MIGRATE_PFN_MIGRATE; 27318315ada7SJérôme Glisse } 27328315ada7SJérôme Glisse 27338763cb45SJérôme Glisse /* 27348763cb45SJérôme Glisse * migrate_vma_pages() - migrate meta-data from src page to dst page 27358763cb45SJérôme Glisse * @migrate: migrate struct containing all migration information 27368763cb45SJérôme Glisse * 27378763cb45SJérôme Glisse * This migrates struct page meta-data from source struct page to destination 27388763cb45SJérôme Glisse * struct page. This effectively finishes the migration from source page to the 27398763cb45SJérôme Glisse * destination page. 27408763cb45SJérôme Glisse */ 27418763cb45SJérôme Glisse static void migrate_vma_pages(struct migrate_vma *migrate) 27428763cb45SJérôme Glisse { 27438763cb45SJérôme Glisse const unsigned long npages = migrate->npages; 27448763cb45SJérôme Glisse const unsigned long start = migrate->start; 27458315ada7SJérôme Glisse struct vm_area_struct *vma = migrate->vma; 27468315ada7SJérôme Glisse struct mm_struct *mm = vma->vm_mm; 27478315ada7SJérôme Glisse unsigned long addr, i, mmu_start; 27488315ada7SJérôme Glisse bool notified = false; 27498763cb45SJérôme Glisse 27508763cb45SJérôme Glisse for (i = 0, addr = start; i < npages; addr += PAGE_SIZE, i++) { 27518763cb45SJérôme Glisse struct page *newpage = migrate_pfn_to_page(migrate->dst[i]); 27528763cb45SJérôme Glisse struct page *page = migrate_pfn_to_page(migrate->src[i]); 27538763cb45SJérôme Glisse struct address_space *mapping; 27548763cb45SJérôme Glisse int r; 27558763cb45SJérôme Glisse 27568315ada7SJérôme Glisse if (!newpage) { 27578315ada7SJérôme Glisse migrate->src[i] &= ~MIGRATE_PFN_MIGRATE; 27588763cb45SJérôme Glisse continue; 27598315ada7SJérôme Glisse } 27608315ada7SJérôme Glisse 27618315ada7SJérôme Glisse if (!page) { 27628315ada7SJérôme Glisse if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE)) { 27638763cb45SJérôme Glisse continue; 27648315ada7SJérôme Glisse } 27658315ada7SJérôme Glisse if (!notified) { 27668315ada7SJérôme Glisse mmu_start = addr; 27678315ada7SJérôme Glisse notified = true; 27688315ada7SJérôme Glisse mmu_notifier_invalidate_range_start(mm, 27698315ada7SJérôme Glisse mmu_start, 27708315ada7SJérôme Glisse migrate->end); 27718315ada7SJérôme Glisse } 27728315ada7SJérôme Glisse migrate_vma_insert_page(migrate, addr, newpage, 27738315ada7SJérôme Glisse &migrate->src[i], 27748315ada7SJérôme Glisse &migrate->dst[i]); 27758315ada7SJérôme Glisse continue; 27768315ada7SJérôme Glisse } 27778763cb45SJérôme Glisse 27788763cb45SJérôme Glisse mapping = page_mapping(page); 27798763cb45SJérôme Glisse 2780a5430ddaSJérôme Glisse if (is_zone_device_page(newpage)) { 2781a5430ddaSJérôme Glisse if (is_device_private_page(newpage)) { 2782a5430ddaSJérôme Glisse /* 2783a5430ddaSJérôme Glisse * For now only support private anonymous when 2784a5430ddaSJérôme Glisse * migrating to un-addressable device memory. 2785a5430ddaSJérôme Glisse */ 2786a5430ddaSJérôme Glisse if (mapping) { 2787a5430ddaSJérôme Glisse migrate->src[i] &= ~MIGRATE_PFN_MIGRATE; 2788a5430ddaSJérôme Glisse continue; 2789a5430ddaSJérôme Glisse } 2790*df6ad698SJérôme Glisse } else if (!is_device_public_page(newpage)) { 2791a5430ddaSJérôme Glisse /* 2792a5430ddaSJérôme Glisse * Other types of ZONE_DEVICE page are not 2793a5430ddaSJérôme Glisse * supported. 2794a5430ddaSJérôme Glisse */ 2795a5430ddaSJérôme Glisse migrate->src[i] &= ~MIGRATE_PFN_MIGRATE; 2796a5430ddaSJérôme Glisse continue; 2797a5430ddaSJérôme Glisse } 2798a5430ddaSJérôme Glisse } 2799a5430ddaSJérôme Glisse 28008763cb45SJérôme Glisse r = migrate_page(mapping, newpage, page, MIGRATE_SYNC_NO_COPY); 28018763cb45SJérôme Glisse if (r != MIGRATEPAGE_SUCCESS) 28028763cb45SJérôme Glisse migrate->src[i] &= ~MIGRATE_PFN_MIGRATE; 28038763cb45SJérôme Glisse } 28048315ada7SJérôme Glisse 28058315ada7SJérôme Glisse if (notified) 28068315ada7SJérôme Glisse mmu_notifier_invalidate_range_end(mm, mmu_start, 28078315ada7SJérôme Glisse migrate->end); 28088763cb45SJérôme Glisse } 28098763cb45SJérôme Glisse 28108763cb45SJérôme Glisse /* 28118763cb45SJérôme Glisse * migrate_vma_finalize() - restore CPU page table entry 28128763cb45SJérôme Glisse * @migrate: migrate struct containing all migration information 28138763cb45SJérôme Glisse * 28148763cb45SJérôme Glisse * This replaces the special migration pte entry with either a mapping to the 28158763cb45SJérôme Glisse * new page if migration was successful for that page, or to the original page 28168763cb45SJérôme Glisse * otherwise. 28178763cb45SJérôme Glisse * 28188763cb45SJérôme Glisse * This also unlocks the pages and puts them back on the lru, or drops the extra 28198763cb45SJérôme Glisse * refcount, for device pages. 28208763cb45SJérôme Glisse */ 28218763cb45SJérôme Glisse static void migrate_vma_finalize(struct migrate_vma *migrate) 28228763cb45SJérôme Glisse { 28238763cb45SJérôme Glisse const unsigned long npages = migrate->npages; 28248763cb45SJérôme Glisse unsigned long i; 28258763cb45SJérôme Glisse 28268763cb45SJérôme Glisse for (i = 0; i < npages; i++) { 28278763cb45SJérôme Glisse struct page *newpage = migrate_pfn_to_page(migrate->dst[i]); 28288763cb45SJérôme Glisse struct page *page = migrate_pfn_to_page(migrate->src[i]); 28298763cb45SJérôme Glisse 28308315ada7SJérôme Glisse if (!page) { 28318315ada7SJérôme Glisse if (newpage) { 28328315ada7SJérôme Glisse unlock_page(newpage); 28338315ada7SJérôme Glisse put_page(newpage); 28348315ada7SJérôme Glisse } 28358763cb45SJérôme Glisse continue; 28368315ada7SJérôme Glisse } 28378315ada7SJérôme Glisse 28388763cb45SJérôme Glisse if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE) || !newpage) { 28398763cb45SJérôme Glisse if (newpage) { 28408763cb45SJérôme Glisse unlock_page(newpage); 28418763cb45SJérôme Glisse put_page(newpage); 28428763cb45SJérôme Glisse } 28438763cb45SJérôme Glisse newpage = page; 28448763cb45SJérôme Glisse } 28458763cb45SJérôme Glisse 28468763cb45SJérôme Glisse remove_migration_ptes(page, newpage, false); 28478763cb45SJérôme Glisse unlock_page(page); 28488763cb45SJérôme Glisse migrate->cpages--; 28498763cb45SJérôme Glisse 2850a5430ddaSJérôme Glisse if (is_zone_device_page(page)) 2851a5430ddaSJérôme Glisse put_page(page); 2852a5430ddaSJérôme Glisse else 28538763cb45SJérôme Glisse putback_lru_page(page); 28548763cb45SJérôme Glisse 28558763cb45SJérôme Glisse if (newpage != page) { 28568763cb45SJérôme Glisse unlock_page(newpage); 2857a5430ddaSJérôme Glisse if (is_zone_device_page(newpage)) 2858a5430ddaSJérôme Glisse put_page(newpage); 2859a5430ddaSJérôme Glisse else 28608763cb45SJérôme Glisse putback_lru_page(newpage); 28618763cb45SJérôme Glisse } 28628763cb45SJérôme Glisse } 28638763cb45SJérôme Glisse } 28648763cb45SJérôme Glisse 28658763cb45SJérôme Glisse /* 28668763cb45SJérôme Glisse * migrate_vma() - migrate a range of memory inside vma 28678763cb45SJérôme Glisse * 28688763cb45SJérôme Glisse * @ops: migration callback for allocating destination memory and copying 28698763cb45SJérôme Glisse * @vma: virtual memory area containing the range to be migrated 28708763cb45SJérôme Glisse * @start: start address of the range to migrate (inclusive) 28718763cb45SJérôme Glisse * @end: end address of the range to migrate (exclusive) 28728763cb45SJérôme Glisse * @src: array of hmm_pfn_t containing source pfns 28738763cb45SJérôme Glisse * @dst: array of hmm_pfn_t containing destination pfns 28748763cb45SJérôme Glisse * @private: pointer passed back to each of the callback 28758763cb45SJérôme Glisse * Returns: 0 on success, error code otherwise 28768763cb45SJérôme Glisse * 28778763cb45SJérôme Glisse * This function tries to migrate a range of memory virtual address range, using 28788763cb45SJérôme Glisse * callbacks to allocate and copy memory from source to destination. First it 28798763cb45SJérôme Glisse * collects all the pages backing each virtual address in the range, saving this 28808763cb45SJérôme Glisse * inside the src array. Then it locks those pages and unmaps them. Once the pages 28818763cb45SJérôme Glisse * are locked and unmapped, it checks whether each page is pinned or not. Pages 28828763cb45SJérôme Glisse * that aren't pinned have the MIGRATE_PFN_MIGRATE flag set (by this function) 28838763cb45SJérôme Glisse * in the corresponding src array entry. It then restores any pages that are 28848763cb45SJérôme Glisse * pinned, by remapping and unlocking those pages. 28858763cb45SJérôme Glisse * 28868763cb45SJérôme Glisse * At this point it calls the alloc_and_copy() callback. For documentation on 28878763cb45SJérôme Glisse * what is expected from that callback, see struct migrate_vma_ops comments in 28888763cb45SJérôme Glisse * include/linux/migrate.h 28898763cb45SJérôme Glisse * 28908763cb45SJérôme Glisse * After the alloc_and_copy() callback, this function goes over each entry in 28918763cb45SJérôme Glisse * the src array that has the MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE flag 28928763cb45SJérôme Glisse * set. If the corresponding entry in dst array has MIGRATE_PFN_VALID flag set, 28938763cb45SJérôme Glisse * then the function tries to migrate struct page information from the source 28948763cb45SJérôme Glisse * struct page to the destination struct page. If it fails to migrate the struct 28958763cb45SJérôme Glisse * page information, then it clears the MIGRATE_PFN_MIGRATE flag in the src 28968763cb45SJérôme Glisse * array. 28978763cb45SJérôme Glisse * 28988763cb45SJérôme Glisse * At this point all successfully migrated pages have an entry in the src 28998763cb45SJérôme Glisse * array with MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE flag set and the dst 29008763cb45SJérôme Glisse * array entry with MIGRATE_PFN_VALID flag set. 29018763cb45SJérôme Glisse * 29028763cb45SJérôme Glisse * It then calls the finalize_and_map() callback. See comments for "struct 29038763cb45SJérôme Glisse * migrate_vma_ops", in include/linux/migrate.h for details about 29048763cb45SJérôme Glisse * finalize_and_map() behavior. 29058763cb45SJérôme Glisse * 29068763cb45SJérôme Glisse * After the finalize_and_map() callback, for successfully migrated pages, this 29078763cb45SJérôme Glisse * function updates the CPU page table to point to new pages, otherwise it 29088763cb45SJérôme Glisse * restores the CPU page table to point to the original source pages. 29098763cb45SJérôme Glisse * 29108763cb45SJérôme Glisse * Function returns 0 after the above steps, even if no pages were migrated 29118763cb45SJérôme Glisse * (The function only returns an error if any of the arguments are invalid.) 29128763cb45SJérôme Glisse * 29138763cb45SJérôme Glisse * Both src and dst array must be big enough for (end - start) >> PAGE_SHIFT 29148763cb45SJérôme Glisse * unsigned long entries. 29158763cb45SJérôme Glisse */ 29168763cb45SJérôme Glisse int migrate_vma(const struct migrate_vma_ops *ops, 29178763cb45SJérôme Glisse struct vm_area_struct *vma, 29188763cb45SJérôme Glisse unsigned long start, 29198763cb45SJérôme Glisse unsigned long end, 29208763cb45SJérôme Glisse unsigned long *src, 29218763cb45SJérôme Glisse unsigned long *dst, 29228763cb45SJérôme Glisse void *private) 29238763cb45SJérôme Glisse { 29248763cb45SJérôme Glisse struct migrate_vma migrate; 29258763cb45SJérôme Glisse 29268763cb45SJérôme Glisse /* Sanity check the arguments */ 29278763cb45SJérôme Glisse start &= PAGE_MASK; 29288763cb45SJérôme Glisse end &= PAGE_MASK; 29298763cb45SJérôme Glisse if (!vma || is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_SPECIAL)) 29308763cb45SJérôme Glisse return -EINVAL; 29318763cb45SJérôme Glisse if (start < vma->vm_start || start >= vma->vm_end) 29328763cb45SJérôme Glisse return -EINVAL; 29338763cb45SJérôme Glisse if (end <= vma->vm_start || end > vma->vm_end) 29348763cb45SJérôme Glisse return -EINVAL; 29358763cb45SJérôme Glisse if (!ops || !src || !dst || start >= end) 29368763cb45SJérôme Glisse return -EINVAL; 29378763cb45SJérôme Glisse 29388763cb45SJérôme Glisse memset(src, 0, sizeof(*src) * ((end - start) >> PAGE_SHIFT)); 29398763cb45SJérôme Glisse migrate.src = src; 29408763cb45SJérôme Glisse migrate.dst = dst; 29418763cb45SJérôme Glisse migrate.start = start; 29428763cb45SJérôme Glisse migrate.npages = 0; 29438763cb45SJérôme Glisse migrate.cpages = 0; 29448763cb45SJérôme Glisse migrate.end = end; 29458763cb45SJérôme Glisse migrate.vma = vma; 29468763cb45SJérôme Glisse 29478763cb45SJérôme Glisse /* Collect, and try to unmap source pages */ 29488763cb45SJérôme Glisse migrate_vma_collect(&migrate); 29498763cb45SJérôme Glisse if (!migrate.cpages) 29508763cb45SJérôme Glisse return 0; 29518763cb45SJérôme Glisse 29528763cb45SJérôme Glisse /* Lock and isolate page */ 29538763cb45SJérôme Glisse migrate_vma_prepare(&migrate); 29548763cb45SJérôme Glisse if (!migrate.cpages) 29558763cb45SJérôme Glisse return 0; 29568763cb45SJérôme Glisse 29578763cb45SJérôme Glisse /* Unmap pages */ 29588763cb45SJérôme Glisse migrate_vma_unmap(&migrate); 29598763cb45SJérôme Glisse if (!migrate.cpages) 29608763cb45SJérôme Glisse return 0; 29618763cb45SJérôme Glisse 29628763cb45SJérôme Glisse /* 29638763cb45SJérôme Glisse * At this point pages are locked and unmapped, and thus they have 29648763cb45SJérôme Glisse * stable content and can safely be copied to destination memory that 29658763cb45SJérôme Glisse * is allocated by the callback. 29668763cb45SJérôme Glisse * 29678763cb45SJérôme Glisse * Note that migration can fail in migrate_vma_struct_page() for each 29688763cb45SJérôme Glisse * individual page. 29698763cb45SJérôme Glisse */ 29708763cb45SJérôme Glisse ops->alloc_and_copy(vma, src, dst, start, end, private); 29718763cb45SJérôme Glisse 29728763cb45SJérôme Glisse /* This does the real migration of struct page */ 29738763cb45SJérôme Glisse migrate_vma_pages(&migrate); 29748763cb45SJérôme Glisse 29758763cb45SJérôme Glisse ops->finalize_and_map(vma, src, dst, start, end, private); 29768763cb45SJérôme Glisse 29778763cb45SJérôme Glisse /* Unlock and remap pages */ 29788763cb45SJérôme Glisse migrate_vma_finalize(&migrate); 29798763cb45SJérôme Glisse 29808763cb45SJérôme Glisse return 0; 29818763cb45SJérôme Glisse } 29828763cb45SJérôme Glisse EXPORT_SYMBOL(migrate_vma); 2983