1748446bbSMel Gorman /* 2748446bbSMel Gorman * linux/mm/compaction.c 3748446bbSMel Gorman * 4748446bbSMel Gorman * Memory compaction for the reduction of external fragmentation. Note that 5748446bbSMel Gorman * this heavily depends upon page migration to do all the real heavy 6748446bbSMel Gorman * lifting 7748446bbSMel Gorman * 8748446bbSMel Gorman * Copyright IBM Corp. 2007-2010 Mel Gorman <mel@csn.ul.ie> 9748446bbSMel Gorman */ 10698b1b30SVlastimil Babka #include <linux/cpu.h> 11748446bbSMel Gorman #include <linux/swap.h> 12748446bbSMel Gorman #include <linux/migrate.h> 13748446bbSMel Gorman #include <linux/compaction.h> 14748446bbSMel Gorman #include <linux/mm_inline.h> 15174cd4b1SIngo Molnar #include <linux/sched/signal.h> 16748446bbSMel Gorman #include <linux/backing-dev.h> 1776ab0f53SMel Gorman #include <linux/sysctl.h> 18ed4a6d7fSMel Gorman #include <linux/sysfs.h> 19194159fbSMinchan Kim #include <linux/page-isolation.h> 20b8c73fc2SAndrey Ryabinin #include <linux/kasan.h> 21698b1b30SVlastimil Babka #include <linux/kthread.h> 22698b1b30SVlastimil Babka #include <linux/freezer.h> 2383358eceSJoonsoo Kim #include <linux/page_owner.h> 24748446bbSMel Gorman #include "internal.h" 25748446bbSMel Gorman 26010fc29aSMinchan Kim #ifdef CONFIG_COMPACTION 27010fc29aSMinchan Kim static inline void count_compact_event(enum vm_event_item item) 28010fc29aSMinchan Kim { 29010fc29aSMinchan Kim count_vm_event(item); 30010fc29aSMinchan Kim } 31010fc29aSMinchan Kim 32010fc29aSMinchan Kim static inline void count_compact_events(enum vm_event_item item, long delta) 33010fc29aSMinchan Kim { 34010fc29aSMinchan Kim count_vm_events(item, delta); 35010fc29aSMinchan Kim } 36010fc29aSMinchan Kim #else 37010fc29aSMinchan Kim #define count_compact_event(item) do { } while (0) 38010fc29aSMinchan Kim #define count_compact_events(item, delta) do { } while (0) 39010fc29aSMinchan Kim #endif 40010fc29aSMinchan Kim 41ff9543fdSMichal Nazarewicz #if defined CONFIG_COMPACTION || defined CONFIG_CMA 42ff9543fdSMichal Nazarewicz 43b7aba698SMel Gorman #define CREATE_TRACE_POINTS 44b7aba698SMel Gorman #include <trace/events/compaction.h> 45b7aba698SMel Gorman 4606b6640aSVlastimil Babka #define block_start_pfn(pfn, order) round_down(pfn, 1UL << (order)) 4706b6640aSVlastimil Babka #define block_end_pfn(pfn, order) ALIGN((pfn) + 1, 1UL << (order)) 4806b6640aSVlastimil Babka #define pageblock_start_pfn(pfn) block_start_pfn(pfn, pageblock_order) 4906b6640aSVlastimil Babka #define pageblock_end_pfn(pfn) block_end_pfn(pfn, pageblock_order) 5006b6640aSVlastimil Babka 51748446bbSMel Gorman static unsigned long release_freepages(struct list_head *freelist) 52748446bbSMel Gorman { 53748446bbSMel Gorman struct page *page, *next; 546bace090SVlastimil Babka unsigned long high_pfn = 0; 55748446bbSMel Gorman 56748446bbSMel Gorman list_for_each_entry_safe(page, next, freelist, lru) { 576bace090SVlastimil Babka unsigned long pfn = page_to_pfn(page); 58748446bbSMel Gorman list_del(&page->lru); 59748446bbSMel Gorman __free_page(page); 606bace090SVlastimil Babka if (pfn > high_pfn) 616bace090SVlastimil Babka high_pfn = pfn; 62748446bbSMel Gorman } 63748446bbSMel Gorman 646bace090SVlastimil Babka return high_pfn; 65748446bbSMel Gorman } 66748446bbSMel Gorman 67ff9543fdSMichal Nazarewicz static void map_pages(struct list_head *list) 68ff9543fdSMichal Nazarewicz { 6966c64223SJoonsoo Kim unsigned int i, order, nr_pages; 7066c64223SJoonsoo Kim struct page *page, *next; 7166c64223SJoonsoo Kim LIST_HEAD(tmp_list); 72ff9543fdSMichal Nazarewicz 7366c64223SJoonsoo Kim list_for_each_entry_safe(page, next, list, lru) { 7466c64223SJoonsoo Kim list_del(&page->lru); 7566c64223SJoonsoo Kim 7666c64223SJoonsoo Kim order = page_private(page); 7766c64223SJoonsoo Kim nr_pages = 1 << order; 7866c64223SJoonsoo Kim 7946f24fd8SJoonsoo Kim post_alloc_hook(page, order, __GFP_MOVABLE); 8066c64223SJoonsoo Kim if (order) 8166c64223SJoonsoo Kim split_page(page, order); 8266c64223SJoonsoo Kim 8366c64223SJoonsoo Kim for (i = 0; i < nr_pages; i++) { 8466c64223SJoonsoo Kim list_add(&page->lru, &tmp_list); 8566c64223SJoonsoo Kim page++; 86ff9543fdSMichal Nazarewicz } 87ff9543fdSMichal Nazarewicz } 88ff9543fdSMichal Nazarewicz 8966c64223SJoonsoo Kim list_splice(&tmp_list, list); 9066c64223SJoonsoo Kim } 9166c64223SJoonsoo Kim 92bb13ffebSMel Gorman #ifdef CONFIG_COMPACTION 9324e2716fSJoonsoo Kim 94bda807d4SMinchan Kim int PageMovable(struct page *page) 95bda807d4SMinchan Kim { 96bda807d4SMinchan Kim struct address_space *mapping; 97bda807d4SMinchan Kim 98bda807d4SMinchan Kim VM_BUG_ON_PAGE(!PageLocked(page), page); 99bda807d4SMinchan Kim if (!__PageMovable(page)) 100bda807d4SMinchan Kim return 0; 101bda807d4SMinchan Kim 102bda807d4SMinchan Kim mapping = page_mapping(page); 103bda807d4SMinchan Kim if (mapping && mapping->a_ops && mapping->a_ops->isolate_page) 104bda807d4SMinchan Kim return 1; 105bda807d4SMinchan Kim 106bda807d4SMinchan Kim return 0; 107bda807d4SMinchan Kim } 108bda807d4SMinchan Kim EXPORT_SYMBOL(PageMovable); 109bda807d4SMinchan Kim 110bda807d4SMinchan Kim void __SetPageMovable(struct page *page, struct address_space *mapping) 111bda807d4SMinchan Kim { 112bda807d4SMinchan Kim VM_BUG_ON_PAGE(!PageLocked(page), page); 113bda807d4SMinchan Kim VM_BUG_ON_PAGE((unsigned long)mapping & PAGE_MAPPING_MOVABLE, page); 114bda807d4SMinchan Kim page->mapping = (void *)((unsigned long)mapping | PAGE_MAPPING_MOVABLE); 115bda807d4SMinchan Kim } 116bda807d4SMinchan Kim EXPORT_SYMBOL(__SetPageMovable); 117bda807d4SMinchan Kim 118bda807d4SMinchan Kim void __ClearPageMovable(struct page *page) 119bda807d4SMinchan Kim { 120bda807d4SMinchan Kim VM_BUG_ON_PAGE(!PageLocked(page), page); 121bda807d4SMinchan Kim VM_BUG_ON_PAGE(!PageMovable(page), page); 122bda807d4SMinchan Kim /* 123bda807d4SMinchan Kim * Clear registered address_space val with keeping PAGE_MAPPING_MOVABLE 124bda807d4SMinchan Kim * flag so that VM can catch up released page by driver after isolation. 125bda807d4SMinchan Kim * With it, VM migration doesn't try to put it back. 126bda807d4SMinchan Kim */ 127bda807d4SMinchan Kim page->mapping = (void *)((unsigned long)page->mapping & 128bda807d4SMinchan Kim PAGE_MAPPING_MOVABLE); 129bda807d4SMinchan Kim } 130bda807d4SMinchan Kim EXPORT_SYMBOL(__ClearPageMovable); 131bda807d4SMinchan Kim 13224e2716fSJoonsoo Kim /* Do not skip compaction more than 64 times */ 13324e2716fSJoonsoo Kim #define COMPACT_MAX_DEFER_SHIFT 6 13424e2716fSJoonsoo Kim 13524e2716fSJoonsoo Kim /* 13624e2716fSJoonsoo Kim * Compaction is deferred when compaction fails to result in a page 13724e2716fSJoonsoo Kim * allocation success. 1 << compact_defer_limit compactions are skipped up 13824e2716fSJoonsoo Kim * to a limit of 1 << COMPACT_MAX_DEFER_SHIFT 13924e2716fSJoonsoo Kim */ 14024e2716fSJoonsoo Kim void defer_compaction(struct zone *zone, int order) 14124e2716fSJoonsoo Kim { 14224e2716fSJoonsoo Kim zone->compact_considered = 0; 14324e2716fSJoonsoo Kim zone->compact_defer_shift++; 14424e2716fSJoonsoo Kim 14524e2716fSJoonsoo Kim if (order < zone->compact_order_failed) 14624e2716fSJoonsoo Kim zone->compact_order_failed = order; 14724e2716fSJoonsoo Kim 14824e2716fSJoonsoo Kim if (zone->compact_defer_shift > COMPACT_MAX_DEFER_SHIFT) 14924e2716fSJoonsoo Kim zone->compact_defer_shift = COMPACT_MAX_DEFER_SHIFT; 15024e2716fSJoonsoo Kim 15124e2716fSJoonsoo Kim trace_mm_compaction_defer_compaction(zone, order); 15224e2716fSJoonsoo Kim } 15324e2716fSJoonsoo Kim 15424e2716fSJoonsoo Kim /* Returns true if compaction should be skipped this time */ 15524e2716fSJoonsoo Kim bool compaction_deferred(struct zone *zone, int order) 15624e2716fSJoonsoo Kim { 15724e2716fSJoonsoo Kim unsigned long defer_limit = 1UL << zone->compact_defer_shift; 15824e2716fSJoonsoo Kim 15924e2716fSJoonsoo Kim if (order < zone->compact_order_failed) 16024e2716fSJoonsoo Kim return false; 16124e2716fSJoonsoo Kim 16224e2716fSJoonsoo Kim /* Avoid possible overflow */ 16324e2716fSJoonsoo Kim if (++zone->compact_considered > defer_limit) 16424e2716fSJoonsoo Kim zone->compact_considered = defer_limit; 16524e2716fSJoonsoo Kim 16624e2716fSJoonsoo Kim if (zone->compact_considered >= defer_limit) 16724e2716fSJoonsoo Kim return false; 16824e2716fSJoonsoo Kim 16924e2716fSJoonsoo Kim trace_mm_compaction_deferred(zone, order); 17024e2716fSJoonsoo Kim 17124e2716fSJoonsoo Kim return true; 17224e2716fSJoonsoo Kim } 17324e2716fSJoonsoo Kim 17424e2716fSJoonsoo Kim /* 17524e2716fSJoonsoo Kim * Update defer tracking counters after successful compaction of given order, 17624e2716fSJoonsoo Kim * which means an allocation either succeeded (alloc_success == true) or is 17724e2716fSJoonsoo Kim * expected to succeed. 17824e2716fSJoonsoo Kim */ 17924e2716fSJoonsoo Kim void compaction_defer_reset(struct zone *zone, int order, 18024e2716fSJoonsoo Kim bool alloc_success) 18124e2716fSJoonsoo Kim { 18224e2716fSJoonsoo Kim if (alloc_success) { 18324e2716fSJoonsoo Kim zone->compact_considered = 0; 18424e2716fSJoonsoo Kim zone->compact_defer_shift = 0; 18524e2716fSJoonsoo Kim } 18624e2716fSJoonsoo Kim if (order >= zone->compact_order_failed) 18724e2716fSJoonsoo Kim zone->compact_order_failed = order + 1; 18824e2716fSJoonsoo Kim 18924e2716fSJoonsoo Kim trace_mm_compaction_defer_reset(zone, order); 19024e2716fSJoonsoo Kim } 19124e2716fSJoonsoo Kim 19224e2716fSJoonsoo Kim /* Returns true if restarting compaction after many failures */ 19324e2716fSJoonsoo Kim bool compaction_restarting(struct zone *zone, int order) 19424e2716fSJoonsoo Kim { 19524e2716fSJoonsoo Kim if (order < zone->compact_order_failed) 19624e2716fSJoonsoo Kim return false; 19724e2716fSJoonsoo Kim 19824e2716fSJoonsoo Kim return zone->compact_defer_shift == COMPACT_MAX_DEFER_SHIFT && 19924e2716fSJoonsoo Kim zone->compact_considered >= 1UL << zone->compact_defer_shift; 20024e2716fSJoonsoo Kim } 20124e2716fSJoonsoo Kim 202bb13ffebSMel Gorman /* Returns true if the pageblock should be scanned for pages to isolate. */ 203bb13ffebSMel Gorman static inline bool isolation_suitable(struct compact_control *cc, 204bb13ffebSMel Gorman struct page *page) 205bb13ffebSMel Gorman { 206bb13ffebSMel Gorman if (cc->ignore_skip_hint) 207bb13ffebSMel Gorman return true; 208bb13ffebSMel Gorman 209bb13ffebSMel Gorman return !get_pageblock_skip(page); 210bb13ffebSMel Gorman } 211bb13ffebSMel Gorman 21202333641SVlastimil Babka static void reset_cached_positions(struct zone *zone) 21302333641SVlastimil Babka { 21402333641SVlastimil Babka zone->compact_cached_migrate_pfn[0] = zone->zone_start_pfn; 21502333641SVlastimil Babka zone->compact_cached_migrate_pfn[1] = zone->zone_start_pfn; 216623446e4SJoonsoo Kim zone->compact_cached_free_pfn = 21706b6640aSVlastimil Babka pageblock_start_pfn(zone_end_pfn(zone) - 1); 21802333641SVlastimil Babka } 21902333641SVlastimil Babka 220bb13ffebSMel Gorman /* 221bb13ffebSMel Gorman * This function is called to clear all cached information on pageblocks that 222bb13ffebSMel Gorman * should be skipped for page isolation when the migrate and free page scanner 223bb13ffebSMel Gorman * meet. 224bb13ffebSMel Gorman */ 22562997027SMel Gorman static void __reset_isolation_suitable(struct zone *zone) 226bb13ffebSMel Gorman { 227bb13ffebSMel Gorman unsigned long start_pfn = zone->zone_start_pfn; 228108bcc96SCody P Schafer unsigned long end_pfn = zone_end_pfn(zone); 229bb13ffebSMel Gorman unsigned long pfn; 230bb13ffebSMel Gorman 23162997027SMel Gorman zone->compact_blockskip_flush = false; 232bb13ffebSMel Gorman 233bb13ffebSMel Gorman /* Walk the zone and mark every pageblock as suitable for isolation */ 234bb13ffebSMel Gorman for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) { 235bb13ffebSMel Gorman struct page *page; 236bb13ffebSMel Gorman 237bb13ffebSMel Gorman cond_resched(); 238bb13ffebSMel Gorman 239ccbe1e4dSMichal Hocko page = pfn_to_online_page(pfn); 240ccbe1e4dSMichal Hocko if (!page) 241bb13ffebSMel Gorman continue; 242bb13ffebSMel Gorman if (zone != page_zone(page)) 243bb13ffebSMel Gorman continue; 244bb13ffebSMel Gorman 245bb13ffebSMel Gorman clear_pageblock_skip(page); 246bb13ffebSMel Gorman } 24702333641SVlastimil Babka 24802333641SVlastimil Babka reset_cached_positions(zone); 249bb13ffebSMel Gorman } 250bb13ffebSMel Gorman 25162997027SMel Gorman void reset_isolation_suitable(pg_data_t *pgdat) 25262997027SMel Gorman { 25362997027SMel Gorman int zoneid; 25462997027SMel Gorman 25562997027SMel Gorman for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) { 25662997027SMel Gorman struct zone *zone = &pgdat->node_zones[zoneid]; 25762997027SMel Gorman if (!populated_zone(zone)) 25862997027SMel Gorman continue; 25962997027SMel Gorman 26062997027SMel Gorman /* Only flush if a full compaction finished recently */ 26162997027SMel Gorman if (zone->compact_blockskip_flush) 26262997027SMel Gorman __reset_isolation_suitable(zone); 26362997027SMel Gorman } 26462997027SMel Gorman } 26562997027SMel Gorman 266bb13ffebSMel Gorman /* 267bb13ffebSMel Gorman * If no pages were isolated then mark this pageblock to be skipped in the 26862997027SMel Gorman * future. The information is later cleared by __reset_isolation_suitable(). 269bb13ffebSMel Gorman */ 270c89511abSMel Gorman static void update_pageblock_skip(struct compact_control *cc, 271c89511abSMel Gorman struct page *page, unsigned long nr_isolated, 272edc2ca61SVlastimil Babka bool migrate_scanner) 273bb13ffebSMel Gorman { 274c89511abSMel Gorman struct zone *zone = cc->zone; 27535979ef3SDavid Rientjes unsigned long pfn; 2766815bf3fSJoonsoo Kim 2776815bf3fSJoonsoo Kim if (cc->ignore_skip_hint) 2786815bf3fSJoonsoo Kim return; 2796815bf3fSJoonsoo Kim 280bb13ffebSMel Gorman if (!page) 281bb13ffebSMel Gorman return; 282bb13ffebSMel Gorman 28335979ef3SDavid Rientjes if (nr_isolated) 28435979ef3SDavid Rientjes return; 28535979ef3SDavid Rientjes 286bb13ffebSMel Gorman set_pageblock_skip(page); 287c89511abSMel Gorman 28835979ef3SDavid Rientjes pfn = page_to_pfn(page); 28935979ef3SDavid Rientjes 29035979ef3SDavid Rientjes /* Update where async and sync compaction should restart */ 291c89511abSMel Gorman if (migrate_scanner) { 29235979ef3SDavid Rientjes if (pfn > zone->compact_cached_migrate_pfn[0]) 29335979ef3SDavid Rientjes zone->compact_cached_migrate_pfn[0] = pfn; 294e0b9daebSDavid Rientjes if (cc->mode != MIGRATE_ASYNC && 295e0b9daebSDavid Rientjes pfn > zone->compact_cached_migrate_pfn[1]) 29635979ef3SDavid Rientjes zone->compact_cached_migrate_pfn[1] = pfn; 297c89511abSMel Gorman } else { 29835979ef3SDavid Rientjes if (pfn < zone->compact_cached_free_pfn) 299c89511abSMel Gorman zone->compact_cached_free_pfn = pfn; 300c89511abSMel Gorman } 301c89511abSMel Gorman } 302bb13ffebSMel Gorman #else 303bb13ffebSMel Gorman static inline bool isolation_suitable(struct compact_control *cc, 304bb13ffebSMel Gorman struct page *page) 305bb13ffebSMel Gorman { 306bb13ffebSMel Gorman return true; 307bb13ffebSMel Gorman } 308bb13ffebSMel Gorman 309c89511abSMel Gorman static void update_pageblock_skip(struct compact_control *cc, 310c89511abSMel Gorman struct page *page, unsigned long nr_isolated, 311edc2ca61SVlastimil Babka bool migrate_scanner) 312bb13ffebSMel Gorman { 313bb13ffebSMel Gorman } 314bb13ffebSMel Gorman #endif /* CONFIG_COMPACTION */ 315bb13ffebSMel Gorman 3161f9efdefSVlastimil Babka /* 3178b44d279SVlastimil Babka * Compaction requires the taking of some coarse locks that are potentially 3188b44d279SVlastimil Babka * very heavily contended. For async compaction, back out if the lock cannot 3198b44d279SVlastimil Babka * be taken immediately. For sync compaction, spin on the lock if needed. 3208b44d279SVlastimil Babka * 3218b44d279SVlastimil Babka * Returns true if the lock is held 3228b44d279SVlastimil Babka * Returns false if the lock is not held and compaction should abort 3231f9efdefSVlastimil Babka */ 3248b44d279SVlastimil Babka static bool compact_trylock_irqsave(spinlock_t *lock, unsigned long *flags, 3258b44d279SVlastimil Babka struct compact_control *cc) 3268b44d279SVlastimil Babka { 3278b44d279SVlastimil Babka if (cc->mode == MIGRATE_ASYNC) { 3288b44d279SVlastimil Babka if (!spin_trylock_irqsave(lock, *flags)) { 329c3486f53SVlastimil Babka cc->contended = true; 3308b44d279SVlastimil Babka return false; 3318b44d279SVlastimil Babka } 3328b44d279SVlastimil Babka } else { 3338b44d279SVlastimil Babka spin_lock_irqsave(lock, *flags); 3348b44d279SVlastimil Babka } 3351f9efdefSVlastimil Babka 3368b44d279SVlastimil Babka return true; 3372a1402aaSMel Gorman } 3382a1402aaSMel Gorman 33985aa125fSMichal Nazarewicz /* 340c67fe375SMel Gorman * Compaction requires the taking of some coarse locks that are potentially 3418b44d279SVlastimil Babka * very heavily contended. The lock should be periodically unlocked to avoid 3428b44d279SVlastimil Babka * having disabled IRQs for a long time, even when there is nobody waiting on 3438b44d279SVlastimil Babka * the lock. It might also be that allowing the IRQs will result in 3448b44d279SVlastimil Babka * need_resched() becoming true. If scheduling is needed, async compaction 3458b44d279SVlastimil Babka * aborts. Sync compaction schedules. 3468b44d279SVlastimil Babka * Either compaction type will also abort if a fatal signal is pending. 3478b44d279SVlastimil Babka * In either case if the lock was locked, it is dropped and not regained. 348c67fe375SMel Gorman * 3498b44d279SVlastimil Babka * Returns true if compaction should abort due to fatal signal pending, or 3508b44d279SVlastimil Babka * async compaction due to need_resched() 3518b44d279SVlastimil Babka * Returns false when compaction can continue (sync compaction might have 3528b44d279SVlastimil Babka * scheduled) 353c67fe375SMel Gorman */ 3548b44d279SVlastimil Babka static bool compact_unlock_should_abort(spinlock_t *lock, 3558b44d279SVlastimil Babka unsigned long flags, bool *locked, struct compact_control *cc) 356c67fe375SMel Gorman { 3578b44d279SVlastimil Babka if (*locked) { 3588b44d279SVlastimil Babka spin_unlock_irqrestore(lock, flags); 3598b44d279SVlastimil Babka *locked = false; 360c67fe375SMel Gorman } 361c67fe375SMel Gorman 3628b44d279SVlastimil Babka if (fatal_signal_pending(current)) { 363c3486f53SVlastimil Babka cc->contended = true; 3648b44d279SVlastimil Babka return true; 3658b44d279SVlastimil Babka } 3668b44d279SVlastimil Babka 3678b44d279SVlastimil Babka if (need_resched()) { 368e0b9daebSDavid Rientjes if (cc->mode == MIGRATE_ASYNC) { 369c3486f53SVlastimil Babka cc->contended = true; 3708b44d279SVlastimil Babka return true; 371c67fe375SMel Gorman } 372c67fe375SMel Gorman cond_resched(); 373c67fe375SMel Gorman } 374c67fe375SMel Gorman 3758b44d279SVlastimil Babka return false; 376c67fe375SMel Gorman } 377c67fe375SMel Gorman 378be976572SVlastimil Babka /* 379be976572SVlastimil Babka * Aside from avoiding lock contention, compaction also periodically checks 380be976572SVlastimil Babka * need_resched() and either schedules in sync compaction or aborts async 3818b44d279SVlastimil Babka * compaction. This is similar to what compact_unlock_should_abort() does, but 382be976572SVlastimil Babka * is used where no lock is concerned. 383be976572SVlastimil Babka * 384be976572SVlastimil Babka * Returns false when no scheduling was needed, or sync compaction scheduled. 385be976572SVlastimil Babka * Returns true when async compaction should abort. 386be976572SVlastimil Babka */ 387be976572SVlastimil Babka static inline bool compact_should_abort(struct compact_control *cc) 388be976572SVlastimil Babka { 389be976572SVlastimil Babka /* async compaction aborts if contended */ 390be976572SVlastimil Babka if (need_resched()) { 391be976572SVlastimil Babka if (cc->mode == MIGRATE_ASYNC) { 392c3486f53SVlastimil Babka cc->contended = true; 393be976572SVlastimil Babka return true; 394be976572SVlastimil Babka } 395be976572SVlastimil Babka 396be976572SVlastimil Babka cond_resched(); 397be976572SVlastimil Babka } 398be976572SVlastimil Babka 399be976572SVlastimil Babka return false; 400be976572SVlastimil Babka } 401be976572SVlastimil Babka 402c67fe375SMel Gorman /* 4039e4be470SJerome Marchand * Isolate free pages onto a private freelist. If @strict is true, will abort 4049e4be470SJerome Marchand * returning 0 on any invalid PFNs or non-free pages inside of the pageblock 4059e4be470SJerome Marchand * (even though it may still end up isolating some pages). 40685aa125fSMichal Nazarewicz */ 407f40d1e42SMel Gorman static unsigned long isolate_freepages_block(struct compact_control *cc, 408e14c720eSVlastimil Babka unsigned long *start_pfn, 40985aa125fSMichal Nazarewicz unsigned long end_pfn, 41085aa125fSMichal Nazarewicz struct list_head *freelist, 41185aa125fSMichal Nazarewicz bool strict) 412748446bbSMel Gorman { 413b7aba698SMel Gorman int nr_scanned = 0, total_isolated = 0; 414bb13ffebSMel Gorman struct page *cursor, *valid_page = NULL; 415b8b2d825SXiubo Li unsigned long flags = 0; 416f40d1e42SMel Gorman bool locked = false; 417e14c720eSVlastimil Babka unsigned long blockpfn = *start_pfn; 41866c64223SJoonsoo Kim unsigned int order; 419748446bbSMel Gorman 420748446bbSMel Gorman cursor = pfn_to_page(blockpfn); 421748446bbSMel Gorman 422f40d1e42SMel Gorman /* Isolate free pages. */ 423748446bbSMel Gorman for (; blockpfn < end_pfn; blockpfn++, cursor++) { 42466c64223SJoonsoo Kim int isolated; 425748446bbSMel Gorman struct page *page = cursor; 426748446bbSMel Gorman 4278b44d279SVlastimil Babka /* 4288b44d279SVlastimil Babka * Periodically drop the lock (if held) regardless of its 4298b44d279SVlastimil Babka * contention, to give chance to IRQs. Abort if fatal signal 4308b44d279SVlastimil Babka * pending or async compaction detects need_resched() 4318b44d279SVlastimil Babka */ 4328b44d279SVlastimil Babka if (!(blockpfn % SWAP_CLUSTER_MAX) 4338b44d279SVlastimil Babka && compact_unlock_should_abort(&cc->zone->lock, flags, 4348b44d279SVlastimil Babka &locked, cc)) 4358b44d279SVlastimil Babka break; 4368b44d279SVlastimil Babka 437b7aba698SMel Gorman nr_scanned++; 438f40d1e42SMel Gorman if (!pfn_valid_within(blockpfn)) 4392af120bcSLaura Abbott goto isolate_fail; 4402af120bcSLaura Abbott 441bb13ffebSMel Gorman if (!valid_page) 442bb13ffebSMel Gorman valid_page = page; 4439fcd6d2eSVlastimil Babka 4449fcd6d2eSVlastimil Babka /* 4459fcd6d2eSVlastimil Babka * For compound pages such as THP and hugetlbfs, we can save 4469fcd6d2eSVlastimil Babka * potentially a lot of iterations if we skip them at once. 4479fcd6d2eSVlastimil Babka * The check is racy, but we can consider only valid values 4489fcd6d2eSVlastimil Babka * and the only danger is skipping too much. 4499fcd6d2eSVlastimil Babka */ 4509fcd6d2eSVlastimil Babka if (PageCompound(page)) { 4519fcd6d2eSVlastimil Babka unsigned int comp_order = compound_order(page); 4529fcd6d2eSVlastimil Babka 4539fcd6d2eSVlastimil Babka if (likely(comp_order < MAX_ORDER)) { 4549fcd6d2eSVlastimil Babka blockpfn += (1UL << comp_order) - 1; 4559fcd6d2eSVlastimil Babka cursor += (1UL << comp_order) - 1; 4569fcd6d2eSVlastimil Babka } 4579fcd6d2eSVlastimil Babka 4589fcd6d2eSVlastimil Babka goto isolate_fail; 4599fcd6d2eSVlastimil Babka } 4609fcd6d2eSVlastimil Babka 461f40d1e42SMel Gorman if (!PageBuddy(page)) 4622af120bcSLaura Abbott goto isolate_fail; 463f40d1e42SMel Gorman 464f40d1e42SMel Gorman /* 46569b7189fSVlastimil Babka * If we already hold the lock, we can skip some rechecking. 46669b7189fSVlastimil Babka * Note that if we hold the lock now, checked_pageblock was 46769b7189fSVlastimil Babka * already set in some previous iteration (or strict is true), 46869b7189fSVlastimil Babka * so it is correct to skip the suitable migration target 46969b7189fSVlastimil Babka * recheck as well. 47069b7189fSVlastimil Babka */ 47169b7189fSVlastimil Babka if (!locked) { 47269b7189fSVlastimil Babka /* 473f40d1e42SMel Gorman * The zone lock must be held to isolate freepages. 474f40d1e42SMel Gorman * Unfortunately this is a very coarse lock and can be 475f40d1e42SMel Gorman * heavily contended if there are parallel allocations 476f40d1e42SMel Gorman * or parallel compactions. For async compaction do not 477f40d1e42SMel Gorman * spin on the lock and we acquire the lock as late as 478f40d1e42SMel Gorman * possible. 479f40d1e42SMel Gorman */ 4808b44d279SVlastimil Babka locked = compact_trylock_irqsave(&cc->zone->lock, 4818b44d279SVlastimil Babka &flags, cc); 482f40d1e42SMel Gorman if (!locked) 483f40d1e42SMel Gorman break; 484f40d1e42SMel Gorman 485f40d1e42SMel Gorman /* Recheck this is a buddy page under lock */ 486f40d1e42SMel Gorman if (!PageBuddy(page)) 4872af120bcSLaura Abbott goto isolate_fail; 48869b7189fSVlastimil Babka } 489748446bbSMel Gorman 49066c64223SJoonsoo Kim /* Found a free page, will break it into order-0 pages */ 49166c64223SJoonsoo Kim order = page_order(page); 49266c64223SJoonsoo Kim isolated = __isolate_free_page(page, order); 493a4f04f2cSDavid Rientjes if (!isolated) 494a4f04f2cSDavid Rientjes break; 49566c64223SJoonsoo Kim set_page_private(page, order); 496a4f04f2cSDavid Rientjes 497748446bbSMel Gorman total_isolated += isolated; 498a4f04f2cSDavid Rientjes cc->nr_freepages += isolated; 49966c64223SJoonsoo Kim list_add_tail(&page->lru, freelist); 50066c64223SJoonsoo Kim 501a4f04f2cSDavid Rientjes if (!strict && cc->nr_migratepages <= cc->nr_freepages) { 502932ff6bbSJoonsoo Kim blockpfn += isolated; 503932ff6bbSJoonsoo Kim break; 504932ff6bbSJoonsoo Kim } 505a4f04f2cSDavid Rientjes /* Advance to the end of split page */ 506748446bbSMel Gorman blockpfn += isolated - 1; 507748446bbSMel Gorman cursor += isolated - 1; 5082af120bcSLaura Abbott continue; 5092af120bcSLaura Abbott 5102af120bcSLaura Abbott isolate_fail: 5112af120bcSLaura Abbott if (strict) 5122af120bcSLaura Abbott break; 5132af120bcSLaura Abbott else 5142af120bcSLaura Abbott continue; 5152af120bcSLaura Abbott 516748446bbSMel Gorman } 517748446bbSMel Gorman 518a4f04f2cSDavid Rientjes if (locked) 519a4f04f2cSDavid Rientjes spin_unlock_irqrestore(&cc->zone->lock, flags); 520a4f04f2cSDavid Rientjes 5219fcd6d2eSVlastimil Babka /* 5229fcd6d2eSVlastimil Babka * There is a tiny chance that we have read bogus compound_order(), 5239fcd6d2eSVlastimil Babka * so be careful to not go outside of the pageblock. 5249fcd6d2eSVlastimil Babka */ 5259fcd6d2eSVlastimil Babka if (unlikely(blockpfn > end_pfn)) 5269fcd6d2eSVlastimil Babka blockpfn = end_pfn; 5279fcd6d2eSVlastimil Babka 528e34d85f0SJoonsoo Kim trace_mm_compaction_isolate_freepages(*start_pfn, blockpfn, 529e34d85f0SJoonsoo Kim nr_scanned, total_isolated); 530e34d85f0SJoonsoo Kim 531e14c720eSVlastimil Babka /* Record how far we have got within the block */ 532e14c720eSVlastimil Babka *start_pfn = blockpfn; 533e14c720eSVlastimil Babka 534f40d1e42SMel Gorman /* 535f40d1e42SMel Gorman * If strict isolation is requested by CMA then check that all the 536f40d1e42SMel Gorman * pages requested were isolated. If there were any failures, 0 is 537f40d1e42SMel Gorman * returned and CMA will fail. 538f40d1e42SMel Gorman */ 5392af120bcSLaura Abbott if (strict && blockpfn < end_pfn) 540f40d1e42SMel Gorman total_isolated = 0; 541f40d1e42SMel Gorman 542bb13ffebSMel Gorman /* Update the pageblock-skip if the whole pageblock was scanned */ 543bb13ffebSMel Gorman if (blockpfn == end_pfn) 544edc2ca61SVlastimil Babka update_pageblock_skip(cc, valid_page, total_isolated, false); 545bb13ffebSMel Gorman 5467f354a54SDavid Rientjes cc->total_free_scanned += nr_scanned; 547397487dbSMel Gorman if (total_isolated) 548010fc29aSMinchan Kim count_compact_events(COMPACTISOLATED, total_isolated); 549748446bbSMel Gorman return total_isolated; 550748446bbSMel Gorman } 551748446bbSMel Gorman 55285aa125fSMichal Nazarewicz /** 55385aa125fSMichal Nazarewicz * isolate_freepages_range() - isolate free pages. 55485aa125fSMichal Nazarewicz * @start_pfn: The first PFN to start isolating. 55585aa125fSMichal Nazarewicz * @end_pfn: The one-past-last PFN. 55685aa125fSMichal Nazarewicz * 55785aa125fSMichal Nazarewicz * Non-free pages, invalid PFNs, or zone boundaries within the 55885aa125fSMichal Nazarewicz * [start_pfn, end_pfn) range are considered errors, cause function to 55985aa125fSMichal Nazarewicz * undo its actions and return zero. 56085aa125fSMichal Nazarewicz * 56185aa125fSMichal Nazarewicz * Otherwise, function returns one-past-the-last PFN of isolated page 56285aa125fSMichal Nazarewicz * (which may be greater then end_pfn if end fell in a middle of 56385aa125fSMichal Nazarewicz * a free page). 56485aa125fSMichal Nazarewicz */ 565ff9543fdSMichal Nazarewicz unsigned long 566bb13ffebSMel Gorman isolate_freepages_range(struct compact_control *cc, 567bb13ffebSMel Gorman unsigned long start_pfn, unsigned long end_pfn) 56885aa125fSMichal Nazarewicz { 569e1409c32SJoonsoo Kim unsigned long isolated, pfn, block_start_pfn, block_end_pfn; 57085aa125fSMichal Nazarewicz LIST_HEAD(freelist); 57185aa125fSMichal Nazarewicz 5727d49d886SVlastimil Babka pfn = start_pfn; 57306b6640aSVlastimil Babka block_start_pfn = pageblock_start_pfn(pfn); 574e1409c32SJoonsoo Kim if (block_start_pfn < cc->zone->zone_start_pfn) 575e1409c32SJoonsoo Kim block_start_pfn = cc->zone->zone_start_pfn; 57606b6640aSVlastimil Babka block_end_pfn = pageblock_end_pfn(pfn); 5777d49d886SVlastimil Babka 5787d49d886SVlastimil Babka for (; pfn < end_pfn; pfn += isolated, 579e1409c32SJoonsoo Kim block_start_pfn = block_end_pfn, 5807d49d886SVlastimil Babka block_end_pfn += pageblock_nr_pages) { 581e14c720eSVlastimil Babka /* Protect pfn from changing by isolate_freepages_block */ 582e14c720eSVlastimil Babka unsigned long isolate_start_pfn = pfn; 5837d49d886SVlastimil Babka 58485aa125fSMichal Nazarewicz block_end_pfn = min(block_end_pfn, end_pfn); 58585aa125fSMichal Nazarewicz 58658420016SJoonsoo Kim /* 58758420016SJoonsoo Kim * pfn could pass the block_end_pfn if isolated freepage 58858420016SJoonsoo Kim * is more than pageblock order. In this case, we adjust 58958420016SJoonsoo Kim * scanning range to right one. 59058420016SJoonsoo Kim */ 59158420016SJoonsoo Kim if (pfn >= block_end_pfn) { 59206b6640aSVlastimil Babka block_start_pfn = pageblock_start_pfn(pfn); 59306b6640aSVlastimil Babka block_end_pfn = pageblock_end_pfn(pfn); 59458420016SJoonsoo Kim block_end_pfn = min(block_end_pfn, end_pfn); 59558420016SJoonsoo Kim } 59658420016SJoonsoo Kim 597e1409c32SJoonsoo Kim if (!pageblock_pfn_to_page(block_start_pfn, 598e1409c32SJoonsoo Kim block_end_pfn, cc->zone)) 5997d49d886SVlastimil Babka break; 6007d49d886SVlastimil Babka 601e14c720eSVlastimil Babka isolated = isolate_freepages_block(cc, &isolate_start_pfn, 602e14c720eSVlastimil Babka block_end_pfn, &freelist, true); 60385aa125fSMichal Nazarewicz 60485aa125fSMichal Nazarewicz /* 60585aa125fSMichal Nazarewicz * In strict mode, isolate_freepages_block() returns 0 if 60685aa125fSMichal Nazarewicz * there are any holes in the block (ie. invalid PFNs or 60785aa125fSMichal Nazarewicz * non-free pages). 60885aa125fSMichal Nazarewicz */ 60985aa125fSMichal Nazarewicz if (!isolated) 61085aa125fSMichal Nazarewicz break; 61185aa125fSMichal Nazarewicz 61285aa125fSMichal Nazarewicz /* 61385aa125fSMichal Nazarewicz * If we managed to isolate pages, it is always (1 << n) * 61485aa125fSMichal Nazarewicz * pageblock_nr_pages for some non-negative n. (Max order 61585aa125fSMichal Nazarewicz * page may span two pageblocks). 61685aa125fSMichal Nazarewicz */ 61785aa125fSMichal Nazarewicz } 61885aa125fSMichal Nazarewicz 61966c64223SJoonsoo Kim /* __isolate_free_page() does not map the pages */ 62085aa125fSMichal Nazarewicz map_pages(&freelist); 62185aa125fSMichal Nazarewicz 62285aa125fSMichal Nazarewicz if (pfn < end_pfn) { 62385aa125fSMichal Nazarewicz /* Loop terminated early, cleanup. */ 62485aa125fSMichal Nazarewicz release_freepages(&freelist); 62585aa125fSMichal Nazarewicz return 0; 62685aa125fSMichal Nazarewicz } 62785aa125fSMichal Nazarewicz 62885aa125fSMichal Nazarewicz /* We don't use freelists for anything. */ 62985aa125fSMichal Nazarewicz return pfn; 63085aa125fSMichal Nazarewicz } 63185aa125fSMichal Nazarewicz 632748446bbSMel Gorman /* Similar to reclaim, but different enough that they don't share logic */ 633748446bbSMel Gorman static bool too_many_isolated(struct zone *zone) 634748446bbSMel Gorman { 635bc693045SMinchan Kim unsigned long active, inactive, isolated; 636748446bbSMel Gorman 637599d0c95SMel Gorman inactive = node_page_state(zone->zone_pgdat, NR_INACTIVE_FILE) + 638599d0c95SMel Gorman node_page_state(zone->zone_pgdat, NR_INACTIVE_ANON); 639599d0c95SMel Gorman active = node_page_state(zone->zone_pgdat, NR_ACTIVE_FILE) + 640599d0c95SMel Gorman node_page_state(zone->zone_pgdat, NR_ACTIVE_ANON); 641599d0c95SMel Gorman isolated = node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE) + 642599d0c95SMel Gorman node_page_state(zone->zone_pgdat, NR_ISOLATED_ANON); 643748446bbSMel Gorman 644bc693045SMinchan Kim return isolated > (inactive + active) / 2; 645748446bbSMel Gorman } 646748446bbSMel Gorman 6472fe86e00SMichal Nazarewicz /** 648edc2ca61SVlastimil Babka * isolate_migratepages_block() - isolate all migrate-able pages within 649edc2ca61SVlastimil Babka * a single pageblock 6502fe86e00SMichal Nazarewicz * @cc: Compaction control structure. 651edc2ca61SVlastimil Babka * @low_pfn: The first PFN to isolate 652edc2ca61SVlastimil Babka * @end_pfn: The one-past-the-last PFN to isolate, within same pageblock 653edc2ca61SVlastimil Babka * @isolate_mode: Isolation mode to be used. 6542fe86e00SMichal Nazarewicz * 6552fe86e00SMichal Nazarewicz * Isolate all pages that can be migrated from the range specified by 656edc2ca61SVlastimil Babka * [low_pfn, end_pfn). The range is expected to be within same pageblock. 657edc2ca61SVlastimil Babka * Returns zero if there is a fatal signal pending, otherwise PFN of the 658edc2ca61SVlastimil Babka * first page that was not scanned (which may be both less, equal to or more 659edc2ca61SVlastimil Babka * than end_pfn). 6602fe86e00SMichal Nazarewicz * 661edc2ca61SVlastimil Babka * The pages are isolated on cc->migratepages list (not required to be empty), 662edc2ca61SVlastimil Babka * and cc->nr_migratepages is updated accordingly. The cc->migrate_pfn field 663edc2ca61SVlastimil Babka * is neither read nor updated. 664748446bbSMel Gorman */ 665edc2ca61SVlastimil Babka static unsigned long 666edc2ca61SVlastimil Babka isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, 667edc2ca61SVlastimil Babka unsigned long end_pfn, isolate_mode_t isolate_mode) 668748446bbSMel Gorman { 669edc2ca61SVlastimil Babka struct zone *zone = cc->zone; 670b7aba698SMel Gorman unsigned long nr_scanned = 0, nr_isolated = 0; 671fa9add64SHugh Dickins struct lruvec *lruvec; 672b8b2d825SXiubo Li unsigned long flags = 0; 6732a1402aaSMel Gorman bool locked = false; 674bb13ffebSMel Gorman struct page *page = NULL, *valid_page = NULL; 675e34d85f0SJoonsoo Kim unsigned long start_pfn = low_pfn; 676fdd048e1SVlastimil Babka bool skip_on_failure = false; 677fdd048e1SVlastimil Babka unsigned long next_skip_pfn = 0; 678748446bbSMel Gorman 679748446bbSMel Gorman /* 680748446bbSMel Gorman * Ensure that there are not too many pages isolated from the LRU 681748446bbSMel Gorman * list by either parallel reclaimers or compaction. If there are, 682748446bbSMel Gorman * delay for some time until fewer pages are isolated 683748446bbSMel Gorman */ 684748446bbSMel Gorman while (unlikely(too_many_isolated(zone))) { 685f9e35b3bSMel Gorman /* async migration should just abort */ 686e0b9daebSDavid Rientjes if (cc->mode == MIGRATE_ASYNC) 6872fe86e00SMichal Nazarewicz return 0; 688f9e35b3bSMel Gorman 689748446bbSMel Gorman congestion_wait(BLK_RW_ASYNC, HZ/10); 690748446bbSMel Gorman 691748446bbSMel Gorman if (fatal_signal_pending(current)) 6922fe86e00SMichal Nazarewicz return 0; 693748446bbSMel Gorman } 694748446bbSMel Gorman 695be976572SVlastimil Babka if (compact_should_abort(cc)) 696aeef4b83SDavid Rientjes return 0; 697aeef4b83SDavid Rientjes 698fdd048e1SVlastimil Babka if (cc->direct_compaction && (cc->mode == MIGRATE_ASYNC)) { 699fdd048e1SVlastimil Babka skip_on_failure = true; 700fdd048e1SVlastimil Babka next_skip_pfn = block_end_pfn(low_pfn, cc->order); 701fdd048e1SVlastimil Babka } 702fdd048e1SVlastimil Babka 703748446bbSMel Gorman /* Time to isolate some pages for migration */ 704748446bbSMel Gorman for (; low_pfn < end_pfn; low_pfn++) { 70529c0dde8SVlastimil Babka 706fdd048e1SVlastimil Babka if (skip_on_failure && low_pfn >= next_skip_pfn) { 707fdd048e1SVlastimil Babka /* 708fdd048e1SVlastimil Babka * We have isolated all migration candidates in the 709fdd048e1SVlastimil Babka * previous order-aligned block, and did not skip it due 710fdd048e1SVlastimil Babka * to failure. We should migrate the pages now and 711fdd048e1SVlastimil Babka * hopefully succeed compaction. 712fdd048e1SVlastimil Babka */ 713fdd048e1SVlastimil Babka if (nr_isolated) 714fdd048e1SVlastimil Babka break; 715fdd048e1SVlastimil Babka 716fdd048e1SVlastimil Babka /* 717fdd048e1SVlastimil Babka * We failed to isolate in the previous order-aligned 718fdd048e1SVlastimil Babka * block. Set the new boundary to the end of the 719fdd048e1SVlastimil Babka * current block. Note we can't simply increase 720fdd048e1SVlastimil Babka * next_skip_pfn by 1 << order, as low_pfn might have 721fdd048e1SVlastimil Babka * been incremented by a higher number due to skipping 722fdd048e1SVlastimil Babka * a compound or a high-order buddy page in the 723fdd048e1SVlastimil Babka * previous loop iteration. 724fdd048e1SVlastimil Babka */ 725fdd048e1SVlastimil Babka next_skip_pfn = block_end_pfn(low_pfn, cc->order); 726fdd048e1SVlastimil Babka } 727fdd048e1SVlastimil Babka 7288b44d279SVlastimil Babka /* 7298b44d279SVlastimil Babka * Periodically drop the lock (if held) regardless of its 7308b44d279SVlastimil Babka * contention, to give chance to IRQs. Abort async compaction 7318b44d279SVlastimil Babka * if contended. 7328b44d279SVlastimil Babka */ 7338b44d279SVlastimil Babka if (!(low_pfn % SWAP_CLUSTER_MAX) 734a52633d8SMel Gorman && compact_unlock_should_abort(zone_lru_lock(zone), flags, 7358b44d279SVlastimil Babka &locked, cc)) 7368b44d279SVlastimil Babka break; 737b2eef8c0SAndrea Arcangeli 738748446bbSMel Gorman if (!pfn_valid_within(low_pfn)) 739fdd048e1SVlastimil Babka goto isolate_fail; 740b7aba698SMel Gorman nr_scanned++; 741748446bbSMel Gorman 742748446bbSMel Gorman page = pfn_to_page(low_pfn); 743dc908600SMel Gorman 744bb13ffebSMel Gorman if (!valid_page) 745bb13ffebSMel Gorman valid_page = page; 746bb13ffebSMel Gorman 747c122b208SJoonsoo Kim /* 74899c0fd5eSVlastimil Babka * Skip if free. We read page order here without zone lock 74999c0fd5eSVlastimil Babka * which is generally unsafe, but the race window is small and 75099c0fd5eSVlastimil Babka * the worst thing that can happen is that we skip some 75199c0fd5eSVlastimil Babka * potential isolation targets. 7526c14466cSMel Gorman */ 75399c0fd5eSVlastimil Babka if (PageBuddy(page)) { 75499c0fd5eSVlastimil Babka unsigned long freepage_order = page_order_unsafe(page); 75599c0fd5eSVlastimil Babka 75699c0fd5eSVlastimil Babka /* 75799c0fd5eSVlastimil Babka * Without lock, we cannot be sure that what we got is 75899c0fd5eSVlastimil Babka * a valid page order. Consider only values in the 75999c0fd5eSVlastimil Babka * valid order range to prevent low_pfn overflow. 76099c0fd5eSVlastimil Babka */ 76199c0fd5eSVlastimil Babka if (freepage_order > 0 && freepage_order < MAX_ORDER) 76299c0fd5eSVlastimil Babka low_pfn += (1UL << freepage_order) - 1; 763748446bbSMel Gorman continue; 76499c0fd5eSVlastimil Babka } 765748446bbSMel Gorman 7669927af74SMel Gorman /* 76729c0dde8SVlastimil Babka * Regardless of being on LRU, compound pages such as THP and 76829c0dde8SVlastimil Babka * hugetlbfs are not to be compacted. We can potentially save 76929c0dde8SVlastimil Babka * a lot of iterations if we skip them at once. The check is 77029c0dde8SVlastimil Babka * racy, but we can consider only valid values and the only 77129c0dde8SVlastimil Babka * danger is skipping too much. 772bc835011SAndrea Arcangeli */ 77329c0dde8SVlastimil Babka if (PageCompound(page)) { 77429c0dde8SVlastimil Babka unsigned int comp_order = compound_order(page); 77529c0dde8SVlastimil Babka 77629c0dde8SVlastimil Babka if (likely(comp_order < MAX_ORDER)) 77729c0dde8SVlastimil Babka low_pfn += (1UL << comp_order) - 1; 778edc2ca61SVlastimil Babka 779fdd048e1SVlastimil Babka goto isolate_fail; 7802a1402aaSMel Gorman } 7812a1402aaSMel Gorman 782bda807d4SMinchan Kim /* 783bda807d4SMinchan Kim * Check may be lockless but that's ok as we recheck later. 784bda807d4SMinchan Kim * It's possible to migrate LRU and non-lru movable pages. 785bda807d4SMinchan Kim * Skip any other type of page 786bda807d4SMinchan Kim */ 787bda807d4SMinchan Kim if (!PageLRU(page)) { 788bda807d4SMinchan Kim /* 789bda807d4SMinchan Kim * __PageMovable can return false positive so we need 790bda807d4SMinchan Kim * to verify it under page_lock. 791bda807d4SMinchan Kim */ 792bda807d4SMinchan Kim if (unlikely(__PageMovable(page)) && 793bda807d4SMinchan Kim !PageIsolated(page)) { 794bda807d4SMinchan Kim if (locked) { 795a52633d8SMel Gorman spin_unlock_irqrestore(zone_lru_lock(zone), 796bda807d4SMinchan Kim flags); 797bda807d4SMinchan Kim locked = false; 798bda807d4SMinchan Kim } 799bda807d4SMinchan Kim 8009e5bcd61SYisheng Xie if (!isolate_movable_page(page, isolate_mode)) 801bda807d4SMinchan Kim goto isolate_success; 802bda807d4SMinchan Kim } 803bda807d4SMinchan Kim 804fdd048e1SVlastimil Babka goto isolate_fail; 805bda807d4SMinchan Kim } 80629c0dde8SVlastimil Babka 807119d6d59SDavid Rientjes /* 808119d6d59SDavid Rientjes * Migration will fail if an anonymous page is pinned in memory, 809119d6d59SDavid Rientjes * so avoid taking lru_lock and isolating it unnecessarily in an 810119d6d59SDavid Rientjes * admittedly racy check. 811119d6d59SDavid Rientjes */ 812119d6d59SDavid Rientjes if (!page_mapping(page) && 813119d6d59SDavid Rientjes page_count(page) > page_mapcount(page)) 814fdd048e1SVlastimil Babka goto isolate_fail; 815119d6d59SDavid Rientjes 81673e64c51SMichal Hocko /* 81773e64c51SMichal Hocko * Only allow to migrate anonymous pages in GFP_NOFS context 81873e64c51SMichal Hocko * because those do not depend on fs locks. 81973e64c51SMichal Hocko */ 82073e64c51SMichal Hocko if (!(cc->gfp_mask & __GFP_FS) && page_mapping(page)) 82173e64c51SMichal Hocko goto isolate_fail; 82273e64c51SMichal Hocko 82369b7189fSVlastimil Babka /* If we already hold the lock, we can skip some rechecking */ 82469b7189fSVlastimil Babka if (!locked) { 825a52633d8SMel Gorman locked = compact_trylock_irqsave(zone_lru_lock(zone), 8268b44d279SVlastimil Babka &flags, cc); 8278b44d279SVlastimil Babka if (!locked) 8282a1402aaSMel Gorman break; 8292a1402aaSMel Gorman 83029c0dde8SVlastimil Babka /* Recheck PageLRU and PageCompound under lock */ 8312a1402aaSMel Gorman if (!PageLRU(page)) 832fdd048e1SVlastimil Babka goto isolate_fail; 83329c0dde8SVlastimil Babka 83429c0dde8SVlastimil Babka /* 83529c0dde8SVlastimil Babka * Page become compound since the non-locked check, 83629c0dde8SVlastimil Babka * and it's on LRU. It can only be a THP so the order 83729c0dde8SVlastimil Babka * is safe to read and it's 0 for tail pages. 83829c0dde8SVlastimil Babka */ 83929c0dde8SVlastimil Babka if (unlikely(PageCompound(page))) { 84029c0dde8SVlastimil Babka low_pfn += (1UL << compound_order(page)) - 1; 841fdd048e1SVlastimil Babka goto isolate_fail; 842bc835011SAndrea Arcangeli } 84369b7189fSVlastimil Babka } 844bc835011SAndrea Arcangeli 845599d0c95SMel Gorman lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat); 846fa9add64SHugh Dickins 847748446bbSMel Gorman /* Try isolate the page */ 848edc2ca61SVlastimil Babka if (__isolate_lru_page(page, isolate_mode) != 0) 849fdd048e1SVlastimil Babka goto isolate_fail; 850748446bbSMel Gorman 85129c0dde8SVlastimil Babka VM_BUG_ON_PAGE(PageCompound(page), page); 852bc835011SAndrea Arcangeli 853748446bbSMel Gorman /* Successfully isolated */ 854fa9add64SHugh Dickins del_page_from_lru_list(page, lruvec, page_lru(page)); 8556afcf8efSMing Ling inc_node_page_state(page, 8566afcf8efSMing Ling NR_ISOLATED_ANON + page_is_file_cache(page)); 857b6c75016SJoonsoo Kim 858b6c75016SJoonsoo Kim isolate_success: 859fdd048e1SVlastimil Babka list_add(&page->lru, &cc->migratepages); 860748446bbSMel Gorman cc->nr_migratepages++; 861b7aba698SMel Gorman nr_isolated++; 862748446bbSMel Gorman 863a34753d2SVlastimil Babka /* 864a34753d2SVlastimil Babka * Record where we could have freed pages by migration and not 865a34753d2SVlastimil Babka * yet flushed them to buddy allocator. 866a34753d2SVlastimil Babka * - this is the lowest page that was isolated and likely be 867a34753d2SVlastimil Babka * then freed by migration. 868a34753d2SVlastimil Babka */ 869a34753d2SVlastimil Babka if (!cc->last_migrated_pfn) 870a34753d2SVlastimil Babka cc->last_migrated_pfn = low_pfn; 871a34753d2SVlastimil Babka 872748446bbSMel Gorman /* Avoid isolating too much */ 87331b8384aSHillf Danton if (cc->nr_migratepages == COMPACT_CLUSTER_MAX) { 87431b8384aSHillf Danton ++low_pfn; 875748446bbSMel Gorman break; 876748446bbSMel Gorman } 877fdd048e1SVlastimil Babka 878fdd048e1SVlastimil Babka continue; 879fdd048e1SVlastimil Babka isolate_fail: 880fdd048e1SVlastimil Babka if (!skip_on_failure) 881fdd048e1SVlastimil Babka continue; 882fdd048e1SVlastimil Babka 883fdd048e1SVlastimil Babka /* 884fdd048e1SVlastimil Babka * We have isolated some pages, but then failed. Release them 885fdd048e1SVlastimil Babka * instead of migrating, as we cannot form the cc->order buddy 886fdd048e1SVlastimil Babka * page anyway. 887fdd048e1SVlastimil Babka */ 888fdd048e1SVlastimil Babka if (nr_isolated) { 889fdd048e1SVlastimil Babka if (locked) { 890a52633d8SMel Gorman spin_unlock_irqrestore(zone_lru_lock(zone), flags); 891fdd048e1SVlastimil Babka locked = false; 892fdd048e1SVlastimil Babka } 893fdd048e1SVlastimil Babka putback_movable_pages(&cc->migratepages); 894fdd048e1SVlastimil Babka cc->nr_migratepages = 0; 895fdd048e1SVlastimil Babka cc->last_migrated_pfn = 0; 896fdd048e1SVlastimil Babka nr_isolated = 0; 897fdd048e1SVlastimil Babka } 898fdd048e1SVlastimil Babka 899fdd048e1SVlastimil Babka if (low_pfn < next_skip_pfn) { 900fdd048e1SVlastimil Babka low_pfn = next_skip_pfn - 1; 901fdd048e1SVlastimil Babka /* 902fdd048e1SVlastimil Babka * The check near the loop beginning would have updated 903fdd048e1SVlastimil Babka * next_skip_pfn too, but this is a bit simpler. 904fdd048e1SVlastimil Babka */ 905fdd048e1SVlastimil Babka next_skip_pfn += 1UL << cc->order; 906fdd048e1SVlastimil Babka } 90731b8384aSHillf Danton } 908748446bbSMel Gorman 90999c0fd5eSVlastimil Babka /* 91099c0fd5eSVlastimil Babka * The PageBuddy() check could have potentially brought us outside 91199c0fd5eSVlastimil Babka * the range to be scanned. 91299c0fd5eSVlastimil Babka */ 91399c0fd5eSVlastimil Babka if (unlikely(low_pfn > end_pfn)) 91499c0fd5eSVlastimil Babka low_pfn = end_pfn; 91599c0fd5eSVlastimil Babka 916c67fe375SMel Gorman if (locked) 917a52633d8SMel Gorman spin_unlock_irqrestore(zone_lru_lock(zone), flags); 918748446bbSMel Gorman 91950b5b094SVlastimil Babka /* 92050b5b094SVlastimil Babka * Update the pageblock-skip information and cached scanner pfn, 92150b5b094SVlastimil Babka * if the whole pageblock was scanned without isolating any page. 92250b5b094SVlastimil Babka */ 92335979ef3SDavid Rientjes if (low_pfn == end_pfn) 924edc2ca61SVlastimil Babka update_pageblock_skip(cc, valid_page, nr_isolated, true); 925bb13ffebSMel Gorman 926e34d85f0SJoonsoo Kim trace_mm_compaction_isolate_migratepages(start_pfn, low_pfn, 927e34d85f0SJoonsoo Kim nr_scanned, nr_isolated); 928b7aba698SMel Gorman 9297f354a54SDavid Rientjes cc->total_migrate_scanned += nr_scanned; 930397487dbSMel Gorman if (nr_isolated) 931010fc29aSMinchan Kim count_compact_events(COMPACTISOLATED, nr_isolated); 932397487dbSMel Gorman 9332fe86e00SMichal Nazarewicz return low_pfn; 9342fe86e00SMichal Nazarewicz } 9352fe86e00SMichal Nazarewicz 936edc2ca61SVlastimil Babka /** 937edc2ca61SVlastimil Babka * isolate_migratepages_range() - isolate migrate-able pages in a PFN range 938edc2ca61SVlastimil Babka * @cc: Compaction control structure. 939edc2ca61SVlastimil Babka * @start_pfn: The first PFN to start isolating. 940edc2ca61SVlastimil Babka * @end_pfn: The one-past-last PFN. 941edc2ca61SVlastimil Babka * 942edc2ca61SVlastimil Babka * Returns zero if isolation fails fatally due to e.g. pending signal. 943edc2ca61SVlastimil Babka * Otherwise, function returns one-past-the-last PFN of isolated page 944edc2ca61SVlastimil Babka * (which may be greater than end_pfn if end fell in a middle of a THP page). 945edc2ca61SVlastimil Babka */ 946edc2ca61SVlastimil Babka unsigned long 947edc2ca61SVlastimil Babka isolate_migratepages_range(struct compact_control *cc, unsigned long start_pfn, 948edc2ca61SVlastimil Babka unsigned long end_pfn) 949edc2ca61SVlastimil Babka { 950e1409c32SJoonsoo Kim unsigned long pfn, block_start_pfn, block_end_pfn; 951edc2ca61SVlastimil Babka 952edc2ca61SVlastimil Babka /* Scan block by block. First and last block may be incomplete */ 953edc2ca61SVlastimil Babka pfn = start_pfn; 95406b6640aSVlastimil Babka block_start_pfn = pageblock_start_pfn(pfn); 955e1409c32SJoonsoo Kim if (block_start_pfn < cc->zone->zone_start_pfn) 956e1409c32SJoonsoo Kim block_start_pfn = cc->zone->zone_start_pfn; 95706b6640aSVlastimil Babka block_end_pfn = pageblock_end_pfn(pfn); 958edc2ca61SVlastimil Babka 959edc2ca61SVlastimil Babka for (; pfn < end_pfn; pfn = block_end_pfn, 960e1409c32SJoonsoo Kim block_start_pfn = block_end_pfn, 961edc2ca61SVlastimil Babka block_end_pfn += pageblock_nr_pages) { 962edc2ca61SVlastimil Babka 963edc2ca61SVlastimil Babka block_end_pfn = min(block_end_pfn, end_pfn); 964edc2ca61SVlastimil Babka 965e1409c32SJoonsoo Kim if (!pageblock_pfn_to_page(block_start_pfn, 966e1409c32SJoonsoo Kim block_end_pfn, cc->zone)) 967edc2ca61SVlastimil Babka continue; 968edc2ca61SVlastimil Babka 969edc2ca61SVlastimil Babka pfn = isolate_migratepages_block(cc, pfn, block_end_pfn, 970edc2ca61SVlastimil Babka ISOLATE_UNEVICTABLE); 971edc2ca61SVlastimil Babka 97214af4a5eSHugh Dickins if (!pfn) 973edc2ca61SVlastimil Babka break; 9746ea41c0cSJoonsoo Kim 9756ea41c0cSJoonsoo Kim if (cc->nr_migratepages == COMPACT_CLUSTER_MAX) 9766ea41c0cSJoonsoo Kim break; 977edc2ca61SVlastimil Babka } 978edc2ca61SVlastimil Babka 979edc2ca61SVlastimil Babka return pfn; 980edc2ca61SVlastimil Babka } 981edc2ca61SVlastimil Babka 982ff9543fdSMichal Nazarewicz #endif /* CONFIG_COMPACTION || CONFIG_CMA */ 983ff9543fdSMichal Nazarewicz #ifdef CONFIG_COMPACTION 984018e9a49SAndrew Morton 985b682debdSVlastimil Babka static bool suitable_migration_source(struct compact_control *cc, 986b682debdSVlastimil Babka struct page *page) 987b682debdSVlastimil Babka { 988282722b0SVlastimil Babka int block_mt; 989282722b0SVlastimil Babka 990282722b0SVlastimil Babka if ((cc->mode != MIGRATE_ASYNC) || !cc->direct_compaction) 991b682debdSVlastimil Babka return true; 992b682debdSVlastimil Babka 993282722b0SVlastimil Babka block_mt = get_pageblock_migratetype(page); 994282722b0SVlastimil Babka 995282722b0SVlastimil Babka if (cc->migratetype == MIGRATE_MOVABLE) 996282722b0SVlastimil Babka return is_migrate_movable(block_mt); 997282722b0SVlastimil Babka else 998282722b0SVlastimil Babka return block_mt == cc->migratetype; 999b682debdSVlastimil Babka } 1000b682debdSVlastimil Babka 1001018e9a49SAndrew Morton /* Returns true if the page is within a block suitable for migration to */ 10029f7e3387SVlastimil Babka static bool suitable_migration_target(struct compact_control *cc, 10039f7e3387SVlastimil Babka struct page *page) 1004018e9a49SAndrew Morton { 1005018e9a49SAndrew Morton /* If the page is a large free page, then disallow migration */ 1006018e9a49SAndrew Morton if (PageBuddy(page)) { 1007018e9a49SAndrew Morton /* 1008018e9a49SAndrew Morton * We are checking page_order without zone->lock taken. But 1009018e9a49SAndrew Morton * the only small danger is that we skip a potentially suitable 1010018e9a49SAndrew Morton * pageblock, so it's not worth to check order for valid range. 1011018e9a49SAndrew Morton */ 1012018e9a49SAndrew Morton if (page_order_unsafe(page) >= pageblock_order) 1013018e9a49SAndrew Morton return false; 1014018e9a49SAndrew Morton } 1015018e9a49SAndrew Morton 10161ef36db2SYisheng Xie if (cc->ignore_block_suitable) 10171ef36db2SYisheng Xie return true; 10181ef36db2SYisheng Xie 1019018e9a49SAndrew Morton /* If the block is MIGRATE_MOVABLE or MIGRATE_CMA, allow migration */ 1020b682debdSVlastimil Babka if (is_migrate_movable(get_pageblock_migratetype(page))) 1021018e9a49SAndrew Morton return true; 1022018e9a49SAndrew Morton 1023018e9a49SAndrew Morton /* Otherwise skip the block */ 1024018e9a49SAndrew Morton return false; 1025018e9a49SAndrew Morton } 1026018e9a49SAndrew Morton 1027ff9543fdSMichal Nazarewicz /* 1028f2849aa0SVlastimil Babka * Test whether the free scanner has reached the same or lower pageblock than 1029f2849aa0SVlastimil Babka * the migration scanner, and compaction should thus terminate. 1030f2849aa0SVlastimil Babka */ 1031f2849aa0SVlastimil Babka static inline bool compact_scanners_met(struct compact_control *cc) 1032f2849aa0SVlastimil Babka { 1033f2849aa0SVlastimil Babka return (cc->free_pfn >> pageblock_order) 1034f2849aa0SVlastimil Babka <= (cc->migrate_pfn >> pageblock_order); 1035f2849aa0SVlastimil Babka } 1036f2849aa0SVlastimil Babka 1037f2849aa0SVlastimil Babka /* 1038ff9543fdSMichal Nazarewicz * Based on information in the current compact_control, find blocks 1039ff9543fdSMichal Nazarewicz * suitable for isolating free pages from and then isolate them. 1040ff9543fdSMichal Nazarewicz */ 1041edc2ca61SVlastimil Babka static void isolate_freepages(struct compact_control *cc) 1042ff9543fdSMichal Nazarewicz { 1043edc2ca61SVlastimil Babka struct zone *zone = cc->zone; 1044ff9543fdSMichal Nazarewicz struct page *page; 1045c96b9e50SVlastimil Babka unsigned long block_start_pfn; /* start of current pageblock */ 1046e14c720eSVlastimil Babka unsigned long isolate_start_pfn; /* exact pfn we start at */ 1047c96b9e50SVlastimil Babka unsigned long block_end_pfn; /* end of current pageblock */ 1048c96b9e50SVlastimil Babka unsigned long low_pfn; /* lowest pfn scanner is able to scan */ 1049ff9543fdSMichal Nazarewicz struct list_head *freelist = &cc->freepages; 10502fe86e00SMichal Nazarewicz 1051ff9543fdSMichal Nazarewicz /* 1052ff9543fdSMichal Nazarewicz * Initialise the free scanner. The starting point is where we last 105349e068f0SVlastimil Babka * successfully isolated from, zone-cached value, or the end of the 1054e14c720eSVlastimil Babka * zone when isolating for the first time. For looping we also need 1055e14c720eSVlastimil Babka * this pfn aligned down to the pageblock boundary, because we do 1056c96b9e50SVlastimil Babka * block_start_pfn -= pageblock_nr_pages in the for loop. 1057c96b9e50SVlastimil Babka * For ending point, take care when isolating in last pageblock of a 1058c96b9e50SVlastimil Babka * a zone which ends in the middle of a pageblock. 105949e068f0SVlastimil Babka * The low boundary is the end of the pageblock the migration scanner 106049e068f0SVlastimil Babka * is using. 1061ff9543fdSMichal Nazarewicz */ 1062e14c720eSVlastimil Babka isolate_start_pfn = cc->free_pfn; 106306b6640aSVlastimil Babka block_start_pfn = pageblock_start_pfn(cc->free_pfn); 1064c96b9e50SVlastimil Babka block_end_pfn = min(block_start_pfn + pageblock_nr_pages, 1065c96b9e50SVlastimil Babka zone_end_pfn(zone)); 106606b6640aSVlastimil Babka low_pfn = pageblock_end_pfn(cc->migrate_pfn); 10672fe86e00SMichal Nazarewicz 1068ff9543fdSMichal Nazarewicz /* 1069ff9543fdSMichal Nazarewicz * Isolate free pages until enough are available to migrate the 1070ff9543fdSMichal Nazarewicz * pages on cc->migratepages. We stop searching if the migrate 1071ff9543fdSMichal Nazarewicz * and free page scanners meet or enough free pages are isolated. 1072ff9543fdSMichal Nazarewicz */ 1073f5f61a32SVlastimil Babka for (; block_start_pfn >= low_pfn; 1074c96b9e50SVlastimil Babka block_end_pfn = block_start_pfn, 1075e14c720eSVlastimil Babka block_start_pfn -= pageblock_nr_pages, 1076e14c720eSVlastimil Babka isolate_start_pfn = block_start_pfn) { 1077f6ea3adbSDavid Rientjes /* 1078f6ea3adbSDavid Rientjes * This can iterate a massively long zone without finding any 1079f6ea3adbSDavid Rientjes * suitable migration targets, so periodically check if we need 1080be976572SVlastimil Babka * to schedule, or even abort async compaction. 1081f6ea3adbSDavid Rientjes */ 1082be976572SVlastimil Babka if (!(block_start_pfn % (SWAP_CLUSTER_MAX * pageblock_nr_pages)) 1083be976572SVlastimil Babka && compact_should_abort(cc)) 1084be976572SVlastimil Babka break; 1085f6ea3adbSDavid Rientjes 10867d49d886SVlastimil Babka page = pageblock_pfn_to_page(block_start_pfn, block_end_pfn, 10877d49d886SVlastimil Babka zone); 10887d49d886SVlastimil Babka if (!page) 1089ff9543fdSMichal Nazarewicz continue; 1090ff9543fdSMichal Nazarewicz 1091ff9543fdSMichal Nazarewicz /* Check the block is suitable for migration */ 10929f7e3387SVlastimil Babka if (!suitable_migration_target(cc, page)) 1093ff9543fdSMichal Nazarewicz continue; 109468e3e926SLinus Torvalds 1095bb13ffebSMel Gorman /* If isolation recently failed, do not retry */ 1096bb13ffebSMel Gorman if (!isolation_suitable(cc, page)) 1097bb13ffebSMel Gorman continue; 1098bb13ffebSMel Gorman 1099e14c720eSVlastimil Babka /* Found a block suitable for isolating free pages from. */ 1100a46cbf3bSDavid Rientjes isolate_freepages_block(cc, &isolate_start_pfn, block_end_pfn, 1101a46cbf3bSDavid Rientjes freelist, false); 1102ff9543fdSMichal Nazarewicz 1103ff9543fdSMichal Nazarewicz /* 1104a46cbf3bSDavid Rientjes * If we isolated enough freepages, or aborted due to lock 1105a46cbf3bSDavid Rientjes * contention, terminate. 1106e14c720eSVlastimil Babka */ 1107f5f61a32SVlastimil Babka if ((cc->nr_freepages >= cc->nr_migratepages) 1108f5f61a32SVlastimil Babka || cc->contended) { 1109a46cbf3bSDavid Rientjes if (isolate_start_pfn >= block_end_pfn) { 1110a46cbf3bSDavid Rientjes /* 1111a46cbf3bSDavid Rientjes * Restart at previous pageblock if more 1112a46cbf3bSDavid Rientjes * freepages can be isolated next time. 1113a46cbf3bSDavid Rientjes */ 1114f5f61a32SVlastimil Babka isolate_start_pfn = 1115e14c720eSVlastimil Babka block_start_pfn - pageblock_nr_pages; 1116a46cbf3bSDavid Rientjes } 1117be976572SVlastimil Babka break; 1118a46cbf3bSDavid Rientjes } else if (isolate_start_pfn < block_end_pfn) { 1119f5f61a32SVlastimil Babka /* 1120a46cbf3bSDavid Rientjes * If isolation failed early, do not continue 1121a46cbf3bSDavid Rientjes * needlessly. 1122f5f61a32SVlastimil Babka */ 1123a46cbf3bSDavid Rientjes break; 1124f5f61a32SVlastimil Babka } 1125c89511abSMel Gorman } 1126ff9543fdSMichal Nazarewicz 112766c64223SJoonsoo Kim /* __isolate_free_page() does not map the pages */ 1128ff9543fdSMichal Nazarewicz map_pages(freelist); 1129ff9543fdSMichal Nazarewicz 11307ed695e0SVlastimil Babka /* 1131f5f61a32SVlastimil Babka * Record where the free scanner will restart next time. Either we 1132f5f61a32SVlastimil Babka * broke from the loop and set isolate_start_pfn based on the last 1133f5f61a32SVlastimil Babka * call to isolate_freepages_block(), or we met the migration scanner 1134f5f61a32SVlastimil Babka * and the loop terminated due to isolate_start_pfn < low_pfn 11357ed695e0SVlastimil Babka */ 1136f5f61a32SVlastimil Babka cc->free_pfn = isolate_start_pfn; 1137748446bbSMel Gorman } 1138748446bbSMel Gorman 1139748446bbSMel Gorman /* 1140748446bbSMel Gorman * This is a migrate-callback that "allocates" freepages by taking pages 1141748446bbSMel Gorman * from the isolated freelists in the block we are migrating to. 1142748446bbSMel Gorman */ 1143748446bbSMel Gorman static struct page *compaction_alloc(struct page *migratepage, 1144748446bbSMel Gorman unsigned long data, 1145748446bbSMel Gorman int **result) 1146748446bbSMel Gorman { 1147748446bbSMel Gorman struct compact_control *cc = (struct compact_control *)data; 1148748446bbSMel Gorman struct page *freepage; 1149748446bbSMel Gorman 1150be976572SVlastimil Babka /* 1151be976572SVlastimil Babka * Isolate free pages if necessary, and if we are not aborting due to 1152be976572SVlastimil Babka * contention. 1153be976572SVlastimil Babka */ 1154748446bbSMel Gorman if (list_empty(&cc->freepages)) { 1155be976572SVlastimil Babka if (!cc->contended) 1156edc2ca61SVlastimil Babka isolate_freepages(cc); 1157748446bbSMel Gorman 1158748446bbSMel Gorman if (list_empty(&cc->freepages)) 1159748446bbSMel Gorman return NULL; 1160748446bbSMel Gorman } 1161748446bbSMel Gorman 1162748446bbSMel Gorman freepage = list_entry(cc->freepages.next, struct page, lru); 1163748446bbSMel Gorman list_del(&freepage->lru); 1164748446bbSMel Gorman cc->nr_freepages--; 1165748446bbSMel Gorman 1166748446bbSMel Gorman return freepage; 1167748446bbSMel Gorman } 1168748446bbSMel Gorman 1169748446bbSMel Gorman /* 1170d53aea3dSDavid Rientjes * This is a migrate-callback that "frees" freepages back to the isolated 1171d53aea3dSDavid Rientjes * freelist. All pages on the freelist are from the same zone, so there is no 1172d53aea3dSDavid Rientjes * special handling needed for NUMA. 1173d53aea3dSDavid Rientjes */ 1174d53aea3dSDavid Rientjes static void compaction_free(struct page *page, unsigned long data) 1175d53aea3dSDavid Rientjes { 1176d53aea3dSDavid Rientjes struct compact_control *cc = (struct compact_control *)data; 1177d53aea3dSDavid Rientjes 1178d53aea3dSDavid Rientjes list_add(&page->lru, &cc->freepages); 1179d53aea3dSDavid Rientjes cc->nr_freepages++; 1180d53aea3dSDavid Rientjes } 1181d53aea3dSDavid Rientjes 1182ff9543fdSMichal Nazarewicz /* possible outcome of isolate_migratepages */ 1183ff9543fdSMichal Nazarewicz typedef enum { 1184ff9543fdSMichal Nazarewicz ISOLATE_ABORT, /* Abort compaction now */ 1185ff9543fdSMichal Nazarewicz ISOLATE_NONE, /* No pages isolated, continue scanning */ 1186ff9543fdSMichal Nazarewicz ISOLATE_SUCCESS, /* Pages isolated, migrate */ 1187ff9543fdSMichal Nazarewicz } isolate_migrate_t; 1188ff9543fdSMichal Nazarewicz 1189ff9543fdSMichal Nazarewicz /* 11905bbe3547SEric B Munson * Allow userspace to control policy on scanning the unevictable LRU for 11915bbe3547SEric B Munson * compactable pages. 11925bbe3547SEric B Munson */ 11935bbe3547SEric B Munson int sysctl_compact_unevictable_allowed __read_mostly = 1; 11945bbe3547SEric B Munson 11955bbe3547SEric B Munson /* 1196edc2ca61SVlastimil Babka * Isolate all pages that can be migrated from the first suitable block, 1197edc2ca61SVlastimil Babka * starting at the block pointed to by the migrate scanner pfn within 1198edc2ca61SVlastimil Babka * compact_control. 1199ff9543fdSMichal Nazarewicz */ 1200ff9543fdSMichal Nazarewicz static isolate_migrate_t isolate_migratepages(struct zone *zone, 1201ff9543fdSMichal Nazarewicz struct compact_control *cc) 1202ff9543fdSMichal Nazarewicz { 1203e1409c32SJoonsoo Kim unsigned long block_start_pfn; 1204e1409c32SJoonsoo Kim unsigned long block_end_pfn; 1205e1409c32SJoonsoo Kim unsigned long low_pfn; 1206edc2ca61SVlastimil Babka struct page *page; 1207edc2ca61SVlastimil Babka const isolate_mode_t isolate_mode = 12085bbe3547SEric B Munson (sysctl_compact_unevictable_allowed ? ISOLATE_UNEVICTABLE : 0) | 12091d2047feSHugh Dickins (cc->mode != MIGRATE_SYNC ? ISOLATE_ASYNC_MIGRATE : 0); 1210ff9543fdSMichal Nazarewicz 1211edc2ca61SVlastimil Babka /* 1212edc2ca61SVlastimil Babka * Start at where we last stopped, or beginning of the zone as 1213edc2ca61SVlastimil Babka * initialized by compact_zone() 1214edc2ca61SVlastimil Babka */ 1215edc2ca61SVlastimil Babka low_pfn = cc->migrate_pfn; 121606b6640aSVlastimil Babka block_start_pfn = pageblock_start_pfn(low_pfn); 1217e1409c32SJoonsoo Kim if (block_start_pfn < zone->zone_start_pfn) 1218e1409c32SJoonsoo Kim block_start_pfn = zone->zone_start_pfn; 1219ff9543fdSMichal Nazarewicz 1220ff9543fdSMichal Nazarewicz /* Only scan within a pageblock boundary */ 122106b6640aSVlastimil Babka block_end_pfn = pageblock_end_pfn(low_pfn); 1222ff9543fdSMichal Nazarewicz 1223edc2ca61SVlastimil Babka /* 1224edc2ca61SVlastimil Babka * Iterate over whole pageblocks until we find the first suitable. 1225edc2ca61SVlastimil Babka * Do not cross the free scanner. 1226edc2ca61SVlastimil Babka */ 1227e1409c32SJoonsoo Kim for (; block_end_pfn <= cc->free_pfn; 1228e1409c32SJoonsoo Kim low_pfn = block_end_pfn, 1229e1409c32SJoonsoo Kim block_start_pfn = block_end_pfn, 1230e1409c32SJoonsoo Kim block_end_pfn += pageblock_nr_pages) { 1231edc2ca61SVlastimil Babka 1232edc2ca61SVlastimil Babka /* 1233edc2ca61SVlastimil Babka * This can potentially iterate a massively long zone with 1234edc2ca61SVlastimil Babka * many pageblocks unsuitable, so periodically check if we 1235edc2ca61SVlastimil Babka * need to schedule, or even abort async compaction. 1236edc2ca61SVlastimil Babka */ 1237edc2ca61SVlastimil Babka if (!(low_pfn % (SWAP_CLUSTER_MAX * pageblock_nr_pages)) 1238edc2ca61SVlastimil Babka && compact_should_abort(cc)) 1239edc2ca61SVlastimil Babka break; 1240edc2ca61SVlastimil Babka 1241e1409c32SJoonsoo Kim page = pageblock_pfn_to_page(block_start_pfn, block_end_pfn, 1242e1409c32SJoonsoo Kim zone); 12437d49d886SVlastimil Babka if (!page) 1244edc2ca61SVlastimil Babka continue; 1245edc2ca61SVlastimil Babka 1246edc2ca61SVlastimil Babka /* If isolation recently failed, do not retry */ 1247edc2ca61SVlastimil Babka if (!isolation_suitable(cc, page)) 1248edc2ca61SVlastimil Babka continue; 1249edc2ca61SVlastimil Babka 1250edc2ca61SVlastimil Babka /* 1251edc2ca61SVlastimil Babka * For async compaction, also only scan in MOVABLE blocks. 1252edc2ca61SVlastimil Babka * Async compaction is optimistic to see if the minimum amount 1253edc2ca61SVlastimil Babka * of work satisfies the allocation. 1254edc2ca61SVlastimil Babka */ 1255b682debdSVlastimil Babka if (!suitable_migration_source(cc, page)) 1256edc2ca61SVlastimil Babka continue; 1257ff9543fdSMichal Nazarewicz 1258ff9543fdSMichal Nazarewicz /* Perform the isolation */ 1259e1409c32SJoonsoo Kim low_pfn = isolate_migratepages_block(cc, low_pfn, 1260e1409c32SJoonsoo Kim block_end_pfn, isolate_mode); 1261edc2ca61SVlastimil Babka 12626afcf8efSMing Ling if (!low_pfn || cc->contended) 1263ff9543fdSMichal Nazarewicz return ISOLATE_ABORT; 1264ff9543fdSMichal Nazarewicz 1265edc2ca61SVlastimil Babka /* 1266edc2ca61SVlastimil Babka * Either we isolated something and proceed with migration. Or 1267edc2ca61SVlastimil Babka * we failed and compact_zone should decide if we should 1268edc2ca61SVlastimil Babka * continue or not. 1269edc2ca61SVlastimil Babka */ 1270edc2ca61SVlastimil Babka break; 1271edc2ca61SVlastimil Babka } 1272edc2ca61SVlastimil Babka 1273f2849aa0SVlastimil Babka /* Record where migration scanner will be restarted. */ 1274f2849aa0SVlastimil Babka cc->migrate_pfn = low_pfn; 1275ff9543fdSMichal Nazarewicz 1276edc2ca61SVlastimil Babka return cc->nr_migratepages ? ISOLATE_SUCCESS : ISOLATE_NONE; 1277ff9543fdSMichal Nazarewicz } 1278ff9543fdSMichal Nazarewicz 127921c527a3SYaowei Bai /* 128021c527a3SYaowei Bai * order == -1 is expected when compacting via 128121c527a3SYaowei Bai * /proc/sys/vm/compact_memory 128221c527a3SYaowei Bai */ 128321c527a3SYaowei Bai static inline bool is_via_compact_memory(int order) 128421c527a3SYaowei Bai { 128521c527a3SYaowei Bai return order == -1; 128621c527a3SYaowei Bai } 128721c527a3SYaowei Bai 1288d39773a0SVlastimil Babka static enum compact_result __compact_finished(struct zone *zone, 1289d39773a0SVlastimil Babka struct compact_control *cc) 1290748446bbSMel Gorman { 12918fb74b9fSMel Gorman unsigned int order; 1292d39773a0SVlastimil Babka const int migratetype = cc->migratetype; 129356de7263SMel Gorman 1294be976572SVlastimil Babka if (cc->contended || fatal_signal_pending(current)) 12952d1e1041SVlastimil Babka return COMPACT_CONTENDED; 1296748446bbSMel Gorman 1297753341a4SMel Gorman /* Compaction run completes if the migrate and free scanner meet */ 1298f2849aa0SVlastimil Babka if (compact_scanners_met(cc)) { 129955b7c4c9SVlastimil Babka /* Let the next compaction start anew. */ 130002333641SVlastimil Babka reset_cached_positions(zone); 130155b7c4c9SVlastimil Babka 130262997027SMel Gorman /* 130362997027SMel Gorman * Mark that the PG_migrate_skip information should be cleared 1304accf6242SVlastimil Babka * by kswapd when it goes to sleep. kcompactd does not set the 130562997027SMel Gorman * flag itself as the decision to be clear should be directly 130662997027SMel Gorman * based on an allocation request. 130762997027SMel Gorman */ 1308accf6242SVlastimil Babka if (cc->direct_compaction) 130962997027SMel Gorman zone->compact_blockskip_flush = true; 131062997027SMel Gorman 1311c8f7de0bSMichal Hocko if (cc->whole_zone) 1312748446bbSMel Gorman return COMPACT_COMPLETE; 1313c8f7de0bSMichal Hocko else 1314c8f7de0bSMichal Hocko return COMPACT_PARTIAL_SKIPPED; 1315bb13ffebSMel Gorman } 1316748446bbSMel Gorman 131721c527a3SYaowei Bai if (is_via_compact_memory(cc->order)) 131856de7263SMel Gorman return COMPACT_CONTINUE; 131956de7263SMel Gorman 1320baf6a9a1SVlastimil Babka if (cc->finishing_block) { 1321baf6a9a1SVlastimil Babka /* 1322baf6a9a1SVlastimil Babka * We have finished the pageblock, but better check again that 1323baf6a9a1SVlastimil Babka * we really succeeded. 1324baf6a9a1SVlastimil Babka */ 1325baf6a9a1SVlastimil Babka if (IS_ALIGNED(cc->migrate_pfn, pageblock_nr_pages)) 1326baf6a9a1SVlastimil Babka cc->finishing_block = false; 1327baf6a9a1SVlastimil Babka else 1328baf6a9a1SVlastimil Babka return COMPACT_CONTINUE; 1329baf6a9a1SVlastimil Babka } 1330baf6a9a1SVlastimil Babka 133156de7263SMel Gorman /* Direct compactor: Is a suitable page free? */ 133256de7263SMel Gorman for (order = cc->order; order < MAX_ORDER; order++) { 13338fb74b9fSMel Gorman struct free_area *area = &zone->free_area[order]; 13342149cdaeSJoonsoo Kim bool can_steal; 13358fb74b9fSMel Gorman 133656de7263SMel Gorman /* Job done if page is free of the right migratetype */ 13376d7ce559SDavid Rientjes if (!list_empty(&area->free_list[migratetype])) 1338cf378319SVlastimil Babka return COMPACT_SUCCESS; 133956de7263SMel Gorman 13402149cdaeSJoonsoo Kim #ifdef CONFIG_CMA 13412149cdaeSJoonsoo Kim /* MIGRATE_MOVABLE can fallback on MIGRATE_CMA */ 13422149cdaeSJoonsoo Kim if (migratetype == MIGRATE_MOVABLE && 13432149cdaeSJoonsoo Kim !list_empty(&area->free_list[MIGRATE_CMA])) 1344cf378319SVlastimil Babka return COMPACT_SUCCESS; 13452149cdaeSJoonsoo Kim #endif 13462149cdaeSJoonsoo Kim /* 13472149cdaeSJoonsoo Kim * Job done if allocation would steal freepages from 13482149cdaeSJoonsoo Kim * other migratetype buddy lists. 13492149cdaeSJoonsoo Kim */ 13502149cdaeSJoonsoo Kim if (find_suitable_fallback(area, order, migratetype, 1351baf6a9a1SVlastimil Babka true, &can_steal) != -1) { 1352baf6a9a1SVlastimil Babka 1353baf6a9a1SVlastimil Babka /* movable pages are OK in any pageblock */ 1354baf6a9a1SVlastimil Babka if (migratetype == MIGRATE_MOVABLE) 1355cf378319SVlastimil Babka return COMPACT_SUCCESS; 1356baf6a9a1SVlastimil Babka 1357baf6a9a1SVlastimil Babka /* 1358baf6a9a1SVlastimil Babka * We are stealing for a non-movable allocation. Make 1359baf6a9a1SVlastimil Babka * sure we finish compacting the current pageblock 1360baf6a9a1SVlastimil Babka * first so it is as free as possible and we won't 1361baf6a9a1SVlastimil Babka * have to steal another one soon. This only applies 1362baf6a9a1SVlastimil Babka * to sync compaction, as async compaction operates 1363baf6a9a1SVlastimil Babka * on pageblocks of the same migratetype. 1364baf6a9a1SVlastimil Babka */ 1365baf6a9a1SVlastimil Babka if (cc->mode == MIGRATE_ASYNC || 1366baf6a9a1SVlastimil Babka IS_ALIGNED(cc->migrate_pfn, 1367baf6a9a1SVlastimil Babka pageblock_nr_pages)) { 1368baf6a9a1SVlastimil Babka return COMPACT_SUCCESS; 1369baf6a9a1SVlastimil Babka } 1370baf6a9a1SVlastimil Babka 1371baf6a9a1SVlastimil Babka cc->finishing_block = true; 1372baf6a9a1SVlastimil Babka return COMPACT_CONTINUE; 1373baf6a9a1SVlastimil Babka } 137456de7263SMel Gorman } 137556de7263SMel Gorman 1376837d026dSJoonsoo Kim return COMPACT_NO_SUITABLE_PAGE; 1377837d026dSJoonsoo Kim } 1378837d026dSJoonsoo Kim 1379ea7ab982SMichal Hocko static enum compact_result compact_finished(struct zone *zone, 1380d39773a0SVlastimil Babka struct compact_control *cc) 1381837d026dSJoonsoo Kim { 1382837d026dSJoonsoo Kim int ret; 1383837d026dSJoonsoo Kim 1384d39773a0SVlastimil Babka ret = __compact_finished(zone, cc); 1385837d026dSJoonsoo Kim trace_mm_compaction_finished(zone, cc->order, ret); 1386837d026dSJoonsoo Kim if (ret == COMPACT_NO_SUITABLE_PAGE) 1387837d026dSJoonsoo Kim ret = COMPACT_CONTINUE; 1388837d026dSJoonsoo Kim 1389837d026dSJoonsoo Kim return ret; 1390748446bbSMel Gorman } 1391748446bbSMel Gorman 13923e7d3449SMel Gorman /* 13933e7d3449SMel Gorman * compaction_suitable: Is this suitable to run compaction on this zone now? 13943e7d3449SMel Gorman * Returns 13953e7d3449SMel Gorman * COMPACT_SKIPPED - If there are too few free pages for compaction 1396cf378319SVlastimil Babka * COMPACT_SUCCESS - If the allocation would succeed without compaction 13973e7d3449SMel Gorman * COMPACT_CONTINUE - If compaction should run now 13983e7d3449SMel Gorman */ 1399ea7ab982SMichal Hocko static enum compact_result __compaction_suitable(struct zone *zone, int order, 1400c603844bSMel Gorman unsigned int alloc_flags, 140186a294a8SMichal Hocko int classzone_idx, 140286a294a8SMichal Hocko unsigned long wmark_target) 14033e7d3449SMel Gorman { 14043e7d3449SMel Gorman unsigned long watermark; 14053e7d3449SMel Gorman 140621c527a3SYaowei Bai if (is_via_compact_memory(order)) 14073957c776SMichal Hocko return COMPACT_CONTINUE; 14083957c776SMichal Hocko 1409f2b8228cSVlastimil Babka watermark = zone->watermark[alloc_flags & ALLOC_WMARK_MASK]; 1410ebff3980SVlastimil Babka /* 1411ebff3980SVlastimil Babka * If watermarks for high-order allocation are already met, there 1412ebff3980SVlastimil Babka * should be no need for compaction at all. 1413ebff3980SVlastimil Babka */ 1414ebff3980SVlastimil Babka if (zone_watermark_ok(zone, order, watermark, classzone_idx, 1415ebff3980SVlastimil Babka alloc_flags)) 1416cf378319SVlastimil Babka return COMPACT_SUCCESS; 1417ebff3980SVlastimil Babka 14183957c776SMichal Hocko /* 14199861a62cSVlastimil Babka * Watermarks for order-0 must be met for compaction to be able to 1420984fdba6SVlastimil Babka * isolate free pages for migration targets. This means that the 1421984fdba6SVlastimil Babka * watermark and alloc_flags have to match, or be more pessimistic than 1422984fdba6SVlastimil Babka * the check in __isolate_free_page(). We don't use the direct 1423984fdba6SVlastimil Babka * compactor's alloc_flags, as they are not relevant for freepage 1424984fdba6SVlastimil Babka * isolation. We however do use the direct compactor's classzone_idx to 1425984fdba6SVlastimil Babka * skip over zones where lowmem reserves would prevent allocation even 1426984fdba6SVlastimil Babka * if compaction succeeds. 14278348faf9SVlastimil Babka * For costly orders, we require low watermark instead of min for 14288348faf9SVlastimil Babka * compaction to proceed to increase its chances. 1429984fdba6SVlastimil Babka * ALLOC_CMA is used, as pages in CMA pageblocks are considered 1430984fdba6SVlastimil Babka * suitable migration targets 14313e7d3449SMel Gorman */ 14328348faf9SVlastimil Babka watermark = (order > PAGE_ALLOC_COSTLY_ORDER) ? 14338348faf9SVlastimil Babka low_wmark_pages(zone) : min_wmark_pages(zone); 14348348faf9SVlastimil Babka watermark += compact_gap(order); 143586a294a8SMichal Hocko if (!__zone_watermark_ok(zone, 0, watermark, classzone_idx, 1436984fdba6SVlastimil Babka ALLOC_CMA, wmark_target)) 14373e7d3449SMel Gorman return COMPACT_SKIPPED; 14383e7d3449SMel Gorman 1439cc5c9f09SVlastimil Babka return COMPACT_CONTINUE; 1440cc5c9f09SVlastimil Babka } 1441cc5c9f09SVlastimil Babka 1442cc5c9f09SVlastimil Babka enum compact_result compaction_suitable(struct zone *zone, int order, 1443cc5c9f09SVlastimil Babka unsigned int alloc_flags, 1444cc5c9f09SVlastimil Babka int classzone_idx) 1445cc5c9f09SVlastimil Babka { 1446cc5c9f09SVlastimil Babka enum compact_result ret; 1447cc5c9f09SVlastimil Babka int fragindex; 1448cc5c9f09SVlastimil Babka 1449cc5c9f09SVlastimil Babka ret = __compaction_suitable(zone, order, alloc_flags, classzone_idx, 1450cc5c9f09SVlastimil Babka zone_page_state(zone, NR_FREE_PAGES)); 14513e7d3449SMel Gorman /* 14523e7d3449SMel Gorman * fragmentation index determines if allocation failures are due to 14533e7d3449SMel Gorman * low memory or external fragmentation 14543e7d3449SMel Gorman * 1455ebff3980SVlastimil Babka * index of -1000 would imply allocations might succeed depending on 1456ebff3980SVlastimil Babka * watermarks, but we already failed the high-order watermark check 14573e7d3449SMel Gorman * index towards 0 implies failure is due to lack of memory 14583e7d3449SMel Gorman * index towards 1000 implies failure is due to fragmentation 14593e7d3449SMel Gorman * 146020311420SVlastimil Babka * Only compact if a failure would be due to fragmentation. Also 146120311420SVlastimil Babka * ignore fragindex for non-costly orders where the alternative to 146220311420SVlastimil Babka * a successful reclaim/compaction is OOM. Fragindex and the 146320311420SVlastimil Babka * vm.extfrag_threshold sysctl is meant as a heuristic to prevent 146420311420SVlastimil Babka * excessive compaction for costly orders, but it should not be at the 146520311420SVlastimil Babka * expense of system stability. 14663e7d3449SMel Gorman */ 146720311420SVlastimil Babka if (ret == COMPACT_CONTINUE && (order > PAGE_ALLOC_COSTLY_ORDER)) { 14683e7d3449SMel Gorman fragindex = fragmentation_index(zone, order); 14693e7d3449SMel Gorman if (fragindex >= 0 && fragindex <= sysctl_extfrag_threshold) 1470cc5c9f09SVlastimil Babka ret = COMPACT_NOT_SUITABLE_ZONE; 14713e7d3449SMel Gorman } 14723e7d3449SMel Gorman 1473837d026dSJoonsoo Kim trace_mm_compaction_suitable(zone, order, ret); 1474837d026dSJoonsoo Kim if (ret == COMPACT_NOT_SUITABLE_ZONE) 1475837d026dSJoonsoo Kim ret = COMPACT_SKIPPED; 1476837d026dSJoonsoo Kim 1477837d026dSJoonsoo Kim return ret; 1478837d026dSJoonsoo Kim } 1479837d026dSJoonsoo Kim 148086a294a8SMichal Hocko bool compaction_zonelist_suitable(struct alloc_context *ac, int order, 148186a294a8SMichal Hocko int alloc_flags) 148286a294a8SMichal Hocko { 148386a294a8SMichal Hocko struct zone *zone; 148486a294a8SMichal Hocko struct zoneref *z; 148586a294a8SMichal Hocko 148686a294a8SMichal Hocko /* 148786a294a8SMichal Hocko * Make sure at least one zone would pass __compaction_suitable if we continue 148886a294a8SMichal Hocko * retrying the reclaim. 148986a294a8SMichal Hocko */ 149086a294a8SMichal Hocko for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx, 149186a294a8SMichal Hocko ac->nodemask) { 149286a294a8SMichal Hocko unsigned long available; 149386a294a8SMichal Hocko enum compact_result compact_result; 149486a294a8SMichal Hocko 149586a294a8SMichal Hocko /* 149686a294a8SMichal Hocko * Do not consider all the reclaimable memory because we do not 149786a294a8SMichal Hocko * want to trash just for a single high order allocation which 149886a294a8SMichal Hocko * is even not guaranteed to appear even if __compaction_suitable 149986a294a8SMichal Hocko * is happy about the watermark check. 150086a294a8SMichal Hocko */ 15015a1c84b4SMel Gorman available = zone_reclaimable_pages(zone) / order; 150286a294a8SMichal Hocko available += zone_page_state_snapshot(zone, NR_FREE_PAGES); 150386a294a8SMichal Hocko compact_result = __compaction_suitable(zone, order, alloc_flags, 150486a294a8SMichal Hocko ac_classzone_idx(ac), available); 1505cc5c9f09SVlastimil Babka if (compact_result != COMPACT_SKIPPED) 150686a294a8SMichal Hocko return true; 150786a294a8SMichal Hocko } 150886a294a8SMichal Hocko 150986a294a8SMichal Hocko return false; 151086a294a8SMichal Hocko } 151186a294a8SMichal Hocko 1512ea7ab982SMichal Hocko static enum compact_result compact_zone(struct zone *zone, struct compact_control *cc) 1513748446bbSMel Gorman { 1514ea7ab982SMichal Hocko enum compact_result ret; 1515c89511abSMel Gorman unsigned long start_pfn = zone->zone_start_pfn; 1516108bcc96SCody P Schafer unsigned long end_pfn = zone_end_pfn(zone); 1517e0b9daebSDavid Rientjes const bool sync = cc->mode != MIGRATE_ASYNC; 1518748446bbSMel Gorman 1519d39773a0SVlastimil Babka cc->migratetype = gfpflags_to_migratetype(cc->gfp_mask); 1520ebff3980SVlastimil Babka ret = compaction_suitable(zone, cc->order, cc->alloc_flags, 1521ebff3980SVlastimil Babka cc->classzone_idx); 15223e7d3449SMel Gorman /* Compaction is likely to fail */ 1523cf378319SVlastimil Babka if (ret == COMPACT_SUCCESS || ret == COMPACT_SKIPPED) 15243e7d3449SMel Gorman return ret; 1525c46649deSMichal Hocko 1526c46649deSMichal Hocko /* huh, compaction_suitable is returning something unexpected */ 1527c46649deSMichal Hocko VM_BUG_ON(ret != COMPACT_CONTINUE); 15283e7d3449SMel Gorman 1529c89511abSMel Gorman /* 1530d3132e4bSVlastimil Babka * Clear pageblock skip if there were failures recently and compaction 1531accf6242SVlastimil Babka * is about to be retried after being deferred. 1532d3132e4bSVlastimil Babka */ 1533accf6242SVlastimil Babka if (compaction_restarting(zone, cc->order)) 1534d3132e4bSVlastimil Babka __reset_isolation_suitable(zone); 1535d3132e4bSVlastimil Babka 1536d3132e4bSVlastimil Babka /* 1537c89511abSMel Gorman * Setup to move all movable pages to the end of the zone. Used cached 153806ed2998SVlastimil Babka * information on where the scanners should start (unless we explicitly 153906ed2998SVlastimil Babka * want to compact the whole zone), but check that it is initialised 154006ed2998SVlastimil Babka * by ensuring the values are within zone boundaries. 1541c89511abSMel Gorman */ 154206ed2998SVlastimil Babka if (cc->whole_zone) { 154306ed2998SVlastimil Babka cc->migrate_pfn = start_pfn; 154406ed2998SVlastimil Babka cc->free_pfn = pageblock_start_pfn(end_pfn - 1); 154506ed2998SVlastimil Babka } else { 1546e0b9daebSDavid Rientjes cc->migrate_pfn = zone->compact_cached_migrate_pfn[sync]; 1547c89511abSMel Gorman cc->free_pfn = zone->compact_cached_free_pfn; 1548623446e4SJoonsoo Kim if (cc->free_pfn < start_pfn || cc->free_pfn >= end_pfn) { 154906b6640aSVlastimil Babka cc->free_pfn = pageblock_start_pfn(end_pfn - 1); 1550c89511abSMel Gorman zone->compact_cached_free_pfn = cc->free_pfn; 1551c89511abSMel Gorman } 1552623446e4SJoonsoo Kim if (cc->migrate_pfn < start_pfn || cc->migrate_pfn >= end_pfn) { 1553c89511abSMel Gorman cc->migrate_pfn = start_pfn; 155435979ef3SDavid Rientjes zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn; 155535979ef3SDavid Rientjes zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn; 1556c89511abSMel Gorman } 1557c8f7de0bSMichal Hocko 1558c8f7de0bSMichal Hocko if (cc->migrate_pfn == start_pfn) 1559c8f7de0bSMichal Hocko cc->whole_zone = true; 156006ed2998SVlastimil Babka } 1561c8f7de0bSMichal Hocko 15621a16718cSJoonsoo Kim cc->last_migrated_pfn = 0; 1563748446bbSMel Gorman 156416c4a097SJoonsoo Kim trace_mm_compaction_begin(start_pfn, cc->migrate_pfn, 156516c4a097SJoonsoo Kim cc->free_pfn, end_pfn, sync); 15660eb927c0SMel Gorman 1567748446bbSMel Gorman migrate_prep_local(); 1568748446bbSMel Gorman 1569d39773a0SVlastimil Babka while ((ret = compact_finished(zone, cc)) == COMPACT_CONTINUE) { 15709d502c1cSMinchan Kim int err; 1571748446bbSMel Gorman 1572f9e35b3bSMel Gorman switch (isolate_migratepages(zone, cc)) { 1573f9e35b3bSMel Gorman case ISOLATE_ABORT: 15742d1e1041SVlastimil Babka ret = COMPACT_CONTENDED; 15755733c7d1SRafael Aquini putback_movable_pages(&cc->migratepages); 1576e64c5237SShaohua Li cc->nr_migratepages = 0; 1577f9e35b3bSMel Gorman goto out; 1578f9e35b3bSMel Gorman case ISOLATE_NONE: 1579fdaf7f5cSVlastimil Babka /* 1580fdaf7f5cSVlastimil Babka * We haven't isolated and migrated anything, but 1581fdaf7f5cSVlastimil Babka * there might still be unflushed migrations from 1582fdaf7f5cSVlastimil Babka * previous cc->order aligned block. 1583fdaf7f5cSVlastimil Babka */ 1584fdaf7f5cSVlastimil Babka goto check_drain; 1585f9e35b3bSMel Gorman case ISOLATE_SUCCESS: 1586f9e35b3bSMel Gorman ; 1587f9e35b3bSMel Gorman } 1588748446bbSMel Gorman 1589d53aea3dSDavid Rientjes err = migrate_pages(&cc->migratepages, compaction_alloc, 1590e0b9daebSDavid Rientjes compaction_free, (unsigned long)cc, cc->mode, 15917b2a2d4aSMel Gorman MR_COMPACTION); 1592748446bbSMel Gorman 1593f8c9301fSVlastimil Babka trace_mm_compaction_migratepages(cc->nr_migratepages, err, 1594f8c9301fSVlastimil Babka &cc->migratepages); 1595748446bbSMel Gorman 1596f8c9301fSVlastimil Babka /* All pages were either migrated or will be released */ 1597f8c9301fSVlastimil Babka cc->nr_migratepages = 0; 15989d502c1cSMinchan Kim if (err) { 15995733c7d1SRafael Aquini putback_movable_pages(&cc->migratepages); 16007ed695e0SVlastimil Babka /* 16017ed695e0SVlastimil Babka * migrate_pages() may return -ENOMEM when scanners meet 16027ed695e0SVlastimil Babka * and we want compact_finished() to detect it 16037ed695e0SVlastimil Babka */ 1604f2849aa0SVlastimil Babka if (err == -ENOMEM && !compact_scanners_met(cc)) { 16052d1e1041SVlastimil Babka ret = COMPACT_CONTENDED; 16064bf2bba3SDavid Rientjes goto out; 1607748446bbSMel Gorman } 1608fdd048e1SVlastimil Babka /* 1609fdd048e1SVlastimil Babka * We failed to migrate at least one page in the current 1610fdd048e1SVlastimil Babka * order-aligned block, so skip the rest of it. 1611fdd048e1SVlastimil Babka */ 1612fdd048e1SVlastimil Babka if (cc->direct_compaction && 1613fdd048e1SVlastimil Babka (cc->mode == MIGRATE_ASYNC)) { 1614fdd048e1SVlastimil Babka cc->migrate_pfn = block_end_pfn( 1615fdd048e1SVlastimil Babka cc->migrate_pfn - 1, cc->order); 1616fdd048e1SVlastimil Babka /* Draining pcplists is useless in this case */ 1617fdd048e1SVlastimil Babka cc->last_migrated_pfn = 0; 1618fdd048e1SVlastimil Babka 1619fdd048e1SVlastimil Babka } 16204bf2bba3SDavid Rientjes } 1621fdaf7f5cSVlastimil Babka 1622fdaf7f5cSVlastimil Babka check_drain: 1623fdaf7f5cSVlastimil Babka /* 1624fdaf7f5cSVlastimil Babka * Has the migration scanner moved away from the previous 1625fdaf7f5cSVlastimil Babka * cc->order aligned block where we migrated from? If yes, 1626fdaf7f5cSVlastimil Babka * flush the pages that were freed, so that they can merge and 1627fdaf7f5cSVlastimil Babka * compact_finished() can detect immediately if allocation 1628fdaf7f5cSVlastimil Babka * would succeed. 1629fdaf7f5cSVlastimil Babka */ 16301a16718cSJoonsoo Kim if (cc->order > 0 && cc->last_migrated_pfn) { 1631fdaf7f5cSVlastimil Babka int cpu; 1632fdaf7f5cSVlastimil Babka unsigned long current_block_start = 163306b6640aSVlastimil Babka block_start_pfn(cc->migrate_pfn, cc->order); 1634fdaf7f5cSVlastimil Babka 16351a16718cSJoonsoo Kim if (cc->last_migrated_pfn < current_block_start) { 1636fdaf7f5cSVlastimil Babka cpu = get_cpu(); 1637fdaf7f5cSVlastimil Babka lru_add_drain_cpu(cpu); 1638fdaf7f5cSVlastimil Babka drain_local_pages(zone); 1639fdaf7f5cSVlastimil Babka put_cpu(); 1640fdaf7f5cSVlastimil Babka /* No more flushing until we migrate again */ 16411a16718cSJoonsoo Kim cc->last_migrated_pfn = 0; 1642fdaf7f5cSVlastimil Babka } 1643fdaf7f5cSVlastimil Babka } 1644fdaf7f5cSVlastimil Babka 1645748446bbSMel Gorman } 1646748446bbSMel Gorman 1647f9e35b3bSMel Gorman out: 16486bace090SVlastimil Babka /* 16496bace090SVlastimil Babka * Release free pages and update where the free scanner should restart, 16506bace090SVlastimil Babka * so we don't leave any returned pages behind in the next attempt. 16516bace090SVlastimil Babka */ 16526bace090SVlastimil Babka if (cc->nr_freepages > 0) { 16536bace090SVlastimil Babka unsigned long free_pfn = release_freepages(&cc->freepages); 16546bace090SVlastimil Babka 16556bace090SVlastimil Babka cc->nr_freepages = 0; 16566bace090SVlastimil Babka VM_BUG_ON(free_pfn == 0); 16576bace090SVlastimil Babka /* The cached pfn is always the first in a pageblock */ 165806b6640aSVlastimil Babka free_pfn = pageblock_start_pfn(free_pfn); 16596bace090SVlastimil Babka /* 16606bace090SVlastimil Babka * Only go back, not forward. The cached pfn might have been 16616bace090SVlastimil Babka * already reset to zone end in compact_finished() 16626bace090SVlastimil Babka */ 16636bace090SVlastimil Babka if (free_pfn > zone->compact_cached_free_pfn) 16646bace090SVlastimil Babka zone->compact_cached_free_pfn = free_pfn; 16656bace090SVlastimil Babka } 1666748446bbSMel Gorman 16677f354a54SDavid Rientjes count_compact_events(COMPACTMIGRATE_SCANNED, cc->total_migrate_scanned); 16687f354a54SDavid Rientjes count_compact_events(COMPACTFREE_SCANNED, cc->total_free_scanned); 16697f354a54SDavid Rientjes 167016c4a097SJoonsoo Kim trace_mm_compaction_end(start_pfn, cc->migrate_pfn, 167116c4a097SJoonsoo Kim cc->free_pfn, end_pfn, sync, ret); 16720eb927c0SMel Gorman 1673748446bbSMel Gorman return ret; 1674748446bbSMel Gorman } 167576ab0f53SMel Gorman 1676ea7ab982SMichal Hocko static enum compact_result compact_zone_order(struct zone *zone, int order, 1677c3486f53SVlastimil Babka gfp_t gfp_mask, enum compact_priority prio, 1678c603844bSMel Gorman unsigned int alloc_flags, int classzone_idx) 167956de7263SMel Gorman { 1680ea7ab982SMichal Hocko enum compact_result ret; 168156de7263SMel Gorman struct compact_control cc = { 168256de7263SMel Gorman .nr_freepages = 0, 168356de7263SMel Gorman .nr_migratepages = 0, 16847f354a54SDavid Rientjes .total_migrate_scanned = 0, 16857f354a54SDavid Rientjes .total_free_scanned = 0, 168656de7263SMel Gorman .order = order, 16876d7ce559SDavid Rientjes .gfp_mask = gfp_mask, 168856de7263SMel Gorman .zone = zone, 1689a5508cd8SVlastimil Babka .mode = (prio == COMPACT_PRIO_ASYNC) ? 1690a5508cd8SVlastimil Babka MIGRATE_ASYNC : MIGRATE_SYNC_LIGHT, 1691ebff3980SVlastimil Babka .alloc_flags = alloc_flags, 1692ebff3980SVlastimil Babka .classzone_idx = classzone_idx, 1693accf6242SVlastimil Babka .direct_compaction = true, 1694a8e025e5SVlastimil Babka .whole_zone = (prio == MIN_COMPACT_PRIORITY), 16959f7e3387SVlastimil Babka .ignore_skip_hint = (prio == MIN_COMPACT_PRIORITY), 16969f7e3387SVlastimil Babka .ignore_block_suitable = (prio == MIN_COMPACT_PRIORITY) 169756de7263SMel Gorman }; 169856de7263SMel Gorman INIT_LIST_HEAD(&cc.freepages); 169956de7263SMel Gorman INIT_LIST_HEAD(&cc.migratepages); 170056de7263SMel Gorman 1701e64c5237SShaohua Li ret = compact_zone(zone, &cc); 1702e64c5237SShaohua Li 1703e64c5237SShaohua Li VM_BUG_ON(!list_empty(&cc.freepages)); 1704e64c5237SShaohua Li VM_BUG_ON(!list_empty(&cc.migratepages)); 1705e64c5237SShaohua Li 1706e64c5237SShaohua Li return ret; 170756de7263SMel Gorman } 170856de7263SMel Gorman 17095e771905SMel Gorman int sysctl_extfrag_threshold = 500; 17105e771905SMel Gorman 171156de7263SMel Gorman /** 171256de7263SMel Gorman * try_to_compact_pages - Direct compact to satisfy a high-order allocation 171356de7263SMel Gorman * @gfp_mask: The GFP mask of the current allocation 17141a6d53a1SVlastimil Babka * @order: The order of the current allocation 17151a6d53a1SVlastimil Babka * @alloc_flags: The allocation flags of the current allocation 17161a6d53a1SVlastimil Babka * @ac: The context of current allocation 1717e0b9daebSDavid Rientjes * @mode: The migration mode for async, sync light, or sync migration 171856de7263SMel Gorman * 171956de7263SMel Gorman * This is the main entry point for direct page compaction. 172056de7263SMel Gorman */ 1721ea7ab982SMichal Hocko enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order, 1722c603844bSMel Gorman unsigned int alloc_flags, const struct alloc_context *ac, 1723c3486f53SVlastimil Babka enum compact_priority prio) 172456de7263SMel Gorman { 172556de7263SMel Gorman int may_perform_io = gfp_mask & __GFP_IO; 172656de7263SMel Gorman struct zoneref *z; 172756de7263SMel Gorman struct zone *zone; 17281d4746d3SMichal Hocko enum compact_result rc = COMPACT_SKIPPED; 172956de7263SMel Gorman 173073e64c51SMichal Hocko /* 173173e64c51SMichal Hocko * Check if the GFP flags allow compaction - GFP_NOIO is really 173273e64c51SMichal Hocko * tricky context because the migration might require IO 173373e64c51SMichal Hocko */ 173473e64c51SMichal Hocko if (!may_perform_io) 173553853e2dSVlastimil Babka return COMPACT_SKIPPED; 173656de7263SMel Gorman 1737a5508cd8SVlastimil Babka trace_mm_compaction_try_to_compact_pages(order, gfp_mask, prio); 1738837d026dSJoonsoo Kim 173956de7263SMel Gorman /* Compact each zone in the list */ 17401a6d53a1SVlastimil Babka for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx, 17411a6d53a1SVlastimil Babka ac->nodemask) { 1742ea7ab982SMichal Hocko enum compact_result status; 174356de7263SMel Gorman 1744a8e025e5SVlastimil Babka if (prio > MIN_COMPACT_PRIORITY 1745a8e025e5SVlastimil Babka && compaction_deferred(zone, order)) { 17461d4746d3SMichal Hocko rc = max_t(enum compact_result, COMPACT_DEFERRED, rc); 174753853e2dSVlastimil Babka continue; 17481d4746d3SMichal Hocko } 174953853e2dSVlastimil Babka 1750a5508cd8SVlastimil Babka status = compact_zone_order(zone, order, gfp_mask, prio, 1751c3486f53SVlastimil Babka alloc_flags, ac_classzone_idx(ac)); 175256de7263SMel Gorman rc = max(status, rc); 175356de7263SMel Gorman 17547ceb009aSVlastimil Babka /* The allocation should succeed, stop compacting */ 17557ceb009aSVlastimil Babka if (status == COMPACT_SUCCESS) { 175653853e2dSVlastimil Babka /* 175753853e2dSVlastimil Babka * We think the allocation will succeed in this zone, 175853853e2dSVlastimil Babka * but it is not certain, hence the false. The caller 175953853e2dSVlastimil Babka * will repeat this with true if allocation indeed 176053853e2dSVlastimil Babka * succeeds in this zone. 176153853e2dSVlastimil Babka */ 176253853e2dSVlastimil Babka compaction_defer_reset(zone, order, false); 17631f9efdefSVlastimil Babka 1764c3486f53SVlastimil Babka break; 17651f9efdefSVlastimil Babka } 17661f9efdefSVlastimil Babka 1767a5508cd8SVlastimil Babka if (prio != COMPACT_PRIO_ASYNC && (status == COMPACT_COMPLETE || 1768c3486f53SVlastimil Babka status == COMPACT_PARTIAL_SKIPPED)) 176953853e2dSVlastimil Babka /* 177053853e2dSVlastimil Babka * We think that allocation won't succeed in this zone 177153853e2dSVlastimil Babka * so we defer compaction there. If it ends up 177253853e2dSVlastimil Babka * succeeding after all, it will be reset. 177353853e2dSVlastimil Babka */ 177453853e2dSVlastimil Babka defer_compaction(zone, order); 17751f9efdefSVlastimil Babka 17761f9efdefSVlastimil Babka /* 17771f9efdefSVlastimil Babka * We might have stopped compacting due to need_resched() in 17781f9efdefSVlastimil Babka * async compaction, or due to a fatal signal detected. In that 1779c3486f53SVlastimil Babka * case do not try further zones 17801f9efdefSVlastimil Babka */ 1781c3486f53SVlastimil Babka if ((prio == COMPACT_PRIO_ASYNC && need_resched()) 1782c3486f53SVlastimil Babka || fatal_signal_pending(current)) 17831f9efdefSVlastimil Babka break; 17841f9efdefSVlastimil Babka } 17851f9efdefSVlastimil Babka 178656de7263SMel Gorman return rc; 178756de7263SMel Gorman } 178856de7263SMel Gorman 178956de7263SMel Gorman 179076ab0f53SMel Gorman /* Compact all zones within a node */ 17917103f16dSAndrew Morton static void compact_node(int nid) 17927be62de9SRik van Riel { 1793791cae96SVlastimil Babka pg_data_t *pgdat = NODE_DATA(nid); 1794791cae96SVlastimil Babka int zoneid; 1795791cae96SVlastimil Babka struct zone *zone; 17967be62de9SRik van Riel struct compact_control cc = { 17977be62de9SRik van Riel .order = -1, 17987f354a54SDavid Rientjes .total_migrate_scanned = 0, 17997f354a54SDavid Rientjes .total_free_scanned = 0, 1800e0b9daebSDavid Rientjes .mode = MIGRATE_SYNC, 180191ca9186SDavid Rientjes .ignore_skip_hint = true, 180206ed2998SVlastimil Babka .whole_zone = true, 180373e64c51SMichal Hocko .gfp_mask = GFP_KERNEL, 18047be62de9SRik van Riel }; 18057be62de9SRik van Riel 1806791cae96SVlastimil Babka 1807791cae96SVlastimil Babka for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) { 1808791cae96SVlastimil Babka 1809791cae96SVlastimil Babka zone = &pgdat->node_zones[zoneid]; 1810791cae96SVlastimil Babka if (!populated_zone(zone)) 1811791cae96SVlastimil Babka continue; 1812791cae96SVlastimil Babka 1813791cae96SVlastimil Babka cc.nr_freepages = 0; 1814791cae96SVlastimil Babka cc.nr_migratepages = 0; 1815791cae96SVlastimil Babka cc.zone = zone; 1816791cae96SVlastimil Babka INIT_LIST_HEAD(&cc.freepages); 1817791cae96SVlastimil Babka INIT_LIST_HEAD(&cc.migratepages); 1818791cae96SVlastimil Babka 1819791cae96SVlastimil Babka compact_zone(zone, &cc); 1820791cae96SVlastimil Babka 1821791cae96SVlastimil Babka VM_BUG_ON(!list_empty(&cc.freepages)); 1822791cae96SVlastimil Babka VM_BUG_ON(!list_empty(&cc.migratepages)); 1823791cae96SVlastimil Babka } 18247be62de9SRik van Riel } 18257be62de9SRik van Riel 182676ab0f53SMel Gorman /* Compact all nodes in the system */ 18277964c06dSJason Liu static void compact_nodes(void) 182876ab0f53SMel Gorman { 182976ab0f53SMel Gorman int nid; 183076ab0f53SMel Gorman 18318575ec29SHugh Dickins /* Flush pending updates to the LRU lists */ 18328575ec29SHugh Dickins lru_add_drain_all(); 18338575ec29SHugh Dickins 183476ab0f53SMel Gorman for_each_online_node(nid) 183576ab0f53SMel Gorman compact_node(nid); 183676ab0f53SMel Gorman } 183776ab0f53SMel Gorman 183876ab0f53SMel Gorman /* The written value is actually unused, all memory is compacted */ 183976ab0f53SMel Gorman int sysctl_compact_memory; 184076ab0f53SMel Gorman 1841fec4eb2cSYaowei Bai /* 1842fec4eb2cSYaowei Bai * This is the entry point for compacting all nodes via 1843fec4eb2cSYaowei Bai * /proc/sys/vm/compact_memory 1844fec4eb2cSYaowei Bai */ 184576ab0f53SMel Gorman int sysctl_compaction_handler(struct ctl_table *table, int write, 184676ab0f53SMel Gorman void __user *buffer, size_t *length, loff_t *ppos) 184776ab0f53SMel Gorman { 184876ab0f53SMel Gorman if (write) 18497964c06dSJason Liu compact_nodes(); 185076ab0f53SMel Gorman 185176ab0f53SMel Gorman return 0; 185276ab0f53SMel Gorman } 1853ed4a6d7fSMel Gorman 18545e771905SMel Gorman int sysctl_extfrag_handler(struct ctl_table *table, int write, 18555e771905SMel Gorman void __user *buffer, size_t *length, loff_t *ppos) 18565e771905SMel Gorman { 18575e771905SMel Gorman proc_dointvec_minmax(table, write, buffer, length, ppos); 18585e771905SMel Gorman 18595e771905SMel Gorman return 0; 18605e771905SMel Gorman } 18615e771905SMel Gorman 1862ed4a6d7fSMel Gorman #if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA) 186374e77fb9SRashika Kheria static ssize_t sysfs_compact_node(struct device *dev, 186410fbcf4cSKay Sievers struct device_attribute *attr, 1865ed4a6d7fSMel Gorman const char *buf, size_t count) 1866ed4a6d7fSMel Gorman { 18678575ec29SHugh Dickins int nid = dev->id; 18688575ec29SHugh Dickins 18698575ec29SHugh Dickins if (nid >= 0 && nid < nr_node_ids && node_online(nid)) { 18708575ec29SHugh Dickins /* Flush pending updates to the LRU lists */ 18718575ec29SHugh Dickins lru_add_drain_all(); 18728575ec29SHugh Dickins 18738575ec29SHugh Dickins compact_node(nid); 18748575ec29SHugh Dickins } 1875ed4a6d7fSMel Gorman 1876ed4a6d7fSMel Gorman return count; 1877ed4a6d7fSMel Gorman } 187810fbcf4cSKay Sievers static DEVICE_ATTR(compact, S_IWUSR, NULL, sysfs_compact_node); 1879ed4a6d7fSMel Gorman 1880ed4a6d7fSMel Gorman int compaction_register_node(struct node *node) 1881ed4a6d7fSMel Gorman { 188210fbcf4cSKay Sievers return device_create_file(&node->dev, &dev_attr_compact); 1883ed4a6d7fSMel Gorman } 1884ed4a6d7fSMel Gorman 1885ed4a6d7fSMel Gorman void compaction_unregister_node(struct node *node) 1886ed4a6d7fSMel Gorman { 188710fbcf4cSKay Sievers return device_remove_file(&node->dev, &dev_attr_compact); 1888ed4a6d7fSMel Gorman } 1889ed4a6d7fSMel Gorman #endif /* CONFIG_SYSFS && CONFIG_NUMA */ 1890ff9543fdSMichal Nazarewicz 1891698b1b30SVlastimil Babka static inline bool kcompactd_work_requested(pg_data_t *pgdat) 1892698b1b30SVlastimil Babka { 1893172400c6SVlastimil Babka return pgdat->kcompactd_max_order > 0 || kthread_should_stop(); 1894698b1b30SVlastimil Babka } 1895698b1b30SVlastimil Babka 1896698b1b30SVlastimil Babka static bool kcompactd_node_suitable(pg_data_t *pgdat) 1897698b1b30SVlastimil Babka { 1898698b1b30SVlastimil Babka int zoneid; 1899698b1b30SVlastimil Babka struct zone *zone; 1900698b1b30SVlastimil Babka enum zone_type classzone_idx = pgdat->kcompactd_classzone_idx; 1901698b1b30SVlastimil Babka 19026cd9dc3eSChen Feng for (zoneid = 0; zoneid <= classzone_idx; zoneid++) { 1903698b1b30SVlastimil Babka zone = &pgdat->node_zones[zoneid]; 1904698b1b30SVlastimil Babka 1905698b1b30SVlastimil Babka if (!populated_zone(zone)) 1906698b1b30SVlastimil Babka continue; 1907698b1b30SVlastimil Babka 1908698b1b30SVlastimil Babka if (compaction_suitable(zone, pgdat->kcompactd_max_order, 0, 1909698b1b30SVlastimil Babka classzone_idx) == COMPACT_CONTINUE) 1910698b1b30SVlastimil Babka return true; 1911698b1b30SVlastimil Babka } 1912698b1b30SVlastimil Babka 1913698b1b30SVlastimil Babka return false; 1914698b1b30SVlastimil Babka } 1915698b1b30SVlastimil Babka 1916698b1b30SVlastimil Babka static void kcompactd_do_work(pg_data_t *pgdat) 1917698b1b30SVlastimil Babka { 1918698b1b30SVlastimil Babka /* 1919698b1b30SVlastimil Babka * With no special task, compact all zones so that a page of requested 1920698b1b30SVlastimil Babka * order is allocatable. 1921698b1b30SVlastimil Babka */ 1922698b1b30SVlastimil Babka int zoneid; 1923698b1b30SVlastimil Babka struct zone *zone; 1924698b1b30SVlastimil Babka struct compact_control cc = { 1925698b1b30SVlastimil Babka .order = pgdat->kcompactd_max_order, 19267f354a54SDavid Rientjes .total_migrate_scanned = 0, 19277f354a54SDavid Rientjes .total_free_scanned = 0, 1928698b1b30SVlastimil Babka .classzone_idx = pgdat->kcompactd_classzone_idx, 1929698b1b30SVlastimil Babka .mode = MIGRATE_SYNC_LIGHT, 1930698b1b30SVlastimil Babka .ignore_skip_hint = true, 193173e64c51SMichal Hocko .gfp_mask = GFP_KERNEL, 1932698b1b30SVlastimil Babka 1933698b1b30SVlastimil Babka }; 1934698b1b30SVlastimil Babka trace_mm_compaction_kcompactd_wake(pgdat->node_id, cc.order, 1935698b1b30SVlastimil Babka cc.classzone_idx); 19367f354a54SDavid Rientjes count_compact_event(KCOMPACTD_WAKE); 1937698b1b30SVlastimil Babka 19386cd9dc3eSChen Feng for (zoneid = 0; zoneid <= cc.classzone_idx; zoneid++) { 1939698b1b30SVlastimil Babka int status; 1940698b1b30SVlastimil Babka 1941698b1b30SVlastimil Babka zone = &pgdat->node_zones[zoneid]; 1942698b1b30SVlastimil Babka if (!populated_zone(zone)) 1943698b1b30SVlastimil Babka continue; 1944698b1b30SVlastimil Babka 1945698b1b30SVlastimil Babka if (compaction_deferred(zone, cc.order)) 1946698b1b30SVlastimil Babka continue; 1947698b1b30SVlastimil Babka 1948698b1b30SVlastimil Babka if (compaction_suitable(zone, cc.order, 0, zoneid) != 1949698b1b30SVlastimil Babka COMPACT_CONTINUE) 1950698b1b30SVlastimil Babka continue; 1951698b1b30SVlastimil Babka 1952698b1b30SVlastimil Babka cc.nr_freepages = 0; 1953698b1b30SVlastimil Babka cc.nr_migratepages = 0; 19547f354a54SDavid Rientjes cc.total_migrate_scanned = 0; 19557f354a54SDavid Rientjes cc.total_free_scanned = 0; 1956698b1b30SVlastimil Babka cc.zone = zone; 1957698b1b30SVlastimil Babka INIT_LIST_HEAD(&cc.freepages); 1958698b1b30SVlastimil Babka INIT_LIST_HEAD(&cc.migratepages); 1959698b1b30SVlastimil Babka 1960172400c6SVlastimil Babka if (kthread_should_stop()) 1961172400c6SVlastimil Babka return; 1962698b1b30SVlastimil Babka status = compact_zone(zone, &cc); 1963698b1b30SVlastimil Babka 19647ceb009aSVlastimil Babka if (status == COMPACT_SUCCESS) { 1965698b1b30SVlastimil Babka compaction_defer_reset(zone, cc.order, false); 1966c8f7de0bSMichal Hocko } else if (status == COMPACT_PARTIAL_SKIPPED || status == COMPACT_COMPLETE) { 1967698b1b30SVlastimil Babka /* 1968698b1b30SVlastimil Babka * We use sync migration mode here, so we defer like 1969698b1b30SVlastimil Babka * sync direct compaction does. 1970698b1b30SVlastimil Babka */ 1971698b1b30SVlastimil Babka defer_compaction(zone, cc.order); 1972698b1b30SVlastimil Babka } 1973698b1b30SVlastimil Babka 19747f354a54SDavid Rientjes count_compact_events(KCOMPACTD_MIGRATE_SCANNED, 19757f354a54SDavid Rientjes cc.total_migrate_scanned); 19767f354a54SDavid Rientjes count_compact_events(KCOMPACTD_FREE_SCANNED, 19777f354a54SDavid Rientjes cc.total_free_scanned); 19787f354a54SDavid Rientjes 1979698b1b30SVlastimil Babka VM_BUG_ON(!list_empty(&cc.freepages)); 1980698b1b30SVlastimil Babka VM_BUG_ON(!list_empty(&cc.migratepages)); 1981698b1b30SVlastimil Babka } 1982698b1b30SVlastimil Babka 1983698b1b30SVlastimil Babka /* 1984698b1b30SVlastimil Babka * Regardless of success, we are done until woken up next. But remember 1985698b1b30SVlastimil Babka * the requested order/classzone_idx in case it was higher/tighter than 1986698b1b30SVlastimil Babka * our current ones 1987698b1b30SVlastimil Babka */ 1988698b1b30SVlastimil Babka if (pgdat->kcompactd_max_order <= cc.order) 1989698b1b30SVlastimil Babka pgdat->kcompactd_max_order = 0; 1990698b1b30SVlastimil Babka if (pgdat->kcompactd_classzone_idx >= cc.classzone_idx) 1991698b1b30SVlastimil Babka pgdat->kcompactd_classzone_idx = pgdat->nr_zones - 1; 1992698b1b30SVlastimil Babka } 1993698b1b30SVlastimil Babka 1994698b1b30SVlastimil Babka void wakeup_kcompactd(pg_data_t *pgdat, int order, int classzone_idx) 1995698b1b30SVlastimil Babka { 1996698b1b30SVlastimil Babka if (!order) 1997698b1b30SVlastimil Babka return; 1998698b1b30SVlastimil Babka 1999698b1b30SVlastimil Babka if (pgdat->kcompactd_max_order < order) 2000698b1b30SVlastimil Babka pgdat->kcompactd_max_order = order; 2001698b1b30SVlastimil Babka 2002698b1b30SVlastimil Babka if (pgdat->kcompactd_classzone_idx > classzone_idx) 2003698b1b30SVlastimil Babka pgdat->kcompactd_classzone_idx = classzone_idx; 2004698b1b30SVlastimil Babka 2005*6818600fSDavidlohr Bueso /* 2006*6818600fSDavidlohr Bueso * Pairs with implicit barrier in wait_event_freezable() 2007*6818600fSDavidlohr Bueso * such that wakeups are not missed. 2008*6818600fSDavidlohr Bueso */ 2009*6818600fSDavidlohr Bueso if (!wq_has_sleeper(&pgdat->kcompactd_wait)) 2010698b1b30SVlastimil Babka return; 2011698b1b30SVlastimil Babka 2012698b1b30SVlastimil Babka if (!kcompactd_node_suitable(pgdat)) 2013698b1b30SVlastimil Babka return; 2014698b1b30SVlastimil Babka 2015698b1b30SVlastimil Babka trace_mm_compaction_wakeup_kcompactd(pgdat->node_id, order, 2016698b1b30SVlastimil Babka classzone_idx); 2017698b1b30SVlastimil Babka wake_up_interruptible(&pgdat->kcompactd_wait); 2018698b1b30SVlastimil Babka } 2019698b1b30SVlastimil Babka 2020698b1b30SVlastimil Babka /* 2021698b1b30SVlastimil Babka * The background compaction daemon, started as a kernel thread 2022698b1b30SVlastimil Babka * from the init process. 2023698b1b30SVlastimil Babka */ 2024698b1b30SVlastimil Babka static int kcompactd(void *p) 2025698b1b30SVlastimil Babka { 2026698b1b30SVlastimil Babka pg_data_t *pgdat = (pg_data_t*)p; 2027698b1b30SVlastimil Babka struct task_struct *tsk = current; 2028698b1b30SVlastimil Babka 2029698b1b30SVlastimil Babka const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); 2030698b1b30SVlastimil Babka 2031698b1b30SVlastimil Babka if (!cpumask_empty(cpumask)) 2032698b1b30SVlastimil Babka set_cpus_allowed_ptr(tsk, cpumask); 2033698b1b30SVlastimil Babka 2034698b1b30SVlastimil Babka set_freezable(); 2035698b1b30SVlastimil Babka 2036698b1b30SVlastimil Babka pgdat->kcompactd_max_order = 0; 2037698b1b30SVlastimil Babka pgdat->kcompactd_classzone_idx = pgdat->nr_zones - 1; 2038698b1b30SVlastimil Babka 2039698b1b30SVlastimil Babka while (!kthread_should_stop()) { 2040698b1b30SVlastimil Babka trace_mm_compaction_kcompactd_sleep(pgdat->node_id); 2041698b1b30SVlastimil Babka wait_event_freezable(pgdat->kcompactd_wait, 2042698b1b30SVlastimil Babka kcompactd_work_requested(pgdat)); 2043698b1b30SVlastimil Babka 2044698b1b30SVlastimil Babka kcompactd_do_work(pgdat); 2045698b1b30SVlastimil Babka } 2046698b1b30SVlastimil Babka 2047698b1b30SVlastimil Babka return 0; 2048698b1b30SVlastimil Babka } 2049698b1b30SVlastimil Babka 2050698b1b30SVlastimil Babka /* 2051698b1b30SVlastimil Babka * This kcompactd start function will be called by init and node-hot-add. 2052698b1b30SVlastimil Babka * On node-hot-add, kcompactd will moved to proper cpus if cpus are hot-added. 2053698b1b30SVlastimil Babka */ 2054698b1b30SVlastimil Babka int kcompactd_run(int nid) 2055698b1b30SVlastimil Babka { 2056698b1b30SVlastimil Babka pg_data_t *pgdat = NODE_DATA(nid); 2057698b1b30SVlastimil Babka int ret = 0; 2058698b1b30SVlastimil Babka 2059698b1b30SVlastimil Babka if (pgdat->kcompactd) 2060698b1b30SVlastimil Babka return 0; 2061698b1b30SVlastimil Babka 2062698b1b30SVlastimil Babka pgdat->kcompactd = kthread_run(kcompactd, pgdat, "kcompactd%d", nid); 2063698b1b30SVlastimil Babka if (IS_ERR(pgdat->kcompactd)) { 2064698b1b30SVlastimil Babka pr_err("Failed to start kcompactd on node %d\n", nid); 2065698b1b30SVlastimil Babka ret = PTR_ERR(pgdat->kcompactd); 2066698b1b30SVlastimil Babka pgdat->kcompactd = NULL; 2067698b1b30SVlastimil Babka } 2068698b1b30SVlastimil Babka return ret; 2069698b1b30SVlastimil Babka } 2070698b1b30SVlastimil Babka 2071698b1b30SVlastimil Babka /* 2072698b1b30SVlastimil Babka * Called by memory hotplug when all memory in a node is offlined. Caller must 2073698b1b30SVlastimil Babka * hold mem_hotplug_begin/end(). 2074698b1b30SVlastimil Babka */ 2075698b1b30SVlastimil Babka void kcompactd_stop(int nid) 2076698b1b30SVlastimil Babka { 2077698b1b30SVlastimil Babka struct task_struct *kcompactd = NODE_DATA(nid)->kcompactd; 2078698b1b30SVlastimil Babka 2079698b1b30SVlastimil Babka if (kcompactd) { 2080698b1b30SVlastimil Babka kthread_stop(kcompactd); 2081698b1b30SVlastimil Babka NODE_DATA(nid)->kcompactd = NULL; 2082698b1b30SVlastimil Babka } 2083698b1b30SVlastimil Babka } 2084698b1b30SVlastimil Babka 2085698b1b30SVlastimil Babka /* 2086698b1b30SVlastimil Babka * It's optimal to keep kcompactd on the same CPUs as their memory, but 2087698b1b30SVlastimil Babka * not required for correctness. So if the last cpu in a node goes 2088698b1b30SVlastimil Babka * away, we get changed to run anywhere: as the first one comes back, 2089698b1b30SVlastimil Babka * restore their cpu bindings. 2090698b1b30SVlastimil Babka */ 2091e46b1db2SAnna-Maria Gleixner static int kcompactd_cpu_online(unsigned int cpu) 2092698b1b30SVlastimil Babka { 2093698b1b30SVlastimil Babka int nid; 2094698b1b30SVlastimil Babka 2095698b1b30SVlastimil Babka for_each_node_state(nid, N_MEMORY) { 2096698b1b30SVlastimil Babka pg_data_t *pgdat = NODE_DATA(nid); 2097698b1b30SVlastimil Babka const struct cpumask *mask; 2098698b1b30SVlastimil Babka 2099698b1b30SVlastimil Babka mask = cpumask_of_node(pgdat->node_id); 2100698b1b30SVlastimil Babka 2101698b1b30SVlastimil Babka if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids) 2102698b1b30SVlastimil Babka /* One of our CPUs online: restore mask */ 2103698b1b30SVlastimil Babka set_cpus_allowed_ptr(pgdat->kcompactd, mask); 2104698b1b30SVlastimil Babka } 2105e46b1db2SAnna-Maria Gleixner return 0; 2106698b1b30SVlastimil Babka } 2107698b1b30SVlastimil Babka 2108698b1b30SVlastimil Babka static int __init kcompactd_init(void) 2109698b1b30SVlastimil Babka { 2110698b1b30SVlastimil Babka int nid; 2111e46b1db2SAnna-Maria Gleixner int ret; 2112e46b1db2SAnna-Maria Gleixner 2113e46b1db2SAnna-Maria Gleixner ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, 2114e46b1db2SAnna-Maria Gleixner "mm/compaction:online", 2115e46b1db2SAnna-Maria Gleixner kcompactd_cpu_online, NULL); 2116e46b1db2SAnna-Maria Gleixner if (ret < 0) { 2117e46b1db2SAnna-Maria Gleixner pr_err("kcompactd: failed to register hotplug callbacks.\n"); 2118e46b1db2SAnna-Maria Gleixner return ret; 2119e46b1db2SAnna-Maria Gleixner } 2120698b1b30SVlastimil Babka 2121698b1b30SVlastimil Babka for_each_node_state(nid, N_MEMORY) 2122698b1b30SVlastimil Babka kcompactd_run(nid); 2123698b1b30SVlastimil Babka return 0; 2124698b1b30SVlastimil Babka } 2125698b1b30SVlastimil Babka subsys_initcall(kcompactd_init) 2126698b1b30SVlastimil Babka 2127ff9543fdSMichal Nazarewicz #endif /* CONFIG_COMPACTION */ 2128