1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 2748446bbSMel Gorman /* 3748446bbSMel Gorman * linux/mm/compaction.c 4748446bbSMel Gorman * 5748446bbSMel Gorman * Memory compaction for the reduction of external fragmentation. Note that 6748446bbSMel Gorman * this heavily depends upon page migration to do all the real heavy 7748446bbSMel Gorman * lifting 8748446bbSMel Gorman * 9748446bbSMel Gorman * Copyright IBM Corp. 2007-2010 Mel Gorman <mel@csn.ul.ie> 10748446bbSMel Gorman */ 11698b1b30SVlastimil Babka #include <linux/cpu.h> 12748446bbSMel Gorman #include <linux/swap.h> 13748446bbSMel Gorman #include <linux/migrate.h> 14748446bbSMel Gorman #include <linux/compaction.h> 15748446bbSMel Gorman #include <linux/mm_inline.h> 16174cd4b1SIngo Molnar #include <linux/sched/signal.h> 17748446bbSMel Gorman #include <linux/backing-dev.h> 1876ab0f53SMel Gorman #include <linux/sysctl.h> 19ed4a6d7fSMel Gorman #include <linux/sysfs.h> 20194159fbSMinchan Kim #include <linux/page-isolation.h> 21b8c73fc2SAndrey Ryabinin #include <linux/kasan.h> 22698b1b30SVlastimil Babka #include <linux/kthread.h> 23698b1b30SVlastimil Babka #include <linux/freezer.h> 2483358eceSJoonsoo Kim #include <linux/page_owner.h> 25eb414681SJohannes Weiner #include <linux/psi.h> 26748446bbSMel Gorman #include "internal.h" 27748446bbSMel Gorman 28010fc29aSMinchan Kim #ifdef CONFIG_COMPACTION 29010fc29aSMinchan Kim static inline void count_compact_event(enum vm_event_item item) 30010fc29aSMinchan Kim { 31010fc29aSMinchan Kim count_vm_event(item); 32010fc29aSMinchan Kim } 33010fc29aSMinchan Kim 34010fc29aSMinchan Kim static inline void count_compact_events(enum vm_event_item item, long delta) 35010fc29aSMinchan Kim { 36010fc29aSMinchan Kim count_vm_events(item, delta); 37010fc29aSMinchan Kim } 38010fc29aSMinchan Kim #else 39010fc29aSMinchan Kim #define count_compact_event(item) do { } while (0) 40010fc29aSMinchan Kim #define count_compact_events(item, delta) do { } while (0) 41010fc29aSMinchan Kim #endif 42010fc29aSMinchan Kim 43ff9543fdSMichal Nazarewicz #if defined CONFIG_COMPACTION || defined CONFIG_CMA 44ff9543fdSMichal Nazarewicz 45b7aba698SMel Gorman #define CREATE_TRACE_POINTS 46b7aba698SMel Gorman #include <trace/events/compaction.h> 47b7aba698SMel Gorman 4806b6640aSVlastimil Babka #define block_start_pfn(pfn, order) round_down(pfn, 1UL << (order)) 4906b6640aSVlastimil Babka #define block_end_pfn(pfn, order) ALIGN((pfn) + 1, 1UL << (order)) 5006b6640aSVlastimil Babka #define pageblock_start_pfn(pfn) block_start_pfn(pfn, pageblock_order) 5106b6640aSVlastimil Babka #define pageblock_end_pfn(pfn) block_end_pfn(pfn, pageblock_order) 5206b6640aSVlastimil Babka 53748446bbSMel Gorman static unsigned long release_freepages(struct list_head *freelist) 54748446bbSMel Gorman { 55748446bbSMel Gorman struct page *page, *next; 566bace090SVlastimil Babka unsigned long high_pfn = 0; 57748446bbSMel Gorman 58748446bbSMel Gorman list_for_each_entry_safe(page, next, freelist, lru) { 596bace090SVlastimil Babka unsigned long pfn = page_to_pfn(page); 60748446bbSMel Gorman list_del(&page->lru); 61748446bbSMel Gorman __free_page(page); 626bace090SVlastimil Babka if (pfn > high_pfn) 636bace090SVlastimil Babka high_pfn = pfn; 64748446bbSMel Gorman } 65748446bbSMel Gorman 666bace090SVlastimil Babka return high_pfn; 67748446bbSMel Gorman } 68748446bbSMel Gorman 694469ab98SMel Gorman static void split_map_pages(struct list_head *list) 70ff9543fdSMichal Nazarewicz { 7166c64223SJoonsoo Kim unsigned int i, order, nr_pages; 7266c64223SJoonsoo Kim struct page *page, *next; 7366c64223SJoonsoo Kim LIST_HEAD(tmp_list); 74ff9543fdSMichal Nazarewicz 7566c64223SJoonsoo Kim list_for_each_entry_safe(page, next, list, lru) { 7666c64223SJoonsoo Kim list_del(&page->lru); 7766c64223SJoonsoo Kim 7866c64223SJoonsoo Kim order = page_private(page); 7966c64223SJoonsoo Kim nr_pages = 1 << order; 8066c64223SJoonsoo Kim 8146f24fd8SJoonsoo Kim post_alloc_hook(page, order, __GFP_MOVABLE); 8266c64223SJoonsoo Kim if (order) 8366c64223SJoonsoo Kim split_page(page, order); 8466c64223SJoonsoo Kim 8566c64223SJoonsoo Kim for (i = 0; i < nr_pages; i++) { 8666c64223SJoonsoo Kim list_add(&page->lru, &tmp_list); 8766c64223SJoonsoo Kim page++; 88ff9543fdSMichal Nazarewicz } 89ff9543fdSMichal Nazarewicz } 90ff9543fdSMichal Nazarewicz 9166c64223SJoonsoo Kim list_splice(&tmp_list, list); 9266c64223SJoonsoo Kim } 9366c64223SJoonsoo Kim 94bb13ffebSMel Gorman #ifdef CONFIG_COMPACTION 9524e2716fSJoonsoo Kim 96bda807d4SMinchan Kim int PageMovable(struct page *page) 97bda807d4SMinchan Kim { 98bda807d4SMinchan Kim struct address_space *mapping; 99bda807d4SMinchan Kim 100bda807d4SMinchan Kim VM_BUG_ON_PAGE(!PageLocked(page), page); 101bda807d4SMinchan Kim if (!__PageMovable(page)) 102bda807d4SMinchan Kim return 0; 103bda807d4SMinchan Kim 104bda807d4SMinchan Kim mapping = page_mapping(page); 105bda807d4SMinchan Kim if (mapping && mapping->a_ops && mapping->a_ops->isolate_page) 106bda807d4SMinchan Kim return 1; 107bda807d4SMinchan Kim 108bda807d4SMinchan Kim return 0; 109bda807d4SMinchan Kim } 110bda807d4SMinchan Kim EXPORT_SYMBOL(PageMovable); 111bda807d4SMinchan Kim 112bda807d4SMinchan Kim void __SetPageMovable(struct page *page, struct address_space *mapping) 113bda807d4SMinchan Kim { 114bda807d4SMinchan Kim VM_BUG_ON_PAGE(!PageLocked(page), page); 115bda807d4SMinchan Kim VM_BUG_ON_PAGE((unsigned long)mapping & PAGE_MAPPING_MOVABLE, page); 116bda807d4SMinchan Kim page->mapping = (void *)((unsigned long)mapping | PAGE_MAPPING_MOVABLE); 117bda807d4SMinchan Kim } 118bda807d4SMinchan Kim EXPORT_SYMBOL(__SetPageMovable); 119bda807d4SMinchan Kim 120bda807d4SMinchan Kim void __ClearPageMovable(struct page *page) 121bda807d4SMinchan Kim { 122bda807d4SMinchan Kim VM_BUG_ON_PAGE(!PageLocked(page), page); 123bda807d4SMinchan Kim VM_BUG_ON_PAGE(!PageMovable(page), page); 124bda807d4SMinchan Kim /* 125bda807d4SMinchan Kim * Clear registered address_space val with keeping PAGE_MAPPING_MOVABLE 126bda807d4SMinchan Kim * flag so that VM can catch up released page by driver after isolation. 127bda807d4SMinchan Kim * With it, VM migration doesn't try to put it back. 128bda807d4SMinchan Kim */ 129bda807d4SMinchan Kim page->mapping = (void *)((unsigned long)page->mapping & 130bda807d4SMinchan Kim PAGE_MAPPING_MOVABLE); 131bda807d4SMinchan Kim } 132bda807d4SMinchan Kim EXPORT_SYMBOL(__ClearPageMovable); 133bda807d4SMinchan Kim 13424e2716fSJoonsoo Kim /* Do not skip compaction more than 64 times */ 13524e2716fSJoonsoo Kim #define COMPACT_MAX_DEFER_SHIFT 6 13624e2716fSJoonsoo Kim 13724e2716fSJoonsoo Kim /* 13824e2716fSJoonsoo Kim * Compaction is deferred when compaction fails to result in a page 13924e2716fSJoonsoo Kim * allocation success. 1 << compact_defer_limit compactions are skipped up 14024e2716fSJoonsoo Kim * to a limit of 1 << COMPACT_MAX_DEFER_SHIFT 14124e2716fSJoonsoo Kim */ 14224e2716fSJoonsoo Kim void defer_compaction(struct zone *zone, int order) 14324e2716fSJoonsoo Kim { 14424e2716fSJoonsoo Kim zone->compact_considered = 0; 14524e2716fSJoonsoo Kim zone->compact_defer_shift++; 14624e2716fSJoonsoo Kim 14724e2716fSJoonsoo Kim if (order < zone->compact_order_failed) 14824e2716fSJoonsoo Kim zone->compact_order_failed = order; 14924e2716fSJoonsoo Kim 15024e2716fSJoonsoo Kim if (zone->compact_defer_shift > COMPACT_MAX_DEFER_SHIFT) 15124e2716fSJoonsoo Kim zone->compact_defer_shift = COMPACT_MAX_DEFER_SHIFT; 15224e2716fSJoonsoo Kim 15324e2716fSJoonsoo Kim trace_mm_compaction_defer_compaction(zone, order); 15424e2716fSJoonsoo Kim } 15524e2716fSJoonsoo Kim 15624e2716fSJoonsoo Kim /* Returns true if compaction should be skipped this time */ 15724e2716fSJoonsoo Kim bool compaction_deferred(struct zone *zone, int order) 15824e2716fSJoonsoo Kim { 15924e2716fSJoonsoo Kim unsigned long defer_limit = 1UL << zone->compact_defer_shift; 16024e2716fSJoonsoo Kim 16124e2716fSJoonsoo Kim if (order < zone->compact_order_failed) 16224e2716fSJoonsoo Kim return false; 16324e2716fSJoonsoo Kim 16424e2716fSJoonsoo Kim /* Avoid possible overflow */ 16524e2716fSJoonsoo Kim if (++zone->compact_considered > defer_limit) 16624e2716fSJoonsoo Kim zone->compact_considered = defer_limit; 16724e2716fSJoonsoo Kim 16824e2716fSJoonsoo Kim if (zone->compact_considered >= defer_limit) 16924e2716fSJoonsoo Kim return false; 17024e2716fSJoonsoo Kim 17124e2716fSJoonsoo Kim trace_mm_compaction_deferred(zone, order); 17224e2716fSJoonsoo Kim 17324e2716fSJoonsoo Kim return true; 17424e2716fSJoonsoo Kim } 17524e2716fSJoonsoo Kim 17624e2716fSJoonsoo Kim /* 17724e2716fSJoonsoo Kim * Update defer tracking counters after successful compaction of given order, 17824e2716fSJoonsoo Kim * which means an allocation either succeeded (alloc_success == true) or is 17924e2716fSJoonsoo Kim * expected to succeed. 18024e2716fSJoonsoo Kim */ 18124e2716fSJoonsoo Kim void compaction_defer_reset(struct zone *zone, int order, 18224e2716fSJoonsoo Kim bool alloc_success) 18324e2716fSJoonsoo Kim { 18424e2716fSJoonsoo Kim if (alloc_success) { 18524e2716fSJoonsoo Kim zone->compact_considered = 0; 18624e2716fSJoonsoo Kim zone->compact_defer_shift = 0; 18724e2716fSJoonsoo Kim } 18824e2716fSJoonsoo Kim if (order >= zone->compact_order_failed) 18924e2716fSJoonsoo Kim zone->compact_order_failed = order + 1; 19024e2716fSJoonsoo Kim 19124e2716fSJoonsoo Kim trace_mm_compaction_defer_reset(zone, order); 19224e2716fSJoonsoo Kim } 19324e2716fSJoonsoo Kim 19424e2716fSJoonsoo Kim /* Returns true if restarting compaction after many failures */ 19524e2716fSJoonsoo Kim bool compaction_restarting(struct zone *zone, int order) 19624e2716fSJoonsoo Kim { 19724e2716fSJoonsoo Kim if (order < zone->compact_order_failed) 19824e2716fSJoonsoo Kim return false; 19924e2716fSJoonsoo Kim 20024e2716fSJoonsoo Kim return zone->compact_defer_shift == COMPACT_MAX_DEFER_SHIFT && 20124e2716fSJoonsoo Kim zone->compact_considered >= 1UL << zone->compact_defer_shift; 20224e2716fSJoonsoo Kim } 20324e2716fSJoonsoo Kim 204bb13ffebSMel Gorman /* Returns true if the pageblock should be scanned for pages to isolate. */ 205bb13ffebSMel Gorman static inline bool isolation_suitable(struct compact_control *cc, 206bb13ffebSMel Gorman struct page *page) 207bb13ffebSMel Gorman { 208bb13ffebSMel Gorman if (cc->ignore_skip_hint) 209bb13ffebSMel Gorman return true; 210bb13ffebSMel Gorman 211bb13ffebSMel Gorman return !get_pageblock_skip(page); 212bb13ffebSMel Gorman } 213bb13ffebSMel Gorman 21402333641SVlastimil Babka static void reset_cached_positions(struct zone *zone) 21502333641SVlastimil Babka { 21602333641SVlastimil Babka zone->compact_cached_migrate_pfn[0] = zone->zone_start_pfn; 21702333641SVlastimil Babka zone->compact_cached_migrate_pfn[1] = zone->zone_start_pfn; 218623446e4SJoonsoo Kim zone->compact_cached_free_pfn = 21906b6640aSVlastimil Babka pageblock_start_pfn(zone_end_pfn(zone) - 1); 22002333641SVlastimil Babka } 22102333641SVlastimil Babka 222bb13ffebSMel Gorman /* 223b527cfe5SVlastimil Babka * Compound pages of >= pageblock_order should consistenly be skipped until 224b527cfe5SVlastimil Babka * released. It is always pointless to compact pages of such order (if they are 225b527cfe5SVlastimil Babka * migratable), and the pageblocks they occupy cannot contain any free pages. 22621dc7e02SDavid Rientjes */ 227b527cfe5SVlastimil Babka static bool pageblock_skip_persistent(struct page *page) 22821dc7e02SDavid Rientjes { 229b527cfe5SVlastimil Babka if (!PageCompound(page)) 23021dc7e02SDavid Rientjes return false; 231b527cfe5SVlastimil Babka 232b527cfe5SVlastimil Babka page = compound_head(page); 233b527cfe5SVlastimil Babka 234b527cfe5SVlastimil Babka if (compound_order(page) >= pageblock_order) 23521dc7e02SDavid Rientjes return true; 236b527cfe5SVlastimil Babka 237b527cfe5SVlastimil Babka return false; 23821dc7e02SDavid Rientjes } 23921dc7e02SDavid Rientjes 24021dc7e02SDavid Rientjes /* 241bb13ffebSMel Gorman * This function is called to clear all cached information on pageblocks that 242bb13ffebSMel Gorman * should be skipped for page isolation when the migrate and free page scanner 243bb13ffebSMel Gorman * meet. 244bb13ffebSMel Gorman */ 24562997027SMel Gorman static void __reset_isolation_suitable(struct zone *zone) 246bb13ffebSMel Gorman { 247bb13ffebSMel Gorman unsigned long start_pfn = zone->zone_start_pfn; 248108bcc96SCody P Schafer unsigned long end_pfn = zone_end_pfn(zone); 249bb13ffebSMel Gorman unsigned long pfn; 250bb13ffebSMel Gorman 25162997027SMel Gorman zone->compact_blockskip_flush = false; 252bb13ffebSMel Gorman 253bb13ffebSMel Gorman /* Walk the zone and mark every pageblock as suitable for isolation */ 254bb13ffebSMel Gorman for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) { 255bb13ffebSMel Gorman struct page *page; 256bb13ffebSMel Gorman 257bb13ffebSMel Gorman cond_resched(); 258bb13ffebSMel Gorman 259ccbe1e4dSMichal Hocko page = pfn_to_online_page(pfn); 260ccbe1e4dSMichal Hocko if (!page) 261bb13ffebSMel Gorman continue; 262bb13ffebSMel Gorman if (zone != page_zone(page)) 263bb13ffebSMel Gorman continue; 264b527cfe5SVlastimil Babka if (pageblock_skip_persistent(page)) 26521dc7e02SDavid Rientjes continue; 266bb13ffebSMel Gorman 267bb13ffebSMel Gorman clear_pageblock_skip(page); 268bb13ffebSMel Gorman } 26902333641SVlastimil Babka 27002333641SVlastimil Babka reset_cached_positions(zone); 271bb13ffebSMel Gorman } 272bb13ffebSMel Gorman 27362997027SMel Gorman void reset_isolation_suitable(pg_data_t *pgdat) 27462997027SMel Gorman { 27562997027SMel Gorman int zoneid; 27662997027SMel Gorman 27762997027SMel Gorman for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) { 27862997027SMel Gorman struct zone *zone = &pgdat->node_zones[zoneid]; 27962997027SMel Gorman if (!populated_zone(zone)) 28062997027SMel Gorman continue; 28162997027SMel Gorman 28262997027SMel Gorman /* Only flush if a full compaction finished recently */ 28362997027SMel Gorman if (zone->compact_blockskip_flush) 28462997027SMel Gorman __reset_isolation_suitable(zone); 28562997027SMel Gorman } 28662997027SMel Gorman } 28762997027SMel Gorman 288bb13ffebSMel Gorman /* 289*e380bebeSMel Gorman * Sets the pageblock skip bit if it was clear. Note that this is a hint as 290*e380bebeSMel Gorman * locks are not required for read/writers. Returns true if it was already set. 291*e380bebeSMel Gorman */ 292*e380bebeSMel Gorman static bool test_and_set_skip(struct compact_control *cc, struct page *page, 293*e380bebeSMel Gorman unsigned long pfn) 294*e380bebeSMel Gorman { 295*e380bebeSMel Gorman bool skip; 296*e380bebeSMel Gorman 297*e380bebeSMel Gorman /* Do no update if skip hint is being ignored */ 298*e380bebeSMel Gorman if (cc->ignore_skip_hint) 299*e380bebeSMel Gorman return false; 300*e380bebeSMel Gorman 301*e380bebeSMel Gorman if (!IS_ALIGNED(pfn, pageblock_nr_pages)) 302*e380bebeSMel Gorman return false; 303*e380bebeSMel Gorman 304*e380bebeSMel Gorman skip = get_pageblock_skip(page); 305*e380bebeSMel Gorman if (!skip && !cc->no_set_skip_hint) 306*e380bebeSMel Gorman set_pageblock_skip(page); 307*e380bebeSMel Gorman 308*e380bebeSMel Gorman return skip; 309*e380bebeSMel Gorman } 310*e380bebeSMel Gorman 311*e380bebeSMel Gorman static void update_cached_migrate(struct compact_control *cc, unsigned long pfn) 312*e380bebeSMel Gorman { 313*e380bebeSMel Gorman struct zone *zone = cc->zone; 314*e380bebeSMel Gorman 315*e380bebeSMel Gorman pfn = pageblock_end_pfn(pfn); 316*e380bebeSMel Gorman 317*e380bebeSMel Gorman /* Set for isolation rather than compaction */ 318*e380bebeSMel Gorman if (cc->no_set_skip_hint) 319*e380bebeSMel Gorman return; 320*e380bebeSMel Gorman 321*e380bebeSMel Gorman if (pfn > zone->compact_cached_migrate_pfn[0]) 322*e380bebeSMel Gorman zone->compact_cached_migrate_pfn[0] = pfn; 323*e380bebeSMel Gorman if (cc->mode != MIGRATE_ASYNC && 324*e380bebeSMel Gorman pfn > zone->compact_cached_migrate_pfn[1]) 325*e380bebeSMel Gorman zone->compact_cached_migrate_pfn[1] = pfn; 326*e380bebeSMel Gorman } 327*e380bebeSMel Gorman 328*e380bebeSMel Gorman /* 329bb13ffebSMel Gorman * If no pages were isolated then mark this pageblock to be skipped in the 33062997027SMel Gorman * future. The information is later cleared by __reset_isolation_suitable(). 331bb13ffebSMel Gorman */ 332c89511abSMel Gorman static void update_pageblock_skip(struct compact_control *cc, 333*e380bebeSMel Gorman struct page *page, unsigned long nr_isolated) 334bb13ffebSMel Gorman { 335c89511abSMel Gorman struct zone *zone = cc->zone; 33635979ef3SDavid Rientjes unsigned long pfn; 3376815bf3fSJoonsoo Kim 3382583d671SVlastimil Babka if (cc->no_set_skip_hint) 3396815bf3fSJoonsoo Kim return; 3406815bf3fSJoonsoo Kim 341bb13ffebSMel Gorman if (!page) 342bb13ffebSMel Gorman return; 343bb13ffebSMel Gorman 34435979ef3SDavid Rientjes if (nr_isolated) 34535979ef3SDavid Rientjes return; 34635979ef3SDavid Rientjes 347bb13ffebSMel Gorman set_pageblock_skip(page); 348c89511abSMel Gorman 34935979ef3SDavid Rientjes pfn = page_to_pfn(page); 35035979ef3SDavid Rientjes 35135979ef3SDavid Rientjes /* Update where async and sync compaction should restart */ 35235979ef3SDavid Rientjes if (pfn < zone->compact_cached_free_pfn) 353c89511abSMel Gorman zone->compact_cached_free_pfn = pfn; 354c89511abSMel Gorman } 355bb13ffebSMel Gorman #else 356bb13ffebSMel Gorman static inline bool isolation_suitable(struct compact_control *cc, 357bb13ffebSMel Gorman struct page *page) 358bb13ffebSMel Gorman { 359bb13ffebSMel Gorman return true; 360bb13ffebSMel Gorman } 361bb13ffebSMel Gorman 362b527cfe5SVlastimil Babka static inline bool pageblock_skip_persistent(struct page *page) 36321dc7e02SDavid Rientjes { 36421dc7e02SDavid Rientjes return false; 36521dc7e02SDavid Rientjes } 36621dc7e02SDavid Rientjes 36721dc7e02SDavid Rientjes static inline void update_pageblock_skip(struct compact_control *cc, 368*e380bebeSMel Gorman struct page *page, unsigned long nr_isolated) 369bb13ffebSMel Gorman { 370bb13ffebSMel Gorman } 371*e380bebeSMel Gorman 372*e380bebeSMel Gorman static void update_cached_migrate(struct compact_control *cc, unsigned long pfn) 373*e380bebeSMel Gorman { 374*e380bebeSMel Gorman } 375*e380bebeSMel Gorman 376*e380bebeSMel Gorman static bool test_and_set_skip(struct compact_control *cc, struct page *page, 377*e380bebeSMel Gorman unsigned long pfn) 378*e380bebeSMel Gorman { 379*e380bebeSMel Gorman return false; 380*e380bebeSMel Gorman } 381bb13ffebSMel Gorman #endif /* CONFIG_COMPACTION */ 382bb13ffebSMel Gorman 3831f9efdefSVlastimil Babka /* 3848b44d279SVlastimil Babka * Compaction requires the taking of some coarse locks that are potentially 3858b44d279SVlastimil Babka * very heavily contended. For async compaction, back out if the lock cannot 3868b44d279SVlastimil Babka * be taken immediately. For sync compaction, spin on the lock if needed. 3878b44d279SVlastimil Babka * 3888b44d279SVlastimil Babka * Returns true if the lock is held 3898b44d279SVlastimil Babka * Returns false if the lock is not held and compaction should abort 3901f9efdefSVlastimil Babka */ 3918b44d279SVlastimil Babka static bool compact_trylock_irqsave(spinlock_t *lock, unsigned long *flags, 3928b44d279SVlastimil Babka struct compact_control *cc) 3938b44d279SVlastimil Babka { 3948b44d279SVlastimil Babka if (cc->mode == MIGRATE_ASYNC) { 3958b44d279SVlastimil Babka if (!spin_trylock_irqsave(lock, *flags)) { 396c3486f53SVlastimil Babka cc->contended = true; 3978b44d279SVlastimil Babka return false; 3988b44d279SVlastimil Babka } 3998b44d279SVlastimil Babka } else { 4008b44d279SVlastimil Babka spin_lock_irqsave(lock, *flags); 4018b44d279SVlastimil Babka } 4021f9efdefSVlastimil Babka 4038b44d279SVlastimil Babka return true; 4042a1402aaSMel Gorman } 4052a1402aaSMel Gorman 40685aa125fSMichal Nazarewicz /* 407c67fe375SMel Gorman * Compaction requires the taking of some coarse locks that are potentially 4088b44d279SVlastimil Babka * very heavily contended. The lock should be periodically unlocked to avoid 4098b44d279SVlastimil Babka * having disabled IRQs for a long time, even when there is nobody waiting on 4108b44d279SVlastimil Babka * the lock. It might also be that allowing the IRQs will result in 4118b44d279SVlastimil Babka * need_resched() becoming true. If scheduling is needed, async compaction 4128b44d279SVlastimil Babka * aborts. Sync compaction schedules. 4138b44d279SVlastimil Babka * Either compaction type will also abort if a fatal signal is pending. 4148b44d279SVlastimil Babka * In either case if the lock was locked, it is dropped and not regained. 415c67fe375SMel Gorman * 4168b44d279SVlastimil Babka * Returns true if compaction should abort due to fatal signal pending, or 4178b44d279SVlastimil Babka * async compaction due to need_resched() 4188b44d279SVlastimil Babka * Returns false when compaction can continue (sync compaction might have 4198b44d279SVlastimil Babka * scheduled) 420c67fe375SMel Gorman */ 4218b44d279SVlastimil Babka static bool compact_unlock_should_abort(spinlock_t *lock, 4228b44d279SVlastimil Babka unsigned long flags, bool *locked, struct compact_control *cc) 423c67fe375SMel Gorman { 4248b44d279SVlastimil Babka if (*locked) { 4258b44d279SVlastimil Babka spin_unlock_irqrestore(lock, flags); 4268b44d279SVlastimil Babka *locked = false; 427c67fe375SMel Gorman } 428c67fe375SMel Gorman 4298b44d279SVlastimil Babka if (fatal_signal_pending(current)) { 430c3486f53SVlastimil Babka cc->contended = true; 4318b44d279SVlastimil Babka return true; 4328b44d279SVlastimil Babka } 4338b44d279SVlastimil Babka 4348b44d279SVlastimil Babka if (need_resched()) { 435e0b9daebSDavid Rientjes if (cc->mode == MIGRATE_ASYNC) { 436c3486f53SVlastimil Babka cc->contended = true; 4378b44d279SVlastimil Babka return true; 438c67fe375SMel Gorman } 439c67fe375SMel Gorman cond_resched(); 440c67fe375SMel Gorman } 441c67fe375SMel Gorman 4428b44d279SVlastimil Babka return false; 443c67fe375SMel Gorman } 444c67fe375SMel Gorman 445be976572SVlastimil Babka /* 446be976572SVlastimil Babka * Aside from avoiding lock contention, compaction also periodically checks 447be976572SVlastimil Babka * need_resched() and either schedules in sync compaction or aborts async 4488b44d279SVlastimil Babka * compaction. This is similar to what compact_unlock_should_abort() does, but 449be976572SVlastimil Babka * is used where no lock is concerned. 450be976572SVlastimil Babka * 451be976572SVlastimil Babka * Returns false when no scheduling was needed, or sync compaction scheduled. 452be976572SVlastimil Babka * Returns true when async compaction should abort. 453be976572SVlastimil Babka */ 454be976572SVlastimil Babka static inline bool compact_should_abort(struct compact_control *cc) 455be976572SVlastimil Babka { 456be976572SVlastimil Babka /* async compaction aborts if contended */ 457be976572SVlastimil Babka if (need_resched()) { 458be976572SVlastimil Babka if (cc->mode == MIGRATE_ASYNC) { 459c3486f53SVlastimil Babka cc->contended = true; 460be976572SVlastimil Babka return true; 461be976572SVlastimil Babka } 462be976572SVlastimil Babka 463be976572SVlastimil Babka cond_resched(); 464be976572SVlastimil Babka } 465be976572SVlastimil Babka 466be976572SVlastimil Babka return false; 467be976572SVlastimil Babka } 468be976572SVlastimil Babka 469c67fe375SMel Gorman /* 4709e4be470SJerome Marchand * Isolate free pages onto a private freelist. If @strict is true, will abort 4719e4be470SJerome Marchand * returning 0 on any invalid PFNs or non-free pages inside of the pageblock 4729e4be470SJerome Marchand * (even though it may still end up isolating some pages). 47385aa125fSMichal Nazarewicz */ 474f40d1e42SMel Gorman static unsigned long isolate_freepages_block(struct compact_control *cc, 475e14c720eSVlastimil Babka unsigned long *start_pfn, 47685aa125fSMichal Nazarewicz unsigned long end_pfn, 47785aa125fSMichal Nazarewicz struct list_head *freelist, 47885aa125fSMichal Nazarewicz bool strict) 479748446bbSMel Gorman { 480b7aba698SMel Gorman int nr_scanned = 0, total_isolated = 0; 481bb13ffebSMel Gorman struct page *cursor, *valid_page = NULL; 482b8b2d825SXiubo Li unsigned long flags = 0; 483f40d1e42SMel Gorman bool locked = false; 484e14c720eSVlastimil Babka unsigned long blockpfn = *start_pfn; 48566c64223SJoonsoo Kim unsigned int order; 486748446bbSMel Gorman 487748446bbSMel Gorman cursor = pfn_to_page(blockpfn); 488748446bbSMel Gorman 489f40d1e42SMel Gorman /* Isolate free pages. */ 490748446bbSMel Gorman for (; blockpfn < end_pfn; blockpfn++, cursor++) { 49166c64223SJoonsoo Kim int isolated; 492748446bbSMel Gorman struct page *page = cursor; 493748446bbSMel Gorman 4948b44d279SVlastimil Babka /* 4958b44d279SVlastimil Babka * Periodically drop the lock (if held) regardless of its 4968b44d279SVlastimil Babka * contention, to give chance to IRQs. Abort if fatal signal 4978b44d279SVlastimil Babka * pending or async compaction detects need_resched() 4988b44d279SVlastimil Babka */ 4998b44d279SVlastimil Babka if (!(blockpfn % SWAP_CLUSTER_MAX) 5008b44d279SVlastimil Babka && compact_unlock_should_abort(&cc->zone->lock, flags, 5018b44d279SVlastimil Babka &locked, cc)) 5028b44d279SVlastimil Babka break; 5038b44d279SVlastimil Babka 504b7aba698SMel Gorman nr_scanned++; 505f40d1e42SMel Gorman if (!pfn_valid_within(blockpfn)) 5062af120bcSLaura Abbott goto isolate_fail; 5072af120bcSLaura Abbott 508bb13ffebSMel Gorman if (!valid_page) 509bb13ffebSMel Gorman valid_page = page; 5109fcd6d2eSVlastimil Babka 5119fcd6d2eSVlastimil Babka /* 5129fcd6d2eSVlastimil Babka * For compound pages such as THP and hugetlbfs, we can save 5139fcd6d2eSVlastimil Babka * potentially a lot of iterations if we skip them at once. 5149fcd6d2eSVlastimil Babka * The check is racy, but we can consider only valid values 5159fcd6d2eSVlastimil Babka * and the only danger is skipping too much. 5169fcd6d2eSVlastimil Babka */ 5179fcd6d2eSVlastimil Babka if (PageCompound(page)) { 51821dc7e02SDavid Rientjes const unsigned int order = compound_order(page); 5199fcd6d2eSVlastimil Babka 520d3c85badSVlastimil Babka if (likely(order < MAX_ORDER)) { 52121dc7e02SDavid Rientjes blockpfn += (1UL << order) - 1; 52221dc7e02SDavid Rientjes cursor += (1UL << order) - 1; 5239fcd6d2eSVlastimil Babka } 5249fcd6d2eSVlastimil Babka goto isolate_fail; 5259fcd6d2eSVlastimil Babka } 5269fcd6d2eSVlastimil Babka 527f40d1e42SMel Gorman if (!PageBuddy(page)) 5282af120bcSLaura Abbott goto isolate_fail; 529f40d1e42SMel Gorman 530f40d1e42SMel Gorman /* 53169b7189fSVlastimil Babka * If we already hold the lock, we can skip some rechecking. 53269b7189fSVlastimil Babka * Note that if we hold the lock now, checked_pageblock was 53369b7189fSVlastimil Babka * already set in some previous iteration (or strict is true), 53469b7189fSVlastimil Babka * so it is correct to skip the suitable migration target 53569b7189fSVlastimil Babka * recheck as well. 53669b7189fSVlastimil Babka */ 53769b7189fSVlastimil Babka if (!locked) { 53869b7189fSVlastimil Babka /* 539f40d1e42SMel Gorman * The zone lock must be held to isolate freepages. 540f40d1e42SMel Gorman * Unfortunately this is a very coarse lock and can be 541f40d1e42SMel Gorman * heavily contended if there are parallel allocations 542f40d1e42SMel Gorman * or parallel compactions. For async compaction do not 543f40d1e42SMel Gorman * spin on the lock and we acquire the lock as late as 544f40d1e42SMel Gorman * possible. 545f40d1e42SMel Gorman */ 5468b44d279SVlastimil Babka locked = compact_trylock_irqsave(&cc->zone->lock, 5478b44d279SVlastimil Babka &flags, cc); 548f40d1e42SMel Gorman if (!locked) 549f40d1e42SMel Gorman break; 550f40d1e42SMel Gorman 551f40d1e42SMel Gorman /* Recheck this is a buddy page under lock */ 552f40d1e42SMel Gorman if (!PageBuddy(page)) 5532af120bcSLaura Abbott goto isolate_fail; 55469b7189fSVlastimil Babka } 555748446bbSMel Gorman 55666c64223SJoonsoo Kim /* Found a free page, will break it into order-0 pages */ 55766c64223SJoonsoo Kim order = page_order(page); 55866c64223SJoonsoo Kim isolated = __isolate_free_page(page, order); 559a4f04f2cSDavid Rientjes if (!isolated) 560a4f04f2cSDavid Rientjes break; 56166c64223SJoonsoo Kim set_page_private(page, order); 562a4f04f2cSDavid Rientjes 563748446bbSMel Gorman total_isolated += isolated; 564a4f04f2cSDavid Rientjes cc->nr_freepages += isolated; 56566c64223SJoonsoo Kim list_add_tail(&page->lru, freelist); 56666c64223SJoonsoo Kim 567a4f04f2cSDavid Rientjes if (!strict && cc->nr_migratepages <= cc->nr_freepages) { 568932ff6bbSJoonsoo Kim blockpfn += isolated; 569932ff6bbSJoonsoo Kim break; 570932ff6bbSJoonsoo Kim } 571a4f04f2cSDavid Rientjes /* Advance to the end of split page */ 572748446bbSMel Gorman blockpfn += isolated - 1; 573748446bbSMel Gorman cursor += isolated - 1; 5742af120bcSLaura Abbott continue; 5752af120bcSLaura Abbott 5762af120bcSLaura Abbott isolate_fail: 5772af120bcSLaura Abbott if (strict) 5782af120bcSLaura Abbott break; 5792af120bcSLaura Abbott else 5802af120bcSLaura Abbott continue; 5812af120bcSLaura Abbott 582748446bbSMel Gorman } 583748446bbSMel Gorman 584a4f04f2cSDavid Rientjes if (locked) 585a4f04f2cSDavid Rientjes spin_unlock_irqrestore(&cc->zone->lock, flags); 586a4f04f2cSDavid Rientjes 5879fcd6d2eSVlastimil Babka /* 5889fcd6d2eSVlastimil Babka * There is a tiny chance that we have read bogus compound_order(), 5899fcd6d2eSVlastimil Babka * so be careful to not go outside of the pageblock. 5909fcd6d2eSVlastimil Babka */ 5919fcd6d2eSVlastimil Babka if (unlikely(blockpfn > end_pfn)) 5929fcd6d2eSVlastimil Babka blockpfn = end_pfn; 5939fcd6d2eSVlastimil Babka 594e34d85f0SJoonsoo Kim trace_mm_compaction_isolate_freepages(*start_pfn, blockpfn, 595e34d85f0SJoonsoo Kim nr_scanned, total_isolated); 596e34d85f0SJoonsoo Kim 597e14c720eSVlastimil Babka /* Record how far we have got within the block */ 598e14c720eSVlastimil Babka *start_pfn = blockpfn; 599e14c720eSVlastimil Babka 600f40d1e42SMel Gorman /* 601f40d1e42SMel Gorman * If strict isolation is requested by CMA then check that all the 602f40d1e42SMel Gorman * pages requested were isolated. If there were any failures, 0 is 603f40d1e42SMel Gorman * returned and CMA will fail. 604f40d1e42SMel Gorman */ 6052af120bcSLaura Abbott if (strict && blockpfn < end_pfn) 606f40d1e42SMel Gorman total_isolated = 0; 607f40d1e42SMel Gorman 608bb13ffebSMel Gorman /* Update the pageblock-skip if the whole pageblock was scanned */ 609bb13ffebSMel Gorman if (blockpfn == end_pfn) 610*e380bebeSMel Gorman update_pageblock_skip(cc, valid_page, total_isolated); 611bb13ffebSMel Gorman 6127f354a54SDavid Rientjes cc->total_free_scanned += nr_scanned; 613397487dbSMel Gorman if (total_isolated) 614010fc29aSMinchan Kim count_compact_events(COMPACTISOLATED, total_isolated); 615748446bbSMel Gorman return total_isolated; 616748446bbSMel Gorman } 617748446bbSMel Gorman 61885aa125fSMichal Nazarewicz /** 61985aa125fSMichal Nazarewicz * isolate_freepages_range() - isolate free pages. 620e8b098fcSMike Rapoport * @cc: Compaction control structure. 62185aa125fSMichal Nazarewicz * @start_pfn: The first PFN to start isolating. 62285aa125fSMichal Nazarewicz * @end_pfn: The one-past-last PFN. 62385aa125fSMichal Nazarewicz * 62485aa125fSMichal Nazarewicz * Non-free pages, invalid PFNs, or zone boundaries within the 62585aa125fSMichal Nazarewicz * [start_pfn, end_pfn) range are considered errors, cause function to 62685aa125fSMichal Nazarewicz * undo its actions and return zero. 62785aa125fSMichal Nazarewicz * 62885aa125fSMichal Nazarewicz * Otherwise, function returns one-past-the-last PFN of isolated page 62985aa125fSMichal Nazarewicz * (which may be greater then end_pfn if end fell in a middle of 63085aa125fSMichal Nazarewicz * a free page). 63185aa125fSMichal Nazarewicz */ 632ff9543fdSMichal Nazarewicz unsigned long 633bb13ffebSMel Gorman isolate_freepages_range(struct compact_control *cc, 634bb13ffebSMel Gorman unsigned long start_pfn, unsigned long end_pfn) 63585aa125fSMichal Nazarewicz { 636e1409c32SJoonsoo Kim unsigned long isolated, pfn, block_start_pfn, block_end_pfn; 63785aa125fSMichal Nazarewicz LIST_HEAD(freelist); 63885aa125fSMichal Nazarewicz 6397d49d886SVlastimil Babka pfn = start_pfn; 64006b6640aSVlastimil Babka block_start_pfn = pageblock_start_pfn(pfn); 641e1409c32SJoonsoo Kim if (block_start_pfn < cc->zone->zone_start_pfn) 642e1409c32SJoonsoo Kim block_start_pfn = cc->zone->zone_start_pfn; 64306b6640aSVlastimil Babka block_end_pfn = pageblock_end_pfn(pfn); 6447d49d886SVlastimil Babka 6457d49d886SVlastimil Babka for (; pfn < end_pfn; pfn += isolated, 646e1409c32SJoonsoo Kim block_start_pfn = block_end_pfn, 6477d49d886SVlastimil Babka block_end_pfn += pageblock_nr_pages) { 648e14c720eSVlastimil Babka /* Protect pfn from changing by isolate_freepages_block */ 649e14c720eSVlastimil Babka unsigned long isolate_start_pfn = pfn; 6507d49d886SVlastimil Babka 65185aa125fSMichal Nazarewicz block_end_pfn = min(block_end_pfn, end_pfn); 65285aa125fSMichal Nazarewicz 65358420016SJoonsoo Kim /* 65458420016SJoonsoo Kim * pfn could pass the block_end_pfn if isolated freepage 65558420016SJoonsoo Kim * is more than pageblock order. In this case, we adjust 65658420016SJoonsoo Kim * scanning range to right one. 65758420016SJoonsoo Kim */ 65858420016SJoonsoo Kim if (pfn >= block_end_pfn) { 65906b6640aSVlastimil Babka block_start_pfn = pageblock_start_pfn(pfn); 66006b6640aSVlastimil Babka block_end_pfn = pageblock_end_pfn(pfn); 66158420016SJoonsoo Kim block_end_pfn = min(block_end_pfn, end_pfn); 66258420016SJoonsoo Kim } 66358420016SJoonsoo Kim 664e1409c32SJoonsoo Kim if (!pageblock_pfn_to_page(block_start_pfn, 665e1409c32SJoonsoo Kim block_end_pfn, cc->zone)) 6667d49d886SVlastimil Babka break; 6677d49d886SVlastimil Babka 668e14c720eSVlastimil Babka isolated = isolate_freepages_block(cc, &isolate_start_pfn, 669e14c720eSVlastimil Babka block_end_pfn, &freelist, true); 67085aa125fSMichal Nazarewicz 67185aa125fSMichal Nazarewicz /* 67285aa125fSMichal Nazarewicz * In strict mode, isolate_freepages_block() returns 0 if 67385aa125fSMichal Nazarewicz * there are any holes in the block (ie. invalid PFNs or 67485aa125fSMichal Nazarewicz * non-free pages). 67585aa125fSMichal Nazarewicz */ 67685aa125fSMichal Nazarewicz if (!isolated) 67785aa125fSMichal Nazarewicz break; 67885aa125fSMichal Nazarewicz 67985aa125fSMichal Nazarewicz /* 68085aa125fSMichal Nazarewicz * If we managed to isolate pages, it is always (1 << n) * 68185aa125fSMichal Nazarewicz * pageblock_nr_pages for some non-negative n. (Max order 68285aa125fSMichal Nazarewicz * page may span two pageblocks). 68385aa125fSMichal Nazarewicz */ 68485aa125fSMichal Nazarewicz } 68585aa125fSMichal Nazarewicz 68666c64223SJoonsoo Kim /* __isolate_free_page() does not map the pages */ 6874469ab98SMel Gorman split_map_pages(&freelist); 68885aa125fSMichal Nazarewicz 68985aa125fSMichal Nazarewicz if (pfn < end_pfn) { 69085aa125fSMichal Nazarewicz /* Loop terminated early, cleanup. */ 69185aa125fSMichal Nazarewicz release_freepages(&freelist); 69285aa125fSMichal Nazarewicz return 0; 69385aa125fSMichal Nazarewicz } 69485aa125fSMichal Nazarewicz 69585aa125fSMichal Nazarewicz /* We don't use freelists for anything. */ 69685aa125fSMichal Nazarewicz return pfn; 69785aa125fSMichal Nazarewicz } 69885aa125fSMichal Nazarewicz 699748446bbSMel Gorman /* Similar to reclaim, but different enough that they don't share logic */ 700748446bbSMel Gorman static bool too_many_isolated(struct zone *zone) 701748446bbSMel Gorman { 702bc693045SMinchan Kim unsigned long active, inactive, isolated; 703748446bbSMel Gorman 704599d0c95SMel Gorman inactive = node_page_state(zone->zone_pgdat, NR_INACTIVE_FILE) + 705599d0c95SMel Gorman node_page_state(zone->zone_pgdat, NR_INACTIVE_ANON); 706599d0c95SMel Gorman active = node_page_state(zone->zone_pgdat, NR_ACTIVE_FILE) + 707599d0c95SMel Gorman node_page_state(zone->zone_pgdat, NR_ACTIVE_ANON); 708599d0c95SMel Gorman isolated = node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE) + 709599d0c95SMel Gorman node_page_state(zone->zone_pgdat, NR_ISOLATED_ANON); 710748446bbSMel Gorman 711bc693045SMinchan Kim return isolated > (inactive + active) / 2; 712748446bbSMel Gorman } 713748446bbSMel Gorman 7142fe86e00SMichal Nazarewicz /** 715edc2ca61SVlastimil Babka * isolate_migratepages_block() - isolate all migrate-able pages within 716edc2ca61SVlastimil Babka * a single pageblock 7172fe86e00SMichal Nazarewicz * @cc: Compaction control structure. 718edc2ca61SVlastimil Babka * @low_pfn: The first PFN to isolate 719edc2ca61SVlastimil Babka * @end_pfn: The one-past-the-last PFN to isolate, within same pageblock 720edc2ca61SVlastimil Babka * @isolate_mode: Isolation mode to be used. 7212fe86e00SMichal Nazarewicz * 7222fe86e00SMichal Nazarewicz * Isolate all pages that can be migrated from the range specified by 723edc2ca61SVlastimil Babka * [low_pfn, end_pfn). The range is expected to be within same pageblock. 724edc2ca61SVlastimil Babka * Returns zero if there is a fatal signal pending, otherwise PFN of the 725edc2ca61SVlastimil Babka * first page that was not scanned (which may be both less, equal to or more 726edc2ca61SVlastimil Babka * than end_pfn). 7272fe86e00SMichal Nazarewicz * 728edc2ca61SVlastimil Babka * The pages are isolated on cc->migratepages list (not required to be empty), 729edc2ca61SVlastimil Babka * and cc->nr_migratepages is updated accordingly. The cc->migrate_pfn field 730edc2ca61SVlastimil Babka * is neither read nor updated. 731748446bbSMel Gorman */ 732edc2ca61SVlastimil Babka static unsigned long 733edc2ca61SVlastimil Babka isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, 734edc2ca61SVlastimil Babka unsigned long end_pfn, isolate_mode_t isolate_mode) 735748446bbSMel Gorman { 736edc2ca61SVlastimil Babka struct zone *zone = cc->zone; 737b7aba698SMel Gorman unsigned long nr_scanned = 0, nr_isolated = 0; 738fa9add64SHugh Dickins struct lruvec *lruvec; 739b8b2d825SXiubo Li unsigned long flags = 0; 7402a1402aaSMel Gorman bool locked = false; 741bb13ffebSMel Gorman struct page *page = NULL, *valid_page = NULL; 742e34d85f0SJoonsoo Kim unsigned long start_pfn = low_pfn; 743fdd048e1SVlastimil Babka bool skip_on_failure = false; 744fdd048e1SVlastimil Babka unsigned long next_skip_pfn = 0; 745*e380bebeSMel Gorman bool skip_updated = false; 746748446bbSMel Gorman 747748446bbSMel Gorman /* 748748446bbSMel Gorman * Ensure that there are not too many pages isolated from the LRU 749748446bbSMel Gorman * list by either parallel reclaimers or compaction. If there are, 750748446bbSMel Gorman * delay for some time until fewer pages are isolated 751748446bbSMel Gorman */ 752748446bbSMel Gorman while (unlikely(too_many_isolated(zone))) { 753f9e35b3bSMel Gorman /* async migration should just abort */ 754e0b9daebSDavid Rientjes if (cc->mode == MIGRATE_ASYNC) 7552fe86e00SMichal Nazarewicz return 0; 756f9e35b3bSMel Gorman 757748446bbSMel Gorman congestion_wait(BLK_RW_ASYNC, HZ/10); 758748446bbSMel Gorman 759748446bbSMel Gorman if (fatal_signal_pending(current)) 7602fe86e00SMichal Nazarewicz return 0; 761748446bbSMel Gorman } 762748446bbSMel Gorman 763be976572SVlastimil Babka if (compact_should_abort(cc)) 764aeef4b83SDavid Rientjes return 0; 765aeef4b83SDavid Rientjes 766fdd048e1SVlastimil Babka if (cc->direct_compaction && (cc->mode == MIGRATE_ASYNC)) { 767fdd048e1SVlastimil Babka skip_on_failure = true; 768fdd048e1SVlastimil Babka next_skip_pfn = block_end_pfn(low_pfn, cc->order); 769fdd048e1SVlastimil Babka } 770fdd048e1SVlastimil Babka 771748446bbSMel Gorman /* Time to isolate some pages for migration */ 772748446bbSMel Gorman for (; low_pfn < end_pfn; low_pfn++) { 77329c0dde8SVlastimil Babka 774fdd048e1SVlastimil Babka if (skip_on_failure && low_pfn >= next_skip_pfn) { 775fdd048e1SVlastimil Babka /* 776fdd048e1SVlastimil Babka * We have isolated all migration candidates in the 777fdd048e1SVlastimil Babka * previous order-aligned block, and did not skip it due 778fdd048e1SVlastimil Babka * to failure. We should migrate the pages now and 779fdd048e1SVlastimil Babka * hopefully succeed compaction. 780fdd048e1SVlastimil Babka */ 781fdd048e1SVlastimil Babka if (nr_isolated) 782fdd048e1SVlastimil Babka break; 783fdd048e1SVlastimil Babka 784fdd048e1SVlastimil Babka /* 785fdd048e1SVlastimil Babka * We failed to isolate in the previous order-aligned 786fdd048e1SVlastimil Babka * block. Set the new boundary to the end of the 787fdd048e1SVlastimil Babka * current block. Note we can't simply increase 788fdd048e1SVlastimil Babka * next_skip_pfn by 1 << order, as low_pfn might have 789fdd048e1SVlastimil Babka * been incremented by a higher number due to skipping 790fdd048e1SVlastimil Babka * a compound or a high-order buddy page in the 791fdd048e1SVlastimil Babka * previous loop iteration. 792fdd048e1SVlastimil Babka */ 793fdd048e1SVlastimil Babka next_skip_pfn = block_end_pfn(low_pfn, cc->order); 794fdd048e1SVlastimil Babka } 795fdd048e1SVlastimil Babka 7968b44d279SVlastimil Babka /* 7978b44d279SVlastimil Babka * Periodically drop the lock (if held) regardless of its 7988b44d279SVlastimil Babka * contention, to give chance to IRQs. Abort async compaction 7998b44d279SVlastimil Babka * if contended. 8008b44d279SVlastimil Babka */ 8018b44d279SVlastimil Babka if (!(low_pfn % SWAP_CLUSTER_MAX) 802a52633d8SMel Gorman && compact_unlock_should_abort(zone_lru_lock(zone), flags, 8038b44d279SVlastimil Babka &locked, cc)) 8048b44d279SVlastimil Babka break; 805b2eef8c0SAndrea Arcangeli 806748446bbSMel Gorman if (!pfn_valid_within(low_pfn)) 807fdd048e1SVlastimil Babka goto isolate_fail; 808b7aba698SMel Gorman nr_scanned++; 809748446bbSMel Gorman 810748446bbSMel Gorman page = pfn_to_page(low_pfn); 811dc908600SMel Gorman 812*e380bebeSMel Gorman /* 813*e380bebeSMel Gorman * Check if the pageblock has already been marked skipped. 814*e380bebeSMel Gorman * Only the aligned PFN is checked as the caller isolates 815*e380bebeSMel Gorman * COMPACT_CLUSTER_MAX at a time so the second call must 816*e380bebeSMel Gorman * not falsely conclude that the block should be skipped. 817*e380bebeSMel Gorman */ 818*e380bebeSMel Gorman if (!valid_page && IS_ALIGNED(low_pfn, pageblock_nr_pages)) { 819*e380bebeSMel Gorman if (!cc->ignore_skip_hint && get_pageblock_skip(page)) { 820*e380bebeSMel Gorman low_pfn = end_pfn; 821*e380bebeSMel Gorman goto isolate_abort; 822*e380bebeSMel Gorman } 823bb13ffebSMel Gorman valid_page = page; 824*e380bebeSMel Gorman } 825bb13ffebSMel Gorman 826c122b208SJoonsoo Kim /* 82799c0fd5eSVlastimil Babka * Skip if free. We read page order here without zone lock 82899c0fd5eSVlastimil Babka * which is generally unsafe, but the race window is small and 82999c0fd5eSVlastimil Babka * the worst thing that can happen is that we skip some 83099c0fd5eSVlastimil Babka * potential isolation targets. 8316c14466cSMel Gorman */ 83299c0fd5eSVlastimil Babka if (PageBuddy(page)) { 83399c0fd5eSVlastimil Babka unsigned long freepage_order = page_order_unsafe(page); 83499c0fd5eSVlastimil Babka 83599c0fd5eSVlastimil Babka /* 83699c0fd5eSVlastimil Babka * Without lock, we cannot be sure that what we got is 83799c0fd5eSVlastimil Babka * a valid page order. Consider only values in the 83899c0fd5eSVlastimil Babka * valid order range to prevent low_pfn overflow. 83999c0fd5eSVlastimil Babka */ 84099c0fd5eSVlastimil Babka if (freepage_order > 0 && freepage_order < MAX_ORDER) 84199c0fd5eSVlastimil Babka low_pfn += (1UL << freepage_order) - 1; 842748446bbSMel Gorman continue; 84399c0fd5eSVlastimil Babka } 844748446bbSMel Gorman 8459927af74SMel Gorman /* 84629c0dde8SVlastimil Babka * Regardless of being on LRU, compound pages such as THP and 84729c0dde8SVlastimil Babka * hugetlbfs are not to be compacted. We can potentially save 84829c0dde8SVlastimil Babka * a lot of iterations if we skip them at once. The check is 84929c0dde8SVlastimil Babka * racy, but we can consider only valid values and the only 85029c0dde8SVlastimil Babka * danger is skipping too much. 851bc835011SAndrea Arcangeli */ 85229c0dde8SVlastimil Babka if (PageCompound(page)) { 85321dc7e02SDavid Rientjes const unsigned int order = compound_order(page); 85429c0dde8SVlastimil Babka 855d3c85badSVlastimil Babka if (likely(order < MAX_ORDER)) 85621dc7e02SDavid Rientjes low_pfn += (1UL << order) - 1; 857fdd048e1SVlastimil Babka goto isolate_fail; 8582a1402aaSMel Gorman } 8592a1402aaSMel Gorman 860bda807d4SMinchan Kim /* 861bda807d4SMinchan Kim * Check may be lockless but that's ok as we recheck later. 862bda807d4SMinchan Kim * It's possible to migrate LRU and non-lru movable pages. 863bda807d4SMinchan Kim * Skip any other type of page 864bda807d4SMinchan Kim */ 865bda807d4SMinchan Kim if (!PageLRU(page)) { 866bda807d4SMinchan Kim /* 867bda807d4SMinchan Kim * __PageMovable can return false positive so we need 868bda807d4SMinchan Kim * to verify it under page_lock. 869bda807d4SMinchan Kim */ 870bda807d4SMinchan Kim if (unlikely(__PageMovable(page)) && 871bda807d4SMinchan Kim !PageIsolated(page)) { 872bda807d4SMinchan Kim if (locked) { 873a52633d8SMel Gorman spin_unlock_irqrestore(zone_lru_lock(zone), 874bda807d4SMinchan Kim flags); 875bda807d4SMinchan Kim locked = false; 876bda807d4SMinchan Kim } 877bda807d4SMinchan Kim 8789e5bcd61SYisheng Xie if (!isolate_movable_page(page, isolate_mode)) 879bda807d4SMinchan Kim goto isolate_success; 880bda807d4SMinchan Kim } 881bda807d4SMinchan Kim 882fdd048e1SVlastimil Babka goto isolate_fail; 883bda807d4SMinchan Kim } 88429c0dde8SVlastimil Babka 885119d6d59SDavid Rientjes /* 886119d6d59SDavid Rientjes * Migration will fail if an anonymous page is pinned in memory, 887119d6d59SDavid Rientjes * so avoid taking lru_lock and isolating it unnecessarily in an 888119d6d59SDavid Rientjes * admittedly racy check. 889119d6d59SDavid Rientjes */ 890119d6d59SDavid Rientjes if (!page_mapping(page) && 891119d6d59SDavid Rientjes page_count(page) > page_mapcount(page)) 892fdd048e1SVlastimil Babka goto isolate_fail; 893119d6d59SDavid Rientjes 89473e64c51SMichal Hocko /* 89573e64c51SMichal Hocko * Only allow to migrate anonymous pages in GFP_NOFS context 89673e64c51SMichal Hocko * because those do not depend on fs locks. 89773e64c51SMichal Hocko */ 89873e64c51SMichal Hocko if (!(cc->gfp_mask & __GFP_FS) && page_mapping(page)) 89973e64c51SMichal Hocko goto isolate_fail; 90073e64c51SMichal Hocko 90169b7189fSVlastimil Babka /* If we already hold the lock, we can skip some rechecking */ 90269b7189fSVlastimil Babka if (!locked) { 903a52633d8SMel Gorman locked = compact_trylock_irqsave(zone_lru_lock(zone), 9048b44d279SVlastimil Babka &flags, cc); 905*e380bebeSMel Gorman 906*e380bebeSMel Gorman /* Allow future scanning if the lock is contended */ 907*e380bebeSMel Gorman if (!locked) { 908*e380bebeSMel Gorman clear_pageblock_skip(page); 9092a1402aaSMel Gorman break; 910*e380bebeSMel Gorman } 911*e380bebeSMel Gorman 912*e380bebeSMel Gorman /* Try get exclusive access under lock */ 913*e380bebeSMel Gorman if (!skip_updated) { 914*e380bebeSMel Gorman skip_updated = true; 915*e380bebeSMel Gorman if (test_and_set_skip(cc, page, low_pfn)) 916*e380bebeSMel Gorman goto isolate_abort; 917*e380bebeSMel Gorman } 9182a1402aaSMel Gorman 91929c0dde8SVlastimil Babka /* Recheck PageLRU and PageCompound under lock */ 9202a1402aaSMel Gorman if (!PageLRU(page)) 921fdd048e1SVlastimil Babka goto isolate_fail; 92229c0dde8SVlastimil Babka 92329c0dde8SVlastimil Babka /* 92429c0dde8SVlastimil Babka * Page become compound since the non-locked check, 92529c0dde8SVlastimil Babka * and it's on LRU. It can only be a THP so the order 92629c0dde8SVlastimil Babka * is safe to read and it's 0 for tail pages. 92729c0dde8SVlastimil Babka */ 92829c0dde8SVlastimil Babka if (unlikely(PageCompound(page))) { 929d3c85badSVlastimil Babka low_pfn += (1UL << compound_order(page)) - 1; 930fdd048e1SVlastimil Babka goto isolate_fail; 931bc835011SAndrea Arcangeli } 93269b7189fSVlastimil Babka } 933bc835011SAndrea Arcangeli 934599d0c95SMel Gorman lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat); 935fa9add64SHugh Dickins 936748446bbSMel Gorman /* Try isolate the page */ 937edc2ca61SVlastimil Babka if (__isolate_lru_page(page, isolate_mode) != 0) 938fdd048e1SVlastimil Babka goto isolate_fail; 939748446bbSMel Gorman 94029c0dde8SVlastimil Babka VM_BUG_ON_PAGE(PageCompound(page), page); 941bc835011SAndrea Arcangeli 942748446bbSMel Gorman /* Successfully isolated */ 943fa9add64SHugh Dickins del_page_from_lru_list(page, lruvec, page_lru(page)); 9446afcf8efSMing Ling inc_node_page_state(page, 9456afcf8efSMing Ling NR_ISOLATED_ANON + page_is_file_cache(page)); 946b6c75016SJoonsoo Kim 947b6c75016SJoonsoo Kim isolate_success: 948fdd048e1SVlastimil Babka list_add(&page->lru, &cc->migratepages); 949748446bbSMel Gorman cc->nr_migratepages++; 950b7aba698SMel Gorman nr_isolated++; 951748446bbSMel Gorman 952748446bbSMel Gorman /* Avoid isolating too much */ 95331b8384aSHillf Danton if (cc->nr_migratepages == COMPACT_CLUSTER_MAX) { 95431b8384aSHillf Danton ++low_pfn; 955748446bbSMel Gorman break; 956748446bbSMel Gorman } 957fdd048e1SVlastimil Babka 958fdd048e1SVlastimil Babka continue; 959fdd048e1SVlastimil Babka isolate_fail: 960fdd048e1SVlastimil Babka if (!skip_on_failure) 961fdd048e1SVlastimil Babka continue; 962fdd048e1SVlastimil Babka 963fdd048e1SVlastimil Babka /* 964fdd048e1SVlastimil Babka * We have isolated some pages, but then failed. Release them 965fdd048e1SVlastimil Babka * instead of migrating, as we cannot form the cc->order buddy 966fdd048e1SVlastimil Babka * page anyway. 967fdd048e1SVlastimil Babka */ 968fdd048e1SVlastimil Babka if (nr_isolated) { 969fdd048e1SVlastimil Babka if (locked) { 970a52633d8SMel Gorman spin_unlock_irqrestore(zone_lru_lock(zone), flags); 971fdd048e1SVlastimil Babka locked = false; 972fdd048e1SVlastimil Babka } 973fdd048e1SVlastimil Babka putback_movable_pages(&cc->migratepages); 974fdd048e1SVlastimil Babka cc->nr_migratepages = 0; 975fdd048e1SVlastimil Babka nr_isolated = 0; 976fdd048e1SVlastimil Babka } 977fdd048e1SVlastimil Babka 978fdd048e1SVlastimil Babka if (low_pfn < next_skip_pfn) { 979fdd048e1SVlastimil Babka low_pfn = next_skip_pfn - 1; 980fdd048e1SVlastimil Babka /* 981fdd048e1SVlastimil Babka * The check near the loop beginning would have updated 982fdd048e1SVlastimil Babka * next_skip_pfn too, but this is a bit simpler. 983fdd048e1SVlastimil Babka */ 984fdd048e1SVlastimil Babka next_skip_pfn += 1UL << cc->order; 985fdd048e1SVlastimil Babka } 98631b8384aSHillf Danton } 987748446bbSMel Gorman 98899c0fd5eSVlastimil Babka /* 98999c0fd5eSVlastimil Babka * The PageBuddy() check could have potentially brought us outside 99099c0fd5eSVlastimil Babka * the range to be scanned. 99199c0fd5eSVlastimil Babka */ 99299c0fd5eSVlastimil Babka if (unlikely(low_pfn > end_pfn)) 99399c0fd5eSVlastimil Babka low_pfn = end_pfn; 99499c0fd5eSVlastimil Babka 995*e380bebeSMel Gorman isolate_abort: 996c67fe375SMel Gorman if (locked) 997a52633d8SMel Gorman spin_unlock_irqrestore(zone_lru_lock(zone), flags); 998748446bbSMel Gorman 99950b5b094SVlastimil Babka /* 1000*e380bebeSMel Gorman * Updated the cached scanner pfn if the pageblock was scanned 1001*e380bebeSMel Gorman * without isolating a page. The pageblock may not be marked 1002*e380bebeSMel Gorman * skipped already if there were no LRU pages in the block. 100350b5b094SVlastimil Babka */ 1004*e380bebeSMel Gorman if (low_pfn == end_pfn && !nr_isolated) { 1005*e380bebeSMel Gorman if (valid_page && !skip_updated) 1006*e380bebeSMel Gorman set_pageblock_skip(valid_page); 1007*e380bebeSMel Gorman update_cached_migrate(cc, low_pfn); 1008*e380bebeSMel Gorman } 1009bb13ffebSMel Gorman 1010e34d85f0SJoonsoo Kim trace_mm_compaction_isolate_migratepages(start_pfn, low_pfn, 1011e34d85f0SJoonsoo Kim nr_scanned, nr_isolated); 1012b7aba698SMel Gorman 10137f354a54SDavid Rientjes cc->total_migrate_scanned += nr_scanned; 1014397487dbSMel Gorman if (nr_isolated) 1015010fc29aSMinchan Kim count_compact_events(COMPACTISOLATED, nr_isolated); 1016397487dbSMel Gorman 10172fe86e00SMichal Nazarewicz return low_pfn; 10182fe86e00SMichal Nazarewicz } 10192fe86e00SMichal Nazarewicz 1020edc2ca61SVlastimil Babka /** 1021edc2ca61SVlastimil Babka * isolate_migratepages_range() - isolate migrate-able pages in a PFN range 1022edc2ca61SVlastimil Babka * @cc: Compaction control structure. 1023edc2ca61SVlastimil Babka * @start_pfn: The first PFN to start isolating. 1024edc2ca61SVlastimil Babka * @end_pfn: The one-past-last PFN. 1025edc2ca61SVlastimil Babka * 1026edc2ca61SVlastimil Babka * Returns zero if isolation fails fatally due to e.g. pending signal. 1027edc2ca61SVlastimil Babka * Otherwise, function returns one-past-the-last PFN of isolated page 1028edc2ca61SVlastimil Babka * (which may be greater than end_pfn if end fell in a middle of a THP page). 1029edc2ca61SVlastimil Babka */ 1030edc2ca61SVlastimil Babka unsigned long 1031edc2ca61SVlastimil Babka isolate_migratepages_range(struct compact_control *cc, unsigned long start_pfn, 1032edc2ca61SVlastimil Babka unsigned long end_pfn) 1033edc2ca61SVlastimil Babka { 1034e1409c32SJoonsoo Kim unsigned long pfn, block_start_pfn, block_end_pfn; 1035edc2ca61SVlastimil Babka 1036edc2ca61SVlastimil Babka /* Scan block by block. First and last block may be incomplete */ 1037edc2ca61SVlastimil Babka pfn = start_pfn; 103806b6640aSVlastimil Babka block_start_pfn = pageblock_start_pfn(pfn); 1039e1409c32SJoonsoo Kim if (block_start_pfn < cc->zone->zone_start_pfn) 1040e1409c32SJoonsoo Kim block_start_pfn = cc->zone->zone_start_pfn; 104106b6640aSVlastimil Babka block_end_pfn = pageblock_end_pfn(pfn); 1042edc2ca61SVlastimil Babka 1043edc2ca61SVlastimil Babka for (; pfn < end_pfn; pfn = block_end_pfn, 1044e1409c32SJoonsoo Kim block_start_pfn = block_end_pfn, 1045edc2ca61SVlastimil Babka block_end_pfn += pageblock_nr_pages) { 1046edc2ca61SVlastimil Babka 1047edc2ca61SVlastimil Babka block_end_pfn = min(block_end_pfn, end_pfn); 1048edc2ca61SVlastimil Babka 1049e1409c32SJoonsoo Kim if (!pageblock_pfn_to_page(block_start_pfn, 1050e1409c32SJoonsoo Kim block_end_pfn, cc->zone)) 1051edc2ca61SVlastimil Babka continue; 1052edc2ca61SVlastimil Babka 1053edc2ca61SVlastimil Babka pfn = isolate_migratepages_block(cc, pfn, block_end_pfn, 1054edc2ca61SVlastimil Babka ISOLATE_UNEVICTABLE); 1055edc2ca61SVlastimil Babka 105614af4a5eSHugh Dickins if (!pfn) 1057edc2ca61SVlastimil Babka break; 10586ea41c0cSJoonsoo Kim 10596ea41c0cSJoonsoo Kim if (cc->nr_migratepages == COMPACT_CLUSTER_MAX) 10606ea41c0cSJoonsoo Kim break; 1061edc2ca61SVlastimil Babka } 1062edc2ca61SVlastimil Babka 1063edc2ca61SVlastimil Babka return pfn; 1064edc2ca61SVlastimil Babka } 1065edc2ca61SVlastimil Babka 1066ff9543fdSMichal Nazarewicz #endif /* CONFIG_COMPACTION || CONFIG_CMA */ 1067ff9543fdSMichal Nazarewicz #ifdef CONFIG_COMPACTION 1068018e9a49SAndrew Morton 1069b682debdSVlastimil Babka static bool suitable_migration_source(struct compact_control *cc, 1070b682debdSVlastimil Babka struct page *page) 1071b682debdSVlastimil Babka { 1072282722b0SVlastimil Babka int block_mt; 1073282722b0SVlastimil Babka 1074282722b0SVlastimil Babka if ((cc->mode != MIGRATE_ASYNC) || !cc->direct_compaction) 1075b682debdSVlastimil Babka return true; 1076b682debdSVlastimil Babka 1077282722b0SVlastimil Babka block_mt = get_pageblock_migratetype(page); 1078282722b0SVlastimil Babka 1079282722b0SVlastimil Babka if (cc->migratetype == MIGRATE_MOVABLE) 1080282722b0SVlastimil Babka return is_migrate_movable(block_mt); 1081282722b0SVlastimil Babka else 1082282722b0SVlastimil Babka return block_mt == cc->migratetype; 1083b682debdSVlastimil Babka } 1084b682debdSVlastimil Babka 1085018e9a49SAndrew Morton /* Returns true if the page is within a block suitable for migration to */ 10869f7e3387SVlastimil Babka static bool suitable_migration_target(struct compact_control *cc, 10879f7e3387SVlastimil Babka struct page *page) 1088018e9a49SAndrew Morton { 1089018e9a49SAndrew Morton /* If the page is a large free page, then disallow migration */ 1090018e9a49SAndrew Morton if (PageBuddy(page)) { 1091018e9a49SAndrew Morton /* 1092018e9a49SAndrew Morton * We are checking page_order without zone->lock taken. But 1093018e9a49SAndrew Morton * the only small danger is that we skip a potentially suitable 1094018e9a49SAndrew Morton * pageblock, so it's not worth to check order for valid range. 1095018e9a49SAndrew Morton */ 1096018e9a49SAndrew Morton if (page_order_unsafe(page) >= pageblock_order) 1097018e9a49SAndrew Morton return false; 1098018e9a49SAndrew Morton } 1099018e9a49SAndrew Morton 11001ef36db2SYisheng Xie if (cc->ignore_block_suitable) 11011ef36db2SYisheng Xie return true; 11021ef36db2SYisheng Xie 1103018e9a49SAndrew Morton /* If the block is MIGRATE_MOVABLE or MIGRATE_CMA, allow migration */ 1104b682debdSVlastimil Babka if (is_migrate_movable(get_pageblock_migratetype(page))) 1105018e9a49SAndrew Morton return true; 1106018e9a49SAndrew Morton 1107018e9a49SAndrew Morton /* Otherwise skip the block */ 1108018e9a49SAndrew Morton return false; 1109018e9a49SAndrew Morton } 1110018e9a49SAndrew Morton 111170b44595SMel Gorman static inline unsigned int 111270b44595SMel Gorman freelist_scan_limit(struct compact_control *cc) 111370b44595SMel Gorman { 111470b44595SMel Gorman return (COMPACT_CLUSTER_MAX >> cc->fast_search_fail) + 1; 111570b44595SMel Gorman } 111670b44595SMel Gorman 1117ff9543fdSMichal Nazarewicz /* 1118f2849aa0SVlastimil Babka * Test whether the free scanner has reached the same or lower pageblock than 1119f2849aa0SVlastimil Babka * the migration scanner, and compaction should thus terminate. 1120f2849aa0SVlastimil Babka */ 1121f2849aa0SVlastimil Babka static inline bool compact_scanners_met(struct compact_control *cc) 1122f2849aa0SVlastimil Babka { 1123f2849aa0SVlastimil Babka return (cc->free_pfn >> pageblock_order) 1124f2849aa0SVlastimil Babka <= (cc->migrate_pfn >> pageblock_order); 1125f2849aa0SVlastimil Babka } 1126f2849aa0SVlastimil Babka 112770b44595SMel Gorman /* Reorder the free list to reduce repeated future searches */ 112870b44595SMel Gorman static void 112970b44595SMel Gorman move_freelist_tail(struct list_head *freelist, struct page *freepage) 113070b44595SMel Gorman { 113170b44595SMel Gorman LIST_HEAD(sublist); 113270b44595SMel Gorman 113370b44595SMel Gorman if (!list_is_first(freelist, &freepage->lru)) { 113470b44595SMel Gorman list_cut_position(&sublist, freelist, &freepage->lru); 113570b44595SMel Gorman if (!list_empty(&sublist)) 113670b44595SMel Gorman list_splice_tail(&sublist, freelist); 113770b44595SMel Gorman } 113870b44595SMel Gorman } 113970b44595SMel Gorman 1140f2849aa0SVlastimil Babka /* 1141ff9543fdSMichal Nazarewicz * Based on information in the current compact_control, find blocks 1142ff9543fdSMichal Nazarewicz * suitable for isolating free pages from and then isolate them. 1143ff9543fdSMichal Nazarewicz */ 1144edc2ca61SVlastimil Babka static void isolate_freepages(struct compact_control *cc) 1145ff9543fdSMichal Nazarewicz { 1146edc2ca61SVlastimil Babka struct zone *zone = cc->zone; 1147ff9543fdSMichal Nazarewicz struct page *page; 1148c96b9e50SVlastimil Babka unsigned long block_start_pfn; /* start of current pageblock */ 1149e14c720eSVlastimil Babka unsigned long isolate_start_pfn; /* exact pfn we start at */ 1150c96b9e50SVlastimil Babka unsigned long block_end_pfn; /* end of current pageblock */ 1151c96b9e50SVlastimil Babka unsigned long low_pfn; /* lowest pfn scanner is able to scan */ 1152ff9543fdSMichal Nazarewicz struct list_head *freelist = &cc->freepages; 11532fe86e00SMichal Nazarewicz 1154ff9543fdSMichal Nazarewicz /* 1155ff9543fdSMichal Nazarewicz * Initialise the free scanner. The starting point is where we last 115649e068f0SVlastimil Babka * successfully isolated from, zone-cached value, or the end of the 1157e14c720eSVlastimil Babka * zone when isolating for the first time. For looping we also need 1158e14c720eSVlastimil Babka * this pfn aligned down to the pageblock boundary, because we do 1159c96b9e50SVlastimil Babka * block_start_pfn -= pageblock_nr_pages in the for loop. 1160c96b9e50SVlastimil Babka * For ending point, take care when isolating in last pageblock of a 1161c96b9e50SVlastimil Babka * a zone which ends in the middle of a pageblock. 116249e068f0SVlastimil Babka * The low boundary is the end of the pageblock the migration scanner 116349e068f0SVlastimil Babka * is using. 1164ff9543fdSMichal Nazarewicz */ 1165e14c720eSVlastimil Babka isolate_start_pfn = cc->free_pfn; 116606b6640aSVlastimil Babka block_start_pfn = pageblock_start_pfn(cc->free_pfn); 1167c96b9e50SVlastimil Babka block_end_pfn = min(block_start_pfn + pageblock_nr_pages, 1168c96b9e50SVlastimil Babka zone_end_pfn(zone)); 116906b6640aSVlastimil Babka low_pfn = pageblock_end_pfn(cc->migrate_pfn); 11702fe86e00SMichal Nazarewicz 1171ff9543fdSMichal Nazarewicz /* 1172ff9543fdSMichal Nazarewicz * Isolate free pages until enough are available to migrate the 1173ff9543fdSMichal Nazarewicz * pages on cc->migratepages. We stop searching if the migrate 1174ff9543fdSMichal Nazarewicz * and free page scanners meet or enough free pages are isolated. 1175ff9543fdSMichal Nazarewicz */ 1176f5f61a32SVlastimil Babka for (; block_start_pfn >= low_pfn; 1177c96b9e50SVlastimil Babka block_end_pfn = block_start_pfn, 1178e14c720eSVlastimil Babka block_start_pfn -= pageblock_nr_pages, 1179e14c720eSVlastimil Babka isolate_start_pfn = block_start_pfn) { 1180f6ea3adbSDavid Rientjes /* 1181f6ea3adbSDavid Rientjes * This can iterate a massively long zone without finding any 1182f6ea3adbSDavid Rientjes * suitable migration targets, so periodically check if we need 1183be976572SVlastimil Babka * to schedule, or even abort async compaction. 1184f6ea3adbSDavid Rientjes */ 1185be976572SVlastimil Babka if (!(block_start_pfn % (SWAP_CLUSTER_MAX * pageblock_nr_pages)) 1186be976572SVlastimil Babka && compact_should_abort(cc)) 1187be976572SVlastimil Babka break; 1188f6ea3adbSDavid Rientjes 11897d49d886SVlastimil Babka page = pageblock_pfn_to_page(block_start_pfn, block_end_pfn, 11907d49d886SVlastimil Babka zone); 11917d49d886SVlastimil Babka if (!page) 1192ff9543fdSMichal Nazarewicz continue; 1193ff9543fdSMichal Nazarewicz 1194ff9543fdSMichal Nazarewicz /* Check the block is suitable for migration */ 11959f7e3387SVlastimil Babka if (!suitable_migration_target(cc, page)) 1196ff9543fdSMichal Nazarewicz continue; 119768e3e926SLinus Torvalds 1198bb13ffebSMel Gorman /* If isolation recently failed, do not retry */ 1199bb13ffebSMel Gorman if (!isolation_suitable(cc, page)) 1200bb13ffebSMel Gorman continue; 1201bb13ffebSMel Gorman 1202e14c720eSVlastimil Babka /* Found a block suitable for isolating free pages from. */ 1203a46cbf3bSDavid Rientjes isolate_freepages_block(cc, &isolate_start_pfn, block_end_pfn, 1204a46cbf3bSDavid Rientjes freelist, false); 1205ff9543fdSMichal Nazarewicz 1206ff9543fdSMichal Nazarewicz /* 1207a46cbf3bSDavid Rientjes * If we isolated enough freepages, or aborted due to lock 1208a46cbf3bSDavid Rientjes * contention, terminate. 1209e14c720eSVlastimil Babka */ 1210f5f61a32SVlastimil Babka if ((cc->nr_freepages >= cc->nr_migratepages) 1211f5f61a32SVlastimil Babka || cc->contended) { 1212a46cbf3bSDavid Rientjes if (isolate_start_pfn >= block_end_pfn) { 1213a46cbf3bSDavid Rientjes /* 1214a46cbf3bSDavid Rientjes * Restart at previous pageblock if more 1215a46cbf3bSDavid Rientjes * freepages can be isolated next time. 1216a46cbf3bSDavid Rientjes */ 1217f5f61a32SVlastimil Babka isolate_start_pfn = 1218e14c720eSVlastimil Babka block_start_pfn - pageblock_nr_pages; 1219a46cbf3bSDavid Rientjes } 1220be976572SVlastimil Babka break; 1221a46cbf3bSDavid Rientjes } else if (isolate_start_pfn < block_end_pfn) { 1222f5f61a32SVlastimil Babka /* 1223a46cbf3bSDavid Rientjes * If isolation failed early, do not continue 1224a46cbf3bSDavid Rientjes * needlessly. 1225f5f61a32SVlastimil Babka */ 1226a46cbf3bSDavid Rientjes break; 1227f5f61a32SVlastimil Babka } 1228c89511abSMel Gorman } 1229ff9543fdSMichal Nazarewicz 123066c64223SJoonsoo Kim /* __isolate_free_page() does not map the pages */ 12314469ab98SMel Gorman split_map_pages(freelist); 1232ff9543fdSMichal Nazarewicz 12337ed695e0SVlastimil Babka /* 1234f5f61a32SVlastimil Babka * Record where the free scanner will restart next time. Either we 1235f5f61a32SVlastimil Babka * broke from the loop and set isolate_start_pfn based on the last 1236f5f61a32SVlastimil Babka * call to isolate_freepages_block(), or we met the migration scanner 1237f5f61a32SVlastimil Babka * and the loop terminated due to isolate_start_pfn < low_pfn 12387ed695e0SVlastimil Babka */ 1239f5f61a32SVlastimil Babka cc->free_pfn = isolate_start_pfn; 1240748446bbSMel Gorman } 1241748446bbSMel Gorman 1242748446bbSMel Gorman /* 1243748446bbSMel Gorman * This is a migrate-callback that "allocates" freepages by taking pages 1244748446bbSMel Gorman * from the isolated freelists in the block we are migrating to. 1245748446bbSMel Gorman */ 1246748446bbSMel Gorman static struct page *compaction_alloc(struct page *migratepage, 1247666feb21SMichal Hocko unsigned long data) 1248748446bbSMel Gorman { 1249748446bbSMel Gorman struct compact_control *cc = (struct compact_control *)data; 1250748446bbSMel Gorman struct page *freepage; 1251748446bbSMel Gorman 1252be976572SVlastimil Babka /* 1253be976572SVlastimil Babka * Isolate free pages if necessary, and if we are not aborting due to 1254be976572SVlastimil Babka * contention. 1255be976572SVlastimil Babka */ 1256748446bbSMel Gorman if (list_empty(&cc->freepages)) { 1257be976572SVlastimil Babka if (!cc->contended) 1258edc2ca61SVlastimil Babka isolate_freepages(cc); 1259748446bbSMel Gorman 1260748446bbSMel Gorman if (list_empty(&cc->freepages)) 1261748446bbSMel Gorman return NULL; 1262748446bbSMel Gorman } 1263748446bbSMel Gorman 1264748446bbSMel Gorman freepage = list_entry(cc->freepages.next, struct page, lru); 1265748446bbSMel Gorman list_del(&freepage->lru); 1266748446bbSMel Gorman cc->nr_freepages--; 1267748446bbSMel Gorman 1268748446bbSMel Gorman return freepage; 1269748446bbSMel Gorman } 1270748446bbSMel Gorman 1271748446bbSMel Gorman /* 1272d53aea3dSDavid Rientjes * This is a migrate-callback that "frees" freepages back to the isolated 1273d53aea3dSDavid Rientjes * freelist. All pages on the freelist are from the same zone, so there is no 1274d53aea3dSDavid Rientjes * special handling needed for NUMA. 1275d53aea3dSDavid Rientjes */ 1276d53aea3dSDavid Rientjes static void compaction_free(struct page *page, unsigned long data) 1277d53aea3dSDavid Rientjes { 1278d53aea3dSDavid Rientjes struct compact_control *cc = (struct compact_control *)data; 1279d53aea3dSDavid Rientjes 1280d53aea3dSDavid Rientjes list_add(&page->lru, &cc->freepages); 1281d53aea3dSDavid Rientjes cc->nr_freepages++; 1282d53aea3dSDavid Rientjes } 1283d53aea3dSDavid Rientjes 1284ff9543fdSMichal Nazarewicz /* possible outcome of isolate_migratepages */ 1285ff9543fdSMichal Nazarewicz typedef enum { 1286ff9543fdSMichal Nazarewicz ISOLATE_ABORT, /* Abort compaction now */ 1287ff9543fdSMichal Nazarewicz ISOLATE_NONE, /* No pages isolated, continue scanning */ 1288ff9543fdSMichal Nazarewicz ISOLATE_SUCCESS, /* Pages isolated, migrate */ 1289ff9543fdSMichal Nazarewicz } isolate_migrate_t; 1290ff9543fdSMichal Nazarewicz 1291ff9543fdSMichal Nazarewicz /* 12925bbe3547SEric B Munson * Allow userspace to control policy on scanning the unevictable LRU for 12935bbe3547SEric B Munson * compactable pages. 12945bbe3547SEric B Munson */ 12955bbe3547SEric B Munson int sysctl_compact_unevictable_allowed __read_mostly = 1; 12965bbe3547SEric B Munson 129770b44595SMel Gorman static inline void 129870b44595SMel Gorman update_fast_start_pfn(struct compact_control *cc, unsigned long pfn) 129970b44595SMel Gorman { 130070b44595SMel Gorman if (cc->fast_start_pfn == ULONG_MAX) 130170b44595SMel Gorman return; 130270b44595SMel Gorman 130370b44595SMel Gorman if (!cc->fast_start_pfn) 130470b44595SMel Gorman cc->fast_start_pfn = pfn; 130570b44595SMel Gorman 130670b44595SMel Gorman cc->fast_start_pfn = min(cc->fast_start_pfn, pfn); 130770b44595SMel Gorman } 130870b44595SMel Gorman 130970b44595SMel Gorman static inline unsigned long 131070b44595SMel Gorman reinit_migrate_pfn(struct compact_control *cc) 131170b44595SMel Gorman { 131270b44595SMel Gorman if (!cc->fast_start_pfn || cc->fast_start_pfn == ULONG_MAX) 131370b44595SMel Gorman return cc->migrate_pfn; 131470b44595SMel Gorman 131570b44595SMel Gorman cc->migrate_pfn = cc->fast_start_pfn; 131670b44595SMel Gorman cc->fast_start_pfn = ULONG_MAX; 131770b44595SMel Gorman 131870b44595SMel Gorman return cc->migrate_pfn; 131970b44595SMel Gorman } 132070b44595SMel Gorman 132170b44595SMel Gorman /* 132270b44595SMel Gorman * Briefly search the free lists for a migration source that already has 132370b44595SMel Gorman * some free pages to reduce the number of pages that need migration 132470b44595SMel Gorman * before a pageblock is free. 132570b44595SMel Gorman */ 132670b44595SMel Gorman static unsigned long fast_find_migrateblock(struct compact_control *cc) 132770b44595SMel Gorman { 132870b44595SMel Gorman unsigned int limit = freelist_scan_limit(cc); 132970b44595SMel Gorman unsigned int nr_scanned = 0; 133070b44595SMel Gorman unsigned long distance; 133170b44595SMel Gorman unsigned long pfn = cc->migrate_pfn; 133270b44595SMel Gorman unsigned long high_pfn; 133370b44595SMel Gorman int order; 133470b44595SMel Gorman 133570b44595SMel Gorman /* Skip hints are relied on to avoid repeats on the fast search */ 133670b44595SMel Gorman if (cc->ignore_skip_hint) 133770b44595SMel Gorman return pfn; 133870b44595SMel Gorman 133970b44595SMel Gorman /* 134070b44595SMel Gorman * If the migrate_pfn is not at the start of a zone or the start 134170b44595SMel Gorman * of a pageblock then assume this is a continuation of a previous 134270b44595SMel Gorman * scan restarted due to COMPACT_CLUSTER_MAX. 134370b44595SMel Gorman */ 134470b44595SMel Gorman if (pfn != cc->zone->zone_start_pfn && pfn != pageblock_start_pfn(pfn)) 134570b44595SMel Gorman return pfn; 134670b44595SMel Gorman 134770b44595SMel Gorman /* 134870b44595SMel Gorman * For smaller orders, just linearly scan as the number of pages 134970b44595SMel Gorman * to migrate should be relatively small and does not necessarily 135070b44595SMel Gorman * justify freeing up a large block for a small allocation. 135170b44595SMel Gorman */ 135270b44595SMel Gorman if (cc->order <= PAGE_ALLOC_COSTLY_ORDER) 135370b44595SMel Gorman return pfn; 135470b44595SMel Gorman 135570b44595SMel Gorman /* 135670b44595SMel Gorman * Only allow kcompactd and direct requests for movable pages to 135770b44595SMel Gorman * quickly clear out a MOVABLE pageblock for allocation. This 135870b44595SMel Gorman * reduces the risk that a large movable pageblock is freed for 135970b44595SMel Gorman * an unmovable/reclaimable small allocation. 136070b44595SMel Gorman */ 136170b44595SMel Gorman if (cc->direct_compaction && cc->migratetype != MIGRATE_MOVABLE) 136270b44595SMel Gorman return pfn; 136370b44595SMel Gorman 136470b44595SMel Gorman /* 136570b44595SMel Gorman * When starting the migration scanner, pick any pageblock within the 136670b44595SMel Gorman * first half of the search space. Otherwise try and pick a pageblock 136770b44595SMel Gorman * within the first eighth to reduce the chances that a migration 136870b44595SMel Gorman * target later becomes a source. 136970b44595SMel Gorman */ 137070b44595SMel Gorman distance = (cc->free_pfn - cc->migrate_pfn) >> 1; 137170b44595SMel Gorman if (cc->migrate_pfn != cc->zone->zone_start_pfn) 137270b44595SMel Gorman distance >>= 2; 137370b44595SMel Gorman high_pfn = pageblock_start_pfn(cc->migrate_pfn + distance); 137470b44595SMel Gorman 137570b44595SMel Gorman for (order = cc->order - 1; 137670b44595SMel Gorman order >= PAGE_ALLOC_COSTLY_ORDER && pfn == cc->migrate_pfn && nr_scanned < limit; 137770b44595SMel Gorman order--) { 137870b44595SMel Gorman struct free_area *area = &cc->zone->free_area[order]; 137970b44595SMel Gorman struct list_head *freelist; 138070b44595SMel Gorman unsigned long flags; 138170b44595SMel Gorman struct page *freepage; 138270b44595SMel Gorman 138370b44595SMel Gorman if (!area->nr_free) 138470b44595SMel Gorman continue; 138570b44595SMel Gorman 138670b44595SMel Gorman spin_lock_irqsave(&cc->zone->lock, flags); 138770b44595SMel Gorman freelist = &area->free_list[MIGRATE_MOVABLE]; 138870b44595SMel Gorman list_for_each_entry(freepage, freelist, lru) { 138970b44595SMel Gorman unsigned long free_pfn; 139070b44595SMel Gorman 139170b44595SMel Gorman nr_scanned++; 139270b44595SMel Gorman free_pfn = page_to_pfn(freepage); 139370b44595SMel Gorman if (free_pfn < high_pfn) { 139470b44595SMel Gorman /* 139570b44595SMel Gorman * Avoid if skipped recently. Ideally it would 139670b44595SMel Gorman * move to the tail but even safe iteration of 139770b44595SMel Gorman * the list assumes an entry is deleted, not 139870b44595SMel Gorman * reordered. 139970b44595SMel Gorman */ 140070b44595SMel Gorman if (get_pageblock_skip(freepage)) { 140170b44595SMel Gorman if (list_is_last(freelist, &freepage->lru)) 140270b44595SMel Gorman break; 140370b44595SMel Gorman 140470b44595SMel Gorman continue; 140570b44595SMel Gorman } 140670b44595SMel Gorman 140770b44595SMel Gorman /* Reorder to so a future search skips recent pages */ 140870b44595SMel Gorman move_freelist_tail(freelist, freepage); 140970b44595SMel Gorman 1410*e380bebeSMel Gorman update_fast_start_pfn(cc, free_pfn); 141170b44595SMel Gorman pfn = pageblock_start_pfn(free_pfn); 141270b44595SMel Gorman cc->fast_search_fail = 0; 141370b44595SMel Gorman set_pageblock_skip(freepage); 141470b44595SMel Gorman break; 141570b44595SMel Gorman } 141670b44595SMel Gorman 141770b44595SMel Gorman if (nr_scanned >= limit) { 141870b44595SMel Gorman cc->fast_search_fail++; 141970b44595SMel Gorman move_freelist_tail(freelist, freepage); 142070b44595SMel Gorman break; 142170b44595SMel Gorman } 142270b44595SMel Gorman } 142370b44595SMel Gorman spin_unlock_irqrestore(&cc->zone->lock, flags); 142470b44595SMel Gorman } 142570b44595SMel Gorman 142670b44595SMel Gorman cc->total_migrate_scanned += nr_scanned; 142770b44595SMel Gorman 142870b44595SMel Gorman /* 142970b44595SMel Gorman * If fast scanning failed then use a cached entry for a page block 143070b44595SMel Gorman * that had free pages as the basis for starting a linear scan. 143170b44595SMel Gorman */ 143270b44595SMel Gorman if (pfn == cc->migrate_pfn) 143370b44595SMel Gorman pfn = reinit_migrate_pfn(cc); 143470b44595SMel Gorman 143570b44595SMel Gorman return pfn; 143670b44595SMel Gorman } 143770b44595SMel Gorman 14385bbe3547SEric B Munson /* 1439edc2ca61SVlastimil Babka * Isolate all pages that can be migrated from the first suitable block, 1440edc2ca61SVlastimil Babka * starting at the block pointed to by the migrate scanner pfn within 1441edc2ca61SVlastimil Babka * compact_control. 1442ff9543fdSMichal Nazarewicz */ 1443ff9543fdSMichal Nazarewicz static isolate_migrate_t isolate_migratepages(struct zone *zone, 1444ff9543fdSMichal Nazarewicz struct compact_control *cc) 1445ff9543fdSMichal Nazarewicz { 1446e1409c32SJoonsoo Kim unsigned long block_start_pfn; 1447e1409c32SJoonsoo Kim unsigned long block_end_pfn; 1448e1409c32SJoonsoo Kim unsigned long low_pfn; 1449edc2ca61SVlastimil Babka struct page *page; 1450edc2ca61SVlastimil Babka const isolate_mode_t isolate_mode = 14515bbe3547SEric B Munson (sysctl_compact_unevictable_allowed ? ISOLATE_UNEVICTABLE : 0) | 14521d2047feSHugh Dickins (cc->mode != MIGRATE_SYNC ? ISOLATE_ASYNC_MIGRATE : 0); 145370b44595SMel Gorman bool fast_find_block; 1454ff9543fdSMichal Nazarewicz 1455edc2ca61SVlastimil Babka /* 1456edc2ca61SVlastimil Babka * Start at where we last stopped, or beginning of the zone as 145770b44595SMel Gorman * initialized by compact_zone(). The first failure will use 145870b44595SMel Gorman * the lowest PFN as the starting point for linear scanning. 1459edc2ca61SVlastimil Babka */ 146070b44595SMel Gorman low_pfn = fast_find_migrateblock(cc); 146106b6640aSVlastimil Babka block_start_pfn = pageblock_start_pfn(low_pfn); 1462e1409c32SJoonsoo Kim if (block_start_pfn < zone->zone_start_pfn) 1463e1409c32SJoonsoo Kim block_start_pfn = zone->zone_start_pfn; 1464ff9543fdSMichal Nazarewicz 146570b44595SMel Gorman /* 146670b44595SMel Gorman * fast_find_migrateblock marks a pageblock skipped so to avoid 146770b44595SMel Gorman * the isolation_suitable check below, check whether the fast 146870b44595SMel Gorman * search was successful. 146970b44595SMel Gorman */ 147070b44595SMel Gorman fast_find_block = low_pfn != cc->migrate_pfn && !cc->fast_search_fail; 147170b44595SMel Gorman 1472ff9543fdSMichal Nazarewicz /* Only scan within a pageblock boundary */ 147306b6640aSVlastimil Babka block_end_pfn = pageblock_end_pfn(low_pfn); 1474ff9543fdSMichal Nazarewicz 1475edc2ca61SVlastimil Babka /* 1476edc2ca61SVlastimil Babka * Iterate over whole pageblocks until we find the first suitable. 1477edc2ca61SVlastimil Babka * Do not cross the free scanner. 1478edc2ca61SVlastimil Babka */ 1479e1409c32SJoonsoo Kim for (; block_end_pfn <= cc->free_pfn; 148070b44595SMel Gorman fast_find_block = false, 1481e1409c32SJoonsoo Kim low_pfn = block_end_pfn, 1482e1409c32SJoonsoo Kim block_start_pfn = block_end_pfn, 1483e1409c32SJoonsoo Kim block_end_pfn += pageblock_nr_pages) { 1484edc2ca61SVlastimil Babka 1485edc2ca61SVlastimil Babka /* 1486edc2ca61SVlastimil Babka * This can potentially iterate a massively long zone with 1487edc2ca61SVlastimil Babka * many pageblocks unsuitable, so periodically check if we 1488edc2ca61SVlastimil Babka * need to schedule, or even abort async compaction. 1489edc2ca61SVlastimil Babka */ 1490edc2ca61SVlastimil Babka if (!(low_pfn % (SWAP_CLUSTER_MAX * pageblock_nr_pages)) 1491edc2ca61SVlastimil Babka && compact_should_abort(cc)) 1492edc2ca61SVlastimil Babka break; 1493edc2ca61SVlastimil Babka 1494e1409c32SJoonsoo Kim page = pageblock_pfn_to_page(block_start_pfn, block_end_pfn, 1495e1409c32SJoonsoo Kim zone); 14967d49d886SVlastimil Babka if (!page) 1497edc2ca61SVlastimil Babka continue; 1498edc2ca61SVlastimil Babka 1499*e380bebeSMel Gorman /* 1500*e380bebeSMel Gorman * If isolation recently failed, do not retry. Only check the 1501*e380bebeSMel Gorman * pageblock once. COMPACT_CLUSTER_MAX causes a pageblock 1502*e380bebeSMel Gorman * to be visited multiple times. Assume skip was checked 1503*e380bebeSMel Gorman * before making it "skip" so other compaction instances do 1504*e380bebeSMel Gorman * not scan the same block. 1505*e380bebeSMel Gorman */ 1506*e380bebeSMel Gorman if (IS_ALIGNED(low_pfn, pageblock_nr_pages) && 1507*e380bebeSMel Gorman !fast_find_block && !isolation_suitable(cc, page)) 1508edc2ca61SVlastimil Babka continue; 1509edc2ca61SVlastimil Babka 1510edc2ca61SVlastimil Babka /* 1511edc2ca61SVlastimil Babka * For async compaction, also only scan in MOVABLE blocks. 1512edc2ca61SVlastimil Babka * Async compaction is optimistic to see if the minimum amount 1513edc2ca61SVlastimil Babka * of work satisfies the allocation. 1514edc2ca61SVlastimil Babka */ 1515b682debdSVlastimil Babka if (!suitable_migration_source(cc, page)) 1516edc2ca61SVlastimil Babka continue; 1517ff9543fdSMichal Nazarewicz 1518ff9543fdSMichal Nazarewicz /* Perform the isolation */ 1519e1409c32SJoonsoo Kim low_pfn = isolate_migratepages_block(cc, low_pfn, 1520e1409c32SJoonsoo Kim block_end_pfn, isolate_mode); 1521edc2ca61SVlastimil Babka 15226afcf8efSMing Ling if (!low_pfn || cc->contended) 1523ff9543fdSMichal Nazarewicz return ISOLATE_ABORT; 1524ff9543fdSMichal Nazarewicz 1525edc2ca61SVlastimil Babka /* 1526edc2ca61SVlastimil Babka * Either we isolated something and proceed with migration. Or 1527edc2ca61SVlastimil Babka * we failed and compact_zone should decide if we should 1528edc2ca61SVlastimil Babka * continue or not. 1529edc2ca61SVlastimil Babka */ 1530edc2ca61SVlastimil Babka break; 1531edc2ca61SVlastimil Babka } 1532edc2ca61SVlastimil Babka 1533f2849aa0SVlastimil Babka /* Record where migration scanner will be restarted. */ 1534f2849aa0SVlastimil Babka cc->migrate_pfn = low_pfn; 1535ff9543fdSMichal Nazarewicz 1536edc2ca61SVlastimil Babka return cc->nr_migratepages ? ISOLATE_SUCCESS : ISOLATE_NONE; 1537ff9543fdSMichal Nazarewicz } 1538ff9543fdSMichal Nazarewicz 153921c527a3SYaowei Bai /* 154021c527a3SYaowei Bai * order == -1 is expected when compacting via 154121c527a3SYaowei Bai * /proc/sys/vm/compact_memory 154221c527a3SYaowei Bai */ 154321c527a3SYaowei Bai static inline bool is_via_compact_memory(int order) 154421c527a3SYaowei Bai { 154521c527a3SYaowei Bai return order == -1; 154621c527a3SYaowei Bai } 154721c527a3SYaowei Bai 154840cacbcbSMel Gorman static enum compact_result __compact_finished(struct compact_control *cc) 1549748446bbSMel Gorman { 15508fb74b9fSMel Gorman unsigned int order; 1551d39773a0SVlastimil Babka const int migratetype = cc->migratetype; 155256de7263SMel Gorman 1553be976572SVlastimil Babka if (cc->contended || fatal_signal_pending(current)) 15542d1e1041SVlastimil Babka return COMPACT_CONTENDED; 1555748446bbSMel Gorman 1556753341a4SMel Gorman /* Compaction run completes if the migrate and free scanner meet */ 1557f2849aa0SVlastimil Babka if (compact_scanners_met(cc)) { 155855b7c4c9SVlastimil Babka /* Let the next compaction start anew. */ 155940cacbcbSMel Gorman reset_cached_positions(cc->zone); 156055b7c4c9SVlastimil Babka 156162997027SMel Gorman /* 156262997027SMel Gorman * Mark that the PG_migrate_skip information should be cleared 1563accf6242SVlastimil Babka * by kswapd when it goes to sleep. kcompactd does not set the 156462997027SMel Gorman * flag itself as the decision to be clear should be directly 156562997027SMel Gorman * based on an allocation request. 156662997027SMel Gorman */ 1567accf6242SVlastimil Babka if (cc->direct_compaction) 156840cacbcbSMel Gorman cc->zone->compact_blockskip_flush = true; 156962997027SMel Gorman 1570c8f7de0bSMichal Hocko if (cc->whole_zone) 1571748446bbSMel Gorman return COMPACT_COMPLETE; 1572c8f7de0bSMichal Hocko else 1573c8f7de0bSMichal Hocko return COMPACT_PARTIAL_SKIPPED; 1574bb13ffebSMel Gorman } 1575748446bbSMel Gorman 157621c527a3SYaowei Bai if (is_via_compact_memory(cc->order)) 157756de7263SMel Gorman return COMPACT_CONTINUE; 157856de7263SMel Gorman 1579baf6a9a1SVlastimil Babka /* 1580efe771c7SMel Gorman * Always finish scanning a pageblock to reduce the possibility of 1581efe771c7SMel Gorman * fallbacks in the future. This is particularly important when 1582efe771c7SMel Gorman * migration source is unmovable/reclaimable but it's not worth 1583efe771c7SMel Gorman * special casing. 1584baf6a9a1SVlastimil Babka */ 1585efe771c7SMel Gorman if (!IS_ALIGNED(cc->migrate_pfn, pageblock_nr_pages)) 1586baf6a9a1SVlastimil Babka return COMPACT_CONTINUE; 1587baf6a9a1SVlastimil Babka 158856de7263SMel Gorman /* Direct compactor: Is a suitable page free? */ 158956de7263SMel Gorman for (order = cc->order; order < MAX_ORDER; order++) { 159040cacbcbSMel Gorman struct free_area *area = &cc->zone->free_area[order]; 15912149cdaeSJoonsoo Kim bool can_steal; 15928fb74b9fSMel Gorman 159356de7263SMel Gorman /* Job done if page is free of the right migratetype */ 15946d7ce559SDavid Rientjes if (!list_empty(&area->free_list[migratetype])) 1595cf378319SVlastimil Babka return COMPACT_SUCCESS; 159656de7263SMel Gorman 15972149cdaeSJoonsoo Kim #ifdef CONFIG_CMA 15982149cdaeSJoonsoo Kim /* MIGRATE_MOVABLE can fallback on MIGRATE_CMA */ 15992149cdaeSJoonsoo Kim if (migratetype == MIGRATE_MOVABLE && 16002149cdaeSJoonsoo Kim !list_empty(&area->free_list[MIGRATE_CMA])) 1601cf378319SVlastimil Babka return COMPACT_SUCCESS; 16022149cdaeSJoonsoo Kim #endif 16032149cdaeSJoonsoo Kim /* 16042149cdaeSJoonsoo Kim * Job done if allocation would steal freepages from 16052149cdaeSJoonsoo Kim * other migratetype buddy lists. 16062149cdaeSJoonsoo Kim */ 16072149cdaeSJoonsoo Kim if (find_suitable_fallback(area, order, migratetype, 1608baf6a9a1SVlastimil Babka true, &can_steal) != -1) { 1609baf6a9a1SVlastimil Babka 1610baf6a9a1SVlastimil Babka /* movable pages are OK in any pageblock */ 1611baf6a9a1SVlastimil Babka if (migratetype == MIGRATE_MOVABLE) 1612cf378319SVlastimil Babka return COMPACT_SUCCESS; 1613baf6a9a1SVlastimil Babka 1614baf6a9a1SVlastimil Babka /* 1615baf6a9a1SVlastimil Babka * We are stealing for a non-movable allocation. Make 1616baf6a9a1SVlastimil Babka * sure we finish compacting the current pageblock 1617baf6a9a1SVlastimil Babka * first so it is as free as possible and we won't 1618baf6a9a1SVlastimil Babka * have to steal another one soon. This only applies 1619baf6a9a1SVlastimil Babka * to sync compaction, as async compaction operates 1620baf6a9a1SVlastimil Babka * on pageblocks of the same migratetype. 1621baf6a9a1SVlastimil Babka */ 1622baf6a9a1SVlastimil Babka if (cc->mode == MIGRATE_ASYNC || 1623baf6a9a1SVlastimil Babka IS_ALIGNED(cc->migrate_pfn, 1624baf6a9a1SVlastimil Babka pageblock_nr_pages)) { 1625baf6a9a1SVlastimil Babka return COMPACT_SUCCESS; 1626baf6a9a1SVlastimil Babka } 1627baf6a9a1SVlastimil Babka 1628baf6a9a1SVlastimil Babka return COMPACT_CONTINUE; 1629baf6a9a1SVlastimil Babka } 163056de7263SMel Gorman } 163156de7263SMel Gorman 1632837d026dSJoonsoo Kim return COMPACT_NO_SUITABLE_PAGE; 1633837d026dSJoonsoo Kim } 1634837d026dSJoonsoo Kim 163540cacbcbSMel Gorman static enum compact_result compact_finished(struct compact_control *cc) 1636837d026dSJoonsoo Kim { 1637837d026dSJoonsoo Kim int ret; 1638837d026dSJoonsoo Kim 163940cacbcbSMel Gorman ret = __compact_finished(cc); 164040cacbcbSMel Gorman trace_mm_compaction_finished(cc->zone, cc->order, ret); 1641837d026dSJoonsoo Kim if (ret == COMPACT_NO_SUITABLE_PAGE) 1642837d026dSJoonsoo Kim ret = COMPACT_CONTINUE; 1643837d026dSJoonsoo Kim 1644837d026dSJoonsoo Kim return ret; 1645748446bbSMel Gorman } 1646748446bbSMel Gorman 16473e7d3449SMel Gorman /* 16483e7d3449SMel Gorman * compaction_suitable: Is this suitable to run compaction on this zone now? 16493e7d3449SMel Gorman * Returns 16503e7d3449SMel Gorman * COMPACT_SKIPPED - If there are too few free pages for compaction 1651cf378319SVlastimil Babka * COMPACT_SUCCESS - If the allocation would succeed without compaction 16523e7d3449SMel Gorman * COMPACT_CONTINUE - If compaction should run now 16533e7d3449SMel Gorman */ 1654ea7ab982SMichal Hocko static enum compact_result __compaction_suitable(struct zone *zone, int order, 1655c603844bSMel Gorman unsigned int alloc_flags, 165686a294a8SMichal Hocko int classzone_idx, 165786a294a8SMichal Hocko unsigned long wmark_target) 16583e7d3449SMel Gorman { 16593e7d3449SMel Gorman unsigned long watermark; 16603e7d3449SMel Gorman 166121c527a3SYaowei Bai if (is_via_compact_memory(order)) 16623957c776SMichal Hocko return COMPACT_CONTINUE; 16633957c776SMichal Hocko 1664a9214443SMel Gorman watermark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK); 1665ebff3980SVlastimil Babka /* 1666ebff3980SVlastimil Babka * If watermarks for high-order allocation are already met, there 1667ebff3980SVlastimil Babka * should be no need for compaction at all. 1668ebff3980SVlastimil Babka */ 1669ebff3980SVlastimil Babka if (zone_watermark_ok(zone, order, watermark, classzone_idx, 1670ebff3980SVlastimil Babka alloc_flags)) 1671cf378319SVlastimil Babka return COMPACT_SUCCESS; 1672ebff3980SVlastimil Babka 16733957c776SMichal Hocko /* 16749861a62cSVlastimil Babka * Watermarks for order-0 must be met for compaction to be able to 1675984fdba6SVlastimil Babka * isolate free pages for migration targets. This means that the 1676984fdba6SVlastimil Babka * watermark and alloc_flags have to match, or be more pessimistic than 1677984fdba6SVlastimil Babka * the check in __isolate_free_page(). We don't use the direct 1678984fdba6SVlastimil Babka * compactor's alloc_flags, as they are not relevant for freepage 1679984fdba6SVlastimil Babka * isolation. We however do use the direct compactor's classzone_idx to 1680984fdba6SVlastimil Babka * skip over zones where lowmem reserves would prevent allocation even 1681984fdba6SVlastimil Babka * if compaction succeeds. 16828348faf9SVlastimil Babka * For costly orders, we require low watermark instead of min for 16838348faf9SVlastimil Babka * compaction to proceed to increase its chances. 1684d883c6cfSJoonsoo Kim * ALLOC_CMA is used, as pages in CMA pageblocks are considered 1685d883c6cfSJoonsoo Kim * suitable migration targets 16863e7d3449SMel Gorman */ 16878348faf9SVlastimil Babka watermark = (order > PAGE_ALLOC_COSTLY_ORDER) ? 16888348faf9SVlastimil Babka low_wmark_pages(zone) : min_wmark_pages(zone); 16898348faf9SVlastimil Babka watermark += compact_gap(order); 169086a294a8SMichal Hocko if (!__zone_watermark_ok(zone, 0, watermark, classzone_idx, 1691d883c6cfSJoonsoo Kim ALLOC_CMA, wmark_target)) 16923e7d3449SMel Gorman return COMPACT_SKIPPED; 16933e7d3449SMel Gorman 1694cc5c9f09SVlastimil Babka return COMPACT_CONTINUE; 1695cc5c9f09SVlastimil Babka } 1696cc5c9f09SVlastimil Babka 1697cc5c9f09SVlastimil Babka enum compact_result compaction_suitable(struct zone *zone, int order, 1698cc5c9f09SVlastimil Babka unsigned int alloc_flags, 1699cc5c9f09SVlastimil Babka int classzone_idx) 1700cc5c9f09SVlastimil Babka { 1701cc5c9f09SVlastimil Babka enum compact_result ret; 1702cc5c9f09SVlastimil Babka int fragindex; 1703cc5c9f09SVlastimil Babka 1704cc5c9f09SVlastimil Babka ret = __compaction_suitable(zone, order, alloc_flags, classzone_idx, 1705cc5c9f09SVlastimil Babka zone_page_state(zone, NR_FREE_PAGES)); 17063e7d3449SMel Gorman /* 17073e7d3449SMel Gorman * fragmentation index determines if allocation failures are due to 17083e7d3449SMel Gorman * low memory or external fragmentation 17093e7d3449SMel Gorman * 1710ebff3980SVlastimil Babka * index of -1000 would imply allocations might succeed depending on 1711ebff3980SVlastimil Babka * watermarks, but we already failed the high-order watermark check 17123e7d3449SMel Gorman * index towards 0 implies failure is due to lack of memory 17133e7d3449SMel Gorman * index towards 1000 implies failure is due to fragmentation 17143e7d3449SMel Gorman * 171520311420SVlastimil Babka * Only compact if a failure would be due to fragmentation. Also 171620311420SVlastimil Babka * ignore fragindex for non-costly orders where the alternative to 171720311420SVlastimil Babka * a successful reclaim/compaction is OOM. Fragindex and the 171820311420SVlastimil Babka * vm.extfrag_threshold sysctl is meant as a heuristic to prevent 171920311420SVlastimil Babka * excessive compaction for costly orders, but it should not be at the 172020311420SVlastimil Babka * expense of system stability. 17213e7d3449SMel Gorman */ 172220311420SVlastimil Babka if (ret == COMPACT_CONTINUE && (order > PAGE_ALLOC_COSTLY_ORDER)) { 17233e7d3449SMel Gorman fragindex = fragmentation_index(zone, order); 17243e7d3449SMel Gorman if (fragindex >= 0 && fragindex <= sysctl_extfrag_threshold) 1725cc5c9f09SVlastimil Babka ret = COMPACT_NOT_SUITABLE_ZONE; 17263e7d3449SMel Gorman } 17273e7d3449SMel Gorman 1728837d026dSJoonsoo Kim trace_mm_compaction_suitable(zone, order, ret); 1729837d026dSJoonsoo Kim if (ret == COMPACT_NOT_SUITABLE_ZONE) 1730837d026dSJoonsoo Kim ret = COMPACT_SKIPPED; 1731837d026dSJoonsoo Kim 1732837d026dSJoonsoo Kim return ret; 1733837d026dSJoonsoo Kim } 1734837d026dSJoonsoo Kim 173586a294a8SMichal Hocko bool compaction_zonelist_suitable(struct alloc_context *ac, int order, 173686a294a8SMichal Hocko int alloc_flags) 173786a294a8SMichal Hocko { 173886a294a8SMichal Hocko struct zone *zone; 173986a294a8SMichal Hocko struct zoneref *z; 174086a294a8SMichal Hocko 174186a294a8SMichal Hocko /* 174286a294a8SMichal Hocko * Make sure at least one zone would pass __compaction_suitable if we continue 174386a294a8SMichal Hocko * retrying the reclaim. 174486a294a8SMichal Hocko */ 174586a294a8SMichal Hocko for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx, 174686a294a8SMichal Hocko ac->nodemask) { 174786a294a8SMichal Hocko unsigned long available; 174886a294a8SMichal Hocko enum compact_result compact_result; 174986a294a8SMichal Hocko 175086a294a8SMichal Hocko /* 175186a294a8SMichal Hocko * Do not consider all the reclaimable memory because we do not 175286a294a8SMichal Hocko * want to trash just for a single high order allocation which 175386a294a8SMichal Hocko * is even not guaranteed to appear even if __compaction_suitable 175486a294a8SMichal Hocko * is happy about the watermark check. 175586a294a8SMichal Hocko */ 17565a1c84b4SMel Gorman available = zone_reclaimable_pages(zone) / order; 175786a294a8SMichal Hocko available += zone_page_state_snapshot(zone, NR_FREE_PAGES); 175886a294a8SMichal Hocko compact_result = __compaction_suitable(zone, order, alloc_flags, 175986a294a8SMichal Hocko ac_classzone_idx(ac), available); 1760cc5c9f09SVlastimil Babka if (compact_result != COMPACT_SKIPPED) 176186a294a8SMichal Hocko return true; 176286a294a8SMichal Hocko } 176386a294a8SMichal Hocko 176486a294a8SMichal Hocko return false; 176586a294a8SMichal Hocko } 176686a294a8SMichal Hocko 176740cacbcbSMel Gorman static enum compact_result compact_zone(struct compact_control *cc) 1768748446bbSMel Gorman { 1769ea7ab982SMichal Hocko enum compact_result ret; 177040cacbcbSMel Gorman unsigned long start_pfn = cc->zone->zone_start_pfn; 177140cacbcbSMel Gorman unsigned long end_pfn = zone_end_pfn(cc->zone); 1772566e54e1SMel Gorman unsigned long last_migrated_pfn; 1773e0b9daebSDavid Rientjes const bool sync = cc->mode != MIGRATE_ASYNC; 1774748446bbSMel Gorman 1775d39773a0SVlastimil Babka cc->migratetype = gfpflags_to_migratetype(cc->gfp_mask); 177640cacbcbSMel Gorman ret = compaction_suitable(cc->zone, cc->order, cc->alloc_flags, 1777ebff3980SVlastimil Babka cc->classzone_idx); 17783e7d3449SMel Gorman /* Compaction is likely to fail */ 1779cf378319SVlastimil Babka if (ret == COMPACT_SUCCESS || ret == COMPACT_SKIPPED) 17803e7d3449SMel Gorman return ret; 1781c46649deSMichal Hocko 1782c46649deSMichal Hocko /* huh, compaction_suitable is returning something unexpected */ 1783c46649deSMichal Hocko VM_BUG_ON(ret != COMPACT_CONTINUE); 17843e7d3449SMel Gorman 1785c89511abSMel Gorman /* 1786d3132e4bSVlastimil Babka * Clear pageblock skip if there were failures recently and compaction 1787accf6242SVlastimil Babka * is about to be retried after being deferred. 1788d3132e4bSVlastimil Babka */ 178940cacbcbSMel Gorman if (compaction_restarting(cc->zone, cc->order)) 179040cacbcbSMel Gorman __reset_isolation_suitable(cc->zone); 1791d3132e4bSVlastimil Babka 1792d3132e4bSVlastimil Babka /* 1793c89511abSMel Gorman * Setup to move all movable pages to the end of the zone. Used cached 179406ed2998SVlastimil Babka * information on where the scanners should start (unless we explicitly 179506ed2998SVlastimil Babka * want to compact the whole zone), but check that it is initialised 179606ed2998SVlastimil Babka * by ensuring the values are within zone boundaries. 1797c89511abSMel Gorman */ 179870b44595SMel Gorman cc->fast_start_pfn = 0; 179906ed2998SVlastimil Babka if (cc->whole_zone) { 180006ed2998SVlastimil Babka cc->migrate_pfn = start_pfn; 180106ed2998SVlastimil Babka cc->free_pfn = pageblock_start_pfn(end_pfn - 1); 180206ed2998SVlastimil Babka } else { 180340cacbcbSMel Gorman cc->migrate_pfn = cc->zone->compact_cached_migrate_pfn[sync]; 180440cacbcbSMel Gorman cc->free_pfn = cc->zone->compact_cached_free_pfn; 1805623446e4SJoonsoo Kim if (cc->free_pfn < start_pfn || cc->free_pfn >= end_pfn) { 180606b6640aSVlastimil Babka cc->free_pfn = pageblock_start_pfn(end_pfn - 1); 180740cacbcbSMel Gorman cc->zone->compact_cached_free_pfn = cc->free_pfn; 1808c89511abSMel Gorman } 1809623446e4SJoonsoo Kim if (cc->migrate_pfn < start_pfn || cc->migrate_pfn >= end_pfn) { 1810c89511abSMel Gorman cc->migrate_pfn = start_pfn; 181140cacbcbSMel Gorman cc->zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn; 181240cacbcbSMel Gorman cc->zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn; 1813c89511abSMel Gorman } 1814c8f7de0bSMichal Hocko 1815c8f7de0bSMichal Hocko if (cc->migrate_pfn == start_pfn) 1816c8f7de0bSMichal Hocko cc->whole_zone = true; 181706ed2998SVlastimil Babka } 1818c8f7de0bSMichal Hocko 1819566e54e1SMel Gorman last_migrated_pfn = 0; 1820748446bbSMel Gorman 182116c4a097SJoonsoo Kim trace_mm_compaction_begin(start_pfn, cc->migrate_pfn, 182216c4a097SJoonsoo Kim cc->free_pfn, end_pfn, sync); 18230eb927c0SMel Gorman 1824748446bbSMel Gorman migrate_prep_local(); 1825748446bbSMel Gorman 182640cacbcbSMel Gorman while ((ret = compact_finished(cc)) == COMPACT_CONTINUE) { 18279d502c1cSMinchan Kim int err; 1828566e54e1SMel Gorman unsigned long start_pfn = cc->migrate_pfn; 1829748446bbSMel Gorman 183040cacbcbSMel Gorman switch (isolate_migratepages(cc->zone, cc)) { 1831f9e35b3bSMel Gorman case ISOLATE_ABORT: 18322d1e1041SVlastimil Babka ret = COMPACT_CONTENDED; 18335733c7d1SRafael Aquini putback_movable_pages(&cc->migratepages); 1834e64c5237SShaohua Li cc->nr_migratepages = 0; 1835566e54e1SMel Gorman last_migrated_pfn = 0; 1836f9e35b3bSMel Gorman goto out; 1837f9e35b3bSMel Gorman case ISOLATE_NONE: 1838fdaf7f5cSVlastimil Babka /* 1839fdaf7f5cSVlastimil Babka * We haven't isolated and migrated anything, but 1840fdaf7f5cSVlastimil Babka * there might still be unflushed migrations from 1841fdaf7f5cSVlastimil Babka * previous cc->order aligned block. 1842fdaf7f5cSVlastimil Babka */ 1843fdaf7f5cSVlastimil Babka goto check_drain; 1844f9e35b3bSMel Gorman case ISOLATE_SUCCESS: 1845566e54e1SMel Gorman last_migrated_pfn = start_pfn; 1846f9e35b3bSMel Gorman ; 1847f9e35b3bSMel Gorman } 1848748446bbSMel Gorman 1849d53aea3dSDavid Rientjes err = migrate_pages(&cc->migratepages, compaction_alloc, 1850e0b9daebSDavid Rientjes compaction_free, (unsigned long)cc, cc->mode, 18517b2a2d4aSMel Gorman MR_COMPACTION); 1852748446bbSMel Gorman 1853f8c9301fSVlastimil Babka trace_mm_compaction_migratepages(cc->nr_migratepages, err, 1854f8c9301fSVlastimil Babka &cc->migratepages); 1855748446bbSMel Gorman 1856f8c9301fSVlastimil Babka /* All pages were either migrated or will be released */ 1857f8c9301fSVlastimil Babka cc->nr_migratepages = 0; 18589d502c1cSMinchan Kim if (err) { 18595733c7d1SRafael Aquini putback_movable_pages(&cc->migratepages); 18607ed695e0SVlastimil Babka /* 18617ed695e0SVlastimil Babka * migrate_pages() may return -ENOMEM when scanners meet 18627ed695e0SVlastimil Babka * and we want compact_finished() to detect it 18637ed695e0SVlastimil Babka */ 1864f2849aa0SVlastimil Babka if (err == -ENOMEM && !compact_scanners_met(cc)) { 18652d1e1041SVlastimil Babka ret = COMPACT_CONTENDED; 18664bf2bba3SDavid Rientjes goto out; 1867748446bbSMel Gorman } 1868fdd048e1SVlastimil Babka /* 1869fdd048e1SVlastimil Babka * We failed to migrate at least one page in the current 1870fdd048e1SVlastimil Babka * order-aligned block, so skip the rest of it. 1871fdd048e1SVlastimil Babka */ 1872fdd048e1SVlastimil Babka if (cc->direct_compaction && 1873fdd048e1SVlastimil Babka (cc->mode == MIGRATE_ASYNC)) { 1874fdd048e1SVlastimil Babka cc->migrate_pfn = block_end_pfn( 1875fdd048e1SVlastimil Babka cc->migrate_pfn - 1, cc->order); 1876fdd048e1SVlastimil Babka /* Draining pcplists is useless in this case */ 1877566e54e1SMel Gorman last_migrated_pfn = 0; 1878fdd048e1SVlastimil Babka } 18794bf2bba3SDavid Rientjes } 1880fdaf7f5cSVlastimil Babka 1881fdaf7f5cSVlastimil Babka check_drain: 1882fdaf7f5cSVlastimil Babka /* 1883fdaf7f5cSVlastimil Babka * Has the migration scanner moved away from the previous 1884fdaf7f5cSVlastimil Babka * cc->order aligned block where we migrated from? If yes, 1885fdaf7f5cSVlastimil Babka * flush the pages that were freed, so that they can merge and 1886fdaf7f5cSVlastimil Babka * compact_finished() can detect immediately if allocation 1887fdaf7f5cSVlastimil Babka * would succeed. 1888fdaf7f5cSVlastimil Babka */ 1889566e54e1SMel Gorman if (cc->order > 0 && last_migrated_pfn) { 1890fdaf7f5cSVlastimil Babka int cpu; 1891fdaf7f5cSVlastimil Babka unsigned long current_block_start = 189206b6640aSVlastimil Babka block_start_pfn(cc->migrate_pfn, cc->order); 1893fdaf7f5cSVlastimil Babka 1894566e54e1SMel Gorman if (last_migrated_pfn < current_block_start) { 1895fdaf7f5cSVlastimil Babka cpu = get_cpu(); 1896fdaf7f5cSVlastimil Babka lru_add_drain_cpu(cpu); 189740cacbcbSMel Gorman drain_local_pages(cc->zone); 1898fdaf7f5cSVlastimil Babka put_cpu(); 1899fdaf7f5cSVlastimil Babka /* No more flushing until we migrate again */ 1900566e54e1SMel Gorman last_migrated_pfn = 0; 1901fdaf7f5cSVlastimil Babka } 1902fdaf7f5cSVlastimil Babka } 1903fdaf7f5cSVlastimil Babka 1904748446bbSMel Gorman } 1905748446bbSMel Gorman 1906f9e35b3bSMel Gorman out: 19076bace090SVlastimil Babka /* 19086bace090SVlastimil Babka * Release free pages and update where the free scanner should restart, 19096bace090SVlastimil Babka * so we don't leave any returned pages behind in the next attempt. 19106bace090SVlastimil Babka */ 19116bace090SVlastimil Babka if (cc->nr_freepages > 0) { 19126bace090SVlastimil Babka unsigned long free_pfn = release_freepages(&cc->freepages); 19136bace090SVlastimil Babka 19146bace090SVlastimil Babka cc->nr_freepages = 0; 19156bace090SVlastimil Babka VM_BUG_ON(free_pfn == 0); 19166bace090SVlastimil Babka /* The cached pfn is always the first in a pageblock */ 191706b6640aSVlastimil Babka free_pfn = pageblock_start_pfn(free_pfn); 19186bace090SVlastimil Babka /* 19196bace090SVlastimil Babka * Only go back, not forward. The cached pfn might have been 19206bace090SVlastimil Babka * already reset to zone end in compact_finished() 19216bace090SVlastimil Babka */ 192240cacbcbSMel Gorman if (free_pfn > cc->zone->compact_cached_free_pfn) 192340cacbcbSMel Gorman cc->zone->compact_cached_free_pfn = free_pfn; 19246bace090SVlastimil Babka } 1925748446bbSMel Gorman 19267f354a54SDavid Rientjes count_compact_events(COMPACTMIGRATE_SCANNED, cc->total_migrate_scanned); 19277f354a54SDavid Rientjes count_compact_events(COMPACTFREE_SCANNED, cc->total_free_scanned); 19287f354a54SDavid Rientjes 192916c4a097SJoonsoo Kim trace_mm_compaction_end(start_pfn, cc->migrate_pfn, 193016c4a097SJoonsoo Kim cc->free_pfn, end_pfn, sync, ret); 19310eb927c0SMel Gorman 1932748446bbSMel Gorman return ret; 1933748446bbSMel Gorman } 193476ab0f53SMel Gorman 1935ea7ab982SMichal Hocko static enum compact_result compact_zone_order(struct zone *zone, int order, 1936c3486f53SVlastimil Babka gfp_t gfp_mask, enum compact_priority prio, 1937c603844bSMel Gorman unsigned int alloc_flags, int classzone_idx) 193856de7263SMel Gorman { 1939ea7ab982SMichal Hocko enum compact_result ret; 194056de7263SMel Gorman struct compact_control cc = { 194156de7263SMel Gorman .nr_freepages = 0, 194256de7263SMel Gorman .nr_migratepages = 0, 19437f354a54SDavid Rientjes .total_migrate_scanned = 0, 19447f354a54SDavid Rientjes .total_free_scanned = 0, 194556de7263SMel Gorman .order = order, 19466d7ce559SDavid Rientjes .gfp_mask = gfp_mask, 194756de7263SMel Gorman .zone = zone, 1948a5508cd8SVlastimil Babka .mode = (prio == COMPACT_PRIO_ASYNC) ? 1949a5508cd8SVlastimil Babka MIGRATE_ASYNC : MIGRATE_SYNC_LIGHT, 1950ebff3980SVlastimil Babka .alloc_flags = alloc_flags, 1951ebff3980SVlastimil Babka .classzone_idx = classzone_idx, 1952accf6242SVlastimil Babka .direct_compaction = true, 1953a8e025e5SVlastimil Babka .whole_zone = (prio == MIN_COMPACT_PRIORITY), 19549f7e3387SVlastimil Babka .ignore_skip_hint = (prio == MIN_COMPACT_PRIORITY), 19559f7e3387SVlastimil Babka .ignore_block_suitable = (prio == MIN_COMPACT_PRIORITY) 195656de7263SMel Gorman }; 195756de7263SMel Gorman INIT_LIST_HEAD(&cc.freepages); 195856de7263SMel Gorman INIT_LIST_HEAD(&cc.migratepages); 195956de7263SMel Gorman 196040cacbcbSMel Gorman ret = compact_zone(&cc); 1961e64c5237SShaohua Li 1962e64c5237SShaohua Li VM_BUG_ON(!list_empty(&cc.freepages)); 1963e64c5237SShaohua Li VM_BUG_ON(!list_empty(&cc.migratepages)); 1964e64c5237SShaohua Li 1965e64c5237SShaohua Li return ret; 196656de7263SMel Gorman } 196756de7263SMel Gorman 19685e771905SMel Gorman int sysctl_extfrag_threshold = 500; 19695e771905SMel Gorman 197056de7263SMel Gorman /** 197156de7263SMel Gorman * try_to_compact_pages - Direct compact to satisfy a high-order allocation 197256de7263SMel Gorman * @gfp_mask: The GFP mask of the current allocation 19731a6d53a1SVlastimil Babka * @order: The order of the current allocation 19741a6d53a1SVlastimil Babka * @alloc_flags: The allocation flags of the current allocation 19751a6d53a1SVlastimil Babka * @ac: The context of current allocation 1976112d2d29SYang Shi * @prio: Determines how hard direct compaction should try to succeed 197756de7263SMel Gorman * 197856de7263SMel Gorman * This is the main entry point for direct page compaction. 197956de7263SMel Gorman */ 1980ea7ab982SMichal Hocko enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order, 1981c603844bSMel Gorman unsigned int alloc_flags, const struct alloc_context *ac, 1982c3486f53SVlastimil Babka enum compact_priority prio) 198356de7263SMel Gorman { 198456de7263SMel Gorman int may_perform_io = gfp_mask & __GFP_IO; 198556de7263SMel Gorman struct zoneref *z; 198656de7263SMel Gorman struct zone *zone; 19871d4746d3SMichal Hocko enum compact_result rc = COMPACT_SKIPPED; 198856de7263SMel Gorman 198973e64c51SMichal Hocko /* 199073e64c51SMichal Hocko * Check if the GFP flags allow compaction - GFP_NOIO is really 199173e64c51SMichal Hocko * tricky context because the migration might require IO 199273e64c51SMichal Hocko */ 199373e64c51SMichal Hocko if (!may_perform_io) 199453853e2dSVlastimil Babka return COMPACT_SKIPPED; 199556de7263SMel Gorman 1996a5508cd8SVlastimil Babka trace_mm_compaction_try_to_compact_pages(order, gfp_mask, prio); 1997837d026dSJoonsoo Kim 199856de7263SMel Gorman /* Compact each zone in the list */ 19991a6d53a1SVlastimil Babka for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx, 20001a6d53a1SVlastimil Babka ac->nodemask) { 2001ea7ab982SMichal Hocko enum compact_result status; 200256de7263SMel Gorman 2003a8e025e5SVlastimil Babka if (prio > MIN_COMPACT_PRIORITY 2004a8e025e5SVlastimil Babka && compaction_deferred(zone, order)) { 20051d4746d3SMichal Hocko rc = max_t(enum compact_result, COMPACT_DEFERRED, rc); 200653853e2dSVlastimil Babka continue; 20071d4746d3SMichal Hocko } 200853853e2dSVlastimil Babka 2009a5508cd8SVlastimil Babka status = compact_zone_order(zone, order, gfp_mask, prio, 2010c3486f53SVlastimil Babka alloc_flags, ac_classzone_idx(ac)); 201156de7263SMel Gorman rc = max(status, rc); 201256de7263SMel Gorman 20137ceb009aSVlastimil Babka /* The allocation should succeed, stop compacting */ 20147ceb009aSVlastimil Babka if (status == COMPACT_SUCCESS) { 201553853e2dSVlastimil Babka /* 201653853e2dSVlastimil Babka * We think the allocation will succeed in this zone, 201753853e2dSVlastimil Babka * but it is not certain, hence the false. The caller 201853853e2dSVlastimil Babka * will repeat this with true if allocation indeed 201953853e2dSVlastimil Babka * succeeds in this zone. 202053853e2dSVlastimil Babka */ 202153853e2dSVlastimil Babka compaction_defer_reset(zone, order, false); 20221f9efdefSVlastimil Babka 2023c3486f53SVlastimil Babka break; 20241f9efdefSVlastimil Babka } 20251f9efdefSVlastimil Babka 2026a5508cd8SVlastimil Babka if (prio != COMPACT_PRIO_ASYNC && (status == COMPACT_COMPLETE || 2027c3486f53SVlastimil Babka status == COMPACT_PARTIAL_SKIPPED)) 202853853e2dSVlastimil Babka /* 202953853e2dSVlastimil Babka * We think that allocation won't succeed in this zone 203053853e2dSVlastimil Babka * so we defer compaction there. If it ends up 203153853e2dSVlastimil Babka * succeeding after all, it will be reset. 203253853e2dSVlastimil Babka */ 203353853e2dSVlastimil Babka defer_compaction(zone, order); 20341f9efdefSVlastimil Babka 20351f9efdefSVlastimil Babka /* 20361f9efdefSVlastimil Babka * We might have stopped compacting due to need_resched() in 20371f9efdefSVlastimil Babka * async compaction, or due to a fatal signal detected. In that 2038c3486f53SVlastimil Babka * case do not try further zones 20391f9efdefSVlastimil Babka */ 2040c3486f53SVlastimil Babka if ((prio == COMPACT_PRIO_ASYNC && need_resched()) 2041c3486f53SVlastimil Babka || fatal_signal_pending(current)) 20421f9efdefSVlastimil Babka break; 20431f9efdefSVlastimil Babka } 20441f9efdefSVlastimil Babka 204556de7263SMel Gorman return rc; 204656de7263SMel Gorman } 204756de7263SMel Gorman 204856de7263SMel Gorman 204976ab0f53SMel Gorman /* Compact all zones within a node */ 20507103f16dSAndrew Morton static void compact_node(int nid) 20517be62de9SRik van Riel { 2052791cae96SVlastimil Babka pg_data_t *pgdat = NODE_DATA(nid); 2053791cae96SVlastimil Babka int zoneid; 2054791cae96SVlastimil Babka struct zone *zone; 20557be62de9SRik van Riel struct compact_control cc = { 20567be62de9SRik van Riel .order = -1, 20577f354a54SDavid Rientjes .total_migrate_scanned = 0, 20587f354a54SDavid Rientjes .total_free_scanned = 0, 2059e0b9daebSDavid Rientjes .mode = MIGRATE_SYNC, 206091ca9186SDavid Rientjes .ignore_skip_hint = true, 206106ed2998SVlastimil Babka .whole_zone = true, 206273e64c51SMichal Hocko .gfp_mask = GFP_KERNEL, 20637be62de9SRik van Riel }; 20647be62de9SRik van Riel 2065791cae96SVlastimil Babka 2066791cae96SVlastimil Babka for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) { 2067791cae96SVlastimil Babka 2068791cae96SVlastimil Babka zone = &pgdat->node_zones[zoneid]; 2069791cae96SVlastimil Babka if (!populated_zone(zone)) 2070791cae96SVlastimil Babka continue; 2071791cae96SVlastimil Babka 2072791cae96SVlastimil Babka cc.nr_freepages = 0; 2073791cae96SVlastimil Babka cc.nr_migratepages = 0; 2074791cae96SVlastimil Babka cc.zone = zone; 2075791cae96SVlastimil Babka INIT_LIST_HEAD(&cc.freepages); 2076791cae96SVlastimil Babka INIT_LIST_HEAD(&cc.migratepages); 2077791cae96SVlastimil Babka 207840cacbcbSMel Gorman compact_zone(&cc); 2079791cae96SVlastimil Babka 2080791cae96SVlastimil Babka VM_BUG_ON(!list_empty(&cc.freepages)); 2081791cae96SVlastimil Babka VM_BUG_ON(!list_empty(&cc.migratepages)); 2082791cae96SVlastimil Babka } 20837be62de9SRik van Riel } 20847be62de9SRik van Riel 208576ab0f53SMel Gorman /* Compact all nodes in the system */ 20867964c06dSJason Liu static void compact_nodes(void) 208776ab0f53SMel Gorman { 208876ab0f53SMel Gorman int nid; 208976ab0f53SMel Gorman 20908575ec29SHugh Dickins /* Flush pending updates to the LRU lists */ 20918575ec29SHugh Dickins lru_add_drain_all(); 20928575ec29SHugh Dickins 209376ab0f53SMel Gorman for_each_online_node(nid) 209476ab0f53SMel Gorman compact_node(nid); 209576ab0f53SMel Gorman } 209676ab0f53SMel Gorman 209776ab0f53SMel Gorman /* The written value is actually unused, all memory is compacted */ 209876ab0f53SMel Gorman int sysctl_compact_memory; 209976ab0f53SMel Gorman 2100fec4eb2cSYaowei Bai /* 2101fec4eb2cSYaowei Bai * This is the entry point for compacting all nodes via 2102fec4eb2cSYaowei Bai * /proc/sys/vm/compact_memory 2103fec4eb2cSYaowei Bai */ 210476ab0f53SMel Gorman int sysctl_compaction_handler(struct ctl_table *table, int write, 210576ab0f53SMel Gorman void __user *buffer, size_t *length, loff_t *ppos) 210676ab0f53SMel Gorman { 210776ab0f53SMel Gorman if (write) 21087964c06dSJason Liu compact_nodes(); 210976ab0f53SMel Gorman 211076ab0f53SMel Gorman return 0; 211176ab0f53SMel Gorman } 2112ed4a6d7fSMel Gorman 2113ed4a6d7fSMel Gorman #if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA) 211474e77fb9SRashika Kheria static ssize_t sysfs_compact_node(struct device *dev, 211510fbcf4cSKay Sievers struct device_attribute *attr, 2116ed4a6d7fSMel Gorman const char *buf, size_t count) 2117ed4a6d7fSMel Gorman { 21188575ec29SHugh Dickins int nid = dev->id; 21198575ec29SHugh Dickins 21208575ec29SHugh Dickins if (nid >= 0 && nid < nr_node_ids && node_online(nid)) { 21218575ec29SHugh Dickins /* Flush pending updates to the LRU lists */ 21228575ec29SHugh Dickins lru_add_drain_all(); 21238575ec29SHugh Dickins 21248575ec29SHugh Dickins compact_node(nid); 21258575ec29SHugh Dickins } 2126ed4a6d7fSMel Gorman 2127ed4a6d7fSMel Gorman return count; 2128ed4a6d7fSMel Gorman } 21290825a6f9SJoe Perches static DEVICE_ATTR(compact, 0200, NULL, sysfs_compact_node); 2130ed4a6d7fSMel Gorman 2131ed4a6d7fSMel Gorman int compaction_register_node(struct node *node) 2132ed4a6d7fSMel Gorman { 213310fbcf4cSKay Sievers return device_create_file(&node->dev, &dev_attr_compact); 2134ed4a6d7fSMel Gorman } 2135ed4a6d7fSMel Gorman 2136ed4a6d7fSMel Gorman void compaction_unregister_node(struct node *node) 2137ed4a6d7fSMel Gorman { 213810fbcf4cSKay Sievers return device_remove_file(&node->dev, &dev_attr_compact); 2139ed4a6d7fSMel Gorman } 2140ed4a6d7fSMel Gorman #endif /* CONFIG_SYSFS && CONFIG_NUMA */ 2141ff9543fdSMichal Nazarewicz 2142698b1b30SVlastimil Babka static inline bool kcompactd_work_requested(pg_data_t *pgdat) 2143698b1b30SVlastimil Babka { 2144172400c6SVlastimil Babka return pgdat->kcompactd_max_order > 0 || kthread_should_stop(); 2145698b1b30SVlastimil Babka } 2146698b1b30SVlastimil Babka 2147698b1b30SVlastimil Babka static bool kcompactd_node_suitable(pg_data_t *pgdat) 2148698b1b30SVlastimil Babka { 2149698b1b30SVlastimil Babka int zoneid; 2150698b1b30SVlastimil Babka struct zone *zone; 2151698b1b30SVlastimil Babka enum zone_type classzone_idx = pgdat->kcompactd_classzone_idx; 2152698b1b30SVlastimil Babka 21536cd9dc3eSChen Feng for (zoneid = 0; zoneid <= classzone_idx; zoneid++) { 2154698b1b30SVlastimil Babka zone = &pgdat->node_zones[zoneid]; 2155698b1b30SVlastimil Babka 2156698b1b30SVlastimil Babka if (!populated_zone(zone)) 2157698b1b30SVlastimil Babka continue; 2158698b1b30SVlastimil Babka 2159698b1b30SVlastimil Babka if (compaction_suitable(zone, pgdat->kcompactd_max_order, 0, 2160698b1b30SVlastimil Babka classzone_idx) == COMPACT_CONTINUE) 2161698b1b30SVlastimil Babka return true; 2162698b1b30SVlastimil Babka } 2163698b1b30SVlastimil Babka 2164698b1b30SVlastimil Babka return false; 2165698b1b30SVlastimil Babka } 2166698b1b30SVlastimil Babka 2167698b1b30SVlastimil Babka static void kcompactd_do_work(pg_data_t *pgdat) 2168698b1b30SVlastimil Babka { 2169698b1b30SVlastimil Babka /* 2170698b1b30SVlastimil Babka * With no special task, compact all zones so that a page of requested 2171698b1b30SVlastimil Babka * order is allocatable. 2172698b1b30SVlastimil Babka */ 2173698b1b30SVlastimil Babka int zoneid; 2174698b1b30SVlastimil Babka struct zone *zone; 2175698b1b30SVlastimil Babka struct compact_control cc = { 2176698b1b30SVlastimil Babka .order = pgdat->kcompactd_max_order, 21777f354a54SDavid Rientjes .total_migrate_scanned = 0, 21787f354a54SDavid Rientjes .total_free_scanned = 0, 2179698b1b30SVlastimil Babka .classzone_idx = pgdat->kcompactd_classzone_idx, 2180698b1b30SVlastimil Babka .mode = MIGRATE_SYNC_LIGHT, 2181a0647dc9SDavid Rientjes .ignore_skip_hint = false, 218273e64c51SMichal Hocko .gfp_mask = GFP_KERNEL, 2183698b1b30SVlastimil Babka }; 2184698b1b30SVlastimil Babka trace_mm_compaction_kcompactd_wake(pgdat->node_id, cc.order, 2185698b1b30SVlastimil Babka cc.classzone_idx); 21867f354a54SDavid Rientjes count_compact_event(KCOMPACTD_WAKE); 2187698b1b30SVlastimil Babka 21886cd9dc3eSChen Feng for (zoneid = 0; zoneid <= cc.classzone_idx; zoneid++) { 2189698b1b30SVlastimil Babka int status; 2190698b1b30SVlastimil Babka 2191698b1b30SVlastimil Babka zone = &pgdat->node_zones[zoneid]; 2192698b1b30SVlastimil Babka if (!populated_zone(zone)) 2193698b1b30SVlastimil Babka continue; 2194698b1b30SVlastimil Babka 2195698b1b30SVlastimil Babka if (compaction_deferred(zone, cc.order)) 2196698b1b30SVlastimil Babka continue; 2197698b1b30SVlastimil Babka 2198698b1b30SVlastimil Babka if (compaction_suitable(zone, cc.order, 0, zoneid) != 2199698b1b30SVlastimil Babka COMPACT_CONTINUE) 2200698b1b30SVlastimil Babka continue; 2201698b1b30SVlastimil Babka 2202698b1b30SVlastimil Babka cc.nr_freepages = 0; 2203698b1b30SVlastimil Babka cc.nr_migratepages = 0; 22047f354a54SDavid Rientjes cc.total_migrate_scanned = 0; 22057f354a54SDavid Rientjes cc.total_free_scanned = 0; 2206698b1b30SVlastimil Babka cc.zone = zone; 2207698b1b30SVlastimil Babka INIT_LIST_HEAD(&cc.freepages); 2208698b1b30SVlastimil Babka INIT_LIST_HEAD(&cc.migratepages); 2209698b1b30SVlastimil Babka 2210172400c6SVlastimil Babka if (kthread_should_stop()) 2211172400c6SVlastimil Babka return; 221240cacbcbSMel Gorman status = compact_zone(&cc); 2213698b1b30SVlastimil Babka 22147ceb009aSVlastimil Babka if (status == COMPACT_SUCCESS) { 2215698b1b30SVlastimil Babka compaction_defer_reset(zone, cc.order, false); 2216c8f7de0bSMichal Hocko } else if (status == COMPACT_PARTIAL_SKIPPED || status == COMPACT_COMPLETE) { 2217698b1b30SVlastimil Babka /* 2218bc3106b2SDavid Rientjes * Buddy pages may become stranded on pcps that could 2219bc3106b2SDavid Rientjes * otherwise coalesce on the zone's free area for 2220bc3106b2SDavid Rientjes * order >= cc.order. This is ratelimited by the 2221bc3106b2SDavid Rientjes * upcoming deferral. 2222bc3106b2SDavid Rientjes */ 2223bc3106b2SDavid Rientjes drain_all_pages(zone); 2224bc3106b2SDavid Rientjes 2225bc3106b2SDavid Rientjes /* 2226698b1b30SVlastimil Babka * We use sync migration mode here, so we defer like 2227698b1b30SVlastimil Babka * sync direct compaction does. 2228698b1b30SVlastimil Babka */ 2229698b1b30SVlastimil Babka defer_compaction(zone, cc.order); 2230698b1b30SVlastimil Babka } 2231698b1b30SVlastimil Babka 22327f354a54SDavid Rientjes count_compact_events(KCOMPACTD_MIGRATE_SCANNED, 22337f354a54SDavid Rientjes cc.total_migrate_scanned); 22347f354a54SDavid Rientjes count_compact_events(KCOMPACTD_FREE_SCANNED, 22357f354a54SDavid Rientjes cc.total_free_scanned); 22367f354a54SDavid Rientjes 2237698b1b30SVlastimil Babka VM_BUG_ON(!list_empty(&cc.freepages)); 2238698b1b30SVlastimil Babka VM_BUG_ON(!list_empty(&cc.migratepages)); 2239698b1b30SVlastimil Babka } 2240698b1b30SVlastimil Babka 2241698b1b30SVlastimil Babka /* 2242698b1b30SVlastimil Babka * Regardless of success, we are done until woken up next. But remember 2243698b1b30SVlastimil Babka * the requested order/classzone_idx in case it was higher/tighter than 2244698b1b30SVlastimil Babka * our current ones 2245698b1b30SVlastimil Babka */ 2246698b1b30SVlastimil Babka if (pgdat->kcompactd_max_order <= cc.order) 2247698b1b30SVlastimil Babka pgdat->kcompactd_max_order = 0; 2248698b1b30SVlastimil Babka if (pgdat->kcompactd_classzone_idx >= cc.classzone_idx) 2249698b1b30SVlastimil Babka pgdat->kcompactd_classzone_idx = pgdat->nr_zones - 1; 2250698b1b30SVlastimil Babka } 2251698b1b30SVlastimil Babka 2252698b1b30SVlastimil Babka void wakeup_kcompactd(pg_data_t *pgdat, int order, int classzone_idx) 2253698b1b30SVlastimil Babka { 2254698b1b30SVlastimil Babka if (!order) 2255698b1b30SVlastimil Babka return; 2256698b1b30SVlastimil Babka 2257698b1b30SVlastimil Babka if (pgdat->kcompactd_max_order < order) 2258698b1b30SVlastimil Babka pgdat->kcompactd_max_order = order; 2259698b1b30SVlastimil Babka 2260698b1b30SVlastimil Babka if (pgdat->kcompactd_classzone_idx > classzone_idx) 2261698b1b30SVlastimil Babka pgdat->kcompactd_classzone_idx = classzone_idx; 2262698b1b30SVlastimil Babka 22636818600fSDavidlohr Bueso /* 22646818600fSDavidlohr Bueso * Pairs with implicit barrier in wait_event_freezable() 22656818600fSDavidlohr Bueso * such that wakeups are not missed. 22666818600fSDavidlohr Bueso */ 22676818600fSDavidlohr Bueso if (!wq_has_sleeper(&pgdat->kcompactd_wait)) 2268698b1b30SVlastimil Babka return; 2269698b1b30SVlastimil Babka 2270698b1b30SVlastimil Babka if (!kcompactd_node_suitable(pgdat)) 2271698b1b30SVlastimil Babka return; 2272698b1b30SVlastimil Babka 2273698b1b30SVlastimil Babka trace_mm_compaction_wakeup_kcompactd(pgdat->node_id, order, 2274698b1b30SVlastimil Babka classzone_idx); 2275698b1b30SVlastimil Babka wake_up_interruptible(&pgdat->kcompactd_wait); 2276698b1b30SVlastimil Babka } 2277698b1b30SVlastimil Babka 2278698b1b30SVlastimil Babka /* 2279698b1b30SVlastimil Babka * The background compaction daemon, started as a kernel thread 2280698b1b30SVlastimil Babka * from the init process. 2281698b1b30SVlastimil Babka */ 2282698b1b30SVlastimil Babka static int kcompactd(void *p) 2283698b1b30SVlastimil Babka { 2284698b1b30SVlastimil Babka pg_data_t *pgdat = (pg_data_t*)p; 2285698b1b30SVlastimil Babka struct task_struct *tsk = current; 2286698b1b30SVlastimil Babka 2287698b1b30SVlastimil Babka const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); 2288698b1b30SVlastimil Babka 2289698b1b30SVlastimil Babka if (!cpumask_empty(cpumask)) 2290698b1b30SVlastimil Babka set_cpus_allowed_ptr(tsk, cpumask); 2291698b1b30SVlastimil Babka 2292698b1b30SVlastimil Babka set_freezable(); 2293698b1b30SVlastimil Babka 2294698b1b30SVlastimil Babka pgdat->kcompactd_max_order = 0; 2295698b1b30SVlastimil Babka pgdat->kcompactd_classzone_idx = pgdat->nr_zones - 1; 2296698b1b30SVlastimil Babka 2297698b1b30SVlastimil Babka while (!kthread_should_stop()) { 2298eb414681SJohannes Weiner unsigned long pflags; 2299eb414681SJohannes Weiner 2300698b1b30SVlastimil Babka trace_mm_compaction_kcompactd_sleep(pgdat->node_id); 2301698b1b30SVlastimil Babka wait_event_freezable(pgdat->kcompactd_wait, 2302698b1b30SVlastimil Babka kcompactd_work_requested(pgdat)); 2303698b1b30SVlastimil Babka 2304eb414681SJohannes Weiner psi_memstall_enter(&pflags); 2305698b1b30SVlastimil Babka kcompactd_do_work(pgdat); 2306eb414681SJohannes Weiner psi_memstall_leave(&pflags); 2307698b1b30SVlastimil Babka } 2308698b1b30SVlastimil Babka 2309698b1b30SVlastimil Babka return 0; 2310698b1b30SVlastimil Babka } 2311698b1b30SVlastimil Babka 2312698b1b30SVlastimil Babka /* 2313698b1b30SVlastimil Babka * This kcompactd start function will be called by init and node-hot-add. 2314698b1b30SVlastimil Babka * On node-hot-add, kcompactd will moved to proper cpus if cpus are hot-added. 2315698b1b30SVlastimil Babka */ 2316698b1b30SVlastimil Babka int kcompactd_run(int nid) 2317698b1b30SVlastimil Babka { 2318698b1b30SVlastimil Babka pg_data_t *pgdat = NODE_DATA(nid); 2319698b1b30SVlastimil Babka int ret = 0; 2320698b1b30SVlastimil Babka 2321698b1b30SVlastimil Babka if (pgdat->kcompactd) 2322698b1b30SVlastimil Babka return 0; 2323698b1b30SVlastimil Babka 2324698b1b30SVlastimil Babka pgdat->kcompactd = kthread_run(kcompactd, pgdat, "kcompactd%d", nid); 2325698b1b30SVlastimil Babka if (IS_ERR(pgdat->kcompactd)) { 2326698b1b30SVlastimil Babka pr_err("Failed to start kcompactd on node %d\n", nid); 2327698b1b30SVlastimil Babka ret = PTR_ERR(pgdat->kcompactd); 2328698b1b30SVlastimil Babka pgdat->kcompactd = NULL; 2329698b1b30SVlastimil Babka } 2330698b1b30SVlastimil Babka return ret; 2331698b1b30SVlastimil Babka } 2332698b1b30SVlastimil Babka 2333698b1b30SVlastimil Babka /* 2334698b1b30SVlastimil Babka * Called by memory hotplug when all memory in a node is offlined. Caller must 2335698b1b30SVlastimil Babka * hold mem_hotplug_begin/end(). 2336698b1b30SVlastimil Babka */ 2337698b1b30SVlastimil Babka void kcompactd_stop(int nid) 2338698b1b30SVlastimil Babka { 2339698b1b30SVlastimil Babka struct task_struct *kcompactd = NODE_DATA(nid)->kcompactd; 2340698b1b30SVlastimil Babka 2341698b1b30SVlastimil Babka if (kcompactd) { 2342698b1b30SVlastimil Babka kthread_stop(kcompactd); 2343698b1b30SVlastimil Babka NODE_DATA(nid)->kcompactd = NULL; 2344698b1b30SVlastimil Babka } 2345698b1b30SVlastimil Babka } 2346698b1b30SVlastimil Babka 2347698b1b30SVlastimil Babka /* 2348698b1b30SVlastimil Babka * It's optimal to keep kcompactd on the same CPUs as their memory, but 2349698b1b30SVlastimil Babka * not required for correctness. So if the last cpu in a node goes 2350698b1b30SVlastimil Babka * away, we get changed to run anywhere: as the first one comes back, 2351698b1b30SVlastimil Babka * restore their cpu bindings. 2352698b1b30SVlastimil Babka */ 2353e46b1db2SAnna-Maria Gleixner static int kcompactd_cpu_online(unsigned int cpu) 2354698b1b30SVlastimil Babka { 2355698b1b30SVlastimil Babka int nid; 2356698b1b30SVlastimil Babka 2357698b1b30SVlastimil Babka for_each_node_state(nid, N_MEMORY) { 2358698b1b30SVlastimil Babka pg_data_t *pgdat = NODE_DATA(nid); 2359698b1b30SVlastimil Babka const struct cpumask *mask; 2360698b1b30SVlastimil Babka 2361698b1b30SVlastimil Babka mask = cpumask_of_node(pgdat->node_id); 2362698b1b30SVlastimil Babka 2363698b1b30SVlastimil Babka if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids) 2364698b1b30SVlastimil Babka /* One of our CPUs online: restore mask */ 2365698b1b30SVlastimil Babka set_cpus_allowed_ptr(pgdat->kcompactd, mask); 2366698b1b30SVlastimil Babka } 2367e46b1db2SAnna-Maria Gleixner return 0; 2368698b1b30SVlastimil Babka } 2369698b1b30SVlastimil Babka 2370698b1b30SVlastimil Babka static int __init kcompactd_init(void) 2371698b1b30SVlastimil Babka { 2372698b1b30SVlastimil Babka int nid; 2373e46b1db2SAnna-Maria Gleixner int ret; 2374e46b1db2SAnna-Maria Gleixner 2375e46b1db2SAnna-Maria Gleixner ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, 2376e46b1db2SAnna-Maria Gleixner "mm/compaction:online", 2377e46b1db2SAnna-Maria Gleixner kcompactd_cpu_online, NULL); 2378e46b1db2SAnna-Maria Gleixner if (ret < 0) { 2379e46b1db2SAnna-Maria Gleixner pr_err("kcompactd: failed to register hotplug callbacks.\n"); 2380e46b1db2SAnna-Maria Gleixner return ret; 2381e46b1db2SAnna-Maria Gleixner } 2382698b1b30SVlastimil Babka 2383698b1b30SVlastimil Babka for_each_node_state(nid, N_MEMORY) 2384698b1b30SVlastimil Babka kcompactd_run(nid); 2385698b1b30SVlastimil Babka return 0; 2386698b1b30SVlastimil Babka } 2387698b1b30SVlastimil Babka subsys_initcall(kcompactd_init) 2388698b1b30SVlastimil Babka 2389ff9543fdSMichal Nazarewicz #endif /* CONFIG_COMPACTION */ 2390