1748446bbSMel Gorman /* 2748446bbSMel Gorman * linux/mm/compaction.c 3748446bbSMel Gorman * 4748446bbSMel Gorman * Memory compaction for the reduction of external fragmentation. Note that 5748446bbSMel Gorman * this heavily depends upon page migration to do all the real heavy 6748446bbSMel Gorman * lifting 7748446bbSMel Gorman * 8748446bbSMel Gorman * Copyright IBM Corp. 2007-2010 Mel Gorman <mel@csn.ul.ie> 9748446bbSMel Gorman */ 10748446bbSMel Gorman #include <linux/swap.h> 11748446bbSMel Gorman #include <linux/migrate.h> 12748446bbSMel Gorman #include <linux/compaction.h> 13748446bbSMel Gorman #include <linux/mm_inline.h> 14748446bbSMel Gorman #include <linux/backing-dev.h> 1576ab0f53SMel Gorman #include <linux/sysctl.h> 16ed4a6d7fSMel Gorman #include <linux/sysfs.h> 17bf6bddf1SRafael Aquini #include <linux/balloon_compaction.h> 18194159fbSMinchan Kim #include <linux/page-isolation.h> 19b8c73fc2SAndrey Ryabinin #include <linux/kasan.h> 20748446bbSMel Gorman #include "internal.h" 21748446bbSMel Gorman 22010fc29aSMinchan Kim #ifdef CONFIG_COMPACTION 23010fc29aSMinchan Kim static inline void count_compact_event(enum vm_event_item item) 24010fc29aSMinchan Kim { 25010fc29aSMinchan Kim count_vm_event(item); 26010fc29aSMinchan Kim } 27010fc29aSMinchan Kim 28010fc29aSMinchan Kim static inline void count_compact_events(enum vm_event_item item, long delta) 29010fc29aSMinchan Kim { 30010fc29aSMinchan Kim count_vm_events(item, delta); 31010fc29aSMinchan Kim } 32010fc29aSMinchan Kim #else 33010fc29aSMinchan Kim #define count_compact_event(item) do { } while (0) 34010fc29aSMinchan Kim #define count_compact_events(item, delta) do { } while (0) 35010fc29aSMinchan Kim #endif 36010fc29aSMinchan Kim 37ff9543fdSMichal Nazarewicz #if defined CONFIG_COMPACTION || defined CONFIG_CMA 3816c4a097SJoonsoo Kim #ifdef CONFIG_TRACEPOINTS 3916c4a097SJoonsoo Kim static const char *const compaction_status_string[] = { 4016c4a097SJoonsoo Kim "deferred", 4116c4a097SJoonsoo Kim "skipped", 4216c4a097SJoonsoo Kim "continue", 4316c4a097SJoonsoo Kim "partial", 4416c4a097SJoonsoo Kim "complete", 45837d026dSJoonsoo Kim "no_suitable_page", 46837d026dSJoonsoo Kim "not_suitable_zone", 4716c4a097SJoonsoo Kim }; 4816c4a097SJoonsoo Kim #endif 49ff9543fdSMichal Nazarewicz 50b7aba698SMel Gorman #define CREATE_TRACE_POINTS 51b7aba698SMel Gorman #include <trace/events/compaction.h> 52b7aba698SMel Gorman 53748446bbSMel Gorman static unsigned long release_freepages(struct list_head *freelist) 54748446bbSMel Gorman { 55748446bbSMel Gorman struct page *page, *next; 566bace090SVlastimil Babka unsigned long high_pfn = 0; 57748446bbSMel Gorman 58748446bbSMel Gorman list_for_each_entry_safe(page, next, freelist, lru) { 596bace090SVlastimil Babka unsigned long pfn = page_to_pfn(page); 60748446bbSMel Gorman list_del(&page->lru); 61748446bbSMel Gorman __free_page(page); 626bace090SVlastimil Babka if (pfn > high_pfn) 636bace090SVlastimil Babka high_pfn = pfn; 64748446bbSMel Gorman } 65748446bbSMel Gorman 666bace090SVlastimil Babka return high_pfn; 67748446bbSMel Gorman } 68748446bbSMel Gorman 69ff9543fdSMichal Nazarewicz static void map_pages(struct list_head *list) 70ff9543fdSMichal Nazarewicz { 71ff9543fdSMichal Nazarewicz struct page *page; 72ff9543fdSMichal Nazarewicz 73ff9543fdSMichal Nazarewicz list_for_each_entry(page, list, lru) { 74ff9543fdSMichal Nazarewicz arch_alloc_page(page, 0); 75ff9543fdSMichal Nazarewicz kernel_map_pages(page, 1, 1); 76b8c73fc2SAndrey Ryabinin kasan_alloc_pages(page, 0); 77ff9543fdSMichal Nazarewicz } 78ff9543fdSMichal Nazarewicz } 79ff9543fdSMichal Nazarewicz 8047118af0SMichal Nazarewicz static inline bool migrate_async_suitable(int migratetype) 8147118af0SMichal Nazarewicz { 8247118af0SMichal Nazarewicz return is_migrate_cma(migratetype) || migratetype == MIGRATE_MOVABLE; 8347118af0SMichal Nazarewicz } 8447118af0SMichal Nazarewicz 857d49d886SVlastimil Babka /* 867d49d886SVlastimil Babka * Check that the whole (or subset of) a pageblock given by the interval of 877d49d886SVlastimil Babka * [start_pfn, end_pfn) is valid and within the same zone, before scanning it 887d49d886SVlastimil Babka * with the migration of free compaction scanner. The scanners then need to 897d49d886SVlastimil Babka * use only pfn_valid_within() check for arches that allow holes within 907d49d886SVlastimil Babka * pageblocks. 917d49d886SVlastimil Babka * 927d49d886SVlastimil Babka * Return struct page pointer of start_pfn, or NULL if checks were not passed. 937d49d886SVlastimil Babka * 947d49d886SVlastimil Babka * It's possible on some configurations to have a setup like node0 node1 node0 957d49d886SVlastimil Babka * i.e. it's possible that all pages within a zones range of pages do not 967d49d886SVlastimil Babka * belong to a single zone. We assume that a border between node0 and node1 977d49d886SVlastimil Babka * can occur within a single pageblock, but not a node0 node1 node0 987d49d886SVlastimil Babka * interleaving within a single pageblock. It is therefore sufficient to check 997d49d886SVlastimil Babka * the first and last page of a pageblock and avoid checking each individual 1007d49d886SVlastimil Babka * page in a pageblock. 1017d49d886SVlastimil Babka */ 1027d49d886SVlastimil Babka static struct page *pageblock_pfn_to_page(unsigned long start_pfn, 1037d49d886SVlastimil Babka unsigned long end_pfn, struct zone *zone) 1047d49d886SVlastimil Babka { 1057d49d886SVlastimil Babka struct page *start_page; 1067d49d886SVlastimil Babka struct page *end_page; 1077d49d886SVlastimil Babka 1087d49d886SVlastimil Babka /* end_pfn is one past the range we are checking */ 1097d49d886SVlastimil Babka end_pfn--; 1107d49d886SVlastimil Babka 1117d49d886SVlastimil Babka if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn)) 1127d49d886SVlastimil Babka return NULL; 1137d49d886SVlastimil Babka 1147d49d886SVlastimil Babka start_page = pfn_to_page(start_pfn); 1157d49d886SVlastimil Babka 1167d49d886SVlastimil Babka if (page_zone(start_page) != zone) 1177d49d886SVlastimil Babka return NULL; 1187d49d886SVlastimil Babka 1197d49d886SVlastimil Babka end_page = pfn_to_page(end_pfn); 1207d49d886SVlastimil Babka 1217d49d886SVlastimil Babka /* This gives a shorter code than deriving page_zone(end_page) */ 1227d49d886SVlastimil Babka if (page_zone_id(start_page) != page_zone_id(end_page)) 1237d49d886SVlastimil Babka return NULL; 1247d49d886SVlastimil Babka 1257d49d886SVlastimil Babka return start_page; 1267d49d886SVlastimil Babka } 1277d49d886SVlastimil Babka 128bb13ffebSMel Gorman #ifdef CONFIG_COMPACTION 12924e2716fSJoonsoo Kim 13024e2716fSJoonsoo Kim /* Do not skip compaction more than 64 times */ 13124e2716fSJoonsoo Kim #define COMPACT_MAX_DEFER_SHIFT 6 13224e2716fSJoonsoo Kim 13324e2716fSJoonsoo Kim /* 13424e2716fSJoonsoo Kim * Compaction is deferred when compaction fails to result in a page 13524e2716fSJoonsoo Kim * allocation success. 1 << compact_defer_limit compactions are skipped up 13624e2716fSJoonsoo Kim * to a limit of 1 << COMPACT_MAX_DEFER_SHIFT 13724e2716fSJoonsoo Kim */ 13824e2716fSJoonsoo Kim void defer_compaction(struct zone *zone, int order) 13924e2716fSJoonsoo Kim { 14024e2716fSJoonsoo Kim zone->compact_considered = 0; 14124e2716fSJoonsoo Kim zone->compact_defer_shift++; 14224e2716fSJoonsoo Kim 14324e2716fSJoonsoo Kim if (order < zone->compact_order_failed) 14424e2716fSJoonsoo Kim zone->compact_order_failed = order; 14524e2716fSJoonsoo Kim 14624e2716fSJoonsoo Kim if (zone->compact_defer_shift > COMPACT_MAX_DEFER_SHIFT) 14724e2716fSJoonsoo Kim zone->compact_defer_shift = COMPACT_MAX_DEFER_SHIFT; 14824e2716fSJoonsoo Kim 14924e2716fSJoonsoo Kim trace_mm_compaction_defer_compaction(zone, order); 15024e2716fSJoonsoo Kim } 15124e2716fSJoonsoo Kim 15224e2716fSJoonsoo Kim /* Returns true if compaction should be skipped this time */ 15324e2716fSJoonsoo Kim bool compaction_deferred(struct zone *zone, int order) 15424e2716fSJoonsoo Kim { 15524e2716fSJoonsoo Kim unsigned long defer_limit = 1UL << zone->compact_defer_shift; 15624e2716fSJoonsoo Kim 15724e2716fSJoonsoo Kim if (order < zone->compact_order_failed) 15824e2716fSJoonsoo Kim return false; 15924e2716fSJoonsoo Kim 16024e2716fSJoonsoo Kim /* Avoid possible overflow */ 16124e2716fSJoonsoo Kim if (++zone->compact_considered > defer_limit) 16224e2716fSJoonsoo Kim zone->compact_considered = defer_limit; 16324e2716fSJoonsoo Kim 16424e2716fSJoonsoo Kim if (zone->compact_considered >= defer_limit) 16524e2716fSJoonsoo Kim return false; 16624e2716fSJoonsoo Kim 16724e2716fSJoonsoo Kim trace_mm_compaction_deferred(zone, order); 16824e2716fSJoonsoo Kim 16924e2716fSJoonsoo Kim return true; 17024e2716fSJoonsoo Kim } 17124e2716fSJoonsoo Kim 17224e2716fSJoonsoo Kim /* 17324e2716fSJoonsoo Kim * Update defer tracking counters after successful compaction of given order, 17424e2716fSJoonsoo Kim * which means an allocation either succeeded (alloc_success == true) or is 17524e2716fSJoonsoo Kim * expected to succeed. 17624e2716fSJoonsoo Kim */ 17724e2716fSJoonsoo Kim void compaction_defer_reset(struct zone *zone, int order, 17824e2716fSJoonsoo Kim bool alloc_success) 17924e2716fSJoonsoo Kim { 18024e2716fSJoonsoo Kim if (alloc_success) { 18124e2716fSJoonsoo Kim zone->compact_considered = 0; 18224e2716fSJoonsoo Kim zone->compact_defer_shift = 0; 18324e2716fSJoonsoo Kim } 18424e2716fSJoonsoo Kim if (order >= zone->compact_order_failed) 18524e2716fSJoonsoo Kim zone->compact_order_failed = order + 1; 18624e2716fSJoonsoo Kim 18724e2716fSJoonsoo Kim trace_mm_compaction_defer_reset(zone, order); 18824e2716fSJoonsoo Kim } 18924e2716fSJoonsoo Kim 19024e2716fSJoonsoo Kim /* Returns true if restarting compaction after many failures */ 19124e2716fSJoonsoo Kim bool compaction_restarting(struct zone *zone, int order) 19224e2716fSJoonsoo Kim { 19324e2716fSJoonsoo Kim if (order < zone->compact_order_failed) 19424e2716fSJoonsoo Kim return false; 19524e2716fSJoonsoo Kim 19624e2716fSJoonsoo Kim return zone->compact_defer_shift == COMPACT_MAX_DEFER_SHIFT && 19724e2716fSJoonsoo Kim zone->compact_considered >= 1UL << zone->compact_defer_shift; 19824e2716fSJoonsoo Kim } 19924e2716fSJoonsoo Kim 200bb13ffebSMel Gorman /* Returns true if the pageblock should be scanned for pages to isolate. */ 201bb13ffebSMel Gorman static inline bool isolation_suitable(struct compact_control *cc, 202bb13ffebSMel Gorman struct page *page) 203bb13ffebSMel Gorman { 204bb13ffebSMel Gorman if (cc->ignore_skip_hint) 205bb13ffebSMel Gorman return true; 206bb13ffebSMel Gorman 207bb13ffebSMel Gorman return !get_pageblock_skip(page); 208bb13ffebSMel Gorman } 209bb13ffebSMel Gorman 21002333641SVlastimil Babka static void reset_cached_positions(struct zone *zone) 21102333641SVlastimil Babka { 21202333641SVlastimil Babka zone->compact_cached_migrate_pfn[0] = zone->zone_start_pfn; 21302333641SVlastimil Babka zone->compact_cached_migrate_pfn[1] = zone->zone_start_pfn; 21402333641SVlastimil Babka zone->compact_cached_free_pfn = zone_end_pfn(zone); 21502333641SVlastimil Babka } 21602333641SVlastimil Babka 217bb13ffebSMel Gorman /* 218bb13ffebSMel Gorman * This function is called to clear all cached information on pageblocks that 219bb13ffebSMel Gorman * should be skipped for page isolation when the migrate and free page scanner 220bb13ffebSMel Gorman * meet. 221bb13ffebSMel Gorman */ 22262997027SMel Gorman static void __reset_isolation_suitable(struct zone *zone) 223bb13ffebSMel Gorman { 224bb13ffebSMel Gorman unsigned long start_pfn = zone->zone_start_pfn; 225108bcc96SCody P Schafer unsigned long end_pfn = zone_end_pfn(zone); 226bb13ffebSMel Gorman unsigned long pfn; 227bb13ffebSMel Gorman 22862997027SMel Gorman zone->compact_blockskip_flush = false; 229bb13ffebSMel Gorman 230bb13ffebSMel Gorman /* Walk the zone and mark every pageblock as suitable for isolation */ 231bb13ffebSMel Gorman for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) { 232bb13ffebSMel Gorman struct page *page; 233bb13ffebSMel Gorman 234bb13ffebSMel Gorman cond_resched(); 235bb13ffebSMel Gorman 236bb13ffebSMel Gorman if (!pfn_valid(pfn)) 237bb13ffebSMel Gorman continue; 238bb13ffebSMel Gorman 239bb13ffebSMel Gorman page = pfn_to_page(pfn); 240bb13ffebSMel Gorman if (zone != page_zone(page)) 241bb13ffebSMel Gorman continue; 242bb13ffebSMel Gorman 243bb13ffebSMel Gorman clear_pageblock_skip(page); 244bb13ffebSMel Gorman } 24502333641SVlastimil Babka 24602333641SVlastimil Babka reset_cached_positions(zone); 247bb13ffebSMel Gorman } 248bb13ffebSMel Gorman 24962997027SMel Gorman void reset_isolation_suitable(pg_data_t *pgdat) 25062997027SMel Gorman { 25162997027SMel Gorman int zoneid; 25262997027SMel Gorman 25362997027SMel Gorman for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) { 25462997027SMel Gorman struct zone *zone = &pgdat->node_zones[zoneid]; 25562997027SMel Gorman if (!populated_zone(zone)) 25662997027SMel Gorman continue; 25762997027SMel Gorman 25862997027SMel Gorman /* Only flush if a full compaction finished recently */ 25962997027SMel Gorman if (zone->compact_blockskip_flush) 26062997027SMel Gorman __reset_isolation_suitable(zone); 26162997027SMel Gorman } 26262997027SMel Gorman } 26362997027SMel Gorman 264bb13ffebSMel Gorman /* 265bb13ffebSMel Gorman * If no pages were isolated then mark this pageblock to be skipped in the 26662997027SMel Gorman * future. The information is later cleared by __reset_isolation_suitable(). 267bb13ffebSMel Gorman */ 268c89511abSMel Gorman static void update_pageblock_skip(struct compact_control *cc, 269c89511abSMel Gorman struct page *page, unsigned long nr_isolated, 270edc2ca61SVlastimil Babka bool migrate_scanner) 271bb13ffebSMel Gorman { 272c89511abSMel Gorman struct zone *zone = cc->zone; 27335979ef3SDavid Rientjes unsigned long pfn; 2746815bf3fSJoonsoo Kim 2756815bf3fSJoonsoo Kim if (cc->ignore_skip_hint) 2766815bf3fSJoonsoo Kim return; 2776815bf3fSJoonsoo Kim 278bb13ffebSMel Gorman if (!page) 279bb13ffebSMel Gorman return; 280bb13ffebSMel Gorman 28135979ef3SDavid Rientjes if (nr_isolated) 28235979ef3SDavid Rientjes return; 28335979ef3SDavid Rientjes 284bb13ffebSMel Gorman set_pageblock_skip(page); 285c89511abSMel Gorman 28635979ef3SDavid Rientjes pfn = page_to_pfn(page); 28735979ef3SDavid Rientjes 28835979ef3SDavid Rientjes /* Update where async and sync compaction should restart */ 289c89511abSMel Gorman if (migrate_scanner) { 29035979ef3SDavid Rientjes if (pfn > zone->compact_cached_migrate_pfn[0]) 29135979ef3SDavid Rientjes zone->compact_cached_migrate_pfn[0] = pfn; 292e0b9daebSDavid Rientjes if (cc->mode != MIGRATE_ASYNC && 293e0b9daebSDavid Rientjes pfn > zone->compact_cached_migrate_pfn[1]) 29435979ef3SDavid Rientjes zone->compact_cached_migrate_pfn[1] = pfn; 295c89511abSMel Gorman } else { 29635979ef3SDavid Rientjes if (pfn < zone->compact_cached_free_pfn) 297c89511abSMel Gorman zone->compact_cached_free_pfn = pfn; 298c89511abSMel Gorman } 299c89511abSMel Gorman } 300bb13ffebSMel Gorman #else 301bb13ffebSMel Gorman static inline bool isolation_suitable(struct compact_control *cc, 302bb13ffebSMel Gorman struct page *page) 303bb13ffebSMel Gorman { 304bb13ffebSMel Gorman return true; 305bb13ffebSMel Gorman } 306bb13ffebSMel Gorman 307c89511abSMel Gorman static void update_pageblock_skip(struct compact_control *cc, 308c89511abSMel Gorman struct page *page, unsigned long nr_isolated, 309edc2ca61SVlastimil Babka bool migrate_scanner) 310bb13ffebSMel Gorman { 311bb13ffebSMel Gorman } 312bb13ffebSMel Gorman #endif /* CONFIG_COMPACTION */ 313bb13ffebSMel Gorman 3141f9efdefSVlastimil Babka /* 3158b44d279SVlastimil Babka * Compaction requires the taking of some coarse locks that are potentially 3168b44d279SVlastimil Babka * very heavily contended. For async compaction, back out if the lock cannot 3178b44d279SVlastimil Babka * be taken immediately. For sync compaction, spin on the lock if needed. 3188b44d279SVlastimil Babka * 3198b44d279SVlastimil Babka * Returns true if the lock is held 3208b44d279SVlastimil Babka * Returns false if the lock is not held and compaction should abort 3211f9efdefSVlastimil Babka */ 3228b44d279SVlastimil Babka static bool compact_trylock_irqsave(spinlock_t *lock, unsigned long *flags, 3238b44d279SVlastimil Babka struct compact_control *cc) 3248b44d279SVlastimil Babka { 3258b44d279SVlastimil Babka if (cc->mode == MIGRATE_ASYNC) { 3268b44d279SVlastimil Babka if (!spin_trylock_irqsave(lock, *flags)) { 3278b44d279SVlastimil Babka cc->contended = COMPACT_CONTENDED_LOCK; 3288b44d279SVlastimil Babka return false; 3298b44d279SVlastimil Babka } 3308b44d279SVlastimil Babka } else { 3318b44d279SVlastimil Babka spin_lock_irqsave(lock, *flags); 3328b44d279SVlastimil Babka } 3331f9efdefSVlastimil Babka 3348b44d279SVlastimil Babka return true; 3352a1402aaSMel Gorman } 3362a1402aaSMel Gorman 33785aa125fSMichal Nazarewicz /* 338c67fe375SMel Gorman * Compaction requires the taking of some coarse locks that are potentially 3398b44d279SVlastimil Babka * very heavily contended. The lock should be periodically unlocked to avoid 3408b44d279SVlastimil Babka * having disabled IRQs for a long time, even when there is nobody waiting on 3418b44d279SVlastimil Babka * the lock. It might also be that allowing the IRQs will result in 3428b44d279SVlastimil Babka * need_resched() becoming true. If scheduling is needed, async compaction 3438b44d279SVlastimil Babka * aborts. Sync compaction schedules. 3448b44d279SVlastimil Babka * Either compaction type will also abort if a fatal signal is pending. 3458b44d279SVlastimil Babka * In either case if the lock was locked, it is dropped and not regained. 346c67fe375SMel Gorman * 3478b44d279SVlastimil Babka * Returns true if compaction should abort due to fatal signal pending, or 3488b44d279SVlastimil Babka * async compaction due to need_resched() 3498b44d279SVlastimil Babka * Returns false when compaction can continue (sync compaction might have 3508b44d279SVlastimil Babka * scheduled) 351c67fe375SMel Gorman */ 3528b44d279SVlastimil Babka static bool compact_unlock_should_abort(spinlock_t *lock, 3538b44d279SVlastimil Babka unsigned long flags, bool *locked, struct compact_control *cc) 354c67fe375SMel Gorman { 3558b44d279SVlastimil Babka if (*locked) { 3568b44d279SVlastimil Babka spin_unlock_irqrestore(lock, flags); 3578b44d279SVlastimil Babka *locked = false; 358c67fe375SMel Gorman } 359c67fe375SMel Gorman 3608b44d279SVlastimil Babka if (fatal_signal_pending(current)) { 3618b44d279SVlastimil Babka cc->contended = COMPACT_CONTENDED_SCHED; 3628b44d279SVlastimil Babka return true; 3638b44d279SVlastimil Babka } 3648b44d279SVlastimil Babka 3658b44d279SVlastimil Babka if (need_resched()) { 366e0b9daebSDavid Rientjes if (cc->mode == MIGRATE_ASYNC) { 3678b44d279SVlastimil Babka cc->contended = COMPACT_CONTENDED_SCHED; 3688b44d279SVlastimil Babka return true; 369c67fe375SMel Gorman } 370c67fe375SMel Gorman cond_resched(); 371c67fe375SMel Gorman } 372c67fe375SMel Gorman 3738b44d279SVlastimil Babka return false; 374c67fe375SMel Gorman } 375c67fe375SMel Gorman 376be976572SVlastimil Babka /* 377be976572SVlastimil Babka * Aside from avoiding lock contention, compaction also periodically checks 378be976572SVlastimil Babka * need_resched() and either schedules in sync compaction or aborts async 3798b44d279SVlastimil Babka * compaction. This is similar to what compact_unlock_should_abort() does, but 380be976572SVlastimil Babka * is used where no lock is concerned. 381be976572SVlastimil Babka * 382be976572SVlastimil Babka * Returns false when no scheduling was needed, or sync compaction scheduled. 383be976572SVlastimil Babka * Returns true when async compaction should abort. 384be976572SVlastimil Babka */ 385be976572SVlastimil Babka static inline bool compact_should_abort(struct compact_control *cc) 386be976572SVlastimil Babka { 387be976572SVlastimil Babka /* async compaction aborts if contended */ 388be976572SVlastimil Babka if (need_resched()) { 389be976572SVlastimil Babka if (cc->mode == MIGRATE_ASYNC) { 3901f9efdefSVlastimil Babka cc->contended = COMPACT_CONTENDED_SCHED; 391be976572SVlastimil Babka return true; 392be976572SVlastimil Babka } 393be976572SVlastimil Babka 394be976572SVlastimil Babka cond_resched(); 395be976572SVlastimil Babka } 396be976572SVlastimil Babka 397be976572SVlastimil Babka return false; 398be976572SVlastimil Babka } 399be976572SVlastimil Babka 400c67fe375SMel Gorman /* 4019e4be470SJerome Marchand * Isolate free pages onto a private freelist. If @strict is true, will abort 4029e4be470SJerome Marchand * returning 0 on any invalid PFNs or non-free pages inside of the pageblock 4039e4be470SJerome Marchand * (even though it may still end up isolating some pages). 40485aa125fSMichal Nazarewicz */ 405f40d1e42SMel Gorman static unsigned long isolate_freepages_block(struct compact_control *cc, 406e14c720eSVlastimil Babka unsigned long *start_pfn, 40785aa125fSMichal Nazarewicz unsigned long end_pfn, 40885aa125fSMichal Nazarewicz struct list_head *freelist, 40985aa125fSMichal Nazarewicz bool strict) 410748446bbSMel Gorman { 411b7aba698SMel Gorman int nr_scanned = 0, total_isolated = 0; 412bb13ffebSMel Gorman struct page *cursor, *valid_page = NULL; 413b8b2d825SXiubo Li unsigned long flags = 0; 414f40d1e42SMel Gorman bool locked = false; 415e14c720eSVlastimil Babka unsigned long blockpfn = *start_pfn; 416748446bbSMel Gorman 417748446bbSMel Gorman cursor = pfn_to_page(blockpfn); 418748446bbSMel Gorman 419f40d1e42SMel Gorman /* Isolate free pages. */ 420748446bbSMel Gorman for (; blockpfn < end_pfn; blockpfn++, cursor++) { 421748446bbSMel Gorman int isolated, i; 422748446bbSMel Gorman struct page *page = cursor; 423748446bbSMel Gorman 4248b44d279SVlastimil Babka /* 4258b44d279SVlastimil Babka * Periodically drop the lock (if held) regardless of its 4268b44d279SVlastimil Babka * contention, to give chance to IRQs. Abort if fatal signal 4278b44d279SVlastimil Babka * pending or async compaction detects need_resched() 4288b44d279SVlastimil Babka */ 4298b44d279SVlastimil Babka if (!(blockpfn % SWAP_CLUSTER_MAX) 4308b44d279SVlastimil Babka && compact_unlock_should_abort(&cc->zone->lock, flags, 4318b44d279SVlastimil Babka &locked, cc)) 4328b44d279SVlastimil Babka break; 4338b44d279SVlastimil Babka 434b7aba698SMel Gorman nr_scanned++; 435f40d1e42SMel Gorman if (!pfn_valid_within(blockpfn)) 4362af120bcSLaura Abbott goto isolate_fail; 4372af120bcSLaura Abbott 438bb13ffebSMel Gorman if (!valid_page) 439bb13ffebSMel Gorman valid_page = page; 440*9fcd6d2eSVlastimil Babka 441*9fcd6d2eSVlastimil Babka /* 442*9fcd6d2eSVlastimil Babka * For compound pages such as THP and hugetlbfs, we can save 443*9fcd6d2eSVlastimil Babka * potentially a lot of iterations if we skip them at once. 444*9fcd6d2eSVlastimil Babka * The check is racy, but we can consider only valid values 445*9fcd6d2eSVlastimil Babka * and the only danger is skipping too much. 446*9fcd6d2eSVlastimil Babka */ 447*9fcd6d2eSVlastimil Babka if (PageCompound(page)) { 448*9fcd6d2eSVlastimil Babka unsigned int comp_order = compound_order(page); 449*9fcd6d2eSVlastimil Babka 450*9fcd6d2eSVlastimil Babka if (likely(comp_order < MAX_ORDER)) { 451*9fcd6d2eSVlastimil Babka blockpfn += (1UL << comp_order) - 1; 452*9fcd6d2eSVlastimil Babka cursor += (1UL << comp_order) - 1; 453*9fcd6d2eSVlastimil Babka } 454*9fcd6d2eSVlastimil Babka 455*9fcd6d2eSVlastimil Babka goto isolate_fail; 456*9fcd6d2eSVlastimil Babka } 457*9fcd6d2eSVlastimil Babka 458f40d1e42SMel Gorman if (!PageBuddy(page)) 4592af120bcSLaura Abbott goto isolate_fail; 460f40d1e42SMel Gorman 461f40d1e42SMel Gorman /* 46269b7189fSVlastimil Babka * If we already hold the lock, we can skip some rechecking. 46369b7189fSVlastimil Babka * Note that if we hold the lock now, checked_pageblock was 46469b7189fSVlastimil Babka * already set in some previous iteration (or strict is true), 46569b7189fSVlastimil Babka * so it is correct to skip the suitable migration target 46669b7189fSVlastimil Babka * recheck as well. 46769b7189fSVlastimil Babka */ 46869b7189fSVlastimil Babka if (!locked) { 46969b7189fSVlastimil Babka /* 470f40d1e42SMel Gorman * The zone lock must be held to isolate freepages. 471f40d1e42SMel Gorman * Unfortunately this is a very coarse lock and can be 472f40d1e42SMel Gorman * heavily contended if there are parallel allocations 473f40d1e42SMel Gorman * or parallel compactions. For async compaction do not 474f40d1e42SMel Gorman * spin on the lock and we acquire the lock as late as 475f40d1e42SMel Gorman * possible. 476f40d1e42SMel Gorman */ 4778b44d279SVlastimil Babka locked = compact_trylock_irqsave(&cc->zone->lock, 4788b44d279SVlastimil Babka &flags, cc); 479f40d1e42SMel Gorman if (!locked) 480f40d1e42SMel Gorman break; 481f40d1e42SMel Gorman 482f40d1e42SMel Gorman /* Recheck this is a buddy page under lock */ 483f40d1e42SMel Gorman if (!PageBuddy(page)) 4842af120bcSLaura Abbott goto isolate_fail; 48569b7189fSVlastimil Babka } 486748446bbSMel Gorman 487748446bbSMel Gorman /* Found a free page, break it into order-0 pages */ 488748446bbSMel Gorman isolated = split_free_page(page); 489748446bbSMel Gorman total_isolated += isolated; 490748446bbSMel Gorman for (i = 0; i < isolated; i++) { 491748446bbSMel Gorman list_add(&page->lru, freelist); 492748446bbSMel Gorman page++; 493748446bbSMel Gorman } 494748446bbSMel Gorman 495748446bbSMel Gorman /* If a page was split, advance to the end of it */ 496748446bbSMel Gorman if (isolated) { 497932ff6bbSJoonsoo Kim cc->nr_freepages += isolated; 498932ff6bbSJoonsoo Kim if (!strict && 499932ff6bbSJoonsoo Kim cc->nr_migratepages <= cc->nr_freepages) { 500932ff6bbSJoonsoo Kim blockpfn += isolated; 501932ff6bbSJoonsoo Kim break; 502932ff6bbSJoonsoo Kim } 503932ff6bbSJoonsoo Kim 504748446bbSMel Gorman blockpfn += isolated - 1; 505748446bbSMel Gorman cursor += isolated - 1; 5062af120bcSLaura Abbott continue; 507748446bbSMel Gorman } 5082af120bcSLaura Abbott 5092af120bcSLaura Abbott isolate_fail: 5102af120bcSLaura Abbott if (strict) 5112af120bcSLaura Abbott break; 5122af120bcSLaura Abbott else 5132af120bcSLaura Abbott continue; 5142af120bcSLaura Abbott 515748446bbSMel Gorman } 516748446bbSMel Gorman 517*9fcd6d2eSVlastimil Babka /* 518*9fcd6d2eSVlastimil Babka * There is a tiny chance that we have read bogus compound_order(), 519*9fcd6d2eSVlastimil Babka * so be careful to not go outside of the pageblock. 520*9fcd6d2eSVlastimil Babka */ 521*9fcd6d2eSVlastimil Babka if (unlikely(blockpfn > end_pfn)) 522*9fcd6d2eSVlastimil Babka blockpfn = end_pfn; 523*9fcd6d2eSVlastimil Babka 524e34d85f0SJoonsoo Kim trace_mm_compaction_isolate_freepages(*start_pfn, blockpfn, 525e34d85f0SJoonsoo Kim nr_scanned, total_isolated); 526e34d85f0SJoonsoo Kim 527e14c720eSVlastimil Babka /* Record how far we have got within the block */ 528e14c720eSVlastimil Babka *start_pfn = blockpfn; 529e14c720eSVlastimil Babka 530f40d1e42SMel Gorman /* 531f40d1e42SMel Gorman * If strict isolation is requested by CMA then check that all the 532f40d1e42SMel Gorman * pages requested were isolated. If there were any failures, 0 is 533f40d1e42SMel Gorman * returned and CMA will fail. 534f40d1e42SMel Gorman */ 5352af120bcSLaura Abbott if (strict && blockpfn < end_pfn) 536f40d1e42SMel Gorman total_isolated = 0; 537f40d1e42SMel Gorman 538f40d1e42SMel Gorman if (locked) 539f40d1e42SMel Gorman spin_unlock_irqrestore(&cc->zone->lock, flags); 540f40d1e42SMel Gorman 541bb13ffebSMel Gorman /* Update the pageblock-skip if the whole pageblock was scanned */ 542bb13ffebSMel Gorman if (blockpfn == end_pfn) 543edc2ca61SVlastimil Babka update_pageblock_skip(cc, valid_page, total_isolated, false); 544bb13ffebSMel Gorman 545010fc29aSMinchan Kim count_compact_events(COMPACTFREE_SCANNED, nr_scanned); 546397487dbSMel Gorman if (total_isolated) 547010fc29aSMinchan Kim count_compact_events(COMPACTISOLATED, total_isolated); 548748446bbSMel Gorman return total_isolated; 549748446bbSMel Gorman } 550748446bbSMel Gorman 55185aa125fSMichal Nazarewicz /** 55285aa125fSMichal Nazarewicz * isolate_freepages_range() - isolate free pages. 55385aa125fSMichal Nazarewicz * @start_pfn: The first PFN to start isolating. 55485aa125fSMichal Nazarewicz * @end_pfn: The one-past-last PFN. 55585aa125fSMichal Nazarewicz * 55685aa125fSMichal Nazarewicz * Non-free pages, invalid PFNs, or zone boundaries within the 55785aa125fSMichal Nazarewicz * [start_pfn, end_pfn) range are considered errors, cause function to 55885aa125fSMichal Nazarewicz * undo its actions and return zero. 55985aa125fSMichal Nazarewicz * 56085aa125fSMichal Nazarewicz * Otherwise, function returns one-past-the-last PFN of isolated page 56185aa125fSMichal Nazarewicz * (which may be greater then end_pfn if end fell in a middle of 56285aa125fSMichal Nazarewicz * a free page). 56385aa125fSMichal Nazarewicz */ 564ff9543fdSMichal Nazarewicz unsigned long 565bb13ffebSMel Gorman isolate_freepages_range(struct compact_control *cc, 566bb13ffebSMel Gorman unsigned long start_pfn, unsigned long end_pfn) 56785aa125fSMichal Nazarewicz { 568f40d1e42SMel Gorman unsigned long isolated, pfn, block_end_pfn; 56985aa125fSMichal Nazarewicz LIST_HEAD(freelist); 57085aa125fSMichal Nazarewicz 5717d49d886SVlastimil Babka pfn = start_pfn; 57285aa125fSMichal Nazarewicz block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages); 5737d49d886SVlastimil Babka 5747d49d886SVlastimil Babka for (; pfn < end_pfn; pfn += isolated, 5757d49d886SVlastimil Babka block_end_pfn += pageblock_nr_pages) { 576e14c720eSVlastimil Babka /* Protect pfn from changing by isolate_freepages_block */ 577e14c720eSVlastimil Babka unsigned long isolate_start_pfn = pfn; 5787d49d886SVlastimil Babka 57985aa125fSMichal Nazarewicz block_end_pfn = min(block_end_pfn, end_pfn); 58085aa125fSMichal Nazarewicz 58158420016SJoonsoo Kim /* 58258420016SJoonsoo Kim * pfn could pass the block_end_pfn if isolated freepage 58358420016SJoonsoo Kim * is more than pageblock order. In this case, we adjust 58458420016SJoonsoo Kim * scanning range to right one. 58558420016SJoonsoo Kim */ 58658420016SJoonsoo Kim if (pfn >= block_end_pfn) { 58758420016SJoonsoo Kim block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages); 58858420016SJoonsoo Kim block_end_pfn = min(block_end_pfn, end_pfn); 58958420016SJoonsoo Kim } 59058420016SJoonsoo Kim 5917d49d886SVlastimil Babka if (!pageblock_pfn_to_page(pfn, block_end_pfn, cc->zone)) 5927d49d886SVlastimil Babka break; 5937d49d886SVlastimil Babka 594e14c720eSVlastimil Babka isolated = isolate_freepages_block(cc, &isolate_start_pfn, 595e14c720eSVlastimil Babka block_end_pfn, &freelist, true); 59685aa125fSMichal Nazarewicz 59785aa125fSMichal Nazarewicz /* 59885aa125fSMichal Nazarewicz * In strict mode, isolate_freepages_block() returns 0 if 59985aa125fSMichal Nazarewicz * there are any holes in the block (ie. invalid PFNs or 60085aa125fSMichal Nazarewicz * non-free pages). 60185aa125fSMichal Nazarewicz */ 60285aa125fSMichal Nazarewicz if (!isolated) 60385aa125fSMichal Nazarewicz break; 60485aa125fSMichal Nazarewicz 60585aa125fSMichal Nazarewicz /* 60685aa125fSMichal Nazarewicz * If we managed to isolate pages, it is always (1 << n) * 60785aa125fSMichal Nazarewicz * pageblock_nr_pages for some non-negative n. (Max order 60885aa125fSMichal Nazarewicz * page may span two pageblocks). 60985aa125fSMichal Nazarewicz */ 61085aa125fSMichal Nazarewicz } 61185aa125fSMichal Nazarewicz 61285aa125fSMichal Nazarewicz /* split_free_page does not map the pages */ 61385aa125fSMichal Nazarewicz map_pages(&freelist); 61485aa125fSMichal Nazarewicz 61585aa125fSMichal Nazarewicz if (pfn < end_pfn) { 61685aa125fSMichal Nazarewicz /* Loop terminated early, cleanup. */ 61785aa125fSMichal Nazarewicz release_freepages(&freelist); 61885aa125fSMichal Nazarewicz return 0; 61985aa125fSMichal Nazarewicz } 62085aa125fSMichal Nazarewicz 62185aa125fSMichal Nazarewicz /* We don't use freelists for anything. */ 62285aa125fSMichal Nazarewicz return pfn; 62385aa125fSMichal Nazarewicz } 62485aa125fSMichal Nazarewicz 625748446bbSMel Gorman /* Update the number of anon and file isolated pages in the zone */ 626edc2ca61SVlastimil Babka static void acct_isolated(struct zone *zone, struct compact_control *cc) 627748446bbSMel Gorman { 628748446bbSMel Gorman struct page *page; 629b9e84ac1SMinchan Kim unsigned int count[2] = { 0, }; 630748446bbSMel Gorman 631edc2ca61SVlastimil Babka if (list_empty(&cc->migratepages)) 632edc2ca61SVlastimil Babka return; 633edc2ca61SVlastimil Babka 634b9e84ac1SMinchan Kim list_for_each_entry(page, &cc->migratepages, lru) 635b9e84ac1SMinchan Kim count[!!page_is_file_cache(page)]++; 636748446bbSMel Gorman 637c67fe375SMel Gorman mod_zone_page_state(zone, NR_ISOLATED_ANON, count[0]); 638c67fe375SMel Gorman mod_zone_page_state(zone, NR_ISOLATED_FILE, count[1]); 639c67fe375SMel Gorman } 640748446bbSMel Gorman 641748446bbSMel Gorman /* Similar to reclaim, but different enough that they don't share logic */ 642748446bbSMel Gorman static bool too_many_isolated(struct zone *zone) 643748446bbSMel Gorman { 644bc693045SMinchan Kim unsigned long active, inactive, isolated; 645748446bbSMel Gorman 646748446bbSMel Gorman inactive = zone_page_state(zone, NR_INACTIVE_FILE) + 647748446bbSMel Gorman zone_page_state(zone, NR_INACTIVE_ANON); 648bc693045SMinchan Kim active = zone_page_state(zone, NR_ACTIVE_FILE) + 649bc693045SMinchan Kim zone_page_state(zone, NR_ACTIVE_ANON); 650748446bbSMel Gorman isolated = zone_page_state(zone, NR_ISOLATED_FILE) + 651748446bbSMel Gorman zone_page_state(zone, NR_ISOLATED_ANON); 652748446bbSMel Gorman 653bc693045SMinchan Kim return isolated > (inactive + active) / 2; 654748446bbSMel Gorman } 655748446bbSMel Gorman 6562fe86e00SMichal Nazarewicz /** 657edc2ca61SVlastimil Babka * isolate_migratepages_block() - isolate all migrate-able pages within 658edc2ca61SVlastimil Babka * a single pageblock 6592fe86e00SMichal Nazarewicz * @cc: Compaction control structure. 660edc2ca61SVlastimil Babka * @low_pfn: The first PFN to isolate 661edc2ca61SVlastimil Babka * @end_pfn: The one-past-the-last PFN to isolate, within same pageblock 662edc2ca61SVlastimil Babka * @isolate_mode: Isolation mode to be used. 6632fe86e00SMichal Nazarewicz * 6642fe86e00SMichal Nazarewicz * Isolate all pages that can be migrated from the range specified by 665edc2ca61SVlastimil Babka * [low_pfn, end_pfn). The range is expected to be within same pageblock. 666edc2ca61SVlastimil Babka * Returns zero if there is a fatal signal pending, otherwise PFN of the 667edc2ca61SVlastimil Babka * first page that was not scanned (which may be both less, equal to or more 668edc2ca61SVlastimil Babka * than end_pfn). 6692fe86e00SMichal Nazarewicz * 670edc2ca61SVlastimil Babka * The pages are isolated on cc->migratepages list (not required to be empty), 671edc2ca61SVlastimil Babka * and cc->nr_migratepages is updated accordingly. The cc->migrate_pfn field 672edc2ca61SVlastimil Babka * is neither read nor updated. 673748446bbSMel Gorman */ 674edc2ca61SVlastimil Babka static unsigned long 675edc2ca61SVlastimil Babka isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, 676edc2ca61SVlastimil Babka unsigned long end_pfn, isolate_mode_t isolate_mode) 677748446bbSMel Gorman { 678edc2ca61SVlastimil Babka struct zone *zone = cc->zone; 679b7aba698SMel Gorman unsigned long nr_scanned = 0, nr_isolated = 0; 680748446bbSMel Gorman struct list_head *migratelist = &cc->migratepages; 681fa9add64SHugh Dickins struct lruvec *lruvec; 682b8b2d825SXiubo Li unsigned long flags = 0; 6832a1402aaSMel Gorman bool locked = false; 684bb13ffebSMel Gorman struct page *page = NULL, *valid_page = NULL; 685e34d85f0SJoonsoo Kim unsigned long start_pfn = low_pfn; 686748446bbSMel Gorman 687748446bbSMel Gorman /* 688748446bbSMel Gorman * Ensure that there are not too many pages isolated from the LRU 689748446bbSMel Gorman * list by either parallel reclaimers or compaction. If there are, 690748446bbSMel Gorman * delay for some time until fewer pages are isolated 691748446bbSMel Gorman */ 692748446bbSMel Gorman while (unlikely(too_many_isolated(zone))) { 693f9e35b3bSMel Gorman /* async migration should just abort */ 694e0b9daebSDavid Rientjes if (cc->mode == MIGRATE_ASYNC) 6952fe86e00SMichal Nazarewicz return 0; 696f9e35b3bSMel Gorman 697748446bbSMel Gorman congestion_wait(BLK_RW_ASYNC, HZ/10); 698748446bbSMel Gorman 699748446bbSMel Gorman if (fatal_signal_pending(current)) 7002fe86e00SMichal Nazarewicz return 0; 701748446bbSMel Gorman } 702748446bbSMel Gorman 703be976572SVlastimil Babka if (compact_should_abort(cc)) 704aeef4b83SDavid Rientjes return 0; 705aeef4b83SDavid Rientjes 706748446bbSMel Gorman /* Time to isolate some pages for migration */ 707748446bbSMel Gorman for (; low_pfn < end_pfn; low_pfn++) { 70829c0dde8SVlastimil Babka bool is_lru; 70929c0dde8SVlastimil Babka 7108b44d279SVlastimil Babka /* 7118b44d279SVlastimil Babka * Periodically drop the lock (if held) regardless of its 7128b44d279SVlastimil Babka * contention, to give chance to IRQs. Abort async compaction 7138b44d279SVlastimil Babka * if contended. 7148b44d279SVlastimil Babka */ 7158b44d279SVlastimil Babka if (!(low_pfn % SWAP_CLUSTER_MAX) 7168b44d279SVlastimil Babka && compact_unlock_should_abort(&zone->lru_lock, flags, 7178b44d279SVlastimil Babka &locked, cc)) 7188b44d279SVlastimil Babka break; 719b2eef8c0SAndrea Arcangeli 720748446bbSMel Gorman if (!pfn_valid_within(low_pfn)) 721748446bbSMel Gorman continue; 722b7aba698SMel Gorman nr_scanned++; 723748446bbSMel Gorman 724748446bbSMel Gorman page = pfn_to_page(low_pfn); 725dc908600SMel Gorman 726bb13ffebSMel Gorman if (!valid_page) 727bb13ffebSMel Gorman valid_page = page; 728bb13ffebSMel Gorman 729c122b208SJoonsoo Kim /* 73099c0fd5eSVlastimil Babka * Skip if free. We read page order here without zone lock 73199c0fd5eSVlastimil Babka * which is generally unsafe, but the race window is small and 73299c0fd5eSVlastimil Babka * the worst thing that can happen is that we skip some 73399c0fd5eSVlastimil Babka * potential isolation targets. 7346c14466cSMel Gorman */ 73599c0fd5eSVlastimil Babka if (PageBuddy(page)) { 73699c0fd5eSVlastimil Babka unsigned long freepage_order = page_order_unsafe(page); 73799c0fd5eSVlastimil Babka 73899c0fd5eSVlastimil Babka /* 73999c0fd5eSVlastimil Babka * Without lock, we cannot be sure that what we got is 74099c0fd5eSVlastimil Babka * a valid page order. Consider only values in the 74199c0fd5eSVlastimil Babka * valid order range to prevent low_pfn overflow. 74299c0fd5eSVlastimil Babka */ 74399c0fd5eSVlastimil Babka if (freepage_order > 0 && freepage_order < MAX_ORDER) 74499c0fd5eSVlastimil Babka low_pfn += (1UL << freepage_order) - 1; 745748446bbSMel Gorman continue; 74699c0fd5eSVlastimil Babka } 747748446bbSMel Gorman 7489927af74SMel Gorman /* 749bf6bddf1SRafael Aquini * Check may be lockless but that's ok as we recheck later. 750bf6bddf1SRafael Aquini * It's possible to migrate LRU pages and balloon pages 751bf6bddf1SRafael Aquini * Skip any other type of page 752bf6bddf1SRafael Aquini */ 75329c0dde8SVlastimil Babka is_lru = PageLRU(page); 75429c0dde8SVlastimil Babka if (!is_lru) { 755bf6bddf1SRafael Aquini if (unlikely(balloon_page_movable(page))) { 756d6d86c0aSKonstantin Khlebnikov if (balloon_page_isolate(page)) { 757bf6bddf1SRafael Aquini /* Successfully isolated */ 758b6c75016SJoonsoo Kim goto isolate_success; 759bf6bddf1SRafael Aquini } 760bf6bddf1SRafael Aquini } 761bf6bddf1SRafael Aquini } 762bc835011SAndrea Arcangeli 763bc835011SAndrea Arcangeli /* 76429c0dde8SVlastimil Babka * Regardless of being on LRU, compound pages such as THP and 76529c0dde8SVlastimil Babka * hugetlbfs are not to be compacted. We can potentially save 76629c0dde8SVlastimil Babka * a lot of iterations if we skip them at once. The check is 76729c0dde8SVlastimil Babka * racy, but we can consider only valid values and the only 76829c0dde8SVlastimil Babka * danger is skipping too much. 769bc835011SAndrea Arcangeli */ 77029c0dde8SVlastimil Babka if (PageCompound(page)) { 77129c0dde8SVlastimil Babka unsigned int comp_order = compound_order(page); 77229c0dde8SVlastimil Babka 77329c0dde8SVlastimil Babka if (likely(comp_order < MAX_ORDER)) 77429c0dde8SVlastimil Babka low_pfn += (1UL << comp_order) - 1; 775edc2ca61SVlastimil Babka 7762a1402aaSMel Gorman continue; 7772a1402aaSMel Gorman } 7782a1402aaSMel Gorman 77929c0dde8SVlastimil Babka if (!is_lru) 78029c0dde8SVlastimil Babka continue; 78129c0dde8SVlastimil Babka 782119d6d59SDavid Rientjes /* 783119d6d59SDavid Rientjes * Migration will fail if an anonymous page is pinned in memory, 784119d6d59SDavid Rientjes * so avoid taking lru_lock and isolating it unnecessarily in an 785119d6d59SDavid Rientjes * admittedly racy check. 786119d6d59SDavid Rientjes */ 787119d6d59SDavid Rientjes if (!page_mapping(page) && 788119d6d59SDavid Rientjes page_count(page) > page_mapcount(page)) 789119d6d59SDavid Rientjes continue; 790119d6d59SDavid Rientjes 79169b7189fSVlastimil Babka /* If we already hold the lock, we can skip some rechecking */ 79269b7189fSVlastimil Babka if (!locked) { 7938b44d279SVlastimil Babka locked = compact_trylock_irqsave(&zone->lru_lock, 7948b44d279SVlastimil Babka &flags, cc); 7958b44d279SVlastimil Babka if (!locked) 7962a1402aaSMel Gorman break; 7972a1402aaSMel Gorman 79829c0dde8SVlastimil Babka /* Recheck PageLRU and PageCompound under lock */ 7992a1402aaSMel Gorman if (!PageLRU(page)) 8002a1402aaSMel Gorman continue; 80129c0dde8SVlastimil Babka 80229c0dde8SVlastimil Babka /* 80329c0dde8SVlastimil Babka * Page become compound since the non-locked check, 80429c0dde8SVlastimil Babka * and it's on LRU. It can only be a THP so the order 80529c0dde8SVlastimil Babka * is safe to read and it's 0 for tail pages. 80629c0dde8SVlastimil Babka */ 80729c0dde8SVlastimil Babka if (unlikely(PageCompound(page))) { 80829c0dde8SVlastimil Babka low_pfn += (1UL << compound_order(page)) - 1; 809bc835011SAndrea Arcangeli continue; 810bc835011SAndrea Arcangeli } 81169b7189fSVlastimil Babka } 812bc835011SAndrea Arcangeli 813fa9add64SHugh Dickins lruvec = mem_cgroup_page_lruvec(page, zone); 814fa9add64SHugh Dickins 815748446bbSMel Gorman /* Try isolate the page */ 816edc2ca61SVlastimil Babka if (__isolate_lru_page(page, isolate_mode) != 0) 817748446bbSMel Gorman continue; 818748446bbSMel Gorman 81929c0dde8SVlastimil Babka VM_BUG_ON_PAGE(PageCompound(page), page); 820bc835011SAndrea Arcangeli 821748446bbSMel Gorman /* Successfully isolated */ 822fa9add64SHugh Dickins del_page_from_lru_list(page, lruvec, page_lru(page)); 823b6c75016SJoonsoo Kim 824b6c75016SJoonsoo Kim isolate_success: 825748446bbSMel Gorman list_add(&page->lru, migratelist); 826748446bbSMel Gorman cc->nr_migratepages++; 827b7aba698SMel Gorman nr_isolated++; 828748446bbSMel Gorman 829748446bbSMel Gorman /* Avoid isolating too much */ 83031b8384aSHillf Danton if (cc->nr_migratepages == COMPACT_CLUSTER_MAX) { 83131b8384aSHillf Danton ++low_pfn; 832748446bbSMel Gorman break; 833748446bbSMel Gorman } 83431b8384aSHillf Danton } 835748446bbSMel Gorman 83699c0fd5eSVlastimil Babka /* 83799c0fd5eSVlastimil Babka * The PageBuddy() check could have potentially brought us outside 83899c0fd5eSVlastimil Babka * the range to be scanned. 83999c0fd5eSVlastimil Babka */ 84099c0fd5eSVlastimil Babka if (unlikely(low_pfn > end_pfn)) 84199c0fd5eSVlastimil Babka low_pfn = end_pfn; 84299c0fd5eSVlastimil Babka 843c67fe375SMel Gorman if (locked) 844c67fe375SMel Gorman spin_unlock_irqrestore(&zone->lru_lock, flags); 845748446bbSMel Gorman 84650b5b094SVlastimil Babka /* 84750b5b094SVlastimil Babka * Update the pageblock-skip information and cached scanner pfn, 84850b5b094SVlastimil Babka * if the whole pageblock was scanned without isolating any page. 84950b5b094SVlastimil Babka */ 85035979ef3SDavid Rientjes if (low_pfn == end_pfn) 851edc2ca61SVlastimil Babka update_pageblock_skip(cc, valid_page, nr_isolated, true); 852bb13ffebSMel Gorman 853e34d85f0SJoonsoo Kim trace_mm_compaction_isolate_migratepages(start_pfn, low_pfn, 854e34d85f0SJoonsoo Kim nr_scanned, nr_isolated); 855b7aba698SMel Gorman 856010fc29aSMinchan Kim count_compact_events(COMPACTMIGRATE_SCANNED, nr_scanned); 857397487dbSMel Gorman if (nr_isolated) 858010fc29aSMinchan Kim count_compact_events(COMPACTISOLATED, nr_isolated); 859397487dbSMel Gorman 8602fe86e00SMichal Nazarewicz return low_pfn; 8612fe86e00SMichal Nazarewicz } 8622fe86e00SMichal Nazarewicz 863edc2ca61SVlastimil Babka /** 864edc2ca61SVlastimil Babka * isolate_migratepages_range() - isolate migrate-able pages in a PFN range 865edc2ca61SVlastimil Babka * @cc: Compaction control structure. 866edc2ca61SVlastimil Babka * @start_pfn: The first PFN to start isolating. 867edc2ca61SVlastimil Babka * @end_pfn: The one-past-last PFN. 868edc2ca61SVlastimil Babka * 869edc2ca61SVlastimil Babka * Returns zero if isolation fails fatally due to e.g. pending signal. 870edc2ca61SVlastimil Babka * Otherwise, function returns one-past-the-last PFN of isolated page 871edc2ca61SVlastimil Babka * (which may be greater than end_pfn if end fell in a middle of a THP page). 872edc2ca61SVlastimil Babka */ 873edc2ca61SVlastimil Babka unsigned long 874edc2ca61SVlastimil Babka isolate_migratepages_range(struct compact_control *cc, unsigned long start_pfn, 875edc2ca61SVlastimil Babka unsigned long end_pfn) 876edc2ca61SVlastimil Babka { 877edc2ca61SVlastimil Babka unsigned long pfn, block_end_pfn; 878edc2ca61SVlastimil Babka 879edc2ca61SVlastimil Babka /* Scan block by block. First and last block may be incomplete */ 880edc2ca61SVlastimil Babka pfn = start_pfn; 881edc2ca61SVlastimil Babka block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages); 882edc2ca61SVlastimil Babka 883edc2ca61SVlastimil Babka for (; pfn < end_pfn; pfn = block_end_pfn, 884edc2ca61SVlastimil Babka block_end_pfn += pageblock_nr_pages) { 885edc2ca61SVlastimil Babka 886edc2ca61SVlastimil Babka block_end_pfn = min(block_end_pfn, end_pfn); 887edc2ca61SVlastimil Babka 8887d49d886SVlastimil Babka if (!pageblock_pfn_to_page(pfn, block_end_pfn, cc->zone)) 889edc2ca61SVlastimil Babka continue; 890edc2ca61SVlastimil Babka 891edc2ca61SVlastimil Babka pfn = isolate_migratepages_block(cc, pfn, block_end_pfn, 892edc2ca61SVlastimil Babka ISOLATE_UNEVICTABLE); 893edc2ca61SVlastimil Babka 894edc2ca61SVlastimil Babka /* 895edc2ca61SVlastimil Babka * In case of fatal failure, release everything that might 896edc2ca61SVlastimil Babka * have been isolated in the previous iteration, and signal 897edc2ca61SVlastimil Babka * the failure back to caller. 898edc2ca61SVlastimil Babka */ 899edc2ca61SVlastimil Babka if (!pfn) { 900edc2ca61SVlastimil Babka putback_movable_pages(&cc->migratepages); 901edc2ca61SVlastimil Babka cc->nr_migratepages = 0; 902edc2ca61SVlastimil Babka break; 903edc2ca61SVlastimil Babka } 9046ea41c0cSJoonsoo Kim 9056ea41c0cSJoonsoo Kim if (cc->nr_migratepages == COMPACT_CLUSTER_MAX) 9066ea41c0cSJoonsoo Kim break; 907edc2ca61SVlastimil Babka } 908edc2ca61SVlastimil Babka acct_isolated(cc->zone, cc); 909edc2ca61SVlastimil Babka 910edc2ca61SVlastimil Babka return pfn; 911edc2ca61SVlastimil Babka } 912edc2ca61SVlastimil Babka 913ff9543fdSMichal Nazarewicz #endif /* CONFIG_COMPACTION || CONFIG_CMA */ 914ff9543fdSMichal Nazarewicz #ifdef CONFIG_COMPACTION 915018e9a49SAndrew Morton 916018e9a49SAndrew Morton /* Returns true if the page is within a block suitable for migration to */ 917018e9a49SAndrew Morton static bool suitable_migration_target(struct page *page) 918018e9a49SAndrew Morton { 919018e9a49SAndrew Morton /* If the page is a large free page, then disallow migration */ 920018e9a49SAndrew Morton if (PageBuddy(page)) { 921018e9a49SAndrew Morton /* 922018e9a49SAndrew Morton * We are checking page_order without zone->lock taken. But 923018e9a49SAndrew Morton * the only small danger is that we skip a potentially suitable 924018e9a49SAndrew Morton * pageblock, so it's not worth to check order for valid range. 925018e9a49SAndrew Morton */ 926018e9a49SAndrew Morton if (page_order_unsafe(page) >= pageblock_order) 927018e9a49SAndrew Morton return false; 928018e9a49SAndrew Morton } 929018e9a49SAndrew Morton 930018e9a49SAndrew Morton /* If the block is MIGRATE_MOVABLE or MIGRATE_CMA, allow migration */ 931018e9a49SAndrew Morton if (migrate_async_suitable(get_pageblock_migratetype(page))) 932018e9a49SAndrew Morton return true; 933018e9a49SAndrew Morton 934018e9a49SAndrew Morton /* Otherwise skip the block */ 935018e9a49SAndrew Morton return false; 936018e9a49SAndrew Morton } 937018e9a49SAndrew Morton 938ff9543fdSMichal Nazarewicz /* 939f2849aa0SVlastimil Babka * Test whether the free scanner has reached the same or lower pageblock than 940f2849aa0SVlastimil Babka * the migration scanner, and compaction should thus terminate. 941f2849aa0SVlastimil Babka */ 942f2849aa0SVlastimil Babka static inline bool compact_scanners_met(struct compact_control *cc) 943f2849aa0SVlastimil Babka { 944f2849aa0SVlastimil Babka return (cc->free_pfn >> pageblock_order) 945f2849aa0SVlastimil Babka <= (cc->migrate_pfn >> pageblock_order); 946f2849aa0SVlastimil Babka } 947f2849aa0SVlastimil Babka 948f2849aa0SVlastimil Babka /* 949ff9543fdSMichal Nazarewicz * Based on information in the current compact_control, find blocks 950ff9543fdSMichal Nazarewicz * suitable for isolating free pages from and then isolate them. 951ff9543fdSMichal Nazarewicz */ 952edc2ca61SVlastimil Babka static void isolate_freepages(struct compact_control *cc) 953ff9543fdSMichal Nazarewicz { 954edc2ca61SVlastimil Babka struct zone *zone = cc->zone; 955ff9543fdSMichal Nazarewicz struct page *page; 956c96b9e50SVlastimil Babka unsigned long block_start_pfn; /* start of current pageblock */ 957e14c720eSVlastimil Babka unsigned long isolate_start_pfn; /* exact pfn we start at */ 958c96b9e50SVlastimil Babka unsigned long block_end_pfn; /* end of current pageblock */ 959c96b9e50SVlastimil Babka unsigned long low_pfn; /* lowest pfn scanner is able to scan */ 960ff9543fdSMichal Nazarewicz struct list_head *freelist = &cc->freepages; 9612fe86e00SMichal Nazarewicz 962ff9543fdSMichal Nazarewicz /* 963ff9543fdSMichal Nazarewicz * Initialise the free scanner. The starting point is where we last 96449e068f0SVlastimil Babka * successfully isolated from, zone-cached value, or the end of the 965e14c720eSVlastimil Babka * zone when isolating for the first time. For looping we also need 966e14c720eSVlastimil Babka * this pfn aligned down to the pageblock boundary, because we do 967c96b9e50SVlastimil Babka * block_start_pfn -= pageblock_nr_pages in the for loop. 968c96b9e50SVlastimil Babka * For ending point, take care when isolating in last pageblock of a 969c96b9e50SVlastimil Babka * a zone which ends in the middle of a pageblock. 97049e068f0SVlastimil Babka * The low boundary is the end of the pageblock the migration scanner 97149e068f0SVlastimil Babka * is using. 972ff9543fdSMichal Nazarewicz */ 973e14c720eSVlastimil Babka isolate_start_pfn = cc->free_pfn; 974c96b9e50SVlastimil Babka block_start_pfn = cc->free_pfn & ~(pageblock_nr_pages-1); 975c96b9e50SVlastimil Babka block_end_pfn = min(block_start_pfn + pageblock_nr_pages, 976c96b9e50SVlastimil Babka zone_end_pfn(zone)); 9777ed695e0SVlastimil Babka low_pfn = ALIGN(cc->migrate_pfn + 1, pageblock_nr_pages); 9782fe86e00SMichal Nazarewicz 979ff9543fdSMichal Nazarewicz /* 980ff9543fdSMichal Nazarewicz * Isolate free pages until enough are available to migrate the 981ff9543fdSMichal Nazarewicz * pages on cc->migratepages. We stop searching if the migrate 982ff9543fdSMichal Nazarewicz * and free page scanners meet or enough free pages are isolated. 983ff9543fdSMichal Nazarewicz */ 984f5f61a32SVlastimil Babka for (; block_start_pfn >= low_pfn; 985c96b9e50SVlastimil Babka block_end_pfn = block_start_pfn, 986e14c720eSVlastimil Babka block_start_pfn -= pageblock_nr_pages, 987e14c720eSVlastimil Babka isolate_start_pfn = block_start_pfn) { 988ff9543fdSMichal Nazarewicz 989f6ea3adbSDavid Rientjes /* 990f6ea3adbSDavid Rientjes * This can iterate a massively long zone without finding any 991f6ea3adbSDavid Rientjes * suitable migration targets, so periodically check if we need 992be976572SVlastimil Babka * to schedule, or even abort async compaction. 993f6ea3adbSDavid Rientjes */ 994be976572SVlastimil Babka if (!(block_start_pfn % (SWAP_CLUSTER_MAX * pageblock_nr_pages)) 995be976572SVlastimil Babka && compact_should_abort(cc)) 996be976572SVlastimil Babka break; 997f6ea3adbSDavid Rientjes 9987d49d886SVlastimil Babka page = pageblock_pfn_to_page(block_start_pfn, block_end_pfn, 9997d49d886SVlastimil Babka zone); 10007d49d886SVlastimil Babka if (!page) 1001ff9543fdSMichal Nazarewicz continue; 1002ff9543fdSMichal Nazarewicz 1003ff9543fdSMichal Nazarewicz /* Check the block is suitable for migration */ 100468e3e926SLinus Torvalds if (!suitable_migration_target(page)) 1005ff9543fdSMichal Nazarewicz continue; 100668e3e926SLinus Torvalds 1007bb13ffebSMel Gorman /* If isolation recently failed, do not retry */ 1008bb13ffebSMel Gorman if (!isolation_suitable(cc, page)) 1009bb13ffebSMel Gorman continue; 1010bb13ffebSMel Gorman 1011e14c720eSVlastimil Babka /* Found a block suitable for isolating free pages from. */ 1012932ff6bbSJoonsoo Kim isolate_freepages_block(cc, &isolate_start_pfn, 1013c96b9e50SVlastimil Babka block_end_pfn, freelist, false); 1014ff9543fdSMichal Nazarewicz 1015ff9543fdSMichal Nazarewicz /* 1016f5f61a32SVlastimil Babka * If we isolated enough freepages, or aborted due to async 1017f5f61a32SVlastimil Babka * compaction being contended, terminate the loop. 1018e14c720eSVlastimil Babka * Remember where the free scanner should restart next time, 1019e14c720eSVlastimil Babka * which is where isolate_freepages_block() left off. 1020e14c720eSVlastimil Babka * But if it scanned the whole pageblock, isolate_start_pfn 1021e14c720eSVlastimil Babka * now points at block_end_pfn, which is the start of the next 1022e14c720eSVlastimil Babka * pageblock. 1023e14c720eSVlastimil Babka * In that case we will however want to restart at the start 1024e14c720eSVlastimil Babka * of the previous pageblock. 1025e14c720eSVlastimil Babka */ 1026f5f61a32SVlastimil Babka if ((cc->nr_freepages >= cc->nr_migratepages) 1027f5f61a32SVlastimil Babka || cc->contended) { 1028f5f61a32SVlastimil Babka if (isolate_start_pfn >= block_end_pfn) 1029f5f61a32SVlastimil Babka isolate_start_pfn = 1030e14c720eSVlastimil Babka block_start_pfn - pageblock_nr_pages; 1031be976572SVlastimil Babka break; 1032f5f61a32SVlastimil Babka } else { 1033f5f61a32SVlastimil Babka /* 1034f5f61a32SVlastimil Babka * isolate_freepages_block() should not terminate 1035f5f61a32SVlastimil Babka * prematurely unless contended, or isolated enough 1036f5f61a32SVlastimil Babka */ 1037f5f61a32SVlastimil Babka VM_BUG_ON(isolate_start_pfn < block_end_pfn); 1038f5f61a32SVlastimil Babka } 1039c89511abSMel Gorman } 1040ff9543fdSMichal Nazarewicz 1041ff9543fdSMichal Nazarewicz /* split_free_page does not map the pages */ 1042ff9543fdSMichal Nazarewicz map_pages(freelist); 1043ff9543fdSMichal Nazarewicz 10447ed695e0SVlastimil Babka /* 1045f5f61a32SVlastimil Babka * Record where the free scanner will restart next time. Either we 1046f5f61a32SVlastimil Babka * broke from the loop and set isolate_start_pfn based on the last 1047f5f61a32SVlastimil Babka * call to isolate_freepages_block(), or we met the migration scanner 1048f5f61a32SVlastimil Babka * and the loop terminated due to isolate_start_pfn < low_pfn 10497ed695e0SVlastimil Babka */ 1050f5f61a32SVlastimil Babka cc->free_pfn = isolate_start_pfn; 1051748446bbSMel Gorman } 1052748446bbSMel Gorman 1053748446bbSMel Gorman /* 1054748446bbSMel Gorman * This is a migrate-callback that "allocates" freepages by taking pages 1055748446bbSMel Gorman * from the isolated freelists in the block we are migrating to. 1056748446bbSMel Gorman */ 1057748446bbSMel Gorman static struct page *compaction_alloc(struct page *migratepage, 1058748446bbSMel Gorman unsigned long data, 1059748446bbSMel Gorman int **result) 1060748446bbSMel Gorman { 1061748446bbSMel Gorman struct compact_control *cc = (struct compact_control *)data; 1062748446bbSMel Gorman struct page *freepage; 1063748446bbSMel Gorman 1064be976572SVlastimil Babka /* 1065be976572SVlastimil Babka * Isolate free pages if necessary, and if we are not aborting due to 1066be976572SVlastimil Babka * contention. 1067be976572SVlastimil Babka */ 1068748446bbSMel Gorman if (list_empty(&cc->freepages)) { 1069be976572SVlastimil Babka if (!cc->contended) 1070edc2ca61SVlastimil Babka isolate_freepages(cc); 1071748446bbSMel Gorman 1072748446bbSMel Gorman if (list_empty(&cc->freepages)) 1073748446bbSMel Gorman return NULL; 1074748446bbSMel Gorman } 1075748446bbSMel Gorman 1076748446bbSMel Gorman freepage = list_entry(cc->freepages.next, struct page, lru); 1077748446bbSMel Gorman list_del(&freepage->lru); 1078748446bbSMel Gorman cc->nr_freepages--; 1079748446bbSMel Gorman 1080748446bbSMel Gorman return freepage; 1081748446bbSMel Gorman } 1082748446bbSMel Gorman 1083748446bbSMel Gorman /* 1084d53aea3dSDavid Rientjes * This is a migrate-callback that "frees" freepages back to the isolated 1085d53aea3dSDavid Rientjes * freelist. All pages on the freelist are from the same zone, so there is no 1086d53aea3dSDavid Rientjes * special handling needed for NUMA. 1087d53aea3dSDavid Rientjes */ 1088d53aea3dSDavid Rientjes static void compaction_free(struct page *page, unsigned long data) 1089d53aea3dSDavid Rientjes { 1090d53aea3dSDavid Rientjes struct compact_control *cc = (struct compact_control *)data; 1091d53aea3dSDavid Rientjes 1092d53aea3dSDavid Rientjes list_add(&page->lru, &cc->freepages); 1093d53aea3dSDavid Rientjes cc->nr_freepages++; 1094d53aea3dSDavid Rientjes } 1095d53aea3dSDavid Rientjes 1096ff9543fdSMichal Nazarewicz /* possible outcome of isolate_migratepages */ 1097ff9543fdSMichal Nazarewicz typedef enum { 1098ff9543fdSMichal Nazarewicz ISOLATE_ABORT, /* Abort compaction now */ 1099ff9543fdSMichal Nazarewicz ISOLATE_NONE, /* No pages isolated, continue scanning */ 1100ff9543fdSMichal Nazarewicz ISOLATE_SUCCESS, /* Pages isolated, migrate */ 1101ff9543fdSMichal Nazarewicz } isolate_migrate_t; 1102ff9543fdSMichal Nazarewicz 1103ff9543fdSMichal Nazarewicz /* 11045bbe3547SEric B Munson * Allow userspace to control policy on scanning the unevictable LRU for 11055bbe3547SEric B Munson * compactable pages. 11065bbe3547SEric B Munson */ 11075bbe3547SEric B Munson int sysctl_compact_unevictable_allowed __read_mostly = 1; 11085bbe3547SEric B Munson 11095bbe3547SEric B Munson /* 1110edc2ca61SVlastimil Babka * Isolate all pages that can be migrated from the first suitable block, 1111edc2ca61SVlastimil Babka * starting at the block pointed to by the migrate scanner pfn within 1112edc2ca61SVlastimil Babka * compact_control. 1113ff9543fdSMichal Nazarewicz */ 1114ff9543fdSMichal Nazarewicz static isolate_migrate_t isolate_migratepages(struct zone *zone, 1115ff9543fdSMichal Nazarewicz struct compact_control *cc) 1116ff9543fdSMichal Nazarewicz { 1117ff9543fdSMichal Nazarewicz unsigned long low_pfn, end_pfn; 1118edc2ca61SVlastimil Babka struct page *page; 1119edc2ca61SVlastimil Babka const isolate_mode_t isolate_mode = 11205bbe3547SEric B Munson (sysctl_compact_unevictable_allowed ? ISOLATE_UNEVICTABLE : 0) | 1121edc2ca61SVlastimil Babka (cc->mode == MIGRATE_ASYNC ? ISOLATE_ASYNC_MIGRATE : 0); 1122ff9543fdSMichal Nazarewicz 1123edc2ca61SVlastimil Babka /* 1124edc2ca61SVlastimil Babka * Start at where we last stopped, or beginning of the zone as 1125edc2ca61SVlastimil Babka * initialized by compact_zone() 1126edc2ca61SVlastimil Babka */ 1127edc2ca61SVlastimil Babka low_pfn = cc->migrate_pfn; 1128ff9543fdSMichal Nazarewicz 1129ff9543fdSMichal Nazarewicz /* Only scan within a pageblock boundary */ 1130a9aacbccSMel Gorman end_pfn = ALIGN(low_pfn + 1, pageblock_nr_pages); 1131ff9543fdSMichal Nazarewicz 1132edc2ca61SVlastimil Babka /* 1133edc2ca61SVlastimil Babka * Iterate over whole pageblocks until we find the first suitable. 1134edc2ca61SVlastimil Babka * Do not cross the free scanner. 1135edc2ca61SVlastimil Babka */ 1136edc2ca61SVlastimil Babka for (; end_pfn <= cc->free_pfn; 1137edc2ca61SVlastimil Babka low_pfn = end_pfn, end_pfn += pageblock_nr_pages) { 1138edc2ca61SVlastimil Babka 1139edc2ca61SVlastimil Babka /* 1140edc2ca61SVlastimil Babka * This can potentially iterate a massively long zone with 1141edc2ca61SVlastimil Babka * many pageblocks unsuitable, so periodically check if we 1142edc2ca61SVlastimil Babka * need to schedule, or even abort async compaction. 1143edc2ca61SVlastimil Babka */ 1144edc2ca61SVlastimil Babka if (!(low_pfn % (SWAP_CLUSTER_MAX * pageblock_nr_pages)) 1145edc2ca61SVlastimil Babka && compact_should_abort(cc)) 1146edc2ca61SVlastimil Babka break; 1147edc2ca61SVlastimil Babka 11487d49d886SVlastimil Babka page = pageblock_pfn_to_page(low_pfn, end_pfn, zone); 11497d49d886SVlastimil Babka if (!page) 1150edc2ca61SVlastimil Babka continue; 1151edc2ca61SVlastimil Babka 1152edc2ca61SVlastimil Babka /* If isolation recently failed, do not retry */ 1153edc2ca61SVlastimil Babka if (!isolation_suitable(cc, page)) 1154edc2ca61SVlastimil Babka continue; 1155edc2ca61SVlastimil Babka 1156edc2ca61SVlastimil Babka /* 1157edc2ca61SVlastimil Babka * For async compaction, also only scan in MOVABLE blocks. 1158edc2ca61SVlastimil Babka * Async compaction is optimistic to see if the minimum amount 1159edc2ca61SVlastimil Babka * of work satisfies the allocation. 1160edc2ca61SVlastimil Babka */ 1161edc2ca61SVlastimil Babka if (cc->mode == MIGRATE_ASYNC && 1162edc2ca61SVlastimil Babka !migrate_async_suitable(get_pageblock_migratetype(page))) 1163edc2ca61SVlastimil Babka continue; 1164ff9543fdSMichal Nazarewicz 1165ff9543fdSMichal Nazarewicz /* Perform the isolation */ 1166edc2ca61SVlastimil Babka low_pfn = isolate_migratepages_block(cc, low_pfn, end_pfn, 1167edc2ca61SVlastimil Babka isolate_mode); 1168edc2ca61SVlastimil Babka 1169ff59909aSHugh Dickins if (!low_pfn || cc->contended) { 1170ff59909aSHugh Dickins acct_isolated(zone, cc); 1171ff9543fdSMichal Nazarewicz return ISOLATE_ABORT; 1172ff59909aSHugh Dickins } 1173ff9543fdSMichal Nazarewicz 1174edc2ca61SVlastimil Babka /* 1175edc2ca61SVlastimil Babka * Either we isolated something and proceed with migration. Or 1176edc2ca61SVlastimil Babka * we failed and compact_zone should decide if we should 1177edc2ca61SVlastimil Babka * continue or not. 1178edc2ca61SVlastimil Babka */ 1179edc2ca61SVlastimil Babka break; 1180edc2ca61SVlastimil Babka } 1181edc2ca61SVlastimil Babka 1182edc2ca61SVlastimil Babka acct_isolated(zone, cc); 1183f2849aa0SVlastimil Babka /* Record where migration scanner will be restarted. */ 1184f2849aa0SVlastimil Babka cc->migrate_pfn = low_pfn; 1185ff9543fdSMichal Nazarewicz 1186edc2ca61SVlastimil Babka return cc->nr_migratepages ? ISOLATE_SUCCESS : ISOLATE_NONE; 1187ff9543fdSMichal Nazarewicz } 1188ff9543fdSMichal Nazarewicz 1189837d026dSJoonsoo Kim static int __compact_finished(struct zone *zone, struct compact_control *cc, 11906d7ce559SDavid Rientjes const int migratetype) 1191748446bbSMel Gorman { 11928fb74b9fSMel Gorman unsigned int order; 11935a03b051SAndrea Arcangeli unsigned long watermark; 119456de7263SMel Gorman 1195be976572SVlastimil Babka if (cc->contended || fatal_signal_pending(current)) 1196748446bbSMel Gorman return COMPACT_PARTIAL; 1197748446bbSMel Gorman 1198753341a4SMel Gorman /* Compaction run completes if the migrate and free scanner meet */ 1199f2849aa0SVlastimil Babka if (compact_scanners_met(cc)) { 120055b7c4c9SVlastimil Babka /* Let the next compaction start anew. */ 120102333641SVlastimil Babka reset_cached_positions(zone); 120255b7c4c9SVlastimil Babka 120362997027SMel Gorman /* 120462997027SMel Gorman * Mark that the PG_migrate_skip information should be cleared 120562997027SMel Gorman * by kswapd when it goes to sleep. kswapd does not set the 120662997027SMel Gorman * flag itself as the decision to be clear should be directly 120762997027SMel Gorman * based on an allocation request. 120862997027SMel Gorman */ 120962997027SMel Gorman if (!current_is_kswapd()) 121062997027SMel Gorman zone->compact_blockskip_flush = true; 121162997027SMel Gorman 1212748446bbSMel Gorman return COMPACT_COMPLETE; 1213bb13ffebSMel Gorman } 1214748446bbSMel Gorman 121582478fb7SJohannes Weiner /* 121682478fb7SJohannes Weiner * order == -1 is expected when compacting via 121782478fb7SJohannes Weiner * /proc/sys/vm/compact_memory 121882478fb7SJohannes Weiner */ 121956de7263SMel Gorman if (cc->order == -1) 122056de7263SMel Gorman return COMPACT_CONTINUE; 122156de7263SMel Gorman 12223957c776SMichal Hocko /* Compaction run is not finished if the watermark is not met */ 12233957c776SMichal Hocko watermark = low_wmark_pages(zone); 12243957c776SMichal Hocko 1225ebff3980SVlastimil Babka if (!zone_watermark_ok(zone, cc->order, watermark, cc->classzone_idx, 1226ebff3980SVlastimil Babka cc->alloc_flags)) 12273957c776SMichal Hocko return COMPACT_CONTINUE; 12283957c776SMichal Hocko 122956de7263SMel Gorman /* Direct compactor: Is a suitable page free? */ 123056de7263SMel Gorman for (order = cc->order; order < MAX_ORDER; order++) { 12318fb74b9fSMel Gorman struct free_area *area = &zone->free_area[order]; 12322149cdaeSJoonsoo Kim bool can_steal; 12338fb74b9fSMel Gorman 123456de7263SMel Gorman /* Job done if page is free of the right migratetype */ 12356d7ce559SDavid Rientjes if (!list_empty(&area->free_list[migratetype])) 123656de7263SMel Gorman return COMPACT_PARTIAL; 123756de7263SMel Gorman 12382149cdaeSJoonsoo Kim #ifdef CONFIG_CMA 12392149cdaeSJoonsoo Kim /* MIGRATE_MOVABLE can fallback on MIGRATE_CMA */ 12402149cdaeSJoonsoo Kim if (migratetype == MIGRATE_MOVABLE && 12412149cdaeSJoonsoo Kim !list_empty(&area->free_list[MIGRATE_CMA])) 12422149cdaeSJoonsoo Kim return COMPACT_PARTIAL; 12432149cdaeSJoonsoo Kim #endif 12442149cdaeSJoonsoo Kim /* 12452149cdaeSJoonsoo Kim * Job done if allocation would steal freepages from 12462149cdaeSJoonsoo Kim * other migratetype buddy lists. 12472149cdaeSJoonsoo Kim */ 12482149cdaeSJoonsoo Kim if (find_suitable_fallback(area, order, migratetype, 12492149cdaeSJoonsoo Kim true, &can_steal) != -1) 125056de7263SMel Gorman return COMPACT_PARTIAL; 125156de7263SMel Gorman } 125256de7263SMel Gorman 1253837d026dSJoonsoo Kim return COMPACT_NO_SUITABLE_PAGE; 1254837d026dSJoonsoo Kim } 1255837d026dSJoonsoo Kim 1256837d026dSJoonsoo Kim static int compact_finished(struct zone *zone, struct compact_control *cc, 1257837d026dSJoonsoo Kim const int migratetype) 1258837d026dSJoonsoo Kim { 1259837d026dSJoonsoo Kim int ret; 1260837d026dSJoonsoo Kim 1261837d026dSJoonsoo Kim ret = __compact_finished(zone, cc, migratetype); 1262837d026dSJoonsoo Kim trace_mm_compaction_finished(zone, cc->order, ret); 1263837d026dSJoonsoo Kim if (ret == COMPACT_NO_SUITABLE_PAGE) 1264837d026dSJoonsoo Kim ret = COMPACT_CONTINUE; 1265837d026dSJoonsoo Kim 1266837d026dSJoonsoo Kim return ret; 1267748446bbSMel Gorman } 1268748446bbSMel Gorman 12693e7d3449SMel Gorman /* 12703e7d3449SMel Gorman * compaction_suitable: Is this suitable to run compaction on this zone now? 12713e7d3449SMel Gorman * Returns 12723e7d3449SMel Gorman * COMPACT_SKIPPED - If there are too few free pages for compaction 12733e7d3449SMel Gorman * COMPACT_PARTIAL - If the allocation would succeed without compaction 12743e7d3449SMel Gorman * COMPACT_CONTINUE - If compaction should run now 12753e7d3449SMel Gorman */ 1276837d026dSJoonsoo Kim static unsigned long __compaction_suitable(struct zone *zone, int order, 1277ebff3980SVlastimil Babka int alloc_flags, int classzone_idx) 12783e7d3449SMel Gorman { 12793e7d3449SMel Gorman int fragindex; 12803e7d3449SMel Gorman unsigned long watermark; 12813e7d3449SMel Gorman 12823e7d3449SMel Gorman /* 12833957c776SMichal Hocko * order == -1 is expected when compacting via 12843957c776SMichal Hocko * /proc/sys/vm/compact_memory 12853957c776SMichal Hocko */ 12863957c776SMichal Hocko if (order == -1) 12873957c776SMichal Hocko return COMPACT_CONTINUE; 12883957c776SMichal Hocko 1289ebff3980SVlastimil Babka watermark = low_wmark_pages(zone); 1290ebff3980SVlastimil Babka /* 1291ebff3980SVlastimil Babka * If watermarks for high-order allocation are already met, there 1292ebff3980SVlastimil Babka * should be no need for compaction at all. 1293ebff3980SVlastimil Babka */ 1294ebff3980SVlastimil Babka if (zone_watermark_ok(zone, order, watermark, classzone_idx, 1295ebff3980SVlastimil Babka alloc_flags)) 1296ebff3980SVlastimil Babka return COMPACT_PARTIAL; 1297ebff3980SVlastimil Babka 12983957c776SMichal Hocko /* 12993e7d3449SMel Gorman * Watermarks for order-0 must be met for compaction. Note the 2UL. 13003e7d3449SMel Gorman * This is because during migration, copies of pages need to be 13013e7d3449SMel Gorman * allocated and for a short time, the footprint is higher 13023e7d3449SMel Gorman */ 1303ebff3980SVlastimil Babka watermark += (2UL << order); 1304ebff3980SVlastimil Babka if (!zone_watermark_ok(zone, 0, watermark, classzone_idx, alloc_flags)) 13053e7d3449SMel Gorman return COMPACT_SKIPPED; 13063e7d3449SMel Gorman 13073e7d3449SMel Gorman /* 13083e7d3449SMel Gorman * fragmentation index determines if allocation failures are due to 13093e7d3449SMel Gorman * low memory or external fragmentation 13103e7d3449SMel Gorman * 1311ebff3980SVlastimil Babka * index of -1000 would imply allocations might succeed depending on 1312ebff3980SVlastimil Babka * watermarks, but we already failed the high-order watermark check 13133e7d3449SMel Gorman * index towards 0 implies failure is due to lack of memory 13143e7d3449SMel Gorman * index towards 1000 implies failure is due to fragmentation 13153e7d3449SMel Gorman * 13163e7d3449SMel Gorman * Only compact if a failure would be due to fragmentation. 13173e7d3449SMel Gorman */ 13183e7d3449SMel Gorman fragindex = fragmentation_index(zone, order); 13193e7d3449SMel Gorman if (fragindex >= 0 && fragindex <= sysctl_extfrag_threshold) 1320837d026dSJoonsoo Kim return COMPACT_NOT_SUITABLE_ZONE; 13213e7d3449SMel Gorman 13223e7d3449SMel Gorman return COMPACT_CONTINUE; 13233e7d3449SMel Gorman } 13243e7d3449SMel Gorman 1325837d026dSJoonsoo Kim unsigned long compaction_suitable(struct zone *zone, int order, 1326837d026dSJoonsoo Kim int alloc_flags, int classzone_idx) 1327837d026dSJoonsoo Kim { 1328837d026dSJoonsoo Kim unsigned long ret; 1329837d026dSJoonsoo Kim 1330837d026dSJoonsoo Kim ret = __compaction_suitable(zone, order, alloc_flags, classzone_idx); 1331837d026dSJoonsoo Kim trace_mm_compaction_suitable(zone, order, ret); 1332837d026dSJoonsoo Kim if (ret == COMPACT_NOT_SUITABLE_ZONE) 1333837d026dSJoonsoo Kim ret = COMPACT_SKIPPED; 1334837d026dSJoonsoo Kim 1335837d026dSJoonsoo Kim return ret; 1336837d026dSJoonsoo Kim } 1337837d026dSJoonsoo Kim 1338748446bbSMel Gorman static int compact_zone(struct zone *zone, struct compact_control *cc) 1339748446bbSMel Gorman { 1340748446bbSMel Gorman int ret; 1341c89511abSMel Gorman unsigned long start_pfn = zone->zone_start_pfn; 1342108bcc96SCody P Schafer unsigned long end_pfn = zone_end_pfn(zone); 13436d7ce559SDavid Rientjes const int migratetype = gfpflags_to_migratetype(cc->gfp_mask); 1344e0b9daebSDavid Rientjes const bool sync = cc->mode != MIGRATE_ASYNC; 1345fdaf7f5cSVlastimil Babka unsigned long last_migrated_pfn = 0; 1346748446bbSMel Gorman 1347ebff3980SVlastimil Babka ret = compaction_suitable(zone, cc->order, cc->alloc_flags, 1348ebff3980SVlastimil Babka cc->classzone_idx); 13493e7d3449SMel Gorman switch (ret) { 13503e7d3449SMel Gorman case COMPACT_PARTIAL: 13513e7d3449SMel Gorman case COMPACT_SKIPPED: 13523e7d3449SMel Gorman /* Compaction is likely to fail */ 13533e7d3449SMel Gorman return ret; 13543e7d3449SMel Gorman case COMPACT_CONTINUE: 13553e7d3449SMel Gorman /* Fall through to compaction */ 13563e7d3449SMel Gorman ; 13573e7d3449SMel Gorman } 13583e7d3449SMel Gorman 1359c89511abSMel Gorman /* 1360d3132e4bSVlastimil Babka * Clear pageblock skip if there were failures recently and compaction 1361d3132e4bSVlastimil Babka * is about to be retried after being deferred. kswapd does not do 1362d3132e4bSVlastimil Babka * this reset as it'll reset the cached information when going to sleep. 1363d3132e4bSVlastimil Babka */ 1364d3132e4bSVlastimil Babka if (compaction_restarting(zone, cc->order) && !current_is_kswapd()) 1365d3132e4bSVlastimil Babka __reset_isolation_suitable(zone); 1366d3132e4bSVlastimil Babka 1367d3132e4bSVlastimil Babka /* 1368c89511abSMel Gorman * Setup to move all movable pages to the end of the zone. Used cached 1369c89511abSMel Gorman * information on where the scanners should start but check that it 1370c89511abSMel Gorman * is initialised by ensuring the values are within zone boundaries. 1371c89511abSMel Gorman */ 1372e0b9daebSDavid Rientjes cc->migrate_pfn = zone->compact_cached_migrate_pfn[sync]; 1373c89511abSMel Gorman cc->free_pfn = zone->compact_cached_free_pfn; 1374c89511abSMel Gorman if (cc->free_pfn < start_pfn || cc->free_pfn > end_pfn) { 1375c89511abSMel Gorman cc->free_pfn = end_pfn & ~(pageblock_nr_pages-1); 1376c89511abSMel Gorman zone->compact_cached_free_pfn = cc->free_pfn; 1377c89511abSMel Gorman } 1378c89511abSMel Gorman if (cc->migrate_pfn < start_pfn || cc->migrate_pfn > end_pfn) { 1379c89511abSMel Gorman cc->migrate_pfn = start_pfn; 138035979ef3SDavid Rientjes zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn; 138135979ef3SDavid Rientjes zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn; 1382c89511abSMel Gorman } 1383748446bbSMel Gorman 138416c4a097SJoonsoo Kim trace_mm_compaction_begin(start_pfn, cc->migrate_pfn, 138516c4a097SJoonsoo Kim cc->free_pfn, end_pfn, sync); 13860eb927c0SMel Gorman 1387748446bbSMel Gorman migrate_prep_local(); 1388748446bbSMel Gorman 13896d7ce559SDavid Rientjes while ((ret = compact_finished(zone, cc, migratetype)) == 13906d7ce559SDavid Rientjes COMPACT_CONTINUE) { 13919d502c1cSMinchan Kim int err; 1392fdaf7f5cSVlastimil Babka unsigned long isolate_start_pfn = cc->migrate_pfn; 1393748446bbSMel Gorman 1394f9e35b3bSMel Gorman switch (isolate_migratepages(zone, cc)) { 1395f9e35b3bSMel Gorman case ISOLATE_ABORT: 1396f9e35b3bSMel Gorman ret = COMPACT_PARTIAL; 13975733c7d1SRafael Aquini putback_movable_pages(&cc->migratepages); 1398e64c5237SShaohua Li cc->nr_migratepages = 0; 1399f9e35b3bSMel Gorman goto out; 1400f9e35b3bSMel Gorman case ISOLATE_NONE: 1401fdaf7f5cSVlastimil Babka /* 1402fdaf7f5cSVlastimil Babka * We haven't isolated and migrated anything, but 1403fdaf7f5cSVlastimil Babka * there might still be unflushed migrations from 1404fdaf7f5cSVlastimil Babka * previous cc->order aligned block. 1405fdaf7f5cSVlastimil Babka */ 1406fdaf7f5cSVlastimil Babka goto check_drain; 1407f9e35b3bSMel Gorman case ISOLATE_SUCCESS: 1408f9e35b3bSMel Gorman ; 1409f9e35b3bSMel Gorman } 1410748446bbSMel Gorman 1411d53aea3dSDavid Rientjes err = migrate_pages(&cc->migratepages, compaction_alloc, 1412e0b9daebSDavid Rientjes compaction_free, (unsigned long)cc, cc->mode, 14137b2a2d4aSMel Gorman MR_COMPACTION); 1414748446bbSMel Gorman 1415f8c9301fSVlastimil Babka trace_mm_compaction_migratepages(cc->nr_migratepages, err, 1416f8c9301fSVlastimil Babka &cc->migratepages); 1417748446bbSMel Gorman 1418f8c9301fSVlastimil Babka /* All pages were either migrated or will be released */ 1419f8c9301fSVlastimil Babka cc->nr_migratepages = 0; 14209d502c1cSMinchan Kim if (err) { 14215733c7d1SRafael Aquini putback_movable_pages(&cc->migratepages); 14227ed695e0SVlastimil Babka /* 14237ed695e0SVlastimil Babka * migrate_pages() may return -ENOMEM when scanners meet 14247ed695e0SVlastimil Babka * and we want compact_finished() to detect it 14257ed695e0SVlastimil Babka */ 1426f2849aa0SVlastimil Babka if (err == -ENOMEM && !compact_scanners_met(cc)) { 14274bf2bba3SDavid Rientjes ret = COMPACT_PARTIAL; 14284bf2bba3SDavid Rientjes goto out; 1429748446bbSMel Gorman } 14304bf2bba3SDavid Rientjes } 1431fdaf7f5cSVlastimil Babka 1432fdaf7f5cSVlastimil Babka /* 1433fdaf7f5cSVlastimil Babka * Record where we could have freed pages by migration and not 1434fdaf7f5cSVlastimil Babka * yet flushed them to buddy allocator. We use the pfn that 1435fdaf7f5cSVlastimil Babka * isolate_migratepages() started from in this loop iteration 1436fdaf7f5cSVlastimil Babka * - this is the lowest page that could have been isolated and 1437fdaf7f5cSVlastimil Babka * then freed by migration. 1438fdaf7f5cSVlastimil Babka */ 1439fdaf7f5cSVlastimil Babka if (!last_migrated_pfn) 1440fdaf7f5cSVlastimil Babka last_migrated_pfn = isolate_start_pfn; 1441fdaf7f5cSVlastimil Babka 1442fdaf7f5cSVlastimil Babka check_drain: 1443fdaf7f5cSVlastimil Babka /* 1444fdaf7f5cSVlastimil Babka * Has the migration scanner moved away from the previous 1445fdaf7f5cSVlastimil Babka * cc->order aligned block where we migrated from? If yes, 1446fdaf7f5cSVlastimil Babka * flush the pages that were freed, so that they can merge and 1447fdaf7f5cSVlastimil Babka * compact_finished() can detect immediately if allocation 1448fdaf7f5cSVlastimil Babka * would succeed. 1449fdaf7f5cSVlastimil Babka */ 1450fdaf7f5cSVlastimil Babka if (cc->order > 0 && last_migrated_pfn) { 1451fdaf7f5cSVlastimil Babka int cpu; 1452fdaf7f5cSVlastimil Babka unsigned long current_block_start = 1453fdaf7f5cSVlastimil Babka cc->migrate_pfn & ~((1UL << cc->order) - 1); 1454fdaf7f5cSVlastimil Babka 1455fdaf7f5cSVlastimil Babka if (last_migrated_pfn < current_block_start) { 1456fdaf7f5cSVlastimil Babka cpu = get_cpu(); 1457fdaf7f5cSVlastimil Babka lru_add_drain_cpu(cpu); 1458fdaf7f5cSVlastimil Babka drain_local_pages(zone); 1459fdaf7f5cSVlastimil Babka put_cpu(); 1460fdaf7f5cSVlastimil Babka /* No more flushing until we migrate again */ 1461fdaf7f5cSVlastimil Babka last_migrated_pfn = 0; 1462fdaf7f5cSVlastimil Babka } 1463fdaf7f5cSVlastimil Babka } 1464fdaf7f5cSVlastimil Babka 1465748446bbSMel Gorman } 1466748446bbSMel Gorman 1467f9e35b3bSMel Gorman out: 14686bace090SVlastimil Babka /* 14696bace090SVlastimil Babka * Release free pages and update where the free scanner should restart, 14706bace090SVlastimil Babka * so we don't leave any returned pages behind in the next attempt. 14716bace090SVlastimil Babka */ 14726bace090SVlastimil Babka if (cc->nr_freepages > 0) { 14736bace090SVlastimil Babka unsigned long free_pfn = release_freepages(&cc->freepages); 14746bace090SVlastimil Babka 14756bace090SVlastimil Babka cc->nr_freepages = 0; 14766bace090SVlastimil Babka VM_BUG_ON(free_pfn == 0); 14776bace090SVlastimil Babka /* The cached pfn is always the first in a pageblock */ 14786bace090SVlastimil Babka free_pfn &= ~(pageblock_nr_pages-1); 14796bace090SVlastimil Babka /* 14806bace090SVlastimil Babka * Only go back, not forward. The cached pfn might have been 14816bace090SVlastimil Babka * already reset to zone end in compact_finished() 14826bace090SVlastimil Babka */ 14836bace090SVlastimil Babka if (free_pfn > zone->compact_cached_free_pfn) 14846bace090SVlastimil Babka zone->compact_cached_free_pfn = free_pfn; 14856bace090SVlastimil Babka } 1486748446bbSMel Gorman 148716c4a097SJoonsoo Kim trace_mm_compaction_end(start_pfn, cc->migrate_pfn, 148816c4a097SJoonsoo Kim cc->free_pfn, end_pfn, sync, ret); 14890eb927c0SMel Gorman 1490748446bbSMel Gorman return ret; 1491748446bbSMel Gorman } 149276ab0f53SMel Gorman 1493e0b9daebSDavid Rientjes static unsigned long compact_zone_order(struct zone *zone, int order, 1494ebff3980SVlastimil Babka gfp_t gfp_mask, enum migrate_mode mode, int *contended, 1495ebff3980SVlastimil Babka int alloc_flags, int classzone_idx) 149656de7263SMel Gorman { 1497e64c5237SShaohua Li unsigned long ret; 149856de7263SMel Gorman struct compact_control cc = { 149956de7263SMel Gorman .nr_freepages = 0, 150056de7263SMel Gorman .nr_migratepages = 0, 150156de7263SMel Gorman .order = order, 15026d7ce559SDavid Rientjes .gfp_mask = gfp_mask, 150356de7263SMel Gorman .zone = zone, 1504e0b9daebSDavid Rientjes .mode = mode, 1505ebff3980SVlastimil Babka .alloc_flags = alloc_flags, 1506ebff3980SVlastimil Babka .classzone_idx = classzone_idx, 150756de7263SMel Gorman }; 150856de7263SMel Gorman INIT_LIST_HEAD(&cc.freepages); 150956de7263SMel Gorman INIT_LIST_HEAD(&cc.migratepages); 151056de7263SMel Gorman 1511e64c5237SShaohua Li ret = compact_zone(zone, &cc); 1512e64c5237SShaohua Li 1513e64c5237SShaohua Li VM_BUG_ON(!list_empty(&cc.freepages)); 1514e64c5237SShaohua Li VM_BUG_ON(!list_empty(&cc.migratepages)); 1515e64c5237SShaohua Li 1516e64c5237SShaohua Li *contended = cc.contended; 1517e64c5237SShaohua Li return ret; 151856de7263SMel Gorman } 151956de7263SMel Gorman 15205e771905SMel Gorman int sysctl_extfrag_threshold = 500; 15215e771905SMel Gorman 152256de7263SMel Gorman /** 152356de7263SMel Gorman * try_to_compact_pages - Direct compact to satisfy a high-order allocation 152456de7263SMel Gorman * @gfp_mask: The GFP mask of the current allocation 15251a6d53a1SVlastimil Babka * @order: The order of the current allocation 15261a6d53a1SVlastimil Babka * @alloc_flags: The allocation flags of the current allocation 15271a6d53a1SVlastimil Babka * @ac: The context of current allocation 1528e0b9daebSDavid Rientjes * @mode: The migration mode for async, sync light, or sync migration 15291f9efdefSVlastimil Babka * @contended: Return value that determines if compaction was aborted due to 15301f9efdefSVlastimil Babka * need_resched() or lock contention 153156de7263SMel Gorman * 153256de7263SMel Gorman * This is the main entry point for direct page compaction. 153356de7263SMel Gorman */ 15341a6d53a1SVlastimil Babka unsigned long try_to_compact_pages(gfp_t gfp_mask, unsigned int order, 15351a6d53a1SVlastimil Babka int alloc_flags, const struct alloc_context *ac, 15361a6d53a1SVlastimil Babka enum migrate_mode mode, int *contended) 153756de7263SMel Gorman { 153856de7263SMel Gorman int may_enter_fs = gfp_mask & __GFP_FS; 153956de7263SMel Gorman int may_perform_io = gfp_mask & __GFP_IO; 154056de7263SMel Gorman struct zoneref *z; 154156de7263SMel Gorman struct zone *zone; 154253853e2dSVlastimil Babka int rc = COMPACT_DEFERRED; 15431f9efdefSVlastimil Babka int all_zones_contended = COMPACT_CONTENDED_LOCK; /* init for &= op */ 15441f9efdefSVlastimil Babka 15451f9efdefSVlastimil Babka *contended = COMPACT_CONTENDED_NONE; 154656de7263SMel Gorman 15474ffb6335SMel Gorman /* Check if the GFP flags allow compaction */ 1548c5a73c3dSAndrea Arcangeli if (!order || !may_enter_fs || !may_perform_io) 154953853e2dSVlastimil Babka return COMPACT_SKIPPED; 155056de7263SMel Gorman 1551837d026dSJoonsoo Kim trace_mm_compaction_try_to_compact_pages(order, gfp_mask, mode); 1552837d026dSJoonsoo Kim 155356de7263SMel Gorman /* Compact each zone in the list */ 15541a6d53a1SVlastimil Babka for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx, 15551a6d53a1SVlastimil Babka ac->nodemask) { 155656de7263SMel Gorman int status; 15571f9efdefSVlastimil Babka int zone_contended; 155856de7263SMel Gorman 155953853e2dSVlastimil Babka if (compaction_deferred(zone, order)) 156053853e2dSVlastimil Babka continue; 156153853e2dSVlastimil Babka 1562e0b9daebSDavid Rientjes status = compact_zone_order(zone, order, gfp_mask, mode, 15631a6d53a1SVlastimil Babka &zone_contended, alloc_flags, 15641a6d53a1SVlastimil Babka ac->classzone_idx); 156556de7263SMel Gorman rc = max(status, rc); 15661f9efdefSVlastimil Babka /* 15671f9efdefSVlastimil Babka * It takes at least one zone that wasn't lock contended 15681f9efdefSVlastimil Babka * to clear all_zones_contended. 15691f9efdefSVlastimil Babka */ 15701f9efdefSVlastimil Babka all_zones_contended &= zone_contended; 157156de7263SMel Gorman 15723e7d3449SMel Gorman /* If a normal allocation would succeed, stop compacting */ 1573ebff3980SVlastimil Babka if (zone_watermark_ok(zone, order, low_wmark_pages(zone), 15741a6d53a1SVlastimil Babka ac->classzone_idx, alloc_flags)) { 157553853e2dSVlastimil Babka /* 157653853e2dSVlastimil Babka * We think the allocation will succeed in this zone, 157753853e2dSVlastimil Babka * but it is not certain, hence the false. The caller 157853853e2dSVlastimil Babka * will repeat this with true if allocation indeed 157953853e2dSVlastimil Babka * succeeds in this zone. 158053853e2dSVlastimil Babka */ 158153853e2dSVlastimil Babka compaction_defer_reset(zone, order, false); 15821f9efdefSVlastimil Babka /* 15831f9efdefSVlastimil Babka * It is possible that async compaction aborted due to 15841f9efdefSVlastimil Babka * need_resched() and the watermarks were ok thanks to 15851f9efdefSVlastimil Babka * somebody else freeing memory. The allocation can 15861f9efdefSVlastimil Babka * however still fail so we better signal the 15871f9efdefSVlastimil Babka * need_resched() contention anyway (this will not 15881f9efdefSVlastimil Babka * prevent the allocation attempt). 15891f9efdefSVlastimil Babka */ 15901f9efdefSVlastimil Babka if (zone_contended == COMPACT_CONTENDED_SCHED) 15911f9efdefSVlastimil Babka *contended = COMPACT_CONTENDED_SCHED; 15921f9efdefSVlastimil Babka 15931f9efdefSVlastimil Babka goto break_loop; 15941f9efdefSVlastimil Babka } 15951f9efdefSVlastimil Babka 1596f8669795SVlastimil Babka if (mode != MIGRATE_ASYNC && status == COMPACT_COMPLETE) { 159753853e2dSVlastimil Babka /* 159853853e2dSVlastimil Babka * We think that allocation won't succeed in this zone 159953853e2dSVlastimil Babka * so we defer compaction there. If it ends up 160053853e2dSVlastimil Babka * succeeding after all, it will be reset. 160153853e2dSVlastimil Babka */ 160253853e2dSVlastimil Babka defer_compaction(zone, order); 160353853e2dSVlastimil Babka } 16041f9efdefSVlastimil Babka 16051f9efdefSVlastimil Babka /* 16061f9efdefSVlastimil Babka * We might have stopped compacting due to need_resched() in 16071f9efdefSVlastimil Babka * async compaction, or due to a fatal signal detected. In that 16081f9efdefSVlastimil Babka * case do not try further zones and signal need_resched() 16091f9efdefSVlastimil Babka * contention. 16101f9efdefSVlastimil Babka */ 16111f9efdefSVlastimil Babka if ((zone_contended == COMPACT_CONTENDED_SCHED) 16121f9efdefSVlastimil Babka || fatal_signal_pending(current)) { 16131f9efdefSVlastimil Babka *contended = COMPACT_CONTENDED_SCHED; 16141f9efdefSVlastimil Babka goto break_loop; 161556de7263SMel Gorman } 161656de7263SMel Gorman 16171f9efdefSVlastimil Babka continue; 16181f9efdefSVlastimil Babka break_loop: 16191f9efdefSVlastimil Babka /* 16201f9efdefSVlastimil Babka * We might not have tried all the zones, so be conservative 16211f9efdefSVlastimil Babka * and assume they are not all lock contended. 16221f9efdefSVlastimil Babka */ 16231f9efdefSVlastimil Babka all_zones_contended = 0; 16241f9efdefSVlastimil Babka break; 16251f9efdefSVlastimil Babka } 16261f9efdefSVlastimil Babka 16271f9efdefSVlastimil Babka /* 16281f9efdefSVlastimil Babka * If at least one zone wasn't deferred or skipped, we report if all 16291f9efdefSVlastimil Babka * zones that were tried were lock contended. 16301f9efdefSVlastimil Babka */ 16311f9efdefSVlastimil Babka if (rc > COMPACT_SKIPPED && all_zones_contended) 16321f9efdefSVlastimil Babka *contended = COMPACT_CONTENDED_LOCK; 16331f9efdefSVlastimil Babka 163456de7263SMel Gorman return rc; 163556de7263SMel Gorman } 163656de7263SMel Gorman 163756de7263SMel Gorman 163876ab0f53SMel Gorman /* Compact all zones within a node */ 16397103f16dSAndrew Morton static void __compact_pgdat(pg_data_t *pgdat, struct compact_control *cc) 164076ab0f53SMel Gorman { 164176ab0f53SMel Gorman int zoneid; 164276ab0f53SMel Gorman struct zone *zone; 164376ab0f53SMel Gorman 164476ab0f53SMel Gorman for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) { 164576ab0f53SMel Gorman 164676ab0f53SMel Gorman zone = &pgdat->node_zones[zoneid]; 164776ab0f53SMel Gorman if (!populated_zone(zone)) 164876ab0f53SMel Gorman continue; 164976ab0f53SMel Gorman 16507be62de9SRik van Riel cc->nr_freepages = 0; 16517be62de9SRik van Riel cc->nr_migratepages = 0; 16527be62de9SRik van Riel cc->zone = zone; 16537be62de9SRik van Riel INIT_LIST_HEAD(&cc->freepages); 16547be62de9SRik van Riel INIT_LIST_HEAD(&cc->migratepages); 165576ab0f53SMel Gorman 1656195b0c60SGioh Kim /* 1657195b0c60SGioh Kim * When called via /proc/sys/vm/compact_memory 1658195b0c60SGioh Kim * this makes sure we compact the whole zone regardless of 1659195b0c60SGioh Kim * cached scanner positions. 1660195b0c60SGioh Kim */ 1661195b0c60SGioh Kim if (cc->order == -1) 1662195b0c60SGioh Kim __reset_isolation_suitable(zone); 1663195b0c60SGioh Kim 1664aad6ec37SDan Carpenter if (cc->order == -1 || !compaction_deferred(zone, cc->order)) 16657be62de9SRik van Riel compact_zone(zone, cc); 166676ab0f53SMel Gorman 1667aff62249SRik van Riel if (cc->order > 0) { 1668de6c60a6SVlastimil Babka if (zone_watermark_ok(zone, cc->order, 1669de6c60a6SVlastimil Babka low_wmark_pages(zone), 0, 0)) 1670de6c60a6SVlastimil Babka compaction_defer_reset(zone, cc->order, false); 1671aff62249SRik van Riel } 1672aff62249SRik van Riel 16737be62de9SRik van Riel VM_BUG_ON(!list_empty(&cc->freepages)); 16747be62de9SRik van Riel VM_BUG_ON(!list_empty(&cc->migratepages)); 167576ab0f53SMel Gorman } 167676ab0f53SMel Gorman } 167776ab0f53SMel Gorman 16787103f16dSAndrew Morton void compact_pgdat(pg_data_t *pgdat, int order) 16797be62de9SRik van Riel { 16807be62de9SRik van Riel struct compact_control cc = { 16817be62de9SRik van Riel .order = order, 1682e0b9daebSDavid Rientjes .mode = MIGRATE_ASYNC, 16837be62de9SRik van Riel }; 16847be62de9SRik van Riel 16853a7200afSMel Gorman if (!order) 16863a7200afSMel Gorman return; 16873a7200afSMel Gorman 16887103f16dSAndrew Morton __compact_pgdat(pgdat, &cc); 16897be62de9SRik van Riel } 16907be62de9SRik van Riel 16917103f16dSAndrew Morton static void compact_node(int nid) 16927be62de9SRik van Riel { 16937be62de9SRik van Riel struct compact_control cc = { 16947be62de9SRik van Riel .order = -1, 1695e0b9daebSDavid Rientjes .mode = MIGRATE_SYNC, 169691ca9186SDavid Rientjes .ignore_skip_hint = true, 16977be62de9SRik van Riel }; 16987be62de9SRik van Riel 16997103f16dSAndrew Morton __compact_pgdat(NODE_DATA(nid), &cc); 17007be62de9SRik van Riel } 17017be62de9SRik van Riel 170276ab0f53SMel Gorman /* Compact all nodes in the system */ 17037964c06dSJason Liu static void compact_nodes(void) 170476ab0f53SMel Gorman { 170576ab0f53SMel Gorman int nid; 170676ab0f53SMel Gorman 17078575ec29SHugh Dickins /* Flush pending updates to the LRU lists */ 17088575ec29SHugh Dickins lru_add_drain_all(); 17098575ec29SHugh Dickins 171076ab0f53SMel Gorman for_each_online_node(nid) 171176ab0f53SMel Gorman compact_node(nid); 171276ab0f53SMel Gorman } 171376ab0f53SMel Gorman 171476ab0f53SMel Gorman /* The written value is actually unused, all memory is compacted */ 171576ab0f53SMel Gorman int sysctl_compact_memory; 171676ab0f53SMel Gorman 171776ab0f53SMel Gorman /* This is the entry point for compacting all nodes via /proc/sys/vm */ 171876ab0f53SMel Gorman int sysctl_compaction_handler(struct ctl_table *table, int write, 171976ab0f53SMel Gorman void __user *buffer, size_t *length, loff_t *ppos) 172076ab0f53SMel Gorman { 172176ab0f53SMel Gorman if (write) 17227964c06dSJason Liu compact_nodes(); 172376ab0f53SMel Gorman 172476ab0f53SMel Gorman return 0; 172576ab0f53SMel Gorman } 1726ed4a6d7fSMel Gorman 17275e771905SMel Gorman int sysctl_extfrag_handler(struct ctl_table *table, int write, 17285e771905SMel Gorman void __user *buffer, size_t *length, loff_t *ppos) 17295e771905SMel Gorman { 17305e771905SMel Gorman proc_dointvec_minmax(table, write, buffer, length, ppos); 17315e771905SMel Gorman 17325e771905SMel Gorman return 0; 17335e771905SMel Gorman } 17345e771905SMel Gorman 1735ed4a6d7fSMel Gorman #if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA) 173674e77fb9SRashika Kheria static ssize_t sysfs_compact_node(struct device *dev, 173710fbcf4cSKay Sievers struct device_attribute *attr, 1738ed4a6d7fSMel Gorman const char *buf, size_t count) 1739ed4a6d7fSMel Gorman { 17408575ec29SHugh Dickins int nid = dev->id; 17418575ec29SHugh Dickins 17428575ec29SHugh Dickins if (nid >= 0 && nid < nr_node_ids && node_online(nid)) { 17438575ec29SHugh Dickins /* Flush pending updates to the LRU lists */ 17448575ec29SHugh Dickins lru_add_drain_all(); 17458575ec29SHugh Dickins 17468575ec29SHugh Dickins compact_node(nid); 17478575ec29SHugh Dickins } 1748ed4a6d7fSMel Gorman 1749ed4a6d7fSMel Gorman return count; 1750ed4a6d7fSMel Gorman } 175110fbcf4cSKay Sievers static DEVICE_ATTR(compact, S_IWUSR, NULL, sysfs_compact_node); 1752ed4a6d7fSMel Gorman 1753ed4a6d7fSMel Gorman int compaction_register_node(struct node *node) 1754ed4a6d7fSMel Gorman { 175510fbcf4cSKay Sievers return device_create_file(&node->dev, &dev_attr_compact); 1756ed4a6d7fSMel Gorman } 1757ed4a6d7fSMel Gorman 1758ed4a6d7fSMel Gorman void compaction_unregister_node(struct node *node) 1759ed4a6d7fSMel Gorman { 176010fbcf4cSKay Sievers return device_remove_file(&node->dev, &dev_attr_compact); 1761ed4a6d7fSMel Gorman } 1762ed4a6d7fSMel Gorman #endif /* CONFIG_SYSFS && CONFIG_NUMA */ 1763ff9543fdSMichal Nazarewicz 1764ff9543fdSMichal Nazarewicz #endif /* CONFIG_COMPACTION */ 1765