126 static void defer_compaction(struct zone *zone, int order)  in defer_compaction()  argument
128 	zone->compact_considered = 0;  in defer_compaction()
129 	zone->compact_defer_shift++;  in defer_compaction()
131 	if (order < zone->compact_order_failed)  in defer_compaction()
132 		zone->compact_order_failed = order;  in defer_compaction()
134 	if (zone->compact_defer_shift > COMPACT_MAX_DEFER_SHIFT)  in defer_compaction()
135 		zone->compact_defer_shift = COMPACT_MAX_DEFER_SHIFT;  in defer_compaction()
137 	trace_mm_compaction_defer_compaction(zone, order);  in defer_compaction()
141 static bool compaction_deferred(struct zone *zone, int order)  in compaction_deferred()  argument
143 	unsigned long defer_limit = 1UL << zone->compact_defer_shift;  in compaction_deferred()
145 	if (order < zone->compact_order_failed)  in compaction_deferred()
149 	if (++zone->compact_considered >= defer_limit) {  in compaction_deferred()
150 		zone->compact_considered = defer_limit;  in compaction_deferred()
154 	trace_mm_compaction_deferred(zone, order);  in compaction_deferred()
164 void compaction_defer_reset(struct zone *zone, int order,  in compaction_defer_reset()  argument
168 		zone->compact_considered = 0;  in compaction_defer_reset()
169 		zone->compact_defer_shift = 0;  in compaction_defer_reset()
171 	if (order >= zone->compact_order_failed)  in compaction_defer_reset()
172 		zone->compact_order_failed = order + 1;  in compaction_defer_reset()
174 	trace_mm_compaction_defer_reset(zone, order);  in compaction_defer_reset()
178 static bool compaction_restarting(struct zone *zone, int order)  in compaction_restarting()  argument
180 	if (order < zone->compact_order_failed)  in compaction_restarting()
183 	return zone->compact_defer_shift == COMPACT_MAX_DEFER_SHIFT &&  in compaction_restarting()
184 		zone->compact_considered >= 1UL << zone->compact_defer_shift;  in compaction_restarting()
197 static void reset_cached_positions(struct zone *zone)  in reset_cached_positions()  argument
199 	zone->compact_cached_migrate_pfn[0] = zone->zone_start_pfn;  in reset_cached_positions()
200 	zone->compact_cached_migrate_pfn[1] = zone->zone_start_pfn;  in reset_cached_positions()
201 	zone->compact_cached_free_pfn =  in reset_cached_positions()
202 				pageblock_start_pfn(zone_end_pfn(zone) - 1);  in reset_cached_positions()
276 __reset_isolation_pfn(struct zone *zone, unsigned long pfn, bool check_source,  in __reset_isolation_pfn()  argument
286 	if (zone != page_zone(page))  in __reset_isolation_pfn()
308 	block_pfn = max(block_pfn, zone->zone_start_pfn);  in __reset_isolation_pfn()
317 	block_pfn = min(block_pfn, zone_end_pfn(zone) - 1);  in __reset_isolation_pfn()
349 static void __reset_isolation_suitable(struct zone *zone)  in __reset_isolation_suitable()  argument
351 	unsigned long migrate_pfn = zone->zone_start_pfn;  in __reset_isolation_suitable()
352 	unsigned long free_pfn = zone_end_pfn(zone) - 1;  in __reset_isolation_suitable()
359 	if (!zone->compact_blockskip_flush)  in __reset_isolation_suitable()
362 	zone->compact_blockskip_flush = false;  in __reset_isolation_suitable()
375 		if (__reset_isolation_pfn(zone, migrate_pfn, true, source_set) &&  in __reset_isolation_suitable()
379 			zone->compact_init_migrate_pfn = reset_migrate;  in __reset_isolation_suitable()
380 			zone->compact_cached_migrate_pfn[0] = reset_migrate;  in __reset_isolation_suitable()
381 			zone->compact_cached_migrate_pfn[1] = reset_migrate;  in __reset_isolation_suitable()
385 		if (__reset_isolation_pfn(zone, free_pfn, free_set, true) &&  in __reset_isolation_suitable()
389 			zone->compact_init_free_pfn = reset_free;  in __reset_isolation_suitable()
390 			zone->compact_cached_free_pfn = reset_free;  in __reset_isolation_suitable()
396 		zone->compact_cached_migrate_pfn[0] = migrate_pfn;  in __reset_isolation_suitable()
397 		zone->compact_cached_migrate_pfn[1] = migrate_pfn;  in __reset_isolation_suitable()
398 		zone->compact_cached_free_pfn = free_pfn;  in __reset_isolation_suitable()
407 		struct zone *zone = &pgdat->node_zones[zoneid];  in reset_isolation_suitable()  local
408 		if (!populated_zone(zone))  in reset_isolation_suitable()
411 		__reset_isolation_suitable(zone);  in reset_isolation_suitable()
436 	struct zone *zone = cc->zone;  in update_cached_migrate()  local
445 	if (pfn > zone->compact_cached_migrate_pfn[0])  in update_cached_migrate()
446 		zone->compact_cached_migrate_pfn[0] = pfn;  in update_cached_migrate()
448 	    pfn > zone->compact_cached_migrate_pfn[1])  in update_cached_migrate()
449 		zone->compact_cached_migrate_pfn[1] = pfn;  in update_cached_migrate()
459 	struct zone *zone = cc->zone;  in update_pageblock_skip()  local
466 	if (pfn < zone->compact_cached_free_pfn)  in update_pageblock_skip()
467 		zone->compact_cached_free_pfn = pfn;  in update_pageblock_skip()
586 		    && compact_unlock_should_abort(&cc->zone->lock, flags,  in isolate_freepages_block()
616 			locked = compact_lock_irqsave(&cc->zone->lock,  in isolate_freepages_block()
652 		spin_unlock_irqrestore(&cc->zone->lock, flags);  in isolate_freepages_block()
706 	if (block_start_pfn < cc->zone->zone_start_pfn)  in isolate_freepages_range()
707 		block_start_pfn = cc->zone->zone_start_pfn;  in isolate_freepages_range()
729 					block_end_pfn, cc->zone))  in isolate_freepages_range()
763 	pg_data_t *pgdat = cc->zone->zone_pgdat;  in too_many_isolated()
840 	pg_data_t *pgdat = cc->zone->zone_pgdat;  in isolate_migratepages_block()
941 				    low_pfn == cc->zone->zone_start_pfn)) {  in isolate_migratepages_block()
1329 	if (block_start_pfn < cc->zone->zone_start_pfn)  in isolate_migratepages_range()
1330 		block_start_pfn = cc->zone->zone_start_pfn;  in isolate_migratepages_range()
1340 					block_end_pfn, cc->zone))  in isolate_migratepages_range()
1472 	start_pfn = max(pageblock_start_pfn(pfn), cc->zone->zone_start_pfn);  in fast_isolate_around()
1473 	end_pfn = min(pageblock_end_pfn(pfn), zone_end_pfn(cc->zone));  in fast_isolate_around()
1475 	page = pageblock_pfn_to_page(start_pfn, end_pfn, cc->zone);  in fast_isolate_around()
1523 	if (cc->free_pfn >= cc->zone->compact_init_free_pfn) {  in fast_isolate_freepages()
1548 		struct free_area *area = &cc->zone->free_area[order];  in fast_isolate_freepages()
1558 		spin_lock_irqsave(&cc->zone->lock, flags);  in fast_isolate_freepages()
1569 					      cc->zone->zone_start_pfn);  in fast_isolate_freepages()
1617 		spin_unlock_irqrestore(&cc->zone->lock, flags);  in fast_isolate_freepages()
1649 						    zone_end_pfn(cc->zone)),  in fast_isolate_freepages()
1650 						cc->zone);  in fast_isolate_freepages()
1660 	if (highest && highest >= cc->zone->compact_cached_free_pfn) {  in fast_isolate_freepages()
1662 		cc->zone->compact_cached_free_pfn = highest;  in fast_isolate_freepages()
1679 	struct zone *zone = cc->zone;  in isolate_freepages()  local
1706 						zone_end_pfn(zone));  in isolate_freepages()
1729 									zone);  in isolate_freepages()
1949 	if (pfn != cc->zone->zone_start_pfn && pfn != pageblock_start_pfn(pfn))  in fast_find_migrateblock()
1976 	if (cc->migrate_pfn != cc->zone->zone_start_pfn)  in fast_find_migrateblock()
1983 		struct free_area *area = &cc->zone->free_area[order];  in fast_find_migrateblock()
1991 		spin_lock_irqsave(&cc->zone->lock, flags);  in fast_find_migrateblock()
2017 				if (pfn < cc->zone->zone_start_pfn)  in fast_find_migrateblock()
2018 					pfn = cc->zone->zone_start_pfn;  in fast_find_migrateblock()
2024 		spin_unlock_irqrestore(&cc->zone->lock, flags);  in fast_find_migrateblock()
2063 	if (block_start_pfn < cc->zone->zone_start_pfn)  in isolate_migratepages()
2064 		block_start_pfn = cc->zone->zone_start_pfn;  in isolate_migratepages()
2095 						block_end_pfn, cc->zone);  in isolate_migratepages()
2113 		     low_pfn == cc->zone->zone_start_pfn) &&  in isolate_migratepages()
2167 static unsigned int fragmentation_score_zone(struct zone *zone)  in fragmentation_score_zone()  argument
2169 	return extfrag_for_order(zone, COMPACTION_HPAGE_ORDER);  in fragmentation_score_zone()
2182 static unsigned int fragmentation_score_zone_weighted(struct zone *zone)  in fragmentation_score_zone_weighted()  argument
2186 	score = zone->present_pages * fragmentation_score_zone(zone);  in fragmentation_score_zone_weighted()
2187 	return div64_ul(score, zone->zone_pgdat->node_present_pages + 1);  in fragmentation_score_zone_weighted()
2203 		struct zone *zone;  in fragmentation_score_node()  local
2205 		zone = &pgdat->node_zones[zoneid];  in fragmentation_score_node()
2206 		if (!populated_zone(zone))  in fragmentation_score_node()
2208 		score += fragmentation_score_zone_weighted(zone);  in fragmentation_score_node()
2243 		reset_cached_positions(cc->zone);  in __compact_finished()
2252 			cc->zone->compact_blockskip_flush = true;  in __compact_finished()
2264 		pgdat = cc->zone->zone_pgdat;  in __compact_finished()
2268 		score = fragmentation_score_zone(cc->zone);  in __compact_finished()
2297 		if (__zone_watermark_ok(cc->zone, cc->order,  in __compact_finished()
2298 					high_wmark_pages(cc->zone),  in __compact_finished()
2300 					zone_page_state(cc->zone,  in __compact_finished()
2310 		struct free_area *area = &cc->zone->free_area[order];  in __compact_finished()
2350 	trace_mm_compaction_finished(cc->zone, cc->order, ret);  in compact_finished()
2357 static bool __compaction_suitable(struct zone *zone, int order,  in __compaction_suitable()  argument
2379 		watermark += low_wmark_pages(zone) - min_wmark_pages(zone);  in __compaction_suitable()
2380 	return __zone_watermark_ok(zone, 0, watermark, highest_zoneidx,  in __compaction_suitable()
2387 bool compaction_suitable(struct zone *zone, int order, unsigned long watermark,  in compaction_suitable()  argument
2393 	suitable = __compaction_suitable(zone, order, watermark, highest_zoneidx,  in compaction_suitable()
2394 					 zone_page_state(zone, NR_FREE_PAGES));  in compaction_suitable()
2414 			int fragindex = fragmentation_index(zone, order);  in compaction_suitable()
2426 	trace_mm_compaction_suitable(zone, order, compact_result);  in compaction_suitable()
2435 	struct zone *zone;  in compaction_zonelist_suitable()  local
2442 	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,  in compaction_zonelist_suitable()
2452 		available = zone_reclaimable_pages(zone) / order;  in compaction_zonelist_suitable()
2453 		available += zone_page_state_snapshot(zone, NR_FREE_PAGES);  in compaction_zonelist_suitable()
2454 		if (__compaction_suitable(zone, order, min_wmark_pages(zone),  in compaction_zonelist_suitable()
2470 compaction_suit_allocation_order(struct zone *zone, unsigned int order,  in compaction_suit_allocation_order()  argument
2478 		free_pages = zone_page_state(zone, NR_FREE_PAGES_BLOCKS);  in compaction_suit_allocation_order()
2480 		free_pages = zone_page_state(zone, NR_FREE_PAGES);  in compaction_suit_allocation_order()
2482 	watermark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK);  in compaction_suit_allocation_order()
2483 	if (__zone_watermark_ok(zone, order, watermark, highest_zoneidx,  in compaction_suit_allocation_order()
2498 		if (!__zone_watermark_ok(zone, 0, watermark + compact_gap(order),  in compaction_suit_allocation_order()
2500 					 zone_page_state(zone, NR_FREE_PAGES)))  in compaction_suit_allocation_order()
2504 	if (!compaction_suitable(zone, order, watermark, highest_zoneidx))  in compaction_suit_allocation_order()
2514 	unsigned long start_pfn = cc->zone->zone_start_pfn;  in compact_zone()
2515 	unsigned long end_pfn = zone_end_pfn(cc->zone);  in compact_zone()
2537 		ret = compaction_suit_allocation_order(cc->zone, cc->order,  in compact_zone()
2550 	if (compaction_restarting(cc->zone, cc->order))  in compact_zone()
2551 		__reset_isolation_suitable(cc->zone);  in compact_zone()
2564 		cc->migrate_pfn = cc->zone->compact_cached_migrate_pfn[sync];  in compact_zone()
2565 		cc->free_pfn = cc->zone->compact_cached_free_pfn;  in compact_zone()
2568 			cc->zone->compact_cached_free_pfn = cc->free_pfn;  in compact_zone()
2572 			cc->zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn;  in compact_zone()
2573 			cc->zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn;  in compact_zone()
2576 		if (cc->migrate_pfn <= cc->zone->compact_init_migrate_pfn)  in compact_zone()
2591 		cc->zone->compact_cached_migrate_pfn[0] == cc->zone->compact_cached_migrate_pfn[1];  in compact_zone()
2625 				cc->zone->compact_cached_migrate_pfn[1] =  in compact_zone()
2626 					cc->zone->compact_cached_migrate_pfn[0];  in compact_zone()
2637 			last_migrated_pfn = max(cc->zone->zone_start_pfn,  in compact_zone()
2712 				lru_add_drain_cpu_zone(cc->zone);  in compact_zone()
2735 		if (free_pfn > cc->zone->compact_cached_free_pfn)  in compact_zone()
2736 			cc->zone->compact_cached_free_pfn = free_pfn;  in compact_zone()
2749 static enum compact_result compact_zone_order(struct zone *zone, int order,  in compact_zone_order()  argument
2759 		.zone = zone,  in compact_zone_order()
2819 	struct zone *zone;  in try_to_compact_pages()  local
2828 	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,  in try_to_compact_pages()
2834 			!__cpuset_zone_allowed(zone, gfp_mask))  in try_to_compact_pages()
2838 					&& compaction_deferred(zone, order)) {  in try_to_compact_pages()
2843 		status = compact_zone_order(zone, order, gfp_mask, prio,  in try_to_compact_pages()
2855 			compaction_defer_reset(zone, order, false);  in try_to_compact_pages()
2867 			defer_compaction(zone, order);  in try_to_compact_pages()
2896 	struct zone *zone;  in compact_node()  local
2907 		zone = &pgdat->node_zones[zoneid];  in compact_node()
2908 		if (!populated_zone(zone))  in compact_node()
2914 		cc.zone = zone;  in compact_node()
3032 	struct zone *zone;  in kcompactd_node_suitable()  local
3039 		zone = &pgdat->node_zones[zoneid];  in kcompactd_node_suitable()
3041 		if (!populated_zone(zone))  in kcompactd_node_suitable()
3044 		ret = compaction_suit_allocation_order(zone,  in kcompactd_node_suitable()
3062 	struct zone *zone;  in kcompactd_do_work()  local
3081 		zone = &pgdat->node_zones[zoneid];  in kcompactd_do_work()
3082 		if (!populated_zone(zone))  in kcompactd_do_work()
3085 		if (compaction_deferred(zone, cc.order))  in kcompactd_do_work()
3088 		ret = compaction_suit_allocation_order(zone,  in kcompactd_do_work()
3097 		cc.zone = zone;  in kcompactd_do_work()
3101 			compaction_defer_reset(zone, cc.order, false);  in kcompactd_do_work()
3109 			drain_all_pages(zone);  in kcompactd_do_work()
3115 			defer_compaction(zone, cc.order);  in kcompactd_do_work()