xref: /linux/mm/compaction.c (revision f73a058be5d70dd81a43f16b2bbff4b1576a7af8)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * linux/mm/compaction.c
4  *
5  * Memory compaction for the reduction of external fragmentation. Note that
6  * this heavily depends upon page migration to do all the real heavy
7  * lifting
8  *
9  * Copyright IBM Corp. 2007-2010 Mel Gorman <mel@csn.ul.ie>
10  */
11 #include <linux/cpu.h>
12 #include <linux/swap.h>
13 #include <linux/migrate.h>
14 #include <linux/compaction.h>
15 #include <linux/mm_inline.h>
16 #include <linux/sched/signal.h>
17 #include <linux/backing-dev.h>
18 #include <linux/sysctl.h>
19 #include <linux/sysfs.h>
20 #include <linux/page-isolation.h>
21 #include <linux/kasan.h>
22 #include <linux/kthread.h>
23 #include <linux/freezer.h>
24 #include <linux/page_owner.h>
25 #include <linux/psi.h>
26 #include "internal.h"
27 
28 #ifdef CONFIG_COMPACTION
29 /*
30  * Fragmentation score check interval for proactive compaction purposes.
31  */
32 #define HPAGE_FRAG_CHECK_INTERVAL_MSEC	(500)
33 
34 static inline void count_compact_event(enum vm_event_item item)
35 {
36 	count_vm_event(item);
37 }
38 
39 static inline void count_compact_events(enum vm_event_item item, long delta)
40 {
41 	count_vm_events(item, delta);
42 }
43 
44 /*
45  * order == -1 is expected when compacting proactively via
46  * 1. /proc/sys/vm/compact_memory
47  * 2. /sys/devices/system/node/nodex/compact
48  * 3. /proc/sys/vm/compaction_proactiveness
49  */
50 static inline bool is_via_compact_memory(int order)
51 {
52 	return order == -1;
53 }
54 
55 #else
56 #define count_compact_event(item) do { } while (0)
57 #define count_compact_events(item, delta) do { } while (0)
58 static inline bool is_via_compact_memory(int order) { return false; }
59 #endif
60 
61 #if defined CONFIG_COMPACTION || defined CONFIG_CMA
62 
63 #define CREATE_TRACE_POINTS
64 #include <trace/events/compaction.h>
65 
66 #define block_start_pfn(pfn, order)	round_down(pfn, 1UL << (order))
67 #define block_end_pfn(pfn, order)	ALIGN((pfn) + 1, 1UL << (order))
68 
69 /*
70  * Page order with-respect-to which proactive compaction
71  * calculates external fragmentation, which is used as
72  * the "fragmentation score" of a node/zone.
73  */
74 #if defined CONFIG_TRANSPARENT_HUGEPAGE
75 #define COMPACTION_HPAGE_ORDER	HPAGE_PMD_ORDER
76 #elif defined CONFIG_HUGETLBFS
77 #define COMPACTION_HPAGE_ORDER	HUGETLB_PAGE_ORDER
78 #else
79 #define COMPACTION_HPAGE_ORDER	(PMD_SHIFT - PAGE_SHIFT)
80 #endif
81 
82 static void split_map_pages(struct list_head *freepages)
83 {
84 	unsigned int i, order;
85 	struct page *page, *next;
86 	LIST_HEAD(tmp_list);
87 
88 	for (order = 0; order < NR_PAGE_ORDERS; order++) {
89 		list_for_each_entry_safe(page, next, &freepages[order], lru) {
90 			unsigned int nr_pages;
91 
92 			list_del(&page->lru);
93 
94 			nr_pages = 1 << order;
95 
96 			post_alloc_hook(page, order, __GFP_MOVABLE);
97 			if (order)
98 				split_page(page, order);
99 
100 			for (i = 0; i < nr_pages; i++) {
101 				list_add(&page->lru, &tmp_list);
102 				page++;
103 			}
104 		}
105 		list_splice_init(&tmp_list, &freepages[0]);
106 	}
107 }
108 
109 static unsigned long release_free_list(struct list_head *freepages)
110 {
111 	int order;
112 	unsigned long high_pfn = 0;
113 
114 	for (order = 0; order < NR_PAGE_ORDERS; order++) {
115 		struct page *page, *next;
116 
117 		list_for_each_entry_safe(page, next, &freepages[order], lru) {
118 			unsigned long pfn = page_to_pfn(page);
119 
120 			list_del(&page->lru);
121 			/*
122 			 * Convert free pages into post allocation pages, so
123 			 * that we can free them via __free_page.
124 			 */
125 			post_alloc_hook(page, order, __GFP_MOVABLE);
126 			__free_pages(page, order);
127 			if (pfn > high_pfn)
128 				high_pfn = pfn;
129 		}
130 	}
131 	return high_pfn;
132 }
133 
134 #ifdef CONFIG_COMPACTION
135 bool PageMovable(struct page *page)
136 {
137 	const struct movable_operations *mops;
138 
139 	VM_BUG_ON_PAGE(!PageLocked(page), page);
140 	if (!__PageMovable(page))
141 		return false;
142 
143 	mops = page_movable_ops(page);
144 	if (mops)
145 		return true;
146 
147 	return false;
148 }
149 
150 void __SetPageMovable(struct page *page, const struct movable_operations *mops)
151 {
152 	VM_BUG_ON_PAGE(!PageLocked(page), page);
153 	VM_BUG_ON_PAGE((unsigned long)mops & PAGE_MAPPING_MOVABLE, page);
154 	page->mapping = (void *)((unsigned long)mops | PAGE_MAPPING_MOVABLE);
155 }
156 EXPORT_SYMBOL(__SetPageMovable);
157 
158 void __ClearPageMovable(struct page *page)
159 {
160 	VM_BUG_ON_PAGE(!PageMovable(page), page);
161 	/*
162 	 * This page still has the type of a movable page, but it's
163 	 * actually not movable any more.
164 	 */
165 	page->mapping = (void *)PAGE_MAPPING_MOVABLE;
166 }
167 EXPORT_SYMBOL(__ClearPageMovable);
168 
169 /* Do not skip compaction more than 64 times */
170 #define COMPACT_MAX_DEFER_SHIFT 6
171 
172 /*
173  * Compaction is deferred when compaction fails to result in a page
174  * allocation success. 1 << compact_defer_shift, compactions are skipped up
175  * to a limit of 1 << COMPACT_MAX_DEFER_SHIFT
176  */
177 static void defer_compaction(struct zone *zone, int order)
178 {
179 	zone->compact_considered = 0;
180 	zone->compact_defer_shift++;
181 
182 	if (order < zone->compact_order_failed)
183 		zone->compact_order_failed = order;
184 
185 	if (zone->compact_defer_shift > COMPACT_MAX_DEFER_SHIFT)
186 		zone->compact_defer_shift = COMPACT_MAX_DEFER_SHIFT;
187 
188 	trace_mm_compaction_defer_compaction(zone, order);
189 }
190 
191 /* Returns true if compaction should be skipped this time */
192 static bool compaction_deferred(struct zone *zone, int order)
193 {
194 	unsigned long defer_limit = 1UL << zone->compact_defer_shift;
195 
196 	if (order < zone->compact_order_failed)
197 		return false;
198 
199 	/* Avoid possible overflow */
200 	if (++zone->compact_considered >= defer_limit) {
201 		zone->compact_considered = defer_limit;
202 		return false;
203 	}
204 
205 	trace_mm_compaction_deferred(zone, order);
206 
207 	return true;
208 }
209 
210 /*
211  * Update defer tracking counters after successful compaction of given order,
212  * which means an allocation either succeeded (alloc_success == true) or is
213  * expected to succeed.
214  */
215 void compaction_defer_reset(struct zone *zone, int order,
216 		bool alloc_success)
217 {
218 	if (alloc_success) {
219 		zone->compact_considered = 0;
220 		zone->compact_defer_shift = 0;
221 	}
222 	if (order >= zone->compact_order_failed)
223 		zone->compact_order_failed = order + 1;
224 
225 	trace_mm_compaction_defer_reset(zone, order);
226 }
227 
228 /* Returns true if restarting compaction after many failures */
229 static bool compaction_restarting(struct zone *zone, int order)
230 {
231 	if (order < zone->compact_order_failed)
232 		return false;
233 
234 	return zone->compact_defer_shift == COMPACT_MAX_DEFER_SHIFT &&
235 		zone->compact_considered >= 1UL << zone->compact_defer_shift;
236 }
237 
238 /* Returns true if the pageblock should be scanned for pages to isolate. */
239 static inline bool isolation_suitable(struct compact_control *cc,
240 					struct page *page)
241 {
242 	if (cc->ignore_skip_hint)
243 		return true;
244 
245 	return !get_pageblock_skip(page);
246 }
247 
248 static void reset_cached_positions(struct zone *zone)
249 {
250 	zone->compact_cached_migrate_pfn[0] = zone->zone_start_pfn;
251 	zone->compact_cached_migrate_pfn[1] = zone->zone_start_pfn;
252 	zone->compact_cached_free_pfn =
253 				pageblock_start_pfn(zone_end_pfn(zone) - 1);
254 }
255 
256 #ifdef CONFIG_SPARSEMEM
257 /*
258  * If the PFN falls into an offline section, return the start PFN of the
259  * next online section. If the PFN falls into an online section or if
260  * there is no next online section, return 0.
261  */
262 static unsigned long skip_offline_sections(unsigned long start_pfn)
263 {
264 	unsigned long start_nr = pfn_to_section_nr(start_pfn);
265 
266 	if (online_section_nr(start_nr))
267 		return 0;
268 
269 	while (++start_nr <= __highest_present_section_nr) {
270 		if (online_section_nr(start_nr))
271 			return section_nr_to_pfn(start_nr);
272 	}
273 
274 	return 0;
275 }
276 
277 /*
278  * If the PFN falls into an offline section, return the end PFN of the
279  * next online section in reverse. If the PFN falls into an online section
280  * or if there is no next online section in reverse, return 0.
281  */
282 static unsigned long skip_offline_sections_reverse(unsigned long start_pfn)
283 {
284 	unsigned long start_nr = pfn_to_section_nr(start_pfn);
285 
286 	if (!start_nr || online_section_nr(start_nr))
287 		return 0;
288 
289 	while (start_nr-- > 0) {
290 		if (online_section_nr(start_nr))
291 			return section_nr_to_pfn(start_nr) + PAGES_PER_SECTION;
292 	}
293 
294 	return 0;
295 }
296 #else
297 static unsigned long skip_offline_sections(unsigned long start_pfn)
298 {
299 	return 0;
300 }
301 
302 static unsigned long skip_offline_sections_reverse(unsigned long start_pfn)
303 {
304 	return 0;
305 }
306 #endif
307 
308 /*
309  * Compound pages of >= pageblock_order should consistently be skipped until
310  * released. It is always pointless to compact pages of such order (if they are
311  * migratable), and the pageblocks they occupy cannot contain any free pages.
312  */
313 static bool pageblock_skip_persistent(struct page *page)
314 {
315 	if (!PageCompound(page))
316 		return false;
317 
318 	page = compound_head(page);
319 
320 	if (compound_order(page) >= pageblock_order)
321 		return true;
322 
323 	return false;
324 }
325 
326 static bool
327 __reset_isolation_pfn(struct zone *zone, unsigned long pfn, bool check_source,
328 							bool check_target)
329 {
330 	struct page *page = pfn_to_online_page(pfn);
331 	struct page *block_page;
332 	struct page *end_page;
333 	unsigned long block_pfn;
334 
335 	if (!page)
336 		return false;
337 	if (zone != page_zone(page))
338 		return false;
339 	if (pageblock_skip_persistent(page))
340 		return false;
341 
342 	/*
343 	 * If skip is already cleared do no further checking once the
344 	 * restart points have been set.
345 	 */
346 	if (check_source && check_target && !get_pageblock_skip(page))
347 		return true;
348 
349 	/*
350 	 * If clearing skip for the target scanner, do not select a
351 	 * non-movable pageblock as the starting point.
352 	 */
353 	if (!check_source && check_target &&
354 	    get_pageblock_migratetype(page) != MIGRATE_MOVABLE)
355 		return false;
356 
357 	/* Ensure the start of the pageblock or zone is online and valid */
358 	block_pfn = pageblock_start_pfn(pfn);
359 	block_pfn = max(block_pfn, zone->zone_start_pfn);
360 	block_page = pfn_to_online_page(block_pfn);
361 	if (block_page) {
362 		page = block_page;
363 		pfn = block_pfn;
364 	}
365 
366 	/* Ensure the end of the pageblock or zone is online and valid */
367 	block_pfn = pageblock_end_pfn(pfn) - 1;
368 	block_pfn = min(block_pfn, zone_end_pfn(zone) - 1);
369 	end_page = pfn_to_online_page(block_pfn);
370 	if (!end_page)
371 		return false;
372 
373 	/*
374 	 * Only clear the hint if a sample indicates there is either a
375 	 * free page or an LRU page in the block. One or other condition
376 	 * is necessary for the block to be a migration source/target.
377 	 */
378 	do {
379 		if (check_source && PageLRU(page)) {
380 			clear_pageblock_skip(page);
381 			return true;
382 		}
383 
384 		if (check_target && PageBuddy(page)) {
385 			clear_pageblock_skip(page);
386 			return true;
387 		}
388 
389 		page += (1 << PAGE_ALLOC_COSTLY_ORDER);
390 	} while (page <= end_page);
391 
392 	return false;
393 }
394 
395 /*
396  * This function is called to clear all cached information on pageblocks that
397  * should be skipped for page isolation when the migrate and free page scanner
398  * meet.
399  */
400 static void __reset_isolation_suitable(struct zone *zone)
401 {
402 	unsigned long migrate_pfn = zone->zone_start_pfn;
403 	unsigned long free_pfn = zone_end_pfn(zone) - 1;
404 	unsigned long reset_migrate = free_pfn;
405 	unsigned long reset_free = migrate_pfn;
406 	bool source_set = false;
407 	bool free_set = false;
408 
409 	/* Only flush if a full compaction finished recently */
410 	if (!zone->compact_blockskip_flush)
411 		return;
412 
413 	zone->compact_blockskip_flush = false;
414 
415 	/*
416 	 * Walk the zone and update pageblock skip information. Source looks
417 	 * for PageLRU while target looks for PageBuddy. When the scanner
418 	 * is found, both PageBuddy and PageLRU are checked as the pageblock
419 	 * is suitable as both source and target.
420 	 */
421 	for (; migrate_pfn < free_pfn; migrate_pfn += pageblock_nr_pages,
422 					free_pfn -= pageblock_nr_pages) {
423 		cond_resched();
424 
425 		/* Update the migrate PFN */
426 		if (__reset_isolation_pfn(zone, migrate_pfn, true, source_set) &&
427 		    migrate_pfn < reset_migrate) {
428 			source_set = true;
429 			reset_migrate = migrate_pfn;
430 			zone->compact_init_migrate_pfn = reset_migrate;
431 			zone->compact_cached_migrate_pfn[0] = reset_migrate;
432 			zone->compact_cached_migrate_pfn[1] = reset_migrate;
433 		}
434 
435 		/* Update the free PFN */
436 		if (__reset_isolation_pfn(zone, free_pfn, free_set, true) &&
437 		    free_pfn > reset_free) {
438 			free_set = true;
439 			reset_free = free_pfn;
440 			zone->compact_init_free_pfn = reset_free;
441 			zone->compact_cached_free_pfn = reset_free;
442 		}
443 	}
444 
445 	/* Leave no distance if no suitable block was reset */
446 	if (reset_migrate >= reset_free) {
447 		zone->compact_cached_migrate_pfn[0] = migrate_pfn;
448 		zone->compact_cached_migrate_pfn[1] = migrate_pfn;
449 		zone->compact_cached_free_pfn = free_pfn;
450 	}
451 }
452 
453 void reset_isolation_suitable(pg_data_t *pgdat)
454 {
455 	int zoneid;
456 
457 	for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {
458 		struct zone *zone = &pgdat->node_zones[zoneid];
459 		if (!populated_zone(zone))
460 			continue;
461 
462 		__reset_isolation_suitable(zone);
463 	}
464 }
465 
466 /*
467  * Sets the pageblock skip bit if it was clear. Note that this is a hint as
468  * locks are not required for read/writers. Returns true if it was already set.
469  */
470 static bool test_and_set_skip(struct compact_control *cc, struct page *page)
471 {
472 	bool skip;
473 
474 	/* Do not update if skip hint is being ignored */
475 	if (cc->ignore_skip_hint)
476 		return false;
477 
478 	skip = get_pageblock_skip(page);
479 	if (!skip && !cc->no_set_skip_hint)
480 		set_pageblock_skip(page);
481 
482 	return skip;
483 }
484 
485 static void update_cached_migrate(struct compact_control *cc, unsigned long pfn)
486 {
487 	struct zone *zone = cc->zone;
488 
489 	/* Set for isolation rather than compaction */
490 	if (cc->no_set_skip_hint)
491 		return;
492 
493 	pfn = pageblock_end_pfn(pfn);
494 
495 	/* Update where async and sync compaction should restart */
496 	if (pfn > zone->compact_cached_migrate_pfn[0])
497 		zone->compact_cached_migrate_pfn[0] = pfn;
498 	if (cc->mode != MIGRATE_ASYNC &&
499 	    pfn > zone->compact_cached_migrate_pfn[1])
500 		zone->compact_cached_migrate_pfn[1] = pfn;
501 }
502 
503 /*
504  * If no pages were isolated then mark this pageblock to be skipped in the
505  * future. The information is later cleared by __reset_isolation_suitable().
506  */
507 static void update_pageblock_skip(struct compact_control *cc,
508 			struct page *page, unsigned long pfn)
509 {
510 	struct zone *zone = cc->zone;
511 
512 	if (cc->no_set_skip_hint)
513 		return;
514 
515 	set_pageblock_skip(page);
516 
517 	if (pfn < zone->compact_cached_free_pfn)
518 		zone->compact_cached_free_pfn = pfn;
519 }
520 #else
521 static inline bool isolation_suitable(struct compact_control *cc,
522 					struct page *page)
523 {
524 	return true;
525 }
526 
527 static inline bool pageblock_skip_persistent(struct page *page)
528 {
529 	return false;
530 }
531 
532 static inline void update_pageblock_skip(struct compact_control *cc,
533 			struct page *page, unsigned long pfn)
534 {
535 }
536 
537 static void update_cached_migrate(struct compact_control *cc, unsigned long pfn)
538 {
539 }
540 
541 static bool test_and_set_skip(struct compact_control *cc, struct page *page)
542 {
543 	return false;
544 }
545 #endif /* CONFIG_COMPACTION */
546 
547 /*
548  * Compaction requires the taking of some coarse locks that are potentially
549  * very heavily contended. For async compaction, trylock and record if the
550  * lock is contended. The lock will still be acquired but compaction will
551  * abort when the current block is finished regardless of success rate.
552  * Sync compaction acquires the lock.
553  *
554  * Always returns true which makes it easier to track lock state in callers.
555  */
556 static bool compact_lock_irqsave(spinlock_t *lock, unsigned long *flags,
557 						struct compact_control *cc)
558 	__acquires(lock)
559 {
560 	/* Track if the lock is contended in async mode */
561 	if (cc->mode == MIGRATE_ASYNC && !cc->contended) {
562 		if (spin_trylock_irqsave(lock, *flags))
563 			return true;
564 
565 		cc->contended = true;
566 	}
567 
568 	spin_lock_irqsave(lock, *flags);
569 	return true;
570 }
571 
572 /*
573  * Compaction requires the taking of some coarse locks that are potentially
574  * very heavily contended. The lock should be periodically unlocked to avoid
575  * having disabled IRQs for a long time, even when there is nobody waiting on
576  * the lock. It might also be that allowing the IRQs will result in
577  * need_resched() becoming true. If scheduling is needed, compaction schedules.
578  * Either compaction type will also abort if a fatal signal is pending.
579  * In either case if the lock was locked, it is dropped and not regained.
580  *
581  * Returns true if compaction should abort due to fatal signal pending.
582  * Returns false when compaction can continue.
583  */
584 static bool compact_unlock_should_abort(spinlock_t *lock,
585 		unsigned long flags, bool *locked, struct compact_control *cc)
586 {
587 	if (*locked) {
588 		spin_unlock_irqrestore(lock, flags);
589 		*locked = false;
590 	}
591 
592 	if (fatal_signal_pending(current)) {
593 		cc->contended = true;
594 		return true;
595 	}
596 
597 	cond_resched();
598 
599 	return false;
600 }
601 
602 /*
603  * Isolate free pages onto a private freelist. If @strict is true, will abort
604  * returning 0 on any invalid PFNs or non-free pages inside of the pageblock
605  * (even though it may still end up isolating some pages).
606  */
607 static unsigned long isolate_freepages_block(struct compact_control *cc,
608 				unsigned long *start_pfn,
609 				unsigned long end_pfn,
610 				struct list_head *freelist,
611 				unsigned int stride,
612 				bool strict)
613 {
614 	int nr_scanned = 0, total_isolated = 0;
615 	struct page *page;
616 	unsigned long flags = 0;
617 	bool locked = false;
618 	unsigned long blockpfn = *start_pfn;
619 	unsigned int order;
620 
621 	/* Strict mode is for isolation, speed is secondary */
622 	if (strict)
623 		stride = 1;
624 
625 	page = pfn_to_page(blockpfn);
626 
627 	/* Isolate free pages. */
628 	for (; blockpfn < end_pfn; blockpfn += stride, page += stride) {
629 		int isolated;
630 
631 		/*
632 		 * Periodically drop the lock (if held) regardless of its
633 		 * contention, to give chance to IRQs. Abort if fatal signal
634 		 * pending.
635 		 */
636 		if (!(blockpfn % COMPACT_CLUSTER_MAX)
637 		    && compact_unlock_should_abort(&cc->zone->lock, flags,
638 								&locked, cc))
639 			break;
640 
641 		nr_scanned++;
642 
643 		/*
644 		 * For compound pages such as THP and hugetlbfs, we can save
645 		 * potentially a lot of iterations if we skip them at once.
646 		 * The check is racy, but we can consider only valid values
647 		 * and the only danger is skipping too much.
648 		 */
649 		if (PageCompound(page)) {
650 			const unsigned int order = compound_order(page);
651 
652 			if (blockpfn + (1UL << order) <= end_pfn) {
653 				blockpfn += (1UL << order) - 1;
654 				page += (1UL << order) - 1;
655 				nr_scanned += (1UL << order) - 1;
656 			}
657 
658 			goto isolate_fail;
659 		}
660 
661 		if (!PageBuddy(page))
662 			goto isolate_fail;
663 
664 		/* If we already hold the lock, we can skip some rechecking. */
665 		if (!locked) {
666 			locked = compact_lock_irqsave(&cc->zone->lock,
667 								&flags, cc);
668 
669 			/* Recheck this is a buddy page under lock */
670 			if (!PageBuddy(page))
671 				goto isolate_fail;
672 		}
673 
674 		/* Found a free page, will break it into order-0 pages */
675 		order = buddy_order(page);
676 		isolated = __isolate_free_page(page, order);
677 		if (!isolated)
678 			break;
679 		set_page_private(page, order);
680 
681 		nr_scanned += isolated - 1;
682 		total_isolated += isolated;
683 		cc->nr_freepages += isolated;
684 		list_add_tail(&page->lru, &freelist[order]);
685 
686 		if (!strict && cc->nr_migratepages <= cc->nr_freepages) {
687 			blockpfn += isolated;
688 			break;
689 		}
690 		/* Advance to the end of split page */
691 		blockpfn += isolated - 1;
692 		page += isolated - 1;
693 		continue;
694 
695 isolate_fail:
696 		if (strict)
697 			break;
698 
699 	}
700 
701 	if (locked)
702 		spin_unlock_irqrestore(&cc->zone->lock, flags);
703 
704 	/*
705 	 * Be careful to not go outside of the pageblock.
706 	 */
707 	if (unlikely(blockpfn > end_pfn))
708 		blockpfn = end_pfn;
709 
710 	trace_mm_compaction_isolate_freepages(*start_pfn, blockpfn,
711 					nr_scanned, total_isolated);
712 
713 	/* Record how far we have got within the block */
714 	*start_pfn = blockpfn;
715 
716 	/*
717 	 * If strict isolation is requested by CMA then check that all the
718 	 * pages requested were isolated. If there were any failures, 0 is
719 	 * returned and CMA will fail.
720 	 */
721 	if (strict && blockpfn < end_pfn)
722 		total_isolated = 0;
723 
724 	cc->total_free_scanned += nr_scanned;
725 	if (total_isolated)
726 		count_compact_events(COMPACTISOLATED, total_isolated);
727 	return total_isolated;
728 }
729 
730 /**
731  * isolate_freepages_range() - isolate free pages.
732  * @cc:        Compaction control structure.
733  * @start_pfn: The first PFN to start isolating.
734  * @end_pfn:   The one-past-last PFN.
735  *
736  * Non-free pages, invalid PFNs, or zone boundaries within the
737  * [start_pfn, end_pfn) range are considered errors, cause function to
738  * undo its actions and return zero.
739  *
740  * Otherwise, function returns one-past-the-last PFN of isolated page
741  * (which may be greater then end_pfn if end fell in a middle of
742  * a free page).
743  */
744 unsigned long
745 isolate_freepages_range(struct compact_control *cc,
746 			unsigned long start_pfn, unsigned long end_pfn)
747 {
748 	unsigned long isolated, pfn, block_start_pfn, block_end_pfn;
749 	int order;
750 	struct list_head tmp_freepages[NR_PAGE_ORDERS];
751 
752 	for (order = 0; order < NR_PAGE_ORDERS; order++)
753 		INIT_LIST_HEAD(&tmp_freepages[order]);
754 
755 	pfn = start_pfn;
756 	block_start_pfn = pageblock_start_pfn(pfn);
757 	if (block_start_pfn < cc->zone->zone_start_pfn)
758 		block_start_pfn = cc->zone->zone_start_pfn;
759 	block_end_pfn = pageblock_end_pfn(pfn);
760 
761 	for (; pfn < end_pfn; pfn += isolated,
762 				block_start_pfn = block_end_pfn,
763 				block_end_pfn += pageblock_nr_pages) {
764 		/* Protect pfn from changing by isolate_freepages_block */
765 		unsigned long isolate_start_pfn = pfn;
766 
767 		/*
768 		 * pfn could pass the block_end_pfn if isolated freepage
769 		 * is more than pageblock order. In this case, we adjust
770 		 * scanning range to right one.
771 		 */
772 		if (pfn >= block_end_pfn) {
773 			block_start_pfn = pageblock_start_pfn(pfn);
774 			block_end_pfn = pageblock_end_pfn(pfn);
775 		}
776 
777 		block_end_pfn = min(block_end_pfn, end_pfn);
778 
779 		if (!pageblock_pfn_to_page(block_start_pfn,
780 					block_end_pfn, cc->zone))
781 			break;
782 
783 		isolated = isolate_freepages_block(cc, &isolate_start_pfn,
784 					block_end_pfn, tmp_freepages, 0, true);
785 
786 		/*
787 		 * In strict mode, isolate_freepages_block() returns 0 if
788 		 * there are any holes in the block (ie. invalid PFNs or
789 		 * non-free pages).
790 		 */
791 		if (!isolated)
792 			break;
793 
794 		/*
795 		 * If we managed to isolate pages, it is always (1 << n) *
796 		 * pageblock_nr_pages for some non-negative n.  (Max order
797 		 * page may span two pageblocks).
798 		 */
799 	}
800 
801 	if (pfn < end_pfn) {
802 		/* Loop terminated early, cleanup. */
803 		release_free_list(tmp_freepages);
804 		return 0;
805 	}
806 
807 	/* __isolate_free_page() does not map the pages */
808 	split_map_pages(tmp_freepages);
809 
810 	/* We don't use freelists for anything. */
811 	return pfn;
812 }
813 
814 /* Similar to reclaim, but different enough that they don't share logic */
815 static bool too_many_isolated(struct compact_control *cc)
816 {
817 	pg_data_t *pgdat = cc->zone->zone_pgdat;
818 	bool too_many;
819 
820 	unsigned long active, inactive, isolated;
821 
822 	inactive = node_page_state(pgdat, NR_INACTIVE_FILE) +
823 			node_page_state(pgdat, NR_INACTIVE_ANON);
824 	active = node_page_state(pgdat, NR_ACTIVE_FILE) +
825 			node_page_state(pgdat, NR_ACTIVE_ANON);
826 	isolated = node_page_state(pgdat, NR_ISOLATED_FILE) +
827 			node_page_state(pgdat, NR_ISOLATED_ANON);
828 
829 	/*
830 	 * Allow GFP_NOFS to isolate past the limit set for regular
831 	 * compaction runs. This prevents an ABBA deadlock when other
832 	 * compactors have already isolated to the limit, but are
833 	 * blocked on filesystem locks held by the GFP_NOFS thread.
834 	 */
835 	if (cc->gfp_mask & __GFP_FS) {
836 		inactive >>= 3;
837 		active >>= 3;
838 	}
839 
840 	too_many = isolated > (inactive + active) / 2;
841 	if (!too_many)
842 		wake_throttle_isolated(pgdat);
843 
844 	return too_many;
845 }
846 
847 /**
848  * skip_isolation_on_order() - determine when to skip folio isolation based on
849  *			       folio order and compaction target order
850  * @order:		to-be-isolated folio order
851  * @target_order:	compaction target order
852  *
853  * This avoids unnecessary folio isolations during compaction.
854  */
855 static bool skip_isolation_on_order(int order, int target_order)
856 {
857 	/*
858 	 * Unless we are performing global compaction (i.e.,
859 	 * is_via_compact_memory), skip any folios that are larger than the
860 	 * target order: we wouldn't be here if we'd have a free folio with
861 	 * the desired target_order, so migrating this folio would likely fail
862 	 * later.
863 	 */
864 	if (!is_via_compact_memory(target_order) && order >= target_order)
865 		return true;
866 	/*
867 	 * We limit memory compaction to pageblocks and won't try
868 	 * creating free blocks of memory that are larger than that.
869 	 */
870 	return order >= pageblock_order;
871 }
872 
873 /**
874  * isolate_migratepages_block() - isolate all migrate-able pages within
875  *				  a single pageblock
876  * @cc:		Compaction control structure.
877  * @low_pfn:	The first PFN to isolate
878  * @end_pfn:	The one-past-the-last PFN to isolate, within same pageblock
879  * @mode:	Isolation mode to be used.
880  *
881  * Isolate all pages that can be migrated from the range specified by
882  * [low_pfn, end_pfn). The range is expected to be within same pageblock.
883  * Returns errno, like -EAGAIN or -EINTR in case e.g signal pending or congestion,
884  * -ENOMEM in case we could not allocate a page, or 0.
885  * cc->migrate_pfn will contain the next pfn to scan.
886  *
887  * The pages are isolated on cc->migratepages list (not required to be empty),
888  * and cc->nr_migratepages is updated accordingly.
889  */
890 static int
891 isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
892 			unsigned long end_pfn, isolate_mode_t mode)
893 {
894 	pg_data_t *pgdat = cc->zone->zone_pgdat;
895 	unsigned long nr_scanned = 0, nr_isolated = 0;
896 	struct lruvec *lruvec;
897 	unsigned long flags = 0;
898 	struct lruvec *locked = NULL;
899 	struct folio *folio = NULL;
900 	struct page *page = NULL, *valid_page = NULL;
901 	struct address_space *mapping;
902 	unsigned long start_pfn = low_pfn;
903 	bool skip_on_failure = false;
904 	unsigned long next_skip_pfn = 0;
905 	bool skip_updated = false;
906 	int ret = 0;
907 
908 	cc->migrate_pfn = low_pfn;
909 
910 	/*
911 	 * Ensure that there are not too many pages isolated from the LRU
912 	 * list by either parallel reclaimers or compaction. If there are,
913 	 * delay for some time until fewer pages are isolated
914 	 */
915 	while (unlikely(too_many_isolated(cc))) {
916 		/* stop isolation if there are still pages not migrated */
917 		if (cc->nr_migratepages)
918 			return -EAGAIN;
919 
920 		/* async migration should just abort */
921 		if (cc->mode == MIGRATE_ASYNC)
922 			return -EAGAIN;
923 
924 		reclaim_throttle(pgdat, VMSCAN_THROTTLE_ISOLATED);
925 
926 		if (fatal_signal_pending(current))
927 			return -EINTR;
928 	}
929 
930 	cond_resched();
931 
932 	if (cc->direct_compaction && (cc->mode == MIGRATE_ASYNC)) {
933 		skip_on_failure = true;
934 		next_skip_pfn = block_end_pfn(low_pfn, cc->order);
935 	}
936 
937 	/* Time to isolate some pages for migration */
938 	for (; low_pfn < end_pfn; low_pfn++) {
939 		bool is_dirty, is_unevictable;
940 
941 		if (skip_on_failure && low_pfn >= next_skip_pfn) {
942 			/*
943 			 * We have isolated all migration candidates in the
944 			 * previous order-aligned block, and did not skip it due
945 			 * to failure. We should migrate the pages now and
946 			 * hopefully succeed compaction.
947 			 */
948 			if (nr_isolated)
949 				break;
950 
951 			/*
952 			 * We failed to isolate in the previous order-aligned
953 			 * block. Set the new boundary to the end of the
954 			 * current block. Note we can't simply increase
955 			 * next_skip_pfn by 1 << order, as low_pfn might have
956 			 * been incremented by a higher number due to skipping
957 			 * a compound or a high-order buddy page in the
958 			 * previous loop iteration.
959 			 */
960 			next_skip_pfn = block_end_pfn(low_pfn, cc->order);
961 		}
962 
963 		/*
964 		 * Periodically drop the lock (if held) regardless of its
965 		 * contention, to give chance to IRQs. Abort completely if
966 		 * a fatal signal is pending.
967 		 */
968 		if (!(low_pfn % COMPACT_CLUSTER_MAX)) {
969 			if (locked) {
970 				unlock_page_lruvec_irqrestore(locked, flags);
971 				locked = NULL;
972 			}
973 
974 			if (fatal_signal_pending(current)) {
975 				cc->contended = true;
976 				ret = -EINTR;
977 
978 				goto fatal_pending;
979 			}
980 
981 			cond_resched();
982 		}
983 
984 		nr_scanned++;
985 
986 		page = pfn_to_page(low_pfn);
987 
988 		/*
989 		 * Check if the pageblock has already been marked skipped.
990 		 * Only the first PFN is checked as the caller isolates
991 		 * COMPACT_CLUSTER_MAX at a time so the second call must
992 		 * not falsely conclude that the block should be skipped.
993 		 */
994 		if (!valid_page && (pageblock_aligned(low_pfn) ||
995 				    low_pfn == cc->zone->zone_start_pfn)) {
996 			if (!isolation_suitable(cc, page)) {
997 				low_pfn = end_pfn;
998 				folio = NULL;
999 				goto isolate_abort;
1000 			}
1001 			valid_page = page;
1002 		}
1003 
1004 		if (PageHuge(page)) {
1005 			/*
1006 			 * skip hugetlbfs if we are not compacting for pages
1007 			 * bigger than its order. THPs and other compound pages
1008 			 * are handled below.
1009 			 */
1010 			if (!cc->alloc_contig) {
1011 				const unsigned int order = compound_order(page);
1012 
1013 				if (order <= MAX_PAGE_ORDER) {
1014 					low_pfn += (1UL << order) - 1;
1015 					nr_scanned += (1UL << order) - 1;
1016 				}
1017 				goto isolate_fail;
1018 			}
1019 			/* for alloc_contig case */
1020 			if (locked) {
1021 				unlock_page_lruvec_irqrestore(locked, flags);
1022 				locked = NULL;
1023 			}
1024 
1025 			ret = isolate_or_dissolve_huge_page(page, &cc->migratepages);
1026 
1027 			/*
1028 			 * Fail isolation in case isolate_or_dissolve_huge_page()
1029 			 * reports an error. In case of -ENOMEM, abort right away.
1030 			 */
1031 			if (ret < 0) {
1032 				 /* Do not report -EBUSY down the chain */
1033 				if (ret == -EBUSY)
1034 					ret = 0;
1035 				low_pfn += compound_nr(page) - 1;
1036 				nr_scanned += compound_nr(page) - 1;
1037 				goto isolate_fail;
1038 			}
1039 
1040 			if (PageHuge(page)) {
1041 				/*
1042 				 * Hugepage was successfully isolated and placed
1043 				 * on the cc->migratepages list.
1044 				 */
1045 				folio = page_folio(page);
1046 				low_pfn += folio_nr_pages(folio) - 1;
1047 				goto isolate_success_no_list;
1048 			}
1049 
1050 			/*
1051 			 * Ok, the hugepage was dissolved. Now these pages are
1052 			 * Buddy and cannot be re-allocated because they are
1053 			 * isolated. Fall-through as the check below handles
1054 			 * Buddy pages.
1055 			 */
1056 		}
1057 
1058 		/*
1059 		 * Skip if free. We read page order here without zone lock
1060 		 * which is generally unsafe, but the race window is small and
1061 		 * the worst thing that can happen is that we skip some
1062 		 * potential isolation targets.
1063 		 */
1064 		if (PageBuddy(page)) {
1065 			unsigned long freepage_order = buddy_order_unsafe(page);
1066 
1067 			/*
1068 			 * Without lock, we cannot be sure that what we got is
1069 			 * a valid page order. Consider only values in the
1070 			 * valid order range to prevent low_pfn overflow.
1071 			 */
1072 			if (freepage_order > 0 && freepage_order <= MAX_PAGE_ORDER) {
1073 				low_pfn += (1UL << freepage_order) - 1;
1074 				nr_scanned += (1UL << freepage_order) - 1;
1075 			}
1076 			continue;
1077 		}
1078 
1079 		/*
1080 		 * Regardless of being on LRU, compound pages such as THP
1081 		 * (hugetlbfs is handled above) are not to be compacted unless
1082 		 * we are attempting an allocation larger than the compound
1083 		 * page size. We can potentially save a lot of iterations if we
1084 		 * skip them at once. The check is racy, but we can consider
1085 		 * only valid values and the only danger is skipping too much.
1086 		 */
1087 		if (PageCompound(page) && !cc->alloc_contig) {
1088 			const unsigned int order = compound_order(page);
1089 
1090 			/* Skip based on page order and compaction target order. */
1091 			if (skip_isolation_on_order(order, cc->order)) {
1092 				if (order <= MAX_PAGE_ORDER) {
1093 					low_pfn += (1UL << order) - 1;
1094 					nr_scanned += (1UL << order) - 1;
1095 				}
1096 				goto isolate_fail;
1097 			}
1098 		}
1099 
1100 		/*
1101 		 * Check may be lockless but that's ok as we recheck later.
1102 		 * It's possible to migrate LRU and non-lru movable pages.
1103 		 * Skip any other type of page
1104 		 */
1105 		if (!PageLRU(page)) {
1106 			/*
1107 			 * __PageMovable can return false positive so we need
1108 			 * to verify it under page_lock.
1109 			 */
1110 			if (unlikely(__PageMovable(page)) &&
1111 					!PageIsolated(page)) {
1112 				if (locked) {
1113 					unlock_page_lruvec_irqrestore(locked, flags);
1114 					locked = NULL;
1115 				}
1116 
1117 				if (isolate_movable_page(page, mode)) {
1118 					folio = page_folio(page);
1119 					goto isolate_success;
1120 				}
1121 			}
1122 
1123 			goto isolate_fail;
1124 		}
1125 
1126 		/*
1127 		 * Be careful not to clear PageLRU until after we're
1128 		 * sure the page is not being freed elsewhere -- the
1129 		 * page release code relies on it.
1130 		 */
1131 		folio = folio_get_nontail_page(page);
1132 		if (unlikely(!folio))
1133 			goto isolate_fail;
1134 
1135 		/*
1136 		 * Migration will fail if an anonymous page is pinned in memory,
1137 		 * so avoid taking lru_lock and isolating it unnecessarily in an
1138 		 * admittedly racy check.
1139 		 */
1140 		mapping = folio_mapping(folio);
1141 		if (!mapping && (folio_ref_count(folio) - 1) > folio_mapcount(folio))
1142 			goto isolate_fail_put;
1143 
1144 		/*
1145 		 * Only allow to migrate anonymous pages in GFP_NOFS context
1146 		 * because those do not depend on fs locks.
1147 		 */
1148 		if (!(cc->gfp_mask & __GFP_FS) && mapping)
1149 			goto isolate_fail_put;
1150 
1151 		/* Only take pages on LRU: a check now makes later tests safe */
1152 		if (!folio_test_lru(folio))
1153 			goto isolate_fail_put;
1154 
1155 		is_unevictable = folio_test_unevictable(folio);
1156 
1157 		/* Compaction might skip unevictable pages but CMA takes them */
1158 		if (!(mode & ISOLATE_UNEVICTABLE) && is_unevictable)
1159 			goto isolate_fail_put;
1160 
1161 		/*
1162 		 * To minimise LRU disruption, the caller can indicate with
1163 		 * ISOLATE_ASYNC_MIGRATE that it only wants to isolate pages
1164 		 * it will be able to migrate without blocking - clean pages
1165 		 * for the most part.  PageWriteback would require blocking.
1166 		 */
1167 		if ((mode & ISOLATE_ASYNC_MIGRATE) && folio_test_writeback(folio))
1168 			goto isolate_fail_put;
1169 
1170 		is_dirty = folio_test_dirty(folio);
1171 
1172 		if (((mode & ISOLATE_ASYNC_MIGRATE) && is_dirty) ||
1173 		    (mapping && is_unevictable)) {
1174 			bool migrate_dirty = true;
1175 			bool is_unmovable;
1176 
1177 			/*
1178 			 * Only folios without mappings or that have
1179 			 * a ->migrate_folio callback are possible to migrate
1180 			 * without blocking.
1181 			 *
1182 			 * Folios from unmovable mappings are not migratable.
1183 			 *
1184 			 * However, we can be racing with truncation, which can
1185 			 * free the mapping that we need to check. Truncation
1186 			 * holds the folio lock until after the folio is removed
1187 			 * from the page so holding it ourselves is sufficient.
1188 			 *
1189 			 * To avoid locking the folio just to check unmovable,
1190 			 * assume every unmovable folio is also unevictable,
1191 			 * which is a cheaper test.  If our assumption goes
1192 			 * wrong, it's not a correctness bug, just potentially
1193 			 * wasted cycles.
1194 			 */
1195 			if (!folio_trylock(folio))
1196 				goto isolate_fail_put;
1197 
1198 			mapping = folio_mapping(folio);
1199 			if ((mode & ISOLATE_ASYNC_MIGRATE) && is_dirty) {
1200 				migrate_dirty = !mapping ||
1201 						mapping->a_ops->migrate_folio;
1202 			}
1203 			is_unmovable = mapping && mapping_unmovable(mapping);
1204 			folio_unlock(folio);
1205 			if (!migrate_dirty || is_unmovable)
1206 				goto isolate_fail_put;
1207 		}
1208 
1209 		/* Try isolate the folio */
1210 		if (!folio_test_clear_lru(folio))
1211 			goto isolate_fail_put;
1212 
1213 		lruvec = folio_lruvec(folio);
1214 
1215 		/* If we already hold the lock, we can skip some rechecking */
1216 		if (lruvec != locked) {
1217 			if (locked)
1218 				unlock_page_lruvec_irqrestore(locked, flags);
1219 
1220 			compact_lock_irqsave(&lruvec->lru_lock, &flags, cc);
1221 			locked = lruvec;
1222 
1223 			lruvec_memcg_debug(lruvec, folio);
1224 
1225 			/*
1226 			 * Try get exclusive access under lock. If marked for
1227 			 * skip, the scan is aborted unless the current context
1228 			 * is a rescan to reach the end of the pageblock.
1229 			 */
1230 			if (!skip_updated && valid_page) {
1231 				skip_updated = true;
1232 				if (test_and_set_skip(cc, valid_page) &&
1233 				    !cc->finish_pageblock) {
1234 					low_pfn = end_pfn;
1235 					goto isolate_abort;
1236 				}
1237 			}
1238 
1239 			/*
1240 			 * Check LRU folio order under the lock
1241 			 */
1242 			if (unlikely(skip_isolation_on_order(folio_order(folio),
1243 							     cc->order) &&
1244 				     !cc->alloc_contig)) {
1245 				low_pfn += folio_nr_pages(folio) - 1;
1246 				nr_scanned += folio_nr_pages(folio) - 1;
1247 				folio_set_lru(folio);
1248 				goto isolate_fail_put;
1249 			}
1250 		}
1251 
1252 		/* The folio is taken off the LRU */
1253 		if (folio_test_large(folio))
1254 			low_pfn += folio_nr_pages(folio) - 1;
1255 
1256 		/* Successfully isolated */
1257 		lruvec_del_folio(lruvec, folio);
1258 		node_stat_mod_folio(folio,
1259 				NR_ISOLATED_ANON + folio_is_file_lru(folio),
1260 				folio_nr_pages(folio));
1261 
1262 isolate_success:
1263 		list_add(&folio->lru, &cc->migratepages);
1264 isolate_success_no_list:
1265 		cc->nr_migratepages += folio_nr_pages(folio);
1266 		nr_isolated += folio_nr_pages(folio);
1267 		nr_scanned += folio_nr_pages(folio) - 1;
1268 
1269 		/*
1270 		 * Avoid isolating too much unless this block is being
1271 		 * fully scanned (e.g. dirty/writeback pages, parallel allocation)
1272 		 * or a lock is contended. For contention, isolate quickly to
1273 		 * potentially remove one source of contention.
1274 		 */
1275 		if (cc->nr_migratepages >= COMPACT_CLUSTER_MAX &&
1276 		    !cc->finish_pageblock && !cc->contended) {
1277 			++low_pfn;
1278 			break;
1279 		}
1280 
1281 		continue;
1282 
1283 isolate_fail_put:
1284 		/* Avoid potential deadlock in freeing page under lru_lock */
1285 		if (locked) {
1286 			unlock_page_lruvec_irqrestore(locked, flags);
1287 			locked = NULL;
1288 		}
1289 		folio_put(folio);
1290 
1291 isolate_fail:
1292 		if (!skip_on_failure && ret != -ENOMEM)
1293 			continue;
1294 
1295 		/*
1296 		 * We have isolated some pages, but then failed. Release them
1297 		 * instead of migrating, as we cannot form the cc->order buddy
1298 		 * page anyway.
1299 		 */
1300 		if (nr_isolated) {
1301 			if (locked) {
1302 				unlock_page_lruvec_irqrestore(locked, flags);
1303 				locked = NULL;
1304 			}
1305 			putback_movable_pages(&cc->migratepages);
1306 			cc->nr_migratepages = 0;
1307 			nr_isolated = 0;
1308 		}
1309 
1310 		if (low_pfn < next_skip_pfn) {
1311 			low_pfn = next_skip_pfn - 1;
1312 			/*
1313 			 * The check near the loop beginning would have updated
1314 			 * next_skip_pfn too, but this is a bit simpler.
1315 			 */
1316 			next_skip_pfn += 1UL << cc->order;
1317 		}
1318 
1319 		if (ret == -ENOMEM)
1320 			break;
1321 	}
1322 
1323 	/*
1324 	 * The PageBuddy() check could have potentially brought us outside
1325 	 * the range to be scanned.
1326 	 */
1327 	if (unlikely(low_pfn > end_pfn))
1328 		low_pfn = end_pfn;
1329 
1330 	folio = NULL;
1331 
1332 isolate_abort:
1333 	if (locked)
1334 		unlock_page_lruvec_irqrestore(locked, flags);
1335 	if (folio) {
1336 		folio_set_lru(folio);
1337 		folio_put(folio);
1338 	}
1339 
1340 	/*
1341 	 * Update the cached scanner pfn once the pageblock has been scanned.
1342 	 * Pages will either be migrated in which case there is no point
1343 	 * scanning in the near future or migration failed in which case the
1344 	 * failure reason may persist. The block is marked for skipping if
1345 	 * there were no pages isolated in the block or if the block is
1346 	 * rescanned twice in a row.
1347 	 */
1348 	if (low_pfn == end_pfn && (!nr_isolated || cc->finish_pageblock)) {
1349 		if (!cc->no_set_skip_hint && valid_page && !skip_updated)
1350 			set_pageblock_skip(valid_page);
1351 		update_cached_migrate(cc, low_pfn);
1352 	}
1353 
1354 	trace_mm_compaction_isolate_migratepages(start_pfn, low_pfn,
1355 						nr_scanned, nr_isolated);
1356 
1357 fatal_pending:
1358 	cc->total_migrate_scanned += nr_scanned;
1359 	if (nr_isolated)
1360 		count_compact_events(COMPACTISOLATED, nr_isolated);
1361 
1362 	cc->migrate_pfn = low_pfn;
1363 
1364 	return ret;
1365 }
1366 
1367 /**
1368  * isolate_migratepages_range() - isolate migrate-able pages in a PFN range
1369  * @cc:        Compaction control structure.
1370  * @start_pfn: The first PFN to start isolating.
1371  * @end_pfn:   The one-past-last PFN.
1372  *
1373  * Returns -EAGAIN when contented, -EINTR in case of a signal pending, -ENOMEM
1374  * in case we could not allocate a page, or 0.
1375  */
1376 int
1377 isolate_migratepages_range(struct compact_control *cc, unsigned long start_pfn,
1378 							unsigned long end_pfn)
1379 {
1380 	unsigned long pfn, block_start_pfn, block_end_pfn;
1381 	int ret = 0;
1382 
1383 	/* Scan block by block. First and last block may be incomplete */
1384 	pfn = start_pfn;
1385 	block_start_pfn = pageblock_start_pfn(pfn);
1386 	if (block_start_pfn < cc->zone->zone_start_pfn)
1387 		block_start_pfn = cc->zone->zone_start_pfn;
1388 	block_end_pfn = pageblock_end_pfn(pfn);
1389 
1390 	for (; pfn < end_pfn; pfn = block_end_pfn,
1391 				block_start_pfn = block_end_pfn,
1392 				block_end_pfn += pageblock_nr_pages) {
1393 
1394 		block_end_pfn = min(block_end_pfn, end_pfn);
1395 
1396 		if (!pageblock_pfn_to_page(block_start_pfn,
1397 					block_end_pfn, cc->zone))
1398 			continue;
1399 
1400 		ret = isolate_migratepages_block(cc, pfn, block_end_pfn,
1401 						 ISOLATE_UNEVICTABLE);
1402 
1403 		if (ret)
1404 			break;
1405 
1406 		if (cc->nr_migratepages >= COMPACT_CLUSTER_MAX)
1407 			break;
1408 	}
1409 
1410 	return ret;
1411 }
1412 
1413 #endif /* CONFIG_COMPACTION || CONFIG_CMA */
1414 #ifdef CONFIG_COMPACTION
1415 
1416 static bool suitable_migration_source(struct compact_control *cc,
1417 							struct page *page)
1418 {
1419 	int block_mt;
1420 
1421 	if (pageblock_skip_persistent(page))
1422 		return false;
1423 
1424 	if ((cc->mode != MIGRATE_ASYNC) || !cc->direct_compaction)
1425 		return true;
1426 
1427 	block_mt = get_pageblock_migratetype(page);
1428 
1429 	if (cc->migratetype == MIGRATE_MOVABLE)
1430 		return is_migrate_movable(block_mt);
1431 	else
1432 		return block_mt == cc->migratetype;
1433 }
1434 
1435 /* Returns true if the page is within a block suitable for migration to */
1436 static bool suitable_migration_target(struct compact_control *cc,
1437 							struct page *page)
1438 {
1439 	/* If the page is a large free page, then disallow migration */
1440 	if (PageBuddy(page)) {
1441 		int order = cc->order > 0 ? cc->order : pageblock_order;
1442 
1443 		/*
1444 		 * We are checking page_order without zone->lock taken. But
1445 		 * the only small danger is that we skip a potentially suitable
1446 		 * pageblock, so it's not worth to check order for valid range.
1447 		 */
1448 		if (buddy_order_unsafe(page) >= order)
1449 			return false;
1450 	}
1451 
1452 	if (cc->ignore_block_suitable)
1453 		return true;
1454 
1455 	/* If the block is MIGRATE_MOVABLE or MIGRATE_CMA, allow migration */
1456 	if (is_migrate_movable(get_pageblock_migratetype(page)))
1457 		return true;
1458 
1459 	/* Otherwise skip the block */
1460 	return false;
1461 }
1462 
1463 static inline unsigned int
1464 freelist_scan_limit(struct compact_control *cc)
1465 {
1466 	unsigned short shift = BITS_PER_LONG - 1;
1467 
1468 	return (COMPACT_CLUSTER_MAX >> min(shift, cc->fast_search_fail)) + 1;
1469 }
1470 
1471 /*
1472  * Test whether the free scanner has reached the same or lower pageblock than
1473  * the migration scanner, and compaction should thus terminate.
1474  */
1475 static inline bool compact_scanners_met(struct compact_control *cc)
1476 {
1477 	return (cc->free_pfn >> pageblock_order)
1478 		<= (cc->migrate_pfn >> pageblock_order);
1479 }
1480 
1481 /*
1482  * Used when scanning for a suitable migration target which scans freelists
1483  * in reverse. Reorders the list such as the unscanned pages are scanned
1484  * first on the next iteration of the free scanner
1485  */
1486 static void
1487 move_freelist_head(struct list_head *freelist, struct page *freepage)
1488 {
1489 	LIST_HEAD(sublist);
1490 
1491 	if (!list_is_first(&freepage->buddy_list, freelist)) {
1492 		list_cut_before(&sublist, freelist, &freepage->buddy_list);
1493 		list_splice_tail(&sublist, freelist);
1494 	}
1495 }
1496 
1497 /*
1498  * Similar to move_freelist_head except used by the migration scanner
1499  * when scanning forward. It's possible for these list operations to
1500  * move against each other if they search the free list exactly in
1501  * lockstep.
1502  */
1503 static void
1504 move_freelist_tail(struct list_head *freelist, struct page *freepage)
1505 {
1506 	LIST_HEAD(sublist);
1507 
1508 	if (!list_is_last(&freepage->buddy_list, freelist)) {
1509 		list_cut_position(&sublist, freelist, &freepage->buddy_list);
1510 		list_splice_tail(&sublist, freelist);
1511 	}
1512 }
1513 
1514 static void
1515 fast_isolate_around(struct compact_control *cc, unsigned long pfn)
1516 {
1517 	unsigned long start_pfn, end_pfn;
1518 	struct page *page;
1519 
1520 	/* Do not search around if there are enough pages already */
1521 	if (cc->nr_freepages >= cc->nr_migratepages)
1522 		return;
1523 
1524 	/* Minimise scanning during async compaction */
1525 	if (cc->direct_compaction && cc->mode == MIGRATE_ASYNC)
1526 		return;
1527 
1528 	/* Pageblock boundaries */
1529 	start_pfn = max(pageblock_start_pfn(pfn), cc->zone->zone_start_pfn);
1530 	end_pfn = min(pageblock_end_pfn(pfn), zone_end_pfn(cc->zone));
1531 
1532 	page = pageblock_pfn_to_page(start_pfn, end_pfn, cc->zone);
1533 	if (!page)
1534 		return;
1535 
1536 	isolate_freepages_block(cc, &start_pfn, end_pfn, cc->freepages, 1, false);
1537 
1538 	/* Skip this pageblock in the future as it's full or nearly full */
1539 	if (start_pfn == end_pfn && !cc->no_set_skip_hint)
1540 		set_pageblock_skip(page);
1541 }
1542 
1543 /* Search orders in round-robin fashion */
1544 static int next_search_order(struct compact_control *cc, int order)
1545 {
1546 	order--;
1547 	if (order < 0)
1548 		order = cc->order - 1;
1549 
1550 	/* Search wrapped around? */
1551 	if (order == cc->search_order) {
1552 		cc->search_order--;
1553 		if (cc->search_order < 0)
1554 			cc->search_order = cc->order - 1;
1555 		return -1;
1556 	}
1557 
1558 	return order;
1559 }
1560 
1561 static void fast_isolate_freepages(struct compact_control *cc)
1562 {
1563 	unsigned int limit = max(1U, freelist_scan_limit(cc) >> 1);
1564 	unsigned int nr_scanned = 0, total_isolated = 0;
1565 	unsigned long low_pfn, min_pfn, highest = 0;
1566 	unsigned long nr_isolated = 0;
1567 	unsigned long distance;
1568 	struct page *page = NULL;
1569 	bool scan_start = false;
1570 	int order;
1571 
1572 	/* Full compaction passes in a negative order */
1573 	if (cc->order <= 0)
1574 		return;
1575 
1576 	/*
1577 	 * If starting the scan, use a deeper search and use the highest
1578 	 * PFN found if a suitable one is not found.
1579 	 */
1580 	if (cc->free_pfn >= cc->zone->compact_init_free_pfn) {
1581 		limit = pageblock_nr_pages >> 1;
1582 		scan_start = true;
1583 	}
1584 
1585 	/*
1586 	 * Preferred point is in the top quarter of the scan space but take
1587 	 * a pfn from the top half if the search is problematic.
1588 	 */
1589 	distance = (cc->free_pfn - cc->migrate_pfn);
1590 	low_pfn = pageblock_start_pfn(cc->free_pfn - (distance >> 2));
1591 	min_pfn = pageblock_start_pfn(cc->free_pfn - (distance >> 1));
1592 
1593 	if (WARN_ON_ONCE(min_pfn > low_pfn))
1594 		low_pfn = min_pfn;
1595 
1596 	/*
1597 	 * Search starts from the last successful isolation order or the next
1598 	 * order to search after a previous failure
1599 	 */
1600 	cc->search_order = min_t(unsigned int, cc->order - 1, cc->search_order);
1601 
1602 	for (order = cc->search_order;
1603 	     !page && order >= 0;
1604 	     order = next_search_order(cc, order)) {
1605 		struct free_area *area = &cc->zone->free_area[order];
1606 		struct list_head *freelist;
1607 		struct page *freepage;
1608 		unsigned long flags;
1609 		unsigned int order_scanned = 0;
1610 		unsigned long high_pfn = 0;
1611 
1612 		if (!area->nr_free)
1613 			continue;
1614 
1615 		spin_lock_irqsave(&cc->zone->lock, flags);
1616 		freelist = &area->free_list[MIGRATE_MOVABLE];
1617 		list_for_each_entry_reverse(freepage, freelist, buddy_list) {
1618 			unsigned long pfn;
1619 
1620 			order_scanned++;
1621 			nr_scanned++;
1622 			pfn = page_to_pfn(freepage);
1623 
1624 			if (pfn >= highest)
1625 				highest = max(pageblock_start_pfn(pfn),
1626 					      cc->zone->zone_start_pfn);
1627 
1628 			if (pfn >= low_pfn) {
1629 				cc->fast_search_fail = 0;
1630 				cc->search_order = order;
1631 				page = freepage;
1632 				break;
1633 			}
1634 
1635 			if (pfn >= min_pfn && pfn > high_pfn) {
1636 				high_pfn = pfn;
1637 
1638 				/* Shorten the scan if a candidate is found */
1639 				limit >>= 1;
1640 			}
1641 
1642 			if (order_scanned >= limit)
1643 				break;
1644 		}
1645 
1646 		/* Use a maximum candidate pfn if a preferred one was not found */
1647 		if (!page && high_pfn) {
1648 			page = pfn_to_page(high_pfn);
1649 
1650 			/* Update freepage for the list reorder below */
1651 			freepage = page;
1652 		}
1653 
1654 		/* Reorder to so a future search skips recent pages */
1655 		move_freelist_head(freelist, freepage);
1656 
1657 		/* Isolate the page if available */
1658 		if (page) {
1659 			if (__isolate_free_page(page, order)) {
1660 				set_page_private(page, order);
1661 				nr_isolated = 1 << order;
1662 				nr_scanned += nr_isolated - 1;
1663 				total_isolated += nr_isolated;
1664 				cc->nr_freepages += nr_isolated;
1665 				list_add_tail(&page->lru, &cc->freepages[order]);
1666 				count_compact_events(COMPACTISOLATED, nr_isolated);
1667 			} else {
1668 				/* If isolation fails, abort the search */
1669 				order = cc->search_order + 1;
1670 				page = NULL;
1671 			}
1672 		}
1673 
1674 		spin_unlock_irqrestore(&cc->zone->lock, flags);
1675 
1676 		/* Skip fast search if enough freepages isolated */
1677 		if (cc->nr_freepages >= cc->nr_migratepages)
1678 			break;
1679 
1680 		/*
1681 		 * Smaller scan on next order so the total scan is related
1682 		 * to freelist_scan_limit.
1683 		 */
1684 		if (order_scanned >= limit)
1685 			limit = max(1U, limit >> 1);
1686 	}
1687 
1688 	trace_mm_compaction_fast_isolate_freepages(min_pfn, cc->free_pfn,
1689 						   nr_scanned, total_isolated);
1690 
1691 	if (!page) {
1692 		cc->fast_search_fail++;
1693 		if (scan_start) {
1694 			/*
1695 			 * Use the highest PFN found above min. If one was
1696 			 * not found, be pessimistic for direct compaction
1697 			 * and use the min mark.
1698 			 */
1699 			if (highest >= min_pfn) {
1700 				page = pfn_to_page(highest);
1701 				cc->free_pfn = highest;
1702 			} else {
1703 				if (cc->direct_compaction && pfn_valid(min_pfn)) {
1704 					page = pageblock_pfn_to_page(min_pfn,
1705 						min(pageblock_end_pfn(min_pfn),
1706 						    zone_end_pfn(cc->zone)),
1707 						cc->zone);
1708 					if (page && !suitable_migration_target(cc, page))
1709 						page = NULL;
1710 
1711 					cc->free_pfn = min_pfn;
1712 				}
1713 			}
1714 		}
1715 	}
1716 
1717 	if (highest && highest >= cc->zone->compact_cached_free_pfn) {
1718 		highest -= pageblock_nr_pages;
1719 		cc->zone->compact_cached_free_pfn = highest;
1720 	}
1721 
1722 	cc->total_free_scanned += nr_scanned;
1723 	if (!page)
1724 		return;
1725 
1726 	low_pfn = page_to_pfn(page);
1727 	fast_isolate_around(cc, low_pfn);
1728 }
1729 
1730 /*
1731  * Based on information in the current compact_control, find blocks
1732  * suitable for isolating free pages from and then isolate them.
1733  */
1734 static void isolate_freepages(struct compact_control *cc)
1735 {
1736 	struct zone *zone = cc->zone;
1737 	struct page *page;
1738 	unsigned long block_start_pfn;	/* start of current pageblock */
1739 	unsigned long isolate_start_pfn; /* exact pfn we start at */
1740 	unsigned long block_end_pfn;	/* end of current pageblock */
1741 	unsigned long low_pfn;	     /* lowest pfn scanner is able to scan */
1742 	unsigned int stride;
1743 
1744 	/* Try a small search of the free lists for a candidate */
1745 	fast_isolate_freepages(cc);
1746 	if (cc->nr_freepages)
1747 		return;
1748 
1749 	/*
1750 	 * Initialise the free scanner. The starting point is where we last
1751 	 * successfully isolated from, zone-cached value, or the end of the
1752 	 * zone when isolating for the first time. For looping we also need
1753 	 * this pfn aligned down to the pageblock boundary, because we do
1754 	 * block_start_pfn -= pageblock_nr_pages in the for loop.
1755 	 * For ending point, take care when isolating in last pageblock of a
1756 	 * zone which ends in the middle of a pageblock.
1757 	 * The low boundary is the end of the pageblock the migration scanner
1758 	 * is using.
1759 	 */
1760 	isolate_start_pfn = cc->free_pfn;
1761 	block_start_pfn = pageblock_start_pfn(isolate_start_pfn);
1762 	block_end_pfn = min(block_start_pfn + pageblock_nr_pages,
1763 						zone_end_pfn(zone));
1764 	low_pfn = pageblock_end_pfn(cc->migrate_pfn);
1765 	stride = cc->mode == MIGRATE_ASYNC ? COMPACT_CLUSTER_MAX : 1;
1766 
1767 	/*
1768 	 * Isolate free pages until enough are available to migrate the
1769 	 * pages on cc->migratepages. We stop searching if the migrate
1770 	 * and free page scanners meet or enough free pages are isolated.
1771 	 */
1772 	for (; block_start_pfn >= low_pfn;
1773 				block_end_pfn = block_start_pfn,
1774 				block_start_pfn -= pageblock_nr_pages,
1775 				isolate_start_pfn = block_start_pfn) {
1776 		unsigned long nr_isolated;
1777 
1778 		/*
1779 		 * This can iterate a massively long zone without finding any
1780 		 * suitable migration targets, so periodically check resched.
1781 		 */
1782 		if (!(block_start_pfn % (COMPACT_CLUSTER_MAX * pageblock_nr_pages)))
1783 			cond_resched();
1784 
1785 		page = pageblock_pfn_to_page(block_start_pfn, block_end_pfn,
1786 									zone);
1787 		if (!page) {
1788 			unsigned long next_pfn;
1789 
1790 			next_pfn = skip_offline_sections_reverse(block_start_pfn);
1791 			if (next_pfn)
1792 				block_start_pfn = max(next_pfn, low_pfn);
1793 
1794 			continue;
1795 		}
1796 
1797 		/* Check the block is suitable for migration */
1798 		if (!suitable_migration_target(cc, page))
1799 			continue;
1800 
1801 		/* If isolation recently failed, do not retry */
1802 		if (!isolation_suitable(cc, page))
1803 			continue;
1804 
1805 		/* Found a block suitable for isolating free pages from. */
1806 		nr_isolated = isolate_freepages_block(cc, &isolate_start_pfn,
1807 					block_end_pfn, cc->freepages, stride, false);
1808 
1809 		/* Update the skip hint if the full pageblock was scanned */
1810 		if (isolate_start_pfn == block_end_pfn)
1811 			update_pageblock_skip(cc, page, block_start_pfn -
1812 					      pageblock_nr_pages);
1813 
1814 		/* Are enough freepages isolated? */
1815 		if (cc->nr_freepages >= cc->nr_migratepages) {
1816 			if (isolate_start_pfn >= block_end_pfn) {
1817 				/*
1818 				 * Restart at previous pageblock if more
1819 				 * freepages can be isolated next time.
1820 				 */
1821 				isolate_start_pfn =
1822 					block_start_pfn - pageblock_nr_pages;
1823 			}
1824 			break;
1825 		} else if (isolate_start_pfn < block_end_pfn) {
1826 			/*
1827 			 * If isolation failed early, do not continue
1828 			 * needlessly.
1829 			 */
1830 			break;
1831 		}
1832 
1833 		/* Adjust stride depending on isolation */
1834 		if (nr_isolated) {
1835 			stride = 1;
1836 			continue;
1837 		}
1838 		stride = min_t(unsigned int, COMPACT_CLUSTER_MAX, stride << 1);
1839 	}
1840 
1841 	/*
1842 	 * Record where the free scanner will restart next time. Either we
1843 	 * broke from the loop and set isolate_start_pfn based on the last
1844 	 * call to isolate_freepages_block(), or we met the migration scanner
1845 	 * and the loop terminated due to isolate_start_pfn < low_pfn
1846 	 */
1847 	cc->free_pfn = isolate_start_pfn;
1848 }
1849 
1850 /*
1851  * This is a migrate-callback that "allocates" freepages by taking pages
1852  * from the isolated freelists in the block we are migrating to.
1853  */
1854 static struct folio *compaction_alloc_noprof(struct folio *src, unsigned long data)
1855 {
1856 	struct compact_control *cc = (struct compact_control *)data;
1857 	struct folio *dst;
1858 	int order = folio_order(src);
1859 	bool has_isolated_pages = false;
1860 	int start_order;
1861 	struct page *freepage;
1862 	unsigned long size;
1863 
1864 again:
1865 	for (start_order = order; start_order < NR_PAGE_ORDERS; start_order++)
1866 		if (!list_empty(&cc->freepages[start_order]))
1867 			break;
1868 
1869 	/* no free pages in the list */
1870 	if (start_order == NR_PAGE_ORDERS) {
1871 		if (has_isolated_pages)
1872 			return NULL;
1873 		isolate_freepages(cc);
1874 		has_isolated_pages = true;
1875 		goto again;
1876 	}
1877 
1878 	freepage = list_first_entry(&cc->freepages[start_order], struct page,
1879 				lru);
1880 	size = 1 << start_order;
1881 
1882 	list_del(&freepage->lru);
1883 
1884 	while (start_order > order) {
1885 		start_order--;
1886 		size >>= 1;
1887 
1888 		list_add(&freepage[size].lru, &cc->freepages[start_order]);
1889 		set_page_private(&freepage[size], start_order);
1890 	}
1891 	dst = (struct folio *)freepage;
1892 
1893 	post_alloc_hook(&dst->page, order, __GFP_MOVABLE);
1894 	if (order)
1895 		prep_compound_page(&dst->page, order);
1896 	cc->nr_freepages -= 1 << order;
1897 	cc->nr_migratepages -= 1 << order;
1898 	return page_rmappable_folio(&dst->page);
1899 }
1900 
1901 static struct folio *compaction_alloc(struct folio *src, unsigned long data)
1902 {
1903 	return alloc_hooks(compaction_alloc_noprof(src, data));
1904 }
1905 
1906 /*
1907  * This is a migrate-callback that "frees" freepages back to the isolated
1908  * freelist.  All pages on the freelist are from the same zone, so there is no
1909  * special handling needed for NUMA.
1910  */
1911 static void compaction_free(struct folio *dst, unsigned long data)
1912 {
1913 	struct compact_control *cc = (struct compact_control *)data;
1914 	int order = folio_order(dst);
1915 	struct page *page = &dst->page;
1916 
1917 	if (folio_put_testzero(dst)) {
1918 		free_pages_prepare(page, order);
1919 		list_add(&dst->lru, &cc->freepages[order]);
1920 		cc->nr_freepages += 1 << order;
1921 	}
1922 	cc->nr_migratepages += 1 << order;
1923 	/*
1924 	 * someone else has referenced the page, we cannot take it back to our
1925 	 * free list.
1926 	 */
1927 }
1928 
1929 /* possible outcome of isolate_migratepages */
1930 typedef enum {
1931 	ISOLATE_ABORT,		/* Abort compaction now */
1932 	ISOLATE_NONE,		/* No pages isolated, continue scanning */
1933 	ISOLATE_SUCCESS,	/* Pages isolated, migrate */
1934 } isolate_migrate_t;
1935 
1936 /*
1937  * Allow userspace to control policy on scanning the unevictable LRU for
1938  * compactable pages.
1939  */
1940 static int sysctl_compact_unevictable_allowed __read_mostly = CONFIG_COMPACT_UNEVICTABLE_DEFAULT;
1941 /*
1942  * Tunable for proactive compaction. It determines how
1943  * aggressively the kernel should compact memory in the
1944  * background. It takes values in the range [0, 100].
1945  */
1946 static unsigned int __read_mostly sysctl_compaction_proactiveness = 20;
1947 static int sysctl_extfrag_threshold = 500;
1948 static int __read_mostly sysctl_compact_memory;
1949 
1950 static inline void
1951 update_fast_start_pfn(struct compact_control *cc, unsigned long pfn)
1952 {
1953 	if (cc->fast_start_pfn == ULONG_MAX)
1954 		return;
1955 
1956 	if (!cc->fast_start_pfn)
1957 		cc->fast_start_pfn = pfn;
1958 
1959 	cc->fast_start_pfn = min(cc->fast_start_pfn, pfn);
1960 }
1961 
1962 static inline unsigned long
1963 reinit_migrate_pfn(struct compact_control *cc)
1964 {
1965 	if (!cc->fast_start_pfn || cc->fast_start_pfn == ULONG_MAX)
1966 		return cc->migrate_pfn;
1967 
1968 	cc->migrate_pfn = cc->fast_start_pfn;
1969 	cc->fast_start_pfn = ULONG_MAX;
1970 
1971 	return cc->migrate_pfn;
1972 }
1973 
1974 /*
1975  * Briefly search the free lists for a migration source that already has
1976  * some free pages to reduce the number of pages that need migration
1977  * before a pageblock is free.
1978  */
1979 static unsigned long fast_find_migrateblock(struct compact_control *cc)
1980 {
1981 	unsigned int limit = freelist_scan_limit(cc);
1982 	unsigned int nr_scanned = 0;
1983 	unsigned long distance;
1984 	unsigned long pfn = cc->migrate_pfn;
1985 	unsigned long high_pfn;
1986 	int order;
1987 	bool found_block = false;
1988 
1989 	/* Skip hints are relied on to avoid repeats on the fast search */
1990 	if (cc->ignore_skip_hint)
1991 		return pfn;
1992 
1993 	/*
1994 	 * If the pageblock should be finished then do not select a different
1995 	 * pageblock.
1996 	 */
1997 	if (cc->finish_pageblock)
1998 		return pfn;
1999 
2000 	/*
2001 	 * If the migrate_pfn is not at the start of a zone or the start
2002 	 * of a pageblock then assume this is a continuation of a previous
2003 	 * scan restarted due to COMPACT_CLUSTER_MAX.
2004 	 */
2005 	if (pfn != cc->zone->zone_start_pfn && pfn != pageblock_start_pfn(pfn))
2006 		return pfn;
2007 
2008 	/*
2009 	 * For smaller orders, just linearly scan as the number of pages
2010 	 * to migrate should be relatively small and does not necessarily
2011 	 * justify freeing up a large block for a small allocation.
2012 	 */
2013 	if (cc->order <= PAGE_ALLOC_COSTLY_ORDER)
2014 		return pfn;
2015 
2016 	/*
2017 	 * Only allow kcompactd and direct requests for movable pages to
2018 	 * quickly clear out a MOVABLE pageblock for allocation. This
2019 	 * reduces the risk that a large movable pageblock is freed for
2020 	 * an unmovable/reclaimable small allocation.
2021 	 */
2022 	if (cc->direct_compaction && cc->migratetype != MIGRATE_MOVABLE)
2023 		return pfn;
2024 
2025 	/*
2026 	 * When starting the migration scanner, pick any pageblock within the
2027 	 * first half of the search space. Otherwise try and pick a pageblock
2028 	 * within the first eighth to reduce the chances that a migration
2029 	 * target later becomes a source.
2030 	 */
2031 	distance = (cc->free_pfn - cc->migrate_pfn) >> 1;
2032 	if (cc->migrate_pfn != cc->zone->zone_start_pfn)
2033 		distance >>= 2;
2034 	high_pfn = pageblock_start_pfn(cc->migrate_pfn + distance);
2035 
2036 	for (order = cc->order - 1;
2037 	     order >= PAGE_ALLOC_COSTLY_ORDER && !found_block && nr_scanned < limit;
2038 	     order--) {
2039 		struct free_area *area = &cc->zone->free_area[order];
2040 		struct list_head *freelist;
2041 		unsigned long flags;
2042 		struct page *freepage;
2043 
2044 		if (!area->nr_free)
2045 			continue;
2046 
2047 		spin_lock_irqsave(&cc->zone->lock, flags);
2048 		freelist = &area->free_list[MIGRATE_MOVABLE];
2049 		list_for_each_entry(freepage, freelist, buddy_list) {
2050 			unsigned long free_pfn;
2051 
2052 			if (nr_scanned++ >= limit) {
2053 				move_freelist_tail(freelist, freepage);
2054 				break;
2055 			}
2056 
2057 			free_pfn = page_to_pfn(freepage);
2058 			if (free_pfn < high_pfn) {
2059 				/*
2060 				 * Avoid if skipped recently. Ideally it would
2061 				 * move to the tail but even safe iteration of
2062 				 * the list assumes an entry is deleted, not
2063 				 * reordered.
2064 				 */
2065 				if (get_pageblock_skip(freepage))
2066 					continue;
2067 
2068 				/* Reorder to so a future search skips recent pages */
2069 				move_freelist_tail(freelist, freepage);
2070 
2071 				update_fast_start_pfn(cc, free_pfn);
2072 				pfn = pageblock_start_pfn(free_pfn);
2073 				if (pfn < cc->zone->zone_start_pfn)
2074 					pfn = cc->zone->zone_start_pfn;
2075 				cc->fast_search_fail = 0;
2076 				found_block = true;
2077 				break;
2078 			}
2079 		}
2080 		spin_unlock_irqrestore(&cc->zone->lock, flags);
2081 	}
2082 
2083 	cc->total_migrate_scanned += nr_scanned;
2084 
2085 	/*
2086 	 * If fast scanning failed then use a cached entry for a page block
2087 	 * that had free pages as the basis for starting a linear scan.
2088 	 */
2089 	if (!found_block) {
2090 		cc->fast_search_fail++;
2091 		pfn = reinit_migrate_pfn(cc);
2092 	}
2093 	return pfn;
2094 }
2095 
2096 /*
2097  * Isolate all pages that can be migrated from the first suitable block,
2098  * starting at the block pointed to by the migrate scanner pfn within
2099  * compact_control.
2100  */
2101 static isolate_migrate_t isolate_migratepages(struct compact_control *cc)
2102 {
2103 	unsigned long block_start_pfn;
2104 	unsigned long block_end_pfn;
2105 	unsigned long low_pfn;
2106 	struct page *page;
2107 	const isolate_mode_t isolate_mode =
2108 		(sysctl_compact_unevictable_allowed ? ISOLATE_UNEVICTABLE : 0) |
2109 		(cc->mode != MIGRATE_SYNC ? ISOLATE_ASYNC_MIGRATE : 0);
2110 	bool fast_find_block;
2111 
2112 	/*
2113 	 * Start at where we last stopped, or beginning of the zone as
2114 	 * initialized by compact_zone(). The first failure will use
2115 	 * the lowest PFN as the starting point for linear scanning.
2116 	 */
2117 	low_pfn = fast_find_migrateblock(cc);
2118 	block_start_pfn = pageblock_start_pfn(low_pfn);
2119 	if (block_start_pfn < cc->zone->zone_start_pfn)
2120 		block_start_pfn = cc->zone->zone_start_pfn;
2121 
2122 	/*
2123 	 * fast_find_migrateblock() has already ensured the pageblock is not
2124 	 * set with a skipped flag, so to avoid the isolation_suitable check
2125 	 * below again, check whether the fast search was successful.
2126 	 */
2127 	fast_find_block = low_pfn != cc->migrate_pfn && !cc->fast_search_fail;
2128 
2129 	/* Only scan within a pageblock boundary */
2130 	block_end_pfn = pageblock_end_pfn(low_pfn);
2131 
2132 	/*
2133 	 * Iterate over whole pageblocks until we find the first suitable.
2134 	 * Do not cross the free scanner.
2135 	 */
2136 	for (; block_end_pfn <= cc->free_pfn;
2137 			fast_find_block = false,
2138 			cc->migrate_pfn = low_pfn = block_end_pfn,
2139 			block_start_pfn = block_end_pfn,
2140 			block_end_pfn += pageblock_nr_pages) {
2141 
2142 		/*
2143 		 * This can potentially iterate a massively long zone with
2144 		 * many pageblocks unsuitable, so periodically check if we
2145 		 * need to schedule.
2146 		 */
2147 		if (!(low_pfn % (COMPACT_CLUSTER_MAX * pageblock_nr_pages)))
2148 			cond_resched();
2149 
2150 		page = pageblock_pfn_to_page(block_start_pfn,
2151 						block_end_pfn, cc->zone);
2152 		if (!page) {
2153 			unsigned long next_pfn;
2154 
2155 			next_pfn = skip_offline_sections(block_start_pfn);
2156 			if (next_pfn)
2157 				block_end_pfn = min(next_pfn, cc->free_pfn);
2158 			continue;
2159 		}
2160 
2161 		/*
2162 		 * If isolation recently failed, do not retry. Only check the
2163 		 * pageblock once. COMPACT_CLUSTER_MAX causes a pageblock
2164 		 * to be visited multiple times. Assume skip was checked
2165 		 * before making it "skip" so other compaction instances do
2166 		 * not scan the same block.
2167 		 */
2168 		if ((pageblock_aligned(low_pfn) ||
2169 		     low_pfn == cc->zone->zone_start_pfn) &&
2170 		    !fast_find_block && !isolation_suitable(cc, page))
2171 			continue;
2172 
2173 		/*
2174 		 * For async direct compaction, only scan the pageblocks of the
2175 		 * same migratetype without huge pages. Async direct compaction
2176 		 * is optimistic to see if the minimum amount of work satisfies
2177 		 * the allocation. The cached PFN is updated as it's possible
2178 		 * that all remaining blocks between source and target are
2179 		 * unsuitable and the compaction scanners fail to meet.
2180 		 */
2181 		if (!suitable_migration_source(cc, page)) {
2182 			update_cached_migrate(cc, block_end_pfn);
2183 			continue;
2184 		}
2185 
2186 		/* Perform the isolation */
2187 		if (isolate_migratepages_block(cc, low_pfn, block_end_pfn,
2188 						isolate_mode))
2189 			return ISOLATE_ABORT;
2190 
2191 		/*
2192 		 * Either we isolated something and proceed with migration. Or
2193 		 * we failed and compact_zone should decide if we should
2194 		 * continue or not.
2195 		 */
2196 		break;
2197 	}
2198 
2199 	return cc->nr_migratepages ? ISOLATE_SUCCESS : ISOLATE_NONE;
2200 }
2201 
2202 /*
2203  * Determine whether kswapd is (or recently was!) running on this node.
2204  *
2205  * pgdat_kswapd_lock() pins pgdat->kswapd, so a concurrent kswapd_stop() can't
2206  * zero it.
2207  */
2208 static bool kswapd_is_running(pg_data_t *pgdat)
2209 {
2210 	bool running;
2211 
2212 	pgdat_kswapd_lock(pgdat);
2213 	running = pgdat->kswapd && task_is_running(pgdat->kswapd);
2214 	pgdat_kswapd_unlock(pgdat);
2215 
2216 	return running;
2217 }
2218 
2219 /*
2220  * A zone's fragmentation score is the external fragmentation wrt to the
2221  * COMPACTION_HPAGE_ORDER. It returns a value in the range [0, 100].
2222  */
2223 static unsigned int fragmentation_score_zone(struct zone *zone)
2224 {
2225 	return extfrag_for_order(zone, COMPACTION_HPAGE_ORDER);
2226 }
2227 
2228 /*
2229  * A weighted zone's fragmentation score is the external fragmentation
2230  * wrt to the COMPACTION_HPAGE_ORDER scaled by the zone's size. It
2231  * returns a value in the range [0, 100].
2232  *
2233  * The scaling factor ensures that proactive compaction focuses on larger
2234  * zones like ZONE_NORMAL, rather than smaller, specialized zones like
2235  * ZONE_DMA32. For smaller zones, the score value remains close to zero,
2236  * and thus never exceeds the high threshold for proactive compaction.
2237  */
2238 static unsigned int fragmentation_score_zone_weighted(struct zone *zone)
2239 {
2240 	unsigned long score;
2241 
2242 	score = zone->present_pages * fragmentation_score_zone(zone);
2243 	return div64_ul(score, zone->zone_pgdat->node_present_pages + 1);
2244 }
2245 
2246 /*
2247  * The per-node proactive (background) compaction process is started by its
2248  * corresponding kcompactd thread when the node's fragmentation score
2249  * exceeds the high threshold. The compaction process remains active till
2250  * the node's score falls below the low threshold, or one of the back-off
2251  * conditions is met.
2252  */
2253 static unsigned int fragmentation_score_node(pg_data_t *pgdat)
2254 {
2255 	unsigned int score = 0;
2256 	int zoneid;
2257 
2258 	for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {
2259 		struct zone *zone;
2260 
2261 		zone = &pgdat->node_zones[zoneid];
2262 		if (!populated_zone(zone))
2263 			continue;
2264 		score += fragmentation_score_zone_weighted(zone);
2265 	}
2266 
2267 	return score;
2268 }
2269 
2270 static unsigned int fragmentation_score_wmark(bool low)
2271 {
2272 	unsigned int wmark_low;
2273 
2274 	/*
2275 	 * Cap the low watermark to avoid excessive compaction
2276 	 * activity in case a user sets the proactiveness tunable
2277 	 * close to 100 (maximum).
2278 	 */
2279 	wmark_low = max(100U - sysctl_compaction_proactiveness, 5U);
2280 	return low ? wmark_low : min(wmark_low + 10, 100U);
2281 }
2282 
2283 static bool should_proactive_compact_node(pg_data_t *pgdat)
2284 {
2285 	int wmark_high;
2286 
2287 	if (!sysctl_compaction_proactiveness || kswapd_is_running(pgdat))
2288 		return false;
2289 
2290 	wmark_high = fragmentation_score_wmark(false);
2291 	return fragmentation_score_node(pgdat) > wmark_high;
2292 }
2293 
2294 static enum compact_result __compact_finished(struct compact_control *cc)
2295 {
2296 	unsigned int order;
2297 	const int migratetype = cc->migratetype;
2298 	int ret;
2299 
2300 	/* Compaction run completes if the migrate and free scanner meet */
2301 	if (compact_scanners_met(cc)) {
2302 		/* Let the next compaction start anew. */
2303 		reset_cached_positions(cc->zone);
2304 
2305 		/*
2306 		 * Mark that the PG_migrate_skip information should be cleared
2307 		 * by kswapd when it goes to sleep. kcompactd does not set the
2308 		 * flag itself as the decision to be clear should be directly
2309 		 * based on an allocation request.
2310 		 */
2311 		if (cc->direct_compaction)
2312 			cc->zone->compact_blockskip_flush = true;
2313 
2314 		if (cc->whole_zone)
2315 			return COMPACT_COMPLETE;
2316 		else
2317 			return COMPACT_PARTIAL_SKIPPED;
2318 	}
2319 
2320 	if (cc->proactive_compaction) {
2321 		int score, wmark_low;
2322 		pg_data_t *pgdat;
2323 
2324 		pgdat = cc->zone->zone_pgdat;
2325 		if (kswapd_is_running(pgdat))
2326 			return COMPACT_PARTIAL_SKIPPED;
2327 
2328 		score = fragmentation_score_zone(cc->zone);
2329 		wmark_low = fragmentation_score_wmark(true);
2330 
2331 		if (score > wmark_low)
2332 			ret = COMPACT_CONTINUE;
2333 		else
2334 			ret = COMPACT_SUCCESS;
2335 
2336 		goto out;
2337 	}
2338 
2339 	if (is_via_compact_memory(cc->order))
2340 		return COMPACT_CONTINUE;
2341 
2342 	/*
2343 	 * Always finish scanning a pageblock to reduce the possibility of
2344 	 * fallbacks in the future. This is particularly important when
2345 	 * migration source is unmovable/reclaimable but it's not worth
2346 	 * special casing.
2347 	 */
2348 	if (!pageblock_aligned(cc->migrate_pfn))
2349 		return COMPACT_CONTINUE;
2350 
2351 	/* Direct compactor: Is a suitable page free? */
2352 	ret = COMPACT_NO_SUITABLE_PAGE;
2353 	for (order = cc->order; order < NR_PAGE_ORDERS; order++) {
2354 		struct free_area *area = &cc->zone->free_area[order];
2355 		bool can_steal;
2356 
2357 		/* Job done if page is free of the right migratetype */
2358 		if (!free_area_empty(area, migratetype))
2359 			return COMPACT_SUCCESS;
2360 
2361 #ifdef CONFIG_CMA
2362 		/* MIGRATE_MOVABLE can fallback on MIGRATE_CMA */
2363 		if (migratetype == MIGRATE_MOVABLE &&
2364 			!free_area_empty(area, MIGRATE_CMA))
2365 			return COMPACT_SUCCESS;
2366 #endif
2367 		/*
2368 		 * Job done if allocation would steal freepages from
2369 		 * other migratetype buddy lists.
2370 		 */
2371 		if (find_suitable_fallback(area, order, migratetype,
2372 						true, &can_steal) != -1)
2373 			/*
2374 			 * Movable pages are OK in any pageblock. If we are
2375 			 * stealing for a non-movable allocation, make sure
2376 			 * we finish compacting the current pageblock first
2377 			 * (which is assured by the above migrate_pfn align
2378 			 * check) so it is as free as possible and we won't
2379 			 * have to steal another one soon.
2380 			 */
2381 			return COMPACT_SUCCESS;
2382 	}
2383 
2384 out:
2385 	if (cc->contended || fatal_signal_pending(current))
2386 		ret = COMPACT_CONTENDED;
2387 
2388 	return ret;
2389 }
2390 
2391 static enum compact_result compact_finished(struct compact_control *cc)
2392 {
2393 	int ret;
2394 
2395 	ret = __compact_finished(cc);
2396 	trace_mm_compaction_finished(cc->zone, cc->order, ret);
2397 	if (ret == COMPACT_NO_SUITABLE_PAGE)
2398 		ret = COMPACT_CONTINUE;
2399 
2400 	return ret;
2401 }
2402 
2403 static bool __compaction_suitable(struct zone *zone, int order,
2404 				  int highest_zoneidx,
2405 				  unsigned long wmark_target)
2406 {
2407 	unsigned long watermark;
2408 	/*
2409 	 * Watermarks for order-0 must be met for compaction to be able to
2410 	 * isolate free pages for migration targets. This means that the
2411 	 * watermark and alloc_flags have to match, or be more pessimistic than
2412 	 * the check in __isolate_free_page(). We don't use the direct
2413 	 * compactor's alloc_flags, as they are not relevant for freepage
2414 	 * isolation. We however do use the direct compactor's highest_zoneidx
2415 	 * to skip over zones where lowmem reserves would prevent allocation
2416 	 * even if compaction succeeds.
2417 	 * For costly orders, we require low watermark instead of min for
2418 	 * compaction to proceed to increase its chances.
2419 	 * ALLOC_CMA is used, as pages in CMA pageblocks are considered
2420 	 * suitable migration targets
2421 	 */
2422 	watermark = (order > PAGE_ALLOC_COSTLY_ORDER) ?
2423 				low_wmark_pages(zone) : min_wmark_pages(zone);
2424 	watermark += compact_gap(order);
2425 	return __zone_watermark_ok(zone, 0, watermark, highest_zoneidx,
2426 				   ALLOC_CMA, wmark_target);
2427 }
2428 
2429 /*
2430  * compaction_suitable: Is this suitable to run compaction on this zone now?
2431  */
2432 bool compaction_suitable(struct zone *zone, int order, int highest_zoneidx)
2433 {
2434 	enum compact_result compact_result;
2435 	bool suitable;
2436 
2437 	suitable = __compaction_suitable(zone, order, highest_zoneidx,
2438 					 zone_page_state(zone, NR_FREE_PAGES));
2439 	/*
2440 	 * fragmentation index determines if allocation failures are due to
2441 	 * low memory or external fragmentation
2442 	 *
2443 	 * index of -1000 would imply allocations might succeed depending on
2444 	 * watermarks, but we already failed the high-order watermark check
2445 	 * index towards 0 implies failure is due to lack of memory
2446 	 * index towards 1000 implies failure is due to fragmentation
2447 	 *
2448 	 * Only compact if a failure would be due to fragmentation. Also
2449 	 * ignore fragindex for non-costly orders where the alternative to
2450 	 * a successful reclaim/compaction is OOM. Fragindex and the
2451 	 * vm.extfrag_threshold sysctl is meant as a heuristic to prevent
2452 	 * excessive compaction for costly orders, but it should not be at the
2453 	 * expense of system stability.
2454 	 */
2455 	if (suitable) {
2456 		compact_result = COMPACT_CONTINUE;
2457 		if (order > PAGE_ALLOC_COSTLY_ORDER) {
2458 			int fragindex = fragmentation_index(zone, order);
2459 
2460 			if (fragindex >= 0 &&
2461 			    fragindex <= sysctl_extfrag_threshold) {
2462 				suitable = false;
2463 				compact_result = COMPACT_NOT_SUITABLE_ZONE;
2464 			}
2465 		}
2466 	} else {
2467 		compact_result = COMPACT_SKIPPED;
2468 	}
2469 
2470 	trace_mm_compaction_suitable(zone, order, compact_result);
2471 
2472 	return suitable;
2473 }
2474 
2475 bool compaction_zonelist_suitable(struct alloc_context *ac, int order,
2476 		int alloc_flags)
2477 {
2478 	struct zone *zone;
2479 	struct zoneref *z;
2480 
2481 	/*
2482 	 * Make sure at least one zone would pass __compaction_suitable if we continue
2483 	 * retrying the reclaim.
2484 	 */
2485 	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
2486 				ac->highest_zoneidx, ac->nodemask) {
2487 		unsigned long available;
2488 
2489 		/*
2490 		 * Do not consider all the reclaimable memory because we do not
2491 		 * want to trash just for a single high order allocation which
2492 		 * is even not guaranteed to appear even if __compaction_suitable
2493 		 * is happy about the watermark check.
2494 		 */
2495 		available = zone_reclaimable_pages(zone) / order;
2496 		available += zone_page_state_snapshot(zone, NR_FREE_PAGES);
2497 		if (__compaction_suitable(zone, order, ac->highest_zoneidx,
2498 					  available))
2499 			return true;
2500 	}
2501 
2502 	return false;
2503 }
2504 
2505 /*
2506  * Should we do compaction for target allocation order.
2507  * Return COMPACT_SUCCESS if allocation for target order can be already
2508  * satisfied
2509  * Return COMPACT_SKIPPED if compaction for target order is likely to fail
2510  * Return COMPACT_CONTINUE if compaction for target order should be ran
2511  */
2512 static enum compact_result
2513 compaction_suit_allocation_order(struct zone *zone, unsigned int order,
2514 				 int highest_zoneidx, unsigned int alloc_flags)
2515 {
2516 	unsigned long watermark;
2517 
2518 	watermark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK);
2519 	if (zone_watermark_ok(zone, order, watermark, highest_zoneidx,
2520 			      alloc_flags))
2521 		return COMPACT_SUCCESS;
2522 
2523 	if (!compaction_suitable(zone, order, highest_zoneidx))
2524 		return COMPACT_SKIPPED;
2525 
2526 	return COMPACT_CONTINUE;
2527 }
2528 
2529 static enum compact_result
2530 compact_zone(struct compact_control *cc, struct capture_control *capc)
2531 {
2532 	enum compact_result ret;
2533 	unsigned long start_pfn = cc->zone->zone_start_pfn;
2534 	unsigned long end_pfn = zone_end_pfn(cc->zone);
2535 	unsigned long last_migrated_pfn;
2536 	const bool sync = cc->mode != MIGRATE_ASYNC;
2537 	bool update_cached;
2538 	unsigned int nr_succeeded = 0, nr_migratepages;
2539 	int order;
2540 
2541 	/*
2542 	 * These counters track activities during zone compaction.  Initialize
2543 	 * them before compacting a new zone.
2544 	 */
2545 	cc->total_migrate_scanned = 0;
2546 	cc->total_free_scanned = 0;
2547 	cc->nr_migratepages = 0;
2548 	cc->nr_freepages = 0;
2549 	for (order = 0; order < NR_PAGE_ORDERS; order++)
2550 		INIT_LIST_HEAD(&cc->freepages[order]);
2551 	INIT_LIST_HEAD(&cc->migratepages);
2552 
2553 	cc->migratetype = gfp_migratetype(cc->gfp_mask);
2554 
2555 	if (!is_via_compact_memory(cc->order)) {
2556 		ret = compaction_suit_allocation_order(cc->zone, cc->order,
2557 						       cc->highest_zoneidx,
2558 						       cc->alloc_flags);
2559 		if (ret != COMPACT_CONTINUE)
2560 			return ret;
2561 	}
2562 
2563 	/*
2564 	 * Clear pageblock skip if there were failures recently and compaction
2565 	 * is about to be retried after being deferred.
2566 	 */
2567 	if (compaction_restarting(cc->zone, cc->order))
2568 		__reset_isolation_suitable(cc->zone);
2569 
2570 	/*
2571 	 * Setup to move all movable pages to the end of the zone. Used cached
2572 	 * information on where the scanners should start (unless we explicitly
2573 	 * want to compact the whole zone), but check that it is initialised
2574 	 * by ensuring the values are within zone boundaries.
2575 	 */
2576 	cc->fast_start_pfn = 0;
2577 	if (cc->whole_zone) {
2578 		cc->migrate_pfn = start_pfn;
2579 		cc->free_pfn = pageblock_start_pfn(end_pfn - 1);
2580 	} else {
2581 		cc->migrate_pfn = cc->zone->compact_cached_migrate_pfn[sync];
2582 		cc->free_pfn = cc->zone->compact_cached_free_pfn;
2583 		if (cc->free_pfn < start_pfn || cc->free_pfn >= end_pfn) {
2584 			cc->free_pfn = pageblock_start_pfn(end_pfn - 1);
2585 			cc->zone->compact_cached_free_pfn = cc->free_pfn;
2586 		}
2587 		if (cc->migrate_pfn < start_pfn || cc->migrate_pfn >= end_pfn) {
2588 			cc->migrate_pfn = start_pfn;
2589 			cc->zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn;
2590 			cc->zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn;
2591 		}
2592 
2593 		if (cc->migrate_pfn <= cc->zone->compact_init_migrate_pfn)
2594 			cc->whole_zone = true;
2595 	}
2596 
2597 	last_migrated_pfn = 0;
2598 
2599 	/*
2600 	 * Migrate has separate cached PFNs for ASYNC and SYNC* migration on
2601 	 * the basis that some migrations will fail in ASYNC mode. However,
2602 	 * if the cached PFNs match and pageblocks are skipped due to having
2603 	 * no isolation candidates, then the sync state does not matter.
2604 	 * Until a pageblock with isolation candidates is found, keep the
2605 	 * cached PFNs in sync to avoid revisiting the same blocks.
2606 	 */
2607 	update_cached = !sync &&
2608 		cc->zone->compact_cached_migrate_pfn[0] == cc->zone->compact_cached_migrate_pfn[1];
2609 
2610 	trace_mm_compaction_begin(cc, start_pfn, end_pfn, sync);
2611 
2612 	/* lru_add_drain_all could be expensive with involving other CPUs */
2613 	lru_add_drain();
2614 
2615 	while ((ret = compact_finished(cc)) == COMPACT_CONTINUE) {
2616 		int err;
2617 		unsigned long iteration_start_pfn = cc->migrate_pfn;
2618 
2619 		/*
2620 		 * Avoid multiple rescans of the same pageblock which can
2621 		 * happen if a page cannot be isolated (dirty/writeback in
2622 		 * async mode) or if the migrated pages are being allocated
2623 		 * before the pageblock is cleared.  The first rescan will
2624 		 * capture the entire pageblock for migration. If it fails,
2625 		 * it'll be marked skip and scanning will proceed as normal.
2626 		 */
2627 		cc->finish_pageblock = false;
2628 		if (pageblock_start_pfn(last_migrated_pfn) ==
2629 		    pageblock_start_pfn(iteration_start_pfn)) {
2630 			cc->finish_pageblock = true;
2631 		}
2632 
2633 rescan:
2634 		switch (isolate_migratepages(cc)) {
2635 		case ISOLATE_ABORT:
2636 			ret = COMPACT_CONTENDED;
2637 			putback_movable_pages(&cc->migratepages);
2638 			cc->nr_migratepages = 0;
2639 			goto out;
2640 		case ISOLATE_NONE:
2641 			if (update_cached) {
2642 				cc->zone->compact_cached_migrate_pfn[1] =
2643 					cc->zone->compact_cached_migrate_pfn[0];
2644 			}
2645 
2646 			/*
2647 			 * We haven't isolated and migrated anything, but
2648 			 * there might still be unflushed migrations from
2649 			 * previous cc->order aligned block.
2650 			 */
2651 			goto check_drain;
2652 		case ISOLATE_SUCCESS:
2653 			update_cached = false;
2654 			last_migrated_pfn = max(cc->zone->zone_start_pfn,
2655 				pageblock_start_pfn(cc->migrate_pfn - 1));
2656 		}
2657 
2658 		/*
2659 		 * Record the number of pages to migrate since the
2660 		 * compaction_alloc/free() will update cc->nr_migratepages
2661 		 * properly.
2662 		 */
2663 		nr_migratepages = cc->nr_migratepages;
2664 		err = migrate_pages(&cc->migratepages, compaction_alloc,
2665 				compaction_free, (unsigned long)cc, cc->mode,
2666 				MR_COMPACTION, &nr_succeeded);
2667 
2668 		trace_mm_compaction_migratepages(nr_migratepages, nr_succeeded);
2669 
2670 		/* All pages were either migrated or will be released */
2671 		cc->nr_migratepages = 0;
2672 		if (err) {
2673 			putback_movable_pages(&cc->migratepages);
2674 			/*
2675 			 * migrate_pages() may return -ENOMEM when scanners meet
2676 			 * and we want compact_finished() to detect it
2677 			 */
2678 			if (err == -ENOMEM && !compact_scanners_met(cc)) {
2679 				ret = COMPACT_CONTENDED;
2680 				goto out;
2681 			}
2682 			/*
2683 			 * If an ASYNC or SYNC_LIGHT fails to migrate a page
2684 			 * within the pageblock_order-aligned block and
2685 			 * fast_find_migrateblock may be used then scan the
2686 			 * remainder of the pageblock. This will mark the
2687 			 * pageblock "skip" to avoid rescanning in the near
2688 			 * future. This will isolate more pages than necessary
2689 			 * for the request but avoid loops due to
2690 			 * fast_find_migrateblock revisiting blocks that were
2691 			 * recently partially scanned.
2692 			 */
2693 			if (!pageblock_aligned(cc->migrate_pfn) &&
2694 			    !cc->ignore_skip_hint && !cc->finish_pageblock &&
2695 			    (cc->mode < MIGRATE_SYNC)) {
2696 				cc->finish_pageblock = true;
2697 
2698 				/*
2699 				 * Draining pcplists does not help THP if
2700 				 * any page failed to migrate. Even after
2701 				 * drain, the pageblock will not be free.
2702 				 */
2703 				if (cc->order == COMPACTION_HPAGE_ORDER)
2704 					last_migrated_pfn = 0;
2705 
2706 				goto rescan;
2707 			}
2708 		}
2709 
2710 		/* Stop if a page has been captured */
2711 		if (capc && capc->page) {
2712 			ret = COMPACT_SUCCESS;
2713 			break;
2714 		}
2715 
2716 check_drain:
2717 		/*
2718 		 * Has the migration scanner moved away from the previous
2719 		 * cc->order aligned block where we migrated from? If yes,
2720 		 * flush the pages that were freed, so that they can merge and
2721 		 * compact_finished() can detect immediately if allocation
2722 		 * would succeed.
2723 		 */
2724 		if (cc->order > 0 && last_migrated_pfn) {
2725 			unsigned long current_block_start =
2726 				block_start_pfn(cc->migrate_pfn, cc->order);
2727 
2728 			if (last_migrated_pfn < current_block_start) {
2729 				lru_add_drain_cpu_zone(cc->zone);
2730 				/* No more flushing until we migrate again */
2731 				last_migrated_pfn = 0;
2732 			}
2733 		}
2734 	}
2735 
2736 out:
2737 	/*
2738 	 * Release free pages and update where the free scanner should restart,
2739 	 * so we don't leave any returned pages behind in the next attempt.
2740 	 */
2741 	if (cc->nr_freepages > 0) {
2742 		unsigned long free_pfn = release_free_list(cc->freepages);
2743 
2744 		cc->nr_freepages = 0;
2745 		VM_BUG_ON(free_pfn == 0);
2746 		/* The cached pfn is always the first in a pageblock */
2747 		free_pfn = pageblock_start_pfn(free_pfn);
2748 		/*
2749 		 * Only go back, not forward. The cached pfn might have been
2750 		 * already reset to zone end in compact_finished()
2751 		 */
2752 		if (free_pfn > cc->zone->compact_cached_free_pfn)
2753 			cc->zone->compact_cached_free_pfn = free_pfn;
2754 	}
2755 
2756 	count_compact_events(COMPACTMIGRATE_SCANNED, cc->total_migrate_scanned);
2757 	count_compact_events(COMPACTFREE_SCANNED, cc->total_free_scanned);
2758 
2759 	trace_mm_compaction_end(cc, start_pfn, end_pfn, sync, ret);
2760 
2761 	VM_BUG_ON(!list_empty(&cc->migratepages));
2762 
2763 	return ret;
2764 }
2765 
2766 static enum compact_result compact_zone_order(struct zone *zone, int order,
2767 		gfp_t gfp_mask, enum compact_priority prio,
2768 		unsigned int alloc_flags, int highest_zoneidx,
2769 		struct page **capture)
2770 {
2771 	enum compact_result ret;
2772 	struct compact_control cc = {
2773 		.order = order,
2774 		.search_order = order,
2775 		.gfp_mask = gfp_mask,
2776 		.zone = zone,
2777 		.mode = (prio == COMPACT_PRIO_ASYNC) ?
2778 					MIGRATE_ASYNC :	MIGRATE_SYNC_LIGHT,
2779 		.alloc_flags = alloc_flags,
2780 		.highest_zoneidx = highest_zoneidx,
2781 		.direct_compaction = true,
2782 		.whole_zone = (prio == MIN_COMPACT_PRIORITY),
2783 		.ignore_skip_hint = (prio == MIN_COMPACT_PRIORITY),
2784 		.ignore_block_suitable = (prio == MIN_COMPACT_PRIORITY)
2785 	};
2786 	struct capture_control capc = {
2787 		.cc = &cc,
2788 		.page = NULL,
2789 	};
2790 
2791 	/*
2792 	 * Make sure the structs are really initialized before we expose the
2793 	 * capture control, in case we are interrupted and the interrupt handler
2794 	 * frees a page.
2795 	 */
2796 	barrier();
2797 	WRITE_ONCE(current->capture_control, &capc);
2798 
2799 	ret = compact_zone(&cc, &capc);
2800 
2801 	/*
2802 	 * Make sure we hide capture control first before we read the captured
2803 	 * page pointer, otherwise an interrupt could free and capture a page
2804 	 * and we would leak it.
2805 	 */
2806 	WRITE_ONCE(current->capture_control, NULL);
2807 	*capture = READ_ONCE(capc.page);
2808 	/*
2809 	 * Technically, it is also possible that compaction is skipped but
2810 	 * the page is still captured out of luck(IRQ came and freed the page).
2811 	 * Returning COMPACT_SUCCESS in such cases helps in properly accounting
2812 	 * the COMPACT[STALL|FAIL] when compaction is skipped.
2813 	 */
2814 	if (*capture)
2815 		ret = COMPACT_SUCCESS;
2816 
2817 	return ret;
2818 }
2819 
2820 /**
2821  * try_to_compact_pages - Direct compact to satisfy a high-order allocation
2822  * @gfp_mask: The GFP mask of the current allocation
2823  * @order: The order of the current allocation
2824  * @alloc_flags: The allocation flags of the current allocation
2825  * @ac: The context of current allocation
2826  * @prio: Determines how hard direct compaction should try to succeed
2827  * @capture: Pointer to free page created by compaction will be stored here
2828  *
2829  * This is the main entry point for direct page compaction.
2830  */
2831 enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order,
2832 		unsigned int alloc_flags, const struct alloc_context *ac,
2833 		enum compact_priority prio, struct page **capture)
2834 {
2835 	struct zoneref *z;
2836 	struct zone *zone;
2837 	enum compact_result rc = COMPACT_SKIPPED;
2838 
2839 	if (!gfp_compaction_allowed(gfp_mask))
2840 		return COMPACT_SKIPPED;
2841 
2842 	trace_mm_compaction_try_to_compact_pages(order, gfp_mask, prio);
2843 
2844 	/* Compact each zone in the list */
2845 	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
2846 					ac->highest_zoneidx, ac->nodemask) {
2847 		enum compact_result status;
2848 
2849 		if (prio > MIN_COMPACT_PRIORITY
2850 					&& compaction_deferred(zone, order)) {
2851 			rc = max_t(enum compact_result, COMPACT_DEFERRED, rc);
2852 			continue;
2853 		}
2854 
2855 		status = compact_zone_order(zone, order, gfp_mask, prio,
2856 				alloc_flags, ac->highest_zoneidx, capture);
2857 		rc = max(status, rc);
2858 
2859 		/* The allocation should succeed, stop compacting */
2860 		if (status == COMPACT_SUCCESS) {
2861 			/*
2862 			 * We think the allocation will succeed in this zone,
2863 			 * but it is not certain, hence the false. The caller
2864 			 * will repeat this with true if allocation indeed
2865 			 * succeeds in this zone.
2866 			 */
2867 			compaction_defer_reset(zone, order, false);
2868 
2869 			break;
2870 		}
2871 
2872 		if (prio != COMPACT_PRIO_ASYNC && (status == COMPACT_COMPLETE ||
2873 					status == COMPACT_PARTIAL_SKIPPED))
2874 			/*
2875 			 * We think that allocation won't succeed in this zone
2876 			 * so we defer compaction there. If it ends up
2877 			 * succeeding after all, it will be reset.
2878 			 */
2879 			defer_compaction(zone, order);
2880 
2881 		/*
2882 		 * We might have stopped compacting due to need_resched() in
2883 		 * async compaction, or due to a fatal signal detected. In that
2884 		 * case do not try further zones
2885 		 */
2886 		if ((prio == COMPACT_PRIO_ASYNC && need_resched())
2887 					|| fatal_signal_pending(current))
2888 			break;
2889 	}
2890 
2891 	return rc;
2892 }
2893 
2894 /*
2895  * compact_node() - compact all zones within a node
2896  * @pgdat: The node page data
2897  * @proactive: Whether the compaction is proactive
2898  *
2899  * For proactive compaction, compact till each zone's fragmentation score
2900  * reaches within proactive compaction thresholds (as determined by the
2901  * proactiveness tunable), it is possible that the function returns before
2902  * reaching score targets due to various back-off conditions, such as,
2903  * contention on per-node or per-zone locks.
2904  */
2905 static int compact_node(pg_data_t *pgdat, bool proactive)
2906 {
2907 	int zoneid;
2908 	struct zone *zone;
2909 	struct compact_control cc = {
2910 		.order = -1,
2911 		.mode = proactive ? MIGRATE_SYNC_LIGHT : MIGRATE_SYNC,
2912 		.ignore_skip_hint = true,
2913 		.whole_zone = true,
2914 		.gfp_mask = GFP_KERNEL,
2915 		.proactive_compaction = proactive,
2916 	};
2917 
2918 	for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {
2919 		zone = &pgdat->node_zones[zoneid];
2920 		if (!populated_zone(zone))
2921 			continue;
2922 
2923 		if (fatal_signal_pending(current))
2924 			return -EINTR;
2925 
2926 		cc.zone = zone;
2927 
2928 		compact_zone(&cc, NULL);
2929 
2930 		if (proactive) {
2931 			count_compact_events(KCOMPACTD_MIGRATE_SCANNED,
2932 					     cc.total_migrate_scanned);
2933 			count_compact_events(KCOMPACTD_FREE_SCANNED,
2934 					     cc.total_free_scanned);
2935 		}
2936 	}
2937 
2938 	return 0;
2939 }
2940 
2941 /* Compact all zones of all nodes in the system */
2942 static int compact_nodes(void)
2943 {
2944 	int ret, nid;
2945 
2946 	/* Flush pending updates to the LRU lists */
2947 	lru_add_drain_all();
2948 
2949 	for_each_online_node(nid) {
2950 		ret = compact_node(NODE_DATA(nid), false);
2951 		if (ret)
2952 			return ret;
2953 	}
2954 
2955 	return 0;
2956 }
2957 
2958 static int compaction_proactiveness_sysctl_handler(struct ctl_table *table, int write,
2959 		void *buffer, size_t *length, loff_t *ppos)
2960 {
2961 	int rc, nid;
2962 
2963 	rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
2964 	if (rc)
2965 		return rc;
2966 
2967 	if (write && sysctl_compaction_proactiveness) {
2968 		for_each_online_node(nid) {
2969 			pg_data_t *pgdat = NODE_DATA(nid);
2970 
2971 			if (pgdat->proactive_compact_trigger)
2972 				continue;
2973 
2974 			pgdat->proactive_compact_trigger = true;
2975 			trace_mm_compaction_wakeup_kcompactd(pgdat->node_id, -1,
2976 							     pgdat->nr_zones - 1);
2977 			wake_up_interruptible(&pgdat->kcompactd_wait);
2978 		}
2979 	}
2980 
2981 	return 0;
2982 }
2983 
2984 /*
2985  * This is the entry point for compacting all nodes via
2986  * /proc/sys/vm/compact_memory
2987  */
2988 static int sysctl_compaction_handler(struct ctl_table *table, int write,
2989 			void *buffer, size_t *length, loff_t *ppos)
2990 {
2991 	int ret;
2992 
2993 	ret = proc_dointvec(table, write, buffer, length, ppos);
2994 	if (ret)
2995 		return ret;
2996 
2997 	if (sysctl_compact_memory != 1)
2998 		return -EINVAL;
2999 
3000 	if (write)
3001 		ret = compact_nodes();
3002 
3003 	return ret;
3004 }
3005 
3006 #if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA)
3007 static ssize_t compact_store(struct device *dev,
3008 			     struct device_attribute *attr,
3009 			     const char *buf, size_t count)
3010 {
3011 	int nid = dev->id;
3012 
3013 	if (nid >= 0 && nid < nr_node_ids && node_online(nid)) {
3014 		/* Flush pending updates to the LRU lists */
3015 		lru_add_drain_all();
3016 
3017 		compact_node(NODE_DATA(nid), false);
3018 	}
3019 
3020 	return count;
3021 }
3022 static DEVICE_ATTR_WO(compact);
3023 
3024 int compaction_register_node(struct node *node)
3025 {
3026 	return device_create_file(&node->dev, &dev_attr_compact);
3027 }
3028 
3029 void compaction_unregister_node(struct node *node)
3030 {
3031 	device_remove_file(&node->dev, &dev_attr_compact);
3032 }
3033 #endif /* CONFIG_SYSFS && CONFIG_NUMA */
3034 
3035 static inline bool kcompactd_work_requested(pg_data_t *pgdat)
3036 {
3037 	return pgdat->kcompactd_max_order > 0 || kthread_should_stop() ||
3038 		pgdat->proactive_compact_trigger;
3039 }
3040 
3041 static bool kcompactd_node_suitable(pg_data_t *pgdat)
3042 {
3043 	int zoneid;
3044 	struct zone *zone;
3045 	enum zone_type highest_zoneidx = pgdat->kcompactd_highest_zoneidx;
3046 	enum compact_result ret;
3047 
3048 	for (zoneid = 0; zoneid <= highest_zoneidx; zoneid++) {
3049 		zone = &pgdat->node_zones[zoneid];
3050 
3051 		if (!populated_zone(zone))
3052 			continue;
3053 
3054 		ret = compaction_suit_allocation_order(zone,
3055 				pgdat->kcompactd_max_order,
3056 				highest_zoneidx, ALLOC_WMARK_MIN);
3057 		if (ret == COMPACT_CONTINUE)
3058 			return true;
3059 	}
3060 
3061 	return false;
3062 }
3063 
3064 static void kcompactd_do_work(pg_data_t *pgdat)
3065 {
3066 	/*
3067 	 * With no special task, compact all zones so that a page of requested
3068 	 * order is allocatable.
3069 	 */
3070 	int zoneid;
3071 	struct zone *zone;
3072 	struct compact_control cc = {
3073 		.order = pgdat->kcompactd_max_order,
3074 		.search_order = pgdat->kcompactd_max_order,
3075 		.highest_zoneidx = pgdat->kcompactd_highest_zoneidx,
3076 		.mode = MIGRATE_SYNC_LIGHT,
3077 		.ignore_skip_hint = false,
3078 		.gfp_mask = GFP_KERNEL,
3079 	};
3080 	enum compact_result ret;
3081 
3082 	trace_mm_compaction_kcompactd_wake(pgdat->node_id, cc.order,
3083 							cc.highest_zoneidx);
3084 	count_compact_event(KCOMPACTD_WAKE);
3085 
3086 	for (zoneid = 0; zoneid <= cc.highest_zoneidx; zoneid++) {
3087 		int status;
3088 
3089 		zone = &pgdat->node_zones[zoneid];
3090 		if (!populated_zone(zone))
3091 			continue;
3092 
3093 		if (compaction_deferred(zone, cc.order))
3094 			continue;
3095 
3096 		ret = compaction_suit_allocation_order(zone,
3097 				cc.order, zoneid, ALLOC_WMARK_MIN);
3098 		if (ret != COMPACT_CONTINUE)
3099 			continue;
3100 
3101 		if (kthread_should_stop())
3102 			return;
3103 
3104 		cc.zone = zone;
3105 		status = compact_zone(&cc, NULL);
3106 
3107 		if (status == COMPACT_SUCCESS) {
3108 			compaction_defer_reset(zone, cc.order, false);
3109 		} else if (status == COMPACT_PARTIAL_SKIPPED || status == COMPACT_COMPLETE) {
3110 			/*
3111 			 * Buddy pages may become stranded on pcps that could
3112 			 * otherwise coalesce on the zone's free area for
3113 			 * order >= cc.order.  This is ratelimited by the
3114 			 * upcoming deferral.
3115 			 */
3116 			drain_all_pages(zone);
3117 
3118 			/*
3119 			 * We use sync migration mode here, so we defer like
3120 			 * sync direct compaction does.
3121 			 */
3122 			defer_compaction(zone, cc.order);
3123 		}
3124 
3125 		count_compact_events(KCOMPACTD_MIGRATE_SCANNED,
3126 				     cc.total_migrate_scanned);
3127 		count_compact_events(KCOMPACTD_FREE_SCANNED,
3128 				     cc.total_free_scanned);
3129 	}
3130 
3131 	/*
3132 	 * Regardless of success, we are done until woken up next. But remember
3133 	 * the requested order/highest_zoneidx in case it was higher/tighter
3134 	 * than our current ones
3135 	 */
3136 	if (pgdat->kcompactd_max_order <= cc.order)
3137 		pgdat->kcompactd_max_order = 0;
3138 	if (pgdat->kcompactd_highest_zoneidx >= cc.highest_zoneidx)
3139 		pgdat->kcompactd_highest_zoneidx = pgdat->nr_zones - 1;
3140 }
3141 
3142 void wakeup_kcompactd(pg_data_t *pgdat, int order, int highest_zoneidx)
3143 {
3144 	if (!order)
3145 		return;
3146 
3147 	if (pgdat->kcompactd_max_order < order)
3148 		pgdat->kcompactd_max_order = order;
3149 
3150 	if (pgdat->kcompactd_highest_zoneidx > highest_zoneidx)
3151 		pgdat->kcompactd_highest_zoneidx = highest_zoneidx;
3152 
3153 	/*
3154 	 * Pairs with implicit barrier in wait_event_freezable()
3155 	 * such that wakeups are not missed.
3156 	 */
3157 	if (!wq_has_sleeper(&pgdat->kcompactd_wait))
3158 		return;
3159 
3160 	if (!kcompactd_node_suitable(pgdat))
3161 		return;
3162 
3163 	trace_mm_compaction_wakeup_kcompactd(pgdat->node_id, order,
3164 							highest_zoneidx);
3165 	wake_up_interruptible(&pgdat->kcompactd_wait);
3166 }
3167 
3168 /*
3169  * The background compaction daemon, started as a kernel thread
3170  * from the init process.
3171  */
3172 static int kcompactd(void *p)
3173 {
3174 	pg_data_t *pgdat = (pg_data_t *)p;
3175 	struct task_struct *tsk = current;
3176 	long default_timeout = msecs_to_jiffies(HPAGE_FRAG_CHECK_INTERVAL_MSEC);
3177 	long timeout = default_timeout;
3178 
3179 	const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
3180 
3181 	if (!cpumask_empty(cpumask))
3182 		set_cpus_allowed_ptr(tsk, cpumask);
3183 
3184 	set_freezable();
3185 
3186 	pgdat->kcompactd_max_order = 0;
3187 	pgdat->kcompactd_highest_zoneidx = pgdat->nr_zones - 1;
3188 
3189 	while (!kthread_should_stop()) {
3190 		unsigned long pflags;
3191 
3192 		/*
3193 		 * Avoid the unnecessary wakeup for proactive compaction
3194 		 * when it is disabled.
3195 		 */
3196 		if (!sysctl_compaction_proactiveness)
3197 			timeout = MAX_SCHEDULE_TIMEOUT;
3198 		trace_mm_compaction_kcompactd_sleep(pgdat->node_id);
3199 		if (wait_event_freezable_timeout(pgdat->kcompactd_wait,
3200 			kcompactd_work_requested(pgdat), timeout) &&
3201 			!pgdat->proactive_compact_trigger) {
3202 
3203 			psi_memstall_enter(&pflags);
3204 			kcompactd_do_work(pgdat);
3205 			psi_memstall_leave(&pflags);
3206 			/*
3207 			 * Reset the timeout value. The defer timeout from
3208 			 * proactive compaction is lost here but that is fine
3209 			 * as the condition of the zone changing substantionally
3210 			 * then carrying on with the previous defer interval is
3211 			 * not useful.
3212 			 */
3213 			timeout = default_timeout;
3214 			continue;
3215 		}
3216 
3217 		/*
3218 		 * Start the proactive work with default timeout. Based
3219 		 * on the fragmentation score, this timeout is updated.
3220 		 */
3221 		timeout = default_timeout;
3222 		if (should_proactive_compact_node(pgdat)) {
3223 			unsigned int prev_score, score;
3224 
3225 			prev_score = fragmentation_score_node(pgdat);
3226 			compact_node(pgdat, true);
3227 			score = fragmentation_score_node(pgdat);
3228 			/*
3229 			 * Defer proactive compaction if the fragmentation
3230 			 * score did not go down i.e. no progress made.
3231 			 */
3232 			if (unlikely(score >= prev_score))
3233 				timeout =
3234 				   default_timeout << COMPACT_MAX_DEFER_SHIFT;
3235 		}
3236 		if (unlikely(pgdat->proactive_compact_trigger))
3237 			pgdat->proactive_compact_trigger = false;
3238 	}
3239 
3240 	return 0;
3241 }
3242 
3243 /*
3244  * This kcompactd start function will be called by init and node-hot-add.
3245  * On node-hot-add, kcompactd will moved to proper cpus if cpus are hot-added.
3246  */
3247 void __meminit kcompactd_run(int nid)
3248 {
3249 	pg_data_t *pgdat = NODE_DATA(nid);
3250 
3251 	if (pgdat->kcompactd)
3252 		return;
3253 
3254 	pgdat->kcompactd = kthread_run(kcompactd, pgdat, "kcompactd%d", nid);
3255 	if (IS_ERR(pgdat->kcompactd)) {
3256 		pr_err("Failed to start kcompactd on node %d\n", nid);
3257 		pgdat->kcompactd = NULL;
3258 	}
3259 }
3260 
3261 /*
3262  * Called by memory hotplug when all memory in a node is offlined. Caller must
3263  * be holding mem_hotplug_begin/done().
3264  */
3265 void __meminit kcompactd_stop(int nid)
3266 {
3267 	struct task_struct *kcompactd = NODE_DATA(nid)->kcompactd;
3268 
3269 	if (kcompactd) {
3270 		kthread_stop(kcompactd);
3271 		NODE_DATA(nid)->kcompactd = NULL;
3272 	}
3273 }
3274 
3275 /*
3276  * It's optimal to keep kcompactd on the same CPUs as their memory, but
3277  * not required for correctness. So if the last cpu in a node goes
3278  * away, we get changed to run anywhere: as the first one comes back,
3279  * restore their cpu bindings.
3280  */
3281 static int kcompactd_cpu_online(unsigned int cpu)
3282 {
3283 	int nid;
3284 
3285 	for_each_node_state(nid, N_MEMORY) {
3286 		pg_data_t *pgdat = NODE_DATA(nid);
3287 		const struct cpumask *mask;
3288 
3289 		mask = cpumask_of_node(pgdat->node_id);
3290 
3291 		if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids)
3292 			/* One of our CPUs online: restore mask */
3293 			if (pgdat->kcompactd)
3294 				set_cpus_allowed_ptr(pgdat->kcompactd, mask);
3295 	}
3296 	return 0;
3297 }
3298 
3299 static int proc_dointvec_minmax_warn_RT_change(struct ctl_table *table,
3300 		int write, void *buffer, size_t *lenp, loff_t *ppos)
3301 {
3302 	int ret, old;
3303 
3304 	if (!IS_ENABLED(CONFIG_PREEMPT_RT) || !write)
3305 		return proc_dointvec_minmax(table, write, buffer, lenp, ppos);
3306 
3307 	old = *(int *)table->data;
3308 	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
3309 	if (ret)
3310 		return ret;
3311 	if (old != *(int *)table->data)
3312 		pr_warn_once("sysctl attribute %s changed by %s[%d]\n",
3313 			     table->procname, current->comm,
3314 			     task_pid_nr(current));
3315 	return ret;
3316 }
3317 
3318 static struct ctl_table vm_compaction[] = {
3319 	{
3320 		.procname	= "compact_memory",
3321 		.data		= &sysctl_compact_memory,
3322 		.maxlen		= sizeof(int),
3323 		.mode		= 0200,
3324 		.proc_handler	= sysctl_compaction_handler,
3325 	},
3326 	{
3327 		.procname	= "compaction_proactiveness",
3328 		.data		= &sysctl_compaction_proactiveness,
3329 		.maxlen		= sizeof(sysctl_compaction_proactiveness),
3330 		.mode		= 0644,
3331 		.proc_handler	= compaction_proactiveness_sysctl_handler,
3332 		.extra1		= SYSCTL_ZERO,
3333 		.extra2		= SYSCTL_ONE_HUNDRED,
3334 	},
3335 	{
3336 		.procname	= "extfrag_threshold",
3337 		.data		= &sysctl_extfrag_threshold,
3338 		.maxlen		= sizeof(int),
3339 		.mode		= 0644,
3340 		.proc_handler	= proc_dointvec_minmax,
3341 		.extra1		= SYSCTL_ZERO,
3342 		.extra2		= SYSCTL_ONE_THOUSAND,
3343 	},
3344 	{
3345 		.procname	= "compact_unevictable_allowed",
3346 		.data		= &sysctl_compact_unevictable_allowed,
3347 		.maxlen		= sizeof(int),
3348 		.mode		= 0644,
3349 		.proc_handler	= proc_dointvec_minmax_warn_RT_change,
3350 		.extra1		= SYSCTL_ZERO,
3351 		.extra2		= SYSCTL_ONE,
3352 	},
3353 };
3354 
3355 static int __init kcompactd_init(void)
3356 {
3357 	int nid;
3358 	int ret;
3359 
3360 	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
3361 					"mm/compaction:online",
3362 					kcompactd_cpu_online, NULL);
3363 	if (ret < 0) {
3364 		pr_err("kcompactd: failed to register hotplug callbacks.\n");
3365 		return ret;
3366 	}
3367 
3368 	for_each_node_state(nid, N_MEMORY)
3369 		kcompactd_run(nid);
3370 	register_sysctl_init("vm", vm_compaction);
3371 	return 0;
3372 }
3373 subsys_initcall(kcompactd_init)
3374 
3375 #endif /* CONFIG_COMPACTION */
3376