Lines Matching full:order
46 * order == -1 is expected when compacting proactively via
51 static inline bool is_via_compact_memory(int order) in is_via_compact_memory() argument
53 return order == -1; in is_via_compact_memory()
59 static inline bool is_via_compact_memory(int order) { return false; } in is_via_compact_memory() argument
67 #define block_start_pfn(pfn, order) round_down(pfn, 1UL << (order)) argument
68 #define block_end_pfn(pfn, order) ALIGN((pfn) + 1, 1UL << (order)) argument
71 * Page order with-respect-to which proactive compaction
83 static struct page *mark_allocated_noprof(struct page *page, unsigned int order, gfp_t gfp_flags) in mark_allocated_noprof() argument
85 post_alloc_hook(page, order, __GFP_MOVABLE); in mark_allocated_noprof()
92 int order; in release_free_list() local
95 for (order = 0; order < NR_PAGE_ORDERS; order++) { in release_free_list()
98 list_for_each_entry_safe(page, next, &freepages[order], lru) { in release_free_list()
106 mark_allocated(page, order, __GFP_MOVABLE); in release_free_list()
107 __free_pages(page, order); in release_free_list()
158 static void defer_compaction(struct zone *zone, int order) in defer_compaction() argument
163 if (order < zone->compact_order_failed) in defer_compaction()
164 zone->compact_order_failed = order; in defer_compaction()
169 trace_mm_compaction_defer_compaction(zone, order); in defer_compaction()
173 static bool compaction_deferred(struct zone *zone, int order) in compaction_deferred() argument
177 if (order < zone->compact_order_failed) in compaction_deferred()
186 trace_mm_compaction_deferred(zone, order); in compaction_deferred()
192 * Update defer tracking counters after successful compaction of given order,
196 void compaction_defer_reset(struct zone *zone, int order, in compaction_defer_reset() argument
203 if (order >= zone->compact_order_failed) in compaction_defer_reset()
204 zone->compact_order_failed = order + 1; in compaction_defer_reset()
206 trace_mm_compaction_defer_reset(zone, order); in compaction_defer_reset()
210 static bool compaction_restarting(struct zone *zone, int order) in compaction_restarting() argument
212 if (order < zone->compact_order_failed) in compaction_restarting()
291 * released. It is always pointless to compact pages of such order (if they are
600 unsigned int order; in isolate_freepages_block() local
631 const unsigned int order = compound_order(page); in isolate_freepages_block() local
633 if (blockpfn + (1UL << order) <= end_pfn) { in isolate_freepages_block()
634 blockpfn += (1UL << order) - 1; in isolate_freepages_block()
635 page += (1UL << order) - 1; in isolate_freepages_block()
636 nr_scanned += (1UL << order) - 1; in isolate_freepages_block()
655 /* Found a free page, will break it into order-0 pages */ in isolate_freepages_block()
656 order = buddy_order(page); in isolate_freepages_block()
657 isolated = __isolate_free_page(page, order); in isolate_freepages_block()
660 set_page_private(page, order); in isolate_freepages_block()
665 list_add_tail(&page->lru, &freelist[order]); in isolate_freepages_block()
730 int order; in isolate_freepages_range() local
732 for (order = 0; order < NR_PAGE_ORDERS; order++) in isolate_freepages_range()
733 INIT_LIST_HEAD(&cc->freepages[order]); in isolate_freepages_range()
749 * is more than pageblock order. In this case, we adjust in isolate_freepages_range()
776 * pageblock_nr_pages for some non-negative n. (Max order in isolate_freepages_range()
826 * folio order and compaction target order
827 * @order: to-be-isolated folio order
828 * @target_order: compaction target order
832 static bool skip_isolation_on_order(int order, int target_order) in skip_isolation_on_order() argument
837 * target order: we wouldn't be here if we'd have a free folio with in skip_isolation_on_order()
841 if (!is_via_compact_memory(target_order) && order >= target_order) in skip_isolation_on_order()
847 return order >= pageblock_order; in skip_isolation_on_order()
911 next_skip_pfn = block_end_pfn(low_pfn, cc->order); in isolate_migratepages_block()
921 * previous order-aligned block, and did not skip it due in isolate_migratepages_block()
929 * We failed to isolate in the previous order-aligned in isolate_migratepages_block()
932 * next_skip_pfn by 1 << order, as low_pfn might have in isolate_migratepages_block()
934 * a compound or a high-order buddy page in the in isolate_migratepages_block()
937 next_skip_pfn = block_end_pfn(low_pfn, cc->order); in isolate_migratepages_block()
984 * bigger than its order. THPs and other compound pages in isolate_migratepages_block()
988 const unsigned int order = compound_order(page); in isolate_migratepages_block() local
990 if (order <= MAX_PAGE_ORDER) { in isolate_migratepages_block()
991 low_pfn += (1UL << order) - 1; in isolate_migratepages_block()
992 nr_scanned += (1UL << order) - 1; in isolate_migratepages_block()
1036 * Skip if free. We read page order here without zone lock in isolate_migratepages_block()
1046 * a valid page order. Consider only values in the in isolate_migratepages_block()
1047 * valid order range to prevent low_pfn overflow. in isolate_migratepages_block()
1065 const unsigned int order = compound_order(page); in isolate_migratepages_block() local
1067 /* Skip based on page order and compaction target order. */ in isolate_migratepages_block()
1068 if (skip_isolation_on_order(order, cc->order)) { in isolate_migratepages_block()
1069 if (order <= MAX_PAGE_ORDER) { in isolate_migratepages_block()
1070 low_pfn += (1UL << order) - 1; in isolate_migratepages_block()
1071 nr_scanned += (1UL << order) - 1; in isolate_migratepages_block()
1217 * Check LRU folio order under the lock in isolate_migratepages_block()
1220 cc->order) && in isolate_migratepages_block()
1274 * instead of migrating, as we cannot form the cc->order buddy in isolate_migratepages_block()
1293 next_skip_pfn += 1UL << cc->order; in isolate_migratepages_block()
1418 int order = cc->order > 0 ? cc->order : pageblock_order; in suitable_migration_target() local
1423 * pageblock, so it's not worth to check order for valid range. in suitable_migration_target()
1425 if (buddy_order_unsafe(page) >= order) in suitable_migration_target()
1521 static int next_search_order(struct compact_control *cc, int order) in next_search_order() argument
1523 order--; in next_search_order()
1524 if (order < 0) in next_search_order()
1525 order = cc->order - 1; in next_search_order()
1528 if (order == cc->search_order) { in next_search_order()
1531 cc->search_order = cc->order - 1; in next_search_order()
1535 return order; in next_search_order()
1547 int order; in fast_isolate_freepages() local
1549 /* Full compaction passes in a negative order */ in fast_isolate_freepages()
1550 if (cc->order <= 0) in fast_isolate_freepages()
1574 * Search starts from the last successful isolation order or the next in fast_isolate_freepages()
1575 * order to search after a previous failure in fast_isolate_freepages()
1577 cc->search_order = min_t(unsigned int, cc->order - 1, cc->search_order); in fast_isolate_freepages()
1579 for (order = cc->search_order; in fast_isolate_freepages()
1580 !page && order >= 0; in fast_isolate_freepages()
1581 order = next_search_order(cc, order)) { in fast_isolate_freepages()
1582 struct free_area *area = &cc->zone->free_area[order]; in fast_isolate_freepages()
1607 cc->search_order = order; in fast_isolate_freepages()
1636 if (__isolate_free_page(page, order)) { in fast_isolate_freepages()
1637 set_page_private(page, order); in fast_isolate_freepages()
1638 nr_isolated = 1 << order; in fast_isolate_freepages()
1642 list_add_tail(&page->lru, &cc->freepages[order]); in fast_isolate_freepages()
1646 order = cc->search_order + 1; in fast_isolate_freepages()
1658 * Smaller scan on next order so the total scan is related in fast_isolate_freepages()
1835 int order = folio_order(src); in compaction_alloc_noprof() local
1842 for (start_order = order; start_order < NR_PAGE_ORDERS; start_order++) in compaction_alloc_noprof()
1861 while (start_order > order) { in compaction_alloc_noprof()
1870 post_alloc_hook(&dst->page, order, __GFP_MOVABLE); in compaction_alloc_noprof()
1871 if (order) in compaction_alloc_noprof()
1872 prep_compound_page(&dst->page, order); in compaction_alloc_noprof()
1873 cc->nr_freepages -= 1 << order; in compaction_alloc_noprof()
1874 cc->nr_migratepages -= 1 << order; in compaction_alloc_noprof()
1891 int order = folio_order(dst); in compaction_free() local
1895 free_pages_prepare(page, order); in compaction_free()
1896 list_add(&dst->lru, &cc->freepages[order]); in compaction_free()
1897 cc->nr_freepages += 1 << order; in compaction_free()
1899 cc->nr_migratepages += 1 << order; in compaction_free()
1963 int order; in fast_find_migrateblock() local
1990 if (cc->order <= PAGE_ALLOC_COSTLY_ORDER) in fast_find_migrateblock()
2013 for (order = cc->order - 1; in fast_find_migrateblock()
2014 order >= PAGE_ALLOC_COSTLY_ORDER && !found_block && nr_scanned < limit; in fast_find_migrateblock()
2015 order--) { in fast_find_migrateblock()
2016 struct free_area *area = &cc->zone->free_area[order]; in fast_find_migrateblock()
2273 unsigned int order; in __compact_finished() local
2316 if (is_via_compact_memory(cc->order)) in __compact_finished()
2330 for (order = cc->order; order < NR_PAGE_ORDERS; order++) { in __compact_finished()
2331 struct free_area *area = &cc->zone->free_area[order]; in __compact_finished()
2348 if (find_suitable_fallback(area, order, migratetype, in __compact_finished()
2373 trace_mm_compaction_finished(cc->zone, cc->order, ret); in compact_finished()
2380 static bool __compaction_suitable(struct zone *zone, int order, in __compaction_suitable() argument
2386 * Watermarks for order-0 must be met for compaction to be able to in __compaction_suitable()
2399 watermark = (order > PAGE_ALLOC_COSTLY_ORDER) ? in __compaction_suitable()
2401 watermark += compact_gap(order); in __compaction_suitable()
2409 bool compaction_suitable(struct zone *zone, int order, int highest_zoneidx) in compaction_suitable() argument
2414 suitable = __compaction_suitable(zone, order, highest_zoneidx, in compaction_suitable()
2421 * watermarks, but we already failed the high-order watermark check in compaction_suitable()
2434 if (order > PAGE_ALLOC_COSTLY_ORDER) { in compaction_suitable()
2435 int fragindex = fragmentation_index(zone, order); in compaction_suitable()
2447 trace_mm_compaction_suitable(zone, order, compact_result); in compaction_suitable()
2452 bool compaction_zonelist_suitable(struct alloc_context *ac, int order, in compaction_zonelist_suitable() argument
2468 * want to trash just for a single high order allocation which in compaction_zonelist_suitable()
2472 available = zone_reclaimable_pages(zone) / order; in compaction_zonelist_suitable()
2474 if (__compaction_suitable(zone, order, ac->highest_zoneidx, in compaction_zonelist_suitable()
2483 * Should we do compaction for target allocation order.
2484 * Return COMPACT_SUCCESS if allocation for target order can be already
2486 * Return COMPACT_SKIPPED if compaction for target order is likely to fail
2487 * Return COMPACT_CONTINUE if compaction for target order should be ran
2490 compaction_suit_allocation_order(struct zone *zone, unsigned int order, in compaction_suit_allocation_order() argument
2496 if (zone_watermark_ok(zone, order, watermark, highest_zoneidx, in compaction_suit_allocation_order()
2500 if (!compaction_suitable(zone, order, highest_zoneidx)) in compaction_suit_allocation_order()
2516 int order; in compact_zone() local
2526 for (order = 0; order < NR_PAGE_ORDERS; order++) in compact_zone()
2527 INIT_LIST_HEAD(&cc->freepages[order]); in compact_zone()
2532 if (!is_via_compact_memory(cc->order)) { in compact_zone()
2533 ret = compaction_suit_allocation_order(cc->zone, cc->order, in compact_zone()
2544 if (compaction_restarting(cc->zone, cc->order)) in compact_zone()
2626 * previous cc->order aligned block. in compact_zone()
2680 if (cc->order == COMPACTION_HPAGE_ORDER) in compact_zone()
2696 * cc->order aligned block where we migrated from? If yes, in compact_zone()
2701 if (cc->order > 0 && last_migrated_pfn) { in compact_zone()
2703 block_start_pfn(cc->migrate_pfn, cc->order); in compact_zone()
2743 static enum compact_result compact_zone_order(struct zone *zone, int order, in compact_zone_order() argument
2750 .order = order, in compact_zone_order()
2751 .search_order = order, in compact_zone_order()
2798 * try_to_compact_pages - Direct compact to satisfy a high-order allocation
2800 * @order: The order of the current allocation
2808 enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order, in try_to_compact_pages() argument
2819 trace_mm_compaction_try_to_compact_pages(order, gfp_mask, prio); in try_to_compact_pages()
2832 && compaction_deferred(zone, order)) { in try_to_compact_pages()
2837 status = compact_zone_order(zone, order, gfp_mask, prio, in try_to_compact_pages()
2849 compaction_defer_reset(zone, order, false); in try_to_compact_pages()
2861 defer_compaction(zone, order); in try_to_compact_pages()
2892 .order = -1, in compact_node()
3050 * order is allocatable. in kcompactd_do_work()
3055 .order = pgdat->kcompactd_max_order, in kcompactd_do_work()
3064 trace_mm_compaction_kcompactd_wake(pgdat->node_id, cc.order, in kcompactd_do_work()
3075 if (compaction_deferred(zone, cc.order)) in kcompactd_do_work()
3079 cc.order, zoneid, ALLOC_WMARK_MIN); in kcompactd_do_work()
3090 compaction_defer_reset(zone, cc.order, false); in kcompactd_do_work()
3095 * order >= cc.order. This is ratelimited by the in kcompactd_do_work()
3104 defer_compaction(zone, cc.order); in kcompactd_do_work()
3115 * the requested order/highest_zoneidx in case it was higher/tighter in kcompactd_do_work()
3118 if (pgdat->kcompactd_max_order <= cc.order) in kcompactd_do_work()
3124 void wakeup_kcompactd(pg_data_t *pgdat, int order, int highest_zoneidx) in wakeup_kcompactd() argument
3126 if (!order) in wakeup_kcompactd()
3129 if (pgdat->kcompactd_max_order < order) in wakeup_kcompactd()
3130 pgdat->kcompactd_max_order = order; in wakeup_kcompactd()
3145 trace_mm_compaction_wakeup_kcompactd(pgdat->node_id, order, in wakeup_kcompactd()