1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * mm-internal API for the page (buddy) allocator. Public API lives in 4 * include/linux/gfp.h. 5 */ 6 #ifndef __MM_PAGE_ALLOC_H 7 #define __MM_PAGE_ALLOC_H 8 9 #include <linux/mm.h> 10 #include <linux/mmzone.h> 11 #include <linux/nodemask.h> 12 #include <linux/types.h> 13 14 #define ALLOC_DEFAULT 0 15 /* The ALLOC_WMARK bits are used as an index to zone->watermark */ 16 #define ALLOC_WMARK_MIN WMARK_MIN 17 #define ALLOC_WMARK_LOW WMARK_LOW 18 #define ALLOC_WMARK_HIGH WMARK_HIGH 19 #define ALLOC_NO_WATERMARKS 0x04 /* don't check watermarks at all */ 20 21 /* Mask to get the watermark bits */ 22 #define ALLOC_WMARK_MASK (ALLOC_NO_WATERMARKS-1) 23 24 /* 25 * Only MMU archs have async oom victim reclaim - aka oom_reaper so we 26 * cannot assume a reduced access to memory reserves is sufficient for 27 * !MMU 28 */ 29 #ifdef CONFIG_MMU 30 #define ALLOC_OOM 0x08 31 #else 32 #define ALLOC_OOM ALLOC_NO_WATERMARKS 33 #endif 34 35 #define ALLOC_NON_BLOCK 0x10 /* Caller cannot block. Allow access 36 * to 25% of the min watermark or 37 * 62.5% if __GFP_HIGH is set. 38 */ 39 #define ALLOC_MIN_RESERVE 0x20 /* __GFP_HIGH set. Allow access to 50% 40 * of the min watermark. 41 */ 42 #define ALLOC_CPUSET 0x40 /* check for correct cpuset */ 43 #define ALLOC_CMA 0x80 /* allow allocations from CMA areas */ 44 #ifdef CONFIG_ZONE_DMA32 45 #define ALLOC_NOFRAGMENT 0x100 /* avoid mixing pageblock types */ 46 #else 47 #define ALLOC_NOFRAGMENT 0x0 48 #endif 49 #define ALLOC_HIGHATOMIC 0x200 /* Allows access to MIGRATE_HIGHATOMIC */ 50 #define ALLOC_NOLOCK 0x400 /* Only use spin_trylock in allocation path */ 51 #define ALLOC_KSWAPD 0x800 /* allow waking of kswapd, __GFP_KSWAPD_RECLAIM set */ 52 /* 53 * Avoid alloc_tag recursion for internal allocations. 54 * 55 * Callers must clear_page_tag_ref() before freeing to avoid warnings from 56 * alloc_tag_sub_check(). 57 */ 58 #define ALLOC_NO_CODETAG 0x1000 59 60 /* Flags that allow allocations below the min watermark. */ 61 #define ALLOC_RESERVES (ALLOC_NON_BLOCK|ALLOC_MIN_RESERVE|ALLOC_HIGHATOMIC|ALLOC_OOM) 62 63 /* 64 * Structure for holding the mostly immutable allocation parameters passed 65 * between functions involved in allocations, including the alloc_pages* 66 * family of functions. 67 * 68 * nodemask, migratetype and highest_zoneidx are initialized only once in 69 * __alloc_pages() and then never change. 70 * 71 * zonelist, preferred_zone and highest_zoneidx are set first in 72 * __alloc_pages() for the fast path, and might be later changed 73 * in __alloc_pages_slowpath(). All other functions pass the whole structure 74 * by a const pointer. 75 */ 76 struct alloc_context { 77 struct zonelist *zonelist; 78 const nodemask_t *nodemask; 79 struct zoneref *preferred_zoneref; 80 int migratetype; 81 82 /* 83 * highest_zoneidx represents highest usable zone index of 84 * the allocation request. Due to the nature of the zone, 85 * memory on lower zone than the highest_zoneidx will be 86 * protected by lowmem_reserve[highest_zoneidx]. 87 * 88 * highest_zoneidx is also used by reclaim/compaction to limit 89 * the target zone since higher zone than this index cannot be 90 * usable for this allocation request. 91 */ 92 enum zone_type highest_zoneidx; 93 bool spread_dirty_pages; 94 /* Only flags that are global to the whole allocation go here. */ 95 unsigned int alloc_flags; 96 }; 97 98 /* 99 * This function returns the order of a free page in the buddy system. In 100 * general, page_zone(page)->lock must be held by the caller to prevent the 101 * page from being allocated in parallel and returning garbage as the order. 102 * If a caller does not hold page_zone(page)->lock, it must guarantee that the 103 * page cannot be allocated or merged in parallel. Alternatively, it must 104 * handle invalid values gracefully, and use buddy_order_unsafe() below. 105 */ 106 static inline unsigned int buddy_order(struct page *page) 107 { 108 /* PageBuddy() must be checked by the caller */ 109 return page_private(page); 110 } 111 112 /* 113 * Like buddy_order(), but for callers who cannot afford to hold the zone lock. 114 * PageBuddy() should be checked first by the caller to minimize race window, 115 * and invalid values must be handled gracefully. 116 * 117 * READ_ONCE is used so that if the caller assigns the result into a local 118 * variable and e.g. tests it for valid range before using, the compiler cannot 119 * decide to remove the variable and inline the page_private(page) multiple 120 * times, potentially observing different values in the tests and the actual 121 * use of the result. 122 */ 123 #define buddy_order_unsafe(page) READ_ONCE(page_private(page)) 124 125 /* 126 * This function checks whether a page is free && is the buddy 127 * we can coalesce a page and its buddy if 128 * (a) the buddy is not in a hole (check before calling!) && 129 * (b) the buddy is in the buddy system && 130 * (c) a page and its buddy have the same order && 131 * (d) a page and its buddy are in the same zone. 132 * 133 * For recording whether a page is in the buddy system, we set PageBuddy. 134 * Setting, clearing, and testing PageBuddy is serialized by zone->lock. 135 * 136 * For recording page's order, we use page_private(page). 137 */ 138 static inline bool page_is_buddy(struct page *page, struct page *buddy, 139 unsigned int order) 140 { 141 if (!page_is_guard(buddy) && !PageBuddy(buddy)) 142 return false; 143 144 if (buddy_order(buddy) != order) 145 return false; 146 147 /* 148 * zone check is done late to avoid uselessly calculating 149 * zone/node ids for pages that could never merge. 150 */ 151 if (page_zone_id(page) != page_zone_id(buddy)) 152 return false; 153 154 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); 155 156 return true; 157 } 158 159 /* 160 * Locate the struct page for both the matching buddy in our 161 * pair (buddy1) and the combined O(n+1) page they form (page). 162 * 163 * 1) Any buddy B1 will have an order O twin B2 which satisfies 164 * the following equation: 165 * B2 = B1 ^ (1 << O) 166 * For example, if the starting buddy (buddy2) is #8 its order 167 * 1 buddy is #10: 168 * B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10 169 * 170 * 2) Any buddy B will have an order O+1 parent P which 171 * satisfies the following equation: 172 * P = B & ~(1 << O) 173 * 174 * Assumption: *_mem_map is contiguous at least up to MAX_PAGE_ORDER 175 */ 176 static inline unsigned long 177 __find_buddy_pfn(unsigned long page_pfn, unsigned int order) 178 { 179 return page_pfn ^ (1 << order); 180 } 181 182 /* 183 * Find the buddy of @page and validate it. 184 * @page: The input page 185 * @pfn: The pfn of the page, it saves a call to page_to_pfn() when the 186 * function is used in the performance-critical __free_one_page(). 187 * @order: The order of the page 188 * @buddy_pfn: The output pointer to the buddy pfn, it also saves a call to 189 * page_to_pfn(). 190 * 191 * The found buddy can be a non PageBuddy, out of @page's zone, or its order is 192 * not the same as @page. The validation is necessary before use it. 193 * 194 * Return: the found buddy page or NULL if not found. 195 */ 196 static inline struct page *find_buddy_page_pfn(struct page *page, 197 unsigned long pfn, unsigned int order, unsigned long *buddy_pfn) 198 { 199 unsigned long __buddy_pfn = __find_buddy_pfn(pfn, order); 200 struct page *buddy; 201 202 buddy = page + (__buddy_pfn - pfn); 203 if (buddy_pfn) 204 *buddy_pfn = __buddy_pfn; 205 206 if (page_is_buddy(page, buddy, order)) 207 return buddy; 208 return NULL; 209 } 210 211 extern struct page *__pageblock_pfn_to_page(unsigned long start_pfn, 212 unsigned long end_pfn, struct zone *zone); 213 214 static inline struct page *pageblock_pfn_to_page(unsigned long start_pfn, 215 unsigned long end_pfn, struct zone *zone) 216 { 217 if (zone->contiguous) 218 return pfn_to_page(start_pfn); 219 220 return __pageblock_pfn_to_page(start_pfn, end_pfn, zone); 221 } 222 223 extern void __free_pages_core(struct page *page, unsigned int order, 224 enum meminit_context context); 225 226 void post_alloc_hook(struct page *page, unsigned int order, gfp_t gfp_flags, 227 unsigned int alloc_flags); 228 extern bool free_pages_prepare(struct page *page, unsigned int order); 229 230 extern int user_min_free_kbytes; 231 232 struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, int nid, 233 nodemask_t *nodemask, unsigned int alloc_flags); 234 #define __alloc_frozen_pages(...) \ 235 alloc_hooks(__alloc_frozen_pages_noprof(__VA_ARGS__)) 236 void free_frozen_pages(struct page *page, unsigned int order); 237 void free_unref_folios(struct folio_batch *fbatch); 238 239 #ifdef CONFIG_NUMA 240 struct page *alloc_frozen_pages_noprof(gfp_t, unsigned int order); 241 #else 242 static inline struct page *alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order) 243 { 244 return __alloc_frozen_pages_noprof(gfp, order, numa_node_id(), NULL, 245 ALLOC_DEFAULT); 246 } 247 #endif 248 249 #define alloc_frozen_pages(...) \ 250 alloc_hooks(alloc_frozen_pages_noprof(__VA_ARGS__)) 251 252 struct page *alloc_frozen_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned int order); 253 #define alloc_frozen_pages_nolock(...) \ 254 alloc_hooks(alloc_frozen_pages_nolock_noprof(__VA_ARGS__)) 255 void free_frozen_pages_nolock(struct page *page, unsigned int order); 256 257 struct page *__alloc_pages_noprof(gfp_t gfp, unsigned int order, int preferred_nid, 258 nodemask_t *nodemask, unsigned int alloc_flags); 259 #define __alloc_pages(...) alloc_hooks(__alloc_pages_noprof(__VA_ARGS__)) 260 261 extern void zone_pcp_reset(struct zone *zone); 262 extern void zone_pcp_disable(struct zone *zone); 263 extern void zone_pcp_enable(struct zone *zone); 264 extern void zone_pcp_init(struct zone *zone); 265 266 enum fallback_result { 267 /* Found suitable migratetype, *mt_out is valid. */ 268 FALLBACK_FOUND, 269 /* No fallback found in requested order. */ 270 FALLBACK_EMPTY, 271 /* Passed @claimable, but claiming whole block is a bad idea. */ 272 FALLBACK_NOCLAIM, 273 }; 274 enum fallback_result 275 find_suitable_fallback(struct free_area *area, unsigned int order, 276 int migratetype, bool claimable, int *mt_out); 277 278 static inline bool free_area_empty(struct free_area *area, int migratetype) 279 { 280 return list_empty(&area->free_list[migratetype]); 281 } 282 283 /* Convert GFP flags to their corresponding migrate type */ 284 #define GFP_MOVABLE_MASK (__GFP_RECLAIMABLE|__GFP_MOVABLE) 285 #define GFP_MOVABLE_SHIFT 3 286 287 static inline int gfp_migratetype(const gfp_t gfp_flags) 288 { 289 VM_WARN_ON((gfp_flags & GFP_MOVABLE_MASK) == GFP_MOVABLE_MASK); 290 BUILD_BUG_ON((1UL << GFP_MOVABLE_SHIFT) != ___GFP_MOVABLE); 291 BUILD_BUG_ON((___GFP_MOVABLE >> GFP_MOVABLE_SHIFT) != MIGRATE_MOVABLE); 292 BUILD_BUG_ON((___GFP_RECLAIMABLE >> GFP_MOVABLE_SHIFT) != MIGRATE_RECLAIMABLE); 293 BUILD_BUG_ON(((___GFP_MOVABLE | ___GFP_RECLAIMABLE) >> 294 GFP_MOVABLE_SHIFT) != MIGRATE_HIGHATOMIC); 295 296 if (unlikely(page_group_by_mobility_disabled)) 297 return MIGRATE_UNMOVABLE; 298 299 /* Group based on mobility */ 300 return (__force unsigned long)(gfp_flags & GFP_MOVABLE_MASK) >> GFP_MOVABLE_SHIFT; 301 } 302 #undef GFP_MOVABLE_MASK 303 #undef GFP_MOVABLE_SHIFT 304 305 bool decay_pcp_high(struct zone *zone, struct per_cpu_pages *pcp); 306 void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp); 307 void drain_all_pages(struct zone *zone); 308 309 void page_alloc_init_cpuhp(void); 310 void page_alloc_sysctl_init(void); 311 312 #endif /* __MM_PAGE_ALLOC_H */ 313