1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Page fragment allocator 3 * 4 * Page Fragment: 5 * An arbitrary-length arbitrary-offset area of memory which resides within a 6 * 0 or higher order page. Multiple fragments within that page are 7 * individually refcounted, in the page's reference counter. 8 * 9 * The page_frag functions provide a simple allocation framework for page 10 * fragments. This is used by the network stack and network device drivers to 11 * provide a backing region of memory for use as either an sk_buff->head, or to 12 * be used in the "frags" portion of skb_shared_info. 13 */ 14 15 #include <linux/build_bug.h> 16 #include <linux/export.h> 17 #include <linux/gfp_types.h> 18 #include <linux/init.h> 19 #include <linux/mm.h> 20 #include <linux/page_frag_cache.h> 21 #include "internal.h" 22 23 static unsigned long encoded_page_create(struct page *page, unsigned int order, 24 bool pfmemalloc) 25 { 26 BUILD_BUG_ON(PAGE_FRAG_CACHE_MAX_ORDER > PAGE_FRAG_CACHE_ORDER_MASK); 27 BUILD_BUG_ON(PAGE_FRAG_CACHE_PFMEMALLOC_BIT >= PAGE_SIZE); 28 29 return (unsigned long)page_address(page) | 30 (order & PAGE_FRAG_CACHE_ORDER_MASK) | 31 ((unsigned long)pfmemalloc * PAGE_FRAG_CACHE_PFMEMALLOC_BIT); 32 } 33 34 static unsigned long encoded_page_decode_order(unsigned long encoded_page) 35 { 36 return encoded_page & PAGE_FRAG_CACHE_ORDER_MASK; 37 } 38 39 static void *encoded_page_decode_virt(unsigned long encoded_page) 40 { 41 return (void *)(encoded_page & PAGE_MASK); 42 } 43 44 static struct page *encoded_page_decode_page(unsigned long encoded_page) 45 { 46 return virt_to_page((void *)encoded_page); 47 } 48 49 static struct page *__page_frag_cache_refill(struct page_frag_cache *nc, 50 gfp_t gfp_mask) 51 { 52 unsigned long order = PAGE_FRAG_CACHE_MAX_ORDER; 53 struct page *page = NULL; 54 gfp_t gfp = gfp_mask; 55 56 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 57 gfp_mask = (gfp_mask & ~__GFP_DIRECT_RECLAIM) | __GFP_COMP | 58 __GFP_NOWARN | __GFP_NORETRY | __GFP_NOMEMALLOC; 59 page = __alloc_pages(gfp_mask, PAGE_FRAG_CACHE_MAX_ORDER, 60 numa_mem_id(), NULL); 61 #endif 62 if (unlikely(!page)) { 63 page = __alloc_pages(gfp, 0, numa_mem_id(), NULL); 64 order = 0; 65 } 66 67 nc->encoded_page = page ? 68 encoded_page_create(page, order, page_is_pfmemalloc(page)) : 0; 69 70 return page; 71 } 72 73 void page_frag_cache_drain(struct page_frag_cache *nc) 74 { 75 if (!nc->encoded_page) 76 return; 77 78 __page_frag_cache_drain(encoded_page_decode_page(nc->encoded_page), 79 nc->pagecnt_bias); 80 nc->encoded_page = 0; 81 } 82 EXPORT_SYMBOL(page_frag_cache_drain); 83 84 void __page_frag_cache_drain(struct page *page, unsigned int count) 85 { 86 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page); 87 88 if (page_ref_sub_and_test(page, count)) 89 free_unref_page(page, compound_order(page)); 90 } 91 EXPORT_SYMBOL(__page_frag_cache_drain); 92 93 void *__page_frag_alloc_align(struct page_frag_cache *nc, 94 unsigned int fragsz, gfp_t gfp_mask, 95 unsigned int align_mask) 96 { 97 unsigned long encoded_page = nc->encoded_page; 98 unsigned int size, offset; 99 struct page *page; 100 101 if (unlikely(!encoded_page)) { 102 refill: 103 page = __page_frag_cache_refill(nc, gfp_mask); 104 if (!page) 105 return NULL; 106 107 encoded_page = nc->encoded_page; 108 109 /* Even if we own the page, we do not use atomic_set(). 110 * This would break get_page_unless_zero() users. 111 */ 112 page_ref_add(page, PAGE_FRAG_CACHE_MAX_SIZE); 113 114 /* reset page count bias and offset to start of new frag */ 115 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; 116 nc->offset = 0; 117 } 118 119 size = PAGE_SIZE << encoded_page_decode_order(encoded_page); 120 offset = __ALIGN_KERNEL_MASK(nc->offset, ~align_mask); 121 if (unlikely(offset + fragsz > size)) { 122 if (unlikely(fragsz > PAGE_SIZE)) { 123 /* 124 * The caller is trying to allocate a fragment 125 * with fragsz > PAGE_SIZE but the cache isn't big 126 * enough to satisfy the request, this may 127 * happen in low memory conditions. 128 * We don't release the cache page because 129 * it could make memory pressure worse 130 * so we simply return NULL here. 131 */ 132 return NULL; 133 } 134 135 page = encoded_page_decode_page(encoded_page); 136 137 if (!page_ref_sub_and_test(page, nc->pagecnt_bias)) 138 goto refill; 139 140 if (unlikely(encoded_page_decode_pfmemalloc(encoded_page))) { 141 free_unref_page(page, 142 encoded_page_decode_order(encoded_page)); 143 goto refill; 144 } 145 146 /* OK, page count is 0, we can safely set it */ 147 set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1); 148 149 /* reset page count bias and offset to start of new frag */ 150 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; 151 offset = 0; 152 } 153 154 nc->pagecnt_bias--; 155 nc->offset = offset + fragsz; 156 157 return encoded_page_decode_virt(encoded_page) + offset; 158 } 159 EXPORT_SYMBOL(__page_frag_alloc_align); 160 161 /* 162 * Frees a page fragment allocated out of either a compound or order 0 page. 163 */ 164 void page_frag_free(void *addr) 165 { 166 struct page *page = virt_to_head_page(addr); 167 168 if (unlikely(put_page_testzero(page))) 169 free_unref_page(page, compound_order(page)); 170 } 171 EXPORT_SYMBOL(page_frag_free); 172