1 // SPDX-License-Identifier: GPL-2.0-only 2 3 #include <linux/mm.h> 4 #include <linux/cma.h> 5 #include <linux/compiler.h> 6 #include <linux/mm_inline.h> 7 8 #include <asm/page.h> 9 #include <asm/setup.h> 10 11 #include <linux/hugetlb.h> 12 #include "internal.h" 13 #include "hugetlb_cma.h" 14 15 16 static struct cma *hugetlb_cma[MAX_NUMNODES] __ro_after_init; 17 static unsigned long hugetlb_cma_size_in_node[MAX_NUMNODES] __initdata; 18 static bool hugetlb_cma_only __ro_after_init; 19 static unsigned long hugetlb_cma_size __ro_after_init; 20 21 void hugetlb_cma_free_frozen_folio(struct folio *folio) 22 { 23 WARN_ON_ONCE(!cma_release_frozen(hugetlb_cma[folio_nid(folio)], 24 &folio->page, folio_nr_pages(folio))); 25 } 26 27 struct folio *hugetlb_cma_alloc_frozen_folio(int order, gfp_t gfp_mask, 28 int nid, nodemask_t *nodemask) 29 { 30 int node; 31 struct folio *folio; 32 struct page *page = NULL; 33 34 if (!hugetlb_cma_size) 35 return NULL; 36 37 if (hugetlb_cma[nid]) 38 page = cma_alloc_frozen_compound(hugetlb_cma[nid], order); 39 40 if (!page && !(gfp_mask & __GFP_THISNODE)) { 41 for_each_node_mask(node, *nodemask) { 42 if (node == nid || !hugetlb_cma[node]) 43 continue; 44 45 page = cma_alloc_frozen_compound(hugetlb_cma[node], order); 46 if (page) 47 break; 48 } 49 } 50 51 if (!page) 52 return NULL; 53 54 folio = page_folio(page); 55 folio_set_hugetlb_cma(folio); 56 return folio; 57 } 58 59 struct huge_bootmem_page * __init 60 hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid, bool node_exact) 61 { 62 struct cma *cma; 63 struct huge_bootmem_page *m; 64 int node = *nid; 65 66 cma = hugetlb_cma[*nid]; 67 m = cma_reserve_early(cma, huge_page_size(h)); 68 if (!m) { 69 if (node_exact) 70 return NULL; 71 72 for_each_node_mask(node, hugetlb_bootmem_nodes) { 73 cma = hugetlb_cma[node]; 74 if (!cma || node == *nid) 75 continue; 76 m = cma_reserve_early(cma, huge_page_size(h)); 77 if (m) { 78 *nid = node; 79 break; 80 } 81 } 82 } 83 84 if (m) { 85 m->flags = HUGE_BOOTMEM_CMA; 86 m->cma = cma; 87 } 88 89 return m; 90 } 91 92 static int __init cmdline_parse_hugetlb_cma(char *p) 93 { 94 int nid, count = 0; 95 unsigned long tmp; 96 char *s = p; 97 98 while (*s) { 99 if (sscanf(s, "%lu%n", &tmp, &count) != 1) 100 break; 101 102 if (s[count] == ':') { 103 if (tmp >= MAX_NUMNODES) 104 break; 105 nid = array_index_nospec(tmp, MAX_NUMNODES); 106 107 s += count + 1; 108 tmp = memparse(s, &s); 109 hugetlb_cma_size_in_node[nid] = tmp; 110 hugetlb_cma_size += tmp; 111 112 /* 113 * Skip the separator if have one, otherwise 114 * break the parsing. 115 */ 116 if (*s == ',') 117 s++; 118 else 119 break; 120 } else { 121 hugetlb_cma_size = memparse(p, &p); 122 break; 123 } 124 } 125 126 return 0; 127 } 128 129 early_param("hugetlb_cma", cmdline_parse_hugetlb_cma); 130 131 static int __init cmdline_parse_hugetlb_cma_only(char *p) 132 { 133 return kstrtobool(p, &hugetlb_cma_only); 134 } 135 136 early_param("hugetlb_cma_only", cmdline_parse_hugetlb_cma_only); 137 138 unsigned int __weak arch_hugetlb_cma_order(void) 139 { 140 return 0; 141 } 142 143 void __init hugetlb_cma_reserve(void) 144 { 145 unsigned long size, reserved, per_node, order, gigantic_page_size; 146 bool node_specific_cma_alloc = false; 147 int nid; 148 149 if (!hugetlb_cma_size) 150 return; 151 152 order = arch_hugetlb_cma_order(); 153 if (!order) { 154 pr_warn("hugetlb_cma: the option isn't supported by current arch\n"); 155 return; 156 } 157 158 /* 159 * HugeTLB CMA reservation is required for gigantic 160 * huge pages which could not be allocated via the 161 * page allocator. Just warn if there is any change 162 * breaking this assumption. 163 */ 164 VM_WARN_ON(order <= MAX_PAGE_ORDER); 165 gigantic_page_size = PAGE_SIZE << order; 166 167 hugetlb_bootmem_set_nodes(); 168 169 for (nid = 0; nid < MAX_NUMNODES; nid++) { 170 size = hugetlb_cma_size_in_node[nid]; 171 if (size == 0) 172 continue; 173 174 if (!node_isset(nid, hugetlb_bootmem_nodes)) { 175 pr_warn("hugetlb_cma: invalid node %d specified\n", nid); 176 } else if (!IS_ALIGNED(size, gigantic_page_size)) { 177 pr_warn("hugetlb_cma: cma area of node %d must be a multiple of %lu MiB\n", 178 nid, gigantic_page_size / SZ_1M); 179 } else { 180 node_specific_cma_alloc = true; 181 continue; 182 } 183 184 hugetlb_cma_size -= size; 185 hugetlb_cma_size_in_node[nid] = 0; 186 } 187 188 /* Validate the CMA size again in case some invalid nodes specified. */ 189 if (!hugetlb_cma_size) 190 return; 191 192 if (!IS_ALIGNED(hugetlb_cma_size, gigantic_page_size)) { 193 pr_warn("hugetlb_cma: cma area must be a multiple of %lu MiB\n", 194 gigantic_page_size / SZ_1M); 195 hugetlb_cma_size = 0; 196 return; 197 } 198 199 if (!node_specific_cma_alloc) { 200 /* 201 * If 3 GB area is requested on a machine with 4 numa nodes, 202 * let's allocate 1 GB on first three nodes and ignore the last one. 203 */ 204 per_node = DIV_ROUND_UP(hugetlb_cma_size, 205 nodes_weight(hugetlb_bootmem_nodes)); 206 per_node = round_up(per_node, gigantic_page_size); 207 pr_info("hugetlb_cma: reserve %lu MiB, up to %lu MiB per node\n", 208 hugetlb_cma_size / SZ_1M, per_node / SZ_1M); 209 } 210 211 reserved = 0; 212 for_each_node_mask(nid, hugetlb_bootmem_nodes) { 213 int res; 214 char name[CMA_MAX_NAME]; 215 216 if (node_specific_cma_alloc) { 217 if (hugetlb_cma_size_in_node[nid] == 0) 218 continue; 219 220 size = hugetlb_cma_size_in_node[nid]; 221 } else { 222 size = min(per_node, hugetlb_cma_size - reserved); 223 } 224 225 snprintf(name, sizeof(name), "hugetlb%d", nid); 226 /* 227 * Note that 'order per bit' is based on smallest size that 228 * may be returned to CMA allocator in the case of 229 * huge page demotion. 230 */ 231 res = cma_declare_contiguous_multi(size, gigantic_page_size, 232 HUGETLB_PAGE_ORDER, name, 233 &hugetlb_cma[nid], nid); 234 if (res) { 235 pr_warn("hugetlb_cma: reservation failed: err %d, node %d", 236 res, nid); 237 continue; 238 } 239 240 reserved += size; 241 pr_info("hugetlb_cma: reserved %lu MiB on node %d\n", 242 size / SZ_1M, nid); 243 244 if (reserved >= hugetlb_cma_size) 245 break; 246 } 247 248 if (!reserved) 249 /* 250 * hugetlb_cma_size is used to determine if allocations from 251 * cma are possible. Set to zero if no cma regions are set up. 252 */ 253 hugetlb_cma_size = 0; 254 } 255 256 bool hugetlb_cma_exclusive_alloc(void) 257 { 258 return hugetlb_cma_only; 259 } 260 261 unsigned long __init hugetlb_cma_total_size(void) 262 { 263 return hugetlb_cma_size; 264 } 265 266 void __init hugetlb_cma_validate_params(void) 267 { 268 if (!hugetlb_cma_size) 269 hugetlb_cma_only = false; 270 } 271 272 bool __init hugetlb_early_cma(struct hstate *h) 273 { 274 if (arch_has_huge_bootmem_alloc()) 275 return false; 276 277 return hstate_is_gigantic(h) && hugetlb_cma_only; 278 } 279