1 // SPDX-License-Identifier: GPL-2.0-only 2 3 #include <linux/mm.h> 4 #include <linux/cma.h> 5 #include <linux/compiler.h> 6 #include <linux/mm_inline.h> 7 8 #include <asm/page.h> 9 #include <asm/setup.h> 10 11 #include <linux/hugetlb.h> 12 #include "internal.h" 13 #include "hugetlb_cma.h" 14 15 16 static struct cma *hugetlb_cma[MAX_NUMNODES] __ro_after_init; 17 static unsigned long hugetlb_cma_size_in_node[MAX_NUMNODES] __initdata; 18 static bool hugetlb_cma_only __ro_after_init; 19 static unsigned long hugetlb_cma_size __ro_after_init; 20 21 void hugetlb_cma_free_frozen_folio(struct folio *folio) 22 { 23 WARN_ON_ONCE(!cma_release_frozen(hugetlb_cma[folio_nid(folio)], 24 &folio->page, folio_nr_pages(folio))); 25 } 26 27 struct folio *hugetlb_cma_alloc_frozen_folio(int order, gfp_t gfp_mask, 28 int nid, nodemask_t *nodemask) 29 { 30 int node; 31 struct folio *folio; 32 struct page *page = NULL; 33 34 if (!hugetlb_cma_size) 35 return NULL; 36 37 if (hugetlb_cma[nid]) 38 page = cma_alloc_frozen_compound(hugetlb_cma[nid], order); 39 40 if (!page && !(gfp_mask & __GFP_THISNODE)) { 41 for_each_node_mask(node, *nodemask) { 42 if (node == nid || !hugetlb_cma[node]) 43 continue; 44 45 page = cma_alloc_frozen_compound(hugetlb_cma[node], order); 46 if (page) 47 break; 48 } 49 } 50 51 if (!page) 52 return NULL; 53 54 folio = page_folio(page); 55 folio_set_hugetlb_cma(folio); 56 return folio; 57 } 58 59 struct huge_bootmem_page * __init 60 hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid, bool node_exact) 61 { 62 struct cma *cma; 63 struct huge_bootmem_page *m; 64 int node = *nid; 65 66 cma = hugetlb_cma[*nid]; 67 m = cma_reserve_early(cma, huge_page_size(h)); 68 if (!m) { 69 if (node_exact) 70 return NULL; 71 72 for_each_node_mask(node, hugetlb_bootmem_nodes) { 73 cma = hugetlb_cma[node]; 74 if (!cma || node == *nid) 75 continue; 76 m = cma_reserve_early(cma, huge_page_size(h)); 77 if (m) { 78 *nid = node; 79 break; 80 } 81 } 82 } 83 84 if (m) { 85 m->flags = HUGE_BOOTMEM_CMA; 86 m->cma = cma; 87 } 88 89 return m; 90 } 91 92 static int __init cmdline_parse_hugetlb_cma(char *p) 93 { 94 int nid, count = 0; 95 unsigned long tmp; 96 char *s = p; 97 98 while (*s) { 99 if (sscanf(s, "%lu%n", &tmp, &count) != 1) 100 break; 101 102 if (s[count] == ':') { 103 if (tmp >= MAX_NUMNODES) 104 break; 105 nid = array_index_nospec(tmp, MAX_NUMNODES); 106 107 s += count + 1; 108 tmp = memparse(s, &s); 109 hugetlb_cma_size_in_node[nid] = tmp; 110 hugetlb_cma_size += tmp; 111 112 /* 113 * Skip the separator if have one, otherwise 114 * break the parsing. 115 */ 116 if (*s == ',') 117 s++; 118 else 119 break; 120 } else { 121 hugetlb_cma_size = memparse(p, &p); 122 break; 123 } 124 } 125 126 return 0; 127 } 128 129 early_param("hugetlb_cma", cmdline_parse_hugetlb_cma); 130 131 static int __init cmdline_parse_hugetlb_cma_only(char *p) 132 { 133 return kstrtobool(p, &hugetlb_cma_only); 134 } 135 136 early_param("hugetlb_cma_only", cmdline_parse_hugetlb_cma_only); 137 138 unsigned int __weak arch_hugetlb_cma_order(void) 139 { 140 return 0; 141 } 142 143 void __init hugetlb_cma_reserve(void) 144 { 145 unsigned long size, reserved, per_node, order; 146 bool node_specific_cma_alloc = false; 147 int nid; 148 149 if (!hugetlb_cma_size) 150 return; 151 152 order = arch_hugetlb_cma_order(); 153 if (!order) { 154 pr_warn("hugetlb_cma: the option isn't supported by current arch\n"); 155 return; 156 } 157 158 /* 159 * HugeTLB CMA reservation is required for gigantic 160 * huge pages which could not be allocated via the 161 * page allocator. Just warn if there is any change 162 * breaking this assumption. 163 */ 164 VM_WARN_ON(order <= MAX_PAGE_ORDER); 165 166 hugetlb_bootmem_set_nodes(); 167 168 for (nid = 0; nid < MAX_NUMNODES; nid++) { 169 if (hugetlb_cma_size_in_node[nid] == 0) 170 continue; 171 172 if (!node_isset(nid, hugetlb_bootmem_nodes)) { 173 pr_warn("hugetlb_cma: invalid node %d specified\n", nid); 174 hugetlb_cma_size -= hugetlb_cma_size_in_node[nid]; 175 hugetlb_cma_size_in_node[nid] = 0; 176 continue; 177 } 178 179 if (hugetlb_cma_size_in_node[nid] < (PAGE_SIZE << order)) { 180 pr_warn("hugetlb_cma: cma area of node %d should be at least %lu MiB\n", 181 nid, (PAGE_SIZE << order) / SZ_1M); 182 hugetlb_cma_size -= hugetlb_cma_size_in_node[nid]; 183 hugetlb_cma_size_in_node[nid] = 0; 184 } else { 185 node_specific_cma_alloc = true; 186 } 187 } 188 189 /* Validate the CMA size again in case some invalid nodes specified. */ 190 if (!hugetlb_cma_size) 191 return; 192 193 if (hugetlb_cma_size < (PAGE_SIZE << order)) { 194 pr_warn("hugetlb_cma: cma area should be at least %lu MiB\n", 195 (PAGE_SIZE << order) / SZ_1M); 196 hugetlb_cma_size = 0; 197 return; 198 } 199 200 if (!node_specific_cma_alloc) { 201 /* 202 * If 3 GB area is requested on a machine with 4 numa nodes, 203 * let's allocate 1 GB on first three nodes and ignore the last one. 204 */ 205 per_node = DIV_ROUND_UP(hugetlb_cma_size, 206 nodes_weight(hugetlb_bootmem_nodes)); 207 pr_info("hugetlb_cma: reserve %lu MiB, up to %lu MiB per node\n", 208 hugetlb_cma_size / SZ_1M, per_node / SZ_1M); 209 } 210 211 reserved = 0; 212 for_each_node_mask(nid, hugetlb_bootmem_nodes) { 213 int res; 214 char name[CMA_MAX_NAME]; 215 216 if (node_specific_cma_alloc) { 217 if (hugetlb_cma_size_in_node[nid] == 0) 218 continue; 219 220 size = hugetlb_cma_size_in_node[nid]; 221 } else { 222 size = min(per_node, hugetlb_cma_size - reserved); 223 } 224 225 size = round_up(size, PAGE_SIZE << order); 226 227 snprintf(name, sizeof(name), "hugetlb%d", nid); 228 /* 229 * Note that 'order per bit' is based on smallest size that 230 * may be returned to CMA allocator in the case of 231 * huge page demotion. 232 */ 233 res = cma_declare_contiguous_multi(size, PAGE_SIZE << order, 234 HUGETLB_PAGE_ORDER, name, 235 &hugetlb_cma[nid], nid); 236 if (res) { 237 pr_warn("hugetlb_cma: reservation failed: err %d, node %d", 238 res, nid); 239 continue; 240 } 241 242 reserved += size; 243 pr_info("hugetlb_cma: reserved %lu MiB on node %d\n", 244 size / SZ_1M, nid); 245 246 if (reserved >= hugetlb_cma_size) 247 break; 248 } 249 250 if (!reserved) 251 /* 252 * hugetlb_cma_size is used to determine if allocations from 253 * cma are possible. Set to zero if no cma regions are set up. 254 */ 255 hugetlb_cma_size = 0; 256 } 257 258 bool hugetlb_cma_exclusive_alloc(void) 259 { 260 return hugetlb_cma_only; 261 } 262 263 unsigned long __init hugetlb_cma_total_size(void) 264 { 265 return hugetlb_cma_size; 266 } 267 268 void __init hugetlb_cma_validate_params(void) 269 { 270 if (!hugetlb_cma_size) 271 hugetlb_cma_only = false; 272 } 273 274 bool __init hugetlb_early_cma(struct hstate *h) 275 { 276 if (arch_has_huge_bootmem_alloc()) 277 return false; 278 279 return hstate_is_gigantic(h) && hugetlb_cma_only; 280 } 281