1 // SPDX-License-Identifier: GPL-2.0-only 2 3 #include <linux/mm.h> 4 #include <linux/cma.h> 5 #include <linux/compiler.h> 6 #include <linux/mm_inline.h> 7 8 #include <asm/page.h> 9 #include <asm/setup.h> 10 11 #include <linux/hugetlb.h> 12 #include "internal.h" 13 #include "hugetlb_cma.h" 14 15 16 static struct cma *hugetlb_cma[MAX_NUMNODES]; 17 static unsigned long hugetlb_cma_size_in_node[MAX_NUMNODES] __initdata; 18 static bool hugetlb_cma_only; 19 static unsigned long hugetlb_cma_size __initdata; 20 21 void hugetlb_cma_free_frozen_folio(struct folio *folio) 22 { 23 WARN_ON_ONCE(!cma_release_frozen(hugetlb_cma[folio_nid(folio)], 24 &folio->page, folio_nr_pages(folio))); 25 } 26 27 struct folio *hugetlb_cma_alloc_frozen_folio(int order, gfp_t gfp_mask, 28 int nid, nodemask_t *nodemask) 29 { 30 int node; 31 struct folio *folio; 32 struct page *page = NULL; 33 34 if (hugetlb_cma[nid]) 35 page = cma_alloc_frozen_compound(hugetlb_cma[nid], order); 36 37 if (!page && !(gfp_mask & __GFP_THISNODE)) { 38 for_each_node_mask(node, *nodemask) { 39 if (node == nid || !hugetlb_cma[node]) 40 continue; 41 42 page = cma_alloc_frozen_compound(hugetlb_cma[node], order); 43 if (page) 44 break; 45 } 46 } 47 48 if (!page) 49 return NULL; 50 51 folio = page_folio(page); 52 folio_set_hugetlb_cma(folio); 53 return folio; 54 } 55 56 struct huge_bootmem_page * __init 57 hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid, bool node_exact) 58 { 59 struct cma *cma; 60 struct huge_bootmem_page *m; 61 int node = *nid; 62 63 cma = hugetlb_cma[*nid]; 64 m = cma_reserve_early(cma, huge_page_size(h)); 65 if (!m) { 66 if (node_exact) 67 return NULL; 68 69 for_each_node_mask(node, hugetlb_bootmem_nodes) { 70 cma = hugetlb_cma[node]; 71 if (!cma || node == *nid) 72 continue; 73 m = cma_reserve_early(cma, huge_page_size(h)); 74 if (m) { 75 *nid = node; 76 break; 77 } 78 } 79 } 80 81 if (m) { 82 m->flags = HUGE_BOOTMEM_CMA; 83 m->cma = cma; 84 } 85 86 return m; 87 } 88 89 static int __init cmdline_parse_hugetlb_cma(char *p) 90 { 91 int nid, count = 0; 92 unsigned long tmp; 93 char *s = p; 94 95 while (*s) { 96 if (sscanf(s, "%lu%n", &tmp, &count) != 1) 97 break; 98 99 if (s[count] == ':') { 100 if (tmp >= MAX_NUMNODES) 101 break; 102 nid = array_index_nospec(tmp, MAX_NUMNODES); 103 104 s += count + 1; 105 tmp = memparse(s, &s); 106 hugetlb_cma_size_in_node[nid] = tmp; 107 hugetlb_cma_size += tmp; 108 109 /* 110 * Skip the separator if have one, otherwise 111 * break the parsing. 112 */ 113 if (*s == ',') 114 s++; 115 else 116 break; 117 } else { 118 hugetlb_cma_size = memparse(p, &p); 119 break; 120 } 121 } 122 123 return 0; 124 } 125 126 early_param("hugetlb_cma", cmdline_parse_hugetlb_cma); 127 128 static int __init cmdline_parse_hugetlb_cma_only(char *p) 129 { 130 return kstrtobool(p, &hugetlb_cma_only); 131 } 132 133 early_param("hugetlb_cma_only", cmdline_parse_hugetlb_cma_only); 134 135 unsigned int __weak arch_hugetlb_cma_order(void) 136 { 137 return 0; 138 } 139 140 void __init hugetlb_cma_reserve(void) 141 { 142 unsigned long size, reserved, per_node, order; 143 bool node_specific_cma_alloc = false; 144 int nid; 145 146 if (!hugetlb_cma_size) 147 return; 148 149 order = arch_hugetlb_cma_order(); 150 if (!order) { 151 pr_warn("hugetlb_cma: the option isn't supported by current arch\n"); 152 return; 153 } 154 155 /* 156 * HugeTLB CMA reservation is required for gigantic 157 * huge pages which could not be allocated via the 158 * page allocator. Just warn if there is any change 159 * breaking this assumption. 160 */ 161 VM_WARN_ON(order <= MAX_PAGE_ORDER); 162 163 hugetlb_bootmem_set_nodes(); 164 165 for (nid = 0; nid < MAX_NUMNODES; nid++) { 166 if (hugetlb_cma_size_in_node[nid] == 0) 167 continue; 168 169 if (!node_isset(nid, hugetlb_bootmem_nodes)) { 170 pr_warn("hugetlb_cma: invalid node %d specified\n", nid); 171 hugetlb_cma_size -= hugetlb_cma_size_in_node[nid]; 172 hugetlb_cma_size_in_node[nid] = 0; 173 continue; 174 } 175 176 if (hugetlb_cma_size_in_node[nid] < (PAGE_SIZE << order)) { 177 pr_warn("hugetlb_cma: cma area of node %d should be at least %lu MiB\n", 178 nid, (PAGE_SIZE << order) / SZ_1M); 179 hugetlb_cma_size -= hugetlb_cma_size_in_node[nid]; 180 hugetlb_cma_size_in_node[nid] = 0; 181 } else { 182 node_specific_cma_alloc = true; 183 } 184 } 185 186 /* Validate the CMA size again in case some invalid nodes specified. */ 187 if (!hugetlb_cma_size) 188 return; 189 190 if (hugetlb_cma_size < (PAGE_SIZE << order)) { 191 pr_warn("hugetlb_cma: cma area should be at least %lu MiB\n", 192 (PAGE_SIZE << order) / SZ_1M); 193 hugetlb_cma_size = 0; 194 return; 195 } 196 197 if (!node_specific_cma_alloc) { 198 /* 199 * If 3 GB area is requested on a machine with 4 numa nodes, 200 * let's allocate 1 GB on first three nodes and ignore the last one. 201 */ 202 per_node = DIV_ROUND_UP(hugetlb_cma_size, 203 nodes_weight(hugetlb_bootmem_nodes)); 204 pr_info("hugetlb_cma: reserve %lu MiB, up to %lu MiB per node\n", 205 hugetlb_cma_size / SZ_1M, per_node / SZ_1M); 206 } 207 208 reserved = 0; 209 for_each_node_mask(nid, hugetlb_bootmem_nodes) { 210 int res; 211 char name[CMA_MAX_NAME]; 212 213 if (node_specific_cma_alloc) { 214 if (hugetlb_cma_size_in_node[nid] == 0) 215 continue; 216 217 size = hugetlb_cma_size_in_node[nid]; 218 } else { 219 size = min(per_node, hugetlb_cma_size - reserved); 220 } 221 222 size = round_up(size, PAGE_SIZE << order); 223 224 snprintf(name, sizeof(name), "hugetlb%d", nid); 225 /* 226 * Note that 'order per bit' is based on smallest size that 227 * may be returned to CMA allocator in the case of 228 * huge page demotion. 229 */ 230 res = cma_declare_contiguous_multi(size, PAGE_SIZE << order, 231 HUGETLB_PAGE_ORDER, name, 232 &hugetlb_cma[nid], nid); 233 if (res) { 234 pr_warn("hugetlb_cma: reservation failed: err %d, node %d", 235 res, nid); 236 continue; 237 } 238 239 reserved += size; 240 pr_info("hugetlb_cma: reserved %lu MiB on node %d\n", 241 size / SZ_1M, nid); 242 243 if (reserved >= hugetlb_cma_size) 244 break; 245 } 246 247 if (!reserved) 248 /* 249 * hugetlb_cma_size is used to determine if allocations from 250 * cma are possible. Set to zero if no cma regions are set up. 251 */ 252 hugetlb_cma_size = 0; 253 } 254 255 bool hugetlb_cma_exclusive_alloc(void) 256 { 257 return hugetlb_cma_only; 258 } 259 260 unsigned long __init hugetlb_cma_total_size(void) 261 { 262 return hugetlb_cma_size; 263 } 264 265 void __init hugetlb_cma_validate_params(void) 266 { 267 if (!hugetlb_cma_size) 268 hugetlb_cma_only = false; 269 } 270 271 bool __init hugetlb_early_cma(struct hstate *h) 272 { 273 if (arch_has_huge_bootmem_alloc()) 274 return false; 275 276 return hstate_is_gigantic(h) && hugetlb_cma_only; 277 } 278