1 // SPDX-License-Identifier: GPL-2.0-only
2
3 #include <linux/mm.h>
4 #include <linux/cma.h>
5 #include <linux/compiler.h>
6 #include <linux/mm_inline.h>
7
8 #include <asm/page.h>
9 #include <asm/setup.h>
10
11 #include <linux/hugetlb.h>
12 #include "internal.h"
13 #include "hugetlb_cma.h"
14
15
16 static struct cma *hugetlb_cma[MAX_NUMNODES] __ro_after_init;
17 static unsigned long hugetlb_cma_size_in_node[MAX_NUMNODES] __initdata;
18 static bool hugetlb_cma_only __ro_after_init;
19 static unsigned long hugetlb_cma_size __ro_after_init;
20
hugetlb_cma_free_frozen_folio(struct folio * folio)21 void hugetlb_cma_free_frozen_folio(struct folio *folio)
22 {
23 WARN_ON_ONCE(!cma_release_frozen(hugetlb_cma[folio_nid(folio)],
24 &folio->page, folio_nr_pages(folio)));
25 }
26
hugetlb_cma_alloc_frozen_folio(int order,gfp_t gfp_mask,int nid,nodemask_t * nodemask)27 struct folio *hugetlb_cma_alloc_frozen_folio(int order, gfp_t gfp_mask,
28 int nid, nodemask_t *nodemask)
29 {
30 int node;
31 struct folio *folio;
32 struct page *page = NULL;
33
34 if (!hugetlb_cma_size)
35 return NULL;
36
37 if (hugetlb_cma[nid])
38 page = cma_alloc_frozen_compound(hugetlb_cma[nid], order);
39
40 if (!page && !(gfp_mask & __GFP_THISNODE)) {
41 for_each_node_mask(node, *nodemask) {
42 if (node == nid || !hugetlb_cma[node])
43 continue;
44
45 page = cma_alloc_frozen_compound(hugetlb_cma[node], order);
46 if (page)
47 break;
48 }
49 }
50
51 if (!page)
52 return NULL;
53
54 folio = page_folio(page);
55 folio_set_hugetlb_cma(folio);
56 return folio;
57 }
58
59 struct huge_bootmem_page * __init
hugetlb_cma_alloc_bootmem(struct hstate * h,int * nid,bool node_exact)60 hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid, bool node_exact)
61 {
62 struct cma *cma;
63 struct huge_bootmem_page *m;
64 int node = *nid;
65
66 cma = hugetlb_cma[*nid];
67 m = cma_reserve_early(cma, huge_page_size(h));
68 if (!m) {
69 if (node_exact)
70 return NULL;
71
72 for_each_node_mask(node, hugetlb_bootmem_nodes) {
73 cma = hugetlb_cma[node];
74 if (!cma || node == *nid)
75 continue;
76 m = cma_reserve_early(cma, huge_page_size(h));
77 if (m) {
78 *nid = node;
79 break;
80 }
81 }
82 }
83
84 if (m) {
85 m->flags = HUGE_BOOTMEM_CMA;
86 m->cma = cma;
87 }
88
89 return m;
90 }
91
cmdline_parse_hugetlb_cma(char * p)92 static int __init cmdline_parse_hugetlb_cma(char *p)
93 {
94 int nid, count = 0;
95 unsigned long tmp;
96 char *s = p;
97
98 while (*s) {
99 if (sscanf(s, "%lu%n", &tmp, &count) != 1)
100 break;
101
102 if (s[count] == ':') {
103 if (tmp >= MAX_NUMNODES)
104 break;
105 nid = array_index_nospec(tmp, MAX_NUMNODES);
106
107 s += count + 1;
108 tmp = memparse(s, &s);
109 hugetlb_cma_size_in_node[nid] = tmp;
110 hugetlb_cma_size += tmp;
111
112 /*
113 * Skip the separator if have one, otherwise
114 * break the parsing.
115 */
116 if (*s == ',')
117 s++;
118 else
119 break;
120 } else {
121 hugetlb_cma_size = memparse(p, &p);
122 break;
123 }
124 }
125
126 return 0;
127 }
128
129 early_param("hugetlb_cma", cmdline_parse_hugetlb_cma);
130
cmdline_parse_hugetlb_cma_only(char * p)131 static int __init cmdline_parse_hugetlb_cma_only(char *p)
132 {
133 return kstrtobool(p, &hugetlb_cma_only);
134 }
135
136 early_param("hugetlb_cma_only", cmdline_parse_hugetlb_cma_only);
137
arch_hugetlb_cma_order(void)138 unsigned int __weak arch_hugetlb_cma_order(void)
139 {
140 return 0;
141 }
142
hugetlb_cma_reserve(void)143 void __init hugetlb_cma_reserve(void)
144 {
145 unsigned long size, reserved, per_node, order;
146 bool node_specific_cma_alloc = false;
147 int nid;
148
149 if (!hugetlb_cma_size)
150 return;
151
152 order = arch_hugetlb_cma_order();
153 if (!order) {
154 pr_warn("hugetlb_cma: the option isn't supported by current arch\n");
155 return;
156 }
157
158 /*
159 * HugeTLB CMA reservation is required for gigantic
160 * huge pages which could not be allocated via the
161 * page allocator. Just warn if there is any change
162 * breaking this assumption.
163 */
164 VM_WARN_ON(order <= MAX_PAGE_ORDER);
165
166 hugetlb_bootmem_set_nodes();
167
168 for (nid = 0; nid < MAX_NUMNODES; nid++) {
169 if (hugetlb_cma_size_in_node[nid] == 0)
170 continue;
171
172 if (!node_isset(nid, hugetlb_bootmem_nodes)) {
173 pr_warn("hugetlb_cma: invalid node %d specified\n", nid);
174 hugetlb_cma_size -= hugetlb_cma_size_in_node[nid];
175 hugetlb_cma_size_in_node[nid] = 0;
176 continue;
177 }
178
179 if (hugetlb_cma_size_in_node[nid] < (PAGE_SIZE << order)) {
180 pr_warn("hugetlb_cma: cma area of node %d should be at least %lu MiB\n",
181 nid, (PAGE_SIZE << order) / SZ_1M);
182 hugetlb_cma_size -= hugetlb_cma_size_in_node[nid];
183 hugetlb_cma_size_in_node[nid] = 0;
184 } else {
185 node_specific_cma_alloc = true;
186 }
187 }
188
189 /* Validate the CMA size again in case some invalid nodes specified. */
190 if (!hugetlb_cma_size)
191 return;
192
193 if (hugetlb_cma_size < (PAGE_SIZE << order)) {
194 pr_warn("hugetlb_cma: cma area should be at least %lu MiB\n",
195 (PAGE_SIZE << order) / SZ_1M);
196 hugetlb_cma_size = 0;
197 return;
198 }
199
200 if (!node_specific_cma_alloc) {
201 /*
202 * If 3 GB area is requested on a machine with 4 numa nodes,
203 * let's allocate 1 GB on first three nodes and ignore the last one.
204 */
205 per_node = DIV_ROUND_UP(hugetlb_cma_size,
206 nodes_weight(hugetlb_bootmem_nodes));
207 per_node = round_up(per_node, PAGE_SIZE << order);
208 pr_info("hugetlb_cma: reserve %lu MiB, up to %lu MiB per node\n",
209 hugetlb_cma_size / SZ_1M, per_node / SZ_1M);
210 }
211
212 reserved = 0;
213 for_each_node_mask(nid, hugetlb_bootmem_nodes) {
214 int res;
215 char name[CMA_MAX_NAME];
216
217 if (node_specific_cma_alloc) {
218 if (hugetlb_cma_size_in_node[nid] == 0)
219 continue;
220
221 size = hugetlb_cma_size_in_node[nid];
222 } else {
223 size = min(per_node, hugetlb_cma_size - reserved);
224 }
225
226 size = round_up(size, PAGE_SIZE << order);
227
228 snprintf(name, sizeof(name), "hugetlb%d", nid);
229 /*
230 * Note that 'order per bit' is based on smallest size that
231 * may be returned to CMA allocator in the case of
232 * huge page demotion.
233 */
234 res = cma_declare_contiguous_multi(size, PAGE_SIZE << order,
235 HUGETLB_PAGE_ORDER, name,
236 &hugetlb_cma[nid], nid);
237 if (res) {
238 pr_warn("hugetlb_cma: reservation failed: err %d, node %d",
239 res, nid);
240 continue;
241 }
242
243 reserved += size;
244 pr_info("hugetlb_cma: reserved %lu MiB on node %d\n",
245 size / SZ_1M, nid);
246
247 if (reserved >= hugetlb_cma_size)
248 break;
249 }
250
251 if (!reserved)
252 /*
253 * hugetlb_cma_size is used to determine if allocations from
254 * cma are possible. Set to zero if no cma regions are set up.
255 */
256 hugetlb_cma_size = 0;
257 }
258
hugetlb_cma_exclusive_alloc(void)259 bool hugetlb_cma_exclusive_alloc(void)
260 {
261 return hugetlb_cma_only;
262 }
263
hugetlb_cma_total_size(void)264 unsigned long __init hugetlb_cma_total_size(void)
265 {
266 return hugetlb_cma_size;
267 }
268
hugetlb_cma_validate_params(void)269 void __init hugetlb_cma_validate_params(void)
270 {
271 if (!hugetlb_cma_size)
272 hugetlb_cma_only = false;
273 }
274
hugetlb_early_cma(struct hstate * h)275 bool __init hugetlb_early_cma(struct hstate *h)
276 {
277 if (arch_has_huge_bootmem_alloc())
278 return false;
279
280 return hstate_is_gigantic(h) && hugetlb_cma_only;
281 }
282