xref: /linux/mm/sparse.c (revision 8026aed072e1221f0a61e5acc48c64546341bd4d)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * sparse memory mappings.
4  */
5 #include <linux/mm.h>
6 #include <linux/slab.h>
7 #include <linux/mmzone.h>
8 #include <linux/memblock.h>
9 #include <linux/compiler.h>
10 #include <linux/highmem.h>
11 #include <linux/export.h>
12 #include <linux/spinlock.h>
13 #include <linux/vmalloc.h>
14 #include <linux/swap.h>
15 #include <linux/swapops.h>
16 #include <linux/bootmem_info.h>
17 #include <linux/vmstat.h>
18 #include "internal.h"
19 #include <asm/dma.h>
20 
21 /*
22  * Permanent SPARSEMEM data:
23  *
24  * 1) mem_section	- memory sections, mem_map's for valid memory
25  */
26 #ifdef CONFIG_SPARSEMEM_EXTREME
27 struct mem_section **mem_section;
28 #else
29 struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT]
30 	____cacheline_internodealigned_in_smp;
31 #endif
32 EXPORT_SYMBOL(mem_section);
33 
34 #ifdef NODE_NOT_IN_PAGE_FLAGS
35 /*
36  * If we did not store the node number in the page then we have to
37  * do a lookup in the section_to_node_table in order to find which
38  * node the page belongs to.
39  */
40 #if MAX_NUMNODES <= 256
41 static u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned;
42 #else
43 static u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned;
44 #endif
45 
page_to_nid(const struct page * page)46 int page_to_nid(const struct page *page)
47 {
48 	return section_to_node_table[page_to_section(page)];
49 }
50 EXPORT_SYMBOL(page_to_nid);
51 
set_section_nid(unsigned long section_nr,int nid)52 static void set_section_nid(unsigned long section_nr, int nid)
53 {
54 	section_to_node_table[section_nr] = nid;
55 }
56 #else /* !NODE_NOT_IN_PAGE_FLAGS */
set_section_nid(unsigned long section_nr,int nid)57 static inline void set_section_nid(unsigned long section_nr, int nid)
58 {
59 }
60 #endif
61 
62 #ifdef CONFIG_SPARSEMEM_EXTREME
sparse_index_alloc(int nid)63 static noinline struct mem_section __ref *sparse_index_alloc(int nid)
64 {
65 	struct mem_section *section = NULL;
66 	unsigned long array_size = SECTIONS_PER_ROOT *
67 				   sizeof(struct mem_section);
68 
69 	if (slab_is_available()) {
70 		section = kzalloc_node(array_size, GFP_KERNEL, nid);
71 	} else {
72 		section = memblock_alloc_node(array_size, SMP_CACHE_BYTES,
73 					      nid);
74 		if (!section)
75 			panic("%s: Failed to allocate %lu bytes nid=%d\n",
76 			      __func__, array_size, nid);
77 	}
78 
79 	return section;
80 }
81 
sparse_index_init(unsigned long section_nr,int nid)82 static int __meminit sparse_index_init(unsigned long section_nr, int nid)
83 {
84 	unsigned long root = SECTION_NR_TO_ROOT(section_nr);
85 	struct mem_section *section;
86 
87 	/*
88 	 * An existing section is possible in the sub-section hotplug
89 	 * case. First hot-add instantiates, follow-on hot-add reuses
90 	 * the existing section.
91 	 *
92 	 * The mem_hotplug_lock resolves the apparent race below.
93 	 */
94 	if (mem_section[root])
95 		return 0;
96 
97 	section = sparse_index_alloc(nid);
98 	if (!section)
99 		return -ENOMEM;
100 
101 	mem_section[root] = section;
102 
103 	return 0;
104 }
105 #else /* !SPARSEMEM_EXTREME */
sparse_index_init(unsigned long section_nr,int nid)106 static inline int sparse_index_init(unsigned long section_nr, int nid)
107 {
108 	return 0;
109 }
110 #endif
111 
112 /*
113  * During early boot, before section_mem_map is used for an actual
114  * mem_map, we use section_mem_map to store the section's NUMA
115  * node.  This keeps us from having to use another data structure.  The
116  * node information is cleared just before we store the real mem_map.
117  */
sparse_encode_early_nid(int nid)118 static inline unsigned long sparse_encode_early_nid(int nid)
119 {
120 	return ((unsigned long)nid << SECTION_NID_SHIFT);
121 }
122 
sparse_early_nid(struct mem_section * section)123 static inline int sparse_early_nid(struct mem_section *section)
124 {
125 	return (section->section_mem_map >> SECTION_NID_SHIFT);
126 }
127 
128 /* Validate the physical addressing limitations of the model */
mminit_validate_memmodel_limits(unsigned long * start_pfn,unsigned long * end_pfn)129 static void __meminit mminit_validate_memmodel_limits(unsigned long *start_pfn,
130 						unsigned long *end_pfn)
131 {
132 	unsigned long max_sparsemem_pfn = (DIRECT_MAP_PHYSMEM_END + 1) >> PAGE_SHIFT;
133 
134 	/*
135 	 * Sanity checks - do not allow an architecture to pass
136 	 * in larger pfns than the maximum scope of sparsemem:
137 	 */
138 	if (*start_pfn > max_sparsemem_pfn) {
139 		mminit_dprintk(MMINIT_WARNING, "pfnvalidation",
140 			"Start of range %lu -> %lu exceeds SPARSEMEM max %lu\n",
141 			*start_pfn, *end_pfn, max_sparsemem_pfn);
142 		WARN_ON_ONCE(1);
143 		*start_pfn = max_sparsemem_pfn;
144 		*end_pfn = max_sparsemem_pfn;
145 	} else if (*end_pfn > max_sparsemem_pfn) {
146 		mminit_dprintk(MMINIT_WARNING, "pfnvalidation",
147 			"End of range %lu -> %lu exceeds SPARSEMEM max %lu\n",
148 			*start_pfn, *end_pfn, max_sparsemem_pfn);
149 		WARN_ON_ONCE(1);
150 		*end_pfn = max_sparsemem_pfn;
151 	}
152 }
153 
154 /*
155  * There are a number of times that we loop over NR_MEM_SECTIONS,
156  * looking for section_present() on each.  But, when we have very
157  * large physical address spaces, NR_MEM_SECTIONS can also be
158  * very large which makes the loops quite long.
159  *
160  * Keeping track of this gives us an easy way to break out of
161  * those loops early.
162  */
163 unsigned long __highest_present_section_nr;
__section_mark_present(struct mem_section * ms,unsigned long section_nr)164 static void __section_mark_present(struct mem_section *ms,
165 		unsigned long section_nr)
166 {
167 	if (section_nr > __highest_present_section_nr)
168 		__highest_present_section_nr = section_nr;
169 
170 	ms->section_mem_map |= SECTION_MARKED_PRESENT;
171 }
172 
first_present_section_nr(void)173 static inline unsigned long first_present_section_nr(void)
174 {
175 	return next_present_section_nr(-1);
176 }
177 
178 #ifdef CONFIG_SPARSEMEM_VMEMMAP
subsection_mask_set(unsigned long * map,unsigned long pfn,unsigned long nr_pages)179 static void subsection_mask_set(unsigned long *map, unsigned long pfn,
180 		unsigned long nr_pages)
181 {
182 	int idx = subsection_map_index(pfn);
183 	int end = subsection_map_index(pfn + nr_pages - 1);
184 
185 	bitmap_set(map, idx, end - idx + 1);
186 }
187 
subsection_map_init(unsigned long pfn,unsigned long nr_pages)188 void __init subsection_map_init(unsigned long pfn, unsigned long nr_pages)
189 {
190 	int end_sec_nr = pfn_to_section_nr(pfn + nr_pages - 1);
191 	unsigned long nr, start_sec_nr = pfn_to_section_nr(pfn);
192 
193 	for (nr = start_sec_nr; nr <= end_sec_nr; nr++) {
194 		struct mem_section *ms;
195 		unsigned long pfns;
196 
197 		pfns = min(nr_pages, PAGES_PER_SECTION
198 				- (pfn & ~PAGE_SECTION_MASK));
199 		ms = __nr_to_section(nr);
200 		subsection_mask_set(ms->usage->subsection_map, pfn, pfns);
201 
202 		pr_debug("%s: sec: %lu pfns: %lu set(%d, %d)\n", __func__, nr,
203 				pfns, subsection_map_index(pfn),
204 				subsection_map_index(pfn + pfns - 1));
205 
206 		pfn += pfns;
207 		nr_pages -= pfns;
208 	}
209 }
210 #else
subsection_map_init(unsigned long pfn,unsigned long nr_pages)211 void __init subsection_map_init(unsigned long pfn, unsigned long nr_pages)
212 {
213 }
214 #endif
215 
216 /* Record a memory area against a node. */
memory_present(int nid,unsigned long start,unsigned long end)217 static void __init memory_present(int nid, unsigned long start, unsigned long end)
218 {
219 	unsigned long pfn;
220 
221 	start &= PAGE_SECTION_MASK;
222 	mminit_validate_memmodel_limits(&start, &end);
223 	for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) {
224 		unsigned long section_nr = pfn_to_section_nr(pfn);
225 		struct mem_section *ms;
226 
227 		sparse_index_init(section_nr, nid);
228 		set_section_nid(section_nr, nid);
229 
230 		ms = __nr_to_section(section_nr);
231 		if (!ms->section_mem_map) {
232 			ms->section_mem_map = sparse_encode_early_nid(nid) |
233 							SECTION_IS_ONLINE;
234 			__section_mark_present(ms, section_nr);
235 		}
236 	}
237 }
238 
239 /*
240  * Mark all memblocks as present using memory_present().
241  * This is a convenience function that is useful to mark all of the systems
242  * memory as present during initialization.
243  */
memblocks_present(void)244 static void __init memblocks_present(void)
245 {
246 	unsigned long start, end;
247 	int i, nid;
248 
249 #ifdef CONFIG_SPARSEMEM_EXTREME
250 	if (unlikely(!mem_section)) {
251 		unsigned long size, align;
252 
253 		size = sizeof(struct mem_section *) * NR_SECTION_ROOTS;
254 		align = 1 << (INTERNODE_CACHE_SHIFT);
255 		mem_section = memblock_alloc_or_panic(size, align);
256 	}
257 #endif
258 
259 	for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid)
260 		memory_present(nid, start, end);
261 }
262 
263 /*
264  * Subtle, we encode the real pfn into the mem_map such that
265  * the identity pfn - section_mem_map will return the actual
266  * physical page frame number.
267  */
sparse_encode_mem_map(struct page * mem_map,unsigned long pnum)268 static unsigned long sparse_encode_mem_map(struct page *mem_map, unsigned long pnum)
269 {
270 	unsigned long coded_mem_map =
271 		(unsigned long)(mem_map - (section_nr_to_pfn(pnum)));
272 	BUILD_BUG_ON(SECTION_MAP_LAST_BIT > PFN_SECTION_SHIFT);
273 	BUG_ON(coded_mem_map & ~SECTION_MAP_MASK);
274 	return coded_mem_map;
275 }
276 
277 #ifdef CONFIG_MEMORY_HOTPLUG
278 /*
279  * Decode mem_map from the coded memmap
280  */
sparse_decode_mem_map(unsigned long coded_mem_map,unsigned long pnum)281 struct page *sparse_decode_mem_map(unsigned long coded_mem_map, unsigned long pnum)
282 {
283 	/* mask off the extra low bits of information */
284 	coded_mem_map &= SECTION_MAP_MASK;
285 	return ((struct page *)coded_mem_map) + section_nr_to_pfn(pnum);
286 }
287 #endif /* CONFIG_MEMORY_HOTPLUG */
288 
sparse_init_one_section(struct mem_section * ms,unsigned long pnum,struct page * mem_map,struct mem_section_usage * usage,unsigned long flags)289 static void __meminit sparse_init_one_section(struct mem_section *ms,
290 		unsigned long pnum, struct page *mem_map,
291 		struct mem_section_usage *usage, unsigned long flags)
292 {
293 	ms->section_mem_map &= ~SECTION_MAP_MASK;
294 	ms->section_mem_map |= sparse_encode_mem_map(mem_map, pnum)
295 		| SECTION_HAS_MEM_MAP | flags;
296 	ms->usage = usage;
297 }
298 
usemap_size(void)299 static unsigned long usemap_size(void)
300 {
301 	return BITS_TO_LONGS(SECTION_BLOCKFLAGS_BITS) * sizeof(unsigned long);
302 }
303 
mem_section_usage_size(void)304 size_t mem_section_usage_size(void)
305 {
306 	return sizeof(struct mem_section_usage) + usemap_size();
307 }
308 
309 #ifdef CONFIG_MEMORY_HOTREMOVE
pgdat_to_phys(struct pglist_data * pgdat)310 static inline phys_addr_t pgdat_to_phys(struct pglist_data *pgdat)
311 {
312 #ifndef CONFIG_NUMA
313 	VM_BUG_ON(pgdat != &contig_page_data);
314 	return __pa_symbol(&contig_page_data);
315 #else
316 	return __pa(pgdat);
317 #endif
318 }
319 
320 static struct mem_section_usage * __init
sparse_early_usemaps_alloc_pgdat_section(struct pglist_data * pgdat,unsigned long size)321 sparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat,
322 					 unsigned long size)
323 {
324 	struct mem_section_usage *usage;
325 	unsigned long goal, limit;
326 	int nid;
327 	/*
328 	 * A page may contain usemaps for other sections preventing the
329 	 * page being freed and making a section unremovable while
330 	 * other sections referencing the usemap remain active. Similarly,
331 	 * a pgdat can prevent a section being removed. If section A
332 	 * contains a pgdat and section B contains the usemap, both
333 	 * sections become inter-dependent. This allocates usemaps
334 	 * from the same section as the pgdat where possible to avoid
335 	 * this problem.
336 	 */
337 	goal = pgdat_to_phys(pgdat) & (PAGE_SECTION_MASK << PAGE_SHIFT);
338 	limit = goal + (1UL << PA_SECTION_SHIFT);
339 	nid = early_pfn_to_nid(goal >> PAGE_SHIFT);
340 again:
341 	usage = memblock_alloc_try_nid(size, SMP_CACHE_BYTES, goal, limit, nid);
342 	if (!usage && limit) {
343 		limit = MEMBLOCK_ALLOC_ACCESSIBLE;
344 		goto again;
345 	}
346 	return usage;
347 }
348 
check_usemap_section_nr(int nid,struct mem_section_usage * usage)349 static void __init check_usemap_section_nr(int nid,
350 		struct mem_section_usage *usage)
351 {
352 	unsigned long usemap_snr, pgdat_snr;
353 	static unsigned long old_usemap_snr;
354 	static unsigned long old_pgdat_snr;
355 	struct pglist_data *pgdat = NODE_DATA(nid);
356 	int usemap_nid;
357 
358 	/* First call */
359 	if (!old_usemap_snr) {
360 		old_usemap_snr = NR_MEM_SECTIONS;
361 		old_pgdat_snr = NR_MEM_SECTIONS;
362 	}
363 
364 	usemap_snr = pfn_to_section_nr(__pa(usage) >> PAGE_SHIFT);
365 	pgdat_snr = pfn_to_section_nr(pgdat_to_phys(pgdat) >> PAGE_SHIFT);
366 	if (usemap_snr == pgdat_snr)
367 		return;
368 
369 	if (old_usemap_snr == usemap_snr && old_pgdat_snr == pgdat_snr)
370 		/* skip redundant message */
371 		return;
372 
373 	old_usemap_snr = usemap_snr;
374 	old_pgdat_snr = pgdat_snr;
375 
376 	usemap_nid = sparse_early_nid(__nr_to_section(usemap_snr));
377 	if (usemap_nid != nid) {
378 		pr_info("node %d must be removed before remove section %ld\n",
379 			nid, usemap_snr);
380 		return;
381 	}
382 	/*
383 	 * There is a circular dependency.
384 	 * Some platforms allow un-removable section because they will just
385 	 * gather other removable sections for dynamic partitioning.
386 	 * Just notify un-removable section's number here.
387 	 */
388 	pr_info("Section %ld and %ld (node %d) have a circular dependency on usemap and pgdat allocations\n",
389 		usemap_snr, pgdat_snr, nid);
390 }
391 #else
392 static struct mem_section_usage * __init
sparse_early_usemaps_alloc_pgdat_section(struct pglist_data * pgdat,unsigned long size)393 sparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat,
394 					 unsigned long size)
395 {
396 	return memblock_alloc_node(size, SMP_CACHE_BYTES, pgdat->node_id);
397 }
398 
check_usemap_section_nr(int nid,struct mem_section_usage * usage)399 static void __init check_usemap_section_nr(int nid,
400 		struct mem_section_usage *usage)
401 {
402 }
403 #endif /* CONFIG_MEMORY_HOTREMOVE */
404 
405 #ifdef CONFIG_SPARSEMEM_VMEMMAP
section_map_size(void)406 unsigned long __init section_map_size(void)
407 {
408 	return ALIGN(sizeof(struct page) * PAGES_PER_SECTION, PMD_SIZE);
409 }
410 
411 #else
section_map_size(void)412 unsigned long __init section_map_size(void)
413 {
414 	return PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION);
415 }
416 
__populate_section_memmap(unsigned long pfn,unsigned long nr_pages,int nid,struct vmem_altmap * altmap,struct dev_pagemap * pgmap)417 struct page __init *__populate_section_memmap(unsigned long pfn,
418 		unsigned long nr_pages, int nid, struct vmem_altmap *altmap,
419 		struct dev_pagemap *pgmap)
420 {
421 	unsigned long size = section_map_size();
422 	struct page *map = sparse_buffer_alloc(size);
423 	phys_addr_t addr = __pa(MAX_DMA_ADDRESS);
424 
425 	if (map)
426 		return map;
427 
428 	map = memmap_alloc(size, size, addr, nid, false);
429 	if (!map)
430 		panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%pa\n",
431 		      __func__, size, PAGE_SIZE, nid, &addr);
432 
433 	return map;
434 }
435 #endif /* !CONFIG_SPARSEMEM_VMEMMAP */
436 
437 static void *sparsemap_buf __meminitdata;
438 static void *sparsemap_buf_end __meminitdata;
439 
sparse_buffer_free(unsigned long size)440 static inline void __meminit sparse_buffer_free(unsigned long size)
441 {
442 	WARN_ON(!sparsemap_buf || size == 0);
443 	memblock_free(sparsemap_buf, size);
444 }
445 
sparse_buffer_init(unsigned long size,int nid)446 static void __init sparse_buffer_init(unsigned long size, int nid)
447 {
448 	phys_addr_t addr = __pa(MAX_DMA_ADDRESS);
449 	WARN_ON(sparsemap_buf);	/* forgot to call sparse_buffer_fini()? */
450 	/*
451 	 * Pre-allocated buffer is mainly used by __populate_section_memmap
452 	 * and we want it to be properly aligned to the section size - this is
453 	 * especially the case for VMEMMAP which maps memmap to PMDs
454 	 */
455 	sparsemap_buf = memmap_alloc(size, section_map_size(), addr, nid, true);
456 	sparsemap_buf_end = sparsemap_buf + size;
457 }
458 
sparse_buffer_fini(void)459 static void __init sparse_buffer_fini(void)
460 {
461 	unsigned long size = sparsemap_buf_end - sparsemap_buf;
462 
463 	if (sparsemap_buf && size > 0)
464 		sparse_buffer_free(size);
465 	sparsemap_buf = NULL;
466 }
467 
sparse_buffer_alloc(unsigned long size)468 void * __meminit sparse_buffer_alloc(unsigned long size)
469 {
470 	void *ptr = NULL;
471 
472 	if (sparsemap_buf) {
473 		ptr = (void *) roundup((unsigned long)sparsemap_buf, size);
474 		if (ptr + size > sparsemap_buf_end)
475 			ptr = NULL;
476 		else {
477 			/* Free redundant aligned space */
478 			if ((unsigned long)(ptr - sparsemap_buf) > 0)
479 				sparse_buffer_free((unsigned long)(ptr - sparsemap_buf));
480 			sparsemap_buf = ptr + size;
481 		}
482 	}
483 	return ptr;
484 }
485 
vmemmap_populate_print_last(void)486 void __weak __meminit vmemmap_populate_print_last(void)
487 {
488 }
489 
490 static void *sparse_usagebuf __meminitdata;
491 static void *sparse_usagebuf_end __meminitdata;
492 
493 /*
494  * Helper function that is used for generic section initialization, and
495  * can also be used by any hooks added above.
496  */
sparse_init_early_section(int nid,struct page * map,unsigned long pnum,unsigned long flags)497 void __init sparse_init_early_section(int nid, struct page *map,
498 				      unsigned long pnum, unsigned long flags)
499 {
500 	BUG_ON(!sparse_usagebuf || sparse_usagebuf >= sparse_usagebuf_end);
501 	check_usemap_section_nr(nid, sparse_usagebuf);
502 	sparse_init_one_section(__nr_to_section(pnum), pnum, map,
503 			sparse_usagebuf, SECTION_IS_EARLY | flags);
504 	sparse_usagebuf = (void *)sparse_usagebuf + mem_section_usage_size();
505 }
506 
sparse_usage_init(int nid,unsigned long map_count)507 static int __init sparse_usage_init(int nid, unsigned long map_count)
508 {
509 	unsigned long size;
510 
511 	size = mem_section_usage_size() * map_count;
512 	sparse_usagebuf = sparse_early_usemaps_alloc_pgdat_section(
513 				NODE_DATA(nid), size);
514 	if (!sparse_usagebuf) {
515 		sparse_usagebuf_end = NULL;
516 		return -ENOMEM;
517 	}
518 
519 	sparse_usagebuf_end = sparse_usagebuf + size;
520 	return 0;
521 }
522 
sparse_usage_fini(void)523 static void __init sparse_usage_fini(void)
524 {
525 	sparse_usagebuf = sparse_usagebuf_end = NULL;
526 }
527 
528 /*
529  * Initialize sparse on a specific node. The node spans [pnum_begin, pnum_end)
530  * And number of present sections in this node is map_count.
531  */
sparse_init_nid(int nid,unsigned long pnum_begin,unsigned long pnum_end,unsigned long map_count)532 static void __init sparse_init_nid(int nid, unsigned long pnum_begin,
533 				   unsigned long pnum_end,
534 				   unsigned long map_count)
535 {
536 	unsigned long pnum;
537 	struct page *map;
538 	struct mem_section *ms;
539 
540 	if (sparse_usage_init(nid, map_count)) {
541 		pr_err("%s: node[%d] usemap allocation failed", __func__, nid);
542 		goto failed;
543 	}
544 
545 	sparse_buffer_init(map_count * section_map_size(), nid);
546 
547 	sparse_vmemmap_init_nid_early(nid);
548 
549 	for_each_present_section_nr(pnum_begin, pnum) {
550 		unsigned long pfn = section_nr_to_pfn(pnum);
551 
552 		if (pnum >= pnum_end)
553 			break;
554 
555 		ms = __nr_to_section(pnum);
556 		if (!preinited_vmemmap_section(ms)) {
557 			map = __populate_section_memmap(pfn, PAGES_PER_SECTION,
558 					nid, NULL, NULL);
559 			if (!map) {
560 				pr_err("%s: node[%d] memory map backing failed. Some memory will not be available.",
561 				       __func__, nid);
562 				pnum_begin = pnum;
563 				sparse_usage_fini();
564 				sparse_buffer_fini();
565 				goto failed;
566 			}
567 			memmap_boot_pages_add(DIV_ROUND_UP(PAGES_PER_SECTION * sizeof(struct page),
568 							   PAGE_SIZE));
569 			sparse_init_early_section(nid, map, pnum, 0);
570 		}
571 	}
572 	sparse_usage_fini();
573 	sparse_buffer_fini();
574 	return;
575 failed:
576 	/*
577 	 * We failed to allocate, mark all the following pnums as not present,
578 	 * except the ones already initialized earlier.
579 	 */
580 	for_each_present_section_nr(pnum_begin, pnum) {
581 		if (pnum >= pnum_end)
582 			break;
583 		ms = __nr_to_section(pnum);
584 		if (!preinited_vmemmap_section(ms))
585 			ms->section_mem_map = 0;
586 		ms->section_mem_map = 0;
587 	}
588 }
589 
590 /*
591  * Allocate the accumulated non-linear sections, allocate a mem_map
592  * for each and record the physical to section mapping.
593  */
sparse_init(void)594 void __init sparse_init(void)
595 {
596 	unsigned long pnum_end, pnum_begin, map_count = 1;
597 	int nid_begin;
598 
599 	/* see include/linux/mmzone.h 'struct mem_section' definition */
600 	BUILD_BUG_ON(!is_power_of_2(sizeof(struct mem_section)));
601 	memblocks_present();
602 
603 	pnum_begin = first_present_section_nr();
604 	nid_begin = sparse_early_nid(__nr_to_section(pnum_begin));
605 
606 	/* Setup pageblock_order for HUGETLB_PAGE_SIZE_VARIABLE */
607 	set_pageblock_order();
608 
609 	for_each_present_section_nr(pnum_begin + 1, pnum_end) {
610 		int nid = sparse_early_nid(__nr_to_section(pnum_end));
611 
612 		if (nid == nid_begin) {
613 			map_count++;
614 			continue;
615 		}
616 		/* Init node with sections in range [pnum_begin, pnum_end) */
617 		sparse_init_nid(nid_begin, pnum_begin, pnum_end, map_count);
618 		nid_begin = nid;
619 		pnum_begin = pnum_end;
620 		map_count = 1;
621 	}
622 	/* cover the last node */
623 	sparse_init_nid(nid_begin, pnum_begin, pnum_end, map_count);
624 	vmemmap_populate_print_last();
625 }
626 
627 #ifdef CONFIG_MEMORY_HOTPLUG
628 
629 /* Mark all memory sections within the pfn range as online */
online_mem_sections(unsigned long start_pfn,unsigned long end_pfn)630 void online_mem_sections(unsigned long start_pfn, unsigned long end_pfn)
631 {
632 	unsigned long pfn;
633 
634 	for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
635 		unsigned long section_nr = pfn_to_section_nr(pfn);
636 		struct mem_section *ms;
637 
638 		/* onlining code should never touch invalid ranges */
639 		if (WARN_ON(!valid_section_nr(section_nr)))
640 			continue;
641 
642 		ms = __nr_to_section(section_nr);
643 		ms->section_mem_map |= SECTION_IS_ONLINE;
644 	}
645 }
646 
647 /* Mark all memory sections within the pfn range as offline */
offline_mem_sections(unsigned long start_pfn,unsigned long end_pfn)648 void offline_mem_sections(unsigned long start_pfn, unsigned long end_pfn)
649 {
650 	unsigned long pfn;
651 
652 	for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
653 		unsigned long section_nr = pfn_to_section_nr(pfn);
654 		struct mem_section *ms;
655 
656 		/*
657 		 * TODO this needs some double checking. Offlining code makes
658 		 * sure to check pfn_valid but those checks might be just bogus
659 		 */
660 		if (WARN_ON(!valid_section_nr(section_nr)))
661 			continue;
662 
663 		ms = __nr_to_section(section_nr);
664 		ms->section_mem_map &= ~SECTION_IS_ONLINE;
665 	}
666 }
667 
668 #ifdef CONFIG_SPARSEMEM_VMEMMAP
populate_section_memmap(unsigned long pfn,unsigned long nr_pages,int nid,struct vmem_altmap * altmap,struct dev_pagemap * pgmap)669 static struct page * __meminit populate_section_memmap(unsigned long pfn,
670 		unsigned long nr_pages, int nid, struct vmem_altmap *altmap,
671 		struct dev_pagemap *pgmap)
672 {
673 	return __populate_section_memmap(pfn, nr_pages, nid, altmap, pgmap);
674 }
675 
depopulate_section_memmap(unsigned long pfn,unsigned long nr_pages,struct vmem_altmap * altmap)676 static void depopulate_section_memmap(unsigned long pfn, unsigned long nr_pages,
677 		struct vmem_altmap *altmap)
678 {
679 	unsigned long start = (unsigned long) pfn_to_page(pfn);
680 	unsigned long end = start + nr_pages * sizeof(struct page);
681 
682 	vmemmap_free(start, end, altmap);
683 }
free_map_bootmem(struct page * memmap)684 static void free_map_bootmem(struct page *memmap)
685 {
686 	unsigned long start = (unsigned long)memmap;
687 	unsigned long end = (unsigned long)(memmap + PAGES_PER_SECTION);
688 
689 	vmemmap_free(start, end, NULL);
690 }
691 
clear_subsection_map(unsigned long pfn,unsigned long nr_pages)692 static int clear_subsection_map(unsigned long pfn, unsigned long nr_pages)
693 {
694 	DECLARE_BITMAP(map, SUBSECTIONS_PER_SECTION) = { 0 };
695 	DECLARE_BITMAP(tmp, SUBSECTIONS_PER_SECTION) = { 0 };
696 	struct mem_section *ms = __pfn_to_section(pfn);
697 	unsigned long *subsection_map = ms->usage
698 		? &ms->usage->subsection_map[0] : NULL;
699 
700 	subsection_mask_set(map, pfn, nr_pages);
701 	if (subsection_map)
702 		bitmap_and(tmp, map, subsection_map, SUBSECTIONS_PER_SECTION);
703 
704 	if (WARN(!subsection_map || !bitmap_equal(tmp, map, SUBSECTIONS_PER_SECTION),
705 				"section already deactivated (%#lx + %ld)\n",
706 				pfn, nr_pages))
707 		return -EINVAL;
708 
709 	bitmap_xor(subsection_map, map, subsection_map, SUBSECTIONS_PER_SECTION);
710 	return 0;
711 }
712 
is_subsection_map_empty(struct mem_section * ms)713 static bool is_subsection_map_empty(struct mem_section *ms)
714 {
715 	return bitmap_empty(&ms->usage->subsection_map[0],
716 			    SUBSECTIONS_PER_SECTION);
717 }
718 
fill_subsection_map(unsigned long pfn,unsigned long nr_pages)719 static int fill_subsection_map(unsigned long pfn, unsigned long nr_pages)
720 {
721 	struct mem_section *ms = __pfn_to_section(pfn);
722 	DECLARE_BITMAP(map, SUBSECTIONS_PER_SECTION) = { 0 };
723 	unsigned long *subsection_map;
724 	int rc = 0;
725 
726 	subsection_mask_set(map, pfn, nr_pages);
727 
728 	subsection_map = &ms->usage->subsection_map[0];
729 
730 	if (bitmap_empty(map, SUBSECTIONS_PER_SECTION))
731 		rc = -EINVAL;
732 	else if (bitmap_intersects(map, subsection_map, SUBSECTIONS_PER_SECTION))
733 		rc = -EEXIST;
734 	else
735 		bitmap_or(subsection_map, map, subsection_map,
736 				SUBSECTIONS_PER_SECTION);
737 
738 	return rc;
739 }
740 #else
populate_section_memmap(unsigned long pfn,unsigned long nr_pages,int nid,struct vmem_altmap * altmap,struct dev_pagemap * pgmap)741 static struct page * __meminit populate_section_memmap(unsigned long pfn,
742 		unsigned long nr_pages, int nid, struct vmem_altmap *altmap,
743 		struct dev_pagemap *pgmap)
744 {
745 	return kvmalloc_node(array_size(sizeof(struct page),
746 					PAGES_PER_SECTION), GFP_KERNEL, nid);
747 }
748 
depopulate_section_memmap(unsigned long pfn,unsigned long nr_pages,struct vmem_altmap * altmap)749 static void depopulate_section_memmap(unsigned long pfn, unsigned long nr_pages,
750 		struct vmem_altmap *altmap)
751 {
752 	kvfree(pfn_to_page(pfn));
753 }
754 
free_map_bootmem(struct page * memmap)755 static void free_map_bootmem(struct page *memmap)
756 {
757 	unsigned long maps_section_nr, removing_section_nr, i;
758 	unsigned long type, nr_pages;
759 	struct page *page = virt_to_page(memmap);
760 
761 	nr_pages = PAGE_ALIGN(PAGES_PER_SECTION * sizeof(struct page))
762 		>> PAGE_SHIFT;
763 
764 	for (i = 0; i < nr_pages; i++, page++) {
765 		type = bootmem_type(page);
766 
767 		BUG_ON(type == NODE_INFO);
768 
769 		maps_section_nr = pfn_to_section_nr(page_to_pfn(page));
770 		removing_section_nr = bootmem_info(page);
771 
772 		/*
773 		 * When this function is called, the removing section is
774 		 * logical offlined state. This means all pages are isolated
775 		 * from page allocator. If removing section's memmap is placed
776 		 * on the same section, it must not be freed.
777 		 * If it is freed, page allocator may allocate it which will
778 		 * be removed physically soon.
779 		 */
780 		if (maps_section_nr != removing_section_nr)
781 			put_page_bootmem(page);
782 	}
783 }
784 
clear_subsection_map(unsigned long pfn,unsigned long nr_pages)785 static int clear_subsection_map(unsigned long pfn, unsigned long nr_pages)
786 {
787 	return 0;
788 }
789 
is_subsection_map_empty(struct mem_section * ms)790 static bool is_subsection_map_empty(struct mem_section *ms)
791 {
792 	return true;
793 }
794 
fill_subsection_map(unsigned long pfn,unsigned long nr_pages)795 static int fill_subsection_map(unsigned long pfn, unsigned long nr_pages)
796 {
797 	return 0;
798 }
799 #endif /* CONFIG_SPARSEMEM_VMEMMAP */
800 
801 /*
802  * To deactivate a memory region, there are 3 cases to handle across
803  * two configurations (SPARSEMEM_VMEMMAP={y,n}):
804  *
805  * 1. deactivation of a partial hot-added section (only possible in
806  *    the SPARSEMEM_VMEMMAP=y case).
807  *      a) section was present at memory init.
808  *      b) section was hot-added post memory init.
809  * 2. deactivation of a complete hot-added section.
810  * 3. deactivation of a complete section from memory init.
811  *
812  * For 1, when subsection_map does not empty we will not be freeing the
813  * usage map, but still need to free the vmemmap range.
814  *
815  * For 2 and 3, the SPARSEMEM_VMEMMAP={y,n} cases are unified
816  */
section_deactivate(unsigned long pfn,unsigned long nr_pages,struct vmem_altmap * altmap)817 static void section_deactivate(unsigned long pfn, unsigned long nr_pages,
818 		struct vmem_altmap *altmap)
819 {
820 	struct mem_section *ms = __pfn_to_section(pfn);
821 	bool section_is_early = early_section(ms);
822 	struct page *memmap = NULL;
823 	bool empty;
824 
825 	if (clear_subsection_map(pfn, nr_pages))
826 		return;
827 
828 	empty = is_subsection_map_empty(ms);
829 	if (empty) {
830 		unsigned long section_nr = pfn_to_section_nr(pfn);
831 
832 		/*
833 		 * Mark the section invalid so that valid_section()
834 		 * return false. This prevents code from dereferencing
835 		 * ms->usage array.
836 		 */
837 		ms->section_mem_map &= ~SECTION_HAS_MEM_MAP;
838 
839 		/*
840 		 * When removing an early section, the usage map is kept (as the
841 		 * usage maps of other sections fall into the same page). It
842 		 * will be re-used when re-adding the section - which is then no
843 		 * longer an early section. If the usage map is PageReserved, it
844 		 * was allocated during boot.
845 		 */
846 		if (!PageReserved(virt_to_page(ms->usage))) {
847 			kfree_rcu(ms->usage, rcu);
848 			WRITE_ONCE(ms->usage, NULL);
849 		}
850 		memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);
851 	}
852 
853 	/*
854 	 * The memmap of early sections is always fully populated. See
855 	 * section_activate() and pfn_valid() .
856 	 */
857 	if (!section_is_early) {
858 		memmap_pages_add(-1L * (DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE)));
859 		depopulate_section_memmap(pfn, nr_pages, altmap);
860 	} else if (memmap) {
861 		memmap_boot_pages_add(-1L * (DIV_ROUND_UP(nr_pages * sizeof(struct page),
862 							  PAGE_SIZE)));
863 		free_map_bootmem(memmap);
864 	}
865 
866 	if (empty)
867 		ms->section_mem_map = (unsigned long)NULL;
868 }
869 
section_activate(int nid,unsigned long pfn,unsigned long nr_pages,struct vmem_altmap * altmap,struct dev_pagemap * pgmap)870 static struct page * __meminit section_activate(int nid, unsigned long pfn,
871 		unsigned long nr_pages, struct vmem_altmap *altmap,
872 		struct dev_pagemap *pgmap)
873 {
874 	struct mem_section *ms = __pfn_to_section(pfn);
875 	struct mem_section_usage *usage = NULL;
876 	struct page *memmap;
877 	int rc;
878 
879 	if (!ms->usage) {
880 		usage = kzalloc(mem_section_usage_size(), GFP_KERNEL);
881 		if (!usage)
882 			return ERR_PTR(-ENOMEM);
883 		ms->usage = usage;
884 	}
885 
886 	rc = fill_subsection_map(pfn, nr_pages);
887 	if (rc) {
888 		if (usage)
889 			ms->usage = NULL;
890 		kfree(usage);
891 		return ERR_PTR(rc);
892 	}
893 
894 	/*
895 	 * The early init code does not consider partially populated
896 	 * initial sections, it simply assumes that memory will never be
897 	 * referenced.  If we hot-add memory into such a section then we
898 	 * do not need to populate the memmap and can simply reuse what
899 	 * is already there.
900 	 */
901 	if (nr_pages < PAGES_PER_SECTION && early_section(ms))
902 		return pfn_to_page(pfn);
903 
904 	memmap = populate_section_memmap(pfn, nr_pages, nid, altmap, pgmap);
905 	if (!memmap) {
906 		section_deactivate(pfn, nr_pages, altmap);
907 		return ERR_PTR(-ENOMEM);
908 	}
909 	memmap_pages_add(DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE));
910 
911 	return memmap;
912 }
913 
914 /**
915  * sparse_add_section - add a memory section, or populate an existing one
916  * @nid: The node to add section on
917  * @start_pfn: start pfn of the memory range
918  * @nr_pages: number of pfns to add in the section
919  * @altmap: alternate pfns to allocate the memmap backing store
920  * @pgmap: alternate compound page geometry for devmap mappings
921  *
922  * This is only intended for hotplug.
923  *
924  * Note that only VMEMMAP supports sub-section aligned hotplug,
925  * the proper alignment and size are gated by check_pfn_span().
926  *
927  *
928  * Return:
929  * * 0		- On success.
930  * * -EEXIST	- Section has been present.
931  * * -ENOMEM	- Out of memory.
932  */
sparse_add_section(int nid,unsigned long start_pfn,unsigned long nr_pages,struct vmem_altmap * altmap,struct dev_pagemap * pgmap)933 int __meminit sparse_add_section(int nid, unsigned long start_pfn,
934 		unsigned long nr_pages, struct vmem_altmap *altmap,
935 		struct dev_pagemap *pgmap)
936 {
937 	unsigned long section_nr = pfn_to_section_nr(start_pfn);
938 	struct mem_section *ms;
939 	struct page *memmap;
940 	int ret;
941 
942 	ret = sparse_index_init(section_nr, nid);
943 	if (ret < 0)
944 		return ret;
945 
946 	memmap = section_activate(nid, start_pfn, nr_pages, altmap, pgmap);
947 	if (IS_ERR(memmap))
948 		return PTR_ERR(memmap);
949 
950 	/*
951 	 * Poison uninitialized struct pages in order to catch invalid flags
952 	 * combinations.
953 	 */
954 	if (!altmap || !altmap->inaccessible)
955 		page_init_poison(memmap, sizeof(struct page) * nr_pages);
956 
957 	ms = __nr_to_section(section_nr);
958 	set_section_nid(section_nr, nid);
959 	__section_mark_present(ms, section_nr);
960 
961 	/* Align memmap to section boundary in the subsection case */
962 	if (section_nr_to_pfn(section_nr) != start_pfn)
963 		memmap = pfn_to_page(section_nr_to_pfn(section_nr));
964 	sparse_init_one_section(ms, section_nr, memmap, ms->usage, 0);
965 
966 	return 0;
967 }
968 
sparse_remove_section(unsigned long pfn,unsigned long nr_pages,struct vmem_altmap * altmap)969 void sparse_remove_section(unsigned long pfn, unsigned long nr_pages,
970 			   struct vmem_altmap *altmap)
971 {
972 	struct mem_section *ms = __pfn_to_section(pfn);
973 
974 	if (WARN_ON_ONCE(!valid_section(ms)))
975 		return;
976 
977 	section_deactivate(pfn, nr_pages, altmap);
978 }
979 #endif /* CONFIG_MEMORY_HOTPLUG */
980