1 // SPDX-License-Identifier: GPL-2.0-or-later
26 	for (i = 0; i < ARRAY_SIZE(mi->blk); i++)  in numa_nodemask_from_meminfo()
27 		if (mi->blk[i].start != mi->blk[i].end &&  in numa_nodemask_from_meminfo()
28 		    mi->blk[i].nid != NUMA_NO_NODE)  in numa_nodemask_from_meminfo()
29 			node_set(mi->blk[i].nid, *nodemask);  in numa_nodemask_from_meminfo()
33  * numa_reset_distance - Reset NUMA distance table
69 		return -ENOMEM;  in numa_alloc_distance()
85  * numa_set_distance - Set NUMA distance from one NUMA to another
133 static int __init numa_add_memblk_to(int nid, u64 start, u64 end,  in numa_add_memblk_to()  argument
137 	if (start == end)  in numa_add_memblk_to()
141 	if (start > end || nid < 0 || nid >= MAX_NUMNODES) {  in numa_add_memblk_to()
142 		pr_warn("Warning: invalid memblk node %d [mem %#010Lx-%#010Lx]\n",  in numa_add_memblk_to()
143 			nid, start, end - 1);  in numa_add_memblk_to()
147 	if (mi->nr_blks >= NR_NODE_MEMBLKS) {  in numa_add_memblk_to()
149 		return -EINVAL;  in numa_add_memblk_to()
152 	mi->blk[mi->nr_blks].start = start;  in numa_add_memblk_to()
153 	mi->blk[mi->nr_blks].end = end;  in numa_add_memblk_to()
154 	mi->blk[mi->nr_blks].nid = nid;  in numa_add_memblk_to()
155 	mi->nr_blks++;  in numa_add_memblk_to()
160  * numa_remove_memblk_from - Remove one numa_memblk from a numa_meminfo
164  * Remove @idx'th numa_memblk from @mi by shifting @mi->blk[] and
165  * decrementing @mi->nr_blks.
169 	mi->nr_blks--;  in numa_remove_memblk_from()
170 	memmove(&mi->blk[idx], &mi->blk[idx + 1],  in numa_remove_memblk_from()
171 		(mi->nr_blks - idx) * sizeof(mi->blk[0]));  in numa_remove_memblk_from()
175  * numa_move_tail_memblk - Move a numa_memblk from one numa_meminfo to another
183 	dst->blk[dst->nr_blks++] = src->blk[idx];  in numa_move_tail_memblk()
188  * numa_add_memblk - Add one numa_memblk to numa_meminfo
191  * @end: End address of the new memblk
196  * 0 on success, -errno on failure.
198 int __init numa_add_memblk(int nid, u64 start, u64 end)  in numa_add_memblk()  argument
200 	return numa_add_memblk_to(nid, start, end, &numa_meminfo);  in numa_add_memblk()
204  * numa_add_reserved_memblk - Add one numa_memblk to numa_reserved_meminfo
207  * @end: End address of the new memblk
218  * 0 on success, -errno on failure.
220 int __init numa_add_reserved_memblk(int nid, u64 start, u64 end)  in numa_add_reserved_memblk()  argument
222 	return numa_add_memblk_to(nid, start, end, &numa_reserved_meminfo);  in numa_add_reserved_memblk()
226  * numa_cleanup_meminfo - Cleanup a numa_meminfo
233  * 0 on success, -errno on failure.
238 	const u64 high = memblock_end_of_DRAM();  in numa_cleanup_meminfo()  local
242 	for (i = 0; i < mi->nr_blks; i++) {  in numa_cleanup_meminfo()
243 		struct numa_memblk *bi = &mi->blk[i];  in numa_cleanup_meminfo()
247 					bi->start, bi->end - bi->start)) {  in numa_cleanup_meminfo()
248 			numa_move_tail_memblk(&numa_reserved_meminfo, i--, mi);  in numa_cleanup_meminfo()
252 		/* make sure all non-reserved blocks are inside the limits */  in numa_cleanup_meminfo()
253 		bi->start = max(bi->start, low);  in numa_cleanup_meminfo()
255 		/* preserve info for non-RAM areas above 'max_pfn': */  in numa_cleanup_meminfo()
256 		if (bi->end > high) {  in numa_cleanup_meminfo()
257 			numa_add_memblk_to(bi->nid, high, bi->end,  in numa_cleanup_meminfo()
259 			bi->end = high;  in numa_cleanup_meminfo()
263 		if (bi->start >= bi->end)  in numa_cleanup_meminfo()
264 			numa_remove_memblk_from(i--, mi);  in numa_cleanup_meminfo()
268 	for (i = 0; i < mi->nr_blks; i++) {  in numa_cleanup_meminfo()
269 		struct numa_memblk *bi = &mi->blk[i];  in numa_cleanup_meminfo()
271 		for (j = i + 1; j < mi->nr_blks; j++) {  in numa_cleanup_meminfo()
272 			struct numa_memblk *bj = &mi->blk[j];  in numa_cleanup_meminfo()
273 			u64 start, end;  in numa_cleanup_meminfo()  local
280 			if (bi->end > bj->start && bi->start < bj->end) {  in numa_cleanup_meminfo()
281 				if (bi->nid != bj->nid) {  in numa_cleanup_meminfo()
282 					pr_err("node %d [mem %#010Lx-%#010Lx] overlaps with node %d [mem %#010Lx-%#010Lx]\n",  in numa_cleanup_meminfo()
283 					       bi->nid, bi->start, bi->end - 1,  in numa_cleanup_meminfo()
284 					       bj->nid, bj->start, bj->end - 1);  in numa_cleanup_meminfo()
285 					return -EINVAL;  in numa_cleanup_meminfo()
287 				pr_warn("Warning: node %d [mem %#010Lx-%#010Lx] overlaps with itself [mem %#010Lx-%#010Lx]\n",  in numa_cleanup_meminfo()
288 					bi->nid, bi->start, bi->end - 1,  in numa_cleanup_meminfo()
289 					bj->start, bj->end - 1);  in numa_cleanup_meminfo()
297 			if (bi->nid != bj->nid)  in numa_cleanup_meminfo()
299 			start = min(bi->start, bj->start);  in numa_cleanup_meminfo()
300 			end = max(bi->end, bj->end);  in numa_cleanup_meminfo()
301 			for (k = 0; k < mi->nr_blks; k++) {  in numa_cleanup_meminfo()
302 				struct numa_memblk *bk = &mi->blk[k];  in numa_cleanup_meminfo()
304 				if (bi->nid == bk->nid)  in numa_cleanup_meminfo()
306 				if (start < bk->end && end > bk->start)  in numa_cleanup_meminfo()
309 			if (k < mi->nr_blks)  in numa_cleanup_meminfo()
311 			pr_info("NUMA: Node %d [mem %#010Lx-%#010Lx] + [mem %#010Lx-%#010Lx] -> [mem %#010Lx-%#010Lx]\n",  in numa_cleanup_meminfo()
312 			       bi->nid, bi->start, bi->end - 1, bj->start,  in numa_cleanup_meminfo()
313 			       bj->end - 1, start, end - 1);  in numa_cleanup_meminfo()
314 			bi->start = start;  in numa_cleanup_meminfo()
315 			bi->end = end;  in numa_cleanup_meminfo()
316 			numa_remove_memblk_from(j--, mi);  in numa_cleanup_meminfo()
321 	for (i = mi->nr_blks; i < ARRAY_SIZE(mi->blk); i++) {  in numa_cleanup_meminfo()
322 		mi->blk[i].start = mi->blk[i].end = 0;  in numa_cleanup_meminfo()
323 		mi->blk[i].nid = NUMA_NO_NODE;  in numa_cleanup_meminfo()
330  * Mark all currently memblock-reserved physical memory (which covers the
331  * kernel's own memory ranges) as hot-unswappable.
357 		ret = memblock_set_node(mb->start, mb->end - mb->start,  in numa_clear_kernel_node_hotplug()
358 					&memblock.reserved, mb->nid);  in numa_clear_kernel_node_hotplug()
383 	 * on nodes that contain kernel memory - entire nodes  in numa_clear_kernel_node_hotplug()
384 	 * become hot-unpluggable:  in numa_clear_kernel_node_hotplug()
389 		if (!node_isset(mb->nid, reserved_nodemask))  in numa_clear_kernel_node_hotplug()
392 		memblock_clear_hotplug(mb->start, mb->end - mb->start);  in numa_clear_kernel_node_hotplug()
404 		return -EINVAL;  in numa_register_meminfo()
406 	for (i = 0; i < mi->nr_blks; i++) {  in numa_register_meminfo()
407 		struct numa_memblk *mb = &mi->blk[i];  in numa_register_meminfo()
409 		memblock_set_node(mb->start, mb->end - mb->start,  in numa_register_meminfo()
410 				  &memblock.memory, mb->nid);  in numa_register_meminfo()
416 	 * node the kernel resides in should be un-hotpluggable.  in numa_register_meminfo()
423 	 * If sections array is gonna be used for pfn -> nid mapping, check  in numa_register_meminfo()
436 			return -EINVAL;  in numa_register_meminfo()
465 	 * We reset memblock back to the top-down direction  in numa_memblks_init()
489 	return (ma->start > mb->start) - (ma->start < mb->start);  in cmp_memblk()
495  * numa_fill_memblks - Fill gaps in numa_meminfo memblks
497  * @end: address to end fill
500  * address range @start-@end
504  * NUMA_NO_MEMBLK : No memblks exist in address range @start-@end
507 int __init numa_fill_memblks(u64 start, u64 end)  in numa_fill_memblks()  argument
516 	 * overlap start, end. The list is used to make in-place  in numa_fill_memblks()
519 	for (int i = 0; i < mi->nr_blks; i++) {  in numa_fill_memblks()
520 		struct numa_memblk *bi = &mi->blk[i];  in numa_fill_memblks()
522 		if (memblock_addrs_overlap(start, end - start, bi->start,  in numa_fill_memblks()
523 					   bi->end - bi->start)) {  in numa_fill_memblks()
524 			blk[count] = &mi->blk[i];  in numa_fill_memblks()
531 	/* Sort the list of pointers in memblk->start order */  in numa_fill_memblks()
534 	/* Make sure the first/last memblks include start/end */  in numa_fill_memblks()
535 	blk[0]->start = min(blk[0]->start, start);  in numa_fill_memblks()
536 	blk[count - 1]->end = max(blk[count - 1]->end, end);  in numa_fill_memblks()
540 	 * end address and backfilling to it if needed.  in numa_fill_memblks()
542 	prev_end = blk[0]->end;  in numa_fill_memblks()
546 		if (prev_end >= curr->start) {  in numa_fill_memblks()
547 			if (prev_end < curr->end)  in numa_fill_memblks()
548 				prev_end = curr->end;  in numa_fill_memblks()
550 			curr->start = prev_end;  in numa_fill_memblks()
551 			prev_end = curr->end;  in numa_fill_memblks()
562 	for (i = 0; i < mi->nr_blks; i++)  in meminfo_to_nid()
563 		if (mi->blk[i].start <= start && mi->blk[i].end > start)  in meminfo_to_nid()
564 			return mi->blk[i].nid;  in meminfo_to_nid()
574 	 * hot-added continue the search with reserved ranges.  in phys_to_target_node()