xref: /freebsd/sys/kern/subr_physmem.c (revision ee97f198b42d50437f87aa4111d478eca2a5be16)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2014 Ian Lepore <ian@freebsd.org>
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  */
28 
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
31 
32 #ifdef _KERNEL
33 #include "opt_acpi.h"
34 #include "opt_ddb.h"
35 #endif
36 
37 /*
38  * Routines for describing and initializing anything related to physical memory.
39  */
40 
41 #include <sys/param.h>
42 #include <sys/systm.h>
43 #include <sys/bus.h>
44 #include <sys/kernel.h>
45 #include <sys/module.h>
46 #include <sys/physmem.h>
47 
48 #ifdef _KERNEL
49 #include <vm/vm.h>
50 #include <vm/vm_param.h>
51 #include <vm/vm_page.h>
52 #include <vm/vm_phys.h>
53 #include <vm/vm_dumpset.h>
54 
55 #include <machine/md_var.h>
56 #include <machine/resource.h>
57 #else
58 #include <stdarg.h>
59 #include <stdio.h>
60 #include <string.h>
61 #endif
62 
63 /*
64  * These structures are used internally to keep track of regions of physical
65  * ram, and regions within the physical ram that need to be excluded.  An
66  * exclusion region can be excluded from crash dumps, from the vm pool of pages
67  * that can be allocated, or both, depending on the exclusion flags associated
68  * with the region.
69  */
70 #ifdef DEV_ACPI
71 #define	MAX_HWCNT	32	/* ACPI needs more regions */
72 #define	MAX_EXCNT	32
73 #else
74 #define	MAX_HWCNT	16
75 #define	MAX_EXCNT	16
76 #endif
77 
78 #if defined(__arm__)
79 #define	MAX_PHYS_ADDR	0xFFFFFFFFull
80 #elif defined(__aarch64__) || defined(__amd64__) || defined(__riscv)
81 #define	MAX_PHYS_ADDR	0xFFFFFFFFFFFFFFFFull
82 #endif
83 
84 struct region {
85 	vm_paddr_t	addr;
86 	vm_size_t	size;
87 	uint32_t	flags;
88 };
89 
90 static struct region hwregions[MAX_HWCNT];
91 static struct region exregions[MAX_EXCNT];
92 
93 static size_t hwcnt;
94 static size_t excnt;
95 
96 /*
97  * realmem is the total number of hardware pages, excluded or not.
98  * Maxmem is one greater than the last physical page number.
99  */
100 long realmem;
101 long Maxmem;
102 
103 #ifndef _KERNEL
104 static void
105 panic(const char *fmt, ...)
106 {
107 	va_list va;
108 
109 	va_start(va, fmt);
110 	vfprintf(stderr, fmt, va);
111 	fprintf(stderr, "\n");
112 	va_end(va);
113 	__builtin_trap();
114 }
115 #endif
116 
117 /*
118  * Print the contents of the physical and excluded region tables using the
119  * provided printf-like output function (which will be either printf or
120  * db_printf).
121  */
122 static void
123 physmem_dump_tables(int (*prfunc)(const char *, ...) __printflike(1, 2))
124 {
125 	size_t i;
126 	int flags;
127 	uintmax_t addr, size;
128 	const unsigned int mbyte = 1024 * 1024;
129 
130 	prfunc("Physical memory chunk(s):\n");
131 	for (i = 0; i < hwcnt; ++i) {
132 		addr = hwregions[i].addr;
133 		size = hwregions[i].size;
134 		prfunc("  0x%08jx - 0x%08jx, %5ju MB (%7ju pages)\n", addr,
135 		    addr + size - 1, size / mbyte, size / PAGE_SIZE);
136 	}
137 
138 	prfunc("Excluded memory regions:\n");
139 	for (i = 0; i < excnt; ++i) {
140 		addr  = exregions[i].addr;
141 		size  = exregions[i].size;
142 		flags = exregions[i].flags;
143 		prfunc("  0x%08jx - 0x%08jx, %5ju MB (%7ju pages) %s %s\n",
144 		    addr, addr + size - 1, size / mbyte, size / PAGE_SIZE,
145 		    (flags & EXFLAG_NOALLOC) ? "NoAlloc" : "",
146 		    (flags & EXFLAG_NODUMP)  ? "NoDump" : "");
147 	}
148 
149 #ifdef DEBUG
150 	prfunc("Avail lists:\n");
151 	for (i = 0; phys_avail[i] != 0; ++i) {
152 		prfunc("  phys_avail[%zu] 0x%08jx\n", i,
153 		    (uintmax_t)phys_avail[i]);
154 	}
155 	for (i = 0; dump_avail[i] != 0; ++i) {
156 		prfunc("  dump_avail[%zu] 0x%08jx\n", i,
157 		    (uintmax_t)dump_avail[i]);
158 	}
159 #endif
160 }
161 
162 /*
163  * Print the contents of the static mapping table.  Used for bootverbose.
164  */
165 void
166 physmem_print_tables(void)
167 {
168 
169 	physmem_dump_tables(printf);
170 }
171 
172 /*
173  * Walk the list of hardware regions, processing it against the list of
174  * exclusions that contain the given exflags, and generating an "avail list".
175  *
176  * If maxphyssz is not zero it sets upper limit, in bytes, for the total
177  * "avail list" size. Walk stops once the limit is reached and the last region
178  * is cut short if necessary.
179  *
180  * Updates the value at *pavail with the sum of all pages in all hw regions.
181  *
182  * Returns the number of pages of non-excluded memory added to the avail list.
183  */
184 static size_t
185 regions_to_avail(vm_paddr_t *avail, uint32_t exflags, size_t maxavail,
186     uint64_t maxphyssz, long *pavail, long *prealmem)
187 {
188 	size_t acnt, exi, hwi;
189 	uint64_t adj, end, start, xend, xstart;
190 	long availmem, totalmem;
191 	const struct region *exp, *hwp;
192 	uint64_t availsz;
193 
194 	totalmem = 0;
195 	availmem = 0;
196 	availsz = 0;
197 	acnt = 0;
198 	for (hwi = 0, hwp = hwregions; hwi < hwcnt; ++hwi, ++hwp) {
199 		adj   = round_page(hwp->addr) - hwp->addr;
200 		start = round_page(hwp->addr);
201 		end   = trunc_page(hwp->size + adj) + start;
202 		totalmem += atop((vm_offset_t)(end - start));
203 		for (exi = 0, exp = exregions; exi < excnt; ++exi, ++exp) {
204 			/*
205 			 * If the excluded region does not match given flags,
206 			 * continue checking with the next excluded region.
207 			 */
208 			if ((exp->flags & exflags) == 0)
209 				continue;
210 			xstart = exp->addr;
211 			xend   = exp->size + xstart;
212 			/*
213 			 * If the excluded region ends before this hw region,
214 			 * continue checking with the next excluded region.
215 			 */
216 			if (xend <= start)
217 				continue;
218 			/*
219 			 * If the excluded region begins after this hw region
220 			 * we're done because both lists are sorted.
221 			 */
222 			if (xstart >= end)
223 				break;
224 			/*
225 			 * If the excluded region completely covers this hw
226 			 * region, shrink this hw region to zero size.
227 			 */
228 			if ((start >= xstart) && (end <= xend)) {
229 				start = xend;
230 				end = xend;
231 				break;
232 			}
233 			/*
234 			 * If the excluded region falls wholly within this hw
235 			 * region without abutting or overlapping the beginning
236 			 * or end, create an available entry from the leading
237 			 * fragment, then adjust the start of this hw region to
238 			 * the end of the excluded region, and continue checking
239 			 * the next excluded region because another exclusion
240 			 * could affect the remainder of this hw region.
241 			 */
242 			if ((xstart > start) && (xend < end)) {
243 
244 				if ((maxphyssz != 0) &&
245 				    (availsz + xstart - start > maxphyssz)) {
246 					xstart = maxphyssz + start - availsz;
247 				}
248 				if (xstart <= start)
249 					continue;
250 				if (acnt > 0 &&
251 				    avail[acnt - 1] == (vm_paddr_t)start) {
252 					avail[acnt - 1] = (vm_paddr_t)xstart;
253 				} else {
254 					avail[acnt++] = (vm_paddr_t)start;
255 					avail[acnt++] = (vm_paddr_t)xstart;
256 				}
257 				availsz += (xstart - start);
258 				availmem += atop((vm_offset_t)(xstart - start));
259 				start = xend;
260 				continue;
261 			}
262 			/*
263 			 * We know the excluded region overlaps either the start
264 			 * or end of this hardware region (but not both), trim
265 			 * the excluded portion off the appropriate end.
266 			 */
267 			if (xstart <= start)
268 				start = xend;
269 			else
270 				end = xstart;
271 		}
272 		/*
273 		 * If the trimming actions above left a non-zero size, create an
274 		 * available entry for it.
275 		 */
276 		if (end > start) {
277 			if ((maxphyssz != 0) &&
278 			    (availsz + end - start > maxphyssz)) {
279 				end = maxphyssz + start - availsz;
280 			}
281 			if (end <= start)
282 				break;
283 
284 			if (acnt > 0 && avail[acnt - 1] == (vm_paddr_t)start) {
285 				avail[acnt - 1] = (vm_paddr_t)end;
286 			} else {
287 				avail[acnt++] = (vm_paddr_t)start;
288 				avail[acnt++] = (vm_paddr_t)end;
289 			}
290 			availsz += end - start;
291 			availmem += atop((vm_offset_t)(end - start));
292 		}
293 		if (acnt >= maxavail)
294 			panic("Not enough space in the dump/phys_avail arrays");
295 	}
296 
297 	if (pavail != NULL)
298 		*pavail = availmem;
299 	if (prealmem != NULL)
300 		*prealmem = totalmem;
301 	return (acnt);
302 }
303 
304 /*
305  * Check if the region at idx can be merged with the region above it.
306  */
307 static size_t
308 merge_upper_regions(struct region *regions, size_t rcnt, size_t idx)
309 {
310 	struct region *lower, *upper;
311 	vm_paddr_t lend, uend;
312 	size_t i, mergecnt, movecnt;
313 
314 	lower = &regions[idx];
315 	lend = lower->addr + lower->size;
316 
317 	/*
318 	 * Continue merging in upper entries as long as we have entries to
319 	 * merge; the new block could have spanned more than one, although one
320 	 * is likely the common case.
321 	 */
322 	for (i = idx + 1; i < rcnt; i++) {
323 		upper = &regions[i];
324 		if (lend < upper->addr || lower->flags != upper->flags)
325 			break;
326 
327 		uend = upper->addr + upper->size;
328 		if (uend > lend) {
329 			lower->size += uend - lend;
330 			lend = lower->addr + lower->size;
331 		}
332 
333 		if (uend >= lend) {
334 			/*
335 			 * If we didn't move past the end of the upper region,
336 			 * then we don't need to bother checking for another
337 			 * merge because it would have been done already.  Just
338 			 * increment i once more to maintain the invariant that
339 			 * i is one past the last entry merged.
340 			 */
341 			i++;
342 			break;
343 		}
344 	}
345 
346 	/*
347 	 * We merged in the entries from [idx + 1, i); physically move the tail
348 	 * end at [i, rcnt) if we need to.
349 	 */
350 	mergecnt = i - (idx + 1);
351 	if (mergecnt > 0) {
352 		movecnt = rcnt - i;
353 		if (movecnt == 0) {
354 			/* Merged all the way to the end, just decrease rcnt. */
355 			rcnt = idx + 1;
356 		} else {
357 			memmove(&regions[idx + 1], &regions[idx + mergecnt + 1],
358 			    movecnt * sizeof(*regions));
359 			rcnt -= mergecnt;
360 		}
361 	}
362 	return (rcnt);
363 }
364 
365 /*
366  * Insertion-sort a new entry into a regions list; sorted by start address.
367  */
368 static size_t
369 insert_region(struct region *regions, size_t rcnt, vm_paddr_t addr,
370     vm_size_t size, uint32_t flags)
371 {
372 	size_t i;
373 	vm_paddr_t nend, rend;
374 	struct region *ep, *rp;
375 
376 	nend = addr + size;
377 	ep = regions + rcnt;
378 	for (i = 0, rp = regions; i < rcnt; ++i, ++rp) {
379 		if (flags == rp->flags) {
380 			rend = rp->addr + rp->size;
381 			if (addr <= rp->addr && nend >= rp->addr) {
382 				/*
383 				 * New mapping overlaps at the beginning, shift
384 				 * for any difference in the beginning then
385 				 * shift if the new mapping extends past.
386 				 */
387 				rp->size += rp->addr - addr;
388 				rp->addr = addr;
389 				if (nend > rend) {
390 					rp->size += nend - rend;
391 					rcnt = merge_upper_regions(regions,
392 					    rcnt, i);
393 				}
394 				return (rcnt);
395 			} else if (addr <= rend && nend > rp->addr) {
396 				/*
397 				 * New mapping is either entirely contained
398 				 * within or it's overlapping at the end.
399 				 */
400 				if (nend > rend) {
401 					rp->size += nend - rend;
402 					rcnt = merge_upper_regions(regions,
403 					    rcnt, i);
404 				}
405 				return (rcnt);
406 			}
407 		}
408 		if (addr < rp->addr) {
409 			bcopy(rp, rp + 1, (ep - rp) * sizeof(*rp));
410 			break;
411 		}
412 	}
413 	rp->addr  = addr;
414 	rp->size  = size;
415 	rp->flags = flags;
416 	rcnt++;
417 
418 	return (rcnt);
419 }
420 
421 /*
422  * Add a hardware memory region.
423  */
424 void
425 physmem_hardware_region(uint64_t pa, uint64_t sz)
426 {
427 	/*
428 	 * Filter out the page at PA 0x00000000.  The VM can't handle it, as
429 	 * pmap_extract() == 0 means failure.
430 	 */
431 	if (pa == 0) {
432 		if (sz <= PAGE_SIZE)
433 			return;
434 		pa  = PAGE_SIZE;
435 		sz -= PAGE_SIZE;
436 	} else if (pa > MAX_PHYS_ADDR) {
437 		/* This range is past usable memory, ignore it */
438 		return;
439 	}
440 
441 	/*
442 	 * Also filter out the page at the end of the physical address space --
443 	 * if addr is non-zero and addr+size is zero we wrapped to the next byte
444 	 * beyond what vm_paddr_t can express.  That leads to a NULL pointer
445 	 * deref early in startup; work around it by leaving the last page out.
446 	 *
447 	 * XXX This just in:  subtract out a whole megabyte, not just 1 page.
448 	 * Reducing the size by anything less than 1MB results in the NULL
449 	 * pointer deref in _vm_map_lock_read().  Better to give up a megabyte
450 	 * than leave some folks with an unusable system while we investigate.
451 	 */
452 	if ((pa + sz) > (MAX_PHYS_ADDR - 1024 * 1024)) {
453 		sz = MAX_PHYS_ADDR - pa + 1;
454 		if (sz <= 1024 * 1024)
455 			return;
456 		sz -= 1024 * 1024;
457 	}
458 
459 	if (sz > 0 && hwcnt < nitems(hwregions))
460 		hwcnt = insert_region(hwregions, hwcnt, pa, sz, 0);
461 }
462 
463 /*
464  * Add an exclusion region.
465  */
466 void
467 physmem_exclude_region(vm_paddr_t pa, vm_size_t sz, uint32_t exflags)
468 {
469 	vm_offset_t adj;
470 
471 	/*
472 	 * Truncate the starting address down to a page boundary, and round the
473 	 * ending page up to a page boundary.
474 	 */
475 	adj = pa - trunc_page(pa);
476 	pa  = trunc_page(pa);
477 	sz  = round_page(sz + adj);
478 
479 	if (excnt >= nitems(exregions))
480 		panic("failed to exclude region %#jx-%#jx", (uintmax_t)pa,
481 		    (uintmax_t)(pa + sz));
482 	excnt = insert_region(exregions, excnt, pa, sz, exflags);
483 }
484 
485 size_t
486 physmem_avail(vm_paddr_t *avail, size_t maxavail)
487 {
488 
489 	return (regions_to_avail(avail, EXFLAG_NOALLOC, maxavail, 0, NULL, NULL));
490 }
491 
492 bool
493 physmem_excluded(vm_paddr_t pa, vm_size_t sz)
494 {
495 	const struct region *exp;
496 	size_t exi;
497 
498 	for (exi = 0, exp = exregions; exi < excnt; ++exi, ++exp) {
499 		if (pa < exp->addr || pa + sz > exp->addr + exp->size)
500 			continue;
501 		return (true);
502 	}
503 	return (false);
504 }
505 
506 #ifdef _KERNEL
507 /*
508  * Process all the regions added earlier into the global avail lists.
509  *
510  * Updates the kernel global 'physmem' with the number of physical pages
511  * available for use (all pages not in any exclusion region).
512  *
513  * Updates the kernel global 'Maxmem' with the page number one greater then the
514  * last page of physical memory in the system.
515  */
516 void
517 physmem_init_kernel_globals(void)
518 {
519 	size_t nextidx;
520 	u_long hwphyssz;
521 
522 	hwphyssz = 0;
523 	TUNABLE_ULONG_FETCH("hw.physmem", &hwphyssz);
524 
525 	regions_to_avail(dump_avail, EXFLAG_NODUMP, PHYS_AVAIL_ENTRIES,
526 	    hwphyssz, NULL, NULL);
527 	nextidx = regions_to_avail(phys_avail, EXFLAG_NOALLOC,
528 	    PHYS_AVAIL_ENTRIES, hwphyssz, &physmem, &realmem);
529 	if (nextidx == 0)
530 		panic("No memory entries in phys_avail");
531 	Maxmem = atop(phys_avail[nextidx - 1]);
532 }
533 
534 #ifdef DDB
535 #include <ddb/ddb.h>
536 
537 DB_SHOW_COMMAND_FLAGS(physmem, db_show_physmem, DB_CMD_MEMSAFE)
538 {
539 
540 	physmem_dump_tables(db_printf);
541 }
542 
543 #endif /* DDB */
544 
545 /*
546  * ram pseudo driver - this reserves I/O space resources corresponding to physical
547  * memory regions.
548  */
549 
550 static void
551 ram_identify(driver_t *driver, device_t parent)
552 {
553 
554 	if (resource_disabled("ram", 0))
555 		return;
556 	if (BUS_ADD_CHILD(parent, 0, "ram", 0) == NULL)
557 		panic("ram_identify");
558 }
559 
560 static int
561 ram_probe(device_t dev)
562 {
563 
564 	device_quiet(dev);
565 	device_set_desc(dev, "System RAM");
566 	return (BUS_PROBE_SPECIFIC);
567 }
568 
569 static int
570 ram_attach(device_t dev)
571 {
572 	vm_paddr_t avail_list[PHYS_AVAIL_COUNT];
573 	rman_res_t start, end;
574 	struct region *hwp;
575 	int rid, i;
576 
577 	rid = 0;
578 
579 	/* Get the avail list. */
580 	bzero(avail_list, sizeof(avail_list));
581 	regions_to_avail(avail_list, EXFLAG_NOALLOC | EXFLAG_NODUMP,
582 	    PHYS_AVAIL_COUNT, 0, NULL, NULL);
583 
584 	/* Reserve all memory regions. */
585 	for (i = 0; avail_list[i + 1] != 0; i += 2) {
586 		start = avail_list[i];
587 		end = avail_list[i + 1];
588 
589 		if (bootverbose)
590 			device_printf(dev,
591 			    "reserving memory region:   %jx-%jx\n",
592 			    (uintmax_t)start, (uintmax_t)end);
593 
594 		if (bus_alloc_resource(dev, SYS_RES_MEMORY, &rid, start, end,
595 		    end - start, 0) == NULL)
596 			panic("ram_attach: resource %d failed to attach", rid);
597 		rid++;
598 	}
599 
600 	/* Now, reserve the excluded memory regions. */
601 	for (i = 0, hwp = exregions; i < excnt; i++, hwp++) {
602 		start = hwp->addr;
603 		end = hwp->addr + hwp->size;
604 
605 		if (bootverbose)
606 			device_printf(dev,
607 			    "reserving excluded region: %jx-%jx\n",
608 			    (uintmax_t)start, (uintmax_t)(end - 1));
609 
610 		/*
611 		 * Best-effort attempt to reserve the range. This may fail, as
612 		 * sometimes the excluded ranges provided by the device tree
613 		 * will cover or overlap some I/O range.
614 		 */
615 		if (bus_alloc_resource(dev, SYS_RES_MEMORY, &rid, start, end,
616 		    end - start, 0) == NULL) {
617 			if (bootverbose)
618 				device_printf(dev, "failed to reserve region\n");
619 			continue;
620 		}
621 		rid++;
622 	}
623 
624 	return (0);
625 }
626 
627 static device_method_t ram_methods[] = {
628 	/* Device interface */
629 	DEVMETHOD(device_identify,	ram_identify),
630 	DEVMETHOD(device_probe,		ram_probe),
631 	DEVMETHOD(device_attach,	ram_attach),
632 
633 	DEVMETHOD_END
634 };
635 
636 DEFINE_CLASS_0(ram, ram_driver, ram_methods, /* no softc */ 1);
637 DRIVER_MODULE(ram, nexus, ram_driver, 0, 0);
638 #endif /* _KERNEL */
639