1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2002-2006 Rice University 5 * Copyright (c) 2007 Alan L. Cox <alc@cs.rice.edu> 6 * All rights reserved. 7 * 8 * This software was developed for the FreeBSD Project by Alan L. Cox, 9 * Olivier Crameri, Peter Druschel, Sitaram Iyer, and Juan Navarro. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 * HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 25 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 26 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 27 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 28 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY 30 * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /* 35 * Physical memory system implementation 36 * 37 * Any external functions defined by this module are only to be used by the 38 * virtual memory system. 39 */ 40 41 #include <sys/cdefs.h> 42 __FBSDID("$FreeBSD$"); 43 44 #include "opt_ddb.h" 45 #include "opt_vm.h" 46 47 #include <sys/param.h> 48 #include <sys/systm.h> 49 #include <sys/lock.h> 50 #include <sys/kernel.h> 51 #include <sys/malloc.h> 52 #include <sys/mutex.h> 53 #include <sys/proc.h> 54 #include <sys/queue.h> 55 #include <sys/rwlock.h> 56 #include <sys/sbuf.h> 57 #include <sys/sysctl.h> 58 #include <sys/tree.h> 59 #include <sys/vmmeter.h> 60 #include <sys/seq.h> 61 62 #include <ddb/ddb.h> 63 64 #include <vm/vm.h> 65 #include <vm/vm_param.h> 66 #include <vm/vm_kern.h> 67 #include <vm/vm_object.h> 68 #include <vm/vm_page.h> 69 #include <vm/vm_phys.h> 70 #include <vm/vm_pagequeue.h> 71 72 _Static_assert(sizeof(long) * NBBY >= VM_PHYSSEG_MAX, 73 "Too many physsegs."); 74 75 #ifdef NUMA 76 struct mem_affinity __read_mostly *mem_affinity; 77 int __read_mostly *mem_locality; 78 #endif 79 80 int __read_mostly vm_ndomains = 1; 81 82 struct vm_phys_seg __read_mostly vm_phys_segs[VM_PHYSSEG_MAX]; 83 int __read_mostly vm_phys_nsegs; 84 85 struct vm_phys_fictitious_seg; 86 static int vm_phys_fictitious_cmp(struct vm_phys_fictitious_seg *, 87 struct vm_phys_fictitious_seg *); 88 89 RB_HEAD(fict_tree, vm_phys_fictitious_seg) vm_phys_fictitious_tree = 90 RB_INITIALIZER(_vm_phys_fictitious_tree); 91 92 struct vm_phys_fictitious_seg { 93 RB_ENTRY(vm_phys_fictitious_seg) node; 94 /* Memory region data */ 95 vm_paddr_t start; 96 vm_paddr_t end; 97 vm_page_t first_page; 98 }; 99 100 RB_GENERATE_STATIC(fict_tree, vm_phys_fictitious_seg, node, 101 vm_phys_fictitious_cmp); 102 103 static struct rwlock_padalign vm_phys_fictitious_reg_lock; 104 MALLOC_DEFINE(M_FICT_PAGES, "vm_fictitious", "Fictitious VM pages"); 105 106 static struct vm_freelist __aligned(CACHE_LINE_SIZE) 107 vm_phys_free_queues[MAXMEMDOM][VM_NFREELIST][VM_NFREEPOOL][VM_NFREEORDER]; 108 109 static int __read_mostly vm_nfreelists; 110 111 /* 112 * Provides the mapping from VM_FREELIST_* to free list indices (flind). 113 */ 114 static int __read_mostly vm_freelist_to_flind[VM_NFREELIST]; 115 116 CTASSERT(VM_FREELIST_DEFAULT == 0); 117 118 #ifdef VM_FREELIST_ISADMA 119 #define VM_ISADMA_BOUNDARY 16777216 120 #endif 121 #ifdef VM_FREELIST_DMA32 122 #define VM_DMA32_BOUNDARY ((vm_paddr_t)1 << 32) 123 #endif 124 125 /* 126 * Enforce the assumptions made by vm_phys_add_seg() and vm_phys_init() about 127 * the ordering of the free list boundaries. 128 */ 129 #if defined(VM_ISADMA_BOUNDARY) && defined(VM_LOWMEM_BOUNDARY) 130 CTASSERT(VM_ISADMA_BOUNDARY < VM_LOWMEM_BOUNDARY); 131 #endif 132 #if defined(VM_LOWMEM_BOUNDARY) && defined(VM_DMA32_BOUNDARY) 133 CTASSERT(VM_LOWMEM_BOUNDARY < VM_DMA32_BOUNDARY); 134 #endif 135 136 static int sysctl_vm_phys_free(SYSCTL_HANDLER_ARGS); 137 SYSCTL_OID(_vm, OID_AUTO, phys_free, CTLTYPE_STRING | CTLFLAG_RD, 138 NULL, 0, sysctl_vm_phys_free, "A", "Phys Free Info"); 139 140 static int sysctl_vm_phys_segs(SYSCTL_HANDLER_ARGS); 141 SYSCTL_OID(_vm, OID_AUTO, phys_segs, CTLTYPE_STRING | CTLFLAG_RD, 142 NULL, 0, sysctl_vm_phys_segs, "A", "Phys Seg Info"); 143 144 #ifdef NUMA 145 static int sysctl_vm_phys_locality(SYSCTL_HANDLER_ARGS); 146 SYSCTL_OID(_vm, OID_AUTO, phys_locality, CTLTYPE_STRING | CTLFLAG_RD, 147 NULL, 0, sysctl_vm_phys_locality, "A", "Phys Locality Info"); 148 #endif 149 150 SYSCTL_INT(_vm, OID_AUTO, ndomains, CTLFLAG_RD, 151 &vm_ndomains, 0, "Number of physical memory domains available."); 152 153 static vm_page_t vm_phys_alloc_seg_contig(struct vm_phys_seg *seg, 154 u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, 155 vm_paddr_t boundary); 156 static void _vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int domain); 157 static void vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end); 158 static void vm_phys_split_pages(vm_page_t m, int oind, struct vm_freelist *fl, 159 int order); 160 161 /* 162 * Red-black tree helpers for vm fictitious range management. 163 */ 164 static inline int 165 vm_phys_fictitious_in_range(struct vm_phys_fictitious_seg *p, 166 struct vm_phys_fictitious_seg *range) 167 { 168 169 KASSERT(range->start != 0 && range->end != 0, 170 ("Invalid range passed on search for vm_fictitious page")); 171 if (p->start >= range->end) 172 return (1); 173 if (p->start < range->start) 174 return (-1); 175 176 return (0); 177 } 178 179 static int 180 vm_phys_fictitious_cmp(struct vm_phys_fictitious_seg *p1, 181 struct vm_phys_fictitious_seg *p2) 182 { 183 184 /* Check if this is a search for a page */ 185 if (p1->end == 0) 186 return (vm_phys_fictitious_in_range(p1, p2)); 187 188 KASSERT(p2->end != 0, 189 ("Invalid range passed as second parameter to vm fictitious comparison")); 190 191 /* Searching to add a new range */ 192 if (p1->end <= p2->start) 193 return (-1); 194 if (p1->start >= p2->end) 195 return (1); 196 197 panic("Trying to add overlapping vm fictitious ranges:\n" 198 "[%#jx:%#jx] and [%#jx:%#jx]", (uintmax_t)p1->start, 199 (uintmax_t)p1->end, (uintmax_t)p2->start, (uintmax_t)p2->end); 200 } 201 202 int 203 vm_phys_domain_match(int prefer, vm_paddr_t low, vm_paddr_t high) 204 { 205 #ifdef NUMA 206 domainset_t mask; 207 int i; 208 209 if (vm_ndomains == 1 || mem_affinity == NULL) 210 return (0); 211 212 DOMAINSET_ZERO(&mask); 213 /* 214 * Check for any memory that overlaps low, high. 215 */ 216 for (i = 0; mem_affinity[i].end != 0; i++) 217 if (mem_affinity[i].start <= high && 218 mem_affinity[i].end >= low) 219 DOMAINSET_SET(mem_affinity[i].domain, &mask); 220 if (prefer != -1 && DOMAINSET_ISSET(prefer, &mask)) 221 return (prefer); 222 if (DOMAINSET_EMPTY(&mask)) 223 panic("vm_phys_domain_match: Impossible constraint"); 224 return (DOMAINSET_FFS(&mask) - 1); 225 #else 226 return (0); 227 #endif 228 } 229 230 /* 231 * Outputs the state of the physical memory allocator, specifically, 232 * the amount of physical memory in each free list. 233 */ 234 static int 235 sysctl_vm_phys_free(SYSCTL_HANDLER_ARGS) 236 { 237 struct sbuf sbuf; 238 struct vm_freelist *fl; 239 int dom, error, flind, oind, pind; 240 241 error = sysctl_wire_old_buffer(req, 0); 242 if (error != 0) 243 return (error); 244 sbuf_new_for_sysctl(&sbuf, NULL, 128 * vm_ndomains, req); 245 for (dom = 0; dom < vm_ndomains; dom++) { 246 sbuf_printf(&sbuf,"\nDOMAIN %d:\n", dom); 247 for (flind = 0; flind < vm_nfreelists; flind++) { 248 sbuf_printf(&sbuf, "\nFREE LIST %d:\n" 249 "\n ORDER (SIZE) | NUMBER" 250 "\n ", flind); 251 for (pind = 0; pind < VM_NFREEPOOL; pind++) 252 sbuf_printf(&sbuf, " | POOL %d", pind); 253 sbuf_printf(&sbuf, "\n-- "); 254 for (pind = 0; pind < VM_NFREEPOOL; pind++) 255 sbuf_printf(&sbuf, "-- -- "); 256 sbuf_printf(&sbuf, "--\n"); 257 for (oind = VM_NFREEORDER - 1; oind >= 0; oind--) { 258 sbuf_printf(&sbuf, " %2d (%6dK)", oind, 259 1 << (PAGE_SHIFT - 10 + oind)); 260 for (pind = 0; pind < VM_NFREEPOOL; pind++) { 261 fl = vm_phys_free_queues[dom][flind][pind]; 262 sbuf_printf(&sbuf, " | %6d", 263 fl[oind].lcnt); 264 } 265 sbuf_printf(&sbuf, "\n"); 266 } 267 } 268 } 269 error = sbuf_finish(&sbuf); 270 sbuf_delete(&sbuf); 271 return (error); 272 } 273 274 /* 275 * Outputs the set of physical memory segments. 276 */ 277 static int 278 sysctl_vm_phys_segs(SYSCTL_HANDLER_ARGS) 279 { 280 struct sbuf sbuf; 281 struct vm_phys_seg *seg; 282 int error, segind; 283 284 error = sysctl_wire_old_buffer(req, 0); 285 if (error != 0) 286 return (error); 287 sbuf_new_for_sysctl(&sbuf, NULL, 128, req); 288 for (segind = 0; segind < vm_phys_nsegs; segind++) { 289 sbuf_printf(&sbuf, "\nSEGMENT %d:\n\n", segind); 290 seg = &vm_phys_segs[segind]; 291 sbuf_printf(&sbuf, "start: %#jx\n", 292 (uintmax_t)seg->start); 293 sbuf_printf(&sbuf, "end: %#jx\n", 294 (uintmax_t)seg->end); 295 sbuf_printf(&sbuf, "domain: %d\n", seg->domain); 296 sbuf_printf(&sbuf, "free list: %p\n", seg->free_queues); 297 } 298 error = sbuf_finish(&sbuf); 299 sbuf_delete(&sbuf); 300 return (error); 301 } 302 303 /* 304 * Return affinity, or -1 if there's no affinity information. 305 */ 306 int 307 vm_phys_mem_affinity(int f, int t) 308 { 309 310 #ifdef NUMA 311 if (mem_locality == NULL) 312 return (-1); 313 if (f >= vm_ndomains || t >= vm_ndomains) 314 return (-1); 315 return (mem_locality[f * vm_ndomains + t]); 316 #else 317 return (-1); 318 #endif 319 } 320 321 #ifdef NUMA 322 /* 323 * Outputs the VM locality table. 324 */ 325 static int 326 sysctl_vm_phys_locality(SYSCTL_HANDLER_ARGS) 327 { 328 struct sbuf sbuf; 329 int error, i, j; 330 331 error = sysctl_wire_old_buffer(req, 0); 332 if (error != 0) 333 return (error); 334 sbuf_new_for_sysctl(&sbuf, NULL, 128, req); 335 336 sbuf_printf(&sbuf, "\n"); 337 338 for (i = 0; i < vm_ndomains; i++) { 339 sbuf_printf(&sbuf, "%d: ", i); 340 for (j = 0; j < vm_ndomains; j++) { 341 sbuf_printf(&sbuf, "%d ", vm_phys_mem_affinity(i, j)); 342 } 343 sbuf_printf(&sbuf, "\n"); 344 } 345 error = sbuf_finish(&sbuf); 346 sbuf_delete(&sbuf); 347 return (error); 348 } 349 #endif 350 351 static void 352 vm_freelist_add(struct vm_freelist *fl, vm_page_t m, int order, int tail) 353 { 354 355 m->order = order; 356 if (tail) 357 TAILQ_INSERT_TAIL(&fl[order].pl, m, plinks.q); 358 else 359 TAILQ_INSERT_HEAD(&fl[order].pl, m, plinks.q); 360 fl[order].lcnt++; 361 } 362 363 static void 364 vm_freelist_rem(struct vm_freelist *fl, vm_page_t m, int order) 365 { 366 367 TAILQ_REMOVE(&fl[order].pl, m, plinks.q); 368 fl[order].lcnt--; 369 m->order = VM_NFREEORDER; 370 } 371 372 /* 373 * Create a physical memory segment. 374 */ 375 static void 376 _vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int domain) 377 { 378 struct vm_phys_seg *seg; 379 380 KASSERT(vm_phys_nsegs < VM_PHYSSEG_MAX, 381 ("vm_phys_create_seg: increase VM_PHYSSEG_MAX")); 382 KASSERT(domain >= 0 && domain < vm_ndomains, 383 ("vm_phys_create_seg: invalid domain provided")); 384 seg = &vm_phys_segs[vm_phys_nsegs++]; 385 while (seg > vm_phys_segs && (seg - 1)->start >= end) { 386 *seg = *(seg - 1); 387 seg--; 388 } 389 seg->start = start; 390 seg->end = end; 391 seg->domain = domain; 392 } 393 394 static void 395 vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end) 396 { 397 #ifdef NUMA 398 int i; 399 400 if (mem_affinity == NULL) { 401 _vm_phys_create_seg(start, end, 0); 402 return; 403 } 404 405 for (i = 0;; i++) { 406 if (mem_affinity[i].end == 0) 407 panic("Reached end of affinity info"); 408 if (mem_affinity[i].end <= start) 409 continue; 410 if (mem_affinity[i].start > start) 411 panic("No affinity info for start %jx", 412 (uintmax_t)start); 413 if (mem_affinity[i].end >= end) { 414 _vm_phys_create_seg(start, end, 415 mem_affinity[i].domain); 416 break; 417 } 418 _vm_phys_create_seg(start, mem_affinity[i].end, 419 mem_affinity[i].domain); 420 start = mem_affinity[i].end; 421 } 422 #else 423 _vm_phys_create_seg(start, end, 0); 424 #endif 425 } 426 427 /* 428 * Add a physical memory segment. 429 */ 430 void 431 vm_phys_add_seg(vm_paddr_t start, vm_paddr_t end) 432 { 433 vm_paddr_t paddr; 434 435 KASSERT((start & PAGE_MASK) == 0, 436 ("vm_phys_define_seg: start is not page aligned")); 437 KASSERT((end & PAGE_MASK) == 0, 438 ("vm_phys_define_seg: end is not page aligned")); 439 440 /* 441 * Split the physical memory segment if it spans two or more free 442 * list boundaries. 443 */ 444 paddr = start; 445 #ifdef VM_FREELIST_ISADMA 446 if (paddr < VM_ISADMA_BOUNDARY && end > VM_ISADMA_BOUNDARY) { 447 vm_phys_create_seg(paddr, VM_ISADMA_BOUNDARY); 448 paddr = VM_ISADMA_BOUNDARY; 449 } 450 #endif 451 #ifdef VM_FREELIST_LOWMEM 452 if (paddr < VM_LOWMEM_BOUNDARY && end > VM_LOWMEM_BOUNDARY) { 453 vm_phys_create_seg(paddr, VM_LOWMEM_BOUNDARY); 454 paddr = VM_LOWMEM_BOUNDARY; 455 } 456 #endif 457 #ifdef VM_FREELIST_DMA32 458 if (paddr < VM_DMA32_BOUNDARY && end > VM_DMA32_BOUNDARY) { 459 vm_phys_create_seg(paddr, VM_DMA32_BOUNDARY); 460 paddr = VM_DMA32_BOUNDARY; 461 } 462 #endif 463 vm_phys_create_seg(paddr, end); 464 } 465 466 /* 467 * Initialize the physical memory allocator. 468 * 469 * Requires that vm_page_array is initialized! 470 */ 471 void 472 vm_phys_init(void) 473 { 474 struct vm_freelist *fl; 475 struct vm_phys_seg *seg; 476 u_long npages; 477 int dom, flind, freelist, oind, pind, segind; 478 479 /* 480 * Compute the number of free lists, and generate the mapping from the 481 * manifest constants VM_FREELIST_* to the free list indices. 482 * 483 * Initially, the entries of vm_freelist_to_flind[] are set to either 484 * 0 or 1 to indicate which free lists should be created. 485 */ 486 npages = 0; 487 for (segind = vm_phys_nsegs - 1; segind >= 0; segind--) { 488 seg = &vm_phys_segs[segind]; 489 #ifdef VM_FREELIST_ISADMA 490 if (seg->end <= VM_ISADMA_BOUNDARY) 491 vm_freelist_to_flind[VM_FREELIST_ISADMA] = 1; 492 else 493 #endif 494 #ifdef VM_FREELIST_LOWMEM 495 if (seg->end <= VM_LOWMEM_BOUNDARY) 496 vm_freelist_to_flind[VM_FREELIST_LOWMEM] = 1; 497 else 498 #endif 499 #ifdef VM_FREELIST_DMA32 500 if ( 501 #ifdef VM_DMA32_NPAGES_THRESHOLD 502 /* 503 * Create the DMA32 free list only if the amount of 504 * physical memory above physical address 4G exceeds the 505 * given threshold. 506 */ 507 npages > VM_DMA32_NPAGES_THRESHOLD && 508 #endif 509 seg->end <= VM_DMA32_BOUNDARY) 510 vm_freelist_to_flind[VM_FREELIST_DMA32] = 1; 511 else 512 #endif 513 { 514 npages += atop(seg->end - seg->start); 515 vm_freelist_to_flind[VM_FREELIST_DEFAULT] = 1; 516 } 517 } 518 /* Change each entry into a running total of the free lists. */ 519 for (freelist = 1; freelist < VM_NFREELIST; freelist++) { 520 vm_freelist_to_flind[freelist] += 521 vm_freelist_to_flind[freelist - 1]; 522 } 523 vm_nfreelists = vm_freelist_to_flind[VM_NFREELIST - 1]; 524 KASSERT(vm_nfreelists > 0, ("vm_phys_init: no free lists")); 525 /* Change each entry into a free list index. */ 526 for (freelist = 0; freelist < VM_NFREELIST; freelist++) 527 vm_freelist_to_flind[freelist]--; 528 529 /* 530 * Initialize the first_page and free_queues fields of each physical 531 * memory segment. 532 */ 533 #ifdef VM_PHYSSEG_SPARSE 534 npages = 0; 535 #endif 536 for (segind = 0; segind < vm_phys_nsegs; segind++) { 537 seg = &vm_phys_segs[segind]; 538 #ifdef VM_PHYSSEG_SPARSE 539 seg->first_page = &vm_page_array[npages]; 540 npages += atop(seg->end - seg->start); 541 #else 542 seg->first_page = PHYS_TO_VM_PAGE(seg->start); 543 #endif 544 #ifdef VM_FREELIST_ISADMA 545 if (seg->end <= VM_ISADMA_BOUNDARY) { 546 flind = vm_freelist_to_flind[VM_FREELIST_ISADMA]; 547 KASSERT(flind >= 0, 548 ("vm_phys_init: ISADMA flind < 0")); 549 } else 550 #endif 551 #ifdef VM_FREELIST_LOWMEM 552 if (seg->end <= VM_LOWMEM_BOUNDARY) { 553 flind = vm_freelist_to_flind[VM_FREELIST_LOWMEM]; 554 KASSERT(flind >= 0, 555 ("vm_phys_init: LOWMEM flind < 0")); 556 } else 557 #endif 558 #ifdef VM_FREELIST_DMA32 559 if (seg->end <= VM_DMA32_BOUNDARY) { 560 flind = vm_freelist_to_flind[VM_FREELIST_DMA32]; 561 KASSERT(flind >= 0, 562 ("vm_phys_init: DMA32 flind < 0")); 563 } else 564 #endif 565 { 566 flind = vm_freelist_to_flind[VM_FREELIST_DEFAULT]; 567 KASSERT(flind >= 0, 568 ("vm_phys_init: DEFAULT flind < 0")); 569 } 570 seg->free_queues = &vm_phys_free_queues[seg->domain][flind]; 571 } 572 573 /* 574 * Initialize the free queues. 575 */ 576 for (dom = 0; dom < vm_ndomains; dom++) { 577 for (flind = 0; flind < vm_nfreelists; flind++) { 578 for (pind = 0; pind < VM_NFREEPOOL; pind++) { 579 fl = vm_phys_free_queues[dom][flind][pind]; 580 for (oind = 0; oind < VM_NFREEORDER; oind++) 581 TAILQ_INIT(&fl[oind].pl); 582 } 583 } 584 } 585 586 rw_init(&vm_phys_fictitious_reg_lock, "vmfctr"); 587 } 588 589 /* 590 * Split a contiguous, power of two-sized set of physical pages. 591 */ 592 static __inline void 593 vm_phys_split_pages(vm_page_t m, int oind, struct vm_freelist *fl, int order) 594 { 595 vm_page_t m_buddy; 596 597 while (oind > order) { 598 oind--; 599 m_buddy = &m[1 << oind]; 600 KASSERT(m_buddy->order == VM_NFREEORDER, 601 ("vm_phys_split_pages: page %p has unexpected order %d", 602 m_buddy, m_buddy->order)); 603 vm_freelist_add(fl, m_buddy, oind, 0); 604 } 605 } 606 607 /* 608 * Allocate a contiguous, power of two-sized set of physical pages 609 * from the free lists. 610 * 611 * The free page queues must be locked. 612 */ 613 vm_page_t 614 vm_phys_alloc_pages(int domain, int pool, int order) 615 { 616 vm_page_t m; 617 int freelist; 618 619 for (freelist = 0; freelist < VM_NFREELIST; freelist++) { 620 m = vm_phys_alloc_freelist_pages(domain, freelist, pool, order); 621 if (m != NULL) 622 return (m); 623 } 624 return (NULL); 625 } 626 627 int 628 vm_phys_alloc_npages(int domain, int pool, vm_page_t *mp, int cnt) 629 { 630 vm_page_t m; 631 int order, freelist; 632 633 for (freelist = 0; freelist < VM_NFREELIST; freelist++) { 634 for (order = fls(cnt) -1; order >= 0; order--) { 635 m = vm_phys_alloc_freelist_pages(domain, freelist, 636 pool, order); 637 if (m != NULL) { 638 *mp = m; 639 return (1 << order); 640 } 641 } 642 } 643 *mp = NULL; 644 return (0); 645 } 646 647 /* 648 * Allocate a contiguous, power of two-sized set of physical pages from the 649 * specified free list. The free list must be specified using one of the 650 * manifest constants VM_FREELIST_*. 651 * 652 * The free page queues must be locked. 653 */ 654 vm_page_t 655 vm_phys_alloc_freelist_pages(int domain, int freelist, int pool, int order) 656 { 657 struct vm_freelist *alt, *fl; 658 vm_page_t m; 659 int oind, pind, flind; 660 661 KASSERT(domain >= 0 && domain < vm_ndomains, 662 ("vm_phys_alloc_freelist_pages: domain %d is out of range", 663 domain)); 664 KASSERT(freelist < VM_NFREELIST, 665 ("vm_phys_alloc_freelist_pages: freelist %d is out of range", 666 freelist)); 667 KASSERT(pool < VM_NFREEPOOL, 668 ("vm_phys_alloc_freelist_pages: pool %d is out of range", pool)); 669 KASSERT(order < VM_NFREEORDER, 670 ("vm_phys_alloc_freelist_pages: order %d is out of range", order)); 671 672 flind = vm_freelist_to_flind[freelist]; 673 /* Check if freelist is present */ 674 if (flind < 0) 675 return (NULL); 676 677 vm_domain_free_assert_locked(VM_DOMAIN(domain)); 678 fl = &vm_phys_free_queues[domain][flind][pool][0]; 679 for (oind = order; oind < VM_NFREEORDER; oind++) { 680 m = TAILQ_FIRST(&fl[oind].pl); 681 if (m != NULL) { 682 vm_freelist_rem(fl, m, oind); 683 vm_phys_split_pages(m, oind, fl, order); 684 return (m); 685 } 686 } 687 688 /* 689 * The given pool was empty. Find the largest 690 * contiguous, power-of-two-sized set of pages in any 691 * pool. Transfer these pages to the given pool, and 692 * use them to satisfy the allocation. 693 */ 694 for (oind = VM_NFREEORDER - 1; oind >= order; oind--) { 695 for (pind = 0; pind < VM_NFREEPOOL; pind++) { 696 alt = &vm_phys_free_queues[domain][flind][pind][0]; 697 m = TAILQ_FIRST(&alt[oind].pl); 698 if (m != NULL) { 699 vm_freelist_rem(alt, m, oind); 700 vm_phys_set_pool(pool, m, oind); 701 vm_phys_split_pages(m, oind, fl, order); 702 return (m); 703 } 704 } 705 } 706 return (NULL); 707 } 708 709 /* 710 * Find the vm_page corresponding to the given physical address. 711 */ 712 vm_page_t 713 vm_phys_paddr_to_vm_page(vm_paddr_t pa) 714 { 715 struct vm_phys_seg *seg; 716 int segind; 717 718 for (segind = 0; segind < vm_phys_nsegs; segind++) { 719 seg = &vm_phys_segs[segind]; 720 if (pa >= seg->start && pa < seg->end) 721 return (&seg->first_page[atop(pa - seg->start)]); 722 } 723 return (NULL); 724 } 725 726 vm_page_t 727 vm_phys_fictitious_to_vm_page(vm_paddr_t pa) 728 { 729 struct vm_phys_fictitious_seg tmp, *seg; 730 vm_page_t m; 731 732 m = NULL; 733 tmp.start = pa; 734 tmp.end = 0; 735 736 rw_rlock(&vm_phys_fictitious_reg_lock); 737 seg = RB_FIND(fict_tree, &vm_phys_fictitious_tree, &tmp); 738 rw_runlock(&vm_phys_fictitious_reg_lock); 739 if (seg == NULL) 740 return (NULL); 741 742 m = &seg->first_page[atop(pa - seg->start)]; 743 KASSERT((m->flags & PG_FICTITIOUS) != 0, ("%p not fictitious", m)); 744 745 return (m); 746 } 747 748 static inline void 749 vm_phys_fictitious_init_range(vm_page_t range, vm_paddr_t start, 750 long page_count, vm_memattr_t memattr) 751 { 752 long i; 753 754 bzero(range, page_count * sizeof(*range)); 755 for (i = 0; i < page_count; i++) { 756 vm_page_initfake(&range[i], start + PAGE_SIZE * i, memattr); 757 range[i].oflags &= ~VPO_UNMANAGED; 758 range[i].busy_lock = VPB_UNBUSIED; 759 } 760 } 761 762 int 763 vm_phys_fictitious_reg_range(vm_paddr_t start, vm_paddr_t end, 764 vm_memattr_t memattr) 765 { 766 struct vm_phys_fictitious_seg *seg; 767 vm_page_t fp; 768 long page_count; 769 #ifdef VM_PHYSSEG_DENSE 770 long pi, pe; 771 long dpage_count; 772 #endif 773 774 KASSERT(start < end, 775 ("Start of segment isn't less than end (start: %jx end: %jx)", 776 (uintmax_t)start, (uintmax_t)end)); 777 778 page_count = (end - start) / PAGE_SIZE; 779 780 #ifdef VM_PHYSSEG_DENSE 781 pi = atop(start); 782 pe = atop(end); 783 if (pi >= first_page && (pi - first_page) < vm_page_array_size) { 784 fp = &vm_page_array[pi - first_page]; 785 if ((pe - first_page) > vm_page_array_size) { 786 /* 787 * We have a segment that starts inside 788 * of vm_page_array, but ends outside of it. 789 * 790 * Use vm_page_array pages for those that are 791 * inside of the vm_page_array range, and 792 * allocate the remaining ones. 793 */ 794 dpage_count = vm_page_array_size - (pi - first_page); 795 vm_phys_fictitious_init_range(fp, start, dpage_count, 796 memattr); 797 page_count -= dpage_count; 798 start += ptoa(dpage_count); 799 goto alloc; 800 } 801 /* 802 * We can allocate the full range from vm_page_array, 803 * so there's no need to register the range in the tree. 804 */ 805 vm_phys_fictitious_init_range(fp, start, page_count, memattr); 806 return (0); 807 } else if (pe > first_page && (pe - first_page) < vm_page_array_size) { 808 /* 809 * We have a segment that ends inside of vm_page_array, 810 * but starts outside of it. 811 */ 812 fp = &vm_page_array[0]; 813 dpage_count = pe - first_page; 814 vm_phys_fictitious_init_range(fp, ptoa(first_page), dpage_count, 815 memattr); 816 end -= ptoa(dpage_count); 817 page_count -= dpage_count; 818 goto alloc; 819 } else if (pi < first_page && pe > (first_page + vm_page_array_size)) { 820 /* 821 * Trying to register a fictitious range that expands before 822 * and after vm_page_array. 823 */ 824 return (EINVAL); 825 } else { 826 alloc: 827 #endif 828 fp = malloc(page_count * sizeof(struct vm_page), M_FICT_PAGES, 829 M_WAITOK); 830 #ifdef VM_PHYSSEG_DENSE 831 } 832 #endif 833 vm_phys_fictitious_init_range(fp, start, page_count, memattr); 834 835 seg = malloc(sizeof(*seg), M_FICT_PAGES, M_WAITOK | M_ZERO); 836 seg->start = start; 837 seg->end = end; 838 seg->first_page = fp; 839 840 rw_wlock(&vm_phys_fictitious_reg_lock); 841 RB_INSERT(fict_tree, &vm_phys_fictitious_tree, seg); 842 rw_wunlock(&vm_phys_fictitious_reg_lock); 843 844 return (0); 845 } 846 847 void 848 vm_phys_fictitious_unreg_range(vm_paddr_t start, vm_paddr_t end) 849 { 850 struct vm_phys_fictitious_seg *seg, tmp; 851 #ifdef VM_PHYSSEG_DENSE 852 long pi, pe; 853 #endif 854 855 KASSERT(start < end, 856 ("Start of segment isn't less than end (start: %jx end: %jx)", 857 (uintmax_t)start, (uintmax_t)end)); 858 859 #ifdef VM_PHYSSEG_DENSE 860 pi = atop(start); 861 pe = atop(end); 862 if (pi >= first_page && (pi - first_page) < vm_page_array_size) { 863 if ((pe - first_page) <= vm_page_array_size) { 864 /* 865 * This segment was allocated using vm_page_array 866 * only, there's nothing to do since those pages 867 * were never added to the tree. 868 */ 869 return; 870 } 871 /* 872 * We have a segment that starts inside 873 * of vm_page_array, but ends outside of it. 874 * 875 * Calculate how many pages were added to the 876 * tree and free them. 877 */ 878 start = ptoa(first_page + vm_page_array_size); 879 } else if (pe > first_page && (pe - first_page) < vm_page_array_size) { 880 /* 881 * We have a segment that ends inside of vm_page_array, 882 * but starts outside of it. 883 */ 884 end = ptoa(first_page); 885 } else if (pi < first_page && pe > (first_page + vm_page_array_size)) { 886 /* Since it's not possible to register such a range, panic. */ 887 panic( 888 "Unregistering not registered fictitious range [%#jx:%#jx]", 889 (uintmax_t)start, (uintmax_t)end); 890 } 891 #endif 892 tmp.start = start; 893 tmp.end = 0; 894 895 rw_wlock(&vm_phys_fictitious_reg_lock); 896 seg = RB_FIND(fict_tree, &vm_phys_fictitious_tree, &tmp); 897 if (seg->start != start || seg->end != end) { 898 rw_wunlock(&vm_phys_fictitious_reg_lock); 899 panic( 900 "Unregistering not registered fictitious range [%#jx:%#jx]", 901 (uintmax_t)start, (uintmax_t)end); 902 } 903 RB_REMOVE(fict_tree, &vm_phys_fictitious_tree, seg); 904 rw_wunlock(&vm_phys_fictitious_reg_lock); 905 free(seg->first_page, M_FICT_PAGES); 906 free(seg, M_FICT_PAGES); 907 } 908 909 /* 910 * Free a contiguous, power of two-sized set of physical pages. 911 * 912 * The free page queues must be locked. 913 */ 914 void 915 vm_phys_free_pages(vm_page_t m, int order) 916 { 917 struct vm_freelist *fl; 918 struct vm_phys_seg *seg; 919 vm_paddr_t pa; 920 vm_page_t m_buddy; 921 922 KASSERT(m->order == VM_NFREEORDER, 923 ("vm_phys_free_pages: page %p has unexpected order %d", 924 m, m->order)); 925 KASSERT(m->pool < VM_NFREEPOOL, 926 ("vm_phys_free_pages: page %p has unexpected pool %d", 927 m, m->pool)); 928 KASSERT(order < VM_NFREEORDER, 929 ("vm_phys_free_pages: order %d is out of range", order)); 930 seg = &vm_phys_segs[m->segind]; 931 vm_domain_free_assert_locked(VM_DOMAIN(seg->domain)); 932 if (order < VM_NFREEORDER - 1) { 933 pa = VM_PAGE_TO_PHYS(m); 934 do { 935 pa ^= ((vm_paddr_t)1 << (PAGE_SHIFT + order)); 936 if (pa < seg->start || pa >= seg->end) 937 break; 938 m_buddy = &seg->first_page[atop(pa - seg->start)]; 939 if (m_buddy->order != order) 940 break; 941 fl = (*seg->free_queues)[m_buddy->pool]; 942 vm_freelist_rem(fl, m_buddy, order); 943 if (m_buddy->pool != m->pool) 944 vm_phys_set_pool(m->pool, m_buddy, order); 945 order++; 946 pa &= ~(((vm_paddr_t)1 << (PAGE_SHIFT + order)) - 1); 947 m = &seg->first_page[atop(pa - seg->start)]; 948 } while (order < VM_NFREEORDER - 1); 949 } 950 fl = (*seg->free_queues)[m->pool]; 951 vm_freelist_add(fl, m, order, 1); 952 } 953 954 /* 955 * Free a contiguous, arbitrarily sized set of physical pages. 956 * 957 * The free page queues must be locked. 958 */ 959 void 960 vm_phys_free_contig(vm_page_t m, u_long npages) 961 { 962 u_int n; 963 int order; 964 965 /* 966 * Avoid unnecessary coalescing by freeing the pages in the largest 967 * possible power-of-two-sized subsets. 968 */ 969 vm_domain_free_assert_locked(vm_pagequeue_domain(m)); 970 for (;; npages -= n) { 971 /* 972 * Unsigned "min" is used here so that "order" is assigned 973 * "VM_NFREEORDER - 1" when "m"'s physical address is zero 974 * or the low-order bits of its physical address are zero 975 * because the size of a physical address exceeds the size of 976 * a long. 977 */ 978 order = min(ffsl(VM_PAGE_TO_PHYS(m) >> PAGE_SHIFT) - 1, 979 VM_NFREEORDER - 1); 980 n = 1 << order; 981 if (npages < n) 982 break; 983 vm_phys_free_pages(m, order); 984 m += n; 985 } 986 /* The residual "npages" is less than "1 << (VM_NFREEORDER - 1)". */ 987 for (; npages > 0; npages -= n) { 988 order = flsl(npages) - 1; 989 n = 1 << order; 990 vm_phys_free_pages(m, order); 991 m += n; 992 } 993 } 994 995 /* 996 * Scan physical memory between the specified addresses "low" and "high" for a 997 * run of contiguous physical pages that satisfy the specified conditions, and 998 * return the lowest page in the run. The specified "alignment" determines 999 * the alignment of the lowest physical page in the run. If the specified 1000 * "boundary" is non-zero, then the run of physical pages cannot span a 1001 * physical address that is a multiple of "boundary". 1002 * 1003 * "npages" must be greater than zero. Both "alignment" and "boundary" must 1004 * be a power of two. 1005 */ 1006 vm_page_t 1007 vm_phys_scan_contig(int domain, u_long npages, vm_paddr_t low, vm_paddr_t high, 1008 u_long alignment, vm_paddr_t boundary, int options) 1009 { 1010 vm_paddr_t pa_end; 1011 vm_page_t m_end, m_run, m_start; 1012 struct vm_phys_seg *seg; 1013 int segind; 1014 1015 KASSERT(npages > 0, ("npages is 0")); 1016 KASSERT(powerof2(alignment), ("alignment is not a power of 2")); 1017 KASSERT(powerof2(boundary), ("boundary is not a power of 2")); 1018 if (low >= high) 1019 return (NULL); 1020 for (segind = 0; segind < vm_phys_nsegs; segind++) { 1021 seg = &vm_phys_segs[segind]; 1022 if (seg->domain != domain) 1023 continue; 1024 if (seg->start >= high) 1025 break; 1026 if (low >= seg->end) 1027 continue; 1028 if (low <= seg->start) 1029 m_start = seg->first_page; 1030 else 1031 m_start = &seg->first_page[atop(low - seg->start)]; 1032 if (high < seg->end) 1033 pa_end = high; 1034 else 1035 pa_end = seg->end; 1036 if (pa_end - VM_PAGE_TO_PHYS(m_start) < ptoa(npages)) 1037 continue; 1038 m_end = &seg->first_page[atop(pa_end - seg->start)]; 1039 m_run = vm_page_scan_contig(npages, m_start, m_end, 1040 alignment, boundary, options); 1041 if (m_run != NULL) 1042 return (m_run); 1043 } 1044 return (NULL); 1045 } 1046 1047 /* 1048 * Set the pool for a contiguous, power of two-sized set of physical pages. 1049 */ 1050 void 1051 vm_phys_set_pool(int pool, vm_page_t m, int order) 1052 { 1053 vm_page_t m_tmp; 1054 1055 for (m_tmp = m; m_tmp < &m[1 << order]; m_tmp++) 1056 m_tmp->pool = pool; 1057 } 1058 1059 /* 1060 * Search for the given physical page "m" in the free lists. If the search 1061 * succeeds, remove "m" from the free lists and return TRUE. Otherwise, return 1062 * FALSE, indicating that "m" is not in the free lists. 1063 * 1064 * The free page queues must be locked. 1065 */ 1066 boolean_t 1067 vm_phys_unfree_page(vm_page_t m) 1068 { 1069 struct vm_freelist *fl; 1070 struct vm_phys_seg *seg; 1071 vm_paddr_t pa, pa_half; 1072 vm_page_t m_set, m_tmp; 1073 int order; 1074 1075 /* 1076 * First, find the contiguous, power of two-sized set of free 1077 * physical pages containing the given physical page "m" and 1078 * assign it to "m_set". 1079 */ 1080 seg = &vm_phys_segs[m->segind]; 1081 vm_domain_free_assert_locked(VM_DOMAIN(seg->domain)); 1082 for (m_set = m, order = 0; m_set->order == VM_NFREEORDER && 1083 order < VM_NFREEORDER - 1; ) { 1084 order++; 1085 pa = m->phys_addr & (~(vm_paddr_t)0 << (PAGE_SHIFT + order)); 1086 if (pa >= seg->start) 1087 m_set = &seg->first_page[atop(pa - seg->start)]; 1088 else 1089 return (FALSE); 1090 } 1091 if (m_set->order < order) 1092 return (FALSE); 1093 if (m_set->order == VM_NFREEORDER) 1094 return (FALSE); 1095 KASSERT(m_set->order < VM_NFREEORDER, 1096 ("vm_phys_unfree_page: page %p has unexpected order %d", 1097 m_set, m_set->order)); 1098 1099 /* 1100 * Next, remove "m_set" from the free lists. Finally, extract 1101 * "m" from "m_set" using an iterative algorithm: While "m_set" 1102 * is larger than a page, shrink "m_set" by returning the half 1103 * of "m_set" that does not contain "m" to the free lists. 1104 */ 1105 fl = (*seg->free_queues)[m_set->pool]; 1106 order = m_set->order; 1107 vm_freelist_rem(fl, m_set, order); 1108 while (order > 0) { 1109 order--; 1110 pa_half = m_set->phys_addr ^ (1 << (PAGE_SHIFT + order)); 1111 if (m->phys_addr < pa_half) 1112 m_tmp = &seg->first_page[atop(pa_half - seg->start)]; 1113 else { 1114 m_tmp = m_set; 1115 m_set = &seg->first_page[atop(pa_half - seg->start)]; 1116 } 1117 vm_freelist_add(fl, m_tmp, order, 0); 1118 } 1119 KASSERT(m_set == m, ("vm_phys_unfree_page: fatal inconsistency")); 1120 return (TRUE); 1121 } 1122 1123 /* 1124 * Allocate a contiguous set of physical pages of the given size 1125 * "npages" from the free lists. All of the physical pages must be at 1126 * or above the given physical address "low" and below the given 1127 * physical address "high". The given value "alignment" determines the 1128 * alignment of the first physical page in the set. If the given value 1129 * "boundary" is non-zero, then the set of physical pages cannot cross 1130 * any physical address boundary that is a multiple of that value. Both 1131 * "alignment" and "boundary" must be a power of two. 1132 */ 1133 vm_page_t 1134 vm_phys_alloc_contig(int domain, u_long npages, vm_paddr_t low, vm_paddr_t high, 1135 u_long alignment, vm_paddr_t boundary) 1136 { 1137 vm_paddr_t pa_end, pa_start; 1138 vm_page_t m_run; 1139 struct vm_phys_seg *seg; 1140 int segind; 1141 1142 KASSERT(npages > 0, ("npages is 0")); 1143 KASSERT(powerof2(alignment), ("alignment is not a power of 2")); 1144 KASSERT(powerof2(boundary), ("boundary is not a power of 2")); 1145 vm_domain_free_assert_locked(VM_DOMAIN(domain)); 1146 if (low >= high) 1147 return (NULL); 1148 m_run = NULL; 1149 for (segind = vm_phys_nsegs - 1; segind >= 0; segind--) { 1150 seg = &vm_phys_segs[segind]; 1151 if (seg->start >= high || seg->domain != domain) 1152 continue; 1153 if (low >= seg->end) 1154 break; 1155 if (low <= seg->start) 1156 pa_start = seg->start; 1157 else 1158 pa_start = low; 1159 if (high < seg->end) 1160 pa_end = high; 1161 else 1162 pa_end = seg->end; 1163 if (pa_end - pa_start < ptoa(npages)) 1164 continue; 1165 m_run = vm_phys_alloc_seg_contig(seg, npages, low, high, 1166 alignment, boundary); 1167 if (m_run != NULL) 1168 break; 1169 } 1170 return (m_run); 1171 } 1172 1173 /* 1174 * Allocate a run of contiguous physical pages from the free list for the 1175 * specified segment. 1176 */ 1177 static vm_page_t 1178 vm_phys_alloc_seg_contig(struct vm_phys_seg *seg, u_long npages, 1179 vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary) 1180 { 1181 struct vm_freelist *fl; 1182 vm_paddr_t pa, pa_end, size; 1183 vm_page_t m, m_ret; 1184 u_long npages_end; 1185 int oind, order, pind; 1186 1187 KASSERT(npages > 0, ("npages is 0")); 1188 KASSERT(powerof2(alignment), ("alignment is not a power of 2")); 1189 KASSERT(powerof2(boundary), ("boundary is not a power of 2")); 1190 vm_domain_free_assert_locked(VM_DOMAIN(seg->domain)); 1191 /* Compute the queue that is the best fit for npages. */ 1192 for (order = 0; (1 << order) < npages; order++); 1193 /* Search for a run satisfying the specified conditions. */ 1194 size = npages << PAGE_SHIFT; 1195 for (oind = min(order, VM_NFREEORDER - 1); oind < VM_NFREEORDER; 1196 oind++) { 1197 for (pind = 0; pind < VM_NFREEPOOL; pind++) { 1198 fl = (*seg->free_queues)[pind]; 1199 TAILQ_FOREACH(m_ret, &fl[oind].pl, plinks.q) { 1200 /* 1201 * Is the size of this allocation request 1202 * larger than the largest block size? 1203 */ 1204 if (order >= VM_NFREEORDER) { 1205 /* 1206 * Determine if a sufficient number of 1207 * subsequent blocks to satisfy the 1208 * allocation request are free. 1209 */ 1210 pa = VM_PAGE_TO_PHYS(m_ret); 1211 pa_end = pa + size; 1212 if (pa_end < pa) 1213 continue; 1214 for (;;) { 1215 pa += 1 << (PAGE_SHIFT + 1216 VM_NFREEORDER - 1); 1217 if (pa >= pa_end || 1218 pa < seg->start || 1219 pa >= seg->end) 1220 break; 1221 m = &seg->first_page[atop(pa - 1222 seg->start)]; 1223 if (m->order != VM_NFREEORDER - 1224 1) 1225 break; 1226 } 1227 /* If not, go to the next block. */ 1228 if (pa < pa_end) 1229 continue; 1230 } 1231 1232 /* 1233 * Determine if the blocks are within the 1234 * given range, satisfy the given alignment, 1235 * and do not cross the given boundary. 1236 */ 1237 pa = VM_PAGE_TO_PHYS(m_ret); 1238 pa_end = pa + size; 1239 if (pa >= low && pa_end <= high && 1240 (pa & (alignment - 1)) == 0 && 1241 rounddown2(pa ^ (pa_end - 1), boundary) == 0) 1242 goto done; 1243 } 1244 } 1245 } 1246 return (NULL); 1247 done: 1248 for (m = m_ret; m < &m_ret[npages]; m = &m[1 << oind]) { 1249 fl = (*seg->free_queues)[m->pool]; 1250 vm_freelist_rem(fl, m, m->order); 1251 } 1252 if (m_ret->pool != VM_FREEPOOL_DEFAULT) 1253 vm_phys_set_pool(VM_FREEPOOL_DEFAULT, m_ret, oind); 1254 fl = (*seg->free_queues)[m_ret->pool]; 1255 vm_phys_split_pages(m_ret, oind, fl, order); 1256 /* Return excess pages to the free lists. */ 1257 npages_end = roundup2(npages, 1 << imin(oind, order)); 1258 if (npages < npages_end) 1259 vm_phys_free_contig(&m_ret[npages], npages_end - npages); 1260 return (m_ret); 1261 } 1262 1263 #ifdef DDB 1264 /* 1265 * Show the number of physical pages in each of the free lists. 1266 */ 1267 DB_SHOW_COMMAND(freepages, db_show_freepages) 1268 { 1269 struct vm_freelist *fl; 1270 int flind, oind, pind, dom; 1271 1272 for (dom = 0; dom < vm_ndomains; dom++) { 1273 db_printf("DOMAIN: %d\n", dom); 1274 for (flind = 0; flind < vm_nfreelists; flind++) { 1275 db_printf("FREE LIST %d:\n" 1276 "\n ORDER (SIZE) | NUMBER" 1277 "\n ", flind); 1278 for (pind = 0; pind < VM_NFREEPOOL; pind++) 1279 db_printf(" | POOL %d", pind); 1280 db_printf("\n-- "); 1281 for (pind = 0; pind < VM_NFREEPOOL; pind++) 1282 db_printf("-- -- "); 1283 db_printf("--\n"); 1284 for (oind = VM_NFREEORDER - 1; oind >= 0; oind--) { 1285 db_printf(" %2.2d (%6.6dK)", oind, 1286 1 << (PAGE_SHIFT - 10 + oind)); 1287 for (pind = 0; pind < VM_NFREEPOOL; pind++) { 1288 fl = vm_phys_free_queues[dom][flind][pind]; 1289 db_printf(" | %6.6d", fl[oind].lcnt); 1290 } 1291 db_printf("\n"); 1292 } 1293 db_printf("\n"); 1294 } 1295 db_printf("\n"); 1296 } 1297 } 1298 #endif 1299