xref: /freebsd/sys/vm/vm_phys.c (revision 282a3889ebf826db9839be296ff1dd903f6d6d6e)
1 /*-
2  * Copyright (c) 2002-2006 Rice University
3  * Copyright (c) 2007 Alan L. Cox <alc@cs.rice.edu>
4  * All rights reserved.
5  *
6  * This software was developed for the FreeBSD Project by Alan L. Cox,
7  * Olivier Crameri, Peter Druschel, Sitaram Iyer, and Juan Navarro.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21  * A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT
22  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
23  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
24  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
25  * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY
28  * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29  * POSSIBILITY OF SUCH DAMAGE.
30  */
31 
32 #include <sys/cdefs.h>
33 __FBSDID("$FreeBSD$");
34 
35 #include "opt_ddb.h"
36 
37 #include <sys/param.h>
38 #include <sys/systm.h>
39 #include <sys/lock.h>
40 #include <sys/kernel.h>
41 #include <sys/malloc.h>
42 #include <sys/mutex.h>
43 #include <sys/queue.h>
44 #include <sys/sbuf.h>
45 #include <sys/sysctl.h>
46 #include <sys/vmmeter.h>
47 
48 #include <ddb/ddb.h>
49 
50 #include <vm/vm.h>
51 #include <vm/vm_param.h>
52 #include <vm/vm_kern.h>
53 #include <vm/vm_object.h>
54 #include <vm/vm_page.h>
55 #include <vm/vm_phys.h>
56 
57 struct vm_freelist {
58 	struct pglist pl;
59 	int lcnt;
60 };
61 
62 struct vm_phys_seg {
63 	vm_paddr_t	start;
64 	vm_paddr_t	end;
65 	vm_page_t	first_page;
66 	struct vm_freelist (*free_queues)[VM_NFREEPOOL][VM_NFREEORDER];
67 };
68 
69 static struct vm_phys_seg vm_phys_segs[VM_PHYSSEG_MAX];
70 
71 static int vm_phys_nsegs;
72 
73 static struct vm_freelist
74     vm_phys_free_queues[VM_NFREELIST][VM_NFREEPOOL][VM_NFREEORDER];
75 
76 static int vm_nfreelists = VM_FREELIST_DEFAULT + 1;
77 
78 static int cnt_prezero;
79 SYSCTL_INT(_vm_stats_misc, OID_AUTO, cnt_prezero, CTLFLAG_RD,
80     &cnt_prezero, 0, "The number of physical pages prezeroed at idle time");
81 
82 static int sysctl_vm_phys_free(SYSCTL_HANDLER_ARGS);
83 SYSCTL_OID(_vm, OID_AUTO, phys_free, CTLTYPE_STRING | CTLFLAG_RD,
84     NULL, 0, sysctl_vm_phys_free, "A", "Phys Free Info");
85 
86 static int sysctl_vm_phys_segs(SYSCTL_HANDLER_ARGS);
87 SYSCTL_OID(_vm, OID_AUTO, phys_segs, CTLTYPE_STRING | CTLFLAG_RD,
88     NULL, 0, sysctl_vm_phys_segs, "A", "Phys Seg Info");
89 
90 static void vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int flind);
91 static int vm_phys_paddr_to_segind(vm_paddr_t pa);
92 static void vm_phys_set_pool(int pool, vm_page_t m, int order);
93 static void vm_phys_split_pages(vm_page_t m, int oind, struct vm_freelist *fl,
94     int order);
95 
96 /*
97  * Outputs the state of the physical memory allocator, specifically,
98  * the amount of physical memory in each free list.
99  */
100 static int
101 sysctl_vm_phys_free(SYSCTL_HANDLER_ARGS)
102 {
103 	struct sbuf sbuf;
104 	struct vm_freelist *fl;
105 	char *cbuf;
106 	const int cbufsize = vm_nfreelists*(VM_NFREEORDER + 1)*81;
107 	int error, flind, oind, pind;
108 
109 	cbuf = malloc(cbufsize, M_TEMP, M_WAITOK | M_ZERO);
110 	sbuf_new(&sbuf, cbuf, cbufsize, SBUF_FIXEDLEN);
111 	for (flind = 0; flind < vm_nfreelists; flind++) {
112 		sbuf_printf(&sbuf, "\nFREE LIST %d:\n"
113 		    "\n  ORDER (SIZE)  |  NUMBER"
114 		    "\n              ", flind);
115 		for (pind = 0; pind < VM_NFREEPOOL; pind++)
116 			sbuf_printf(&sbuf, "  |  POOL %d", pind);
117 		sbuf_printf(&sbuf, "\n--            ");
118 		for (pind = 0; pind < VM_NFREEPOOL; pind++)
119 			sbuf_printf(&sbuf, "-- --      ");
120 		sbuf_printf(&sbuf, "--\n");
121 		for (oind = VM_NFREEORDER - 1; oind >= 0; oind--) {
122 			sbuf_printf(&sbuf, "  %2.2d (%6.6dK)", oind,
123 			    1 << (PAGE_SHIFT - 10 + oind));
124 			for (pind = 0; pind < VM_NFREEPOOL; pind++) {
125 				fl = vm_phys_free_queues[flind][pind];
126 				sbuf_printf(&sbuf, "  |  %6.6d", fl[oind].lcnt);
127 			}
128 			sbuf_printf(&sbuf, "\n");
129 		}
130 	}
131 	sbuf_finish(&sbuf);
132 	error = SYSCTL_OUT(req, sbuf_data(&sbuf), sbuf_len(&sbuf));
133 	sbuf_delete(&sbuf);
134 	free(cbuf, M_TEMP);
135 	return (error);
136 }
137 
138 /*
139  * Outputs the set of physical memory segments.
140  */
141 static int
142 sysctl_vm_phys_segs(SYSCTL_HANDLER_ARGS)
143 {
144 	struct sbuf sbuf;
145 	struct vm_phys_seg *seg;
146 	char *cbuf;
147 	const int cbufsize = VM_PHYSSEG_MAX*(VM_NFREEORDER + 1)*81;
148 	int error, segind;
149 
150 	cbuf = malloc(cbufsize, M_TEMP, M_WAITOK | M_ZERO);
151 	sbuf_new(&sbuf, cbuf, cbufsize, SBUF_FIXEDLEN);
152 	for (segind = 0; segind < vm_phys_nsegs; segind++) {
153 		sbuf_printf(&sbuf, "\nSEGMENT %d:\n\n", segind);
154 		seg = &vm_phys_segs[segind];
155 		sbuf_printf(&sbuf, "start:     %#jx\n",
156 		    (uintmax_t)seg->start);
157 		sbuf_printf(&sbuf, "end:       %#jx\n",
158 		    (uintmax_t)seg->end);
159 		sbuf_printf(&sbuf, "free list: %p\n", seg->free_queues);
160 	}
161 	sbuf_finish(&sbuf);
162 	error = SYSCTL_OUT(req, sbuf_data(&sbuf), sbuf_len(&sbuf));
163 	sbuf_delete(&sbuf);
164 	free(cbuf, M_TEMP);
165 	return (error);
166 }
167 
168 /*
169  * Create a physical memory segment.
170  */
171 static void
172 vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int flind)
173 {
174 	struct vm_phys_seg *seg;
175 #ifdef VM_PHYSSEG_SPARSE
176 	long pages;
177 	int segind;
178 
179 	pages = 0;
180 	for (segind = 0; segind < vm_phys_nsegs; segind++) {
181 		seg = &vm_phys_segs[segind];
182 		pages += atop(seg->end - seg->start);
183 	}
184 #endif
185 	KASSERT(vm_phys_nsegs < VM_PHYSSEG_MAX,
186 	    ("vm_phys_create_seg: increase VM_PHYSSEG_MAX"));
187 	seg = &vm_phys_segs[vm_phys_nsegs++];
188 	seg->start = start;
189 	seg->end = end;
190 #ifdef VM_PHYSSEG_SPARSE
191 	seg->first_page = &vm_page_array[pages];
192 #else
193 	seg->first_page = PHYS_TO_VM_PAGE(start);
194 #endif
195 	seg->free_queues = &vm_phys_free_queues[flind];
196 }
197 
198 /*
199  * Initialize the physical memory allocator.
200  */
201 void
202 vm_phys_init(void)
203 {
204 	struct vm_freelist *fl;
205 	int flind, i, oind, pind;
206 
207 	for (i = 0; phys_avail[i + 1] != 0; i += 2) {
208 #ifdef	VM_FREELIST_ISADMA
209 		if (phys_avail[i] < 16777216) {
210 			if (phys_avail[i + 1] > 16777216) {
211 				vm_phys_create_seg(phys_avail[i], 16777216,
212 				    VM_FREELIST_ISADMA);
213 				vm_phys_create_seg(16777216, phys_avail[i + 1],
214 				    VM_FREELIST_DEFAULT);
215 			} else {
216 				vm_phys_create_seg(phys_avail[i],
217 				    phys_avail[i + 1], VM_FREELIST_ISADMA);
218 			}
219 			if (VM_FREELIST_ISADMA >= vm_nfreelists)
220 				vm_nfreelists = VM_FREELIST_ISADMA + 1;
221 		} else
222 #endif
223 #ifdef	VM_FREELIST_HIGHMEM
224 		if (phys_avail[i + 1] > VM_HIGHMEM_ADDRESS) {
225 			if (phys_avail[i] < VM_HIGHMEM_ADDRESS) {
226 				vm_phys_create_seg(phys_avail[i],
227 				    VM_HIGHMEM_ADDRESS, VM_FREELIST_DEFAULT);
228 				vm_phys_create_seg(VM_HIGHMEM_ADDRESS,
229 				    phys_avail[i + 1], VM_FREELIST_HIGHMEM);
230 			} else {
231 				vm_phys_create_seg(phys_avail[i],
232 				    phys_avail[i + 1], VM_FREELIST_HIGHMEM);
233 			}
234 			if (VM_FREELIST_HIGHMEM >= vm_nfreelists)
235 				vm_nfreelists = VM_FREELIST_HIGHMEM + 1;
236 		} else
237 #endif
238 		vm_phys_create_seg(phys_avail[i], phys_avail[i + 1],
239 		    VM_FREELIST_DEFAULT);
240 	}
241 	for (flind = 0; flind < vm_nfreelists; flind++) {
242 		for (pind = 0; pind < VM_NFREEPOOL; pind++) {
243 			fl = vm_phys_free_queues[flind][pind];
244 			for (oind = 0; oind < VM_NFREEORDER; oind++)
245 				TAILQ_INIT(&fl[oind].pl);
246 		}
247 	}
248 }
249 
250 /*
251  * Split a contiguous, power of two-sized set of physical pages.
252  */
253 static __inline void
254 vm_phys_split_pages(vm_page_t m, int oind, struct vm_freelist *fl, int order)
255 {
256 	vm_page_t m_buddy;
257 
258 	while (oind > order) {
259 		oind--;
260 		m_buddy = &m[1 << oind];
261 		KASSERT(m_buddy->order == VM_NFREEORDER,
262 		    ("vm_phys_split_pages: page %p has unexpected order %d",
263 		    m_buddy, m_buddy->order));
264 		m_buddy->order = oind;
265 		TAILQ_INSERT_HEAD(&fl[oind].pl, m_buddy, pageq);
266 		fl[oind].lcnt++;
267         }
268 }
269 
270 /*
271  * Initialize a physical page and add it to the free lists.
272  */
273 void
274 vm_phys_add_page(vm_paddr_t pa)
275 {
276 	vm_page_t m;
277 
278 	cnt.v_page_count++;
279 	m = vm_phys_paddr_to_vm_page(pa);
280 	m->phys_addr = pa;
281 	m->segind = vm_phys_paddr_to_segind(pa);
282 	m->flags = PG_FREE;
283 	KASSERT(m->order == VM_NFREEORDER,
284 	    ("vm_phys_add_page: page %p has unexpected order %d",
285 	    m, m->order));
286 	m->pool = VM_FREEPOOL_DEFAULT;
287 	pmap_page_init(m);
288 	mtx_lock(&vm_page_queue_free_mtx);
289 	vm_phys_free_pages(m, 0);
290 	mtx_unlock(&vm_page_queue_free_mtx);
291 }
292 
293 /*
294  * Allocate a contiguous, power of two-sized set of physical pages
295  * from the free lists.
296  *
297  * The free page queues must be locked.
298  */
299 vm_page_t
300 vm_phys_alloc_pages(int pool, int order)
301 {
302 	struct vm_freelist *fl;
303 	struct vm_freelist *alt;
304 	int flind, oind, pind;
305 	vm_page_t m;
306 
307 	KASSERT(pool < VM_NFREEPOOL,
308 	    ("vm_phys_alloc_pages: pool %d is out of range", pool));
309 	KASSERT(order < VM_NFREEORDER,
310 	    ("vm_phys_alloc_pages: order %d is out of range", order));
311 	mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
312 	for (flind = 0; flind < vm_nfreelists; flind++) {
313 		fl = vm_phys_free_queues[flind][pool];
314 		for (oind = order; oind < VM_NFREEORDER; oind++) {
315 			m = TAILQ_FIRST(&fl[oind].pl);
316 			if (m != NULL) {
317 				TAILQ_REMOVE(&fl[oind].pl, m, pageq);
318 				fl[oind].lcnt--;
319 				m->order = VM_NFREEORDER;
320 				vm_phys_split_pages(m, oind, fl, order);
321 				cnt.v_free_count -= 1 << order;
322 				return (m);
323 			}
324 		}
325 
326 		/*
327 		 * The given pool was empty.  Find the largest
328 		 * contiguous, power-of-two-sized set of pages in any
329 		 * pool.  Transfer these pages to the given pool, and
330 		 * use them to satisfy the allocation.
331 		 */
332 		for (oind = VM_NFREEORDER - 1; oind >= order; oind--) {
333 			for (pind = 0; pind < VM_NFREEPOOL; pind++) {
334 				alt = vm_phys_free_queues[flind][pind];
335 				m = TAILQ_FIRST(&alt[oind].pl);
336 				if (m != NULL) {
337 					TAILQ_REMOVE(&alt[oind].pl, m, pageq);
338 					alt[oind].lcnt--;
339 					m->order = VM_NFREEORDER;
340 					vm_phys_set_pool(pool, m, oind);
341 					vm_phys_split_pages(m, oind, fl, order);
342 					cnt.v_free_count -= 1 << order;
343 					return (m);
344 				}
345 			}
346 		}
347 	}
348 	return (NULL);
349 }
350 
351 /*
352  * Allocate physical memory from phys_avail[].
353  */
354 vm_paddr_t
355 vm_phys_bootstrap_alloc(vm_size_t size, unsigned long alignment)
356 {
357 	vm_paddr_t pa;
358 	int i;
359 
360 	size = round_page(size);
361 	for (i = 0; phys_avail[i + 1] != 0; i += 2) {
362 		if (phys_avail[i + 1] - phys_avail[i] < size)
363 			continue;
364 		pa = phys_avail[i];
365 		phys_avail[i] += size;
366 		return (pa);
367 	}
368 	panic("vm_phys_bootstrap_alloc");
369 }
370 
371 /*
372  * Find the vm_page corresponding to the given physical address.
373  */
374 vm_page_t
375 vm_phys_paddr_to_vm_page(vm_paddr_t pa)
376 {
377 	struct vm_phys_seg *seg;
378 	int segind;
379 
380 	for (segind = 0; segind < vm_phys_nsegs; segind++) {
381 		seg = &vm_phys_segs[segind];
382 		if (pa >= seg->start && pa < seg->end)
383 			return (&seg->first_page[atop(pa - seg->start)]);
384 	}
385 	panic("vm_phys_paddr_to_vm_page: paddr %#jx is not in any segment",
386 	    (uintmax_t)pa);
387 }
388 
389 /*
390  * Find the segment containing the given physical address.
391  */
392 static int
393 vm_phys_paddr_to_segind(vm_paddr_t pa)
394 {
395 	struct vm_phys_seg *seg;
396 	int segind;
397 
398 	for (segind = 0; segind < vm_phys_nsegs; segind++) {
399 		seg = &vm_phys_segs[segind];
400 		if (pa >= seg->start && pa < seg->end)
401 			return (segind);
402 	}
403 	panic("vm_phys_paddr_to_segind: paddr %#jx is not in any segment" ,
404 	    (uintmax_t)pa);
405 }
406 
407 /*
408  * Free a contiguous, power of two-sized set of physical pages.
409  *
410  * The free page queues must be locked.
411  */
412 void
413 vm_phys_free_pages(vm_page_t m, int order)
414 {
415 	struct vm_freelist *fl;
416 	struct vm_phys_seg *seg;
417 	vm_paddr_t pa, pa_buddy;
418 	vm_page_t m_buddy;
419 
420 	KASSERT(m->order == VM_NFREEORDER,
421 	    ("vm_phys_free_pages: page %p has unexpected order %d",
422 	    m, m->order));
423 	KASSERT(m->pool < VM_NFREEPOOL,
424 	    ("vm_phys_free_pages: page %p has unexpected pool %d",
425 	    m, m->pool));
426 	KASSERT(order < VM_NFREEORDER,
427 	    ("vm_phys_free_pages: order %d is out of range", order));
428 	mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
429 	pa = VM_PAGE_TO_PHYS(m);
430 	seg = &vm_phys_segs[m->segind];
431 	cnt.v_free_count += 1 << order;
432 	while (order < VM_NFREEORDER - 1) {
433 		pa_buddy = pa ^ (1 << (PAGE_SHIFT + order));
434 		if (pa_buddy < seg->start ||
435 		    pa_buddy >= seg->end)
436 			break;
437 		m_buddy = &seg->first_page[atop(pa_buddy - seg->start)];
438 		if (m_buddy->order != order)
439 			break;
440 		fl = (*seg->free_queues)[m_buddy->pool];
441 		TAILQ_REMOVE(&fl[m_buddy->order].pl, m_buddy, pageq);
442 		fl[m_buddy->order].lcnt--;
443 		m_buddy->order = VM_NFREEORDER;
444 		if (m_buddy->pool != m->pool)
445 			vm_phys_set_pool(m->pool, m_buddy, order);
446 		order++;
447 		pa &= ~((1 << (PAGE_SHIFT + order)) - 1);
448 		m = &seg->first_page[atop(pa - seg->start)];
449 	}
450 	m->order = order;
451 	fl = (*seg->free_queues)[m->pool];
452 	TAILQ_INSERT_TAIL(&fl[order].pl, m, pageq);
453 	fl[order].lcnt++;
454 }
455 
456 /*
457  * Set the pool for a contiguous, power of two-sized set of physical pages.
458  */
459 static void
460 vm_phys_set_pool(int pool, vm_page_t m, int order)
461 {
462 	vm_page_t m_tmp;
463 
464 	for (m_tmp = m; m_tmp < &m[1 << order]; m_tmp++)
465 		m_tmp->pool = pool;
466 }
467 
468 /*
469  * Try to zero one or more physical pages.  Used by an idle priority thread.
470  */
471 boolean_t
472 vm_phys_zero_pages_idle(void)
473 {
474 	struct vm_freelist *fl;
475 	vm_page_t m, m_tmp;
476 	int flind, pind, q, zeroed;
477 
478 	mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
479 	for (flind = 0; flind < vm_nfreelists; flind++) {
480 		pind = VM_FREEPOOL_DEFAULT;
481 		fl = vm_phys_free_queues[flind][pind];
482 		for (q = 0; q < VM_NFREEORDER; q++) {
483 			m = TAILQ_FIRST(&fl[q].pl);
484 			if (m != NULL && (m->flags & PG_ZERO) == 0) {
485 				TAILQ_REMOVE(&fl[q].pl, m, pageq);
486 				fl[q].lcnt--;
487 				m->order = VM_NFREEORDER;
488 				cnt.v_free_count -= 1 << q;
489 				mtx_unlock(&vm_page_queue_free_mtx);
490 				zeroed = 0;
491 				for (m_tmp = m; m_tmp < &m[1 << q]; m_tmp++) {
492 					if ((m_tmp->flags & PG_ZERO) == 0) {
493 						pmap_zero_page_idle(m_tmp);
494 						m_tmp->flags |= PG_ZERO;
495 						zeroed++;
496 					}
497 				}
498 				cnt_prezero += zeroed;
499 				mtx_lock(&vm_page_queue_free_mtx);
500 				vm_phys_free_pages(m, q);
501 				vm_page_zero_count += zeroed;
502 				return (TRUE);
503 			}
504 		}
505 	}
506 	return (FALSE);
507 }
508 
509 /*
510  * Allocate a contiguous set of physical pages of the given size
511  * "npages" from the free lists.  All of the physical pages must be at
512  * or above the given physical address "low" and below the given
513  * physical address "high".  The given value "alignment" determines the
514  * alignment of the first physical page in the set.  If the given value
515  * "boundary" is non-zero, then the set of physical pages cannot cross
516  * any physical address boundary that is a multiple of that value.  Both
517  * "alignment" and "boundary" must be a power of two.
518  */
519 vm_page_t
520 vm_phys_alloc_contig(unsigned long npages, vm_paddr_t low, vm_paddr_t high,
521     unsigned long alignment, unsigned long boundary)
522 {
523 	struct vm_freelist *fl;
524 	struct vm_phys_seg *seg;
525 	vm_paddr_t pa, pa_last, size;
526 	vm_page_t m, m_ret;
527 	int flind, i, oind, order, pind;
528 
529 	size = npages << PAGE_SHIFT;
530 	KASSERT(size != 0,
531 	    ("vm_phys_alloc_contig: size must not be 0"));
532 	KASSERT((alignment & (alignment - 1)) == 0,
533 	    ("vm_phys_alloc_contig: alignment must be a power of 2"));
534 	KASSERT((boundary & (boundary - 1)) == 0,
535 	    ("vm_phys_alloc_contig: boundary must be a power of 2"));
536 	/* Compute the queue that is the best fit for npages. */
537 	for (order = 0; (1 << order) < npages; order++);
538 	mtx_lock(&vm_page_queue_free_mtx);
539 	for (flind = 0; flind < vm_nfreelists; flind++) {
540 		for (oind = min(order, VM_NFREEORDER - 1); oind < VM_NFREEORDER; oind++) {
541 			for (pind = 0; pind < VM_NFREEPOOL; pind++) {
542 				fl = vm_phys_free_queues[flind][pind];
543 				TAILQ_FOREACH(m_ret, &fl[oind].pl, pageq) {
544 					/*
545 					 * A free list may contain physical pages
546 					 * from one or more segments.
547 					 */
548 					seg = &vm_phys_segs[m_ret->segind];
549 					if (seg->start > high ||
550 					    low >= seg->end)
551 						continue;
552 
553 					/*
554 					 * Is the size of this allocation request
555 					 * larger than the largest block size?
556 					 */
557 					if (order >= VM_NFREEORDER) {
558 						/*
559 						 * Determine if a sufficient number
560 						 * of subsequent blocks to satisfy
561 						 * the allocation request are free.
562 						 */
563 						pa = VM_PAGE_TO_PHYS(m_ret);
564 						pa_last = pa + size;
565 						for (;;) {
566 							pa += 1 << (PAGE_SHIFT + VM_NFREEORDER - 1);
567 							if (pa >= pa_last)
568 								break;
569 							if (pa < seg->start ||
570 							    pa >= seg->end)
571 								break;
572 							m = &seg->first_page[atop(pa - seg->start)];
573 							if (m->order != VM_NFREEORDER - 1)
574 								break;
575 						}
576 						/* If not, continue to the next block. */
577 						if (pa < pa_last)
578 							continue;
579 					}
580 
581 					/*
582 					 * Determine if the blocks are within the given range,
583 					 * satisfy the given alignment, and do not cross the
584 					 * given boundary.
585 					 */
586 					pa = VM_PAGE_TO_PHYS(m_ret);
587 					if (pa >= low &&
588 					    pa + size <= high &&
589 					    (pa & (alignment - 1)) == 0 &&
590 					    ((pa ^ (pa + size - 1)) & ~(boundary - 1)) == 0)
591 						goto done;
592 				}
593 			}
594 		}
595 	}
596 	mtx_unlock(&vm_page_queue_free_mtx);
597 	return (NULL);
598 done:
599 	for (m = m_ret; m < &m_ret[npages]; m = &m[1 << oind]) {
600 		fl = (*seg->free_queues)[m->pool];
601 		TAILQ_REMOVE(&fl[m->order].pl, m, pageq);
602 		fl[m->order].lcnt--;
603 		m->order = VM_NFREEORDER;
604 	}
605 	if (m_ret->pool != VM_FREEPOOL_DEFAULT)
606 		vm_phys_set_pool(VM_FREEPOOL_DEFAULT, m_ret, oind);
607 	fl = (*seg->free_queues)[m_ret->pool];
608 	vm_phys_split_pages(m_ret, oind, fl, order);
609 	cnt.v_free_count -= roundup2(npages, 1 << imin(oind, order));
610 	for (i = 0; i < npages; i++) {
611 		m = &m_ret[i];
612 		KASSERT(m->queue == PQ_NONE,
613 		    ("vm_phys_alloc_contig: page %p has unexpected queue %d",
614 		    m, m->queue));
615 		m->valid = VM_PAGE_BITS_ALL;
616 		if (m->flags & PG_ZERO)
617 			vm_page_zero_count--;
618 		/* Don't clear the PG_ZERO flag; we'll need it later. */
619 		m->flags = PG_UNMANAGED | (m->flags & PG_ZERO);
620 		m->oflags = 0;
621 		KASSERT(m->dirty == 0,
622 		    ("vm_phys_alloc_contig: page %p was dirty", m));
623 		m->wire_count = 0;
624 		m->busy = 0;
625 	}
626 	for (; i < roundup2(npages, 1 << imin(oind, order)); i++) {
627 		m = &m_ret[i];
628 		KASSERT(m->order == VM_NFREEORDER,
629 		    ("vm_phys_alloc_contig: page %p has unexpected order %d",
630 		    m, m->order));
631 		vm_phys_free_pages(m, 0);
632 	}
633 	mtx_unlock(&vm_page_queue_free_mtx);
634 	return (m_ret);
635 }
636 
637 #ifdef DDB
638 /*
639  * Show the number of physical pages in each of the free lists.
640  */
641 DB_SHOW_COMMAND(freepages, db_show_freepages)
642 {
643 	struct vm_freelist *fl;
644 	int flind, oind, pind;
645 
646 	for (flind = 0; flind < vm_nfreelists; flind++) {
647 		db_printf("FREE LIST %d:\n"
648 		    "\n  ORDER (SIZE)  |  NUMBER"
649 		    "\n              ", flind);
650 		for (pind = 0; pind < VM_NFREEPOOL; pind++)
651 			db_printf("  |  POOL %d", pind);
652 		db_printf("\n--            ");
653 		for (pind = 0; pind < VM_NFREEPOOL; pind++)
654 			db_printf("-- --      ");
655 		db_printf("--\n");
656 		for (oind = VM_NFREEORDER - 1; oind >= 0; oind--) {
657 			db_printf("  %2.2d (%6.6dK)", oind,
658 			    1 << (PAGE_SHIFT - 10 + oind));
659 			for (pind = 0; pind < VM_NFREEPOOL; pind++) {
660 				fl = vm_phys_free_queues[flind][pind];
661 				db_printf("  |  %6.6d", fl[oind].lcnt);
662 			}
663 			db_printf("\n");
664 		}
665 		db_printf("\n");
666 	}
667 }
668 #endif
669