1f8a47341SAlan Cox /*- 2fe267a55SPedro F. Giffuni * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3fe267a55SPedro F. Giffuni * 4f8a47341SAlan Cox * Copyright (c) 2002-2006 Rice University 5ec179322SAlan Cox * Copyright (c) 2007-2011 Alan L. Cox <alc@cs.rice.edu> 6f8a47341SAlan Cox * All rights reserved. 7f8a47341SAlan Cox * 8f8a47341SAlan Cox * This software was developed for the FreeBSD Project by Alan L. Cox, 9f8a47341SAlan Cox * Olivier Crameri, Peter Druschel, Sitaram Iyer, and Juan Navarro. 10f8a47341SAlan Cox * 11f8a47341SAlan Cox * Redistribution and use in source and binary forms, with or without 12f8a47341SAlan Cox * modification, are permitted provided that the following conditions 13f8a47341SAlan Cox * are met: 14f8a47341SAlan Cox * 1. Redistributions of source code must retain the above copyright 15f8a47341SAlan Cox * notice, this list of conditions and the following disclaimer. 16f8a47341SAlan Cox * 2. Redistributions in binary form must reproduce the above copyright 17f8a47341SAlan Cox * notice, this list of conditions and the following disclaimer in the 18f8a47341SAlan Cox * documentation and/or other materials provided with the distribution. 19f8a47341SAlan Cox * 20f8a47341SAlan Cox * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21f8a47341SAlan Cox * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22f8a47341SAlan Cox * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23f8a47341SAlan Cox * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24f8a47341SAlan Cox * HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 25f8a47341SAlan Cox * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 26f8a47341SAlan Cox * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 27f8a47341SAlan Cox * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 28f8a47341SAlan Cox * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29f8a47341SAlan Cox * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY 30f8a47341SAlan Cox * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31f8a47341SAlan Cox * POSSIBILITY OF SUCH DAMAGE. 32f8a47341SAlan Cox */ 33f8a47341SAlan Cox 34f8a47341SAlan Cox /* 35f8a47341SAlan Cox * Superpage reservation management module 36c68c3537SAlan Cox * 37c68c3537SAlan Cox * Any external functions defined by this module are only to be used by the 38c68c3537SAlan Cox * virtual memory system. 39f8a47341SAlan Cox */ 40f8a47341SAlan Cox 41f8a47341SAlan Cox #include <sys/cdefs.h> 42f8a47341SAlan Cox __FBSDID("$FreeBSD$"); 43f8a47341SAlan Cox 44f8a47341SAlan Cox #include "opt_vm.h" 45f8a47341SAlan Cox 46f8a47341SAlan Cox #include <sys/param.h> 47f8a47341SAlan Cox #include <sys/kernel.h> 48f8a47341SAlan Cox #include <sys/lock.h> 49f8a47341SAlan Cox #include <sys/malloc.h> 50f8a47341SAlan Cox #include <sys/mutex.h> 51f8a47341SAlan Cox #include <sys/queue.h> 5289f6b863SAttilio Rao #include <sys/rwlock.h> 53f8a47341SAlan Cox #include <sys/sbuf.h> 54f8a47341SAlan Cox #include <sys/sysctl.h> 55f8a47341SAlan Cox #include <sys/systm.h> 569ed01c32SGleb Smirnoff #include <sys/vmmeter.h> 57f8a47341SAlan Cox 58f8a47341SAlan Cox #include <vm/vm.h> 59f8a47341SAlan Cox #include <vm/vm_param.h> 60f8a47341SAlan Cox #include <vm/vm_object.h> 61f8a47341SAlan Cox #include <vm/vm_page.h> 62e2068d0bSJeff Roberson #include <vm/vm_pageout.h> 63f8a47341SAlan Cox #include <vm/vm_phys.h> 64e2068d0bSJeff Roberson #include <vm/vm_pagequeue.h> 65774d251dSAttilio Rao #include <vm/vm_radix.h> 66f8a47341SAlan Cox #include <vm/vm_reserv.h> 67f8a47341SAlan Cox 68f8a47341SAlan Cox /* 69f8a47341SAlan Cox * The reservation system supports the speculative allocation of large physical 703453bca8SAlan Cox * pages ("superpages"). Speculative allocation enables the fully automatic 71f8a47341SAlan Cox * utilization of superpages by the virtual memory system. In other words, no 72f8a47341SAlan Cox * programmatic directives are required to use superpages. 73f8a47341SAlan Cox */ 74f8a47341SAlan Cox 75f8a47341SAlan Cox #if VM_NRESERVLEVEL > 0 76f8a47341SAlan Cox 77f8a47341SAlan Cox /* 78f8a47341SAlan Cox * The number of small pages that are contained in a level 0 reservation 79f8a47341SAlan Cox */ 80f8a47341SAlan Cox #define VM_LEVEL_0_NPAGES (1 << VM_LEVEL_0_ORDER) 81f8a47341SAlan Cox 82f8a47341SAlan Cox /* 83f8a47341SAlan Cox * The number of bits by which a physical address is shifted to obtain the 84f8a47341SAlan Cox * reservation number 85f8a47341SAlan Cox */ 86f8a47341SAlan Cox #define VM_LEVEL_0_SHIFT (VM_LEVEL_0_ORDER + PAGE_SHIFT) 87f8a47341SAlan Cox 88f8a47341SAlan Cox /* 89f8a47341SAlan Cox * The size of a level 0 reservation in bytes 90f8a47341SAlan Cox */ 91f8a47341SAlan Cox #define VM_LEVEL_0_SIZE (1 << VM_LEVEL_0_SHIFT) 92f8a47341SAlan Cox 93f8a47341SAlan Cox /* 94f8a47341SAlan Cox * Computes the index of the small page underlying the given (object, pindex) 95f8a47341SAlan Cox * within the reservation's array of small pages. 96f8a47341SAlan Cox */ 97f8a47341SAlan Cox #define VM_RESERV_INDEX(object, pindex) \ 98f8a47341SAlan Cox (((object)->pg_color + (pindex)) & (VM_LEVEL_0_NPAGES - 1)) 99f8a47341SAlan Cox 100f8a47341SAlan Cox /* 101ec179322SAlan Cox * The size of a population map entry 102ec179322SAlan Cox */ 103ec179322SAlan Cox typedef u_long popmap_t; 104ec179322SAlan Cox 105ec179322SAlan Cox /* 106ec179322SAlan Cox * The number of bits in a population map entry 107ec179322SAlan Cox */ 108ec179322SAlan Cox #define NBPOPMAP (NBBY * sizeof(popmap_t)) 109ec179322SAlan Cox 110ec179322SAlan Cox /* 111ec179322SAlan Cox * The number of population map entries in a reservation 112ec179322SAlan Cox */ 113ec179322SAlan Cox #define NPOPMAP howmany(VM_LEVEL_0_NPAGES, NBPOPMAP) 114ec179322SAlan Cox 115ec179322SAlan Cox /* 1163180f757SAlan Cox * Clear a bit in the population map. 1173180f757SAlan Cox */ 1183180f757SAlan Cox static __inline void 1193180f757SAlan Cox popmap_clear(popmap_t popmap[], int i) 1203180f757SAlan Cox { 1213180f757SAlan Cox 1223180f757SAlan Cox popmap[i / NBPOPMAP] &= ~(1UL << (i % NBPOPMAP)); 1233180f757SAlan Cox } 1243180f757SAlan Cox 1253180f757SAlan Cox /* 1263180f757SAlan Cox * Set a bit in the population map. 1273180f757SAlan Cox */ 1283180f757SAlan Cox static __inline void 1293180f757SAlan Cox popmap_set(popmap_t popmap[], int i) 1303180f757SAlan Cox { 1313180f757SAlan Cox 1323180f757SAlan Cox popmap[i / NBPOPMAP] |= 1UL << (i % NBPOPMAP); 1333180f757SAlan Cox } 1343180f757SAlan Cox 1353180f757SAlan Cox /* 1363180f757SAlan Cox * Is a bit in the population map clear? 1373180f757SAlan Cox */ 1383180f757SAlan Cox static __inline boolean_t 1393180f757SAlan Cox popmap_is_clear(popmap_t popmap[], int i) 1403180f757SAlan Cox { 1413180f757SAlan Cox 1423180f757SAlan Cox return ((popmap[i / NBPOPMAP] & (1UL << (i % NBPOPMAP))) == 0); 1433180f757SAlan Cox } 1443180f757SAlan Cox 1453180f757SAlan Cox /* 1463180f757SAlan Cox * Is a bit in the population map set? 1473180f757SAlan Cox */ 1483180f757SAlan Cox static __inline boolean_t 1493180f757SAlan Cox popmap_is_set(popmap_t popmap[], int i) 1503180f757SAlan Cox { 1513180f757SAlan Cox 1523180f757SAlan Cox return ((popmap[i / NBPOPMAP] & (1UL << (i % NBPOPMAP))) != 0); 1533180f757SAlan Cox } 1543180f757SAlan Cox 1553180f757SAlan Cox /* 156f8a47341SAlan Cox * The reservation structure 157f8a47341SAlan Cox * 158f8a47341SAlan Cox * A reservation structure is constructed whenever a large physical page is 159f8a47341SAlan Cox * speculatively allocated to an object. The reservation provides the small 160f8a47341SAlan Cox * physical pages for the range [pindex, pindex + VM_LEVEL_0_NPAGES) of offsets 161f8a47341SAlan Cox * within that object. The reservation's "popcnt" tracks the number of these 162f8a47341SAlan Cox * small physical pages that are in use at any given time. When and if the 1633453bca8SAlan Cox * reservation is not fully utilized, it appears in the queue of partially 164f8a47341SAlan Cox * populated reservations. The reservation always appears on the containing 165f8a47341SAlan Cox * object's list of reservations. 166f8a47341SAlan Cox * 1673453bca8SAlan Cox * A partially populated reservation can be broken and reclaimed at any time. 168e2068d0bSJeff Roberson * 169e2068d0bSJeff Roberson * f - vm_domain_free_lock 170e2068d0bSJeff Roberson * o - vm_reserv_object_lock 171e2068d0bSJeff Roberson * c - constant after boot 172f8a47341SAlan Cox */ 173f8a47341SAlan Cox struct vm_reserv { 174e2068d0bSJeff Roberson TAILQ_ENTRY(vm_reserv) partpopq; /* (f) per-domain queue. */ 175e2068d0bSJeff Roberson LIST_ENTRY(vm_reserv) objq; /* (o, f) object queue */ 176e2068d0bSJeff Roberson vm_object_t object; /* (o, f) containing object */ 177e2068d0bSJeff Roberson vm_pindex_t pindex; /* (o, f) offset in object */ 178e2068d0bSJeff Roberson vm_page_t pages; /* (c) first page */ 179e2068d0bSJeff Roberson int domain; /* (c) NUMA domain. */ 180e2068d0bSJeff Roberson int popcnt; /* (f) # of pages in use */ 181e2068d0bSJeff Roberson char inpartpopq; /* (f) */ 182e2068d0bSJeff Roberson popmap_t popmap[NPOPMAP]; /* (f) bit vector, used pages */ 183f8a47341SAlan Cox }; 184f8a47341SAlan Cox 185f8a47341SAlan Cox /* 186f8a47341SAlan Cox * The reservation array 187f8a47341SAlan Cox * 188f8a47341SAlan Cox * This array is analoguous in function to vm_page_array. It differs in the 189f8a47341SAlan Cox * respect that it may contain a greater number of useful reservation 190f8a47341SAlan Cox * structures than there are (physical) superpages. These "invalid" 191f8a47341SAlan Cox * reservation structures exist to trade-off space for time in the 192f8a47341SAlan Cox * implementation of vm_reserv_from_page(). Invalid reservation structures are 193f8a47341SAlan Cox * distinguishable from "valid" reservation structures by inspecting the 194f8a47341SAlan Cox * reservation's "pages" field. Invalid reservation structures have a NULL 195f8a47341SAlan Cox * "pages" field. 196f8a47341SAlan Cox * 197f8a47341SAlan Cox * vm_reserv_from_page() maps a small (physical) page to an element of this 198f8a47341SAlan Cox * array by computing a physical reservation number from the page's physical 199f8a47341SAlan Cox * address. The physical reservation number is used as the array index. 200f8a47341SAlan Cox * 201f8a47341SAlan Cox * An "active" reservation is a valid reservation structure that has a non-NULL 202f8a47341SAlan Cox * "object" field and a non-zero "popcnt" field. In other words, every active 203f8a47341SAlan Cox * reservation belongs to a particular object. Moreover, every active 204f8a47341SAlan Cox * reservation has an entry in the containing object's list of reservations. 205f8a47341SAlan Cox */ 206f8a47341SAlan Cox static vm_reserv_t vm_reserv_array; 207f8a47341SAlan Cox 208f8a47341SAlan Cox /* 2093453bca8SAlan Cox * The partially populated reservation queue 210f8a47341SAlan Cox * 2113453bca8SAlan Cox * This queue enables the fast recovery of an unused free small page from a 2123453bca8SAlan Cox * partially populated reservation. The reservation at the head of this queue 2133453bca8SAlan Cox * is the least recently changed, partially populated reservation. 214f8a47341SAlan Cox * 215f8a47341SAlan Cox * Access to this queue is synchronized by the free page queue lock. 216f8a47341SAlan Cox */ 217ef435ae7SJeff Roberson static TAILQ_HEAD(, vm_reserv) vm_rvq_partpop[MAXMEMDOM]; 218f8a47341SAlan Cox 219f8a47341SAlan Cox static SYSCTL_NODE(_vm, OID_AUTO, reserv, CTLFLAG_RD, 0, "Reservation Info"); 220f8a47341SAlan Cox 221f8a47341SAlan Cox static long vm_reserv_broken; 222f8a47341SAlan Cox SYSCTL_LONG(_vm_reserv, OID_AUTO, broken, CTLFLAG_RD, 223f8a47341SAlan Cox &vm_reserv_broken, 0, "Cumulative number of broken reservations"); 224f8a47341SAlan Cox 225f8a47341SAlan Cox static long vm_reserv_freed; 226f8a47341SAlan Cox SYSCTL_LONG(_vm_reserv, OID_AUTO, freed, CTLFLAG_RD, 227f8a47341SAlan Cox &vm_reserv_freed, 0, "Cumulative number of freed reservations"); 228f8a47341SAlan Cox 229e0a63baaSAlan Cox static int sysctl_vm_reserv_fullpop(SYSCTL_HANDLER_ARGS); 230e0a63baaSAlan Cox 231e0a63baaSAlan Cox SYSCTL_PROC(_vm_reserv, OID_AUTO, fullpop, CTLTYPE_INT | CTLFLAG_RD, NULL, 0, 232e0a63baaSAlan Cox sysctl_vm_reserv_fullpop, "I", "Current number of full reservations"); 233e0a63baaSAlan Cox 234f8a47341SAlan Cox static int sysctl_vm_reserv_partpopq(SYSCTL_HANDLER_ARGS); 235f8a47341SAlan Cox 236f8a47341SAlan Cox SYSCTL_OID(_vm_reserv, OID_AUTO, partpopq, CTLTYPE_STRING | CTLFLAG_RD, NULL, 0, 2373453bca8SAlan Cox sysctl_vm_reserv_partpopq, "A", "Partially populated reservation queues"); 238f8a47341SAlan Cox 239f8a47341SAlan Cox static long vm_reserv_reclaimed; 240f8a47341SAlan Cox SYSCTL_LONG(_vm_reserv, OID_AUTO, reclaimed, CTLFLAG_RD, 241f8a47341SAlan Cox &vm_reserv_reclaimed, 0, "Cumulative number of reclaimed reservations"); 242f8a47341SAlan Cox 243e2068d0bSJeff Roberson /* 244e2068d0bSJeff Roberson * The object lock pool is used to synchronize the rvq. We can not use a 245e2068d0bSJeff Roberson * pool mutex because it is required before malloc works. 246e2068d0bSJeff Roberson * 247e2068d0bSJeff Roberson * The "hash" function could be made faster without divide and modulo. 248e2068d0bSJeff Roberson */ 249e2068d0bSJeff Roberson #define VM_RESERV_OBJ_LOCK_COUNT MAXCPU 250e2068d0bSJeff Roberson 251e2068d0bSJeff Roberson struct mtx_padalign vm_reserv_object_mtx[VM_RESERV_OBJ_LOCK_COUNT]; 252e2068d0bSJeff Roberson 253e2068d0bSJeff Roberson #define vm_reserv_object_lock_idx(object) \ 254e2068d0bSJeff Roberson (((uintptr_t)object / sizeof(*object)) % VM_RESERV_OBJ_LOCK_COUNT) 255e2068d0bSJeff Roberson #define vm_reserv_object_lock_ptr(object) \ 256e2068d0bSJeff Roberson &vm_reserv_object_mtx[vm_reserv_object_lock_idx((object))] 257e2068d0bSJeff Roberson #define vm_reserv_object_lock(object) \ 258e2068d0bSJeff Roberson mtx_lock(vm_reserv_object_lock_ptr((object))) 259e2068d0bSJeff Roberson #define vm_reserv_object_unlock(object) \ 260e2068d0bSJeff Roberson mtx_unlock(vm_reserv_object_lock_ptr((object))) 261e2068d0bSJeff Roberson 262ada27a3bSKonstantin Belousov static void vm_reserv_break(vm_reserv_t rv); 263ec179322SAlan Cox static void vm_reserv_depopulate(vm_reserv_t rv, int index); 264f8a47341SAlan Cox static vm_reserv_t vm_reserv_from_page(vm_page_t m); 265f8a47341SAlan Cox static boolean_t vm_reserv_has_pindex(vm_reserv_t rv, 266f8a47341SAlan Cox vm_pindex_t pindex); 267ec179322SAlan Cox static void vm_reserv_populate(vm_reserv_t rv, int index); 26844aab2c3SAlan Cox static void vm_reserv_reclaim(vm_reserv_t rv); 269f8a47341SAlan Cox 270f8a47341SAlan Cox /* 271e0a63baaSAlan Cox * Returns the current number of full reservations. 272e0a63baaSAlan Cox * 273e0a63baaSAlan Cox * Since the number of full reservations is computed without acquiring the 274e0a63baaSAlan Cox * free page queue lock, the returned value may be inexact. 275e0a63baaSAlan Cox */ 276e0a63baaSAlan Cox static int 277e0a63baaSAlan Cox sysctl_vm_reserv_fullpop(SYSCTL_HANDLER_ARGS) 278e0a63baaSAlan Cox { 279e0a63baaSAlan Cox vm_paddr_t paddr; 280e0a63baaSAlan Cox struct vm_phys_seg *seg; 281e0a63baaSAlan Cox vm_reserv_t rv; 282e0a63baaSAlan Cox int fullpop, segind; 283e0a63baaSAlan Cox 284e0a63baaSAlan Cox fullpop = 0; 285e0a63baaSAlan Cox for (segind = 0; segind < vm_phys_nsegs; segind++) { 286e0a63baaSAlan Cox seg = &vm_phys_segs[segind]; 287e0a63baaSAlan Cox paddr = roundup2(seg->start, VM_LEVEL_0_SIZE); 288e0a63baaSAlan Cox while (paddr + VM_LEVEL_0_SIZE <= seg->end) { 289e0a63baaSAlan Cox rv = &vm_reserv_array[paddr >> VM_LEVEL_0_SHIFT]; 290e0a63baaSAlan Cox fullpop += rv->popcnt == VM_LEVEL_0_NPAGES; 291e0a63baaSAlan Cox paddr += VM_LEVEL_0_SIZE; 292e0a63baaSAlan Cox } 293e0a63baaSAlan Cox } 294e0a63baaSAlan Cox return (sysctl_handle_int(oidp, &fullpop, 0, req)); 295e0a63baaSAlan Cox } 296e0a63baaSAlan Cox 297e0a63baaSAlan Cox /* 2983453bca8SAlan Cox * Describes the current state of the partially populated reservation queue. 299f8a47341SAlan Cox */ 300f8a47341SAlan Cox static int 301f8a47341SAlan Cox sysctl_vm_reserv_partpopq(SYSCTL_HANDLER_ARGS) 302f8a47341SAlan Cox { 303f8a47341SAlan Cox struct sbuf sbuf; 304f8a47341SAlan Cox vm_reserv_t rv; 305ef435ae7SJeff Roberson int counter, error, domain, level, unused_pages; 306f8a47341SAlan Cox 30700f0e671SMatthew D Fleming error = sysctl_wire_old_buffer(req, 0); 30800f0e671SMatthew D Fleming if (error != 0) 30900f0e671SMatthew D Fleming return (error); 3104e657159SMatthew D Fleming sbuf_new_for_sysctl(&sbuf, NULL, 128, req); 311ef435ae7SJeff Roberson sbuf_printf(&sbuf, "\nDOMAIN LEVEL SIZE NUMBER\n\n"); 312ef435ae7SJeff Roberson for (domain = 0; domain < vm_ndomains; domain++) { 313f8a47341SAlan Cox for (level = -1; level <= VM_NRESERVLEVEL - 2; level++) { 314f8a47341SAlan Cox counter = 0; 315f8a47341SAlan Cox unused_pages = 0; 316e2068d0bSJeff Roberson vm_domain_free_lock(VM_DOMAIN(domain)); 317ef435ae7SJeff Roberson TAILQ_FOREACH(rv, &vm_rvq_partpop[domain], partpopq) { 318f8a47341SAlan Cox counter++; 319f8a47341SAlan Cox unused_pages += VM_LEVEL_0_NPAGES - rv->popcnt; 320f8a47341SAlan Cox } 321e2068d0bSJeff Roberson vm_domain_free_unlock(VM_DOMAIN(domain)); 322ef435ae7SJeff Roberson sbuf_printf(&sbuf, "%6d, %7d, %6dK, %6d\n", 323ef435ae7SJeff Roberson domain, level, 3242cf36c8fSAlan Cox unused_pages * ((int)PAGE_SIZE / 1024), counter); 325f8a47341SAlan Cox } 326ef435ae7SJeff Roberson } 3274e657159SMatthew D Fleming error = sbuf_finish(&sbuf); 328f8a47341SAlan Cox sbuf_delete(&sbuf); 329f8a47341SAlan Cox return (error); 330f8a47341SAlan Cox } 331f8a47341SAlan Cox 332f8a47341SAlan Cox /* 333e2068d0bSJeff Roberson * Remove a reservation from the object's objq. 334e2068d0bSJeff Roberson */ 335e2068d0bSJeff Roberson static void 336e2068d0bSJeff Roberson vm_reserv_remove(vm_reserv_t rv) 337e2068d0bSJeff Roberson { 338e2068d0bSJeff Roberson vm_object_t object; 339e2068d0bSJeff Roberson 340e2068d0bSJeff Roberson KASSERT(rv->object != NULL, 341e2068d0bSJeff Roberson ("vm_reserv_remove: reserv %p is free", rv)); 342e2068d0bSJeff Roberson KASSERT(!rv->inpartpopq, 343e2068d0bSJeff Roberson ("vm_reserv_remove: reserv %p's inpartpopq is TRUE", rv)); 344e2068d0bSJeff Roberson object = rv->object; 345e2068d0bSJeff Roberson vm_reserv_object_lock(object); 346e2068d0bSJeff Roberson LIST_REMOVE(rv, objq); 347e2068d0bSJeff Roberson rv->object = NULL; 348e2068d0bSJeff Roberson vm_reserv_object_unlock(object); 349e2068d0bSJeff Roberson } 350e2068d0bSJeff Roberson 351e2068d0bSJeff Roberson /* 352e2068d0bSJeff Roberson * Insert a new reservation into the object's objq. 353e2068d0bSJeff Roberson */ 354e2068d0bSJeff Roberson static void 355e2068d0bSJeff Roberson vm_reserv_insert(vm_reserv_t rv, vm_object_t object, vm_pindex_t pindex) 356e2068d0bSJeff Roberson { 357e2068d0bSJeff Roberson int i; 358e2068d0bSJeff Roberson 359e2068d0bSJeff Roberson KASSERT(rv->object == NULL, 360e2068d0bSJeff Roberson ("vm_reserv_insert: reserv %p isn't free", rv)); 361e2068d0bSJeff Roberson KASSERT(rv->popcnt == 0, 362e2068d0bSJeff Roberson ("vm_reserv_insert: reserv %p's popcnt is corrupted", rv)); 363e2068d0bSJeff Roberson KASSERT(!rv->inpartpopq, 364e2068d0bSJeff Roberson ("vm_reserv_insert: reserv %p's inpartpopq is TRUE", rv)); 365e2068d0bSJeff Roberson for (i = 0; i < NPOPMAP; i++) 366e2068d0bSJeff Roberson KASSERT(rv->popmap[i] == 0, 367e2068d0bSJeff Roberson ("vm_reserv_insert: reserv %p's popmap is corrupted", rv)); 368e2068d0bSJeff Roberson vm_reserv_object_lock(object); 369e2068d0bSJeff Roberson rv->pindex = pindex; 370e2068d0bSJeff Roberson rv->object = object; 371e2068d0bSJeff Roberson LIST_INSERT_HEAD(&object->rvq, rv, objq); 372e2068d0bSJeff Roberson vm_reserv_object_unlock(object); 373e2068d0bSJeff Roberson } 374e2068d0bSJeff Roberson 375e2068d0bSJeff Roberson /* 376f8a47341SAlan Cox * Reduces the given reservation's population count. If the population count 377f8a47341SAlan Cox * becomes zero, the reservation is destroyed. Additionally, moves the 3783453bca8SAlan Cox * reservation to the tail of the partially populated reservation queue if the 379f8a47341SAlan Cox * population count is non-zero. 380f8a47341SAlan Cox * 381f8a47341SAlan Cox * The free page queue lock must be held. 382f8a47341SAlan Cox */ 383f8a47341SAlan Cox static void 384ec179322SAlan Cox vm_reserv_depopulate(vm_reserv_t rv, int index) 385f8a47341SAlan Cox { 386f8a47341SAlan Cox 387e2068d0bSJeff Roberson vm_domain_free_assert_locked(VM_DOMAIN(rv->domain)); 388f8a47341SAlan Cox KASSERT(rv->object != NULL, 389f8a47341SAlan Cox ("vm_reserv_depopulate: reserv %p is free", rv)); 3903180f757SAlan Cox KASSERT(popmap_is_set(rv->popmap, index), 391a08c1515SAlan Cox ("vm_reserv_depopulate: reserv %p's popmap[%d] is clear", rv, 392a08c1515SAlan Cox index)); 393f8a47341SAlan Cox KASSERT(rv->popcnt > 0, 394f8a47341SAlan Cox ("vm_reserv_depopulate: reserv %p's popcnt is corrupted", rv)); 395ef435ae7SJeff Roberson KASSERT(rv->domain >= 0 && rv->domain < vm_ndomains, 396ef435ae7SJeff Roberson ("vm_reserv_depopulate: reserv %p's domain is corrupted %d", 397ef435ae7SJeff Roberson rv, rv->domain)); 398f8a47341SAlan Cox if (rv->inpartpopq) { 399ef435ae7SJeff Roberson TAILQ_REMOVE(&vm_rvq_partpop[rv->domain], rv, partpopq); 400f8a47341SAlan Cox rv->inpartpopq = FALSE; 401dd05fa19SAlan Cox } else { 402dd05fa19SAlan Cox KASSERT(rv->pages->psind == 1, 403dd05fa19SAlan Cox ("vm_reserv_depopulate: reserv %p is already demoted", 404dd05fa19SAlan Cox rv)); 405dd05fa19SAlan Cox rv->pages->psind = 0; 406f8a47341SAlan Cox } 4073180f757SAlan Cox popmap_clear(rv->popmap, index); 408f8a47341SAlan Cox rv->popcnt--; 409f8a47341SAlan Cox if (rv->popcnt == 0) { 410e2068d0bSJeff Roberson vm_reserv_remove(rv); 411f8a47341SAlan Cox vm_phys_free_pages(rv->pages, VM_LEVEL_0_ORDER); 412f8a47341SAlan Cox vm_reserv_freed++; 413f8a47341SAlan Cox } else { 414f8a47341SAlan Cox rv->inpartpopq = TRUE; 415ef435ae7SJeff Roberson TAILQ_INSERT_TAIL(&vm_rvq_partpop[rv->domain], rv, partpopq); 416f8a47341SAlan Cox } 417f8a47341SAlan Cox } 418f8a47341SAlan Cox 419f8a47341SAlan Cox /* 420f8a47341SAlan Cox * Returns the reservation to which the given page might belong. 421f8a47341SAlan Cox */ 422f8a47341SAlan Cox static __inline vm_reserv_t 423f8a47341SAlan Cox vm_reserv_from_page(vm_page_t m) 424f8a47341SAlan Cox { 425f8a47341SAlan Cox 426f8a47341SAlan Cox return (&vm_reserv_array[VM_PAGE_TO_PHYS(m) >> VM_LEVEL_0_SHIFT]); 427f8a47341SAlan Cox } 428f8a47341SAlan Cox 429f8a47341SAlan Cox /* 430e2068d0bSJeff Roberson * Returns an existing reservation or NULL and initialized successor pointer. 431e2068d0bSJeff Roberson */ 432e2068d0bSJeff Roberson static vm_reserv_t 433e2068d0bSJeff Roberson vm_reserv_from_object(vm_object_t object, vm_pindex_t pindex, 434e2068d0bSJeff Roberson vm_page_t mpred, vm_page_t *msuccp) 435e2068d0bSJeff Roberson { 436e2068d0bSJeff Roberson vm_reserv_t rv; 437e2068d0bSJeff Roberson vm_page_t msucc; 438e2068d0bSJeff Roberson 439e2068d0bSJeff Roberson msucc = NULL; 440e2068d0bSJeff Roberson if (mpred != NULL) { 441e2068d0bSJeff Roberson KASSERT(mpred->object == object, 442e2068d0bSJeff Roberson ("vm_reserv_from_object: object doesn't contain mpred")); 443e2068d0bSJeff Roberson KASSERT(mpred->pindex < pindex, 444e2068d0bSJeff Roberson ("vm_reserv_from_object: mpred doesn't precede pindex")); 445e2068d0bSJeff Roberson rv = vm_reserv_from_page(mpred); 446e2068d0bSJeff Roberson if (rv->object == object && vm_reserv_has_pindex(rv, pindex)) 447e2068d0bSJeff Roberson goto found; 448e2068d0bSJeff Roberson msucc = TAILQ_NEXT(mpred, listq); 449e2068d0bSJeff Roberson } else 450e2068d0bSJeff Roberson msucc = TAILQ_FIRST(&object->memq); 451e2068d0bSJeff Roberson if (msucc != NULL) { 452e2068d0bSJeff Roberson KASSERT(msucc->pindex > pindex, 453e2068d0bSJeff Roberson ("vm_reserv_from_object: msucc doesn't succeed pindex")); 454e2068d0bSJeff Roberson rv = vm_reserv_from_page(msucc); 455e2068d0bSJeff Roberson if (rv->object == object && vm_reserv_has_pindex(rv, pindex)) 456e2068d0bSJeff Roberson goto found; 457e2068d0bSJeff Roberson } 458e2068d0bSJeff Roberson rv = NULL; 459e2068d0bSJeff Roberson 460e2068d0bSJeff Roberson found: 461e2068d0bSJeff Roberson *msuccp = msucc; 462e2068d0bSJeff Roberson 463e2068d0bSJeff Roberson return (rv); 464e2068d0bSJeff Roberson } 465e2068d0bSJeff Roberson 466e2068d0bSJeff Roberson /* 467f8a47341SAlan Cox * Returns TRUE if the given reservation contains the given page index and 468f8a47341SAlan Cox * FALSE otherwise. 469f8a47341SAlan Cox */ 470f8a47341SAlan Cox static __inline boolean_t 471f8a47341SAlan Cox vm_reserv_has_pindex(vm_reserv_t rv, vm_pindex_t pindex) 472f8a47341SAlan Cox { 473f8a47341SAlan Cox 474f8a47341SAlan Cox return (((pindex - rv->pindex) & ~(VM_LEVEL_0_NPAGES - 1)) == 0); 475f8a47341SAlan Cox } 476f8a47341SAlan Cox 477f8a47341SAlan Cox /* 478f8a47341SAlan Cox * Increases the given reservation's population count. Moves the reservation 4793453bca8SAlan Cox * to the tail of the partially populated reservation queue. 480f8a47341SAlan Cox * 481f8a47341SAlan Cox * The free page queue must be locked. 482f8a47341SAlan Cox */ 483f8a47341SAlan Cox static void 484ec179322SAlan Cox vm_reserv_populate(vm_reserv_t rv, int index) 485f8a47341SAlan Cox { 486f8a47341SAlan Cox 487e2068d0bSJeff Roberson vm_domain_free_assert_locked(VM_DOMAIN(rv->domain)); 488f8a47341SAlan Cox KASSERT(rv->object != NULL, 489f8a47341SAlan Cox ("vm_reserv_populate: reserv %p is free", rv)); 4903180f757SAlan Cox KASSERT(popmap_is_clear(rv->popmap, index), 491a08c1515SAlan Cox ("vm_reserv_populate: reserv %p's popmap[%d] is set", rv, 492a08c1515SAlan Cox index)); 493f8a47341SAlan Cox KASSERT(rv->popcnt < VM_LEVEL_0_NPAGES, 494f8a47341SAlan Cox ("vm_reserv_populate: reserv %p is already full", rv)); 495dd05fa19SAlan Cox KASSERT(rv->pages->psind == 0, 496dd05fa19SAlan Cox ("vm_reserv_populate: reserv %p is already promoted", rv)); 497ef435ae7SJeff Roberson KASSERT(rv->domain >= 0 && rv->domain < vm_ndomains, 498ef435ae7SJeff Roberson ("vm_reserv_populate: reserv %p's domain is corrupted %d", 499ef435ae7SJeff Roberson rv, rv->domain)); 500f8a47341SAlan Cox if (rv->inpartpopq) { 501ef435ae7SJeff Roberson TAILQ_REMOVE(&vm_rvq_partpop[rv->domain], rv, partpopq); 502f8a47341SAlan Cox rv->inpartpopq = FALSE; 503f8a47341SAlan Cox } 5043180f757SAlan Cox popmap_set(rv->popmap, index); 505f8a47341SAlan Cox rv->popcnt++; 506f8a47341SAlan Cox if (rv->popcnt < VM_LEVEL_0_NPAGES) { 507f8a47341SAlan Cox rv->inpartpopq = TRUE; 508ef435ae7SJeff Roberson TAILQ_INSERT_TAIL(&vm_rvq_partpop[rv->domain], rv, partpopq); 509dd05fa19SAlan Cox } else 510dd05fa19SAlan Cox rv->pages->psind = 1; 511f8a47341SAlan Cox } 512f8a47341SAlan Cox 513f8a47341SAlan Cox /* 5145f70fb14SMark Johnston * Attempts to allocate a contiguous set of physical pages from existing 5155f70fb14SMark Johnston * reservations. See vm_reserv_alloc_contig() for a description of the 5165f70fb14SMark Johnston * function's parameters. 517c68c3537SAlan Cox * 518920da7e4SAlan Cox * The page "mpred" must immediately precede the offset "pindex" within the 519920da7e4SAlan Cox * specified object. 520920da7e4SAlan Cox * 5215f70fb14SMark Johnston * The object must be locked. 522c68c3537SAlan Cox */ 523c68c3537SAlan Cox vm_page_t 524e2068d0bSJeff Roberson vm_reserv_extend_contig(int req, vm_object_t object, vm_pindex_t pindex, 525e2068d0bSJeff Roberson int domain, u_long npages, vm_paddr_t low, vm_paddr_t high, 526e2068d0bSJeff Roberson u_long alignment, vm_paddr_t boundary, vm_page_t mpred) 527e2068d0bSJeff Roberson { 528e2068d0bSJeff Roberson struct vm_domain *vmd; 529e2068d0bSJeff Roberson vm_paddr_t pa, size; 530e2068d0bSJeff Roberson vm_page_t m, msucc; 531e2068d0bSJeff Roberson vm_reserv_t rv; 532e2068d0bSJeff Roberson int i, index; 533e2068d0bSJeff Roberson 534e2068d0bSJeff Roberson VM_OBJECT_ASSERT_WLOCKED(object); 535e2068d0bSJeff Roberson KASSERT(npages != 0, ("vm_reserv_alloc_contig: npages is 0")); 536e2068d0bSJeff Roberson 537e2068d0bSJeff Roberson /* 538e2068d0bSJeff Roberson * Is a reservation fundamentally impossible? 539e2068d0bSJeff Roberson */ 540e2068d0bSJeff Roberson if (pindex < VM_RESERV_INDEX(object, pindex) || 541e2068d0bSJeff Roberson pindex + npages > object->size || object->resident_page_count == 0) 542e2068d0bSJeff Roberson return (NULL); 543e2068d0bSJeff Roberson 544e2068d0bSJeff Roberson /* 545e2068d0bSJeff Roberson * All reservations of a particular size have the same alignment. 546e2068d0bSJeff Roberson * Assuming that the first page is allocated from a reservation, the 547e2068d0bSJeff Roberson * least significant bits of its physical address can be determined 548e2068d0bSJeff Roberson * from its offset from the beginning of the reservation and the size 549e2068d0bSJeff Roberson * of the reservation. 550e2068d0bSJeff Roberson * 551e2068d0bSJeff Roberson * Could the specified index within a reservation of the smallest 552e2068d0bSJeff Roberson * possible size satisfy the alignment and boundary requirements? 553e2068d0bSJeff Roberson */ 554e2068d0bSJeff Roberson pa = VM_RESERV_INDEX(object, pindex) << PAGE_SHIFT; 555e2068d0bSJeff Roberson if ((pa & (alignment - 1)) != 0) 556e2068d0bSJeff Roberson return (NULL); 557e2068d0bSJeff Roberson size = npages << PAGE_SHIFT; 558e2068d0bSJeff Roberson if (((pa ^ (pa + size - 1)) & ~(boundary - 1)) != 0) 559e2068d0bSJeff Roberson return (NULL); 560e2068d0bSJeff Roberson 561e2068d0bSJeff Roberson /* 562e2068d0bSJeff Roberson * Look for an existing reservation. 563e2068d0bSJeff Roberson */ 564e2068d0bSJeff Roberson rv = vm_reserv_from_object(object, pindex, mpred, &msucc); 565e2068d0bSJeff Roberson if (rv == NULL) 566e2068d0bSJeff Roberson return (NULL); 567e2068d0bSJeff Roberson KASSERT(object != kernel_object || rv->domain == domain, 568e2068d0bSJeff Roberson ("vm_reserv_extend_contig: Domain mismatch from reservation.")); 569e2068d0bSJeff Roberson index = VM_RESERV_INDEX(object, pindex); 570e2068d0bSJeff Roberson /* Does the allocation fit within the reservation? */ 571e2068d0bSJeff Roberson if (index + npages > VM_LEVEL_0_NPAGES) 572e2068d0bSJeff Roberson return (NULL); 573e2068d0bSJeff Roberson domain = rv->domain; 574e2068d0bSJeff Roberson vmd = VM_DOMAIN(domain); 575e2068d0bSJeff Roberson vm_domain_free_lock(vmd); 576e2068d0bSJeff Roberson if (rv->object != object || !vm_domain_available(vmd, req, npages)) { 577e2068d0bSJeff Roberson m = NULL; 578e2068d0bSJeff Roberson goto out; 579e2068d0bSJeff Roberson } 580e2068d0bSJeff Roberson m = &rv->pages[index]; 581e2068d0bSJeff Roberson pa = VM_PAGE_TO_PHYS(m); 582e2068d0bSJeff Roberson if (pa < low || pa + size > high || (pa & (alignment - 1)) != 0 || 583e2068d0bSJeff Roberson ((pa ^ (pa + size - 1)) & ~(boundary - 1)) != 0) { 584e2068d0bSJeff Roberson m = NULL; 585e2068d0bSJeff Roberson goto out; 586e2068d0bSJeff Roberson } 587e2068d0bSJeff Roberson /* Handle vm_page_rename(m, new_object, ...). */ 588e2068d0bSJeff Roberson for (i = 0; i < npages; i++) { 589e2068d0bSJeff Roberson if (popmap_is_set(rv->popmap, index + i)) { 590e2068d0bSJeff Roberson m = NULL; 591e2068d0bSJeff Roberson goto out; 592e2068d0bSJeff Roberson } 593e2068d0bSJeff Roberson } 594e2068d0bSJeff Roberson for (i = 0; i < npages; i++) 595e2068d0bSJeff Roberson vm_reserv_populate(rv, index + i); 596*30fbfddaSJeff Roberson vm_domain_freecnt_dec(vmd, npages); 597e2068d0bSJeff Roberson out: 598e2068d0bSJeff Roberson vm_domain_free_unlock(vmd); 599e2068d0bSJeff Roberson return (m); 600e2068d0bSJeff Roberson } 601e2068d0bSJeff Roberson 602e2068d0bSJeff Roberson /* 603e2068d0bSJeff Roberson * Allocates a contiguous set of physical pages of the given size "npages" 6045f70fb14SMark Johnston * from newly created reservations. All of the physical pages 605e2068d0bSJeff Roberson * must be at or above the given physical address "low" and below the given 606e2068d0bSJeff Roberson * physical address "high". The given value "alignment" determines the 607e2068d0bSJeff Roberson * alignment of the first physical page in the set. If the given value 608e2068d0bSJeff Roberson * "boundary" is non-zero, then the set of physical pages cannot cross any 609e2068d0bSJeff Roberson * physical address boundary that is a multiple of that value. Both 610e2068d0bSJeff Roberson * "alignment" and "boundary" must be a power of two. 611e2068d0bSJeff Roberson * 6125f70fb14SMark Johnston * Callers should first invoke vm_reserv_extend_contig() to attempt an 6135f70fb14SMark Johnston * allocation from existing reservations. 6145f70fb14SMark Johnston * 615e2068d0bSJeff Roberson * The page "mpred" must immediately precede the offset "pindex" within the 616e2068d0bSJeff Roberson * specified object. 617e2068d0bSJeff Roberson * 618e2068d0bSJeff Roberson * The object and free page queue must be locked. 619e2068d0bSJeff Roberson */ 620e2068d0bSJeff Roberson vm_page_t 621ef435ae7SJeff Roberson vm_reserv_alloc_contig(vm_object_t object, vm_pindex_t pindex, int domain, 622ef435ae7SJeff Roberson u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, 623ef435ae7SJeff Roberson vm_paddr_t boundary, vm_page_t mpred) 624c68c3537SAlan Cox { 625c68c3537SAlan Cox vm_paddr_t pa, size; 626920da7e4SAlan Cox vm_page_t m, m_ret, msucc; 627c68c3537SAlan Cox vm_pindex_t first, leftcap, rightcap; 628c68c3537SAlan Cox vm_reserv_t rv; 629c68c3537SAlan Cox u_long allocpages, maxpages, minpages; 630c68c3537SAlan Cox int i, index, n; 631c68c3537SAlan Cox 632e2068d0bSJeff Roberson vm_domain_free_assert_locked(VM_DOMAIN(domain)); 63389f6b863SAttilio Rao VM_OBJECT_ASSERT_WLOCKED(object); 634c68c3537SAlan Cox KASSERT(npages != 0, ("vm_reserv_alloc_contig: npages is 0")); 635c68c3537SAlan Cox 636c68c3537SAlan Cox /* 637c68c3537SAlan Cox * Is a reservation fundamentally impossible? 638c68c3537SAlan Cox */ 639c68c3537SAlan Cox if (pindex < VM_RESERV_INDEX(object, pindex) || 640c68c3537SAlan Cox pindex + npages > object->size) 641c68c3537SAlan Cox return (NULL); 642c68c3537SAlan Cox 643c68c3537SAlan Cox /* 644c68c3537SAlan Cox * All reservations of a particular size have the same alignment. 645c68c3537SAlan Cox * Assuming that the first page is allocated from a reservation, the 646c68c3537SAlan Cox * least significant bits of its physical address can be determined 647c68c3537SAlan Cox * from its offset from the beginning of the reservation and the size 648c68c3537SAlan Cox * of the reservation. 649c68c3537SAlan Cox * 650c68c3537SAlan Cox * Could the specified index within a reservation of the smallest 651c68c3537SAlan Cox * possible size satisfy the alignment and boundary requirements? 652c68c3537SAlan Cox */ 653c68c3537SAlan Cox pa = VM_RESERV_INDEX(object, pindex) << PAGE_SHIFT; 654c68c3537SAlan Cox if ((pa & (alignment - 1)) != 0) 655c68c3537SAlan Cox return (NULL); 656c68c3537SAlan Cox size = npages << PAGE_SHIFT; 657c68c3537SAlan Cox if (((pa ^ (pa + size - 1)) & ~(boundary - 1)) != 0) 658c68c3537SAlan Cox return (NULL); 659c68c3537SAlan Cox 660c68c3537SAlan Cox /* 661e2068d0bSJeff Roberson * Callers should've extended an existing reservation prior to 662e2068d0bSJeff Roberson * calling this function. If a reservation exists it is 663e2068d0bSJeff Roberson * incompatible with the allocation. 664c68c3537SAlan Cox */ 665e2068d0bSJeff Roberson rv = vm_reserv_from_object(object, pindex, mpred, &msucc); 666e2068d0bSJeff Roberson if (rv != NULL) 667e2068d0bSJeff Roberson return (NULL); 668c68c3537SAlan Cox 669c68c3537SAlan Cox /* 670c68c3537SAlan Cox * Could at least one reservation fit between the first index to the 67164f096eeSAlan Cox * left that can be used ("leftcap") and the first index to the right 67264f096eeSAlan Cox * that cannot be used ("rightcap")? 673e2068d0bSJeff Roberson * 674e2068d0bSJeff Roberson * We must synchronize with the reserv object lock to protect the 675e2068d0bSJeff Roberson * pindex/object of the resulting reservations against rename while 676e2068d0bSJeff Roberson * we are inspecting. 677c68c3537SAlan Cox */ 678c68c3537SAlan Cox first = pindex - VM_RESERV_INDEX(object, pindex); 679e2068d0bSJeff Roberson minpages = VM_RESERV_INDEX(object, pindex) + npages; 680e2068d0bSJeff Roberson maxpages = roundup2(minpages, VM_LEVEL_0_NPAGES); 681e2068d0bSJeff Roberson allocpages = maxpages; 682e2068d0bSJeff Roberson vm_reserv_object_lock(object); 683c68c3537SAlan Cox if (mpred != NULL) { 684c68c3537SAlan Cox if ((rv = vm_reserv_from_page(mpred))->object != object) 685c68c3537SAlan Cox leftcap = mpred->pindex + 1; 686c68c3537SAlan Cox else 687c68c3537SAlan Cox leftcap = rv->pindex + VM_LEVEL_0_NPAGES; 688e2068d0bSJeff Roberson if (leftcap > first) { 689e2068d0bSJeff Roberson vm_reserv_object_unlock(object); 690c68c3537SAlan Cox return (NULL); 691c68c3537SAlan Cox } 692e2068d0bSJeff Roberson } 693c68c3537SAlan Cox if (msucc != NULL) { 694c68c3537SAlan Cox if ((rv = vm_reserv_from_page(msucc))->object != object) 695c68c3537SAlan Cox rightcap = msucc->pindex; 696c68c3537SAlan Cox else 697c68c3537SAlan Cox rightcap = rv->pindex; 698c68c3537SAlan Cox if (first + maxpages > rightcap) { 699e2068d0bSJeff Roberson if (maxpages == VM_LEVEL_0_NPAGES) { 700e2068d0bSJeff Roberson vm_reserv_object_unlock(object); 701c68c3537SAlan Cox return (NULL); 702e2068d0bSJeff Roberson } 70364f096eeSAlan Cox 70464f096eeSAlan Cox /* 70564f096eeSAlan Cox * At least one reservation will fit between "leftcap" 70664f096eeSAlan Cox * and "rightcap". However, a reservation for the 70764f096eeSAlan Cox * last of the requested pages will not fit. Reduce 70864f096eeSAlan Cox * the size of the upcoming allocation accordingly. 70964f096eeSAlan Cox */ 710c68c3537SAlan Cox allocpages = minpages; 711c68c3537SAlan Cox } 712c68c3537SAlan Cox } 713e2068d0bSJeff Roberson vm_reserv_object_unlock(object); 714c68c3537SAlan Cox 715c68c3537SAlan Cox /* 716c68c3537SAlan Cox * Would the last new reservation extend past the end of the object? 717c68c3537SAlan Cox */ 718c68c3537SAlan Cox if (first + maxpages > object->size) { 719c68c3537SAlan Cox /* 720c68c3537SAlan Cox * Don't allocate the last new reservation if the object is a 721c68c3537SAlan Cox * vnode or backed by another object that is a vnode. 722c68c3537SAlan Cox */ 723c68c3537SAlan Cox if (object->type == OBJT_VNODE || 724c68c3537SAlan Cox (object->backing_object != NULL && 725c68c3537SAlan Cox object->backing_object->type == OBJT_VNODE)) { 726c68c3537SAlan Cox if (maxpages == VM_LEVEL_0_NPAGES) 727c68c3537SAlan Cox return (NULL); 728c68c3537SAlan Cox allocpages = minpages; 729c68c3537SAlan Cox } 730c68c3537SAlan Cox /* Speculate that the object may grow. */ 731c68c3537SAlan Cox } 732c68c3537SAlan Cox 733c68c3537SAlan Cox /* 73464f096eeSAlan Cox * Allocate the physical pages. The alignment and boundary specified 73564f096eeSAlan Cox * for this allocation may be different from the alignment and 73664f096eeSAlan Cox * boundary specified for the requested pages. For instance, the 73764f096eeSAlan Cox * specified index may not be the first page within the first new 73864f096eeSAlan Cox * reservation. 739c68c3537SAlan Cox */ 740ef435ae7SJeff Roberson m = vm_phys_alloc_contig(domain, allocpages, low, high, ulmax(alignment, 741c68c3537SAlan Cox VM_LEVEL_0_SIZE), boundary > VM_LEVEL_0_SIZE ? boundary : 0); 742c68c3537SAlan Cox if (m == NULL) 743c68c3537SAlan Cox return (NULL); 744e2068d0bSJeff Roberson KASSERT(vm_phys_domain(m) == domain, 7457a469c8eSJeff Roberson ("vm_reserv_alloc_contig: Page domain does not match requested.")); 74664f096eeSAlan Cox 74764f096eeSAlan Cox /* 74864f096eeSAlan Cox * The allocated physical pages always begin at a reservation 74964f096eeSAlan Cox * boundary, but they do not always end at a reservation boundary. 75064f096eeSAlan Cox * Initialize every reservation that is completely covered by the 75164f096eeSAlan Cox * allocated physical pages. 75264f096eeSAlan Cox */ 753c68c3537SAlan Cox m_ret = NULL; 754c68c3537SAlan Cox index = VM_RESERV_INDEX(object, pindex); 755c68c3537SAlan Cox do { 756c68c3537SAlan Cox rv = vm_reserv_from_page(m); 757c68c3537SAlan Cox KASSERT(rv->pages == m, 758c68c3537SAlan Cox ("vm_reserv_alloc_contig: reserv %p's pages is corrupted", 759c68c3537SAlan Cox rv)); 760e2068d0bSJeff Roberson vm_reserv_insert(rv, object, first); 761c68c3537SAlan Cox n = ulmin(VM_LEVEL_0_NPAGES - index, npages); 762c68c3537SAlan Cox for (i = 0; i < n; i++) 763ec179322SAlan Cox vm_reserv_populate(rv, index + i); 764c68c3537SAlan Cox npages -= n; 765c68c3537SAlan Cox if (m_ret == NULL) { 766c68c3537SAlan Cox m_ret = &rv->pages[index]; 767c68c3537SAlan Cox index = 0; 768c68c3537SAlan Cox } 769c68c3537SAlan Cox m += VM_LEVEL_0_NPAGES; 770c68c3537SAlan Cox first += VM_LEVEL_0_NPAGES; 771c68c3537SAlan Cox allocpages -= VM_LEVEL_0_NPAGES; 77264f096eeSAlan Cox } while (allocpages >= VM_LEVEL_0_NPAGES); 773c68c3537SAlan Cox return (m_ret); 774e2068d0bSJeff Roberson } 775c68c3537SAlan Cox 776c68c3537SAlan Cox /* 777e2068d0bSJeff Roberson * Attempts to extend an existing reservation and allocate the page to the 778e2068d0bSJeff Roberson * object. 779e2068d0bSJeff Roberson * 780e2068d0bSJeff Roberson * The page "mpred" must immediately precede the offset "pindex" within the 781e2068d0bSJeff Roberson * specified object. 782e2068d0bSJeff Roberson * 783e2068d0bSJeff Roberson * The object must be locked. 784c68c3537SAlan Cox */ 785e2068d0bSJeff Roberson vm_page_t 786e2068d0bSJeff Roberson vm_reserv_extend(int req, vm_object_t object, vm_pindex_t pindex, int domain, 787e2068d0bSJeff Roberson vm_page_t mpred) 788e2068d0bSJeff Roberson { 789e2068d0bSJeff Roberson struct vm_domain *vmd; 790e2068d0bSJeff Roberson vm_page_t m, msucc; 791e2068d0bSJeff Roberson vm_reserv_t rv; 792*30fbfddaSJeff Roberson int index; 793e2068d0bSJeff Roberson 794e2068d0bSJeff Roberson VM_OBJECT_ASSERT_WLOCKED(object); 795e2068d0bSJeff Roberson 796e2068d0bSJeff Roberson /* 797e2068d0bSJeff Roberson * Could a reservation currently exist? 798e2068d0bSJeff Roberson */ 799e2068d0bSJeff Roberson if (pindex < VM_RESERV_INDEX(object, pindex) || 800e2068d0bSJeff Roberson pindex >= object->size || object->resident_page_count == 0) 801e2068d0bSJeff Roberson return (NULL); 802e2068d0bSJeff Roberson 803e2068d0bSJeff Roberson /* 804e2068d0bSJeff Roberson * Look for an existing reservation. 805e2068d0bSJeff Roberson */ 806e2068d0bSJeff Roberson rv = vm_reserv_from_object(object, pindex, mpred, &msucc); 807e2068d0bSJeff Roberson if (rv == NULL) 808e2068d0bSJeff Roberson return (NULL); 809e2068d0bSJeff Roberson 810e2068d0bSJeff Roberson KASSERT(object != kernel_object || rv->domain == domain, 811e2068d0bSJeff Roberson ("vm_reserv_extend: Domain mismatch from reservation.")); 812e2068d0bSJeff Roberson domain = rv->domain; 813e2068d0bSJeff Roberson vmd = VM_DOMAIN(domain); 814c68c3537SAlan Cox index = VM_RESERV_INDEX(object, pindex); 815c68c3537SAlan Cox m = &rv->pages[index]; 816e2068d0bSJeff Roberson vm_domain_free_lock(vmd); 817e2068d0bSJeff Roberson if (vm_domain_available(vmd, req, 1) == 0 || 818e2068d0bSJeff Roberson /* Handle reclaim race. */ 819e2068d0bSJeff Roberson rv->object != object || 820c68c3537SAlan Cox /* Handle vm_page_rename(m, new_object, ...). */ 821e2068d0bSJeff Roberson popmap_is_set(rv->popmap, index)) 822e2068d0bSJeff Roberson m = NULL; 823e2068d0bSJeff Roberson if (m != NULL) { 824e2068d0bSJeff Roberson vm_reserv_populate(rv, index); 825*30fbfddaSJeff Roberson vm_domain_freecnt_dec(vmd, 1); 826*30fbfddaSJeff Roberson } 827e2068d0bSJeff Roberson vm_domain_free_unlock(vmd); 828e2068d0bSJeff Roberson 829c68c3537SAlan Cox return (m); 830c68c3537SAlan Cox } 831c68c3537SAlan Cox 832c68c3537SAlan Cox /* 8335f70fb14SMark Johnston * Attempts to allocate a new reservation for the object, and allocates a 8345f70fb14SMark Johnston * page from that reservation. Callers should first invoke vm_reserv_extend() 8355f70fb14SMark Johnston * to attempt an allocation from an existing reservation. 836f8a47341SAlan Cox * 837404eb1b3SAlan Cox * The page "mpred" must immediately precede the offset "pindex" within the 838404eb1b3SAlan Cox * specified object. 839404eb1b3SAlan Cox * 840f8a47341SAlan Cox * The object and free page queue must be locked. 841f8a47341SAlan Cox */ 842f8a47341SAlan Cox vm_page_t 843ef435ae7SJeff Roberson vm_reserv_alloc_page(vm_object_t object, vm_pindex_t pindex, int domain, 844ef435ae7SJeff Roberson vm_page_t mpred) 845f8a47341SAlan Cox { 846404eb1b3SAlan Cox vm_page_t m, msucc; 847f8a47341SAlan Cox vm_pindex_t first, leftcap, rightcap; 848f8a47341SAlan Cox vm_reserv_t rv; 849e2068d0bSJeff Roberson int index; 850f8a47341SAlan Cox 851e2068d0bSJeff Roberson vm_domain_free_assert_locked(VM_DOMAIN(domain)); 85289f6b863SAttilio Rao VM_OBJECT_ASSERT_WLOCKED(object); 853f8a47341SAlan Cox 854f8a47341SAlan Cox /* 855c68c3537SAlan Cox * Is a reservation fundamentally impossible? 856f8a47341SAlan Cox */ 857f8a47341SAlan Cox if (pindex < VM_RESERV_INDEX(object, pindex) || 858f8a47341SAlan Cox pindex >= object->size) 859f8a47341SAlan Cox return (NULL); 860f8a47341SAlan Cox 861f8a47341SAlan Cox /* 862e2068d0bSJeff Roberson * Callers should've extended an existing reservation prior to 863e2068d0bSJeff Roberson * calling this function. If a reservation exists it is 864e2068d0bSJeff Roberson * incompatible with the allocation. 865f8a47341SAlan Cox */ 866e2068d0bSJeff Roberson rv = vm_reserv_from_object(object, pindex, mpred, &msucc); 867e2068d0bSJeff Roberson if (rv != NULL) 868e2068d0bSJeff Roberson return (NULL); 869f8a47341SAlan Cox 870f8a47341SAlan Cox /* 871c68c3537SAlan Cox * Could a reservation fit between the first index to the left that 872c68c3537SAlan Cox * can be used and the first index to the right that cannot be used? 873e2068d0bSJeff Roberson * 874e2068d0bSJeff Roberson * We must synchronize with the reserv object lock to protect the 875e2068d0bSJeff Roberson * pindex/object of the resulting reservations against rename while 876e2068d0bSJeff Roberson * we are inspecting. 877f8a47341SAlan Cox */ 878c68c3537SAlan Cox first = pindex - VM_RESERV_INDEX(object, pindex); 879e2068d0bSJeff Roberson vm_reserv_object_lock(object); 880c68c3537SAlan Cox if (mpred != NULL) { 881c68c3537SAlan Cox if ((rv = vm_reserv_from_page(mpred))->object != object) 882f8a47341SAlan Cox leftcap = mpred->pindex + 1; 883f8a47341SAlan Cox else 884f8a47341SAlan Cox leftcap = rv->pindex + VM_LEVEL_0_NPAGES; 885e2068d0bSJeff Roberson if (leftcap > first) { 886e2068d0bSJeff Roberson vm_reserv_object_unlock(object); 887c68c3537SAlan Cox return (NULL); 888c68c3537SAlan Cox } 889e2068d0bSJeff Roberson } 890c68c3537SAlan Cox if (msucc != NULL) { 891c68c3537SAlan Cox if ((rv = vm_reserv_from_page(msucc))->object != object) 892f8a47341SAlan Cox rightcap = msucc->pindex; 893f8a47341SAlan Cox else 894f8a47341SAlan Cox rightcap = rv->pindex; 895e2068d0bSJeff Roberson if (first + VM_LEVEL_0_NPAGES > rightcap) { 896e2068d0bSJeff Roberson vm_reserv_object_unlock(object); 897f8a47341SAlan Cox return (NULL); 898c68c3537SAlan Cox } 899e2068d0bSJeff Roberson } 900e2068d0bSJeff Roberson vm_reserv_object_unlock(object); 901f8a47341SAlan Cox 902f8a47341SAlan Cox /* 903c68c3537SAlan Cox * Would a new reservation extend past the end of the object? 904f8a47341SAlan Cox */ 905c68c3537SAlan Cox if (first + VM_LEVEL_0_NPAGES > object->size) { 906f8a47341SAlan Cox /* 907f8a47341SAlan Cox * Don't allocate a new reservation if the object is a vnode or 908f8a47341SAlan Cox * backed by another object that is a vnode. 909f8a47341SAlan Cox */ 910f8a47341SAlan Cox if (object->type == OBJT_VNODE || 911f8a47341SAlan Cox (object->backing_object != NULL && 912f8a47341SAlan Cox object->backing_object->type == OBJT_VNODE)) 913f8a47341SAlan Cox return (NULL); 914f8a47341SAlan Cox /* Speculate that the object may grow. */ 915f8a47341SAlan Cox } 916f8a47341SAlan Cox 917f8a47341SAlan Cox /* 918c68c3537SAlan Cox * Allocate and populate the new reservation. 919f8a47341SAlan Cox */ 920ef435ae7SJeff Roberson m = vm_phys_alloc_pages(domain, VM_FREEPOOL_DEFAULT, VM_LEVEL_0_ORDER); 921c68c3537SAlan Cox if (m == NULL) 922c68c3537SAlan Cox return (NULL); 923f8a47341SAlan Cox rv = vm_reserv_from_page(m); 924f8a47341SAlan Cox KASSERT(rv->pages == m, 925c68c3537SAlan Cox ("vm_reserv_alloc_page: reserv %p's pages is corrupted", rv)); 926e2068d0bSJeff Roberson vm_reserv_insert(rv, object, first); 927ec179322SAlan Cox index = VM_RESERV_INDEX(object, pindex); 928ec179322SAlan Cox vm_reserv_populate(rv, index); 929ec179322SAlan Cox return (&rv->pages[index]); 930f8a47341SAlan Cox } 931f8a47341SAlan Cox 932f8a47341SAlan Cox /* 933ada27a3bSKonstantin Belousov * Breaks the given reservation. All free pages in the reservation 934ada27a3bSKonstantin Belousov * are returned to the physical memory allocator. The reservation's 935ada27a3bSKonstantin Belousov * population count and map are reset to their initial state. 936ec179322SAlan Cox * 9373453bca8SAlan Cox * The given reservation must not be in the partially populated reservation 938ec179322SAlan Cox * queue. The free page queue lock must be held. 939ec179322SAlan Cox */ 940ec179322SAlan Cox static void 941ada27a3bSKonstantin Belousov vm_reserv_break(vm_reserv_t rv) 942ec179322SAlan Cox { 943ec179322SAlan Cox int begin_zeroes, hi, i, lo; 944ec179322SAlan Cox 945e2068d0bSJeff Roberson vm_domain_free_assert_locked(VM_DOMAIN(rv->domain)); 946e2068d0bSJeff Roberson vm_reserv_remove(rv); 947c4be9169SKonstantin Belousov rv->pages->psind = 0; 948ec179322SAlan Cox i = hi = 0; 949ec179322SAlan Cox do { 950ec179322SAlan Cox /* Find the next 0 bit. Any previous 0 bits are < "hi". */ 951ec179322SAlan Cox lo = ffsl(~(((1UL << hi) - 1) | rv->popmap[i])); 952ec179322SAlan Cox if (lo == 0) { 953ec179322SAlan Cox /* Redundantly clears bits < "hi". */ 954ec179322SAlan Cox rv->popmap[i] = 0; 955ec179322SAlan Cox rv->popcnt -= NBPOPMAP - hi; 956ec179322SAlan Cox while (++i < NPOPMAP) { 957ec179322SAlan Cox lo = ffsl(~rv->popmap[i]); 958ec179322SAlan Cox if (lo == 0) { 959ec179322SAlan Cox rv->popmap[i] = 0; 960ec179322SAlan Cox rv->popcnt -= NBPOPMAP; 961ec179322SAlan Cox } else 962ec179322SAlan Cox break; 963ec179322SAlan Cox } 964ec179322SAlan Cox if (i == NPOPMAP) 965ec179322SAlan Cox break; 966ec179322SAlan Cox hi = 0; 967ec179322SAlan Cox } 968ec179322SAlan Cox KASSERT(lo > 0, ("vm_reserv_break: lo is %d", lo)); 969ec179322SAlan Cox /* Convert from ffsl() to ordinary bit numbering. */ 970ec179322SAlan Cox lo--; 971ec179322SAlan Cox if (lo > 0) { 972ec179322SAlan Cox /* Redundantly clears bits < "hi". */ 973ec179322SAlan Cox rv->popmap[i] &= ~((1UL << lo) - 1); 974ec179322SAlan Cox rv->popcnt -= lo - hi; 975ec179322SAlan Cox } 976ec179322SAlan Cox begin_zeroes = NBPOPMAP * i + lo; 977ec179322SAlan Cox /* Find the next 1 bit. */ 978ec179322SAlan Cox do 979ec179322SAlan Cox hi = ffsl(rv->popmap[i]); 980ec179322SAlan Cox while (hi == 0 && ++i < NPOPMAP); 981ec179322SAlan Cox if (i != NPOPMAP) 982ec179322SAlan Cox /* Convert from ffsl() to ordinary bit numbering. */ 983ec179322SAlan Cox hi--; 984ec179322SAlan Cox vm_phys_free_contig(&rv->pages[begin_zeroes], NBPOPMAP * i + 985ec179322SAlan Cox hi - begin_zeroes); 986ec179322SAlan Cox } while (i < NPOPMAP); 987ec179322SAlan Cox KASSERT(rv->popcnt == 0, 988ec179322SAlan Cox ("vm_reserv_break: reserv %p's popcnt is corrupted", rv)); 989ec179322SAlan Cox vm_reserv_broken++; 990ec179322SAlan Cox } 991ec179322SAlan Cox 992ec179322SAlan Cox /* 993f8a47341SAlan Cox * Breaks all reservations belonging to the given object. 994f8a47341SAlan Cox */ 995f8a47341SAlan Cox void 996f8a47341SAlan Cox vm_reserv_break_all(vm_object_t object) 997f8a47341SAlan Cox { 998f8a47341SAlan Cox vm_reserv_t rv; 999e2068d0bSJeff Roberson struct vm_domain *vmd; 1000f8a47341SAlan Cox 1001e2068d0bSJeff Roberson /* 1002e2068d0bSJeff Roberson * This access of object->rvq is unsynchronized so that the 1003e2068d0bSJeff Roberson * object rvq lock can nest after the domain_free lock. We 1004e2068d0bSJeff Roberson * must check for races in the results. However, the object 1005e2068d0bSJeff Roberson * lock prevents new additions, so we are guaranteed that when 1006e2068d0bSJeff Roberson * it returns NULL the object is properly empty. 1007e2068d0bSJeff Roberson */ 1008e2068d0bSJeff Roberson vmd = NULL; 1009f8a47341SAlan Cox while ((rv = LIST_FIRST(&object->rvq)) != NULL) { 1010e2068d0bSJeff Roberson if (vmd != VM_DOMAIN(rv->domain)) { 1011e2068d0bSJeff Roberson if (vmd != NULL) 1012e2068d0bSJeff Roberson vm_domain_free_unlock(vmd); 1013e2068d0bSJeff Roberson vmd = VM_DOMAIN(rv->domain); 1014e2068d0bSJeff Roberson vm_domain_free_lock(vmd); 1015e2068d0bSJeff Roberson } 1016e2068d0bSJeff Roberson /* Reclaim race. */ 1017e2068d0bSJeff Roberson if (rv->object != object) 1018e2068d0bSJeff Roberson continue; 1019f8a47341SAlan Cox KASSERT(rv->object == object, 1020f8a47341SAlan Cox ("vm_reserv_break_all: reserv %p is corrupted", rv)); 1021f8a47341SAlan Cox if (rv->inpartpopq) { 1022ef435ae7SJeff Roberson TAILQ_REMOVE(&vm_rvq_partpop[rv->domain], rv, partpopq); 1023f8a47341SAlan Cox rv->inpartpopq = FALSE; 1024f8a47341SAlan Cox } 1025ada27a3bSKonstantin Belousov vm_reserv_break(rv); 1026f8a47341SAlan Cox } 1027e2068d0bSJeff Roberson if (vmd != NULL) 1028e2068d0bSJeff Roberson vm_domain_free_unlock(vmd); 1029f8a47341SAlan Cox } 1030f8a47341SAlan Cox 1031f8a47341SAlan Cox /* 1032f8a47341SAlan Cox * Frees the given page if it belongs to a reservation. Returns TRUE if the 1033f8a47341SAlan Cox * page is freed and FALSE otherwise. 1034f8a47341SAlan Cox * 1035f8a47341SAlan Cox * The free page queue lock must be held. 1036f8a47341SAlan Cox */ 1037f8a47341SAlan Cox boolean_t 1038f8a47341SAlan Cox vm_reserv_free_page(vm_page_t m) 1039f8a47341SAlan Cox { 1040f8a47341SAlan Cox vm_reserv_t rv; 1041f8a47341SAlan Cox 1042f8a47341SAlan Cox rv = vm_reserv_from_page(m); 1043908e3da1SAlan Cox if (rv->object == NULL) 1044908e3da1SAlan Cox return (FALSE); 1045f4af5959SJeff Roberson vm_domain_free_assert_locked(VM_DOMAIN(rv->domain)); 1046ec179322SAlan Cox vm_reserv_depopulate(rv, m - rv->pages); 1047f8a47341SAlan Cox return (TRUE); 1048f8a47341SAlan Cox } 1049f8a47341SAlan Cox 1050f8a47341SAlan Cox /* 1051f8a47341SAlan Cox * Initializes the reservation management system. Specifically, initializes 1052f8a47341SAlan Cox * the reservation array. 1053f8a47341SAlan Cox * 1054f8a47341SAlan Cox * Requires that vm_page_array and first_page are initialized! 1055f8a47341SAlan Cox */ 1056f8a47341SAlan Cox void 1057f8a47341SAlan Cox vm_reserv_init(void) 1058f8a47341SAlan Cox { 1059f8a47341SAlan Cox vm_paddr_t paddr; 106009e5f3c4SAlan Cox struct vm_phys_seg *seg; 1061ef435ae7SJeff Roberson int i, segind; 1062f8a47341SAlan Cox 1063f8a47341SAlan Cox /* 1064f8a47341SAlan Cox * Initialize the reservation array. Specifically, initialize the 1065f8a47341SAlan Cox * "pages" field for every element that has an underlying superpage. 1066f8a47341SAlan Cox */ 106709e5f3c4SAlan Cox for (segind = 0; segind < vm_phys_nsegs; segind++) { 106809e5f3c4SAlan Cox seg = &vm_phys_segs[segind]; 106909e5f3c4SAlan Cox paddr = roundup2(seg->start, VM_LEVEL_0_SIZE); 107009e5f3c4SAlan Cox while (paddr + VM_LEVEL_0_SIZE <= seg->end) { 1071f8a47341SAlan Cox vm_reserv_array[paddr >> VM_LEVEL_0_SHIFT].pages = 1072f8a47341SAlan Cox PHYS_TO_VM_PAGE(paddr); 1073e2068d0bSJeff Roberson vm_reserv_array[paddr >> VM_LEVEL_0_SHIFT].domain = 1074e2068d0bSJeff Roberson seg->domain; 1075f8a47341SAlan Cox paddr += VM_LEVEL_0_SIZE; 1076f8a47341SAlan Cox } 1077f8a47341SAlan Cox } 1078ef435ae7SJeff Roberson for (i = 0; i < MAXMEMDOM; i++) 1079ef435ae7SJeff Roberson TAILQ_INIT(&vm_rvq_partpop[i]); 1080f8a47341SAlan Cox } 1081f8a47341SAlan Cox 1082f8a47341SAlan Cox /* 1083c869e672SAlan Cox * Returns true if the given page belongs to a reservation and that page is 1084c869e672SAlan Cox * free. Otherwise, returns false. 1085c869e672SAlan Cox */ 1086c869e672SAlan Cox bool 1087c869e672SAlan Cox vm_reserv_is_page_free(vm_page_t m) 1088c869e672SAlan Cox { 1089c869e672SAlan Cox vm_reserv_t rv; 1090c869e672SAlan Cox 1091c869e672SAlan Cox rv = vm_reserv_from_page(m); 1092c869e672SAlan Cox if (rv->object == NULL) 1093c869e672SAlan Cox return (false); 1094f4af5959SJeff Roberson vm_domain_free_assert_locked(VM_DOMAIN(rv->domain)); 1095c869e672SAlan Cox return (popmap_is_clear(rv->popmap, m - rv->pages)); 1096c869e672SAlan Cox } 1097c869e672SAlan Cox 1098c869e672SAlan Cox /* 1099c869e672SAlan Cox * If the given page belongs to a reservation, returns the level of that 1100c869e672SAlan Cox * reservation. Otherwise, returns -1. 1101c869e672SAlan Cox */ 1102c869e672SAlan Cox int 1103c869e672SAlan Cox vm_reserv_level(vm_page_t m) 1104c869e672SAlan Cox { 1105c869e672SAlan Cox vm_reserv_t rv; 1106c869e672SAlan Cox 1107c869e672SAlan Cox rv = vm_reserv_from_page(m); 1108c869e672SAlan Cox return (rv->object != NULL ? 0 : -1); 1109c869e672SAlan Cox } 1110c869e672SAlan Cox 1111c869e672SAlan Cox /* 11123453bca8SAlan Cox * Returns a reservation level if the given page belongs to a fully populated 1113f8a47341SAlan Cox * reservation and -1 otherwise. 1114f8a47341SAlan Cox */ 1115f8a47341SAlan Cox int 1116f8a47341SAlan Cox vm_reserv_level_iffullpop(vm_page_t m) 1117f8a47341SAlan Cox { 1118f8a47341SAlan Cox vm_reserv_t rv; 1119f8a47341SAlan Cox 1120f8a47341SAlan Cox rv = vm_reserv_from_page(m); 1121f8a47341SAlan Cox return (rv->popcnt == VM_LEVEL_0_NPAGES ? 0 : -1); 1122f8a47341SAlan Cox } 1123f8a47341SAlan Cox 1124f8a47341SAlan Cox /* 11253453bca8SAlan Cox * Breaks the given partially populated reservation, releasing its free pages 11263453bca8SAlan Cox * to the physical memory allocator. 1127f8a47341SAlan Cox * 1128f8a47341SAlan Cox * The free page queue lock must be held. 1129f8a47341SAlan Cox */ 113044aab2c3SAlan Cox static void 113144aab2c3SAlan Cox vm_reserv_reclaim(vm_reserv_t rv) 1132f8a47341SAlan Cox { 1133f8a47341SAlan Cox 1134e2068d0bSJeff Roberson vm_domain_free_assert_locked(VM_DOMAIN(rv->domain)); 1135f8a47341SAlan Cox KASSERT(rv->inpartpopq, 1136ec179322SAlan Cox ("vm_reserv_reclaim: reserv %p's inpartpopq is FALSE", rv)); 1137ef435ae7SJeff Roberson KASSERT(rv->domain >= 0 && rv->domain < vm_ndomains, 1138ef435ae7SJeff Roberson ("vm_reserv_reclaim: reserv %p's domain is corrupted %d", 1139ef435ae7SJeff Roberson rv, rv->domain)); 1140ef435ae7SJeff Roberson TAILQ_REMOVE(&vm_rvq_partpop[rv->domain], rv, partpopq); 1141f8a47341SAlan Cox rv->inpartpopq = FALSE; 1142ada27a3bSKonstantin Belousov vm_reserv_break(rv); 1143f8a47341SAlan Cox vm_reserv_reclaimed++; 114444aab2c3SAlan Cox } 114544aab2c3SAlan Cox 114644aab2c3SAlan Cox /* 11473453bca8SAlan Cox * Breaks the reservation at the head of the partially populated reservation 11483453bca8SAlan Cox * queue, releasing its free pages to the physical memory allocator. Returns 11493453bca8SAlan Cox * TRUE if a reservation is broken and FALSE otherwise. 115044aab2c3SAlan Cox * 115144aab2c3SAlan Cox * The free page queue lock must be held. 115244aab2c3SAlan Cox */ 115344aab2c3SAlan Cox boolean_t 1154ef435ae7SJeff Roberson vm_reserv_reclaim_inactive(int domain) 115544aab2c3SAlan Cox { 115644aab2c3SAlan Cox vm_reserv_t rv; 115744aab2c3SAlan Cox 1158e2068d0bSJeff Roberson vm_domain_free_assert_locked(VM_DOMAIN(domain)); 1159ef435ae7SJeff Roberson if ((rv = TAILQ_FIRST(&vm_rvq_partpop[domain])) != NULL) { 116044aab2c3SAlan Cox vm_reserv_reclaim(rv); 1161f8a47341SAlan Cox return (TRUE); 1162f8a47341SAlan Cox } 1163f8a47341SAlan Cox return (FALSE); 1164f8a47341SAlan Cox } 1165f8a47341SAlan Cox 1166f8a47341SAlan Cox /* 11673453bca8SAlan Cox * Searches the partially populated reservation queue for the least recently 11683453bca8SAlan Cox * changed reservation with free pages that satisfy the given request for 11693453bca8SAlan Cox * contiguous physical memory. If a satisfactory reservation is found, it is 11703453bca8SAlan Cox * broken. Returns TRUE if a reservation is broken and FALSE otherwise. 117144aab2c3SAlan Cox * 117244aab2c3SAlan Cox * The free page queue lock must be held. 117344aab2c3SAlan Cox */ 117444aab2c3SAlan Cox boolean_t 1175ef435ae7SJeff Roberson vm_reserv_reclaim_contig(int domain, u_long npages, vm_paddr_t low, 1176ef435ae7SJeff Roberson vm_paddr_t high, u_long alignment, vm_paddr_t boundary) 117744aab2c3SAlan Cox { 1178ec179322SAlan Cox vm_paddr_t pa, size; 117944aab2c3SAlan Cox vm_reserv_t rv; 118067b7e434SAlan Cox int hi, i, lo, low_index, next_free; 118144aab2c3SAlan Cox 1182e2068d0bSJeff Roberson vm_domain_free_assert_locked(VM_DOMAIN(domain)); 1183c68c3537SAlan Cox if (npages > VM_LEVEL_0_NPAGES - 1) 118444aab2c3SAlan Cox return (FALSE); 1185c68c3537SAlan Cox size = npages << PAGE_SHIFT; 1186ef435ae7SJeff Roberson TAILQ_FOREACH(rv, &vm_rvq_partpop[domain], partpopq) { 118744aab2c3SAlan Cox pa = VM_PAGE_TO_PHYS(&rv->pages[VM_LEVEL_0_NPAGES - 1]); 118844aab2c3SAlan Cox if (pa + PAGE_SIZE - size < low) { 1189ec179322SAlan Cox /* This entire reservation is too low; go to next. */ 119044aab2c3SAlan Cox continue; 119144aab2c3SAlan Cox } 1192ec179322SAlan Cox pa = VM_PAGE_TO_PHYS(&rv->pages[0]); 119344aab2c3SAlan Cox if (pa + size > high) { 1194ec179322SAlan Cox /* This entire reservation is too high; go to next. */ 1195ec179322SAlan Cox continue; 119685f2a0c9SMax Laier } 1197ec179322SAlan Cox if (pa < low) { 1198ec179322SAlan Cox /* Start the search for free pages at "low". */ 119967b7e434SAlan Cox low_index = (low + PAGE_MASK - pa) >> PAGE_SHIFT; 120067b7e434SAlan Cox i = low_index / NBPOPMAP; 120167b7e434SAlan Cox hi = low_index % NBPOPMAP; 1202ec179322SAlan Cox } else 1203ec179322SAlan Cox i = hi = 0; 1204ec179322SAlan Cox do { 1205ec179322SAlan Cox /* Find the next free page. */ 1206ec179322SAlan Cox lo = ffsl(~(((1UL << hi) - 1) | rv->popmap[i])); 1207ec179322SAlan Cox while (lo == 0 && ++i < NPOPMAP) 1208ec179322SAlan Cox lo = ffsl(~rv->popmap[i]); 1209ec179322SAlan Cox if (i == NPOPMAP) 1210ec179322SAlan Cox break; 1211ec179322SAlan Cox /* Convert from ffsl() to ordinary bit numbering. */ 1212ec179322SAlan Cox lo--; 1213ec179322SAlan Cox next_free = NBPOPMAP * i + lo; 1214ec179322SAlan Cox pa = VM_PAGE_TO_PHYS(&rv->pages[next_free]); 1215ec179322SAlan Cox KASSERT(pa >= low, 1216ec179322SAlan Cox ("vm_reserv_reclaim_contig: pa is too low")); 1217ec179322SAlan Cox if (pa + size > high) { 1218ec179322SAlan Cox /* The rest of this reservation is too high. */ 1219ec179322SAlan Cox break; 1220ec179322SAlan Cox } else if ((pa & (alignment - 1)) != 0 || 1221ec179322SAlan Cox ((pa ^ (pa + size - 1)) & ~(boundary - 1)) != 0) { 12226a93e36bSAlan Cox /* 12236a93e36bSAlan Cox * The current page doesn't meet the alignment 12246a93e36bSAlan Cox * and/or boundary requirements. Continue 12256a93e36bSAlan Cox * searching this reservation until the rest 12266a93e36bSAlan Cox * of its free pages are either excluded or 12276a93e36bSAlan Cox * exhausted. 12286a93e36bSAlan Cox */ 12296a93e36bSAlan Cox hi = lo + 1; 12306a93e36bSAlan Cox if (hi >= NBPOPMAP) { 12316a93e36bSAlan Cox hi = 0; 12326a93e36bSAlan Cox i++; 12336a93e36bSAlan Cox } 1234ec179322SAlan Cox continue; 1235ec179322SAlan Cox } 1236ec179322SAlan Cox /* Find the next used page. */ 1237ec179322SAlan Cox hi = ffsl(rv->popmap[i] & ~((1UL << lo) - 1)); 1238ec179322SAlan Cox while (hi == 0 && ++i < NPOPMAP) { 1239ec179322SAlan Cox if ((NBPOPMAP * i - next_free) * PAGE_SIZE >= 1240ec179322SAlan Cox size) { 124144aab2c3SAlan Cox vm_reserv_reclaim(rv); 124244aab2c3SAlan Cox return (TRUE); 124344aab2c3SAlan Cox } 1244ec179322SAlan Cox hi = ffsl(rv->popmap[i]); 1245ec179322SAlan Cox } 1246ec179322SAlan Cox /* Convert from ffsl() to ordinary bit numbering. */ 1247ec179322SAlan Cox if (i != NPOPMAP) 1248ec179322SAlan Cox hi--; 1249ec179322SAlan Cox if ((NBPOPMAP * i + hi - next_free) * PAGE_SIZE >= 1250ec179322SAlan Cox size) { 1251ec179322SAlan Cox vm_reserv_reclaim(rv); 1252ec179322SAlan Cox return (TRUE); 1253ec179322SAlan Cox } 1254ec179322SAlan Cox } while (i < NPOPMAP); 125544aab2c3SAlan Cox } 125644aab2c3SAlan Cox return (FALSE); 125744aab2c3SAlan Cox } 125844aab2c3SAlan Cox 125944aab2c3SAlan Cox /* 1260f8a47341SAlan Cox * Transfers the reservation underlying the given page to a new object. 1261f8a47341SAlan Cox * 1262f8a47341SAlan Cox * The object must be locked. 1263f8a47341SAlan Cox */ 1264f8a47341SAlan Cox void 1265f8a47341SAlan Cox vm_reserv_rename(vm_page_t m, vm_object_t new_object, vm_object_t old_object, 1266f8a47341SAlan Cox vm_pindex_t old_object_offset) 1267f8a47341SAlan Cox { 1268f8a47341SAlan Cox vm_reserv_t rv; 1269f8a47341SAlan Cox 127089f6b863SAttilio Rao VM_OBJECT_ASSERT_WLOCKED(new_object); 1271f8a47341SAlan Cox rv = vm_reserv_from_page(m); 1272f8a47341SAlan Cox if (rv->object == old_object) { 1273e2068d0bSJeff Roberson vm_domain_free_lock(VM_DOMAIN(rv->domain)); 1274f8a47341SAlan Cox if (rv->object == old_object) { 1275e2068d0bSJeff Roberson vm_reserv_object_lock(old_object); 1276e2068d0bSJeff Roberson rv->object = NULL; 1277f8a47341SAlan Cox LIST_REMOVE(rv, objq); 1278e2068d0bSJeff Roberson vm_reserv_object_unlock(old_object); 1279e2068d0bSJeff Roberson vm_reserv_object_lock(new_object); 1280f8a47341SAlan Cox rv->object = new_object; 1281f8a47341SAlan Cox rv->pindex -= old_object_offset; 1282e2068d0bSJeff Roberson LIST_INSERT_HEAD(&new_object->rvq, rv, objq); 1283e2068d0bSJeff Roberson vm_reserv_object_unlock(new_object); 1284f8a47341SAlan Cox } 1285e2068d0bSJeff Roberson vm_domain_free_unlock(VM_DOMAIN(rv->domain)); 1286f8a47341SAlan Cox } 1287f8a47341SAlan Cox } 1288f8a47341SAlan Cox 1289f8a47341SAlan Cox /* 1290c869e672SAlan Cox * Returns the size (in bytes) of a reservation of the specified level. 1291c869e672SAlan Cox */ 1292c869e672SAlan Cox int 1293c869e672SAlan Cox vm_reserv_size(int level) 1294c869e672SAlan Cox { 1295c869e672SAlan Cox 1296c869e672SAlan Cox switch (level) { 1297c869e672SAlan Cox case 0: 1298c869e672SAlan Cox return (VM_LEVEL_0_SIZE); 1299c869e672SAlan Cox case -1: 1300c869e672SAlan Cox return (PAGE_SIZE); 1301c869e672SAlan Cox default: 1302c869e672SAlan Cox return (0); 1303c869e672SAlan Cox } 1304c869e672SAlan Cox } 1305c869e672SAlan Cox 1306c869e672SAlan Cox /* 1307f8a47341SAlan Cox * Allocates the virtual and physical memory required by the reservation 1308f8a47341SAlan Cox * management system's data structures, in particular, the reservation array. 1309f8a47341SAlan Cox */ 1310f8a47341SAlan Cox vm_paddr_t 1311f8a47341SAlan Cox vm_reserv_startup(vm_offset_t *vaddr, vm_paddr_t end, vm_paddr_t high_water) 1312f8a47341SAlan Cox { 1313f8a47341SAlan Cox vm_paddr_t new_end; 1314f8a47341SAlan Cox size_t size; 1315e2068d0bSJeff Roberson int i; 1316f8a47341SAlan Cox 1317f8a47341SAlan Cox /* 1318f8a47341SAlan Cox * Calculate the size (in bytes) of the reservation array. Round up 1319f8a47341SAlan Cox * from "high_water" because every small page is mapped to an element 1320f8a47341SAlan Cox * in the reservation array based on its physical address. Thus, the 1321f8a47341SAlan Cox * number of elements in the reservation array can be greater than the 1322f8a47341SAlan Cox * number of superpages. 1323f8a47341SAlan Cox */ 1324f8a47341SAlan Cox size = howmany(high_water, VM_LEVEL_0_SIZE) * sizeof(struct vm_reserv); 1325f8a47341SAlan Cox 1326f8a47341SAlan Cox /* 1327f8a47341SAlan Cox * Allocate and map the physical memory for the reservation array. The 1328f8a47341SAlan Cox * next available virtual address is returned by reference. 1329f8a47341SAlan Cox */ 1330f8a47341SAlan Cox new_end = end - round_page(size); 1331f8a47341SAlan Cox vm_reserv_array = (void *)(uintptr_t)pmap_map(vaddr, new_end, end, 1332f8a47341SAlan Cox VM_PROT_READ | VM_PROT_WRITE); 1333f8a47341SAlan Cox bzero(vm_reserv_array, size); 1334f8a47341SAlan Cox 1335e2068d0bSJeff Roberson for (i = 0; i < VM_RESERV_OBJ_LOCK_COUNT; i++) 1336e2068d0bSJeff Roberson mtx_init(&vm_reserv_object_mtx[i], "resv obj lock", NULL, 1337e2068d0bSJeff Roberson MTX_DEF); 1338e2068d0bSJeff Roberson 1339f8a47341SAlan Cox /* 1340f8a47341SAlan Cox * Return the next available physical address. 1341f8a47341SAlan Cox */ 1342f8a47341SAlan Cox return (new_end); 1343f8a47341SAlan Cox } 1344f8a47341SAlan Cox 13458b5e1472SAlan Cox /* 13468b5e1472SAlan Cox * Returns the superpage containing the given page. 13478b5e1472SAlan Cox */ 13488b5e1472SAlan Cox vm_page_t 13498b5e1472SAlan Cox vm_reserv_to_superpage(vm_page_t m) 13508b5e1472SAlan Cox { 13518b5e1472SAlan Cox vm_reserv_t rv; 13528b5e1472SAlan Cox 13538b5e1472SAlan Cox VM_OBJECT_ASSERT_LOCKED(m->object); 13548b5e1472SAlan Cox rv = vm_reserv_from_page(m); 13558b5e1472SAlan Cox return (rv->object == m->object && rv->popcnt == VM_LEVEL_0_NPAGES ? 13568b5e1472SAlan Cox rv->pages : NULL); 13578b5e1472SAlan Cox } 13588b5e1472SAlan Cox 1359f8a47341SAlan Cox #endif /* VM_NRESERVLEVEL > 0 */ 1360