1*e2068d0bSJeff Roberson /*- 2*e2068d0bSJeff Roberson * SPDX-License-Identifier: (BSD-3-Clause AND MIT-CMU) 3*e2068d0bSJeff Roberson * 4*e2068d0bSJeff Roberson * Copyright (c) 1991, 1993 5*e2068d0bSJeff Roberson * The Regents of the University of California. All rights reserved. 6*e2068d0bSJeff Roberson * 7*e2068d0bSJeff Roberson * This code is derived from software contributed to Berkeley by 8*e2068d0bSJeff Roberson * The Mach Operating System project at Carnegie-Mellon University. 9*e2068d0bSJeff Roberson * 10*e2068d0bSJeff Roberson * Redistribution and use in source and binary forms, with or without 11*e2068d0bSJeff Roberson * modification, are permitted provided that the following conditions 12*e2068d0bSJeff Roberson * are met: 13*e2068d0bSJeff Roberson * 1. Redistributions of source code must retain the above copyright 14*e2068d0bSJeff Roberson * notice, this list of conditions and the following disclaimer. 15*e2068d0bSJeff Roberson * 2. Redistributions in binary form must reproduce the above copyright 16*e2068d0bSJeff Roberson * notice, this list of conditions and the following disclaimer in the 17*e2068d0bSJeff Roberson * documentation and/or other materials provided with the distribution. 18*e2068d0bSJeff Roberson * 3. Neither the name of the University nor the names of its contributors 19*e2068d0bSJeff Roberson * may be used to endorse or promote products derived from this software 20*e2068d0bSJeff Roberson * without specific prior written permission. 21*e2068d0bSJeff Roberson * 22*e2068d0bSJeff Roberson * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 23*e2068d0bSJeff Roberson * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24*e2068d0bSJeff Roberson * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25*e2068d0bSJeff Roberson * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 26*e2068d0bSJeff Roberson * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27*e2068d0bSJeff Roberson * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28*e2068d0bSJeff Roberson * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29*e2068d0bSJeff Roberson * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30*e2068d0bSJeff Roberson * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31*e2068d0bSJeff Roberson * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32*e2068d0bSJeff Roberson * SUCH DAMAGE. 33*e2068d0bSJeff Roberson * 34*e2068d0bSJeff Roberson * from: @(#)vm_page.h 8.2 (Berkeley) 12/13/93 35*e2068d0bSJeff Roberson * 36*e2068d0bSJeff Roberson * 37*e2068d0bSJeff Roberson * Copyright (c) 1987, 1990 Carnegie-Mellon University. 38*e2068d0bSJeff Roberson * All rights reserved. 39*e2068d0bSJeff Roberson * 40*e2068d0bSJeff Roberson * Authors: Avadis Tevanian, Jr., Michael Wayne Young 41*e2068d0bSJeff Roberson * 42*e2068d0bSJeff Roberson * Permission to use, copy, modify and distribute this software and 43*e2068d0bSJeff Roberson * its documentation is hereby granted, provided that both the copyright 44*e2068d0bSJeff Roberson * notice and this permission notice appear in all copies of the 45*e2068d0bSJeff Roberson * software, derivative works or modified versions, and any portions 46*e2068d0bSJeff Roberson * thereof, and that both notices appear in supporting documentation. 47*e2068d0bSJeff Roberson * 48*e2068d0bSJeff Roberson * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 49*e2068d0bSJeff Roberson * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 50*e2068d0bSJeff Roberson * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 51*e2068d0bSJeff Roberson * 52*e2068d0bSJeff Roberson * Carnegie Mellon requests users of this software to return to 53*e2068d0bSJeff Roberson * 54*e2068d0bSJeff Roberson * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 55*e2068d0bSJeff Roberson * School of Computer Science 56*e2068d0bSJeff Roberson * Carnegie Mellon University 57*e2068d0bSJeff Roberson * Pittsburgh PA 15213-3890 58*e2068d0bSJeff Roberson * 59*e2068d0bSJeff Roberson * any improvements or extensions that they make and grant Carnegie the 60*e2068d0bSJeff Roberson * rights to redistribute these changes. 61*e2068d0bSJeff Roberson * 62*e2068d0bSJeff Roberson * $FreeBSD$ 63*e2068d0bSJeff Roberson */ 64*e2068d0bSJeff Roberson 65*e2068d0bSJeff Roberson #ifndef _VM_PAGEQUEUE_ 66*e2068d0bSJeff Roberson #define _VM_PAGEQUEUE_ 67*e2068d0bSJeff Roberson 68*e2068d0bSJeff Roberson #ifdef _KERNEL 69*e2068d0bSJeff Roberson struct vm_pagequeue { 70*e2068d0bSJeff Roberson struct mtx pq_mutex; 71*e2068d0bSJeff Roberson struct pglist pq_pl; 72*e2068d0bSJeff Roberson int pq_cnt; 73*e2068d0bSJeff Roberson const char * const pq_name; 74*e2068d0bSJeff Roberson } __aligned(CACHE_LINE_SIZE); 75*e2068d0bSJeff Roberson 76*e2068d0bSJeff Roberson 77*e2068d0bSJeff Roberson struct vm_domain { 78*e2068d0bSJeff Roberson struct vm_pagequeue vmd_pagequeues[PQ_COUNT]; 79*e2068d0bSJeff Roberson struct mtx_padalign vmd_free_mtx; 80*e2068d0bSJeff Roberson struct vmem *vmd_kernel_arena; 81*e2068d0bSJeff Roberson u_int vmd_domain; /* Domain number. */ 82*e2068d0bSJeff Roberson u_int vmd_page_count; 83*e2068d0bSJeff Roberson long vmd_segs; /* bitmask of the segments */ 84*e2068d0bSJeff Roberson 85*e2068d0bSJeff Roberson /* Paging control variables, locked by domain_free_mtx. */ 86*e2068d0bSJeff Roberson u_int vmd_free_count; 87*e2068d0bSJeff Roberson boolean_t vmd_oom; 88*e2068d0bSJeff Roberson int vmd_oom_seq; 89*e2068d0bSJeff Roberson int vmd_last_active_scan; 90*e2068d0bSJeff Roberson struct vm_page vmd_laundry_marker; 91*e2068d0bSJeff Roberson struct vm_page vmd_marker; /* marker for pagedaemon private use */ 92*e2068d0bSJeff Roberson struct vm_page vmd_inacthead; /* marker for LRU-defeating insertions */ 93*e2068d0bSJeff Roberson 94*e2068d0bSJeff Roberson int vmd_pageout_pages_needed; /* page daemon waiting for pages? */ 95*e2068d0bSJeff Roberson int vmd_pageout_deficit; /* Estimated number of pages deficit */ 96*e2068d0bSJeff Roberson int vmd_waiters; /* Pageout waiters. */ 97*e2068d0bSJeff Roberson bool vmd_pages_needed; /* Are threads waiting for free pages? */ 98*e2068d0bSJeff Roberson bool vmd_pageout_wanted; /* pageout daemon wait channel */ 99*e2068d0bSJeff Roberson bool vmd_minset; /* Are we in vm_min_domains? */ 100*e2068d0bSJeff Roberson bool vmd_severeset; /* Are we in vm_severe_domains? */ 101*e2068d0bSJeff Roberson int vmd_inactq_scans; 102*e2068d0bSJeff Roberson enum { 103*e2068d0bSJeff Roberson VM_LAUNDRY_IDLE = 0, 104*e2068d0bSJeff Roberson VM_LAUNDRY_BACKGROUND, 105*e2068d0bSJeff Roberson VM_LAUNDRY_SHORTFALL 106*e2068d0bSJeff Roberson } vmd_laundry_request; 107*e2068d0bSJeff Roberson 108*e2068d0bSJeff Roberson /* Paging thresholds. */ 109*e2068d0bSJeff Roberson u_int vmd_background_launder_target; 110*e2068d0bSJeff Roberson u_int vmd_free_reserved; /* (c) pages reserved for deadlock */ 111*e2068d0bSJeff Roberson u_int vmd_free_target; /* (c) pages desired free */ 112*e2068d0bSJeff Roberson u_int vmd_free_min; /* (c) pages desired free */ 113*e2068d0bSJeff Roberson u_int vmd_inactive_target; /* (c) pages desired inactive */ 114*e2068d0bSJeff Roberson u_int vmd_pageout_free_min; /* (c) min pages reserved for kernel */ 115*e2068d0bSJeff Roberson u_int vmd_pageout_wakeup_thresh;/* (c) min pages to wake pagedaemon */ 116*e2068d0bSJeff Roberson u_int vmd_interrupt_free_min; /* (c) reserved pages for int code */ 117*e2068d0bSJeff Roberson u_int vmd_free_severe; /* (c) severe page depletion point */ 118*e2068d0bSJeff Roberson } __aligned(CACHE_LINE_SIZE); 119*e2068d0bSJeff Roberson 120*e2068d0bSJeff Roberson extern struct vm_domain vm_dom[MAXMEMDOM]; 121*e2068d0bSJeff Roberson 122*e2068d0bSJeff Roberson #define VM_DOMAIN(n) (&vm_dom[(n)]) 123*e2068d0bSJeff Roberson 124*e2068d0bSJeff Roberson #define vm_pagequeue_assert_locked(pq) mtx_assert(&(pq)->pq_mutex, MA_OWNED) 125*e2068d0bSJeff Roberson #define vm_pagequeue_lock(pq) mtx_lock(&(pq)->pq_mutex) 126*e2068d0bSJeff Roberson #define vm_pagequeue_lockptr(pq) (&(pq)->pq_mutex) 127*e2068d0bSJeff Roberson #define vm_pagequeue_unlock(pq) mtx_unlock(&(pq)->pq_mutex) 128*e2068d0bSJeff Roberson 129*e2068d0bSJeff Roberson #define vm_domain_free_assert_locked(n) \ 130*e2068d0bSJeff Roberson mtx_assert(vm_domain_free_lockptr((n)), MA_OWNED) 131*e2068d0bSJeff Roberson #define vm_domain_free_assert_unlocked(n) \ 132*e2068d0bSJeff Roberson mtx_assert(vm_domain_free_lockptr((n)), MA_NOTOWNED) 133*e2068d0bSJeff Roberson #define vm_domain_free_lock(d) \ 134*e2068d0bSJeff Roberson mtx_lock(vm_domain_free_lockptr((d))) 135*e2068d0bSJeff Roberson #define vm_domain_free_lockptr(d) \ 136*e2068d0bSJeff Roberson (&(d)->vmd_free_mtx) 137*e2068d0bSJeff Roberson #define vm_domain_free_unlock(d) \ 138*e2068d0bSJeff Roberson mtx_unlock(vm_domain_free_lockptr((d))) 139*e2068d0bSJeff Roberson 140*e2068d0bSJeff Roberson static __inline void 141*e2068d0bSJeff Roberson vm_pagequeue_cnt_add(struct vm_pagequeue *pq, int addend) 142*e2068d0bSJeff Roberson { 143*e2068d0bSJeff Roberson 144*e2068d0bSJeff Roberson #ifdef notyet 145*e2068d0bSJeff Roberson vm_pagequeue_assert_locked(pq); 146*e2068d0bSJeff Roberson #endif 147*e2068d0bSJeff Roberson pq->pq_cnt += addend; 148*e2068d0bSJeff Roberson } 149*e2068d0bSJeff Roberson #define vm_pagequeue_cnt_inc(pq) vm_pagequeue_cnt_add((pq), 1) 150*e2068d0bSJeff Roberson #define vm_pagequeue_cnt_dec(pq) vm_pagequeue_cnt_add((pq), -1) 151*e2068d0bSJeff Roberson 152*e2068d0bSJeff Roberson void vm_domain_set(struct vm_domain *vmd); 153*e2068d0bSJeff Roberson int vm_domain_available(struct vm_domain *vmd, int req, int npages); 154*e2068d0bSJeff Roberson 155*e2068d0bSJeff Roberson /* 156*e2068d0bSJeff Roberson * vm_pagequeue_domain: 157*e2068d0bSJeff Roberson * 158*e2068d0bSJeff Roberson * Return the memory domain the page belongs to. 159*e2068d0bSJeff Roberson */ 160*e2068d0bSJeff Roberson static inline struct vm_domain * 161*e2068d0bSJeff Roberson vm_pagequeue_domain(vm_page_t m) 162*e2068d0bSJeff Roberson { 163*e2068d0bSJeff Roberson 164*e2068d0bSJeff Roberson return (VM_DOMAIN(vm_phys_domain(m))); 165*e2068d0bSJeff Roberson } 166*e2068d0bSJeff Roberson 167*e2068d0bSJeff Roberson /* 168*e2068d0bSJeff Roberson * Return the number of pages we need to free-up or cache 169*e2068d0bSJeff Roberson * A positive number indicates that we do not have enough free pages. 170*e2068d0bSJeff Roberson */ 171*e2068d0bSJeff Roberson static inline int 172*e2068d0bSJeff Roberson vm_paging_target(struct vm_domain *vmd) 173*e2068d0bSJeff Roberson { 174*e2068d0bSJeff Roberson 175*e2068d0bSJeff Roberson return (vmd->vmd_free_target - vmd->vmd_free_count); 176*e2068d0bSJeff Roberson } 177*e2068d0bSJeff Roberson 178*e2068d0bSJeff Roberson /* 179*e2068d0bSJeff Roberson * Returns TRUE if the pagedaemon needs to be woken up. 180*e2068d0bSJeff Roberson */ 181*e2068d0bSJeff Roberson static inline int 182*e2068d0bSJeff Roberson vm_paging_needed(struct vm_domain *vmd, u_int free_count) 183*e2068d0bSJeff Roberson { 184*e2068d0bSJeff Roberson 185*e2068d0bSJeff Roberson return (free_count < vmd->vmd_pageout_wakeup_thresh); 186*e2068d0bSJeff Roberson } 187*e2068d0bSJeff Roberson 188*e2068d0bSJeff Roberson /* 189*e2068d0bSJeff Roberson * Returns TRUE if the domain is below the min paging target. 190*e2068d0bSJeff Roberson */ 191*e2068d0bSJeff Roberson static inline int 192*e2068d0bSJeff Roberson vm_paging_min(struct vm_domain *vmd) 193*e2068d0bSJeff Roberson { 194*e2068d0bSJeff Roberson 195*e2068d0bSJeff Roberson return (vmd->vmd_free_min > vmd->vmd_free_count); 196*e2068d0bSJeff Roberson } 197*e2068d0bSJeff Roberson 198*e2068d0bSJeff Roberson /* 199*e2068d0bSJeff Roberson * Returns TRUE if the domain is below the severe paging target. 200*e2068d0bSJeff Roberson */ 201*e2068d0bSJeff Roberson static inline int 202*e2068d0bSJeff Roberson vm_paging_severe(struct vm_domain *vmd) 203*e2068d0bSJeff Roberson { 204*e2068d0bSJeff Roberson 205*e2068d0bSJeff Roberson return (vmd->vmd_free_severe > vmd->vmd_free_count); 206*e2068d0bSJeff Roberson } 207*e2068d0bSJeff Roberson 208*e2068d0bSJeff Roberson /* 209*e2068d0bSJeff Roberson * Return the number of pages we need to launder. 210*e2068d0bSJeff Roberson * A positive number indicates that we have a shortfall of clean pages. 211*e2068d0bSJeff Roberson */ 212*e2068d0bSJeff Roberson static inline int 213*e2068d0bSJeff Roberson vm_laundry_target(struct vm_domain *vmd) 214*e2068d0bSJeff Roberson { 215*e2068d0bSJeff Roberson 216*e2068d0bSJeff Roberson return (vm_paging_target(vmd)); 217*e2068d0bSJeff Roberson } 218*e2068d0bSJeff Roberson 219*e2068d0bSJeff Roberson static inline u_int 220*e2068d0bSJeff Roberson vm_domain_freecnt_adj(struct vm_domain *vmd, int adj) 221*e2068d0bSJeff Roberson { 222*e2068d0bSJeff Roberson u_int ret; 223*e2068d0bSJeff Roberson 224*e2068d0bSJeff Roberson vm_domain_free_assert_locked(vmd); 225*e2068d0bSJeff Roberson ret = vmd->vmd_free_count += adj; 226*e2068d0bSJeff Roberson if ((!vmd->vmd_minset && vm_paging_min(vmd)) || 227*e2068d0bSJeff Roberson (!vmd->vmd_severeset && vm_paging_severe(vmd))) 228*e2068d0bSJeff Roberson vm_domain_set(vmd); 229*e2068d0bSJeff Roberson 230*e2068d0bSJeff Roberson return (ret); 231*e2068d0bSJeff Roberson } 232*e2068d0bSJeff Roberson 233*e2068d0bSJeff Roberson 234*e2068d0bSJeff Roberson #endif /* _KERNEL */ 235*e2068d0bSJeff Roberson #endif /* !_VM_PAGEQUEUE_ */ 236