1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2000 Peter Wemm 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/param.h> 29 #include <sys/systm.h> 30 #include <sys/conf.h> 31 #include <sys/kernel.h> 32 #include <sys/lock.h> 33 #include <sys/proc.h> 34 #include <sys/mutex.h> 35 #include <sys/mman.h> 36 #include <sys/rwlock.h> 37 #include <sys/sysctl.h> 38 #include <sys/user.h> 39 40 #include <vm/vm.h> 41 #include <vm/vm_param.h> 42 #include <vm/vm_object.h> 43 #include <vm/vm_page.h> 44 #include <vm/vm_pageout.h> 45 #include <vm/vm_pager.h> 46 47 /* list of phys pager objects */ 48 static struct pagerlst phys_pager_object_list; 49 /* protect access to phys_pager_object_list */ 50 static struct mtx phys_pager_mtx; 51 52 static int default_phys_pager_getpages(vm_object_t object, vm_page_t *m, 53 int count, int *rbehind, int *rahead); 54 static int default_phys_pager_populate(vm_object_t object, vm_pindex_t pidx, 55 int fault_type, vm_prot_t max_prot, vm_pindex_t *first, vm_pindex_t *last); 56 static boolean_t default_phys_pager_haspage(vm_object_t object, 57 vm_pindex_t pindex, int *before, int *after); 58 const struct phys_pager_ops default_phys_pg_ops = { 59 .phys_pg_getpages = default_phys_pager_getpages, 60 .phys_pg_populate = default_phys_pager_populate, 61 .phys_pg_haspage = default_phys_pager_haspage, 62 .phys_pg_ctor = NULL, 63 .phys_pg_dtor = NULL, 64 }; 65 66 static void 67 phys_pager_init(void) 68 { 69 70 TAILQ_INIT(&phys_pager_object_list); 71 mtx_init(&phys_pager_mtx, "phys_pager list", NULL, MTX_DEF); 72 } 73 74 vm_object_t 75 phys_pager_allocate(void *handle, const struct phys_pager_ops *ops, void *data, 76 vm_ooffset_t size, vm_prot_t prot, vm_ooffset_t foff, struct ucred *cred) 77 { 78 vm_object_t object, object1; 79 vm_pindex_t pindex; 80 bool init; 81 82 /* 83 * Offset should be page aligned. 84 */ 85 if (foff & PAGE_MASK) 86 return (NULL); 87 88 pindex = OFF_TO_IDX(foff + PAGE_MASK + size); 89 init = true; 90 91 if (handle != NULL) { 92 mtx_lock(&phys_pager_mtx); 93 /* 94 * Look up pager, creating as necessary. 95 */ 96 object1 = NULL; 97 object = vm_pager_object_lookup(&phys_pager_object_list, handle); 98 if (object == NULL) { 99 /* 100 * Allocate object and associate it with the pager. 101 */ 102 mtx_unlock(&phys_pager_mtx); 103 object1 = vm_object_allocate(OBJT_PHYS, pindex); 104 mtx_lock(&phys_pager_mtx); 105 object = vm_pager_object_lookup(&phys_pager_object_list, 106 handle); 107 if (object != NULL) { 108 /* 109 * We raced with other thread while 110 * allocating object. 111 */ 112 if (pindex > object->size) 113 object->size = pindex; 114 init = false; 115 } else { 116 object = object1; 117 object1 = NULL; 118 object->handle = handle; 119 object->un_pager.phys.ops = ops; 120 object->un_pager.phys.data_ptr = data; 121 if (ops->phys_pg_populate != NULL) 122 vm_object_set_flag(object, OBJ_POPULATE); 123 TAILQ_INSERT_TAIL(&phys_pager_object_list, 124 object, pager_object_list); 125 } 126 } else { 127 if (pindex > object->size) 128 object->size = pindex; 129 } 130 mtx_unlock(&phys_pager_mtx); 131 vm_object_deallocate(object1); 132 } else { 133 object = vm_object_allocate(OBJT_PHYS, pindex); 134 object->un_pager.phys.ops = ops; 135 object->un_pager.phys.data_ptr = data; 136 if (ops->phys_pg_populate != NULL) 137 vm_object_set_flag(object, OBJ_POPULATE); 138 } 139 if (init && ops->phys_pg_ctor != NULL) 140 ops->phys_pg_ctor(object, prot, foff, cred); 141 142 return (object); 143 } 144 145 static vm_object_t 146 phys_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot, 147 vm_ooffset_t foff, struct ucred *ucred) 148 { 149 return (phys_pager_allocate(handle, &default_phys_pg_ops, NULL, 150 size, prot, foff, ucred)); 151 } 152 153 static void 154 phys_pager_dealloc(vm_object_t object) 155 { 156 157 if (object->handle != NULL) { 158 VM_OBJECT_WUNLOCK(object); 159 mtx_lock(&phys_pager_mtx); 160 TAILQ_REMOVE(&phys_pager_object_list, object, pager_object_list); 161 mtx_unlock(&phys_pager_mtx); 162 VM_OBJECT_WLOCK(object); 163 } 164 object->type = OBJT_DEAD; 165 if (object->un_pager.phys.ops->phys_pg_dtor != NULL) 166 object->un_pager.phys.ops->phys_pg_dtor(object); 167 object->handle = NULL; 168 } 169 170 /* 171 * Fill as many pages as vm_fault has allocated for us. 172 */ 173 static int 174 default_phys_pager_getpages(vm_object_t object, vm_page_t *m, int count, 175 int *rbehind, int *rahead) 176 { 177 int i; 178 179 for (i = 0; i < count; i++) { 180 if (vm_page_none_valid(m[i])) { 181 if ((m[i]->flags & PG_ZERO) == 0) 182 pmap_zero_page(m[i]); 183 vm_page_valid(m[i]); 184 } 185 KASSERT(vm_page_all_valid(m[i]), 186 ("phys_pager_getpages: partially valid page %p", m[i])); 187 KASSERT(m[i]->dirty == 0, 188 ("phys_pager_getpages: dirty page %p", m[i])); 189 } 190 if (rbehind) 191 *rbehind = 0; 192 if (rahead) 193 *rahead = 0; 194 return (VM_PAGER_OK); 195 } 196 197 static int 198 phys_pager_getpages(vm_object_t object, vm_page_t *m, int count, int *rbehind, 199 int *rahead) 200 { 201 return (object->un_pager.phys.ops->phys_pg_getpages(object, m, 202 count, rbehind, rahead)); 203 } 204 205 /* 206 * Implement a pretty aggressive clustered getpages strategy. Hint that 207 * everything in an entire 4MB window should be prefaulted at once. 208 * 209 * 4MB (1024 slots per page table page) is convenient for x86, 210 * but may not be for other arches. 211 */ 212 #ifndef PHYSCLUSTER 213 #define PHYSCLUSTER 1024 214 #endif 215 static int phys_pager_cluster = PHYSCLUSTER; 216 SYSCTL_INT(_vm, OID_AUTO, phys_pager_cluster, CTLFLAG_RWTUN, 217 &phys_pager_cluster, 0, 218 "prefault window size for phys pager"); 219 220 /* 221 * Max hint to vm_page_alloc() about the further allocation needs 222 * inside the phys_pager_populate() loop. The number of bits used to 223 * implement VM_ALLOC_COUNT() determines the hard limit on this value. 224 * That limit is currently 65535. 225 */ 226 #define PHYSALLOC 16 227 228 static int 229 default_phys_pager_populate(vm_object_t object, vm_pindex_t pidx, 230 int fault_type __unused, vm_prot_t max_prot __unused, vm_pindex_t *first, 231 vm_pindex_t *last) 232 { 233 vm_page_t m; 234 vm_pindex_t base, end, i; 235 int ahead; 236 237 base = rounddown(pidx, phys_pager_cluster); 238 end = base + phys_pager_cluster - 1; 239 if (end >= object->size) 240 end = object->size - 1; 241 if (*first > base) 242 base = *first; 243 if (end > *last) 244 end = *last; 245 *first = base; 246 *last = end; 247 248 for (i = base; i <= end; i++) { 249 ahead = MIN(end - i, PHYSALLOC); 250 m = vm_page_grab(object, i, 251 VM_ALLOC_NORMAL | VM_ALLOC_COUNT(ahead)); 252 if (!vm_page_all_valid(m)) 253 vm_page_zero_invalid(m, TRUE); 254 KASSERT(m->dirty == 0, 255 ("phys_pager_populate: dirty page %p", m)); 256 } 257 return (VM_PAGER_OK); 258 } 259 260 static int 261 phys_pager_populate(vm_object_t object, vm_pindex_t pidx, int fault_type, 262 vm_prot_t max_prot, vm_pindex_t *first, vm_pindex_t *last) 263 { 264 return (object->un_pager.phys.ops->phys_pg_populate(object, pidx, 265 fault_type, max_prot, first, last)); 266 } 267 268 static void 269 phys_pager_putpages(vm_object_t object, vm_page_t *m, int count, int flags, 270 int *rtvals) 271 { 272 273 panic("phys_pager_putpage called"); 274 } 275 276 static boolean_t 277 default_phys_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, 278 int *after) 279 { 280 vm_pindex_t base, end; 281 282 base = rounddown(pindex, phys_pager_cluster); 283 end = base + phys_pager_cluster - 1; 284 if (before != NULL) 285 *before = pindex - base; 286 if (after != NULL) 287 *after = end - pindex; 288 return (TRUE); 289 } 290 291 static boolean_t 292 phys_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, 293 int *after) 294 { 295 return (object->un_pager.phys.ops->phys_pg_haspage(object, pindex, 296 before, after)); 297 } 298 299 const struct pagerops physpagerops = { 300 .pgo_kvme_type = KVME_TYPE_PHYS, 301 .pgo_init = phys_pager_init, 302 .pgo_alloc = phys_pager_alloc, 303 .pgo_dealloc = phys_pager_dealloc, 304 .pgo_getpages = phys_pager_getpages, 305 .pgo_putpages = phys_pager_putpages, 306 .pgo_haspage = phys_pager_haspage, 307 .pgo_populate = phys_pager_populate, 308 }; 309