1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2011 NetApp, Inc. 5 * All rights reserved. 6 */ 7 8 #include <sys/types.h> 9 #include <sys/lock.h> 10 #include <sys/sx.h> 11 #include <sys/systm.h> 12 13 #include <machine/vmm.h> 14 15 #include <vm/vm.h> 16 #include <vm/vm_param.h> 17 #include <vm/vm_extern.h> 18 #include <vm/pmap.h> 19 #include <vm/vm_map.h> 20 #include <vm/vm_object.h> 21 #include <vm/vm_page.h> 22 23 #include <dev/vmm/vmm_dev.h> 24 #include <dev/vmm/vmm_mem.h> 25 26 static void vm_free_memmap(struct vm *vm, int ident); 27 28 void 29 vm_mem_init(struct vm_mem *mem) 30 { 31 sx_init(&mem->mem_segs_lock, "vm_mem_segs"); 32 } 33 34 static bool 35 sysmem_mapping(struct vm_mem *mem, int idx) 36 { 37 if (mem->mem_maps[idx].len != 0 && 38 mem->mem_segs[mem->mem_maps[idx].segid].sysmem) 39 return (true); 40 else 41 return (false); 42 } 43 44 bool 45 vm_memseg_sysmem(struct vm *vm, int ident) 46 { 47 struct vm_mem *mem; 48 49 mem = vm_mem(vm); 50 vm_assert_memseg_locked(vm); 51 52 if (ident < 0 || ident >= VM_MAX_MEMSEGS) 53 return (false); 54 55 return (mem->mem_segs[ident].sysmem); 56 } 57 58 void 59 vm_mem_cleanup(struct vm *vm) 60 { 61 struct vm_mem *mem; 62 63 mem = vm_mem(vm); 64 65 /* 66 * System memory is removed from the guest address space only when 67 * the VM is destroyed. This is because the mapping remains the same 68 * across VM reset. 69 * 70 * Device memory can be relocated by the guest (e.g. using PCI BARs) 71 * so those mappings are removed on a VM reset. 72 */ 73 for (int i = 0; i < VM_MAX_MEMMAPS; i++) { 74 if (!sysmem_mapping(mem, i)) 75 vm_free_memmap(vm, i); 76 } 77 } 78 79 void 80 vm_mem_destroy(struct vm *vm) 81 { 82 struct vm_mem *mem; 83 84 mem = vm_mem(vm); 85 vm_assert_memseg_xlocked(vm); 86 87 for (int i = 0; i < VM_MAX_MEMMAPS; i++) { 88 if (sysmem_mapping(mem, i)) 89 vm_free_memmap(vm, i); 90 } 91 92 for (int i = 0; i < VM_MAX_MEMSEGS; i++) 93 vm_free_memseg(vm, i); 94 95 sx_xunlock(&mem->mem_segs_lock); 96 sx_destroy(&mem->mem_segs_lock); 97 } 98 99 void 100 vm_slock_memsegs(struct vm *vm) 101 { 102 sx_slock(&vm_mem(vm)->mem_segs_lock); 103 } 104 105 void 106 vm_xlock_memsegs(struct vm *vm) 107 { 108 sx_xlock(&vm_mem(vm)->mem_segs_lock); 109 } 110 111 void 112 vm_unlock_memsegs(struct vm *vm) 113 { 114 sx_unlock(&vm_mem(vm)->mem_segs_lock); 115 } 116 117 void 118 vm_assert_memseg_locked(struct vm *vm) 119 { 120 sx_assert(&vm_mem(vm)->mem_segs_lock, SX_LOCKED); 121 } 122 123 void 124 vm_assert_memseg_xlocked(struct vm *vm) 125 { 126 sx_assert(&vm_mem(vm)->mem_segs_lock, SX_XLOCKED); 127 } 128 129 /* 130 * Return 'true' if 'gpa' is allocated in the guest address space. 131 * 132 * This function is called in the context of a running vcpu which acts as 133 * an implicit lock on 'vm->mem_maps[]'. 134 */ 135 bool 136 vm_mem_allocated(struct vcpu *vcpu, vm_paddr_t gpa) 137 { 138 struct vm *vm = vcpu_vm(vcpu); 139 struct vm_mem_map *mm; 140 int i; 141 142 #ifdef INVARIANTS 143 int hostcpu, state; 144 state = vcpu_get_state(vcpu, &hostcpu); 145 KASSERT(state == VCPU_RUNNING && hostcpu == curcpu, 146 ("%s: invalid vcpu state %d/%d", __func__, state, hostcpu)); 147 #endif 148 149 for (i = 0; i < VM_MAX_MEMMAPS; i++) { 150 mm = &vm_mem(vm)->mem_maps[i]; 151 if (mm->len != 0 && gpa >= mm->gpa && gpa < mm->gpa + mm->len) 152 return (true); /* 'gpa' is sysmem or devmem */ 153 } 154 155 return (false); 156 } 157 158 int 159 vm_alloc_memseg(struct vm *vm, int ident, size_t len, bool sysmem) 160 { 161 struct vm_mem *mem; 162 struct vm_mem_seg *seg; 163 vm_object_t obj; 164 165 mem = vm_mem(vm); 166 vm_assert_memseg_xlocked(vm); 167 168 if (ident < 0 || ident >= VM_MAX_MEMSEGS) 169 return (EINVAL); 170 171 if (len == 0 || (len & PAGE_MASK)) 172 return (EINVAL); 173 174 seg = &mem->mem_segs[ident]; 175 if (seg->object != NULL) { 176 if (seg->len == len && seg->sysmem == sysmem) 177 return (EEXIST); 178 else 179 return (EINVAL); 180 } 181 182 obj = vm_object_allocate(OBJT_SWAP, len >> PAGE_SHIFT); 183 if (obj == NULL) 184 return (ENOMEM); 185 186 seg->len = len; 187 seg->object = obj; 188 seg->sysmem = sysmem; 189 return (0); 190 } 191 192 int 193 vm_get_memseg(struct vm *vm, int ident, size_t *len, bool *sysmem, 194 vm_object_t *objptr) 195 { 196 struct vm_mem *mem; 197 struct vm_mem_seg *seg; 198 199 mem = vm_mem(vm); 200 201 vm_assert_memseg_locked(vm); 202 203 if (ident < 0 || ident >= VM_MAX_MEMSEGS) 204 return (EINVAL); 205 206 seg = &mem->mem_segs[ident]; 207 if (len) 208 *len = seg->len; 209 if (sysmem) 210 *sysmem = seg->sysmem; 211 if (objptr) 212 *objptr = seg->object; 213 return (0); 214 } 215 216 void 217 vm_free_memseg(struct vm *vm, int ident) 218 { 219 struct vm_mem_seg *seg; 220 221 KASSERT(ident >= 0 && ident < VM_MAX_MEMSEGS, 222 ("%s: invalid memseg ident %d", __func__, ident)); 223 224 seg = &vm_mem(vm)->mem_segs[ident]; 225 if (seg->object != NULL) { 226 vm_object_deallocate(seg->object); 227 bzero(seg, sizeof(struct vm_mem_seg)); 228 } 229 } 230 231 int 232 vm_mmap_memseg(struct vm *vm, vm_paddr_t gpa, int segid, vm_ooffset_t first, 233 size_t len, int prot, int flags) 234 { 235 struct vm_mem *mem; 236 struct vm_mem_seg *seg; 237 struct vm_mem_map *m, *map; 238 struct vmspace *vmspace; 239 vm_ooffset_t last; 240 int i, error; 241 242 if (prot == 0 || (prot & ~(VM_PROT_ALL)) != 0) 243 return (EINVAL); 244 245 if (flags & ~VM_MEMMAP_F_WIRED) 246 return (EINVAL); 247 248 if (segid < 0 || segid >= VM_MAX_MEMSEGS) 249 return (EINVAL); 250 251 mem = vm_mem(vm); 252 seg = &mem->mem_segs[segid]; 253 if (seg->object == NULL) 254 return (EINVAL); 255 256 last = first + len; 257 if (first < 0 || first >= last || last > seg->len) 258 return (EINVAL); 259 260 if ((gpa | first | last) & PAGE_MASK) 261 return (EINVAL); 262 263 map = NULL; 264 for (i = 0; i < VM_MAX_MEMMAPS; i++) { 265 m = &mem->mem_maps[i]; 266 if (m->len == 0) { 267 map = m; 268 break; 269 } 270 } 271 if (map == NULL) 272 return (ENOSPC); 273 274 vmspace = vm_vmspace(vm); 275 error = vm_map_find(&vmspace->vm_map, seg->object, first, &gpa, 276 len, 0, VMFS_NO_SPACE, prot, prot, 0); 277 if (error != KERN_SUCCESS) 278 return (EFAULT); 279 280 vm_object_reference(seg->object); 281 282 if (flags & VM_MEMMAP_F_WIRED) { 283 error = vm_map_wire(&vmspace->vm_map, gpa, gpa + len, 284 VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); 285 if (error != KERN_SUCCESS) { 286 vm_map_remove(&vmspace->vm_map, gpa, gpa + len); 287 return (error == KERN_RESOURCE_SHORTAGE ? ENOMEM : 288 EFAULT); 289 } 290 } 291 292 map->gpa = gpa; 293 map->len = len; 294 map->segoff = first; 295 map->segid = segid; 296 map->prot = prot; 297 map->flags = flags; 298 return (0); 299 } 300 301 int 302 vm_munmap_memseg(struct vm *vm, vm_paddr_t gpa, size_t len) 303 { 304 struct vm_mem *mem; 305 struct vm_mem_map *m; 306 int i; 307 308 mem = vm_mem(vm); 309 for (i = 0; i < VM_MAX_MEMMAPS; i++) { 310 m = &mem->mem_maps[i]; 311 #ifdef VM_MEMMAP_F_IOMMU 312 if ((m->flags & VM_MEMMAP_F_IOMMU) != 0) 313 continue; 314 #endif 315 if (m->gpa == gpa && m->len == len) { 316 vm_free_memmap(vm, i); 317 return (0); 318 } 319 } 320 321 return (EINVAL); 322 } 323 324 int 325 vm_mmap_getnext(struct vm *vm, vm_paddr_t *gpa, int *segid, 326 vm_ooffset_t *segoff, size_t *len, int *prot, int *flags) 327 { 328 struct vm_mem *mem; 329 struct vm_mem_map *mm, *mmnext; 330 int i; 331 332 mem = vm_mem(vm); 333 334 mmnext = NULL; 335 for (i = 0; i < VM_MAX_MEMMAPS; i++) { 336 mm = &mem->mem_maps[i]; 337 if (mm->len == 0 || mm->gpa < *gpa) 338 continue; 339 if (mmnext == NULL || mm->gpa < mmnext->gpa) 340 mmnext = mm; 341 } 342 343 if (mmnext != NULL) { 344 *gpa = mmnext->gpa; 345 if (segid) 346 *segid = mmnext->segid; 347 if (segoff) 348 *segoff = mmnext->segoff; 349 if (len) 350 *len = mmnext->len; 351 if (prot) 352 *prot = mmnext->prot; 353 if (flags) 354 *flags = mmnext->flags; 355 return (0); 356 } else { 357 return (ENOENT); 358 } 359 } 360 361 static void 362 vm_free_memmap(struct vm *vm, int ident) 363 { 364 struct vm_mem_map *mm; 365 int error __diagused; 366 367 mm = &vm_mem(vm)->mem_maps[ident]; 368 if (mm->len) { 369 error = vm_map_remove(&vm_vmspace(vm)->vm_map, mm->gpa, 370 mm->gpa + mm->len); 371 KASSERT(error == KERN_SUCCESS, ("%s: vm_map_remove error %d", 372 __func__, error)); 373 bzero(mm, sizeof(struct vm_mem_map)); 374 } 375 } 376 377 vm_paddr_t 378 vmm_sysmem_maxaddr(struct vm *vm) 379 { 380 struct vm_mem *mem; 381 struct vm_mem_map *mm; 382 vm_paddr_t maxaddr; 383 int i; 384 385 mem = vm_mem(vm); 386 maxaddr = 0; 387 for (i = 0; i < VM_MAX_MEMMAPS; i++) { 388 mm = &mem->mem_maps[i]; 389 if (sysmem_mapping(mem, i)) { 390 if (maxaddr < mm->gpa + mm->len) 391 maxaddr = mm->gpa + mm->len; 392 } 393 } 394 return (maxaddr); 395 } 396 397 static void * 398 _vm_gpa_hold(struct vm *vm, vm_paddr_t gpa, size_t len, int reqprot, 399 void **cookie) 400 { 401 struct vm_mem_map *mm; 402 vm_page_t m; 403 int i, count, pageoff; 404 405 pageoff = gpa & PAGE_MASK; 406 if (len > PAGE_SIZE - pageoff) 407 panic("vm_gpa_hold: invalid gpa/len: 0x%016lx/%lu", gpa, len); 408 409 count = 0; 410 for (i = 0; i < VM_MAX_MEMMAPS; i++) { 411 mm = &vm_mem(vm)->mem_maps[i]; 412 if (gpa >= mm->gpa && gpa < mm->gpa + mm->len) { 413 count = vm_fault_quick_hold_pages( 414 &vm_vmspace(vm)->vm_map, trunc_page(gpa), 415 PAGE_SIZE, reqprot, &m, 1); 416 break; 417 } 418 } 419 420 if (count == 1) { 421 *cookie = m; 422 return ((void *)(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)) + pageoff)); 423 } else { 424 *cookie = NULL; 425 return (NULL); 426 } 427 } 428 429 void * 430 vm_gpa_hold(struct vcpu *vcpu, vm_paddr_t gpa, size_t len, int reqprot, 431 void **cookie) 432 { 433 #ifdef INVARIANTS 434 /* 435 * The current vcpu should be frozen to ensure 'vm_memmap[]' 436 * stability. 437 */ 438 int state = vcpu_get_state(vcpu, NULL); 439 KASSERT(state == VCPU_FROZEN, ("%s: invalid vcpu state %d", 440 __func__, state)); 441 #endif 442 return (_vm_gpa_hold(vcpu_vm(vcpu), gpa, len, reqprot, cookie)); 443 } 444 445 void * 446 vm_gpa_hold_global(struct vm *vm, vm_paddr_t gpa, size_t len, int reqprot, 447 void **cookie) 448 { 449 vm_assert_memseg_locked(vm); 450 return (_vm_gpa_hold(vm, gpa, len, reqprot, cookie)); 451 } 452 453 void 454 vm_gpa_release(void *cookie) 455 { 456 vm_page_t m = cookie; 457 458 vm_page_unwire(m, PQ_ACTIVE); 459 } 460