1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2011 NetApp, Inc. 5 * All rights reserved. 6 */ 7 8 #include <sys/types.h> 9 #include <sys/lock.h> 10 #include <sys/malloc.h> 11 #include <sys/sx.h> 12 #include <sys/systm.h> 13 14 #include <machine/vmm.h> 15 16 #include <vm/vm.h> 17 #include <vm/vm_param.h> 18 #include <vm/vm_extern.h> 19 #include <vm/pmap.h> 20 #include <vm/vm_map.h> 21 #include <vm/vm_object.h> 22 #include <vm/vm_page.h> 23 24 #include <dev/vmm/vmm_dev.h> 25 #include <dev/vmm/vmm_mem.h> 26 #include <dev/vmm/vmm_vm.h> 27 28 static void vm_free_memmap(struct vm *vm, int ident); 29 30 int 31 vm_mem_init(struct vm_mem *mem, vm_offset_t lo, vm_offset_t hi) 32 { 33 mem->mem_vmspace = vmmops_vmspace_alloc(lo, hi); 34 if (mem->mem_vmspace == NULL) 35 return (ENOMEM); 36 sx_init(&mem->mem_segs_lock, "vm_mem_segs"); 37 return (0); 38 } 39 40 static bool 41 sysmem_mapping(struct vm_mem *mem, int idx) 42 { 43 if (mem->mem_maps[idx].len != 0 && 44 mem->mem_segs[mem->mem_maps[idx].segid].sysmem) 45 return (true); 46 else 47 return (false); 48 } 49 50 bool 51 vm_memseg_sysmem(struct vm *vm, int ident) 52 { 53 struct vm_mem *mem; 54 55 mem = vm_mem(vm); 56 vm_assert_memseg_locked(vm); 57 58 if (ident < 0 || ident >= VM_MAX_MEMSEGS) 59 return (false); 60 61 return (mem->mem_segs[ident].sysmem); 62 } 63 64 void 65 vm_mem_cleanup(struct vm *vm) 66 { 67 struct vm_mem *mem; 68 69 mem = vm_mem(vm); 70 71 /* 72 * System memory is removed from the guest address space only when 73 * the VM is destroyed. This is because the mapping remains the same 74 * across VM reset. 75 * 76 * Device memory can be relocated by the guest (e.g. using PCI BARs) 77 * so those mappings are removed on a VM reset. 78 */ 79 for (int i = 0; i < VM_MAX_MEMMAPS; i++) { 80 if (!sysmem_mapping(mem, i)) 81 vm_free_memmap(vm, i); 82 } 83 } 84 85 void 86 vm_mem_destroy(struct vm *vm) 87 { 88 struct vm_mem *mem; 89 90 mem = vm_mem(vm); 91 vm_assert_memseg_xlocked(vm); 92 93 for (int i = 0; i < VM_MAX_MEMMAPS; i++) { 94 if (sysmem_mapping(mem, i)) 95 vm_free_memmap(vm, i); 96 } 97 98 for (int i = 0; i < VM_MAX_MEMSEGS; i++) 99 vm_free_memseg(vm, i); 100 101 vmmops_vmspace_free(mem->mem_vmspace); 102 103 sx_xunlock(&mem->mem_segs_lock); 104 sx_destroy(&mem->mem_segs_lock); 105 } 106 107 struct vmspace * 108 vm_vmspace(struct vm *vm) 109 { 110 struct vm_mem *mem; 111 112 mem = vm_mem(vm); 113 return (mem->mem_vmspace); 114 } 115 116 void 117 vm_slock_memsegs(struct vm *vm) 118 { 119 sx_slock(&vm_mem(vm)->mem_segs_lock); 120 } 121 122 void 123 vm_xlock_memsegs(struct vm *vm) 124 { 125 sx_xlock(&vm_mem(vm)->mem_segs_lock); 126 } 127 128 void 129 vm_unlock_memsegs(struct vm *vm) 130 { 131 sx_unlock(&vm_mem(vm)->mem_segs_lock); 132 } 133 134 void 135 vm_assert_memseg_locked(struct vm *vm) 136 { 137 sx_assert(&vm_mem(vm)->mem_segs_lock, SX_LOCKED); 138 } 139 140 void 141 vm_assert_memseg_xlocked(struct vm *vm) 142 { 143 sx_assert(&vm_mem(vm)->mem_segs_lock, SX_XLOCKED); 144 } 145 146 /* 147 * Return 'true' if 'gpa' is allocated in the guest address space. 148 * 149 * This function is called in the context of a running vcpu which acts as 150 * an implicit lock on 'vm->mem_maps[]'. 151 */ 152 bool 153 vm_mem_allocated(struct vcpu *vcpu, vm_paddr_t gpa) 154 { 155 struct vm *vm = vcpu_vm(vcpu); 156 struct vm_mem_map *mm; 157 int i; 158 159 #ifdef INVARIANTS 160 int hostcpu, state; 161 state = vcpu_get_state(vcpu, &hostcpu); 162 KASSERT(state == VCPU_RUNNING && hostcpu == curcpu, 163 ("%s: invalid vcpu state %d/%d", __func__, state, hostcpu)); 164 #endif 165 166 for (i = 0; i < VM_MAX_MEMMAPS; i++) { 167 mm = &vm_mem(vm)->mem_maps[i]; 168 if (mm->len != 0 && gpa >= mm->gpa && gpa < mm->gpa + mm->len) 169 return (true); /* 'gpa' is sysmem or devmem */ 170 } 171 172 return (false); 173 } 174 175 int 176 vm_alloc_memseg(struct vm *vm, int ident, size_t len, bool sysmem, 177 struct domainset *obj_domainset) 178 { 179 struct vm_mem_seg *seg; 180 struct vm_mem *mem; 181 vm_object_t obj; 182 183 mem = vm_mem(vm); 184 vm_assert_memseg_xlocked(vm); 185 186 if (ident < 0 || ident >= VM_MAX_MEMSEGS) 187 return (EINVAL); 188 189 if (len == 0 || (len & PAGE_MASK)) 190 return (EINVAL); 191 192 seg = &mem->mem_segs[ident]; 193 if (seg->object != NULL) { 194 if (seg->len == len && seg->sysmem == sysmem) 195 return (EEXIST); 196 else 197 return (EINVAL); 198 } 199 200 /* 201 * When given an impossible policy, signal an 202 * error to the user. 203 */ 204 if (obj_domainset != NULL && domainset_empty_vm(obj_domainset)) 205 return (EINVAL); 206 obj = vm_object_allocate(OBJT_SWAP, len >> PAGE_SHIFT); 207 if (obj == NULL) 208 return (ENOMEM); 209 210 seg->len = len; 211 seg->object = obj; 212 if (obj_domainset != NULL) 213 seg->object->domain.dr_policy = obj_domainset; 214 seg->sysmem = sysmem; 215 216 return (0); 217 } 218 219 int 220 vm_get_memseg(struct vm *vm, int ident, size_t *len, bool *sysmem, 221 vm_object_t *objptr) 222 { 223 struct vm_mem *mem; 224 struct vm_mem_seg *seg; 225 226 mem = vm_mem(vm); 227 228 vm_assert_memseg_locked(vm); 229 230 if (ident < 0 || ident >= VM_MAX_MEMSEGS) 231 return (EINVAL); 232 233 seg = &mem->mem_segs[ident]; 234 if (len) 235 *len = seg->len; 236 if (sysmem) 237 *sysmem = seg->sysmem; 238 if (objptr) 239 *objptr = seg->object; 240 return (0); 241 } 242 243 void 244 vm_free_memseg(struct vm *vm, int ident) 245 { 246 struct vm_mem_seg *seg; 247 248 KASSERT(ident >= 0 && ident < VM_MAX_MEMSEGS, 249 ("%s: invalid memseg ident %d", __func__, ident)); 250 251 seg = &vm_mem(vm)->mem_segs[ident]; 252 if (seg->object != NULL) { 253 vm_object_deallocate(seg->object); 254 bzero(seg, sizeof(struct vm_mem_seg)); 255 } 256 } 257 258 int 259 vm_mmap_memseg(struct vm *vm, vm_paddr_t gpa, int segid, vm_ooffset_t first, 260 size_t len, int prot, int flags) 261 { 262 struct vm_mem *mem; 263 struct vm_mem_seg *seg; 264 struct vm_mem_map *m, *map; 265 struct vm_map *vmmap; 266 vm_ooffset_t last; 267 int i, error; 268 269 if (prot == 0 || (prot & ~(VM_PROT_ALL)) != 0) 270 return (EINVAL); 271 272 if (flags & ~VM_MEMMAP_F_WIRED) 273 return (EINVAL); 274 275 if (segid < 0 || segid >= VM_MAX_MEMSEGS) 276 return (EINVAL); 277 278 mem = vm_mem(vm); 279 seg = &mem->mem_segs[segid]; 280 if (seg->object == NULL) 281 return (EINVAL); 282 283 if (first + len < first || gpa + len < gpa) 284 return (EINVAL); 285 last = first + len; 286 if (first >= last || last > seg->len) 287 return (EINVAL); 288 289 if ((gpa | first | last) & PAGE_MASK) 290 return (EINVAL); 291 292 map = NULL; 293 for (i = 0; i < VM_MAX_MEMMAPS; i++) { 294 m = &mem->mem_maps[i]; 295 if (m->len == 0) { 296 map = m; 297 break; 298 } 299 } 300 if (map == NULL) 301 return (ENOSPC); 302 303 vmmap = &mem->mem_vmspace->vm_map; 304 vm_map_lock(vmmap); 305 error = vm_map_insert(vmmap, seg->object, first, gpa, gpa + len, 306 prot, prot, 0); 307 vm_map_unlock(vmmap); 308 if (error != KERN_SUCCESS) 309 return (vm_mmap_to_errno(error)); 310 vm_object_reference(seg->object); 311 312 if (flags & VM_MEMMAP_F_WIRED) { 313 error = vm_map_wire(vmmap, gpa, gpa + len, 314 VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); 315 if (error != KERN_SUCCESS) { 316 vm_map_remove(vmmap, gpa, gpa + len); 317 return (error == KERN_RESOURCE_SHORTAGE ? ENOMEM : 318 EFAULT); 319 } 320 } 321 322 map->gpa = gpa; 323 map->len = len; 324 map->segoff = first; 325 map->segid = segid; 326 map->prot = prot; 327 map->flags = flags; 328 return (0); 329 } 330 331 int 332 vm_munmap_memseg(struct vm *vm, vm_paddr_t gpa, size_t len) 333 { 334 struct vm_mem *mem; 335 struct vm_mem_map *m; 336 int i; 337 338 mem = vm_mem(vm); 339 for (i = 0; i < VM_MAX_MEMMAPS; i++) { 340 m = &mem->mem_maps[i]; 341 #ifdef VM_MEMMAP_F_IOMMU 342 if ((m->flags & VM_MEMMAP_F_IOMMU) != 0) 343 continue; 344 #endif 345 if (m->gpa == gpa && m->len == len) { 346 vm_free_memmap(vm, i); 347 return (0); 348 } 349 } 350 351 return (EINVAL); 352 } 353 354 int 355 vm_mmap_getnext(struct vm *vm, vm_paddr_t *gpa, int *segid, 356 vm_ooffset_t *segoff, size_t *len, int *prot, int *flags) 357 { 358 struct vm_mem *mem; 359 struct vm_mem_map *mm, *mmnext; 360 int i; 361 362 mem = vm_mem(vm); 363 364 mmnext = NULL; 365 for (i = 0; i < VM_MAX_MEMMAPS; i++) { 366 mm = &mem->mem_maps[i]; 367 if (mm->len == 0 || mm->gpa < *gpa) 368 continue; 369 if (mmnext == NULL || mm->gpa < mmnext->gpa) 370 mmnext = mm; 371 } 372 373 if (mmnext != NULL) { 374 *gpa = mmnext->gpa; 375 if (segid) 376 *segid = mmnext->segid; 377 if (segoff) 378 *segoff = mmnext->segoff; 379 if (len) 380 *len = mmnext->len; 381 if (prot) 382 *prot = mmnext->prot; 383 if (flags) 384 *flags = mmnext->flags; 385 return (0); 386 } else { 387 return (ENOENT); 388 } 389 } 390 391 static void 392 vm_free_memmap(struct vm *vm, int ident) 393 { 394 struct vm_mem_map *mm; 395 int error __diagused; 396 397 mm = &vm_mem(vm)->mem_maps[ident]; 398 if (mm->len) { 399 error = vm_map_remove(&vm_vmspace(vm)->vm_map, mm->gpa, 400 mm->gpa + mm->len); 401 KASSERT(error == KERN_SUCCESS, ("%s: vm_map_remove error %d", 402 __func__, error)); 403 bzero(mm, sizeof(struct vm_mem_map)); 404 } 405 } 406 407 vm_paddr_t 408 vmm_sysmem_maxaddr(struct vm *vm) 409 { 410 struct vm_mem *mem; 411 struct vm_mem_map *mm; 412 vm_paddr_t maxaddr; 413 int i; 414 415 mem = vm_mem(vm); 416 maxaddr = 0; 417 for (i = 0; i < VM_MAX_MEMMAPS; i++) { 418 mm = &mem->mem_maps[i]; 419 if (sysmem_mapping(mem, i)) { 420 if (maxaddr < mm->gpa + mm->len) 421 maxaddr = mm->gpa + mm->len; 422 } 423 } 424 return (maxaddr); 425 } 426 427 static void * 428 _vm_gpa_hold(struct vm *vm, vm_paddr_t gpa, size_t len, int reqprot, 429 void **cookie) 430 { 431 struct vm_mem_map *mm; 432 vm_page_t m; 433 int i, count, pageoff; 434 435 pageoff = gpa & PAGE_MASK; 436 if (len > PAGE_SIZE - pageoff) 437 panic("vm_gpa_hold: invalid gpa/len: 0x%016lx/%lu", gpa, len); 438 439 count = 0; 440 for (i = 0; i < VM_MAX_MEMMAPS; i++) { 441 mm = &vm_mem(vm)->mem_maps[i]; 442 if (gpa >= mm->gpa && gpa < mm->gpa + mm->len) { 443 count = vm_fault_quick_hold_pages( 444 &vm_vmspace(vm)->vm_map, trunc_page(gpa), 445 PAGE_SIZE, reqprot, &m, 1); 446 break; 447 } 448 } 449 450 if (count == 1) { 451 *cookie = m; 452 return ((void *)(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)) + pageoff)); 453 } else { 454 *cookie = NULL; 455 return (NULL); 456 } 457 } 458 459 void * 460 vm_gpa_hold(struct vcpu *vcpu, vm_paddr_t gpa, size_t len, int reqprot, 461 void **cookie) 462 { 463 #ifdef INVARIANTS 464 /* 465 * The current vcpu should be frozen to ensure 'vm_memmap[]' 466 * stability. 467 */ 468 int state = vcpu_get_state(vcpu, NULL); 469 KASSERT(state == VCPU_FROZEN, ("%s: invalid vcpu state %d", 470 __func__, state)); 471 #endif 472 return (_vm_gpa_hold(vcpu_vm(vcpu), gpa, len, reqprot, cookie)); 473 } 474 475 void * 476 vm_gpa_hold_global(struct vm *vm, vm_paddr_t gpa, size_t len, int reqprot, 477 void **cookie) 478 { 479 vm_assert_memseg_locked(vm); 480 return (_vm_gpa_hold(vm, gpa, len, reqprot, cookie)); 481 } 482 483 void 484 vm_gpa_release(void *cookie) 485 { 486 vm_page_t m = cookie; 487 488 vm_page_unwire(m, PQ_ACTIVE); 489 } 490