1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_vm.h" 7 8 #include <linux/dma-fence-array.h> 9 10 #include <drm/ttm/ttm_execbuf_util.h> 11 #include <drm/ttm/ttm_tt.h> 12 #include <drm/xe_drm.h> 13 #include <linux/delay.h> 14 #include <linux/kthread.h> 15 #include <linux/mm.h> 16 #include <linux/swap.h> 17 18 #include "xe_bo.h" 19 #include "xe_device.h" 20 #include "xe_engine.h" 21 #include "xe_gt.h" 22 #include "xe_gt_pagefault.h" 23 #include "xe_gt_tlb_invalidation.h" 24 #include "xe_migrate.h" 25 #include "xe_pm.h" 26 #include "xe_preempt_fence.h" 27 #include "xe_pt.h" 28 #include "xe_res_cursor.h" 29 #include "xe_sync.h" 30 #include "xe_trace.h" 31 32 #define TEST_VM_ASYNC_OPS_ERROR 33 34 /** 35 * xe_vma_userptr_check_repin() - Advisory check for repin needed 36 * @vma: The userptr vma 37 * 38 * Check if the userptr vma has been invalidated since last successful 39 * repin. The check is advisory only and can the function can be called 40 * without the vm->userptr.notifier_lock held. There is no guarantee that the 41 * vma userptr will remain valid after a lockless check, so typically 42 * the call needs to be followed by a proper check under the notifier_lock. 43 * 44 * Return: 0 if userptr vma is valid, -EAGAIN otherwise; repin recommended. 45 */ 46 int xe_vma_userptr_check_repin(struct xe_vma *vma) 47 { 48 return mmu_interval_check_retry(&vma->userptr.notifier, 49 vma->userptr.notifier_seq) ? 50 -EAGAIN : 0; 51 } 52 53 int xe_vma_userptr_pin_pages(struct xe_vma *vma) 54 { 55 struct xe_vm *vm = vma->vm; 56 struct xe_device *xe = vm->xe; 57 const unsigned long num_pages = 58 (vma->end - vma->start + 1) >> PAGE_SHIFT; 59 struct page **pages; 60 bool in_kthread = !current->mm; 61 unsigned long notifier_seq; 62 int pinned, ret, i; 63 bool read_only = vma->pte_flags & XE_PTE_READ_ONLY; 64 65 lockdep_assert_held(&vm->lock); 66 XE_BUG_ON(!xe_vma_is_userptr(vma)); 67 retry: 68 if (vma->destroyed) 69 return 0; 70 71 notifier_seq = mmu_interval_read_begin(&vma->userptr.notifier); 72 if (notifier_seq == vma->userptr.notifier_seq) 73 return 0; 74 75 pages = kvmalloc_array(num_pages, sizeof(*pages), GFP_KERNEL); 76 if (!pages) 77 return -ENOMEM; 78 79 if (vma->userptr.sg) { 80 dma_unmap_sgtable(xe->drm.dev, 81 vma->userptr.sg, 82 read_only ? DMA_TO_DEVICE : 83 DMA_BIDIRECTIONAL, 0); 84 sg_free_table(vma->userptr.sg); 85 vma->userptr.sg = NULL; 86 } 87 88 pinned = ret = 0; 89 if (in_kthread) { 90 if (!mmget_not_zero(vma->userptr.notifier.mm)) { 91 ret = -EFAULT; 92 goto mm_closed; 93 } 94 kthread_use_mm(vma->userptr.notifier.mm); 95 } 96 97 while (pinned < num_pages) { 98 ret = get_user_pages_fast(vma->userptr.ptr + pinned * PAGE_SIZE, 99 num_pages - pinned, 100 read_only ? 0 : FOLL_WRITE, 101 &pages[pinned]); 102 if (ret < 0) { 103 if (in_kthread) 104 ret = 0; 105 break; 106 } 107 108 pinned += ret; 109 ret = 0; 110 } 111 112 if (in_kthread) { 113 kthread_unuse_mm(vma->userptr.notifier.mm); 114 mmput(vma->userptr.notifier.mm); 115 } 116 mm_closed: 117 if (ret) 118 goto out; 119 120 ret = sg_alloc_table_from_pages_segment(&vma->userptr.sgt, pages, 121 pinned, 0, 122 (u64)pinned << PAGE_SHIFT, 123 xe_sg_segment_size(xe->drm.dev), 124 GFP_KERNEL); 125 if (ret) { 126 vma->userptr.sg = NULL; 127 goto out; 128 } 129 vma->userptr.sg = &vma->userptr.sgt; 130 131 ret = dma_map_sgtable(xe->drm.dev, vma->userptr.sg, 132 read_only ? DMA_TO_DEVICE : 133 DMA_BIDIRECTIONAL, 134 DMA_ATTR_SKIP_CPU_SYNC | 135 DMA_ATTR_NO_KERNEL_MAPPING); 136 if (ret) { 137 sg_free_table(vma->userptr.sg); 138 vma->userptr.sg = NULL; 139 goto out; 140 } 141 142 for (i = 0; i < pinned; ++i) { 143 if (!read_only) { 144 lock_page(pages[i]); 145 set_page_dirty(pages[i]); 146 unlock_page(pages[i]); 147 } 148 149 mark_page_accessed(pages[i]); 150 } 151 152 out: 153 release_pages(pages, pinned); 154 kvfree(pages); 155 156 if (!(ret < 0)) { 157 vma->userptr.notifier_seq = notifier_seq; 158 if (xe_vma_userptr_check_repin(vma) == -EAGAIN) 159 goto retry; 160 } 161 162 return ret < 0 ? ret : 0; 163 } 164 165 static bool preempt_fences_waiting(struct xe_vm *vm) 166 { 167 struct xe_engine *e; 168 169 lockdep_assert_held(&vm->lock); 170 xe_vm_assert_held(vm); 171 172 list_for_each_entry(e, &vm->preempt.engines, compute.link) { 173 if (!e->compute.pfence || (e->compute.pfence && 174 test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, 175 &e->compute.pfence->flags))) { 176 return true; 177 } 178 } 179 180 return false; 181 } 182 183 static void free_preempt_fences(struct list_head *list) 184 { 185 struct list_head *link, *next; 186 187 list_for_each_safe(link, next, list) 188 xe_preempt_fence_free(to_preempt_fence_from_link(link)); 189 } 190 191 static int alloc_preempt_fences(struct xe_vm *vm, struct list_head *list, 192 unsigned int *count) 193 { 194 lockdep_assert_held(&vm->lock); 195 xe_vm_assert_held(vm); 196 197 if (*count >= vm->preempt.num_engines) 198 return 0; 199 200 for (; *count < vm->preempt.num_engines; ++(*count)) { 201 struct xe_preempt_fence *pfence = xe_preempt_fence_alloc(); 202 203 if (IS_ERR(pfence)) 204 return PTR_ERR(pfence); 205 206 list_move_tail(xe_preempt_fence_link(pfence), list); 207 } 208 209 return 0; 210 } 211 212 static int wait_for_existing_preempt_fences(struct xe_vm *vm) 213 { 214 struct xe_engine *e; 215 216 xe_vm_assert_held(vm); 217 218 list_for_each_entry(e, &vm->preempt.engines, compute.link) { 219 if (e->compute.pfence) { 220 long timeout = dma_fence_wait(e->compute.pfence, false); 221 222 if (timeout < 0) 223 return -ETIME; 224 dma_fence_put(e->compute.pfence); 225 e->compute.pfence = NULL; 226 } 227 } 228 229 return 0; 230 } 231 232 static bool xe_vm_is_idle(struct xe_vm *vm) 233 { 234 struct xe_engine *e; 235 236 xe_vm_assert_held(vm); 237 list_for_each_entry(e, &vm->preempt.engines, compute.link) { 238 if (!xe_engine_is_idle(e)) 239 return false; 240 } 241 242 return true; 243 } 244 245 static void arm_preempt_fences(struct xe_vm *vm, struct list_head *list) 246 { 247 struct list_head *link; 248 struct xe_engine *e; 249 250 list_for_each_entry(e, &vm->preempt.engines, compute.link) { 251 struct dma_fence *fence; 252 253 link = list->next; 254 XE_BUG_ON(link == list); 255 256 fence = xe_preempt_fence_arm(to_preempt_fence_from_link(link), 257 e, e->compute.context, 258 ++e->compute.seqno); 259 dma_fence_put(e->compute.pfence); 260 e->compute.pfence = fence; 261 } 262 } 263 264 static int add_preempt_fences(struct xe_vm *vm, struct xe_bo *bo) 265 { 266 struct xe_engine *e; 267 struct ww_acquire_ctx ww; 268 int err; 269 270 err = xe_bo_lock(bo, &ww, vm->preempt.num_engines, true); 271 if (err) 272 return err; 273 274 list_for_each_entry(e, &vm->preempt.engines, compute.link) 275 if (e->compute.pfence) { 276 dma_resv_add_fence(bo->ttm.base.resv, 277 e->compute.pfence, 278 DMA_RESV_USAGE_BOOKKEEP); 279 } 280 281 xe_bo_unlock(bo, &ww); 282 return 0; 283 } 284 285 /** 286 * xe_vm_fence_all_extobjs() - Add a fence to vm's external objects' resv 287 * @vm: The vm. 288 * @fence: The fence to add. 289 * @usage: The resv usage for the fence. 290 * 291 * Loops over all of the vm's external object bindings and adds a @fence 292 * with the given @usage to all of the external object's reservation 293 * objects. 294 */ 295 void xe_vm_fence_all_extobjs(struct xe_vm *vm, struct dma_fence *fence, 296 enum dma_resv_usage usage) 297 { 298 struct xe_vma *vma; 299 300 list_for_each_entry(vma, &vm->extobj.list, extobj.link) 301 dma_resv_add_fence(vma->bo->ttm.base.resv, fence, usage); 302 } 303 304 static void resume_and_reinstall_preempt_fences(struct xe_vm *vm) 305 { 306 struct xe_engine *e; 307 308 lockdep_assert_held(&vm->lock); 309 xe_vm_assert_held(vm); 310 311 list_for_each_entry(e, &vm->preempt.engines, compute.link) { 312 e->ops->resume(e); 313 314 dma_resv_add_fence(&vm->resv, e->compute.pfence, 315 DMA_RESV_USAGE_BOOKKEEP); 316 xe_vm_fence_all_extobjs(vm, e->compute.pfence, 317 DMA_RESV_USAGE_BOOKKEEP); 318 } 319 } 320 321 int xe_vm_add_compute_engine(struct xe_vm *vm, struct xe_engine *e) 322 { 323 struct ttm_validate_buffer tv_onstack[XE_ONSTACK_TV]; 324 struct ttm_validate_buffer *tv; 325 struct ww_acquire_ctx ww; 326 struct list_head objs; 327 struct dma_fence *pfence; 328 int err; 329 bool wait; 330 331 XE_BUG_ON(!xe_vm_in_compute_mode(vm)); 332 333 down_write(&vm->lock); 334 335 err = xe_vm_lock_dma_resv(vm, &ww, tv_onstack, &tv, &objs, true, 1); 336 if (err) 337 goto out_unlock_outer; 338 339 pfence = xe_preempt_fence_create(e, e->compute.context, 340 ++e->compute.seqno); 341 if (!pfence) { 342 err = -ENOMEM; 343 goto out_unlock; 344 } 345 346 list_add(&e->compute.link, &vm->preempt.engines); 347 ++vm->preempt.num_engines; 348 e->compute.pfence = pfence; 349 350 down_read(&vm->userptr.notifier_lock); 351 352 dma_resv_add_fence(&vm->resv, pfence, 353 DMA_RESV_USAGE_BOOKKEEP); 354 355 xe_vm_fence_all_extobjs(vm, pfence, DMA_RESV_USAGE_BOOKKEEP); 356 357 /* 358 * Check to see if a preemption on VM is in flight or userptr 359 * invalidation, if so trigger this preempt fence to sync state with 360 * other preempt fences on the VM. 361 */ 362 wait = __xe_vm_userptr_needs_repin(vm) || preempt_fences_waiting(vm); 363 if (wait) 364 dma_fence_enable_sw_signaling(pfence); 365 366 up_read(&vm->userptr.notifier_lock); 367 368 out_unlock: 369 xe_vm_unlock_dma_resv(vm, tv_onstack, tv, &ww, &objs); 370 out_unlock_outer: 371 up_write(&vm->lock); 372 373 return err; 374 } 375 376 /** 377 * __xe_vm_userptr_needs_repin() - Check whether the VM does have userptrs 378 * that need repinning. 379 * @vm: The VM. 380 * 381 * This function checks for whether the VM has userptrs that need repinning, 382 * and provides a release-type barrier on the userptr.notifier_lock after 383 * checking. 384 * 385 * Return: 0 if there are no userptrs needing repinning, -EAGAIN if there are. 386 */ 387 int __xe_vm_userptr_needs_repin(struct xe_vm *vm) 388 { 389 lockdep_assert_held_read(&vm->userptr.notifier_lock); 390 391 return (list_empty(&vm->userptr.repin_list) && 392 list_empty(&vm->userptr.invalidated)) ? 0 : -EAGAIN; 393 } 394 395 /** 396 * xe_vm_lock_dma_resv() - Lock the vm dma_resv object and the dma_resv 397 * objects of the vm's external buffer objects. 398 * @vm: The vm. 399 * @ww: Pointer to a struct ww_acquire_ctx locking context. 400 * @tv_onstack: Array size XE_ONSTACK_TV of storage for the struct 401 * ttm_validate_buffers used for locking. 402 * @tv: Pointer to a pointer that on output contains the actual storage used. 403 * @objs: List head for the buffer objects locked. 404 * @intr: Whether to lock interruptible. 405 * @num_shared: Number of dma-fence slots to reserve in the locked objects. 406 * 407 * Locks the vm dma-resv objects and all the dma-resv objects of the 408 * buffer objects on the vm external object list. The TTM utilities require 409 * a list of struct ttm_validate_buffers pointing to the actual buffer 410 * objects to lock. Storage for those struct ttm_validate_buffers should 411 * be provided in @tv_onstack, and is typically reserved on the stack 412 * of the caller. If the size of @tv_onstack isn't sufficient, then 413 * storage will be allocated internally using kvmalloc(). 414 * 415 * The function performs deadlock handling internally, and after a 416 * successful return the ww locking transaction should be considered 417 * sealed. 418 * 419 * Return: 0 on success, Negative error code on error. In particular if 420 * @intr is set to true, -EINTR or -ERESTARTSYS may be returned. In case 421 * of error, any locking performed has been reverted. 422 */ 423 int xe_vm_lock_dma_resv(struct xe_vm *vm, struct ww_acquire_ctx *ww, 424 struct ttm_validate_buffer *tv_onstack, 425 struct ttm_validate_buffer **tv, 426 struct list_head *objs, 427 bool intr, 428 unsigned int num_shared) 429 { 430 struct ttm_validate_buffer *tv_vm, *tv_bo; 431 struct xe_vma *vma, *next; 432 LIST_HEAD(dups); 433 int err; 434 435 lockdep_assert_held(&vm->lock); 436 437 if (vm->extobj.entries < XE_ONSTACK_TV) { 438 tv_vm = tv_onstack; 439 } else { 440 tv_vm = kvmalloc_array(vm->extobj.entries + 1, sizeof(*tv_vm), 441 GFP_KERNEL); 442 if (!tv_vm) 443 return -ENOMEM; 444 } 445 tv_bo = tv_vm + 1; 446 447 INIT_LIST_HEAD(objs); 448 list_for_each_entry(vma, &vm->extobj.list, extobj.link) { 449 tv_bo->num_shared = num_shared; 450 tv_bo->bo = &vma->bo->ttm; 451 452 list_add_tail(&tv_bo->head, objs); 453 tv_bo++; 454 } 455 tv_vm->num_shared = num_shared; 456 tv_vm->bo = xe_vm_ttm_bo(vm); 457 list_add_tail(&tv_vm->head, objs); 458 err = ttm_eu_reserve_buffers(ww, objs, intr, &dups); 459 if (err) 460 goto out_err; 461 462 spin_lock(&vm->notifier.list_lock); 463 list_for_each_entry_safe(vma, next, &vm->notifier.rebind_list, 464 notifier.rebind_link) { 465 xe_bo_assert_held(vma->bo); 466 467 list_del_init(&vma->notifier.rebind_link); 468 if (vma->tile_present && !vma->destroyed) 469 list_move_tail(&vma->rebind_link, &vm->rebind_list); 470 } 471 spin_unlock(&vm->notifier.list_lock); 472 473 *tv = tv_vm; 474 return 0; 475 476 out_err: 477 if (tv_vm != tv_onstack) 478 kvfree(tv_vm); 479 480 return err; 481 } 482 483 /** 484 * xe_vm_unlock_dma_resv() - Unlock reservation objects locked by 485 * xe_vm_lock_dma_resv() 486 * @vm: The vm. 487 * @tv_onstack: The @tv_onstack array given to xe_vm_lock_dma_resv(). 488 * @tv: The value of *@tv given by xe_vm_lock_dma_resv(). 489 * @ww: The ww_acquire_context used for locking. 490 * @objs: The list returned from xe_vm_lock_dma_resv(). 491 * 492 * Unlocks the reservation objects and frees any memory allocated by 493 * xe_vm_lock_dma_resv(). 494 */ 495 void xe_vm_unlock_dma_resv(struct xe_vm *vm, 496 struct ttm_validate_buffer *tv_onstack, 497 struct ttm_validate_buffer *tv, 498 struct ww_acquire_ctx *ww, 499 struct list_head *objs) 500 { 501 /* 502 * Nothing should've been able to enter the list while we were locked, 503 * since we've held the dma-resvs of all the vm's external objects, 504 * and holding the dma_resv of an object is required for list 505 * addition, and we shouldn't add ourselves. 506 */ 507 XE_WARN_ON(!list_empty(&vm->notifier.rebind_list)); 508 509 ttm_eu_backoff_reservation(ww, objs); 510 if (tv && tv != tv_onstack) 511 kvfree(tv); 512 } 513 514 #define XE_VM_REBIND_RETRY_TIMEOUT_MS 1000 515 516 static void preempt_rebind_work_func(struct work_struct *w) 517 { 518 struct xe_vm *vm = container_of(w, struct xe_vm, preempt.rebind_work); 519 struct xe_vma *vma; 520 struct ttm_validate_buffer tv_onstack[XE_ONSTACK_TV]; 521 struct ttm_validate_buffer *tv; 522 struct ww_acquire_ctx ww; 523 struct list_head objs; 524 struct dma_fence *rebind_fence; 525 unsigned int fence_count = 0; 526 LIST_HEAD(preempt_fences); 527 ktime_t end = 0; 528 int err; 529 long wait; 530 int __maybe_unused tries = 0; 531 532 XE_BUG_ON(!xe_vm_in_compute_mode(vm)); 533 trace_xe_vm_rebind_worker_enter(vm); 534 535 if (xe_vm_is_closed(vm)) { 536 trace_xe_vm_rebind_worker_exit(vm); 537 return; 538 } 539 540 down_write(&vm->lock); 541 542 retry: 543 if (vm->async_ops.error) 544 goto out_unlock_outer; 545 546 /* 547 * Extreme corner where we exit a VM error state with a munmap style VM 548 * unbind inflight which requires a rebind. In this case the rebind 549 * needs to install some fences into the dma-resv slots. The worker to 550 * do this queued, let that worker make progress by dropping vm->lock 551 * and trying this again. 552 */ 553 if (vm->async_ops.munmap_rebind_inflight) { 554 up_write(&vm->lock); 555 flush_work(&vm->async_ops.work); 556 goto retry; 557 } 558 559 if (xe_vm_userptr_check_repin(vm)) { 560 err = xe_vm_userptr_pin(vm); 561 if (err) 562 goto out_unlock_outer; 563 } 564 565 err = xe_vm_lock_dma_resv(vm, &ww, tv_onstack, &tv, &objs, 566 false, vm->preempt.num_engines); 567 if (err) 568 goto out_unlock_outer; 569 570 if (xe_vm_is_idle(vm)) { 571 vm->preempt.rebind_deactivated = true; 572 goto out_unlock; 573 } 574 575 /* Fresh preempt fences already installed. Everyting is running. */ 576 if (!preempt_fences_waiting(vm)) 577 goto out_unlock; 578 579 /* 580 * This makes sure vm is completely suspended and also balances 581 * xe_engine suspend- and resume; we resume *all* vm engines below. 582 */ 583 err = wait_for_existing_preempt_fences(vm); 584 if (err) 585 goto out_unlock; 586 587 err = alloc_preempt_fences(vm, &preempt_fences, &fence_count); 588 if (err) 589 goto out_unlock; 590 591 list_for_each_entry(vma, &vm->rebind_list, rebind_link) { 592 if (xe_vma_is_userptr(vma) || vma->destroyed) 593 continue; 594 595 err = xe_bo_validate(vma->bo, vm, false); 596 if (err) 597 goto out_unlock; 598 } 599 600 rebind_fence = xe_vm_rebind(vm, true); 601 if (IS_ERR(rebind_fence)) { 602 err = PTR_ERR(rebind_fence); 603 goto out_unlock; 604 } 605 606 if (rebind_fence) { 607 dma_fence_wait(rebind_fence, false); 608 dma_fence_put(rebind_fence); 609 } 610 611 /* Wait on munmap style VM unbinds */ 612 wait = dma_resv_wait_timeout(&vm->resv, 613 DMA_RESV_USAGE_KERNEL, 614 false, MAX_SCHEDULE_TIMEOUT); 615 if (wait <= 0) { 616 err = -ETIME; 617 goto out_unlock; 618 } 619 620 #define retry_required(__tries, __vm) \ 621 (IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT) ? \ 622 (!(__tries)++ || __xe_vm_userptr_needs_repin(__vm)) : \ 623 __xe_vm_userptr_needs_repin(__vm)) 624 625 down_read(&vm->userptr.notifier_lock); 626 if (retry_required(tries, vm)) { 627 up_read(&vm->userptr.notifier_lock); 628 err = -EAGAIN; 629 goto out_unlock; 630 } 631 632 #undef retry_required 633 634 /* Point of no return. */ 635 arm_preempt_fences(vm, &preempt_fences); 636 resume_and_reinstall_preempt_fences(vm); 637 up_read(&vm->userptr.notifier_lock); 638 639 out_unlock: 640 xe_vm_unlock_dma_resv(vm, tv_onstack, tv, &ww, &objs); 641 out_unlock_outer: 642 if (err == -EAGAIN) { 643 trace_xe_vm_rebind_worker_retry(vm); 644 goto retry; 645 } 646 647 /* 648 * With multiple active VMs, under memory pressure, it is possible that 649 * ttm_bo_validate() run into -EDEADLK and in such case returns -ENOMEM. 650 * Until ttm properly handles locking in such scenarios, best thing the 651 * driver can do is retry with a timeout. Killing the VM or putting it 652 * in error state after timeout or other error scenarios is still TBD. 653 */ 654 if (err == -ENOMEM) { 655 ktime_t cur = ktime_get(); 656 657 end = end ? : ktime_add_ms(cur, XE_VM_REBIND_RETRY_TIMEOUT_MS); 658 if (ktime_before(cur, end)) { 659 msleep(20); 660 trace_xe_vm_rebind_worker_retry(vm); 661 goto retry; 662 } 663 } 664 up_write(&vm->lock); 665 666 free_preempt_fences(&preempt_fences); 667 668 XE_WARN_ON(err < 0); /* TODO: Kill VM or put in error state */ 669 trace_xe_vm_rebind_worker_exit(vm); 670 } 671 672 struct async_op_fence; 673 static int __xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, 674 struct xe_engine *e, struct xe_sync_entry *syncs, 675 u32 num_syncs, struct async_op_fence *afence); 676 677 static bool vma_userptr_invalidate(struct mmu_interval_notifier *mni, 678 const struct mmu_notifier_range *range, 679 unsigned long cur_seq) 680 { 681 struct xe_vma *vma = container_of(mni, struct xe_vma, userptr.notifier); 682 struct xe_vm *vm = vma->vm; 683 struct dma_resv_iter cursor; 684 struct dma_fence *fence; 685 long err; 686 687 XE_BUG_ON(!xe_vma_is_userptr(vma)); 688 trace_xe_vma_userptr_invalidate(vma); 689 690 if (!mmu_notifier_range_blockable(range)) 691 return false; 692 693 down_write(&vm->userptr.notifier_lock); 694 mmu_interval_set_seq(mni, cur_seq); 695 696 /* No need to stop gpu access if the userptr is not yet bound. */ 697 if (!vma->userptr.initial_bind) { 698 up_write(&vm->userptr.notifier_lock); 699 return true; 700 } 701 702 /* 703 * Tell exec and rebind worker they need to repin and rebind this 704 * userptr. 705 */ 706 if (!xe_vm_in_fault_mode(vm) && !vma->destroyed && vma->tile_present) { 707 spin_lock(&vm->userptr.invalidated_lock); 708 list_move_tail(&vma->userptr.invalidate_link, 709 &vm->userptr.invalidated); 710 spin_unlock(&vm->userptr.invalidated_lock); 711 } 712 713 up_write(&vm->userptr.notifier_lock); 714 715 /* 716 * Preempt fences turn into schedule disables, pipeline these. 717 * Note that even in fault mode, we need to wait for binds and 718 * unbinds to complete, and those are attached as BOOKMARK fences 719 * to the vm. 720 */ 721 dma_resv_iter_begin(&cursor, &vm->resv, 722 DMA_RESV_USAGE_BOOKKEEP); 723 dma_resv_for_each_fence_unlocked(&cursor, fence) 724 dma_fence_enable_sw_signaling(fence); 725 dma_resv_iter_end(&cursor); 726 727 err = dma_resv_wait_timeout(&vm->resv, 728 DMA_RESV_USAGE_BOOKKEEP, 729 false, MAX_SCHEDULE_TIMEOUT); 730 XE_WARN_ON(err <= 0); 731 732 if (xe_vm_in_fault_mode(vm)) { 733 err = xe_vm_invalidate_vma(vma); 734 XE_WARN_ON(err); 735 } 736 737 trace_xe_vma_userptr_invalidate_complete(vma); 738 739 return true; 740 } 741 742 static const struct mmu_interval_notifier_ops vma_userptr_notifier_ops = { 743 .invalidate = vma_userptr_invalidate, 744 }; 745 746 int xe_vm_userptr_pin(struct xe_vm *vm) 747 { 748 struct xe_vma *vma, *next; 749 int err = 0; 750 LIST_HEAD(tmp_evict); 751 752 lockdep_assert_held_write(&vm->lock); 753 754 /* Collect invalidated userptrs */ 755 spin_lock(&vm->userptr.invalidated_lock); 756 list_for_each_entry_safe(vma, next, &vm->userptr.invalidated, 757 userptr.invalidate_link) { 758 list_del_init(&vma->userptr.invalidate_link); 759 list_move_tail(&vma->userptr_link, &vm->userptr.repin_list); 760 } 761 spin_unlock(&vm->userptr.invalidated_lock); 762 763 /* Pin and move to temporary list */ 764 list_for_each_entry_safe(vma, next, &vm->userptr.repin_list, userptr_link) { 765 err = xe_vma_userptr_pin_pages(vma); 766 if (err < 0) 767 goto out_err; 768 769 list_move_tail(&vma->userptr_link, &tmp_evict); 770 } 771 772 /* Take lock and move to rebind_list for rebinding. */ 773 err = dma_resv_lock_interruptible(&vm->resv, NULL); 774 if (err) 775 goto out_err; 776 777 list_for_each_entry_safe(vma, next, &tmp_evict, userptr_link) { 778 list_del_init(&vma->userptr_link); 779 list_move_tail(&vma->rebind_link, &vm->rebind_list); 780 } 781 782 dma_resv_unlock(&vm->resv); 783 784 return 0; 785 786 out_err: 787 list_splice_tail(&tmp_evict, &vm->userptr.repin_list); 788 789 return err; 790 } 791 792 /** 793 * xe_vm_userptr_check_repin() - Check whether the VM might have userptrs 794 * that need repinning. 795 * @vm: The VM. 796 * 797 * This function does an advisory check for whether the VM has userptrs that 798 * need repinning. 799 * 800 * Return: 0 if there are no indications of userptrs needing repinning, 801 * -EAGAIN if there are. 802 */ 803 int xe_vm_userptr_check_repin(struct xe_vm *vm) 804 { 805 return (list_empty_careful(&vm->userptr.repin_list) && 806 list_empty_careful(&vm->userptr.invalidated)) ? 0 : -EAGAIN; 807 } 808 809 static struct dma_fence * 810 xe_vm_bind_vma(struct xe_vma *vma, struct xe_engine *e, 811 struct xe_sync_entry *syncs, u32 num_syncs); 812 813 struct dma_fence *xe_vm_rebind(struct xe_vm *vm, bool rebind_worker) 814 { 815 struct dma_fence *fence = NULL; 816 struct xe_vma *vma, *next; 817 818 lockdep_assert_held(&vm->lock); 819 if (xe_vm_no_dma_fences(vm) && !rebind_worker) 820 return NULL; 821 822 xe_vm_assert_held(vm); 823 list_for_each_entry_safe(vma, next, &vm->rebind_list, rebind_link) { 824 XE_WARN_ON(!vma->tile_present); 825 826 list_del_init(&vma->rebind_link); 827 dma_fence_put(fence); 828 if (rebind_worker) 829 trace_xe_vma_rebind_worker(vma); 830 else 831 trace_xe_vma_rebind_exec(vma); 832 fence = xe_vm_bind_vma(vma, NULL, NULL, 0); 833 if (IS_ERR(fence)) 834 return fence; 835 } 836 837 return fence; 838 } 839 840 static struct xe_vma *xe_vma_create(struct xe_vm *vm, 841 struct xe_bo *bo, 842 u64 bo_offset_or_userptr, 843 u64 start, u64 end, 844 bool read_only, 845 u64 tile_mask) 846 { 847 struct xe_vma *vma; 848 struct xe_tile *tile; 849 u8 id; 850 851 XE_BUG_ON(start >= end); 852 XE_BUG_ON(end >= vm->size); 853 854 vma = kzalloc(sizeof(*vma), GFP_KERNEL); 855 if (!vma) { 856 vma = ERR_PTR(-ENOMEM); 857 return vma; 858 } 859 860 INIT_LIST_HEAD(&vma->rebind_link); 861 INIT_LIST_HEAD(&vma->unbind_link); 862 INIT_LIST_HEAD(&vma->userptr_link); 863 INIT_LIST_HEAD(&vma->userptr.invalidate_link); 864 INIT_LIST_HEAD(&vma->notifier.rebind_link); 865 INIT_LIST_HEAD(&vma->extobj.link); 866 867 vma->vm = vm; 868 vma->start = start; 869 vma->end = end; 870 if (read_only) 871 vma->pte_flags = XE_PTE_READ_ONLY; 872 873 if (tile_mask) { 874 vma->tile_mask = tile_mask; 875 } else { 876 for_each_tile(tile, vm->xe, id) 877 vma->tile_mask |= 0x1 << id; 878 } 879 880 if (vm->xe->info.platform == XE_PVC) 881 vma->use_atomic_access_pte_bit = true; 882 883 if (bo) { 884 xe_bo_assert_held(bo); 885 vma->bo_offset = bo_offset_or_userptr; 886 vma->bo = xe_bo_get(bo); 887 list_add_tail(&vma->bo_link, &bo->vmas); 888 } else /* userptr */ { 889 u64 size = end - start + 1; 890 int err; 891 892 vma->userptr.ptr = bo_offset_or_userptr; 893 894 err = mmu_interval_notifier_insert(&vma->userptr.notifier, 895 current->mm, 896 vma->userptr.ptr, size, 897 &vma_userptr_notifier_ops); 898 if (err) { 899 kfree(vma); 900 vma = ERR_PTR(err); 901 return vma; 902 } 903 904 vma->userptr.notifier_seq = LONG_MAX; 905 xe_vm_get(vm); 906 } 907 908 return vma; 909 } 910 911 static bool vm_remove_extobj(struct xe_vma *vma) 912 { 913 if (!list_empty(&vma->extobj.link)) { 914 vma->vm->extobj.entries--; 915 list_del_init(&vma->extobj.link); 916 return true; 917 } 918 return false; 919 } 920 921 static void xe_vma_destroy_late(struct xe_vma *vma) 922 { 923 struct xe_vm *vm = vma->vm; 924 struct xe_device *xe = vm->xe; 925 bool read_only = vma->pte_flags & XE_PTE_READ_ONLY; 926 927 if (xe_vma_is_userptr(vma)) { 928 if (vma->userptr.sg) { 929 dma_unmap_sgtable(xe->drm.dev, 930 vma->userptr.sg, 931 read_only ? DMA_TO_DEVICE : 932 DMA_BIDIRECTIONAL, 0); 933 sg_free_table(vma->userptr.sg); 934 vma->userptr.sg = NULL; 935 } 936 937 /* 938 * Since userptr pages are not pinned, we can't remove 939 * the notifer until we're sure the GPU is not accessing 940 * them anymore 941 */ 942 mmu_interval_notifier_remove(&vma->userptr.notifier); 943 xe_vm_put(vm); 944 } else { 945 xe_bo_put(vma->bo); 946 } 947 948 kfree(vma); 949 } 950 951 static void vma_destroy_work_func(struct work_struct *w) 952 { 953 struct xe_vma *vma = 954 container_of(w, struct xe_vma, destroy_work); 955 956 xe_vma_destroy_late(vma); 957 } 958 959 static struct xe_vma * 960 bo_has_vm_references_locked(struct xe_bo *bo, struct xe_vm *vm, 961 struct xe_vma *ignore) 962 { 963 struct xe_vma *vma; 964 965 list_for_each_entry(vma, &bo->vmas, bo_link) { 966 if (vma != ignore && vma->vm == vm) 967 return vma; 968 } 969 970 return NULL; 971 } 972 973 static bool bo_has_vm_references(struct xe_bo *bo, struct xe_vm *vm, 974 struct xe_vma *ignore) 975 { 976 struct ww_acquire_ctx ww; 977 bool ret; 978 979 xe_bo_lock(bo, &ww, 0, false); 980 ret = !!bo_has_vm_references_locked(bo, vm, ignore); 981 xe_bo_unlock(bo, &ww); 982 983 return ret; 984 } 985 986 static void __vm_insert_extobj(struct xe_vm *vm, struct xe_vma *vma) 987 { 988 list_add(&vma->extobj.link, &vm->extobj.list); 989 vm->extobj.entries++; 990 } 991 992 static void vm_insert_extobj(struct xe_vm *vm, struct xe_vma *vma) 993 { 994 struct xe_bo *bo = vma->bo; 995 996 lockdep_assert_held_write(&vm->lock); 997 998 if (bo_has_vm_references(bo, vm, vma)) 999 return; 1000 1001 __vm_insert_extobj(vm, vma); 1002 } 1003 1004 static void vma_destroy_cb(struct dma_fence *fence, 1005 struct dma_fence_cb *cb) 1006 { 1007 struct xe_vma *vma = container_of(cb, struct xe_vma, destroy_cb); 1008 1009 INIT_WORK(&vma->destroy_work, vma_destroy_work_func); 1010 queue_work(system_unbound_wq, &vma->destroy_work); 1011 } 1012 1013 static void xe_vma_destroy(struct xe_vma *vma, struct dma_fence *fence) 1014 { 1015 struct xe_vm *vm = vma->vm; 1016 1017 lockdep_assert_held_write(&vm->lock); 1018 XE_BUG_ON(!list_empty(&vma->unbind_link)); 1019 1020 if (xe_vma_is_userptr(vma)) { 1021 XE_WARN_ON(!vma->destroyed); 1022 spin_lock(&vm->userptr.invalidated_lock); 1023 list_del_init(&vma->userptr.invalidate_link); 1024 spin_unlock(&vm->userptr.invalidated_lock); 1025 list_del(&vma->userptr_link); 1026 } else { 1027 xe_bo_assert_held(vma->bo); 1028 list_del(&vma->bo_link); 1029 1030 spin_lock(&vm->notifier.list_lock); 1031 list_del(&vma->notifier.rebind_link); 1032 spin_unlock(&vm->notifier.list_lock); 1033 1034 if (!vma->bo->vm && vm_remove_extobj(vma)) { 1035 struct xe_vma *other; 1036 1037 other = bo_has_vm_references_locked(vma->bo, vm, NULL); 1038 1039 if (other) 1040 __vm_insert_extobj(vm, other); 1041 } 1042 } 1043 1044 xe_vm_assert_held(vm); 1045 if (!list_empty(&vma->rebind_link)) 1046 list_del(&vma->rebind_link); 1047 1048 if (fence) { 1049 int ret = dma_fence_add_callback(fence, &vma->destroy_cb, 1050 vma_destroy_cb); 1051 1052 if (ret) { 1053 XE_WARN_ON(ret != -ENOENT); 1054 xe_vma_destroy_late(vma); 1055 } 1056 } else { 1057 xe_vma_destroy_late(vma); 1058 } 1059 } 1060 1061 static void xe_vma_destroy_unlocked(struct xe_vma *vma) 1062 { 1063 struct ttm_validate_buffer tv[2]; 1064 struct ww_acquire_ctx ww; 1065 struct xe_bo *bo = vma->bo; 1066 LIST_HEAD(objs); 1067 LIST_HEAD(dups); 1068 int err; 1069 1070 memset(tv, 0, sizeof(tv)); 1071 tv[0].bo = xe_vm_ttm_bo(vma->vm); 1072 list_add(&tv[0].head, &objs); 1073 1074 if (bo) { 1075 tv[1].bo = &xe_bo_get(bo)->ttm; 1076 list_add(&tv[1].head, &objs); 1077 } 1078 err = ttm_eu_reserve_buffers(&ww, &objs, false, &dups); 1079 XE_WARN_ON(err); 1080 1081 xe_vma_destroy(vma, NULL); 1082 1083 ttm_eu_backoff_reservation(&ww, &objs); 1084 if (bo) 1085 xe_bo_put(bo); 1086 } 1087 1088 static struct xe_vma *to_xe_vma(const struct rb_node *node) 1089 { 1090 BUILD_BUG_ON(offsetof(struct xe_vma, vm_node) != 0); 1091 return (struct xe_vma *)node; 1092 } 1093 1094 static int xe_vma_cmp(const struct xe_vma *a, const struct xe_vma *b) 1095 { 1096 if (a->end < b->start) { 1097 return -1; 1098 } else if (b->end < a->start) { 1099 return 1; 1100 } else { 1101 return 0; 1102 } 1103 } 1104 1105 static bool xe_vma_less_cb(struct rb_node *a, const struct rb_node *b) 1106 { 1107 return xe_vma_cmp(to_xe_vma(a), to_xe_vma(b)) < 0; 1108 } 1109 1110 int xe_vma_cmp_vma_cb(const void *key, const struct rb_node *node) 1111 { 1112 struct xe_vma *cmp = to_xe_vma(node); 1113 const struct xe_vma *own = key; 1114 1115 if (own->start > cmp->end) 1116 return 1; 1117 1118 if (own->end < cmp->start) 1119 return -1; 1120 1121 return 0; 1122 } 1123 1124 struct xe_vma * 1125 xe_vm_find_overlapping_vma(struct xe_vm *vm, const struct xe_vma *vma) 1126 { 1127 struct rb_node *node; 1128 1129 if (xe_vm_is_closed(vm)) 1130 return NULL; 1131 1132 XE_BUG_ON(vma->end >= vm->size); 1133 lockdep_assert_held(&vm->lock); 1134 1135 node = rb_find(vma, &vm->vmas, xe_vma_cmp_vma_cb); 1136 1137 return node ? to_xe_vma(node) : NULL; 1138 } 1139 1140 static void xe_vm_insert_vma(struct xe_vm *vm, struct xe_vma *vma) 1141 { 1142 XE_BUG_ON(vma->vm != vm); 1143 lockdep_assert_held(&vm->lock); 1144 1145 rb_add(&vma->vm_node, &vm->vmas, xe_vma_less_cb); 1146 } 1147 1148 static void xe_vm_remove_vma(struct xe_vm *vm, struct xe_vma *vma) 1149 { 1150 XE_BUG_ON(vma->vm != vm); 1151 lockdep_assert_held(&vm->lock); 1152 1153 rb_erase(&vma->vm_node, &vm->vmas); 1154 if (vm->usm.last_fault_vma == vma) 1155 vm->usm.last_fault_vma = NULL; 1156 } 1157 1158 static void async_op_work_func(struct work_struct *w); 1159 static void vm_destroy_work_func(struct work_struct *w); 1160 1161 struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags) 1162 { 1163 struct xe_vm *vm; 1164 int err, i = 0, number_tiles = 0; 1165 struct xe_tile *tile; 1166 u8 id; 1167 1168 vm = kzalloc(sizeof(*vm), GFP_KERNEL); 1169 if (!vm) 1170 return ERR_PTR(-ENOMEM); 1171 1172 vm->xe = xe; 1173 kref_init(&vm->refcount); 1174 dma_resv_init(&vm->resv); 1175 1176 vm->size = 1ull << xe_pt_shift(xe->info.vm_max_level + 1); 1177 1178 vm->vmas = RB_ROOT; 1179 vm->flags = flags; 1180 1181 init_rwsem(&vm->lock); 1182 1183 INIT_LIST_HEAD(&vm->rebind_list); 1184 1185 INIT_LIST_HEAD(&vm->userptr.repin_list); 1186 INIT_LIST_HEAD(&vm->userptr.invalidated); 1187 init_rwsem(&vm->userptr.notifier_lock); 1188 spin_lock_init(&vm->userptr.invalidated_lock); 1189 1190 INIT_LIST_HEAD(&vm->notifier.rebind_list); 1191 spin_lock_init(&vm->notifier.list_lock); 1192 1193 INIT_LIST_HEAD(&vm->async_ops.pending); 1194 INIT_WORK(&vm->async_ops.work, async_op_work_func); 1195 spin_lock_init(&vm->async_ops.lock); 1196 1197 INIT_WORK(&vm->destroy_work, vm_destroy_work_func); 1198 1199 INIT_LIST_HEAD(&vm->preempt.engines); 1200 vm->preempt.min_run_period_ms = 10; /* FIXME: Wire up to uAPI */ 1201 1202 INIT_LIST_HEAD(&vm->extobj.list); 1203 1204 if (!(flags & XE_VM_FLAG_MIGRATION)) { 1205 /* We need to immeditatelly exit from any D3 state */ 1206 xe_pm_runtime_get(xe); 1207 xe_device_mem_access_get(xe); 1208 } 1209 1210 err = dma_resv_lock_interruptible(&vm->resv, NULL); 1211 if (err) 1212 goto err_put; 1213 1214 if (IS_DGFX(xe) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) 1215 vm->flags |= XE_VM_FLAGS_64K; 1216 1217 for_each_tile(tile, xe, id) { 1218 if (flags & XE_VM_FLAG_MIGRATION && 1219 tile->id != XE_VM_FLAG_GT_ID(flags)) 1220 continue; 1221 1222 vm->pt_root[id] = xe_pt_create(vm, tile, xe->info.vm_max_level); 1223 if (IS_ERR(vm->pt_root[id])) { 1224 err = PTR_ERR(vm->pt_root[id]); 1225 vm->pt_root[id] = NULL; 1226 goto err_destroy_root; 1227 } 1228 } 1229 1230 if (flags & XE_VM_FLAG_SCRATCH_PAGE) { 1231 for_each_tile(tile, xe, id) { 1232 if (!vm->pt_root[id]) 1233 continue; 1234 1235 err = xe_pt_create_scratch(xe, tile, vm); 1236 if (err) 1237 goto err_scratch_pt; 1238 } 1239 } 1240 1241 if (flags & DRM_XE_VM_CREATE_COMPUTE_MODE) { 1242 INIT_WORK(&vm->preempt.rebind_work, preempt_rebind_work_func); 1243 vm->flags |= XE_VM_FLAG_COMPUTE_MODE; 1244 } 1245 1246 if (flags & DRM_XE_VM_CREATE_ASYNC_BIND_OPS) { 1247 vm->async_ops.fence.context = dma_fence_context_alloc(1); 1248 vm->flags |= XE_VM_FLAG_ASYNC_BIND_OPS; 1249 } 1250 1251 /* Fill pt_root after allocating scratch tables */ 1252 for_each_tile(tile, xe, id) { 1253 if (!vm->pt_root[id]) 1254 continue; 1255 1256 xe_pt_populate_empty(tile, vm, vm->pt_root[id]); 1257 } 1258 dma_resv_unlock(&vm->resv); 1259 1260 /* Kernel migration VM shouldn't have a circular loop.. */ 1261 if (!(flags & XE_VM_FLAG_MIGRATION)) { 1262 for_each_tile(tile, xe, id) { 1263 struct xe_gt *gt = tile->primary_gt; 1264 struct xe_vm *migrate_vm; 1265 struct xe_engine *eng; 1266 1267 if (!vm->pt_root[id]) 1268 continue; 1269 1270 migrate_vm = xe_migrate_get_vm(tile->migrate); 1271 eng = xe_engine_create_class(xe, gt, migrate_vm, 1272 XE_ENGINE_CLASS_COPY, 1273 ENGINE_FLAG_VM); 1274 xe_vm_put(migrate_vm); 1275 if (IS_ERR(eng)) { 1276 xe_vm_close_and_put(vm); 1277 return ERR_CAST(eng); 1278 } 1279 vm->eng[id] = eng; 1280 number_tiles++; 1281 } 1282 } 1283 1284 if (number_tiles > 1) 1285 vm->composite_fence_ctx = dma_fence_context_alloc(1); 1286 1287 mutex_lock(&xe->usm.lock); 1288 if (flags & XE_VM_FLAG_FAULT_MODE) 1289 xe->usm.num_vm_in_fault_mode++; 1290 else if (!(flags & XE_VM_FLAG_MIGRATION)) 1291 xe->usm.num_vm_in_non_fault_mode++; 1292 mutex_unlock(&xe->usm.lock); 1293 1294 trace_xe_vm_create(vm); 1295 1296 return vm; 1297 1298 err_scratch_pt: 1299 for_each_tile(tile, xe, id) { 1300 if (!vm->pt_root[id]) 1301 continue; 1302 1303 i = vm->pt_root[id]->level; 1304 while (i) 1305 if (vm->scratch_pt[id][--i]) 1306 xe_pt_destroy(vm->scratch_pt[id][i], 1307 vm->flags, NULL); 1308 xe_bo_unpin(vm->scratch_bo[id]); 1309 xe_bo_put(vm->scratch_bo[id]); 1310 } 1311 err_destroy_root: 1312 for_each_tile(tile, xe, id) { 1313 if (vm->pt_root[id]) 1314 xe_pt_destroy(vm->pt_root[id], vm->flags, NULL); 1315 } 1316 dma_resv_unlock(&vm->resv); 1317 err_put: 1318 dma_resv_fini(&vm->resv); 1319 kfree(vm); 1320 if (!(flags & XE_VM_FLAG_MIGRATION)) { 1321 xe_device_mem_access_put(xe); 1322 xe_pm_runtime_put(xe); 1323 } 1324 return ERR_PTR(err); 1325 } 1326 1327 static void flush_async_ops(struct xe_vm *vm) 1328 { 1329 queue_work(system_unbound_wq, &vm->async_ops.work); 1330 flush_work(&vm->async_ops.work); 1331 } 1332 1333 static void vm_error_capture(struct xe_vm *vm, int err, 1334 u32 op, u64 addr, u64 size) 1335 { 1336 struct drm_xe_vm_bind_op_error_capture capture; 1337 u64 __user *address = 1338 u64_to_user_ptr(vm->async_ops.error_capture.addr); 1339 bool in_kthread = !current->mm; 1340 1341 capture.error = err; 1342 capture.op = op; 1343 capture.addr = addr; 1344 capture.size = size; 1345 1346 if (in_kthread) { 1347 if (!mmget_not_zero(vm->async_ops.error_capture.mm)) 1348 goto mm_closed; 1349 kthread_use_mm(vm->async_ops.error_capture.mm); 1350 } 1351 1352 if (copy_to_user(address, &capture, sizeof(capture))) 1353 XE_WARN_ON("Copy to user failed"); 1354 1355 if (in_kthread) { 1356 kthread_unuse_mm(vm->async_ops.error_capture.mm); 1357 mmput(vm->async_ops.error_capture.mm); 1358 } 1359 1360 mm_closed: 1361 wake_up_all(&vm->async_ops.error_capture.wq); 1362 } 1363 1364 void xe_vm_close_and_put(struct xe_vm *vm) 1365 { 1366 struct rb_root contested = RB_ROOT; 1367 struct ww_acquire_ctx ww; 1368 struct xe_device *xe = vm->xe; 1369 struct xe_tile *tile; 1370 u8 id; 1371 1372 XE_BUG_ON(vm->preempt.num_engines); 1373 1374 vm->size = 0; 1375 smp_mb(); 1376 flush_async_ops(vm); 1377 if (xe_vm_in_compute_mode(vm)) 1378 flush_work(&vm->preempt.rebind_work); 1379 1380 for_each_tile(tile, xe, id) { 1381 if (vm->eng[id]) { 1382 xe_engine_kill(vm->eng[id]); 1383 xe_engine_put(vm->eng[id]); 1384 vm->eng[id] = NULL; 1385 } 1386 } 1387 1388 down_write(&vm->lock); 1389 xe_vm_lock(vm, &ww, 0, false); 1390 while (vm->vmas.rb_node) { 1391 struct xe_vma *vma = to_xe_vma(vm->vmas.rb_node); 1392 1393 if (xe_vma_is_userptr(vma)) { 1394 down_read(&vm->userptr.notifier_lock); 1395 vma->destroyed = true; 1396 up_read(&vm->userptr.notifier_lock); 1397 } 1398 1399 rb_erase(&vma->vm_node, &vm->vmas); 1400 1401 /* easy case, remove from VMA? */ 1402 if (xe_vma_is_userptr(vma) || vma->bo->vm) { 1403 xe_vma_destroy(vma, NULL); 1404 continue; 1405 } 1406 1407 rb_add(&vma->vm_node, &contested, xe_vma_less_cb); 1408 } 1409 1410 /* 1411 * All vm operations will add shared fences to resv. 1412 * The only exception is eviction for a shared object, 1413 * but even so, the unbind when evicted would still 1414 * install a fence to resv. Hence it's safe to 1415 * destroy the pagetables immediately. 1416 */ 1417 for_each_tile(tile, xe, id) { 1418 if (vm->scratch_bo[id]) { 1419 u32 i; 1420 1421 xe_bo_unpin(vm->scratch_bo[id]); 1422 xe_bo_put(vm->scratch_bo[id]); 1423 for (i = 0; i < vm->pt_root[id]->level; i++) 1424 xe_pt_destroy(vm->scratch_pt[id][i], vm->flags, 1425 NULL); 1426 } 1427 } 1428 xe_vm_unlock(vm, &ww); 1429 1430 if (contested.rb_node) { 1431 1432 /* 1433 * VM is now dead, cannot re-add nodes to vm->vmas if it's NULL 1434 * Since we hold a refcount to the bo, we can remove and free 1435 * the members safely without locking. 1436 */ 1437 while (contested.rb_node) { 1438 struct xe_vma *vma = to_xe_vma(contested.rb_node); 1439 1440 rb_erase(&vma->vm_node, &contested); 1441 xe_vma_destroy_unlocked(vma); 1442 } 1443 } 1444 1445 if (vm->async_ops.error_capture.addr) 1446 wake_up_all(&vm->async_ops.error_capture.wq); 1447 1448 XE_WARN_ON(!list_empty(&vm->extobj.list)); 1449 up_write(&vm->lock); 1450 1451 mutex_lock(&xe->usm.lock); 1452 if (vm->flags & XE_VM_FLAG_FAULT_MODE) 1453 xe->usm.num_vm_in_fault_mode--; 1454 else if (!(vm->flags & XE_VM_FLAG_MIGRATION)) 1455 xe->usm.num_vm_in_non_fault_mode--; 1456 mutex_unlock(&xe->usm.lock); 1457 1458 xe_vm_put(vm); 1459 } 1460 1461 static void vm_destroy_work_func(struct work_struct *w) 1462 { 1463 struct xe_vm *vm = 1464 container_of(w, struct xe_vm, destroy_work); 1465 struct ww_acquire_ctx ww; 1466 struct xe_device *xe = vm->xe; 1467 struct xe_tile *tile; 1468 u8 id; 1469 void *lookup; 1470 1471 /* xe_vm_close_and_put was not called? */ 1472 XE_WARN_ON(vm->size); 1473 1474 if (!(vm->flags & XE_VM_FLAG_MIGRATION)) { 1475 xe_device_mem_access_put(xe); 1476 xe_pm_runtime_put(xe); 1477 1478 if (xe->info.has_asid) { 1479 mutex_lock(&xe->usm.lock); 1480 lookup = xa_erase(&xe->usm.asid_to_vm, vm->usm.asid); 1481 XE_WARN_ON(lookup != vm); 1482 mutex_unlock(&xe->usm.lock); 1483 } 1484 } 1485 1486 /* 1487 * XXX: We delay destroying the PT root until the VM if freed as PT root 1488 * is needed for xe_vm_lock to work. If we remove that dependency this 1489 * can be moved to xe_vm_close_and_put. 1490 */ 1491 xe_vm_lock(vm, &ww, 0, false); 1492 for_each_tile(tile, xe, id) { 1493 if (vm->pt_root[id]) { 1494 xe_pt_destroy(vm->pt_root[id], vm->flags, NULL); 1495 vm->pt_root[id] = NULL; 1496 } 1497 } 1498 xe_vm_unlock(vm, &ww); 1499 1500 trace_xe_vm_free(vm); 1501 dma_fence_put(vm->rebind_fence); 1502 dma_resv_fini(&vm->resv); 1503 kfree(vm); 1504 } 1505 1506 void xe_vm_free(struct kref *ref) 1507 { 1508 struct xe_vm *vm = container_of(ref, struct xe_vm, refcount); 1509 1510 /* To destroy the VM we need to be able to sleep */ 1511 queue_work(system_unbound_wq, &vm->destroy_work); 1512 } 1513 1514 struct xe_vm *xe_vm_lookup(struct xe_file *xef, u32 id) 1515 { 1516 struct xe_vm *vm; 1517 1518 mutex_lock(&xef->vm.lock); 1519 vm = xa_load(&xef->vm.xa, id); 1520 mutex_unlock(&xef->vm.lock); 1521 1522 if (vm) 1523 xe_vm_get(vm); 1524 1525 return vm; 1526 } 1527 1528 u64 xe_vm_pdp4_descriptor(struct xe_vm *vm, struct xe_tile *tile) 1529 { 1530 return gen8_pde_encode(vm->pt_root[tile->id]->bo, 0, 1531 XE_CACHE_WB); 1532 } 1533 1534 static struct dma_fence * 1535 xe_vm_unbind_vma(struct xe_vma *vma, struct xe_engine *e, 1536 struct xe_sync_entry *syncs, u32 num_syncs) 1537 { 1538 struct xe_tile *tile; 1539 struct dma_fence *fence = NULL; 1540 struct dma_fence **fences = NULL; 1541 struct dma_fence_array *cf = NULL; 1542 struct xe_vm *vm = vma->vm; 1543 int cur_fence = 0, i; 1544 int number_tiles = hweight_long(vma->tile_present); 1545 int err; 1546 u8 id; 1547 1548 trace_xe_vma_unbind(vma); 1549 1550 if (number_tiles > 1) { 1551 fences = kmalloc_array(number_tiles, sizeof(*fences), 1552 GFP_KERNEL); 1553 if (!fences) 1554 return ERR_PTR(-ENOMEM); 1555 } 1556 1557 for_each_tile(tile, vm->xe, id) { 1558 if (!(vma->tile_present & BIT(id))) 1559 goto next; 1560 1561 fence = __xe_pt_unbind_vma(tile, vma, e, syncs, num_syncs); 1562 if (IS_ERR(fence)) { 1563 err = PTR_ERR(fence); 1564 goto err_fences; 1565 } 1566 1567 if (fences) 1568 fences[cur_fence++] = fence; 1569 1570 next: 1571 if (e && vm->pt_root[id] && !list_empty(&e->multi_gt_list)) 1572 e = list_next_entry(e, multi_gt_list); 1573 } 1574 1575 if (fences) { 1576 cf = dma_fence_array_create(number_tiles, fences, 1577 vm->composite_fence_ctx, 1578 vm->composite_fence_seqno++, 1579 false); 1580 if (!cf) { 1581 --vm->composite_fence_seqno; 1582 err = -ENOMEM; 1583 goto err_fences; 1584 } 1585 } 1586 1587 for (i = 0; i < num_syncs; i++) 1588 xe_sync_entry_signal(&syncs[i], NULL, cf ? &cf->base : fence); 1589 1590 return cf ? &cf->base : !fence ? dma_fence_get_stub() : fence; 1591 1592 err_fences: 1593 if (fences) { 1594 while (cur_fence) { 1595 /* FIXME: Rewind the previous binds? */ 1596 dma_fence_put(fences[--cur_fence]); 1597 } 1598 kfree(fences); 1599 } 1600 1601 return ERR_PTR(err); 1602 } 1603 1604 static struct dma_fence * 1605 xe_vm_bind_vma(struct xe_vma *vma, struct xe_engine *e, 1606 struct xe_sync_entry *syncs, u32 num_syncs) 1607 { 1608 struct xe_tile *tile; 1609 struct dma_fence *fence; 1610 struct dma_fence **fences = NULL; 1611 struct dma_fence_array *cf = NULL; 1612 struct xe_vm *vm = vma->vm; 1613 int cur_fence = 0, i; 1614 int number_tiles = hweight_long(vma->tile_mask); 1615 int err; 1616 u8 id; 1617 1618 trace_xe_vma_bind(vma); 1619 1620 if (number_tiles > 1) { 1621 fences = kmalloc_array(number_tiles, sizeof(*fences), 1622 GFP_KERNEL); 1623 if (!fences) 1624 return ERR_PTR(-ENOMEM); 1625 } 1626 1627 for_each_tile(tile, vm->xe, id) { 1628 if (!(vma->tile_mask & BIT(id))) 1629 goto next; 1630 1631 fence = __xe_pt_bind_vma(tile, vma, e, syncs, num_syncs, 1632 vma->tile_present & BIT(id)); 1633 if (IS_ERR(fence)) { 1634 err = PTR_ERR(fence); 1635 goto err_fences; 1636 } 1637 1638 if (fences) 1639 fences[cur_fence++] = fence; 1640 1641 next: 1642 if (e && vm->pt_root[id] && !list_empty(&e->multi_gt_list)) 1643 e = list_next_entry(e, multi_gt_list); 1644 } 1645 1646 if (fences) { 1647 cf = dma_fence_array_create(number_tiles, fences, 1648 vm->composite_fence_ctx, 1649 vm->composite_fence_seqno++, 1650 false); 1651 if (!cf) { 1652 --vm->composite_fence_seqno; 1653 err = -ENOMEM; 1654 goto err_fences; 1655 } 1656 } 1657 1658 for (i = 0; i < num_syncs; i++) 1659 xe_sync_entry_signal(&syncs[i], NULL, cf ? &cf->base : fence); 1660 1661 return cf ? &cf->base : fence; 1662 1663 err_fences: 1664 if (fences) { 1665 while (cur_fence) { 1666 /* FIXME: Rewind the previous binds? */ 1667 dma_fence_put(fences[--cur_fence]); 1668 } 1669 kfree(fences); 1670 } 1671 1672 return ERR_PTR(err); 1673 } 1674 1675 struct async_op_fence { 1676 struct dma_fence fence; 1677 struct dma_fence *wait_fence; 1678 struct dma_fence_cb cb; 1679 struct xe_vm *vm; 1680 wait_queue_head_t wq; 1681 bool started; 1682 }; 1683 1684 static const char *async_op_fence_get_driver_name(struct dma_fence *dma_fence) 1685 { 1686 return "xe"; 1687 } 1688 1689 static const char * 1690 async_op_fence_get_timeline_name(struct dma_fence *dma_fence) 1691 { 1692 return "async_op_fence"; 1693 } 1694 1695 static const struct dma_fence_ops async_op_fence_ops = { 1696 .get_driver_name = async_op_fence_get_driver_name, 1697 .get_timeline_name = async_op_fence_get_timeline_name, 1698 }; 1699 1700 static void async_op_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb) 1701 { 1702 struct async_op_fence *afence = 1703 container_of(cb, struct async_op_fence, cb); 1704 1705 afence->fence.error = afence->wait_fence->error; 1706 dma_fence_signal(&afence->fence); 1707 xe_vm_put(afence->vm); 1708 dma_fence_put(afence->wait_fence); 1709 dma_fence_put(&afence->fence); 1710 } 1711 1712 static void add_async_op_fence_cb(struct xe_vm *vm, 1713 struct dma_fence *fence, 1714 struct async_op_fence *afence) 1715 { 1716 int ret; 1717 1718 if (!xe_vm_no_dma_fences(vm)) { 1719 afence->started = true; 1720 smp_wmb(); 1721 wake_up_all(&afence->wq); 1722 } 1723 1724 afence->wait_fence = dma_fence_get(fence); 1725 afence->vm = xe_vm_get(vm); 1726 dma_fence_get(&afence->fence); 1727 ret = dma_fence_add_callback(fence, &afence->cb, async_op_fence_cb); 1728 if (ret == -ENOENT) { 1729 afence->fence.error = afence->wait_fence->error; 1730 dma_fence_signal(&afence->fence); 1731 } 1732 if (ret) { 1733 xe_vm_put(vm); 1734 dma_fence_put(afence->wait_fence); 1735 dma_fence_put(&afence->fence); 1736 } 1737 XE_WARN_ON(ret && ret != -ENOENT); 1738 } 1739 1740 int xe_vm_async_fence_wait_start(struct dma_fence *fence) 1741 { 1742 if (fence->ops == &async_op_fence_ops) { 1743 struct async_op_fence *afence = 1744 container_of(fence, struct async_op_fence, fence); 1745 1746 XE_BUG_ON(xe_vm_no_dma_fences(afence->vm)); 1747 1748 smp_rmb(); 1749 return wait_event_interruptible(afence->wq, afence->started); 1750 } 1751 1752 return 0; 1753 } 1754 1755 static int __xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, 1756 struct xe_engine *e, struct xe_sync_entry *syncs, 1757 u32 num_syncs, struct async_op_fence *afence) 1758 { 1759 struct dma_fence *fence; 1760 1761 xe_vm_assert_held(vm); 1762 1763 fence = xe_vm_bind_vma(vma, e, syncs, num_syncs); 1764 if (IS_ERR(fence)) 1765 return PTR_ERR(fence); 1766 if (afence) 1767 add_async_op_fence_cb(vm, fence, afence); 1768 1769 dma_fence_put(fence); 1770 return 0; 1771 } 1772 1773 static int xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, struct xe_engine *e, 1774 struct xe_bo *bo, struct xe_sync_entry *syncs, 1775 u32 num_syncs, struct async_op_fence *afence) 1776 { 1777 int err; 1778 1779 xe_vm_assert_held(vm); 1780 xe_bo_assert_held(bo); 1781 1782 if (bo) { 1783 err = xe_bo_validate(bo, vm, true); 1784 if (err) 1785 return err; 1786 } 1787 1788 return __xe_vm_bind(vm, vma, e, syncs, num_syncs, afence); 1789 } 1790 1791 static int xe_vm_unbind(struct xe_vm *vm, struct xe_vma *vma, 1792 struct xe_engine *e, struct xe_sync_entry *syncs, 1793 u32 num_syncs, struct async_op_fence *afence) 1794 { 1795 struct dma_fence *fence; 1796 1797 xe_vm_assert_held(vm); 1798 xe_bo_assert_held(vma->bo); 1799 1800 fence = xe_vm_unbind_vma(vma, e, syncs, num_syncs); 1801 if (IS_ERR(fence)) 1802 return PTR_ERR(fence); 1803 if (afence) 1804 add_async_op_fence_cb(vm, fence, afence); 1805 1806 xe_vma_destroy(vma, fence); 1807 dma_fence_put(fence); 1808 1809 return 0; 1810 } 1811 1812 static int vm_set_error_capture_address(struct xe_device *xe, struct xe_vm *vm, 1813 u64 value) 1814 { 1815 if (XE_IOCTL_ERR(xe, !value)) 1816 return -EINVAL; 1817 1818 if (XE_IOCTL_ERR(xe, !(vm->flags & XE_VM_FLAG_ASYNC_BIND_OPS))) 1819 return -ENOTSUPP; 1820 1821 if (XE_IOCTL_ERR(xe, vm->async_ops.error_capture.addr)) 1822 return -ENOTSUPP; 1823 1824 vm->async_ops.error_capture.mm = current->mm; 1825 vm->async_ops.error_capture.addr = value; 1826 init_waitqueue_head(&vm->async_ops.error_capture.wq); 1827 1828 return 0; 1829 } 1830 1831 typedef int (*xe_vm_set_property_fn)(struct xe_device *xe, struct xe_vm *vm, 1832 u64 value); 1833 1834 static const xe_vm_set_property_fn vm_set_property_funcs[] = { 1835 [XE_VM_PROPERTY_BIND_OP_ERROR_CAPTURE_ADDRESS] = 1836 vm_set_error_capture_address, 1837 }; 1838 1839 static int vm_user_ext_set_property(struct xe_device *xe, struct xe_vm *vm, 1840 u64 extension) 1841 { 1842 u64 __user *address = u64_to_user_ptr(extension); 1843 struct drm_xe_ext_vm_set_property ext; 1844 int err; 1845 1846 err = __copy_from_user(&ext, address, sizeof(ext)); 1847 if (XE_IOCTL_ERR(xe, err)) 1848 return -EFAULT; 1849 1850 if (XE_IOCTL_ERR(xe, ext.property >= 1851 ARRAY_SIZE(vm_set_property_funcs)) || 1852 XE_IOCTL_ERR(xe, ext.pad) || 1853 XE_IOCTL_ERR(xe, ext.reserved[0] || ext.reserved[1])) 1854 return -EINVAL; 1855 1856 return vm_set_property_funcs[ext.property](xe, vm, ext.value); 1857 } 1858 1859 typedef int (*xe_vm_user_extension_fn)(struct xe_device *xe, struct xe_vm *vm, 1860 u64 extension); 1861 1862 static const xe_vm_set_property_fn vm_user_extension_funcs[] = { 1863 [XE_VM_EXTENSION_SET_PROPERTY] = vm_user_ext_set_property, 1864 }; 1865 1866 #define MAX_USER_EXTENSIONS 16 1867 static int vm_user_extensions(struct xe_device *xe, struct xe_vm *vm, 1868 u64 extensions, int ext_number) 1869 { 1870 u64 __user *address = u64_to_user_ptr(extensions); 1871 struct xe_user_extension ext; 1872 int err; 1873 1874 if (XE_IOCTL_ERR(xe, ext_number >= MAX_USER_EXTENSIONS)) 1875 return -E2BIG; 1876 1877 err = __copy_from_user(&ext, address, sizeof(ext)); 1878 if (XE_IOCTL_ERR(xe, err)) 1879 return -EFAULT; 1880 1881 if (XE_IOCTL_ERR(xe, ext.pad) || 1882 XE_IOCTL_ERR(xe, ext.name >= 1883 ARRAY_SIZE(vm_user_extension_funcs))) 1884 return -EINVAL; 1885 1886 err = vm_user_extension_funcs[ext.name](xe, vm, extensions); 1887 if (XE_IOCTL_ERR(xe, err)) 1888 return err; 1889 1890 if (ext.next_extension) 1891 return vm_user_extensions(xe, vm, ext.next_extension, 1892 ++ext_number); 1893 1894 return 0; 1895 } 1896 1897 #define ALL_DRM_XE_VM_CREATE_FLAGS (DRM_XE_VM_CREATE_SCRATCH_PAGE | \ 1898 DRM_XE_VM_CREATE_COMPUTE_MODE | \ 1899 DRM_XE_VM_CREATE_ASYNC_BIND_OPS | \ 1900 DRM_XE_VM_CREATE_FAULT_MODE) 1901 1902 int xe_vm_create_ioctl(struct drm_device *dev, void *data, 1903 struct drm_file *file) 1904 { 1905 struct xe_device *xe = to_xe_device(dev); 1906 struct xe_file *xef = to_xe_file(file); 1907 struct drm_xe_vm_create *args = data; 1908 struct xe_vm *vm; 1909 u32 id, asid; 1910 int err; 1911 u32 flags = 0; 1912 1913 if (XE_IOCTL_ERR(xe, args->reserved[0] || args->reserved[1])) 1914 return -EINVAL; 1915 1916 if (XE_IOCTL_ERR(xe, args->flags & ~ALL_DRM_XE_VM_CREATE_FLAGS)) 1917 return -EINVAL; 1918 1919 if (XE_IOCTL_ERR(xe, args->flags & DRM_XE_VM_CREATE_SCRATCH_PAGE && 1920 args->flags & DRM_XE_VM_CREATE_FAULT_MODE)) 1921 return -EINVAL; 1922 1923 if (XE_IOCTL_ERR(xe, args->flags & DRM_XE_VM_CREATE_COMPUTE_MODE && 1924 args->flags & DRM_XE_VM_CREATE_FAULT_MODE)) 1925 return -EINVAL; 1926 1927 if (XE_IOCTL_ERR(xe, args->flags & DRM_XE_VM_CREATE_FAULT_MODE && 1928 xe_device_in_non_fault_mode(xe))) 1929 return -EINVAL; 1930 1931 if (XE_IOCTL_ERR(xe, !(args->flags & DRM_XE_VM_CREATE_FAULT_MODE) && 1932 xe_device_in_fault_mode(xe))) 1933 return -EINVAL; 1934 1935 if (XE_IOCTL_ERR(xe, args->flags & DRM_XE_VM_CREATE_FAULT_MODE && 1936 !xe->info.supports_usm)) 1937 return -EINVAL; 1938 1939 if (args->flags & DRM_XE_VM_CREATE_SCRATCH_PAGE) 1940 flags |= XE_VM_FLAG_SCRATCH_PAGE; 1941 if (args->flags & DRM_XE_VM_CREATE_COMPUTE_MODE) 1942 flags |= XE_VM_FLAG_COMPUTE_MODE; 1943 if (args->flags & DRM_XE_VM_CREATE_ASYNC_BIND_OPS) 1944 flags |= XE_VM_FLAG_ASYNC_BIND_OPS; 1945 if (args->flags & DRM_XE_VM_CREATE_FAULT_MODE) 1946 flags |= XE_VM_FLAG_FAULT_MODE; 1947 1948 vm = xe_vm_create(xe, flags); 1949 if (IS_ERR(vm)) 1950 return PTR_ERR(vm); 1951 1952 if (args->extensions) { 1953 err = vm_user_extensions(xe, vm, args->extensions, 0); 1954 if (XE_IOCTL_ERR(xe, err)) { 1955 xe_vm_close_and_put(vm); 1956 return err; 1957 } 1958 } 1959 1960 mutex_lock(&xef->vm.lock); 1961 err = xa_alloc(&xef->vm.xa, &id, vm, xa_limit_32b, GFP_KERNEL); 1962 mutex_unlock(&xef->vm.lock); 1963 if (err) { 1964 xe_vm_close_and_put(vm); 1965 return err; 1966 } 1967 1968 if (xe->info.has_asid) { 1969 mutex_lock(&xe->usm.lock); 1970 err = xa_alloc_cyclic(&xe->usm.asid_to_vm, &asid, vm, 1971 XA_LIMIT(0, XE_MAX_ASID - 1), 1972 &xe->usm.next_asid, GFP_KERNEL); 1973 mutex_unlock(&xe->usm.lock); 1974 if (err) { 1975 xe_vm_close_and_put(vm); 1976 return err; 1977 } 1978 vm->usm.asid = asid; 1979 } 1980 1981 args->vm_id = id; 1982 1983 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEM) 1984 /* Warning: Security issue - never enable by default */ 1985 args->reserved[0] = xe_bo_main_addr(vm->pt_root[0]->bo, XE_PAGE_SIZE); 1986 #endif 1987 1988 return 0; 1989 } 1990 1991 int xe_vm_destroy_ioctl(struct drm_device *dev, void *data, 1992 struct drm_file *file) 1993 { 1994 struct xe_device *xe = to_xe_device(dev); 1995 struct xe_file *xef = to_xe_file(file); 1996 struct drm_xe_vm_destroy *args = data; 1997 struct xe_vm *vm; 1998 1999 if (XE_IOCTL_ERR(xe, args->pad) || 2000 XE_IOCTL_ERR(xe, args->reserved[0] || args->reserved[1])) 2001 return -EINVAL; 2002 2003 vm = xe_vm_lookup(xef, args->vm_id); 2004 if (XE_IOCTL_ERR(xe, !vm)) 2005 return -ENOENT; 2006 xe_vm_put(vm); 2007 2008 /* FIXME: Extend this check to non-compute mode VMs */ 2009 if (XE_IOCTL_ERR(xe, vm->preempt.num_engines)) 2010 return -EBUSY; 2011 2012 mutex_lock(&xef->vm.lock); 2013 xa_erase(&xef->vm.xa, args->vm_id); 2014 mutex_unlock(&xef->vm.lock); 2015 2016 xe_vm_close_and_put(vm); 2017 2018 return 0; 2019 } 2020 2021 static const u32 region_to_mem_type[] = { 2022 XE_PL_TT, 2023 XE_PL_VRAM0, 2024 XE_PL_VRAM1, 2025 }; 2026 2027 static int xe_vm_prefetch(struct xe_vm *vm, struct xe_vma *vma, 2028 struct xe_engine *e, u32 region, 2029 struct xe_sync_entry *syncs, u32 num_syncs, 2030 struct async_op_fence *afence) 2031 { 2032 int err; 2033 2034 XE_BUG_ON(region > ARRAY_SIZE(region_to_mem_type)); 2035 2036 if (!xe_vma_is_userptr(vma)) { 2037 err = xe_bo_migrate(vma->bo, region_to_mem_type[region]); 2038 if (err) 2039 return err; 2040 } 2041 2042 if (vma->tile_mask != (vma->tile_present & ~vma->usm.tile_invalidated)) { 2043 return xe_vm_bind(vm, vma, e, vma->bo, syncs, num_syncs, 2044 afence); 2045 } else { 2046 int i; 2047 2048 /* Nothing to do, signal fences now */ 2049 for (i = 0; i < num_syncs; i++) 2050 xe_sync_entry_signal(&syncs[i], NULL, 2051 dma_fence_get_stub()); 2052 if (afence) 2053 dma_fence_signal(&afence->fence); 2054 return 0; 2055 } 2056 } 2057 2058 #define VM_BIND_OP(op) (op & 0xffff) 2059 2060 static int __vm_bind_ioctl(struct xe_vm *vm, struct xe_vma *vma, 2061 struct xe_engine *e, struct xe_bo *bo, u32 op, 2062 u32 region, struct xe_sync_entry *syncs, 2063 u32 num_syncs, struct async_op_fence *afence) 2064 { 2065 switch (VM_BIND_OP(op)) { 2066 case XE_VM_BIND_OP_MAP: 2067 return xe_vm_bind(vm, vma, e, bo, syncs, num_syncs, afence); 2068 case XE_VM_BIND_OP_UNMAP: 2069 case XE_VM_BIND_OP_UNMAP_ALL: 2070 return xe_vm_unbind(vm, vma, e, syncs, num_syncs, afence); 2071 case XE_VM_BIND_OP_MAP_USERPTR: 2072 return xe_vm_bind(vm, vma, e, NULL, syncs, num_syncs, afence); 2073 case XE_VM_BIND_OP_PREFETCH: 2074 return xe_vm_prefetch(vm, vma, e, region, syncs, num_syncs, 2075 afence); 2076 break; 2077 default: 2078 XE_BUG_ON("NOT POSSIBLE"); 2079 return -EINVAL; 2080 } 2081 } 2082 2083 struct ttm_buffer_object *xe_vm_ttm_bo(struct xe_vm *vm) 2084 { 2085 int idx = vm->flags & XE_VM_FLAG_MIGRATION ? 2086 XE_VM_FLAG_GT_ID(vm->flags) : 0; 2087 2088 /* Safe to use index 0 as all BO in the VM share a single dma-resv lock */ 2089 return &vm->pt_root[idx]->bo->ttm; 2090 } 2091 2092 static void xe_vm_tv_populate(struct xe_vm *vm, struct ttm_validate_buffer *tv) 2093 { 2094 tv->num_shared = 1; 2095 tv->bo = xe_vm_ttm_bo(vm); 2096 } 2097 2098 static bool is_map_op(u32 op) 2099 { 2100 return VM_BIND_OP(op) == XE_VM_BIND_OP_MAP || 2101 VM_BIND_OP(op) == XE_VM_BIND_OP_MAP_USERPTR; 2102 } 2103 2104 static bool is_unmap_op(u32 op) 2105 { 2106 return VM_BIND_OP(op) == XE_VM_BIND_OP_UNMAP || 2107 VM_BIND_OP(op) == XE_VM_BIND_OP_UNMAP_ALL; 2108 } 2109 2110 static int vm_bind_ioctl(struct xe_vm *vm, struct xe_vma *vma, 2111 struct xe_engine *e, struct xe_bo *bo, 2112 struct drm_xe_vm_bind_op *bind_op, 2113 struct xe_sync_entry *syncs, u32 num_syncs, 2114 struct async_op_fence *afence) 2115 { 2116 LIST_HEAD(objs); 2117 LIST_HEAD(dups); 2118 struct ttm_validate_buffer tv_bo, tv_vm; 2119 struct ww_acquire_ctx ww; 2120 struct xe_bo *vbo; 2121 int err, i; 2122 2123 lockdep_assert_held(&vm->lock); 2124 XE_BUG_ON(!list_empty(&vma->unbind_link)); 2125 2126 /* Binds deferred to faults, signal fences now */ 2127 if (xe_vm_in_fault_mode(vm) && is_map_op(bind_op->op) && 2128 !(bind_op->op & XE_VM_BIND_FLAG_IMMEDIATE)) { 2129 for (i = 0; i < num_syncs; i++) 2130 xe_sync_entry_signal(&syncs[i], NULL, 2131 dma_fence_get_stub()); 2132 if (afence) 2133 dma_fence_signal(&afence->fence); 2134 return 0; 2135 } 2136 2137 xe_vm_tv_populate(vm, &tv_vm); 2138 list_add_tail(&tv_vm.head, &objs); 2139 vbo = vma->bo; 2140 if (vbo) { 2141 /* 2142 * An unbind can drop the last reference to the BO and 2143 * the BO is needed for ttm_eu_backoff_reservation so 2144 * take a reference here. 2145 */ 2146 xe_bo_get(vbo); 2147 2148 tv_bo.bo = &vbo->ttm; 2149 tv_bo.num_shared = 1; 2150 list_add(&tv_bo.head, &objs); 2151 } 2152 2153 again: 2154 err = ttm_eu_reserve_buffers(&ww, &objs, true, &dups); 2155 if (!err) { 2156 err = __vm_bind_ioctl(vm, vma, e, bo, 2157 bind_op->op, bind_op->region, syncs, 2158 num_syncs, afence); 2159 ttm_eu_backoff_reservation(&ww, &objs); 2160 if (err == -EAGAIN && xe_vma_is_userptr(vma)) { 2161 lockdep_assert_held_write(&vm->lock); 2162 err = xe_vma_userptr_pin_pages(vma); 2163 if (!err) 2164 goto again; 2165 } 2166 } 2167 xe_bo_put(vbo); 2168 2169 return err; 2170 } 2171 2172 struct async_op { 2173 struct xe_vma *vma; 2174 struct xe_engine *engine; 2175 struct xe_bo *bo; 2176 struct drm_xe_vm_bind_op bind_op; 2177 struct xe_sync_entry *syncs; 2178 u32 num_syncs; 2179 struct list_head link; 2180 struct async_op_fence *fence; 2181 }; 2182 2183 static void async_op_cleanup(struct xe_vm *vm, struct async_op *op) 2184 { 2185 while (op->num_syncs--) 2186 xe_sync_entry_cleanup(&op->syncs[op->num_syncs]); 2187 kfree(op->syncs); 2188 xe_bo_put(op->bo); 2189 if (op->engine) 2190 xe_engine_put(op->engine); 2191 xe_vm_put(vm); 2192 if (op->fence) 2193 dma_fence_put(&op->fence->fence); 2194 kfree(op); 2195 } 2196 2197 static struct async_op *next_async_op(struct xe_vm *vm) 2198 { 2199 return list_first_entry_or_null(&vm->async_ops.pending, 2200 struct async_op, link); 2201 } 2202 2203 static void vm_set_async_error(struct xe_vm *vm, int err) 2204 { 2205 lockdep_assert_held(&vm->lock); 2206 vm->async_ops.error = err; 2207 } 2208 2209 static void async_op_work_func(struct work_struct *w) 2210 { 2211 struct xe_vm *vm = container_of(w, struct xe_vm, async_ops.work); 2212 2213 for (;;) { 2214 struct async_op *op; 2215 int err; 2216 2217 if (vm->async_ops.error && !xe_vm_is_closed(vm)) 2218 break; 2219 2220 spin_lock_irq(&vm->async_ops.lock); 2221 op = next_async_op(vm); 2222 if (op) 2223 list_del_init(&op->link); 2224 spin_unlock_irq(&vm->async_ops.lock); 2225 2226 if (!op) 2227 break; 2228 2229 if (!xe_vm_is_closed(vm)) { 2230 bool first, last; 2231 2232 down_write(&vm->lock); 2233 again: 2234 first = op->vma->first_munmap_rebind; 2235 last = op->vma->last_munmap_rebind; 2236 #ifdef TEST_VM_ASYNC_OPS_ERROR 2237 #define FORCE_ASYNC_OP_ERROR BIT(31) 2238 if (!(op->bind_op.op & FORCE_ASYNC_OP_ERROR)) { 2239 err = vm_bind_ioctl(vm, op->vma, op->engine, 2240 op->bo, &op->bind_op, 2241 op->syncs, op->num_syncs, 2242 op->fence); 2243 } else { 2244 err = -ENOMEM; 2245 op->bind_op.op &= ~FORCE_ASYNC_OP_ERROR; 2246 } 2247 #else 2248 err = vm_bind_ioctl(vm, op->vma, op->engine, op->bo, 2249 &op->bind_op, op->syncs, 2250 op->num_syncs, op->fence); 2251 #endif 2252 /* 2253 * In order for the fencing to work (stall behind 2254 * existing jobs / prevent new jobs from running) all 2255 * the dma-resv slots need to be programmed in a batch 2256 * relative to execs / the rebind worker. The vm->lock 2257 * ensure this. 2258 */ 2259 if (!err && ((first && VM_BIND_OP(op->bind_op.op) == 2260 XE_VM_BIND_OP_UNMAP) || 2261 vm->async_ops.munmap_rebind_inflight)) { 2262 if (last) { 2263 op->vma->last_munmap_rebind = false; 2264 vm->async_ops.munmap_rebind_inflight = 2265 false; 2266 } else { 2267 vm->async_ops.munmap_rebind_inflight = 2268 true; 2269 2270 async_op_cleanup(vm, op); 2271 2272 spin_lock_irq(&vm->async_ops.lock); 2273 op = next_async_op(vm); 2274 XE_BUG_ON(!op); 2275 list_del_init(&op->link); 2276 spin_unlock_irq(&vm->async_ops.lock); 2277 2278 goto again; 2279 } 2280 } 2281 if (err) { 2282 trace_xe_vma_fail(op->vma); 2283 drm_warn(&vm->xe->drm, "Async VM op(%d) failed with %d", 2284 VM_BIND_OP(op->bind_op.op), 2285 err); 2286 2287 spin_lock_irq(&vm->async_ops.lock); 2288 list_add(&op->link, &vm->async_ops.pending); 2289 spin_unlock_irq(&vm->async_ops.lock); 2290 2291 vm_set_async_error(vm, err); 2292 up_write(&vm->lock); 2293 2294 if (vm->async_ops.error_capture.addr) 2295 vm_error_capture(vm, err, 2296 op->bind_op.op, 2297 op->bind_op.addr, 2298 op->bind_op.range); 2299 break; 2300 } 2301 up_write(&vm->lock); 2302 } else { 2303 trace_xe_vma_flush(op->vma); 2304 2305 if (is_unmap_op(op->bind_op.op)) { 2306 down_write(&vm->lock); 2307 xe_vma_destroy_unlocked(op->vma); 2308 up_write(&vm->lock); 2309 } 2310 2311 if (op->fence && !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, 2312 &op->fence->fence.flags)) { 2313 if (!xe_vm_no_dma_fences(vm)) { 2314 op->fence->started = true; 2315 smp_wmb(); 2316 wake_up_all(&op->fence->wq); 2317 } 2318 dma_fence_signal(&op->fence->fence); 2319 } 2320 } 2321 2322 async_op_cleanup(vm, op); 2323 } 2324 } 2325 2326 static int __vm_bind_ioctl_async(struct xe_vm *vm, struct xe_vma *vma, 2327 struct xe_engine *e, struct xe_bo *bo, 2328 struct drm_xe_vm_bind_op *bind_op, 2329 struct xe_sync_entry *syncs, u32 num_syncs) 2330 { 2331 struct async_op *op; 2332 bool installed = false; 2333 u64 seqno; 2334 int i; 2335 2336 lockdep_assert_held(&vm->lock); 2337 2338 op = kmalloc(sizeof(*op), GFP_KERNEL); 2339 if (!op) { 2340 return -ENOMEM; 2341 } 2342 2343 if (num_syncs) { 2344 op->fence = kmalloc(sizeof(*op->fence), GFP_KERNEL); 2345 if (!op->fence) { 2346 kfree(op); 2347 return -ENOMEM; 2348 } 2349 2350 seqno = e ? ++e->bind.fence_seqno : ++vm->async_ops.fence.seqno; 2351 dma_fence_init(&op->fence->fence, &async_op_fence_ops, 2352 &vm->async_ops.lock, e ? e->bind.fence_ctx : 2353 vm->async_ops.fence.context, seqno); 2354 2355 if (!xe_vm_no_dma_fences(vm)) { 2356 op->fence->vm = vm; 2357 op->fence->started = false; 2358 init_waitqueue_head(&op->fence->wq); 2359 } 2360 } else { 2361 op->fence = NULL; 2362 } 2363 op->vma = vma; 2364 op->engine = e; 2365 op->bo = bo; 2366 op->bind_op = *bind_op; 2367 op->syncs = syncs; 2368 op->num_syncs = num_syncs; 2369 INIT_LIST_HEAD(&op->link); 2370 2371 for (i = 0; i < num_syncs; i++) 2372 installed |= xe_sync_entry_signal(&syncs[i], NULL, 2373 &op->fence->fence); 2374 2375 if (!installed && op->fence) 2376 dma_fence_signal(&op->fence->fence); 2377 2378 spin_lock_irq(&vm->async_ops.lock); 2379 list_add_tail(&op->link, &vm->async_ops.pending); 2380 spin_unlock_irq(&vm->async_ops.lock); 2381 2382 if (!vm->async_ops.error) 2383 queue_work(system_unbound_wq, &vm->async_ops.work); 2384 2385 return 0; 2386 } 2387 2388 static int vm_bind_ioctl_async(struct xe_vm *vm, struct xe_vma *vma, 2389 struct xe_engine *e, struct xe_bo *bo, 2390 struct drm_xe_vm_bind_op *bind_op, 2391 struct xe_sync_entry *syncs, u32 num_syncs) 2392 { 2393 struct xe_vma *__vma, *next; 2394 struct list_head rebind_list; 2395 struct xe_sync_entry *in_syncs = NULL, *out_syncs = NULL; 2396 u32 num_in_syncs = 0, num_out_syncs = 0; 2397 bool first = true, last; 2398 int err; 2399 int i; 2400 2401 lockdep_assert_held(&vm->lock); 2402 2403 /* Not a linked list of unbinds + rebinds, easy */ 2404 if (list_empty(&vma->unbind_link)) 2405 return __vm_bind_ioctl_async(vm, vma, e, bo, bind_op, 2406 syncs, num_syncs); 2407 2408 /* 2409 * Linked list of unbinds + rebinds, decompose syncs into 'in / out' 2410 * passing the 'in' to the first operation and 'out' to the last. Also 2411 * the reference counting is a little tricky, increment the VM / bind 2412 * engine ref count on all but the last operation and increment the BOs 2413 * ref count on each rebind. 2414 */ 2415 2416 XE_BUG_ON(VM_BIND_OP(bind_op->op) != XE_VM_BIND_OP_UNMAP && 2417 VM_BIND_OP(bind_op->op) != XE_VM_BIND_OP_UNMAP_ALL && 2418 VM_BIND_OP(bind_op->op) != XE_VM_BIND_OP_PREFETCH); 2419 2420 /* Decompose syncs */ 2421 if (num_syncs) { 2422 in_syncs = kmalloc(sizeof(*in_syncs) * num_syncs, GFP_KERNEL); 2423 out_syncs = kmalloc(sizeof(*out_syncs) * num_syncs, GFP_KERNEL); 2424 if (!in_syncs || !out_syncs) { 2425 err = -ENOMEM; 2426 goto out_error; 2427 } 2428 2429 for (i = 0; i < num_syncs; ++i) { 2430 bool signal = syncs[i].flags & DRM_XE_SYNC_SIGNAL; 2431 2432 if (signal) 2433 out_syncs[num_out_syncs++] = syncs[i]; 2434 else 2435 in_syncs[num_in_syncs++] = syncs[i]; 2436 } 2437 } 2438 2439 /* Do unbinds + move rebinds to new list */ 2440 INIT_LIST_HEAD(&rebind_list); 2441 list_for_each_entry_safe(__vma, next, &vma->unbind_link, unbind_link) { 2442 if (__vma->destroyed || 2443 VM_BIND_OP(bind_op->op) == XE_VM_BIND_OP_PREFETCH) { 2444 list_del_init(&__vma->unbind_link); 2445 xe_bo_get(bo); 2446 err = __vm_bind_ioctl_async(xe_vm_get(vm), __vma, 2447 e ? xe_engine_get(e) : NULL, 2448 bo, bind_op, first ? 2449 in_syncs : NULL, 2450 first ? num_in_syncs : 0); 2451 if (err) { 2452 xe_bo_put(bo); 2453 xe_vm_put(vm); 2454 if (e) 2455 xe_engine_put(e); 2456 goto out_error; 2457 } 2458 in_syncs = NULL; 2459 first = false; 2460 } else { 2461 list_move_tail(&__vma->unbind_link, &rebind_list); 2462 } 2463 } 2464 last = list_empty(&rebind_list); 2465 if (!last) { 2466 xe_vm_get(vm); 2467 if (e) 2468 xe_engine_get(e); 2469 } 2470 err = __vm_bind_ioctl_async(vm, vma, e, 2471 bo, bind_op, 2472 first ? in_syncs : 2473 last ? out_syncs : NULL, 2474 first ? num_in_syncs : 2475 last ? num_out_syncs : 0); 2476 if (err) { 2477 if (!last) { 2478 xe_vm_put(vm); 2479 if (e) 2480 xe_engine_put(e); 2481 } 2482 goto out_error; 2483 } 2484 in_syncs = NULL; 2485 2486 /* Do rebinds */ 2487 list_for_each_entry_safe(__vma, next, &rebind_list, unbind_link) { 2488 list_del_init(&__vma->unbind_link); 2489 last = list_empty(&rebind_list); 2490 2491 if (xe_vma_is_userptr(__vma)) { 2492 bind_op->op = XE_VM_BIND_FLAG_ASYNC | 2493 XE_VM_BIND_OP_MAP_USERPTR; 2494 } else { 2495 bind_op->op = XE_VM_BIND_FLAG_ASYNC | 2496 XE_VM_BIND_OP_MAP; 2497 xe_bo_get(__vma->bo); 2498 } 2499 2500 if (!last) { 2501 xe_vm_get(vm); 2502 if (e) 2503 xe_engine_get(e); 2504 } 2505 2506 err = __vm_bind_ioctl_async(vm, __vma, e, 2507 __vma->bo, bind_op, last ? 2508 out_syncs : NULL, 2509 last ? num_out_syncs : 0); 2510 if (err) { 2511 if (!last) { 2512 xe_vm_put(vm); 2513 if (e) 2514 xe_engine_put(e); 2515 } 2516 goto out_error; 2517 } 2518 } 2519 2520 kfree(syncs); 2521 return 0; 2522 2523 out_error: 2524 kfree(in_syncs); 2525 kfree(out_syncs); 2526 kfree(syncs); 2527 2528 return err; 2529 } 2530 2531 static int __vm_bind_ioctl_lookup_vma(struct xe_vm *vm, struct xe_bo *bo, 2532 u64 addr, u64 range, u32 op) 2533 { 2534 struct xe_device *xe = vm->xe; 2535 struct xe_vma *vma, lookup; 2536 bool async = !!(op & XE_VM_BIND_FLAG_ASYNC); 2537 2538 lockdep_assert_held(&vm->lock); 2539 2540 lookup.start = addr; 2541 lookup.end = addr + range - 1; 2542 2543 switch (VM_BIND_OP(op)) { 2544 case XE_VM_BIND_OP_MAP: 2545 case XE_VM_BIND_OP_MAP_USERPTR: 2546 vma = xe_vm_find_overlapping_vma(vm, &lookup); 2547 if (XE_IOCTL_ERR(xe, vma)) 2548 return -EBUSY; 2549 break; 2550 case XE_VM_BIND_OP_UNMAP: 2551 case XE_VM_BIND_OP_PREFETCH: 2552 vma = xe_vm_find_overlapping_vma(vm, &lookup); 2553 if (XE_IOCTL_ERR(xe, !vma) || 2554 XE_IOCTL_ERR(xe, (vma->start != addr || 2555 vma->end != addr + range - 1) && !async)) 2556 return -EINVAL; 2557 break; 2558 case XE_VM_BIND_OP_UNMAP_ALL: 2559 break; 2560 default: 2561 XE_BUG_ON("NOT POSSIBLE"); 2562 return -EINVAL; 2563 } 2564 2565 return 0; 2566 } 2567 2568 static void prep_vma_destroy(struct xe_vm *vm, struct xe_vma *vma) 2569 { 2570 down_read(&vm->userptr.notifier_lock); 2571 vma->destroyed = true; 2572 up_read(&vm->userptr.notifier_lock); 2573 xe_vm_remove_vma(vm, vma); 2574 } 2575 2576 static int prep_replacement_vma(struct xe_vm *vm, struct xe_vma *vma) 2577 { 2578 int err; 2579 2580 if (vma->bo && !vma->bo->vm) { 2581 vm_insert_extobj(vm, vma); 2582 err = add_preempt_fences(vm, vma->bo); 2583 if (err) 2584 return err; 2585 } 2586 2587 return 0; 2588 } 2589 2590 /* 2591 * Find all overlapping VMAs in lookup range and add to a list in the returned 2592 * VMA, all of VMAs found will be unbound. Also possibly add 2 new VMAs that 2593 * need to be bound if first / last VMAs are not fully unbound. This is akin to 2594 * how munmap works. 2595 */ 2596 static struct xe_vma *vm_unbind_lookup_vmas(struct xe_vm *vm, 2597 struct xe_vma *lookup) 2598 { 2599 struct xe_vma *vma = xe_vm_find_overlapping_vma(vm, lookup); 2600 struct rb_node *node; 2601 struct xe_vma *first = vma, *last = vma, *new_first = NULL, 2602 *new_last = NULL, *__vma, *next; 2603 int err = 0; 2604 bool first_munmap_rebind = false; 2605 2606 lockdep_assert_held(&vm->lock); 2607 XE_BUG_ON(!vma); 2608 2609 node = &vma->vm_node; 2610 while ((node = rb_next(node))) { 2611 if (!xe_vma_cmp_vma_cb(lookup, node)) { 2612 __vma = to_xe_vma(node); 2613 list_add_tail(&__vma->unbind_link, &vma->unbind_link); 2614 last = __vma; 2615 } else { 2616 break; 2617 } 2618 } 2619 2620 node = &vma->vm_node; 2621 while ((node = rb_prev(node))) { 2622 if (!xe_vma_cmp_vma_cb(lookup, node)) { 2623 __vma = to_xe_vma(node); 2624 list_add(&__vma->unbind_link, &vma->unbind_link); 2625 first = __vma; 2626 } else { 2627 break; 2628 } 2629 } 2630 2631 if (first->start != lookup->start) { 2632 struct ww_acquire_ctx ww; 2633 2634 if (first->bo) 2635 err = xe_bo_lock(first->bo, &ww, 0, true); 2636 if (err) 2637 goto unwind; 2638 new_first = xe_vma_create(first->vm, first->bo, 2639 first->bo ? first->bo_offset : 2640 first->userptr.ptr, 2641 first->start, 2642 lookup->start - 1, 2643 (first->pte_flags & XE_PTE_READ_ONLY), 2644 first->tile_mask); 2645 if (first->bo) 2646 xe_bo_unlock(first->bo, &ww); 2647 if (!new_first) { 2648 err = -ENOMEM; 2649 goto unwind; 2650 } 2651 if (!first->bo) { 2652 err = xe_vma_userptr_pin_pages(new_first); 2653 if (err) 2654 goto unwind; 2655 } 2656 err = prep_replacement_vma(vm, new_first); 2657 if (err) 2658 goto unwind; 2659 } 2660 2661 if (last->end != lookup->end) { 2662 struct ww_acquire_ctx ww; 2663 u64 chunk = lookup->end + 1 - last->start; 2664 2665 if (last->bo) 2666 err = xe_bo_lock(last->bo, &ww, 0, true); 2667 if (err) 2668 goto unwind; 2669 new_last = xe_vma_create(last->vm, last->bo, 2670 last->bo ? last->bo_offset + chunk : 2671 last->userptr.ptr + chunk, 2672 last->start + chunk, 2673 last->end, 2674 (last->pte_flags & XE_PTE_READ_ONLY), 2675 last->tile_mask); 2676 if (last->bo) 2677 xe_bo_unlock(last->bo, &ww); 2678 if (!new_last) { 2679 err = -ENOMEM; 2680 goto unwind; 2681 } 2682 if (!last->bo) { 2683 err = xe_vma_userptr_pin_pages(new_last); 2684 if (err) 2685 goto unwind; 2686 } 2687 err = prep_replacement_vma(vm, new_last); 2688 if (err) 2689 goto unwind; 2690 } 2691 2692 prep_vma_destroy(vm, vma); 2693 if (list_empty(&vma->unbind_link) && (new_first || new_last)) 2694 vma->first_munmap_rebind = true; 2695 list_for_each_entry(__vma, &vma->unbind_link, unbind_link) { 2696 if ((new_first || new_last) && !first_munmap_rebind) { 2697 __vma->first_munmap_rebind = true; 2698 first_munmap_rebind = true; 2699 } 2700 prep_vma_destroy(vm, __vma); 2701 } 2702 if (new_first) { 2703 xe_vm_insert_vma(vm, new_first); 2704 list_add_tail(&new_first->unbind_link, &vma->unbind_link); 2705 if (!new_last) 2706 new_first->last_munmap_rebind = true; 2707 } 2708 if (new_last) { 2709 xe_vm_insert_vma(vm, new_last); 2710 list_add_tail(&new_last->unbind_link, &vma->unbind_link); 2711 new_last->last_munmap_rebind = true; 2712 } 2713 2714 return vma; 2715 2716 unwind: 2717 list_for_each_entry_safe(__vma, next, &vma->unbind_link, unbind_link) 2718 list_del_init(&__vma->unbind_link); 2719 if (new_last) { 2720 prep_vma_destroy(vm, new_last); 2721 xe_vma_destroy_unlocked(new_last); 2722 } 2723 if (new_first) { 2724 prep_vma_destroy(vm, new_first); 2725 xe_vma_destroy_unlocked(new_first); 2726 } 2727 2728 return ERR_PTR(err); 2729 } 2730 2731 /* 2732 * Similar to vm_unbind_lookup_vmas, find all VMAs in lookup range to prefetch 2733 */ 2734 static struct xe_vma *vm_prefetch_lookup_vmas(struct xe_vm *vm, 2735 struct xe_vma *lookup, 2736 u32 region) 2737 { 2738 struct xe_vma *vma = xe_vm_find_overlapping_vma(vm, lookup), *__vma, 2739 *next; 2740 struct rb_node *node; 2741 2742 if (!xe_vma_is_userptr(vma)) { 2743 if (!xe_bo_can_migrate(vma->bo, region_to_mem_type[region])) 2744 return ERR_PTR(-EINVAL); 2745 } 2746 2747 node = &vma->vm_node; 2748 while ((node = rb_next(node))) { 2749 if (!xe_vma_cmp_vma_cb(lookup, node)) { 2750 __vma = to_xe_vma(node); 2751 if (!xe_vma_is_userptr(__vma)) { 2752 if (!xe_bo_can_migrate(__vma->bo, region_to_mem_type[region])) 2753 goto flush_list; 2754 } 2755 list_add_tail(&__vma->unbind_link, &vma->unbind_link); 2756 } else { 2757 break; 2758 } 2759 } 2760 2761 node = &vma->vm_node; 2762 while ((node = rb_prev(node))) { 2763 if (!xe_vma_cmp_vma_cb(lookup, node)) { 2764 __vma = to_xe_vma(node); 2765 if (!xe_vma_is_userptr(__vma)) { 2766 if (!xe_bo_can_migrate(__vma->bo, region_to_mem_type[region])) 2767 goto flush_list; 2768 } 2769 list_add(&__vma->unbind_link, &vma->unbind_link); 2770 } else { 2771 break; 2772 } 2773 } 2774 2775 return vma; 2776 2777 flush_list: 2778 list_for_each_entry_safe(__vma, next, &vma->unbind_link, 2779 unbind_link) 2780 list_del_init(&__vma->unbind_link); 2781 2782 return ERR_PTR(-EINVAL); 2783 } 2784 2785 static struct xe_vma *vm_unbind_all_lookup_vmas(struct xe_vm *vm, 2786 struct xe_bo *bo) 2787 { 2788 struct xe_vma *first = NULL, *vma; 2789 2790 lockdep_assert_held(&vm->lock); 2791 xe_bo_assert_held(bo); 2792 2793 list_for_each_entry(vma, &bo->vmas, bo_link) { 2794 if (vma->vm != vm) 2795 continue; 2796 2797 prep_vma_destroy(vm, vma); 2798 if (!first) 2799 first = vma; 2800 else 2801 list_add_tail(&vma->unbind_link, &first->unbind_link); 2802 } 2803 2804 return first; 2805 } 2806 2807 static struct xe_vma *vm_bind_ioctl_lookup_vma(struct xe_vm *vm, 2808 struct xe_bo *bo, 2809 u64 bo_offset_or_userptr, 2810 u64 addr, u64 range, u32 op, 2811 u64 tile_mask, u32 region) 2812 { 2813 struct ww_acquire_ctx ww; 2814 struct xe_vma *vma, lookup; 2815 int err; 2816 2817 lockdep_assert_held(&vm->lock); 2818 2819 lookup.start = addr; 2820 lookup.end = addr + range - 1; 2821 2822 switch (VM_BIND_OP(op)) { 2823 case XE_VM_BIND_OP_MAP: 2824 XE_BUG_ON(!bo); 2825 2826 err = xe_bo_lock(bo, &ww, 0, true); 2827 if (err) 2828 return ERR_PTR(err); 2829 vma = xe_vma_create(vm, bo, bo_offset_or_userptr, addr, 2830 addr + range - 1, 2831 op & XE_VM_BIND_FLAG_READONLY, 2832 tile_mask); 2833 xe_bo_unlock(bo, &ww); 2834 if (!vma) 2835 return ERR_PTR(-ENOMEM); 2836 2837 xe_vm_insert_vma(vm, vma); 2838 if (!bo->vm) { 2839 vm_insert_extobj(vm, vma); 2840 err = add_preempt_fences(vm, bo); 2841 if (err) { 2842 prep_vma_destroy(vm, vma); 2843 xe_vma_destroy_unlocked(vma); 2844 2845 return ERR_PTR(err); 2846 } 2847 } 2848 break; 2849 case XE_VM_BIND_OP_UNMAP: 2850 vma = vm_unbind_lookup_vmas(vm, &lookup); 2851 break; 2852 case XE_VM_BIND_OP_PREFETCH: 2853 vma = vm_prefetch_lookup_vmas(vm, &lookup, region); 2854 break; 2855 case XE_VM_BIND_OP_UNMAP_ALL: 2856 XE_BUG_ON(!bo); 2857 2858 err = xe_bo_lock(bo, &ww, 0, true); 2859 if (err) 2860 return ERR_PTR(err); 2861 vma = vm_unbind_all_lookup_vmas(vm, bo); 2862 if (!vma) 2863 vma = ERR_PTR(-EINVAL); 2864 xe_bo_unlock(bo, &ww); 2865 break; 2866 case XE_VM_BIND_OP_MAP_USERPTR: 2867 XE_BUG_ON(bo); 2868 2869 vma = xe_vma_create(vm, NULL, bo_offset_or_userptr, addr, 2870 addr + range - 1, 2871 op & XE_VM_BIND_FLAG_READONLY, 2872 tile_mask); 2873 if (!vma) 2874 return ERR_PTR(-ENOMEM); 2875 2876 err = xe_vma_userptr_pin_pages(vma); 2877 if (err) { 2878 prep_vma_destroy(vm, vma); 2879 xe_vma_destroy_unlocked(vma); 2880 2881 return ERR_PTR(err); 2882 } else { 2883 xe_vm_insert_vma(vm, vma); 2884 } 2885 break; 2886 default: 2887 XE_BUG_ON("NOT POSSIBLE"); 2888 vma = ERR_PTR(-EINVAL); 2889 } 2890 2891 return vma; 2892 } 2893 2894 #ifdef TEST_VM_ASYNC_OPS_ERROR 2895 #define SUPPORTED_FLAGS \ 2896 (FORCE_ASYNC_OP_ERROR | XE_VM_BIND_FLAG_ASYNC | \ 2897 XE_VM_BIND_FLAG_READONLY | XE_VM_BIND_FLAG_IMMEDIATE | 0xffff) 2898 #else 2899 #define SUPPORTED_FLAGS \ 2900 (XE_VM_BIND_FLAG_ASYNC | XE_VM_BIND_FLAG_READONLY | \ 2901 XE_VM_BIND_FLAG_IMMEDIATE | 0xffff) 2902 #endif 2903 #define XE_64K_PAGE_MASK 0xffffull 2904 2905 #define MAX_BINDS 512 /* FIXME: Picking random upper limit */ 2906 2907 static int vm_bind_ioctl_check_args(struct xe_device *xe, 2908 struct drm_xe_vm_bind *args, 2909 struct drm_xe_vm_bind_op **bind_ops, 2910 bool *async) 2911 { 2912 int err; 2913 int i; 2914 2915 if (XE_IOCTL_ERR(xe, args->extensions) || 2916 XE_IOCTL_ERR(xe, args->pad || args->pad2) || 2917 XE_IOCTL_ERR(xe, args->reserved[0] || args->reserved[1]) || 2918 XE_IOCTL_ERR(xe, !args->num_binds) || 2919 XE_IOCTL_ERR(xe, args->num_binds > MAX_BINDS)) 2920 return -EINVAL; 2921 2922 if (args->num_binds > 1) { 2923 u64 __user *bind_user = 2924 u64_to_user_ptr(args->vector_of_binds); 2925 2926 *bind_ops = kmalloc(sizeof(struct drm_xe_vm_bind_op) * 2927 args->num_binds, GFP_KERNEL); 2928 if (!*bind_ops) 2929 return -ENOMEM; 2930 2931 err = __copy_from_user(*bind_ops, bind_user, 2932 sizeof(struct drm_xe_vm_bind_op) * 2933 args->num_binds); 2934 if (XE_IOCTL_ERR(xe, err)) { 2935 err = -EFAULT; 2936 goto free_bind_ops; 2937 } 2938 } else { 2939 *bind_ops = &args->bind; 2940 } 2941 2942 for (i = 0; i < args->num_binds; ++i) { 2943 u64 range = (*bind_ops)[i].range; 2944 u64 addr = (*bind_ops)[i].addr; 2945 u32 op = (*bind_ops)[i].op; 2946 u32 obj = (*bind_ops)[i].obj; 2947 u64 obj_offset = (*bind_ops)[i].obj_offset; 2948 u32 region = (*bind_ops)[i].region; 2949 2950 if (XE_IOCTL_ERR(xe, (*bind_ops)[i].pad) || 2951 XE_IOCTL_ERR(xe, (*bind_ops)[i].reserved[0] || 2952 (*bind_ops)[i].reserved[1])) { 2953 err = -EINVAL; 2954 goto free_bind_ops; 2955 } 2956 2957 if (i == 0) { 2958 *async = !!(op & XE_VM_BIND_FLAG_ASYNC); 2959 } else if (XE_IOCTL_ERR(xe, !*async) || 2960 XE_IOCTL_ERR(xe, !(op & XE_VM_BIND_FLAG_ASYNC)) || 2961 XE_IOCTL_ERR(xe, VM_BIND_OP(op) == 2962 XE_VM_BIND_OP_RESTART)) { 2963 err = -EINVAL; 2964 goto free_bind_ops; 2965 } 2966 2967 if (XE_IOCTL_ERR(xe, !*async && 2968 VM_BIND_OP(op) == XE_VM_BIND_OP_UNMAP_ALL)) { 2969 err = -EINVAL; 2970 goto free_bind_ops; 2971 } 2972 2973 if (XE_IOCTL_ERR(xe, !*async && 2974 VM_BIND_OP(op) == XE_VM_BIND_OP_PREFETCH)) { 2975 err = -EINVAL; 2976 goto free_bind_ops; 2977 } 2978 2979 if (XE_IOCTL_ERR(xe, VM_BIND_OP(op) > 2980 XE_VM_BIND_OP_PREFETCH) || 2981 XE_IOCTL_ERR(xe, op & ~SUPPORTED_FLAGS) || 2982 XE_IOCTL_ERR(xe, !obj && 2983 VM_BIND_OP(op) == XE_VM_BIND_OP_MAP) || 2984 XE_IOCTL_ERR(xe, !obj && 2985 VM_BIND_OP(op) == XE_VM_BIND_OP_UNMAP_ALL) || 2986 XE_IOCTL_ERR(xe, addr && 2987 VM_BIND_OP(op) == XE_VM_BIND_OP_UNMAP_ALL) || 2988 XE_IOCTL_ERR(xe, range && 2989 VM_BIND_OP(op) == XE_VM_BIND_OP_UNMAP_ALL) || 2990 XE_IOCTL_ERR(xe, obj && 2991 VM_BIND_OP(op) == XE_VM_BIND_OP_MAP_USERPTR) || 2992 XE_IOCTL_ERR(xe, obj && 2993 VM_BIND_OP(op) == XE_VM_BIND_OP_PREFETCH) || 2994 XE_IOCTL_ERR(xe, region && 2995 VM_BIND_OP(op) != XE_VM_BIND_OP_PREFETCH) || 2996 XE_IOCTL_ERR(xe, !(BIT(region) & 2997 xe->info.mem_region_mask)) || 2998 XE_IOCTL_ERR(xe, obj && 2999 VM_BIND_OP(op) == XE_VM_BIND_OP_UNMAP)) { 3000 err = -EINVAL; 3001 goto free_bind_ops; 3002 } 3003 3004 if (XE_IOCTL_ERR(xe, obj_offset & ~PAGE_MASK) || 3005 XE_IOCTL_ERR(xe, addr & ~PAGE_MASK) || 3006 XE_IOCTL_ERR(xe, range & ~PAGE_MASK) || 3007 XE_IOCTL_ERR(xe, !range && VM_BIND_OP(op) != 3008 XE_VM_BIND_OP_RESTART && 3009 VM_BIND_OP(op) != XE_VM_BIND_OP_UNMAP_ALL)) { 3010 err = -EINVAL; 3011 goto free_bind_ops; 3012 } 3013 } 3014 3015 return 0; 3016 3017 free_bind_ops: 3018 if (args->num_binds > 1) 3019 kfree(*bind_ops); 3020 return err; 3021 } 3022 3023 int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 3024 { 3025 struct xe_device *xe = to_xe_device(dev); 3026 struct xe_file *xef = to_xe_file(file); 3027 struct drm_xe_vm_bind *args = data; 3028 struct drm_xe_sync __user *syncs_user; 3029 struct xe_bo **bos = NULL; 3030 struct xe_vma **vmas = NULL; 3031 struct xe_vm *vm; 3032 struct xe_engine *e = NULL; 3033 u32 num_syncs; 3034 struct xe_sync_entry *syncs = NULL; 3035 struct drm_xe_vm_bind_op *bind_ops; 3036 bool async; 3037 int err; 3038 int i, j = 0; 3039 3040 err = vm_bind_ioctl_check_args(xe, args, &bind_ops, &async); 3041 if (err) 3042 return err; 3043 3044 vm = xe_vm_lookup(xef, args->vm_id); 3045 if (XE_IOCTL_ERR(xe, !vm)) { 3046 err = -EINVAL; 3047 goto free_objs; 3048 } 3049 3050 if (XE_IOCTL_ERR(xe, xe_vm_is_closed(vm))) { 3051 DRM_ERROR("VM closed while we began looking up?\n"); 3052 err = -ENOENT; 3053 goto put_vm; 3054 } 3055 3056 if (args->engine_id) { 3057 e = xe_engine_lookup(xef, args->engine_id); 3058 if (XE_IOCTL_ERR(xe, !e)) { 3059 err = -ENOENT; 3060 goto put_vm; 3061 } 3062 if (XE_IOCTL_ERR(xe, !(e->flags & ENGINE_FLAG_VM))) { 3063 err = -EINVAL; 3064 goto put_engine; 3065 } 3066 } 3067 3068 if (VM_BIND_OP(bind_ops[0].op) == XE_VM_BIND_OP_RESTART) { 3069 if (XE_IOCTL_ERR(xe, !(vm->flags & XE_VM_FLAG_ASYNC_BIND_OPS))) 3070 err = -ENOTSUPP; 3071 if (XE_IOCTL_ERR(xe, !err && args->num_syncs)) 3072 err = EINVAL; 3073 if (XE_IOCTL_ERR(xe, !err && !vm->async_ops.error)) 3074 err = -EPROTO; 3075 3076 if (!err) { 3077 down_write(&vm->lock); 3078 trace_xe_vm_restart(vm); 3079 vm_set_async_error(vm, 0); 3080 up_write(&vm->lock); 3081 3082 queue_work(system_unbound_wq, &vm->async_ops.work); 3083 3084 /* Rebinds may have been blocked, give worker a kick */ 3085 if (xe_vm_in_compute_mode(vm)) 3086 queue_work(vm->xe->ordered_wq, 3087 &vm->preempt.rebind_work); 3088 } 3089 3090 goto put_engine; 3091 } 3092 3093 if (XE_IOCTL_ERR(xe, !vm->async_ops.error && 3094 async != !!(vm->flags & XE_VM_FLAG_ASYNC_BIND_OPS))) { 3095 err = -ENOTSUPP; 3096 goto put_engine; 3097 } 3098 3099 for (i = 0; i < args->num_binds; ++i) { 3100 u64 range = bind_ops[i].range; 3101 u64 addr = bind_ops[i].addr; 3102 3103 if (XE_IOCTL_ERR(xe, range > vm->size) || 3104 XE_IOCTL_ERR(xe, addr > vm->size - range)) { 3105 err = -EINVAL; 3106 goto put_engine; 3107 } 3108 3109 if (bind_ops[i].tile_mask) { 3110 u64 valid_tiles = BIT(xe->info.tile_count) - 1; 3111 3112 if (XE_IOCTL_ERR(xe, bind_ops[i].tile_mask & 3113 ~valid_tiles)) { 3114 err = -EINVAL; 3115 goto put_engine; 3116 } 3117 } 3118 } 3119 3120 bos = kzalloc(sizeof(*bos) * args->num_binds, GFP_KERNEL); 3121 if (!bos) { 3122 err = -ENOMEM; 3123 goto put_engine; 3124 } 3125 3126 vmas = kzalloc(sizeof(*vmas) * args->num_binds, GFP_KERNEL); 3127 if (!vmas) { 3128 err = -ENOMEM; 3129 goto put_engine; 3130 } 3131 3132 for (i = 0; i < args->num_binds; ++i) { 3133 struct drm_gem_object *gem_obj; 3134 u64 range = bind_ops[i].range; 3135 u64 addr = bind_ops[i].addr; 3136 u32 obj = bind_ops[i].obj; 3137 u64 obj_offset = bind_ops[i].obj_offset; 3138 3139 if (!obj) 3140 continue; 3141 3142 gem_obj = drm_gem_object_lookup(file, obj); 3143 if (XE_IOCTL_ERR(xe, !gem_obj)) { 3144 err = -ENOENT; 3145 goto put_obj; 3146 } 3147 bos[i] = gem_to_xe_bo(gem_obj); 3148 3149 if (XE_IOCTL_ERR(xe, range > bos[i]->size) || 3150 XE_IOCTL_ERR(xe, obj_offset > 3151 bos[i]->size - range)) { 3152 err = -EINVAL; 3153 goto put_obj; 3154 } 3155 3156 if (bos[i]->flags & XE_BO_INTERNAL_64K) { 3157 if (XE_IOCTL_ERR(xe, obj_offset & 3158 XE_64K_PAGE_MASK) || 3159 XE_IOCTL_ERR(xe, addr & XE_64K_PAGE_MASK) || 3160 XE_IOCTL_ERR(xe, range & XE_64K_PAGE_MASK)) { 3161 err = -EINVAL; 3162 goto put_obj; 3163 } 3164 } 3165 } 3166 3167 if (args->num_syncs) { 3168 syncs = kcalloc(args->num_syncs, sizeof(*syncs), GFP_KERNEL); 3169 if (!syncs) { 3170 err = -ENOMEM; 3171 goto put_obj; 3172 } 3173 } 3174 3175 syncs_user = u64_to_user_ptr(args->syncs); 3176 for (num_syncs = 0; num_syncs < args->num_syncs; num_syncs++) { 3177 err = xe_sync_entry_parse(xe, xef, &syncs[num_syncs], 3178 &syncs_user[num_syncs], false, 3179 xe_vm_in_fault_mode(vm)); 3180 if (err) 3181 goto free_syncs; 3182 } 3183 3184 err = down_write_killable(&vm->lock); 3185 if (err) 3186 goto free_syncs; 3187 3188 /* Do some error checking first to make the unwind easier */ 3189 for (i = 0; i < args->num_binds; ++i) { 3190 u64 range = bind_ops[i].range; 3191 u64 addr = bind_ops[i].addr; 3192 u32 op = bind_ops[i].op; 3193 3194 err = __vm_bind_ioctl_lookup_vma(vm, bos[i], addr, range, op); 3195 if (err) 3196 goto release_vm_lock; 3197 } 3198 3199 for (i = 0; i < args->num_binds; ++i) { 3200 u64 range = bind_ops[i].range; 3201 u64 addr = bind_ops[i].addr; 3202 u32 op = bind_ops[i].op; 3203 u64 obj_offset = bind_ops[i].obj_offset; 3204 u64 tile_mask = bind_ops[i].tile_mask; 3205 u32 region = bind_ops[i].region; 3206 3207 vmas[i] = vm_bind_ioctl_lookup_vma(vm, bos[i], obj_offset, 3208 addr, range, op, tile_mask, 3209 region); 3210 if (IS_ERR(vmas[i])) { 3211 err = PTR_ERR(vmas[i]); 3212 vmas[i] = NULL; 3213 goto destroy_vmas; 3214 } 3215 } 3216 3217 for (j = 0; j < args->num_binds; ++j) { 3218 struct xe_sync_entry *__syncs; 3219 u32 __num_syncs = 0; 3220 bool first_or_last = j == 0 || j == args->num_binds - 1; 3221 3222 if (args->num_binds == 1) { 3223 __num_syncs = num_syncs; 3224 __syncs = syncs; 3225 } else if (first_or_last && num_syncs) { 3226 bool first = j == 0; 3227 3228 __syncs = kmalloc(sizeof(*__syncs) * num_syncs, 3229 GFP_KERNEL); 3230 if (!__syncs) { 3231 err = ENOMEM; 3232 break; 3233 } 3234 3235 /* in-syncs on first bind, out-syncs on last bind */ 3236 for (i = 0; i < num_syncs; ++i) { 3237 bool signal = syncs[i].flags & 3238 DRM_XE_SYNC_SIGNAL; 3239 3240 if ((first && !signal) || (!first && signal)) 3241 __syncs[__num_syncs++] = syncs[i]; 3242 } 3243 } else { 3244 __num_syncs = 0; 3245 __syncs = NULL; 3246 } 3247 3248 if (async) { 3249 bool last = j == args->num_binds - 1; 3250 3251 /* 3252 * Each pass of async worker drops the ref, take a ref 3253 * here, 1 set of refs taken above 3254 */ 3255 if (!last) { 3256 if (e) 3257 xe_engine_get(e); 3258 xe_vm_get(vm); 3259 } 3260 3261 err = vm_bind_ioctl_async(vm, vmas[j], e, bos[j], 3262 bind_ops + j, __syncs, 3263 __num_syncs); 3264 if (err && !last) { 3265 if (e) 3266 xe_engine_put(e); 3267 xe_vm_put(vm); 3268 } 3269 if (err) 3270 break; 3271 } else { 3272 XE_BUG_ON(j != 0); /* Not supported */ 3273 err = vm_bind_ioctl(vm, vmas[j], e, bos[j], 3274 bind_ops + j, __syncs, 3275 __num_syncs, NULL); 3276 break; /* Needed so cleanup loops work */ 3277 } 3278 } 3279 3280 /* Most of cleanup owned by the async bind worker */ 3281 if (async && !err) { 3282 up_write(&vm->lock); 3283 if (args->num_binds > 1) 3284 kfree(syncs); 3285 goto free_objs; 3286 } 3287 3288 destroy_vmas: 3289 for (i = j; err && i < args->num_binds; ++i) { 3290 u32 op = bind_ops[i].op; 3291 struct xe_vma *vma, *next; 3292 3293 if (!vmas[i]) 3294 break; 3295 3296 list_for_each_entry_safe(vma, next, &vma->unbind_link, 3297 unbind_link) { 3298 list_del_init(&vma->unbind_link); 3299 if (!vma->destroyed) { 3300 prep_vma_destroy(vm, vma); 3301 xe_vma_destroy_unlocked(vma); 3302 } 3303 } 3304 3305 switch (VM_BIND_OP(op)) { 3306 case XE_VM_BIND_OP_MAP: 3307 prep_vma_destroy(vm, vmas[i]); 3308 xe_vma_destroy_unlocked(vmas[i]); 3309 break; 3310 case XE_VM_BIND_OP_MAP_USERPTR: 3311 prep_vma_destroy(vm, vmas[i]); 3312 xe_vma_destroy_unlocked(vmas[i]); 3313 break; 3314 } 3315 } 3316 release_vm_lock: 3317 up_write(&vm->lock); 3318 free_syncs: 3319 while (num_syncs--) { 3320 if (async && j && 3321 !(syncs[num_syncs].flags & DRM_XE_SYNC_SIGNAL)) 3322 continue; /* Still in async worker */ 3323 xe_sync_entry_cleanup(&syncs[num_syncs]); 3324 } 3325 3326 kfree(syncs); 3327 put_obj: 3328 for (i = j; i < args->num_binds; ++i) 3329 xe_bo_put(bos[i]); 3330 put_engine: 3331 if (e) 3332 xe_engine_put(e); 3333 put_vm: 3334 xe_vm_put(vm); 3335 free_objs: 3336 kfree(bos); 3337 kfree(vmas); 3338 if (args->num_binds > 1) 3339 kfree(bind_ops); 3340 return err; 3341 } 3342 3343 /* 3344 * XXX: Using the TTM wrappers for now, likely can call into dma-resv code 3345 * directly to optimize. Also this likely should be an inline function. 3346 */ 3347 int xe_vm_lock(struct xe_vm *vm, struct ww_acquire_ctx *ww, 3348 int num_resv, bool intr) 3349 { 3350 struct ttm_validate_buffer tv_vm; 3351 LIST_HEAD(objs); 3352 LIST_HEAD(dups); 3353 3354 XE_BUG_ON(!ww); 3355 3356 tv_vm.num_shared = num_resv; 3357 tv_vm.bo = xe_vm_ttm_bo(vm);; 3358 list_add_tail(&tv_vm.head, &objs); 3359 3360 return ttm_eu_reserve_buffers(ww, &objs, intr, &dups); 3361 } 3362 3363 void xe_vm_unlock(struct xe_vm *vm, struct ww_acquire_ctx *ww) 3364 { 3365 dma_resv_unlock(&vm->resv); 3366 ww_acquire_fini(ww); 3367 } 3368 3369 /** 3370 * xe_vm_invalidate_vma - invalidate GPU mappings for VMA without a lock 3371 * @vma: VMA to invalidate 3372 * 3373 * Walks a list of page tables leaves which it memset the entries owned by this 3374 * VMA to zero, invalidates the TLBs, and block until TLBs invalidation is 3375 * complete. 3376 * 3377 * Returns 0 for success, negative error code otherwise. 3378 */ 3379 int xe_vm_invalidate_vma(struct xe_vma *vma) 3380 { 3381 struct xe_device *xe = vma->vm->xe; 3382 struct xe_tile *tile; 3383 u32 tile_needs_invalidate = 0; 3384 int seqno[XE_MAX_TILES_PER_DEVICE]; 3385 u8 id; 3386 int ret; 3387 3388 XE_BUG_ON(!xe_vm_in_fault_mode(vma->vm)); 3389 trace_xe_vma_usm_invalidate(vma); 3390 3391 /* Check that we don't race with page-table updates */ 3392 if (IS_ENABLED(CONFIG_PROVE_LOCKING)) { 3393 if (xe_vma_is_userptr(vma)) { 3394 WARN_ON_ONCE(!mmu_interval_check_retry 3395 (&vma->userptr.notifier, 3396 vma->userptr.notifier_seq)); 3397 WARN_ON_ONCE(!dma_resv_test_signaled(&vma->vm->resv, 3398 DMA_RESV_USAGE_BOOKKEEP)); 3399 3400 } else { 3401 xe_bo_assert_held(vma->bo); 3402 } 3403 } 3404 3405 for_each_tile(tile, xe, id) { 3406 if (xe_pt_zap_ptes(tile, vma)) { 3407 tile_needs_invalidate |= BIT(id); 3408 xe_device_wmb(xe); 3409 /* 3410 * FIXME: We potentially need to invalidate multiple 3411 * GTs within the tile 3412 */ 3413 seqno[id] = xe_gt_tlb_invalidation_vma(tile->primary_gt, NULL, vma); 3414 if (seqno[id] < 0) 3415 return seqno[id]; 3416 } 3417 } 3418 3419 for_each_tile(tile, xe, id) { 3420 if (tile_needs_invalidate & BIT(id)) { 3421 ret = xe_gt_tlb_invalidation_wait(tile->primary_gt, seqno[id]); 3422 if (ret < 0) 3423 return ret; 3424 } 3425 } 3426 3427 vma->usm.tile_invalidated = vma->tile_mask; 3428 3429 return 0; 3430 } 3431 3432 int xe_analyze_vm(struct drm_printer *p, struct xe_vm *vm, int gt_id) 3433 { 3434 struct rb_node *node; 3435 bool is_vram; 3436 uint64_t addr; 3437 3438 if (!down_read_trylock(&vm->lock)) { 3439 drm_printf(p, " Failed to acquire VM lock to dump capture"); 3440 return 0; 3441 } 3442 if (vm->pt_root[gt_id]) { 3443 addr = xe_bo_addr(vm->pt_root[gt_id]->bo, 0, XE_PAGE_SIZE, 3444 &is_vram); 3445 drm_printf(p, " VM root: A:0x%llx %s\n", addr, is_vram ? "VRAM" : "SYS"); 3446 } 3447 3448 for (node = rb_first(&vm->vmas); node; node = rb_next(node)) { 3449 struct xe_vma *vma = to_xe_vma(node); 3450 bool is_userptr = xe_vma_is_userptr(vma); 3451 3452 if (is_userptr) { 3453 struct xe_res_cursor cur; 3454 3455 xe_res_first_sg(vma->userptr.sg, 0, XE_PAGE_SIZE, 3456 &cur); 3457 addr = xe_res_dma(&cur); 3458 } else { 3459 addr = __xe_bo_addr(vma->bo, 0, XE_PAGE_SIZE, &is_vram); 3460 } 3461 drm_printf(p, " [%016llx-%016llx] S:0x%016llx A:%016llx %s\n", 3462 vma->start, vma->end, vma->end - vma->start + 1ull, 3463 addr, is_userptr ? "USR" : is_vram ? "VRAM" : "SYS"); 3464 } 3465 up_read(&vm->lock); 3466 3467 return 0; 3468 } 3469