1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_vm.h" 7 8 #include <linux/dma-fence-array.h> 9 #include <linux/nospec.h> 10 11 #include <drm/drm_exec.h> 12 #include <drm/drm_print.h> 13 #include <drm/ttm/ttm_execbuf_util.h> 14 #include <drm/ttm/ttm_tt.h> 15 #include <drm/xe_drm.h> 16 #include <linux/ascii85.h> 17 #include <linux/delay.h> 18 #include <linux/kthread.h> 19 #include <linux/mm.h> 20 #include <linux/swap.h> 21 22 #include <generated/xe_wa_oob.h> 23 24 #include "xe_assert.h" 25 #include "xe_bo.h" 26 #include "xe_device.h" 27 #include "xe_drm_client.h" 28 #include "xe_exec_queue.h" 29 #include "xe_gt.h" 30 #include "xe_gt_pagefault.h" 31 #include "xe_gt_tlb_invalidation.h" 32 #include "xe_migrate.h" 33 #include "xe_pat.h" 34 #include "xe_pm.h" 35 #include "xe_preempt_fence.h" 36 #include "xe_pt.h" 37 #include "xe_res_cursor.h" 38 #include "xe_sync.h" 39 #include "xe_trace.h" 40 #include "xe_wa.h" 41 42 static struct drm_gem_object *xe_vm_obj(struct xe_vm *vm) 43 { 44 return vm->gpuvm.r_obj; 45 } 46 47 /** 48 * xe_vma_userptr_check_repin() - Advisory check for repin needed 49 * @uvma: The userptr vma 50 * 51 * Check if the userptr vma has been invalidated since last successful 52 * repin. The check is advisory only and can the function can be called 53 * without the vm->userptr.notifier_lock held. There is no guarantee that the 54 * vma userptr will remain valid after a lockless check, so typically 55 * the call needs to be followed by a proper check under the notifier_lock. 56 * 57 * Return: 0 if userptr vma is valid, -EAGAIN otherwise; repin recommended. 58 */ 59 int xe_vma_userptr_check_repin(struct xe_userptr_vma *uvma) 60 { 61 return mmu_interval_check_retry(&uvma->userptr.notifier, 62 uvma->userptr.notifier_seq) ? 63 -EAGAIN : 0; 64 } 65 66 int xe_vma_userptr_pin_pages(struct xe_userptr_vma *uvma) 67 { 68 struct xe_userptr *userptr = &uvma->userptr; 69 struct xe_vma *vma = &uvma->vma; 70 struct xe_vm *vm = xe_vma_vm(vma); 71 struct xe_device *xe = vm->xe; 72 const unsigned long num_pages = xe_vma_size(vma) >> PAGE_SHIFT; 73 struct page **pages; 74 bool in_kthread = !current->mm; 75 unsigned long notifier_seq; 76 int pinned, ret, i; 77 bool read_only = xe_vma_read_only(vma); 78 79 lockdep_assert_held(&vm->lock); 80 xe_assert(xe, xe_vma_is_userptr(vma)); 81 retry: 82 if (vma->gpuva.flags & XE_VMA_DESTROYED) 83 return 0; 84 85 notifier_seq = mmu_interval_read_begin(&userptr->notifier); 86 if (notifier_seq == userptr->notifier_seq) 87 return 0; 88 89 pages = kvmalloc_array(num_pages, sizeof(*pages), GFP_KERNEL); 90 if (!pages) 91 return -ENOMEM; 92 93 if (userptr->sg) { 94 dma_unmap_sgtable(xe->drm.dev, 95 userptr->sg, 96 read_only ? DMA_TO_DEVICE : 97 DMA_BIDIRECTIONAL, 0); 98 sg_free_table(userptr->sg); 99 userptr->sg = NULL; 100 } 101 102 pinned = ret = 0; 103 if (in_kthread) { 104 if (!mmget_not_zero(userptr->notifier.mm)) { 105 ret = -EFAULT; 106 goto mm_closed; 107 } 108 kthread_use_mm(userptr->notifier.mm); 109 } 110 111 while (pinned < num_pages) { 112 ret = get_user_pages_fast(xe_vma_userptr(vma) + 113 pinned * PAGE_SIZE, 114 num_pages - pinned, 115 read_only ? 0 : FOLL_WRITE, 116 &pages[pinned]); 117 if (ret < 0) 118 break; 119 120 pinned += ret; 121 ret = 0; 122 } 123 124 if (in_kthread) { 125 kthread_unuse_mm(userptr->notifier.mm); 126 mmput(userptr->notifier.mm); 127 } 128 mm_closed: 129 if (ret) 130 goto out; 131 132 ret = sg_alloc_table_from_pages_segment(&userptr->sgt, pages, 133 pinned, 0, 134 (u64)pinned << PAGE_SHIFT, 135 xe_sg_segment_size(xe->drm.dev), 136 GFP_KERNEL); 137 if (ret) { 138 userptr->sg = NULL; 139 goto out; 140 } 141 userptr->sg = &userptr->sgt; 142 143 ret = dma_map_sgtable(xe->drm.dev, userptr->sg, 144 read_only ? DMA_TO_DEVICE : 145 DMA_BIDIRECTIONAL, 146 DMA_ATTR_SKIP_CPU_SYNC | 147 DMA_ATTR_NO_KERNEL_MAPPING); 148 if (ret) { 149 sg_free_table(userptr->sg); 150 userptr->sg = NULL; 151 goto out; 152 } 153 154 for (i = 0; i < pinned; ++i) { 155 if (!read_only) { 156 lock_page(pages[i]); 157 set_page_dirty(pages[i]); 158 unlock_page(pages[i]); 159 } 160 161 mark_page_accessed(pages[i]); 162 } 163 164 out: 165 release_pages(pages, pinned); 166 kvfree(pages); 167 168 if (!(ret < 0)) { 169 userptr->notifier_seq = notifier_seq; 170 if (xe_vma_userptr_check_repin(uvma) == -EAGAIN) 171 goto retry; 172 } 173 174 return ret < 0 ? ret : 0; 175 } 176 177 static bool preempt_fences_waiting(struct xe_vm *vm) 178 { 179 struct xe_exec_queue *q; 180 181 lockdep_assert_held(&vm->lock); 182 xe_vm_assert_held(vm); 183 184 list_for_each_entry(q, &vm->preempt.exec_queues, compute.link) { 185 if (!q->compute.pfence || 186 (q->compute.pfence && test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, 187 &q->compute.pfence->flags))) { 188 return true; 189 } 190 } 191 192 return false; 193 } 194 195 static void free_preempt_fences(struct list_head *list) 196 { 197 struct list_head *link, *next; 198 199 list_for_each_safe(link, next, list) 200 xe_preempt_fence_free(to_preempt_fence_from_link(link)); 201 } 202 203 static int alloc_preempt_fences(struct xe_vm *vm, struct list_head *list, 204 unsigned int *count) 205 { 206 lockdep_assert_held(&vm->lock); 207 xe_vm_assert_held(vm); 208 209 if (*count >= vm->preempt.num_exec_queues) 210 return 0; 211 212 for (; *count < vm->preempt.num_exec_queues; ++(*count)) { 213 struct xe_preempt_fence *pfence = xe_preempt_fence_alloc(); 214 215 if (IS_ERR(pfence)) 216 return PTR_ERR(pfence); 217 218 list_move_tail(xe_preempt_fence_link(pfence), list); 219 } 220 221 return 0; 222 } 223 224 static int wait_for_existing_preempt_fences(struct xe_vm *vm) 225 { 226 struct xe_exec_queue *q; 227 228 xe_vm_assert_held(vm); 229 230 list_for_each_entry(q, &vm->preempt.exec_queues, compute.link) { 231 if (q->compute.pfence) { 232 long timeout = dma_fence_wait(q->compute.pfence, false); 233 234 if (timeout < 0) 235 return -ETIME; 236 dma_fence_put(q->compute.pfence); 237 q->compute.pfence = NULL; 238 } 239 } 240 241 return 0; 242 } 243 244 static bool xe_vm_is_idle(struct xe_vm *vm) 245 { 246 struct xe_exec_queue *q; 247 248 xe_vm_assert_held(vm); 249 list_for_each_entry(q, &vm->preempt.exec_queues, compute.link) { 250 if (!xe_exec_queue_is_idle(q)) 251 return false; 252 } 253 254 return true; 255 } 256 257 static void arm_preempt_fences(struct xe_vm *vm, struct list_head *list) 258 { 259 struct list_head *link; 260 struct xe_exec_queue *q; 261 262 list_for_each_entry(q, &vm->preempt.exec_queues, compute.link) { 263 struct dma_fence *fence; 264 265 link = list->next; 266 xe_assert(vm->xe, link != list); 267 268 fence = xe_preempt_fence_arm(to_preempt_fence_from_link(link), 269 q, q->compute.context, 270 ++q->compute.seqno); 271 dma_fence_put(q->compute.pfence); 272 q->compute.pfence = fence; 273 } 274 } 275 276 static int add_preempt_fences(struct xe_vm *vm, struct xe_bo *bo) 277 { 278 struct xe_exec_queue *q; 279 int err; 280 281 if (!vm->preempt.num_exec_queues) 282 return 0; 283 284 err = xe_bo_lock(bo, true); 285 if (err) 286 return err; 287 288 err = dma_resv_reserve_fences(bo->ttm.base.resv, vm->preempt.num_exec_queues); 289 if (err) 290 goto out_unlock; 291 292 list_for_each_entry(q, &vm->preempt.exec_queues, compute.link) 293 if (q->compute.pfence) { 294 dma_resv_add_fence(bo->ttm.base.resv, 295 q->compute.pfence, 296 DMA_RESV_USAGE_BOOKKEEP); 297 } 298 299 out_unlock: 300 xe_bo_unlock(bo); 301 return err; 302 } 303 304 static void resume_and_reinstall_preempt_fences(struct xe_vm *vm, 305 struct drm_exec *exec) 306 { 307 struct xe_exec_queue *q; 308 309 lockdep_assert_held(&vm->lock); 310 xe_vm_assert_held(vm); 311 312 list_for_each_entry(q, &vm->preempt.exec_queues, compute.link) { 313 q->ops->resume(q); 314 315 drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, q->compute.pfence, 316 DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_BOOKKEEP); 317 } 318 } 319 320 int xe_vm_add_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q) 321 { 322 struct drm_gpuvm_exec vm_exec = { 323 .vm = &vm->gpuvm, 324 .flags = DRM_EXEC_INTERRUPTIBLE_WAIT, 325 .num_fences = 1, 326 }; 327 struct drm_exec *exec = &vm_exec.exec; 328 struct dma_fence *pfence; 329 int err; 330 bool wait; 331 332 xe_assert(vm->xe, xe_vm_in_preempt_fence_mode(vm)); 333 334 down_write(&vm->lock); 335 err = drm_gpuvm_exec_lock(&vm_exec); 336 if (err) 337 goto out_up_write; 338 339 pfence = xe_preempt_fence_create(q, q->compute.context, 340 ++q->compute.seqno); 341 if (!pfence) { 342 err = -ENOMEM; 343 goto out_fini; 344 } 345 346 list_add(&q->compute.link, &vm->preempt.exec_queues); 347 ++vm->preempt.num_exec_queues; 348 q->compute.pfence = pfence; 349 350 down_read(&vm->userptr.notifier_lock); 351 352 drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, pfence, 353 DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_BOOKKEEP); 354 355 /* 356 * Check to see if a preemption on VM is in flight or userptr 357 * invalidation, if so trigger this preempt fence to sync state with 358 * other preempt fences on the VM. 359 */ 360 wait = __xe_vm_userptr_needs_repin(vm) || preempt_fences_waiting(vm); 361 if (wait) 362 dma_fence_enable_sw_signaling(pfence); 363 364 up_read(&vm->userptr.notifier_lock); 365 366 out_fini: 367 drm_exec_fini(exec); 368 out_up_write: 369 up_write(&vm->lock); 370 371 return err; 372 } 373 374 /** 375 * xe_vm_remove_compute_exec_queue() - Remove compute exec queue from VM 376 * @vm: The VM. 377 * @q: The exec_queue 378 */ 379 void xe_vm_remove_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q) 380 { 381 if (!xe_vm_in_preempt_fence_mode(vm)) 382 return; 383 384 down_write(&vm->lock); 385 list_del(&q->compute.link); 386 --vm->preempt.num_exec_queues; 387 if (q->compute.pfence) { 388 dma_fence_enable_sw_signaling(q->compute.pfence); 389 dma_fence_put(q->compute.pfence); 390 q->compute.pfence = NULL; 391 } 392 up_write(&vm->lock); 393 } 394 395 /** 396 * __xe_vm_userptr_needs_repin() - Check whether the VM does have userptrs 397 * that need repinning. 398 * @vm: The VM. 399 * 400 * This function checks for whether the VM has userptrs that need repinning, 401 * and provides a release-type barrier on the userptr.notifier_lock after 402 * checking. 403 * 404 * Return: 0 if there are no userptrs needing repinning, -EAGAIN if there are. 405 */ 406 int __xe_vm_userptr_needs_repin(struct xe_vm *vm) 407 { 408 lockdep_assert_held_read(&vm->userptr.notifier_lock); 409 410 return (list_empty(&vm->userptr.repin_list) && 411 list_empty(&vm->userptr.invalidated)) ? 0 : -EAGAIN; 412 } 413 414 #define XE_VM_REBIND_RETRY_TIMEOUT_MS 1000 415 416 static void xe_vm_kill(struct xe_vm *vm) 417 { 418 struct xe_exec_queue *q; 419 420 lockdep_assert_held(&vm->lock); 421 422 xe_vm_lock(vm, false); 423 vm->flags |= XE_VM_FLAG_BANNED; 424 trace_xe_vm_kill(vm); 425 426 list_for_each_entry(q, &vm->preempt.exec_queues, compute.link) 427 q->ops->kill(q); 428 xe_vm_unlock(vm); 429 430 /* TODO: Inform user the VM is banned */ 431 } 432 433 /** 434 * xe_vm_validate_should_retry() - Whether to retry after a validate error. 435 * @exec: The drm_exec object used for locking before validation. 436 * @err: The error returned from ttm_bo_validate(). 437 * @end: A ktime_t cookie that should be set to 0 before first use and 438 * that should be reused on subsequent calls. 439 * 440 * With multiple active VMs, under memory pressure, it is possible that 441 * ttm_bo_validate() run into -EDEADLK and in such case returns -ENOMEM. 442 * Until ttm properly handles locking in such scenarios, best thing the 443 * driver can do is retry with a timeout. Check if that is necessary, and 444 * if so unlock the drm_exec's objects while keeping the ticket to prepare 445 * for a rerun. 446 * 447 * Return: true if a retry after drm_exec_init() is recommended; 448 * false otherwise. 449 */ 450 bool xe_vm_validate_should_retry(struct drm_exec *exec, int err, ktime_t *end) 451 { 452 ktime_t cur; 453 454 if (err != -ENOMEM) 455 return false; 456 457 cur = ktime_get(); 458 *end = *end ? : ktime_add_ms(cur, XE_VM_REBIND_RETRY_TIMEOUT_MS); 459 if (!ktime_before(cur, *end)) 460 return false; 461 462 msleep(20); 463 return true; 464 } 465 466 static int xe_gpuvm_validate(struct drm_gpuvm_bo *vm_bo, struct drm_exec *exec) 467 { 468 struct xe_vm *vm = gpuvm_to_vm(vm_bo->vm); 469 struct drm_gpuva *gpuva; 470 int ret; 471 472 lockdep_assert_held(&vm->lock); 473 drm_gpuvm_bo_for_each_va(gpuva, vm_bo) 474 list_move_tail(&gpuva_to_vma(gpuva)->combined_links.rebind, 475 &vm->rebind_list); 476 477 ret = xe_bo_validate(gem_to_xe_bo(vm_bo->obj), vm, false); 478 if (ret) 479 return ret; 480 481 vm_bo->evicted = false; 482 return 0; 483 } 484 485 /** 486 * xe_vm_validate_rebind() - Validate buffer objects and rebind vmas 487 * @vm: The vm for which we are rebinding. 488 * @exec: The struct drm_exec with the locked GEM objects. 489 * @num_fences: The number of fences to reserve for the operation, not 490 * including rebinds and validations. 491 * 492 * Validates all evicted gem objects and rebinds their vmas. Note that 493 * rebindings may cause evictions and hence the validation-rebind 494 * sequence is rerun until there are no more objects to validate. 495 * 496 * Return: 0 on success, negative error code on error. In particular, 497 * may return -EINTR or -ERESTARTSYS if interrupted, and -EDEADLK if 498 * the drm_exec transaction needs to be restarted. 499 */ 500 int xe_vm_validate_rebind(struct xe_vm *vm, struct drm_exec *exec, 501 unsigned int num_fences) 502 { 503 struct drm_gem_object *obj; 504 unsigned long index; 505 int ret; 506 507 do { 508 ret = drm_gpuvm_validate(&vm->gpuvm, exec); 509 if (ret) 510 return ret; 511 512 ret = xe_vm_rebind(vm, false); 513 if (ret) 514 return ret; 515 } while (!list_empty(&vm->gpuvm.evict.list)); 516 517 drm_exec_for_each_locked_object(exec, index, obj) { 518 ret = dma_resv_reserve_fences(obj->resv, num_fences); 519 if (ret) 520 return ret; 521 } 522 523 return 0; 524 } 525 526 static int xe_preempt_work_begin(struct drm_exec *exec, struct xe_vm *vm, 527 bool *done) 528 { 529 int err; 530 531 err = drm_gpuvm_prepare_vm(&vm->gpuvm, exec, 0); 532 if (err) 533 return err; 534 535 if (xe_vm_is_idle(vm)) { 536 vm->preempt.rebind_deactivated = true; 537 *done = true; 538 return 0; 539 } 540 541 if (!preempt_fences_waiting(vm)) { 542 *done = true; 543 return 0; 544 } 545 546 err = drm_gpuvm_prepare_objects(&vm->gpuvm, exec, 0); 547 if (err) 548 return err; 549 550 err = wait_for_existing_preempt_fences(vm); 551 if (err) 552 return err; 553 554 /* 555 * Add validation and rebinding to the locking loop since both can 556 * cause evictions which may require blocing dma_resv locks. 557 * The fence reservation here is intended for the new preempt fences 558 * we attach at the end of the rebind work. 559 */ 560 return xe_vm_validate_rebind(vm, exec, vm->preempt.num_exec_queues); 561 } 562 563 static void preempt_rebind_work_func(struct work_struct *w) 564 { 565 struct xe_vm *vm = container_of(w, struct xe_vm, preempt.rebind_work); 566 struct drm_exec exec; 567 unsigned int fence_count = 0; 568 LIST_HEAD(preempt_fences); 569 ktime_t end = 0; 570 int err = 0; 571 long wait; 572 int __maybe_unused tries = 0; 573 574 xe_assert(vm->xe, xe_vm_in_preempt_fence_mode(vm)); 575 trace_xe_vm_rebind_worker_enter(vm); 576 577 down_write(&vm->lock); 578 579 if (xe_vm_is_closed_or_banned(vm)) { 580 up_write(&vm->lock); 581 trace_xe_vm_rebind_worker_exit(vm); 582 return; 583 } 584 585 retry: 586 if (xe_vm_userptr_check_repin(vm)) { 587 err = xe_vm_userptr_pin(vm); 588 if (err) 589 goto out_unlock_outer; 590 } 591 592 drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0); 593 594 drm_exec_until_all_locked(&exec) { 595 bool done = false; 596 597 err = xe_preempt_work_begin(&exec, vm, &done); 598 drm_exec_retry_on_contention(&exec); 599 if (err || done) { 600 drm_exec_fini(&exec); 601 if (err && xe_vm_validate_should_retry(&exec, err, &end)) 602 err = -EAGAIN; 603 604 goto out_unlock_outer; 605 } 606 } 607 608 err = alloc_preempt_fences(vm, &preempt_fences, &fence_count); 609 if (err) 610 goto out_unlock; 611 612 err = xe_vm_rebind(vm, true); 613 if (err) 614 goto out_unlock; 615 616 /* Wait on rebinds and munmap style VM unbinds */ 617 wait = dma_resv_wait_timeout(xe_vm_resv(vm), 618 DMA_RESV_USAGE_KERNEL, 619 false, MAX_SCHEDULE_TIMEOUT); 620 if (wait <= 0) { 621 err = -ETIME; 622 goto out_unlock; 623 } 624 625 #define retry_required(__tries, __vm) \ 626 (IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT) ? \ 627 (!(__tries)++ || __xe_vm_userptr_needs_repin(__vm)) : \ 628 __xe_vm_userptr_needs_repin(__vm)) 629 630 down_read(&vm->userptr.notifier_lock); 631 if (retry_required(tries, vm)) { 632 up_read(&vm->userptr.notifier_lock); 633 err = -EAGAIN; 634 goto out_unlock; 635 } 636 637 #undef retry_required 638 639 spin_lock(&vm->xe->ttm.lru_lock); 640 ttm_lru_bulk_move_tail(&vm->lru_bulk_move); 641 spin_unlock(&vm->xe->ttm.lru_lock); 642 643 /* Point of no return. */ 644 arm_preempt_fences(vm, &preempt_fences); 645 resume_and_reinstall_preempt_fences(vm, &exec); 646 up_read(&vm->userptr.notifier_lock); 647 648 out_unlock: 649 drm_exec_fini(&exec); 650 out_unlock_outer: 651 if (err == -EAGAIN) { 652 trace_xe_vm_rebind_worker_retry(vm); 653 goto retry; 654 } 655 656 if (err) { 657 drm_warn(&vm->xe->drm, "VM worker error: %d\n", err); 658 xe_vm_kill(vm); 659 } 660 up_write(&vm->lock); 661 662 free_preempt_fences(&preempt_fences); 663 664 trace_xe_vm_rebind_worker_exit(vm); 665 } 666 667 static bool vma_userptr_invalidate(struct mmu_interval_notifier *mni, 668 const struct mmu_notifier_range *range, 669 unsigned long cur_seq) 670 { 671 struct xe_userptr *userptr = container_of(mni, typeof(*userptr), notifier); 672 struct xe_userptr_vma *uvma = container_of(userptr, typeof(*uvma), userptr); 673 struct xe_vma *vma = &uvma->vma; 674 struct xe_vm *vm = xe_vma_vm(vma); 675 struct dma_resv_iter cursor; 676 struct dma_fence *fence; 677 long err; 678 679 xe_assert(vm->xe, xe_vma_is_userptr(vma)); 680 trace_xe_vma_userptr_invalidate(vma); 681 682 if (!mmu_notifier_range_blockable(range)) 683 return false; 684 685 down_write(&vm->userptr.notifier_lock); 686 mmu_interval_set_seq(mni, cur_seq); 687 688 /* No need to stop gpu access if the userptr is not yet bound. */ 689 if (!userptr->initial_bind) { 690 up_write(&vm->userptr.notifier_lock); 691 return true; 692 } 693 694 /* 695 * Tell exec and rebind worker they need to repin and rebind this 696 * userptr. 697 */ 698 if (!xe_vm_in_fault_mode(vm) && 699 !(vma->gpuva.flags & XE_VMA_DESTROYED) && vma->tile_present) { 700 spin_lock(&vm->userptr.invalidated_lock); 701 list_move_tail(&userptr->invalidate_link, 702 &vm->userptr.invalidated); 703 spin_unlock(&vm->userptr.invalidated_lock); 704 } 705 706 up_write(&vm->userptr.notifier_lock); 707 708 /* 709 * Preempt fences turn into schedule disables, pipeline these. 710 * Note that even in fault mode, we need to wait for binds and 711 * unbinds to complete, and those are attached as BOOKMARK fences 712 * to the vm. 713 */ 714 dma_resv_iter_begin(&cursor, xe_vm_resv(vm), 715 DMA_RESV_USAGE_BOOKKEEP); 716 dma_resv_for_each_fence_unlocked(&cursor, fence) 717 dma_fence_enable_sw_signaling(fence); 718 dma_resv_iter_end(&cursor); 719 720 err = dma_resv_wait_timeout(xe_vm_resv(vm), 721 DMA_RESV_USAGE_BOOKKEEP, 722 false, MAX_SCHEDULE_TIMEOUT); 723 XE_WARN_ON(err <= 0); 724 725 if (xe_vm_in_fault_mode(vm)) { 726 err = xe_vm_invalidate_vma(vma); 727 XE_WARN_ON(err); 728 } 729 730 trace_xe_vma_userptr_invalidate_complete(vma); 731 732 return true; 733 } 734 735 static const struct mmu_interval_notifier_ops vma_userptr_notifier_ops = { 736 .invalidate = vma_userptr_invalidate, 737 }; 738 739 int xe_vm_userptr_pin(struct xe_vm *vm) 740 { 741 struct xe_userptr_vma *uvma, *next; 742 int err = 0; 743 LIST_HEAD(tmp_evict); 744 745 xe_assert(vm->xe, !xe_vm_in_fault_mode(vm)); 746 lockdep_assert_held_write(&vm->lock); 747 748 /* Collect invalidated userptrs */ 749 spin_lock(&vm->userptr.invalidated_lock); 750 list_for_each_entry_safe(uvma, next, &vm->userptr.invalidated, 751 userptr.invalidate_link) { 752 list_del_init(&uvma->userptr.invalidate_link); 753 list_move_tail(&uvma->userptr.repin_link, 754 &vm->userptr.repin_list); 755 } 756 spin_unlock(&vm->userptr.invalidated_lock); 757 758 /* Pin and move to temporary list */ 759 list_for_each_entry_safe(uvma, next, &vm->userptr.repin_list, 760 userptr.repin_link) { 761 err = xe_vma_userptr_pin_pages(uvma); 762 if (err == -EFAULT) { 763 list_del_init(&uvma->userptr.repin_link); 764 765 /* Wait for pending binds */ 766 xe_vm_lock(vm, false); 767 dma_resv_wait_timeout(xe_vm_resv(vm), 768 DMA_RESV_USAGE_BOOKKEEP, 769 false, MAX_SCHEDULE_TIMEOUT); 770 771 err = xe_vm_invalidate_vma(&uvma->vma); 772 xe_vm_unlock(vm); 773 if (err) 774 return err; 775 } else { 776 if (err < 0) 777 return err; 778 779 list_del_init(&uvma->userptr.repin_link); 780 list_move_tail(&uvma->vma.combined_links.rebind, 781 &vm->rebind_list); 782 } 783 } 784 785 return 0; 786 } 787 788 /** 789 * xe_vm_userptr_check_repin() - Check whether the VM might have userptrs 790 * that need repinning. 791 * @vm: The VM. 792 * 793 * This function does an advisory check for whether the VM has userptrs that 794 * need repinning. 795 * 796 * Return: 0 if there are no indications of userptrs needing repinning, 797 * -EAGAIN if there are. 798 */ 799 int xe_vm_userptr_check_repin(struct xe_vm *vm) 800 { 801 return (list_empty_careful(&vm->userptr.repin_list) && 802 list_empty_careful(&vm->userptr.invalidated)) ? 0 : -EAGAIN; 803 } 804 805 static struct dma_fence * 806 xe_vm_bind_vma(struct xe_vma *vma, struct xe_exec_queue *q, 807 struct xe_sync_entry *syncs, u32 num_syncs, 808 bool first_op, bool last_op); 809 810 int xe_vm_rebind(struct xe_vm *vm, bool rebind_worker) 811 { 812 struct dma_fence *fence; 813 struct xe_vma *vma, *next; 814 815 lockdep_assert_held(&vm->lock); 816 if (xe_vm_in_lr_mode(vm) && !rebind_worker) 817 return 0; 818 819 xe_vm_assert_held(vm); 820 list_for_each_entry_safe(vma, next, &vm->rebind_list, 821 combined_links.rebind) { 822 xe_assert(vm->xe, vma->tile_present); 823 824 list_del_init(&vma->combined_links.rebind); 825 if (rebind_worker) 826 trace_xe_vma_rebind_worker(vma); 827 else 828 trace_xe_vma_rebind_exec(vma); 829 fence = xe_vm_bind_vma(vma, NULL, NULL, 0, false, false); 830 if (IS_ERR(fence)) 831 return PTR_ERR(fence); 832 dma_fence_put(fence); 833 } 834 835 return 0; 836 } 837 838 static void xe_vma_free(struct xe_vma *vma) 839 { 840 if (xe_vma_is_userptr(vma)) 841 kfree(to_userptr_vma(vma)); 842 else 843 kfree(vma); 844 } 845 846 #define VMA_CREATE_FLAG_READ_ONLY BIT(0) 847 #define VMA_CREATE_FLAG_IS_NULL BIT(1) 848 #define VMA_CREATE_FLAG_DUMPABLE BIT(2) 849 850 static struct xe_vma *xe_vma_create(struct xe_vm *vm, 851 struct xe_bo *bo, 852 u64 bo_offset_or_userptr, 853 u64 start, u64 end, 854 u16 pat_index, unsigned int flags) 855 { 856 struct xe_vma *vma; 857 struct xe_tile *tile; 858 u8 id; 859 bool read_only = (flags & VMA_CREATE_FLAG_READ_ONLY); 860 bool is_null = (flags & VMA_CREATE_FLAG_IS_NULL); 861 bool dumpable = (flags & VMA_CREATE_FLAG_DUMPABLE); 862 863 xe_assert(vm->xe, start < end); 864 xe_assert(vm->xe, end < vm->size); 865 866 /* 867 * Allocate and ensure that the xe_vma_is_userptr() return 868 * matches what was allocated. 869 */ 870 if (!bo && !is_null) { 871 struct xe_userptr_vma *uvma = kzalloc(sizeof(*uvma), GFP_KERNEL); 872 873 if (!uvma) 874 return ERR_PTR(-ENOMEM); 875 876 vma = &uvma->vma; 877 } else { 878 vma = kzalloc(sizeof(*vma), GFP_KERNEL); 879 if (!vma) 880 return ERR_PTR(-ENOMEM); 881 882 if (is_null) 883 vma->gpuva.flags |= DRM_GPUVA_SPARSE; 884 if (bo) 885 vma->gpuva.gem.obj = &bo->ttm.base; 886 } 887 888 INIT_LIST_HEAD(&vma->combined_links.rebind); 889 890 INIT_LIST_HEAD(&vma->gpuva.gem.entry); 891 vma->gpuva.vm = &vm->gpuvm; 892 vma->gpuva.va.addr = start; 893 vma->gpuva.va.range = end - start + 1; 894 if (read_only) 895 vma->gpuva.flags |= XE_VMA_READ_ONLY; 896 if (dumpable) 897 vma->gpuva.flags |= XE_VMA_DUMPABLE; 898 899 for_each_tile(tile, vm->xe, id) 900 vma->tile_mask |= 0x1 << id; 901 902 if (GRAPHICS_VER(vm->xe) >= 20 || vm->xe->info.platform == XE_PVC) 903 vma->gpuva.flags |= XE_VMA_ATOMIC_PTE_BIT; 904 905 vma->pat_index = pat_index; 906 907 if (bo) { 908 struct drm_gpuvm_bo *vm_bo; 909 910 xe_bo_assert_held(bo); 911 912 vm_bo = drm_gpuvm_bo_obtain(vma->gpuva.vm, &bo->ttm.base); 913 if (IS_ERR(vm_bo)) { 914 xe_vma_free(vma); 915 return ERR_CAST(vm_bo); 916 } 917 918 drm_gpuvm_bo_extobj_add(vm_bo); 919 drm_gem_object_get(&bo->ttm.base); 920 vma->gpuva.gem.offset = bo_offset_or_userptr; 921 drm_gpuva_link(&vma->gpuva, vm_bo); 922 drm_gpuvm_bo_put(vm_bo); 923 } else /* userptr or null */ { 924 if (!is_null) { 925 struct xe_userptr *userptr = &to_userptr_vma(vma)->userptr; 926 u64 size = end - start + 1; 927 int err; 928 929 INIT_LIST_HEAD(&userptr->invalidate_link); 930 INIT_LIST_HEAD(&userptr->repin_link); 931 vma->gpuva.gem.offset = bo_offset_or_userptr; 932 933 err = mmu_interval_notifier_insert(&userptr->notifier, 934 current->mm, 935 xe_vma_userptr(vma), size, 936 &vma_userptr_notifier_ops); 937 if (err) { 938 xe_vma_free(vma); 939 return ERR_PTR(err); 940 } 941 942 userptr->notifier_seq = LONG_MAX; 943 } 944 945 xe_vm_get(vm); 946 } 947 948 return vma; 949 } 950 951 static void xe_vma_destroy_late(struct xe_vma *vma) 952 { 953 struct xe_vm *vm = xe_vma_vm(vma); 954 struct xe_device *xe = vm->xe; 955 bool read_only = xe_vma_read_only(vma); 956 957 if (vma->ufence) { 958 xe_sync_ufence_put(vma->ufence); 959 vma->ufence = NULL; 960 } 961 962 if (xe_vma_is_userptr(vma)) { 963 struct xe_userptr *userptr = &to_userptr_vma(vma)->userptr; 964 965 if (userptr->sg) { 966 dma_unmap_sgtable(xe->drm.dev, 967 userptr->sg, 968 read_only ? DMA_TO_DEVICE : 969 DMA_BIDIRECTIONAL, 0); 970 sg_free_table(userptr->sg); 971 userptr->sg = NULL; 972 } 973 974 /* 975 * Since userptr pages are not pinned, we can't remove 976 * the notifer until we're sure the GPU is not accessing 977 * them anymore 978 */ 979 mmu_interval_notifier_remove(&userptr->notifier); 980 xe_vm_put(vm); 981 } else if (xe_vma_is_null(vma)) { 982 xe_vm_put(vm); 983 } else { 984 xe_bo_put(xe_vma_bo(vma)); 985 } 986 987 xe_vma_free(vma); 988 } 989 990 static void vma_destroy_work_func(struct work_struct *w) 991 { 992 struct xe_vma *vma = 993 container_of(w, struct xe_vma, destroy_work); 994 995 xe_vma_destroy_late(vma); 996 } 997 998 static void vma_destroy_cb(struct dma_fence *fence, 999 struct dma_fence_cb *cb) 1000 { 1001 struct xe_vma *vma = container_of(cb, struct xe_vma, destroy_cb); 1002 1003 INIT_WORK(&vma->destroy_work, vma_destroy_work_func); 1004 queue_work(system_unbound_wq, &vma->destroy_work); 1005 } 1006 1007 static void xe_vma_destroy(struct xe_vma *vma, struct dma_fence *fence) 1008 { 1009 struct xe_vm *vm = xe_vma_vm(vma); 1010 1011 lockdep_assert_held_write(&vm->lock); 1012 xe_assert(vm->xe, list_empty(&vma->combined_links.destroy)); 1013 1014 if (xe_vma_is_userptr(vma)) { 1015 xe_assert(vm->xe, vma->gpuva.flags & XE_VMA_DESTROYED); 1016 1017 spin_lock(&vm->userptr.invalidated_lock); 1018 list_del(&to_userptr_vma(vma)->userptr.invalidate_link); 1019 spin_unlock(&vm->userptr.invalidated_lock); 1020 } else if (!xe_vma_is_null(vma)) { 1021 xe_bo_assert_held(xe_vma_bo(vma)); 1022 1023 drm_gpuva_unlink(&vma->gpuva); 1024 } 1025 1026 xe_vm_assert_held(vm); 1027 if (fence) { 1028 int ret = dma_fence_add_callback(fence, &vma->destroy_cb, 1029 vma_destroy_cb); 1030 1031 if (ret) { 1032 XE_WARN_ON(ret != -ENOENT); 1033 xe_vma_destroy_late(vma); 1034 } 1035 } else { 1036 xe_vma_destroy_late(vma); 1037 } 1038 } 1039 1040 /** 1041 * xe_vm_lock_vma() - drm_exec utility to lock a vma 1042 * @exec: The drm_exec object we're currently locking for. 1043 * @vma: The vma for witch we want to lock the vm resv and any attached 1044 * object's resv. 1045 * 1046 * Return: 0 on success, negative error code on error. In particular 1047 * may return -EDEADLK on WW transaction contention and -EINTR if 1048 * an interruptible wait is terminated by a signal. 1049 */ 1050 int xe_vm_lock_vma(struct drm_exec *exec, struct xe_vma *vma) 1051 { 1052 struct xe_vm *vm = xe_vma_vm(vma); 1053 struct xe_bo *bo = xe_vma_bo(vma); 1054 int err; 1055 1056 XE_WARN_ON(!vm); 1057 1058 err = drm_exec_lock_obj(exec, xe_vm_obj(vm)); 1059 if (!err && bo && !bo->vm) 1060 err = drm_exec_lock_obj(exec, &bo->ttm.base); 1061 1062 return err; 1063 } 1064 1065 static void xe_vma_destroy_unlocked(struct xe_vma *vma) 1066 { 1067 struct drm_exec exec; 1068 int err; 1069 1070 drm_exec_init(&exec, 0, 0); 1071 drm_exec_until_all_locked(&exec) { 1072 err = xe_vm_lock_vma(&exec, vma); 1073 drm_exec_retry_on_contention(&exec); 1074 if (XE_WARN_ON(err)) 1075 break; 1076 } 1077 1078 xe_vma_destroy(vma, NULL); 1079 1080 drm_exec_fini(&exec); 1081 } 1082 1083 struct xe_vma * 1084 xe_vm_find_overlapping_vma(struct xe_vm *vm, u64 start, u64 range) 1085 { 1086 struct drm_gpuva *gpuva; 1087 1088 lockdep_assert_held(&vm->lock); 1089 1090 if (xe_vm_is_closed_or_banned(vm)) 1091 return NULL; 1092 1093 xe_assert(vm->xe, start + range <= vm->size); 1094 1095 gpuva = drm_gpuva_find_first(&vm->gpuvm, start, range); 1096 1097 return gpuva ? gpuva_to_vma(gpuva) : NULL; 1098 } 1099 1100 static int xe_vm_insert_vma(struct xe_vm *vm, struct xe_vma *vma) 1101 { 1102 int err; 1103 1104 xe_assert(vm->xe, xe_vma_vm(vma) == vm); 1105 lockdep_assert_held(&vm->lock); 1106 1107 mutex_lock(&vm->snap_mutex); 1108 err = drm_gpuva_insert(&vm->gpuvm, &vma->gpuva); 1109 mutex_unlock(&vm->snap_mutex); 1110 XE_WARN_ON(err); /* Shouldn't be possible */ 1111 1112 return err; 1113 } 1114 1115 static void xe_vm_remove_vma(struct xe_vm *vm, struct xe_vma *vma) 1116 { 1117 xe_assert(vm->xe, xe_vma_vm(vma) == vm); 1118 lockdep_assert_held(&vm->lock); 1119 1120 mutex_lock(&vm->snap_mutex); 1121 drm_gpuva_remove(&vma->gpuva); 1122 mutex_unlock(&vm->snap_mutex); 1123 if (vm->usm.last_fault_vma == vma) 1124 vm->usm.last_fault_vma = NULL; 1125 } 1126 1127 static struct drm_gpuva_op *xe_vm_op_alloc(void) 1128 { 1129 struct xe_vma_op *op; 1130 1131 op = kzalloc(sizeof(*op), GFP_KERNEL); 1132 1133 if (unlikely(!op)) 1134 return NULL; 1135 1136 return &op->base; 1137 } 1138 1139 static void xe_vm_free(struct drm_gpuvm *gpuvm); 1140 1141 static const struct drm_gpuvm_ops gpuvm_ops = { 1142 .op_alloc = xe_vm_op_alloc, 1143 .vm_bo_validate = xe_gpuvm_validate, 1144 .vm_free = xe_vm_free, 1145 }; 1146 1147 static u64 pde_encode_pat_index(struct xe_device *xe, u16 pat_index) 1148 { 1149 u64 pte = 0; 1150 1151 if (pat_index & BIT(0)) 1152 pte |= XE_PPGTT_PTE_PAT0; 1153 1154 if (pat_index & BIT(1)) 1155 pte |= XE_PPGTT_PTE_PAT1; 1156 1157 return pte; 1158 } 1159 1160 static u64 pte_encode_pat_index(struct xe_device *xe, u16 pat_index, 1161 u32 pt_level) 1162 { 1163 u64 pte = 0; 1164 1165 if (pat_index & BIT(0)) 1166 pte |= XE_PPGTT_PTE_PAT0; 1167 1168 if (pat_index & BIT(1)) 1169 pte |= XE_PPGTT_PTE_PAT1; 1170 1171 if (pat_index & BIT(2)) { 1172 if (pt_level) 1173 pte |= XE_PPGTT_PDE_PDPE_PAT2; 1174 else 1175 pte |= XE_PPGTT_PTE_PAT2; 1176 } 1177 1178 if (pat_index & BIT(3)) 1179 pte |= XELPG_PPGTT_PTE_PAT3; 1180 1181 if (pat_index & (BIT(4))) 1182 pte |= XE2_PPGTT_PTE_PAT4; 1183 1184 return pte; 1185 } 1186 1187 static u64 pte_encode_ps(u32 pt_level) 1188 { 1189 XE_WARN_ON(pt_level > MAX_HUGEPTE_LEVEL); 1190 1191 if (pt_level == 1) 1192 return XE_PDE_PS_2M; 1193 else if (pt_level == 2) 1194 return XE_PDPE_PS_1G; 1195 1196 return 0; 1197 } 1198 1199 static u64 xelp_pde_encode_bo(struct xe_bo *bo, u64 bo_offset, 1200 const u16 pat_index) 1201 { 1202 struct xe_device *xe = xe_bo_device(bo); 1203 u64 pde; 1204 1205 pde = xe_bo_addr(bo, bo_offset, XE_PAGE_SIZE); 1206 pde |= XE_PAGE_PRESENT | XE_PAGE_RW; 1207 pde |= pde_encode_pat_index(xe, pat_index); 1208 1209 return pde; 1210 } 1211 1212 static u64 xelp_pte_encode_bo(struct xe_bo *bo, u64 bo_offset, 1213 u16 pat_index, u32 pt_level) 1214 { 1215 struct xe_device *xe = xe_bo_device(bo); 1216 u64 pte; 1217 1218 pte = xe_bo_addr(bo, bo_offset, XE_PAGE_SIZE); 1219 pte |= XE_PAGE_PRESENT | XE_PAGE_RW; 1220 pte |= pte_encode_pat_index(xe, pat_index, pt_level); 1221 pte |= pte_encode_ps(pt_level); 1222 1223 if (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo)) 1224 pte |= XE_PPGTT_PTE_DM; 1225 1226 return pte; 1227 } 1228 1229 static u64 xelp_pte_encode_vma(u64 pte, struct xe_vma *vma, 1230 u16 pat_index, u32 pt_level) 1231 { 1232 struct xe_device *xe = xe_vma_vm(vma)->xe; 1233 1234 pte |= XE_PAGE_PRESENT; 1235 1236 if (likely(!xe_vma_read_only(vma))) 1237 pte |= XE_PAGE_RW; 1238 1239 pte |= pte_encode_pat_index(xe, pat_index, pt_level); 1240 pte |= pte_encode_ps(pt_level); 1241 1242 if (unlikely(xe_vma_is_null(vma))) 1243 pte |= XE_PTE_NULL; 1244 1245 return pte; 1246 } 1247 1248 static u64 xelp_pte_encode_addr(struct xe_device *xe, u64 addr, 1249 u16 pat_index, 1250 u32 pt_level, bool devmem, u64 flags) 1251 { 1252 u64 pte; 1253 1254 /* Avoid passing random bits directly as flags */ 1255 xe_assert(xe, !(flags & ~XE_PTE_PS64)); 1256 1257 pte = addr; 1258 pte |= XE_PAGE_PRESENT | XE_PAGE_RW; 1259 pte |= pte_encode_pat_index(xe, pat_index, pt_level); 1260 pte |= pte_encode_ps(pt_level); 1261 1262 if (devmem) 1263 pte |= XE_PPGTT_PTE_DM; 1264 1265 pte |= flags; 1266 1267 return pte; 1268 } 1269 1270 static const struct xe_pt_ops xelp_pt_ops = { 1271 .pte_encode_bo = xelp_pte_encode_bo, 1272 .pte_encode_vma = xelp_pte_encode_vma, 1273 .pte_encode_addr = xelp_pte_encode_addr, 1274 .pde_encode_bo = xelp_pde_encode_bo, 1275 }; 1276 1277 static void vm_destroy_work_func(struct work_struct *w); 1278 1279 /** 1280 * xe_vm_create_scratch() - Setup a scratch memory pagetable tree for the 1281 * given tile and vm. 1282 * @xe: xe device. 1283 * @tile: tile to set up for. 1284 * @vm: vm to set up for. 1285 * 1286 * Sets up a pagetable tree with one page-table per level and a single 1287 * leaf PTE. All pagetable entries point to the single page-table or, 1288 * for MAX_HUGEPTE_LEVEL, a NULL huge PTE returning 0 on read and 1289 * writes become NOPs. 1290 * 1291 * Return: 0 on success, negative error code on error. 1292 */ 1293 static int xe_vm_create_scratch(struct xe_device *xe, struct xe_tile *tile, 1294 struct xe_vm *vm) 1295 { 1296 u8 id = tile->id; 1297 int i; 1298 1299 for (i = MAX_HUGEPTE_LEVEL; i < vm->pt_root[id]->level; i++) { 1300 vm->scratch_pt[id][i] = xe_pt_create(vm, tile, i); 1301 if (IS_ERR(vm->scratch_pt[id][i])) 1302 return PTR_ERR(vm->scratch_pt[id][i]); 1303 1304 xe_pt_populate_empty(tile, vm, vm->scratch_pt[id][i]); 1305 } 1306 1307 return 0; 1308 } 1309 1310 static void xe_vm_free_scratch(struct xe_vm *vm) 1311 { 1312 struct xe_tile *tile; 1313 u8 id; 1314 1315 if (!xe_vm_has_scratch(vm)) 1316 return; 1317 1318 for_each_tile(tile, vm->xe, id) { 1319 u32 i; 1320 1321 if (!vm->pt_root[id]) 1322 continue; 1323 1324 for (i = MAX_HUGEPTE_LEVEL; i < vm->pt_root[id]->level; ++i) 1325 if (vm->scratch_pt[id][i]) 1326 xe_pt_destroy(vm->scratch_pt[id][i], vm->flags, NULL); 1327 } 1328 } 1329 1330 struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags) 1331 { 1332 struct drm_gem_object *vm_resv_obj; 1333 struct xe_vm *vm; 1334 int err, number_tiles = 0; 1335 struct xe_tile *tile; 1336 u8 id; 1337 1338 vm = kzalloc(sizeof(*vm), GFP_KERNEL); 1339 if (!vm) 1340 return ERR_PTR(-ENOMEM); 1341 1342 vm->xe = xe; 1343 1344 vm->size = 1ull << xe->info.va_bits; 1345 1346 vm->flags = flags; 1347 1348 init_rwsem(&vm->lock); 1349 mutex_init(&vm->snap_mutex); 1350 1351 INIT_LIST_HEAD(&vm->rebind_list); 1352 1353 INIT_LIST_HEAD(&vm->userptr.repin_list); 1354 INIT_LIST_HEAD(&vm->userptr.invalidated); 1355 init_rwsem(&vm->userptr.notifier_lock); 1356 spin_lock_init(&vm->userptr.invalidated_lock); 1357 1358 INIT_WORK(&vm->destroy_work, vm_destroy_work_func); 1359 1360 INIT_LIST_HEAD(&vm->preempt.exec_queues); 1361 vm->preempt.min_run_period_ms = 10; /* FIXME: Wire up to uAPI */ 1362 1363 for_each_tile(tile, xe, id) 1364 xe_range_fence_tree_init(&vm->rftree[id]); 1365 1366 vm->pt_ops = &xelp_pt_ops; 1367 1368 if (!(flags & XE_VM_FLAG_MIGRATION)) 1369 xe_device_mem_access_get(xe); 1370 1371 vm_resv_obj = drm_gpuvm_resv_object_alloc(&xe->drm); 1372 if (!vm_resv_obj) { 1373 err = -ENOMEM; 1374 goto err_no_resv; 1375 } 1376 1377 drm_gpuvm_init(&vm->gpuvm, "Xe VM", DRM_GPUVM_RESV_PROTECTED, &xe->drm, 1378 vm_resv_obj, 0, vm->size, 0, 0, &gpuvm_ops); 1379 1380 drm_gem_object_put(vm_resv_obj); 1381 1382 err = dma_resv_lock_interruptible(xe_vm_resv(vm), NULL); 1383 if (err) 1384 goto err_close; 1385 1386 if (IS_DGFX(xe) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) 1387 vm->flags |= XE_VM_FLAG_64K; 1388 1389 for_each_tile(tile, xe, id) { 1390 if (flags & XE_VM_FLAG_MIGRATION && 1391 tile->id != XE_VM_FLAG_TILE_ID(flags)) 1392 continue; 1393 1394 vm->pt_root[id] = xe_pt_create(vm, tile, xe->info.vm_max_level); 1395 if (IS_ERR(vm->pt_root[id])) { 1396 err = PTR_ERR(vm->pt_root[id]); 1397 vm->pt_root[id] = NULL; 1398 goto err_unlock_close; 1399 } 1400 } 1401 1402 if (xe_vm_has_scratch(vm)) { 1403 for_each_tile(tile, xe, id) { 1404 if (!vm->pt_root[id]) 1405 continue; 1406 1407 err = xe_vm_create_scratch(xe, tile, vm); 1408 if (err) 1409 goto err_unlock_close; 1410 } 1411 vm->batch_invalidate_tlb = true; 1412 } 1413 1414 if (flags & XE_VM_FLAG_LR_MODE) { 1415 INIT_WORK(&vm->preempt.rebind_work, preempt_rebind_work_func); 1416 vm->flags |= XE_VM_FLAG_LR_MODE; 1417 vm->batch_invalidate_tlb = false; 1418 } 1419 1420 /* Fill pt_root after allocating scratch tables */ 1421 for_each_tile(tile, xe, id) { 1422 if (!vm->pt_root[id]) 1423 continue; 1424 1425 xe_pt_populate_empty(tile, vm, vm->pt_root[id]); 1426 } 1427 dma_resv_unlock(xe_vm_resv(vm)); 1428 1429 /* Kernel migration VM shouldn't have a circular loop.. */ 1430 if (!(flags & XE_VM_FLAG_MIGRATION)) { 1431 for_each_tile(tile, xe, id) { 1432 struct xe_gt *gt = tile->primary_gt; 1433 struct xe_vm *migrate_vm; 1434 struct xe_exec_queue *q; 1435 u32 create_flags = EXEC_QUEUE_FLAG_VM; 1436 1437 if (!vm->pt_root[id]) 1438 continue; 1439 1440 migrate_vm = xe_migrate_get_vm(tile->migrate); 1441 q = xe_exec_queue_create_class(xe, gt, migrate_vm, 1442 XE_ENGINE_CLASS_COPY, 1443 create_flags); 1444 xe_vm_put(migrate_vm); 1445 if (IS_ERR(q)) { 1446 err = PTR_ERR(q); 1447 goto err_close; 1448 } 1449 vm->q[id] = q; 1450 number_tiles++; 1451 } 1452 } 1453 1454 if (number_tiles > 1) 1455 vm->composite_fence_ctx = dma_fence_context_alloc(1); 1456 1457 mutex_lock(&xe->usm.lock); 1458 if (flags & XE_VM_FLAG_FAULT_MODE) 1459 xe->usm.num_vm_in_fault_mode++; 1460 else if (!(flags & XE_VM_FLAG_MIGRATION)) 1461 xe->usm.num_vm_in_non_fault_mode++; 1462 mutex_unlock(&xe->usm.lock); 1463 1464 trace_xe_vm_create(vm); 1465 1466 return vm; 1467 1468 err_unlock_close: 1469 dma_resv_unlock(xe_vm_resv(vm)); 1470 err_close: 1471 xe_vm_close_and_put(vm); 1472 return ERR_PTR(err); 1473 1474 err_no_resv: 1475 mutex_destroy(&vm->snap_mutex); 1476 for_each_tile(tile, xe, id) 1477 xe_range_fence_tree_fini(&vm->rftree[id]); 1478 kfree(vm); 1479 if (!(flags & XE_VM_FLAG_MIGRATION)) 1480 xe_device_mem_access_put(xe); 1481 return ERR_PTR(err); 1482 } 1483 1484 static void xe_vm_close(struct xe_vm *vm) 1485 { 1486 down_write(&vm->lock); 1487 vm->size = 0; 1488 up_write(&vm->lock); 1489 } 1490 1491 void xe_vm_close_and_put(struct xe_vm *vm) 1492 { 1493 LIST_HEAD(contested); 1494 struct xe_device *xe = vm->xe; 1495 struct xe_tile *tile; 1496 struct xe_vma *vma, *next_vma; 1497 struct drm_gpuva *gpuva, *next; 1498 u8 id; 1499 1500 xe_assert(xe, !vm->preempt.num_exec_queues); 1501 1502 xe_vm_close(vm); 1503 if (xe_vm_in_preempt_fence_mode(vm)) 1504 flush_work(&vm->preempt.rebind_work); 1505 1506 down_write(&vm->lock); 1507 for_each_tile(tile, xe, id) { 1508 if (vm->q[id]) 1509 xe_exec_queue_last_fence_put(vm->q[id], vm); 1510 } 1511 up_write(&vm->lock); 1512 1513 for_each_tile(tile, xe, id) { 1514 if (vm->q[id]) { 1515 xe_exec_queue_kill(vm->q[id]); 1516 xe_exec_queue_put(vm->q[id]); 1517 vm->q[id] = NULL; 1518 } 1519 } 1520 1521 down_write(&vm->lock); 1522 xe_vm_lock(vm, false); 1523 drm_gpuvm_for_each_va_safe(gpuva, next, &vm->gpuvm) { 1524 vma = gpuva_to_vma(gpuva); 1525 1526 if (xe_vma_has_no_bo(vma)) { 1527 down_read(&vm->userptr.notifier_lock); 1528 vma->gpuva.flags |= XE_VMA_DESTROYED; 1529 up_read(&vm->userptr.notifier_lock); 1530 } 1531 1532 xe_vm_remove_vma(vm, vma); 1533 1534 /* easy case, remove from VMA? */ 1535 if (xe_vma_has_no_bo(vma) || xe_vma_bo(vma)->vm) { 1536 list_del_init(&vma->combined_links.rebind); 1537 xe_vma_destroy(vma, NULL); 1538 continue; 1539 } 1540 1541 list_move_tail(&vma->combined_links.destroy, &contested); 1542 vma->gpuva.flags |= XE_VMA_DESTROYED; 1543 } 1544 1545 /* 1546 * All vm operations will add shared fences to resv. 1547 * The only exception is eviction for a shared object, 1548 * but even so, the unbind when evicted would still 1549 * install a fence to resv. Hence it's safe to 1550 * destroy the pagetables immediately. 1551 */ 1552 xe_vm_free_scratch(vm); 1553 1554 for_each_tile(tile, xe, id) { 1555 if (vm->pt_root[id]) { 1556 xe_pt_destroy(vm->pt_root[id], vm->flags, NULL); 1557 vm->pt_root[id] = NULL; 1558 } 1559 } 1560 xe_vm_unlock(vm); 1561 1562 /* 1563 * VM is now dead, cannot re-add nodes to vm->vmas if it's NULL 1564 * Since we hold a refcount to the bo, we can remove and free 1565 * the members safely without locking. 1566 */ 1567 list_for_each_entry_safe(vma, next_vma, &contested, 1568 combined_links.destroy) { 1569 list_del_init(&vma->combined_links.destroy); 1570 xe_vma_destroy_unlocked(vma); 1571 } 1572 1573 up_write(&vm->lock); 1574 1575 mutex_lock(&xe->usm.lock); 1576 if (vm->flags & XE_VM_FLAG_FAULT_MODE) 1577 xe->usm.num_vm_in_fault_mode--; 1578 else if (!(vm->flags & XE_VM_FLAG_MIGRATION)) 1579 xe->usm.num_vm_in_non_fault_mode--; 1580 mutex_unlock(&xe->usm.lock); 1581 1582 for_each_tile(tile, xe, id) 1583 xe_range_fence_tree_fini(&vm->rftree[id]); 1584 1585 xe_vm_put(vm); 1586 } 1587 1588 static void vm_destroy_work_func(struct work_struct *w) 1589 { 1590 struct xe_vm *vm = 1591 container_of(w, struct xe_vm, destroy_work); 1592 struct xe_device *xe = vm->xe; 1593 struct xe_tile *tile; 1594 u8 id; 1595 void *lookup; 1596 1597 /* xe_vm_close_and_put was not called? */ 1598 xe_assert(xe, !vm->size); 1599 1600 mutex_destroy(&vm->snap_mutex); 1601 1602 if (!(vm->flags & XE_VM_FLAG_MIGRATION)) { 1603 xe_device_mem_access_put(xe); 1604 1605 if (xe->info.has_asid && vm->usm.asid) { 1606 mutex_lock(&xe->usm.lock); 1607 lookup = xa_erase(&xe->usm.asid_to_vm, vm->usm.asid); 1608 xe_assert(xe, lookup == vm); 1609 mutex_unlock(&xe->usm.lock); 1610 } 1611 } 1612 1613 for_each_tile(tile, xe, id) 1614 XE_WARN_ON(vm->pt_root[id]); 1615 1616 trace_xe_vm_free(vm); 1617 kfree(vm); 1618 } 1619 1620 static void xe_vm_free(struct drm_gpuvm *gpuvm) 1621 { 1622 struct xe_vm *vm = container_of(gpuvm, struct xe_vm, gpuvm); 1623 1624 /* To destroy the VM we need to be able to sleep */ 1625 queue_work(system_unbound_wq, &vm->destroy_work); 1626 } 1627 1628 struct xe_vm *xe_vm_lookup(struct xe_file *xef, u32 id) 1629 { 1630 struct xe_vm *vm; 1631 1632 mutex_lock(&xef->vm.lock); 1633 vm = xa_load(&xef->vm.xa, id); 1634 if (vm) 1635 xe_vm_get(vm); 1636 mutex_unlock(&xef->vm.lock); 1637 1638 return vm; 1639 } 1640 1641 u64 xe_vm_pdp4_descriptor(struct xe_vm *vm, struct xe_tile *tile) 1642 { 1643 return vm->pt_ops->pde_encode_bo(vm->pt_root[tile->id]->bo, 0, 1644 tile_to_xe(tile)->pat.idx[XE_CACHE_WB]); 1645 } 1646 1647 static struct xe_exec_queue * 1648 to_wait_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q) 1649 { 1650 return q ? q : vm->q[0]; 1651 } 1652 1653 static struct dma_fence * 1654 xe_vm_unbind_vma(struct xe_vma *vma, struct xe_exec_queue *q, 1655 struct xe_sync_entry *syncs, u32 num_syncs, 1656 bool first_op, bool last_op) 1657 { 1658 struct xe_vm *vm = xe_vma_vm(vma); 1659 struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q); 1660 struct xe_tile *tile; 1661 struct dma_fence *fence = NULL; 1662 struct dma_fence **fences = NULL; 1663 struct dma_fence_array *cf = NULL; 1664 int cur_fence = 0, i; 1665 int number_tiles = hweight8(vma->tile_present); 1666 int err; 1667 u8 id; 1668 1669 trace_xe_vma_unbind(vma); 1670 1671 if (vma->ufence) { 1672 struct xe_user_fence * const f = vma->ufence; 1673 1674 if (!xe_sync_ufence_get_status(f)) 1675 return ERR_PTR(-EBUSY); 1676 1677 vma->ufence = NULL; 1678 xe_sync_ufence_put(f); 1679 } 1680 1681 if (number_tiles > 1) { 1682 fences = kmalloc_array(number_tiles, sizeof(*fences), 1683 GFP_KERNEL); 1684 if (!fences) 1685 return ERR_PTR(-ENOMEM); 1686 } 1687 1688 for_each_tile(tile, vm->xe, id) { 1689 if (!(vma->tile_present & BIT(id))) 1690 goto next; 1691 1692 fence = __xe_pt_unbind_vma(tile, vma, q ? q : vm->q[id], 1693 first_op ? syncs : NULL, 1694 first_op ? num_syncs : 0); 1695 if (IS_ERR(fence)) { 1696 err = PTR_ERR(fence); 1697 goto err_fences; 1698 } 1699 1700 if (fences) 1701 fences[cur_fence++] = fence; 1702 1703 next: 1704 if (q && vm->pt_root[id] && !list_empty(&q->multi_gt_list)) 1705 q = list_next_entry(q, multi_gt_list); 1706 } 1707 1708 if (fences) { 1709 cf = dma_fence_array_create(number_tiles, fences, 1710 vm->composite_fence_ctx, 1711 vm->composite_fence_seqno++, 1712 false); 1713 if (!cf) { 1714 --vm->composite_fence_seqno; 1715 err = -ENOMEM; 1716 goto err_fences; 1717 } 1718 } 1719 1720 fence = cf ? &cf->base : !fence ? 1721 xe_exec_queue_last_fence_get(wait_exec_queue, vm) : fence; 1722 if (last_op) { 1723 for (i = 0; i < num_syncs; i++) 1724 xe_sync_entry_signal(&syncs[i], NULL, fence); 1725 } 1726 1727 return fence; 1728 1729 err_fences: 1730 if (fences) { 1731 while (cur_fence) 1732 dma_fence_put(fences[--cur_fence]); 1733 kfree(fences); 1734 } 1735 1736 return ERR_PTR(err); 1737 } 1738 1739 static struct dma_fence * 1740 xe_vm_bind_vma(struct xe_vma *vma, struct xe_exec_queue *q, 1741 struct xe_sync_entry *syncs, u32 num_syncs, 1742 bool first_op, bool last_op) 1743 { 1744 struct xe_tile *tile; 1745 struct dma_fence *fence; 1746 struct dma_fence **fences = NULL; 1747 struct dma_fence_array *cf = NULL; 1748 struct xe_vm *vm = xe_vma_vm(vma); 1749 int cur_fence = 0, i; 1750 int number_tiles = hweight8(vma->tile_mask); 1751 int err; 1752 u8 id; 1753 1754 trace_xe_vma_bind(vma); 1755 1756 if (number_tiles > 1) { 1757 fences = kmalloc_array(number_tiles, sizeof(*fences), 1758 GFP_KERNEL); 1759 if (!fences) 1760 return ERR_PTR(-ENOMEM); 1761 } 1762 1763 for_each_tile(tile, vm->xe, id) { 1764 if (!(vma->tile_mask & BIT(id))) 1765 goto next; 1766 1767 fence = __xe_pt_bind_vma(tile, vma, q ? q : vm->q[id], 1768 first_op ? syncs : NULL, 1769 first_op ? num_syncs : 0, 1770 vma->tile_present & BIT(id)); 1771 if (IS_ERR(fence)) { 1772 err = PTR_ERR(fence); 1773 goto err_fences; 1774 } 1775 1776 if (fences) 1777 fences[cur_fence++] = fence; 1778 1779 next: 1780 if (q && vm->pt_root[id] && !list_empty(&q->multi_gt_list)) 1781 q = list_next_entry(q, multi_gt_list); 1782 } 1783 1784 if (fences) { 1785 cf = dma_fence_array_create(number_tiles, fences, 1786 vm->composite_fence_ctx, 1787 vm->composite_fence_seqno++, 1788 false); 1789 if (!cf) { 1790 --vm->composite_fence_seqno; 1791 err = -ENOMEM; 1792 goto err_fences; 1793 } 1794 } 1795 1796 if (last_op) { 1797 for (i = 0; i < num_syncs; i++) 1798 xe_sync_entry_signal(&syncs[i], NULL, 1799 cf ? &cf->base : fence); 1800 } 1801 1802 return cf ? &cf->base : fence; 1803 1804 err_fences: 1805 if (fences) { 1806 while (cur_fence) 1807 dma_fence_put(fences[--cur_fence]); 1808 kfree(fences); 1809 } 1810 1811 return ERR_PTR(err); 1812 } 1813 1814 static struct xe_user_fence * 1815 find_ufence_get(struct xe_sync_entry *syncs, u32 num_syncs) 1816 { 1817 unsigned int i; 1818 1819 for (i = 0; i < num_syncs; i++) { 1820 struct xe_sync_entry *e = &syncs[i]; 1821 1822 if (xe_sync_is_ufence(e)) 1823 return xe_sync_ufence_get(e); 1824 } 1825 1826 return NULL; 1827 } 1828 1829 static int __xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, 1830 struct xe_exec_queue *q, struct xe_sync_entry *syncs, 1831 u32 num_syncs, bool immediate, bool first_op, 1832 bool last_op) 1833 { 1834 struct dma_fence *fence; 1835 struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q); 1836 struct xe_user_fence *ufence; 1837 1838 xe_vm_assert_held(vm); 1839 1840 ufence = find_ufence_get(syncs, num_syncs); 1841 if (vma->ufence && ufence) 1842 xe_sync_ufence_put(vma->ufence); 1843 1844 vma->ufence = ufence ?: vma->ufence; 1845 1846 if (immediate) { 1847 fence = xe_vm_bind_vma(vma, q, syncs, num_syncs, first_op, 1848 last_op); 1849 if (IS_ERR(fence)) 1850 return PTR_ERR(fence); 1851 } else { 1852 int i; 1853 1854 xe_assert(vm->xe, xe_vm_in_fault_mode(vm)); 1855 1856 fence = xe_exec_queue_last_fence_get(wait_exec_queue, vm); 1857 if (last_op) { 1858 for (i = 0; i < num_syncs; i++) 1859 xe_sync_entry_signal(&syncs[i], NULL, fence); 1860 } 1861 } 1862 1863 if (last_op) 1864 xe_exec_queue_last_fence_set(wait_exec_queue, vm, fence); 1865 dma_fence_put(fence); 1866 1867 return 0; 1868 } 1869 1870 static int xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, struct xe_exec_queue *q, 1871 struct xe_bo *bo, struct xe_sync_entry *syncs, 1872 u32 num_syncs, bool immediate, bool first_op, 1873 bool last_op) 1874 { 1875 int err; 1876 1877 xe_vm_assert_held(vm); 1878 xe_bo_assert_held(bo); 1879 1880 if (bo && immediate) { 1881 err = xe_bo_validate(bo, vm, true); 1882 if (err) 1883 return err; 1884 } 1885 1886 return __xe_vm_bind(vm, vma, q, syncs, num_syncs, immediate, first_op, 1887 last_op); 1888 } 1889 1890 static int xe_vm_unbind(struct xe_vm *vm, struct xe_vma *vma, 1891 struct xe_exec_queue *q, struct xe_sync_entry *syncs, 1892 u32 num_syncs, bool first_op, bool last_op) 1893 { 1894 struct dma_fence *fence; 1895 struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q); 1896 1897 xe_vm_assert_held(vm); 1898 xe_bo_assert_held(xe_vma_bo(vma)); 1899 1900 fence = xe_vm_unbind_vma(vma, q, syncs, num_syncs, first_op, last_op); 1901 if (IS_ERR(fence)) 1902 return PTR_ERR(fence); 1903 1904 xe_vma_destroy(vma, fence); 1905 if (last_op) 1906 xe_exec_queue_last_fence_set(wait_exec_queue, vm, fence); 1907 dma_fence_put(fence); 1908 1909 return 0; 1910 } 1911 1912 #define ALL_DRM_XE_VM_CREATE_FLAGS (DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE | \ 1913 DRM_XE_VM_CREATE_FLAG_LR_MODE | \ 1914 DRM_XE_VM_CREATE_FLAG_FAULT_MODE) 1915 1916 int xe_vm_create_ioctl(struct drm_device *dev, void *data, 1917 struct drm_file *file) 1918 { 1919 struct xe_device *xe = to_xe_device(dev); 1920 struct xe_file *xef = to_xe_file(file); 1921 struct drm_xe_vm_create *args = data; 1922 struct xe_tile *tile; 1923 struct xe_vm *vm; 1924 u32 id, asid; 1925 int err; 1926 u32 flags = 0; 1927 1928 if (XE_IOCTL_DBG(xe, args->extensions)) 1929 return -EINVAL; 1930 1931 if (XE_WA(xe_root_mmio_gt(xe), 14016763929)) 1932 args->flags |= DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE; 1933 1934 if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE && 1935 !xe->info.has_usm)) 1936 return -EINVAL; 1937 1938 if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 1939 return -EINVAL; 1940 1941 if (XE_IOCTL_DBG(xe, args->flags & ~ALL_DRM_XE_VM_CREATE_FLAGS)) 1942 return -EINVAL; 1943 1944 if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE && 1945 args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE)) 1946 return -EINVAL; 1947 1948 if (XE_IOCTL_DBG(xe, !(args->flags & DRM_XE_VM_CREATE_FLAG_LR_MODE) && 1949 args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE)) 1950 return -EINVAL; 1951 1952 if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE && 1953 xe_device_in_non_fault_mode(xe))) 1954 return -EINVAL; 1955 1956 if (XE_IOCTL_DBG(xe, !(args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE) && 1957 xe_device_in_fault_mode(xe))) 1958 return -EINVAL; 1959 1960 if (XE_IOCTL_DBG(xe, args->extensions)) 1961 return -EINVAL; 1962 1963 if (args->flags & DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE) 1964 flags |= XE_VM_FLAG_SCRATCH_PAGE; 1965 if (args->flags & DRM_XE_VM_CREATE_FLAG_LR_MODE) 1966 flags |= XE_VM_FLAG_LR_MODE; 1967 if (args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE) 1968 flags |= XE_VM_FLAG_FAULT_MODE; 1969 1970 vm = xe_vm_create(xe, flags); 1971 if (IS_ERR(vm)) 1972 return PTR_ERR(vm); 1973 1974 mutex_lock(&xef->vm.lock); 1975 err = xa_alloc(&xef->vm.xa, &id, vm, xa_limit_32b, GFP_KERNEL); 1976 mutex_unlock(&xef->vm.lock); 1977 if (err) 1978 goto err_close_and_put; 1979 1980 if (xe->info.has_asid) { 1981 mutex_lock(&xe->usm.lock); 1982 err = xa_alloc_cyclic(&xe->usm.asid_to_vm, &asid, vm, 1983 XA_LIMIT(1, XE_MAX_ASID - 1), 1984 &xe->usm.next_asid, GFP_KERNEL); 1985 mutex_unlock(&xe->usm.lock); 1986 if (err < 0) 1987 goto err_free_id; 1988 1989 vm->usm.asid = asid; 1990 } 1991 1992 args->vm_id = id; 1993 vm->xef = xef; 1994 1995 /* Record BO memory for VM pagetable created against client */ 1996 for_each_tile(tile, xe, id) 1997 if (vm->pt_root[id]) 1998 xe_drm_client_add_bo(vm->xef->client, vm->pt_root[id]->bo); 1999 2000 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEM) 2001 /* Warning: Security issue - never enable by default */ 2002 args->reserved[0] = xe_bo_main_addr(vm->pt_root[0]->bo, XE_PAGE_SIZE); 2003 #endif 2004 2005 return 0; 2006 2007 err_free_id: 2008 mutex_lock(&xef->vm.lock); 2009 xa_erase(&xef->vm.xa, id); 2010 mutex_unlock(&xef->vm.lock); 2011 err_close_and_put: 2012 xe_vm_close_and_put(vm); 2013 2014 return err; 2015 } 2016 2017 int xe_vm_destroy_ioctl(struct drm_device *dev, void *data, 2018 struct drm_file *file) 2019 { 2020 struct xe_device *xe = to_xe_device(dev); 2021 struct xe_file *xef = to_xe_file(file); 2022 struct drm_xe_vm_destroy *args = data; 2023 struct xe_vm *vm; 2024 int err = 0; 2025 2026 if (XE_IOCTL_DBG(xe, args->pad) || 2027 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 2028 return -EINVAL; 2029 2030 mutex_lock(&xef->vm.lock); 2031 vm = xa_load(&xef->vm.xa, args->vm_id); 2032 if (XE_IOCTL_DBG(xe, !vm)) 2033 err = -ENOENT; 2034 else if (XE_IOCTL_DBG(xe, vm->preempt.num_exec_queues)) 2035 err = -EBUSY; 2036 else 2037 xa_erase(&xef->vm.xa, args->vm_id); 2038 mutex_unlock(&xef->vm.lock); 2039 2040 if (!err) 2041 xe_vm_close_and_put(vm); 2042 2043 return err; 2044 } 2045 2046 static const u32 region_to_mem_type[] = { 2047 XE_PL_TT, 2048 XE_PL_VRAM0, 2049 XE_PL_VRAM1, 2050 }; 2051 2052 static int xe_vm_prefetch(struct xe_vm *vm, struct xe_vma *vma, 2053 struct xe_exec_queue *q, u32 region, 2054 struct xe_sync_entry *syncs, u32 num_syncs, 2055 bool first_op, bool last_op) 2056 { 2057 struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q); 2058 int err; 2059 2060 xe_assert(vm->xe, region <= ARRAY_SIZE(region_to_mem_type)); 2061 2062 if (!xe_vma_has_no_bo(vma)) { 2063 err = xe_bo_migrate(xe_vma_bo(vma), region_to_mem_type[region]); 2064 if (err) 2065 return err; 2066 } 2067 2068 if (vma->tile_mask != (vma->tile_present & ~vma->tile_invalidated)) { 2069 return xe_vm_bind(vm, vma, q, xe_vma_bo(vma), syncs, num_syncs, 2070 true, first_op, last_op); 2071 } else { 2072 int i; 2073 2074 /* Nothing to do, signal fences now */ 2075 if (last_op) { 2076 for (i = 0; i < num_syncs; i++) { 2077 struct dma_fence *fence = 2078 xe_exec_queue_last_fence_get(wait_exec_queue, vm); 2079 2080 xe_sync_entry_signal(&syncs[i], NULL, fence); 2081 dma_fence_put(fence); 2082 } 2083 } 2084 2085 return 0; 2086 } 2087 } 2088 2089 static void prep_vma_destroy(struct xe_vm *vm, struct xe_vma *vma, 2090 bool post_commit) 2091 { 2092 down_read(&vm->userptr.notifier_lock); 2093 vma->gpuva.flags |= XE_VMA_DESTROYED; 2094 up_read(&vm->userptr.notifier_lock); 2095 if (post_commit) 2096 xe_vm_remove_vma(vm, vma); 2097 } 2098 2099 #undef ULL 2100 #define ULL unsigned long long 2101 2102 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM) 2103 static void print_op(struct xe_device *xe, struct drm_gpuva_op *op) 2104 { 2105 struct xe_vma *vma; 2106 2107 switch (op->op) { 2108 case DRM_GPUVA_OP_MAP: 2109 vm_dbg(&xe->drm, "MAP: addr=0x%016llx, range=0x%016llx", 2110 (ULL)op->map.va.addr, (ULL)op->map.va.range); 2111 break; 2112 case DRM_GPUVA_OP_REMAP: 2113 vma = gpuva_to_vma(op->remap.unmap->va); 2114 vm_dbg(&xe->drm, "REMAP:UNMAP: addr=0x%016llx, range=0x%016llx, keep=%d", 2115 (ULL)xe_vma_start(vma), (ULL)xe_vma_size(vma), 2116 op->remap.unmap->keep ? 1 : 0); 2117 if (op->remap.prev) 2118 vm_dbg(&xe->drm, 2119 "REMAP:PREV: addr=0x%016llx, range=0x%016llx", 2120 (ULL)op->remap.prev->va.addr, 2121 (ULL)op->remap.prev->va.range); 2122 if (op->remap.next) 2123 vm_dbg(&xe->drm, 2124 "REMAP:NEXT: addr=0x%016llx, range=0x%016llx", 2125 (ULL)op->remap.next->va.addr, 2126 (ULL)op->remap.next->va.range); 2127 break; 2128 case DRM_GPUVA_OP_UNMAP: 2129 vma = gpuva_to_vma(op->unmap.va); 2130 vm_dbg(&xe->drm, "UNMAP: addr=0x%016llx, range=0x%016llx, keep=%d", 2131 (ULL)xe_vma_start(vma), (ULL)xe_vma_size(vma), 2132 op->unmap.keep ? 1 : 0); 2133 break; 2134 case DRM_GPUVA_OP_PREFETCH: 2135 vma = gpuva_to_vma(op->prefetch.va); 2136 vm_dbg(&xe->drm, "PREFETCH: addr=0x%016llx, range=0x%016llx", 2137 (ULL)xe_vma_start(vma), (ULL)xe_vma_size(vma)); 2138 break; 2139 default: 2140 drm_warn(&xe->drm, "NOT POSSIBLE"); 2141 } 2142 } 2143 #else 2144 static void print_op(struct xe_device *xe, struct drm_gpuva_op *op) 2145 { 2146 } 2147 #endif 2148 2149 /* 2150 * Create operations list from IOCTL arguments, setup operations fields so parse 2151 * and commit steps are decoupled from IOCTL arguments. This step can fail. 2152 */ 2153 static struct drm_gpuva_ops * 2154 vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_bo *bo, 2155 u64 bo_offset_or_userptr, u64 addr, u64 range, 2156 u32 operation, u32 flags, 2157 u32 prefetch_region, u16 pat_index) 2158 { 2159 struct drm_gem_object *obj = bo ? &bo->ttm.base : NULL; 2160 struct drm_gpuva_ops *ops; 2161 struct drm_gpuva_op *__op; 2162 struct drm_gpuvm_bo *vm_bo; 2163 int err; 2164 2165 lockdep_assert_held_write(&vm->lock); 2166 2167 vm_dbg(&vm->xe->drm, 2168 "op=%d, addr=0x%016llx, range=0x%016llx, bo_offset_or_userptr=0x%016llx", 2169 operation, (ULL)addr, (ULL)range, 2170 (ULL)bo_offset_or_userptr); 2171 2172 switch (operation) { 2173 case DRM_XE_VM_BIND_OP_MAP: 2174 case DRM_XE_VM_BIND_OP_MAP_USERPTR: 2175 ops = drm_gpuvm_sm_map_ops_create(&vm->gpuvm, addr, range, 2176 obj, bo_offset_or_userptr); 2177 break; 2178 case DRM_XE_VM_BIND_OP_UNMAP: 2179 ops = drm_gpuvm_sm_unmap_ops_create(&vm->gpuvm, addr, range); 2180 break; 2181 case DRM_XE_VM_BIND_OP_PREFETCH: 2182 ops = drm_gpuvm_prefetch_ops_create(&vm->gpuvm, addr, range); 2183 break; 2184 case DRM_XE_VM_BIND_OP_UNMAP_ALL: 2185 xe_assert(vm->xe, bo); 2186 2187 err = xe_bo_lock(bo, true); 2188 if (err) 2189 return ERR_PTR(err); 2190 2191 vm_bo = drm_gpuvm_bo_obtain(&vm->gpuvm, obj); 2192 if (IS_ERR(vm_bo)) { 2193 xe_bo_unlock(bo); 2194 return ERR_CAST(vm_bo); 2195 } 2196 2197 ops = drm_gpuvm_bo_unmap_ops_create(vm_bo); 2198 drm_gpuvm_bo_put(vm_bo); 2199 xe_bo_unlock(bo); 2200 break; 2201 default: 2202 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); 2203 ops = ERR_PTR(-EINVAL); 2204 } 2205 if (IS_ERR(ops)) 2206 return ops; 2207 2208 drm_gpuva_for_each_op(__op, ops) { 2209 struct xe_vma_op *op = gpuva_op_to_vma_op(__op); 2210 2211 if (__op->op == DRM_GPUVA_OP_MAP) { 2212 op->map.is_null = flags & DRM_XE_VM_BIND_FLAG_NULL; 2213 op->map.dumpable = flags & DRM_XE_VM_BIND_FLAG_DUMPABLE; 2214 op->map.pat_index = pat_index; 2215 } else if (__op->op == DRM_GPUVA_OP_PREFETCH) { 2216 op->prefetch.region = prefetch_region; 2217 } 2218 2219 print_op(vm->xe, __op); 2220 } 2221 2222 return ops; 2223 } 2224 2225 static struct xe_vma *new_vma(struct xe_vm *vm, struct drm_gpuva_op_map *op, 2226 u16 pat_index, unsigned int flags) 2227 { 2228 struct xe_bo *bo = op->gem.obj ? gem_to_xe_bo(op->gem.obj) : NULL; 2229 struct drm_exec exec; 2230 struct xe_vma *vma; 2231 int err; 2232 2233 lockdep_assert_held_write(&vm->lock); 2234 2235 if (bo) { 2236 drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0); 2237 drm_exec_until_all_locked(&exec) { 2238 err = 0; 2239 if (!bo->vm) { 2240 err = drm_exec_lock_obj(&exec, xe_vm_obj(vm)); 2241 drm_exec_retry_on_contention(&exec); 2242 } 2243 if (!err) { 2244 err = drm_exec_lock_obj(&exec, &bo->ttm.base); 2245 drm_exec_retry_on_contention(&exec); 2246 } 2247 if (err) { 2248 drm_exec_fini(&exec); 2249 return ERR_PTR(err); 2250 } 2251 } 2252 } 2253 vma = xe_vma_create(vm, bo, op->gem.offset, 2254 op->va.addr, op->va.addr + 2255 op->va.range - 1, pat_index, flags); 2256 if (bo) 2257 drm_exec_fini(&exec); 2258 2259 if (xe_vma_is_userptr(vma)) { 2260 err = xe_vma_userptr_pin_pages(to_userptr_vma(vma)); 2261 if (err) { 2262 prep_vma_destroy(vm, vma, false); 2263 xe_vma_destroy_unlocked(vma); 2264 return ERR_PTR(err); 2265 } 2266 } else if (!xe_vma_has_no_bo(vma) && !bo->vm) { 2267 err = add_preempt_fences(vm, bo); 2268 if (err) { 2269 prep_vma_destroy(vm, vma, false); 2270 xe_vma_destroy_unlocked(vma); 2271 return ERR_PTR(err); 2272 } 2273 } 2274 2275 return vma; 2276 } 2277 2278 static u64 xe_vma_max_pte_size(struct xe_vma *vma) 2279 { 2280 if (vma->gpuva.flags & XE_VMA_PTE_1G) 2281 return SZ_1G; 2282 else if (vma->gpuva.flags & (XE_VMA_PTE_2M | XE_VMA_PTE_COMPACT)) 2283 return SZ_2M; 2284 else if (vma->gpuva.flags & XE_VMA_PTE_64K) 2285 return SZ_64K; 2286 else if (vma->gpuva.flags & XE_VMA_PTE_4K) 2287 return SZ_4K; 2288 2289 return SZ_1G; /* Uninitialized, used max size */ 2290 } 2291 2292 static void xe_vma_set_pte_size(struct xe_vma *vma, u64 size) 2293 { 2294 switch (size) { 2295 case SZ_1G: 2296 vma->gpuva.flags |= XE_VMA_PTE_1G; 2297 break; 2298 case SZ_2M: 2299 vma->gpuva.flags |= XE_VMA_PTE_2M; 2300 break; 2301 case SZ_64K: 2302 vma->gpuva.flags |= XE_VMA_PTE_64K; 2303 break; 2304 case SZ_4K: 2305 vma->gpuva.flags |= XE_VMA_PTE_4K; 2306 break; 2307 } 2308 } 2309 2310 static int xe_vma_op_commit(struct xe_vm *vm, struct xe_vma_op *op) 2311 { 2312 int err = 0; 2313 2314 lockdep_assert_held_write(&vm->lock); 2315 2316 switch (op->base.op) { 2317 case DRM_GPUVA_OP_MAP: 2318 err |= xe_vm_insert_vma(vm, op->map.vma); 2319 if (!err) 2320 op->flags |= XE_VMA_OP_COMMITTED; 2321 break; 2322 case DRM_GPUVA_OP_REMAP: 2323 { 2324 u8 tile_present = 2325 gpuva_to_vma(op->base.remap.unmap->va)->tile_present; 2326 2327 prep_vma_destroy(vm, gpuva_to_vma(op->base.remap.unmap->va), 2328 true); 2329 op->flags |= XE_VMA_OP_COMMITTED; 2330 2331 if (op->remap.prev) { 2332 err |= xe_vm_insert_vma(vm, op->remap.prev); 2333 if (!err) 2334 op->flags |= XE_VMA_OP_PREV_COMMITTED; 2335 if (!err && op->remap.skip_prev) { 2336 op->remap.prev->tile_present = 2337 tile_present; 2338 op->remap.prev = NULL; 2339 } 2340 } 2341 if (op->remap.next) { 2342 err |= xe_vm_insert_vma(vm, op->remap.next); 2343 if (!err) 2344 op->flags |= XE_VMA_OP_NEXT_COMMITTED; 2345 if (!err && op->remap.skip_next) { 2346 op->remap.next->tile_present = 2347 tile_present; 2348 op->remap.next = NULL; 2349 } 2350 } 2351 2352 /* Adjust for partial unbind after removin VMA from VM */ 2353 if (!err) { 2354 op->base.remap.unmap->va->va.addr = op->remap.start; 2355 op->base.remap.unmap->va->va.range = op->remap.range; 2356 } 2357 break; 2358 } 2359 case DRM_GPUVA_OP_UNMAP: 2360 prep_vma_destroy(vm, gpuva_to_vma(op->base.unmap.va), true); 2361 op->flags |= XE_VMA_OP_COMMITTED; 2362 break; 2363 case DRM_GPUVA_OP_PREFETCH: 2364 op->flags |= XE_VMA_OP_COMMITTED; 2365 break; 2366 default: 2367 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); 2368 } 2369 2370 return err; 2371 } 2372 2373 2374 static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q, 2375 struct drm_gpuva_ops *ops, 2376 struct xe_sync_entry *syncs, u32 num_syncs, 2377 struct list_head *ops_list, bool last) 2378 { 2379 struct xe_device *xe = vm->xe; 2380 struct xe_vma_op *last_op = NULL; 2381 struct drm_gpuva_op *__op; 2382 int err = 0; 2383 2384 lockdep_assert_held_write(&vm->lock); 2385 2386 drm_gpuva_for_each_op(__op, ops) { 2387 struct xe_vma_op *op = gpuva_op_to_vma_op(__op); 2388 struct xe_vma *vma; 2389 bool first = list_empty(ops_list); 2390 unsigned int flags = 0; 2391 2392 INIT_LIST_HEAD(&op->link); 2393 list_add_tail(&op->link, ops_list); 2394 2395 if (first) { 2396 op->flags |= XE_VMA_OP_FIRST; 2397 op->num_syncs = num_syncs; 2398 op->syncs = syncs; 2399 } 2400 2401 op->q = q; 2402 2403 switch (op->base.op) { 2404 case DRM_GPUVA_OP_MAP: 2405 { 2406 flags |= op->map.is_null ? 2407 VMA_CREATE_FLAG_IS_NULL : 0; 2408 flags |= op->map.dumpable ? 2409 VMA_CREATE_FLAG_DUMPABLE : 0; 2410 2411 vma = new_vma(vm, &op->base.map, op->map.pat_index, 2412 flags); 2413 if (IS_ERR(vma)) 2414 return PTR_ERR(vma); 2415 2416 op->map.vma = vma; 2417 break; 2418 } 2419 case DRM_GPUVA_OP_REMAP: 2420 { 2421 struct xe_vma *old = 2422 gpuva_to_vma(op->base.remap.unmap->va); 2423 2424 op->remap.start = xe_vma_start(old); 2425 op->remap.range = xe_vma_size(old); 2426 2427 if (op->base.remap.prev) { 2428 flags |= op->base.remap.unmap->va->flags & 2429 XE_VMA_READ_ONLY ? 2430 VMA_CREATE_FLAG_READ_ONLY : 0; 2431 flags |= op->base.remap.unmap->va->flags & 2432 DRM_GPUVA_SPARSE ? 2433 VMA_CREATE_FLAG_IS_NULL : 0; 2434 flags |= op->base.remap.unmap->va->flags & 2435 XE_VMA_DUMPABLE ? 2436 VMA_CREATE_FLAG_DUMPABLE : 0; 2437 2438 vma = new_vma(vm, op->base.remap.prev, 2439 old->pat_index, flags); 2440 if (IS_ERR(vma)) 2441 return PTR_ERR(vma); 2442 2443 op->remap.prev = vma; 2444 2445 /* 2446 * Userptr creates a new SG mapping so 2447 * we must also rebind. 2448 */ 2449 op->remap.skip_prev = !xe_vma_is_userptr(old) && 2450 IS_ALIGNED(xe_vma_end(vma), 2451 xe_vma_max_pte_size(old)); 2452 if (op->remap.skip_prev) { 2453 xe_vma_set_pte_size(vma, xe_vma_max_pte_size(old)); 2454 op->remap.range -= 2455 xe_vma_end(vma) - 2456 xe_vma_start(old); 2457 op->remap.start = xe_vma_end(vma); 2458 vm_dbg(&xe->drm, "REMAP:SKIP_PREV: addr=0x%016llx, range=0x%016llx", 2459 (ULL)op->remap.start, 2460 (ULL)op->remap.range); 2461 } 2462 } 2463 2464 if (op->base.remap.next) { 2465 flags |= op->base.remap.unmap->va->flags & 2466 XE_VMA_READ_ONLY ? 2467 VMA_CREATE_FLAG_READ_ONLY : 0; 2468 flags |= op->base.remap.unmap->va->flags & 2469 DRM_GPUVA_SPARSE ? 2470 VMA_CREATE_FLAG_IS_NULL : 0; 2471 flags |= op->base.remap.unmap->va->flags & 2472 XE_VMA_DUMPABLE ? 2473 VMA_CREATE_FLAG_DUMPABLE : 0; 2474 2475 vma = new_vma(vm, op->base.remap.next, 2476 old->pat_index, flags); 2477 if (IS_ERR(vma)) 2478 return PTR_ERR(vma); 2479 2480 op->remap.next = vma; 2481 2482 /* 2483 * Userptr creates a new SG mapping so 2484 * we must also rebind. 2485 */ 2486 op->remap.skip_next = !xe_vma_is_userptr(old) && 2487 IS_ALIGNED(xe_vma_start(vma), 2488 xe_vma_max_pte_size(old)); 2489 if (op->remap.skip_next) { 2490 xe_vma_set_pte_size(vma, xe_vma_max_pte_size(old)); 2491 op->remap.range -= 2492 xe_vma_end(old) - 2493 xe_vma_start(vma); 2494 vm_dbg(&xe->drm, "REMAP:SKIP_NEXT: addr=0x%016llx, range=0x%016llx", 2495 (ULL)op->remap.start, 2496 (ULL)op->remap.range); 2497 } 2498 } 2499 break; 2500 } 2501 case DRM_GPUVA_OP_UNMAP: 2502 case DRM_GPUVA_OP_PREFETCH: 2503 /* Nothing to do */ 2504 break; 2505 default: 2506 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); 2507 } 2508 2509 last_op = op; 2510 2511 err = xe_vma_op_commit(vm, op); 2512 if (err) 2513 return err; 2514 } 2515 2516 /* FIXME: Unhandled corner case */ 2517 XE_WARN_ON(!last_op && last && !list_empty(ops_list)); 2518 2519 if (!last_op) 2520 return 0; 2521 2522 last_op->ops = ops; 2523 if (last) { 2524 last_op->flags |= XE_VMA_OP_LAST; 2525 last_op->num_syncs = num_syncs; 2526 last_op->syncs = syncs; 2527 } 2528 2529 return 0; 2530 } 2531 2532 static int op_execute(struct drm_exec *exec, struct xe_vm *vm, 2533 struct xe_vma *vma, struct xe_vma_op *op) 2534 { 2535 int err; 2536 2537 lockdep_assert_held_write(&vm->lock); 2538 2539 err = xe_vm_lock_vma(exec, vma); 2540 if (err) 2541 return err; 2542 2543 xe_vm_assert_held(vm); 2544 xe_bo_assert_held(xe_vma_bo(vma)); 2545 2546 switch (op->base.op) { 2547 case DRM_GPUVA_OP_MAP: 2548 err = xe_vm_bind(vm, vma, op->q, xe_vma_bo(vma), 2549 op->syncs, op->num_syncs, 2550 !xe_vm_in_fault_mode(vm), 2551 op->flags & XE_VMA_OP_FIRST, 2552 op->flags & XE_VMA_OP_LAST); 2553 break; 2554 case DRM_GPUVA_OP_REMAP: 2555 { 2556 bool prev = !!op->remap.prev; 2557 bool next = !!op->remap.next; 2558 2559 if (!op->remap.unmap_done) { 2560 if (prev || next) 2561 vma->gpuva.flags |= XE_VMA_FIRST_REBIND; 2562 err = xe_vm_unbind(vm, vma, op->q, op->syncs, 2563 op->num_syncs, 2564 op->flags & XE_VMA_OP_FIRST, 2565 op->flags & XE_VMA_OP_LAST && 2566 !prev && !next); 2567 if (err) 2568 break; 2569 op->remap.unmap_done = true; 2570 } 2571 2572 if (prev) { 2573 op->remap.prev->gpuva.flags |= XE_VMA_LAST_REBIND; 2574 err = xe_vm_bind(vm, op->remap.prev, op->q, 2575 xe_vma_bo(op->remap.prev), op->syncs, 2576 op->num_syncs, true, false, 2577 op->flags & XE_VMA_OP_LAST && !next); 2578 op->remap.prev->gpuva.flags &= ~XE_VMA_LAST_REBIND; 2579 if (err) 2580 break; 2581 op->remap.prev = NULL; 2582 } 2583 2584 if (next) { 2585 op->remap.next->gpuva.flags |= XE_VMA_LAST_REBIND; 2586 err = xe_vm_bind(vm, op->remap.next, op->q, 2587 xe_vma_bo(op->remap.next), 2588 op->syncs, op->num_syncs, 2589 true, false, 2590 op->flags & XE_VMA_OP_LAST); 2591 op->remap.next->gpuva.flags &= ~XE_VMA_LAST_REBIND; 2592 if (err) 2593 break; 2594 op->remap.next = NULL; 2595 } 2596 2597 break; 2598 } 2599 case DRM_GPUVA_OP_UNMAP: 2600 err = xe_vm_unbind(vm, vma, op->q, op->syncs, 2601 op->num_syncs, op->flags & XE_VMA_OP_FIRST, 2602 op->flags & XE_VMA_OP_LAST); 2603 break; 2604 case DRM_GPUVA_OP_PREFETCH: 2605 err = xe_vm_prefetch(vm, vma, op->q, op->prefetch.region, 2606 op->syncs, op->num_syncs, 2607 op->flags & XE_VMA_OP_FIRST, 2608 op->flags & XE_VMA_OP_LAST); 2609 break; 2610 default: 2611 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); 2612 } 2613 2614 if (err) 2615 trace_xe_vma_fail(vma); 2616 2617 return err; 2618 } 2619 2620 static int __xe_vma_op_execute(struct xe_vm *vm, struct xe_vma *vma, 2621 struct xe_vma_op *op) 2622 { 2623 struct drm_exec exec; 2624 int err; 2625 2626 retry_userptr: 2627 drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0); 2628 drm_exec_until_all_locked(&exec) { 2629 err = op_execute(&exec, vm, vma, op); 2630 drm_exec_retry_on_contention(&exec); 2631 if (err) 2632 break; 2633 } 2634 drm_exec_fini(&exec); 2635 2636 if (err == -EAGAIN) { 2637 lockdep_assert_held_write(&vm->lock); 2638 2639 if (op->base.op == DRM_GPUVA_OP_REMAP) { 2640 if (!op->remap.unmap_done) 2641 vma = gpuva_to_vma(op->base.remap.unmap->va); 2642 else if (op->remap.prev) 2643 vma = op->remap.prev; 2644 else 2645 vma = op->remap.next; 2646 } 2647 2648 if (xe_vma_is_userptr(vma)) { 2649 err = xe_vma_userptr_pin_pages(to_userptr_vma(vma)); 2650 if (!err) 2651 goto retry_userptr; 2652 2653 trace_xe_vma_fail(vma); 2654 } 2655 } 2656 2657 return err; 2658 } 2659 2660 static int xe_vma_op_execute(struct xe_vm *vm, struct xe_vma_op *op) 2661 { 2662 int ret = 0; 2663 2664 lockdep_assert_held_write(&vm->lock); 2665 2666 switch (op->base.op) { 2667 case DRM_GPUVA_OP_MAP: 2668 ret = __xe_vma_op_execute(vm, op->map.vma, op); 2669 break; 2670 case DRM_GPUVA_OP_REMAP: 2671 { 2672 struct xe_vma *vma; 2673 2674 if (!op->remap.unmap_done) 2675 vma = gpuva_to_vma(op->base.remap.unmap->va); 2676 else if (op->remap.prev) 2677 vma = op->remap.prev; 2678 else 2679 vma = op->remap.next; 2680 2681 ret = __xe_vma_op_execute(vm, vma, op); 2682 break; 2683 } 2684 case DRM_GPUVA_OP_UNMAP: 2685 ret = __xe_vma_op_execute(vm, gpuva_to_vma(op->base.unmap.va), 2686 op); 2687 break; 2688 case DRM_GPUVA_OP_PREFETCH: 2689 ret = __xe_vma_op_execute(vm, 2690 gpuva_to_vma(op->base.prefetch.va), 2691 op); 2692 break; 2693 default: 2694 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); 2695 } 2696 2697 return ret; 2698 } 2699 2700 static void xe_vma_op_cleanup(struct xe_vm *vm, struct xe_vma_op *op) 2701 { 2702 bool last = op->flags & XE_VMA_OP_LAST; 2703 2704 if (last) { 2705 while (op->num_syncs--) 2706 xe_sync_entry_cleanup(&op->syncs[op->num_syncs]); 2707 kfree(op->syncs); 2708 if (op->q) 2709 xe_exec_queue_put(op->q); 2710 } 2711 if (!list_empty(&op->link)) 2712 list_del(&op->link); 2713 if (op->ops) 2714 drm_gpuva_ops_free(&vm->gpuvm, op->ops); 2715 if (last) 2716 xe_vm_put(vm); 2717 } 2718 2719 static void xe_vma_op_unwind(struct xe_vm *vm, struct xe_vma_op *op, 2720 bool post_commit, bool prev_post_commit, 2721 bool next_post_commit) 2722 { 2723 lockdep_assert_held_write(&vm->lock); 2724 2725 switch (op->base.op) { 2726 case DRM_GPUVA_OP_MAP: 2727 if (op->map.vma) { 2728 prep_vma_destroy(vm, op->map.vma, post_commit); 2729 xe_vma_destroy_unlocked(op->map.vma); 2730 } 2731 break; 2732 case DRM_GPUVA_OP_UNMAP: 2733 { 2734 struct xe_vma *vma = gpuva_to_vma(op->base.unmap.va); 2735 2736 if (vma) { 2737 down_read(&vm->userptr.notifier_lock); 2738 vma->gpuva.flags &= ~XE_VMA_DESTROYED; 2739 up_read(&vm->userptr.notifier_lock); 2740 if (post_commit) 2741 xe_vm_insert_vma(vm, vma); 2742 } 2743 break; 2744 } 2745 case DRM_GPUVA_OP_REMAP: 2746 { 2747 struct xe_vma *vma = gpuva_to_vma(op->base.remap.unmap->va); 2748 2749 if (op->remap.prev) { 2750 prep_vma_destroy(vm, op->remap.prev, prev_post_commit); 2751 xe_vma_destroy_unlocked(op->remap.prev); 2752 } 2753 if (op->remap.next) { 2754 prep_vma_destroy(vm, op->remap.next, next_post_commit); 2755 xe_vma_destroy_unlocked(op->remap.next); 2756 } 2757 if (vma) { 2758 down_read(&vm->userptr.notifier_lock); 2759 vma->gpuva.flags &= ~XE_VMA_DESTROYED; 2760 up_read(&vm->userptr.notifier_lock); 2761 if (post_commit) 2762 xe_vm_insert_vma(vm, vma); 2763 } 2764 break; 2765 } 2766 case DRM_GPUVA_OP_PREFETCH: 2767 /* Nothing to do */ 2768 break; 2769 default: 2770 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); 2771 } 2772 } 2773 2774 static void vm_bind_ioctl_ops_unwind(struct xe_vm *vm, 2775 struct drm_gpuva_ops **ops, 2776 int num_ops_list) 2777 { 2778 int i; 2779 2780 for (i = num_ops_list - 1; i >= 0; --i) { 2781 struct drm_gpuva_ops *__ops = ops[i]; 2782 struct drm_gpuva_op *__op; 2783 2784 if (!__ops) 2785 continue; 2786 2787 drm_gpuva_for_each_op_reverse(__op, __ops) { 2788 struct xe_vma_op *op = gpuva_op_to_vma_op(__op); 2789 2790 xe_vma_op_unwind(vm, op, 2791 op->flags & XE_VMA_OP_COMMITTED, 2792 op->flags & XE_VMA_OP_PREV_COMMITTED, 2793 op->flags & XE_VMA_OP_NEXT_COMMITTED); 2794 } 2795 2796 drm_gpuva_ops_free(&vm->gpuvm, __ops); 2797 } 2798 } 2799 2800 static int vm_bind_ioctl_ops_execute(struct xe_vm *vm, 2801 struct list_head *ops_list) 2802 { 2803 struct xe_vma_op *op, *next; 2804 int err; 2805 2806 lockdep_assert_held_write(&vm->lock); 2807 2808 list_for_each_entry_safe(op, next, ops_list, link) { 2809 err = xe_vma_op_execute(vm, op); 2810 if (err) { 2811 drm_warn(&vm->xe->drm, "VM op(%d) failed with %d", 2812 op->base.op, err); 2813 /* 2814 * FIXME: Killing VM rather than proper error handling 2815 */ 2816 xe_vm_kill(vm); 2817 return -ENOSPC; 2818 } 2819 xe_vma_op_cleanup(vm, op); 2820 } 2821 2822 return 0; 2823 } 2824 2825 #define SUPPORTED_FLAGS (DRM_XE_VM_BIND_FLAG_NULL | \ 2826 DRM_XE_VM_BIND_FLAG_DUMPABLE) 2827 #define XE_64K_PAGE_MASK 0xffffull 2828 #define ALL_DRM_XE_SYNCS_FLAGS (DRM_XE_SYNCS_FLAG_WAIT_FOR_OP) 2829 2830 static int vm_bind_ioctl_check_args(struct xe_device *xe, 2831 struct drm_xe_vm_bind *args, 2832 struct drm_xe_vm_bind_op **bind_ops) 2833 { 2834 int err; 2835 int i; 2836 2837 if (XE_IOCTL_DBG(xe, args->pad || args->pad2) || 2838 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 2839 return -EINVAL; 2840 2841 if (XE_IOCTL_DBG(xe, args->extensions)) 2842 return -EINVAL; 2843 2844 if (args->num_binds > 1) { 2845 u64 __user *bind_user = 2846 u64_to_user_ptr(args->vector_of_binds); 2847 2848 *bind_ops = kvmalloc_array(args->num_binds, 2849 sizeof(struct drm_xe_vm_bind_op), 2850 GFP_KERNEL | __GFP_ACCOUNT); 2851 if (!*bind_ops) 2852 return -ENOMEM; 2853 2854 err = __copy_from_user(*bind_ops, bind_user, 2855 sizeof(struct drm_xe_vm_bind_op) * 2856 args->num_binds); 2857 if (XE_IOCTL_DBG(xe, err)) { 2858 err = -EFAULT; 2859 goto free_bind_ops; 2860 } 2861 } else { 2862 *bind_ops = &args->bind; 2863 } 2864 2865 for (i = 0; i < args->num_binds; ++i) { 2866 u64 range = (*bind_ops)[i].range; 2867 u64 addr = (*bind_ops)[i].addr; 2868 u32 op = (*bind_ops)[i].op; 2869 u32 flags = (*bind_ops)[i].flags; 2870 u32 obj = (*bind_ops)[i].obj; 2871 u64 obj_offset = (*bind_ops)[i].obj_offset; 2872 u32 prefetch_region = (*bind_ops)[i].prefetch_mem_region_instance; 2873 bool is_null = flags & DRM_XE_VM_BIND_FLAG_NULL; 2874 u16 pat_index = (*bind_ops)[i].pat_index; 2875 u16 coh_mode; 2876 2877 if (XE_IOCTL_DBG(xe, pat_index >= xe->pat.n_entries)) { 2878 err = -EINVAL; 2879 goto free_bind_ops; 2880 } 2881 2882 pat_index = array_index_nospec(pat_index, xe->pat.n_entries); 2883 (*bind_ops)[i].pat_index = pat_index; 2884 coh_mode = xe_pat_index_get_coh_mode(xe, pat_index); 2885 if (XE_IOCTL_DBG(xe, !coh_mode)) { /* hw reserved */ 2886 err = -EINVAL; 2887 goto free_bind_ops; 2888 } 2889 2890 if (XE_WARN_ON(coh_mode > XE_COH_AT_LEAST_1WAY)) { 2891 err = -EINVAL; 2892 goto free_bind_ops; 2893 } 2894 2895 if (XE_IOCTL_DBG(xe, op > DRM_XE_VM_BIND_OP_PREFETCH) || 2896 XE_IOCTL_DBG(xe, flags & ~SUPPORTED_FLAGS) || 2897 XE_IOCTL_DBG(xe, obj && is_null) || 2898 XE_IOCTL_DBG(xe, obj_offset && is_null) || 2899 XE_IOCTL_DBG(xe, op != DRM_XE_VM_BIND_OP_MAP && 2900 is_null) || 2901 XE_IOCTL_DBG(xe, !obj && 2902 op == DRM_XE_VM_BIND_OP_MAP && 2903 !is_null) || 2904 XE_IOCTL_DBG(xe, !obj && 2905 op == DRM_XE_VM_BIND_OP_UNMAP_ALL) || 2906 XE_IOCTL_DBG(xe, addr && 2907 op == DRM_XE_VM_BIND_OP_UNMAP_ALL) || 2908 XE_IOCTL_DBG(xe, range && 2909 op == DRM_XE_VM_BIND_OP_UNMAP_ALL) || 2910 XE_IOCTL_DBG(xe, obj && 2911 op == DRM_XE_VM_BIND_OP_MAP_USERPTR) || 2912 XE_IOCTL_DBG(xe, coh_mode == XE_COH_NONE && 2913 op == DRM_XE_VM_BIND_OP_MAP_USERPTR) || 2914 XE_IOCTL_DBG(xe, obj && 2915 op == DRM_XE_VM_BIND_OP_PREFETCH) || 2916 XE_IOCTL_DBG(xe, prefetch_region && 2917 op != DRM_XE_VM_BIND_OP_PREFETCH) || 2918 XE_IOCTL_DBG(xe, !(BIT(prefetch_region) & 2919 xe->info.mem_region_mask)) || 2920 XE_IOCTL_DBG(xe, obj && 2921 op == DRM_XE_VM_BIND_OP_UNMAP)) { 2922 err = -EINVAL; 2923 goto free_bind_ops; 2924 } 2925 2926 if (XE_IOCTL_DBG(xe, obj_offset & ~PAGE_MASK) || 2927 XE_IOCTL_DBG(xe, addr & ~PAGE_MASK) || 2928 XE_IOCTL_DBG(xe, range & ~PAGE_MASK) || 2929 XE_IOCTL_DBG(xe, !range && 2930 op != DRM_XE_VM_BIND_OP_UNMAP_ALL)) { 2931 err = -EINVAL; 2932 goto free_bind_ops; 2933 } 2934 } 2935 2936 return 0; 2937 2938 free_bind_ops: 2939 if (args->num_binds > 1) 2940 kvfree(*bind_ops); 2941 return err; 2942 } 2943 2944 static int vm_bind_ioctl_signal_fences(struct xe_vm *vm, 2945 struct xe_exec_queue *q, 2946 struct xe_sync_entry *syncs, 2947 int num_syncs) 2948 { 2949 struct dma_fence *fence; 2950 int i, err = 0; 2951 2952 fence = xe_sync_in_fence_get(syncs, num_syncs, 2953 to_wait_exec_queue(vm, q), vm); 2954 if (IS_ERR(fence)) 2955 return PTR_ERR(fence); 2956 2957 for (i = 0; i < num_syncs; i++) 2958 xe_sync_entry_signal(&syncs[i], NULL, fence); 2959 2960 xe_exec_queue_last_fence_set(to_wait_exec_queue(vm, q), vm, 2961 fence); 2962 dma_fence_put(fence); 2963 2964 return err; 2965 } 2966 2967 int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 2968 { 2969 struct xe_device *xe = to_xe_device(dev); 2970 struct xe_file *xef = to_xe_file(file); 2971 struct drm_xe_vm_bind *args = data; 2972 struct drm_xe_sync __user *syncs_user; 2973 struct xe_bo **bos = NULL; 2974 struct drm_gpuva_ops **ops = NULL; 2975 struct xe_vm *vm; 2976 struct xe_exec_queue *q = NULL; 2977 u32 num_syncs, num_ufence = 0; 2978 struct xe_sync_entry *syncs = NULL; 2979 struct drm_xe_vm_bind_op *bind_ops; 2980 LIST_HEAD(ops_list); 2981 int err; 2982 int i; 2983 2984 err = vm_bind_ioctl_check_args(xe, args, &bind_ops); 2985 if (err) 2986 return err; 2987 2988 if (args->exec_queue_id) { 2989 q = xe_exec_queue_lookup(xef, args->exec_queue_id); 2990 if (XE_IOCTL_DBG(xe, !q)) { 2991 err = -ENOENT; 2992 goto free_objs; 2993 } 2994 2995 if (XE_IOCTL_DBG(xe, !(q->flags & EXEC_QUEUE_FLAG_VM))) { 2996 err = -EINVAL; 2997 goto put_exec_queue; 2998 } 2999 } 3000 3001 vm = xe_vm_lookup(xef, args->vm_id); 3002 if (XE_IOCTL_DBG(xe, !vm)) { 3003 err = -EINVAL; 3004 goto put_exec_queue; 3005 } 3006 3007 err = down_write_killable(&vm->lock); 3008 if (err) 3009 goto put_vm; 3010 3011 if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) { 3012 err = -ENOENT; 3013 goto release_vm_lock; 3014 } 3015 3016 for (i = 0; i < args->num_binds; ++i) { 3017 u64 range = bind_ops[i].range; 3018 u64 addr = bind_ops[i].addr; 3019 3020 if (XE_IOCTL_DBG(xe, range > vm->size) || 3021 XE_IOCTL_DBG(xe, addr > vm->size - range)) { 3022 err = -EINVAL; 3023 goto release_vm_lock; 3024 } 3025 } 3026 3027 if (args->num_binds) { 3028 bos = kvcalloc(args->num_binds, sizeof(*bos), 3029 GFP_KERNEL | __GFP_ACCOUNT); 3030 if (!bos) { 3031 err = -ENOMEM; 3032 goto release_vm_lock; 3033 } 3034 3035 ops = kvcalloc(args->num_binds, sizeof(*ops), 3036 GFP_KERNEL | __GFP_ACCOUNT); 3037 if (!ops) { 3038 err = -ENOMEM; 3039 goto release_vm_lock; 3040 } 3041 } 3042 3043 for (i = 0; i < args->num_binds; ++i) { 3044 struct drm_gem_object *gem_obj; 3045 u64 range = bind_ops[i].range; 3046 u64 addr = bind_ops[i].addr; 3047 u32 obj = bind_ops[i].obj; 3048 u64 obj_offset = bind_ops[i].obj_offset; 3049 u16 pat_index = bind_ops[i].pat_index; 3050 u16 coh_mode; 3051 3052 if (!obj) 3053 continue; 3054 3055 gem_obj = drm_gem_object_lookup(file, obj); 3056 if (XE_IOCTL_DBG(xe, !gem_obj)) { 3057 err = -ENOENT; 3058 goto put_obj; 3059 } 3060 bos[i] = gem_to_xe_bo(gem_obj); 3061 3062 if (XE_IOCTL_DBG(xe, range > bos[i]->size) || 3063 XE_IOCTL_DBG(xe, obj_offset > 3064 bos[i]->size - range)) { 3065 err = -EINVAL; 3066 goto put_obj; 3067 } 3068 3069 if (bos[i]->flags & XE_BO_INTERNAL_64K) { 3070 if (XE_IOCTL_DBG(xe, obj_offset & 3071 XE_64K_PAGE_MASK) || 3072 XE_IOCTL_DBG(xe, addr & XE_64K_PAGE_MASK) || 3073 XE_IOCTL_DBG(xe, range & XE_64K_PAGE_MASK)) { 3074 err = -EINVAL; 3075 goto put_obj; 3076 } 3077 } 3078 3079 coh_mode = xe_pat_index_get_coh_mode(xe, pat_index); 3080 if (bos[i]->cpu_caching) { 3081 if (XE_IOCTL_DBG(xe, coh_mode == XE_COH_NONE && 3082 bos[i]->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB)) { 3083 err = -EINVAL; 3084 goto put_obj; 3085 } 3086 } else if (XE_IOCTL_DBG(xe, coh_mode == XE_COH_NONE)) { 3087 /* 3088 * Imported dma-buf from a different device should 3089 * require 1way or 2way coherency since we don't know 3090 * how it was mapped on the CPU. Just assume is it 3091 * potentially cached on CPU side. 3092 */ 3093 err = -EINVAL; 3094 goto put_obj; 3095 } 3096 } 3097 3098 if (args->num_syncs) { 3099 syncs = kcalloc(args->num_syncs, sizeof(*syncs), GFP_KERNEL); 3100 if (!syncs) { 3101 err = -ENOMEM; 3102 goto put_obj; 3103 } 3104 } 3105 3106 syncs_user = u64_to_user_ptr(args->syncs); 3107 for (num_syncs = 0; num_syncs < args->num_syncs; num_syncs++) { 3108 err = xe_sync_entry_parse(xe, xef, &syncs[num_syncs], 3109 &syncs_user[num_syncs], 3110 (xe_vm_in_lr_mode(vm) ? 3111 SYNC_PARSE_FLAG_LR_MODE : 0) | 3112 (!args->num_binds ? 3113 SYNC_PARSE_FLAG_DISALLOW_USER_FENCE : 0)); 3114 if (err) 3115 goto free_syncs; 3116 3117 if (xe_sync_is_ufence(&syncs[num_syncs])) 3118 num_ufence++; 3119 } 3120 3121 if (XE_IOCTL_DBG(xe, num_ufence > 1)) { 3122 err = -EINVAL; 3123 goto free_syncs; 3124 } 3125 3126 if (!args->num_binds) { 3127 err = -ENODATA; 3128 goto free_syncs; 3129 } 3130 3131 for (i = 0; i < args->num_binds; ++i) { 3132 u64 range = bind_ops[i].range; 3133 u64 addr = bind_ops[i].addr; 3134 u32 op = bind_ops[i].op; 3135 u32 flags = bind_ops[i].flags; 3136 u64 obj_offset = bind_ops[i].obj_offset; 3137 u32 prefetch_region = bind_ops[i].prefetch_mem_region_instance; 3138 u16 pat_index = bind_ops[i].pat_index; 3139 3140 ops[i] = vm_bind_ioctl_ops_create(vm, bos[i], obj_offset, 3141 addr, range, op, flags, 3142 prefetch_region, pat_index); 3143 if (IS_ERR(ops[i])) { 3144 err = PTR_ERR(ops[i]); 3145 ops[i] = NULL; 3146 goto unwind_ops; 3147 } 3148 3149 err = vm_bind_ioctl_ops_parse(vm, q, ops[i], syncs, num_syncs, 3150 &ops_list, 3151 i == args->num_binds - 1); 3152 if (err) 3153 goto unwind_ops; 3154 } 3155 3156 /* Nothing to do */ 3157 if (list_empty(&ops_list)) { 3158 err = -ENODATA; 3159 goto unwind_ops; 3160 } 3161 3162 xe_vm_get(vm); 3163 if (q) 3164 xe_exec_queue_get(q); 3165 3166 err = vm_bind_ioctl_ops_execute(vm, &ops_list); 3167 3168 up_write(&vm->lock); 3169 3170 if (q) 3171 xe_exec_queue_put(q); 3172 xe_vm_put(vm); 3173 3174 for (i = 0; bos && i < args->num_binds; ++i) 3175 xe_bo_put(bos[i]); 3176 3177 kvfree(bos); 3178 kvfree(ops); 3179 if (args->num_binds > 1) 3180 kvfree(bind_ops); 3181 3182 return err; 3183 3184 unwind_ops: 3185 vm_bind_ioctl_ops_unwind(vm, ops, args->num_binds); 3186 free_syncs: 3187 if (err == -ENODATA) 3188 err = vm_bind_ioctl_signal_fences(vm, q, syncs, num_syncs); 3189 while (num_syncs--) 3190 xe_sync_entry_cleanup(&syncs[num_syncs]); 3191 3192 kfree(syncs); 3193 put_obj: 3194 for (i = 0; i < args->num_binds; ++i) 3195 xe_bo_put(bos[i]); 3196 release_vm_lock: 3197 up_write(&vm->lock); 3198 put_vm: 3199 xe_vm_put(vm); 3200 put_exec_queue: 3201 if (q) 3202 xe_exec_queue_put(q); 3203 free_objs: 3204 kvfree(bos); 3205 kvfree(ops); 3206 if (args->num_binds > 1) 3207 kvfree(bind_ops); 3208 return err; 3209 } 3210 3211 /** 3212 * xe_vm_lock() - Lock the vm's dma_resv object 3213 * @vm: The struct xe_vm whose lock is to be locked 3214 * @intr: Whether to perform any wait interruptible 3215 * 3216 * Return: 0 on success, -EINTR if @intr is true and the wait for a 3217 * contended lock was interrupted. If @intr is false, the function 3218 * always returns 0. 3219 */ 3220 int xe_vm_lock(struct xe_vm *vm, bool intr) 3221 { 3222 if (intr) 3223 return dma_resv_lock_interruptible(xe_vm_resv(vm), NULL); 3224 3225 return dma_resv_lock(xe_vm_resv(vm), NULL); 3226 } 3227 3228 /** 3229 * xe_vm_unlock() - Unlock the vm's dma_resv object 3230 * @vm: The struct xe_vm whose lock is to be released. 3231 * 3232 * Unlock a buffer object lock that was locked by xe_vm_lock(). 3233 */ 3234 void xe_vm_unlock(struct xe_vm *vm) 3235 { 3236 dma_resv_unlock(xe_vm_resv(vm)); 3237 } 3238 3239 /** 3240 * xe_vm_invalidate_vma - invalidate GPU mappings for VMA without a lock 3241 * @vma: VMA to invalidate 3242 * 3243 * Walks a list of page tables leaves which it memset the entries owned by this 3244 * VMA to zero, invalidates the TLBs, and block until TLBs invalidation is 3245 * complete. 3246 * 3247 * Returns 0 for success, negative error code otherwise. 3248 */ 3249 int xe_vm_invalidate_vma(struct xe_vma *vma) 3250 { 3251 struct xe_device *xe = xe_vma_vm(vma)->xe; 3252 struct xe_tile *tile; 3253 u32 tile_needs_invalidate = 0; 3254 int seqno[XE_MAX_TILES_PER_DEVICE]; 3255 u8 id; 3256 int ret; 3257 3258 xe_assert(xe, !xe_vma_is_null(vma)); 3259 trace_xe_vma_invalidate(vma); 3260 3261 /* Check that we don't race with page-table updates */ 3262 if (IS_ENABLED(CONFIG_PROVE_LOCKING)) { 3263 if (xe_vma_is_userptr(vma)) { 3264 WARN_ON_ONCE(!mmu_interval_check_retry 3265 (&to_userptr_vma(vma)->userptr.notifier, 3266 to_userptr_vma(vma)->userptr.notifier_seq)); 3267 WARN_ON_ONCE(!dma_resv_test_signaled(xe_vm_resv(xe_vma_vm(vma)), 3268 DMA_RESV_USAGE_BOOKKEEP)); 3269 3270 } else { 3271 xe_bo_assert_held(xe_vma_bo(vma)); 3272 } 3273 } 3274 3275 for_each_tile(tile, xe, id) { 3276 if (xe_pt_zap_ptes(tile, vma)) { 3277 tile_needs_invalidate |= BIT(id); 3278 xe_device_wmb(xe); 3279 /* 3280 * FIXME: We potentially need to invalidate multiple 3281 * GTs within the tile 3282 */ 3283 seqno[id] = xe_gt_tlb_invalidation_vma(tile->primary_gt, NULL, vma); 3284 if (seqno[id] < 0) 3285 return seqno[id]; 3286 } 3287 } 3288 3289 for_each_tile(tile, xe, id) { 3290 if (tile_needs_invalidate & BIT(id)) { 3291 ret = xe_gt_tlb_invalidation_wait(tile->primary_gt, seqno[id]); 3292 if (ret < 0) 3293 return ret; 3294 } 3295 } 3296 3297 vma->tile_invalidated = vma->tile_mask; 3298 3299 return 0; 3300 } 3301 3302 int xe_analyze_vm(struct drm_printer *p, struct xe_vm *vm, int gt_id) 3303 { 3304 struct drm_gpuva *gpuva; 3305 bool is_vram; 3306 uint64_t addr; 3307 3308 if (!down_read_trylock(&vm->lock)) { 3309 drm_printf(p, " Failed to acquire VM lock to dump capture"); 3310 return 0; 3311 } 3312 if (vm->pt_root[gt_id]) { 3313 addr = xe_bo_addr(vm->pt_root[gt_id]->bo, 0, XE_PAGE_SIZE); 3314 is_vram = xe_bo_is_vram(vm->pt_root[gt_id]->bo); 3315 drm_printf(p, " VM root: A:0x%llx %s\n", addr, 3316 is_vram ? "VRAM" : "SYS"); 3317 } 3318 3319 drm_gpuvm_for_each_va(gpuva, &vm->gpuvm) { 3320 struct xe_vma *vma = gpuva_to_vma(gpuva); 3321 bool is_userptr = xe_vma_is_userptr(vma); 3322 bool is_null = xe_vma_is_null(vma); 3323 3324 if (is_null) { 3325 addr = 0; 3326 } else if (is_userptr) { 3327 struct sg_table *sg = to_userptr_vma(vma)->userptr.sg; 3328 struct xe_res_cursor cur; 3329 3330 if (sg) { 3331 xe_res_first_sg(sg, 0, XE_PAGE_SIZE, &cur); 3332 addr = xe_res_dma(&cur); 3333 } else { 3334 addr = 0; 3335 } 3336 } else { 3337 addr = __xe_bo_addr(xe_vma_bo(vma), 0, XE_PAGE_SIZE); 3338 is_vram = xe_bo_is_vram(xe_vma_bo(vma)); 3339 } 3340 drm_printf(p, " [%016llx-%016llx] S:0x%016llx A:%016llx %s\n", 3341 xe_vma_start(vma), xe_vma_end(vma) - 1, 3342 xe_vma_size(vma), 3343 addr, is_null ? "NULL" : is_userptr ? "USR" : 3344 is_vram ? "VRAM" : "SYS"); 3345 } 3346 up_read(&vm->lock); 3347 3348 return 0; 3349 } 3350 3351 struct xe_vm_snapshot { 3352 unsigned long num_snaps; 3353 struct { 3354 u64 ofs, bo_ofs; 3355 unsigned long len; 3356 struct xe_bo *bo; 3357 void *data; 3358 struct mm_struct *mm; 3359 } snap[]; 3360 }; 3361 3362 struct xe_vm_snapshot *xe_vm_snapshot_capture(struct xe_vm *vm) 3363 { 3364 unsigned long num_snaps = 0, i; 3365 struct xe_vm_snapshot *snap = NULL; 3366 struct drm_gpuva *gpuva; 3367 3368 if (!vm) 3369 return NULL; 3370 3371 mutex_lock(&vm->snap_mutex); 3372 drm_gpuvm_for_each_va(gpuva, &vm->gpuvm) { 3373 if (gpuva->flags & XE_VMA_DUMPABLE) 3374 num_snaps++; 3375 } 3376 3377 if (num_snaps) 3378 snap = kvzalloc(offsetof(struct xe_vm_snapshot, snap[num_snaps]), GFP_NOWAIT); 3379 if (!snap) 3380 goto out_unlock; 3381 3382 snap->num_snaps = num_snaps; 3383 i = 0; 3384 drm_gpuvm_for_each_va(gpuva, &vm->gpuvm) { 3385 struct xe_vma *vma = gpuva_to_vma(gpuva); 3386 struct xe_bo *bo = vma->gpuva.gem.obj ? 3387 gem_to_xe_bo(vma->gpuva.gem.obj) : NULL; 3388 3389 if (!(gpuva->flags & XE_VMA_DUMPABLE)) 3390 continue; 3391 3392 snap->snap[i].ofs = xe_vma_start(vma); 3393 snap->snap[i].len = xe_vma_size(vma); 3394 if (bo) { 3395 snap->snap[i].bo = xe_bo_get(bo); 3396 snap->snap[i].bo_ofs = xe_vma_bo_offset(vma); 3397 } else if (xe_vma_is_userptr(vma)) { 3398 struct mm_struct *mm = 3399 to_userptr_vma(vma)->userptr.notifier.mm; 3400 3401 if (mmget_not_zero(mm)) 3402 snap->snap[i].mm = mm; 3403 else 3404 snap->snap[i].data = ERR_PTR(-EFAULT); 3405 3406 snap->snap[i].bo_ofs = xe_vma_userptr(vma); 3407 } else { 3408 snap->snap[i].data = ERR_PTR(-ENOENT); 3409 } 3410 i++; 3411 } 3412 3413 out_unlock: 3414 mutex_unlock(&vm->snap_mutex); 3415 return snap; 3416 } 3417 3418 void xe_vm_snapshot_capture_delayed(struct xe_vm_snapshot *snap) 3419 { 3420 for (int i = 0; i < snap->num_snaps; i++) { 3421 struct xe_bo *bo = snap->snap[i].bo; 3422 struct iosys_map src; 3423 int err; 3424 3425 if (IS_ERR(snap->snap[i].data)) 3426 continue; 3427 3428 snap->snap[i].data = kvmalloc(snap->snap[i].len, GFP_USER); 3429 if (!snap->snap[i].data) { 3430 snap->snap[i].data = ERR_PTR(-ENOMEM); 3431 goto cleanup_bo; 3432 } 3433 3434 if (bo) { 3435 dma_resv_lock(bo->ttm.base.resv, NULL); 3436 err = ttm_bo_vmap(&bo->ttm, &src); 3437 if (!err) { 3438 xe_map_memcpy_from(xe_bo_device(bo), 3439 snap->snap[i].data, 3440 &src, snap->snap[i].bo_ofs, 3441 snap->snap[i].len); 3442 ttm_bo_vunmap(&bo->ttm, &src); 3443 } 3444 dma_resv_unlock(bo->ttm.base.resv); 3445 } else { 3446 void __user *userptr = (void __user *)(size_t)snap->snap[i].bo_ofs; 3447 3448 kthread_use_mm(snap->snap[i].mm); 3449 if (!copy_from_user(snap->snap[i].data, userptr, snap->snap[i].len)) 3450 err = 0; 3451 else 3452 err = -EFAULT; 3453 kthread_unuse_mm(snap->snap[i].mm); 3454 3455 mmput(snap->snap[i].mm); 3456 snap->snap[i].mm = NULL; 3457 } 3458 3459 if (err) { 3460 kvfree(snap->snap[i].data); 3461 snap->snap[i].data = ERR_PTR(err); 3462 } 3463 3464 cleanup_bo: 3465 xe_bo_put(bo); 3466 snap->snap[i].bo = NULL; 3467 } 3468 } 3469 3470 void xe_vm_snapshot_print(struct xe_vm_snapshot *snap, struct drm_printer *p) 3471 { 3472 unsigned long i, j; 3473 3474 for (i = 0; i < snap->num_snaps; i++) { 3475 if (IS_ERR(snap->snap[i].data)) 3476 goto uncaptured; 3477 3478 drm_printf(p, "[%llx].length: 0x%lx\n", snap->snap[i].ofs, snap->snap[i].len); 3479 drm_printf(p, "[%llx].data: ", 3480 snap->snap[i].ofs); 3481 3482 for (j = 0; j < snap->snap[i].len; j += sizeof(u32)) { 3483 u32 *val = snap->snap[i].data + j; 3484 char dumped[ASCII85_BUFSZ]; 3485 3486 drm_puts(p, ascii85_encode(*val, dumped)); 3487 } 3488 3489 drm_puts(p, "\n"); 3490 continue; 3491 3492 uncaptured: 3493 drm_printf(p, "Unable to capture range [%llx-%llx]: %li\n", 3494 snap->snap[i].ofs, snap->snap[i].ofs + snap->snap[i].len - 1, 3495 PTR_ERR(snap->snap[i].data)); 3496 } 3497 } 3498 3499 void xe_vm_snapshot_free(struct xe_vm_snapshot *snap) 3500 { 3501 unsigned long i; 3502 3503 if (!snap) 3504 return; 3505 3506 for (i = 0; i < snap->num_snaps; i++) { 3507 if (!IS_ERR(snap->snap[i].data)) 3508 kvfree(snap->snap[i].data); 3509 xe_bo_put(snap->snap[i].bo); 3510 if (snap->snap[i].mm) 3511 mmput(snap->snap[i].mm); 3512 } 3513 kvfree(snap); 3514 } 3515