1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_vm.h" 7 8 #include <linux/dma-fence-array.h> 9 #include <linux/nospec.h> 10 11 #include <drm/drm_exec.h> 12 #include <drm/drm_print.h> 13 #include <drm/ttm/ttm_execbuf_util.h> 14 #include <drm/ttm/ttm_tt.h> 15 #include <drm/xe_drm.h> 16 #include <linux/delay.h> 17 #include <linux/kthread.h> 18 #include <linux/mm.h> 19 #include <linux/swap.h> 20 21 #include "xe_assert.h" 22 #include "xe_bo.h" 23 #include "xe_device.h" 24 #include "xe_drm_client.h" 25 #include "xe_exec_queue.h" 26 #include "xe_gt.h" 27 #include "xe_gt_pagefault.h" 28 #include "xe_gt_tlb_invalidation.h" 29 #include "xe_migrate.h" 30 #include "xe_pat.h" 31 #include "xe_pm.h" 32 #include "xe_preempt_fence.h" 33 #include "xe_pt.h" 34 #include "xe_res_cursor.h" 35 #include "xe_sync.h" 36 #include "xe_trace.h" 37 #include "generated/xe_wa_oob.h" 38 #include "xe_wa.h" 39 40 static struct drm_gem_object *xe_vm_obj(struct xe_vm *vm) 41 { 42 return vm->gpuvm.r_obj; 43 } 44 45 /** 46 * xe_vma_userptr_check_repin() - Advisory check for repin needed 47 * @uvma: The userptr vma 48 * 49 * Check if the userptr vma has been invalidated since last successful 50 * repin. The check is advisory only and can the function can be called 51 * without the vm->userptr.notifier_lock held. There is no guarantee that the 52 * vma userptr will remain valid after a lockless check, so typically 53 * the call needs to be followed by a proper check under the notifier_lock. 54 * 55 * Return: 0 if userptr vma is valid, -EAGAIN otherwise; repin recommended. 56 */ 57 int xe_vma_userptr_check_repin(struct xe_userptr_vma *uvma) 58 { 59 return mmu_interval_check_retry(&uvma->userptr.notifier, 60 uvma->userptr.notifier_seq) ? 61 -EAGAIN : 0; 62 } 63 64 int xe_vma_userptr_pin_pages(struct xe_userptr_vma *uvma) 65 { 66 struct xe_userptr *userptr = &uvma->userptr; 67 struct xe_vma *vma = &uvma->vma; 68 struct xe_vm *vm = xe_vma_vm(vma); 69 struct xe_device *xe = vm->xe; 70 const unsigned long num_pages = xe_vma_size(vma) >> PAGE_SHIFT; 71 struct page **pages; 72 bool in_kthread = !current->mm; 73 unsigned long notifier_seq; 74 int pinned, ret, i; 75 bool read_only = xe_vma_read_only(vma); 76 77 lockdep_assert_held(&vm->lock); 78 xe_assert(xe, xe_vma_is_userptr(vma)); 79 retry: 80 if (vma->gpuva.flags & XE_VMA_DESTROYED) 81 return 0; 82 83 notifier_seq = mmu_interval_read_begin(&userptr->notifier); 84 if (notifier_seq == userptr->notifier_seq) 85 return 0; 86 87 pages = kvmalloc_array(num_pages, sizeof(*pages), GFP_KERNEL); 88 if (!pages) 89 return -ENOMEM; 90 91 if (userptr->sg) { 92 dma_unmap_sgtable(xe->drm.dev, 93 userptr->sg, 94 read_only ? DMA_TO_DEVICE : 95 DMA_BIDIRECTIONAL, 0); 96 sg_free_table(userptr->sg); 97 userptr->sg = NULL; 98 } 99 100 pinned = ret = 0; 101 if (in_kthread) { 102 if (!mmget_not_zero(userptr->notifier.mm)) { 103 ret = -EFAULT; 104 goto mm_closed; 105 } 106 kthread_use_mm(userptr->notifier.mm); 107 } 108 109 while (pinned < num_pages) { 110 ret = get_user_pages_fast(xe_vma_userptr(vma) + 111 pinned * PAGE_SIZE, 112 num_pages - pinned, 113 read_only ? 0 : FOLL_WRITE, 114 &pages[pinned]); 115 if (ret < 0) 116 break; 117 118 pinned += ret; 119 ret = 0; 120 } 121 122 if (in_kthread) { 123 kthread_unuse_mm(userptr->notifier.mm); 124 mmput(userptr->notifier.mm); 125 } 126 mm_closed: 127 if (ret) 128 goto out; 129 130 ret = sg_alloc_table_from_pages_segment(&userptr->sgt, pages, 131 pinned, 0, 132 (u64)pinned << PAGE_SHIFT, 133 xe_sg_segment_size(xe->drm.dev), 134 GFP_KERNEL); 135 if (ret) { 136 userptr->sg = NULL; 137 goto out; 138 } 139 userptr->sg = &userptr->sgt; 140 141 ret = dma_map_sgtable(xe->drm.dev, userptr->sg, 142 read_only ? DMA_TO_DEVICE : 143 DMA_BIDIRECTIONAL, 144 DMA_ATTR_SKIP_CPU_SYNC | 145 DMA_ATTR_NO_KERNEL_MAPPING); 146 if (ret) { 147 sg_free_table(userptr->sg); 148 userptr->sg = NULL; 149 goto out; 150 } 151 152 for (i = 0; i < pinned; ++i) { 153 if (!read_only) { 154 lock_page(pages[i]); 155 set_page_dirty(pages[i]); 156 unlock_page(pages[i]); 157 } 158 159 mark_page_accessed(pages[i]); 160 } 161 162 out: 163 release_pages(pages, pinned); 164 kvfree(pages); 165 166 if (!(ret < 0)) { 167 userptr->notifier_seq = notifier_seq; 168 if (xe_vma_userptr_check_repin(uvma) == -EAGAIN) 169 goto retry; 170 } 171 172 return ret < 0 ? ret : 0; 173 } 174 175 static bool preempt_fences_waiting(struct xe_vm *vm) 176 { 177 struct xe_exec_queue *q; 178 179 lockdep_assert_held(&vm->lock); 180 xe_vm_assert_held(vm); 181 182 list_for_each_entry(q, &vm->preempt.exec_queues, compute.link) { 183 if (!q->compute.pfence || 184 (q->compute.pfence && test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, 185 &q->compute.pfence->flags))) { 186 return true; 187 } 188 } 189 190 return false; 191 } 192 193 static void free_preempt_fences(struct list_head *list) 194 { 195 struct list_head *link, *next; 196 197 list_for_each_safe(link, next, list) 198 xe_preempt_fence_free(to_preempt_fence_from_link(link)); 199 } 200 201 static int alloc_preempt_fences(struct xe_vm *vm, struct list_head *list, 202 unsigned int *count) 203 { 204 lockdep_assert_held(&vm->lock); 205 xe_vm_assert_held(vm); 206 207 if (*count >= vm->preempt.num_exec_queues) 208 return 0; 209 210 for (; *count < vm->preempt.num_exec_queues; ++(*count)) { 211 struct xe_preempt_fence *pfence = xe_preempt_fence_alloc(); 212 213 if (IS_ERR(pfence)) 214 return PTR_ERR(pfence); 215 216 list_move_tail(xe_preempt_fence_link(pfence), list); 217 } 218 219 return 0; 220 } 221 222 static int wait_for_existing_preempt_fences(struct xe_vm *vm) 223 { 224 struct xe_exec_queue *q; 225 226 xe_vm_assert_held(vm); 227 228 list_for_each_entry(q, &vm->preempt.exec_queues, compute.link) { 229 if (q->compute.pfence) { 230 long timeout = dma_fence_wait(q->compute.pfence, false); 231 232 if (timeout < 0) 233 return -ETIME; 234 dma_fence_put(q->compute.pfence); 235 q->compute.pfence = NULL; 236 } 237 } 238 239 return 0; 240 } 241 242 static bool xe_vm_is_idle(struct xe_vm *vm) 243 { 244 struct xe_exec_queue *q; 245 246 xe_vm_assert_held(vm); 247 list_for_each_entry(q, &vm->preempt.exec_queues, compute.link) { 248 if (!xe_exec_queue_is_idle(q)) 249 return false; 250 } 251 252 return true; 253 } 254 255 static void arm_preempt_fences(struct xe_vm *vm, struct list_head *list) 256 { 257 struct list_head *link; 258 struct xe_exec_queue *q; 259 260 list_for_each_entry(q, &vm->preempt.exec_queues, compute.link) { 261 struct dma_fence *fence; 262 263 link = list->next; 264 xe_assert(vm->xe, link != list); 265 266 fence = xe_preempt_fence_arm(to_preempt_fence_from_link(link), 267 q, q->compute.context, 268 ++q->compute.seqno); 269 dma_fence_put(q->compute.pfence); 270 q->compute.pfence = fence; 271 } 272 } 273 274 static int add_preempt_fences(struct xe_vm *vm, struct xe_bo *bo) 275 { 276 struct xe_exec_queue *q; 277 int err; 278 279 if (!vm->preempt.num_exec_queues) 280 return 0; 281 282 err = xe_bo_lock(bo, true); 283 if (err) 284 return err; 285 286 err = dma_resv_reserve_fences(bo->ttm.base.resv, vm->preempt.num_exec_queues); 287 if (err) 288 goto out_unlock; 289 290 list_for_each_entry(q, &vm->preempt.exec_queues, compute.link) 291 if (q->compute.pfence) { 292 dma_resv_add_fence(bo->ttm.base.resv, 293 q->compute.pfence, 294 DMA_RESV_USAGE_BOOKKEEP); 295 } 296 297 out_unlock: 298 xe_bo_unlock(bo); 299 return err; 300 } 301 302 static void resume_and_reinstall_preempt_fences(struct xe_vm *vm, 303 struct drm_exec *exec) 304 { 305 struct xe_exec_queue *q; 306 307 lockdep_assert_held(&vm->lock); 308 xe_vm_assert_held(vm); 309 310 list_for_each_entry(q, &vm->preempt.exec_queues, compute.link) { 311 q->ops->resume(q); 312 313 drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, q->compute.pfence, 314 DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_BOOKKEEP); 315 } 316 } 317 318 int xe_vm_add_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q) 319 { 320 struct drm_gpuvm_exec vm_exec = { 321 .vm = &vm->gpuvm, 322 .flags = DRM_EXEC_INTERRUPTIBLE_WAIT, 323 .num_fences = 1, 324 }; 325 struct drm_exec *exec = &vm_exec.exec; 326 struct dma_fence *pfence; 327 int err; 328 bool wait; 329 330 xe_assert(vm->xe, xe_vm_in_preempt_fence_mode(vm)); 331 332 down_write(&vm->lock); 333 err = drm_gpuvm_exec_lock(&vm_exec); 334 if (err) 335 goto out_up_write; 336 337 pfence = xe_preempt_fence_create(q, q->compute.context, 338 ++q->compute.seqno); 339 if (!pfence) { 340 err = -ENOMEM; 341 goto out_fini; 342 } 343 344 list_add(&q->compute.link, &vm->preempt.exec_queues); 345 ++vm->preempt.num_exec_queues; 346 q->compute.pfence = pfence; 347 348 down_read(&vm->userptr.notifier_lock); 349 350 drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, pfence, 351 DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_BOOKKEEP); 352 353 /* 354 * Check to see if a preemption on VM is in flight or userptr 355 * invalidation, if so trigger this preempt fence to sync state with 356 * other preempt fences on the VM. 357 */ 358 wait = __xe_vm_userptr_needs_repin(vm) || preempt_fences_waiting(vm); 359 if (wait) 360 dma_fence_enable_sw_signaling(pfence); 361 362 up_read(&vm->userptr.notifier_lock); 363 364 out_fini: 365 drm_exec_fini(exec); 366 out_up_write: 367 up_write(&vm->lock); 368 369 return err; 370 } 371 372 /** 373 * xe_vm_remove_compute_exec_queue() - Remove compute exec queue from VM 374 * @vm: The VM. 375 * @q: The exec_queue 376 */ 377 void xe_vm_remove_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q) 378 { 379 if (!xe_vm_in_preempt_fence_mode(vm)) 380 return; 381 382 down_write(&vm->lock); 383 list_del(&q->compute.link); 384 --vm->preempt.num_exec_queues; 385 if (q->compute.pfence) { 386 dma_fence_enable_sw_signaling(q->compute.pfence); 387 dma_fence_put(q->compute.pfence); 388 q->compute.pfence = NULL; 389 } 390 up_write(&vm->lock); 391 } 392 393 /** 394 * __xe_vm_userptr_needs_repin() - Check whether the VM does have userptrs 395 * that need repinning. 396 * @vm: The VM. 397 * 398 * This function checks for whether the VM has userptrs that need repinning, 399 * and provides a release-type barrier on the userptr.notifier_lock after 400 * checking. 401 * 402 * Return: 0 if there are no userptrs needing repinning, -EAGAIN if there are. 403 */ 404 int __xe_vm_userptr_needs_repin(struct xe_vm *vm) 405 { 406 lockdep_assert_held_read(&vm->userptr.notifier_lock); 407 408 return (list_empty(&vm->userptr.repin_list) && 409 list_empty(&vm->userptr.invalidated)) ? 0 : -EAGAIN; 410 } 411 412 #define XE_VM_REBIND_RETRY_TIMEOUT_MS 1000 413 414 static void xe_vm_kill(struct xe_vm *vm) 415 { 416 struct xe_exec_queue *q; 417 418 lockdep_assert_held(&vm->lock); 419 420 xe_vm_lock(vm, false); 421 vm->flags |= XE_VM_FLAG_BANNED; 422 trace_xe_vm_kill(vm); 423 424 list_for_each_entry(q, &vm->preempt.exec_queues, compute.link) 425 q->ops->kill(q); 426 xe_vm_unlock(vm); 427 428 /* TODO: Inform user the VM is banned */ 429 } 430 431 /** 432 * xe_vm_validate_should_retry() - Whether to retry after a validate error. 433 * @exec: The drm_exec object used for locking before validation. 434 * @err: The error returned from ttm_bo_validate(). 435 * @end: A ktime_t cookie that should be set to 0 before first use and 436 * that should be reused on subsequent calls. 437 * 438 * With multiple active VMs, under memory pressure, it is possible that 439 * ttm_bo_validate() run into -EDEADLK and in such case returns -ENOMEM. 440 * Until ttm properly handles locking in such scenarios, best thing the 441 * driver can do is retry with a timeout. Check if that is necessary, and 442 * if so unlock the drm_exec's objects while keeping the ticket to prepare 443 * for a rerun. 444 * 445 * Return: true if a retry after drm_exec_init() is recommended; 446 * false otherwise. 447 */ 448 bool xe_vm_validate_should_retry(struct drm_exec *exec, int err, ktime_t *end) 449 { 450 ktime_t cur; 451 452 if (err != -ENOMEM) 453 return false; 454 455 cur = ktime_get(); 456 *end = *end ? : ktime_add_ms(cur, XE_VM_REBIND_RETRY_TIMEOUT_MS); 457 if (!ktime_before(cur, *end)) 458 return false; 459 460 msleep(20); 461 return true; 462 } 463 464 static int xe_gpuvm_validate(struct drm_gpuvm_bo *vm_bo, struct drm_exec *exec) 465 { 466 struct xe_vm *vm = gpuvm_to_vm(vm_bo->vm); 467 struct drm_gpuva *gpuva; 468 int ret; 469 470 lockdep_assert_held(&vm->lock); 471 drm_gpuvm_bo_for_each_va(gpuva, vm_bo) 472 list_move_tail(&gpuva_to_vma(gpuva)->combined_links.rebind, 473 &vm->rebind_list); 474 475 ret = xe_bo_validate(gem_to_xe_bo(vm_bo->obj), vm, false); 476 if (ret) 477 return ret; 478 479 vm_bo->evicted = false; 480 return 0; 481 } 482 483 static int xe_preempt_work_begin(struct drm_exec *exec, struct xe_vm *vm, 484 bool *done) 485 { 486 int err; 487 488 /* 489 * 1 fence for each preempt fence plus a fence for each tile from a 490 * possible rebind 491 */ 492 err = drm_gpuvm_prepare_vm(&vm->gpuvm, exec, vm->preempt.num_exec_queues + 493 vm->xe->info.tile_count); 494 if (err) 495 return err; 496 497 if (xe_vm_is_idle(vm)) { 498 vm->preempt.rebind_deactivated = true; 499 *done = true; 500 return 0; 501 } 502 503 if (!preempt_fences_waiting(vm)) { 504 *done = true; 505 return 0; 506 } 507 508 err = drm_gpuvm_prepare_objects(&vm->gpuvm, exec, vm->preempt.num_exec_queues); 509 if (err) 510 return err; 511 512 err = wait_for_existing_preempt_fences(vm); 513 if (err) 514 return err; 515 516 return drm_gpuvm_validate(&vm->gpuvm, exec); 517 } 518 519 static void preempt_rebind_work_func(struct work_struct *w) 520 { 521 struct xe_vm *vm = container_of(w, struct xe_vm, preempt.rebind_work); 522 struct drm_exec exec; 523 struct dma_fence *rebind_fence; 524 unsigned int fence_count = 0; 525 LIST_HEAD(preempt_fences); 526 ktime_t end = 0; 527 int err = 0; 528 long wait; 529 int __maybe_unused tries = 0; 530 531 xe_assert(vm->xe, xe_vm_in_preempt_fence_mode(vm)); 532 trace_xe_vm_rebind_worker_enter(vm); 533 534 down_write(&vm->lock); 535 536 if (xe_vm_is_closed_or_banned(vm)) { 537 up_write(&vm->lock); 538 trace_xe_vm_rebind_worker_exit(vm); 539 return; 540 } 541 542 retry: 543 if (xe_vm_userptr_check_repin(vm)) { 544 err = xe_vm_userptr_pin(vm); 545 if (err) 546 goto out_unlock_outer; 547 } 548 549 drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0); 550 551 drm_exec_until_all_locked(&exec) { 552 bool done = false; 553 554 err = xe_preempt_work_begin(&exec, vm, &done); 555 drm_exec_retry_on_contention(&exec); 556 if (err || done) { 557 drm_exec_fini(&exec); 558 if (err && xe_vm_validate_should_retry(&exec, err, &end)) 559 err = -EAGAIN; 560 561 goto out_unlock_outer; 562 } 563 } 564 565 err = alloc_preempt_fences(vm, &preempt_fences, &fence_count); 566 if (err) 567 goto out_unlock; 568 569 rebind_fence = xe_vm_rebind(vm, true); 570 if (IS_ERR(rebind_fence)) { 571 err = PTR_ERR(rebind_fence); 572 goto out_unlock; 573 } 574 575 if (rebind_fence) { 576 dma_fence_wait(rebind_fence, false); 577 dma_fence_put(rebind_fence); 578 } 579 580 /* Wait on munmap style VM unbinds */ 581 wait = dma_resv_wait_timeout(xe_vm_resv(vm), 582 DMA_RESV_USAGE_KERNEL, 583 false, MAX_SCHEDULE_TIMEOUT); 584 if (wait <= 0) { 585 err = -ETIME; 586 goto out_unlock; 587 } 588 589 #define retry_required(__tries, __vm) \ 590 (IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT) ? \ 591 (!(__tries)++ || __xe_vm_userptr_needs_repin(__vm)) : \ 592 __xe_vm_userptr_needs_repin(__vm)) 593 594 down_read(&vm->userptr.notifier_lock); 595 if (retry_required(tries, vm)) { 596 up_read(&vm->userptr.notifier_lock); 597 err = -EAGAIN; 598 goto out_unlock; 599 } 600 601 #undef retry_required 602 603 spin_lock(&vm->xe->ttm.lru_lock); 604 ttm_lru_bulk_move_tail(&vm->lru_bulk_move); 605 spin_unlock(&vm->xe->ttm.lru_lock); 606 607 /* Point of no return. */ 608 arm_preempt_fences(vm, &preempt_fences); 609 resume_and_reinstall_preempt_fences(vm, &exec); 610 up_read(&vm->userptr.notifier_lock); 611 612 out_unlock: 613 drm_exec_fini(&exec); 614 out_unlock_outer: 615 if (err == -EAGAIN) { 616 trace_xe_vm_rebind_worker_retry(vm); 617 goto retry; 618 } 619 620 if (err) { 621 drm_warn(&vm->xe->drm, "VM worker error: %d\n", err); 622 xe_vm_kill(vm); 623 } 624 up_write(&vm->lock); 625 626 free_preempt_fences(&preempt_fences); 627 628 trace_xe_vm_rebind_worker_exit(vm); 629 } 630 631 static bool vma_userptr_invalidate(struct mmu_interval_notifier *mni, 632 const struct mmu_notifier_range *range, 633 unsigned long cur_seq) 634 { 635 struct xe_userptr *userptr = container_of(mni, typeof(*userptr), notifier); 636 struct xe_userptr_vma *uvma = container_of(userptr, typeof(*uvma), userptr); 637 struct xe_vma *vma = &uvma->vma; 638 struct xe_vm *vm = xe_vma_vm(vma); 639 struct dma_resv_iter cursor; 640 struct dma_fence *fence; 641 long err; 642 643 xe_assert(vm->xe, xe_vma_is_userptr(vma)); 644 trace_xe_vma_userptr_invalidate(vma); 645 646 if (!mmu_notifier_range_blockable(range)) 647 return false; 648 649 down_write(&vm->userptr.notifier_lock); 650 mmu_interval_set_seq(mni, cur_seq); 651 652 /* No need to stop gpu access if the userptr is not yet bound. */ 653 if (!userptr->initial_bind) { 654 up_write(&vm->userptr.notifier_lock); 655 return true; 656 } 657 658 /* 659 * Tell exec and rebind worker they need to repin and rebind this 660 * userptr. 661 */ 662 if (!xe_vm_in_fault_mode(vm) && 663 !(vma->gpuva.flags & XE_VMA_DESTROYED) && vma->tile_present) { 664 spin_lock(&vm->userptr.invalidated_lock); 665 list_move_tail(&userptr->invalidate_link, 666 &vm->userptr.invalidated); 667 spin_unlock(&vm->userptr.invalidated_lock); 668 } 669 670 up_write(&vm->userptr.notifier_lock); 671 672 /* 673 * Preempt fences turn into schedule disables, pipeline these. 674 * Note that even in fault mode, we need to wait for binds and 675 * unbinds to complete, and those are attached as BOOKMARK fences 676 * to the vm. 677 */ 678 dma_resv_iter_begin(&cursor, xe_vm_resv(vm), 679 DMA_RESV_USAGE_BOOKKEEP); 680 dma_resv_for_each_fence_unlocked(&cursor, fence) 681 dma_fence_enable_sw_signaling(fence); 682 dma_resv_iter_end(&cursor); 683 684 err = dma_resv_wait_timeout(xe_vm_resv(vm), 685 DMA_RESV_USAGE_BOOKKEEP, 686 false, MAX_SCHEDULE_TIMEOUT); 687 XE_WARN_ON(err <= 0); 688 689 if (xe_vm_in_fault_mode(vm)) { 690 err = xe_vm_invalidate_vma(vma); 691 XE_WARN_ON(err); 692 } 693 694 trace_xe_vma_userptr_invalidate_complete(vma); 695 696 return true; 697 } 698 699 static const struct mmu_interval_notifier_ops vma_userptr_notifier_ops = { 700 .invalidate = vma_userptr_invalidate, 701 }; 702 703 int xe_vm_userptr_pin(struct xe_vm *vm) 704 { 705 struct xe_userptr_vma *uvma, *next; 706 int err = 0; 707 LIST_HEAD(tmp_evict); 708 709 lockdep_assert_held_write(&vm->lock); 710 711 /* Collect invalidated userptrs */ 712 spin_lock(&vm->userptr.invalidated_lock); 713 list_for_each_entry_safe(uvma, next, &vm->userptr.invalidated, 714 userptr.invalidate_link) { 715 list_del_init(&uvma->userptr.invalidate_link); 716 list_move_tail(&uvma->userptr.repin_link, 717 &vm->userptr.repin_list); 718 } 719 spin_unlock(&vm->userptr.invalidated_lock); 720 721 /* Pin and move to temporary list */ 722 list_for_each_entry_safe(uvma, next, &vm->userptr.repin_list, 723 userptr.repin_link) { 724 err = xe_vma_userptr_pin_pages(uvma); 725 if (err < 0) 726 return err; 727 728 list_del_init(&uvma->userptr.repin_link); 729 list_move_tail(&uvma->vma.combined_links.rebind, &vm->rebind_list); 730 } 731 732 return 0; 733 } 734 735 /** 736 * xe_vm_userptr_check_repin() - Check whether the VM might have userptrs 737 * that need repinning. 738 * @vm: The VM. 739 * 740 * This function does an advisory check for whether the VM has userptrs that 741 * need repinning. 742 * 743 * Return: 0 if there are no indications of userptrs needing repinning, 744 * -EAGAIN if there are. 745 */ 746 int xe_vm_userptr_check_repin(struct xe_vm *vm) 747 { 748 return (list_empty_careful(&vm->userptr.repin_list) && 749 list_empty_careful(&vm->userptr.invalidated)) ? 0 : -EAGAIN; 750 } 751 752 static struct dma_fence * 753 xe_vm_bind_vma(struct xe_vma *vma, struct xe_exec_queue *q, 754 struct xe_sync_entry *syncs, u32 num_syncs, 755 bool first_op, bool last_op); 756 757 struct dma_fence *xe_vm_rebind(struct xe_vm *vm, bool rebind_worker) 758 { 759 struct dma_fence *fence = NULL; 760 struct xe_vma *vma, *next; 761 762 lockdep_assert_held(&vm->lock); 763 if (xe_vm_in_lr_mode(vm) && !rebind_worker) 764 return NULL; 765 766 xe_vm_assert_held(vm); 767 list_for_each_entry_safe(vma, next, &vm->rebind_list, 768 combined_links.rebind) { 769 xe_assert(vm->xe, vma->tile_present); 770 771 list_del_init(&vma->combined_links.rebind); 772 dma_fence_put(fence); 773 if (rebind_worker) 774 trace_xe_vma_rebind_worker(vma); 775 else 776 trace_xe_vma_rebind_exec(vma); 777 fence = xe_vm_bind_vma(vma, NULL, NULL, 0, false, false); 778 if (IS_ERR(fence)) 779 return fence; 780 } 781 782 return fence; 783 } 784 785 static void xe_vma_free(struct xe_vma *vma) 786 { 787 if (xe_vma_is_userptr(vma)) 788 kfree(to_userptr_vma(vma)); 789 else 790 kfree(vma); 791 } 792 793 #define VMA_CREATE_FLAG_READ_ONLY BIT(0) 794 #define VMA_CREATE_FLAG_IS_NULL BIT(1) 795 796 static struct xe_vma *xe_vma_create(struct xe_vm *vm, 797 struct xe_bo *bo, 798 u64 bo_offset_or_userptr, 799 u64 start, u64 end, 800 u16 pat_index, unsigned int flags) 801 { 802 struct xe_vma *vma; 803 struct xe_tile *tile; 804 u8 id; 805 bool read_only = (flags & VMA_CREATE_FLAG_READ_ONLY); 806 bool is_null = (flags & VMA_CREATE_FLAG_IS_NULL); 807 808 xe_assert(vm->xe, start < end); 809 xe_assert(vm->xe, end < vm->size); 810 811 /* 812 * Allocate and ensure that the xe_vma_is_userptr() return 813 * matches what was allocated. 814 */ 815 if (!bo && !is_null) { 816 struct xe_userptr_vma *uvma = kzalloc(sizeof(*uvma), GFP_KERNEL); 817 818 if (!uvma) 819 return ERR_PTR(-ENOMEM); 820 821 vma = &uvma->vma; 822 } else { 823 vma = kzalloc(sizeof(*vma), GFP_KERNEL); 824 if (!vma) 825 return ERR_PTR(-ENOMEM); 826 827 if (is_null) 828 vma->gpuva.flags |= DRM_GPUVA_SPARSE; 829 if (bo) 830 vma->gpuva.gem.obj = &bo->ttm.base; 831 } 832 833 INIT_LIST_HEAD(&vma->combined_links.rebind); 834 835 INIT_LIST_HEAD(&vma->gpuva.gem.entry); 836 vma->gpuva.vm = &vm->gpuvm; 837 vma->gpuva.va.addr = start; 838 vma->gpuva.va.range = end - start + 1; 839 if (read_only) 840 vma->gpuva.flags |= XE_VMA_READ_ONLY; 841 842 for_each_tile(tile, vm->xe, id) 843 vma->tile_mask |= 0x1 << id; 844 845 if (GRAPHICS_VER(vm->xe) >= 20 || vm->xe->info.platform == XE_PVC) 846 vma->gpuva.flags |= XE_VMA_ATOMIC_PTE_BIT; 847 848 vma->pat_index = pat_index; 849 850 if (bo) { 851 struct drm_gpuvm_bo *vm_bo; 852 853 xe_bo_assert_held(bo); 854 855 vm_bo = drm_gpuvm_bo_obtain(vma->gpuva.vm, &bo->ttm.base); 856 if (IS_ERR(vm_bo)) { 857 xe_vma_free(vma); 858 return ERR_CAST(vm_bo); 859 } 860 861 drm_gpuvm_bo_extobj_add(vm_bo); 862 drm_gem_object_get(&bo->ttm.base); 863 vma->gpuva.gem.offset = bo_offset_or_userptr; 864 drm_gpuva_link(&vma->gpuva, vm_bo); 865 drm_gpuvm_bo_put(vm_bo); 866 } else /* userptr or null */ { 867 if (!is_null) { 868 struct xe_userptr *userptr = &to_userptr_vma(vma)->userptr; 869 u64 size = end - start + 1; 870 int err; 871 872 INIT_LIST_HEAD(&userptr->invalidate_link); 873 INIT_LIST_HEAD(&userptr->repin_link); 874 vma->gpuva.gem.offset = bo_offset_or_userptr; 875 876 err = mmu_interval_notifier_insert(&userptr->notifier, 877 current->mm, 878 xe_vma_userptr(vma), size, 879 &vma_userptr_notifier_ops); 880 if (err) { 881 xe_vma_free(vma); 882 return ERR_PTR(err); 883 } 884 885 userptr->notifier_seq = LONG_MAX; 886 } 887 888 xe_vm_get(vm); 889 } 890 891 return vma; 892 } 893 894 static void xe_vma_destroy_late(struct xe_vma *vma) 895 { 896 struct xe_vm *vm = xe_vma_vm(vma); 897 struct xe_device *xe = vm->xe; 898 bool read_only = xe_vma_read_only(vma); 899 900 if (xe_vma_is_userptr(vma)) { 901 struct xe_userptr *userptr = &to_userptr_vma(vma)->userptr; 902 903 if (userptr->sg) { 904 dma_unmap_sgtable(xe->drm.dev, 905 userptr->sg, 906 read_only ? DMA_TO_DEVICE : 907 DMA_BIDIRECTIONAL, 0); 908 sg_free_table(userptr->sg); 909 userptr->sg = NULL; 910 } 911 912 /* 913 * Since userptr pages are not pinned, we can't remove 914 * the notifer until we're sure the GPU is not accessing 915 * them anymore 916 */ 917 mmu_interval_notifier_remove(&userptr->notifier); 918 xe_vm_put(vm); 919 } else if (xe_vma_is_null(vma)) { 920 xe_vm_put(vm); 921 } else { 922 xe_bo_put(xe_vma_bo(vma)); 923 } 924 925 xe_vma_free(vma); 926 } 927 928 static void vma_destroy_work_func(struct work_struct *w) 929 { 930 struct xe_vma *vma = 931 container_of(w, struct xe_vma, destroy_work); 932 933 xe_vma_destroy_late(vma); 934 } 935 936 static void vma_destroy_cb(struct dma_fence *fence, 937 struct dma_fence_cb *cb) 938 { 939 struct xe_vma *vma = container_of(cb, struct xe_vma, destroy_cb); 940 941 INIT_WORK(&vma->destroy_work, vma_destroy_work_func); 942 queue_work(system_unbound_wq, &vma->destroy_work); 943 } 944 945 static void xe_vma_destroy(struct xe_vma *vma, struct dma_fence *fence) 946 { 947 struct xe_vm *vm = xe_vma_vm(vma); 948 949 lockdep_assert_held_write(&vm->lock); 950 xe_assert(vm->xe, list_empty(&vma->combined_links.destroy)); 951 952 if (xe_vma_is_userptr(vma)) { 953 xe_assert(vm->xe, vma->gpuva.flags & XE_VMA_DESTROYED); 954 955 spin_lock(&vm->userptr.invalidated_lock); 956 list_del(&to_userptr_vma(vma)->userptr.invalidate_link); 957 spin_unlock(&vm->userptr.invalidated_lock); 958 } else if (!xe_vma_is_null(vma)) { 959 xe_bo_assert_held(xe_vma_bo(vma)); 960 961 drm_gpuva_unlink(&vma->gpuva); 962 } 963 964 xe_vm_assert_held(vm); 965 if (fence) { 966 int ret = dma_fence_add_callback(fence, &vma->destroy_cb, 967 vma_destroy_cb); 968 969 if (ret) { 970 XE_WARN_ON(ret != -ENOENT); 971 xe_vma_destroy_late(vma); 972 } 973 } else { 974 xe_vma_destroy_late(vma); 975 } 976 } 977 978 /** 979 * xe_vm_prepare_vma() - drm_exec utility to lock a vma 980 * @exec: The drm_exec object we're currently locking for. 981 * @vma: The vma for witch we want to lock the vm resv and any attached 982 * object's resv. 983 * @num_shared: The number of dma-fence slots to pre-allocate in the 984 * objects' reservation objects. 985 * 986 * Return: 0 on success, negative error code on error. In particular 987 * may return -EDEADLK on WW transaction contention and -EINTR if 988 * an interruptible wait is terminated by a signal. 989 */ 990 int xe_vm_prepare_vma(struct drm_exec *exec, struct xe_vma *vma, 991 unsigned int num_shared) 992 { 993 struct xe_vm *vm = xe_vma_vm(vma); 994 struct xe_bo *bo = xe_vma_bo(vma); 995 int err; 996 997 XE_WARN_ON(!vm); 998 err = drm_exec_prepare_obj(exec, xe_vm_obj(vm), num_shared); 999 if (!err && bo && !bo->vm) 1000 err = drm_exec_prepare_obj(exec, &bo->ttm.base, num_shared); 1001 1002 return err; 1003 } 1004 1005 static void xe_vma_destroy_unlocked(struct xe_vma *vma) 1006 { 1007 struct drm_exec exec; 1008 int err; 1009 1010 drm_exec_init(&exec, 0, 0); 1011 drm_exec_until_all_locked(&exec) { 1012 err = xe_vm_prepare_vma(&exec, vma, 0); 1013 drm_exec_retry_on_contention(&exec); 1014 if (XE_WARN_ON(err)) 1015 break; 1016 } 1017 1018 xe_vma_destroy(vma, NULL); 1019 1020 drm_exec_fini(&exec); 1021 } 1022 1023 struct xe_vma * 1024 xe_vm_find_overlapping_vma(struct xe_vm *vm, u64 start, u64 range) 1025 { 1026 struct drm_gpuva *gpuva; 1027 1028 lockdep_assert_held(&vm->lock); 1029 1030 if (xe_vm_is_closed_or_banned(vm)) 1031 return NULL; 1032 1033 xe_assert(vm->xe, start + range <= vm->size); 1034 1035 gpuva = drm_gpuva_find_first(&vm->gpuvm, start, range); 1036 1037 return gpuva ? gpuva_to_vma(gpuva) : NULL; 1038 } 1039 1040 static int xe_vm_insert_vma(struct xe_vm *vm, struct xe_vma *vma) 1041 { 1042 int err; 1043 1044 xe_assert(vm->xe, xe_vma_vm(vma) == vm); 1045 lockdep_assert_held(&vm->lock); 1046 1047 err = drm_gpuva_insert(&vm->gpuvm, &vma->gpuva); 1048 XE_WARN_ON(err); /* Shouldn't be possible */ 1049 1050 return err; 1051 } 1052 1053 static void xe_vm_remove_vma(struct xe_vm *vm, struct xe_vma *vma) 1054 { 1055 xe_assert(vm->xe, xe_vma_vm(vma) == vm); 1056 lockdep_assert_held(&vm->lock); 1057 1058 drm_gpuva_remove(&vma->gpuva); 1059 if (vm->usm.last_fault_vma == vma) 1060 vm->usm.last_fault_vma = NULL; 1061 } 1062 1063 static struct drm_gpuva_op *xe_vm_op_alloc(void) 1064 { 1065 struct xe_vma_op *op; 1066 1067 op = kzalloc(sizeof(*op), GFP_KERNEL); 1068 1069 if (unlikely(!op)) 1070 return NULL; 1071 1072 return &op->base; 1073 } 1074 1075 static void xe_vm_free(struct drm_gpuvm *gpuvm); 1076 1077 static struct drm_gpuvm_ops gpuvm_ops = { 1078 .op_alloc = xe_vm_op_alloc, 1079 .vm_bo_validate = xe_gpuvm_validate, 1080 .vm_free = xe_vm_free, 1081 }; 1082 1083 static u64 pde_encode_pat_index(struct xe_device *xe, u16 pat_index) 1084 { 1085 u64 pte = 0; 1086 1087 if (pat_index & BIT(0)) 1088 pte |= XE_PPGTT_PTE_PAT0; 1089 1090 if (pat_index & BIT(1)) 1091 pte |= XE_PPGTT_PTE_PAT1; 1092 1093 return pte; 1094 } 1095 1096 static u64 pte_encode_pat_index(struct xe_device *xe, u16 pat_index, 1097 u32 pt_level) 1098 { 1099 u64 pte = 0; 1100 1101 if (pat_index & BIT(0)) 1102 pte |= XE_PPGTT_PTE_PAT0; 1103 1104 if (pat_index & BIT(1)) 1105 pte |= XE_PPGTT_PTE_PAT1; 1106 1107 if (pat_index & BIT(2)) { 1108 if (pt_level) 1109 pte |= XE_PPGTT_PDE_PDPE_PAT2; 1110 else 1111 pte |= XE_PPGTT_PTE_PAT2; 1112 } 1113 1114 if (pat_index & BIT(3)) 1115 pte |= XELPG_PPGTT_PTE_PAT3; 1116 1117 if (pat_index & (BIT(4))) 1118 pte |= XE2_PPGTT_PTE_PAT4; 1119 1120 return pte; 1121 } 1122 1123 static u64 pte_encode_ps(u32 pt_level) 1124 { 1125 XE_WARN_ON(pt_level > MAX_HUGEPTE_LEVEL); 1126 1127 if (pt_level == 1) 1128 return XE_PDE_PS_2M; 1129 else if (pt_level == 2) 1130 return XE_PDPE_PS_1G; 1131 1132 return 0; 1133 } 1134 1135 static u64 xelp_pde_encode_bo(struct xe_bo *bo, u64 bo_offset, 1136 const u16 pat_index) 1137 { 1138 struct xe_device *xe = xe_bo_device(bo); 1139 u64 pde; 1140 1141 pde = xe_bo_addr(bo, bo_offset, XE_PAGE_SIZE); 1142 pde |= XE_PAGE_PRESENT | XE_PAGE_RW; 1143 pde |= pde_encode_pat_index(xe, pat_index); 1144 1145 return pde; 1146 } 1147 1148 static u64 xelp_pte_encode_bo(struct xe_bo *bo, u64 bo_offset, 1149 u16 pat_index, u32 pt_level) 1150 { 1151 struct xe_device *xe = xe_bo_device(bo); 1152 u64 pte; 1153 1154 pte = xe_bo_addr(bo, bo_offset, XE_PAGE_SIZE); 1155 pte |= XE_PAGE_PRESENT | XE_PAGE_RW; 1156 pte |= pte_encode_pat_index(xe, pat_index, pt_level); 1157 pte |= pte_encode_ps(pt_level); 1158 1159 if (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo)) 1160 pte |= XE_PPGTT_PTE_DM; 1161 1162 return pte; 1163 } 1164 1165 static u64 xelp_pte_encode_vma(u64 pte, struct xe_vma *vma, 1166 u16 pat_index, u32 pt_level) 1167 { 1168 struct xe_device *xe = xe_vma_vm(vma)->xe; 1169 1170 pte |= XE_PAGE_PRESENT; 1171 1172 if (likely(!xe_vma_read_only(vma))) 1173 pte |= XE_PAGE_RW; 1174 1175 pte |= pte_encode_pat_index(xe, pat_index, pt_level); 1176 pte |= pte_encode_ps(pt_level); 1177 1178 if (unlikely(xe_vma_is_null(vma))) 1179 pte |= XE_PTE_NULL; 1180 1181 return pte; 1182 } 1183 1184 static u64 xelp_pte_encode_addr(struct xe_device *xe, u64 addr, 1185 u16 pat_index, 1186 u32 pt_level, bool devmem, u64 flags) 1187 { 1188 u64 pte; 1189 1190 /* Avoid passing random bits directly as flags */ 1191 xe_assert(xe, !(flags & ~XE_PTE_PS64)); 1192 1193 pte = addr; 1194 pte |= XE_PAGE_PRESENT | XE_PAGE_RW; 1195 pte |= pte_encode_pat_index(xe, pat_index, pt_level); 1196 pte |= pte_encode_ps(pt_level); 1197 1198 if (devmem) 1199 pte |= XE_PPGTT_PTE_DM; 1200 1201 pte |= flags; 1202 1203 return pte; 1204 } 1205 1206 static const struct xe_pt_ops xelp_pt_ops = { 1207 .pte_encode_bo = xelp_pte_encode_bo, 1208 .pte_encode_vma = xelp_pte_encode_vma, 1209 .pte_encode_addr = xelp_pte_encode_addr, 1210 .pde_encode_bo = xelp_pde_encode_bo, 1211 }; 1212 1213 static void vm_destroy_work_func(struct work_struct *w); 1214 1215 /** 1216 * xe_vm_create_scratch() - Setup a scratch memory pagetable tree for the 1217 * given tile and vm. 1218 * @xe: xe device. 1219 * @tile: tile to set up for. 1220 * @vm: vm to set up for. 1221 * 1222 * Sets up a pagetable tree with one page-table per level and a single 1223 * leaf PTE. All pagetable entries point to the single page-table or, 1224 * for MAX_HUGEPTE_LEVEL, a NULL huge PTE returning 0 on read and 1225 * writes become NOPs. 1226 * 1227 * Return: 0 on success, negative error code on error. 1228 */ 1229 static int xe_vm_create_scratch(struct xe_device *xe, struct xe_tile *tile, 1230 struct xe_vm *vm) 1231 { 1232 u8 id = tile->id; 1233 int i; 1234 1235 for (i = MAX_HUGEPTE_LEVEL; i < vm->pt_root[id]->level; i++) { 1236 vm->scratch_pt[id][i] = xe_pt_create(vm, tile, i); 1237 if (IS_ERR(vm->scratch_pt[id][i])) 1238 return PTR_ERR(vm->scratch_pt[id][i]); 1239 1240 xe_pt_populate_empty(tile, vm, vm->scratch_pt[id][i]); 1241 } 1242 1243 return 0; 1244 } 1245 1246 static void xe_vm_free_scratch(struct xe_vm *vm) 1247 { 1248 struct xe_tile *tile; 1249 u8 id; 1250 1251 if (!xe_vm_has_scratch(vm)) 1252 return; 1253 1254 for_each_tile(tile, vm->xe, id) { 1255 u32 i; 1256 1257 if (!vm->pt_root[id]) 1258 continue; 1259 1260 for (i = MAX_HUGEPTE_LEVEL; i < vm->pt_root[id]->level; ++i) 1261 if (vm->scratch_pt[id][i]) 1262 xe_pt_destroy(vm->scratch_pt[id][i], vm->flags, NULL); 1263 } 1264 } 1265 1266 struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags) 1267 { 1268 struct drm_gem_object *vm_resv_obj; 1269 struct xe_vm *vm; 1270 int err, number_tiles = 0; 1271 struct xe_tile *tile; 1272 u8 id; 1273 1274 vm = kzalloc(sizeof(*vm), GFP_KERNEL); 1275 if (!vm) 1276 return ERR_PTR(-ENOMEM); 1277 1278 vm->xe = xe; 1279 1280 vm->size = 1ull << xe->info.va_bits; 1281 1282 vm->flags = flags; 1283 1284 init_rwsem(&vm->lock); 1285 1286 INIT_LIST_HEAD(&vm->rebind_list); 1287 1288 INIT_LIST_HEAD(&vm->userptr.repin_list); 1289 INIT_LIST_HEAD(&vm->userptr.invalidated); 1290 init_rwsem(&vm->userptr.notifier_lock); 1291 spin_lock_init(&vm->userptr.invalidated_lock); 1292 1293 INIT_WORK(&vm->destroy_work, vm_destroy_work_func); 1294 1295 INIT_LIST_HEAD(&vm->preempt.exec_queues); 1296 vm->preempt.min_run_period_ms = 10; /* FIXME: Wire up to uAPI */ 1297 1298 for_each_tile(tile, xe, id) 1299 xe_range_fence_tree_init(&vm->rftree[id]); 1300 1301 vm->pt_ops = &xelp_pt_ops; 1302 1303 if (!(flags & XE_VM_FLAG_MIGRATION)) 1304 xe_device_mem_access_get(xe); 1305 1306 vm_resv_obj = drm_gpuvm_resv_object_alloc(&xe->drm); 1307 if (!vm_resv_obj) { 1308 err = -ENOMEM; 1309 goto err_no_resv; 1310 } 1311 1312 drm_gpuvm_init(&vm->gpuvm, "Xe VM", DRM_GPUVM_RESV_PROTECTED, &xe->drm, 1313 vm_resv_obj, 0, vm->size, 0, 0, &gpuvm_ops); 1314 1315 drm_gem_object_put(vm_resv_obj); 1316 1317 err = dma_resv_lock_interruptible(xe_vm_resv(vm), NULL); 1318 if (err) 1319 goto err_close; 1320 1321 if (IS_DGFX(xe) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) 1322 vm->flags |= XE_VM_FLAG_64K; 1323 1324 for_each_tile(tile, xe, id) { 1325 if (flags & XE_VM_FLAG_MIGRATION && 1326 tile->id != XE_VM_FLAG_TILE_ID(flags)) 1327 continue; 1328 1329 vm->pt_root[id] = xe_pt_create(vm, tile, xe->info.vm_max_level); 1330 if (IS_ERR(vm->pt_root[id])) { 1331 err = PTR_ERR(vm->pt_root[id]); 1332 vm->pt_root[id] = NULL; 1333 goto err_unlock_close; 1334 } 1335 } 1336 1337 if (xe_vm_has_scratch(vm)) { 1338 for_each_tile(tile, xe, id) { 1339 if (!vm->pt_root[id]) 1340 continue; 1341 1342 err = xe_vm_create_scratch(xe, tile, vm); 1343 if (err) 1344 goto err_unlock_close; 1345 } 1346 vm->batch_invalidate_tlb = true; 1347 } 1348 1349 if (flags & XE_VM_FLAG_LR_MODE) { 1350 INIT_WORK(&vm->preempt.rebind_work, preempt_rebind_work_func); 1351 vm->flags |= XE_VM_FLAG_LR_MODE; 1352 vm->batch_invalidate_tlb = false; 1353 } 1354 1355 /* Fill pt_root after allocating scratch tables */ 1356 for_each_tile(tile, xe, id) { 1357 if (!vm->pt_root[id]) 1358 continue; 1359 1360 xe_pt_populate_empty(tile, vm, vm->pt_root[id]); 1361 } 1362 dma_resv_unlock(xe_vm_resv(vm)); 1363 1364 /* Kernel migration VM shouldn't have a circular loop.. */ 1365 if (!(flags & XE_VM_FLAG_MIGRATION)) { 1366 for_each_tile(tile, xe, id) { 1367 struct xe_gt *gt = tile->primary_gt; 1368 struct xe_vm *migrate_vm; 1369 struct xe_exec_queue *q; 1370 u32 create_flags = EXEC_QUEUE_FLAG_VM; 1371 1372 if (!vm->pt_root[id]) 1373 continue; 1374 1375 migrate_vm = xe_migrate_get_vm(tile->migrate); 1376 q = xe_exec_queue_create_class(xe, gt, migrate_vm, 1377 XE_ENGINE_CLASS_COPY, 1378 create_flags); 1379 xe_vm_put(migrate_vm); 1380 if (IS_ERR(q)) { 1381 err = PTR_ERR(q); 1382 goto err_close; 1383 } 1384 vm->q[id] = q; 1385 number_tiles++; 1386 } 1387 } 1388 1389 if (number_tiles > 1) 1390 vm->composite_fence_ctx = dma_fence_context_alloc(1); 1391 1392 mutex_lock(&xe->usm.lock); 1393 if (flags & XE_VM_FLAG_FAULT_MODE) 1394 xe->usm.num_vm_in_fault_mode++; 1395 else if (!(flags & XE_VM_FLAG_MIGRATION)) 1396 xe->usm.num_vm_in_non_fault_mode++; 1397 mutex_unlock(&xe->usm.lock); 1398 1399 trace_xe_vm_create(vm); 1400 1401 return vm; 1402 1403 err_unlock_close: 1404 dma_resv_unlock(xe_vm_resv(vm)); 1405 err_close: 1406 xe_vm_close_and_put(vm); 1407 return ERR_PTR(err); 1408 1409 err_no_resv: 1410 for_each_tile(tile, xe, id) 1411 xe_range_fence_tree_fini(&vm->rftree[id]); 1412 kfree(vm); 1413 if (!(flags & XE_VM_FLAG_MIGRATION)) 1414 xe_device_mem_access_put(xe); 1415 return ERR_PTR(err); 1416 } 1417 1418 static void xe_vm_close(struct xe_vm *vm) 1419 { 1420 down_write(&vm->lock); 1421 vm->size = 0; 1422 up_write(&vm->lock); 1423 } 1424 1425 void xe_vm_close_and_put(struct xe_vm *vm) 1426 { 1427 LIST_HEAD(contested); 1428 struct xe_device *xe = vm->xe; 1429 struct xe_tile *tile; 1430 struct xe_vma *vma, *next_vma; 1431 struct drm_gpuva *gpuva, *next; 1432 u8 id; 1433 1434 xe_assert(xe, !vm->preempt.num_exec_queues); 1435 1436 xe_vm_close(vm); 1437 if (xe_vm_in_preempt_fence_mode(vm)) 1438 flush_work(&vm->preempt.rebind_work); 1439 1440 down_write(&vm->lock); 1441 for_each_tile(tile, xe, id) { 1442 if (vm->q[id]) 1443 xe_exec_queue_last_fence_put(vm->q[id], vm); 1444 } 1445 up_write(&vm->lock); 1446 1447 for_each_tile(tile, xe, id) { 1448 if (vm->q[id]) { 1449 xe_exec_queue_kill(vm->q[id]); 1450 xe_exec_queue_put(vm->q[id]); 1451 vm->q[id] = NULL; 1452 } 1453 } 1454 1455 down_write(&vm->lock); 1456 xe_vm_lock(vm, false); 1457 drm_gpuvm_for_each_va_safe(gpuva, next, &vm->gpuvm) { 1458 vma = gpuva_to_vma(gpuva); 1459 1460 if (xe_vma_has_no_bo(vma)) { 1461 down_read(&vm->userptr.notifier_lock); 1462 vma->gpuva.flags |= XE_VMA_DESTROYED; 1463 up_read(&vm->userptr.notifier_lock); 1464 } 1465 1466 xe_vm_remove_vma(vm, vma); 1467 1468 /* easy case, remove from VMA? */ 1469 if (xe_vma_has_no_bo(vma) || xe_vma_bo(vma)->vm) { 1470 list_del_init(&vma->combined_links.rebind); 1471 xe_vma_destroy(vma, NULL); 1472 continue; 1473 } 1474 1475 list_move_tail(&vma->combined_links.destroy, &contested); 1476 vma->gpuva.flags |= XE_VMA_DESTROYED; 1477 } 1478 1479 /* 1480 * All vm operations will add shared fences to resv. 1481 * The only exception is eviction for a shared object, 1482 * but even so, the unbind when evicted would still 1483 * install a fence to resv. Hence it's safe to 1484 * destroy the pagetables immediately. 1485 */ 1486 xe_vm_free_scratch(vm); 1487 1488 for_each_tile(tile, xe, id) { 1489 if (vm->pt_root[id]) { 1490 xe_pt_destroy(vm->pt_root[id], vm->flags, NULL); 1491 vm->pt_root[id] = NULL; 1492 } 1493 } 1494 xe_vm_unlock(vm); 1495 1496 /* 1497 * VM is now dead, cannot re-add nodes to vm->vmas if it's NULL 1498 * Since we hold a refcount to the bo, we can remove and free 1499 * the members safely without locking. 1500 */ 1501 list_for_each_entry_safe(vma, next_vma, &contested, 1502 combined_links.destroy) { 1503 list_del_init(&vma->combined_links.destroy); 1504 xe_vma_destroy_unlocked(vma); 1505 } 1506 1507 up_write(&vm->lock); 1508 1509 mutex_lock(&xe->usm.lock); 1510 if (vm->flags & XE_VM_FLAG_FAULT_MODE) 1511 xe->usm.num_vm_in_fault_mode--; 1512 else if (!(vm->flags & XE_VM_FLAG_MIGRATION)) 1513 xe->usm.num_vm_in_non_fault_mode--; 1514 mutex_unlock(&xe->usm.lock); 1515 1516 for_each_tile(tile, xe, id) 1517 xe_range_fence_tree_fini(&vm->rftree[id]); 1518 1519 xe_vm_put(vm); 1520 } 1521 1522 static void vm_destroy_work_func(struct work_struct *w) 1523 { 1524 struct xe_vm *vm = 1525 container_of(w, struct xe_vm, destroy_work); 1526 struct xe_device *xe = vm->xe; 1527 struct xe_tile *tile; 1528 u8 id; 1529 void *lookup; 1530 1531 /* xe_vm_close_and_put was not called? */ 1532 xe_assert(xe, !vm->size); 1533 1534 if (!(vm->flags & XE_VM_FLAG_MIGRATION)) { 1535 xe_device_mem_access_put(xe); 1536 1537 if (xe->info.has_asid && vm->usm.asid) { 1538 mutex_lock(&xe->usm.lock); 1539 lookup = xa_erase(&xe->usm.asid_to_vm, vm->usm.asid); 1540 xe_assert(xe, lookup == vm); 1541 mutex_unlock(&xe->usm.lock); 1542 } 1543 } 1544 1545 for_each_tile(tile, xe, id) 1546 XE_WARN_ON(vm->pt_root[id]); 1547 1548 trace_xe_vm_free(vm); 1549 dma_fence_put(vm->rebind_fence); 1550 kfree(vm); 1551 } 1552 1553 static void xe_vm_free(struct drm_gpuvm *gpuvm) 1554 { 1555 struct xe_vm *vm = container_of(gpuvm, struct xe_vm, gpuvm); 1556 1557 /* To destroy the VM we need to be able to sleep */ 1558 queue_work(system_unbound_wq, &vm->destroy_work); 1559 } 1560 1561 struct xe_vm *xe_vm_lookup(struct xe_file *xef, u32 id) 1562 { 1563 struct xe_vm *vm; 1564 1565 mutex_lock(&xef->vm.lock); 1566 vm = xa_load(&xef->vm.xa, id); 1567 if (vm) 1568 xe_vm_get(vm); 1569 mutex_unlock(&xef->vm.lock); 1570 1571 return vm; 1572 } 1573 1574 u64 xe_vm_pdp4_descriptor(struct xe_vm *vm, struct xe_tile *tile) 1575 { 1576 return vm->pt_ops->pde_encode_bo(vm->pt_root[tile->id]->bo, 0, 1577 tile_to_xe(tile)->pat.idx[XE_CACHE_WB]); 1578 } 1579 1580 static struct xe_exec_queue * 1581 to_wait_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q) 1582 { 1583 return q ? q : vm->q[0]; 1584 } 1585 1586 static struct dma_fence * 1587 xe_vm_unbind_vma(struct xe_vma *vma, struct xe_exec_queue *q, 1588 struct xe_sync_entry *syncs, u32 num_syncs, 1589 bool first_op, bool last_op) 1590 { 1591 struct xe_vm *vm = xe_vma_vm(vma); 1592 struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q); 1593 struct xe_tile *tile; 1594 struct dma_fence *fence = NULL; 1595 struct dma_fence **fences = NULL; 1596 struct dma_fence_array *cf = NULL; 1597 int cur_fence = 0, i; 1598 int number_tiles = hweight8(vma->tile_present); 1599 int err; 1600 u8 id; 1601 1602 trace_xe_vma_unbind(vma); 1603 1604 if (number_tiles > 1) { 1605 fences = kmalloc_array(number_tiles, sizeof(*fences), 1606 GFP_KERNEL); 1607 if (!fences) 1608 return ERR_PTR(-ENOMEM); 1609 } 1610 1611 for_each_tile(tile, vm->xe, id) { 1612 if (!(vma->tile_present & BIT(id))) 1613 goto next; 1614 1615 fence = __xe_pt_unbind_vma(tile, vma, q ? q : vm->q[id], 1616 first_op ? syncs : NULL, 1617 first_op ? num_syncs : 0); 1618 if (IS_ERR(fence)) { 1619 err = PTR_ERR(fence); 1620 goto err_fences; 1621 } 1622 1623 if (fences) 1624 fences[cur_fence++] = fence; 1625 1626 next: 1627 if (q && vm->pt_root[id] && !list_empty(&q->multi_gt_list)) 1628 q = list_next_entry(q, multi_gt_list); 1629 } 1630 1631 if (fences) { 1632 cf = dma_fence_array_create(number_tiles, fences, 1633 vm->composite_fence_ctx, 1634 vm->composite_fence_seqno++, 1635 false); 1636 if (!cf) { 1637 --vm->composite_fence_seqno; 1638 err = -ENOMEM; 1639 goto err_fences; 1640 } 1641 } 1642 1643 fence = cf ? &cf->base : !fence ? 1644 xe_exec_queue_last_fence_get(wait_exec_queue, vm) : fence; 1645 if (last_op) { 1646 for (i = 0; i < num_syncs; i++) 1647 xe_sync_entry_signal(&syncs[i], NULL, fence); 1648 } 1649 1650 return fence; 1651 1652 err_fences: 1653 if (fences) { 1654 while (cur_fence) 1655 dma_fence_put(fences[--cur_fence]); 1656 kfree(fences); 1657 } 1658 1659 return ERR_PTR(err); 1660 } 1661 1662 static struct dma_fence * 1663 xe_vm_bind_vma(struct xe_vma *vma, struct xe_exec_queue *q, 1664 struct xe_sync_entry *syncs, u32 num_syncs, 1665 bool first_op, bool last_op) 1666 { 1667 struct xe_tile *tile; 1668 struct dma_fence *fence; 1669 struct dma_fence **fences = NULL; 1670 struct dma_fence_array *cf = NULL; 1671 struct xe_vm *vm = xe_vma_vm(vma); 1672 int cur_fence = 0, i; 1673 int number_tiles = hweight8(vma->tile_mask); 1674 int err; 1675 u8 id; 1676 1677 trace_xe_vma_bind(vma); 1678 1679 if (number_tiles > 1) { 1680 fences = kmalloc_array(number_tiles, sizeof(*fences), 1681 GFP_KERNEL); 1682 if (!fences) 1683 return ERR_PTR(-ENOMEM); 1684 } 1685 1686 for_each_tile(tile, vm->xe, id) { 1687 if (!(vma->tile_mask & BIT(id))) 1688 goto next; 1689 1690 fence = __xe_pt_bind_vma(tile, vma, q ? q : vm->q[id], 1691 first_op ? syncs : NULL, 1692 first_op ? num_syncs : 0, 1693 vma->tile_present & BIT(id)); 1694 if (IS_ERR(fence)) { 1695 err = PTR_ERR(fence); 1696 goto err_fences; 1697 } 1698 1699 if (fences) 1700 fences[cur_fence++] = fence; 1701 1702 next: 1703 if (q && vm->pt_root[id] && !list_empty(&q->multi_gt_list)) 1704 q = list_next_entry(q, multi_gt_list); 1705 } 1706 1707 if (fences) { 1708 cf = dma_fence_array_create(number_tiles, fences, 1709 vm->composite_fence_ctx, 1710 vm->composite_fence_seqno++, 1711 false); 1712 if (!cf) { 1713 --vm->composite_fence_seqno; 1714 err = -ENOMEM; 1715 goto err_fences; 1716 } 1717 } 1718 1719 if (last_op) { 1720 for (i = 0; i < num_syncs; i++) 1721 xe_sync_entry_signal(&syncs[i], NULL, 1722 cf ? &cf->base : fence); 1723 } 1724 1725 return cf ? &cf->base : fence; 1726 1727 err_fences: 1728 if (fences) { 1729 while (cur_fence) 1730 dma_fence_put(fences[--cur_fence]); 1731 kfree(fences); 1732 } 1733 1734 return ERR_PTR(err); 1735 } 1736 1737 static int __xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, 1738 struct xe_exec_queue *q, struct xe_sync_entry *syncs, 1739 u32 num_syncs, bool immediate, bool first_op, 1740 bool last_op) 1741 { 1742 struct dma_fence *fence; 1743 struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q); 1744 1745 xe_vm_assert_held(vm); 1746 1747 if (immediate) { 1748 fence = xe_vm_bind_vma(vma, q, syncs, num_syncs, first_op, 1749 last_op); 1750 if (IS_ERR(fence)) 1751 return PTR_ERR(fence); 1752 } else { 1753 int i; 1754 1755 xe_assert(vm->xe, xe_vm_in_fault_mode(vm)); 1756 1757 fence = xe_exec_queue_last_fence_get(wait_exec_queue, vm); 1758 if (last_op) { 1759 for (i = 0; i < num_syncs; i++) 1760 xe_sync_entry_signal(&syncs[i], NULL, fence); 1761 } 1762 } 1763 1764 if (last_op) 1765 xe_exec_queue_last_fence_set(wait_exec_queue, vm, fence); 1766 dma_fence_put(fence); 1767 1768 return 0; 1769 } 1770 1771 static int xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, struct xe_exec_queue *q, 1772 struct xe_bo *bo, struct xe_sync_entry *syncs, 1773 u32 num_syncs, bool immediate, bool first_op, 1774 bool last_op) 1775 { 1776 int err; 1777 1778 xe_vm_assert_held(vm); 1779 xe_bo_assert_held(bo); 1780 1781 if (bo && immediate) { 1782 err = xe_bo_validate(bo, vm, true); 1783 if (err) 1784 return err; 1785 } 1786 1787 return __xe_vm_bind(vm, vma, q, syncs, num_syncs, immediate, first_op, 1788 last_op); 1789 } 1790 1791 static int xe_vm_unbind(struct xe_vm *vm, struct xe_vma *vma, 1792 struct xe_exec_queue *q, struct xe_sync_entry *syncs, 1793 u32 num_syncs, bool first_op, bool last_op) 1794 { 1795 struct dma_fence *fence; 1796 struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q); 1797 1798 xe_vm_assert_held(vm); 1799 xe_bo_assert_held(xe_vma_bo(vma)); 1800 1801 fence = xe_vm_unbind_vma(vma, q, syncs, num_syncs, first_op, last_op); 1802 if (IS_ERR(fence)) 1803 return PTR_ERR(fence); 1804 1805 xe_vma_destroy(vma, fence); 1806 if (last_op) 1807 xe_exec_queue_last_fence_set(wait_exec_queue, vm, fence); 1808 dma_fence_put(fence); 1809 1810 return 0; 1811 } 1812 1813 #define ALL_DRM_XE_VM_CREATE_FLAGS (DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE | \ 1814 DRM_XE_VM_CREATE_FLAG_LR_MODE | \ 1815 DRM_XE_VM_CREATE_FLAG_FAULT_MODE) 1816 1817 int xe_vm_create_ioctl(struct drm_device *dev, void *data, 1818 struct drm_file *file) 1819 { 1820 struct xe_device *xe = to_xe_device(dev); 1821 struct xe_file *xef = to_xe_file(file); 1822 struct drm_xe_vm_create *args = data; 1823 struct xe_tile *tile; 1824 struct xe_vm *vm; 1825 u32 id, asid; 1826 int err; 1827 u32 flags = 0; 1828 1829 if (XE_IOCTL_DBG(xe, args->extensions)) 1830 return -EINVAL; 1831 1832 if (XE_WA(xe_root_mmio_gt(xe), 14016763929)) 1833 args->flags |= DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE; 1834 1835 if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE && 1836 !xe->info.has_usm)) 1837 return -EINVAL; 1838 1839 if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 1840 return -EINVAL; 1841 1842 if (XE_IOCTL_DBG(xe, args->flags & ~ALL_DRM_XE_VM_CREATE_FLAGS)) 1843 return -EINVAL; 1844 1845 if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE && 1846 args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE)) 1847 return -EINVAL; 1848 1849 if (XE_IOCTL_DBG(xe, !(args->flags & DRM_XE_VM_CREATE_FLAG_LR_MODE) && 1850 args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE)) 1851 return -EINVAL; 1852 1853 if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE && 1854 xe_device_in_non_fault_mode(xe))) 1855 return -EINVAL; 1856 1857 if (XE_IOCTL_DBG(xe, !(args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE) && 1858 xe_device_in_fault_mode(xe))) 1859 return -EINVAL; 1860 1861 if (XE_IOCTL_DBG(xe, args->extensions)) 1862 return -EINVAL; 1863 1864 if (args->flags & DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE) 1865 flags |= XE_VM_FLAG_SCRATCH_PAGE; 1866 if (args->flags & DRM_XE_VM_CREATE_FLAG_LR_MODE) 1867 flags |= XE_VM_FLAG_LR_MODE; 1868 if (args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE) 1869 flags |= XE_VM_FLAG_FAULT_MODE; 1870 1871 vm = xe_vm_create(xe, flags); 1872 if (IS_ERR(vm)) 1873 return PTR_ERR(vm); 1874 1875 mutex_lock(&xef->vm.lock); 1876 err = xa_alloc(&xef->vm.xa, &id, vm, xa_limit_32b, GFP_KERNEL); 1877 mutex_unlock(&xef->vm.lock); 1878 if (err) 1879 goto err_close_and_put; 1880 1881 if (xe->info.has_asid) { 1882 mutex_lock(&xe->usm.lock); 1883 err = xa_alloc_cyclic(&xe->usm.asid_to_vm, &asid, vm, 1884 XA_LIMIT(1, XE_MAX_ASID - 1), 1885 &xe->usm.next_asid, GFP_KERNEL); 1886 mutex_unlock(&xe->usm.lock); 1887 if (err < 0) 1888 goto err_free_id; 1889 1890 vm->usm.asid = asid; 1891 } 1892 1893 args->vm_id = id; 1894 vm->xef = xef; 1895 1896 /* Record BO memory for VM pagetable created against client */ 1897 for_each_tile(tile, xe, id) 1898 if (vm->pt_root[id]) 1899 xe_drm_client_add_bo(vm->xef->client, vm->pt_root[id]->bo); 1900 1901 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEM) 1902 /* Warning: Security issue - never enable by default */ 1903 args->reserved[0] = xe_bo_main_addr(vm->pt_root[0]->bo, XE_PAGE_SIZE); 1904 #endif 1905 1906 return 0; 1907 1908 err_free_id: 1909 mutex_lock(&xef->vm.lock); 1910 xa_erase(&xef->vm.xa, id); 1911 mutex_unlock(&xef->vm.lock); 1912 err_close_and_put: 1913 xe_vm_close_and_put(vm); 1914 1915 return err; 1916 } 1917 1918 int xe_vm_destroy_ioctl(struct drm_device *dev, void *data, 1919 struct drm_file *file) 1920 { 1921 struct xe_device *xe = to_xe_device(dev); 1922 struct xe_file *xef = to_xe_file(file); 1923 struct drm_xe_vm_destroy *args = data; 1924 struct xe_vm *vm; 1925 int err = 0; 1926 1927 if (XE_IOCTL_DBG(xe, args->pad) || 1928 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 1929 return -EINVAL; 1930 1931 mutex_lock(&xef->vm.lock); 1932 vm = xa_load(&xef->vm.xa, args->vm_id); 1933 if (XE_IOCTL_DBG(xe, !vm)) 1934 err = -ENOENT; 1935 else if (XE_IOCTL_DBG(xe, vm->preempt.num_exec_queues)) 1936 err = -EBUSY; 1937 else 1938 xa_erase(&xef->vm.xa, args->vm_id); 1939 mutex_unlock(&xef->vm.lock); 1940 1941 if (!err) 1942 xe_vm_close_and_put(vm); 1943 1944 return err; 1945 } 1946 1947 static const u32 region_to_mem_type[] = { 1948 XE_PL_TT, 1949 XE_PL_VRAM0, 1950 XE_PL_VRAM1, 1951 }; 1952 1953 static int xe_vm_prefetch(struct xe_vm *vm, struct xe_vma *vma, 1954 struct xe_exec_queue *q, u32 region, 1955 struct xe_sync_entry *syncs, u32 num_syncs, 1956 bool first_op, bool last_op) 1957 { 1958 struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q); 1959 int err; 1960 1961 xe_assert(vm->xe, region <= ARRAY_SIZE(region_to_mem_type)); 1962 1963 if (!xe_vma_has_no_bo(vma)) { 1964 err = xe_bo_migrate(xe_vma_bo(vma), region_to_mem_type[region]); 1965 if (err) 1966 return err; 1967 } 1968 1969 if (vma->tile_mask != (vma->tile_present & ~vma->usm.tile_invalidated)) { 1970 return xe_vm_bind(vm, vma, q, xe_vma_bo(vma), syncs, num_syncs, 1971 true, first_op, last_op); 1972 } else { 1973 int i; 1974 1975 /* Nothing to do, signal fences now */ 1976 if (last_op) { 1977 for (i = 0; i < num_syncs; i++) { 1978 struct dma_fence *fence = 1979 xe_exec_queue_last_fence_get(wait_exec_queue, vm); 1980 1981 xe_sync_entry_signal(&syncs[i], NULL, fence); 1982 dma_fence_put(fence); 1983 } 1984 } 1985 1986 return 0; 1987 } 1988 } 1989 1990 static void prep_vma_destroy(struct xe_vm *vm, struct xe_vma *vma, 1991 bool post_commit) 1992 { 1993 down_read(&vm->userptr.notifier_lock); 1994 vma->gpuva.flags |= XE_VMA_DESTROYED; 1995 up_read(&vm->userptr.notifier_lock); 1996 if (post_commit) 1997 xe_vm_remove_vma(vm, vma); 1998 } 1999 2000 #undef ULL 2001 #define ULL unsigned long long 2002 2003 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM) 2004 static void print_op(struct xe_device *xe, struct drm_gpuva_op *op) 2005 { 2006 struct xe_vma *vma; 2007 2008 switch (op->op) { 2009 case DRM_GPUVA_OP_MAP: 2010 vm_dbg(&xe->drm, "MAP: addr=0x%016llx, range=0x%016llx", 2011 (ULL)op->map.va.addr, (ULL)op->map.va.range); 2012 break; 2013 case DRM_GPUVA_OP_REMAP: 2014 vma = gpuva_to_vma(op->remap.unmap->va); 2015 vm_dbg(&xe->drm, "REMAP:UNMAP: addr=0x%016llx, range=0x%016llx, keep=%d", 2016 (ULL)xe_vma_start(vma), (ULL)xe_vma_size(vma), 2017 op->remap.unmap->keep ? 1 : 0); 2018 if (op->remap.prev) 2019 vm_dbg(&xe->drm, 2020 "REMAP:PREV: addr=0x%016llx, range=0x%016llx", 2021 (ULL)op->remap.prev->va.addr, 2022 (ULL)op->remap.prev->va.range); 2023 if (op->remap.next) 2024 vm_dbg(&xe->drm, 2025 "REMAP:NEXT: addr=0x%016llx, range=0x%016llx", 2026 (ULL)op->remap.next->va.addr, 2027 (ULL)op->remap.next->va.range); 2028 break; 2029 case DRM_GPUVA_OP_UNMAP: 2030 vma = gpuva_to_vma(op->unmap.va); 2031 vm_dbg(&xe->drm, "UNMAP: addr=0x%016llx, range=0x%016llx, keep=%d", 2032 (ULL)xe_vma_start(vma), (ULL)xe_vma_size(vma), 2033 op->unmap.keep ? 1 : 0); 2034 break; 2035 case DRM_GPUVA_OP_PREFETCH: 2036 vma = gpuva_to_vma(op->prefetch.va); 2037 vm_dbg(&xe->drm, "PREFETCH: addr=0x%016llx, range=0x%016llx", 2038 (ULL)xe_vma_start(vma), (ULL)xe_vma_size(vma)); 2039 break; 2040 default: 2041 drm_warn(&xe->drm, "NOT POSSIBLE"); 2042 } 2043 } 2044 #else 2045 static void print_op(struct xe_device *xe, struct drm_gpuva_op *op) 2046 { 2047 } 2048 #endif 2049 2050 /* 2051 * Create operations list from IOCTL arguments, setup operations fields so parse 2052 * and commit steps are decoupled from IOCTL arguments. This step can fail. 2053 */ 2054 static struct drm_gpuva_ops * 2055 vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_bo *bo, 2056 u64 bo_offset_or_userptr, u64 addr, u64 range, 2057 u32 operation, u32 flags, 2058 u32 prefetch_region, u16 pat_index) 2059 { 2060 struct drm_gem_object *obj = bo ? &bo->ttm.base : NULL; 2061 struct drm_gpuva_ops *ops; 2062 struct drm_gpuva_op *__op; 2063 struct drm_gpuvm_bo *vm_bo; 2064 int err; 2065 2066 lockdep_assert_held_write(&vm->lock); 2067 2068 vm_dbg(&vm->xe->drm, 2069 "op=%d, addr=0x%016llx, range=0x%016llx, bo_offset_or_userptr=0x%016llx", 2070 operation, (ULL)addr, (ULL)range, 2071 (ULL)bo_offset_or_userptr); 2072 2073 switch (operation) { 2074 case DRM_XE_VM_BIND_OP_MAP: 2075 case DRM_XE_VM_BIND_OP_MAP_USERPTR: 2076 ops = drm_gpuvm_sm_map_ops_create(&vm->gpuvm, addr, range, 2077 obj, bo_offset_or_userptr); 2078 break; 2079 case DRM_XE_VM_BIND_OP_UNMAP: 2080 ops = drm_gpuvm_sm_unmap_ops_create(&vm->gpuvm, addr, range); 2081 break; 2082 case DRM_XE_VM_BIND_OP_PREFETCH: 2083 ops = drm_gpuvm_prefetch_ops_create(&vm->gpuvm, addr, range); 2084 break; 2085 case DRM_XE_VM_BIND_OP_UNMAP_ALL: 2086 xe_assert(vm->xe, bo); 2087 2088 err = xe_bo_lock(bo, true); 2089 if (err) 2090 return ERR_PTR(err); 2091 2092 vm_bo = drm_gpuvm_bo_obtain(&vm->gpuvm, obj); 2093 if (IS_ERR(vm_bo)) { 2094 xe_bo_unlock(bo); 2095 return ERR_CAST(vm_bo); 2096 } 2097 2098 ops = drm_gpuvm_bo_unmap_ops_create(vm_bo); 2099 drm_gpuvm_bo_put(vm_bo); 2100 xe_bo_unlock(bo); 2101 break; 2102 default: 2103 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); 2104 ops = ERR_PTR(-EINVAL); 2105 } 2106 if (IS_ERR(ops)) 2107 return ops; 2108 2109 drm_gpuva_for_each_op(__op, ops) { 2110 struct xe_vma_op *op = gpuva_op_to_vma_op(__op); 2111 2112 if (__op->op == DRM_GPUVA_OP_MAP) { 2113 op->map.immediate = 2114 flags & DRM_XE_VM_BIND_FLAG_IMMEDIATE; 2115 op->map.read_only = 2116 flags & DRM_XE_VM_BIND_FLAG_READONLY; 2117 op->map.is_null = flags & DRM_XE_VM_BIND_FLAG_NULL; 2118 op->map.pat_index = pat_index; 2119 } else if (__op->op == DRM_GPUVA_OP_PREFETCH) { 2120 op->prefetch.region = prefetch_region; 2121 } 2122 2123 print_op(vm->xe, __op); 2124 } 2125 2126 return ops; 2127 } 2128 2129 static struct xe_vma *new_vma(struct xe_vm *vm, struct drm_gpuva_op_map *op, 2130 u16 pat_index, unsigned int flags) 2131 { 2132 struct xe_bo *bo = op->gem.obj ? gem_to_xe_bo(op->gem.obj) : NULL; 2133 struct drm_exec exec; 2134 struct xe_vma *vma; 2135 int err; 2136 2137 lockdep_assert_held_write(&vm->lock); 2138 2139 if (bo) { 2140 drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0); 2141 drm_exec_until_all_locked(&exec) { 2142 err = 0; 2143 if (!bo->vm) { 2144 err = drm_exec_lock_obj(&exec, xe_vm_obj(vm)); 2145 drm_exec_retry_on_contention(&exec); 2146 } 2147 if (!err) { 2148 err = drm_exec_lock_obj(&exec, &bo->ttm.base); 2149 drm_exec_retry_on_contention(&exec); 2150 } 2151 if (err) { 2152 drm_exec_fini(&exec); 2153 return ERR_PTR(err); 2154 } 2155 } 2156 } 2157 vma = xe_vma_create(vm, bo, op->gem.offset, 2158 op->va.addr, op->va.addr + 2159 op->va.range - 1, pat_index, flags); 2160 if (bo) 2161 drm_exec_fini(&exec); 2162 2163 if (xe_vma_is_userptr(vma)) { 2164 err = xe_vma_userptr_pin_pages(to_userptr_vma(vma)); 2165 if (err) { 2166 prep_vma_destroy(vm, vma, false); 2167 xe_vma_destroy_unlocked(vma); 2168 return ERR_PTR(err); 2169 } 2170 } else if (!xe_vma_has_no_bo(vma) && !bo->vm) { 2171 err = add_preempt_fences(vm, bo); 2172 if (err) { 2173 prep_vma_destroy(vm, vma, false); 2174 xe_vma_destroy_unlocked(vma); 2175 return ERR_PTR(err); 2176 } 2177 } 2178 2179 return vma; 2180 } 2181 2182 static u64 xe_vma_max_pte_size(struct xe_vma *vma) 2183 { 2184 if (vma->gpuva.flags & XE_VMA_PTE_1G) 2185 return SZ_1G; 2186 else if (vma->gpuva.flags & XE_VMA_PTE_2M) 2187 return SZ_2M; 2188 else if (vma->gpuva.flags & XE_VMA_PTE_4K) 2189 return SZ_4K; 2190 2191 return SZ_1G; /* Uninitialized, used max size */ 2192 } 2193 2194 static u64 xe_vma_set_pte_size(struct xe_vma *vma, u64 size) 2195 { 2196 switch (size) { 2197 case SZ_1G: 2198 vma->gpuva.flags |= XE_VMA_PTE_1G; 2199 break; 2200 case SZ_2M: 2201 vma->gpuva.flags |= XE_VMA_PTE_2M; 2202 break; 2203 } 2204 2205 return SZ_4K; 2206 } 2207 2208 static int xe_vma_op_commit(struct xe_vm *vm, struct xe_vma_op *op) 2209 { 2210 int err = 0; 2211 2212 lockdep_assert_held_write(&vm->lock); 2213 2214 switch (op->base.op) { 2215 case DRM_GPUVA_OP_MAP: 2216 err |= xe_vm_insert_vma(vm, op->map.vma); 2217 if (!err) 2218 op->flags |= XE_VMA_OP_COMMITTED; 2219 break; 2220 case DRM_GPUVA_OP_REMAP: 2221 { 2222 u8 tile_present = 2223 gpuva_to_vma(op->base.remap.unmap->va)->tile_present; 2224 2225 prep_vma_destroy(vm, gpuva_to_vma(op->base.remap.unmap->va), 2226 true); 2227 op->flags |= XE_VMA_OP_COMMITTED; 2228 2229 if (op->remap.prev) { 2230 err |= xe_vm_insert_vma(vm, op->remap.prev); 2231 if (!err) 2232 op->flags |= XE_VMA_OP_PREV_COMMITTED; 2233 if (!err && op->remap.skip_prev) { 2234 op->remap.prev->tile_present = 2235 tile_present; 2236 op->remap.prev = NULL; 2237 } 2238 } 2239 if (op->remap.next) { 2240 err |= xe_vm_insert_vma(vm, op->remap.next); 2241 if (!err) 2242 op->flags |= XE_VMA_OP_NEXT_COMMITTED; 2243 if (!err && op->remap.skip_next) { 2244 op->remap.next->tile_present = 2245 tile_present; 2246 op->remap.next = NULL; 2247 } 2248 } 2249 2250 /* Adjust for partial unbind after removin VMA from VM */ 2251 if (!err) { 2252 op->base.remap.unmap->va->va.addr = op->remap.start; 2253 op->base.remap.unmap->va->va.range = op->remap.range; 2254 } 2255 break; 2256 } 2257 case DRM_GPUVA_OP_UNMAP: 2258 prep_vma_destroy(vm, gpuva_to_vma(op->base.unmap.va), true); 2259 op->flags |= XE_VMA_OP_COMMITTED; 2260 break; 2261 case DRM_GPUVA_OP_PREFETCH: 2262 op->flags |= XE_VMA_OP_COMMITTED; 2263 break; 2264 default: 2265 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); 2266 } 2267 2268 return err; 2269 } 2270 2271 2272 static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q, 2273 struct drm_gpuva_ops *ops, 2274 struct xe_sync_entry *syncs, u32 num_syncs, 2275 struct list_head *ops_list, bool last) 2276 { 2277 struct xe_vma_op *last_op = NULL; 2278 struct drm_gpuva_op *__op; 2279 int err = 0; 2280 2281 lockdep_assert_held_write(&vm->lock); 2282 2283 drm_gpuva_for_each_op(__op, ops) { 2284 struct xe_vma_op *op = gpuva_op_to_vma_op(__op); 2285 struct xe_vma *vma; 2286 bool first = list_empty(ops_list); 2287 unsigned int flags = 0; 2288 2289 INIT_LIST_HEAD(&op->link); 2290 list_add_tail(&op->link, ops_list); 2291 2292 if (first) { 2293 op->flags |= XE_VMA_OP_FIRST; 2294 op->num_syncs = num_syncs; 2295 op->syncs = syncs; 2296 } 2297 2298 op->q = q; 2299 2300 switch (op->base.op) { 2301 case DRM_GPUVA_OP_MAP: 2302 { 2303 flags |= op->map.read_only ? 2304 VMA_CREATE_FLAG_READ_ONLY : 0; 2305 flags |= op->map.is_null ? 2306 VMA_CREATE_FLAG_IS_NULL : 0; 2307 2308 vma = new_vma(vm, &op->base.map, op->map.pat_index, 2309 flags); 2310 if (IS_ERR(vma)) 2311 return PTR_ERR(vma); 2312 2313 op->map.vma = vma; 2314 break; 2315 } 2316 case DRM_GPUVA_OP_REMAP: 2317 { 2318 struct xe_vma *old = 2319 gpuva_to_vma(op->base.remap.unmap->va); 2320 2321 op->remap.start = xe_vma_start(old); 2322 op->remap.range = xe_vma_size(old); 2323 2324 if (op->base.remap.prev) { 2325 flags |= op->base.remap.unmap->va->flags & 2326 XE_VMA_READ_ONLY ? 2327 VMA_CREATE_FLAG_READ_ONLY : 0; 2328 flags |= op->base.remap.unmap->va->flags & 2329 DRM_GPUVA_SPARSE ? 2330 VMA_CREATE_FLAG_IS_NULL : 0; 2331 2332 vma = new_vma(vm, op->base.remap.prev, 2333 old->pat_index, flags); 2334 if (IS_ERR(vma)) 2335 return PTR_ERR(vma); 2336 2337 op->remap.prev = vma; 2338 2339 /* 2340 * Userptr creates a new SG mapping so 2341 * we must also rebind. 2342 */ 2343 op->remap.skip_prev = !xe_vma_is_userptr(old) && 2344 IS_ALIGNED(xe_vma_end(vma), 2345 xe_vma_max_pte_size(old)); 2346 if (op->remap.skip_prev) { 2347 xe_vma_set_pte_size(vma, xe_vma_max_pte_size(old)); 2348 op->remap.range -= 2349 xe_vma_end(vma) - 2350 xe_vma_start(old); 2351 op->remap.start = xe_vma_end(vma); 2352 } 2353 } 2354 2355 if (op->base.remap.next) { 2356 flags |= op->base.remap.unmap->va->flags & 2357 XE_VMA_READ_ONLY ? 2358 VMA_CREATE_FLAG_READ_ONLY : 0; 2359 flags |= op->base.remap.unmap->va->flags & 2360 DRM_GPUVA_SPARSE ? 2361 VMA_CREATE_FLAG_IS_NULL : 0; 2362 2363 vma = new_vma(vm, op->base.remap.next, 2364 old->pat_index, flags); 2365 if (IS_ERR(vma)) 2366 return PTR_ERR(vma); 2367 2368 op->remap.next = vma; 2369 2370 /* 2371 * Userptr creates a new SG mapping so 2372 * we must also rebind. 2373 */ 2374 op->remap.skip_next = !xe_vma_is_userptr(old) && 2375 IS_ALIGNED(xe_vma_start(vma), 2376 xe_vma_max_pte_size(old)); 2377 if (op->remap.skip_next) { 2378 xe_vma_set_pte_size(vma, xe_vma_max_pte_size(old)); 2379 op->remap.range -= 2380 xe_vma_end(old) - 2381 xe_vma_start(vma); 2382 } 2383 } 2384 break; 2385 } 2386 case DRM_GPUVA_OP_UNMAP: 2387 case DRM_GPUVA_OP_PREFETCH: 2388 /* Nothing to do */ 2389 break; 2390 default: 2391 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); 2392 } 2393 2394 last_op = op; 2395 2396 err = xe_vma_op_commit(vm, op); 2397 if (err) 2398 return err; 2399 } 2400 2401 /* FIXME: Unhandled corner case */ 2402 XE_WARN_ON(!last_op && last && !list_empty(ops_list)); 2403 2404 if (!last_op) 2405 return 0; 2406 2407 last_op->ops = ops; 2408 if (last) { 2409 last_op->flags |= XE_VMA_OP_LAST; 2410 last_op->num_syncs = num_syncs; 2411 last_op->syncs = syncs; 2412 } 2413 2414 return 0; 2415 } 2416 2417 static int op_execute(struct drm_exec *exec, struct xe_vm *vm, 2418 struct xe_vma *vma, struct xe_vma_op *op) 2419 { 2420 int err; 2421 2422 lockdep_assert_held_write(&vm->lock); 2423 2424 err = xe_vm_prepare_vma(exec, vma, 1); 2425 if (err) 2426 return err; 2427 2428 xe_vm_assert_held(vm); 2429 xe_bo_assert_held(xe_vma_bo(vma)); 2430 2431 switch (op->base.op) { 2432 case DRM_GPUVA_OP_MAP: 2433 err = xe_vm_bind(vm, vma, op->q, xe_vma_bo(vma), 2434 op->syncs, op->num_syncs, 2435 op->map.immediate || !xe_vm_in_fault_mode(vm), 2436 op->flags & XE_VMA_OP_FIRST, 2437 op->flags & XE_VMA_OP_LAST); 2438 break; 2439 case DRM_GPUVA_OP_REMAP: 2440 { 2441 bool prev = !!op->remap.prev; 2442 bool next = !!op->remap.next; 2443 2444 if (!op->remap.unmap_done) { 2445 if (prev || next) 2446 vma->gpuva.flags |= XE_VMA_FIRST_REBIND; 2447 err = xe_vm_unbind(vm, vma, op->q, op->syncs, 2448 op->num_syncs, 2449 op->flags & XE_VMA_OP_FIRST, 2450 op->flags & XE_VMA_OP_LAST && 2451 !prev && !next); 2452 if (err) 2453 break; 2454 op->remap.unmap_done = true; 2455 } 2456 2457 if (prev) { 2458 op->remap.prev->gpuva.flags |= XE_VMA_LAST_REBIND; 2459 err = xe_vm_bind(vm, op->remap.prev, op->q, 2460 xe_vma_bo(op->remap.prev), op->syncs, 2461 op->num_syncs, true, false, 2462 op->flags & XE_VMA_OP_LAST && !next); 2463 op->remap.prev->gpuva.flags &= ~XE_VMA_LAST_REBIND; 2464 if (err) 2465 break; 2466 op->remap.prev = NULL; 2467 } 2468 2469 if (next) { 2470 op->remap.next->gpuva.flags |= XE_VMA_LAST_REBIND; 2471 err = xe_vm_bind(vm, op->remap.next, op->q, 2472 xe_vma_bo(op->remap.next), 2473 op->syncs, op->num_syncs, 2474 true, false, 2475 op->flags & XE_VMA_OP_LAST); 2476 op->remap.next->gpuva.flags &= ~XE_VMA_LAST_REBIND; 2477 if (err) 2478 break; 2479 op->remap.next = NULL; 2480 } 2481 2482 break; 2483 } 2484 case DRM_GPUVA_OP_UNMAP: 2485 err = xe_vm_unbind(vm, vma, op->q, op->syncs, 2486 op->num_syncs, op->flags & XE_VMA_OP_FIRST, 2487 op->flags & XE_VMA_OP_LAST); 2488 break; 2489 case DRM_GPUVA_OP_PREFETCH: 2490 err = xe_vm_prefetch(vm, vma, op->q, op->prefetch.region, 2491 op->syncs, op->num_syncs, 2492 op->flags & XE_VMA_OP_FIRST, 2493 op->flags & XE_VMA_OP_LAST); 2494 break; 2495 default: 2496 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); 2497 } 2498 2499 if (err) 2500 trace_xe_vma_fail(vma); 2501 2502 return err; 2503 } 2504 2505 static int __xe_vma_op_execute(struct xe_vm *vm, struct xe_vma *vma, 2506 struct xe_vma_op *op) 2507 { 2508 struct drm_exec exec; 2509 int err; 2510 2511 retry_userptr: 2512 drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0); 2513 drm_exec_until_all_locked(&exec) { 2514 err = op_execute(&exec, vm, vma, op); 2515 drm_exec_retry_on_contention(&exec); 2516 if (err) 2517 break; 2518 } 2519 drm_exec_fini(&exec); 2520 2521 if (err == -EAGAIN) { 2522 lockdep_assert_held_write(&vm->lock); 2523 2524 if (op->base.op == DRM_GPUVA_OP_REMAP) { 2525 if (!op->remap.unmap_done) 2526 vma = gpuva_to_vma(op->base.remap.unmap->va); 2527 else if (op->remap.prev) 2528 vma = op->remap.prev; 2529 else 2530 vma = op->remap.next; 2531 } 2532 2533 if (xe_vma_is_userptr(vma)) { 2534 err = xe_vma_userptr_pin_pages(to_userptr_vma(vma)); 2535 if (!err) 2536 goto retry_userptr; 2537 2538 trace_xe_vma_fail(vma); 2539 } 2540 } 2541 2542 return err; 2543 } 2544 2545 static int xe_vma_op_execute(struct xe_vm *vm, struct xe_vma_op *op) 2546 { 2547 int ret = 0; 2548 2549 lockdep_assert_held_write(&vm->lock); 2550 2551 switch (op->base.op) { 2552 case DRM_GPUVA_OP_MAP: 2553 ret = __xe_vma_op_execute(vm, op->map.vma, op); 2554 break; 2555 case DRM_GPUVA_OP_REMAP: 2556 { 2557 struct xe_vma *vma; 2558 2559 if (!op->remap.unmap_done) 2560 vma = gpuva_to_vma(op->base.remap.unmap->va); 2561 else if (op->remap.prev) 2562 vma = op->remap.prev; 2563 else 2564 vma = op->remap.next; 2565 2566 ret = __xe_vma_op_execute(vm, vma, op); 2567 break; 2568 } 2569 case DRM_GPUVA_OP_UNMAP: 2570 ret = __xe_vma_op_execute(vm, gpuva_to_vma(op->base.unmap.va), 2571 op); 2572 break; 2573 case DRM_GPUVA_OP_PREFETCH: 2574 ret = __xe_vma_op_execute(vm, 2575 gpuva_to_vma(op->base.prefetch.va), 2576 op); 2577 break; 2578 default: 2579 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); 2580 } 2581 2582 return ret; 2583 } 2584 2585 static void xe_vma_op_cleanup(struct xe_vm *vm, struct xe_vma_op *op) 2586 { 2587 bool last = op->flags & XE_VMA_OP_LAST; 2588 2589 if (last) { 2590 while (op->num_syncs--) 2591 xe_sync_entry_cleanup(&op->syncs[op->num_syncs]); 2592 kfree(op->syncs); 2593 if (op->q) 2594 xe_exec_queue_put(op->q); 2595 } 2596 if (!list_empty(&op->link)) 2597 list_del(&op->link); 2598 if (op->ops) 2599 drm_gpuva_ops_free(&vm->gpuvm, op->ops); 2600 if (last) 2601 xe_vm_put(vm); 2602 } 2603 2604 static void xe_vma_op_unwind(struct xe_vm *vm, struct xe_vma_op *op, 2605 bool post_commit, bool prev_post_commit, 2606 bool next_post_commit) 2607 { 2608 lockdep_assert_held_write(&vm->lock); 2609 2610 switch (op->base.op) { 2611 case DRM_GPUVA_OP_MAP: 2612 if (op->map.vma) { 2613 prep_vma_destroy(vm, op->map.vma, post_commit); 2614 xe_vma_destroy_unlocked(op->map.vma); 2615 } 2616 break; 2617 case DRM_GPUVA_OP_UNMAP: 2618 { 2619 struct xe_vma *vma = gpuva_to_vma(op->base.unmap.va); 2620 2621 if (vma) { 2622 down_read(&vm->userptr.notifier_lock); 2623 vma->gpuva.flags &= ~XE_VMA_DESTROYED; 2624 up_read(&vm->userptr.notifier_lock); 2625 if (post_commit) 2626 xe_vm_insert_vma(vm, vma); 2627 } 2628 break; 2629 } 2630 case DRM_GPUVA_OP_REMAP: 2631 { 2632 struct xe_vma *vma = gpuva_to_vma(op->base.remap.unmap->va); 2633 2634 if (op->remap.prev) { 2635 prep_vma_destroy(vm, op->remap.prev, prev_post_commit); 2636 xe_vma_destroy_unlocked(op->remap.prev); 2637 } 2638 if (op->remap.next) { 2639 prep_vma_destroy(vm, op->remap.next, next_post_commit); 2640 xe_vma_destroy_unlocked(op->remap.next); 2641 } 2642 if (vma) { 2643 down_read(&vm->userptr.notifier_lock); 2644 vma->gpuva.flags &= ~XE_VMA_DESTROYED; 2645 up_read(&vm->userptr.notifier_lock); 2646 if (post_commit) 2647 xe_vm_insert_vma(vm, vma); 2648 } 2649 break; 2650 } 2651 case DRM_GPUVA_OP_PREFETCH: 2652 /* Nothing to do */ 2653 break; 2654 default: 2655 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); 2656 } 2657 } 2658 2659 static void vm_bind_ioctl_ops_unwind(struct xe_vm *vm, 2660 struct drm_gpuva_ops **ops, 2661 int num_ops_list) 2662 { 2663 int i; 2664 2665 for (i = num_ops_list - 1; i >= 0; --i) { 2666 struct drm_gpuva_ops *__ops = ops[i]; 2667 struct drm_gpuva_op *__op; 2668 2669 if (!__ops) 2670 continue; 2671 2672 drm_gpuva_for_each_op_reverse(__op, __ops) { 2673 struct xe_vma_op *op = gpuva_op_to_vma_op(__op); 2674 2675 xe_vma_op_unwind(vm, op, 2676 op->flags & XE_VMA_OP_COMMITTED, 2677 op->flags & XE_VMA_OP_PREV_COMMITTED, 2678 op->flags & XE_VMA_OP_NEXT_COMMITTED); 2679 } 2680 2681 drm_gpuva_ops_free(&vm->gpuvm, __ops); 2682 } 2683 } 2684 2685 static int vm_bind_ioctl_ops_execute(struct xe_vm *vm, 2686 struct list_head *ops_list) 2687 { 2688 struct xe_vma_op *op, *next; 2689 int err; 2690 2691 lockdep_assert_held_write(&vm->lock); 2692 2693 list_for_each_entry_safe(op, next, ops_list, link) { 2694 err = xe_vma_op_execute(vm, op); 2695 if (err) { 2696 drm_warn(&vm->xe->drm, "VM op(%d) failed with %d", 2697 op->base.op, err); 2698 /* 2699 * FIXME: Killing VM rather than proper error handling 2700 */ 2701 xe_vm_kill(vm); 2702 return -ENOSPC; 2703 } 2704 xe_vma_op_cleanup(vm, op); 2705 } 2706 2707 return 0; 2708 } 2709 2710 #define SUPPORTED_FLAGS \ 2711 (DRM_XE_VM_BIND_FLAG_READONLY | \ 2712 DRM_XE_VM_BIND_FLAG_IMMEDIATE | DRM_XE_VM_BIND_FLAG_NULL) 2713 #define XE_64K_PAGE_MASK 0xffffull 2714 #define ALL_DRM_XE_SYNCS_FLAGS (DRM_XE_SYNCS_FLAG_WAIT_FOR_OP) 2715 2716 #define MAX_BINDS 512 /* FIXME: Picking random upper limit */ 2717 2718 static int vm_bind_ioctl_check_args(struct xe_device *xe, 2719 struct drm_xe_vm_bind *args, 2720 struct drm_xe_vm_bind_op **bind_ops) 2721 { 2722 int err; 2723 int i; 2724 2725 if (XE_IOCTL_DBG(xe, args->pad || args->pad2) || 2726 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 2727 return -EINVAL; 2728 2729 if (XE_IOCTL_DBG(xe, args->extensions) || 2730 XE_IOCTL_DBG(xe, args->num_binds > MAX_BINDS)) 2731 return -EINVAL; 2732 2733 if (args->num_binds > 1) { 2734 u64 __user *bind_user = 2735 u64_to_user_ptr(args->vector_of_binds); 2736 2737 *bind_ops = kmalloc(sizeof(struct drm_xe_vm_bind_op) * 2738 args->num_binds, GFP_KERNEL); 2739 if (!*bind_ops) 2740 return -ENOMEM; 2741 2742 err = __copy_from_user(*bind_ops, bind_user, 2743 sizeof(struct drm_xe_vm_bind_op) * 2744 args->num_binds); 2745 if (XE_IOCTL_DBG(xe, err)) { 2746 err = -EFAULT; 2747 goto free_bind_ops; 2748 } 2749 } else { 2750 *bind_ops = &args->bind; 2751 } 2752 2753 for (i = 0; i < args->num_binds; ++i) { 2754 u64 range = (*bind_ops)[i].range; 2755 u64 addr = (*bind_ops)[i].addr; 2756 u32 op = (*bind_ops)[i].op; 2757 u32 flags = (*bind_ops)[i].flags; 2758 u32 obj = (*bind_ops)[i].obj; 2759 u64 obj_offset = (*bind_ops)[i].obj_offset; 2760 u32 prefetch_region = (*bind_ops)[i].prefetch_mem_region_instance; 2761 bool is_null = flags & DRM_XE_VM_BIND_FLAG_NULL; 2762 u16 pat_index = (*bind_ops)[i].pat_index; 2763 u16 coh_mode; 2764 2765 if (XE_IOCTL_DBG(xe, pat_index >= xe->pat.n_entries)) { 2766 err = -EINVAL; 2767 goto free_bind_ops; 2768 } 2769 2770 pat_index = array_index_nospec(pat_index, xe->pat.n_entries); 2771 (*bind_ops)[i].pat_index = pat_index; 2772 coh_mode = xe_pat_index_get_coh_mode(xe, pat_index); 2773 if (XE_IOCTL_DBG(xe, !coh_mode)) { /* hw reserved */ 2774 err = -EINVAL; 2775 goto free_bind_ops; 2776 } 2777 2778 if (XE_WARN_ON(coh_mode > XE_COH_AT_LEAST_1WAY)) { 2779 err = -EINVAL; 2780 goto free_bind_ops; 2781 } 2782 2783 if (XE_IOCTL_DBG(xe, op > DRM_XE_VM_BIND_OP_PREFETCH) || 2784 XE_IOCTL_DBG(xe, flags & ~SUPPORTED_FLAGS) || 2785 XE_IOCTL_DBG(xe, obj && is_null) || 2786 XE_IOCTL_DBG(xe, obj_offset && is_null) || 2787 XE_IOCTL_DBG(xe, op != DRM_XE_VM_BIND_OP_MAP && 2788 is_null) || 2789 XE_IOCTL_DBG(xe, !obj && 2790 op == DRM_XE_VM_BIND_OP_MAP && 2791 !is_null) || 2792 XE_IOCTL_DBG(xe, !obj && 2793 op == DRM_XE_VM_BIND_OP_UNMAP_ALL) || 2794 XE_IOCTL_DBG(xe, addr && 2795 op == DRM_XE_VM_BIND_OP_UNMAP_ALL) || 2796 XE_IOCTL_DBG(xe, range && 2797 op == DRM_XE_VM_BIND_OP_UNMAP_ALL) || 2798 XE_IOCTL_DBG(xe, obj && 2799 op == DRM_XE_VM_BIND_OP_MAP_USERPTR) || 2800 XE_IOCTL_DBG(xe, coh_mode == XE_COH_NONE && 2801 op == DRM_XE_VM_BIND_OP_MAP_USERPTR) || 2802 XE_IOCTL_DBG(xe, obj && 2803 op == DRM_XE_VM_BIND_OP_PREFETCH) || 2804 XE_IOCTL_DBG(xe, prefetch_region && 2805 op != DRM_XE_VM_BIND_OP_PREFETCH) || 2806 XE_IOCTL_DBG(xe, !(BIT(prefetch_region) & 2807 xe->info.mem_region_mask)) || 2808 XE_IOCTL_DBG(xe, obj && 2809 op == DRM_XE_VM_BIND_OP_UNMAP)) { 2810 err = -EINVAL; 2811 goto free_bind_ops; 2812 } 2813 2814 if (XE_IOCTL_DBG(xe, obj_offset & ~PAGE_MASK) || 2815 XE_IOCTL_DBG(xe, addr & ~PAGE_MASK) || 2816 XE_IOCTL_DBG(xe, range & ~PAGE_MASK) || 2817 XE_IOCTL_DBG(xe, !range && 2818 op != DRM_XE_VM_BIND_OP_UNMAP_ALL)) { 2819 err = -EINVAL; 2820 goto free_bind_ops; 2821 } 2822 } 2823 2824 return 0; 2825 2826 free_bind_ops: 2827 if (args->num_binds > 1) 2828 kfree(*bind_ops); 2829 return err; 2830 } 2831 2832 static int vm_bind_ioctl_signal_fences(struct xe_vm *vm, 2833 struct xe_exec_queue *q, 2834 struct xe_sync_entry *syncs, 2835 int num_syncs) 2836 { 2837 struct dma_fence *fence; 2838 int i, err = 0; 2839 2840 fence = xe_sync_in_fence_get(syncs, num_syncs, 2841 to_wait_exec_queue(vm, q), vm); 2842 if (IS_ERR(fence)) 2843 return PTR_ERR(fence); 2844 2845 for (i = 0; i < num_syncs; i++) 2846 xe_sync_entry_signal(&syncs[i], NULL, fence); 2847 2848 xe_exec_queue_last_fence_set(to_wait_exec_queue(vm, q), vm, 2849 fence); 2850 dma_fence_put(fence); 2851 2852 return err; 2853 } 2854 2855 int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 2856 { 2857 struct xe_device *xe = to_xe_device(dev); 2858 struct xe_file *xef = to_xe_file(file); 2859 struct drm_xe_vm_bind *args = data; 2860 struct drm_xe_sync __user *syncs_user; 2861 struct xe_bo **bos = NULL; 2862 struct drm_gpuva_ops **ops = NULL; 2863 struct xe_vm *vm; 2864 struct xe_exec_queue *q = NULL; 2865 u32 num_syncs, num_ufence = 0; 2866 struct xe_sync_entry *syncs = NULL; 2867 struct drm_xe_vm_bind_op *bind_ops; 2868 LIST_HEAD(ops_list); 2869 int err; 2870 int i; 2871 2872 err = vm_bind_ioctl_check_args(xe, args, &bind_ops); 2873 if (err) 2874 return err; 2875 2876 if (args->exec_queue_id) { 2877 q = xe_exec_queue_lookup(xef, args->exec_queue_id); 2878 if (XE_IOCTL_DBG(xe, !q)) { 2879 err = -ENOENT; 2880 goto free_objs; 2881 } 2882 2883 if (XE_IOCTL_DBG(xe, !(q->flags & EXEC_QUEUE_FLAG_VM))) { 2884 err = -EINVAL; 2885 goto put_exec_queue; 2886 } 2887 } 2888 2889 vm = xe_vm_lookup(xef, args->vm_id); 2890 if (XE_IOCTL_DBG(xe, !vm)) { 2891 err = -EINVAL; 2892 goto put_exec_queue; 2893 } 2894 2895 err = down_write_killable(&vm->lock); 2896 if (err) 2897 goto put_vm; 2898 2899 if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) { 2900 err = -ENOENT; 2901 goto release_vm_lock; 2902 } 2903 2904 for (i = 0; i < args->num_binds; ++i) { 2905 u64 range = bind_ops[i].range; 2906 u64 addr = bind_ops[i].addr; 2907 2908 if (XE_IOCTL_DBG(xe, range > vm->size) || 2909 XE_IOCTL_DBG(xe, addr > vm->size - range)) { 2910 err = -EINVAL; 2911 goto release_vm_lock; 2912 } 2913 } 2914 2915 if (args->num_binds) { 2916 bos = kcalloc(args->num_binds, sizeof(*bos), GFP_KERNEL); 2917 if (!bos) { 2918 err = -ENOMEM; 2919 goto release_vm_lock; 2920 } 2921 2922 ops = kcalloc(args->num_binds, sizeof(*ops), GFP_KERNEL); 2923 if (!ops) { 2924 err = -ENOMEM; 2925 goto release_vm_lock; 2926 } 2927 } 2928 2929 for (i = 0; i < args->num_binds; ++i) { 2930 struct drm_gem_object *gem_obj; 2931 u64 range = bind_ops[i].range; 2932 u64 addr = bind_ops[i].addr; 2933 u32 obj = bind_ops[i].obj; 2934 u64 obj_offset = bind_ops[i].obj_offset; 2935 u16 pat_index = bind_ops[i].pat_index; 2936 u16 coh_mode; 2937 2938 if (!obj) 2939 continue; 2940 2941 gem_obj = drm_gem_object_lookup(file, obj); 2942 if (XE_IOCTL_DBG(xe, !gem_obj)) { 2943 err = -ENOENT; 2944 goto put_obj; 2945 } 2946 bos[i] = gem_to_xe_bo(gem_obj); 2947 2948 if (XE_IOCTL_DBG(xe, range > bos[i]->size) || 2949 XE_IOCTL_DBG(xe, obj_offset > 2950 bos[i]->size - range)) { 2951 err = -EINVAL; 2952 goto put_obj; 2953 } 2954 2955 if (bos[i]->flags & XE_BO_INTERNAL_64K) { 2956 if (XE_IOCTL_DBG(xe, obj_offset & 2957 XE_64K_PAGE_MASK) || 2958 XE_IOCTL_DBG(xe, addr & XE_64K_PAGE_MASK) || 2959 XE_IOCTL_DBG(xe, range & XE_64K_PAGE_MASK)) { 2960 err = -EINVAL; 2961 goto put_obj; 2962 } 2963 } 2964 2965 coh_mode = xe_pat_index_get_coh_mode(xe, pat_index); 2966 if (bos[i]->cpu_caching) { 2967 if (XE_IOCTL_DBG(xe, coh_mode == XE_COH_NONE && 2968 bos[i]->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB)) { 2969 err = -EINVAL; 2970 goto put_obj; 2971 } 2972 } else if (XE_IOCTL_DBG(xe, coh_mode == XE_COH_NONE)) { 2973 /* 2974 * Imported dma-buf from a different device should 2975 * require 1way or 2way coherency since we don't know 2976 * how it was mapped on the CPU. Just assume is it 2977 * potentially cached on CPU side. 2978 */ 2979 err = -EINVAL; 2980 goto put_obj; 2981 } 2982 } 2983 2984 if (args->num_syncs) { 2985 syncs = kcalloc(args->num_syncs, sizeof(*syncs), GFP_KERNEL); 2986 if (!syncs) { 2987 err = -ENOMEM; 2988 goto put_obj; 2989 } 2990 } 2991 2992 syncs_user = u64_to_user_ptr(args->syncs); 2993 for (num_syncs = 0; num_syncs < args->num_syncs; num_syncs++) { 2994 err = xe_sync_entry_parse(xe, xef, &syncs[num_syncs], 2995 &syncs_user[num_syncs], 2996 (xe_vm_in_lr_mode(vm) ? 2997 SYNC_PARSE_FLAG_LR_MODE : 0) | 2998 (!args->num_binds ? 2999 SYNC_PARSE_FLAG_DISALLOW_USER_FENCE : 0)); 3000 if (err) 3001 goto free_syncs; 3002 3003 if (xe_sync_is_ufence(&syncs[num_syncs])) 3004 num_ufence++; 3005 } 3006 3007 if (XE_IOCTL_DBG(xe, num_ufence > 1)) { 3008 err = -EINVAL; 3009 goto free_syncs; 3010 } 3011 3012 if (!args->num_binds) { 3013 err = -ENODATA; 3014 goto free_syncs; 3015 } 3016 3017 for (i = 0; i < args->num_binds; ++i) { 3018 u64 range = bind_ops[i].range; 3019 u64 addr = bind_ops[i].addr; 3020 u32 op = bind_ops[i].op; 3021 u32 flags = bind_ops[i].flags; 3022 u64 obj_offset = bind_ops[i].obj_offset; 3023 u32 prefetch_region = bind_ops[i].prefetch_mem_region_instance; 3024 u16 pat_index = bind_ops[i].pat_index; 3025 3026 ops[i] = vm_bind_ioctl_ops_create(vm, bos[i], obj_offset, 3027 addr, range, op, flags, 3028 prefetch_region, pat_index); 3029 if (IS_ERR(ops[i])) { 3030 err = PTR_ERR(ops[i]); 3031 ops[i] = NULL; 3032 goto unwind_ops; 3033 } 3034 3035 err = vm_bind_ioctl_ops_parse(vm, q, ops[i], syncs, num_syncs, 3036 &ops_list, 3037 i == args->num_binds - 1); 3038 if (err) 3039 goto unwind_ops; 3040 } 3041 3042 /* Nothing to do */ 3043 if (list_empty(&ops_list)) { 3044 err = -ENODATA; 3045 goto unwind_ops; 3046 } 3047 3048 xe_vm_get(vm); 3049 if (q) 3050 xe_exec_queue_get(q); 3051 3052 err = vm_bind_ioctl_ops_execute(vm, &ops_list); 3053 3054 up_write(&vm->lock); 3055 3056 if (q) 3057 xe_exec_queue_put(q); 3058 xe_vm_put(vm); 3059 3060 for (i = 0; bos && i < args->num_binds; ++i) 3061 xe_bo_put(bos[i]); 3062 3063 kfree(bos); 3064 kfree(ops); 3065 if (args->num_binds > 1) 3066 kfree(bind_ops); 3067 3068 return err; 3069 3070 unwind_ops: 3071 vm_bind_ioctl_ops_unwind(vm, ops, args->num_binds); 3072 free_syncs: 3073 if (err == -ENODATA) 3074 err = vm_bind_ioctl_signal_fences(vm, q, syncs, num_syncs); 3075 while (num_syncs--) 3076 xe_sync_entry_cleanup(&syncs[num_syncs]); 3077 3078 kfree(syncs); 3079 put_obj: 3080 for (i = 0; i < args->num_binds; ++i) 3081 xe_bo_put(bos[i]); 3082 release_vm_lock: 3083 up_write(&vm->lock); 3084 put_vm: 3085 xe_vm_put(vm); 3086 put_exec_queue: 3087 if (q) 3088 xe_exec_queue_put(q); 3089 free_objs: 3090 kfree(bos); 3091 kfree(ops); 3092 if (args->num_binds > 1) 3093 kfree(bind_ops); 3094 return err; 3095 } 3096 3097 /** 3098 * xe_vm_lock() - Lock the vm's dma_resv object 3099 * @vm: The struct xe_vm whose lock is to be locked 3100 * @intr: Whether to perform any wait interruptible 3101 * 3102 * Return: 0 on success, -EINTR if @intr is true and the wait for a 3103 * contended lock was interrupted. If @intr is false, the function 3104 * always returns 0. 3105 */ 3106 int xe_vm_lock(struct xe_vm *vm, bool intr) 3107 { 3108 if (intr) 3109 return dma_resv_lock_interruptible(xe_vm_resv(vm), NULL); 3110 3111 return dma_resv_lock(xe_vm_resv(vm), NULL); 3112 } 3113 3114 /** 3115 * xe_vm_unlock() - Unlock the vm's dma_resv object 3116 * @vm: The struct xe_vm whose lock is to be released. 3117 * 3118 * Unlock a buffer object lock that was locked by xe_vm_lock(). 3119 */ 3120 void xe_vm_unlock(struct xe_vm *vm) 3121 { 3122 dma_resv_unlock(xe_vm_resv(vm)); 3123 } 3124 3125 /** 3126 * xe_vm_invalidate_vma - invalidate GPU mappings for VMA without a lock 3127 * @vma: VMA to invalidate 3128 * 3129 * Walks a list of page tables leaves which it memset the entries owned by this 3130 * VMA to zero, invalidates the TLBs, and block until TLBs invalidation is 3131 * complete. 3132 * 3133 * Returns 0 for success, negative error code otherwise. 3134 */ 3135 int xe_vm_invalidate_vma(struct xe_vma *vma) 3136 { 3137 struct xe_device *xe = xe_vma_vm(vma)->xe; 3138 struct xe_tile *tile; 3139 u32 tile_needs_invalidate = 0; 3140 int seqno[XE_MAX_TILES_PER_DEVICE]; 3141 u8 id; 3142 int ret; 3143 3144 xe_assert(xe, xe_vm_in_fault_mode(xe_vma_vm(vma))); 3145 xe_assert(xe, !xe_vma_is_null(vma)); 3146 trace_xe_vma_usm_invalidate(vma); 3147 3148 /* Check that we don't race with page-table updates */ 3149 if (IS_ENABLED(CONFIG_PROVE_LOCKING)) { 3150 if (xe_vma_is_userptr(vma)) { 3151 WARN_ON_ONCE(!mmu_interval_check_retry 3152 (&to_userptr_vma(vma)->userptr.notifier, 3153 to_userptr_vma(vma)->userptr.notifier_seq)); 3154 WARN_ON_ONCE(!dma_resv_test_signaled(xe_vm_resv(xe_vma_vm(vma)), 3155 DMA_RESV_USAGE_BOOKKEEP)); 3156 3157 } else { 3158 xe_bo_assert_held(xe_vma_bo(vma)); 3159 } 3160 } 3161 3162 for_each_tile(tile, xe, id) { 3163 if (xe_pt_zap_ptes(tile, vma)) { 3164 tile_needs_invalidate |= BIT(id); 3165 xe_device_wmb(xe); 3166 /* 3167 * FIXME: We potentially need to invalidate multiple 3168 * GTs within the tile 3169 */ 3170 seqno[id] = xe_gt_tlb_invalidation_vma(tile->primary_gt, NULL, vma); 3171 if (seqno[id] < 0) 3172 return seqno[id]; 3173 } 3174 } 3175 3176 for_each_tile(tile, xe, id) { 3177 if (tile_needs_invalidate & BIT(id)) { 3178 ret = xe_gt_tlb_invalidation_wait(tile->primary_gt, seqno[id]); 3179 if (ret < 0) 3180 return ret; 3181 } 3182 } 3183 3184 vma->usm.tile_invalidated = vma->tile_mask; 3185 3186 return 0; 3187 } 3188 3189 int xe_analyze_vm(struct drm_printer *p, struct xe_vm *vm, int gt_id) 3190 { 3191 struct drm_gpuva *gpuva; 3192 bool is_vram; 3193 uint64_t addr; 3194 3195 if (!down_read_trylock(&vm->lock)) { 3196 drm_printf(p, " Failed to acquire VM lock to dump capture"); 3197 return 0; 3198 } 3199 if (vm->pt_root[gt_id]) { 3200 addr = xe_bo_addr(vm->pt_root[gt_id]->bo, 0, XE_PAGE_SIZE); 3201 is_vram = xe_bo_is_vram(vm->pt_root[gt_id]->bo); 3202 drm_printf(p, " VM root: A:0x%llx %s\n", addr, 3203 is_vram ? "VRAM" : "SYS"); 3204 } 3205 3206 drm_gpuvm_for_each_va(gpuva, &vm->gpuvm) { 3207 struct xe_vma *vma = gpuva_to_vma(gpuva); 3208 bool is_userptr = xe_vma_is_userptr(vma); 3209 bool is_null = xe_vma_is_null(vma); 3210 3211 if (is_null) { 3212 addr = 0; 3213 } else if (is_userptr) { 3214 struct sg_table *sg = to_userptr_vma(vma)->userptr.sg; 3215 struct xe_res_cursor cur; 3216 3217 if (sg) { 3218 xe_res_first_sg(sg, 0, XE_PAGE_SIZE, &cur); 3219 addr = xe_res_dma(&cur); 3220 } else { 3221 addr = 0; 3222 } 3223 } else { 3224 addr = __xe_bo_addr(xe_vma_bo(vma), 0, XE_PAGE_SIZE); 3225 is_vram = xe_bo_is_vram(xe_vma_bo(vma)); 3226 } 3227 drm_printf(p, " [%016llx-%016llx] S:0x%016llx A:%016llx %s\n", 3228 xe_vma_start(vma), xe_vma_end(vma) - 1, 3229 xe_vma_size(vma), 3230 addr, is_null ? "NULL" : is_userptr ? "USR" : 3231 is_vram ? "VRAM" : "SYS"); 3232 } 3233 up_read(&vm->lock); 3234 3235 return 0; 3236 } 3237