1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_vm.h" 7 8 #include <linux/dma-fence-array.h> 9 #include <linux/nospec.h> 10 11 #include <drm/drm_drv.h> 12 #include <drm/drm_exec.h> 13 #include <drm/drm_print.h> 14 #include <drm/ttm/ttm_tt.h> 15 #include <uapi/drm/xe_drm.h> 16 #include <linux/ascii85.h> 17 #include <linux/delay.h> 18 #include <linux/kthread.h> 19 #include <linux/mm.h> 20 #include <linux/swap.h> 21 22 #include <generated/xe_wa_oob.h> 23 24 #include "regs/xe_gtt_defs.h" 25 #include "xe_assert.h" 26 #include "xe_bo.h" 27 #include "xe_device.h" 28 #include "xe_drm_client.h" 29 #include "xe_exec_queue.h" 30 #include "xe_gt.h" 31 #include "xe_migrate.h" 32 #include "xe_pat.h" 33 #include "xe_pm.h" 34 #include "xe_preempt_fence.h" 35 #include "xe_pt.h" 36 #include "xe_pxp.h" 37 #include "xe_sriov_vf.h" 38 #include "xe_svm.h" 39 #include "xe_sync.h" 40 #include "xe_tile.h" 41 #include "xe_tlb_inval.h" 42 #include "xe_trace_bo.h" 43 #include "xe_vm_madvise.h" 44 #include "xe_wa.h" 45 46 static struct drm_gem_object *xe_vm_obj(struct xe_vm *vm) 47 { 48 return vm->gpuvm.r_obj; 49 } 50 51 /** 52 * xe_vm_drm_exec_lock() - Lock the vm's resv with a drm_exec transaction 53 * @vm: The vm whose resv is to be locked. 54 * @exec: The drm_exec transaction. 55 * 56 * Helper to lock the vm's resv as part of a drm_exec transaction. 57 * 58 * Return: %0 on success. See drm_exec_lock_obj() for error codes. 59 */ 60 int xe_vm_drm_exec_lock(struct xe_vm *vm, struct drm_exec *exec) 61 { 62 return drm_exec_lock_obj(exec, xe_vm_obj(vm)); 63 } 64 65 static bool preempt_fences_waiting(struct xe_vm *vm) 66 { 67 struct xe_exec_queue *q; 68 69 lockdep_assert_held(&vm->lock); 70 xe_vm_assert_held(vm); 71 72 list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) { 73 if (!q->lr.pfence || 74 test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, 75 &q->lr.pfence->flags)) { 76 return true; 77 } 78 } 79 80 return false; 81 } 82 83 static void free_preempt_fences(struct list_head *list) 84 { 85 struct list_head *link, *next; 86 87 list_for_each_safe(link, next, list) 88 xe_preempt_fence_free(to_preempt_fence_from_link(link)); 89 } 90 91 static int alloc_preempt_fences(struct xe_vm *vm, struct list_head *list, 92 unsigned int *count) 93 { 94 lockdep_assert_held(&vm->lock); 95 xe_vm_assert_held(vm); 96 97 if (*count >= vm->preempt.num_exec_queues) 98 return 0; 99 100 for (; *count < vm->preempt.num_exec_queues; ++(*count)) { 101 struct xe_preempt_fence *pfence = xe_preempt_fence_alloc(); 102 103 if (IS_ERR(pfence)) 104 return PTR_ERR(pfence); 105 106 list_move_tail(xe_preempt_fence_link(pfence), list); 107 } 108 109 return 0; 110 } 111 112 static int wait_for_existing_preempt_fences(struct xe_vm *vm) 113 { 114 struct xe_exec_queue *q; 115 bool vf_migration = IS_SRIOV_VF(vm->xe) && 116 xe_sriov_vf_migration_supported(vm->xe); 117 signed long wait_time = vf_migration ? HZ / 5 : MAX_SCHEDULE_TIMEOUT; 118 119 xe_vm_assert_held(vm); 120 121 list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) { 122 if (q->lr.pfence) { 123 long timeout; 124 125 timeout = dma_fence_wait_timeout(q->lr.pfence, false, 126 wait_time); 127 if (!timeout) { 128 xe_assert(vm->xe, vf_migration); 129 return -EAGAIN; 130 } 131 132 /* Only -ETIME on fence indicates VM needs to be killed */ 133 if (timeout < 0 || q->lr.pfence->error == -ETIME) 134 return -ETIME; 135 136 dma_fence_put(q->lr.pfence); 137 q->lr.pfence = NULL; 138 } 139 } 140 141 return 0; 142 } 143 144 static bool xe_vm_is_idle(struct xe_vm *vm) 145 { 146 struct xe_exec_queue *q; 147 148 xe_vm_assert_held(vm); 149 list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) { 150 if (!xe_exec_queue_is_idle(q)) 151 return false; 152 } 153 154 return true; 155 } 156 157 static void arm_preempt_fences(struct xe_vm *vm, struct list_head *list) 158 { 159 struct list_head *link; 160 struct xe_exec_queue *q; 161 162 list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) { 163 struct dma_fence *fence; 164 165 link = list->next; 166 xe_assert(vm->xe, link != list); 167 168 fence = xe_preempt_fence_arm(to_preempt_fence_from_link(link), 169 q, q->lr.context, 170 ++q->lr.seqno); 171 dma_fence_put(q->lr.pfence); 172 q->lr.pfence = fence; 173 } 174 } 175 176 static int add_preempt_fences(struct xe_vm *vm, struct xe_bo *bo) 177 { 178 struct xe_exec_queue *q; 179 int err; 180 181 xe_bo_assert_held(bo); 182 183 if (!vm->preempt.num_exec_queues) 184 return 0; 185 186 err = dma_resv_reserve_fences(bo->ttm.base.resv, vm->preempt.num_exec_queues); 187 if (err) 188 return err; 189 190 list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) 191 if (q->lr.pfence) { 192 dma_resv_add_fence(bo->ttm.base.resv, 193 q->lr.pfence, 194 DMA_RESV_USAGE_BOOKKEEP); 195 } 196 197 return 0; 198 } 199 200 static void resume_and_reinstall_preempt_fences(struct xe_vm *vm, 201 struct drm_exec *exec) 202 { 203 struct xe_exec_queue *q; 204 205 lockdep_assert_held(&vm->lock); 206 xe_vm_assert_held(vm); 207 208 list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) { 209 /* 210 * Only resume queues whose suspend() actually succeeded. A 211 * failed suspend() (e.g. killed/banned/wedged) leaves the queue 212 * un-suspended, so it must not be resumed. 213 * 214 * Also skip queues that have since been reset/killed/banned/ 215 * wedged: their suspend may not have completed (suspend_pending 216 * can still be set, e.g. a preempt fence signalled with -ENOENT 217 * without waiting), so resuming would trip the !suspend_pending 218 * assert in the backend. Such queues are being torn down anyway, 219 * so leave them marked suspended and let teardown resolve their 220 * state. 221 */ 222 if (READ_ONCE(q->lr.suspended) && !q->ops->reset_status(q)) { 223 WRITE_ONCE(q->lr.suspended, false); 224 q->ops->resume(q); 225 } 226 227 drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, q->lr.pfence, 228 DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_BOOKKEEP); 229 } 230 } 231 232 int xe_vm_add_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q) 233 { 234 struct drm_gpuvm_exec vm_exec = { 235 .vm = &vm->gpuvm, 236 .flags = DRM_EXEC_INTERRUPTIBLE_WAIT, 237 .num_fences = 1, 238 }; 239 struct drm_exec *exec = &vm_exec.exec; 240 struct xe_validation_ctx ctx; 241 struct dma_fence *pfence; 242 int err; 243 bool wait; 244 245 xe_assert(vm->xe, xe_vm_in_preempt_fence_mode(vm)); 246 247 down_write(&vm->lock); 248 err = xe_validation_exec_lock(&ctx, &vm_exec, &vm->xe->val); 249 if (err) 250 goto out_up_write; 251 252 pfence = xe_preempt_fence_create(q, q->lr.context, 253 ++q->lr.seqno); 254 if (IS_ERR(pfence)) { 255 err = PTR_ERR(pfence); 256 goto out_fini; 257 } 258 259 list_add(&q->lr.link, &vm->preempt.exec_queues); 260 ++vm->preempt.num_exec_queues; 261 q->lr.pfence = pfence; 262 263 xe_svm_notifier_lock(vm); 264 265 drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, pfence, 266 DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_BOOKKEEP); 267 268 /* 269 * Check to see if a preemption on VM is in flight or userptr 270 * invalidation, if so trigger this preempt fence to sync state with 271 * other preempt fences on the VM. 272 */ 273 wait = __xe_vm_userptr_needs_repin(vm) || preempt_fences_waiting(vm); 274 if (wait) 275 dma_fence_enable_signaling(pfence); 276 277 xe_svm_notifier_unlock(vm); 278 279 out_fini: 280 xe_validation_ctx_fini(&ctx); 281 out_up_write: 282 up_write(&vm->lock); 283 284 return err; 285 } 286 ALLOW_ERROR_INJECTION(xe_vm_add_compute_exec_queue, ERRNO); 287 288 /** 289 * xe_vm_remove_compute_exec_queue() - Remove compute exec queue from VM 290 * @vm: The VM. 291 * @q: The exec_queue 292 * 293 * Note that this function might be called multiple times on the same queue. 294 */ 295 void xe_vm_remove_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q) 296 { 297 if (!xe_vm_in_preempt_fence_mode(vm)) 298 return; 299 300 down_write(&vm->lock); 301 if (!list_empty(&q->lr.link)) { 302 list_del_init(&q->lr.link); 303 --vm->preempt.num_exec_queues; 304 } 305 if (q->lr.pfence) { 306 dma_fence_enable_signaling(q->lr.pfence); 307 dma_fence_put(q->lr.pfence); 308 q->lr.pfence = NULL; 309 } 310 up_write(&vm->lock); 311 } 312 313 #define XE_VM_REBIND_RETRY_TIMEOUT_MS 1000 314 315 /** 316 * xe_vm_kill() - VM Kill 317 * @vm: The VM. 318 * @unlocked: Flag indicates the VM's dma-resv is not held 319 * 320 * Kill the VM by setting banned flag indicated VM is no longer available for 321 * use. If in preempt fence mode, also kill all exec queue attached to the VM. 322 */ 323 void xe_vm_kill(struct xe_vm *vm, bool unlocked) 324 { 325 struct xe_exec_queue *q; 326 327 lockdep_assert_held(&vm->lock); 328 329 if (unlocked) 330 xe_vm_lock(vm, false); 331 332 vm->flags |= XE_VM_FLAG_BANNED; 333 trace_xe_vm_kill(vm); 334 335 list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) 336 q->ops->kill(q); 337 338 if (unlocked) 339 xe_vm_unlock(vm); 340 341 /* TODO: Inform user the VM is banned */ 342 } 343 344 static int xe_gpuvm_validate(struct drm_gpuvm_bo *vm_bo, struct drm_exec *exec) 345 { 346 struct xe_vm *vm = gpuvm_to_vm(vm_bo->vm); 347 struct xe_bo *bo = gem_to_xe_bo(vm_bo->obj); 348 struct drm_gpuva *gpuva; 349 int ret; 350 351 lockdep_assert_held(&vm->lock); 352 drm_gpuvm_bo_for_each_va(gpuva, vm_bo) 353 list_move_tail(&gpuva_to_vma(gpuva)->combined_links.rebind, 354 &vm->rebind_list); 355 356 /* Skip re-populating purged BOs, rebind maps scratch pages. */ 357 if (xe_bo_is_purged(bo)) { 358 vm_bo->evicted = false; 359 return 0; 360 } 361 362 if (!try_wait_for_completion(&vm->xe->pm_block)) 363 return -EAGAIN; 364 365 ret = xe_bo_validate(bo, vm, false, exec); 366 if (ret) 367 return ret; 368 369 vm_bo->evicted = false; 370 return 0; 371 } 372 373 /** 374 * xe_vm_validate_rebind() - Validate buffer objects and rebind vmas 375 * @vm: The vm for which we are rebinding. 376 * @exec: The struct drm_exec with the locked GEM objects. 377 * @num_fences: The number of fences to reserve for the operation, not 378 * including rebinds and validations. 379 * 380 * Validates all evicted gem objects and rebinds their vmas. Note that 381 * rebindings may cause evictions and hence the validation-rebind 382 * sequence is rerun until there are no more objects to validate. 383 * 384 * Return: 0 on success, negative error code on error. In particular, 385 * may return -EINTR or -ERESTARTSYS if interrupted, and -EDEADLK if 386 * the drm_exec transaction needs to be restarted. 387 */ 388 int xe_vm_validate_rebind(struct xe_vm *vm, struct drm_exec *exec, 389 unsigned int num_fences) 390 { 391 struct drm_gem_object *obj; 392 int ret; 393 394 do { 395 ret = drm_gpuvm_validate(&vm->gpuvm, exec); 396 if (ret) 397 return ret; 398 399 ret = xe_vm_rebind(vm, false); 400 if (ret) 401 return ret; 402 } while (!list_empty(&vm->gpuvm.evict.list)); 403 404 drm_exec_for_each_locked_object(exec, obj) { 405 ret = dma_resv_reserve_fences(obj->resv, num_fences); 406 if (ret) 407 return ret; 408 } 409 410 return 0; 411 } 412 413 /** 414 * xe_vm_bump_lru() - Bump the LRU position of BOs used by a submission 415 * @vm: The vm for which we are bumping LRUs. 416 * @exec: The struct drm_exec with the locked GEM objects. 417 * 418 * Move the VM's private BOs, and all external objects locked by @exec, 419 * to the LRU tail. Private BOs are moved in bulk using @vm's 420 * &xe_vm.lru_bulk_move. External objects, which may be shared with other 421 * VMs, are not part of that bulk move and are bumped individually. 422 */ 423 void xe_vm_bump_lru(struct xe_vm *vm, struct drm_exec *exec) 424 { 425 spinlock_t *lru_lock = &vm->xe->ttm.lru_lock; 426 struct drm_gem_object *obj; 427 428 spin_lock(lru_lock); 429 ttm_lru_bulk_move_tail(&vm->lru_bulk_move); 430 drm_exec_for_each_locked_object(exec, obj) { 431 if (!drm_gpuvm_is_extobj(&vm->gpuvm, obj)) 432 continue; 433 434 ttm_bo_move_to_lru_tail(&gem_to_xe_bo(obj)->ttm); 435 cond_resched_lock(lru_lock); 436 } 437 spin_unlock(lru_lock); 438 } 439 440 static int xe_preempt_work_begin(struct drm_exec *exec, struct xe_vm *vm, 441 bool *done) 442 { 443 int err; 444 445 err = drm_gpuvm_prepare_vm(&vm->gpuvm, exec, 0); 446 if (err) 447 return err; 448 449 if (xe_vm_is_idle(vm)) { 450 vm->preempt.rebind_deactivated = true; 451 *done = true; 452 return 0; 453 } 454 455 if (!preempt_fences_waiting(vm)) { 456 *done = true; 457 return 0; 458 } 459 460 err = drm_gpuvm_prepare_objects(&vm->gpuvm, exec, 0); 461 if (err) 462 return err; 463 464 err = wait_for_existing_preempt_fences(vm); 465 if (err) 466 return err; 467 468 /* 469 * Add validation and rebinding to the locking loop since both can 470 * cause evictions which may require blocing dma_resv locks. 471 * The fence reservation here is intended for the new preempt fences 472 * we attach at the end of the rebind work. 473 */ 474 return xe_vm_validate_rebind(vm, exec, vm->preempt.num_exec_queues); 475 } 476 477 static bool vm_suspend_rebind_worker(struct xe_vm *vm) 478 { 479 struct xe_device *xe = vm->xe; 480 bool ret = false; 481 482 mutex_lock(&xe->rebind_resume_lock); 483 if (!try_wait_for_completion(&vm->xe->pm_block)) { 484 ret = true; 485 list_move_tail(&vm->preempt.pm_activate_link, &xe->rebind_resume_list); 486 } 487 mutex_unlock(&xe->rebind_resume_lock); 488 489 return ret; 490 } 491 492 /** 493 * xe_vm_resume_rebind_worker() - Resume the rebind worker. 494 * @vm: The vm whose preempt worker to resume. 495 * 496 * Resume a preempt worker that was previously suspended by 497 * vm_suspend_rebind_worker(). 498 */ 499 void xe_vm_resume_rebind_worker(struct xe_vm *vm) 500 { 501 queue_work(vm->xe->ordered_wq, &vm->preempt.rebind_work); 502 } 503 504 static void preempt_rebind_work_func(struct work_struct *w) 505 { 506 struct xe_vm *vm = container_of(w, struct xe_vm, preempt.rebind_work); 507 struct xe_validation_ctx ctx; 508 struct drm_exec exec; 509 unsigned int fence_count = 0; 510 LIST_HEAD(preempt_fences); 511 int err = 0; 512 long wait; 513 int __maybe_unused tries = 0; 514 515 xe_assert(vm->xe, xe_vm_in_preempt_fence_mode(vm)); 516 trace_xe_vm_rebind_worker_enter(vm); 517 518 down_write(&vm->lock); 519 520 if (xe_vm_is_closed_or_banned(vm)) { 521 up_write(&vm->lock); 522 trace_xe_vm_rebind_worker_exit(vm); 523 return; 524 } 525 526 retry: 527 if (!try_wait_for_completion(&vm->xe->pm_block) && vm_suspend_rebind_worker(vm)) { 528 up_write(&vm->lock); 529 /* We don't actually block but don't make progress. */ 530 xe_pm_might_block_on_suspend(); 531 return; 532 } 533 534 if (xe_vm_userptr_check_repin(vm)) { 535 err = xe_vm_userptr_pin(vm); 536 if (err) 537 goto out_unlock_outer; 538 } 539 540 err = xe_validation_ctx_init(&ctx, &vm->xe->val, &exec, 541 (struct xe_val_flags) {.interruptible = true}); 542 if (err) 543 goto out_unlock_outer; 544 545 drm_exec_until_all_locked(&exec) { 546 bool done = false; 547 548 err = xe_preempt_work_begin(&exec, vm, &done); 549 drm_exec_retry_on_contention(&exec); 550 xe_validation_retry_on_oom(&ctx, &err); 551 if (err || done) { 552 xe_validation_ctx_fini(&ctx); 553 goto out_unlock_outer; 554 } 555 } 556 557 err = alloc_preempt_fences(vm, &preempt_fences, &fence_count); 558 if (err) 559 goto out_unlock; 560 561 xe_vm_set_validation_exec(vm, &exec); 562 err = xe_vm_rebind(vm, true); 563 xe_vm_set_validation_exec(vm, NULL); 564 if (err) 565 goto out_unlock; 566 567 /* Wait on rebinds and munmap style VM unbinds */ 568 wait = dma_resv_wait_timeout(xe_vm_resv(vm), 569 DMA_RESV_USAGE_KERNEL, 570 false, MAX_SCHEDULE_TIMEOUT); 571 if (wait <= 0) { 572 err = -ETIME; 573 goto out_unlock; 574 } 575 576 #define retry_required(__tries, __vm) \ 577 (IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT) ? \ 578 (!(__tries)++ || __xe_vm_userptr_needs_repin(__vm)) : \ 579 __xe_vm_userptr_needs_repin(__vm)) 580 581 xe_svm_notifier_lock(vm); 582 if (retry_required(tries, vm)) { 583 xe_svm_notifier_unlock(vm); 584 err = -EAGAIN; 585 goto out_unlock; 586 } 587 588 #undef retry_required 589 590 xe_vm_bump_lru(vm, &exec); 591 592 /* Point of no return. */ 593 arm_preempt_fences(vm, &preempt_fences); 594 resume_and_reinstall_preempt_fences(vm, &exec); 595 xe_svm_notifier_unlock(vm); 596 597 out_unlock: 598 xe_validation_ctx_fini(&ctx); 599 out_unlock_outer: 600 if (err == -EAGAIN) { 601 trace_xe_vm_rebind_worker_retry(vm); 602 603 /* 604 * We can't block in workers on a VF which supports migration 605 * given this can block the VF post-migration workers from 606 * getting scheduled. 607 */ 608 if (IS_SRIOV_VF(vm->xe) && 609 xe_sriov_vf_migration_supported(vm->xe)) { 610 up_write(&vm->lock); 611 xe_vm_queue_rebind_worker(vm); 612 return; 613 } 614 615 goto retry; 616 } 617 618 if (err) { 619 drm_warn(&vm->xe->drm, "VM worker error: %d\n", err); 620 xe_vm_kill(vm, true); 621 } 622 up_write(&vm->lock); 623 624 free_preempt_fences(&preempt_fences); 625 626 trace_xe_vm_rebind_worker_exit(vm); 627 } 628 629 /** 630 * xe_vm_add_fault_entry_pf() - Add pagefault to vm fault list 631 * @vm: The VM. 632 * @pf: The pagefault. 633 * 634 * This function takes the data from the pagefault @pf and saves it to @vm->faults.list. 635 * 636 * The function exits silently if the list is full, and reports a warning if the pagefault 637 * could not be saved to the list. 638 */ 639 void xe_vm_add_fault_entry_pf(struct xe_vm *vm, struct xe_pagefault *pf) 640 { 641 struct xe_vm_fault_entry *e; 642 struct xe_hw_engine *hwe; 643 644 /* Do not report faults on reserved engines */ 645 hwe = xe_gt_hw_engine(pf->gt, pf->consumer.engine_class, 646 pf->consumer.engine_instance, false); 647 if (!hwe || xe_hw_engine_is_reserved(hwe)) 648 return; 649 650 e = kzalloc_obj(*e); 651 if (!e) { 652 drm_warn(&vm->xe->drm, 653 "Could not allocate memory for fault!\n"); 654 return; 655 } 656 657 guard(spinlock)(&vm->faults.lock); 658 659 /* 660 * Limit the number of faults in the fault list to prevent 661 * memory overuse. 662 */ 663 if (vm->faults.len >= MAX_FAULTS_SAVED_PER_VM) { 664 kfree(e); 665 return; 666 } 667 668 e->address = pf->consumer.page_addr; 669 /* 670 * TODO: 671 * Address precision is currently always SZ_4K, but this may change 672 * in the future. 673 */ 674 e->address_precision = SZ_4K; 675 e->access_type = pf->consumer.access_type; 676 e->fault_type = FIELD_GET(XE_PAGEFAULT_TYPE_MASK, 677 pf->consumer.fault_type_level); 678 e->fault_level = FIELD_GET(XE_PAGEFAULT_LEVEL_MASK, 679 pf->consumer.fault_type_level); 680 681 list_add_tail(&e->list, &vm->faults.list); 682 vm->faults.len++; 683 } 684 685 static void xe_vm_clear_fault_entries(struct xe_vm *vm) 686 { 687 struct xe_vm_fault_entry *e, *tmp; 688 689 guard(spinlock)(&vm->faults.lock); 690 list_for_each_entry_safe(e, tmp, &vm->faults.list, list) { 691 list_del(&e->list); 692 kfree(e); 693 } 694 vm->faults.len = 0; 695 } 696 697 static int xe_vma_ops_alloc(struct xe_vma_ops *vops, bool array_of_binds) 698 { 699 int i; 700 701 for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i) { 702 if (!vops->pt_update_ops[i].num_ops) 703 continue; 704 705 vops->pt_update_ops[i].ops = 706 kmalloc_objs(*vops->pt_update_ops[i].ops, 707 vops->pt_update_ops[i].num_ops, 708 GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN); 709 if (!vops->pt_update_ops[i].ops) 710 return array_of_binds ? -ENOBUFS : -ENOMEM; 711 } 712 713 return 0; 714 } 715 ALLOW_ERROR_INJECTION(xe_vma_ops_alloc, ERRNO); 716 717 static void xe_vma_svm_prefetch_op_fini(struct xe_vma_op *op) 718 { 719 struct xe_vma *vma; 720 721 vma = gpuva_to_vma(op->base.prefetch.va); 722 723 if (op->base.op == DRM_GPUVA_OP_PREFETCH && xe_vma_is_cpu_addr_mirror(vma)) 724 xa_destroy(&op->prefetch_range.range); 725 } 726 727 static void xe_vma_svm_prefetch_ops_fini(struct xe_vma_ops *vops) 728 { 729 struct xe_vma_op *op; 730 731 if (!(vops->flags & XE_VMA_OPS_FLAG_HAS_SVM_PREFETCH)) 732 return; 733 734 list_for_each_entry(op, &vops->list, link) 735 xe_vma_svm_prefetch_op_fini(op); 736 } 737 738 static void xe_vma_ops_fini(struct xe_vma_ops *vops) 739 { 740 int i; 741 742 xe_vma_svm_prefetch_ops_fini(vops); 743 744 for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i) 745 kfree(vops->pt_update_ops[i].ops); 746 } 747 748 static void xe_vma_ops_incr_pt_update_ops(struct xe_vma_ops *vops, u8 tile_mask, int inc_val) 749 { 750 int i; 751 752 if (!inc_val) 753 return; 754 755 for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i) 756 if (BIT(i) & tile_mask) 757 vops->pt_update_ops[i].num_ops += inc_val; 758 } 759 760 #define XE_VMA_CREATE_MASK ( \ 761 XE_VMA_READ_ONLY | \ 762 XE_VMA_DUMPABLE | \ 763 XE_VMA_SYSTEM_ALLOCATOR | \ 764 DRM_GPUVA_SPARSE | \ 765 XE_VMA_MADV_AUTORESET) 766 767 static void xe_vm_populate_rebind(struct xe_vma_op *op, struct xe_vma *vma, 768 u8 tile_mask) 769 { 770 INIT_LIST_HEAD(&op->link); 771 op->tile_mask = tile_mask; 772 op->base.op = DRM_GPUVA_OP_MAP; 773 op->base.map.va.addr = vma->gpuva.va.addr; 774 op->base.map.va.range = vma->gpuva.va.range; 775 op->base.map.gem.obj = vma->gpuva.gem.obj; 776 op->base.map.gem.offset = vma->gpuva.gem.offset; 777 op->map.vma = vma; 778 op->map.immediate = true; 779 op->map.vma_flags = vma->gpuva.flags & XE_VMA_CREATE_MASK; 780 } 781 782 static int xe_vm_ops_add_rebind(struct xe_vma_ops *vops, struct xe_vma *vma, 783 u8 tile_mask) 784 { 785 struct xe_vma_op *op; 786 787 op = kzalloc_obj(*op); 788 if (!op) 789 return -ENOMEM; 790 791 xe_vm_populate_rebind(op, vma, tile_mask); 792 list_add_tail(&op->link, &vops->list); 793 xe_vma_ops_incr_pt_update_ops(vops, tile_mask, 1); 794 795 return 0; 796 } 797 798 static struct dma_fence *ops_execute(struct xe_vm *vm, 799 struct xe_vma_ops *vops); 800 static void xe_vma_ops_init(struct xe_vma_ops *vops, struct xe_vm *vm, 801 struct xe_exec_queue *q, 802 struct xe_sync_entry *syncs, u32 num_syncs); 803 804 int xe_vm_rebind(struct xe_vm *vm, bool rebind_worker) 805 { 806 struct dma_fence *fence; 807 struct xe_vma *vma, *next; 808 struct xe_vma_ops vops; 809 struct xe_vma_op *op, *next_op; 810 int err, i; 811 812 lockdep_assert_held(&vm->lock); 813 if ((xe_vm_in_lr_mode(vm) && !rebind_worker) || 814 list_empty(&vm->rebind_list)) 815 return 0; 816 817 xe_vma_ops_init(&vops, vm, NULL, NULL, 0); 818 for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i) 819 vops.pt_update_ops[i].wait_vm_bookkeep = true; 820 821 xe_vm_assert_held(vm); 822 list_for_each_entry(vma, &vm->rebind_list, combined_links.rebind) { 823 xe_assert(vm->xe, vma->tile_present); 824 825 if (rebind_worker) 826 trace_xe_vma_rebind_worker(vma); 827 else 828 trace_xe_vma_rebind_exec(vma); 829 830 err = xe_vm_ops_add_rebind(&vops, vma, 831 vma->tile_present); 832 if (err) 833 goto free_ops; 834 } 835 836 err = xe_vma_ops_alloc(&vops, false); 837 if (err) 838 goto free_ops; 839 840 fence = ops_execute(vm, &vops); 841 if (IS_ERR(fence)) { 842 err = PTR_ERR(fence); 843 } else { 844 dma_fence_put(fence); 845 list_for_each_entry_safe(vma, next, &vm->rebind_list, 846 combined_links.rebind) 847 list_del_init(&vma->combined_links.rebind); 848 } 849 free_ops: 850 list_for_each_entry_safe(op, next_op, &vops.list, link) { 851 list_del(&op->link); 852 kfree(op); 853 } 854 xe_vma_ops_fini(&vops); 855 856 return err; 857 } 858 859 struct dma_fence *xe_vma_rebind(struct xe_vm *vm, struct xe_vma *vma, u8 tile_mask) 860 { 861 struct dma_fence *fence = NULL; 862 struct xe_vma_ops vops; 863 struct xe_vma_op *op, *next_op; 864 struct xe_tile *tile; 865 u8 id; 866 int err; 867 868 lockdep_assert_held(&vm->lock); 869 xe_vm_assert_held(vm); 870 xe_assert(vm->xe, xe_vm_in_fault_mode(vm)); 871 872 xe_vma_ops_init(&vops, vm, NULL, NULL, 0); 873 vops.flags |= XE_VMA_OPS_FLAG_SKIP_TLB_WAIT; 874 for_each_tile(tile, vm->xe, id) { 875 vops.pt_update_ops[id].wait_vm_bookkeep = true; 876 vops.pt_update_ops[tile->id].q = 877 xe_migrate_exec_queue(tile->migrate); 878 } 879 880 err = xe_vm_ops_add_rebind(&vops, vma, tile_mask); 881 if (err) 882 return ERR_PTR(err); 883 884 err = xe_vma_ops_alloc(&vops, false); 885 if (err) { 886 fence = ERR_PTR(err); 887 goto free_ops; 888 } 889 890 fence = ops_execute(vm, &vops); 891 892 free_ops: 893 list_for_each_entry_safe(op, next_op, &vops.list, link) { 894 list_del(&op->link); 895 kfree(op); 896 } 897 xe_vma_ops_fini(&vops); 898 899 return fence; 900 } 901 902 static void xe_vm_populate_range_rebind(struct xe_vma_op *op, 903 struct xe_vma *vma, 904 struct xe_svm_range *range, 905 u8 tile_mask) 906 { 907 INIT_LIST_HEAD(&op->link); 908 op->tile_mask = tile_mask; 909 op->base.op = DRM_GPUVA_OP_DRIVER; 910 op->subop = XE_VMA_SUBOP_MAP_RANGE; 911 op->map_range.vma = vma; 912 op->map_range.range = range; 913 } 914 915 static int 916 xe_vm_ops_add_range_rebind(struct xe_vma_ops *vops, 917 struct xe_vma *vma, 918 struct xe_svm_range *range, 919 u8 tile_mask) 920 { 921 struct xe_vma_op *op; 922 923 op = kzalloc_obj(*op); 924 if (!op) 925 return -ENOMEM; 926 927 xe_vm_populate_range_rebind(op, vma, range, tile_mask); 928 list_add_tail(&op->link, &vops->list); 929 xe_vma_ops_incr_pt_update_ops(vops, tile_mask, 1); 930 931 return 0; 932 } 933 934 /** 935 * xe_vm_range_rebind() - VM range (re)bind 936 * @vm: The VM which the range belongs to. 937 * @vma: The VMA which the range belongs to. 938 * @range: SVM range to rebind. 939 * @tile_mask: Tile mask to bind the range to. 940 * 941 * (re)bind SVM range setting up GPU page tables for the range. 942 * 943 * Return: dma fence for rebind to signal completion on success, ERR_PTR on 944 * failure 945 */ 946 struct dma_fence *xe_vm_range_rebind(struct xe_vm *vm, 947 struct xe_vma *vma, 948 struct xe_svm_range *range, 949 u8 tile_mask) 950 { 951 struct dma_fence *fence = NULL; 952 struct xe_vma_ops vops; 953 struct xe_vma_op *op, *next_op; 954 struct xe_tile *tile; 955 u8 id; 956 int err; 957 958 lockdep_assert_held(&vm->lock); 959 xe_vm_assert_held(vm); 960 xe_assert(vm->xe, xe_vm_in_fault_mode(vm)); 961 xe_assert(vm->xe, xe_vma_is_cpu_addr_mirror(vma)); 962 963 xe_vma_ops_init(&vops, vm, NULL, NULL, 0); 964 vops.flags |= XE_VMA_OPS_FLAG_SKIP_TLB_WAIT; 965 for_each_tile(tile, vm->xe, id) { 966 vops.pt_update_ops[id].wait_vm_bookkeep = true; 967 vops.pt_update_ops[tile->id].q = 968 xe_migrate_exec_queue(tile->migrate); 969 } 970 971 err = xe_vm_ops_add_range_rebind(&vops, vma, range, tile_mask); 972 if (err) 973 return ERR_PTR(err); 974 975 err = xe_vma_ops_alloc(&vops, false); 976 if (err) { 977 fence = ERR_PTR(err); 978 goto free_ops; 979 } 980 981 fence = ops_execute(vm, &vops); 982 983 free_ops: 984 list_for_each_entry_safe(op, next_op, &vops.list, link) { 985 list_del(&op->link); 986 kfree(op); 987 } 988 xe_vma_ops_fini(&vops); 989 990 return fence; 991 } 992 993 static void xe_vm_populate_range_unbind(struct xe_vma_op *op, 994 struct xe_svm_range *range) 995 { 996 INIT_LIST_HEAD(&op->link); 997 op->tile_mask = range->tile_present; 998 op->base.op = DRM_GPUVA_OP_DRIVER; 999 op->subop = XE_VMA_SUBOP_UNMAP_RANGE; 1000 op->unmap_range.range = range; 1001 } 1002 1003 static int 1004 xe_vm_ops_add_range_unbind(struct xe_vma_ops *vops, 1005 struct xe_svm_range *range) 1006 { 1007 struct xe_vma_op *op; 1008 1009 op = kzalloc_obj(*op); 1010 if (!op) 1011 return -ENOMEM; 1012 1013 xe_vm_populate_range_unbind(op, range); 1014 list_add_tail(&op->link, &vops->list); 1015 xe_vma_ops_incr_pt_update_ops(vops, range->tile_present, 1); 1016 1017 return 0; 1018 } 1019 1020 /** 1021 * xe_vm_range_unbind() - VM range unbind 1022 * @vm: The VM which the range belongs to. 1023 * @range: SVM range to rebind. 1024 * 1025 * Unbind SVM range removing the GPU page tables for the range. 1026 * 1027 * Return: dma fence for unbind to signal completion on success, ERR_PTR on 1028 * failure 1029 */ 1030 struct dma_fence *xe_vm_range_unbind(struct xe_vm *vm, 1031 struct xe_svm_range *range) 1032 { 1033 struct dma_fence *fence = NULL; 1034 struct xe_vma_ops vops; 1035 struct xe_vma_op *op, *next_op; 1036 struct xe_tile *tile; 1037 u8 id; 1038 int err; 1039 1040 lockdep_assert_held(&vm->lock); 1041 xe_vm_assert_held(vm); 1042 xe_assert(vm->xe, xe_vm_in_fault_mode(vm)); 1043 1044 if (!range->tile_present) 1045 return dma_fence_get_stub(); 1046 1047 xe_vma_ops_init(&vops, vm, NULL, NULL, 0); 1048 for_each_tile(tile, vm->xe, id) { 1049 vops.pt_update_ops[id].wait_vm_bookkeep = true; 1050 vops.pt_update_ops[tile->id].q = 1051 xe_migrate_exec_queue(tile->migrate); 1052 } 1053 1054 err = xe_vm_ops_add_range_unbind(&vops, range); 1055 if (err) 1056 return ERR_PTR(err); 1057 1058 err = xe_vma_ops_alloc(&vops, false); 1059 if (err) { 1060 fence = ERR_PTR(err); 1061 goto free_ops; 1062 } 1063 1064 fence = ops_execute(vm, &vops); 1065 1066 free_ops: 1067 list_for_each_entry_safe(op, next_op, &vops.list, link) { 1068 list_del(&op->link); 1069 kfree(op); 1070 } 1071 xe_vma_ops_fini(&vops); 1072 1073 return fence; 1074 } 1075 1076 static void xe_vma_mem_attr_fini(struct xe_vma_mem_attr *attr) 1077 { 1078 drm_pagemap_put(attr->preferred_loc.dpagemap); 1079 } 1080 1081 static void xe_vma_free(struct xe_vma *vma) 1082 { 1083 xe_vma_mem_attr_fini(&vma->attr); 1084 1085 if (xe_vma_is_userptr(vma)) 1086 kfree(to_userptr_vma(vma)); 1087 else 1088 kfree(vma); 1089 } 1090 1091 /** 1092 * xe_vma_mem_attr_copy() - copy an xe_vma_mem_attr structure. 1093 * @to: Destination. 1094 * @from: Source. 1095 * 1096 * Copies an xe_vma_mem_attr structure taking care to get reference 1097 * counting of individual members right. 1098 */ 1099 void xe_vma_mem_attr_copy(struct xe_vma_mem_attr *to, struct xe_vma_mem_attr *from) 1100 { 1101 xe_vma_mem_attr_fini(to); 1102 *to = *from; 1103 if (to->preferred_loc.dpagemap) 1104 drm_pagemap_get(to->preferred_loc.dpagemap); 1105 } 1106 1107 static struct xe_vma *xe_vma_create(struct xe_vm *vm, 1108 struct xe_bo *bo, 1109 u64 bo_offset_or_userptr, 1110 u64 start, u64 end, 1111 struct xe_vma_mem_attr *attr, 1112 unsigned int flags) 1113 { 1114 struct xe_vma *vma; 1115 struct xe_tile *tile; 1116 u8 id; 1117 bool is_null = (flags & DRM_GPUVA_SPARSE); 1118 bool is_cpu_addr_mirror = (flags & XE_VMA_SYSTEM_ALLOCATOR); 1119 1120 xe_assert(vm->xe, start < end); 1121 xe_assert(vm->xe, end < vm->size); 1122 1123 /* 1124 * Allocate and ensure that the xe_vma_is_userptr() return 1125 * matches what was allocated. 1126 */ 1127 if (!bo && !is_null && !is_cpu_addr_mirror) { 1128 struct xe_userptr_vma *uvma = kzalloc_obj(*uvma); 1129 1130 if (!uvma) 1131 return ERR_PTR(-ENOMEM); 1132 1133 vma = &uvma->vma; 1134 } else { 1135 vma = kzalloc_obj(*vma); 1136 if (!vma) 1137 return ERR_PTR(-ENOMEM); 1138 1139 if (bo) 1140 vma->gpuva.gem.obj = &bo->ttm.base; 1141 } 1142 1143 INIT_LIST_HEAD(&vma->combined_links.rebind); 1144 1145 INIT_LIST_HEAD(&vma->gpuva.gem.entry); 1146 vma->gpuva.vm = &vm->gpuvm; 1147 vma->gpuva.va.addr = start; 1148 vma->gpuva.va.range = end - start + 1; 1149 vma->gpuva.flags = flags; 1150 1151 for_each_tile(tile, vm->xe, id) 1152 vma->tile_mask |= 0x1 << id; 1153 1154 if (vm->xe->info.has_atomic_enable_pte_bit) 1155 vma->gpuva.flags |= XE_VMA_ATOMIC_PTE_BIT; 1156 1157 xe_vma_mem_attr_copy(&vma->attr, attr); 1158 if (bo) { 1159 struct drm_gpuvm_bo *vm_bo; 1160 1161 xe_bo_assert_held(bo); 1162 1163 /* 1164 * Reject only WILLNEED mappings on DONTNEED/PURGED BOs. This 1165 * gates new vm_bind ioctls (user supplies WILLNEED) while 1166 * still allowing partial-unbind / remap splits whose new VMAs 1167 * inherit the parent's DONTNEED attr. It must also run before 1168 * xe_bo_willneed_get_locked() below so a 0->1 holder bump 1169 * cannot silently promote DONTNEED back to WILLNEED. 1170 */ 1171 if (vma->attr.purgeable_state == XE_MADV_PURGEABLE_WILLNEED) { 1172 if (xe_bo_madv_is_dontneed(bo)) { 1173 xe_vma_free(vma); 1174 return ERR_PTR(-EBUSY); 1175 } 1176 if (xe_bo_is_purged(bo)) { 1177 xe_vma_free(vma); 1178 return ERR_PTR(-EINVAL); 1179 } 1180 } 1181 1182 vm_bo = drm_gpuvm_bo_obtain_locked(vma->gpuva.vm, &bo->ttm.base); 1183 if (IS_ERR(vm_bo)) { 1184 xe_vma_free(vma); 1185 return ERR_CAST(vm_bo); 1186 } 1187 1188 drm_gpuvm_bo_extobj_add(vm_bo); 1189 drm_gem_object_get(&bo->ttm.base); 1190 vma->gpuva.gem.offset = bo_offset_or_userptr; 1191 drm_gpuva_link(&vma->gpuva, vm_bo); 1192 drm_gpuvm_bo_put(vm_bo); 1193 1194 xe_bo_vma_count_inc_locked(bo); 1195 if (vma->attr.purgeable_state == XE_MADV_PURGEABLE_WILLNEED) 1196 xe_bo_willneed_get_locked(bo); 1197 } else /* userptr or null */ { 1198 if (!is_null && !is_cpu_addr_mirror) { 1199 struct xe_userptr_vma *uvma = to_userptr_vma(vma); 1200 u64 size = end - start + 1; 1201 int err; 1202 1203 vma->gpuva.gem.offset = bo_offset_or_userptr; 1204 1205 err = xe_userptr_setup(uvma, xe_vma_userptr(vma), size); 1206 if (err) { 1207 xe_vma_free(vma); 1208 return ERR_PTR(err); 1209 } 1210 } 1211 1212 xe_vm_get(vm); 1213 } 1214 1215 return vma; 1216 } 1217 1218 static void xe_vma_destroy_late(struct xe_vma *vma) 1219 { 1220 struct xe_vm *vm = xe_vma_vm(vma); 1221 struct xe_bo *bo = xe_vma_bo(vma); 1222 1223 if (vma->ufence) { 1224 xe_sync_ufence_put(vma->ufence); 1225 vma->ufence = NULL; 1226 } 1227 1228 if (xe_vma_is_userptr(vma)) { 1229 struct xe_userptr_vma *uvma = to_userptr_vma(vma); 1230 1231 xe_userptr_remove(uvma); 1232 xe_vm_put(vm); 1233 } else if (xe_vma_is_null(vma) || xe_vma_is_cpu_addr_mirror(vma)) { 1234 xe_vm_put(vm); 1235 } else { 1236 xe_bo_put(bo); 1237 } 1238 1239 xe_vma_free(vma); 1240 } 1241 1242 static void vma_destroy_work_func(struct work_struct *w) 1243 { 1244 struct xe_vma *vma = 1245 container_of(w, struct xe_vma, destroy_work); 1246 1247 xe_vma_destroy_late(vma); 1248 } 1249 1250 static void vma_destroy_cb(struct dma_fence *fence, 1251 struct dma_fence_cb *cb) 1252 { 1253 struct xe_vma *vma = container_of(cb, struct xe_vma, destroy_cb); 1254 1255 INIT_WORK(&vma->destroy_work, vma_destroy_work_func); 1256 queue_work(system_dfl_wq, &vma->destroy_work); 1257 } 1258 1259 static void xe_vma_destroy(struct xe_vma *vma, struct dma_fence *fence) 1260 { 1261 struct xe_vm *vm = xe_vma_vm(vma); 1262 struct xe_bo *bo = xe_vma_bo(vma); 1263 1264 lockdep_assert_held_write(&vm->lock); 1265 xe_assert(vm->xe, list_empty(&vma->combined_links.destroy)); 1266 1267 if (xe_vma_is_userptr(vma)) { 1268 xe_assert(vm->xe, vma->gpuva.flags & XE_VMA_DESTROYED); 1269 xe_userptr_destroy(to_userptr_vma(vma)); 1270 } else if (!xe_vma_is_null(vma) && !xe_vma_is_cpu_addr_mirror(vma)) { 1271 xe_bo_assert_held(bo); 1272 1273 drm_gpuva_unlink(&vma->gpuva); 1274 1275 xe_bo_vma_count_dec_locked(bo); 1276 if (vma->attr.purgeable_state == XE_MADV_PURGEABLE_WILLNEED) 1277 xe_bo_willneed_put_locked(bo); 1278 } 1279 1280 xe_vm_assert_held(vm); 1281 if (fence) { 1282 int ret = dma_fence_add_callback(fence, &vma->destroy_cb, 1283 vma_destroy_cb); 1284 1285 if (ret) { 1286 XE_WARN_ON(ret != -ENOENT); 1287 xe_vma_destroy_late(vma); 1288 } 1289 } else { 1290 xe_vma_destroy_late(vma); 1291 } 1292 } 1293 1294 /** 1295 * xe_vm_lock_vma() - drm_exec utility to lock a vma 1296 * @exec: The drm_exec object we're currently locking for. 1297 * @vma: The vma for witch we want to lock the vm resv and any attached 1298 * object's resv. 1299 * 1300 * Return: 0 on success, negative error code on error. In particular 1301 * may return -EDEADLK on WW transaction contention and -EINTR if 1302 * an interruptible wait is terminated by a signal. 1303 */ 1304 int xe_vm_lock_vma(struct drm_exec *exec, struct xe_vma *vma) 1305 { 1306 struct xe_vm *vm = xe_vma_vm(vma); 1307 struct xe_bo *bo = xe_vma_bo(vma); 1308 int err; 1309 1310 XE_WARN_ON(!vm); 1311 1312 err = drm_exec_lock_obj(exec, xe_vm_obj(vm)); 1313 if (!err && bo && !bo->vm) 1314 err = drm_exec_lock_obj(exec, &bo->ttm.base); 1315 1316 return err; 1317 } 1318 1319 static void xe_vma_destroy_unlocked(struct xe_vma *vma) 1320 { 1321 struct xe_device *xe = xe_vma_vm(vma)->xe; 1322 struct xe_validation_ctx ctx; 1323 struct drm_exec exec; 1324 int err = 0; 1325 1326 xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {}, err) { 1327 err = xe_vm_lock_vma(&exec, vma); 1328 drm_exec_retry_on_contention(&exec); 1329 if (XE_WARN_ON(err)) 1330 break; 1331 xe_vma_destroy(vma, NULL); 1332 } 1333 xe_assert(xe, !err); 1334 } 1335 1336 struct xe_vma * 1337 xe_vm_find_overlapping_vma(struct xe_vm *vm, u64 start, u64 range) 1338 { 1339 struct drm_gpuva *gpuva; 1340 1341 lockdep_assert_held(&vm->lock); 1342 1343 if (xe_vm_is_closed_or_banned(vm)) 1344 return NULL; 1345 1346 xe_assert(vm->xe, start + range <= vm->size); 1347 1348 gpuva = drm_gpuva_find_first(&vm->gpuvm, start, range); 1349 1350 return gpuva ? gpuva_to_vma(gpuva) : NULL; 1351 } 1352 1353 static int xe_vm_insert_vma(struct xe_vm *vm, struct xe_vma *vma) 1354 { 1355 int err; 1356 1357 xe_assert(vm->xe, xe_vma_vm(vma) == vm); 1358 lockdep_assert_held(&vm->lock); 1359 1360 mutex_lock(&vm->snap_mutex); 1361 err = drm_gpuva_insert(&vm->gpuvm, &vma->gpuva); 1362 mutex_unlock(&vm->snap_mutex); 1363 XE_WARN_ON(err); /* Shouldn't be possible */ 1364 1365 return err; 1366 } 1367 1368 static void xe_vm_remove_vma(struct xe_vm *vm, struct xe_vma *vma) 1369 { 1370 xe_assert(vm->xe, xe_vma_vm(vma) == vm); 1371 lockdep_assert_held(&vm->lock); 1372 1373 mutex_lock(&vm->snap_mutex); 1374 drm_gpuva_remove(&vma->gpuva); 1375 mutex_unlock(&vm->snap_mutex); 1376 if (vm->usm.last_fault_vma == vma) 1377 vm->usm.last_fault_vma = NULL; 1378 } 1379 1380 static struct drm_gpuva_op *xe_vm_op_alloc(void) 1381 { 1382 struct xe_vma_op *op; 1383 1384 op = kzalloc_obj(*op); 1385 1386 if (unlikely(!op)) 1387 return NULL; 1388 1389 return &op->base; 1390 } 1391 1392 static void xe_vm_free(struct drm_gpuvm *gpuvm); 1393 1394 static const struct drm_gpuvm_ops gpuvm_ops = { 1395 .op_alloc = xe_vm_op_alloc, 1396 .vm_bo_validate = xe_gpuvm_validate, 1397 .vm_free = xe_vm_free, 1398 }; 1399 1400 static u64 pde_encode_pat_index(u16 pat_index) 1401 { 1402 u64 pte = 0; 1403 1404 if (pat_index & BIT(0)) 1405 pte |= XE_PPGTT_PTE_PAT0; 1406 1407 if (pat_index & BIT(1)) 1408 pte |= XE_PPGTT_PTE_PAT1; 1409 1410 return pte; 1411 } 1412 1413 static u64 pte_encode_pat_index(u16 pat_index, u32 pt_level) 1414 { 1415 u64 pte = 0; 1416 1417 if (pat_index & BIT(0)) 1418 pte |= XE_PPGTT_PTE_PAT0; 1419 1420 if (pat_index & BIT(1)) 1421 pte |= XE_PPGTT_PTE_PAT1; 1422 1423 if (pat_index & BIT(2)) { 1424 if (pt_level) 1425 pte |= XE_PPGTT_PDE_PDPE_PAT2; 1426 else 1427 pte |= XE_PPGTT_PTE_PAT2; 1428 } 1429 1430 if (pat_index & BIT(3)) 1431 pte |= XELPG_PPGTT_PTE_PAT3; 1432 1433 if (pat_index & (BIT(4))) 1434 pte |= XE2_PPGTT_PTE_PAT4; 1435 1436 return pte; 1437 } 1438 1439 static u64 pte_encode_ps(u32 pt_level) 1440 { 1441 XE_WARN_ON(pt_level > MAX_HUGEPTE_LEVEL); 1442 1443 if (pt_level == 1) 1444 return XE_PDE_PS_2M; 1445 else if (pt_level == 2) 1446 return XE_PDPE_PS_1G; 1447 1448 return 0; 1449 } 1450 1451 static u16 pde_pat_index(struct xe_bo *bo) 1452 { 1453 struct xe_device *xe = xe_bo_device(bo); 1454 u16 pat_index; 1455 1456 /* 1457 * We only have two bits to encode the PAT index in non-leaf nodes, but 1458 * these only point to other paging structures so we only need a minimal 1459 * selection of options. The user PAT index is only for encoding leaf 1460 * nodes, where we have use of more bits to do the encoding. The 1461 * non-leaf nodes are instead under driver control so the chosen index 1462 * here should be distinct from the user PAT index. Also the 1463 * corresponding coherency of the PAT index should be tied to the 1464 * allocation type of the page table (or at least we should pick 1465 * something which is always safe). 1466 */ 1467 if (!xe_bo_is_vram(bo) && bo->ttm.ttm->caching == ttm_cached) 1468 pat_index = xe_cache_pat_idx(xe, XE_CACHE_WB); 1469 else 1470 pat_index = xe_cache_pat_idx(xe, XE_CACHE_NONE); 1471 1472 xe_assert(xe, pat_index <= 3); 1473 1474 return pat_index; 1475 } 1476 1477 static u64 xelp_pde_encode_bo(struct xe_bo *bo, u64 bo_offset) 1478 { 1479 u64 pde; 1480 1481 pde = xe_bo_addr(bo, bo_offset, XE_PAGE_SIZE); 1482 pde |= XE_PAGE_PRESENT | XE_PAGE_RW; 1483 pde |= pde_encode_pat_index(pde_pat_index(bo)); 1484 1485 return pde; 1486 } 1487 1488 static u64 xelp_pte_encode_bo(struct xe_bo *bo, u64 bo_offset, 1489 u16 pat_index, u32 pt_level) 1490 { 1491 u64 pte; 1492 1493 pte = xe_bo_addr(bo, bo_offset, XE_PAGE_SIZE); 1494 pte |= XE_PAGE_PRESENT | XE_PAGE_RW; 1495 pte |= pte_encode_pat_index(pat_index, pt_level); 1496 pte |= pte_encode_ps(pt_level); 1497 1498 if (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo)) 1499 pte |= XE_PPGTT_PTE_DM; 1500 1501 return pte; 1502 } 1503 1504 static u64 xelp_pte_encode_vma(u64 pte, struct xe_vma *vma, 1505 u16 pat_index, u32 pt_level) 1506 { 1507 struct xe_bo *bo = xe_vma_bo(vma); 1508 struct xe_vm *vm = xe_vma_vm(vma); 1509 1510 pte |= XE_PAGE_PRESENT; 1511 1512 if (likely(!xe_vma_read_only(vma))) 1513 pte |= XE_PAGE_RW; 1514 1515 pte |= pte_encode_pat_index(pat_index, pt_level); 1516 pte |= pte_encode_ps(pt_level); 1517 1518 /* 1519 * NULL PTEs redirect to scratch page (return zeros on read). 1520 * Set for: 1) explicit null VMAs, 2) purged BOs on scratch VMs. 1521 * Never set NULL flag without scratch page - causes undefined behavior. 1522 */ 1523 if (unlikely(xe_vma_is_null(vma) || 1524 (bo && xe_bo_is_purged(bo) && xe_vm_has_scratch(vm)))) 1525 pte |= XE_PTE_NULL; 1526 1527 return pte; 1528 } 1529 1530 static u64 xelp_pte_encode_addr(struct xe_device *xe, u64 addr, 1531 u16 pat_index, 1532 u32 pt_level, bool devmem, u64 flags) 1533 { 1534 u64 pte; 1535 1536 /* Avoid passing random bits directly as flags */ 1537 xe_assert(xe, !(flags & ~XE_PTE_PS64)); 1538 1539 pte = addr; 1540 pte |= XE_PAGE_PRESENT | XE_PAGE_RW; 1541 pte |= pte_encode_pat_index(pat_index, pt_level); 1542 pte |= pte_encode_ps(pt_level); 1543 1544 if (devmem) 1545 pte |= XE_PPGTT_PTE_DM; 1546 1547 pte |= flags; 1548 1549 return pte; 1550 } 1551 1552 static const struct xe_pt_ops xelp_pt_ops = { 1553 .pte_encode_bo = xelp_pte_encode_bo, 1554 .pte_encode_vma = xelp_pte_encode_vma, 1555 .pte_encode_addr = xelp_pte_encode_addr, 1556 .pde_encode_bo = xelp_pde_encode_bo, 1557 }; 1558 1559 static void vm_destroy_work_func(struct work_struct *w); 1560 1561 /** 1562 * xe_vm_create_scratch() - Setup a scratch memory pagetable tree for the 1563 * given tile and vm. 1564 * @xe: xe device. 1565 * @tile: tile to set up for. 1566 * @vm: vm to set up for. 1567 * @exec: The struct drm_exec object used to lock the vm resv. 1568 * 1569 * Sets up a pagetable tree with one page-table per level and a single 1570 * leaf PTE. All pagetable entries point to the single page-table or, 1571 * for MAX_HUGEPTE_LEVEL, a NULL huge PTE returning 0 on read and 1572 * writes become NOPs. 1573 * 1574 * Return: 0 on success, negative error code on error. 1575 */ 1576 static int xe_vm_create_scratch(struct xe_device *xe, struct xe_tile *tile, 1577 struct xe_vm *vm, struct drm_exec *exec) 1578 { 1579 u8 id = tile->id; 1580 int i; 1581 1582 for (i = MAX_HUGEPTE_LEVEL; i < vm->pt_root[id]->level; i++) { 1583 vm->scratch_pt[id][i] = xe_pt_create(vm, tile, i, exec); 1584 if (IS_ERR(vm->scratch_pt[id][i])) { 1585 int err = PTR_ERR(vm->scratch_pt[id][i]); 1586 1587 vm->scratch_pt[id][i] = NULL; 1588 return err; 1589 } 1590 xe_pt_populate_empty(tile, vm, vm->scratch_pt[id][i]); 1591 } 1592 1593 return 0; 1594 } 1595 ALLOW_ERROR_INJECTION(xe_vm_create_scratch, ERRNO); 1596 1597 static void xe_vm_free_scratch(struct xe_vm *vm) 1598 { 1599 struct xe_tile *tile; 1600 u8 id; 1601 1602 if (!xe_vm_has_scratch(vm)) 1603 return; 1604 1605 for_each_tile(tile, vm->xe, id) { 1606 u32 i; 1607 1608 if (!vm->pt_root[id]) 1609 continue; 1610 1611 for (i = MAX_HUGEPTE_LEVEL; i < vm->pt_root[id]->level; ++i) 1612 if (vm->scratch_pt[id][i]) 1613 xe_pt_destroy(vm->scratch_pt[id][i], vm->flags, NULL); 1614 } 1615 } 1616 1617 static void xe_vm_pt_destroy(struct xe_vm *vm) 1618 { 1619 struct xe_tile *tile; 1620 u8 id; 1621 1622 xe_vm_assert_held(vm); 1623 1624 for_each_tile(tile, vm->xe, id) { 1625 if (vm->pt_root[id]) { 1626 xe_pt_destroy(vm->pt_root[id], vm->flags, NULL); 1627 vm->pt_root[id] = NULL; 1628 } 1629 } 1630 } 1631 1632 static void xe_vm_init_prove_locking(struct xe_device *xe, struct xe_vm *vm) 1633 { 1634 if (!IS_ENABLED(CONFIG_PROVE_LOCKING)) 1635 return; 1636 1637 fs_reclaim_acquire(GFP_KERNEL); 1638 might_lock(&vm->exec_queues.lock); 1639 fs_reclaim_release(GFP_KERNEL); 1640 1641 down_read(&vm->exec_queues.lock); 1642 might_lock(&xe_root_mmio_gt(xe)->uc.guc.ct.lock); 1643 up_read(&vm->exec_queues.lock); 1644 } 1645 1646 struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags, struct xe_file *xef) 1647 { 1648 struct drm_gem_object *vm_resv_obj; 1649 struct xe_validation_ctx ctx; 1650 struct drm_exec exec; 1651 struct xe_vm *vm; 1652 int err; 1653 struct xe_tile *tile; 1654 u8 id; 1655 1656 /* 1657 * Since the GSCCS is not user-accessible, we don't expect a GSC VM to 1658 * ever be in faulting mode. 1659 */ 1660 xe_assert(xe, !((flags & XE_VM_FLAG_GSC) && (flags & XE_VM_FLAG_FAULT_MODE))); 1661 1662 vm = kzalloc(sizeof(*vm), GFP_KERNEL); 1663 if (!vm) 1664 return ERR_PTR(-ENOMEM); 1665 1666 vm->xe = xe; 1667 1668 vm->size = 1ull << xe->info.va_bits; 1669 vm->flags = flags; 1670 1671 if (xef) 1672 vm->xef = xe_file_get(xef); 1673 /* 1674 * GSC VMs are kernel-owned, only used for PXP ops and can sometimes be 1675 * manipulated under the PXP mutex. However, the PXP mutex can be taken 1676 * under a user-VM lock when the PXP session is started at exec_queue 1677 * creation time. Those are different VMs and therefore there is no risk 1678 * of deadlock, but we need to tell lockdep that this is the case or it 1679 * will print a warning. 1680 */ 1681 if (flags & XE_VM_FLAG_GSC) { 1682 static struct lock_class_key gsc_vm_key; 1683 1684 __init_rwsem(&vm->lock, "gsc_vm", &gsc_vm_key); 1685 } else { 1686 init_rwsem(&vm->lock); 1687 } 1688 mutex_init(&vm->snap_mutex); 1689 1690 INIT_LIST_HEAD(&vm->rebind_list); 1691 1692 INIT_LIST_HEAD(&vm->userptr.repin_list); 1693 INIT_LIST_HEAD(&vm->userptr.invalidated); 1694 spin_lock_init(&vm->userptr.invalidated_lock); 1695 1696 INIT_LIST_HEAD(&vm->faults.list); 1697 spin_lock_init(&vm->faults.lock); 1698 1699 ttm_lru_bulk_move_init(&vm->lru_bulk_move); 1700 1701 INIT_WORK(&vm->destroy_work, vm_destroy_work_func); 1702 1703 INIT_LIST_HEAD(&vm->preempt.exec_queues); 1704 for (id = 0; id < XE_MAX_TILES_PER_DEVICE * XE_MAX_GT_PER_TILE; ++id) 1705 INIT_LIST_HEAD(&vm->exec_queues.list[id]); 1706 if (flags & XE_VM_FLAG_FAULT_MODE) 1707 vm->preempt.min_run_period_ms = xe->min_run_period_pf_ms; 1708 else 1709 vm->preempt.min_run_period_ms = xe->min_run_period_lr_ms; 1710 1711 init_rwsem(&vm->exec_queues.lock); 1712 xe_vm_init_prove_locking(xe, vm); 1713 1714 for_each_tile(tile, xe, id) 1715 xe_range_fence_tree_init(&vm->rftree[id]); 1716 1717 vm->pt_ops = &xelp_pt_ops; 1718 1719 /* 1720 * Long-running workloads are not protected by the scheduler references. 1721 * By design, run_job for long-running workloads returns NULL and the 1722 * scheduler drops all the references of it, hence protecting the VM 1723 * for this case is necessary. 1724 */ 1725 if (flags & XE_VM_FLAG_LR_MODE) { 1726 INIT_WORK(&vm->preempt.rebind_work, preempt_rebind_work_func); 1727 xe_pm_runtime_get_noresume(xe); 1728 INIT_LIST_HEAD(&vm->preempt.pm_activate_link); 1729 } 1730 1731 err = xe_svm_init(vm); 1732 if (err) 1733 goto err_no_resv; 1734 1735 vm_resv_obj = drm_gpuvm_resv_object_alloc(&xe->drm); 1736 if (!vm_resv_obj) { 1737 err = -ENOMEM; 1738 goto err_svm_fini; 1739 } 1740 1741 drm_gpuvm_init(&vm->gpuvm, "Xe VM", DRM_GPUVM_RESV_PROTECTED, &xe->drm, 1742 vm_resv_obj, 0, vm->size, 0, 0, &gpuvm_ops); 1743 1744 drm_gem_object_put(vm_resv_obj); 1745 1746 err = 0; 1747 xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {.interruptible = true}, 1748 err) { 1749 err = xe_vm_drm_exec_lock(vm, &exec); 1750 drm_exec_retry_on_contention(&exec); 1751 1752 if (IS_DGFX(xe) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) 1753 vm->flags |= XE_VM_FLAG_64K; 1754 1755 for_each_tile(tile, xe, id) { 1756 if (flags & XE_VM_FLAG_MIGRATION && 1757 tile->id != XE_VM_FLAG_TILE_ID(flags)) 1758 continue; 1759 1760 vm->pt_root[id] = xe_pt_create(vm, tile, xe->info.vm_max_level, 1761 &exec); 1762 if (IS_ERR(vm->pt_root[id])) { 1763 err = PTR_ERR(vm->pt_root[id]); 1764 vm->pt_root[id] = NULL; 1765 xe_vm_pt_destroy(vm); 1766 drm_exec_retry_on_contention(&exec); 1767 xe_validation_retry_on_oom(&ctx, &err); 1768 break; 1769 } 1770 } 1771 if (err) 1772 break; 1773 1774 if (xe_vm_has_scratch(vm)) { 1775 for_each_tile(tile, xe, id) { 1776 if (!vm->pt_root[id]) 1777 continue; 1778 1779 err = xe_vm_create_scratch(xe, tile, vm, &exec); 1780 if (err) { 1781 xe_vm_free_scratch(vm); 1782 xe_vm_pt_destroy(vm); 1783 drm_exec_retry_on_contention(&exec); 1784 xe_validation_retry_on_oom(&ctx, &err); 1785 break; 1786 } 1787 } 1788 if (err) 1789 break; 1790 vm->batch_invalidate_tlb = true; 1791 } 1792 1793 if (vm->flags & XE_VM_FLAG_LR_MODE) 1794 vm->batch_invalidate_tlb = false; 1795 1796 /* Fill pt_root after allocating scratch tables */ 1797 for_each_tile(tile, xe, id) { 1798 if (!vm->pt_root[id]) 1799 continue; 1800 1801 xe_pt_populate_empty(tile, vm, vm->pt_root[id]); 1802 } 1803 } 1804 if (err) 1805 goto err_close; 1806 1807 /* Kernel migration VM shouldn't have a circular loop.. */ 1808 if (!(flags & XE_VM_FLAG_MIGRATION)) { 1809 for_each_tile(tile, xe, id) { 1810 struct xe_exec_queue *q; 1811 u32 create_flags = EXEC_QUEUE_FLAG_VM; 1812 1813 if (!vm->pt_root[id]) 1814 continue; 1815 1816 if (!xef) /* Not from userspace */ 1817 create_flags |= EXEC_QUEUE_FLAG_KERNEL; 1818 1819 q = xe_exec_queue_create_bind(xe, tile, vm, create_flags, 0); 1820 if (IS_ERR(q)) { 1821 err = PTR_ERR(q); 1822 goto err_close; 1823 } 1824 vm->q[id] = q; 1825 } 1826 } 1827 1828 if (xef && xe->info.has_asid) { 1829 u32 asid; 1830 1831 down_write(&xe->usm.lock); 1832 err = xa_alloc_cyclic(&xe->usm.asid_to_vm, &asid, vm, 1833 XA_LIMIT(1, XE_MAX_ASID - 1), 1834 &xe->usm.next_asid, GFP_NOWAIT); 1835 up_write(&xe->usm.lock); 1836 if (err < 0) 1837 goto err_close; 1838 1839 vm->usm.asid = asid; 1840 } 1841 1842 trace_xe_vm_create(vm); 1843 1844 return vm; 1845 1846 err_close: 1847 xe_vm_close_and_put(vm); 1848 return ERR_PTR(err); 1849 1850 err_svm_fini: 1851 vm->size = 0; /* close the vm */ 1852 if (flags & XE_VM_FLAG_FAULT_MODE) 1853 xe_svm_close(vm); 1854 xe_svm_fini(vm); 1855 err_no_resv: 1856 mutex_destroy(&vm->snap_mutex); 1857 for_each_tile(tile, xe, id) 1858 xe_range_fence_tree_fini(&vm->rftree[id]); 1859 ttm_lru_bulk_move_fini(&xe->ttm, &vm->lru_bulk_move); 1860 if (vm->xef) 1861 xe_file_put(vm->xef); 1862 kfree(vm); 1863 if (flags & XE_VM_FLAG_LR_MODE) 1864 xe_pm_runtime_put(xe); 1865 return ERR_PTR(err); 1866 } 1867 1868 static void xe_vm_close(struct xe_vm *vm) 1869 { 1870 struct xe_device *xe = vm->xe; 1871 bool bound; 1872 int idx; 1873 1874 bound = drm_dev_enter(&xe->drm, &idx); 1875 1876 down_write(&vm->lock); 1877 if (xe_vm_in_fault_mode(vm)) 1878 xe_svm_notifier_lock(vm); 1879 1880 vm->size = 0; 1881 1882 if (!((vm->flags & XE_VM_FLAG_MIGRATION))) { 1883 struct xe_tile *tile; 1884 struct xe_gt *gt; 1885 u8 id; 1886 1887 /* Wait for pending binds */ 1888 dma_resv_wait_timeout(xe_vm_resv(vm), 1889 DMA_RESV_USAGE_BOOKKEEP, 1890 false, MAX_SCHEDULE_TIMEOUT); 1891 1892 if (bound) { 1893 for_each_tile(tile, xe, id) 1894 if (vm->pt_root[id]) 1895 xe_pt_clear(xe, vm->pt_root[id]); 1896 1897 for_each_gt(gt, xe, id) 1898 xe_tlb_inval_vm(>->tlb_inval, vm); 1899 } 1900 } 1901 1902 if (xe_vm_in_fault_mode(vm)) 1903 xe_svm_notifier_unlock(vm); 1904 up_write(&vm->lock); 1905 1906 if (bound) 1907 drm_dev_exit(idx); 1908 } 1909 1910 void xe_vm_close_and_put(struct xe_vm *vm) 1911 { 1912 LIST_HEAD(contested); 1913 struct xe_device *xe = vm->xe; 1914 struct xe_tile *tile; 1915 struct xe_vma *vma, *next_vma; 1916 struct drm_gpuva *gpuva, *next; 1917 u8 id; 1918 1919 xe_assert(xe, !vm->preempt.num_exec_queues); 1920 1921 xe_vm_close(vm); 1922 if (xe_vm_in_preempt_fence_mode(vm)) { 1923 mutex_lock(&xe->rebind_resume_lock); 1924 list_del_init(&vm->preempt.pm_activate_link); 1925 mutex_unlock(&xe->rebind_resume_lock); 1926 flush_work(&vm->preempt.rebind_work); 1927 } 1928 if (xe_vm_in_fault_mode(vm)) 1929 xe_svm_close(vm); 1930 1931 down_write(&vm->lock); 1932 for_each_tile(tile, xe, id) { 1933 if (vm->q[id]) { 1934 int i; 1935 1936 xe_exec_queue_last_fence_put(vm->q[id], vm); 1937 for_each_tlb_inval(i) 1938 xe_exec_queue_tlb_inval_last_fence_put(vm->q[id], vm, i); 1939 } 1940 } 1941 up_write(&vm->lock); 1942 1943 for_each_tile(tile, xe, id) { 1944 if (vm->q[id]) { 1945 xe_exec_queue_kill(vm->q[id]); 1946 xe_exec_queue_put(vm->q[id]); 1947 vm->q[id] = NULL; 1948 } 1949 } 1950 1951 down_write(&vm->lock); 1952 xe_vm_lock(vm, false); 1953 drm_gpuvm_for_each_va_safe(gpuva, next, &vm->gpuvm) { 1954 vma = gpuva_to_vma(gpuva); 1955 1956 if (xe_vma_has_no_bo(vma)) { 1957 xe_svm_notifier_lock(vm); 1958 vma->gpuva.flags |= XE_VMA_DESTROYED; 1959 xe_svm_notifier_unlock(vm); 1960 } 1961 1962 xe_vm_remove_vma(vm, vma); 1963 1964 /* easy case, remove from VMA? */ 1965 if (xe_vma_has_no_bo(vma) || xe_vma_bo(vma)->vm) { 1966 list_del_init(&vma->combined_links.rebind); 1967 xe_vma_destroy(vma, NULL); 1968 continue; 1969 } 1970 1971 list_move_tail(&vma->combined_links.destroy, &contested); 1972 vma->gpuva.flags |= XE_VMA_DESTROYED; 1973 } 1974 1975 xe_vm_unlock(vm); 1976 1977 /* 1978 * Unlink and destroy all contested external-BO VMAs before destroying 1979 * the page tables. Otherwise, concurrent eviction holding only bo->resv 1980 * can walk the BO's VMAs and attempt to invalidate/zap page tables that 1981 * have already been freed. 1982 */ 1983 list_for_each_entry_safe(vma, next_vma, &contested, 1984 combined_links.destroy) { 1985 list_del_init(&vma->combined_links.destroy); 1986 xe_vma_destroy_unlocked(vma); 1987 } 1988 1989 xe_vm_lock(vm, false); 1990 xe_vm_free_scratch(vm); 1991 xe_vm_pt_destroy(vm); 1992 xe_vm_unlock(vm); 1993 1994 xe_svm_fini(vm); 1995 1996 up_write(&vm->lock); 1997 1998 down_write(&xe->usm.lock); 1999 if (vm->usm.asid) { 2000 void *lookup; 2001 2002 xe_assert(xe, xe->info.has_asid); 2003 xe_assert(xe, !(vm->flags & XE_VM_FLAG_MIGRATION)); 2004 2005 lookup = xa_erase(&xe->usm.asid_to_vm, vm->usm.asid); 2006 xe_assert(xe, lookup == vm); 2007 } 2008 up_write(&xe->usm.lock); 2009 2010 xe_vm_clear_fault_entries(vm); 2011 2012 for_each_tile(tile, xe, id) 2013 xe_range_fence_tree_fini(&vm->rftree[id]); 2014 2015 xe_vm_put(vm); 2016 } 2017 2018 static void vm_destroy_work_func(struct work_struct *w) 2019 { 2020 struct xe_vm *vm = 2021 container_of(w, struct xe_vm, destroy_work); 2022 struct xe_device *xe = vm->xe; 2023 struct xe_tile *tile; 2024 u8 id; 2025 2026 /* xe_vm_close_and_put was not called? */ 2027 xe_assert(xe, !vm->size); 2028 2029 if (xe_vm_in_preempt_fence_mode(vm)) 2030 flush_work(&vm->preempt.rebind_work); 2031 2032 mutex_destroy(&vm->snap_mutex); 2033 2034 if (vm->flags & XE_VM_FLAG_LR_MODE) 2035 xe_pm_runtime_put(xe); 2036 2037 for_each_tile(tile, xe, id) 2038 XE_WARN_ON(vm->pt_root[id]); 2039 2040 trace_xe_vm_free(vm); 2041 2042 ttm_lru_bulk_move_fini(&xe->ttm, &vm->lru_bulk_move); 2043 2044 if (vm->xef) 2045 xe_file_put(vm->xef); 2046 2047 kfree(vm); 2048 } 2049 2050 static void xe_vm_free(struct drm_gpuvm *gpuvm) 2051 { 2052 struct xe_vm *vm = container_of(gpuvm, struct xe_vm, gpuvm); 2053 2054 /* To destroy the VM we need to be able to sleep */ 2055 queue_work(system_dfl_wq, &vm->destroy_work); 2056 } 2057 2058 struct xe_vm *xe_vm_lookup(struct xe_file *xef, u32 id) 2059 { 2060 struct xe_vm *vm; 2061 2062 mutex_lock(&xef->vm.lock); 2063 vm = xa_load(&xef->vm.xa, id); 2064 if (vm) 2065 xe_vm_get(vm); 2066 mutex_unlock(&xef->vm.lock); 2067 2068 return vm; 2069 } 2070 2071 u64 xe_vm_pdp4_descriptor(struct xe_vm *vm, struct xe_tile *tile) 2072 { 2073 return vm->pt_ops->pde_encode_bo(vm->pt_root[tile->id]->bo, 0); 2074 } 2075 2076 static struct xe_exec_queue * 2077 to_wait_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q) 2078 { 2079 return q ? q : vm->q[0]; 2080 } 2081 2082 static struct xe_user_fence * 2083 find_ufence_get(struct xe_sync_entry *syncs, u32 num_syncs) 2084 { 2085 unsigned int i; 2086 2087 for (i = 0; i < num_syncs; i++) { 2088 struct xe_sync_entry *e = &syncs[i]; 2089 2090 if (xe_sync_is_ufence(e)) 2091 return xe_sync_ufence_get(e); 2092 } 2093 2094 return NULL; 2095 } 2096 2097 #define ALL_DRM_XE_VM_CREATE_FLAGS (DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE | \ 2098 DRM_XE_VM_CREATE_FLAG_LR_MODE | \ 2099 DRM_XE_VM_CREATE_FLAG_FAULT_MODE | \ 2100 DRM_XE_VM_CREATE_FLAG_NO_VM_OVERCOMMIT) 2101 2102 int xe_vm_create_ioctl(struct drm_device *dev, void *data, 2103 struct drm_file *file) 2104 { 2105 struct xe_device *xe = to_xe_device(dev); 2106 struct xe_file *xef = to_xe_file(file); 2107 struct drm_xe_vm_create *args = data; 2108 struct xe_gt *wa_gt = xe_root_mmio_gt(xe); 2109 struct xe_vm *vm; 2110 u32 id; 2111 int err; 2112 u32 flags = 0; 2113 2114 if (XE_IOCTL_DBG(xe, args->extensions)) 2115 return -EINVAL; 2116 2117 if (wa_gt && XE_GT_WA(wa_gt, 22014953428)) 2118 args->flags |= DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE; 2119 2120 if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE && 2121 !xe->info.has_usm)) 2122 return -EINVAL; 2123 2124 if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 2125 return -EINVAL; 2126 2127 if (XE_IOCTL_DBG(xe, args->flags & ~ALL_DRM_XE_VM_CREATE_FLAGS)) 2128 return -EINVAL; 2129 2130 if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE && 2131 args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE && 2132 !xe->info.needs_scratch)) 2133 return -EINVAL; 2134 2135 if (XE_IOCTL_DBG(xe, !(args->flags & DRM_XE_VM_CREATE_FLAG_LR_MODE) && 2136 args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE)) 2137 return -EINVAL; 2138 2139 if (XE_IOCTL_DBG(xe, !(args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE) && 2140 args->flags & DRM_XE_VM_CREATE_FLAG_NO_VM_OVERCOMMIT)) 2141 return -EINVAL; 2142 2143 if (args->flags & DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE) 2144 flags |= XE_VM_FLAG_SCRATCH_PAGE; 2145 if (args->flags & DRM_XE_VM_CREATE_FLAG_LR_MODE) 2146 flags |= XE_VM_FLAG_LR_MODE; 2147 if (args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE) 2148 flags |= XE_VM_FLAG_FAULT_MODE; 2149 if (args->flags & DRM_XE_VM_CREATE_FLAG_NO_VM_OVERCOMMIT) 2150 flags |= XE_VM_FLAG_NO_VM_OVERCOMMIT; 2151 2152 vm = xe_vm_create(xe, flags, xef); 2153 if (IS_ERR(vm)) 2154 return PTR_ERR(vm); 2155 2156 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEM) 2157 /* Warning: Security issue - never enable by default */ 2158 args->reserved[0] = xe_bo_main_addr(vm->pt_root[0]->bo, XE_PAGE_SIZE); 2159 #endif 2160 2161 /* user id alloc must always be last in ioctl to prevent UAF */ 2162 err = xa_alloc(&xef->vm.xa, &id, vm, xa_limit_32b, GFP_KERNEL); 2163 if (err) 2164 goto err_close_and_put; 2165 2166 args->vm_id = id; 2167 2168 return 0; 2169 2170 err_close_and_put: 2171 xe_vm_close_and_put(vm); 2172 2173 return err; 2174 } 2175 2176 int xe_vm_destroy_ioctl(struct drm_device *dev, void *data, 2177 struct drm_file *file) 2178 { 2179 struct xe_device *xe = to_xe_device(dev); 2180 struct xe_file *xef = to_xe_file(file); 2181 struct drm_xe_vm_destroy *args = data; 2182 struct xe_vm *vm; 2183 int err = 0; 2184 2185 if (XE_IOCTL_DBG(xe, args->pad) || 2186 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 2187 return -EINVAL; 2188 2189 mutex_lock(&xef->vm.lock); 2190 vm = xa_load(&xef->vm.xa, args->vm_id); 2191 if (XE_IOCTL_DBG(xe, !vm)) 2192 err = -ENOENT; 2193 else if (XE_IOCTL_DBG(xe, vm->preempt.num_exec_queues)) 2194 err = -EBUSY; 2195 else 2196 xa_erase(&xef->vm.xa, args->vm_id); 2197 mutex_unlock(&xef->vm.lock); 2198 2199 if (!err) 2200 xe_vm_close_and_put(vm); 2201 2202 return err; 2203 } 2204 2205 static int xe_vm_query_vmas(struct xe_vm *vm, u64 start, u64 end) 2206 { 2207 struct drm_gpuva *gpuva; 2208 u32 num_vmas = 0; 2209 2210 lockdep_assert_held(&vm->lock); 2211 drm_gpuvm_for_each_va_range(gpuva, &vm->gpuvm, start, end) 2212 num_vmas++; 2213 2214 return num_vmas; 2215 } 2216 2217 static int get_mem_attrs(struct xe_vm *vm, u32 *num_vmas, u64 start, 2218 u64 end, struct drm_xe_mem_range_attr *attrs) 2219 { 2220 struct drm_gpuva *gpuva; 2221 int i = 0; 2222 2223 lockdep_assert_held(&vm->lock); 2224 2225 drm_gpuvm_for_each_va_range(gpuva, &vm->gpuvm, start, end) { 2226 struct xe_vma *vma = gpuva_to_vma(gpuva); 2227 2228 if (i == *num_vmas) 2229 return -ENOSPC; 2230 2231 attrs[i].start = xe_vma_start(vma); 2232 attrs[i].end = xe_vma_end(vma); 2233 attrs[i].atomic.val = vma->attr.atomic_access; 2234 attrs[i].pat_index.val = vma->attr.pat_index; 2235 attrs[i].preferred_mem_loc.devmem_fd = vma->attr.preferred_loc.devmem_fd; 2236 attrs[i].preferred_mem_loc.migration_policy = 2237 vma->attr.preferred_loc.migration_policy; 2238 2239 i++; 2240 } 2241 2242 *num_vmas = i; 2243 return 0; 2244 } 2245 2246 int xe_vm_query_vmas_attrs_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 2247 { 2248 struct xe_device *xe = to_xe_device(dev); 2249 struct xe_file *xef = to_xe_file(file); 2250 struct drm_xe_mem_range_attr *mem_attrs; 2251 struct drm_xe_vm_query_mem_range_attr *args = data; 2252 u64 __user *attrs_user = u64_to_user_ptr(args->vector_of_mem_attr); 2253 struct xe_vm *vm; 2254 int err = 0; 2255 2256 if (XE_IOCTL_DBG(xe, 2257 ((args->num_mem_ranges == 0 && 2258 (attrs_user || args->sizeof_mem_range_attr != 0)) || 2259 (args->num_mem_ranges > 0 && 2260 (!attrs_user || 2261 args->sizeof_mem_range_attr != 2262 sizeof(struct drm_xe_mem_range_attr)))))) 2263 return -EINVAL; 2264 2265 vm = xe_vm_lookup(xef, args->vm_id); 2266 if (XE_IOCTL_DBG(xe, !vm)) 2267 return -EINVAL; 2268 2269 err = down_read_interruptible(&vm->lock); 2270 if (err) 2271 goto put_vm; 2272 2273 attrs_user = u64_to_user_ptr(args->vector_of_mem_attr); 2274 2275 if (args->num_mem_ranges == 0 && !attrs_user) { 2276 args->num_mem_ranges = xe_vm_query_vmas(vm, args->start, args->start + args->range); 2277 args->sizeof_mem_range_attr = sizeof(struct drm_xe_mem_range_attr); 2278 goto unlock_vm; 2279 } 2280 2281 mem_attrs = kvmalloc_array(args->num_mem_ranges, args->sizeof_mem_range_attr, 2282 GFP_KERNEL | __GFP_ACCOUNT | 2283 __GFP_RETRY_MAYFAIL | __GFP_NOWARN); 2284 if (!mem_attrs) { 2285 err = args->num_mem_ranges > 1 ? -ENOBUFS : -ENOMEM; 2286 goto unlock_vm; 2287 } 2288 2289 memset(mem_attrs, 0, args->num_mem_ranges * args->sizeof_mem_range_attr); 2290 err = get_mem_attrs(vm, &args->num_mem_ranges, args->start, 2291 args->start + args->range, mem_attrs); 2292 if (err) 2293 goto free_mem_attrs; 2294 2295 err = copy_to_user(attrs_user, mem_attrs, 2296 args->sizeof_mem_range_attr * args->num_mem_ranges); 2297 if (err) 2298 err = -EFAULT; 2299 2300 free_mem_attrs: 2301 kvfree(mem_attrs); 2302 unlock_vm: 2303 up_read(&vm->lock); 2304 put_vm: 2305 xe_vm_put(vm); 2306 return err; 2307 } 2308 2309 static bool vma_matches(struct xe_vma *vma, u64 page_addr) 2310 { 2311 if (page_addr > xe_vma_end(vma) - 1 || 2312 page_addr + SZ_4K - 1 < xe_vma_start(vma)) 2313 return false; 2314 2315 return true; 2316 } 2317 2318 /** 2319 * xe_vm_find_vma_by_addr() - Find a VMA by its address 2320 * 2321 * @vm: the xe_vm the vma belongs to 2322 * @page_addr: address to look up 2323 */ 2324 struct xe_vma *xe_vm_find_vma_by_addr(struct xe_vm *vm, u64 page_addr) 2325 { 2326 struct xe_vma *vma = NULL; 2327 2328 if (vm->usm.last_fault_vma) { /* Fast lookup */ 2329 if (vma_matches(vm->usm.last_fault_vma, page_addr)) 2330 vma = vm->usm.last_fault_vma; 2331 } 2332 if (!vma) 2333 vma = xe_vm_find_overlapping_vma(vm, page_addr, SZ_4K); 2334 2335 return vma; 2336 } 2337 2338 static const u32 region_to_mem_type[] = { 2339 XE_PL_TT, 2340 XE_PL_VRAM0, 2341 XE_PL_VRAM1, 2342 }; 2343 2344 static void prep_vma_destroy(struct xe_vm *vm, struct xe_vma *vma, 2345 bool post_commit) 2346 { 2347 xe_svm_notifier_lock(vm); 2348 vma->gpuva.flags |= XE_VMA_DESTROYED; 2349 xe_svm_notifier_unlock(vm); 2350 if (post_commit) 2351 xe_vm_remove_vma(vm, vma); 2352 } 2353 2354 #undef ULL 2355 #define ULL unsigned long long 2356 2357 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM) 2358 static void print_op(struct xe_device *xe, struct drm_gpuva_op *op) 2359 { 2360 struct xe_vma *vma; 2361 2362 switch (op->op) { 2363 case DRM_GPUVA_OP_MAP: 2364 vm_dbg(&xe->drm, "MAP: addr=0x%016llx, range=0x%016llx", 2365 (ULL)op->map.va.addr, (ULL)op->map.va.range); 2366 break; 2367 case DRM_GPUVA_OP_REMAP: 2368 vma = gpuva_to_vma(op->remap.unmap->va); 2369 vm_dbg(&xe->drm, "REMAP:UNMAP: addr=0x%016llx, range=0x%016llx, keep=%d", 2370 (ULL)xe_vma_start(vma), (ULL)xe_vma_size(vma), 2371 op->remap.unmap->keep ? 1 : 0); 2372 if (op->remap.prev) 2373 vm_dbg(&xe->drm, 2374 "REMAP:PREV: addr=0x%016llx, range=0x%016llx", 2375 (ULL)op->remap.prev->va.addr, 2376 (ULL)op->remap.prev->va.range); 2377 if (op->remap.next) 2378 vm_dbg(&xe->drm, 2379 "REMAP:NEXT: addr=0x%016llx, range=0x%016llx", 2380 (ULL)op->remap.next->va.addr, 2381 (ULL)op->remap.next->va.range); 2382 break; 2383 case DRM_GPUVA_OP_UNMAP: 2384 vma = gpuva_to_vma(op->unmap.va); 2385 vm_dbg(&xe->drm, "UNMAP: addr=0x%016llx, range=0x%016llx, keep=%d", 2386 (ULL)xe_vma_start(vma), (ULL)xe_vma_size(vma), 2387 op->unmap.keep ? 1 : 0); 2388 break; 2389 case DRM_GPUVA_OP_PREFETCH: 2390 vma = gpuva_to_vma(op->prefetch.va); 2391 vm_dbg(&xe->drm, "PREFETCH: addr=0x%016llx, range=0x%016llx", 2392 (ULL)xe_vma_start(vma), (ULL)xe_vma_size(vma)); 2393 break; 2394 default: 2395 drm_warn(&xe->drm, "NOT POSSIBLE\n"); 2396 } 2397 } 2398 #else 2399 static void print_op(struct xe_device *xe, struct drm_gpuva_op *op) 2400 { 2401 } 2402 #endif 2403 2404 static bool __xe_vm_needs_clear_scratch_pages(struct xe_vm *vm, u32 bind_flags) 2405 { 2406 if (!xe_vm_in_fault_mode(vm)) 2407 return false; 2408 2409 if (!xe_vm_has_scratch(vm)) 2410 return false; 2411 2412 if (bind_flags & DRM_XE_VM_BIND_FLAG_IMMEDIATE) 2413 return false; 2414 2415 return true; 2416 } 2417 2418 static void xe_svm_prefetch_gpuva_ops_fini(struct drm_gpuva_ops *ops) 2419 { 2420 struct drm_gpuva_op *__op; 2421 2422 drm_gpuva_for_each_op(__op, ops) { 2423 struct xe_vma_op *op = gpuva_op_to_vma_op(__op); 2424 2425 xe_vma_svm_prefetch_op_fini(op); 2426 } 2427 } 2428 2429 /* 2430 * Create operations list from IOCTL arguments, setup operations fields so parse 2431 * and commit steps are decoupled from IOCTL arguments. This step can fail. 2432 */ 2433 static struct drm_gpuva_ops * 2434 vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_vma_ops *vops, 2435 struct xe_bo *bo, u64 bo_offset_or_userptr, 2436 u64 addr, u64 range, 2437 u32 operation, u32 flags, 2438 u32 prefetch_region, u16 pat_index) 2439 { 2440 struct drm_gem_object *obj = bo ? &bo->ttm.base : NULL; 2441 struct drm_gpuva_ops *ops; 2442 struct drm_gpuva_op *__op; 2443 struct drm_gpuvm_bo *vm_bo; 2444 u64 range_start = addr; 2445 u64 range_end = addr + range; 2446 int err; 2447 2448 lockdep_assert_held_write(&vm->lock); 2449 2450 vm_dbg(&vm->xe->drm, 2451 "op=%d, addr=0x%016llx, range=0x%016llx, bo_offset_or_userptr=0x%016llx", 2452 operation, (ULL)addr, (ULL)range, 2453 (ULL)bo_offset_or_userptr); 2454 2455 switch (operation) { 2456 case DRM_XE_VM_BIND_OP_MAP: 2457 if (flags & DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR) { 2458 xe_vm_find_cpu_addr_mirror_vma_range(vm, &range_start, &range_end); 2459 vops->flags |= XE_VMA_OPS_FLAG_ALLOW_SVM_UNMAP; 2460 } 2461 2462 fallthrough; 2463 case DRM_XE_VM_BIND_OP_MAP_USERPTR: { 2464 struct drm_gpuvm_map_req map_req = { 2465 .map.va.addr = range_start, 2466 .map.va.range = range_end - range_start, 2467 .map.gem.obj = obj, 2468 .map.gem.offset = bo_offset_or_userptr, 2469 }; 2470 2471 ops = drm_gpuvm_sm_map_ops_create(&vm->gpuvm, &map_req); 2472 break; 2473 } 2474 case DRM_XE_VM_BIND_OP_UNMAP: 2475 ops = drm_gpuvm_sm_unmap_ops_create(&vm->gpuvm, addr, range); 2476 break; 2477 case DRM_XE_VM_BIND_OP_PREFETCH: 2478 ops = drm_gpuvm_prefetch_ops_create(&vm->gpuvm, addr, range); 2479 break; 2480 case DRM_XE_VM_BIND_OP_UNMAP_ALL: 2481 xe_assert(vm->xe, bo); 2482 2483 err = xe_bo_lock(bo, true); 2484 if (err) 2485 return ERR_PTR(err); 2486 2487 vm_bo = drm_gpuvm_bo_obtain_locked(&vm->gpuvm, obj); 2488 if (IS_ERR(vm_bo)) { 2489 xe_bo_unlock(bo); 2490 return ERR_CAST(vm_bo); 2491 } 2492 2493 ops = drm_gpuvm_bo_unmap_ops_create(vm_bo); 2494 drm_gpuvm_bo_put(vm_bo); 2495 xe_bo_unlock(bo); 2496 break; 2497 default: 2498 drm_warn(&vm->xe->drm, "NOT POSSIBLE\n"); 2499 ops = ERR_PTR(-EINVAL); 2500 } 2501 if (IS_ERR(ops)) 2502 return ops; 2503 2504 drm_gpuva_for_each_op(__op, ops) { 2505 struct xe_vma_op *op = gpuva_op_to_vma_op(__op); 2506 2507 if (__op->op == DRM_GPUVA_OP_MAP) { 2508 op->map.immediate = 2509 flags & DRM_XE_VM_BIND_FLAG_IMMEDIATE; 2510 if (flags & DRM_XE_VM_BIND_FLAG_READONLY) 2511 op->map.vma_flags |= XE_VMA_READ_ONLY; 2512 if (flags & DRM_XE_VM_BIND_FLAG_NULL) 2513 op->map.vma_flags |= DRM_GPUVA_SPARSE; 2514 if (flags & DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR) 2515 op->map.vma_flags |= XE_VMA_SYSTEM_ALLOCATOR; 2516 if (flags & DRM_XE_VM_BIND_FLAG_DUMPABLE) 2517 op->map.vma_flags |= XE_VMA_DUMPABLE; 2518 if (flags & DRM_XE_VM_BIND_FLAG_MADVISE_AUTORESET) 2519 op->map.vma_flags |= XE_VMA_MADV_AUTORESET; 2520 op->map.request_decompress = flags & DRM_XE_VM_BIND_FLAG_DECOMPRESS; 2521 op->map.pat_index = pat_index; 2522 op->map.invalidate_on_bind = 2523 __xe_vm_needs_clear_scratch_pages(vm, flags); 2524 } else if (__op->op == DRM_GPUVA_OP_PREFETCH) { 2525 struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va); 2526 struct xe_tile *tile; 2527 struct xe_svm_range *svm_range; 2528 struct drm_gpusvm_ctx ctx = {}; 2529 struct drm_pagemap *dpagemap = NULL; 2530 u8 id, tile_mask = 0; 2531 u32 i; 2532 2533 if (!xe_vma_is_cpu_addr_mirror(vma)) { 2534 op->prefetch.region = prefetch_region; 2535 break; 2536 } 2537 2538 ctx.read_only = xe_vma_read_only(vma); 2539 ctx.devmem_possible = IS_DGFX(vm->xe) && 2540 IS_ENABLED(CONFIG_DRM_XE_PAGEMAP); 2541 2542 for_each_tile(tile, vm->xe, id) 2543 tile_mask |= 0x1 << id; 2544 2545 xa_init_flags(&op->prefetch_range.range, XA_FLAGS_ALLOC); 2546 op->prefetch_range.ranges_count = 0; 2547 2548 if (prefetch_region == DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC) { 2549 dpagemap = xe_vma_resolve_pagemap(vma, 2550 xe_device_get_root_tile(vm->xe)); 2551 } else if (prefetch_region) { 2552 tile = &vm->xe->tiles[region_to_mem_type[prefetch_region] - 2553 XE_PL_VRAM0]; 2554 dpagemap = xe_tile_local_pagemap(tile); 2555 } 2556 2557 op->prefetch_range.dpagemap = dpagemap; 2558 alloc_next_range: 2559 svm_range = xe_svm_range_find_or_insert(vm, addr, vma, &ctx); 2560 2561 if (PTR_ERR(svm_range) == -ENOENT) { 2562 u64 ret = xe_svm_find_vma_start(vm, addr, range_end, vma); 2563 2564 addr = ret == ULONG_MAX ? 0 : ret; 2565 if (addr) 2566 goto alloc_next_range; 2567 else 2568 goto print_op_label; 2569 } 2570 2571 if (IS_ERR(svm_range)) { 2572 err = PTR_ERR(svm_range); 2573 goto unwind_prefetch_ops; 2574 } 2575 2576 if (xe_svm_range_validate(vm, svm_range, tile_mask, dpagemap)) { 2577 xe_svm_range_debug(svm_range, "PREFETCH - RANGE IS VALID"); 2578 goto check_next_range; 2579 } 2580 2581 err = xa_alloc(&op->prefetch_range.range, 2582 &i, svm_range, xa_limit_32b, 2583 GFP_KERNEL); 2584 2585 if (err) 2586 goto unwind_prefetch_ops; 2587 2588 op->prefetch_range.ranges_count++; 2589 vops->flags |= XE_VMA_OPS_FLAG_HAS_SVM_PREFETCH; 2590 xe_svm_range_debug(svm_range, "PREFETCH - RANGE CREATED"); 2591 check_next_range: 2592 if (range_end > xe_svm_range_end(svm_range) && 2593 xe_svm_range_end(svm_range) < xe_vma_end(vma)) { 2594 addr = xe_svm_range_end(svm_range); 2595 goto alloc_next_range; 2596 } 2597 } 2598 print_op_label: 2599 print_op(vm->xe, __op); 2600 } 2601 2602 return ops; 2603 2604 unwind_prefetch_ops: 2605 xe_svm_prefetch_gpuva_ops_fini(ops); 2606 drm_gpuva_ops_free(&vm->gpuvm, ops); 2607 return ERR_PTR(err); 2608 } 2609 2610 ALLOW_ERROR_INJECTION(vm_bind_ioctl_ops_create, ERRNO); 2611 2612 static struct xe_vma *new_vma(struct xe_vm *vm, struct drm_gpuva_op_map *op, 2613 struct xe_vma_mem_attr *attr, unsigned int flags) 2614 { 2615 struct xe_bo *bo = op->gem.obj ? gem_to_xe_bo(op->gem.obj) : NULL; 2616 struct xe_validation_ctx ctx; 2617 struct drm_exec exec; 2618 struct xe_vma *vma; 2619 int err = 0; 2620 2621 lockdep_assert_held_write(&vm->lock); 2622 2623 if (bo) { 2624 err = 0; 2625 xe_validation_guard(&ctx, &vm->xe->val, &exec, 2626 (struct xe_val_flags) {.interruptible = true}, err) { 2627 if (!bo->vm) { 2628 err = drm_exec_lock_obj(&exec, xe_vm_obj(vm)); 2629 drm_exec_retry_on_contention(&exec); 2630 } 2631 if (!err) { 2632 err = drm_exec_lock_obj(&exec, &bo->ttm.base); 2633 drm_exec_retry_on_contention(&exec); 2634 } 2635 if (err) 2636 return ERR_PTR(err); 2637 2638 vma = xe_vma_create(vm, bo, op->gem.offset, 2639 op->va.addr, op->va.addr + 2640 op->va.range - 1, attr, flags); 2641 if (IS_ERR(vma)) 2642 return vma; 2643 2644 if (!bo->vm) { 2645 err = add_preempt_fences(vm, bo); 2646 if (err) { 2647 prep_vma_destroy(vm, vma, false); 2648 xe_vma_destroy(vma, NULL); 2649 } 2650 } 2651 } 2652 if (err) 2653 return ERR_PTR(err); 2654 } else { 2655 vma = xe_vma_create(vm, NULL, op->gem.offset, 2656 op->va.addr, op->va.addr + 2657 op->va.range - 1, attr, flags); 2658 if (IS_ERR(vma)) 2659 return vma; 2660 2661 if (xe_vma_is_userptr(vma)) { 2662 err = xe_vma_userptr_pin_pages(to_userptr_vma(vma)); 2663 /* 2664 * -EBUSY has dedicated meaning that a user fence 2665 * attached to the VMA is busy, in practice 2666 * xe_vma_userptr_pin_pages can only fail with -EBUSY if 2667 * we are low on memory so convert this to -ENOMEM. 2668 */ 2669 if (err == -EBUSY) 2670 err = -ENOMEM; 2671 } 2672 } 2673 if (err) { 2674 prep_vma_destroy(vm, vma, false); 2675 xe_vma_destroy_unlocked(vma); 2676 vma = ERR_PTR(err); 2677 } 2678 2679 return vma; 2680 } 2681 2682 static u64 xe_vma_max_pte_size(struct xe_vma *vma) 2683 { 2684 if (vma->gpuva.flags & XE_VMA_PTE_1G) 2685 return SZ_1G; 2686 else if (vma->gpuva.flags & (XE_VMA_PTE_2M | XE_VMA_PTE_COMPACT)) 2687 return SZ_2M; 2688 else if (vma->gpuva.flags & XE_VMA_PTE_64K) 2689 return SZ_64K; 2690 else if (vma->gpuva.flags & XE_VMA_PTE_4K) 2691 return SZ_4K; 2692 2693 return SZ_1G; /* Uninitialized, used max size */ 2694 } 2695 2696 static void xe_vma_set_pte_size(struct xe_vma *vma, u64 size) 2697 { 2698 switch (size) { 2699 case SZ_1G: 2700 vma->gpuva.flags |= XE_VMA_PTE_1G; 2701 break; 2702 case SZ_2M: 2703 vma->gpuva.flags |= XE_VMA_PTE_2M; 2704 break; 2705 case SZ_64K: 2706 vma->gpuva.flags |= XE_VMA_PTE_64K; 2707 break; 2708 case SZ_4K: 2709 vma->gpuva.flags |= XE_VMA_PTE_4K; 2710 break; 2711 } 2712 } 2713 2714 static int xe_vma_op_commit(struct xe_vm *vm, struct xe_vma_op *op) 2715 { 2716 int err = 0; 2717 2718 lockdep_assert_held_write(&vm->lock); 2719 2720 switch (op->base.op) { 2721 case DRM_GPUVA_OP_MAP: 2722 err |= xe_vm_insert_vma(vm, op->map.vma); 2723 if (!err) 2724 op->flags |= XE_VMA_OP_COMMITTED; 2725 break; 2726 case DRM_GPUVA_OP_REMAP: 2727 { 2728 u8 tile_present = 2729 gpuva_to_vma(op->base.remap.unmap->va)->tile_present; 2730 2731 prep_vma_destroy(vm, gpuva_to_vma(op->base.remap.unmap->va), 2732 true); 2733 op->flags |= XE_VMA_OP_COMMITTED; 2734 2735 if (op->remap.prev) { 2736 err |= xe_vm_insert_vma(vm, op->remap.prev); 2737 if (!err) 2738 op->flags |= XE_VMA_OP_PREV_COMMITTED; 2739 if (!err && op->remap.skip_prev) { 2740 op->remap.prev->tile_present = 2741 tile_present; 2742 } 2743 } 2744 if (op->remap.next) { 2745 err |= xe_vm_insert_vma(vm, op->remap.next); 2746 if (!err) 2747 op->flags |= XE_VMA_OP_NEXT_COMMITTED; 2748 if (!err && op->remap.skip_next) { 2749 op->remap.next->tile_present = 2750 tile_present; 2751 } 2752 } 2753 2754 /* 2755 * Adjust for partial unbind after removing VMA from VM. In case 2756 * of unwind we might need to undo this later. 2757 */ 2758 if (!err) { 2759 op->base.remap.unmap->va->va.addr = op->remap.start; 2760 op->base.remap.unmap->va->va.range = op->remap.range; 2761 } 2762 break; 2763 } 2764 case DRM_GPUVA_OP_UNMAP: 2765 prep_vma_destroy(vm, gpuva_to_vma(op->base.unmap.va), true); 2766 op->flags |= XE_VMA_OP_COMMITTED; 2767 break; 2768 case DRM_GPUVA_OP_PREFETCH: 2769 op->flags |= XE_VMA_OP_COMMITTED; 2770 break; 2771 default: 2772 drm_warn(&vm->xe->drm, "NOT POSSIBLE\n"); 2773 } 2774 2775 return err; 2776 } 2777 2778 /** 2779 * xe_vma_has_default_mem_attrs - Check if a VMA has default memory attributes 2780 * @vma: Pointer to the xe_vma structure to check 2781 * 2782 * This function determines whether the given VMA (Virtual Memory Area) 2783 * has its memory attributes set to their default values. Specifically, 2784 * it checks the following conditions: 2785 * 2786 * - `atomic_access` is `DRM_XE_VMA_ATOMIC_UNDEFINED` 2787 * - `pat_index` is equal to `default_pat_index` 2788 * - `preferred_loc.devmem_fd` is `DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE` 2789 * - `preferred_loc.migration_policy` is `DRM_XE_MIGRATE_ALL_PAGES` 2790 * 2791 * Return: true if all attributes are at their default values, false otherwise. 2792 */ 2793 bool xe_vma_has_default_mem_attrs(struct xe_vma *vma) 2794 { 2795 return (vma->attr.atomic_access == DRM_XE_ATOMIC_UNDEFINED && 2796 vma->attr.pat_index == vma->attr.default_pat_index && 2797 vma->attr.preferred_loc.devmem_fd == DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE && 2798 vma->attr.preferred_loc.migration_policy == DRM_XE_MIGRATE_ALL_PAGES); 2799 } 2800 2801 static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct drm_gpuva_ops *ops, 2802 struct xe_vma_ops *vops) 2803 { 2804 struct xe_device *xe = vm->xe; 2805 struct drm_gpuva_op *__op; 2806 struct xe_tile *tile; 2807 u8 id, tile_mask = 0; 2808 int err = 0; 2809 2810 lockdep_assert_held_write(&vm->lock); 2811 2812 for_each_tile(tile, vm->xe, id) 2813 tile_mask |= 0x1 << id; 2814 2815 drm_gpuva_for_each_op(__op, ops) { 2816 struct xe_vma_op *op = gpuva_op_to_vma_op(__op); 2817 struct xe_vma *vma; 2818 unsigned int flags = 0; 2819 2820 INIT_LIST_HEAD(&op->link); 2821 list_add_tail(&op->link, &vops->list); 2822 op->tile_mask = tile_mask; 2823 2824 switch (op->base.op) { 2825 case DRM_GPUVA_OP_MAP: 2826 { 2827 struct xe_vma_mem_attr default_attr = { 2828 .preferred_loc = { 2829 .devmem_fd = DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE, 2830 .migration_policy = DRM_XE_MIGRATE_ALL_PAGES, 2831 }, 2832 .atomic_access = DRM_XE_ATOMIC_UNDEFINED, 2833 .default_pat_index = op->map.pat_index, 2834 .pat_index = op->map.pat_index, 2835 .purgeable_state = XE_MADV_PURGEABLE_WILLNEED, 2836 }; 2837 2838 flags |= op->map.vma_flags & XE_VMA_CREATE_MASK; 2839 2840 vma = new_vma(vm, &op->base.map, &default_attr, 2841 flags); 2842 if (IS_ERR(vma)) 2843 return PTR_ERR(vma); 2844 2845 op->map.vma = vma; 2846 if (((op->map.immediate || !xe_vm_in_fault_mode(vm)) && 2847 !(op->map.vma_flags & XE_VMA_SYSTEM_ALLOCATOR)) || 2848 op->map.invalidate_on_bind) 2849 xe_vma_ops_incr_pt_update_ops(vops, 2850 op->tile_mask, 1); 2851 break; 2852 } 2853 case DRM_GPUVA_OP_REMAP: 2854 { 2855 struct xe_vma *old = 2856 gpuva_to_vma(op->base.remap.unmap->va); 2857 bool skip = xe_vma_is_cpu_addr_mirror(old); 2858 u64 start = xe_vma_start(old), end = xe_vma_end(old); 2859 int num_remap_ops = 0; 2860 2861 if (op->base.remap.prev) 2862 start = op->base.remap.prev->va.addr + 2863 op->base.remap.prev->va.range; 2864 if (op->base.remap.next) 2865 end = op->base.remap.next->va.addr; 2866 2867 if (xe_vma_is_cpu_addr_mirror(old) && 2868 xe_svm_has_mapping(vm, start, end)) { 2869 if (vops->flags & XE_VMA_OPS_FLAG_MADVISE) 2870 xe_svm_unmap_address_range(vm, start, end); 2871 else 2872 return -EBUSY; 2873 } 2874 2875 op->remap.start = xe_vma_start(old); 2876 op->remap.range = xe_vma_size(old); 2877 op->remap.old_start = op->remap.start; 2878 op->remap.old_range = op->remap.range; 2879 2880 flags |= op->base.remap.unmap->va->flags & XE_VMA_CREATE_MASK; 2881 if (op->base.remap.prev) { 2882 vma = new_vma(vm, op->base.remap.prev, 2883 &old->attr, flags); 2884 if (IS_ERR(vma)) 2885 return PTR_ERR(vma); 2886 2887 op->remap.prev = vma; 2888 2889 /* 2890 * Userptr creates a new SG mapping so 2891 * we must also rebind. 2892 */ 2893 op->remap.skip_prev = skip || 2894 (!xe_vma_is_userptr(old) && 2895 IS_ALIGNED(xe_vma_end(vma), 2896 xe_vma_max_pte_size(old))); 2897 if (op->remap.skip_prev) { 2898 xe_vma_set_pte_size(vma, xe_vma_max_pte_size(old)); 2899 op->remap.range -= 2900 xe_vma_end(vma) - 2901 xe_vma_start(old); 2902 op->remap.start = xe_vma_end(vma); 2903 vm_dbg(&xe->drm, "REMAP:SKIP_PREV: addr=0x%016llx, range=0x%016llx", 2904 (ULL)op->remap.start, 2905 (ULL)op->remap.range); 2906 } else { 2907 num_remap_ops++; 2908 } 2909 } 2910 2911 if (op->base.remap.next) { 2912 vma = new_vma(vm, op->base.remap.next, 2913 &old->attr, flags); 2914 if (IS_ERR(vma)) 2915 return PTR_ERR(vma); 2916 2917 op->remap.next = vma; 2918 2919 /* 2920 * Userptr creates a new SG mapping so 2921 * we must also rebind. 2922 */ 2923 op->remap.skip_next = skip || 2924 (!xe_vma_is_userptr(old) && 2925 IS_ALIGNED(xe_vma_start(vma), 2926 xe_vma_max_pte_size(old))); 2927 if (op->remap.skip_next) { 2928 xe_vma_set_pte_size(vma, xe_vma_max_pte_size(old)); 2929 op->remap.range -= 2930 xe_vma_end(old) - 2931 xe_vma_start(vma); 2932 vm_dbg(&xe->drm, "REMAP:SKIP_NEXT: addr=0x%016llx, range=0x%016llx", 2933 (ULL)op->remap.start, 2934 (ULL)op->remap.range); 2935 } else { 2936 num_remap_ops++; 2937 } 2938 } 2939 if (!skip) 2940 num_remap_ops++; 2941 2942 xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask, num_remap_ops); 2943 break; 2944 } 2945 case DRM_GPUVA_OP_UNMAP: 2946 vma = gpuva_to_vma(op->base.unmap.va); 2947 2948 if (xe_vma_is_cpu_addr_mirror(vma) && 2949 xe_svm_has_mapping(vm, xe_vma_start(vma), 2950 xe_vma_end(vma)) && 2951 !(vops->flags & XE_VMA_OPS_FLAG_ALLOW_SVM_UNMAP)) 2952 return -EBUSY; 2953 2954 if (!xe_vma_is_cpu_addr_mirror(vma)) 2955 xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask, 1); 2956 break; 2957 case DRM_GPUVA_OP_PREFETCH: 2958 vma = gpuva_to_vma(op->base.prefetch.va); 2959 2960 if (xe_vma_is_userptr(vma)) { 2961 err = xe_vma_userptr_pin_pages(to_userptr_vma(vma)); 2962 if (err) 2963 return err; 2964 } 2965 2966 if (xe_vma_is_cpu_addr_mirror(vma)) 2967 xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask, 2968 op->prefetch_range.ranges_count); 2969 else 2970 xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask, 1); 2971 2972 break; 2973 default: 2974 drm_warn(&vm->xe->drm, "NOT POSSIBLE\n"); 2975 } 2976 2977 err = xe_vma_op_commit(vm, op); 2978 if (err) 2979 return err; 2980 } 2981 2982 return 0; 2983 } 2984 2985 static void xe_vma_op_unwind(struct xe_vm *vm, struct xe_vma_op *op, 2986 bool post_commit, bool prev_post_commit, 2987 bool next_post_commit) 2988 { 2989 lockdep_assert_held_write(&vm->lock); 2990 2991 switch (op->base.op) { 2992 case DRM_GPUVA_OP_MAP: 2993 if (op->map.vma) { 2994 prep_vma_destroy(vm, op->map.vma, post_commit); 2995 xe_vma_destroy_unlocked(op->map.vma); 2996 } 2997 break; 2998 case DRM_GPUVA_OP_UNMAP: 2999 { 3000 struct xe_vma *vma = gpuva_to_vma(op->base.unmap.va); 3001 3002 if (vma) { 3003 xe_svm_notifier_lock(vm); 3004 vma->gpuva.flags &= ~XE_VMA_DESTROYED; 3005 xe_svm_notifier_unlock(vm); 3006 if (post_commit) 3007 xe_vm_insert_vma(vm, vma); 3008 } 3009 break; 3010 } 3011 case DRM_GPUVA_OP_REMAP: 3012 { 3013 struct xe_vma *vma = gpuva_to_vma(op->base.remap.unmap->va); 3014 3015 if (op->remap.prev) { 3016 prep_vma_destroy(vm, op->remap.prev, prev_post_commit); 3017 xe_vma_destroy_unlocked(op->remap.prev); 3018 } 3019 if (op->remap.next) { 3020 prep_vma_destroy(vm, op->remap.next, next_post_commit); 3021 xe_vma_destroy_unlocked(op->remap.next); 3022 } 3023 if (vma) { 3024 xe_svm_notifier_lock(vm); 3025 vma->gpuva.flags &= ~XE_VMA_DESTROYED; 3026 xe_svm_notifier_unlock(vm); 3027 if (post_commit) { 3028 /* 3029 * Restore the old va range, in case of the 3030 * prev/next skip optimisation. Otherwise what 3031 * we re-insert here could be smaller than the 3032 * original range. 3033 */ 3034 op->base.remap.unmap->va->va.addr = 3035 op->remap.old_start; 3036 op->base.remap.unmap->va->va.range = 3037 op->remap.old_range; 3038 xe_vm_insert_vma(vm, vma); 3039 } 3040 } 3041 break; 3042 } 3043 case DRM_GPUVA_OP_PREFETCH: 3044 /* Nothing to do */ 3045 break; 3046 default: 3047 drm_warn(&vm->xe->drm, "NOT POSSIBLE\n"); 3048 } 3049 } 3050 3051 static void vm_bind_ioctl_ops_unwind(struct xe_vm *vm, 3052 struct drm_gpuva_ops **ops, 3053 int num_ops_list) 3054 { 3055 int i; 3056 3057 for (i = num_ops_list - 1; i >= 0; --i) { 3058 struct drm_gpuva_ops *__ops = ops[i]; 3059 struct drm_gpuva_op *__op; 3060 3061 if (!__ops) 3062 continue; 3063 3064 drm_gpuva_for_each_op_reverse(__op, __ops) { 3065 struct xe_vma_op *op = gpuva_op_to_vma_op(__op); 3066 3067 xe_vma_op_unwind(vm, op, 3068 op->flags & XE_VMA_OP_COMMITTED, 3069 op->flags & XE_VMA_OP_PREV_COMMITTED, 3070 op->flags & XE_VMA_OP_NEXT_COMMITTED); 3071 } 3072 } 3073 } 3074 3075 /** 3076 * struct xe_vma_lock_and_validate_flags - Flags for vma_lock_and_validate() 3077 * @res_evict: Allow evicting resources during validation 3078 * @validate: Perform BO validation 3079 * @request_decompress: Request BO decompression 3080 * @check_purged: Reject operation if BO is DONTNEED or PURGED 3081 */ 3082 struct xe_vma_lock_and_validate_flags { 3083 u32 res_evict : 1; 3084 u32 validate : 1; 3085 u32 request_decompress : 1; 3086 u32 check_purged : 1; 3087 }; 3088 3089 static int vma_lock_and_validate(struct drm_exec *exec, struct xe_vma *vma, 3090 struct xe_vma_lock_and_validate_flags flags) 3091 { 3092 struct xe_bo *bo = xe_vma_bo(vma); 3093 struct xe_vm *vm = xe_vma_vm(vma); 3094 bool validate_bo = flags.validate; 3095 int err = 0; 3096 3097 if (bo) { 3098 if (!bo->vm) 3099 err = drm_exec_lock_obj(exec, &bo->ttm.base); 3100 3101 /* Reject new mappings to DONTNEED/purged BOs; allow cleanup operations */ 3102 if (!err && flags.check_purged) { 3103 if (xe_bo_madv_is_dontneed(bo)) 3104 err = -EBUSY; /* BO marked purgeable */ 3105 else if (xe_bo_is_purged(bo)) 3106 err = -EINVAL; /* BO already purged */ 3107 } 3108 3109 /* Don't validate the BO for DONTNEED/PURGED remap remnants. */ 3110 if (vma->attr.purgeable_state != XE_MADV_PURGEABLE_WILLNEED) 3111 validate_bo = false; 3112 3113 if (!err && validate_bo) 3114 err = xe_bo_validate(bo, vm, 3115 xe_vm_allow_vm_eviction(vm) && 3116 flags.res_evict, exec); 3117 3118 if (err) 3119 return err; 3120 3121 if (flags.request_decompress) 3122 err = xe_bo_decompress(bo); 3123 } 3124 3125 return err; 3126 } 3127 3128 static int check_ufence(struct xe_vma *vma) 3129 { 3130 if (vma->ufence) { 3131 struct xe_user_fence * const f = vma->ufence; 3132 3133 if (!xe_sync_ufence_get_status(f)) 3134 return -EBUSY; 3135 3136 vma->ufence = NULL; 3137 xe_sync_ufence_put(f); 3138 } 3139 3140 return 0; 3141 } 3142 3143 static int prefetch_ranges(struct xe_vm *vm, struct xe_vma_op *op) 3144 { 3145 bool devmem_possible = IS_DGFX(vm->xe) && IS_ENABLED(CONFIG_DRM_XE_PAGEMAP); 3146 struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va); 3147 struct drm_pagemap *dpagemap = op->prefetch_range.dpagemap; 3148 int err = 0; 3149 3150 struct xe_svm_range *svm_range; 3151 struct drm_gpusvm_ctx ctx = {}; 3152 unsigned long i; 3153 3154 if (!xe_vma_is_cpu_addr_mirror(vma)) 3155 return 0; 3156 3157 ctx.read_only = xe_vma_read_only(vma); 3158 ctx.devmem_possible = devmem_possible; 3159 ctx.check_pages_threshold = devmem_possible ? SZ_64K : 0; 3160 ctx.device_private_page_owner = xe_svm_private_page_owner(vm, !dpagemap); 3161 3162 /* TODO: Threading the migration */ 3163 xa_for_each(&op->prefetch_range.range, i, svm_range) { 3164 if (!dpagemap) 3165 xe_svm_range_migrate_to_smem(vm, svm_range); 3166 3167 if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)) { 3168 drm_dbg(&vm->xe->drm, 3169 "Prefetch pagemap is %s start 0x%016lx end 0x%016lx\n", 3170 dpagemap ? dpagemap->drm->unique : "system", 3171 xe_svm_range_start(svm_range), xe_svm_range_end(svm_range)); 3172 } 3173 3174 if (xe_svm_range_needs_migrate_to_vram(svm_range, vma, dpagemap)) { 3175 err = xe_svm_alloc_vram(svm_range, &ctx, dpagemap); 3176 if (err) { 3177 drm_dbg(&vm->xe->drm, "VRAM allocation failed, retry from userspace, asid=%u, gpusvm=%p, errno=%pe\n", 3178 vm->usm.asid, &vm->svm.gpusvm, ERR_PTR(err)); 3179 return -ENODATA; 3180 } 3181 xe_svm_range_debug(svm_range, "PREFETCH - RANGE MIGRATED TO VRAM"); 3182 } 3183 3184 err = xe_svm_range_get_pages(vm, svm_range, &ctx); 3185 if (err) { 3186 drm_dbg(&vm->xe->drm, "Get pages failed, asid=%u, gpusvm=%p, errno=%pe\n", 3187 vm->usm.asid, &vm->svm.gpusvm, ERR_PTR(err)); 3188 if (err == -EOPNOTSUPP || err == -EFAULT || err == -EPERM) 3189 err = -ENODATA; 3190 return err; 3191 } 3192 xe_svm_range_debug(svm_range, "PREFETCH - RANGE GET PAGES DONE"); 3193 } 3194 3195 return err; 3196 } 3197 3198 static int op_lock_and_prep(struct drm_exec *exec, struct xe_vm *vm, 3199 struct xe_vma_ops *vops, struct xe_vma_op *op) 3200 { 3201 int err = 0; 3202 bool res_evict; 3203 3204 /* 3205 * We only allow evicting a BO within the VM if it is not part of an 3206 * array of binds, as an array of binds can evict another BO within the 3207 * bind. 3208 */ 3209 res_evict = !(vops->flags & XE_VMA_OPS_ARRAY_OF_BINDS); 3210 3211 switch (op->base.op) { 3212 case DRM_GPUVA_OP_MAP: 3213 if (!op->map.invalidate_on_bind) 3214 err = vma_lock_and_validate(exec, op->map.vma, 3215 (struct xe_vma_lock_and_validate_flags) { 3216 .res_evict = res_evict, 3217 .validate = !xe_vm_in_fault_mode(vm) || 3218 op->map.immediate, 3219 .request_decompress = 3220 op->map.request_decompress, 3221 .check_purged = false, 3222 }); 3223 break; 3224 case DRM_GPUVA_OP_REMAP: 3225 err = check_ufence(gpuva_to_vma(op->base.remap.unmap->va)); 3226 if (err) 3227 break; 3228 3229 err = vma_lock_and_validate(exec, 3230 gpuva_to_vma(op->base.remap.unmap->va), 3231 (struct xe_vma_lock_and_validate_flags) { 3232 .res_evict = res_evict, 3233 .validate = false, 3234 .request_decompress = false, 3235 .check_purged = false, 3236 }); 3237 if (!err && op->remap.prev) 3238 err = vma_lock_and_validate(exec, op->remap.prev, 3239 (struct xe_vma_lock_and_validate_flags) { 3240 .res_evict = res_evict, 3241 .validate = true, 3242 .request_decompress = false, 3243 .check_purged = false, 3244 }); 3245 if (!err && op->remap.next) 3246 err = vma_lock_and_validate(exec, op->remap.next, 3247 (struct xe_vma_lock_and_validate_flags) { 3248 .res_evict = res_evict, 3249 .validate = true, 3250 .request_decompress = false, 3251 .check_purged = false, 3252 }); 3253 break; 3254 case DRM_GPUVA_OP_UNMAP: 3255 err = check_ufence(gpuva_to_vma(op->base.unmap.va)); 3256 if (err) 3257 break; 3258 3259 err = vma_lock_and_validate(exec, 3260 gpuva_to_vma(op->base.unmap.va), 3261 (struct xe_vma_lock_and_validate_flags) { 3262 .res_evict = res_evict, 3263 .validate = false, 3264 .request_decompress = false, 3265 .check_purged = false, 3266 }); 3267 break; 3268 case DRM_GPUVA_OP_PREFETCH: 3269 { 3270 struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va); 3271 u32 region; 3272 3273 if (!xe_vma_is_cpu_addr_mirror(vma)) { 3274 region = op->prefetch.region; 3275 xe_assert(vm->xe, region == DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC || 3276 region <= ARRAY_SIZE(region_to_mem_type)); 3277 } 3278 3279 /* 3280 * PREFETCH is the only op that still gates on BO purge state. 3281 * MAP/REMAP handle this inside xe_vma_create() so partial 3282 * unbind on a DONTNEED BO still works. PREFETCH skips 3283 * xe_vma_create() and would migrate a BO with no backing 3284 * store, so reject DONTNEED/PURGED here. 3285 */ 3286 err = vma_lock_and_validate(exec, 3287 gpuva_to_vma(op->base.prefetch.va), 3288 (struct xe_vma_lock_and_validate_flags) { 3289 .res_evict = res_evict, 3290 .validate = false, 3291 .request_decompress = false, 3292 .check_purged = true, 3293 }); 3294 if (!err && !xe_vma_has_no_bo(vma)) { 3295 struct xe_bo *bo = xe_vma_bo(vma); 3296 u32 mem_type; 3297 3298 if (region == DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC) { 3299 unsigned int i; 3300 3301 mem_type = XE_PL_TT; 3302 for (i = 0; i < bo->placement.num_placement; i++) { 3303 if (mem_type_is_vram(bo->placements[i].mem_type)) { 3304 mem_type = bo->placements[i].mem_type; 3305 break; 3306 } 3307 } 3308 } else { 3309 mem_type = region_to_mem_type[region]; 3310 } 3311 3312 err = xe_bo_migrate(bo, mem_type, NULL, exec); 3313 } 3314 break; 3315 } 3316 default: 3317 drm_warn(&vm->xe->drm, "NOT POSSIBLE\n"); 3318 } 3319 3320 return err; 3321 } 3322 3323 static int vm_bind_ioctl_ops_prefetch_ranges(struct xe_vm *vm, struct xe_vma_ops *vops) 3324 { 3325 struct xe_vma_op *op; 3326 int err; 3327 3328 if (!(vops->flags & XE_VMA_OPS_FLAG_HAS_SVM_PREFETCH)) 3329 return 0; 3330 3331 list_for_each_entry(op, &vops->list, link) { 3332 if (op->base.op == DRM_GPUVA_OP_PREFETCH) { 3333 err = prefetch_ranges(vm, op); 3334 if (err) 3335 return err; 3336 } 3337 } 3338 3339 return 0; 3340 } 3341 3342 static int vm_bind_ioctl_ops_lock_and_prep(struct drm_exec *exec, 3343 struct xe_vm *vm, 3344 struct xe_vma_ops *vops) 3345 { 3346 struct xe_vma_op *op; 3347 int err; 3348 3349 err = drm_exec_lock_obj(exec, xe_vm_obj(vm)); 3350 if (err) 3351 return err; 3352 3353 list_for_each_entry(op, &vops->list, link) { 3354 err = op_lock_and_prep(exec, vm, vops, op); 3355 if (err) 3356 return err; 3357 } 3358 3359 #ifdef TEST_VM_OPS_ERROR 3360 if (vops->inject_error && 3361 vm->xe->vm_inject_error_position == FORCE_OP_ERROR_LOCK) 3362 return -ENOSPC; 3363 #endif 3364 3365 return 0; 3366 } 3367 3368 static void op_trace(struct xe_vma_op *op) 3369 { 3370 switch (op->base.op) { 3371 case DRM_GPUVA_OP_MAP: 3372 trace_xe_vma_bind(op->map.vma); 3373 break; 3374 case DRM_GPUVA_OP_REMAP: 3375 trace_xe_vma_unbind(gpuva_to_vma(op->base.remap.unmap->va)); 3376 if (op->remap.prev) 3377 trace_xe_vma_bind(op->remap.prev); 3378 if (op->remap.next) 3379 trace_xe_vma_bind(op->remap.next); 3380 break; 3381 case DRM_GPUVA_OP_UNMAP: 3382 trace_xe_vma_unbind(gpuva_to_vma(op->base.unmap.va)); 3383 break; 3384 case DRM_GPUVA_OP_PREFETCH: 3385 trace_xe_vma_bind(gpuva_to_vma(op->base.prefetch.va)); 3386 break; 3387 case DRM_GPUVA_OP_DRIVER: 3388 break; 3389 default: 3390 XE_WARN_ON("NOT POSSIBLE"); 3391 } 3392 } 3393 3394 static void trace_xe_vm_ops_execute(struct xe_vma_ops *vops) 3395 { 3396 struct xe_vma_op *op; 3397 3398 list_for_each_entry(op, &vops->list, link) 3399 op_trace(op); 3400 } 3401 3402 static int vm_ops_setup_tile_args(struct xe_vm *vm, struct xe_vma_ops *vops) 3403 { 3404 struct xe_exec_queue *q = vops->q; 3405 struct xe_tile *tile; 3406 int number_tiles = 0; 3407 u8 id; 3408 3409 for_each_tile(tile, vm->xe, id) { 3410 if (vops->pt_update_ops[id].num_ops) 3411 ++number_tiles; 3412 3413 if (vops->pt_update_ops[id].q) 3414 continue; 3415 3416 if (q) { 3417 vops->pt_update_ops[id].q = q; 3418 if (vm->pt_root[id] && !list_empty(&q->multi_gt_list)) 3419 q = list_next_entry(q, multi_gt_list); 3420 } else { 3421 vops->pt_update_ops[id].q = vm->q[id]; 3422 } 3423 } 3424 3425 return number_tiles; 3426 } 3427 3428 static struct dma_fence *ops_execute(struct xe_vm *vm, 3429 struct xe_vma_ops *vops) 3430 { 3431 struct xe_tile *tile; 3432 struct dma_fence *fence = NULL; 3433 struct dma_fence **fences = NULL; 3434 struct dma_fence_array *cf = NULL; 3435 int number_tiles = 0, current_fence = 0, n_fence = 0, err, i; 3436 u8 id; 3437 3438 number_tiles = vm_ops_setup_tile_args(vm, vops); 3439 if (number_tiles == 0) 3440 return ERR_PTR(-ENODATA); 3441 3442 for_each_tile(tile, vm->xe, id) { 3443 ++n_fence; 3444 3445 if (!(vops->flags & XE_VMA_OPS_FLAG_SKIP_TLB_WAIT)) 3446 for_each_tlb_inval(i) 3447 ++n_fence; 3448 } 3449 3450 fences = kmalloc_objs(*fences, n_fence); 3451 if (!fences) { 3452 fence = ERR_PTR(-ENOMEM); 3453 goto err_trace; 3454 } 3455 3456 cf = dma_fence_array_alloc(n_fence); 3457 if (!cf) { 3458 fence = ERR_PTR(-ENOMEM); 3459 goto err_out; 3460 } 3461 3462 for_each_tile(tile, vm->xe, id) { 3463 if (!vops->pt_update_ops[id].num_ops) 3464 continue; 3465 3466 err = xe_pt_update_ops_prepare(tile, vops); 3467 if (err) { 3468 fence = ERR_PTR(err); 3469 goto err_out; 3470 } 3471 } 3472 3473 trace_xe_vm_ops_execute(vops); 3474 3475 for_each_tile(tile, vm->xe, id) { 3476 struct xe_exec_queue *q = vops->pt_update_ops[tile->id].q; 3477 3478 fence = NULL; 3479 if (!vops->pt_update_ops[id].num_ops) 3480 goto collect_fences; 3481 3482 fence = xe_pt_update_ops_run(tile, vops); 3483 if (IS_ERR(fence)) 3484 goto err_out; 3485 3486 collect_fences: 3487 fences[current_fence++] = fence ?: dma_fence_get_stub(); 3488 if (vops->flags & XE_VMA_OPS_FLAG_SKIP_TLB_WAIT) 3489 continue; 3490 3491 xe_migrate_job_lock(tile->migrate, q); 3492 for_each_tlb_inval(i) 3493 fences[current_fence++] = 3494 xe_exec_queue_tlb_inval_last_fence_get(q, vm, i); 3495 xe_migrate_job_unlock(tile->migrate, q); 3496 } 3497 3498 xe_assert(vm->xe, current_fence == n_fence); 3499 dma_fence_array_init(cf, n_fence, fences, dma_fence_context_alloc(1), 3500 1); 3501 fence = &cf->base; 3502 3503 for_each_tile(tile, vm->xe, id) { 3504 if (!vops->pt_update_ops[id].num_ops) 3505 continue; 3506 3507 xe_pt_update_ops_fini(tile, vops); 3508 } 3509 3510 return fence; 3511 3512 err_out: 3513 for_each_tile(tile, vm->xe, id) { 3514 if (!vops->pt_update_ops[id].num_ops) 3515 continue; 3516 3517 xe_pt_update_ops_abort(tile, vops); 3518 } 3519 while (current_fence) 3520 dma_fence_put(fences[--current_fence]); 3521 kfree(fences); 3522 kfree(cf); 3523 3524 err_trace: 3525 trace_xe_vm_ops_fail(vm); 3526 return fence; 3527 } 3528 3529 static void vma_add_ufence(struct xe_vma *vma, struct xe_user_fence *ufence) 3530 { 3531 if (vma->ufence) 3532 xe_sync_ufence_put(vma->ufence); 3533 vma->ufence = __xe_sync_ufence_get(ufence); 3534 } 3535 3536 static void op_add_ufence(struct xe_vm *vm, struct xe_vma_op *op, 3537 struct xe_user_fence *ufence) 3538 { 3539 switch (op->base.op) { 3540 case DRM_GPUVA_OP_MAP: 3541 if (!xe_vma_is_cpu_addr_mirror(op->map.vma)) 3542 vma_add_ufence(op->map.vma, ufence); 3543 break; 3544 case DRM_GPUVA_OP_REMAP: 3545 if (op->remap.prev) 3546 vma_add_ufence(op->remap.prev, ufence); 3547 if (op->remap.next) 3548 vma_add_ufence(op->remap.next, ufence); 3549 break; 3550 case DRM_GPUVA_OP_UNMAP: 3551 break; 3552 case DRM_GPUVA_OP_PREFETCH: 3553 vma_add_ufence(gpuva_to_vma(op->base.prefetch.va), ufence); 3554 break; 3555 default: 3556 drm_warn(&vm->xe->drm, "NOT POSSIBLE\n"); 3557 } 3558 } 3559 3560 static void vm_bind_ioctl_ops_fini(struct xe_vm *vm, struct xe_vma_ops *vops, 3561 struct dma_fence *fence) 3562 { 3563 struct xe_user_fence *ufence; 3564 struct xe_vma_op *op; 3565 int i; 3566 3567 ufence = find_ufence_get(vops->syncs, vops->num_syncs); 3568 list_for_each_entry(op, &vops->list, link) { 3569 if (ufence) 3570 op_add_ufence(vm, op, ufence); 3571 3572 if (op->base.op == DRM_GPUVA_OP_UNMAP) 3573 xe_vma_destroy(gpuva_to_vma(op->base.unmap.va), fence); 3574 else if (op->base.op == DRM_GPUVA_OP_REMAP) 3575 xe_vma_destroy(gpuva_to_vma(op->base.remap.unmap->va), 3576 fence); 3577 } 3578 if (ufence) 3579 xe_sync_ufence_put(ufence); 3580 if (fence) { 3581 for (i = 0; i < vops->num_syncs; i++) 3582 xe_sync_entry_signal(vops->syncs + i, fence); 3583 } 3584 } 3585 3586 static struct dma_fence *vm_bind_ioctl_ops_execute(struct xe_vm *vm, 3587 struct xe_vma_ops *vops) 3588 { 3589 struct xe_validation_ctx ctx; 3590 struct drm_exec exec; 3591 struct dma_fence *fence; 3592 int err = 0; 3593 3594 lockdep_assert_held_write(&vm->lock); 3595 3596 xe_validation_guard(&ctx, &vm->xe->val, &exec, 3597 ((struct xe_val_flags) { 3598 .interruptible = true, 3599 .exec_ignore_duplicates = true, 3600 }), err) { 3601 err = vm_bind_ioctl_ops_lock_and_prep(&exec, vm, vops); 3602 drm_exec_retry_on_contention(&exec); 3603 xe_validation_retry_on_oom(&ctx, &err); 3604 if (err) 3605 return ERR_PTR(err); 3606 3607 xe_vm_set_validation_exec(vm, &exec); 3608 fence = ops_execute(vm, vops); 3609 xe_vm_set_validation_exec(vm, NULL); 3610 if (IS_ERR(fence)) { 3611 if (PTR_ERR(fence) == -ENODATA) 3612 vm_bind_ioctl_ops_fini(vm, vops, NULL); 3613 return fence; 3614 } 3615 3616 vm_bind_ioctl_ops_fini(vm, vops, fence); 3617 } 3618 3619 return err ? ERR_PTR(err) : fence; 3620 } 3621 ALLOW_ERROR_INJECTION(vm_bind_ioctl_ops_execute, ERRNO); 3622 3623 #define SUPPORTED_FLAGS_STUB \ 3624 (DRM_XE_VM_BIND_FLAG_READONLY | \ 3625 DRM_XE_VM_BIND_FLAG_IMMEDIATE | \ 3626 DRM_XE_VM_BIND_FLAG_NULL | \ 3627 DRM_XE_VM_BIND_FLAG_DUMPABLE | \ 3628 DRM_XE_VM_BIND_FLAG_CHECK_PXP | \ 3629 DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR | \ 3630 DRM_XE_VM_BIND_FLAG_MADVISE_AUTORESET | \ 3631 DRM_XE_VM_BIND_FLAG_DECOMPRESS) 3632 3633 #ifdef TEST_VM_OPS_ERROR 3634 #define SUPPORTED_FLAGS (SUPPORTED_FLAGS_STUB | FORCE_OP_ERROR) 3635 #else 3636 #define SUPPORTED_FLAGS SUPPORTED_FLAGS_STUB 3637 #endif 3638 3639 #define XE_64K_PAGE_MASK 0xffffull 3640 #define ALL_DRM_XE_SYNCS_FLAGS (DRM_XE_SYNCS_FLAG_WAIT_FOR_OP) 3641 3642 static int vm_bind_ioctl_check_args(struct xe_device *xe, struct xe_vm *vm, 3643 struct drm_xe_vm_bind *args, 3644 struct drm_xe_vm_bind_op **bind_ops) 3645 { 3646 int err; 3647 int i; 3648 3649 if (XE_IOCTL_DBG(xe, args->pad || args->pad2) || 3650 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 3651 return -EINVAL; 3652 3653 if (XE_IOCTL_DBG(xe, args->extensions)) 3654 return -EINVAL; 3655 3656 if (XE_IOCTL_DBG(xe, args->num_syncs > DRM_XE_MAX_SYNCS)) 3657 return -EINVAL; 3658 3659 if (args->num_binds > 1) { 3660 u64 __user *bind_user = 3661 u64_to_user_ptr(args->vector_of_binds); 3662 3663 *bind_ops = kvmalloc_objs(struct drm_xe_vm_bind_op, 3664 args->num_binds, 3665 GFP_KERNEL | __GFP_ACCOUNT | __GFP_RETRY_MAYFAIL | __GFP_NOWARN); 3666 if (!*bind_ops) 3667 return args->num_binds > 1 ? -ENOBUFS : -ENOMEM; 3668 3669 err = copy_from_user(*bind_ops, bind_user, 3670 sizeof(struct drm_xe_vm_bind_op) * 3671 args->num_binds); 3672 if (XE_IOCTL_DBG(xe, err)) { 3673 err = -EFAULT; 3674 goto free_bind_ops; 3675 } 3676 } else { 3677 *bind_ops = &args->bind; 3678 } 3679 3680 for (i = 0; i < args->num_binds; ++i) { 3681 u64 range = (*bind_ops)[i].range; 3682 u64 addr = (*bind_ops)[i].addr; 3683 u32 op = (*bind_ops)[i].op; 3684 u32 flags = (*bind_ops)[i].flags; 3685 u32 obj = (*bind_ops)[i].obj; 3686 u64 obj_offset = (*bind_ops)[i].obj_offset; 3687 u32 prefetch_region = (*bind_ops)[i].prefetch_mem_region_instance; 3688 bool is_null = flags & DRM_XE_VM_BIND_FLAG_NULL; 3689 bool is_cpu_addr_mirror = flags & 3690 DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR; 3691 bool is_decompress = flags & DRM_XE_VM_BIND_FLAG_DECOMPRESS; 3692 u16 pat_index = (*bind_ops)[i].pat_index; 3693 u16 coh_mode; 3694 bool comp_en; 3695 3696 if (XE_IOCTL_DBG(xe, is_cpu_addr_mirror && 3697 (!xe_vm_in_fault_mode(vm) || 3698 !IS_ENABLED(CONFIG_DRM_XE_GPUSVM)))) { 3699 err = -EINVAL; 3700 goto free_bind_ops; 3701 } 3702 3703 if (XE_IOCTL_DBG(xe, pat_index >= xe->pat.n_entries)) { 3704 err = -EINVAL; 3705 goto free_bind_ops; 3706 } 3707 3708 pat_index = array_index_nospec(pat_index, xe->pat.n_entries); 3709 (*bind_ops)[i].pat_index = pat_index; 3710 coh_mode = xe_pat_index_get_coh_mode(xe, pat_index); 3711 comp_en = xe_pat_index_get_comp_en(xe, pat_index); 3712 if (XE_IOCTL_DBG(xe, !coh_mode)) { /* hw reserved */ 3713 err = -EINVAL; 3714 goto free_bind_ops; 3715 } 3716 3717 if (XE_WARN_ON(coh_mode > XE_COH_2WAY)) { 3718 err = -EINVAL; 3719 goto free_bind_ops; 3720 } 3721 3722 if (XE_IOCTL_DBG(xe, op > DRM_XE_VM_BIND_OP_PREFETCH) || 3723 XE_IOCTL_DBG(xe, flags & ~SUPPORTED_FLAGS) || 3724 XE_IOCTL_DBG(xe, obj && (is_null || is_cpu_addr_mirror)) || 3725 XE_IOCTL_DBG(xe, obj_offset && (is_null || 3726 is_cpu_addr_mirror)) || 3727 XE_IOCTL_DBG(xe, op != DRM_XE_VM_BIND_OP_MAP && 3728 (is_decompress || is_null || is_cpu_addr_mirror)) || 3729 XE_IOCTL_DBG(xe, is_decompress && 3730 xe_pat_index_get_comp_en(xe, pat_index)) || 3731 XE_IOCTL_DBG(xe, !obj && 3732 op == DRM_XE_VM_BIND_OP_MAP && 3733 !is_null && !is_cpu_addr_mirror) || 3734 XE_IOCTL_DBG(xe, !obj && 3735 op == DRM_XE_VM_BIND_OP_UNMAP_ALL) || 3736 XE_IOCTL_DBG(xe, addr && 3737 op == DRM_XE_VM_BIND_OP_UNMAP_ALL) || 3738 XE_IOCTL_DBG(xe, range && 3739 op == DRM_XE_VM_BIND_OP_UNMAP_ALL) || 3740 XE_IOCTL_DBG(xe, obj && 3741 op == DRM_XE_VM_BIND_OP_MAP_USERPTR) || 3742 XE_IOCTL_DBG(xe, coh_mode == XE_COH_NONE && 3743 op == DRM_XE_VM_BIND_OP_MAP_USERPTR) || 3744 XE_IOCTL_DBG(xe, !IS_DGFX(xe) && coh_mode == XE_COH_NONE && 3745 is_cpu_addr_mirror) || 3746 XE_IOCTL_DBG(xe, xe_device_is_l2_flush_optimized(xe) && 3747 (op == DRM_XE_VM_BIND_OP_MAP_USERPTR || 3748 is_cpu_addr_mirror) && 3749 (pat_index != 19 && coh_mode != XE_COH_2WAY)) || 3750 XE_IOCTL_DBG(xe, comp_en && 3751 op == DRM_XE_VM_BIND_OP_MAP_USERPTR) || 3752 XE_IOCTL_DBG(xe, op == DRM_XE_VM_BIND_OP_MAP_USERPTR && 3753 !IS_ENABLED(CONFIG_DRM_GPUSVM)) || 3754 XE_IOCTL_DBG(xe, obj && 3755 op == DRM_XE_VM_BIND_OP_PREFETCH) || 3756 XE_IOCTL_DBG(xe, prefetch_region && 3757 op != DRM_XE_VM_BIND_OP_PREFETCH) || 3758 XE_IOCTL_DBG(xe, (prefetch_region != DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC && 3759 /* Guard against undefined shift in BIT(prefetch_region) */ 3760 (prefetch_region >= (sizeof(xe->info.mem_region_mask) * 8) || 3761 !(BIT(prefetch_region) & xe->info.mem_region_mask)))) || 3762 XE_IOCTL_DBG(xe, obj && 3763 op == DRM_XE_VM_BIND_OP_UNMAP) || 3764 XE_IOCTL_DBG(xe, (flags & DRM_XE_VM_BIND_FLAG_MADVISE_AUTORESET) && 3765 (!is_cpu_addr_mirror || op != DRM_XE_VM_BIND_OP_MAP))) { 3766 err = -EINVAL; 3767 goto free_bind_ops; 3768 } 3769 3770 if (XE_IOCTL_DBG(xe, obj_offset & ~PAGE_MASK) || 3771 XE_IOCTL_DBG(xe, addr & ~PAGE_MASK) || 3772 XE_IOCTL_DBG(xe, range & ~PAGE_MASK) || 3773 XE_IOCTL_DBG(xe, !range && 3774 op != DRM_XE_VM_BIND_OP_UNMAP_ALL)) { 3775 err = -EINVAL; 3776 goto free_bind_ops; 3777 } 3778 3779 if (is_decompress && (XE_IOCTL_DBG(xe, !xe_device_has_flat_ccs(xe)) || 3780 XE_IOCTL_DBG(xe, GRAPHICS_VER(xe) < 20) || 3781 XE_IOCTL_DBG(xe, !IS_DGFX(xe)))) { 3782 err = -EOPNOTSUPP; 3783 goto free_bind_ops; 3784 } 3785 } 3786 3787 return 0; 3788 3789 free_bind_ops: 3790 if (args->num_binds > 1) 3791 kvfree(*bind_ops); 3792 *bind_ops = NULL; 3793 return err; 3794 } 3795 3796 static int vm_bind_ioctl_signal_fences(struct xe_vm *vm, 3797 struct xe_exec_queue *q, 3798 struct xe_sync_entry *syncs, 3799 int num_syncs) 3800 { 3801 struct dma_fence *fence = NULL; 3802 int i, err = 0; 3803 3804 if (num_syncs) { 3805 fence = xe_sync_in_fence_get(syncs, num_syncs, 3806 to_wait_exec_queue(vm, q), vm); 3807 if (IS_ERR(fence)) 3808 return PTR_ERR(fence); 3809 3810 for (i = 0; i < num_syncs; i++) 3811 xe_sync_entry_signal(&syncs[i], fence); 3812 } 3813 3814 dma_fence_put(fence); 3815 3816 return err; 3817 } 3818 3819 static void xe_vma_ops_init(struct xe_vma_ops *vops, struct xe_vm *vm, 3820 struct xe_exec_queue *q, 3821 struct xe_sync_entry *syncs, u32 num_syncs) 3822 { 3823 memset(vops, 0, sizeof(*vops)); 3824 INIT_LIST_HEAD(&vops->list); 3825 vops->vm = vm; 3826 vops->q = q; 3827 vops->syncs = syncs; 3828 vops->num_syncs = num_syncs; 3829 vops->flags = 0; 3830 } 3831 3832 static int xe_vm_bind_ioctl_validate_bo(struct xe_device *xe, struct xe_bo *bo, 3833 u64 addr, u64 range, u64 obj_offset, 3834 u16 pat_index, u32 op, u32 bind_flags) 3835 { 3836 u16 coh_mode; 3837 bool comp_en; 3838 3839 if (XE_IOCTL_DBG(xe, (bo->flags & XE_BO_FLAG_NO_COMPRESSION) && 3840 xe_pat_index_get_comp_en(xe, pat_index))) 3841 return -EINVAL; 3842 3843 if (XE_IOCTL_DBG(xe, range > xe_bo_size(bo)) || 3844 XE_IOCTL_DBG(xe, obj_offset > 3845 xe_bo_size(bo) - range)) { 3846 return -EINVAL; 3847 } 3848 3849 /* 3850 * Some platforms require 64k VM_BIND alignment, 3851 * specifically those with XE_VRAM_FLAGS_NEED64K. 3852 * 3853 * Other platforms may have BO's set to 64k physical placement, 3854 * but can be mapped at 4k offsets anyway. This check is only 3855 * there for the former case. 3856 */ 3857 if ((bo->flags & XE_BO_FLAG_INTERNAL_64K) && 3858 (xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K)) { 3859 if (XE_IOCTL_DBG(xe, obj_offset & 3860 XE_64K_PAGE_MASK) || 3861 XE_IOCTL_DBG(xe, addr & XE_64K_PAGE_MASK) || 3862 XE_IOCTL_DBG(xe, range & XE_64K_PAGE_MASK)) { 3863 return -EINVAL; 3864 } 3865 } 3866 3867 coh_mode = xe_pat_index_get_coh_mode(xe, pat_index); 3868 if (bo->cpu_caching) { 3869 if (XE_IOCTL_DBG(xe, coh_mode == XE_COH_NONE && 3870 bo->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB)) { 3871 return -EINVAL; 3872 } 3873 } else if (XE_IOCTL_DBG(xe, coh_mode == XE_COH_NONE)) { 3874 /* 3875 * Imported dma-buf from a different device should 3876 * require 1way or 2way coherency since we don't know 3877 * how it was mapped on the CPU. Just assume is it 3878 * potentially cached on CPU side. 3879 */ 3880 return -EINVAL; 3881 } 3882 3883 /* 3884 * Ensures that imported buffer objects (dma-bufs) are not mapped 3885 * with a PAT index that enables compression. 3886 */ 3887 comp_en = xe_pat_index_get_comp_en(xe, pat_index); 3888 if (XE_IOCTL_DBG(xe, bo->ttm.base.import_attach && comp_en)) 3889 return -EINVAL; 3890 3891 if (XE_IOCTL_DBG(xe, bo->ttm.base.import_attach && xe_device_is_l2_flush_optimized(xe) && 3892 (pat_index != 19 && coh_mode != XE_COH_2WAY))) 3893 return -EINVAL; 3894 3895 /* If a BO is protected it can only be mapped if the key is still valid */ 3896 if ((bind_flags & DRM_XE_VM_BIND_FLAG_CHECK_PXP) && xe_bo_is_protected(bo) && 3897 op != DRM_XE_VM_BIND_OP_UNMAP && op != DRM_XE_VM_BIND_OP_UNMAP_ALL) 3898 if (XE_IOCTL_DBG(xe, xe_pxp_bo_key_check(xe->pxp, bo) != 0)) 3899 return -ENOEXEC; 3900 3901 return 0; 3902 } 3903 3904 int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 3905 { 3906 struct xe_device *xe = to_xe_device(dev); 3907 struct xe_file *xef = to_xe_file(file); 3908 struct drm_xe_vm_bind *args = data; 3909 struct drm_xe_sync __user *syncs_user; 3910 struct xe_bo **bos = NULL; 3911 struct drm_gpuva_ops **ops = NULL; 3912 struct xe_vm *vm; 3913 struct xe_exec_queue *q = NULL; 3914 u32 num_syncs, num_ufence = 0; 3915 struct xe_sync_entry *syncs = NULL; 3916 struct drm_xe_vm_bind_op *bind_ops = NULL; 3917 struct xe_vma_ops vops; 3918 struct dma_fence *fence; 3919 int err; 3920 int i; 3921 3922 vm = xe_vm_lookup(xef, args->vm_id); 3923 if (XE_IOCTL_DBG(xe, !vm)) 3924 return -EINVAL; 3925 3926 err = vm_bind_ioctl_check_args(xe, vm, args, &bind_ops); 3927 if (err) 3928 goto put_vm; 3929 3930 if (args->exec_queue_id) { 3931 q = xe_exec_queue_lookup(xef, args->exec_queue_id); 3932 if (XE_IOCTL_DBG(xe, !q)) { 3933 err = -ENOENT; 3934 goto free_bind_ops; 3935 } 3936 3937 if (XE_IOCTL_DBG(xe, !(q->flags & EXEC_QUEUE_FLAG_VM))) { 3938 err = -EINVAL; 3939 goto put_exec_queue; 3940 } 3941 } 3942 3943 if (XE_IOCTL_DBG(xe, q && vm != q->user_vm)) { 3944 err = -EINVAL; 3945 goto put_exec_queue; 3946 } 3947 3948 /* Ensure all UNMAPs visible */ 3949 xe_svm_flush(vm); 3950 3951 err = down_write_killable(&vm->lock); 3952 if (err) 3953 goto put_exec_queue; 3954 3955 if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) { 3956 err = -ENOENT; 3957 goto release_vm_lock; 3958 } 3959 3960 for (i = 0; i < args->num_binds; ++i) { 3961 u64 range = bind_ops[i].range; 3962 u64 addr = bind_ops[i].addr; 3963 3964 if (XE_IOCTL_DBG(xe, range > vm->size) || 3965 XE_IOCTL_DBG(xe, addr > vm->size - range)) { 3966 err = -EINVAL; 3967 goto release_vm_lock; 3968 } 3969 } 3970 3971 if (args->num_binds) { 3972 bos = kvzalloc_objs(*bos, args->num_binds, 3973 GFP_KERNEL | __GFP_ACCOUNT | __GFP_RETRY_MAYFAIL | __GFP_NOWARN); 3974 if (!bos) { 3975 err = -ENOMEM; 3976 goto release_vm_lock; 3977 } 3978 3979 ops = kvzalloc_objs(*ops, args->num_binds, 3980 GFP_KERNEL | __GFP_ACCOUNT | __GFP_RETRY_MAYFAIL | __GFP_NOWARN); 3981 if (!ops) { 3982 err = -ENOMEM; 3983 goto free_bos; 3984 } 3985 } 3986 3987 for (i = 0; i < args->num_binds; ++i) { 3988 struct drm_gem_object *gem_obj; 3989 u64 range = bind_ops[i].range; 3990 u64 addr = bind_ops[i].addr; 3991 u32 obj = bind_ops[i].obj; 3992 u64 obj_offset = bind_ops[i].obj_offset; 3993 u16 pat_index = bind_ops[i].pat_index; 3994 u32 op = bind_ops[i].op; 3995 u32 bind_flags = bind_ops[i].flags; 3996 3997 if (!obj) 3998 continue; 3999 4000 gem_obj = drm_gem_object_lookup(file, obj); 4001 if (XE_IOCTL_DBG(xe, !gem_obj)) { 4002 err = -ENOENT; 4003 goto put_obj; 4004 } 4005 bos[i] = gem_to_xe_bo(gem_obj); 4006 4007 err = xe_vm_bind_ioctl_validate_bo(xe, bos[i], addr, range, 4008 obj_offset, pat_index, op, 4009 bind_flags); 4010 if (err) 4011 goto put_obj; 4012 } 4013 4014 if (args->num_syncs) { 4015 syncs = kzalloc_objs(*syncs, args->num_syncs); 4016 if (!syncs) { 4017 err = -ENOMEM; 4018 goto put_obj; 4019 } 4020 } 4021 4022 syncs_user = u64_to_user_ptr(args->syncs); 4023 for (num_syncs = 0; num_syncs < args->num_syncs; num_syncs++) { 4024 struct xe_exec_queue *__q = q ?: vm->q[0]; 4025 4026 err = xe_sync_entry_parse(xe, xef, &syncs[num_syncs], 4027 &syncs_user[num_syncs], 4028 __q->ufence_syncobj, 4029 ++__q->ufence_timeline_value, 4030 (xe_vm_in_lr_mode(vm) ? 4031 SYNC_PARSE_FLAG_LR_MODE : 0) | 4032 (!args->num_binds ? 4033 SYNC_PARSE_FLAG_DISALLOW_USER_FENCE : 0)); 4034 if (err) 4035 goto free_syncs; 4036 4037 if (xe_sync_is_ufence(&syncs[num_syncs])) 4038 num_ufence++; 4039 } 4040 4041 if (XE_IOCTL_DBG(xe, num_ufence > 1)) { 4042 err = -EINVAL; 4043 goto free_syncs; 4044 } 4045 4046 if (!args->num_binds) { 4047 err = -ENODATA; 4048 goto free_syncs; 4049 } 4050 4051 xe_vma_ops_init(&vops, vm, q, syncs, num_syncs); 4052 if (args->num_binds > 1) 4053 vops.flags |= XE_VMA_OPS_ARRAY_OF_BINDS; 4054 for (i = 0; i < args->num_binds; ++i) { 4055 u64 range = bind_ops[i].range; 4056 u64 addr = bind_ops[i].addr; 4057 u32 op = bind_ops[i].op; 4058 u32 flags = bind_ops[i].flags; 4059 u64 obj_offset = bind_ops[i].obj_offset; 4060 u32 prefetch_region = bind_ops[i].prefetch_mem_region_instance; 4061 u16 pat_index = bind_ops[i].pat_index; 4062 4063 ops[i] = vm_bind_ioctl_ops_create(vm, &vops, bos[i], obj_offset, 4064 addr, range, op, flags, 4065 prefetch_region, pat_index); 4066 if (IS_ERR(ops[i])) { 4067 err = PTR_ERR(ops[i]); 4068 ops[i] = NULL; 4069 goto unwind_ops; 4070 } 4071 4072 err = vm_bind_ioctl_ops_parse(vm, ops[i], &vops); 4073 if (err) 4074 goto unwind_ops; 4075 4076 #ifdef TEST_VM_OPS_ERROR 4077 if (flags & FORCE_OP_ERROR) { 4078 vops.inject_error = true; 4079 vm->xe->vm_inject_error_position = 4080 (vm->xe->vm_inject_error_position + 1) % 4081 FORCE_OP_ERROR_COUNT; 4082 } 4083 #endif 4084 } 4085 4086 /* Nothing to do */ 4087 if (list_empty(&vops.list)) { 4088 err = -ENODATA; 4089 goto unwind_ops; 4090 } 4091 4092 err = xe_vma_ops_alloc(&vops, args->num_binds > 1); 4093 if (err) 4094 goto unwind_ops; 4095 4096 err = vm_bind_ioctl_ops_prefetch_ranges(vm, &vops); 4097 if (err) 4098 goto unwind_ops; 4099 4100 fence = vm_bind_ioctl_ops_execute(vm, &vops); 4101 if (IS_ERR(fence)) 4102 err = PTR_ERR(fence); 4103 else 4104 dma_fence_put(fence); 4105 4106 unwind_ops: 4107 if (err && err != -ENODATA) 4108 vm_bind_ioctl_ops_unwind(vm, ops, args->num_binds); 4109 xe_vma_ops_fini(&vops); 4110 for (i = args->num_binds - 1; i >= 0; --i) 4111 if (ops[i]) 4112 drm_gpuva_ops_free(&vm->gpuvm, ops[i]); 4113 free_syncs: 4114 if (err == -ENODATA) 4115 err = vm_bind_ioctl_signal_fences(vm, q, syncs, num_syncs); 4116 while (num_syncs--) 4117 xe_sync_entry_cleanup(&syncs[num_syncs]); 4118 4119 kfree(syncs); 4120 put_obj: 4121 for (i = 0; i < args->num_binds; ++i) 4122 xe_bo_put(bos[i]); 4123 4124 kvfree(ops); 4125 free_bos: 4126 kvfree(bos); 4127 release_vm_lock: 4128 up_write(&vm->lock); 4129 put_exec_queue: 4130 if (q) 4131 xe_exec_queue_put(q); 4132 free_bind_ops: 4133 if (args->num_binds > 1) 4134 kvfree(bind_ops); 4135 put_vm: 4136 xe_vm_put(vm); 4137 return err; 4138 } 4139 4140 /* 4141 * Map access type, fault type, and fault level from current bspec 4142 * specification to user spec abstraction. The current mapping is 4143 * approximately 1-to-1, with access type being the only notable 4144 * exception as it carries additional data with respect to prefetch 4145 * status that needs to be masked out. 4146 */ 4147 static u8 xe_to_user_access_type(u8 access_type) 4148 { 4149 return access_type & XE_PAGEFAULT_ACCESS_TYPE_MASK; 4150 } 4151 4152 static u8 xe_to_user_fault_type(u8 fault_type) 4153 { 4154 return fault_type; 4155 } 4156 4157 static u8 xe_to_user_fault_level(u8 fault_level) 4158 { 4159 return fault_level; 4160 } 4161 4162 static int fill_faults(struct xe_vm *vm, 4163 struct drm_xe_vm_get_property *args) 4164 { 4165 struct xe_vm_fault __user *usr_ptr = u64_to_user_ptr(args->data); 4166 struct xe_vm_fault *fault_list, fault_entry = { 0 }; 4167 struct xe_vm_fault_entry *entry; 4168 int ret = 0, i = 0, count, entry_size; 4169 4170 entry_size = sizeof(struct xe_vm_fault); 4171 count = args->size / entry_size; 4172 4173 fault_list = kzalloc_objs(struct xe_vm_fault, count); 4174 if (!fault_list) 4175 return -ENOMEM; 4176 4177 spin_lock(&vm->faults.lock); 4178 list_for_each_entry(entry, &vm->faults.list, list) { 4179 if (i == count) 4180 break; 4181 4182 fault_entry.address = xe_device_canonicalize_addr(vm->xe, entry->address); 4183 fault_entry.address_precision = entry->address_precision; 4184 4185 fault_entry.access_type = xe_to_user_access_type(entry->access_type); 4186 fault_entry.fault_type = xe_to_user_fault_type(entry->fault_type); 4187 fault_entry.fault_level = xe_to_user_fault_level(entry->fault_level); 4188 4189 memcpy(&fault_list[i], &fault_entry, entry_size); 4190 4191 i++; 4192 } 4193 spin_unlock(&vm->faults.lock); 4194 4195 ret = copy_to_user(usr_ptr, fault_list, args->size); 4196 4197 kfree(fault_list); 4198 return ret ? -EFAULT : 0; 4199 } 4200 4201 static int xe_vm_get_property_helper(struct xe_vm *vm, 4202 struct drm_xe_vm_get_property *args) 4203 { 4204 size_t size; 4205 4206 switch (args->property) { 4207 case DRM_XE_VM_GET_PROPERTY_FAULTS: 4208 spin_lock(&vm->faults.lock); 4209 size = size_mul(sizeof(struct xe_vm_fault), vm->faults.len); 4210 spin_unlock(&vm->faults.lock); 4211 4212 if (!args->size) { 4213 args->size = size; 4214 return 0; 4215 } 4216 4217 /* 4218 * Number of faults may increase between calls to 4219 * xe_vm_get_property_ioctl, so just report the number of 4220 * faults the user requests if it's less than or equal to 4221 * the number of faults in the VM fault array. 4222 * 4223 * We should also at least assert that the args->size value 4224 * is a multiple of the xe_vm_fault struct size. 4225 */ 4226 if (args->size > size || args->size % sizeof(struct xe_vm_fault)) 4227 return -EINVAL; 4228 4229 return fill_faults(vm, args); 4230 } 4231 return -EINVAL; 4232 } 4233 4234 int xe_vm_get_property_ioctl(struct drm_device *drm, void *data, 4235 struct drm_file *file) 4236 { 4237 struct xe_device *xe = to_xe_device(drm); 4238 struct xe_file *xef = to_xe_file(file); 4239 struct drm_xe_vm_get_property *args = data; 4240 struct xe_vm *vm; 4241 int ret = 0; 4242 4243 if (XE_IOCTL_DBG(xe, (args->reserved[0] || args->reserved[1] || 4244 args->reserved[2] || args->extensions || 4245 args->pad))) 4246 return -EINVAL; 4247 4248 vm = xe_vm_lookup(xef, args->vm_id); 4249 if (XE_IOCTL_DBG(xe, !vm)) 4250 return -ENOENT; 4251 4252 ret = xe_vm_get_property_helper(vm, args); 4253 4254 xe_vm_put(vm); 4255 return ret; 4256 } 4257 4258 /** 4259 * xe_vm_bind_kernel_bo - bind a kernel BO to a VM 4260 * @vm: VM to bind the BO to 4261 * @bo: BO to bind 4262 * @q: exec queue to use for the bind (optional) 4263 * @addr: address at which to bind the BO 4264 * @cache_lvl: PAT cache level to use 4265 * 4266 * Execute a VM bind map operation on a kernel-owned BO to bind it into a 4267 * kernel-owned VM. 4268 * 4269 * Returns a dma_fence to track the binding completion if the job to do so was 4270 * successfully submitted, an error pointer otherwise. 4271 */ 4272 struct dma_fence *xe_vm_bind_kernel_bo(struct xe_vm *vm, struct xe_bo *bo, 4273 struct xe_exec_queue *q, u64 addr, 4274 enum xe_cache_level cache_lvl) 4275 { 4276 struct xe_vma_ops vops; 4277 struct drm_gpuva_ops *ops = NULL; 4278 struct dma_fence *fence; 4279 int err; 4280 4281 xe_bo_get(bo); 4282 xe_vm_get(vm); 4283 if (q) 4284 xe_exec_queue_get(q); 4285 4286 down_write(&vm->lock); 4287 4288 xe_vma_ops_init(&vops, vm, q, NULL, 0); 4289 4290 ops = vm_bind_ioctl_ops_create(vm, &vops, bo, 0, addr, xe_bo_size(bo), 4291 DRM_XE_VM_BIND_OP_MAP, 0, 0, 4292 xe_cache_pat_idx(vm->xe, cache_lvl)); 4293 if (IS_ERR(ops)) { 4294 err = PTR_ERR(ops); 4295 goto release_vm_lock; 4296 } 4297 4298 err = vm_bind_ioctl_ops_parse(vm, ops, &vops); 4299 if (err) 4300 goto release_vm_lock; 4301 4302 xe_assert(vm->xe, !list_empty(&vops.list)); 4303 4304 err = xe_vma_ops_alloc(&vops, false); 4305 if (err) 4306 goto unwind_ops; 4307 4308 fence = vm_bind_ioctl_ops_execute(vm, &vops); 4309 if (IS_ERR(fence)) 4310 err = PTR_ERR(fence); 4311 4312 unwind_ops: 4313 if (err && err != -ENODATA) 4314 vm_bind_ioctl_ops_unwind(vm, &ops, 1); 4315 4316 xe_vma_ops_fini(&vops); 4317 drm_gpuva_ops_free(&vm->gpuvm, ops); 4318 4319 release_vm_lock: 4320 up_write(&vm->lock); 4321 4322 if (q) 4323 xe_exec_queue_put(q); 4324 xe_vm_put(vm); 4325 xe_bo_put(bo); 4326 4327 if (err) 4328 fence = ERR_PTR(err); 4329 4330 return fence; 4331 } 4332 4333 /** 4334 * xe_vm_lock() - Lock the vm's dma_resv object 4335 * @vm: The struct xe_vm whose lock is to be locked 4336 * @intr: Whether to perform any wait interruptible 4337 * 4338 * Return: 0 on success, -EINTR if @intr is true and the wait for a 4339 * contended lock was interrupted. If @intr is false, the function 4340 * always returns 0. 4341 */ 4342 int xe_vm_lock(struct xe_vm *vm, bool intr) 4343 { 4344 int ret; 4345 4346 if (intr) 4347 ret = dma_resv_lock_interruptible(xe_vm_resv(vm), NULL); 4348 else 4349 ret = dma_resv_lock(xe_vm_resv(vm), NULL); 4350 4351 return ret; 4352 } 4353 4354 /** 4355 * xe_vm_unlock() - Unlock the vm's dma_resv object 4356 * @vm: The struct xe_vm whose lock is to be released. 4357 * 4358 * Unlock a buffer object lock that was locked by xe_vm_lock(). 4359 */ 4360 void xe_vm_unlock(struct xe_vm *vm) 4361 { 4362 dma_resv_unlock(xe_vm_resv(vm)); 4363 } 4364 4365 /** 4366 * xe_vm_invalidate_vma_submit - Submit a job to invalidate GPU mappings for 4367 * VMA. 4368 * @vma: VMA to invalidate 4369 * @batch: TLB invalidation batch to populate; caller must later call 4370 * xe_tlb_inval_batch_wait() on it to wait for completion 4371 * 4372 * Walks a list of page tables leaves which it memset the entries owned by this 4373 * VMA to zero, invalidates the TLBs, but doesn't block waiting for TLB flush 4374 * to complete, but instead populates @batch which can be waited on using 4375 * xe_tlb_inval_batch_wait(). 4376 * 4377 * Returns 0 for success, negative error code otherwise. 4378 */ 4379 int xe_vm_invalidate_vma_submit(struct xe_vma *vma, struct xe_tlb_inval_batch *batch) 4380 { 4381 struct xe_device *xe = xe_vma_vm(vma)->xe; 4382 struct xe_vm *vm = xe_vma_vm(vma); 4383 struct xe_tile *tile; 4384 u8 tile_mask = 0; 4385 int ret = 0; 4386 u8 id; 4387 4388 xe_assert(xe, !xe_vma_is_null(vma)); 4389 xe_assert(xe, !xe_vma_is_cpu_addr_mirror(vma)); 4390 trace_xe_vma_invalidate(vma); 4391 4392 vm_dbg(&vm->xe->drm, 4393 "INVALIDATE: addr=0x%016llx, range=0x%016llx", 4394 xe_vma_start(vma), xe_vma_size(vma)); 4395 4396 /* 4397 * Check that we don't race with page-table updates, tile_invalidated 4398 * update is safe 4399 */ 4400 if (IS_ENABLED(CONFIG_PROVE_LOCKING)) { 4401 if (xe_vma_is_userptr(vma)) { 4402 lockdep_assert(lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 0) || 4403 (lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 1) && 4404 lockdep_is_held(&xe_vm_resv(vm)->lock.base))); 4405 4406 WARN_ON_ONCE(!mmu_interval_check_retry 4407 (&to_userptr_vma(vma)->userptr.notifier, 4408 to_userptr_vma(vma)->userptr.pages.notifier_seq)); 4409 WARN_ON_ONCE(!dma_resv_test_signaled(xe_vm_resv(vm), 4410 DMA_RESV_USAGE_BOOKKEEP)); 4411 4412 } else { 4413 xe_bo_assert_held(xe_vma_bo(vma)); 4414 } 4415 } 4416 4417 for_each_tile(tile, xe, id) 4418 if (xe_pt_zap_ptes(tile, vma)) 4419 tile_mask |= BIT(id); 4420 4421 xe_device_wmb(xe); 4422 4423 ret = xe_tlb_inval_range_tilemask_submit(xe, xe_vma_vm(vma)->usm.asid, 4424 xe_vma_start(vma), xe_vma_end(vma), 4425 tile_mask, batch); 4426 4427 /* WRITE_ONCE pairs with READ_ONCE in xe_vm_has_valid_gpu_mapping() */ 4428 WRITE_ONCE(vma->tile_invalidated, vma->tile_mask); 4429 return ret; 4430 } 4431 4432 /** 4433 * xe_vm_invalidate_vma - invalidate GPU mappings for VMA without a lock 4434 * @vma: VMA to invalidate 4435 * 4436 * Walks a list of page tables leaves which it memset the entries owned by this 4437 * VMA to zero, invalidates the TLBs, and block until TLBs invalidation is 4438 * complete. 4439 * 4440 * Returns 0 for success, negative error code otherwise. 4441 */ 4442 int xe_vm_invalidate_vma(struct xe_vma *vma) 4443 { 4444 struct xe_tlb_inval_batch batch; 4445 int ret; 4446 4447 ret = xe_vm_invalidate_vma_submit(vma, &batch); 4448 if (ret) 4449 return ret; 4450 4451 xe_tlb_inval_batch_wait(&batch); 4452 return ret; 4453 } 4454 4455 int xe_vm_validate_protected(struct xe_vm *vm) 4456 { 4457 struct drm_gpuva *gpuva; 4458 int err = 0; 4459 4460 if (!vm) 4461 return -ENODEV; 4462 4463 mutex_lock(&vm->snap_mutex); 4464 4465 drm_gpuvm_for_each_va(gpuva, &vm->gpuvm) { 4466 struct xe_vma *vma = gpuva_to_vma(gpuva); 4467 struct xe_bo *bo = vma->gpuva.gem.obj ? 4468 gem_to_xe_bo(vma->gpuva.gem.obj) : NULL; 4469 4470 if (!bo) 4471 continue; 4472 4473 if (xe_bo_is_protected(bo)) { 4474 err = xe_pxp_bo_key_check(vm->xe->pxp, bo); 4475 if (err) 4476 break; 4477 } 4478 } 4479 4480 mutex_unlock(&vm->snap_mutex); 4481 return err; 4482 } 4483 4484 struct xe_vm_snapshot { 4485 int uapi_flags; 4486 unsigned long num_snaps; 4487 struct { 4488 u64 ofs, bo_ofs; 4489 unsigned long len; 4490 #define XE_VM_SNAP_FLAG_USERPTR BIT(0) 4491 #define XE_VM_SNAP_FLAG_READ_ONLY BIT(1) 4492 #define XE_VM_SNAP_FLAG_IS_NULL BIT(2) 4493 unsigned long flags; 4494 int uapi_mem_region; 4495 u16 pat_index; 4496 int cpu_caching; 4497 struct xe_bo *bo; 4498 void *data; 4499 struct mm_struct *mm; 4500 } snap[]; 4501 }; 4502 4503 struct xe_vm_snapshot *xe_vm_snapshot_capture(struct xe_vm *vm) 4504 { 4505 unsigned long num_snaps = 0, i; 4506 struct xe_vm_snapshot *snap = NULL; 4507 struct drm_gpuva *gpuva; 4508 4509 if (!vm) 4510 return NULL; 4511 4512 mutex_lock(&vm->snap_mutex); 4513 drm_gpuvm_for_each_va(gpuva, &vm->gpuvm) { 4514 if (gpuva->flags & XE_VMA_DUMPABLE) 4515 num_snaps++; 4516 } 4517 4518 if (num_snaps) 4519 snap = kvzalloc(offsetof(struct xe_vm_snapshot, snap[num_snaps]), GFP_NOWAIT); 4520 if (!snap) { 4521 snap = num_snaps ? ERR_PTR(-ENOMEM) : ERR_PTR(-ENODEV); 4522 goto out_unlock; 4523 } 4524 4525 if (vm->flags & XE_VM_FLAG_FAULT_MODE) 4526 snap->uapi_flags |= DRM_XE_VM_CREATE_FLAG_FAULT_MODE; 4527 if (vm->flags & XE_VM_FLAG_LR_MODE) 4528 snap->uapi_flags |= DRM_XE_VM_CREATE_FLAG_LR_MODE; 4529 if (vm->flags & XE_VM_FLAG_SCRATCH_PAGE) 4530 snap->uapi_flags |= DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE; 4531 4532 snap->num_snaps = num_snaps; 4533 i = 0; 4534 drm_gpuvm_for_each_va(gpuva, &vm->gpuvm) { 4535 struct xe_vma *vma = gpuva_to_vma(gpuva); 4536 struct xe_bo *bo = vma->gpuva.gem.obj ? 4537 gem_to_xe_bo(vma->gpuva.gem.obj) : NULL; 4538 4539 if (!(gpuva->flags & XE_VMA_DUMPABLE)) 4540 continue; 4541 4542 snap->snap[i].ofs = xe_vma_start(vma); 4543 snap->snap[i].len = xe_vma_size(vma); 4544 snap->snap[i].flags = xe_vma_read_only(vma) ? 4545 XE_VM_SNAP_FLAG_READ_ONLY : 0; 4546 snap->snap[i].pat_index = vma->attr.pat_index; 4547 if (bo) { 4548 snap->snap[i].cpu_caching = bo->cpu_caching; 4549 snap->snap[i].bo = xe_bo_get(bo); 4550 snap->snap[i].bo_ofs = xe_vma_bo_offset(vma); 4551 switch (bo->ttm.resource->mem_type) { 4552 case XE_PL_SYSTEM: 4553 case XE_PL_TT: 4554 snap->snap[i].uapi_mem_region = 0; 4555 break; 4556 case XE_PL_VRAM0: 4557 snap->snap[i].uapi_mem_region = 1; 4558 break; 4559 case XE_PL_VRAM1: 4560 snap->snap[i].uapi_mem_region = 2; 4561 break; 4562 } 4563 } else if (xe_vma_is_userptr(vma)) { 4564 struct mm_struct *mm = 4565 to_userptr_vma(vma)->userptr.notifier.mm; 4566 4567 if (mmget_not_zero(mm)) 4568 snap->snap[i].mm = mm; 4569 else 4570 snap->snap[i].data = ERR_PTR(-EFAULT); 4571 4572 snap->snap[i].bo_ofs = xe_vma_userptr(vma); 4573 snap->snap[i].flags |= XE_VM_SNAP_FLAG_USERPTR; 4574 snap->snap[i].uapi_mem_region = 0; 4575 } else if (xe_vma_is_null(vma)) { 4576 snap->snap[i].flags |= XE_VM_SNAP_FLAG_IS_NULL; 4577 snap->snap[i].uapi_mem_region = -1; 4578 } else { 4579 snap->snap[i].data = ERR_PTR(-ENOENT); 4580 snap->snap[i].uapi_mem_region = -1; 4581 } 4582 i++; 4583 } 4584 4585 out_unlock: 4586 mutex_unlock(&vm->snap_mutex); 4587 return snap; 4588 } 4589 4590 void xe_vm_snapshot_capture_delayed(struct xe_vm_snapshot *snap) 4591 { 4592 if (IS_ERR_OR_NULL(snap)) 4593 return; 4594 4595 for (int i = 0; i < snap->num_snaps; i++) { 4596 struct xe_bo *bo = snap->snap[i].bo; 4597 int err; 4598 4599 if (IS_ERR(snap->snap[i].data) || 4600 snap->snap[i].flags & XE_VM_SNAP_FLAG_IS_NULL) 4601 continue; 4602 4603 snap->snap[i].data = kvmalloc(snap->snap[i].len, GFP_USER); 4604 if (!snap->snap[i].data) { 4605 snap->snap[i].data = ERR_PTR(-ENOMEM); 4606 goto cleanup_bo; 4607 } 4608 4609 if (bo) { 4610 err = xe_bo_read(bo, snap->snap[i].bo_ofs, 4611 snap->snap[i].data, snap->snap[i].len); 4612 } else { 4613 void __user *userptr = (void __user *)(size_t)snap->snap[i].bo_ofs; 4614 4615 kthread_use_mm(snap->snap[i].mm); 4616 if (!copy_from_user(snap->snap[i].data, userptr, snap->snap[i].len)) 4617 err = 0; 4618 else 4619 err = -EFAULT; 4620 kthread_unuse_mm(snap->snap[i].mm); 4621 4622 mmput(snap->snap[i].mm); 4623 snap->snap[i].mm = NULL; 4624 } 4625 4626 if (err) { 4627 kvfree(snap->snap[i].data); 4628 snap->snap[i].data = ERR_PTR(err); 4629 } 4630 4631 cleanup_bo: 4632 xe_bo_put(bo); 4633 snap->snap[i].bo = NULL; 4634 } 4635 } 4636 4637 void xe_vm_snapshot_print(struct xe_vm_snapshot *snap, struct drm_printer *p) 4638 { 4639 unsigned long i, j; 4640 4641 if (IS_ERR_OR_NULL(snap)) { 4642 drm_printf(p, "[0].error: %li\n", PTR_ERR(snap)); 4643 return; 4644 } 4645 4646 drm_printf(p, "VM.uapi_flags: 0x%x\n", snap->uapi_flags); 4647 for (i = 0; i < snap->num_snaps; i++) { 4648 drm_printf(p, "[%llx].length: 0x%lx\n", snap->snap[i].ofs, snap->snap[i].len); 4649 4650 drm_printf(p, "[%llx].properties: %s|%s|mem_region=0x%lx|pat_index=%d|cpu_caching=%d\n", 4651 snap->snap[i].ofs, 4652 snap->snap[i].flags & XE_VM_SNAP_FLAG_READ_ONLY ? 4653 "read_only" : "read_write", 4654 snap->snap[i].flags & XE_VM_SNAP_FLAG_IS_NULL ? 4655 "null_sparse" : 4656 snap->snap[i].flags & XE_VM_SNAP_FLAG_USERPTR ? 4657 "userptr" : "bo", 4658 snap->snap[i].uapi_mem_region == -1 ? 0 : 4659 BIT(snap->snap[i].uapi_mem_region), 4660 snap->snap[i].pat_index, 4661 snap->snap[i].cpu_caching); 4662 4663 if (IS_ERR(snap->snap[i].data)) { 4664 drm_printf(p, "[%llx].error: %li\n", snap->snap[i].ofs, 4665 PTR_ERR(snap->snap[i].data)); 4666 continue; 4667 } 4668 4669 if (snap->snap[i].flags & XE_VM_SNAP_FLAG_IS_NULL) 4670 continue; 4671 4672 drm_printf(p, "[%llx].data: ", snap->snap[i].ofs); 4673 4674 for (j = 0; j < snap->snap[i].len; j += sizeof(u32)) { 4675 u32 *val = snap->snap[i].data + j; 4676 char dumped[ASCII85_BUFSZ]; 4677 4678 drm_puts(p, ascii85_encode(*val, dumped)); 4679 } 4680 4681 drm_puts(p, "\n"); 4682 4683 if (drm_coredump_printer_is_full(p)) 4684 return; 4685 } 4686 } 4687 4688 void xe_vm_snapshot_free(struct xe_vm_snapshot *snap) 4689 { 4690 unsigned long i; 4691 4692 if (IS_ERR_OR_NULL(snap)) 4693 return; 4694 4695 for (i = 0; i < snap->num_snaps; i++) { 4696 if (!IS_ERR(snap->snap[i].data)) 4697 kvfree(snap->snap[i].data); 4698 xe_bo_put(snap->snap[i].bo); 4699 if (snap->snap[i].mm) 4700 mmput(snap->snap[i].mm); 4701 } 4702 kvfree(snap); 4703 } 4704 4705 /** 4706 * xe_vma_need_vram_for_atomic - Check if VMA needs VRAM migration for atomic operations 4707 * @xe: Pointer to the Xe device structure 4708 * @vma: Pointer to the virtual memory area (VMA) structure 4709 * @is_atomic: In pagefault path and atomic operation 4710 * 4711 * This function determines whether the given VMA needs to be migrated to 4712 * VRAM in order to do atomic GPU operation. 4713 * 4714 * Return: 4715 * 1 - Migration to VRAM is required 4716 * 0 - Migration is not required 4717 * -EACCES - Invalid access for atomic memory attr 4718 * 4719 */ 4720 int xe_vma_need_vram_for_atomic(struct xe_device *xe, struct xe_vma *vma, bool is_atomic) 4721 { 4722 u32 atomic_access = xe_vma_bo(vma) ? xe_vma_bo(vma)->attr.atomic_access : 4723 vma->attr.atomic_access; 4724 4725 if (!IS_DGFX(xe) || !is_atomic) 4726 return false; 4727 4728 /* 4729 * NOTE: The checks implemented here are platform-specific. For 4730 * instance, on a device supporting CXL atomics, these would ideally 4731 * work universally without additional handling. 4732 */ 4733 switch (atomic_access) { 4734 case DRM_XE_ATOMIC_DEVICE: 4735 return !xe->info.has_device_atomics_on_smem; 4736 4737 case DRM_XE_ATOMIC_CPU: 4738 return -EACCES; 4739 4740 case DRM_XE_ATOMIC_UNDEFINED: 4741 case DRM_XE_ATOMIC_GLOBAL: 4742 default: 4743 return 1; 4744 } 4745 } 4746 4747 static int xe_vm_alloc_vma(struct xe_vm *vm, 4748 struct drm_gpuvm_map_req *map_req, 4749 bool is_madvise) 4750 { 4751 struct xe_vma_ops vops; 4752 struct drm_gpuva_ops *ops = NULL; 4753 struct drm_gpuva_op *__op; 4754 unsigned int vma_flags = 0; 4755 bool remap_op = false; 4756 struct xe_vma_mem_attr tmp_attr = {}; 4757 u16 default_pat; 4758 int err; 4759 4760 lockdep_assert_held_write(&vm->lock); 4761 4762 if (is_madvise) 4763 ops = drm_gpuvm_madvise_ops_create(&vm->gpuvm, map_req); 4764 else 4765 ops = drm_gpuvm_sm_map_ops_create(&vm->gpuvm, map_req); 4766 4767 if (IS_ERR(ops)) 4768 return PTR_ERR(ops); 4769 4770 if (list_empty(&ops->list)) { 4771 err = 0; 4772 goto free_ops; 4773 } 4774 4775 drm_gpuva_for_each_op(__op, ops) { 4776 struct xe_vma_op *op = gpuva_op_to_vma_op(__op); 4777 struct xe_vma *vma = NULL; 4778 4779 if (!is_madvise) { 4780 if (__op->op == DRM_GPUVA_OP_UNMAP) { 4781 vma = gpuva_to_vma(op->base.unmap.va); 4782 XE_WARN_ON(!xe_vma_has_default_mem_attrs(vma)); 4783 default_pat = vma->attr.default_pat_index; 4784 vma_flags = vma->gpuva.flags; 4785 } 4786 4787 if (__op->op == DRM_GPUVA_OP_REMAP) { 4788 vma = gpuva_to_vma(op->base.remap.unmap->va); 4789 default_pat = vma->attr.default_pat_index; 4790 vma_flags = vma->gpuva.flags; 4791 } 4792 4793 if (__op->op == DRM_GPUVA_OP_MAP) { 4794 op->map.vma_flags |= vma_flags & XE_VMA_CREATE_MASK; 4795 op->map.pat_index = default_pat; 4796 } 4797 } else { 4798 if (__op->op == DRM_GPUVA_OP_REMAP) { 4799 vma = gpuva_to_vma(op->base.remap.unmap->va); 4800 xe_assert(vm->xe, !remap_op); 4801 xe_assert(vm->xe, xe_vma_has_no_bo(vma)); 4802 remap_op = true; 4803 vma_flags = vma->gpuva.flags; 4804 } 4805 4806 if (__op->op == DRM_GPUVA_OP_MAP) { 4807 xe_assert(vm->xe, remap_op); 4808 remap_op = false; 4809 /* 4810 * In case of madvise ops DRM_GPUVA_OP_MAP is 4811 * always after DRM_GPUVA_OP_REMAP, so ensure 4812 * to propagate the flags from the vma we're 4813 * unmapping. 4814 */ 4815 op->map.vma_flags |= vma_flags & XE_VMA_CREATE_MASK; 4816 } 4817 } 4818 print_op(vm->xe, __op); 4819 } 4820 4821 xe_vma_ops_init(&vops, vm, NULL, NULL, 0); 4822 4823 if (is_madvise) 4824 vops.flags |= XE_VMA_OPS_FLAG_MADVISE; 4825 else 4826 vops.flags |= XE_VMA_OPS_FLAG_ALLOW_SVM_UNMAP; 4827 4828 err = vm_bind_ioctl_ops_parse(vm, ops, &vops); 4829 if (err) 4830 goto unwind_ops; 4831 4832 xe_vm_lock(vm, false); 4833 4834 drm_gpuva_for_each_op(__op, ops) { 4835 struct xe_vma_op *op = gpuva_op_to_vma_op(__op); 4836 struct xe_vma *vma; 4837 4838 if (__op->op == DRM_GPUVA_OP_UNMAP) { 4839 vma = gpuva_to_vma(op->base.unmap.va); 4840 /* There should be no unmap for madvise */ 4841 if (is_madvise) 4842 XE_WARN_ON("UNEXPECTED UNMAP"); 4843 4844 xe_vma_destroy(vma, NULL); 4845 } else if (__op->op == DRM_GPUVA_OP_REMAP) { 4846 vma = gpuva_to_vma(op->base.remap.unmap->va); 4847 /* In case of madvise ops Store attributes for REMAP UNMAPPED 4848 * VMA, so they can be assigned to newly MAP created vma. 4849 */ 4850 if (is_madvise) 4851 xe_vma_mem_attr_copy(&tmp_attr, &vma->attr); 4852 4853 xe_vma_destroy(gpuva_to_vma(op->base.remap.unmap->va), NULL); 4854 } else if (__op->op == DRM_GPUVA_OP_MAP) { 4855 vma = op->map.vma; 4856 /* In case of madvise call, MAP will always be followed by REMAP. 4857 * Therefore temp_attr will always have sane values, making it safe to 4858 * copy them to new vma. 4859 */ 4860 if (is_madvise) 4861 xe_vma_mem_attr_copy(&vma->attr, &tmp_attr); 4862 } 4863 } 4864 4865 xe_vm_unlock(vm); 4866 drm_gpuva_ops_free(&vm->gpuvm, ops); 4867 xe_vma_mem_attr_fini(&tmp_attr); 4868 return 0; 4869 4870 unwind_ops: 4871 vm_bind_ioctl_ops_unwind(vm, &ops, 1); 4872 free_ops: 4873 drm_gpuva_ops_free(&vm->gpuvm, ops); 4874 return err; 4875 } 4876 4877 /** 4878 * xe_vm_alloc_madvise_vma - Allocate VMA's with madvise ops 4879 * @vm: Pointer to the xe_vm structure 4880 * @start: Starting input address 4881 * @range: Size of the input range 4882 * 4883 * This function splits existing vma to create new vma for user provided input range 4884 * 4885 * Return: 0 if success 4886 */ 4887 int xe_vm_alloc_madvise_vma(struct xe_vm *vm, uint64_t start, uint64_t range) 4888 { 4889 struct drm_gpuvm_map_req map_req = { 4890 .map.va.addr = start, 4891 .map.va.range = range, 4892 }; 4893 4894 lockdep_assert_held_write(&vm->lock); 4895 4896 vm_dbg(&vm->xe->drm, "MADVISE_OPS_CREATE: addr=0x%016llx, size=0x%016llx", start, range); 4897 4898 return xe_vm_alloc_vma(vm, &map_req, true); 4899 } 4900 4901 static bool is_cpu_addr_vma_with_default_attr(struct xe_vma *vma) 4902 { 4903 return vma && xe_vma_is_cpu_addr_mirror(vma) && 4904 xe_vma_has_default_mem_attrs(vma); 4905 } 4906 4907 /** 4908 * xe_vm_find_cpu_addr_mirror_vma_range - Extend a VMA range to include adjacent CPU-mirrored VMAs 4909 * @vm: VM to search within 4910 * @start: Input/output pointer to the starting address of the range 4911 * @end: Input/output pointer to the end address of the range 4912 * 4913 * Given a range defined by @start and @range, this function checks the VMAs 4914 * immediately before and after the range. If those neighboring VMAs are 4915 * CPU-address-mirrored and have default memory attributes, the function 4916 * updates @start and @range to include them. This extended range can then 4917 * be used for merging or other operations that require a unified VMA. 4918 * 4919 * The function does not perform the merge itself; it only computes the 4920 * mergeable boundaries. 4921 */ 4922 void xe_vm_find_cpu_addr_mirror_vma_range(struct xe_vm *vm, u64 *start, u64 *end) 4923 { 4924 struct xe_vma *prev, *next; 4925 4926 lockdep_assert_held(&vm->lock); 4927 4928 if (*start >= SZ_4K) { 4929 prev = xe_vm_find_vma_by_addr(vm, *start - SZ_4K); 4930 if (is_cpu_addr_vma_with_default_attr(prev)) 4931 *start = xe_vma_start(prev); 4932 } 4933 4934 if (*end < vm->size) { 4935 next = xe_vm_find_vma_by_addr(vm, *end + 1); 4936 if (is_cpu_addr_vma_with_default_attr(next)) 4937 *end = xe_vma_end(next); 4938 } 4939 } 4940 4941 /** 4942 * xe_vm_alloc_cpu_addr_mirror_vma - Allocate CPU addr mirror vma 4943 * @vm: Pointer to the xe_vm structure 4944 * @start: Starting input address 4945 * @range: Size of the input range 4946 * 4947 * This function splits/merges existing vma to create new vma for user provided input range 4948 * 4949 * Return: 0 if success 4950 */ 4951 int xe_vm_alloc_cpu_addr_mirror_vma(struct xe_vm *vm, uint64_t start, uint64_t range) 4952 { 4953 struct drm_gpuvm_map_req map_req = { 4954 .map.va.addr = start, 4955 .map.va.range = range, 4956 }; 4957 4958 lockdep_assert_held_write(&vm->lock); 4959 4960 vm_dbg(&vm->xe->drm, "CPU_ADDR_MIRROR_VMA_OPS_CREATE: addr=0x%016llx, size=0x%016llx", 4961 start, range); 4962 4963 return xe_vm_alloc_vma(vm, &map_req, false); 4964 } 4965 4966 /** 4967 * xe_vm_add_exec_queue() - Add exec queue to VM 4968 * @vm: The VM. 4969 * @q: The exec_queue 4970 * 4971 * Add exec queue to VM, skipped if the device does not have context based TLB 4972 * invalidations. 4973 */ 4974 void xe_vm_add_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q) 4975 { 4976 struct xe_device *xe = vm->xe; 4977 4978 /* User VMs and queues only */ 4979 xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_KERNEL)); 4980 xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_PERMANENT)); 4981 xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_VM)); 4982 xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_MIGRATE)); 4983 xe_assert(xe, vm->xef); 4984 xe_assert(xe, vm == q->vm); 4985 4986 if (!xe->info.has_ctx_tlb_inval) 4987 return; 4988 4989 down_write(&vm->exec_queues.lock); 4990 list_add(&q->vm_exec_queue_link, &vm->exec_queues.list[q->gt->info.id]); 4991 ++vm->exec_queues.count[q->gt->info.id]; 4992 up_write(&vm->exec_queues.lock); 4993 } 4994 4995 /** 4996 * xe_vm_remove_exec_queue() - Remove exec queue from VM 4997 * @vm: The VM. 4998 * @q: The exec_queue 4999 * 5000 * Remove exec queue from VM, skipped if the device does not have context based 5001 * TLB invalidations. 5002 */ 5003 void xe_vm_remove_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q) 5004 { 5005 if (!vm->xe->info.has_ctx_tlb_inval) 5006 return; 5007 5008 down_write(&vm->exec_queues.lock); 5009 if (!list_empty(&q->vm_exec_queue_link)) { 5010 list_del(&q->vm_exec_queue_link); 5011 --vm->exec_queues.count[q->gt->info.id]; 5012 } 5013 up_write(&vm->exec_queues.lock); 5014 } 5015