1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include <linux/dma-fence-array.h> 7 8 #include "xe_pt.h" 9 10 #include "regs/xe_gtt_defs.h" 11 #include "xe_bo.h" 12 #include "xe_device.h" 13 #include "xe_drm_client.h" 14 #include "xe_exec_queue.h" 15 #include "xe_gt.h" 16 #include "xe_gt_tlb_invalidation.h" 17 #include "xe_migrate.h" 18 #include "xe_pt_types.h" 19 #include "xe_pt_walk.h" 20 #include "xe_res_cursor.h" 21 #include "xe_sched_job.h" 22 #include "xe_sync.h" 23 #include "xe_svm.h" 24 #include "xe_trace.h" 25 #include "xe_ttm_stolen_mgr.h" 26 #include "xe_vm.h" 27 28 struct xe_pt_dir { 29 struct xe_pt pt; 30 /** @children: Array of page-table child nodes */ 31 struct xe_ptw *children[XE_PDES]; 32 /** @staging: Array of page-table staging nodes */ 33 struct xe_ptw *staging[XE_PDES]; 34 }; 35 36 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM) 37 #define xe_pt_set_addr(__xe_pt, __addr) ((__xe_pt)->addr = (__addr)) 38 #define xe_pt_addr(__xe_pt) ((__xe_pt)->addr) 39 #else 40 #define xe_pt_set_addr(__xe_pt, __addr) 41 #define xe_pt_addr(__xe_pt) 0ull 42 #endif 43 44 static const u64 xe_normal_pt_shifts[] = {12, 21, 30, 39, 48}; 45 static const u64 xe_compact_pt_shifts[] = {16, 21, 30, 39, 48}; 46 47 #define XE_PT_HIGHEST_LEVEL (ARRAY_SIZE(xe_normal_pt_shifts) - 1) 48 49 static struct xe_pt_dir *as_xe_pt_dir(struct xe_pt *pt) 50 { 51 return container_of(pt, struct xe_pt_dir, pt); 52 } 53 54 static struct xe_pt * 55 xe_pt_entry_staging(struct xe_pt_dir *pt_dir, unsigned int index) 56 { 57 return container_of(pt_dir->staging[index], struct xe_pt, base); 58 } 59 60 static u64 __xe_pt_empty_pte(struct xe_tile *tile, struct xe_vm *vm, 61 unsigned int level) 62 { 63 struct xe_device *xe = tile_to_xe(tile); 64 u16 pat_index = xe->pat.idx[XE_CACHE_WB]; 65 u8 id = tile->id; 66 67 if (!xe_vm_has_scratch(vm)) 68 return 0; 69 70 if (level > MAX_HUGEPTE_LEVEL) 71 return vm->pt_ops->pde_encode_bo(vm->scratch_pt[id][level - 1]->bo, 72 0, pat_index); 73 74 return vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, IS_DGFX(xe), 0) | 75 XE_PTE_NULL; 76 } 77 78 static void xe_pt_free(struct xe_pt *pt) 79 { 80 if (pt->level) 81 kfree(as_xe_pt_dir(pt)); 82 else 83 kfree(pt); 84 } 85 86 /** 87 * xe_pt_create() - Create a page-table. 88 * @vm: The vm to create for. 89 * @tile: The tile to create for. 90 * @level: The page-table level. 91 * 92 * Allocate and initialize a single struct xe_pt metadata structure. Also 93 * create the corresponding page-table bo, but don't initialize it. If the 94 * level is grater than zero, then it's assumed to be a directory page- 95 * table and the directory structure is also allocated and initialized to 96 * NULL pointers. 97 * 98 * Return: A valid struct xe_pt pointer on success, Pointer error code on 99 * error. 100 */ 101 struct xe_pt *xe_pt_create(struct xe_vm *vm, struct xe_tile *tile, 102 unsigned int level) 103 { 104 struct xe_pt *pt; 105 struct xe_bo *bo; 106 u32 bo_flags; 107 int err; 108 109 if (level) { 110 struct xe_pt_dir *dir = kzalloc(sizeof(*dir), GFP_KERNEL); 111 112 pt = (dir) ? &dir->pt : NULL; 113 } else { 114 pt = kzalloc(sizeof(*pt), GFP_KERNEL); 115 } 116 if (!pt) 117 return ERR_PTR(-ENOMEM); 118 119 bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile) | 120 XE_BO_FLAG_IGNORE_MIN_PAGE_SIZE | 121 XE_BO_FLAG_NO_RESV_EVICT | XE_BO_FLAG_PAGETABLE; 122 if (vm->xef) /* userspace */ 123 bo_flags |= XE_BO_FLAG_PINNED_LATE_RESTORE; 124 125 pt->level = level; 126 bo = xe_bo_create_pin_map(vm->xe, tile, vm, SZ_4K, 127 ttm_bo_type_kernel, 128 bo_flags); 129 if (IS_ERR(bo)) { 130 err = PTR_ERR(bo); 131 goto err_kfree; 132 } 133 pt->bo = bo; 134 pt->base.children = level ? as_xe_pt_dir(pt)->children : NULL; 135 pt->base.staging = level ? as_xe_pt_dir(pt)->staging : NULL; 136 137 if (vm->xef) 138 xe_drm_client_add_bo(vm->xef->client, pt->bo); 139 xe_tile_assert(tile, level <= XE_VM_MAX_LEVEL); 140 141 return pt; 142 143 err_kfree: 144 xe_pt_free(pt); 145 return ERR_PTR(err); 146 } 147 ALLOW_ERROR_INJECTION(xe_pt_create, ERRNO); 148 149 /** 150 * xe_pt_populate_empty() - Populate a page-table bo with scratch- or zero 151 * entries. 152 * @tile: The tile the scratch pagetable of which to use. 153 * @vm: The vm we populate for. 154 * @pt: The pagetable the bo of which to initialize. 155 * 156 * Populate the page-table bo of @pt with entries pointing into the tile's 157 * scratch page-table tree if any. Otherwise populate with zeros. 158 */ 159 void xe_pt_populate_empty(struct xe_tile *tile, struct xe_vm *vm, 160 struct xe_pt *pt) 161 { 162 struct iosys_map *map = &pt->bo->vmap; 163 u64 empty; 164 int i; 165 166 if (!xe_vm_has_scratch(vm)) { 167 /* 168 * FIXME: Some memory is allocated already allocated to zero? 169 * Find out which memory that is and avoid this memset... 170 */ 171 xe_map_memset(vm->xe, map, 0, 0, SZ_4K); 172 } else { 173 empty = __xe_pt_empty_pte(tile, vm, pt->level); 174 for (i = 0; i < XE_PDES; i++) 175 xe_pt_write(vm->xe, map, i, empty); 176 } 177 } 178 179 /** 180 * xe_pt_shift() - Return the ilog2 value of the size of the address range of 181 * a page-table at a certain level. 182 * @level: The level. 183 * 184 * Return: The ilog2 value of the size of the address range of a page-table 185 * at level @level. 186 */ 187 unsigned int xe_pt_shift(unsigned int level) 188 { 189 return XE_PTE_SHIFT + XE_PDE_SHIFT * level; 190 } 191 192 /** 193 * xe_pt_destroy() - Destroy a page-table tree. 194 * @pt: The root of the page-table tree to destroy. 195 * @flags: vm flags. Currently unused. 196 * @deferred: List head of lockless list for deferred putting. NULL for 197 * immediate putting. 198 * 199 * Puts the page-table bo, recursively calls xe_pt_destroy on all children 200 * and finally frees @pt. TODO: Can we remove the @flags argument? 201 */ 202 void xe_pt_destroy(struct xe_pt *pt, u32 flags, struct llist_head *deferred) 203 { 204 int i; 205 206 if (!pt) 207 return; 208 209 XE_WARN_ON(!list_empty(&pt->bo->ttm.base.gpuva.list)); 210 xe_bo_unpin(pt->bo); 211 xe_bo_put_deferred(pt->bo, deferred); 212 213 if (pt->level > 0 && pt->num_live) { 214 struct xe_pt_dir *pt_dir = as_xe_pt_dir(pt); 215 216 for (i = 0; i < XE_PDES; i++) { 217 if (xe_pt_entry_staging(pt_dir, i)) 218 xe_pt_destroy(xe_pt_entry_staging(pt_dir, i), flags, 219 deferred); 220 } 221 } 222 xe_pt_free(pt); 223 } 224 225 /** 226 * xe_pt_clear() - Clear a page-table. 227 * @xe: xe device. 228 * @pt: The page-table. 229 * 230 * Clears page-table by setting to zero. 231 */ 232 void xe_pt_clear(struct xe_device *xe, struct xe_pt *pt) 233 { 234 struct iosys_map *map = &pt->bo->vmap; 235 236 xe_map_memset(xe, map, 0, 0, SZ_4K); 237 } 238 239 /** 240 * DOC: Pagetable building 241 * 242 * Below we use the term "page-table" for both page-directories, containing 243 * pointers to lower level page-directories or page-tables, and level 0 244 * page-tables that contain only page-table-entries pointing to memory pages. 245 * 246 * When inserting an address range in an already existing page-table tree 247 * there will typically be a set of page-tables that are shared with other 248 * address ranges, and a set that are private to this address range. 249 * The set of shared page-tables can be at most two per level, 250 * and those can't be updated immediately because the entries of those 251 * page-tables may still be in use by the gpu for other mappings. Therefore 252 * when inserting entries into those, we instead stage those insertions by 253 * adding insertion data into struct xe_vm_pgtable_update structures. This 254 * data, (subtrees for the cpu and page-table-entries for the gpu) is then 255 * added in a separate commit step. CPU-data is committed while still under the 256 * vm lock, the object lock and for userptr, the notifier lock in read mode. 257 * The GPU async data is committed either by the GPU or CPU after fulfilling 258 * relevant dependencies. 259 * For non-shared page-tables (and, in fact, for shared ones that aren't 260 * existing at the time of staging), we add the data in-place without the 261 * special update structures. This private part of the page-table tree will 262 * remain disconnected from the vm page-table tree until data is committed to 263 * the shared page tables of the vm tree in the commit phase. 264 */ 265 266 struct xe_pt_update { 267 /** @update: The update structure we're building for this parent. */ 268 struct xe_vm_pgtable_update *update; 269 /** @parent: The parent. Used to detect a parent change. */ 270 struct xe_pt *parent; 271 /** @preexisting: Whether the parent was pre-existing or allocated */ 272 bool preexisting; 273 }; 274 275 /** 276 * struct xe_pt_stage_bind_walk - Walk state for the stage_bind walk. 277 */ 278 struct xe_pt_stage_bind_walk { 279 /** @base: The base class. */ 280 struct xe_pt_walk base; 281 282 /* Input parameters for the walk */ 283 /** @vm: The vm we're building for. */ 284 struct xe_vm *vm; 285 /** @tile: The tile we're building for. */ 286 struct xe_tile *tile; 287 /** @default_vram_pte: PTE flag only template for VRAM. No address is associated */ 288 u64 default_vram_pte; 289 /** @default_system_pte: PTE flag only template for System. No address is associated */ 290 u64 default_system_pte; 291 /** @dma_offset: DMA offset to add to the PTE. */ 292 u64 dma_offset; 293 /** 294 * @needs_64K: This address range enforces 64K alignment and 295 * granularity on VRAM. 296 */ 297 bool needs_64K; 298 /** @clear_pt: clear page table entries during the bind walk */ 299 bool clear_pt; 300 /** 301 * @vma: VMA being mapped 302 */ 303 struct xe_vma *vma; 304 305 /* Also input, but is updated during the walk*/ 306 /** @curs: The DMA address cursor. */ 307 struct xe_res_cursor *curs; 308 /** @va_curs_start: The Virtual address corresponding to @curs->start */ 309 u64 va_curs_start; 310 311 /* Output */ 312 /** @wupd: Walk output data for page-table updates. */ 313 struct xe_walk_update { 314 /** @wupd.entries: Caller provided storage. */ 315 struct xe_vm_pgtable_update *entries; 316 /** @wupd.num_used_entries: Number of update @entries used. */ 317 unsigned int num_used_entries; 318 /** @wupd.updates: Tracks the update entry at a given level */ 319 struct xe_pt_update updates[XE_VM_MAX_LEVEL + 1]; 320 } wupd; 321 322 /* Walk state */ 323 /** 324 * @l0_end_addr: The end address of the current l0 leaf. Used for 325 * 64K granularity detection. 326 */ 327 u64 l0_end_addr; 328 /** @addr_64K: The start address of the current 64K chunk. */ 329 u64 addr_64K; 330 /** @found_64K: Whether @add_64K actually points to a 64K chunk. */ 331 bool found_64K; 332 }; 333 334 static int 335 xe_pt_new_shared(struct xe_walk_update *wupd, struct xe_pt *parent, 336 pgoff_t offset, bool alloc_entries) 337 { 338 struct xe_pt_update *upd = &wupd->updates[parent->level]; 339 struct xe_vm_pgtable_update *entry; 340 341 /* 342 * For *each level*, we could only have one active 343 * struct xt_pt_update at any one time. Once we move on to a 344 * new parent and page-directory, the old one is complete, and 345 * updates are either already stored in the build tree or in 346 * @wupd->entries 347 */ 348 if (likely(upd->parent == parent)) 349 return 0; 350 351 upd->parent = parent; 352 upd->preexisting = true; 353 354 if (wupd->num_used_entries == XE_VM_MAX_LEVEL * 2 + 1) 355 return -EINVAL; 356 357 entry = wupd->entries + wupd->num_used_entries++; 358 upd->update = entry; 359 entry->ofs = offset; 360 entry->pt_bo = parent->bo; 361 entry->pt = parent; 362 entry->flags = 0; 363 entry->qwords = 0; 364 entry->pt_bo->update_index = -1; 365 366 if (alloc_entries) { 367 entry->pt_entries = kmalloc_array(XE_PDES, 368 sizeof(*entry->pt_entries), 369 GFP_KERNEL); 370 if (!entry->pt_entries) 371 return -ENOMEM; 372 } 373 374 return 0; 375 } 376 377 /* 378 * NOTE: This is a very frequently called function so we allow ourselves 379 * to annotate (using branch prediction hints) the fastpath of updating a 380 * non-pre-existing pagetable with leaf ptes. 381 */ 382 static int 383 xe_pt_insert_entry(struct xe_pt_stage_bind_walk *xe_walk, struct xe_pt *parent, 384 pgoff_t offset, struct xe_pt *xe_child, u64 pte) 385 { 386 struct xe_pt_update *upd = &xe_walk->wupd.updates[parent->level]; 387 struct xe_pt_update *child_upd = xe_child ? 388 &xe_walk->wupd.updates[xe_child->level] : NULL; 389 int ret; 390 391 ret = xe_pt_new_shared(&xe_walk->wupd, parent, offset, true); 392 if (unlikely(ret)) 393 return ret; 394 395 /* 396 * Register this new pagetable so that it won't be recognized as 397 * a shared pagetable by a subsequent insertion. 398 */ 399 if (unlikely(child_upd)) { 400 child_upd->update = NULL; 401 child_upd->parent = xe_child; 402 child_upd->preexisting = false; 403 } 404 405 if (likely(!upd->preexisting)) { 406 /* Continue building a non-connected subtree. */ 407 struct iosys_map *map = &parent->bo->vmap; 408 409 if (unlikely(xe_child)) { 410 parent->base.children[offset] = &xe_child->base; 411 parent->base.staging[offset] = &xe_child->base; 412 } 413 414 xe_pt_write(xe_walk->vm->xe, map, offset, pte); 415 parent->num_live++; 416 } else { 417 /* Shared pt. Stage update. */ 418 unsigned int idx; 419 struct xe_vm_pgtable_update *entry = upd->update; 420 421 idx = offset - entry->ofs; 422 entry->pt_entries[idx].pt = xe_child; 423 entry->pt_entries[idx].pte = pte; 424 entry->qwords++; 425 } 426 427 return 0; 428 } 429 430 static bool xe_pt_hugepte_possible(u64 addr, u64 next, unsigned int level, 431 struct xe_pt_stage_bind_walk *xe_walk) 432 { 433 u64 size, dma; 434 435 if (level > MAX_HUGEPTE_LEVEL) 436 return false; 437 438 /* Does the virtual range requested cover a huge pte? */ 439 if (!xe_pt_covers(addr, next, level, &xe_walk->base)) 440 return false; 441 442 /* Does the DMA segment cover the whole pte? */ 443 if (next - xe_walk->va_curs_start > xe_walk->curs->size) 444 return false; 445 446 /* null VMA's do not have dma addresses */ 447 if (xe_vma_is_null(xe_walk->vma)) 448 return true; 449 450 /* if we are clearing page table, no dma addresses*/ 451 if (xe_walk->clear_pt) 452 return true; 453 454 /* Is the DMA address huge PTE size aligned? */ 455 size = next - addr; 456 dma = addr - xe_walk->va_curs_start + xe_res_dma(xe_walk->curs); 457 458 return IS_ALIGNED(dma, size); 459 } 460 461 /* 462 * Scan the requested mapping to check whether it can be done entirely 463 * with 64K PTEs. 464 */ 465 static bool 466 xe_pt_scan_64K(u64 addr, u64 next, struct xe_pt_stage_bind_walk *xe_walk) 467 { 468 struct xe_res_cursor curs = *xe_walk->curs; 469 470 if (!IS_ALIGNED(addr, SZ_64K)) 471 return false; 472 473 if (next > xe_walk->l0_end_addr) 474 return false; 475 476 /* null VMA's do not have dma addresses */ 477 if (xe_vma_is_null(xe_walk->vma)) 478 return true; 479 480 xe_res_next(&curs, addr - xe_walk->va_curs_start); 481 for (; addr < next; addr += SZ_64K) { 482 if (!IS_ALIGNED(xe_res_dma(&curs), SZ_64K) || curs.size < SZ_64K) 483 return false; 484 485 xe_res_next(&curs, SZ_64K); 486 } 487 488 return addr == next; 489 } 490 491 /* 492 * For non-compact "normal" 4K level-0 pagetables, we want to try to group 493 * addresses together in 64K-contigous regions to add a 64K TLB hint for the 494 * device to the PTE. 495 * This function determines whether the address is part of such a 496 * segment. For VRAM in normal pagetables, this is strictly necessary on 497 * some devices. 498 */ 499 static bool 500 xe_pt_is_pte_ps64K(u64 addr, u64 next, struct xe_pt_stage_bind_walk *xe_walk) 501 { 502 /* Address is within an already found 64k region */ 503 if (xe_walk->found_64K && addr - xe_walk->addr_64K < SZ_64K) 504 return true; 505 506 xe_walk->found_64K = xe_pt_scan_64K(addr, addr + SZ_64K, xe_walk); 507 xe_walk->addr_64K = addr; 508 509 return xe_walk->found_64K; 510 } 511 512 static int 513 xe_pt_stage_bind_entry(struct xe_ptw *parent, pgoff_t offset, 514 unsigned int level, u64 addr, u64 next, 515 struct xe_ptw **child, 516 enum page_walk_action *action, 517 struct xe_pt_walk *walk) 518 { 519 struct xe_pt_stage_bind_walk *xe_walk = 520 container_of(walk, typeof(*xe_walk), base); 521 u16 pat_index = xe_walk->vma->pat_index; 522 struct xe_pt *xe_parent = container_of(parent, typeof(*xe_parent), base); 523 struct xe_vm *vm = xe_walk->vm; 524 struct xe_pt *xe_child; 525 bool covers; 526 int ret = 0; 527 u64 pte; 528 529 /* Is this a leaf entry ?*/ 530 if (level == 0 || xe_pt_hugepte_possible(addr, next, level, xe_walk)) { 531 struct xe_res_cursor *curs = xe_walk->curs; 532 bool is_null = xe_vma_is_null(xe_walk->vma); 533 bool is_vram = is_null ? false : xe_res_is_vram(curs); 534 535 XE_WARN_ON(xe_walk->va_curs_start != addr); 536 537 if (xe_walk->clear_pt) { 538 pte = 0; 539 } else { 540 pte = vm->pt_ops->pte_encode_vma(is_null ? 0 : 541 xe_res_dma(curs) + 542 xe_walk->dma_offset, 543 xe_walk->vma, 544 pat_index, level); 545 if (!is_null) 546 pte |= is_vram ? xe_walk->default_vram_pte : 547 xe_walk->default_system_pte; 548 549 /* 550 * Set the XE_PTE_PS64 hint if possible, otherwise if 551 * this device *requires* 64K PTE size for VRAM, fail. 552 */ 553 if (level == 0 && !xe_parent->is_compact) { 554 if (xe_pt_is_pte_ps64K(addr, next, xe_walk)) { 555 xe_walk->vma->gpuva.flags |= 556 XE_VMA_PTE_64K; 557 pte |= XE_PTE_PS64; 558 } else if (XE_WARN_ON(xe_walk->needs_64K && 559 is_vram)) { 560 return -EINVAL; 561 } 562 } 563 } 564 565 ret = xe_pt_insert_entry(xe_walk, xe_parent, offset, NULL, pte); 566 if (unlikely(ret)) 567 return ret; 568 569 if (!is_null && !xe_walk->clear_pt) 570 xe_res_next(curs, next - addr); 571 xe_walk->va_curs_start = next; 572 xe_walk->vma->gpuva.flags |= (XE_VMA_PTE_4K << level); 573 *action = ACTION_CONTINUE; 574 575 return ret; 576 } 577 578 /* 579 * Descending to lower level. Determine if we need to allocate a 580 * new page table or -directory, which we do if there is no 581 * previous one or there is one we can completely replace. 582 */ 583 if (level == 1) { 584 walk->shifts = xe_normal_pt_shifts; 585 xe_walk->l0_end_addr = next; 586 } 587 588 covers = xe_pt_covers(addr, next, level, &xe_walk->base); 589 if (covers || !*child) { 590 u64 flags = 0; 591 592 xe_child = xe_pt_create(xe_walk->vm, xe_walk->tile, level - 1); 593 if (IS_ERR(xe_child)) 594 return PTR_ERR(xe_child); 595 596 xe_pt_set_addr(xe_child, 597 round_down(addr, 1ull << walk->shifts[level])); 598 599 if (!covers) 600 xe_pt_populate_empty(xe_walk->tile, xe_walk->vm, xe_child); 601 602 *child = &xe_child->base; 603 604 /* 605 * Prefer the compact pagetable layout for L0 if possible. Only 606 * possible if VMA covers entire 2MB region as compact 64k and 607 * 4k pages cannot be mixed within a 2MB region. 608 * TODO: Suballocate the pt bo to avoid wasting a lot of 609 * memory. 610 */ 611 if (GRAPHICS_VERx100(tile_to_xe(xe_walk->tile)) >= 1250 && level == 1 && 612 covers && xe_pt_scan_64K(addr, next, xe_walk)) { 613 walk->shifts = xe_compact_pt_shifts; 614 xe_walk->vma->gpuva.flags |= XE_VMA_PTE_COMPACT; 615 flags |= XE_PDE_64K; 616 xe_child->is_compact = true; 617 } 618 619 pte = vm->pt_ops->pde_encode_bo(xe_child->bo, 0, pat_index) | flags; 620 ret = xe_pt_insert_entry(xe_walk, xe_parent, offset, xe_child, 621 pte); 622 } 623 624 *action = ACTION_SUBTREE; 625 return ret; 626 } 627 628 static const struct xe_pt_walk_ops xe_pt_stage_bind_ops = { 629 .pt_entry = xe_pt_stage_bind_entry, 630 }; 631 632 /* 633 * Default atomic expectations for different allocation scenarios are as follows: 634 * 635 * 1. Traditional API: When the VM is not in LR mode: 636 * - Device atomics are expected to function with all allocations. 637 * 638 * 2. Compute/SVM API: When the VM is in LR mode: 639 * - Device atomics are the default behavior when the bo is placed in a single region. 640 * - In all other cases device atomics will be disabled with AE=0 until an application 641 * request differently using a ioctl like madvise. 642 */ 643 static bool xe_atomic_for_vram(struct xe_vm *vm) 644 { 645 return true; 646 } 647 648 static bool xe_atomic_for_system(struct xe_vm *vm, struct xe_bo *bo) 649 { 650 struct xe_device *xe = vm->xe; 651 652 if (!xe->info.has_device_atomics_on_smem) 653 return false; 654 655 /* 656 * If a SMEM+LMEM allocation is backed by SMEM, a device 657 * atomics will cause a gpu page fault and which then 658 * gets migrated to LMEM, bind such allocations with 659 * device atomics enabled. 660 * 661 * TODO: Revisit this. Perhaps add something like a 662 * fault_on_atomics_in_system UAPI flag. 663 * Note that this also prohibits GPU atomics in LR mode for 664 * userptr and system memory on DGFX. 665 */ 666 return (!IS_DGFX(xe) || (!xe_vm_in_lr_mode(vm) || 667 (bo && xe_bo_has_single_placement(bo)))); 668 } 669 670 /** 671 * xe_pt_stage_bind() - Build a disconnected page-table tree for a given address 672 * range. 673 * @tile: The tile we're building for. 674 * @vma: The vma indicating the address range. 675 * @range: The range indicating the address range. 676 * @entries: Storage for the update entries used for connecting the tree to 677 * the main tree at commit time. 678 * @num_entries: On output contains the number of @entries used. 679 * @clear_pt: Clear the page table entries. 680 * 681 * This function builds a disconnected page-table tree for a given address 682 * range. The tree is connected to the main vm tree for the gpu using 683 * xe_migrate_update_pgtables() and for the cpu using xe_pt_commit_bind(). 684 * The function builds xe_vm_pgtable_update structures for already existing 685 * shared page-tables, and non-existing shared and non-shared page-tables 686 * are built and populated directly. 687 * 688 * Return 0 on success, negative error code on error. 689 */ 690 static int 691 xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma, 692 struct xe_svm_range *range, 693 struct xe_vm_pgtable_update *entries, 694 u32 *num_entries, bool clear_pt) 695 { 696 struct xe_device *xe = tile_to_xe(tile); 697 struct xe_bo *bo = xe_vma_bo(vma); 698 struct xe_res_cursor curs; 699 struct xe_vm *vm = xe_vma_vm(vma); 700 struct xe_pt_stage_bind_walk xe_walk = { 701 .base = { 702 .ops = &xe_pt_stage_bind_ops, 703 .shifts = xe_normal_pt_shifts, 704 .max_level = XE_PT_HIGHEST_LEVEL, 705 .staging = true, 706 }, 707 .vm = vm, 708 .tile = tile, 709 .curs = &curs, 710 .va_curs_start = range ? range->base.itree.start : 711 xe_vma_start(vma), 712 .vma = vma, 713 .wupd.entries = entries, 714 .clear_pt = clear_pt, 715 }; 716 struct xe_pt *pt = vm->pt_root[tile->id]; 717 int ret; 718 719 if (range) { 720 /* Move this entire thing to xe_svm.c? */ 721 xe_svm_notifier_lock(vm); 722 if (!xe_svm_range_pages_valid(range)) { 723 xe_svm_range_debug(range, "BIND PREPARE - RETRY"); 724 xe_svm_notifier_unlock(vm); 725 return -EAGAIN; 726 } 727 if (xe_svm_range_has_dma_mapping(range)) { 728 xe_res_first_dma(range->base.dma_addr, 0, 729 range->base.itree.last + 1 - range->base.itree.start, 730 &curs); 731 xe_svm_range_debug(range, "BIND PREPARE - MIXED"); 732 } else { 733 xe_assert(xe, false); 734 } 735 /* 736 * Note, when unlocking the resource cursor dma addresses may become 737 * stale, but the bind will be aborted anyway at commit time. 738 */ 739 xe_svm_notifier_unlock(vm); 740 } 741 742 xe_walk.needs_64K = (vm->flags & XE_VM_FLAG_64K); 743 if (clear_pt) 744 goto walk_pt; 745 746 if (vma->gpuva.flags & XE_VMA_ATOMIC_PTE_BIT) { 747 xe_walk.default_vram_pte = xe_atomic_for_vram(vm) ? XE_USM_PPGTT_PTE_AE : 0; 748 xe_walk.default_system_pte = xe_atomic_for_system(vm, bo) ? 749 XE_USM_PPGTT_PTE_AE : 0; 750 } 751 752 xe_walk.default_vram_pte |= XE_PPGTT_PTE_DM; 753 xe_walk.dma_offset = bo ? vram_region_gpu_offset(bo->ttm.resource) : 0; 754 if (!range) 755 xe_bo_assert_held(bo); 756 757 if (!xe_vma_is_null(vma) && !range) { 758 if (xe_vma_is_userptr(vma)) 759 xe_res_first_sg(to_userptr_vma(vma)->userptr.sg, 0, 760 xe_vma_size(vma), &curs); 761 else if (xe_bo_is_vram(bo) || xe_bo_is_stolen(bo)) 762 xe_res_first(bo->ttm.resource, xe_vma_bo_offset(vma), 763 xe_vma_size(vma), &curs); 764 else 765 xe_res_first_sg(xe_bo_sg(bo), xe_vma_bo_offset(vma), 766 xe_vma_size(vma), &curs); 767 } else if (!range) { 768 curs.size = xe_vma_size(vma); 769 } 770 771 walk_pt: 772 ret = xe_pt_walk_range(&pt->base, pt->level, 773 range ? range->base.itree.start : xe_vma_start(vma), 774 range ? range->base.itree.last + 1 : xe_vma_end(vma), 775 &xe_walk.base); 776 777 *num_entries = xe_walk.wupd.num_used_entries; 778 return ret; 779 } 780 781 /** 782 * xe_pt_nonshared_offsets() - Determine the non-shared entry offsets of a 783 * shared pagetable. 784 * @addr: The start address within the non-shared pagetable. 785 * @end: The end address within the non-shared pagetable. 786 * @level: The level of the non-shared pagetable. 787 * @walk: Walk info. The function adjusts the walk action. 788 * @action: next action to perform (see enum page_walk_action) 789 * @offset: Ignored on input, First non-shared entry on output. 790 * @end_offset: Ignored on input, Last non-shared entry + 1 on output. 791 * 792 * A non-shared page-table has some entries that belong to the address range 793 * and others that don't. This function determines the entries that belong 794 * fully to the address range. Depending on level, some entries may 795 * partially belong to the address range (that can't happen at level 0). 796 * The function detects that and adjust those offsets to not include those 797 * partial entries. Iff it does detect partial entries, we know that there must 798 * be shared page tables also at lower levels, so it adjusts the walk action 799 * accordingly. 800 * 801 * Return: true if there were non-shared entries, false otherwise. 802 */ 803 static bool xe_pt_nonshared_offsets(u64 addr, u64 end, unsigned int level, 804 struct xe_pt_walk *walk, 805 enum page_walk_action *action, 806 pgoff_t *offset, pgoff_t *end_offset) 807 { 808 u64 size = 1ull << walk->shifts[level]; 809 810 *offset = xe_pt_offset(addr, level, walk); 811 *end_offset = xe_pt_num_entries(addr, end, level, walk) + *offset; 812 813 if (!level) 814 return true; 815 816 /* 817 * If addr or next are not size aligned, there are shared pts at lower 818 * level, so in that case traverse down the subtree 819 */ 820 *action = ACTION_CONTINUE; 821 if (!IS_ALIGNED(addr, size)) { 822 *action = ACTION_SUBTREE; 823 (*offset)++; 824 } 825 826 if (!IS_ALIGNED(end, size)) { 827 *action = ACTION_SUBTREE; 828 (*end_offset)--; 829 } 830 831 return *end_offset > *offset; 832 } 833 834 struct xe_pt_zap_ptes_walk { 835 /** @base: The walk base-class */ 836 struct xe_pt_walk base; 837 838 /* Input parameters for the walk */ 839 /** @tile: The tile we're building for */ 840 struct xe_tile *tile; 841 842 /* Output */ 843 /** @needs_invalidate: Whether we need to invalidate TLB*/ 844 bool needs_invalidate; 845 }; 846 847 static int xe_pt_zap_ptes_entry(struct xe_ptw *parent, pgoff_t offset, 848 unsigned int level, u64 addr, u64 next, 849 struct xe_ptw **child, 850 enum page_walk_action *action, 851 struct xe_pt_walk *walk) 852 { 853 struct xe_pt_zap_ptes_walk *xe_walk = 854 container_of(walk, typeof(*xe_walk), base); 855 struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base); 856 pgoff_t end_offset; 857 858 XE_WARN_ON(!*child); 859 XE_WARN_ON(!level); 860 861 /* 862 * Note that we're called from an entry callback, and we're dealing 863 * with the child of that entry rather than the parent, so need to 864 * adjust level down. 865 */ 866 if (xe_pt_nonshared_offsets(addr, next, --level, walk, action, &offset, 867 &end_offset)) { 868 xe_map_memset(tile_to_xe(xe_walk->tile), &xe_child->bo->vmap, 869 offset * sizeof(u64), 0, 870 (end_offset - offset) * sizeof(u64)); 871 xe_walk->needs_invalidate = true; 872 } 873 874 return 0; 875 } 876 877 static const struct xe_pt_walk_ops xe_pt_zap_ptes_ops = { 878 .pt_entry = xe_pt_zap_ptes_entry, 879 }; 880 881 /** 882 * xe_pt_zap_ptes() - Zap (zero) gpu ptes of an address range 883 * @tile: The tile we're zapping for. 884 * @vma: GPU VMA detailing address range. 885 * 886 * Eviction and Userptr invalidation needs to be able to zap the 887 * gpu ptes of a given address range in pagefaulting mode. 888 * In order to be able to do that, that function needs access to the shared 889 * page-table entrieaso it can either clear the leaf PTEs or 890 * clear the pointers to lower-level page-tables. The caller is required 891 * to hold the necessary locks to ensure neither the page-table connectivity 892 * nor the page-table entries of the range is updated from under us. 893 * 894 * Return: Whether ptes were actually updated and a TLB invalidation is 895 * required. 896 */ 897 bool xe_pt_zap_ptes(struct xe_tile *tile, struct xe_vma *vma) 898 { 899 struct xe_pt_zap_ptes_walk xe_walk = { 900 .base = { 901 .ops = &xe_pt_zap_ptes_ops, 902 .shifts = xe_normal_pt_shifts, 903 .max_level = XE_PT_HIGHEST_LEVEL, 904 }, 905 .tile = tile, 906 }; 907 struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id]; 908 u8 pt_mask = (vma->tile_present & ~vma->tile_invalidated); 909 910 if (!(pt_mask & BIT(tile->id))) 911 return false; 912 913 (void)xe_pt_walk_shared(&pt->base, pt->level, xe_vma_start(vma), 914 xe_vma_end(vma), &xe_walk.base); 915 916 return xe_walk.needs_invalidate; 917 } 918 919 /** 920 * xe_pt_zap_ptes_range() - Zap (zero) gpu ptes of a SVM range 921 * @tile: The tile we're zapping for. 922 * @vm: The VM we're zapping for. 923 * @range: The SVM range we're zapping for. 924 * 925 * SVM invalidation needs to be able to zap the gpu ptes of a given address 926 * range. In order to be able to do that, that function needs access to the 927 * shared page-table entries so it can either clear the leaf PTEs or 928 * clear the pointers to lower-level page-tables. The caller is required 929 * to hold the SVM notifier lock. 930 * 931 * Return: Whether ptes were actually updated and a TLB invalidation is 932 * required. 933 */ 934 bool xe_pt_zap_ptes_range(struct xe_tile *tile, struct xe_vm *vm, 935 struct xe_svm_range *range) 936 { 937 struct xe_pt_zap_ptes_walk xe_walk = { 938 .base = { 939 .ops = &xe_pt_zap_ptes_ops, 940 .shifts = xe_normal_pt_shifts, 941 .max_level = XE_PT_HIGHEST_LEVEL, 942 }, 943 .tile = tile, 944 }; 945 struct xe_pt *pt = vm->pt_root[tile->id]; 946 u8 pt_mask = (range->tile_present & ~range->tile_invalidated); 947 948 xe_svm_assert_in_notifier(vm); 949 950 if (!(pt_mask & BIT(tile->id))) 951 return false; 952 953 (void)xe_pt_walk_shared(&pt->base, pt->level, range->base.itree.start, 954 range->base.itree.last + 1, &xe_walk.base); 955 956 return xe_walk.needs_invalidate; 957 } 958 959 static void 960 xe_vm_populate_pgtable(struct xe_migrate_pt_update *pt_update, struct xe_tile *tile, 961 struct iosys_map *map, void *data, 962 u32 qword_ofs, u32 num_qwords, 963 const struct xe_vm_pgtable_update *update) 964 { 965 struct xe_pt_entry *ptes = update->pt_entries; 966 u64 *ptr = data; 967 u32 i; 968 969 for (i = 0; i < num_qwords; i++) { 970 if (map) 971 xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) * 972 sizeof(u64), u64, ptes[i].pte); 973 else 974 ptr[i] = ptes[i].pte; 975 } 976 } 977 978 static void xe_pt_cancel_bind(struct xe_vma *vma, 979 struct xe_vm_pgtable_update *entries, 980 u32 num_entries) 981 { 982 u32 i, j; 983 984 for (i = 0; i < num_entries; i++) { 985 struct xe_pt *pt = entries[i].pt; 986 987 if (!pt) 988 continue; 989 990 if (pt->level) { 991 for (j = 0; j < entries[i].qwords; j++) 992 xe_pt_destroy(entries[i].pt_entries[j].pt, 993 xe_vma_vm(vma)->flags, NULL); 994 } 995 996 kfree(entries[i].pt_entries); 997 entries[i].pt_entries = NULL; 998 entries[i].qwords = 0; 999 } 1000 } 1001 1002 #define XE_INVALID_VMA ((struct xe_vma *)(0xdeaddeadull)) 1003 1004 static void xe_pt_commit_prepare_locks_assert(struct xe_vma *vma) 1005 { 1006 struct xe_vm *vm; 1007 1008 if (vma == XE_INVALID_VMA) 1009 return; 1010 1011 vm = xe_vma_vm(vma); 1012 lockdep_assert_held(&vm->lock); 1013 1014 if (!xe_vma_has_no_bo(vma)) 1015 dma_resv_assert_held(xe_vma_bo(vma)->ttm.base.resv); 1016 1017 xe_vm_assert_held(vm); 1018 } 1019 1020 static void xe_pt_commit_locks_assert(struct xe_vma *vma) 1021 { 1022 struct xe_vm *vm; 1023 1024 if (vma == XE_INVALID_VMA) 1025 return; 1026 1027 vm = xe_vma_vm(vma); 1028 xe_pt_commit_prepare_locks_assert(vma); 1029 1030 if (xe_vma_is_userptr(vma)) 1031 lockdep_assert_held_read(&vm->userptr.notifier_lock); 1032 } 1033 1034 static void xe_pt_commit(struct xe_vma *vma, 1035 struct xe_vm_pgtable_update *entries, 1036 u32 num_entries, struct llist_head *deferred) 1037 { 1038 u32 i, j; 1039 1040 xe_pt_commit_locks_assert(vma); 1041 1042 for (i = 0; i < num_entries; i++) { 1043 struct xe_pt *pt = entries[i].pt; 1044 struct xe_pt_dir *pt_dir; 1045 1046 if (!pt->level) 1047 continue; 1048 1049 pt_dir = as_xe_pt_dir(pt); 1050 for (j = 0; j < entries[i].qwords; j++) { 1051 struct xe_pt *oldpte = entries[i].pt_entries[j].pt; 1052 int j_ = j + entries[i].ofs; 1053 1054 pt_dir->children[j_] = pt_dir->staging[j_]; 1055 xe_pt_destroy(oldpte, (vma == XE_INVALID_VMA) ? 0 : 1056 xe_vma_vm(vma)->flags, deferred); 1057 } 1058 } 1059 } 1060 1061 static void xe_pt_abort_bind(struct xe_vma *vma, 1062 struct xe_vm_pgtable_update *entries, 1063 u32 num_entries, bool rebind) 1064 { 1065 int i, j; 1066 1067 xe_pt_commit_prepare_locks_assert(vma); 1068 1069 for (i = num_entries - 1; i >= 0; --i) { 1070 struct xe_pt *pt = entries[i].pt; 1071 struct xe_pt_dir *pt_dir; 1072 1073 if (!rebind) 1074 pt->num_live -= entries[i].qwords; 1075 1076 if (!pt->level) 1077 continue; 1078 1079 pt_dir = as_xe_pt_dir(pt); 1080 for (j = 0; j < entries[i].qwords; j++) { 1081 u32 j_ = j + entries[i].ofs; 1082 struct xe_pt *newpte = xe_pt_entry_staging(pt_dir, j_); 1083 struct xe_pt *oldpte = entries[i].pt_entries[j].pt; 1084 1085 pt_dir->staging[j_] = oldpte ? &oldpte->base : 0; 1086 xe_pt_destroy(newpte, xe_vma_vm(vma)->flags, NULL); 1087 } 1088 } 1089 } 1090 1091 static void xe_pt_commit_prepare_bind(struct xe_vma *vma, 1092 struct xe_vm_pgtable_update *entries, 1093 u32 num_entries, bool rebind) 1094 { 1095 u32 i, j; 1096 1097 xe_pt_commit_prepare_locks_assert(vma); 1098 1099 for (i = 0; i < num_entries; i++) { 1100 struct xe_pt *pt = entries[i].pt; 1101 struct xe_pt_dir *pt_dir; 1102 1103 if (!rebind) 1104 pt->num_live += entries[i].qwords; 1105 1106 if (!pt->level) 1107 continue; 1108 1109 pt_dir = as_xe_pt_dir(pt); 1110 for (j = 0; j < entries[i].qwords; j++) { 1111 u32 j_ = j + entries[i].ofs; 1112 struct xe_pt *newpte = entries[i].pt_entries[j].pt; 1113 struct xe_pt *oldpte = NULL; 1114 1115 if (xe_pt_entry_staging(pt_dir, j_)) 1116 oldpte = xe_pt_entry_staging(pt_dir, j_); 1117 1118 pt_dir->staging[j_] = &newpte->base; 1119 entries[i].pt_entries[j].pt = oldpte; 1120 } 1121 } 1122 } 1123 1124 static void xe_pt_free_bind(struct xe_vm_pgtable_update *entries, 1125 u32 num_entries) 1126 { 1127 u32 i; 1128 1129 for (i = 0; i < num_entries; i++) 1130 kfree(entries[i].pt_entries); 1131 } 1132 1133 static int 1134 xe_pt_prepare_bind(struct xe_tile *tile, struct xe_vma *vma, 1135 struct xe_svm_range *range, 1136 struct xe_vm_pgtable_update *entries, 1137 u32 *num_entries, bool invalidate_on_bind) 1138 { 1139 int err; 1140 1141 *num_entries = 0; 1142 err = xe_pt_stage_bind(tile, vma, range, entries, num_entries, 1143 invalidate_on_bind); 1144 if (!err) 1145 xe_tile_assert(tile, *num_entries); 1146 1147 return err; 1148 } 1149 1150 static void xe_vm_dbg_print_entries(struct xe_device *xe, 1151 const struct xe_vm_pgtable_update *entries, 1152 unsigned int num_entries, bool bind) 1153 #if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)) 1154 { 1155 unsigned int i; 1156 1157 vm_dbg(&xe->drm, "%s: %u entries to update\n", bind ? "bind" : "unbind", 1158 num_entries); 1159 for (i = 0; i < num_entries; i++) { 1160 const struct xe_vm_pgtable_update *entry = &entries[i]; 1161 struct xe_pt *xe_pt = entry->pt; 1162 u64 page_size = 1ull << xe_pt_shift(xe_pt->level); 1163 u64 end; 1164 u64 start; 1165 1166 xe_assert(xe, !entry->pt->is_compact); 1167 start = entry->ofs * page_size; 1168 end = start + page_size * entry->qwords; 1169 vm_dbg(&xe->drm, 1170 "\t%u: Update level %u at (%u + %u) [%llx...%llx) f:%x\n", 1171 i, xe_pt->level, entry->ofs, entry->qwords, 1172 xe_pt_addr(xe_pt) + start, xe_pt_addr(xe_pt) + end, 0); 1173 } 1174 } 1175 #else 1176 {} 1177 #endif 1178 1179 static bool no_in_syncs(struct xe_sync_entry *syncs, u32 num_syncs) 1180 { 1181 int i; 1182 1183 for (i = 0; i < num_syncs; i++) { 1184 struct dma_fence *fence = syncs[i].fence; 1185 1186 if (fence && !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, 1187 &fence->flags)) 1188 return false; 1189 } 1190 1191 return true; 1192 } 1193 1194 static int job_test_add_deps(struct xe_sched_job *job, 1195 struct dma_resv *resv, 1196 enum dma_resv_usage usage) 1197 { 1198 if (!job) { 1199 if (!dma_resv_test_signaled(resv, usage)) 1200 return -ETIME; 1201 1202 return 0; 1203 } 1204 1205 return xe_sched_job_add_deps(job, resv, usage); 1206 } 1207 1208 static int vma_add_deps(struct xe_vma *vma, struct xe_sched_job *job) 1209 { 1210 struct xe_bo *bo = xe_vma_bo(vma); 1211 1212 xe_bo_assert_held(bo); 1213 1214 if (bo && !bo->vm) 1215 return job_test_add_deps(job, bo->ttm.base.resv, 1216 DMA_RESV_USAGE_KERNEL); 1217 1218 return 0; 1219 } 1220 1221 static int op_add_deps(struct xe_vm *vm, struct xe_vma_op *op, 1222 struct xe_sched_job *job) 1223 { 1224 int err = 0; 1225 1226 /* 1227 * No need to check for is_cpu_addr_mirror here as vma_add_deps is a 1228 * NOP if VMA is_cpu_addr_mirror 1229 */ 1230 1231 switch (op->base.op) { 1232 case DRM_GPUVA_OP_MAP: 1233 if (!op->map.immediate && xe_vm_in_fault_mode(vm)) 1234 break; 1235 1236 err = vma_add_deps(op->map.vma, job); 1237 break; 1238 case DRM_GPUVA_OP_REMAP: 1239 if (op->remap.prev) 1240 err = vma_add_deps(op->remap.prev, job); 1241 if (!err && op->remap.next) 1242 err = vma_add_deps(op->remap.next, job); 1243 break; 1244 case DRM_GPUVA_OP_UNMAP: 1245 break; 1246 case DRM_GPUVA_OP_PREFETCH: 1247 err = vma_add_deps(gpuva_to_vma(op->base.prefetch.va), job); 1248 break; 1249 case DRM_GPUVA_OP_DRIVER: 1250 break; 1251 default: 1252 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); 1253 } 1254 1255 return err; 1256 } 1257 1258 static int xe_pt_vm_dependencies(struct xe_sched_job *job, 1259 struct xe_vm *vm, 1260 struct xe_vma_ops *vops, 1261 struct xe_vm_pgtable_update_ops *pt_update_ops, 1262 struct xe_range_fence_tree *rftree) 1263 { 1264 struct xe_range_fence *rtfence; 1265 struct dma_fence *fence; 1266 struct xe_vma_op *op; 1267 int err = 0, i; 1268 1269 xe_vm_assert_held(vm); 1270 1271 if (!job && !no_in_syncs(vops->syncs, vops->num_syncs)) 1272 return -ETIME; 1273 1274 if (!job && !xe_exec_queue_is_idle(pt_update_ops->q)) 1275 return -ETIME; 1276 1277 if (pt_update_ops->wait_vm_bookkeep || pt_update_ops->wait_vm_kernel) { 1278 err = job_test_add_deps(job, xe_vm_resv(vm), 1279 pt_update_ops->wait_vm_bookkeep ? 1280 DMA_RESV_USAGE_BOOKKEEP : 1281 DMA_RESV_USAGE_KERNEL); 1282 if (err) 1283 return err; 1284 } 1285 1286 rtfence = xe_range_fence_tree_first(rftree, pt_update_ops->start, 1287 pt_update_ops->last); 1288 while (rtfence) { 1289 fence = rtfence->fence; 1290 1291 if (!dma_fence_is_signaled(fence)) { 1292 /* 1293 * Is this a CPU update? GPU is busy updating, so return 1294 * an error 1295 */ 1296 if (!job) 1297 return -ETIME; 1298 1299 dma_fence_get(fence); 1300 err = drm_sched_job_add_dependency(&job->drm, fence); 1301 if (err) 1302 return err; 1303 } 1304 1305 rtfence = xe_range_fence_tree_next(rtfence, 1306 pt_update_ops->start, 1307 pt_update_ops->last); 1308 } 1309 1310 list_for_each_entry(op, &vops->list, link) { 1311 err = op_add_deps(vm, op, job); 1312 if (err) 1313 return err; 1314 } 1315 1316 if (!(pt_update_ops->q->flags & EXEC_QUEUE_FLAG_KERNEL)) { 1317 if (job) 1318 err = xe_sched_job_last_fence_add_dep(job, vm); 1319 else 1320 err = xe_exec_queue_last_fence_test_dep(pt_update_ops->q, vm); 1321 } 1322 1323 for (i = 0; job && !err && i < vops->num_syncs; i++) 1324 err = xe_sync_entry_add_deps(&vops->syncs[i], job); 1325 1326 return err; 1327 } 1328 1329 static int xe_pt_pre_commit(struct xe_migrate_pt_update *pt_update) 1330 { 1331 struct xe_vma_ops *vops = pt_update->vops; 1332 struct xe_vm *vm = vops->vm; 1333 struct xe_range_fence_tree *rftree = &vm->rftree[pt_update->tile_id]; 1334 struct xe_vm_pgtable_update_ops *pt_update_ops = 1335 &vops->pt_update_ops[pt_update->tile_id]; 1336 1337 return xe_pt_vm_dependencies(pt_update->job, vm, pt_update->vops, 1338 pt_update_ops, rftree); 1339 } 1340 1341 #ifdef CONFIG_DRM_XE_USERPTR_INVAL_INJECT 1342 1343 static bool xe_pt_userptr_inject_eagain(struct xe_userptr_vma *uvma) 1344 { 1345 u32 divisor = uvma->userptr.divisor ? uvma->userptr.divisor : 2; 1346 static u32 count; 1347 1348 if (count++ % divisor == divisor - 1) { 1349 uvma->userptr.divisor = divisor << 1; 1350 return true; 1351 } 1352 1353 return false; 1354 } 1355 1356 #else 1357 1358 static bool xe_pt_userptr_inject_eagain(struct xe_userptr_vma *uvma) 1359 { 1360 return false; 1361 } 1362 1363 #endif 1364 1365 static int vma_check_userptr(struct xe_vm *vm, struct xe_vma *vma, 1366 struct xe_vm_pgtable_update_ops *pt_update) 1367 { 1368 struct xe_userptr_vma *uvma; 1369 unsigned long notifier_seq; 1370 1371 lockdep_assert_held_read(&vm->userptr.notifier_lock); 1372 1373 if (!xe_vma_is_userptr(vma)) 1374 return 0; 1375 1376 uvma = to_userptr_vma(vma); 1377 if (xe_pt_userptr_inject_eagain(uvma)) 1378 xe_vma_userptr_force_invalidate(uvma); 1379 1380 notifier_seq = uvma->userptr.notifier_seq; 1381 1382 if (!mmu_interval_read_retry(&uvma->userptr.notifier, 1383 notifier_seq)) 1384 return 0; 1385 1386 if (xe_vm_in_fault_mode(vm)) 1387 return -EAGAIN; 1388 1389 /* 1390 * Just continue the operation since exec or rebind worker 1391 * will take care of rebinding. 1392 */ 1393 return 0; 1394 } 1395 1396 static int op_check_userptr(struct xe_vm *vm, struct xe_vma_op *op, 1397 struct xe_vm_pgtable_update_ops *pt_update) 1398 { 1399 int err = 0; 1400 1401 lockdep_assert_held_read(&vm->userptr.notifier_lock); 1402 1403 switch (op->base.op) { 1404 case DRM_GPUVA_OP_MAP: 1405 if (!op->map.immediate && xe_vm_in_fault_mode(vm)) 1406 break; 1407 1408 err = vma_check_userptr(vm, op->map.vma, pt_update); 1409 break; 1410 case DRM_GPUVA_OP_REMAP: 1411 if (op->remap.prev) 1412 err = vma_check_userptr(vm, op->remap.prev, pt_update); 1413 if (!err && op->remap.next) 1414 err = vma_check_userptr(vm, op->remap.next, pt_update); 1415 break; 1416 case DRM_GPUVA_OP_UNMAP: 1417 break; 1418 case DRM_GPUVA_OP_PREFETCH: 1419 err = vma_check_userptr(vm, gpuva_to_vma(op->base.prefetch.va), 1420 pt_update); 1421 break; 1422 default: 1423 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); 1424 } 1425 1426 return err; 1427 } 1428 1429 static int xe_pt_userptr_pre_commit(struct xe_migrate_pt_update *pt_update) 1430 { 1431 struct xe_vm *vm = pt_update->vops->vm; 1432 struct xe_vma_ops *vops = pt_update->vops; 1433 struct xe_vm_pgtable_update_ops *pt_update_ops = 1434 &vops->pt_update_ops[pt_update->tile_id]; 1435 struct xe_vma_op *op; 1436 int err; 1437 1438 err = xe_pt_pre_commit(pt_update); 1439 if (err) 1440 return err; 1441 1442 down_read(&vm->userptr.notifier_lock); 1443 1444 list_for_each_entry(op, &vops->list, link) { 1445 err = op_check_userptr(vm, op, pt_update_ops); 1446 if (err) { 1447 up_read(&vm->userptr.notifier_lock); 1448 break; 1449 } 1450 } 1451 1452 return err; 1453 } 1454 1455 #if IS_ENABLED(CONFIG_DRM_XE_GPUSVM) 1456 static int xe_pt_svm_pre_commit(struct xe_migrate_pt_update *pt_update) 1457 { 1458 struct xe_vm *vm = pt_update->vops->vm; 1459 struct xe_vma_ops *vops = pt_update->vops; 1460 struct xe_vma_op *op; 1461 int err; 1462 1463 err = xe_pt_pre_commit(pt_update); 1464 if (err) 1465 return err; 1466 1467 xe_svm_notifier_lock(vm); 1468 1469 list_for_each_entry(op, &vops->list, link) { 1470 struct xe_svm_range *range = op->map_range.range; 1471 1472 if (op->subop == XE_VMA_SUBOP_UNMAP_RANGE) 1473 continue; 1474 1475 xe_svm_range_debug(range, "PRE-COMMIT"); 1476 1477 xe_assert(vm->xe, xe_vma_is_cpu_addr_mirror(op->map_range.vma)); 1478 xe_assert(vm->xe, op->subop == XE_VMA_SUBOP_MAP_RANGE); 1479 1480 if (!xe_svm_range_pages_valid(range)) { 1481 xe_svm_range_debug(range, "PRE-COMMIT - RETRY"); 1482 xe_svm_notifier_unlock(vm); 1483 return -EAGAIN; 1484 } 1485 } 1486 1487 return 0; 1488 } 1489 #endif 1490 1491 struct invalidation_fence { 1492 struct xe_gt_tlb_invalidation_fence base; 1493 struct xe_gt *gt; 1494 struct dma_fence *fence; 1495 struct dma_fence_cb cb; 1496 struct work_struct work; 1497 u64 start; 1498 u64 end; 1499 u32 asid; 1500 }; 1501 1502 static void invalidation_fence_cb(struct dma_fence *fence, 1503 struct dma_fence_cb *cb) 1504 { 1505 struct invalidation_fence *ifence = 1506 container_of(cb, struct invalidation_fence, cb); 1507 struct xe_device *xe = gt_to_xe(ifence->gt); 1508 1509 trace_xe_gt_tlb_invalidation_fence_cb(xe, &ifence->base); 1510 if (!ifence->fence->error) { 1511 queue_work(system_wq, &ifence->work); 1512 } else { 1513 ifence->base.base.error = ifence->fence->error; 1514 xe_gt_tlb_invalidation_fence_signal(&ifence->base); 1515 } 1516 dma_fence_put(ifence->fence); 1517 } 1518 1519 static void invalidation_fence_work_func(struct work_struct *w) 1520 { 1521 struct invalidation_fence *ifence = 1522 container_of(w, struct invalidation_fence, work); 1523 struct xe_device *xe = gt_to_xe(ifence->gt); 1524 1525 trace_xe_gt_tlb_invalidation_fence_work_func(xe, &ifence->base); 1526 xe_gt_tlb_invalidation_range(ifence->gt, &ifence->base, ifence->start, 1527 ifence->end, ifence->asid); 1528 } 1529 1530 static void invalidation_fence_init(struct xe_gt *gt, 1531 struct invalidation_fence *ifence, 1532 struct dma_fence *fence, 1533 u64 start, u64 end, u32 asid) 1534 { 1535 int ret; 1536 1537 trace_xe_gt_tlb_invalidation_fence_create(gt_to_xe(gt), &ifence->base); 1538 1539 xe_gt_tlb_invalidation_fence_init(gt, &ifence->base, false); 1540 1541 ifence->fence = fence; 1542 ifence->gt = gt; 1543 ifence->start = start; 1544 ifence->end = end; 1545 ifence->asid = asid; 1546 1547 INIT_WORK(&ifence->work, invalidation_fence_work_func); 1548 ret = dma_fence_add_callback(fence, &ifence->cb, invalidation_fence_cb); 1549 if (ret == -ENOENT) { 1550 dma_fence_put(ifence->fence); /* Usually dropped in CB */ 1551 invalidation_fence_work_func(&ifence->work); 1552 } else if (ret) { 1553 dma_fence_put(&ifence->base.base); /* Caller ref */ 1554 dma_fence_put(&ifence->base.base); /* Creation ref */ 1555 } 1556 1557 xe_gt_assert(gt, !ret || ret == -ENOENT); 1558 } 1559 1560 struct xe_pt_stage_unbind_walk { 1561 /** @base: The pagewalk base-class. */ 1562 struct xe_pt_walk base; 1563 1564 /* Input parameters for the walk */ 1565 /** @tile: The tile we're unbinding from. */ 1566 struct xe_tile *tile; 1567 1568 /** 1569 * @modified_start: Walk range start, modified to include any 1570 * shared pagetables that we're the only user of and can thus 1571 * treat as private. 1572 */ 1573 u64 modified_start; 1574 /** @modified_end: Walk range start, modified like @modified_start. */ 1575 u64 modified_end; 1576 1577 /* Output */ 1578 /* @wupd: Structure to track the page-table updates we're building */ 1579 struct xe_walk_update wupd; 1580 }; 1581 1582 /* 1583 * Check whether this range is the only one populating this pagetable, 1584 * and in that case, update the walk range checks so that higher levels don't 1585 * view us as a shared pagetable. 1586 */ 1587 static bool xe_pt_check_kill(u64 addr, u64 next, unsigned int level, 1588 const struct xe_pt *child, 1589 enum page_walk_action *action, 1590 struct xe_pt_walk *walk) 1591 { 1592 struct xe_pt_stage_unbind_walk *xe_walk = 1593 container_of(walk, typeof(*xe_walk), base); 1594 unsigned int shift = walk->shifts[level]; 1595 u64 size = 1ull << shift; 1596 1597 if (IS_ALIGNED(addr, size) && IS_ALIGNED(next, size) && 1598 ((next - addr) >> shift) == child->num_live) { 1599 u64 size = 1ull << walk->shifts[level + 1]; 1600 1601 *action = ACTION_CONTINUE; 1602 1603 if (xe_walk->modified_start >= addr) 1604 xe_walk->modified_start = round_down(addr, size); 1605 if (xe_walk->modified_end <= next) 1606 xe_walk->modified_end = round_up(next, size); 1607 1608 return true; 1609 } 1610 1611 return false; 1612 } 1613 1614 static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset, 1615 unsigned int level, u64 addr, u64 next, 1616 struct xe_ptw **child, 1617 enum page_walk_action *action, 1618 struct xe_pt_walk *walk) 1619 { 1620 struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base); 1621 1622 XE_WARN_ON(!*child); 1623 XE_WARN_ON(!level); 1624 1625 xe_pt_check_kill(addr, next, level - 1, xe_child, action, walk); 1626 1627 return 0; 1628 } 1629 1630 static int 1631 xe_pt_stage_unbind_post_descend(struct xe_ptw *parent, pgoff_t offset, 1632 unsigned int level, u64 addr, u64 next, 1633 struct xe_ptw **child, 1634 enum page_walk_action *action, 1635 struct xe_pt_walk *walk) 1636 { 1637 struct xe_pt_stage_unbind_walk *xe_walk = 1638 container_of(walk, typeof(*xe_walk), base); 1639 struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base); 1640 pgoff_t end_offset; 1641 u64 size = 1ull << walk->shifts[--level]; 1642 int err; 1643 1644 if (!IS_ALIGNED(addr, size)) 1645 addr = xe_walk->modified_start; 1646 if (!IS_ALIGNED(next, size)) 1647 next = xe_walk->modified_end; 1648 1649 /* Parent == *child is the root pt. Don't kill it. */ 1650 if (parent != *child && 1651 xe_pt_check_kill(addr, next, level, xe_child, action, walk)) 1652 return 0; 1653 1654 if (!xe_pt_nonshared_offsets(addr, next, level, walk, action, &offset, 1655 &end_offset)) 1656 return 0; 1657 1658 err = xe_pt_new_shared(&xe_walk->wupd, xe_child, offset, true); 1659 if (err) 1660 return err; 1661 1662 xe_walk->wupd.updates[level].update->qwords = end_offset - offset; 1663 1664 return 0; 1665 } 1666 1667 static const struct xe_pt_walk_ops xe_pt_stage_unbind_ops = { 1668 .pt_entry = xe_pt_stage_unbind_entry, 1669 .pt_post_descend = xe_pt_stage_unbind_post_descend, 1670 }; 1671 1672 /** 1673 * xe_pt_stage_unbind() - Build page-table update structures for an unbind 1674 * operation 1675 * @tile: The tile we're unbinding for. 1676 * @vm: The vm 1677 * @vma: The vma we're unbinding. 1678 * @range: The range we're unbinding. 1679 * @entries: Caller-provided storage for the update structures. 1680 * 1681 * Builds page-table update structures for an unbind operation. The function 1682 * will attempt to remove all page-tables that we're the only user 1683 * of, and for that to work, the unbind operation must be committed in the 1684 * same critical section that blocks racing binds to the same page-table tree. 1685 * 1686 * Return: The number of entries used. 1687 */ 1688 static unsigned int xe_pt_stage_unbind(struct xe_tile *tile, 1689 struct xe_vm *vm, 1690 struct xe_vma *vma, 1691 struct xe_svm_range *range, 1692 struct xe_vm_pgtable_update *entries) 1693 { 1694 u64 start = range ? range->base.itree.start : xe_vma_start(vma); 1695 u64 end = range ? range->base.itree.last + 1 : xe_vma_end(vma); 1696 struct xe_pt_stage_unbind_walk xe_walk = { 1697 .base = { 1698 .ops = &xe_pt_stage_unbind_ops, 1699 .shifts = xe_normal_pt_shifts, 1700 .max_level = XE_PT_HIGHEST_LEVEL, 1701 .staging = true, 1702 }, 1703 .tile = tile, 1704 .modified_start = start, 1705 .modified_end = end, 1706 .wupd.entries = entries, 1707 }; 1708 struct xe_pt *pt = vm->pt_root[tile->id]; 1709 1710 (void)xe_pt_walk_shared(&pt->base, pt->level, start, end, 1711 &xe_walk.base); 1712 1713 return xe_walk.wupd.num_used_entries; 1714 } 1715 1716 static void 1717 xe_migrate_clear_pgtable_callback(struct xe_migrate_pt_update *pt_update, 1718 struct xe_tile *tile, struct iosys_map *map, 1719 void *ptr, u32 qword_ofs, u32 num_qwords, 1720 const struct xe_vm_pgtable_update *update) 1721 { 1722 struct xe_vm *vm = pt_update->vops->vm; 1723 u64 empty = __xe_pt_empty_pte(tile, vm, update->pt->level); 1724 int i; 1725 1726 if (map && map->is_iomem) 1727 for (i = 0; i < num_qwords; ++i) 1728 xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) * 1729 sizeof(u64), u64, empty); 1730 else if (map) 1731 memset64(map->vaddr + qword_ofs * sizeof(u64), empty, 1732 num_qwords); 1733 else 1734 memset64(ptr, empty, num_qwords); 1735 } 1736 1737 static void xe_pt_abort_unbind(struct xe_vma *vma, 1738 struct xe_vm_pgtable_update *entries, 1739 u32 num_entries) 1740 { 1741 int i, j; 1742 1743 xe_pt_commit_prepare_locks_assert(vma); 1744 1745 for (i = num_entries - 1; i >= 0; --i) { 1746 struct xe_vm_pgtable_update *entry = &entries[i]; 1747 struct xe_pt *pt = entry->pt; 1748 struct xe_pt_dir *pt_dir = as_xe_pt_dir(pt); 1749 1750 pt->num_live += entry->qwords; 1751 1752 if (!pt->level) 1753 continue; 1754 1755 for (j = entry->ofs; j < entry->ofs + entry->qwords; j++) 1756 pt_dir->staging[j] = 1757 entries[i].pt_entries[j - entry->ofs].pt ? 1758 &entries[i].pt_entries[j - entry->ofs].pt->base : NULL; 1759 } 1760 } 1761 1762 static void 1763 xe_pt_commit_prepare_unbind(struct xe_vma *vma, 1764 struct xe_vm_pgtable_update *entries, 1765 u32 num_entries) 1766 { 1767 int i, j; 1768 1769 xe_pt_commit_prepare_locks_assert(vma); 1770 1771 for (i = 0; i < num_entries; ++i) { 1772 struct xe_vm_pgtable_update *entry = &entries[i]; 1773 struct xe_pt *pt = entry->pt; 1774 struct xe_pt_dir *pt_dir; 1775 1776 pt->num_live -= entry->qwords; 1777 if (!pt->level) 1778 continue; 1779 1780 pt_dir = as_xe_pt_dir(pt); 1781 for (j = entry->ofs; j < entry->ofs + entry->qwords; j++) { 1782 entry->pt_entries[j - entry->ofs].pt = 1783 xe_pt_entry_staging(pt_dir, j); 1784 pt_dir->staging[j] = NULL; 1785 } 1786 } 1787 } 1788 1789 static void 1790 xe_pt_update_ops_rfence_interval(struct xe_vm_pgtable_update_ops *pt_update_ops, 1791 u64 start, u64 end) 1792 { 1793 u64 last; 1794 u32 current_op = pt_update_ops->current_op; 1795 struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op]; 1796 int i, level = 0; 1797 1798 for (i = 0; i < pt_op->num_entries; i++) { 1799 const struct xe_vm_pgtable_update *entry = &pt_op->entries[i]; 1800 1801 if (entry->pt->level > level) 1802 level = entry->pt->level; 1803 } 1804 1805 /* Greedy (non-optimal) calculation but simple */ 1806 start = ALIGN_DOWN(start, 0x1ull << xe_pt_shift(level)); 1807 last = ALIGN(end, 0x1ull << xe_pt_shift(level)) - 1; 1808 1809 if (start < pt_update_ops->start) 1810 pt_update_ops->start = start; 1811 if (last > pt_update_ops->last) 1812 pt_update_ops->last = last; 1813 } 1814 1815 static int vma_reserve_fences(struct xe_device *xe, struct xe_vma *vma) 1816 { 1817 int shift = xe_device_get_root_tile(xe)->media_gt ? 1 : 0; 1818 1819 if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) 1820 return dma_resv_reserve_fences(xe_vma_bo(vma)->ttm.base.resv, 1821 xe->info.tile_count << shift); 1822 1823 return 0; 1824 } 1825 1826 static int bind_op_prepare(struct xe_vm *vm, struct xe_tile *tile, 1827 struct xe_vm_pgtable_update_ops *pt_update_ops, 1828 struct xe_vma *vma, bool invalidate_on_bind) 1829 { 1830 u32 current_op = pt_update_ops->current_op; 1831 struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op]; 1832 int err; 1833 1834 xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma)); 1835 xe_bo_assert_held(xe_vma_bo(vma)); 1836 1837 vm_dbg(&xe_vma_vm(vma)->xe->drm, 1838 "Preparing bind, with range [%llx...%llx)\n", 1839 xe_vma_start(vma), xe_vma_end(vma) - 1); 1840 1841 pt_op->vma = NULL; 1842 pt_op->bind = true; 1843 pt_op->rebind = BIT(tile->id) & vma->tile_present; 1844 1845 err = vma_reserve_fences(tile_to_xe(tile), vma); 1846 if (err) 1847 return err; 1848 1849 err = xe_pt_prepare_bind(tile, vma, NULL, pt_op->entries, 1850 &pt_op->num_entries, invalidate_on_bind); 1851 if (!err) { 1852 xe_tile_assert(tile, pt_op->num_entries <= 1853 ARRAY_SIZE(pt_op->entries)); 1854 xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries, 1855 pt_op->num_entries, true); 1856 1857 xe_pt_update_ops_rfence_interval(pt_update_ops, 1858 xe_vma_start(vma), 1859 xe_vma_end(vma)); 1860 ++pt_update_ops->current_op; 1861 pt_update_ops->needs_userptr_lock |= xe_vma_is_userptr(vma); 1862 1863 /* 1864 * If rebind, we have to invalidate TLB on !LR vms to invalidate 1865 * cached PTEs point to freed memory. On LR vms this is done 1866 * automatically when the context is re-enabled by the rebind worker, 1867 * or in fault mode it was invalidated on PTE zapping. 1868 * 1869 * If !rebind, and scratch enabled VMs, there is a chance the scratch 1870 * PTE is already cached in the TLB so it needs to be invalidated. 1871 * On !LR VMs this is done in the ring ops preceding a batch, but on 1872 * LR, in particular on user-space batch buffer chaining, it needs to 1873 * be done here. 1874 */ 1875 if ((!pt_op->rebind && xe_vm_has_scratch(vm) && 1876 xe_vm_in_lr_mode(vm))) 1877 pt_update_ops->needs_invalidation = true; 1878 else if (pt_op->rebind && !xe_vm_in_lr_mode(vm)) 1879 /* We bump also if batch_invalidate_tlb is true */ 1880 vm->tlb_flush_seqno++; 1881 1882 vma->tile_staged |= BIT(tile->id); 1883 pt_op->vma = vma; 1884 xe_pt_commit_prepare_bind(vma, pt_op->entries, 1885 pt_op->num_entries, pt_op->rebind); 1886 } else { 1887 xe_pt_cancel_bind(vma, pt_op->entries, pt_op->num_entries); 1888 } 1889 1890 return err; 1891 } 1892 1893 static int bind_range_prepare(struct xe_vm *vm, struct xe_tile *tile, 1894 struct xe_vm_pgtable_update_ops *pt_update_ops, 1895 struct xe_vma *vma, struct xe_svm_range *range) 1896 { 1897 u32 current_op = pt_update_ops->current_op; 1898 struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op]; 1899 int err; 1900 1901 xe_tile_assert(tile, xe_vma_is_cpu_addr_mirror(vma)); 1902 1903 vm_dbg(&xe_vma_vm(vma)->xe->drm, 1904 "Preparing bind, with range [%lx...%lx)\n", 1905 range->base.itree.start, range->base.itree.last); 1906 1907 pt_op->vma = NULL; 1908 pt_op->bind = true; 1909 pt_op->rebind = BIT(tile->id) & range->tile_present; 1910 1911 err = xe_pt_prepare_bind(tile, vma, range, pt_op->entries, 1912 &pt_op->num_entries, false); 1913 if (!err) { 1914 xe_tile_assert(tile, pt_op->num_entries <= 1915 ARRAY_SIZE(pt_op->entries)); 1916 xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries, 1917 pt_op->num_entries, true); 1918 1919 xe_pt_update_ops_rfence_interval(pt_update_ops, 1920 range->base.itree.start, 1921 range->base.itree.last + 1); 1922 ++pt_update_ops->current_op; 1923 pt_update_ops->needs_svm_lock = true; 1924 1925 pt_op->vma = vma; 1926 xe_pt_commit_prepare_bind(vma, pt_op->entries, 1927 pt_op->num_entries, pt_op->rebind); 1928 } else { 1929 xe_pt_cancel_bind(vma, pt_op->entries, pt_op->num_entries); 1930 } 1931 1932 return err; 1933 } 1934 1935 static int unbind_op_prepare(struct xe_tile *tile, 1936 struct xe_vm_pgtable_update_ops *pt_update_ops, 1937 struct xe_vma *vma) 1938 { 1939 u32 current_op = pt_update_ops->current_op; 1940 struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op]; 1941 int err; 1942 1943 if (!((vma->tile_present | vma->tile_staged) & BIT(tile->id))) 1944 return 0; 1945 1946 xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma)); 1947 xe_bo_assert_held(xe_vma_bo(vma)); 1948 1949 vm_dbg(&xe_vma_vm(vma)->xe->drm, 1950 "Preparing unbind, with range [%llx...%llx)\n", 1951 xe_vma_start(vma), xe_vma_end(vma) - 1); 1952 1953 pt_op->vma = vma; 1954 pt_op->bind = false; 1955 pt_op->rebind = false; 1956 1957 err = vma_reserve_fences(tile_to_xe(tile), vma); 1958 if (err) 1959 return err; 1960 1961 pt_op->num_entries = xe_pt_stage_unbind(tile, xe_vma_vm(vma), 1962 vma, NULL, pt_op->entries); 1963 1964 xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries, 1965 pt_op->num_entries, false); 1966 xe_pt_update_ops_rfence_interval(pt_update_ops, xe_vma_start(vma), 1967 xe_vma_end(vma)); 1968 ++pt_update_ops->current_op; 1969 pt_update_ops->needs_userptr_lock |= xe_vma_is_userptr(vma); 1970 pt_update_ops->needs_invalidation = true; 1971 1972 xe_pt_commit_prepare_unbind(vma, pt_op->entries, pt_op->num_entries); 1973 1974 return 0; 1975 } 1976 1977 static int unbind_range_prepare(struct xe_vm *vm, 1978 struct xe_tile *tile, 1979 struct xe_vm_pgtable_update_ops *pt_update_ops, 1980 struct xe_svm_range *range) 1981 { 1982 u32 current_op = pt_update_ops->current_op; 1983 struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op]; 1984 1985 if (!(range->tile_present & BIT(tile->id))) 1986 return 0; 1987 1988 vm_dbg(&vm->xe->drm, 1989 "Preparing unbind, with range [%lx...%lx)\n", 1990 range->base.itree.start, range->base.itree.last); 1991 1992 pt_op->vma = XE_INVALID_VMA; 1993 pt_op->bind = false; 1994 pt_op->rebind = false; 1995 1996 pt_op->num_entries = xe_pt_stage_unbind(tile, vm, NULL, range, 1997 pt_op->entries); 1998 1999 xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries, 2000 pt_op->num_entries, false); 2001 xe_pt_update_ops_rfence_interval(pt_update_ops, range->base.itree.start, 2002 range->base.itree.last + 1); 2003 ++pt_update_ops->current_op; 2004 pt_update_ops->needs_svm_lock = true; 2005 pt_update_ops->needs_invalidation = true; 2006 2007 xe_pt_commit_prepare_unbind(XE_INVALID_VMA, pt_op->entries, 2008 pt_op->num_entries); 2009 2010 return 0; 2011 } 2012 2013 static int op_prepare(struct xe_vm *vm, 2014 struct xe_tile *tile, 2015 struct xe_vm_pgtable_update_ops *pt_update_ops, 2016 struct xe_vma_op *op) 2017 { 2018 int err = 0; 2019 2020 xe_vm_assert_held(vm); 2021 2022 switch (op->base.op) { 2023 case DRM_GPUVA_OP_MAP: 2024 if ((!op->map.immediate && xe_vm_in_fault_mode(vm) && 2025 !op->map.invalidate_on_bind) || 2026 op->map.is_cpu_addr_mirror) 2027 break; 2028 2029 err = bind_op_prepare(vm, tile, pt_update_ops, op->map.vma, 2030 op->map.invalidate_on_bind); 2031 pt_update_ops->wait_vm_kernel = true; 2032 break; 2033 case DRM_GPUVA_OP_REMAP: 2034 { 2035 struct xe_vma *old = gpuva_to_vma(op->base.remap.unmap->va); 2036 2037 if (xe_vma_is_cpu_addr_mirror(old)) 2038 break; 2039 2040 err = unbind_op_prepare(tile, pt_update_ops, old); 2041 2042 if (!err && op->remap.prev) { 2043 err = bind_op_prepare(vm, tile, pt_update_ops, 2044 op->remap.prev, false); 2045 pt_update_ops->wait_vm_bookkeep = true; 2046 } 2047 if (!err && op->remap.next) { 2048 err = bind_op_prepare(vm, tile, pt_update_ops, 2049 op->remap.next, false); 2050 pt_update_ops->wait_vm_bookkeep = true; 2051 } 2052 break; 2053 } 2054 case DRM_GPUVA_OP_UNMAP: 2055 { 2056 struct xe_vma *vma = gpuva_to_vma(op->base.unmap.va); 2057 2058 if (xe_vma_is_cpu_addr_mirror(vma)) 2059 break; 2060 2061 err = unbind_op_prepare(tile, pt_update_ops, vma); 2062 break; 2063 } 2064 case DRM_GPUVA_OP_PREFETCH: 2065 { 2066 struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va); 2067 2068 if (xe_vma_is_cpu_addr_mirror(vma)) 2069 break; 2070 2071 err = bind_op_prepare(vm, tile, pt_update_ops, vma, false); 2072 pt_update_ops->wait_vm_kernel = true; 2073 break; 2074 } 2075 case DRM_GPUVA_OP_DRIVER: 2076 if (op->subop == XE_VMA_SUBOP_MAP_RANGE) { 2077 xe_assert(vm->xe, xe_vma_is_cpu_addr_mirror(op->map_range.vma)); 2078 2079 err = bind_range_prepare(vm, tile, pt_update_ops, 2080 op->map_range.vma, 2081 op->map_range.range); 2082 } else if (op->subop == XE_VMA_SUBOP_UNMAP_RANGE) { 2083 err = unbind_range_prepare(vm, tile, pt_update_ops, 2084 op->unmap_range.range); 2085 } 2086 break; 2087 default: 2088 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); 2089 } 2090 2091 return err; 2092 } 2093 2094 static void 2095 xe_pt_update_ops_init(struct xe_vm_pgtable_update_ops *pt_update_ops) 2096 { 2097 init_llist_head(&pt_update_ops->deferred); 2098 pt_update_ops->start = ~0x0ull; 2099 pt_update_ops->last = 0x0ull; 2100 } 2101 2102 /** 2103 * xe_pt_update_ops_prepare() - Prepare PT update operations 2104 * @tile: Tile of PT update operations 2105 * @vops: VMA operationa 2106 * 2107 * Prepare PT update operations which includes updating internal PT state, 2108 * allocate memory for page tables, populate page table being pruned in, and 2109 * create PT update operations for leaf insertion / removal. 2110 * 2111 * Return: 0 on success, negative error code on error. 2112 */ 2113 int xe_pt_update_ops_prepare(struct xe_tile *tile, struct xe_vma_ops *vops) 2114 { 2115 struct xe_vm_pgtable_update_ops *pt_update_ops = 2116 &vops->pt_update_ops[tile->id]; 2117 struct xe_vma_op *op; 2118 int shift = tile->media_gt ? 1 : 0; 2119 int err; 2120 2121 lockdep_assert_held(&vops->vm->lock); 2122 xe_vm_assert_held(vops->vm); 2123 2124 xe_pt_update_ops_init(pt_update_ops); 2125 2126 err = dma_resv_reserve_fences(xe_vm_resv(vops->vm), 2127 tile_to_xe(tile)->info.tile_count << shift); 2128 if (err) 2129 return err; 2130 2131 list_for_each_entry(op, &vops->list, link) { 2132 err = op_prepare(vops->vm, tile, pt_update_ops, op); 2133 2134 if (err) 2135 return err; 2136 } 2137 2138 xe_tile_assert(tile, pt_update_ops->current_op <= 2139 pt_update_ops->num_ops); 2140 2141 #ifdef TEST_VM_OPS_ERROR 2142 if (vops->inject_error && 2143 vops->vm->xe->vm_inject_error_position == FORCE_OP_ERROR_PREPARE) 2144 return -ENOSPC; 2145 #endif 2146 2147 return 0; 2148 } 2149 ALLOW_ERROR_INJECTION(xe_pt_update_ops_prepare, ERRNO); 2150 2151 static void bind_op_commit(struct xe_vm *vm, struct xe_tile *tile, 2152 struct xe_vm_pgtable_update_ops *pt_update_ops, 2153 struct xe_vma *vma, struct dma_fence *fence, 2154 struct dma_fence *fence2, bool invalidate_on_bind) 2155 { 2156 xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma)); 2157 2158 if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) { 2159 dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence, 2160 pt_update_ops->wait_vm_bookkeep ? 2161 DMA_RESV_USAGE_KERNEL : 2162 DMA_RESV_USAGE_BOOKKEEP); 2163 if (fence2) 2164 dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence2, 2165 pt_update_ops->wait_vm_bookkeep ? 2166 DMA_RESV_USAGE_KERNEL : 2167 DMA_RESV_USAGE_BOOKKEEP); 2168 } 2169 vma->tile_present |= BIT(tile->id); 2170 vma->tile_staged &= ~BIT(tile->id); 2171 if (invalidate_on_bind) 2172 vma->tile_invalidated |= BIT(tile->id); 2173 if (xe_vma_is_userptr(vma)) { 2174 lockdep_assert_held_read(&vm->userptr.notifier_lock); 2175 to_userptr_vma(vma)->userptr.initial_bind = true; 2176 } 2177 2178 /* 2179 * Kick rebind worker if this bind triggers preempt fences and not in 2180 * the rebind worker 2181 */ 2182 if (pt_update_ops->wait_vm_bookkeep && 2183 xe_vm_in_preempt_fence_mode(vm) && 2184 !current->mm) 2185 xe_vm_queue_rebind_worker(vm); 2186 } 2187 2188 static void unbind_op_commit(struct xe_vm *vm, struct xe_tile *tile, 2189 struct xe_vm_pgtable_update_ops *pt_update_ops, 2190 struct xe_vma *vma, struct dma_fence *fence, 2191 struct dma_fence *fence2) 2192 { 2193 xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma)); 2194 2195 if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) { 2196 dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence, 2197 pt_update_ops->wait_vm_bookkeep ? 2198 DMA_RESV_USAGE_KERNEL : 2199 DMA_RESV_USAGE_BOOKKEEP); 2200 if (fence2) 2201 dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence2, 2202 pt_update_ops->wait_vm_bookkeep ? 2203 DMA_RESV_USAGE_KERNEL : 2204 DMA_RESV_USAGE_BOOKKEEP); 2205 } 2206 vma->tile_present &= ~BIT(tile->id); 2207 if (!vma->tile_present) { 2208 list_del_init(&vma->combined_links.rebind); 2209 if (xe_vma_is_userptr(vma)) { 2210 lockdep_assert_held_read(&vm->userptr.notifier_lock); 2211 2212 spin_lock(&vm->userptr.invalidated_lock); 2213 list_del_init(&to_userptr_vma(vma)->userptr.invalidate_link); 2214 spin_unlock(&vm->userptr.invalidated_lock); 2215 } 2216 } 2217 } 2218 2219 static void op_commit(struct xe_vm *vm, 2220 struct xe_tile *tile, 2221 struct xe_vm_pgtable_update_ops *pt_update_ops, 2222 struct xe_vma_op *op, struct dma_fence *fence, 2223 struct dma_fence *fence2) 2224 { 2225 xe_vm_assert_held(vm); 2226 2227 switch (op->base.op) { 2228 case DRM_GPUVA_OP_MAP: 2229 if ((!op->map.immediate && xe_vm_in_fault_mode(vm)) || 2230 op->map.is_cpu_addr_mirror) 2231 break; 2232 2233 bind_op_commit(vm, tile, pt_update_ops, op->map.vma, fence, 2234 fence2, op->map.invalidate_on_bind); 2235 break; 2236 case DRM_GPUVA_OP_REMAP: 2237 { 2238 struct xe_vma *old = gpuva_to_vma(op->base.remap.unmap->va); 2239 2240 if (xe_vma_is_cpu_addr_mirror(old)) 2241 break; 2242 2243 unbind_op_commit(vm, tile, pt_update_ops, old, fence, fence2); 2244 2245 if (op->remap.prev) 2246 bind_op_commit(vm, tile, pt_update_ops, op->remap.prev, 2247 fence, fence2, false); 2248 if (op->remap.next) 2249 bind_op_commit(vm, tile, pt_update_ops, op->remap.next, 2250 fence, fence2, false); 2251 break; 2252 } 2253 case DRM_GPUVA_OP_UNMAP: 2254 { 2255 struct xe_vma *vma = gpuva_to_vma(op->base.unmap.va); 2256 2257 if (!xe_vma_is_cpu_addr_mirror(vma)) 2258 unbind_op_commit(vm, tile, pt_update_ops, vma, fence, 2259 fence2); 2260 break; 2261 } 2262 case DRM_GPUVA_OP_PREFETCH: 2263 { 2264 struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va); 2265 2266 if (!xe_vma_is_cpu_addr_mirror(vma)) 2267 bind_op_commit(vm, tile, pt_update_ops, vma, fence, 2268 fence2, false); 2269 break; 2270 } 2271 case DRM_GPUVA_OP_DRIVER: 2272 { 2273 /* WRITE_ONCE pairs with READ_ONCE in xe_svm.c */ 2274 2275 if (op->subop == XE_VMA_SUBOP_MAP_RANGE) { 2276 WRITE_ONCE(op->map_range.range->tile_present, 2277 op->map_range.range->tile_present | 2278 BIT(tile->id)); 2279 WRITE_ONCE(op->map_range.range->tile_invalidated, 2280 op->map_range.range->tile_invalidated & 2281 ~BIT(tile->id)); 2282 } else if (op->subop == XE_VMA_SUBOP_UNMAP_RANGE) { 2283 WRITE_ONCE(op->unmap_range.range->tile_present, 2284 op->unmap_range.range->tile_present & 2285 ~BIT(tile->id)); 2286 } 2287 break; 2288 } 2289 default: 2290 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); 2291 } 2292 } 2293 2294 static const struct xe_migrate_pt_update_ops migrate_ops = { 2295 .populate = xe_vm_populate_pgtable, 2296 .clear = xe_migrate_clear_pgtable_callback, 2297 .pre_commit = xe_pt_pre_commit, 2298 }; 2299 2300 static const struct xe_migrate_pt_update_ops userptr_migrate_ops = { 2301 .populate = xe_vm_populate_pgtable, 2302 .clear = xe_migrate_clear_pgtable_callback, 2303 .pre_commit = xe_pt_userptr_pre_commit, 2304 }; 2305 2306 #if IS_ENABLED(CONFIG_DRM_XE_GPUSVM) 2307 static const struct xe_migrate_pt_update_ops svm_migrate_ops = { 2308 .populate = xe_vm_populate_pgtable, 2309 .clear = xe_migrate_clear_pgtable_callback, 2310 .pre_commit = xe_pt_svm_pre_commit, 2311 }; 2312 #else 2313 static const struct xe_migrate_pt_update_ops svm_migrate_ops; 2314 #endif 2315 2316 /** 2317 * xe_pt_update_ops_run() - Run PT update operations 2318 * @tile: Tile of PT update operations 2319 * @vops: VMA operationa 2320 * 2321 * Run PT update operations which includes committing internal PT state changes, 2322 * creating job for PT update operations for leaf insertion / removal, and 2323 * installing job fence in various places. 2324 * 2325 * Return: fence on success, negative ERR_PTR on error. 2326 */ 2327 struct dma_fence * 2328 xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops) 2329 { 2330 struct xe_vm *vm = vops->vm; 2331 struct xe_vm_pgtable_update_ops *pt_update_ops = 2332 &vops->pt_update_ops[tile->id]; 2333 struct dma_fence *fence; 2334 struct invalidation_fence *ifence = NULL, *mfence = NULL; 2335 struct dma_fence **fences = NULL; 2336 struct dma_fence_array *cf = NULL; 2337 struct xe_range_fence *rfence; 2338 struct xe_vma_op *op; 2339 int err = 0, i; 2340 struct xe_migrate_pt_update update = { 2341 .ops = pt_update_ops->needs_svm_lock ? 2342 &svm_migrate_ops : 2343 pt_update_ops->needs_userptr_lock ? 2344 &userptr_migrate_ops : 2345 &migrate_ops, 2346 .vops = vops, 2347 .tile_id = tile->id, 2348 }; 2349 2350 lockdep_assert_held(&vm->lock); 2351 xe_vm_assert_held(vm); 2352 2353 if (!pt_update_ops->current_op) { 2354 xe_tile_assert(tile, xe_vm_in_fault_mode(vm)); 2355 2356 return dma_fence_get_stub(); 2357 } 2358 2359 #ifdef TEST_VM_OPS_ERROR 2360 if (vops->inject_error && 2361 vm->xe->vm_inject_error_position == FORCE_OP_ERROR_RUN) 2362 return ERR_PTR(-ENOSPC); 2363 #endif 2364 2365 if (pt_update_ops->needs_invalidation) { 2366 ifence = kzalloc(sizeof(*ifence), GFP_KERNEL); 2367 if (!ifence) { 2368 err = -ENOMEM; 2369 goto kill_vm_tile1; 2370 } 2371 if (tile->media_gt) { 2372 mfence = kzalloc(sizeof(*ifence), GFP_KERNEL); 2373 if (!mfence) { 2374 err = -ENOMEM; 2375 goto free_ifence; 2376 } 2377 fences = kmalloc_array(2, sizeof(*fences), GFP_KERNEL); 2378 if (!fences) { 2379 err = -ENOMEM; 2380 goto free_ifence; 2381 } 2382 cf = dma_fence_array_alloc(2); 2383 if (!cf) { 2384 err = -ENOMEM; 2385 goto free_ifence; 2386 } 2387 } 2388 } 2389 2390 rfence = kzalloc(sizeof(*rfence), GFP_KERNEL); 2391 if (!rfence) { 2392 err = -ENOMEM; 2393 goto free_ifence; 2394 } 2395 2396 fence = xe_migrate_update_pgtables(tile->migrate, &update); 2397 if (IS_ERR(fence)) { 2398 err = PTR_ERR(fence); 2399 goto free_rfence; 2400 } 2401 2402 /* Point of no return - VM killed if failure after this */ 2403 for (i = 0; i < pt_update_ops->current_op; ++i) { 2404 struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i]; 2405 2406 xe_pt_commit(pt_op->vma, pt_op->entries, 2407 pt_op->num_entries, &pt_update_ops->deferred); 2408 pt_op->vma = NULL; /* skip in xe_pt_update_ops_abort */ 2409 } 2410 2411 if (xe_range_fence_insert(&vm->rftree[tile->id], rfence, 2412 &xe_range_fence_kfree_ops, 2413 pt_update_ops->start, 2414 pt_update_ops->last, fence)) 2415 dma_fence_wait(fence, false); 2416 2417 /* tlb invalidation must be done before signaling rebind */ 2418 if (ifence) { 2419 if (mfence) 2420 dma_fence_get(fence); 2421 invalidation_fence_init(tile->primary_gt, ifence, fence, 2422 pt_update_ops->start, 2423 pt_update_ops->last, vm->usm.asid); 2424 if (mfence) { 2425 invalidation_fence_init(tile->media_gt, mfence, fence, 2426 pt_update_ops->start, 2427 pt_update_ops->last, vm->usm.asid); 2428 fences[0] = &ifence->base.base; 2429 fences[1] = &mfence->base.base; 2430 dma_fence_array_init(cf, 2, fences, 2431 vm->composite_fence_ctx, 2432 vm->composite_fence_seqno++, 2433 false); 2434 fence = &cf->base; 2435 } else { 2436 fence = &ifence->base.base; 2437 } 2438 } 2439 2440 if (!mfence) { 2441 dma_resv_add_fence(xe_vm_resv(vm), fence, 2442 pt_update_ops->wait_vm_bookkeep ? 2443 DMA_RESV_USAGE_KERNEL : 2444 DMA_RESV_USAGE_BOOKKEEP); 2445 2446 list_for_each_entry(op, &vops->list, link) 2447 op_commit(vops->vm, tile, pt_update_ops, op, fence, NULL); 2448 } else { 2449 dma_resv_add_fence(xe_vm_resv(vm), &ifence->base.base, 2450 pt_update_ops->wait_vm_bookkeep ? 2451 DMA_RESV_USAGE_KERNEL : 2452 DMA_RESV_USAGE_BOOKKEEP); 2453 2454 dma_resv_add_fence(xe_vm_resv(vm), &mfence->base.base, 2455 pt_update_ops->wait_vm_bookkeep ? 2456 DMA_RESV_USAGE_KERNEL : 2457 DMA_RESV_USAGE_BOOKKEEP); 2458 2459 list_for_each_entry(op, &vops->list, link) 2460 op_commit(vops->vm, tile, pt_update_ops, op, 2461 &ifence->base.base, &mfence->base.base); 2462 } 2463 2464 if (pt_update_ops->needs_svm_lock) 2465 xe_svm_notifier_unlock(vm); 2466 if (pt_update_ops->needs_userptr_lock) 2467 up_read(&vm->userptr.notifier_lock); 2468 2469 return fence; 2470 2471 free_rfence: 2472 kfree(rfence); 2473 free_ifence: 2474 kfree(cf); 2475 kfree(fences); 2476 kfree(mfence); 2477 kfree(ifence); 2478 kill_vm_tile1: 2479 if (err != -EAGAIN && tile->id) 2480 xe_vm_kill(vops->vm, false); 2481 2482 return ERR_PTR(err); 2483 } 2484 ALLOW_ERROR_INJECTION(xe_pt_update_ops_run, ERRNO); 2485 2486 /** 2487 * xe_pt_update_ops_fini() - Finish PT update operations 2488 * @tile: Tile of PT update operations 2489 * @vops: VMA operations 2490 * 2491 * Finish PT update operations by committing to destroy page table memory 2492 */ 2493 void xe_pt_update_ops_fini(struct xe_tile *tile, struct xe_vma_ops *vops) 2494 { 2495 struct xe_vm_pgtable_update_ops *pt_update_ops = 2496 &vops->pt_update_ops[tile->id]; 2497 int i; 2498 2499 lockdep_assert_held(&vops->vm->lock); 2500 xe_vm_assert_held(vops->vm); 2501 2502 for (i = 0; i < pt_update_ops->current_op; ++i) { 2503 struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i]; 2504 2505 xe_pt_free_bind(pt_op->entries, pt_op->num_entries); 2506 } 2507 xe_bo_put_commit(&vops->pt_update_ops[tile->id].deferred); 2508 } 2509 2510 /** 2511 * xe_pt_update_ops_abort() - Abort PT update operations 2512 * @tile: Tile of PT update operations 2513 * @vops: VMA operationa 2514 * 2515 * Abort PT update operations by unwinding internal PT state 2516 */ 2517 void xe_pt_update_ops_abort(struct xe_tile *tile, struct xe_vma_ops *vops) 2518 { 2519 struct xe_vm_pgtable_update_ops *pt_update_ops = 2520 &vops->pt_update_ops[tile->id]; 2521 int i; 2522 2523 lockdep_assert_held(&vops->vm->lock); 2524 xe_vm_assert_held(vops->vm); 2525 2526 for (i = pt_update_ops->num_ops - 1; i >= 0; --i) { 2527 struct xe_vm_pgtable_update_op *pt_op = 2528 &pt_update_ops->ops[i]; 2529 2530 if (!pt_op->vma || i >= pt_update_ops->current_op) 2531 continue; 2532 2533 if (pt_op->bind) 2534 xe_pt_abort_bind(pt_op->vma, pt_op->entries, 2535 pt_op->num_entries, 2536 pt_op->rebind); 2537 else 2538 xe_pt_abort_unbind(pt_op->vma, pt_op->entries, 2539 pt_op->num_entries); 2540 } 2541 2542 xe_pt_update_ops_fini(tile, vops); 2543 } 2544