Lines Matching +full:smem +full:- +full:part
1 // SPDX-License-Identifier: MIT
6 #include <linux/dma-fence-array.h>
29 /** @children: Array of page-table child nodes */
34 #define xe_pt_set_addr(__xe_pt, __addr) ((__xe_pt)->addr = (__addr))
35 #define xe_pt_addr(__xe_pt) ((__xe_pt)->addr)
44 #define XE_PT_HIGHEST_LEVEL (ARRAY_SIZE(xe_normal_pt_shifts) - 1)
53 return container_of(pt_dir->children[index], struct xe_pt, base); in xe_pt_entry()
60 u16 pat_index = xe->pat.idx[XE_CACHE_WB]; in __xe_pt_empty_pte()
61 u8 id = tile->id; in __xe_pt_empty_pte()
67 return vm->pt_ops->pde_encode_bo(vm->scratch_pt[id][level - 1]->bo, in __xe_pt_empty_pte()
70 return vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, IS_DGFX(xe), 0) | in __xe_pt_empty_pte()
76 if (pt->level) in xe_pt_free()
83 * xe_pt_create() - Create a page-table.
86 * @level: The page-table level.
89 * create the corresponding page-table bo, but don't initialize it. If the
90 * level is grater than zero, then it's assumed to be a directory page-
107 pt = (dir) ? &dir->pt : NULL; in xe_pt_create()
112 return ERR_PTR(-ENOMEM); in xe_pt_create()
114 pt->level = level; in xe_pt_create()
115 bo = xe_bo_create_pin_map(vm->xe, tile, vm, SZ_4K, in xe_pt_create()
126 pt->bo = bo; in xe_pt_create()
127 pt->base.children = level ? as_xe_pt_dir(pt)->children : NULL; in xe_pt_create()
129 if (vm->xef) in xe_pt_create()
130 xe_drm_client_add_bo(vm->xef->client, pt->bo); in xe_pt_create()
141 * xe_pt_populate_empty() - Populate a page-table bo with scratch- or zero
147 * Populate the page-table bo of @pt with entries pointing into the tile's
148 * scratch page-table tree if any. Otherwise populate with zeros.
153 struct iosys_map *map = &pt->bo->vmap; in xe_pt_populate_empty()
162 xe_map_memset(vm->xe, map, 0, 0, SZ_4K); in xe_pt_populate_empty()
164 empty = __xe_pt_empty_pte(tile, vm, pt->level); in xe_pt_populate_empty()
166 xe_pt_write(vm->xe, map, i, empty); in xe_pt_populate_empty()
171 * xe_pt_shift() - Return the ilog2 value of the size of the address range of
172 * a page-table at a certain level.
175 * Return: The ilog2 value of the size of the address range of a page-table
184 * xe_pt_destroy() - Destroy a page-table tree.
185 * @pt: The root of the page-table tree to destroy.
190 * Puts the page-table bo, recursively calls xe_pt_destroy on all children
200 XE_WARN_ON(!list_empty(&pt->bo->ttm.base.gpuva.list)); in xe_pt_destroy()
201 xe_bo_unpin(pt->bo); in xe_pt_destroy()
202 xe_bo_put_deferred(pt->bo, deferred); in xe_pt_destroy()
204 if (pt->level > 0 && pt->num_live) { in xe_pt_destroy()
219 * Below we use the term "page-table" for both page-directories, containing
220 * pointers to lower level page-directories or page-tables, and level 0
221 * page-tables that contain only page-table-entries pointing to memory pages.
223 * When inserting an address range in an already existing page-table tree
224 * there will typically be a set of page-tables that are shared with other
226 * The set of shared page-tables can be at most two per level,
228 * page-tables may still be in use by the gpu for other mappings. Therefore
231 * data, (subtrees for the cpu and page-table-entries for the gpu) is then
232 * added in a separate commit step. CPU-data is committed while still under the
236 * For non-shared page-tables (and, in fact, for shared ones that aren't
237 * existing at the time of staging), we add the data in-place without the
238 * special update structures. This private part of the page-table tree will
239 * remain disconnected from the vm page-table tree until data is committed to
248 /** @preexisting: Whether the parent was pre-existing or allocated */
278 /** @va_curs_start: The Virtual address coresponding to @curs->start */
307 struct xe_pt_update *upd = &wupd->updates[parent->level]; in xe_pt_new_shared()
313 * new parent and page-directory, the old one is complete, and in xe_pt_new_shared()
315 * @wupd->entries in xe_pt_new_shared()
317 if (likely(upd->parent == parent)) in xe_pt_new_shared()
320 upd->parent = parent; in xe_pt_new_shared()
321 upd->preexisting = true; in xe_pt_new_shared()
323 if (wupd->num_used_entries == XE_VM_MAX_LEVEL * 2 + 1) in xe_pt_new_shared()
324 return -EINVAL; in xe_pt_new_shared()
326 entry = wupd->entries + wupd->num_used_entries++; in xe_pt_new_shared()
327 upd->update = entry; in xe_pt_new_shared()
328 entry->ofs = offset; in xe_pt_new_shared()
329 entry->pt_bo = parent->bo; in xe_pt_new_shared()
330 entry->pt = parent; in xe_pt_new_shared()
331 entry->flags = 0; in xe_pt_new_shared()
332 entry->qwords = 0; in xe_pt_new_shared()
333 entry->pt_bo->update_index = -1; in xe_pt_new_shared()
336 entry->pt_entries = kmalloc_array(XE_PDES, in xe_pt_new_shared()
337 sizeof(*entry->pt_entries), in xe_pt_new_shared()
339 if (!entry->pt_entries) in xe_pt_new_shared()
340 return -ENOMEM; in xe_pt_new_shared()
349 * non-pre-existing pagetable with leaf ptes.
355 struct xe_pt_update *upd = &xe_walk->wupd.updates[parent->level]; in xe_pt_insert_entry()
357 &xe_walk->wupd.updates[xe_child->level] : NULL; in xe_pt_insert_entry()
360 ret = xe_pt_new_shared(&xe_walk->wupd, parent, offset, true); in xe_pt_insert_entry()
369 child_upd->update = NULL; in xe_pt_insert_entry()
370 child_upd->parent = xe_child; in xe_pt_insert_entry()
371 child_upd->preexisting = false; in xe_pt_insert_entry()
374 if (likely(!upd->preexisting)) { in xe_pt_insert_entry()
375 /* Continue building a non-connected subtree. */ in xe_pt_insert_entry()
376 struct iosys_map *map = &parent->bo->vmap; in xe_pt_insert_entry()
379 parent->base.children[offset] = &xe_child->base; in xe_pt_insert_entry()
381 xe_pt_write(xe_walk->vm->xe, map, offset, pte); in xe_pt_insert_entry()
382 parent->num_live++; in xe_pt_insert_entry()
386 struct xe_vm_pgtable_update *entry = upd->update; in xe_pt_insert_entry()
388 idx = offset - entry->ofs; in xe_pt_insert_entry()
389 entry->pt_entries[idx].pt = xe_child; in xe_pt_insert_entry()
390 entry->pt_entries[idx].pte = pte; in xe_pt_insert_entry()
391 entry->qwords++; in xe_pt_insert_entry()
406 if (!xe_pt_covers(addr, next, level, &xe_walk->base)) in xe_pt_hugepte_possible()
410 if (next - xe_walk->va_curs_start > xe_walk->curs->size) in xe_pt_hugepte_possible()
414 if (xe_vma_is_null(xe_walk->vma)) in xe_pt_hugepte_possible()
418 size = next - addr; in xe_pt_hugepte_possible()
419 dma = addr - xe_walk->va_curs_start + xe_res_dma(xe_walk->curs); in xe_pt_hugepte_possible()
431 struct xe_res_cursor curs = *xe_walk->curs; in xe_pt_scan_64K()
436 if (next > xe_walk->l0_end_addr) in xe_pt_scan_64K()
440 if (xe_vma_is_null(xe_walk->vma)) in xe_pt_scan_64K()
443 xe_res_next(&curs, addr - xe_walk->va_curs_start); in xe_pt_scan_64K()
455 * For non-compact "normal" 4K level-0 pagetables, we want to try to group
456 * addresses together in 64K-contigous regions to add a 64K TLB hint for the
458 * This function determines whether the address is part of such a
466 if (xe_walk->found_64K && addr - xe_walk->addr_64K < SZ_64K) in xe_pt_is_pte_ps64K()
469 xe_walk->found_64K = xe_pt_scan_64K(addr, addr + SZ_64K, xe_walk); in xe_pt_is_pte_ps64K()
470 xe_walk->addr_64K = addr; in xe_pt_is_pte_ps64K()
472 return xe_walk->found_64K; in xe_pt_is_pte_ps64K()
484 u16 pat_index = xe_walk->vma->pat_index; in xe_pt_stage_bind_entry()
486 struct xe_vm *vm = xe_walk->vm; in xe_pt_stage_bind_entry()
494 struct xe_res_cursor *curs = xe_walk->curs; in xe_pt_stage_bind_entry()
495 bool is_null = xe_vma_is_null(xe_walk->vma); in xe_pt_stage_bind_entry()
497 XE_WARN_ON(xe_walk->va_curs_start != addr); in xe_pt_stage_bind_entry()
499 pte = vm->pt_ops->pte_encode_vma(is_null ? 0 : in xe_pt_stage_bind_entry()
500 xe_res_dma(curs) + xe_walk->dma_offset, in xe_pt_stage_bind_entry()
501 xe_walk->vma, pat_index, level); in xe_pt_stage_bind_entry()
502 pte |= xe_walk->default_pte; in xe_pt_stage_bind_entry()
508 if (level == 0 && !xe_parent->is_compact) { in xe_pt_stage_bind_entry()
510 xe_walk->vma->gpuva.flags |= XE_VMA_PTE_64K; in xe_pt_stage_bind_entry()
512 } else if (XE_WARN_ON(xe_walk->needs_64K)) { in xe_pt_stage_bind_entry()
513 return -EINVAL; in xe_pt_stage_bind_entry()
522 xe_res_next(curs, next - addr); in xe_pt_stage_bind_entry()
523 xe_walk->va_curs_start = next; in xe_pt_stage_bind_entry()
524 xe_walk->vma->gpuva.flags |= (XE_VMA_PTE_4K << level); in xe_pt_stage_bind_entry()
532 * new page table or -directory, which we do if there is no in xe_pt_stage_bind_entry()
536 walk->shifts = xe_normal_pt_shifts; in xe_pt_stage_bind_entry()
537 xe_walk->l0_end_addr = next; in xe_pt_stage_bind_entry()
540 covers = xe_pt_covers(addr, next, level, &xe_walk->base); in xe_pt_stage_bind_entry()
544 xe_child = xe_pt_create(xe_walk->vm, xe_walk->tile, level - 1); in xe_pt_stage_bind_entry()
549 round_down(addr, 1ull << walk->shifts[level])); in xe_pt_stage_bind_entry()
552 xe_pt_populate_empty(xe_walk->tile, xe_walk->vm, xe_child); in xe_pt_stage_bind_entry()
554 *child = &xe_child->base; in xe_pt_stage_bind_entry()
563 if (GRAPHICS_VERx100(tile_to_xe(xe_walk->tile)) >= 1250 && level == 1 && in xe_pt_stage_bind_entry()
565 walk->shifts = xe_compact_pt_shifts; in xe_pt_stage_bind_entry()
566 xe_walk->vma->gpuva.flags |= XE_VMA_PTE_COMPACT; in xe_pt_stage_bind_entry()
568 xe_child->is_compact = true; in xe_pt_stage_bind_entry()
571 pte = vm->pt_ops->pde_encode_bo(xe_child->bo, 0, pat_index) | flags; in xe_pt_stage_bind_entry()
585 * xe_pt_stage_bind() - Build a disconnected page-table tree for a given address
593 * This function builds a disconnected page-table tree for a given address
597 * shared page-tables, and non-existing shared and non-shared page-tables
623 .needs_64K = (xe_vma_vm(vma)->flags & XE_VM_FLAG_64K) && is_devmem, in xe_pt_stage_bind()
625 struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id]; in xe_pt_stage_bind()
632 * - Device atomics are expected to function with all allocations. in xe_pt_stage_bind()
635 * - Device atomics are the default behavior when the bo is placed in a single region. in xe_pt_stage_bind()
636 * - In all other cases device atomics will be disabled with AE=0 until an application in xe_pt_stage_bind()
639 if (vma->gpuva.flags & XE_VMA_ATOMIC_PTE_BIT) { in xe_pt_stage_bind()
644 * If a SMEM+LMEM allocation is backed by SMEM, a device in xe_pt_stage_bind()
659 if (!xe->info.has_device_atomics_on_smem && !is_devmem) in xe_pt_stage_bind()
665 xe_walk.dma_offset = vram_region_gpu_offset(bo->ttm.resource); in xe_pt_stage_bind()
675 xe_res_first_sg(to_userptr_vma(vma)->userptr.sg, 0, in xe_pt_stage_bind()
678 xe_res_first(bo->ttm.resource, xe_vma_bo_offset(vma), in xe_pt_stage_bind()
687 ret = xe_pt_walk_range(&pt->base, pt->level, xe_vma_start(vma), in xe_pt_stage_bind()
695 * xe_pt_nonshared_offsets() - Determine the non-shared entry offsets of a
697 * @addr: The start address within the non-shared pagetable.
698 * @end: The end address within the non-shared pagetable.
699 * @level: The level of the non-shared pagetable.
702 * @offset: Ignored on input, First non-shared entry on output.
703 * @end_offset: Ignored on input, Last non-shared entry + 1 on output.
705 * A non-shared page-table has some entries that belong to the address range
714 * Return: true if there were non-shared entries, false otherwise.
721 u64 size = 1ull << walk->shifts[level]; in xe_pt_nonshared_offsets()
741 (*end_offset)--; in xe_pt_nonshared_offsets()
748 /** @base: The walk base-class */
779 if (xe_pt_nonshared_offsets(addr, next, --level, walk, action, &offset, in xe_pt_zap_ptes_entry()
781 xe_map_memset(tile_to_xe(xe_walk->tile), &xe_child->bo->vmap, in xe_pt_zap_ptes_entry()
783 (end_offset - offset) * sizeof(u64)); in xe_pt_zap_ptes_entry()
784 xe_walk->needs_invalidate = true; in xe_pt_zap_ptes_entry()
795 * xe_pt_zap_ptes() - Zap (zero) gpu ptes of an address range
802 * page-table entrieaso it can either clear the leaf PTEs or
803 * clear the pointers to lower-level page-tables. The caller is required
804 * to hold the necessary locks to ensure neither the page-table connectivity
805 * nor the page-table entries of the range is updated from under us.
820 struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id]; in xe_pt_zap_ptes()
821 u8 pt_mask = (vma->tile_present & ~vma->tile_invalidated); in xe_pt_zap_ptes()
823 if (!(pt_mask & BIT(tile->id))) in xe_pt_zap_ptes()
826 (void)xe_pt_walk_shared(&pt->base, pt->level, xe_vma_start(vma), in xe_pt_zap_ptes()
838 struct xe_pt_entry *ptes = update->pt_entries; in xe_vm_populate_pgtable()
863 if (pt->level) { in xe_pt_cancel_bind()
866 xe_vma_vm(vma)->flags, NULL); in xe_pt_cancel_bind()
879 lockdep_assert_held(&vm->lock); in xe_pt_commit_locks_assert()
882 dma_resv_assert_held(xe_vma_bo(vma)->ttm.base.resv); in xe_pt_commit_locks_assert()
898 if (!pt->level) in xe_pt_commit()
904 xe_pt_destroy(oldpte, xe_vma_vm(vma)->flags, deferred); in xe_pt_commit()
917 for (i = num_entries - 1; i >= 0; --i) { in xe_pt_abort_bind()
922 pt->num_live -= entries[i].qwords; in xe_pt_abort_bind()
924 if (!pt->level) in xe_pt_abort_bind()
933 pt_dir->children[j_] = oldpte ? &oldpte->base : 0; in xe_pt_abort_bind()
934 xe_pt_destroy(newpte, xe_vma_vm(vma)->flags, NULL); in xe_pt_abort_bind()
952 pt->num_live += entries[i].qwords; in xe_pt_commit_prepare_bind()
954 if (!pt->level) in xe_pt_commit_prepare_bind()
966 pt_dir->children[j_] = &newpte->base; in xe_pt_commit_prepare_bind()
1002 vm_dbg(&xe->drm, "%s: %u entries to update\n", bind ? "bind" : "unbind", in xe_vm_dbg_print_entries()
1006 struct xe_pt *xe_pt = entry->pt; in xe_vm_dbg_print_entries()
1007 u64 page_size = 1ull << xe_pt_shift(xe_pt->level); in xe_vm_dbg_print_entries()
1011 xe_assert(xe, !entry->pt->is_compact); in xe_vm_dbg_print_entries()
1012 start = entry->ofs * page_size; in xe_vm_dbg_print_entries()
1013 end = start + page_size * entry->qwords; in xe_vm_dbg_print_entries()
1014 vm_dbg(&xe->drm, in xe_vm_dbg_print_entries()
1016 i, xe_pt->level, entry->ofs, entry->qwords, in xe_vm_dbg_print_entries()
1032 &fence->flags)) in no_in_syncs()
1045 return -ETIME; in job_test_add_deps()
1059 if (bo && !bo->vm) in vma_add_deps()
1060 return job_test_add_deps(job, bo->ttm.base.resv, in vma_add_deps()
1071 switch (op->base.op) { in op_add_deps()
1073 if (!op->map.immediate && xe_vm_in_fault_mode(vm)) in op_add_deps()
1076 err = vma_add_deps(op->map.vma, job); in op_add_deps()
1079 if (op->remap.prev) in op_add_deps()
1080 err = vma_add_deps(op->remap.prev, job); in op_add_deps()
1081 if (!err && op->remap.next) in op_add_deps()
1082 err = vma_add_deps(op->remap.next, job); in op_add_deps()
1087 err = vma_add_deps(gpuva_to_vma(op->base.prefetch.va), job); in op_add_deps()
1090 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); in op_add_deps()
1109 if (!job && !no_in_syncs(vops->syncs, vops->num_syncs)) in xe_pt_vm_dependencies()
1110 return -ETIME; in xe_pt_vm_dependencies()
1112 if (!job && !xe_exec_queue_is_idle(pt_update_ops->q)) in xe_pt_vm_dependencies()
1113 return -ETIME; in xe_pt_vm_dependencies()
1115 if (pt_update_ops->wait_vm_bookkeep || pt_update_ops->wait_vm_kernel) { in xe_pt_vm_dependencies()
1117 pt_update_ops->wait_vm_bookkeep ? in xe_pt_vm_dependencies()
1124 rtfence = xe_range_fence_tree_first(rftree, pt_update_ops->start, in xe_pt_vm_dependencies()
1125 pt_update_ops->last); in xe_pt_vm_dependencies()
1127 fence = rtfence->fence; in xe_pt_vm_dependencies()
1135 return -ETIME; in xe_pt_vm_dependencies()
1138 err = drm_sched_job_add_dependency(&job->drm, fence); in xe_pt_vm_dependencies()
1144 pt_update_ops->start, in xe_pt_vm_dependencies()
1145 pt_update_ops->last); in xe_pt_vm_dependencies()
1148 list_for_each_entry(op, &vops->list, link) { in xe_pt_vm_dependencies()
1154 if (!(pt_update_ops->q->flags & EXEC_QUEUE_FLAG_KERNEL)) { in xe_pt_vm_dependencies()
1158 err = xe_exec_queue_last_fence_test_dep(pt_update_ops->q, vm); in xe_pt_vm_dependencies()
1161 for (i = 0; job && !err && i < vops->num_syncs; i++) in xe_pt_vm_dependencies()
1162 err = xe_sync_entry_add_deps(&vops->syncs[i], job); in xe_pt_vm_dependencies()
1169 struct xe_vma_ops *vops = pt_update->vops; in xe_pt_pre_commit()
1170 struct xe_vm *vm = vops->vm; in xe_pt_pre_commit()
1171 struct xe_range_fence_tree *rftree = &vm->rftree[pt_update->tile_id]; in xe_pt_pre_commit()
1173 &vops->pt_update_ops[pt_update->tile_id]; in xe_pt_pre_commit()
1175 return xe_pt_vm_dependencies(pt_update->job, vm, pt_update->vops, in xe_pt_pre_commit()
1183 u32 divisor = uvma->userptr.divisor ? uvma->userptr.divisor : 2; in xe_pt_userptr_inject_eagain()
1186 if (count++ % divisor == divisor - 1) { in xe_pt_userptr_inject_eagain()
1187 uvma->userptr.divisor = divisor << 1; in xe_pt_userptr_inject_eagain()
1209 lockdep_assert_held_read(&vm->userptr.notifier_lock); in vma_check_userptr()
1215 notifier_seq = uvma->userptr.notifier_seq; in vma_check_userptr()
1217 if (uvma->userptr.initial_bind && !xe_vm_in_fault_mode(vm)) in vma_check_userptr()
1220 if (!mmu_interval_read_retry(&uvma->userptr.notifier, in vma_check_userptr()
1226 return -EAGAIN; in vma_check_userptr()
1228 spin_lock(&vm->userptr.invalidated_lock); in vma_check_userptr()
1229 list_move_tail(&uvma->userptr.invalidate_link, in vma_check_userptr()
1230 &vm->userptr.invalidated); in vma_check_userptr()
1231 spin_unlock(&vm->userptr.invalidated_lock); in vma_check_userptr()
1259 lockdep_assert_held_read(&vm->userptr.notifier_lock); in op_check_userptr()
1261 switch (op->base.op) { in op_check_userptr()
1263 if (!op->map.immediate && xe_vm_in_fault_mode(vm)) in op_check_userptr()
1266 err = vma_check_userptr(vm, op->map.vma, pt_update); in op_check_userptr()
1269 if (op->remap.prev) in op_check_userptr()
1270 err = vma_check_userptr(vm, op->remap.prev, pt_update); in op_check_userptr()
1271 if (!err && op->remap.next) in op_check_userptr()
1272 err = vma_check_userptr(vm, op->remap.next, pt_update); in op_check_userptr()
1277 err = vma_check_userptr(vm, gpuva_to_vma(op->base.prefetch.va), in op_check_userptr()
1281 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); in op_check_userptr()
1289 struct xe_vm *vm = pt_update->vops->vm; in xe_pt_userptr_pre_commit()
1290 struct xe_vma_ops *vops = pt_update->vops; in xe_pt_userptr_pre_commit()
1292 &vops->pt_update_ops[pt_update->tile_id]; in xe_pt_userptr_pre_commit()
1300 down_read(&vm->userptr.notifier_lock); in xe_pt_userptr_pre_commit()
1302 list_for_each_entry(op, &vops->list, link) { in xe_pt_userptr_pre_commit()
1305 up_read(&vm->userptr.notifier_lock); in xe_pt_userptr_pre_commit()
1329 struct xe_device *xe = gt_to_xe(ifence->gt); in invalidation_fence_cb()
1331 trace_xe_gt_tlb_invalidation_fence_cb(xe, &ifence->base); in invalidation_fence_cb()
1332 if (!ifence->fence->error) { in invalidation_fence_cb()
1333 queue_work(system_wq, &ifence->work); in invalidation_fence_cb()
1335 ifence->base.base.error = ifence->fence->error; in invalidation_fence_cb()
1336 xe_gt_tlb_invalidation_fence_signal(&ifence->base); in invalidation_fence_cb()
1338 dma_fence_put(ifence->fence); in invalidation_fence_cb()
1345 struct xe_device *xe = gt_to_xe(ifence->gt); in invalidation_fence_work_func()
1347 trace_xe_gt_tlb_invalidation_fence_work_func(xe, &ifence->base); in invalidation_fence_work_func()
1348 xe_gt_tlb_invalidation_range(ifence->gt, &ifence->base, ifence->start, in invalidation_fence_work_func()
1349 ifence->end, ifence->asid); in invalidation_fence_work_func()
1359 trace_xe_gt_tlb_invalidation_fence_create(gt_to_xe(gt), &ifence->base); in invalidation_fence_init()
1361 xe_gt_tlb_invalidation_fence_init(gt, &ifence->base, false); in invalidation_fence_init()
1363 ifence->fence = fence; in invalidation_fence_init()
1364 ifence->gt = gt; in invalidation_fence_init()
1365 ifence->start = start; in invalidation_fence_init()
1366 ifence->end = end; in invalidation_fence_init()
1367 ifence->asid = asid; in invalidation_fence_init()
1369 INIT_WORK(&ifence->work, invalidation_fence_work_func); in invalidation_fence_init()
1370 ret = dma_fence_add_callback(fence, &ifence->cb, invalidation_fence_cb); in invalidation_fence_init()
1371 if (ret == -ENOENT) { in invalidation_fence_init()
1372 dma_fence_put(ifence->fence); /* Usually dropped in CB */ in invalidation_fence_init()
1373 invalidation_fence_work_func(&ifence->work); in invalidation_fence_init()
1375 dma_fence_put(&ifence->base.base); /* Caller ref */ in invalidation_fence_init()
1376 dma_fence_put(&ifence->base.base); /* Creation ref */ in invalidation_fence_init()
1379 xe_gt_assert(gt, !ret || ret == -ENOENT); in invalidation_fence_init()
1383 /** @base: The pagewalk base-class. */
1400 /* @wupd: Structure to track the page-table updates we're building */
1416 unsigned int shift = walk->shifts[level]; in xe_pt_check_kill()
1420 ((next - addr) >> shift) == child->num_live) { in xe_pt_check_kill()
1421 u64 size = 1ull << walk->shifts[level + 1]; in xe_pt_check_kill()
1425 if (xe_walk->modified_start >= addr) in xe_pt_check_kill()
1426 xe_walk->modified_start = round_down(addr, size); in xe_pt_check_kill()
1427 if (xe_walk->modified_end <= next) in xe_pt_check_kill()
1428 xe_walk->modified_end = round_up(next, size); in xe_pt_check_kill()
1447 xe_pt_check_kill(addr, next, level - 1, xe_child, action, walk); in xe_pt_stage_unbind_entry()
1463 u64 size = 1ull << walk->shifts[--level]; in xe_pt_stage_unbind_post_descend()
1467 addr = xe_walk->modified_start; in xe_pt_stage_unbind_post_descend()
1469 next = xe_walk->modified_end; in xe_pt_stage_unbind_post_descend()
1480 err = xe_pt_new_shared(&xe_walk->wupd, xe_child, offset, true); in xe_pt_stage_unbind_post_descend()
1484 xe_walk->wupd.updates[level].update->qwords = end_offset - offset; in xe_pt_stage_unbind_post_descend()
1495 * xe_pt_stage_unbind() - Build page-table update structures for an unbind
1499 * @entries: Caller-provided storage for the update structures.
1501 * Builds page-table update structures for an unbind operation. The function
1502 * will attempt to remove all page-tables that we're the only user
1504 * same critical section that blocks racing binds to the same page-table tree.
1522 struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id]; in xe_pt_stage_unbind()
1524 (void)xe_pt_walk_shared(&pt->base, pt->level, xe_vma_start(vma), in xe_pt_stage_unbind()
1536 struct xe_vm *vm = pt_update->vops->vm; in xe_migrate_clear_pgtable_callback()
1537 u64 empty = __xe_pt_empty_pte(tile, vm, update->pt->level); in xe_migrate_clear_pgtable_callback()
1540 if (map && map->is_iomem) in xe_migrate_clear_pgtable_callback()
1545 memset64(map->vaddr + qword_ofs * sizeof(u64), empty, in xe_migrate_clear_pgtable_callback()
1559 for (i = num_entries - 1; i >= 0; --i) { in xe_pt_abort_unbind()
1561 struct xe_pt *pt = entry->pt; in xe_pt_abort_unbind()
1564 pt->num_live += entry->qwords; in xe_pt_abort_unbind()
1566 if (!pt->level) in xe_pt_abort_unbind()
1569 for (j = entry->ofs; j < entry->ofs + entry->qwords; j++) in xe_pt_abort_unbind()
1570 pt_dir->children[j] = in xe_pt_abort_unbind()
1571 entries[i].pt_entries[j - entry->ofs].pt ? in xe_pt_abort_unbind()
1572 &entries[i].pt_entries[j - entry->ofs].pt->base : NULL; in xe_pt_abort_unbind()
1587 struct xe_pt *pt = entry->pt; in xe_pt_commit_prepare_unbind()
1590 pt->num_live -= entry->qwords; in xe_pt_commit_prepare_unbind()
1591 if (!pt->level) in xe_pt_commit_prepare_unbind()
1595 for (j = entry->ofs; j < entry->ofs + entry->qwords; j++) { in xe_pt_commit_prepare_unbind()
1596 entry->pt_entries[j - entry->ofs].pt = in xe_pt_commit_prepare_unbind()
1598 pt_dir->children[j] = NULL; in xe_pt_commit_prepare_unbind()
1607 u32 current_op = pt_update_ops->current_op; in xe_pt_update_ops_rfence_interval()
1608 struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op]; in xe_pt_update_ops_rfence_interval()
1612 for (i = 0; i < pt_op->num_entries; i++) { in xe_pt_update_ops_rfence_interval()
1613 const struct xe_vm_pgtable_update *entry = &pt_op->entries[i]; in xe_pt_update_ops_rfence_interval()
1615 if (entry->pt->level > level) in xe_pt_update_ops_rfence_interval()
1616 level = entry->pt->level; in xe_pt_update_ops_rfence_interval()
1619 /* Greedy (non-optimal) calculation but simple */ in xe_pt_update_ops_rfence_interval()
1621 last = ALIGN(xe_vma_end(vma), 0x1ull << xe_pt_shift(level)) - 1; in xe_pt_update_ops_rfence_interval()
1623 if (start < pt_update_ops->start) in xe_pt_update_ops_rfence_interval()
1624 pt_update_ops->start = start; in xe_pt_update_ops_rfence_interval()
1625 if (last > pt_update_ops->last) in xe_pt_update_ops_rfence_interval()
1626 pt_update_ops->last = last; in xe_pt_update_ops_rfence_interval()
1631 int shift = xe_device_get_root_tile(xe)->media_gt ? 1 : 0; in vma_reserve_fences()
1633 if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) in vma_reserve_fences()
1634 return dma_resv_reserve_fences(xe_vma_bo(vma)->ttm.base.resv, in vma_reserve_fences()
1635 xe->info.tile_count << shift); in vma_reserve_fences()
1644 u32 current_op = pt_update_ops->current_op; in bind_op_prepare()
1645 struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op]; in bind_op_prepare()
1650 vm_dbg(&xe_vma_vm(vma)->xe->drm, in bind_op_prepare()
1652 xe_vma_start(vma), xe_vma_end(vma) - 1); in bind_op_prepare()
1654 pt_op->vma = NULL; in bind_op_prepare()
1655 pt_op->bind = true; in bind_op_prepare()
1656 pt_op->rebind = BIT(tile->id) & vma->tile_present; in bind_op_prepare()
1662 err = xe_pt_prepare_bind(tile, vma, pt_op->entries, in bind_op_prepare()
1663 &pt_op->num_entries); in bind_op_prepare()
1665 xe_tile_assert(tile, pt_op->num_entries <= in bind_op_prepare()
1666 ARRAY_SIZE(pt_op->entries)); in bind_op_prepare()
1667 xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries, in bind_op_prepare()
1668 pt_op->num_entries, true); in bind_op_prepare()
1671 ++pt_update_ops->current_op; in bind_op_prepare()
1672 pt_update_ops->needs_userptr_lock |= xe_vma_is_userptr(vma); in bind_op_prepare()
1677 * automatically when the context is re-enabled by the rebind worker, in bind_op_prepare()
1683 * non-faulting LR, in particular on user-space batch buffer chaining, in bind_op_prepare()
1686 if ((!pt_op->rebind && xe_vm_has_scratch(vm) && in bind_op_prepare()
1688 pt_update_ops->needs_invalidation = true; in bind_op_prepare()
1689 else if (pt_op->rebind && !xe_vm_in_lr_mode(vm)) in bind_op_prepare()
1691 vm->tlb_flush_seqno++; in bind_op_prepare()
1693 vma->tile_staged |= BIT(tile->id); in bind_op_prepare()
1694 pt_op->vma = vma; in bind_op_prepare()
1695 xe_pt_commit_prepare_bind(vma, pt_op->entries, in bind_op_prepare()
1696 pt_op->num_entries, pt_op->rebind); in bind_op_prepare()
1698 xe_pt_cancel_bind(vma, pt_op->entries, pt_op->num_entries); in bind_op_prepare()
1708 u32 current_op = pt_update_ops->current_op; in unbind_op_prepare()
1709 struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op]; in unbind_op_prepare()
1712 if (!((vma->tile_present | vma->tile_staged) & BIT(tile->id))) in unbind_op_prepare()
1717 vm_dbg(&xe_vma_vm(vma)->xe->drm, in unbind_op_prepare()
1719 xe_vma_start(vma), xe_vma_end(vma) - 1); in unbind_op_prepare()
1726 mmu_interval_read_begin(&to_userptr_vma(vma)->userptr.notifier); in unbind_op_prepare()
1728 pt_op->vma = vma; in unbind_op_prepare()
1729 pt_op->bind = false; in unbind_op_prepare()
1730 pt_op->rebind = false; in unbind_op_prepare()
1736 pt_op->num_entries = xe_pt_stage_unbind(tile, vma, pt_op->entries); in unbind_op_prepare()
1738 xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries, in unbind_op_prepare()
1739 pt_op->num_entries, false); in unbind_op_prepare()
1741 ++pt_update_ops->current_op; in unbind_op_prepare()
1742 pt_update_ops->needs_userptr_lock |= xe_vma_is_userptr(vma); in unbind_op_prepare()
1743 pt_update_ops->needs_invalidation = true; in unbind_op_prepare()
1745 xe_pt_commit_prepare_unbind(vma, pt_op->entries, pt_op->num_entries); in unbind_op_prepare()
1759 switch (op->base.op) { in op_prepare()
1761 if (!op->map.immediate && xe_vm_in_fault_mode(vm)) in op_prepare()
1764 err = bind_op_prepare(vm, tile, pt_update_ops, op->map.vma); in op_prepare()
1765 pt_update_ops->wait_vm_kernel = true; in op_prepare()
1769 gpuva_to_vma(op->base.remap.unmap->va)); in op_prepare()
1771 if (!err && op->remap.prev) { in op_prepare()
1773 op->remap.prev); in op_prepare()
1774 pt_update_ops->wait_vm_bookkeep = true; in op_prepare()
1776 if (!err && op->remap.next) { in op_prepare()
1778 op->remap.next); in op_prepare()
1779 pt_update_ops->wait_vm_bookkeep = true; in op_prepare()
1784 gpuva_to_vma(op->base.unmap.va)); in op_prepare()
1788 gpuva_to_vma(op->base.prefetch.va)); in op_prepare()
1789 pt_update_ops->wait_vm_kernel = true; in op_prepare()
1792 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); in op_prepare()
1801 init_llist_head(&pt_update_ops->deferred); in xe_pt_update_ops_init()
1802 pt_update_ops->start = ~0x0ull; in xe_pt_update_ops_init()
1803 pt_update_ops->last = 0x0ull; in xe_pt_update_ops_init()
1807 * xe_pt_update_ops_prepare() - Prepare PT update operations
1820 &vops->pt_update_ops[tile->id]; in xe_pt_update_ops_prepare()
1822 int shift = tile->media_gt ? 1 : 0; in xe_pt_update_ops_prepare()
1825 lockdep_assert_held(&vops->vm->lock); in xe_pt_update_ops_prepare()
1826 xe_vm_assert_held(vops->vm); in xe_pt_update_ops_prepare()
1830 err = dma_resv_reserve_fences(xe_vm_resv(vops->vm), in xe_pt_update_ops_prepare()
1831 tile_to_xe(tile)->info.tile_count << shift); in xe_pt_update_ops_prepare()
1835 list_for_each_entry(op, &vops->list, link) { in xe_pt_update_ops_prepare()
1836 err = op_prepare(vops->vm, tile, pt_update_ops, op); in xe_pt_update_ops_prepare()
1842 xe_tile_assert(tile, pt_update_ops->current_op <= in xe_pt_update_ops_prepare()
1843 pt_update_ops->num_ops); in xe_pt_update_ops_prepare()
1846 if (vops->inject_error && in xe_pt_update_ops_prepare()
1847 vops->vm->xe->vm_inject_error_position == FORCE_OP_ERROR_PREPARE) in xe_pt_update_ops_prepare()
1848 return -ENOSPC; in xe_pt_update_ops_prepare()
1859 if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) { in bind_op_commit()
1860 dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence, in bind_op_commit()
1861 pt_update_ops->wait_vm_bookkeep ? in bind_op_commit()
1865 dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence2, in bind_op_commit()
1866 pt_update_ops->wait_vm_bookkeep ? in bind_op_commit()
1870 vma->tile_present |= BIT(tile->id); in bind_op_commit()
1871 vma->tile_staged &= ~BIT(tile->id); in bind_op_commit()
1873 lockdep_assert_held_read(&vm->userptr.notifier_lock); in bind_op_commit()
1874 to_userptr_vma(vma)->userptr.initial_bind = true; in bind_op_commit()
1881 if (pt_update_ops->wait_vm_bookkeep && in bind_op_commit()
1883 !current->mm) in bind_op_commit()
1892 if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) { in unbind_op_commit()
1893 dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence, in unbind_op_commit()
1894 pt_update_ops->wait_vm_bookkeep ? in unbind_op_commit()
1898 dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence2, in unbind_op_commit()
1899 pt_update_ops->wait_vm_bookkeep ? in unbind_op_commit()
1903 vma->tile_present &= ~BIT(tile->id); in unbind_op_commit()
1904 if (!vma->tile_present) { in unbind_op_commit()
1905 list_del_init(&vma->combined_links.rebind); in unbind_op_commit()
1907 lockdep_assert_held_read(&vm->userptr.notifier_lock); in unbind_op_commit()
1909 spin_lock(&vm->userptr.invalidated_lock); in unbind_op_commit()
1910 list_del_init(&to_userptr_vma(vma)->userptr.invalidate_link); in unbind_op_commit()
1911 spin_unlock(&vm->userptr.invalidated_lock); in unbind_op_commit()
1924 switch (op->base.op) { in op_commit()
1926 if (!op->map.immediate && xe_vm_in_fault_mode(vm)) in op_commit()
1929 bind_op_commit(vm, tile, pt_update_ops, op->map.vma, fence, in op_commit()
1934 gpuva_to_vma(op->base.remap.unmap->va), fence, in op_commit()
1937 if (op->remap.prev) in op_commit()
1938 bind_op_commit(vm, tile, pt_update_ops, op->remap.prev, in op_commit()
1940 if (op->remap.next) in op_commit()
1941 bind_op_commit(vm, tile, pt_update_ops, op->remap.next, in op_commit()
1946 gpuva_to_vma(op->base.unmap.va), fence, fence2); in op_commit()
1950 gpuva_to_vma(op->base.prefetch.va), fence, fence2); in op_commit()
1953 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); in op_commit()
1970 * xe_pt_update_ops_run() - Run PT update operations
1983 struct xe_vm *vm = vops->vm; in xe_pt_update_ops_run()
1985 &vops->pt_update_ops[tile->id]; in xe_pt_update_ops_run()
1994 .ops = pt_update_ops->needs_userptr_lock ? in xe_pt_update_ops_run()
1998 .tile_id = tile->id, in xe_pt_update_ops_run()
2001 lockdep_assert_held(&vm->lock); in xe_pt_update_ops_run()
2004 if (!pt_update_ops->current_op) { in xe_pt_update_ops_run()
2011 if (vops->inject_error && in xe_pt_update_ops_run()
2012 vm->xe->vm_inject_error_position == FORCE_OP_ERROR_RUN) in xe_pt_update_ops_run()
2013 return ERR_PTR(-ENOSPC); in xe_pt_update_ops_run()
2016 if (pt_update_ops->needs_invalidation) { in xe_pt_update_ops_run()
2019 err = -ENOMEM; in xe_pt_update_ops_run()
2022 if (tile->media_gt) { in xe_pt_update_ops_run()
2025 err = -ENOMEM; in xe_pt_update_ops_run()
2030 err = -ENOMEM; in xe_pt_update_ops_run()
2035 err = -ENOMEM; in xe_pt_update_ops_run()
2043 err = -ENOMEM; in xe_pt_update_ops_run()
2047 fence = xe_migrate_update_pgtables(tile->migrate, &update); in xe_pt_update_ops_run()
2053 /* Point of no return - VM killed if failure after this */ in xe_pt_update_ops_run()
2054 for (i = 0; i < pt_update_ops->current_op; ++i) { in xe_pt_update_ops_run()
2055 struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i]; in xe_pt_update_ops_run()
2057 xe_pt_commit(pt_op->vma, pt_op->entries, in xe_pt_update_ops_run()
2058 pt_op->num_entries, &pt_update_ops->deferred); in xe_pt_update_ops_run()
2059 pt_op->vma = NULL; /* skip in xe_pt_update_ops_abort */ in xe_pt_update_ops_run()
2062 if (xe_range_fence_insert(&vm->rftree[tile->id], rfence, in xe_pt_update_ops_run()
2064 pt_update_ops->start, in xe_pt_update_ops_run()
2065 pt_update_ops->last, fence)) in xe_pt_update_ops_run()
2072 invalidation_fence_init(tile->primary_gt, ifence, fence, in xe_pt_update_ops_run()
2073 pt_update_ops->start, in xe_pt_update_ops_run()
2074 pt_update_ops->last, vm->usm.asid); in xe_pt_update_ops_run()
2076 invalidation_fence_init(tile->media_gt, mfence, fence, in xe_pt_update_ops_run()
2077 pt_update_ops->start, in xe_pt_update_ops_run()
2078 pt_update_ops->last, vm->usm.asid); in xe_pt_update_ops_run()
2079 fences[0] = &ifence->base.base; in xe_pt_update_ops_run()
2080 fences[1] = &mfence->base.base; in xe_pt_update_ops_run()
2082 vm->composite_fence_ctx, in xe_pt_update_ops_run()
2083 vm->composite_fence_seqno++, in xe_pt_update_ops_run()
2085 fence = &cf->base; in xe_pt_update_ops_run()
2087 fence = &ifence->base.base; in xe_pt_update_ops_run()
2093 pt_update_ops->wait_vm_bookkeep ? in xe_pt_update_ops_run()
2097 list_for_each_entry(op, &vops->list, link) in xe_pt_update_ops_run()
2098 op_commit(vops->vm, tile, pt_update_ops, op, fence, NULL); in xe_pt_update_ops_run()
2100 dma_resv_add_fence(xe_vm_resv(vm), &ifence->base.base, in xe_pt_update_ops_run()
2101 pt_update_ops->wait_vm_bookkeep ? in xe_pt_update_ops_run()
2105 dma_resv_add_fence(xe_vm_resv(vm), &mfence->base.base, in xe_pt_update_ops_run()
2106 pt_update_ops->wait_vm_bookkeep ? in xe_pt_update_ops_run()
2110 list_for_each_entry(op, &vops->list, link) in xe_pt_update_ops_run()
2111 op_commit(vops->vm, tile, pt_update_ops, op, in xe_pt_update_ops_run()
2112 &ifence->base.base, &mfence->base.base); in xe_pt_update_ops_run()
2115 if (pt_update_ops->needs_userptr_lock) in xe_pt_update_ops_run()
2116 up_read(&vm->userptr.notifier_lock); in xe_pt_update_ops_run()
2128 if (err != -EAGAIN && tile->id) in xe_pt_update_ops_run()
2129 xe_vm_kill(vops->vm, false); in xe_pt_update_ops_run()
2135 * xe_pt_update_ops_fini() - Finish PT update operations
2144 &vops->pt_update_ops[tile->id]; in xe_pt_update_ops_fini()
2147 lockdep_assert_held(&vops->vm->lock); in xe_pt_update_ops_fini()
2148 xe_vm_assert_held(vops->vm); in xe_pt_update_ops_fini()
2150 for (i = 0; i < pt_update_ops->current_op; ++i) { in xe_pt_update_ops_fini()
2151 struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i]; in xe_pt_update_ops_fini()
2153 xe_pt_free_bind(pt_op->entries, pt_op->num_entries); in xe_pt_update_ops_fini()
2155 xe_bo_put_commit(&vops->pt_update_ops[tile->id].deferred); in xe_pt_update_ops_fini()
2159 * xe_pt_update_ops_abort() - Abort PT update operations
2168 &vops->pt_update_ops[tile->id]; in xe_pt_update_ops_abort()
2171 lockdep_assert_held(&vops->vm->lock); in xe_pt_update_ops_abort()
2172 xe_vm_assert_held(vops->vm); in xe_pt_update_ops_abort()
2174 for (i = pt_update_ops->num_ops - 1; i >= 0; --i) { in xe_pt_update_ops_abort()
2176 &pt_update_ops->ops[i]; in xe_pt_update_ops_abort()
2178 if (!pt_op->vma || i >= pt_update_ops->current_op) in xe_pt_update_ops_abort()
2181 if (pt_op->bind) in xe_pt_update_ops_abort()
2182 xe_pt_abort_bind(pt_op->vma, pt_op->entries, in xe_pt_update_ops_abort()
2183 pt_op->num_entries, in xe_pt_update_ops_abort()
2184 pt_op->rebind); in xe_pt_update_ops_abort()
2186 xe_pt_abort_unbind(pt_op->vma, pt_op->entries, in xe_pt_update_ops_abort()
2187 pt_op->num_entries); in xe_pt_update_ops_abort()