Lines Matching +full:0 +full:xe
53 .fpfn = 0,
54 .lpfn = 0,
56 .flags = 0,
68 .fpfn = 0,
69 .lpfn = 0,
74 .fpfn = 0,
75 .lpfn = 0,
87 for (unsigned int __bit_tmp = BIT(0); __bit_tmp <= XE_BO_FLAG_VRAM_MASK; __bit_tmp <<= 1) \
95 static bool resource_is_stolen_vram(struct xe_device *xe, struct ttm_resource *res) in resource_is_stolen_vram() argument
97 return res->mem_type == XE_PL_STOLEN && IS_DGFX(xe); in resource_is_stolen_vram()
167 mem_type_to_migrate(struct xe_device *xe, u32 mem_type) in mem_type_to_migrate() argument
171 xe_assert(xe, mem_type == XE_PL_STOLEN || mem_type_is_vram(mem_type)); in mem_type_to_migrate()
172 tile = &xe->tiles[mem_type == XE_PL_STOLEN ? 0 : (mem_type - XE_PL_VRAM0)]; in mem_type_to_migrate()
178 struct xe_device *xe = ttm_to_xe_device(res->bo->bdev); in res_to_mem_region() local
182 xe_assert(xe, resource_is_vram(res)); in res_to_mem_region()
183 mgr = ttm_manager_type(&xe->ttm, res->mem_type); in res_to_mem_region()
189 static void try_add_system(struct xe_device *xe, struct xe_bo *bo, in try_add_system() argument
193 xe_assert(xe, *c < ARRAY_SIZE(bo->placements)); in try_add_system()
198 TTM_PL_FLAG_FALLBACK : 0, in try_add_system()
222 static u8 vram_bo_flag_to_tile_id(struct xe_device *xe, u32 vram_bo_flag) in vram_bo_flag_to_tile_id() argument
224 xe_assert(xe, vram_bo_flag & XE_BO_FLAG_VRAM_MASK); in vram_bo_flag_to_tile_id()
225 xe_assert(xe, (vram_bo_flag & (vram_bo_flag - 1)) == 0); in vram_bo_flag_to_tile_id()
230 static u32 bo_vram_flags_to_vram_placement(struct xe_device *xe, u32 bo_flags, u32 vram_flag, in bo_vram_flags_to_vram_placement() argument
233 u8 tile_id = vram_bo_flag_to_tile_id(xe, vram_flag); in bo_vram_flags_to_vram_placement()
235 xe_assert(xe, tile_id < xe->info.tile_count); in bo_vram_flags_to_vram_placement()
238 return xe->tiles[tile_id].mem.kernel_vram->placement; in bo_vram_flags_to_vram_placement()
240 return xe->tiles[tile_id].mem.vram->placement; in bo_vram_flags_to_vram_placement()
243 static void add_vram(struct xe_device *xe, struct xe_bo *bo, in add_vram() argument
247 struct ttm_resource_manager *mgr = ttm_manager_type(&xe->ttm, mem_type); in add_vram()
253 xe_assert(xe, *c < ARRAY_SIZE(bo->placements)); in add_vram()
256 xe_assert(xe, vram && vram->usable_size); in add_vram()
264 place.fpfn = 0; in add_vram()
274 static void try_add_vram(struct xe_device *xe, struct xe_bo *bo, in try_add_vram() argument
280 u32 pl = bo_vram_flags_to_vram_placement(xe, bo_flags, vram_flag, type); in try_add_vram()
282 add_vram(xe, bo, bo->placements, bo_flags, pl, c); in try_add_vram()
286 static void try_add_stolen(struct xe_device *xe, struct xe_bo *bo, in try_add_stolen() argument
290 xe_assert(xe, *c < ARRAY_SIZE(bo->placements)); in try_add_stolen()
295 TTM_PL_FLAG_CONTIGUOUS : 0, in try_add_stolen()
301 static int __xe_bo_placement_for_flags(struct xe_device *xe, struct xe_bo *bo, in __xe_bo_placement_for_flags() argument
304 u32 c = 0; in __xe_bo_placement_for_flags()
306 try_add_vram(xe, bo, bo_flags, type, &c); in __xe_bo_placement_for_flags()
307 try_add_system(xe, bo, bo_flags, &c); in __xe_bo_placement_for_flags()
308 try_add_stolen(xe, bo, bo_flags, &c); in __xe_bo_placement_for_flags()
318 return 0; in __xe_bo_placement_for_flags()
321 int xe_bo_placement_for_flags(struct xe_device *xe, struct xe_bo *bo, in xe_bo_placement_for_flags() argument
325 return __xe_bo_placement_for_flags(xe, bo, bo_flags, type); in xe_bo_placement_for_flags()
331 struct xe_device *xe = container_of(tbo->bdev, typeof(*xe), ttm); in xe_evict_flags() local
332 bool device_unplugged = drm_dev_is_unplugged(&xe->drm); in xe_evict_flags()
338 placement->num_placement = 0; in xe_evict_flags()
358 * For xe, sg bos that are evicted to system just triggers a in xe_evict_flags()
374 /* struct xe_ttm_tt - Subclassed ttm_tt for xe */
383 static int xe_tt_map_sg(struct xe_device *xe, struct ttm_tt *tt) in xe_tt_map_sg() argument
393 return 0; in xe_tt_map_sg()
396 num_pages, 0, in xe_tt_map_sg()
398 xe_sg_segment_size(xe->drm.dev), in xe_tt_map_sg()
404 ret = dma_map_sgtable(xe->drm.dev, xe_tt->sg, DMA_BIDIRECTIONAL, in xe_tt_map_sg()
412 return 0; in xe_tt_map_sg()
415 static void xe_tt_unmap_sg(struct xe_device *xe, struct ttm_tt *tt) in xe_tt_unmap_sg() argument
420 dma_unmap_sgtable(xe->drm.dev, xe_tt->sg, in xe_tt_unmap_sg()
421 DMA_BIDIRECTIONAL, 0); in xe_tt_unmap_sg()
439 static void xe_ttm_tt_account_add(struct xe_device *xe, struct ttm_tt *tt) in xe_ttm_tt_account_add() argument
444 xe_shrinker_mod_pages(xe->mem.shrinker, 0, tt->num_pages); in xe_ttm_tt_account_add()
446 xe_shrinker_mod_pages(xe->mem.shrinker, tt->num_pages, 0); in xe_ttm_tt_account_add()
449 static void xe_ttm_tt_account_subtract(struct xe_device *xe, struct ttm_tt *tt) in xe_ttm_tt_account_subtract() argument
454 xe_shrinker_mod_pages(xe->mem.shrinker, 0, -(long)tt->num_pages); in xe_ttm_tt_account_subtract()
456 xe_shrinker_mod_pages(xe->mem.shrinker, -(long)tt->num_pages, 0); in xe_ttm_tt_account_subtract()
463 struct xe_device *xe = ttm_to_xe_device(ttm_dev); in update_global_total_pages() local
465 atomic64_add_return(num_pages, &xe->global_total_pages); in update_global_total_pages()
467 trace_gpu_mem_total(xe->drm.primary->index, 0, in update_global_total_pages()
476 struct xe_device *xe = xe_bo_device(bo); in xe_ttm_tt_create() local
489 extra_pages = 0; in xe_ttm_tt_create()
491 extra_pages = DIV_ROUND_UP(xe_device_ccs_bytes(xe, xe_bo_size(bo)), in xe_ttm_tt_create()
500 if (!IS_DGFX(xe)) { in xe_ttm_tt_create()
517 (!xe->info.has_cached_pt && bo->flags & XE_BO_FLAG_PAGETABLE)) in xe_ttm_tt_create()
526 xe_assert(xe, bo->cpu_caching == 0); in xe_ttm_tt_create()
563 return 0; in xe_ttm_tt_populate()
578 return 0; in xe_ttm_tt_populate()
583 struct xe_device *xe = ttm_to_xe_device(ttm_dev); in xe_ttm_tt_unpopulate() local
589 xe_tt_unmap_sg(xe, tt); in xe_ttm_tt_unpopulate()
592 xe_ttm_tt_account_subtract(xe, tt); in xe_ttm_tt_unpopulate()
630 struct xe_device *xe = ttm_to_xe_device(bdev); in xe_ttm_io_mem_reserve() local
635 return 0; in xe_ttm_io_mem_reserve()
656 return 0; in xe_ttm_io_mem_reserve()
658 return xe_ttm_stolen_io_mem_reserve(xe, mem); in xe_ttm_io_mem_reserve()
664 static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo, in xe_bo_trigger_rebind() argument
672 int ret = 0; in xe_bo_trigger_rebind()
694 if (!xe_device_is_l2_flush_optimized(xe)) in xe_bo_trigger_rebind()
712 if (timeout < 0) in xe_bo_trigger_rebind()
746 struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev); in xe_bo_move_dmabuf() local
747 bool device_unplugged = drm_dev_is_unplugged(&xe->drm); in xe_bo_move_dmabuf()
750 xe_assert(xe, attach); in xe_bo_move_dmabuf()
751 xe_assert(xe, ttm_bo->ttm); in xe_bo_move_dmabuf()
779 return 0; in xe_bo_move_dmabuf()
797 * Return: 0 on success, -EINTR or -ERESTARTSYS if interrupted in fault mode,
804 struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev); in xe_bo_move_notify() local
818 ret = xe_bo_trigger_rebind(xe, bo, ctx); in xe_bo_move_notify()
832 mutex_lock(&xe->mem_access.vram_userfault.lock); in xe_bo_move_notify()
835 mutex_unlock(&xe->mem_access.vram_userfault.lock); in xe_bo_move_notify()
838 return 0; in xe_bo_move_notify()
854 struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev); in xe_bo_set_purgeable_shrinker() local
869 xe_shrinker_mod_pages(xe->mem.shrinker, -tt_pages, tt_pages); in xe_bo_set_purgeable_shrinker()
873 xe_shrinker_mod_pages(xe->mem.shrinker, tt_pages, -tt_pages); in xe_bo_set_purgeable_shrinker()
890 struct xe_device *xe = xe_bo_device(bo); in xe_bo_set_purgeable_state() local
895 xe_assert(xe, new_state == XE_MADV_PURGEABLE_WILLNEED || in xe_bo_set_purgeable_state()
900 xe_assert(xe, !(bo->purgeable.state == XE_MADV_PURGEABLE_PURGED && in xe_bo_set_purgeable_state()
917 * Return: 0 on success, negative error code on failure
928 return 0; in xe_ttm_bo_purge()
931 return 0; in xe_ttm_bo_purge()
954 return 0; in xe_ttm_bo_purge()
962 struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev); in xe_bo_move() local
972 bool handle_system_ccs = (!IS_DGFX(xe) && xe_bo_needs_ccs_pages(bo) && in xe_bo_move()
974 int ret = 0; in xe_bo_move()
987 return 0; in xe_bo_move()
993 ret = xe_tt_map_sg(xe, ttm); in xe_bo_move()
1016 ret = xe_tt_map_sg(xe, ttm); in xe_bo_move()
1030 drm_dbg(&xe->drm, "Evict system allocator BO success\n"); in xe_bo_move()
1033 drm_dbg(&xe->drm, "Evict system allocator BO failed=%pe\n", in xe_bo_move()
1067 if (timeout < 0) { in xe_bo_move()
1082 hop->fpfn = 0; in xe_bo_move()
1083 hop->lpfn = 0; in xe_bo_move()
1093 migrate = mem_type_to_migrate(xe, new_mem->mem_type); in xe_bo_move()
1095 migrate = mem_type_to_migrate(xe, old_mem_type); in xe_bo_move()
1097 migrate = xe->tiles[0].migrate; in xe_bo_move()
1099 xe_assert(xe, migrate); in xe_bo_move()
1101 if (xe_rpm_reclaim_safe(xe)) { in xe_bo_move()
1106 xe_pm_runtime_get(xe); in xe_bo_move()
1108 drm_WARN_ON(&xe->drm, handle_system_ccs); in xe_bo_move()
1109 xe_pm_runtime_get_noresume(xe); in xe_bo_move()
1113 u32 flags = 0; in xe_bo_move()
1127 xe_pm_runtime_put(xe); in xe_bo_move()
1136 ret = 0; in xe_bo_move()
1150 xe_pm_runtime_put(xe); in xe_bo_move()
1156 if (IS_VF_CCS_READY(xe) && old_mem_type == XE_PL_TT && in xe_bo_move()
1160 if (IS_VF_CCS_READY(xe) && in xe_bo_move()
1173 if (timeout < 0) in xe_bo_move()
1176 if (IS_VF_CCS_READY(xe)) in xe_bo_move()
1179 xe_tt_unmap_sg(xe, ttm_bo->ttm); in xe_bo_move()
1189 struct xe_device *xe = ttm_to_xe_device(bo->bdev); in xe_bo_shrink_purge() local
1205 xe_tt_unmap_sg(xe, bo->ttm); in xe_bo_shrink_purge()
1215 if (lret > 0) { in xe_bo_shrink_purge()
1216 xe_ttm_tt_account_subtract(xe, bo->ttm); in xe_bo_shrink_purge()
1243 * xe_bo_shrink() - Try to shrink an xe bo.
1251 * Note that we need to be able to handle also non xe bos
1268 struct xe_device *xe = ttm_to_xe_device(bo->bdev); in xe_bo_shrink() local
1270 long lret = 0L; in xe_bo_shrink()
1287 if (lret > 0 && xe_bo_madv_is_dontneed(xe_bo)) in xe_bo_shrink()
1294 needs_rpm = (!IS_DGFX(xe) && bo->resource->mem_type != XE_PL_SYSTEM && in xe_bo_shrink()
1296 if (needs_rpm && !xe_pm_runtime_get_if_active(xe)) in xe_bo_shrink()
1305 xe_pm_runtime_put(xe); in xe_bo_shrink()
1307 if (lret > 0) { in xe_bo_shrink()
1308 xe_ttm_tt_account_subtract(xe, tt); in xe_bo_shrink()
1326 * Return: 0 on success. Negative error code on failure.
1330 struct xe_device *xe = ttm_to_xe_device(bo->ttm.bdev); in xe_bo_notifier_prepare_pinned() local
1334 int ret = 0; in xe_bo_notifier_prepare_pinned()
1336 xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {.exclusive = true}, ret) { in xe_bo_notifier_prepare_pinned()
1339 xe_assert(xe, !ret); in xe_bo_notifier_prepare_pinned()
1340 xe_assert(xe, !bo->backup_obj); in xe_bo_notifier_prepare_pinned()
1356 backup = xe_bo_init_locked(xe, NULL, NULL, bo->ttm.base.resv, NULL, xe_bo_size(bo), in xe_bo_notifier_prepare_pinned()
1379 * Always returns 0. The backup object is removed, if still present. Expectation
1382 * Return: Always returns 0.
1394 return 0; in xe_bo_notifier_unprepare_pinned()
1399 struct xe_device *xe = xe_bo_device(bo); in xe_bo_evict_pinned_copy() local
1401 int ret = 0; in xe_bo_evict_pinned_copy()
1410 migrate = mem_type_to_migrate(xe, bo->ttm.resource->mem_type); in xe_bo_evict_pinned_copy()
1412 xe_assert(xe, bo->ttm.base.resv == backup->ttm.base.resv); in xe_bo_evict_pinned_copy()
1439 xe_map_memcpy_from(xe, backup->vmap.vaddr, &bo->vmap, 0, in xe_bo_evict_pinned_copy()
1463 * Return: 0 on success. Negative error code on failure.
1467 struct xe_device *xe = ttm_to_xe_device(bo->ttm.bdev); in xe_bo_evict_pinned() local
1472 int ret = 0; in xe_bo_evict_pinned()
1474 xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {.exclusive = true}, ret) { in xe_bo_evict_pinned()
1477 xe_assert(xe, !ret); in xe_bo_evict_pinned()
1496 backup = xe_bo_init_locked(xe, NULL, NULL, bo->ttm.base.resv, NULL, in xe_bo_evict_pinned()
1529 * Return: 0 on success. Negative error code on failure.
1537 struct xe_device *xe = ttm_to_xe_device(bo->ttm.bdev); in xe_bo_restore_pinned() local
1543 return 0; in xe_bo_restore_pinned()
1560 migrate = mem_type_to_migrate(xe, bo->ttm.resource->mem_type); in xe_bo_restore_pinned()
1589 xe_map_memcpy_to(xe, &bo->vmap, 0, backup->vmap.vaddr, in xe_bo_restore_pinned()
1626 DMA_BIDIRECTIONAL, 0); in xe_bo_dma_unmap_pinned()
1632 return 0; in xe_bo_dma_unmap_pinned()
1646 xe_res_first(ttm_bo->resource, (u64)page_offset << PAGE_SHIFT, 0, &cursor); in xe_ttm_io_mem_pfn()
1658 struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev); in xe_ttm_bo_lock_in_destructor() local
1661 xe_assert(xe, !kref_read(&ttm_bo->kref)); in xe_ttm_bo_lock_in_destructor()
1672 xe_assert(xe, locked); in xe_ttm_bo_lock_in_destructor()
1763 struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev); in xe_ttm_access_memory() local
1768 int err = 0; in xe_ttm_access_memory()
1771 xe_device_assert_mem_access(xe); in xe_ttm_access_memory()
1778 mem_type_to_migrate(xe, ttm_bo->resource->mem_type); in xe_ttm_access_memory()
1796 xe_map_memcpy_to(xe, &vmap, page_offset, buf, byte_count); in xe_ttm_access_memory()
1798 xe_map_memcpy_from(xe, buf, &vmap, page_offset, byte_count); in xe_ttm_access_memory()
1830 struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev); in xe_ttm_bo_destroy() local
1838 xe_assert(xe, list_empty(&ttm_bo->base.gpuva.list)); in xe_ttm_bo_destroy()
1840 for_each_tile(tile, xe, id) in xe_ttm_bo_destroy()
1855 mutex_lock(&xe->mem_access.vram_userfault.lock); in xe_ttm_bo_destroy()
1858 mutex_unlock(&xe->mem_access.vram_userfault.lock); in xe_ttm_bo_destroy()
1914 0 : -EBUSY; in xe_bo_wait_usage_kernel()
1918 if (lerr < 0) in xe_bo_wait_usage_kernel()
1920 if (lerr == 0) in xe_bo_wait_usage_kernel()
1923 return 0; in xe_bo_wait_usage_kernel()
1931 int err = 0; in xe_bo_fault_migrate()
1946 static vm_fault_t __xe_bo_cpu_fault(struct vm_fault *vmf, struct xe_device *xe, struct xe_bo *bo) in __xe_bo_cpu_fault() argument
1958 xe_assert(xe, ret != VM_FAULT_RETRY); in __xe_bo_cpu_fault()
1962 mutex_lock(&xe->mem_access.vram_userfault.lock); in __xe_bo_cpu_fault()
1965 &xe->mem_access.vram_userfault.list); in __xe_bo_cpu_fault()
1966 mutex_unlock(&xe->mem_access.vram_userfault.lock); in __xe_bo_cpu_fault()
1975 case 0: in xe_err_to_fault_t()
1998 static vm_fault_t xe_bo_cpu_fault_fastpath(struct vm_fault *vmf, struct xe_device *xe, in xe_bo_cpu_fault_fastpath() argument
2012 if (needs_rpm && !xe_pm_runtime_get_if_active(xe)) in xe_bo_cpu_fault_fastpath()
2015 err = xe_validation_ctx_init(&ctx, &xe->val, NULL, in xe_bo_cpu_fault_fastpath()
2038 drm_dbg(&xe->drm, "CPU trying to access an imported buffer object.\n"); in xe_bo_cpu_fault_fastpath()
2051 ret = __xe_bo_cpu_fault(vmf, xe, bo); in xe_bo_cpu_fault_fastpath()
2059 xe_pm_runtime_put(xe); in xe_bo_cpu_fault_fastpath()
2068 struct xe_device *xe = to_xe_device(ddev); in xe_bo_cpu_fault() local
2075 int err = 0; in xe_bo_cpu_fault()
2078 if (xe_device_wedged(xe) || !drm_dev_enter(&xe->drm, &idx)) in xe_bo_cpu_fault()
2081 ret = xe_bo_cpu_fault_fastpath(vmf, xe, bo, needs_rpm); in xe_bo_cpu_fault()
2111 xe_pm_runtime_get(xe); in xe_bo_cpu_fault()
2113 xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {.interruptible = true}, in xe_bo_cpu_fault()
2137 drm_dbg(&xe->drm, "CPU trying to access an imported buffer object.\n"); in xe_bo_cpu_fault()
2153 ret = __xe_bo_cpu_fault(vmf, xe, bo); in xe_bo_cpu_fault()
2160 xe_pm_runtime_put(xe); in xe_bo_cpu_fault()
2175 struct xe_device *xe = xe_bo_device(bo); in xe_bo_vm_access() local
2177 guard(xe_pm_runtime)(xe); in xe_bo_vm_access()
2196 ret = ttm_bo_access(&bo->ttm, offset, dst, size, 0); in xe_bo_read()
2197 if (ret >= 0 && ret != size) in xe_bo_read()
2200 ret = 0; in xe_bo_read()
2215 int err = 0; in xe_gem_object_mmap()
2281 * @xe: The xe device.
2295 * Initialize or create an xe buffer object. On failure, any allocated buffer
2300 struct xe_bo *xe_bo_init_locked(struct xe_device *xe, struct xe_bo *bo, in xe_bo_init_locked() argument
2317 xe_assert(xe, !tile || type == ttm_bo_type_kernel); in xe_bo_init_locked()
2332 ((xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) || in xe_bo_init_locked()
2373 drm_gem_private_object_init(&xe->drm, &bo->ttm.base, size); in xe_bo_init_locked()
2380 xe_validation_assert_exec(xe, exec, &bo->ttm.base); in xe_bo_init_locked()
2382 err = __xe_bo_placement_for_flags(xe, bo, bo->flags, type); in xe_bo_init_locked()
2393 err = ttm_bo_init_reserved(&xe->ttm, &bo->ttm, type, in xe_bo_init_locked()
2422 if (timeout < 0) { in xe_bo_init_locked()
2439 static int __xe_bo_fixed_placement(struct xe_device *xe, in __xe_bo_fixed_placement() argument
2451 xe_assert(xe, !IS_SRIOV_VF(xe) || !(bo->flags & XE_BO_FLAG_GGTT)); in __xe_bo_fixed_placement()
2470 place->mem_type = bo_vram_flags_to_vram_placement(xe, flags, vram_flag, type); in __xe_bo_fixed_placement()
2477 return 0; in __xe_bo_fixed_placement()
2481 __xe_bo_create_locked(struct xe_device *xe, in __xe_bo_create_locked() argument
2493 if (start || end != ~0ULL) { in __xe_bo_create_locked()
2499 err = __xe_bo_fixed_placement(xe, bo, type, flags, start, end, size); in __xe_bo_create_locked()
2506 bo = xe_bo_init_locked(xe, bo, tile, vm ? xe_vm_resv(vm) : NULL, in __xe_bo_create_locked()
2533 tile = xe_device_get_root_tile(xe); in __xe_bo_create_locked()
2535 xe_assert(xe, tile); in __xe_bo_create_locked()
2538 for_each_tile(t, xe, id) { in __xe_bo_create_locked()
2566 * @xe: The xe device.
2575 * Create a locked xe BO with no range- nor alignment restrictions.
2579 struct xe_bo *xe_bo_create_locked(struct xe_device *xe, struct xe_tile *tile, in xe_bo_create_locked() argument
2584 return __xe_bo_create_locked(xe, tile, vm, size, 0, ~0ULL, 0, type, in xe_bo_create_locked()
2585 flags, 0, exec); in xe_bo_create_locked()
2588 static struct xe_bo *xe_bo_create_novm(struct xe_device *xe, struct xe_tile *tile, in xe_bo_create_novm() argument
2596 int ret = 0; in xe_bo_create_novm()
2598 xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {.interruptible = intr}, in xe_bo_create_novm()
2600 bo = __xe_bo_create_locked(xe, tile, NULL, size, 0, ~0ULL, in xe_bo_create_novm()
2616 * @xe: The xe device.
2628 struct xe_bo *xe_bo_create_user(struct xe_device *xe, in xe_bo_create_user() argument
2638 xe_assert(xe, exec); in xe_bo_create_user()
2639 bo = __xe_bo_create_locked(xe, NULL, vm, size, 0, ~0ULL, in xe_bo_create_user()
2641 flags, 0, exec); in xe_bo_create_user()
2645 bo = xe_bo_create_novm(xe, NULL, size, cpu_caching, in xe_bo_create_user()
2646 ttm_bo_type_device, flags, 0, true); in xe_bo_create_user()
2654 * @xe: The xe device.
2658 * @start: Start of fixed VRAM range or 0.
2659 * @end: End of fixed VRAM range or ~0ULL.
2663 * Create an Xe BO with range- and options. If @start and @end indicate
2669 struct xe_bo *xe_bo_create_pin_range_novm(struct xe_device *xe, struct xe_tile *tile, in xe_bo_create_pin_range_novm() argument
2676 int err = 0; in xe_bo_create_pin_range_novm()
2678 xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {}, err) { in xe_bo_create_pin_range_novm()
2679 bo = __xe_bo_create_locked(xe, tile, NULL, size, start, end, in xe_bo_create_pin_range_novm()
2680 0, type, flags, 0, &exec); in xe_bo_create_pin_range_novm()
2701 static struct xe_bo *xe_bo_create_pin_map_at_aligned(struct xe_device *xe, in xe_bo_create_pin_map_at_aligned() argument
2710 u64 start = offset == ~0ull ? 0 : offset; in xe_bo_create_pin_map_at_aligned()
2711 u64 end = offset == ~0ull ? ~0ull : start + size; in xe_bo_create_pin_map_at_aligned()
2714 xe_ttm_stolen_cpu_access_needs_ggtt(xe)) in xe_bo_create_pin_map_at_aligned()
2717 bo = __xe_bo_create_locked(xe, tile, vm, size, start, end, 0, type, in xe_bo_create_pin_map_at_aligned()
2745 * @xe: The xe device.
2749 * @offset: Optional VRAM offset or %~0ull for don't care.
2763 xe_bo_create_pin_map_at_novm(struct xe_device *xe, struct xe_tile *tile, in xe_bo_create_pin_map_at_novm() argument
2770 int ret = 0; in xe_bo_create_pin_map_at_novm()
2772 xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {.interruptible = intr}, in xe_bo_create_pin_map_at_novm()
2774 bo = xe_bo_create_pin_map_at_aligned(xe, tile, NULL, size, offset, in xe_bo_create_pin_map_at_novm()
2788 * @xe: The xe device.
2806 struct xe_bo *xe_bo_create_pin_map(struct xe_device *xe, struct xe_tile *tile, in xe_bo_create_pin_map() argument
2811 return xe_bo_create_pin_map_at_aligned(xe, tile, vm, size, ~0ull, type, flags, in xe_bo_create_pin_map()
2812 0, exec); in xe_bo_create_pin_map()
2817 * @xe: The xe device.
2832 struct xe_bo *xe_bo_create_pin_map_novm(struct xe_device *xe, struct xe_tile *tile, in xe_bo_create_pin_map_novm() argument
2836 return xe_bo_create_pin_map_at_novm(xe, tile, size, ~0ull, type, flags, 0, intr); in xe_bo_create_pin_map_novm()
2844 struct xe_bo *xe_managed_bo_create_pin_map(struct xe_device *xe, struct xe_tile *tile, in xe_managed_bo_create_pin_map() argument
2850 KUNIT_STATIC_STUB_REDIRECT(xe_managed_bo_create_pin_map, xe, tile, size, flags); in xe_managed_bo_create_pin_map()
2851 bo = xe_bo_create_pin_map_novm(xe, tile, size, ttm_bo_type_kernel, flags, true); in xe_managed_bo_create_pin_map()
2855 ret = devm_add_action_or_reset(xe->drm.dev, __xe_bo_unpin_map_no_vm, bo); in xe_managed_bo_create_pin_map()
2867 struct xe_bo *xe_managed_bo_create_from_data(struct xe_device *xe, struct xe_tile *tile, in xe_managed_bo_create_from_data() argument
2870 struct xe_bo *bo = xe_managed_bo_create_pin_map(xe, tile, ALIGN(size, PAGE_SIZE), flags); in xe_managed_bo_create_from_data()
2875 xe_map_memcpy_to(xe, &bo->vmap, 0, data, size); in xe_managed_bo_create_from_data()
2882 * @xe: xe device
2891 * Returns 0 for success, negative error code otherwise.
2893 int xe_managed_bo_reinit_in_vram(struct xe_device *xe, struct xe_tile *tile, struct xe_bo **src) in xe_managed_bo_reinit_in_vram() argument
2901 xe_assert(xe, IS_DGFX(xe)); in xe_managed_bo_reinit_in_vram()
2902 xe_assert(xe, !(*src)->vmap.is_iomem); in xe_managed_bo_reinit_in_vram()
2904 bo = xe_managed_bo_create_from_data(xe, tile, (*src)->vmap.vaddr, in xe_managed_bo_reinit_in_vram()
2909 devm_release_action(xe->drm.dev, __xe_bo_unpin_map_no_vm, *src); in xe_managed_bo_reinit_in_vram()
2912 return 0; in xe_managed_bo_reinit_in_vram()
2921 struct xe_device *xe = ttm_to_xe_device(res->bo->bdev); in vram_region_gpu_offset() local
2925 return xe_ttm_stolen_gpu_offset(xe); in vram_region_gpu_offset()
2928 return 0; in vram_region_gpu_offset()
2932 return 0; in vram_region_gpu_offset()
2945 * Returns 0 for success, negative error code otherwise.
2949 struct xe_device *xe = xe_bo_device(bo); in xe_bo_pin_external() local
2952 xe_assert(xe, !bo->vm); in xe_bo_pin_external()
2953 xe_assert(xe, xe_bo_is_user(bo)); in xe_bo_pin_external()
2962 spin_lock(&xe->pinned.lock); in xe_bo_pin_external()
2963 list_add_tail(&bo->pinned_link, &xe->pinned.late.external); in xe_bo_pin_external()
2964 spin_unlock(&xe->pinned.lock); in xe_bo_pin_external()
2969 xe_ttm_tt_account_subtract(xe, bo->ttm.ttm); in xe_bo_pin_external()
2977 return 0; in xe_bo_pin_external()
2988 * Return: %0 on success, negative error code on migration failure.
2992 struct ttm_place *place = &bo->placements[0]; in xe_bo_pin()
2993 struct xe_device *xe = xe_bo_device(bo); in xe_bo_pin() local
2997 xe_assert(xe, !xe_bo_is_user(bo)); in xe_bo_pin()
3000 xe_assert(xe, bo->flags & (XE_BO_FLAG_PINNED | in xe_bo_pin()
3007 xe_assert(xe, !bo->ttm.base.import_attach); in xe_bo_pin()
3010 xe_assert(xe, !xe_bo_is_pinned(bo)); in xe_bo_pin()
3017 spin_lock(&xe->pinned.lock); in xe_bo_pin()
3019 list_add_tail(&bo->pinned_link, &xe->pinned.late.kernel_bo_present); in xe_bo_pin()
3021 list_add_tail(&bo->pinned_link, &xe->pinned.early.kernel_bo_present); in xe_bo_pin()
3022 spin_unlock(&xe->pinned.lock); in xe_bo_pin()
3027 xe_ttm_tt_account_subtract(xe, bo->ttm.ttm); in xe_bo_pin()
3035 return 0; in xe_bo_pin()
3046 * Returns 0 for success, negative error code otherwise.
3050 struct xe_device *xe = xe_bo_device(bo); in xe_bo_unpin_external() local
3052 xe_assert(xe, !bo->vm); in xe_bo_unpin_external()
3053 xe_assert(xe, xe_bo_is_pinned(bo)); in xe_bo_unpin_external()
3054 xe_assert(xe, xe_bo_is_user(bo)); in xe_bo_unpin_external()
3056 spin_lock(&xe->pinned.lock); in xe_bo_unpin_external()
3059 spin_unlock(&xe->pinned.lock); in xe_bo_unpin_external()
3063 xe_ttm_tt_account_add(xe, bo->ttm.ttm); in xe_bo_unpin_external()
3074 struct ttm_place *place = &bo->placements[0]; in xe_bo_unpin()
3075 struct xe_device *xe = xe_bo_device(bo); in xe_bo_unpin() local
3077 xe_assert(xe, !bo->ttm.base.import_attach); in xe_bo_unpin()
3078 xe_assert(xe, xe_bo_is_pinned(bo)); in xe_bo_unpin()
3081 spin_lock(&xe->pinned.lock); in xe_bo_unpin()
3082 xe_assert(xe, !list_empty(&bo->pinned_link)); in xe_bo_unpin()
3084 spin_unlock(&xe->pinned.lock); in xe_bo_unpin()
3095 xe_ttm_tt_account_add(xe, bo->ttm.ttm); in xe_bo_unpin()
3112 * Return: 0 on success, negative error code on failure. May return
3126 return 0; in xe_bo_validate()
3161 struct xe_device *xe = xe_bo_device(bo); in __xe_bo_addr() local
3165 xe_assert(xe, page_size <= PAGE_SIZE); in __xe_bo_addr()
3170 xe_assert(xe, bo->ttm.ttm); in __xe_bo_addr()
3193 struct xe_device *xe = ttm_to_xe_device(bo->ttm.bdev); in xe_bo_vmap() local
3200 if (drm_WARN_ON(&xe->drm, !(bo->flags & XE_BO_FLAG_NEEDS_CPU_ACCESS) || in xe_bo_vmap()
3205 return 0; in xe_bo_vmap()
3214 ret = ttm_bo_kmap(&bo->ttm, 0, xe_bo_size(bo) >> PAGE_SHIFT, &bo->kmap); in xe_bo_vmap()
3224 return 0; in xe_bo_vmap()
3241 static int gem_create_set_pxp_type(struct xe_device *xe, struct xe_bo *bo, u64 value) in gem_create_set_pxp_type() argument
3244 return 0; in gem_create_set_pxp_type()
3247 if (XE_IOCTL_DBG(xe, value != DRM_XE_PXP_TYPE_HWDRM)) in gem_create_set_pxp_type()
3250 return xe_pxp_key_assign(xe->pxp, bo); in gem_create_set_pxp_type()
3253 typedef int (*xe_gem_create_set_property_fn)(struct xe_device *xe,
3261 static int gem_create_user_ext_set_property(struct xe_device *xe, in gem_create_user_ext_set_property() argument
3271 if (XE_IOCTL_DBG(xe, err)) in gem_create_user_ext_set_property()
3274 if (XE_IOCTL_DBG(xe, ext.property >= in gem_create_user_ext_set_property()
3276 XE_IOCTL_DBG(xe, ext.pad) || in gem_create_user_ext_set_property()
3277 XE_IOCTL_DBG(xe, ext.property != DRM_XE_GEM_CREATE_EXTENSION_SET_PROPERTY)) in gem_create_user_ext_set_property()
3284 return gem_create_set_property_funcs[idx](xe, bo, ext.value); in gem_create_user_ext_set_property()
3287 typedef int (*xe_gem_create_user_extension_fn)(struct xe_device *xe,
3296 static int gem_create_user_extensions(struct xe_device *xe, struct xe_bo *bo, in gem_create_user_extensions() argument
3304 if (XE_IOCTL_DBG(xe, ext_number >= MAX_USER_EXTENSIONS)) in gem_create_user_extensions()
3308 if (XE_IOCTL_DBG(xe, err)) in gem_create_user_extensions()
3311 if (XE_IOCTL_DBG(xe, ext.pad) || in gem_create_user_extensions()
3312 XE_IOCTL_DBG(xe, ext.name >= ARRAY_SIZE(gem_create_user_extension_funcs))) in gem_create_user_extensions()
3317 err = gem_create_user_extension_funcs[idx](xe, bo, extensions); in gem_create_user_extensions()
3318 if (XE_IOCTL_DBG(xe, err)) in gem_create_user_extensions()
3322 return gem_create_user_extensions(xe, bo, ext.next_extension, in gem_create_user_extensions()
3325 return 0; in gem_create_user_extensions()
3331 struct xe_device *xe = to_xe_device(dev); in xe_gem_create_ioctl() local
3342 if (XE_IOCTL_DBG(xe, args->pad[0] || args->pad[1] || args->pad[2]) || in xe_gem_create_ioctl()
3343 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) in xe_gem_create_ioctl()
3347 if (XE_IOCTL_DBG(xe, (args->placement & ~xe->info.mem_region_mask) || in xe_gem_create_ioctl()
3351 if (XE_IOCTL_DBG(xe, args->flags & in xe_gem_create_ioctl()
3358 if (XE_IOCTL_DBG(xe, args->handle)) in xe_gem_create_ioctl()
3361 if (XE_IOCTL_DBG(xe, !args->size)) in xe_gem_create_ioctl()
3364 if (XE_IOCTL_DBG(xe, args->size > SIZE_MAX)) in xe_gem_create_ioctl()
3367 if (XE_IOCTL_DBG(xe, args->size & ~PAGE_MASK)) in xe_gem_create_ioctl()
3370 bo_flags = 0; in xe_gem_create_ioctl()
3381 if (XE_IOCTL_DBG(xe, GRAPHICS_VER(xe) < 20)) in xe_gem_create_ioctl()
3391 !(xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) && in xe_gem_create_ioctl()
3396 if (XE_IOCTL_DBG(xe, !(bo_flags & XE_BO_FLAG_VRAM_MASK))) in xe_gem_create_ioctl()
3402 if (XE_IOCTL_DBG(xe, !args->cpu_caching || in xe_gem_create_ioctl()
3406 if (XE_IOCTL_DBG(xe, bo_flags & XE_BO_FLAG_VRAM_MASK && in xe_gem_create_ioctl()
3410 if (XE_IOCTL_DBG(xe, bo_flags & XE_BO_FLAG_FORCE_WC && in xe_gem_create_ioctl()
3416 if (XE_IOCTL_DBG(xe, !vm)) in xe_gem_create_ioctl()
3420 err = 0; in xe_gem_create_ioctl()
3421 xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {.interruptible = true}, in xe_gem_create_ioctl()
3429 bo = xe_bo_create_user(xe, vm, args->size, args->cpu_caching, in xe_gem_create_ioctl()
3442 err = gem_create_user_extensions(xe, bo, args->extensions, 0); in xe_gem_create_ioctl()
3472 struct xe_device *xe = to_xe_device(dev); in xe_gem_mmap_offset_ioctl() local
3476 if (XE_IOCTL_DBG(xe, args->extensions) || in xe_gem_mmap_offset_ioctl()
3477 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) in xe_gem_mmap_offset_ioctl()
3480 if (XE_IOCTL_DBG(xe, args->flags & in xe_gem_mmap_offset_ioctl()
3485 if (XE_IOCTL_DBG(xe, !IS_DGFX(xe))) in xe_gem_mmap_offset_ioctl()
3488 if (XE_IOCTL_DBG(xe, args->handle)) in xe_gem_mmap_offset_ioctl()
3491 if (XE_IOCTL_DBG(xe, PAGE_SIZE > SZ_4K)) in xe_gem_mmap_offset_ioctl()
3497 return 0; in xe_gem_mmap_offset_ioctl()
3501 if (XE_IOCTL_DBG(xe, !gem_obj)) in xe_gem_mmap_offset_ioctl()
3508 return 0; in xe_gem_mmap_offset_ioctl()
3523 * Returns 0 on success, negative errno on error.
3527 struct xe_device *xe = xe_bo_device(bo); in xe_bo_decompress() local
3528 struct xe_tile *tile = xe_device_get_root_tile(xe); in xe_bo_decompress()
3535 int err = 0; in xe_bo_decompress()
3539 return 0; in xe_bo_decompress()
3563 return 0; in xe_bo_decompress()
3574 * Return: 0 on success, -EINTR if @intr is true and the wait for a
3576 * function always returns 0.
3585 return 0; in xe_bo_lock()
3624 for (cur_place = 0; cur_place < bo->placement.num_placement; in xe_bo_can_migrate()
3635 memset(place, 0, sizeof(*place)); in xe_place_from_ttm_type()
3654 * Return: 0 on success. Negative error code on failure. In particular may
3660 struct xe_device *xe = ttm_to_xe_device(bo->ttm.bdev); in xe_bo_migrate() local
3673 return 0; in xe_bo_migrate()
3689 drm_WARN_ON(&xe->drm, mem_type == XE_PL_STOLEN); in xe_bo_migrate()
3692 u32 c = 0; in xe_bo_migrate()
3694 add_vram(xe, bo, &requested, bo->flags, mem_type, &c); in xe_bo_migrate()
3710 * Return: 0 on success. Negative error code on failure.
3730 return 0; in xe_bo_evict()
3742 struct xe_device *xe = xe_bo_device(bo); in xe_bo_needs_ccs_pages() local
3744 if (GRAPHICS_VER(xe) >= 20 && IS_DGFX(xe)) in xe_bo_needs_ccs_pages()
3747 if (!xe_device_has_flat_ccs(xe) || bo->ttm.type != ttm_bo_type_device) in xe_bo_needs_ccs_pages()
3755 if (IS_DGFX(xe) && (bo->flags & XE_BO_FLAG_SYSTEM)) in xe_bo_needs_ccs_pages()
3767 xe->pat.idx[XE_CACHE_WB_COMPRESSION] == XE_PAT_INVALID_IDX) in xe_bo_needs_ccs_pages()
3864 struct xe_device *xe = to_xe_device(dev); in xe_bo_dumb_create() local
3869 xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? SZ_64K : SZ_4K); in xe_bo_dumb_create()
3875 bo = xe_bo_create_user(xe, NULL, args->size, in xe_bo_dumb_create()
3877 XE_BO_FLAG_VRAM_IF_DGFX(xe_device_get_root_tile(xe)) | in xe_bo_dumb_create()