1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_sa.h" 7 8 #include <linux/kernel.h> 9 10 #include <drm/drm_managed.h> 11 12 #include "xe_bo.h" 13 #include "xe_device_types.h" 14 #include "xe_map.h" 15 16 static void xe_sa_bo_manager_fini(struct drm_device *drm, void *arg) 17 { 18 struct xe_sa_manager *sa_manager = arg; 19 struct xe_bo *bo = sa_manager->bo; 20 21 if (!bo) { 22 drm_err(drm, "no bo for sa manager\n"); 23 return; 24 } 25 26 drm_suballoc_manager_fini(&sa_manager->base); 27 28 if (sa_manager->is_iomem) 29 kvfree(sa_manager->cpu_ptr); 30 31 sa_manager->bo = NULL; 32 sa_manager->shadow = NULL; 33 } 34 35 /** 36 * __xe_sa_bo_manager_init() - Create and initialize the suballocator 37 * @tile: the &xe_tile where allocate 38 * @size: number of bytes to allocate 39 * @guard: number of bytes to exclude from suballocations 40 * @align: alignment for each suballocated chunk 41 * @flags: flags for suballocator 42 * 43 * Prepares the suballocation manager for suballocations. 44 * 45 * Return: a pointer to the &xe_sa_manager or an ERR_PTR on failure. 46 */ 47 struct xe_sa_manager *__xe_sa_bo_manager_init(struct xe_tile *tile, u32 size, 48 u32 guard, u32 align, u32 flags) 49 { 50 struct xe_device *xe = tile_to_xe(tile); 51 struct xe_sa_manager *sa_manager; 52 u32 managed_size; 53 struct xe_bo *bo; 54 int ret; 55 56 xe_tile_assert(tile, size > guard); 57 managed_size = size - guard; 58 59 sa_manager = drmm_kzalloc(&xe->drm, sizeof(*sa_manager), GFP_KERNEL); 60 if (!sa_manager) 61 return ERR_PTR(-ENOMEM); 62 63 bo = xe_managed_bo_create_pin_map(xe, tile, size, 64 XE_BO_FLAG_VRAM_IF_DGFX(tile) | 65 XE_BO_FLAG_GGTT | 66 XE_BO_FLAG_GGTT_INVALIDATE | 67 XE_BO_FLAG_PINNED_NORESTORE); 68 if (IS_ERR(bo)) { 69 drm_err(&xe->drm, "Failed to prepare %uKiB BO for SA manager (%pe)\n", 70 size / SZ_1K, bo); 71 return ERR_CAST(bo); 72 } 73 sa_manager->bo = bo; 74 sa_manager->is_iomem = bo->vmap.is_iomem; 75 76 if (bo->vmap.is_iomem) { 77 sa_manager->cpu_ptr = kvzalloc(managed_size, GFP_KERNEL); 78 if (!sa_manager->cpu_ptr) 79 return ERR_PTR(-ENOMEM); 80 } else { 81 sa_manager->cpu_ptr = bo->vmap.vaddr; 82 memset(sa_manager->cpu_ptr, 0, bo->ttm.base.size); 83 } 84 85 if (flags & XE_SA_BO_MANAGER_FLAG_SHADOW) { 86 struct xe_bo *shadow; 87 88 ret = drmm_mutex_init(&xe->drm, &sa_manager->swap_guard); 89 if (ret) 90 return ERR_PTR(ret); 91 92 shadow = xe_managed_bo_create_pin_map(xe, tile, size, 93 XE_BO_FLAG_VRAM_IF_DGFX(tile) | 94 XE_BO_FLAG_GGTT | 95 XE_BO_FLAG_GGTT_INVALIDATE | 96 XE_BO_FLAG_PINNED_NORESTORE); 97 if (IS_ERR(shadow)) { 98 drm_err(&xe->drm, "Failed to prepare %uKiB BO for SA manager (%pe)\n", 99 size / SZ_1K, shadow); 100 return ERR_CAST(shadow); 101 } 102 sa_manager->shadow = shadow; 103 } 104 105 drm_suballoc_manager_init(&sa_manager->base, managed_size, align); 106 ret = drmm_add_action_or_reset(&xe->drm, xe_sa_bo_manager_fini, 107 sa_manager); 108 if (ret) 109 return ERR_PTR(ret); 110 111 return sa_manager; 112 } 113 114 /** 115 * xe_sa_bo_swap_shadow() - Swap the SA BO with shadow BO. 116 * @sa_manager: the XE sub allocator manager 117 * 118 * Swaps the sub-allocator primary buffer object with shadow buffer object. 119 * 120 * Return: None. 121 */ 122 void xe_sa_bo_swap_shadow(struct xe_sa_manager *sa_manager) 123 { 124 struct xe_device *xe = tile_to_xe(sa_manager->bo->tile); 125 126 xe_assert(xe, sa_manager->shadow); 127 lockdep_assert_held(&sa_manager->swap_guard); 128 129 swap(sa_manager->bo, sa_manager->shadow); 130 if (!sa_manager->bo->vmap.is_iomem) 131 sa_manager->cpu_ptr = sa_manager->bo->vmap.vaddr; 132 } 133 134 /** 135 * xe_sa_bo_sync_shadow() - Sync the SA Shadow BO with primary BO. 136 * @sa_bo: the sub-allocator buffer object. 137 * 138 * Synchronize sub-allocator shadow buffer object with primary buffer object. 139 * 140 * Return: None. 141 */ 142 void xe_sa_bo_sync_shadow(struct drm_suballoc *sa_bo) 143 { 144 struct xe_sa_manager *sa_manager = to_xe_sa_manager(sa_bo->manager); 145 struct xe_device *xe = tile_to_xe(sa_manager->bo->tile); 146 147 xe_assert(xe, sa_manager->shadow); 148 lockdep_assert_held(&sa_manager->swap_guard); 149 150 xe_map_memcpy_to(xe, &sa_manager->shadow->vmap, 151 drm_suballoc_soffset(sa_bo), 152 xe_sa_bo_cpu_addr(sa_bo), 153 drm_suballoc_size(sa_bo)); 154 } 155 156 /** 157 * __xe_sa_bo_new() - Make a suballocation but use custom gfp flags. 158 * @sa_manager: the &xe_sa_manager 159 * @size: number of bytes we want to suballocate 160 * @gfp: gfp flags used for memory allocation. Typically GFP_KERNEL. 161 * 162 * Try to make a suballocation of size @size. 163 * 164 * Return: a &drm_suballoc, or an ERR_PTR. 165 */ 166 struct drm_suballoc *__xe_sa_bo_new(struct xe_sa_manager *sa_manager, u32 size, gfp_t gfp) 167 { 168 /* 169 * BB to large, return -ENOBUFS indicating user should split 170 * array of binds into smaller chunks. 171 */ 172 if (size > sa_manager->base.size) 173 return ERR_PTR(-ENOBUFS); 174 175 return drm_suballoc_new(&sa_manager->base, size, gfp, true, 0); 176 } 177 178 /** 179 * xe_sa_bo_flush_write() - Copy the data from the sub-allocation to the GPU memory. 180 * @sa_bo: the &drm_suballoc to flush 181 */ 182 void xe_sa_bo_flush_write(struct drm_suballoc *sa_bo) 183 { 184 struct xe_sa_manager *sa_manager = to_xe_sa_manager(sa_bo->manager); 185 struct xe_device *xe = tile_to_xe(sa_manager->bo->tile); 186 187 if (!sa_manager->bo->vmap.is_iomem) 188 return; 189 190 xe_map_memcpy_to(xe, &sa_manager->bo->vmap, drm_suballoc_soffset(sa_bo), 191 xe_sa_bo_cpu_addr(sa_bo), 192 drm_suballoc_size(sa_bo)); 193 } 194 195 /** 196 * xe_sa_bo_sync_read() - Copy the data from GPU memory to the sub-allocation. 197 * @sa_bo: the &drm_suballoc to sync 198 */ 199 void xe_sa_bo_sync_read(struct drm_suballoc *sa_bo) 200 { 201 struct xe_sa_manager *sa_manager = to_xe_sa_manager(sa_bo->manager); 202 struct xe_device *xe = tile_to_xe(sa_manager->bo->tile); 203 204 if (!sa_manager->bo->vmap.is_iomem) 205 return; 206 207 xe_map_memcpy_from(xe, xe_sa_bo_cpu_addr(sa_bo), &sa_manager->bo->vmap, 208 drm_suballoc_soffset(sa_bo), 209 drm_suballoc_size(sa_bo)); 210 } 211 212 void xe_sa_bo_free(struct drm_suballoc *sa_bo, 213 struct dma_fence *fence) 214 { 215 drm_suballoc_free(sa_bo, fence); 216 } 217