1 /* SPDX-License-Identifier: MIT */ 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #ifndef _XE_VM_TYPES_H_ 7 #define _XE_VM_TYPES_H_ 8 9 #include <drm/drm_gpuvm.h> 10 11 #include <linux/dma-resv.h> 12 #include <linux/kref.h> 13 #include <linux/mmu_notifier.h> 14 #include <linux/scatterlist.h> 15 16 #include "xe_device_types.h" 17 #include "xe_pt_types.h" 18 #include "xe_range_fence.h" 19 20 struct xe_bo; 21 struct xe_sync_entry; 22 struct xe_user_fence; 23 struct xe_vm; 24 struct xe_vm_pgtable_update_op; 25 26 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 27 #define TEST_VM_OPS_ERROR 28 #define FORCE_OP_ERROR BIT(31) 29 30 #define FORCE_OP_ERROR_LOCK 0 31 #define FORCE_OP_ERROR_PREPARE 1 32 #define FORCE_OP_ERROR_RUN 2 33 #define FORCE_OP_ERROR_COUNT 3 34 #endif 35 36 #define XE_VMA_READ_ONLY DRM_GPUVA_USERBITS 37 #define XE_VMA_DESTROYED (DRM_GPUVA_USERBITS << 1) 38 #define XE_VMA_ATOMIC_PTE_BIT (DRM_GPUVA_USERBITS << 2) 39 #define XE_VMA_PTE_4K (DRM_GPUVA_USERBITS << 3) 40 #define XE_VMA_PTE_2M (DRM_GPUVA_USERBITS << 4) 41 #define XE_VMA_PTE_1G (DRM_GPUVA_USERBITS << 5) 42 #define XE_VMA_PTE_64K (DRM_GPUVA_USERBITS << 6) 43 #define XE_VMA_PTE_COMPACT (DRM_GPUVA_USERBITS << 7) 44 #define XE_VMA_DUMPABLE (DRM_GPUVA_USERBITS << 8) 45 46 /** struct xe_userptr - User pointer */ 47 struct xe_userptr { 48 /** @invalidate_link: Link for the vm::userptr.invalidated list */ 49 struct list_head invalidate_link; 50 /** @userptr: link into VM repin list if userptr. */ 51 struct list_head repin_link; 52 /** 53 * @notifier: MMU notifier for user pointer (invalidation call back) 54 */ 55 struct mmu_interval_notifier notifier; 56 /** @sgt: storage for a scatter gather table */ 57 struct sg_table sgt; 58 /** @sg: allocated scatter gather table */ 59 struct sg_table *sg; 60 /** @notifier_seq: notifier sequence number */ 61 unsigned long notifier_seq; 62 /** 63 * @initial_bind: user pointer has been bound at least once. 64 * write: vm->userptr.notifier_lock in read mode and vm->resv held. 65 * read: vm->userptr.notifier_lock in write mode or vm->resv held. 66 */ 67 bool initial_bind; 68 #if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT) 69 u32 divisor; 70 #endif 71 }; 72 73 struct xe_vma { 74 /** @gpuva: Base GPUVA object */ 75 struct drm_gpuva gpuva; 76 77 /** 78 * @combined_links: links into lists which are mutually exclusive. 79 * Locking: vm lock in write mode OR vm lock in read mode and the vm's 80 * resv. 81 */ 82 union { 83 /** @rebind: link into VM if this VMA needs rebinding. */ 84 struct list_head rebind; 85 /** @destroy: link to contested list when VM is being closed. */ 86 struct list_head destroy; 87 } combined_links; 88 89 union { 90 /** @destroy_cb: callback to destroy VMA when unbind job is done */ 91 struct dma_fence_cb destroy_cb; 92 /** @destroy_work: worker to destroy this BO */ 93 struct work_struct destroy_work; 94 }; 95 96 /** @tile_invalidated: VMA has been invalidated */ 97 u8 tile_invalidated; 98 99 /** @tile_mask: Tile mask of where to create binding for this VMA */ 100 u8 tile_mask; 101 102 /** 103 * @tile_present: GT mask of binding are present for this VMA. 104 * protected by vm->lock, vm->resv and for userptrs, 105 * vm->userptr.notifier_lock for writing. Needs either for reading, 106 * but if reading is done under the vm->lock only, it needs to be held 107 * in write mode. 108 */ 109 u8 tile_present; 110 111 /** @tile_staged: bind is staged for this VMA */ 112 u8 tile_staged; 113 114 /** 115 * @pat_index: The pat index to use when encoding the PTEs for this vma. 116 */ 117 u16 pat_index; 118 119 /** 120 * @ufence: The user fence that was provided with MAP. 121 * Needs to be signalled before UNMAP can be processed. 122 */ 123 struct xe_user_fence *ufence; 124 }; 125 126 /** 127 * struct xe_userptr_vma - A userptr vma subclass 128 * @vma: The vma. 129 * @userptr: Additional userptr information. 130 */ 131 struct xe_userptr_vma { 132 struct xe_vma vma; 133 struct xe_userptr userptr; 134 }; 135 136 struct xe_device; 137 138 struct xe_vm { 139 /** @gpuvm: base GPUVM used to track VMAs */ 140 struct drm_gpuvm gpuvm; 141 142 struct xe_device *xe; 143 144 /* exec queue used for (un)binding vma's */ 145 struct xe_exec_queue *q[XE_MAX_TILES_PER_DEVICE]; 146 147 /** @lru_bulk_move: Bulk LRU move list for this VM's BOs */ 148 struct ttm_lru_bulk_move lru_bulk_move; 149 150 u64 size; 151 152 struct xe_pt *pt_root[XE_MAX_TILES_PER_DEVICE]; 153 struct xe_pt *scratch_pt[XE_MAX_TILES_PER_DEVICE][XE_VM_MAX_LEVEL]; 154 155 /** 156 * @flags: flags for this VM, statically setup a creation time aside 157 * from XE_VM_FLAG_BANNED which requires vm->lock to set / read safely 158 */ 159 #define XE_VM_FLAG_64K BIT(0) 160 #define XE_VM_FLAG_LR_MODE BIT(1) 161 #define XE_VM_FLAG_MIGRATION BIT(2) 162 #define XE_VM_FLAG_SCRATCH_PAGE BIT(3) 163 #define XE_VM_FLAG_FAULT_MODE BIT(4) 164 #define XE_VM_FLAG_BANNED BIT(5) 165 #define XE_VM_FLAG_TILE_ID(flags) FIELD_GET(GENMASK(7, 6), flags) 166 #define XE_VM_FLAG_SET_TILE_ID(tile) FIELD_PREP(GENMASK(7, 6), (tile)->id) 167 unsigned long flags; 168 169 /** @composite_fence_ctx: context composite fence */ 170 u64 composite_fence_ctx; 171 /** @composite_fence_seqno: seqno for composite fence */ 172 u32 composite_fence_seqno; 173 174 /** 175 * @lock: outer most lock, protects objects of anything attached to this 176 * VM 177 */ 178 struct rw_semaphore lock; 179 /** 180 * @snap_mutex: Mutex used to guard insertions and removals from gpuva, 181 * so we can take a snapshot safely from devcoredump. 182 */ 183 struct mutex snap_mutex; 184 185 /** 186 * @rebind_list: list of VMAs that need rebinding. Protected by the 187 * vm->lock in write mode, OR (the vm->lock in read mode and the 188 * vm resv). 189 */ 190 struct list_head rebind_list; 191 192 /** 193 * @destroy_work: worker to destroy VM, needed as a dma_fence signaling 194 * from an irq context can be last put and the destroy needs to be able 195 * to sleep. 196 */ 197 struct work_struct destroy_work; 198 199 /** 200 * @rftree: range fence tree to track updates to page table structure. 201 * Used to implement conflict tracking between independent bind engines. 202 */ 203 struct xe_range_fence_tree rftree[XE_MAX_TILES_PER_DEVICE]; 204 205 const struct xe_pt_ops *pt_ops; 206 207 /** @userptr: user pointer state */ 208 struct { 209 /** 210 * @userptr.repin_list: list of VMAs which are user pointers, 211 * and needs repinning. Protected by @lock. 212 */ 213 struct list_head repin_list; 214 /** 215 * @notifier_lock: protects notifier in write mode and 216 * submission in read mode. 217 */ 218 struct rw_semaphore notifier_lock; 219 /** 220 * @userptr.invalidated_lock: Protects the 221 * @userptr.invalidated list. 222 */ 223 spinlock_t invalidated_lock; 224 /** 225 * @userptr.invalidated: List of invalidated userptrs, not yet 226 * picked 227 * up for revalidation. Protected from access with the 228 * @invalidated_lock. Removing items from the list 229 * additionally requires @lock in write mode, and adding 230 * items to the list requires the @userptr.notifer_lock in 231 * write mode. 232 */ 233 struct list_head invalidated; 234 } userptr; 235 236 /** @preempt: preempt state */ 237 struct { 238 /** 239 * @min_run_period_ms: The minimum run period before preempting 240 * an engine again 241 */ 242 s64 min_run_period_ms; 243 /** @exec_queues: list of exec queues attached to this VM */ 244 struct list_head exec_queues; 245 /** @num_exec_queues: number exec queues attached to this VM */ 246 int num_exec_queues; 247 /** 248 * @rebind_deactivated: Whether rebind has been temporarily deactivated 249 * due to no work available. Protected by the vm resv. 250 */ 251 bool rebind_deactivated; 252 /** 253 * @rebind_work: worker to rebind invalidated userptrs / evicted 254 * BOs 255 */ 256 struct work_struct rebind_work; 257 } preempt; 258 259 /** @um: unified memory state */ 260 struct { 261 /** @asid: address space ID, unique to each VM */ 262 u32 asid; 263 /** 264 * @last_fault_vma: Last fault VMA, used for fast lookup when we 265 * get a flood of faults to the same VMA 266 */ 267 struct xe_vma *last_fault_vma; 268 } usm; 269 270 /** @error_capture: allow to track errors */ 271 struct { 272 /** @capture_once: capture only one error per VM */ 273 bool capture_once; 274 } error_capture; 275 276 /** 277 * @tlb_flush_seqno: Required TLB flush seqno for the next exec. 278 * protected by the vm resv. 279 */ 280 u64 tlb_flush_seqno; 281 /** @batch_invalidate_tlb: Always invalidate TLB before batch start */ 282 bool batch_invalidate_tlb; 283 /** @xef: XE file handle for tracking this VM's drm client */ 284 struct xe_file *xef; 285 }; 286 287 /** struct xe_vma_op_map - VMA map operation */ 288 struct xe_vma_op_map { 289 /** @vma: VMA to map */ 290 struct xe_vma *vma; 291 /** @immediate: Immediate bind */ 292 bool immediate; 293 /** @read_only: Read only */ 294 bool read_only; 295 /** @is_null: is NULL binding */ 296 bool is_null; 297 /** @dumpable: whether BO is dumped on GPU hang */ 298 bool dumpable; 299 /** @pat_index: The pat index to use for this operation. */ 300 u16 pat_index; 301 }; 302 303 /** struct xe_vma_op_remap - VMA remap operation */ 304 struct xe_vma_op_remap { 305 /** @prev: VMA preceding part of a split mapping */ 306 struct xe_vma *prev; 307 /** @next: VMA subsequent part of a split mapping */ 308 struct xe_vma *next; 309 /** @start: start of the VMA unmap */ 310 u64 start; 311 /** @range: range of the VMA unmap */ 312 u64 range; 313 /** @skip_prev: skip prev rebind */ 314 bool skip_prev; 315 /** @skip_next: skip next rebind */ 316 bool skip_next; 317 /** @unmap_done: unmap operation in done */ 318 bool unmap_done; 319 }; 320 321 /** struct xe_vma_op_prefetch - VMA prefetch operation */ 322 struct xe_vma_op_prefetch { 323 /** @region: memory region to prefetch to */ 324 u32 region; 325 }; 326 327 /** enum xe_vma_op_flags - flags for VMA operation */ 328 enum xe_vma_op_flags { 329 /** @XE_VMA_OP_COMMITTED: VMA operation committed */ 330 XE_VMA_OP_COMMITTED = BIT(0), 331 /** @XE_VMA_OP_PREV_COMMITTED: Previous VMA operation committed */ 332 XE_VMA_OP_PREV_COMMITTED = BIT(1), 333 /** @XE_VMA_OP_NEXT_COMMITTED: Next VMA operation committed */ 334 XE_VMA_OP_NEXT_COMMITTED = BIT(2), 335 }; 336 337 /** struct xe_vma_op - VMA operation */ 338 struct xe_vma_op { 339 /** @base: GPUVA base operation */ 340 struct drm_gpuva_op base; 341 /** @link: async operation link */ 342 struct list_head link; 343 /** @flags: operation flags */ 344 enum xe_vma_op_flags flags; 345 /** @tile_mask: Tile mask for operation */ 346 u8 tile_mask; 347 348 union { 349 /** @map: VMA map operation specific data */ 350 struct xe_vma_op_map map; 351 /** @remap: VMA remap operation specific data */ 352 struct xe_vma_op_remap remap; 353 /** @prefetch: VMA prefetch operation specific data */ 354 struct xe_vma_op_prefetch prefetch; 355 }; 356 }; 357 358 /** struct xe_vma_ops - VMA operations */ 359 struct xe_vma_ops { 360 /** @list: list of VMA operations */ 361 struct list_head list; 362 /** @vm: VM */ 363 struct xe_vm *vm; 364 /** @q: exec queue for VMA operations */ 365 struct xe_exec_queue *q; 366 /** @syncs: syncs these operation */ 367 struct xe_sync_entry *syncs; 368 /** @num_syncs: number of syncs */ 369 u32 num_syncs; 370 /** @pt_update_ops: page table update operations */ 371 struct xe_vm_pgtable_update_ops pt_update_ops[XE_MAX_TILES_PER_DEVICE]; 372 #ifdef TEST_VM_OPS_ERROR 373 /** @inject_error: inject error to test error handling */ 374 bool inject_error; 375 #endif 376 }; 377 378 #endif 379