1 /* SPDX-License-Identifier: MIT */ 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #ifndef _XE_VM_TYPES_H_ 7 #define _XE_VM_TYPES_H_ 8 9 #include <drm/drm_gpuvm.h> 10 11 #include <linux/dma-resv.h> 12 #include <linux/kref.h> 13 #include <linux/mmu_notifier.h> 14 #include <linux/scatterlist.h> 15 16 #include "xe_device_types.h" 17 #include "xe_pt_types.h" 18 #include "xe_range_fence.h" 19 20 struct xe_bo; 21 struct xe_sync_entry; 22 struct xe_user_fence; 23 struct xe_vm; 24 struct xe_vm_pgtable_update_op; 25 26 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 27 #define TEST_VM_OPS_ERROR 28 #define FORCE_OP_ERROR BIT(31) 29 30 #define FORCE_OP_ERROR_LOCK 0 31 #define FORCE_OP_ERROR_PREPARE 1 32 #define FORCE_OP_ERROR_RUN 2 33 #define FORCE_OP_ERROR_COUNT 3 34 #endif 35 36 #define XE_VMA_READ_ONLY DRM_GPUVA_USERBITS 37 #define XE_VMA_DESTROYED (DRM_GPUVA_USERBITS << 1) 38 #define XE_VMA_ATOMIC_PTE_BIT (DRM_GPUVA_USERBITS << 2) 39 #define XE_VMA_PTE_4K (DRM_GPUVA_USERBITS << 3) 40 #define XE_VMA_PTE_2M (DRM_GPUVA_USERBITS << 4) 41 #define XE_VMA_PTE_1G (DRM_GPUVA_USERBITS << 5) 42 #define XE_VMA_PTE_64K (DRM_GPUVA_USERBITS << 6) 43 #define XE_VMA_PTE_COMPACT (DRM_GPUVA_USERBITS << 7) 44 #define XE_VMA_DUMPABLE (DRM_GPUVA_USERBITS << 8) 45 46 /** struct xe_userptr - User pointer */ 47 struct xe_userptr { 48 /** @invalidate_link: Link for the vm::userptr.invalidated list */ 49 struct list_head invalidate_link; 50 /** @userptr: link into VM repin list if userptr. */ 51 struct list_head repin_link; 52 /** 53 * @notifier: MMU notifier for user pointer (invalidation call back) 54 */ 55 struct mmu_interval_notifier notifier; 56 /** @sgt: storage for a scatter gather table */ 57 struct sg_table sgt; 58 /** @sg: allocated scatter gather table */ 59 struct sg_table *sg; 60 /** @notifier_seq: notifier sequence number */ 61 unsigned long notifier_seq; 62 /** @unmap_mutex: Mutex protecting dma-unmapping */ 63 struct mutex unmap_mutex; 64 /** 65 * @initial_bind: user pointer has been bound at least once. 66 * write: vm->userptr.notifier_lock in read mode and vm->resv held. 67 * read: vm->userptr.notifier_lock in write mode or vm->resv held. 68 */ 69 bool initial_bind; 70 /** @mapped: Whether the @sgt sg-table is dma-mapped. Protected by @unmap_mutex. */ 71 bool mapped; 72 #if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT) 73 u32 divisor; 74 #endif 75 }; 76 77 struct xe_vma { 78 /** @gpuva: Base GPUVA object */ 79 struct drm_gpuva gpuva; 80 81 /** 82 * @combined_links: links into lists which are mutually exclusive. 83 * Locking: vm lock in write mode OR vm lock in read mode and the vm's 84 * resv. 85 */ 86 union { 87 /** @rebind: link into VM if this VMA needs rebinding. */ 88 struct list_head rebind; 89 /** @destroy: link to contested list when VM is being closed. */ 90 struct list_head destroy; 91 } combined_links; 92 93 union { 94 /** @destroy_cb: callback to destroy VMA when unbind job is done */ 95 struct dma_fence_cb destroy_cb; 96 /** @destroy_work: worker to destroy this BO */ 97 struct work_struct destroy_work; 98 }; 99 100 /** @tile_invalidated: VMA has been invalidated */ 101 u8 tile_invalidated; 102 103 /** @tile_mask: Tile mask of where to create binding for this VMA */ 104 u8 tile_mask; 105 106 /** 107 * @tile_present: GT mask of binding are present for this VMA. 108 * protected by vm->lock, vm->resv and for userptrs, 109 * vm->userptr.notifier_lock for writing. Needs either for reading, 110 * but if reading is done under the vm->lock only, it needs to be held 111 * in write mode. 112 */ 113 u8 tile_present; 114 115 /** @tile_staged: bind is staged for this VMA */ 116 u8 tile_staged; 117 118 /** 119 * @pat_index: The pat index to use when encoding the PTEs for this vma. 120 */ 121 u16 pat_index; 122 123 /** 124 * @ufence: The user fence that was provided with MAP. 125 * Needs to be signalled before UNMAP can be processed. 126 */ 127 struct xe_user_fence *ufence; 128 }; 129 130 /** 131 * struct xe_userptr_vma - A userptr vma subclass 132 * @vma: The vma. 133 * @userptr: Additional userptr information. 134 */ 135 struct xe_userptr_vma { 136 struct xe_vma vma; 137 struct xe_userptr userptr; 138 }; 139 140 struct xe_device; 141 142 struct xe_vm { 143 /** @gpuvm: base GPUVM used to track VMAs */ 144 struct drm_gpuvm gpuvm; 145 146 struct xe_device *xe; 147 148 /* exec queue used for (un)binding vma's */ 149 struct xe_exec_queue *q[XE_MAX_TILES_PER_DEVICE]; 150 151 /** @lru_bulk_move: Bulk LRU move list for this VM's BOs */ 152 struct ttm_lru_bulk_move lru_bulk_move; 153 154 u64 size; 155 156 struct xe_pt *pt_root[XE_MAX_TILES_PER_DEVICE]; 157 struct xe_pt *scratch_pt[XE_MAX_TILES_PER_DEVICE][XE_VM_MAX_LEVEL]; 158 159 /** 160 * @flags: flags for this VM, statically setup a creation time aside 161 * from XE_VM_FLAG_BANNED which requires vm->lock to set / read safely 162 */ 163 #define XE_VM_FLAG_64K BIT(0) 164 #define XE_VM_FLAG_LR_MODE BIT(1) 165 #define XE_VM_FLAG_MIGRATION BIT(2) 166 #define XE_VM_FLAG_SCRATCH_PAGE BIT(3) 167 #define XE_VM_FLAG_FAULT_MODE BIT(4) 168 #define XE_VM_FLAG_BANNED BIT(5) 169 #define XE_VM_FLAG_TILE_ID(flags) FIELD_GET(GENMASK(7, 6), flags) 170 #define XE_VM_FLAG_SET_TILE_ID(tile) FIELD_PREP(GENMASK(7, 6), (tile)->id) 171 unsigned long flags; 172 173 /** @composite_fence_ctx: context composite fence */ 174 u64 composite_fence_ctx; 175 /** @composite_fence_seqno: seqno for composite fence */ 176 u32 composite_fence_seqno; 177 178 /** 179 * @lock: outer most lock, protects objects of anything attached to this 180 * VM 181 */ 182 struct rw_semaphore lock; 183 /** 184 * @snap_mutex: Mutex used to guard insertions and removals from gpuva, 185 * so we can take a snapshot safely from devcoredump. 186 */ 187 struct mutex snap_mutex; 188 189 /** 190 * @rebind_list: list of VMAs that need rebinding. Protected by the 191 * vm->lock in write mode, OR (the vm->lock in read mode and the 192 * vm resv). 193 */ 194 struct list_head rebind_list; 195 196 /** 197 * @destroy_work: worker to destroy VM, needed as a dma_fence signaling 198 * from an irq context can be last put and the destroy needs to be able 199 * to sleep. 200 */ 201 struct work_struct destroy_work; 202 203 /** 204 * @rftree: range fence tree to track updates to page table structure. 205 * Used to implement conflict tracking between independent bind engines. 206 */ 207 struct xe_range_fence_tree rftree[XE_MAX_TILES_PER_DEVICE]; 208 209 const struct xe_pt_ops *pt_ops; 210 211 /** @userptr: user pointer state */ 212 struct { 213 /** 214 * @userptr.repin_list: list of VMAs which are user pointers, 215 * and needs repinning. Protected by @lock. 216 */ 217 struct list_head repin_list; 218 /** 219 * @notifier_lock: protects notifier in write mode and 220 * submission in read mode. 221 */ 222 struct rw_semaphore notifier_lock; 223 /** 224 * @userptr.invalidated_lock: Protects the 225 * @userptr.invalidated list. 226 */ 227 spinlock_t invalidated_lock; 228 /** 229 * @userptr.invalidated: List of invalidated userptrs, not yet 230 * picked 231 * up for revalidation. Protected from access with the 232 * @invalidated_lock. Removing items from the list 233 * additionally requires @lock in write mode, and adding 234 * items to the list requires either the @userptr.notifer_lock in 235 * write mode, OR @lock in write mode. 236 */ 237 struct list_head invalidated; 238 } userptr; 239 240 /** @preempt: preempt state */ 241 struct { 242 /** 243 * @min_run_period_ms: The minimum run period before preempting 244 * an engine again 245 */ 246 s64 min_run_period_ms; 247 /** @exec_queues: list of exec queues attached to this VM */ 248 struct list_head exec_queues; 249 /** @num_exec_queues: number exec queues attached to this VM */ 250 int num_exec_queues; 251 /** 252 * @rebind_deactivated: Whether rebind has been temporarily deactivated 253 * due to no work available. Protected by the vm resv. 254 */ 255 bool rebind_deactivated; 256 /** 257 * @rebind_work: worker to rebind invalidated userptrs / evicted 258 * BOs 259 */ 260 struct work_struct rebind_work; 261 } preempt; 262 263 /** @um: unified memory state */ 264 struct { 265 /** @asid: address space ID, unique to each VM */ 266 u32 asid; 267 /** 268 * @last_fault_vma: Last fault VMA, used for fast lookup when we 269 * get a flood of faults to the same VMA 270 */ 271 struct xe_vma *last_fault_vma; 272 } usm; 273 274 /** @error_capture: allow to track errors */ 275 struct { 276 /** @capture_once: capture only one error per VM */ 277 bool capture_once; 278 } error_capture; 279 280 /** 281 * @tlb_flush_seqno: Required TLB flush seqno for the next exec. 282 * protected by the vm resv. 283 */ 284 u64 tlb_flush_seqno; 285 /** @batch_invalidate_tlb: Always invalidate TLB before batch start */ 286 bool batch_invalidate_tlb; 287 /** @xef: XE file handle for tracking this VM's drm client */ 288 struct xe_file *xef; 289 }; 290 291 /** struct xe_vma_op_map - VMA map operation */ 292 struct xe_vma_op_map { 293 /** @vma: VMA to map */ 294 struct xe_vma *vma; 295 /** @immediate: Immediate bind */ 296 bool immediate; 297 /** @read_only: Read only */ 298 bool read_only; 299 /** @is_null: is NULL binding */ 300 bool is_null; 301 /** @dumpable: whether BO is dumped on GPU hang */ 302 bool dumpable; 303 /** @pat_index: The pat index to use for this operation. */ 304 u16 pat_index; 305 }; 306 307 /** struct xe_vma_op_remap - VMA remap operation */ 308 struct xe_vma_op_remap { 309 /** @prev: VMA preceding part of a split mapping */ 310 struct xe_vma *prev; 311 /** @next: VMA subsequent part of a split mapping */ 312 struct xe_vma *next; 313 /** @start: start of the VMA unmap */ 314 u64 start; 315 /** @range: range of the VMA unmap */ 316 u64 range; 317 /** @skip_prev: skip prev rebind */ 318 bool skip_prev; 319 /** @skip_next: skip next rebind */ 320 bool skip_next; 321 /** @unmap_done: unmap operation in done */ 322 bool unmap_done; 323 }; 324 325 /** struct xe_vma_op_prefetch - VMA prefetch operation */ 326 struct xe_vma_op_prefetch { 327 /** @region: memory region to prefetch to */ 328 u32 region; 329 }; 330 331 /** enum xe_vma_op_flags - flags for VMA operation */ 332 enum xe_vma_op_flags { 333 /** @XE_VMA_OP_COMMITTED: VMA operation committed */ 334 XE_VMA_OP_COMMITTED = BIT(0), 335 /** @XE_VMA_OP_PREV_COMMITTED: Previous VMA operation committed */ 336 XE_VMA_OP_PREV_COMMITTED = BIT(1), 337 /** @XE_VMA_OP_NEXT_COMMITTED: Next VMA operation committed */ 338 XE_VMA_OP_NEXT_COMMITTED = BIT(2), 339 }; 340 341 /** struct xe_vma_op - VMA operation */ 342 struct xe_vma_op { 343 /** @base: GPUVA base operation */ 344 struct drm_gpuva_op base; 345 /** @link: async operation link */ 346 struct list_head link; 347 /** @flags: operation flags */ 348 enum xe_vma_op_flags flags; 349 /** @tile_mask: Tile mask for operation */ 350 u8 tile_mask; 351 352 union { 353 /** @map: VMA map operation specific data */ 354 struct xe_vma_op_map map; 355 /** @remap: VMA remap operation specific data */ 356 struct xe_vma_op_remap remap; 357 /** @prefetch: VMA prefetch operation specific data */ 358 struct xe_vma_op_prefetch prefetch; 359 }; 360 }; 361 362 /** struct xe_vma_ops - VMA operations */ 363 struct xe_vma_ops { 364 /** @list: list of VMA operations */ 365 struct list_head list; 366 /** @vm: VM */ 367 struct xe_vm *vm; 368 /** @q: exec queue for VMA operations */ 369 struct xe_exec_queue *q; 370 /** @syncs: syncs these operation */ 371 struct xe_sync_entry *syncs; 372 /** @num_syncs: number of syncs */ 373 u32 num_syncs; 374 /** @pt_update_ops: page table update operations */ 375 struct xe_vm_pgtable_update_ops pt_update_ops[XE_MAX_TILES_PER_DEVICE]; 376 #ifdef TEST_VM_OPS_ERROR 377 /** @inject_error: inject error to test error handling */ 378 bool inject_error; 379 #endif 380 }; 381 382 #endif 383