1 /* SPDX-License-Identifier: MIT */ 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #ifndef _XE_VM_TYPES_H_ 7 #define _XE_VM_TYPES_H_ 8 9 #include <drm/drm_gpusvm.h> 10 #include <drm/drm_gpuvm.h> 11 12 #include <linux/dma-resv.h> 13 #include <linux/kref.h> 14 #include <linux/mmu_notifier.h> 15 #include <linux/scatterlist.h> 16 17 #include "xe_device_types.h" 18 #include "xe_pt_types.h" 19 #include "xe_range_fence.h" 20 21 struct xe_bo; 22 struct xe_svm_range; 23 struct xe_sync_entry; 24 struct xe_user_fence; 25 struct xe_vm; 26 struct xe_vm_pgtable_update_op; 27 28 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 29 #define TEST_VM_OPS_ERROR 30 #define FORCE_OP_ERROR BIT(31) 31 32 #define FORCE_OP_ERROR_LOCK 0 33 #define FORCE_OP_ERROR_PREPARE 1 34 #define FORCE_OP_ERROR_RUN 2 35 #define FORCE_OP_ERROR_COUNT 3 36 #endif 37 38 #define XE_VMA_READ_ONLY DRM_GPUVA_USERBITS 39 #define XE_VMA_DESTROYED (DRM_GPUVA_USERBITS << 1) 40 #define XE_VMA_ATOMIC_PTE_BIT (DRM_GPUVA_USERBITS << 2) 41 #define XE_VMA_PTE_4K (DRM_GPUVA_USERBITS << 3) 42 #define XE_VMA_PTE_2M (DRM_GPUVA_USERBITS << 4) 43 #define XE_VMA_PTE_1G (DRM_GPUVA_USERBITS << 5) 44 #define XE_VMA_PTE_64K (DRM_GPUVA_USERBITS << 6) 45 #define XE_VMA_PTE_COMPACT (DRM_GPUVA_USERBITS << 7) 46 #define XE_VMA_DUMPABLE (DRM_GPUVA_USERBITS << 8) 47 #define XE_VMA_SYSTEM_ALLOCATOR (DRM_GPUVA_USERBITS << 9) 48 49 /** struct xe_userptr - User pointer */ 50 struct xe_userptr { 51 /** @invalidate_link: Link for the vm::userptr.invalidated list */ 52 struct list_head invalidate_link; 53 /** @userptr: link into VM repin list if userptr. */ 54 struct list_head repin_link; 55 /** 56 * @notifier: MMU notifier for user pointer (invalidation call back) 57 */ 58 struct mmu_interval_notifier notifier; 59 /** @sgt: storage for a scatter gather table */ 60 struct sg_table sgt; 61 /** @sg: allocated scatter gather table */ 62 struct sg_table *sg; 63 /** @notifier_seq: notifier sequence number */ 64 unsigned long notifier_seq; 65 /** @unmap_mutex: Mutex protecting dma-unmapping */ 66 struct mutex unmap_mutex; 67 /** 68 * @initial_bind: user pointer has been bound at least once. 69 * write: vm->userptr.notifier_lock in read mode and vm->resv held. 70 * read: vm->userptr.notifier_lock in write mode or vm->resv held. 71 */ 72 bool initial_bind; 73 /** @mapped: Whether the @sgt sg-table is dma-mapped. Protected by @unmap_mutex. */ 74 bool mapped; 75 #if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT) 76 u32 divisor; 77 #endif 78 }; 79 80 struct xe_vma { 81 /** @gpuva: Base GPUVA object */ 82 struct drm_gpuva gpuva; 83 84 /** 85 * @combined_links: links into lists which are mutually exclusive. 86 * Locking: vm lock in write mode OR vm lock in read mode and the vm's 87 * resv. 88 */ 89 union { 90 /** @rebind: link into VM if this VMA needs rebinding. */ 91 struct list_head rebind; 92 /** @destroy: link to contested list when VM is being closed. */ 93 struct list_head destroy; 94 } combined_links; 95 96 union { 97 /** @destroy_cb: callback to destroy VMA when unbind job is done */ 98 struct dma_fence_cb destroy_cb; 99 /** @destroy_work: worker to destroy this BO */ 100 struct work_struct destroy_work; 101 }; 102 103 /** 104 * @tile_invalidated: Tile mask of binding are invalidated for this VMA. 105 * protected by BO's resv and for userptrs, vm->userptr.notifier_lock in 106 * write mode for writing or vm->userptr.notifier_lock in read mode and 107 * the vm->resv. For stable reading, BO's resv or userptr 108 * vm->userptr.notifier_lock in read mode is required. Can be 109 * opportunistically read with READ_ONCE outside of locks. 110 */ 111 u8 tile_invalidated; 112 113 /** @tile_mask: Tile mask of where to create binding for this VMA */ 114 u8 tile_mask; 115 116 /** 117 * @tile_present: Tile mask of binding are present for this VMA. 118 * protected by vm->lock, vm->resv and for userptrs, 119 * vm->userptr.notifier_lock for writing. Needs either for reading, 120 * but if reading is done under the vm->lock only, it needs to be held 121 * in write mode. 122 */ 123 u8 tile_present; 124 125 /** @tile_staged: bind is staged for this VMA */ 126 u8 tile_staged; 127 128 /** 129 * @pat_index: The pat index to use when encoding the PTEs for this vma. 130 */ 131 u16 pat_index; 132 133 /** 134 * @ufence: The user fence that was provided with MAP. 135 * Needs to be signalled before UNMAP can be processed. 136 */ 137 struct xe_user_fence *ufence; 138 }; 139 140 /** 141 * struct xe_userptr_vma - A userptr vma subclass 142 * @vma: The vma. 143 * @userptr: Additional userptr information. 144 */ 145 struct xe_userptr_vma { 146 struct xe_vma vma; 147 struct xe_userptr userptr; 148 }; 149 150 struct xe_device; 151 152 struct xe_vm { 153 /** @gpuvm: base GPUVM used to track VMAs */ 154 struct drm_gpuvm gpuvm; 155 156 /** @svm: Shared virtual memory state */ 157 struct { 158 /** @svm.gpusvm: base GPUSVM used to track fault allocations */ 159 struct drm_gpusvm gpusvm; 160 /** 161 * @svm.garbage_collector: Garbage collector which is used unmap 162 * SVM range's GPU bindings and destroy the ranges. 163 */ 164 struct { 165 /** @svm.garbage_collector.lock: Protect's range list */ 166 spinlock_t lock; 167 /** 168 * @svm.garbage_collector.range_list: List of SVM ranges 169 * in the garbage collector. 170 */ 171 struct list_head range_list; 172 /** 173 * @svm.garbage_collector.work: Worker which the 174 * garbage collector runs on. 175 */ 176 struct work_struct work; 177 } garbage_collector; 178 } svm; 179 180 struct xe_device *xe; 181 182 /* exec queue used for (un)binding vma's */ 183 struct xe_exec_queue *q[XE_MAX_TILES_PER_DEVICE]; 184 185 /** @lru_bulk_move: Bulk LRU move list for this VM's BOs */ 186 struct ttm_lru_bulk_move lru_bulk_move; 187 188 u64 size; 189 190 struct xe_pt *pt_root[XE_MAX_TILES_PER_DEVICE]; 191 struct xe_pt *scratch_pt[XE_MAX_TILES_PER_DEVICE][XE_VM_MAX_LEVEL]; 192 193 /** 194 * @flags: flags for this VM, statically setup a creation time aside 195 * from XE_VM_FLAG_BANNED which requires vm->lock to set / read safely 196 */ 197 #define XE_VM_FLAG_64K BIT(0) 198 #define XE_VM_FLAG_LR_MODE BIT(1) 199 #define XE_VM_FLAG_MIGRATION BIT(2) 200 #define XE_VM_FLAG_SCRATCH_PAGE BIT(3) 201 #define XE_VM_FLAG_FAULT_MODE BIT(4) 202 #define XE_VM_FLAG_BANNED BIT(5) 203 #define XE_VM_FLAG_TILE_ID(flags) FIELD_GET(GENMASK(7, 6), flags) 204 #define XE_VM_FLAG_SET_TILE_ID(tile) FIELD_PREP(GENMASK(7, 6), (tile)->id) 205 #define XE_VM_FLAG_GSC BIT(8) 206 unsigned long flags; 207 208 /** @composite_fence_ctx: context composite fence */ 209 u64 composite_fence_ctx; 210 /** @composite_fence_seqno: seqno for composite fence */ 211 u32 composite_fence_seqno; 212 213 /** 214 * @lock: outer most lock, protects objects of anything attached to this 215 * VM 216 */ 217 struct rw_semaphore lock; 218 /** 219 * @snap_mutex: Mutex used to guard insertions and removals from gpuva, 220 * so we can take a snapshot safely from devcoredump. 221 */ 222 struct mutex snap_mutex; 223 224 /** 225 * @rebind_list: list of VMAs that need rebinding. Protected by the 226 * vm->lock in write mode, OR (the vm->lock in read mode and the 227 * vm resv). 228 */ 229 struct list_head rebind_list; 230 231 /** 232 * @destroy_work: worker to destroy VM, needed as a dma_fence signaling 233 * from an irq context can be last put and the destroy needs to be able 234 * to sleep. 235 */ 236 struct work_struct destroy_work; 237 238 /** 239 * @rftree: range fence tree to track updates to page table structure. 240 * Used to implement conflict tracking between independent bind engines. 241 */ 242 struct xe_range_fence_tree rftree[XE_MAX_TILES_PER_DEVICE]; 243 244 const struct xe_pt_ops *pt_ops; 245 246 /** @userptr: user pointer state */ 247 struct { 248 /** 249 * @userptr.repin_list: list of VMAs which are user pointers, 250 * and needs repinning. Protected by @lock. 251 */ 252 struct list_head repin_list; 253 /** 254 * @notifier_lock: protects notifier in write mode and 255 * submission in read mode. 256 */ 257 struct rw_semaphore notifier_lock; 258 /** 259 * @userptr.invalidated_lock: Protects the 260 * @userptr.invalidated list. 261 */ 262 spinlock_t invalidated_lock; 263 /** 264 * @userptr.invalidated: List of invalidated userptrs, not yet 265 * picked 266 * up for revalidation. Protected from access with the 267 * @invalidated_lock. Removing items from the list 268 * additionally requires @lock in write mode, and adding 269 * items to the list requires either the @userptr.notifier_lock in 270 * write mode, OR @lock in write mode. 271 */ 272 struct list_head invalidated; 273 } userptr; 274 275 /** @preempt: preempt state */ 276 struct { 277 /** 278 * @min_run_period_ms: The minimum run period before preempting 279 * an engine again 280 */ 281 s64 min_run_period_ms; 282 /** @exec_queues: list of exec queues attached to this VM */ 283 struct list_head exec_queues; 284 /** @num_exec_queues: number exec queues attached to this VM */ 285 int num_exec_queues; 286 /** 287 * @rebind_deactivated: Whether rebind has been temporarily deactivated 288 * due to no work available. Protected by the vm resv. 289 */ 290 bool rebind_deactivated; 291 /** 292 * @rebind_work: worker to rebind invalidated userptrs / evicted 293 * BOs 294 */ 295 struct work_struct rebind_work; 296 /** 297 * @preempt.pm_activate_link: Link to list of rebind workers to be 298 * kicked on resume. 299 */ 300 struct list_head pm_activate_link; 301 } preempt; 302 303 /** @um: unified memory state */ 304 struct { 305 /** @asid: address space ID, unique to each VM */ 306 u32 asid; 307 /** 308 * @last_fault_vma: Last fault VMA, used for fast lookup when we 309 * get a flood of faults to the same VMA 310 */ 311 struct xe_vma *last_fault_vma; 312 } usm; 313 314 /** @error_capture: allow to track errors */ 315 struct { 316 /** @capture_once: capture only one error per VM */ 317 bool capture_once; 318 } error_capture; 319 320 /** 321 * @tlb_flush_seqno: Required TLB flush seqno for the next exec. 322 * protected by the vm resv. 323 */ 324 u64 tlb_flush_seqno; 325 /** 326 * @validating: The task that is currently making bos resident for this vm. 327 * Protected by the VM's resv for writing. Opportunistic reading can be done 328 * using READ_ONCE. Note: This is a workaround for the 329 * TTM eviction_valuable() callback not being passed a struct 330 * ttm_operation_context(). Future work might want to address this. 331 */ 332 struct task_struct *validating; 333 /** @batch_invalidate_tlb: Always invalidate TLB before batch start */ 334 bool batch_invalidate_tlb; 335 /** @xef: XE file handle for tracking this VM's drm client */ 336 struct xe_file *xef; 337 }; 338 339 /** struct xe_vma_op_map - VMA map operation */ 340 struct xe_vma_op_map { 341 /** @vma: VMA to map */ 342 struct xe_vma *vma; 343 /** @immediate: Immediate bind */ 344 bool immediate; 345 /** @read_only: Read only */ 346 bool read_only; 347 /** @is_null: is NULL binding */ 348 bool is_null; 349 /** @is_cpu_addr_mirror: is CPU address mirror binding */ 350 bool is_cpu_addr_mirror; 351 /** @dumpable: whether BO is dumped on GPU hang */ 352 bool dumpable; 353 /** @invalidate: invalidate the VMA before bind */ 354 bool invalidate_on_bind; 355 /** @pat_index: The pat index to use for this operation. */ 356 u16 pat_index; 357 }; 358 359 /** struct xe_vma_op_remap - VMA remap operation */ 360 struct xe_vma_op_remap { 361 /** @prev: VMA preceding part of a split mapping */ 362 struct xe_vma *prev; 363 /** @next: VMA subsequent part of a split mapping */ 364 struct xe_vma *next; 365 /** @start: start of the VMA unmap */ 366 u64 start; 367 /** @range: range of the VMA unmap */ 368 u64 range; 369 /** @skip_prev: skip prev rebind */ 370 bool skip_prev; 371 /** @skip_next: skip next rebind */ 372 bool skip_next; 373 /** @unmap_done: unmap operation in done */ 374 bool unmap_done; 375 }; 376 377 /** struct xe_vma_op_prefetch - VMA prefetch operation */ 378 struct xe_vma_op_prefetch { 379 /** @region: memory region to prefetch to */ 380 u32 region; 381 }; 382 383 /** struct xe_vma_op_map_range - VMA map range operation */ 384 struct xe_vma_op_map_range { 385 /** @vma: VMA to map (system allocator VMA) */ 386 struct xe_vma *vma; 387 /** @range: SVM range to map */ 388 struct xe_svm_range *range; 389 }; 390 391 /** struct xe_vma_op_unmap_range - VMA unmap range operation */ 392 struct xe_vma_op_unmap_range { 393 /** @range: SVM range to unmap */ 394 struct xe_svm_range *range; 395 }; 396 397 /** struct xe_vma_op_prefetch_range - VMA prefetch range operation */ 398 struct xe_vma_op_prefetch_range { 399 /** @range: xarray for SVM ranges data */ 400 struct xarray range; 401 /** @ranges_count: number of svm ranges to map */ 402 u32 ranges_count; 403 /** @region: memory region to prefetch to */ 404 u32 region; 405 }; 406 407 /** enum xe_vma_op_flags - flags for VMA operation */ 408 enum xe_vma_op_flags { 409 /** @XE_VMA_OP_COMMITTED: VMA operation committed */ 410 XE_VMA_OP_COMMITTED = BIT(0), 411 /** @XE_VMA_OP_PREV_COMMITTED: Previous VMA operation committed */ 412 XE_VMA_OP_PREV_COMMITTED = BIT(1), 413 /** @XE_VMA_OP_NEXT_COMMITTED: Next VMA operation committed */ 414 XE_VMA_OP_NEXT_COMMITTED = BIT(2), 415 }; 416 417 /** enum xe_vma_subop - VMA sub-operation */ 418 enum xe_vma_subop { 419 /** @XE_VMA_SUBOP_MAP_RANGE: Map range */ 420 XE_VMA_SUBOP_MAP_RANGE, 421 /** @XE_VMA_SUBOP_UNMAP_RANGE: Unmap range */ 422 XE_VMA_SUBOP_UNMAP_RANGE, 423 }; 424 425 /** struct xe_vma_op - VMA operation */ 426 struct xe_vma_op { 427 /** @base: GPUVA base operation */ 428 struct drm_gpuva_op base; 429 /** @link: async operation link */ 430 struct list_head link; 431 /** @flags: operation flags */ 432 enum xe_vma_op_flags flags; 433 /** @subop: user defined sub-operation */ 434 enum xe_vma_subop subop; 435 /** @tile_mask: Tile mask for operation */ 436 u8 tile_mask; 437 438 union { 439 /** @map: VMA map operation specific data */ 440 struct xe_vma_op_map map; 441 /** @remap: VMA remap operation specific data */ 442 struct xe_vma_op_remap remap; 443 /** @prefetch: VMA prefetch operation specific data */ 444 struct xe_vma_op_prefetch prefetch; 445 /** @map_range: VMA map range operation specific data */ 446 struct xe_vma_op_map_range map_range; 447 /** @unmap_range: VMA unmap range operation specific data */ 448 struct xe_vma_op_unmap_range unmap_range; 449 /** @prefetch_range: VMA prefetch range operation specific data */ 450 struct xe_vma_op_prefetch_range prefetch_range; 451 }; 452 }; 453 454 /** struct xe_vma_ops - VMA operations */ 455 struct xe_vma_ops { 456 /** @list: list of VMA operations */ 457 struct list_head list; 458 /** @vm: VM */ 459 struct xe_vm *vm; 460 /** @q: exec queue for VMA operations */ 461 struct xe_exec_queue *q; 462 /** @syncs: syncs these operation */ 463 struct xe_sync_entry *syncs; 464 /** @num_syncs: number of syncs */ 465 u32 num_syncs; 466 /** @pt_update_ops: page table update operations */ 467 struct xe_vm_pgtable_update_ops pt_update_ops[XE_MAX_TILES_PER_DEVICE]; 468 /** @flag: signify the properties within xe_vma_ops*/ 469 #define XE_VMA_OPS_FLAG_HAS_SVM_PREFETCH BIT(0) 470 u32 flags; 471 #ifdef TEST_VM_OPS_ERROR 472 /** @inject_error: inject error to test error handling */ 473 bool inject_error; 474 #endif 475 }; 476 477 #endif 478