1 /* SPDX-License-Identifier: MIT */ 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #ifndef _XE_VM_TYPES_H_ 7 #define _XE_VM_TYPES_H_ 8 9 #include <drm/drm_gpusvm.h> 10 #include <drm/drm_gpuvm.h> 11 12 #include <linux/dma-resv.h> 13 #include <linux/kref.h> 14 #include <linux/mmu_notifier.h> 15 #include <linux/scatterlist.h> 16 17 #include "xe_device_types.h" 18 #include "xe_pt_types.h" 19 #include "xe_range_fence.h" 20 #include "xe_userptr.h" 21 22 struct xe_bo; 23 struct xe_svm_range; 24 struct xe_sync_entry; 25 struct xe_user_fence; 26 struct xe_vm; 27 struct xe_vm_pgtable_update_op; 28 29 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 30 #define TEST_VM_OPS_ERROR 31 #define FORCE_OP_ERROR BIT(31) 32 33 #define FORCE_OP_ERROR_LOCK 0 34 #define FORCE_OP_ERROR_PREPARE 1 35 #define FORCE_OP_ERROR_RUN 2 36 #define FORCE_OP_ERROR_COUNT 3 37 #endif 38 39 #define XE_VMA_READ_ONLY DRM_GPUVA_USERBITS 40 #define XE_VMA_DESTROYED (DRM_GPUVA_USERBITS << 1) 41 #define XE_VMA_ATOMIC_PTE_BIT (DRM_GPUVA_USERBITS << 2) 42 #define XE_VMA_PTE_4K (DRM_GPUVA_USERBITS << 3) 43 #define XE_VMA_PTE_2M (DRM_GPUVA_USERBITS << 4) 44 #define XE_VMA_PTE_1G (DRM_GPUVA_USERBITS << 5) 45 #define XE_VMA_PTE_64K (DRM_GPUVA_USERBITS << 6) 46 #define XE_VMA_PTE_COMPACT (DRM_GPUVA_USERBITS << 7) 47 #define XE_VMA_DUMPABLE (DRM_GPUVA_USERBITS << 8) 48 #define XE_VMA_SYSTEM_ALLOCATOR (DRM_GPUVA_USERBITS << 9) 49 #define XE_VMA_MADV_AUTORESET (DRM_GPUVA_USERBITS << 10) 50 51 /** 52 * struct xe_vma_mem_attr - memory attributes associated with vma 53 */ 54 struct xe_vma_mem_attr { 55 /** @preferred_loc: preferred memory_location */ 56 struct { 57 /** @preferred_loc.migration_policy: Pages migration policy */ 58 u32 migration_policy; 59 60 /** 61 * @preferred_loc.devmem_fd: used for determining pagemap_fd 62 * requested by user DRM_XE_PREFERRED_LOC_DEFAULT_SYSTEM and 63 * DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE mean system memory or 64 * closest device memory respectively. 65 */ 66 u32 devmem_fd; 67 } preferred_loc; 68 69 /** 70 * @atomic_access: The atomic access type for the vma 71 * See %DRM_XE_VMA_ATOMIC_UNDEFINED, %DRM_XE_VMA_ATOMIC_DEVICE, 72 * %DRM_XE_VMA_ATOMIC_GLOBAL, and %DRM_XE_VMA_ATOMIC_CPU for possible 73 * values. These are defined in uapi/drm/xe_drm.h. 74 */ 75 u32 atomic_access; 76 77 /** 78 * @default_pat_index: The pat index for VMA set during first bind by user. 79 */ 80 u16 default_pat_index; 81 82 /** 83 * @pat_index: The pat index to use when encoding the PTEs for this vma. 84 * same as default_pat_index unless overwritten by madvise. 85 */ 86 u16 pat_index; 87 }; 88 89 struct xe_vma { 90 /** @gpuva: Base GPUVA object */ 91 struct drm_gpuva gpuva; 92 93 /** 94 * @combined_links: links into lists which are mutually exclusive. 95 * Locking: vm lock in write mode OR vm lock in read mode and the vm's 96 * resv. 97 */ 98 union { 99 /** @rebind: link into VM if this VMA needs rebinding. */ 100 struct list_head rebind; 101 /** @destroy: link to contested list when VM is being closed. */ 102 struct list_head destroy; 103 } combined_links; 104 105 union { 106 /** @destroy_cb: callback to destroy VMA when unbind job is done */ 107 struct dma_fence_cb destroy_cb; 108 /** @destroy_work: worker to destroy this BO */ 109 struct work_struct destroy_work; 110 }; 111 112 /** 113 * @tile_invalidated: Tile mask of binding are invalidated for this VMA. 114 * protected by BO's resv and for userptrs, vm->svm.gpusvm.notifier_lock in 115 * write mode for writing or vm->svm.gpusvm.notifier_lock in read mode and 116 * the vm->resv. For stable reading, BO's resv or userptr 117 * vm->svm.gpusvm.notifier_lock in read mode is required. Can be 118 * opportunistically read with READ_ONCE outside of locks. 119 */ 120 u8 tile_invalidated; 121 122 /** @tile_mask: Tile mask of where to create binding for this VMA */ 123 u8 tile_mask; 124 125 /** 126 * @tile_present: Tile mask of binding are present for this VMA. 127 * protected by vm->lock, vm->resv and for userptrs, 128 * vm->svm.gpusvm.notifier_lock for writing. Needs either for reading, 129 * but if reading is done under the vm->lock only, it needs to be held 130 * in write mode. 131 */ 132 u8 tile_present; 133 134 /** @tile_staged: bind is staged for this VMA */ 135 u8 tile_staged; 136 137 /** 138 * @skip_invalidation: Used in madvise to avoid invalidation 139 * if mem attributes doesn't change 140 */ 141 bool skip_invalidation; 142 143 /** 144 * @ufence: The user fence that was provided with MAP. 145 * Needs to be signalled before UNMAP can be processed. 146 */ 147 struct xe_user_fence *ufence; 148 149 /** 150 * @attr: The attributes of vma which determines the migration policy 151 * and encoding of the PTEs for this vma. 152 */ 153 struct xe_vma_mem_attr attr; 154 }; 155 156 /** 157 * struct xe_userptr_vma - A userptr vma subclass 158 * @vma: The vma. 159 * @userptr: Additional userptr information. 160 */ 161 struct xe_userptr_vma { 162 struct xe_vma vma; 163 struct xe_userptr userptr; 164 }; 165 166 struct xe_device; 167 168 struct xe_vm { 169 /** @gpuvm: base GPUVM used to track VMAs */ 170 struct drm_gpuvm gpuvm; 171 172 /** @svm: Shared virtual memory state */ 173 struct { 174 /** @svm.gpusvm: base GPUSVM used to track fault allocations */ 175 struct drm_gpusvm gpusvm; 176 /** 177 * @svm.garbage_collector: Garbage collector which is used unmap 178 * SVM range's GPU bindings and destroy the ranges. 179 */ 180 struct { 181 /** @svm.garbage_collector.lock: Protect's range list */ 182 spinlock_t lock; 183 /** 184 * @svm.garbage_collector.range_list: List of SVM ranges 185 * in the garbage collector. 186 */ 187 struct list_head range_list; 188 /** 189 * @svm.garbage_collector.work: Worker which the 190 * garbage collector runs on. 191 */ 192 struct work_struct work; 193 } garbage_collector; 194 } svm; 195 196 struct xe_device *xe; 197 198 /* exec queue used for (un)binding vma's */ 199 struct xe_exec_queue *q[XE_MAX_TILES_PER_DEVICE]; 200 201 /** @lru_bulk_move: Bulk LRU move list for this VM's BOs */ 202 struct ttm_lru_bulk_move lru_bulk_move; 203 204 u64 size; 205 206 struct xe_pt *pt_root[XE_MAX_TILES_PER_DEVICE]; 207 struct xe_pt *scratch_pt[XE_MAX_TILES_PER_DEVICE][XE_VM_MAX_LEVEL]; 208 209 /** 210 * @flags: flags for this VM, statically setup a creation time aside 211 * from XE_VM_FLAG_BANNED which requires vm->lock to set / read safely 212 */ 213 #define XE_VM_FLAG_64K BIT(0) 214 #define XE_VM_FLAG_LR_MODE BIT(1) 215 #define XE_VM_FLAG_MIGRATION BIT(2) 216 #define XE_VM_FLAG_SCRATCH_PAGE BIT(3) 217 #define XE_VM_FLAG_FAULT_MODE BIT(4) 218 #define XE_VM_FLAG_BANNED BIT(5) 219 #define XE_VM_FLAG_TILE_ID(flags) FIELD_GET(GENMASK(7, 6), flags) 220 #define XE_VM_FLAG_SET_TILE_ID(tile) FIELD_PREP(GENMASK(7, 6), (tile)->id) 221 #define XE_VM_FLAG_GSC BIT(8) 222 unsigned long flags; 223 224 /** 225 * @lock: outer most lock, protects objects of anything attached to this 226 * VM 227 */ 228 struct rw_semaphore lock; 229 /** 230 * @snap_mutex: Mutex used to guard insertions and removals from gpuva, 231 * so we can take a snapshot safely from devcoredump. 232 */ 233 struct mutex snap_mutex; 234 235 /** 236 * @rebind_list: list of VMAs that need rebinding. Protected by the 237 * vm->lock in write mode, OR (the vm->lock in read mode and the 238 * vm resv). 239 */ 240 struct list_head rebind_list; 241 242 /** 243 * @destroy_work: worker to destroy VM, needed as a dma_fence signaling 244 * from an irq context can be last put and the destroy needs to be able 245 * to sleep. 246 */ 247 struct work_struct destroy_work; 248 249 /** 250 * @rftree: range fence tree to track updates to page table structure. 251 * Used to implement conflict tracking between independent bind engines. 252 */ 253 struct xe_range_fence_tree rftree[XE_MAX_TILES_PER_DEVICE]; 254 255 const struct xe_pt_ops *pt_ops; 256 257 /** @userptr: user pointer state */ 258 struct xe_userptr_vm userptr; 259 260 /** @preempt: preempt state */ 261 struct { 262 /** 263 * @min_run_period_ms: The minimum run period before preempting 264 * an engine again 265 */ 266 s64 min_run_period_ms; 267 /** @exec_queues: list of exec queues attached to this VM */ 268 struct list_head exec_queues; 269 /** @num_exec_queues: number exec queues attached to this VM */ 270 int num_exec_queues; 271 /** 272 * @rebind_deactivated: Whether rebind has been temporarily deactivated 273 * due to no work available. Protected by the vm resv. 274 */ 275 bool rebind_deactivated; 276 /** 277 * @rebind_work: worker to rebind invalidated userptrs / evicted 278 * BOs 279 */ 280 struct work_struct rebind_work; 281 /** 282 * @preempt.pm_activate_link: Link to list of rebind workers to be 283 * kicked on resume. 284 */ 285 struct list_head pm_activate_link; 286 } preempt; 287 288 /** @um: unified memory state */ 289 struct { 290 /** @asid: address space ID, unique to each VM */ 291 u32 asid; 292 /** 293 * @last_fault_vma: Last fault VMA, used for fast lookup when we 294 * get a flood of faults to the same VMA 295 */ 296 struct xe_vma *last_fault_vma; 297 } usm; 298 299 /** @error_capture: allow to track errors */ 300 struct { 301 /** @capture_once: capture only one error per VM */ 302 bool capture_once; 303 } error_capture; 304 305 /** 306 * @validation: Validation data only valid with the vm resv held. 307 * Note: This is really task state of the task holding the vm resv, 308 * and moving forward we should 309 * come up with a better way of passing this down the call- 310 * chain. 311 */ 312 struct { 313 /** 314 * @validation.validating: The task that is currently making bos resident. 315 * for this vm. 316 * Protected by the VM's resv for writing. Opportunistic reading can be done 317 * using READ_ONCE. Note: This is a workaround for the 318 * TTM eviction_valuable() callback not being passed a struct 319 * ttm_operation_context(). Future work might want to address this. 320 */ 321 struct task_struct *validating; 322 /** 323 * @validation.exec The drm_exec context used when locking the vm resv. 324 * Protected by the vm's resv. 325 */ 326 struct drm_exec *_exec; 327 } validation; 328 329 /** 330 * @tlb_flush_seqno: Required TLB flush seqno for the next exec. 331 * protected by the vm resv. 332 */ 333 u64 tlb_flush_seqno; 334 /** @batch_invalidate_tlb: Always invalidate TLB before batch start */ 335 bool batch_invalidate_tlb; 336 /** @xef: Xe file handle for tracking this VM's drm client */ 337 struct xe_file *xef; 338 }; 339 340 /** struct xe_vma_op_map - VMA map operation */ 341 struct xe_vma_op_map { 342 /** @vma: VMA to map */ 343 struct xe_vma *vma; 344 unsigned int vma_flags; 345 /** @immediate: Immediate bind */ 346 bool immediate; 347 /** @read_only: Read only */ 348 bool invalidate_on_bind; 349 /** @pat_index: The pat index to use for this operation. */ 350 u16 pat_index; 351 }; 352 353 /** struct xe_vma_op_remap - VMA remap operation */ 354 struct xe_vma_op_remap { 355 /** @prev: VMA preceding part of a split mapping */ 356 struct xe_vma *prev; 357 /** @next: VMA subsequent part of a split mapping */ 358 struct xe_vma *next; 359 /** @start: start of the VMA unmap */ 360 u64 start; 361 /** @range: range of the VMA unmap */ 362 u64 range; 363 /** @skip_prev: skip prev rebind */ 364 bool skip_prev; 365 /** @skip_next: skip next rebind */ 366 bool skip_next; 367 /** @unmap_done: unmap operation in done */ 368 bool unmap_done; 369 }; 370 371 /** struct xe_vma_op_prefetch - VMA prefetch operation */ 372 struct xe_vma_op_prefetch { 373 /** @region: memory region to prefetch to */ 374 u32 region; 375 }; 376 377 /** struct xe_vma_op_map_range - VMA map range operation */ 378 struct xe_vma_op_map_range { 379 /** @vma: VMA to map (system allocator VMA) */ 380 struct xe_vma *vma; 381 /** @range: SVM range to map */ 382 struct xe_svm_range *range; 383 }; 384 385 /** struct xe_vma_op_unmap_range - VMA unmap range operation */ 386 struct xe_vma_op_unmap_range { 387 /** @range: SVM range to unmap */ 388 struct xe_svm_range *range; 389 }; 390 391 /** struct xe_vma_op_prefetch_range - VMA prefetch range operation */ 392 struct xe_vma_op_prefetch_range { 393 /** @range: xarray for SVM ranges data */ 394 struct xarray range; 395 /** @ranges_count: number of svm ranges to map */ 396 u32 ranges_count; 397 /** 398 * @tile: Pointer to the tile structure containing memory to prefetch. 399 * NULL if prefetch requested region is smem 400 */ 401 struct xe_tile *tile; 402 }; 403 404 /** enum xe_vma_op_flags - flags for VMA operation */ 405 enum xe_vma_op_flags { 406 /** @XE_VMA_OP_COMMITTED: VMA operation committed */ 407 XE_VMA_OP_COMMITTED = BIT(0), 408 /** @XE_VMA_OP_PREV_COMMITTED: Previous VMA operation committed */ 409 XE_VMA_OP_PREV_COMMITTED = BIT(1), 410 /** @XE_VMA_OP_NEXT_COMMITTED: Next VMA operation committed */ 411 XE_VMA_OP_NEXT_COMMITTED = BIT(2), 412 }; 413 414 /** enum xe_vma_subop - VMA sub-operation */ 415 enum xe_vma_subop { 416 /** @XE_VMA_SUBOP_MAP_RANGE: Map range */ 417 XE_VMA_SUBOP_MAP_RANGE, 418 /** @XE_VMA_SUBOP_UNMAP_RANGE: Unmap range */ 419 XE_VMA_SUBOP_UNMAP_RANGE, 420 }; 421 422 /** struct xe_vma_op - VMA operation */ 423 struct xe_vma_op { 424 /** @base: GPUVA base operation */ 425 struct drm_gpuva_op base; 426 /** @link: async operation link */ 427 struct list_head link; 428 /** @flags: operation flags */ 429 enum xe_vma_op_flags flags; 430 /** @subop: user defined sub-operation */ 431 enum xe_vma_subop subop; 432 /** @tile_mask: Tile mask for operation */ 433 u8 tile_mask; 434 435 union { 436 /** @map: VMA map operation specific data */ 437 struct xe_vma_op_map map; 438 /** @remap: VMA remap operation specific data */ 439 struct xe_vma_op_remap remap; 440 /** @prefetch: VMA prefetch operation specific data */ 441 struct xe_vma_op_prefetch prefetch; 442 /** @map_range: VMA map range operation specific data */ 443 struct xe_vma_op_map_range map_range; 444 /** @unmap_range: VMA unmap range operation specific data */ 445 struct xe_vma_op_unmap_range unmap_range; 446 /** @prefetch_range: VMA prefetch range operation specific data */ 447 struct xe_vma_op_prefetch_range prefetch_range; 448 }; 449 }; 450 451 /** struct xe_vma_ops - VMA operations */ 452 struct xe_vma_ops { 453 /** @list: list of VMA operations */ 454 struct list_head list; 455 /** @vm: VM */ 456 struct xe_vm *vm; 457 /** @q: exec queue for VMA operations */ 458 struct xe_exec_queue *q; 459 /** @syncs: syncs these operation */ 460 struct xe_sync_entry *syncs; 461 /** @num_syncs: number of syncs */ 462 u32 num_syncs; 463 /** @pt_update_ops: page table update operations */ 464 struct xe_vm_pgtable_update_ops pt_update_ops[XE_MAX_TILES_PER_DEVICE]; 465 /** @flag: signify the properties within xe_vma_ops*/ 466 #define XE_VMA_OPS_FLAG_HAS_SVM_PREFETCH BIT(0) 467 #define XE_VMA_OPS_FLAG_MADVISE BIT(1) 468 #define XE_VMA_OPS_ARRAY_OF_BINDS BIT(2) 469 #define XE_VMA_OPS_FLAG_SKIP_TLB_WAIT BIT(3) 470 u32 flags; 471 #ifdef TEST_VM_OPS_ERROR 472 /** @inject_error: inject error to test error handling */ 473 bool inject_error; 474 #endif 475 }; 476 477 #endif 478