1 /* SPDX-License-Identifier: MIT */ 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #ifndef _XE_VM_TYPES_H_ 7 #define _XE_VM_TYPES_H_ 8 9 #include <drm/drm_gpusvm.h> 10 #include <drm/drm_gpuvm.h> 11 #include <drm/drm_pagemap_util.h> 12 13 #include <linux/dma-resv.h> 14 #include <linux/kref.h> 15 #include <linux/mmu_notifier.h> 16 #include <linux/scatterlist.h> 17 18 #include "xe_device_types.h" 19 #include "xe_pt_types.h" 20 #include "xe_range_fence.h" 21 #include "xe_userptr.h" 22 23 struct drm_pagemap; 24 25 struct xe_bo; 26 struct xe_svm_range; 27 struct xe_sync_entry; 28 struct xe_user_fence; 29 struct xe_vm; 30 struct xe_vm_pgtable_update_op; 31 32 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 33 #define TEST_VM_OPS_ERROR 34 #define FORCE_OP_ERROR BIT(31) 35 36 #define FORCE_OP_ERROR_LOCK 0 37 #define FORCE_OP_ERROR_PREPARE 1 38 #define FORCE_OP_ERROR_RUN 2 39 #define FORCE_OP_ERROR_COUNT 3 40 #endif 41 42 #define XE_VMA_READ_ONLY DRM_GPUVA_USERBITS 43 #define XE_VMA_DESTROYED (DRM_GPUVA_USERBITS << 1) 44 #define XE_VMA_ATOMIC_PTE_BIT (DRM_GPUVA_USERBITS << 2) 45 #define XE_VMA_PTE_4K (DRM_GPUVA_USERBITS << 3) 46 #define XE_VMA_PTE_2M (DRM_GPUVA_USERBITS << 4) 47 #define XE_VMA_PTE_1G (DRM_GPUVA_USERBITS << 5) 48 #define XE_VMA_PTE_64K (DRM_GPUVA_USERBITS << 6) 49 #define XE_VMA_PTE_COMPACT (DRM_GPUVA_USERBITS << 7) 50 #define XE_VMA_DUMPABLE (DRM_GPUVA_USERBITS << 8) 51 #define XE_VMA_SYSTEM_ALLOCATOR (DRM_GPUVA_USERBITS << 9) 52 #define XE_VMA_MADV_AUTORESET (DRM_GPUVA_USERBITS << 10) 53 54 /** 55 * struct xe_vma_mem_attr - memory attributes associated with vma 56 */ 57 struct xe_vma_mem_attr { 58 /** @preferred_loc: preferred memory_location */ 59 struct xe_vma_preferred_loc { 60 /** @preferred_loc.migration_policy: Pages migration policy */ 61 u32 migration_policy; 62 63 /** 64 * @preferred_loc.devmem_fd: used for determining pagemap_fd 65 * requested by user DRM_XE_PREFERRED_LOC_DEFAULT_SYSTEM and 66 * DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE mean system memory or 67 * closest device memory respectively. 68 */ 69 u32 devmem_fd; 70 /** 71 * @preferred_loc.dpagemap: Reference-counted pointer to the drm_pagemap preferred 72 * for migration on a SVM page-fault. The pointer is protected by the 73 * vm lock, and is %NULL if @devmem_fd should be consulted for special 74 * values. 75 */ 76 struct drm_pagemap *dpagemap; 77 } preferred_loc; 78 79 /** 80 * @atomic_access: The atomic access type for the vma 81 * See %DRM_XE_VMA_ATOMIC_UNDEFINED, %DRM_XE_VMA_ATOMIC_DEVICE, 82 * %DRM_XE_VMA_ATOMIC_GLOBAL, and %DRM_XE_VMA_ATOMIC_CPU for possible 83 * values. These are defined in uapi/drm/xe_drm.h. 84 */ 85 u32 atomic_access; 86 87 /** 88 * @default_pat_index: The pat index for VMA set during first bind by user. 89 */ 90 u16 default_pat_index; 91 92 /** 93 * @pat_index: The pat index to use when encoding the PTEs for this vma. 94 * same as default_pat_index unless overwritten by madvise. 95 */ 96 u16 pat_index; 97 }; 98 99 struct xe_vma { 100 /** @gpuva: Base GPUVA object */ 101 struct drm_gpuva gpuva; 102 103 /** 104 * @combined_links: links into lists which are mutually exclusive. 105 * Locking: vm lock in write mode OR vm lock in read mode and the vm's 106 * resv. 107 */ 108 union { 109 /** @rebind: link into VM if this VMA needs rebinding. */ 110 struct list_head rebind; 111 /** @destroy: link to contested list when VM is being closed. */ 112 struct list_head destroy; 113 } combined_links; 114 115 union { 116 /** @destroy_cb: callback to destroy VMA when unbind job is done */ 117 struct dma_fence_cb destroy_cb; 118 /** @destroy_work: worker to destroy this BO */ 119 struct work_struct destroy_work; 120 }; 121 122 /** 123 * @tile_invalidated: Tile mask of binding are invalidated for this VMA. 124 * protected by BO's resv and for userptrs, vm->svm.gpusvm.notifier_lock in 125 * write mode for writing or vm->svm.gpusvm.notifier_lock in read mode and 126 * the vm->resv. For stable reading, BO's resv or userptr 127 * vm->svm.gpusvm.notifier_lock in read mode is required. Can be 128 * opportunistically read with READ_ONCE outside of locks. 129 */ 130 u8 tile_invalidated; 131 132 /** @tile_mask: Tile mask of where to create binding for this VMA */ 133 u8 tile_mask; 134 135 /** 136 * @tile_present: Tile mask of binding are present for this VMA. 137 * protected by vm->lock, vm->resv and for userptrs, 138 * vm->svm.gpusvm.notifier_lock for writing. Needs either for reading, 139 * but if reading is done under the vm->lock only, it needs to be held 140 * in write mode. 141 */ 142 u8 tile_present; 143 144 /** @tile_staged: bind is staged for this VMA */ 145 u8 tile_staged; 146 147 /** 148 * @skip_invalidation: Used in madvise to avoid invalidation 149 * if mem attributes doesn't change 150 */ 151 bool skip_invalidation; 152 153 /** 154 * @ufence: The user fence that was provided with MAP. 155 * Needs to be signalled before UNMAP can be processed. 156 */ 157 struct xe_user_fence *ufence; 158 159 /** 160 * @attr: The attributes of vma which determines the migration policy 161 * and encoding of the PTEs for this vma. 162 */ 163 struct xe_vma_mem_attr attr; 164 }; 165 166 /** 167 * struct xe_userptr_vma - A userptr vma subclass 168 * @vma: The vma. 169 * @userptr: Additional userptr information. 170 */ 171 struct xe_userptr_vma { 172 struct xe_vma vma; 173 struct xe_userptr userptr; 174 }; 175 176 struct xe_device; 177 178 struct xe_vm { 179 /** @gpuvm: base GPUVM used to track VMAs */ 180 struct drm_gpuvm gpuvm; 181 182 /** @svm: Shared virtual memory state */ 183 struct { 184 /** @svm.gpusvm: base GPUSVM used to track fault allocations */ 185 struct drm_gpusvm gpusvm; 186 /** 187 * @svm.garbage_collector: Garbage collector which is used unmap 188 * SVM range's GPU bindings and destroy the ranges. 189 */ 190 struct { 191 /** @svm.garbage_collector.lock: Protect's range list */ 192 spinlock_t lock; 193 /** 194 * @svm.garbage_collector.range_list: List of SVM ranges 195 * in the garbage collector. 196 */ 197 struct list_head range_list; 198 /** 199 * @svm.garbage_collector.work: Worker which the 200 * garbage collector runs on. 201 */ 202 struct work_struct work; 203 } garbage_collector; 204 struct xe_pagemap *pagemaps[XE_MAX_TILES_PER_DEVICE]; 205 /** @svm.peer: Used for pagemap connectivity computations. */ 206 struct drm_pagemap_peer peer; 207 } svm; 208 209 struct xe_device *xe; 210 211 /* exec queue used for (un)binding vma's */ 212 struct xe_exec_queue *q[XE_MAX_TILES_PER_DEVICE]; 213 214 /** @lru_bulk_move: Bulk LRU move list for this VM's BOs */ 215 struct ttm_lru_bulk_move lru_bulk_move; 216 217 u64 size; 218 219 struct xe_pt *pt_root[XE_MAX_TILES_PER_DEVICE]; 220 struct xe_pt *scratch_pt[XE_MAX_TILES_PER_DEVICE][XE_VM_MAX_LEVEL]; 221 222 /** 223 * @flags: flags for this VM, statically setup a creation time aside 224 * from XE_VM_FLAG_BANNED which requires vm->lock to set / read safely 225 */ 226 #define XE_VM_FLAG_64K BIT(0) 227 #define XE_VM_FLAG_LR_MODE BIT(1) 228 #define XE_VM_FLAG_MIGRATION BIT(2) 229 #define XE_VM_FLAG_SCRATCH_PAGE BIT(3) 230 #define XE_VM_FLAG_FAULT_MODE BIT(4) 231 #define XE_VM_FLAG_BANNED BIT(5) 232 #define XE_VM_FLAG_TILE_ID(flags) FIELD_GET(GENMASK(7, 6), flags) 233 #define XE_VM_FLAG_SET_TILE_ID(tile) FIELD_PREP(GENMASK(7, 6), (tile)->id) 234 #define XE_VM_FLAG_GSC BIT(8) 235 unsigned long flags; 236 237 /** 238 * @lock: outer most lock, protects objects of anything attached to this 239 * VM 240 */ 241 struct rw_semaphore lock; 242 /** 243 * @snap_mutex: Mutex used to guard insertions and removals from gpuva, 244 * so we can take a snapshot safely from devcoredump. 245 */ 246 struct mutex snap_mutex; 247 248 /** 249 * @rebind_list: list of VMAs that need rebinding. Protected by the 250 * vm->lock in write mode, OR (the vm->lock in read mode and the 251 * vm resv). 252 */ 253 struct list_head rebind_list; 254 255 /** 256 * @destroy_work: worker to destroy VM, needed as a dma_fence signaling 257 * from an irq context can be last put and the destroy needs to be able 258 * to sleep. 259 */ 260 struct work_struct destroy_work; 261 262 /** 263 * @rftree: range fence tree to track updates to page table structure. 264 * Used to implement conflict tracking between independent bind engines. 265 */ 266 struct xe_range_fence_tree rftree[XE_MAX_TILES_PER_DEVICE]; 267 268 const struct xe_pt_ops *pt_ops; 269 270 /** @userptr: user pointer state */ 271 struct xe_userptr_vm userptr; 272 273 /** @preempt: preempt state */ 274 struct { 275 /** 276 * @min_run_period_ms: The minimum run period before preempting 277 * an engine again 278 */ 279 unsigned int min_run_period_ms; 280 /** @exec_queues: list of exec queues attached to this VM */ 281 struct list_head exec_queues; 282 /** @num_exec_queues: number exec queues attached to this VM */ 283 int num_exec_queues; 284 /** 285 * @rebind_deactivated: Whether rebind has been temporarily deactivated 286 * due to no work available. Protected by the vm resv. 287 */ 288 bool rebind_deactivated; 289 /** 290 * @rebind_work: worker to rebind invalidated userptrs / evicted 291 * BOs 292 */ 293 struct work_struct rebind_work; 294 /** 295 * @preempt.pm_activate_link: Link to list of rebind workers to be 296 * kicked on resume. 297 */ 298 struct list_head pm_activate_link; 299 } preempt; 300 301 /** @um: unified memory state */ 302 struct { 303 /** @asid: address space ID, unique to each VM */ 304 u32 asid; 305 /** 306 * @last_fault_vma: Last fault VMA, used for fast lookup when we 307 * get a flood of faults to the same VMA 308 */ 309 struct xe_vma *last_fault_vma; 310 } usm; 311 312 /** @error_capture: allow to track errors */ 313 struct { 314 /** @capture_once: capture only one error per VM */ 315 bool capture_once; 316 } error_capture; 317 318 /** 319 * @validation: Validation data only valid with the vm resv held. 320 * Note: This is really task state of the task holding the vm resv, 321 * and moving forward we should 322 * come up with a better way of passing this down the call- 323 * chain. 324 */ 325 struct { 326 /** 327 * @validation.validating: The task that is currently making bos resident. 328 * for this vm. 329 * Protected by the VM's resv for writing. Opportunistic reading can be done 330 * using READ_ONCE. Note: This is a workaround for the 331 * TTM eviction_valuable() callback not being passed a struct 332 * ttm_operation_context(). Future work might want to address this. 333 */ 334 struct task_struct *validating; 335 /** 336 * @validation.exec The drm_exec context used when locking the vm resv. 337 * Protected by the vm's resv. 338 */ 339 struct drm_exec *_exec; 340 } validation; 341 342 /** 343 * @tlb_flush_seqno: Required TLB flush seqno for the next exec. 344 * protected by the vm resv. 345 */ 346 u64 tlb_flush_seqno; 347 /** @batch_invalidate_tlb: Always invalidate TLB before batch start */ 348 bool batch_invalidate_tlb; 349 /** @xef: Xe file handle for tracking this VM's drm client */ 350 struct xe_file *xef; 351 }; 352 353 /** struct xe_vma_op_map - VMA map operation */ 354 struct xe_vma_op_map { 355 /** @vma: VMA to map */ 356 struct xe_vma *vma; 357 unsigned int vma_flags; 358 /** @immediate: Immediate bind */ 359 bool immediate; 360 /** @read_only: Read only */ 361 bool invalidate_on_bind; 362 /** @pat_index: The pat index to use for this operation. */ 363 u16 pat_index; 364 }; 365 366 /** struct xe_vma_op_remap - VMA remap operation */ 367 struct xe_vma_op_remap { 368 /** @prev: VMA preceding part of a split mapping */ 369 struct xe_vma *prev; 370 /** @next: VMA subsequent part of a split mapping */ 371 struct xe_vma *next; 372 /** @start: start of the VMA unmap */ 373 u64 start; 374 /** @range: range of the VMA unmap */ 375 u64 range; 376 /** @skip_prev: skip prev rebind */ 377 bool skip_prev; 378 /** @skip_next: skip next rebind */ 379 bool skip_next; 380 /** @unmap_done: unmap operation in done */ 381 bool unmap_done; 382 }; 383 384 /** struct xe_vma_op_prefetch - VMA prefetch operation */ 385 struct xe_vma_op_prefetch { 386 /** @region: memory region to prefetch to */ 387 u32 region; 388 }; 389 390 /** struct xe_vma_op_map_range - VMA map range operation */ 391 struct xe_vma_op_map_range { 392 /** @vma: VMA to map (system allocator VMA) */ 393 struct xe_vma *vma; 394 /** @range: SVM range to map */ 395 struct xe_svm_range *range; 396 }; 397 398 /** struct xe_vma_op_unmap_range - VMA unmap range operation */ 399 struct xe_vma_op_unmap_range { 400 /** @range: SVM range to unmap */ 401 struct xe_svm_range *range; 402 }; 403 404 /** struct xe_vma_op_prefetch_range - VMA prefetch range operation */ 405 struct xe_vma_op_prefetch_range { 406 /** @range: xarray for SVM ranges data */ 407 struct xarray range; 408 /** @ranges_count: number of svm ranges to map */ 409 u32 ranges_count; 410 /** 411 * @dpagemap: Pointer to the dpagemap structure containing memory to prefetch. 412 * NULL if prefetch requested region is smem 413 */ 414 struct drm_pagemap *dpagemap; 415 }; 416 417 /** enum xe_vma_op_flags - flags for VMA operation */ 418 enum xe_vma_op_flags { 419 /** @XE_VMA_OP_COMMITTED: VMA operation committed */ 420 XE_VMA_OP_COMMITTED = BIT(0), 421 /** @XE_VMA_OP_PREV_COMMITTED: Previous VMA operation committed */ 422 XE_VMA_OP_PREV_COMMITTED = BIT(1), 423 /** @XE_VMA_OP_NEXT_COMMITTED: Next VMA operation committed */ 424 XE_VMA_OP_NEXT_COMMITTED = BIT(2), 425 }; 426 427 /** enum xe_vma_subop - VMA sub-operation */ 428 enum xe_vma_subop { 429 /** @XE_VMA_SUBOP_MAP_RANGE: Map range */ 430 XE_VMA_SUBOP_MAP_RANGE, 431 /** @XE_VMA_SUBOP_UNMAP_RANGE: Unmap range */ 432 XE_VMA_SUBOP_UNMAP_RANGE, 433 }; 434 435 /** struct xe_vma_op - VMA operation */ 436 struct xe_vma_op { 437 /** @base: GPUVA base operation */ 438 struct drm_gpuva_op base; 439 /** @link: async operation link */ 440 struct list_head link; 441 /** @flags: operation flags */ 442 enum xe_vma_op_flags flags; 443 /** @subop: user defined sub-operation */ 444 enum xe_vma_subop subop; 445 /** @tile_mask: Tile mask for operation */ 446 u8 tile_mask; 447 448 union { 449 /** @map: VMA map operation specific data */ 450 struct xe_vma_op_map map; 451 /** @remap: VMA remap operation specific data */ 452 struct xe_vma_op_remap remap; 453 /** @prefetch: VMA prefetch operation specific data */ 454 struct xe_vma_op_prefetch prefetch; 455 /** @map_range: VMA map range operation specific data */ 456 struct xe_vma_op_map_range map_range; 457 /** @unmap_range: VMA unmap range operation specific data */ 458 struct xe_vma_op_unmap_range unmap_range; 459 /** @prefetch_range: VMA prefetch range operation specific data */ 460 struct xe_vma_op_prefetch_range prefetch_range; 461 }; 462 }; 463 464 /** struct xe_vma_ops - VMA operations */ 465 struct xe_vma_ops { 466 /** @list: list of VMA operations */ 467 struct list_head list; 468 /** @vm: VM */ 469 struct xe_vm *vm; 470 /** @q: exec queue for VMA operations */ 471 struct xe_exec_queue *q; 472 /** @syncs: syncs these operation */ 473 struct xe_sync_entry *syncs; 474 /** @num_syncs: number of syncs */ 475 u32 num_syncs; 476 /** @pt_update_ops: page table update operations */ 477 struct xe_vm_pgtable_update_ops pt_update_ops[XE_MAX_TILES_PER_DEVICE]; 478 /** @flag: signify the properties within xe_vma_ops*/ 479 #define XE_VMA_OPS_FLAG_HAS_SVM_PREFETCH BIT(0) 480 #define XE_VMA_OPS_FLAG_MADVISE BIT(1) 481 #define XE_VMA_OPS_ARRAY_OF_BINDS BIT(2) 482 #define XE_VMA_OPS_FLAG_SKIP_TLB_WAIT BIT(3) 483 #define XE_VMA_OPS_FLAG_ALLOW_SVM_UNMAP BIT(4) 484 u32 flags; 485 #ifdef TEST_VM_OPS_ERROR 486 /** @inject_error: inject error to test error handling */ 487 bool inject_error; 488 #endif 489 }; 490 491 #endif 492