1 /* SPDX-License-Identifier: MIT */ 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #ifndef _XE_VM_TYPES_H_ 7 #define _XE_VM_TYPES_H_ 8 9 #include <drm/drm_gpusvm.h> 10 #include <drm/drm_gpuvm.h> 11 #include <drm/drm_pagemap_util.h> 12 13 #include <linux/dma-resv.h> 14 #include <linux/kref.h> 15 #include <linux/mmu_notifier.h> 16 #include <linux/scatterlist.h> 17 18 #include "xe_device_types.h" 19 #include "xe_pt_types.h" 20 #include "xe_range_fence.h" 21 #include "xe_tlb_inval_types.h" 22 #include "xe_userptr.h" 23 24 struct drm_pagemap; 25 26 struct xe_bo; 27 struct xe_pagefault; 28 struct xe_svm_range; 29 struct xe_sync_entry; 30 struct xe_user_fence; 31 struct xe_vm; 32 struct xe_vm_pgtable_update_op; 33 34 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 35 #define TEST_VM_OPS_ERROR 36 #define FORCE_OP_ERROR BIT(31) 37 38 #define FORCE_OP_ERROR_LOCK 0 39 #define FORCE_OP_ERROR_PREPARE 1 40 #define FORCE_OP_ERROR_RUN 2 41 #define FORCE_OP_ERROR_COUNT 3 42 #endif 43 44 #define XE_VMA_READ_ONLY DRM_GPUVA_USERBITS 45 #define XE_VMA_DESTROYED (DRM_GPUVA_USERBITS << 1) 46 #define XE_VMA_ATOMIC_PTE_BIT (DRM_GPUVA_USERBITS << 2) 47 #define XE_VMA_PTE_4K (DRM_GPUVA_USERBITS << 3) 48 #define XE_VMA_PTE_2M (DRM_GPUVA_USERBITS << 4) 49 #define XE_VMA_PTE_1G (DRM_GPUVA_USERBITS << 5) 50 #define XE_VMA_PTE_64K (DRM_GPUVA_USERBITS << 6) 51 #define XE_VMA_PTE_COMPACT (DRM_GPUVA_USERBITS << 7) 52 #define XE_VMA_DUMPABLE (DRM_GPUVA_USERBITS << 8) 53 #define XE_VMA_SYSTEM_ALLOCATOR (DRM_GPUVA_USERBITS << 9) 54 #define XE_VMA_MADV_AUTORESET (DRM_GPUVA_USERBITS << 10) 55 56 /** 57 * struct xe_vma_mem_attr - memory attributes associated with vma 58 */ 59 struct xe_vma_mem_attr { 60 /** @preferred_loc: preferred memory_location */ 61 struct xe_vma_preferred_loc { 62 /** @preferred_loc.migration_policy: Pages migration policy */ 63 u32 migration_policy; 64 65 /** 66 * @preferred_loc.devmem_fd: used for determining pagemap_fd 67 * requested by user DRM_XE_PREFERRED_LOC_DEFAULT_SYSTEM and 68 * DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE mean system memory or 69 * closest device memory respectively. 70 */ 71 u32 devmem_fd; 72 /** 73 * @preferred_loc.dpagemap: Reference-counted pointer to the drm_pagemap preferred 74 * for migration on a SVM page-fault. The pointer is protected by the 75 * vm lock, and is %NULL if @devmem_fd should be consulted for special 76 * values. 77 */ 78 struct drm_pagemap *dpagemap; 79 } preferred_loc; 80 81 /** 82 * @atomic_access: The atomic access type for the vma 83 * See %DRM_XE_VMA_ATOMIC_UNDEFINED, %DRM_XE_VMA_ATOMIC_DEVICE, 84 * %DRM_XE_VMA_ATOMIC_GLOBAL, and %DRM_XE_VMA_ATOMIC_CPU for possible 85 * values. These are defined in uapi/drm/xe_drm.h. 86 */ 87 u32 atomic_access; 88 89 /** 90 * @default_pat_index: The pat index for VMA set during first bind by user. 91 */ 92 u16 default_pat_index; 93 94 /** 95 * @pat_index: The pat index to use when encoding the PTEs for this vma. 96 * same as default_pat_index unless overwritten by madvise. 97 */ 98 u16 pat_index; 99 100 /** 101 * @purgeable_state: Purgeable hint for this VMA mapping 102 * 103 * Per-VMA purgeable state from madvise. Valid states are WILLNEED (0) 104 * or DONTNEED (1). Shared BOs require all VMAs to be DONTNEED before 105 * the BO can be purged. PURGED state exists only at BO level. 106 * 107 * Protected by BO dma-resv lock. Set via DRM_IOCTL_XE_MADVISE. 108 */ 109 u32 purgeable_state; 110 }; 111 112 struct xe_vma { 113 /** @gpuva: Base GPUVA object */ 114 struct drm_gpuva gpuva; 115 116 /** 117 * @combined_links: links into lists which are mutually exclusive. 118 * Locking: vm lock in write mode OR vm lock in read mode and the vm's 119 * resv. 120 */ 121 union { 122 /** @rebind: link into VM if this VMA needs rebinding. */ 123 struct list_head rebind; 124 /** @destroy: link to contested list when VM is being closed. */ 125 struct list_head destroy; 126 } combined_links; 127 128 union { 129 /** @destroy_cb: callback to destroy VMA when unbind job is done */ 130 struct dma_fence_cb destroy_cb; 131 /** @destroy_work: worker to destroy this BO */ 132 struct work_struct destroy_work; 133 }; 134 135 /** 136 * @tile_invalidated: Tile mask of binding are invalidated for this VMA. 137 * protected by BO's resv and for userptrs, vm->svm.gpusvm.notifier_lock in 138 * write mode for writing or vm->svm.gpusvm.notifier_lock in read mode and 139 * the vm->resv. For stable reading, BO's resv or userptr 140 * vm->svm.gpusvm.notifier_lock in read mode is required. Can be 141 * opportunistically read with READ_ONCE outside of locks. 142 */ 143 u8 tile_invalidated; 144 145 /** @tile_mask: Tile mask of where to create binding for this VMA */ 146 u8 tile_mask; 147 148 /** 149 * @tile_present: Tile mask of binding are present for this VMA. 150 * protected by vm->lock, vm->resv and for userptrs, 151 * vm->svm.gpusvm.notifier_lock for writing. Needs either for reading, 152 * but if reading is done under the vm->lock only, it needs to be held 153 * in write mode. 154 */ 155 u8 tile_present; 156 157 /** @tile_staged: bind is staged for this VMA */ 158 u8 tile_staged; 159 160 /** 161 * @skip_invalidation: Used in madvise to avoid invalidation 162 * if mem attributes doesn't change 163 */ 164 bool skip_invalidation; 165 166 /** 167 * @ufence: The user fence that was provided with MAP. 168 * Needs to be signalled before UNMAP can be processed. 169 */ 170 struct xe_user_fence *ufence; 171 172 /** 173 * @attr: The attributes of vma which determines the migration policy 174 * and encoding of the PTEs for this vma. 175 */ 176 struct xe_vma_mem_attr attr; 177 }; 178 179 /** 180 * struct xe_userptr_vma - A userptr vma subclass 181 * @vma: The vma. 182 * @userptr: Additional userptr information. 183 */ 184 struct xe_userptr_vma { 185 struct xe_vma vma; 186 struct xe_userptr userptr; 187 }; 188 189 struct xe_device; 190 191 /** 192 * struct xe_vm_fault_entry - Elements of vm->faults.list 193 * @list: link into @xe_vm.faults.list 194 * @address: address of the fault 195 * @address_precision: precision of faulted address 196 * @access_type: type of address access that resulted in fault 197 * @fault_type: type of fault reported 198 * @fault_level: fault level of the fault 199 */ 200 struct xe_vm_fault_entry { 201 struct list_head list; 202 u64 address; 203 u32 address_precision; 204 u8 access_type; 205 u8 fault_type; 206 u8 fault_level; 207 }; 208 209 struct xe_vm { 210 /** @gpuvm: base GPUVM used to track VMAs */ 211 struct drm_gpuvm gpuvm; 212 213 /** @svm: Shared virtual memory state */ 214 struct { 215 /** @svm.gpusvm: base GPUSVM used to track fault allocations */ 216 struct drm_gpusvm gpusvm; 217 /** 218 * @svm.garbage_collector: Garbage collector which is used unmap 219 * SVM range's GPU bindings and destroy the ranges. 220 */ 221 struct { 222 /** @svm.garbage_collector.lock: Protect's range list */ 223 spinlock_t lock; 224 /** 225 * @svm.garbage_collector.range_list: List of SVM ranges 226 * in the garbage collector. 227 */ 228 struct list_head range_list; 229 /** 230 * @svm.garbage_collector.work: Worker which the 231 * garbage collector runs on. 232 */ 233 struct work_struct work; 234 } garbage_collector; 235 struct xe_pagemap *pagemaps[XE_MAX_TILES_PER_DEVICE]; 236 /** @svm.peer: Used for pagemap connectivity computations. */ 237 struct drm_pagemap_peer peer; 238 } svm; 239 240 struct xe_device *xe; 241 242 /* exec queue used for (un)binding vma's */ 243 struct xe_exec_queue *q[XE_MAX_TILES_PER_DEVICE]; 244 245 /** @lru_bulk_move: Bulk LRU move list for this VM's BOs */ 246 struct ttm_lru_bulk_move lru_bulk_move; 247 248 u64 size; 249 250 struct xe_pt *pt_root[XE_MAX_TILES_PER_DEVICE]; 251 struct xe_pt *scratch_pt[XE_MAX_TILES_PER_DEVICE][XE_VM_MAX_LEVEL]; 252 253 /** 254 * @flags: flags for this VM, statically setup a creation time aside 255 * from XE_VM_FLAG_BANNED which requires vm->lock to set / read safely 256 */ 257 #define XE_VM_FLAG_64K BIT(0) 258 #define XE_VM_FLAG_LR_MODE BIT(1) 259 #define XE_VM_FLAG_MIGRATION BIT(2) 260 #define XE_VM_FLAG_SCRATCH_PAGE BIT(3) 261 #define XE_VM_FLAG_FAULT_MODE BIT(4) 262 #define XE_VM_FLAG_BANNED BIT(5) 263 #define XE_VM_FLAG_TILE_ID(flags) FIELD_GET(GENMASK(7, 6), flags) 264 #define XE_VM_FLAG_SET_TILE_ID(tile) FIELD_PREP(GENMASK(7, 6), (tile)->id) 265 #define XE_VM_FLAG_GSC BIT(8) 266 #define XE_VM_FLAG_NO_VM_OVERCOMMIT BIT(9) 267 unsigned long flags; 268 269 /** 270 * @lock: outer most lock, protects objects of anything attached to this 271 * VM 272 */ 273 struct rw_semaphore lock; 274 /** 275 * @snap_mutex: Mutex used to guard insertions and removals from gpuva, 276 * so we can take a snapshot safely from devcoredump. 277 */ 278 struct mutex snap_mutex; 279 280 /** 281 * @rebind_list: list of VMAs that need rebinding. Protected by the 282 * vm->lock in write mode, OR (the vm->lock in read mode and the 283 * vm resv). 284 */ 285 struct list_head rebind_list; 286 287 /** 288 * @destroy_work: worker to destroy VM, needed as a dma_fence signaling 289 * from an irq context can be last put and the destroy needs to be able 290 * to sleep. 291 */ 292 struct work_struct destroy_work; 293 294 /** 295 * @rftree: range fence tree to track updates to page table structure. 296 * Used to implement conflict tracking between independent bind engines. 297 */ 298 struct xe_range_fence_tree rftree[XE_MAX_TILES_PER_DEVICE]; 299 300 const struct xe_pt_ops *pt_ops; 301 302 /** @userptr: user pointer state */ 303 struct xe_userptr_vm userptr; 304 305 /** @preempt: preempt state */ 306 struct { 307 /** 308 * @min_run_period_ms: The minimum run period before preempting 309 * an engine again 310 */ 311 unsigned int min_run_period_ms; 312 /** @exec_queues: list of exec queues attached to this VM */ 313 struct list_head exec_queues; 314 /** @num_exec_queues: number exec queues attached to this VM */ 315 int num_exec_queues; 316 /** 317 * @rebind_deactivated: Whether rebind has been temporarily deactivated 318 * due to no work available. Protected by the vm resv. 319 */ 320 bool rebind_deactivated; 321 /** 322 * @rebind_work: worker to rebind invalidated userptrs / evicted 323 * BOs 324 */ 325 struct work_struct rebind_work; 326 /** 327 * @preempt.pm_activate_link: Link to list of rebind workers to be 328 * kicked on resume. 329 */ 330 struct list_head pm_activate_link; 331 } preempt; 332 333 /** @exec_queues: Manages list of exec queues attached to this VM, protected by lock. */ 334 struct { 335 /** 336 * @exec_queues.list: list of exec queues attached to this VM, 337 * per GT 338 */ 339 struct list_head list[XE_MAX_TILES_PER_DEVICE * XE_MAX_GT_PER_TILE]; 340 /** 341 * @exec_queues.count: count of exec queues attached to this VM, 342 * per GT 343 */ 344 int count[XE_MAX_TILES_PER_DEVICE * XE_MAX_GT_PER_TILE]; 345 /** @exec_queues.lock: lock to protect exec_queues list */ 346 struct rw_semaphore lock; 347 } exec_queues; 348 349 /** @um: unified memory state */ 350 struct { 351 /** @asid: address space ID, unique to each VM */ 352 u32 asid; 353 /** 354 * @last_fault_vma: Last fault VMA, used for fast lookup when we 355 * get a flood of faults to the same VMA 356 */ 357 struct xe_vma *last_fault_vma; 358 } usm; 359 360 /** @error_capture: allow to track errors */ 361 struct { 362 /** @capture_once: capture only one error per VM */ 363 bool capture_once; 364 } error_capture; 365 366 /** @faults: List of all faults associated with this VM */ 367 struct { 368 /** @faults.lock: lock protecting @faults.list */ 369 spinlock_t lock; 370 /** @faults.list: list of xe_vm_fault_entry entries */ 371 struct list_head list; 372 /** @faults.len: length of @faults.list */ 373 unsigned int len; 374 } faults; 375 376 /** 377 * @validation: Validation data only valid with the vm resv held. 378 * Note: This is really task state of the task holding the vm resv, 379 * and moving forward we should 380 * come up with a better way of passing this down the call- 381 * chain. 382 */ 383 struct { 384 /** 385 * @validation.validating: The task that is currently making bos resident. 386 * for this vm. 387 * Protected by the VM's resv for writing. Opportunistic reading can be done 388 * using READ_ONCE. Note: This is a workaround for the 389 * TTM eviction_valuable() callback not being passed a struct 390 * ttm_operation_context(). Future work might want to address this. 391 */ 392 struct task_struct *validating; 393 /** 394 * @validation.exec The drm_exec context used when locking the vm resv. 395 * Protected by the vm's resv. 396 */ 397 struct drm_exec *_exec; 398 } validation; 399 400 /** 401 * @tlb_flush_seqno: Required TLB flush seqno for the next exec. 402 * protected by the vm resv. 403 */ 404 u64 tlb_flush_seqno; 405 /** @batch_invalidate_tlb: Always invalidate TLB before batch start */ 406 bool batch_invalidate_tlb; 407 /** @xef: Xe file handle for tracking this VM's drm client */ 408 struct xe_file *xef; 409 }; 410 411 /** struct xe_vma_op_map - VMA map operation */ 412 struct xe_vma_op_map { 413 /** @vma: VMA to map */ 414 struct xe_vma *vma; 415 /** @vma_flags: VMA flags for this operation */ 416 unsigned int vma_flags; 417 /** @immediate: Immediate bind */ 418 bool immediate; 419 /** @invalidate_on_bind: Invalidate on bind */ 420 bool invalidate_on_bind; 421 /** @request_decompress: schedule decompression for GPU map */ 422 bool request_decompress; 423 /** @pat_index: The pat index to use for this operation. */ 424 u16 pat_index; 425 }; 426 427 /** struct xe_vma_op_remap - VMA remap operation */ 428 struct xe_vma_op_remap { 429 /** @prev: VMA preceding part of a split mapping */ 430 struct xe_vma *prev; 431 /** @next: VMA subsequent part of a split mapping */ 432 struct xe_vma *next; 433 /** @start: start of the VMA unmap */ 434 u64 start; 435 /** @range: range of the VMA unmap */ 436 u64 range; 437 /** @old_start: Original start of the VMA we unmap */ 438 u64 old_start; 439 /** @old_range: Original range of the VMA we unmap */ 440 u64 old_range; 441 /** @skip_prev: skip prev rebind */ 442 bool skip_prev; 443 /** @skip_next: skip next rebind */ 444 bool skip_next; 445 /** @unmap_done: unmap operation in done */ 446 bool unmap_done; 447 }; 448 449 /** struct xe_vma_op_prefetch - VMA prefetch operation */ 450 struct xe_vma_op_prefetch { 451 /** @region: memory region to prefetch to */ 452 u32 region; 453 }; 454 455 /** struct xe_vma_op_map_range - VMA map range operation */ 456 struct xe_vma_op_map_range { 457 /** @vma: VMA to map (system allocator VMA) */ 458 struct xe_vma *vma; 459 /** @range: SVM range to map */ 460 struct xe_svm_range *range; 461 }; 462 463 /** struct xe_vma_op_unmap_range - VMA unmap range operation */ 464 struct xe_vma_op_unmap_range { 465 /** @range: SVM range to unmap */ 466 struct xe_svm_range *range; 467 }; 468 469 /** struct xe_vma_op_prefetch_range - VMA prefetch range operation */ 470 struct xe_vma_op_prefetch_range { 471 /** @range: xarray for SVM ranges data */ 472 struct xarray range; 473 /** @ranges_count: number of svm ranges to map */ 474 u32 ranges_count; 475 /** 476 * @dpagemap: Pointer to the dpagemap structure containing memory to prefetch. 477 * NULL if prefetch requested region is smem 478 */ 479 struct drm_pagemap *dpagemap; 480 }; 481 482 /** enum xe_vma_op_flags - flags for VMA operation */ 483 enum xe_vma_op_flags { 484 /** @XE_VMA_OP_COMMITTED: VMA operation committed */ 485 XE_VMA_OP_COMMITTED = BIT(0), 486 /** @XE_VMA_OP_PREV_COMMITTED: Previous VMA operation committed */ 487 XE_VMA_OP_PREV_COMMITTED = BIT(1), 488 /** @XE_VMA_OP_NEXT_COMMITTED: Next VMA operation committed */ 489 XE_VMA_OP_NEXT_COMMITTED = BIT(2), 490 }; 491 492 /** enum xe_vma_subop - VMA sub-operation */ 493 enum xe_vma_subop { 494 /** @XE_VMA_SUBOP_MAP_RANGE: Map range */ 495 XE_VMA_SUBOP_MAP_RANGE, 496 /** @XE_VMA_SUBOP_UNMAP_RANGE: Unmap range */ 497 XE_VMA_SUBOP_UNMAP_RANGE, 498 }; 499 500 /** struct xe_vma_op - VMA operation */ 501 struct xe_vma_op { 502 /** @base: GPUVA base operation */ 503 struct drm_gpuva_op base; 504 /** @link: async operation link */ 505 struct list_head link; 506 /** @flags: operation flags */ 507 enum xe_vma_op_flags flags; 508 /** @subop: user defined sub-operation */ 509 enum xe_vma_subop subop; 510 /** @tile_mask: Tile mask for operation */ 511 u8 tile_mask; 512 513 union { 514 /** @map: VMA map operation specific data */ 515 struct xe_vma_op_map map; 516 /** @remap: VMA remap operation specific data */ 517 struct xe_vma_op_remap remap; 518 /** @prefetch: VMA prefetch operation specific data */ 519 struct xe_vma_op_prefetch prefetch; 520 /** @map_range: VMA map range operation specific data */ 521 struct xe_vma_op_map_range map_range; 522 /** @unmap_range: VMA unmap range operation specific data */ 523 struct xe_vma_op_unmap_range unmap_range; 524 /** @prefetch_range: VMA prefetch range operation specific data */ 525 struct xe_vma_op_prefetch_range prefetch_range; 526 }; 527 }; 528 529 /** struct xe_vma_ops - VMA operations */ 530 struct xe_vma_ops { 531 /** @list: list of VMA operations */ 532 struct list_head list; 533 /** @vm: VM */ 534 struct xe_vm *vm; 535 /** @q: exec queue for VMA operations */ 536 struct xe_exec_queue *q; 537 /** @syncs: syncs these operation */ 538 struct xe_sync_entry *syncs; 539 /** @num_syncs: number of syncs */ 540 u32 num_syncs; 541 /** @pt_update_ops: page table update operations */ 542 struct xe_vm_pgtable_update_ops pt_update_ops[XE_MAX_TILES_PER_DEVICE]; 543 /** @flag: signify the properties within xe_vma_ops*/ 544 #define XE_VMA_OPS_FLAG_HAS_SVM_PREFETCH BIT(0) 545 #define XE_VMA_OPS_FLAG_MADVISE BIT(1) 546 #define XE_VMA_OPS_ARRAY_OF_BINDS BIT(2) 547 #define XE_VMA_OPS_FLAG_SKIP_TLB_WAIT BIT(3) 548 #define XE_VMA_OPS_FLAG_ALLOW_SVM_UNMAP BIT(4) 549 u32 flags; 550 #ifdef TEST_VM_OPS_ERROR 551 /** @inject_error: inject error to test error handling */ 552 bool inject_error; 553 #endif 554 }; 555 556 #endif 557