1 /* SPDX-License-Identifier: MIT */ 2 /* 3 * Copyright © 2024 Intel Corporation 4 */ 5 6 #ifndef _XE_SVM_H_ 7 #define _XE_SVM_H_ 8 9 #if IS_ENABLED(CONFIG_DRM_XE_GPUSVM) 10 11 #include <drm/drm_pagemap.h> 12 #include <drm/drm_gpusvm.h> 13 #include <drm/drm_pagemap_util.h> 14 15 #define XE_INTERCONNECT_VRAM DRM_INTERCONNECT_DRIVER 16 #define XE_INTERCONNECT_P2P (XE_INTERCONNECT_VRAM + 1) 17 18 struct drm_device; 19 struct drm_file; 20 21 struct xe_bo; 22 struct xe_gt; 23 struct xe_device; 24 struct xe_pagefault; 25 struct xe_vram_region; 26 struct xe_tile; 27 struct xe_vm; 28 struct xe_vma; 29 struct xe_vram_region; 30 31 /** struct xe_svm_range - SVM range */ 32 struct xe_svm_range { 33 /** @base: base drm_gpusvm_range */ 34 struct drm_gpusvm_range base; 35 /** @pages: Page/DMA mapping state for this range (single drm_device). */ 36 struct drm_gpusvm_pages pages; 37 /** 38 * @garbage_collector_link: Link into VM's garbage collect SVM range 39 * list. Protected by VM's garbage collect lock. 40 */ 41 struct list_head garbage_collector_link; 42 /** 43 * @lock: Protects fault handler, garbage collector, and prefetch 44 * critical sections, ensuring only one thread operates on a range at a 45 * time. Locking order: inside vm->lock and garbage collector, outside 46 * dma-resv locks, vm->svm.range_lock. 47 */ 48 struct mutex lock; 49 /** 50 * @tile_present: Tile mask of binding is present for this range. 51 * Protected by GPU SVM notifier lock. 52 */ 53 u8 tile_present; 54 /** 55 * @tile_invalidated: Tile mask of binding is invalidated for this 56 * range. Protected by GPU SVM notifier lock. 57 */ 58 u8 tile_invalidated; 59 /** 60 * @removed: Range has been removed from GPU SVM tree, protected by 61 * @lock. 62 */ 63 bool removed; 64 }; 65 66 /** 67 * xe_svm_range_put() - SVM range put 68 * @range: SVM range 69 */ 70 static inline void xe_svm_range_put(struct xe_svm_range *range) 71 { 72 drm_gpusvm_range_put(&range->base); 73 } 74 75 /** 76 * struct xe_pagemap - Manages xe device_private memory for SVM. 77 * @pagemap: The struct dev_pagemap providing the struct pages. 78 * @dpagemap: The drm_pagemap managing allocation and migration. 79 * @destroy_work: Handles asnynchronous destruction and caching. 80 * @peer: Used for pagemap owner computation. 81 * @hpa_base: The host physical address base for the managemd memory. 82 * @vr: Backpointer to the xe_vram region. 83 */ 84 struct xe_pagemap { 85 struct dev_pagemap pagemap; 86 struct drm_pagemap dpagemap; 87 struct work_struct destroy_work; 88 struct drm_pagemap_peer peer; 89 resource_size_t hpa_base; 90 struct xe_vram_region *vr; 91 }; 92 93 /** 94 * xe_svm_range_pages_valid() - SVM range pages valid 95 * @range: SVM range 96 * 97 * Return: True if SVM range pages are valid, False otherwise 98 */ 99 static inline bool xe_svm_range_pages_valid(struct xe_svm_range *range) 100 { 101 return drm_gpusvm_pages_valid(range->base.gpusvm, &range->pages); 102 } 103 104 int xe_devm_add(struct xe_tile *tile, struct xe_vram_region *vr); 105 106 int xe_svm_init(struct xe_vm *vm); 107 108 void xe_svm_fini(struct xe_vm *vm); 109 110 void xe_svm_close(struct xe_vm *vm); 111 112 int xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma, 113 struct xe_pagefault *pf, struct xe_gt *gt, 114 u64 fault_addr, bool atomic); 115 116 bool xe_svm_has_mapping(struct xe_vm *vm, u64 start, u64 end); 117 118 int xe_svm_bo_evict(struct xe_bo *bo); 119 120 void xe_svm_range_debug(struct xe_svm_range *range, const char *operation); 121 122 int xe_svm_alloc_vram(struct xe_svm_range *range, const struct drm_gpusvm_ctx *ctx, 123 struct drm_pagemap *dpagemap); 124 125 struct xe_svm_range *xe_svm_range_find_or_insert(struct xe_vm *vm, u64 addr, 126 struct xe_vma *vma, struct drm_gpusvm_ctx *ctx); 127 128 int xe_svm_range_get_pages(struct xe_vm *vm, struct xe_svm_range *range, 129 struct drm_gpusvm_ctx *ctx); 130 131 void xe_svm_devmem_lru_bump(struct drm_pagemap_devmem *devmem_allocation); 132 133 void xe_svm_range_prefetch_lru_bump(struct xe_vm *vm, struct xe_vma *vma, 134 struct xe_svm_range *range, 135 struct drm_pagemap *dpagemap); 136 137 bool xe_svm_range_needs_migrate_to_vram(struct xe_svm_range *range, struct xe_vma *vma, 138 const struct drm_pagemap *dpagemap); 139 140 void xe_svm_range_migrate_to_smem(struct xe_vm *vm, struct xe_svm_range *range); 141 142 bool xe_svm_range_validate(struct xe_vm *vm, 143 struct xe_svm_range *range, 144 u8 tile_mask, const struct drm_pagemap *dpagemap, 145 bool *valid_pages); 146 147 u64 xe_svm_find_vma_start(struct xe_vm *vm, u64 addr, u64 end, struct xe_vma *vma); 148 149 void xe_svm_unmap_address_range(struct xe_vm *vm, u64 start, u64 end); 150 151 u8 xe_svm_ranges_zap_ptes_in_range(struct xe_vm *vm, u64 start, u64 end); 152 153 struct drm_pagemap *xe_vma_resolve_pagemap(struct xe_vma *vma, struct xe_tile *tile); 154 155 void *xe_svm_private_page_owner(struct xe_vm *vm, bool force_smem); 156 157 /** 158 * xe_svm_range_has_dma_mapping() - SVM range has DMA mapping 159 * @range: SVM range 160 * 161 * Return: True if SVM range has a DMA mapping, False otherwise 162 */ 163 static inline bool xe_svm_range_has_dma_mapping(struct xe_svm_range *range) 164 { 165 lockdep_assert_held(&range->base.gpusvm->notifier_lock); 166 return range->pages.flags.has_dma_mapping; 167 } 168 169 /** 170 * xe_svm_range_is_removed() - SVM range is removed from GPU SVM tree 171 * @range: SVM range 172 * 173 * Return: True if SVM range is removed from GPU SVM tree, False otherwise 174 */ 175 static inline bool xe_svm_range_is_removed(struct xe_svm_range *range) 176 { 177 lockdep_assert_held(&range->lock); 178 179 return range->removed; 180 } 181 182 /** 183 * to_xe_range - Convert a drm_gpusvm_range pointer to a xe_svm_range 184 * @r: Pointer to the drm_gpusvm_range structure 185 * 186 * This function takes a pointer to a drm_gpusvm_range structure and 187 * converts it to a pointer to the containing xe_svm_range structure. 188 * 189 * Return: Pointer to the xe_svm_range structure 190 */ 191 static inline struct xe_svm_range *to_xe_range(struct drm_gpusvm_range *r) 192 { 193 return container_of(r, struct xe_svm_range, base); 194 } 195 196 /** 197 * xe_svm_range_start() - SVM range start address 198 * @range: SVM range 199 * 200 * Return: start address of range. 201 */ 202 static inline unsigned long xe_svm_range_start(struct xe_svm_range *range) 203 { 204 return drm_gpusvm_range_start(&range->base); 205 } 206 207 /** 208 * xe_svm_range_end() - SVM range end address 209 * @range: SVM range 210 * 211 * Return: end address of range. 212 */ 213 static inline unsigned long xe_svm_range_end(struct xe_svm_range *range) 214 { 215 return drm_gpusvm_range_end(&range->base); 216 } 217 218 /** 219 * xe_svm_range_size() - SVM range size 220 * @range: SVM range 221 * 222 * Return: Size of range. 223 */ 224 static inline unsigned long xe_svm_range_size(struct xe_svm_range *range) 225 { 226 return drm_gpusvm_range_size(&range->base); 227 } 228 229 /** 230 * xe_svm_range_first_dma() - Resolve the device address array of a SVM range 231 * @range: SVM range 232 * @contiguous: Where to store whether one entry spans the whole range 233 * 234 * Return: Pointer to the first device address, NULL if none is populated. 235 */ 236 static inline const struct drm_pagemap_addr * 237 xe_svm_range_first_dma(struct xe_svm_range *range, bool *contiguous) 238 { 239 return drm_gpusvm_pages_first_dma(&range->pages, contiguous); 240 } 241 242 void xe_svm_flush(struct xe_vm *vm); 243 244 int xe_pagemap_shrinker_create(struct xe_device *xe); 245 246 int xe_pagemap_cache_create(struct xe_tile *tile); 247 248 struct drm_pagemap *xe_drm_pagemap_from_fd(int fd, u32 region_instance); 249 250 #else 251 #include <linux/interval_tree.h> 252 #include "xe_vm.h" 253 254 struct drm_pagemap_addr; 255 struct drm_gpusvm_ctx; 256 struct drm_gpusvm_range; 257 struct xe_bo; 258 struct xe_device; 259 struct xe_vm; 260 struct xe_vma; 261 struct xe_tile; 262 struct xe_vram_region; 263 264 #define XE_INTERCONNECT_VRAM 1 265 #define XE_INTERCONNECT_P2P (XE_INTERCONNECT_VRAM + 1) 266 267 struct xe_svm_range { 268 struct { 269 struct interval_tree_node itree; 270 } base; 271 struct { 272 const struct drm_pagemap_addr *dma_addr; 273 } pages; 274 struct mutex lock; 275 u32 tile_present; 276 u32 tile_invalidated; 277 }; 278 279 static inline void xe_svm_range_put(struct xe_svm_range *range) 280 { 281 } 282 283 static inline bool xe_svm_range_pages_valid(struct xe_svm_range *range) 284 { 285 return false; 286 } 287 288 static inline 289 int xe_devm_add(struct xe_tile *tile, struct xe_vram_region *vr) 290 { 291 return 0; 292 } 293 294 static inline 295 int xe_svm_init(struct xe_vm *vm) 296 { 297 #if IS_ENABLED(CONFIG_DRM_GPUSVM) 298 return drm_gpusvm_init(&vm->svm.gpusvm, "Xe SVM (simple)", 299 NULL, 0, 0, 0, NULL, NULL, 0); 300 #else 301 return 0; 302 #endif 303 } 304 305 static inline 306 void xe_svm_fini(struct xe_vm *vm) 307 { 308 #if IS_ENABLED(CONFIG_DRM_GPUSVM) 309 xe_assert(vm->xe, xe_vm_is_closed(vm)); 310 drm_gpusvm_fini(&vm->svm.gpusvm); 311 #endif 312 } 313 314 static inline 315 void xe_svm_close(struct xe_vm *vm) 316 { 317 } 318 319 static inline 320 int xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma, 321 struct xe_pagefault *pf, struct xe_gt *gt, 322 u64 fault_addr, bool atomic) 323 { 324 return 0; 325 } 326 327 static inline 328 bool xe_svm_has_mapping(struct xe_vm *vm, u64 start, u64 end) 329 { 330 return false; 331 } 332 333 static inline 334 int xe_svm_bo_evict(struct xe_bo *bo) 335 { 336 return 0; 337 } 338 339 static inline 340 void xe_svm_range_debug(struct xe_svm_range *range, const char *operation) 341 { 342 } 343 344 static inline int 345 xe_svm_alloc_vram(struct xe_svm_range *range, const struct drm_gpusvm_ctx *ctx, 346 struct drm_pagemap *dpagemap) 347 { 348 return -EOPNOTSUPP; 349 } 350 351 static inline 352 struct xe_svm_range *xe_svm_range_find_or_insert(struct xe_vm *vm, u64 addr, 353 struct xe_vma *vma, struct drm_gpusvm_ctx *ctx) 354 { 355 return ERR_PTR(-EINVAL); 356 } 357 358 static inline 359 int xe_svm_range_get_pages(struct xe_vm *vm, struct xe_svm_range *range, 360 struct drm_gpusvm_ctx *ctx) 361 { 362 return -EINVAL; 363 } 364 365 static inline 366 void xe_svm_devmem_lru_bump(struct drm_pagemap_devmem *devmem_allocation) 367 { 368 } 369 370 static inline 371 void xe_svm_range_prefetch_lru_bump(struct xe_vm *vm, struct xe_vma *vma, 372 struct xe_svm_range *range, 373 struct drm_pagemap *dpagemap) 374 { 375 } 376 377 static inline struct xe_svm_range *to_xe_range(struct drm_gpusvm_range *r) 378 { 379 return NULL; 380 } 381 382 static inline unsigned long xe_svm_range_start(struct xe_svm_range *range) 383 { 384 return 0; 385 } 386 387 static inline unsigned long xe_svm_range_end(struct xe_svm_range *range) 388 { 389 return 0; 390 } 391 392 static inline unsigned long xe_svm_range_size(struct xe_svm_range *range) 393 { 394 return 0; 395 } 396 397 static inline 398 bool xe_svm_range_needs_migrate_to_vram(struct xe_svm_range *range, struct xe_vma *vma, 399 const struct drm_pagemap *dpagemap) 400 { 401 return false; 402 } 403 404 static inline 405 void xe_svm_range_migrate_to_smem(struct xe_vm *vm, struct xe_svm_range *range) 406 { 407 } 408 409 static inline 410 bool xe_svm_range_validate(struct xe_vm *vm, 411 struct xe_svm_range *range, 412 u8 tile_mask, const struct drm_pagemap *dpagemap, 413 bool *valid_pages) 414 { 415 return false; 416 } 417 418 static inline 419 u64 xe_svm_find_vma_start(struct xe_vm *vm, u64 addr, u64 end, struct xe_vma *vma) 420 { 421 return ULONG_MAX; 422 } 423 424 static inline 425 void xe_svm_unmap_address_range(struct xe_vm *vm, u64 start, u64 end) 426 { 427 } 428 429 static inline 430 u8 xe_svm_ranges_zap_ptes_in_range(struct xe_vm *vm, u64 start, u64 end) 431 { 432 return 0; 433 } 434 435 static inline 436 struct drm_pagemap *xe_vma_resolve_pagemap(struct xe_vma *vma, struct xe_tile *tile) 437 { 438 return NULL; 439 } 440 441 static inline void *xe_svm_private_page_owner(struct xe_vm *vm, bool force_smem) 442 { 443 return NULL; 444 } 445 446 static inline void xe_svm_flush(struct xe_vm *vm) 447 { 448 } 449 450 static inline int xe_pagemap_shrinker_create(struct xe_device *xe) 451 { 452 return 0; 453 } 454 455 static inline int xe_pagemap_cache_create(struct xe_tile *tile) 456 { 457 return 0; 458 } 459 460 static inline struct drm_pagemap *xe_drm_pagemap_from_fd(int fd, u32 region_instance) 461 { 462 return ERR_PTR(-ENOENT); 463 } 464 465 static inline bool xe_svm_range_is_removed(struct xe_svm_range *range) 466 { 467 return false; 468 } 469 470 static inline const struct drm_pagemap_addr * 471 xe_svm_range_first_dma(struct xe_svm_range *range, bool *contiguous) 472 { 473 *contiguous = false; 474 return NULL; 475 } 476 477 #define xe_svm_range_has_dma_mapping(...) false 478 #endif /* CONFIG_DRM_XE_GPUSVM */ 479 480 #if IS_ENABLED(CONFIG_DRM_GPUSVM) /* Need to support userptr without XE_GPUSVM */ 481 #define xe_svm_assert_in_notifier(vm__) \ 482 lockdep_assert_held_write(&(vm__)->svm.gpusvm.notifier_lock) 483 484 /* 485 * Assert the svm notifier_lock is held. Read mode by default; write mode 486 * when CONFIG_DRM_XE_USERPTR_INVAL_INJECT is on, because that path forces 487 * a userptr invalidation that ends in drm_gpusvm_unmap_pages() with 488 * ctx->in_notifier=true, which requires the lock held for write. 489 */ 490 #if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT) 491 #define xe_svm_assert_held_read_or_inject_write(vm__) \ 492 lockdep_assert_held_write(&(vm__)->svm.gpusvm.notifier_lock) 493 #else 494 #define xe_svm_assert_held_read_or_inject_write(vm__) \ 495 lockdep_assert_held_read(&(vm__)->svm.gpusvm.notifier_lock) 496 #endif 497 498 #define xe_svm_notifier_lock(vm__) \ 499 drm_gpusvm_notifier_lock(&(vm__)->svm.gpusvm) 500 501 #define xe_svm_notifier_lock_interruptible(vm__) \ 502 down_read_interruptible(&(vm__)->svm.gpusvm.notifier_lock) 503 504 #define xe_svm_notifier_unlock(vm__) \ 505 drm_gpusvm_notifier_unlock(&(vm__)->svm.gpusvm) 506 507 #else 508 #define xe_svm_assert_in_notifier(...) do {} while (0) 509 510 static inline void xe_svm_assert_held_read_or_inject_write(struct xe_vm *vm) 511 { 512 } 513 514 static inline void xe_svm_notifier_lock(struct xe_vm *vm) 515 { 516 } 517 518 static inline int xe_svm_notifier_lock_interruptible(struct xe_vm *vm) 519 { 520 return 0; 521 } 522 523 static inline void xe_svm_notifier_unlock(struct xe_vm *vm) 524 { 525 } 526 #endif /* CONFIG_DRM_GPUSVM */ 527 528 #endif 529