1 /* SPDX-License-Identifier: MIT */ 2 /* 3 * Copyright © 2024 Intel Corporation 4 */ 5 6 #ifndef _XE_SVM_H_ 7 #define _XE_SVM_H_ 8 9 #if IS_ENABLED(CONFIG_DRM_XE_GPUSVM) 10 11 #include <drm/drm_pagemap.h> 12 #include <drm/drm_gpusvm.h> 13 #include <drm/drm_pagemap_util.h> 14 15 #define XE_INTERCONNECT_VRAM DRM_INTERCONNECT_DRIVER 16 #define XE_INTERCONNECT_P2P (XE_INTERCONNECT_VRAM + 1) 17 18 struct drm_device; 19 struct drm_file; 20 21 struct xe_bo; 22 struct xe_gt; 23 struct xe_device; 24 struct xe_vram_region; 25 struct xe_tile; 26 struct xe_vm; 27 struct xe_vma; 28 struct xe_vram_region; 29 30 /** struct xe_svm_range - SVM range */ 31 struct xe_svm_range { 32 /** @base: base drm_gpusvm_range */ 33 struct drm_gpusvm_range base; 34 /** @pages: Page/DMA mapping state for this range (single drm_device). */ 35 struct drm_gpusvm_pages pages; 36 /** 37 * @garbage_collector_link: Link into VM's garbage collect SVM range 38 * list. Protected by VM's garbage collect lock. 39 */ 40 struct list_head garbage_collector_link; 41 /** 42 * @tile_present: Tile mask of binding is present for this range. 43 * Protected by GPU SVM notifier lock. 44 */ 45 u8 tile_present; 46 /** 47 * @tile_invalidated: Tile mask of binding is invalidated for this 48 * range. Protected by GPU SVM notifier lock. 49 */ 50 u8 tile_invalidated; 51 }; 52 53 /** 54 * struct xe_pagemap - Manages xe device_private memory for SVM. 55 * @pagemap: The struct dev_pagemap providing the struct pages. 56 * @dpagemap: The drm_pagemap managing allocation and migration. 57 * @destroy_work: Handles asnynchronous destruction and caching. 58 * @peer: Used for pagemap owner computation. 59 * @hpa_base: The host physical address base for the managemd memory. 60 * @vr: Backpointer to the xe_vram region. 61 */ 62 struct xe_pagemap { 63 struct dev_pagemap pagemap; 64 struct drm_pagemap dpagemap; 65 struct work_struct destroy_work; 66 struct drm_pagemap_peer peer; 67 resource_size_t hpa_base; 68 struct xe_vram_region *vr; 69 }; 70 71 /** 72 * xe_svm_range_pages_valid() - SVM range pages valid 73 * @range: SVM range 74 * 75 * Return: True if SVM range pages are valid, False otherwise 76 */ 77 static inline bool xe_svm_range_pages_valid(struct xe_svm_range *range) 78 { 79 return drm_gpusvm_pages_valid(range->base.gpusvm, &range->pages); 80 } 81 82 int xe_devm_add(struct xe_tile *tile, struct xe_vram_region *vr); 83 84 int xe_svm_init(struct xe_vm *vm); 85 86 void xe_svm_fini(struct xe_vm *vm); 87 88 void xe_svm_close(struct xe_vm *vm); 89 90 int xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma, 91 struct xe_gt *gt, u64 fault_addr, 92 bool atomic); 93 94 bool xe_svm_has_mapping(struct xe_vm *vm, u64 start, u64 end); 95 96 int xe_svm_bo_evict(struct xe_bo *bo); 97 98 void xe_svm_range_debug(struct xe_svm_range *range, const char *operation); 99 100 int xe_svm_alloc_vram(struct xe_svm_range *range, const struct drm_gpusvm_ctx *ctx, 101 struct drm_pagemap *dpagemap); 102 103 struct xe_svm_range *xe_svm_range_find_or_insert(struct xe_vm *vm, u64 addr, 104 struct xe_vma *vma, struct drm_gpusvm_ctx *ctx); 105 106 int xe_svm_range_get_pages(struct xe_vm *vm, struct xe_svm_range *range, 107 struct drm_gpusvm_ctx *ctx); 108 109 bool xe_svm_range_needs_migrate_to_vram(struct xe_svm_range *range, struct xe_vma *vma, 110 const struct drm_pagemap *dpagemap); 111 112 void xe_svm_range_migrate_to_smem(struct xe_vm *vm, struct xe_svm_range *range); 113 114 bool xe_svm_range_validate(struct xe_vm *vm, 115 struct xe_svm_range *range, 116 u8 tile_mask, const struct drm_pagemap *dpagemap); 117 118 u64 xe_svm_find_vma_start(struct xe_vm *vm, u64 addr, u64 end, struct xe_vma *vma); 119 120 void xe_svm_unmap_address_range(struct xe_vm *vm, u64 start, u64 end); 121 122 u8 xe_svm_ranges_zap_ptes_in_range(struct xe_vm *vm, u64 start, u64 end); 123 124 struct drm_pagemap *xe_vma_resolve_pagemap(struct xe_vma *vma, struct xe_tile *tile); 125 126 void *xe_svm_private_page_owner(struct xe_vm *vm, bool force_smem); 127 128 /** 129 * xe_svm_range_has_dma_mapping() - SVM range has DMA mapping 130 * @range: SVM range 131 * 132 * Return: True if SVM range has a DMA mapping, False otherwise 133 */ 134 static inline bool xe_svm_range_has_dma_mapping(struct xe_svm_range *range) 135 { 136 lockdep_assert_held(&range->base.gpusvm->notifier_lock); 137 return range->pages.flags.has_dma_mapping; 138 } 139 140 /** 141 * to_xe_range - Convert a drm_gpusvm_range pointer to a xe_svm_range 142 * @r: Pointer to the drm_gpusvm_range structure 143 * 144 * This function takes a pointer to a drm_gpusvm_range structure and 145 * converts it to a pointer to the containing xe_svm_range structure. 146 * 147 * Return: Pointer to the xe_svm_range structure 148 */ 149 static inline struct xe_svm_range *to_xe_range(struct drm_gpusvm_range *r) 150 { 151 return container_of(r, struct xe_svm_range, base); 152 } 153 154 /** 155 * xe_svm_range_start() - SVM range start address 156 * @range: SVM range 157 * 158 * Return: start address of range. 159 */ 160 static inline unsigned long xe_svm_range_start(struct xe_svm_range *range) 161 { 162 return drm_gpusvm_range_start(&range->base); 163 } 164 165 /** 166 * xe_svm_range_end() - SVM range end address 167 * @range: SVM range 168 * 169 * Return: end address of range. 170 */ 171 static inline unsigned long xe_svm_range_end(struct xe_svm_range *range) 172 { 173 return drm_gpusvm_range_end(&range->base); 174 } 175 176 /** 177 * xe_svm_range_size() - SVM range size 178 * @range: SVM range 179 * 180 * Return: Size of range. 181 */ 182 static inline unsigned long xe_svm_range_size(struct xe_svm_range *range) 183 { 184 return drm_gpusvm_range_size(&range->base); 185 } 186 187 void xe_svm_flush(struct xe_vm *vm); 188 189 int xe_pagemap_shrinker_create(struct xe_device *xe); 190 191 int xe_pagemap_cache_create(struct xe_tile *tile); 192 193 struct drm_pagemap *xe_drm_pagemap_from_fd(int fd, u32 region_instance); 194 195 #else 196 #include <linux/interval_tree.h> 197 #include "xe_vm.h" 198 199 struct drm_pagemap_addr; 200 struct drm_gpusvm_ctx; 201 struct drm_gpusvm_range; 202 struct xe_bo; 203 struct xe_device; 204 struct xe_vm; 205 struct xe_vma; 206 struct xe_tile; 207 struct xe_vram_region; 208 209 #define XE_INTERCONNECT_VRAM 1 210 #define XE_INTERCONNECT_P2P (XE_INTERCONNECT_VRAM + 1) 211 212 struct xe_svm_range { 213 struct { 214 struct interval_tree_node itree; 215 } base; 216 struct { 217 const struct drm_pagemap_addr *dma_addr; 218 } pages; 219 u32 tile_present; 220 u32 tile_invalidated; 221 }; 222 223 static inline bool xe_svm_range_pages_valid(struct xe_svm_range *range) 224 { 225 return false; 226 } 227 228 static inline 229 int xe_devm_add(struct xe_tile *tile, struct xe_vram_region *vr) 230 { 231 return 0; 232 } 233 234 static inline 235 int xe_svm_init(struct xe_vm *vm) 236 { 237 #if IS_ENABLED(CONFIG_DRM_GPUSVM) 238 return drm_gpusvm_init(&vm->svm.gpusvm, "Xe SVM (simple)", 239 NULL, 0, 0, 0, NULL, NULL, 0); 240 #else 241 return 0; 242 #endif 243 } 244 245 static inline 246 void xe_svm_fini(struct xe_vm *vm) 247 { 248 #if IS_ENABLED(CONFIG_DRM_GPUSVM) 249 xe_assert(vm->xe, xe_vm_is_closed(vm)); 250 drm_gpusvm_fini(&vm->svm.gpusvm); 251 #endif 252 } 253 254 static inline 255 void xe_svm_close(struct xe_vm *vm) 256 { 257 } 258 259 static inline 260 int xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma, 261 struct xe_gt *gt, u64 fault_addr, 262 bool atomic) 263 { 264 return 0; 265 } 266 267 static inline 268 bool xe_svm_has_mapping(struct xe_vm *vm, u64 start, u64 end) 269 { 270 return false; 271 } 272 273 static inline 274 int xe_svm_bo_evict(struct xe_bo *bo) 275 { 276 return 0; 277 } 278 279 static inline 280 void xe_svm_range_debug(struct xe_svm_range *range, const char *operation) 281 { 282 } 283 284 static inline int 285 xe_svm_alloc_vram(struct xe_svm_range *range, const struct drm_gpusvm_ctx *ctx, 286 struct drm_pagemap *dpagemap) 287 { 288 return -EOPNOTSUPP; 289 } 290 291 static inline 292 struct xe_svm_range *xe_svm_range_find_or_insert(struct xe_vm *vm, u64 addr, 293 struct xe_vma *vma, struct drm_gpusvm_ctx *ctx) 294 { 295 return ERR_PTR(-EINVAL); 296 } 297 298 static inline 299 int xe_svm_range_get_pages(struct xe_vm *vm, struct xe_svm_range *range, 300 struct drm_gpusvm_ctx *ctx) 301 { 302 return -EINVAL; 303 } 304 305 static inline struct xe_svm_range *to_xe_range(struct drm_gpusvm_range *r) 306 { 307 return NULL; 308 } 309 310 static inline unsigned long xe_svm_range_start(struct xe_svm_range *range) 311 { 312 return 0; 313 } 314 315 static inline unsigned long xe_svm_range_end(struct xe_svm_range *range) 316 { 317 return 0; 318 } 319 320 static inline unsigned long xe_svm_range_size(struct xe_svm_range *range) 321 { 322 return 0; 323 } 324 325 static inline 326 bool xe_svm_range_needs_migrate_to_vram(struct xe_svm_range *range, struct xe_vma *vma, 327 const struct drm_pagemap *dpagemap) 328 { 329 return false; 330 } 331 332 static inline 333 void xe_svm_range_migrate_to_smem(struct xe_vm *vm, struct xe_svm_range *range) 334 { 335 } 336 337 static inline 338 bool xe_svm_range_validate(struct xe_vm *vm, 339 struct xe_svm_range *range, 340 u8 tile_mask, bool devmem_preferred) 341 { 342 return false; 343 } 344 345 static inline 346 u64 xe_svm_find_vma_start(struct xe_vm *vm, u64 addr, u64 end, struct xe_vma *vma) 347 { 348 return ULONG_MAX; 349 } 350 351 static inline 352 void xe_svm_unmap_address_range(struct xe_vm *vm, u64 start, u64 end) 353 { 354 } 355 356 static inline 357 u8 xe_svm_ranges_zap_ptes_in_range(struct xe_vm *vm, u64 start, u64 end) 358 { 359 return 0; 360 } 361 362 static inline 363 struct drm_pagemap *xe_vma_resolve_pagemap(struct xe_vma *vma, struct xe_tile *tile) 364 { 365 return NULL; 366 } 367 368 static inline void *xe_svm_private_page_owner(struct xe_vm *vm, bool force_smem) 369 { 370 return NULL; 371 } 372 373 static inline void xe_svm_flush(struct xe_vm *vm) 374 { 375 } 376 377 static inline int xe_pagemap_shrinker_create(struct xe_device *xe) 378 { 379 return 0; 380 } 381 382 static inline int xe_pagemap_cache_create(struct xe_tile *tile) 383 { 384 return 0; 385 } 386 387 static inline struct drm_pagemap *xe_drm_pagemap_from_fd(int fd, u32 region_instance) 388 { 389 return ERR_PTR(-ENOENT); 390 } 391 392 #define xe_svm_range_has_dma_mapping(...) false 393 #endif /* CONFIG_DRM_XE_GPUSVM */ 394 395 #if IS_ENABLED(CONFIG_DRM_GPUSVM) /* Need to support userptr without XE_GPUSVM */ 396 #define xe_svm_assert_in_notifier(vm__) \ 397 lockdep_assert_held_write(&(vm__)->svm.gpusvm.notifier_lock) 398 399 /* 400 * Assert the svm notifier_lock is held. Read mode by default; write mode 401 * when CONFIG_DRM_XE_USERPTR_INVAL_INJECT is on, because that path forces 402 * a userptr invalidation that ends in drm_gpusvm_unmap_pages() with 403 * ctx->in_notifier=true, which requires the lock held for write. 404 */ 405 #if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT) 406 #define xe_svm_assert_held_read_or_inject_write(vm__) \ 407 lockdep_assert_held_write(&(vm__)->svm.gpusvm.notifier_lock) 408 #else 409 #define xe_svm_assert_held_read_or_inject_write(vm__) \ 410 lockdep_assert_held_read(&(vm__)->svm.gpusvm.notifier_lock) 411 #endif 412 413 #define xe_svm_notifier_lock(vm__) \ 414 drm_gpusvm_notifier_lock(&(vm__)->svm.gpusvm) 415 416 #define xe_svm_notifier_lock_interruptible(vm__) \ 417 down_read_interruptible(&(vm__)->svm.gpusvm.notifier_lock) 418 419 #define xe_svm_notifier_unlock(vm__) \ 420 drm_gpusvm_notifier_unlock(&(vm__)->svm.gpusvm) 421 422 #else 423 #define xe_svm_assert_in_notifier(...) do {} while (0) 424 425 static inline void xe_svm_assert_held_read_or_inject_write(struct xe_vm *vm) 426 { 427 } 428 429 static inline void xe_svm_notifier_lock(struct xe_vm *vm) 430 { 431 } 432 433 static inline int xe_svm_notifier_lock_interruptible(struct xe_vm *vm) 434 { 435 return 0; 436 } 437 438 static inline void xe_svm_notifier_unlock(struct xe_vm *vm) 439 { 440 } 441 #endif /* CONFIG_DRM_GPUSVM */ 442 443 #endif 444