1 /* SPDX-License-Identifier: GPL-2.0-only OR MIT */ 2 /* 3 * Copyright © 2024 Intel Corporation 4 */ 5 6 #ifndef __DRM_GPUSVM_H__ 7 #define __DRM_GPUSVM_H__ 8 9 #include <linux/kref.h> 10 #include <linux/interval_tree.h> 11 #include <linux/mmu_notifier.h> 12 13 struct dev_pagemap_ops; 14 struct drm_device; 15 struct drm_gpusvm; 16 struct drm_gpusvm_notifier; 17 struct drm_gpusvm_ops; 18 struct drm_gpusvm_range; 19 struct drm_gpusvm_devmem; 20 struct drm_pagemap; 21 struct drm_pagemap_device_addr; 22 23 /** 24 * struct drm_gpusvm_devmem_ops - Operations structure for GPU SVM device memory 25 * 26 * This structure defines the operations for GPU Shared Virtual Memory (SVM) 27 * device memory. These operations are provided by the GPU driver to manage device memory 28 * allocations and perform operations such as migration between device memory and system 29 * RAM. 30 */ 31 struct drm_gpusvm_devmem_ops { 32 /** 33 * @devmem_release: Release device memory allocation (optional) 34 * @devmem_allocation: device memory allocation 35 * 36 * Release device memory allocation and drop a reference to device 37 * memory allocation. 38 */ 39 void (*devmem_release)(struct drm_gpusvm_devmem *devmem_allocation); 40 41 /** 42 * @populate_devmem_pfn: Populate device memory PFN (required for migration) 43 * @devmem_allocation: device memory allocation 44 * @npages: Number of pages to populate 45 * @pfn: Array of page frame numbers to populate 46 * 47 * Populate device memory page frame numbers (PFN). 48 * 49 * Return: 0 on success, a negative error code on failure. 50 */ 51 int (*populate_devmem_pfn)(struct drm_gpusvm_devmem *devmem_allocation, 52 unsigned long npages, unsigned long *pfn); 53 54 /** 55 * @copy_to_devmem: Copy to device memory (required for migration) 56 * @pages: Pointer to array of device memory pages (destination) 57 * @dma_addr: Pointer to array of DMA addresses (source) 58 * @npages: Number of pages to copy 59 * 60 * Copy pages to device memory. 61 * 62 * Return: 0 on success, a negative error code on failure. 63 */ 64 int (*copy_to_devmem)(struct page **pages, 65 dma_addr_t *dma_addr, 66 unsigned long npages); 67 68 /** 69 * @copy_to_ram: Copy to system RAM (required for migration) 70 * @pages: Pointer to array of device memory pages (source) 71 * @dma_addr: Pointer to array of DMA addresses (destination) 72 * @npages: Number of pages to copy 73 * 74 * Copy pages to system RAM. 75 * 76 * Return: 0 on success, a negative error code on failure. 77 */ 78 int (*copy_to_ram)(struct page **pages, 79 dma_addr_t *dma_addr, 80 unsigned long npages); 81 }; 82 83 /** 84 * struct drm_gpusvm_devmem - Structure representing a GPU SVM device memory allocation 85 * 86 * @dev: Pointer to the device structure which device memory allocation belongs to 87 * @mm: Pointer to the mm_struct for the address space 88 * @detached: device memory allocations is detached from device pages 89 * @ops: Pointer to the operations structure for GPU SVM device memory 90 * @dpagemap: The struct drm_pagemap of the pages this allocation belongs to. 91 * @size: Size of device memory allocation 92 * @timeslice_expiration: Timeslice expiration in jiffies 93 */ 94 struct drm_gpusvm_devmem { 95 struct device *dev; 96 struct mm_struct *mm; 97 struct completion detached; 98 const struct drm_gpusvm_devmem_ops *ops; 99 struct drm_pagemap *dpagemap; 100 size_t size; 101 u64 timeslice_expiration; 102 }; 103 104 /** 105 * struct drm_gpusvm_ops - Operations structure for GPU SVM 106 * 107 * This structure defines the operations for GPU Shared Virtual Memory (SVM). 108 * These operations are provided by the GPU driver to manage SVM ranges and 109 * notifiers. 110 */ 111 struct drm_gpusvm_ops { 112 /** 113 * @notifier_alloc: Allocate a GPU SVM notifier (optional) 114 * 115 * Allocate a GPU SVM notifier. 116 * 117 * Return: Pointer to the allocated GPU SVM notifier on success, NULL on failure. 118 */ 119 struct drm_gpusvm_notifier *(*notifier_alloc)(void); 120 121 /** 122 * @notifier_free: Free a GPU SVM notifier (optional) 123 * @notifier: Pointer to the GPU SVM notifier to be freed 124 * 125 * Free a GPU SVM notifier. 126 */ 127 void (*notifier_free)(struct drm_gpusvm_notifier *notifier); 128 129 /** 130 * @range_alloc: Allocate a GPU SVM range (optional) 131 * @gpusvm: Pointer to the GPU SVM 132 * 133 * Allocate a GPU SVM range. 134 * 135 * Return: Pointer to the allocated GPU SVM range on success, NULL on failure. 136 */ 137 struct drm_gpusvm_range *(*range_alloc)(struct drm_gpusvm *gpusvm); 138 139 /** 140 * @range_free: Free a GPU SVM range (optional) 141 * @range: Pointer to the GPU SVM range to be freed 142 * 143 * Free a GPU SVM range. 144 */ 145 void (*range_free)(struct drm_gpusvm_range *range); 146 147 /** 148 * @invalidate: Invalidate GPU SVM notifier (required) 149 * @gpusvm: Pointer to the GPU SVM 150 * @notifier: Pointer to the GPU SVM notifier 151 * @mmu_range: Pointer to the mmu_notifier_range structure 152 * 153 * Invalidate the GPU page tables. It can safely walk the notifier range 154 * RB tree/list in this function. Called while holding the notifier lock. 155 */ 156 void (*invalidate)(struct drm_gpusvm *gpusvm, 157 struct drm_gpusvm_notifier *notifier, 158 const struct mmu_notifier_range *mmu_range); 159 }; 160 161 /** 162 * struct drm_gpusvm_notifier - Structure representing a GPU SVM notifier 163 * 164 * @gpusvm: Pointer to the GPU SVM structure 165 * @notifier: MMU interval notifier 166 * @itree: Interval tree node for the notifier (inserted in GPU SVM) 167 * @entry: List entry to fast interval tree traversal 168 * @root: Cached root node of the RB tree containing ranges 169 * @range_list: List head containing of ranges in the same order they appear in 170 * interval tree. This is useful to keep iterating ranges while 171 * doing modifications to RB tree. 172 * @flags: Flags for notifier 173 * @flags.removed: Flag indicating whether the MMU interval notifier has been 174 * removed 175 * 176 * This structure represents a GPU SVM notifier. 177 */ 178 struct drm_gpusvm_notifier { 179 struct drm_gpusvm *gpusvm; 180 struct mmu_interval_notifier notifier; 181 struct interval_tree_node itree; 182 struct list_head entry; 183 struct rb_root_cached root; 184 struct list_head range_list; 185 struct { 186 u32 removed : 1; 187 } flags; 188 }; 189 190 /** 191 * struct drm_gpusvm_range_flags - Structure representing a GPU SVM range flags 192 * 193 * @migrate_devmem: Flag indicating whether the range can be migrated to device memory 194 * @unmapped: Flag indicating if the range has been unmapped 195 * @partial_unmap: Flag indicating if the range has been partially unmapped 196 * @has_devmem_pages: Flag indicating if the range has devmem pages 197 * @has_dma_mapping: Flag indicating if the range has a DMA mapping 198 * @__flags: Flags for range in u16 form (used for READ_ONCE) 199 */ 200 struct drm_gpusvm_range_flags { 201 union { 202 struct { 203 /* All flags below must be set upon creation */ 204 u16 migrate_devmem : 1; 205 /* All flags below must be set / cleared under notifier lock */ 206 u16 unmapped : 1; 207 u16 partial_unmap : 1; 208 u16 has_devmem_pages : 1; 209 u16 has_dma_mapping : 1; 210 }; 211 u16 __flags; 212 }; 213 }; 214 215 /** 216 * struct drm_gpusvm_range - Structure representing a GPU SVM range 217 * 218 * @gpusvm: Pointer to the GPU SVM structure 219 * @notifier: Pointer to the GPU SVM notifier 220 * @refcount: Reference count for the range 221 * @itree: Interval tree node for the range (inserted in GPU SVM notifier) 222 * @entry: List entry to fast interval tree traversal 223 * @notifier_seq: Notifier sequence number of the range's pages 224 * @dma_addr: Device address array 225 * @dpagemap: The struct drm_pagemap of the device pages we're dma-mapping. 226 * Note this is assuming only one drm_pagemap per range is allowed. 227 * @flags: Flags for range 228 * 229 * This structure represents a GPU SVM range used for tracking memory ranges 230 * mapped in a DRM device. 231 */ 232 struct drm_gpusvm_range { 233 struct drm_gpusvm *gpusvm; 234 struct drm_gpusvm_notifier *notifier; 235 struct kref refcount; 236 struct interval_tree_node itree; 237 struct list_head entry; 238 unsigned long notifier_seq; 239 struct drm_pagemap_device_addr *dma_addr; 240 struct drm_pagemap *dpagemap; 241 struct drm_gpusvm_range_flags flags; 242 }; 243 244 /** 245 * struct drm_gpusvm - GPU SVM structure 246 * 247 * @name: Name of the GPU SVM 248 * @drm: Pointer to the DRM device structure 249 * @mm: Pointer to the mm_struct for the address space 250 * @device_private_page_owner: Device private pages owner 251 * @mm_start: Start address of GPU SVM 252 * @mm_range: Range of the GPU SVM 253 * @notifier_size: Size of individual notifiers 254 * @ops: Pointer to the operations structure for GPU SVM 255 * @chunk_sizes: Pointer to the array of chunk sizes used in range allocation. 256 * Entries should be powers of 2 in descending order. 257 * @num_chunks: Number of chunks 258 * @notifier_lock: Read-write semaphore for protecting notifier operations 259 * @root: Cached root node of the Red-Black tree containing GPU SVM notifiers 260 * @notifier_list: list head containing of notifiers in the same order they 261 * appear in interval tree. This is useful to keep iterating 262 * notifiers while doing modifications to RB tree. 263 * 264 * This structure represents a GPU SVM (Shared Virtual Memory) used for tracking 265 * memory ranges mapped in a DRM (Direct Rendering Manager) device. 266 * 267 * No reference counting is provided, as this is expected to be embedded in the 268 * driver VM structure along with the struct drm_gpuvm, which handles reference 269 * counting. 270 */ 271 struct drm_gpusvm { 272 const char *name; 273 struct drm_device *drm; 274 struct mm_struct *mm; 275 void *device_private_page_owner; 276 unsigned long mm_start; 277 unsigned long mm_range; 278 unsigned long notifier_size; 279 const struct drm_gpusvm_ops *ops; 280 const unsigned long *chunk_sizes; 281 int num_chunks; 282 struct rw_semaphore notifier_lock; 283 struct rb_root_cached root; 284 struct list_head notifier_list; 285 #ifdef CONFIG_LOCKDEP 286 /** 287 * @lock_dep_map: Annotates drm_gpusvm_range_find_or_insert and 288 * drm_gpusvm_range_remove with a driver provided lock. 289 */ 290 struct lockdep_map *lock_dep_map; 291 #endif 292 }; 293 294 /** 295 * struct drm_gpusvm_ctx - DRM GPU SVM context 296 * 297 * @check_pages_threshold: Check CPU pages for present if chunk is less than or 298 * equal to threshold. If not present, reduce chunk 299 * size. 300 * @timeslice_ms: The timeslice MS which in minimum time a piece of memory 301 * remains with either exclusive GPU or CPU access. 302 * @in_notifier: entering from a MMU notifier 303 * @read_only: operating on read-only memory 304 * @devmem_possible: possible to use device memory 305 * @devmem_only: use only device memory 306 * 307 * Context that is DRM GPUSVM is operating in (i.e. user arguments). 308 */ 309 struct drm_gpusvm_ctx { 310 unsigned long check_pages_threshold; 311 unsigned long timeslice_ms; 312 unsigned int in_notifier :1; 313 unsigned int read_only :1; 314 unsigned int devmem_possible :1; 315 unsigned int devmem_only :1; 316 }; 317 318 int drm_gpusvm_init(struct drm_gpusvm *gpusvm, 319 const char *name, struct drm_device *drm, 320 struct mm_struct *mm, void *device_private_page_owner, 321 unsigned long mm_start, unsigned long mm_range, 322 unsigned long notifier_size, 323 const struct drm_gpusvm_ops *ops, 324 const unsigned long *chunk_sizes, int num_chunks); 325 326 void drm_gpusvm_fini(struct drm_gpusvm *gpusvm); 327 328 void drm_gpusvm_free(struct drm_gpusvm *gpusvm); 329 330 unsigned long 331 drm_gpusvm_find_vma_start(struct drm_gpusvm *gpusvm, 332 unsigned long start, 333 unsigned long end); 334 335 struct drm_gpusvm_range * 336 drm_gpusvm_range_find_or_insert(struct drm_gpusvm *gpusvm, 337 unsigned long fault_addr, 338 unsigned long gpuva_start, 339 unsigned long gpuva_end, 340 const struct drm_gpusvm_ctx *ctx); 341 342 void drm_gpusvm_range_remove(struct drm_gpusvm *gpusvm, 343 struct drm_gpusvm_range *range); 344 345 int drm_gpusvm_range_evict(struct drm_gpusvm *gpusvm, 346 struct drm_gpusvm_range *range); 347 348 struct drm_gpusvm_range * 349 drm_gpusvm_range_get(struct drm_gpusvm_range *range); 350 351 void drm_gpusvm_range_put(struct drm_gpusvm_range *range); 352 353 bool drm_gpusvm_range_pages_valid(struct drm_gpusvm *gpusvm, 354 struct drm_gpusvm_range *range); 355 356 int drm_gpusvm_range_get_pages(struct drm_gpusvm *gpusvm, 357 struct drm_gpusvm_range *range, 358 const struct drm_gpusvm_ctx *ctx); 359 360 void drm_gpusvm_range_unmap_pages(struct drm_gpusvm *gpusvm, 361 struct drm_gpusvm_range *range, 362 const struct drm_gpusvm_ctx *ctx); 363 364 int drm_gpusvm_migrate_to_devmem(struct drm_gpusvm *gpusvm, 365 struct drm_gpusvm_range *range, 366 struct drm_gpusvm_devmem *devmem_allocation, 367 const struct drm_gpusvm_ctx *ctx); 368 369 int drm_gpusvm_evict_to_ram(struct drm_gpusvm_devmem *devmem_allocation); 370 371 const struct dev_pagemap_ops *drm_gpusvm_pagemap_ops_get(void); 372 373 bool drm_gpusvm_has_mapping(struct drm_gpusvm *gpusvm, unsigned long start, 374 unsigned long end); 375 376 struct drm_gpusvm_range * 377 drm_gpusvm_range_find(struct drm_gpusvm_notifier *notifier, unsigned long start, 378 unsigned long end); 379 380 void drm_gpusvm_range_set_unmapped(struct drm_gpusvm_range *range, 381 const struct mmu_notifier_range *mmu_range); 382 383 void drm_gpusvm_devmem_init(struct drm_gpusvm_devmem *devmem_allocation, 384 struct device *dev, struct mm_struct *mm, 385 const struct drm_gpusvm_devmem_ops *ops, 386 struct drm_pagemap *dpagemap, size_t size); 387 388 #ifdef CONFIG_LOCKDEP 389 /** 390 * drm_gpusvm_driver_set_lock() - Set the lock protecting accesses to GPU SVM 391 * @gpusvm: Pointer to the GPU SVM structure. 392 * @lock: the lock used to protect the gpuva list. The locking primitive 393 * must contain a dep_map field. 394 * 395 * Call this to annotate drm_gpusvm_range_find_or_insert and 396 * drm_gpusvm_range_remove. 397 */ 398 #define drm_gpusvm_driver_set_lock(gpusvm, lock) \ 399 do { \ 400 if (!WARN((gpusvm)->lock_dep_map, \ 401 "GPUSVM range lock should be set only once."))\ 402 (gpusvm)->lock_dep_map = &(lock)->dep_map; \ 403 } while (0) 404 #else 405 #define drm_gpusvm_driver_set_lock(gpusvm, lock) do {} while (0) 406 #endif 407 408 /** 409 * drm_gpusvm_notifier_lock() - Lock GPU SVM notifier 410 * @gpusvm__: Pointer to the GPU SVM structure. 411 * 412 * Abstract client usage GPU SVM notifier lock, take lock 413 */ 414 #define drm_gpusvm_notifier_lock(gpusvm__) \ 415 down_read(&(gpusvm__)->notifier_lock) 416 417 /** 418 * drm_gpusvm_notifier_unlock() - Unlock GPU SVM notifier 419 * @gpusvm__: Pointer to the GPU SVM structure. 420 * 421 * Abstract client usage GPU SVM notifier lock, drop lock 422 */ 423 #define drm_gpusvm_notifier_unlock(gpusvm__) \ 424 up_read(&(gpusvm__)->notifier_lock) 425 426 /** 427 * drm_gpusvm_range_start() - GPU SVM range start address 428 * @range: Pointer to the GPU SVM range 429 * 430 * Return: GPU SVM range start address 431 */ 432 static inline unsigned long 433 drm_gpusvm_range_start(struct drm_gpusvm_range *range) 434 { 435 return range->itree.start; 436 } 437 438 /** 439 * drm_gpusvm_range_end() - GPU SVM range end address 440 * @range: Pointer to the GPU SVM range 441 * 442 * Return: GPU SVM range end address 443 */ 444 static inline unsigned long 445 drm_gpusvm_range_end(struct drm_gpusvm_range *range) 446 { 447 return range->itree.last + 1; 448 } 449 450 /** 451 * drm_gpusvm_range_size() - GPU SVM range size 452 * @range: Pointer to the GPU SVM range 453 * 454 * Return: GPU SVM range size 455 */ 456 static inline unsigned long 457 drm_gpusvm_range_size(struct drm_gpusvm_range *range) 458 { 459 return drm_gpusvm_range_end(range) - drm_gpusvm_range_start(range); 460 } 461 462 /** 463 * drm_gpusvm_notifier_start() - GPU SVM notifier start address 464 * @notifier: Pointer to the GPU SVM notifier 465 * 466 * Return: GPU SVM notifier start address 467 */ 468 static inline unsigned long 469 drm_gpusvm_notifier_start(struct drm_gpusvm_notifier *notifier) 470 { 471 return notifier->itree.start; 472 } 473 474 /** 475 * drm_gpusvm_notifier_end() - GPU SVM notifier end address 476 * @notifier: Pointer to the GPU SVM notifier 477 * 478 * Return: GPU SVM notifier end address 479 */ 480 static inline unsigned long 481 drm_gpusvm_notifier_end(struct drm_gpusvm_notifier *notifier) 482 { 483 return notifier->itree.last + 1; 484 } 485 486 /** 487 * drm_gpusvm_notifier_size() - GPU SVM notifier size 488 * @notifier: Pointer to the GPU SVM notifier 489 * 490 * Return: GPU SVM notifier size 491 */ 492 static inline unsigned long 493 drm_gpusvm_notifier_size(struct drm_gpusvm_notifier *notifier) 494 { 495 return drm_gpusvm_notifier_end(notifier) - 496 drm_gpusvm_notifier_start(notifier); 497 } 498 499 /** 500 * __drm_gpusvm_range_next() - Get the next GPU SVM range in the list 501 * @range: a pointer to the current GPU SVM range 502 * 503 * Return: A pointer to the next drm_gpusvm_range if available, or NULL if the 504 * current range is the last one or if the input range is NULL. 505 */ 506 static inline struct drm_gpusvm_range * 507 __drm_gpusvm_range_next(struct drm_gpusvm_range *range) 508 { 509 if (range && !list_is_last(&range->entry, 510 &range->notifier->range_list)) 511 return list_next_entry(range, entry); 512 513 return NULL; 514 } 515 516 /** 517 * drm_gpusvm_for_each_range() - Iterate over GPU SVM ranges in a notifier 518 * @range__: Iterator variable for the ranges. If set, it indicates the start of 519 * the iterator. If NULL, call drm_gpusvm_range_find() to get the range. 520 * @notifier__: Pointer to the GPU SVM notifier 521 * @start__: Start address of the range 522 * @end__: End address of the range 523 * 524 * This macro is used to iterate over GPU SVM ranges in a notifier. It is safe 525 * to use while holding the driver SVM lock or the notifier lock. 526 */ 527 #define drm_gpusvm_for_each_range(range__, notifier__, start__, end__) \ 528 for ((range__) = (range__) ?: \ 529 drm_gpusvm_range_find((notifier__), (start__), (end__)); \ 530 (range__) && (drm_gpusvm_range_start(range__) < (end__)); \ 531 (range__) = __drm_gpusvm_range_next(range__)) 532 533 #endif /* __DRM_GPUSVM_H__ */ 534