1 /* SPDX-License-Identifier: GPL-2.0-only OR MIT */ 2 /* 3 * Copyright © 2024 Intel Corporation 4 */ 5 6 #ifndef __DRM_GPUSVM_H__ 7 #define __DRM_GPUSVM_H__ 8 9 #include <linux/kref.h> 10 #include <linux/interval_tree.h> 11 #include <linux/mmu_notifier.h> 12 13 struct dev_pagemap_ops; 14 struct drm_device; 15 struct drm_gpusvm; 16 struct drm_gpusvm_notifier; 17 struct drm_gpusvm_ops; 18 struct drm_gpusvm_range; 19 struct drm_gpusvm_devmem; 20 struct drm_pagemap; 21 struct drm_pagemap_device_addr; 22 23 /** 24 * struct drm_gpusvm_devmem_ops - Operations structure for GPU SVM device memory 25 * 26 * This structure defines the operations for GPU Shared Virtual Memory (SVM) 27 * device memory. These operations are provided by the GPU driver to manage device memory 28 * allocations and perform operations such as migration between device memory and system 29 * RAM. 30 */ 31 struct drm_gpusvm_devmem_ops { 32 /** 33 * @devmem_release: Release device memory allocation (optional) 34 * @devmem_allocation: device memory allocation 35 * 36 * Release device memory allocation and drop a reference to device 37 * memory allocation. 38 */ 39 void (*devmem_release)(struct drm_gpusvm_devmem *devmem_allocation); 40 41 /** 42 * @populate_devmem_pfn: Populate device memory PFN (required for migration) 43 * @devmem_allocation: device memory allocation 44 * @npages: Number of pages to populate 45 * @pfn: Array of page frame numbers to populate 46 * 47 * Populate device memory page frame numbers (PFN). 48 * 49 * Return: 0 on success, a negative error code on failure. 50 */ 51 int (*populate_devmem_pfn)(struct drm_gpusvm_devmem *devmem_allocation, 52 unsigned long npages, unsigned long *pfn); 53 54 /** 55 * @copy_to_devmem: Copy to device memory (required for migration) 56 * @pages: Pointer to array of device memory pages (destination) 57 * @dma_addr: Pointer to array of DMA addresses (source) 58 * @npages: Number of pages to copy 59 * 60 * Copy pages to device memory. 61 * 62 * Return: 0 on success, a negative error code on failure. 63 */ 64 int (*copy_to_devmem)(struct page **pages, 65 dma_addr_t *dma_addr, 66 unsigned long npages); 67 68 /** 69 * @copy_to_ram: Copy to system RAM (required for migration) 70 * @pages: Pointer to array of device memory pages (source) 71 * @dma_addr: Pointer to array of DMA addresses (destination) 72 * @npages: Number of pages to copy 73 * 74 * Copy pages to system RAM. 75 * 76 * Return: 0 on success, a negative error code on failure. 77 */ 78 int (*copy_to_ram)(struct page **pages, 79 dma_addr_t *dma_addr, 80 unsigned long npages); 81 }; 82 83 /** 84 * struct drm_gpusvm_devmem - Structure representing a GPU SVM device memory allocation 85 * 86 * @dev: Pointer to the device structure which device memory allocation belongs to 87 * @mm: Pointer to the mm_struct for the address space 88 * @detached: device memory allocations is detached from device pages 89 * @ops: Pointer to the operations structure for GPU SVM device memory 90 * @dpagemap: The struct drm_pagemap of the pages this allocation belongs to. 91 * @size: Size of device memory allocation 92 */ 93 struct drm_gpusvm_devmem { 94 struct device *dev; 95 struct mm_struct *mm; 96 struct completion detached; 97 const struct drm_gpusvm_devmem_ops *ops; 98 struct drm_pagemap *dpagemap; 99 size_t size; 100 }; 101 102 /** 103 * struct drm_gpusvm_ops - Operations structure for GPU SVM 104 * 105 * This structure defines the operations for GPU Shared Virtual Memory (SVM). 106 * These operations are provided by the GPU driver to manage SVM ranges and 107 * notifiers. 108 */ 109 struct drm_gpusvm_ops { 110 /** 111 * @notifier_alloc: Allocate a GPU SVM notifier (optional) 112 * 113 * Allocate a GPU SVM notifier. 114 * 115 * Return: Pointer to the allocated GPU SVM notifier on success, NULL on failure. 116 */ 117 struct drm_gpusvm_notifier *(*notifier_alloc)(void); 118 119 /** 120 * @notifier_free: Free a GPU SVM notifier (optional) 121 * @notifier: Pointer to the GPU SVM notifier to be freed 122 * 123 * Free a GPU SVM notifier. 124 */ 125 void (*notifier_free)(struct drm_gpusvm_notifier *notifier); 126 127 /** 128 * @range_alloc: Allocate a GPU SVM range (optional) 129 * @gpusvm: Pointer to the GPU SVM 130 * 131 * Allocate a GPU SVM range. 132 * 133 * Return: Pointer to the allocated GPU SVM range on success, NULL on failure. 134 */ 135 struct drm_gpusvm_range *(*range_alloc)(struct drm_gpusvm *gpusvm); 136 137 /** 138 * @range_free: Free a GPU SVM range (optional) 139 * @range: Pointer to the GPU SVM range to be freed 140 * 141 * Free a GPU SVM range. 142 */ 143 void (*range_free)(struct drm_gpusvm_range *range); 144 145 /** 146 * @invalidate: Invalidate GPU SVM notifier (required) 147 * @gpusvm: Pointer to the GPU SVM 148 * @notifier: Pointer to the GPU SVM notifier 149 * @mmu_range: Pointer to the mmu_notifier_range structure 150 * 151 * Invalidate the GPU page tables. It can safely walk the notifier range 152 * RB tree/list in this function. Called while holding the notifier lock. 153 */ 154 void (*invalidate)(struct drm_gpusvm *gpusvm, 155 struct drm_gpusvm_notifier *notifier, 156 const struct mmu_notifier_range *mmu_range); 157 }; 158 159 /** 160 * struct drm_gpusvm_notifier - Structure representing a GPU SVM notifier 161 * 162 * @gpusvm: Pointer to the GPU SVM structure 163 * @notifier: MMU interval notifier 164 * @itree: Interval tree node for the notifier (inserted in GPU SVM) 165 * @entry: List entry to fast interval tree traversal 166 * @root: Cached root node of the RB tree containing ranges 167 * @range_list: List head containing of ranges in the same order they appear in 168 * interval tree. This is useful to keep iterating ranges while 169 * doing modifications to RB tree. 170 * @flags: Flags for notifier 171 * @flags.removed: Flag indicating whether the MMU interval notifier has been 172 * removed 173 * 174 * This structure represents a GPU SVM notifier. 175 */ 176 struct drm_gpusvm_notifier { 177 struct drm_gpusvm *gpusvm; 178 struct mmu_interval_notifier notifier; 179 struct interval_tree_node itree; 180 struct list_head entry; 181 struct rb_root_cached root; 182 struct list_head range_list; 183 struct { 184 u32 removed : 1; 185 } flags; 186 }; 187 188 /** 189 * struct drm_gpusvm_range - Structure representing a GPU SVM range 190 * 191 * @gpusvm: Pointer to the GPU SVM structure 192 * @notifier: Pointer to the GPU SVM notifier 193 * @refcount: Reference count for the range 194 * @itree: Interval tree node for the range (inserted in GPU SVM notifier) 195 * @entry: List entry to fast interval tree traversal 196 * @notifier_seq: Notifier sequence number of the range's pages 197 * @dma_addr: Device address array 198 * @dpagemap: The struct drm_pagemap of the device pages we're dma-mapping. 199 * Note this is assuming only one drm_pagemap per range is allowed. 200 * @flags: Flags for range 201 * @flags.migrate_devmem: Flag indicating whether the range can be migrated to device memory 202 * @flags.unmapped: Flag indicating if the range has been unmapped 203 * @flags.partial_unmap: Flag indicating if the range has been partially unmapped 204 * @flags.has_devmem_pages: Flag indicating if the range has devmem pages 205 * @flags.has_dma_mapping: Flag indicating if the range has a DMA mapping 206 * 207 * This structure represents a GPU SVM range used for tracking memory ranges 208 * mapped in a DRM device. 209 */ 210 struct drm_gpusvm_range { 211 struct drm_gpusvm *gpusvm; 212 struct drm_gpusvm_notifier *notifier; 213 struct kref refcount; 214 struct interval_tree_node itree; 215 struct list_head entry; 216 unsigned long notifier_seq; 217 struct drm_pagemap_device_addr *dma_addr; 218 struct drm_pagemap *dpagemap; 219 struct { 220 /* All flags below must be set upon creation */ 221 u16 migrate_devmem : 1; 222 /* All flags below must be set / cleared under notifier lock */ 223 u16 unmapped : 1; 224 u16 partial_unmap : 1; 225 u16 has_devmem_pages : 1; 226 u16 has_dma_mapping : 1; 227 } flags; 228 }; 229 230 /** 231 * struct drm_gpusvm - GPU SVM structure 232 * 233 * @name: Name of the GPU SVM 234 * @drm: Pointer to the DRM device structure 235 * @mm: Pointer to the mm_struct for the address space 236 * @device_private_page_owner: Device private pages owner 237 * @mm_start: Start address of GPU SVM 238 * @mm_range: Range of the GPU SVM 239 * @notifier_size: Size of individual notifiers 240 * @ops: Pointer to the operations structure for GPU SVM 241 * @chunk_sizes: Pointer to the array of chunk sizes used in range allocation. 242 * Entries should be powers of 2 in descending order. 243 * @num_chunks: Number of chunks 244 * @notifier_lock: Read-write semaphore for protecting notifier operations 245 * @root: Cached root node of the Red-Black tree containing GPU SVM notifiers 246 * @notifier_list: list head containing of notifiers in the same order they 247 * appear in interval tree. This is useful to keep iterating 248 * notifiers while doing modifications to RB tree. 249 * 250 * This structure represents a GPU SVM (Shared Virtual Memory) used for tracking 251 * memory ranges mapped in a DRM (Direct Rendering Manager) device. 252 * 253 * No reference counting is provided, as this is expected to be embedded in the 254 * driver VM structure along with the struct drm_gpuvm, which handles reference 255 * counting. 256 */ 257 struct drm_gpusvm { 258 const char *name; 259 struct drm_device *drm; 260 struct mm_struct *mm; 261 void *device_private_page_owner; 262 unsigned long mm_start; 263 unsigned long mm_range; 264 unsigned long notifier_size; 265 const struct drm_gpusvm_ops *ops; 266 const unsigned long *chunk_sizes; 267 int num_chunks; 268 struct rw_semaphore notifier_lock; 269 struct rb_root_cached root; 270 struct list_head notifier_list; 271 #ifdef CONFIG_LOCKDEP 272 /** 273 * @lock_dep_map: Annotates drm_gpusvm_range_find_or_insert and 274 * drm_gpusvm_range_remove with a driver provided lock. 275 */ 276 struct lockdep_map *lock_dep_map; 277 #endif 278 }; 279 280 /** 281 * struct drm_gpusvm_ctx - DRM GPU SVM context 282 * 283 * @check_pages_threshold: Check CPU pages for present if chunk is less than or 284 * equal to threshold. If not present, reduce chunk 285 * size. 286 * @in_notifier: entering from a MMU notifier 287 * @read_only: operating on read-only memory 288 * @devmem_possible: possible to use device memory 289 * 290 * Context that is DRM GPUSVM is operating in (i.e. user arguments). 291 */ 292 struct drm_gpusvm_ctx { 293 unsigned long check_pages_threshold; 294 unsigned int in_notifier :1; 295 unsigned int read_only :1; 296 unsigned int devmem_possible :1; 297 }; 298 299 int drm_gpusvm_init(struct drm_gpusvm *gpusvm, 300 const char *name, struct drm_device *drm, 301 struct mm_struct *mm, void *device_private_page_owner, 302 unsigned long mm_start, unsigned long mm_range, 303 unsigned long notifier_size, 304 const struct drm_gpusvm_ops *ops, 305 const unsigned long *chunk_sizes, int num_chunks); 306 307 void drm_gpusvm_fini(struct drm_gpusvm *gpusvm); 308 309 void drm_gpusvm_free(struct drm_gpusvm *gpusvm); 310 311 struct drm_gpusvm_range * 312 drm_gpusvm_range_find_or_insert(struct drm_gpusvm *gpusvm, 313 unsigned long fault_addr, 314 unsigned long gpuva_start, 315 unsigned long gpuva_end, 316 const struct drm_gpusvm_ctx *ctx); 317 318 void drm_gpusvm_range_remove(struct drm_gpusvm *gpusvm, 319 struct drm_gpusvm_range *range); 320 321 int drm_gpusvm_range_evict(struct drm_gpusvm *gpusvm, 322 struct drm_gpusvm_range *range); 323 324 struct drm_gpusvm_range * 325 drm_gpusvm_range_get(struct drm_gpusvm_range *range); 326 327 void drm_gpusvm_range_put(struct drm_gpusvm_range *range); 328 329 bool drm_gpusvm_range_pages_valid(struct drm_gpusvm *gpusvm, 330 struct drm_gpusvm_range *range); 331 332 int drm_gpusvm_range_get_pages(struct drm_gpusvm *gpusvm, 333 struct drm_gpusvm_range *range, 334 const struct drm_gpusvm_ctx *ctx); 335 336 void drm_gpusvm_range_unmap_pages(struct drm_gpusvm *gpusvm, 337 struct drm_gpusvm_range *range, 338 const struct drm_gpusvm_ctx *ctx); 339 340 int drm_gpusvm_migrate_to_devmem(struct drm_gpusvm *gpusvm, 341 struct drm_gpusvm_range *range, 342 struct drm_gpusvm_devmem *devmem_allocation, 343 const struct drm_gpusvm_ctx *ctx); 344 345 int drm_gpusvm_evict_to_ram(struct drm_gpusvm_devmem *devmem_allocation); 346 347 const struct dev_pagemap_ops *drm_gpusvm_pagemap_ops_get(void); 348 349 bool drm_gpusvm_has_mapping(struct drm_gpusvm *gpusvm, unsigned long start, 350 unsigned long end); 351 352 struct drm_gpusvm_range * 353 drm_gpusvm_range_find(struct drm_gpusvm_notifier *notifier, unsigned long start, 354 unsigned long end); 355 356 void drm_gpusvm_range_set_unmapped(struct drm_gpusvm_range *range, 357 const struct mmu_notifier_range *mmu_range); 358 359 void drm_gpusvm_devmem_init(struct drm_gpusvm_devmem *devmem_allocation, 360 struct device *dev, struct mm_struct *mm, 361 const struct drm_gpusvm_devmem_ops *ops, 362 struct drm_pagemap *dpagemap, size_t size); 363 364 #ifdef CONFIG_LOCKDEP 365 /** 366 * drm_gpusvm_driver_set_lock() - Set the lock protecting accesses to GPU SVM 367 * @gpusvm: Pointer to the GPU SVM structure. 368 * @lock: the lock used to protect the gpuva list. The locking primitive 369 * must contain a dep_map field. 370 * 371 * Call this to annotate drm_gpusvm_range_find_or_insert and 372 * drm_gpusvm_range_remove. 373 */ 374 #define drm_gpusvm_driver_set_lock(gpusvm, lock) \ 375 do { \ 376 if (!WARN((gpusvm)->lock_dep_map, \ 377 "GPUSVM range lock should be set only once."))\ 378 (gpusvm)->lock_dep_map = &(lock)->dep_map; \ 379 } while (0) 380 #else 381 #define drm_gpusvm_driver_set_lock(gpusvm, lock) do {} while (0) 382 #endif 383 384 /** 385 * drm_gpusvm_notifier_lock() - Lock GPU SVM notifier 386 * @gpusvm__: Pointer to the GPU SVM structure. 387 * 388 * Abstract client usage GPU SVM notifier lock, take lock 389 */ 390 #define drm_gpusvm_notifier_lock(gpusvm__) \ 391 down_read(&(gpusvm__)->notifier_lock) 392 393 /** 394 * drm_gpusvm_notifier_unlock() - Unlock GPU SVM notifier 395 * @gpusvm__: Pointer to the GPU SVM structure. 396 * 397 * Abstract client usage GPU SVM notifier lock, drop lock 398 */ 399 #define drm_gpusvm_notifier_unlock(gpusvm__) \ 400 up_read(&(gpusvm__)->notifier_lock) 401 402 /** 403 * drm_gpusvm_range_start() - GPU SVM range start address 404 * @range: Pointer to the GPU SVM range 405 * 406 * Return: GPU SVM range start address 407 */ 408 static inline unsigned long 409 drm_gpusvm_range_start(struct drm_gpusvm_range *range) 410 { 411 return range->itree.start; 412 } 413 414 /** 415 * drm_gpusvm_range_end() - GPU SVM range end address 416 * @range: Pointer to the GPU SVM range 417 * 418 * Return: GPU SVM range end address 419 */ 420 static inline unsigned long 421 drm_gpusvm_range_end(struct drm_gpusvm_range *range) 422 { 423 return range->itree.last + 1; 424 } 425 426 /** 427 * drm_gpusvm_range_size() - GPU SVM range size 428 * @range: Pointer to the GPU SVM range 429 * 430 * Return: GPU SVM range size 431 */ 432 static inline unsigned long 433 drm_gpusvm_range_size(struct drm_gpusvm_range *range) 434 { 435 return drm_gpusvm_range_end(range) - drm_gpusvm_range_start(range); 436 } 437 438 /** 439 * drm_gpusvm_notifier_start() - GPU SVM notifier start address 440 * @notifier: Pointer to the GPU SVM notifier 441 * 442 * Return: GPU SVM notifier start address 443 */ 444 static inline unsigned long 445 drm_gpusvm_notifier_start(struct drm_gpusvm_notifier *notifier) 446 { 447 return notifier->itree.start; 448 } 449 450 /** 451 * drm_gpusvm_notifier_end() - GPU SVM notifier end address 452 * @notifier: Pointer to the GPU SVM notifier 453 * 454 * Return: GPU SVM notifier end address 455 */ 456 static inline unsigned long 457 drm_gpusvm_notifier_end(struct drm_gpusvm_notifier *notifier) 458 { 459 return notifier->itree.last + 1; 460 } 461 462 /** 463 * drm_gpusvm_notifier_size() - GPU SVM notifier size 464 * @notifier: Pointer to the GPU SVM notifier 465 * 466 * Return: GPU SVM notifier size 467 */ 468 static inline unsigned long 469 drm_gpusvm_notifier_size(struct drm_gpusvm_notifier *notifier) 470 { 471 return drm_gpusvm_notifier_end(notifier) - 472 drm_gpusvm_notifier_start(notifier); 473 } 474 475 /** 476 * __drm_gpusvm_range_next() - Get the next GPU SVM range in the list 477 * @range: a pointer to the current GPU SVM range 478 * 479 * Return: A pointer to the next drm_gpusvm_range if available, or NULL if the 480 * current range is the last one or if the input range is NULL. 481 */ 482 static inline struct drm_gpusvm_range * 483 __drm_gpusvm_range_next(struct drm_gpusvm_range *range) 484 { 485 if (range && !list_is_last(&range->entry, 486 &range->notifier->range_list)) 487 return list_next_entry(range, entry); 488 489 return NULL; 490 } 491 492 /** 493 * drm_gpusvm_for_each_range() - Iterate over GPU SVM ranges in a notifier 494 * @range__: Iterator variable for the ranges. If set, it indicates the start of 495 * the iterator. If NULL, call drm_gpusvm_range_find() to get the range. 496 * @notifier__: Pointer to the GPU SVM notifier 497 * @start__: Start address of the range 498 * @end__: End address of the range 499 * 500 * This macro is used to iterate over GPU SVM ranges in a notifier. It is safe 501 * to use while holding the driver SVM lock or the notifier lock. 502 */ 503 #define drm_gpusvm_for_each_range(range__, notifier__, start__, end__) \ 504 for ((range__) = (range__) ?: \ 505 drm_gpusvm_range_find((notifier__), (start__), (end__)); \ 506 (range__) && (drm_gpusvm_range_start(range__) < (end__)); \ 507 (range__) = __drm_gpusvm_range_next(range__)) 508 509 #endif /* __DRM_GPUSVM_H__ */ 510