1 /* SPDX-License-Identifier: GPL-2.0-only OR MIT */ 2 /* 3 * Copyright © 2024 Intel Corporation 4 */ 5 6 #ifndef __DRM_GPUSVM_H__ 7 #define __DRM_GPUSVM_H__ 8 9 #include <linux/dma-mapping.h> 10 #include <linux/kref.h> 11 #include <linux/interval_tree.h> 12 #include <linux/mmu_notifier.h> 13 14 struct dev_pagemap_ops; 15 struct drm_device; 16 struct drm_gpusvm; 17 struct drm_gpusvm_notifier; 18 struct drm_gpusvm_ops; 19 struct drm_gpusvm_range; 20 struct drm_pagemap; 21 struct drm_pagemap_addr; 22 23 /** 24 * struct drm_gpusvm_ops - Operations structure for GPU SVM 25 * 26 * This structure defines the operations for GPU Shared Virtual Memory (SVM). 27 * These operations are provided by the GPU driver to manage SVM ranges and 28 * notifiers. 29 */ 30 struct drm_gpusvm_ops { 31 /** 32 * @notifier_alloc: Allocate a GPU SVM notifier (optional) 33 * 34 * Allocate a GPU SVM notifier. 35 * 36 * Return: Pointer to the allocated GPU SVM notifier on success, NULL on failure. 37 */ 38 struct drm_gpusvm_notifier *(*notifier_alloc)(void); 39 40 /** 41 * @notifier_free: Free a GPU SVM notifier (optional) 42 * @notifier: Pointer to the GPU SVM notifier to be freed 43 * 44 * Free a GPU SVM notifier. 45 */ 46 void (*notifier_free)(struct drm_gpusvm_notifier *notifier); 47 48 /** 49 * @range_alloc: Allocate a GPU SVM range (optional) 50 * @gpusvm: Pointer to the GPU SVM 51 * 52 * Allocate a GPU SVM range. 53 * 54 * Return: Pointer to the allocated GPU SVM range on success, NULL on failure. 55 */ 56 struct drm_gpusvm_range *(*range_alloc)(struct drm_gpusvm *gpusvm); 57 58 /** 59 * @range_free: Free a GPU SVM range (optional) 60 * @range: Pointer to the GPU SVM range to be freed 61 * 62 * Free a GPU SVM range. 63 */ 64 void (*range_free)(struct drm_gpusvm_range *range); 65 66 /** 67 * @invalidate: Invalidate GPU SVM notifier (required) 68 * @gpusvm: Pointer to the GPU SVM 69 * @notifier: Pointer to the GPU SVM notifier 70 * @mmu_range: Pointer to the mmu_notifier_range structure 71 * 72 * Invalidate the GPU page tables. It can safely walk the notifier range 73 * RB tree/list in this function. Called while holding the notifier lock. 74 */ 75 void (*invalidate)(struct drm_gpusvm *gpusvm, 76 struct drm_gpusvm_notifier *notifier, 77 const struct mmu_notifier_range *mmu_range); 78 }; 79 80 /** 81 * struct drm_gpusvm_notifier - Structure representing a GPU SVM notifier 82 * 83 * @gpusvm: Pointer to the GPU SVM structure 84 * @notifier: MMU interval notifier 85 * @itree: Interval tree node for the notifier (inserted in GPU SVM) 86 * @entry: List entry to fast interval tree traversal 87 * @root: Cached root node of the RB tree containing ranges 88 * @range_list: List head containing of ranges in the same order they appear in 89 * interval tree. This is useful to keep iterating ranges while 90 * doing modifications to RB tree. 91 * @flags: Flags for notifier 92 * @flags.removed: Flag indicating whether the MMU interval notifier has been 93 * removed 94 * 95 * This structure represents a GPU SVM notifier. 96 */ 97 struct drm_gpusvm_notifier { 98 struct drm_gpusvm *gpusvm; 99 struct mmu_interval_notifier notifier; 100 struct interval_tree_node itree; 101 struct list_head entry; 102 struct rb_root_cached root; 103 struct list_head range_list; 104 struct { 105 u32 removed : 1; 106 } flags; 107 }; 108 109 /** 110 * struct drm_gpusvm_pages_flags - Structure representing a GPU SVM pages flags 111 * 112 * @unmapped: Flag indicating if the pages has been unmapped 113 * @has_devmem_pages: Flag indicating if the pages has devmem pages 114 * @has_dma_mapping: Flag indicating if the pages has a DMA mapping 115 * @__flags: Flags for pages in u16 form (used for READ_ONCE) 116 */ 117 struct drm_gpusvm_pages_flags { 118 union { 119 struct { 120 /* All flags below must be set / cleared under notifier lock */ 121 u16 unmapped : 1; 122 u16 has_devmem_pages : 1; 123 u16 has_dma_mapping : 1; 124 }; 125 u16 __flags; 126 }; 127 }; 128 129 /** 130 * struct drm_gpusvm_pages - Structure representing a GPU SVM mapped pages 131 * 132 * @drm: The DRM device that owns the dma mappings 133 * @dma_addr: Device address array 134 * @dpagemap: The struct drm_pagemap of the device pages we're dma-mapping. 135 * Note this is assuming only one drm_pagemap per range is allowed. 136 * @state: DMA IOVA state for mapping. 137 * @state_offset: DMA IOVA offset for mapping. 138 * @notifier_seq: Notifier sequence number of the range's pages 139 * @flags: Flags for the range; see &struct drm_gpusvm_pages_flags 140 */ 141 struct drm_gpusvm_pages { 142 struct drm_device *drm; 143 struct drm_pagemap_addr *dma_addr; 144 struct drm_pagemap *dpagemap; 145 struct dma_iova_state state; 146 unsigned long state_offset; 147 unsigned long notifier_seq; 148 struct drm_gpusvm_pages_flags flags; 149 }; 150 151 /** 152 * struct drm_gpusvm_range_flags - Range-level GPU SVM flags 153 * 154 * @migrate_devmem: Flag indicating whether the range can be migrated to device memory 155 * @unmapped: Flag indicating if the range has been unmapped 156 * @partial_unmap: Flag indicating if the range has been partially unmapped 157 * @__flags: All flags in u16 form (used for READ_ONCE) 158 */ 159 struct drm_gpusvm_range_flags { 160 union { 161 struct { 162 /* All flags below must be set upon creation */ 163 u16 migrate_devmem : 1; 164 /* All flags below must be set / cleared under notifier lock */ 165 u16 unmapped : 1; 166 u16 partial_unmap : 1; 167 }; 168 u16 __flags; 169 }; 170 }; 171 172 /** 173 * struct drm_gpusvm_range - Structure representing a GPU SVM range 174 * 175 * @gpusvm: Pointer to the GPU SVM structure 176 * @notifier: Pointer to the GPU SVM notifier 177 * @refcount: Reference count for the range 178 * @itree: Interval tree node for the range (inserted in GPU SVM notifier) 179 * @entry: List entry to fast interval tree traversal 180 * @flags: Flags for range see &struct drm_gpusvm_range_flags 181 * 182 * This structure represents a GPU SVM range used for tracking memory ranges 183 * mapped in a DRM device. 184 */ 185 struct drm_gpusvm_range { 186 struct drm_gpusvm *gpusvm; 187 struct drm_gpusvm_notifier *notifier; 188 struct kref refcount; 189 struct interval_tree_node itree; 190 struct list_head entry; 191 struct drm_gpusvm_range_flags flags; 192 }; 193 194 /** 195 * struct drm_gpusvm - GPU SVM structure 196 * 197 * @name: Name of the GPU SVM 198 * @mm: Pointer to the mm_struct for the address space 199 * @mm_start: Start address of GPU SVM 200 * @mm_range: Range of the GPU SVM 201 * @notifier_size: Size of individual notifiers 202 * @ops: Pointer to the operations structure for GPU SVM 203 * @chunk_sizes: Pointer to the array of chunk sizes used in range allocation. 204 * Entries should be powers of 2 in descending order. 205 * @num_chunks: Number of chunks 206 * @notifier_lock: Read-write semaphore for protecting notifier operations 207 * @root: Cached root node of the Red-Black tree containing GPU SVM notifiers 208 * @notifier_list: list head containing of notifiers in the same order they 209 * appear in interval tree. This is useful to keep iterating 210 * notifiers while doing modifications to RB tree. 211 * 212 * This structure represents a GPU SVM (Shared Virtual Memory) used for tracking 213 * memory ranges mapped in a DRM (Direct Rendering Manager) device. 214 * 215 * No reference counting is provided, as this is expected to be embedded in the 216 * driver VM structure along with the struct drm_gpuvm, which handles reference 217 * counting. 218 */ 219 struct drm_gpusvm { 220 const char *name; 221 struct mm_struct *mm; 222 unsigned long mm_start; 223 unsigned long mm_range; 224 unsigned long notifier_size; 225 const struct drm_gpusvm_ops *ops; 226 const unsigned long *chunk_sizes; 227 int num_chunks; 228 struct rw_semaphore notifier_lock; 229 struct rb_root_cached root; 230 struct list_head notifier_list; 231 #ifdef CONFIG_LOCKDEP 232 /** 233 * @lock_dep_map: Annotates drm_gpusvm_range_find_or_insert and 234 * drm_gpusvm_range_remove with a driver provided lock. 235 */ 236 struct lockdep_map *lock_dep_map; 237 #endif 238 }; 239 240 /** 241 * struct drm_gpusvm_ctx - DRM GPU SVM context 242 * 243 * @device_private_page_owner: The device-private page owner to use for 244 * this operation 245 * @check_pages_threshold: Check CPU pages for present if chunk is less than or 246 * equal to threshold. If not present, reduce chunk 247 * size. 248 * @timeslice_ms: The timeslice MS which in minimum time a piece of memory 249 * remains with either exclusive GPU or CPU access. 250 * @in_notifier: entering from a MMU notifier 251 * @read_only: operating on read-only memory 252 * @devmem_possible: possible to use device memory 253 * @devmem_only: use only device memory 254 * @allow_mixed: Allow mixed mappings in get pages. Mixing between system and 255 * single dpagemap is supported, mixing between multiple dpagemap 256 * is unsupported. 257 * 258 * Context that is DRM GPUSVM is operating in (i.e. user arguments). 259 */ 260 struct drm_gpusvm_ctx { 261 void *device_private_page_owner; 262 unsigned long check_pages_threshold; 263 unsigned long timeslice_ms; 264 unsigned int in_notifier :1; 265 unsigned int read_only :1; 266 unsigned int devmem_possible :1; 267 unsigned int devmem_only :1; 268 unsigned int allow_mixed :1; 269 }; 270 271 int drm_gpusvm_init(struct drm_gpusvm *gpusvm, 272 const char *name, 273 struct mm_struct *mm, 274 unsigned long mm_start, unsigned long mm_range, 275 unsigned long notifier_size, 276 const struct drm_gpusvm_ops *ops, 277 const unsigned long *chunk_sizes, int num_chunks); 278 279 void drm_gpusvm_fini(struct drm_gpusvm *gpusvm); 280 281 void drm_gpusvm_free(struct drm_gpusvm *gpusvm); 282 283 unsigned long 284 drm_gpusvm_find_vma_start(struct drm_gpusvm *gpusvm, 285 unsigned long start, 286 unsigned long end); 287 288 struct drm_gpusvm_range * 289 drm_gpusvm_range_find_or_insert(struct drm_gpusvm *gpusvm, 290 unsigned long fault_addr, 291 unsigned long gpuva_start, 292 unsigned long gpuva_end, 293 const struct drm_gpusvm_ctx *ctx); 294 295 void drm_gpusvm_range_remove(struct drm_gpusvm *gpusvm, 296 struct drm_gpusvm_range *range); 297 298 int drm_gpusvm_range_evict(struct drm_gpusvm *gpusvm, 299 struct drm_gpusvm_range *range); 300 301 struct drm_gpusvm_range * 302 drm_gpusvm_range_get(struct drm_gpusvm_range *range); 303 304 void drm_gpusvm_range_put(struct drm_gpusvm_range *range); 305 306 bool drm_gpusvm_pages_valid(struct drm_gpusvm *gpusvm, 307 struct drm_gpusvm_pages *svm_pages); 308 309 bool drm_gpusvm_has_mapping(struct drm_gpusvm *gpusvm, unsigned long start, 310 unsigned long end); 311 312 struct drm_gpusvm_notifier * 313 drm_gpusvm_notifier_find(struct drm_gpusvm *gpusvm, unsigned long start, 314 unsigned long end); 315 316 struct drm_gpusvm_range * 317 drm_gpusvm_range_find(struct drm_gpusvm_notifier *notifier, unsigned long start, 318 unsigned long end); 319 320 void drm_gpusvm_range_set_unmapped(struct drm_gpusvm_range *range, 321 struct drm_gpusvm_pages *pages, 322 unsigned int pages_count, 323 const struct mmu_notifier_range *mmu_range); 324 325 int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, 326 struct drm_gpusvm_pages *svm_pages, 327 struct mm_struct *mm, 328 struct mmu_interval_notifier *notifier, 329 unsigned long pages_start, unsigned long pages_end, 330 const struct drm_gpusvm_ctx *ctx); 331 332 void drm_gpusvm_unmap_pages(struct drm_gpusvm *gpusvm, 333 struct drm_gpusvm_pages *svm_pages, 334 unsigned long npages, 335 const struct drm_gpusvm_ctx *ctx); 336 337 void drm_gpusvm_free_pages(struct drm_gpusvm *gpusvm, 338 struct drm_gpusvm_pages *svm_pages, 339 unsigned long npages); 340 341 /** 342 * drm_gpusvm_init_pages() - Initialize a freshly allocated drm_gpusvm_pages 343 * @svm_pages: Pointer to the drm_gpusvm_pages to initialize. 344 * @drm: The DRM device that will own DMA mappings for this pages object. 345 * 346 * Drivers that embed one or more drm_gpusvm_pages in their own range 347 * structure must call this once on each pages instance after allocation, 348 * before the first drm_gpusvm_get_pages() / unmap / free. 349 */ 350 static inline void drm_gpusvm_init_pages(struct drm_gpusvm_pages *svm_pages, 351 struct drm_device *drm) 352 { 353 memset(svm_pages, 0, sizeof(*svm_pages)); 354 svm_pages->drm = drm; 355 svm_pages->notifier_seq = LONG_MAX; 356 } 357 358 /** 359 * enum drm_gpusvm_scan_result - Scan result from the drm_gpusvm_scan_mm() function. 360 * @DRM_GPUSVM_SCAN_UNPOPULATED: At least one page was not present or inaccessible. 361 * @DRM_GPUSVM_SCAN_EQUAL: All pages belong to the struct dev_pagemap indicated as 362 * the @pagemap argument to the drm_gpusvm_scan_mm() function. 363 * @DRM_GPUSVM_SCAN_OTHER: All pages belong to exactly one dev_pagemap, which is 364 * *NOT* the @pagemap argument to the drm_gpusvm_scan_mm(). All pages belong to 365 * the same device private owner. 366 * @DRM_GPUSVM_SCAN_SYSTEM: All pages are present and system pages. 367 * @DRM_GPUSVM_SCAN_MIXED_DEVICE: All pages are device pages and belong to at least 368 * two different struct dev_pagemaps. All pages belong to the same device private 369 * owner. 370 * @DRM_GPUSVM_SCAN_MIXED: Pages are present and are a mix of system pages 371 * and device-private pages. All device-private pages belong to the same device 372 * private owner. 373 */ 374 enum drm_gpusvm_scan_result { 375 DRM_GPUSVM_SCAN_UNPOPULATED, 376 DRM_GPUSVM_SCAN_EQUAL, 377 DRM_GPUSVM_SCAN_OTHER, 378 DRM_GPUSVM_SCAN_SYSTEM, 379 DRM_GPUSVM_SCAN_MIXED_DEVICE, 380 DRM_GPUSVM_SCAN_MIXED, 381 }; 382 383 enum drm_gpusvm_scan_result drm_gpusvm_scan_mm(struct drm_gpusvm_range *range, 384 void *dev_private_owner, 385 const struct dev_pagemap *pagemap); 386 387 #ifdef CONFIG_LOCKDEP 388 /** 389 * drm_gpusvm_driver_set_lock() - Set the lock protecting accesses to GPU SVM 390 * @gpusvm: Pointer to the GPU SVM structure. 391 * @lock: the lock used to protect the gpuva list. The locking primitive 392 * must contain a dep_map field. 393 * 394 * Call this to annotate drm_gpusvm_range_find_or_insert and 395 * drm_gpusvm_range_remove. 396 */ 397 #define drm_gpusvm_driver_set_lock(gpusvm, lock) \ 398 do { \ 399 if (!WARN((gpusvm)->lock_dep_map, \ 400 "GPUSVM range lock should be set only once."))\ 401 (gpusvm)->lock_dep_map = &(lock)->dep_map; \ 402 } while (0) 403 #else 404 #define drm_gpusvm_driver_set_lock(gpusvm, lock) do {} while (0) 405 #endif 406 407 /** 408 * drm_gpusvm_notifier_lock() - Lock GPU SVM notifier 409 * @gpusvm__: Pointer to the GPU SVM structure. 410 * 411 * Abstract client usage GPU SVM notifier lock, take lock 412 */ 413 #define drm_gpusvm_notifier_lock(gpusvm__) \ 414 down_read(&(gpusvm__)->notifier_lock) 415 416 /** 417 * drm_gpusvm_notifier_unlock() - Unlock GPU SVM notifier 418 * @gpusvm__: Pointer to the GPU SVM structure. 419 * 420 * Abstract client usage GPU SVM notifier lock, drop lock 421 */ 422 #define drm_gpusvm_notifier_unlock(gpusvm__) \ 423 up_read(&(gpusvm__)->notifier_lock) 424 425 /** 426 * drm_gpusvm_range_start() - GPU SVM range start address 427 * @range: Pointer to the GPU SVM range 428 * 429 * Return: GPU SVM range start address 430 */ 431 static inline unsigned long 432 drm_gpusvm_range_start(struct drm_gpusvm_range *range) 433 { 434 return range->itree.start; 435 } 436 437 /** 438 * drm_gpusvm_range_end() - GPU SVM range end address 439 * @range: Pointer to the GPU SVM range 440 * 441 * Return: GPU SVM range end address 442 */ 443 static inline unsigned long 444 drm_gpusvm_range_end(struct drm_gpusvm_range *range) 445 { 446 return range->itree.last + 1; 447 } 448 449 /** 450 * drm_gpusvm_range_size() - GPU SVM range size 451 * @range: Pointer to the GPU SVM range 452 * 453 * Return: GPU SVM range size 454 */ 455 static inline unsigned long 456 drm_gpusvm_range_size(struct drm_gpusvm_range *range) 457 { 458 return drm_gpusvm_range_end(range) - drm_gpusvm_range_start(range); 459 } 460 461 /** 462 * drm_gpusvm_notifier_start() - GPU SVM notifier start address 463 * @notifier: Pointer to the GPU SVM notifier 464 * 465 * Return: GPU SVM notifier start address 466 */ 467 static inline unsigned long 468 drm_gpusvm_notifier_start(struct drm_gpusvm_notifier *notifier) 469 { 470 return notifier->itree.start; 471 } 472 473 /** 474 * drm_gpusvm_notifier_end() - GPU SVM notifier end address 475 * @notifier: Pointer to the GPU SVM notifier 476 * 477 * Return: GPU SVM notifier end address 478 */ 479 static inline unsigned long 480 drm_gpusvm_notifier_end(struct drm_gpusvm_notifier *notifier) 481 { 482 return notifier->itree.last + 1; 483 } 484 485 /** 486 * drm_gpusvm_notifier_size() - GPU SVM notifier size 487 * @notifier: Pointer to the GPU SVM notifier 488 * 489 * Return: GPU SVM notifier size 490 */ 491 static inline unsigned long 492 drm_gpusvm_notifier_size(struct drm_gpusvm_notifier *notifier) 493 { 494 return drm_gpusvm_notifier_end(notifier) - 495 drm_gpusvm_notifier_start(notifier); 496 } 497 498 /** 499 * __drm_gpusvm_range_next() - Get the next GPU SVM range in the list 500 * @range: a pointer to the current GPU SVM range 501 * 502 * Return: A pointer to the next drm_gpusvm_range if available, or NULL if the 503 * current range is the last one or if the input range is NULL. 504 */ 505 static inline struct drm_gpusvm_range * 506 __drm_gpusvm_range_next(struct drm_gpusvm_range *range) 507 { 508 if (range && !list_is_last(&range->entry, 509 &range->notifier->range_list)) 510 return list_next_entry(range, entry); 511 512 return NULL; 513 } 514 515 /** 516 * drm_gpusvm_for_each_range() - Iterate over GPU SVM ranges in a notifier 517 * @range__: Iterator variable for the ranges. If set, it indicates the start of 518 * the iterator. If NULL, call drm_gpusvm_range_find() to get the range. 519 * @notifier__: Pointer to the GPU SVM notifier 520 * @start__: Start address of the range 521 * @end__: End address of the range 522 * 523 * This macro is used to iterate over GPU SVM ranges in a notifier. It is safe 524 * to use while holding the driver SVM lock or the notifier lock. 525 */ 526 #define drm_gpusvm_for_each_range(range__, notifier__, start__, end__) \ 527 for ((range__) = (range__) ?: \ 528 drm_gpusvm_range_find((notifier__), (start__), (end__)); \ 529 (range__) && (drm_gpusvm_range_start(range__) < (end__)); \ 530 (range__) = __drm_gpusvm_range_next(range__)) 531 532 /** 533 * drm_gpusvm_for_each_range_safe() - Safely iterate over GPU SVM ranges in a notifier 534 * @range__: Iterator variable for the ranges 535 * @next__: Iterator variable for the ranges temporay storage 536 * @notifier__: Pointer to the GPU SVM notifier 537 * @start__: Start address of the range 538 * @end__: End address of the range 539 * 540 * This macro is used to iterate over GPU SVM ranges in a notifier while 541 * removing ranges from it. 542 */ 543 #define drm_gpusvm_for_each_range_safe(range__, next__, notifier__, start__, end__) \ 544 for ((range__) = drm_gpusvm_range_find((notifier__), (start__), (end__)), \ 545 (next__) = __drm_gpusvm_range_next(range__); \ 546 (range__) && (drm_gpusvm_range_start(range__) < (end__)); \ 547 (range__) = (next__), (next__) = __drm_gpusvm_range_next(range__)) 548 549 /** 550 * __drm_gpusvm_notifier_next() - get the next drm_gpusvm_notifier in the list 551 * @notifier: a pointer to the current drm_gpusvm_notifier 552 * 553 * Return: A pointer to the next drm_gpusvm_notifier if available, or NULL if 554 * the current notifier is the last one or if the input notifier is 555 * NULL. 556 */ 557 static inline struct drm_gpusvm_notifier * 558 __drm_gpusvm_notifier_next(struct drm_gpusvm_notifier *notifier) 559 { 560 if (notifier && !list_is_last(¬ifier->entry, 561 ¬ifier->gpusvm->notifier_list)) 562 return list_next_entry(notifier, entry); 563 564 return NULL; 565 } 566 567 /** 568 * drm_gpusvm_for_each_notifier() - Iterate over GPU SVM notifiers in a gpusvm 569 * @notifier__: Iterator variable for the notifiers 570 * @gpusvm__: Pointer to the GPU SVM notifier 571 * @start__: Start address of the notifier 572 * @end__: End address of the notifier 573 * 574 * This macro is used to iterate over GPU SVM notifiers in a gpusvm. 575 */ 576 #define drm_gpusvm_for_each_notifier(notifier__, gpusvm__, start__, end__) \ 577 for ((notifier__) = drm_gpusvm_notifier_find((gpusvm__), (start__), (end__)); \ 578 (notifier__) && (drm_gpusvm_notifier_start(notifier__) < (end__)); \ 579 (notifier__) = __drm_gpusvm_notifier_next(notifier__)) 580 581 /** 582 * drm_gpusvm_for_each_notifier_safe() - Safely iterate over GPU SVM notifiers in a gpusvm 583 * @notifier__: Iterator variable for the notifiers 584 * @next__: Iterator variable for the notifiers temporay storage 585 * @gpusvm__: Pointer to the GPU SVM notifier 586 * @start__: Start address of the notifier 587 * @end__: End address of the notifier 588 * 589 * This macro is used to iterate over GPU SVM notifiers in a gpusvm while 590 * removing notifiers from it. 591 */ 592 #define drm_gpusvm_for_each_notifier_safe(notifier__, next__, gpusvm__, start__, end__) \ 593 for ((notifier__) = drm_gpusvm_notifier_find((gpusvm__), (start__), (end__)), \ 594 (next__) = __drm_gpusvm_notifier_next(notifier__); \ 595 (notifier__) && (drm_gpusvm_notifier_start(notifier__) < (end__)); \ 596 (notifier__) = (next__), (next__) = __drm_gpusvm_notifier_next(notifier__)) 597 598 #endif /* __DRM_GPUSVM_H__ */ 599