xref: /linux/include/drm/drm_gpusvm.h (revision fdc290ff4ab19c7e0dde36c4cd1e2771b61f6bf5)
1 /* SPDX-License-Identifier: GPL-2.0-only OR MIT */
2 /*
3  * Copyright © 2024 Intel Corporation
4  */
5 
6 #ifndef __DRM_GPUSVM_H__
7 #define __DRM_GPUSVM_H__
8 
9 #include <linux/dma-mapping.h>
10 #include <linux/kref.h>
11 #include <linux/interval_tree.h>
12 #include <linux/mmu_notifier.h>
13 
14 struct dev_pagemap_ops;
15 struct drm_device;
16 struct drm_gpusvm;
17 struct drm_gpusvm_notifier;
18 struct drm_gpusvm_ops;
19 struct drm_gpusvm_range;
20 struct drm_pagemap;
21 struct drm_pagemap_addr;
22 
23 /**
24  * struct drm_gpusvm_ops - Operations structure for GPU SVM
25  *
26  * This structure defines the operations for GPU Shared Virtual Memory (SVM).
27  * These operations are provided by the GPU driver to manage SVM ranges and
28  * notifiers.
29  */
30 struct drm_gpusvm_ops {
31 	/**
32 	 * @notifier_alloc: Allocate a GPU SVM notifier (optional)
33 	 *
34 	 * Allocate a GPU SVM notifier.
35 	 *
36 	 * Return: Pointer to the allocated GPU SVM notifier on success, NULL on failure.
37 	 */
38 	struct drm_gpusvm_notifier *(*notifier_alloc)(void);
39 
40 	/**
41 	 * @notifier_free: Free a GPU SVM notifier (optional)
42 	 * @notifier: Pointer to the GPU SVM notifier to be freed
43 	 *
44 	 * Free a GPU SVM notifier.
45 	 */
46 	void (*notifier_free)(struct drm_gpusvm_notifier *notifier);
47 
48 	/**
49 	 * @range_alloc: Allocate a GPU SVM range (optional)
50 	 * @gpusvm: Pointer to the GPU SVM
51 	 *
52 	 * Allocate a GPU SVM range.
53 	 *
54 	 * Return: Pointer to the allocated GPU SVM range on success, NULL on failure.
55 	 */
56 	struct drm_gpusvm_range *(*range_alloc)(struct drm_gpusvm *gpusvm);
57 
58 	/**
59 	 * @range_free: Free a GPU SVM range (optional)
60 	 * @range: Pointer to the GPU SVM range to be freed
61 	 *
62 	 * Free a GPU SVM range.
63 	 */
64 	void (*range_free)(struct drm_gpusvm_range *range);
65 
66 	/**
67 	 * @invalidate: Invalidate GPU SVM notifier (required)
68 	 * @gpusvm: Pointer to the GPU SVM
69 	 * @notifier: Pointer to the GPU SVM notifier
70 	 * @mmu_range: Pointer to the mmu_notifier_range structure
71 	 *
72 	 * Invalidate the GPU page tables. It can safely walk the notifier range
73 	 * RB tree/list in this function. Called while holding the notifier lock.
74 	 */
75 	void (*invalidate)(struct drm_gpusvm *gpusvm,
76 			   struct drm_gpusvm_notifier *notifier,
77 			   const struct mmu_notifier_range *mmu_range);
78 };
79 
80 /**
81  * struct drm_gpusvm_notifier - Structure representing a GPU SVM notifier
82  *
83  * @gpusvm: Pointer to the GPU SVM structure
84  * @notifier: MMU interval notifier
85  * @itree: Interval tree node for the notifier (inserted in GPU SVM)
86  * @entry: List entry to fast interval tree traversal
87  * @root: Cached root node of the RB tree containing ranges
88  * @range_list: List head containing of ranges in the same order they appear in
89  *              interval tree. This is useful to keep iterating ranges while
90  *              doing modifications to RB tree.
91  * @flags: Flags for notifier
92  * @flags.removed: Flag indicating whether the MMU interval notifier has been
93  *                 removed
94  *
95  * This structure represents a GPU SVM notifier.
96  */
97 struct drm_gpusvm_notifier {
98 	struct drm_gpusvm *gpusvm;
99 	struct mmu_interval_notifier notifier;
100 	struct interval_tree_node itree;
101 	struct list_head entry;
102 	struct rb_root_cached root;
103 	struct list_head range_list;
104 	struct {
105 		u32 removed : 1;
106 	} flags;
107 };
108 
109 /**
110  * struct drm_gpusvm_pages_flags - Structure representing a GPU SVM pages flags
111  *
112  * @unmapped: Flag indicating if the pages has been unmapped
113  * @has_devmem_pages: Flag indicating if the pages has devmem pages
114  * @has_dma_mapping: Flag indicating if the pages has a DMA mapping
115  * @__flags: Flags for pages in u16 form (used for READ_ONCE)
116  */
117 struct drm_gpusvm_pages_flags {
118 	union {
119 		struct {
120 			/* All flags below must be set / cleared under notifier lock */
121 			u16 unmapped : 1;
122 			u16 has_devmem_pages : 1;
123 			u16 has_dma_mapping : 1;
124 		};
125 		u16 __flags;
126 	};
127 };
128 
129 /**
130  * struct drm_gpusvm_pages - Structure representing a GPU SVM mapped pages
131  *
132  * @drm: The DRM device that owns the dma mappings
133  * @dma_addr: Device address array
134  * @dpagemap: The struct drm_pagemap of the device pages we're dma-mapping.
135  *            Note this is assuming only one drm_pagemap per range is allowed.
136  * @state: DMA IOVA state for mapping.
137  * @state_offset: DMA IOVA offset for mapping.
138  * @notifier_seq: Notifier sequence number of the range's pages
139  * @flags: Flags for the range; see &struct drm_gpusvm_pages_flags
140  */
141 struct drm_gpusvm_pages {
142 	struct drm_device *drm;
143 	struct drm_pagemap_addr *dma_addr;
144 	struct drm_pagemap *dpagemap;
145 	struct dma_iova_state state;
146 	unsigned long state_offset;
147 	unsigned long notifier_seq;
148 	struct drm_gpusvm_pages_flags flags;
149 };
150 
151 /**
152  * struct drm_gpusvm_range_flags - Range-level GPU SVM flags
153  *
154  * @migrate_devmem: Flag indicating whether the range can be migrated to device memory
155  * @unmapped: Flag indicating if the range has been unmapped
156  * @partial_unmap: Flag indicating if the range has been partially unmapped
157  * @__flags: All flags in u16 form (used for READ_ONCE)
158  */
159 struct drm_gpusvm_range_flags {
160 	union {
161 		struct {
162 			/* All flags below must be set upon creation */
163 			u16 migrate_devmem	: 1;
164 			/* All flags below must be set / cleared under notifier lock */
165 			u16 unmapped		: 1;
166 			u16 partial_unmap	: 1;
167 		};
168 		u16 __flags;
169 	};
170 };
171 
172 /**
173  * struct drm_gpusvm_range - Structure representing a GPU SVM range
174  *
175  * @gpusvm: Pointer to the GPU SVM structure
176  * @notifier: Pointer to the GPU SVM notifier
177  * @refcount: Reference count for the range
178  * @itree: Interval tree node for the range (inserted in GPU SVM notifier)
179  * @entry: List entry to fast interval tree traversal
180  * @flags: Flags for range see &struct drm_gpusvm_range_flags
181  *
182  * This structure represents a GPU SVM range used for tracking memory ranges
183  * mapped in a DRM device.
184  */
185 struct drm_gpusvm_range {
186 	struct drm_gpusvm *gpusvm;
187 	struct drm_gpusvm_notifier *notifier;
188 	struct kref refcount;
189 	struct interval_tree_node itree;
190 	struct list_head entry;
191 	struct drm_gpusvm_range_flags flags;
192 };
193 
194 /**
195  * struct drm_gpusvm - GPU SVM structure
196  *
197  * @name: Name of the GPU SVM
198  * @mm: Pointer to the mm_struct for the address space
199  * @mm_start: Start address of GPU SVM
200  * @mm_range: Range of the GPU SVM
201  * @notifier_size: Size of individual notifiers
202  * @ops: Pointer to the operations structure for GPU SVM
203  * @chunk_sizes: Pointer to the array of chunk sizes used in range allocation.
204  *               Entries should be powers of 2 in descending order.
205  * @num_chunks: Number of chunks
206  * @notifier_lock: Read-write semaphore for protecting notifier operations
207  * @root: Cached root node of the Red-Black tree containing GPU SVM notifiers
208  * @notifier_list: list head containing of notifiers in the same order they
209  *                 appear in interval tree. This is useful to keep iterating
210  *                 notifiers while doing modifications to RB tree.
211  *
212  * This structure represents a GPU SVM (Shared Virtual Memory) used for tracking
213  * memory ranges mapped in a DRM (Direct Rendering Manager) device.
214  *
215  * No reference counting is provided, as this is expected to be embedded in the
216  * driver VM structure along with the struct drm_gpuvm, which handles reference
217  * counting.
218  */
219 struct drm_gpusvm {
220 	const char *name;
221 	struct mm_struct *mm;
222 	unsigned long mm_start;
223 	unsigned long mm_range;
224 	unsigned long notifier_size;
225 	const struct drm_gpusvm_ops *ops;
226 	const unsigned long *chunk_sizes;
227 	int num_chunks;
228 	struct rw_semaphore notifier_lock;
229 	struct rb_root_cached root;
230 	struct list_head notifier_list;
231 #ifdef CONFIG_LOCKDEP
232 	/**
233 	 * @lock_dep_map: Annotates drm_gpusvm_range_find_or_insert and
234 	 * drm_gpusvm_range_remove with a driver provided lock.
235 	 */
236 	struct lockdep_map *lock_dep_map;
237 #endif
238 };
239 
240 /**
241  * struct drm_gpusvm_ctx - DRM GPU SVM context
242  *
243  * @device_private_page_owner: The device-private page owner to use for
244  * this operation
245  * @check_pages_threshold: Check CPU pages for present if chunk is less than or
246  *                         equal to threshold. If not present, reduce chunk
247  *                         size.
248  * @timeslice_ms: The timeslice MS which in minimum time a piece of memory
249  *		  remains with either exclusive GPU or CPU access.
250  * @in_notifier: entering from a MMU notifier
251  * @read_only: operating on read-only memory
252  * @devmem_possible: possible to use device memory
253  * @devmem_only: use only device memory
254  * @allow_mixed: Allow mixed mappings in get pages. Mixing between system and
255  *               single dpagemap is supported, mixing between multiple dpagemap
256  *               is unsupported.
257  *
258  * Context that is DRM GPUSVM is operating in (i.e. user arguments).
259  */
260 struct drm_gpusvm_ctx {
261 	void *device_private_page_owner;
262 	unsigned long check_pages_threshold;
263 	unsigned long timeslice_ms;
264 	unsigned int in_notifier :1;
265 	unsigned int read_only :1;
266 	unsigned int devmem_possible :1;
267 	unsigned int devmem_only :1;
268 	unsigned int allow_mixed :1;
269 };
270 
271 int drm_gpusvm_init(struct drm_gpusvm *gpusvm,
272 		    const char *name,
273 		    struct mm_struct *mm,
274 		    unsigned long mm_start, unsigned long mm_range,
275 		    unsigned long notifier_size,
276 		    const struct drm_gpusvm_ops *ops,
277 		    const unsigned long *chunk_sizes, int num_chunks);
278 
279 void drm_gpusvm_fini(struct drm_gpusvm *gpusvm);
280 
281 void drm_gpusvm_free(struct drm_gpusvm *gpusvm);
282 
283 unsigned long
284 drm_gpusvm_find_vma_start(struct drm_gpusvm *gpusvm,
285 			  unsigned long start,
286 			  unsigned long end);
287 
288 struct drm_gpusvm_range *
289 drm_gpusvm_range_find_or_insert(struct drm_gpusvm *gpusvm,
290 				unsigned long fault_addr,
291 				unsigned long gpuva_start,
292 				unsigned long gpuva_end,
293 				const struct drm_gpusvm_ctx *ctx);
294 
295 void drm_gpusvm_range_remove(struct drm_gpusvm *gpusvm,
296 			     struct drm_gpusvm_range *range);
297 
298 int drm_gpusvm_range_evict(struct drm_gpusvm *gpusvm,
299 			   struct drm_gpusvm_range *range);
300 
301 struct drm_gpusvm_range *
302 drm_gpusvm_range_get(struct drm_gpusvm_range *range);
303 
304 void drm_gpusvm_range_put(struct drm_gpusvm_range *range);
305 
306 bool drm_gpusvm_pages_valid(struct drm_gpusvm *gpusvm,
307 			    struct drm_gpusvm_pages *svm_pages);
308 
309 bool drm_gpusvm_has_mapping(struct drm_gpusvm *gpusvm, unsigned long start,
310 			    unsigned long end);
311 
312 struct drm_gpusvm_notifier *
313 drm_gpusvm_notifier_find(struct drm_gpusvm *gpusvm, unsigned long start,
314 			 unsigned long end);
315 
316 struct drm_gpusvm_range *
317 drm_gpusvm_range_find(struct drm_gpusvm_notifier *notifier, unsigned long start,
318 		      unsigned long end);
319 
320 void drm_gpusvm_range_set_unmapped(struct drm_gpusvm_range *range,
321 				   struct drm_gpusvm_pages *pages,
322 				   unsigned int pages_count,
323 				   const struct mmu_notifier_range *mmu_range);
324 
325 int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
326 			 struct drm_gpusvm_pages *svm_pages,
327 			 struct mm_struct *mm,
328 			 struct mmu_interval_notifier *notifier,
329 			 unsigned long pages_start, unsigned long pages_end,
330 			 const struct drm_gpusvm_ctx *ctx);
331 
332 void drm_gpusvm_unmap_pages(struct drm_gpusvm *gpusvm,
333 			    struct drm_gpusvm_pages *svm_pages,
334 			    unsigned long npages,
335 			    const struct drm_gpusvm_ctx *ctx);
336 
337 void drm_gpusvm_free_pages(struct drm_gpusvm *gpusvm,
338 			   struct drm_gpusvm_pages *svm_pages,
339 			   unsigned long npages);
340 
341 /**
342  * drm_gpusvm_init_pages() - Initialize a freshly allocated drm_gpusvm_pages
343  * @svm_pages: Pointer to the drm_gpusvm_pages to initialize.
344  * @drm: The DRM device that will own DMA mappings for this pages object.
345  *
346  * Drivers that embed one or more drm_gpusvm_pages in their own range
347  * structure must call this once on each pages instance after allocation,
348  * before the first drm_gpusvm_get_pages() / unmap / free.
349  */
350 static inline void drm_gpusvm_init_pages(struct drm_gpusvm_pages *svm_pages,
351 					 struct drm_device *drm)
352 {
353 	memset(svm_pages, 0, sizeof(*svm_pages));
354 	svm_pages->drm = drm;
355 	svm_pages->notifier_seq = LONG_MAX;
356 }
357 
358 /**
359  * enum drm_gpusvm_scan_result - Scan result from the drm_gpusvm_scan_mm() function.
360  * @DRM_GPUSVM_SCAN_UNPOPULATED: At least one page was not present or inaccessible.
361  * @DRM_GPUSVM_SCAN_EQUAL: All pages belong to the struct dev_pagemap indicated as
362  * the @pagemap argument to the drm_gpusvm_scan_mm() function.
363  * @DRM_GPUSVM_SCAN_OTHER: All pages belong to exactly one dev_pagemap, which is
364  * *NOT* the @pagemap argument to the drm_gpusvm_scan_mm(). All pages belong to
365  * the same device private owner.
366  * @DRM_GPUSVM_SCAN_SYSTEM: All pages are present and system pages.
367  * @DRM_GPUSVM_SCAN_MIXED_DEVICE: All pages are device pages and belong to at least
368  * two different struct dev_pagemaps. All pages belong to the same device private
369  * owner.
370  * @DRM_GPUSVM_SCAN_MIXED: Pages are present and are a mix of system pages
371  * and device-private pages. All device-private pages belong to the same device
372  * private owner.
373  */
374 enum drm_gpusvm_scan_result {
375 	DRM_GPUSVM_SCAN_UNPOPULATED,
376 	DRM_GPUSVM_SCAN_EQUAL,
377 	DRM_GPUSVM_SCAN_OTHER,
378 	DRM_GPUSVM_SCAN_SYSTEM,
379 	DRM_GPUSVM_SCAN_MIXED_DEVICE,
380 	DRM_GPUSVM_SCAN_MIXED,
381 };
382 
383 enum drm_gpusvm_scan_result drm_gpusvm_scan_mm(struct drm_gpusvm_range *range,
384 					       void *dev_private_owner,
385 					       const struct dev_pagemap *pagemap);
386 
387 #ifdef CONFIG_LOCKDEP
388 /**
389  * drm_gpusvm_driver_set_lock() - Set the lock protecting accesses to GPU SVM
390  * @gpusvm: Pointer to the GPU SVM structure.
391  * @lock: the lock used to protect the gpuva list. The locking primitive
392  * must contain a dep_map field.
393  *
394  * Call this to annotate drm_gpusvm_range_find_or_insert and
395  * drm_gpusvm_range_remove.
396  */
397 #define drm_gpusvm_driver_set_lock(gpusvm, lock) \
398 	do { \
399 		if (!WARN((gpusvm)->lock_dep_map, \
400 			  "GPUSVM range lock should be set only once."))\
401 			(gpusvm)->lock_dep_map = &(lock)->dep_map;	\
402 	} while (0)
403 #else
404 #define drm_gpusvm_driver_set_lock(gpusvm, lock) do {} while (0)
405 #endif
406 
407 /**
408  * drm_gpusvm_notifier_lock() - Lock GPU SVM notifier
409  * @gpusvm__: Pointer to the GPU SVM structure.
410  *
411  * Abstract client usage GPU SVM notifier lock, take lock
412  */
413 #define drm_gpusvm_notifier_lock(gpusvm__)	\
414 	down_read(&(gpusvm__)->notifier_lock)
415 
416 /**
417  * drm_gpusvm_notifier_unlock() - Unlock GPU SVM notifier
418  * @gpusvm__: Pointer to the GPU SVM structure.
419  *
420  * Abstract client usage GPU SVM notifier lock, drop lock
421  */
422 #define drm_gpusvm_notifier_unlock(gpusvm__)	\
423 	up_read(&(gpusvm__)->notifier_lock)
424 
425 /**
426  * drm_gpusvm_range_start() - GPU SVM range start address
427  * @range: Pointer to the GPU SVM range
428  *
429  * Return: GPU SVM range start address
430  */
431 static inline unsigned long
432 drm_gpusvm_range_start(struct drm_gpusvm_range *range)
433 {
434 	return range->itree.start;
435 }
436 
437 /**
438  * drm_gpusvm_range_end() - GPU SVM range end address
439  * @range: Pointer to the GPU SVM range
440  *
441  * Return: GPU SVM range end address
442  */
443 static inline unsigned long
444 drm_gpusvm_range_end(struct drm_gpusvm_range *range)
445 {
446 	return range->itree.last + 1;
447 }
448 
449 /**
450  * drm_gpusvm_range_size() - GPU SVM range size
451  * @range: Pointer to the GPU SVM range
452  *
453  * Return: GPU SVM range size
454  */
455 static inline unsigned long
456 drm_gpusvm_range_size(struct drm_gpusvm_range *range)
457 {
458 	return drm_gpusvm_range_end(range) - drm_gpusvm_range_start(range);
459 }
460 
461 /**
462  * drm_gpusvm_notifier_start() - GPU SVM notifier start address
463  * @notifier: Pointer to the GPU SVM notifier
464  *
465  * Return: GPU SVM notifier start address
466  */
467 static inline unsigned long
468 drm_gpusvm_notifier_start(struct drm_gpusvm_notifier *notifier)
469 {
470 	return notifier->itree.start;
471 }
472 
473 /**
474  * drm_gpusvm_notifier_end() - GPU SVM notifier end address
475  * @notifier: Pointer to the GPU SVM notifier
476  *
477  * Return: GPU SVM notifier end address
478  */
479 static inline unsigned long
480 drm_gpusvm_notifier_end(struct drm_gpusvm_notifier *notifier)
481 {
482 	return notifier->itree.last + 1;
483 }
484 
485 /**
486  * drm_gpusvm_notifier_size() - GPU SVM notifier size
487  * @notifier: Pointer to the GPU SVM notifier
488  *
489  * Return: GPU SVM notifier size
490  */
491 static inline unsigned long
492 drm_gpusvm_notifier_size(struct drm_gpusvm_notifier *notifier)
493 {
494 	return drm_gpusvm_notifier_end(notifier) -
495 		drm_gpusvm_notifier_start(notifier);
496 }
497 
498 /**
499  * __drm_gpusvm_range_next() - Get the next GPU SVM range in the list
500  * @range: a pointer to the current GPU SVM range
501  *
502  * Return: A pointer to the next drm_gpusvm_range if available, or NULL if the
503  *         current range is the last one or if the input range is NULL.
504  */
505 static inline struct drm_gpusvm_range *
506 __drm_gpusvm_range_next(struct drm_gpusvm_range *range)
507 {
508 	if (range && !list_is_last(&range->entry,
509 				   &range->notifier->range_list))
510 		return list_next_entry(range, entry);
511 
512 	return NULL;
513 }
514 
515 /**
516  * drm_gpusvm_for_each_range() - Iterate over GPU SVM ranges in a notifier
517  * @range__: Iterator variable for the ranges. If set, it indicates the start of
518  *	     the iterator. If NULL, call drm_gpusvm_range_find() to get the range.
519  * @notifier__: Pointer to the GPU SVM notifier
520  * @start__: Start address of the range
521  * @end__: End address of the range
522  *
523  * This macro is used to iterate over GPU SVM ranges in a notifier. It is safe
524  * to use while holding the driver SVM lock or the notifier lock.
525  */
526 #define drm_gpusvm_for_each_range(range__, notifier__, start__, end__)	\
527 	for ((range__) = (range__) ?:					\
528 	     drm_gpusvm_range_find((notifier__), (start__), (end__));	\
529 	     (range__) && (drm_gpusvm_range_start(range__) < (end__));	\
530 	     (range__) = __drm_gpusvm_range_next(range__))
531 
532 /**
533  * drm_gpusvm_for_each_range_safe() - Safely iterate over GPU SVM ranges in a notifier
534  * @range__: Iterator variable for the ranges
535  * @next__: Iterator variable for the ranges temporay storage
536  * @notifier__: Pointer to the GPU SVM notifier
537  * @start__: Start address of the range
538  * @end__: End address of the range
539  *
540  * This macro is used to iterate over GPU SVM ranges in a notifier while
541  * removing ranges from it.
542  */
543 #define drm_gpusvm_for_each_range_safe(range__, next__, notifier__, start__, end__)	\
544 	for ((range__) = drm_gpusvm_range_find((notifier__), (start__), (end__)),	\
545 	     (next__) = __drm_gpusvm_range_next(range__);				\
546 	     (range__) && (drm_gpusvm_range_start(range__) < (end__));			\
547 	     (range__) = (next__), (next__) = __drm_gpusvm_range_next(range__))
548 
549 /**
550  * __drm_gpusvm_notifier_next() - get the next drm_gpusvm_notifier in the list
551  * @notifier: a pointer to the current drm_gpusvm_notifier
552  *
553  * Return: A pointer to the next drm_gpusvm_notifier if available, or NULL if
554  *         the current notifier is the last one or if the input notifier is
555  *         NULL.
556  */
557 static inline struct drm_gpusvm_notifier *
558 __drm_gpusvm_notifier_next(struct drm_gpusvm_notifier *notifier)
559 {
560 	if (notifier && !list_is_last(&notifier->entry,
561 				      &notifier->gpusvm->notifier_list))
562 		return list_next_entry(notifier, entry);
563 
564 	return NULL;
565 }
566 
567 /**
568  * drm_gpusvm_for_each_notifier() - Iterate over GPU SVM notifiers in a gpusvm
569  * @notifier__: Iterator variable for the notifiers
570  * @gpusvm__: Pointer to the GPU SVM notifier
571  * @start__: Start address of the notifier
572  * @end__: End address of the notifier
573  *
574  * This macro is used to iterate over GPU SVM notifiers in a gpusvm.
575  */
576 #define drm_gpusvm_for_each_notifier(notifier__, gpusvm__, start__, end__)		\
577 	for ((notifier__) = drm_gpusvm_notifier_find((gpusvm__), (start__), (end__));	\
578 	     (notifier__) && (drm_gpusvm_notifier_start(notifier__) < (end__));		\
579 	     (notifier__) = __drm_gpusvm_notifier_next(notifier__))
580 
581 /**
582  * drm_gpusvm_for_each_notifier_safe() - Safely iterate over GPU SVM notifiers in a gpusvm
583  * @notifier__: Iterator variable for the notifiers
584  * @next__: Iterator variable for the notifiers temporay storage
585  * @gpusvm__: Pointer to the GPU SVM notifier
586  * @start__: Start address of the notifier
587  * @end__: End address of the notifier
588  *
589  * This macro is used to iterate over GPU SVM notifiers in a gpusvm while
590  * removing notifiers from it.
591  */
592 #define drm_gpusvm_for_each_notifier_safe(notifier__, next__, gpusvm__, start__, end__)	\
593 	for ((notifier__) = drm_gpusvm_notifier_find((gpusvm__), (start__), (end__)),	\
594 	     (next__) = __drm_gpusvm_notifier_next(notifier__);				\
595 	     (notifier__) && (drm_gpusvm_notifier_start(notifier__) < (end__));		\
596 	     (notifier__) = (next__), (next__) = __drm_gpusvm_notifier_next(notifier__))
597 
598 #endif /* __DRM_GPUSVM_H__ */
599