xref: /linux/drivers/gpu/drm/drm_gpusvm.c (revision 4dd10fbbeb9c33709edc89916e19c03c780a0751)
1 // SPDX-License-Identifier: GPL-2.0-only OR MIT
2 /*
3  * Copyright © 2024 Intel Corporation
4  *
5  * Authors:
6  *     Matthew Brost <matthew.brost@intel.com>
7  */
8 
9 #include <linux/dma-mapping.h>
10 #include <linux/export.h>
11 #include <linux/hmm.h>
12 #include <linux/memremap.h>
13 #include <linux/mm_types.h>
14 #include <linux/mm.h>
15 #include <linux/slab.h>
16 
17 #include <drm/drm_device.h>
18 #include <drm/drm_gpusvm.h>
19 #include <drm/drm_pagemap.h>
20 #include <drm/drm_print.h>
21 
22 /**
23  * DOC: Overview
24  *
25  * GPU Shared Virtual Memory (GPU SVM) layer for the Direct Rendering Manager (DRM)
26  * is a component of the DRM framework designed to manage shared virtual memory
27  * between the CPU and GPU. It enables efficient data exchange and processing
28  * for GPU-accelerated applications by allowing memory sharing and
29  * synchronization between the CPU's and GPU's virtual address spaces.
30  *
31  * Key GPU SVM Components:
32  *
33  * - Notifiers:
34  *	Used for tracking memory intervals and notifying the GPU of changes,
35  *	notifiers are sized based on a GPU SVM initialization parameter, with a
36  *	recommendation of 512M or larger. They maintain a Red-BlacK tree and a
37  *	list of ranges that fall within the notifier interval.  Notifiers are
38  *	tracked within a GPU SVM Red-BlacK tree and list and are dynamically
39  *	inserted or removed as ranges within the interval are created or
40  *	destroyed.
41  * - Ranges:
42  *	Represent memory ranges mapped in a DRM device and managed by GPU SVM.
43  *	They are sized based on an array of chunk sizes, which is a GPU SVM
44  *	initialization parameter, and the CPU address space.  Upon GPU fault,
45  *	the largest aligned chunk that fits within the faulting CPU address
46  *	space is chosen for the range size. Ranges are expected to be
47  *	dynamically allocated on GPU fault and removed on an MMU notifier UNMAP
48  *	event. As mentioned above, ranges are tracked in a notifier's Red-Black
49  *	tree.
50  *
51  * - Pages:
52  *	struct drm_gpusvm_pages holds the DMA mapping state for a range of
53  *	CPU virtual addresses: the DMA mapped device addresses,
54  *	the device private pagemap, the IOVA state, the per mapping
55  *	notifier sequence number, and the drm_device that owns the DMA
56  *	mappings.
57  *	A driver embeds one or more struct drm_gpusvm_pages alongside its
58  *	struct drm_gpusvm_range, choosing one of two layouts:
59  *
60  *	1:1 - one drm_gpusvm_pages per range (one drm_device). Simplest
61  *	layout; to mirror a VA range on several devices a driver uses a
62  *	separate range (and notifier) per device, so the HMM fault is taken
63  *	once per device.
64  *
65  *	N:1 - one drm_gpusvm_pages per drm_device, all sharing one range and
66  *	notifier; only the per-device DMA mapping differs. The instances must
67  *	sit in contiguous memory so a single drm_gpusvm_range_set_unmapped()
68  *	can mark them all. A driver can keep one instance inline for the single
69  *	device case and switch to a heap array only when more devices join,
70  *	e.g.:
71  *
72  *	.. code-block:: c
73  *
74  *		struct driver_range {
75  *			struct drm_gpusvm_range base;
76  *			unsigned int num_pages;	// 1: inline_pages, >1: pages[]
77  *			union {
78  *				struct drm_gpusvm_pages inline_pages;
79  *				struct drm_gpusvm_pages *pages;
80  *			};
81  *		};
82  *
83  *		static struct drm_gpusvm_pages *
84  *		driver_pages(struct driver_range *drange)
85  *		{
86  *			return drange->num_pages == 1 ? &drange->inline_pages :
87  *							drange->pages;
88  *		}
89  *
90  *	In the N:1 case the driver allocates the pages array with a zeroing
91  *	allocator (e.g. kcalloc(num_pages, ...)), initialises each entry with
92  *	drm_gpusvm_init_pages(), and frees each entry with
93  *	drm_gpusvm_free_pages() plus the array itself, from its range free
94  *	callback. Each drm_gpusvm_pages is mapped independently by their own
95  *	drm_device.
96  *	Each drm_gpusvm_pages must be zero-initialised and initialised with
97  *	drm_gpusvm_init_pages(), called once per entry.
98  *
99  *	The 1:1 examples below pass @num_pages == 1 and &drange->pages. In the
100  *	N:1 case the driver instead passes the whole array and its count, so a
101  *	single call faults the CPU range once and DMA maps it for every owning
102  *	drm_device, e.g.:
103  *
104  *	.. code-block:: c
105  *
106  *		// GPU fault handler: one fault, one DMA mapping per device
107  *		err = drm_gpusvm_get_pages(gpusvm, driver_pages(drange),
108  *					   drange->num_pages, gpusvm->mm,
109  *					   &range->notifier->notifier,
110  *					   drm_gpusvm_range_start(range),
111  *					   drm_gpusvm_range_end(range), &ctx);
112  *
113  *		// Notifier callback: mark every instance unmapped in one call
114  *		drm_gpusvm_range_set_unmapped(range, driver_pages(drange),
115  *					      drange->num_pages, mmu_range);
116  *
117  *	The unmap and free paths stay per-instance: iterate @num_pages over
118  *	driver_pages(drange) and call drm_gpusvm_unmap_pages() /
119  *	drm_gpusvm_free_pages() for each entry.
120  *
121  * - Operations:
122  *	Define the interface for driver-specific GPU SVM operations such as
123  *	range allocation, notifier allocation, and invalidations.
124  *
125  * - Device Memory Allocations:
126  *	Embedded structure containing enough information for GPU SVM to migrate
127  *	to / from device memory.
128  *
129  * - Device Memory Operations:
130  *	Define the interface for driver-specific device memory operations
131  *	release memory, populate pfns, and copy to / from device memory.
132  *
133  * This layer provides interfaces for allocating, mapping, migrating, and
134  * releasing memory ranges between the CPU and GPU. It handles all core memory
135  * management interactions (DMA mapping, HMM, and migration) and provides
136  * driver-specific virtual functions (vfuncs). This infrastructure is sufficient
137  * to build the expected driver components for an SVM implementation as detailed
138  * below.
139  *
140  * Expected Driver Components:
141  *
142  * - GPU page fault handler:
143  *	Used to create ranges and notifiers based on the fault address,
144  *	optionally migrate the range to device memory, and create GPU bindings.
145  *
146  * - Garbage collector:
147  *	Used to unmap and destroy GPU bindings for ranges.  Ranges are expected
148  *	to be added to the garbage collector upon a MMU_NOTIFY_UNMAP event in
149  *	notifier callback.
150  *
151  * - Notifier callback:
152  *	Used to invalidate and DMA unmap GPU bindings for ranges.
153  */
154 
155 /**
156  * DOC: Locking
157  *
158  * GPU SVM handles locking for core MM interactions, i.e., it locks/unlocks the
159  * mmap lock as needed.
160  *
161  * GPU SVM introduces a global notifier lock, which safeguards the notifier's
162  * range RB tree and list, as well as the range's DMA mappings and sequence
163  * number. GPU SVM manages all necessary locking and unlocking operations,
164  * except for the recheck range's pages being valid
165  * (drm_gpusvm_pages_valid) when the driver is committing GPU bindings.
166  * This lock corresponds to the ``driver->update`` lock mentioned in
167  * Documentation/mm/hmm.rst. Future revisions may transition from a GPU SVM
168  * global lock to a per-notifier lock if finer-grained locking is deemed
169  * necessary.
170  *
171  * In addition to the locking mentioned above, the driver should implement a
172  * lock to safeguard core GPU SVM function calls that modify state, such as
173  * drm_gpusvm_range_find_or_insert and drm_gpusvm_range_remove. This lock is
174  * denoted as 'driver_svm_lock' in code examples. Finer grained driver side
175  * locking should also be possible for concurrent GPU fault processing within a
176  * single GPU SVM. The 'driver_svm_lock' can be via drm_gpusvm_driver_set_lock
177  * to add annotations to GPU SVM.
178  */
179 
180 /**
181  * DOC: Partial Unmapping of Ranges
182  *
183  * Partial unmapping of ranges (e.g., 1M out of 2M is unmapped by CPU resulting
184  * in MMU_NOTIFY_UNMAP event) presents several challenges, with the main one
185  * being that a subset of the range still has CPU and GPU mappings. If the
186  * backing store for the range is in device memory, a subset of the backing
187  * store has references. One option would be to split the range and device
188  * memory backing store, but the implementation for this would be quite
189  * complicated. Given that partial unmappings are rare and driver-defined range
190  * sizes are relatively small, GPU SVM does not support splitting of ranges.
191  *
192  * With no support for range splitting, upon partial unmapping of a range, the
193  * driver is expected to invalidate and destroy the entire range. If the range
194  * has device memory as its backing, the driver is also expected to migrate any
195  * remaining pages back to RAM.
196  */
197 
198 /**
199  * DOC: Examples
200  *
201  * This section provides three examples of how to build the expected driver
202  * components: the GPU page fault handler, the garbage collector, and the
203  * notifier callback.
204  *
205  * The generic code provided does not include logic for complex migration
206  * policies, optimized invalidations, fined grained driver locking, or other
207  * potentially required driver locking (e.g., DMA-resv locks).
208  *
209  * 1) GPU page fault handler
210  *
211  * .. code-block:: c
212  *
213  *	struct driver_range {
214  *		struct drm_gpusvm_range base;
215  *		struct drm_gpusvm_pages pages;
216  *	};
217  *
218  *	int driver_bind_range(struct drm_gpusvm *gpusvm, struct driver_range *drange)
219  *	{
220  *		int err = 0;
221  *
222  *		driver_alloc_and_setup_memory_for_bind(gpusvm, drange);
223  *
224  *		drm_gpusvm_notifier_lock(gpusvm);
225  *		if (drm_gpusvm_pages_valid(gpusvm, &drange->pages))
226  *			driver_commit_bind(gpusvm, drange);
227  *		else
228  *			err = -EAGAIN;
229  *		drm_gpusvm_notifier_unlock(gpusvm);
230  *
231  *		return err;
232  *	}
233  *
234  *	int driver_gpu_fault(struct drm_gpusvm *gpusvm, unsigned long fault_addr,
235  *			     unsigned long gpuva_start, unsigned long gpuva_end)
236  *	{
237  *		struct drm_gpusvm_ctx ctx = {};
238  *		struct driver_range *drange;
239  *		struct drm_gpusvm_range *range;
240  *		int err;
241  *
242  *		driver_svm_lock();
243  *	retry:
244  *		// Always process UNMAPs first so view of GPU SVM ranges is current
245  *		driver_garbage_collector(gpusvm);
246  *
247  *		range = drm_gpusvm_range_find_or_insert(gpusvm, fault_addr,
248  *							gpuva_start, gpuva_end,
249  *						        &ctx);
250  *		if (IS_ERR(range)) {
251  *			err = PTR_ERR(range);
252  *			goto unlock;
253  *		}
254  *		drange = container_of(range, struct driver_range, base);
255  *
256  *		if (driver_migration_policy(range)) {
257  *			err = drm_pagemap_populate_mm(driver_choose_drm_pagemap(),
258  *						      gpuva_start, gpuva_end, gpusvm->mm,
259  *						      ctx->timeslice_ms);
260  *			if (err)	// CPU mappings may have changed
261  *				goto retry;
262  *		}
263  *
264  *		err = drm_gpusvm_get_pages(gpusvm, &drange->pages, 1,
265  *					   gpusvm->mm, &range->notifier->notifier,
266  *					   drm_gpusvm_range_start(range),
267  *					   drm_gpusvm_range_end(range), &ctx);
268  *		if (err == -EOPNOTSUPP || err == -EFAULT || err == -EPERM) {	// CPU mappings changed
269  *			if (err == -EOPNOTSUPP)
270  *				drm_gpusvm_range_evict(gpusvm, range);
271  *			goto retry;
272  *		} else if (err) {
273  *			goto unlock;
274  *		}
275  *
276  *		err = driver_bind_range(gpusvm, drange);
277  *		if (err == -EAGAIN)	// CPU mappings changed
278  *			goto retry
279  *
280  *	unlock:
281  *		driver_svm_unlock();
282  *		return err;
283  *	}
284  *
285  * 2) Garbage Collector
286  *
287  * .. code-block:: c
288  *
289  *	// The driver owns the drm_gpusvm_pages lifecycle. ops->range_free is
290  *	// the final fallback: drm_gpusvm_free_pages() unmaps any
291  *	// lingering DMA mapping and a no-op if already unmapped and frees the
292  *	// dma_addr array. The normal flow is to DMA unmap before
293  *	// drm_gpusvm_range_remove() (before the range leaves the tree).
294  *	void driver_range_free(struct drm_gpusvm_range *range)
295  *	{
296  *		struct driver_range *drange =
297  *			container_of(range, struct driver_range, base);
298  *
299  *		drm_gpusvm_free_pages(range->gpusvm, &drange->pages,
300  *				      drm_gpusvm_range_size(range) >> PAGE_SHIFT);
301  *		kfree(drange);
302  *	}
303  *
304  *	void __driver_garbage_collector(struct drm_gpusvm *gpusvm,
305  *					struct drm_gpusvm_range *range)
306  *	{
307  *		assert_driver_svm_locked(gpusvm);
308  *
309  *		// Partial unmap, migrate any remaining device memory pages back to RAM
310  *		if (range->flags.partial_unmap)
311  *			drm_gpusvm_range_evict(gpusvm, range);
312  *
313  *		driver_unbind_range(range);
314  *		// The pages must be DMA unmapped before drm_gpusvm_range_remove()
315  *		// , so a range is never off the MMU interval tree while still DMA
316  *		// mapped as the original drmsvm design flow. Otherwise a concurrent CPU
317  *		// munmap's notifier could miss this range and free pages still mapped
318  *		// for device DMA. This is the normal unmap point.
319  *		drm_gpusvm_unmap_pages(gpusvm, &drange->pages,
320  *				       drm_gpusvm_range_size(range) >> PAGE_SHIFT,
321  *				       &(struct drm_gpusvm_ctx){ .in_notifier = false });
322  *		drm_gpusvm_range_remove(gpusvm, range);
323  *	}
324  *
325  *	void driver_garbage_collector(struct drm_gpusvm *gpusvm)
326  *	{
327  *		assert_driver_svm_locked(gpusvm);
328  *
329  *		for_each_range_in_garbage_collector(gpusvm, range)
330  *			__driver_garbage_collector(gpusvm, range);
331  *	}
332  *
333  * 3) Notifier callback
334  *
335  * .. code-block:: c
336  *
337  *	void driver_invalidation(struct drm_gpusvm *gpusvm,
338  *				 struct drm_gpusvm_notifier *notifier,
339  *				 const struct mmu_notifier_range *mmu_range)
340  *	{
341  *		struct drm_gpusvm_ctx ctx = { .in_notifier = true, };
342  *		struct drm_gpusvm_range *range = NULL;
343  *		struct driver_range *drange;
344  *
345  *		driver_invalidate_device_pages(gpusvm, mmu_range->start, mmu_range->end);
346  *
347  *		drm_gpusvm_for_each_range(range, notifier, mmu_range->start,
348  *					  mmu_range->end) {
349  *			drange = container_of(range, struct driver_range, base);
350  *
351  *			drm_gpusvm_unmap_pages(gpusvm, &drange->pages,
352  *					       drm_gpusvm_range_size(range) >> PAGE_SHIFT,
353  *					       &ctx);
354  *
355  *			if (mmu_range->event != MMU_NOTIFY_UNMAP)
356  *				continue;
357  *
358  *			drm_gpusvm_range_set_unmapped(range, &drange->pages, 1, mmu_range);
359  *			driver_garbage_collector_add(gpusvm, range);
360  *		}
361  *	}
362  */
363 
364 /**
365  * npages_in_range() - Calculate the number of pages in a given range
366  * @start: The start address of the range
367  * @end: The end address of the range
368  *
369  * This macro calculates the number of pages in a given memory range,
370  * specified by the start and end addresses. It divides the difference
371  * between the end and start addresses by the page size (PAGE_SIZE) to
372  * determine the number of pages in the range.
373  *
374  * Return: The number of pages in the specified range.
375  */
376 static unsigned long
377 npages_in_range(unsigned long start, unsigned long end)
378 {
379 	return (end - start) >> PAGE_SHIFT;
380 }
381 
382 /**
383  * drm_gpusvm_notifier_find() - Find GPU SVM notifier from GPU SVM
384  * @gpusvm: Pointer to the GPU SVM structure.
385  * @start: Start address of the notifier
386  * @end: End address of the notifier
387  *
388  * Return: A pointer to the drm_gpusvm_notifier if found or NULL
389  */
390 struct drm_gpusvm_notifier *
391 drm_gpusvm_notifier_find(struct drm_gpusvm *gpusvm, unsigned long start,
392 			 unsigned long end)
393 {
394 	struct interval_tree_node *itree;
395 
396 	itree = interval_tree_iter_first(&gpusvm->root, start, end - 1);
397 
398 	if (itree)
399 		return container_of(itree, struct drm_gpusvm_notifier, itree);
400 	else
401 		return NULL;
402 }
403 EXPORT_SYMBOL_GPL(drm_gpusvm_notifier_find);
404 
405 /**
406  * drm_gpusvm_range_find() - Find GPU SVM range from GPU SVM notifier
407  * @notifier: Pointer to the GPU SVM notifier structure.
408  * @start: Start address of the range
409  * @end: End address of the range
410  *
411  * Return: A pointer to the drm_gpusvm_range if found or NULL
412  */
413 struct drm_gpusvm_range *
414 drm_gpusvm_range_find(struct drm_gpusvm_notifier *notifier, unsigned long start,
415 		      unsigned long end)
416 {
417 	struct interval_tree_node *itree;
418 
419 	itree = interval_tree_iter_first(&notifier->root, start, end - 1);
420 
421 	if (itree)
422 		return container_of(itree, struct drm_gpusvm_range, itree);
423 	else
424 		return NULL;
425 }
426 EXPORT_SYMBOL_GPL(drm_gpusvm_range_find);
427 
428 /**
429  * drm_gpusvm_notifier_invalidate() - Invalidate a GPU SVM notifier.
430  * @mni: Pointer to the mmu_interval_notifier structure.
431  * @mmu_range: Pointer to the mmu_notifier_range structure.
432  * @cur_seq: Current sequence number.
433  *
434  * This function serves as a generic MMU notifier for GPU SVM. It sets the MMU
435  * notifier sequence number and calls the driver invalidate vfunc under
436  * gpusvm->notifier_lock.
437  *
438  * Return: true if the operation succeeds, false otherwise.
439  */
440 static bool
441 drm_gpusvm_notifier_invalidate(struct mmu_interval_notifier *mni,
442 			       const struct mmu_notifier_range *mmu_range,
443 			       unsigned long cur_seq)
444 {
445 	struct drm_gpusvm_notifier *notifier =
446 		container_of(mni, typeof(*notifier), notifier);
447 	struct drm_gpusvm *gpusvm = notifier->gpusvm;
448 
449 	if (!mmu_notifier_range_blockable(mmu_range))
450 		return false;
451 
452 	down_write(&gpusvm->notifier_lock);
453 	mmu_interval_set_seq(mni, cur_seq);
454 	gpusvm->ops->invalidate(gpusvm, notifier, mmu_range);
455 	up_write(&gpusvm->notifier_lock);
456 
457 	return true;
458 }
459 
460 /*
461  * drm_gpusvm_notifier_ops - MMU interval notifier operations for GPU SVM
462  */
463 static const struct mmu_interval_notifier_ops drm_gpusvm_notifier_ops = {
464 	.invalidate = drm_gpusvm_notifier_invalidate,
465 };
466 
467 /**
468  * drm_gpusvm_init() - Initialize the GPU SVM.
469  * @gpusvm: Pointer to the GPU SVM structure.
470  * @name: Name of the GPU SVM.
471  * @mm: Pointer to the mm_struct for the address space.
472  * @mm_start: Start address of GPU SVM.
473  * @mm_range: Range of the GPU SVM.
474  * @notifier_size: Size of individual notifiers.
475  * @ops: Pointer to the operations structure for GPU SVM.
476  * @chunk_sizes: Pointer to the array of chunk sizes used in range allocation.
477  *               Entries should be powers of 2 in descending order with last
478  *               entry being SZ_4K.
479  * @num_chunks: Number of chunks.
480  *
481  * This function initializes the GPU SVM.
482  *
483  * Note: If only using the simple drm_gpusvm_pages API (get/unmap/free),
484  * then only @gpusvm and @name are expected. The @drm drm_device for dma
485  * mappings is bound per-pages via drm_gpusvm_init_pages() before the first
486  * drm_gpusvm_get_pages() call. However, the same base
487  * @gpusvm can also be used with both modes together in which case the full
488  * setup is needed, where the core drm_gpusvm_pages API will simply never use
489  * the other fields.
490  *
491  * Return: 0 on success, a negative error code on failure.
492  */
493 int drm_gpusvm_init(struct drm_gpusvm *gpusvm,
494 		    const char *name,
495 		    struct mm_struct *mm,
496 		    unsigned long mm_start, unsigned long mm_range,
497 		    unsigned long notifier_size,
498 		    const struct drm_gpusvm_ops *ops,
499 		    const unsigned long *chunk_sizes, int num_chunks)
500 {
501 	if (mm) {
502 		if (!ops->invalidate || !num_chunks)
503 			return -EINVAL;
504 		mmgrab(mm);
505 	} else {
506 		/* No full SVM mode, only core drm_gpusvm_pages API. */
507 		if (ops || num_chunks || mm_range || notifier_size)
508 			return -EINVAL;
509 	}
510 
511 	gpusvm->name = name;
512 	gpusvm->mm = mm;
513 	gpusvm->mm_start = mm_start;
514 	gpusvm->mm_range = mm_range;
515 	gpusvm->notifier_size = notifier_size;
516 	gpusvm->ops = ops;
517 	gpusvm->chunk_sizes = chunk_sizes;
518 	gpusvm->num_chunks = num_chunks;
519 
520 	gpusvm->root = RB_ROOT_CACHED;
521 	INIT_LIST_HEAD(&gpusvm->notifier_list);
522 
523 	init_rwsem(&gpusvm->notifier_lock);
524 
525 	fs_reclaim_acquire(GFP_KERNEL);
526 	might_lock(&gpusvm->notifier_lock);
527 	fs_reclaim_release(GFP_KERNEL);
528 
529 #ifdef CONFIG_LOCKDEP
530 	gpusvm->lock_dep_map = NULL;
531 #endif
532 
533 	return 0;
534 }
535 EXPORT_SYMBOL_GPL(drm_gpusvm_init);
536 
537 /**
538  * to_drm_gpusvm_notifier() - retrieve the container struct for a given rbtree node
539  * @node: a pointer to the rbtree node embedded within a drm_gpusvm_notifier struct
540  *
541  * Return: A pointer to the containing drm_gpusvm_notifier structure.
542  */
543 static struct drm_gpusvm_notifier *to_drm_gpusvm_notifier(struct rb_node *node)
544 {
545 	return container_of(node, struct drm_gpusvm_notifier, itree.rb);
546 }
547 
548 /**
549  * drm_gpusvm_notifier_insert() - Insert GPU SVM notifier
550  * @gpusvm: Pointer to the GPU SVM structure
551  * @notifier: Pointer to the GPU SVM notifier structure
552  *
553  * This function inserts the GPU SVM notifier into the GPU SVM RB tree and list.
554  */
555 static void drm_gpusvm_notifier_insert(struct drm_gpusvm *gpusvm,
556 				       struct drm_gpusvm_notifier *notifier)
557 {
558 	struct rb_node *node;
559 	struct list_head *head;
560 
561 	interval_tree_insert(&notifier->itree, &gpusvm->root);
562 
563 	node = rb_prev(&notifier->itree.rb);
564 	if (node)
565 		head = &(to_drm_gpusvm_notifier(node))->entry;
566 	else
567 		head = &gpusvm->notifier_list;
568 
569 	list_add(&notifier->entry, head);
570 }
571 
572 /**
573  * drm_gpusvm_notifier_remove() - Remove GPU SVM notifier
574  * @gpusvm: Pointer to the GPU SVM tructure
575  * @notifier: Pointer to the GPU SVM notifier structure
576  *
577  * This function removes the GPU SVM notifier from the GPU SVM RB tree and list.
578  */
579 static void drm_gpusvm_notifier_remove(struct drm_gpusvm *gpusvm,
580 				       struct drm_gpusvm_notifier *notifier)
581 {
582 	interval_tree_remove(&notifier->itree, &gpusvm->root);
583 	list_del(&notifier->entry);
584 }
585 
586 /**
587  * drm_gpusvm_fini() - Finalize the GPU SVM.
588  * @gpusvm: Pointer to the GPU SVM structure.
589  *
590  * This function finalizes the GPU SVM by cleaning up any remaining ranges and
591  * notifiers, and dropping a reference to struct MM.
592  */
593 void drm_gpusvm_fini(struct drm_gpusvm *gpusvm)
594 {
595 	struct drm_gpusvm_notifier *notifier, *next;
596 
597 	drm_gpusvm_for_each_notifier_safe(notifier, next, gpusvm, 0, LONG_MAX) {
598 		struct drm_gpusvm_range *range, *__next;
599 
600 		/*
601 		 * Remove notifier first to avoid racing with any invalidation
602 		 */
603 		mmu_interval_notifier_remove(&notifier->notifier);
604 		notifier->flags.removed = true;
605 
606 		drm_gpusvm_for_each_range_safe(range, __next, notifier, 0,
607 					       LONG_MAX)
608 			drm_gpusvm_range_remove(gpusvm, range);
609 	}
610 
611 	if (gpusvm->mm)
612 		mmdrop(gpusvm->mm);
613 	WARN_ON(!RB_EMPTY_ROOT(&gpusvm->root.rb_root));
614 }
615 EXPORT_SYMBOL_GPL(drm_gpusvm_fini);
616 
617 /**
618  * drm_gpusvm_notifier_alloc() - Allocate GPU SVM notifier
619  * @gpusvm: Pointer to the GPU SVM structure
620  * @fault_addr: Fault address
621  *
622  * This function allocates and initializes the GPU SVM notifier structure.
623  *
624  * Return: Pointer to the allocated GPU SVM notifier on success, ERR_PTR() on failure.
625  */
626 static struct drm_gpusvm_notifier *
627 drm_gpusvm_notifier_alloc(struct drm_gpusvm *gpusvm, unsigned long fault_addr)
628 {
629 	struct drm_gpusvm_notifier *notifier;
630 
631 	if (gpusvm->ops->notifier_alloc)
632 		notifier = gpusvm->ops->notifier_alloc();
633 	else
634 		notifier = kzalloc_obj(*notifier);
635 
636 	if (!notifier)
637 		return ERR_PTR(-ENOMEM);
638 
639 	notifier->gpusvm = gpusvm;
640 	notifier->itree.start = ALIGN_DOWN(fault_addr, gpusvm->notifier_size);
641 	notifier->itree.last = ALIGN(fault_addr + 1, gpusvm->notifier_size) - 1;
642 	INIT_LIST_HEAD(&notifier->entry);
643 	notifier->root = RB_ROOT_CACHED;
644 	INIT_LIST_HEAD(&notifier->range_list);
645 
646 	return notifier;
647 }
648 
649 /**
650  * drm_gpusvm_notifier_free() - Free GPU SVM notifier
651  * @gpusvm: Pointer to the GPU SVM structure
652  * @notifier: Pointer to the GPU SVM notifier structure
653  *
654  * This function frees the GPU SVM notifier structure.
655  */
656 static void drm_gpusvm_notifier_free(struct drm_gpusvm *gpusvm,
657 				     struct drm_gpusvm_notifier *notifier)
658 {
659 	WARN_ON(!RB_EMPTY_ROOT(&notifier->root.rb_root));
660 
661 	if (gpusvm->ops->notifier_free)
662 		gpusvm->ops->notifier_free(notifier);
663 	else
664 		kfree(notifier);
665 }
666 
667 /**
668  * to_drm_gpusvm_range() - retrieve the container struct for a given rbtree node
669  * @node: a pointer to the rbtree node embedded within a drm_gpusvm_range struct
670  *
671  * Return: A pointer to the containing drm_gpusvm_range structure.
672  */
673 static struct drm_gpusvm_range *to_drm_gpusvm_range(struct rb_node *node)
674 {
675 	return container_of(node, struct drm_gpusvm_range, itree.rb);
676 }
677 
678 /**
679  * drm_gpusvm_range_insert() - Insert GPU SVM range
680  * @notifier: Pointer to the GPU SVM notifier structure
681  * @range: Pointer to the GPU SVM range structure
682  *
683  * This function inserts the GPU SVM range into the notifier RB tree and list.
684  */
685 static void drm_gpusvm_range_insert(struct drm_gpusvm_notifier *notifier,
686 				    struct drm_gpusvm_range *range)
687 {
688 	struct rb_node *node;
689 	struct list_head *head;
690 
691 	drm_gpusvm_notifier_lock(notifier->gpusvm);
692 	interval_tree_insert(&range->itree, &notifier->root);
693 
694 	node = rb_prev(&range->itree.rb);
695 	if (node)
696 		head = &(to_drm_gpusvm_range(node))->entry;
697 	else
698 		head = &notifier->range_list;
699 
700 	list_add(&range->entry, head);
701 	drm_gpusvm_notifier_unlock(notifier->gpusvm);
702 }
703 
704 /**
705  * __drm_gpusvm_range_remove() - Remove GPU SVM range
706  * @notifier: Pointer to the GPU SVM notifier structure
707  * @range: Pointer to the GPU SVM range structure
708  *
709  * This macro removes the GPU SVM range from the notifier RB tree and list.
710  */
711 static void __drm_gpusvm_range_remove(struct drm_gpusvm_notifier *notifier,
712 				      struct drm_gpusvm_range *range)
713 {
714 	interval_tree_remove(&range->itree, &notifier->root);
715 	list_del(&range->entry);
716 }
717 
718 /**
719  * drm_gpusvm_range_alloc() - Allocate GPU SVM range
720  * @gpusvm: Pointer to the GPU SVM structure
721  * @notifier: Pointer to the GPU SVM notifier structure
722  * @fault_addr: Fault address
723  * @chunk_size: Chunk size
724  * @migrate_devmem: Flag indicating whether to migrate device memory
725  *
726  * This function allocates and initializes the GPU SVM range structure.
727  *
728  * Return: Pointer to the allocated GPU SVM range on success, ERR_PTR() on failure.
729  */
730 static struct drm_gpusvm_range *
731 drm_gpusvm_range_alloc(struct drm_gpusvm *gpusvm,
732 		       struct drm_gpusvm_notifier *notifier,
733 		       unsigned long fault_addr, unsigned long chunk_size,
734 		       bool migrate_devmem)
735 {
736 	struct drm_gpusvm_range *range;
737 
738 	if (gpusvm->ops->range_alloc)
739 		range = gpusvm->ops->range_alloc(gpusvm);
740 	else
741 		range = kzalloc_obj(*range);
742 
743 	if (!range)
744 		return ERR_PTR(-ENOMEM);
745 
746 	kref_init(&range->refcount);
747 	range->gpusvm = gpusvm;
748 	range->notifier = notifier;
749 	range->itree.start = ALIGN_DOWN(fault_addr, chunk_size);
750 	range->itree.last = ALIGN(fault_addr + 1, chunk_size) - 1;
751 	INIT_LIST_HEAD(&range->entry);
752 	range->flags.migrate_devmem = migrate_devmem ? 1 : 0;
753 
754 	return range;
755 }
756 
757 /**
758  * drm_gpusvm_hmm_pfn_to_order() - Get the largest CPU mapping order.
759  * @hmm_pfn: The current hmm_pfn.
760  * @hmm_pfn_index: Index of the @hmm_pfn within the pfn array.
761  * @npages: Number of pages within the pfn array i.e the hmm range size.
762  *
763  * To allow skipping PFNs with the same flags (like when they belong to
764  * the same huge PTE) when looping over the pfn array, take a given a hmm_pfn,
765  * and return the largest order that will fit inside the CPU PTE, but also
766  * crucially accounting for the original hmm range boundaries.
767  *
768  * Return: The largest order that will safely fit within the size of the hmm_pfn
769  * CPU PTE.
770  */
771 static unsigned int drm_gpusvm_hmm_pfn_to_order(unsigned long hmm_pfn,
772 						unsigned long hmm_pfn_index,
773 						unsigned long npages)
774 {
775 	unsigned long size;
776 
777 	size = 1UL << hmm_pfn_to_map_order(hmm_pfn);
778 	size -= (hmm_pfn & ~HMM_PFN_FLAGS) & (size - 1);
779 	hmm_pfn_index += size;
780 	if (hmm_pfn_index > npages)
781 		size -= (hmm_pfn_index - npages);
782 
783 	return ilog2(size);
784 }
785 
786 /**
787  * drm_gpusvm_check_pages() - Check pages
788  * @gpusvm: Pointer to the GPU SVM structure
789  * @notifier: Pointer to the GPU SVM notifier structure
790  * @start: Start address
791  * @end: End address
792  * @dev_private_owner: The device private page owner
793  *
794  * Check if pages between start and end have been faulted in on the CPU. Use to
795  * prevent migration of pages without CPU backing store.
796  *
797  * Return: True if pages have been faulted into CPU, False otherwise
798  */
799 static bool drm_gpusvm_check_pages(struct drm_gpusvm *gpusvm,
800 				   struct drm_gpusvm_notifier *notifier,
801 				   unsigned long start, unsigned long end,
802 				   void *dev_private_owner)
803 {
804 	struct hmm_range hmm_range = {
805 		.default_flags = 0,
806 		.notifier = &notifier->notifier,
807 		.start = start,
808 		.end = end,
809 		.dev_private_owner = dev_private_owner,
810 	};
811 	unsigned long timeout =
812 		jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
813 	unsigned long *pfns;
814 	unsigned long npages = npages_in_range(start, end);
815 	int err, i;
816 
817 	mmap_assert_locked(gpusvm->mm);
818 
819 	pfns = kvmalloc_objs(*pfns, npages);
820 	if (!pfns)
821 		return false;
822 
823 	hmm_range.notifier_seq = mmu_interval_read_begin(&notifier->notifier);
824 	hmm_range.hmm_pfns = pfns;
825 
826 	while (true) {
827 		err = hmm_range_fault(&hmm_range);
828 		if (err == -EBUSY) {
829 			if (time_after(jiffies, timeout))
830 				break;
831 
832 			hmm_range.notifier_seq =
833 				mmu_interval_read_begin(&notifier->notifier);
834 			continue;
835 		}
836 		break;
837 	}
838 	if (err)
839 		goto err_free;
840 
841 	for (i = 0; i < npages;) {
842 		if (!(pfns[i] & HMM_PFN_VALID)) {
843 			err = -EFAULT;
844 			goto err_free;
845 		}
846 		i += 0x1 << drm_gpusvm_hmm_pfn_to_order(pfns[i], i, npages);
847 	}
848 
849 err_free:
850 	kvfree(pfns);
851 	return err ? false : true;
852 }
853 
854 /**
855  * drm_gpusvm_scan_mm() - Check the migration state of a drm_gpusvm_range
856  * @range: Pointer to the struct drm_gpusvm_range to check.
857  * @dev_private_owner: The struct dev_private_owner to use to determine
858  * compatible device-private pages.
859  * @pagemap: The struct dev_pagemap pointer to use for pagemap-specific
860  * checks.
861  *
862  * Scan the CPU address space corresponding to @range and return the
863  * current migration state. Note that the result may be invalid as
864  * soon as the function returns. It's an advisory check.
865  *
866  * TODO: Bail early and call hmm_range_fault() for subranges.
867  *
868  * Return: See &enum drm_gpusvm_scan_result.
869  */
870 enum drm_gpusvm_scan_result drm_gpusvm_scan_mm(struct drm_gpusvm_range *range,
871 					       void *dev_private_owner,
872 					       const struct dev_pagemap *pagemap)
873 {
874 	struct mmu_interval_notifier *notifier = &range->notifier->notifier;
875 	unsigned long start = drm_gpusvm_range_start(range);
876 	unsigned long end = drm_gpusvm_range_end(range);
877 	struct hmm_range hmm_range = {
878 		.default_flags = 0,
879 		.notifier = notifier,
880 		.start = start,
881 		.end = end,
882 		.dev_private_owner = dev_private_owner,
883 	};
884 	unsigned long timeout = msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
885 	enum drm_gpusvm_scan_result state = DRM_GPUSVM_SCAN_UNPOPULATED, new_state;
886 	unsigned long *pfns;
887 	unsigned long npages = npages_in_range(start, end);
888 	const struct dev_pagemap *other = NULL;
889 	int err, i;
890 
891 	pfns = kvzalloc_objs(*pfns, npages);
892 	if (!pfns)
893 		return DRM_GPUSVM_SCAN_UNPOPULATED;
894 
895 	hmm_range.hmm_pfns = pfns;
896 
897 retry:
898 	err = hmm_range_fault_unlocked_timeout(&hmm_range, timeout);
899 	if (err)
900 		goto err_free;
901 
902 	drm_gpusvm_notifier_lock(range->gpusvm);
903 	if (mmu_interval_read_retry(notifier, hmm_range.notifier_seq)) {
904 		drm_gpusvm_notifier_unlock(range->gpusvm);
905 		goto retry;
906 	}
907 
908 	for (i = 0; i < npages;) {
909 		struct page *page;
910 		const struct dev_pagemap *cur = NULL;
911 
912 		if (!(pfns[i] & HMM_PFN_VALID)) {
913 			state = DRM_GPUSVM_SCAN_UNPOPULATED;
914 			break;
915 		}
916 
917 		page = hmm_pfn_to_page(pfns[i]);
918 		if (is_device_private_page(page) ||
919 		    is_device_coherent_page(page))
920 			cur = page_pgmap(page);
921 
922 		if (cur == pagemap) {
923 			new_state = DRM_GPUSVM_SCAN_EQUAL;
924 		} else if (cur && (cur == other || !other)) {
925 			new_state = DRM_GPUSVM_SCAN_OTHER;
926 			other = cur;
927 		} else if (cur) {
928 			new_state = DRM_GPUSVM_SCAN_MIXED_DEVICE;
929 		} else {
930 			new_state = DRM_GPUSVM_SCAN_SYSTEM;
931 		}
932 
933 		/*
934 		 * TODO: Could use an array for state
935 		 * transitions, and caller might want it
936 		 * to bail early for some results.
937 		 */
938 		if (state == DRM_GPUSVM_SCAN_UNPOPULATED) {
939 			state = new_state;
940 		} else if (state != new_state) {
941 			if (new_state == DRM_GPUSVM_SCAN_SYSTEM ||
942 			    state == DRM_GPUSVM_SCAN_SYSTEM)
943 				state = DRM_GPUSVM_SCAN_MIXED;
944 			else if (state != DRM_GPUSVM_SCAN_MIXED)
945 				state = DRM_GPUSVM_SCAN_MIXED_DEVICE;
946 		}
947 
948 		i += 1ul << drm_gpusvm_hmm_pfn_to_order(pfns[i], i, npages);
949 	}
950 
951 	drm_gpusvm_notifier_unlock(range->gpusvm);
952 
953 err_free:
954 	kvfree(pfns);
955 	return state;
956 }
957 EXPORT_SYMBOL(drm_gpusvm_scan_mm);
958 
959 /**
960  * drm_gpusvm_range_chunk_size() - Determine chunk size for GPU SVM range
961  * @gpusvm: Pointer to the GPU SVM structure
962  * @notifier: Pointer to the GPU SVM notifier structure
963  * @vas: Pointer to the virtual memory area structure
964  * @fault_addr: Fault address
965  * @gpuva_start: Start address of GPUVA which mirrors CPU
966  * @gpuva_end: End address of GPUVA which mirrors CPU
967  * @check_pages_threshold: Check CPU pages for present threshold
968  * @dev_private_owner: The device private page owner
969  *
970  * This function determines the chunk size for the GPU SVM range based on the
971  * fault address, GPU SVM chunk sizes, existing GPU SVM ranges, and the virtual
972  * memory area boundaries.
973  *
974  * Return: Chunk size on success, LONG_MAX on failure.
975  */
976 static unsigned long
977 drm_gpusvm_range_chunk_size(struct drm_gpusvm *gpusvm,
978 			    struct drm_gpusvm_notifier *notifier,
979 			    struct vm_area_struct *vas,
980 			    unsigned long fault_addr,
981 			    unsigned long gpuva_start,
982 			    unsigned long gpuva_end,
983 			    unsigned long check_pages_threshold,
984 			    void *dev_private_owner)
985 {
986 	unsigned long start, end;
987 	int i = 0;
988 
989 retry:
990 	for (; i < gpusvm->num_chunks; ++i) {
991 		start = ALIGN_DOWN(fault_addr, gpusvm->chunk_sizes[i]);
992 		end = ALIGN(fault_addr + 1, gpusvm->chunk_sizes[i]);
993 
994 		if (start >= vas->vm_start && end <= vas->vm_end &&
995 		    start >= drm_gpusvm_notifier_start(notifier) &&
996 		    end <= drm_gpusvm_notifier_end(notifier) &&
997 		    start >= gpuva_start && end <= gpuva_end)
998 			break;
999 	}
1000 
1001 	if (i == gpusvm->num_chunks)
1002 		return LONG_MAX;
1003 
1004 	/*
1005 	 * If allocation more than page, ensure not to overlap with existing
1006 	 * ranges.
1007 	 */
1008 	if (end - start != SZ_4K) {
1009 		struct drm_gpusvm_range *range;
1010 
1011 		range = drm_gpusvm_range_find(notifier, start, end);
1012 		if (range) {
1013 			++i;
1014 			goto retry;
1015 		}
1016 
1017 		/*
1018 		 * XXX: Only create range on pages CPU has faulted in. Without
1019 		 * this check, or prefault, on BMG 'xe_exec_system_allocator --r
1020 		 * process-many-malloc' fails. In the failure case, each process
1021 		 * mallocs 16k but the CPU VMA is ~128k which results in 64k SVM
1022 		 * ranges. When migrating the SVM ranges, some processes fail in
1023 		 * drm_pagemap_migrate_to_devmem with 'migrate.cpages != npages'
1024 		 * and then upon drm_gpusvm_get_pages device pages from
1025 		 * other processes are collected + faulted in which creates all
1026 		 * sorts of problems. Unsure exactly how this happening, also
1027 		 * problem goes away if 'xe_exec_system_allocator --r
1028 		 * process-many-malloc' mallocs at least 64k at a time.
1029 		 */
1030 		if (end - start <= check_pages_threshold &&
1031 		    !drm_gpusvm_check_pages(gpusvm, notifier, start, end, dev_private_owner)) {
1032 			++i;
1033 			goto retry;
1034 		}
1035 	}
1036 
1037 	return end - start;
1038 }
1039 
1040 #ifdef CONFIG_LOCKDEP
1041 /**
1042  * drm_gpusvm_driver_lock_held() - Assert GPU SVM driver lock is held
1043  * @gpusvm: Pointer to the GPU SVM structure.
1044  *
1045  * Ensure driver lock is held.
1046  */
1047 static void drm_gpusvm_driver_lock_held(struct drm_gpusvm *gpusvm)
1048 {
1049 	if ((gpusvm)->lock_dep_map)
1050 		lockdep_assert(lock_is_held_type((gpusvm)->lock_dep_map, 0));
1051 }
1052 #else
1053 static void drm_gpusvm_driver_lock_held(struct drm_gpusvm *gpusvm)
1054 {
1055 }
1056 #endif
1057 
1058 /**
1059  * drm_gpusvm_find_vma_start() - Find start address for first VMA in range
1060  * @gpusvm: Pointer to the GPU SVM structure
1061  * @start: The inclusive start user address.
1062  * @end: The exclusive end user address.
1063  *
1064  * Returns: The start address of first VMA within the provided range,
1065  * ULONG_MAX otherwise. Assumes start_addr < end_addr.
1066  */
1067 unsigned long
1068 drm_gpusvm_find_vma_start(struct drm_gpusvm *gpusvm,
1069 			  unsigned long start,
1070 			  unsigned long end)
1071 {
1072 	struct mm_struct *mm = gpusvm->mm;
1073 	struct vm_area_struct *vma;
1074 	unsigned long addr = ULONG_MAX;
1075 
1076 	if (!mmget_not_zero(mm))
1077 		return addr;
1078 
1079 	mmap_read_lock(mm);
1080 
1081 	vma = find_vma_intersection(mm, start, end);
1082 	if (vma)
1083 		addr =  vma->vm_start;
1084 
1085 	mmap_read_unlock(mm);
1086 	mmput(mm);
1087 
1088 	return addr;
1089 }
1090 EXPORT_SYMBOL_GPL(drm_gpusvm_find_vma_start);
1091 
1092 /**
1093  * drm_gpusvm_range_find_or_insert() - Find or insert GPU SVM range
1094  * @gpusvm: Pointer to the GPU SVM structure
1095  * @fault_addr: Fault address
1096  * @gpuva_start: Start address of GPUVA which mirrors CPU
1097  * @gpuva_end: End address of GPUVA which mirrors CPU
1098  * @ctx: GPU SVM context
1099  *
1100  * This function finds or inserts a newly allocated a GPU SVM range based on the
1101  * fault address. Caller must hold a lock to protect range lookup and insertion.
1102  *
1103  * Return: Pointer to the GPU SVM range on success, ERR_PTR() on failure.
1104  */
1105 struct drm_gpusvm_range *
1106 drm_gpusvm_range_find_or_insert(struct drm_gpusvm *gpusvm,
1107 				unsigned long fault_addr,
1108 				unsigned long gpuva_start,
1109 				unsigned long gpuva_end,
1110 				const struct drm_gpusvm_ctx *ctx)
1111 {
1112 	struct drm_gpusvm_notifier *notifier;
1113 	struct drm_gpusvm_range *range;
1114 	struct mm_struct *mm = gpusvm->mm;
1115 	struct vm_area_struct *vas;
1116 	bool notifier_alloc = false;
1117 	unsigned long chunk_size;
1118 	int err;
1119 	bool migrate_devmem;
1120 
1121 	drm_gpusvm_driver_lock_held(gpusvm);
1122 
1123 	if (fault_addr < gpusvm->mm_start ||
1124 	    fault_addr > gpusvm->mm_start + gpusvm->mm_range)
1125 		return ERR_PTR(-EINVAL);
1126 
1127 	if (!mmget_not_zero(mm))
1128 		return ERR_PTR(-EFAULT);
1129 
1130 	notifier = drm_gpusvm_notifier_find(gpusvm, fault_addr, fault_addr + 1);
1131 	if (!notifier) {
1132 		notifier = drm_gpusvm_notifier_alloc(gpusvm, fault_addr);
1133 		if (IS_ERR(notifier)) {
1134 			err = PTR_ERR(notifier);
1135 			goto err_mmunlock;
1136 		}
1137 		notifier_alloc = true;
1138 		err = mmu_interval_notifier_insert(&notifier->notifier,
1139 						   mm,
1140 						   drm_gpusvm_notifier_start(notifier),
1141 						   drm_gpusvm_notifier_size(notifier),
1142 						   &drm_gpusvm_notifier_ops);
1143 		if (err)
1144 			goto err_notifier;
1145 	}
1146 
1147 	mmap_read_lock(mm);
1148 
1149 	vas = vma_lookup(mm, fault_addr);
1150 	if (!vas) {
1151 		err = -ENOENT;
1152 		goto err_notifier_remove;
1153 	}
1154 
1155 	if (!ctx->read_only && !(vas->vm_flags & VM_WRITE)) {
1156 		err = -EPERM;
1157 		goto err_notifier_remove;
1158 	}
1159 
1160 	if (vas->vm_flags & (VM_IO | VM_PFNMAP)) {
1161 		err = -EIO;
1162 		goto err_notifier_remove;
1163 	}
1164 
1165 	range = drm_gpusvm_range_find(notifier, fault_addr, fault_addr + 1);
1166 	if (range)
1167 		goto out_mmunlock;
1168 	/*
1169 	 * XXX: Short-circuiting migration based on migrate_vma_* current
1170 	 * limitations. If/when migrate_vma_* add more support, this logic will
1171 	 * have to change.
1172 	 */
1173 	migrate_devmem = ctx->devmem_possible && vma_is_anonymous(vas);
1174 
1175 	chunk_size = drm_gpusvm_range_chunk_size(gpusvm, notifier, vas,
1176 						 fault_addr, gpuva_start,
1177 						 gpuva_end,
1178 						 ctx->check_pages_threshold,
1179 						 ctx->device_private_page_owner);
1180 	if (chunk_size == LONG_MAX) {
1181 		err = -EINVAL;
1182 		goto err_notifier_remove;
1183 	}
1184 
1185 	range = drm_gpusvm_range_alloc(gpusvm, notifier, fault_addr, chunk_size,
1186 				       migrate_devmem);
1187 	if (IS_ERR(range)) {
1188 		err = PTR_ERR(range);
1189 		goto err_notifier_remove;
1190 	}
1191 
1192 	drm_gpusvm_range_insert(notifier, range);
1193 	if (notifier_alloc)
1194 		drm_gpusvm_notifier_insert(gpusvm, notifier);
1195 
1196 out_mmunlock:
1197 	mmap_read_unlock(mm);
1198 	mmput(mm);
1199 
1200 	return range;
1201 
1202 err_notifier_remove:
1203 	mmap_read_unlock(mm);
1204 	if (notifier_alloc)
1205 		mmu_interval_notifier_remove(&notifier->notifier);
1206 err_notifier:
1207 	if (notifier_alloc)
1208 		drm_gpusvm_notifier_free(gpusvm, notifier);
1209 err_mmunlock:
1210 	mmput(mm);
1211 	return ERR_PTR(err);
1212 }
1213 EXPORT_SYMBOL_GPL(drm_gpusvm_range_find_or_insert);
1214 
1215 /**
1216  * __drm_gpusvm_unmap_pages() - Unmap pages associated with GPU SVM pages (internal)
1217  * @gpusvm: Pointer to the GPU SVM structure
1218  * @svm_pages: Pointer to the GPU SVM pages structure
1219  * @npages: Number of pages to unmap
1220  *
1221  * This function unmap pages associated with a GPU SVM pages struct. Assumes and
1222  * asserts correct locking is in place when called.
1223  */
1224 static void __drm_gpusvm_unmap_pages(struct drm_gpusvm *gpusvm,
1225 				     struct drm_gpusvm_pages *svm_pages,
1226 				     unsigned long npages)
1227 {
1228 	struct drm_pagemap *dpagemap = svm_pages->dpagemap;
1229 	struct device *dev;
1230 	unsigned long i, j;
1231 
1232 	lockdep_assert_held(&gpusvm->notifier_lock);
1233 
1234 	if (!svm_pages->drm)
1235 		return;
1236 
1237 	dev = svm_pages->drm->dev;
1238 
1239 	if (svm_pages->flags.has_dma_mapping) {
1240 		struct drm_gpusvm_pages_flags flags = {
1241 			.__flags = svm_pages->flags.__flags,
1242 		};
1243 		const struct drm_pagemap_addr *addrs =
1244 			drm_gpusvm_pages_first_dma(svm_pages, NULL);
1245 		bool use_iova = dma_use_iova(&svm_pages->state);
1246 
1247 		/*
1248 		 * IOVA is reserved for the whole range but only the linked
1249 		 * system pages (state_offset bytes) need unlinking; free the
1250 		 * entire reservation to avoid leaking the device-page part.
1251 		 * On the error path state_offset is 0, so just free it.
1252 		 */
1253 		if (use_iova) {
1254 			if (svm_pages->state_offset)
1255 				dma_iova_unlink(dev, &svm_pages->state, 0,
1256 						svm_pages->state_offset,
1257 						addrs[0].dir, 0);
1258 			dma_iova_free(dev, &svm_pages->state);
1259 		}
1260 
1261 		/*
1262 		 * With IOVA and no device page the unlink above tore every
1263 		 * entry down, and that is also when the range may be folded
1264 		 * to one entry, which must not be walked per entry. dpagemap
1265 		 * is set before the first device_map(), so it is also right
1266 		 * on the error path, where the flags are not published yet.
1267 		 */
1268 		for (i = 0, j = 0;
1269 		     (!use_iova || dpagemap) && i < npages; j++) {
1270 			const struct drm_pagemap_addr *addr = &addrs[j];
1271 
1272 			if (addr->proto == DRM_INTERCONNECT_SYSTEM) {
1273 				/*
1274 				 * Linked IOVA pages were already torn down by
1275 				 * the dma_iova_unlink()/dma_iova_free() above;
1276 				 * only the non-IOVA mappings need unmap here.
1277 				 */
1278 				if (!use_iova)
1279 					dma_unmap_page(dev,
1280 						       addr->addr,
1281 						       PAGE_SIZE << addr->order,
1282 						       addr->dir);
1283 			} else if (dpagemap && dpagemap->ops->device_unmap)
1284 				dpagemap->ops->device_unmap(dpagemap,
1285 							    dev, addr);
1286 			i += 1 << addr->order;
1287 		}
1288 
1289 		/* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */
1290 		flags.has_devmem_pages = false;
1291 		flags.has_dma_mapping = false;
1292 		WRITE_ONCE(svm_pages->flags.__flags, flags.__flags);
1293 
1294 		drm_pagemap_put(svm_pages->dpagemap);
1295 		svm_pages->dpagemap = NULL;
1296 	}
1297 }
1298 
1299 /**
1300  * __drm_gpusvm_free_pages() - Free dma array associated with GPU SVM pages
1301  * @gpusvm: Pointer to the GPU SVM structure
1302  * @svm_pages: Pointer to the GPU SVM pages structure
1303  *
1304  * This function frees the dma address array associated with a GPU SVM range.
1305  */
1306 static void __drm_gpusvm_free_pages(struct drm_gpusvm *gpusvm,
1307 				    struct drm_gpusvm_pages *svm_pages)
1308 {
1309 	lockdep_assert_held(&gpusvm->notifier_lock);
1310 
1311 	if (svm_pages->flags.inline_dma_mapping) {
1312 		struct drm_gpusvm_pages_flags flags = {
1313 			.__flags = svm_pages->flags.__flags,
1314 		};
1315 
1316 		svm_pages->inline_addr = (struct drm_pagemap_addr){};
1317 		flags.inline_dma_mapping = false;
1318 		/* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */
1319 		WRITE_ONCE(svm_pages->flags.__flags, flags.__flags);
1320 		return;
1321 	}
1322 
1323 	if (svm_pages->dma_addr) {
1324 		kvfree(svm_pages->dma_addr);
1325 		svm_pages->dma_addr = NULL;
1326 	}
1327 }
1328 
1329 /**
1330  * drm_gpusvm_free_pages() - Free dma-mapping associated with GPU SVM pages
1331  * struct
1332  * @gpusvm: Pointer to the GPU SVM structure
1333  * @svm_pages: Pointer to the GPU SVM pages structure
1334  * @npages: Number of mapped pages
1335  *
1336  * This function unmaps and frees the dma address array associated with a GPU
1337  * SVM pages struct.
1338  */
1339 void drm_gpusvm_free_pages(struct drm_gpusvm *gpusvm,
1340 			   struct drm_gpusvm_pages *svm_pages,
1341 			   unsigned long npages)
1342 {
1343 	drm_gpusvm_notifier_lock(gpusvm);
1344 	__drm_gpusvm_unmap_pages(gpusvm, svm_pages, npages);
1345 	__drm_gpusvm_free_pages(gpusvm, svm_pages);
1346 	drm_gpusvm_notifier_unlock(gpusvm);
1347 }
1348 EXPORT_SYMBOL_GPL(drm_gpusvm_free_pages);
1349 
1350 /**
1351  * drm_gpusvm_range_remove() - Remove GPU SVM range
1352  * @gpusvm: Pointer to the GPU SVM structure
1353  * @range: Pointer to the GPU SVM range to be removed
1354  *
1355  * This function removes the specified GPU SVM range and also removes the parent
1356  * GPU SVM notifier if no more ranges remain in the notifier. The caller must
1357  * hold a lock to protect range and notifier removal.
1358  *
1359  * This function does not unmap or free the drm_gpusvm_pages, the driver owns
1360  * that lifecycle. The caller must DMA unmap the range's pages before calling
1361  * this function, so a range is never removed from the MMU interval tree while
1362  * still DMA mapped. Typically the driver calls drm_gpusvm_unmap_pages() first.
1363  * And the range_free callback's drm_gpusvm_free_pages() is a final fallback safe
1364  * net.
1365  */
1366 void drm_gpusvm_range_remove(struct drm_gpusvm *gpusvm,
1367 			     struct drm_gpusvm_range *range)
1368 {
1369 	struct drm_gpusvm_notifier *notifier;
1370 
1371 	drm_gpusvm_driver_lock_held(gpusvm);
1372 
1373 	notifier = drm_gpusvm_notifier_find(gpusvm,
1374 					    drm_gpusvm_range_start(range),
1375 					    drm_gpusvm_range_start(range) + 1);
1376 	if (WARN_ON_ONCE(!notifier))
1377 		return;
1378 
1379 	drm_gpusvm_notifier_lock(gpusvm);
1380 	__drm_gpusvm_range_remove(notifier, range);
1381 	drm_gpusvm_notifier_unlock(gpusvm);
1382 
1383 	drm_gpusvm_range_put(range);
1384 
1385 	if (RB_EMPTY_ROOT(&notifier->root.rb_root)) {
1386 		if (!notifier->flags.removed)
1387 			mmu_interval_notifier_remove(&notifier->notifier);
1388 		drm_gpusvm_notifier_remove(gpusvm, notifier);
1389 		drm_gpusvm_notifier_free(gpusvm, notifier);
1390 	}
1391 }
1392 EXPORT_SYMBOL_GPL(drm_gpusvm_range_remove);
1393 
1394 /**
1395  * drm_gpusvm_range_get() - Get a reference to GPU SVM range
1396  * @range: Pointer to the GPU SVM range
1397  *
1398  * This function increments the reference count of the specified GPU SVM range.
1399  *
1400  * Return: Pointer to the GPU SVM range.
1401  */
1402 struct drm_gpusvm_range *
1403 drm_gpusvm_range_get(struct drm_gpusvm_range *range)
1404 {
1405 	kref_get(&range->refcount);
1406 
1407 	return range;
1408 }
1409 EXPORT_SYMBOL_GPL(drm_gpusvm_range_get);
1410 
1411 /**
1412  * drm_gpusvm_range_destroy() - Destroy GPU SVM range
1413  * @refcount: Pointer to the reference counter embedded in the GPU SVM range
1414  *
1415  * This function destroys the specified GPU SVM range when its reference count
1416  * reaches zero. If a custom range-free function is provided, it is invoked to
1417  * free the range; otherwise, the range is deallocated using kfree().
1418  */
1419 static void drm_gpusvm_range_destroy(struct kref *refcount)
1420 {
1421 	struct drm_gpusvm_range *range =
1422 		container_of(refcount, struct drm_gpusvm_range, refcount);
1423 	struct drm_gpusvm *gpusvm = range->gpusvm;
1424 
1425 	if (gpusvm->ops->range_free)
1426 		gpusvm->ops->range_free(range);
1427 	else
1428 		kfree(range);
1429 }
1430 
1431 /**
1432  * drm_gpusvm_range_put() - Put a reference to GPU SVM range
1433  * @range: Pointer to the GPU SVM range
1434  *
1435  * This function decrements the reference count of the specified GPU SVM range
1436  * and frees it when the count reaches zero.
1437  */
1438 void drm_gpusvm_range_put(struct drm_gpusvm_range *range)
1439 {
1440 	kref_put(&range->refcount, drm_gpusvm_range_destroy);
1441 }
1442 EXPORT_SYMBOL_GPL(drm_gpusvm_range_put);
1443 
1444 /**
1445  * drm_gpusvm_pages_valid() - GPU SVM range pages valid
1446  * @gpusvm: Pointer to the GPU SVM structure
1447  * @svm_pages: Pointer to the GPU SVM pages structure
1448  *
1449  * This function determines if a GPU SVM range pages are valid. Expected be
1450  * called holding gpusvm->notifier_lock and as the last step before committing a
1451  * GPU binding. This is akin to a notifier seqno check in the HMM documentation
1452  * but due to wider notifiers (i.e., notifiers which span multiple ranges) this
1453  * function is required for finer grained checking (i.e., per range) if pages
1454  * are valid.
1455  *
1456  * Return: True if GPU SVM range has valid pages, False otherwise
1457  */
1458 bool drm_gpusvm_pages_valid(struct drm_gpusvm *gpusvm,
1459 			    struct drm_gpusvm_pages *svm_pages)
1460 {
1461 	lockdep_assert_held(&gpusvm->notifier_lock);
1462 
1463 	return svm_pages->flags.has_devmem_pages || svm_pages->flags.has_dma_mapping;
1464 }
1465 EXPORT_SYMBOL_GPL(drm_gpusvm_pages_valid);
1466 
1467 /**
1468  * drm_gpusvm_pages_valid_unlocked() - GPU SVM pages valid unlocked
1469  * @gpusvm: Pointer to the GPU SVM structure
1470  * @svm_pages: Array of GPU SVM pages structures
1471  * @num_pages: Number of drm_gpusvm_pages instances in @svm_pages
1472  *
1473  * This function determines if every GPU SVM pages instance is valid, resetting
1474  * every instance which is not so that get_pages() maps it afresh. It therefore
1475  * has to walk them all. Expected be called without holding
1476  * gpusvm->notifier_lock.
1477  *
1478  * Return: True if all GPU SVM pages are valid, False otherwise
1479  */
1480 static bool drm_gpusvm_pages_valid_unlocked(struct drm_gpusvm *gpusvm,
1481 					    struct drm_gpusvm_pages *svm_pages,
1482 					    unsigned int num_pages)
1483 {
1484 	bool pages_valid = true;
1485 	unsigned int p;
1486 
1487 	drm_gpusvm_notifier_lock(gpusvm);
1488 	for (p = 0; p < num_pages; ++p) {
1489 		if (drm_gpusvm_pages_valid(gpusvm, &svm_pages[p]))
1490 			continue;
1491 		__drm_gpusvm_free_pages(gpusvm, &svm_pages[p]);
1492 		pages_valid = false;
1493 	}
1494 	drm_gpusvm_notifier_unlock(gpusvm);
1495 
1496 	return pages_valid;
1497 }
1498 
1499 /**
1500  * drm_gpusvm_pages_inlinable() - Whether the dma address can be inlined
1501  * @svm_pages: The SVM pages instance that was just mapped
1502  * @nentries: Number of entries the mapping loop produced
1503  * @npages: Number of pages in the CPU range
1504  *
1505  * A THP maps as one huge page, and an IOVA reservation links every page of
1506  * the range at the next offset, so the device addresses run contiguously from
1507  * entry 0. Either way one entry describes the whole range, so the dma_addr
1508  * array can be freed and the address kept inline.
1509  *
1510  * state_offset advances only on the IOVA branch, so reaching the full range
1511  * length proves no device page was mapped in between. Only single page
1512  * entries fold, so the order kept is 0 and describes the range truthfully.
1513  * Larger chunks, several huge pages among them, stay an array that is
1514  * already short and that a consumer places with one PTE each.
1515  *
1516  * Return: True if the mapping fits in a single drm_pagemap_addr.
1517  */
1518 static bool drm_gpusvm_pages_inlinable(struct drm_gpusvm_pages *svm_pages,
1519 				       unsigned long nentries,
1520 				       unsigned long npages)
1521 {
1522 	if (nentries == 1)
1523 		return true;
1524 
1525 	return nentries == npages && dma_use_iova(&svm_pages->state) &&
1526 	       svm_pages->state_offset == npages * PAGE_SIZE;
1527 }
1528 
1529 /**
1530  * drm_gpusvm_walk_devmem() - Invoke the devmem callback across faulted pages
1531  * @gpusvm: Pointer to the GPU SVM structure
1532  * @pfns: The already-faulted pfn array (size @npages)
1533  * @npages: Number of pages in the CPU range
1534  * @ctx: GPU SVM context, with a non-NULL &drm_gpusvm_ctx.devmem_fn
1535  *
1536  * Invoke &drm_gpusvm_ctx.devmem_fn once per contiguous run of @pfns backed by
1537  * the same device-memory allocation. Must be called under the notifier lock.
1538  */
1539 static void drm_gpusvm_walk_devmem(struct drm_gpusvm *gpusvm,
1540 				   unsigned long *pfns,
1541 				   unsigned long npages,
1542 				   const struct drm_gpusvm_ctx *ctx)
1543 {
1544 	struct drm_pagemap_devmem *last = NULL;
1545 	unsigned int order = 0;
1546 	unsigned long i;
1547 
1548 	lockdep_assert_held(&gpusvm->notifier_lock);
1549 
1550 	for (i = 0; i < npages; i += 1 << order) {
1551 		struct page *page = hmm_pfn_to_page(pfns[i]);
1552 		struct drm_pagemap_devmem *devmem;
1553 
1554 		order = drm_gpusvm_hmm_pfn_to_order(pfns[i], i, npages);
1555 
1556 		if (!is_device_private_page(page) &&
1557 		    !is_device_coherent_page(page)) {
1558 			last = NULL;
1559 			continue;
1560 		}
1561 
1562 		devmem = drm_pagemap_page_to_devmem(page);
1563 		if (devmem == last)
1564 			continue;
1565 
1566 		last = devmem;
1567 		ctx->devmem_fn(devmem);
1568 	}
1569 }
1570 
1571 /**
1572  * drm_gpusvm_dma_map_pages() - Walk and DMA map one drm_gpusvm_pages instance
1573  * @gpusvm: Pointer to the GPU SVM structure
1574  * @svm_pages: The SVM pages instance to populate with dma-addresses
1575  * @pfns: The already-faulted pfn array (size @npages)
1576  * @npages: Number of pages in the CPU range
1577  * @ctx: GPU SVM context
1578  * @dma_dir: DMA data direction for the mappings
1579  *
1580  * Walk the faulted @pfns and map them into @svm_pages for DMA access through
1581  * its owning drm_device. Must be called under the notifier lock and only for
1582  * an instance without a live mapping. On failure this unwinds the partial
1583  * mapping of this instance before returning.
1584  *
1585  * Return: 0 on success, negative error code on failure.
1586  */
1587 static int drm_gpusvm_dma_map_pages(struct drm_gpusvm *gpusvm,
1588 				    struct drm_gpusvm_pages *svm_pages,
1589 				    unsigned long *pfns,
1590 				    unsigned long npages,
1591 				    const struct drm_gpusvm_ctx *ctx,
1592 				    enum dma_data_direction dma_dir)
1593 {
1594 	void *zdd = NULL;
1595 	unsigned long i, j;
1596 	unsigned long num_dma_mapped = 0;
1597 	unsigned int order = 0;
1598 	int err = 0;
1599 	struct dev_pagemap *pagemap = NULL;
1600 	struct drm_pagemap *dpagemap;
1601 	struct drm_gpusvm_pages_flags flags;
1602 	struct dma_iova_state *state = &svm_pages->state;
1603 
1604 	lockdep_assert_held(&gpusvm->notifier_lock);
1605 
1606 	*state = (struct dma_iova_state){};
1607 	svm_pages->state_offset = 0;
1608 
1609 	flags.__flags = svm_pages->flags.__flags;
1610 
1611 	for (i = 0, j = 0; i < npages; ++j) {
1612 		struct page *page = hmm_pfn_to_page(pfns[i]);
1613 
1614 		order = drm_gpusvm_hmm_pfn_to_order(pfns[i], i, npages);
1615 		if (is_device_private_page(page) ||
1616 		    is_device_coherent_page(page)) {
1617 			struct drm_pagemap_zdd *__zdd =
1618 				drm_pagemap_page_zone_device_data(page);
1619 
1620 			if (!ctx->allow_mixed &&
1621 			    zdd != __zdd && i > 0) {
1622 				err = -EOPNOTSUPP;
1623 				goto err_unmap;
1624 			}
1625 			zdd = __zdd;
1626 			if (pagemap != page_pgmap(page)) {
1627 				if (pagemap) {
1628 					err = -EOPNOTSUPP;
1629 					goto err_unmap;
1630 				}
1631 
1632 				pagemap = page_pgmap(page);
1633 				dpagemap = drm_pagemap_page_to_dpagemap(page);
1634 				if (drm_WARN_ON(svm_pages->drm, !dpagemap)) {
1635 					/*
1636 					 * Raced. This is not supposed to happen
1637 					 * since hmm_range_fault() should've migrated
1638 					 * this page to system.
1639 					 */
1640 					err = -EAGAIN;
1641 					goto err_unmap;
1642 				}
1643 
1644 				/*
1645 				 * Set the dpagemap as soon as the first
1646 				 * device page is mapped so the err_unmap path
1647 				 * can device_unmap() the device mappings that
1648 				 * have already been created.
1649 				 */
1650 				drm_pagemap_get(dpagemap);
1651 				drm_pagemap_put(svm_pages->dpagemap);
1652 				svm_pages->dpagemap = dpagemap;
1653 			}
1654 			svm_pages->dma_addr[j] =
1655 				dpagemap->ops->device_map(dpagemap,
1656 							  svm_pages->drm->dev,
1657 							  page, order,
1658 							  dma_dir);
1659 			if (dma_mapping_error(svm_pages->drm->dev,
1660 					      svm_pages->dma_addr[j].addr)) {
1661 				err = -EFAULT;
1662 				goto err_unmap;
1663 			}
1664 		} else {
1665 			dma_addr_t addr;
1666 
1667 			if (is_zone_device_page(page) ||
1668 			    (pagemap && !ctx->allow_mixed)) {
1669 				err = -EOPNOTSUPP;
1670 				goto err_unmap;
1671 			}
1672 
1673 			if (ctx->devmem_only) {
1674 				err = -EFAULT;
1675 				goto err_unmap;
1676 			}
1677 
1678 			if (!i)
1679 				dma_iova_try_alloc(svm_pages->drm->dev, state,
1680 						   0, npages * PAGE_SIZE);
1681 
1682 			if (dma_use_iova(state)) {
1683 				err = dma_iova_link(svm_pages->drm->dev, state,
1684 						    hmm_pfn_to_phys(pfns[i]),
1685 						    svm_pages->state_offset,
1686 						    PAGE_SIZE << order,
1687 						    dma_dir, 0);
1688 				if (err)
1689 					goto err_unmap;
1690 
1691 				addr = state->addr + svm_pages->state_offset;
1692 				svm_pages->state_offset += PAGE_SIZE << order;
1693 			} else {
1694 				addr = dma_map_page(svm_pages->drm->dev,
1695 						    page, 0,
1696 						    PAGE_SIZE << order,
1697 						    dma_dir);
1698 				if (dma_mapping_error(svm_pages->drm->dev, addr)) {
1699 					err = -EFAULT;
1700 					goto err_unmap;
1701 				}
1702 			}
1703 
1704 			svm_pages->dma_addr[j] = drm_pagemap_addr_encode
1705 				(addr, DRM_INTERCONNECT_SYSTEM, order,
1706 				 dma_dir);
1707 		}
1708 		i += 1 << order;
1709 		num_dma_mapped = i;
1710 		flags.has_dma_mapping = true;
1711 	}
1712 
1713 	if (dma_use_iova(state)) {
1714 		err = dma_iova_sync(svm_pages->drm->dev, state, 0,
1715 				    svm_pages->state_offset);
1716 		if (err)
1717 			goto err_unmap;
1718 	}
1719 
1720 	if (pagemap)
1721 		flags.has_devmem_pages = true;
1722 
1723 	if (drm_gpusvm_pages_inlinable(svm_pages, j, npages)) {
1724 		struct drm_pagemap_addr addr = svm_pages->dma_addr[0];
1725 
1726 		kvfree(svm_pages->dma_addr);
1727 		svm_pages->inline_addr = addr;
1728 		flags.inline_dma_mapping = true;
1729 	}
1730 
1731 	/* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */
1732 	WRITE_ONCE(svm_pages->flags.__flags, flags.__flags);
1733 
1734 	return 0;
1735 
1736 err_unmap:
1737 	svm_pages->flags.has_dma_mapping = true;
1738 	__drm_gpusvm_unmap_pages(gpusvm, svm_pages, num_dma_mapped);
1739 	return err;
1740 }
1741 
1742 /**
1743  * drm_gpusvm_get_pages() - Get pages and populate GPU SVM pages struct
1744  * @gpusvm: Pointer to the GPU SVM structure
1745  * @svm_pages: Array of SVM pages instances to populate with dma addresses
1746  * @num_pages: Number of drm_gpusvm_pages instances in @svm_pages, must not be 0
1747  * @mm: The mm corresponding to the CPU range
1748  * @notifier: The corresponding notifier for the given CPU range
1749  * @pages_start: Start CPU address for the pages
1750  * @pages_end: End CPU address for the pages (exclusive)
1751  * @ctx: GPU SVM context
1752  *
1753  * This function gets and maps pages for a CPU range and ensures they are
1754  * mapped for DMA access. The HMM fault for the CPU range is performed once,
1755  * the DMA mapping by drm_gpusvm_dma_map_pages() is then done per instance,
1756  * one per owning drm_device. The retry against notifier races is kept here
1757  * in common code so drivers never open code it.
1758  *
1759  * On error the instances mapped before the failing one stay mapped, so the
1760  * caller must unmap and free every instance regardless of the return value.
1761  *
1762  * With &drm_gpusvm_ctx.no_dma_map no mapping state is recorded, so
1763  * drm_gpusvm_pages_valid() never returns true and success is only a snapshot:
1764  * the caller must recheck mmu_interval_read_retry() against the recorded
1765  * &drm_gpusvm_pages.notifier_seq under the notifier lock, and hold it until
1766  * its work is visible to invalidation.
1767  *
1768  * Return: 0 on success, negative error code on failure.
1769  */
1770 int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
1771 			 struct drm_gpusvm_pages *svm_pages,
1772 			 unsigned int num_pages,
1773 			 struct mm_struct *mm,
1774 			 struct mmu_interval_notifier *notifier,
1775 			 unsigned long pages_start, unsigned long pages_end,
1776 			 const struct drm_gpusvm_ctx *ctx)
1777 {
1778 	struct hmm_range hmm_range = {
1779 		.default_flags = HMM_PFN_REQ_FAULT | (ctx->read_only ? 0 :
1780 			HMM_PFN_REQ_WRITE),
1781 		.notifier = notifier,
1782 		.start = pages_start,
1783 		.end = pages_end,
1784 		.dev_private_owner = ctx->device_private_page_owner,
1785 	};
1786 	unsigned long timeout =
1787 		jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
1788 	unsigned long remaining;
1789 	unsigned long npages = npages_in_range(pages_start, pages_end);
1790 	unsigned long *pfns;
1791 	int err = 0;
1792 	enum dma_data_direction dma_dir = ctx->read_only ? DMA_TO_DEVICE :
1793 							   DMA_BIDIRECTIONAL;
1794 	const bool map_dma = !ctx->no_dma_map;
1795 	unsigned int p;
1796 
1797 	if (!num_pages)
1798 		return -EINVAL;
1799 
1800 	if (ctx->no_dma_map && ctx->devmem_only)
1801 		return -EINVAL;
1802 
1803 	if (map_dma) {
1804 		for (p = 0; p < num_pages; ++p) {
1805 			if (!svm_pages[p].drm)
1806 				return -EINVAL;
1807 		}
1808 	}
1809 
1810 retry:
1811 	remaining = timeout - jiffies;
1812 
1813 	if (time_after_eq(jiffies, timeout))
1814 		return -EBUSY;
1815 
1816 	hmm_range.notifier_seq = mmu_interval_read_begin(notifier);
1817 
1818 	/*
1819 	 * drm_gpusvm_pages_valid_unlocked() also resets any instance whose
1820 	 * mapping was invalidated, so it must run on the map_dma path even when
1821 	 * devmem_fn suppresses the fast path; otherwise a stale inline mapping
1822 	 * is later mistaken for a dma_addr array by drm_gpusvm_dma_map_pages().
1823 	 */
1824 	if (map_dma) {
1825 		bool pages_valid = drm_gpusvm_pages_valid_unlocked(gpusvm,
1826 								   svm_pages,
1827 								   num_pages);
1828 
1829 		if (!ctx->devmem_fn && pages_valid)
1830 			goto set_seqno;
1831 	}
1832 
1833 	pfns = kvmalloc_objs(*pfns, npages);
1834 	if (!pfns)
1835 		return -ENOMEM;
1836 
1837 	if (!mmget_not_zero(mm)) {
1838 		err = -EFAULT;
1839 		goto err_free;
1840 	}
1841 
1842 	hmm_range.hmm_pfns = pfns;
1843 	err = hmm_range_fault_unlocked_timeout(&hmm_range, remaining);
1844 	mmput(mm);
1845 	if (err)
1846 		goto err_free;
1847 
1848 	if (map_dma) {
1849 		for (p = 0; p < num_pages; ++p) {
1850 			if (drm_gpusvm_pages_first_dma(&svm_pages[p], NULL))
1851 				continue;
1852 			svm_pages[p].dma_addr =
1853 				kvzalloc_objs(*svm_pages[p].dma_addr, npages);
1854 			if (!svm_pages[p].dma_addr) {
1855 				err = -ENOMEM;
1856 				goto err_free;
1857 			}
1858 		}
1859 	}
1860 
1861 	/*
1862 	 * Perform all dma mappings under the notifier lock to not
1863 	 * access freed pages. A notifier will either block on
1864 	 * the notifier lock or unmap dma.
1865 	 */
1866 	drm_gpusvm_notifier_lock(gpusvm);
1867 
1868 	/*
1869 	 * drm_gpusvm_range_set_unmapped() flags the whole array in one go under
1870 	 * the write lock, so any instance answers for all of them here.
1871 	 */
1872 	if (svm_pages[0].flags.unmapped) {
1873 		drm_gpusvm_notifier_unlock(gpusvm);
1874 		err = -EFAULT;
1875 		goto err_free;
1876 	}
1877 
1878 	if (mmu_interval_read_retry(notifier, hmm_range.notifier_seq)) {
1879 		drm_gpusvm_notifier_unlock(gpusvm);
1880 		kvfree(pfns);
1881 		goto retry;
1882 	}
1883 
1884 	if (ctx->devmem_fn)
1885 		drm_gpusvm_walk_devmem(gpusvm, pfns, npages, ctx);
1886 
1887 	if (map_dma) {
1888 		for (p = 0; p < num_pages; ++p) {
1889 			if (drm_gpusvm_pages_valid(gpusvm, &svm_pages[p]))
1890 				continue;
1891 
1892 			err = drm_gpusvm_dma_map_pages(gpusvm, &svm_pages[p], pfns,
1893 						       npages, ctx, dma_dir);
1894 			if (err) {
1895 				/*
1896 				 * The failing instance was unwound by the
1897 				 * helper. Keep the ones mapped earlier: the
1898 				 * -EAGAIN retry reuses them, and the driver
1899 				 * unmaps every instance with the range on the
1900 				 * other error paths.
1901 				 */
1902 				drm_gpusvm_notifier_unlock(gpusvm);
1903 				goto err_free;
1904 			}
1905 		}
1906 	}
1907 
1908 	drm_gpusvm_notifier_unlock(gpusvm);
1909 	kvfree(pfns);
1910 set_seqno:
1911 	for (p = 0; p < num_pages; ++p)
1912 		svm_pages[p].notifier_seq = hmm_range.notifier_seq;
1913 
1914 	return 0;
1915 
1916 err_free:
1917 	kvfree(pfns);
1918 	if (err == -EAGAIN)
1919 		goto retry;
1920 	return err;
1921 }
1922 EXPORT_SYMBOL_GPL(drm_gpusvm_get_pages);
1923 
1924 /**
1925  * drm_gpusvm_unmap_pages() - Unmap GPU svm pages
1926  * @gpusvm: Pointer to the GPU SVM structure
1927  * @svm_pages: Pointer to the GPU SVM pages structure
1928  * @npages: Number of pages in @svm_pages.
1929  * @ctx: GPU SVM context
1930  *
1931  * This function unmaps pages associated with a GPU SVM pages struct. If
1932  * @in_notifier is set, it is assumed that gpusvm->notifier_lock is held in
1933  * write mode; if it is clear, it acquires gpusvm->notifier_lock in read mode.
1934  * Must be called in the invalidate() callback of the corresponding notifier for
1935  * IOMMU security model.
1936  */
1937 void drm_gpusvm_unmap_pages(struct drm_gpusvm *gpusvm,
1938 			    struct drm_gpusvm_pages *svm_pages,
1939 			    unsigned long npages,
1940 			    const struct drm_gpusvm_ctx *ctx)
1941 {
1942 	if (ctx->in_notifier)
1943 		lockdep_assert_held_write(&gpusvm->notifier_lock);
1944 	else
1945 		drm_gpusvm_notifier_lock(gpusvm);
1946 
1947 	__drm_gpusvm_unmap_pages(gpusvm, svm_pages, npages);
1948 
1949 	if (!ctx->in_notifier)
1950 		drm_gpusvm_notifier_unlock(gpusvm);
1951 }
1952 EXPORT_SYMBOL_GPL(drm_gpusvm_unmap_pages);
1953 
1954 /**
1955  * drm_gpusvm_range_evict() - Evict GPU SVM range
1956  * @gpusvm: Pointer to the GPU SVM structure
1957  * @range: Pointer to the GPU SVM range to be removed
1958  *
1959  * This function evicts the specified GPU SVM range.
1960  *
1961  * Return: 0 on success, a negative error code on failure.
1962  */
1963 int drm_gpusvm_range_evict(struct drm_gpusvm *gpusvm,
1964 			   struct drm_gpusvm_range *range)
1965 {
1966 	struct mmu_interval_notifier *notifier = &range->notifier->notifier;
1967 	struct hmm_range hmm_range = {
1968 		.default_flags = HMM_PFN_REQ_FAULT,
1969 		.notifier = notifier,
1970 		.start = drm_gpusvm_range_start(range),
1971 		.end = drm_gpusvm_range_end(range),
1972 		.dev_private_owner = NULL,
1973 	};
1974 	unsigned long timeout = msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
1975 	unsigned long *pfns;
1976 	unsigned long npages = npages_in_range(drm_gpusvm_range_start(range),
1977 					       drm_gpusvm_range_end(range));
1978 	int err = 0;
1979 	struct mm_struct *mm = gpusvm->mm;
1980 
1981 	if (!mmget_not_zero(mm))
1982 		return -EFAULT;
1983 
1984 	pfns = kvmalloc_objs(*pfns, npages);
1985 	if (!pfns) {
1986 		mmput(mm);
1987 		return -ENOMEM;
1988 	}
1989 
1990 	hmm_range.hmm_pfns = pfns;
1991 	err = hmm_range_fault_unlocked_timeout(&hmm_range, timeout);
1992 
1993 	kvfree(pfns);
1994 	mmput(mm);
1995 
1996 	return err == -EBUSY ? -ETIME : err;
1997 }
1998 EXPORT_SYMBOL_GPL(drm_gpusvm_range_evict);
1999 
2000 /**
2001  * drm_gpusvm_has_mapping() - Check if GPU SVM has mapping for the given address range
2002  * @gpusvm: Pointer to the GPU SVM structure.
2003  * @start: Start address
2004  * @end: End address
2005  *
2006  * Return: True if GPU SVM has mapping, False otherwise
2007  */
2008 bool drm_gpusvm_has_mapping(struct drm_gpusvm *gpusvm, unsigned long start,
2009 			    unsigned long end)
2010 {
2011 	struct drm_gpusvm_notifier *notifier;
2012 
2013 	drm_gpusvm_for_each_notifier(notifier, gpusvm, start, end) {
2014 		struct drm_gpusvm_range *range = NULL;
2015 
2016 		drm_gpusvm_for_each_range(range, notifier, start, end)
2017 			return true;
2018 	}
2019 
2020 	return false;
2021 }
2022 EXPORT_SYMBOL_GPL(drm_gpusvm_has_mapping);
2023 
2024 /**
2025  * drm_gpusvm_range_set_unmapped() - Mark a GPU SVM range as unmapped
2026  * @range: Pointer to the GPU SVM range structure.
2027  * @pages: Pointer to the GPU SVM pages structure(s).
2028  * @pages_count: Number of GPU SVM pages structure(s) passed in.
2029  * @mmu_range: Pointer to the MMU notifier range structure.
2030  *
2031  * This function marks a GPU SVM range as unmapped and sets the partial_unmap flag
2032  * if the range partially falls within the provided MMU notifier range.
2033  */
2034 void drm_gpusvm_range_set_unmapped(struct drm_gpusvm_range *range,
2035 				   struct drm_gpusvm_pages *pages,
2036 				   unsigned int pages_count,
2037 				   const struct mmu_notifier_range *mmu_range)
2038 {
2039 	struct drm_gpusvm_range_flags range_flags = {
2040 		.__flags = range->flags.__flags,
2041 	};
2042 	unsigned int i;
2043 
2044 	lockdep_assert_held_write(&range->gpusvm->notifier_lock);
2045 
2046 	range_flags.unmapped = true;
2047 	for (i = 0; i < pages_count; ++i) {
2048 		struct drm_gpusvm_pages_flags flags = {
2049 			.__flags = pages[i].flags.__flags,
2050 		};
2051 
2052 		flags.unmapped = true;
2053 		/* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */
2054 		WRITE_ONCE(pages[i].flags.__flags, flags.__flags);
2055 	}
2056 	if (drm_gpusvm_range_start(range) < mmu_range->start ||
2057 	    drm_gpusvm_range_end(range) > mmu_range->end)
2058 		range_flags.partial_unmap = true;
2059 	/* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */
2060 	WRITE_ONCE(range->flags.__flags, range_flags.__flags);
2061 }
2062 EXPORT_SYMBOL_GPL(drm_gpusvm_range_set_unmapped);
2063 
2064 MODULE_DESCRIPTION("DRM GPUSVM");
2065 MODULE_LICENSE("GPL");
2066