xref: /linux/drivers/gpu/drm/drm_gpusvm.c (revision cb8fef4b07f71da01843457b66e579ad248a625e)
1 // SPDX-License-Identifier: GPL-2.0-only OR MIT
2 /*
3  * Copyright © 2024 Intel Corporation
4  *
5  * Authors:
6  *     Matthew Brost <matthew.brost@intel.com>
7  */
8 
9 #include <linux/dma-mapping.h>
10 #include <linux/export.h>
11 #include <linux/hmm.h>
12 #include <linux/hugetlb_inline.h>
13 #include <linux/memremap.h>
14 #include <linux/mm_types.h>
15 #include <linux/slab.h>
16 
17 #include <drm/drm_device.h>
18 #include <drm/drm_gpusvm.h>
19 #include <drm/drm_pagemap.h>
20 #include <drm/drm_print.h>
21 
22 /**
23  * DOC: Overview
24  *
25  * GPU Shared Virtual Memory (GPU SVM) layer for the Direct Rendering Manager (DRM)
26  * is a component of the DRM framework designed to manage shared virtual memory
27  * between the CPU and GPU. It enables efficient data exchange and processing
28  * for GPU-accelerated applications by allowing memory sharing and
29  * synchronization between the CPU's and GPU's virtual address spaces.
30  *
31  * Key GPU SVM Components:
32  *
33  * - Notifiers:
34  *	Used for tracking memory intervals and notifying the GPU of changes,
35  *	notifiers are sized based on a GPU SVM initialization parameter, with a
36  *	recommendation of 512M or larger. They maintain a Red-BlacK tree and a
37  *	list of ranges that fall within the notifier interval.  Notifiers are
38  *	tracked within a GPU SVM Red-BlacK tree and list and are dynamically
39  *	inserted or removed as ranges within the interval are created or
40  *	destroyed.
41  * - Ranges:
42  *	Represent memory ranges mapped in a DRM device and managed by GPU SVM.
43  *	They are sized based on an array of chunk sizes, which is a GPU SVM
44  *	initialization parameter, and the CPU address space.  Upon GPU fault,
45  *	the largest aligned chunk that fits within the faulting CPU address
46  *	space is chosen for the range size. Ranges are expected to be
47  *	dynamically allocated on GPU fault and removed on an MMU notifier UNMAP
48  *	event. As mentioned above, ranges are tracked in a notifier's Red-Black
49  *	tree.
50  *
51  * - Pages:
52  *	struct drm_gpusvm_pages holds the DMA mapping state for a range of
53  *	CPU virtual addresses: the DMA mapped device addresses,
54  *	the device private pagemap, the IOVA state, the per mapping
55  *	notifier sequence number, and the drm_device that owns the DMA
56  *	mappings.
57  *	A driver embeds one or more struct drm_gpusvm_pages alongside its
58  *	struct drm_gpusvm_range, choosing one of two layouts:
59  *
60  *	1:1 - one drm_gpusvm_pages per range (one drm_device). Simplest
61  *	layout; to mirror a VA range on several devices a driver uses a
62  *	separate range (and notifier) per device, so the HMM fault is taken
63  *	once per device.
64  *
65  *	N:1 - one drm_gpusvm_pages per drm_device, all sharing one range and
66  *	notifier; only the per-device DMA mapping differs. The instances must
67  *	sit in contiguous memory so a single drm_gpusvm_range_set_unmapped()
68  *	can mark them all. A driver can keep one instance inline for the single
69  *	device case and switch to a heap array only when more devices join,
70  *	e.g.:
71  *
72  *	.. code-block:: c
73  *
74  *		struct driver_range {
75  *			struct drm_gpusvm_range base;
76  *			unsigned int num_pages;	// 1: inline_pages, >1: pages[]
77  *			union {
78  *				struct drm_gpusvm_pages inline_pages;
79  *				struct drm_gpusvm_pages *pages;
80  *			};
81  *		};
82  *
83  *		static struct drm_gpusvm_pages *
84  *		driver_pages(struct driver_range *drange)
85  *		{
86  *			return drange->num_pages == 1 ? &drange->inline_pages :
87  *							drange->pages;
88  *		}
89  *
90  *	In the N:1 case the driver allocates the pages array with a zeroing
91  *	allocator (e.g. kcalloc(num_pages, ...)), initialises each entry with
92  *	drm_gpusvm_init_pages(), and frees each entry with
93  *	drm_gpusvm_free_pages() plus the array itself, from its range free
94  *	callback. Each drm_gpusvm_pages is mapped independently by their own
95  *	drm_device.
96  *	Each drm_gpusvm_pages must be zero-initialised and initialised with
97  *	drm_gpusvm_init_pages(), called once per entry.
98  *
99  *	The 1:1 examples below pass @num_pages == 1 and &drange->pages. In the
100  *	N:1 case the driver instead passes the whole array and its count, so a
101  *	single call faults the CPU range once and DMA maps it for every owning
102  *	drm_device, e.g.:
103  *
104  *	.. code-block:: c
105  *
106  *		// GPU fault handler: one fault, one DMA mapping per device
107  *		err = drm_gpusvm_get_pages(gpusvm, driver_pages(drange),
108  *					   drange->num_pages, gpusvm->mm,
109  *					   &range->notifier->notifier,
110  *					   drm_gpusvm_range_start(range),
111  *					   drm_gpusvm_range_end(range), &ctx);
112  *
113  *		// Notifier callback: mark every instance unmapped in one call
114  *		drm_gpusvm_range_set_unmapped(range, driver_pages(drange),
115  *					      drange->num_pages, mmu_range);
116  *
117  *	The unmap and free paths stay per-instance: iterate @num_pages over
118  *	driver_pages(drange) and call drm_gpusvm_unmap_pages() /
119  *	drm_gpusvm_free_pages() for each entry.
120  *
121  * - Operations:
122  *	Define the interface for driver-specific GPU SVM operations such as
123  *	range allocation, notifier allocation, and invalidations.
124  *
125  * - Device Memory Allocations:
126  *	Embedded structure containing enough information for GPU SVM to migrate
127  *	to / from device memory.
128  *
129  * - Device Memory Operations:
130  *	Define the interface for driver-specific device memory operations
131  *	release memory, populate pfns, and copy to / from device memory.
132  *
133  * This layer provides interfaces for allocating, mapping, migrating, and
134  * releasing memory ranges between the CPU and GPU. It handles all core memory
135  * management interactions (DMA mapping, HMM, and migration) and provides
136  * driver-specific virtual functions (vfuncs). This infrastructure is sufficient
137  * to build the expected driver components for an SVM implementation as detailed
138  * below.
139  *
140  * Expected Driver Components:
141  *
142  * - GPU page fault handler:
143  *	Used to create ranges and notifiers based on the fault address,
144  *	optionally migrate the range to device memory, and create GPU bindings.
145  *
146  * - Garbage collector:
147  *	Used to unmap and destroy GPU bindings for ranges.  Ranges are expected
148  *	to be added to the garbage collector upon a MMU_NOTIFY_UNMAP event in
149  *	notifier callback.
150  *
151  * - Notifier callback:
152  *	Used to invalidate and DMA unmap GPU bindings for ranges.
153  */
154 
155 /**
156  * DOC: Locking
157  *
158  * GPU SVM handles locking for core MM interactions, i.e., it locks/unlocks the
159  * mmap lock as needed.
160  *
161  * GPU SVM introduces a global notifier lock, which safeguards the notifier's
162  * range RB tree and list, as well as the range's DMA mappings and sequence
163  * number. GPU SVM manages all necessary locking and unlocking operations,
164  * except for the recheck range's pages being valid
165  * (drm_gpusvm_pages_valid) when the driver is committing GPU bindings.
166  * This lock corresponds to the ``driver->update`` lock mentioned in
167  * Documentation/mm/hmm.rst. Future revisions may transition from a GPU SVM
168  * global lock to a per-notifier lock if finer-grained locking is deemed
169  * necessary.
170  *
171  * In addition to the locking mentioned above, the driver should implement a
172  * lock to safeguard core GPU SVM function calls that modify state, such as
173  * drm_gpusvm_range_find_or_insert and drm_gpusvm_range_remove. This lock is
174  * denoted as 'driver_svm_lock' in code examples. Finer grained driver side
175  * locking should also be possible for concurrent GPU fault processing within a
176  * single GPU SVM. The 'driver_svm_lock' can be via drm_gpusvm_driver_set_lock
177  * to add annotations to GPU SVM.
178  */
179 
180 /**
181  * DOC: Partial Unmapping of Ranges
182  *
183  * Partial unmapping of ranges (e.g., 1M out of 2M is unmapped by CPU resulting
184  * in MMU_NOTIFY_UNMAP event) presents several challenges, with the main one
185  * being that a subset of the range still has CPU and GPU mappings. If the
186  * backing store for the range is in device memory, a subset of the backing
187  * store has references. One option would be to split the range and device
188  * memory backing store, but the implementation for this would be quite
189  * complicated. Given that partial unmappings are rare and driver-defined range
190  * sizes are relatively small, GPU SVM does not support splitting of ranges.
191  *
192  * With no support for range splitting, upon partial unmapping of a range, the
193  * driver is expected to invalidate and destroy the entire range. If the range
194  * has device memory as its backing, the driver is also expected to migrate any
195  * remaining pages back to RAM.
196  */
197 
198 /**
199  * DOC: Examples
200  *
201  * This section provides three examples of how to build the expected driver
202  * components: the GPU page fault handler, the garbage collector, and the
203  * notifier callback.
204  *
205  * The generic code provided does not include logic for complex migration
206  * policies, optimized invalidations, fined grained driver locking, or other
207  * potentially required driver locking (e.g., DMA-resv locks).
208  *
209  * 1) GPU page fault handler
210  *
211  * .. code-block:: c
212  *
213  *	struct driver_range {
214  *		struct drm_gpusvm_range base;
215  *		struct drm_gpusvm_pages pages;
216  *	};
217  *
218  *	int driver_bind_range(struct drm_gpusvm *gpusvm, struct driver_range *drange)
219  *	{
220  *		int err = 0;
221  *
222  *		driver_alloc_and_setup_memory_for_bind(gpusvm, drange);
223  *
224  *		drm_gpusvm_notifier_lock(gpusvm);
225  *		if (drm_gpusvm_pages_valid(gpusvm, &drange->pages))
226  *			driver_commit_bind(gpusvm, drange);
227  *		else
228  *			err = -EAGAIN;
229  *		drm_gpusvm_notifier_unlock(gpusvm);
230  *
231  *		return err;
232  *	}
233  *
234  *	int driver_gpu_fault(struct drm_gpusvm *gpusvm, unsigned long fault_addr,
235  *			     unsigned long gpuva_start, unsigned long gpuva_end)
236  *	{
237  *		struct drm_gpusvm_ctx ctx = {};
238  *		struct driver_range *drange;
239  *		struct drm_gpusvm_range *range;
240  *		int err;
241  *
242  *		driver_svm_lock();
243  *	retry:
244  *		// Always process UNMAPs first so view of GPU SVM ranges is current
245  *		driver_garbage_collector(gpusvm);
246  *
247  *		range = drm_gpusvm_range_find_or_insert(gpusvm, fault_addr,
248  *							gpuva_start, gpuva_end,
249  *						        &ctx);
250  *		if (IS_ERR(range)) {
251  *			err = PTR_ERR(range);
252  *			goto unlock;
253  *		}
254  *		drange = container_of(range, struct driver_range, base);
255  *
256  *		if (driver_migration_policy(range)) {
257  *			err = drm_pagemap_populate_mm(driver_choose_drm_pagemap(),
258  *						      gpuva_start, gpuva_end, gpusvm->mm,
259  *						      ctx->timeslice_ms);
260  *			if (err)	// CPU mappings may have changed
261  *				goto retry;
262  *		}
263  *
264  *		err = drm_gpusvm_get_pages(gpusvm, &drange->pages, 1,
265  *					   gpusvm->mm, &range->notifier->notifier,
266  *					   drm_gpusvm_range_start(range),
267  *					   drm_gpusvm_range_end(range), &ctx);
268  *		if (err == -EOPNOTSUPP || err == -EFAULT || err == -EPERM) {	// CPU mappings changed
269  *			if (err == -EOPNOTSUPP)
270  *				drm_gpusvm_range_evict(gpusvm, range);
271  *			goto retry;
272  *		} else if (err) {
273  *			goto unlock;
274  *		}
275  *
276  *		err = driver_bind_range(gpusvm, drange);
277  *		if (err == -EAGAIN)	// CPU mappings changed
278  *			goto retry
279  *
280  *	unlock:
281  *		driver_svm_unlock();
282  *		return err;
283  *	}
284  *
285  * 2) Garbage Collector
286  *
287  * .. code-block:: c
288  *
289  *	// The driver owns the drm_gpusvm_pages lifecycle. ops->range_free is
290  *	// the final fallback: drm_gpusvm_free_pages() unmaps any
291  *	// lingering DMA mapping and a no-op if already unmapped and frees the
292  *	// dma_addr array. The normal flow is to DMA unmap before
293  *	// drm_gpusvm_range_remove() (before the range leaves the tree).
294  *	void driver_range_free(struct drm_gpusvm_range *range)
295  *	{
296  *		struct driver_range *drange =
297  *			container_of(range, struct driver_range, base);
298  *
299  *		drm_gpusvm_free_pages(range->gpusvm, &drange->pages,
300  *				      drm_gpusvm_range_size(range) >> PAGE_SHIFT);
301  *		kfree(drange);
302  *	}
303  *
304  *	void __driver_garbage_collector(struct drm_gpusvm *gpusvm,
305  *					struct drm_gpusvm_range *range)
306  *	{
307  *		assert_driver_svm_locked(gpusvm);
308  *
309  *		// Partial unmap, migrate any remaining device memory pages back to RAM
310  *		if (range->flags.partial_unmap)
311  *			drm_gpusvm_range_evict(gpusvm, range);
312  *
313  *		driver_unbind_range(range);
314  *		// The pages must be DMA unmapped before drm_gpusvm_range_remove()
315  *		// , so a range is never off the MMU interval tree while still DMA
316  *		// mapped as the original drmsvm design flow. Otherwise a concurrent CPU
317  *		// munmap's notifier could miss this range and free pages still mapped
318  *		// for device DMA. This is the normal unmap point.
319  *		drm_gpusvm_unmap_pages(gpusvm, &drange->pages,
320  *				       drm_gpusvm_range_size(range) >> PAGE_SHIFT,
321  *				       &(struct drm_gpusvm_ctx){ .in_notifier = false });
322  *		drm_gpusvm_range_remove(gpusvm, range);
323  *	}
324  *
325  *	void driver_garbage_collector(struct drm_gpusvm *gpusvm)
326  *	{
327  *		assert_driver_svm_locked(gpusvm);
328  *
329  *		for_each_range_in_garbage_collector(gpusvm, range)
330  *			__driver_garbage_collector(gpusvm, range);
331  *	}
332  *
333  * 3) Notifier callback
334  *
335  * .. code-block:: c
336  *
337  *	void driver_invalidation(struct drm_gpusvm *gpusvm,
338  *				 struct drm_gpusvm_notifier *notifier,
339  *				 const struct mmu_notifier_range *mmu_range)
340  *	{
341  *		struct drm_gpusvm_ctx ctx = { .in_notifier = true, };
342  *		struct drm_gpusvm_range *range = NULL;
343  *		struct driver_range *drange;
344  *
345  *		driver_invalidate_device_pages(gpusvm, mmu_range->start, mmu_range->end);
346  *
347  *		drm_gpusvm_for_each_range(range, notifier, mmu_range->start,
348  *					  mmu_range->end) {
349  *			drange = container_of(range, struct driver_range, base);
350  *
351  *			drm_gpusvm_unmap_pages(gpusvm, &drange->pages,
352  *					       drm_gpusvm_range_size(range) >> PAGE_SHIFT,
353  *					       &ctx);
354  *
355  *			if (mmu_range->event != MMU_NOTIFY_UNMAP)
356  *				continue;
357  *
358  *			drm_gpusvm_range_set_unmapped(range, &drange->pages, 1, mmu_range);
359  *			driver_garbage_collector_add(gpusvm, range);
360  *		}
361  *	}
362  */
363 
364 /**
365  * npages_in_range() - Calculate the number of pages in a given range
366  * @start: The start address of the range
367  * @end: The end address of the range
368  *
369  * This macro calculates the number of pages in a given memory range,
370  * specified by the start and end addresses. It divides the difference
371  * between the end and start addresses by the page size (PAGE_SIZE) to
372  * determine the number of pages in the range.
373  *
374  * Return: The number of pages in the specified range.
375  */
376 static unsigned long
377 npages_in_range(unsigned long start, unsigned long end)
378 {
379 	return (end - start) >> PAGE_SHIFT;
380 }
381 
382 /**
383  * drm_gpusvm_notifier_find() - Find GPU SVM notifier from GPU SVM
384  * @gpusvm: Pointer to the GPU SVM structure.
385  * @start: Start address of the notifier
386  * @end: End address of the notifier
387  *
388  * Return: A pointer to the drm_gpusvm_notifier if found or NULL
389  */
390 struct drm_gpusvm_notifier *
391 drm_gpusvm_notifier_find(struct drm_gpusvm *gpusvm, unsigned long start,
392 			 unsigned long end)
393 {
394 	struct interval_tree_node *itree;
395 
396 	itree = interval_tree_iter_first(&gpusvm->root, start, end - 1);
397 
398 	if (itree)
399 		return container_of(itree, struct drm_gpusvm_notifier, itree);
400 	else
401 		return NULL;
402 }
403 EXPORT_SYMBOL_GPL(drm_gpusvm_notifier_find);
404 
405 /**
406  * drm_gpusvm_range_find() - Find GPU SVM range from GPU SVM notifier
407  * @notifier: Pointer to the GPU SVM notifier structure.
408  * @start: Start address of the range
409  * @end: End address of the range
410  *
411  * Return: A pointer to the drm_gpusvm_range if found or NULL
412  */
413 struct drm_gpusvm_range *
414 drm_gpusvm_range_find(struct drm_gpusvm_notifier *notifier, unsigned long start,
415 		      unsigned long end)
416 {
417 	struct interval_tree_node *itree;
418 
419 	itree = interval_tree_iter_first(&notifier->root, start, end - 1);
420 
421 	if (itree)
422 		return container_of(itree, struct drm_gpusvm_range, itree);
423 	else
424 		return NULL;
425 }
426 EXPORT_SYMBOL_GPL(drm_gpusvm_range_find);
427 
428 /**
429  * drm_gpusvm_notifier_invalidate() - Invalidate a GPU SVM notifier.
430  * @mni: Pointer to the mmu_interval_notifier structure.
431  * @mmu_range: Pointer to the mmu_notifier_range structure.
432  * @cur_seq: Current sequence number.
433  *
434  * This function serves as a generic MMU notifier for GPU SVM. It sets the MMU
435  * notifier sequence number and calls the driver invalidate vfunc under
436  * gpusvm->notifier_lock.
437  *
438  * Return: true if the operation succeeds, false otherwise.
439  */
440 static bool
441 drm_gpusvm_notifier_invalidate(struct mmu_interval_notifier *mni,
442 			       const struct mmu_notifier_range *mmu_range,
443 			       unsigned long cur_seq)
444 {
445 	struct drm_gpusvm_notifier *notifier =
446 		container_of(mni, typeof(*notifier), notifier);
447 	struct drm_gpusvm *gpusvm = notifier->gpusvm;
448 
449 	if (!mmu_notifier_range_blockable(mmu_range))
450 		return false;
451 
452 	down_write(&gpusvm->notifier_lock);
453 	mmu_interval_set_seq(mni, cur_seq);
454 	gpusvm->ops->invalidate(gpusvm, notifier, mmu_range);
455 	up_write(&gpusvm->notifier_lock);
456 
457 	return true;
458 }
459 
460 /*
461  * drm_gpusvm_notifier_ops - MMU interval notifier operations for GPU SVM
462  */
463 static const struct mmu_interval_notifier_ops drm_gpusvm_notifier_ops = {
464 	.invalidate = drm_gpusvm_notifier_invalidate,
465 };
466 
467 /**
468  * drm_gpusvm_init() - Initialize the GPU SVM.
469  * @gpusvm: Pointer to the GPU SVM structure.
470  * @name: Name of the GPU SVM.
471  * @mm: Pointer to the mm_struct for the address space.
472  * @mm_start: Start address of GPU SVM.
473  * @mm_range: Range of the GPU SVM.
474  * @notifier_size: Size of individual notifiers.
475  * @ops: Pointer to the operations structure for GPU SVM.
476  * @chunk_sizes: Pointer to the array of chunk sizes used in range allocation.
477  *               Entries should be powers of 2 in descending order with last
478  *               entry being SZ_4K.
479  * @num_chunks: Number of chunks.
480  *
481  * This function initializes the GPU SVM.
482  *
483  * Note: If only using the simple drm_gpusvm_pages API (get/unmap/free),
484  * then only @gpusvm and @name are expected. The @drm drm_device for dma
485  * mappings is bound per-pages via drm_gpusvm_init_pages() before the first
486  * drm_gpusvm_get_pages() call. However, the same base
487  * @gpusvm can also be used with both modes together in which case the full
488  * setup is needed, where the core drm_gpusvm_pages API will simply never use
489  * the other fields.
490  *
491  * Return: 0 on success, a negative error code on failure.
492  */
493 int drm_gpusvm_init(struct drm_gpusvm *gpusvm,
494 		    const char *name,
495 		    struct mm_struct *mm,
496 		    unsigned long mm_start, unsigned long mm_range,
497 		    unsigned long notifier_size,
498 		    const struct drm_gpusvm_ops *ops,
499 		    const unsigned long *chunk_sizes, int num_chunks)
500 {
501 	if (mm) {
502 		if (!ops->invalidate || !num_chunks)
503 			return -EINVAL;
504 		mmgrab(mm);
505 	} else {
506 		/* No full SVM mode, only core drm_gpusvm_pages API. */
507 		if (ops || num_chunks || mm_range || notifier_size)
508 			return -EINVAL;
509 	}
510 
511 	gpusvm->name = name;
512 	gpusvm->mm = mm;
513 	gpusvm->mm_start = mm_start;
514 	gpusvm->mm_range = mm_range;
515 	gpusvm->notifier_size = notifier_size;
516 	gpusvm->ops = ops;
517 	gpusvm->chunk_sizes = chunk_sizes;
518 	gpusvm->num_chunks = num_chunks;
519 
520 	gpusvm->root = RB_ROOT_CACHED;
521 	INIT_LIST_HEAD(&gpusvm->notifier_list);
522 
523 	init_rwsem(&gpusvm->notifier_lock);
524 
525 	fs_reclaim_acquire(GFP_KERNEL);
526 	might_lock(&gpusvm->notifier_lock);
527 	fs_reclaim_release(GFP_KERNEL);
528 
529 #ifdef CONFIG_LOCKDEP
530 	gpusvm->lock_dep_map = NULL;
531 #endif
532 
533 	return 0;
534 }
535 EXPORT_SYMBOL_GPL(drm_gpusvm_init);
536 
537 /**
538  * to_drm_gpusvm_notifier() - retrieve the container struct for a given rbtree node
539  * @node: a pointer to the rbtree node embedded within a drm_gpusvm_notifier struct
540  *
541  * Return: A pointer to the containing drm_gpusvm_notifier structure.
542  */
543 static struct drm_gpusvm_notifier *to_drm_gpusvm_notifier(struct rb_node *node)
544 {
545 	return container_of(node, struct drm_gpusvm_notifier, itree.rb);
546 }
547 
548 /**
549  * drm_gpusvm_notifier_insert() - Insert GPU SVM notifier
550  * @gpusvm: Pointer to the GPU SVM structure
551  * @notifier: Pointer to the GPU SVM notifier structure
552  *
553  * This function inserts the GPU SVM notifier into the GPU SVM RB tree and list.
554  */
555 static void drm_gpusvm_notifier_insert(struct drm_gpusvm *gpusvm,
556 				       struct drm_gpusvm_notifier *notifier)
557 {
558 	struct rb_node *node;
559 	struct list_head *head;
560 
561 	interval_tree_insert(&notifier->itree, &gpusvm->root);
562 
563 	node = rb_prev(&notifier->itree.rb);
564 	if (node)
565 		head = &(to_drm_gpusvm_notifier(node))->entry;
566 	else
567 		head = &gpusvm->notifier_list;
568 
569 	list_add(&notifier->entry, head);
570 }
571 
572 /**
573  * drm_gpusvm_notifier_remove() - Remove GPU SVM notifier
574  * @gpusvm: Pointer to the GPU SVM tructure
575  * @notifier: Pointer to the GPU SVM notifier structure
576  *
577  * This function removes the GPU SVM notifier from the GPU SVM RB tree and list.
578  */
579 static void drm_gpusvm_notifier_remove(struct drm_gpusvm *gpusvm,
580 				       struct drm_gpusvm_notifier *notifier)
581 {
582 	interval_tree_remove(&notifier->itree, &gpusvm->root);
583 	list_del(&notifier->entry);
584 }
585 
586 /**
587  * drm_gpusvm_fini() - Finalize the GPU SVM.
588  * @gpusvm: Pointer to the GPU SVM structure.
589  *
590  * This function finalizes the GPU SVM by cleaning up any remaining ranges and
591  * notifiers, and dropping a reference to struct MM.
592  */
593 void drm_gpusvm_fini(struct drm_gpusvm *gpusvm)
594 {
595 	struct drm_gpusvm_notifier *notifier, *next;
596 
597 	drm_gpusvm_for_each_notifier_safe(notifier, next, gpusvm, 0, LONG_MAX) {
598 		struct drm_gpusvm_range *range, *__next;
599 
600 		/*
601 		 * Remove notifier first to avoid racing with any invalidation
602 		 */
603 		mmu_interval_notifier_remove(&notifier->notifier);
604 		notifier->flags.removed = true;
605 
606 		drm_gpusvm_for_each_range_safe(range, __next, notifier, 0,
607 					       LONG_MAX)
608 			drm_gpusvm_range_remove(gpusvm, range);
609 	}
610 
611 	if (gpusvm->mm)
612 		mmdrop(gpusvm->mm);
613 	WARN_ON(!RB_EMPTY_ROOT(&gpusvm->root.rb_root));
614 }
615 EXPORT_SYMBOL_GPL(drm_gpusvm_fini);
616 
617 /**
618  * drm_gpusvm_notifier_alloc() - Allocate GPU SVM notifier
619  * @gpusvm: Pointer to the GPU SVM structure
620  * @fault_addr: Fault address
621  *
622  * This function allocates and initializes the GPU SVM notifier structure.
623  *
624  * Return: Pointer to the allocated GPU SVM notifier on success, ERR_PTR() on failure.
625  */
626 static struct drm_gpusvm_notifier *
627 drm_gpusvm_notifier_alloc(struct drm_gpusvm *gpusvm, unsigned long fault_addr)
628 {
629 	struct drm_gpusvm_notifier *notifier;
630 
631 	if (gpusvm->ops->notifier_alloc)
632 		notifier = gpusvm->ops->notifier_alloc();
633 	else
634 		notifier = kzalloc_obj(*notifier);
635 
636 	if (!notifier)
637 		return ERR_PTR(-ENOMEM);
638 
639 	notifier->gpusvm = gpusvm;
640 	notifier->itree.start = ALIGN_DOWN(fault_addr, gpusvm->notifier_size);
641 	notifier->itree.last = ALIGN(fault_addr + 1, gpusvm->notifier_size) - 1;
642 	INIT_LIST_HEAD(&notifier->entry);
643 	notifier->root = RB_ROOT_CACHED;
644 	INIT_LIST_HEAD(&notifier->range_list);
645 
646 	return notifier;
647 }
648 
649 /**
650  * drm_gpusvm_notifier_free() - Free GPU SVM notifier
651  * @gpusvm: Pointer to the GPU SVM structure
652  * @notifier: Pointer to the GPU SVM notifier structure
653  *
654  * This function frees the GPU SVM notifier structure.
655  */
656 static void drm_gpusvm_notifier_free(struct drm_gpusvm *gpusvm,
657 				     struct drm_gpusvm_notifier *notifier)
658 {
659 	WARN_ON(!RB_EMPTY_ROOT(&notifier->root.rb_root));
660 
661 	if (gpusvm->ops->notifier_free)
662 		gpusvm->ops->notifier_free(notifier);
663 	else
664 		kfree(notifier);
665 }
666 
667 /**
668  * to_drm_gpusvm_range() - retrieve the container struct for a given rbtree node
669  * @node: a pointer to the rbtree node embedded within a drm_gpusvm_range struct
670  *
671  * Return: A pointer to the containing drm_gpusvm_range structure.
672  */
673 static struct drm_gpusvm_range *to_drm_gpusvm_range(struct rb_node *node)
674 {
675 	return container_of(node, struct drm_gpusvm_range, itree.rb);
676 }
677 
678 /**
679  * drm_gpusvm_range_insert() - Insert GPU SVM range
680  * @notifier: Pointer to the GPU SVM notifier structure
681  * @range: Pointer to the GPU SVM range structure
682  *
683  * This function inserts the GPU SVM range into the notifier RB tree and list.
684  */
685 static void drm_gpusvm_range_insert(struct drm_gpusvm_notifier *notifier,
686 				    struct drm_gpusvm_range *range)
687 {
688 	struct rb_node *node;
689 	struct list_head *head;
690 
691 	drm_gpusvm_notifier_lock(notifier->gpusvm);
692 	interval_tree_insert(&range->itree, &notifier->root);
693 
694 	node = rb_prev(&range->itree.rb);
695 	if (node)
696 		head = &(to_drm_gpusvm_range(node))->entry;
697 	else
698 		head = &notifier->range_list;
699 
700 	list_add(&range->entry, head);
701 	drm_gpusvm_notifier_unlock(notifier->gpusvm);
702 }
703 
704 /**
705  * __drm_gpusvm_range_remove() - Remove GPU SVM range
706  * @notifier: Pointer to the GPU SVM notifier structure
707  * @range: Pointer to the GPU SVM range structure
708  *
709  * This macro removes the GPU SVM range from the notifier RB tree and list.
710  */
711 static void __drm_gpusvm_range_remove(struct drm_gpusvm_notifier *notifier,
712 				      struct drm_gpusvm_range *range)
713 {
714 	interval_tree_remove(&range->itree, &notifier->root);
715 	list_del(&range->entry);
716 }
717 
718 /**
719  * drm_gpusvm_range_alloc() - Allocate GPU SVM range
720  * @gpusvm: Pointer to the GPU SVM structure
721  * @notifier: Pointer to the GPU SVM notifier structure
722  * @fault_addr: Fault address
723  * @chunk_size: Chunk size
724  * @migrate_devmem: Flag indicating whether to migrate device memory
725  *
726  * This function allocates and initializes the GPU SVM range structure.
727  *
728  * Return: Pointer to the allocated GPU SVM range on success, ERR_PTR() on failure.
729  */
730 static struct drm_gpusvm_range *
731 drm_gpusvm_range_alloc(struct drm_gpusvm *gpusvm,
732 		       struct drm_gpusvm_notifier *notifier,
733 		       unsigned long fault_addr, unsigned long chunk_size,
734 		       bool migrate_devmem)
735 {
736 	struct drm_gpusvm_range *range;
737 
738 	if (gpusvm->ops->range_alloc)
739 		range = gpusvm->ops->range_alloc(gpusvm);
740 	else
741 		range = kzalloc_obj(*range);
742 
743 	if (!range)
744 		return ERR_PTR(-ENOMEM);
745 
746 	kref_init(&range->refcount);
747 	range->gpusvm = gpusvm;
748 	range->notifier = notifier;
749 	range->itree.start = ALIGN_DOWN(fault_addr, chunk_size);
750 	range->itree.last = ALIGN(fault_addr + 1, chunk_size) - 1;
751 	INIT_LIST_HEAD(&range->entry);
752 	range->flags.migrate_devmem = migrate_devmem ? 1 : 0;
753 
754 	return range;
755 }
756 
757 /**
758  * drm_gpusvm_hmm_pfn_to_order() - Get the largest CPU mapping order.
759  * @hmm_pfn: The current hmm_pfn.
760  * @hmm_pfn_index: Index of the @hmm_pfn within the pfn array.
761  * @npages: Number of pages within the pfn array i.e the hmm range size.
762  *
763  * To allow skipping PFNs with the same flags (like when they belong to
764  * the same huge PTE) when looping over the pfn array, take a given a hmm_pfn,
765  * and return the largest order that will fit inside the CPU PTE, but also
766  * crucially accounting for the original hmm range boundaries.
767  *
768  * Return: The largest order that will safely fit within the size of the hmm_pfn
769  * CPU PTE.
770  */
771 static unsigned int drm_gpusvm_hmm_pfn_to_order(unsigned long hmm_pfn,
772 						unsigned long hmm_pfn_index,
773 						unsigned long npages)
774 {
775 	unsigned long size;
776 
777 	size = 1UL << hmm_pfn_to_map_order(hmm_pfn);
778 	size -= (hmm_pfn & ~HMM_PFN_FLAGS) & (size - 1);
779 	hmm_pfn_index += size;
780 	if (hmm_pfn_index > npages)
781 		size -= (hmm_pfn_index - npages);
782 
783 	return ilog2(size);
784 }
785 
786 /**
787  * drm_gpusvm_check_pages() - Check pages
788  * @gpusvm: Pointer to the GPU SVM structure
789  * @notifier: Pointer to the GPU SVM notifier structure
790  * @start: Start address
791  * @end: End address
792  * @dev_private_owner: The device private page owner
793  *
794  * Check if pages between start and end have been faulted in on the CPU. Use to
795  * prevent migration of pages without CPU backing store.
796  *
797  * Return: True if pages have been faulted into CPU, False otherwise
798  */
799 static bool drm_gpusvm_check_pages(struct drm_gpusvm *gpusvm,
800 				   struct drm_gpusvm_notifier *notifier,
801 				   unsigned long start, unsigned long end,
802 				   void *dev_private_owner)
803 {
804 	struct hmm_range hmm_range = {
805 		.default_flags = 0,
806 		.notifier = &notifier->notifier,
807 		.start = start,
808 		.end = end,
809 		.dev_private_owner = dev_private_owner,
810 	};
811 	unsigned long timeout =
812 		jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
813 	unsigned long *pfns;
814 	unsigned long npages = npages_in_range(start, end);
815 	int err, i;
816 
817 	mmap_assert_locked(gpusvm->mm);
818 
819 	pfns = kvmalloc_array(npages, sizeof(*pfns), GFP_KERNEL);
820 	if (!pfns)
821 		return false;
822 
823 	hmm_range.notifier_seq = mmu_interval_read_begin(&notifier->notifier);
824 	hmm_range.hmm_pfns = pfns;
825 
826 	while (true) {
827 		err = hmm_range_fault(&hmm_range);
828 		if (err == -EBUSY) {
829 			if (time_after(jiffies, timeout))
830 				break;
831 
832 			hmm_range.notifier_seq =
833 				mmu_interval_read_begin(&notifier->notifier);
834 			continue;
835 		}
836 		break;
837 	}
838 	if (err)
839 		goto err_free;
840 
841 	for (i = 0; i < npages;) {
842 		if (!(pfns[i] & HMM_PFN_VALID)) {
843 			err = -EFAULT;
844 			goto err_free;
845 		}
846 		i += 0x1 << drm_gpusvm_hmm_pfn_to_order(pfns[i], i, npages);
847 	}
848 
849 err_free:
850 	kvfree(pfns);
851 	return err ? false : true;
852 }
853 
854 /**
855  * drm_gpusvm_scan_mm() - Check the migration state of a drm_gpusvm_range
856  * @range: Pointer to the struct drm_gpusvm_range to check.
857  * @dev_private_owner: The struct dev_private_owner to use to determine
858  * compatible device-private pages.
859  * @pagemap: The struct dev_pagemap pointer to use for pagemap-specific
860  * checks.
861  *
862  * Scan the CPU address space corresponding to @range and return the
863  * current migration state. Note that the result may be invalid as
864  * soon as the function returns. It's an advisory check.
865  *
866  * TODO: Bail early and call hmm_range_fault() for subranges.
867  *
868  * Return: See &enum drm_gpusvm_scan_result.
869  */
870 enum drm_gpusvm_scan_result drm_gpusvm_scan_mm(struct drm_gpusvm_range *range,
871 					       void *dev_private_owner,
872 					       const struct dev_pagemap *pagemap)
873 {
874 	struct mmu_interval_notifier *notifier = &range->notifier->notifier;
875 	unsigned long start = drm_gpusvm_range_start(range);
876 	unsigned long end = drm_gpusvm_range_end(range);
877 	struct hmm_range hmm_range = {
878 		.default_flags = 0,
879 		.notifier = notifier,
880 		.start = start,
881 		.end = end,
882 		.dev_private_owner = dev_private_owner,
883 	};
884 	unsigned long timeout = msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
885 	enum drm_gpusvm_scan_result state = DRM_GPUSVM_SCAN_UNPOPULATED, new_state;
886 	unsigned long *pfns;
887 	unsigned long npages = npages_in_range(start, end);
888 	const struct dev_pagemap *other = NULL;
889 	int err, i;
890 
891 	pfns = kvcalloc(npages, sizeof(*pfns), GFP_KERNEL);
892 	if (!pfns)
893 		return DRM_GPUSVM_SCAN_UNPOPULATED;
894 
895 	hmm_range.hmm_pfns = pfns;
896 
897 retry:
898 	err = hmm_range_fault_unlocked_timeout(&hmm_range, timeout);
899 	if (err)
900 		goto err_free;
901 
902 	drm_gpusvm_notifier_lock(range->gpusvm);
903 	if (mmu_interval_read_retry(notifier, hmm_range.notifier_seq)) {
904 		drm_gpusvm_notifier_unlock(range->gpusvm);
905 		goto retry;
906 	}
907 
908 	for (i = 0; i < npages;) {
909 		struct page *page;
910 		const struct dev_pagemap *cur = NULL;
911 
912 		if (!(pfns[i] & HMM_PFN_VALID)) {
913 			state = DRM_GPUSVM_SCAN_UNPOPULATED;
914 			break;
915 		}
916 
917 		page = hmm_pfn_to_page(pfns[i]);
918 		if (is_device_private_page(page) ||
919 		    is_device_coherent_page(page))
920 			cur = page_pgmap(page);
921 
922 		if (cur == pagemap) {
923 			new_state = DRM_GPUSVM_SCAN_EQUAL;
924 		} else if (cur && (cur == other || !other)) {
925 			new_state = DRM_GPUSVM_SCAN_OTHER;
926 			other = cur;
927 		} else if (cur) {
928 			new_state = DRM_GPUSVM_SCAN_MIXED_DEVICE;
929 		} else {
930 			new_state = DRM_GPUSVM_SCAN_SYSTEM;
931 		}
932 
933 		/*
934 		 * TODO: Could use an array for state
935 		 * transitions, and caller might want it
936 		 * to bail early for some results.
937 		 */
938 		if (state == DRM_GPUSVM_SCAN_UNPOPULATED) {
939 			state = new_state;
940 		} else if (state != new_state) {
941 			if (new_state == DRM_GPUSVM_SCAN_SYSTEM ||
942 			    state == DRM_GPUSVM_SCAN_SYSTEM)
943 				state = DRM_GPUSVM_SCAN_MIXED;
944 			else if (state != DRM_GPUSVM_SCAN_MIXED)
945 				state = DRM_GPUSVM_SCAN_MIXED_DEVICE;
946 		}
947 
948 		i += 1ul << drm_gpusvm_hmm_pfn_to_order(pfns[i], i, npages);
949 	}
950 
951 	drm_gpusvm_notifier_unlock(range->gpusvm);
952 
953 err_free:
954 	kvfree(pfns);
955 	return state;
956 }
957 EXPORT_SYMBOL(drm_gpusvm_scan_mm);
958 
959 /**
960  * drm_gpusvm_range_chunk_size() - Determine chunk size for GPU SVM range
961  * @gpusvm: Pointer to the GPU SVM structure
962  * @notifier: Pointer to the GPU SVM notifier structure
963  * @vas: Pointer to the virtual memory area structure
964  * @fault_addr: Fault address
965  * @gpuva_start: Start address of GPUVA which mirrors CPU
966  * @gpuva_end: End address of GPUVA which mirrors CPU
967  * @check_pages_threshold: Check CPU pages for present threshold
968  * @dev_private_owner: The device private page owner
969  *
970  * This function determines the chunk size for the GPU SVM range based on the
971  * fault address, GPU SVM chunk sizes, existing GPU SVM ranges, and the virtual
972  * memory area boundaries.
973  *
974  * Return: Chunk size on success, LONG_MAX on failure.
975  */
976 static unsigned long
977 drm_gpusvm_range_chunk_size(struct drm_gpusvm *gpusvm,
978 			    struct drm_gpusvm_notifier *notifier,
979 			    struct vm_area_struct *vas,
980 			    unsigned long fault_addr,
981 			    unsigned long gpuva_start,
982 			    unsigned long gpuva_end,
983 			    unsigned long check_pages_threshold,
984 			    void *dev_private_owner)
985 {
986 	unsigned long start, end;
987 	int i = 0;
988 
989 retry:
990 	for (; i < gpusvm->num_chunks; ++i) {
991 		start = ALIGN_DOWN(fault_addr, gpusvm->chunk_sizes[i]);
992 		end = ALIGN(fault_addr + 1, gpusvm->chunk_sizes[i]);
993 
994 		if (start >= vas->vm_start && end <= vas->vm_end &&
995 		    start >= drm_gpusvm_notifier_start(notifier) &&
996 		    end <= drm_gpusvm_notifier_end(notifier) &&
997 		    start >= gpuva_start && end <= gpuva_end)
998 			break;
999 	}
1000 
1001 	if (i == gpusvm->num_chunks)
1002 		return LONG_MAX;
1003 
1004 	/*
1005 	 * If allocation more than page, ensure not to overlap with existing
1006 	 * ranges.
1007 	 */
1008 	if (end - start != SZ_4K) {
1009 		struct drm_gpusvm_range *range;
1010 
1011 		range = drm_gpusvm_range_find(notifier, start, end);
1012 		if (range) {
1013 			++i;
1014 			goto retry;
1015 		}
1016 
1017 		/*
1018 		 * XXX: Only create range on pages CPU has faulted in. Without
1019 		 * this check, or prefault, on BMG 'xe_exec_system_allocator --r
1020 		 * process-many-malloc' fails. In the failure case, each process
1021 		 * mallocs 16k but the CPU VMA is ~128k which results in 64k SVM
1022 		 * ranges. When migrating the SVM ranges, some processes fail in
1023 		 * drm_pagemap_migrate_to_devmem with 'migrate.cpages != npages'
1024 		 * and then upon drm_gpusvm_get_pages device pages from
1025 		 * other processes are collected + faulted in which creates all
1026 		 * sorts of problems. Unsure exactly how this happening, also
1027 		 * problem goes away if 'xe_exec_system_allocator --r
1028 		 * process-many-malloc' mallocs at least 64k at a time.
1029 		 */
1030 		if (end - start <= check_pages_threshold &&
1031 		    !drm_gpusvm_check_pages(gpusvm, notifier, start, end, dev_private_owner)) {
1032 			++i;
1033 			goto retry;
1034 		}
1035 	}
1036 
1037 	return end - start;
1038 }
1039 
1040 #ifdef CONFIG_LOCKDEP
1041 /**
1042  * drm_gpusvm_driver_lock_held() - Assert GPU SVM driver lock is held
1043  * @gpusvm: Pointer to the GPU SVM structure.
1044  *
1045  * Ensure driver lock is held.
1046  */
1047 static void drm_gpusvm_driver_lock_held(struct drm_gpusvm *gpusvm)
1048 {
1049 	if ((gpusvm)->lock_dep_map)
1050 		lockdep_assert(lock_is_held_type((gpusvm)->lock_dep_map, 0));
1051 }
1052 #else
1053 static void drm_gpusvm_driver_lock_held(struct drm_gpusvm *gpusvm)
1054 {
1055 }
1056 #endif
1057 
1058 /**
1059  * drm_gpusvm_find_vma_start() - Find start address for first VMA in range
1060  * @gpusvm: Pointer to the GPU SVM structure
1061  * @start: The inclusive start user address.
1062  * @end: The exclusive end user address.
1063  *
1064  * Returns: The start address of first VMA within the provided range,
1065  * ULONG_MAX otherwise. Assumes start_addr < end_addr.
1066  */
1067 unsigned long
1068 drm_gpusvm_find_vma_start(struct drm_gpusvm *gpusvm,
1069 			  unsigned long start,
1070 			  unsigned long end)
1071 {
1072 	struct mm_struct *mm = gpusvm->mm;
1073 	struct vm_area_struct *vma;
1074 	unsigned long addr = ULONG_MAX;
1075 
1076 	if (!mmget_not_zero(mm))
1077 		return addr;
1078 
1079 	mmap_read_lock(mm);
1080 
1081 	vma = find_vma_intersection(mm, start, end);
1082 	if (vma)
1083 		addr =  vma->vm_start;
1084 
1085 	mmap_read_unlock(mm);
1086 	mmput(mm);
1087 
1088 	return addr;
1089 }
1090 EXPORT_SYMBOL_GPL(drm_gpusvm_find_vma_start);
1091 
1092 /**
1093  * drm_gpusvm_range_find_or_insert() - Find or insert GPU SVM range
1094  * @gpusvm: Pointer to the GPU SVM structure
1095  * @fault_addr: Fault address
1096  * @gpuva_start: Start address of GPUVA which mirrors CPU
1097  * @gpuva_end: End address of GPUVA which mirrors CPU
1098  * @ctx: GPU SVM context
1099  *
1100  * This function finds or inserts a newly allocated a GPU SVM range based on the
1101  * fault address. Caller must hold a lock to protect range lookup and insertion.
1102  *
1103  * Return: Pointer to the GPU SVM range on success, ERR_PTR() on failure.
1104  */
1105 struct drm_gpusvm_range *
1106 drm_gpusvm_range_find_or_insert(struct drm_gpusvm *gpusvm,
1107 				unsigned long fault_addr,
1108 				unsigned long gpuva_start,
1109 				unsigned long gpuva_end,
1110 				const struct drm_gpusvm_ctx *ctx)
1111 {
1112 	struct drm_gpusvm_notifier *notifier;
1113 	struct drm_gpusvm_range *range;
1114 	struct mm_struct *mm = gpusvm->mm;
1115 	struct vm_area_struct *vas;
1116 	bool notifier_alloc = false;
1117 	unsigned long chunk_size;
1118 	int err;
1119 	bool migrate_devmem;
1120 
1121 	drm_gpusvm_driver_lock_held(gpusvm);
1122 
1123 	if (fault_addr < gpusvm->mm_start ||
1124 	    fault_addr > gpusvm->mm_start + gpusvm->mm_range)
1125 		return ERR_PTR(-EINVAL);
1126 
1127 	if (!mmget_not_zero(mm))
1128 		return ERR_PTR(-EFAULT);
1129 
1130 	notifier = drm_gpusvm_notifier_find(gpusvm, fault_addr, fault_addr + 1);
1131 	if (!notifier) {
1132 		notifier = drm_gpusvm_notifier_alloc(gpusvm, fault_addr);
1133 		if (IS_ERR(notifier)) {
1134 			err = PTR_ERR(notifier);
1135 			goto err_mmunlock;
1136 		}
1137 		notifier_alloc = true;
1138 		err = mmu_interval_notifier_insert(&notifier->notifier,
1139 						   mm,
1140 						   drm_gpusvm_notifier_start(notifier),
1141 						   drm_gpusvm_notifier_size(notifier),
1142 						   &drm_gpusvm_notifier_ops);
1143 		if (err)
1144 			goto err_notifier;
1145 	}
1146 
1147 	mmap_read_lock(mm);
1148 
1149 	vas = vma_lookup(mm, fault_addr);
1150 	if (!vas) {
1151 		err = -ENOENT;
1152 		goto err_notifier_remove;
1153 	}
1154 
1155 	if (!ctx->read_only && !(vas->vm_flags & VM_WRITE)) {
1156 		err = -EPERM;
1157 		goto err_notifier_remove;
1158 	}
1159 
1160 	if (vas->vm_flags & (VM_IO | VM_PFNMAP)) {
1161 		err = -EIO;
1162 		goto err_notifier_remove;
1163 	}
1164 
1165 	range = drm_gpusvm_range_find(notifier, fault_addr, fault_addr + 1);
1166 	if (range)
1167 		goto out_mmunlock;
1168 	/*
1169 	 * XXX: Short-circuiting migration based on migrate_vma_* current
1170 	 * limitations. If/when migrate_vma_* add more support, this logic will
1171 	 * have to change.
1172 	 */
1173 	migrate_devmem = ctx->devmem_possible &&
1174 		vma_is_anonymous(vas) && !is_vm_hugetlb_page(vas);
1175 
1176 	chunk_size = drm_gpusvm_range_chunk_size(gpusvm, notifier, vas,
1177 						 fault_addr, gpuva_start,
1178 						 gpuva_end,
1179 						 ctx->check_pages_threshold,
1180 						 ctx->device_private_page_owner);
1181 	if (chunk_size == LONG_MAX) {
1182 		err = -EINVAL;
1183 		goto err_notifier_remove;
1184 	}
1185 
1186 	range = drm_gpusvm_range_alloc(gpusvm, notifier, fault_addr, chunk_size,
1187 				       migrate_devmem);
1188 	if (IS_ERR(range)) {
1189 		err = PTR_ERR(range);
1190 		goto err_notifier_remove;
1191 	}
1192 
1193 	drm_gpusvm_range_insert(notifier, range);
1194 	if (notifier_alloc)
1195 		drm_gpusvm_notifier_insert(gpusvm, notifier);
1196 
1197 out_mmunlock:
1198 	mmap_read_unlock(mm);
1199 	mmput(mm);
1200 
1201 	return range;
1202 
1203 err_notifier_remove:
1204 	mmap_read_unlock(mm);
1205 	if (notifier_alloc)
1206 		mmu_interval_notifier_remove(&notifier->notifier);
1207 err_notifier:
1208 	if (notifier_alloc)
1209 		drm_gpusvm_notifier_free(gpusvm, notifier);
1210 err_mmunlock:
1211 	mmput(mm);
1212 	return ERR_PTR(err);
1213 }
1214 EXPORT_SYMBOL_GPL(drm_gpusvm_range_find_or_insert);
1215 
1216 /**
1217  * __drm_gpusvm_unmap_pages() - Unmap pages associated with GPU SVM pages (internal)
1218  * @gpusvm: Pointer to the GPU SVM structure
1219  * @svm_pages: Pointer to the GPU SVM pages structure
1220  * @npages: Number of pages to unmap
1221  *
1222  * This function unmap pages associated with a GPU SVM pages struct. Assumes and
1223  * asserts correct locking is in place when called.
1224  */
1225 static void __drm_gpusvm_unmap_pages(struct drm_gpusvm *gpusvm,
1226 				     struct drm_gpusvm_pages *svm_pages,
1227 				     unsigned long npages)
1228 {
1229 	struct drm_pagemap *dpagemap = svm_pages->dpagemap;
1230 	struct device *dev;
1231 	unsigned long i, j;
1232 
1233 	lockdep_assert_held(&gpusvm->notifier_lock);
1234 
1235 	if (!svm_pages->drm)
1236 		return;
1237 
1238 	dev = svm_pages->drm->dev;
1239 
1240 	if (svm_pages->flags.has_dma_mapping) {
1241 		struct drm_gpusvm_pages_flags flags = {
1242 			.__flags = svm_pages->flags.__flags,
1243 		};
1244 		const struct drm_pagemap_addr *addrs =
1245 			drm_gpusvm_pages_first_dma(svm_pages, NULL);
1246 		bool use_iova = dma_use_iova(&svm_pages->state);
1247 
1248 		/*
1249 		 * IOVA is reserved for the whole range but only the linked
1250 		 * system pages (state_offset bytes) need unlinking; free the
1251 		 * entire reservation to avoid leaking the device-page part.
1252 		 * On the error path state_offset is 0, so just free it.
1253 		 */
1254 		if (use_iova) {
1255 			if (svm_pages->state_offset)
1256 				dma_iova_unlink(dev, &svm_pages->state, 0,
1257 						svm_pages->state_offset,
1258 						addrs[0].dir, 0);
1259 			dma_iova_free(dev, &svm_pages->state);
1260 		}
1261 
1262 		/*
1263 		 * With IOVA and no device page the unlink above tore every
1264 		 * entry down, and that is also when the range may be folded
1265 		 * to one entry, which must not be walked per entry. dpagemap
1266 		 * is set before the first device_map(), so it is also right
1267 		 * on the error path, where the flags are not published yet.
1268 		 */
1269 		for (i = 0, j = 0;
1270 		     (!use_iova || dpagemap) && i < npages; j++) {
1271 			const struct drm_pagemap_addr *addr = &addrs[j];
1272 
1273 			if (addr->proto == DRM_INTERCONNECT_SYSTEM) {
1274 				/*
1275 				 * Linked IOVA pages were already torn down by
1276 				 * the dma_iova_unlink()/dma_iova_free() above;
1277 				 * only the non-IOVA mappings need unmap here.
1278 				 */
1279 				if (!use_iova)
1280 					dma_unmap_page(dev,
1281 						       addr->addr,
1282 						       PAGE_SIZE << addr->order,
1283 						       addr->dir);
1284 			} else if (dpagemap && dpagemap->ops->device_unmap)
1285 				dpagemap->ops->device_unmap(dpagemap,
1286 							    dev, addr);
1287 			i += 1 << addr->order;
1288 		}
1289 
1290 		/* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */
1291 		flags.has_devmem_pages = false;
1292 		flags.has_dma_mapping = false;
1293 		WRITE_ONCE(svm_pages->flags.__flags, flags.__flags);
1294 
1295 		drm_pagemap_put(svm_pages->dpagemap);
1296 		svm_pages->dpagemap = NULL;
1297 	}
1298 }
1299 
1300 /**
1301  * __drm_gpusvm_free_pages() - Free dma array associated with GPU SVM pages
1302  * @gpusvm: Pointer to the GPU SVM structure
1303  * @svm_pages: Pointer to the GPU SVM pages structure
1304  *
1305  * This function frees the dma address array associated with a GPU SVM range.
1306  */
1307 static void __drm_gpusvm_free_pages(struct drm_gpusvm *gpusvm,
1308 				    struct drm_gpusvm_pages *svm_pages)
1309 {
1310 	lockdep_assert_held(&gpusvm->notifier_lock);
1311 
1312 	if (svm_pages->flags.inline_dma_mapping) {
1313 		struct drm_gpusvm_pages_flags flags = {
1314 			.__flags = svm_pages->flags.__flags,
1315 		};
1316 
1317 		svm_pages->inline_addr = (struct drm_pagemap_addr){};
1318 		flags.inline_dma_mapping = false;
1319 		/* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */
1320 		WRITE_ONCE(svm_pages->flags.__flags, flags.__flags);
1321 		return;
1322 	}
1323 
1324 	if (svm_pages->dma_addr) {
1325 		kvfree(svm_pages->dma_addr);
1326 		svm_pages->dma_addr = NULL;
1327 	}
1328 }
1329 
1330 /**
1331  * drm_gpusvm_free_pages() - Free dma-mapping associated with GPU SVM pages
1332  * struct
1333  * @gpusvm: Pointer to the GPU SVM structure
1334  * @svm_pages: Pointer to the GPU SVM pages structure
1335  * @npages: Number of mapped pages
1336  *
1337  * This function unmaps and frees the dma address array associated with a GPU
1338  * SVM pages struct.
1339  */
1340 void drm_gpusvm_free_pages(struct drm_gpusvm *gpusvm,
1341 			   struct drm_gpusvm_pages *svm_pages,
1342 			   unsigned long npages)
1343 {
1344 	drm_gpusvm_notifier_lock(gpusvm);
1345 	__drm_gpusvm_unmap_pages(gpusvm, svm_pages, npages);
1346 	__drm_gpusvm_free_pages(gpusvm, svm_pages);
1347 	drm_gpusvm_notifier_unlock(gpusvm);
1348 }
1349 EXPORT_SYMBOL_GPL(drm_gpusvm_free_pages);
1350 
1351 /**
1352  * drm_gpusvm_range_remove() - Remove GPU SVM range
1353  * @gpusvm: Pointer to the GPU SVM structure
1354  * @range: Pointer to the GPU SVM range to be removed
1355  *
1356  * This function removes the specified GPU SVM range and also removes the parent
1357  * GPU SVM notifier if no more ranges remain in the notifier. The caller must
1358  * hold a lock to protect range and notifier removal.
1359  *
1360  * This function does not unmap or free the drm_gpusvm_pages, the driver owns
1361  * that lifecycle. The caller must DMA unmap the range's pages before calling
1362  * this function, so a range is never removed from the MMU interval tree while
1363  * still DMA mapped. Typically the driver calls drm_gpusvm_unmap_pages() first.
1364  * And the range_free callback's drm_gpusvm_free_pages() is a final fallback safe
1365  * net.
1366  */
1367 void drm_gpusvm_range_remove(struct drm_gpusvm *gpusvm,
1368 			     struct drm_gpusvm_range *range)
1369 {
1370 	struct drm_gpusvm_notifier *notifier;
1371 
1372 	drm_gpusvm_driver_lock_held(gpusvm);
1373 
1374 	notifier = drm_gpusvm_notifier_find(gpusvm,
1375 					    drm_gpusvm_range_start(range),
1376 					    drm_gpusvm_range_start(range) + 1);
1377 	if (WARN_ON_ONCE(!notifier))
1378 		return;
1379 
1380 	drm_gpusvm_notifier_lock(gpusvm);
1381 	__drm_gpusvm_range_remove(notifier, range);
1382 	drm_gpusvm_notifier_unlock(gpusvm);
1383 
1384 	drm_gpusvm_range_put(range);
1385 
1386 	if (RB_EMPTY_ROOT(&notifier->root.rb_root)) {
1387 		if (!notifier->flags.removed)
1388 			mmu_interval_notifier_remove(&notifier->notifier);
1389 		drm_gpusvm_notifier_remove(gpusvm, notifier);
1390 		drm_gpusvm_notifier_free(gpusvm, notifier);
1391 	}
1392 }
1393 EXPORT_SYMBOL_GPL(drm_gpusvm_range_remove);
1394 
1395 /**
1396  * drm_gpusvm_range_get() - Get a reference to GPU SVM range
1397  * @range: Pointer to the GPU SVM range
1398  *
1399  * This function increments the reference count of the specified GPU SVM range.
1400  *
1401  * Return: Pointer to the GPU SVM range.
1402  */
1403 struct drm_gpusvm_range *
1404 drm_gpusvm_range_get(struct drm_gpusvm_range *range)
1405 {
1406 	kref_get(&range->refcount);
1407 
1408 	return range;
1409 }
1410 EXPORT_SYMBOL_GPL(drm_gpusvm_range_get);
1411 
1412 /**
1413  * drm_gpusvm_range_destroy() - Destroy GPU SVM range
1414  * @refcount: Pointer to the reference counter embedded in the GPU SVM range
1415  *
1416  * This function destroys the specified GPU SVM range when its reference count
1417  * reaches zero. If a custom range-free function is provided, it is invoked to
1418  * free the range; otherwise, the range is deallocated using kfree().
1419  */
1420 static void drm_gpusvm_range_destroy(struct kref *refcount)
1421 {
1422 	struct drm_gpusvm_range *range =
1423 		container_of(refcount, struct drm_gpusvm_range, refcount);
1424 	struct drm_gpusvm *gpusvm = range->gpusvm;
1425 
1426 	if (gpusvm->ops->range_free)
1427 		gpusvm->ops->range_free(range);
1428 	else
1429 		kfree(range);
1430 }
1431 
1432 /**
1433  * drm_gpusvm_range_put() - Put a reference to GPU SVM range
1434  * @range: Pointer to the GPU SVM range
1435  *
1436  * This function decrements the reference count of the specified GPU SVM range
1437  * and frees it when the count reaches zero.
1438  */
1439 void drm_gpusvm_range_put(struct drm_gpusvm_range *range)
1440 {
1441 	kref_put(&range->refcount, drm_gpusvm_range_destroy);
1442 }
1443 EXPORT_SYMBOL_GPL(drm_gpusvm_range_put);
1444 
1445 /**
1446  * drm_gpusvm_pages_valid() - GPU SVM range pages valid
1447  * @gpusvm: Pointer to the GPU SVM structure
1448  * @svm_pages: Pointer to the GPU SVM pages structure
1449  *
1450  * This function determines if a GPU SVM range pages are valid. Expected be
1451  * called holding gpusvm->notifier_lock and as the last step before committing a
1452  * GPU binding. This is akin to a notifier seqno check in the HMM documentation
1453  * but due to wider notifiers (i.e., notifiers which span multiple ranges) this
1454  * function is required for finer grained checking (i.e., per range) if pages
1455  * are valid.
1456  *
1457  * Return: True if GPU SVM range has valid pages, False otherwise
1458  */
1459 bool drm_gpusvm_pages_valid(struct drm_gpusvm *gpusvm,
1460 			    struct drm_gpusvm_pages *svm_pages)
1461 {
1462 	lockdep_assert_held(&gpusvm->notifier_lock);
1463 
1464 	return svm_pages->flags.has_devmem_pages || svm_pages->flags.has_dma_mapping;
1465 }
1466 EXPORT_SYMBOL_GPL(drm_gpusvm_pages_valid);
1467 
1468 /**
1469  * drm_gpusvm_pages_valid_unlocked() - GPU SVM pages valid unlocked
1470  * @gpusvm: Pointer to the GPU SVM structure
1471  * @svm_pages: Array of GPU SVM pages structures
1472  * @num_pages: Number of drm_gpusvm_pages instances in @svm_pages
1473  *
1474  * This function determines if every GPU SVM pages instance is valid, resetting
1475  * every instance which is not so that get_pages() maps it afresh. It therefore
1476  * has to walk them all. Expected be called without holding
1477  * gpusvm->notifier_lock.
1478  *
1479  * Return: True if all GPU SVM pages are valid, False otherwise
1480  */
1481 static bool drm_gpusvm_pages_valid_unlocked(struct drm_gpusvm *gpusvm,
1482 					    struct drm_gpusvm_pages *svm_pages,
1483 					    unsigned int num_pages)
1484 {
1485 	bool pages_valid = true;
1486 	unsigned int p;
1487 
1488 	drm_gpusvm_notifier_lock(gpusvm);
1489 	for (p = 0; p < num_pages; ++p) {
1490 		if (drm_gpusvm_pages_valid(gpusvm, &svm_pages[p]))
1491 			continue;
1492 		__drm_gpusvm_free_pages(gpusvm, &svm_pages[p]);
1493 		pages_valid = false;
1494 	}
1495 	drm_gpusvm_notifier_unlock(gpusvm);
1496 
1497 	return pages_valid;
1498 }
1499 
1500 /**
1501  * drm_gpusvm_pages_inlinable() - Whether the dma address can be inlined
1502  * @svm_pages: The SVM pages instance that was just mapped
1503  * @nentries: Number of entries the mapping loop produced
1504  * @npages: Number of pages in the CPU range
1505  *
1506  * A THP maps as one huge page, and an IOVA reservation links every page of
1507  * the range at the next offset, so the device addresses run contiguously from
1508  * entry 0. Either way one entry describes the whole range, so the dma_addr
1509  * array can be freed and the address kept inline.
1510  *
1511  * state_offset advances only on the IOVA branch, so reaching the full range
1512  * length proves no device page was mapped in between. Only single page
1513  * entries fold, so the order kept is 0 and describes the range truthfully.
1514  * Larger chunks, several huge pages among them, stay an array that is
1515  * already short and that a consumer places with one PTE each.
1516  *
1517  * Return: True if the mapping fits in a single drm_pagemap_addr.
1518  */
1519 static bool drm_gpusvm_pages_inlinable(struct drm_gpusvm_pages *svm_pages,
1520 				       unsigned long nentries,
1521 				       unsigned long npages)
1522 {
1523 	if (nentries == 1)
1524 		return true;
1525 
1526 	return nentries == npages && dma_use_iova(&svm_pages->state) &&
1527 	       svm_pages->state_offset == npages * PAGE_SIZE;
1528 }
1529 
1530 /**
1531  * drm_gpusvm_walk_devmem() - Invoke the devmem callback across faulted pages
1532  * @gpusvm: Pointer to the GPU SVM structure
1533  * @pfns: The already-faulted pfn array (size @npages)
1534  * @npages: Number of pages in the CPU range
1535  * @ctx: GPU SVM context, with a non-NULL &drm_gpusvm_ctx.devmem_fn
1536  *
1537  * Invoke &drm_gpusvm_ctx.devmem_fn once per contiguous run of @pfns backed by
1538  * the same device-memory allocation. Must be called under the notifier lock.
1539  */
1540 static void drm_gpusvm_walk_devmem(struct drm_gpusvm *gpusvm,
1541 				   unsigned long *pfns,
1542 				   unsigned long npages,
1543 				   const struct drm_gpusvm_ctx *ctx)
1544 {
1545 	struct drm_pagemap_devmem *last = NULL;
1546 	unsigned int order = 0;
1547 	unsigned long i;
1548 
1549 	lockdep_assert_held(&gpusvm->notifier_lock);
1550 
1551 	for (i = 0; i < npages; i += 1 << order) {
1552 		struct page *page = hmm_pfn_to_page(pfns[i]);
1553 		struct drm_pagemap_devmem *devmem;
1554 
1555 		order = drm_gpusvm_hmm_pfn_to_order(pfns[i], i, npages);
1556 
1557 		if (!is_device_private_page(page) &&
1558 		    !is_device_coherent_page(page)) {
1559 			last = NULL;
1560 			continue;
1561 		}
1562 
1563 		devmem = drm_pagemap_page_to_devmem(page);
1564 		if (devmem == last)
1565 			continue;
1566 
1567 		last = devmem;
1568 		ctx->devmem_fn(devmem);
1569 	}
1570 }
1571 
1572 /**
1573  * drm_gpusvm_dma_map_pages() - Walk and DMA map one drm_gpusvm_pages instance
1574  * @gpusvm: Pointer to the GPU SVM structure
1575  * @svm_pages: The SVM pages instance to populate with dma-addresses
1576  * @pfns: The already-faulted pfn array (size @npages)
1577  * @npages: Number of pages in the CPU range
1578  * @ctx: GPU SVM context
1579  * @dma_dir: DMA data direction for the mappings
1580  *
1581  * Walk the faulted @pfns and map them into @svm_pages for DMA access through
1582  * its owning drm_device. Must be called under the notifier lock and only for
1583  * an instance without a live mapping. On failure this unwinds the partial
1584  * mapping of this instance before returning.
1585  *
1586  * Return: 0 on success, negative error code on failure.
1587  */
1588 static int drm_gpusvm_dma_map_pages(struct drm_gpusvm *gpusvm,
1589 				    struct drm_gpusvm_pages *svm_pages,
1590 				    unsigned long *pfns,
1591 				    unsigned long npages,
1592 				    const struct drm_gpusvm_ctx *ctx,
1593 				    enum dma_data_direction dma_dir)
1594 {
1595 	void *zdd = NULL;
1596 	unsigned long i, j;
1597 	unsigned long num_dma_mapped = 0;
1598 	unsigned int order = 0;
1599 	int err = 0;
1600 	struct dev_pagemap *pagemap = NULL;
1601 	struct drm_pagemap *dpagemap;
1602 	struct drm_gpusvm_pages_flags flags;
1603 	struct dma_iova_state *state = &svm_pages->state;
1604 
1605 	lockdep_assert_held(&gpusvm->notifier_lock);
1606 
1607 	*state = (struct dma_iova_state){};
1608 	svm_pages->state_offset = 0;
1609 
1610 	flags.__flags = svm_pages->flags.__flags;
1611 
1612 	for (i = 0, j = 0; i < npages; ++j) {
1613 		struct page *page = hmm_pfn_to_page(pfns[i]);
1614 
1615 		order = drm_gpusvm_hmm_pfn_to_order(pfns[i], i, npages);
1616 		if (is_device_private_page(page) ||
1617 		    is_device_coherent_page(page)) {
1618 			struct drm_pagemap_zdd *__zdd =
1619 				drm_pagemap_page_zone_device_data(page);
1620 
1621 			if (!ctx->allow_mixed &&
1622 			    zdd != __zdd && i > 0) {
1623 				err = -EOPNOTSUPP;
1624 				goto err_unmap;
1625 			}
1626 			zdd = __zdd;
1627 			if (pagemap != page_pgmap(page)) {
1628 				if (pagemap) {
1629 					err = -EOPNOTSUPP;
1630 					goto err_unmap;
1631 				}
1632 
1633 				pagemap = page_pgmap(page);
1634 				dpagemap = drm_pagemap_page_to_dpagemap(page);
1635 				if (drm_WARN_ON(svm_pages->drm, !dpagemap)) {
1636 					/*
1637 					 * Raced. This is not supposed to happen
1638 					 * since hmm_range_fault() should've migrated
1639 					 * this page to system.
1640 					 */
1641 					err = -EAGAIN;
1642 					goto err_unmap;
1643 				}
1644 
1645 				/*
1646 				 * Set the dpagemap as soon as the first
1647 				 * device page is mapped so the err_unmap path
1648 				 * can device_unmap() the device mappings that
1649 				 * have already been created.
1650 				 */
1651 				drm_pagemap_get(dpagemap);
1652 				drm_pagemap_put(svm_pages->dpagemap);
1653 				svm_pages->dpagemap = dpagemap;
1654 			}
1655 			svm_pages->dma_addr[j] =
1656 				dpagemap->ops->device_map(dpagemap,
1657 							  svm_pages->drm->dev,
1658 							  page, order,
1659 							  dma_dir);
1660 			if (dma_mapping_error(svm_pages->drm->dev,
1661 					      svm_pages->dma_addr[j].addr)) {
1662 				err = -EFAULT;
1663 				goto err_unmap;
1664 			}
1665 		} else {
1666 			dma_addr_t addr;
1667 
1668 			if (is_zone_device_page(page) ||
1669 			    (pagemap && !ctx->allow_mixed)) {
1670 				err = -EOPNOTSUPP;
1671 				goto err_unmap;
1672 			}
1673 
1674 			if (ctx->devmem_only) {
1675 				err = -EFAULT;
1676 				goto err_unmap;
1677 			}
1678 
1679 			if (!i)
1680 				dma_iova_try_alloc(svm_pages->drm->dev, state,
1681 						   0, npages * PAGE_SIZE);
1682 
1683 			if (dma_use_iova(state)) {
1684 				err = dma_iova_link(svm_pages->drm->dev, state,
1685 						    hmm_pfn_to_phys(pfns[i]),
1686 						    svm_pages->state_offset,
1687 						    PAGE_SIZE << order,
1688 						    dma_dir, 0);
1689 				if (err)
1690 					goto err_unmap;
1691 
1692 				addr = state->addr + svm_pages->state_offset;
1693 				svm_pages->state_offset += PAGE_SIZE << order;
1694 			} else {
1695 				addr = dma_map_page(svm_pages->drm->dev,
1696 						    page, 0,
1697 						    PAGE_SIZE << order,
1698 						    dma_dir);
1699 				if (dma_mapping_error(svm_pages->drm->dev, addr)) {
1700 					err = -EFAULT;
1701 					goto err_unmap;
1702 				}
1703 			}
1704 
1705 			svm_pages->dma_addr[j] = drm_pagemap_addr_encode
1706 				(addr, DRM_INTERCONNECT_SYSTEM, order,
1707 				 dma_dir);
1708 		}
1709 		i += 1 << order;
1710 		num_dma_mapped = i;
1711 		flags.has_dma_mapping = true;
1712 	}
1713 
1714 	if (dma_use_iova(state)) {
1715 		err = dma_iova_sync(svm_pages->drm->dev, state, 0,
1716 				    svm_pages->state_offset);
1717 		if (err)
1718 			goto err_unmap;
1719 	}
1720 
1721 	if (pagemap)
1722 		flags.has_devmem_pages = true;
1723 
1724 	if (drm_gpusvm_pages_inlinable(svm_pages, j, npages)) {
1725 		struct drm_pagemap_addr addr = svm_pages->dma_addr[0];
1726 
1727 		kvfree(svm_pages->dma_addr);
1728 		svm_pages->inline_addr = addr;
1729 		flags.inline_dma_mapping = true;
1730 	}
1731 
1732 	/* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */
1733 	WRITE_ONCE(svm_pages->flags.__flags, flags.__flags);
1734 
1735 	return 0;
1736 
1737 err_unmap:
1738 	svm_pages->flags.has_dma_mapping = true;
1739 	__drm_gpusvm_unmap_pages(gpusvm, svm_pages, num_dma_mapped);
1740 	return err;
1741 }
1742 
1743 /**
1744  * drm_gpusvm_get_pages() - Get pages and populate GPU SVM pages struct
1745  * @gpusvm: Pointer to the GPU SVM structure
1746  * @svm_pages: Array of SVM pages instances to populate with dma addresses
1747  * @num_pages: Number of drm_gpusvm_pages instances in @svm_pages, must not be 0
1748  * @mm: The mm corresponding to the CPU range
1749  * @notifier: The corresponding notifier for the given CPU range
1750  * @pages_start: Start CPU address for the pages
1751  * @pages_end: End CPU address for the pages (exclusive)
1752  * @ctx: GPU SVM context
1753  *
1754  * This function gets and maps pages for a CPU range and ensures they are
1755  * mapped for DMA access. The HMM fault for the CPU range is performed once,
1756  * the DMA mapping by drm_gpusvm_dma_map_pages() is then done per instance,
1757  * one per owning drm_device. The retry against notifier races is kept here
1758  * in common code so drivers never open code it.
1759  *
1760  * On error the instances mapped before the failing one stay mapped, so the
1761  * caller must unmap and free every instance regardless of the return value.
1762  *
1763  * With &drm_gpusvm_ctx.no_dma_map no mapping state is recorded, so
1764  * drm_gpusvm_pages_valid() never returns true and success is only a snapshot:
1765  * the caller must recheck mmu_interval_read_retry() against the recorded
1766  * &drm_gpusvm_pages.notifier_seq under the notifier lock, and hold it until
1767  * its work is visible to invalidation.
1768  *
1769  * Return: 0 on success, negative error code on failure.
1770  */
1771 int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
1772 			 struct drm_gpusvm_pages *svm_pages,
1773 			 unsigned int num_pages,
1774 			 struct mm_struct *mm,
1775 			 struct mmu_interval_notifier *notifier,
1776 			 unsigned long pages_start, unsigned long pages_end,
1777 			 const struct drm_gpusvm_ctx *ctx)
1778 {
1779 	struct hmm_range hmm_range = {
1780 		.default_flags = HMM_PFN_REQ_FAULT | (ctx->read_only ? 0 :
1781 			HMM_PFN_REQ_WRITE),
1782 		.notifier = notifier,
1783 		.start = pages_start,
1784 		.end = pages_end,
1785 		.dev_private_owner = ctx->device_private_page_owner,
1786 	};
1787 	unsigned long timeout =
1788 		jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
1789 	unsigned long remaining;
1790 	unsigned long npages = npages_in_range(pages_start, pages_end);
1791 	unsigned long *pfns;
1792 	int err = 0;
1793 	enum dma_data_direction dma_dir = ctx->read_only ? DMA_TO_DEVICE :
1794 							   DMA_BIDIRECTIONAL;
1795 	const bool map_dma = !ctx->no_dma_map;
1796 	unsigned int p;
1797 
1798 	if (!num_pages)
1799 		return -EINVAL;
1800 
1801 	if (ctx->no_dma_map && ctx->devmem_only)
1802 		return -EINVAL;
1803 
1804 	if (map_dma) {
1805 		for (p = 0; p < num_pages; ++p) {
1806 			if (!svm_pages[p].drm)
1807 				return -EINVAL;
1808 		}
1809 	}
1810 
1811 retry:
1812 	remaining = timeout - jiffies;
1813 
1814 	if (time_after_eq(jiffies, timeout))
1815 		return -EBUSY;
1816 
1817 	hmm_range.notifier_seq = mmu_interval_read_begin(notifier);
1818 
1819 	/*
1820 	 * drm_gpusvm_pages_valid_unlocked() also resets any instance whose
1821 	 * mapping was invalidated, so it must run on the map_dma path even when
1822 	 * devmem_fn suppresses the fast path; otherwise a stale inline mapping
1823 	 * is later mistaken for a dma_addr array by drm_gpusvm_dma_map_pages().
1824 	 */
1825 	if (map_dma) {
1826 		bool pages_valid = drm_gpusvm_pages_valid_unlocked(gpusvm,
1827 								   svm_pages,
1828 								   num_pages);
1829 
1830 		if (!ctx->devmem_fn && pages_valid)
1831 			goto set_seqno;
1832 	}
1833 
1834 	pfns = kvmalloc_array(npages, sizeof(*pfns), GFP_KERNEL);
1835 	if (!pfns)
1836 		return -ENOMEM;
1837 
1838 	if (!mmget_not_zero(mm)) {
1839 		err = -EFAULT;
1840 		goto err_free;
1841 	}
1842 
1843 	hmm_range.hmm_pfns = pfns;
1844 	err = hmm_range_fault_unlocked_timeout(&hmm_range, remaining);
1845 	mmput(mm);
1846 	if (err)
1847 		goto err_free;
1848 
1849 	if (map_dma) {
1850 		for (p = 0; p < num_pages; ++p) {
1851 			if (drm_gpusvm_pages_first_dma(&svm_pages[p], NULL))
1852 				continue;
1853 			svm_pages[p].dma_addr =
1854 				kvzalloc_objs(*svm_pages[p].dma_addr, npages);
1855 			if (!svm_pages[p].dma_addr) {
1856 				err = -ENOMEM;
1857 				goto err_free;
1858 			}
1859 		}
1860 	}
1861 
1862 	/*
1863 	 * Perform all dma mappings under the notifier lock to not
1864 	 * access freed pages. A notifier will either block on
1865 	 * the notifier lock or unmap dma.
1866 	 */
1867 	drm_gpusvm_notifier_lock(gpusvm);
1868 
1869 	/*
1870 	 * drm_gpusvm_range_set_unmapped() flags the whole array in one go under
1871 	 * the write lock, so any instance answers for all of them here.
1872 	 */
1873 	if (svm_pages[0].flags.unmapped) {
1874 		drm_gpusvm_notifier_unlock(gpusvm);
1875 		err = -EFAULT;
1876 		goto err_free;
1877 	}
1878 
1879 	if (mmu_interval_read_retry(notifier, hmm_range.notifier_seq)) {
1880 		drm_gpusvm_notifier_unlock(gpusvm);
1881 		kvfree(pfns);
1882 		goto retry;
1883 	}
1884 
1885 	if (ctx->devmem_fn)
1886 		drm_gpusvm_walk_devmem(gpusvm, pfns, npages, ctx);
1887 
1888 	if (map_dma) {
1889 		for (p = 0; p < num_pages; ++p) {
1890 			if (drm_gpusvm_pages_valid(gpusvm, &svm_pages[p]))
1891 				continue;
1892 
1893 			err = drm_gpusvm_dma_map_pages(gpusvm, &svm_pages[p], pfns,
1894 						       npages, ctx, dma_dir);
1895 			if (err) {
1896 				/*
1897 				 * The failing instance was unwound by the
1898 				 * helper. Keep the ones mapped earlier: the
1899 				 * -EAGAIN retry reuses them, and the driver
1900 				 * unmaps every instance with the range on the
1901 				 * other error paths.
1902 				 */
1903 				drm_gpusvm_notifier_unlock(gpusvm);
1904 				goto err_free;
1905 			}
1906 		}
1907 	}
1908 
1909 	drm_gpusvm_notifier_unlock(gpusvm);
1910 	kvfree(pfns);
1911 set_seqno:
1912 	for (p = 0; p < num_pages; ++p)
1913 		svm_pages[p].notifier_seq = hmm_range.notifier_seq;
1914 
1915 	return 0;
1916 
1917 err_free:
1918 	kvfree(pfns);
1919 	if (err == -EAGAIN)
1920 		goto retry;
1921 	return err;
1922 }
1923 EXPORT_SYMBOL_GPL(drm_gpusvm_get_pages);
1924 
1925 /**
1926  * drm_gpusvm_unmap_pages() - Unmap GPU svm pages
1927  * @gpusvm: Pointer to the GPU SVM structure
1928  * @svm_pages: Pointer to the GPU SVM pages structure
1929  * @npages: Number of pages in @svm_pages.
1930  * @ctx: GPU SVM context
1931  *
1932  * This function unmaps pages associated with a GPU SVM pages struct. If
1933  * @in_notifier is set, it is assumed that gpusvm->notifier_lock is held in
1934  * write mode; if it is clear, it acquires gpusvm->notifier_lock in read mode.
1935  * Must be called in the invalidate() callback of the corresponding notifier for
1936  * IOMMU security model.
1937  */
1938 void drm_gpusvm_unmap_pages(struct drm_gpusvm *gpusvm,
1939 			    struct drm_gpusvm_pages *svm_pages,
1940 			    unsigned long npages,
1941 			    const struct drm_gpusvm_ctx *ctx)
1942 {
1943 	if (ctx->in_notifier)
1944 		lockdep_assert_held_write(&gpusvm->notifier_lock);
1945 	else
1946 		drm_gpusvm_notifier_lock(gpusvm);
1947 
1948 	__drm_gpusvm_unmap_pages(gpusvm, svm_pages, npages);
1949 
1950 	if (!ctx->in_notifier)
1951 		drm_gpusvm_notifier_unlock(gpusvm);
1952 }
1953 EXPORT_SYMBOL_GPL(drm_gpusvm_unmap_pages);
1954 
1955 /**
1956  * drm_gpusvm_range_evict() - Evict GPU SVM range
1957  * @gpusvm: Pointer to the GPU SVM structure
1958  * @range: Pointer to the GPU SVM range to be removed
1959  *
1960  * This function evicts the specified GPU SVM range.
1961  *
1962  * Return: 0 on success, a negative error code on failure.
1963  */
1964 int drm_gpusvm_range_evict(struct drm_gpusvm *gpusvm,
1965 			   struct drm_gpusvm_range *range)
1966 {
1967 	struct mmu_interval_notifier *notifier = &range->notifier->notifier;
1968 	struct hmm_range hmm_range = {
1969 		.default_flags = HMM_PFN_REQ_FAULT,
1970 		.notifier = notifier,
1971 		.start = drm_gpusvm_range_start(range),
1972 		.end = drm_gpusvm_range_end(range),
1973 		.dev_private_owner = NULL,
1974 	};
1975 	unsigned long timeout = msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
1976 	unsigned long *pfns;
1977 	unsigned long npages = npages_in_range(drm_gpusvm_range_start(range),
1978 					       drm_gpusvm_range_end(range));
1979 	int err = 0;
1980 	struct mm_struct *mm = gpusvm->mm;
1981 
1982 	if (!mmget_not_zero(mm))
1983 		return -EFAULT;
1984 
1985 	pfns = kvmalloc_array(npages, sizeof(*pfns), GFP_KERNEL);
1986 	if (!pfns) {
1987 		mmput(mm);
1988 		return -ENOMEM;
1989 	}
1990 
1991 	hmm_range.hmm_pfns = pfns;
1992 	err = hmm_range_fault_unlocked_timeout(&hmm_range, timeout);
1993 
1994 	kvfree(pfns);
1995 	mmput(mm);
1996 
1997 	return err == -EBUSY ? -ETIME : err;
1998 }
1999 EXPORT_SYMBOL_GPL(drm_gpusvm_range_evict);
2000 
2001 /**
2002  * drm_gpusvm_has_mapping() - Check if GPU SVM has mapping for the given address range
2003  * @gpusvm: Pointer to the GPU SVM structure.
2004  * @start: Start address
2005  * @end: End address
2006  *
2007  * Return: True if GPU SVM has mapping, False otherwise
2008  */
2009 bool drm_gpusvm_has_mapping(struct drm_gpusvm *gpusvm, unsigned long start,
2010 			    unsigned long end)
2011 {
2012 	struct drm_gpusvm_notifier *notifier;
2013 
2014 	drm_gpusvm_for_each_notifier(notifier, gpusvm, start, end) {
2015 		struct drm_gpusvm_range *range = NULL;
2016 
2017 		drm_gpusvm_for_each_range(range, notifier, start, end)
2018 			return true;
2019 	}
2020 
2021 	return false;
2022 }
2023 EXPORT_SYMBOL_GPL(drm_gpusvm_has_mapping);
2024 
2025 /**
2026  * drm_gpusvm_range_set_unmapped() - Mark a GPU SVM range as unmapped
2027  * @range: Pointer to the GPU SVM range structure.
2028  * @pages: Pointer to the GPU SVM pages structure(s).
2029  * @pages_count: Number of GPU SVM pages structure(s) passed in.
2030  * @mmu_range: Pointer to the MMU notifier range structure.
2031  *
2032  * This function marks a GPU SVM range as unmapped and sets the partial_unmap flag
2033  * if the range partially falls within the provided MMU notifier range.
2034  */
2035 void drm_gpusvm_range_set_unmapped(struct drm_gpusvm_range *range,
2036 				   struct drm_gpusvm_pages *pages,
2037 				   unsigned int pages_count,
2038 				   const struct mmu_notifier_range *mmu_range)
2039 {
2040 	struct drm_gpusvm_range_flags range_flags = {
2041 		.__flags = range->flags.__flags,
2042 	};
2043 	unsigned int i;
2044 
2045 	lockdep_assert_held_write(&range->gpusvm->notifier_lock);
2046 
2047 	range_flags.unmapped = true;
2048 	for (i = 0; i < pages_count; ++i) {
2049 		struct drm_gpusvm_pages_flags flags = {
2050 			.__flags = pages[i].flags.__flags,
2051 		};
2052 
2053 		flags.unmapped = true;
2054 		/* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */
2055 		WRITE_ONCE(pages[i].flags.__flags, flags.__flags);
2056 	}
2057 	if (drm_gpusvm_range_start(range) < mmu_range->start ||
2058 	    drm_gpusvm_range_end(range) > mmu_range->end)
2059 		range_flags.partial_unmap = true;
2060 	/* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */
2061 	WRITE_ONCE(range->flags.__flags, range_flags.__flags);
2062 }
2063 EXPORT_SYMBOL_GPL(drm_gpusvm_range_set_unmapped);
2064 
2065 MODULE_DESCRIPTION("DRM GPUSVM");
2066 MODULE_LICENSE("GPL");
2067