1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2024 Intel Corporation
4 */
5
6 #include <linux/scatterlist.h>
7 #include <linux/mmu_notifier.h>
8 #include <linux/dma-mapping.h>
9 #include <linux/memremap.h>
10 #include <linux/swap.h>
11 #include <linux/hmm.h>
12 #include <linux/mm.h>
13 #include "xe_hmm.h"
14 #include "xe_vm.h"
15 #include "xe_bo.h"
16
xe_npages_in_range(unsigned long start,unsigned long end)17 static u64 xe_npages_in_range(unsigned long start, unsigned long end)
18 {
19 return (end - start) >> PAGE_SHIFT;
20 }
21
22 /*
23 * xe_mark_range_accessed() - mark a range is accessed, so core mm
24 * have such information for memory eviction or write back to
25 * hard disk
26 *
27 * @range: the range to mark
28 * @write: if write to this range, we mark pages in this range
29 * as dirty
30 */
xe_mark_range_accessed(struct hmm_range * range,bool write)31 static void xe_mark_range_accessed(struct hmm_range *range, bool write)
32 {
33 struct page *page;
34 u64 i, npages;
35
36 npages = xe_npages_in_range(range->start, range->end);
37 for (i = 0; i < npages; i++) {
38 page = hmm_pfn_to_page(range->hmm_pfns[i]);
39 if (write)
40 set_page_dirty_lock(page);
41
42 mark_page_accessed(page);
43 }
44 }
45
46 /*
47 * xe_build_sg() - build a scatter gather table for all the physical pages/pfn
48 * in a hmm_range. dma-map pages if necessary. dma-address is save in sg table
49 * and will be used to program GPU page table later.
50 *
51 * @xe: the xe device who will access the dma-address in sg table
52 * @range: the hmm range that we build the sg table from. range->hmm_pfns[]
53 * has the pfn numbers of pages that back up this hmm address range.
54 * @st: pointer to the sg table.
55 * @write: whether we write to this range. This decides dma map direction
56 * for system pages. If write we map it bi-diretional; otherwise
57 * DMA_TO_DEVICE
58 *
59 * All the contiguous pfns will be collapsed into one entry in
60 * the scatter gather table. This is for the purpose of efficiently
61 * programming GPU page table.
62 *
63 * The dma_address in the sg table will later be used by GPU to
64 * access memory. So if the memory is system memory, we need to
65 * do a dma-mapping so it can be accessed by GPU/DMA.
66 *
67 * FIXME: This function currently only support pages in system
68 * memory. If the memory is GPU local memory (of the GPU who
69 * is going to access memory), we need gpu dpa (device physical
70 * address), and there is no need of dma-mapping. This is TBD.
71 *
72 * FIXME: dma-mapping for peer gpu device to access remote gpu's
73 * memory. Add this when you support p2p
74 *
75 * This function allocates the storage of the sg table. It is
76 * caller's responsibility to free it calling sg_free_table.
77 *
78 * Returns 0 if successful; -ENOMEM if fails to allocate memory
79 */
xe_build_sg(struct xe_device * xe,struct hmm_range * range,struct sg_table * st,bool write)80 static int xe_build_sg(struct xe_device *xe, struct hmm_range *range,
81 struct sg_table *st, bool write)
82 {
83 struct device *dev = xe->drm.dev;
84 struct page **pages;
85 u64 i, npages;
86 int ret;
87
88 npages = xe_npages_in_range(range->start, range->end);
89 pages = kvmalloc_array(npages, sizeof(*pages), GFP_KERNEL);
90 if (!pages)
91 return -ENOMEM;
92
93 for (i = 0; i < npages; i++) {
94 pages[i] = hmm_pfn_to_page(range->hmm_pfns[i]);
95 xe_assert(xe, !is_device_private_page(pages[i]));
96 }
97
98 ret = sg_alloc_table_from_pages_segment(st, pages, npages, 0, npages << PAGE_SHIFT,
99 xe_sg_segment_size(dev), GFP_KERNEL);
100 if (ret)
101 goto free_pages;
102
103 ret = dma_map_sgtable(dev, st, write ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE,
104 DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_NO_KERNEL_MAPPING);
105 if (ret) {
106 sg_free_table(st);
107 st = NULL;
108 }
109
110 free_pages:
111 kvfree(pages);
112 return ret;
113 }
114
115 /*
116 * xe_hmm_userptr_free_sg() - Free the scatter gather table of userptr
117 *
118 * @uvma: the userptr vma which hold the scatter gather table
119 *
120 * With function xe_userptr_populate_range, we allocate storage of
121 * the userptr sg table. This is a helper function to free this
122 * sg table, and dma unmap the address in the table.
123 */
xe_hmm_userptr_free_sg(struct xe_userptr_vma * uvma)124 void xe_hmm_userptr_free_sg(struct xe_userptr_vma *uvma)
125 {
126 struct xe_userptr *userptr = &uvma->userptr;
127 struct xe_vma *vma = &uvma->vma;
128 bool write = !xe_vma_read_only(vma);
129 struct xe_vm *vm = xe_vma_vm(vma);
130 struct xe_device *xe = vm->xe;
131 struct device *dev = xe->drm.dev;
132
133 xe_assert(xe, userptr->sg);
134 dma_unmap_sgtable(dev, userptr->sg,
135 write ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE, 0);
136
137 sg_free_table(userptr->sg);
138 userptr->sg = NULL;
139 }
140
141 /**
142 * xe_hmm_userptr_populate_range() - Populate physical pages of a virtual
143 * address range
144 *
145 * @uvma: userptr vma which has information of the range to populate.
146 * @is_mm_mmap_locked: True if mmap_read_lock is already acquired by caller.
147 *
148 * This function populate the physical pages of a virtual
149 * address range. The populated physical pages is saved in
150 * userptr's sg table. It is similar to get_user_pages but call
151 * hmm_range_fault.
152 *
153 * This function also read mmu notifier sequence # (
154 * mmu_interval_read_begin), for the purpose of later
155 * comparison (through mmu_interval_read_retry).
156 *
157 * This must be called with mmap read or write lock held.
158 *
159 * This function allocates the storage of the userptr sg table.
160 * It is caller's responsibility to free it calling sg_free_table.
161 *
162 * returns: 0 for succuss; negative error no on failure
163 */
xe_hmm_userptr_populate_range(struct xe_userptr_vma * uvma,bool is_mm_mmap_locked)164 int xe_hmm_userptr_populate_range(struct xe_userptr_vma *uvma,
165 bool is_mm_mmap_locked)
166 {
167 unsigned long timeout =
168 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
169 unsigned long *pfns, flags = HMM_PFN_REQ_FAULT;
170 struct xe_userptr *userptr;
171 struct xe_vma *vma = &uvma->vma;
172 u64 userptr_start = xe_vma_userptr(vma);
173 u64 userptr_end = userptr_start + xe_vma_size(vma);
174 struct xe_vm *vm = xe_vma_vm(vma);
175 struct hmm_range hmm_range;
176 bool write = !xe_vma_read_only(vma);
177 unsigned long notifier_seq;
178 u64 npages;
179 int ret;
180
181 userptr = &uvma->userptr;
182
183 if (is_mm_mmap_locked)
184 mmap_assert_locked(userptr->notifier.mm);
185
186 if (vma->gpuva.flags & XE_VMA_DESTROYED)
187 return 0;
188
189 notifier_seq = mmu_interval_read_begin(&userptr->notifier);
190 if (notifier_seq == userptr->notifier_seq)
191 return 0;
192
193 if (userptr->sg)
194 xe_hmm_userptr_free_sg(uvma);
195
196 npages = xe_npages_in_range(userptr_start, userptr_end);
197 pfns = kvmalloc_array(npages, sizeof(*pfns), GFP_KERNEL);
198 if (unlikely(!pfns))
199 return -ENOMEM;
200
201 if (write)
202 flags |= HMM_PFN_REQ_WRITE;
203
204 if (!mmget_not_zero(userptr->notifier.mm)) {
205 ret = -EFAULT;
206 goto free_pfns;
207 }
208
209 hmm_range.default_flags = flags;
210 hmm_range.hmm_pfns = pfns;
211 hmm_range.notifier = &userptr->notifier;
212 hmm_range.start = userptr_start;
213 hmm_range.end = userptr_end;
214 hmm_range.dev_private_owner = vm->xe;
215
216 while (true) {
217 hmm_range.notifier_seq = mmu_interval_read_begin(&userptr->notifier);
218
219 if (!is_mm_mmap_locked)
220 mmap_read_lock(userptr->notifier.mm);
221
222 ret = hmm_range_fault(&hmm_range);
223
224 if (!is_mm_mmap_locked)
225 mmap_read_unlock(userptr->notifier.mm);
226
227 if (ret == -EBUSY) {
228 if (time_after(jiffies, timeout))
229 break;
230
231 continue;
232 }
233 break;
234 }
235
236 mmput(userptr->notifier.mm);
237
238 if (ret)
239 goto free_pfns;
240
241 ret = xe_build_sg(vm->xe, &hmm_range, &userptr->sgt, write);
242 if (ret)
243 goto free_pfns;
244
245 xe_mark_range_accessed(&hmm_range, write);
246 userptr->sg = &userptr->sgt;
247 userptr->notifier_seq = hmm_range.notifier_seq;
248
249 free_pfns:
250 kvfree(pfns);
251 return ret;
252 }
253
254