1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2024 Intel Corporation 4 */ 5 6 #include <linux/scatterlist.h> 7 #include <linux/mmu_notifier.h> 8 #include <linux/dma-mapping.h> 9 #include <linux/memremap.h> 10 #include <linux/swap.h> 11 #include <linux/hmm.h> 12 #include <linux/mm.h> 13 #include "xe_hmm.h" 14 #include "xe_vm.h" 15 #include "xe_bo.h" 16 17 static u64 xe_npages_in_range(unsigned long start, unsigned long end) 18 { 19 return (end - start) >> PAGE_SHIFT; 20 } 21 22 /* 23 * xe_mark_range_accessed() - mark a range is accessed, so core mm 24 * have such information for memory eviction or write back to 25 * hard disk 26 * 27 * @range: the range to mark 28 * @write: if write to this range, we mark pages in this range 29 * as dirty 30 */ 31 static void xe_mark_range_accessed(struct hmm_range *range, bool write) 32 { 33 struct page *page; 34 u64 i, npages; 35 36 npages = xe_npages_in_range(range->start, range->end); 37 for (i = 0; i < npages; i++) { 38 page = hmm_pfn_to_page(range->hmm_pfns[i]); 39 if (write) 40 set_page_dirty_lock(page); 41 42 mark_page_accessed(page); 43 } 44 } 45 46 /* 47 * xe_build_sg() - build a scatter gather table for all the physical pages/pfn 48 * in a hmm_range. dma-map pages if necessary. dma-address is save in sg table 49 * and will be used to program GPU page table later. 50 * 51 * @xe: the xe device who will access the dma-address in sg table 52 * @range: the hmm range that we build the sg table from. range->hmm_pfns[] 53 * has the pfn numbers of pages that back up this hmm address range. 54 * @st: pointer to the sg table. 55 * @write: whether we write to this range. This decides dma map direction 56 * for system pages. If write we map it bi-diretional; otherwise 57 * DMA_TO_DEVICE 58 * 59 * All the contiguous pfns will be collapsed into one entry in 60 * the scatter gather table. This is for the purpose of efficiently 61 * programming GPU page table. 62 * 63 * The dma_address in the sg table will later be used by GPU to 64 * access memory. So if the memory is system memory, we need to 65 * do a dma-mapping so it can be accessed by GPU/DMA. 66 * 67 * FIXME: This function currently only support pages in system 68 * memory. If the memory is GPU local memory (of the GPU who 69 * is going to access memory), we need gpu dpa (device physical 70 * address), and there is no need of dma-mapping. This is TBD. 71 * 72 * FIXME: dma-mapping for peer gpu device to access remote gpu's 73 * memory. Add this when you support p2p 74 * 75 * This function allocates the storage of the sg table. It is 76 * caller's responsibility to free it calling sg_free_table. 77 * 78 * Returns 0 if successful; -ENOMEM if fails to allocate memory 79 */ 80 static int xe_build_sg(struct xe_device *xe, struct hmm_range *range, 81 struct sg_table *st, bool write) 82 { 83 struct device *dev = xe->drm.dev; 84 struct page **pages; 85 u64 i, npages; 86 int ret; 87 88 npages = xe_npages_in_range(range->start, range->end); 89 pages = kvmalloc_array(npages, sizeof(*pages), GFP_KERNEL); 90 if (!pages) 91 return -ENOMEM; 92 93 for (i = 0; i < npages; i++) { 94 pages[i] = hmm_pfn_to_page(range->hmm_pfns[i]); 95 xe_assert(xe, !is_device_private_page(pages[i])); 96 } 97 98 ret = sg_alloc_table_from_pages_segment(st, pages, npages, 0, npages << PAGE_SHIFT, 99 xe_sg_segment_size(dev), GFP_KERNEL); 100 if (ret) 101 goto free_pages; 102 103 ret = dma_map_sgtable(dev, st, write ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE, 104 DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_NO_KERNEL_MAPPING); 105 if (ret) { 106 sg_free_table(st); 107 st = NULL; 108 } 109 110 free_pages: 111 kvfree(pages); 112 return ret; 113 } 114 115 /* 116 * xe_hmm_userptr_free_sg() - Free the scatter gather table of userptr 117 * 118 * @uvma: the userptr vma which hold the scatter gather table 119 * 120 * With function xe_userptr_populate_range, we allocate storage of 121 * the userptr sg table. This is a helper function to free this 122 * sg table, and dma unmap the address in the table. 123 */ 124 void xe_hmm_userptr_free_sg(struct xe_userptr_vma *uvma) 125 { 126 struct xe_userptr *userptr = &uvma->userptr; 127 struct xe_vma *vma = &uvma->vma; 128 bool write = !xe_vma_read_only(vma); 129 struct xe_vm *vm = xe_vma_vm(vma); 130 struct xe_device *xe = vm->xe; 131 struct device *dev = xe->drm.dev; 132 133 xe_assert(xe, userptr->sg); 134 dma_unmap_sgtable(dev, userptr->sg, 135 write ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE, 0); 136 137 sg_free_table(userptr->sg); 138 userptr->sg = NULL; 139 } 140 141 /** 142 * xe_hmm_userptr_populate_range() - Populate physical pages of a virtual 143 * address range 144 * 145 * @uvma: userptr vma which has information of the range to populate. 146 * @is_mm_mmap_locked: True if mmap_read_lock is already acquired by caller. 147 * 148 * This function populate the physical pages of a virtual 149 * address range. The populated physical pages is saved in 150 * userptr's sg table. It is similar to get_user_pages but call 151 * hmm_range_fault. 152 * 153 * This function also read mmu notifier sequence # ( 154 * mmu_interval_read_begin), for the purpose of later 155 * comparison (through mmu_interval_read_retry). 156 * 157 * This must be called with mmap read or write lock held. 158 * 159 * This function allocates the storage of the userptr sg table. 160 * It is caller's responsibility to free it calling sg_free_table. 161 * 162 * returns: 0 for succuss; negative error no on failure 163 */ 164 int xe_hmm_userptr_populate_range(struct xe_userptr_vma *uvma, 165 bool is_mm_mmap_locked) 166 { 167 unsigned long timeout = 168 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 169 unsigned long *pfns, flags = HMM_PFN_REQ_FAULT; 170 struct xe_userptr *userptr; 171 struct xe_vma *vma = &uvma->vma; 172 u64 userptr_start = xe_vma_userptr(vma); 173 u64 userptr_end = userptr_start + xe_vma_size(vma); 174 struct xe_vm *vm = xe_vma_vm(vma); 175 struct hmm_range hmm_range; 176 bool write = !xe_vma_read_only(vma); 177 unsigned long notifier_seq; 178 u64 npages; 179 int ret; 180 181 userptr = &uvma->userptr; 182 183 if (is_mm_mmap_locked) 184 mmap_assert_locked(userptr->notifier.mm); 185 186 if (vma->gpuva.flags & XE_VMA_DESTROYED) 187 return 0; 188 189 notifier_seq = mmu_interval_read_begin(&userptr->notifier); 190 if (notifier_seq == userptr->notifier_seq) 191 return 0; 192 193 if (userptr->sg) 194 xe_hmm_userptr_free_sg(uvma); 195 196 npages = xe_npages_in_range(userptr_start, userptr_end); 197 pfns = kvmalloc_array(npages, sizeof(*pfns), GFP_KERNEL); 198 if (unlikely(!pfns)) 199 return -ENOMEM; 200 201 if (write) 202 flags |= HMM_PFN_REQ_WRITE; 203 204 if (!mmget_not_zero(userptr->notifier.mm)) { 205 ret = -EFAULT; 206 goto free_pfns; 207 } 208 209 hmm_range.default_flags = flags; 210 hmm_range.hmm_pfns = pfns; 211 hmm_range.notifier = &userptr->notifier; 212 hmm_range.start = userptr_start; 213 hmm_range.end = userptr_end; 214 hmm_range.dev_private_owner = vm->xe; 215 216 while (true) { 217 hmm_range.notifier_seq = mmu_interval_read_begin(&userptr->notifier); 218 219 if (!is_mm_mmap_locked) 220 mmap_read_lock(userptr->notifier.mm); 221 222 ret = hmm_range_fault(&hmm_range); 223 224 if (!is_mm_mmap_locked) 225 mmap_read_unlock(userptr->notifier.mm); 226 227 if (ret == -EBUSY) { 228 if (time_after(jiffies, timeout)) 229 break; 230 231 continue; 232 } 233 break; 234 } 235 236 mmput(userptr->notifier.mm); 237 238 if (ret) 239 goto free_pfns; 240 241 ret = xe_build_sg(vm->xe, &hmm_range, &userptr->sgt, write); 242 if (ret) 243 goto free_pfns; 244 245 xe_mark_range_accessed(&hmm_range, write); 246 userptr->sg = &userptr->sgt; 247 userptr->notifier_seq = hmm_range.notifier_seq; 248 249 free_pfns: 250 kvfree(pfns); 251 return ret; 252 } 253 254