1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kernel.h> 3 #include <linux/errno.h> 4 #include <linux/err.h> 5 #include <linux/mm.h> 6 #include <linux/slab.h> 7 #include <linux/vmalloc.h> 8 #include <linux/pagemap.h> 9 #include <linux/sched.h> 10 11 #include <media/frame_vector.h> 12 13 /** 14 * get_vaddr_frames() - map virtual addresses to pfns 15 * @start: starting user address 16 * @nr_frames: number of pages / pfns from start to map 17 * @vec: structure which receives pages / pfns of the addresses mapped. 18 * It should have space for at least nr_frames entries. 19 * 20 * This function maps virtual addresses from @start and fills @vec structure 21 * with page frame numbers or page pointers to corresponding pages (choice 22 * depends on the type of the vma underlying the virtual address). If @start 23 * belongs to a normal vma, the function grabs reference to each of the pages 24 * to pin them in memory. If @start belongs to VM_IO | VM_PFNMAP vma, we don't 25 * touch page structures and the caller must make sure pfns aren't reused for 26 * anything else while he is using them. 27 * 28 * The function returns number of pages mapped which may be less than 29 * @nr_frames. In particular we stop mapping if there are more vmas of 30 * different type underlying the specified range of virtual addresses. 31 * When the function isn't able to map a single page, it returns error. 32 * 33 * This function takes care of grabbing mmap_lock as necessary. 34 */ 35 int get_vaddr_frames(unsigned long start, unsigned int nr_frames, 36 struct frame_vector *vec) 37 { 38 struct mm_struct *mm = current->mm; 39 struct vm_area_struct *vma; 40 int ret_pin_user_pages_fast = 0; 41 int ret = 0; 42 int err; 43 44 if (nr_frames == 0) 45 return 0; 46 47 if (WARN_ON_ONCE(nr_frames > vec->nr_allocated)) 48 nr_frames = vec->nr_allocated; 49 50 start = untagged_addr(start); 51 52 ret = pin_user_pages_fast(start, nr_frames, 53 FOLL_FORCE | FOLL_WRITE | FOLL_LONGTERM, 54 (struct page **)(vec->ptrs)); 55 if (ret > 0) { 56 vec->got_ref = true; 57 vec->is_pfns = false; 58 goto out_unlocked; 59 } 60 ret_pin_user_pages_fast = ret; 61 62 mmap_read_lock(mm); 63 vec->got_ref = false; 64 vec->is_pfns = true; 65 ret = 0; 66 do { 67 unsigned long *nums = frame_vector_pfns(vec); 68 69 vma = vma_lookup(mm, start); 70 if (!vma) 71 break; 72 73 while (ret < nr_frames && start + PAGE_SIZE <= vma->vm_end) { 74 err = follow_pfn(vma, start, &nums[ret]); 75 if (err) { 76 if (ret) 77 goto out; 78 // If follow_pfn() returns -EINVAL, then this 79 // is not an IO mapping or a raw PFN mapping. 80 // In that case, return the original error from 81 // pin_user_pages_fast(). Otherwise this 82 // function would return -EINVAL when 83 // pin_user_pages_fast() returned -ENOMEM, 84 // which makes debugging hard. 85 if (err == -EINVAL && ret_pin_user_pages_fast) 86 ret = ret_pin_user_pages_fast; 87 else 88 ret = err; 89 goto out; 90 } 91 start += PAGE_SIZE; 92 ret++; 93 } 94 /* Bail out if VMA doesn't completely cover the tail page. */ 95 if (start < vma->vm_end) 96 break; 97 } while (ret < nr_frames); 98 out: 99 mmap_read_unlock(mm); 100 out_unlocked: 101 if (!ret) 102 ret = -EFAULT; 103 if (ret > 0) 104 vec->nr_frames = ret; 105 return ret; 106 } 107 EXPORT_SYMBOL(get_vaddr_frames); 108 109 /** 110 * put_vaddr_frames() - drop references to pages if get_vaddr_frames() acquired 111 * them 112 * @vec: frame vector to put 113 * 114 * Drop references to pages if get_vaddr_frames() acquired them. We also 115 * invalidate the frame vector so that it is prepared for the next call into 116 * get_vaddr_frames(). 117 */ 118 void put_vaddr_frames(struct frame_vector *vec) 119 { 120 struct page **pages; 121 122 if (!vec->got_ref) 123 goto out; 124 pages = frame_vector_pages(vec); 125 /* 126 * frame_vector_pages() might needed to do a conversion when 127 * get_vaddr_frames() got pages but vec was later converted to pfns. 128 * But it shouldn't really fail to convert pfns back... 129 */ 130 if (WARN_ON(IS_ERR(pages))) 131 goto out; 132 133 unpin_user_pages(pages, vec->nr_frames); 134 vec->got_ref = false; 135 out: 136 vec->nr_frames = 0; 137 } 138 EXPORT_SYMBOL(put_vaddr_frames); 139 140 /** 141 * frame_vector_to_pages - convert frame vector to contain page pointers 142 * @vec: frame vector to convert 143 * 144 * Convert @vec to contain array of page pointers. If the conversion is 145 * successful, return 0. Otherwise return an error. Note that we do not grab 146 * page references for the page structures. 147 */ 148 int frame_vector_to_pages(struct frame_vector *vec) 149 { 150 int i; 151 unsigned long *nums; 152 struct page **pages; 153 154 if (!vec->is_pfns) 155 return 0; 156 nums = frame_vector_pfns(vec); 157 for (i = 0; i < vec->nr_frames; i++) 158 if (!pfn_valid(nums[i])) 159 return -EINVAL; 160 pages = (struct page **)nums; 161 for (i = 0; i < vec->nr_frames; i++) 162 pages[i] = pfn_to_page(nums[i]); 163 vec->is_pfns = false; 164 return 0; 165 } 166 EXPORT_SYMBOL(frame_vector_to_pages); 167 168 /** 169 * frame_vector_to_pfns - convert frame vector to contain pfns 170 * @vec: frame vector to convert 171 * 172 * Convert @vec to contain array of pfns. 173 */ 174 void frame_vector_to_pfns(struct frame_vector *vec) 175 { 176 int i; 177 unsigned long *nums; 178 struct page **pages; 179 180 if (vec->is_pfns) 181 return; 182 pages = (struct page **)(vec->ptrs); 183 nums = (unsigned long *)pages; 184 for (i = 0; i < vec->nr_frames; i++) 185 nums[i] = page_to_pfn(pages[i]); 186 vec->is_pfns = true; 187 } 188 EXPORT_SYMBOL(frame_vector_to_pfns); 189 190 /** 191 * frame_vector_create() - allocate & initialize structure for pinned pfns 192 * @nr_frames: number of pfns slots we should reserve 193 * 194 * Allocate and initialize struct pinned_pfns to be able to hold @nr_pfns 195 * pfns. 196 */ 197 struct frame_vector *frame_vector_create(unsigned int nr_frames) 198 { 199 struct frame_vector *vec; 200 int size = sizeof(struct frame_vector) + sizeof(void *) * nr_frames; 201 202 if (WARN_ON_ONCE(nr_frames == 0)) 203 return NULL; 204 /* 205 * This is absurdly high. It's here just to avoid strange effects when 206 * arithmetics overflows. 207 */ 208 if (WARN_ON_ONCE(nr_frames > INT_MAX / sizeof(void *) / 2)) 209 return NULL; 210 /* 211 * Avoid higher order allocations, use vmalloc instead. It should 212 * be rare anyway. 213 */ 214 vec = kvmalloc(size, GFP_KERNEL); 215 if (!vec) 216 return NULL; 217 vec->nr_allocated = nr_frames; 218 vec->nr_frames = 0; 219 return vec; 220 } 221 EXPORT_SYMBOL(frame_vector_create); 222 223 /** 224 * frame_vector_destroy() - free memory allocated to carry frame vector 225 * @vec: Frame vector to free 226 * 227 * Free structure allocated by frame_vector_create() to carry frames. 228 */ 229 void frame_vector_destroy(struct frame_vector *vec) 230 { 231 /* Make sure put_vaddr_frames() got called properly... */ 232 VM_BUG_ON(vec->nr_frames > 0); 233 kvfree(vec); 234 } 235 EXPORT_SYMBOL(frame_vector_destroy); 236