1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2025, Advanced Micro Devices, Inc. 4 */ 5 6 #include <drm/amdxdna_accel.h> 7 #include <drm/drm_device.h> 8 #include <drm/drm_print.h> 9 #include <linux/dma-buf.h> 10 #include <linux/overflow.h> 11 #include <linux/pagemap.h> 12 #include <linux/vmalloc.h> 13 14 #include "amdxdna_pci_drv.h" 15 #include "amdxdna_ubuf.h" 16 17 struct amdxdna_ubuf_priv { 18 struct page **pages; 19 u64 nr_pages; 20 enum amdxdna_ubuf_flag flags; 21 struct mm_struct *mm; 22 }; 23 24 static struct sg_table *amdxdna_ubuf_map(struct dma_buf_attachment *attach, 25 enum dma_data_direction direction) 26 { 27 struct amdxdna_ubuf_priv *ubuf = attach->dmabuf->priv; 28 struct sg_table *sg; 29 int ret; 30 31 sg = kzalloc_obj(*sg); 32 if (!sg) 33 return ERR_PTR(-ENOMEM); 34 35 ret = sg_alloc_table_from_pages(sg, ubuf->pages, ubuf->nr_pages, 0, 36 ubuf->nr_pages << PAGE_SHIFT, GFP_KERNEL); 37 if (ret) 38 goto err_free_sg; 39 40 if (ubuf->flags & AMDXDNA_UBUF_FLAG_MAP_DMA) { 41 ret = dma_map_sgtable(attach->dev, sg, direction, 0); 42 if (ret) 43 goto err_free_table; 44 } 45 46 return sg; 47 48 err_free_table: 49 sg_free_table(sg); 50 err_free_sg: 51 kfree(sg); 52 return ERR_PTR(ret); 53 } 54 55 static void amdxdna_ubuf_unmap(struct dma_buf_attachment *attach, 56 struct sg_table *sg, 57 enum dma_data_direction direction) 58 { 59 struct amdxdna_ubuf_priv *ubuf = attach->dmabuf->priv; 60 61 if (ubuf->flags & AMDXDNA_UBUF_FLAG_MAP_DMA) 62 dma_unmap_sgtable(attach->dev, sg, direction, 0); 63 64 sg_free_table(sg); 65 kfree(sg); 66 } 67 68 static void amdxdna_ubuf_release(struct dma_buf *dbuf) 69 { 70 struct amdxdna_ubuf_priv *ubuf = dbuf->priv; 71 72 unpin_user_pages(ubuf->pages, ubuf->nr_pages); 73 kvfree(ubuf->pages); 74 atomic64_sub(ubuf->nr_pages, &ubuf->mm->pinned_vm); 75 mmdrop(ubuf->mm); 76 kfree(ubuf); 77 } 78 79 static vm_fault_t amdxdna_ubuf_vm_fault(struct vm_fault *vmf) 80 { 81 struct vm_area_struct *vma = vmf->vma; 82 struct amdxdna_ubuf_priv *ubuf; 83 unsigned long pfn; 84 pgoff_t pgoff; 85 86 ubuf = vma->vm_private_data; 87 pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT; 88 89 pfn = page_to_pfn(ubuf->pages[pgoff]); 90 return vmf_insert_pfn(vma, vmf->address, pfn); 91 } 92 93 static const struct vm_operations_struct amdxdna_ubuf_vm_ops = { 94 .fault = amdxdna_ubuf_vm_fault, 95 }; 96 97 static int amdxdna_ubuf_mmap(struct dma_buf *dbuf, struct vm_area_struct *vma) 98 { 99 struct amdxdna_ubuf_priv *ubuf = dbuf->priv; 100 101 vma->vm_ops = &amdxdna_ubuf_vm_ops; 102 vma->vm_private_data = ubuf; 103 vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); 104 105 return 0; 106 } 107 108 static int amdxdna_ubuf_vmap(struct dma_buf *dbuf, struct iosys_map *map) 109 { 110 struct amdxdna_ubuf_priv *ubuf = dbuf->priv; 111 void *kva; 112 113 kva = vmap(ubuf->pages, ubuf->nr_pages, VM_MAP, PAGE_KERNEL); 114 if (!kva) 115 return -EINVAL; 116 117 iosys_map_set_vaddr(map, kva); 118 return 0; 119 } 120 121 static void amdxdna_ubuf_vunmap(struct dma_buf *dbuf, struct iosys_map *map) 122 { 123 vunmap(map->vaddr); 124 } 125 126 static const struct dma_buf_ops amdxdna_ubuf_dmabuf_ops = { 127 .map_dma_buf = amdxdna_ubuf_map, 128 .unmap_dma_buf = amdxdna_ubuf_unmap, 129 .release = amdxdna_ubuf_release, 130 .mmap = amdxdna_ubuf_mmap, 131 .vmap = amdxdna_ubuf_vmap, 132 .vunmap = amdxdna_ubuf_vunmap, 133 }; 134 135 struct dma_buf *amdxdna_get_ubuf(struct drm_device *dev, 136 enum amdxdna_ubuf_flag flags, 137 u32 num_entries, void __user *va_entries) 138 { 139 struct amdxdna_dev *xdna = to_xdna_dev(dev); 140 unsigned long lock_limit, new_pinned; 141 struct amdxdna_drm_va_entry *va_ent; 142 struct amdxdna_ubuf_priv *ubuf; 143 u32 npages, start = 0; 144 struct dma_buf *dbuf; 145 int i, ret; 146 DEFINE_DMA_BUF_EXPORT_INFO(exp_info); 147 148 if (!can_do_mlock()) 149 return ERR_PTR(-EPERM); 150 151 ubuf = kzalloc_obj(*ubuf); 152 if (!ubuf) 153 return ERR_PTR(-ENOMEM); 154 155 ubuf->flags = flags; 156 ubuf->mm = current->mm; 157 mmgrab(ubuf->mm); 158 159 va_ent = kvzalloc_objs(*va_ent, num_entries); 160 if (!va_ent) { 161 ret = -ENOMEM; 162 goto free_ubuf; 163 } 164 165 if (copy_from_user(va_ent, va_entries, sizeof(*va_ent) * num_entries)) { 166 XDNA_DBG(xdna, "Access va entries failed"); 167 ret = -EINVAL; 168 goto free_ent; 169 } 170 171 for (i = 0, exp_info.size = 0; i < num_entries; i++) { 172 if (!IS_ALIGNED(va_ent[i].vaddr, PAGE_SIZE) || 173 !IS_ALIGNED(va_ent[i].len, PAGE_SIZE)) { 174 XDNA_ERR(xdna, "Invalid address or len %llx, %llx", 175 va_ent[i].vaddr, va_ent[i].len); 176 ret = -EINVAL; 177 goto free_ent; 178 } 179 180 if (check_add_overflow(exp_info.size, va_ent[i].len, &exp_info.size)) { 181 ret = -EINVAL; 182 goto free_ent; 183 } 184 } 185 186 ubuf->nr_pages = exp_info.size >> PAGE_SHIFT; 187 lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 188 new_pinned = atomic64_add_return(ubuf->nr_pages, &ubuf->mm->pinned_vm); 189 if (new_pinned > lock_limit && !capable(CAP_IPC_LOCK)) { 190 XDNA_DBG(xdna, "New pin %ld, limit %ld, cap %d", 191 new_pinned, lock_limit, capable(CAP_IPC_LOCK)); 192 ret = -ENOMEM; 193 goto sub_pin_cnt; 194 } 195 196 ubuf->pages = kvmalloc_objs(*ubuf->pages, ubuf->nr_pages); 197 if (!ubuf->pages) { 198 ret = -ENOMEM; 199 goto sub_pin_cnt; 200 } 201 202 for (i = 0; i < num_entries; i++) { 203 npages = va_ent[i].len >> PAGE_SHIFT; 204 205 ret = pin_user_pages_fast(va_ent[i].vaddr, npages, 206 FOLL_WRITE | FOLL_LONGTERM, 207 &ubuf->pages[start]); 208 if (ret < 0 || ret != npages) { 209 ret = -ENOMEM; 210 XDNA_ERR(xdna, "Failed to pin pages ret %d", ret); 211 goto destroy_pages; 212 } 213 214 start += ret; 215 } 216 217 exp_info.ops = &amdxdna_ubuf_dmabuf_ops; 218 exp_info.priv = ubuf; 219 exp_info.flags = O_RDWR | O_CLOEXEC; 220 221 dbuf = dma_buf_export(&exp_info); 222 if (IS_ERR(dbuf)) { 223 ret = PTR_ERR(dbuf); 224 goto destroy_pages; 225 } 226 kvfree(va_ent); 227 228 return dbuf; 229 230 destroy_pages: 231 if (start) 232 unpin_user_pages(ubuf->pages, start); 233 kvfree(ubuf->pages); 234 sub_pin_cnt: 235 atomic64_sub(ubuf->nr_pages, &ubuf->mm->pinned_vm); 236 free_ent: 237 kvfree(va_ent); 238 free_ubuf: 239 mmdrop(ubuf->mm); 240 kfree(ubuf); 241 return ERR_PTR(ret); 242 } 243