1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2025, Advanced Micro Devices, Inc. 4 */ 5 6 #include <drm/amdxdna_accel.h> 7 #include <drm/drm_device.h> 8 #include <drm/drm_print.h> 9 #include <linux/dma-buf.h> 10 #include <linux/overflow.h> 11 #include <linux/pagemap.h> 12 #include <linux/vmalloc.h> 13 14 #include "amdxdna_pci_drv.h" 15 #include "amdxdna_ubuf.h" 16 17 struct amdxdna_ubuf_priv { 18 struct page **pages; 19 u64 nr_pages; 20 struct mm_struct *mm; 21 }; 22 23 static struct sg_table *amdxdna_ubuf_map(struct dma_buf_attachment *attach, 24 enum dma_data_direction direction) 25 { 26 struct amdxdna_ubuf_priv *ubuf = attach->dmabuf->priv; 27 struct sg_table *sg; 28 int ret; 29 30 sg = kzalloc_obj(*sg); 31 if (!sg) 32 return ERR_PTR(-ENOMEM); 33 34 ret = sg_alloc_table_from_pages(sg, ubuf->pages, ubuf->nr_pages, 0, 35 ubuf->nr_pages << PAGE_SHIFT, GFP_KERNEL); 36 if (ret) 37 goto err_free_sg; 38 39 ret = dma_map_sgtable(attach->dev, sg, direction, 0); 40 if (ret) 41 goto err_free_table; 42 43 return sg; 44 45 err_free_table: 46 sg_free_table(sg); 47 err_free_sg: 48 kfree(sg); 49 return ERR_PTR(ret); 50 } 51 52 static void amdxdna_ubuf_unmap(struct dma_buf_attachment *attach, 53 struct sg_table *sg, 54 enum dma_data_direction direction) 55 { 56 dma_unmap_sgtable(attach->dev, sg, direction, 0); 57 sg_free_table(sg); 58 kfree(sg); 59 } 60 61 static void amdxdna_ubuf_release(struct dma_buf *dbuf) 62 { 63 struct amdxdna_ubuf_priv *ubuf = dbuf->priv; 64 65 unpin_user_pages(ubuf->pages, ubuf->nr_pages); 66 kvfree(ubuf->pages); 67 atomic64_sub(ubuf->nr_pages, &ubuf->mm->pinned_vm); 68 mmdrop(ubuf->mm); 69 kfree(ubuf); 70 } 71 72 static const struct dma_buf_ops amdxdna_ubuf_dmabuf_ops = { 73 .map_dma_buf = amdxdna_ubuf_map, 74 .unmap_dma_buf = amdxdna_ubuf_unmap, 75 .release = amdxdna_ubuf_release, 76 }; 77 78 struct dma_buf *amdxdna_get_ubuf(struct drm_device *dev, 79 u32 num_entries, void __user *va_entries) 80 { 81 struct amdxdna_dev *xdna = to_xdna_dev(dev); 82 unsigned long lock_limit, new_pinned; 83 struct amdxdna_drm_va_entry *va_ent; 84 struct amdxdna_ubuf_priv *ubuf; 85 u32 npages, start = 0; 86 struct dma_buf *dbuf; 87 int i, ret; 88 DEFINE_DMA_BUF_EXPORT_INFO(exp_info); 89 90 if (!can_do_mlock()) 91 return ERR_PTR(-EPERM); 92 93 ubuf = kzalloc_obj(*ubuf); 94 if (!ubuf) 95 return ERR_PTR(-ENOMEM); 96 97 ubuf->mm = current->mm; 98 mmgrab(ubuf->mm); 99 100 va_ent = kvzalloc_objs(*va_ent, num_entries); 101 if (!va_ent) { 102 ret = -ENOMEM; 103 goto free_ubuf; 104 } 105 106 if (copy_from_user(va_ent, va_entries, sizeof(*va_ent) * num_entries)) { 107 XDNA_DBG(xdna, "Access va entries failed"); 108 ret = -EINVAL; 109 goto free_ent; 110 } 111 112 for (i = 0, exp_info.size = 0; i < num_entries; i++) { 113 if (!IS_ALIGNED(va_ent[i].vaddr, PAGE_SIZE) || 114 !IS_ALIGNED(va_ent[i].len, PAGE_SIZE)) { 115 XDNA_ERR(xdna, "Invalid address or len %llx, %llx", 116 va_ent[i].vaddr, va_ent[i].len); 117 ret = -EINVAL; 118 goto free_ent; 119 } 120 121 if (check_add_overflow(exp_info.size, va_ent[i].len, &exp_info.size)) { 122 ret = -EINVAL; 123 goto free_ent; 124 } 125 } 126 127 ubuf->nr_pages = exp_info.size >> PAGE_SHIFT; 128 lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 129 new_pinned = atomic64_add_return(ubuf->nr_pages, &ubuf->mm->pinned_vm); 130 if (new_pinned > lock_limit && !capable(CAP_IPC_LOCK)) { 131 XDNA_DBG(xdna, "New pin %ld, limit %ld, cap %d", 132 new_pinned, lock_limit, capable(CAP_IPC_LOCK)); 133 ret = -ENOMEM; 134 goto sub_pin_cnt; 135 } 136 137 ubuf->pages = kvmalloc_objs(*ubuf->pages, ubuf->nr_pages); 138 if (!ubuf->pages) { 139 ret = -ENOMEM; 140 goto sub_pin_cnt; 141 } 142 143 for (i = 0; i < num_entries; i++) { 144 npages = va_ent[i].len >> PAGE_SHIFT; 145 146 ret = pin_user_pages_fast(va_ent[i].vaddr, npages, 147 FOLL_WRITE | FOLL_LONGTERM, 148 &ubuf->pages[start]); 149 if (ret >= 0) { 150 start += ret; 151 if (ret != npages) { 152 XDNA_ERR(xdna, "Partially pinned pages %d/%u", ret, npages); 153 ret = -ENOMEM; 154 goto destroy_pages; 155 } 156 } else { 157 XDNA_ERR(xdna, "Failed to pin pages ret %d", ret); 158 goto destroy_pages; 159 } 160 } 161 162 exp_info.ops = &amdxdna_ubuf_dmabuf_ops; 163 exp_info.priv = ubuf; 164 exp_info.flags = O_RDWR | O_CLOEXEC; 165 166 dbuf = dma_buf_export(&exp_info); 167 if (IS_ERR(dbuf)) { 168 ret = PTR_ERR(dbuf); 169 goto destroy_pages; 170 } 171 kvfree(va_ent); 172 173 return dbuf; 174 175 destroy_pages: 176 if (start) 177 unpin_user_pages(ubuf->pages, start); 178 kvfree(ubuf->pages); 179 sub_pin_cnt: 180 atomic64_sub(ubuf->nr_pages, &ubuf->mm->pinned_vm); 181 free_ent: 182 kvfree(va_ent); 183 free_ubuf: 184 mmdrop(ubuf->mm); 185 kfree(ubuf); 186 return ERR_PTR(ret); 187 } 188