1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2025, Advanced Micro Devices, Inc.
4 */
5
6 #include <drm/amdxdna_accel.h>
7 #include <drm/drm_device.h>
8 #include <drm/drm_print.h>
9 #include <linux/dma-buf.h>
10 #include <linux/overflow.h>
11 #include <linux/pagemap.h>
12 #include <linux/vmalloc.h>
13
14 #include "amdxdna_pci_drv.h"
15 #include "amdxdna_ubuf.h"
16
17 struct amdxdna_ubuf_priv {
18 struct page **pages;
19 u64 nr_pages;
20 struct mm_struct *mm;
21 };
22
amdxdna_ubuf_map(struct dma_buf_attachment * attach,enum dma_data_direction direction)23 static struct sg_table *amdxdna_ubuf_map(struct dma_buf_attachment *attach,
24 enum dma_data_direction direction)
25 {
26 struct amdxdna_ubuf_priv *ubuf = attach->dmabuf->priv;
27 struct sg_table *sg;
28 int ret;
29
30 sg = kzalloc_obj(*sg);
31 if (!sg)
32 return ERR_PTR(-ENOMEM);
33
34 ret = sg_alloc_table_from_pages(sg, ubuf->pages, ubuf->nr_pages, 0,
35 ubuf->nr_pages << PAGE_SHIFT, GFP_KERNEL);
36 if (ret)
37 goto err_free_sg;
38
39 ret = dma_map_sgtable(attach->dev, sg, direction, 0);
40 if (ret)
41 goto err_free_table;
42
43 return sg;
44
45 err_free_table:
46 sg_free_table(sg);
47 err_free_sg:
48 kfree(sg);
49 return ERR_PTR(ret);
50 }
51
amdxdna_ubuf_unmap(struct dma_buf_attachment * attach,struct sg_table * sg,enum dma_data_direction direction)52 static void amdxdna_ubuf_unmap(struct dma_buf_attachment *attach,
53 struct sg_table *sg,
54 enum dma_data_direction direction)
55 {
56 dma_unmap_sgtable(attach->dev, sg, direction, 0);
57 sg_free_table(sg);
58 kfree(sg);
59 }
60
amdxdna_ubuf_release(struct dma_buf * dbuf)61 static void amdxdna_ubuf_release(struct dma_buf *dbuf)
62 {
63 struct amdxdna_ubuf_priv *ubuf = dbuf->priv;
64
65 unpin_user_pages(ubuf->pages, ubuf->nr_pages);
66 kvfree(ubuf->pages);
67 atomic64_sub(ubuf->nr_pages, &ubuf->mm->pinned_vm);
68 mmdrop(ubuf->mm);
69 kfree(ubuf);
70 }
71
72 static const struct dma_buf_ops amdxdna_ubuf_dmabuf_ops = {
73 .map_dma_buf = amdxdna_ubuf_map,
74 .unmap_dma_buf = amdxdna_ubuf_unmap,
75 .release = amdxdna_ubuf_release,
76 };
77
amdxdna_get_ubuf(struct drm_device * dev,u32 num_entries,void __user * va_entries)78 struct dma_buf *amdxdna_get_ubuf(struct drm_device *dev,
79 u32 num_entries, void __user *va_entries)
80 {
81 struct amdxdna_dev *xdna = to_xdna_dev(dev);
82 unsigned long lock_limit, new_pinned;
83 struct amdxdna_drm_va_entry *va_ent;
84 struct amdxdna_ubuf_priv *ubuf;
85 u32 npages, start = 0;
86 struct dma_buf *dbuf;
87 int i, ret;
88 DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
89
90 if (!can_do_mlock())
91 return ERR_PTR(-EPERM);
92
93 ubuf = kzalloc_obj(*ubuf);
94 if (!ubuf)
95 return ERR_PTR(-ENOMEM);
96
97 ubuf->mm = current->mm;
98 mmgrab(ubuf->mm);
99
100 va_ent = kvzalloc_objs(*va_ent, num_entries);
101 if (!va_ent) {
102 ret = -ENOMEM;
103 goto free_ubuf;
104 }
105
106 if (copy_from_user(va_ent, va_entries, sizeof(*va_ent) * num_entries)) {
107 XDNA_DBG(xdna, "Access va entries failed");
108 ret = -EINVAL;
109 goto free_ent;
110 }
111
112 for (i = 0, exp_info.size = 0; i < num_entries; i++) {
113 if (!IS_ALIGNED(va_ent[i].vaddr, PAGE_SIZE) ||
114 !IS_ALIGNED(va_ent[i].len, PAGE_SIZE)) {
115 XDNA_ERR(xdna, "Invalid address or len %llx, %llx",
116 va_ent[i].vaddr, va_ent[i].len);
117 ret = -EINVAL;
118 goto free_ent;
119 }
120
121 if (check_add_overflow(exp_info.size, va_ent[i].len, &exp_info.size)) {
122 ret = -EINVAL;
123 goto free_ent;
124 }
125 }
126
127 ubuf->nr_pages = exp_info.size >> PAGE_SHIFT;
128 lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
129 new_pinned = atomic64_add_return(ubuf->nr_pages, &ubuf->mm->pinned_vm);
130 if (new_pinned > lock_limit && !capable(CAP_IPC_LOCK)) {
131 XDNA_DBG(xdna, "New pin %ld, limit %ld, cap %d",
132 new_pinned, lock_limit, capable(CAP_IPC_LOCK));
133 ret = -ENOMEM;
134 goto sub_pin_cnt;
135 }
136
137 ubuf->pages = kvmalloc_objs(*ubuf->pages, ubuf->nr_pages);
138 if (!ubuf->pages) {
139 ret = -ENOMEM;
140 goto sub_pin_cnt;
141 }
142
143 for (i = 0; i < num_entries; i++) {
144 npages = va_ent[i].len >> PAGE_SHIFT;
145
146 ret = pin_user_pages_fast(va_ent[i].vaddr, npages,
147 FOLL_WRITE | FOLL_LONGTERM,
148 &ubuf->pages[start]);
149 if (ret < 0 || ret != npages) {
150 ret = -ENOMEM;
151 XDNA_ERR(xdna, "Failed to pin pages ret %d", ret);
152 goto destroy_pages;
153 }
154
155 start += ret;
156 }
157
158 exp_info.ops = &amdxdna_ubuf_dmabuf_ops;
159 exp_info.priv = ubuf;
160 exp_info.flags = O_RDWR | O_CLOEXEC;
161
162 dbuf = dma_buf_export(&exp_info);
163 if (IS_ERR(dbuf)) {
164 ret = PTR_ERR(dbuf);
165 goto destroy_pages;
166 }
167 kvfree(va_ent);
168
169 return dbuf;
170
171 destroy_pages:
172 if (start)
173 unpin_user_pages(ubuf->pages, start);
174 kvfree(ubuf->pages);
175 sub_pin_cnt:
176 atomic64_sub(ubuf->nr_pages, &ubuf->mm->pinned_vm);
177 free_ent:
178 kvfree(va_ent);
179 free_ubuf:
180 mmdrop(ubuf->mm);
181 kfree(ubuf);
182 return ERR_PTR(ret);
183 }
184