1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright 2018-2020 Broadcom.
4 */
5 #include <linux/dma-mapping.h>
6 #include <linux/mm.h>
7 #include <linux/pagemap.h>
8 #include <linux/pgtable.h>
9 #include <linux/vmalloc.h>
10
11 #include <asm/page.h>
12 #include <linux/unaligned.h>
13
14 #include <uapi/linux/misc/bcm_vk.h>
15
16 #include "bcm_vk.h"
17 #include "bcm_vk_msg.h"
18 #include "bcm_vk_sg.h"
19
20 /*
21 * Valkyrie has a hardware limitation of 16M transfer size.
22 * So limit the SGL chunks to 16M.
23 */
24 #define BCM_VK_MAX_SGL_CHUNK SZ_16M
25
26 static int bcm_vk_dma_alloc(struct device *dev,
27 struct bcm_vk_dma *dma,
28 int dir,
29 struct _vk_data *vkdata);
30 static int bcm_vk_dma_free(struct device *dev, struct bcm_vk_dma *dma);
31
32 /* Uncomment to dump SGLIST */
33 /* #define BCM_VK_DUMP_SGLIST */
34
bcm_vk_dma_alloc(struct device * dev,struct bcm_vk_dma * dma,int direction,struct _vk_data * vkdata)35 static int bcm_vk_dma_alloc(struct device *dev,
36 struct bcm_vk_dma *dma,
37 int direction,
38 struct _vk_data *vkdata)
39 {
40 dma_addr_t addr, sg_addr;
41 int err;
42 int i;
43 int offset;
44 u32 size;
45 u32 remaining_size;
46 u32 transfer_size;
47 u64 data;
48 unsigned long first, last;
49 struct _vk_data *sgdata;
50
51 /* Get 64-bit user address */
52 data = get_unaligned(&vkdata->address);
53
54 /* offset into first page */
55 offset = offset_in_page(data);
56
57 /* Calculate number of pages */
58 first = (data & PAGE_MASK) >> PAGE_SHIFT;
59 last = ((data + vkdata->size - 1) & PAGE_MASK) >> PAGE_SHIFT;
60 dma->nr_pages = last - first + 1;
61
62 /* Allocate DMA pages */
63 dma->pages = kmalloc_objs(struct page *, dma->nr_pages);
64 if (!dma->pages)
65 return -ENOMEM;
66
67 dev_dbg(dev, "Alloc DMA Pages [0x%llx+0x%x => %d pages]\n",
68 data, vkdata->size, dma->nr_pages);
69
70 dma->direction = direction;
71
72 /* Get user pages into memory */
73 err = get_user_pages_fast(data & PAGE_MASK,
74 dma->nr_pages,
75 direction == DMA_FROM_DEVICE,
76 dma->pages);
77 if (err != dma->nr_pages) {
78 dma->nr_pages = (err >= 0) ? err : 0;
79 dev_err(dev, "get_user_pages_fast, err=%d [%d]\n",
80 err, dma->nr_pages);
81 return err < 0 ? err : -EINVAL;
82 }
83
84 /* Max size of sg list is 1 per mapped page + fields at start */
85 dma->sglen = (dma->nr_pages * sizeof(*sgdata)) +
86 (sizeof(u32) * SGLIST_VKDATA_START);
87
88 /* Allocate sglist */
89 dma->sglist = dma_alloc_coherent(dev,
90 dma->sglen,
91 &dma->handle,
92 GFP_KERNEL);
93 if (!dma->sglist)
94 return -ENOMEM;
95
96 dma->sglist[SGLIST_NUM_SG] = 0;
97 dma->sglist[SGLIST_TOTALSIZE] = vkdata->size;
98 remaining_size = vkdata->size;
99 sgdata = (struct _vk_data *)&dma->sglist[SGLIST_VKDATA_START];
100
101 /* Map all pages into DMA */
102 size = min_t(size_t, PAGE_SIZE - offset, remaining_size);
103 remaining_size -= size;
104 sg_addr = dma_map_page(dev,
105 dma->pages[0],
106 offset,
107 size,
108 dma->direction);
109 transfer_size = size;
110 if (unlikely(dma_mapping_error(dev, sg_addr))) {
111 __free_page(dma->pages[0]);
112 return -EIO;
113 }
114
115 for (i = 1; i < dma->nr_pages; i++) {
116 size = min_t(size_t, PAGE_SIZE, remaining_size);
117 remaining_size -= size;
118 addr = dma_map_page(dev,
119 dma->pages[i],
120 0,
121 size,
122 dma->direction);
123 if (unlikely(dma_mapping_error(dev, addr))) {
124 __free_page(dma->pages[i]);
125 return -EIO;
126 }
127
128 /*
129 * Compress SG list entry when pages are contiguous
130 * and transfer size less or equal to BCM_VK_MAX_SGL_CHUNK
131 */
132 if ((addr == (sg_addr + transfer_size)) &&
133 ((transfer_size + size) <= BCM_VK_MAX_SGL_CHUNK)) {
134 /* pages are contiguous, add to same sg entry */
135 transfer_size += size;
136 } else {
137 /* pages are not contiguous, write sg entry */
138 sgdata->size = transfer_size;
139 put_unaligned(sg_addr, (u64 *)&sgdata->address);
140 dma->sglist[SGLIST_NUM_SG]++;
141
142 /* start new sg entry */
143 sgdata++;
144 sg_addr = addr;
145 transfer_size = size;
146 }
147 }
148 /* Write last sg list entry */
149 sgdata->size = transfer_size;
150 put_unaligned(sg_addr, (u64 *)&sgdata->address);
151 dma->sglist[SGLIST_NUM_SG]++;
152
153 /* Update pointers and size field to point to sglist */
154 put_unaligned((u64)dma->handle, &vkdata->address);
155 vkdata->size = (dma->sglist[SGLIST_NUM_SG] * sizeof(*sgdata)) +
156 (sizeof(u32) * SGLIST_VKDATA_START);
157
158 #ifdef BCM_VK_DUMP_SGLIST
159 dev_dbg(dev,
160 "sgl 0x%llx handle 0x%llx, sglen: 0x%x sgsize: 0x%x\n",
161 (u64)dma->sglist,
162 dma->handle,
163 dma->sglen,
164 vkdata->size);
165 for (i = 0; i < vkdata->size / sizeof(u32); i++)
166 dev_dbg(dev, "i:0x%x 0x%x\n", i, dma->sglist[i]);
167 #endif
168
169 return 0;
170 }
171
bcm_vk_sg_alloc(struct device * dev,struct bcm_vk_dma * dma,int dir,struct _vk_data * vkdata,int num)172 int bcm_vk_sg_alloc(struct device *dev,
173 struct bcm_vk_dma *dma,
174 int dir,
175 struct _vk_data *vkdata,
176 int num)
177 {
178 int i;
179 int rc = -EINVAL;
180
181 /* Convert user addresses to DMA SG List */
182 for (i = 0; i < num; i++) {
183 if (vkdata[i].size && vkdata[i].address) {
184 /*
185 * If both size and address are non-zero
186 * then DMA alloc.
187 */
188 rc = bcm_vk_dma_alloc(dev,
189 &dma[i],
190 dir,
191 &vkdata[i]);
192 } else if (vkdata[i].size ||
193 vkdata[i].address) {
194 /*
195 * If one of size and address are zero
196 * there is a problem.
197 */
198 dev_err(dev,
199 "Invalid vkdata %x 0x%x 0x%llx\n",
200 i, vkdata[i].size, vkdata[i].address);
201 rc = -EINVAL;
202 } else {
203 /*
204 * If size and address are both zero
205 * don't convert, but return success.
206 */
207 rc = 0;
208 }
209
210 if (rc)
211 goto fail_alloc;
212 }
213 return rc;
214
215 fail_alloc:
216 while (i > 0) {
217 i--;
218 if (dma[i].sglist)
219 bcm_vk_dma_free(dev, &dma[i]);
220 }
221 return rc;
222 }
223
bcm_vk_dma_free(struct device * dev,struct bcm_vk_dma * dma)224 static int bcm_vk_dma_free(struct device *dev, struct bcm_vk_dma *dma)
225 {
226 dma_addr_t addr;
227 int i;
228 int num_sg;
229 u32 size;
230 struct _vk_data *vkdata;
231
232 dev_dbg(dev, "free sglist=%p sglen=0x%x\n", dma->sglist, dma->sglen);
233
234 /* Unmap all pages in the sglist */
235 num_sg = dma->sglist[SGLIST_NUM_SG];
236 vkdata = (struct _vk_data *)&dma->sglist[SGLIST_VKDATA_START];
237 for (i = 0; i < num_sg; i++) {
238 size = vkdata[i].size;
239 addr = get_unaligned(&vkdata[i].address);
240
241 dma_unmap_page(dev, addr, size, dma->direction);
242 }
243
244 /* Free allocated sglist */
245 dma_free_coherent(dev, dma->sglen, dma->sglist, dma->handle);
246
247 /* Release lock on all pages */
248 for (i = 0; i < dma->nr_pages; i++)
249 put_page(dma->pages[i]);
250
251 /* Free allocated dma pages */
252 kfree(dma->pages);
253 dma->sglist = NULL;
254
255 return 0;
256 }
257
bcm_vk_sg_free(struct device * dev,struct bcm_vk_dma * dma,int num,int * proc_cnt)258 int bcm_vk_sg_free(struct device *dev, struct bcm_vk_dma *dma, int num,
259 int *proc_cnt)
260 {
261 int i;
262
263 *proc_cnt = 0;
264 /* Unmap and free all pages and sglists */
265 for (i = 0; i < num; i++) {
266 if (dma[i].sglist) {
267 bcm_vk_dma_free(dev, &dma[i]);
268 *proc_cnt += 1;
269 }
270 }
271
272 return 0;
273 }
274