1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright 2018-2020 Broadcom. 4 */ 5 #include <linux/dma-mapping.h> 6 #include <linux/mm.h> 7 #include <linux/pagemap.h> 8 #include <linux/pgtable.h> 9 #include <linux/vmalloc.h> 10 11 #include <asm/page.h> 12 #include <linux/unaligned.h> 13 14 #include <uapi/linux/misc/bcm_vk.h> 15 16 #include "bcm_vk.h" 17 #include "bcm_vk_msg.h" 18 #include "bcm_vk_sg.h" 19 20 /* 21 * Valkyrie has a hardware limitation of 16M transfer size. 22 * So limit the SGL chunks to 16M. 23 */ 24 #define BCM_VK_MAX_SGL_CHUNK SZ_16M 25 26 static int bcm_vk_dma_alloc(struct device *dev, 27 struct bcm_vk_dma *dma, 28 int dir, 29 struct _vk_data *vkdata); 30 static int bcm_vk_dma_free(struct device *dev, struct bcm_vk_dma *dma); 31 32 /* Uncomment to dump SGLIST */ 33 /* #define BCM_VK_DUMP_SGLIST */ 34 35 static int bcm_vk_dma_alloc(struct device *dev, 36 struct bcm_vk_dma *dma, 37 int direction, 38 struct _vk_data *vkdata) 39 { 40 dma_addr_t addr, sg_addr; 41 int err; 42 int i; 43 int offset; 44 u32 size; 45 u32 remaining_size; 46 u32 transfer_size; 47 u64 data; 48 unsigned long first, last; 49 struct _vk_data *sgdata; 50 51 /* Get 64-bit user address */ 52 data = get_unaligned(&vkdata->address); 53 54 /* offset into first page */ 55 offset = offset_in_page(data); 56 57 /* Calculate number of pages */ 58 first = (data & PAGE_MASK) >> PAGE_SHIFT; 59 last = ((data + vkdata->size - 1) & PAGE_MASK) >> PAGE_SHIFT; 60 dma->nr_pages = last - first + 1; 61 62 /* Allocate DMA pages */ 63 dma->pages = kmalloc_objs(struct page *, dma->nr_pages); 64 if (!dma->pages) 65 return -ENOMEM; 66 67 dev_dbg(dev, "Alloc DMA Pages [0x%llx+0x%x => %d pages]\n", 68 data, vkdata->size, dma->nr_pages); 69 70 dma->direction = direction; 71 72 /* Get user pages into memory */ 73 err = get_user_pages_fast(data & PAGE_MASK, 74 dma->nr_pages, 75 direction == DMA_FROM_DEVICE, 76 dma->pages); 77 if (err != dma->nr_pages) { 78 dma->nr_pages = (err >= 0) ? err : 0; 79 dev_err(dev, "get_user_pages_fast, err=%d [%d]\n", 80 err, dma->nr_pages); 81 return err < 0 ? err : -EINVAL; 82 } 83 84 /* Max size of sg list is 1 per mapped page + fields at start */ 85 dma->sglen = (dma->nr_pages * sizeof(*sgdata)) + 86 (sizeof(u32) * SGLIST_VKDATA_START); 87 88 /* Allocate sglist */ 89 dma->sglist = dma_alloc_coherent(dev, 90 dma->sglen, 91 &dma->handle, 92 GFP_KERNEL); 93 if (!dma->sglist) 94 return -ENOMEM; 95 96 dma->sglist[SGLIST_NUM_SG] = 0; 97 dma->sglist[SGLIST_TOTALSIZE] = vkdata->size; 98 remaining_size = vkdata->size; 99 sgdata = (struct _vk_data *)&dma->sglist[SGLIST_VKDATA_START]; 100 101 /* Map all pages into DMA */ 102 size = min_t(size_t, PAGE_SIZE - offset, remaining_size); 103 remaining_size -= size; 104 sg_addr = dma_map_page(dev, 105 dma->pages[0], 106 offset, 107 size, 108 dma->direction); 109 transfer_size = size; 110 if (unlikely(dma_mapping_error(dev, sg_addr))) { 111 __free_page(dma->pages[0]); 112 return -EIO; 113 } 114 115 for (i = 1; i < dma->nr_pages; i++) { 116 size = min_t(size_t, PAGE_SIZE, remaining_size); 117 remaining_size -= size; 118 addr = dma_map_page(dev, 119 dma->pages[i], 120 0, 121 size, 122 dma->direction); 123 if (unlikely(dma_mapping_error(dev, addr))) { 124 __free_page(dma->pages[i]); 125 return -EIO; 126 } 127 128 /* 129 * Compress SG list entry when pages are contiguous 130 * and transfer size less or equal to BCM_VK_MAX_SGL_CHUNK 131 */ 132 if ((addr == (sg_addr + transfer_size)) && 133 ((transfer_size + size) <= BCM_VK_MAX_SGL_CHUNK)) { 134 /* pages are contiguous, add to same sg entry */ 135 transfer_size += size; 136 } else { 137 /* pages are not contiguous, write sg entry */ 138 sgdata->size = transfer_size; 139 put_unaligned(sg_addr, (u64 *)&sgdata->address); 140 dma->sglist[SGLIST_NUM_SG]++; 141 142 /* start new sg entry */ 143 sgdata++; 144 sg_addr = addr; 145 transfer_size = size; 146 } 147 } 148 /* Write last sg list entry */ 149 sgdata->size = transfer_size; 150 put_unaligned(sg_addr, (u64 *)&sgdata->address); 151 dma->sglist[SGLIST_NUM_SG]++; 152 153 /* Update pointers and size field to point to sglist */ 154 put_unaligned((u64)dma->handle, &vkdata->address); 155 vkdata->size = (dma->sglist[SGLIST_NUM_SG] * sizeof(*sgdata)) + 156 (sizeof(u32) * SGLIST_VKDATA_START); 157 158 #ifdef BCM_VK_DUMP_SGLIST 159 dev_dbg(dev, 160 "sgl 0x%llx handle 0x%llx, sglen: 0x%x sgsize: 0x%x\n", 161 (u64)dma->sglist, 162 dma->handle, 163 dma->sglen, 164 vkdata->size); 165 for (i = 0; i < vkdata->size / sizeof(u32); i++) 166 dev_dbg(dev, "i:0x%x 0x%x\n", i, dma->sglist[i]); 167 #endif 168 169 return 0; 170 } 171 172 int bcm_vk_sg_alloc(struct device *dev, 173 struct bcm_vk_dma *dma, 174 int dir, 175 struct _vk_data *vkdata, 176 int num) 177 { 178 int i; 179 int rc = -EINVAL; 180 181 /* Convert user addresses to DMA SG List */ 182 for (i = 0; i < num; i++) { 183 if (vkdata[i].size && vkdata[i].address) { 184 /* 185 * If both size and address are non-zero 186 * then DMA alloc. 187 */ 188 rc = bcm_vk_dma_alloc(dev, 189 &dma[i], 190 dir, 191 &vkdata[i]); 192 } else if (vkdata[i].size || 193 vkdata[i].address) { 194 /* 195 * If one of size and address are zero 196 * there is a problem. 197 */ 198 dev_err(dev, 199 "Invalid vkdata %x 0x%x 0x%llx\n", 200 i, vkdata[i].size, vkdata[i].address); 201 rc = -EINVAL; 202 } else { 203 /* 204 * If size and address are both zero 205 * don't convert, but return success. 206 */ 207 rc = 0; 208 } 209 210 if (rc) 211 goto fail_alloc; 212 } 213 return rc; 214 215 fail_alloc: 216 while (i > 0) { 217 i--; 218 if (dma[i].sglist) 219 bcm_vk_dma_free(dev, &dma[i]); 220 } 221 return rc; 222 } 223 224 static int bcm_vk_dma_free(struct device *dev, struct bcm_vk_dma *dma) 225 { 226 dma_addr_t addr; 227 int i; 228 int num_sg; 229 u32 size; 230 struct _vk_data *vkdata; 231 232 dev_dbg(dev, "free sglist=%p sglen=0x%x\n", dma->sglist, dma->sglen); 233 234 /* Unmap all pages in the sglist */ 235 num_sg = dma->sglist[SGLIST_NUM_SG]; 236 vkdata = (struct _vk_data *)&dma->sglist[SGLIST_VKDATA_START]; 237 for (i = 0; i < num_sg; i++) { 238 size = vkdata[i].size; 239 addr = get_unaligned(&vkdata[i].address); 240 241 dma_unmap_page(dev, addr, size, dma->direction); 242 } 243 244 /* Free allocated sglist */ 245 dma_free_coherent(dev, dma->sglen, dma->sglist, dma->handle); 246 247 /* Release lock on all pages */ 248 for (i = 0; i < dma->nr_pages; i++) 249 put_page(dma->pages[i]); 250 251 /* Free allocated dma pages */ 252 kfree(dma->pages); 253 dma->sglist = NULL; 254 255 return 0; 256 } 257 258 int bcm_vk_sg_free(struct device *dev, struct bcm_vk_dma *dma, int num, 259 int *proc_cnt) 260 { 261 int i; 262 263 *proc_cnt = 0; 264 /* Unmap and free all pages and sglists */ 265 for (i = 0; i < num; i++) { 266 if (dma[i].sglist) { 267 bcm_vk_dma_free(dev, &dma[i]); 268 *proc_cnt += 1; 269 } 270 } 271 272 return 0; 273 } 274