1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright 2018-2020 Broadcom. 4 */ 5 #include <linux/dma-mapping.h> 6 #include <linux/mm.h> 7 #include <linux/pagemap.h> 8 #include <linux/pgtable.h> 9 #include <linux/vmalloc.h> 10 11 #include <asm/page.h> 12 #include <linux/unaligned.h> 13 14 #include <uapi/linux/misc/bcm_vk.h> 15 16 #include "bcm_vk.h" 17 #include "bcm_vk_msg.h" 18 #include "bcm_vk_sg.h" 19 20 /* 21 * Valkyrie has a hardware limitation of 16M transfer size. 22 * So limit the SGL chunks to 16M. 23 */ 24 #define BCM_VK_MAX_SGL_CHUNK SZ_16M 25 26 static int bcm_vk_dma_alloc(struct device *dev, 27 struct bcm_vk_dma *dma, 28 int dir, 29 struct _vk_data *vkdata); 30 static int bcm_vk_dma_free(struct device *dev, struct bcm_vk_dma *dma); 31 32 /* Uncomment to dump SGLIST */ 33 /* #define BCM_VK_DUMP_SGLIST */ 34 35 static int bcm_vk_dma_alloc(struct device *dev, 36 struct bcm_vk_dma *dma, 37 int direction, 38 struct _vk_data *vkdata) 39 { 40 dma_addr_t addr, sg_addr; 41 int err; 42 int i; 43 int offset; 44 u32 size; 45 u32 remaining_size; 46 u32 transfer_size; 47 u64 data; 48 unsigned long first, last; 49 struct _vk_data *sgdata; 50 51 /* Get 64-bit user address */ 52 data = get_unaligned(&vkdata->address); 53 54 /* offset into first page */ 55 offset = offset_in_page(data); 56 57 /* Calculate number of pages */ 58 first = (data & PAGE_MASK) >> PAGE_SHIFT; 59 last = ((data + vkdata->size - 1) & PAGE_MASK) >> PAGE_SHIFT; 60 dma->nr_pages = last - first + 1; 61 62 /* Allocate DMA pages */ 63 dma->pages = kmalloc_array(dma->nr_pages, 64 sizeof(struct page *), 65 GFP_KERNEL); 66 if (!dma->pages) 67 return -ENOMEM; 68 69 dev_dbg(dev, "Alloc DMA Pages [0x%llx+0x%x => %d pages]\n", 70 data, vkdata->size, dma->nr_pages); 71 72 dma->direction = direction; 73 74 /* Get user pages into memory */ 75 err = get_user_pages_fast(data & PAGE_MASK, 76 dma->nr_pages, 77 direction == DMA_FROM_DEVICE, 78 dma->pages); 79 if (err != dma->nr_pages) { 80 dma->nr_pages = (err >= 0) ? err : 0; 81 dev_err(dev, "get_user_pages_fast, err=%d [%d]\n", 82 err, dma->nr_pages); 83 return err < 0 ? err : -EINVAL; 84 } 85 86 /* Max size of sg list is 1 per mapped page + fields at start */ 87 dma->sglen = (dma->nr_pages * sizeof(*sgdata)) + 88 (sizeof(u32) * SGLIST_VKDATA_START); 89 90 /* Allocate sglist */ 91 dma->sglist = dma_alloc_coherent(dev, 92 dma->sglen, 93 &dma->handle, 94 GFP_KERNEL); 95 if (!dma->sglist) 96 return -ENOMEM; 97 98 dma->sglist[SGLIST_NUM_SG] = 0; 99 dma->sglist[SGLIST_TOTALSIZE] = vkdata->size; 100 remaining_size = vkdata->size; 101 sgdata = (struct _vk_data *)&dma->sglist[SGLIST_VKDATA_START]; 102 103 /* Map all pages into DMA */ 104 size = min_t(size_t, PAGE_SIZE - offset, remaining_size); 105 remaining_size -= size; 106 sg_addr = dma_map_page(dev, 107 dma->pages[0], 108 offset, 109 size, 110 dma->direction); 111 transfer_size = size; 112 if (unlikely(dma_mapping_error(dev, sg_addr))) { 113 __free_page(dma->pages[0]); 114 return -EIO; 115 } 116 117 for (i = 1; i < dma->nr_pages; i++) { 118 size = min_t(size_t, PAGE_SIZE, remaining_size); 119 remaining_size -= size; 120 addr = dma_map_page(dev, 121 dma->pages[i], 122 0, 123 size, 124 dma->direction); 125 if (unlikely(dma_mapping_error(dev, addr))) { 126 __free_page(dma->pages[i]); 127 return -EIO; 128 } 129 130 /* 131 * Compress SG list entry when pages are contiguous 132 * and transfer size less or equal to BCM_VK_MAX_SGL_CHUNK 133 */ 134 if ((addr == (sg_addr + transfer_size)) && 135 ((transfer_size + size) <= BCM_VK_MAX_SGL_CHUNK)) { 136 /* pages are contiguous, add to same sg entry */ 137 transfer_size += size; 138 } else { 139 /* pages are not contiguous, write sg entry */ 140 sgdata->size = transfer_size; 141 put_unaligned(sg_addr, (u64 *)&sgdata->address); 142 dma->sglist[SGLIST_NUM_SG]++; 143 144 /* start new sg entry */ 145 sgdata++; 146 sg_addr = addr; 147 transfer_size = size; 148 } 149 } 150 /* Write last sg list entry */ 151 sgdata->size = transfer_size; 152 put_unaligned(sg_addr, (u64 *)&sgdata->address); 153 dma->sglist[SGLIST_NUM_SG]++; 154 155 /* Update pointers and size field to point to sglist */ 156 put_unaligned((u64)dma->handle, &vkdata->address); 157 vkdata->size = (dma->sglist[SGLIST_NUM_SG] * sizeof(*sgdata)) + 158 (sizeof(u32) * SGLIST_VKDATA_START); 159 160 #ifdef BCM_VK_DUMP_SGLIST 161 dev_dbg(dev, 162 "sgl 0x%llx handle 0x%llx, sglen: 0x%x sgsize: 0x%x\n", 163 (u64)dma->sglist, 164 dma->handle, 165 dma->sglen, 166 vkdata->size); 167 for (i = 0; i < vkdata->size / sizeof(u32); i++) 168 dev_dbg(dev, "i:0x%x 0x%x\n", i, dma->sglist[i]); 169 #endif 170 171 return 0; 172 } 173 174 int bcm_vk_sg_alloc(struct device *dev, 175 struct bcm_vk_dma *dma, 176 int dir, 177 struct _vk_data *vkdata, 178 int num) 179 { 180 int i; 181 int rc = -EINVAL; 182 183 /* Convert user addresses to DMA SG List */ 184 for (i = 0; i < num; i++) { 185 if (vkdata[i].size && vkdata[i].address) { 186 /* 187 * If both size and address are non-zero 188 * then DMA alloc. 189 */ 190 rc = bcm_vk_dma_alloc(dev, 191 &dma[i], 192 dir, 193 &vkdata[i]); 194 } else if (vkdata[i].size || 195 vkdata[i].address) { 196 /* 197 * If one of size and address are zero 198 * there is a problem. 199 */ 200 dev_err(dev, 201 "Invalid vkdata %x 0x%x 0x%llx\n", 202 i, vkdata[i].size, vkdata[i].address); 203 rc = -EINVAL; 204 } else { 205 /* 206 * If size and address are both zero 207 * don't convert, but return success. 208 */ 209 rc = 0; 210 } 211 212 if (rc) 213 goto fail_alloc; 214 } 215 return rc; 216 217 fail_alloc: 218 while (i > 0) { 219 i--; 220 if (dma[i].sglist) 221 bcm_vk_dma_free(dev, &dma[i]); 222 } 223 return rc; 224 } 225 226 static int bcm_vk_dma_free(struct device *dev, struct bcm_vk_dma *dma) 227 { 228 dma_addr_t addr; 229 int i; 230 int num_sg; 231 u32 size; 232 struct _vk_data *vkdata; 233 234 dev_dbg(dev, "free sglist=%p sglen=0x%x\n", dma->sglist, dma->sglen); 235 236 /* Unmap all pages in the sglist */ 237 num_sg = dma->sglist[SGLIST_NUM_SG]; 238 vkdata = (struct _vk_data *)&dma->sglist[SGLIST_VKDATA_START]; 239 for (i = 0; i < num_sg; i++) { 240 size = vkdata[i].size; 241 addr = get_unaligned(&vkdata[i].address); 242 243 dma_unmap_page(dev, addr, size, dma->direction); 244 } 245 246 /* Free allocated sglist */ 247 dma_free_coherent(dev, dma->sglen, dma->sglist, dma->handle); 248 249 /* Release lock on all pages */ 250 for (i = 0; i < dma->nr_pages; i++) 251 put_page(dma->pages[i]); 252 253 /* Free allocated dma pages */ 254 kfree(dma->pages); 255 dma->sglist = NULL; 256 257 return 0; 258 } 259 260 int bcm_vk_sg_free(struct device *dev, struct bcm_vk_dma *dma, int num, 261 int *proc_cnt) 262 { 263 int i; 264 265 *proc_cnt = 0; 266 /* Unmap and free all pages and sglists */ 267 for (i = 0; i < num; i++) { 268 if (dma[i].sglist) { 269 bcm_vk_dma_free(dev, &dma[i]); 270 *proc_cnt += 1; 271 } 272 } 273 274 return 0; 275 } 276