1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2014 - 2022 Intel Corporation */ 3 #include <linux/device.h> 4 #include <linux/dma-mapping.h> 5 #include <linux/pci.h> 6 #include <linux/scatterlist.h> 7 #include <linux/slab.h> 8 #include <linux/types.h> 9 #include "adf_accel_devices.h" 10 #include "qat_bl.h" 11 #include "qat_crypto.h" 12 13 void qat_bl_free_bufl(struct adf_accel_dev *accel_dev, 14 struct qat_request_buffs *buf) 15 { 16 struct device *dev = &GET_DEV(accel_dev); 17 struct qat_alg_buf_list *bl = buf->bl; 18 struct qat_alg_buf_list *blout = buf->blout; 19 dma_addr_t blp = buf->blp; 20 dma_addr_t blpout = buf->bloutp; 21 size_t sz = buf->sz; 22 size_t sz_out = buf->sz_out; 23 int bl_dma_dir; 24 int i; 25 26 bl_dma_dir = blp != blpout ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; 27 28 for (i = 0; i < bl->num_bufs; i++) 29 dma_unmap_single(dev, bl->buffers[i].addr, 30 bl->buffers[i].len, bl_dma_dir); 31 32 dma_unmap_single(dev, blp, sz, DMA_TO_DEVICE); 33 34 if (!buf->sgl_src_valid) 35 kfree(bl); 36 37 if (blp != blpout) { 38 for (i = 0; i < blout->num_mapped_bufs; i++) { 39 dma_unmap_single(dev, blout->buffers[i].addr, 40 blout->buffers[i].len, 41 DMA_FROM_DEVICE); 42 } 43 dma_unmap_single(dev, blpout, sz_out, DMA_TO_DEVICE); 44 45 if (!buf->sgl_dst_valid) 46 kfree(blout); 47 } 48 } 49 50 static int __qat_bl_sgl_to_bufl(struct adf_accel_dev *accel_dev, 51 struct scatterlist *sgl, 52 struct scatterlist *sglout, 53 struct qat_request_buffs *buf, 54 dma_addr_t extra_dst_buff, 55 size_t sz_extra_dst_buff, 56 unsigned int sskip, 57 unsigned int dskip, 58 gfp_t flags) 59 { 60 struct device *dev = &GET_DEV(accel_dev); 61 int i, sg_nctr = 0; 62 int n = sg_nents(sgl); 63 struct qat_alg_buf_list *bufl; 64 struct qat_alg_buf_list *buflout = NULL; 65 dma_addr_t blp = DMA_MAPPING_ERROR; 66 dma_addr_t bloutp = DMA_MAPPING_ERROR; 67 struct scatterlist *sg; 68 size_t sz_out, sz = struct_size(bufl, buffers, n); 69 int node = dev_to_node(&GET_DEV(accel_dev)); 70 unsigned int left; 71 int bufl_dma_dir; 72 73 if (unlikely(!n)) 74 return -EINVAL; 75 76 buf->sgl_src_valid = false; 77 buf->sgl_dst_valid = false; 78 79 if (n > QAT_MAX_BUFF_DESC) { 80 bufl = kzalloc_node(sz, flags, node); 81 if (unlikely(!bufl)) 82 return -ENOMEM; 83 } else { 84 bufl = container_of(&buf->sgl_src.sgl_hdr, 85 struct qat_alg_buf_list, hdr); 86 memset(bufl, 0, sizeof(struct qat_alg_buf_list)); 87 buf->sgl_src_valid = true; 88 } 89 90 bufl_dma_dir = sgl != sglout ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; 91 92 for (i = 0; i < n; i++) 93 bufl->buffers[i].addr = DMA_MAPPING_ERROR; 94 95 left = sskip; 96 97 for_each_sg(sgl, sg, n, i) { 98 int y = sg_nctr; 99 100 if (!sg->length) 101 continue; 102 103 if (left >= sg->length) { 104 left -= sg->length; 105 continue; 106 } 107 bufl->buffers[y].addr = dma_map_single(dev, sg_virt(sg) + left, 108 sg->length - left, 109 bufl_dma_dir); 110 bufl->buffers[y].len = sg->length; 111 if (unlikely(dma_mapping_error(dev, bufl->buffers[y].addr))) 112 goto err_in; 113 sg_nctr++; 114 if (left) { 115 bufl->buffers[y].len -= left; 116 left = 0; 117 } 118 } 119 bufl->num_bufs = sg_nctr; 120 blp = dma_map_single(dev, bufl, sz, DMA_TO_DEVICE); 121 if (unlikely(dma_mapping_error(dev, blp))) 122 goto err_in; 123 buf->bl = bufl; 124 buf->blp = blp; 125 buf->sz = sz; 126 /* Handle out of place operation */ 127 if (sgl != sglout) { 128 struct qat_alg_buf *buffers; 129 int extra_buff = extra_dst_buff ? 1 : 0; 130 int n_sglout = sg_nents(sglout); 131 132 n = n_sglout + extra_buff; 133 sz_out = struct_size(buflout, buffers, n); 134 left = dskip; 135 136 sg_nctr = 0; 137 138 if (n > QAT_MAX_BUFF_DESC) { 139 buflout = kzalloc_node(sz_out, flags, node); 140 if (unlikely(!buflout)) 141 goto err_in; 142 } else { 143 buflout = container_of(&buf->sgl_dst.sgl_hdr, 144 struct qat_alg_buf_list, hdr); 145 memset(buflout, 0, sizeof(struct qat_alg_buf_list)); 146 buf->sgl_dst_valid = true; 147 } 148 149 buffers = buflout->buffers; 150 for (i = 0; i < n; i++) 151 buffers[i].addr = DMA_MAPPING_ERROR; 152 153 for_each_sg(sglout, sg, n_sglout, i) { 154 int y = sg_nctr; 155 156 if (!sg->length) 157 continue; 158 159 if (left >= sg->length) { 160 left -= sg->length; 161 continue; 162 } 163 buffers[y].addr = dma_map_single(dev, sg_virt(sg) + left, 164 sg->length - left, 165 DMA_FROM_DEVICE); 166 if (unlikely(dma_mapping_error(dev, buffers[y].addr))) 167 goto err_out; 168 buffers[y].len = sg->length; 169 sg_nctr++; 170 if (left) { 171 buffers[y].len -= left; 172 left = 0; 173 } 174 } 175 if (extra_buff) { 176 buffers[sg_nctr].addr = extra_dst_buff; 177 buffers[sg_nctr].len = sz_extra_dst_buff; 178 } 179 180 buflout->num_bufs = sg_nctr; 181 buflout->num_bufs += extra_buff; 182 buflout->num_mapped_bufs = sg_nctr; 183 bloutp = dma_map_single(dev, buflout, sz_out, DMA_TO_DEVICE); 184 if (unlikely(dma_mapping_error(dev, bloutp))) 185 goto err_out; 186 buf->blout = buflout; 187 buf->bloutp = bloutp; 188 buf->sz_out = sz_out; 189 } else { 190 /* Otherwise set the src and dst to the same address */ 191 buf->bloutp = buf->blp; 192 buf->sz_out = 0; 193 } 194 return 0; 195 196 err_out: 197 if (!dma_mapping_error(dev, bloutp)) 198 dma_unmap_single(dev, bloutp, sz_out, DMA_TO_DEVICE); 199 200 n = sg_nents(sglout); 201 for (i = 0; i < n; i++) { 202 if (buflout->buffers[i].addr == extra_dst_buff) 203 break; 204 if (!dma_mapping_error(dev, buflout->buffers[i].addr)) 205 dma_unmap_single(dev, buflout->buffers[i].addr, 206 buflout->buffers[i].len, 207 DMA_FROM_DEVICE); 208 } 209 210 if (!buf->sgl_dst_valid) 211 kfree(buflout); 212 213 err_in: 214 if (!dma_mapping_error(dev, blp)) 215 dma_unmap_single(dev, blp, sz, DMA_TO_DEVICE); 216 217 n = sg_nents(sgl); 218 for (i = 0; i < n; i++) 219 if (!dma_mapping_error(dev, bufl->buffers[i].addr)) 220 dma_unmap_single(dev, bufl->buffers[i].addr, 221 bufl->buffers[i].len, 222 bufl_dma_dir); 223 224 if (!buf->sgl_src_valid) 225 kfree(bufl); 226 227 dev_err(dev, "Failed to map buf for dma\n"); 228 return -ENOMEM; 229 } 230 231 int qat_bl_sgl_to_bufl(struct adf_accel_dev *accel_dev, 232 struct scatterlist *sgl, 233 struct scatterlist *sglout, 234 struct qat_request_buffs *buf, 235 struct qat_sgl_to_bufl_params *params, 236 gfp_t flags) 237 { 238 dma_addr_t extra_dst_buff = 0; 239 size_t sz_extra_dst_buff = 0; 240 unsigned int sskip = 0; 241 unsigned int dskip = 0; 242 243 if (params) { 244 extra_dst_buff = params->extra_dst_buff; 245 sz_extra_dst_buff = params->sz_extra_dst_buff; 246 sskip = params->sskip; 247 dskip = params->dskip; 248 } 249 250 return __qat_bl_sgl_to_bufl(accel_dev, sgl, sglout, buf, 251 extra_dst_buff, sz_extra_dst_buff, 252 sskip, dskip, flags); 253 } 254 255 static void qat_bl_sgl_unmap(struct adf_accel_dev *accel_dev, 256 struct qat_alg_buf_list *bl) 257 { 258 struct device *dev = &GET_DEV(accel_dev); 259 int n = bl->num_bufs; 260 int i; 261 262 for (i = 0; i < n; i++) 263 if (!dma_mapping_error(dev, bl->buffers[i].addr)) 264 dma_unmap_single(dev, bl->buffers[i].addr, 265 bl->buffers[i].len, DMA_FROM_DEVICE); 266 } 267 268 static int qat_bl_sgl_map(struct adf_accel_dev *accel_dev, 269 struct scatterlist *sgl, 270 struct qat_alg_buf_list **bl) 271 { 272 struct device *dev = &GET_DEV(accel_dev); 273 struct qat_alg_buf_list *bufl; 274 int node = dev_to_node(dev); 275 struct scatterlist *sg; 276 int n, i, sg_nctr; 277 size_t sz; 278 279 n = sg_nents(sgl); 280 sz = struct_size(bufl, buffers, n); 281 bufl = kzalloc_node(sz, GFP_KERNEL, node); 282 if (unlikely(!bufl)) 283 return -ENOMEM; 284 285 for (i = 0; i < n; i++) 286 bufl->buffers[i].addr = DMA_MAPPING_ERROR; 287 288 sg_nctr = 0; 289 for_each_sg(sgl, sg, n, i) { 290 int y = sg_nctr; 291 292 if (!sg->length) 293 continue; 294 295 bufl->buffers[y].addr = dma_map_single(dev, sg_virt(sg), 296 sg->length, 297 DMA_FROM_DEVICE); 298 bufl->buffers[y].len = sg->length; 299 if (unlikely(dma_mapping_error(dev, bufl->buffers[y].addr))) 300 goto err_map; 301 sg_nctr++; 302 } 303 bufl->num_bufs = sg_nctr; 304 bufl->num_mapped_bufs = sg_nctr; 305 306 *bl = bufl; 307 308 return 0; 309 310 err_map: 311 for (i = 0; i < n; i++) 312 if (!dma_mapping_error(dev, bufl->buffers[i].addr)) 313 dma_unmap_single(dev, bufl->buffers[i].addr, 314 bufl->buffers[i].len, 315 DMA_FROM_DEVICE); 316 kfree(bufl); 317 *bl = NULL; 318 319 return -ENOMEM; 320 } 321 322 static void qat_bl_sgl_free_unmap(struct adf_accel_dev *accel_dev, 323 struct scatterlist *sgl, 324 struct qat_alg_buf_list *bl, 325 bool free_bl) 326 { 327 if (bl) { 328 qat_bl_sgl_unmap(accel_dev, bl); 329 330 if (free_bl) 331 kfree(bl); 332 } 333 if (sgl) 334 sgl_free(sgl); 335 } 336 337 static int qat_bl_sgl_alloc_map(struct adf_accel_dev *accel_dev, 338 struct scatterlist **sgl, 339 struct qat_alg_buf_list **bl, 340 unsigned int dlen, 341 gfp_t gfp) 342 { 343 struct scatterlist *dst; 344 int ret; 345 346 dst = sgl_alloc(dlen, gfp, NULL); 347 if (!dst) { 348 dev_err(&GET_DEV(accel_dev), "sg_alloc failed\n"); 349 return -ENOMEM; 350 } 351 352 ret = qat_bl_sgl_map(accel_dev, dst, bl); 353 if (ret) 354 goto err; 355 356 *sgl = dst; 357 358 return 0; 359 360 err: 361 sgl_free(dst); 362 *sgl = NULL; 363 return ret; 364 } 365 366 int qat_bl_realloc_map_new_dst(struct adf_accel_dev *accel_dev, 367 struct scatterlist **sg, 368 unsigned int dlen, 369 struct qat_request_buffs *qat_bufs, 370 gfp_t gfp) 371 { 372 struct device *dev = &GET_DEV(accel_dev); 373 dma_addr_t new_blp = DMA_MAPPING_ERROR; 374 struct qat_alg_buf_list *new_bl; 375 struct scatterlist *new_sg; 376 size_t new_bl_size; 377 int ret; 378 379 ret = qat_bl_sgl_alloc_map(accel_dev, &new_sg, &new_bl, dlen, gfp); 380 if (ret) 381 return ret; 382 383 new_bl_size = struct_size(new_bl, buffers, new_bl->num_bufs); 384 385 /* Map new firmware SGL descriptor */ 386 new_blp = dma_map_single(dev, new_bl, new_bl_size, DMA_TO_DEVICE); 387 if (unlikely(dma_mapping_error(dev, new_blp))) 388 goto err; 389 390 /* Unmap old firmware SGL descriptor */ 391 dma_unmap_single(dev, qat_bufs->bloutp, qat_bufs->sz_out, DMA_TO_DEVICE); 392 393 /* Free and unmap old scatterlist */ 394 qat_bl_sgl_free_unmap(accel_dev, *sg, qat_bufs->blout, 395 !qat_bufs->sgl_dst_valid); 396 397 qat_bufs->sgl_dst_valid = false; 398 qat_bufs->blout = new_bl; 399 qat_bufs->bloutp = new_blp; 400 qat_bufs->sz_out = new_bl_size; 401 402 *sg = new_sg; 403 404 return 0; 405 err: 406 qat_bl_sgl_free_unmap(accel_dev, new_sg, new_bl, true); 407 408 if (!dma_mapping_error(dev, new_blp)) 409 dma_unmap_single(dev, new_blp, new_bl_size, DMA_TO_DEVICE); 410 411 return -ENOMEM; 412 } 413