1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2019 HiSilicon Limited. */ 3 #include <crypto/internal/acompress.h> 4 #include <linux/bitfield.h> 5 #include <linux/bitmap.h> 6 #include <linux/dma-mapping.h> 7 #include <linux/scatterlist.h> 8 #include "zip.h" 9 10 /* hisi_zip_sqe dw3 */ 11 #define HZIP_BD_STATUS_M GENMASK(7, 0) 12 /* hisi_zip_sqe dw7 */ 13 #define HZIP_IN_SGE_DATA_OFFSET_M GENMASK(23, 0) 14 #define HZIP_SQE_TYPE_M GENMASK(31, 28) 15 /* hisi_zip_sqe dw8 */ 16 #define HZIP_OUT_SGE_DATA_OFFSET_M GENMASK(23, 0) 17 /* hisi_zip_sqe dw9 */ 18 #define HZIP_REQ_TYPE_M GENMASK(7, 0) 19 #define HZIP_ALG_TYPE_ZLIB 0x02 20 #define HZIP_ALG_TYPE_GZIP 0x03 21 #define HZIP_BUF_TYPE_M GENMASK(11, 8) 22 #define HZIP_PBUFFER 0x0 23 #define HZIP_SGL 0x1 24 25 #define HZIP_ZLIB_HEAD_SIZE 2 26 #define HZIP_GZIP_HEAD_SIZE 10 27 28 #define GZIP_HEAD_FHCRC_BIT BIT(1) 29 #define GZIP_HEAD_FEXTRA_BIT BIT(2) 30 #define GZIP_HEAD_FNAME_BIT BIT(3) 31 #define GZIP_HEAD_FCOMMENT_BIT BIT(4) 32 33 #define GZIP_HEAD_FLG_SHIFT 3 34 #define GZIP_HEAD_FEXTRA_SHIFT 10 35 #define GZIP_HEAD_FEXTRA_XLEN 2UL 36 #define GZIP_HEAD_FHCRC_SIZE 2 37 38 #define HZIP_GZIP_HEAD_BUF 256 39 #define HZIP_ALG_PRIORITY 300 40 #define HZIP_SGL_SGE_NR 10 41 42 static const u8 zlib_head[HZIP_ZLIB_HEAD_SIZE] = {0x78, 0x9c}; 43 static const u8 gzip_head[HZIP_GZIP_HEAD_SIZE] = { 44 0x1f, 0x8b, 0x08, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x03 45 }; 46 47 enum hisi_zip_alg_type { 48 HZIP_ALG_TYPE_COMP = 0, 49 HZIP_ALG_TYPE_DECOMP = 1, 50 }; 51 52 enum { 53 HZIP_QPC_COMP, 54 HZIP_QPC_DECOMP, 55 HZIP_CTX_Q_NUM 56 }; 57 58 #define COMP_NAME_TO_TYPE(alg_name) \ 59 (!strcmp((alg_name), "zlib-deflate") ? HZIP_ALG_TYPE_ZLIB : \ 60 !strcmp((alg_name), "gzip") ? HZIP_ALG_TYPE_GZIP : 0) \ 61 62 #define TO_HEAD_SIZE(req_type) \ 63 (((req_type) == HZIP_ALG_TYPE_ZLIB) ? sizeof(zlib_head) : \ 64 ((req_type) == HZIP_ALG_TYPE_GZIP) ? sizeof(gzip_head) : 0) \ 65 66 #define TO_HEAD(req_type) \ 67 (((req_type) == HZIP_ALG_TYPE_ZLIB) ? zlib_head : \ 68 ((req_type) == HZIP_ALG_TYPE_GZIP) ? gzip_head : NULL) \ 69 70 struct hisi_zip_req { 71 struct acomp_req *req; 72 u32 sskip; 73 u32 dskip; 74 struct hisi_acc_hw_sgl *hw_src; 75 struct hisi_acc_hw_sgl *hw_dst; 76 dma_addr_t dma_src; 77 dma_addr_t dma_dst; 78 u16 req_id; 79 }; 80 81 struct hisi_zip_req_q { 82 struct hisi_zip_req *q; 83 unsigned long *req_bitmap; 84 rwlock_t req_lock; 85 u16 size; 86 }; 87 88 struct hisi_zip_qp_ctx { 89 struct hisi_qp *qp; 90 struct hisi_zip_req_q req_q; 91 struct hisi_acc_sgl_pool *sgl_pool; 92 struct hisi_zip *zip_dev; 93 struct hisi_zip_ctx *ctx; 94 }; 95 96 struct hisi_zip_sqe_ops { 97 u8 sqe_type; 98 void (*fill_addr)(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req); 99 void (*fill_buf_size)(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req); 100 void (*fill_buf_type)(struct hisi_zip_sqe *sqe, u8 buf_type); 101 void (*fill_req_type)(struct hisi_zip_sqe *sqe, u8 req_type); 102 void (*fill_tag)(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req); 103 void (*fill_sqe_type)(struct hisi_zip_sqe *sqe, u8 sqe_type); 104 u32 (*get_tag)(struct hisi_zip_sqe *sqe); 105 u32 (*get_status)(struct hisi_zip_sqe *sqe); 106 u32 (*get_dstlen)(struct hisi_zip_sqe *sqe); 107 }; 108 109 struct hisi_zip_ctx { 110 struct hisi_zip_qp_ctx qp_ctx[HZIP_CTX_Q_NUM]; 111 const struct hisi_zip_sqe_ops *ops; 112 }; 113 114 static int sgl_sge_nr_set(const char *val, const struct kernel_param *kp) 115 { 116 int ret; 117 u16 n; 118 119 if (!val) 120 return -EINVAL; 121 122 ret = kstrtou16(val, 10, &n); 123 if (ret || n == 0 || n > HISI_ACC_SGL_SGE_NR_MAX) 124 return -EINVAL; 125 126 return param_set_ushort(val, kp); 127 } 128 129 static const struct kernel_param_ops sgl_sge_nr_ops = { 130 .set = sgl_sge_nr_set, 131 .get = param_get_ushort, 132 }; 133 134 static u16 sgl_sge_nr = HZIP_SGL_SGE_NR; 135 module_param_cb(sgl_sge_nr, &sgl_sge_nr_ops, &sgl_sge_nr, 0444); 136 MODULE_PARM_DESC(sgl_sge_nr, "Number of sge in sgl(1-255)"); 137 138 static u32 get_extra_field_size(const u8 *start) 139 { 140 return *((u16 *)start) + GZIP_HEAD_FEXTRA_XLEN; 141 } 142 143 static u32 get_name_field_size(const u8 *start) 144 { 145 return strlen(start) + 1; 146 } 147 148 static u32 get_comment_field_size(const u8 *start) 149 { 150 return strlen(start) + 1; 151 } 152 153 static u32 __get_gzip_head_size(const u8 *src) 154 { 155 u8 head_flg = *(src + GZIP_HEAD_FLG_SHIFT); 156 u32 size = GZIP_HEAD_FEXTRA_SHIFT; 157 158 if (head_flg & GZIP_HEAD_FEXTRA_BIT) 159 size += get_extra_field_size(src + size); 160 if (head_flg & GZIP_HEAD_FNAME_BIT) 161 size += get_name_field_size(src + size); 162 if (head_flg & GZIP_HEAD_FCOMMENT_BIT) 163 size += get_comment_field_size(src + size); 164 if (head_flg & GZIP_HEAD_FHCRC_BIT) 165 size += GZIP_HEAD_FHCRC_SIZE; 166 167 return size; 168 } 169 170 static u32 __maybe_unused get_gzip_head_size(struct scatterlist *sgl) 171 { 172 char buf[HZIP_GZIP_HEAD_BUF]; 173 174 sg_copy_to_buffer(sgl, sg_nents(sgl), buf, sizeof(buf)); 175 176 return __get_gzip_head_size(buf); 177 } 178 179 static int add_comp_head(struct scatterlist *dst, u8 req_type) 180 { 181 int head_size = TO_HEAD_SIZE(req_type); 182 const u8 *head = TO_HEAD(req_type); 183 int ret; 184 185 ret = sg_copy_from_buffer(dst, sg_nents(dst), head, head_size); 186 if (unlikely(ret != head_size)) { 187 pr_err("the head size of buffer is wrong (%d)!\n", ret); 188 return -ENOMEM; 189 } 190 191 return head_size; 192 } 193 194 static int get_comp_head_size(struct acomp_req *acomp_req, u8 req_type) 195 { 196 if (unlikely(!acomp_req->src || !acomp_req->slen)) 197 return -EINVAL; 198 199 if (unlikely(req_type == HZIP_ALG_TYPE_GZIP && 200 acomp_req->slen < GZIP_HEAD_FEXTRA_SHIFT)) 201 return -EINVAL; 202 203 switch (req_type) { 204 case HZIP_ALG_TYPE_ZLIB: 205 return TO_HEAD_SIZE(HZIP_ALG_TYPE_ZLIB); 206 case HZIP_ALG_TYPE_GZIP: 207 return TO_HEAD_SIZE(HZIP_ALG_TYPE_GZIP); 208 default: 209 pr_err("request type does not support!\n"); 210 return -EINVAL; 211 } 212 } 213 214 static struct hisi_zip_req *hisi_zip_create_req(struct acomp_req *req, 215 struct hisi_zip_qp_ctx *qp_ctx, 216 size_t head_size, bool is_comp) 217 { 218 struct hisi_zip_req_q *req_q = &qp_ctx->req_q; 219 struct hisi_zip_req *q = req_q->q; 220 struct hisi_zip_req *req_cache; 221 int req_id; 222 223 write_lock(&req_q->req_lock); 224 225 req_id = find_first_zero_bit(req_q->req_bitmap, req_q->size); 226 if (req_id >= req_q->size) { 227 write_unlock(&req_q->req_lock); 228 dev_dbg(&qp_ctx->qp->qm->pdev->dev, "req cache is full!\n"); 229 return ERR_PTR(-EAGAIN); 230 } 231 set_bit(req_id, req_q->req_bitmap); 232 233 write_unlock(&req_q->req_lock); 234 235 req_cache = q + req_id; 236 req_cache->req_id = req_id; 237 req_cache->req = req; 238 239 if (is_comp) { 240 req_cache->sskip = 0; 241 req_cache->dskip = head_size; 242 } else { 243 req_cache->sskip = head_size; 244 req_cache->dskip = 0; 245 } 246 247 return req_cache; 248 } 249 250 static void hisi_zip_remove_req(struct hisi_zip_qp_ctx *qp_ctx, 251 struct hisi_zip_req *req) 252 { 253 struct hisi_zip_req_q *req_q = &qp_ctx->req_q; 254 255 write_lock(&req_q->req_lock); 256 clear_bit(req->req_id, req_q->req_bitmap); 257 write_unlock(&req_q->req_lock); 258 } 259 260 static void hisi_zip_fill_addr(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req) 261 { 262 sqe->source_addr_l = lower_32_bits(req->dma_src); 263 sqe->source_addr_h = upper_32_bits(req->dma_src); 264 sqe->dest_addr_l = lower_32_bits(req->dma_dst); 265 sqe->dest_addr_h = upper_32_bits(req->dma_dst); 266 } 267 268 static void hisi_zip_fill_buf_size(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req) 269 { 270 struct acomp_req *a_req = req->req; 271 272 sqe->input_data_length = a_req->slen - req->sskip; 273 sqe->dest_avail_out = a_req->dlen - req->dskip; 274 sqe->dw7 = FIELD_PREP(HZIP_IN_SGE_DATA_OFFSET_M, req->sskip); 275 sqe->dw8 = FIELD_PREP(HZIP_OUT_SGE_DATA_OFFSET_M, req->dskip); 276 } 277 278 static void hisi_zip_fill_buf_type(struct hisi_zip_sqe *sqe, u8 buf_type) 279 { 280 u32 val; 281 282 val = sqe->dw9 & ~HZIP_BUF_TYPE_M; 283 val |= FIELD_PREP(HZIP_BUF_TYPE_M, buf_type); 284 sqe->dw9 = val; 285 } 286 287 static void hisi_zip_fill_req_type(struct hisi_zip_sqe *sqe, u8 req_type) 288 { 289 u32 val; 290 291 val = sqe->dw9 & ~HZIP_REQ_TYPE_M; 292 val |= FIELD_PREP(HZIP_REQ_TYPE_M, req_type); 293 sqe->dw9 = val; 294 } 295 296 static void hisi_zip_fill_tag_v1(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req) 297 { 298 sqe->dw13 = req->req_id; 299 } 300 301 static void hisi_zip_fill_tag_v2(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req) 302 { 303 sqe->dw26 = req->req_id; 304 } 305 306 static void hisi_zip_fill_sqe_type(struct hisi_zip_sqe *sqe, u8 sqe_type) 307 { 308 u32 val; 309 310 val = sqe->dw7 & ~HZIP_SQE_TYPE_M; 311 val |= FIELD_PREP(HZIP_SQE_TYPE_M, sqe_type); 312 sqe->dw7 = val; 313 } 314 315 static void hisi_zip_fill_sqe(struct hisi_zip_ctx *ctx, struct hisi_zip_sqe *sqe, 316 u8 req_type, struct hisi_zip_req *req) 317 { 318 const struct hisi_zip_sqe_ops *ops = ctx->ops; 319 320 memset(sqe, 0, sizeof(struct hisi_zip_sqe)); 321 322 ops->fill_addr(sqe, req); 323 ops->fill_buf_size(sqe, req); 324 ops->fill_buf_type(sqe, HZIP_SGL); 325 ops->fill_req_type(sqe, req_type); 326 ops->fill_tag(sqe, req); 327 ops->fill_sqe_type(sqe, ops->sqe_type); 328 } 329 330 static int hisi_zip_do_work(struct hisi_zip_req *req, 331 struct hisi_zip_qp_ctx *qp_ctx) 332 { 333 struct hisi_acc_sgl_pool *pool = qp_ctx->sgl_pool; 334 struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx; 335 struct acomp_req *a_req = req->req; 336 struct hisi_qp *qp = qp_ctx->qp; 337 struct device *dev = &qp->qm->pdev->dev; 338 struct hisi_zip_sqe zip_sqe; 339 int ret; 340 341 if (unlikely(!a_req->src || !a_req->slen || !a_req->dst || !a_req->dlen)) 342 return -EINVAL; 343 344 req->hw_src = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->src, pool, 345 req->req_id << 1, &req->dma_src); 346 if (IS_ERR(req->hw_src)) { 347 dev_err(dev, "failed to map the src buffer to hw sgl (%ld)!\n", 348 PTR_ERR(req->hw_src)); 349 return PTR_ERR(req->hw_src); 350 } 351 352 req->hw_dst = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->dst, pool, 353 (req->req_id << 1) + 1, 354 &req->dma_dst); 355 if (IS_ERR(req->hw_dst)) { 356 ret = PTR_ERR(req->hw_dst); 357 dev_err(dev, "failed to map the dst buffer to hw slg (%d)!\n", 358 ret); 359 goto err_unmap_input; 360 } 361 362 hisi_zip_fill_sqe(qp_ctx->ctx, &zip_sqe, qp->req_type, req); 363 364 /* send command to start a task */ 365 atomic64_inc(&dfx->send_cnt); 366 ret = hisi_qp_send(qp, &zip_sqe); 367 if (unlikely(ret < 0)) { 368 atomic64_inc(&dfx->send_busy_cnt); 369 ret = -EAGAIN; 370 dev_dbg_ratelimited(dev, "failed to send request!\n"); 371 goto err_unmap_output; 372 } 373 374 return -EINPROGRESS; 375 376 err_unmap_output: 377 hisi_acc_sg_buf_unmap(dev, a_req->dst, req->hw_dst); 378 err_unmap_input: 379 hisi_acc_sg_buf_unmap(dev, a_req->src, req->hw_src); 380 return ret; 381 } 382 383 static u32 hisi_zip_get_tag_v1(struct hisi_zip_sqe *sqe) 384 { 385 return sqe->dw13; 386 } 387 388 static u32 hisi_zip_get_tag_v2(struct hisi_zip_sqe *sqe) 389 { 390 return sqe->dw26; 391 } 392 393 static u32 hisi_zip_get_status(struct hisi_zip_sqe *sqe) 394 { 395 return sqe->dw3 & HZIP_BD_STATUS_M; 396 } 397 398 static u32 hisi_zip_get_dstlen(struct hisi_zip_sqe *sqe) 399 { 400 return sqe->produced; 401 } 402 403 static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data) 404 { 405 struct hisi_zip_qp_ctx *qp_ctx = qp->qp_ctx; 406 const struct hisi_zip_sqe_ops *ops = qp_ctx->ctx->ops; 407 struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx; 408 struct hisi_zip_req_q *req_q = &qp_ctx->req_q; 409 struct device *dev = &qp->qm->pdev->dev; 410 struct hisi_zip_sqe *sqe = data; 411 u32 tag = ops->get_tag(sqe); 412 struct hisi_zip_req *req = req_q->q + tag; 413 struct acomp_req *acomp_req = req->req; 414 u32 status, dlen, head_size; 415 int err = 0; 416 417 atomic64_inc(&dfx->recv_cnt); 418 status = ops->get_status(sqe); 419 if (unlikely(status != 0 && status != HZIP_NC_ERR)) { 420 dev_err(dev, "%scompress fail in qp%u: %u, output: %u\n", 421 (qp->alg_type == 0) ? "" : "de", qp->qp_id, status, 422 sqe->produced); 423 atomic64_inc(&dfx->err_bd_cnt); 424 err = -EIO; 425 } 426 427 dlen = ops->get_dstlen(sqe); 428 429 hisi_acc_sg_buf_unmap(dev, acomp_req->src, req->hw_src); 430 hisi_acc_sg_buf_unmap(dev, acomp_req->dst, req->hw_dst); 431 432 head_size = (qp->alg_type == 0) ? TO_HEAD_SIZE(qp->req_type) : 0; 433 acomp_req->dlen = dlen + head_size; 434 435 if (acomp_req->base.complete) 436 acomp_request_complete(acomp_req, err); 437 438 hisi_zip_remove_req(qp_ctx, req); 439 } 440 441 static int hisi_zip_acompress(struct acomp_req *acomp_req) 442 { 443 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); 444 struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_COMP]; 445 struct device *dev = &qp_ctx->qp->qm->pdev->dev; 446 struct hisi_zip_req *req; 447 int head_size; 448 int ret; 449 450 /* let's output compression head now */ 451 head_size = add_comp_head(acomp_req->dst, qp_ctx->qp->req_type); 452 if (unlikely(head_size < 0)) { 453 dev_err_ratelimited(dev, "failed to add comp head (%d)!\n", 454 head_size); 455 return head_size; 456 } 457 458 req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, true); 459 if (IS_ERR(req)) 460 return PTR_ERR(req); 461 462 ret = hisi_zip_do_work(req, qp_ctx); 463 if (unlikely(ret != -EINPROGRESS)) { 464 dev_info_ratelimited(dev, "failed to do compress (%d)!\n", ret); 465 hisi_zip_remove_req(qp_ctx, req); 466 } 467 468 return ret; 469 } 470 471 static int hisi_zip_adecompress(struct acomp_req *acomp_req) 472 { 473 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); 474 struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_DECOMP]; 475 struct device *dev = &qp_ctx->qp->qm->pdev->dev; 476 struct hisi_zip_req *req; 477 int head_size, ret; 478 479 head_size = get_comp_head_size(acomp_req, qp_ctx->qp->req_type); 480 if (unlikely(head_size < 0)) { 481 dev_err_ratelimited(dev, "failed to get comp head size (%d)!\n", 482 head_size); 483 return head_size; 484 } 485 486 req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, false); 487 if (IS_ERR(req)) 488 return PTR_ERR(req); 489 490 ret = hisi_zip_do_work(req, qp_ctx); 491 if (unlikely(ret != -EINPROGRESS)) { 492 dev_info_ratelimited(dev, "failed to do decompress (%d)!\n", 493 ret); 494 hisi_zip_remove_req(qp_ctx, req); 495 } 496 497 return ret; 498 } 499 500 static int hisi_zip_start_qp(struct hisi_qp *qp, struct hisi_zip_qp_ctx *qp_ctx, 501 int alg_type, int req_type) 502 { 503 struct device *dev = &qp->qm->pdev->dev; 504 int ret; 505 506 qp->req_type = req_type; 507 qp->alg_type = alg_type; 508 qp->qp_ctx = qp_ctx; 509 510 ret = hisi_qm_start_qp(qp, 0); 511 if (ret < 0) { 512 dev_err(dev, "failed to start qp (%d)!\n", ret); 513 return ret; 514 } 515 516 qp_ctx->qp = qp; 517 518 return 0; 519 } 520 521 static void hisi_zip_release_qp(struct hisi_zip_qp_ctx *qp_ctx) 522 { 523 hisi_qm_stop_qp(qp_ctx->qp); 524 hisi_qm_free_qps(&qp_ctx->qp, 1); 525 } 526 527 static const struct hisi_zip_sqe_ops hisi_zip_ops_v1 = { 528 .sqe_type = 0, 529 .fill_addr = hisi_zip_fill_addr, 530 .fill_buf_size = hisi_zip_fill_buf_size, 531 .fill_buf_type = hisi_zip_fill_buf_type, 532 .fill_req_type = hisi_zip_fill_req_type, 533 .fill_tag = hisi_zip_fill_tag_v1, 534 .fill_sqe_type = hisi_zip_fill_sqe_type, 535 .get_tag = hisi_zip_get_tag_v1, 536 .get_status = hisi_zip_get_status, 537 .get_dstlen = hisi_zip_get_dstlen, 538 }; 539 540 static const struct hisi_zip_sqe_ops hisi_zip_ops_v2 = { 541 .sqe_type = 0x3, 542 .fill_addr = hisi_zip_fill_addr, 543 .fill_buf_size = hisi_zip_fill_buf_size, 544 .fill_buf_type = hisi_zip_fill_buf_type, 545 .fill_req_type = hisi_zip_fill_req_type, 546 .fill_tag = hisi_zip_fill_tag_v2, 547 .fill_sqe_type = hisi_zip_fill_sqe_type, 548 .get_tag = hisi_zip_get_tag_v2, 549 .get_status = hisi_zip_get_status, 550 .get_dstlen = hisi_zip_get_dstlen, 551 }; 552 553 static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type, int node) 554 { 555 struct hisi_qp *qps[HZIP_CTX_Q_NUM] = { NULL }; 556 struct hisi_zip_qp_ctx *qp_ctx; 557 struct hisi_zip *hisi_zip; 558 int ret, i, j; 559 560 ret = zip_create_qps(qps, HZIP_CTX_Q_NUM, node); 561 if (ret) { 562 pr_err("failed to create zip qps (%d)!\n", ret); 563 return -ENODEV; 564 } 565 566 hisi_zip = container_of(qps[0]->qm, struct hisi_zip, qm); 567 568 for (i = 0; i < HZIP_CTX_Q_NUM; i++) { 569 /* alg_type = 0 for compress, 1 for decompress in hw sqe */ 570 qp_ctx = &hisi_zip_ctx->qp_ctx[i]; 571 qp_ctx->ctx = hisi_zip_ctx; 572 ret = hisi_zip_start_qp(qps[i], qp_ctx, i, req_type); 573 if (ret) { 574 for (j = i - 1; j >= 0; j--) 575 hisi_qm_stop_qp(hisi_zip_ctx->qp_ctx[j].qp); 576 577 hisi_qm_free_qps(qps, HZIP_CTX_Q_NUM); 578 return ret; 579 } 580 581 qp_ctx->zip_dev = hisi_zip; 582 } 583 584 if (hisi_zip->qm.ver < QM_HW_V3) 585 hisi_zip_ctx->ops = &hisi_zip_ops_v1; 586 else 587 hisi_zip_ctx->ops = &hisi_zip_ops_v2; 588 589 return 0; 590 } 591 592 static void hisi_zip_ctx_exit(struct hisi_zip_ctx *hisi_zip_ctx) 593 { 594 int i; 595 596 for (i = 0; i < HZIP_CTX_Q_NUM; i++) 597 hisi_zip_release_qp(&hisi_zip_ctx->qp_ctx[i]); 598 } 599 600 static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx) 601 { 602 u16 q_depth = ctx->qp_ctx[0].qp->sq_depth; 603 struct hisi_zip_req_q *req_q; 604 int i, ret; 605 606 for (i = 0; i < HZIP_CTX_Q_NUM; i++) { 607 req_q = &ctx->qp_ctx[i].req_q; 608 req_q->size = q_depth; 609 610 req_q->req_bitmap = bitmap_zalloc(req_q->size, GFP_KERNEL); 611 if (!req_q->req_bitmap) { 612 ret = -ENOMEM; 613 if (i == 0) 614 return ret; 615 616 goto err_free_comp_q; 617 } 618 rwlock_init(&req_q->req_lock); 619 620 req_q->q = kcalloc(req_q->size, sizeof(struct hisi_zip_req), 621 GFP_KERNEL); 622 if (!req_q->q) { 623 ret = -ENOMEM; 624 if (i == 0) 625 goto err_free_comp_bitmap; 626 else 627 goto err_free_decomp_bitmap; 628 } 629 } 630 631 return 0; 632 633 err_free_decomp_bitmap: 634 bitmap_free(ctx->qp_ctx[HZIP_QPC_DECOMP].req_q.req_bitmap); 635 err_free_comp_q: 636 kfree(ctx->qp_ctx[HZIP_QPC_COMP].req_q.q); 637 err_free_comp_bitmap: 638 bitmap_free(ctx->qp_ctx[HZIP_QPC_COMP].req_q.req_bitmap); 639 return ret; 640 } 641 642 static void hisi_zip_release_req_q(struct hisi_zip_ctx *ctx) 643 { 644 int i; 645 646 for (i = 0; i < HZIP_CTX_Q_NUM; i++) { 647 kfree(ctx->qp_ctx[i].req_q.q); 648 bitmap_free(ctx->qp_ctx[i].req_q.req_bitmap); 649 } 650 } 651 652 static int hisi_zip_create_sgl_pool(struct hisi_zip_ctx *ctx) 653 { 654 u16 q_depth = ctx->qp_ctx[0].qp->sq_depth; 655 struct hisi_zip_qp_ctx *tmp; 656 struct device *dev; 657 int i; 658 659 for (i = 0; i < HZIP_CTX_Q_NUM; i++) { 660 tmp = &ctx->qp_ctx[i]; 661 dev = &tmp->qp->qm->pdev->dev; 662 tmp->sgl_pool = hisi_acc_create_sgl_pool(dev, q_depth << 1, 663 sgl_sge_nr); 664 if (IS_ERR(tmp->sgl_pool)) { 665 if (i == 1) 666 goto err_free_sgl_pool0; 667 return -ENOMEM; 668 } 669 } 670 671 return 0; 672 673 err_free_sgl_pool0: 674 hisi_acc_free_sgl_pool(&ctx->qp_ctx[HZIP_QPC_COMP].qp->qm->pdev->dev, 675 ctx->qp_ctx[HZIP_QPC_COMP].sgl_pool); 676 return -ENOMEM; 677 } 678 679 static void hisi_zip_release_sgl_pool(struct hisi_zip_ctx *ctx) 680 { 681 int i; 682 683 for (i = 0; i < HZIP_CTX_Q_NUM; i++) 684 hisi_acc_free_sgl_pool(&ctx->qp_ctx[i].qp->qm->pdev->dev, 685 ctx->qp_ctx[i].sgl_pool); 686 } 687 688 static void hisi_zip_set_acomp_cb(struct hisi_zip_ctx *ctx, 689 void (*fn)(struct hisi_qp *, void *)) 690 { 691 int i; 692 693 for (i = 0; i < HZIP_CTX_Q_NUM; i++) 694 ctx->qp_ctx[i].qp->req_cb = fn; 695 } 696 697 static int hisi_zip_acomp_init(struct crypto_acomp *tfm) 698 { 699 const char *alg_name = crypto_tfm_alg_name(&tfm->base); 700 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(&tfm->base); 701 struct device *dev; 702 int ret; 703 704 ret = hisi_zip_ctx_init(ctx, COMP_NAME_TO_TYPE(alg_name), tfm->base.node); 705 if (ret) { 706 pr_err("failed to init ctx (%d)!\n", ret); 707 return ret; 708 } 709 710 dev = &ctx->qp_ctx[0].qp->qm->pdev->dev; 711 712 ret = hisi_zip_create_req_q(ctx); 713 if (ret) { 714 dev_err(dev, "failed to create request queue (%d)!\n", ret); 715 goto err_ctx_exit; 716 } 717 718 ret = hisi_zip_create_sgl_pool(ctx); 719 if (ret) { 720 dev_err(dev, "failed to create sgl pool (%d)!\n", ret); 721 goto err_release_req_q; 722 } 723 724 hisi_zip_set_acomp_cb(ctx, hisi_zip_acomp_cb); 725 726 return 0; 727 728 err_release_req_q: 729 hisi_zip_release_req_q(ctx); 730 err_ctx_exit: 731 hisi_zip_ctx_exit(ctx); 732 return ret; 733 } 734 735 static void hisi_zip_acomp_exit(struct crypto_acomp *tfm) 736 { 737 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(&tfm->base); 738 739 hisi_zip_set_acomp_cb(ctx, NULL); 740 hisi_zip_release_sgl_pool(ctx); 741 hisi_zip_release_req_q(ctx); 742 hisi_zip_ctx_exit(ctx); 743 } 744 745 static struct acomp_alg hisi_zip_acomp_zlib = { 746 .init = hisi_zip_acomp_init, 747 .exit = hisi_zip_acomp_exit, 748 .compress = hisi_zip_acompress, 749 .decompress = hisi_zip_adecompress, 750 .base = { 751 .cra_name = "zlib-deflate", 752 .cra_driver_name = "hisi-zlib-acomp", 753 .cra_module = THIS_MODULE, 754 .cra_priority = HZIP_ALG_PRIORITY, 755 .cra_ctxsize = sizeof(struct hisi_zip_ctx), 756 } 757 }; 758 759 static struct acomp_alg hisi_zip_acomp_gzip = { 760 .init = hisi_zip_acomp_init, 761 .exit = hisi_zip_acomp_exit, 762 .compress = hisi_zip_acompress, 763 .decompress = hisi_zip_adecompress, 764 .base = { 765 .cra_name = "gzip", 766 .cra_driver_name = "hisi-gzip-acomp", 767 .cra_module = THIS_MODULE, 768 .cra_priority = HZIP_ALG_PRIORITY, 769 .cra_ctxsize = sizeof(struct hisi_zip_ctx), 770 } 771 }; 772 773 int hisi_zip_register_to_crypto(struct hisi_qm *qm) 774 { 775 int ret; 776 777 ret = crypto_register_acomp(&hisi_zip_acomp_zlib); 778 if (ret) { 779 pr_err("failed to register to zlib (%d)!\n", ret); 780 return ret; 781 } 782 783 ret = crypto_register_acomp(&hisi_zip_acomp_gzip); 784 if (ret) { 785 pr_err("failed to register to gzip (%d)!\n", ret); 786 crypto_unregister_acomp(&hisi_zip_acomp_zlib); 787 } 788 789 return ret; 790 } 791 792 void hisi_zip_unregister_from_crypto(struct hisi_qm *qm) 793 { 794 crypto_unregister_acomp(&hisi_zip_acomp_gzip); 795 crypto_unregister_acomp(&hisi_zip_acomp_zlib); 796 } 797