1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2019 HiSilicon Limited. */ 3 4 #include <crypto/aes.h> 5 #include <crypto/aead.h> 6 #include <crypto/algapi.h> 7 #include <crypto/authenc.h> 8 #include <crypto/des.h> 9 #include <crypto/hash.h> 10 #include <crypto/internal/aead.h> 11 #include <crypto/internal/des.h> 12 #include <crypto/sha1.h> 13 #include <crypto/sha2.h> 14 #include <crypto/skcipher.h> 15 #include <crypto/xts.h> 16 #include <linux/crypto.h> 17 #include <linux/dma-mapping.h> 18 #include <linux/idr.h> 19 20 #include "sec.h" 21 #include "sec_crypto.h" 22 23 #define SEC_PRIORITY 4001 24 #define SEC_XTS_MIN_KEY_SIZE (2 * AES_MIN_KEY_SIZE) 25 #define SEC_XTS_MID_KEY_SIZE (3 * AES_MIN_KEY_SIZE) 26 #define SEC_XTS_MAX_KEY_SIZE (2 * AES_MAX_KEY_SIZE) 27 #define SEC_DES3_2KEY_SIZE (2 * DES_KEY_SIZE) 28 #define SEC_DES3_3KEY_SIZE (3 * DES_KEY_SIZE) 29 30 /* SEC sqe(bd) bit operational relative MACRO */ 31 #define SEC_DE_OFFSET 1 32 #define SEC_CIPHER_OFFSET 4 33 #define SEC_SCENE_OFFSET 3 34 #define SEC_DST_SGL_OFFSET 2 35 #define SEC_SRC_SGL_OFFSET 7 36 #define SEC_CKEY_OFFSET 9 37 #define SEC_CMODE_OFFSET 12 38 #define SEC_AKEY_OFFSET 5 39 #define SEC_AEAD_ALG_OFFSET 11 40 #define SEC_AUTH_OFFSET 6 41 42 #define SEC_DE_OFFSET_V3 9 43 #define SEC_SCENE_OFFSET_V3 5 44 #define SEC_CKEY_OFFSET_V3 13 45 #define SEC_CTR_CNT_OFFSET 25 46 #define SEC_CTR_CNT_ROLLOVER 2 47 #define SEC_SRC_SGL_OFFSET_V3 11 48 #define SEC_DST_SGL_OFFSET_V3 14 49 #define SEC_CALG_OFFSET_V3 4 50 #define SEC_AKEY_OFFSET_V3 9 51 #define SEC_MAC_OFFSET_V3 4 52 #define SEC_AUTH_ALG_OFFSET_V3 15 53 #define SEC_CIPHER_AUTH_V3 0xbf 54 #define SEC_AUTH_CIPHER_V3 0x40 55 #define SEC_FLAG_OFFSET 7 56 #define SEC_FLAG_MASK 0x0780 57 #define SEC_TYPE_MASK 0x0F 58 #define SEC_DONE_MASK 0x0001 59 #define SEC_ICV_MASK 0x000E 60 61 #define SEC_TOTAL_IV_SZ(depth) (SEC_IV_SIZE * (depth)) 62 #define SEC_SGL_SGE_NR 128 63 #define SEC_CIPHER_AUTH 0xfe 64 #define SEC_AUTH_CIPHER 0x1 65 #define SEC_MAX_MAC_LEN 64 66 #define SEC_MAX_AAD_LEN 65535 67 #define SEC_MAX_CCM_AAD_LEN 65279 68 #define SEC_TOTAL_MAC_SZ(depth) (SEC_MAX_MAC_LEN * (depth)) 69 70 #define SEC_PBUF_IV_OFFSET SEC_PBUF_SZ 71 #define SEC_PBUF_MAC_OFFSET (SEC_PBUF_SZ + SEC_IV_SIZE) 72 #define SEC_PBUF_PKG (SEC_PBUF_SZ + SEC_IV_SIZE + \ 73 SEC_MAX_MAC_LEN * 2) 74 #define SEC_PBUF_NUM (PAGE_SIZE / SEC_PBUF_PKG) 75 #define SEC_PBUF_PAGE_NUM(depth) ((depth) / SEC_PBUF_NUM) 76 #define SEC_PBUF_LEFT_SZ(depth) (SEC_PBUF_PKG * ((depth) - \ 77 SEC_PBUF_PAGE_NUM(depth) * SEC_PBUF_NUM)) 78 #define SEC_TOTAL_PBUF_SZ(depth) (PAGE_SIZE * SEC_PBUF_PAGE_NUM(depth) + \ 79 SEC_PBUF_LEFT_SZ(depth)) 80 81 #define SEC_SQE_CFLAG 2 82 #define SEC_SQE_AEAD_FLAG 3 83 #define SEC_SQE_DONE 0x1 84 #define SEC_ICV_ERR 0x2 85 #define MAC_LEN_MASK 0x1U 86 #define MAX_INPUT_DATA_LEN 0xFFFE00 87 #define BITS_MASK 0xFF 88 #define WORD_MASK 0x3 89 #define BYTE_BITS 0x8 90 #define BYTES_TO_WORDS(bcount) ((bcount) >> 2) 91 #define SEC_XTS_NAME_SZ 0x3 92 #define IV_CM_CAL_NUM 2 93 #define IV_CL_MASK 0x7 94 #define IV_CL_MIN 2 95 #define IV_CL_MID 4 96 #define IV_CL_MAX 8 97 #define IV_FLAGS_OFFSET 0x6 98 #define IV_CM_OFFSET 0x3 99 #define IV_LAST_BYTE1 1 100 #define IV_LAST_BYTE2 2 101 #define IV_LAST_BYTE_MASK 0xFF 102 #define IV_CTR_INIT 0x1 103 #define IV_BYTE_OFFSET 0x8 104 #define SEC_GCM_MIN_AUTH_SZ 0x8 105 #define SEC_RETRY_MAX_CNT 5U 106 107 static DEFINE_MUTEX(sec_algs_lock); 108 static unsigned int sec_available_devs; 109 110 struct sec_skcipher { 111 u64 alg_msk; 112 struct skcipher_alg alg; 113 }; 114 115 struct sec_aead { 116 u64 alg_msk; 117 struct aead_alg alg; 118 }; 119 120 static int sec_aead_soft_crypto(struct sec_ctx *ctx, 121 struct aead_request *aead_req, 122 bool encrypt); 123 static int sec_skcipher_soft_crypto(struct sec_ctx *ctx, 124 struct skcipher_request *sreq, bool encrypt); 125 126 static int sec_alloc_req_id(struct sec_req *req, struct sec_qp_ctx *qp_ctx) 127 { 128 int req_id; 129 130 spin_lock_bh(&qp_ctx->id_lock); 131 req_id = idr_alloc_cyclic(&qp_ctx->req_idr, NULL, 0, qp_ctx->qp->sq_depth, GFP_ATOMIC); 132 spin_unlock_bh(&qp_ctx->id_lock); 133 return req_id; 134 } 135 136 static void sec_free_req_id(struct sec_req *req) 137 { 138 struct sec_qp_ctx *qp_ctx = req->qp_ctx; 139 int req_id = req->req_id; 140 141 if (unlikely(req_id < 0 || req_id >= qp_ctx->qp->sq_depth)) { 142 dev_err(req->ctx->dev, "free request id invalid!\n"); 143 return; 144 } 145 146 spin_lock_bh(&qp_ctx->id_lock); 147 idr_remove(&qp_ctx->req_idr, req_id); 148 spin_unlock_bh(&qp_ctx->id_lock); 149 } 150 151 static u8 pre_parse_finished_bd(struct bd_status *status, void *resp) 152 { 153 struct sec_sqe *bd = resp; 154 155 status->done = le16_to_cpu(bd->type2.done_flag) & SEC_DONE_MASK; 156 status->icv = (le16_to_cpu(bd->type2.done_flag) & SEC_ICV_MASK) >> 1; 157 status->flag = (le16_to_cpu(bd->type2.done_flag) & 158 SEC_FLAG_MASK) >> SEC_FLAG_OFFSET; 159 status->tag = le16_to_cpu(bd->type2.tag); 160 status->err_type = bd->type2.error_type; 161 162 return bd->type_cipher_auth & SEC_TYPE_MASK; 163 } 164 165 static u8 pre_parse_finished_bd3(struct bd_status *status, void *resp) 166 { 167 struct sec_sqe3 *bd3 = resp; 168 169 status->done = le16_to_cpu(bd3->done_flag) & SEC_DONE_MASK; 170 status->icv = (le16_to_cpu(bd3->done_flag) & SEC_ICV_MASK) >> 1; 171 status->flag = (le16_to_cpu(bd3->done_flag) & 172 SEC_FLAG_MASK) >> SEC_FLAG_OFFSET; 173 status->tag = le64_to_cpu(bd3->tag); 174 status->err_type = bd3->error_type; 175 176 return le32_to_cpu(bd3->bd_param) & SEC_TYPE_MASK; 177 } 178 179 static int sec_cb_status_check(struct sec_req *req, 180 struct bd_status *status) 181 { 182 struct sec_ctx *ctx = req->ctx; 183 184 if (unlikely(req->err_type || status->done != SEC_SQE_DONE)) { 185 dev_err_ratelimited(ctx->dev, "err_type[%d], done[%u]\n", 186 req->err_type, status->done); 187 return -EIO; 188 } 189 190 if (unlikely(ctx->alg_type == SEC_SKCIPHER)) { 191 if (unlikely(status->flag != SEC_SQE_CFLAG)) { 192 dev_err_ratelimited(ctx->dev, "flag[%u]\n", 193 status->flag); 194 return -EIO; 195 } 196 } else if (unlikely(ctx->alg_type == SEC_AEAD)) { 197 if (unlikely(status->flag != SEC_SQE_AEAD_FLAG || 198 status->icv == SEC_ICV_ERR)) { 199 dev_err_ratelimited(ctx->dev, 200 "flag[%u], icv[%u]\n", 201 status->flag, status->icv); 202 return -EBADMSG; 203 } 204 } 205 206 return 0; 207 } 208 209 static int qp_send_message(struct sec_req *req) 210 { 211 struct sec_qp_ctx *qp_ctx = req->qp_ctx; 212 int ret; 213 214 if (atomic_read(&qp_ctx->qp->qp_status.used) == qp_ctx->qp->sq_depth - 1) 215 return -EBUSY; 216 217 spin_lock_bh(&qp_ctx->req_lock); 218 if (atomic_read(&qp_ctx->qp->qp_status.used) == qp_ctx->qp->sq_depth - 1) { 219 spin_unlock_bh(&qp_ctx->req_lock); 220 return -EBUSY; 221 } 222 223 if (qp_ctx->ctx->type_supported == SEC_BD_TYPE2) { 224 req->sec_sqe.type2.tag = cpu_to_le16((u16)qp_ctx->send_head); 225 qp_ctx->req_list[qp_ctx->send_head] = req; 226 } 227 228 ret = hisi_qp_send(qp_ctx->qp, &req->sec_sqe); 229 if (ret) { 230 spin_unlock_bh(&qp_ctx->req_lock); 231 return ret; 232 } 233 if (qp_ctx->ctx->type_supported == SEC_BD_TYPE2) 234 qp_ctx->send_head = (qp_ctx->send_head + 1) % qp_ctx->qp->sq_depth; 235 236 spin_unlock_bh(&qp_ctx->req_lock); 237 238 atomic64_inc(&req->ctx->sec->debug.dfx.send_cnt); 239 return -EINPROGRESS; 240 } 241 242 static void sec_alg_send_backlog_soft(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) 243 { 244 struct sec_req *req, *tmp; 245 int ret; 246 247 list_for_each_entry_safe(req, tmp, &qp_ctx->backlog.list, list) { 248 list_del(&req->list); 249 ctx->req_op->buf_unmap(ctx, req); 250 if (req->req_id >= 0) 251 sec_free_req_id(req); 252 253 if (ctx->alg_type == SEC_AEAD) 254 ret = sec_aead_soft_crypto(ctx, req->aead_req.aead_req, 255 req->c_req.encrypt); 256 else 257 ret = sec_skcipher_soft_crypto(ctx, req->c_req.sk_req, 258 req->c_req.encrypt); 259 260 /* Wake up the busy thread first, then return the errno. */ 261 crypto_request_complete(req->base, -EINPROGRESS); 262 crypto_request_complete(req->base, ret); 263 } 264 } 265 266 static void sec_alg_send_backlog(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) 267 { 268 struct sec_req *req, *tmp; 269 int ret; 270 271 spin_lock_bh(&qp_ctx->backlog.lock); 272 list_for_each_entry_safe(req, tmp, &qp_ctx->backlog.list, list) { 273 ret = qp_send_message(req); 274 switch (ret) { 275 case -EINPROGRESS: 276 list_del(&req->list); 277 crypto_request_complete(req->base, -EINPROGRESS); 278 break; 279 case -EBUSY: 280 /* Device is busy and stop send any request. */ 281 goto unlock; 282 default: 283 /* Release memory resources and send all requests through software. */ 284 sec_alg_send_backlog_soft(ctx, qp_ctx); 285 goto unlock; 286 } 287 } 288 289 unlock: 290 spin_unlock_bh(&qp_ctx->backlog.lock); 291 } 292 293 static void sec_req_cb(struct hisi_qp *qp, void *resp) 294 { 295 struct sec_qp_ctx *qp_ctx = qp->qp_ctx; 296 struct sec_dfx *dfx = &qp_ctx->ctx->sec->debug.dfx; 297 u8 type_supported = qp_ctx->ctx->type_supported; 298 struct bd_status status; 299 struct sec_ctx *ctx; 300 struct sec_req *req; 301 int err; 302 u8 type; 303 304 if (type_supported == SEC_BD_TYPE2) { 305 type = pre_parse_finished_bd(&status, resp); 306 req = qp_ctx->req_list[status.tag]; 307 } else { 308 type = pre_parse_finished_bd3(&status, resp); 309 req = (void *)(uintptr_t)status.tag; 310 } 311 312 if (unlikely(type != type_supported)) { 313 atomic64_inc(&dfx->err_bd_cnt); 314 pr_err("err bd type [%u]\n", type); 315 return; 316 } 317 318 if (unlikely(!req)) { 319 atomic64_inc(&dfx->invalid_req_cnt); 320 atomic_inc(&qp->qp_status.used); 321 return; 322 } 323 324 req->err_type = status.err_type; 325 ctx = req->ctx; 326 err = sec_cb_status_check(req, &status); 327 if (err) 328 atomic64_inc(&dfx->done_flag_cnt); 329 330 atomic64_inc(&dfx->recv_cnt); 331 332 ctx->req_op->buf_unmap(ctx, req); 333 334 ctx->req_op->callback(ctx, req, err); 335 } 336 337 static int sec_alg_send_message_retry(struct sec_req *req) 338 { 339 int ctr = 0; 340 int ret; 341 342 do { 343 ret = qp_send_message(req); 344 } while (ret == -EBUSY && ctr++ < SEC_RETRY_MAX_CNT); 345 346 return ret; 347 } 348 349 static int sec_alg_try_enqueue(struct sec_req *req) 350 { 351 /* Check if any request is already backlogged */ 352 if (!list_empty(&req->backlog->list)) 353 return -EBUSY; 354 355 /* Try to enqueue to HW ring */ 356 return qp_send_message(req); 357 } 358 359 360 static int sec_alg_send_message_maybacklog(struct sec_req *req) 361 { 362 int ret; 363 364 ret = sec_alg_try_enqueue(req); 365 if (ret != -EBUSY) 366 return ret; 367 368 spin_lock_bh(&req->backlog->lock); 369 ret = sec_alg_try_enqueue(req); 370 if (ret == -EBUSY) 371 list_add_tail(&req->list, &req->backlog->list); 372 spin_unlock_bh(&req->backlog->lock); 373 374 return ret; 375 } 376 377 static int sec_bd_send(struct sec_ctx *ctx, struct sec_req *req) 378 { 379 if (req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG) 380 return sec_alg_send_message_maybacklog(req); 381 382 return sec_alg_send_message_retry(req); 383 } 384 385 static int sec_alloc_civ_resource(struct device *dev, struct sec_alg_res *res) 386 { 387 u16 q_depth = res->depth; 388 int i; 389 390 res->c_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ(q_depth), 391 &res->c_ivin_dma, GFP_KERNEL); 392 if (!res->c_ivin) 393 return -ENOMEM; 394 395 for (i = 1; i < q_depth; i++) { 396 res[i].c_ivin_dma = res->c_ivin_dma + i * SEC_IV_SIZE; 397 res[i].c_ivin = res->c_ivin + i * SEC_IV_SIZE; 398 } 399 400 return 0; 401 } 402 403 static void sec_free_civ_resource(struct device *dev, struct sec_alg_res *res) 404 { 405 if (res->c_ivin) 406 dma_free_coherent(dev, SEC_TOTAL_IV_SZ(res->depth), 407 res->c_ivin, res->c_ivin_dma); 408 } 409 410 static int sec_alloc_aiv_resource(struct device *dev, struct sec_alg_res *res) 411 { 412 u16 q_depth = res->depth; 413 int i; 414 415 res->a_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ(q_depth), 416 &res->a_ivin_dma, GFP_KERNEL); 417 if (!res->a_ivin) 418 return -ENOMEM; 419 420 for (i = 1; i < q_depth; i++) { 421 res[i].a_ivin_dma = res->a_ivin_dma + i * SEC_IV_SIZE; 422 res[i].a_ivin = res->a_ivin + i * SEC_IV_SIZE; 423 } 424 425 return 0; 426 } 427 428 static void sec_free_aiv_resource(struct device *dev, struct sec_alg_res *res) 429 { 430 if (res->a_ivin) 431 dma_free_coherent(dev, SEC_TOTAL_IV_SZ(res->depth), 432 res->a_ivin, res->a_ivin_dma); 433 } 434 435 static int sec_alloc_mac_resource(struct device *dev, struct sec_alg_res *res) 436 { 437 u16 q_depth = res->depth; 438 int i; 439 440 res->out_mac = dma_alloc_coherent(dev, SEC_TOTAL_MAC_SZ(q_depth) << 1, 441 &res->out_mac_dma, GFP_KERNEL); 442 if (!res->out_mac) 443 return -ENOMEM; 444 445 for (i = 1; i < q_depth; i++) { 446 res[i].out_mac_dma = res->out_mac_dma + 447 i * (SEC_MAX_MAC_LEN << 1); 448 res[i].out_mac = res->out_mac + i * (SEC_MAX_MAC_LEN << 1); 449 } 450 451 return 0; 452 } 453 454 static void sec_free_mac_resource(struct device *dev, struct sec_alg_res *res) 455 { 456 if (res->out_mac) 457 dma_free_coherent(dev, SEC_TOTAL_MAC_SZ(res->depth) << 1, 458 res->out_mac, res->out_mac_dma); 459 } 460 461 static void sec_free_pbuf_resource(struct device *dev, struct sec_alg_res *res) 462 { 463 if (res->pbuf) 464 dma_free_coherent(dev, SEC_TOTAL_PBUF_SZ(res->depth), 465 res->pbuf, res->pbuf_dma); 466 } 467 468 /* 469 * To improve performance, pbuffer is used for 470 * small packets (< 512Bytes) as IOMMU translation using. 471 */ 472 static int sec_alloc_pbuf_resource(struct device *dev, struct sec_alg_res *res) 473 { 474 u16 q_depth = res->depth; 475 int size = SEC_PBUF_PAGE_NUM(q_depth); 476 int pbuf_page_offset; 477 int i, j, k; 478 479 res->pbuf = dma_alloc_coherent(dev, SEC_TOTAL_PBUF_SZ(q_depth), 480 &res->pbuf_dma, GFP_KERNEL); 481 if (!res->pbuf) 482 return -ENOMEM; 483 484 /* 485 * SEC_PBUF_PKG contains data pbuf, iv and 486 * out_mac : <SEC_PBUF|SEC_IV|SEC_MAC> 487 * Every PAGE contains six SEC_PBUF_PKG 488 * The sec_qp_ctx contains QM_Q_DEPTH numbers of SEC_PBUF_PKG 489 * So we need SEC_PBUF_PAGE_NUM numbers of PAGE 490 * for the SEC_TOTAL_PBUF_SZ 491 */ 492 for (i = 0; i <= size; i++) { 493 pbuf_page_offset = PAGE_SIZE * i; 494 for (j = 0; j < SEC_PBUF_NUM; j++) { 495 k = i * SEC_PBUF_NUM + j; 496 if (k == q_depth) 497 break; 498 res[k].pbuf = res->pbuf + 499 j * SEC_PBUF_PKG + pbuf_page_offset; 500 res[k].pbuf_dma = res->pbuf_dma + 501 j * SEC_PBUF_PKG + pbuf_page_offset; 502 } 503 } 504 505 return 0; 506 } 507 508 static int sec_alg_resource_alloc(struct sec_ctx *ctx, 509 struct sec_qp_ctx *qp_ctx) 510 { 511 struct sec_alg_res *res = qp_ctx->res; 512 struct device *dev = ctx->dev; 513 int ret; 514 515 ret = sec_alloc_civ_resource(dev, res); 516 if (ret) 517 return ret; 518 519 if (ctx->alg_type == SEC_AEAD) { 520 ret = sec_alloc_aiv_resource(dev, res); 521 if (ret) 522 goto alloc_aiv_fail; 523 524 ret = sec_alloc_mac_resource(dev, res); 525 if (ret) 526 goto alloc_mac_fail; 527 } 528 if (ctx->pbuf_supported) { 529 ret = sec_alloc_pbuf_resource(dev, res); 530 if (ret) { 531 dev_err(dev, "fail to alloc pbuf dma resource!\n"); 532 goto alloc_pbuf_fail; 533 } 534 } 535 536 return 0; 537 538 alloc_pbuf_fail: 539 if (ctx->alg_type == SEC_AEAD) 540 sec_free_mac_resource(dev, qp_ctx->res); 541 alloc_mac_fail: 542 if (ctx->alg_type == SEC_AEAD) 543 sec_free_aiv_resource(dev, res); 544 alloc_aiv_fail: 545 sec_free_civ_resource(dev, res); 546 return ret; 547 } 548 549 static void sec_alg_resource_free(struct sec_ctx *ctx, 550 struct sec_qp_ctx *qp_ctx) 551 { 552 struct device *dev = ctx->dev; 553 554 sec_free_civ_resource(dev, qp_ctx->res); 555 556 if (ctx->pbuf_supported) 557 sec_free_pbuf_resource(dev, qp_ctx->res); 558 if (ctx->alg_type == SEC_AEAD) { 559 sec_free_mac_resource(dev, qp_ctx->res); 560 sec_free_aiv_resource(dev, qp_ctx->res); 561 } 562 } 563 564 static int sec_alloc_qp_ctx_resource(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) 565 { 566 u16 q_depth = qp_ctx->qp->sq_depth; 567 struct device *dev = ctx->dev; 568 int ret = -ENOMEM; 569 570 qp_ctx->req_list = kcalloc(q_depth, sizeof(struct sec_req *), GFP_KERNEL); 571 if (!qp_ctx->req_list) 572 return ret; 573 574 qp_ctx->res = kcalloc(q_depth, sizeof(struct sec_alg_res), GFP_KERNEL); 575 if (!qp_ctx->res) 576 goto err_free_req_list; 577 qp_ctx->res->depth = q_depth; 578 579 qp_ctx->c_in_pool = hisi_acc_create_sgl_pool(dev, q_depth, SEC_SGL_SGE_NR); 580 if (IS_ERR(qp_ctx->c_in_pool)) { 581 dev_err(dev, "fail to create sgl pool for input!\n"); 582 goto err_free_res; 583 } 584 585 qp_ctx->c_out_pool = hisi_acc_create_sgl_pool(dev, q_depth, SEC_SGL_SGE_NR); 586 if (IS_ERR(qp_ctx->c_out_pool)) { 587 dev_err(dev, "fail to create sgl pool for output!\n"); 588 goto err_free_c_in_pool; 589 } 590 591 ret = sec_alg_resource_alloc(ctx, qp_ctx); 592 if (ret) 593 goto err_free_c_out_pool; 594 595 return 0; 596 597 err_free_c_out_pool: 598 hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool); 599 err_free_c_in_pool: 600 hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool); 601 err_free_res: 602 kfree(qp_ctx->res); 603 err_free_req_list: 604 kfree(qp_ctx->req_list); 605 return ret; 606 } 607 608 static void sec_free_qp_ctx_resource(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) 609 { 610 struct device *dev = ctx->dev; 611 612 sec_alg_resource_free(ctx, qp_ctx); 613 hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool); 614 hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool); 615 kfree(qp_ctx->res); 616 kfree(qp_ctx->req_list); 617 } 618 619 static int sec_create_qp_ctx(struct sec_ctx *ctx, int qp_ctx_id) 620 { 621 struct sec_qp_ctx *qp_ctx; 622 struct hisi_qp *qp; 623 int ret; 624 625 qp_ctx = &ctx->qp_ctx[qp_ctx_id]; 626 qp = ctx->qps[qp_ctx_id]; 627 qp->req_type = 0; 628 qp->qp_ctx = qp_ctx; 629 qp_ctx->qp = qp; 630 qp_ctx->ctx = ctx; 631 632 qp->req_cb = sec_req_cb; 633 634 spin_lock_init(&qp_ctx->req_lock); 635 idr_init(&qp_ctx->req_idr); 636 spin_lock_init(&qp_ctx->backlog.lock); 637 spin_lock_init(&qp_ctx->id_lock); 638 INIT_LIST_HEAD(&qp_ctx->backlog.list); 639 qp_ctx->send_head = 0; 640 641 ret = sec_alloc_qp_ctx_resource(ctx, qp_ctx); 642 if (ret) 643 goto err_destroy_idr; 644 645 ret = hisi_qm_start_qp(qp, 0); 646 if (ret < 0) 647 goto err_resource_free; 648 649 return 0; 650 651 err_resource_free: 652 sec_free_qp_ctx_resource(ctx, qp_ctx); 653 err_destroy_idr: 654 idr_destroy(&qp_ctx->req_idr); 655 return ret; 656 } 657 658 static void sec_release_qp_ctx(struct sec_ctx *ctx, 659 struct sec_qp_ctx *qp_ctx) 660 { 661 hisi_qm_stop_qp(qp_ctx->qp); 662 sec_free_qp_ctx_resource(ctx, qp_ctx); 663 idr_destroy(&qp_ctx->req_idr); 664 } 665 666 static int sec_ctx_base_init(struct sec_ctx *ctx) 667 { 668 struct sec_dev *sec; 669 int i, ret; 670 671 ctx->qps = sec_create_qps(); 672 if (!ctx->qps) { 673 pr_err("Can not create sec qps!\n"); 674 return -ENODEV; 675 } 676 677 sec = container_of(ctx->qps[0]->qm, struct sec_dev, qm); 678 ctx->sec = sec; 679 ctx->dev = &sec->qm.pdev->dev; 680 ctx->hlf_q_num = sec->ctx_q_num >> 1; 681 682 ctx->pbuf_supported = ctx->sec->iommu_used; 683 ctx->qp_ctx = kcalloc(sec->ctx_q_num, sizeof(struct sec_qp_ctx), 684 GFP_KERNEL); 685 if (!ctx->qp_ctx) { 686 ret = -ENOMEM; 687 goto err_destroy_qps; 688 } 689 690 for (i = 0; i < sec->ctx_q_num; i++) { 691 ret = sec_create_qp_ctx(ctx, i); 692 if (ret) 693 goto err_sec_release_qp_ctx; 694 } 695 696 return 0; 697 698 err_sec_release_qp_ctx: 699 for (i = i - 1; i >= 0; i--) 700 sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]); 701 kfree(ctx->qp_ctx); 702 err_destroy_qps: 703 sec_destroy_qps(ctx->qps, sec->ctx_q_num); 704 return ret; 705 } 706 707 static void sec_ctx_base_uninit(struct sec_ctx *ctx) 708 { 709 int i; 710 711 for (i = 0; i < ctx->sec->ctx_q_num; i++) 712 sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]); 713 714 sec_destroy_qps(ctx->qps, ctx->sec->ctx_q_num); 715 kfree(ctx->qp_ctx); 716 } 717 718 static int sec_cipher_init(struct sec_ctx *ctx) 719 { 720 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 721 722 c_ctx->c_key = dma_alloc_coherent(ctx->dev, SEC_MAX_KEY_SIZE, 723 &c_ctx->c_key_dma, GFP_KERNEL); 724 if (!c_ctx->c_key) 725 return -ENOMEM; 726 727 return 0; 728 } 729 730 static void sec_cipher_uninit(struct sec_ctx *ctx) 731 { 732 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 733 734 memzero_explicit(c_ctx->c_key, SEC_MAX_KEY_SIZE); 735 dma_free_coherent(ctx->dev, SEC_MAX_KEY_SIZE, 736 c_ctx->c_key, c_ctx->c_key_dma); 737 } 738 739 static int sec_auth_init(struct sec_ctx *ctx) 740 { 741 struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 742 743 a_ctx->a_key = dma_alloc_coherent(ctx->dev, SEC_MAX_AKEY_SIZE, 744 &a_ctx->a_key_dma, GFP_KERNEL); 745 if (!a_ctx->a_key) 746 return -ENOMEM; 747 748 return 0; 749 } 750 751 static void sec_auth_uninit(struct sec_ctx *ctx) 752 { 753 struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 754 755 memzero_explicit(a_ctx->a_key, SEC_MAX_AKEY_SIZE); 756 dma_free_coherent(ctx->dev, SEC_MAX_AKEY_SIZE, 757 a_ctx->a_key, a_ctx->a_key_dma); 758 } 759 760 static int sec_skcipher_fbtfm_init(struct crypto_skcipher *tfm) 761 { 762 const char *alg = crypto_tfm_alg_name(&tfm->base); 763 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 764 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 765 766 c_ctx->fallback = false; 767 768 c_ctx->fbtfm = crypto_alloc_sync_skcipher(alg, 0, 769 CRYPTO_ALG_NEED_FALLBACK); 770 if (IS_ERR(c_ctx->fbtfm)) { 771 pr_err("failed to alloc fallback tfm for %s!\n", alg); 772 return PTR_ERR(c_ctx->fbtfm); 773 } 774 775 return 0; 776 } 777 778 static int sec_skcipher_init(struct crypto_skcipher *tfm) 779 { 780 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 781 int ret; 782 783 ctx->alg_type = SEC_SKCIPHER; 784 crypto_skcipher_set_reqsize_dma(tfm, sizeof(struct sec_req)); 785 ctx->c_ctx.ivsize = crypto_skcipher_ivsize(tfm); 786 if (ctx->c_ctx.ivsize > SEC_IV_SIZE) { 787 pr_err("get error skcipher iv size!\n"); 788 return -EINVAL; 789 } 790 791 ret = sec_ctx_base_init(ctx); 792 if (ret) 793 return ret; 794 795 ret = sec_cipher_init(ctx); 796 if (ret) 797 goto err_cipher_init; 798 799 ret = sec_skcipher_fbtfm_init(tfm); 800 if (ret) 801 goto err_fbtfm_init; 802 803 return 0; 804 805 err_fbtfm_init: 806 sec_cipher_uninit(ctx); 807 err_cipher_init: 808 sec_ctx_base_uninit(ctx); 809 return ret; 810 } 811 812 static void sec_skcipher_uninit(struct crypto_skcipher *tfm) 813 { 814 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 815 816 if (ctx->c_ctx.fbtfm) 817 crypto_free_sync_skcipher(ctx->c_ctx.fbtfm); 818 819 sec_cipher_uninit(ctx); 820 sec_ctx_base_uninit(ctx); 821 } 822 823 static int sec_skcipher_3des_setkey(struct crypto_skcipher *tfm, const u8 *key, const u32 keylen) 824 { 825 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 826 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 827 int ret; 828 829 ret = verify_skcipher_des3_key(tfm, key); 830 if (ret) 831 return ret; 832 833 switch (keylen) { 834 case SEC_DES3_2KEY_SIZE: 835 c_ctx->c_key_len = SEC_CKEY_3DES_2KEY; 836 break; 837 case SEC_DES3_3KEY_SIZE: 838 c_ctx->c_key_len = SEC_CKEY_3DES_3KEY; 839 break; 840 default: 841 return -EINVAL; 842 } 843 844 return 0; 845 } 846 847 static int sec_skcipher_aes_sm4_setkey(struct sec_cipher_ctx *c_ctx, 848 const u32 keylen, 849 const enum sec_cmode c_mode) 850 { 851 if (c_mode == SEC_CMODE_XTS) { 852 switch (keylen) { 853 case SEC_XTS_MIN_KEY_SIZE: 854 c_ctx->c_key_len = SEC_CKEY_128BIT; 855 break; 856 case SEC_XTS_MID_KEY_SIZE: 857 c_ctx->fallback = true; 858 break; 859 case SEC_XTS_MAX_KEY_SIZE: 860 c_ctx->c_key_len = SEC_CKEY_256BIT; 861 break; 862 default: 863 pr_err("hisi_sec2: xts mode key error!\n"); 864 return -EINVAL; 865 } 866 } else { 867 if (c_ctx->c_alg == SEC_CALG_SM4 && 868 keylen != AES_KEYSIZE_128) { 869 pr_err("hisi_sec2: sm4 key error!\n"); 870 return -EINVAL; 871 } else { 872 switch (keylen) { 873 case AES_KEYSIZE_128: 874 c_ctx->c_key_len = SEC_CKEY_128BIT; 875 break; 876 case AES_KEYSIZE_192: 877 c_ctx->c_key_len = SEC_CKEY_192BIT; 878 break; 879 case AES_KEYSIZE_256: 880 c_ctx->c_key_len = SEC_CKEY_256BIT; 881 break; 882 default: 883 pr_err("hisi_sec2: aes key error!\n"); 884 return -EINVAL; 885 } 886 } 887 } 888 889 return 0; 890 } 891 892 static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, 893 const u32 keylen, const enum sec_calg c_alg, 894 const enum sec_cmode c_mode) 895 { 896 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 897 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 898 struct device *dev = ctx->dev; 899 int ret; 900 901 if (c_mode == SEC_CMODE_XTS) { 902 ret = xts_verify_key(tfm, key, keylen); 903 if (ret) { 904 dev_err(dev, "xts mode key err!\n"); 905 return ret; 906 } 907 } 908 909 c_ctx->c_alg = c_alg; 910 c_ctx->c_mode = c_mode; 911 912 switch (c_alg) { 913 case SEC_CALG_3DES: 914 ret = sec_skcipher_3des_setkey(tfm, key, keylen); 915 break; 916 case SEC_CALG_AES: 917 case SEC_CALG_SM4: 918 ret = sec_skcipher_aes_sm4_setkey(c_ctx, keylen, c_mode); 919 break; 920 default: 921 dev_err(dev, "sec c_alg err!\n"); 922 return -EINVAL; 923 } 924 925 if (ret) { 926 dev_err(dev, "set sec key err!\n"); 927 return ret; 928 } 929 930 memcpy(c_ctx->c_key, key, keylen); 931 if (c_ctx->fbtfm) { 932 ret = crypto_sync_skcipher_setkey(c_ctx->fbtfm, key, keylen); 933 if (ret) { 934 dev_err(dev, "failed to set fallback skcipher key!\n"); 935 return ret; 936 } 937 } 938 return 0; 939 } 940 941 #define GEN_SEC_SETKEY_FUNC(name, c_alg, c_mode) \ 942 static int sec_setkey_##name(struct crypto_skcipher *tfm, const u8 *key,\ 943 u32 keylen) \ 944 { \ 945 return sec_skcipher_setkey(tfm, key, keylen, c_alg, c_mode); \ 946 } 947 948 GEN_SEC_SETKEY_FUNC(aes_ecb, SEC_CALG_AES, SEC_CMODE_ECB) 949 GEN_SEC_SETKEY_FUNC(aes_cbc, SEC_CALG_AES, SEC_CMODE_CBC) 950 GEN_SEC_SETKEY_FUNC(aes_xts, SEC_CALG_AES, SEC_CMODE_XTS) 951 GEN_SEC_SETKEY_FUNC(aes_ctr, SEC_CALG_AES, SEC_CMODE_CTR) 952 GEN_SEC_SETKEY_FUNC(3des_ecb, SEC_CALG_3DES, SEC_CMODE_ECB) 953 GEN_SEC_SETKEY_FUNC(3des_cbc, SEC_CALG_3DES, SEC_CMODE_CBC) 954 GEN_SEC_SETKEY_FUNC(sm4_xts, SEC_CALG_SM4, SEC_CMODE_XTS) 955 GEN_SEC_SETKEY_FUNC(sm4_cbc, SEC_CALG_SM4, SEC_CMODE_CBC) 956 GEN_SEC_SETKEY_FUNC(sm4_ctr, SEC_CALG_SM4, SEC_CMODE_CTR) 957 958 static int sec_cipher_pbuf_map(struct sec_ctx *ctx, struct sec_req *req, 959 struct scatterlist *src) 960 { 961 struct aead_request *aead_req = req->aead_req.aead_req; 962 struct sec_cipher_req *c_req = &req->c_req; 963 struct sec_qp_ctx *qp_ctx = req->qp_ctx; 964 struct sec_request_buf *buf = &req->buf; 965 struct device *dev = ctx->dev; 966 int copy_size, pbuf_length; 967 int req_id = req->req_id; 968 struct crypto_aead *tfm; 969 u8 *mac_offset, *pbuf; 970 size_t authsize; 971 972 if (ctx->alg_type == SEC_AEAD) 973 copy_size = aead_req->cryptlen + aead_req->assoclen; 974 else 975 copy_size = c_req->c_len; 976 977 978 pbuf = req->req_id < 0 ? buf->pbuf : qp_ctx->res[req_id].pbuf; 979 pbuf_length = sg_copy_to_buffer(src, sg_nents(src), pbuf, copy_size); 980 if (unlikely(pbuf_length != copy_size)) { 981 dev_err(dev, "copy src data to pbuf error!\n"); 982 return -EINVAL; 983 } 984 if (!c_req->encrypt && ctx->alg_type == SEC_AEAD) { 985 tfm = crypto_aead_reqtfm(aead_req); 986 authsize = crypto_aead_authsize(tfm); 987 mac_offset = pbuf + copy_size - authsize; 988 memcpy(req->aead_req.out_mac, mac_offset, authsize); 989 } 990 991 if (req->req_id < 0) { 992 buf->in_dma = dma_map_single(dev, buf->pbuf, SEC_PBUF_SZ, DMA_BIDIRECTIONAL); 993 if (unlikely(dma_mapping_error(dev, buf->in_dma))) 994 return -ENOMEM; 995 996 buf->out_dma = buf->in_dma; 997 return 0; 998 } 999 1000 req->in_dma = qp_ctx->res[req_id].pbuf_dma; 1001 c_req->c_out_dma = req->in_dma; 1002 1003 return 0; 1004 } 1005 1006 static void sec_cipher_pbuf_unmap(struct sec_ctx *ctx, struct sec_req *req, 1007 struct scatterlist *dst) 1008 { 1009 struct aead_request *aead_req = req->aead_req.aead_req; 1010 struct sec_cipher_req *c_req = &req->c_req; 1011 struct sec_qp_ctx *qp_ctx = req->qp_ctx; 1012 struct sec_request_buf *buf = &req->buf; 1013 int copy_size, pbuf_length; 1014 int req_id = req->req_id; 1015 1016 if (ctx->alg_type == SEC_AEAD) 1017 copy_size = c_req->c_len + aead_req->assoclen; 1018 else 1019 copy_size = c_req->c_len; 1020 1021 if (req->req_id < 0) 1022 pbuf_length = sg_copy_from_buffer(dst, sg_nents(dst), buf->pbuf, copy_size); 1023 else 1024 pbuf_length = sg_copy_from_buffer(dst, sg_nents(dst), qp_ctx->res[req_id].pbuf, 1025 copy_size); 1026 if (unlikely(pbuf_length != copy_size)) 1027 dev_err(ctx->dev, "copy pbuf data to dst error!\n"); 1028 1029 if (req->req_id < 0) 1030 dma_unmap_single(ctx->dev, buf->in_dma, SEC_PBUF_SZ, DMA_BIDIRECTIONAL); 1031 } 1032 1033 static int sec_aead_mac_init(struct sec_aead_req *req) 1034 { 1035 struct aead_request *aead_req = req->aead_req; 1036 struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req); 1037 size_t authsize = crypto_aead_authsize(tfm); 1038 struct scatterlist *sgl = aead_req->src; 1039 u8 *mac_out = req->out_mac; 1040 size_t copy_size; 1041 off_t skip_size; 1042 1043 /* Copy input mac */ 1044 skip_size = aead_req->assoclen + aead_req->cryptlen - authsize; 1045 copy_size = sg_pcopy_to_buffer(sgl, sg_nents(sgl), mac_out, authsize, skip_size); 1046 if (unlikely(copy_size != authsize)) 1047 return -EINVAL; 1048 1049 return 0; 1050 } 1051 1052 static void fill_sg_to_hw_sge(struct scatterlist *sgl, struct sec_hw_sge *hw_sge) 1053 { 1054 hw_sge->buf = sg_dma_address(sgl); 1055 hw_sge->len = cpu_to_le32(sg_dma_len(sgl)); 1056 hw_sge->page_ctrl = sg_virt(sgl); 1057 } 1058 1059 static int sec_cipher_to_hw_sgl(struct device *dev, struct scatterlist *src, 1060 struct sec_hw_sgl *src_in, dma_addr_t *hw_sgl_dma, 1061 int dma_dir) 1062 { 1063 struct sec_hw_sge *curr_hw_sge = src_in->sge_entries; 1064 u32 i, sg_n, sg_n_mapped; 1065 struct scatterlist *sg; 1066 u32 sge_var = 0; 1067 1068 sg_n = sg_nents(src); 1069 sg_n_mapped = dma_map_sg(dev, src, sg_n, dma_dir); 1070 if (unlikely(!sg_n_mapped)) { 1071 dev_err(dev, "dma mapping for SG error!\n"); 1072 return -EINVAL; 1073 } else if (unlikely(sg_n_mapped > SEC_SGE_NR_NUM)) { 1074 dev_err(dev, "the number of entries in input scatterlist error!\n"); 1075 dma_unmap_sg(dev, src, sg_n, dma_dir); 1076 return -EINVAL; 1077 } 1078 1079 for_each_sg(src, sg, sg_n_mapped, i) { 1080 fill_sg_to_hw_sge(sg, curr_hw_sge); 1081 curr_hw_sge++; 1082 sge_var++; 1083 } 1084 1085 src_in->entry_sum_in_sgl = cpu_to_le16(sge_var); 1086 src_in->entry_sum_in_chain = cpu_to_le16(SEC_SGE_NR_NUM); 1087 src_in->entry_length_in_sgl = cpu_to_le16(SEC_SGE_NR_NUM); 1088 *hw_sgl_dma = dma_map_single(dev, src_in, sizeof(struct sec_hw_sgl), dma_dir); 1089 if (unlikely(dma_mapping_error(dev, *hw_sgl_dma))) { 1090 dma_unmap_sg(dev, src, sg_n, dma_dir); 1091 return -ENOMEM; 1092 } 1093 1094 return 0; 1095 } 1096 1097 static void sec_cipher_put_hw_sgl(struct device *dev, struct scatterlist *src, 1098 dma_addr_t src_in, int dma_dir) 1099 { 1100 dma_unmap_single(dev, src_in, sizeof(struct sec_hw_sgl), dma_dir); 1101 dma_unmap_sg(dev, src, sg_nents(src), dma_dir); 1102 } 1103 1104 static int sec_cipher_map_sgl(struct device *dev, struct sec_req *req, 1105 struct scatterlist *src, struct scatterlist *dst) 1106 { 1107 struct sec_hw_sgl *src_in = &req->buf.data_buf.in; 1108 struct sec_hw_sgl *dst_out = &req->buf.data_buf.out; 1109 int ret; 1110 1111 if (dst == src) { 1112 ret = sec_cipher_to_hw_sgl(dev, src, src_in, &req->buf.in_dma, 1113 DMA_BIDIRECTIONAL); 1114 req->buf.out_dma = req->buf.in_dma; 1115 return ret; 1116 } 1117 1118 ret = sec_cipher_to_hw_sgl(dev, src, src_in, &req->buf.in_dma, DMA_TO_DEVICE); 1119 if (unlikely(ret)) 1120 return ret; 1121 1122 ret = sec_cipher_to_hw_sgl(dev, dst, dst_out, &req->buf.out_dma, 1123 DMA_FROM_DEVICE); 1124 if (unlikely(ret)) { 1125 sec_cipher_put_hw_sgl(dev, src, req->buf.in_dma, DMA_TO_DEVICE); 1126 return ret; 1127 } 1128 1129 return 0; 1130 } 1131 1132 static int sec_cipher_map_inner(struct sec_ctx *ctx, struct sec_req *req, 1133 struct scatterlist *src, struct scatterlist *dst) 1134 { 1135 struct sec_cipher_req *c_req = &req->c_req; 1136 struct sec_aead_req *a_req = &req->aead_req; 1137 struct sec_qp_ctx *qp_ctx = req->qp_ctx; 1138 struct sec_alg_res *res = &qp_ctx->res[req->req_id]; 1139 struct device *dev = ctx->dev; 1140 enum dma_data_direction src_direction; 1141 int ret; 1142 1143 if (req->use_pbuf) { 1144 c_req->c_ivin = res->pbuf + SEC_PBUF_IV_OFFSET; 1145 c_req->c_ivin_dma = res->pbuf_dma + SEC_PBUF_IV_OFFSET; 1146 if (ctx->alg_type == SEC_AEAD) { 1147 a_req->a_ivin = res->a_ivin; 1148 a_req->a_ivin_dma = res->a_ivin_dma; 1149 a_req->out_mac = res->pbuf + SEC_PBUF_MAC_OFFSET; 1150 a_req->out_mac_dma = res->pbuf_dma + 1151 SEC_PBUF_MAC_OFFSET; 1152 } 1153 return sec_cipher_pbuf_map(ctx, req, src); 1154 } 1155 1156 c_req->c_ivin = res->c_ivin; 1157 c_req->c_ivin_dma = res->c_ivin_dma; 1158 if (ctx->alg_type == SEC_AEAD) { 1159 a_req->a_ivin = res->a_ivin; 1160 a_req->a_ivin_dma = res->a_ivin_dma; 1161 a_req->out_mac = res->out_mac; 1162 a_req->out_mac_dma = res->out_mac_dma; 1163 } 1164 1165 src_direction = dst == src ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 1166 req->in = hisi_acc_sg_buf_map_to_hw_sgl(dev, src, 1167 qp_ctx->c_in_pool, 1168 req->req_id, 1169 &req->in_dma, src_direction); 1170 if (IS_ERR(req->in)) { 1171 dev_err(dev, "fail to dma map input sgl buffers!\n"); 1172 return PTR_ERR(req->in); 1173 } 1174 1175 if (!c_req->encrypt && ctx->alg_type == SEC_AEAD) { 1176 ret = sec_aead_mac_init(a_req); 1177 if (unlikely(ret)) { 1178 dev_err(dev, "fail to init mac data for ICV!\n"); 1179 hisi_acc_sg_buf_unmap(dev, src, req->in, src_direction); 1180 return ret; 1181 } 1182 } 1183 1184 if (dst == src) { 1185 c_req->c_out = req->in; 1186 c_req->c_out_dma = req->in_dma; 1187 } else { 1188 c_req->c_out = hisi_acc_sg_buf_map_to_hw_sgl(dev, dst, 1189 qp_ctx->c_out_pool, 1190 req->req_id, 1191 &c_req->c_out_dma, 1192 DMA_FROM_DEVICE); 1193 1194 if (IS_ERR(c_req->c_out)) { 1195 dev_err(dev, "fail to dma map output sgl buffers!\n"); 1196 hisi_acc_sg_buf_unmap(dev, src, req->in, src_direction); 1197 return PTR_ERR(c_req->c_out); 1198 } 1199 } 1200 1201 return 0; 1202 } 1203 1204 static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req, 1205 struct scatterlist *src, struct scatterlist *dst) 1206 { 1207 struct sec_aead_req *a_req = &req->aead_req; 1208 struct sec_cipher_req *c_req = &req->c_req; 1209 bool is_aead = (ctx->alg_type == SEC_AEAD); 1210 struct device *dev = ctx->dev; 1211 int ret = -ENOMEM; 1212 1213 if (req->req_id >= 0) 1214 return sec_cipher_map_inner(ctx, req, src, dst); 1215 1216 c_req->c_ivin = c_req->c_ivin_buf; 1217 c_req->c_ivin_dma = dma_map_single(dev, c_req->c_ivin, 1218 SEC_IV_SIZE, DMA_TO_DEVICE); 1219 if (unlikely(dma_mapping_error(dev, c_req->c_ivin_dma))) 1220 return -ENOMEM; 1221 1222 if (is_aead) { 1223 a_req->a_ivin = a_req->a_ivin_buf; 1224 a_req->out_mac = a_req->out_mac_buf; 1225 a_req->a_ivin_dma = dma_map_single(dev, a_req->a_ivin, 1226 SEC_IV_SIZE, DMA_TO_DEVICE); 1227 if (unlikely(dma_mapping_error(dev, a_req->a_ivin_dma))) 1228 goto free_c_ivin_dma; 1229 1230 a_req->out_mac_dma = dma_map_single(dev, a_req->out_mac, 1231 SEC_MAX_MAC_LEN, DMA_BIDIRECTIONAL); 1232 if (unlikely(dma_mapping_error(dev, a_req->out_mac_dma))) 1233 goto free_a_ivin_dma; 1234 } 1235 if (req->use_pbuf) { 1236 ret = sec_cipher_pbuf_map(ctx, req, src); 1237 if (unlikely(ret)) 1238 goto free_out_mac_dma; 1239 1240 return 0; 1241 } 1242 1243 if (!c_req->encrypt && is_aead) { 1244 ret = sec_aead_mac_init(a_req); 1245 if (unlikely(ret)) { 1246 dev_err(dev, "fail to init mac data for ICV!\n"); 1247 goto free_out_mac_dma; 1248 } 1249 } 1250 1251 ret = sec_cipher_map_sgl(dev, req, src, dst); 1252 if (unlikely(ret)) { 1253 dev_err(dev, "fail to dma map input sgl buffers!\n"); 1254 goto free_out_mac_dma; 1255 } 1256 1257 return 0; 1258 1259 free_out_mac_dma: 1260 if (is_aead) 1261 dma_unmap_single(dev, a_req->out_mac_dma, SEC_MAX_MAC_LEN, DMA_BIDIRECTIONAL); 1262 free_a_ivin_dma: 1263 if (is_aead) 1264 dma_unmap_single(dev, a_req->a_ivin_dma, SEC_IV_SIZE, DMA_TO_DEVICE); 1265 free_c_ivin_dma: 1266 dma_unmap_single(dev, c_req->c_ivin_dma, SEC_IV_SIZE, DMA_TO_DEVICE); 1267 return ret; 1268 } 1269 1270 static void sec_cipher_unmap(struct sec_ctx *ctx, struct sec_req *req, 1271 struct scatterlist *src, struct scatterlist *dst) 1272 { 1273 struct sec_aead_req *a_req = &req->aead_req; 1274 struct sec_cipher_req *c_req = &req->c_req; 1275 struct device *dev = ctx->dev; 1276 1277 if (req->req_id >= 0) { 1278 if (req->use_pbuf) { 1279 sec_cipher_pbuf_unmap(ctx, req, dst); 1280 } else { 1281 if (dst != src) { 1282 hisi_acc_sg_buf_unmap(dev, dst, c_req->c_out, DMA_FROM_DEVICE); 1283 hisi_acc_sg_buf_unmap(dev, src, req->in, DMA_TO_DEVICE); 1284 } else { 1285 hisi_acc_sg_buf_unmap(dev, src, req->in, DMA_BIDIRECTIONAL); 1286 } 1287 } 1288 return; 1289 } 1290 1291 if (req->use_pbuf) { 1292 sec_cipher_pbuf_unmap(ctx, req, dst); 1293 } else { 1294 if (dst != src) { 1295 sec_cipher_put_hw_sgl(dev, dst, req->buf.out_dma, DMA_FROM_DEVICE); 1296 sec_cipher_put_hw_sgl(dev, src, req->buf.in_dma, DMA_TO_DEVICE); 1297 } else { 1298 sec_cipher_put_hw_sgl(dev, src, req->buf.in_dma, DMA_BIDIRECTIONAL); 1299 } 1300 } 1301 1302 dma_unmap_single(dev, c_req->c_ivin_dma, SEC_IV_SIZE, DMA_TO_DEVICE); 1303 if (ctx->alg_type == SEC_AEAD) { 1304 dma_unmap_single(dev, a_req->a_ivin_dma, SEC_IV_SIZE, DMA_TO_DEVICE); 1305 dma_unmap_single(dev, a_req->out_mac_dma, SEC_MAX_MAC_LEN, DMA_BIDIRECTIONAL); 1306 } 1307 } 1308 1309 static int sec_skcipher_sgl_map(struct sec_ctx *ctx, struct sec_req *req) 1310 { 1311 struct skcipher_request *sq = req->c_req.sk_req; 1312 1313 return sec_cipher_map(ctx, req, sq->src, sq->dst); 1314 } 1315 1316 static void sec_skcipher_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req) 1317 { 1318 struct skcipher_request *sq = req->c_req.sk_req; 1319 1320 sec_cipher_unmap(ctx, req, sq->src, sq->dst); 1321 } 1322 1323 static int sec_aead_aes_set_key(struct sec_cipher_ctx *c_ctx, 1324 struct crypto_authenc_keys *keys) 1325 { 1326 switch (keys->enckeylen) { 1327 case AES_KEYSIZE_128: 1328 c_ctx->c_key_len = SEC_CKEY_128BIT; 1329 break; 1330 case AES_KEYSIZE_192: 1331 c_ctx->c_key_len = SEC_CKEY_192BIT; 1332 break; 1333 case AES_KEYSIZE_256: 1334 c_ctx->c_key_len = SEC_CKEY_256BIT; 1335 break; 1336 default: 1337 pr_err("hisi_sec2: aead aes key error!\n"); 1338 return -EINVAL; 1339 } 1340 memcpy(c_ctx->c_key, keys->enckey, keys->enckeylen); 1341 1342 return 0; 1343 } 1344 1345 static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx, 1346 struct crypto_authenc_keys *keys) 1347 { 1348 struct crypto_shash *hash_tfm = ctx->hash_tfm; 1349 int blocksize, digestsize, ret; 1350 1351 blocksize = crypto_shash_blocksize(hash_tfm); 1352 digestsize = crypto_shash_digestsize(hash_tfm); 1353 if (keys->authkeylen > blocksize) { 1354 ret = crypto_shash_tfm_digest(hash_tfm, keys->authkey, 1355 keys->authkeylen, ctx->a_key); 1356 if (ret) { 1357 pr_err("hisi_sec2: aead auth digest error!\n"); 1358 return -EINVAL; 1359 } 1360 ctx->a_key_len = digestsize; 1361 } else { 1362 if (keys->authkeylen) 1363 memcpy(ctx->a_key, keys->authkey, keys->authkeylen); 1364 ctx->a_key_len = keys->authkeylen; 1365 } 1366 1367 return 0; 1368 } 1369 1370 static int sec_aead_setauthsize(struct crypto_aead *aead, unsigned int authsize) 1371 { 1372 struct crypto_tfm *tfm = crypto_aead_tfm(aead); 1373 struct sec_ctx *ctx = crypto_tfm_ctx(tfm); 1374 struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 1375 1376 return crypto_aead_setauthsize(a_ctx->fallback_aead_tfm, authsize); 1377 } 1378 1379 static int sec_aead_fallback_setkey(struct sec_auth_ctx *a_ctx, 1380 struct crypto_aead *tfm, const u8 *key, 1381 unsigned int keylen) 1382 { 1383 crypto_aead_clear_flags(a_ctx->fallback_aead_tfm, CRYPTO_TFM_REQ_MASK); 1384 crypto_aead_set_flags(a_ctx->fallback_aead_tfm, 1385 crypto_aead_get_flags(tfm) & CRYPTO_TFM_REQ_MASK); 1386 return crypto_aead_setkey(a_ctx->fallback_aead_tfm, key, keylen); 1387 } 1388 1389 static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, 1390 const u32 keylen, const enum sec_hash_alg a_alg, 1391 const enum sec_calg c_alg, 1392 const enum sec_cmode c_mode) 1393 { 1394 struct sec_ctx *ctx = crypto_aead_ctx(tfm); 1395 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 1396 struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 1397 struct device *dev = ctx->dev; 1398 struct crypto_authenc_keys keys; 1399 int ret; 1400 1401 ctx->a_ctx.a_alg = a_alg; 1402 ctx->c_ctx.c_alg = c_alg; 1403 c_ctx->c_mode = c_mode; 1404 1405 if (c_mode == SEC_CMODE_CCM || c_mode == SEC_CMODE_GCM) { 1406 ret = sec_skcipher_aes_sm4_setkey(c_ctx, keylen, c_mode); 1407 if (ret) { 1408 dev_err(dev, "set sec aes ccm cipher key err!\n"); 1409 return ret; 1410 } 1411 memcpy(c_ctx->c_key, key, keylen); 1412 1413 return sec_aead_fallback_setkey(a_ctx, tfm, key, keylen); 1414 } 1415 1416 ret = crypto_authenc_extractkeys(&keys, key, keylen); 1417 if (ret) { 1418 dev_err(dev, "sec extract aead keys err!\n"); 1419 goto bad_key; 1420 } 1421 1422 ret = sec_aead_aes_set_key(c_ctx, &keys); 1423 if (ret) { 1424 dev_err(dev, "set sec cipher key err!\n"); 1425 goto bad_key; 1426 } 1427 1428 ret = sec_aead_auth_set_key(&ctx->a_ctx, &keys); 1429 if (ret) { 1430 dev_err(dev, "set sec auth key err!\n"); 1431 goto bad_key; 1432 } 1433 1434 ret = sec_aead_fallback_setkey(a_ctx, tfm, key, keylen); 1435 if (ret) { 1436 dev_err(dev, "set sec fallback key err!\n"); 1437 goto bad_key; 1438 } 1439 1440 return 0; 1441 1442 bad_key: 1443 memzero_explicit(&keys, sizeof(struct crypto_authenc_keys)); 1444 return ret; 1445 } 1446 1447 1448 #define GEN_SEC_AEAD_SETKEY_FUNC(name, aalg, calg, cmode) \ 1449 static int sec_setkey_##name(struct crypto_aead *tfm, const u8 *key, u32 keylen) \ 1450 { \ 1451 return sec_aead_setkey(tfm, key, keylen, aalg, calg, cmode); \ 1452 } 1453 1454 GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha1, SEC_A_HMAC_SHA1, SEC_CALG_AES, SEC_CMODE_CBC) 1455 GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha256, SEC_A_HMAC_SHA256, SEC_CALG_AES, SEC_CMODE_CBC) 1456 GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha512, SEC_A_HMAC_SHA512, SEC_CALG_AES, SEC_CMODE_CBC) 1457 GEN_SEC_AEAD_SETKEY_FUNC(aes_ccm, 0, SEC_CALG_AES, SEC_CMODE_CCM) 1458 GEN_SEC_AEAD_SETKEY_FUNC(aes_gcm, 0, SEC_CALG_AES, SEC_CMODE_GCM) 1459 GEN_SEC_AEAD_SETKEY_FUNC(sm4_ccm, 0, SEC_CALG_SM4, SEC_CMODE_CCM) 1460 GEN_SEC_AEAD_SETKEY_FUNC(sm4_gcm, 0, SEC_CALG_SM4, SEC_CMODE_GCM) 1461 1462 static int sec_aead_sgl_map(struct sec_ctx *ctx, struct sec_req *req) 1463 { 1464 struct aead_request *aq = req->aead_req.aead_req; 1465 1466 return sec_cipher_map(ctx, req, aq->src, aq->dst); 1467 } 1468 1469 static void sec_aead_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req) 1470 { 1471 struct aead_request *aq = req->aead_req.aead_req; 1472 1473 sec_cipher_unmap(ctx, req, aq->src, aq->dst); 1474 } 1475 1476 static int sec_request_transfer(struct sec_ctx *ctx, struct sec_req *req) 1477 { 1478 int ret; 1479 1480 ret = ctx->req_op->buf_map(ctx, req); 1481 if (unlikely(ret)) 1482 return ret; 1483 1484 ctx->req_op->do_transfer(ctx, req); 1485 1486 ret = ctx->req_op->bd_fill(ctx, req); 1487 if (unlikely(ret)) 1488 goto unmap_req_buf; 1489 1490 return ret; 1491 1492 unmap_req_buf: 1493 ctx->req_op->buf_unmap(ctx, req); 1494 return ret; 1495 } 1496 1497 static void sec_request_untransfer(struct sec_ctx *ctx, struct sec_req *req) 1498 { 1499 ctx->req_op->buf_unmap(ctx, req); 1500 } 1501 1502 static void sec_skcipher_copy_iv(struct sec_ctx *ctx, struct sec_req *req) 1503 { 1504 struct skcipher_request *sk_req = req->c_req.sk_req; 1505 struct sec_cipher_req *c_req = &req->c_req; 1506 1507 memcpy(c_req->c_ivin, sk_req->iv, ctx->c_ctx.ivsize); 1508 } 1509 1510 static int sec_skcipher_bd_fill(struct sec_ctx *ctx, struct sec_req *req) 1511 { 1512 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 1513 struct sec_cipher_req *c_req = &req->c_req; 1514 struct sec_sqe *sec_sqe = &req->sec_sqe; 1515 u8 scene, sa_type, da_type; 1516 u8 bd_type, cipher; 1517 u8 de = 0; 1518 1519 memset(sec_sqe, 0, sizeof(struct sec_sqe)); 1520 1521 sec_sqe->type2.c_key_addr = cpu_to_le64(c_ctx->c_key_dma); 1522 sec_sqe->type2.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma); 1523 if (req->req_id < 0) { 1524 sec_sqe->type2.data_src_addr = cpu_to_le64(req->buf.in_dma); 1525 sec_sqe->type2.data_dst_addr = cpu_to_le64(req->buf.out_dma); 1526 } else { 1527 sec_sqe->type2.data_src_addr = cpu_to_le64(req->in_dma); 1528 sec_sqe->type2.data_dst_addr = cpu_to_le64(c_req->c_out_dma); 1529 } 1530 if (sec_sqe->type2.data_src_addr != sec_sqe->type2.data_dst_addr) 1531 de = 0x1 << SEC_DE_OFFSET; 1532 1533 sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_mode) << 1534 SEC_CMODE_OFFSET); 1535 sec_sqe->type2.c_alg = c_ctx->c_alg; 1536 sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_key_len) << 1537 SEC_CKEY_OFFSET); 1538 1539 bd_type = SEC_BD_TYPE2; 1540 if (c_req->encrypt) 1541 cipher = SEC_CIPHER_ENC << SEC_CIPHER_OFFSET; 1542 else 1543 cipher = SEC_CIPHER_DEC << SEC_CIPHER_OFFSET; 1544 sec_sqe->type_cipher_auth = bd_type | cipher; 1545 1546 /* Set destination and source address type */ 1547 if (req->use_pbuf) { 1548 sa_type = SEC_PBUF << SEC_SRC_SGL_OFFSET; 1549 da_type = SEC_PBUF << SEC_DST_SGL_OFFSET; 1550 } else { 1551 sa_type = SEC_SGL << SEC_SRC_SGL_OFFSET; 1552 da_type = SEC_SGL << SEC_DST_SGL_OFFSET; 1553 } 1554 1555 sec_sqe->sdm_addr_type |= da_type; 1556 scene = SEC_COMM_SCENE << SEC_SCENE_OFFSET; 1557 1558 sec_sqe->sds_sa_type = (de | scene | sa_type); 1559 1560 sec_sqe->type2.clen_ivhlen |= cpu_to_le32(c_req->c_len); 1561 1562 return 0; 1563 } 1564 1565 static int sec_skcipher_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req) 1566 { 1567 struct sec_sqe3 *sec_sqe3 = &req->sec_sqe3; 1568 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 1569 struct sec_cipher_req *c_req = &req->c_req; 1570 u32 bd_param = 0; 1571 u16 cipher; 1572 1573 memset(sec_sqe3, 0, sizeof(struct sec_sqe3)); 1574 1575 sec_sqe3->c_key_addr = cpu_to_le64(c_ctx->c_key_dma); 1576 sec_sqe3->no_scene.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma); 1577 if (req->req_id < 0) { 1578 sec_sqe3->data_src_addr = cpu_to_le64(req->buf.in_dma); 1579 sec_sqe3->data_dst_addr = cpu_to_le64(req->buf.out_dma); 1580 } else { 1581 sec_sqe3->data_src_addr = cpu_to_le64(req->in_dma); 1582 sec_sqe3->data_dst_addr = cpu_to_le64(c_req->c_out_dma); 1583 } 1584 if (sec_sqe3->data_src_addr != sec_sqe3->data_dst_addr) 1585 bd_param |= 0x1 << SEC_DE_OFFSET_V3; 1586 1587 sec_sqe3->c_mode_alg = ((u8)c_ctx->c_alg << SEC_CALG_OFFSET_V3) | 1588 c_ctx->c_mode; 1589 sec_sqe3->c_icv_key |= cpu_to_le16(((u16)c_ctx->c_key_len) << 1590 SEC_CKEY_OFFSET_V3); 1591 1592 if (c_req->encrypt) 1593 cipher = SEC_CIPHER_ENC; 1594 else 1595 cipher = SEC_CIPHER_DEC; 1596 sec_sqe3->c_icv_key |= cpu_to_le16(cipher); 1597 1598 /* Set the CTR counter mode is 128bit rollover */ 1599 sec_sqe3->auth_mac_key = cpu_to_le32((u32)SEC_CTR_CNT_ROLLOVER << 1600 SEC_CTR_CNT_OFFSET); 1601 1602 if (req->use_pbuf) { 1603 bd_param |= SEC_PBUF << SEC_SRC_SGL_OFFSET_V3; 1604 bd_param |= SEC_PBUF << SEC_DST_SGL_OFFSET_V3; 1605 } else { 1606 bd_param |= SEC_SGL << SEC_SRC_SGL_OFFSET_V3; 1607 bd_param |= SEC_SGL << SEC_DST_SGL_OFFSET_V3; 1608 } 1609 1610 bd_param |= SEC_COMM_SCENE << SEC_SCENE_OFFSET_V3; 1611 1612 bd_param |= SEC_BD_TYPE3; 1613 sec_sqe3->bd_param = cpu_to_le32(bd_param); 1614 1615 sec_sqe3->c_len_ivin |= cpu_to_le32(c_req->c_len); 1616 sec_sqe3->tag = cpu_to_le64((unsigned long)req); 1617 1618 return 0; 1619 } 1620 1621 /* increment counter (128-bit int) */ 1622 static void ctr_iv_inc(__u8 *counter, __u8 bits, __u32 nums) 1623 { 1624 do { 1625 --bits; 1626 nums += counter[bits]; 1627 counter[bits] = nums & BITS_MASK; 1628 nums >>= BYTE_BITS; 1629 } while (bits && nums); 1630 } 1631 1632 static void sec_update_iv(struct sec_req *req, enum sec_alg_type alg_type) 1633 { 1634 struct aead_request *aead_req = req->aead_req.aead_req; 1635 struct skcipher_request *sk_req = req->c_req.sk_req; 1636 u32 iv_size = req->ctx->c_ctx.ivsize; 1637 struct scatterlist *sgl; 1638 unsigned int cryptlen; 1639 size_t sz; 1640 u8 *iv; 1641 1642 if (alg_type == SEC_SKCIPHER) { 1643 sgl = req->c_req.encrypt ? sk_req->dst : sk_req->src; 1644 iv = sk_req->iv; 1645 cryptlen = sk_req->cryptlen; 1646 } else { 1647 sgl = req->c_req.encrypt ? aead_req->dst : aead_req->src; 1648 iv = aead_req->iv; 1649 cryptlen = aead_req->cryptlen; 1650 } 1651 1652 if (req->ctx->c_ctx.c_mode == SEC_CMODE_CBC) { 1653 sz = sg_pcopy_to_buffer(sgl, sg_nents(sgl), iv, iv_size, 1654 cryptlen - iv_size); 1655 if (unlikely(sz != iv_size)) 1656 dev_err(req->ctx->dev, "copy output iv error!\n"); 1657 } else { 1658 sz = (cryptlen + iv_size - 1) / iv_size; 1659 ctr_iv_inc(iv, iv_size, sz); 1660 } 1661 } 1662 1663 static void sec_skcipher_callback(struct sec_ctx *ctx, struct sec_req *req, 1664 int err) 1665 { 1666 struct sec_qp_ctx *qp_ctx = req->qp_ctx; 1667 1668 if (req->req_id >= 0) 1669 sec_free_req_id(req); 1670 1671 /* IV output at encrypto of CBC/CTR mode */ 1672 if (!err && (ctx->c_ctx.c_mode == SEC_CMODE_CBC || 1673 ctx->c_ctx.c_mode == SEC_CMODE_CTR) && req->c_req.encrypt) 1674 sec_update_iv(req, SEC_SKCIPHER); 1675 1676 crypto_request_complete(req->base, err); 1677 sec_alg_send_backlog(ctx, qp_ctx); 1678 } 1679 1680 static void set_aead_auth_iv(struct sec_ctx *ctx, struct sec_req *req) 1681 { 1682 struct aead_request *aead_req = req->aead_req.aead_req; 1683 struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req); 1684 size_t authsize = crypto_aead_authsize(tfm); 1685 struct sec_aead_req *a_req = &req->aead_req; 1686 struct sec_cipher_req *c_req = &req->c_req; 1687 u32 data_size = aead_req->cryptlen; 1688 u8 flage = 0; 1689 u8 cm, cl; 1690 1691 /* the specification has been checked in aead_iv_demension_check() */ 1692 cl = c_req->c_ivin[0] + 1; 1693 c_req->c_ivin[ctx->c_ctx.ivsize - cl] = 0x00; 1694 memset(&c_req->c_ivin[ctx->c_ctx.ivsize - cl], 0, cl); 1695 c_req->c_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE1] = IV_CTR_INIT; 1696 1697 /* the last 3bit is L' */ 1698 flage |= c_req->c_ivin[0] & IV_CL_MASK; 1699 1700 /* the M' is bit3~bit5, the Flags is bit6 */ 1701 cm = (authsize - IV_CM_CAL_NUM) / IV_CM_CAL_NUM; 1702 flage |= cm << IV_CM_OFFSET; 1703 if (aead_req->assoclen) 1704 flage |= 0x01 << IV_FLAGS_OFFSET; 1705 1706 memcpy(a_req->a_ivin, c_req->c_ivin, ctx->c_ctx.ivsize); 1707 a_req->a_ivin[0] = flage; 1708 1709 /* 1710 * the last 32bit is counter's initial number, 1711 * but the nonce uses the first 16bit 1712 * the tail 16bit fill with the cipher length 1713 */ 1714 if (!c_req->encrypt) 1715 data_size = aead_req->cryptlen - authsize; 1716 1717 a_req->a_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE1] = 1718 data_size & IV_LAST_BYTE_MASK; 1719 data_size >>= IV_BYTE_OFFSET; 1720 a_req->a_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE2] = 1721 data_size & IV_LAST_BYTE_MASK; 1722 } 1723 1724 static void sec_aead_set_iv(struct sec_ctx *ctx, struct sec_req *req) 1725 { 1726 struct aead_request *aead_req = req->aead_req.aead_req; 1727 struct sec_aead_req *a_req = &req->aead_req; 1728 struct sec_cipher_req *c_req = &req->c_req; 1729 1730 memcpy(c_req->c_ivin, aead_req->iv, ctx->c_ctx.ivsize); 1731 1732 if (ctx->c_ctx.c_mode == SEC_CMODE_CCM) { 1733 /* 1734 * CCM 16Byte Cipher_IV: {1B_Flage,13B_IV,2B_counter}, 1735 * the counter must set to 0x01 1736 * CCM 16Byte Auth_IV: {1B_AFlage,13B_IV,2B_Ptext_length} 1737 */ 1738 set_aead_auth_iv(ctx, req); 1739 } else if (ctx->c_ctx.c_mode == SEC_CMODE_GCM) { 1740 /* GCM 12Byte Cipher_IV == Auth_IV */ 1741 memcpy(a_req->a_ivin, c_req->c_ivin, SEC_AIV_SIZE); 1742 } 1743 } 1744 1745 static void sec_auth_bd_fill_xcm(struct sec_auth_ctx *ctx, int dir, 1746 struct sec_req *req, struct sec_sqe *sec_sqe) 1747 { 1748 struct sec_aead_req *a_req = &req->aead_req; 1749 struct aead_request *aq = a_req->aead_req; 1750 struct crypto_aead *tfm = crypto_aead_reqtfm(aq); 1751 size_t authsize = crypto_aead_authsize(tfm); 1752 1753 /* C_ICV_Len is MAC size, 0x4 ~ 0x10 */ 1754 sec_sqe->type2.icvw_kmode |= cpu_to_le16((u16)authsize); 1755 1756 /* mode set to CCM/GCM, don't set {A_Alg, AKey_Len, MAC_Len} */ 1757 sec_sqe->type2.a_key_addr = sec_sqe->type2.c_key_addr; 1758 sec_sqe->type2.a_ivin_addr = cpu_to_le64(a_req->a_ivin_dma); 1759 sec_sqe->type_cipher_auth |= SEC_NO_AUTH << SEC_AUTH_OFFSET; 1760 1761 if (dir) 1762 sec_sqe->sds_sa_type &= SEC_CIPHER_AUTH; 1763 else 1764 sec_sqe->sds_sa_type |= SEC_AUTH_CIPHER; 1765 1766 sec_sqe->type2.alen_ivllen = cpu_to_le32(aq->assoclen); 1767 sec_sqe->type2.auth_src_offset = cpu_to_le16(0x0); 1768 sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen); 1769 1770 sec_sqe->type2.mac_addr = cpu_to_le64(a_req->out_mac_dma); 1771 } 1772 1773 static void sec_auth_bd_fill_xcm_v3(struct sec_auth_ctx *ctx, int dir, 1774 struct sec_req *req, struct sec_sqe3 *sqe3) 1775 { 1776 struct sec_aead_req *a_req = &req->aead_req; 1777 struct aead_request *aq = a_req->aead_req; 1778 struct crypto_aead *tfm = crypto_aead_reqtfm(aq); 1779 size_t authsize = crypto_aead_authsize(tfm); 1780 1781 /* C_ICV_Len is MAC size, 0x4 ~ 0x10 */ 1782 sqe3->c_icv_key |= cpu_to_le16((u16)authsize << SEC_MAC_OFFSET_V3); 1783 1784 /* mode set to CCM/GCM, don't set {A_Alg, AKey_Len, MAC_Len} */ 1785 sqe3->a_key_addr = sqe3->c_key_addr; 1786 sqe3->auth_ivin.a_ivin_addr = cpu_to_le64(a_req->a_ivin_dma); 1787 sqe3->auth_mac_key |= SEC_NO_AUTH; 1788 1789 if (dir) 1790 sqe3->huk_iv_seq &= SEC_CIPHER_AUTH_V3; 1791 else 1792 sqe3->huk_iv_seq |= SEC_AUTH_CIPHER_V3; 1793 1794 sqe3->a_len_key = cpu_to_le32(aq->assoclen); 1795 sqe3->auth_src_offset = cpu_to_le16(0x0); 1796 sqe3->cipher_src_offset = cpu_to_le16((u16)aq->assoclen); 1797 sqe3->mac_addr = cpu_to_le64(a_req->out_mac_dma); 1798 } 1799 1800 static void sec_auth_bd_fill_ex(struct sec_auth_ctx *ctx, int dir, 1801 struct sec_req *req, struct sec_sqe *sec_sqe) 1802 { 1803 struct sec_aead_req *a_req = &req->aead_req; 1804 struct sec_cipher_req *c_req = &req->c_req; 1805 struct aead_request *aq = a_req->aead_req; 1806 struct crypto_aead *tfm = crypto_aead_reqtfm(aq); 1807 size_t authsize = crypto_aead_authsize(tfm); 1808 1809 sec_sqe->type2.a_key_addr = cpu_to_le64(ctx->a_key_dma); 1810 1811 sec_sqe->type2.mac_key_alg = cpu_to_le32(BYTES_TO_WORDS(authsize)); 1812 1813 sec_sqe->type2.mac_key_alg |= 1814 cpu_to_le32((u32)BYTES_TO_WORDS(ctx->a_key_len) << SEC_AKEY_OFFSET); 1815 1816 sec_sqe->type2.mac_key_alg |= 1817 cpu_to_le32((u32)(ctx->a_alg) << SEC_AEAD_ALG_OFFSET); 1818 1819 if (dir) { 1820 sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE1 << SEC_AUTH_OFFSET; 1821 sec_sqe->sds_sa_type &= SEC_CIPHER_AUTH; 1822 } else { 1823 sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE2 << SEC_AUTH_OFFSET; 1824 sec_sqe->sds_sa_type |= SEC_AUTH_CIPHER; 1825 } 1826 sec_sqe->type2.alen_ivllen = cpu_to_le32(c_req->c_len + aq->assoclen); 1827 1828 sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen); 1829 1830 sec_sqe->type2.mac_addr = cpu_to_le64(a_req->out_mac_dma); 1831 } 1832 1833 static int sec_aead_bd_fill(struct sec_ctx *ctx, struct sec_req *req) 1834 { 1835 struct sec_auth_ctx *auth_ctx = &ctx->a_ctx; 1836 struct sec_sqe *sec_sqe = &req->sec_sqe; 1837 int ret; 1838 1839 ret = sec_skcipher_bd_fill(ctx, req); 1840 if (unlikely(ret)) { 1841 dev_err(ctx->dev, "skcipher bd fill is error!\n"); 1842 return ret; 1843 } 1844 1845 if (ctx->c_ctx.c_mode == SEC_CMODE_CCM || 1846 ctx->c_ctx.c_mode == SEC_CMODE_GCM) 1847 sec_auth_bd_fill_xcm(auth_ctx, req->c_req.encrypt, req, sec_sqe); 1848 else 1849 sec_auth_bd_fill_ex(auth_ctx, req->c_req.encrypt, req, sec_sqe); 1850 1851 return 0; 1852 } 1853 1854 static void sec_auth_bd_fill_ex_v3(struct sec_auth_ctx *ctx, int dir, 1855 struct sec_req *req, struct sec_sqe3 *sqe3) 1856 { 1857 struct sec_aead_req *a_req = &req->aead_req; 1858 struct sec_cipher_req *c_req = &req->c_req; 1859 struct aead_request *aq = a_req->aead_req; 1860 struct crypto_aead *tfm = crypto_aead_reqtfm(aq); 1861 size_t authsize = crypto_aead_authsize(tfm); 1862 1863 sqe3->a_key_addr = cpu_to_le64(ctx->a_key_dma); 1864 1865 sqe3->auth_mac_key |= 1866 cpu_to_le32(BYTES_TO_WORDS(authsize) << SEC_MAC_OFFSET_V3); 1867 1868 sqe3->auth_mac_key |= 1869 cpu_to_le32((u32)BYTES_TO_WORDS(ctx->a_key_len) << SEC_AKEY_OFFSET_V3); 1870 1871 sqe3->auth_mac_key |= 1872 cpu_to_le32((u32)(ctx->a_alg) << SEC_AUTH_ALG_OFFSET_V3); 1873 1874 if (dir) { 1875 sqe3->auth_mac_key |= cpu_to_le32((u32)SEC_AUTH_TYPE1); 1876 sqe3->huk_iv_seq &= SEC_CIPHER_AUTH_V3; 1877 } else { 1878 sqe3->auth_mac_key |= cpu_to_le32((u32)SEC_AUTH_TYPE2); 1879 sqe3->huk_iv_seq |= SEC_AUTH_CIPHER_V3; 1880 } 1881 sqe3->a_len_key = cpu_to_le32(c_req->c_len + aq->assoclen); 1882 1883 sqe3->cipher_src_offset = cpu_to_le16((u16)aq->assoclen); 1884 1885 sqe3->mac_addr = cpu_to_le64(a_req->out_mac_dma); 1886 } 1887 1888 static int sec_aead_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req) 1889 { 1890 struct sec_auth_ctx *auth_ctx = &ctx->a_ctx; 1891 struct sec_sqe3 *sec_sqe3 = &req->sec_sqe3; 1892 int ret; 1893 1894 ret = sec_skcipher_bd_fill_v3(ctx, req); 1895 if (unlikely(ret)) { 1896 dev_err(ctx->dev, "skcipher bd3 fill is error!\n"); 1897 return ret; 1898 } 1899 1900 if (ctx->c_ctx.c_mode == SEC_CMODE_CCM || 1901 ctx->c_ctx.c_mode == SEC_CMODE_GCM) 1902 sec_auth_bd_fill_xcm_v3(auth_ctx, req->c_req.encrypt, 1903 req, sec_sqe3); 1904 else 1905 sec_auth_bd_fill_ex_v3(auth_ctx, req->c_req.encrypt, 1906 req, sec_sqe3); 1907 1908 return 0; 1909 } 1910 1911 static void sec_aead_callback(struct sec_ctx *c, struct sec_req *req, int err) 1912 { 1913 struct aead_request *a_req = req->aead_req.aead_req; 1914 struct crypto_aead *tfm = crypto_aead_reqtfm(a_req); 1915 size_t authsize = crypto_aead_authsize(tfm); 1916 struct sec_qp_ctx *qp_ctx = req->qp_ctx; 1917 size_t sz; 1918 1919 if (!err && req->c_req.encrypt) { 1920 if (c->c_ctx.c_mode == SEC_CMODE_CBC) 1921 sec_update_iv(req, SEC_AEAD); 1922 1923 sz = sg_pcopy_from_buffer(a_req->dst, sg_nents(a_req->dst), req->aead_req.out_mac, 1924 authsize, a_req->cryptlen + a_req->assoclen); 1925 if (unlikely(sz != authsize)) { 1926 dev_err(c->dev, "copy out mac err!\n"); 1927 err = -EINVAL; 1928 } 1929 } 1930 1931 if (req->req_id >= 0) 1932 sec_free_req_id(req); 1933 1934 crypto_request_complete(req->base, err); 1935 sec_alg_send_backlog(c, qp_ctx); 1936 } 1937 1938 static void sec_request_uninit(struct sec_req *req) 1939 { 1940 if (req->req_id >= 0) 1941 sec_free_req_id(req); 1942 } 1943 1944 static int sec_request_init(struct sec_ctx *ctx, struct sec_req *req) 1945 { 1946 struct sec_qp_ctx *qp_ctx; 1947 int i; 1948 1949 for (i = 0; i < ctx->sec->ctx_q_num; i++) { 1950 qp_ctx = &ctx->qp_ctx[i]; 1951 req->req_id = sec_alloc_req_id(req, qp_ctx); 1952 if (req->req_id >= 0) 1953 break; 1954 } 1955 1956 req->qp_ctx = qp_ctx; 1957 req->backlog = &qp_ctx->backlog; 1958 1959 return 0; 1960 } 1961 1962 static int sec_process(struct sec_ctx *ctx, struct sec_req *req) 1963 { 1964 int ret; 1965 1966 ret = sec_request_init(ctx, req); 1967 if (unlikely(ret)) 1968 return ret; 1969 1970 ret = sec_request_transfer(ctx, req); 1971 if (unlikely(ret)) 1972 goto err_uninit_req; 1973 1974 /* Output IV as decrypto */ 1975 if (!req->c_req.encrypt && (ctx->c_ctx.c_mode == SEC_CMODE_CBC || 1976 ctx->c_ctx.c_mode == SEC_CMODE_CTR)) 1977 sec_update_iv(req, ctx->alg_type); 1978 1979 ret = ctx->req_op->bd_send(ctx, req); 1980 if (unlikely((ret != -EBUSY && ret != -EINPROGRESS))) { 1981 dev_err_ratelimited(ctx->dev, "send sec request failed!\n"); 1982 goto err_send_req; 1983 } 1984 1985 return ret; 1986 1987 err_send_req: 1988 /* As failing, restore the IV from user */ 1989 if (ctx->c_ctx.c_mode == SEC_CMODE_CBC && !req->c_req.encrypt) { 1990 if (ctx->alg_type == SEC_SKCIPHER) 1991 memcpy(req->c_req.sk_req->iv, req->c_req.c_ivin, 1992 ctx->c_ctx.ivsize); 1993 else 1994 memcpy(req->aead_req.aead_req->iv, req->c_req.c_ivin, 1995 ctx->c_ctx.ivsize); 1996 } 1997 1998 sec_request_untransfer(ctx, req); 1999 2000 err_uninit_req: 2001 sec_request_uninit(req); 2002 if (ctx->alg_type == SEC_AEAD) 2003 ret = sec_aead_soft_crypto(ctx, req->aead_req.aead_req, 2004 req->c_req.encrypt); 2005 else 2006 ret = sec_skcipher_soft_crypto(ctx, req->c_req.sk_req, 2007 req->c_req.encrypt); 2008 return ret; 2009 } 2010 2011 static const struct sec_req_op sec_skcipher_req_ops = { 2012 .buf_map = sec_skcipher_sgl_map, 2013 .buf_unmap = sec_skcipher_sgl_unmap, 2014 .do_transfer = sec_skcipher_copy_iv, 2015 .bd_fill = sec_skcipher_bd_fill, 2016 .bd_send = sec_bd_send, 2017 .callback = sec_skcipher_callback, 2018 .process = sec_process, 2019 }; 2020 2021 static const struct sec_req_op sec_aead_req_ops = { 2022 .buf_map = sec_aead_sgl_map, 2023 .buf_unmap = sec_aead_sgl_unmap, 2024 .do_transfer = sec_aead_set_iv, 2025 .bd_fill = sec_aead_bd_fill, 2026 .bd_send = sec_bd_send, 2027 .callback = sec_aead_callback, 2028 .process = sec_process, 2029 }; 2030 2031 static const struct sec_req_op sec_skcipher_req_ops_v3 = { 2032 .buf_map = sec_skcipher_sgl_map, 2033 .buf_unmap = sec_skcipher_sgl_unmap, 2034 .do_transfer = sec_skcipher_copy_iv, 2035 .bd_fill = sec_skcipher_bd_fill_v3, 2036 .bd_send = sec_bd_send, 2037 .callback = sec_skcipher_callback, 2038 .process = sec_process, 2039 }; 2040 2041 static const struct sec_req_op sec_aead_req_ops_v3 = { 2042 .buf_map = sec_aead_sgl_map, 2043 .buf_unmap = sec_aead_sgl_unmap, 2044 .do_transfer = sec_aead_set_iv, 2045 .bd_fill = sec_aead_bd_fill_v3, 2046 .bd_send = sec_bd_send, 2047 .callback = sec_aead_callback, 2048 .process = sec_process, 2049 }; 2050 2051 static int sec_skcipher_ctx_init(struct crypto_skcipher *tfm) 2052 { 2053 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 2054 int ret; 2055 2056 ret = sec_skcipher_init(tfm); 2057 if (ret) 2058 return ret; 2059 2060 if (ctx->sec->qm.ver < QM_HW_V3) { 2061 ctx->type_supported = SEC_BD_TYPE2; 2062 ctx->req_op = &sec_skcipher_req_ops; 2063 } else { 2064 ctx->type_supported = SEC_BD_TYPE3; 2065 ctx->req_op = &sec_skcipher_req_ops_v3; 2066 } 2067 2068 return ret; 2069 } 2070 2071 static void sec_skcipher_ctx_exit(struct crypto_skcipher *tfm) 2072 { 2073 sec_skcipher_uninit(tfm); 2074 } 2075 2076 static int sec_aead_init(struct crypto_aead *tfm) 2077 { 2078 struct sec_ctx *ctx = crypto_aead_ctx(tfm); 2079 int ret; 2080 2081 crypto_aead_set_reqsize_dma(tfm, sizeof(struct sec_req)); 2082 ctx->alg_type = SEC_AEAD; 2083 ctx->c_ctx.ivsize = crypto_aead_ivsize(tfm); 2084 if (ctx->c_ctx.ivsize < SEC_AIV_SIZE || 2085 ctx->c_ctx.ivsize > SEC_IV_SIZE) { 2086 pr_err("get error aead iv size!\n"); 2087 return -EINVAL; 2088 } 2089 2090 ret = sec_ctx_base_init(ctx); 2091 if (ret) 2092 return ret; 2093 if (ctx->sec->qm.ver < QM_HW_V3) { 2094 ctx->type_supported = SEC_BD_TYPE2; 2095 ctx->req_op = &sec_aead_req_ops; 2096 } else { 2097 ctx->type_supported = SEC_BD_TYPE3; 2098 ctx->req_op = &sec_aead_req_ops_v3; 2099 } 2100 2101 ret = sec_auth_init(ctx); 2102 if (ret) 2103 goto err_auth_init; 2104 2105 ret = sec_cipher_init(ctx); 2106 if (ret) 2107 goto err_cipher_init; 2108 2109 return ret; 2110 2111 err_cipher_init: 2112 sec_auth_uninit(ctx); 2113 err_auth_init: 2114 sec_ctx_base_uninit(ctx); 2115 return ret; 2116 } 2117 2118 static void sec_aead_exit(struct crypto_aead *tfm) 2119 { 2120 struct sec_ctx *ctx = crypto_aead_ctx(tfm); 2121 2122 sec_cipher_uninit(ctx); 2123 sec_auth_uninit(ctx); 2124 sec_ctx_base_uninit(ctx); 2125 } 2126 2127 static int sec_aead_ctx_init(struct crypto_aead *tfm, const char *hash_name) 2128 { 2129 struct aead_alg *alg = crypto_aead_alg(tfm); 2130 struct sec_ctx *ctx = crypto_aead_ctx(tfm); 2131 struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 2132 const char *aead_name = alg->base.cra_name; 2133 int ret; 2134 2135 ret = sec_aead_init(tfm); 2136 if (ret) { 2137 pr_err("hisi_sec2: aead init error!\n"); 2138 return ret; 2139 } 2140 2141 a_ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0); 2142 if (IS_ERR(a_ctx->hash_tfm)) { 2143 dev_err(ctx->dev, "aead alloc shash error!\n"); 2144 sec_aead_exit(tfm); 2145 return PTR_ERR(a_ctx->hash_tfm); 2146 } 2147 2148 a_ctx->fallback_aead_tfm = crypto_alloc_aead(aead_name, 0, 2149 CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ASYNC); 2150 if (IS_ERR(a_ctx->fallback_aead_tfm)) { 2151 dev_err(ctx->dev, "aead driver alloc fallback tfm error!\n"); 2152 crypto_free_shash(ctx->a_ctx.hash_tfm); 2153 sec_aead_exit(tfm); 2154 return PTR_ERR(a_ctx->fallback_aead_tfm); 2155 } 2156 2157 return 0; 2158 } 2159 2160 static void sec_aead_ctx_exit(struct crypto_aead *tfm) 2161 { 2162 struct sec_ctx *ctx = crypto_aead_ctx(tfm); 2163 2164 crypto_free_aead(ctx->a_ctx.fallback_aead_tfm); 2165 crypto_free_shash(ctx->a_ctx.hash_tfm); 2166 sec_aead_exit(tfm); 2167 } 2168 2169 static int sec_aead_xcm_ctx_init(struct crypto_aead *tfm) 2170 { 2171 struct aead_alg *alg = crypto_aead_alg(tfm); 2172 struct sec_ctx *ctx = crypto_aead_ctx(tfm); 2173 struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 2174 const char *aead_name = alg->base.cra_name; 2175 int ret; 2176 2177 ret = sec_aead_init(tfm); 2178 if (ret) { 2179 dev_err(ctx->dev, "hisi_sec2: aead xcm init error!\n"); 2180 return ret; 2181 } 2182 2183 a_ctx->fallback_aead_tfm = crypto_alloc_aead(aead_name, 0, 2184 CRYPTO_ALG_NEED_FALLBACK | 2185 CRYPTO_ALG_ASYNC); 2186 if (IS_ERR(a_ctx->fallback_aead_tfm)) { 2187 dev_err(ctx->dev, "aead driver alloc fallback tfm error!\n"); 2188 sec_aead_exit(tfm); 2189 return PTR_ERR(a_ctx->fallback_aead_tfm); 2190 } 2191 2192 return 0; 2193 } 2194 2195 static void sec_aead_xcm_ctx_exit(struct crypto_aead *tfm) 2196 { 2197 struct sec_ctx *ctx = crypto_aead_ctx(tfm); 2198 2199 crypto_free_aead(ctx->a_ctx.fallback_aead_tfm); 2200 sec_aead_exit(tfm); 2201 } 2202 2203 static int sec_aead_sha1_ctx_init(struct crypto_aead *tfm) 2204 { 2205 return sec_aead_ctx_init(tfm, "sha1"); 2206 } 2207 2208 static int sec_aead_sha256_ctx_init(struct crypto_aead *tfm) 2209 { 2210 return sec_aead_ctx_init(tfm, "sha256"); 2211 } 2212 2213 static int sec_aead_sha512_ctx_init(struct crypto_aead *tfm) 2214 { 2215 return sec_aead_ctx_init(tfm, "sha512"); 2216 } 2217 2218 static int sec_skcipher_cryptlen_check(struct sec_ctx *ctx, struct sec_req *sreq) 2219 { 2220 u32 cryptlen = sreq->c_req.sk_req->cryptlen; 2221 struct device *dev = ctx->dev; 2222 u8 c_mode = ctx->c_ctx.c_mode; 2223 int ret = 0; 2224 2225 switch (c_mode) { 2226 case SEC_CMODE_XTS: 2227 if (unlikely(cryptlen < AES_BLOCK_SIZE)) { 2228 dev_err(dev, "skcipher XTS mode input length error!\n"); 2229 ret = -EINVAL; 2230 } 2231 break; 2232 case SEC_CMODE_ECB: 2233 case SEC_CMODE_CBC: 2234 if (unlikely(cryptlen & (AES_BLOCK_SIZE - 1))) { 2235 dev_err(dev, "skcipher AES input length error!\n"); 2236 ret = -EINVAL; 2237 } 2238 break; 2239 case SEC_CMODE_CTR: 2240 break; 2241 default: 2242 ret = -EINVAL; 2243 } 2244 2245 return ret; 2246 } 2247 2248 static int sec_skcipher_param_check(struct sec_ctx *ctx, 2249 struct sec_req *sreq, bool *need_fallback) 2250 { 2251 struct skcipher_request *sk_req = sreq->c_req.sk_req; 2252 struct device *dev = ctx->dev; 2253 u8 c_alg = ctx->c_ctx.c_alg; 2254 2255 if (unlikely(!sk_req->src || !sk_req->dst)) { 2256 dev_err(dev, "skcipher input param error!\n"); 2257 return -EINVAL; 2258 } 2259 2260 if (sk_req->cryptlen > MAX_INPUT_DATA_LEN) 2261 *need_fallback = true; 2262 2263 sreq->c_req.c_len = sk_req->cryptlen; 2264 2265 if (ctx->pbuf_supported && sk_req->cryptlen <= SEC_PBUF_SZ) 2266 sreq->use_pbuf = true; 2267 else 2268 sreq->use_pbuf = false; 2269 2270 if (c_alg == SEC_CALG_3DES) { 2271 if (unlikely(sk_req->cryptlen & (DES3_EDE_BLOCK_SIZE - 1))) { 2272 dev_err(dev, "skcipher 3des input length error!\n"); 2273 return -EINVAL; 2274 } 2275 return 0; 2276 } else if (c_alg == SEC_CALG_AES || c_alg == SEC_CALG_SM4) { 2277 return sec_skcipher_cryptlen_check(ctx, sreq); 2278 } 2279 2280 dev_err(dev, "skcipher algorithm error!\n"); 2281 2282 return -EINVAL; 2283 } 2284 2285 static int sec_skcipher_soft_crypto(struct sec_ctx *ctx, 2286 struct skcipher_request *sreq, bool encrypt) 2287 { 2288 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 2289 SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, c_ctx->fbtfm); 2290 struct device *dev = ctx->dev; 2291 int ret; 2292 2293 if (!c_ctx->fbtfm) { 2294 dev_err_ratelimited(dev, "the soft tfm isn't supported in the current system.\n"); 2295 return -EINVAL; 2296 } 2297 2298 skcipher_request_set_sync_tfm(subreq, c_ctx->fbtfm); 2299 2300 /* software need sync mode to do crypto */ 2301 skcipher_request_set_callback(subreq, sreq->base.flags, 2302 NULL, NULL); 2303 skcipher_request_set_crypt(subreq, sreq->src, sreq->dst, 2304 sreq->cryptlen, sreq->iv); 2305 if (encrypt) 2306 ret = crypto_skcipher_encrypt(subreq); 2307 else 2308 ret = crypto_skcipher_decrypt(subreq); 2309 2310 skcipher_request_zero(subreq); 2311 2312 return ret; 2313 } 2314 2315 static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt) 2316 { 2317 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(sk_req); 2318 struct sec_req *req = skcipher_request_ctx_dma(sk_req); 2319 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 2320 bool need_fallback = false; 2321 int ret; 2322 2323 if (!sk_req->cryptlen) { 2324 if (ctx->c_ctx.c_mode == SEC_CMODE_XTS) 2325 return -EINVAL; 2326 return 0; 2327 } 2328 2329 req->flag = sk_req->base.flags; 2330 req->c_req.sk_req = sk_req; 2331 req->c_req.encrypt = encrypt; 2332 req->ctx = ctx; 2333 req->base = &sk_req->base; 2334 2335 ret = sec_skcipher_param_check(ctx, req, &need_fallback); 2336 if (unlikely(ret)) 2337 return -EINVAL; 2338 2339 if (unlikely(ctx->c_ctx.fallback || need_fallback)) 2340 return sec_skcipher_soft_crypto(ctx, sk_req, encrypt); 2341 2342 return ctx->req_op->process(ctx, req); 2343 } 2344 2345 static int sec_skcipher_encrypt(struct skcipher_request *sk_req) 2346 { 2347 return sec_skcipher_crypto(sk_req, true); 2348 } 2349 2350 static int sec_skcipher_decrypt(struct skcipher_request *sk_req) 2351 { 2352 return sec_skcipher_crypto(sk_req, false); 2353 } 2354 2355 #define SEC_SKCIPHER_ALG(sec_cra_name, sec_set_key, \ 2356 sec_min_key_size, sec_max_key_size, blk_size, iv_size)\ 2357 {\ 2358 .base = {\ 2359 .cra_name = sec_cra_name,\ 2360 .cra_driver_name = "hisi_sec_"sec_cra_name,\ 2361 .cra_priority = SEC_PRIORITY,\ 2362 .cra_flags = CRYPTO_ALG_ASYNC |\ 2363 CRYPTO_ALG_NEED_FALLBACK,\ 2364 .cra_blocksize = blk_size,\ 2365 .cra_ctxsize = sizeof(struct sec_ctx),\ 2366 .cra_module = THIS_MODULE,\ 2367 },\ 2368 .init = sec_skcipher_ctx_init,\ 2369 .exit = sec_skcipher_ctx_exit,\ 2370 .setkey = sec_set_key,\ 2371 .decrypt = sec_skcipher_decrypt,\ 2372 .encrypt = sec_skcipher_encrypt,\ 2373 .min_keysize = sec_min_key_size,\ 2374 .max_keysize = sec_max_key_size,\ 2375 .ivsize = iv_size,\ 2376 } 2377 2378 static struct sec_skcipher sec_skciphers[] = { 2379 { 2380 .alg_msk = BIT(0), 2381 .alg = SEC_SKCIPHER_ALG("ecb(aes)", sec_setkey_aes_ecb, AES_MIN_KEY_SIZE, 2382 AES_MAX_KEY_SIZE, AES_BLOCK_SIZE, 0), 2383 }, 2384 { 2385 .alg_msk = BIT(1), 2386 .alg = SEC_SKCIPHER_ALG("cbc(aes)", sec_setkey_aes_cbc, AES_MIN_KEY_SIZE, 2387 AES_MAX_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE), 2388 }, 2389 { 2390 .alg_msk = BIT(2), 2391 .alg = SEC_SKCIPHER_ALG("ctr(aes)", sec_setkey_aes_ctr, AES_MIN_KEY_SIZE, 2392 AES_MAX_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), 2393 }, 2394 { 2395 .alg_msk = BIT(3), 2396 .alg = SEC_SKCIPHER_ALG("xts(aes)", sec_setkey_aes_xts, SEC_XTS_MIN_KEY_SIZE, 2397 SEC_XTS_MAX_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE), 2398 }, 2399 { 2400 .alg_msk = BIT(12), 2401 .alg = SEC_SKCIPHER_ALG("cbc(sm4)", sec_setkey_sm4_cbc, AES_MIN_KEY_SIZE, 2402 AES_MIN_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE), 2403 }, 2404 { 2405 .alg_msk = BIT(13), 2406 .alg = SEC_SKCIPHER_ALG("ctr(sm4)", sec_setkey_sm4_ctr, AES_MIN_KEY_SIZE, 2407 AES_MIN_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), 2408 }, 2409 { 2410 .alg_msk = BIT(14), 2411 .alg = SEC_SKCIPHER_ALG("xts(sm4)", sec_setkey_sm4_xts, SEC_XTS_MIN_KEY_SIZE, 2412 SEC_XTS_MIN_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE), 2413 }, 2414 { 2415 .alg_msk = BIT(23), 2416 .alg = SEC_SKCIPHER_ALG("ecb(des3_ede)", sec_setkey_3des_ecb, SEC_DES3_3KEY_SIZE, 2417 SEC_DES3_3KEY_SIZE, DES3_EDE_BLOCK_SIZE, 0), 2418 }, 2419 { 2420 .alg_msk = BIT(24), 2421 .alg = SEC_SKCIPHER_ALG("cbc(des3_ede)", sec_setkey_3des_cbc, SEC_DES3_3KEY_SIZE, 2422 SEC_DES3_3KEY_SIZE, DES3_EDE_BLOCK_SIZE, 2423 DES3_EDE_BLOCK_SIZE), 2424 }, 2425 }; 2426 2427 static int aead_iv_demension_check(struct aead_request *aead_req) 2428 { 2429 u8 cl; 2430 2431 cl = aead_req->iv[0] + 1; 2432 if (cl < IV_CL_MIN || cl > IV_CL_MAX) 2433 return -EINVAL; 2434 2435 if (cl < IV_CL_MID && aead_req->cryptlen >> (BYTE_BITS * cl)) 2436 return -EOVERFLOW; 2437 2438 return 0; 2439 } 2440 2441 static int sec_aead_spec_check(struct sec_ctx *ctx, struct sec_req *sreq) 2442 { 2443 struct aead_request *req = sreq->aead_req.aead_req; 2444 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2445 size_t sz = crypto_aead_authsize(tfm); 2446 u8 c_mode = ctx->c_ctx.c_mode; 2447 int ret; 2448 2449 if (unlikely(ctx->sec->qm.ver == QM_HW_V2 && !sreq->c_req.c_len)) 2450 return -EINVAL; 2451 2452 if (unlikely(req->cryptlen + req->assoclen > MAX_INPUT_DATA_LEN || 2453 req->assoclen > SEC_MAX_AAD_LEN)) 2454 return -EINVAL; 2455 2456 if (c_mode == SEC_CMODE_CCM) { 2457 if (unlikely(req->assoclen > SEC_MAX_CCM_AAD_LEN)) 2458 return -EINVAL; 2459 2460 ret = aead_iv_demension_check(req); 2461 if (unlikely(ret)) 2462 return -EINVAL; 2463 } else if (c_mode == SEC_CMODE_CBC) { 2464 if (unlikely(sz & WORD_MASK)) 2465 return -EINVAL; 2466 if (unlikely(ctx->a_ctx.a_key_len & WORD_MASK)) 2467 return -EINVAL; 2468 } else if (c_mode == SEC_CMODE_GCM) { 2469 if (unlikely(sz < SEC_GCM_MIN_AUTH_SZ)) 2470 return -EINVAL; 2471 } 2472 2473 return 0; 2474 } 2475 2476 static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq, bool *need_fallback) 2477 { 2478 struct aead_request *req = sreq->aead_req.aead_req; 2479 struct device *dev = ctx->dev; 2480 u8 c_alg = ctx->c_ctx.c_alg; 2481 2482 if (unlikely(!req->src || !req->dst)) { 2483 dev_err(dev, "aead input param error!\n"); 2484 return -EINVAL; 2485 } 2486 2487 if (unlikely(ctx->c_ctx.c_mode == SEC_CMODE_CBC && 2488 sreq->c_req.c_len & (AES_BLOCK_SIZE - 1))) { 2489 dev_err(dev, "aead cbc mode input data length error!\n"); 2490 return -EINVAL; 2491 } 2492 2493 /* Support AES or SM4 */ 2494 if (unlikely(c_alg != SEC_CALG_AES && c_alg != SEC_CALG_SM4)) { 2495 dev_err(dev, "aead crypto alg error!\n"); 2496 return -EINVAL; 2497 } 2498 2499 if (unlikely(sec_aead_spec_check(ctx, sreq))) { 2500 *need_fallback = true; 2501 return -EINVAL; 2502 } 2503 2504 if (ctx->pbuf_supported && (req->cryptlen + req->assoclen) <= 2505 SEC_PBUF_SZ) 2506 sreq->use_pbuf = true; 2507 else 2508 sreq->use_pbuf = false; 2509 2510 return 0; 2511 } 2512 2513 static int sec_aead_soft_crypto(struct sec_ctx *ctx, 2514 struct aead_request *aead_req, 2515 bool encrypt) 2516 { 2517 struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 2518 struct aead_request *subreq; 2519 int ret; 2520 2521 subreq = aead_request_alloc(a_ctx->fallback_aead_tfm, GFP_KERNEL); 2522 if (!subreq) 2523 return -ENOMEM; 2524 2525 aead_request_set_tfm(subreq, a_ctx->fallback_aead_tfm); 2526 aead_request_set_callback(subreq, aead_req->base.flags, 2527 aead_req->base.complete, aead_req->base.data); 2528 aead_request_set_crypt(subreq, aead_req->src, aead_req->dst, 2529 aead_req->cryptlen, aead_req->iv); 2530 aead_request_set_ad(subreq, aead_req->assoclen); 2531 2532 if (encrypt) 2533 ret = crypto_aead_encrypt(subreq); 2534 else 2535 ret = crypto_aead_decrypt(subreq); 2536 aead_request_free(subreq); 2537 2538 return ret; 2539 } 2540 2541 static int sec_aead_crypto(struct aead_request *a_req, bool encrypt) 2542 { 2543 struct crypto_aead *tfm = crypto_aead_reqtfm(a_req); 2544 struct sec_req *req = aead_request_ctx_dma(a_req); 2545 struct sec_ctx *ctx = crypto_aead_ctx(tfm); 2546 size_t sz = crypto_aead_authsize(tfm); 2547 bool need_fallback = false; 2548 int ret; 2549 2550 req->flag = a_req->base.flags; 2551 req->aead_req.aead_req = a_req; 2552 req->c_req.encrypt = encrypt; 2553 req->ctx = ctx; 2554 req->base = &a_req->base; 2555 req->c_req.c_len = a_req->cryptlen - (req->c_req.encrypt ? 0 : sz); 2556 2557 ret = sec_aead_param_check(ctx, req, &need_fallback); 2558 if (unlikely(ret)) { 2559 if (need_fallback) 2560 return sec_aead_soft_crypto(ctx, a_req, encrypt); 2561 return -EINVAL; 2562 } 2563 2564 return ctx->req_op->process(ctx, req); 2565 } 2566 2567 static int sec_aead_encrypt(struct aead_request *a_req) 2568 { 2569 return sec_aead_crypto(a_req, true); 2570 } 2571 2572 static int sec_aead_decrypt(struct aead_request *a_req) 2573 { 2574 return sec_aead_crypto(a_req, false); 2575 } 2576 2577 #define SEC_AEAD_ALG(sec_cra_name, sec_set_key, ctx_init,\ 2578 ctx_exit, blk_size, iv_size, max_authsize)\ 2579 {\ 2580 .base = {\ 2581 .cra_name = sec_cra_name,\ 2582 .cra_driver_name = "hisi_sec_"sec_cra_name,\ 2583 .cra_priority = SEC_PRIORITY,\ 2584 .cra_flags = CRYPTO_ALG_ASYNC |\ 2585 CRYPTO_ALG_NEED_FALLBACK,\ 2586 .cra_blocksize = blk_size,\ 2587 .cra_ctxsize = sizeof(struct sec_ctx),\ 2588 .cra_module = THIS_MODULE,\ 2589 },\ 2590 .init = ctx_init,\ 2591 .exit = ctx_exit,\ 2592 .setkey = sec_set_key,\ 2593 .setauthsize = sec_aead_setauthsize,\ 2594 .decrypt = sec_aead_decrypt,\ 2595 .encrypt = sec_aead_encrypt,\ 2596 .ivsize = iv_size,\ 2597 .maxauthsize = max_authsize,\ 2598 } 2599 2600 static struct sec_aead sec_aeads[] = { 2601 { 2602 .alg_msk = BIT(6), 2603 .alg = SEC_AEAD_ALG("ccm(aes)", sec_setkey_aes_ccm, sec_aead_xcm_ctx_init, 2604 sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE, 2605 AES_BLOCK_SIZE), 2606 }, 2607 { 2608 .alg_msk = BIT(7), 2609 .alg = SEC_AEAD_ALG("gcm(aes)", sec_setkey_aes_gcm, sec_aead_xcm_ctx_init, 2610 sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, SEC_AIV_SIZE, 2611 AES_BLOCK_SIZE), 2612 }, 2613 { 2614 .alg_msk = BIT(17), 2615 .alg = SEC_AEAD_ALG("ccm(sm4)", sec_setkey_sm4_ccm, sec_aead_xcm_ctx_init, 2616 sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE, 2617 AES_BLOCK_SIZE), 2618 }, 2619 { 2620 .alg_msk = BIT(18), 2621 .alg = SEC_AEAD_ALG("gcm(sm4)", sec_setkey_sm4_gcm, sec_aead_xcm_ctx_init, 2622 sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, SEC_AIV_SIZE, 2623 AES_BLOCK_SIZE), 2624 }, 2625 { 2626 .alg_msk = BIT(43), 2627 .alg = SEC_AEAD_ALG("authenc(hmac(sha1),cbc(aes))", sec_setkey_aes_cbc_sha1, 2628 sec_aead_sha1_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE, 2629 AES_BLOCK_SIZE, SHA1_DIGEST_SIZE), 2630 }, 2631 { 2632 .alg_msk = BIT(44), 2633 .alg = SEC_AEAD_ALG("authenc(hmac(sha256),cbc(aes))", sec_setkey_aes_cbc_sha256, 2634 sec_aead_sha256_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE, 2635 AES_BLOCK_SIZE, SHA256_DIGEST_SIZE), 2636 }, 2637 { 2638 .alg_msk = BIT(45), 2639 .alg = SEC_AEAD_ALG("authenc(hmac(sha512),cbc(aes))", sec_setkey_aes_cbc_sha512, 2640 sec_aead_sha512_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE, 2641 AES_BLOCK_SIZE, SHA512_DIGEST_SIZE), 2642 }, 2643 }; 2644 2645 static void sec_unregister_skcipher(u64 alg_mask, int end) 2646 { 2647 int i; 2648 2649 for (i = 0; i < end; i++) 2650 if (sec_skciphers[i].alg_msk & alg_mask) 2651 crypto_unregister_skcipher(&sec_skciphers[i].alg); 2652 } 2653 2654 static int sec_register_skcipher(u64 alg_mask) 2655 { 2656 int i, ret, count; 2657 2658 count = ARRAY_SIZE(sec_skciphers); 2659 2660 for (i = 0; i < count; i++) { 2661 if (!(sec_skciphers[i].alg_msk & alg_mask)) 2662 continue; 2663 2664 ret = crypto_register_skcipher(&sec_skciphers[i].alg); 2665 if (ret) 2666 goto err; 2667 } 2668 2669 return 0; 2670 2671 err: 2672 sec_unregister_skcipher(alg_mask, i); 2673 2674 return ret; 2675 } 2676 2677 static void sec_unregister_aead(u64 alg_mask, int end) 2678 { 2679 int i; 2680 2681 for (i = 0; i < end; i++) 2682 if (sec_aeads[i].alg_msk & alg_mask) 2683 crypto_unregister_aead(&sec_aeads[i].alg); 2684 } 2685 2686 static int sec_register_aead(u64 alg_mask) 2687 { 2688 int i, ret, count; 2689 2690 count = ARRAY_SIZE(sec_aeads); 2691 2692 for (i = 0; i < count; i++) { 2693 if (!(sec_aeads[i].alg_msk & alg_mask)) 2694 continue; 2695 2696 ret = crypto_register_aead(&sec_aeads[i].alg); 2697 if (ret) 2698 goto err; 2699 } 2700 2701 return 0; 2702 2703 err: 2704 sec_unregister_aead(alg_mask, i); 2705 2706 return ret; 2707 } 2708 2709 int sec_register_to_crypto(struct hisi_qm *qm) 2710 { 2711 u64 alg_mask; 2712 int ret = 0; 2713 2714 alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH_TB, 2715 SEC_DRV_ALG_BITMAP_LOW_TB); 2716 2717 mutex_lock(&sec_algs_lock); 2718 if (sec_available_devs) { 2719 sec_available_devs++; 2720 goto unlock; 2721 } 2722 2723 ret = sec_register_skcipher(alg_mask); 2724 if (ret) 2725 goto unlock; 2726 2727 ret = sec_register_aead(alg_mask); 2728 if (ret) 2729 goto unreg_skcipher; 2730 2731 sec_available_devs++; 2732 mutex_unlock(&sec_algs_lock); 2733 2734 return 0; 2735 2736 unreg_skcipher: 2737 sec_unregister_skcipher(alg_mask, ARRAY_SIZE(sec_skciphers)); 2738 unlock: 2739 mutex_unlock(&sec_algs_lock); 2740 return ret; 2741 } 2742 2743 void sec_unregister_from_crypto(struct hisi_qm *qm) 2744 { 2745 u64 alg_mask; 2746 2747 alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH_TB, 2748 SEC_DRV_ALG_BITMAP_LOW_TB); 2749 2750 mutex_lock(&sec_algs_lock); 2751 if (--sec_available_devs) 2752 goto unlock; 2753 2754 sec_unregister_aead(alg_mask, ARRAY_SIZE(sec_aeads)); 2755 sec_unregister_skcipher(alg_mask, ARRAY_SIZE(sec_skciphers)); 2756 2757 unlock: 2758 mutex_unlock(&sec_algs_lock); 2759 } 2760