1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2019 HiSilicon Limited. */ 3 4 #include <crypto/aes.h> 5 #include <crypto/aead.h> 6 #include <crypto/algapi.h> 7 #include <crypto/authenc.h> 8 #include <crypto/des.h> 9 #include <crypto/hash.h> 10 #include <crypto/internal/aead.h> 11 #include <crypto/internal/des.h> 12 #include <crypto/sha1.h> 13 #include <crypto/sha2.h> 14 #include <crypto/skcipher.h> 15 #include <crypto/xts.h> 16 #include <linux/crypto.h> 17 #include <linux/dma-mapping.h> 18 #include <linux/idr.h> 19 20 #include "sec.h" 21 #include "sec_crypto.h" 22 23 #define SEC_PRIORITY 80 24 #define SEC_XTS_MIN_KEY_SIZE (2 * AES_MIN_KEY_SIZE) 25 #define SEC_XTS_MID_KEY_SIZE (3 * AES_MIN_KEY_SIZE) 26 #define SEC_XTS_MAX_KEY_SIZE (2 * AES_MAX_KEY_SIZE) 27 #define SEC_DES3_2KEY_SIZE (2 * DES_KEY_SIZE) 28 #define SEC_DES3_3KEY_SIZE (3 * DES_KEY_SIZE) 29 30 /* SEC sqe(bd) bit operational relative MACRO */ 31 #define SEC_DE_OFFSET 1 32 #define SEC_CIPHER_OFFSET 4 33 #define SEC_SCENE_OFFSET 3 34 #define SEC_DST_SGL_OFFSET 2 35 #define SEC_SRC_SGL_OFFSET 7 36 #define SEC_CKEY_OFFSET 9 37 #define SEC_CMODE_OFFSET 12 38 #define SEC_AKEY_OFFSET 5 39 #define SEC_AEAD_ALG_OFFSET 11 40 #define SEC_AUTH_OFFSET 6 41 42 #define SEC_DE_OFFSET_V3 9 43 #define SEC_SCENE_OFFSET_V3 5 44 #define SEC_CKEY_OFFSET_V3 13 45 #define SEC_CTR_CNT_OFFSET 25 46 #define SEC_CTR_CNT_ROLLOVER 2 47 #define SEC_SRC_SGL_OFFSET_V3 11 48 #define SEC_DST_SGL_OFFSET_V3 14 49 #define SEC_CALG_OFFSET_V3 4 50 #define SEC_AKEY_OFFSET_V3 9 51 #define SEC_MAC_OFFSET_V3 4 52 #define SEC_AUTH_ALG_OFFSET_V3 15 53 #define SEC_CIPHER_AUTH_V3 0xbf 54 #define SEC_AUTH_CIPHER_V3 0x40 55 #define SEC_FLAG_OFFSET 7 56 #define SEC_FLAG_MASK 0x0780 57 #define SEC_DONE_MASK 0x0001 58 #define SEC_ICV_MASK 0x000E 59 60 #define SEC_TOTAL_IV_SZ(depth) (SEC_IV_SIZE * (depth)) 61 #define SEC_SGL_SGE_NR 128 62 #define SEC_CIPHER_AUTH 0xfe 63 #define SEC_AUTH_CIPHER 0x1 64 #define SEC_MAX_MAC_LEN 64 65 #define SEC_MAX_AAD_LEN 65535 66 #define SEC_MAX_CCM_AAD_LEN 65279 67 #define SEC_TOTAL_MAC_SZ(depth) (SEC_MAX_MAC_LEN * (depth)) 68 69 #define SEC_PBUF_IV_OFFSET SEC_PBUF_SZ 70 #define SEC_PBUF_MAC_OFFSET (SEC_PBUF_SZ + SEC_IV_SIZE) 71 #define SEC_PBUF_PKG (SEC_PBUF_SZ + SEC_IV_SIZE + \ 72 SEC_MAX_MAC_LEN * 2) 73 #define SEC_PBUF_NUM (PAGE_SIZE / SEC_PBUF_PKG) 74 #define SEC_PBUF_PAGE_NUM(depth) ((depth) / SEC_PBUF_NUM) 75 #define SEC_PBUF_LEFT_SZ(depth) (SEC_PBUF_PKG * ((depth) - \ 76 SEC_PBUF_PAGE_NUM(depth) * SEC_PBUF_NUM)) 77 #define SEC_TOTAL_PBUF_SZ(depth) (PAGE_SIZE * SEC_PBUF_PAGE_NUM(depth) + \ 78 SEC_PBUF_LEFT_SZ(depth)) 79 80 #define SEC_SQE_CFLAG 2 81 #define SEC_SQE_AEAD_FLAG 3 82 #define SEC_SQE_DONE 0x1 83 #define SEC_ICV_ERR 0x2 84 #define MAC_LEN_MASK 0x1U 85 #define MAX_INPUT_DATA_LEN 0xFFFE00 86 #define BITS_MASK 0xFF 87 #define WORD_MASK 0x3 88 #define BYTE_BITS 0x8 89 #define BYTES_TO_WORDS(bcount) ((bcount) >> 2) 90 #define SEC_XTS_NAME_SZ 0x3 91 #define IV_CM_CAL_NUM 2 92 #define IV_CL_MASK 0x7 93 #define IV_CL_MIN 2 94 #define IV_CL_MID 4 95 #define IV_CL_MAX 8 96 #define IV_FLAGS_OFFSET 0x6 97 #define IV_CM_OFFSET 0x3 98 #define IV_LAST_BYTE1 1 99 #define IV_LAST_BYTE_MASK 0xFF 100 #define IV_CTR_INIT 0x1 101 #define IV_BYTE_OFFSET 0x8 102 #define SEC_GCM_MIN_AUTH_SZ 0x8 103 #define SEC_RETRY_MAX_CNT 5U 104 105 static DEFINE_MUTEX(sec_algs_lock); 106 static unsigned int sec_available_devs; 107 108 struct sec_skcipher { 109 u64 alg_msk; 110 struct skcipher_alg alg; 111 }; 112 113 struct sec_aead { 114 u64 alg_msk; 115 struct aead_alg alg; 116 }; 117 118 static int sec_aead_soft_crypto(struct sec_ctx *ctx, 119 struct aead_request *aead_req, 120 bool encrypt); 121 static int sec_skcipher_soft_crypto(struct sec_ctx *ctx, 122 struct skcipher_request *sreq, bool encrypt); 123 124 static int sec_alloc_req_id(struct sec_req *req, struct sec_qp_ctx *qp_ctx) 125 { 126 int req_id; 127 128 spin_lock_bh(&qp_ctx->id_lock); 129 req_id = idr_alloc_cyclic(&qp_ctx->req_idr, NULL, 0, qp_ctx->qp->sq_depth, GFP_ATOMIC); 130 spin_unlock_bh(&qp_ctx->id_lock); 131 return req_id; 132 } 133 134 static void sec_free_req_id(struct sec_req *req) 135 { 136 struct sec_qp_ctx *qp_ctx = req->qp_ctx; 137 int req_id = req->req_id; 138 139 if (unlikely(req_id < 0 || req_id >= qp_ctx->qp->sq_depth)) { 140 dev_err(req->ctx->dev, "free request id invalid!\n"); 141 return; 142 } 143 144 spin_lock_bh(&qp_ctx->id_lock); 145 idr_remove(&qp_ctx->req_idr, req_id); 146 spin_unlock_bh(&qp_ctx->id_lock); 147 } 148 149 static void pre_parse_finished_bd(struct bd_status *status, void *resp) 150 { 151 struct sec_sqe *bd = resp; 152 153 status->done = le16_to_cpu(bd->type2.done_flag) & SEC_DONE_MASK; 154 status->icv = (le16_to_cpu(bd->type2.done_flag) & SEC_ICV_MASK) >> 1; 155 status->flag = (le16_to_cpu(bd->type2.done_flag) & 156 SEC_FLAG_MASK) >> SEC_FLAG_OFFSET; 157 status->tag = le16_to_cpu(bd->type2.tag); 158 status->err_type = bd->type2.error_type; 159 } 160 161 static void pre_parse_finished_bd3(struct bd_status *status, void *resp) 162 { 163 struct sec_sqe3 *bd3 = resp; 164 165 status->done = le16_to_cpu(bd3->done_flag) & SEC_DONE_MASK; 166 status->icv = (le16_to_cpu(bd3->done_flag) & SEC_ICV_MASK) >> 1; 167 status->flag = (le16_to_cpu(bd3->done_flag) & 168 SEC_FLAG_MASK) >> SEC_FLAG_OFFSET; 169 status->tag = le64_to_cpu(bd3->tag); 170 status->err_type = bd3->error_type; 171 } 172 173 static int sec_cb_status_check(struct sec_req *req, 174 struct bd_status *status) 175 { 176 struct sec_ctx *ctx = req->ctx; 177 178 if (unlikely(req->err_type || status->done != SEC_SQE_DONE)) { 179 dev_err_ratelimited(ctx->dev, "err_type[%d], done[%u]\n", 180 req->err_type, status->done); 181 return -EIO; 182 } 183 184 if (unlikely(ctx->alg_type == SEC_SKCIPHER)) { 185 if (unlikely(status->flag != SEC_SQE_CFLAG)) { 186 dev_err_ratelimited(ctx->dev, "flag[%u]\n", 187 status->flag); 188 return -EIO; 189 } 190 } else if (unlikely(ctx->alg_type == SEC_AEAD)) { 191 if (unlikely(status->flag != SEC_SQE_AEAD_FLAG || 192 status->icv == SEC_ICV_ERR)) { 193 dev_err_ratelimited(ctx->dev, 194 "flag[%u], icv[%u]\n", 195 status->flag, status->icv); 196 return -EBADMSG; 197 } 198 } 199 200 return 0; 201 } 202 203 static int qp_send_message(struct sec_req *req) 204 { 205 struct sec_qp_ctx *qp_ctx = req->qp_ctx; 206 int ret; 207 208 if (atomic_read(&qp_ctx->qp->qp_status.used) == qp_ctx->qp->sq_depth - 1) 209 return -EBUSY; 210 211 spin_lock_bh(&qp_ctx->req_lock); 212 if (atomic_read(&qp_ctx->qp->qp_status.used) == qp_ctx->qp->sq_depth - 1) { 213 spin_unlock_bh(&qp_ctx->req_lock); 214 return -EBUSY; 215 } 216 217 if (qp_ctx->ctx->type_supported == SEC_BD_TYPE2) { 218 req->sec_sqe.type2.tag = cpu_to_le16((u16)qp_ctx->send_head); 219 qp_ctx->req_list[qp_ctx->send_head] = req; 220 } 221 222 ret = hisi_qp_send(qp_ctx->qp, &req->sec_sqe); 223 if (ret) { 224 spin_unlock_bh(&qp_ctx->req_lock); 225 return ret; 226 } 227 if (qp_ctx->ctx->type_supported == SEC_BD_TYPE2) 228 qp_ctx->send_head = (qp_ctx->send_head + 1) % qp_ctx->qp->sq_depth; 229 230 spin_unlock_bh(&qp_ctx->req_lock); 231 232 atomic64_inc(&qp_ctx->ctx->sec->debug.dfx.send_cnt); 233 return -EINPROGRESS; 234 } 235 236 static void sec_alg_send_backlog_soft(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) 237 { 238 struct sec_req *req, *tmp; 239 int ret; 240 241 list_for_each_entry_safe(req, tmp, &qp_ctx->qp->backlog.list, list) { 242 list_del(&req->list); 243 ctx->req_op->buf_unmap(ctx, req); 244 if (req->req_id >= 0) 245 sec_free_req_id(req); 246 247 if (ctx->alg_type == SEC_AEAD) 248 ret = sec_aead_soft_crypto(ctx, req->aead_req.aead_req, 249 req->c_req.encrypt); 250 else 251 ret = sec_skcipher_soft_crypto(ctx, req->c_req.sk_req, 252 req->c_req.encrypt); 253 254 /* Wake up the busy thread first, then return the errno. */ 255 crypto_request_complete(req->base, -EINPROGRESS); 256 crypto_request_complete(req->base, ret); 257 } 258 } 259 260 static void sec_alg_send_backlog(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) 261 { 262 struct hisi_qp *qp = qp_ctx->qp; 263 struct sec_req *req, *tmp; 264 int ret; 265 266 spin_lock_bh(&qp->backlog.lock); 267 list_for_each_entry_safe(req, tmp, &qp->backlog.list, list) { 268 ret = qp_send_message(req); 269 switch (ret) { 270 case -EINPROGRESS: 271 list_del(&req->list); 272 crypto_request_complete(req->base, -EINPROGRESS); 273 break; 274 case -EBUSY: 275 /* Device is busy and stop send any request. */ 276 goto unlock; 277 default: 278 /* Release memory resources and send all requests through software. */ 279 sec_alg_send_backlog_soft(ctx, qp_ctx); 280 goto unlock; 281 } 282 } 283 284 unlock: 285 spin_unlock_bh(&qp->backlog.lock); 286 } 287 288 static void sec_req_cb(struct hisi_qp *qp, void *resp) 289 { 290 const struct sec_sqe *sqe = qp->msg[qp->qp_status.cq_head]; 291 struct sec_req *req = container_of(sqe, struct sec_req, sec_sqe); 292 struct sec_ctx *ctx = req->ctx; 293 struct sec_dfx *dfx = &ctx->sec->debug.dfx; 294 struct bd_status status; 295 int err; 296 297 pre_parse_finished_bd(&status, resp); 298 299 req->err_type = status.err_type; 300 err = sec_cb_status_check(req, &status); 301 if (err) 302 atomic64_inc(&dfx->done_flag_cnt); 303 304 atomic64_inc(&dfx->recv_cnt); 305 306 ctx->req_op->buf_unmap(ctx, req); 307 ctx->req_op->callback(ctx, req, err); 308 } 309 310 static void sec_req_cb3(struct hisi_qp *qp, void *resp) 311 { 312 struct bd_status status; 313 struct sec_ctx *ctx; 314 struct sec_dfx *dfx; 315 struct sec_req *req; 316 int err; 317 318 pre_parse_finished_bd3(&status, resp); 319 320 req = (void *)(uintptr_t)status.tag; 321 req->err_type = status.err_type; 322 ctx = req->ctx; 323 dfx = &ctx->sec->debug.dfx; 324 325 err = sec_cb_status_check(req, &status); 326 if (err) 327 atomic64_inc(&dfx->done_flag_cnt); 328 329 atomic64_inc(&dfx->recv_cnt); 330 331 ctx->req_op->buf_unmap(ctx, req); 332 ctx->req_op->callback(ctx, req, err); 333 } 334 335 static int sec_alg_send_message_retry(struct sec_req *req) 336 { 337 int ctr = 0; 338 int ret; 339 340 do { 341 ret = qp_send_message(req); 342 } while (ret == -EBUSY && ctr++ < SEC_RETRY_MAX_CNT); 343 344 return ret; 345 } 346 347 static int sec_alg_try_enqueue(struct sec_req *req) 348 { 349 struct hisi_qp *qp = req->qp_ctx->qp; 350 351 /* Check if any request is already backlogged */ 352 if (!list_empty(&qp->backlog.list)) 353 return -EBUSY; 354 355 /* Try to enqueue to HW ring */ 356 return qp_send_message(req); 357 } 358 359 360 static int sec_alg_send_message_maybacklog(struct sec_req *req) 361 { 362 struct hisi_qp *qp = req->qp_ctx->qp; 363 int ret; 364 365 ret = sec_alg_try_enqueue(req); 366 if (ret != -EBUSY) 367 return ret; 368 369 spin_lock_bh(&qp->backlog.lock); 370 ret = sec_alg_try_enqueue(req); 371 if (ret == -EBUSY) 372 list_add_tail(&req->list, &qp->backlog.list); 373 spin_unlock_bh(&qp->backlog.lock); 374 375 return ret; 376 } 377 378 static int sec_bd_send(struct sec_ctx *ctx, struct sec_req *req) 379 { 380 if (req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG) 381 return sec_alg_send_message_maybacklog(req); 382 383 return sec_alg_send_message_retry(req); 384 } 385 386 static int sec_alloc_civ_resource(struct device *dev, struct sec_alg_res *res) 387 { 388 u16 q_depth = res->depth; 389 int i; 390 391 res->c_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ(q_depth), 392 &res->c_ivin_dma, GFP_KERNEL); 393 if (!res->c_ivin) 394 return -ENOMEM; 395 396 for (i = 1; i < q_depth; i++) { 397 res[i].c_ivin_dma = res->c_ivin_dma + i * SEC_IV_SIZE; 398 res[i].c_ivin = res->c_ivin + i * SEC_IV_SIZE; 399 } 400 401 return 0; 402 } 403 404 static void sec_free_civ_resource(struct device *dev, struct sec_alg_res *res) 405 { 406 if (res->c_ivin) 407 dma_free_coherent(dev, SEC_TOTAL_IV_SZ(res->depth), 408 res->c_ivin, res->c_ivin_dma); 409 } 410 411 static int sec_alloc_aiv_resource(struct device *dev, struct sec_alg_res *res) 412 { 413 u16 q_depth = res->depth; 414 int i; 415 416 res->a_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ(q_depth), 417 &res->a_ivin_dma, GFP_KERNEL); 418 if (!res->a_ivin) 419 return -ENOMEM; 420 421 for (i = 1; i < q_depth; i++) { 422 res[i].a_ivin_dma = res->a_ivin_dma + i * SEC_IV_SIZE; 423 res[i].a_ivin = res->a_ivin + i * SEC_IV_SIZE; 424 } 425 426 return 0; 427 } 428 429 static void sec_free_aiv_resource(struct device *dev, struct sec_alg_res *res) 430 { 431 if (res->a_ivin) 432 dma_free_coherent(dev, SEC_TOTAL_IV_SZ(res->depth), 433 res->a_ivin, res->a_ivin_dma); 434 } 435 436 static int sec_alloc_mac_resource(struct device *dev, struct sec_alg_res *res) 437 { 438 u16 q_depth = res->depth; 439 int i; 440 441 res->out_mac = dma_alloc_coherent(dev, SEC_TOTAL_MAC_SZ(q_depth) << 1, 442 &res->out_mac_dma, GFP_KERNEL); 443 if (!res->out_mac) 444 return -ENOMEM; 445 446 for (i = 1; i < q_depth; i++) { 447 res[i].out_mac_dma = res->out_mac_dma + 448 i * (SEC_MAX_MAC_LEN << 1); 449 res[i].out_mac = res->out_mac + i * (SEC_MAX_MAC_LEN << 1); 450 } 451 452 return 0; 453 } 454 455 static void sec_free_mac_resource(struct device *dev, struct sec_alg_res *res) 456 { 457 if (res->out_mac) 458 dma_free_coherent(dev, SEC_TOTAL_MAC_SZ(res->depth) << 1, 459 res->out_mac, res->out_mac_dma); 460 } 461 462 static void sec_free_pbuf_resource(struct device *dev, struct sec_alg_res *res) 463 { 464 if (res->pbuf) 465 dma_free_coherent(dev, SEC_TOTAL_PBUF_SZ(res->depth), 466 res->pbuf, res->pbuf_dma); 467 } 468 469 /* 470 * To improve performance, pbuffer is used for 471 * small packets (< 512Bytes) as IOMMU translation using. 472 */ 473 static int sec_alloc_pbuf_resource(struct device *dev, struct sec_alg_res *res) 474 { 475 u16 q_depth = res->depth; 476 int size = SEC_PBUF_PAGE_NUM(q_depth); 477 int pbuf_page_offset; 478 int i, j, k; 479 480 res->pbuf = dma_alloc_coherent(dev, SEC_TOTAL_PBUF_SZ(q_depth), 481 &res->pbuf_dma, GFP_KERNEL); 482 if (!res->pbuf) 483 return -ENOMEM; 484 485 /* 486 * SEC_PBUF_PKG contains data pbuf, iv and 487 * out_mac : <SEC_PBUF|SEC_IV|SEC_MAC> 488 * Every PAGE contains six SEC_PBUF_PKG 489 * The sec_qp_ctx contains QM_Q_DEPTH numbers of SEC_PBUF_PKG 490 * So we need SEC_PBUF_PAGE_NUM numbers of PAGE 491 * for the SEC_TOTAL_PBUF_SZ 492 */ 493 for (i = 0; i <= size; i++) { 494 pbuf_page_offset = PAGE_SIZE * i; 495 for (j = 0; j < SEC_PBUF_NUM; j++) { 496 k = i * SEC_PBUF_NUM + j; 497 if (k == q_depth) 498 break; 499 res[k].pbuf = res->pbuf + 500 j * SEC_PBUF_PKG + pbuf_page_offset; 501 res[k].pbuf_dma = res->pbuf_dma + 502 j * SEC_PBUF_PKG + pbuf_page_offset; 503 } 504 } 505 506 return 0; 507 } 508 509 static int sec_alg_resource_alloc(struct sec_ctx *ctx, 510 struct sec_qp_ctx *qp_ctx) 511 { 512 struct sec_alg_res *res = qp_ctx->res; 513 struct device *dev = ctx->dev; 514 int ret; 515 516 ret = sec_alloc_civ_resource(dev, res); 517 if (ret) 518 return ret; 519 520 if (ctx->alg_type == SEC_AEAD) { 521 ret = sec_alloc_aiv_resource(dev, res); 522 if (ret) 523 goto alloc_aiv_fail; 524 525 ret = sec_alloc_mac_resource(dev, res); 526 if (ret) 527 goto alloc_mac_fail; 528 } 529 if (ctx->pbuf_supported) { 530 ret = sec_alloc_pbuf_resource(dev, res); 531 if (ret) { 532 dev_err(dev, "fail to alloc pbuf dma resource!\n"); 533 goto alloc_pbuf_fail; 534 } 535 } 536 537 return 0; 538 539 alloc_pbuf_fail: 540 if (ctx->alg_type == SEC_AEAD) 541 sec_free_mac_resource(dev, qp_ctx->res); 542 alloc_mac_fail: 543 if (ctx->alg_type == SEC_AEAD) 544 sec_free_aiv_resource(dev, res); 545 alloc_aiv_fail: 546 sec_free_civ_resource(dev, res); 547 return ret; 548 } 549 550 static void sec_alg_resource_free(struct sec_ctx *ctx, 551 struct sec_qp_ctx *qp_ctx) 552 { 553 struct device *dev = ctx->dev; 554 555 sec_free_civ_resource(dev, qp_ctx->res); 556 557 if (ctx->pbuf_supported) 558 sec_free_pbuf_resource(dev, qp_ctx->res); 559 if (ctx->alg_type == SEC_AEAD) { 560 sec_free_mac_resource(dev, qp_ctx->res); 561 sec_free_aiv_resource(dev, qp_ctx->res); 562 } 563 } 564 565 static int sec_alloc_qp_ctx_resource(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) 566 { 567 u16 q_depth = qp_ctx->qp->sq_depth; 568 struct device *dev = ctx->dev; 569 int ret = -ENOMEM; 570 571 qp_ctx->req_list = kzalloc_objs(struct sec_req *, q_depth); 572 if (!qp_ctx->req_list) 573 return ret; 574 575 qp_ctx->res = kzalloc_objs(struct sec_alg_res, q_depth); 576 if (!qp_ctx->res) 577 goto err_free_req_list; 578 qp_ctx->res->depth = q_depth; 579 580 qp_ctx->c_in_pool = hisi_acc_create_sgl_pool(dev, q_depth, SEC_SGL_SGE_NR); 581 if (IS_ERR(qp_ctx->c_in_pool)) { 582 dev_err(dev, "fail to create sgl pool for input!\n"); 583 goto err_free_res; 584 } 585 586 qp_ctx->c_out_pool = hisi_acc_create_sgl_pool(dev, q_depth, SEC_SGL_SGE_NR); 587 if (IS_ERR(qp_ctx->c_out_pool)) { 588 dev_err(dev, "fail to create sgl pool for output!\n"); 589 goto err_free_c_in_pool; 590 } 591 592 ret = sec_alg_resource_alloc(ctx, qp_ctx); 593 if (ret) 594 goto err_free_c_out_pool; 595 596 return 0; 597 598 err_free_c_out_pool: 599 hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool); 600 err_free_c_in_pool: 601 hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool); 602 err_free_res: 603 kfree(qp_ctx->res); 604 err_free_req_list: 605 kfree(qp_ctx->req_list); 606 return ret; 607 } 608 609 static void sec_free_qp_ctx_resource(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) 610 { 611 struct device *dev = ctx->dev; 612 613 sec_alg_resource_free(ctx, qp_ctx); 614 hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool); 615 hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool); 616 kfree(qp_ctx->res); 617 kfree(qp_ctx->req_list); 618 } 619 620 static int sec_create_qp_ctx(struct sec_ctx *ctx, int qp_ctx_id) 621 { 622 struct sec_qp_ctx *qp_ctx; 623 struct hisi_qp *qp; 624 int ret; 625 626 qp_ctx = &ctx->qp_ctx[qp_ctx_id]; 627 qp = ctx->qps[qp_ctx_id]; 628 qp_ctx->qp = qp; 629 qp_ctx->ctx = ctx; 630 631 if (ctx->type_supported == SEC_BD_TYPE3) 632 qp->req_cb = sec_req_cb3; 633 else 634 qp->req_cb = sec_req_cb; 635 636 spin_lock_init(&qp_ctx->req_lock); 637 idr_init(&qp_ctx->req_idr); 638 spin_lock_init(&qp_ctx->id_lock); 639 qp_ctx->send_head = 0; 640 641 ret = sec_alloc_qp_ctx_resource(ctx, qp_ctx); 642 if (ret) 643 goto err_destroy_idr; 644 645 return 0; 646 647 err_destroy_idr: 648 idr_destroy(&qp_ctx->req_idr); 649 return ret; 650 } 651 652 static void sec_release_qp_ctx(struct sec_ctx *ctx, 653 struct sec_qp_ctx *qp_ctx) 654 { 655 sec_free_qp_ctx_resource(ctx, qp_ctx); 656 idr_destroy(&qp_ctx->req_idr); 657 } 658 659 static int sec_ctx_base_init(struct sec_ctx *ctx) 660 { 661 struct sec_dev *sec; 662 int i, ret; 663 664 ctx->qps = sec_create_qps(); 665 if (!ctx->qps) 666 return -ENODEV; 667 668 sec = container_of(ctx->qps[0]->qm, struct sec_dev, qm); 669 ctx->sec = sec; 670 ctx->dev = &sec->qm.pdev->dev; 671 672 ctx->pbuf_supported = ctx->sec->iommu_used; 673 if (sec->qm.ver < QM_HW_V3) 674 ctx->type_supported = SEC_BD_TYPE2; 675 else 676 ctx->type_supported = SEC_BD_TYPE3; 677 678 ctx->qp_ctx = kzalloc_objs(struct sec_qp_ctx, sec->ctx_q_num); 679 if (!ctx->qp_ctx) { 680 ret = -ENOMEM; 681 goto err_destroy_qps; 682 } 683 684 for (i = 0; i < sec->ctx_q_num; i++) { 685 ret = sec_create_qp_ctx(ctx, i); 686 if (ret) 687 goto err_sec_release_qp_ctx; 688 } 689 690 return 0; 691 692 err_sec_release_qp_ctx: 693 for (i = i - 1; i >= 0; i--) 694 sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]); 695 kfree(ctx->qp_ctx); 696 err_destroy_qps: 697 sec_destroy_qps(ctx->qps, sec->ctx_q_num); 698 return ret; 699 } 700 701 static void sec_ctx_base_uninit(struct sec_ctx *ctx) 702 { 703 int i; 704 705 if (!ctx->qps) 706 return; 707 708 for (i = 0; i < ctx->sec->ctx_q_num; i++) 709 sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]); 710 711 sec_destroy_qps(ctx->qps, ctx->sec->ctx_q_num); 712 kfree(ctx->qp_ctx); 713 } 714 715 static int sec_cipher_init(struct sec_ctx *ctx) 716 { 717 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 718 719 if (!ctx->qps) 720 return 0; 721 722 c_ctx->c_key = dma_alloc_coherent(ctx->dev, SEC_MAX_KEY_SIZE, 723 &c_ctx->c_key_dma, GFP_KERNEL); 724 if (!c_ctx->c_key) 725 return -ENOMEM; 726 727 return 0; 728 } 729 730 static void sec_cipher_uninit(struct sec_ctx *ctx) 731 { 732 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 733 734 if (!ctx->qps) 735 return; 736 737 memzero_explicit(c_ctx->c_key, SEC_MAX_KEY_SIZE); 738 dma_free_coherent(ctx->dev, SEC_MAX_KEY_SIZE, 739 c_ctx->c_key, c_ctx->c_key_dma); 740 } 741 742 static int sec_auth_init(struct sec_ctx *ctx) 743 { 744 struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 745 746 a_ctx->a_key = dma_alloc_coherent(ctx->dev, SEC_MAX_AKEY_SIZE, 747 &a_ctx->a_key_dma, GFP_KERNEL); 748 if (!a_ctx->a_key) 749 return -ENOMEM; 750 751 return 0; 752 } 753 754 static void sec_auth_uninit(struct sec_ctx *ctx) 755 { 756 struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 757 758 if (!ctx->qps) 759 return; 760 761 memzero_explicit(a_ctx->a_key, SEC_MAX_AKEY_SIZE); 762 dma_free_coherent(ctx->dev, SEC_MAX_AKEY_SIZE, 763 a_ctx->a_key, a_ctx->a_key_dma); 764 } 765 766 static int sec_skcipher_fbtfm_init(struct crypto_skcipher *tfm) 767 { 768 const char *alg = crypto_tfm_alg_name(&tfm->base); 769 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 770 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 771 772 c_ctx->fallback = false; 773 774 c_ctx->fbtfm = crypto_alloc_sync_skcipher(alg, 0, 775 CRYPTO_ALG_NEED_FALLBACK); 776 if (IS_ERR(c_ctx->fbtfm)) { 777 pr_err("failed to alloc fallback tfm for %s!\n", alg); 778 return PTR_ERR(c_ctx->fbtfm); 779 } 780 781 return 0; 782 } 783 784 static int sec_skcipher_init(struct crypto_skcipher *tfm) 785 { 786 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 787 int ret; 788 789 ctx->alg_type = SEC_SKCIPHER; 790 crypto_skcipher_set_reqsize_dma(tfm, sizeof(struct sec_req)); 791 ctx->c_ctx.ivsize = crypto_skcipher_ivsize(tfm); 792 if (ctx->c_ctx.ivsize > SEC_IV_SIZE) { 793 pr_err("get error skcipher iv size!\n"); 794 return -EINVAL; 795 } 796 797 ret = sec_ctx_base_init(ctx); 798 if (ret && ret != -ENODEV) 799 return ret; 800 801 ret = sec_cipher_init(ctx); 802 if (ret) 803 goto err_cipher_init; 804 805 ret = sec_skcipher_fbtfm_init(tfm); 806 if (ret) 807 goto err_fbtfm_init; 808 809 return 0; 810 811 err_fbtfm_init: 812 sec_cipher_uninit(ctx); 813 err_cipher_init: 814 sec_ctx_base_uninit(ctx); 815 return ret; 816 } 817 818 static void sec_skcipher_uninit(struct crypto_skcipher *tfm) 819 { 820 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 821 822 if (ctx->c_ctx.fbtfm) 823 crypto_free_sync_skcipher(ctx->c_ctx.fbtfm); 824 825 sec_cipher_uninit(ctx); 826 sec_ctx_base_uninit(ctx); 827 } 828 829 static int sec_skcipher_3des_setkey(struct crypto_skcipher *tfm, const u8 *key, const u32 keylen) 830 { 831 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 832 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 833 int ret; 834 835 ret = verify_skcipher_des3_key(tfm, key); 836 if (ret) 837 return ret; 838 839 switch (keylen) { 840 case SEC_DES3_2KEY_SIZE: 841 c_ctx->c_key_len = SEC_CKEY_3DES_2KEY; 842 break; 843 case SEC_DES3_3KEY_SIZE: 844 c_ctx->c_key_len = SEC_CKEY_3DES_3KEY; 845 break; 846 default: 847 return -EINVAL; 848 } 849 850 return 0; 851 } 852 853 static int sec_skcipher_aes_sm4_setkey(struct sec_cipher_ctx *c_ctx, 854 const u32 keylen, 855 const enum sec_cmode c_mode) 856 { 857 if (c_mode == SEC_CMODE_XTS) { 858 switch (keylen) { 859 case SEC_XTS_MIN_KEY_SIZE: 860 c_ctx->c_key_len = SEC_CKEY_128BIT; 861 break; 862 case SEC_XTS_MID_KEY_SIZE: 863 c_ctx->fallback = true; 864 break; 865 case SEC_XTS_MAX_KEY_SIZE: 866 c_ctx->c_key_len = SEC_CKEY_256BIT; 867 break; 868 default: 869 pr_err("hisi_sec2: xts mode key error!\n"); 870 return -EINVAL; 871 } 872 } else { 873 if (c_ctx->c_alg == SEC_CALG_SM4 && 874 keylen != AES_KEYSIZE_128) { 875 pr_err("hisi_sec2: sm4 key error!\n"); 876 return -EINVAL; 877 } else { 878 switch (keylen) { 879 case AES_KEYSIZE_128: 880 c_ctx->c_key_len = SEC_CKEY_128BIT; 881 break; 882 case AES_KEYSIZE_192: 883 c_ctx->c_key_len = SEC_CKEY_192BIT; 884 break; 885 case AES_KEYSIZE_256: 886 c_ctx->c_key_len = SEC_CKEY_256BIT; 887 break; 888 default: 889 pr_err("hisi_sec2: aes key error!\n"); 890 return -EINVAL; 891 } 892 } 893 } 894 895 return 0; 896 } 897 898 static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, 899 const u32 keylen, const enum sec_calg c_alg, 900 const enum sec_cmode c_mode) 901 { 902 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 903 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 904 struct device *dev = ctx->dev; 905 int ret; 906 907 if (!ctx->qps) 908 goto set_soft_key; 909 910 if (c_mode == SEC_CMODE_XTS) { 911 ret = xts_verify_key(tfm, key, keylen); 912 if (ret) { 913 dev_err(dev, "xts mode key err!\n"); 914 return ret; 915 } 916 } 917 918 c_ctx->c_alg = c_alg; 919 c_ctx->c_mode = c_mode; 920 921 switch (c_alg) { 922 case SEC_CALG_3DES: 923 ret = sec_skcipher_3des_setkey(tfm, key, keylen); 924 break; 925 case SEC_CALG_AES: 926 case SEC_CALG_SM4: 927 ret = sec_skcipher_aes_sm4_setkey(c_ctx, keylen, c_mode); 928 break; 929 default: 930 dev_err(dev, "sec c_alg err!\n"); 931 return -EINVAL; 932 } 933 934 if (ret) { 935 dev_err(dev, "set sec key err!\n"); 936 return ret; 937 } 938 939 memcpy(c_ctx->c_key, key, keylen); 940 941 set_soft_key: 942 ret = crypto_sync_skcipher_setkey(c_ctx->fbtfm, key, keylen); 943 if (ret) { 944 dev_err(dev, "failed to set fallback skcipher key!\n"); 945 return ret; 946 } 947 948 return 0; 949 } 950 951 #define GEN_SEC_SETKEY_FUNC(name, c_alg, c_mode) \ 952 static int sec_setkey_##name(struct crypto_skcipher *tfm, const u8 *key,\ 953 u32 keylen) \ 954 { \ 955 return sec_skcipher_setkey(tfm, key, keylen, c_alg, c_mode); \ 956 } 957 958 GEN_SEC_SETKEY_FUNC(aes_ecb, SEC_CALG_AES, SEC_CMODE_ECB) 959 GEN_SEC_SETKEY_FUNC(aes_cbc, SEC_CALG_AES, SEC_CMODE_CBC) 960 GEN_SEC_SETKEY_FUNC(aes_xts, SEC_CALG_AES, SEC_CMODE_XTS) 961 GEN_SEC_SETKEY_FUNC(aes_ctr, SEC_CALG_AES, SEC_CMODE_CTR) 962 GEN_SEC_SETKEY_FUNC(3des_ecb, SEC_CALG_3DES, SEC_CMODE_ECB) 963 GEN_SEC_SETKEY_FUNC(3des_cbc, SEC_CALG_3DES, SEC_CMODE_CBC) 964 GEN_SEC_SETKEY_FUNC(sm4_xts, SEC_CALG_SM4, SEC_CMODE_XTS) 965 GEN_SEC_SETKEY_FUNC(sm4_cbc, SEC_CALG_SM4, SEC_CMODE_CBC) 966 GEN_SEC_SETKEY_FUNC(sm4_ctr, SEC_CALG_SM4, SEC_CMODE_CTR) 967 968 static int sec_cipher_pbuf_map(struct sec_ctx *ctx, struct sec_req *req, 969 struct scatterlist *src) 970 { 971 struct aead_request *aead_req = req->aead_req.aead_req; 972 struct sec_cipher_req *c_req = &req->c_req; 973 struct sec_qp_ctx *qp_ctx = req->qp_ctx; 974 struct sec_request_buf *buf = &req->buf; 975 struct device *dev = ctx->dev; 976 int copy_size, pbuf_length; 977 int req_id = req->req_id; 978 struct crypto_aead *tfm; 979 u8 *mac_offset, *pbuf; 980 size_t authsize; 981 982 if (ctx->alg_type == SEC_AEAD) 983 copy_size = aead_req->cryptlen + aead_req->assoclen; 984 else 985 copy_size = c_req->c_len; 986 987 988 pbuf = req->req_id < 0 ? buf->pbuf : qp_ctx->res[req_id].pbuf; 989 pbuf_length = sg_copy_to_buffer(src, sg_nents(src), pbuf, copy_size); 990 if (unlikely(pbuf_length != copy_size)) { 991 dev_err(dev, "copy src data to pbuf error!\n"); 992 return -EINVAL; 993 } 994 if (!c_req->encrypt && ctx->alg_type == SEC_AEAD) { 995 tfm = crypto_aead_reqtfm(aead_req); 996 authsize = crypto_aead_authsize(tfm); 997 mac_offset = pbuf + copy_size - authsize; 998 memcpy(req->aead_req.out_mac, mac_offset, authsize); 999 } 1000 1001 if (req->req_id < 0) { 1002 buf->in_dma = dma_map_single(dev, buf->pbuf, SEC_PBUF_SZ, DMA_BIDIRECTIONAL); 1003 if (unlikely(dma_mapping_error(dev, buf->in_dma))) 1004 return -ENOMEM; 1005 1006 buf->out_dma = buf->in_dma; 1007 return 0; 1008 } 1009 1010 req->in_dma = qp_ctx->res[req_id].pbuf_dma; 1011 c_req->c_out_dma = req->in_dma; 1012 1013 return 0; 1014 } 1015 1016 static void sec_cipher_pbuf_unmap(struct sec_ctx *ctx, struct sec_req *req, 1017 struct scatterlist *dst) 1018 { 1019 struct aead_request *aead_req = req->aead_req.aead_req; 1020 struct sec_cipher_req *c_req = &req->c_req; 1021 struct sec_qp_ctx *qp_ctx = req->qp_ctx; 1022 struct sec_request_buf *buf = &req->buf; 1023 int copy_size, pbuf_length; 1024 int req_id = req->req_id; 1025 1026 if (ctx->alg_type == SEC_AEAD) 1027 copy_size = c_req->c_len + aead_req->assoclen; 1028 else 1029 copy_size = c_req->c_len; 1030 1031 if (req->req_id < 0) 1032 pbuf_length = sg_copy_from_buffer(dst, sg_nents(dst), buf->pbuf, copy_size); 1033 else 1034 pbuf_length = sg_copy_from_buffer(dst, sg_nents(dst), qp_ctx->res[req_id].pbuf, 1035 copy_size); 1036 if (unlikely(pbuf_length != copy_size)) 1037 dev_err(ctx->dev, "copy pbuf data to dst error!\n"); 1038 1039 if (req->req_id < 0) 1040 dma_unmap_single(ctx->dev, buf->in_dma, SEC_PBUF_SZ, DMA_BIDIRECTIONAL); 1041 } 1042 1043 static int sec_aead_mac_init(struct sec_aead_req *req) 1044 { 1045 struct aead_request *aead_req = req->aead_req; 1046 struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req); 1047 size_t authsize = crypto_aead_authsize(tfm); 1048 struct scatterlist *sgl = aead_req->src; 1049 u8 *mac_out = req->out_mac; 1050 size_t copy_size; 1051 off_t skip_size; 1052 1053 /* Copy input mac */ 1054 skip_size = aead_req->assoclen + aead_req->cryptlen - authsize; 1055 copy_size = sg_pcopy_to_buffer(sgl, sg_nents(sgl), mac_out, authsize, skip_size); 1056 if (unlikely(copy_size != authsize)) 1057 return -EINVAL; 1058 1059 return 0; 1060 } 1061 1062 static void fill_sg_to_hw_sge(struct scatterlist *sgl, struct sec_hw_sge *hw_sge) 1063 { 1064 hw_sge->buf = sg_dma_address(sgl); 1065 hw_sge->len = cpu_to_le32(sg_dma_len(sgl)); 1066 hw_sge->page_ctrl = sg_virt(sgl); 1067 } 1068 1069 static int sec_cipher_to_hw_sgl(struct device *dev, struct scatterlist *src, 1070 struct sec_hw_sgl *src_in, dma_addr_t *hw_sgl_dma, 1071 int dma_dir) 1072 { 1073 struct sec_hw_sge *curr_hw_sge = src_in->sge_entries; 1074 u32 i, sg_n, sg_n_mapped; 1075 struct scatterlist *sg; 1076 u32 sge_var = 0; 1077 1078 sg_n = sg_nents(src); 1079 sg_n_mapped = dma_map_sg(dev, src, sg_n, dma_dir); 1080 if (unlikely(!sg_n_mapped)) { 1081 dev_err(dev, "dma mapping for SG error!\n"); 1082 return -EINVAL; 1083 } else if (unlikely(sg_n_mapped > SEC_SGE_NR_NUM)) { 1084 dev_err(dev, "the number of entries in input scatterlist error!\n"); 1085 dma_unmap_sg(dev, src, sg_n, dma_dir); 1086 return -EINVAL; 1087 } 1088 1089 for_each_sg(src, sg, sg_n_mapped, i) { 1090 fill_sg_to_hw_sge(sg, curr_hw_sge); 1091 curr_hw_sge++; 1092 sge_var++; 1093 } 1094 1095 src_in->entry_sum_in_sgl = cpu_to_le16(sge_var); 1096 src_in->entry_sum_in_chain = cpu_to_le16(SEC_SGE_NR_NUM); 1097 src_in->entry_length_in_sgl = cpu_to_le16(SEC_SGE_NR_NUM); 1098 *hw_sgl_dma = dma_map_single(dev, src_in, sizeof(struct sec_hw_sgl), dma_dir); 1099 if (unlikely(dma_mapping_error(dev, *hw_sgl_dma))) { 1100 dma_unmap_sg(dev, src, sg_n, dma_dir); 1101 return -ENOMEM; 1102 } 1103 1104 return 0; 1105 } 1106 1107 static void sec_cipher_put_hw_sgl(struct device *dev, struct scatterlist *src, 1108 dma_addr_t src_in, int dma_dir) 1109 { 1110 dma_unmap_single(dev, src_in, sizeof(struct sec_hw_sgl), dma_dir); 1111 dma_unmap_sg(dev, src, sg_nents(src), dma_dir); 1112 } 1113 1114 static int sec_cipher_map_sgl(struct device *dev, struct sec_req *req, 1115 struct scatterlist *src, struct scatterlist *dst) 1116 { 1117 struct sec_hw_sgl *src_in = &req->buf.data_buf.in; 1118 struct sec_hw_sgl *dst_out = &req->buf.data_buf.out; 1119 int ret; 1120 1121 if (dst == src) { 1122 ret = sec_cipher_to_hw_sgl(dev, src, src_in, &req->buf.in_dma, 1123 DMA_BIDIRECTIONAL); 1124 req->buf.out_dma = req->buf.in_dma; 1125 return ret; 1126 } 1127 1128 ret = sec_cipher_to_hw_sgl(dev, src, src_in, &req->buf.in_dma, DMA_TO_DEVICE); 1129 if (unlikely(ret)) 1130 return ret; 1131 1132 ret = sec_cipher_to_hw_sgl(dev, dst, dst_out, &req->buf.out_dma, 1133 DMA_FROM_DEVICE); 1134 if (unlikely(ret)) { 1135 sec_cipher_put_hw_sgl(dev, src, req->buf.in_dma, DMA_TO_DEVICE); 1136 return ret; 1137 } 1138 1139 return 0; 1140 } 1141 1142 static int sec_cipher_map_inner(struct sec_ctx *ctx, struct sec_req *req, 1143 struct scatterlist *src, struct scatterlist *dst) 1144 { 1145 struct sec_cipher_req *c_req = &req->c_req; 1146 struct sec_aead_req *a_req = &req->aead_req; 1147 struct sec_qp_ctx *qp_ctx = req->qp_ctx; 1148 struct sec_alg_res *res = &qp_ctx->res[req->req_id]; 1149 struct device *dev = ctx->dev; 1150 enum dma_data_direction src_direction; 1151 int ret; 1152 1153 if (req->use_pbuf) { 1154 c_req->c_ivin = res->pbuf + SEC_PBUF_IV_OFFSET; 1155 c_req->c_ivin_dma = res->pbuf_dma + SEC_PBUF_IV_OFFSET; 1156 if (ctx->alg_type == SEC_AEAD) { 1157 a_req->a_ivin = res->a_ivin; 1158 a_req->a_ivin_dma = res->a_ivin_dma; 1159 a_req->out_mac = res->pbuf + SEC_PBUF_MAC_OFFSET; 1160 a_req->out_mac_dma = res->pbuf_dma + 1161 SEC_PBUF_MAC_OFFSET; 1162 } 1163 return sec_cipher_pbuf_map(ctx, req, src); 1164 } 1165 1166 c_req->c_ivin = res->c_ivin; 1167 c_req->c_ivin_dma = res->c_ivin_dma; 1168 if (ctx->alg_type == SEC_AEAD) { 1169 a_req->a_ivin = res->a_ivin; 1170 a_req->a_ivin_dma = res->a_ivin_dma; 1171 a_req->out_mac = res->out_mac; 1172 a_req->out_mac_dma = res->out_mac_dma; 1173 } 1174 1175 src_direction = dst == src ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 1176 req->in = hisi_acc_sg_buf_map_to_hw_sgl(dev, src, 1177 qp_ctx->c_in_pool, 1178 req->req_id, 1179 &req->in_dma, src_direction); 1180 if (IS_ERR(req->in)) { 1181 dev_err(dev, "fail to dma map input sgl buffers!\n"); 1182 return PTR_ERR(req->in); 1183 } 1184 1185 if (!c_req->encrypt && ctx->alg_type == SEC_AEAD) { 1186 ret = sec_aead_mac_init(a_req); 1187 if (unlikely(ret)) { 1188 dev_err(dev, "fail to init mac data for ICV!\n"); 1189 hisi_acc_sg_buf_unmap(dev, src, req->in, src_direction); 1190 return ret; 1191 } 1192 } 1193 1194 if (dst == src) { 1195 c_req->c_out = req->in; 1196 c_req->c_out_dma = req->in_dma; 1197 } else { 1198 c_req->c_out = hisi_acc_sg_buf_map_to_hw_sgl(dev, dst, 1199 qp_ctx->c_out_pool, 1200 req->req_id, 1201 &c_req->c_out_dma, 1202 DMA_FROM_DEVICE); 1203 1204 if (IS_ERR(c_req->c_out)) { 1205 dev_err(dev, "fail to dma map output sgl buffers!\n"); 1206 hisi_acc_sg_buf_unmap(dev, src, req->in, src_direction); 1207 return PTR_ERR(c_req->c_out); 1208 } 1209 } 1210 1211 return 0; 1212 } 1213 1214 static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req, 1215 struct scatterlist *src, struct scatterlist *dst) 1216 { 1217 struct sec_aead_req *a_req = &req->aead_req; 1218 struct sec_cipher_req *c_req = &req->c_req; 1219 bool is_aead = (ctx->alg_type == SEC_AEAD); 1220 struct device *dev = ctx->dev; 1221 int ret = -ENOMEM; 1222 1223 if (req->req_id >= 0) 1224 return sec_cipher_map_inner(ctx, req, src, dst); 1225 1226 c_req->c_ivin = c_req->c_ivin_buf; 1227 c_req->c_ivin_dma = dma_map_single(dev, c_req->c_ivin, 1228 SEC_IV_SIZE, DMA_TO_DEVICE); 1229 if (unlikely(dma_mapping_error(dev, c_req->c_ivin_dma))) 1230 return -ENOMEM; 1231 1232 if (is_aead) { 1233 a_req->a_ivin = a_req->a_ivin_buf; 1234 a_req->out_mac = a_req->out_mac_buf; 1235 a_req->a_ivin_dma = dma_map_single(dev, a_req->a_ivin, 1236 SEC_IV_SIZE, DMA_TO_DEVICE); 1237 if (unlikely(dma_mapping_error(dev, a_req->a_ivin_dma))) 1238 goto free_c_ivin_dma; 1239 1240 a_req->out_mac_dma = dma_map_single(dev, a_req->out_mac, 1241 SEC_MAX_MAC_LEN, DMA_BIDIRECTIONAL); 1242 if (unlikely(dma_mapping_error(dev, a_req->out_mac_dma))) 1243 goto free_a_ivin_dma; 1244 } 1245 if (req->use_pbuf) { 1246 ret = sec_cipher_pbuf_map(ctx, req, src); 1247 if (unlikely(ret)) 1248 goto free_out_mac_dma; 1249 1250 return 0; 1251 } 1252 1253 if (!c_req->encrypt && is_aead) { 1254 ret = sec_aead_mac_init(a_req); 1255 if (unlikely(ret)) { 1256 dev_err(dev, "fail to init mac data for ICV!\n"); 1257 goto free_out_mac_dma; 1258 } 1259 } 1260 1261 ret = sec_cipher_map_sgl(dev, req, src, dst); 1262 if (unlikely(ret)) { 1263 dev_err(dev, "fail to dma map input sgl buffers!\n"); 1264 goto free_out_mac_dma; 1265 } 1266 1267 return 0; 1268 1269 free_out_mac_dma: 1270 if (is_aead) 1271 dma_unmap_single(dev, a_req->out_mac_dma, SEC_MAX_MAC_LEN, DMA_BIDIRECTIONAL); 1272 free_a_ivin_dma: 1273 if (is_aead) 1274 dma_unmap_single(dev, a_req->a_ivin_dma, SEC_IV_SIZE, DMA_TO_DEVICE); 1275 free_c_ivin_dma: 1276 dma_unmap_single(dev, c_req->c_ivin_dma, SEC_IV_SIZE, DMA_TO_DEVICE); 1277 return ret; 1278 } 1279 1280 static void sec_cipher_unmap(struct sec_ctx *ctx, struct sec_req *req, 1281 struct scatterlist *src, struct scatterlist *dst) 1282 { 1283 struct sec_aead_req *a_req = &req->aead_req; 1284 struct sec_cipher_req *c_req = &req->c_req; 1285 struct device *dev = ctx->dev; 1286 1287 if (req->req_id >= 0) { 1288 if (req->use_pbuf) { 1289 sec_cipher_pbuf_unmap(ctx, req, dst); 1290 } else { 1291 if (dst != src) { 1292 hisi_acc_sg_buf_unmap(dev, dst, c_req->c_out, DMA_FROM_DEVICE); 1293 hisi_acc_sg_buf_unmap(dev, src, req->in, DMA_TO_DEVICE); 1294 } else { 1295 hisi_acc_sg_buf_unmap(dev, src, req->in, DMA_BIDIRECTIONAL); 1296 } 1297 } 1298 return; 1299 } 1300 1301 if (req->use_pbuf) { 1302 sec_cipher_pbuf_unmap(ctx, req, dst); 1303 } else { 1304 if (dst != src) { 1305 sec_cipher_put_hw_sgl(dev, dst, req->buf.out_dma, DMA_FROM_DEVICE); 1306 sec_cipher_put_hw_sgl(dev, src, req->buf.in_dma, DMA_TO_DEVICE); 1307 } else { 1308 sec_cipher_put_hw_sgl(dev, src, req->buf.in_dma, DMA_BIDIRECTIONAL); 1309 } 1310 } 1311 1312 dma_unmap_single(dev, c_req->c_ivin_dma, SEC_IV_SIZE, DMA_TO_DEVICE); 1313 if (ctx->alg_type == SEC_AEAD) { 1314 dma_unmap_single(dev, a_req->a_ivin_dma, SEC_IV_SIZE, DMA_TO_DEVICE); 1315 dma_unmap_single(dev, a_req->out_mac_dma, SEC_MAX_MAC_LEN, DMA_BIDIRECTIONAL); 1316 } 1317 } 1318 1319 static int sec_skcipher_sgl_map(struct sec_ctx *ctx, struct sec_req *req) 1320 { 1321 struct skcipher_request *sq = req->c_req.sk_req; 1322 1323 return sec_cipher_map(ctx, req, sq->src, sq->dst); 1324 } 1325 1326 static void sec_skcipher_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req) 1327 { 1328 struct skcipher_request *sq = req->c_req.sk_req; 1329 1330 sec_cipher_unmap(ctx, req, sq->src, sq->dst); 1331 } 1332 1333 static int sec_aead_aes_set_key(struct sec_cipher_ctx *c_ctx, 1334 struct crypto_authenc_keys *keys) 1335 { 1336 switch (keys->enckeylen) { 1337 case AES_KEYSIZE_128: 1338 c_ctx->c_key_len = SEC_CKEY_128BIT; 1339 break; 1340 case AES_KEYSIZE_192: 1341 c_ctx->c_key_len = SEC_CKEY_192BIT; 1342 break; 1343 case AES_KEYSIZE_256: 1344 c_ctx->c_key_len = SEC_CKEY_256BIT; 1345 break; 1346 default: 1347 pr_err("hisi_sec2: aead aes key error!\n"); 1348 return -EINVAL; 1349 } 1350 memcpy(c_ctx->c_key, keys->enckey, keys->enckeylen); 1351 1352 return 0; 1353 } 1354 1355 static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx, 1356 struct crypto_authenc_keys *keys) 1357 { 1358 struct crypto_shash *hash_tfm = ctx->hash_tfm; 1359 int blocksize, digestsize, ret; 1360 1361 blocksize = crypto_shash_blocksize(hash_tfm); 1362 digestsize = crypto_shash_digestsize(hash_tfm); 1363 if (keys->authkeylen > blocksize) { 1364 ret = crypto_shash_tfm_digest(hash_tfm, keys->authkey, 1365 keys->authkeylen, ctx->a_key); 1366 if (ret) { 1367 pr_err("hisi_sec2: aead auth digest error!\n"); 1368 return -EINVAL; 1369 } 1370 ctx->a_key_len = digestsize; 1371 } else { 1372 if (keys->authkeylen) 1373 memcpy(ctx->a_key, keys->authkey, keys->authkeylen); 1374 ctx->a_key_len = keys->authkeylen; 1375 } 1376 1377 return 0; 1378 } 1379 1380 static int sec_aead_setauthsize(struct crypto_aead *aead, unsigned int authsize) 1381 { 1382 struct crypto_tfm *tfm = crypto_aead_tfm(aead); 1383 struct sec_ctx *ctx = crypto_tfm_ctx(tfm); 1384 struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 1385 1386 return crypto_aead_setauthsize(a_ctx->fallback_aead_tfm, authsize); 1387 } 1388 1389 static int sec_aead_fallback_setkey(struct sec_auth_ctx *a_ctx, 1390 struct crypto_aead *tfm, const u8 *key, 1391 unsigned int keylen) 1392 { 1393 crypto_aead_clear_flags(a_ctx->fallback_aead_tfm, CRYPTO_TFM_REQ_MASK); 1394 crypto_aead_set_flags(a_ctx->fallback_aead_tfm, 1395 crypto_aead_get_flags(tfm) & CRYPTO_TFM_REQ_MASK); 1396 return crypto_aead_setkey(a_ctx->fallback_aead_tfm, key, keylen); 1397 } 1398 1399 static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, 1400 const u32 keylen, const enum sec_hash_alg a_alg, 1401 const enum sec_calg c_alg, 1402 const enum sec_cmode c_mode) 1403 { 1404 struct sec_ctx *ctx = crypto_aead_ctx(tfm); 1405 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 1406 struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 1407 struct device *dev = ctx->dev; 1408 struct crypto_authenc_keys keys; 1409 int ret; 1410 1411 if (!ctx->qps) 1412 return sec_aead_fallback_setkey(a_ctx, tfm, key, keylen); 1413 1414 ctx->a_ctx.a_alg = a_alg; 1415 ctx->c_ctx.c_alg = c_alg; 1416 c_ctx->c_mode = c_mode; 1417 1418 if (c_mode == SEC_CMODE_CCM || c_mode == SEC_CMODE_GCM) { 1419 ret = sec_skcipher_aes_sm4_setkey(c_ctx, keylen, c_mode); 1420 if (ret) { 1421 dev_err(dev, "set sec aes ccm cipher key err!\n"); 1422 return ret; 1423 } 1424 memcpy(c_ctx->c_key, key, keylen); 1425 1426 return sec_aead_fallback_setkey(a_ctx, tfm, key, keylen); 1427 } 1428 1429 ret = crypto_authenc_extractkeys(&keys, key, keylen); 1430 if (ret) { 1431 dev_err(dev, "sec extract aead keys err!\n"); 1432 goto bad_key; 1433 } 1434 1435 ret = sec_aead_aes_set_key(c_ctx, &keys); 1436 if (ret) { 1437 dev_err(dev, "set sec cipher key err!\n"); 1438 goto bad_key; 1439 } 1440 1441 ret = sec_aead_auth_set_key(&ctx->a_ctx, &keys); 1442 if (ret) { 1443 dev_err(dev, "set sec auth key err!\n"); 1444 goto bad_key; 1445 } 1446 1447 ret = sec_aead_fallback_setkey(a_ctx, tfm, key, keylen); 1448 if (ret) { 1449 dev_err(dev, "set sec fallback key err!\n"); 1450 goto bad_key; 1451 } 1452 1453 return 0; 1454 1455 bad_key: 1456 memzero_explicit(&keys, sizeof(struct crypto_authenc_keys)); 1457 return ret; 1458 } 1459 1460 1461 #define GEN_SEC_AEAD_SETKEY_FUNC(name, aalg, calg, cmode) \ 1462 static int sec_setkey_##name(struct crypto_aead *tfm, const u8 *key, u32 keylen) \ 1463 { \ 1464 return sec_aead_setkey(tfm, key, keylen, aalg, calg, cmode); \ 1465 } 1466 1467 GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha1, SEC_A_HMAC_SHA1, SEC_CALG_AES, SEC_CMODE_CBC) 1468 GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha256, SEC_A_HMAC_SHA256, SEC_CALG_AES, SEC_CMODE_CBC) 1469 GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha512, SEC_A_HMAC_SHA512, SEC_CALG_AES, SEC_CMODE_CBC) 1470 GEN_SEC_AEAD_SETKEY_FUNC(aes_ccm, 0, SEC_CALG_AES, SEC_CMODE_CCM) 1471 GEN_SEC_AEAD_SETKEY_FUNC(aes_gcm, 0, SEC_CALG_AES, SEC_CMODE_GCM) 1472 GEN_SEC_AEAD_SETKEY_FUNC(sm4_ccm, 0, SEC_CALG_SM4, SEC_CMODE_CCM) 1473 GEN_SEC_AEAD_SETKEY_FUNC(sm4_gcm, 0, SEC_CALG_SM4, SEC_CMODE_GCM) 1474 1475 static int sec_aead_sgl_map(struct sec_ctx *ctx, struct sec_req *req) 1476 { 1477 struct aead_request *aq = req->aead_req.aead_req; 1478 1479 return sec_cipher_map(ctx, req, aq->src, aq->dst); 1480 } 1481 1482 static void sec_aead_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req) 1483 { 1484 struct aead_request *aq = req->aead_req.aead_req; 1485 1486 sec_cipher_unmap(ctx, req, aq->src, aq->dst); 1487 } 1488 1489 static int sec_request_transfer(struct sec_ctx *ctx, struct sec_req *req) 1490 { 1491 int ret; 1492 1493 ret = ctx->req_op->buf_map(ctx, req); 1494 if (unlikely(ret)) 1495 return ret; 1496 1497 ctx->req_op->do_transfer(ctx, req); 1498 1499 ret = ctx->req_op->bd_fill(ctx, req); 1500 if (unlikely(ret)) 1501 goto unmap_req_buf; 1502 1503 return ret; 1504 1505 unmap_req_buf: 1506 ctx->req_op->buf_unmap(ctx, req); 1507 return ret; 1508 } 1509 1510 static void sec_request_untransfer(struct sec_ctx *ctx, struct sec_req *req) 1511 { 1512 ctx->req_op->buf_unmap(ctx, req); 1513 } 1514 1515 static void sec_skcipher_copy_iv(struct sec_ctx *ctx, struct sec_req *req) 1516 { 1517 struct skcipher_request *sk_req = req->c_req.sk_req; 1518 struct sec_cipher_req *c_req = &req->c_req; 1519 1520 memcpy(c_req->c_ivin, sk_req->iv, ctx->c_ctx.ivsize); 1521 } 1522 1523 static int sec_skcipher_bd_fill(struct sec_ctx *ctx, struct sec_req *req) 1524 { 1525 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 1526 struct sec_cipher_req *c_req = &req->c_req; 1527 struct sec_sqe *sec_sqe = &req->sec_sqe; 1528 u8 scene, sa_type, da_type; 1529 u8 bd_type, cipher; 1530 u8 de = 0; 1531 1532 memset(sec_sqe, 0, sizeof(struct sec_sqe)); 1533 1534 sec_sqe->type2.c_key_addr = cpu_to_le64(c_ctx->c_key_dma); 1535 sec_sqe->type2.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma); 1536 if (req->req_id < 0) { 1537 sec_sqe->type2.data_src_addr = cpu_to_le64(req->buf.in_dma); 1538 sec_sqe->type2.data_dst_addr = cpu_to_le64(req->buf.out_dma); 1539 } else { 1540 sec_sqe->type2.data_src_addr = cpu_to_le64(req->in_dma); 1541 sec_sqe->type2.data_dst_addr = cpu_to_le64(c_req->c_out_dma); 1542 } 1543 if (sec_sqe->type2.data_src_addr != sec_sqe->type2.data_dst_addr) 1544 de = 0x1 << SEC_DE_OFFSET; 1545 1546 sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_mode) << 1547 SEC_CMODE_OFFSET); 1548 sec_sqe->type2.c_alg = c_ctx->c_alg; 1549 sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_key_len) << 1550 SEC_CKEY_OFFSET); 1551 1552 bd_type = SEC_BD_TYPE2; 1553 if (c_req->encrypt) 1554 cipher = SEC_CIPHER_ENC << SEC_CIPHER_OFFSET; 1555 else 1556 cipher = SEC_CIPHER_DEC << SEC_CIPHER_OFFSET; 1557 sec_sqe->type_cipher_auth = bd_type | cipher; 1558 1559 /* Set destination and source address type */ 1560 if (req->use_pbuf) { 1561 sa_type = SEC_PBUF << SEC_SRC_SGL_OFFSET; 1562 da_type = SEC_PBUF << SEC_DST_SGL_OFFSET; 1563 } else { 1564 sa_type = SEC_SGL << SEC_SRC_SGL_OFFSET; 1565 da_type = SEC_SGL << SEC_DST_SGL_OFFSET; 1566 } 1567 1568 sec_sqe->sdm_addr_type |= da_type; 1569 scene = SEC_COMM_SCENE << SEC_SCENE_OFFSET; 1570 1571 sec_sqe->sds_sa_type = (de | scene | sa_type); 1572 1573 sec_sqe->type2.clen_ivhlen |= cpu_to_le32(c_req->c_len); 1574 1575 return 0; 1576 } 1577 1578 static int sec_skcipher_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req) 1579 { 1580 struct sec_sqe3 *sec_sqe3 = &req->sec_sqe3; 1581 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 1582 struct sec_cipher_req *c_req = &req->c_req; 1583 u32 bd_param = 0; 1584 u16 cipher; 1585 1586 memset(sec_sqe3, 0, sizeof(struct sec_sqe3)); 1587 1588 sec_sqe3->c_key_addr = cpu_to_le64(c_ctx->c_key_dma); 1589 sec_sqe3->no_scene.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma); 1590 if (req->req_id < 0) { 1591 sec_sqe3->data_src_addr = cpu_to_le64(req->buf.in_dma); 1592 sec_sqe3->data_dst_addr = cpu_to_le64(req->buf.out_dma); 1593 } else { 1594 sec_sqe3->data_src_addr = cpu_to_le64(req->in_dma); 1595 sec_sqe3->data_dst_addr = cpu_to_le64(c_req->c_out_dma); 1596 } 1597 if (sec_sqe3->data_src_addr != sec_sqe3->data_dst_addr) 1598 bd_param |= 0x1 << SEC_DE_OFFSET_V3; 1599 1600 sec_sqe3->c_mode_alg = ((u8)c_ctx->c_alg << SEC_CALG_OFFSET_V3) | 1601 c_ctx->c_mode; 1602 sec_sqe3->c_icv_key |= cpu_to_le16(((u16)c_ctx->c_key_len) << 1603 SEC_CKEY_OFFSET_V3); 1604 1605 if (c_req->encrypt) 1606 cipher = SEC_CIPHER_ENC; 1607 else 1608 cipher = SEC_CIPHER_DEC; 1609 sec_sqe3->c_icv_key |= cpu_to_le16(cipher); 1610 1611 /* Set the CTR counter mode is 128bit rollover */ 1612 sec_sqe3->auth_mac_key = cpu_to_le32((u32)SEC_CTR_CNT_ROLLOVER << 1613 SEC_CTR_CNT_OFFSET); 1614 1615 if (req->use_pbuf) { 1616 bd_param |= SEC_PBUF << SEC_SRC_SGL_OFFSET_V3; 1617 bd_param |= SEC_PBUF << SEC_DST_SGL_OFFSET_V3; 1618 } else { 1619 bd_param |= SEC_SGL << SEC_SRC_SGL_OFFSET_V3; 1620 bd_param |= SEC_SGL << SEC_DST_SGL_OFFSET_V3; 1621 } 1622 1623 bd_param |= SEC_COMM_SCENE << SEC_SCENE_OFFSET_V3; 1624 1625 bd_param |= SEC_BD_TYPE3; 1626 sec_sqe3->bd_param = cpu_to_le32(bd_param); 1627 1628 sec_sqe3->c_len_ivin |= cpu_to_le32(c_req->c_len); 1629 sec_sqe3->tag = cpu_to_le64((unsigned long)req); 1630 1631 return 0; 1632 } 1633 1634 /* increment counter (128-bit int) */ 1635 static void ctr_iv_inc(__u8 *counter, __u8 bits, __u32 nums) 1636 { 1637 do { 1638 --bits; 1639 nums += counter[bits]; 1640 counter[bits] = nums & BITS_MASK; 1641 nums >>= BYTE_BITS; 1642 } while (bits && nums); 1643 } 1644 1645 static void sec_update_iv(struct sec_req *req, enum sec_alg_type alg_type) 1646 { 1647 struct aead_request *aead_req = req->aead_req.aead_req; 1648 struct skcipher_request *sk_req = req->c_req.sk_req; 1649 u32 iv_size = req->ctx->c_ctx.ivsize; 1650 struct scatterlist *sgl; 1651 unsigned int cryptlen; 1652 size_t sz; 1653 u8 *iv; 1654 1655 if (alg_type == SEC_SKCIPHER) { 1656 sgl = req->c_req.encrypt ? sk_req->dst : sk_req->src; 1657 iv = sk_req->iv; 1658 cryptlen = sk_req->cryptlen; 1659 } else { 1660 sgl = req->c_req.encrypt ? aead_req->dst : aead_req->src; 1661 iv = aead_req->iv; 1662 cryptlen = aead_req->cryptlen; 1663 } 1664 1665 if (req->ctx->c_ctx.c_mode == SEC_CMODE_CBC) { 1666 sz = sg_pcopy_to_buffer(sgl, sg_nents(sgl), iv, iv_size, 1667 cryptlen - iv_size); 1668 if (unlikely(sz != iv_size)) 1669 dev_err(req->ctx->dev, "copy output iv error!\n"); 1670 } else { 1671 sz = (cryptlen + iv_size - 1) / iv_size; 1672 ctr_iv_inc(iv, iv_size, sz); 1673 } 1674 } 1675 1676 static void sec_skcipher_callback(struct sec_ctx *ctx, struct sec_req *req, 1677 int err) 1678 { 1679 struct sec_qp_ctx *qp_ctx = req->qp_ctx; 1680 1681 if (req->req_id >= 0) 1682 sec_free_req_id(req); 1683 1684 /* IV output at encrypto of CBC/CTR mode */ 1685 if (!err && (ctx->c_ctx.c_mode == SEC_CMODE_CBC || 1686 ctx->c_ctx.c_mode == SEC_CMODE_CTR) && req->c_req.encrypt) 1687 sec_update_iv(req, SEC_SKCIPHER); 1688 1689 crypto_request_complete(req->base, err); 1690 sec_alg_send_backlog(ctx, qp_ctx); 1691 } 1692 1693 static void set_aead_auth_iv(struct sec_ctx *ctx, struct sec_req *req) 1694 { 1695 struct aead_request *aead_req = req->aead_req.aead_req; 1696 struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req); 1697 size_t authsize = crypto_aead_authsize(tfm); 1698 struct sec_aead_req *a_req = &req->aead_req; 1699 struct sec_cipher_req *c_req = &req->c_req; 1700 u32 data_size = aead_req->cryptlen; 1701 u8 flage = 0; 1702 u8 cm, cl, i; 1703 1704 /* the specification has been checked in aead_iv_demension_check() */ 1705 cl = c_req->c_ivin[0] + 1; 1706 c_req->c_ivin[ctx->c_ctx.ivsize - cl] = 0x00; 1707 memset(&c_req->c_ivin[ctx->c_ctx.ivsize - cl], 0, cl); 1708 c_req->c_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE1] = IV_CTR_INIT; 1709 1710 /* the last 3bit is L' */ 1711 flage |= c_req->c_ivin[0] & IV_CL_MASK; 1712 1713 /* the M' is bit3~bit5, the Flags is bit6 */ 1714 cm = (authsize - IV_CM_CAL_NUM) / IV_CM_CAL_NUM; 1715 flage |= cm << IV_CM_OFFSET; 1716 if (aead_req->assoclen) 1717 flage |= 0x01 << IV_FLAGS_OFFSET; 1718 1719 memcpy(a_req->a_ivin, c_req->c_ivin, ctx->c_ctx.ivsize); 1720 a_req->a_ivin[0] = flage; 1721 1722 /* 1723 * the last 32bit is counter's initial number, 1724 * but the nonce uses the first 16bit 1725 * the tail 16bit fill with the cipher length 1726 * When CL is 3, the tail 24bit fill with the cipher length. 1727 */ 1728 if (!c_req->encrypt) 1729 data_size = aead_req->cryptlen - authsize; 1730 1731 for (i = 1; i <= cl; i++) { 1732 a_req->a_ivin[ctx->c_ctx.ivsize - i] = 1733 data_size & IV_LAST_BYTE_MASK; 1734 data_size >>= IV_BYTE_OFFSET; 1735 } 1736 } 1737 1738 static void sec_aead_set_iv(struct sec_ctx *ctx, struct sec_req *req) 1739 { 1740 struct aead_request *aead_req = req->aead_req.aead_req; 1741 struct sec_aead_req *a_req = &req->aead_req; 1742 struct sec_cipher_req *c_req = &req->c_req; 1743 1744 memcpy(c_req->c_ivin, aead_req->iv, ctx->c_ctx.ivsize); 1745 1746 if (ctx->c_ctx.c_mode == SEC_CMODE_CCM) { 1747 /* 1748 * CCM 16Byte Cipher_IV: {1B_Flage,13B_IV,2B_counter}, 1749 * the counter must set to 0x01 1750 * CCM 16Byte Auth_IV: {1B_AFlage,13B_IV,2B_Ptext_length} 1751 */ 1752 set_aead_auth_iv(ctx, req); 1753 } else if (ctx->c_ctx.c_mode == SEC_CMODE_GCM) { 1754 /* GCM 12Byte Cipher_IV == Auth_IV */ 1755 memcpy(a_req->a_ivin, c_req->c_ivin, SEC_AIV_SIZE); 1756 } 1757 } 1758 1759 static void sec_auth_bd_fill_xcm(struct sec_auth_ctx *ctx, int dir, 1760 struct sec_req *req, struct sec_sqe *sec_sqe) 1761 { 1762 struct sec_aead_req *a_req = &req->aead_req; 1763 struct aead_request *aq = a_req->aead_req; 1764 struct crypto_aead *tfm = crypto_aead_reqtfm(aq); 1765 size_t authsize = crypto_aead_authsize(tfm); 1766 1767 /* C_ICV_Len is MAC size, 0x4 ~ 0x10 */ 1768 sec_sqe->type2.icvw_kmode |= cpu_to_le16((u16)authsize); 1769 1770 /* mode set to CCM/GCM, don't set {A_Alg, AKey_Len, MAC_Len} */ 1771 sec_sqe->type2.a_key_addr = sec_sqe->type2.c_key_addr; 1772 sec_sqe->type2.a_ivin_addr = cpu_to_le64(a_req->a_ivin_dma); 1773 sec_sqe->type_cipher_auth |= SEC_NO_AUTH << SEC_AUTH_OFFSET; 1774 1775 if (dir) 1776 sec_sqe->sds_sa_type &= SEC_CIPHER_AUTH; 1777 else 1778 sec_sqe->sds_sa_type |= SEC_AUTH_CIPHER; 1779 1780 sec_sqe->type2.alen_ivllen = cpu_to_le32(aq->assoclen); 1781 sec_sqe->type2.auth_src_offset = cpu_to_le16(0x0); 1782 sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen); 1783 1784 sec_sqe->type2.mac_addr = cpu_to_le64(a_req->out_mac_dma); 1785 } 1786 1787 static void sec_auth_bd_fill_xcm_v3(struct sec_auth_ctx *ctx, int dir, 1788 struct sec_req *req, struct sec_sqe3 *sqe3) 1789 { 1790 struct sec_aead_req *a_req = &req->aead_req; 1791 struct aead_request *aq = a_req->aead_req; 1792 struct crypto_aead *tfm = crypto_aead_reqtfm(aq); 1793 size_t authsize = crypto_aead_authsize(tfm); 1794 1795 /* C_ICV_Len is MAC size, 0x4 ~ 0x10 */ 1796 sqe3->c_icv_key |= cpu_to_le16((u16)authsize << SEC_MAC_OFFSET_V3); 1797 1798 /* mode set to CCM/GCM, don't set {A_Alg, AKey_Len, MAC_Len} */ 1799 sqe3->a_key_addr = sqe3->c_key_addr; 1800 sqe3->auth_ivin.a_ivin_addr = cpu_to_le64(a_req->a_ivin_dma); 1801 sqe3->auth_mac_key |= SEC_NO_AUTH; 1802 1803 if (dir) 1804 sqe3->huk_iv_seq &= SEC_CIPHER_AUTH_V3; 1805 else 1806 sqe3->huk_iv_seq |= SEC_AUTH_CIPHER_V3; 1807 1808 sqe3->a_len_key = cpu_to_le32(aq->assoclen); 1809 sqe3->auth_src_offset = cpu_to_le16(0x0); 1810 sqe3->cipher_src_offset = cpu_to_le16((u16)aq->assoclen); 1811 sqe3->mac_addr = cpu_to_le64(a_req->out_mac_dma); 1812 } 1813 1814 static void sec_auth_bd_fill_ex(struct sec_auth_ctx *ctx, int dir, 1815 struct sec_req *req, struct sec_sqe *sec_sqe) 1816 { 1817 struct sec_aead_req *a_req = &req->aead_req; 1818 struct sec_cipher_req *c_req = &req->c_req; 1819 struct aead_request *aq = a_req->aead_req; 1820 struct crypto_aead *tfm = crypto_aead_reqtfm(aq); 1821 size_t authsize = crypto_aead_authsize(tfm); 1822 1823 sec_sqe->type2.a_key_addr = cpu_to_le64(ctx->a_key_dma); 1824 1825 sec_sqe->type2.mac_key_alg = cpu_to_le32(BYTES_TO_WORDS(authsize)); 1826 1827 sec_sqe->type2.mac_key_alg |= 1828 cpu_to_le32((u32)BYTES_TO_WORDS(ctx->a_key_len) << SEC_AKEY_OFFSET); 1829 1830 sec_sqe->type2.mac_key_alg |= 1831 cpu_to_le32((u32)(ctx->a_alg) << SEC_AEAD_ALG_OFFSET); 1832 1833 if (dir) { 1834 sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE1 << SEC_AUTH_OFFSET; 1835 sec_sqe->sds_sa_type &= SEC_CIPHER_AUTH; 1836 } else { 1837 sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE2 << SEC_AUTH_OFFSET; 1838 sec_sqe->sds_sa_type |= SEC_AUTH_CIPHER; 1839 } 1840 sec_sqe->type2.alen_ivllen = cpu_to_le32(c_req->c_len + aq->assoclen); 1841 1842 sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen); 1843 1844 sec_sqe->type2.mac_addr = cpu_to_le64(a_req->out_mac_dma); 1845 } 1846 1847 static int sec_aead_bd_fill(struct sec_ctx *ctx, struct sec_req *req) 1848 { 1849 struct sec_auth_ctx *auth_ctx = &ctx->a_ctx; 1850 struct sec_sqe *sec_sqe = &req->sec_sqe; 1851 int ret; 1852 1853 ret = sec_skcipher_bd_fill(ctx, req); 1854 if (unlikely(ret)) { 1855 dev_err(ctx->dev, "skcipher bd fill is error!\n"); 1856 return ret; 1857 } 1858 1859 if (ctx->c_ctx.c_mode == SEC_CMODE_CCM || 1860 ctx->c_ctx.c_mode == SEC_CMODE_GCM) 1861 sec_auth_bd_fill_xcm(auth_ctx, req->c_req.encrypt, req, sec_sqe); 1862 else 1863 sec_auth_bd_fill_ex(auth_ctx, req->c_req.encrypt, req, sec_sqe); 1864 1865 return 0; 1866 } 1867 1868 static void sec_auth_bd_fill_ex_v3(struct sec_auth_ctx *ctx, int dir, 1869 struct sec_req *req, struct sec_sqe3 *sqe3) 1870 { 1871 struct sec_aead_req *a_req = &req->aead_req; 1872 struct sec_cipher_req *c_req = &req->c_req; 1873 struct aead_request *aq = a_req->aead_req; 1874 struct crypto_aead *tfm = crypto_aead_reqtfm(aq); 1875 size_t authsize = crypto_aead_authsize(tfm); 1876 1877 sqe3->a_key_addr = cpu_to_le64(ctx->a_key_dma); 1878 1879 sqe3->auth_mac_key |= 1880 cpu_to_le32(BYTES_TO_WORDS(authsize) << SEC_MAC_OFFSET_V3); 1881 1882 sqe3->auth_mac_key |= 1883 cpu_to_le32((u32)BYTES_TO_WORDS(ctx->a_key_len) << SEC_AKEY_OFFSET_V3); 1884 1885 sqe3->auth_mac_key |= 1886 cpu_to_le32((u32)(ctx->a_alg) << SEC_AUTH_ALG_OFFSET_V3); 1887 1888 if (dir) { 1889 sqe3->auth_mac_key |= cpu_to_le32((u32)SEC_AUTH_TYPE1); 1890 sqe3->huk_iv_seq &= SEC_CIPHER_AUTH_V3; 1891 } else { 1892 sqe3->auth_mac_key |= cpu_to_le32((u32)SEC_AUTH_TYPE2); 1893 sqe3->huk_iv_seq |= SEC_AUTH_CIPHER_V3; 1894 } 1895 sqe3->a_len_key = cpu_to_le32(c_req->c_len + aq->assoclen); 1896 1897 sqe3->cipher_src_offset = cpu_to_le16((u16)aq->assoclen); 1898 1899 sqe3->mac_addr = cpu_to_le64(a_req->out_mac_dma); 1900 } 1901 1902 static int sec_aead_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req) 1903 { 1904 struct sec_auth_ctx *auth_ctx = &ctx->a_ctx; 1905 struct sec_sqe3 *sec_sqe3 = &req->sec_sqe3; 1906 int ret; 1907 1908 ret = sec_skcipher_bd_fill_v3(ctx, req); 1909 if (unlikely(ret)) { 1910 dev_err(ctx->dev, "skcipher bd3 fill is error!\n"); 1911 return ret; 1912 } 1913 1914 if (ctx->c_ctx.c_mode == SEC_CMODE_CCM || 1915 ctx->c_ctx.c_mode == SEC_CMODE_GCM) 1916 sec_auth_bd_fill_xcm_v3(auth_ctx, req->c_req.encrypt, 1917 req, sec_sqe3); 1918 else 1919 sec_auth_bd_fill_ex_v3(auth_ctx, req->c_req.encrypt, 1920 req, sec_sqe3); 1921 1922 return 0; 1923 } 1924 1925 static void sec_aead_callback(struct sec_ctx *c, struct sec_req *req, int err) 1926 { 1927 struct aead_request *a_req = req->aead_req.aead_req; 1928 struct crypto_aead *tfm = crypto_aead_reqtfm(a_req); 1929 size_t authsize = crypto_aead_authsize(tfm); 1930 struct sec_qp_ctx *qp_ctx = req->qp_ctx; 1931 size_t sz; 1932 1933 if (!err && req->c_req.encrypt) { 1934 if (c->c_ctx.c_mode == SEC_CMODE_CBC) 1935 sec_update_iv(req, SEC_AEAD); 1936 1937 sz = sg_pcopy_from_buffer(a_req->dst, sg_nents(a_req->dst), req->aead_req.out_mac, 1938 authsize, a_req->cryptlen + a_req->assoclen); 1939 if (unlikely(sz != authsize)) { 1940 dev_err(c->dev, "copy out mac err!\n"); 1941 err = -EINVAL; 1942 } 1943 } 1944 1945 if (req->req_id >= 0) 1946 sec_free_req_id(req); 1947 1948 crypto_request_complete(req->base, err); 1949 sec_alg_send_backlog(c, qp_ctx); 1950 } 1951 1952 static void sec_request_uninit(struct sec_req *req) 1953 { 1954 if (req->req_id >= 0) 1955 sec_free_req_id(req); 1956 } 1957 1958 static int sec_request_init(struct sec_ctx *ctx, struct sec_req *req) 1959 { 1960 struct sec_qp_ctx *qp_ctx; 1961 int i = 0; 1962 1963 do { 1964 qp_ctx = &ctx->qp_ctx[i]; 1965 req->req_id = sec_alloc_req_id(req, qp_ctx); 1966 } while (req->req_id < 0 && ++i < ctx->sec->ctx_q_num); 1967 1968 req->qp_ctx = qp_ctx; 1969 1970 return 0; 1971 } 1972 1973 static int sec_process(struct sec_ctx *ctx, struct sec_req *req) 1974 { 1975 int ret; 1976 1977 ret = sec_request_init(ctx, req); 1978 if (unlikely(ret)) 1979 return ret; 1980 1981 ret = sec_request_transfer(ctx, req); 1982 if (unlikely(ret)) 1983 goto err_uninit_req; 1984 1985 /* Output IV as decrypto */ 1986 if (!req->c_req.encrypt && (ctx->c_ctx.c_mode == SEC_CMODE_CBC || 1987 ctx->c_ctx.c_mode == SEC_CMODE_CTR)) 1988 sec_update_iv(req, ctx->alg_type); 1989 1990 ret = ctx->req_op->bd_send(ctx, req); 1991 if (unlikely((ret != -EBUSY && ret != -EINPROGRESS))) { 1992 dev_err_ratelimited(ctx->dev, "send sec request failed!\n"); 1993 goto err_send_req; 1994 } 1995 1996 return ret; 1997 1998 err_send_req: 1999 /* As failing, restore the IV from user */ 2000 if (ctx->c_ctx.c_mode == SEC_CMODE_CBC && !req->c_req.encrypt) { 2001 if (ctx->alg_type == SEC_SKCIPHER) 2002 memcpy(req->c_req.sk_req->iv, req->c_req.c_ivin, 2003 ctx->c_ctx.ivsize); 2004 else 2005 memcpy(req->aead_req.aead_req->iv, req->c_req.c_ivin, 2006 ctx->c_ctx.ivsize); 2007 } 2008 2009 sec_request_untransfer(ctx, req); 2010 2011 err_uninit_req: 2012 sec_request_uninit(req); 2013 if (ctx->alg_type == SEC_AEAD) 2014 ret = sec_aead_soft_crypto(ctx, req->aead_req.aead_req, 2015 req->c_req.encrypt); 2016 else 2017 ret = sec_skcipher_soft_crypto(ctx, req->c_req.sk_req, 2018 req->c_req.encrypt); 2019 return ret; 2020 } 2021 2022 static const struct sec_req_op sec_skcipher_req_ops = { 2023 .buf_map = sec_skcipher_sgl_map, 2024 .buf_unmap = sec_skcipher_sgl_unmap, 2025 .do_transfer = sec_skcipher_copy_iv, 2026 .bd_fill = sec_skcipher_bd_fill, 2027 .bd_send = sec_bd_send, 2028 .callback = sec_skcipher_callback, 2029 .process = sec_process, 2030 }; 2031 2032 static const struct sec_req_op sec_aead_req_ops = { 2033 .buf_map = sec_aead_sgl_map, 2034 .buf_unmap = sec_aead_sgl_unmap, 2035 .do_transfer = sec_aead_set_iv, 2036 .bd_fill = sec_aead_bd_fill, 2037 .bd_send = sec_bd_send, 2038 .callback = sec_aead_callback, 2039 .process = sec_process, 2040 }; 2041 2042 static const struct sec_req_op sec_skcipher_req_ops_v3 = { 2043 .buf_map = sec_skcipher_sgl_map, 2044 .buf_unmap = sec_skcipher_sgl_unmap, 2045 .do_transfer = sec_skcipher_copy_iv, 2046 .bd_fill = sec_skcipher_bd_fill_v3, 2047 .bd_send = sec_bd_send, 2048 .callback = sec_skcipher_callback, 2049 .process = sec_process, 2050 }; 2051 2052 static const struct sec_req_op sec_aead_req_ops_v3 = { 2053 .buf_map = sec_aead_sgl_map, 2054 .buf_unmap = sec_aead_sgl_unmap, 2055 .do_transfer = sec_aead_set_iv, 2056 .bd_fill = sec_aead_bd_fill_v3, 2057 .bd_send = sec_bd_send, 2058 .callback = sec_aead_callback, 2059 .process = sec_process, 2060 }; 2061 2062 static int sec_skcipher_ctx_init(struct crypto_skcipher *tfm) 2063 { 2064 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 2065 int ret; 2066 2067 ret = sec_skcipher_init(tfm); 2068 if (ret) 2069 return ret; 2070 2071 if (!ctx->qps) 2072 return 0; 2073 2074 if (ctx->type_supported == SEC_BD_TYPE3) 2075 ctx->req_op = &sec_skcipher_req_ops_v3; 2076 else 2077 ctx->req_op = &sec_skcipher_req_ops; 2078 2079 return 0; 2080 } 2081 2082 static void sec_skcipher_ctx_exit(struct crypto_skcipher *tfm) 2083 { 2084 sec_skcipher_uninit(tfm); 2085 } 2086 2087 static int sec_aead_init(struct crypto_aead *tfm) 2088 { 2089 struct sec_ctx *ctx = crypto_aead_ctx(tfm); 2090 int ret; 2091 2092 crypto_aead_set_reqsize_dma(tfm, sizeof(struct sec_req)); 2093 ctx->alg_type = SEC_AEAD; 2094 ctx->c_ctx.ivsize = crypto_aead_ivsize(tfm); 2095 if (ctx->c_ctx.ivsize < SEC_AIV_SIZE || 2096 ctx->c_ctx.ivsize > SEC_IV_SIZE) { 2097 pr_err("get error aead iv size!\n"); 2098 return -EINVAL; 2099 } 2100 2101 ret = sec_ctx_base_init(ctx); 2102 if (ret) 2103 return ret; 2104 2105 if (ctx->type_supported == SEC_BD_TYPE3) 2106 ctx->req_op = &sec_aead_req_ops_v3; 2107 else 2108 ctx->req_op = &sec_aead_req_ops; 2109 2110 ret = sec_auth_init(ctx); 2111 if (ret) 2112 goto err_auth_init; 2113 2114 ret = sec_cipher_init(ctx); 2115 if (ret) 2116 goto err_cipher_init; 2117 2118 return ret; 2119 2120 err_cipher_init: 2121 sec_auth_uninit(ctx); 2122 err_auth_init: 2123 sec_ctx_base_uninit(ctx); 2124 return ret; 2125 } 2126 2127 static void sec_aead_exit(struct crypto_aead *tfm) 2128 { 2129 struct sec_ctx *ctx = crypto_aead_ctx(tfm); 2130 2131 sec_cipher_uninit(ctx); 2132 sec_auth_uninit(ctx); 2133 sec_ctx_base_uninit(ctx); 2134 } 2135 2136 static int sec_aead_ctx_init(struct crypto_aead *tfm, const char *hash_name) 2137 { 2138 struct aead_alg *alg = crypto_aead_alg(tfm); 2139 struct sec_ctx *ctx = crypto_aead_ctx(tfm); 2140 struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 2141 const char *aead_name = alg->base.cra_name; 2142 int ret; 2143 2144 ret = sec_aead_init(tfm); 2145 if (ret && ret != -ENODEV) { 2146 pr_err("hisi_sec2: aead init error!\n"); 2147 return ret; 2148 } 2149 2150 a_ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0); 2151 if (IS_ERR(a_ctx->hash_tfm)) { 2152 dev_err(ctx->dev, "aead alloc shash error!\n"); 2153 sec_aead_exit(tfm); 2154 return PTR_ERR(a_ctx->hash_tfm); 2155 } 2156 2157 a_ctx->fallback_aead_tfm = crypto_alloc_aead(aead_name, 0, 2158 CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ASYNC); 2159 if (IS_ERR(a_ctx->fallback_aead_tfm)) { 2160 dev_err(ctx->dev, "aead driver alloc fallback tfm error!\n"); 2161 crypto_free_shash(ctx->a_ctx.hash_tfm); 2162 sec_aead_exit(tfm); 2163 return PTR_ERR(a_ctx->fallback_aead_tfm); 2164 } 2165 2166 return 0; 2167 } 2168 2169 static void sec_aead_ctx_exit(struct crypto_aead *tfm) 2170 { 2171 struct sec_ctx *ctx = crypto_aead_ctx(tfm); 2172 2173 crypto_free_aead(ctx->a_ctx.fallback_aead_tfm); 2174 crypto_free_shash(ctx->a_ctx.hash_tfm); 2175 sec_aead_exit(tfm); 2176 } 2177 2178 static int sec_aead_xcm_ctx_init(struct crypto_aead *tfm) 2179 { 2180 struct aead_alg *alg = crypto_aead_alg(tfm); 2181 struct sec_ctx *ctx = crypto_aead_ctx(tfm); 2182 struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 2183 const char *aead_name = alg->base.cra_name; 2184 int ret; 2185 2186 ret = sec_aead_init(tfm); 2187 if (ret && ret != -ENODEV) { 2188 dev_err(ctx->dev, "hisi_sec2: aead xcm init error!\n"); 2189 return ret; 2190 } 2191 2192 a_ctx->fallback_aead_tfm = crypto_alloc_aead(aead_name, 0, 2193 CRYPTO_ALG_NEED_FALLBACK | 2194 CRYPTO_ALG_ASYNC); 2195 if (IS_ERR(a_ctx->fallback_aead_tfm)) { 2196 dev_err(ctx->dev, "aead driver alloc fallback tfm error!\n"); 2197 sec_aead_exit(tfm); 2198 return PTR_ERR(a_ctx->fallback_aead_tfm); 2199 } 2200 2201 return 0; 2202 } 2203 2204 static void sec_aead_xcm_ctx_exit(struct crypto_aead *tfm) 2205 { 2206 struct sec_ctx *ctx = crypto_aead_ctx(tfm); 2207 2208 crypto_free_aead(ctx->a_ctx.fallback_aead_tfm); 2209 sec_aead_exit(tfm); 2210 } 2211 2212 static int sec_aead_sha1_ctx_init(struct crypto_aead *tfm) 2213 { 2214 return sec_aead_ctx_init(tfm, "sha1"); 2215 } 2216 2217 static int sec_aead_sha256_ctx_init(struct crypto_aead *tfm) 2218 { 2219 return sec_aead_ctx_init(tfm, "sha256"); 2220 } 2221 2222 static int sec_aead_sha512_ctx_init(struct crypto_aead *tfm) 2223 { 2224 return sec_aead_ctx_init(tfm, "sha512"); 2225 } 2226 2227 static int sec_skcipher_cryptlen_check(struct sec_ctx *ctx, struct sec_req *sreq) 2228 { 2229 u32 cryptlen = sreq->c_req.sk_req->cryptlen; 2230 struct device *dev = ctx->dev; 2231 u8 c_mode = ctx->c_ctx.c_mode; 2232 int ret = 0; 2233 2234 switch (c_mode) { 2235 case SEC_CMODE_XTS: 2236 if (unlikely(cryptlen < AES_BLOCK_SIZE)) { 2237 dev_err(dev, "skcipher XTS mode input length error!\n"); 2238 ret = -EINVAL; 2239 } 2240 break; 2241 case SEC_CMODE_ECB: 2242 case SEC_CMODE_CBC: 2243 if (unlikely(cryptlen & (AES_BLOCK_SIZE - 1))) { 2244 dev_err(dev, "skcipher AES input length error!\n"); 2245 ret = -EINVAL; 2246 } 2247 break; 2248 case SEC_CMODE_CTR: 2249 break; 2250 default: 2251 ret = -EINVAL; 2252 } 2253 2254 return ret; 2255 } 2256 2257 static int sec_skcipher_param_check(struct sec_ctx *ctx, 2258 struct sec_req *sreq, bool *need_fallback) 2259 { 2260 struct skcipher_request *sk_req = sreq->c_req.sk_req; 2261 struct device *dev = ctx->dev; 2262 u8 c_alg = ctx->c_ctx.c_alg; 2263 2264 if (unlikely(!sk_req->src || !sk_req->dst)) { 2265 dev_err(dev, "skcipher input param error!\n"); 2266 return -EINVAL; 2267 } 2268 2269 if (sk_req->cryptlen > MAX_INPUT_DATA_LEN) 2270 *need_fallback = true; 2271 2272 sreq->c_req.c_len = sk_req->cryptlen; 2273 2274 if (ctx->pbuf_supported && sk_req->cryptlen <= SEC_PBUF_SZ) 2275 sreq->use_pbuf = true; 2276 else 2277 sreq->use_pbuf = false; 2278 2279 if (c_alg == SEC_CALG_3DES) { 2280 if (unlikely(sk_req->cryptlen & (DES3_EDE_BLOCK_SIZE - 1))) { 2281 dev_err(dev, "skcipher 3des input length error!\n"); 2282 return -EINVAL; 2283 } 2284 return 0; 2285 } else if (c_alg == SEC_CALG_AES || c_alg == SEC_CALG_SM4) { 2286 return sec_skcipher_cryptlen_check(ctx, sreq); 2287 } 2288 2289 dev_err(dev, "skcipher algorithm error!\n"); 2290 2291 return -EINVAL; 2292 } 2293 2294 static int sec_skcipher_soft_crypto(struct sec_ctx *ctx, 2295 struct skcipher_request *sreq, bool encrypt) 2296 { 2297 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 2298 SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, c_ctx->fbtfm); 2299 struct device *dev = ctx->dev; 2300 int ret; 2301 2302 if (!c_ctx->fbtfm) { 2303 dev_err_ratelimited(dev, "the soft tfm isn't supported in the current system.\n"); 2304 return -EINVAL; 2305 } 2306 2307 skcipher_request_set_sync_tfm(subreq, c_ctx->fbtfm); 2308 2309 /* software need sync mode to do crypto */ 2310 skcipher_request_set_callback(subreq, sreq->base.flags, 2311 NULL, NULL); 2312 skcipher_request_set_crypt(subreq, sreq->src, sreq->dst, 2313 sreq->cryptlen, sreq->iv); 2314 if (encrypt) 2315 ret = crypto_skcipher_encrypt(subreq); 2316 else 2317 ret = crypto_skcipher_decrypt(subreq); 2318 2319 skcipher_request_zero(subreq); 2320 2321 return ret; 2322 } 2323 2324 static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt) 2325 { 2326 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(sk_req); 2327 struct sec_req *req = skcipher_request_ctx_dma(sk_req); 2328 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 2329 bool need_fallback = false; 2330 int ret; 2331 2332 if (!ctx->qps) 2333 goto soft_crypto; 2334 2335 if (!sk_req->cryptlen) { 2336 if (ctx->c_ctx.c_mode == SEC_CMODE_XTS) 2337 return -EINVAL; 2338 return 0; 2339 } 2340 2341 req->flag = sk_req->base.flags; 2342 req->c_req.sk_req = sk_req; 2343 req->c_req.encrypt = encrypt; 2344 req->ctx = ctx; 2345 req->base = &sk_req->base; 2346 2347 ret = sec_skcipher_param_check(ctx, req, &need_fallback); 2348 if (unlikely(ret)) 2349 return -EINVAL; 2350 2351 if (unlikely(ctx->c_ctx.fallback || need_fallback)) 2352 goto soft_crypto; 2353 2354 return ctx->req_op->process(ctx, req); 2355 2356 soft_crypto: 2357 return sec_skcipher_soft_crypto(ctx, sk_req, encrypt); 2358 } 2359 2360 static int sec_skcipher_encrypt(struct skcipher_request *sk_req) 2361 { 2362 return sec_skcipher_crypto(sk_req, true); 2363 } 2364 2365 static int sec_skcipher_decrypt(struct skcipher_request *sk_req) 2366 { 2367 return sec_skcipher_crypto(sk_req, false); 2368 } 2369 2370 #define SEC_SKCIPHER_ALG(sec_cra_name, sec_set_key, \ 2371 sec_min_key_size, sec_max_key_size, blk_size, iv_size)\ 2372 {\ 2373 .base = {\ 2374 .cra_name = sec_cra_name,\ 2375 .cra_driver_name = "hisi_sec_"sec_cra_name,\ 2376 .cra_priority = SEC_PRIORITY,\ 2377 .cra_flags = CRYPTO_ALG_ASYNC |\ 2378 CRYPTO_ALG_NEED_FALLBACK,\ 2379 .cra_blocksize = blk_size,\ 2380 .cra_ctxsize = sizeof(struct sec_ctx),\ 2381 .cra_module = THIS_MODULE,\ 2382 },\ 2383 .init = sec_skcipher_ctx_init,\ 2384 .exit = sec_skcipher_ctx_exit,\ 2385 .setkey = sec_set_key,\ 2386 .decrypt = sec_skcipher_decrypt,\ 2387 .encrypt = sec_skcipher_encrypt,\ 2388 .min_keysize = sec_min_key_size,\ 2389 .max_keysize = sec_max_key_size,\ 2390 .ivsize = iv_size,\ 2391 } 2392 2393 static struct sec_skcipher sec_skciphers[] = { 2394 { 2395 .alg_msk = BIT(0), 2396 .alg = SEC_SKCIPHER_ALG("ecb(aes)", sec_setkey_aes_ecb, AES_MIN_KEY_SIZE, 2397 AES_MAX_KEY_SIZE, AES_BLOCK_SIZE, 0), 2398 }, 2399 { 2400 .alg_msk = BIT(1), 2401 .alg = SEC_SKCIPHER_ALG("cbc(aes)", sec_setkey_aes_cbc, AES_MIN_KEY_SIZE, 2402 AES_MAX_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE), 2403 }, 2404 { 2405 .alg_msk = BIT(2), 2406 .alg = SEC_SKCIPHER_ALG("ctr(aes)", sec_setkey_aes_ctr, AES_MIN_KEY_SIZE, 2407 AES_MAX_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), 2408 }, 2409 { 2410 .alg_msk = BIT(3), 2411 .alg = SEC_SKCIPHER_ALG("xts(aes)", sec_setkey_aes_xts, SEC_XTS_MIN_KEY_SIZE, 2412 SEC_XTS_MAX_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE), 2413 }, 2414 { 2415 .alg_msk = BIT(12), 2416 .alg = SEC_SKCIPHER_ALG("cbc(sm4)", sec_setkey_sm4_cbc, AES_MIN_KEY_SIZE, 2417 AES_MIN_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE), 2418 }, 2419 { 2420 .alg_msk = BIT(13), 2421 .alg = SEC_SKCIPHER_ALG("ctr(sm4)", sec_setkey_sm4_ctr, AES_MIN_KEY_SIZE, 2422 AES_MIN_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), 2423 }, 2424 { 2425 .alg_msk = BIT(14), 2426 .alg = SEC_SKCIPHER_ALG("xts(sm4)", sec_setkey_sm4_xts, SEC_XTS_MIN_KEY_SIZE, 2427 SEC_XTS_MIN_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE), 2428 }, 2429 { 2430 .alg_msk = BIT(23), 2431 .alg = SEC_SKCIPHER_ALG("ecb(des3_ede)", sec_setkey_3des_ecb, SEC_DES3_3KEY_SIZE, 2432 SEC_DES3_3KEY_SIZE, DES3_EDE_BLOCK_SIZE, 0), 2433 }, 2434 { 2435 .alg_msk = BIT(24), 2436 .alg = SEC_SKCIPHER_ALG("cbc(des3_ede)", sec_setkey_3des_cbc, SEC_DES3_3KEY_SIZE, 2437 SEC_DES3_3KEY_SIZE, DES3_EDE_BLOCK_SIZE, 2438 DES3_EDE_BLOCK_SIZE), 2439 }, 2440 }; 2441 2442 static int aead_iv_demension_check(struct aead_request *aead_req) 2443 { 2444 u8 cl; 2445 2446 cl = aead_req->iv[0] + 1; 2447 if (cl < IV_CL_MIN || cl > IV_CL_MAX) 2448 return -EINVAL; 2449 2450 if (cl < IV_CL_MID && aead_req->cryptlen >> (BYTE_BITS * cl)) 2451 return -EOVERFLOW; 2452 2453 return 0; 2454 } 2455 2456 static int sec_aead_spec_check(struct sec_ctx *ctx, struct sec_req *sreq) 2457 { 2458 struct aead_request *req = sreq->aead_req.aead_req; 2459 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2460 size_t sz = crypto_aead_authsize(tfm); 2461 u8 c_mode = ctx->c_ctx.c_mode; 2462 int ret; 2463 2464 if (unlikely(ctx->sec->qm.ver == QM_HW_V2 && !sreq->c_req.c_len)) 2465 return -EINVAL; 2466 2467 if (unlikely(req->cryptlen + req->assoclen > MAX_INPUT_DATA_LEN || 2468 req->assoclen > SEC_MAX_AAD_LEN)) 2469 return -EINVAL; 2470 2471 if (c_mode == SEC_CMODE_CCM) { 2472 if (unlikely(req->assoclen > SEC_MAX_CCM_AAD_LEN)) 2473 return -EINVAL; 2474 2475 ret = aead_iv_demension_check(req); 2476 if (unlikely(ret)) 2477 return -EINVAL; 2478 } else if (c_mode == SEC_CMODE_CBC) { 2479 if (unlikely(sz & WORD_MASK)) 2480 return -EINVAL; 2481 if (unlikely(ctx->a_ctx.a_key_len & WORD_MASK)) 2482 return -EINVAL; 2483 } else if (c_mode == SEC_CMODE_GCM) { 2484 if (unlikely(sz < SEC_GCM_MIN_AUTH_SZ)) 2485 return -EINVAL; 2486 } 2487 2488 return 0; 2489 } 2490 2491 static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq, bool *need_fallback) 2492 { 2493 struct aead_request *req = sreq->aead_req.aead_req; 2494 struct device *dev = ctx->dev; 2495 u8 c_alg = ctx->c_ctx.c_alg; 2496 2497 if (unlikely(!req->src || !req->dst)) { 2498 dev_err(dev, "aead input param error!\n"); 2499 return -EINVAL; 2500 } 2501 2502 if (unlikely(ctx->c_ctx.c_mode == SEC_CMODE_CBC && 2503 sreq->c_req.c_len & (AES_BLOCK_SIZE - 1))) { 2504 dev_err(dev, "aead cbc mode input data length error!\n"); 2505 return -EINVAL; 2506 } 2507 2508 /* Support AES or SM4 */ 2509 if (unlikely(c_alg != SEC_CALG_AES && c_alg != SEC_CALG_SM4)) { 2510 dev_err(dev, "aead crypto alg error!\n"); 2511 return -EINVAL; 2512 } 2513 2514 if (unlikely(sec_aead_spec_check(ctx, sreq))) { 2515 *need_fallback = true; 2516 return -EINVAL; 2517 } 2518 2519 if (ctx->pbuf_supported && (req->cryptlen + req->assoclen) <= 2520 SEC_PBUF_SZ) 2521 sreq->use_pbuf = true; 2522 else 2523 sreq->use_pbuf = false; 2524 2525 return 0; 2526 } 2527 2528 static int sec_aead_soft_crypto(struct sec_ctx *ctx, 2529 struct aead_request *aead_req, 2530 bool encrypt) 2531 { 2532 struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 2533 struct aead_request *subreq; 2534 int ret; 2535 2536 subreq = aead_request_alloc(a_ctx->fallback_aead_tfm, GFP_KERNEL); 2537 if (!subreq) 2538 return -ENOMEM; 2539 2540 aead_request_set_tfm(subreq, a_ctx->fallback_aead_tfm); 2541 aead_request_set_callback(subreq, aead_req->base.flags, 2542 aead_req->base.complete, aead_req->base.data); 2543 aead_request_set_crypt(subreq, aead_req->src, aead_req->dst, 2544 aead_req->cryptlen, aead_req->iv); 2545 aead_request_set_ad(subreq, aead_req->assoclen); 2546 2547 if (encrypt) 2548 ret = crypto_aead_encrypt(subreq); 2549 else 2550 ret = crypto_aead_decrypt(subreq); 2551 aead_request_free(subreq); 2552 2553 return ret; 2554 } 2555 2556 static int sec_aead_crypto(struct aead_request *a_req, bool encrypt) 2557 { 2558 struct crypto_aead *tfm = crypto_aead_reqtfm(a_req); 2559 struct sec_req *req = aead_request_ctx_dma(a_req); 2560 struct sec_ctx *ctx = crypto_aead_ctx(tfm); 2561 size_t sz = crypto_aead_authsize(tfm); 2562 bool need_fallback = false; 2563 int ret; 2564 2565 if (!ctx->qps) 2566 goto soft_crypto; 2567 2568 req->flag = a_req->base.flags; 2569 req->aead_req.aead_req = a_req; 2570 req->c_req.encrypt = encrypt; 2571 req->ctx = ctx; 2572 req->base = &a_req->base; 2573 req->c_req.c_len = a_req->cryptlen - (req->c_req.encrypt ? 0 : sz); 2574 2575 ret = sec_aead_param_check(ctx, req, &need_fallback); 2576 if (unlikely(ret)) { 2577 if (need_fallback) 2578 goto soft_crypto; 2579 return -EINVAL; 2580 } 2581 2582 return ctx->req_op->process(ctx, req); 2583 2584 soft_crypto: 2585 return sec_aead_soft_crypto(ctx, a_req, encrypt); 2586 } 2587 2588 static int sec_aead_encrypt(struct aead_request *a_req) 2589 { 2590 return sec_aead_crypto(a_req, true); 2591 } 2592 2593 static int sec_aead_decrypt(struct aead_request *a_req) 2594 { 2595 return sec_aead_crypto(a_req, false); 2596 } 2597 2598 #define SEC_AEAD_ALG(sec_cra_name, sec_set_key, ctx_init,\ 2599 ctx_exit, blk_size, iv_size, max_authsize)\ 2600 {\ 2601 .base = {\ 2602 .cra_name = sec_cra_name,\ 2603 .cra_driver_name = "hisi_sec_"sec_cra_name,\ 2604 .cra_priority = SEC_PRIORITY,\ 2605 .cra_flags = CRYPTO_ALG_ASYNC |\ 2606 CRYPTO_ALG_NEED_FALLBACK,\ 2607 .cra_blocksize = blk_size,\ 2608 .cra_ctxsize = sizeof(struct sec_ctx),\ 2609 .cra_module = THIS_MODULE,\ 2610 },\ 2611 .init = ctx_init,\ 2612 .exit = ctx_exit,\ 2613 .setkey = sec_set_key,\ 2614 .setauthsize = sec_aead_setauthsize,\ 2615 .decrypt = sec_aead_decrypt,\ 2616 .encrypt = sec_aead_encrypt,\ 2617 .ivsize = iv_size,\ 2618 .maxauthsize = max_authsize,\ 2619 } 2620 2621 static struct sec_aead sec_aeads[] = { 2622 { 2623 .alg_msk = BIT(6), 2624 .alg = SEC_AEAD_ALG("ccm(aes)", sec_setkey_aes_ccm, sec_aead_xcm_ctx_init, 2625 sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE, 2626 AES_BLOCK_SIZE), 2627 }, 2628 { 2629 .alg_msk = BIT(7), 2630 .alg = SEC_AEAD_ALG("gcm(aes)", sec_setkey_aes_gcm, sec_aead_xcm_ctx_init, 2631 sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, SEC_AIV_SIZE, 2632 AES_BLOCK_SIZE), 2633 }, 2634 { 2635 .alg_msk = BIT(17), 2636 .alg = SEC_AEAD_ALG("ccm(sm4)", sec_setkey_sm4_ccm, sec_aead_xcm_ctx_init, 2637 sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE, 2638 AES_BLOCK_SIZE), 2639 }, 2640 { 2641 .alg_msk = BIT(18), 2642 .alg = SEC_AEAD_ALG("gcm(sm4)", sec_setkey_sm4_gcm, sec_aead_xcm_ctx_init, 2643 sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, SEC_AIV_SIZE, 2644 AES_BLOCK_SIZE), 2645 }, 2646 { 2647 .alg_msk = BIT(43), 2648 .alg = SEC_AEAD_ALG("authenc(hmac(sha1),cbc(aes))", sec_setkey_aes_cbc_sha1, 2649 sec_aead_sha1_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE, 2650 AES_BLOCK_SIZE, SHA1_DIGEST_SIZE), 2651 }, 2652 { 2653 .alg_msk = BIT(44), 2654 .alg = SEC_AEAD_ALG("authenc(hmac(sha256),cbc(aes))", sec_setkey_aes_cbc_sha256, 2655 sec_aead_sha256_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE, 2656 AES_BLOCK_SIZE, SHA256_DIGEST_SIZE), 2657 }, 2658 { 2659 .alg_msk = BIT(45), 2660 .alg = SEC_AEAD_ALG("authenc(hmac(sha512),cbc(aes))", sec_setkey_aes_cbc_sha512, 2661 sec_aead_sha512_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE, 2662 AES_BLOCK_SIZE, SHA512_DIGEST_SIZE), 2663 }, 2664 }; 2665 2666 static void sec_unregister_skcipher(u64 alg_mask, int end) 2667 { 2668 int i; 2669 2670 for (i = 0; i < end; i++) 2671 if (sec_skciphers[i].alg_msk & alg_mask) 2672 crypto_unregister_skcipher(&sec_skciphers[i].alg); 2673 } 2674 2675 static int sec_register_skcipher(u64 alg_mask) 2676 { 2677 int i, ret, count; 2678 2679 count = ARRAY_SIZE(sec_skciphers); 2680 2681 for (i = 0; i < count; i++) { 2682 if (!(sec_skciphers[i].alg_msk & alg_mask)) 2683 continue; 2684 2685 ret = crypto_register_skcipher(&sec_skciphers[i].alg); 2686 if (ret) 2687 goto err; 2688 } 2689 2690 return 0; 2691 2692 err: 2693 sec_unregister_skcipher(alg_mask, i); 2694 2695 return ret; 2696 } 2697 2698 static void sec_unregister_aead(u64 alg_mask, int end) 2699 { 2700 int i; 2701 2702 for (i = 0; i < end; i++) 2703 if (sec_aeads[i].alg_msk & alg_mask) 2704 crypto_unregister_aead(&sec_aeads[i].alg); 2705 } 2706 2707 static int sec_register_aead(u64 alg_mask) 2708 { 2709 int i, ret, count; 2710 2711 count = ARRAY_SIZE(sec_aeads); 2712 2713 for (i = 0; i < count; i++) { 2714 if (!(sec_aeads[i].alg_msk & alg_mask)) 2715 continue; 2716 2717 ret = crypto_register_aead(&sec_aeads[i].alg); 2718 if (ret) 2719 goto err; 2720 } 2721 2722 return 0; 2723 2724 err: 2725 sec_unregister_aead(alg_mask, i); 2726 2727 return ret; 2728 } 2729 2730 int sec_register_to_crypto(struct hisi_qm *qm) 2731 { 2732 u64 alg_mask; 2733 int ret = 0; 2734 2735 alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH_TB, 2736 SEC_DRV_ALG_BITMAP_LOW_TB); 2737 2738 mutex_lock(&sec_algs_lock); 2739 if (sec_available_devs) { 2740 sec_available_devs++; 2741 goto unlock; 2742 } 2743 2744 ret = sec_register_skcipher(alg_mask); 2745 if (ret) 2746 goto unlock; 2747 2748 ret = sec_register_aead(alg_mask); 2749 if (ret) 2750 goto unreg_skcipher; 2751 2752 sec_available_devs++; 2753 mutex_unlock(&sec_algs_lock); 2754 2755 return 0; 2756 2757 unreg_skcipher: 2758 sec_unregister_skcipher(alg_mask, ARRAY_SIZE(sec_skciphers)); 2759 unlock: 2760 mutex_unlock(&sec_algs_lock); 2761 return ret; 2762 } 2763 2764 void sec_unregister_from_crypto(struct hisi_qm *qm) 2765 { 2766 u64 alg_mask; 2767 2768 alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH_TB, 2769 SEC_DRV_ALG_BITMAP_LOW_TB); 2770 2771 mutex_lock(&sec_algs_lock); 2772 if (--sec_available_devs) 2773 goto unlock; 2774 2775 sec_unregister_aead(alg_mask, ARRAY_SIZE(sec_aeads)); 2776 sec_unregister_skcipher(alg_mask, ARRAY_SIZE(sec_skciphers)); 2777 2778 unlock: 2779 mutex_unlock(&sec_algs_lock); 2780 } 2781