1 // SPDX-License-Identifier: GPL-2.0-only 2 3 /* 4 * Copyright (C) 2021, Linaro Limited. All rights reserved. 5 */ 6 #include <linux/dma-mapping.h> 7 #include <linux/interrupt.h> 8 #include <linux/string.h> 9 #include <crypto/gcm.h> 10 #include <crypto/authenc.h> 11 #include <crypto/internal/aead.h> 12 #include <crypto/sha2.h> 13 #include <crypto/scatterwalk.h> 14 #include "aead.h" 15 16 #define CCM_NONCE_ADATA_SHIFT 6 17 #define CCM_NONCE_AUTHSIZE_SHIFT 3 18 #define MAX_CCM_ADATA_HEADER_LEN 6 19 20 static LIST_HEAD(aead_algs); 21 22 static void qce_aead_done(void *data) 23 { 24 struct crypto_async_request *async_req = data; 25 struct aead_request *req = aead_request_cast(async_req); 26 struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req); 27 struct qce_aead_ctx *ctx = crypto_tfm_ctx(async_req->tfm); 28 struct qce_alg_template *tmpl = to_aead_tmpl(crypto_aead_reqtfm(req)); 29 struct qce_device *qce = tmpl->qce; 30 struct qce_result_dump *result_buf = qce->dma.result_buf; 31 enum dma_data_direction dir_src, dir_dst; 32 bool diff_dst; 33 int error; 34 u32 status; 35 unsigned int totallen; 36 unsigned char tag[SHA256_DIGEST_SIZE] = {0}; 37 38 diff_dst = (req->src != req->dst) ? true : false; 39 dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; 40 dir_dst = diff_dst ? DMA_FROM_DEVICE : DMA_BIDIRECTIONAL; 41 42 error = qce_dma_terminate_all(&qce->dma); 43 if (error) 44 dev_dbg(qce->dev, "aead dma termination error (%d)\n", 45 error); 46 if (diff_dst) 47 dma_unmap_sg(qce->dev, rctx->src_sg, rctx->src_nents, dir_src); 48 49 dma_unmap_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst); 50 51 if (IS_CCM(rctx->flags)) { 52 if (req->assoclen) { 53 sg_free_table(&rctx->src_tbl); 54 if (diff_dst) 55 sg_free_table(&rctx->dst_tbl); 56 } else { 57 if (!(IS_DECRYPT(rctx->flags) && !diff_dst)) 58 sg_free_table(&rctx->dst_tbl); 59 } 60 } else { 61 sg_free_table(&rctx->dst_tbl); 62 } 63 64 error = qce_check_status(qce, &status); 65 if (error < 0 && (error != -EBADMSG)) 66 dev_err(qce->dev, "aead operation error (%x)\n", status); 67 68 if (IS_ENCRYPT(rctx->flags)) { 69 totallen = req->cryptlen + req->assoclen; 70 if (IS_CCM(rctx->flags)) 71 scatterwalk_map_and_copy(rctx->ccmresult_buf, req->dst, 72 totallen, ctx->authsize, 1); 73 else 74 scatterwalk_map_and_copy(result_buf->auth_iv, req->dst, 75 totallen, ctx->authsize, 1); 76 77 } else if (!IS_CCM(rctx->flags)) { 78 totallen = req->cryptlen + req->assoclen - ctx->authsize; 79 scatterwalk_map_and_copy(tag, req->src, totallen, ctx->authsize, 0); 80 if (memcmp(result_buf->auth_iv, tag, ctx->authsize)) { 81 pr_err("Bad message error\n"); 82 error = -EBADMSG; 83 } 84 } 85 86 qce->async_req_done(qce, error); 87 } 88 89 static struct scatterlist * 90 qce_aead_prepare_result_buf(struct sg_table *tbl, struct aead_request *req) 91 { 92 struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req); 93 struct qce_alg_template *tmpl = to_aead_tmpl(crypto_aead_reqtfm(req)); 94 struct qce_device *qce = tmpl->qce; 95 96 sg_init_one(&rctx->result_sg, qce->dma.result_buf, QCE_RESULT_BUF_SZ); 97 return qce_sgtable_add(tbl, &rctx->result_sg, QCE_RESULT_BUF_SZ); 98 } 99 100 static struct scatterlist * 101 qce_aead_prepare_ccm_result_buf(struct sg_table *tbl, struct aead_request *req) 102 { 103 struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req); 104 105 sg_init_one(&rctx->result_sg, rctx->ccmresult_buf, QCE_BAM_BURST_SIZE); 106 return qce_sgtable_add(tbl, &rctx->result_sg, QCE_BAM_BURST_SIZE); 107 } 108 109 static struct scatterlist * 110 qce_aead_prepare_dst_buf(struct aead_request *req) 111 { 112 struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req); 113 struct qce_alg_template *tmpl = to_aead_tmpl(crypto_aead_reqtfm(req)); 114 struct qce_device *qce = tmpl->qce; 115 struct scatterlist *sg, *msg_sg, __sg[2]; 116 gfp_t gfp; 117 unsigned int assoclen = req->assoclen; 118 unsigned int totallen; 119 int ret; 120 121 totallen = rctx->cryptlen + assoclen; 122 rctx->dst_nents = sg_nents_for_len(req->dst, totallen); 123 if (rctx->dst_nents < 0) { 124 dev_err(qce->dev, "Invalid numbers of dst SG.\n"); 125 return ERR_PTR(-EINVAL); 126 } 127 if (IS_CCM(rctx->flags)) 128 rctx->dst_nents += 2; 129 else 130 rctx->dst_nents += 1; 131 132 gfp = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 133 GFP_KERNEL : GFP_ATOMIC; 134 ret = sg_alloc_table(&rctx->dst_tbl, rctx->dst_nents, gfp); 135 if (ret) 136 return ERR_PTR(ret); 137 138 if (IS_CCM(rctx->flags) && assoclen) { 139 /* Get the dst buffer */ 140 msg_sg = scatterwalk_ffwd(__sg, req->dst, assoclen); 141 142 sg = qce_sgtable_add(&rctx->dst_tbl, &rctx->adata_sg, 143 rctx->assoclen); 144 if (IS_ERR(sg)) 145 goto dst_tbl_free; 146 /* dst buffer */ 147 sg = qce_sgtable_add(&rctx->dst_tbl, msg_sg, rctx->cryptlen); 148 if (IS_ERR(sg)) 149 goto dst_tbl_free; 150 totallen = rctx->cryptlen + rctx->assoclen; 151 } else { 152 if (totallen) { 153 sg = qce_sgtable_add(&rctx->dst_tbl, req->dst, totallen); 154 if (IS_ERR(sg)) 155 goto dst_tbl_free; 156 } 157 } 158 if (IS_CCM(rctx->flags)) 159 sg = qce_aead_prepare_ccm_result_buf(&rctx->dst_tbl, req); 160 else 161 sg = qce_aead_prepare_result_buf(&rctx->dst_tbl, req); 162 163 if (IS_ERR(sg)) 164 goto dst_tbl_free; 165 166 sg_mark_end(sg); 167 rctx->dst_sg = rctx->dst_tbl.sgl; 168 rctx->dst_nents = sg_nents_for_len(rctx->dst_sg, totallen) + 1; 169 170 return sg; 171 172 dst_tbl_free: 173 sg_free_table(&rctx->dst_tbl); 174 return sg; 175 } 176 177 static int 178 qce_aead_ccm_prepare_buf_assoclen(struct aead_request *req) 179 { 180 struct scatterlist *sg, *msg_sg, __sg[2]; 181 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 182 struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req); 183 struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); 184 unsigned int assoclen = rctx->assoclen; 185 unsigned int adata_header_len, cryptlen, totallen; 186 gfp_t gfp; 187 bool diff_dst; 188 int ret; 189 190 if (IS_DECRYPT(rctx->flags)) 191 cryptlen = rctx->cryptlen + ctx->authsize; 192 else 193 cryptlen = rctx->cryptlen; 194 totallen = cryptlen + req->assoclen; 195 196 /* Get the msg */ 197 msg_sg = scatterwalk_ffwd(__sg, req->src, req->assoclen); 198 199 rctx->adata = kzalloc(ALIGN(assoclen + MAX_CCM_ADATA_HEADER_LEN, 16) * 200 sizeof(unsigned char), GFP_ATOMIC); 201 if (!rctx->adata) 202 return -ENOMEM; 203 204 /* 205 * Format associated data (RFC3610 and NIST 800-38C) 206 * Even though specification allows for AAD to be up to 2^64 - 1 bytes, 207 * the assoclen field in aead_request is unsigned int and thus limits 208 * the AAD to be up to 2^32 - 1 bytes. So we handle only two scenarios 209 * while forming the header for AAD. 210 */ 211 if (assoclen < 0xff00) { 212 adata_header_len = 2; 213 *(__be16 *)rctx->adata = cpu_to_be16(assoclen); 214 } else { 215 adata_header_len = 6; 216 *(__be16 *)rctx->adata = cpu_to_be16(0xfffe); 217 *(__be32 *)(rctx->adata + 2) = cpu_to_be32(assoclen); 218 } 219 220 /* Copy the associated data */ 221 if (sg_copy_to_buffer(req->src, sg_nents_for_len(req->src, assoclen), 222 rctx->adata + adata_header_len, 223 assoclen) != assoclen) 224 return -EINVAL; 225 226 /* Pad associated data to block size */ 227 rctx->assoclen = ALIGN(assoclen + adata_header_len, 16); 228 229 diff_dst = (req->src != req->dst) ? true : false; 230 231 if (diff_dst) 232 rctx->src_nents = sg_nents_for_len(req->src, totallen) + 1; 233 else 234 rctx->src_nents = sg_nents_for_len(req->src, totallen) + 2; 235 236 gfp = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; 237 ret = sg_alloc_table(&rctx->src_tbl, rctx->src_nents, gfp); 238 if (ret) 239 return ret; 240 241 /* Associated Data */ 242 sg_init_one(&rctx->adata_sg, rctx->adata, rctx->assoclen); 243 sg = qce_sgtable_add(&rctx->src_tbl, &rctx->adata_sg, 244 rctx->assoclen); 245 if (IS_ERR(sg)) { 246 ret = PTR_ERR(sg); 247 goto err_free; 248 } 249 /* src msg */ 250 sg = qce_sgtable_add(&rctx->src_tbl, msg_sg, cryptlen); 251 if (IS_ERR(sg)) { 252 ret = PTR_ERR(sg); 253 goto err_free; 254 } 255 if (!diff_dst) { 256 /* 257 * For decrypt, when src and dst buffers are same, there is already space 258 * in the buffer for padded 0's which is output in lieu of 259 * the MAC that is input. So skip the below. 260 */ 261 if (!IS_DECRYPT(rctx->flags)) { 262 sg = qce_aead_prepare_ccm_result_buf(&rctx->src_tbl, req); 263 if (IS_ERR(sg)) { 264 ret = PTR_ERR(sg); 265 goto err_free; 266 } 267 } 268 } 269 sg_mark_end(sg); 270 rctx->src_sg = rctx->src_tbl.sgl; 271 totallen = cryptlen + rctx->assoclen; 272 rctx->src_nents = sg_nents_for_len(rctx->src_sg, totallen); 273 274 if (diff_dst) { 275 sg = qce_aead_prepare_dst_buf(req); 276 if (IS_ERR(sg)) { 277 ret = PTR_ERR(sg); 278 goto err_free; 279 } 280 } else { 281 if (IS_ENCRYPT(rctx->flags)) 282 rctx->dst_nents = rctx->src_nents + 1; 283 else 284 rctx->dst_nents = rctx->src_nents; 285 rctx->dst_sg = rctx->src_sg; 286 } 287 288 return 0; 289 err_free: 290 sg_free_table(&rctx->src_tbl); 291 return ret; 292 } 293 294 static int qce_aead_prepare_buf(struct aead_request *req) 295 { 296 struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req); 297 struct qce_alg_template *tmpl = to_aead_tmpl(crypto_aead_reqtfm(req)); 298 struct qce_device *qce = tmpl->qce; 299 struct scatterlist *sg; 300 bool diff_dst = (req->src != req->dst) ? true : false; 301 unsigned int totallen; 302 303 totallen = rctx->cryptlen + rctx->assoclen; 304 305 sg = qce_aead_prepare_dst_buf(req); 306 if (IS_ERR(sg)) 307 return PTR_ERR(sg); 308 if (diff_dst) { 309 rctx->src_nents = sg_nents_for_len(req->src, totallen); 310 if (rctx->src_nents < 0) { 311 dev_err(qce->dev, "Invalid numbers of src SG.\n"); 312 return -EINVAL; 313 } 314 rctx->src_sg = req->src; 315 } else { 316 rctx->src_nents = rctx->dst_nents - 1; 317 rctx->src_sg = rctx->dst_sg; 318 } 319 return 0; 320 } 321 322 static int qce_aead_ccm_prepare_buf(struct aead_request *req) 323 { 324 struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req); 325 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 326 struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); 327 struct scatterlist *sg; 328 bool diff_dst = (req->src != req->dst) ? true : false; 329 unsigned int cryptlen; 330 331 if (rctx->assoclen) 332 return qce_aead_ccm_prepare_buf_assoclen(req); 333 334 if (IS_ENCRYPT(rctx->flags)) 335 return qce_aead_prepare_buf(req); 336 337 cryptlen = rctx->cryptlen + ctx->authsize; 338 if (diff_dst) { 339 rctx->src_nents = sg_nents_for_len(req->src, cryptlen); 340 rctx->src_sg = req->src; 341 sg = qce_aead_prepare_dst_buf(req); 342 if (IS_ERR(sg)) 343 return PTR_ERR(sg); 344 } else { 345 rctx->src_nents = sg_nents_for_len(req->src, cryptlen); 346 rctx->src_sg = req->src; 347 rctx->dst_nents = rctx->src_nents; 348 rctx->dst_sg = rctx->src_sg; 349 } 350 351 return 0; 352 } 353 354 static int qce_aead_create_ccm_nonce(struct qce_aead_reqctx *rctx, struct qce_aead_ctx *ctx) 355 { 356 unsigned int msglen_size, ivsize; 357 u8 msg_len[4]; 358 int i; 359 360 if (!rctx || !rctx->iv) 361 return -EINVAL; 362 363 msglen_size = rctx->iv[0] + 1; 364 365 /* Verify that msg len size is valid */ 366 if (msglen_size < 2 || msglen_size > 8) 367 return -EINVAL; 368 369 ivsize = rctx->ivsize; 370 371 /* 372 * Clear the msglen bytes in IV. 373 * Else the h/w engine and nonce will use any stray value pending there. 374 */ 375 if (!IS_CCM_RFC4309(rctx->flags)) { 376 for (i = 0; i < msglen_size; i++) 377 rctx->iv[ivsize - i - 1] = 0; 378 } 379 380 /* 381 * The crypto framework encodes cryptlen as unsigned int. Thus, even though 382 * spec allows for upto 8 bytes to encode msg_len only 4 bytes are needed. 383 */ 384 if (msglen_size > 4) 385 msglen_size = 4; 386 387 memcpy(&msg_len[0], &rctx->cryptlen, 4); 388 389 memcpy(&rctx->ccm_nonce[0], rctx->iv, rctx->ivsize); 390 if (rctx->assoclen) 391 rctx->ccm_nonce[0] |= 1 << CCM_NONCE_ADATA_SHIFT; 392 rctx->ccm_nonce[0] |= ((ctx->authsize - 2) / 2) << 393 CCM_NONCE_AUTHSIZE_SHIFT; 394 for (i = 0; i < msglen_size; i++) 395 rctx->ccm_nonce[QCE_MAX_NONCE - i - 1] = msg_len[i]; 396 397 return 0; 398 } 399 400 static int 401 qce_aead_async_req_handle(struct crypto_async_request *async_req) 402 { 403 struct aead_request *req = aead_request_cast(async_req); 404 struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req); 405 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 406 struct qce_aead_ctx *ctx = crypto_tfm_ctx(async_req->tfm); 407 struct qce_alg_template *tmpl = to_aead_tmpl(crypto_aead_reqtfm(req)); 408 struct qce_device *qce = tmpl->qce; 409 enum dma_data_direction dir_src, dir_dst; 410 bool diff_dst; 411 int dst_nents, src_nents, ret; 412 413 if (IS_CCM_RFC4309(rctx->flags)) { 414 memset(rctx->ccm_rfc4309_iv, 0, QCE_MAX_IV_SIZE); 415 rctx->ccm_rfc4309_iv[0] = 3; 416 memcpy(&rctx->ccm_rfc4309_iv[1], ctx->ccm4309_salt, QCE_CCM4309_SALT_SIZE); 417 memcpy(&rctx->ccm_rfc4309_iv[4], req->iv, 8); 418 rctx->iv = rctx->ccm_rfc4309_iv; 419 rctx->ivsize = AES_BLOCK_SIZE; 420 } else { 421 rctx->iv = req->iv; 422 rctx->ivsize = crypto_aead_ivsize(tfm); 423 } 424 if (IS_CCM_RFC4309(rctx->flags)) 425 rctx->assoclen = req->assoclen - 8; 426 else 427 rctx->assoclen = req->assoclen; 428 429 diff_dst = (req->src != req->dst) ? true : false; 430 dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; 431 dir_dst = diff_dst ? DMA_FROM_DEVICE : DMA_BIDIRECTIONAL; 432 433 if (IS_CCM(rctx->flags)) { 434 ret = qce_aead_create_ccm_nonce(rctx, ctx); 435 if (ret) 436 return ret; 437 } 438 if (IS_CCM(rctx->flags)) 439 ret = qce_aead_ccm_prepare_buf(req); 440 else 441 ret = qce_aead_prepare_buf(req); 442 443 if (ret) 444 return ret; 445 dst_nents = dma_map_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst); 446 if (!dst_nents) { 447 ret = -EIO; 448 goto error_free; 449 } 450 451 if (diff_dst) { 452 src_nents = dma_map_sg(qce->dev, rctx->src_sg, rctx->src_nents, dir_src); 453 if (src_nents < 0) { 454 ret = src_nents; 455 goto error_unmap_dst; 456 } 457 } else { 458 if (IS_CCM(rctx->flags) && IS_DECRYPT(rctx->flags)) 459 src_nents = dst_nents; 460 else 461 src_nents = dst_nents - 1; 462 } 463 464 ret = qce_dma_prep_sgs(&qce->dma, rctx->src_sg, src_nents, rctx->dst_sg, dst_nents, 465 qce_aead_done, async_req); 466 if (ret) 467 goto error_unmap_src; 468 469 qce_dma_issue_pending(&qce->dma); 470 471 ret = qce_start(async_req, tmpl->crypto_alg_type); 472 if (ret) 473 goto error_terminate; 474 475 return 0; 476 477 error_terminate: 478 qce_dma_terminate_all(&qce->dma); 479 error_unmap_src: 480 if (diff_dst) 481 dma_unmap_sg(qce->dev, req->src, rctx->src_nents, dir_src); 482 error_unmap_dst: 483 dma_unmap_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst); 484 error_free: 485 if (IS_CCM(rctx->flags) && rctx->assoclen) { 486 sg_free_table(&rctx->src_tbl); 487 if (diff_dst) 488 sg_free_table(&rctx->dst_tbl); 489 } else { 490 sg_free_table(&rctx->dst_tbl); 491 } 492 return ret; 493 } 494 495 static int qce_aead_crypt(struct aead_request *req, int encrypt) 496 { 497 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 498 struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req); 499 struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); 500 struct qce_alg_template *tmpl = to_aead_tmpl(tfm); 501 unsigned int blocksize = crypto_aead_blocksize(tfm); 502 503 rctx->flags = tmpl->alg_flags; 504 rctx->flags |= encrypt ? QCE_ENCRYPT : QCE_DECRYPT; 505 506 if (encrypt) 507 rctx->cryptlen = req->cryptlen; 508 else 509 rctx->cryptlen = req->cryptlen - ctx->authsize; 510 511 /* CE does not handle 0 length messages */ 512 if (!rctx->cryptlen) { 513 if (!(IS_CCM(rctx->flags) && IS_DECRYPT(rctx->flags))) 514 ctx->need_fallback = true; 515 } 516 517 /* If fallback is needed, schedule and exit */ 518 if (ctx->need_fallback) { 519 /* Reset need_fallback in case the same ctx is used for another transaction */ 520 ctx->need_fallback = false; 521 522 aead_request_set_tfm(&rctx->fallback_req, ctx->fallback); 523 aead_request_set_callback(&rctx->fallback_req, req->base.flags, 524 req->base.complete, req->base.data); 525 aead_request_set_crypt(&rctx->fallback_req, req->src, 526 req->dst, req->cryptlen, req->iv); 527 aead_request_set_ad(&rctx->fallback_req, req->assoclen); 528 529 return encrypt ? crypto_aead_encrypt(&rctx->fallback_req) : 530 crypto_aead_decrypt(&rctx->fallback_req); 531 } 532 533 /* 534 * CBC algorithms require message lengths to be 535 * multiples of block size. 536 */ 537 if (IS_CBC(rctx->flags) && !IS_ALIGNED(rctx->cryptlen, blocksize)) 538 return -EINVAL; 539 540 /* RFC4309 supported AAD size 16 bytes/20 bytes */ 541 if (IS_CCM_RFC4309(rctx->flags)) 542 if (crypto_ipsec_check_assoclen(req->assoclen)) 543 return -EINVAL; 544 545 return tmpl->qce->async_req_enqueue(tmpl->qce, &req->base); 546 } 547 548 static int qce_aead_encrypt(struct aead_request *req) 549 { 550 return qce_aead_crypt(req, 1); 551 } 552 553 static int qce_aead_decrypt(struct aead_request *req) 554 { 555 return qce_aead_crypt(req, 0); 556 } 557 558 static int qce_aead_ccm_setkey(struct crypto_aead *tfm, const u8 *key, 559 unsigned int keylen) 560 { 561 struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); 562 unsigned long flags = to_aead_tmpl(tfm)->alg_flags; 563 564 if (IS_CCM_RFC4309(flags)) { 565 if (keylen < QCE_CCM4309_SALT_SIZE) 566 return -EINVAL; 567 keylen -= QCE_CCM4309_SALT_SIZE; 568 memcpy(ctx->ccm4309_salt, key + keylen, QCE_CCM4309_SALT_SIZE); 569 } 570 571 if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_256 && keylen != AES_KEYSIZE_192) 572 return -EINVAL; 573 574 ctx->enc_keylen = keylen; 575 ctx->auth_keylen = keylen; 576 577 memcpy(ctx->enc_key, key, keylen); 578 memcpy(ctx->auth_key, key, keylen); 579 580 if (keylen == AES_KEYSIZE_192) 581 ctx->need_fallback = true; 582 583 return IS_CCM_RFC4309(flags) ? 584 crypto_aead_setkey(ctx->fallback, key, keylen + QCE_CCM4309_SALT_SIZE) : 585 crypto_aead_setkey(ctx->fallback, key, keylen); 586 } 587 588 static int qce_aead_setkey(struct crypto_aead *tfm, const u8 *key, unsigned int keylen) 589 { 590 struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); 591 struct crypto_authenc_keys authenc_keys; 592 unsigned long flags = to_aead_tmpl(tfm)->alg_flags; 593 int err; 594 595 err = crypto_authenc_extractkeys(&authenc_keys, key, keylen); 596 if (err) 597 return err; 598 599 if (authenc_keys.enckeylen > QCE_MAX_KEY_SIZE || 600 authenc_keys.authkeylen > QCE_MAX_KEY_SIZE) 601 return -EINVAL; 602 603 if (IS_AES(flags)) { 604 /* No random key sizes */ 605 if (authenc_keys.enckeylen != AES_KEYSIZE_128 && 606 authenc_keys.enckeylen != AES_KEYSIZE_192 && 607 authenc_keys.enckeylen != AES_KEYSIZE_256) 608 return -EINVAL; 609 if (authenc_keys.enckeylen == AES_KEYSIZE_192) 610 ctx->need_fallback = true; 611 } 612 613 ctx->enc_keylen = authenc_keys.enckeylen; 614 ctx->auth_keylen = authenc_keys.authkeylen; 615 616 memcpy(ctx->enc_key, authenc_keys.enckey, authenc_keys.enckeylen); 617 618 memcpy_and_pad(ctx->auth_key, sizeof(ctx->auth_key), 619 authenc_keys.authkey, authenc_keys.authkeylen, 0); 620 621 return crypto_aead_setkey(ctx->fallback, key, keylen); 622 } 623 624 static int qce_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 625 { 626 struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); 627 unsigned long flags = to_aead_tmpl(tfm)->alg_flags; 628 629 if (IS_CCM(flags)) { 630 if (authsize < 4 || authsize > 16 || authsize % 2) 631 return -EINVAL; 632 if (IS_CCM_RFC4309(flags) && (authsize < 8 || authsize % 4)) 633 return -EINVAL; 634 } 635 ctx->authsize = authsize; 636 637 return crypto_aead_setauthsize(ctx->fallback, authsize); 638 } 639 640 static int qce_aead_init(struct crypto_aead *tfm) 641 { 642 struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); 643 644 ctx->need_fallback = false; 645 ctx->fallback = crypto_alloc_aead(crypto_tfm_alg_name(&tfm->base), 646 0, CRYPTO_ALG_NEED_FALLBACK); 647 648 if (IS_ERR(ctx->fallback)) 649 return PTR_ERR(ctx->fallback); 650 651 crypto_aead_set_reqsize_dma(tfm, sizeof(struct qce_aead_reqctx) + 652 crypto_aead_reqsize(ctx->fallback)); 653 return 0; 654 } 655 656 static void qce_aead_exit(struct crypto_aead *tfm) 657 { 658 struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); 659 660 crypto_free_aead(ctx->fallback); 661 } 662 663 struct qce_aead_def { 664 unsigned long flags; 665 const char *name; 666 const char *drv_name; 667 unsigned int blocksize; 668 unsigned int chunksize; 669 unsigned int ivsize; 670 unsigned int maxauthsize; 671 }; 672 673 static const struct qce_aead_def aead_def[] = { 674 { 675 .flags = QCE_ALG_AES | QCE_MODE_CBC | QCE_HASH_SHA256_HMAC, 676 .name = "authenc(hmac(sha256),cbc(aes))", 677 .drv_name = "authenc-hmac-sha256-cbc-aes-qce", 678 .blocksize = AES_BLOCK_SIZE, 679 .ivsize = AES_BLOCK_SIZE, 680 .maxauthsize = SHA256_DIGEST_SIZE, 681 }, 682 { 683 .flags = QCE_ALG_AES | QCE_MODE_CCM, 684 .name = "ccm(aes)", 685 .drv_name = "ccm-aes-qce", 686 .blocksize = 1, 687 .ivsize = AES_BLOCK_SIZE, 688 .maxauthsize = AES_BLOCK_SIZE, 689 }, 690 { 691 .flags = QCE_ALG_AES | QCE_MODE_CCM | QCE_MODE_CCM_RFC4309, 692 .name = "rfc4309(ccm(aes))", 693 .drv_name = "rfc4309-ccm-aes-qce", 694 .blocksize = 1, 695 .ivsize = 8, 696 .maxauthsize = AES_BLOCK_SIZE, 697 }, 698 }; 699 700 static int qce_aead_register_one(const struct qce_aead_def *def, struct qce_device *qce) 701 { 702 struct qce_alg_template *tmpl; 703 struct aead_alg *alg; 704 int ret; 705 706 tmpl = kzalloc_obj(*tmpl); 707 if (!tmpl) 708 return -ENOMEM; 709 710 alg = &tmpl->alg.aead; 711 712 strscpy(alg->base.cra_name, def->name); 713 strscpy(alg->base.cra_driver_name, def->drv_name); 714 715 alg->base.cra_blocksize = def->blocksize; 716 alg->chunksize = def->chunksize; 717 alg->ivsize = def->ivsize; 718 alg->maxauthsize = def->maxauthsize; 719 if (IS_CCM(def->flags)) 720 alg->setkey = qce_aead_ccm_setkey; 721 else 722 alg->setkey = qce_aead_setkey; 723 alg->setauthsize = qce_aead_setauthsize; 724 alg->encrypt = qce_aead_encrypt; 725 alg->decrypt = qce_aead_decrypt; 726 alg->init = qce_aead_init; 727 alg->exit = qce_aead_exit; 728 729 alg->base.cra_priority = 275; 730 alg->base.cra_flags = CRYPTO_ALG_ASYNC | 731 CRYPTO_ALG_ALLOCATES_MEMORY | 732 CRYPTO_ALG_KERN_DRIVER_ONLY | 733 CRYPTO_ALG_NEED_FALLBACK; 734 alg->base.cra_ctxsize = sizeof(struct qce_aead_ctx); 735 alg->base.cra_alignmask = 0; 736 alg->base.cra_module = THIS_MODULE; 737 738 INIT_LIST_HEAD(&tmpl->entry); 739 tmpl->crypto_alg_type = CRYPTO_ALG_TYPE_AEAD; 740 tmpl->alg_flags = def->flags; 741 tmpl->qce = qce; 742 743 ret = crypto_register_aead(alg); 744 if (ret) { 745 dev_err(qce->dev, "%s registration failed\n", alg->base.cra_name); 746 kfree(tmpl); 747 return ret; 748 } 749 750 list_add_tail(&tmpl->entry, &aead_algs); 751 dev_dbg(qce->dev, "%s is registered\n", alg->base.cra_name); 752 return 0; 753 } 754 755 static void qce_aead_unregister(struct qce_device *qce) 756 { 757 struct qce_alg_template *tmpl, *n; 758 759 list_for_each_entry_safe(tmpl, n, &aead_algs, entry) { 760 crypto_unregister_aead(&tmpl->alg.aead); 761 list_del(&tmpl->entry); 762 kfree(tmpl); 763 } 764 } 765 766 static int qce_aead_register(struct qce_device *qce) 767 { 768 int ret, i; 769 770 for (i = 0; i < ARRAY_SIZE(aead_def); i++) { 771 ret = qce_aead_register_one(&aead_def[i], qce); 772 if (ret) 773 goto err; 774 } 775 776 return 0; 777 err: 778 qce_aead_unregister(qce); 779 return ret; 780 } 781 782 const struct qce_algo_ops aead_ops = { 783 .type = CRYPTO_ALG_TYPE_AEAD, 784 .register_algs = qce_aead_register, 785 .unregister_algs = qce_aead_unregister, 786 .async_req_handle = qce_aead_async_req_handle, 787 }; 788