1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2017 Marvell 4 * 5 * Antoine Tenart <antoine.tenart@free-electrons.com> 6 */ 7 8 #include <crypto/aes.h> 9 #include <crypto/hmac.h> 10 #include <crypto/md5.h> 11 #include <crypto/sha1.h> 12 #include <crypto/sha2.h> 13 #include <crypto/sha3.h> 14 #include <crypto/skcipher.h> 15 #include <crypto/sm3.h> 16 #include <crypto/internal/cipher.h> 17 #include <linux/device.h> 18 #include <linux/dma-mapping.h> 19 #include <linux/dmapool.h> 20 21 #include "safexcel.h" 22 23 struct safexcel_ahash_ctx { 24 struct safexcel_context base; 25 26 u32 alg; 27 u8 key_sz; 28 bool cbcmac; 29 bool do_fallback; 30 bool fb_init_done; 31 bool fb_do_setkey; 32 33 struct crypto_aes_ctx *aes; 34 struct crypto_ahash *fback; 35 struct crypto_shash *shpre; 36 struct shash_desc *shdesc; 37 }; 38 39 struct safexcel_ahash_req { 40 bool last_req; 41 bool finish; 42 bool hmac; 43 bool needs_inv; 44 bool hmac_zlen; 45 bool len_is_le; 46 bool not_first; 47 bool xcbcmac; 48 49 int nents; 50 dma_addr_t result_dma; 51 52 u32 digest; 53 54 u8 state_sz; /* expected state size, only set once */ 55 u8 block_sz; /* block size, only set once */ 56 u8 digest_sz; /* output digest size, only set once */ 57 __le32 state[SHA3_512_BLOCK_SIZE / 58 sizeof(__le32)] __aligned(sizeof(__le32)); 59 60 u64 len; 61 u64 processed; 62 63 u8 cache[HASH_CACHE_SIZE] __aligned(sizeof(u32)); 64 dma_addr_t cache_dma; 65 unsigned int cache_sz; 66 67 u8 cache_next[HASH_CACHE_SIZE] __aligned(sizeof(u32)); 68 }; 69 70 static inline u64 safexcel_queued_len(struct safexcel_ahash_req *req) 71 { 72 return req->len - req->processed; 73 } 74 75 static void safexcel_hash_token(struct safexcel_command_desc *cdesc, 76 u32 input_length, u32 result_length, 77 bool cbcmac) 78 { 79 struct safexcel_token *token = 80 (struct safexcel_token *)cdesc->control_data.token; 81 82 token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION; 83 token[0].packet_length = input_length; 84 token[0].instructions = EIP197_TOKEN_INS_TYPE_HASH; 85 86 input_length &= 15; 87 if (unlikely(cbcmac && input_length)) { 88 token[0].stat = 0; 89 token[1].opcode = EIP197_TOKEN_OPCODE_INSERT; 90 token[1].packet_length = 16 - input_length; 91 token[1].stat = EIP197_TOKEN_STAT_LAST_HASH; 92 token[1].instructions = EIP197_TOKEN_INS_TYPE_HASH; 93 } else { 94 token[0].stat = EIP197_TOKEN_STAT_LAST_HASH; 95 eip197_noop_token(&token[1]); 96 } 97 98 token[2].opcode = EIP197_TOKEN_OPCODE_INSERT; 99 token[2].stat = EIP197_TOKEN_STAT_LAST_HASH | 100 EIP197_TOKEN_STAT_LAST_PACKET; 101 token[2].packet_length = result_length; 102 token[2].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | 103 EIP197_TOKEN_INS_INSERT_HASH_DIGEST; 104 105 eip197_noop_token(&token[3]); 106 } 107 108 static void safexcel_context_control(struct safexcel_ahash_ctx *ctx, 109 struct safexcel_ahash_req *req, 110 struct safexcel_command_desc *cdesc) 111 { 112 struct safexcel_crypto_priv *priv = ctx->base.priv; 113 u64 count = 0; 114 115 cdesc->control_data.control0 = ctx->alg; 116 cdesc->control_data.control1 = 0; 117 118 /* 119 * Copy the input digest if needed, and setup the context 120 * fields. Do this now as we need it to setup the first command 121 * descriptor. 122 */ 123 if (unlikely(req->digest == CONTEXT_CONTROL_DIGEST_XCM)) { 124 if (req->xcbcmac) 125 memcpy(ctx->base.ctxr->data, &ctx->base.ipad, ctx->key_sz); 126 else 127 memcpy(ctx->base.ctxr->data, req->state, req->state_sz); 128 129 if (!req->finish && req->xcbcmac) 130 cdesc->control_data.control0 |= 131 CONTEXT_CONTROL_DIGEST_XCM | 132 CONTEXT_CONTROL_TYPE_HASH_OUT | 133 CONTEXT_CONTROL_NO_FINISH_HASH | 134 CONTEXT_CONTROL_SIZE(req->state_sz / 135 sizeof(u32)); 136 else 137 cdesc->control_data.control0 |= 138 CONTEXT_CONTROL_DIGEST_XCM | 139 CONTEXT_CONTROL_TYPE_HASH_OUT | 140 CONTEXT_CONTROL_SIZE(req->state_sz / 141 sizeof(u32)); 142 return; 143 } else if (!req->processed) { 144 /* First - and possibly only - block of basic hash only */ 145 if (req->finish) 146 cdesc->control_data.control0 |= req->digest | 147 CONTEXT_CONTROL_TYPE_HASH_OUT | 148 CONTEXT_CONTROL_RESTART_HASH | 149 /* ensure its not 0! */ 150 CONTEXT_CONTROL_SIZE(1); 151 else 152 cdesc->control_data.control0 |= req->digest | 153 CONTEXT_CONTROL_TYPE_HASH_OUT | 154 CONTEXT_CONTROL_RESTART_HASH | 155 CONTEXT_CONTROL_NO_FINISH_HASH | 156 /* ensure its not 0! */ 157 CONTEXT_CONTROL_SIZE(1); 158 return; 159 } 160 161 /* Hash continuation or HMAC, setup (inner) digest from state */ 162 memcpy(ctx->base.ctxr->data, req->state, req->state_sz); 163 164 if (req->finish) { 165 /* Compute digest count for hash/HMAC finish operations */ 166 if ((req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) || 167 req->hmac_zlen || (req->processed != req->block_sz)) { 168 count = req->processed / EIP197_COUNTER_BLOCK_SIZE; 169 170 /* This is a hardware limitation, as the 171 * counter must fit into an u32. This represents 172 * a fairly big amount of input data, so we 173 * shouldn't see this. 174 */ 175 if (unlikely(count & 0xffffffff00000000ULL)) { 176 dev_warn(priv->dev, 177 "Input data is too big\n"); 178 return; 179 } 180 } 181 182 if ((req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) || 183 /* Special case: zero length HMAC */ 184 req->hmac_zlen || 185 /* PE HW < 4.4 cannot do HMAC continue, fake using hash */ 186 (req->processed != req->block_sz)) { 187 /* Basic hash continue operation, need digest + cnt */ 188 cdesc->control_data.control0 |= 189 CONTEXT_CONTROL_SIZE((req->state_sz >> 2) + 1) | 190 CONTEXT_CONTROL_TYPE_HASH_OUT | 191 CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 192 /* For zero-len HMAC, don't finalize, already padded! */ 193 if (req->hmac_zlen) 194 cdesc->control_data.control0 |= 195 CONTEXT_CONTROL_NO_FINISH_HASH; 196 cdesc->control_data.control1 |= 197 CONTEXT_CONTROL_DIGEST_CNT; 198 ctx->base.ctxr->data[req->state_sz >> 2] = 199 cpu_to_le32(count); 200 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 201 202 /* Clear zero-length HMAC flag for next operation! */ 203 req->hmac_zlen = false; 204 } else { /* HMAC */ 205 /* Need outer digest for HMAC finalization */ 206 memcpy(ctx->base.ctxr->data + (req->state_sz >> 2), 207 &ctx->base.opad, req->state_sz); 208 209 /* Single pass HMAC - no digest count */ 210 cdesc->control_data.control0 |= 211 CONTEXT_CONTROL_SIZE(req->state_sz >> 1) | 212 CONTEXT_CONTROL_TYPE_HASH_OUT | 213 CONTEXT_CONTROL_DIGEST_HMAC; 214 } 215 } else { /* Hash continuation, do not finish yet */ 216 cdesc->control_data.control0 |= 217 CONTEXT_CONTROL_SIZE(req->state_sz >> 2) | 218 CONTEXT_CONTROL_DIGEST_PRECOMPUTED | 219 CONTEXT_CONTROL_TYPE_HASH_OUT | 220 CONTEXT_CONTROL_NO_FINISH_HASH; 221 } 222 } 223 224 static int safexcel_ahash_enqueue(struct ahash_request *areq); 225 226 static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, 227 int ring, 228 struct crypto_async_request *async, 229 bool *should_complete, int *ret) 230 { 231 struct safexcel_result_desc *rdesc; 232 struct ahash_request *areq = ahash_request_cast(async); 233 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); 234 struct safexcel_ahash_req *sreq = ahash_request_ctx_dma(areq); 235 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(ahash); 236 u64 cache_len; 237 238 *ret = 0; 239 240 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); 241 if (IS_ERR(rdesc)) { 242 dev_err(priv->dev, 243 "hash: result: could not retrieve the result descriptor\n"); 244 *ret = PTR_ERR(rdesc); 245 } else { 246 *ret = safexcel_rdesc_check_errors(priv, rdesc); 247 } 248 249 safexcel_complete(priv, ring); 250 251 if (sreq->nents) { 252 dma_unmap_sg(priv->dev, areq->src, 253 sg_nents_for_len(areq->src, areq->nbytes), 254 DMA_TO_DEVICE); 255 sreq->nents = 0; 256 } 257 258 if (sreq->result_dma) { 259 dma_unmap_single(priv->dev, sreq->result_dma, sreq->digest_sz, 260 DMA_FROM_DEVICE); 261 sreq->result_dma = 0; 262 } 263 264 if (sreq->cache_dma) { 265 dma_unmap_single(priv->dev, sreq->cache_dma, sreq->cache_sz, 266 DMA_TO_DEVICE); 267 sreq->cache_dma = 0; 268 sreq->cache_sz = 0; 269 } 270 271 if (sreq->finish) { 272 if (sreq->hmac && 273 (sreq->digest != CONTEXT_CONTROL_DIGEST_HMAC)) { 274 /* Faking HMAC using hash - need to do outer hash */ 275 memcpy(sreq->cache, sreq->state, 276 crypto_ahash_digestsize(ahash)); 277 278 memcpy(sreq->state, &ctx->base.opad, sreq->digest_sz); 279 280 sreq->len = sreq->block_sz + 281 crypto_ahash_digestsize(ahash); 282 sreq->processed = sreq->block_sz; 283 sreq->hmac = 0; 284 285 if (priv->flags & EIP197_TRC_CACHE) 286 ctx->base.needs_inv = true; 287 areq->nbytes = 0; 288 safexcel_ahash_enqueue(areq); 289 290 *should_complete = false; /* Not done yet */ 291 return 1; 292 } 293 294 memcpy(areq->result, sreq->state, 295 crypto_ahash_digestsize(ahash)); 296 } 297 298 cache_len = safexcel_queued_len(sreq); 299 if (cache_len) 300 memcpy(sreq->cache, sreq->cache_next, cache_len); 301 302 *should_complete = true; 303 304 return 1; 305 } 306 307 static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, 308 int *commands, int *results) 309 { 310 struct ahash_request *areq = ahash_request_cast(async); 311 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 312 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 313 struct safexcel_crypto_priv *priv = ctx->base.priv; 314 struct safexcel_command_desc *cdesc, *first_cdesc = NULL; 315 struct safexcel_result_desc *rdesc; 316 struct scatterlist *sg; 317 struct safexcel_token *dmmy; 318 int i, extra = 0, n_cdesc = 0, ret = 0, cache_len, skip = 0; 319 u64 queued, len; 320 321 queued = safexcel_queued_len(req); 322 if (queued <= HASH_CACHE_SIZE) 323 cache_len = queued; 324 else 325 cache_len = queued - areq->nbytes; 326 327 if (!req->finish && !req->last_req) { 328 /* If this is not the last request and the queued data does not 329 * fit into full cache blocks, cache it for the next send call. 330 */ 331 extra = queued & (HASH_CACHE_SIZE - 1); 332 333 /* If this is not the last request and the queued data 334 * is a multiple of a block, cache the last one for now. 335 */ 336 if (!extra) 337 extra = HASH_CACHE_SIZE; 338 339 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), 340 req->cache_next, extra, 341 areq->nbytes - extra); 342 343 queued -= extra; 344 345 if (!queued) { 346 *commands = 0; 347 *results = 0; 348 return 0; 349 } 350 351 extra = 0; 352 } 353 354 if (unlikely(req->xcbcmac && req->processed > AES_BLOCK_SIZE)) { 355 if (unlikely(cache_len < AES_BLOCK_SIZE)) { 356 /* 357 * Cache contains less than 1 full block, complete. 358 */ 359 extra = AES_BLOCK_SIZE - cache_len; 360 if (queued > cache_len) { 361 /* More data follows: borrow bytes */ 362 u64 tmp = queued - cache_len; 363 364 skip = min_t(u64, tmp, extra); 365 sg_pcopy_to_buffer(areq->src, 366 sg_nents(areq->src), 367 req->cache + cache_len, 368 skip, 0); 369 } 370 extra -= skip; 371 memset(req->cache + cache_len + skip, 0, extra); 372 if (!ctx->cbcmac && extra) { 373 // 10- padding for XCBCMAC & CMAC 374 req->cache[cache_len + skip] = 0x80; 375 // HW will use K2 iso K3 - compensate! 376 for (i = 0; i < AES_BLOCK_SIZE / 4; i++) { 377 u32 *cache = (void *)req->cache; 378 u32 *ipad = ctx->base.ipad.word; 379 u32 x; 380 381 x = ipad[i] ^ ipad[i + 4]; 382 cache[i] ^= swab32(x); 383 } 384 } 385 cache_len = AES_BLOCK_SIZE; 386 queued = queued + extra; 387 } 388 389 /* XCBC continue: XOR previous result into 1st word */ 390 crypto_xor(req->cache, (const u8 *)req->state, AES_BLOCK_SIZE); 391 } 392 393 len = queued; 394 /* Add a command descriptor for the cached data, if any */ 395 if (cache_len) { 396 req->cache_dma = dma_map_single(priv->dev, req->cache, 397 cache_len, DMA_TO_DEVICE); 398 if (dma_mapping_error(priv->dev, req->cache_dma)) 399 return -EINVAL; 400 401 req->cache_sz = cache_len; 402 first_cdesc = safexcel_add_cdesc(priv, ring, 1, 403 (cache_len == len), 404 req->cache_dma, cache_len, 405 len, ctx->base.ctxr_dma, 406 &dmmy); 407 if (IS_ERR(first_cdesc)) { 408 ret = PTR_ERR(first_cdesc); 409 goto unmap_cache; 410 } 411 n_cdesc++; 412 413 queued -= cache_len; 414 if (!queued) 415 goto send_command; 416 } 417 418 /* Now handle the current ahash request buffer(s) */ 419 req->nents = dma_map_sg(priv->dev, areq->src, 420 sg_nents_for_len(areq->src, 421 areq->nbytes), 422 DMA_TO_DEVICE); 423 if (!req->nents) { 424 ret = -ENOMEM; 425 goto cdesc_rollback; 426 } 427 428 for_each_sg(areq->src, sg, req->nents, i) { 429 int sglen = sg_dma_len(sg); 430 431 if (unlikely(sglen <= skip)) { 432 skip -= sglen; 433 continue; 434 } 435 436 /* Do not overflow the request */ 437 if ((queued + skip) <= sglen) 438 sglen = queued; 439 else 440 sglen -= skip; 441 442 cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc, 443 !(queued - sglen), 444 sg_dma_address(sg) + skip, sglen, 445 len, ctx->base.ctxr_dma, &dmmy); 446 if (IS_ERR(cdesc)) { 447 ret = PTR_ERR(cdesc); 448 goto unmap_sg; 449 } 450 451 if (!n_cdesc) 452 first_cdesc = cdesc; 453 n_cdesc++; 454 455 queued -= sglen; 456 if (!queued) 457 break; 458 skip = 0; 459 } 460 461 send_command: 462 /* Setup the context options */ 463 safexcel_context_control(ctx, req, first_cdesc); 464 465 /* Add the token */ 466 safexcel_hash_token(first_cdesc, len, req->digest_sz, ctx->cbcmac); 467 468 req->result_dma = dma_map_single(priv->dev, req->state, req->digest_sz, 469 DMA_FROM_DEVICE); 470 if (dma_mapping_error(priv->dev, req->result_dma)) { 471 ret = -EINVAL; 472 goto unmap_sg; 473 } 474 475 /* Add a result descriptor */ 476 rdesc = safexcel_add_rdesc(priv, ring, 1, 1, req->result_dma, 477 req->digest_sz); 478 if (IS_ERR(rdesc)) { 479 ret = PTR_ERR(rdesc); 480 goto unmap_result; 481 } 482 483 safexcel_rdr_req_set(priv, ring, rdesc, &areq->base); 484 485 req->processed += len - extra; 486 487 *commands = n_cdesc; 488 *results = 1; 489 return 0; 490 491 unmap_result: 492 dma_unmap_single(priv->dev, req->result_dma, req->digest_sz, 493 DMA_FROM_DEVICE); 494 unmap_sg: 495 if (req->nents) { 496 dma_unmap_sg(priv->dev, areq->src, 497 sg_nents_for_len(areq->src, areq->nbytes), 498 DMA_TO_DEVICE); 499 req->nents = 0; 500 } 501 cdesc_rollback: 502 for (i = 0; i < n_cdesc; i++) 503 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr); 504 unmap_cache: 505 if (req->cache_dma) { 506 dma_unmap_single(priv->dev, req->cache_dma, req->cache_sz, 507 DMA_TO_DEVICE); 508 req->cache_dma = 0; 509 req->cache_sz = 0; 510 } 511 512 return ret; 513 } 514 515 static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv, 516 int ring, 517 struct crypto_async_request *async, 518 bool *should_complete, int *ret) 519 { 520 struct safexcel_result_desc *rdesc; 521 struct ahash_request *areq = ahash_request_cast(async); 522 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); 523 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(ahash); 524 int enq_ret; 525 526 *ret = 0; 527 528 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); 529 if (IS_ERR(rdesc)) { 530 dev_err(priv->dev, 531 "hash: invalidate: could not retrieve the result descriptor\n"); 532 *ret = PTR_ERR(rdesc); 533 } else { 534 *ret = safexcel_rdesc_check_errors(priv, rdesc); 535 } 536 537 safexcel_complete(priv, ring); 538 539 if (ctx->base.exit_inv) { 540 dma_pool_free(priv->context_pool, ctx->base.ctxr, 541 ctx->base.ctxr_dma); 542 543 *should_complete = true; 544 return 1; 545 } 546 547 ring = safexcel_select_ring(priv); 548 ctx->base.ring = ring; 549 550 spin_lock_bh(&priv->ring[ring].queue_lock); 551 enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, async); 552 spin_unlock_bh(&priv->ring[ring].queue_lock); 553 554 if (enq_ret != -EINPROGRESS) 555 *ret = enq_ret; 556 557 queue_work(priv->ring[ring].workqueue, 558 &priv->ring[ring].work_data.work); 559 560 *should_complete = false; 561 562 return 1; 563 } 564 565 static int safexcel_handle_result(struct safexcel_crypto_priv *priv, int ring, 566 struct crypto_async_request *async, 567 bool *should_complete, int *ret) 568 { 569 struct ahash_request *areq = ahash_request_cast(async); 570 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 571 int err; 572 573 BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && req->needs_inv); 574 575 if (req->needs_inv) { 576 req->needs_inv = false; 577 err = safexcel_handle_inv_result(priv, ring, async, 578 should_complete, ret); 579 } else { 580 err = safexcel_handle_req_result(priv, ring, async, 581 should_complete, ret); 582 } 583 584 return err; 585 } 586 587 static int safexcel_ahash_send_inv(struct crypto_async_request *async, 588 int ring, int *commands, int *results) 589 { 590 struct ahash_request *areq = ahash_request_cast(async); 591 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 592 int ret; 593 594 ret = safexcel_invalidate_cache(async, ctx->base.priv, 595 ctx->base.ctxr_dma, ring); 596 if (unlikely(ret)) 597 return ret; 598 599 *commands = 1; 600 *results = 1; 601 602 return 0; 603 } 604 605 static int safexcel_ahash_send(struct crypto_async_request *async, 606 int ring, int *commands, int *results) 607 { 608 struct ahash_request *areq = ahash_request_cast(async); 609 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 610 int ret; 611 612 if (req->needs_inv) 613 ret = safexcel_ahash_send_inv(async, ring, commands, results); 614 else 615 ret = safexcel_ahash_send_req(async, ring, commands, results); 616 617 return ret; 618 } 619 620 static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm) 621 { 622 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 623 struct safexcel_crypto_priv *priv = ctx->base.priv; 624 EIP197_REQUEST_ON_STACK(req, ahash, EIP197_AHASH_REQ_SIZE); 625 struct safexcel_ahash_req *rctx = ahash_request_ctx_dma(req); 626 DECLARE_CRYPTO_WAIT(result); 627 int ring = ctx->base.ring; 628 int err; 629 630 memset(req, 0, EIP197_AHASH_REQ_SIZE); 631 632 /* create invalidation request */ 633 init_completion(&result.completion); 634 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 635 crypto_req_done, &result); 636 637 ahash_request_set_tfm(req, __crypto_ahash_cast(tfm)); 638 ctx = crypto_tfm_ctx(req->base.tfm); 639 ctx->base.exit_inv = true; 640 rctx->needs_inv = true; 641 642 spin_lock_bh(&priv->ring[ring].queue_lock); 643 crypto_enqueue_request(&priv->ring[ring].queue, &req->base); 644 spin_unlock_bh(&priv->ring[ring].queue_lock); 645 646 queue_work(priv->ring[ring].workqueue, 647 &priv->ring[ring].work_data.work); 648 649 err = crypto_wait_req(-EINPROGRESS, &result); 650 651 if (err) { 652 dev_warn(priv->dev, "hash: completion error (%d)\n", err); 653 return err; 654 } 655 656 return 0; 657 } 658 659 /* safexcel_ahash_cache: cache data until at least one request can be sent to 660 * the engine, aka. when there is at least 1 block size in the pipe. 661 */ 662 static int safexcel_ahash_cache(struct ahash_request *areq) 663 { 664 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 665 u64 cache_len; 666 667 /* cache_len: everything accepted by the driver but not sent yet, 668 * tot sz handled by update() - last req sz - tot sz handled by send() 669 */ 670 cache_len = safexcel_queued_len(req); 671 672 /* 673 * In case there isn't enough bytes to proceed (less than a 674 * block size), cache the data until we have enough. 675 */ 676 if (cache_len + areq->nbytes <= HASH_CACHE_SIZE) { 677 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), 678 req->cache + cache_len, 679 areq->nbytes, 0); 680 return 0; 681 } 682 683 /* We couldn't cache all the data */ 684 return -E2BIG; 685 } 686 687 static int safexcel_ahash_enqueue(struct ahash_request *areq) 688 { 689 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 690 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 691 struct safexcel_crypto_priv *priv = ctx->base.priv; 692 int ret, ring; 693 694 req->needs_inv = false; 695 696 if (ctx->base.ctxr) { 697 if (priv->flags & EIP197_TRC_CACHE && !ctx->base.needs_inv && 698 /* invalidate for *any* non-XCBC continuation */ 699 ((req->not_first && !req->xcbcmac) || 700 /* invalidate if (i)digest changed */ 701 memcmp(ctx->base.ctxr->data, req->state, req->state_sz) || 702 /* invalidate for HMAC finish with odigest changed */ 703 (req->finish && req->hmac && 704 memcmp(ctx->base.ctxr->data + (req->state_sz>>2), 705 &ctx->base.opad, req->state_sz)))) 706 /* 707 * We're still setting needs_inv here, even though it is 708 * cleared right away, because the needs_inv flag can be 709 * set in other functions and we want to keep the same 710 * logic. 711 */ 712 ctx->base.needs_inv = true; 713 714 if (ctx->base.needs_inv) { 715 ctx->base.needs_inv = false; 716 req->needs_inv = true; 717 } 718 } else { 719 ctx->base.ring = safexcel_select_ring(priv); 720 ctx->base.ctxr = dma_pool_zalloc(priv->context_pool, 721 EIP197_GFP_FLAGS(areq->base), 722 &ctx->base.ctxr_dma); 723 if (!ctx->base.ctxr) 724 return -ENOMEM; 725 } 726 req->not_first = true; 727 728 ring = ctx->base.ring; 729 730 spin_lock_bh(&priv->ring[ring].queue_lock); 731 ret = crypto_enqueue_request(&priv->ring[ring].queue, &areq->base); 732 spin_unlock_bh(&priv->ring[ring].queue_lock); 733 734 queue_work(priv->ring[ring].workqueue, 735 &priv->ring[ring].work_data.work); 736 737 return ret; 738 } 739 740 static int safexcel_ahash_update(struct ahash_request *areq) 741 { 742 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 743 int ret; 744 745 /* If the request is 0 length, do nothing */ 746 if (!areq->nbytes) 747 return 0; 748 749 /* Add request to the cache if it fits */ 750 ret = safexcel_ahash_cache(areq); 751 752 /* Update total request length */ 753 req->len += areq->nbytes; 754 755 /* If not all data could fit into the cache, go process the excess. 756 * Also go process immediately for an HMAC IV precompute, which 757 * will never be finished at all, but needs to be processed anyway. 758 */ 759 if ((ret && !req->finish) || req->last_req) 760 return safexcel_ahash_enqueue(areq); 761 762 return 0; 763 } 764 765 static int safexcel_ahash_final(struct ahash_request *areq) 766 { 767 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 768 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 769 770 req->finish = true; 771 772 if (unlikely(!req->len && !areq->nbytes)) { 773 /* 774 * If we have an overall 0 length *hash* request: 775 * The HW cannot do 0 length hash, so we provide the correct 776 * result directly here. 777 */ 778 if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5) 779 memcpy(areq->result, md5_zero_message_hash, 780 MD5_DIGEST_SIZE); 781 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA1) 782 memcpy(areq->result, sha1_zero_message_hash, 783 SHA1_DIGEST_SIZE); 784 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA224) 785 memcpy(areq->result, sha224_zero_message_hash, 786 SHA224_DIGEST_SIZE); 787 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA256) 788 memcpy(areq->result, sha256_zero_message_hash, 789 SHA256_DIGEST_SIZE); 790 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA384) 791 memcpy(areq->result, sha384_zero_message_hash, 792 SHA384_DIGEST_SIZE); 793 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA512) 794 memcpy(areq->result, sha512_zero_message_hash, 795 SHA512_DIGEST_SIZE); 796 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SM3) { 797 memcpy(areq->result, 798 EIP197_SM3_ZEROM_HASH, SM3_DIGEST_SIZE); 799 } 800 801 return 0; 802 } else if (unlikely(req->digest == CONTEXT_CONTROL_DIGEST_XCM && 803 ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5 && 804 req->len == sizeof(u32) && !areq->nbytes)) { 805 /* Zero length CRC32 */ 806 memcpy(areq->result, &ctx->base.ipad, sizeof(u32)); 807 return 0; 808 } else if (unlikely(ctx->cbcmac && req->len == AES_BLOCK_SIZE && 809 !areq->nbytes)) { 810 /* Zero length CBC MAC */ 811 memset(areq->result, 0, AES_BLOCK_SIZE); 812 return 0; 813 } else if (unlikely(req->xcbcmac && req->len == AES_BLOCK_SIZE && 814 !areq->nbytes)) { 815 /* Zero length (X)CBC/CMAC */ 816 int i; 817 818 for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) { 819 u32 *result = (void *)areq->result; 820 821 /* K3 */ 822 result[i] = swab32(ctx->base.ipad.word[i + 4]); 823 } 824 areq->result[0] ^= 0x80; // 10- padding 825 aes_encrypt(ctx->aes, areq->result, areq->result); 826 return 0; 827 } else if (unlikely(req->hmac && 828 (req->len == req->block_sz) && 829 !areq->nbytes)) { 830 /* 831 * If we have an overall 0 length *HMAC* request: 832 * For HMAC, we need to finalize the inner digest 833 * and then perform the outer hash. 834 */ 835 836 /* generate pad block in the cache */ 837 /* start with a hash block of all zeroes */ 838 memset(req->cache, 0, req->block_sz); 839 /* set the first byte to 0x80 to 'append a 1 bit' */ 840 req->cache[0] = 0x80; 841 /* add the length in bits in the last 2 bytes */ 842 if (req->len_is_le) { 843 /* Little endian length word (e.g. MD5) */ 844 req->cache[req->block_sz-8] = (req->block_sz << 3) & 845 255; 846 req->cache[req->block_sz-7] = (req->block_sz >> 5); 847 } else { 848 /* Big endian length word (e.g. any SHA) */ 849 req->cache[req->block_sz-2] = (req->block_sz >> 5); 850 req->cache[req->block_sz-1] = (req->block_sz << 3) & 851 255; 852 } 853 854 req->len += req->block_sz; /* plus 1 hash block */ 855 856 /* Set special zero-length HMAC flag */ 857 req->hmac_zlen = true; 858 859 /* Finalize HMAC */ 860 req->digest = CONTEXT_CONTROL_DIGEST_HMAC; 861 } else if (req->hmac) { 862 /* Finalize HMAC */ 863 req->digest = CONTEXT_CONTROL_DIGEST_HMAC; 864 } 865 866 return safexcel_ahash_enqueue(areq); 867 } 868 869 static int safexcel_ahash_finup(struct ahash_request *areq) 870 { 871 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 872 873 req->finish = true; 874 875 safexcel_ahash_update(areq); 876 return safexcel_ahash_final(areq); 877 } 878 879 static int safexcel_ahash_export(struct ahash_request *areq, void *out) 880 { 881 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 882 struct safexcel_ahash_export_state *export = out; 883 884 export->len = req->len; 885 export->processed = req->processed; 886 887 export->digest = req->digest; 888 889 memcpy(export->state, req->state, req->state_sz); 890 memcpy(export->cache, req->cache, HASH_CACHE_SIZE); 891 892 return 0; 893 } 894 895 static int safexcel_ahash_import(struct ahash_request *areq, const void *in) 896 { 897 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 898 const struct safexcel_ahash_export_state *export = in; 899 int ret; 900 901 ret = crypto_ahash_init(areq); 902 if (ret) 903 return ret; 904 905 req->len = export->len; 906 req->processed = export->processed; 907 908 req->digest = export->digest; 909 910 memcpy(req->cache, export->cache, HASH_CACHE_SIZE); 911 memcpy(req->state, export->state, req->state_sz); 912 913 return 0; 914 } 915 916 static int safexcel_ahash_cra_init(struct crypto_tfm *tfm) 917 { 918 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 919 struct safexcel_alg_template *tmpl = 920 container_of(__crypto_ahash_alg(tfm->__crt_alg), 921 struct safexcel_alg_template, alg.ahash); 922 923 ctx->base.priv = tmpl->priv; 924 ctx->base.send = safexcel_ahash_send; 925 ctx->base.handle_result = safexcel_handle_result; 926 ctx->fb_do_setkey = false; 927 928 crypto_ahash_set_reqsize_dma(__crypto_ahash_cast(tfm), 929 sizeof(struct safexcel_ahash_req)); 930 return 0; 931 } 932 933 static int safexcel_sha1_init(struct ahash_request *areq) 934 { 935 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 936 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 937 938 memset(req, 0, sizeof(*req)); 939 940 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1; 941 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 942 req->state_sz = SHA1_DIGEST_SIZE; 943 req->digest_sz = SHA1_DIGEST_SIZE; 944 req->block_sz = SHA1_BLOCK_SIZE; 945 946 return 0; 947 } 948 949 static int safexcel_sha1_digest(struct ahash_request *areq) 950 { 951 int ret = safexcel_sha1_init(areq); 952 953 if (ret) 954 return ret; 955 956 return safexcel_ahash_finup(areq); 957 } 958 959 static void safexcel_ahash_cra_exit(struct crypto_tfm *tfm) 960 { 961 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 962 struct safexcel_crypto_priv *priv = ctx->base.priv; 963 int ret; 964 965 /* context not allocated, skip invalidation */ 966 if (!ctx->base.ctxr) 967 return; 968 969 if (priv->flags & EIP197_TRC_CACHE) { 970 ret = safexcel_ahash_exit_inv(tfm); 971 if (ret) 972 dev_warn(priv->dev, "hash: invalidation error %d\n", ret); 973 } else { 974 dma_pool_free(priv->context_pool, ctx->base.ctxr, 975 ctx->base.ctxr_dma); 976 } 977 } 978 979 struct safexcel_alg_template safexcel_alg_sha1 = { 980 .type = SAFEXCEL_ALG_TYPE_AHASH, 981 .algo_mask = SAFEXCEL_ALG_SHA1, 982 .alg.ahash = { 983 .init = safexcel_sha1_init, 984 .update = safexcel_ahash_update, 985 .final = safexcel_ahash_final, 986 .finup = safexcel_ahash_finup, 987 .digest = safexcel_sha1_digest, 988 .export = safexcel_ahash_export, 989 .import = safexcel_ahash_import, 990 .halg = { 991 .digestsize = SHA1_DIGEST_SIZE, 992 .statesize = sizeof(struct safexcel_ahash_export_state), 993 .base = { 994 .cra_name = "sha1", 995 .cra_driver_name = "safexcel-sha1", 996 .cra_priority = SAFEXCEL_CRA_PRIORITY, 997 .cra_flags = CRYPTO_ALG_ASYNC | 998 CRYPTO_ALG_ALLOCATES_MEMORY | 999 CRYPTO_ALG_KERN_DRIVER_ONLY, 1000 .cra_blocksize = SHA1_BLOCK_SIZE, 1001 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1002 .cra_init = safexcel_ahash_cra_init, 1003 .cra_exit = safexcel_ahash_cra_exit, 1004 .cra_module = THIS_MODULE, 1005 }, 1006 }, 1007 }, 1008 }; 1009 1010 static int safexcel_hmac_sha1_init(struct ahash_request *areq) 1011 { 1012 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1013 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1014 1015 memset(req, 0, sizeof(*req)); 1016 1017 /* Start from ipad precompute */ 1018 memcpy(req->state, &ctx->base.ipad, SHA1_DIGEST_SIZE); 1019 /* Already processed the key^ipad part now! */ 1020 req->len = SHA1_BLOCK_SIZE; 1021 req->processed = SHA1_BLOCK_SIZE; 1022 1023 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1; 1024 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1025 req->state_sz = SHA1_DIGEST_SIZE; 1026 req->digest_sz = SHA1_DIGEST_SIZE; 1027 req->block_sz = SHA1_BLOCK_SIZE; 1028 req->hmac = true; 1029 1030 return 0; 1031 } 1032 1033 static int safexcel_hmac_sha1_digest(struct ahash_request *areq) 1034 { 1035 int ret = safexcel_hmac_sha1_init(areq); 1036 1037 if (ret) 1038 return ret; 1039 1040 return safexcel_ahash_finup(areq); 1041 } 1042 1043 static int safexcel_hmac_init_pad(struct ahash_request *areq, 1044 unsigned int blocksize, const u8 *key, 1045 unsigned int keylen, u8 *ipad, u8 *opad) 1046 { 1047 DECLARE_CRYPTO_WAIT(result); 1048 struct scatterlist sg; 1049 int ret, i; 1050 u8 *keydup; 1051 1052 if (keylen <= blocksize) { 1053 memcpy(ipad, key, keylen); 1054 } else { 1055 keydup = kmemdup(key, keylen, GFP_KERNEL); 1056 if (!keydup) 1057 return -ENOMEM; 1058 1059 ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG, 1060 crypto_req_done, &result); 1061 sg_init_one(&sg, keydup, keylen); 1062 ahash_request_set_crypt(areq, &sg, ipad, keylen); 1063 1064 ret = crypto_ahash_digest(areq); 1065 ret = crypto_wait_req(ret, &result); 1066 1067 /* Avoid leaking */ 1068 kfree_sensitive(keydup); 1069 1070 if (ret) 1071 return ret; 1072 1073 keylen = crypto_ahash_digestsize(crypto_ahash_reqtfm(areq)); 1074 } 1075 1076 memset(ipad + keylen, 0, blocksize - keylen); 1077 memcpy(opad, ipad, blocksize); 1078 1079 for (i = 0; i < blocksize; i++) { 1080 ipad[i] ^= HMAC_IPAD_VALUE; 1081 opad[i] ^= HMAC_OPAD_VALUE; 1082 } 1083 1084 return 0; 1085 } 1086 1087 static int safexcel_hmac_init_iv(struct ahash_request *areq, 1088 unsigned int blocksize, u8 *pad, void *state) 1089 { 1090 struct safexcel_ahash_req *req; 1091 DECLARE_CRYPTO_WAIT(result); 1092 struct scatterlist sg; 1093 int ret; 1094 1095 ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG, 1096 crypto_req_done, &result); 1097 sg_init_one(&sg, pad, blocksize); 1098 ahash_request_set_crypt(areq, &sg, pad, blocksize); 1099 1100 ret = crypto_ahash_init(areq); 1101 if (ret) 1102 return ret; 1103 1104 req = ahash_request_ctx_dma(areq); 1105 req->hmac = true; 1106 req->last_req = true; 1107 1108 ret = crypto_ahash_update(areq); 1109 ret = crypto_wait_req(ret, &result); 1110 1111 return ret ?: crypto_ahash_export(areq, state); 1112 } 1113 1114 static int __safexcel_hmac_setkey(const char *alg, const u8 *key, 1115 unsigned int keylen, 1116 void *istate, void *ostate) 1117 { 1118 struct ahash_request *areq; 1119 struct crypto_ahash *tfm; 1120 unsigned int blocksize; 1121 u8 *ipad, *opad; 1122 int ret; 1123 1124 tfm = crypto_alloc_ahash(alg, 0, 0); 1125 if (IS_ERR(tfm)) 1126 return PTR_ERR(tfm); 1127 1128 areq = ahash_request_alloc(tfm, GFP_KERNEL); 1129 if (!areq) { 1130 ret = -ENOMEM; 1131 goto free_ahash; 1132 } 1133 1134 crypto_ahash_clear_flags(tfm, ~0); 1135 blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 1136 1137 ipad = kcalloc(2, blocksize, GFP_KERNEL); 1138 if (!ipad) { 1139 ret = -ENOMEM; 1140 goto free_request; 1141 } 1142 1143 opad = ipad + blocksize; 1144 1145 ret = safexcel_hmac_init_pad(areq, blocksize, key, keylen, ipad, opad); 1146 if (ret) 1147 goto free_ipad; 1148 1149 ret = safexcel_hmac_init_iv(areq, blocksize, ipad, istate); 1150 if (ret) 1151 goto free_ipad; 1152 1153 ret = safexcel_hmac_init_iv(areq, blocksize, opad, ostate); 1154 1155 free_ipad: 1156 kfree(ipad); 1157 free_request: 1158 ahash_request_free(areq); 1159 free_ahash: 1160 crypto_free_ahash(tfm); 1161 1162 return ret; 1163 } 1164 1165 int safexcel_hmac_setkey(struct safexcel_context *base, const u8 *key, 1166 unsigned int keylen, const char *alg, 1167 unsigned int state_sz) 1168 { 1169 struct safexcel_crypto_priv *priv = base->priv; 1170 struct safexcel_ahash_export_state istate, ostate; 1171 int ret; 1172 1173 ret = __safexcel_hmac_setkey(alg, key, keylen, &istate, &ostate); 1174 if (ret) 1175 return ret; 1176 1177 if (priv->flags & EIP197_TRC_CACHE && base->ctxr && 1178 (memcmp(&base->ipad, istate.state, state_sz) || 1179 memcmp(&base->opad, ostate.state, state_sz))) 1180 base->needs_inv = true; 1181 1182 memcpy(&base->ipad, &istate.state, state_sz); 1183 memcpy(&base->opad, &ostate.state, state_sz); 1184 1185 return 0; 1186 } 1187 1188 static int safexcel_hmac_alg_setkey(struct crypto_ahash *tfm, const u8 *key, 1189 unsigned int keylen, const char *alg, 1190 unsigned int state_sz) 1191 { 1192 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 1193 1194 return safexcel_hmac_setkey(&ctx->base, key, keylen, alg, state_sz); 1195 } 1196 1197 static int safexcel_hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key, 1198 unsigned int keylen) 1199 { 1200 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha1", 1201 SHA1_DIGEST_SIZE); 1202 } 1203 1204 struct safexcel_alg_template safexcel_alg_hmac_sha1 = { 1205 .type = SAFEXCEL_ALG_TYPE_AHASH, 1206 .algo_mask = SAFEXCEL_ALG_SHA1, 1207 .alg.ahash = { 1208 .init = safexcel_hmac_sha1_init, 1209 .update = safexcel_ahash_update, 1210 .final = safexcel_ahash_final, 1211 .finup = safexcel_ahash_finup, 1212 .digest = safexcel_hmac_sha1_digest, 1213 .setkey = safexcel_hmac_sha1_setkey, 1214 .export = safexcel_ahash_export, 1215 .import = safexcel_ahash_import, 1216 .halg = { 1217 .digestsize = SHA1_DIGEST_SIZE, 1218 .statesize = sizeof(struct safexcel_ahash_export_state), 1219 .base = { 1220 .cra_name = "hmac(sha1)", 1221 .cra_driver_name = "safexcel-hmac-sha1", 1222 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1223 .cra_flags = CRYPTO_ALG_ASYNC | 1224 CRYPTO_ALG_ALLOCATES_MEMORY | 1225 CRYPTO_ALG_KERN_DRIVER_ONLY, 1226 .cra_blocksize = SHA1_BLOCK_SIZE, 1227 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1228 .cra_init = safexcel_ahash_cra_init, 1229 .cra_exit = safexcel_ahash_cra_exit, 1230 .cra_module = THIS_MODULE, 1231 }, 1232 }, 1233 }, 1234 }; 1235 1236 static int safexcel_sha256_init(struct ahash_request *areq) 1237 { 1238 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1239 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1240 1241 memset(req, 0, sizeof(*req)); 1242 1243 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256; 1244 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1245 req->state_sz = SHA256_DIGEST_SIZE; 1246 req->digest_sz = SHA256_DIGEST_SIZE; 1247 req->block_sz = SHA256_BLOCK_SIZE; 1248 1249 return 0; 1250 } 1251 1252 static int safexcel_sha256_digest(struct ahash_request *areq) 1253 { 1254 int ret = safexcel_sha256_init(areq); 1255 1256 if (ret) 1257 return ret; 1258 1259 return safexcel_ahash_finup(areq); 1260 } 1261 1262 struct safexcel_alg_template safexcel_alg_sha256 = { 1263 .type = SAFEXCEL_ALG_TYPE_AHASH, 1264 .algo_mask = SAFEXCEL_ALG_SHA2_256, 1265 .alg.ahash = { 1266 .init = safexcel_sha256_init, 1267 .update = safexcel_ahash_update, 1268 .final = safexcel_ahash_final, 1269 .finup = safexcel_ahash_finup, 1270 .digest = safexcel_sha256_digest, 1271 .export = safexcel_ahash_export, 1272 .import = safexcel_ahash_import, 1273 .halg = { 1274 .digestsize = SHA256_DIGEST_SIZE, 1275 .statesize = sizeof(struct safexcel_ahash_export_state), 1276 .base = { 1277 .cra_name = "sha256", 1278 .cra_driver_name = "safexcel-sha256", 1279 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1280 .cra_flags = CRYPTO_ALG_ASYNC | 1281 CRYPTO_ALG_ALLOCATES_MEMORY | 1282 CRYPTO_ALG_KERN_DRIVER_ONLY, 1283 .cra_blocksize = SHA256_BLOCK_SIZE, 1284 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1285 .cra_init = safexcel_ahash_cra_init, 1286 .cra_exit = safexcel_ahash_cra_exit, 1287 .cra_module = THIS_MODULE, 1288 }, 1289 }, 1290 }, 1291 }; 1292 1293 static int safexcel_sha224_init(struct ahash_request *areq) 1294 { 1295 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1296 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1297 1298 memset(req, 0, sizeof(*req)); 1299 1300 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224; 1301 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1302 req->state_sz = SHA256_DIGEST_SIZE; 1303 req->digest_sz = SHA256_DIGEST_SIZE; 1304 req->block_sz = SHA256_BLOCK_SIZE; 1305 1306 return 0; 1307 } 1308 1309 static int safexcel_sha224_digest(struct ahash_request *areq) 1310 { 1311 int ret = safexcel_sha224_init(areq); 1312 1313 if (ret) 1314 return ret; 1315 1316 return safexcel_ahash_finup(areq); 1317 } 1318 1319 struct safexcel_alg_template safexcel_alg_sha224 = { 1320 .type = SAFEXCEL_ALG_TYPE_AHASH, 1321 .algo_mask = SAFEXCEL_ALG_SHA2_256, 1322 .alg.ahash = { 1323 .init = safexcel_sha224_init, 1324 .update = safexcel_ahash_update, 1325 .final = safexcel_ahash_final, 1326 .finup = safexcel_ahash_finup, 1327 .digest = safexcel_sha224_digest, 1328 .export = safexcel_ahash_export, 1329 .import = safexcel_ahash_import, 1330 .halg = { 1331 .digestsize = SHA224_DIGEST_SIZE, 1332 .statesize = sizeof(struct safexcel_ahash_export_state), 1333 .base = { 1334 .cra_name = "sha224", 1335 .cra_driver_name = "safexcel-sha224", 1336 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1337 .cra_flags = CRYPTO_ALG_ASYNC | 1338 CRYPTO_ALG_ALLOCATES_MEMORY | 1339 CRYPTO_ALG_KERN_DRIVER_ONLY, 1340 .cra_blocksize = SHA224_BLOCK_SIZE, 1341 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1342 .cra_init = safexcel_ahash_cra_init, 1343 .cra_exit = safexcel_ahash_cra_exit, 1344 .cra_module = THIS_MODULE, 1345 }, 1346 }, 1347 }, 1348 }; 1349 1350 static int safexcel_hmac_sha224_setkey(struct crypto_ahash *tfm, const u8 *key, 1351 unsigned int keylen) 1352 { 1353 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha224", 1354 SHA256_DIGEST_SIZE); 1355 } 1356 1357 static int safexcel_hmac_sha224_init(struct ahash_request *areq) 1358 { 1359 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1360 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1361 1362 memset(req, 0, sizeof(*req)); 1363 1364 /* Start from ipad precompute */ 1365 memcpy(req->state, &ctx->base.ipad, SHA256_DIGEST_SIZE); 1366 /* Already processed the key^ipad part now! */ 1367 req->len = SHA256_BLOCK_SIZE; 1368 req->processed = SHA256_BLOCK_SIZE; 1369 1370 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224; 1371 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1372 req->state_sz = SHA256_DIGEST_SIZE; 1373 req->digest_sz = SHA256_DIGEST_SIZE; 1374 req->block_sz = SHA256_BLOCK_SIZE; 1375 req->hmac = true; 1376 1377 return 0; 1378 } 1379 1380 static int safexcel_hmac_sha224_digest(struct ahash_request *areq) 1381 { 1382 int ret = safexcel_hmac_sha224_init(areq); 1383 1384 if (ret) 1385 return ret; 1386 1387 return safexcel_ahash_finup(areq); 1388 } 1389 1390 struct safexcel_alg_template safexcel_alg_hmac_sha224 = { 1391 .type = SAFEXCEL_ALG_TYPE_AHASH, 1392 .algo_mask = SAFEXCEL_ALG_SHA2_256, 1393 .alg.ahash = { 1394 .init = safexcel_hmac_sha224_init, 1395 .update = safexcel_ahash_update, 1396 .final = safexcel_ahash_final, 1397 .finup = safexcel_ahash_finup, 1398 .digest = safexcel_hmac_sha224_digest, 1399 .setkey = safexcel_hmac_sha224_setkey, 1400 .export = safexcel_ahash_export, 1401 .import = safexcel_ahash_import, 1402 .halg = { 1403 .digestsize = SHA224_DIGEST_SIZE, 1404 .statesize = sizeof(struct safexcel_ahash_export_state), 1405 .base = { 1406 .cra_name = "hmac(sha224)", 1407 .cra_driver_name = "safexcel-hmac-sha224", 1408 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1409 .cra_flags = CRYPTO_ALG_ASYNC | 1410 CRYPTO_ALG_ALLOCATES_MEMORY | 1411 CRYPTO_ALG_KERN_DRIVER_ONLY, 1412 .cra_blocksize = SHA224_BLOCK_SIZE, 1413 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1414 .cra_init = safexcel_ahash_cra_init, 1415 .cra_exit = safexcel_ahash_cra_exit, 1416 .cra_module = THIS_MODULE, 1417 }, 1418 }, 1419 }, 1420 }; 1421 1422 static int safexcel_hmac_sha256_setkey(struct crypto_ahash *tfm, const u8 *key, 1423 unsigned int keylen) 1424 { 1425 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha256", 1426 SHA256_DIGEST_SIZE); 1427 } 1428 1429 static int safexcel_hmac_sha256_init(struct ahash_request *areq) 1430 { 1431 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1432 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1433 1434 memset(req, 0, sizeof(*req)); 1435 1436 /* Start from ipad precompute */ 1437 memcpy(req->state, &ctx->base.ipad, SHA256_DIGEST_SIZE); 1438 /* Already processed the key^ipad part now! */ 1439 req->len = SHA256_BLOCK_SIZE; 1440 req->processed = SHA256_BLOCK_SIZE; 1441 1442 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256; 1443 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1444 req->state_sz = SHA256_DIGEST_SIZE; 1445 req->digest_sz = SHA256_DIGEST_SIZE; 1446 req->block_sz = SHA256_BLOCK_SIZE; 1447 req->hmac = true; 1448 1449 return 0; 1450 } 1451 1452 static int safexcel_hmac_sha256_digest(struct ahash_request *areq) 1453 { 1454 int ret = safexcel_hmac_sha256_init(areq); 1455 1456 if (ret) 1457 return ret; 1458 1459 return safexcel_ahash_finup(areq); 1460 } 1461 1462 struct safexcel_alg_template safexcel_alg_hmac_sha256 = { 1463 .type = SAFEXCEL_ALG_TYPE_AHASH, 1464 .algo_mask = SAFEXCEL_ALG_SHA2_256, 1465 .alg.ahash = { 1466 .init = safexcel_hmac_sha256_init, 1467 .update = safexcel_ahash_update, 1468 .final = safexcel_ahash_final, 1469 .finup = safexcel_ahash_finup, 1470 .digest = safexcel_hmac_sha256_digest, 1471 .setkey = safexcel_hmac_sha256_setkey, 1472 .export = safexcel_ahash_export, 1473 .import = safexcel_ahash_import, 1474 .halg = { 1475 .digestsize = SHA256_DIGEST_SIZE, 1476 .statesize = sizeof(struct safexcel_ahash_export_state), 1477 .base = { 1478 .cra_name = "hmac(sha256)", 1479 .cra_driver_name = "safexcel-hmac-sha256", 1480 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1481 .cra_flags = CRYPTO_ALG_ASYNC | 1482 CRYPTO_ALG_ALLOCATES_MEMORY | 1483 CRYPTO_ALG_KERN_DRIVER_ONLY, 1484 .cra_blocksize = SHA256_BLOCK_SIZE, 1485 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1486 .cra_init = safexcel_ahash_cra_init, 1487 .cra_exit = safexcel_ahash_cra_exit, 1488 .cra_module = THIS_MODULE, 1489 }, 1490 }, 1491 }, 1492 }; 1493 1494 static int safexcel_sha512_init(struct ahash_request *areq) 1495 { 1496 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1497 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1498 1499 memset(req, 0, sizeof(*req)); 1500 1501 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512; 1502 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1503 req->state_sz = SHA512_DIGEST_SIZE; 1504 req->digest_sz = SHA512_DIGEST_SIZE; 1505 req->block_sz = SHA512_BLOCK_SIZE; 1506 1507 return 0; 1508 } 1509 1510 static int safexcel_sha512_digest(struct ahash_request *areq) 1511 { 1512 int ret = safexcel_sha512_init(areq); 1513 1514 if (ret) 1515 return ret; 1516 1517 return safexcel_ahash_finup(areq); 1518 } 1519 1520 struct safexcel_alg_template safexcel_alg_sha512 = { 1521 .type = SAFEXCEL_ALG_TYPE_AHASH, 1522 .algo_mask = SAFEXCEL_ALG_SHA2_512, 1523 .alg.ahash = { 1524 .init = safexcel_sha512_init, 1525 .update = safexcel_ahash_update, 1526 .final = safexcel_ahash_final, 1527 .finup = safexcel_ahash_finup, 1528 .digest = safexcel_sha512_digest, 1529 .export = safexcel_ahash_export, 1530 .import = safexcel_ahash_import, 1531 .halg = { 1532 .digestsize = SHA512_DIGEST_SIZE, 1533 .statesize = sizeof(struct safexcel_ahash_export_state), 1534 .base = { 1535 .cra_name = "sha512", 1536 .cra_driver_name = "safexcel-sha512", 1537 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1538 .cra_flags = CRYPTO_ALG_ASYNC | 1539 CRYPTO_ALG_ALLOCATES_MEMORY | 1540 CRYPTO_ALG_KERN_DRIVER_ONLY, 1541 .cra_blocksize = SHA512_BLOCK_SIZE, 1542 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1543 .cra_init = safexcel_ahash_cra_init, 1544 .cra_exit = safexcel_ahash_cra_exit, 1545 .cra_module = THIS_MODULE, 1546 }, 1547 }, 1548 }, 1549 }; 1550 1551 static int safexcel_sha384_init(struct ahash_request *areq) 1552 { 1553 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1554 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1555 1556 memset(req, 0, sizeof(*req)); 1557 1558 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384; 1559 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1560 req->state_sz = SHA512_DIGEST_SIZE; 1561 req->digest_sz = SHA512_DIGEST_SIZE; 1562 req->block_sz = SHA512_BLOCK_SIZE; 1563 1564 return 0; 1565 } 1566 1567 static int safexcel_sha384_digest(struct ahash_request *areq) 1568 { 1569 int ret = safexcel_sha384_init(areq); 1570 1571 if (ret) 1572 return ret; 1573 1574 return safexcel_ahash_finup(areq); 1575 } 1576 1577 struct safexcel_alg_template safexcel_alg_sha384 = { 1578 .type = SAFEXCEL_ALG_TYPE_AHASH, 1579 .algo_mask = SAFEXCEL_ALG_SHA2_512, 1580 .alg.ahash = { 1581 .init = safexcel_sha384_init, 1582 .update = safexcel_ahash_update, 1583 .final = safexcel_ahash_final, 1584 .finup = safexcel_ahash_finup, 1585 .digest = safexcel_sha384_digest, 1586 .export = safexcel_ahash_export, 1587 .import = safexcel_ahash_import, 1588 .halg = { 1589 .digestsize = SHA384_DIGEST_SIZE, 1590 .statesize = sizeof(struct safexcel_ahash_export_state), 1591 .base = { 1592 .cra_name = "sha384", 1593 .cra_driver_name = "safexcel-sha384", 1594 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1595 .cra_flags = CRYPTO_ALG_ASYNC | 1596 CRYPTO_ALG_ALLOCATES_MEMORY | 1597 CRYPTO_ALG_KERN_DRIVER_ONLY, 1598 .cra_blocksize = SHA384_BLOCK_SIZE, 1599 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1600 .cra_init = safexcel_ahash_cra_init, 1601 .cra_exit = safexcel_ahash_cra_exit, 1602 .cra_module = THIS_MODULE, 1603 }, 1604 }, 1605 }, 1606 }; 1607 1608 static int safexcel_hmac_sha512_setkey(struct crypto_ahash *tfm, const u8 *key, 1609 unsigned int keylen) 1610 { 1611 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha512", 1612 SHA512_DIGEST_SIZE); 1613 } 1614 1615 static int safexcel_hmac_sha512_init(struct ahash_request *areq) 1616 { 1617 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1618 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1619 1620 memset(req, 0, sizeof(*req)); 1621 1622 /* Start from ipad precompute */ 1623 memcpy(req->state, &ctx->base.ipad, SHA512_DIGEST_SIZE); 1624 /* Already processed the key^ipad part now! */ 1625 req->len = SHA512_BLOCK_SIZE; 1626 req->processed = SHA512_BLOCK_SIZE; 1627 1628 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512; 1629 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1630 req->state_sz = SHA512_DIGEST_SIZE; 1631 req->digest_sz = SHA512_DIGEST_SIZE; 1632 req->block_sz = SHA512_BLOCK_SIZE; 1633 req->hmac = true; 1634 1635 return 0; 1636 } 1637 1638 static int safexcel_hmac_sha512_digest(struct ahash_request *areq) 1639 { 1640 int ret = safexcel_hmac_sha512_init(areq); 1641 1642 if (ret) 1643 return ret; 1644 1645 return safexcel_ahash_finup(areq); 1646 } 1647 1648 struct safexcel_alg_template safexcel_alg_hmac_sha512 = { 1649 .type = SAFEXCEL_ALG_TYPE_AHASH, 1650 .algo_mask = SAFEXCEL_ALG_SHA2_512, 1651 .alg.ahash = { 1652 .init = safexcel_hmac_sha512_init, 1653 .update = safexcel_ahash_update, 1654 .final = safexcel_ahash_final, 1655 .finup = safexcel_ahash_finup, 1656 .digest = safexcel_hmac_sha512_digest, 1657 .setkey = safexcel_hmac_sha512_setkey, 1658 .export = safexcel_ahash_export, 1659 .import = safexcel_ahash_import, 1660 .halg = { 1661 .digestsize = SHA512_DIGEST_SIZE, 1662 .statesize = sizeof(struct safexcel_ahash_export_state), 1663 .base = { 1664 .cra_name = "hmac(sha512)", 1665 .cra_driver_name = "safexcel-hmac-sha512", 1666 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1667 .cra_flags = CRYPTO_ALG_ASYNC | 1668 CRYPTO_ALG_ALLOCATES_MEMORY | 1669 CRYPTO_ALG_KERN_DRIVER_ONLY, 1670 .cra_blocksize = SHA512_BLOCK_SIZE, 1671 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1672 .cra_init = safexcel_ahash_cra_init, 1673 .cra_exit = safexcel_ahash_cra_exit, 1674 .cra_module = THIS_MODULE, 1675 }, 1676 }, 1677 }, 1678 }; 1679 1680 static int safexcel_hmac_sha384_setkey(struct crypto_ahash *tfm, const u8 *key, 1681 unsigned int keylen) 1682 { 1683 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha384", 1684 SHA512_DIGEST_SIZE); 1685 } 1686 1687 static int safexcel_hmac_sha384_init(struct ahash_request *areq) 1688 { 1689 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1690 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1691 1692 memset(req, 0, sizeof(*req)); 1693 1694 /* Start from ipad precompute */ 1695 memcpy(req->state, &ctx->base.ipad, SHA512_DIGEST_SIZE); 1696 /* Already processed the key^ipad part now! */ 1697 req->len = SHA512_BLOCK_SIZE; 1698 req->processed = SHA512_BLOCK_SIZE; 1699 1700 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384; 1701 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1702 req->state_sz = SHA512_DIGEST_SIZE; 1703 req->digest_sz = SHA512_DIGEST_SIZE; 1704 req->block_sz = SHA512_BLOCK_SIZE; 1705 req->hmac = true; 1706 1707 return 0; 1708 } 1709 1710 static int safexcel_hmac_sha384_digest(struct ahash_request *areq) 1711 { 1712 int ret = safexcel_hmac_sha384_init(areq); 1713 1714 if (ret) 1715 return ret; 1716 1717 return safexcel_ahash_finup(areq); 1718 } 1719 1720 struct safexcel_alg_template safexcel_alg_hmac_sha384 = { 1721 .type = SAFEXCEL_ALG_TYPE_AHASH, 1722 .algo_mask = SAFEXCEL_ALG_SHA2_512, 1723 .alg.ahash = { 1724 .init = safexcel_hmac_sha384_init, 1725 .update = safexcel_ahash_update, 1726 .final = safexcel_ahash_final, 1727 .finup = safexcel_ahash_finup, 1728 .digest = safexcel_hmac_sha384_digest, 1729 .setkey = safexcel_hmac_sha384_setkey, 1730 .export = safexcel_ahash_export, 1731 .import = safexcel_ahash_import, 1732 .halg = { 1733 .digestsize = SHA384_DIGEST_SIZE, 1734 .statesize = sizeof(struct safexcel_ahash_export_state), 1735 .base = { 1736 .cra_name = "hmac(sha384)", 1737 .cra_driver_name = "safexcel-hmac-sha384", 1738 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1739 .cra_flags = CRYPTO_ALG_ASYNC | 1740 CRYPTO_ALG_ALLOCATES_MEMORY | 1741 CRYPTO_ALG_KERN_DRIVER_ONLY, 1742 .cra_blocksize = SHA384_BLOCK_SIZE, 1743 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1744 .cra_init = safexcel_ahash_cra_init, 1745 .cra_exit = safexcel_ahash_cra_exit, 1746 .cra_module = THIS_MODULE, 1747 }, 1748 }, 1749 }, 1750 }; 1751 1752 static int safexcel_md5_init(struct ahash_request *areq) 1753 { 1754 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1755 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1756 1757 memset(req, 0, sizeof(*req)); 1758 1759 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5; 1760 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1761 req->state_sz = MD5_DIGEST_SIZE; 1762 req->digest_sz = MD5_DIGEST_SIZE; 1763 req->block_sz = MD5_HMAC_BLOCK_SIZE; 1764 1765 return 0; 1766 } 1767 1768 static int safexcel_md5_digest(struct ahash_request *areq) 1769 { 1770 int ret = safexcel_md5_init(areq); 1771 1772 if (ret) 1773 return ret; 1774 1775 return safexcel_ahash_finup(areq); 1776 } 1777 1778 struct safexcel_alg_template safexcel_alg_md5 = { 1779 .type = SAFEXCEL_ALG_TYPE_AHASH, 1780 .algo_mask = SAFEXCEL_ALG_MD5, 1781 .alg.ahash = { 1782 .init = safexcel_md5_init, 1783 .update = safexcel_ahash_update, 1784 .final = safexcel_ahash_final, 1785 .finup = safexcel_ahash_finup, 1786 .digest = safexcel_md5_digest, 1787 .export = safexcel_ahash_export, 1788 .import = safexcel_ahash_import, 1789 .halg = { 1790 .digestsize = MD5_DIGEST_SIZE, 1791 .statesize = sizeof(struct safexcel_ahash_export_state), 1792 .base = { 1793 .cra_name = "md5", 1794 .cra_driver_name = "safexcel-md5", 1795 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1796 .cra_flags = CRYPTO_ALG_ASYNC | 1797 CRYPTO_ALG_ALLOCATES_MEMORY | 1798 CRYPTO_ALG_KERN_DRIVER_ONLY, 1799 .cra_blocksize = MD5_HMAC_BLOCK_SIZE, 1800 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1801 .cra_init = safexcel_ahash_cra_init, 1802 .cra_exit = safexcel_ahash_cra_exit, 1803 .cra_module = THIS_MODULE, 1804 }, 1805 }, 1806 }, 1807 }; 1808 1809 static int safexcel_hmac_md5_init(struct ahash_request *areq) 1810 { 1811 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1812 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1813 1814 memset(req, 0, sizeof(*req)); 1815 1816 /* Start from ipad precompute */ 1817 memcpy(req->state, &ctx->base.ipad, MD5_DIGEST_SIZE); 1818 /* Already processed the key^ipad part now! */ 1819 req->len = MD5_HMAC_BLOCK_SIZE; 1820 req->processed = MD5_HMAC_BLOCK_SIZE; 1821 1822 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5; 1823 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1824 req->state_sz = MD5_DIGEST_SIZE; 1825 req->digest_sz = MD5_DIGEST_SIZE; 1826 req->block_sz = MD5_HMAC_BLOCK_SIZE; 1827 req->len_is_le = true; /* MD5 is little endian! ... */ 1828 req->hmac = true; 1829 1830 return 0; 1831 } 1832 1833 static int safexcel_hmac_md5_setkey(struct crypto_ahash *tfm, const u8 *key, 1834 unsigned int keylen) 1835 { 1836 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-md5", 1837 MD5_DIGEST_SIZE); 1838 } 1839 1840 static int safexcel_hmac_md5_digest(struct ahash_request *areq) 1841 { 1842 int ret = safexcel_hmac_md5_init(areq); 1843 1844 if (ret) 1845 return ret; 1846 1847 return safexcel_ahash_finup(areq); 1848 } 1849 1850 struct safexcel_alg_template safexcel_alg_hmac_md5 = { 1851 .type = SAFEXCEL_ALG_TYPE_AHASH, 1852 .algo_mask = SAFEXCEL_ALG_MD5, 1853 .alg.ahash = { 1854 .init = safexcel_hmac_md5_init, 1855 .update = safexcel_ahash_update, 1856 .final = safexcel_ahash_final, 1857 .finup = safexcel_ahash_finup, 1858 .digest = safexcel_hmac_md5_digest, 1859 .setkey = safexcel_hmac_md5_setkey, 1860 .export = safexcel_ahash_export, 1861 .import = safexcel_ahash_import, 1862 .halg = { 1863 .digestsize = MD5_DIGEST_SIZE, 1864 .statesize = sizeof(struct safexcel_ahash_export_state), 1865 .base = { 1866 .cra_name = "hmac(md5)", 1867 .cra_driver_name = "safexcel-hmac-md5", 1868 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1869 .cra_flags = CRYPTO_ALG_ASYNC | 1870 CRYPTO_ALG_ALLOCATES_MEMORY | 1871 CRYPTO_ALG_KERN_DRIVER_ONLY, 1872 .cra_blocksize = MD5_HMAC_BLOCK_SIZE, 1873 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1874 .cra_init = safexcel_ahash_cra_init, 1875 .cra_exit = safexcel_ahash_cra_exit, 1876 .cra_module = THIS_MODULE, 1877 }, 1878 }, 1879 }, 1880 }; 1881 1882 static int safexcel_cbcmac_init(struct ahash_request *areq) 1883 { 1884 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1885 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1886 1887 memset(req, 0, sizeof(*req)); 1888 1889 /* Start from loaded keys */ 1890 memcpy(req->state, &ctx->base.ipad, ctx->key_sz); 1891 /* Set processed to non-zero to enable invalidation detection */ 1892 req->len = AES_BLOCK_SIZE; 1893 req->processed = AES_BLOCK_SIZE; 1894 1895 req->digest = CONTEXT_CONTROL_DIGEST_XCM; 1896 req->state_sz = ctx->key_sz; 1897 req->digest_sz = AES_BLOCK_SIZE; 1898 req->block_sz = AES_BLOCK_SIZE; 1899 req->xcbcmac = true; 1900 1901 return 0; 1902 } 1903 1904 static int safexcel_cbcmac_setkey(struct crypto_ahash *tfm, const u8 *key, 1905 unsigned int len) 1906 { 1907 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 1908 struct crypto_aes_ctx aes; 1909 int ret, i; 1910 1911 ret = aes_expandkey(&aes, key, len); 1912 if (ret) 1913 return ret; 1914 1915 memset(&ctx->base.ipad, 0, 2 * AES_BLOCK_SIZE); 1916 for (i = 0; i < len / sizeof(u32); i++) 1917 ctx->base.ipad.be[i + 8] = cpu_to_be32(aes.key_enc[i]); 1918 1919 if (len == AES_KEYSIZE_192) { 1920 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192; 1921 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE; 1922 } else if (len == AES_KEYSIZE_256) { 1923 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256; 1924 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE; 1925 } else { 1926 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128; 1927 ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE; 1928 } 1929 ctx->cbcmac = true; 1930 1931 memzero_explicit(&aes, sizeof(aes)); 1932 return 0; 1933 } 1934 1935 static int safexcel_cbcmac_digest(struct ahash_request *areq) 1936 { 1937 return safexcel_cbcmac_init(areq) ?: safexcel_ahash_finup(areq); 1938 } 1939 1940 struct safexcel_alg_template safexcel_alg_cbcmac = { 1941 .type = SAFEXCEL_ALG_TYPE_AHASH, 1942 .algo_mask = 0, 1943 .alg.ahash = { 1944 .init = safexcel_cbcmac_init, 1945 .update = safexcel_ahash_update, 1946 .final = safexcel_ahash_final, 1947 .finup = safexcel_ahash_finup, 1948 .digest = safexcel_cbcmac_digest, 1949 .setkey = safexcel_cbcmac_setkey, 1950 .export = safexcel_ahash_export, 1951 .import = safexcel_ahash_import, 1952 .halg = { 1953 .digestsize = AES_BLOCK_SIZE, 1954 .statesize = sizeof(struct safexcel_ahash_export_state), 1955 .base = { 1956 .cra_name = "cbcmac(aes)", 1957 .cra_driver_name = "safexcel-cbcmac-aes", 1958 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1959 .cra_flags = CRYPTO_ALG_ASYNC | 1960 CRYPTO_ALG_ALLOCATES_MEMORY | 1961 CRYPTO_ALG_KERN_DRIVER_ONLY, 1962 .cra_blocksize = AES_BLOCK_SIZE, 1963 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1964 .cra_init = safexcel_ahash_cra_init, 1965 .cra_exit = safexcel_ahash_cra_exit, 1966 .cra_module = THIS_MODULE, 1967 }, 1968 }, 1969 }, 1970 }; 1971 1972 static int safexcel_xcbcmac_setkey(struct crypto_ahash *tfm, const u8 *key, 1973 unsigned int len) 1974 { 1975 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 1976 u32 key_tmp[3 * AES_BLOCK_SIZE / sizeof(u32)]; 1977 int ret, i; 1978 1979 ret = aes_expandkey(ctx->aes, key, len); 1980 if (ret) 1981 return ret; 1982 1983 /* precompute the XCBC key material */ 1984 aes_encrypt(ctx->aes, (u8 *)key_tmp + 2 * AES_BLOCK_SIZE, 1985 "\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1"); 1986 aes_encrypt(ctx->aes, (u8 *)key_tmp, 1987 "\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2"); 1988 aes_encrypt(ctx->aes, (u8 *)key_tmp + AES_BLOCK_SIZE, 1989 "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3"); 1990 for (i = 0; i < 3 * AES_BLOCK_SIZE / sizeof(u32); i++) 1991 ctx->base.ipad.word[i] = swab32(key_tmp[i]); 1992 1993 ret = aes_expandkey(ctx->aes, 1994 (u8 *)key_tmp + 2 * AES_BLOCK_SIZE, 1995 AES_MIN_KEY_SIZE); 1996 if (ret) 1997 return ret; 1998 1999 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128; 2000 ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE; 2001 ctx->cbcmac = false; 2002 2003 return 0; 2004 } 2005 2006 static int safexcel_xcbcmac_cra_init(struct crypto_tfm *tfm) 2007 { 2008 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 2009 2010 safexcel_ahash_cra_init(tfm); 2011 ctx->aes = kmalloc(sizeof(*ctx->aes), GFP_KERNEL); 2012 return ctx->aes == NULL ? -ENOMEM : 0; 2013 } 2014 2015 static void safexcel_xcbcmac_cra_exit(struct crypto_tfm *tfm) 2016 { 2017 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 2018 2019 kfree(ctx->aes); 2020 safexcel_ahash_cra_exit(tfm); 2021 } 2022 2023 struct safexcel_alg_template safexcel_alg_xcbcmac = { 2024 .type = SAFEXCEL_ALG_TYPE_AHASH, 2025 .algo_mask = 0, 2026 .alg.ahash = { 2027 .init = safexcel_cbcmac_init, 2028 .update = safexcel_ahash_update, 2029 .final = safexcel_ahash_final, 2030 .finup = safexcel_ahash_finup, 2031 .digest = safexcel_cbcmac_digest, 2032 .setkey = safexcel_xcbcmac_setkey, 2033 .export = safexcel_ahash_export, 2034 .import = safexcel_ahash_import, 2035 .halg = { 2036 .digestsize = AES_BLOCK_SIZE, 2037 .statesize = sizeof(struct safexcel_ahash_export_state), 2038 .base = { 2039 .cra_name = "xcbc(aes)", 2040 .cra_driver_name = "safexcel-xcbc-aes", 2041 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2042 .cra_flags = CRYPTO_ALG_ASYNC | 2043 CRYPTO_ALG_ALLOCATES_MEMORY | 2044 CRYPTO_ALG_KERN_DRIVER_ONLY, 2045 .cra_blocksize = AES_BLOCK_SIZE, 2046 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 2047 .cra_init = safexcel_xcbcmac_cra_init, 2048 .cra_exit = safexcel_xcbcmac_cra_exit, 2049 .cra_module = THIS_MODULE, 2050 }, 2051 }, 2052 }, 2053 }; 2054 2055 static int safexcel_cmac_setkey(struct crypto_ahash *tfm, const u8 *key, 2056 unsigned int len) 2057 { 2058 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 2059 __be64 consts[4]; 2060 u64 _const[2]; 2061 u8 msb_mask, gfmask; 2062 int ret, i; 2063 2064 /* precompute the CMAC key material */ 2065 ret = aes_expandkey(ctx->aes, key, len); 2066 if (ret) 2067 return ret; 2068 2069 for (i = 0; i < len / sizeof(u32); i++) 2070 ctx->base.ipad.word[i + 8] = swab32(ctx->aes->key_enc[i]); 2071 2072 /* code below borrowed from crypto/cmac.c */ 2073 /* encrypt the zero block */ 2074 memset(consts, 0, AES_BLOCK_SIZE); 2075 aes_encrypt(ctx->aes, (u8 *)consts, (u8 *)consts); 2076 2077 gfmask = 0x87; 2078 _const[0] = be64_to_cpu(consts[1]); 2079 _const[1] = be64_to_cpu(consts[0]); 2080 2081 /* gf(2^128) multiply zero-ciphertext with u and u^2 */ 2082 for (i = 0; i < 4; i += 2) { 2083 msb_mask = ((s64)_const[1] >> 63) & gfmask; 2084 _const[1] = (_const[1] << 1) | (_const[0] >> 63); 2085 _const[0] = (_const[0] << 1) ^ msb_mask; 2086 2087 consts[i + 0] = cpu_to_be64(_const[1]); 2088 consts[i + 1] = cpu_to_be64(_const[0]); 2089 } 2090 /* end of code borrowed from crypto/cmac.c */ 2091 2092 for (i = 0; i < 2 * AES_BLOCK_SIZE / sizeof(u32); i++) 2093 ctx->base.ipad.be[i] = cpu_to_be32(((u32 *)consts)[i]); 2094 2095 if (len == AES_KEYSIZE_192) { 2096 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192; 2097 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE; 2098 } else if (len == AES_KEYSIZE_256) { 2099 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256; 2100 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE; 2101 } else { 2102 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128; 2103 ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE; 2104 } 2105 ctx->cbcmac = false; 2106 2107 return 0; 2108 } 2109 2110 struct safexcel_alg_template safexcel_alg_cmac = { 2111 .type = SAFEXCEL_ALG_TYPE_AHASH, 2112 .algo_mask = 0, 2113 .alg.ahash = { 2114 .init = safexcel_cbcmac_init, 2115 .update = safexcel_ahash_update, 2116 .final = safexcel_ahash_final, 2117 .finup = safexcel_ahash_finup, 2118 .digest = safexcel_cbcmac_digest, 2119 .setkey = safexcel_cmac_setkey, 2120 .export = safexcel_ahash_export, 2121 .import = safexcel_ahash_import, 2122 .halg = { 2123 .digestsize = AES_BLOCK_SIZE, 2124 .statesize = sizeof(struct safexcel_ahash_export_state), 2125 .base = { 2126 .cra_name = "cmac(aes)", 2127 .cra_driver_name = "safexcel-cmac-aes", 2128 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2129 .cra_flags = CRYPTO_ALG_ASYNC | 2130 CRYPTO_ALG_ALLOCATES_MEMORY | 2131 CRYPTO_ALG_KERN_DRIVER_ONLY, 2132 .cra_blocksize = AES_BLOCK_SIZE, 2133 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 2134 .cra_init = safexcel_xcbcmac_cra_init, 2135 .cra_exit = safexcel_xcbcmac_cra_exit, 2136 .cra_module = THIS_MODULE, 2137 }, 2138 }, 2139 }, 2140 }; 2141 2142 static int safexcel_sm3_init(struct ahash_request *areq) 2143 { 2144 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 2145 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 2146 2147 memset(req, 0, sizeof(*req)); 2148 2149 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3; 2150 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 2151 req->state_sz = SM3_DIGEST_SIZE; 2152 req->digest_sz = SM3_DIGEST_SIZE; 2153 req->block_sz = SM3_BLOCK_SIZE; 2154 2155 return 0; 2156 } 2157 2158 static int safexcel_sm3_digest(struct ahash_request *areq) 2159 { 2160 int ret = safexcel_sm3_init(areq); 2161 2162 if (ret) 2163 return ret; 2164 2165 return safexcel_ahash_finup(areq); 2166 } 2167 2168 struct safexcel_alg_template safexcel_alg_sm3 = { 2169 .type = SAFEXCEL_ALG_TYPE_AHASH, 2170 .algo_mask = SAFEXCEL_ALG_SM3, 2171 .alg.ahash = { 2172 .init = safexcel_sm3_init, 2173 .update = safexcel_ahash_update, 2174 .final = safexcel_ahash_final, 2175 .finup = safexcel_ahash_finup, 2176 .digest = safexcel_sm3_digest, 2177 .export = safexcel_ahash_export, 2178 .import = safexcel_ahash_import, 2179 .halg = { 2180 .digestsize = SM3_DIGEST_SIZE, 2181 .statesize = sizeof(struct safexcel_ahash_export_state), 2182 .base = { 2183 .cra_name = "sm3", 2184 .cra_driver_name = "safexcel-sm3", 2185 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2186 .cra_flags = CRYPTO_ALG_ASYNC | 2187 CRYPTO_ALG_ALLOCATES_MEMORY | 2188 CRYPTO_ALG_KERN_DRIVER_ONLY, 2189 .cra_blocksize = SM3_BLOCK_SIZE, 2190 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 2191 .cra_init = safexcel_ahash_cra_init, 2192 .cra_exit = safexcel_ahash_cra_exit, 2193 .cra_module = THIS_MODULE, 2194 }, 2195 }, 2196 }, 2197 }; 2198 2199 static int safexcel_hmac_sm3_setkey(struct crypto_ahash *tfm, const u8 *key, 2200 unsigned int keylen) 2201 { 2202 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sm3", 2203 SM3_DIGEST_SIZE); 2204 } 2205 2206 static int safexcel_hmac_sm3_init(struct ahash_request *areq) 2207 { 2208 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 2209 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 2210 2211 memset(req, 0, sizeof(*req)); 2212 2213 /* Start from ipad precompute */ 2214 memcpy(req->state, &ctx->base.ipad, SM3_DIGEST_SIZE); 2215 /* Already processed the key^ipad part now! */ 2216 req->len = SM3_BLOCK_SIZE; 2217 req->processed = SM3_BLOCK_SIZE; 2218 2219 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3; 2220 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 2221 req->state_sz = SM3_DIGEST_SIZE; 2222 req->digest_sz = SM3_DIGEST_SIZE; 2223 req->block_sz = SM3_BLOCK_SIZE; 2224 req->hmac = true; 2225 2226 return 0; 2227 } 2228 2229 static int safexcel_hmac_sm3_digest(struct ahash_request *areq) 2230 { 2231 int ret = safexcel_hmac_sm3_init(areq); 2232 2233 if (ret) 2234 return ret; 2235 2236 return safexcel_ahash_finup(areq); 2237 } 2238 2239 struct safexcel_alg_template safexcel_alg_hmac_sm3 = { 2240 .type = SAFEXCEL_ALG_TYPE_AHASH, 2241 .algo_mask = SAFEXCEL_ALG_SM3, 2242 .alg.ahash = { 2243 .init = safexcel_hmac_sm3_init, 2244 .update = safexcel_ahash_update, 2245 .final = safexcel_ahash_final, 2246 .finup = safexcel_ahash_finup, 2247 .digest = safexcel_hmac_sm3_digest, 2248 .setkey = safexcel_hmac_sm3_setkey, 2249 .export = safexcel_ahash_export, 2250 .import = safexcel_ahash_import, 2251 .halg = { 2252 .digestsize = SM3_DIGEST_SIZE, 2253 .statesize = sizeof(struct safexcel_ahash_export_state), 2254 .base = { 2255 .cra_name = "hmac(sm3)", 2256 .cra_driver_name = "safexcel-hmac-sm3", 2257 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2258 .cra_flags = CRYPTO_ALG_ASYNC | 2259 CRYPTO_ALG_ALLOCATES_MEMORY | 2260 CRYPTO_ALG_KERN_DRIVER_ONLY, 2261 .cra_blocksize = SM3_BLOCK_SIZE, 2262 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 2263 .cra_init = safexcel_ahash_cra_init, 2264 .cra_exit = safexcel_ahash_cra_exit, 2265 .cra_module = THIS_MODULE, 2266 }, 2267 }, 2268 }, 2269 }; 2270 2271 static int safexcel_sha3_224_init(struct ahash_request *areq) 2272 { 2273 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2274 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2275 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 2276 2277 memset(req, 0, sizeof(*req)); 2278 2279 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_224; 2280 req->digest = CONTEXT_CONTROL_DIGEST_INITIAL; 2281 req->state_sz = SHA3_224_DIGEST_SIZE; 2282 req->digest_sz = SHA3_224_DIGEST_SIZE; 2283 req->block_sz = SHA3_224_BLOCK_SIZE; 2284 ctx->do_fallback = false; 2285 ctx->fb_init_done = false; 2286 return 0; 2287 } 2288 2289 static int safexcel_sha3_fbcheck(struct ahash_request *req) 2290 { 2291 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 2292 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2293 struct ahash_request *subreq = ahash_request_ctx_dma(req); 2294 int ret = 0; 2295 2296 if (ctx->do_fallback) { 2297 ahash_request_set_tfm(subreq, ctx->fback); 2298 ahash_request_set_callback(subreq, req->base.flags, 2299 req->base.complete, req->base.data); 2300 ahash_request_set_crypt(subreq, req->src, req->result, 2301 req->nbytes); 2302 if (!ctx->fb_init_done) { 2303 if (ctx->fb_do_setkey) { 2304 /* Set fallback cipher HMAC key */ 2305 u8 key[SHA3_224_BLOCK_SIZE]; 2306 2307 memcpy(key, &ctx->base.ipad, 2308 crypto_ahash_blocksize(ctx->fback) / 2); 2309 memcpy(key + 2310 crypto_ahash_blocksize(ctx->fback) / 2, 2311 &ctx->base.opad, 2312 crypto_ahash_blocksize(ctx->fback) / 2); 2313 ret = crypto_ahash_setkey(ctx->fback, key, 2314 crypto_ahash_blocksize(ctx->fback)); 2315 memzero_explicit(key, 2316 crypto_ahash_blocksize(ctx->fback)); 2317 ctx->fb_do_setkey = false; 2318 } 2319 ret = ret ?: crypto_ahash_init(subreq); 2320 ctx->fb_init_done = true; 2321 } 2322 } 2323 return ret; 2324 } 2325 2326 static int safexcel_sha3_update(struct ahash_request *req) 2327 { 2328 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 2329 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2330 struct ahash_request *subreq = ahash_request_ctx_dma(req); 2331 2332 ctx->do_fallback = true; 2333 return safexcel_sha3_fbcheck(req) ?: crypto_ahash_update(subreq); 2334 } 2335 2336 static int safexcel_sha3_final(struct ahash_request *req) 2337 { 2338 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 2339 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2340 struct ahash_request *subreq = ahash_request_ctx_dma(req); 2341 2342 ctx->do_fallback = true; 2343 return safexcel_sha3_fbcheck(req) ?: crypto_ahash_final(subreq); 2344 } 2345 2346 static int safexcel_sha3_finup(struct ahash_request *req) 2347 { 2348 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 2349 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2350 struct ahash_request *subreq = ahash_request_ctx_dma(req); 2351 2352 ctx->do_fallback |= !req->nbytes; 2353 if (ctx->do_fallback) 2354 /* Update or ex/import happened or len 0, cannot use the HW */ 2355 return safexcel_sha3_fbcheck(req) ?: 2356 crypto_ahash_finup(subreq); 2357 else 2358 return safexcel_ahash_finup(req); 2359 } 2360 2361 static int safexcel_sha3_digest_fallback(struct ahash_request *req) 2362 { 2363 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 2364 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2365 struct ahash_request *subreq = ahash_request_ctx_dma(req); 2366 2367 ctx->do_fallback = true; 2368 ctx->fb_init_done = false; 2369 return safexcel_sha3_fbcheck(req) ?: crypto_ahash_finup(subreq); 2370 } 2371 2372 static int safexcel_sha3_224_digest(struct ahash_request *req) 2373 { 2374 if (req->nbytes) 2375 return safexcel_sha3_224_init(req) ?: safexcel_ahash_finup(req); 2376 2377 /* HW cannot do zero length hash, use fallback instead */ 2378 return safexcel_sha3_digest_fallback(req); 2379 } 2380 2381 static int safexcel_sha3_export(struct ahash_request *req, void *out) 2382 { 2383 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 2384 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2385 struct ahash_request *subreq = ahash_request_ctx_dma(req); 2386 2387 ctx->do_fallback = true; 2388 return safexcel_sha3_fbcheck(req) ?: crypto_ahash_export(subreq, out); 2389 } 2390 2391 static int safexcel_sha3_import(struct ahash_request *req, const void *in) 2392 { 2393 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 2394 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2395 struct ahash_request *subreq = ahash_request_ctx_dma(req); 2396 2397 ctx->do_fallback = true; 2398 return safexcel_sha3_fbcheck(req) ?: crypto_ahash_import(subreq, in); 2399 // return safexcel_ahash_import(req, in); 2400 } 2401 2402 static int safexcel_sha3_cra_init(struct crypto_tfm *tfm) 2403 { 2404 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); 2405 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 2406 2407 safexcel_ahash_cra_init(tfm); 2408 2409 /* Allocate fallback implementation */ 2410 ctx->fback = crypto_alloc_ahash(crypto_tfm_alg_name(tfm), 0, 2411 CRYPTO_ALG_ASYNC | 2412 CRYPTO_ALG_NEED_FALLBACK); 2413 if (IS_ERR(ctx->fback)) 2414 return PTR_ERR(ctx->fback); 2415 2416 /* Update statesize from fallback algorithm! */ 2417 crypto_hash_alg_common(ahash)->statesize = 2418 crypto_ahash_statesize(ctx->fback); 2419 crypto_ahash_set_reqsize_dma( 2420 ahash, max(sizeof(struct safexcel_ahash_req), 2421 sizeof(struct ahash_request) + 2422 crypto_ahash_reqsize(ctx->fback))); 2423 return 0; 2424 } 2425 2426 static void safexcel_sha3_cra_exit(struct crypto_tfm *tfm) 2427 { 2428 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 2429 2430 crypto_free_ahash(ctx->fback); 2431 safexcel_ahash_cra_exit(tfm); 2432 } 2433 2434 struct safexcel_alg_template safexcel_alg_sha3_224 = { 2435 .type = SAFEXCEL_ALG_TYPE_AHASH, 2436 .algo_mask = SAFEXCEL_ALG_SHA3, 2437 .alg.ahash = { 2438 .init = safexcel_sha3_224_init, 2439 .update = safexcel_sha3_update, 2440 .final = safexcel_sha3_final, 2441 .finup = safexcel_sha3_finup, 2442 .digest = safexcel_sha3_224_digest, 2443 .export = safexcel_sha3_export, 2444 .import = safexcel_sha3_import, 2445 .halg = { 2446 .digestsize = SHA3_224_DIGEST_SIZE, 2447 .statesize = sizeof(struct safexcel_ahash_export_state), 2448 .base = { 2449 .cra_name = "sha3-224", 2450 .cra_driver_name = "safexcel-sha3-224", 2451 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2452 .cra_flags = CRYPTO_ALG_ASYNC | 2453 CRYPTO_ALG_KERN_DRIVER_ONLY | 2454 CRYPTO_ALG_NEED_FALLBACK, 2455 .cra_blocksize = SHA3_224_BLOCK_SIZE, 2456 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 2457 .cra_init = safexcel_sha3_cra_init, 2458 .cra_exit = safexcel_sha3_cra_exit, 2459 .cra_module = THIS_MODULE, 2460 }, 2461 }, 2462 }, 2463 }; 2464 2465 static int safexcel_sha3_256_init(struct ahash_request *areq) 2466 { 2467 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2468 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2469 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 2470 2471 memset(req, 0, sizeof(*req)); 2472 2473 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_256; 2474 req->digest = CONTEXT_CONTROL_DIGEST_INITIAL; 2475 req->state_sz = SHA3_256_DIGEST_SIZE; 2476 req->digest_sz = SHA3_256_DIGEST_SIZE; 2477 req->block_sz = SHA3_256_BLOCK_SIZE; 2478 ctx->do_fallback = false; 2479 ctx->fb_init_done = false; 2480 return 0; 2481 } 2482 2483 static int safexcel_sha3_256_digest(struct ahash_request *req) 2484 { 2485 if (req->nbytes) 2486 return safexcel_sha3_256_init(req) ?: safexcel_ahash_finup(req); 2487 2488 /* HW cannot do zero length hash, use fallback instead */ 2489 return safexcel_sha3_digest_fallback(req); 2490 } 2491 2492 struct safexcel_alg_template safexcel_alg_sha3_256 = { 2493 .type = SAFEXCEL_ALG_TYPE_AHASH, 2494 .algo_mask = SAFEXCEL_ALG_SHA3, 2495 .alg.ahash = { 2496 .init = safexcel_sha3_256_init, 2497 .update = safexcel_sha3_update, 2498 .final = safexcel_sha3_final, 2499 .finup = safexcel_sha3_finup, 2500 .digest = safexcel_sha3_256_digest, 2501 .export = safexcel_sha3_export, 2502 .import = safexcel_sha3_import, 2503 .halg = { 2504 .digestsize = SHA3_256_DIGEST_SIZE, 2505 .statesize = sizeof(struct safexcel_ahash_export_state), 2506 .base = { 2507 .cra_name = "sha3-256", 2508 .cra_driver_name = "safexcel-sha3-256", 2509 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2510 .cra_flags = CRYPTO_ALG_ASYNC | 2511 CRYPTO_ALG_KERN_DRIVER_ONLY | 2512 CRYPTO_ALG_NEED_FALLBACK, 2513 .cra_blocksize = SHA3_256_BLOCK_SIZE, 2514 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 2515 .cra_init = safexcel_sha3_cra_init, 2516 .cra_exit = safexcel_sha3_cra_exit, 2517 .cra_module = THIS_MODULE, 2518 }, 2519 }, 2520 }, 2521 }; 2522 2523 static int safexcel_sha3_384_init(struct ahash_request *areq) 2524 { 2525 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2526 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2527 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 2528 2529 memset(req, 0, sizeof(*req)); 2530 2531 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_384; 2532 req->digest = CONTEXT_CONTROL_DIGEST_INITIAL; 2533 req->state_sz = SHA3_384_DIGEST_SIZE; 2534 req->digest_sz = SHA3_384_DIGEST_SIZE; 2535 req->block_sz = SHA3_384_BLOCK_SIZE; 2536 ctx->do_fallback = false; 2537 ctx->fb_init_done = false; 2538 return 0; 2539 } 2540 2541 static int safexcel_sha3_384_digest(struct ahash_request *req) 2542 { 2543 if (req->nbytes) 2544 return safexcel_sha3_384_init(req) ?: safexcel_ahash_finup(req); 2545 2546 /* HW cannot do zero length hash, use fallback instead */ 2547 return safexcel_sha3_digest_fallback(req); 2548 } 2549 2550 struct safexcel_alg_template safexcel_alg_sha3_384 = { 2551 .type = SAFEXCEL_ALG_TYPE_AHASH, 2552 .algo_mask = SAFEXCEL_ALG_SHA3, 2553 .alg.ahash = { 2554 .init = safexcel_sha3_384_init, 2555 .update = safexcel_sha3_update, 2556 .final = safexcel_sha3_final, 2557 .finup = safexcel_sha3_finup, 2558 .digest = safexcel_sha3_384_digest, 2559 .export = safexcel_sha3_export, 2560 .import = safexcel_sha3_import, 2561 .halg = { 2562 .digestsize = SHA3_384_DIGEST_SIZE, 2563 .statesize = sizeof(struct safexcel_ahash_export_state), 2564 .base = { 2565 .cra_name = "sha3-384", 2566 .cra_driver_name = "safexcel-sha3-384", 2567 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2568 .cra_flags = CRYPTO_ALG_ASYNC | 2569 CRYPTO_ALG_KERN_DRIVER_ONLY | 2570 CRYPTO_ALG_NEED_FALLBACK, 2571 .cra_blocksize = SHA3_384_BLOCK_SIZE, 2572 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 2573 .cra_init = safexcel_sha3_cra_init, 2574 .cra_exit = safexcel_sha3_cra_exit, 2575 .cra_module = THIS_MODULE, 2576 }, 2577 }, 2578 }, 2579 }; 2580 2581 static int safexcel_sha3_512_init(struct ahash_request *areq) 2582 { 2583 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2584 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2585 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 2586 2587 memset(req, 0, sizeof(*req)); 2588 2589 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_512; 2590 req->digest = CONTEXT_CONTROL_DIGEST_INITIAL; 2591 req->state_sz = SHA3_512_DIGEST_SIZE; 2592 req->digest_sz = SHA3_512_DIGEST_SIZE; 2593 req->block_sz = SHA3_512_BLOCK_SIZE; 2594 ctx->do_fallback = false; 2595 ctx->fb_init_done = false; 2596 return 0; 2597 } 2598 2599 static int safexcel_sha3_512_digest(struct ahash_request *req) 2600 { 2601 if (req->nbytes) 2602 return safexcel_sha3_512_init(req) ?: safexcel_ahash_finup(req); 2603 2604 /* HW cannot do zero length hash, use fallback instead */ 2605 return safexcel_sha3_digest_fallback(req); 2606 } 2607 2608 struct safexcel_alg_template safexcel_alg_sha3_512 = { 2609 .type = SAFEXCEL_ALG_TYPE_AHASH, 2610 .algo_mask = SAFEXCEL_ALG_SHA3, 2611 .alg.ahash = { 2612 .init = safexcel_sha3_512_init, 2613 .update = safexcel_sha3_update, 2614 .final = safexcel_sha3_final, 2615 .finup = safexcel_sha3_finup, 2616 .digest = safexcel_sha3_512_digest, 2617 .export = safexcel_sha3_export, 2618 .import = safexcel_sha3_import, 2619 .halg = { 2620 .digestsize = SHA3_512_DIGEST_SIZE, 2621 .statesize = sizeof(struct safexcel_ahash_export_state), 2622 .base = { 2623 .cra_name = "sha3-512", 2624 .cra_driver_name = "safexcel-sha3-512", 2625 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2626 .cra_flags = CRYPTO_ALG_ASYNC | 2627 CRYPTO_ALG_KERN_DRIVER_ONLY | 2628 CRYPTO_ALG_NEED_FALLBACK, 2629 .cra_blocksize = SHA3_512_BLOCK_SIZE, 2630 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 2631 .cra_init = safexcel_sha3_cra_init, 2632 .cra_exit = safexcel_sha3_cra_exit, 2633 .cra_module = THIS_MODULE, 2634 }, 2635 }, 2636 }, 2637 }; 2638 2639 static int safexcel_hmac_sha3_cra_init(struct crypto_tfm *tfm, const char *alg) 2640 { 2641 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 2642 int ret; 2643 2644 ret = safexcel_sha3_cra_init(tfm); 2645 if (ret) 2646 return ret; 2647 2648 /* Allocate precalc basic digest implementation */ 2649 ctx->shpre = crypto_alloc_shash(alg, 0, CRYPTO_ALG_NEED_FALLBACK); 2650 if (IS_ERR(ctx->shpre)) 2651 return PTR_ERR(ctx->shpre); 2652 2653 ctx->shdesc = kmalloc(sizeof(*ctx->shdesc) + 2654 crypto_shash_descsize(ctx->shpre), GFP_KERNEL); 2655 if (!ctx->shdesc) { 2656 crypto_free_shash(ctx->shpre); 2657 return -ENOMEM; 2658 } 2659 ctx->shdesc->tfm = ctx->shpre; 2660 return 0; 2661 } 2662 2663 static void safexcel_hmac_sha3_cra_exit(struct crypto_tfm *tfm) 2664 { 2665 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 2666 2667 crypto_free_ahash(ctx->fback); 2668 crypto_free_shash(ctx->shpre); 2669 kfree(ctx->shdesc); 2670 safexcel_ahash_cra_exit(tfm); 2671 } 2672 2673 static int safexcel_hmac_sha3_setkey(struct crypto_ahash *tfm, const u8 *key, 2674 unsigned int keylen) 2675 { 2676 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2677 int ret = 0; 2678 2679 if (keylen > crypto_ahash_blocksize(tfm)) { 2680 /* 2681 * If the key is larger than the blocksize, then hash it 2682 * first using our fallback cipher 2683 */ 2684 ret = crypto_shash_digest(ctx->shdesc, key, keylen, 2685 ctx->base.ipad.byte); 2686 keylen = crypto_shash_digestsize(ctx->shpre); 2687 2688 /* 2689 * If the digest is larger than half the blocksize, we need to 2690 * move the rest to opad due to the way our HMAC infra works. 2691 */ 2692 if (keylen > crypto_ahash_blocksize(tfm) / 2) 2693 /* Buffers overlap, need to use memmove iso memcpy! */ 2694 memmove(&ctx->base.opad, 2695 ctx->base.ipad.byte + 2696 crypto_ahash_blocksize(tfm) / 2, 2697 keylen - crypto_ahash_blocksize(tfm) / 2); 2698 } else { 2699 /* 2700 * Copy the key to our ipad & opad buffers 2701 * Note that ipad and opad each contain one half of the key, 2702 * to match the existing HMAC driver infrastructure. 2703 */ 2704 if (keylen <= crypto_ahash_blocksize(tfm) / 2) { 2705 memcpy(&ctx->base.ipad, key, keylen); 2706 } else { 2707 memcpy(&ctx->base.ipad, key, 2708 crypto_ahash_blocksize(tfm) / 2); 2709 memcpy(&ctx->base.opad, 2710 key + crypto_ahash_blocksize(tfm) / 2, 2711 keylen - crypto_ahash_blocksize(tfm) / 2); 2712 } 2713 } 2714 2715 /* Pad key with zeroes */ 2716 if (keylen <= crypto_ahash_blocksize(tfm) / 2) { 2717 memset(ctx->base.ipad.byte + keylen, 0, 2718 crypto_ahash_blocksize(tfm) / 2 - keylen); 2719 memset(&ctx->base.opad, 0, crypto_ahash_blocksize(tfm) / 2); 2720 } else { 2721 memset(ctx->base.opad.byte + keylen - 2722 crypto_ahash_blocksize(tfm) / 2, 0, 2723 crypto_ahash_blocksize(tfm) - keylen); 2724 } 2725 2726 /* If doing fallback, still need to set the new key! */ 2727 ctx->fb_do_setkey = true; 2728 return ret; 2729 } 2730 2731 static int safexcel_hmac_sha3_224_init(struct ahash_request *areq) 2732 { 2733 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2734 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2735 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 2736 2737 memset(req, 0, sizeof(*req)); 2738 2739 /* Copy (half of) the key */ 2740 memcpy(req->state, &ctx->base.ipad, SHA3_224_BLOCK_SIZE / 2); 2741 /* Start of HMAC should have len == processed == blocksize */ 2742 req->len = SHA3_224_BLOCK_SIZE; 2743 req->processed = SHA3_224_BLOCK_SIZE; 2744 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_224; 2745 req->digest = CONTEXT_CONTROL_DIGEST_HMAC; 2746 req->state_sz = SHA3_224_BLOCK_SIZE / 2; 2747 req->digest_sz = SHA3_224_DIGEST_SIZE; 2748 req->block_sz = SHA3_224_BLOCK_SIZE; 2749 req->hmac = true; 2750 ctx->do_fallback = false; 2751 ctx->fb_init_done = false; 2752 return 0; 2753 } 2754 2755 static int safexcel_hmac_sha3_224_digest(struct ahash_request *req) 2756 { 2757 if (req->nbytes) 2758 return safexcel_hmac_sha3_224_init(req) ?: 2759 safexcel_ahash_finup(req); 2760 2761 /* HW cannot do zero length HMAC, use fallback instead */ 2762 return safexcel_sha3_digest_fallback(req); 2763 } 2764 2765 static int safexcel_hmac_sha3_224_cra_init(struct crypto_tfm *tfm) 2766 { 2767 return safexcel_hmac_sha3_cra_init(tfm, "sha3-224"); 2768 } 2769 2770 struct safexcel_alg_template safexcel_alg_hmac_sha3_224 = { 2771 .type = SAFEXCEL_ALG_TYPE_AHASH, 2772 .algo_mask = SAFEXCEL_ALG_SHA3, 2773 .alg.ahash = { 2774 .init = safexcel_hmac_sha3_224_init, 2775 .update = safexcel_sha3_update, 2776 .final = safexcel_sha3_final, 2777 .finup = safexcel_sha3_finup, 2778 .digest = safexcel_hmac_sha3_224_digest, 2779 .setkey = safexcel_hmac_sha3_setkey, 2780 .export = safexcel_sha3_export, 2781 .import = safexcel_sha3_import, 2782 .halg = { 2783 .digestsize = SHA3_224_DIGEST_SIZE, 2784 .statesize = sizeof(struct safexcel_ahash_export_state), 2785 .base = { 2786 .cra_name = "hmac(sha3-224)", 2787 .cra_driver_name = "safexcel-hmac-sha3-224", 2788 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2789 .cra_flags = CRYPTO_ALG_ASYNC | 2790 CRYPTO_ALG_KERN_DRIVER_ONLY | 2791 CRYPTO_ALG_NEED_FALLBACK, 2792 .cra_blocksize = SHA3_224_BLOCK_SIZE, 2793 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 2794 .cra_init = safexcel_hmac_sha3_224_cra_init, 2795 .cra_exit = safexcel_hmac_sha3_cra_exit, 2796 .cra_module = THIS_MODULE, 2797 }, 2798 }, 2799 }, 2800 }; 2801 2802 static int safexcel_hmac_sha3_256_init(struct ahash_request *areq) 2803 { 2804 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2805 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2806 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 2807 2808 memset(req, 0, sizeof(*req)); 2809 2810 /* Copy (half of) the key */ 2811 memcpy(req->state, &ctx->base.ipad, SHA3_256_BLOCK_SIZE / 2); 2812 /* Start of HMAC should have len == processed == blocksize */ 2813 req->len = SHA3_256_BLOCK_SIZE; 2814 req->processed = SHA3_256_BLOCK_SIZE; 2815 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_256; 2816 req->digest = CONTEXT_CONTROL_DIGEST_HMAC; 2817 req->state_sz = SHA3_256_BLOCK_SIZE / 2; 2818 req->digest_sz = SHA3_256_DIGEST_SIZE; 2819 req->block_sz = SHA3_256_BLOCK_SIZE; 2820 req->hmac = true; 2821 ctx->do_fallback = false; 2822 ctx->fb_init_done = false; 2823 return 0; 2824 } 2825 2826 static int safexcel_hmac_sha3_256_digest(struct ahash_request *req) 2827 { 2828 if (req->nbytes) 2829 return safexcel_hmac_sha3_256_init(req) ?: 2830 safexcel_ahash_finup(req); 2831 2832 /* HW cannot do zero length HMAC, use fallback instead */ 2833 return safexcel_sha3_digest_fallback(req); 2834 } 2835 2836 static int safexcel_hmac_sha3_256_cra_init(struct crypto_tfm *tfm) 2837 { 2838 return safexcel_hmac_sha3_cra_init(tfm, "sha3-256"); 2839 } 2840 2841 struct safexcel_alg_template safexcel_alg_hmac_sha3_256 = { 2842 .type = SAFEXCEL_ALG_TYPE_AHASH, 2843 .algo_mask = SAFEXCEL_ALG_SHA3, 2844 .alg.ahash = { 2845 .init = safexcel_hmac_sha3_256_init, 2846 .update = safexcel_sha3_update, 2847 .final = safexcel_sha3_final, 2848 .finup = safexcel_sha3_finup, 2849 .digest = safexcel_hmac_sha3_256_digest, 2850 .setkey = safexcel_hmac_sha3_setkey, 2851 .export = safexcel_sha3_export, 2852 .import = safexcel_sha3_import, 2853 .halg = { 2854 .digestsize = SHA3_256_DIGEST_SIZE, 2855 .statesize = sizeof(struct safexcel_ahash_export_state), 2856 .base = { 2857 .cra_name = "hmac(sha3-256)", 2858 .cra_driver_name = "safexcel-hmac-sha3-256", 2859 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2860 .cra_flags = CRYPTO_ALG_ASYNC | 2861 CRYPTO_ALG_KERN_DRIVER_ONLY | 2862 CRYPTO_ALG_NEED_FALLBACK, 2863 .cra_blocksize = SHA3_256_BLOCK_SIZE, 2864 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 2865 .cra_init = safexcel_hmac_sha3_256_cra_init, 2866 .cra_exit = safexcel_hmac_sha3_cra_exit, 2867 .cra_module = THIS_MODULE, 2868 }, 2869 }, 2870 }, 2871 }; 2872 2873 static int safexcel_hmac_sha3_384_init(struct ahash_request *areq) 2874 { 2875 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2876 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2877 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 2878 2879 memset(req, 0, sizeof(*req)); 2880 2881 /* Copy (half of) the key */ 2882 memcpy(req->state, &ctx->base.ipad, SHA3_384_BLOCK_SIZE / 2); 2883 /* Start of HMAC should have len == processed == blocksize */ 2884 req->len = SHA3_384_BLOCK_SIZE; 2885 req->processed = SHA3_384_BLOCK_SIZE; 2886 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_384; 2887 req->digest = CONTEXT_CONTROL_DIGEST_HMAC; 2888 req->state_sz = SHA3_384_BLOCK_SIZE / 2; 2889 req->digest_sz = SHA3_384_DIGEST_SIZE; 2890 req->block_sz = SHA3_384_BLOCK_SIZE; 2891 req->hmac = true; 2892 ctx->do_fallback = false; 2893 ctx->fb_init_done = false; 2894 return 0; 2895 } 2896 2897 static int safexcel_hmac_sha3_384_digest(struct ahash_request *req) 2898 { 2899 if (req->nbytes) 2900 return safexcel_hmac_sha3_384_init(req) ?: 2901 safexcel_ahash_finup(req); 2902 2903 /* HW cannot do zero length HMAC, use fallback instead */ 2904 return safexcel_sha3_digest_fallback(req); 2905 } 2906 2907 static int safexcel_hmac_sha3_384_cra_init(struct crypto_tfm *tfm) 2908 { 2909 return safexcel_hmac_sha3_cra_init(tfm, "sha3-384"); 2910 } 2911 2912 struct safexcel_alg_template safexcel_alg_hmac_sha3_384 = { 2913 .type = SAFEXCEL_ALG_TYPE_AHASH, 2914 .algo_mask = SAFEXCEL_ALG_SHA3, 2915 .alg.ahash = { 2916 .init = safexcel_hmac_sha3_384_init, 2917 .update = safexcel_sha3_update, 2918 .final = safexcel_sha3_final, 2919 .finup = safexcel_sha3_finup, 2920 .digest = safexcel_hmac_sha3_384_digest, 2921 .setkey = safexcel_hmac_sha3_setkey, 2922 .export = safexcel_sha3_export, 2923 .import = safexcel_sha3_import, 2924 .halg = { 2925 .digestsize = SHA3_384_DIGEST_SIZE, 2926 .statesize = sizeof(struct safexcel_ahash_export_state), 2927 .base = { 2928 .cra_name = "hmac(sha3-384)", 2929 .cra_driver_name = "safexcel-hmac-sha3-384", 2930 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2931 .cra_flags = CRYPTO_ALG_ASYNC | 2932 CRYPTO_ALG_KERN_DRIVER_ONLY | 2933 CRYPTO_ALG_NEED_FALLBACK, 2934 .cra_blocksize = SHA3_384_BLOCK_SIZE, 2935 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 2936 .cra_init = safexcel_hmac_sha3_384_cra_init, 2937 .cra_exit = safexcel_hmac_sha3_cra_exit, 2938 .cra_module = THIS_MODULE, 2939 }, 2940 }, 2941 }, 2942 }; 2943 2944 static int safexcel_hmac_sha3_512_init(struct ahash_request *areq) 2945 { 2946 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2947 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2948 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 2949 2950 memset(req, 0, sizeof(*req)); 2951 2952 /* Copy (half of) the key */ 2953 memcpy(req->state, &ctx->base.ipad, SHA3_512_BLOCK_SIZE / 2); 2954 /* Start of HMAC should have len == processed == blocksize */ 2955 req->len = SHA3_512_BLOCK_SIZE; 2956 req->processed = SHA3_512_BLOCK_SIZE; 2957 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_512; 2958 req->digest = CONTEXT_CONTROL_DIGEST_HMAC; 2959 req->state_sz = SHA3_512_BLOCK_SIZE / 2; 2960 req->digest_sz = SHA3_512_DIGEST_SIZE; 2961 req->block_sz = SHA3_512_BLOCK_SIZE; 2962 req->hmac = true; 2963 ctx->do_fallback = false; 2964 ctx->fb_init_done = false; 2965 return 0; 2966 } 2967 2968 static int safexcel_hmac_sha3_512_digest(struct ahash_request *req) 2969 { 2970 if (req->nbytes) 2971 return safexcel_hmac_sha3_512_init(req) ?: 2972 safexcel_ahash_finup(req); 2973 2974 /* HW cannot do zero length HMAC, use fallback instead */ 2975 return safexcel_sha3_digest_fallback(req); 2976 } 2977 2978 static int safexcel_hmac_sha3_512_cra_init(struct crypto_tfm *tfm) 2979 { 2980 return safexcel_hmac_sha3_cra_init(tfm, "sha3-512"); 2981 } 2982 struct safexcel_alg_template safexcel_alg_hmac_sha3_512 = { 2983 .type = SAFEXCEL_ALG_TYPE_AHASH, 2984 .algo_mask = SAFEXCEL_ALG_SHA3, 2985 .alg.ahash = { 2986 .init = safexcel_hmac_sha3_512_init, 2987 .update = safexcel_sha3_update, 2988 .final = safexcel_sha3_final, 2989 .finup = safexcel_sha3_finup, 2990 .digest = safexcel_hmac_sha3_512_digest, 2991 .setkey = safexcel_hmac_sha3_setkey, 2992 .export = safexcel_sha3_export, 2993 .import = safexcel_sha3_import, 2994 .halg = { 2995 .digestsize = SHA3_512_DIGEST_SIZE, 2996 .statesize = sizeof(struct safexcel_ahash_export_state), 2997 .base = { 2998 .cra_name = "hmac(sha3-512)", 2999 .cra_driver_name = "safexcel-hmac-sha3-512", 3000 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3001 .cra_flags = CRYPTO_ALG_ASYNC | 3002 CRYPTO_ALG_KERN_DRIVER_ONLY | 3003 CRYPTO_ALG_NEED_FALLBACK, 3004 .cra_blocksize = SHA3_512_BLOCK_SIZE, 3005 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 3006 .cra_init = safexcel_hmac_sha3_512_cra_init, 3007 .cra_exit = safexcel_hmac_sha3_cra_exit, 3008 .cra_module = THIS_MODULE, 3009 }, 3010 }, 3011 }, 3012 }; 3013