1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2017 Marvell 4 * 5 * Antoine Tenart <antoine.tenart@free-electrons.com> 6 */ 7 8 #include <crypto/aes.h> 9 #include <crypto/hmac.h> 10 #include <crypto/md5.h> 11 #include <crypto/sha1.h> 12 #include <crypto/sha2.h> 13 #include <crypto/sha3.h> 14 #include <crypto/skcipher.h> 15 #include <crypto/sm3.h> 16 #include <crypto/internal/cipher.h> 17 #include <linux/device.h> 18 #include <linux/dma-mapping.h> 19 #include <linux/dmapool.h> 20 21 #include "safexcel.h" 22 23 struct safexcel_ahash_ctx { 24 struct safexcel_context base; 25 26 u32 alg; 27 u8 key_sz; 28 bool cbcmac; 29 bool do_fallback; 30 bool fb_init_done; 31 bool fb_do_setkey; 32 33 struct crypto_aes_ctx *aes; 34 struct crypto_ahash *fback; 35 struct crypto_shash *shpre; 36 struct shash_desc *shdesc; 37 }; 38 39 struct safexcel_ahash_req { 40 bool last_req; 41 bool finish; 42 bool hmac; 43 bool needs_inv; 44 bool hmac_zlen; 45 bool len_is_le; 46 bool not_first; 47 bool xcbcmac; 48 49 int nents; 50 dma_addr_t result_dma; 51 52 u32 digest; 53 54 u8 state_sz; /* expected state size, only set once */ 55 u8 block_sz; /* block size, only set once */ 56 u8 digest_sz; /* output digest size, only set once */ 57 __le32 state[SHA3_512_BLOCK_SIZE / 58 sizeof(__le32)] __aligned(sizeof(__le32)); 59 60 u64 len; 61 u64 processed; 62 63 u8 cache[HASH_CACHE_SIZE] __aligned(sizeof(u32)); 64 dma_addr_t cache_dma; 65 unsigned int cache_sz; 66 67 u8 cache_next[HASH_CACHE_SIZE] __aligned(sizeof(u32)); 68 }; 69 70 static inline u64 safexcel_queued_len(struct safexcel_ahash_req *req) 71 { 72 return req->len - req->processed; 73 } 74 75 static void safexcel_hash_token(struct safexcel_command_desc *cdesc, 76 u32 input_length, u32 result_length, 77 bool cbcmac) 78 { 79 struct safexcel_token *token = 80 (struct safexcel_token *)cdesc->control_data.token; 81 82 token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION; 83 token[0].packet_length = input_length; 84 token[0].instructions = EIP197_TOKEN_INS_TYPE_HASH; 85 86 input_length &= 15; 87 if (unlikely(cbcmac && input_length)) { 88 token[0].stat = 0; 89 token[1].opcode = EIP197_TOKEN_OPCODE_INSERT; 90 token[1].packet_length = 16 - input_length; 91 token[1].stat = EIP197_TOKEN_STAT_LAST_HASH; 92 token[1].instructions = EIP197_TOKEN_INS_TYPE_HASH; 93 } else { 94 token[0].stat = EIP197_TOKEN_STAT_LAST_HASH; 95 eip197_noop_token(&token[1]); 96 } 97 98 token[2].opcode = EIP197_TOKEN_OPCODE_INSERT; 99 token[2].stat = EIP197_TOKEN_STAT_LAST_HASH | 100 EIP197_TOKEN_STAT_LAST_PACKET; 101 token[2].packet_length = result_length; 102 token[2].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | 103 EIP197_TOKEN_INS_INSERT_HASH_DIGEST; 104 105 eip197_noop_token(&token[3]); 106 } 107 108 static void safexcel_context_control(struct safexcel_ahash_ctx *ctx, 109 struct safexcel_ahash_req *req, 110 struct safexcel_command_desc *cdesc) 111 { 112 struct safexcel_crypto_priv *priv = ctx->base.priv; 113 u64 count = 0; 114 115 cdesc->control_data.control0 = ctx->alg; 116 cdesc->control_data.control1 = 0; 117 118 /* 119 * Copy the input digest if needed, and setup the context 120 * fields. Do this now as we need it to setup the first command 121 * descriptor. 122 */ 123 if (unlikely(req->digest == CONTEXT_CONTROL_DIGEST_XCM)) { 124 if (req->xcbcmac) 125 memcpy(ctx->base.ctxr->data, &ctx->base.ipad, ctx->key_sz); 126 else 127 memcpy(ctx->base.ctxr->data, req->state, req->state_sz); 128 129 if (!req->finish && req->xcbcmac) 130 cdesc->control_data.control0 |= 131 CONTEXT_CONTROL_DIGEST_XCM | 132 CONTEXT_CONTROL_TYPE_HASH_OUT | 133 CONTEXT_CONTROL_NO_FINISH_HASH | 134 CONTEXT_CONTROL_SIZE(req->state_sz / 135 sizeof(u32)); 136 else 137 cdesc->control_data.control0 |= 138 CONTEXT_CONTROL_DIGEST_XCM | 139 CONTEXT_CONTROL_TYPE_HASH_OUT | 140 CONTEXT_CONTROL_SIZE(req->state_sz / 141 sizeof(u32)); 142 return; 143 } else if (!req->processed) { 144 /* First - and possibly only - block of basic hash only */ 145 if (req->finish) 146 cdesc->control_data.control0 |= req->digest | 147 CONTEXT_CONTROL_TYPE_HASH_OUT | 148 CONTEXT_CONTROL_RESTART_HASH | 149 /* ensure its not 0! */ 150 CONTEXT_CONTROL_SIZE(1); 151 else 152 cdesc->control_data.control0 |= req->digest | 153 CONTEXT_CONTROL_TYPE_HASH_OUT | 154 CONTEXT_CONTROL_RESTART_HASH | 155 CONTEXT_CONTROL_NO_FINISH_HASH | 156 /* ensure its not 0! */ 157 CONTEXT_CONTROL_SIZE(1); 158 return; 159 } 160 161 /* Hash continuation or HMAC, setup (inner) digest from state */ 162 memcpy(ctx->base.ctxr->data, req->state, req->state_sz); 163 164 if (req->finish) { 165 /* Compute digest count for hash/HMAC finish operations */ 166 if ((req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) || 167 req->hmac_zlen || (req->processed != req->block_sz)) { 168 count = req->processed / EIP197_COUNTER_BLOCK_SIZE; 169 170 /* This is a hardware limitation, as the 171 * counter must fit into an u32. This represents 172 * a fairly big amount of input data, so we 173 * shouldn't see this. 174 */ 175 if (unlikely(count & 0xffffffff00000000ULL)) { 176 dev_warn(priv->dev, 177 "Input data is too big\n"); 178 return; 179 } 180 } 181 182 if ((req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) || 183 /* Special case: zero length HMAC */ 184 req->hmac_zlen || 185 /* PE HW < 4.4 cannot do HMAC continue, fake using hash */ 186 (req->processed != req->block_sz)) { 187 /* Basic hash continue operation, need digest + cnt */ 188 cdesc->control_data.control0 |= 189 CONTEXT_CONTROL_SIZE((req->state_sz >> 2) + 1) | 190 CONTEXT_CONTROL_TYPE_HASH_OUT | 191 CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 192 /* For zero-len HMAC, don't finalize, already padded! */ 193 if (req->hmac_zlen) 194 cdesc->control_data.control0 |= 195 CONTEXT_CONTROL_NO_FINISH_HASH; 196 cdesc->control_data.control1 |= 197 CONTEXT_CONTROL_DIGEST_CNT; 198 ctx->base.ctxr->data[req->state_sz >> 2] = 199 cpu_to_le32(count); 200 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 201 202 /* Clear zero-length HMAC flag for next operation! */ 203 req->hmac_zlen = false; 204 } else { /* HMAC */ 205 /* Need outer digest for HMAC finalization */ 206 memcpy(ctx->base.ctxr->data + (req->state_sz >> 2), 207 &ctx->base.opad, req->state_sz); 208 209 /* Single pass HMAC - no digest count */ 210 cdesc->control_data.control0 |= 211 CONTEXT_CONTROL_SIZE(req->state_sz >> 1) | 212 CONTEXT_CONTROL_TYPE_HASH_OUT | 213 CONTEXT_CONTROL_DIGEST_HMAC; 214 } 215 } else { /* Hash continuation, do not finish yet */ 216 cdesc->control_data.control0 |= 217 CONTEXT_CONTROL_SIZE(req->state_sz >> 2) | 218 CONTEXT_CONTROL_DIGEST_PRECOMPUTED | 219 CONTEXT_CONTROL_TYPE_HASH_OUT | 220 CONTEXT_CONTROL_NO_FINISH_HASH; 221 } 222 } 223 224 static int safexcel_ahash_enqueue(struct ahash_request *areq); 225 226 static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, 227 int ring, 228 struct crypto_async_request *async, 229 bool *should_complete, int *ret) 230 { 231 struct safexcel_result_desc *rdesc; 232 struct ahash_request *areq = ahash_request_cast(async); 233 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); 234 struct safexcel_ahash_req *sreq = ahash_request_ctx_dma(areq); 235 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(ahash); 236 u64 cache_len; 237 238 *ret = 0; 239 240 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); 241 if (IS_ERR(rdesc)) { 242 dev_err(priv->dev, 243 "hash: result: could not retrieve the result descriptor\n"); 244 *ret = PTR_ERR(rdesc); 245 } else { 246 *ret = safexcel_rdesc_check_errors(priv, rdesc); 247 } 248 249 safexcel_complete(priv, ring); 250 251 if (sreq->nents) { 252 dma_unmap_sg(priv->dev, areq->src, sreq->nents, DMA_TO_DEVICE); 253 sreq->nents = 0; 254 } 255 256 if (sreq->result_dma) { 257 dma_unmap_single(priv->dev, sreq->result_dma, sreq->digest_sz, 258 DMA_FROM_DEVICE); 259 sreq->result_dma = 0; 260 } 261 262 if (sreq->cache_dma) { 263 dma_unmap_single(priv->dev, sreq->cache_dma, sreq->cache_sz, 264 DMA_TO_DEVICE); 265 sreq->cache_dma = 0; 266 sreq->cache_sz = 0; 267 } 268 269 if (sreq->finish) { 270 if (sreq->hmac && 271 (sreq->digest != CONTEXT_CONTROL_DIGEST_HMAC)) { 272 /* Faking HMAC using hash - need to do outer hash */ 273 memcpy(sreq->cache, sreq->state, 274 crypto_ahash_digestsize(ahash)); 275 276 memcpy(sreq->state, &ctx->base.opad, sreq->digest_sz); 277 278 sreq->len = sreq->block_sz + 279 crypto_ahash_digestsize(ahash); 280 sreq->processed = sreq->block_sz; 281 sreq->hmac = 0; 282 283 if (priv->flags & EIP197_TRC_CACHE) 284 ctx->base.needs_inv = true; 285 areq->nbytes = 0; 286 safexcel_ahash_enqueue(areq); 287 288 *should_complete = false; /* Not done yet */ 289 return 1; 290 } 291 292 memcpy(areq->result, sreq->state, 293 crypto_ahash_digestsize(ahash)); 294 } 295 296 cache_len = safexcel_queued_len(sreq); 297 if (cache_len) 298 memcpy(sreq->cache, sreq->cache_next, cache_len); 299 300 *should_complete = true; 301 302 return 1; 303 } 304 305 static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, 306 int *commands, int *results) 307 { 308 struct ahash_request *areq = ahash_request_cast(async); 309 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 310 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 311 struct safexcel_crypto_priv *priv = ctx->base.priv; 312 struct safexcel_command_desc *cdesc, *first_cdesc = NULL; 313 struct safexcel_result_desc *rdesc; 314 struct scatterlist *sg; 315 struct safexcel_token *dmmy; 316 int i, extra = 0, n_cdesc = 0, ret = 0, cache_len, skip = 0; 317 u64 queued, len; 318 319 queued = safexcel_queued_len(req); 320 if (queued <= HASH_CACHE_SIZE) 321 cache_len = queued; 322 else 323 cache_len = queued - areq->nbytes; 324 325 if (!req->finish && !req->last_req) { 326 /* If this is not the last request and the queued data does not 327 * fit into full cache blocks, cache it for the next send call. 328 */ 329 extra = queued & (HASH_CACHE_SIZE - 1); 330 331 /* If this is not the last request and the queued data 332 * is a multiple of a block, cache the last one for now. 333 */ 334 if (!extra) 335 extra = HASH_CACHE_SIZE; 336 337 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), 338 req->cache_next, extra, 339 areq->nbytes - extra); 340 341 queued -= extra; 342 343 if (!queued) { 344 *commands = 0; 345 *results = 0; 346 return 0; 347 } 348 349 extra = 0; 350 } 351 352 if (unlikely(req->xcbcmac && req->processed > AES_BLOCK_SIZE)) { 353 if (unlikely(cache_len < AES_BLOCK_SIZE)) { 354 /* 355 * Cache contains less than 1 full block, complete. 356 */ 357 extra = AES_BLOCK_SIZE - cache_len; 358 if (queued > cache_len) { 359 /* More data follows: borrow bytes */ 360 u64 tmp = queued - cache_len; 361 362 skip = min_t(u64, tmp, extra); 363 sg_pcopy_to_buffer(areq->src, 364 sg_nents(areq->src), 365 req->cache + cache_len, 366 skip, 0); 367 } 368 extra -= skip; 369 memset(req->cache + cache_len + skip, 0, extra); 370 if (!ctx->cbcmac && extra) { 371 // 10- padding for XCBCMAC & CMAC 372 req->cache[cache_len + skip] = 0x80; 373 // HW will use K2 iso K3 - compensate! 374 for (i = 0; i < AES_BLOCK_SIZE / 4; i++) { 375 u32 *cache = (void *)req->cache; 376 u32 *ipad = ctx->base.ipad.word; 377 u32 x; 378 379 x = ipad[i] ^ ipad[i + 4]; 380 cache[i] ^= swab32(x); 381 } 382 } 383 cache_len = AES_BLOCK_SIZE; 384 queued = queued + extra; 385 } 386 387 /* XCBC continue: XOR previous result into 1st word */ 388 crypto_xor(req->cache, (const u8 *)req->state, AES_BLOCK_SIZE); 389 } 390 391 len = queued; 392 /* Add a command descriptor for the cached data, if any */ 393 if (cache_len) { 394 req->cache_dma = dma_map_single(priv->dev, req->cache, 395 cache_len, DMA_TO_DEVICE); 396 if (dma_mapping_error(priv->dev, req->cache_dma)) 397 return -EINVAL; 398 399 req->cache_sz = cache_len; 400 first_cdesc = safexcel_add_cdesc(priv, ring, 1, 401 (cache_len == len), 402 req->cache_dma, cache_len, 403 len, ctx->base.ctxr_dma, 404 &dmmy); 405 if (IS_ERR(first_cdesc)) { 406 ret = PTR_ERR(first_cdesc); 407 goto unmap_cache; 408 } 409 n_cdesc++; 410 411 queued -= cache_len; 412 if (!queued) 413 goto send_command; 414 } 415 416 /* Now handle the current ahash request buffer(s) */ 417 req->nents = dma_map_sg(priv->dev, areq->src, 418 sg_nents_for_len(areq->src, 419 areq->nbytes), 420 DMA_TO_DEVICE); 421 if (!req->nents) { 422 ret = -ENOMEM; 423 goto cdesc_rollback; 424 } 425 426 for_each_sg(areq->src, sg, req->nents, i) { 427 int sglen = sg_dma_len(sg); 428 429 if (unlikely(sglen <= skip)) { 430 skip -= sglen; 431 continue; 432 } 433 434 /* Do not overflow the request */ 435 if ((queued + skip) <= sglen) 436 sglen = queued; 437 else 438 sglen -= skip; 439 440 cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc, 441 !(queued - sglen), 442 sg_dma_address(sg) + skip, sglen, 443 len, ctx->base.ctxr_dma, &dmmy); 444 if (IS_ERR(cdesc)) { 445 ret = PTR_ERR(cdesc); 446 goto unmap_sg; 447 } 448 449 if (!n_cdesc) 450 first_cdesc = cdesc; 451 n_cdesc++; 452 453 queued -= sglen; 454 if (!queued) 455 break; 456 skip = 0; 457 } 458 459 send_command: 460 /* Setup the context options */ 461 safexcel_context_control(ctx, req, first_cdesc); 462 463 /* Add the token */ 464 safexcel_hash_token(first_cdesc, len, req->digest_sz, ctx->cbcmac); 465 466 req->result_dma = dma_map_single(priv->dev, req->state, req->digest_sz, 467 DMA_FROM_DEVICE); 468 if (dma_mapping_error(priv->dev, req->result_dma)) { 469 ret = -EINVAL; 470 goto unmap_sg; 471 } 472 473 /* Add a result descriptor */ 474 rdesc = safexcel_add_rdesc(priv, ring, 1, 1, req->result_dma, 475 req->digest_sz); 476 if (IS_ERR(rdesc)) { 477 ret = PTR_ERR(rdesc); 478 goto unmap_result; 479 } 480 481 safexcel_rdr_req_set(priv, ring, rdesc, &areq->base); 482 483 req->processed += len - extra; 484 485 *commands = n_cdesc; 486 *results = 1; 487 return 0; 488 489 unmap_result: 490 dma_unmap_single(priv->dev, req->result_dma, req->digest_sz, 491 DMA_FROM_DEVICE); 492 unmap_sg: 493 if (req->nents) { 494 dma_unmap_sg(priv->dev, areq->src, req->nents, DMA_TO_DEVICE); 495 req->nents = 0; 496 } 497 cdesc_rollback: 498 for (i = 0; i < n_cdesc; i++) 499 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr); 500 unmap_cache: 501 if (req->cache_dma) { 502 dma_unmap_single(priv->dev, req->cache_dma, req->cache_sz, 503 DMA_TO_DEVICE); 504 req->cache_dma = 0; 505 req->cache_sz = 0; 506 } 507 508 return ret; 509 } 510 511 static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv, 512 int ring, 513 struct crypto_async_request *async, 514 bool *should_complete, int *ret) 515 { 516 struct safexcel_result_desc *rdesc; 517 struct ahash_request *areq = ahash_request_cast(async); 518 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); 519 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(ahash); 520 int enq_ret; 521 522 *ret = 0; 523 524 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); 525 if (IS_ERR(rdesc)) { 526 dev_err(priv->dev, 527 "hash: invalidate: could not retrieve the result descriptor\n"); 528 *ret = PTR_ERR(rdesc); 529 } else { 530 *ret = safexcel_rdesc_check_errors(priv, rdesc); 531 } 532 533 safexcel_complete(priv, ring); 534 535 if (ctx->base.exit_inv) { 536 dma_pool_free(priv->context_pool, ctx->base.ctxr, 537 ctx->base.ctxr_dma); 538 539 *should_complete = true; 540 return 1; 541 } 542 543 ring = safexcel_select_ring(priv); 544 ctx->base.ring = ring; 545 546 spin_lock_bh(&priv->ring[ring].queue_lock); 547 enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, async); 548 spin_unlock_bh(&priv->ring[ring].queue_lock); 549 550 if (enq_ret != -EINPROGRESS) 551 *ret = enq_ret; 552 553 queue_work(priv->ring[ring].workqueue, 554 &priv->ring[ring].work_data.work); 555 556 *should_complete = false; 557 558 return 1; 559 } 560 561 static int safexcel_handle_result(struct safexcel_crypto_priv *priv, int ring, 562 struct crypto_async_request *async, 563 bool *should_complete, int *ret) 564 { 565 struct ahash_request *areq = ahash_request_cast(async); 566 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 567 int err; 568 569 BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && req->needs_inv); 570 571 if (req->needs_inv) { 572 req->needs_inv = false; 573 err = safexcel_handle_inv_result(priv, ring, async, 574 should_complete, ret); 575 } else { 576 err = safexcel_handle_req_result(priv, ring, async, 577 should_complete, ret); 578 } 579 580 return err; 581 } 582 583 static int safexcel_ahash_send_inv(struct crypto_async_request *async, 584 int ring, int *commands, int *results) 585 { 586 struct ahash_request *areq = ahash_request_cast(async); 587 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 588 int ret; 589 590 ret = safexcel_invalidate_cache(async, ctx->base.priv, 591 ctx->base.ctxr_dma, ring); 592 if (unlikely(ret)) 593 return ret; 594 595 *commands = 1; 596 *results = 1; 597 598 return 0; 599 } 600 601 static int safexcel_ahash_send(struct crypto_async_request *async, 602 int ring, int *commands, int *results) 603 { 604 struct ahash_request *areq = ahash_request_cast(async); 605 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 606 int ret; 607 608 if (req->needs_inv) 609 ret = safexcel_ahash_send_inv(async, ring, commands, results); 610 else 611 ret = safexcel_ahash_send_req(async, ring, commands, results); 612 613 return ret; 614 } 615 616 static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm) 617 { 618 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 619 struct safexcel_crypto_priv *priv = ctx->base.priv; 620 EIP197_REQUEST_ON_STACK(req, ahash, EIP197_AHASH_REQ_SIZE); 621 struct safexcel_ahash_req *rctx = ahash_request_ctx_dma(req); 622 DECLARE_CRYPTO_WAIT(result); 623 int ring = ctx->base.ring; 624 int err; 625 626 memset(req, 0, EIP197_AHASH_REQ_SIZE); 627 628 /* create invalidation request */ 629 init_completion(&result.completion); 630 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 631 crypto_req_done, &result); 632 633 ahash_request_set_tfm(req, __crypto_ahash_cast(tfm)); 634 ctx = crypto_tfm_ctx(req->base.tfm); 635 ctx->base.exit_inv = true; 636 rctx->needs_inv = true; 637 638 spin_lock_bh(&priv->ring[ring].queue_lock); 639 crypto_enqueue_request(&priv->ring[ring].queue, &req->base); 640 spin_unlock_bh(&priv->ring[ring].queue_lock); 641 642 queue_work(priv->ring[ring].workqueue, 643 &priv->ring[ring].work_data.work); 644 645 err = crypto_wait_req(-EINPROGRESS, &result); 646 647 if (err) { 648 dev_warn(priv->dev, "hash: completion error (%d)\n", err); 649 return err; 650 } 651 652 return 0; 653 } 654 655 /* safexcel_ahash_cache: cache data until at least one request can be sent to 656 * the engine, aka. when there is at least 1 block size in the pipe. 657 */ 658 static int safexcel_ahash_cache(struct ahash_request *areq) 659 { 660 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 661 u64 cache_len; 662 663 /* cache_len: everything accepted by the driver but not sent yet, 664 * tot sz handled by update() - last req sz - tot sz handled by send() 665 */ 666 cache_len = safexcel_queued_len(req); 667 668 /* 669 * In case there isn't enough bytes to proceed (less than a 670 * block size), cache the data until we have enough. 671 */ 672 if (cache_len + areq->nbytes <= HASH_CACHE_SIZE) { 673 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), 674 req->cache + cache_len, 675 areq->nbytes, 0); 676 return 0; 677 } 678 679 /* We couldn't cache all the data */ 680 return -E2BIG; 681 } 682 683 static int safexcel_ahash_enqueue(struct ahash_request *areq) 684 { 685 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 686 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 687 struct safexcel_crypto_priv *priv = ctx->base.priv; 688 int ret, ring; 689 690 req->needs_inv = false; 691 692 if (ctx->base.ctxr) { 693 if (priv->flags & EIP197_TRC_CACHE && !ctx->base.needs_inv && 694 /* invalidate for *any* non-XCBC continuation */ 695 ((req->not_first && !req->xcbcmac) || 696 /* invalidate if (i)digest changed */ 697 memcmp(ctx->base.ctxr->data, req->state, req->state_sz) || 698 /* invalidate for HMAC finish with odigest changed */ 699 (req->finish && req->hmac && 700 memcmp(ctx->base.ctxr->data + (req->state_sz>>2), 701 &ctx->base.opad, req->state_sz)))) 702 /* 703 * We're still setting needs_inv here, even though it is 704 * cleared right away, because the needs_inv flag can be 705 * set in other functions and we want to keep the same 706 * logic. 707 */ 708 ctx->base.needs_inv = true; 709 710 if (ctx->base.needs_inv) { 711 ctx->base.needs_inv = false; 712 req->needs_inv = true; 713 } 714 } else { 715 ctx->base.ring = safexcel_select_ring(priv); 716 ctx->base.ctxr = dma_pool_zalloc(priv->context_pool, 717 EIP197_GFP_FLAGS(areq->base), 718 &ctx->base.ctxr_dma); 719 if (!ctx->base.ctxr) 720 return -ENOMEM; 721 } 722 req->not_first = true; 723 724 ring = ctx->base.ring; 725 726 spin_lock_bh(&priv->ring[ring].queue_lock); 727 ret = crypto_enqueue_request(&priv->ring[ring].queue, &areq->base); 728 spin_unlock_bh(&priv->ring[ring].queue_lock); 729 730 queue_work(priv->ring[ring].workqueue, 731 &priv->ring[ring].work_data.work); 732 733 return ret; 734 } 735 736 static int safexcel_ahash_update(struct ahash_request *areq) 737 { 738 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 739 int ret; 740 741 /* If the request is 0 length, do nothing */ 742 if (!areq->nbytes) 743 return 0; 744 745 /* Add request to the cache if it fits */ 746 ret = safexcel_ahash_cache(areq); 747 748 /* Update total request length */ 749 req->len += areq->nbytes; 750 751 /* If not all data could fit into the cache, go process the excess. 752 * Also go process immediately for an HMAC IV precompute, which 753 * will never be finished at all, but needs to be processed anyway. 754 */ 755 if ((ret && !req->finish) || req->last_req) 756 return safexcel_ahash_enqueue(areq); 757 758 return 0; 759 } 760 761 static int safexcel_ahash_final(struct ahash_request *areq) 762 { 763 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 764 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 765 766 req->finish = true; 767 768 if (unlikely(!req->len && !areq->nbytes)) { 769 /* 770 * If we have an overall 0 length *hash* request: 771 * The HW cannot do 0 length hash, so we provide the correct 772 * result directly here. 773 */ 774 if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5) 775 memcpy(areq->result, md5_zero_message_hash, 776 MD5_DIGEST_SIZE); 777 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA1) 778 memcpy(areq->result, sha1_zero_message_hash, 779 SHA1_DIGEST_SIZE); 780 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA224) 781 memcpy(areq->result, sha224_zero_message_hash, 782 SHA224_DIGEST_SIZE); 783 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA256) 784 memcpy(areq->result, sha256_zero_message_hash, 785 SHA256_DIGEST_SIZE); 786 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA384) 787 memcpy(areq->result, sha384_zero_message_hash, 788 SHA384_DIGEST_SIZE); 789 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA512) 790 memcpy(areq->result, sha512_zero_message_hash, 791 SHA512_DIGEST_SIZE); 792 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SM3) { 793 memcpy(areq->result, 794 EIP197_SM3_ZEROM_HASH, SM3_DIGEST_SIZE); 795 } 796 797 return 0; 798 } else if (unlikely(req->digest == CONTEXT_CONTROL_DIGEST_XCM && 799 ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5 && 800 req->len == sizeof(u32) && !areq->nbytes)) { 801 /* Zero length CRC32 */ 802 memcpy(areq->result, &ctx->base.ipad, sizeof(u32)); 803 return 0; 804 } else if (unlikely(ctx->cbcmac && req->len == AES_BLOCK_SIZE && 805 !areq->nbytes)) { 806 /* Zero length CBC MAC */ 807 memset(areq->result, 0, AES_BLOCK_SIZE); 808 return 0; 809 } else if (unlikely(req->xcbcmac && req->len == AES_BLOCK_SIZE && 810 !areq->nbytes)) { 811 /* Zero length (X)CBC/CMAC */ 812 int i; 813 814 for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) { 815 u32 *result = (void *)areq->result; 816 817 /* K3 */ 818 result[i] = swab32(ctx->base.ipad.word[i + 4]); 819 } 820 areq->result[0] ^= 0x80; // 10- padding 821 aes_encrypt(ctx->aes, areq->result, areq->result); 822 return 0; 823 } else if (unlikely(req->hmac && 824 (req->len == req->block_sz) && 825 !areq->nbytes)) { 826 /* 827 * If we have an overall 0 length *HMAC* request: 828 * For HMAC, we need to finalize the inner digest 829 * and then perform the outer hash. 830 */ 831 832 /* generate pad block in the cache */ 833 /* start with a hash block of all zeroes */ 834 memset(req->cache, 0, req->block_sz); 835 /* set the first byte to 0x80 to 'append a 1 bit' */ 836 req->cache[0] = 0x80; 837 /* add the length in bits in the last 2 bytes */ 838 if (req->len_is_le) { 839 /* Little endian length word (e.g. MD5) */ 840 req->cache[req->block_sz-8] = (req->block_sz << 3) & 841 255; 842 req->cache[req->block_sz-7] = (req->block_sz >> 5); 843 } else { 844 /* Big endian length word (e.g. any SHA) */ 845 req->cache[req->block_sz-2] = (req->block_sz >> 5); 846 req->cache[req->block_sz-1] = (req->block_sz << 3) & 847 255; 848 } 849 850 req->len += req->block_sz; /* plus 1 hash block */ 851 852 /* Set special zero-length HMAC flag */ 853 req->hmac_zlen = true; 854 855 /* Finalize HMAC */ 856 req->digest = CONTEXT_CONTROL_DIGEST_HMAC; 857 } else if (req->hmac) { 858 /* Finalize HMAC */ 859 req->digest = CONTEXT_CONTROL_DIGEST_HMAC; 860 } 861 862 return safexcel_ahash_enqueue(areq); 863 } 864 865 static int safexcel_ahash_finup(struct ahash_request *areq) 866 { 867 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 868 869 req->finish = true; 870 871 safexcel_ahash_update(areq); 872 return safexcel_ahash_final(areq); 873 } 874 875 static int safexcel_ahash_export(struct ahash_request *areq, void *out) 876 { 877 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 878 struct safexcel_ahash_export_state *export = out; 879 880 export->len = req->len; 881 export->processed = req->processed; 882 883 export->digest = req->digest; 884 885 memcpy(export->state, req->state, req->state_sz); 886 memcpy(export->cache, req->cache, HASH_CACHE_SIZE); 887 888 return 0; 889 } 890 891 static int safexcel_ahash_import(struct ahash_request *areq, const void *in) 892 { 893 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 894 const struct safexcel_ahash_export_state *export = in; 895 int ret; 896 897 ret = crypto_ahash_init(areq); 898 if (ret) 899 return ret; 900 901 req->len = export->len; 902 req->processed = export->processed; 903 904 req->digest = export->digest; 905 906 memcpy(req->cache, export->cache, HASH_CACHE_SIZE); 907 memcpy(req->state, export->state, req->state_sz); 908 909 return 0; 910 } 911 912 static int safexcel_ahash_cra_init(struct crypto_tfm *tfm) 913 { 914 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 915 struct safexcel_alg_template *tmpl = 916 container_of(__crypto_ahash_alg(tfm->__crt_alg), 917 struct safexcel_alg_template, alg.ahash); 918 919 ctx->base.priv = tmpl->priv; 920 ctx->base.send = safexcel_ahash_send; 921 ctx->base.handle_result = safexcel_handle_result; 922 ctx->fb_do_setkey = false; 923 924 crypto_ahash_set_reqsize_dma(__crypto_ahash_cast(tfm), 925 sizeof(struct safexcel_ahash_req)); 926 return 0; 927 } 928 929 static int safexcel_sha1_init(struct ahash_request *areq) 930 { 931 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 932 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 933 934 memset(req, 0, sizeof(*req)); 935 936 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1; 937 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 938 req->state_sz = SHA1_DIGEST_SIZE; 939 req->digest_sz = SHA1_DIGEST_SIZE; 940 req->block_sz = SHA1_BLOCK_SIZE; 941 942 return 0; 943 } 944 945 static int safexcel_sha1_digest(struct ahash_request *areq) 946 { 947 int ret = safexcel_sha1_init(areq); 948 949 if (ret) 950 return ret; 951 952 return safexcel_ahash_finup(areq); 953 } 954 955 static void safexcel_ahash_cra_exit(struct crypto_tfm *tfm) 956 { 957 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 958 struct safexcel_crypto_priv *priv = ctx->base.priv; 959 int ret; 960 961 /* context not allocated, skip invalidation */ 962 if (!ctx->base.ctxr) 963 return; 964 965 if (priv->flags & EIP197_TRC_CACHE) { 966 ret = safexcel_ahash_exit_inv(tfm); 967 if (ret) 968 dev_warn(priv->dev, "hash: invalidation error %d\n", ret); 969 } else { 970 dma_pool_free(priv->context_pool, ctx->base.ctxr, 971 ctx->base.ctxr_dma); 972 } 973 } 974 975 struct safexcel_alg_template safexcel_alg_sha1 = { 976 .type = SAFEXCEL_ALG_TYPE_AHASH, 977 .algo_mask = SAFEXCEL_ALG_SHA1, 978 .alg.ahash = { 979 .init = safexcel_sha1_init, 980 .update = safexcel_ahash_update, 981 .final = safexcel_ahash_final, 982 .finup = safexcel_ahash_finup, 983 .digest = safexcel_sha1_digest, 984 .export = safexcel_ahash_export, 985 .import = safexcel_ahash_import, 986 .halg = { 987 .digestsize = SHA1_DIGEST_SIZE, 988 .statesize = sizeof(struct safexcel_ahash_export_state), 989 .base = { 990 .cra_name = "sha1", 991 .cra_driver_name = "safexcel-sha1", 992 .cra_priority = SAFEXCEL_CRA_PRIORITY, 993 .cra_flags = CRYPTO_ALG_ASYNC | 994 CRYPTO_ALG_ALLOCATES_MEMORY | 995 CRYPTO_ALG_KERN_DRIVER_ONLY, 996 .cra_blocksize = SHA1_BLOCK_SIZE, 997 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 998 .cra_init = safexcel_ahash_cra_init, 999 .cra_exit = safexcel_ahash_cra_exit, 1000 .cra_module = THIS_MODULE, 1001 }, 1002 }, 1003 }, 1004 }; 1005 1006 static int safexcel_hmac_sha1_init(struct ahash_request *areq) 1007 { 1008 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1009 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1010 1011 memset(req, 0, sizeof(*req)); 1012 1013 /* Start from ipad precompute */ 1014 memcpy(req->state, &ctx->base.ipad, SHA1_DIGEST_SIZE); 1015 /* Already processed the key^ipad part now! */ 1016 req->len = SHA1_BLOCK_SIZE; 1017 req->processed = SHA1_BLOCK_SIZE; 1018 1019 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1; 1020 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1021 req->state_sz = SHA1_DIGEST_SIZE; 1022 req->digest_sz = SHA1_DIGEST_SIZE; 1023 req->block_sz = SHA1_BLOCK_SIZE; 1024 req->hmac = true; 1025 1026 return 0; 1027 } 1028 1029 static int safexcel_hmac_sha1_digest(struct ahash_request *areq) 1030 { 1031 int ret = safexcel_hmac_sha1_init(areq); 1032 1033 if (ret) 1034 return ret; 1035 1036 return safexcel_ahash_finup(areq); 1037 } 1038 1039 static int safexcel_hmac_init_pad(struct ahash_request *areq, 1040 unsigned int blocksize, const u8 *key, 1041 unsigned int keylen, u8 *ipad, u8 *opad) 1042 { 1043 DECLARE_CRYPTO_WAIT(result); 1044 struct scatterlist sg; 1045 int ret, i; 1046 u8 *keydup; 1047 1048 if (keylen <= blocksize) { 1049 memcpy(ipad, key, keylen); 1050 } else { 1051 keydup = kmemdup(key, keylen, GFP_KERNEL); 1052 if (!keydup) 1053 return -ENOMEM; 1054 1055 ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG, 1056 crypto_req_done, &result); 1057 sg_init_one(&sg, keydup, keylen); 1058 ahash_request_set_crypt(areq, &sg, ipad, keylen); 1059 1060 ret = crypto_ahash_digest(areq); 1061 ret = crypto_wait_req(ret, &result); 1062 1063 /* Avoid leaking */ 1064 kfree_sensitive(keydup); 1065 1066 if (ret) 1067 return ret; 1068 1069 keylen = crypto_ahash_digestsize(crypto_ahash_reqtfm(areq)); 1070 } 1071 1072 memset(ipad + keylen, 0, blocksize - keylen); 1073 memcpy(opad, ipad, blocksize); 1074 1075 for (i = 0; i < blocksize; i++) { 1076 ipad[i] ^= HMAC_IPAD_VALUE; 1077 opad[i] ^= HMAC_OPAD_VALUE; 1078 } 1079 1080 return 0; 1081 } 1082 1083 static int safexcel_hmac_init_iv(struct ahash_request *areq, 1084 unsigned int blocksize, u8 *pad, void *state) 1085 { 1086 struct safexcel_ahash_req *req; 1087 DECLARE_CRYPTO_WAIT(result); 1088 struct scatterlist sg; 1089 int ret; 1090 1091 ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG, 1092 crypto_req_done, &result); 1093 sg_init_one(&sg, pad, blocksize); 1094 ahash_request_set_crypt(areq, &sg, pad, blocksize); 1095 1096 ret = crypto_ahash_init(areq); 1097 if (ret) 1098 return ret; 1099 1100 req = ahash_request_ctx_dma(areq); 1101 req->hmac = true; 1102 req->last_req = true; 1103 1104 ret = crypto_ahash_update(areq); 1105 ret = crypto_wait_req(ret, &result); 1106 1107 return ret ?: crypto_ahash_export(areq, state); 1108 } 1109 1110 static int __safexcel_hmac_setkey(const char *alg, const u8 *key, 1111 unsigned int keylen, 1112 void *istate, void *ostate) 1113 { 1114 struct ahash_request *areq; 1115 struct crypto_ahash *tfm; 1116 unsigned int blocksize; 1117 u8 *ipad, *opad; 1118 int ret; 1119 1120 tfm = crypto_alloc_ahash(alg, 0, 0); 1121 if (IS_ERR(tfm)) 1122 return PTR_ERR(tfm); 1123 1124 areq = ahash_request_alloc(tfm, GFP_KERNEL); 1125 if (!areq) { 1126 ret = -ENOMEM; 1127 goto free_ahash; 1128 } 1129 1130 crypto_ahash_clear_flags(tfm, ~0); 1131 blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 1132 1133 ipad = kcalloc(2, blocksize, GFP_KERNEL); 1134 if (!ipad) { 1135 ret = -ENOMEM; 1136 goto free_request; 1137 } 1138 1139 opad = ipad + blocksize; 1140 1141 ret = safexcel_hmac_init_pad(areq, blocksize, key, keylen, ipad, opad); 1142 if (ret) 1143 goto free_ipad; 1144 1145 ret = safexcel_hmac_init_iv(areq, blocksize, ipad, istate); 1146 if (ret) 1147 goto free_ipad; 1148 1149 ret = safexcel_hmac_init_iv(areq, blocksize, opad, ostate); 1150 1151 free_ipad: 1152 kfree(ipad); 1153 free_request: 1154 ahash_request_free(areq); 1155 free_ahash: 1156 crypto_free_ahash(tfm); 1157 1158 return ret; 1159 } 1160 1161 int safexcel_hmac_setkey(struct safexcel_context *base, const u8 *key, 1162 unsigned int keylen, const char *alg, 1163 unsigned int state_sz) 1164 { 1165 struct safexcel_crypto_priv *priv = base->priv; 1166 struct safexcel_ahash_export_state istate, ostate; 1167 int ret; 1168 1169 ret = __safexcel_hmac_setkey(alg, key, keylen, &istate, &ostate); 1170 if (ret) 1171 return ret; 1172 1173 if (priv->flags & EIP197_TRC_CACHE && base->ctxr && 1174 (memcmp(&base->ipad, istate.state, state_sz) || 1175 memcmp(&base->opad, ostate.state, state_sz))) 1176 base->needs_inv = true; 1177 1178 memcpy(&base->ipad, &istate.state, state_sz); 1179 memcpy(&base->opad, &ostate.state, state_sz); 1180 1181 return 0; 1182 } 1183 1184 static int safexcel_hmac_alg_setkey(struct crypto_ahash *tfm, const u8 *key, 1185 unsigned int keylen, const char *alg, 1186 unsigned int state_sz) 1187 { 1188 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 1189 1190 return safexcel_hmac_setkey(&ctx->base, key, keylen, alg, state_sz); 1191 } 1192 1193 static int safexcel_hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key, 1194 unsigned int keylen) 1195 { 1196 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha1", 1197 SHA1_DIGEST_SIZE); 1198 } 1199 1200 struct safexcel_alg_template safexcel_alg_hmac_sha1 = { 1201 .type = SAFEXCEL_ALG_TYPE_AHASH, 1202 .algo_mask = SAFEXCEL_ALG_SHA1, 1203 .alg.ahash = { 1204 .init = safexcel_hmac_sha1_init, 1205 .update = safexcel_ahash_update, 1206 .final = safexcel_ahash_final, 1207 .finup = safexcel_ahash_finup, 1208 .digest = safexcel_hmac_sha1_digest, 1209 .setkey = safexcel_hmac_sha1_setkey, 1210 .export = safexcel_ahash_export, 1211 .import = safexcel_ahash_import, 1212 .halg = { 1213 .digestsize = SHA1_DIGEST_SIZE, 1214 .statesize = sizeof(struct safexcel_ahash_export_state), 1215 .base = { 1216 .cra_name = "hmac(sha1)", 1217 .cra_driver_name = "safexcel-hmac-sha1", 1218 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1219 .cra_flags = CRYPTO_ALG_ASYNC | 1220 CRYPTO_ALG_ALLOCATES_MEMORY | 1221 CRYPTO_ALG_KERN_DRIVER_ONLY, 1222 .cra_blocksize = SHA1_BLOCK_SIZE, 1223 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1224 .cra_init = safexcel_ahash_cra_init, 1225 .cra_exit = safexcel_ahash_cra_exit, 1226 .cra_module = THIS_MODULE, 1227 }, 1228 }, 1229 }, 1230 }; 1231 1232 static int safexcel_sha256_init(struct ahash_request *areq) 1233 { 1234 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1235 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1236 1237 memset(req, 0, sizeof(*req)); 1238 1239 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256; 1240 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1241 req->state_sz = SHA256_DIGEST_SIZE; 1242 req->digest_sz = SHA256_DIGEST_SIZE; 1243 req->block_sz = SHA256_BLOCK_SIZE; 1244 1245 return 0; 1246 } 1247 1248 static int safexcel_sha256_digest(struct ahash_request *areq) 1249 { 1250 int ret = safexcel_sha256_init(areq); 1251 1252 if (ret) 1253 return ret; 1254 1255 return safexcel_ahash_finup(areq); 1256 } 1257 1258 struct safexcel_alg_template safexcel_alg_sha256 = { 1259 .type = SAFEXCEL_ALG_TYPE_AHASH, 1260 .algo_mask = SAFEXCEL_ALG_SHA2_256, 1261 .alg.ahash = { 1262 .init = safexcel_sha256_init, 1263 .update = safexcel_ahash_update, 1264 .final = safexcel_ahash_final, 1265 .finup = safexcel_ahash_finup, 1266 .digest = safexcel_sha256_digest, 1267 .export = safexcel_ahash_export, 1268 .import = safexcel_ahash_import, 1269 .halg = { 1270 .digestsize = SHA256_DIGEST_SIZE, 1271 .statesize = sizeof(struct safexcel_ahash_export_state), 1272 .base = { 1273 .cra_name = "sha256", 1274 .cra_driver_name = "safexcel-sha256", 1275 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1276 .cra_flags = CRYPTO_ALG_ASYNC | 1277 CRYPTO_ALG_ALLOCATES_MEMORY | 1278 CRYPTO_ALG_KERN_DRIVER_ONLY, 1279 .cra_blocksize = SHA256_BLOCK_SIZE, 1280 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1281 .cra_init = safexcel_ahash_cra_init, 1282 .cra_exit = safexcel_ahash_cra_exit, 1283 .cra_module = THIS_MODULE, 1284 }, 1285 }, 1286 }, 1287 }; 1288 1289 static int safexcel_sha224_init(struct ahash_request *areq) 1290 { 1291 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1292 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1293 1294 memset(req, 0, sizeof(*req)); 1295 1296 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224; 1297 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1298 req->state_sz = SHA256_DIGEST_SIZE; 1299 req->digest_sz = SHA256_DIGEST_SIZE; 1300 req->block_sz = SHA256_BLOCK_SIZE; 1301 1302 return 0; 1303 } 1304 1305 static int safexcel_sha224_digest(struct ahash_request *areq) 1306 { 1307 int ret = safexcel_sha224_init(areq); 1308 1309 if (ret) 1310 return ret; 1311 1312 return safexcel_ahash_finup(areq); 1313 } 1314 1315 struct safexcel_alg_template safexcel_alg_sha224 = { 1316 .type = SAFEXCEL_ALG_TYPE_AHASH, 1317 .algo_mask = SAFEXCEL_ALG_SHA2_256, 1318 .alg.ahash = { 1319 .init = safexcel_sha224_init, 1320 .update = safexcel_ahash_update, 1321 .final = safexcel_ahash_final, 1322 .finup = safexcel_ahash_finup, 1323 .digest = safexcel_sha224_digest, 1324 .export = safexcel_ahash_export, 1325 .import = safexcel_ahash_import, 1326 .halg = { 1327 .digestsize = SHA224_DIGEST_SIZE, 1328 .statesize = sizeof(struct safexcel_ahash_export_state), 1329 .base = { 1330 .cra_name = "sha224", 1331 .cra_driver_name = "safexcel-sha224", 1332 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1333 .cra_flags = CRYPTO_ALG_ASYNC | 1334 CRYPTO_ALG_ALLOCATES_MEMORY | 1335 CRYPTO_ALG_KERN_DRIVER_ONLY, 1336 .cra_blocksize = SHA224_BLOCK_SIZE, 1337 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1338 .cra_init = safexcel_ahash_cra_init, 1339 .cra_exit = safexcel_ahash_cra_exit, 1340 .cra_module = THIS_MODULE, 1341 }, 1342 }, 1343 }, 1344 }; 1345 1346 static int safexcel_hmac_sha224_setkey(struct crypto_ahash *tfm, const u8 *key, 1347 unsigned int keylen) 1348 { 1349 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha224", 1350 SHA256_DIGEST_SIZE); 1351 } 1352 1353 static int safexcel_hmac_sha224_init(struct ahash_request *areq) 1354 { 1355 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1356 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1357 1358 memset(req, 0, sizeof(*req)); 1359 1360 /* Start from ipad precompute */ 1361 memcpy(req->state, &ctx->base.ipad, SHA256_DIGEST_SIZE); 1362 /* Already processed the key^ipad part now! */ 1363 req->len = SHA256_BLOCK_SIZE; 1364 req->processed = SHA256_BLOCK_SIZE; 1365 1366 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224; 1367 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1368 req->state_sz = SHA256_DIGEST_SIZE; 1369 req->digest_sz = SHA256_DIGEST_SIZE; 1370 req->block_sz = SHA256_BLOCK_SIZE; 1371 req->hmac = true; 1372 1373 return 0; 1374 } 1375 1376 static int safexcel_hmac_sha224_digest(struct ahash_request *areq) 1377 { 1378 int ret = safexcel_hmac_sha224_init(areq); 1379 1380 if (ret) 1381 return ret; 1382 1383 return safexcel_ahash_finup(areq); 1384 } 1385 1386 struct safexcel_alg_template safexcel_alg_hmac_sha224 = { 1387 .type = SAFEXCEL_ALG_TYPE_AHASH, 1388 .algo_mask = SAFEXCEL_ALG_SHA2_256, 1389 .alg.ahash = { 1390 .init = safexcel_hmac_sha224_init, 1391 .update = safexcel_ahash_update, 1392 .final = safexcel_ahash_final, 1393 .finup = safexcel_ahash_finup, 1394 .digest = safexcel_hmac_sha224_digest, 1395 .setkey = safexcel_hmac_sha224_setkey, 1396 .export = safexcel_ahash_export, 1397 .import = safexcel_ahash_import, 1398 .halg = { 1399 .digestsize = SHA224_DIGEST_SIZE, 1400 .statesize = sizeof(struct safexcel_ahash_export_state), 1401 .base = { 1402 .cra_name = "hmac(sha224)", 1403 .cra_driver_name = "safexcel-hmac-sha224", 1404 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1405 .cra_flags = CRYPTO_ALG_ASYNC | 1406 CRYPTO_ALG_ALLOCATES_MEMORY | 1407 CRYPTO_ALG_KERN_DRIVER_ONLY, 1408 .cra_blocksize = SHA224_BLOCK_SIZE, 1409 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1410 .cra_init = safexcel_ahash_cra_init, 1411 .cra_exit = safexcel_ahash_cra_exit, 1412 .cra_module = THIS_MODULE, 1413 }, 1414 }, 1415 }, 1416 }; 1417 1418 static int safexcel_hmac_sha256_setkey(struct crypto_ahash *tfm, const u8 *key, 1419 unsigned int keylen) 1420 { 1421 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha256", 1422 SHA256_DIGEST_SIZE); 1423 } 1424 1425 static int safexcel_hmac_sha256_init(struct ahash_request *areq) 1426 { 1427 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1428 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1429 1430 memset(req, 0, sizeof(*req)); 1431 1432 /* Start from ipad precompute */ 1433 memcpy(req->state, &ctx->base.ipad, SHA256_DIGEST_SIZE); 1434 /* Already processed the key^ipad part now! */ 1435 req->len = SHA256_BLOCK_SIZE; 1436 req->processed = SHA256_BLOCK_SIZE; 1437 1438 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256; 1439 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1440 req->state_sz = SHA256_DIGEST_SIZE; 1441 req->digest_sz = SHA256_DIGEST_SIZE; 1442 req->block_sz = SHA256_BLOCK_SIZE; 1443 req->hmac = true; 1444 1445 return 0; 1446 } 1447 1448 static int safexcel_hmac_sha256_digest(struct ahash_request *areq) 1449 { 1450 int ret = safexcel_hmac_sha256_init(areq); 1451 1452 if (ret) 1453 return ret; 1454 1455 return safexcel_ahash_finup(areq); 1456 } 1457 1458 struct safexcel_alg_template safexcel_alg_hmac_sha256 = { 1459 .type = SAFEXCEL_ALG_TYPE_AHASH, 1460 .algo_mask = SAFEXCEL_ALG_SHA2_256, 1461 .alg.ahash = { 1462 .init = safexcel_hmac_sha256_init, 1463 .update = safexcel_ahash_update, 1464 .final = safexcel_ahash_final, 1465 .finup = safexcel_ahash_finup, 1466 .digest = safexcel_hmac_sha256_digest, 1467 .setkey = safexcel_hmac_sha256_setkey, 1468 .export = safexcel_ahash_export, 1469 .import = safexcel_ahash_import, 1470 .halg = { 1471 .digestsize = SHA256_DIGEST_SIZE, 1472 .statesize = sizeof(struct safexcel_ahash_export_state), 1473 .base = { 1474 .cra_name = "hmac(sha256)", 1475 .cra_driver_name = "safexcel-hmac-sha256", 1476 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1477 .cra_flags = CRYPTO_ALG_ASYNC | 1478 CRYPTO_ALG_ALLOCATES_MEMORY | 1479 CRYPTO_ALG_KERN_DRIVER_ONLY, 1480 .cra_blocksize = SHA256_BLOCK_SIZE, 1481 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1482 .cra_init = safexcel_ahash_cra_init, 1483 .cra_exit = safexcel_ahash_cra_exit, 1484 .cra_module = THIS_MODULE, 1485 }, 1486 }, 1487 }, 1488 }; 1489 1490 static int safexcel_sha512_init(struct ahash_request *areq) 1491 { 1492 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1493 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1494 1495 memset(req, 0, sizeof(*req)); 1496 1497 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512; 1498 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1499 req->state_sz = SHA512_DIGEST_SIZE; 1500 req->digest_sz = SHA512_DIGEST_SIZE; 1501 req->block_sz = SHA512_BLOCK_SIZE; 1502 1503 return 0; 1504 } 1505 1506 static int safexcel_sha512_digest(struct ahash_request *areq) 1507 { 1508 int ret = safexcel_sha512_init(areq); 1509 1510 if (ret) 1511 return ret; 1512 1513 return safexcel_ahash_finup(areq); 1514 } 1515 1516 struct safexcel_alg_template safexcel_alg_sha512 = { 1517 .type = SAFEXCEL_ALG_TYPE_AHASH, 1518 .algo_mask = SAFEXCEL_ALG_SHA2_512, 1519 .alg.ahash = { 1520 .init = safexcel_sha512_init, 1521 .update = safexcel_ahash_update, 1522 .final = safexcel_ahash_final, 1523 .finup = safexcel_ahash_finup, 1524 .digest = safexcel_sha512_digest, 1525 .export = safexcel_ahash_export, 1526 .import = safexcel_ahash_import, 1527 .halg = { 1528 .digestsize = SHA512_DIGEST_SIZE, 1529 .statesize = sizeof(struct safexcel_ahash_export_state), 1530 .base = { 1531 .cra_name = "sha512", 1532 .cra_driver_name = "safexcel-sha512", 1533 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1534 .cra_flags = CRYPTO_ALG_ASYNC | 1535 CRYPTO_ALG_ALLOCATES_MEMORY | 1536 CRYPTO_ALG_KERN_DRIVER_ONLY, 1537 .cra_blocksize = SHA512_BLOCK_SIZE, 1538 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1539 .cra_init = safexcel_ahash_cra_init, 1540 .cra_exit = safexcel_ahash_cra_exit, 1541 .cra_module = THIS_MODULE, 1542 }, 1543 }, 1544 }, 1545 }; 1546 1547 static int safexcel_sha384_init(struct ahash_request *areq) 1548 { 1549 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1550 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1551 1552 memset(req, 0, sizeof(*req)); 1553 1554 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384; 1555 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1556 req->state_sz = SHA512_DIGEST_SIZE; 1557 req->digest_sz = SHA512_DIGEST_SIZE; 1558 req->block_sz = SHA512_BLOCK_SIZE; 1559 1560 return 0; 1561 } 1562 1563 static int safexcel_sha384_digest(struct ahash_request *areq) 1564 { 1565 int ret = safexcel_sha384_init(areq); 1566 1567 if (ret) 1568 return ret; 1569 1570 return safexcel_ahash_finup(areq); 1571 } 1572 1573 struct safexcel_alg_template safexcel_alg_sha384 = { 1574 .type = SAFEXCEL_ALG_TYPE_AHASH, 1575 .algo_mask = SAFEXCEL_ALG_SHA2_512, 1576 .alg.ahash = { 1577 .init = safexcel_sha384_init, 1578 .update = safexcel_ahash_update, 1579 .final = safexcel_ahash_final, 1580 .finup = safexcel_ahash_finup, 1581 .digest = safexcel_sha384_digest, 1582 .export = safexcel_ahash_export, 1583 .import = safexcel_ahash_import, 1584 .halg = { 1585 .digestsize = SHA384_DIGEST_SIZE, 1586 .statesize = sizeof(struct safexcel_ahash_export_state), 1587 .base = { 1588 .cra_name = "sha384", 1589 .cra_driver_name = "safexcel-sha384", 1590 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1591 .cra_flags = CRYPTO_ALG_ASYNC | 1592 CRYPTO_ALG_ALLOCATES_MEMORY | 1593 CRYPTO_ALG_KERN_DRIVER_ONLY, 1594 .cra_blocksize = SHA384_BLOCK_SIZE, 1595 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1596 .cra_init = safexcel_ahash_cra_init, 1597 .cra_exit = safexcel_ahash_cra_exit, 1598 .cra_module = THIS_MODULE, 1599 }, 1600 }, 1601 }, 1602 }; 1603 1604 static int safexcel_hmac_sha512_setkey(struct crypto_ahash *tfm, const u8 *key, 1605 unsigned int keylen) 1606 { 1607 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha512", 1608 SHA512_DIGEST_SIZE); 1609 } 1610 1611 static int safexcel_hmac_sha512_init(struct ahash_request *areq) 1612 { 1613 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1614 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1615 1616 memset(req, 0, sizeof(*req)); 1617 1618 /* Start from ipad precompute */ 1619 memcpy(req->state, &ctx->base.ipad, SHA512_DIGEST_SIZE); 1620 /* Already processed the key^ipad part now! */ 1621 req->len = SHA512_BLOCK_SIZE; 1622 req->processed = SHA512_BLOCK_SIZE; 1623 1624 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512; 1625 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1626 req->state_sz = SHA512_DIGEST_SIZE; 1627 req->digest_sz = SHA512_DIGEST_SIZE; 1628 req->block_sz = SHA512_BLOCK_SIZE; 1629 req->hmac = true; 1630 1631 return 0; 1632 } 1633 1634 static int safexcel_hmac_sha512_digest(struct ahash_request *areq) 1635 { 1636 int ret = safexcel_hmac_sha512_init(areq); 1637 1638 if (ret) 1639 return ret; 1640 1641 return safexcel_ahash_finup(areq); 1642 } 1643 1644 struct safexcel_alg_template safexcel_alg_hmac_sha512 = { 1645 .type = SAFEXCEL_ALG_TYPE_AHASH, 1646 .algo_mask = SAFEXCEL_ALG_SHA2_512, 1647 .alg.ahash = { 1648 .init = safexcel_hmac_sha512_init, 1649 .update = safexcel_ahash_update, 1650 .final = safexcel_ahash_final, 1651 .finup = safexcel_ahash_finup, 1652 .digest = safexcel_hmac_sha512_digest, 1653 .setkey = safexcel_hmac_sha512_setkey, 1654 .export = safexcel_ahash_export, 1655 .import = safexcel_ahash_import, 1656 .halg = { 1657 .digestsize = SHA512_DIGEST_SIZE, 1658 .statesize = sizeof(struct safexcel_ahash_export_state), 1659 .base = { 1660 .cra_name = "hmac(sha512)", 1661 .cra_driver_name = "safexcel-hmac-sha512", 1662 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1663 .cra_flags = CRYPTO_ALG_ASYNC | 1664 CRYPTO_ALG_ALLOCATES_MEMORY | 1665 CRYPTO_ALG_KERN_DRIVER_ONLY, 1666 .cra_blocksize = SHA512_BLOCK_SIZE, 1667 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1668 .cra_init = safexcel_ahash_cra_init, 1669 .cra_exit = safexcel_ahash_cra_exit, 1670 .cra_module = THIS_MODULE, 1671 }, 1672 }, 1673 }, 1674 }; 1675 1676 static int safexcel_hmac_sha384_setkey(struct crypto_ahash *tfm, const u8 *key, 1677 unsigned int keylen) 1678 { 1679 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha384", 1680 SHA512_DIGEST_SIZE); 1681 } 1682 1683 static int safexcel_hmac_sha384_init(struct ahash_request *areq) 1684 { 1685 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1686 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1687 1688 memset(req, 0, sizeof(*req)); 1689 1690 /* Start from ipad precompute */ 1691 memcpy(req->state, &ctx->base.ipad, SHA512_DIGEST_SIZE); 1692 /* Already processed the key^ipad part now! */ 1693 req->len = SHA512_BLOCK_SIZE; 1694 req->processed = SHA512_BLOCK_SIZE; 1695 1696 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384; 1697 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1698 req->state_sz = SHA512_DIGEST_SIZE; 1699 req->digest_sz = SHA512_DIGEST_SIZE; 1700 req->block_sz = SHA512_BLOCK_SIZE; 1701 req->hmac = true; 1702 1703 return 0; 1704 } 1705 1706 static int safexcel_hmac_sha384_digest(struct ahash_request *areq) 1707 { 1708 int ret = safexcel_hmac_sha384_init(areq); 1709 1710 if (ret) 1711 return ret; 1712 1713 return safexcel_ahash_finup(areq); 1714 } 1715 1716 struct safexcel_alg_template safexcel_alg_hmac_sha384 = { 1717 .type = SAFEXCEL_ALG_TYPE_AHASH, 1718 .algo_mask = SAFEXCEL_ALG_SHA2_512, 1719 .alg.ahash = { 1720 .init = safexcel_hmac_sha384_init, 1721 .update = safexcel_ahash_update, 1722 .final = safexcel_ahash_final, 1723 .finup = safexcel_ahash_finup, 1724 .digest = safexcel_hmac_sha384_digest, 1725 .setkey = safexcel_hmac_sha384_setkey, 1726 .export = safexcel_ahash_export, 1727 .import = safexcel_ahash_import, 1728 .halg = { 1729 .digestsize = SHA384_DIGEST_SIZE, 1730 .statesize = sizeof(struct safexcel_ahash_export_state), 1731 .base = { 1732 .cra_name = "hmac(sha384)", 1733 .cra_driver_name = "safexcel-hmac-sha384", 1734 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1735 .cra_flags = CRYPTO_ALG_ASYNC | 1736 CRYPTO_ALG_ALLOCATES_MEMORY | 1737 CRYPTO_ALG_KERN_DRIVER_ONLY, 1738 .cra_blocksize = SHA384_BLOCK_SIZE, 1739 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1740 .cra_init = safexcel_ahash_cra_init, 1741 .cra_exit = safexcel_ahash_cra_exit, 1742 .cra_module = THIS_MODULE, 1743 }, 1744 }, 1745 }, 1746 }; 1747 1748 static int safexcel_md5_init(struct ahash_request *areq) 1749 { 1750 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1751 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1752 1753 memset(req, 0, sizeof(*req)); 1754 1755 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5; 1756 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1757 req->state_sz = MD5_DIGEST_SIZE; 1758 req->digest_sz = MD5_DIGEST_SIZE; 1759 req->block_sz = MD5_HMAC_BLOCK_SIZE; 1760 1761 return 0; 1762 } 1763 1764 static int safexcel_md5_digest(struct ahash_request *areq) 1765 { 1766 int ret = safexcel_md5_init(areq); 1767 1768 if (ret) 1769 return ret; 1770 1771 return safexcel_ahash_finup(areq); 1772 } 1773 1774 struct safexcel_alg_template safexcel_alg_md5 = { 1775 .type = SAFEXCEL_ALG_TYPE_AHASH, 1776 .algo_mask = SAFEXCEL_ALG_MD5, 1777 .alg.ahash = { 1778 .init = safexcel_md5_init, 1779 .update = safexcel_ahash_update, 1780 .final = safexcel_ahash_final, 1781 .finup = safexcel_ahash_finup, 1782 .digest = safexcel_md5_digest, 1783 .export = safexcel_ahash_export, 1784 .import = safexcel_ahash_import, 1785 .halg = { 1786 .digestsize = MD5_DIGEST_SIZE, 1787 .statesize = sizeof(struct safexcel_ahash_export_state), 1788 .base = { 1789 .cra_name = "md5", 1790 .cra_driver_name = "safexcel-md5", 1791 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1792 .cra_flags = CRYPTO_ALG_ASYNC | 1793 CRYPTO_ALG_ALLOCATES_MEMORY | 1794 CRYPTO_ALG_KERN_DRIVER_ONLY, 1795 .cra_blocksize = MD5_HMAC_BLOCK_SIZE, 1796 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1797 .cra_init = safexcel_ahash_cra_init, 1798 .cra_exit = safexcel_ahash_cra_exit, 1799 .cra_module = THIS_MODULE, 1800 }, 1801 }, 1802 }, 1803 }; 1804 1805 static int safexcel_hmac_md5_init(struct ahash_request *areq) 1806 { 1807 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1808 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1809 1810 memset(req, 0, sizeof(*req)); 1811 1812 /* Start from ipad precompute */ 1813 memcpy(req->state, &ctx->base.ipad, MD5_DIGEST_SIZE); 1814 /* Already processed the key^ipad part now! */ 1815 req->len = MD5_HMAC_BLOCK_SIZE; 1816 req->processed = MD5_HMAC_BLOCK_SIZE; 1817 1818 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5; 1819 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1820 req->state_sz = MD5_DIGEST_SIZE; 1821 req->digest_sz = MD5_DIGEST_SIZE; 1822 req->block_sz = MD5_HMAC_BLOCK_SIZE; 1823 req->len_is_le = true; /* MD5 is little endian! ... */ 1824 req->hmac = true; 1825 1826 return 0; 1827 } 1828 1829 static int safexcel_hmac_md5_setkey(struct crypto_ahash *tfm, const u8 *key, 1830 unsigned int keylen) 1831 { 1832 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-md5", 1833 MD5_DIGEST_SIZE); 1834 } 1835 1836 static int safexcel_hmac_md5_digest(struct ahash_request *areq) 1837 { 1838 int ret = safexcel_hmac_md5_init(areq); 1839 1840 if (ret) 1841 return ret; 1842 1843 return safexcel_ahash_finup(areq); 1844 } 1845 1846 struct safexcel_alg_template safexcel_alg_hmac_md5 = { 1847 .type = SAFEXCEL_ALG_TYPE_AHASH, 1848 .algo_mask = SAFEXCEL_ALG_MD5, 1849 .alg.ahash = { 1850 .init = safexcel_hmac_md5_init, 1851 .update = safexcel_ahash_update, 1852 .final = safexcel_ahash_final, 1853 .finup = safexcel_ahash_finup, 1854 .digest = safexcel_hmac_md5_digest, 1855 .setkey = safexcel_hmac_md5_setkey, 1856 .export = safexcel_ahash_export, 1857 .import = safexcel_ahash_import, 1858 .halg = { 1859 .digestsize = MD5_DIGEST_SIZE, 1860 .statesize = sizeof(struct safexcel_ahash_export_state), 1861 .base = { 1862 .cra_name = "hmac(md5)", 1863 .cra_driver_name = "safexcel-hmac-md5", 1864 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1865 .cra_flags = CRYPTO_ALG_ASYNC | 1866 CRYPTO_ALG_ALLOCATES_MEMORY | 1867 CRYPTO_ALG_KERN_DRIVER_ONLY, 1868 .cra_blocksize = MD5_HMAC_BLOCK_SIZE, 1869 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1870 .cra_init = safexcel_ahash_cra_init, 1871 .cra_exit = safexcel_ahash_cra_exit, 1872 .cra_module = THIS_MODULE, 1873 }, 1874 }, 1875 }, 1876 }; 1877 1878 static int safexcel_cbcmac_init(struct ahash_request *areq) 1879 { 1880 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1881 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1882 1883 memset(req, 0, sizeof(*req)); 1884 1885 /* Start from loaded keys */ 1886 memcpy(req->state, &ctx->base.ipad, ctx->key_sz); 1887 /* Set processed to non-zero to enable invalidation detection */ 1888 req->len = AES_BLOCK_SIZE; 1889 req->processed = AES_BLOCK_SIZE; 1890 1891 req->digest = CONTEXT_CONTROL_DIGEST_XCM; 1892 req->state_sz = ctx->key_sz; 1893 req->digest_sz = AES_BLOCK_SIZE; 1894 req->block_sz = AES_BLOCK_SIZE; 1895 req->xcbcmac = true; 1896 1897 return 0; 1898 } 1899 1900 static int safexcel_cbcmac_setkey(struct crypto_ahash *tfm, const u8 *key, 1901 unsigned int len) 1902 { 1903 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 1904 struct crypto_aes_ctx aes; 1905 int ret, i; 1906 1907 ret = aes_expandkey(&aes, key, len); 1908 if (ret) 1909 return ret; 1910 1911 memset(&ctx->base.ipad, 0, 2 * AES_BLOCK_SIZE); 1912 for (i = 0; i < len / sizeof(u32); i++) 1913 ctx->base.ipad.be[i + 8] = cpu_to_be32(aes.key_enc[i]); 1914 1915 if (len == AES_KEYSIZE_192) { 1916 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192; 1917 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE; 1918 } else if (len == AES_KEYSIZE_256) { 1919 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256; 1920 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE; 1921 } else { 1922 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128; 1923 ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE; 1924 } 1925 ctx->cbcmac = true; 1926 1927 memzero_explicit(&aes, sizeof(aes)); 1928 return 0; 1929 } 1930 1931 static int safexcel_cbcmac_digest(struct ahash_request *areq) 1932 { 1933 return safexcel_cbcmac_init(areq) ?: safexcel_ahash_finup(areq); 1934 } 1935 1936 struct safexcel_alg_template safexcel_alg_cbcmac = { 1937 .type = SAFEXCEL_ALG_TYPE_AHASH, 1938 .algo_mask = 0, 1939 .alg.ahash = { 1940 .init = safexcel_cbcmac_init, 1941 .update = safexcel_ahash_update, 1942 .final = safexcel_ahash_final, 1943 .finup = safexcel_ahash_finup, 1944 .digest = safexcel_cbcmac_digest, 1945 .setkey = safexcel_cbcmac_setkey, 1946 .export = safexcel_ahash_export, 1947 .import = safexcel_ahash_import, 1948 .halg = { 1949 .digestsize = AES_BLOCK_SIZE, 1950 .statesize = sizeof(struct safexcel_ahash_export_state), 1951 .base = { 1952 .cra_name = "cbcmac(aes)", 1953 .cra_driver_name = "safexcel-cbcmac-aes", 1954 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1955 .cra_flags = CRYPTO_ALG_ASYNC | 1956 CRYPTO_ALG_ALLOCATES_MEMORY | 1957 CRYPTO_ALG_KERN_DRIVER_ONLY, 1958 .cra_blocksize = AES_BLOCK_SIZE, 1959 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1960 .cra_init = safexcel_ahash_cra_init, 1961 .cra_exit = safexcel_ahash_cra_exit, 1962 .cra_module = THIS_MODULE, 1963 }, 1964 }, 1965 }, 1966 }; 1967 1968 static int safexcel_xcbcmac_setkey(struct crypto_ahash *tfm, const u8 *key, 1969 unsigned int len) 1970 { 1971 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 1972 u32 key_tmp[3 * AES_BLOCK_SIZE / sizeof(u32)]; 1973 int ret, i; 1974 1975 ret = aes_expandkey(ctx->aes, key, len); 1976 if (ret) 1977 return ret; 1978 1979 /* precompute the XCBC key material */ 1980 aes_encrypt(ctx->aes, (u8 *)key_tmp + 2 * AES_BLOCK_SIZE, 1981 "\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1"); 1982 aes_encrypt(ctx->aes, (u8 *)key_tmp, 1983 "\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2"); 1984 aes_encrypt(ctx->aes, (u8 *)key_tmp + AES_BLOCK_SIZE, 1985 "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3"); 1986 for (i = 0; i < 3 * AES_BLOCK_SIZE / sizeof(u32); i++) 1987 ctx->base.ipad.word[i] = swab32(key_tmp[i]); 1988 1989 ret = aes_expandkey(ctx->aes, 1990 (u8 *)key_tmp + 2 * AES_BLOCK_SIZE, 1991 AES_MIN_KEY_SIZE); 1992 if (ret) 1993 return ret; 1994 1995 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128; 1996 ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE; 1997 ctx->cbcmac = false; 1998 1999 return 0; 2000 } 2001 2002 static int safexcel_xcbcmac_cra_init(struct crypto_tfm *tfm) 2003 { 2004 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 2005 2006 safexcel_ahash_cra_init(tfm); 2007 ctx->aes = kmalloc(sizeof(*ctx->aes), GFP_KERNEL); 2008 return ctx->aes == NULL ? -ENOMEM : 0; 2009 } 2010 2011 static void safexcel_xcbcmac_cra_exit(struct crypto_tfm *tfm) 2012 { 2013 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 2014 2015 kfree(ctx->aes); 2016 safexcel_ahash_cra_exit(tfm); 2017 } 2018 2019 struct safexcel_alg_template safexcel_alg_xcbcmac = { 2020 .type = SAFEXCEL_ALG_TYPE_AHASH, 2021 .algo_mask = 0, 2022 .alg.ahash = { 2023 .init = safexcel_cbcmac_init, 2024 .update = safexcel_ahash_update, 2025 .final = safexcel_ahash_final, 2026 .finup = safexcel_ahash_finup, 2027 .digest = safexcel_cbcmac_digest, 2028 .setkey = safexcel_xcbcmac_setkey, 2029 .export = safexcel_ahash_export, 2030 .import = safexcel_ahash_import, 2031 .halg = { 2032 .digestsize = AES_BLOCK_SIZE, 2033 .statesize = sizeof(struct safexcel_ahash_export_state), 2034 .base = { 2035 .cra_name = "xcbc(aes)", 2036 .cra_driver_name = "safexcel-xcbc-aes", 2037 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2038 .cra_flags = CRYPTO_ALG_ASYNC | 2039 CRYPTO_ALG_ALLOCATES_MEMORY | 2040 CRYPTO_ALG_KERN_DRIVER_ONLY, 2041 .cra_blocksize = AES_BLOCK_SIZE, 2042 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 2043 .cra_init = safexcel_xcbcmac_cra_init, 2044 .cra_exit = safexcel_xcbcmac_cra_exit, 2045 .cra_module = THIS_MODULE, 2046 }, 2047 }, 2048 }, 2049 }; 2050 2051 static int safexcel_cmac_setkey(struct crypto_ahash *tfm, const u8 *key, 2052 unsigned int len) 2053 { 2054 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 2055 __be64 consts[4]; 2056 u64 _const[2]; 2057 u8 msb_mask, gfmask; 2058 int ret, i; 2059 2060 /* precompute the CMAC key material */ 2061 ret = aes_expandkey(ctx->aes, key, len); 2062 if (ret) 2063 return ret; 2064 2065 for (i = 0; i < len / sizeof(u32); i++) 2066 ctx->base.ipad.word[i + 8] = swab32(ctx->aes->key_enc[i]); 2067 2068 /* code below borrowed from crypto/cmac.c */ 2069 /* encrypt the zero block */ 2070 memset(consts, 0, AES_BLOCK_SIZE); 2071 aes_encrypt(ctx->aes, (u8 *)consts, (u8 *)consts); 2072 2073 gfmask = 0x87; 2074 _const[0] = be64_to_cpu(consts[1]); 2075 _const[1] = be64_to_cpu(consts[0]); 2076 2077 /* gf(2^128) multiply zero-ciphertext with u and u^2 */ 2078 for (i = 0; i < 4; i += 2) { 2079 msb_mask = ((s64)_const[1] >> 63) & gfmask; 2080 _const[1] = (_const[1] << 1) | (_const[0] >> 63); 2081 _const[0] = (_const[0] << 1) ^ msb_mask; 2082 2083 consts[i + 0] = cpu_to_be64(_const[1]); 2084 consts[i + 1] = cpu_to_be64(_const[0]); 2085 } 2086 /* end of code borrowed from crypto/cmac.c */ 2087 2088 for (i = 0; i < 2 * AES_BLOCK_SIZE / sizeof(u32); i++) 2089 ctx->base.ipad.be[i] = cpu_to_be32(((u32 *)consts)[i]); 2090 2091 if (len == AES_KEYSIZE_192) { 2092 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192; 2093 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE; 2094 } else if (len == AES_KEYSIZE_256) { 2095 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256; 2096 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE; 2097 } else { 2098 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128; 2099 ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE; 2100 } 2101 ctx->cbcmac = false; 2102 2103 return 0; 2104 } 2105 2106 struct safexcel_alg_template safexcel_alg_cmac = { 2107 .type = SAFEXCEL_ALG_TYPE_AHASH, 2108 .algo_mask = 0, 2109 .alg.ahash = { 2110 .init = safexcel_cbcmac_init, 2111 .update = safexcel_ahash_update, 2112 .final = safexcel_ahash_final, 2113 .finup = safexcel_ahash_finup, 2114 .digest = safexcel_cbcmac_digest, 2115 .setkey = safexcel_cmac_setkey, 2116 .export = safexcel_ahash_export, 2117 .import = safexcel_ahash_import, 2118 .halg = { 2119 .digestsize = AES_BLOCK_SIZE, 2120 .statesize = sizeof(struct safexcel_ahash_export_state), 2121 .base = { 2122 .cra_name = "cmac(aes)", 2123 .cra_driver_name = "safexcel-cmac-aes", 2124 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2125 .cra_flags = CRYPTO_ALG_ASYNC | 2126 CRYPTO_ALG_ALLOCATES_MEMORY | 2127 CRYPTO_ALG_KERN_DRIVER_ONLY, 2128 .cra_blocksize = AES_BLOCK_SIZE, 2129 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 2130 .cra_init = safexcel_xcbcmac_cra_init, 2131 .cra_exit = safexcel_xcbcmac_cra_exit, 2132 .cra_module = THIS_MODULE, 2133 }, 2134 }, 2135 }, 2136 }; 2137 2138 static int safexcel_sm3_init(struct ahash_request *areq) 2139 { 2140 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 2141 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 2142 2143 memset(req, 0, sizeof(*req)); 2144 2145 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3; 2146 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 2147 req->state_sz = SM3_DIGEST_SIZE; 2148 req->digest_sz = SM3_DIGEST_SIZE; 2149 req->block_sz = SM3_BLOCK_SIZE; 2150 2151 return 0; 2152 } 2153 2154 static int safexcel_sm3_digest(struct ahash_request *areq) 2155 { 2156 int ret = safexcel_sm3_init(areq); 2157 2158 if (ret) 2159 return ret; 2160 2161 return safexcel_ahash_finup(areq); 2162 } 2163 2164 struct safexcel_alg_template safexcel_alg_sm3 = { 2165 .type = SAFEXCEL_ALG_TYPE_AHASH, 2166 .algo_mask = SAFEXCEL_ALG_SM3, 2167 .alg.ahash = { 2168 .init = safexcel_sm3_init, 2169 .update = safexcel_ahash_update, 2170 .final = safexcel_ahash_final, 2171 .finup = safexcel_ahash_finup, 2172 .digest = safexcel_sm3_digest, 2173 .export = safexcel_ahash_export, 2174 .import = safexcel_ahash_import, 2175 .halg = { 2176 .digestsize = SM3_DIGEST_SIZE, 2177 .statesize = sizeof(struct safexcel_ahash_export_state), 2178 .base = { 2179 .cra_name = "sm3", 2180 .cra_driver_name = "safexcel-sm3", 2181 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2182 .cra_flags = CRYPTO_ALG_ASYNC | 2183 CRYPTO_ALG_ALLOCATES_MEMORY | 2184 CRYPTO_ALG_KERN_DRIVER_ONLY, 2185 .cra_blocksize = SM3_BLOCK_SIZE, 2186 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 2187 .cra_init = safexcel_ahash_cra_init, 2188 .cra_exit = safexcel_ahash_cra_exit, 2189 .cra_module = THIS_MODULE, 2190 }, 2191 }, 2192 }, 2193 }; 2194 2195 static int safexcel_hmac_sm3_setkey(struct crypto_ahash *tfm, const u8 *key, 2196 unsigned int keylen) 2197 { 2198 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sm3", 2199 SM3_DIGEST_SIZE); 2200 } 2201 2202 static int safexcel_hmac_sm3_init(struct ahash_request *areq) 2203 { 2204 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 2205 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 2206 2207 memset(req, 0, sizeof(*req)); 2208 2209 /* Start from ipad precompute */ 2210 memcpy(req->state, &ctx->base.ipad, SM3_DIGEST_SIZE); 2211 /* Already processed the key^ipad part now! */ 2212 req->len = SM3_BLOCK_SIZE; 2213 req->processed = SM3_BLOCK_SIZE; 2214 2215 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3; 2216 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 2217 req->state_sz = SM3_DIGEST_SIZE; 2218 req->digest_sz = SM3_DIGEST_SIZE; 2219 req->block_sz = SM3_BLOCK_SIZE; 2220 req->hmac = true; 2221 2222 return 0; 2223 } 2224 2225 static int safexcel_hmac_sm3_digest(struct ahash_request *areq) 2226 { 2227 int ret = safexcel_hmac_sm3_init(areq); 2228 2229 if (ret) 2230 return ret; 2231 2232 return safexcel_ahash_finup(areq); 2233 } 2234 2235 struct safexcel_alg_template safexcel_alg_hmac_sm3 = { 2236 .type = SAFEXCEL_ALG_TYPE_AHASH, 2237 .algo_mask = SAFEXCEL_ALG_SM3, 2238 .alg.ahash = { 2239 .init = safexcel_hmac_sm3_init, 2240 .update = safexcel_ahash_update, 2241 .final = safexcel_ahash_final, 2242 .finup = safexcel_ahash_finup, 2243 .digest = safexcel_hmac_sm3_digest, 2244 .setkey = safexcel_hmac_sm3_setkey, 2245 .export = safexcel_ahash_export, 2246 .import = safexcel_ahash_import, 2247 .halg = { 2248 .digestsize = SM3_DIGEST_SIZE, 2249 .statesize = sizeof(struct safexcel_ahash_export_state), 2250 .base = { 2251 .cra_name = "hmac(sm3)", 2252 .cra_driver_name = "safexcel-hmac-sm3", 2253 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2254 .cra_flags = CRYPTO_ALG_ASYNC | 2255 CRYPTO_ALG_ALLOCATES_MEMORY | 2256 CRYPTO_ALG_KERN_DRIVER_ONLY, 2257 .cra_blocksize = SM3_BLOCK_SIZE, 2258 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 2259 .cra_init = safexcel_ahash_cra_init, 2260 .cra_exit = safexcel_ahash_cra_exit, 2261 .cra_module = THIS_MODULE, 2262 }, 2263 }, 2264 }, 2265 }; 2266 2267 static int safexcel_sha3_224_init(struct ahash_request *areq) 2268 { 2269 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2270 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2271 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 2272 2273 memset(req, 0, sizeof(*req)); 2274 2275 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_224; 2276 req->digest = CONTEXT_CONTROL_DIGEST_INITIAL; 2277 req->state_sz = SHA3_224_DIGEST_SIZE; 2278 req->digest_sz = SHA3_224_DIGEST_SIZE; 2279 req->block_sz = SHA3_224_BLOCK_SIZE; 2280 ctx->do_fallback = false; 2281 ctx->fb_init_done = false; 2282 return 0; 2283 } 2284 2285 static int safexcel_sha3_fbcheck(struct ahash_request *req) 2286 { 2287 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 2288 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2289 struct ahash_request *subreq = ahash_request_ctx_dma(req); 2290 int ret = 0; 2291 2292 if (ctx->do_fallback) { 2293 ahash_request_set_tfm(subreq, ctx->fback); 2294 ahash_request_set_callback(subreq, req->base.flags, 2295 req->base.complete, req->base.data); 2296 ahash_request_set_crypt(subreq, req->src, req->result, 2297 req->nbytes); 2298 if (!ctx->fb_init_done) { 2299 if (ctx->fb_do_setkey) { 2300 /* Set fallback cipher HMAC key */ 2301 u8 key[SHA3_224_BLOCK_SIZE]; 2302 2303 memcpy(key, &ctx->base.ipad, 2304 crypto_ahash_blocksize(ctx->fback) / 2); 2305 memcpy(key + 2306 crypto_ahash_blocksize(ctx->fback) / 2, 2307 &ctx->base.opad, 2308 crypto_ahash_blocksize(ctx->fback) / 2); 2309 ret = crypto_ahash_setkey(ctx->fback, key, 2310 crypto_ahash_blocksize(ctx->fback)); 2311 memzero_explicit(key, 2312 crypto_ahash_blocksize(ctx->fback)); 2313 ctx->fb_do_setkey = false; 2314 } 2315 ret = ret ?: crypto_ahash_init(subreq); 2316 ctx->fb_init_done = true; 2317 } 2318 } 2319 return ret; 2320 } 2321 2322 static int safexcel_sha3_update(struct ahash_request *req) 2323 { 2324 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 2325 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2326 struct ahash_request *subreq = ahash_request_ctx_dma(req); 2327 2328 ctx->do_fallback = true; 2329 return safexcel_sha3_fbcheck(req) ?: crypto_ahash_update(subreq); 2330 } 2331 2332 static int safexcel_sha3_final(struct ahash_request *req) 2333 { 2334 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 2335 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2336 struct ahash_request *subreq = ahash_request_ctx_dma(req); 2337 2338 ctx->do_fallback = true; 2339 return safexcel_sha3_fbcheck(req) ?: crypto_ahash_final(subreq); 2340 } 2341 2342 static int safexcel_sha3_finup(struct ahash_request *req) 2343 { 2344 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 2345 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2346 struct ahash_request *subreq = ahash_request_ctx_dma(req); 2347 2348 ctx->do_fallback |= !req->nbytes; 2349 if (ctx->do_fallback) 2350 /* Update or ex/import happened or len 0, cannot use the HW */ 2351 return safexcel_sha3_fbcheck(req) ?: 2352 crypto_ahash_finup(subreq); 2353 else 2354 return safexcel_ahash_finup(req); 2355 } 2356 2357 static int safexcel_sha3_digest_fallback(struct ahash_request *req) 2358 { 2359 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 2360 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2361 struct ahash_request *subreq = ahash_request_ctx_dma(req); 2362 2363 ctx->do_fallback = true; 2364 ctx->fb_init_done = false; 2365 return safexcel_sha3_fbcheck(req) ?: crypto_ahash_finup(subreq); 2366 } 2367 2368 static int safexcel_sha3_224_digest(struct ahash_request *req) 2369 { 2370 if (req->nbytes) 2371 return safexcel_sha3_224_init(req) ?: safexcel_ahash_finup(req); 2372 2373 /* HW cannot do zero length hash, use fallback instead */ 2374 return safexcel_sha3_digest_fallback(req); 2375 } 2376 2377 static int safexcel_sha3_export(struct ahash_request *req, void *out) 2378 { 2379 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 2380 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2381 struct ahash_request *subreq = ahash_request_ctx_dma(req); 2382 2383 ctx->do_fallback = true; 2384 return safexcel_sha3_fbcheck(req) ?: crypto_ahash_export(subreq, out); 2385 } 2386 2387 static int safexcel_sha3_import(struct ahash_request *req, const void *in) 2388 { 2389 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 2390 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2391 struct ahash_request *subreq = ahash_request_ctx_dma(req); 2392 2393 ctx->do_fallback = true; 2394 return safexcel_sha3_fbcheck(req) ?: crypto_ahash_import(subreq, in); 2395 // return safexcel_ahash_import(req, in); 2396 } 2397 2398 static int safexcel_sha3_cra_init(struct crypto_tfm *tfm) 2399 { 2400 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); 2401 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 2402 2403 safexcel_ahash_cra_init(tfm); 2404 2405 /* Allocate fallback implementation */ 2406 ctx->fback = crypto_alloc_ahash(crypto_tfm_alg_name(tfm), 0, 2407 CRYPTO_ALG_ASYNC | 2408 CRYPTO_ALG_NEED_FALLBACK); 2409 if (IS_ERR(ctx->fback)) 2410 return PTR_ERR(ctx->fback); 2411 2412 /* Update statesize from fallback algorithm! */ 2413 crypto_hash_alg_common(ahash)->statesize = 2414 crypto_ahash_statesize(ctx->fback); 2415 crypto_ahash_set_reqsize_dma( 2416 ahash, max(sizeof(struct safexcel_ahash_req), 2417 sizeof(struct ahash_request) + 2418 crypto_ahash_reqsize(ctx->fback))); 2419 return 0; 2420 } 2421 2422 static void safexcel_sha3_cra_exit(struct crypto_tfm *tfm) 2423 { 2424 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 2425 2426 crypto_free_ahash(ctx->fback); 2427 safexcel_ahash_cra_exit(tfm); 2428 } 2429 2430 struct safexcel_alg_template safexcel_alg_sha3_224 = { 2431 .type = SAFEXCEL_ALG_TYPE_AHASH, 2432 .algo_mask = SAFEXCEL_ALG_SHA3, 2433 .alg.ahash = { 2434 .init = safexcel_sha3_224_init, 2435 .update = safexcel_sha3_update, 2436 .final = safexcel_sha3_final, 2437 .finup = safexcel_sha3_finup, 2438 .digest = safexcel_sha3_224_digest, 2439 .export = safexcel_sha3_export, 2440 .import = safexcel_sha3_import, 2441 .halg = { 2442 .digestsize = SHA3_224_DIGEST_SIZE, 2443 .statesize = sizeof(struct safexcel_ahash_export_state), 2444 .base = { 2445 .cra_name = "sha3-224", 2446 .cra_driver_name = "safexcel-sha3-224", 2447 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2448 .cra_flags = CRYPTO_ALG_ASYNC | 2449 CRYPTO_ALG_KERN_DRIVER_ONLY | 2450 CRYPTO_ALG_NEED_FALLBACK, 2451 .cra_blocksize = SHA3_224_BLOCK_SIZE, 2452 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 2453 .cra_init = safexcel_sha3_cra_init, 2454 .cra_exit = safexcel_sha3_cra_exit, 2455 .cra_module = THIS_MODULE, 2456 }, 2457 }, 2458 }, 2459 }; 2460 2461 static int safexcel_sha3_256_init(struct ahash_request *areq) 2462 { 2463 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2464 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2465 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 2466 2467 memset(req, 0, sizeof(*req)); 2468 2469 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_256; 2470 req->digest = CONTEXT_CONTROL_DIGEST_INITIAL; 2471 req->state_sz = SHA3_256_DIGEST_SIZE; 2472 req->digest_sz = SHA3_256_DIGEST_SIZE; 2473 req->block_sz = SHA3_256_BLOCK_SIZE; 2474 ctx->do_fallback = false; 2475 ctx->fb_init_done = false; 2476 return 0; 2477 } 2478 2479 static int safexcel_sha3_256_digest(struct ahash_request *req) 2480 { 2481 if (req->nbytes) 2482 return safexcel_sha3_256_init(req) ?: safexcel_ahash_finup(req); 2483 2484 /* HW cannot do zero length hash, use fallback instead */ 2485 return safexcel_sha3_digest_fallback(req); 2486 } 2487 2488 struct safexcel_alg_template safexcel_alg_sha3_256 = { 2489 .type = SAFEXCEL_ALG_TYPE_AHASH, 2490 .algo_mask = SAFEXCEL_ALG_SHA3, 2491 .alg.ahash = { 2492 .init = safexcel_sha3_256_init, 2493 .update = safexcel_sha3_update, 2494 .final = safexcel_sha3_final, 2495 .finup = safexcel_sha3_finup, 2496 .digest = safexcel_sha3_256_digest, 2497 .export = safexcel_sha3_export, 2498 .import = safexcel_sha3_import, 2499 .halg = { 2500 .digestsize = SHA3_256_DIGEST_SIZE, 2501 .statesize = sizeof(struct safexcel_ahash_export_state), 2502 .base = { 2503 .cra_name = "sha3-256", 2504 .cra_driver_name = "safexcel-sha3-256", 2505 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2506 .cra_flags = CRYPTO_ALG_ASYNC | 2507 CRYPTO_ALG_KERN_DRIVER_ONLY | 2508 CRYPTO_ALG_NEED_FALLBACK, 2509 .cra_blocksize = SHA3_256_BLOCK_SIZE, 2510 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 2511 .cra_init = safexcel_sha3_cra_init, 2512 .cra_exit = safexcel_sha3_cra_exit, 2513 .cra_module = THIS_MODULE, 2514 }, 2515 }, 2516 }, 2517 }; 2518 2519 static int safexcel_sha3_384_init(struct ahash_request *areq) 2520 { 2521 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2522 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2523 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 2524 2525 memset(req, 0, sizeof(*req)); 2526 2527 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_384; 2528 req->digest = CONTEXT_CONTROL_DIGEST_INITIAL; 2529 req->state_sz = SHA3_384_DIGEST_SIZE; 2530 req->digest_sz = SHA3_384_DIGEST_SIZE; 2531 req->block_sz = SHA3_384_BLOCK_SIZE; 2532 ctx->do_fallback = false; 2533 ctx->fb_init_done = false; 2534 return 0; 2535 } 2536 2537 static int safexcel_sha3_384_digest(struct ahash_request *req) 2538 { 2539 if (req->nbytes) 2540 return safexcel_sha3_384_init(req) ?: safexcel_ahash_finup(req); 2541 2542 /* HW cannot do zero length hash, use fallback instead */ 2543 return safexcel_sha3_digest_fallback(req); 2544 } 2545 2546 struct safexcel_alg_template safexcel_alg_sha3_384 = { 2547 .type = SAFEXCEL_ALG_TYPE_AHASH, 2548 .algo_mask = SAFEXCEL_ALG_SHA3, 2549 .alg.ahash = { 2550 .init = safexcel_sha3_384_init, 2551 .update = safexcel_sha3_update, 2552 .final = safexcel_sha3_final, 2553 .finup = safexcel_sha3_finup, 2554 .digest = safexcel_sha3_384_digest, 2555 .export = safexcel_sha3_export, 2556 .import = safexcel_sha3_import, 2557 .halg = { 2558 .digestsize = SHA3_384_DIGEST_SIZE, 2559 .statesize = sizeof(struct safexcel_ahash_export_state), 2560 .base = { 2561 .cra_name = "sha3-384", 2562 .cra_driver_name = "safexcel-sha3-384", 2563 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2564 .cra_flags = CRYPTO_ALG_ASYNC | 2565 CRYPTO_ALG_KERN_DRIVER_ONLY | 2566 CRYPTO_ALG_NEED_FALLBACK, 2567 .cra_blocksize = SHA3_384_BLOCK_SIZE, 2568 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 2569 .cra_init = safexcel_sha3_cra_init, 2570 .cra_exit = safexcel_sha3_cra_exit, 2571 .cra_module = THIS_MODULE, 2572 }, 2573 }, 2574 }, 2575 }; 2576 2577 static int safexcel_sha3_512_init(struct ahash_request *areq) 2578 { 2579 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2580 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2581 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 2582 2583 memset(req, 0, sizeof(*req)); 2584 2585 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_512; 2586 req->digest = CONTEXT_CONTROL_DIGEST_INITIAL; 2587 req->state_sz = SHA3_512_DIGEST_SIZE; 2588 req->digest_sz = SHA3_512_DIGEST_SIZE; 2589 req->block_sz = SHA3_512_BLOCK_SIZE; 2590 ctx->do_fallback = false; 2591 ctx->fb_init_done = false; 2592 return 0; 2593 } 2594 2595 static int safexcel_sha3_512_digest(struct ahash_request *req) 2596 { 2597 if (req->nbytes) 2598 return safexcel_sha3_512_init(req) ?: safexcel_ahash_finup(req); 2599 2600 /* HW cannot do zero length hash, use fallback instead */ 2601 return safexcel_sha3_digest_fallback(req); 2602 } 2603 2604 struct safexcel_alg_template safexcel_alg_sha3_512 = { 2605 .type = SAFEXCEL_ALG_TYPE_AHASH, 2606 .algo_mask = SAFEXCEL_ALG_SHA3, 2607 .alg.ahash = { 2608 .init = safexcel_sha3_512_init, 2609 .update = safexcel_sha3_update, 2610 .final = safexcel_sha3_final, 2611 .finup = safexcel_sha3_finup, 2612 .digest = safexcel_sha3_512_digest, 2613 .export = safexcel_sha3_export, 2614 .import = safexcel_sha3_import, 2615 .halg = { 2616 .digestsize = SHA3_512_DIGEST_SIZE, 2617 .statesize = sizeof(struct safexcel_ahash_export_state), 2618 .base = { 2619 .cra_name = "sha3-512", 2620 .cra_driver_name = "safexcel-sha3-512", 2621 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2622 .cra_flags = CRYPTO_ALG_ASYNC | 2623 CRYPTO_ALG_KERN_DRIVER_ONLY | 2624 CRYPTO_ALG_NEED_FALLBACK, 2625 .cra_blocksize = SHA3_512_BLOCK_SIZE, 2626 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 2627 .cra_init = safexcel_sha3_cra_init, 2628 .cra_exit = safexcel_sha3_cra_exit, 2629 .cra_module = THIS_MODULE, 2630 }, 2631 }, 2632 }, 2633 }; 2634 2635 static int safexcel_hmac_sha3_cra_init(struct crypto_tfm *tfm, const char *alg) 2636 { 2637 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 2638 int ret; 2639 2640 ret = safexcel_sha3_cra_init(tfm); 2641 if (ret) 2642 return ret; 2643 2644 /* Allocate precalc basic digest implementation */ 2645 ctx->shpre = crypto_alloc_shash(alg, 0, CRYPTO_ALG_NEED_FALLBACK); 2646 if (IS_ERR(ctx->shpre)) 2647 return PTR_ERR(ctx->shpre); 2648 2649 ctx->shdesc = kmalloc(sizeof(*ctx->shdesc) + 2650 crypto_shash_descsize(ctx->shpre), GFP_KERNEL); 2651 if (!ctx->shdesc) { 2652 crypto_free_shash(ctx->shpre); 2653 return -ENOMEM; 2654 } 2655 ctx->shdesc->tfm = ctx->shpre; 2656 return 0; 2657 } 2658 2659 static void safexcel_hmac_sha3_cra_exit(struct crypto_tfm *tfm) 2660 { 2661 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 2662 2663 crypto_free_ahash(ctx->fback); 2664 crypto_free_shash(ctx->shpre); 2665 kfree(ctx->shdesc); 2666 safexcel_ahash_cra_exit(tfm); 2667 } 2668 2669 static int safexcel_hmac_sha3_setkey(struct crypto_ahash *tfm, const u8 *key, 2670 unsigned int keylen) 2671 { 2672 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2673 int ret = 0; 2674 2675 if (keylen > crypto_ahash_blocksize(tfm)) { 2676 /* 2677 * If the key is larger than the blocksize, then hash it 2678 * first using our fallback cipher 2679 */ 2680 ret = crypto_shash_digest(ctx->shdesc, key, keylen, 2681 ctx->base.ipad.byte); 2682 keylen = crypto_shash_digestsize(ctx->shpre); 2683 2684 /* 2685 * If the digest is larger than half the blocksize, we need to 2686 * move the rest to opad due to the way our HMAC infra works. 2687 */ 2688 if (keylen > crypto_ahash_blocksize(tfm) / 2) 2689 /* Buffers overlap, need to use memmove iso memcpy! */ 2690 memmove(&ctx->base.opad, 2691 ctx->base.ipad.byte + 2692 crypto_ahash_blocksize(tfm) / 2, 2693 keylen - crypto_ahash_blocksize(tfm) / 2); 2694 } else { 2695 /* 2696 * Copy the key to our ipad & opad buffers 2697 * Note that ipad and opad each contain one half of the key, 2698 * to match the existing HMAC driver infrastructure. 2699 */ 2700 if (keylen <= crypto_ahash_blocksize(tfm) / 2) { 2701 memcpy(&ctx->base.ipad, key, keylen); 2702 } else { 2703 memcpy(&ctx->base.ipad, key, 2704 crypto_ahash_blocksize(tfm) / 2); 2705 memcpy(&ctx->base.opad, 2706 key + crypto_ahash_blocksize(tfm) / 2, 2707 keylen - crypto_ahash_blocksize(tfm) / 2); 2708 } 2709 } 2710 2711 /* Pad key with zeroes */ 2712 if (keylen <= crypto_ahash_blocksize(tfm) / 2) { 2713 memset(ctx->base.ipad.byte + keylen, 0, 2714 crypto_ahash_blocksize(tfm) / 2 - keylen); 2715 memset(&ctx->base.opad, 0, crypto_ahash_blocksize(tfm) / 2); 2716 } else { 2717 memset(ctx->base.opad.byte + keylen - 2718 crypto_ahash_blocksize(tfm) / 2, 0, 2719 crypto_ahash_blocksize(tfm) - keylen); 2720 } 2721 2722 /* If doing fallback, still need to set the new key! */ 2723 ctx->fb_do_setkey = true; 2724 return ret; 2725 } 2726 2727 static int safexcel_hmac_sha3_224_init(struct ahash_request *areq) 2728 { 2729 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2730 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2731 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 2732 2733 memset(req, 0, sizeof(*req)); 2734 2735 /* Copy (half of) the key */ 2736 memcpy(req->state, &ctx->base.ipad, SHA3_224_BLOCK_SIZE / 2); 2737 /* Start of HMAC should have len == processed == blocksize */ 2738 req->len = SHA3_224_BLOCK_SIZE; 2739 req->processed = SHA3_224_BLOCK_SIZE; 2740 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_224; 2741 req->digest = CONTEXT_CONTROL_DIGEST_HMAC; 2742 req->state_sz = SHA3_224_BLOCK_SIZE / 2; 2743 req->digest_sz = SHA3_224_DIGEST_SIZE; 2744 req->block_sz = SHA3_224_BLOCK_SIZE; 2745 req->hmac = true; 2746 ctx->do_fallback = false; 2747 ctx->fb_init_done = false; 2748 return 0; 2749 } 2750 2751 static int safexcel_hmac_sha3_224_digest(struct ahash_request *req) 2752 { 2753 if (req->nbytes) 2754 return safexcel_hmac_sha3_224_init(req) ?: 2755 safexcel_ahash_finup(req); 2756 2757 /* HW cannot do zero length HMAC, use fallback instead */ 2758 return safexcel_sha3_digest_fallback(req); 2759 } 2760 2761 static int safexcel_hmac_sha3_224_cra_init(struct crypto_tfm *tfm) 2762 { 2763 return safexcel_hmac_sha3_cra_init(tfm, "sha3-224"); 2764 } 2765 2766 struct safexcel_alg_template safexcel_alg_hmac_sha3_224 = { 2767 .type = SAFEXCEL_ALG_TYPE_AHASH, 2768 .algo_mask = SAFEXCEL_ALG_SHA3, 2769 .alg.ahash = { 2770 .init = safexcel_hmac_sha3_224_init, 2771 .update = safexcel_sha3_update, 2772 .final = safexcel_sha3_final, 2773 .finup = safexcel_sha3_finup, 2774 .digest = safexcel_hmac_sha3_224_digest, 2775 .setkey = safexcel_hmac_sha3_setkey, 2776 .export = safexcel_sha3_export, 2777 .import = safexcel_sha3_import, 2778 .halg = { 2779 .digestsize = SHA3_224_DIGEST_SIZE, 2780 .statesize = sizeof(struct safexcel_ahash_export_state), 2781 .base = { 2782 .cra_name = "hmac(sha3-224)", 2783 .cra_driver_name = "safexcel-hmac-sha3-224", 2784 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2785 .cra_flags = CRYPTO_ALG_ASYNC | 2786 CRYPTO_ALG_KERN_DRIVER_ONLY | 2787 CRYPTO_ALG_NEED_FALLBACK, 2788 .cra_blocksize = SHA3_224_BLOCK_SIZE, 2789 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 2790 .cra_init = safexcel_hmac_sha3_224_cra_init, 2791 .cra_exit = safexcel_hmac_sha3_cra_exit, 2792 .cra_module = THIS_MODULE, 2793 }, 2794 }, 2795 }, 2796 }; 2797 2798 static int safexcel_hmac_sha3_256_init(struct ahash_request *areq) 2799 { 2800 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2801 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2802 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 2803 2804 memset(req, 0, sizeof(*req)); 2805 2806 /* Copy (half of) the key */ 2807 memcpy(req->state, &ctx->base.ipad, SHA3_256_BLOCK_SIZE / 2); 2808 /* Start of HMAC should have len == processed == blocksize */ 2809 req->len = SHA3_256_BLOCK_SIZE; 2810 req->processed = SHA3_256_BLOCK_SIZE; 2811 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_256; 2812 req->digest = CONTEXT_CONTROL_DIGEST_HMAC; 2813 req->state_sz = SHA3_256_BLOCK_SIZE / 2; 2814 req->digest_sz = SHA3_256_DIGEST_SIZE; 2815 req->block_sz = SHA3_256_BLOCK_SIZE; 2816 req->hmac = true; 2817 ctx->do_fallback = false; 2818 ctx->fb_init_done = false; 2819 return 0; 2820 } 2821 2822 static int safexcel_hmac_sha3_256_digest(struct ahash_request *req) 2823 { 2824 if (req->nbytes) 2825 return safexcel_hmac_sha3_256_init(req) ?: 2826 safexcel_ahash_finup(req); 2827 2828 /* HW cannot do zero length HMAC, use fallback instead */ 2829 return safexcel_sha3_digest_fallback(req); 2830 } 2831 2832 static int safexcel_hmac_sha3_256_cra_init(struct crypto_tfm *tfm) 2833 { 2834 return safexcel_hmac_sha3_cra_init(tfm, "sha3-256"); 2835 } 2836 2837 struct safexcel_alg_template safexcel_alg_hmac_sha3_256 = { 2838 .type = SAFEXCEL_ALG_TYPE_AHASH, 2839 .algo_mask = SAFEXCEL_ALG_SHA3, 2840 .alg.ahash = { 2841 .init = safexcel_hmac_sha3_256_init, 2842 .update = safexcel_sha3_update, 2843 .final = safexcel_sha3_final, 2844 .finup = safexcel_sha3_finup, 2845 .digest = safexcel_hmac_sha3_256_digest, 2846 .setkey = safexcel_hmac_sha3_setkey, 2847 .export = safexcel_sha3_export, 2848 .import = safexcel_sha3_import, 2849 .halg = { 2850 .digestsize = SHA3_256_DIGEST_SIZE, 2851 .statesize = sizeof(struct safexcel_ahash_export_state), 2852 .base = { 2853 .cra_name = "hmac(sha3-256)", 2854 .cra_driver_name = "safexcel-hmac-sha3-256", 2855 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2856 .cra_flags = CRYPTO_ALG_ASYNC | 2857 CRYPTO_ALG_KERN_DRIVER_ONLY | 2858 CRYPTO_ALG_NEED_FALLBACK, 2859 .cra_blocksize = SHA3_256_BLOCK_SIZE, 2860 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 2861 .cra_init = safexcel_hmac_sha3_256_cra_init, 2862 .cra_exit = safexcel_hmac_sha3_cra_exit, 2863 .cra_module = THIS_MODULE, 2864 }, 2865 }, 2866 }, 2867 }; 2868 2869 static int safexcel_hmac_sha3_384_init(struct ahash_request *areq) 2870 { 2871 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2872 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2873 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 2874 2875 memset(req, 0, sizeof(*req)); 2876 2877 /* Copy (half of) the key */ 2878 memcpy(req->state, &ctx->base.ipad, SHA3_384_BLOCK_SIZE / 2); 2879 /* Start of HMAC should have len == processed == blocksize */ 2880 req->len = SHA3_384_BLOCK_SIZE; 2881 req->processed = SHA3_384_BLOCK_SIZE; 2882 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_384; 2883 req->digest = CONTEXT_CONTROL_DIGEST_HMAC; 2884 req->state_sz = SHA3_384_BLOCK_SIZE / 2; 2885 req->digest_sz = SHA3_384_DIGEST_SIZE; 2886 req->block_sz = SHA3_384_BLOCK_SIZE; 2887 req->hmac = true; 2888 ctx->do_fallback = false; 2889 ctx->fb_init_done = false; 2890 return 0; 2891 } 2892 2893 static int safexcel_hmac_sha3_384_digest(struct ahash_request *req) 2894 { 2895 if (req->nbytes) 2896 return safexcel_hmac_sha3_384_init(req) ?: 2897 safexcel_ahash_finup(req); 2898 2899 /* HW cannot do zero length HMAC, use fallback instead */ 2900 return safexcel_sha3_digest_fallback(req); 2901 } 2902 2903 static int safexcel_hmac_sha3_384_cra_init(struct crypto_tfm *tfm) 2904 { 2905 return safexcel_hmac_sha3_cra_init(tfm, "sha3-384"); 2906 } 2907 2908 struct safexcel_alg_template safexcel_alg_hmac_sha3_384 = { 2909 .type = SAFEXCEL_ALG_TYPE_AHASH, 2910 .algo_mask = SAFEXCEL_ALG_SHA3, 2911 .alg.ahash = { 2912 .init = safexcel_hmac_sha3_384_init, 2913 .update = safexcel_sha3_update, 2914 .final = safexcel_sha3_final, 2915 .finup = safexcel_sha3_finup, 2916 .digest = safexcel_hmac_sha3_384_digest, 2917 .setkey = safexcel_hmac_sha3_setkey, 2918 .export = safexcel_sha3_export, 2919 .import = safexcel_sha3_import, 2920 .halg = { 2921 .digestsize = SHA3_384_DIGEST_SIZE, 2922 .statesize = sizeof(struct safexcel_ahash_export_state), 2923 .base = { 2924 .cra_name = "hmac(sha3-384)", 2925 .cra_driver_name = "safexcel-hmac-sha3-384", 2926 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2927 .cra_flags = CRYPTO_ALG_ASYNC | 2928 CRYPTO_ALG_KERN_DRIVER_ONLY | 2929 CRYPTO_ALG_NEED_FALLBACK, 2930 .cra_blocksize = SHA3_384_BLOCK_SIZE, 2931 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 2932 .cra_init = safexcel_hmac_sha3_384_cra_init, 2933 .cra_exit = safexcel_hmac_sha3_cra_exit, 2934 .cra_module = THIS_MODULE, 2935 }, 2936 }, 2937 }, 2938 }; 2939 2940 static int safexcel_hmac_sha3_512_init(struct ahash_request *areq) 2941 { 2942 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2943 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2944 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 2945 2946 memset(req, 0, sizeof(*req)); 2947 2948 /* Copy (half of) the key */ 2949 memcpy(req->state, &ctx->base.ipad, SHA3_512_BLOCK_SIZE / 2); 2950 /* Start of HMAC should have len == processed == blocksize */ 2951 req->len = SHA3_512_BLOCK_SIZE; 2952 req->processed = SHA3_512_BLOCK_SIZE; 2953 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_512; 2954 req->digest = CONTEXT_CONTROL_DIGEST_HMAC; 2955 req->state_sz = SHA3_512_BLOCK_SIZE / 2; 2956 req->digest_sz = SHA3_512_DIGEST_SIZE; 2957 req->block_sz = SHA3_512_BLOCK_SIZE; 2958 req->hmac = true; 2959 ctx->do_fallback = false; 2960 ctx->fb_init_done = false; 2961 return 0; 2962 } 2963 2964 static int safexcel_hmac_sha3_512_digest(struct ahash_request *req) 2965 { 2966 if (req->nbytes) 2967 return safexcel_hmac_sha3_512_init(req) ?: 2968 safexcel_ahash_finup(req); 2969 2970 /* HW cannot do zero length HMAC, use fallback instead */ 2971 return safexcel_sha3_digest_fallback(req); 2972 } 2973 2974 static int safexcel_hmac_sha3_512_cra_init(struct crypto_tfm *tfm) 2975 { 2976 return safexcel_hmac_sha3_cra_init(tfm, "sha3-512"); 2977 } 2978 struct safexcel_alg_template safexcel_alg_hmac_sha3_512 = { 2979 .type = SAFEXCEL_ALG_TYPE_AHASH, 2980 .algo_mask = SAFEXCEL_ALG_SHA3, 2981 .alg.ahash = { 2982 .init = safexcel_hmac_sha3_512_init, 2983 .update = safexcel_sha3_update, 2984 .final = safexcel_sha3_final, 2985 .finup = safexcel_sha3_finup, 2986 .digest = safexcel_hmac_sha3_512_digest, 2987 .setkey = safexcel_hmac_sha3_setkey, 2988 .export = safexcel_sha3_export, 2989 .import = safexcel_sha3_import, 2990 .halg = { 2991 .digestsize = SHA3_512_DIGEST_SIZE, 2992 .statesize = sizeof(struct safexcel_ahash_export_state), 2993 .base = { 2994 .cra_name = "hmac(sha3-512)", 2995 .cra_driver_name = "safexcel-hmac-sha3-512", 2996 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2997 .cra_flags = CRYPTO_ALG_ASYNC | 2998 CRYPTO_ALG_KERN_DRIVER_ONLY | 2999 CRYPTO_ALG_NEED_FALLBACK, 3000 .cra_blocksize = SHA3_512_BLOCK_SIZE, 3001 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 3002 .cra_init = safexcel_hmac_sha3_512_cra_init, 3003 .cra_exit = safexcel_hmac_sha3_cra_exit, 3004 .cra_module = THIS_MODULE, 3005 }, 3006 }, 3007 }, 3008 }; 3009