1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2017 Marvell 4 * 5 * Antoine Tenart <antoine.tenart@free-electrons.com> 6 */ 7 8 #include <crypto/aes.h> 9 #include <crypto/hmac.h> 10 #include <crypto/md5.h> 11 #include <crypto/sha1.h> 12 #include <crypto/sha2.h> 13 #include <crypto/sha3.h> 14 #include <crypto/skcipher.h> 15 #include <crypto/sm3.h> 16 #include <crypto/internal/cipher.h> 17 #include <linux/device.h> 18 #include <linux/dma-mapping.h> 19 #include <linux/dmapool.h> 20 21 #include "safexcel.h" 22 23 struct safexcel_ahash_ctx { 24 struct safexcel_context base; 25 26 u32 alg; 27 u8 key_sz; 28 bool cbcmac; 29 bool do_fallback; 30 bool fb_init_done; 31 bool fb_do_setkey; 32 33 struct crypto_aes_ctx *aes; 34 struct crypto_ahash *fback; 35 struct crypto_shash *shpre; 36 struct shash_desc *shdesc; 37 }; 38 39 struct safexcel_ahash_req { 40 bool last_req; 41 bool finish; 42 bool hmac; 43 bool needs_inv; 44 bool hmac_zlen; 45 bool len_is_le; 46 bool not_first; 47 bool xcbcmac; 48 49 int nents; 50 dma_addr_t result_dma; 51 52 u32 digest; 53 54 u8 state_sz; /* expected state size, only set once */ 55 u8 block_sz; /* block size, only set once */ 56 u8 digest_sz; /* output digest size, only set once */ 57 __le32 state[SHA3_512_BLOCK_SIZE / 58 sizeof(__le32)] __aligned(sizeof(__le32)); 59 60 u64 len; 61 u64 processed; 62 63 u8 cache[HASH_CACHE_SIZE] __aligned(sizeof(u32)); 64 dma_addr_t cache_dma; 65 unsigned int cache_sz; 66 67 u8 cache_next[HASH_CACHE_SIZE] __aligned(sizeof(u32)); 68 }; 69 70 static inline u64 safexcel_queued_len(struct safexcel_ahash_req *req) 71 { 72 return req->len - req->processed; 73 } 74 75 static void safexcel_hash_token(struct safexcel_command_desc *cdesc, 76 u32 input_length, u32 result_length, 77 bool cbcmac) 78 { 79 struct safexcel_token *token = 80 (struct safexcel_token *)cdesc->control_data.token; 81 82 token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION; 83 token[0].packet_length = input_length; 84 token[0].instructions = EIP197_TOKEN_INS_TYPE_HASH; 85 86 input_length &= 15; 87 if (unlikely(cbcmac && input_length)) { 88 token[0].stat = 0; 89 token[1].opcode = EIP197_TOKEN_OPCODE_INSERT; 90 token[1].packet_length = 16 - input_length; 91 token[1].stat = EIP197_TOKEN_STAT_LAST_HASH; 92 token[1].instructions = EIP197_TOKEN_INS_TYPE_HASH; 93 } else { 94 token[0].stat = EIP197_TOKEN_STAT_LAST_HASH; 95 eip197_noop_token(&token[1]); 96 } 97 98 token[2].opcode = EIP197_TOKEN_OPCODE_INSERT; 99 token[2].stat = EIP197_TOKEN_STAT_LAST_HASH | 100 EIP197_TOKEN_STAT_LAST_PACKET; 101 token[2].packet_length = result_length; 102 token[2].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | 103 EIP197_TOKEN_INS_INSERT_HASH_DIGEST; 104 105 eip197_noop_token(&token[3]); 106 } 107 108 static void safexcel_context_control(struct safexcel_ahash_ctx *ctx, 109 struct safexcel_ahash_req *req, 110 struct safexcel_command_desc *cdesc) 111 { 112 struct safexcel_crypto_priv *priv = ctx->base.priv; 113 u64 count = 0; 114 115 cdesc->control_data.control0 = ctx->alg; 116 cdesc->control_data.control1 = 0; 117 118 /* 119 * Copy the input digest if needed, and setup the context 120 * fields. Do this now as we need it to setup the first command 121 * descriptor. 122 */ 123 if (unlikely(req->digest == CONTEXT_CONTROL_DIGEST_XCM)) { 124 if (req->xcbcmac) 125 memcpy(ctx->base.ctxr->data, &ctx->base.ipad, ctx->key_sz); 126 else 127 memcpy(ctx->base.ctxr->data, req->state, req->state_sz); 128 129 if (!req->finish && req->xcbcmac) 130 cdesc->control_data.control0 |= 131 CONTEXT_CONTROL_DIGEST_XCM | 132 CONTEXT_CONTROL_TYPE_HASH_OUT | 133 CONTEXT_CONTROL_NO_FINISH_HASH | 134 CONTEXT_CONTROL_SIZE(req->state_sz / 135 sizeof(u32)); 136 else 137 cdesc->control_data.control0 |= 138 CONTEXT_CONTROL_DIGEST_XCM | 139 CONTEXT_CONTROL_TYPE_HASH_OUT | 140 CONTEXT_CONTROL_SIZE(req->state_sz / 141 sizeof(u32)); 142 return; 143 } else if (!req->processed) { 144 /* First - and possibly only - block of basic hash only */ 145 if (req->finish) 146 cdesc->control_data.control0 |= req->digest | 147 CONTEXT_CONTROL_TYPE_HASH_OUT | 148 CONTEXT_CONTROL_RESTART_HASH | 149 /* ensure its not 0! */ 150 CONTEXT_CONTROL_SIZE(1); 151 else 152 cdesc->control_data.control0 |= req->digest | 153 CONTEXT_CONTROL_TYPE_HASH_OUT | 154 CONTEXT_CONTROL_RESTART_HASH | 155 CONTEXT_CONTROL_NO_FINISH_HASH | 156 /* ensure its not 0! */ 157 CONTEXT_CONTROL_SIZE(1); 158 return; 159 } 160 161 /* Hash continuation or HMAC, setup (inner) digest from state */ 162 memcpy(ctx->base.ctxr->data, req->state, req->state_sz); 163 164 if (req->finish) { 165 /* Compute digest count for hash/HMAC finish operations */ 166 if ((req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) || 167 req->hmac_zlen || (req->processed != req->block_sz)) { 168 count = req->processed / EIP197_COUNTER_BLOCK_SIZE; 169 170 /* This is a hardware limitation, as the 171 * counter must fit into an u32. This represents 172 * a fairly big amount of input data, so we 173 * shouldn't see this. 174 */ 175 if (unlikely(count & 0xffffffff00000000ULL)) { 176 dev_warn(priv->dev, 177 "Input data is too big\n"); 178 return; 179 } 180 } 181 182 if ((req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) || 183 /* Special case: zero length HMAC */ 184 req->hmac_zlen || 185 /* PE HW < 4.4 cannot do HMAC continue, fake using hash */ 186 (req->processed != req->block_sz)) { 187 /* Basic hash continue operation, need digest + cnt */ 188 cdesc->control_data.control0 |= 189 CONTEXT_CONTROL_SIZE((req->state_sz >> 2) + 1) | 190 CONTEXT_CONTROL_TYPE_HASH_OUT | 191 CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 192 /* For zero-len HMAC, don't finalize, already padded! */ 193 if (req->hmac_zlen) 194 cdesc->control_data.control0 |= 195 CONTEXT_CONTROL_NO_FINISH_HASH; 196 cdesc->control_data.control1 |= 197 CONTEXT_CONTROL_DIGEST_CNT; 198 ctx->base.ctxr->data[req->state_sz >> 2] = 199 cpu_to_le32(count); 200 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 201 202 /* Clear zero-length HMAC flag for next operation! */ 203 req->hmac_zlen = false; 204 } else { /* HMAC */ 205 /* Need outer digest for HMAC finalization */ 206 memcpy(ctx->base.ctxr->data + (req->state_sz >> 2), 207 &ctx->base.opad, req->state_sz); 208 209 /* Single pass HMAC - no digest count */ 210 cdesc->control_data.control0 |= 211 CONTEXT_CONTROL_SIZE(req->state_sz >> 1) | 212 CONTEXT_CONTROL_TYPE_HASH_OUT | 213 CONTEXT_CONTROL_DIGEST_HMAC; 214 } 215 } else { /* Hash continuation, do not finish yet */ 216 cdesc->control_data.control0 |= 217 CONTEXT_CONTROL_SIZE(req->state_sz >> 2) | 218 CONTEXT_CONTROL_DIGEST_PRECOMPUTED | 219 CONTEXT_CONTROL_TYPE_HASH_OUT | 220 CONTEXT_CONTROL_NO_FINISH_HASH; 221 } 222 } 223 224 static int safexcel_ahash_enqueue(struct ahash_request *areq); 225 226 static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, 227 int ring, 228 struct crypto_async_request *async, 229 bool *should_complete, int *ret) 230 { 231 struct safexcel_result_desc *rdesc; 232 struct ahash_request *areq = ahash_request_cast(async); 233 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); 234 struct safexcel_ahash_req *sreq = ahash_request_ctx_dma(areq); 235 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(ahash); 236 u64 cache_len; 237 238 *ret = 0; 239 240 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); 241 if (IS_ERR(rdesc)) { 242 dev_err(priv->dev, 243 "hash: result: could not retrieve the result descriptor\n"); 244 *ret = PTR_ERR(rdesc); 245 } else { 246 *ret = safexcel_rdesc_check_errors(priv, rdesc); 247 } 248 249 safexcel_complete(priv, ring); 250 251 if (sreq->nents) { 252 dma_unmap_sg(priv->dev, areq->src, sreq->nents, DMA_TO_DEVICE); 253 sreq->nents = 0; 254 } 255 256 if (sreq->result_dma) { 257 dma_unmap_single(priv->dev, sreq->result_dma, sreq->digest_sz, 258 DMA_FROM_DEVICE); 259 sreq->result_dma = 0; 260 } 261 262 if (sreq->cache_dma) { 263 dma_unmap_single(priv->dev, sreq->cache_dma, sreq->cache_sz, 264 DMA_TO_DEVICE); 265 sreq->cache_dma = 0; 266 sreq->cache_sz = 0; 267 } 268 269 if (sreq->finish) { 270 if (sreq->hmac && 271 (sreq->digest != CONTEXT_CONTROL_DIGEST_HMAC)) { 272 /* Faking HMAC using hash - need to do outer hash */ 273 memcpy(sreq->cache, sreq->state, 274 crypto_ahash_digestsize(ahash)); 275 276 memcpy(sreq->state, &ctx->base.opad, sreq->digest_sz); 277 278 sreq->len = sreq->block_sz + 279 crypto_ahash_digestsize(ahash); 280 sreq->processed = sreq->block_sz; 281 sreq->hmac = 0; 282 283 if (priv->flags & EIP197_TRC_CACHE) 284 ctx->base.needs_inv = true; 285 areq->nbytes = 0; 286 safexcel_ahash_enqueue(areq); 287 288 *should_complete = false; /* Not done yet */ 289 return 1; 290 } 291 292 if (unlikely(sreq->digest == CONTEXT_CONTROL_DIGEST_XCM && 293 ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_CRC32)) { 294 /* Undo final XOR with 0xffffffff ...*/ 295 *(__le32 *)areq->result = ~sreq->state[0]; 296 } else { 297 memcpy(areq->result, sreq->state, 298 crypto_ahash_digestsize(ahash)); 299 } 300 } 301 302 cache_len = safexcel_queued_len(sreq); 303 if (cache_len) 304 memcpy(sreq->cache, sreq->cache_next, cache_len); 305 306 *should_complete = true; 307 308 return 1; 309 } 310 311 static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, 312 int *commands, int *results) 313 { 314 struct ahash_request *areq = ahash_request_cast(async); 315 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 316 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 317 struct safexcel_crypto_priv *priv = ctx->base.priv; 318 struct safexcel_command_desc *cdesc, *first_cdesc = NULL; 319 struct safexcel_result_desc *rdesc; 320 struct scatterlist *sg; 321 struct safexcel_token *dmmy; 322 int i, extra = 0, n_cdesc = 0, ret = 0, cache_len, skip = 0; 323 u64 queued, len; 324 325 queued = safexcel_queued_len(req); 326 if (queued <= HASH_CACHE_SIZE) 327 cache_len = queued; 328 else 329 cache_len = queued - areq->nbytes; 330 331 if (!req->finish && !req->last_req) { 332 /* If this is not the last request and the queued data does not 333 * fit into full cache blocks, cache it for the next send call. 334 */ 335 extra = queued & (HASH_CACHE_SIZE - 1); 336 337 /* If this is not the last request and the queued data 338 * is a multiple of a block, cache the last one for now. 339 */ 340 if (!extra) 341 extra = HASH_CACHE_SIZE; 342 343 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), 344 req->cache_next, extra, 345 areq->nbytes - extra); 346 347 queued -= extra; 348 349 if (!queued) { 350 *commands = 0; 351 *results = 0; 352 return 0; 353 } 354 355 extra = 0; 356 } 357 358 if (unlikely(req->xcbcmac && req->processed > AES_BLOCK_SIZE)) { 359 if (unlikely(cache_len < AES_BLOCK_SIZE)) { 360 /* 361 * Cache contains less than 1 full block, complete. 362 */ 363 extra = AES_BLOCK_SIZE - cache_len; 364 if (queued > cache_len) { 365 /* More data follows: borrow bytes */ 366 u64 tmp = queued - cache_len; 367 368 skip = min_t(u64, tmp, extra); 369 sg_pcopy_to_buffer(areq->src, 370 sg_nents(areq->src), 371 req->cache + cache_len, 372 skip, 0); 373 } 374 extra -= skip; 375 memset(req->cache + cache_len + skip, 0, extra); 376 if (!ctx->cbcmac && extra) { 377 // 10- padding for XCBCMAC & CMAC 378 req->cache[cache_len + skip] = 0x80; 379 // HW will use K2 iso K3 - compensate! 380 for (i = 0; i < AES_BLOCK_SIZE / 4; i++) { 381 u32 *cache = (void *)req->cache; 382 u32 *ipad = ctx->base.ipad.word; 383 u32 x; 384 385 x = ipad[i] ^ ipad[i + 4]; 386 cache[i] ^= swab32(x); 387 } 388 } 389 cache_len = AES_BLOCK_SIZE; 390 queued = queued + extra; 391 } 392 393 /* XCBC continue: XOR previous result into 1st word */ 394 crypto_xor(req->cache, (const u8 *)req->state, AES_BLOCK_SIZE); 395 } 396 397 len = queued; 398 /* Add a command descriptor for the cached data, if any */ 399 if (cache_len) { 400 req->cache_dma = dma_map_single(priv->dev, req->cache, 401 cache_len, DMA_TO_DEVICE); 402 if (dma_mapping_error(priv->dev, req->cache_dma)) 403 return -EINVAL; 404 405 req->cache_sz = cache_len; 406 first_cdesc = safexcel_add_cdesc(priv, ring, 1, 407 (cache_len == len), 408 req->cache_dma, cache_len, 409 len, ctx->base.ctxr_dma, 410 &dmmy); 411 if (IS_ERR(first_cdesc)) { 412 ret = PTR_ERR(first_cdesc); 413 goto unmap_cache; 414 } 415 n_cdesc++; 416 417 queued -= cache_len; 418 if (!queued) 419 goto send_command; 420 } 421 422 /* Now handle the current ahash request buffer(s) */ 423 req->nents = dma_map_sg(priv->dev, areq->src, 424 sg_nents_for_len(areq->src, 425 areq->nbytes), 426 DMA_TO_DEVICE); 427 if (!req->nents) { 428 ret = -ENOMEM; 429 goto cdesc_rollback; 430 } 431 432 for_each_sg(areq->src, sg, req->nents, i) { 433 int sglen = sg_dma_len(sg); 434 435 if (unlikely(sglen <= skip)) { 436 skip -= sglen; 437 continue; 438 } 439 440 /* Do not overflow the request */ 441 if ((queued + skip) <= sglen) 442 sglen = queued; 443 else 444 sglen -= skip; 445 446 cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc, 447 !(queued - sglen), 448 sg_dma_address(sg) + skip, sglen, 449 len, ctx->base.ctxr_dma, &dmmy); 450 if (IS_ERR(cdesc)) { 451 ret = PTR_ERR(cdesc); 452 goto unmap_sg; 453 } 454 455 if (!n_cdesc) 456 first_cdesc = cdesc; 457 n_cdesc++; 458 459 queued -= sglen; 460 if (!queued) 461 break; 462 skip = 0; 463 } 464 465 send_command: 466 /* Setup the context options */ 467 safexcel_context_control(ctx, req, first_cdesc); 468 469 /* Add the token */ 470 safexcel_hash_token(first_cdesc, len, req->digest_sz, ctx->cbcmac); 471 472 req->result_dma = dma_map_single(priv->dev, req->state, req->digest_sz, 473 DMA_FROM_DEVICE); 474 if (dma_mapping_error(priv->dev, req->result_dma)) { 475 ret = -EINVAL; 476 goto unmap_sg; 477 } 478 479 /* Add a result descriptor */ 480 rdesc = safexcel_add_rdesc(priv, ring, 1, 1, req->result_dma, 481 req->digest_sz); 482 if (IS_ERR(rdesc)) { 483 ret = PTR_ERR(rdesc); 484 goto unmap_result; 485 } 486 487 safexcel_rdr_req_set(priv, ring, rdesc, &areq->base); 488 489 req->processed += len - extra; 490 491 *commands = n_cdesc; 492 *results = 1; 493 return 0; 494 495 unmap_result: 496 dma_unmap_single(priv->dev, req->result_dma, req->digest_sz, 497 DMA_FROM_DEVICE); 498 unmap_sg: 499 if (req->nents) { 500 dma_unmap_sg(priv->dev, areq->src, req->nents, DMA_TO_DEVICE); 501 req->nents = 0; 502 } 503 cdesc_rollback: 504 for (i = 0; i < n_cdesc; i++) 505 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr); 506 unmap_cache: 507 if (req->cache_dma) { 508 dma_unmap_single(priv->dev, req->cache_dma, req->cache_sz, 509 DMA_TO_DEVICE); 510 req->cache_dma = 0; 511 req->cache_sz = 0; 512 } 513 514 return ret; 515 } 516 517 static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv, 518 int ring, 519 struct crypto_async_request *async, 520 bool *should_complete, int *ret) 521 { 522 struct safexcel_result_desc *rdesc; 523 struct ahash_request *areq = ahash_request_cast(async); 524 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); 525 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(ahash); 526 int enq_ret; 527 528 *ret = 0; 529 530 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); 531 if (IS_ERR(rdesc)) { 532 dev_err(priv->dev, 533 "hash: invalidate: could not retrieve the result descriptor\n"); 534 *ret = PTR_ERR(rdesc); 535 } else { 536 *ret = safexcel_rdesc_check_errors(priv, rdesc); 537 } 538 539 safexcel_complete(priv, ring); 540 541 if (ctx->base.exit_inv) { 542 dma_pool_free(priv->context_pool, ctx->base.ctxr, 543 ctx->base.ctxr_dma); 544 545 *should_complete = true; 546 return 1; 547 } 548 549 ring = safexcel_select_ring(priv); 550 ctx->base.ring = ring; 551 552 spin_lock_bh(&priv->ring[ring].queue_lock); 553 enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, async); 554 spin_unlock_bh(&priv->ring[ring].queue_lock); 555 556 if (enq_ret != -EINPROGRESS) 557 *ret = enq_ret; 558 559 queue_work(priv->ring[ring].workqueue, 560 &priv->ring[ring].work_data.work); 561 562 *should_complete = false; 563 564 return 1; 565 } 566 567 static int safexcel_handle_result(struct safexcel_crypto_priv *priv, int ring, 568 struct crypto_async_request *async, 569 bool *should_complete, int *ret) 570 { 571 struct ahash_request *areq = ahash_request_cast(async); 572 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 573 int err; 574 575 BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && req->needs_inv); 576 577 if (req->needs_inv) { 578 req->needs_inv = false; 579 err = safexcel_handle_inv_result(priv, ring, async, 580 should_complete, ret); 581 } else { 582 err = safexcel_handle_req_result(priv, ring, async, 583 should_complete, ret); 584 } 585 586 return err; 587 } 588 589 static int safexcel_ahash_send_inv(struct crypto_async_request *async, 590 int ring, int *commands, int *results) 591 { 592 struct ahash_request *areq = ahash_request_cast(async); 593 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 594 int ret; 595 596 ret = safexcel_invalidate_cache(async, ctx->base.priv, 597 ctx->base.ctxr_dma, ring); 598 if (unlikely(ret)) 599 return ret; 600 601 *commands = 1; 602 *results = 1; 603 604 return 0; 605 } 606 607 static int safexcel_ahash_send(struct crypto_async_request *async, 608 int ring, int *commands, int *results) 609 { 610 struct ahash_request *areq = ahash_request_cast(async); 611 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 612 int ret; 613 614 if (req->needs_inv) 615 ret = safexcel_ahash_send_inv(async, ring, commands, results); 616 else 617 ret = safexcel_ahash_send_req(async, ring, commands, results); 618 619 return ret; 620 } 621 622 static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm) 623 { 624 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 625 struct safexcel_crypto_priv *priv = ctx->base.priv; 626 EIP197_REQUEST_ON_STACK(req, ahash, EIP197_AHASH_REQ_SIZE); 627 struct safexcel_ahash_req *rctx = ahash_request_ctx_dma(req); 628 DECLARE_CRYPTO_WAIT(result); 629 int ring = ctx->base.ring; 630 int err; 631 632 memset(req, 0, EIP197_AHASH_REQ_SIZE); 633 634 /* create invalidation request */ 635 init_completion(&result.completion); 636 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 637 crypto_req_done, &result); 638 639 ahash_request_set_tfm(req, __crypto_ahash_cast(tfm)); 640 ctx = crypto_tfm_ctx(req->base.tfm); 641 ctx->base.exit_inv = true; 642 rctx->needs_inv = true; 643 644 spin_lock_bh(&priv->ring[ring].queue_lock); 645 crypto_enqueue_request(&priv->ring[ring].queue, &req->base); 646 spin_unlock_bh(&priv->ring[ring].queue_lock); 647 648 queue_work(priv->ring[ring].workqueue, 649 &priv->ring[ring].work_data.work); 650 651 err = crypto_wait_req(-EINPROGRESS, &result); 652 653 if (err) { 654 dev_warn(priv->dev, "hash: completion error (%d)\n", err); 655 return err; 656 } 657 658 return 0; 659 } 660 661 /* safexcel_ahash_cache: cache data until at least one request can be sent to 662 * the engine, aka. when there is at least 1 block size in the pipe. 663 */ 664 static int safexcel_ahash_cache(struct ahash_request *areq) 665 { 666 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 667 u64 cache_len; 668 669 /* cache_len: everything accepted by the driver but not sent yet, 670 * tot sz handled by update() - last req sz - tot sz handled by send() 671 */ 672 cache_len = safexcel_queued_len(req); 673 674 /* 675 * In case there isn't enough bytes to proceed (less than a 676 * block size), cache the data until we have enough. 677 */ 678 if (cache_len + areq->nbytes <= HASH_CACHE_SIZE) { 679 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), 680 req->cache + cache_len, 681 areq->nbytes, 0); 682 return 0; 683 } 684 685 /* We couldn't cache all the data */ 686 return -E2BIG; 687 } 688 689 static int safexcel_ahash_enqueue(struct ahash_request *areq) 690 { 691 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 692 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 693 struct safexcel_crypto_priv *priv = ctx->base.priv; 694 int ret, ring; 695 696 req->needs_inv = false; 697 698 if (ctx->base.ctxr) { 699 if (priv->flags & EIP197_TRC_CACHE && !ctx->base.needs_inv && 700 /* invalidate for *any* non-XCBC continuation */ 701 ((req->not_first && !req->xcbcmac) || 702 /* invalidate if (i)digest changed */ 703 memcmp(ctx->base.ctxr->data, req->state, req->state_sz) || 704 /* invalidate for HMAC finish with odigest changed */ 705 (req->finish && req->hmac && 706 memcmp(ctx->base.ctxr->data + (req->state_sz>>2), 707 &ctx->base.opad, req->state_sz)))) 708 /* 709 * We're still setting needs_inv here, even though it is 710 * cleared right away, because the needs_inv flag can be 711 * set in other functions and we want to keep the same 712 * logic. 713 */ 714 ctx->base.needs_inv = true; 715 716 if (ctx->base.needs_inv) { 717 ctx->base.needs_inv = false; 718 req->needs_inv = true; 719 } 720 } else { 721 ctx->base.ring = safexcel_select_ring(priv); 722 ctx->base.ctxr = dma_pool_zalloc(priv->context_pool, 723 EIP197_GFP_FLAGS(areq->base), 724 &ctx->base.ctxr_dma); 725 if (!ctx->base.ctxr) 726 return -ENOMEM; 727 } 728 req->not_first = true; 729 730 ring = ctx->base.ring; 731 732 spin_lock_bh(&priv->ring[ring].queue_lock); 733 ret = crypto_enqueue_request(&priv->ring[ring].queue, &areq->base); 734 spin_unlock_bh(&priv->ring[ring].queue_lock); 735 736 queue_work(priv->ring[ring].workqueue, 737 &priv->ring[ring].work_data.work); 738 739 return ret; 740 } 741 742 static int safexcel_ahash_update(struct ahash_request *areq) 743 { 744 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 745 int ret; 746 747 /* If the request is 0 length, do nothing */ 748 if (!areq->nbytes) 749 return 0; 750 751 /* Add request to the cache if it fits */ 752 ret = safexcel_ahash_cache(areq); 753 754 /* Update total request length */ 755 req->len += areq->nbytes; 756 757 /* If not all data could fit into the cache, go process the excess. 758 * Also go process immediately for an HMAC IV precompute, which 759 * will never be finished at all, but needs to be processed anyway. 760 */ 761 if ((ret && !req->finish) || req->last_req) 762 return safexcel_ahash_enqueue(areq); 763 764 return 0; 765 } 766 767 static int safexcel_ahash_final(struct ahash_request *areq) 768 { 769 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 770 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 771 772 req->finish = true; 773 774 if (unlikely(!req->len && !areq->nbytes)) { 775 /* 776 * If we have an overall 0 length *hash* request: 777 * The HW cannot do 0 length hash, so we provide the correct 778 * result directly here. 779 */ 780 if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5) 781 memcpy(areq->result, md5_zero_message_hash, 782 MD5_DIGEST_SIZE); 783 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA1) 784 memcpy(areq->result, sha1_zero_message_hash, 785 SHA1_DIGEST_SIZE); 786 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA224) 787 memcpy(areq->result, sha224_zero_message_hash, 788 SHA224_DIGEST_SIZE); 789 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA256) 790 memcpy(areq->result, sha256_zero_message_hash, 791 SHA256_DIGEST_SIZE); 792 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA384) 793 memcpy(areq->result, sha384_zero_message_hash, 794 SHA384_DIGEST_SIZE); 795 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA512) 796 memcpy(areq->result, sha512_zero_message_hash, 797 SHA512_DIGEST_SIZE); 798 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SM3) { 799 memcpy(areq->result, 800 EIP197_SM3_ZEROM_HASH, SM3_DIGEST_SIZE); 801 } 802 803 return 0; 804 } else if (unlikely(req->digest == CONTEXT_CONTROL_DIGEST_XCM && 805 ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5 && 806 req->len == sizeof(u32) && !areq->nbytes)) { 807 /* Zero length CRC32 */ 808 memcpy(areq->result, &ctx->base.ipad, sizeof(u32)); 809 return 0; 810 } else if (unlikely(ctx->cbcmac && req->len == AES_BLOCK_SIZE && 811 !areq->nbytes)) { 812 /* Zero length CBC MAC */ 813 memset(areq->result, 0, AES_BLOCK_SIZE); 814 return 0; 815 } else if (unlikely(req->xcbcmac && req->len == AES_BLOCK_SIZE && 816 !areq->nbytes)) { 817 /* Zero length (X)CBC/CMAC */ 818 int i; 819 820 for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) { 821 u32 *result = (void *)areq->result; 822 823 /* K3 */ 824 result[i] = swab32(ctx->base.ipad.word[i + 4]); 825 } 826 areq->result[0] ^= 0x80; // 10- padding 827 aes_encrypt(ctx->aes, areq->result, areq->result); 828 return 0; 829 } else if (unlikely(req->hmac && 830 (req->len == req->block_sz) && 831 !areq->nbytes)) { 832 /* 833 * If we have an overall 0 length *HMAC* request: 834 * For HMAC, we need to finalize the inner digest 835 * and then perform the outer hash. 836 */ 837 838 /* generate pad block in the cache */ 839 /* start with a hash block of all zeroes */ 840 memset(req->cache, 0, req->block_sz); 841 /* set the first byte to 0x80 to 'append a 1 bit' */ 842 req->cache[0] = 0x80; 843 /* add the length in bits in the last 2 bytes */ 844 if (req->len_is_le) { 845 /* Little endian length word (e.g. MD5) */ 846 req->cache[req->block_sz-8] = (req->block_sz << 3) & 847 255; 848 req->cache[req->block_sz-7] = (req->block_sz >> 5); 849 } else { 850 /* Big endian length word (e.g. any SHA) */ 851 req->cache[req->block_sz-2] = (req->block_sz >> 5); 852 req->cache[req->block_sz-1] = (req->block_sz << 3) & 853 255; 854 } 855 856 req->len += req->block_sz; /* plus 1 hash block */ 857 858 /* Set special zero-length HMAC flag */ 859 req->hmac_zlen = true; 860 861 /* Finalize HMAC */ 862 req->digest = CONTEXT_CONTROL_DIGEST_HMAC; 863 } else if (req->hmac) { 864 /* Finalize HMAC */ 865 req->digest = CONTEXT_CONTROL_DIGEST_HMAC; 866 } 867 868 return safexcel_ahash_enqueue(areq); 869 } 870 871 static int safexcel_ahash_finup(struct ahash_request *areq) 872 { 873 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 874 875 req->finish = true; 876 877 safexcel_ahash_update(areq); 878 return safexcel_ahash_final(areq); 879 } 880 881 static int safexcel_ahash_export(struct ahash_request *areq, void *out) 882 { 883 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 884 struct safexcel_ahash_export_state *export = out; 885 886 export->len = req->len; 887 export->processed = req->processed; 888 889 export->digest = req->digest; 890 891 memcpy(export->state, req->state, req->state_sz); 892 memcpy(export->cache, req->cache, HASH_CACHE_SIZE); 893 894 return 0; 895 } 896 897 static int safexcel_ahash_import(struct ahash_request *areq, const void *in) 898 { 899 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 900 const struct safexcel_ahash_export_state *export = in; 901 int ret; 902 903 ret = crypto_ahash_init(areq); 904 if (ret) 905 return ret; 906 907 req->len = export->len; 908 req->processed = export->processed; 909 910 req->digest = export->digest; 911 912 memcpy(req->cache, export->cache, HASH_CACHE_SIZE); 913 memcpy(req->state, export->state, req->state_sz); 914 915 return 0; 916 } 917 918 static int safexcel_ahash_cra_init(struct crypto_tfm *tfm) 919 { 920 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 921 struct safexcel_alg_template *tmpl = 922 container_of(__crypto_ahash_alg(tfm->__crt_alg), 923 struct safexcel_alg_template, alg.ahash); 924 925 ctx->base.priv = tmpl->priv; 926 ctx->base.send = safexcel_ahash_send; 927 ctx->base.handle_result = safexcel_handle_result; 928 ctx->fb_do_setkey = false; 929 930 crypto_ahash_set_reqsize_dma(__crypto_ahash_cast(tfm), 931 sizeof(struct safexcel_ahash_req)); 932 return 0; 933 } 934 935 static int safexcel_sha1_init(struct ahash_request *areq) 936 { 937 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 938 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 939 940 memset(req, 0, sizeof(*req)); 941 942 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1; 943 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 944 req->state_sz = SHA1_DIGEST_SIZE; 945 req->digest_sz = SHA1_DIGEST_SIZE; 946 req->block_sz = SHA1_BLOCK_SIZE; 947 948 return 0; 949 } 950 951 static int safexcel_sha1_digest(struct ahash_request *areq) 952 { 953 int ret = safexcel_sha1_init(areq); 954 955 if (ret) 956 return ret; 957 958 return safexcel_ahash_finup(areq); 959 } 960 961 static void safexcel_ahash_cra_exit(struct crypto_tfm *tfm) 962 { 963 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 964 struct safexcel_crypto_priv *priv = ctx->base.priv; 965 int ret; 966 967 /* context not allocated, skip invalidation */ 968 if (!ctx->base.ctxr) 969 return; 970 971 if (priv->flags & EIP197_TRC_CACHE) { 972 ret = safexcel_ahash_exit_inv(tfm); 973 if (ret) 974 dev_warn(priv->dev, "hash: invalidation error %d\n", ret); 975 } else { 976 dma_pool_free(priv->context_pool, ctx->base.ctxr, 977 ctx->base.ctxr_dma); 978 } 979 } 980 981 struct safexcel_alg_template safexcel_alg_sha1 = { 982 .type = SAFEXCEL_ALG_TYPE_AHASH, 983 .algo_mask = SAFEXCEL_ALG_SHA1, 984 .alg.ahash = { 985 .init = safexcel_sha1_init, 986 .update = safexcel_ahash_update, 987 .final = safexcel_ahash_final, 988 .finup = safexcel_ahash_finup, 989 .digest = safexcel_sha1_digest, 990 .export = safexcel_ahash_export, 991 .import = safexcel_ahash_import, 992 .halg = { 993 .digestsize = SHA1_DIGEST_SIZE, 994 .statesize = sizeof(struct safexcel_ahash_export_state), 995 .base = { 996 .cra_name = "sha1", 997 .cra_driver_name = "safexcel-sha1", 998 .cra_priority = SAFEXCEL_CRA_PRIORITY, 999 .cra_flags = CRYPTO_ALG_ASYNC | 1000 CRYPTO_ALG_ALLOCATES_MEMORY | 1001 CRYPTO_ALG_KERN_DRIVER_ONLY, 1002 .cra_blocksize = SHA1_BLOCK_SIZE, 1003 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1004 .cra_init = safexcel_ahash_cra_init, 1005 .cra_exit = safexcel_ahash_cra_exit, 1006 .cra_module = THIS_MODULE, 1007 }, 1008 }, 1009 }, 1010 }; 1011 1012 static int safexcel_hmac_sha1_init(struct ahash_request *areq) 1013 { 1014 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1015 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1016 1017 memset(req, 0, sizeof(*req)); 1018 1019 /* Start from ipad precompute */ 1020 memcpy(req->state, &ctx->base.ipad, SHA1_DIGEST_SIZE); 1021 /* Already processed the key^ipad part now! */ 1022 req->len = SHA1_BLOCK_SIZE; 1023 req->processed = SHA1_BLOCK_SIZE; 1024 1025 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1; 1026 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1027 req->state_sz = SHA1_DIGEST_SIZE; 1028 req->digest_sz = SHA1_DIGEST_SIZE; 1029 req->block_sz = SHA1_BLOCK_SIZE; 1030 req->hmac = true; 1031 1032 return 0; 1033 } 1034 1035 static int safexcel_hmac_sha1_digest(struct ahash_request *areq) 1036 { 1037 int ret = safexcel_hmac_sha1_init(areq); 1038 1039 if (ret) 1040 return ret; 1041 1042 return safexcel_ahash_finup(areq); 1043 } 1044 1045 static int safexcel_hmac_init_pad(struct ahash_request *areq, 1046 unsigned int blocksize, const u8 *key, 1047 unsigned int keylen, u8 *ipad, u8 *opad) 1048 { 1049 DECLARE_CRYPTO_WAIT(result); 1050 struct scatterlist sg; 1051 int ret, i; 1052 u8 *keydup; 1053 1054 if (keylen <= blocksize) { 1055 memcpy(ipad, key, keylen); 1056 } else { 1057 keydup = kmemdup(key, keylen, GFP_KERNEL); 1058 if (!keydup) 1059 return -ENOMEM; 1060 1061 ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG, 1062 crypto_req_done, &result); 1063 sg_init_one(&sg, keydup, keylen); 1064 ahash_request_set_crypt(areq, &sg, ipad, keylen); 1065 1066 ret = crypto_ahash_digest(areq); 1067 ret = crypto_wait_req(ret, &result); 1068 1069 /* Avoid leaking */ 1070 kfree_sensitive(keydup); 1071 1072 if (ret) 1073 return ret; 1074 1075 keylen = crypto_ahash_digestsize(crypto_ahash_reqtfm(areq)); 1076 } 1077 1078 memset(ipad + keylen, 0, blocksize - keylen); 1079 memcpy(opad, ipad, blocksize); 1080 1081 for (i = 0; i < blocksize; i++) { 1082 ipad[i] ^= HMAC_IPAD_VALUE; 1083 opad[i] ^= HMAC_OPAD_VALUE; 1084 } 1085 1086 return 0; 1087 } 1088 1089 static int safexcel_hmac_init_iv(struct ahash_request *areq, 1090 unsigned int blocksize, u8 *pad, void *state) 1091 { 1092 struct safexcel_ahash_req *req; 1093 DECLARE_CRYPTO_WAIT(result); 1094 struct scatterlist sg; 1095 int ret; 1096 1097 ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG, 1098 crypto_req_done, &result); 1099 sg_init_one(&sg, pad, blocksize); 1100 ahash_request_set_crypt(areq, &sg, pad, blocksize); 1101 1102 ret = crypto_ahash_init(areq); 1103 if (ret) 1104 return ret; 1105 1106 req = ahash_request_ctx_dma(areq); 1107 req->hmac = true; 1108 req->last_req = true; 1109 1110 ret = crypto_ahash_update(areq); 1111 ret = crypto_wait_req(ret, &result); 1112 1113 return ret ?: crypto_ahash_export(areq, state); 1114 } 1115 1116 static int __safexcel_hmac_setkey(const char *alg, const u8 *key, 1117 unsigned int keylen, 1118 void *istate, void *ostate) 1119 { 1120 struct ahash_request *areq; 1121 struct crypto_ahash *tfm; 1122 unsigned int blocksize; 1123 u8 *ipad, *opad; 1124 int ret; 1125 1126 tfm = crypto_alloc_ahash(alg, 0, 0); 1127 if (IS_ERR(tfm)) 1128 return PTR_ERR(tfm); 1129 1130 areq = ahash_request_alloc(tfm, GFP_KERNEL); 1131 if (!areq) { 1132 ret = -ENOMEM; 1133 goto free_ahash; 1134 } 1135 1136 crypto_ahash_clear_flags(tfm, ~0); 1137 blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 1138 1139 ipad = kcalloc(2, blocksize, GFP_KERNEL); 1140 if (!ipad) { 1141 ret = -ENOMEM; 1142 goto free_request; 1143 } 1144 1145 opad = ipad + blocksize; 1146 1147 ret = safexcel_hmac_init_pad(areq, blocksize, key, keylen, ipad, opad); 1148 if (ret) 1149 goto free_ipad; 1150 1151 ret = safexcel_hmac_init_iv(areq, blocksize, ipad, istate); 1152 if (ret) 1153 goto free_ipad; 1154 1155 ret = safexcel_hmac_init_iv(areq, blocksize, opad, ostate); 1156 1157 free_ipad: 1158 kfree(ipad); 1159 free_request: 1160 ahash_request_free(areq); 1161 free_ahash: 1162 crypto_free_ahash(tfm); 1163 1164 return ret; 1165 } 1166 1167 int safexcel_hmac_setkey(struct safexcel_context *base, const u8 *key, 1168 unsigned int keylen, const char *alg, 1169 unsigned int state_sz) 1170 { 1171 struct safexcel_crypto_priv *priv = base->priv; 1172 struct safexcel_ahash_export_state istate, ostate; 1173 int ret; 1174 1175 ret = __safexcel_hmac_setkey(alg, key, keylen, &istate, &ostate); 1176 if (ret) 1177 return ret; 1178 1179 if (priv->flags & EIP197_TRC_CACHE && base->ctxr && 1180 (memcmp(&base->ipad, istate.state, state_sz) || 1181 memcmp(&base->opad, ostate.state, state_sz))) 1182 base->needs_inv = true; 1183 1184 memcpy(&base->ipad, &istate.state, state_sz); 1185 memcpy(&base->opad, &ostate.state, state_sz); 1186 1187 return 0; 1188 } 1189 1190 static int safexcel_hmac_alg_setkey(struct crypto_ahash *tfm, const u8 *key, 1191 unsigned int keylen, const char *alg, 1192 unsigned int state_sz) 1193 { 1194 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 1195 1196 return safexcel_hmac_setkey(&ctx->base, key, keylen, alg, state_sz); 1197 } 1198 1199 static int safexcel_hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key, 1200 unsigned int keylen) 1201 { 1202 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha1", 1203 SHA1_DIGEST_SIZE); 1204 } 1205 1206 struct safexcel_alg_template safexcel_alg_hmac_sha1 = { 1207 .type = SAFEXCEL_ALG_TYPE_AHASH, 1208 .algo_mask = SAFEXCEL_ALG_SHA1, 1209 .alg.ahash = { 1210 .init = safexcel_hmac_sha1_init, 1211 .update = safexcel_ahash_update, 1212 .final = safexcel_ahash_final, 1213 .finup = safexcel_ahash_finup, 1214 .digest = safexcel_hmac_sha1_digest, 1215 .setkey = safexcel_hmac_sha1_setkey, 1216 .export = safexcel_ahash_export, 1217 .import = safexcel_ahash_import, 1218 .halg = { 1219 .digestsize = SHA1_DIGEST_SIZE, 1220 .statesize = sizeof(struct safexcel_ahash_export_state), 1221 .base = { 1222 .cra_name = "hmac(sha1)", 1223 .cra_driver_name = "safexcel-hmac-sha1", 1224 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1225 .cra_flags = CRYPTO_ALG_ASYNC | 1226 CRYPTO_ALG_ALLOCATES_MEMORY | 1227 CRYPTO_ALG_KERN_DRIVER_ONLY, 1228 .cra_blocksize = SHA1_BLOCK_SIZE, 1229 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1230 .cra_init = safexcel_ahash_cra_init, 1231 .cra_exit = safexcel_ahash_cra_exit, 1232 .cra_module = THIS_MODULE, 1233 }, 1234 }, 1235 }, 1236 }; 1237 1238 static int safexcel_sha256_init(struct ahash_request *areq) 1239 { 1240 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1241 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1242 1243 memset(req, 0, sizeof(*req)); 1244 1245 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256; 1246 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1247 req->state_sz = SHA256_DIGEST_SIZE; 1248 req->digest_sz = SHA256_DIGEST_SIZE; 1249 req->block_sz = SHA256_BLOCK_SIZE; 1250 1251 return 0; 1252 } 1253 1254 static int safexcel_sha256_digest(struct ahash_request *areq) 1255 { 1256 int ret = safexcel_sha256_init(areq); 1257 1258 if (ret) 1259 return ret; 1260 1261 return safexcel_ahash_finup(areq); 1262 } 1263 1264 struct safexcel_alg_template safexcel_alg_sha256 = { 1265 .type = SAFEXCEL_ALG_TYPE_AHASH, 1266 .algo_mask = SAFEXCEL_ALG_SHA2_256, 1267 .alg.ahash = { 1268 .init = safexcel_sha256_init, 1269 .update = safexcel_ahash_update, 1270 .final = safexcel_ahash_final, 1271 .finup = safexcel_ahash_finup, 1272 .digest = safexcel_sha256_digest, 1273 .export = safexcel_ahash_export, 1274 .import = safexcel_ahash_import, 1275 .halg = { 1276 .digestsize = SHA256_DIGEST_SIZE, 1277 .statesize = sizeof(struct safexcel_ahash_export_state), 1278 .base = { 1279 .cra_name = "sha256", 1280 .cra_driver_name = "safexcel-sha256", 1281 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1282 .cra_flags = CRYPTO_ALG_ASYNC | 1283 CRYPTO_ALG_ALLOCATES_MEMORY | 1284 CRYPTO_ALG_KERN_DRIVER_ONLY, 1285 .cra_blocksize = SHA256_BLOCK_SIZE, 1286 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1287 .cra_init = safexcel_ahash_cra_init, 1288 .cra_exit = safexcel_ahash_cra_exit, 1289 .cra_module = THIS_MODULE, 1290 }, 1291 }, 1292 }, 1293 }; 1294 1295 static int safexcel_sha224_init(struct ahash_request *areq) 1296 { 1297 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1298 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1299 1300 memset(req, 0, sizeof(*req)); 1301 1302 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224; 1303 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1304 req->state_sz = SHA256_DIGEST_SIZE; 1305 req->digest_sz = SHA256_DIGEST_SIZE; 1306 req->block_sz = SHA256_BLOCK_SIZE; 1307 1308 return 0; 1309 } 1310 1311 static int safexcel_sha224_digest(struct ahash_request *areq) 1312 { 1313 int ret = safexcel_sha224_init(areq); 1314 1315 if (ret) 1316 return ret; 1317 1318 return safexcel_ahash_finup(areq); 1319 } 1320 1321 struct safexcel_alg_template safexcel_alg_sha224 = { 1322 .type = SAFEXCEL_ALG_TYPE_AHASH, 1323 .algo_mask = SAFEXCEL_ALG_SHA2_256, 1324 .alg.ahash = { 1325 .init = safexcel_sha224_init, 1326 .update = safexcel_ahash_update, 1327 .final = safexcel_ahash_final, 1328 .finup = safexcel_ahash_finup, 1329 .digest = safexcel_sha224_digest, 1330 .export = safexcel_ahash_export, 1331 .import = safexcel_ahash_import, 1332 .halg = { 1333 .digestsize = SHA224_DIGEST_SIZE, 1334 .statesize = sizeof(struct safexcel_ahash_export_state), 1335 .base = { 1336 .cra_name = "sha224", 1337 .cra_driver_name = "safexcel-sha224", 1338 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1339 .cra_flags = CRYPTO_ALG_ASYNC | 1340 CRYPTO_ALG_ALLOCATES_MEMORY | 1341 CRYPTO_ALG_KERN_DRIVER_ONLY, 1342 .cra_blocksize = SHA224_BLOCK_SIZE, 1343 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1344 .cra_init = safexcel_ahash_cra_init, 1345 .cra_exit = safexcel_ahash_cra_exit, 1346 .cra_module = THIS_MODULE, 1347 }, 1348 }, 1349 }, 1350 }; 1351 1352 static int safexcel_hmac_sha224_setkey(struct crypto_ahash *tfm, const u8 *key, 1353 unsigned int keylen) 1354 { 1355 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha224", 1356 SHA256_DIGEST_SIZE); 1357 } 1358 1359 static int safexcel_hmac_sha224_init(struct ahash_request *areq) 1360 { 1361 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1362 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1363 1364 memset(req, 0, sizeof(*req)); 1365 1366 /* Start from ipad precompute */ 1367 memcpy(req->state, &ctx->base.ipad, SHA256_DIGEST_SIZE); 1368 /* Already processed the key^ipad part now! */ 1369 req->len = SHA256_BLOCK_SIZE; 1370 req->processed = SHA256_BLOCK_SIZE; 1371 1372 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224; 1373 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1374 req->state_sz = SHA256_DIGEST_SIZE; 1375 req->digest_sz = SHA256_DIGEST_SIZE; 1376 req->block_sz = SHA256_BLOCK_SIZE; 1377 req->hmac = true; 1378 1379 return 0; 1380 } 1381 1382 static int safexcel_hmac_sha224_digest(struct ahash_request *areq) 1383 { 1384 int ret = safexcel_hmac_sha224_init(areq); 1385 1386 if (ret) 1387 return ret; 1388 1389 return safexcel_ahash_finup(areq); 1390 } 1391 1392 struct safexcel_alg_template safexcel_alg_hmac_sha224 = { 1393 .type = SAFEXCEL_ALG_TYPE_AHASH, 1394 .algo_mask = SAFEXCEL_ALG_SHA2_256, 1395 .alg.ahash = { 1396 .init = safexcel_hmac_sha224_init, 1397 .update = safexcel_ahash_update, 1398 .final = safexcel_ahash_final, 1399 .finup = safexcel_ahash_finup, 1400 .digest = safexcel_hmac_sha224_digest, 1401 .setkey = safexcel_hmac_sha224_setkey, 1402 .export = safexcel_ahash_export, 1403 .import = safexcel_ahash_import, 1404 .halg = { 1405 .digestsize = SHA224_DIGEST_SIZE, 1406 .statesize = sizeof(struct safexcel_ahash_export_state), 1407 .base = { 1408 .cra_name = "hmac(sha224)", 1409 .cra_driver_name = "safexcel-hmac-sha224", 1410 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1411 .cra_flags = CRYPTO_ALG_ASYNC | 1412 CRYPTO_ALG_ALLOCATES_MEMORY | 1413 CRYPTO_ALG_KERN_DRIVER_ONLY, 1414 .cra_blocksize = SHA224_BLOCK_SIZE, 1415 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1416 .cra_init = safexcel_ahash_cra_init, 1417 .cra_exit = safexcel_ahash_cra_exit, 1418 .cra_module = THIS_MODULE, 1419 }, 1420 }, 1421 }, 1422 }; 1423 1424 static int safexcel_hmac_sha256_setkey(struct crypto_ahash *tfm, const u8 *key, 1425 unsigned int keylen) 1426 { 1427 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha256", 1428 SHA256_DIGEST_SIZE); 1429 } 1430 1431 static int safexcel_hmac_sha256_init(struct ahash_request *areq) 1432 { 1433 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1434 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1435 1436 memset(req, 0, sizeof(*req)); 1437 1438 /* Start from ipad precompute */ 1439 memcpy(req->state, &ctx->base.ipad, SHA256_DIGEST_SIZE); 1440 /* Already processed the key^ipad part now! */ 1441 req->len = SHA256_BLOCK_SIZE; 1442 req->processed = SHA256_BLOCK_SIZE; 1443 1444 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256; 1445 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1446 req->state_sz = SHA256_DIGEST_SIZE; 1447 req->digest_sz = SHA256_DIGEST_SIZE; 1448 req->block_sz = SHA256_BLOCK_SIZE; 1449 req->hmac = true; 1450 1451 return 0; 1452 } 1453 1454 static int safexcel_hmac_sha256_digest(struct ahash_request *areq) 1455 { 1456 int ret = safexcel_hmac_sha256_init(areq); 1457 1458 if (ret) 1459 return ret; 1460 1461 return safexcel_ahash_finup(areq); 1462 } 1463 1464 struct safexcel_alg_template safexcel_alg_hmac_sha256 = { 1465 .type = SAFEXCEL_ALG_TYPE_AHASH, 1466 .algo_mask = SAFEXCEL_ALG_SHA2_256, 1467 .alg.ahash = { 1468 .init = safexcel_hmac_sha256_init, 1469 .update = safexcel_ahash_update, 1470 .final = safexcel_ahash_final, 1471 .finup = safexcel_ahash_finup, 1472 .digest = safexcel_hmac_sha256_digest, 1473 .setkey = safexcel_hmac_sha256_setkey, 1474 .export = safexcel_ahash_export, 1475 .import = safexcel_ahash_import, 1476 .halg = { 1477 .digestsize = SHA256_DIGEST_SIZE, 1478 .statesize = sizeof(struct safexcel_ahash_export_state), 1479 .base = { 1480 .cra_name = "hmac(sha256)", 1481 .cra_driver_name = "safexcel-hmac-sha256", 1482 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1483 .cra_flags = CRYPTO_ALG_ASYNC | 1484 CRYPTO_ALG_ALLOCATES_MEMORY | 1485 CRYPTO_ALG_KERN_DRIVER_ONLY, 1486 .cra_blocksize = SHA256_BLOCK_SIZE, 1487 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1488 .cra_init = safexcel_ahash_cra_init, 1489 .cra_exit = safexcel_ahash_cra_exit, 1490 .cra_module = THIS_MODULE, 1491 }, 1492 }, 1493 }, 1494 }; 1495 1496 static int safexcel_sha512_init(struct ahash_request *areq) 1497 { 1498 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1499 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1500 1501 memset(req, 0, sizeof(*req)); 1502 1503 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512; 1504 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1505 req->state_sz = SHA512_DIGEST_SIZE; 1506 req->digest_sz = SHA512_DIGEST_SIZE; 1507 req->block_sz = SHA512_BLOCK_SIZE; 1508 1509 return 0; 1510 } 1511 1512 static int safexcel_sha512_digest(struct ahash_request *areq) 1513 { 1514 int ret = safexcel_sha512_init(areq); 1515 1516 if (ret) 1517 return ret; 1518 1519 return safexcel_ahash_finup(areq); 1520 } 1521 1522 struct safexcel_alg_template safexcel_alg_sha512 = { 1523 .type = SAFEXCEL_ALG_TYPE_AHASH, 1524 .algo_mask = SAFEXCEL_ALG_SHA2_512, 1525 .alg.ahash = { 1526 .init = safexcel_sha512_init, 1527 .update = safexcel_ahash_update, 1528 .final = safexcel_ahash_final, 1529 .finup = safexcel_ahash_finup, 1530 .digest = safexcel_sha512_digest, 1531 .export = safexcel_ahash_export, 1532 .import = safexcel_ahash_import, 1533 .halg = { 1534 .digestsize = SHA512_DIGEST_SIZE, 1535 .statesize = sizeof(struct safexcel_ahash_export_state), 1536 .base = { 1537 .cra_name = "sha512", 1538 .cra_driver_name = "safexcel-sha512", 1539 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1540 .cra_flags = CRYPTO_ALG_ASYNC | 1541 CRYPTO_ALG_ALLOCATES_MEMORY | 1542 CRYPTO_ALG_KERN_DRIVER_ONLY, 1543 .cra_blocksize = SHA512_BLOCK_SIZE, 1544 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1545 .cra_init = safexcel_ahash_cra_init, 1546 .cra_exit = safexcel_ahash_cra_exit, 1547 .cra_module = THIS_MODULE, 1548 }, 1549 }, 1550 }, 1551 }; 1552 1553 static int safexcel_sha384_init(struct ahash_request *areq) 1554 { 1555 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1556 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1557 1558 memset(req, 0, sizeof(*req)); 1559 1560 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384; 1561 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1562 req->state_sz = SHA512_DIGEST_SIZE; 1563 req->digest_sz = SHA512_DIGEST_SIZE; 1564 req->block_sz = SHA512_BLOCK_SIZE; 1565 1566 return 0; 1567 } 1568 1569 static int safexcel_sha384_digest(struct ahash_request *areq) 1570 { 1571 int ret = safexcel_sha384_init(areq); 1572 1573 if (ret) 1574 return ret; 1575 1576 return safexcel_ahash_finup(areq); 1577 } 1578 1579 struct safexcel_alg_template safexcel_alg_sha384 = { 1580 .type = SAFEXCEL_ALG_TYPE_AHASH, 1581 .algo_mask = SAFEXCEL_ALG_SHA2_512, 1582 .alg.ahash = { 1583 .init = safexcel_sha384_init, 1584 .update = safexcel_ahash_update, 1585 .final = safexcel_ahash_final, 1586 .finup = safexcel_ahash_finup, 1587 .digest = safexcel_sha384_digest, 1588 .export = safexcel_ahash_export, 1589 .import = safexcel_ahash_import, 1590 .halg = { 1591 .digestsize = SHA384_DIGEST_SIZE, 1592 .statesize = sizeof(struct safexcel_ahash_export_state), 1593 .base = { 1594 .cra_name = "sha384", 1595 .cra_driver_name = "safexcel-sha384", 1596 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1597 .cra_flags = CRYPTO_ALG_ASYNC | 1598 CRYPTO_ALG_ALLOCATES_MEMORY | 1599 CRYPTO_ALG_KERN_DRIVER_ONLY, 1600 .cra_blocksize = SHA384_BLOCK_SIZE, 1601 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1602 .cra_init = safexcel_ahash_cra_init, 1603 .cra_exit = safexcel_ahash_cra_exit, 1604 .cra_module = THIS_MODULE, 1605 }, 1606 }, 1607 }, 1608 }; 1609 1610 static int safexcel_hmac_sha512_setkey(struct crypto_ahash *tfm, const u8 *key, 1611 unsigned int keylen) 1612 { 1613 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha512", 1614 SHA512_DIGEST_SIZE); 1615 } 1616 1617 static int safexcel_hmac_sha512_init(struct ahash_request *areq) 1618 { 1619 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1620 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1621 1622 memset(req, 0, sizeof(*req)); 1623 1624 /* Start from ipad precompute */ 1625 memcpy(req->state, &ctx->base.ipad, SHA512_DIGEST_SIZE); 1626 /* Already processed the key^ipad part now! */ 1627 req->len = SHA512_BLOCK_SIZE; 1628 req->processed = SHA512_BLOCK_SIZE; 1629 1630 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512; 1631 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1632 req->state_sz = SHA512_DIGEST_SIZE; 1633 req->digest_sz = SHA512_DIGEST_SIZE; 1634 req->block_sz = SHA512_BLOCK_SIZE; 1635 req->hmac = true; 1636 1637 return 0; 1638 } 1639 1640 static int safexcel_hmac_sha512_digest(struct ahash_request *areq) 1641 { 1642 int ret = safexcel_hmac_sha512_init(areq); 1643 1644 if (ret) 1645 return ret; 1646 1647 return safexcel_ahash_finup(areq); 1648 } 1649 1650 struct safexcel_alg_template safexcel_alg_hmac_sha512 = { 1651 .type = SAFEXCEL_ALG_TYPE_AHASH, 1652 .algo_mask = SAFEXCEL_ALG_SHA2_512, 1653 .alg.ahash = { 1654 .init = safexcel_hmac_sha512_init, 1655 .update = safexcel_ahash_update, 1656 .final = safexcel_ahash_final, 1657 .finup = safexcel_ahash_finup, 1658 .digest = safexcel_hmac_sha512_digest, 1659 .setkey = safexcel_hmac_sha512_setkey, 1660 .export = safexcel_ahash_export, 1661 .import = safexcel_ahash_import, 1662 .halg = { 1663 .digestsize = SHA512_DIGEST_SIZE, 1664 .statesize = sizeof(struct safexcel_ahash_export_state), 1665 .base = { 1666 .cra_name = "hmac(sha512)", 1667 .cra_driver_name = "safexcel-hmac-sha512", 1668 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1669 .cra_flags = CRYPTO_ALG_ASYNC | 1670 CRYPTO_ALG_ALLOCATES_MEMORY | 1671 CRYPTO_ALG_KERN_DRIVER_ONLY, 1672 .cra_blocksize = SHA512_BLOCK_SIZE, 1673 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1674 .cra_init = safexcel_ahash_cra_init, 1675 .cra_exit = safexcel_ahash_cra_exit, 1676 .cra_module = THIS_MODULE, 1677 }, 1678 }, 1679 }, 1680 }; 1681 1682 static int safexcel_hmac_sha384_setkey(struct crypto_ahash *tfm, const u8 *key, 1683 unsigned int keylen) 1684 { 1685 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha384", 1686 SHA512_DIGEST_SIZE); 1687 } 1688 1689 static int safexcel_hmac_sha384_init(struct ahash_request *areq) 1690 { 1691 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1692 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1693 1694 memset(req, 0, sizeof(*req)); 1695 1696 /* Start from ipad precompute */ 1697 memcpy(req->state, &ctx->base.ipad, SHA512_DIGEST_SIZE); 1698 /* Already processed the key^ipad part now! */ 1699 req->len = SHA512_BLOCK_SIZE; 1700 req->processed = SHA512_BLOCK_SIZE; 1701 1702 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384; 1703 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1704 req->state_sz = SHA512_DIGEST_SIZE; 1705 req->digest_sz = SHA512_DIGEST_SIZE; 1706 req->block_sz = SHA512_BLOCK_SIZE; 1707 req->hmac = true; 1708 1709 return 0; 1710 } 1711 1712 static int safexcel_hmac_sha384_digest(struct ahash_request *areq) 1713 { 1714 int ret = safexcel_hmac_sha384_init(areq); 1715 1716 if (ret) 1717 return ret; 1718 1719 return safexcel_ahash_finup(areq); 1720 } 1721 1722 struct safexcel_alg_template safexcel_alg_hmac_sha384 = { 1723 .type = SAFEXCEL_ALG_TYPE_AHASH, 1724 .algo_mask = SAFEXCEL_ALG_SHA2_512, 1725 .alg.ahash = { 1726 .init = safexcel_hmac_sha384_init, 1727 .update = safexcel_ahash_update, 1728 .final = safexcel_ahash_final, 1729 .finup = safexcel_ahash_finup, 1730 .digest = safexcel_hmac_sha384_digest, 1731 .setkey = safexcel_hmac_sha384_setkey, 1732 .export = safexcel_ahash_export, 1733 .import = safexcel_ahash_import, 1734 .halg = { 1735 .digestsize = SHA384_DIGEST_SIZE, 1736 .statesize = sizeof(struct safexcel_ahash_export_state), 1737 .base = { 1738 .cra_name = "hmac(sha384)", 1739 .cra_driver_name = "safexcel-hmac-sha384", 1740 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1741 .cra_flags = CRYPTO_ALG_ASYNC | 1742 CRYPTO_ALG_ALLOCATES_MEMORY | 1743 CRYPTO_ALG_KERN_DRIVER_ONLY, 1744 .cra_blocksize = SHA384_BLOCK_SIZE, 1745 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1746 .cra_init = safexcel_ahash_cra_init, 1747 .cra_exit = safexcel_ahash_cra_exit, 1748 .cra_module = THIS_MODULE, 1749 }, 1750 }, 1751 }, 1752 }; 1753 1754 static int safexcel_md5_init(struct ahash_request *areq) 1755 { 1756 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1757 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1758 1759 memset(req, 0, sizeof(*req)); 1760 1761 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5; 1762 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1763 req->state_sz = MD5_DIGEST_SIZE; 1764 req->digest_sz = MD5_DIGEST_SIZE; 1765 req->block_sz = MD5_HMAC_BLOCK_SIZE; 1766 1767 return 0; 1768 } 1769 1770 static int safexcel_md5_digest(struct ahash_request *areq) 1771 { 1772 int ret = safexcel_md5_init(areq); 1773 1774 if (ret) 1775 return ret; 1776 1777 return safexcel_ahash_finup(areq); 1778 } 1779 1780 struct safexcel_alg_template safexcel_alg_md5 = { 1781 .type = SAFEXCEL_ALG_TYPE_AHASH, 1782 .algo_mask = SAFEXCEL_ALG_MD5, 1783 .alg.ahash = { 1784 .init = safexcel_md5_init, 1785 .update = safexcel_ahash_update, 1786 .final = safexcel_ahash_final, 1787 .finup = safexcel_ahash_finup, 1788 .digest = safexcel_md5_digest, 1789 .export = safexcel_ahash_export, 1790 .import = safexcel_ahash_import, 1791 .halg = { 1792 .digestsize = MD5_DIGEST_SIZE, 1793 .statesize = sizeof(struct safexcel_ahash_export_state), 1794 .base = { 1795 .cra_name = "md5", 1796 .cra_driver_name = "safexcel-md5", 1797 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1798 .cra_flags = CRYPTO_ALG_ASYNC | 1799 CRYPTO_ALG_ALLOCATES_MEMORY | 1800 CRYPTO_ALG_KERN_DRIVER_ONLY, 1801 .cra_blocksize = MD5_HMAC_BLOCK_SIZE, 1802 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1803 .cra_init = safexcel_ahash_cra_init, 1804 .cra_exit = safexcel_ahash_cra_exit, 1805 .cra_module = THIS_MODULE, 1806 }, 1807 }, 1808 }, 1809 }; 1810 1811 static int safexcel_hmac_md5_init(struct ahash_request *areq) 1812 { 1813 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1814 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1815 1816 memset(req, 0, sizeof(*req)); 1817 1818 /* Start from ipad precompute */ 1819 memcpy(req->state, &ctx->base.ipad, MD5_DIGEST_SIZE); 1820 /* Already processed the key^ipad part now! */ 1821 req->len = MD5_HMAC_BLOCK_SIZE; 1822 req->processed = MD5_HMAC_BLOCK_SIZE; 1823 1824 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5; 1825 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1826 req->state_sz = MD5_DIGEST_SIZE; 1827 req->digest_sz = MD5_DIGEST_SIZE; 1828 req->block_sz = MD5_HMAC_BLOCK_SIZE; 1829 req->len_is_le = true; /* MD5 is little endian! ... */ 1830 req->hmac = true; 1831 1832 return 0; 1833 } 1834 1835 static int safexcel_hmac_md5_setkey(struct crypto_ahash *tfm, const u8 *key, 1836 unsigned int keylen) 1837 { 1838 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-md5", 1839 MD5_DIGEST_SIZE); 1840 } 1841 1842 static int safexcel_hmac_md5_digest(struct ahash_request *areq) 1843 { 1844 int ret = safexcel_hmac_md5_init(areq); 1845 1846 if (ret) 1847 return ret; 1848 1849 return safexcel_ahash_finup(areq); 1850 } 1851 1852 struct safexcel_alg_template safexcel_alg_hmac_md5 = { 1853 .type = SAFEXCEL_ALG_TYPE_AHASH, 1854 .algo_mask = SAFEXCEL_ALG_MD5, 1855 .alg.ahash = { 1856 .init = safexcel_hmac_md5_init, 1857 .update = safexcel_ahash_update, 1858 .final = safexcel_ahash_final, 1859 .finup = safexcel_ahash_finup, 1860 .digest = safexcel_hmac_md5_digest, 1861 .setkey = safexcel_hmac_md5_setkey, 1862 .export = safexcel_ahash_export, 1863 .import = safexcel_ahash_import, 1864 .halg = { 1865 .digestsize = MD5_DIGEST_SIZE, 1866 .statesize = sizeof(struct safexcel_ahash_export_state), 1867 .base = { 1868 .cra_name = "hmac(md5)", 1869 .cra_driver_name = "safexcel-hmac-md5", 1870 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1871 .cra_flags = CRYPTO_ALG_ASYNC | 1872 CRYPTO_ALG_ALLOCATES_MEMORY | 1873 CRYPTO_ALG_KERN_DRIVER_ONLY, 1874 .cra_blocksize = MD5_HMAC_BLOCK_SIZE, 1875 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1876 .cra_init = safexcel_ahash_cra_init, 1877 .cra_exit = safexcel_ahash_cra_exit, 1878 .cra_module = THIS_MODULE, 1879 }, 1880 }, 1881 }, 1882 }; 1883 1884 static int safexcel_crc32_cra_init(struct crypto_tfm *tfm) 1885 { 1886 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 1887 int ret = safexcel_ahash_cra_init(tfm); 1888 1889 /* Default 'key' is all zeroes */ 1890 memset(&ctx->base.ipad, 0, sizeof(u32)); 1891 return ret; 1892 } 1893 1894 static int safexcel_crc32_init(struct ahash_request *areq) 1895 { 1896 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1897 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1898 1899 memset(req, 0, sizeof(*req)); 1900 1901 /* Start from loaded key */ 1902 req->state[0] = cpu_to_le32(~ctx->base.ipad.word[0]); 1903 /* Set processed to non-zero to enable invalidation detection */ 1904 req->len = sizeof(u32); 1905 req->processed = sizeof(u32); 1906 1907 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_CRC32; 1908 req->digest = CONTEXT_CONTROL_DIGEST_XCM; 1909 req->state_sz = sizeof(u32); 1910 req->digest_sz = sizeof(u32); 1911 req->block_sz = sizeof(u32); 1912 1913 return 0; 1914 } 1915 1916 static int safexcel_crc32_setkey(struct crypto_ahash *tfm, const u8 *key, 1917 unsigned int keylen) 1918 { 1919 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 1920 1921 if (keylen != sizeof(u32)) 1922 return -EINVAL; 1923 1924 memcpy(&ctx->base.ipad, key, sizeof(u32)); 1925 return 0; 1926 } 1927 1928 static int safexcel_crc32_digest(struct ahash_request *areq) 1929 { 1930 return safexcel_crc32_init(areq) ?: safexcel_ahash_finup(areq); 1931 } 1932 1933 struct safexcel_alg_template safexcel_alg_crc32 = { 1934 .type = SAFEXCEL_ALG_TYPE_AHASH, 1935 .algo_mask = 0, 1936 .alg.ahash = { 1937 .init = safexcel_crc32_init, 1938 .update = safexcel_ahash_update, 1939 .final = safexcel_ahash_final, 1940 .finup = safexcel_ahash_finup, 1941 .digest = safexcel_crc32_digest, 1942 .setkey = safexcel_crc32_setkey, 1943 .export = safexcel_ahash_export, 1944 .import = safexcel_ahash_import, 1945 .halg = { 1946 .digestsize = sizeof(u32), 1947 .statesize = sizeof(struct safexcel_ahash_export_state), 1948 .base = { 1949 .cra_name = "crc32", 1950 .cra_driver_name = "safexcel-crc32", 1951 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1952 .cra_flags = CRYPTO_ALG_OPTIONAL_KEY | 1953 CRYPTO_ALG_ASYNC | 1954 CRYPTO_ALG_ALLOCATES_MEMORY | 1955 CRYPTO_ALG_KERN_DRIVER_ONLY, 1956 .cra_blocksize = 1, 1957 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1958 .cra_init = safexcel_crc32_cra_init, 1959 .cra_exit = safexcel_ahash_cra_exit, 1960 .cra_module = THIS_MODULE, 1961 }, 1962 }, 1963 }, 1964 }; 1965 1966 static int safexcel_cbcmac_init(struct ahash_request *areq) 1967 { 1968 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1969 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1970 1971 memset(req, 0, sizeof(*req)); 1972 1973 /* Start from loaded keys */ 1974 memcpy(req->state, &ctx->base.ipad, ctx->key_sz); 1975 /* Set processed to non-zero to enable invalidation detection */ 1976 req->len = AES_BLOCK_SIZE; 1977 req->processed = AES_BLOCK_SIZE; 1978 1979 req->digest = CONTEXT_CONTROL_DIGEST_XCM; 1980 req->state_sz = ctx->key_sz; 1981 req->digest_sz = AES_BLOCK_SIZE; 1982 req->block_sz = AES_BLOCK_SIZE; 1983 req->xcbcmac = true; 1984 1985 return 0; 1986 } 1987 1988 static int safexcel_cbcmac_setkey(struct crypto_ahash *tfm, const u8 *key, 1989 unsigned int len) 1990 { 1991 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 1992 struct crypto_aes_ctx aes; 1993 int ret, i; 1994 1995 ret = aes_expandkey(&aes, key, len); 1996 if (ret) 1997 return ret; 1998 1999 memset(&ctx->base.ipad, 0, 2 * AES_BLOCK_SIZE); 2000 for (i = 0; i < len / sizeof(u32); i++) 2001 ctx->base.ipad.be[i + 8] = cpu_to_be32(aes.key_enc[i]); 2002 2003 if (len == AES_KEYSIZE_192) { 2004 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192; 2005 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE; 2006 } else if (len == AES_KEYSIZE_256) { 2007 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256; 2008 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE; 2009 } else { 2010 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128; 2011 ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE; 2012 } 2013 ctx->cbcmac = true; 2014 2015 memzero_explicit(&aes, sizeof(aes)); 2016 return 0; 2017 } 2018 2019 static int safexcel_cbcmac_digest(struct ahash_request *areq) 2020 { 2021 return safexcel_cbcmac_init(areq) ?: safexcel_ahash_finup(areq); 2022 } 2023 2024 struct safexcel_alg_template safexcel_alg_cbcmac = { 2025 .type = SAFEXCEL_ALG_TYPE_AHASH, 2026 .algo_mask = 0, 2027 .alg.ahash = { 2028 .init = safexcel_cbcmac_init, 2029 .update = safexcel_ahash_update, 2030 .final = safexcel_ahash_final, 2031 .finup = safexcel_ahash_finup, 2032 .digest = safexcel_cbcmac_digest, 2033 .setkey = safexcel_cbcmac_setkey, 2034 .export = safexcel_ahash_export, 2035 .import = safexcel_ahash_import, 2036 .halg = { 2037 .digestsize = AES_BLOCK_SIZE, 2038 .statesize = sizeof(struct safexcel_ahash_export_state), 2039 .base = { 2040 .cra_name = "cbcmac(aes)", 2041 .cra_driver_name = "safexcel-cbcmac-aes", 2042 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2043 .cra_flags = CRYPTO_ALG_ASYNC | 2044 CRYPTO_ALG_ALLOCATES_MEMORY | 2045 CRYPTO_ALG_KERN_DRIVER_ONLY, 2046 .cra_blocksize = 1, 2047 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 2048 .cra_init = safexcel_ahash_cra_init, 2049 .cra_exit = safexcel_ahash_cra_exit, 2050 .cra_module = THIS_MODULE, 2051 }, 2052 }, 2053 }, 2054 }; 2055 2056 static int safexcel_xcbcmac_setkey(struct crypto_ahash *tfm, const u8 *key, 2057 unsigned int len) 2058 { 2059 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 2060 u32 key_tmp[3 * AES_BLOCK_SIZE / sizeof(u32)]; 2061 int ret, i; 2062 2063 ret = aes_expandkey(ctx->aes, key, len); 2064 if (ret) 2065 return ret; 2066 2067 /* precompute the XCBC key material */ 2068 aes_encrypt(ctx->aes, (u8 *)key_tmp + 2 * AES_BLOCK_SIZE, 2069 "\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1"); 2070 aes_encrypt(ctx->aes, (u8 *)key_tmp, 2071 "\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2"); 2072 aes_encrypt(ctx->aes, (u8 *)key_tmp + AES_BLOCK_SIZE, 2073 "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3"); 2074 for (i = 0; i < 3 * AES_BLOCK_SIZE / sizeof(u32); i++) 2075 ctx->base.ipad.word[i] = swab32(key_tmp[i]); 2076 2077 ret = aes_expandkey(ctx->aes, 2078 (u8 *)key_tmp + 2 * AES_BLOCK_SIZE, 2079 AES_MIN_KEY_SIZE); 2080 if (ret) 2081 return ret; 2082 2083 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128; 2084 ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE; 2085 ctx->cbcmac = false; 2086 2087 return 0; 2088 } 2089 2090 static int safexcel_xcbcmac_cra_init(struct crypto_tfm *tfm) 2091 { 2092 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 2093 2094 safexcel_ahash_cra_init(tfm); 2095 ctx->aes = kmalloc(sizeof(*ctx->aes), GFP_KERNEL); 2096 return PTR_ERR_OR_ZERO(ctx->aes); 2097 } 2098 2099 static void safexcel_xcbcmac_cra_exit(struct crypto_tfm *tfm) 2100 { 2101 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 2102 2103 kfree(ctx->aes); 2104 safexcel_ahash_cra_exit(tfm); 2105 } 2106 2107 struct safexcel_alg_template safexcel_alg_xcbcmac = { 2108 .type = SAFEXCEL_ALG_TYPE_AHASH, 2109 .algo_mask = 0, 2110 .alg.ahash = { 2111 .init = safexcel_cbcmac_init, 2112 .update = safexcel_ahash_update, 2113 .final = safexcel_ahash_final, 2114 .finup = safexcel_ahash_finup, 2115 .digest = safexcel_cbcmac_digest, 2116 .setkey = safexcel_xcbcmac_setkey, 2117 .export = safexcel_ahash_export, 2118 .import = safexcel_ahash_import, 2119 .halg = { 2120 .digestsize = AES_BLOCK_SIZE, 2121 .statesize = sizeof(struct safexcel_ahash_export_state), 2122 .base = { 2123 .cra_name = "xcbc(aes)", 2124 .cra_driver_name = "safexcel-xcbc-aes", 2125 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2126 .cra_flags = CRYPTO_ALG_ASYNC | 2127 CRYPTO_ALG_ALLOCATES_MEMORY | 2128 CRYPTO_ALG_KERN_DRIVER_ONLY, 2129 .cra_blocksize = AES_BLOCK_SIZE, 2130 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 2131 .cra_init = safexcel_xcbcmac_cra_init, 2132 .cra_exit = safexcel_xcbcmac_cra_exit, 2133 .cra_module = THIS_MODULE, 2134 }, 2135 }, 2136 }, 2137 }; 2138 2139 static int safexcel_cmac_setkey(struct crypto_ahash *tfm, const u8 *key, 2140 unsigned int len) 2141 { 2142 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 2143 __be64 consts[4]; 2144 u64 _const[2]; 2145 u8 msb_mask, gfmask; 2146 int ret, i; 2147 2148 /* precompute the CMAC key material */ 2149 ret = aes_expandkey(ctx->aes, key, len); 2150 if (ret) 2151 return ret; 2152 2153 for (i = 0; i < len / sizeof(u32); i++) 2154 ctx->base.ipad.word[i + 8] = swab32(ctx->aes->key_enc[i]); 2155 2156 /* code below borrowed from crypto/cmac.c */ 2157 /* encrypt the zero block */ 2158 memset(consts, 0, AES_BLOCK_SIZE); 2159 aes_encrypt(ctx->aes, (u8 *)consts, (u8 *)consts); 2160 2161 gfmask = 0x87; 2162 _const[0] = be64_to_cpu(consts[1]); 2163 _const[1] = be64_to_cpu(consts[0]); 2164 2165 /* gf(2^128) multiply zero-ciphertext with u and u^2 */ 2166 for (i = 0; i < 4; i += 2) { 2167 msb_mask = ((s64)_const[1] >> 63) & gfmask; 2168 _const[1] = (_const[1] << 1) | (_const[0] >> 63); 2169 _const[0] = (_const[0] << 1) ^ msb_mask; 2170 2171 consts[i + 0] = cpu_to_be64(_const[1]); 2172 consts[i + 1] = cpu_to_be64(_const[0]); 2173 } 2174 /* end of code borrowed from crypto/cmac.c */ 2175 2176 for (i = 0; i < 2 * AES_BLOCK_SIZE / sizeof(u32); i++) 2177 ctx->base.ipad.be[i] = cpu_to_be32(((u32 *)consts)[i]); 2178 2179 if (len == AES_KEYSIZE_192) { 2180 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192; 2181 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE; 2182 } else if (len == AES_KEYSIZE_256) { 2183 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256; 2184 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE; 2185 } else { 2186 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128; 2187 ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE; 2188 } 2189 ctx->cbcmac = false; 2190 2191 return 0; 2192 } 2193 2194 struct safexcel_alg_template safexcel_alg_cmac = { 2195 .type = SAFEXCEL_ALG_TYPE_AHASH, 2196 .algo_mask = 0, 2197 .alg.ahash = { 2198 .init = safexcel_cbcmac_init, 2199 .update = safexcel_ahash_update, 2200 .final = safexcel_ahash_final, 2201 .finup = safexcel_ahash_finup, 2202 .digest = safexcel_cbcmac_digest, 2203 .setkey = safexcel_cmac_setkey, 2204 .export = safexcel_ahash_export, 2205 .import = safexcel_ahash_import, 2206 .halg = { 2207 .digestsize = AES_BLOCK_SIZE, 2208 .statesize = sizeof(struct safexcel_ahash_export_state), 2209 .base = { 2210 .cra_name = "cmac(aes)", 2211 .cra_driver_name = "safexcel-cmac-aes", 2212 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2213 .cra_flags = CRYPTO_ALG_ASYNC | 2214 CRYPTO_ALG_ALLOCATES_MEMORY | 2215 CRYPTO_ALG_KERN_DRIVER_ONLY, 2216 .cra_blocksize = AES_BLOCK_SIZE, 2217 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 2218 .cra_init = safexcel_xcbcmac_cra_init, 2219 .cra_exit = safexcel_xcbcmac_cra_exit, 2220 .cra_module = THIS_MODULE, 2221 }, 2222 }, 2223 }, 2224 }; 2225 2226 static int safexcel_sm3_init(struct ahash_request *areq) 2227 { 2228 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 2229 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 2230 2231 memset(req, 0, sizeof(*req)); 2232 2233 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3; 2234 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 2235 req->state_sz = SM3_DIGEST_SIZE; 2236 req->digest_sz = SM3_DIGEST_SIZE; 2237 req->block_sz = SM3_BLOCK_SIZE; 2238 2239 return 0; 2240 } 2241 2242 static int safexcel_sm3_digest(struct ahash_request *areq) 2243 { 2244 int ret = safexcel_sm3_init(areq); 2245 2246 if (ret) 2247 return ret; 2248 2249 return safexcel_ahash_finup(areq); 2250 } 2251 2252 struct safexcel_alg_template safexcel_alg_sm3 = { 2253 .type = SAFEXCEL_ALG_TYPE_AHASH, 2254 .algo_mask = SAFEXCEL_ALG_SM3, 2255 .alg.ahash = { 2256 .init = safexcel_sm3_init, 2257 .update = safexcel_ahash_update, 2258 .final = safexcel_ahash_final, 2259 .finup = safexcel_ahash_finup, 2260 .digest = safexcel_sm3_digest, 2261 .export = safexcel_ahash_export, 2262 .import = safexcel_ahash_import, 2263 .halg = { 2264 .digestsize = SM3_DIGEST_SIZE, 2265 .statesize = sizeof(struct safexcel_ahash_export_state), 2266 .base = { 2267 .cra_name = "sm3", 2268 .cra_driver_name = "safexcel-sm3", 2269 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2270 .cra_flags = CRYPTO_ALG_ASYNC | 2271 CRYPTO_ALG_ALLOCATES_MEMORY | 2272 CRYPTO_ALG_KERN_DRIVER_ONLY, 2273 .cra_blocksize = SM3_BLOCK_SIZE, 2274 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 2275 .cra_init = safexcel_ahash_cra_init, 2276 .cra_exit = safexcel_ahash_cra_exit, 2277 .cra_module = THIS_MODULE, 2278 }, 2279 }, 2280 }, 2281 }; 2282 2283 static int safexcel_hmac_sm3_setkey(struct crypto_ahash *tfm, const u8 *key, 2284 unsigned int keylen) 2285 { 2286 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sm3", 2287 SM3_DIGEST_SIZE); 2288 } 2289 2290 static int safexcel_hmac_sm3_init(struct ahash_request *areq) 2291 { 2292 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 2293 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 2294 2295 memset(req, 0, sizeof(*req)); 2296 2297 /* Start from ipad precompute */ 2298 memcpy(req->state, &ctx->base.ipad, SM3_DIGEST_SIZE); 2299 /* Already processed the key^ipad part now! */ 2300 req->len = SM3_BLOCK_SIZE; 2301 req->processed = SM3_BLOCK_SIZE; 2302 2303 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3; 2304 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 2305 req->state_sz = SM3_DIGEST_SIZE; 2306 req->digest_sz = SM3_DIGEST_SIZE; 2307 req->block_sz = SM3_BLOCK_SIZE; 2308 req->hmac = true; 2309 2310 return 0; 2311 } 2312 2313 static int safexcel_hmac_sm3_digest(struct ahash_request *areq) 2314 { 2315 int ret = safexcel_hmac_sm3_init(areq); 2316 2317 if (ret) 2318 return ret; 2319 2320 return safexcel_ahash_finup(areq); 2321 } 2322 2323 struct safexcel_alg_template safexcel_alg_hmac_sm3 = { 2324 .type = SAFEXCEL_ALG_TYPE_AHASH, 2325 .algo_mask = SAFEXCEL_ALG_SM3, 2326 .alg.ahash = { 2327 .init = safexcel_hmac_sm3_init, 2328 .update = safexcel_ahash_update, 2329 .final = safexcel_ahash_final, 2330 .finup = safexcel_ahash_finup, 2331 .digest = safexcel_hmac_sm3_digest, 2332 .setkey = safexcel_hmac_sm3_setkey, 2333 .export = safexcel_ahash_export, 2334 .import = safexcel_ahash_import, 2335 .halg = { 2336 .digestsize = SM3_DIGEST_SIZE, 2337 .statesize = sizeof(struct safexcel_ahash_export_state), 2338 .base = { 2339 .cra_name = "hmac(sm3)", 2340 .cra_driver_name = "safexcel-hmac-sm3", 2341 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2342 .cra_flags = CRYPTO_ALG_ASYNC | 2343 CRYPTO_ALG_ALLOCATES_MEMORY | 2344 CRYPTO_ALG_KERN_DRIVER_ONLY, 2345 .cra_blocksize = SM3_BLOCK_SIZE, 2346 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 2347 .cra_init = safexcel_ahash_cra_init, 2348 .cra_exit = safexcel_ahash_cra_exit, 2349 .cra_module = THIS_MODULE, 2350 }, 2351 }, 2352 }, 2353 }; 2354 2355 static int safexcel_sha3_224_init(struct ahash_request *areq) 2356 { 2357 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2358 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2359 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 2360 2361 memset(req, 0, sizeof(*req)); 2362 2363 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_224; 2364 req->digest = CONTEXT_CONTROL_DIGEST_INITIAL; 2365 req->state_sz = SHA3_224_DIGEST_SIZE; 2366 req->digest_sz = SHA3_224_DIGEST_SIZE; 2367 req->block_sz = SHA3_224_BLOCK_SIZE; 2368 ctx->do_fallback = false; 2369 ctx->fb_init_done = false; 2370 return 0; 2371 } 2372 2373 static int safexcel_sha3_fbcheck(struct ahash_request *req) 2374 { 2375 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 2376 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2377 struct ahash_request *subreq = ahash_request_ctx_dma(req); 2378 int ret = 0; 2379 2380 if (ctx->do_fallback) { 2381 ahash_request_set_tfm(subreq, ctx->fback); 2382 ahash_request_set_callback(subreq, req->base.flags, 2383 req->base.complete, req->base.data); 2384 ahash_request_set_crypt(subreq, req->src, req->result, 2385 req->nbytes); 2386 if (!ctx->fb_init_done) { 2387 if (ctx->fb_do_setkey) { 2388 /* Set fallback cipher HMAC key */ 2389 u8 key[SHA3_224_BLOCK_SIZE]; 2390 2391 memcpy(key, &ctx->base.ipad, 2392 crypto_ahash_blocksize(ctx->fback) / 2); 2393 memcpy(key + 2394 crypto_ahash_blocksize(ctx->fback) / 2, 2395 &ctx->base.opad, 2396 crypto_ahash_blocksize(ctx->fback) / 2); 2397 ret = crypto_ahash_setkey(ctx->fback, key, 2398 crypto_ahash_blocksize(ctx->fback)); 2399 memzero_explicit(key, 2400 crypto_ahash_blocksize(ctx->fback)); 2401 ctx->fb_do_setkey = false; 2402 } 2403 ret = ret ?: crypto_ahash_init(subreq); 2404 ctx->fb_init_done = true; 2405 } 2406 } 2407 return ret; 2408 } 2409 2410 static int safexcel_sha3_update(struct ahash_request *req) 2411 { 2412 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 2413 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2414 struct ahash_request *subreq = ahash_request_ctx_dma(req); 2415 2416 ctx->do_fallback = true; 2417 return safexcel_sha3_fbcheck(req) ?: crypto_ahash_update(subreq); 2418 } 2419 2420 static int safexcel_sha3_final(struct ahash_request *req) 2421 { 2422 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 2423 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2424 struct ahash_request *subreq = ahash_request_ctx_dma(req); 2425 2426 ctx->do_fallback = true; 2427 return safexcel_sha3_fbcheck(req) ?: crypto_ahash_final(subreq); 2428 } 2429 2430 static int safexcel_sha3_finup(struct ahash_request *req) 2431 { 2432 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 2433 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2434 struct ahash_request *subreq = ahash_request_ctx_dma(req); 2435 2436 ctx->do_fallback |= !req->nbytes; 2437 if (ctx->do_fallback) 2438 /* Update or ex/import happened or len 0, cannot use the HW */ 2439 return safexcel_sha3_fbcheck(req) ?: 2440 crypto_ahash_finup(subreq); 2441 else 2442 return safexcel_ahash_finup(req); 2443 } 2444 2445 static int safexcel_sha3_digest_fallback(struct ahash_request *req) 2446 { 2447 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 2448 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2449 struct ahash_request *subreq = ahash_request_ctx_dma(req); 2450 2451 ctx->do_fallback = true; 2452 ctx->fb_init_done = false; 2453 return safexcel_sha3_fbcheck(req) ?: crypto_ahash_finup(subreq); 2454 } 2455 2456 static int safexcel_sha3_224_digest(struct ahash_request *req) 2457 { 2458 if (req->nbytes) 2459 return safexcel_sha3_224_init(req) ?: safexcel_ahash_finup(req); 2460 2461 /* HW cannot do zero length hash, use fallback instead */ 2462 return safexcel_sha3_digest_fallback(req); 2463 } 2464 2465 static int safexcel_sha3_export(struct ahash_request *req, void *out) 2466 { 2467 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 2468 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2469 struct ahash_request *subreq = ahash_request_ctx_dma(req); 2470 2471 ctx->do_fallback = true; 2472 return safexcel_sha3_fbcheck(req) ?: crypto_ahash_export(subreq, out); 2473 } 2474 2475 static int safexcel_sha3_import(struct ahash_request *req, const void *in) 2476 { 2477 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 2478 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2479 struct ahash_request *subreq = ahash_request_ctx_dma(req); 2480 2481 ctx->do_fallback = true; 2482 return safexcel_sha3_fbcheck(req) ?: crypto_ahash_import(subreq, in); 2483 // return safexcel_ahash_import(req, in); 2484 } 2485 2486 static int safexcel_sha3_cra_init(struct crypto_tfm *tfm) 2487 { 2488 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); 2489 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 2490 2491 safexcel_ahash_cra_init(tfm); 2492 2493 /* Allocate fallback implementation */ 2494 ctx->fback = crypto_alloc_ahash(crypto_tfm_alg_name(tfm), 0, 2495 CRYPTO_ALG_ASYNC | 2496 CRYPTO_ALG_NEED_FALLBACK); 2497 if (IS_ERR(ctx->fback)) 2498 return PTR_ERR(ctx->fback); 2499 2500 /* Update statesize from fallback algorithm! */ 2501 crypto_hash_alg_common(ahash)->statesize = 2502 crypto_ahash_statesize(ctx->fback); 2503 crypto_ahash_set_reqsize_dma( 2504 ahash, max(sizeof(struct safexcel_ahash_req), 2505 sizeof(struct ahash_request) + 2506 crypto_ahash_reqsize(ctx->fback))); 2507 return 0; 2508 } 2509 2510 static void safexcel_sha3_cra_exit(struct crypto_tfm *tfm) 2511 { 2512 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 2513 2514 crypto_free_ahash(ctx->fback); 2515 safexcel_ahash_cra_exit(tfm); 2516 } 2517 2518 struct safexcel_alg_template safexcel_alg_sha3_224 = { 2519 .type = SAFEXCEL_ALG_TYPE_AHASH, 2520 .algo_mask = SAFEXCEL_ALG_SHA3, 2521 .alg.ahash = { 2522 .init = safexcel_sha3_224_init, 2523 .update = safexcel_sha3_update, 2524 .final = safexcel_sha3_final, 2525 .finup = safexcel_sha3_finup, 2526 .digest = safexcel_sha3_224_digest, 2527 .export = safexcel_sha3_export, 2528 .import = safexcel_sha3_import, 2529 .halg = { 2530 .digestsize = SHA3_224_DIGEST_SIZE, 2531 .statesize = sizeof(struct safexcel_ahash_export_state), 2532 .base = { 2533 .cra_name = "sha3-224", 2534 .cra_driver_name = "safexcel-sha3-224", 2535 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2536 .cra_flags = CRYPTO_ALG_ASYNC | 2537 CRYPTO_ALG_KERN_DRIVER_ONLY | 2538 CRYPTO_ALG_NEED_FALLBACK, 2539 .cra_blocksize = SHA3_224_BLOCK_SIZE, 2540 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 2541 .cra_init = safexcel_sha3_cra_init, 2542 .cra_exit = safexcel_sha3_cra_exit, 2543 .cra_module = THIS_MODULE, 2544 }, 2545 }, 2546 }, 2547 }; 2548 2549 static int safexcel_sha3_256_init(struct ahash_request *areq) 2550 { 2551 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2552 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2553 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 2554 2555 memset(req, 0, sizeof(*req)); 2556 2557 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_256; 2558 req->digest = CONTEXT_CONTROL_DIGEST_INITIAL; 2559 req->state_sz = SHA3_256_DIGEST_SIZE; 2560 req->digest_sz = SHA3_256_DIGEST_SIZE; 2561 req->block_sz = SHA3_256_BLOCK_SIZE; 2562 ctx->do_fallback = false; 2563 ctx->fb_init_done = false; 2564 return 0; 2565 } 2566 2567 static int safexcel_sha3_256_digest(struct ahash_request *req) 2568 { 2569 if (req->nbytes) 2570 return safexcel_sha3_256_init(req) ?: safexcel_ahash_finup(req); 2571 2572 /* HW cannot do zero length hash, use fallback instead */ 2573 return safexcel_sha3_digest_fallback(req); 2574 } 2575 2576 struct safexcel_alg_template safexcel_alg_sha3_256 = { 2577 .type = SAFEXCEL_ALG_TYPE_AHASH, 2578 .algo_mask = SAFEXCEL_ALG_SHA3, 2579 .alg.ahash = { 2580 .init = safexcel_sha3_256_init, 2581 .update = safexcel_sha3_update, 2582 .final = safexcel_sha3_final, 2583 .finup = safexcel_sha3_finup, 2584 .digest = safexcel_sha3_256_digest, 2585 .export = safexcel_sha3_export, 2586 .import = safexcel_sha3_import, 2587 .halg = { 2588 .digestsize = SHA3_256_DIGEST_SIZE, 2589 .statesize = sizeof(struct safexcel_ahash_export_state), 2590 .base = { 2591 .cra_name = "sha3-256", 2592 .cra_driver_name = "safexcel-sha3-256", 2593 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2594 .cra_flags = CRYPTO_ALG_ASYNC | 2595 CRYPTO_ALG_KERN_DRIVER_ONLY | 2596 CRYPTO_ALG_NEED_FALLBACK, 2597 .cra_blocksize = SHA3_256_BLOCK_SIZE, 2598 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 2599 .cra_init = safexcel_sha3_cra_init, 2600 .cra_exit = safexcel_sha3_cra_exit, 2601 .cra_module = THIS_MODULE, 2602 }, 2603 }, 2604 }, 2605 }; 2606 2607 static int safexcel_sha3_384_init(struct ahash_request *areq) 2608 { 2609 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2610 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2611 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 2612 2613 memset(req, 0, sizeof(*req)); 2614 2615 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_384; 2616 req->digest = CONTEXT_CONTROL_DIGEST_INITIAL; 2617 req->state_sz = SHA3_384_DIGEST_SIZE; 2618 req->digest_sz = SHA3_384_DIGEST_SIZE; 2619 req->block_sz = SHA3_384_BLOCK_SIZE; 2620 ctx->do_fallback = false; 2621 ctx->fb_init_done = false; 2622 return 0; 2623 } 2624 2625 static int safexcel_sha3_384_digest(struct ahash_request *req) 2626 { 2627 if (req->nbytes) 2628 return safexcel_sha3_384_init(req) ?: safexcel_ahash_finup(req); 2629 2630 /* HW cannot do zero length hash, use fallback instead */ 2631 return safexcel_sha3_digest_fallback(req); 2632 } 2633 2634 struct safexcel_alg_template safexcel_alg_sha3_384 = { 2635 .type = SAFEXCEL_ALG_TYPE_AHASH, 2636 .algo_mask = SAFEXCEL_ALG_SHA3, 2637 .alg.ahash = { 2638 .init = safexcel_sha3_384_init, 2639 .update = safexcel_sha3_update, 2640 .final = safexcel_sha3_final, 2641 .finup = safexcel_sha3_finup, 2642 .digest = safexcel_sha3_384_digest, 2643 .export = safexcel_sha3_export, 2644 .import = safexcel_sha3_import, 2645 .halg = { 2646 .digestsize = SHA3_384_DIGEST_SIZE, 2647 .statesize = sizeof(struct safexcel_ahash_export_state), 2648 .base = { 2649 .cra_name = "sha3-384", 2650 .cra_driver_name = "safexcel-sha3-384", 2651 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2652 .cra_flags = CRYPTO_ALG_ASYNC | 2653 CRYPTO_ALG_KERN_DRIVER_ONLY | 2654 CRYPTO_ALG_NEED_FALLBACK, 2655 .cra_blocksize = SHA3_384_BLOCK_SIZE, 2656 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 2657 .cra_init = safexcel_sha3_cra_init, 2658 .cra_exit = safexcel_sha3_cra_exit, 2659 .cra_module = THIS_MODULE, 2660 }, 2661 }, 2662 }, 2663 }; 2664 2665 static int safexcel_sha3_512_init(struct ahash_request *areq) 2666 { 2667 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2668 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2669 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 2670 2671 memset(req, 0, sizeof(*req)); 2672 2673 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_512; 2674 req->digest = CONTEXT_CONTROL_DIGEST_INITIAL; 2675 req->state_sz = SHA3_512_DIGEST_SIZE; 2676 req->digest_sz = SHA3_512_DIGEST_SIZE; 2677 req->block_sz = SHA3_512_BLOCK_SIZE; 2678 ctx->do_fallback = false; 2679 ctx->fb_init_done = false; 2680 return 0; 2681 } 2682 2683 static int safexcel_sha3_512_digest(struct ahash_request *req) 2684 { 2685 if (req->nbytes) 2686 return safexcel_sha3_512_init(req) ?: safexcel_ahash_finup(req); 2687 2688 /* HW cannot do zero length hash, use fallback instead */ 2689 return safexcel_sha3_digest_fallback(req); 2690 } 2691 2692 struct safexcel_alg_template safexcel_alg_sha3_512 = { 2693 .type = SAFEXCEL_ALG_TYPE_AHASH, 2694 .algo_mask = SAFEXCEL_ALG_SHA3, 2695 .alg.ahash = { 2696 .init = safexcel_sha3_512_init, 2697 .update = safexcel_sha3_update, 2698 .final = safexcel_sha3_final, 2699 .finup = safexcel_sha3_finup, 2700 .digest = safexcel_sha3_512_digest, 2701 .export = safexcel_sha3_export, 2702 .import = safexcel_sha3_import, 2703 .halg = { 2704 .digestsize = SHA3_512_DIGEST_SIZE, 2705 .statesize = sizeof(struct safexcel_ahash_export_state), 2706 .base = { 2707 .cra_name = "sha3-512", 2708 .cra_driver_name = "safexcel-sha3-512", 2709 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2710 .cra_flags = CRYPTO_ALG_ASYNC | 2711 CRYPTO_ALG_KERN_DRIVER_ONLY | 2712 CRYPTO_ALG_NEED_FALLBACK, 2713 .cra_blocksize = SHA3_512_BLOCK_SIZE, 2714 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 2715 .cra_init = safexcel_sha3_cra_init, 2716 .cra_exit = safexcel_sha3_cra_exit, 2717 .cra_module = THIS_MODULE, 2718 }, 2719 }, 2720 }, 2721 }; 2722 2723 static int safexcel_hmac_sha3_cra_init(struct crypto_tfm *tfm, const char *alg) 2724 { 2725 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 2726 int ret; 2727 2728 ret = safexcel_sha3_cra_init(tfm); 2729 if (ret) 2730 return ret; 2731 2732 /* Allocate precalc basic digest implementation */ 2733 ctx->shpre = crypto_alloc_shash(alg, 0, CRYPTO_ALG_NEED_FALLBACK); 2734 if (IS_ERR(ctx->shpre)) 2735 return PTR_ERR(ctx->shpre); 2736 2737 ctx->shdesc = kmalloc(sizeof(*ctx->shdesc) + 2738 crypto_shash_descsize(ctx->shpre), GFP_KERNEL); 2739 if (!ctx->shdesc) { 2740 crypto_free_shash(ctx->shpre); 2741 return -ENOMEM; 2742 } 2743 ctx->shdesc->tfm = ctx->shpre; 2744 return 0; 2745 } 2746 2747 static void safexcel_hmac_sha3_cra_exit(struct crypto_tfm *tfm) 2748 { 2749 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 2750 2751 crypto_free_ahash(ctx->fback); 2752 crypto_free_shash(ctx->shpre); 2753 kfree(ctx->shdesc); 2754 safexcel_ahash_cra_exit(tfm); 2755 } 2756 2757 static int safexcel_hmac_sha3_setkey(struct crypto_ahash *tfm, const u8 *key, 2758 unsigned int keylen) 2759 { 2760 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2761 int ret = 0; 2762 2763 if (keylen > crypto_ahash_blocksize(tfm)) { 2764 /* 2765 * If the key is larger than the blocksize, then hash it 2766 * first using our fallback cipher 2767 */ 2768 ret = crypto_shash_digest(ctx->shdesc, key, keylen, 2769 ctx->base.ipad.byte); 2770 keylen = crypto_shash_digestsize(ctx->shpre); 2771 2772 /* 2773 * If the digest is larger than half the blocksize, we need to 2774 * move the rest to opad due to the way our HMAC infra works. 2775 */ 2776 if (keylen > crypto_ahash_blocksize(tfm) / 2) 2777 /* Buffers overlap, need to use memmove iso memcpy! */ 2778 memmove(&ctx->base.opad, 2779 ctx->base.ipad.byte + 2780 crypto_ahash_blocksize(tfm) / 2, 2781 keylen - crypto_ahash_blocksize(tfm) / 2); 2782 } else { 2783 /* 2784 * Copy the key to our ipad & opad buffers 2785 * Note that ipad and opad each contain one half of the key, 2786 * to match the existing HMAC driver infrastructure. 2787 */ 2788 if (keylen <= crypto_ahash_blocksize(tfm) / 2) { 2789 memcpy(&ctx->base.ipad, key, keylen); 2790 } else { 2791 memcpy(&ctx->base.ipad, key, 2792 crypto_ahash_blocksize(tfm) / 2); 2793 memcpy(&ctx->base.opad, 2794 key + crypto_ahash_blocksize(tfm) / 2, 2795 keylen - crypto_ahash_blocksize(tfm) / 2); 2796 } 2797 } 2798 2799 /* Pad key with zeroes */ 2800 if (keylen <= crypto_ahash_blocksize(tfm) / 2) { 2801 memset(ctx->base.ipad.byte + keylen, 0, 2802 crypto_ahash_blocksize(tfm) / 2 - keylen); 2803 memset(&ctx->base.opad, 0, crypto_ahash_blocksize(tfm) / 2); 2804 } else { 2805 memset(ctx->base.opad.byte + keylen - 2806 crypto_ahash_blocksize(tfm) / 2, 0, 2807 crypto_ahash_blocksize(tfm) - keylen); 2808 } 2809 2810 /* If doing fallback, still need to set the new key! */ 2811 ctx->fb_do_setkey = true; 2812 return ret; 2813 } 2814 2815 static int safexcel_hmac_sha3_224_init(struct ahash_request *areq) 2816 { 2817 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2818 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2819 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 2820 2821 memset(req, 0, sizeof(*req)); 2822 2823 /* Copy (half of) the key */ 2824 memcpy(req->state, &ctx->base.ipad, SHA3_224_BLOCK_SIZE / 2); 2825 /* Start of HMAC should have len == processed == blocksize */ 2826 req->len = SHA3_224_BLOCK_SIZE; 2827 req->processed = SHA3_224_BLOCK_SIZE; 2828 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_224; 2829 req->digest = CONTEXT_CONTROL_DIGEST_HMAC; 2830 req->state_sz = SHA3_224_BLOCK_SIZE / 2; 2831 req->digest_sz = SHA3_224_DIGEST_SIZE; 2832 req->block_sz = SHA3_224_BLOCK_SIZE; 2833 req->hmac = true; 2834 ctx->do_fallback = false; 2835 ctx->fb_init_done = false; 2836 return 0; 2837 } 2838 2839 static int safexcel_hmac_sha3_224_digest(struct ahash_request *req) 2840 { 2841 if (req->nbytes) 2842 return safexcel_hmac_sha3_224_init(req) ?: 2843 safexcel_ahash_finup(req); 2844 2845 /* HW cannot do zero length HMAC, use fallback instead */ 2846 return safexcel_sha3_digest_fallback(req); 2847 } 2848 2849 static int safexcel_hmac_sha3_224_cra_init(struct crypto_tfm *tfm) 2850 { 2851 return safexcel_hmac_sha3_cra_init(tfm, "sha3-224"); 2852 } 2853 2854 struct safexcel_alg_template safexcel_alg_hmac_sha3_224 = { 2855 .type = SAFEXCEL_ALG_TYPE_AHASH, 2856 .algo_mask = SAFEXCEL_ALG_SHA3, 2857 .alg.ahash = { 2858 .init = safexcel_hmac_sha3_224_init, 2859 .update = safexcel_sha3_update, 2860 .final = safexcel_sha3_final, 2861 .finup = safexcel_sha3_finup, 2862 .digest = safexcel_hmac_sha3_224_digest, 2863 .setkey = safexcel_hmac_sha3_setkey, 2864 .export = safexcel_sha3_export, 2865 .import = safexcel_sha3_import, 2866 .halg = { 2867 .digestsize = SHA3_224_DIGEST_SIZE, 2868 .statesize = sizeof(struct safexcel_ahash_export_state), 2869 .base = { 2870 .cra_name = "hmac(sha3-224)", 2871 .cra_driver_name = "safexcel-hmac-sha3-224", 2872 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2873 .cra_flags = CRYPTO_ALG_ASYNC | 2874 CRYPTO_ALG_KERN_DRIVER_ONLY | 2875 CRYPTO_ALG_NEED_FALLBACK, 2876 .cra_blocksize = SHA3_224_BLOCK_SIZE, 2877 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 2878 .cra_init = safexcel_hmac_sha3_224_cra_init, 2879 .cra_exit = safexcel_hmac_sha3_cra_exit, 2880 .cra_module = THIS_MODULE, 2881 }, 2882 }, 2883 }, 2884 }; 2885 2886 static int safexcel_hmac_sha3_256_init(struct ahash_request *areq) 2887 { 2888 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2889 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2890 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 2891 2892 memset(req, 0, sizeof(*req)); 2893 2894 /* Copy (half of) the key */ 2895 memcpy(req->state, &ctx->base.ipad, SHA3_256_BLOCK_SIZE / 2); 2896 /* Start of HMAC should have len == processed == blocksize */ 2897 req->len = SHA3_256_BLOCK_SIZE; 2898 req->processed = SHA3_256_BLOCK_SIZE; 2899 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_256; 2900 req->digest = CONTEXT_CONTROL_DIGEST_HMAC; 2901 req->state_sz = SHA3_256_BLOCK_SIZE / 2; 2902 req->digest_sz = SHA3_256_DIGEST_SIZE; 2903 req->block_sz = SHA3_256_BLOCK_SIZE; 2904 req->hmac = true; 2905 ctx->do_fallback = false; 2906 ctx->fb_init_done = false; 2907 return 0; 2908 } 2909 2910 static int safexcel_hmac_sha3_256_digest(struct ahash_request *req) 2911 { 2912 if (req->nbytes) 2913 return safexcel_hmac_sha3_256_init(req) ?: 2914 safexcel_ahash_finup(req); 2915 2916 /* HW cannot do zero length HMAC, use fallback instead */ 2917 return safexcel_sha3_digest_fallback(req); 2918 } 2919 2920 static int safexcel_hmac_sha3_256_cra_init(struct crypto_tfm *tfm) 2921 { 2922 return safexcel_hmac_sha3_cra_init(tfm, "sha3-256"); 2923 } 2924 2925 struct safexcel_alg_template safexcel_alg_hmac_sha3_256 = { 2926 .type = SAFEXCEL_ALG_TYPE_AHASH, 2927 .algo_mask = SAFEXCEL_ALG_SHA3, 2928 .alg.ahash = { 2929 .init = safexcel_hmac_sha3_256_init, 2930 .update = safexcel_sha3_update, 2931 .final = safexcel_sha3_final, 2932 .finup = safexcel_sha3_finup, 2933 .digest = safexcel_hmac_sha3_256_digest, 2934 .setkey = safexcel_hmac_sha3_setkey, 2935 .export = safexcel_sha3_export, 2936 .import = safexcel_sha3_import, 2937 .halg = { 2938 .digestsize = SHA3_256_DIGEST_SIZE, 2939 .statesize = sizeof(struct safexcel_ahash_export_state), 2940 .base = { 2941 .cra_name = "hmac(sha3-256)", 2942 .cra_driver_name = "safexcel-hmac-sha3-256", 2943 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2944 .cra_flags = CRYPTO_ALG_ASYNC | 2945 CRYPTO_ALG_KERN_DRIVER_ONLY | 2946 CRYPTO_ALG_NEED_FALLBACK, 2947 .cra_blocksize = SHA3_256_BLOCK_SIZE, 2948 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 2949 .cra_init = safexcel_hmac_sha3_256_cra_init, 2950 .cra_exit = safexcel_hmac_sha3_cra_exit, 2951 .cra_module = THIS_MODULE, 2952 }, 2953 }, 2954 }, 2955 }; 2956 2957 static int safexcel_hmac_sha3_384_init(struct ahash_request *areq) 2958 { 2959 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2960 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 2961 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 2962 2963 memset(req, 0, sizeof(*req)); 2964 2965 /* Copy (half of) the key */ 2966 memcpy(req->state, &ctx->base.ipad, SHA3_384_BLOCK_SIZE / 2); 2967 /* Start of HMAC should have len == processed == blocksize */ 2968 req->len = SHA3_384_BLOCK_SIZE; 2969 req->processed = SHA3_384_BLOCK_SIZE; 2970 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_384; 2971 req->digest = CONTEXT_CONTROL_DIGEST_HMAC; 2972 req->state_sz = SHA3_384_BLOCK_SIZE / 2; 2973 req->digest_sz = SHA3_384_DIGEST_SIZE; 2974 req->block_sz = SHA3_384_BLOCK_SIZE; 2975 req->hmac = true; 2976 ctx->do_fallback = false; 2977 ctx->fb_init_done = false; 2978 return 0; 2979 } 2980 2981 static int safexcel_hmac_sha3_384_digest(struct ahash_request *req) 2982 { 2983 if (req->nbytes) 2984 return safexcel_hmac_sha3_384_init(req) ?: 2985 safexcel_ahash_finup(req); 2986 2987 /* HW cannot do zero length HMAC, use fallback instead */ 2988 return safexcel_sha3_digest_fallback(req); 2989 } 2990 2991 static int safexcel_hmac_sha3_384_cra_init(struct crypto_tfm *tfm) 2992 { 2993 return safexcel_hmac_sha3_cra_init(tfm, "sha3-384"); 2994 } 2995 2996 struct safexcel_alg_template safexcel_alg_hmac_sha3_384 = { 2997 .type = SAFEXCEL_ALG_TYPE_AHASH, 2998 .algo_mask = SAFEXCEL_ALG_SHA3, 2999 .alg.ahash = { 3000 .init = safexcel_hmac_sha3_384_init, 3001 .update = safexcel_sha3_update, 3002 .final = safexcel_sha3_final, 3003 .finup = safexcel_sha3_finup, 3004 .digest = safexcel_hmac_sha3_384_digest, 3005 .setkey = safexcel_hmac_sha3_setkey, 3006 .export = safexcel_sha3_export, 3007 .import = safexcel_sha3_import, 3008 .halg = { 3009 .digestsize = SHA3_384_DIGEST_SIZE, 3010 .statesize = sizeof(struct safexcel_ahash_export_state), 3011 .base = { 3012 .cra_name = "hmac(sha3-384)", 3013 .cra_driver_name = "safexcel-hmac-sha3-384", 3014 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3015 .cra_flags = CRYPTO_ALG_ASYNC | 3016 CRYPTO_ALG_KERN_DRIVER_ONLY | 3017 CRYPTO_ALG_NEED_FALLBACK, 3018 .cra_blocksize = SHA3_384_BLOCK_SIZE, 3019 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 3020 .cra_init = safexcel_hmac_sha3_384_cra_init, 3021 .cra_exit = safexcel_hmac_sha3_cra_exit, 3022 .cra_module = THIS_MODULE, 3023 }, 3024 }, 3025 }, 3026 }; 3027 3028 static int safexcel_hmac_sha3_512_init(struct ahash_request *areq) 3029 { 3030 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 3031 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); 3032 struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 3033 3034 memset(req, 0, sizeof(*req)); 3035 3036 /* Copy (half of) the key */ 3037 memcpy(req->state, &ctx->base.ipad, SHA3_512_BLOCK_SIZE / 2); 3038 /* Start of HMAC should have len == processed == blocksize */ 3039 req->len = SHA3_512_BLOCK_SIZE; 3040 req->processed = SHA3_512_BLOCK_SIZE; 3041 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_512; 3042 req->digest = CONTEXT_CONTROL_DIGEST_HMAC; 3043 req->state_sz = SHA3_512_BLOCK_SIZE / 2; 3044 req->digest_sz = SHA3_512_DIGEST_SIZE; 3045 req->block_sz = SHA3_512_BLOCK_SIZE; 3046 req->hmac = true; 3047 ctx->do_fallback = false; 3048 ctx->fb_init_done = false; 3049 return 0; 3050 } 3051 3052 static int safexcel_hmac_sha3_512_digest(struct ahash_request *req) 3053 { 3054 if (req->nbytes) 3055 return safexcel_hmac_sha3_512_init(req) ?: 3056 safexcel_ahash_finup(req); 3057 3058 /* HW cannot do zero length HMAC, use fallback instead */ 3059 return safexcel_sha3_digest_fallback(req); 3060 } 3061 3062 static int safexcel_hmac_sha3_512_cra_init(struct crypto_tfm *tfm) 3063 { 3064 return safexcel_hmac_sha3_cra_init(tfm, "sha3-512"); 3065 } 3066 struct safexcel_alg_template safexcel_alg_hmac_sha3_512 = { 3067 .type = SAFEXCEL_ALG_TYPE_AHASH, 3068 .algo_mask = SAFEXCEL_ALG_SHA3, 3069 .alg.ahash = { 3070 .init = safexcel_hmac_sha3_512_init, 3071 .update = safexcel_sha3_update, 3072 .final = safexcel_sha3_final, 3073 .finup = safexcel_sha3_finup, 3074 .digest = safexcel_hmac_sha3_512_digest, 3075 .setkey = safexcel_hmac_sha3_setkey, 3076 .export = safexcel_sha3_export, 3077 .import = safexcel_sha3_import, 3078 .halg = { 3079 .digestsize = SHA3_512_DIGEST_SIZE, 3080 .statesize = sizeof(struct safexcel_ahash_export_state), 3081 .base = { 3082 .cra_name = "hmac(sha3-512)", 3083 .cra_driver_name = "safexcel-hmac-sha3-512", 3084 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3085 .cra_flags = CRYPTO_ALG_ASYNC | 3086 CRYPTO_ALG_KERN_DRIVER_ONLY | 3087 CRYPTO_ALG_NEED_FALLBACK, 3088 .cra_blocksize = SHA3_512_BLOCK_SIZE, 3089 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 3090 .cra_init = safexcel_hmac_sha3_512_cra_init, 3091 .cra_exit = safexcel_hmac_sha3_cra_exit, 3092 .cra_module = THIS_MODULE, 3093 }, 3094 }, 3095 }, 3096 }; 3097