1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2017 Marvell 4 * 5 * Antoine Tenart <antoine.tenart@free-electrons.com> 6 */ 7 8 #include <crypto/hmac.h> 9 #include <crypto/md5.h> 10 #include <crypto/sha.h> 11 #include <linux/device.h> 12 #include <linux/dma-mapping.h> 13 #include <linux/dmapool.h> 14 15 #include "safexcel.h" 16 17 struct safexcel_ahash_ctx { 18 struct safexcel_context base; 19 struct safexcel_crypto_priv *priv; 20 21 u32 alg; 22 23 u32 ipad[SHA512_DIGEST_SIZE / sizeof(u32)]; 24 u32 opad[SHA512_DIGEST_SIZE / sizeof(u32)]; 25 }; 26 27 struct safexcel_ahash_req { 28 bool last_req; 29 bool finish; 30 bool hmac; 31 bool needs_inv; 32 33 int nents; 34 dma_addr_t result_dma; 35 36 u32 digest; 37 38 u8 state_sz; /* expected sate size, only set once */ 39 u32 state[SHA512_DIGEST_SIZE / sizeof(u32)] __aligned(sizeof(u32)); 40 41 u64 len[2]; 42 u64 processed[2]; 43 44 u8 cache[SHA512_BLOCK_SIZE] __aligned(sizeof(u32)); 45 dma_addr_t cache_dma; 46 unsigned int cache_sz; 47 48 u8 cache_next[SHA512_BLOCK_SIZE] __aligned(sizeof(u32)); 49 }; 50 51 static inline u64 safexcel_queued_len(struct safexcel_ahash_req *req) 52 { 53 if (req->len[1] > req->processed[1]) 54 return 0xffffffff - (req->len[0] - req->processed[0]); 55 56 return req->len[0] - req->processed[0]; 57 } 58 59 static void safexcel_hash_token(struct safexcel_command_desc *cdesc, 60 u32 input_length, u32 result_length) 61 { 62 struct safexcel_token *token = 63 (struct safexcel_token *)cdesc->control_data.token; 64 65 token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION; 66 token[0].packet_length = input_length; 67 token[0].stat = EIP197_TOKEN_STAT_LAST_HASH; 68 token[0].instructions = EIP197_TOKEN_INS_TYPE_HASH; 69 70 token[1].opcode = EIP197_TOKEN_OPCODE_INSERT; 71 token[1].packet_length = result_length; 72 token[1].stat = EIP197_TOKEN_STAT_LAST_HASH | 73 EIP197_TOKEN_STAT_LAST_PACKET; 74 token[1].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | 75 EIP197_TOKEN_INS_INSERT_HASH_DIGEST; 76 } 77 78 static void safexcel_context_control(struct safexcel_ahash_ctx *ctx, 79 struct safexcel_ahash_req *req, 80 struct safexcel_command_desc *cdesc, 81 unsigned int digestsize) 82 { 83 struct safexcel_crypto_priv *priv = ctx->priv; 84 int i; 85 86 cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_HASH_OUT; 87 cdesc->control_data.control0 |= ctx->alg; 88 cdesc->control_data.control0 |= req->digest; 89 90 if (req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) { 91 if (req->processed[0] || req->processed[1]) { 92 if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5) 93 cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(5); 94 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA1) 95 cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(6); 96 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA224 || 97 ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA256) 98 cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(9); 99 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA384 || 100 ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA512) 101 cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(17); 102 103 cdesc->control_data.control1 |= CONTEXT_CONTROL_DIGEST_CNT; 104 } else { 105 cdesc->control_data.control0 |= CONTEXT_CONTROL_RESTART_HASH; 106 } 107 108 if (!req->finish) 109 cdesc->control_data.control0 |= CONTEXT_CONTROL_NO_FINISH_HASH; 110 111 /* 112 * Copy the input digest if needed, and setup the context 113 * fields. Do this now as we need it to setup the first command 114 * descriptor. 115 */ 116 if (req->processed[0] || req->processed[1]) { 117 for (i = 0; i < digestsize / sizeof(u32); i++) 118 ctx->base.ctxr->data[i] = cpu_to_le32(req->state[i]); 119 120 if (req->finish) { 121 u64 count = req->processed[0] / EIP197_COUNTER_BLOCK_SIZE; 122 count += ((0xffffffff / EIP197_COUNTER_BLOCK_SIZE) * 123 req->processed[1]); 124 125 /* This is a haredware limitation, as the 126 * counter must fit into an u32. This represents 127 * a farily big amount of input data, so we 128 * shouldn't see this. 129 */ 130 if (unlikely(count & 0xffff0000)) { 131 dev_warn(priv->dev, 132 "Input data is too big\n"); 133 return; 134 } 135 136 ctx->base.ctxr->data[i] = cpu_to_le32(count); 137 } 138 } 139 } else if (req->digest == CONTEXT_CONTROL_DIGEST_HMAC) { 140 cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(2 * req->state_sz / sizeof(u32)); 141 142 memcpy(ctx->base.ctxr->data, ctx->ipad, req->state_sz); 143 memcpy(ctx->base.ctxr->data + req->state_sz / sizeof(u32), 144 ctx->opad, req->state_sz); 145 } 146 } 147 148 static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int ring, 149 struct crypto_async_request *async, 150 bool *should_complete, int *ret) 151 { 152 struct safexcel_result_desc *rdesc; 153 struct ahash_request *areq = ahash_request_cast(async); 154 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); 155 struct safexcel_ahash_req *sreq = ahash_request_ctx(areq); 156 u64 cache_len; 157 158 *ret = 0; 159 160 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); 161 if (IS_ERR(rdesc)) { 162 dev_err(priv->dev, 163 "hash: result: could not retrieve the result descriptor\n"); 164 *ret = PTR_ERR(rdesc); 165 } else { 166 *ret = safexcel_rdesc_check_errors(priv, rdesc); 167 } 168 169 safexcel_complete(priv, ring); 170 171 if (sreq->nents) { 172 dma_unmap_sg(priv->dev, areq->src, sreq->nents, DMA_TO_DEVICE); 173 sreq->nents = 0; 174 } 175 176 if (sreq->result_dma) { 177 dma_unmap_single(priv->dev, sreq->result_dma, sreq->state_sz, 178 DMA_FROM_DEVICE); 179 sreq->result_dma = 0; 180 } 181 182 if (sreq->cache_dma) { 183 dma_unmap_single(priv->dev, sreq->cache_dma, sreq->cache_sz, 184 DMA_TO_DEVICE); 185 sreq->cache_dma = 0; 186 } 187 188 if (sreq->finish) 189 memcpy(areq->result, sreq->state, 190 crypto_ahash_digestsize(ahash)); 191 192 cache_len = safexcel_queued_len(sreq); 193 if (cache_len) 194 memcpy(sreq->cache, sreq->cache_next, cache_len); 195 196 *should_complete = true; 197 198 return 1; 199 } 200 201 static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, 202 int *commands, int *results) 203 { 204 struct ahash_request *areq = ahash_request_cast(async); 205 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); 206 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 207 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 208 struct safexcel_crypto_priv *priv = ctx->priv; 209 struct safexcel_command_desc *cdesc, *first_cdesc = NULL; 210 struct safexcel_result_desc *rdesc; 211 struct scatterlist *sg; 212 int i, extra, n_cdesc = 0, ret = 0; 213 u64 queued, len, cache_len; 214 215 queued = len = safexcel_queued_len(req); 216 if (queued <= crypto_ahash_blocksize(ahash)) 217 cache_len = queued; 218 else 219 cache_len = queued - areq->nbytes; 220 221 if (!req->last_req) { 222 /* If this is not the last request and the queued data does not 223 * fit into full blocks, cache it for the next send() call. 224 */ 225 extra = queued & (crypto_ahash_blocksize(ahash) - 1); 226 if (!extra) 227 /* If this is not the last request and the queued data 228 * is a multiple of a block, cache the last one for now. 229 */ 230 extra = crypto_ahash_blocksize(ahash); 231 232 if (extra) { 233 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), 234 req->cache_next, extra, 235 areq->nbytes - extra); 236 237 queued -= extra; 238 len -= extra; 239 240 if (!queued) { 241 *commands = 0; 242 *results = 0; 243 return 0; 244 } 245 } 246 } 247 248 /* Add a command descriptor for the cached data, if any */ 249 if (cache_len) { 250 req->cache_dma = dma_map_single(priv->dev, req->cache, 251 cache_len, DMA_TO_DEVICE); 252 if (dma_mapping_error(priv->dev, req->cache_dma)) 253 return -EINVAL; 254 255 req->cache_sz = cache_len; 256 first_cdesc = safexcel_add_cdesc(priv, ring, 1, 257 (cache_len == len), 258 req->cache_dma, cache_len, len, 259 ctx->base.ctxr_dma); 260 if (IS_ERR(first_cdesc)) { 261 ret = PTR_ERR(first_cdesc); 262 goto unmap_cache; 263 } 264 n_cdesc++; 265 266 queued -= cache_len; 267 if (!queued) 268 goto send_command; 269 } 270 271 /* Now handle the current ahash request buffer(s) */ 272 req->nents = dma_map_sg(priv->dev, areq->src, 273 sg_nents_for_len(areq->src, areq->nbytes), 274 DMA_TO_DEVICE); 275 if (!req->nents) { 276 ret = -ENOMEM; 277 goto cdesc_rollback; 278 } 279 280 for_each_sg(areq->src, sg, req->nents, i) { 281 int sglen = sg_dma_len(sg); 282 283 /* Do not overflow the request */ 284 if (queued < sglen) 285 sglen = queued; 286 287 cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc, 288 !(queued - sglen), sg_dma_address(sg), 289 sglen, len, ctx->base.ctxr_dma); 290 if (IS_ERR(cdesc)) { 291 ret = PTR_ERR(cdesc); 292 goto unmap_sg; 293 } 294 n_cdesc++; 295 296 if (n_cdesc == 1) 297 first_cdesc = cdesc; 298 299 queued -= sglen; 300 if (!queued) 301 break; 302 } 303 304 send_command: 305 /* Setup the context options */ 306 safexcel_context_control(ctx, req, first_cdesc, req->state_sz); 307 308 /* Add the token */ 309 safexcel_hash_token(first_cdesc, len, req->state_sz); 310 311 req->result_dma = dma_map_single(priv->dev, req->state, req->state_sz, 312 DMA_FROM_DEVICE); 313 if (dma_mapping_error(priv->dev, req->result_dma)) { 314 ret = -EINVAL; 315 goto unmap_sg; 316 } 317 318 /* Add a result descriptor */ 319 rdesc = safexcel_add_rdesc(priv, ring, 1, 1, req->result_dma, 320 req->state_sz); 321 if (IS_ERR(rdesc)) { 322 ret = PTR_ERR(rdesc); 323 goto unmap_result; 324 } 325 326 safexcel_rdr_req_set(priv, ring, rdesc, &areq->base); 327 328 req->processed[0] += len; 329 if (req->processed[0] < len) 330 req->processed[1]++; 331 332 *commands = n_cdesc; 333 *results = 1; 334 return 0; 335 336 unmap_result: 337 dma_unmap_single(priv->dev, req->result_dma, req->state_sz, 338 DMA_FROM_DEVICE); 339 unmap_sg: 340 dma_unmap_sg(priv->dev, areq->src, req->nents, DMA_TO_DEVICE); 341 cdesc_rollback: 342 for (i = 0; i < n_cdesc; i++) 343 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr); 344 unmap_cache: 345 if (req->cache_dma) { 346 dma_unmap_single(priv->dev, req->cache_dma, req->cache_sz, 347 DMA_TO_DEVICE); 348 req->cache_sz = 0; 349 } 350 351 return ret; 352 } 353 354 static inline bool safexcel_ahash_needs_inv_get(struct ahash_request *areq) 355 { 356 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 357 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 358 unsigned int state_w_sz = req->state_sz / sizeof(u32); 359 u64 processed; 360 int i; 361 362 processed = req->processed[0] / EIP197_COUNTER_BLOCK_SIZE; 363 processed += (0xffffffff / EIP197_COUNTER_BLOCK_SIZE) * req->processed[1]; 364 365 for (i = 0; i < state_w_sz; i++) 366 if (ctx->base.ctxr->data[i] != cpu_to_le32(req->state[i])) 367 return true; 368 369 if (ctx->base.ctxr->data[state_w_sz] != cpu_to_le32(processed)) 370 return true; 371 372 return false; 373 } 374 375 static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv, 376 int ring, 377 struct crypto_async_request *async, 378 bool *should_complete, int *ret) 379 { 380 struct safexcel_result_desc *rdesc; 381 struct ahash_request *areq = ahash_request_cast(async); 382 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); 383 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(ahash); 384 int enq_ret; 385 386 *ret = 0; 387 388 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); 389 if (IS_ERR(rdesc)) { 390 dev_err(priv->dev, 391 "hash: invalidate: could not retrieve the result descriptor\n"); 392 *ret = PTR_ERR(rdesc); 393 } else { 394 *ret = safexcel_rdesc_check_errors(priv, rdesc); 395 } 396 397 safexcel_complete(priv, ring); 398 399 if (ctx->base.exit_inv) { 400 dma_pool_free(priv->context_pool, ctx->base.ctxr, 401 ctx->base.ctxr_dma); 402 403 *should_complete = true; 404 return 1; 405 } 406 407 ring = safexcel_select_ring(priv); 408 ctx->base.ring = ring; 409 410 spin_lock_bh(&priv->ring[ring].queue_lock); 411 enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, async); 412 spin_unlock_bh(&priv->ring[ring].queue_lock); 413 414 if (enq_ret != -EINPROGRESS) 415 *ret = enq_ret; 416 417 queue_work(priv->ring[ring].workqueue, 418 &priv->ring[ring].work_data.work); 419 420 *should_complete = false; 421 422 return 1; 423 } 424 425 static int safexcel_handle_result(struct safexcel_crypto_priv *priv, int ring, 426 struct crypto_async_request *async, 427 bool *should_complete, int *ret) 428 { 429 struct ahash_request *areq = ahash_request_cast(async); 430 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 431 int err; 432 433 BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && req->needs_inv); 434 435 if (req->needs_inv) { 436 req->needs_inv = false; 437 err = safexcel_handle_inv_result(priv, ring, async, 438 should_complete, ret); 439 } else { 440 err = safexcel_handle_req_result(priv, ring, async, 441 should_complete, ret); 442 } 443 444 return err; 445 } 446 447 static int safexcel_ahash_send_inv(struct crypto_async_request *async, 448 int ring, int *commands, int *results) 449 { 450 struct ahash_request *areq = ahash_request_cast(async); 451 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 452 int ret; 453 454 ret = safexcel_invalidate_cache(async, ctx->priv, 455 ctx->base.ctxr_dma, ring); 456 if (unlikely(ret)) 457 return ret; 458 459 *commands = 1; 460 *results = 1; 461 462 return 0; 463 } 464 465 static int safexcel_ahash_send(struct crypto_async_request *async, 466 int ring, int *commands, int *results) 467 { 468 struct ahash_request *areq = ahash_request_cast(async); 469 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 470 int ret; 471 472 if (req->needs_inv) 473 ret = safexcel_ahash_send_inv(async, ring, commands, results); 474 else 475 ret = safexcel_ahash_send_req(async, ring, commands, results); 476 477 return ret; 478 } 479 480 static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm) 481 { 482 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 483 struct safexcel_crypto_priv *priv = ctx->priv; 484 EIP197_REQUEST_ON_STACK(req, ahash, EIP197_AHASH_REQ_SIZE); 485 struct safexcel_ahash_req *rctx = ahash_request_ctx(req); 486 struct safexcel_inv_result result = {}; 487 int ring = ctx->base.ring; 488 489 memset(req, 0, sizeof(struct ahash_request)); 490 491 /* create invalidation request */ 492 init_completion(&result.completion); 493 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 494 safexcel_inv_complete, &result); 495 496 ahash_request_set_tfm(req, __crypto_ahash_cast(tfm)); 497 ctx = crypto_tfm_ctx(req->base.tfm); 498 ctx->base.exit_inv = true; 499 rctx->needs_inv = true; 500 501 spin_lock_bh(&priv->ring[ring].queue_lock); 502 crypto_enqueue_request(&priv->ring[ring].queue, &req->base); 503 spin_unlock_bh(&priv->ring[ring].queue_lock); 504 505 queue_work(priv->ring[ring].workqueue, 506 &priv->ring[ring].work_data.work); 507 508 wait_for_completion(&result.completion); 509 510 if (result.error) { 511 dev_warn(priv->dev, "hash: completion error (%d)\n", 512 result.error); 513 return result.error; 514 } 515 516 return 0; 517 } 518 519 /* safexcel_ahash_cache: cache data until at least one request can be sent to 520 * the engine, aka. when there is at least 1 block size in the pipe. 521 */ 522 static int safexcel_ahash_cache(struct ahash_request *areq) 523 { 524 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 525 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); 526 u64 queued, cache_len; 527 528 /* queued: everything accepted by the driver which will be handled by 529 * the next send() calls. 530 * tot sz handled by update() - tot sz handled by send() 531 */ 532 queued = safexcel_queued_len(req); 533 /* cache_len: everything accepted by the driver but not sent yet, 534 * tot sz handled by update() - last req sz - tot sz handled by send() 535 */ 536 cache_len = queued - areq->nbytes; 537 538 /* 539 * In case there isn't enough bytes to proceed (less than a 540 * block size), cache the data until we have enough. 541 */ 542 if (cache_len + areq->nbytes <= crypto_ahash_blocksize(ahash)) { 543 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), 544 req->cache + cache_len, 545 areq->nbytes, 0); 546 return areq->nbytes; 547 } 548 549 /* We couldn't cache all the data */ 550 return -E2BIG; 551 } 552 553 static int safexcel_ahash_enqueue(struct ahash_request *areq) 554 { 555 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 556 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 557 struct safexcel_crypto_priv *priv = ctx->priv; 558 int ret, ring; 559 560 req->needs_inv = false; 561 562 if (ctx->base.ctxr) { 563 if (priv->flags & EIP197_TRC_CACHE && !ctx->base.needs_inv && 564 (req->processed[0] || req->processed[1]) && 565 req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) 566 /* We're still setting needs_inv here, even though it is 567 * cleared right away, because the needs_inv flag can be 568 * set in other functions and we want to keep the same 569 * logic. 570 */ 571 ctx->base.needs_inv = safexcel_ahash_needs_inv_get(areq); 572 573 if (ctx->base.needs_inv) { 574 ctx->base.needs_inv = false; 575 req->needs_inv = true; 576 } 577 } else { 578 ctx->base.ring = safexcel_select_ring(priv); 579 ctx->base.ctxr = dma_pool_zalloc(priv->context_pool, 580 EIP197_GFP_FLAGS(areq->base), 581 &ctx->base.ctxr_dma); 582 if (!ctx->base.ctxr) 583 return -ENOMEM; 584 } 585 586 ring = ctx->base.ring; 587 588 spin_lock_bh(&priv->ring[ring].queue_lock); 589 ret = crypto_enqueue_request(&priv->ring[ring].queue, &areq->base); 590 spin_unlock_bh(&priv->ring[ring].queue_lock); 591 592 queue_work(priv->ring[ring].workqueue, 593 &priv->ring[ring].work_data.work); 594 595 return ret; 596 } 597 598 static int safexcel_ahash_update(struct ahash_request *areq) 599 { 600 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 601 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); 602 603 /* If the request is 0 length, do nothing */ 604 if (!areq->nbytes) 605 return 0; 606 607 req->len[0] += areq->nbytes; 608 if (req->len[0] < areq->nbytes) 609 req->len[1]++; 610 611 safexcel_ahash_cache(areq); 612 613 /* 614 * We're not doing partial updates when performing an hmac request. 615 * Everything will be handled by the final() call. 616 */ 617 if (req->digest == CONTEXT_CONTROL_DIGEST_HMAC) 618 return 0; 619 620 if (req->hmac) 621 return safexcel_ahash_enqueue(areq); 622 623 if (!req->last_req && 624 safexcel_queued_len(req) > crypto_ahash_blocksize(ahash)) 625 return safexcel_ahash_enqueue(areq); 626 627 return 0; 628 } 629 630 static int safexcel_ahash_final(struct ahash_request *areq) 631 { 632 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 633 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 634 635 req->last_req = true; 636 req->finish = true; 637 638 /* If we have an overall 0 length request */ 639 if (!req->len[0] && !req->len[1] && !areq->nbytes) { 640 if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5) 641 memcpy(areq->result, md5_zero_message_hash, 642 MD5_DIGEST_SIZE); 643 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA1) 644 memcpy(areq->result, sha1_zero_message_hash, 645 SHA1_DIGEST_SIZE); 646 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA224) 647 memcpy(areq->result, sha224_zero_message_hash, 648 SHA224_DIGEST_SIZE); 649 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA256) 650 memcpy(areq->result, sha256_zero_message_hash, 651 SHA256_DIGEST_SIZE); 652 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA384) 653 memcpy(areq->result, sha384_zero_message_hash, 654 SHA384_DIGEST_SIZE); 655 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA512) 656 memcpy(areq->result, sha512_zero_message_hash, 657 SHA512_DIGEST_SIZE); 658 659 return 0; 660 } 661 662 return safexcel_ahash_enqueue(areq); 663 } 664 665 static int safexcel_ahash_finup(struct ahash_request *areq) 666 { 667 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 668 669 req->last_req = true; 670 req->finish = true; 671 672 safexcel_ahash_update(areq); 673 return safexcel_ahash_final(areq); 674 } 675 676 static int safexcel_ahash_export(struct ahash_request *areq, void *out) 677 { 678 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); 679 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 680 struct safexcel_ahash_export_state *export = out; 681 682 export->len[0] = req->len[0]; 683 export->len[1] = req->len[1]; 684 export->processed[0] = req->processed[0]; 685 export->processed[1] = req->processed[1]; 686 687 export->digest = req->digest; 688 689 memcpy(export->state, req->state, req->state_sz); 690 memcpy(export->cache, req->cache, crypto_ahash_blocksize(ahash)); 691 692 return 0; 693 } 694 695 static int safexcel_ahash_import(struct ahash_request *areq, const void *in) 696 { 697 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); 698 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 699 const struct safexcel_ahash_export_state *export = in; 700 int ret; 701 702 ret = crypto_ahash_init(areq); 703 if (ret) 704 return ret; 705 706 req->len[0] = export->len[0]; 707 req->len[1] = export->len[1]; 708 req->processed[0] = export->processed[0]; 709 req->processed[1] = export->processed[1]; 710 711 req->digest = export->digest; 712 713 memcpy(req->cache, export->cache, crypto_ahash_blocksize(ahash)); 714 memcpy(req->state, export->state, req->state_sz); 715 716 return 0; 717 } 718 719 static int safexcel_ahash_cra_init(struct crypto_tfm *tfm) 720 { 721 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 722 struct safexcel_alg_template *tmpl = 723 container_of(__crypto_ahash_alg(tfm->__crt_alg), 724 struct safexcel_alg_template, alg.ahash); 725 726 ctx->priv = tmpl->priv; 727 ctx->base.send = safexcel_ahash_send; 728 ctx->base.handle_result = safexcel_handle_result; 729 730 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 731 sizeof(struct safexcel_ahash_req)); 732 return 0; 733 } 734 735 static int safexcel_sha1_init(struct ahash_request *areq) 736 { 737 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 738 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 739 740 memset(req, 0, sizeof(*req)); 741 742 req->state[0] = SHA1_H0; 743 req->state[1] = SHA1_H1; 744 req->state[2] = SHA1_H2; 745 req->state[3] = SHA1_H3; 746 req->state[4] = SHA1_H4; 747 748 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1; 749 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 750 req->state_sz = SHA1_DIGEST_SIZE; 751 752 return 0; 753 } 754 755 static int safexcel_sha1_digest(struct ahash_request *areq) 756 { 757 int ret = safexcel_sha1_init(areq); 758 759 if (ret) 760 return ret; 761 762 return safexcel_ahash_finup(areq); 763 } 764 765 static void safexcel_ahash_cra_exit(struct crypto_tfm *tfm) 766 { 767 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 768 struct safexcel_crypto_priv *priv = ctx->priv; 769 int ret; 770 771 /* context not allocated, skip invalidation */ 772 if (!ctx->base.ctxr) 773 return; 774 775 if (priv->flags & EIP197_TRC_CACHE) { 776 ret = safexcel_ahash_exit_inv(tfm); 777 if (ret) 778 dev_warn(priv->dev, "hash: invalidation error %d\n", ret); 779 } else { 780 dma_pool_free(priv->context_pool, ctx->base.ctxr, 781 ctx->base.ctxr_dma); 782 } 783 } 784 785 struct safexcel_alg_template safexcel_alg_sha1 = { 786 .type = SAFEXCEL_ALG_TYPE_AHASH, 787 .engines = EIP97IES | EIP197B | EIP197D, 788 .alg.ahash = { 789 .init = safexcel_sha1_init, 790 .update = safexcel_ahash_update, 791 .final = safexcel_ahash_final, 792 .finup = safexcel_ahash_finup, 793 .digest = safexcel_sha1_digest, 794 .export = safexcel_ahash_export, 795 .import = safexcel_ahash_import, 796 .halg = { 797 .digestsize = SHA1_DIGEST_SIZE, 798 .statesize = sizeof(struct safexcel_ahash_export_state), 799 .base = { 800 .cra_name = "sha1", 801 .cra_driver_name = "safexcel-sha1", 802 .cra_priority = 300, 803 .cra_flags = CRYPTO_ALG_ASYNC | 804 CRYPTO_ALG_KERN_DRIVER_ONLY, 805 .cra_blocksize = SHA1_BLOCK_SIZE, 806 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 807 .cra_init = safexcel_ahash_cra_init, 808 .cra_exit = safexcel_ahash_cra_exit, 809 .cra_module = THIS_MODULE, 810 }, 811 }, 812 }, 813 }; 814 815 static int safexcel_hmac_sha1_init(struct ahash_request *areq) 816 { 817 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 818 819 safexcel_sha1_init(areq); 820 req->digest = CONTEXT_CONTROL_DIGEST_HMAC; 821 return 0; 822 } 823 824 static int safexcel_hmac_sha1_digest(struct ahash_request *areq) 825 { 826 int ret = safexcel_hmac_sha1_init(areq); 827 828 if (ret) 829 return ret; 830 831 return safexcel_ahash_finup(areq); 832 } 833 834 struct safexcel_ahash_result { 835 struct completion completion; 836 int error; 837 }; 838 839 static void safexcel_ahash_complete(struct crypto_async_request *req, int error) 840 { 841 struct safexcel_ahash_result *result = req->data; 842 843 if (error == -EINPROGRESS) 844 return; 845 846 result->error = error; 847 complete(&result->completion); 848 } 849 850 static int safexcel_hmac_init_pad(struct ahash_request *areq, 851 unsigned int blocksize, const u8 *key, 852 unsigned int keylen, u8 *ipad, u8 *opad) 853 { 854 struct safexcel_ahash_result result; 855 struct scatterlist sg; 856 int ret, i; 857 u8 *keydup; 858 859 if (keylen <= blocksize) { 860 memcpy(ipad, key, keylen); 861 } else { 862 keydup = kmemdup(key, keylen, GFP_KERNEL); 863 if (!keydup) 864 return -ENOMEM; 865 866 ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG, 867 safexcel_ahash_complete, &result); 868 sg_init_one(&sg, keydup, keylen); 869 ahash_request_set_crypt(areq, &sg, ipad, keylen); 870 init_completion(&result.completion); 871 872 ret = crypto_ahash_digest(areq); 873 if (ret == -EINPROGRESS || ret == -EBUSY) { 874 wait_for_completion_interruptible(&result.completion); 875 ret = result.error; 876 } 877 878 /* Avoid leaking */ 879 memzero_explicit(keydup, keylen); 880 kfree(keydup); 881 882 if (ret) 883 return ret; 884 885 keylen = crypto_ahash_digestsize(crypto_ahash_reqtfm(areq)); 886 } 887 888 memset(ipad + keylen, 0, blocksize - keylen); 889 memcpy(opad, ipad, blocksize); 890 891 for (i = 0; i < blocksize; i++) { 892 ipad[i] ^= HMAC_IPAD_VALUE; 893 opad[i] ^= HMAC_OPAD_VALUE; 894 } 895 896 return 0; 897 } 898 899 static int safexcel_hmac_init_iv(struct ahash_request *areq, 900 unsigned int blocksize, u8 *pad, void *state) 901 { 902 struct safexcel_ahash_result result; 903 struct safexcel_ahash_req *req; 904 struct scatterlist sg; 905 int ret; 906 907 ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG, 908 safexcel_ahash_complete, &result); 909 sg_init_one(&sg, pad, blocksize); 910 ahash_request_set_crypt(areq, &sg, pad, blocksize); 911 init_completion(&result.completion); 912 913 ret = crypto_ahash_init(areq); 914 if (ret) 915 return ret; 916 917 req = ahash_request_ctx(areq); 918 req->hmac = true; 919 req->last_req = true; 920 921 ret = crypto_ahash_update(areq); 922 if (ret && ret != -EINPROGRESS && ret != -EBUSY) 923 return ret; 924 925 wait_for_completion_interruptible(&result.completion); 926 if (result.error) 927 return result.error; 928 929 return crypto_ahash_export(areq, state); 930 } 931 932 int safexcel_hmac_setkey(const char *alg, const u8 *key, unsigned int keylen, 933 void *istate, void *ostate) 934 { 935 struct ahash_request *areq; 936 struct crypto_ahash *tfm; 937 unsigned int blocksize; 938 u8 *ipad, *opad; 939 int ret; 940 941 tfm = crypto_alloc_ahash(alg, 0, 0); 942 if (IS_ERR(tfm)) 943 return PTR_ERR(tfm); 944 945 areq = ahash_request_alloc(tfm, GFP_KERNEL); 946 if (!areq) { 947 ret = -ENOMEM; 948 goto free_ahash; 949 } 950 951 crypto_ahash_clear_flags(tfm, ~0); 952 blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 953 954 ipad = kcalloc(2, blocksize, GFP_KERNEL); 955 if (!ipad) { 956 ret = -ENOMEM; 957 goto free_request; 958 } 959 960 opad = ipad + blocksize; 961 962 ret = safexcel_hmac_init_pad(areq, blocksize, key, keylen, ipad, opad); 963 if (ret) 964 goto free_ipad; 965 966 ret = safexcel_hmac_init_iv(areq, blocksize, ipad, istate); 967 if (ret) 968 goto free_ipad; 969 970 ret = safexcel_hmac_init_iv(areq, blocksize, opad, ostate); 971 972 free_ipad: 973 kfree(ipad); 974 free_request: 975 ahash_request_free(areq); 976 free_ahash: 977 crypto_free_ahash(tfm); 978 979 return ret; 980 } 981 982 static int safexcel_hmac_alg_setkey(struct crypto_ahash *tfm, const u8 *key, 983 unsigned int keylen, const char *alg, 984 unsigned int state_sz) 985 { 986 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 987 struct safexcel_crypto_priv *priv = ctx->priv; 988 struct safexcel_ahash_export_state istate, ostate; 989 int ret, i; 990 991 ret = safexcel_hmac_setkey(alg, key, keylen, &istate, &ostate); 992 if (ret) 993 return ret; 994 995 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr) { 996 for (i = 0; i < state_sz / sizeof(u32); i++) { 997 if (ctx->ipad[i] != le32_to_cpu(istate.state[i]) || 998 ctx->opad[i] != le32_to_cpu(ostate.state[i])) { 999 ctx->base.needs_inv = true; 1000 break; 1001 } 1002 } 1003 } 1004 1005 memcpy(ctx->ipad, &istate.state, state_sz); 1006 memcpy(ctx->opad, &ostate.state, state_sz); 1007 1008 return 0; 1009 } 1010 1011 static int safexcel_hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key, 1012 unsigned int keylen) 1013 { 1014 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha1", 1015 SHA1_DIGEST_SIZE); 1016 } 1017 1018 struct safexcel_alg_template safexcel_alg_hmac_sha1 = { 1019 .type = SAFEXCEL_ALG_TYPE_AHASH, 1020 .engines = EIP97IES | EIP197B | EIP197D, 1021 .alg.ahash = { 1022 .init = safexcel_hmac_sha1_init, 1023 .update = safexcel_ahash_update, 1024 .final = safexcel_ahash_final, 1025 .finup = safexcel_ahash_finup, 1026 .digest = safexcel_hmac_sha1_digest, 1027 .setkey = safexcel_hmac_sha1_setkey, 1028 .export = safexcel_ahash_export, 1029 .import = safexcel_ahash_import, 1030 .halg = { 1031 .digestsize = SHA1_DIGEST_SIZE, 1032 .statesize = sizeof(struct safexcel_ahash_export_state), 1033 .base = { 1034 .cra_name = "hmac(sha1)", 1035 .cra_driver_name = "safexcel-hmac-sha1", 1036 .cra_priority = 300, 1037 .cra_flags = CRYPTO_ALG_ASYNC | 1038 CRYPTO_ALG_KERN_DRIVER_ONLY, 1039 .cra_blocksize = SHA1_BLOCK_SIZE, 1040 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1041 .cra_init = safexcel_ahash_cra_init, 1042 .cra_exit = safexcel_ahash_cra_exit, 1043 .cra_module = THIS_MODULE, 1044 }, 1045 }, 1046 }, 1047 }; 1048 1049 static int safexcel_sha256_init(struct ahash_request *areq) 1050 { 1051 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1052 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 1053 1054 memset(req, 0, sizeof(*req)); 1055 1056 req->state[0] = SHA256_H0; 1057 req->state[1] = SHA256_H1; 1058 req->state[2] = SHA256_H2; 1059 req->state[3] = SHA256_H3; 1060 req->state[4] = SHA256_H4; 1061 req->state[5] = SHA256_H5; 1062 req->state[6] = SHA256_H6; 1063 req->state[7] = SHA256_H7; 1064 1065 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256; 1066 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1067 req->state_sz = SHA256_DIGEST_SIZE; 1068 1069 return 0; 1070 } 1071 1072 static int safexcel_sha256_digest(struct ahash_request *areq) 1073 { 1074 int ret = safexcel_sha256_init(areq); 1075 1076 if (ret) 1077 return ret; 1078 1079 return safexcel_ahash_finup(areq); 1080 } 1081 1082 struct safexcel_alg_template safexcel_alg_sha256 = { 1083 .type = SAFEXCEL_ALG_TYPE_AHASH, 1084 .engines = EIP97IES | EIP197B | EIP197D, 1085 .alg.ahash = { 1086 .init = safexcel_sha256_init, 1087 .update = safexcel_ahash_update, 1088 .final = safexcel_ahash_final, 1089 .finup = safexcel_ahash_finup, 1090 .digest = safexcel_sha256_digest, 1091 .export = safexcel_ahash_export, 1092 .import = safexcel_ahash_import, 1093 .halg = { 1094 .digestsize = SHA256_DIGEST_SIZE, 1095 .statesize = sizeof(struct safexcel_ahash_export_state), 1096 .base = { 1097 .cra_name = "sha256", 1098 .cra_driver_name = "safexcel-sha256", 1099 .cra_priority = 300, 1100 .cra_flags = CRYPTO_ALG_ASYNC | 1101 CRYPTO_ALG_KERN_DRIVER_ONLY, 1102 .cra_blocksize = SHA256_BLOCK_SIZE, 1103 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1104 .cra_init = safexcel_ahash_cra_init, 1105 .cra_exit = safexcel_ahash_cra_exit, 1106 .cra_module = THIS_MODULE, 1107 }, 1108 }, 1109 }, 1110 }; 1111 1112 static int safexcel_sha224_init(struct ahash_request *areq) 1113 { 1114 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1115 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 1116 1117 memset(req, 0, sizeof(*req)); 1118 1119 req->state[0] = SHA224_H0; 1120 req->state[1] = SHA224_H1; 1121 req->state[2] = SHA224_H2; 1122 req->state[3] = SHA224_H3; 1123 req->state[4] = SHA224_H4; 1124 req->state[5] = SHA224_H5; 1125 req->state[6] = SHA224_H6; 1126 req->state[7] = SHA224_H7; 1127 1128 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224; 1129 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1130 req->state_sz = SHA256_DIGEST_SIZE; 1131 1132 return 0; 1133 } 1134 1135 static int safexcel_sha224_digest(struct ahash_request *areq) 1136 { 1137 int ret = safexcel_sha224_init(areq); 1138 1139 if (ret) 1140 return ret; 1141 1142 return safexcel_ahash_finup(areq); 1143 } 1144 1145 struct safexcel_alg_template safexcel_alg_sha224 = { 1146 .type = SAFEXCEL_ALG_TYPE_AHASH, 1147 .engines = EIP97IES | EIP197B | EIP197D, 1148 .alg.ahash = { 1149 .init = safexcel_sha224_init, 1150 .update = safexcel_ahash_update, 1151 .final = safexcel_ahash_final, 1152 .finup = safexcel_ahash_finup, 1153 .digest = safexcel_sha224_digest, 1154 .export = safexcel_ahash_export, 1155 .import = safexcel_ahash_import, 1156 .halg = { 1157 .digestsize = SHA224_DIGEST_SIZE, 1158 .statesize = sizeof(struct safexcel_ahash_export_state), 1159 .base = { 1160 .cra_name = "sha224", 1161 .cra_driver_name = "safexcel-sha224", 1162 .cra_priority = 300, 1163 .cra_flags = CRYPTO_ALG_ASYNC | 1164 CRYPTO_ALG_KERN_DRIVER_ONLY, 1165 .cra_blocksize = SHA224_BLOCK_SIZE, 1166 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1167 .cra_init = safexcel_ahash_cra_init, 1168 .cra_exit = safexcel_ahash_cra_exit, 1169 .cra_module = THIS_MODULE, 1170 }, 1171 }, 1172 }, 1173 }; 1174 1175 static int safexcel_hmac_sha224_setkey(struct crypto_ahash *tfm, const u8 *key, 1176 unsigned int keylen) 1177 { 1178 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha224", 1179 SHA256_DIGEST_SIZE); 1180 } 1181 1182 static int safexcel_hmac_sha224_init(struct ahash_request *areq) 1183 { 1184 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 1185 1186 safexcel_sha224_init(areq); 1187 req->digest = CONTEXT_CONTROL_DIGEST_HMAC; 1188 return 0; 1189 } 1190 1191 static int safexcel_hmac_sha224_digest(struct ahash_request *areq) 1192 { 1193 int ret = safexcel_hmac_sha224_init(areq); 1194 1195 if (ret) 1196 return ret; 1197 1198 return safexcel_ahash_finup(areq); 1199 } 1200 1201 struct safexcel_alg_template safexcel_alg_hmac_sha224 = { 1202 .type = SAFEXCEL_ALG_TYPE_AHASH, 1203 .engines = EIP97IES | EIP197B | EIP197D, 1204 .alg.ahash = { 1205 .init = safexcel_hmac_sha224_init, 1206 .update = safexcel_ahash_update, 1207 .final = safexcel_ahash_final, 1208 .finup = safexcel_ahash_finup, 1209 .digest = safexcel_hmac_sha224_digest, 1210 .setkey = safexcel_hmac_sha224_setkey, 1211 .export = safexcel_ahash_export, 1212 .import = safexcel_ahash_import, 1213 .halg = { 1214 .digestsize = SHA224_DIGEST_SIZE, 1215 .statesize = sizeof(struct safexcel_ahash_export_state), 1216 .base = { 1217 .cra_name = "hmac(sha224)", 1218 .cra_driver_name = "safexcel-hmac-sha224", 1219 .cra_priority = 300, 1220 .cra_flags = CRYPTO_ALG_ASYNC | 1221 CRYPTO_ALG_KERN_DRIVER_ONLY, 1222 .cra_blocksize = SHA224_BLOCK_SIZE, 1223 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1224 .cra_init = safexcel_ahash_cra_init, 1225 .cra_exit = safexcel_ahash_cra_exit, 1226 .cra_module = THIS_MODULE, 1227 }, 1228 }, 1229 }, 1230 }; 1231 1232 static int safexcel_hmac_sha256_setkey(struct crypto_ahash *tfm, const u8 *key, 1233 unsigned int keylen) 1234 { 1235 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha256", 1236 SHA256_DIGEST_SIZE); 1237 } 1238 1239 static int safexcel_hmac_sha256_init(struct ahash_request *areq) 1240 { 1241 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 1242 1243 safexcel_sha256_init(areq); 1244 req->digest = CONTEXT_CONTROL_DIGEST_HMAC; 1245 return 0; 1246 } 1247 1248 static int safexcel_hmac_sha256_digest(struct ahash_request *areq) 1249 { 1250 int ret = safexcel_hmac_sha256_init(areq); 1251 1252 if (ret) 1253 return ret; 1254 1255 return safexcel_ahash_finup(areq); 1256 } 1257 1258 struct safexcel_alg_template safexcel_alg_hmac_sha256 = { 1259 .type = SAFEXCEL_ALG_TYPE_AHASH, 1260 .engines = EIP97IES | EIP197B | EIP197D, 1261 .alg.ahash = { 1262 .init = safexcel_hmac_sha256_init, 1263 .update = safexcel_ahash_update, 1264 .final = safexcel_ahash_final, 1265 .finup = safexcel_ahash_finup, 1266 .digest = safexcel_hmac_sha256_digest, 1267 .setkey = safexcel_hmac_sha256_setkey, 1268 .export = safexcel_ahash_export, 1269 .import = safexcel_ahash_import, 1270 .halg = { 1271 .digestsize = SHA256_DIGEST_SIZE, 1272 .statesize = sizeof(struct safexcel_ahash_export_state), 1273 .base = { 1274 .cra_name = "hmac(sha256)", 1275 .cra_driver_name = "safexcel-hmac-sha256", 1276 .cra_priority = 300, 1277 .cra_flags = CRYPTO_ALG_ASYNC | 1278 CRYPTO_ALG_KERN_DRIVER_ONLY, 1279 .cra_blocksize = SHA256_BLOCK_SIZE, 1280 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1281 .cra_init = safexcel_ahash_cra_init, 1282 .cra_exit = safexcel_ahash_cra_exit, 1283 .cra_module = THIS_MODULE, 1284 }, 1285 }, 1286 }, 1287 }; 1288 1289 static int safexcel_sha512_init(struct ahash_request *areq) 1290 { 1291 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1292 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 1293 1294 memset(req, 0, sizeof(*req)); 1295 1296 req->state[0] = lower_32_bits(SHA512_H0); 1297 req->state[1] = upper_32_bits(SHA512_H0); 1298 req->state[2] = lower_32_bits(SHA512_H1); 1299 req->state[3] = upper_32_bits(SHA512_H1); 1300 req->state[4] = lower_32_bits(SHA512_H2); 1301 req->state[5] = upper_32_bits(SHA512_H2); 1302 req->state[6] = lower_32_bits(SHA512_H3); 1303 req->state[7] = upper_32_bits(SHA512_H3); 1304 req->state[8] = lower_32_bits(SHA512_H4); 1305 req->state[9] = upper_32_bits(SHA512_H4); 1306 req->state[10] = lower_32_bits(SHA512_H5); 1307 req->state[11] = upper_32_bits(SHA512_H5); 1308 req->state[12] = lower_32_bits(SHA512_H6); 1309 req->state[13] = upper_32_bits(SHA512_H6); 1310 req->state[14] = lower_32_bits(SHA512_H7); 1311 req->state[15] = upper_32_bits(SHA512_H7); 1312 1313 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512; 1314 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1315 req->state_sz = SHA512_DIGEST_SIZE; 1316 1317 return 0; 1318 } 1319 1320 static int safexcel_sha512_digest(struct ahash_request *areq) 1321 { 1322 int ret = safexcel_sha512_init(areq); 1323 1324 if (ret) 1325 return ret; 1326 1327 return safexcel_ahash_finup(areq); 1328 } 1329 1330 struct safexcel_alg_template safexcel_alg_sha512 = { 1331 .type = SAFEXCEL_ALG_TYPE_AHASH, 1332 .engines = EIP97IES | EIP197B | EIP197D, 1333 .alg.ahash = { 1334 .init = safexcel_sha512_init, 1335 .update = safexcel_ahash_update, 1336 .final = safexcel_ahash_final, 1337 .finup = safexcel_ahash_finup, 1338 .digest = safexcel_sha512_digest, 1339 .export = safexcel_ahash_export, 1340 .import = safexcel_ahash_import, 1341 .halg = { 1342 .digestsize = SHA512_DIGEST_SIZE, 1343 .statesize = sizeof(struct safexcel_ahash_export_state), 1344 .base = { 1345 .cra_name = "sha512", 1346 .cra_driver_name = "safexcel-sha512", 1347 .cra_priority = 300, 1348 .cra_flags = CRYPTO_ALG_ASYNC | 1349 CRYPTO_ALG_KERN_DRIVER_ONLY, 1350 .cra_blocksize = SHA512_BLOCK_SIZE, 1351 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1352 .cra_init = safexcel_ahash_cra_init, 1353 .cra_exit = safexcel_ahash_cra_exit, 1354 .cra_module = THIS_MODULE, 1355 }, 1356 }, 1357 }, 1358 }; 1359 1360 static int safexcel_sha384_init(struct ahash_request *areq) 1361 { 1362 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1363 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 1364 1365 memset(req, 0, sizeof(*req)); 1366 1367 req->state[0] = lower_32_bits(SHA384_H0); 1368 req->state[1] = upper_32_bits(SHA384_H0); 1369 req->state[2] = lower_32_bits(SHA384_H1); 1370 req->state[3] = upper_32_bits(SHA384_H1); 1371 req->state[4] = lower_32_bits(SHA384_H2); 1372 req->state[5] = upper_32_bits(SHA384_H2); 1373 req->state[6] = lower_32_bits(SHA384_H3); 1374 req->state[7] = upper_32_bits(SHA384_H3); 1375 req->state[8] = lower_32_bits(SHA384_H4); 1376 req->state[9] = upper_32_bits(SHA384_H4); 1377 req->state[10] = lower_32_bits(SHA384_H5); 1378 req->state[11] = upper_32_bits(SHA384_H5); 1379 req->state[12] = lower_32_bits(SHA384_H6); 1380 req->state[13] = upper_32_bits(SHA384_H6); 1381 req->state[14] = lower_32_bits(SHA384_H7); 1382 req->state[15] = upper_32_bits(SHA384_H7); 1383 1384 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384; 1385 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1386 req->state_sz = SHA512_DIGEST_SIZE; 1387 1388 return 0; 1389 } 1390 1391 static int safexcel_sha384_digest(struct ahash_request *areq) 1392 { 1393 int ret = safexcel_sha384_init(areq); 1394 1395 if (ret) 1396 return ret; 1397 1398 return safexcel_ahash_finup(areq); 1399 } 1400 1401 struct safexcel_alg_template safexcel_alg_sha384 = { 1402 .type = SAFEXCEL_ALG_TYPE_AHASH, 1403 .engines = EIP97IES | EIP197B | EIP197D, 1404 .alg.ahash = { 1405 .init = safexcel_sha384_init, 1406 .update = safexcel_ahash_update, 1407 .final = safexcel_ahash_final, 1408 .finup = safexcel_ahash_finup, 1409 .digest = safexcel_sha384_digest, 1410 .export = safexcel_ahash_export, 1411 .import = safexcel_ahash_import, 1412 .halg = { 1413 .digestsize = SHA384_DIGEST_SIZE, 1414 .statesize = sizeof(struct safexcel_ahash_export_state), 1415 .base = { 1416 .cra_name = "sha384", 1417 .cra_driver_name = "safexcel-sha384", 1418 .cra_priority = 300, 1419 .cra_flags = CRYPTO_ALG_ASYNC | 1420 CRYPTO_ALG_KERN_DRIVER_ONLY, 1421 .cra_blocksize = SHA384_BLOCK_SIZE, 1422 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1423 .cra_init = safexcel_ahash_cra_init, 1424 .cra_exit = safexcel_ahash_cra_exit, 1425 .cra_module = THIS_MODULE, 1426 }, 1427 }, 1428 }, 1429 }; 1430 1431 static int safexcel_hmac_sha512_setkey(struct crypto_ahash *tfm, const u8 *key, 1432 unsigned int keylen) 1433 { 1434 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha512", 1435 SHA512_DIGEST_SIZE); 1436 } 1437 1438 static int safexcel_hmac_sha512_init(struct ahash_request *areq) 1439 { 1440 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 1441 1442 safexcel_sha512_init(areq); 1443 req->digest = CONTEXT_CONTROL_DIGEST_HMAC; 1444 return 0; 1445 } 1446 1447 static int safexcel_hmac_sha512_digest(struct ahash_request *areq) 1448 { 1449 int ret = safexcel_hmac_sha512_init(areq); 1450 1451 if (ret) 1452 return ret; 1453 1454 return safexcel_ahash_finup(areq); 1455 } 1456 1457 struct safexcel_alg_template safexcel_alg_hmac_sha512 = { 1458 .type = SAFEXCEL_ALG_TYPE_AHASH, 1459 .engines = EIP97IES | EIP197B | EIP197D, 1460 .alg.ahash = { 1461 .init = safexcel_hmac_sha512_init, 1462 .update = safexcel_ahash_update, 1463 .final = safexcel_ahash_final, 1464 .finup = safexcel_ahash_finup, 1465 .digest = safexcel_hmac_sha512_digest, 1466 .setkey = safexcel_hmac_sha512_setkey, 1467 .export = safexcel_ahash_export, 1468 .import = safexcel_ahash_import, 1469 .halg = { 1470 .digestsize = SHA512_DIGEST_SIZE, 1471 .statesize = sizeof(struct safexcel_ahash_export_state), 1472 .base = { 1473 .cra_name = "hmac(sha512)", 1474 .cra_driver_name = "safexcel-hmac-sha512", 1475 .cra_priority = 300, 1476 .cra_flags = CRYPTO_ALG_ASYNC | 1477 CRYPTO_ALG_KERN_DRIVER_ONLY, 1478 .cra_blocksize = SHA512_BLOCK_SIZE, 1479 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1480 .cra_init = safexcel_ahash_cra_init, 1481 .cra_exit = safexcel_ahash_cra_exit, 1482 .cra_module = THIS_MODULE, 1483 }, 1484 }, 1485 }, 1486 }; 1487 1488 static int safexcel_hmac_sha384_setkey(struct crypto_ahash *tfm, const u8 *key, 1489 unsigned int keylen) 1490 { 1491 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha384", 1492 SHA512_DIGEST_SIZE); 1493 } 1494 1495 static int safexcel_hmac_sha384_init(struct ahash_request *areq) 1496 { 1497 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 1498 1499 safexcel_sha384_init(areq); 1500 req->digest = CONTEXT_CONTROL_DIGEST_HMAC; 1501 return 0; 1502 } 1503 1504 static int safexcel_hmac_sha384_digest(struct ahash_request *areq) 1505 { 1506 int ret = safexcel_hmac_sha384_init(areq); 1507 1508 if (ret) 1509 return ret; 1510 1511 return safexcel_ahash_finup(areq); 1512 } 1513 1514 struct safexcel_alg_template safexcel_alg_hmac_sha384 = { 1515 .type = SAFEXCEL_ALG_TYPE_AHASH, 1516 .engines = EIP97IES | EIP197B | EIP197D, 1517 .alg.ahash = { 1518 .init = safexcel_hmac_sha384_init, 1519 .update = safexcel_ahash_update, 1520 .final = safexcel_ahash_final, 1521 .finup = safexcel_ahash_finup, 1522 .digest = safexcel_hmac_sha384_digest, 1523 .setkey = safexcel_hmac_sha384_setkey, 1524 .export = safexcel_ahash_export, 1525 .import = safexcel_ahash_import, 1526 .halg = { 1527 .digestsize = SHA384_DIGEST_SIZE, 1528 .statesize = sizeof(struct safexcel_ahash_export_state), 1529 .base = { 1530 .cra_name = "hmac(sha384)", 1531 .cra_driver_name = "safexcel-hmac-sha384", 1532 .cra_priority = 300, 1533 .cra_flags = CRYPTO_ALG_ASYNC | 1534 CRYPTO_ALG_KERN_DRIVER_ONLY, 1535 .cra_blocksize = SHA384_BLOCK_SIZE, 1536 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1537 .cra_init = safexcel_ahash_cra_init, 1538 .cra_exit = safexcel_ahash_cra_exit, 1539 .cra_module = THIS_MODULE, 1540 }, 1541 }, 1542 }, 1543 }; 1544 1545 static int safexcel_md5_init(struct ahash_request *areq) 1546 { 1547 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1548 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 1549 1550 memset(req, 0, sizeof(*req)); 1551 1552 req->state[0] = MD5_H0; 1553 req->state[1] = MD5_H1; 1554 req->state[2] = MD5_H2; 1555 req->state[3] = MD5_H3; 1556 1557 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5; 1558 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1559 req->state_sz = MD5_DIGEST_SIZE; 1560 1561 return 0; 1562 } 1563 1564 static int safexcel_md5_digest(struct ahash_request *areq) 1565 { 1566 int ret = safexcel_md5_init(areq); 1567 1568 if (ret) 1569 return ret; 1570 1571 return safexcel_ahash_finup(areq); 1572 } 1573 1574 struct safexcel_alg_template safexcel_alg_md5 = { 1575 .type = SAFEXCEL_ALG_TYPE_AHASH, 1576 .engines = EIP97IES | EIP197B | EIP197D, 1577 .alg.ahash = { 1578 .init = safexcel_md5_init, 1579 .update = safexcel_ahash_update, 1580 .final = safexcel_ahash_final, 1581 .finup = safexcel_ahash_finup, 1582 .digest = safexcel_md5_digest, 1583 .export = safexcel_ahash_export, 1584 .import = safexcel_ahash_import, 1585 .halg = { 1586 .digestsize = MD5_DIGEST_SIZE, 1587 .statesize = sizeof(struct safexcel_ahash_export_state), 1588 .base = { 1589 .cra_name = "md5", 1590 .cra_driver_name = "safexcel-md5", 1591 .cra_priority = 300, 1592 .cra_flags = CRYPTO_ALG_ASYNC | 1593 CRYPTO_ALG_KERN_DRIVER_ONLY, 1594 .cra_blocksize = MD5_HMAC_BLOCK_SIZE, 1595 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1596 .cra_init = safexcel_ahash_cra_init, 1597 .cra_exit = safexcel_ahash_cra_exit, 1598 .cra_module = THIS_MODULE, 1599 }, 1600 }, 1601 }, 1602 }; 1603 1604 static int safexcel_hmac_md5_init(struct ahash_request *areq) 1605 { 1606 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 1607 1608 safexcel_md5_init(areq); 1609 req->digest = CONTEXT_CONTROL_DIGEST_HMAC; 1610 return 0; 1611 } 1612 1613 static int safexcel_hmac_md5_setkey(struct crypto_ahash *tfm, const u8 *key, 1614 unsigned int keylen) 1615 { 1616 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-md5", 1617 MD5_DIGEST_SIZE); 1618 } 1619 1620 static int safexcel_hmac_md5_digest(struct ahash_request *areq) 1621 { 1622 int ret = safexcel_hmac_md5_init(areq); 1623 1624 if (ret) 1625 return ret; 1626 1627 return safexcel_ahash_finup(areq); 1628 } 1629 1630 struct safexcel_alg_template safexcel_alg_hmac_md5 = { 1631 .type = SAFEXCEL_ALG_TYPE_AHASH, 1632 .engines = EIP97IES | EIP197B | EIP197D, 1633 .alg.ahash = { 1634 .init = safexcel_hmac_md5_init, 1635 .update = safexcel_ahash_update, 1636 .final = safexcel_ahash_final, 1637 .finup = safexcel_ahash_finup, 1638 .digest = safexcel_hmac_md5_digest, 1639 .setkey = safexcel_hmac_md5_setkey, 1640 .export = safexcel_ahash_export, 1641 .import = safexcel_ahash_import, 1642 .halg = { 1643 .digestsize = MD5_DIGEST_SIZE, 1644 .statesize = sizeof(struct safexcel_ahash_export_state), 1645 .base = { 1646 .cra_name = "hmac(md5)", 1647 .cra_driver_name = "safexcel-hmac-md5", 1648 .cra_priority = 300, 1649 .cra_flags = CRYPTO_ALG_ASYNC | 1650 CRYPTO_ALG_KERN_DRIVER_ONLY, 1651 .cra_blocksize = MD5_HMAC_BLOCK_SIZE, 1652 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1653 .cra_init = safexcel_ahash_cra_init, 1654 .cra_exit = safexcel_ahash_cra_exit, 1655 .cra_module = THIS_MODULE, 1656 }, 1657 }, 1658 }, 1659 }; 1660