1 // SPDX-License-Identifier: GPL-2.0-only 2 // SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 /* 4 * Crypto driver to handle block cipher algorithms using NVIDIA Security Engine. 5 */ 6 7 #include <linux/bottom_half.h> 8 #include <linux/clk.h> 9 #include <linux/dma-mapping.h> 10 #include <linux/module.h> 11 #include <linux/of_device.h> 12 #include <linux/platform_device.h> 13 14 #include <crypto/aead.h> 15 #include <crypto/aes.h> 16 #include <crypto/engine.h> 17 #include <crypto/gcm.h> 18 #include <crypto/scatterwalk.h> 19 #include <crypto/xts.h> 20 #include <crypto/internal/aead.h> 21 #include <crypto/internal/hash.h> 22 #include <crypto/internal/skcipher.h> 23 24 #include "tegra-se.h" 25 26 struct tegra_aes_ctx { 27 struct tegra_se *se; 28 u32 alg; 29 u32 ivsize; 30 u32 key1_id; 31 u32 key2_id; 32 u32 keylen; 33 u8 key1[AES_MAX_KEY_SIZE]; 34 u8 key2[AES_MAX_KEY_SIZE]; 35 }; 36 37 struct tegra_aes_reqctx { 38 struct tegra_se_datbuf datbuf; 39 bool encrypt; 40 u32 config; 41 u32 crypto_config; 42 u32 len; 43 u32 *iv; 44 }; 45 46 struct tegra_aead_ctx { 47 struct tegra_se *se; 48 u32 alg; 49 u32 key_id; 50 u32 keylen; 51 u8 key[AES_MAX_KEY_SIZE]; 52 }; 53 54 struct tegra_aead_reqctx { 55 struct tegra_se_datbuf inbuf; 56 struct tegra_se_datbuf outbuf; 57 struct scatterlist *src_sg; 58 struct scatterlist *dst_sg; 59 unsigned int assoclen; 60 unsigned int cryptlen; 61 unsigned int authsize; 62 bool encrypt; 63 u32 crypto_config; 64 u32 config; 65 u32 key_id; 66 u32 iv[4]; 67 u8 authdata[16]; 68 }; 69 70 struct tegra_cmac_ctx { 71 struct tegra_se *se; 72 unsigned int alg; 73 u32 key_id; 74 u32 keylen; 75 u8 key[AES_MAX_KEY_SIZE]; 76 struct crypto_shash *fallback_tfm; 77 }; 78 79 struct tegra_cmac_reqctx { 80 struct scatterlist *src_sg; 81 struct tegra_se_datbuf datbuf; 82 struct tegra_se_datbuf residue; 83 unsigned int total_len; 84 unsigned int blk_size; 85 unsigned int task; 86 u32 crypto_config; 87 u32 config; 88 u32 key_id; 89 u32 *iv; 90 u32 result[CMAC_RESULT_REG_COUNT]; 91 }; 92 93 /* increment counter (128-bit int) */ 94 static void ctr_iv_inc(__u8 *counter, __u8 bits, __u32 nums) 95 { 96 do { 97 --bits; 98 nums += counter[bits]; 99 counter[bits] = nums & 0xff; 100 nums >>= 8; 101 } while (bits && nums); 102 } 103 104 static void tegra_cbc_iv_copyback(struct skcipher_request *req, struct tegra_aes_ctx *ctx) 105 { 106 struct tegra_aes_reqctx *rctx = skcipher_request_ctx(req); 107 unsigned int offset; 108 109 offset = req->cryptlen - ctx->ivsize; 110 111 if (rctx->encrypt) 112 memcpy(req->iv, rctx->datbuf.buf + offset, ctx->ivsize); 113 else 114 scatterwalk_map_and_copy(req->iv, req->src, offset, ctx->ivsize, 0); 115 } 116 117 static void tegra_aes_update_iv(struct skcipher_request *req, struct tegra_aes_ctx *ctx) 118 { 119 int num; 120 121 if (ctx->alg == SE_ALG_CBC) { 122 tegra_cbc_iv_copyback(req, ctx); 123 } else if (ctx->alg == SE_ALG_CTR) { 124 num = req->cryptlen / ctx->ivsize; 125 if (req->cryptlen % ctx->ivsize) 126 num++; 127 128 ctr_iv_inc(req->iv, ctx->ivsize, num); 129 } 130 } 131 132 static int tegra234_aes_crypto_cfg(u32 alg, bool encrypt) 133 { 134 switch (alg) { 135 case SE_ALG_CMAC: 136 case SE_ALG_GMAC: 137 case SE_ALG_GCM: 138 case SE_ALG_GCM_FINAL: 139 return 0; 140 case SE_ALG_CBC: 141 if (encrypt) 142 return SE_CRYPTO_CFG_CBC_ENCRYPT; 143 else 144 return SE_CRYPTO_CFG_CBC_DECRYPT; 145 case SE_ALG_ECB: 146 if (encrypt) 147 return SE_CRYPTO_CFG_ECB_ENCRYPT; 148 else 149 return SE_CRYPTO_CFG_ECB_DECRYPT; 150 case SE_ALG_XTS: 151 if (encrypt) 152 return SE_CRYPTO_CFG_XTS_ENCRYPT; 153 else 154 return SE_CRYPTO_CFG_XTS_DECRYPT; 155 156 case SE_ALG_CTR: 157 return SE_CRYPTO_CFG_CTR; 158 case SE_ALG_CBC_MAC: 159 return SE_CRYPTO_CFG_CBC_MAC; 160 161 default: 162 break; 163 } 164 165 return -EINVAL; 166 } 167 168 static int tegra234_aes_cfg(u32 alg, bool encrypt) 169 { 170 switch (alg) { 171 case SE_ALG_CBC: 172 case SE_ALG_ECB: 173 case SE_ALG_XTS: 174 case SE_ALG_CTR: 175 if (encrypt) 176 return SE_CFG_AES_ENCRYPT; 177 else 178 return SE_CFG_AES_DECRYPT; 179 180 case SE_ALG_GMAC: 181 if (encrypt) 182 return SE_CFG_GMAC_ENCRYPT; 183 else 184 return SE_CFG_GMAC_DECRYPT; 185 186 case SE_ALG_GCM: 187 if (encrypt) 188 return SE_CFG_GCM_ENCRYPT; 189 else 190 return SE_CFG_GCM_DECRYPT; 191 192 case SE_ALG_GCM_FINAL: 193 if (encrypt) 194 return SE_CFG_GCM_FINAL_ENCRYPT; 195 else 196 return SE_CFG_GCM_FINAL_DECRYPT; 197 198 case SE_ALG_CMAC: 199 return SE_CFG_CMAC; 200 201 case SE_ALG_CBC_MAC: 202 return SE_AES_ENC_ALG_AES_ENC | 203 SE_AES_DST_HASH_REG; 204 } 205 return -EINVAL; 206 } 207 208 static unsigned int tegra_aes_prep_cmd(struct tegra_aes_ctx *ctx, 209 struct tegra_aes_reqctx *rctx) 210 { 211 unsigned int data_count, res_bits, i = 0, j; 212 struct tegra_se *se = ctx->se; 213 u32 *cpuvaddr = se->cmdbuf->addr; 214 dma_addr_t addr = rctx->datbuf.addr; 215 216 data_count = rctx->len / AES_BLOCK_SIZE; 217 res_bits = (rctx->len % AES_BLOCK_SIZE) * 8; 218 219 /* 220 * Hardware processes data_count + 1 blocks. 221 * Reduce 1 block if there is no residue 222 */ 223 if (!res_bits) 224 data_count--; 225 226 if (rctx->iv) { 227 cpuvaddr[i++] = host1x_opcode_setpayload(SE_CRYPTO_CTR_REG_COUNT); 228 cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->linear_ctr); 229 for (j = 0; j < SE_CRYPTO_CTR_REG_COUNT; j++) 230 cpuvaddr[i++] = rctx->iv[j]; 231 } 232 233 cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->last_blk, 1); 234 cpuvaddr[i++] = SE_LAST_BLOCK_VAL(data_count) | 235 SE_LAST_BLOCK_RES_BITS(res_bits); 236 237 cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 6); 238 cpuvaddr[i++] = rctx->config; 239 cpuvaddr[i++] = rctx->crypto_config; 240 241 /* Source address setting */ 242 cpuvaddr[i++] = lower_32_bits(addr); 243 cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(addr)) | SE_ADDR_HI_SZ(rctx->len); 244 245 /* Destination address setting */ 246 cpuvaddr[i++] = lower_32_bits(addr); 247 cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(addr)) | 248 SE_ADDR_HI_SZ(rctx->len); 249 250 cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1); 251 cpuvaddr[i++] = SE_AES_OP_WRSTALL | SE_AES_OP_LASTBUF | 252 SE_AES_OP_START; 253 254 cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1); 255 cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) | 256 host1x_uclass_incr_syncpt_indx_f(se->syncpt_id); 257 258 dev_dbg(se->dev, "cfg %#x crypto cfg %#x\n", rctx->config, rctx->crypto_config); 259 260 return i; 261 } 262 263 static int tegra_aes_do_one_req(struct crypto_engine *engine, void *areq) 264 { 265 struct skcipher_request *req = container_of(areq, struct skcipher_request, base); 266 struct tegra_aes_ctx *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); 267 struct tegra_aes_reqctx *rctx = skcipher_request_ctx(req); 268 struct tegra_se *se = ctx->se; 269 unsigned int cmdlen, key1_id, key2_id; 270 int ret; 271 272 rctx->iv = (ctx->alg == SE_ALG_ECB) ? NULL : (u32 *)req->iv; 273 rctx->len = req->cryptlen; 274 key1_id = ctx->key1_id; 275 key2_id = ctx->key2_id; 276 277 /* Pad input to AES Block size */ 278 if (ctx->alg != SE_ALG_XTS) { 279 if (rctx->len % AES_BLOCK_SIZE) 280 rctx->len += AES_BLOCK_SIZE - (rctx->len % AES_BLOCK_SIZE); 281 } 282 283 rctx->datbuf.size = rctx->len; 284 rctx->datbuf.buf = dma_alloc_coherent(se->dev, rctx->datbuf.size, 285 &rctx->datbuf.addr, GFP_KERNEL); 286 if (!rctx->datbuf.buf) { 287 ret = -ENOMEM; 288 goto out_finalize; 289 } 290 291 scatterwalk_map_and_copy(rctx->datbuf.buf, req->src, 0, req->cryptlen, 0); 292 293 rctx->config = tegra234_aes_cfg(ctx->alg, rctx->encrypt); 294 rctx->crypto_config = tegra234_aes_crypto_cfg(ctx->alg, rctx->encrypt); 295 296 if (!key1_id) { 297 ret = tegra_key_submit_reserved_aes(ctx->se, ctx->key1, 298 ctx->keylen, ctx->alg, &key1_id); 299 if (ret) 300 goto out; 301 } 302 303 rctx->crypto_config |= SE_AES_KEY_INDEX(key1_id); 304 305 if (ctx->alg == SE_ALG_XTS) { 306 if (!key2_id) { 307 ret = tegra_key_submit_reserved_xts(ctx->se, ctx->key2, 308 ctx->keylen, ctx->alg, &key2_id); 309 if (ret) 310 goto out; 311 } 312 313 rctx->crypto_config |= SE_AES_KEY2_INDEX(key2_id); 314 } 315 316 /* Prepare the command and submit for execution */ 317 cmdlen = tegra_aes_prep_cmd(ctx, rctx); 318 ret = tegra_se_host1x_submit(se, se->cmdbuf, cmdlen); 319 320 /* Copy the result */ 321 tegra_aes_update_iv(req, ctx); 322 scatterwalk_map_and_copy(rctx->datbuf.buf, req->dst, 0, req->cryptlen, 1); 323 324 out: 325 /* Free the buffer */ 326 dma_free_coherent(ctx->se->dev, rctx->datbuf.size, 327 rctx->datbuf.buf, rctx->datbuf.addr); 328 329 if (tegra_key_is_reserved(key1_id)) 330 tegra_key_invalidate_reserved(ctx->se, key1_id, ctx->alg); 331 332 if (tegra_key_is_reserved(key2_id)) 333 tegra_key_invalidate_reserved(ctx->se, key2_id, ctx->alg); 334 335 out_finalize: 336 local_bh_disable(); 337 crypto_finalize_skcipher_request(se->engine, req, ret); 338 local_bh_enable(); 339 340 return 0; 341 } 342 343 static int tegra_aes_cra_init(struct crypto_skcipher *tfm) 344 { 345 struct tegra_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 346 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 347 struct tegra_se_alg *se_alg; 348 const char *algname; 349 int ret; 350 351 se_alg = container_of(alg, struct tegra_se_alg, alg.skcipher.base); 352 353 crypto_skcipher_set_reqsize(tfm, sizeof(struct tegra_aes_reqctx)); 354 355 ctx->ivsize = crypto_skcipher_ivsize(tfm); 356 ctx->se = se_alg->se_dev; 357 ctx->key1_id = 0; 358 ctx->key2_id = 0; 359 ctx->keylen = 0; 360 361 algname = crypto_tfm_alg_name(&tfm->base); 362 ret = se_algname_to_algid(algname); 363 if (ret < 0) { 364 dev_err(ctx->se->dev, "invalid algorithm\n"); 365 return ret; 366 } 367 368 ctx->alg = ret; 369 370 return 0; 371 } 372 373 static void tegra_aes_cra_exit(struct crypto_skcipher *tfm) 374 { 375 struct tegra_aes_ctx *ctx = crypto_tfm_ctx(&tfm->base); 376 377 if (ctx->key1_id) 378 tegra_key_invalidate(ctx->se, ctx->key1_id, ctx->alg); 379 380 if (ctx->key2_id) 381 tegra_key_invalidate(ctx->se, ctx->key2_id, ctx->alg); 382 } 383 384 static int tegra_aes_setkey(struct crypto_skcipher *tfm, 385 const u8 *key, u32 keylen) 386 { 387 struct tegra_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 388 int ret; 389 390 if (aes_check_keylen(keylen)) { 391 dev_dbg(ctx->se->dev, "invalid key length (%d)\n", keylen); 392 return -EINVAL; 393 } 394 395 ret = tegra_key_submit(ctx->se, key, keylen, ctx->alg, &ctx->key1_id); 396 if (ret) { 397 ctx->keylen = keylen; 398 memcpy(ctx->key1, key, keylen); 399 } 400 401 return 0; 402 } 403 404 static int tegra_xts_setkey(struct crypto_skcipher *tfm, 405 const u8 *key, u32 keylen) 406 { 407 struct tegra_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 408 u32 len = keylen / 2; 409 int ret; 410 411 ret = xts_verify_key(tfm, key, keylen); 412 if (ret || aes_check_keylen(len)) { 413 dev_dbg(ctx->se->dev, "invalid key length (%d)\n", keylen); 414 return -EINVAL; 415 } 416 417 ret = tegra_key_submit(ctx->se, key, len, 418 ctx->alg, &ctx->key1_id); 419 if (ret) { 420 ctx->keylen = len; 421 memcpy(ctx->key1, key, len); 422 } 423 424 ret = tegra_key_submit(ctx->se, key + len, len, 425 ctx->alg, &ctx->key2_id); 426 if (ret) { 427 ctx->keylen = len; 428 memcpy(ctx->key2, key + len, len); 429 } 430 431 return 0; 432 } 433 434 static int tegra_aes_kac_manifest(u32 user, u32 alg, u32 keylen) 435 { 436 int manifest; 437 438 manifest = SE_KAC_USER_NS; 439 440 switch (alg) { 441 case SE_ALG_CBC: 442 case SE_ALG_ECB: 443 case SE_ALG_CTR: 444 manifest |= SE_KAC_ENC; 445 break; 446 case SE_ALG_XTS: 447 manifest |= SE_KAC_XTS; 448 break; 449 case SE_ALG_GCM: 450 manifest |= SE_KAC_GCM; 451 break; 452 case SE_ALG_CMAC: 453 manifest |= SE_KAC_CMAC; 454 break; 455 case SE_ALG_CBC_MAC: 456 manifest |= SE_KAC_ENC; 457 break; 458 default: 459 return -EINVAL; 460 } 461 462 switch (keylen) { 463 case AES_KEYSIZE_128: 464 manifest |= SE_KAC_SIZE_128; 465 break; 466 case AES_KEYSIZE_192: 467 manifest |= SE_KAC_SIZE_192; 468 break; 469 case AES_KEYSIZE_256: 470 manifest |= SE_KAC_SIZE_256; 471 break; 472 default: 473 return -EINVAL; 474 } 475 476 return manifest; 477 } 478 479 static int tegra_aes_crypt(struct skcipher_request *req, bool encrypt) 480 481 { 482 struct crypto_skcipher *tfm; 483 struct tegra_aes_ctx *ctx; 484 struct tegra_aes_reqctx *rctx; 485 486 tfm = crypto_skcipher_reqtfm(req); 487 ctx = crypto_skcipher_ctx(tfm); 488 rctx = skcipher_request_ctx(req); 489 490 if (ctx->alg != SE_ALG_XTS) { 491 if (!IS_ALIGNED(req->cryptlen, crypto_skcipher_blocksize(tfm))) { 492 dev_dbg(ctx->se->dev, "invalid length (%d)", req->cryptlen); 493 return -EINVAL; 494 } 495 } else if (req->cryptlen < XTS_BLOCK_SIZE) { 496 dev_dbg(ctx->se->dev, "invalid length (%d)", req->cryptlen); 497 return -EINVAL; 498 } 499 500 if (!req->cryptlen) 501 return 0; 502 503 rctx->encrypt = encrypt; 504 505 return crypto_transfer_skcipher_request_to_engine(ctx->se->engine, req); 506 } 507 508 static int tegra_aes_encrypt(struct skcipher_request *req) 509 { 510 return tegra_aes_crypt(req, true); 511 } 512 513 static int tegra_aes_decrypt(struct skcipher_request *req) 514 { 515 return tegra_aes_crypt(req, false); 516 } 517 518 static struct tegra_se_alg tegra_aes_algs[] = { 519 { 520 .alg.skcipher.op.do_one_request = tegra_aes_do_one_req, 521 .alg.skcipher.base = { 522 .init = tegra_aes_cra_init, 523 .exit = tegra_aes_cra_exit, 524 .setkey = tegra_aes_setkey, 525 .encrypt = tegra_aes_encrypt, 526 .decrypt = tegra_aes_decrypt, 527 .min_keysize = AES_MIN_KEY_SIZE, 528 .max_keysize = AES_MAX_KEY_SIZE, 529 .ivsize = AES_BLOCK_SIZE, 530 .base = { 531 .cra_name = "cbc(aes)", 532 .cra_driver_name = "cbc-aes-tegra", 533 .cra_priority = 500, 534 .cra_flags = CRYPTO_ALG_ASYNC, 535 .cra_blocksize = AES_BLOCK_SIZE, 536 .cra_ctxsize = sizeof(struct tegra_aes_ctx), 537 .cra_alignmask = 0xf, 538 .cra_module = THIS_MODULE, 539 }, 540 } 541 }, { 542 .alg.skcipher.op.do_one_request = tegra_aes_do_one_req, 543 .alg.skcipher.base = { 544 .init = tegra_aes_cra_init, 545 .exit = tegra_aes_cra_exit, 546 .setkey = tegra_aes_setkey, 547 .encrypt = tegra_aes_encrypt, 548 .decrypt = tegra_aes_decrypt, 549 .min_keysize = AES_MIN_KEY_SIZE, 550 .max_keysize = AES_MAX_KEY_SIZE, 551 .base = { 552 .cra_name = "ecb(aes)", 553 .cra_driver_name = "ecb-aes-tegra", 554 .cra_priority = 500, 555 .cra_flags = CRYPTO_ALG_ASYNC, 556 .cra_blocksize = AES_BLOCK_SIZE, 557 .cra_ctxsize = sizeof(struct tegra_aes_ctx), 558 .cra_alignmask = 0xf, 559 .cra_module = THIS_MODULE, 560 }, 561 } 562 }, { 563 .alg.skcipher.op.do_one_request = tegra_aes_do_one_req, 564 .alg.skcipher.base = { 565 .init = tegra_aes_cra_init, 566 .exit = tegra_aes_cra_exit, 567 .setkey = tegra_aes_setkey, 568 .encrypt = tegra_aes_encrypt, 569 .decrypt = tegra_aes_decrypt, 570 .min_keysize = AES_MIN_KEY_SIZE, 571 .max_keysize = AES_MAX_KEY_SIZE, 572 .ivsize = AES_BLOCK_SIZE, 573 .base = { 574 .cra_name = "ctr(aes)", 575 .cra_driver_name = "ctr-aes-tegra", 576 .cra_priority = 500, 577 .cra_flags = CRYPTO_ALG_ASYNC, 578 .cra_blocksize = 1, 579 .cra_ctxsize = sizeof(struct tegra_aes_ctx), 580 .cra_alignmask = 0xf, 581 .cra_module = THIS_MODULE, 582 }, 583 } 584 }, { 585 .alg.skcipher.op.do_one_request = tegra_aes_do_one_req, 586 .alg.skcipher.base = { 587 .init = tegra_aes_cra_init, 588 .exit = tegra_aes_cra_exit, 589 .setkey = tegra_xts_setkey, 590 .encrypt = tegra_aes_encrypt, 591 .decrypt = tegra_aes_decrypt, 592 .min_keysize = 2 * AES_MIN_KEY_SIZE, 593 .max_keysize = 2 * AES_MAX_KEY_SIZE, 594 .ivsize = AES_BLOCK_SIZE, 595 .base = { 596 .cra_name = "xts(aes)", 597 .cra_driver_name = "xts-aes-tegra", 598 .cra_priority = 500, 599 .cra_flags = CRYPTO_ALG_ASYNC, 600 .cra_blocksize = AES_BLOCK_SIZE, 601 .cra_ctxsize = sizeof(struct tegra_aes_ctx), 602 .cra_alignmask = (__alignof__(u64) - 1), 603 .cra_module = THIS_MODULE, 604 }, 605 } 606 }, 607 }; 608 609 static unsigned int tegra_gmac_prep_cmd(struct tegra_aead_ctx *ctx, 610 struct tegra_aead_reqctx *rctx) 611 { 612 unsigned int data_count, res_bits, i = 0; 613 struct tegra_se *se = ctx->se; 614 u32 *cpuvaddr = se->cmdbuf->addr; 615 616 data_count = (rctx->assoclen / AES_BLOCK_SIZE); 617 res_bits = (rctx->assoclen % AES_BLOCK_SIZE) * 8; 618 619 /* 620 * Hardware processes data_count + 1 blocks. 621 * Reduce 1 block if there is no residue 622 */ 623 if (!res_bits) 624 data_count--; 625 626 cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->last_blk, 1); 627 cpuvaddr[i++] = SE_LAST_BLOCK_VAL(data_count) | 628 SE_LAST_BLOCK_RES_BITS(res_bits); 629 630 cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 4); 631 cpuvaddr[i++] = rctx->config; 632 cpuvaddr[i++] = rctx->crypto_config; 633 cpuvaddr[i++] = lower_32_bits(rctx->inbuf.addr); 634 cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->inbuf.addr)) | 635 SE_ADDR_HI_SZ(rctx->assoclen); 636 637 cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1); 638 cpuvaddr[i++] = SE_AES_OP_WRSTALL | SE_AES_OP_FINAL | 639 SE_AES_OP_INIT | SE_AES_OP_LASTBUF | 640 SE_AES_OP_START; 641 642 cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1); 643 cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) | 644 host1x_uclass_incr_syncpt_indx_f(se->syncpt_id); 645 646 return i; 647 } 648 649 static unsigned int tegra_gcm_crypt_prep_cmd(struct tegra_aead_ctx *ctx, 650 struct tegra_aead_reqctx *rctx) 651 { 652 unsigned int data_count, res_bits, i = 0, j; 653 struct tegra_se *se = ctx->se; 654 u32 *cpuvaddr = se->cmdbuf->addr, op; 655 656 data_count = (rctx->cryptlen / AES_BLOCK_SIZE); 657 res_bits = (rctx->cryptlen % AES_BLOCK_SIZE) * 8; 658 op = SE_AES_OP_WRSTALL | SE_AES_OP_FINAL | 659 SE_AES_OP_LASTBUF | SE_AES_OP_START; 660 661 /* 662 * If there is no assoc data, 663 * this will be the init command 664 */ 665 if (!rctx->assoclen) 666 op |= SE_AES_OP_INIT; 667 668 /* 669 * Hardware processes data_count + 1 blocks. 670 * Reduce 1 block if there is no residue 671 */ 672 if (!res_bits) 673 data_count--; 674 675 cpuvaddr[i++] = host1x_opcode_setpayload(SE_CRYPTO_CTR_REG_COUNT); 676 cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->linear_ctr); 677 for (j = 0; j < SE_CRYPTO_CTR_REG_COUNT; j++) 678 cpuvaddr[i++] = rctx->iv[j]; 679 680 cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->last_blk, 1); 681 cpuvaddr[i++] = SE_LAST_BLOCK_VAL(data_count) | 682 SE_LAST_BLOCK_RES_BITS(res_bits); 683 684 cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 6); 685 cpuvaddr[i++] = rctx->config; 686 cpuvaddr[i++] = rctx->crypto_config; 687 688 /* Source Address */ 689 cpuvaddr[i++] = lower_32_bits(rctx->inbuf.addr); 690 cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->inbuf.addr)) | 691 SE_ADDR_HI_SZ(rctx->cryptlen); 692 693 /* Destination Address */ 694 cpuvaddr[i++] = lower_32_bits(rctx->outbuf.addr); 695 cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->outbuf.addr)) | 696 SE_ADDR_HI_SZ(rctx->cryptlen); 697 698 cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1); 699 cpuvaddr[i++] = op; 700 701 cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1); 702 cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) | 703 host1x_uclass_incr_syncpt_indx_f(se->syncpt_id); 704 705 dev_dbg(se->dev, "cfg %#x crypto cfg %#x\n", rctx->config, rctx->crypto_config); 706 return i; 707 } 708 709 static int tegra_gcm_prep_final_cmd(struct tegra_se *se, u32 *cpuvaddr, 710 struct tegra_aead_reqctx *rctx) 711 { 712 unsigned int i = 0, j; 713 u32 op; 714 715 op = SE_AES_OP_WRSTALL | SE_AES_OP_FINAL | 716 SE_AES_OP_LASTBUF | SE_AES_OP_START; 717 718 /* 719 * Set init for zero sized vector 720 */ 721 if (!rctx->assoclen && !rctx->cryptlen) 722 op |= SE_AES_OP_INIT; 723 724 cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->aad_len, 2); 725 cpuvaddr[i++] = rctx->assoclen * 8; 726 cpuvaddr[i++] = 0; 727 728 cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->cryp_msg_len, 2); 729 cpuvaddr[i++] = rctx->cryptlen * 8; 730 cpuvaddr[i++] = 0; 731 732 cpuvaddr[i++] = host1x_opcode_setpayload(SE_CRYPTO_CTR_REG_COUNT); 733 cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->linear_ctr); 734 for (j = 0; j < SE_CRYPTO_CTR_REG_COUNT; j++) 735 cpuvaddr[i++] = rctx->iv[j]; 736 737 cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 6); 738 cpuvaddr[i++] = rctx->config; 739 cpuvaddr[i++] = rctx->crypto_config; 740 cpuvaddr[i++] = 0; 741 cpuvaddr[i++] = 0; 742 743 /* Destination Address */ 744 cpuvaddr[i++] = lower_32_bits(rctx->outbuf.addr); 745 cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->outbuf.addr)) | 746 SE_ADDR_HI_SZ(0x10); /* HW always generates 128-bit tag */ 747 748 cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1); 749 cpuvaddr[i++] = op; 750 751 cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1); 752 cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) | 753 host1x_uclass_incr_syncpt_indx_f(se->syncpt_id); 754 755 dev_dbg(se->dev, "cfg %#x crypto cfg %#x\n", rctx->config, rctx->crypto_config); 756 757 return i; 758 } 759 760 static int tegra_gcm_do_gmac(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx) 761 { 762 struct tegra_se *se = ctx->se; 763 unsigned int cmdlen; 764 765 scatterwalk_map_and_copy(rctx->inbuf.buf, 766 rctx->src_sg, 0, rctx->assoclen, 0); 767 768 rctx->config = tegra234_aes_cfg(SE_ALG_GMAC, rctx->encrypt); 769 rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_GMAC, rctx->encrypt) | 770 SE_AES_KEY_INDEX(rctx->key_id); 771 772 cmdlen = tegra_gmac_prep_cmd(ctx, rctx); 773 774 return tegra_se_host1x_submit(se, se->cmdbuf, cmdlen); 775 } 776 777 static int tegra_gcm_do_crypt(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx) 778 { 779 struct tegra_se *se = ctx->se; 780 int cmdlen, ret; 781 782 scatterwalk_map_and_copy(rctx->inbuf.buf, rctx->src_sg, 783 rctx->assoclen, rctx->cryptlen, 0); 784 785 rctx->config = tegra234_aes_cfg(SE_ALG_GCM, rctx->encrypt); 786 rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_GCM, rctx->encrypt) | 787 SE_AES_KEY_INDEX(rctx->key_id); 788 789 /* Prepare command and submit */ 790 cmdlen = tegra_gcm_crypt_prep_cmd(ctx, rctx); 791 ret = tegra_se_host1x_submit(se, se->cmdbuf, cmdlen); 792 if (ret) 793 return ret; 794 795 /* Copy the result */ 796 scatterwalk_map_and_copy(rctx->outbuf.buf, rctx->dst_sg, 797 rctx->assoclen, rctx->cryptlen, 1); 798 799 return 0; 800 } 801 802 static int tegra_gcm_do_final(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx) 803 { 804 struct tegra_se *se = ctx->se; 805 u32 *cpuvaddr = se->cmdbuf->addr; 806 int cmdlen, ret, offset; 807 808 rctx->config = tegra234_aes_cfg(SE_ALG_GCM_FINAL, rctx->encrypt); 809 rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_GCM_FINAL, rctx->encrypt) | 810 SE_AES_KEY_INDEX(rctx->key_id); 811 812 /* Prepare command and submit */ 813 cmdlen = tegra_gcm_prep_final_cmd(se, cpuvaddr, rctx); 814 ret = tegra_se_host1x_submit(se, se->cmdbuf, cmdlen); 815 if (ret) 816 return ret; 817 818 if (rctx->encrypt) { 819 /* Copy the result */ 820 offset = rctx->assoclen + rctx->cryptlen; 821 scatterwalk_map_and_copy(rctx->outbuf.buf, rctx->dst_sg, 822 offset, rctx->authsize, 1); 823 } 824 825 return 0; 826 } 827 828 static int tegra_gcm_do_verify(struct tegra_se *se, struct tegra_aead_reqctx *rctx) 829 { 830 unsigned int offset; 831 u8 mac[16]; 832 833 offset = rctx->assoclen + rctx->cryptlen; 834 scatterwalk_map_and_copy(mac, rctx->src_sg, offset, rctx->authsize, 0); 835 836 if (crypto_memneq(rctx->outbuf.buf, mac, rctx->authsize)) 837 return -EBADMSG; 838 839 return 0; 840 } 841 842 static inline int tegra_ccm_check_iv(const u8 *iv) 843 { 844 /* iv[0] gives value of q-1 845 * 2 <= q <= 8 as per NIST 800-38C notation 846 * 2 <= L <= 8, so 1 <= L' <= 7. as per rfc 3610 notation 847 */ 848 if (iv[0] < 1 || iv[0] > 7) { 849 pr_debug("ccm_check_iv failed %d\n", iv[0]); 850 return -EINVAL; 851 } 852 853 return 0; 854 } 855 856 static unsigned int tegra_cbcmac_prep_cmd(struct tegra_aead_ctx *ctx, 857 struct tegra_aead_reqctx *rctx) 858 { 859 unsigned int data_count, i = 0; 860 struct tegra_se *se = ctx->se; 861 u32 *cpuvaddr = se->cmdbuf->addr; 862 863 data_count = (rctx->inbuf.size / AES_BLOCK_SIZE) - 1; 864 865 cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->last_blk, 1); 866 cpuvaddr[i++] = SE_LAST_BLOCK_VAL(data_count); 867 868 cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 6); 869 cpuvaddr[i++] = rctx->config; 870 cpuvaddr[i++] = rctx->crypto_config; 871 872 cpuvaddr[i++] = lower_32_bits(rctx->inbuf.addr); 873 cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->inbuf.addr)) | 874 SE_ADDR_HI_SZ(rctx->inbuf.size); 875 876 cpuvaddr[i++] = lower_32_bits(rctx->outbuf.addr); 877 cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->outbuf.addr)) | 878 SE_ADDR_HI_SZ(0x10); /* HW always generates 128 bit tag */ 879 880 cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1); 881 cpuvaddr[i++] = SE_AES_OP_WRSTALL | 882 SE_AES_OP_LASTBUF | SE_AES_OP_START; 883 884 cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1); 885 cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) | 886 host1x_uclass_incr_syncpt_indx_f(se->syncpt_id); 887 888 return i; 889 } 890 891 static unsigned int tegra_ctr_prep_cmd(struct tegra_aead_ctx *ctx, 892 struct tegra_aead_reqctx *rctx) 893 { 894 unsigned int i = 0, j; 895 struct tegra_se *se = ctx->se; 896 u32 *cpuvaddr = se->cmdbuf->addr; 897 898 cpuvaddr[i++] = host1x_opcode_setpayload(SE_CRYPTO_CTR_REG_COUNT); 899 cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->linear_ctr); 900 for (j = 0; j < SE_CRYPTO_CTR_REG_COUNT; j++) 901 cpuvaddr[i++] = rctx->iv[j]; 902 903 cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->last_blk, 1); 904 cpuvaddr[i++] = (rctx->inbuf.size / AES_BLOCK_SIZE) - 1; 905 cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 6); 906 cpuvaddr[i++] = rctx->config; 907 cpuvaddr[i++] = rctx->crypto_config; 908 909 /* Source address setting */ 910 cpuvaddr[i++] = lower_32_bits(rctx->inbuf.addr); 911 cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->inbuf.addr)) | 912 SE_ADDR_HI_SZ(rctx->inbuf.size); 913 914 /* Destination address setting */ 915 cpuvaddr[i++] = lower_32_bits(rctx->outbuf.addr); 916 cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->outbuf.addr)) | 917 SE_ADDR_HI_SZ(rctx->inbuf.size); 918 919 cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1); 920 cpuvaddr[i++] = SE_AES_OP_WRSTALL | SE_AES_OP_LASTBUF | 921 SE_AES_OP_START; 922 923 cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1); 924 cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) | 925 host1x_uclass_incr_syncpt_indx_f(se->syncpt_id); 926 927 dev_dbg(se->dev, "cfg %#x crypto cfg %#x\n", 928 rctx->config, rctx->crypto_config); 929 930 return i; 931 } 932 933 static int tegra_ccm_do_cbcmac(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx) 934 { 935 struct tegra_se *se = ctx->se; 936 int cmdlen; 937 938 rctx->config = tegra234_aes_cfg(SE_ALG_CBC_MAC, rctx->encrypt); 939 rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_CBC_MAC, 940 rctx->encrypt) | 941 SE_AES_KEY_INDEX(rctx->key_id); 942 943 /* Prepare command and submit */ 944 cmdlen = tegra_cbcmac_prep_cmd(ctx, rctx); 945 946 return tegra_se_host1x_submit(se, se->cmdbuf, cmdlen); 947 } 948 949 static int tegra_ccm_set_msg_len(u8 *block, unsigned int msglen, int csize) 950 { 951 __be32 data; 952 953 memset(block, 0, csize); 954 block += csize; 955 956 if (csize >= 4) 957 csize = 4; 958 else if (msglen > (1 << (8 * csize))) 959 return -EOVERFLOW; 960 961 data = cpu_to_be32(msglen); 962 memcpy(block - csize, (u8 *)&data + 4 - csize, csize); 963 964 return 0; 965 } 966 967 static int tegra_ccm_format_nonce(struct tegra_aead_reqctx *rctx, u8 *nonce) 968 { 969 unsigned int q, t; 970 u8 *q_ptr, *iv = (u8 *)rctx->iv; 971 972 memcpy(nonce, rctx->iv, 16); 973 974 /*** 1. Prepare Flags Octet ***/ 975 976 /* Encode t (mac length) */ 977 t = rctx->authsize; 978 nonce[0] |= (((t - 2) / 2) << 3); 979 980 /* Adata */ 981 if (rctx->assoclen) 982 nonce[0] |= (1 << 6); 983 984 /*** Encode Q - message length ***/ 985 q = iv[0] + 1; 986 q_ptr = nonce + 16 - q; 987 988 return tegra_ccm_set_msg_len(q_ptr, rctx->cryptlen, q); 989 } 990 991 static int tegra_ccm_format_adata(u8 *adata, unsigned int a) 992 { 993 int len = 0; 994 995 /* add control info for associated data 996 * RFC 3610 and NIST Special Publication 800-38C 997 */ 998 if (a < 65280) { 999 *(__be16 *)adata = cpu_to_be16(a); 1000 len = 2; 1001 } else { 1002 *(__be16 *)adata = cpu_to_be16(0xfffe); 1003 *(__be32 *)&adata[2] = cpu_to_be32(a); 1004 len = 6; 1005 } 1006 1007 return len; 1008 } 1009 1010 static int tegra_ccm_add_padding(u8 *buf, unsigned int len) 1011 { 1012 unsigned int padlen = 16 - (len % 16); 1013 u8 padding[16] = {0}; 1014 1015 if (padlen == 16) 1016 return 0; 1017 1018 memcpy(buf, padding, padlen); 1019 1020 return padlen; 1021 } 1022 1023 static int tegra_ccm_format_blocks(struct tegra_aead_reqctx *rctx) 1024 { 1025 unsigned int alen = 0, offset = 0; 1026 u8 nonce[16], adata[16]; 1027 int ret; 1028 1029 ret = tegra_ccm_format_nonce(rctx, nonce); 1030 if (ret) 1031 return ret; 1032 1033 memcpy(rctx->inbuf.buf, nonce, 16); 1034 offset = 16; 1035 1036 if (rctx->assoclen) { 1037 alen = tegra_ccm_format_adata(adata, rctx->assoclen); 1038 memcpy(rctx->inbuf.buf + offset, adata, alen); 1039 offset += alen; 1040 1041 scatterwalk_map_and_copy(rctx->inbuf.buf + offset, 1042 rctx->src_sg, 0, rctx->assoclen, 0); 1043 1044 offset += rctx->assoclen; 1045 offset += tegra_ccm_add_padding(rctx->inbuf.buf + offset, 1046 rctx->assoclen + alen); 1047 } 1048 1049 return offset; 1050 } 1051 1052 static int tegra_ccm_mac_result(struct tegra_se *se, struct tegra_aead_reqctx *rctx) 1053 { 1054 u32 result[16]; 1055 int i, ret; 1056 1057 /* Read and clear Result */ 1058 for (i = 0; i < CMAC_RESULT_REG_COUNT; i++) 1059 result[i] = readl(se->base + se->hw->regs->result + (i * 4)); 1060 1061 for (i = 0; i < CMAC_RESULT_REG_COUNT; i++) 1062 writel(0, se->base + se->hw->regs->result + (i * 4)); 1063 1064 if (rctx->encrypt) { 1065 memcpy(rctx->authdata, result, rctx->authsize); 1066 } else { 1067 ret = crypto_memneq(rctx->authdata, result, rctx->authsize); 1068 if (ret) 1069 return -EBADMSG; 1070 } 1071 1072 return 0; 1073 } 1074 1075 static int tegra_ccm_ctr_result(struct tegra_se *se, struct tegra_aead_reqctx *rctx) 1076 { 1077 /* Copy result */ 1078 scatterwalk_map_and_copy(rctx->outbuf.buf + 16, rctx->dst_sg, 1079 rctx->assoclen, rctx->cryptlen, 1); 1080 1081 if (rctx->encrypt) 1082 scatterwalk_map_and_copy(rctx->outbuf.buf, rctx->dst_sg, 1083 rctx->assoclen + rctx->cryptlen, 1084 rctx->authsize, 1); 1085 else 1086 memcpy(rctx->authdata, rctx->outbuf.buf, rctx->authsize); 1087 1088 return 0; 1089 } 1090 1091 static int tegra_ccm_compute_auth(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx) 1092 { 1093 struct tegra_se *se = ctx->se; 1094 struct scatterlist *sg; 1095 int offset, ret; 1096 1097 offset = tegra_ccm_format_blocks(rctx); 1098 if (offset < 0) 1099 return -EINVAL; 1100 1101 /* Copy plain text to the buffer */ 1102 sg = rctx->encrypt ? rctx->src_sg : rctx->dst_sg; 1103 1104 scatterwalk_map_and_copy(rctx->inbuf.buf + offset, 1105 sg, rctx->assoclen, 1106 rctx->cryptlen, 0); 1107 offset += rctx->cryptlen; 1108 offset += tegra_ccm_add_padding(rctx->inbuf.buf + offset, rctx->cryptlen); 1109 1110 rctx->inbuf.size = offset; 1111 1112 ret = tegra_ccm_do_cbcmac(ctx, rctx); 1113 if (ret) 1114 return ret; 1115 1116 return tegra_ccm_mac_result(se, rctx); 1117 } 1118 1119 static int tegra_ccm_do_ctr(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx) 1120 { 1121 struct tegra_se *se = ctx->se; 1122 unsigned int cmdlen, offset = 0; 1123 struct scatterlist *sg = rctx->src_sg; 1124 int ret; 1125 1126 rctx->config = tegra234_aes_cfg(SE_ALG_CTR, rctx->encrypt); 1127 rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_CTR, rctx->encrypt) | 1128 SE_AES_KEY_INDEX(rctx->key_id); 1129 1130 /* Copy authdata in the top of buffer for encryption/decryption */ 1131 if (rctx->encrypt) 1132 memcpy(rctx->inbuf.buf, rctx->authdata, rctx->authsize); 1133 else 1134 scatterwalk_map_and_copy(rctx->inbuf.buf, sg, 1135 rctx->assoclen + rctx->cryptlen, 1136 rctx->authsize, 0); 1137 1138 offset += rctx->authsize; 1139 offset += tegra_ccm_add_padding(rctx->inbuf.buf + offset, rctx->authsize); 1140 1141 /* If there is no cryptlen, proceed to submit the task */ 1142 if (rctx->cryptlen) { 1143 scatterwalk_map_and_copy(rctx->inbuf.buf + offset, sg, 1144 rctx->assoclen, rctx->cryptlen, 0); 1145 offset += rctx->cryptlen; 1146 offset += tegra_ccm_add_padding(rctx->inbuf.buf + offset, rctx->cryptlen); 1147 } 1148 1149 rctx->inbuf.size = offset; 1150 1151 /* Prepare command and submit */ 1152 cmdlen = tegra_ctr_prep_cmd(ctx, rctx); 1153 ret = tegra_se_host1x_submit(se, se->cmdbuf, cmdlen); 1154 if (ret) 1155 return ret; 1156 1157 return tegra_ccm_ctr_result(se, rctx); 1158 } 1159 1160 static int tegra_ccm_crypt_init(struct aead_request *req, struct tegra_se *se, 1161 struct tegra_aead_reqctx *rctx) 1162 { 1163 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1164 u8 *iv = (u8 *)rctx->iv; 1165 int ret, i; 1166 1167 rctx->src_sg = req->src; 1168 rctx->dst_sg = req->dst; 1169 rctx->assoclen = req->assoclen; 1170 rctx->authsize = crypto_aead_authsize(tfm); 1171 1172 if (rctx->encrypt) 1173 rctx->cryptlen = req->cryptlen; 1174 else 1175 rctx->cryptlen = req->cryptlen - rctx->authsize; 1176 1177 memcpy(iv, req->iv, 16); 1178 1179 ret = tegra_ccm_check_iv(iv); 1180 if (ret) 1181 return ret; 1182 1183 /* Note: rfc 3610 and NIST 800-38C require counter (ctr_0) of 1184 * zero to encrypt auth tag. 1185 * req->iv has the formatted ctr_0 (i.e. Flags || N || 0). 1186 */ 1187 memset(iv + 15 - iv[0], 0, iv[0] + 1); 1188 1189 /* Clear any previous result */ 1190 for (i = 0; i < CMAC_RESULT_REG_COUNT; i++) 1191 writel(0, se->base + se->hw->regs->result + (i * 4)); 1192 1193 return 0; 1194 } 1195 1196 static int tegra_ccm_do_one_req(struct crypto_engine *engine, void *areq) 1197 { 1198 struct aead_request *req = container_of(areq, struct aead_request, base); 1199 struct tegra_aead_reqctx *rctx = aead_request_ctx(req); 1200 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1201 struct tegra_aead_ctx *ctx = crypto_aead_ctx(tfm); 1202 struct tegra_se *se = ctx->se; 1203 unsigned int bufsize; 1204 int ret; 1205 1206 ret = tegra_ccm_crypt_init(req, se, rctx); 1207 if (ret) 1208 goto out_finalize; 1209 1210 rctx->key_id = ctx->key_id; 1211 1212 /* Allocate buffers required */ 1213 bufsize = rctx->assoclen + rctx->authsize + rctx->cryptlen + 100; 1214 rctx->inbuf.size = bufsize; 1215 rctx->inbuf.buf = dma_alloc_coherent(ctx->se->dev, bufsize, 1216 &rctx->inbuf.addr, GFP_KERNEL); 1217 ret = -ENOMEM; 1218 if (!rctx->inbuf.buf) 1219 goto out_finalize; 1220 1221 rctx->outbuf.size = bufsize; 1222 rctx->outbuf.buf = dma_alloc_coherent(ctx->se->dev, bufsize, 1223 &rctx->outbuf.addr, GFP_KERNEL); 1224 if (!rctx->outbuf.buf) 1225 goto out_free_inbuf; 1226 1227 if (!ctx->key_id) { 1228 ret = tegra_key_submit_reserved_aes(ctx->se, ctx->key, 1229 ctx->keylen, ctx->alg, &rctx->key_id); 1230 if (ret) 1231 goto out; 1232 } 1233 1234 if (rctx->encrypt) { 1235 /* CBC MAC Operation */ 1236 ret = tegra_ccm_compute_auth(ctx, rctx); 1237 if (ret) 1238 goto out; 1239 1240 /* CTR operation */ 1241 ret = tegra_ccm_do_ctr(ctx, rctx); 1242 if (ret) 1243 goto out; 1244 } else { 1245 /* CTR operation */ 1246 ret = tegra_ccm_do_ctr(ctx, rctx); 1247 if (ret) 1248 goto out; 1249 1250 /* CBC MAC Operation */ 1251 ret = tegra_ccm_compute_auth(ctx, rctx); 1252 if (ret) 1253 goto out; 1254 } 1255 1256 out: 1257 dma_free_coherent(ctx->se->dev, bufsize, 1258 rctx->outbuf.buf, rctx->outbuf.addr); 1259 1260 out_free_inbuf: 1261 dma_free_coherent(ctx->se->dev, bufsize, 1262 rctx->inbuf.buf, rctx->inbuf.addr); 1263 1264 if (tegra_key_is_reserved(rctx->key_id)) 1265 tegra_key_invalidate_reserved(ctx->se, rctx->key_id, ctx->alg); 1266 1267 out_finalize: 1268 local_bh_disable(); 1269 crypto_finalize_aead_request(ctx->se->engine, req, ret); 1270 local_bh_enable(); 1271 1272 return 0; 1273 } 1274 1275 static int tegra_gcm_do_one_req(struct crypto_engine *engine, void *areq) 1276 { 1277 struct aead_request *req = container_of(areq, struct aead_request, base); 1278 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1279 struct tegra_aead_ctx *ctx = crypto_aead_ctx(tfm); 1280 struct tegra_aead_reqctx *rctx = aead_request_ctx(req); 1281 unsigned int bufsize; 1282 int ret; 1283 1284 rctx->src_sg = req->src; 1285 rctx->dst_sg = req->dst; 1286 rctx->assoclen = req->assoclen; 1287 rctx->authsize = crypto_aead_authsize(tfm); 1288 1289 if (rctx->encrypt) 1290 rctx->cryptlen = req->cryptlen; 1291 else 1292 rctx->cryptlen = req->cryptlen - rctx->authsize; 1293 1294 memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE); 1295 rctx->iv[3] = (1 << 24); 1296 1297 rctx->key_id = ctx->key_id; 1298 1299 /* Allocate buffers required */ 1300 bufsize = rctx->assoclen + rctx->authsize + rctx->cryptlen; 1301 rctx->inbuf.size = bufsize; 1302 rctx->inbuf.buf = dma_alloc_coherent(ctx->se->dev, bufsize, 1303 &rctx->inbuf.addr, GFP_KERNEL); 1304 if (!rctx->inbuf.buf) { 1305 ret = -ENOMEM; 1306 goto out_finalize; 1307 } 1308 1309 rctx->outbuf.size = bufsize; 1310 rctx->outbuf.buf = dma_alloc_coherent(ctx->se->dev, bufsize, 1311 &rctx->outbuf.addr, GFP_KERNEL); 1312 if (!rctx->outbuf.buf) { 1313 ret = -ENOMEM; 1314 goto out_free_inbuf; 1315 } 1316 1317 if (!ctx->key_id) { 1318 ret = tegra_key_submit_reserved_aes(ctx->se, ctx->key, 1319 ctx->keylen, ctx->alg, &rctx->key_id); 1320 if (ret) 1321 goto out; 1322 } 1323 1324 /* If there is associated data perform GMAC operation */ 1325 if (rctx->assoclen) { 1326 ret = tegra_gcm_do_gmac(ctx, rctx); 1327 if (ret) 1328 goto out; 1329 } 1330 1331 /* GCM Encryption/Decryption operation */ 1332 if (rctx->cryptlen) { 1333 ret = tegra_gcm_do_crypt(ctx, rctx); 1334 if (ret) 1335 goto out; 1336 } 1337 1338 /* GCM_FINAL operation */ 1339 ret = tegra_gcm_do_final(ctx, rctx); 1340 if (ret) 1341 goto out; 1342 1343 if (!rctx->encrypt) 1344 ret = tegra_gcm_do_verify(ctx->se, rctx); 1345 1346 out: 1347 dma_free_coherent(ctx->se->dev, bufsize, 1348 rctx->outbuf.buf, rctx->outbuf.addr); 1349 1350 out_free_inbuf: 1351 dma_free_coherent(ctx->se->dev, bufsize, 1352 rctx->inbuf.buf, rctx->inbuf.addr); 1353 1354 if (tegra_key_is_reserved(rctx->key_id)) 1355 tegra_key_invalidate_reserved(ctx->se, rctx->key_id, ctx->alg); 1356 1357 out_finalize: 1358 local_bh_disable(); 1359 crypto_finalize_aead_request(ctx->se->engine, req, ret); 1360 local_bh_enable(); 1361 1362 return 0; 1363 } 1364 1365 static int tegra_aead_cra_init(struct crypto_aead *tfm) 1366 { 1367 struct tegra_aead_ctx *ctx = crypto_aead_ctx(tfm); 1368 struct aead_alg *alg = crypto_aead_alg(tfm); 1369 struct tegra_se_alg *se_alg; 1370 const char *algname; 1371 int ret; 1372 1373 algname = crypto_tfm_alg_name(&tfm->base); 1374 1375 se_alg = container_of(alg, struct tegra_se_alg, alg.aead.base); 1376 1377 crypto_aead_set_reqsize(tfm, sizeof(struct tegra_aead_reqctx)); 1378 1379 ctx->se = se_alg->se_dev; 1380 ctx->key_id = 0; 1381 ctx->keylen = 0; 1382 1383 ret = se_algname_to_algid(algname); 1384 if (ret < 0) { 1385 dev_err(ctx->se->dev, "invalid algorithm\n"); 1386 return ret; 1387 } 1388 1389 ctx->alg = ret; 1390 1391 return 0; 1392 } 1393 1394 static int tegra_ccm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 1395 { 1396 switch (authsize) { 1397 case 4: 1398 case 6: 1399 case 8: 1400 case 10: 1401 case 12: 1402 case 14: 1403 case 16: 1404 return 0; 1405 default: 1406 return -EINVAL; 1407 } 1408 } 1409 1410 static int tegra_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 1411 { 1412 return crypto_gcm_check_authsize(authsize); 1413 } 1414 1415 static void tegra_aead_cra_exit(struct crypto_aead *tfm) 1416 { 1417 struct tegra_aead_ctx *ctx = crypto_tfm_ctx(&tfm->base); 1418 1419 if (ctx->key_id) 1420 tegra_key_invalidate(ctx->se, ctx->key_id, ctx->alg); 1421 } 1422 1423 static int tegra_aead_crypt(struct aead_request *req, bool encrypt) 1424 { 1425 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1426 struct tegra_aead_ctx *ctx = crypto_aead_ctx(tfm); 1427 struct tegra_aead_reqctx *rctx = aead_request_ctx(req); 1428 1429 rctx->encrypt = encrypt; 1430 1431 return crypto_transfer_aead_request_to_engine(ctx->se->engine, req); 1432 } 1433 1434 static int tegra_aead_encrypt(struct aead_request *req) 1435 { 1436 return tegra_aead_crypt(req, true); 1437 } 1438 1439 static int tegra_aead_decrypt(struct aead_request *req) 1440 { 1441 return tegra_aead_crypt(req, false); 1442 } 1443 1444 static int tegra_aead_setkey(struct crypto_aead *tfm, 1445 const u8 *key, u32 keylen) 1446 { 1447 struct tegra_aead_ctx *ctx = crypto_aead_ctx(tfm); 1448 int ret; 1449 1450 if (aes_check_keylen(keylen)) { 1451 dev_dbg(ctx->se->dev, "invalid key length (%d)\n", keylen); 1452 return -EINVAL; 1453 } 1454 1455 ret = tegra_key_submit(ctx->se, key, keylen, ctx->alg, &ctx->key_id); 1456 if (ret) { 1457 ctx->keylen = keylen; 1458 memcpy(ctx->key, key, keylen); 1459 } 1460 1461 return 0; 1462 } 1463 1464 static unsigned int tegra_cmac_prep_cmd(struct tegra_cmac_ctx *ctx, 1465 struct tegra_cmac_reqctx *rctx) 1466 { 1467 unsigned int data_count, res_bits = 0, i = 0, j; 1468 struct tegra_se *se = ctx->se; 1469 u32 *cpuvaddr = se->cmdbuf->addr, op; 1470 1471 data_count = (rctx->datbuf.size / AES_BLOCK_SIZE); 1472 1473 op = SE_AES_OP_WRSTALL | SE_AES_OP_START | SE_AES_OP_LASTBUF; 1474 1475 if (!(rctx->task & SHA_UPDATE)) { 1476 op |= SE_AES_OP_FINAL; 1477 res_bits = (rctx->datbuf.size % AES_BLOCK_SIZE) * 8; 1478 } 1479 1480 if (!res_bits && data_count) 1481 data_count--; 1482 1483 if (rctx->task & SHA_FIRST) { 1484 rctx->task &= ~SHA_FIRST; 1485 1486 cpuvaddr[i++] = host1x_opcode_setpayload(SE_CRYPTO_CTR_REG_COUNT); 1487 cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->linear_ctr); 1488 /* Load 0 IV */ 1489 for (j = 0; j < SE_CRYPTO_CTR_REG_COUNT; j++) 1490 cpuvaddr[i++] = 0; 1491 } 1492 1493 cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->last_blk, 1); 1494 cpuvaddr[i++] = SE_LAST_BLOCK_VAL(data_count) | 1495 SE_LAST_BLOCK_RES_BITS(res_bits); 1496 1497 cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 6); 1498 cpuvaddr[i++] = rctx->config; 1499 cpuvaddr[i++] = rctx->crypto_config; 1500 1501 /* Source Address */ 1502 cpuvaddr[i++] = lower_32_bits(rctx->datbuf.addr); 1503 cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->datbuf.addr)) | 1504 SE_ADDR_HI_SZ(rctx->datbuf.size); 1505 cpuvaddr[i++] = 0; 1506 cpuvaddr[i++] = SE_ADDR_HI_SZ(AES_BLOCK_SIZE); 1507 1508 cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1); 1509 cpuvaddr[i++] = op; 1510 1511 cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1); 1512 cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) | 1513 host1x_uclass_incr_syncpt_indx_f(se->syncpt_id); 1514 1515 return i; 1516 } 1517 1518 static void tegra_cmac_copy_result(struct tegra_se *se, struct tegra_cmac_reqctx *rctx) 1519 { 1520 int i; 1521 1522 for (i = 0; i < CMAC_RESULT_REG_COUNT; i++) 1523 rctx->result[i] = readl(se->base + se->hw->regs->result + (i * 4)); 1524 } 1525 1526 static void tegra_cmac_paste_result(struct tegra_se *se, struct tegra_cmac_reqctx *rctx) 1527 { 1528 int i; 1529 1530 for (i = 0; i < CMAC_RESULT_REG_COUNT; i++) 1531 writel(rctx->result[i], 1532 se->base + se->hw->regs->result + (i * 4)); 1533 } 1534 1535 static int tegra_cmac_do_init(struct ahash_request *req) 1536 { 1537 struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); 1538 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1539 struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); 1540 struct tegra_se *se = ctx->se; 1541 int i; 1542 1543 rctx->total_len = 0; 1544 rctx->datbuf.size = 0; 1545 rctx->residue.size = 0; 1546 rctx->key_id = ctx->key_id; 1547 rctx->task |= SHA_FIRST; 1548 rctx->blk_size = crypto_ahash_blocksize(tfm); 1549 1550 rctx->residue.buf = dma_alloc_coherent(se->dev, rctx->blk_size * 2, 1551 &rctx->residue.addr, GFP_KERNEL); 1552 if (!rctx->residue.buf) 1553 return -ENOMEM; 1554 1555 rctx->residue.size = 0; 1556 1557 /* Clear any previous result */ 1558 for (i = 0; i < CMAC_RESULT_REG_COUNT; i++) 1559 writel(0, se->base + se->hw->regs->result + (i * 4)); 1560 1561 return 0; 1562 } 1563 1564 static int tegra_cmac_do_update(struct ahash_request *req) 1565 { 1566 struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); 1567 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1568 struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); 1569 struct tegra_se *se = ctx->se; 1570 unsigned int nblks, nresidue, cmdlen; 1571 int ret; 1572 1573 if (!req->nbytes) 1574 return 0; 1575 1576 nresidue = (req->nbytes + rctx->residue.size) % rctx->blk_size; 1577 nblks = (req->nbytes + rctx->residue.size) / rctx->blk_size; 1578 1579 /* 1580 * Reserve the last block as residue during final() to process. 1581 */ 1582 if (!nresidue && nblks) { 1583 nresidue += rctx->blk_size; 1584 nblks--; 1585 } 1586 1587 rctx->src_sg = req->src; 1588 rctx->datbuf.size = (req->nbytes + rctx->residue.size) - nresidue; 1589 rctx->total_len += rctx->datbuf.size; 1590 rctx->config = tegra234_aes_cfg(SE_ALG_CMAC, 0); 1591 rctx->crypto_config = SE_AES_KEY_INDEX(rctx->key_id); 1592 1593 /* 1594 * Keep one block and residue bytes in residue and 1595 * return. The bytes will be processed in final() 1596 */ 1597 if (nblks < 1) { 1598 scatterwalk_map_and_copy(rctx->residue.buf + rctx->residue.size, 1599 rctx->src_sg, 0, req->nbytes, 0); 1600 1601 rctx->residue.size += req->nbytes; 1602 return 0; 1603 } 1604 1605 rctx->datbuf.buf = dma_alloc_coherent(se->dev, rctx->datbuf.size, 1606 &rctx->datbuf.addr, GFP_KERNEL); 1607 if (!rctx->datbuf.buf) 1608 return -ENOMEM; 1609 1610 /* Copy the previous residue first */ 1611 if (rctx->residue.size) 1612 memcpy(rctx->datbuf.buf, rctx->residue.buf, rctx->residue.size); 1613 1614 scatterwalk_map_and_copy(rctx->datbuf.buf + rctx->residue.size, 1615 rctx->src_sg, 0, req->nbytes - nresidue, 0); 1616 1617 scatterwalk_map_and_copy(rctx->residue.buf, rctx->src_sg, 1618 req->nbytes - nresidue, nresidue, 0); 1619 1620 /* Update residue value with the residue after current block */ 1621 rctx->residue.size = nresidue; 1622 1623 /* 1624 * If this is not the first task, paste the previous copied 1625 * intermediate results to the registers so that it gets picked up. 1626 */ 1627 if (!(rctx->task & SHA_FIRST)) 1628 tegra_cmac_paste_result(ctx->se, rctx); 1629 1630 cmdlen = tegra_cmac_prep_cmd(ctx, rctx); 1631 ret = tegra_se_host1x_submit(se, se->cmdbuf, cmdlen); 1632 1633 tegra_cmac_copy_result(ctx->se, rctx); 1634 1635 dma_free_coherent(ctx->se->dev, rctx->datbuf.size, 1636 rctx->datbuf.buf, rctx->datbuf.addr); 1637 1638 return ret; 1639 } 1640 1641 static int tegra_cmac_do_final(struct ahash_request *req) 1642 { 1643 struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); 1644 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1645 struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); 1646 struct tegra_se *se = ctx->se; 1647 u32 *result = (u32 *)req->result; 1648 int ret = 0, i, cmdlen; 1649 1650 if (!req->nbytes && !rctx->total_len && ctx->fallback_tfm) { 1651 return crypto_shash_tfm_digest(ctx->fallback_tfm, 1652 NULL, 0, req->result); 1653 } 1654 1655 if (rctx->residue.size) { 1656 rctx->datbuf.buf = dma_alloc_coherent(se->dev, rctx->residue.size, 1657 &rctx->datbuf.addr, GFP_KERNEL); 1658 if (!rctx->datbuf.buf) { 1659 ret = -ENOMEM; 1660 goto out_free; 1661 } 1662 1663 memcpy(rctx->datbuf.buf, rctx->residue.buf, rctx->residue.size); 1664 } 1665 1666 rctx->datbuf.size = rctx->residue.size; 1667 rctx->total_len += rctx->residue.size; 1668 rctx->config = tegra234_aes_cfg(SE_ALG_CMAC, 0); 1669 1670 /* 1671 * If this is not the first task, paste the previous copied 1672 * intermediate results to the registers so that it gets picked up. 1673 */ 1674 if (!(rctx->task & SHA_FIRST)) 1675 tegra_cmac_paste_result(ctx->se, rctx); 1676 1677 /* Prepare command and submit */ 1678 cmdlen = tegra_cmac_prep_cmd(ctx, rctx); 1679 ret = tegra_se_host1x_submit(se, se->cmdbuf, cmdlen); 1680 if (ret) 1681 goto out; 1682 1683 /* Read and clear Result register */ 1684 for (i = 0; i < CMAC_RESULT_REG_COUNT; i++) 1685 result[i] = readl(se->base + se->hw->regs->result + (i * 4)); 1686 1687 for (i = 0; i < CMAC_RESULT_REG_COUNT; i++) 1688 writel(0, se->base + se->hw->regs->result + (i * 4)); 1689 1690 out: 1691 if (rctx->residue.size) 1692 dma_free_coherent(se->dev, rctx->datbuf.size, 1693 rctx->datbuf.buf, rctx->datbuf.addr); 1694 out_free: 1695 dma_free_coherent(se->dev, crypto_ahash_blocksize(tfm) * 2, 1696 rctx->residue.buf, rctx->residue.addr); 1697 return ret; 1698 } 1699 1700 static int tegra_cmac_do_one_req(struct crypto_engine *engine, void *areq) 1701 { 1702 struct ahash_request *req = ahash_request_cast(areq); 1703 struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); 1704 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1705 struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); 1706 struct tegra_se *se = ctx->se; 1707 int ret = 0; 1708 1709 if (rctx->task & SHA_INIT) { 1710 ret = tegra_cmac_do_init(req); 1711 if (ret) 1712 goto out; 1713 1714 rctx->task &= ~SHA_INIT; 1715 } 1716 1717 if (!ctx->key_id) { 1718 ret = tegra_key_submit_reserved_aes(ctx->se, ctx->key, 1719 ctx->keylen, ctx->alg, &rctx->key_id); 1720 if (ret) 1721 goto out; 1722 } 1723 1724 if (rctx->task & SHA_UPDATE) { 1725 ret = tegra_cmac_do_update(req); 1726 if (ret) 1727 goto out; 1728 1729 rctx->task &= ~SHA_UPDATE; 1730 } 1731 1732 if (rctx->task & SHA_FINAL) { 1733 ret = tegra_cmac_do_final(req); 1734 if (ret) 1735 goto out; 1736 1737 rctx->task &= ~SHA_FINAL; 1738 } 1739 out: 1740 if (tegra_key_is_reserved(rctx->key_id)) 1741 tegra_key_invalidate_reserved(ctx->se, rctx->key_id, ctx->alg); 1742 1743 local_bh_disable(); 1744 crypto_finalize_hash_request(se->engine, req, ret); 1745 local_bh_enable(); 1746 1747 return 0; 1748 } 1749 1750 static void tegra_cmac_init_fallback(struct crypto_ahash *tfm, struct tegra_cmac_ctx *ctx, 1751 const char *algname) 1752 { 1753 unsigned int statesize; 1754 1755 ctx->fallback_tfm = crypto_alloc_shash(algname, 0, CRYPTO_ALG_NEED_FALLBACK); 1756 1757 if (IS_ERR(ctx->fallback_tfm)) { 1758 dev_warn(ctx->se->dev, "failed to allocate fallback for %s\n", algname); 1759 ctx->fallback_tfm = NULL; 1760 return; 1761 } 1762 1763 statesize = crypto_shash_statesize(ctx->fallback_tfm); 1764 1765 if (statesize > sizeof(struct tegra_cmac_reqctx)) 1766 crypto_ahash_set_statesize(tfm, statesize); 1767 } 1768 1769 static int tegra_cmac_cra_init(struct crypto_tfm *tfm) 1770 { 1771 struct tegra_cmac_ctx *ctx = crypto_tfm_ctx(tfm); 1772 struct crypto_ahash *ahash_tfm = __crypto_ahash_cast(tfm); 1773 struct ahash_alg *alg = __crypto_ahash_alg(tfm->__crt_alg); 1774 struct tegra_se_alg *se_alg; 1775 const char *algname; 1776 int ret; 1777 1778 algname = crypto_tfm_alg_name(tfm); 1779 se_alg = container_of(alg, struct tegra_se_alg, alg.ahash.base); 1780 1781 crypto_ahash_set_reqsize(ahash_tfm, sizeof(struct tegra_cmac_reqctx)); 1782 1783 ctx->se = se_alg->se_dev; 1784 ctx->key_id = 0; 1785 ctx->keylen = 0; 1786 1787 ret = se_algname_to_algid(algname); 1788 if (ret < 0) { 1789 dev_err(ctx->se->dev, "invalid algorithm\n"); 1790 return ret; 1791 } 1792 1793 ctx->alg = ret; 1794 1795 tegra_cmac_init_fallback(ahash_tfm, ctx, algname); 1796 1797 return 0; 1798 } 1799 1800 static void tegra_cmac_cra_exit(struct crypto_tfm *tfm) 1801 { 1802 struct tegra_cmac_ctx *ctx = crypto_tfm_ctx(tfm); 1803 1804 if (ctx->fallback_tfm) 1805 crypto_free_shash(ctx->fallback_tfm); 1806 1807 tegra_key_invalidate(ctx->se, ctx->key_id, ctx->alg); 1808 } 1809 1810 static int tegra_cmac_setkey(struct crypto_ahash *tfm, const u8 *key, 1811 unsigned int keylen) 1812 { 1813 struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); 1814 int ret; 1815 1816 if (aes_check_keylen(keylen)) { 1817 dev_dbg(ctx->se->dev, "invalid key length (%d)\n", keylen); 1818 return -EINVAL; 1819 } 1820 1821 if (ctx->fallback_tfm) 1822 crypto_shash_setkey(ctx->fallback_tfm, key, keylen); 1823 1824 ret = tegra_key_submit(ctx->se, key, keylen, ctx->alg, &ctx->key_id); 1825 if (ret) { 1826 ctx->keylen = keylen; 1827 memcpy(ctx->key, key, keylen); 1828 } 1829 1830 return 0; 1831 } 1832 1833 static int tegra_cmac_init(struct ahash_request *req) 1834 { 1835 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1836 struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); 1837 struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); 1838 1839 rctx->task = SHA_INIT; 1840 1841 return crypto_transfer_hash_request_to_engine(ctx->se->engine, req); 1842 } 1843 1844 static int tegra_cmac_update(struct ahash_request *req) 1845 { 1846 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1847 struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); 1848 struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); 1849 1850 rctx->task |= SHA_UPDATE; 1851 1852 return crypto_transfer_hash_request_to_engine(ctx->se->engine, req); 1853 } 1854 1855 static int tegra_cmac_final(struct ahash_request *req) 1856 { 1857 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1858 struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); 1859 struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); 1860 1861 rctx->task |= SHA_FINAL; 1862 1863 return crypto_transfer_hash_request_to_engine(ctx->se->engine, req); 1864 } 1865 1866 static int tegra_cmac_finup(struct ahash_request *req) 1867 { 1868 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1869 struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); 1870 struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); 1871 1872 rctx->task |= SHA_UPDATE | SHA_FINAL; 1873 1874 return crypto_transfer_hash_request_to_engine(ctx->se->engine, req); 1875 } 1876 1877 static int tegra_cmac_digest(struct ahash_request *req) 1878 { 1879 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1880 struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); 1881 struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); 1882 1883 rctx->task |= SHA_INIT | SHA_UPDATE | SHA_FINAL; 1884 1885 return crypto_transfer_hash_request_to_engine(ctx->se->engine, req); 1886 } 1887 1888 static int tegra_cmac_export(struct ahash_request *req, void *out) 1889 { 1890 struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); 1891 1892 memcpy(out, rctx, sizeof(*rctx)); 1893 1894 return 0; 1895 } 1896 1897 static int tegra_cmac_import(struct ahash_request *req, const void *in) 1898 { 1899 struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); 1900 1901 memcpy(rctx, in, sizeof(*rctx)); 1902 1903 return 0; 1904 } 1905 1906 static struct tegra_se_alg tegra_aead_algs[] = { 1907 { 1908 .alg.aead.op.do_one_request = tegra_gcm_do_one_req, 1909 .alg.aead.base = { 1910 .init = tegra_aead_cra_init, 1911 .exit = tegra_aead_cra_exit, 1912 .setkey = tegra_aead_setkey, 1913 .setauthsize = tegra_gcm_setauthsize, 1914 .encrypt = tegra_aead_encrypt, 1915 .decrypt = tegra_aead_decrypt, 1916 .maxauthsize = AES_BLOCK_SIZE, 1917 .ivsize = GCM_AES_IV_SIZE, 1918 .base = { 1919 .cra_name = "gcm(aes)", 1920 .cra_driver_name = "gcm-aes-tegra", 1921 .cra_priority = 500, 1922 .cra_flags = CRYPTO_ALG_ASYNC, 1923 .cra_blocksize = 1, 1924 .cra_ctxsize = sizeof(struct tegra_aead_ctx), 1925 .cra_alignmask = 0xf, 1926 .cra_module = THIS_MODULE, 1927 }, 1928 } 1929 }, { 1930 .alg.aead.op.do_one_request = tegra_ccm_do_one_req, 1931 .alg.aead.base = { 1932 .init = tegra_aead_cra_init, 1933 .exit = tegra_aead_cra_exit, 1934 .setkey = tegra_aead_setkey, 1935 .setauthsize = tegra_ccm_setauthsize, 1936 .encrypt = tegra_aead_encrypt, 1937 .decrypt = tegra_aead_decrypt, 1938 .maxauthsize = AES_BLOCK_SIZE, 1939 .ivsize = AES_BLOCK_SIZE, 1940 .chunksize = AES_BLOCK_SIZE, 1941 .base = { 1942 .cra_name = "ccm(aes)", 1943 .cra_driver_name = "ccm-aes-tegra", 1944 .cra_priority = 500, 1945 .cra_flags = CRYPTO_ALG_ASYNC, 1946 .cra_blocksize = 1, 1947 .cra_ctxsize = sizeof(struct tegra_aead_ctx), 1948 .cra_alignmask = 0xf, 1949 .cra_module = THIS_MODULE, 1950 }, 1951 } 1952 } 1953 }; 1954 1955 static struct tegra_se_alg tegra_cmac_algs[] = { 1956 { 1957 .alg.ahash.op.do_one_request = tegra_cmac_do_one_req, 1958 .alg.ahash.base = { 1959 .init = tegra_cmac_init, 1960 .setkey = tegra_cmac_setkey, 1961 .update = tegra_cmac_update, 1962 .final = tegra_cmac_final, 1963 .finup = tegra_cmac_finup, 1964 .digest = tegra_cmac_digest, 1965 .export = tegra_cmac_export, 1966 .import = tegra_cmac_import, 1967 .halg.digestsize = AES_BLOCK_SIZE, 1968 .halg.statesize = sizeof(struct tegra_cmac_reqctx), 1969 .halg.base = { 1970 .cra_name = "cmac(aes)", 1971 .cra_driver_name = "tegra-se-cmac", 1972 .cra_priority = 300, 1973 .cra_flags = CRYPTO_ALG_ASYNC, 1974 .cra_blocksize = AES_BLOCK_SIZE, 1975 .cra_ctxsize = sizeof(struct tegra_cmac_ctx), 1976 .cra_alignmask = 0, 1977 .cra_module = THIS_MODULE, 1978 .cra_init = tegra_cmac_cra_init, 1979 .cra_exit = tegra_cmac_cra_exit, 1980 } 1981 } 1982 } 1983 }; 1984 1985 int tegra_init_aes(struct tegra_se *se) 1986 { 1987 struct aead_engine_alg *aead_alg; 1988 struct ahash_engine_alg *ahash_alg; 1989 struct skcipher_engine_alg *sk_alg; 1990 int i, ret; 1991 1992 se->manifest = tegra_aes_kac_manifest; 1993 1994 for (i = 0; i < ARRAY_SIZE(tegra_aes_algs); i++) { 1995 sk_alg = &tegra_aes_algs[i].alg.skcipher; 1996 tegra_aes_algs[i].se_dev = se; 1997 1998 ret = crypto_engine_register_skcipher(sk_alg); 1999 if (ret) { 2000 dev_err(se->dev, "failed to register %s\n", 2001 sk_alg->base.base.cra_name); 2002 goto err_aes; 2003 } 2004 } 2005 2006 for (i = 0; i < ARRAY_SIZE(tegra_aead_algs); i++) { 2007 aead_alg = &tegra_aead_algs[i].alg.aead; 2008 tegra_aead_algs[i].se_dev = se; 2009 2010 ret = crypto_engine_register_aead(aead_alg); 2011 if (ret) { 2012 dev_err(se->dev, "failed to register %s\n", 2013 aead_alg->base.base.cra_name); 2014 goto err_aead; 2015 } 2016 } 2017 2018 for (i = 0; i < ARRAY_SIZE(tegra_cmac_algs); i++) { 2019 ahash_alg = &tegra_cmac_algs[i].alg.ahash; 2020 tegra_cmac_algs[i].se_dev = se; 2021 2022 ret = crypto_engine_register_ahash(ahash_alg); 2023 if (ret) { 2024 dev_err(se->dev, "failed to register %s\n", 2025 ahash_alg->base.halg.base.cra_name); 2026 goto err_cmac; 2027 } 2028 } 2029 2030 return 0; 2031 2032 err_cmac: 2033 while (i--) 2034 crypto_engine_unregister_ahash(&tegra_cmac_algs[i].alg.ahash); 2035 2036 i = ARRAY_SIZE(tegra_aead_algs); 2037 err_aead: 2038 while (i--) 2039 crypto_engine_unregister_aead(&tegra_aead_algs[i].alg.aead); 2040 2041 i = ARRAY_SIZE(tegra_aes_algs); 2042 err_aes: 2043 while (i--) 2044 crypto_engine_unregister_skcipher(&tegra_aes_algs[i].alg.skcipher); 2045 2046 return ret; 2047 } 2048 2049 void tegra_deinit_aes(struct tegra_se *se) 2050 { 2051 int i; 2052 2053 for (i = 0; i < ARRAY_SIZE(tegra_aes_algs); i++) 2054 crypto_engine_unregister_skcipher(&tegra_aes_algs[i].alg.skcipher); 2055 2056 for (i = 0; i < ARRAY_SIZE(tegra_aead_algs); i++) 2057 crypto_engine_unregister_aead(&tegra_aead_algs[i].alg.aead); 2058 2059 for (i = 0; i < ARRAY_SIZE(tegra_cmac_algs); i++) 2060 crypto_engine_unregister_ahash(&tegra_cmac_algs[i].alg.ahash); 2061 } 2062