1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Freescale FSL CAAM support for crypto API over QI backend. 4 * Based on caamalg.c 5 * 6 * Copyright 2013-2016 Freescale Semiconductor, Inc. 7 * Copyright 2016-2019 NXP 8 */ 9 10 #include "compat.h" 11 #include "ctrl.h" 12 #include "regs.h" 13 #include "intern.h" 14 #include "desc_constr.h" 15 #include "error.h" 16 #include "sg_sw_qm.h" 17 #include "key_gen.h" 18 #include "qi.h" 19 #include "jr.h" 20 #include "caamalg_desc.h" 21 #include <crypto/xts.h> 22 #include <linux/unaligned.h> 23 #include <linux/device.h> 24 #include <linux/err.h> 25 #include <linux/dma-mapping.h> 26 #include <linux/kernel.h> 27 #include <linux/string.h> 28 29 /* 30 * crypto alg 31 */ 32 #define CAAM_CRA_PRIORITY 2000 33 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ 34 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ 35 SHA512_DIGEST_SIZE * 2) 36 37 #define DESC_MAX_USED_BYTES (DESC_QI_AEAD_GIVENC_LEN + \ 38 CAAM_MAX_KEY_SIZE) 39 #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) 40 41 struct caam_alg_entry { 42 int class1_alg_type; 43 int class2_alg_type; 44 bool rfc3686; 45 bool geniv; 46 bool nodkp; 47 }; 48 49 struct caam_aead_alg { 50 struct aead_alg aead; 51 struct caam_alg_entry caam; 52 bool registered; 53 }; 54 55 struct caam_skcipher_alg { 56 struct skcipher_alg skcipher; 57 struct caam_alg_entry caam; 58 bool registered; 59 }; 60 61 /* 62 * per-session context 63 */ 64 struct caam_ctx { 65 struct device *jrdev; 66 u32 sh_desc_enc[DESC_MAX_USED_LEN]; 67 u32 sh_desc_dec[DESC_MAX_USED_LEN]; 68 u8 key[CAAM_MAX_KEY_SIZE]; 69 dma_addr_t key_dma; 70 enum dma_data_direction dir; 71 struct alginfo adata; 72 struct alginfo cdata; 73 unsigned int authsize; 74 struct device *qidev; 75 spinlock_t lock; /* Protects multiple init of driver context */ 76 struct caam_drv_ctx *drv_ctx[NUM_OP]; 77 bool xts_key_fallback; 78 struct crypto_skcipher *fallback; 79 }; 80 81 struct caam_skcipher_req_ctx { 82 struct skcipher_request fallback_req; 83 }; 84 85 static int aead_set_sh_desc(struct crypto_aead *aead) 86 { 87 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 88 typeof(*alg), aead); 89 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 90 unsigned int ivsize = crypto_aead_ivsize(aead); 91 u32 ctx1_iv_off = 0; 92 u32 *nonce = NULL; 93 unsigned int data_len[2]; 94 u32 inl_mask; 95 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 96 OP_ALG_AAI_CTR_MOD128); 97 const bool is_rfc3686 = alg->caam.rfc3686; 98 struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); 99 100 if (!ctx->cdata.keylen || !ctx->authsize) 101 return 0; 102 103 /* 104 * AES-CTR needs to load IV in CONTEXT1 reg 105 * at an offset of 128bits (16bytes) 106 * CONTEXT1[255:128] = IV 107 */ 108 if (ctr_mode) 109 ctx1_iv_off = 16; 110 111 /* 112 * RFC3686 specific: 113 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 114 */ 115 if (is_rfc3686) { 116 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 117 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + 118 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); 119 } 120 121 /* 122 * In case |user key| > |derived key|, using DKP<imm,imm> would result 123 * in invalid opcodes (last bytes of user key) in the resulting 124 * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key 125 * addresses are needed. 126 */ 127 ctx->adata.key_virt = ctx->key; 128 ctx->adata.key_dma = ctx->key_dma; 129 130 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 131 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 132 133 data_len[0] = ctx->adata.keylen_pad; 134 data_len[1] = ctx->cdata.keylen; 135 136 if (alg->caam.geniv) 137 goto skip_enc; 138 139 /* aead_encrypt shared descriptor */ 140 if (desc_inline_query(DESC_QI_AEAD_ENC_LEN + 141 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 142 DESC_JOB_IO_LEN, data_len, &inl_mask, 143 ARRAY_SIZE(data_len)) < 0) 144 return -EINVAL; 145 146 ctx->adata.key_inline = !!(inl_mask & 1); 147 ctx->cdata.key_inline = !!(inl_mask & 2); 148 149 cnstr_shdsc_aead_encap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata, 150 ivsize, ctx->authsize, is_rfc3686, nonce, 151 ctx1_iv_off, true, ctrlpriv->era); 152 153 skip_enc: 154 /* aead_decrypt shared descriptor */ 155 if (desc_inline_query(DESC_QI_AEAD_DEC_LEN + 156 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 157 DESC_JOB_IO_LEN, data_len, &inl_mask, 158 ARRAY_SIZE(data_len)) < 0) 159 return -EINVAL; 160 161 ctx->adata.key_inline = !!(inl_mask & 1); 162 ctx->cdata.key_inline = !!(inl_mask & 2); 163 164 cnstr_shdsc_aead_decap(ctx->sh_desc_dec, &ctx->cdata, &ctx->adata, 165 ivsize, ctx->authsize, alg->caam.geniv, 166 is_rfc3686, nonce, ctx1_iv_off, true, 167 ctrlpriv->era); 168 169 if (!alg->caam.geniv) 170 goto skip_givenc; 171 172 /* aead_givencrypt shared descriptor */ 173 if (desc_inline_query(DESC_QI_AEAD_GIVENC_LEN + 174 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 175 DESC_JOB_IO_LEN, data_len, &inl_mask, 176 ARRAY_SIZE(data_len)) < 0) 177 return -EINVAL; 178 179 ctx->adata.key_inline = !!(inl_mask & 1); 180 ctx->cdata.key_inline = !!(inl_mask & 2); 181 182 cnstr_shdsc_aead_givencap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata, 183 ivsize, ctx->authsize, is_rfc3686, nonce, 184 ctx1_iv_off, true, ctrlpriv->era); 185 186 skip_givenc: 187 return 0; 188 } 189 190 static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 191 { 192 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); 193 194 ctx->authsize = authsize; 195 aead_set_sh_desc(authenc); 196 197 return 0; 198 } 199 200 static int aead_setkey(struct crypto_aead *aead, const u8 *key, 201 unsigned int keylen) 202 { 203 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 204 struct device *jrdev = ctx->jrdev; 205 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); 206 struct crypto_authenc_keys keys; 207 int ret = 0; 208 209 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 210 goto badkey; 211 212 dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n", 213 keys.authkeylen + keys.enckeylen, keys.enckeylen, 214 keys.authkeylen); 215 print_hex_dump_devel("key in @" __stringify(__LINE__)": ", 216 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 217 218 /* 219 * If DKP is supported, use it in the shared descriptor to generate 220 * the split key. 221 */ 222 if (ctrlpriv->era >= 6) { 223 ctx->adata.keylen = keys.authkeylen; 224 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & 225 OP_ALG_ALGSEL_MASK); 226 227 if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) 228 goto badkey; 229 230 memcpy(ctx->key, keys.authkey, keys.authkeylen); 231 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, 232 keys.enckeylen); 233 dma_sync_single_for_device(jrdev->parent, ctx->key_dma, 234 ctx->adata.keylen_pad + 235 keys.enckeylen, ctx->dir); 236 goto skip_split_key; 237 } 238 239 ret = gen_split_key(jrdev, ctx->key, &ctx->adata, keys.authkey, 240 keys.authkeylen, CAAM_MAX_KEY_SIZE - 241 keys.enckeylen); 242 if (ret) 243 goto badkey; 244 245 /* postpend encryption key to auth split key */ 246 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); 247 dma_sync_single_for_device(jrdev->parent, ctx->key_dma, 248 ctx->adata.keylen_pad + keys.enckeylen, 249 ctx->dir); 250 251 print_hex_dump_devel("ctx.key@" __stringify(__LINE__)": ", 252 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 253 ctx->adata.keylen_pad + keys.enckeylen, 1); 254 255 skip_split_key: 256 ctx->cdata.keylen = keys.enckeylen; 257 258 ret = aead_set_sh_desc(aead); 259 if (ret) 260 goto badkey; 261 262 /* Now update the driver contexts with the new shared descriptor */ 263 if (ctx->drv_ctx[ENCRYPT]) { 264 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 265 ctx->sh_desc_enc); 266 if (ret) { 267 dev_err(jrdev, "driver enc context update failed\n"); 268 goto badkey; 269 } 270 } 271 272 if (ctx->drv_ctx[DECRYPT]) { 273 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 274 ctx->sh_desc_dec); 275 if (ret) { 276 dev_err(jrdev, "driver dec context update failed\n"); 277 goto badkey; 278 } 279 } 280 281 memzero_explicit(&keys, sizeof(keys)); 282 return ret; 283 badkey: 284 memzero_explicit(&keys, sizeof(keys)); 285 return -EINVAL; 286 } 287 288 static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key, 289 unsigned int keylen) 290 { 291 struct crypto_authenc_keys keys; 292 int err; 293 294 err = crypto_authenc_extractkeys(&keys, key, keylen); 295 if (unlikely(err)) 296 return err; 297 298 err = verify_aead_des3_key(aead, keys.enckey, keys.enckeylen) ?: 299 aead_setkey(aead, key, keylen); 300 301 memzero_explicit(&keys, sizeof(keys)); 302 return err; 303 } 304 305 static int gcm_set_sh_desc(struct crypto_aead *aead) 306 { 307 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 308 unsigned int ivsize = crypto_aead_ivsize(aead); 309 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 310 ctx->cdata.keylen; 311 312 if (!ctx->cdata.keylen || !ctx->authsize) 313 return 0; 314 315 /* 316 * Job Descriptor and Shared Descriptor 317 * must fit into the 64-word Descriptor h/w Buffer 318 */ 319 if (rem_bytes >= DESC_QI_GCM_ENC_LEN) { 320 ctx->cdata.key_inline = true; 321 ctx->cdata.key_virt = ctx->key; 322 } else { 323 ctx->cdata.key_inline = false; 324 ctx->cdata.key_dma = ctx->key_dma; 325 } 326 327 cnstr_shdsc_gcm_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, 328 ctx->authsize, true); 329 330 /* 331 * Job Descriptor and Shared Descriptor 332 * must fit into the 64-word Descriptor h/w Buffer 333 */ 334 if (rem_bytes >= DESC_QI_GCM_DEC_LEN) { 335 ctx->cdata.key_inline = true; 336 ctx->cdata.key_virt = ctx->key; 337 } else { 338 ctx->cdata.key_inline = false; 339 ctx->cdata.key_dma = ctx->key_dma; 340 } 341 342 cnstr_shdsc_gcm_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, 343 ctx->authsize, true); 344 345 return 0; 346 } 347 348 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 349 { 350 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); 351 int err; 352 353 err = crypto_gcm_check_authsize(authsize); 354 if (err) 355 return err; 356 357 ctx->authsize = authsize; 358 gcm_set_sh_desc(authenc); 359 360 return 0; 361 } 362 363 static int gcm_setkey(struct crypto_aead *aead, 364 const u8 *key, unsigned int keylen) 365 { 366 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 367 struct device *jrdev = ctx->jrdev; 368 int ret; 369 370 ret = aes_check_keylen(keylen); 371 if (ret) 372 return ret; 373 374 print_hex_dump_devel("key in @" __stringify(__LINE__)": ", 375 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 376 377 memcpy(ctx->key, key, keylen); 378 dma_sync_single_for_device(jrdev->parent, ctx->key_dma, keylen, 379 ctx->dir); 380 ctx->cdata.keylen = keylen; 381 382 ret = gcm_set_sh_desc(aead); 383 if (ret) 384 return ret; 385 386 /* Now update the driver contexts with the new shared descriptor */ 387 if (ctx->drv_ctx[ENCRYPT]) { 388 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 389 ctx->sh_desc_enc); 390 if (ret) { 391 dev_err(jrdev, "driver enc context update failed\n"); 392 return ret; 393 } 394 } 395 396 if (ctx->drv_ctx[DECRYPT]) { 397 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 398 ctx->sh_desc_dec); 399 if (ret) { 400 dev_err(jrdev, "driver dec context update failed\n"); 401 return ret; 402 } 403 } 404 405 return 0; 406 } 407 408 static int rfc4106_set_sh_desc(struct crypto_aead *aead) 409 { 410 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 411 unsigned int ivsize = crypto_aead_ivsize(aead); 412 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 413 ctx->cdata.keylen; 414 415 if (!ctx->cdata.keylen || !ctx->authsize) 416 return 0; 417 418 ctx->cdata.key_virt = ctx->key; 419 420 /* 421 * Job Descriptor and Shared Descriptor 422 * must fit into the 64-word Descriptor h/w Buffer 423 */ 424 if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) { 425 ctx->cdata.key_inline = true; 426 } else { 427 ctx->cdata.key_inline = false; 428 ctx->cdata.key_dma = ctx->key_dma; 429 } 430 431 cnstr_shdsc_rfc4106_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, 432 ctx->authsize, true); 433 434 /* 435 * Job Descriptor and Shared Descriptor 436 * must fit into the 64-word Descriptor h/w Buffer 437 */ 438 if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) { 439 ctx->cdata.key_inline = true; 440 } else { 441 ctx->cdata.key_inline = false; 442 ctx->cdata.key_dma = ctx->key_dma; 443 } 444 445 cnstr_shdsc_rfc4106_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, 446 ctx->authsize, true); 447 448 return 0; 449 } 450 451 static int rfc4106_setauthsize(struct crypto_aead *authenc, 452 unsigned int authsize) 453 { 454 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); 455 int err; 456 457 err = crypto_rfc4106_check_authsize(authsize); 458 if (err) 459 return err; 460 461 ctx->authsize = authsize; 462 rfc4106_set_sh_desc(authenc); 463 464 return 0; 465 } 466 467 static int rfc4106_setkey(struct crypto_aead *aead, 468 const u8 *key, unsigned int keylen) 469 { 470 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 471 struct device *jrdev = ctx->jrdev; 472 int ret; 473 474 ret = aes_check_keylen(keylen - 4); 475 if (ret) 476 return ret; 477 478 print_hex_dump_devel("key in @" __stringify(__LINE__)": ", 479 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 480 481 memcpy(ctx->key, key, keylen); 482 /* 483 * The last four bytes of the key material are used as the salt value 484 * in the nonce. Update the AES key length. 485 */ 486 ctx->cdata.keylen = keylen - 4; 487 dma_sync_single_for_device(jrdev->parent, ctx->key_dma, 488 ctx->cdata.keylen, ctx->dir); 489 490 ret = rfc4106_set_sh_desc(aead); 491 if (ret) 492 return ret; 493 494 /* Now update the driver contexts with the new shared descriptor */ 495 if (ctx->drv_ctx[ENCRYPT]) { 496 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 497 ctx->sh_desc_enc); 498 if (ret) { 499 dev_err(jrdev, "driver enc context update failed\n"); 500 return ret; 501 } 502 } 503 504 if (ctx->drv_ctx[DECRYPT]) { 505 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 506 ctx->sh_desc_dec); 507 if (ret) { 508 dev_err(jrdev, "driver dec context update failed\n"); 509 return ret; 510 } 511 } 512 513 return 0; 514 } 515 516 static int rfc4543_set_sh_desc(struct crypto_aead *aead) 517 { 518 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 519 unsigned int ivsize = crypto_aead_ivsize(aead); 520 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 521 ctx->cdata.keylen; 522 523 if (!ctx->cdata.keylen || !ctx->authsize) 524 return 0; 525 526 ctx->cdata.key_virt = ctx->key; 527 528 /* 529 * Job Descriptor and Shared Descriptor 530 * must fit into the 64-word Descriptor h/w Buffer 531 */ 532 if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) { 533 ctx->cdata.key_inline = true; 534 } else { 535 ctx->cdata.key_inline = false; 536 ctx->cdata.key_dma = ctx->key_dma; 537 } 538 539 cnstr_shdsc_rfc4543_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, 540 ctx->authsize, true); 541 542 /* 543 * Job Descriptor and Shared Descriptor 544 * must fit into the 64-word Descriptor h/w Buffer 545 */ 546 if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) { 547 ctx->cdata.key_inline = true; 548 } else { 549 ctx->cdata.key_inline = false; 550 ctx->cdata.key_dma = ctx->key_dma; 551 } 552 553 cnstr_shdsc_rfc4543_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, 554 ctx->authsize, true); 555 556 return 0; 557 } 558 559 static int rfc4543_setauthsize(struct crypto_aead *authenc, 560 unsigned int authsize) 561 { 562 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); 563 564 if (authsize != 16) 565 return -EINVAL; 566 567 ctx->authsize = authsize; 568 rfc4543_set_sh_desc(authenc); 569 570 return 0; 571 } 572 573 static int rfc4543_setkey(struct crypto_aead *aead, 574 const u8 *key, unsigned int keylen) 575 { 576 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 577 struct device *jrdev = ctx->jrdev; 578 int ret; 579 580 ret = aes_check_keylen(keylen - 4); 581 if (ret) 582 return ret; 583 584 print_hex_dump_devel("key in @" __stringify(__LINE__)": ", 585 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 586 587 memcpy(ctx->key, key, keylen); 588 /* 589 * The last four bytes of the key material are used as the salt value 590 * in the nonce. Update the AES key length. 591 */ 592 ctx->cdata.keylen = keylen - 4; 593 dma_sync_single_for_device(jrdev->parent, ctx->key_dma, 594 ctx->cdata.keylen, ctx->dir); 595 596 ret = rfc4543_set_sh_desc(aead); 597 if (ret) 598 return ret; 599 600 /* Now update the driver contexts with the new shared descriptor */ 601 if (ctx->drv_ctx[ENCRYPT]) { 602 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 603 ctx->sh_desc_enc); 604 if (ret) { 605 dev_err(jrdev, "driver enc context update failed\n"); 606 return ret; 607 } 608 } 609 610 if (ctx->drv_ctx[DECRYPT]) { 611 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 612 ctx->sh_desc_dec); 613 if (ret) { 614 dev_err(jrdev, "driver dec context update failed\n"); 615 return ret; 616 } 617 } 618 619 return 0; 620 } 621 622 static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 623 unsigned int keylen, const u32 ctx1_iv_off) 624 { 625 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 626 struct caam_skcipher_alg *alg = 627 container_of(crypto_skcipher_alg(skcipher), typeof(*alg), 628 skcipher); 629 struct device *jrdev = ctx->jrdev; 630 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 631 const bool is_rfc3686 = alg->caam.rfc3686; 632 int ret = 0; 633 634 print_hex_dump_devel("key in @" __stringify(__LINE__)": ", 635 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 636 637 ctx->cdata.keylen = keylen; 638 ctx->cdata.key_virt = key; 639 ctx->cdata.key_inline = true; 640 ctx->cdata.plain_keylen = keylen; 641 ctx->cdata.key_cmd_opt = 0; 642 643 /* skcipher encrypt, decrypt shared descriptors */ 644 cnstr_shdsc_skcipher_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, 645 is_rfc3686, ctx1_iv_off); 646 cnstr_shdsc_skcipher_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, 647 is_rfc3686, ctx1_iv_off); 648 649 /* Now update the driver contexts with the new shared descriptor */ 650 if (ctx->drv_ctx[ENCRYPT]) { 651 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 652 ctx->sh_desc_enc); 653 if (ret) { 654 dev_err(jrdev, "driver enc context update failed\n"); 655 return -EINVAL; 656 } 657 } 658 659 if (ctx->drv_ctx[DECRYPT]) { 660 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 661 ctx->sh_desc_dec); 662 if (ret) { 663 dev_err(jrdev, "driver dec context update failed\n"); 664 return -EINVAL; 665 } 666 } 667 668 return ret; 669 } 670 671 static int aes_skcipher_setkey(struct crypto_skcipher *skcipher, 672 const u8 *key, unsigned int keylen) 673 { 674 int err; 675 676 err = aes_check_keylen(keylen); 677 if (err) 678 return err; 679 680 return skcipher_setkey(skcipher, key, keylen, 0); 681 } 682 683 static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher, 684 const u8 *key, unsigned int keylen) 685 { 686 u32 ctx1_iv_off; 687 int err; 688 689 /* 690 * RFC3686 specific: 691 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} 692 * | *key = {KEY, NONCE} 693 */ 694 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 695 keylen -= CTR_RFC3686_NONCE_SIZE; 696 697 err = aes_check_keylen(keylen); 698 if (err) 699 return err; 700 701 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); 702 } 703 704 static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher, 705 const u8 *key, unsigned int keylen) 706 { 707 u32 ctx1_iv_off; 708 int err; 709 710 /* 711 * AES-CTR needs to load IV in CONTEXT1 reg 712 * at an offset of 128bits (16bytes) 713 * CONTEXT1[255:128] = IV 714 */ 715 ctx1_iv_off = 16; 716 717 err = aes_check_keylen(keylen); 718 if (err) 719 return err; 720 721 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); 722 } 723 724 static int des3_skcipher_setkey(struct crypto_skcipher *skcipher, 725 const u8 *key, unsigned int keylen) 726 { 727 return verify_skcipher_des3_key(skcipher, key) ?: 728 skcipher_setkey(skcipher, key, keylen, 0); 729 } 730 731 static int des_skcipher_setkey(struct crypto_skcipher *skcipher, 732 const u8 *key, unsigned int keylen) 733 { 734 return verify_skcipher_des_key(skcipher, key) ?: 735 skcipher_setkey(skcipher, key, keylen, 0); 736 } 737 738 static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 739 unsigned int keylen) 740 { 741 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 742 struct device *jrdev = ctx->jrdev; 743 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); 744 int ret = 0; 745 int err; 746 747 err = xts_verify_key(skcipher, key, keylen); 748 if (err) { 749 dev_dbg(jrdev, "key size mismatch\n"); 750 return err; 751 } 752 753 if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256) 754 ctx->xts_key_fallback = true; 755 756 if (ctrlpriv->era <= 8 || ctx->xts_key_fallback) { 757 err = crypto_skcipher_setkey(ctx->fallback, key, keylen); 758 if (err) 759 return err; 760 } 761 762 ctx->cdata.keylen = keylen; 763 ctx->cdata.key_virt = key; 764 ctx->cdata.key_inline = true; 765 766 /* xts skcipher encrypt, decrypt shared descriptors */ 767 cnstr_shdsc_xts_skcipher_encap(ctx->sh_desc_enc, &ctx->cdata); 768 cnstr_shdsc_xts_skcipher_decap(ctx->sh_desc_dec, &ctx->cdata); 769 770 /* Now update the driver contexts with the new shared descriptor */ 771 if (ctx->drv_ctx[ENCRYPT]) { 772 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 773 ctx->sh_desc_enc); 774 if (ret) { 775 dev_err(jrdev, "driver enc context update failed\n"); 776 return -EINVAL; 777 } 778 } 779 780 if (ctx->drv_ctx[DECRYPT]) { 781 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 782 ctx->sh_desc_dec); 783 if (ret) { 784 dev_err(jrdev, "driver dec context update failed\n"); 785 return -EINVAL; 786 } 787 } 788 789 return ret; 790 } 791 792 /* 793 * aead_edesc - s/w-extended aead descriptor 794 * @src_nents: number of segments in input scatterlist 795 * @dst_nents: number of segments in output scatterlist 796 * @iv_dma: dma address of iv for checking continuity and link table 797 * @qm_sg_bytes: length of dma mapped h/w link table 798 * @qm_sg_dma: bus physical mapped address of h/w link table 799 * @assoclen: associated data length, in CAAM endianness 800 * @assoclen_dma: bus physical mapped address of req->assoclen 801 * @drv_req: driver-specific request structure 802 * @sgt: the h/w link table, followed by IV 803 */ 804 struct aead_edesc { 805 int src_nents; 806 int dst_nents; 807 dma_addr_t iv_dma; 808 int qm_sg_bytes; 809 dma_addr_t qm_sg_dma; 810 unsigned int assoclen; 811 dma_addr_t assoclen_dma; 812 struct caam_drv_req drv_req; 813 struct qm_sg_entry sgt[]; 814 }; 815 816 /* 817 * skcipher_edesc - s/w-extended skcipher descriptor 818 * @src_nents: number of segments in input scatterlist 819 * @dst_nents: number of segments in output scatterlist 820 * @iv_dma: dma address of iv for checking continuity and link table 821 * @qm_sg_bytes: length of dma mapped h/w link table 822 * @qm_sg_dma: bus physical mapped address of h/w link table 823 * @drv_req: driver-specific request structure 824 * @sgt: the h/w link table, followed by IV 825 */ 826 struct skcipher_edesc { 827 int src_nents; 828 int dst_nents; 829 dma_addr_t iv_dma; 830 int qm_sg_bytes; 831 dma_addr_t qm_sg_dma; 832 struct caam_drv_req drv_req; 833 struct qm_sg_entry sgt[]; 834 }; 835 836 static struct caam_drv_ctx *get_drv_ctx(struct caam_ctx *ctx, 837 enum optype type) 838 { 839 /* 840 * This function is called on the fast path with values of 'type' 841 * known at compile time. Invalid arguments are not expected and 842 * thus no checks are made. 843 */ 844 struct caam_drv_ctx *drv_ctx = ctx->drv_ctx[type]; 845 u32 *desc; 846 847 if (unlikely(!drv_ctx)) { 848 spin_lock(&ctx->lock); 849 850 /* Read again to check if some other core init drv_ctx */ 851 drv_ctx = ctx->drv_ctx[type]; 852 if (!drv_ctx) { 853 int cpu; 854 855 if (type == ENCRYPT) 856 desc = ctx->sh_desc_enc; 857 else /* (type == DECRYPT) */ 858 desc = ctx->sh_desc_dec; 859 860 cpu = smp_processor_id(); 861 drv_ctx = caam_drv_ctx_init(ctx->qidev, &cpu, desc); 862 if (!IS_ERR(drv_ctx)) 863 drv_ctx->op_type = type; 864 865 ctx->drv_ctx[type] = drv_ctx; 866 } 867 868 spin_unlock(&ctx->lock); 869 } 870 871 return drv_ctx; 872 } 873 874 static void caam_unmap(struct device *dev, struct scatterlist *src, 875 struct scatterlist *dst, int src_nents, 876 int dst_nents, dma_addr_t iv_dma, int ivsize, 877 enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma, 878 int qm_sg_bytes) 879 { 880 if (dst != src) { 881 if (src_nents) 882 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); 883 if (dst_nents) 884 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); 885 } else { 886 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); 887 } 888 889 if (iv_dma) 890 dma_unmap_single(dev, iv_dma, ivsize, iv_dir); 891 if (qm_sg_bytes) 892 dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE); 893 } 894 895 static void aead_unmap(struct device *dev, 896 struct aead_edesc *edesc, 897 struct aead_request *req) 898 { 899 struct crypto_aead *aead = crypto_aead_reqtfm(req); 900 int ivsize = crypto_aead_ivsize(aead); 901 902 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 903 edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma, 904 edesc->qm_sg_bytes); 905 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 906 } 907 908 static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc, 909 struct skcipher_request *req) 910 { 911 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 912 int ivsize = crypto_skcipher_ivsize(skcipher); 913 914 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 915 edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma, 916 edesc->qm_sg_bytes); 917 } 918 919 static void aead_done(struct caam_drv_req *drv_req, u32 status) 920 { 921 struct device *qidev; 922 struct aead_edesc *edesc; 923 struct aead_request *aead_req = drv_req->app_ctx; 924 struct crypto_aead *aead = crypto_aead_reqtfm(aead_req); 925 struct caam_ctx *caam_ctx = crypto_aead_ctx_dma(aead); 926 int ecode = 0; 927 928 qidev = caam_ctx->qidev; 929 930 if (unlikely(status)) 931 ecode = caam_jr_strstatus(qidev, status); 932 933 edesc = container_of(drv_req, typeof(*edesc), drv_req); 934 aead_unmap(qidev, edesc, aead_req); 935 936 aead_request_complete(aead_req, ecode); 937 qi_cache_free(edesc); 938 } 939 940 /* 941 * allocate and map the aead extended descriptor 942 */ 943 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 944 bool encrypt) 945 { 946 struct crypto_aead *aead = crypto_aead_reqtfm(req); 947 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 948 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 949 typeof(*alg), aead); 950 struct device *qidev = ctx->qidev; 951 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 952 GFP_KERNEL : GFP_ATOMIC; 953 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 954 int src_len, dst_len = 0; 955 struct aead_edesc *edesc; 956 dma_addr_t qm_sg_dma, iv_dma = 0; 957 int ivsize = 0; 958 unsigned int authsize = ctx->authsize; 959 int qm_sg_index = 0, qm_sg_ents = 0, qm_sg_bytes; 960 int in_len, out_len; 961 struct qm_sg_entry *sg_table, *fd_sgt; 962 struct caam_drv_ctx *drv_ctx; 963 964 drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT); 965 if (IS_ERR(drv_ctx)) 966 return ERR_CAST(drv_ctx); 967 968 /* allocate space for base edesc and hw desc commands, link tables */ 969 edesc = qi_cache_alloc(flags); 970 if (unlikely(!edesc)) { 971 dev_err(qidev, "could not allocate extended descriptor\n"); 972 return ERR_PTR(-ENOMEM); 973 } 974 975 if (likely(req->src == req->dst)) { 976 src_len = req->assoclen + req->cryptlen + 977 (encrypt ? authsize : 0); 978 979 src_nents = sg_nents_for_len(req->src, src_len); 980 if (unlikely(src_nents < 0)) { 981 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", 982 src_len); 983 qi_cache_free(edesc); 984 return ERR_PTR(src_nents); 985 } 986 987 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, 988 DMA_BIDIRECTIONAL); 989 if (unlikely(!mapped_src_nents)) { 990 dev_err(qidev, "unable to map source\n"); 991 qi_cache_free(edesc); 992 return ERR_PTR(-ENOMEM); 993 } 994 } else { 995 src_len = req->assoclen + req->cryptlen; 996 dst_len = src_len + (encrypt ? authsize : (-authsize)); 997 998 src_nents = sg_nents_for_len(req->src, src_len); 999 if (unlikely(src_nents < 0)) { 1000 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", 1001 src_len); 1002 qi_cache_free(edesc); 1003 return ERR_PTR(src_nents); 1004 } 1005 1006 dst_nents = sg_nents_for_len(req->dst, dst_len); 1007 if (unlikely(dst_nents < 0)) { 1008 dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n", 1009 dst_len); 1010 qi_cache_free(edesc); 1011 return ERR_PTR(dst_nents); 1012 } 1013 1014 if (src_nents) { 1015 mapped_src_nents = dma_map_sg(qidev, req->src, 1016 src_nents, DMA_TO_DEVICE); 1017 if (unlikely(!mapped_src_nents)) { 1018 dev_err(qidev, "unable to map source\n"); 1019 qi_cache_free(edesc); 1020 return ERR_PTR(-ENOMEM); 1021 } 1022 } else { 1023 mapped_src_nents = 0; 1024 } 1025 1026 if (dst_nents) { 1027 mapped_dst_nents = dma_map_sg(qidev, req->dst, 1028 dst_nents, 1029 DMA_FROM_DEVICE); 1030 if (unlikely(!mapped_dst_nents)) { 1031 dev_err(qidev, "unable to map destination\n"); 1032 dma_unmap_sg(qidev, req->src, src_nents, 1033 DMA_TO_DEVICE); 1034 qi_cache_free(edesc); 1035 return ERR_PTR(-ENOMEM); 1036 } 1037 } else { 1038 mapped_dst_nents = 0; 1039 } 1040 } 1041 1042 if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv) 1043 ivsize = crypto_aead_ivsize(aead); 1044 1045 /* 1046 * Create S/G table: req->assoclen, [IV,] req->src [, req->dst]. 1047 * Input is not contiguous. 1048 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond 1049 * the end of the table by allocating more S/G entries. Logic: 1050 * if (src != dst && output S/G) 1051 * pad output S/G, if needed 1052 * else if (src == dst && S/G) 1053 * overlapping S/Gs; pad one of them 1054 * else if (input S/G) ... 1055 * pad input S/G, if needed 1056 */ 1057 qm_sg_ents = 1 + !!ivsize + mapped_src_nents; 1058 if (mapped_dst_nents > 1) 1059 qm_sg_ents += pad_sg_nents(mapped_dst_nents); 1060 else if ((req->src == req->dst) && (mapped_src_nents > 1)) 1061 qm_sg_ents = max(pad_sg_nents(qm_sg_ents), 1062 1 + !!ivsize + pad_sg_nents(mapped_src_nents)); 1063 else 1064 qm_sg_ents = pad_sg_nents(qm_sg_ents); 1065 1066 sg_table = &edesc->sgt[0]; 1067 qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); 1068 if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize > 1069 CAAM_QI_MEMCACHE_SIZE)) { 1070 dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n", 1071 qm_sg_ents, ivsize); 1072 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, 1073 0, DMA_NONE, 0, 0); 1074 qi_cache_free(edesc); 1075 return ERR_PTR(-ENOMEM); 1076 } 1077 1078 if (ivsize) { 1079 u8 *iv = (u8 *)(sg_table + qm_sg_ents); 1080 1081 /* Make sure IV is located in a DMAable area */ 1082 memcpy(iv, req->iv, ivsize); 1083 1084 iv_dma = dma_map_single(qidev, iv, ivsize, DMA_TO_DEVICE); 1085 if (dma_mapping_error(qidev, iv_dma)) { 1086 dev_err(qidev, "unable to map IV\n"); 1087 caam_unmap(qidev, req->src, req->dst, src_nents, 1088 dst_nents, 0, 0, DMA_NONE, 0, 0); 1089 qi_cache_free(edesc); 1090 return ERR_PTR(-ENOMEM); 1091 } 1092 } 1093 1094 edesc->src_nents = src_nents; 1095 edesc->dst_nents = dst_nents; 1096 edesc->iv_dma = iv_dma; 1097 edesc->drv_req.app_ctx = req; 1098 edesc->drv_req.cbk = aead_done; 1099 edesc->drv_req.drv_ctx = drv_ctx; 1100 1101 edesc->assoclen = cpu_to_caam32(req->assoclen); 1102 edesc->assoclen_dma = dma_map_single(qidev, &edesc->assoclen, 4, 1103 DMA_TO_DEVICE); 1104 if (dma_mapping_error(qidev, edesc->assoclen_dma)) { 1105 dev_err(qidev, "unable to map assoclen\n"); 1106 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 1107 iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); 1108 qi_cache_free(edesc); 1109 return ERR_PTR(-ENOMEM); 1110 } 1111 1112 dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0); 1113 qm_sg_index++; 1114 if (ivsize) { 1115 dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0); 1116 qm_sg_index++; 1117 } 1118 sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0); 1119 qm_sg_index += mapped_src_nents; 1120 1121 if (mapped_dst_nents > 1) 1122 sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0); 1123 1124 qm_sg_dma = dma_map_single(qidev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); 1125 if (dma_mapping_error(qidev, qm_sg_dma)) { 1126 dev_err(qidev, "unable to map S/G table\n"); 1127 dma_unmap_single(qidev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 1128 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 1129 iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); 1130 qi_cache_free(edesc); 1131 return ERR_PTR(-ENOMEM); 1132 } 1133 1134 edesc->qm_sg_dma = qm_sg_dma; 1135 edesc->qm_sg_bytes = qm_sg_bytes; 1136 1137 out_len = req->assoclen + req->cryptlen + 1138 (encrypt ? ctx->authsize : (-ctx->authsize)); 1139 in_len = 4 + ivsize + req->assoclen + req->cryptlen; 1140 1141 fd_sgt = &edesc->drv_req.fd_sgt[0]; 1142 dma_to_qm_sg_one_last_ext(&fd_sgt[1], qm_sg_dma, in_len, 0); 1143 1144 if (req->dst == req->src) { 1145 if (mapped_src_nents == 1) 1146 dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->src), 1147 out_len, 0); 1148 else 1149 dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + 1150 (1 + !!ivsize) * sizeof(*sg_table), 1151 out_len, 0); 1152 } else if (mapped_dst_nents <= 1) { 1153 dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), out_len, 1154 0); 1155 } else { 1156 dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + sizeof(*sg_table) * 1157 qm_sg_index, out_len, 0); 1158 } 1159 1160 return edesc; 1161 } 1162 1163 static inline int aead_crypt(struct aead_request *req, bool encrypt) 1164 { 1165 struct aead_edesc *edesc; 1166 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1167 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 1168 int ret; 1169 1170 if (unlikely(caam_congested)) 1171 return -EAGAIN; 1172 1173 /* allocate extended descriptor */ 1174 edesc = aead_edesc_alloc(req, encrypt); 1175 if (IS_ERR(edesc)) 1176 return PTR_ERR(edesc); 1177 1178 /* Create and submit job descriptor */ 1179 ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req); 1180 if (!ret) { 1181 ret = -EINPROGRESS; 1182 } else { 1183 aead_unmap(ctx->qidev, edesc, req); 1184 qi_cache_free(edesc); 1185 } 1186 1187 return ret; 1188 } 1189 1190 static int aead_encrypt(struct aead_request *req) 1191 { 1192 return aead_crypt(req, true); 1193 } 1194 1195 static int aead_decrypt(struct aead_request *req) 1196 { 1197 return aead_crypt(req, false); 1198 } 1199 1200 static int ipsec_gcm_encrypt(struct aead_request *req) 1201 { 1202 return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_crypt(req, 1203 true); 1204 } 1205 1206 static int ipsec_gcm_decrypt(struct aead_request *req) 1207 { 1208 return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_crypt(req, 1209 false); 1210 } 1211 1212 static inline u8 *skcipher_edesc_iv(struct skcipher_edesc *edesc) 1213 { 1214 return PTR_ALIGN((u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, 1215 dma_get_cache_alignment()); 1216 } 1217 1218 static void skcipher_done(struct caam_drv_req *drv_req, u32 status) 1219 { 1220 struct skcipher_edesc *edesc; 1221 struct skcipher_request *req = drv_req->app_ctx; 1222 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1223 struct caam_ctx *caam_ctx = crypto_skcipher_ctx_dma(skcipher); 1224 struct device *qidev = caam_ctx->qidev; 1225 int ivsize = crypto_skcipher_ivsize(skcipher); 1226 int ecode = 0; 1227 1228 dev_dbg(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status); 1229 1230 edesc = container_of(drv_req, typeof(*edesc), drv_req); 1231 1232 if (status) 1233 ecode = caam_jr_strstatus(qidev, status); 1234 1235 print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", 1236 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 1237 edesc->src_nents > 1 ? 100 : ivsize, 1); 1238 caam_dump_sg("dst @" __stringify(__LINE__)": ", 1239 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 1240 edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); 1241 1242 skcipher_unmap(qidev, edesc, req); 1243 1244 /* 1245 * The crypto API expects us to set the IV (req->iv) to the last 1246 * ciphertext block (CBC mode) or last counter (CTR mode). 1247 * This is used e.g. by the CTS mode. 1248 */ 1249 if (!ecode) 1250 memcpy(req->iv, skcipher_edesc_iv(edesc), ivsize); 1251 1252 qi_cache_free(edesc); 1253 skcipher_request_complete(req, ecode); 1254 } 1255 1256 static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, 1257 bool encrypt) 1258 { 1259 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1260 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 1261 struct device *qidev = ctx->qidev; 1262 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1263 GFP_KERNEL : GFP_ATOMIC; 1264 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 1265 struct skcipher_edesc *edesc; 1266 dma_addr_t iv_dma; 1267 u8 *iv; 1268 int ivsize = crypto_skcipher_ivsize(skcipher); 1269 int dst_sg_idx, qm_sg_ents, qm_sg_bytes; 1270 struct qm_sg_entry *sg_table, *fd_sgt; 1271 struct caam_drv_ctx *drv_ctx; 1272 unsigned int len; 1273 1274 drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT); 1275 if (IS_ERR(drv_ctx)) 1276 return ERR_CAST(drv_ctx); 1277 1278 src_nents = sg_nents_for_len(req->src, req->cryptlen); 1279 if (unlikely(src_nents < 0)) { 1280 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", 1281 req->cryptlen); 1282 return ERR_PTR(src_nents); 1283 } 1284 1285 if (unlikely(req->src != req->dst)) { 1286 dst_nents = sg_nents_for_len(req->dst, req->cryptlen); 1287 if (unlikely(dst_nents < 0)) { 1288 dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n", 1289 req->cryptlen); 1290 return ERR_PTR(dst_nents); 1291 } 1292 1293 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, 1294 DMA_TO_DEVICE); 1295 if (unlikely(!mapped_src_nents)) { 1296 dev_err(qidev, "unable to map source\n"); 1297 return ERR_PTR(-ENOMEM); 1298 } 1299 1300 mapped_dst_nents = dma_map_sg(qidev, req->dst, dst_nents, 1301 DMA_FROM_DEVICE); 1302 if (unlikely(!mapped_dst_nents)) { 1303 dev_err(qidev, "unable to map destination\n"); 1304 dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE); 1305 return ERR_PTR(-ENOMEM); 1306 } 1307 } else { 1308 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, 1309 DMA_BIDIRECTIONAL); 1310 if (unlikely(!mapped_src_nents)) { 1311 dev_err(qidev, "unable to map source\n"); 1312 return ERR_PTR(-ENOMEM); 1313 } 1314 } 1315 1316 qm_sg_ents = 1 + mapped_src_nents; 1317 dst_sg_idx = qm_sg_ents; 1318 1319 /* 1320 * Input, output HW S/G tables: [IV, src][dst, IV] 1321 * IV entries point to the same buffer 1322 * If src == dst, S/G entries are reused (S/G tables overlap) 1323 * 1324 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond 1325 * the end of the table by allocating more S/G entries. 1326 */ 1327 if (req->src != req->dst) 1328 qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1); 1329 else 1330 qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents); 1331 1332 qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry); 1333 1334 len = offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes; 1335 len = ALIGN(len, dma_get_cache_alignment()); 1336 len += ivsize; 1337 1338 if (unlikely(len > CAAM_QI_MEMCACHE_SIZE)) { 1339 dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n", 1340 qm_sg_ents, ivsize); 1341 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, 1342 0, DMA_NONE, 0, 0); 1343 return ERR_PTR(-ENOMEM); 1344 } 1345 1346 /* allocate space for base edesc, link tables and IV */ 1347 edesc = qi_cache_alloc(flags); 1348 if (unlikely(!edesc)) { 1349 dev_err(qidev, "could not allocate extended descriptor\n"); 1350 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, 1351 0, DMA_NONE, 0, 0); 1352 return ERR_PTR(-ENOMEM); 1353 } 1354 1355 edesc->src_nents = src_nents; 1356 edesc->dst_nents = dst_nents; 1357 edesc->qm_sg_bytes = qm_sg_bytes; 1358 edesc->drv_req.app_ctx = req; 1359 edesc->drv_req.cbk = skcipher_done; 1360 edesc->drv_req.drv_ctx = drv_ctx; 1361 1362 /* Make sure IV is located in a DMAable area */ 1363 sg_table = &edesc->sgt[0]; 1364 iv = skcipher_edesc_iv(edesc); 1365 memcpy(iv, req->iv, ivsize); 1366 1367 iv_dma = dma_map_single(qidev, iv, ivsize, DMA_BIDIRECTIONAL); 1368 if (dma_mapping_error(qidev, iv_dma)) { 1369 dev_err(qidev, "unable to map IV\n"); 1370 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, 1371 0, DMA_NONE, 0, 0); 1372 qi_cache_free(edesc); 1373 return ERR_PTR(-ENOMEM); 1374 } 1375 1376 edesc->iv_dma = iv_dma; 1377 1378 dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); 1379 sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0); 1380 1381 if (req->src != req->dst) 1382 sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0); 1383 1384 dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma, 1385 ivsize, 0); 1386 1387 edesc->qm_sg_dma = dma_map_single(qidev, sg_table, edesc->qm_sg_bytes, 1388 DMA_TO_DEVICE); 1389 if (dma_mapping_error(qidev, edesc->qm_sg_dma)) { 1390 dev_err(qidev, "unable to map S/G table\n"); 1391 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 1392 iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0); 1393 qi_cache_free(edesc); 1394 return ERR_PTR(-ENOMEM); 1395 } 1396 1397 fd_sgt = &edesc->drv_req.fd_sgt[0]; 1398 1399 dma_to_qm_sg_one_last_ext(&fd_sgt[1], edesc->qm_sg_dma, 1400 ivsize + req->cryptlen, 0); 1401 1402 if (req->src == req->dst) 1403 dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + 1404 sizeof(*sg_table), req->cryptlen + ivsize, 1405 0); 1406 else 1407 dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx * 1408 sizeof(*sg_table), req->cryptlen + ivsize, 1409 0); 1410 1411 return edesc; 1412 } 1413 1414 static inline bool xts_skcipher_ivsize(struct skcipher_request *req) 1415 { 1416 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1417 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 1418 1419 return !!get_unaligned((u64 *)(req->iv + (ivsize / 2))); 1420 } 1421 1422 static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt) 1423 { 1424 struct skcipher_edesc *edesc; 1425 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1426 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 1427 struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); 1428 int ret; 1429 1430 /* 1431 * XTS is expected to return an error even for input length = 0 1432 * Note that the case input length < block size will be caught during 1433 * HW offloading and return an error. 1434 */ 1435 if (!req->cryptlen && !ctx->fallback) 1436 return 0; 1437 1438 if (ctx->fallback && ((ctrlpriv->era <= 8 && xts_skcipher_ivsize(req)) || 1439 ctx->xts_key_fallback)) { 1440 struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req); 1441 1442 skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback); 1443 skcipher_request_set_callback(&rctx->fallback_req, 1444 req->base.flags, 1445 req->base.complete, 1446 req->base.data); 1447 skcipher_request_set_crypt(&rctx->fallback_req, req->src, 1448 req->dst, req->cryptlen, req->iv); 1449 1450 return encrypt ? crypto_skcipher_encrypt(&rctx->fallback_req) : 1451 crypto_skcipher_decrypt(&rctx->fallback_req); 1452 } 1453 1454 if (unlikely(caam_congested)) 1455 return -EAGAIN; 1456 1457 /* allocate extended descriptor */ 1458 edesc = skcipher_edesc_alloc(req, encrypt); 1459 if (IS_ERR(edesc)) 1460 return PTR_ERR(edesc); 1461 1462 ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req); 1463 if (!ret) { 1464 ret = -EINPROGRESS; 1465 } else { 1466 skcipher_unmap(ctx->qidev, edesc, req); 1467 qi_cache_free(edesc); 1468 } 1469 1470 return ret; 1471 } 1472 1473 static int skcipher_encrypt(struct skcipher_request *req) 1474 { 1475 return skcipher_crypt(req, true); 1476 } 1477 1478 static int skcipher_decrypt(struct skcipher_request *req) 1479 { 1480 return skcipher_crypt(req, false); 1481 } 1482 1483 static struct caam_skcipher_alg driver_algs[] = { 1484 { 1485 .skcipher = { 1486 .base = { 1487 .cra_name = "cbc(aes)", 1488 .cra_driver_name = "cbc-aes-caam-qi", 1489 .cra_blocksize = AES_BLOCK_SIZE, 1490 }, 1491 .setkey = aes_skcipher_setkey, 1492 .encrypt = skcipher_encrypt, 1493 .decrypt = skcipher_decrypt, 1494 .min_keysize = AES_MIN_KEY_SIZE, 1495 .max_keysize = AES_MAX_KEY_SIZE, 1496 .ivsize = AES_BLOCK_SIZE, 1497 }, 1498 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1499 }, 1500 { 1501 .skcipher = { 1502 .base = { 1503 .cra_name = "cbc(des3_ede)", 1504 .cra_driver_name = "cbc-3des-caam-qi", 1505 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1506 }, 1507 .setkey = des3_skcipher_setkey, 1508 .encrypt = skcipher_encrypt, 1509 .decrypt = skcipher_decrypt, 1510 .min_keysize = DES3_EDE_KEY_SIZE, 1511 .max_keysize = DES3_EDE_KEY_SIZE, 1512 .ivsize = DES3_EDE_BLOCK_SIZE, 1513 }, 1514 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1515 }, 1516 { 1517 .skcipher = { 1518 .base = { 1519 .cra_name = "cbc(des)", 1520 .cra_driver_name = "cbc-des-caam-qi", 1521 .cra_blocksize = DES_BLOCK_SIZE, 1522 }, 1523 .setkey = des_skcipher_setkey, 1524 .encrypt = skcipher_encrypt, 1525 .decrypt = skcipher_decrypt, 1526 .min_keysize = DES_KEY_SIZE, 1527 .max_keysize = DES_KEY_SIZE, 1528 .ivsize = DES_BLOCK_SIZE, 1529 }, 1530 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1531 }, 1532 { 1533 .skcipher = { 1534 .base = { 1535 .cra_name = "ctr(aes)", 1536 .cra_driver_name = "ctr-aes-caam-qi", 1537 .cra_blocksize = 1, 1538 }, 1539 .setkey = ctr_skcipher_setkey, 1540 .encrypt = skcipher_encrypt, 1541 .decrypt = skcipher_decrypt, 1542 .min_keysize = AES_MIN_KEY_SIZE, 1543 .max_keysize = AES_MAX_KEY_SIZE, 1544 .ivsize = AES_BLOCK_SIZE, 1545 .chunksize = AES_BLOCK_SIZE, 1546 }, 1547 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | 1548 OP_ALG_AAI_CTR_MOD128, 1549 }, 1550 { 1551 .skcipher = { 1552 .base = { 1553 .cra_name = "rfc3686(ctr(aes))", 1554 .cra_driver_name = "rfc3686-ctr-aes-caam-qi", 1555 .cra_blocksize = 1, 1556 }, 1557 .setkey = rfc3686_skcipher_setkey, 1558 .encrypt = skcipher_encrypt, 1559 .decrypt = skcipher_decrypt, 1560 .min_keysize = AES_MIN_KEY_SIZE + 1561 CTR_RFC3686_NONCE_SIZE, 1562 .max_keysize = AES_MAX_KEY_SIZE + 1563 CTR_RFC3686_NONCE_SIZE, 1564 .ivsize = CTR_RFC3686_IV_SIZE, 1565 .chunksize = AES_BLOCK_SIZE, 1566 }, 1567 .caam = { 1568 .class1_alg_type = OP_ALG_ALGSEL_AES | 1569 OP_ALG_AAI_CTR_MOD128, 1570 .rfc3686 = true, 1571 }, 1572 }, 1573 { 1574 .skcipher = { 1575 .base = { 1576 .cra_name = "xts(aes)", 1577 .cra_driver_name = "xts-aes-caam-qi", 1578 .cra_flags = CRYPTO_ALG_NEED_FALLBACK, 1579 .cra_blocksize = AES_BLOCK_SIZE, 1580 }, 1581 .setkey = xts_skcipher_setkey, 1582 .encrypt = skcipher_encrypt, 1583 .decrypt = skcipher_decrypt, 1584 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1585 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1586 .ivsize = AES_BLOCK_SIZE, 1587 }, 1588 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, 1589 }, 1590 }; 1591 1592 static struct caam_aead_alg driver_aeads[] = { 1593 { 1594 .aead = { 1595 .base = { 1596 .cra_name = "rfc4106(gcm(aes))", 1597 .cra_driver_name = "rfc4106-gcm-aes-caam-qi", 1598 .cra_blocksize = 1, 1599 }, 1600 .setkey = rfc4106_setkey, 1601 .setauthsize = rfc4106_setauthsize, 1602 .encrypt = ipsec_gcm_encrypt, 1603 .decrypt = ipsec_gcm_decrypt, 1604 .ivsize = 8, 1605 .maxauthsize = AES_BLOCK_SIZE, 1606 }, 1607 .caam = { 1608 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1609 .nodkp = true, 1610 }, 1611 }, 1612 { 1613 .aead = { 1614 .base = { 1615 .cra_name = "rfc4543(gcm(aes))", 1616 .cra_driver_name = "rfc4543-gcm-aes-caam-qi", 1617 .cra_blocksize = 1, 1618 }, 1619 .setkey = rfc4543_setkey, 1620 .setauthsize = rfc4543_setauthsize, 1621 .encrypt = ipsec_gcm_encrypt, 1622 .decrypt = ipsec_gcm_decrypt, 1623 .ivsize = 8, 1624 .maxauthsize = AES_BLOCK_SIZE, 1625 }, 1626 .caam = { 1627 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1628 .nodkp = true, 1629 }, 1630 }, 1631 /* Galois Counter Mode */ 1632 { 1633 .aead = { 1634 .base = { 1635 .cra_name = "gcm(aes)", 1636 .cra_driver_name = "gcm-aes-caam-qi", 1637 .cra_blocksize = 1, 1638 }, 1639 .setkey = gcm_setkey, 1640 .setauthsize = gcm_setauthsize, 1641 .encrypt = aead_encrypt, 1642 .decrypt = aead_decrypt, 1643 .ivsize = 12, 1644 .maxauthsize = AES_BLOCK_SIZE, 1645 }, 1646 .caam = { 1647 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1648 .nodkp = true, 1649 } 1650 }, 1651 /* single-pass ipsec_esp descriptor */ 1652 { 1653 .aead = { 1654 .base = { 1655 .cra_name = "authenc(hmac(md5),cbc(aes))", 1656 .cra_driver_name = "authenc-hmac-md5-" 1657 "cbc-aes-caam-qi", 1658 .cra_blocksize = AES_BLOCK_SIZE, 1659 }, 1660 .setkey = aead_setkey, 1661 .setauthsize = aead_setauthsize, 1662 .encrypt = aead_encrypt, 1663 .decrypt = aead_decrypt, 1664 .ivsize = AES_BLOCK_SIZE, 1665 .maxauthsize = MD5_DIGEST_SIZE, 1666 }, 1667 .caam = { 1668 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1669 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1670 OP_ALG_AAI_HMAC_PRECOMP, 1671 } 1672 }, 1673 { 1674 .aead = { 1675 .base = { 1676 .cra_name = "echainiv(authenc(hmac(md5)," 1677 "cbc(aes)))", 1678 .cra_driver_name = "echainiv-authenc-hmac-md5-" 1679 "cbc-aes-caam-qi", 1680 .cra_blocksize = AES_BLOCK_SIZE, 1681 }, 1682 .setkey = aead_setkey, 1683 .setauthsize = aead_setauthsize, 1684 .encrypt = aead_encrypt, 1685 .decrypt = aead_decrypt, 1686 .ivsize = AES_BLOCK_SIZE, 1687 .maxauthsize = MD5_DIGEST_SIZE, 1688 }, 1689 .caam = { 1690 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1691 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1692 OP_ALG_AAI_HMAC_PRECOMP, 1693 .geniv = true, 1694 } 1695 }, 1696 { 1697 .aead = { 1698 .base = { 1699 .cra_name = "authenc(hmac(sha1),cbc(aes))", 1700 .cra_driver_name = "authenc-hmac-sha1-" 1701 "cbc-aes-caam-qi", 1702 .cra_blocksize = AES_BLOCK_SIZE, 1703 }, 1704 .setkey = aead_setkey, 1705 .setauthsize = aead_setauthsize, 1706 .encrypt = aead_encrypt, 1707 .decrypt = aead_decrypt, 1708 .ivsize = AES_BLOCK_SIZE, 1709 .maxauthsize = SHA1_DIGEST_SIZE, 1710 }, 1711 .caam = { 1712 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1713 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1714 OP_ALG_AAI_HMAC_PRECOMP, 1715 } 1716 }, 1717 { 1718 .aead = { 1719 .base = { 1720 .cra_name = "echainiv(authenc(hmac(sha1)," 1721 "cbc(aes)))", 1722 .cra_driver_name = "echainiv-authenc-" 1723 "hmac-sha1-cbc-aes-caam-qi", 1724 .cra_blocksize = AES_BLOCK_SIZE, 1725 }, 1726 .setkey = aead_setkey, 1727 .setauthsize = aead_setauthsize, 1728 .encrypt = aead_encrypt, 1729 .decrypt = aead_decrypt, 1730 .ivsize = AES_BLOCK_SIZE, 1731 .maxauthsize = SHA1_DIGEST_SIZE, 1732 }, 1733 .caam = { 1734 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1735 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1736 OP_ALG_AAI_HMAC_PRECOMP, 1737 .geniv = true, 1738 }, 1739 }, 1740 { 1741 .aead = { 1742 .base = { 1743 .cra_name = "authenc(hmac(sha224),cbc(aes))", 1744 .cra_driver_name = "authenc-hmac-sha224-" 1745 "cbc-aes-caam-qi", 1746 .cra_blocksize = AES_BLOCK_SIZE, 1747 }, 1748 .setkey = aead_setkey, 1749 .setauthsize = aead_setauthsize, 1750 .encrypt = aead_encrypt, 1751 .decrypt = aead_decrypt, 1752 .ivsize = AES_BLOCK_SIZE, 1753 .maxauthsize = SHA224_DIGEST_SIZE, 1754 }, 1755 .caam = { 1756 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1757 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1758 OP_ALG_AAI_HMAC_PRECOMP, 1759 } 1760 }, 1761 { 1762 .aead = { 1763 .base = { 1764 .cra_name = "echainiv(authenc(hmac(sha224)," 1765 "cbc(aes)))", 1766 .cra_driver_name = "echainiv-authenc-" 1767 "hmac-sha224-cbc-aes-caam-qi", 1768 .cra_blocksize = AES_BLOCK_SIZE, 1769 }, 1770 .setkey = aead_setkey, 1771 .setauthsize = aead_setauthsize, 1772 .encrypt = aead_encrypt, 1773 .decrypt = aead_decrypt, 1774 .ivsize = AES_BLOCK_SIZE, 1775 .maxauthsize = SHA224_DIGEST_SIZE, 1776 }, 1777 .caam = { 1778 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1779 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1780 OP_ALG_AAI_HMAC_PRECOMP, 1781 .geniv = true, 1782 } 1783 }, 1784 { 1785 .aead = { 1786 .base = { 1787 .cra_name = "authenc(hmac(sha256),cbc(aes))", 1788 .cra_driver_name = "authenc-hmac-sha256-" 1789 "cbc-aes-caam-qi", 1790 .cra_blocksize = AES_BLOCK_SIZE, 1791 }, 1792 .setkey = aead_setkey, 1793 .setauthsize = aead_setauthsize, 1794 .encrypt = aead_encrypt, 1795 .decrypt = aead_decrypt, 1796 .ivsize = AES_BLOCK_SIZE, 1797 .maxauthsize = SHA256_DIGEST_SIZE, 1798 }, 1799 .caam = { 1800 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1801 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1802 OP_ALG_AAI_HMAC_PRECOMP, 1803 } 1804 }, 1805 { 1806 .aead = { 1807 .base = { 1808 .cra_name = "echainiv(authenc(hmac(sha256)," 1809 "cbc(aes)))", 1810 .cra_driver_name = "echainiv-authenc-" 1811 "hmac-sha256-cbc-aes-" 1812 "caam-qi", 1813 .cra_blocksize = AES_BLOCK_SIZE, 1814 }, 1815 .setkey = aead_setkey, 1816 .setauthsize = aead_setauthsize, 1817 .encrypt = aead_encrypt, 1818 .decrypt = aead_decrypt, 1819 .ivsize = AES_BLOCK_SIZE, 1820 .maxauthsize = SHA256_DIGEST_SIZE, 1821 }, 1822 .caam = { 1823 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1824 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1825 OP_ALG_AAI_HMAC_PRECOMP, 1826 .geniv = true, 1827 } 1828 }, 1829 { 1830 .aead = { 1831 .base = { 1832 .cra_name = "authenc(hmac(sha384),cbc(aes))", 1833 .cra_driver_name = "authenc-hmac-sha384-" 1834 "cbc-aes-caam-qi", 1835 .cra_blocksize = AES_BLOCK_SIZE, 1836 }, 1837 .setkey = aead_setkey, 1838 .setauthsize = aead_setauthsize, 1839 .encrypt = aead_encrypt, 1840 .decrypt = aead_decrypt, 1841 .ivsize = AES_BLOCK_SIZE, 1842 .maxauthsize = SHA384_DIGEST_SIZE, 1843 }, 1844 .caam = { 1845 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1846 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 1847 OP_ALG_AAI_HMAC_PRECOMP, 1848 } 1849 }, 1850 { 1851 .aead = { 1852 .base = { 1853 .cra_name = "echainiv(authenc(hmac(sha384)," 1854 "cbc(aes)))", 1855 .cra_driver_name = "echainiv-authenc-" 1856 "hmac-sha384-cbc-aes-" 1857 "caam-qi", 1858 .cra_blocksize = AES_BLOCK_SIZE, 1859 }, 1860 .setkey = aead_setkey, 1861 .setauthsize = aead_setauthsize, 1862 .encrypt = aead_encrypt, 1863 .decrypt = aead_decrypt, 1864 .ivsize = AES_BLOCK_SIZE, 1865 .maxauthsize = SHA384_DIGEST_SIZE, 1866 }, 1867 .caam = { 1868 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1869 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 1870 OP_ALG_AAI_HMAC_PRECOMP, 1871 .geniv = true, 1872 } 1873 }, 1874 { 1875 .aead = { 1876 .base = { 1877 .cra_name = "authenc(hmac(sha512),cbc(aes))", 1878 .cra_driver_name = "authenc-hmac-sha512-" 1879 "cbc-aes-caam-qi", 1880 .cra_blocksize = AES_BLOCK_SIZE, 1881 }, 1882 .setkey = aead_setkey, 1883 .setauthsize = aead_setauthsize, 1884 .encrypt = aead_encrypt, 1885 .decrypt = aead_decrypt, 1886 .ivsize = AES_BLOCK_SIZE, 1887 .maxauthsize = SHA512_DIGEST_SIZE, 1888 }, 1889 .caam = { 1890 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1891 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 1892 OP_ALG_AAI_HMAC_PRECOMP, 1893 } 1894 }, 1895 { 1896 .aead = { 1897 .base = { 1898 .cra_name = "echainiv(authenc(hmac(sha512)," 1899 "cbc(aes)))", 1900 .cra_driver_name = "echainiv-authenc-" 1901 "hmac-sha512-cbc-aes-" 1902 "caam-qi", 1903 .cra_blocksize = AES_BLOCK_SIZE, 1904 }, 1905 .setkey = aead_setkey, 1906 .setauthsize = aead_setauthsize, 1907 .encrypt = aead_encrypt, 1908 .decrypt = aead_decrypt, 1909 .ivsize = AES_BLOCK_SIZE, 1910 .maxauthsize = SHA512_DIGEST_SIZE, 1911 }, 1912 .caam = { 1913 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1914 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 1915 OP_ALG_AAI_HMAC_PRECOMP, 1916 .geniv = true, 1917 } 1918 }, 1919 { 1920 .aead = { 1921 .base = { 1922 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 1923 .cra_driver_name = "authenc-hmac-md5-" 1924 "cbc-des3_ede-caam-qi", 1925 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1926 }, 1927 .setkey = des3_aead_setkey, 1928 .setauthsize = aead_setauthsize, 1929 .encrypt = aead_encrypt, 1930 .decrypt = aead_decrypt, 1931 .ivsize = DES3_EDE_BLOCK_SIZE, 1932 .maxauthsize = MD5_DIGEST_SIZE, 1933 }, 1934 .caam = { 1935 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1936 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1937 OP_ALG_AAI_HMAC_PRECOMP, 1938 } 1939 }, 1940 { 1941 .aead = { 1942 .base = { 1943 .cra_name = "echainiv(authenc(hmac(md5)," 1944 "cbc(des3_ede)))", 1945 .cra_driver_name = "echainiv-authenc-hmac-md5-" 1946 "cbc-des3_ede-caam-qi", 1947 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1948 }, 1949 .setkey = des3_aead_setkey, 1950 .setauthsize = aead_setauthsize, 1951 .encrypt = aead_encrypt, 1952 .decrypt = aead_decrypt, 1953 .ivsize = DES3_EDE_BLOCK_SIZE, 1954 .maxauthsize = MD5_DIGEST_SIZE, 1955 }, 1956 .caam = { 1957 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1958 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1959 OP_ALG_AAI_HMAC_PRECOMP, 1960 .geniv = true, 1961 } 1962 }, 1963 { 1964 .aead = { 1965 .base = { 1966 .cra_name = "authenc(hmac(sha1)," 1967 "cbc(des3_ede))", 1968 .cra_driver_name = "authenc-hmac-sha1-" 1969 "cbc-des3_ede-caam-qi", 1970 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1971 }, 1972 .setkey = des3_aead_setkey, 1973 .setauthsize = aead_setauthsize, 1974 .encrypt = aead_encrypt, 1975 .decrypt = aead_decrypt, 1976 .ivsize = DES3_EDE_BLOCK_SIZE, 1977 .maxauthsize = SHA1_DIGEST_SIZE, 1978 }, 1979 .caam = { 1980 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1981 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1982 OP_ALG_AAI_HMAC_PRECOMP, 1983 }, 1984 }, 1985 { 1986 .aead = { 1987 .base = { 1988 .cra_name = "echainiv(authenc(hmac(sha1)," 1989 "cbc(des3_ede)))", 1990 .cra_driver_name = "echainiv-authenc-" 1991 "hmac-sha1-" 1992 "cbc-des3_ede-caam-qi", 1993 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1994 }, 1995 .setkey = des3_aead_setkey, 1996 .setauthsize = aead_setauthsize, 1997 .encrypt = aead_encrypt, 1998 .decrypt = aead_decrypt, 1999 .ivsize = DES3_EDE_BLOCK_SIZE, 2000 .maxauthsize = SHA1_DIGEST_SIZE, 2001 }, 2002 .caam = { 2003 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2004 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2005 OP_ALG_AAI_HMAC_PRECOMP, 2006 .geniv = true, 2007 } 2008 }, 2009 { 2010 .aead = { 2011 .base = { 2012 .cra_name = "authenc(hmac(sha224)," 2013 "cbc(des3_ede))", 2014 .cra_driver_name = "authenc-hmac-sha224-" 2015 "cbc-des3_ede-caam-qi", 2016 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2017 }, 2018 .setkey = des3_aead_setkey, 2019 .setauthsize = aead_setauthsize, 2020 .encrypt = aead_encrypt, 2021 .decrypt = aead_decrypt, 2022 .ivsize = DES3_EDE_BLOCK_SIZE, 2023 .maxauthsize = SHA224_DIGEST_SIZE, 2024 }, 2025 .caam = { 2026 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2027 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2028 OP_ALG_AAI_HMAC_PRECOMP, 2029 }, 2030 }, 2031 { 2032 .aead = { 2033 .base = { 2034 .cra_name = "echainiv(authenc(hmac(sha224)," 2035 "cbc(des3_ede)))", 2036 .cra_driver_name = "echainiv-authenc-" 2037 "hmac-sha224-" 2038 "cbc-des3_ede-caam-qi", 2039 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2040 }, 2041 .setkey = des3_aead_setkey, 2042 .setauthsize = aead_setauthsize, 2043 .encrypt = aead_encrypt, 2044 .decrypt = aead_decrypt, 2045 .ivsize = DES3_EDE_BLOCK_SIZE, 2046 .maxauthsize = SHA224_DIGEST_SIZE, 2047 }, 2048 .caam = { 2049 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2050 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2051 OP_ALG_AAI_HMAC_PRECOMP, 2052 .geniv = true, 2053 } 2054 }, 2055 { 2056 .aead = { 2057 .base = { 2058 .cra_name = "authenc(hmac(sha256)," 2059 "cbc(des3_ede))", 2060 .cra_driver_name = "authenc-hmac-sha256-" 2061 "cbc-des3_ede-caam-qi", 2062 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2063 }, 2064 .setkey = des3_aead_setkey, 2065 .setauthsize = aead_setauthsize, 2066 .encrypt = aead_encrypt, 2067 .decrypt = aead_decrypt, 2068 .ivsize = DES3_EDE_BLOCK_SIZE, 2069 .maxauthsize = SHA256_DIGEST_SIZE, 2070 }, 2071 .caam = { 2072 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2073 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2074 OP_ALG_AAI_HMAC_PRECOMP, 2075 }, 2076 }, 2077 { 2078 .aead = { 2079 .base = { 2080 .cra_name = "echainiv(authenc(hmac(sha256)," 2081 "cbc(des3_ede)))", 2082 .cra_driver_name = "echainiv-authenc-" 2083 "hmac-sha256-" 2084 "cbc-des3_ede-caam-qi", 2085 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2086 }, 2087 .setkey = des3_aead_setkey, 2088 .setauthsize = aead_setauthsize, 2089 .encrypt = aead_encrypt, 2090 .decrypt = aead_decrypt, 2091 .ivsize = DES3_EDE_BLOCK_SIZE, 2092 .maxauthsize = SHA256_DIGEST_SIZE, 2093 }, 2094 .caam = { 2095 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2096 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2097 OP_ALG_AAI_HMAC_PRECOMP, 2098 .geniv = true, 2099 } 2100 }, 2101 { 2102 .aead = { 2103 .base = { 2104 .cra_name = "authenc(hmac(sha384)," 2105 "cbc(des3_ede))", 2106 .cra_driver_name = "authenc-hmac-sha384-" 2107 "cbc-des3_ede-caam-qi", 2108 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2109 }, 2110 .setkey = des3_aead_setkey, 2111 .setauthsize = aead_setauthsize, 2112 .encrypt = aead_encrypt, 2113 .decrypt = aead_decrypt, 2114 .ivsize = DES3_EDE_BLOCK_SIZE, 2115 .maxauthsize = SHA384_DIGEST_SIZE, 2116 }, 2117 .caam = { 2118 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2119 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2120 OP_ALG_AAI_HMAC_PRECOMP, 2121 }, 2122 }, 2123 { 2124 .aead = { 2125 .base = { 2126 .cra_name = "echainiv(authenc(hmac(sha384)," 2127 "cbc(des3_ede)))", 2128 .cra_driver_name = "echainiv-authenc-" 2129 "hmac-sha384-" 2130 "cbc-des3_ede-caam-qi", 2131 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2132 }, 2133 .setkey = des3_aead_setkey, 2134 .setauthsize = aead_setauthsize, 2135 .encrypt = aead_encrypt, 2136 .decrypt = aead_decrypt, 2137 .ivsize = DES3_EDE_BLOCK_SIZE, 2138 .maxauthsize = SHA384_DIGEST_SIZE, 2139 }, 2140 .caam = { 2141 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2142 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2143 OP_ALG_AAI_HMAC_PRECOMP, 2144 .geniv = true, 2145 } 2146 }, 2147 { 2148 .aead = { 2149 .base = { 2150 .cra_name = "authenc(hmac(sha512)," 2151 "cbc(des3_ede))", 2152 .cra_driver_name = "authenc-hmac-sha512-" 2153 "cbc-des3_ede-caam-qi", 2154 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2155 }, 2156 .setkey = des3_aead_setkey, 2157 .setauthsize = aead_setauthsize, 2158 .encrypt = aead_encrypt, 2159 .decrypt = aead_decrypt, 2160 .ivsize = DES3_EDE_BLOCK_SIZE, 2161 .maxauthsize = SHA512_DIGEST_SIZE, 2162 }, 2163 .caam = { 2164 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2165 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2166 OP_ALG_AAI_HMAC_PRECOMP, 2167 }, 2168 }, 2169 { 2170 .aead = { 2171 .base = { 2172 .cra_name = "echainiv(authenc(hmac(sha512)," 2173 "cbc(des3_ede)))", 2174 .cra_driver_name = "echainiv-authenc-" 2175 "hmac-sha512-" 2176 "cbc-des3_ede-caam-qi", 2177 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2178 }, 2179 .setkey = des3_aead_setkey, 2180 .setauthsize = aead_setauthsize, 2181 .encrypt = aead_encrypt, 2182 .decrypt = aead_decrypt, 2183 .ivsize = DES3_EDE_BLOCK_SIZE, 2184 .maxauthsize = SHA512_DIGEST_SIZE, 2185 }, 2186 .caam = { 2187 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2188 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2189 OP_ALG_AAI_HMAC_PRECOMP, 2190 .geniv = true, 2191 } 2192 }, 2193 { 2194 .aead = { 2195 .base = { 2196 .cra_name = "authenc(hmac(md5),cbc(des))", 2197 .cra_driver_name = "authenc-hmac-md5-" 2198 "cbc-des-caam-qi", 2199 .cra_blocksize = DES_BLOCK_SIZE, 2200 }, 2201 .setkey = aead_setkey, 2202 .setauthsize = aead_setauthsize, 2203 .encrypt = aead_encrypt, 2204 .decrypt = aead_decrypt, 2205 .ivsize = DES_BLOCK_SIZE, 2206 .maxauthsize = MD5_DIGEST_SIZE, 2207 }, 2208 .caam = { 2209 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2210 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2211 OP_ALG_AAI_HMAC_PRECOMP, 2212 }, 2213 }, 2214 { 2215 .aead = { 2216 .base = { 2217 .cra_name = "echainiv(authenc(hmac(md5)," 2218 "cbc(des)))", 2219 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2220 "cbc-des-caam-qi", 2221 .cra_blocksize = DES_BLOCK_SIZE, 2222 }, 2223 .setkey = aead_setkey, 2224 .setauthsize = aead_setauthsize, 2225 .encrypt = aead_encrypt, 2226 .decrypt = aead_decrypt, 2227 .ivsize = DES_BLOCK_SIZE, 2228 .maxauthsize = MD5_DIGEST_SIZE, 2229 }, 2230 .caam = { 2231 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2232 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2233 OP_ALG_AAI_HMAC_PRECOMP, 2234 .geniv = true, 2235 } 2236 }, 2237 { 2238 .aead = { 2239 .base = { 2240 .cra_name = "authenc(hmac(sha1),cbc(des))", 2241 .cra_driver_name = "authenc-hmac-sha1-" 2242 "cbc-des-caam-qi", 2243 .cra_blocksize = DES_BLOCK_SIZE, 2244 }, 2245 .setkey = aead_setkey, 2246 .setauthsize = aead_setauthsize, 2247 .encrypt = aead_encrypt, 2248 .decrypt = aead_decrypt, 2249 .ivsize = DES_BLOCK_SIZE, 2250 .maxauthsize = SHA1_DIGEST_SIZE, 2251 }, 2252 .caam = { 2253 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2254 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2255 OP_ALG_AAI_HMAC_PRECOMP, 2256 }, 2257 }, 2258 { 2259 .aead = { 2260 .base = { 2261 .cra_name = "echainiv(authenc(hmac(sha1)," 2262 "cbc(des)))", 2263 .cra_driver_name = "echainiv-authenc-" 2264 "hmac-sha1-cbc-des-caam-qi", 2265 .cra_blocksize = DES_BLOCK_SIZE, 2266 }, 2267 .setkey = aead_setkey, 2268 .setauthsize = aead_setauthsize, 2269 .encrypt = aead_encrypt, 2270 .decrypt = aead_decrypt, 2271 .ivsize = DES_BLOCK_SIZE, 2272 .maxauthsize = SHA1_DIGEST_SIZE, 2273 }, 2274 .caam = { 2275 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2276 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2277 OP_ALG_AAI_HMAC_PRECOMP, 2278 .geniv = true, 2279 } 2280 }, 2281 { 2282 .aead = { 2283 .base = { 2284 .cra_name = "authenc(hmac(sha224),cbc(des))", 2285 .cra_driver_name = "authenc-hmac-sha224-" 2286 "cbc-des-caam-qi", 2287 .cra_blocksize = DES_BLOCK_SIZE, 2288 }, 2289 .setkey = aead_setkey, 2290 .setauthsize = aead_setauthsize, 2291 .encrypt = aead_encrypt, 2292 .decrypt = aead_decrypt, 2293 .ivsize = DES_BLOCK_SIZE, 2294 .maxauthsize = SHA224_DIGEST_SIZE, 2295 }, 2296 .caam = { 2297 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2298 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2299 OP_ALG_AAI_HMAC_PRECOMP, 2300 }, 2301 }, 2302 { 2303 .aead = { 2304 .base = { 2305 .cra_name = "echainiv(authenc(hmac(sha224)," 2306 "cbc(des)))", 2307 .cra_driver_name = "echainiv-authenc-" 2308 "hmac-sha224-cbc-des-" 2309 "caam-qi", 2310 .cra_blocksize = DES_BLOCK_SIZE, 2311 }, 2312 .setkey = aead_setkey, 2313 .setauthsize = aead_setauthsize, 2314 .encrypt = aead_encrypt, 2315 .decrypt = aead_decrypt, 2316 .ivsize = DES_BLOCK_SIZE, 2317 .maxauthsize = SHA224_DIGEST_SIZE, 2318 }, 2319 .caam = { 2320 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2321 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2322 OP_ALG_AAI_HMAC_PRECOMP, 2323 .geniv = true, 2324 } 2325 }, 2326 { 2327 .aead = { 2328 .base = { 2329 .cra_name = "authenc(hmac(sha256),cbc(des))", 2330 .cra_driver_name = "authenc-hmac-sha256-" 2331 "cbc-des-caam-qi", 2332 .cra_blocksize = DES_BLOCK_SIZE, 2333 }, 2334 .setkey = aead_setkey, 2335 .setauthsize = aead_setauthsize, 2336 .encrypt = aead_encrypt, 2337 .decrypt = aead_decrypt, 2338 .ivsize = DES_BLOCK_SIZE, 2339 .maxauthsize = SHA256_DIGEST_SIZE, 2340 }, 2341 .caam = { 2342 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2343 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2344 OP_ALG_AAI_HMAC_PRECOMP, 2345 }, 2346 }, 2347 { 2348 .aead = { 2349 .base = { 2350 .cra_name = "echainiv(authenc(hmac(sha256)," 2351 "cbc(des)))", 2352 .cra_driver_name = "echainiv-authenc-" 2353 "hmac-sha256-cbc-des-" 2354 "caam-qi", 2355 .cra_blocksize = DES_BLOCK_SIZE, 2356 }, 2357 .setkey = aead_setkey, 2358 .setauthsize = aead_setauthsize, 2359 .encrypt = aead_encrypt, 2360 .decrypt = aead_decrypt, 2361 .ivsize = DES_BLOCK_SIZE, 2362 .maxauthsize = SHA256_DIGEST_SIZE, 2363 }, 2364 .caam = { 2365 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2366 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2367 OP_ALG_AAI_HMAC_PRECOMP, 2368 .geniv = true, 2369 }, 2370 }, 2371 { 2372 .aead = { 2373 .base = { 2374 .cra_name = "authenc(hmac(sha384),cbc(des))", 2375 .cra_driver_name = "authenc-hmac-sha384-" 2376 "cbc-des-caam-qi", 2377 .cra_blocksize = DES_BLOCK_SIZE, 2378 }, 2379 .setkey = aead_setkey, 2380 .setauthsize = aead_setauthsize, 2381 .encrypt = aead_encrypt, 2382 .decrypt = aead_decrypt, 2383 .ivsize = DES_BLOCK_SIZE, 2384 .maxauthsize = SHA384_DIGEST_SIZE, 2385 }, 2386 .caam = { 2387 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2388 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2389 OP_ALG_AAI_HMAC_PRECOMP, 2390 }, 2391 }, 2392 { 2393 .aead = { 2394 .base = { 2395 .cra_name = "echainiv(authenc(hmac(sha384)," 2396 "cbc(des)))", 2397 .cra_driver_name = "echainiv-authenc-" 2398 "hmac-sha384-cbc-des-" 2399 "caam-qi", 2400 .cra_blocksize = DES_BLOCK_SIZE, 2401 }, 2402 .setkey = aead_setkey, 2403 .setauthsize = aead_setauthsize, 2404 .encrypt = aead_encrypt, 2405 .decrypt = aead_decrypt, 2406 .ivsize = DES_BLOCK_SIZE, 2407 .maxauthsize = SHA384_DIGEST_SIZE, 2408 }, 2409 .caam = { 2410 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2411 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2412 OP_ALG_AAI_HMAC_PRECOMP, 2413 .geniv = true, 2414 } 2415 }, 2416 { 2417 .aead = { 2418 .base = { 2419 .cra_name = "authenc(hmac(sha512),cbc(des))", 2420 .cra_driver_name = "authenc-hmac-sha512-" 2421 "cbc-des-caam-qi", 2422 .cra_blocksize = DES_BLOCK_SIZE, 2423 }, 2424 .setkey = aead_setkey, 2425 .setauthsize = aead_setauthsize, 2426 .encrypt = aead_encrypt, 2427 .decrypt = aead_decrypt, 2428 .ivsize = DES_BLOCK_SIZE, 2429 .maxauthsize = SHA512_DIGEST_SIZE, 2430 }, 2431 .caam = { 2432 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2433 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2434 OP_ALG_AAI_HMAC_PRECOMP, 2435 } 2436 }, 2437 { 2438 .aead = { 2439 .base = { 2440 .cra_name = "echainiv(authenc(hmac(sha512)," 2441 "cbc(des)))", 2442 .cra_driver_name = "echainiv-authenc-" 2443 "hmac-sha512-cbc-des-" 2444 "caam-qi", 2445 .cra_blocksize = DES_BLOCK_SIZE, 2446 }, 2447 .setkey = aead_setkey, 2448 .setauthsize = aead_setauthsize, 2449 .encrypt = aead_encrypt, 2450 .decrypt = aead_decrypt, 2451 .ivsize = DES_BLOCK_SIZE, 2452 .maxauthsize = SHA512_DIGEST_SIZE, 2453 }, 2454 .caam = { 2455 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2456 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2457 OP_ALG_AAI_HMAC_PRECOMP, 2458 .geniv = true, 2459 } 2460 }, 2461 }; 2462 2463 static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam, 2464 bool uses_dkp) 2465 { 2466 struct caam_drv_private *priv; 2467 struct device *dev; 2468 2469 /* 2470 * distribute tfms across job rings to ensure in-order 2471 * crypto request processing per tfm 2472 */ 2473 ctx->jrdev = caam_jr_alloc(); 2474 if (IS_ERR(ctx->jrdev)) { 2475 pr_err("Job Ring Device allocation for transform failed\n"); 2476 return PTR_ERR(ctx->jrdev); 2477 } 2478 2479 dev = ctx->jrdev->parent; 2480 priv = dev_get_drvdata(dev); 2481 if (priv->era >= 6 && uses_dkp) 2482 ctx->dir = DMA_BIDIRECTIONAL; 2483 else 2484 ctx->dir = DMA_TO_DEVICE; 2485 2486 ctx->key_dma = dma_map_single(dev, ctx->key, sizeof(ctx->key), 2487 ctx->dir); 2488 if (dma_mapping_error(dev, ctx->key_dma)) { 2489 dev_err(dev, "unable to map key\n"); 2490 caam_jr_free(ctx->jrdev); 2491 return -ENOMEM; 2492 } 2493 2494 /* copy descriptor header template value */ 2495 ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; 2496 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; 2497 2498 ctx->qidev = dev; 2499 2500 spin_lock_init(&ctx->lock); 2501 ctx->drv_ctx[ENCRYPT] = NULL; 2502 ctx->drv_ctx[DECRYPT] = NULL; 2503 2504 return 0; 2505 } 2506 2507 static int caam_cra_init(struct crypto_skcipher *tfm) 2508 { 2509 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 2510 struct caam_skcipher_alg *caam_alg = 2511 container_of(alg, typeof(*caam_alg), skcipher); 2512 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm); 2513 u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; 2514 int ret = 0; 2515 2516 if (alg_aai == OP_ALG_AAI_XTS) { 2517 const char *tfm_name = crypto_tfm_alg_name(&tfm->base); 2518 struct crypto_skcipher *fallback; 2519 2520 fallback = crypto_alloc_skcipher(tfm_name, 0, 2521 CRYPTO_ALG_NEED_FALLBACK); 2522 if (IS_ERR(fallback)) { 2523 pr_err("Failed to allocate %s fallback: %ld\n", 2524 tfm_name, PTR_ERR(fallback)); 2525 return PTR_ERR(fallback); 2526 } 2527 2528 ctx->fallback = fallback; 2529 crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx) + 2530 crypto_skcipher_reqsize(fallback)); 2531 } 2532 2533 ret = caam_init_common(ctx, &caam_alg->caam, false); 2534 if (ret && ctx->fallback) 2535 crypto_free_skcipher(ctx->fallback); 2536 2537 return ret; 2538 } 2539 2540 static int caam_aead_init(struct crypto_aead *tfm) 2541 { 2542 struct aead_alg *alg = crypto_aead_alg(tfm); 2543 struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg), 2544 aead); 2545 struct caam_ctx *ctx = crypto_aead_ctx_dma(tfm); 2546 2547 return caam_init_common(ctx, &caam_alg->caam, !caam_alg->caam.nodkp); 2548 } 2549 2550 static void caam_exit_common(struct caam_ctx *ctx) 2551 { 2552 caam_drv_ctx_rel(ctx->drv_ctx[ENCRYPT]); 2553 caam_drv_ctx_rel(ctx->drv_ctx[DECRYPT]); 2554 2555 dma_unmap_single(ctx->jrdev->parent, ctx->key_dma, sizeof(ctx->key), 2556 ctx->dir); 2557 2558 caam_jr_free(ctx->jrdev); 2559 } 2560 2561 static void caam_cra_exit(struct crypto_skcipher *tfm) 2562 { 2563 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm); 2564 2565 if (ctx->fallback) 2566 crypto_free_skcipher(ctx->fallback); 2567 caam_exit_common(ctx); 2568 } 2569 2570 static void caam_aead_exit(struct crypto_aead *tfm) 2571 { 2572 caam_exit_common(crypto_aead_ctx_dma(tfm)); 2573 } 2574 2575 void caam_qi_algapi_exit(void) 2576 { 2577 int i; 2578 2579 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 2580 struct caam_aead_alg *t_alg = driver_aeads + i; 2581 2582 if (t_alg->registered) 2583 crypto_unregister_aead(&t_alg->aead); 2584 } 2585 2586 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 2587 struct caam_skcipher_alg *t_alg = driver_algs + i; 2588 2589 if (t_alg->registered) 2590 crypto_unregister_skcipher(&t_alg->skcipher); 2591 } 2592 } 2593 2594 static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg) 2595 { 2596 struct skcipher_alg *alg = &t_alg->skcipher; 2597 2598 alg->base.cra_module = THIS_MODULE; 2599 alg->base.cra_priority = CAAM_CRA_PRIORITY; 2600 alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding(); 2601 alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | 2602 CRYPTO_ALG_KERN_DRIVER_ONLY); 2603 2604 alg->init = caam_cra_init; 2605 alg->exit = caam_cra_exit; 2606 } 2607 2608 static void caam_aead_alg_init(struct caam_aead_alg *t_alg) 2609 { 2610 struct aead_alg *alg = &t_alg->aead; 2611 2612 alg->base.cra_module = THIS_MODULE; 2613 alg->base.cra_priority = CAAM_CRA_PRIORITY; 2614 alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding(); 2615 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | 2616 CRYPTO_ALG_KERN_DRIVER_ONLY; 2617 2618 alg->init = caam_aead_init; 2619 alg->exit = caam_aead_exit; 2620 } 2621 2622 int caam_qi_algapi_init(struct device *ctrldev) 2623 { 2624 struct caam_drv_private *priv = dev_get_drvdata(ctrldev); 2625 int i = 0, err = 0; 2626 u32 aes_vid, aes_inst, des_inst, md_vid, md_inst; 2627 unsigned int md_limit = SHA512_DIGEST_SIZE; 2628 bool registered = false; 2629 2630 /* Make sure this runs only on (DPAA 1.x) QI */ 2631 if (!priv->qi_present || caam_dpaa2) 2632 return 0; 2633 2634 /* 2635 * Register crypto algorithms the device supports. 2636 * First, detect presence and attributes of DES, AES, and MD blocks. 2637 */ 2638 if (priv->era < 10) { 2639 u32 cha_vid, cha_inst; 2640 2641 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls); 2642 aes_vid = cha_vid & CHA_ID_LS_AES_MASK; 2643 md_vid = (cha_vid & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; 2644 2645 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls); 2646 des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >> 2647 CHA_ID_LS_DES_SHIFT; 2648 aes_inst = cha_inst & CHA_ID_LS_AES_MASK; 2649 md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; 2650 } else { 2651 u32 aesa, mdha; 2652 2653 aesa = rd_reg32(&priv->ctrl->vreg.aesa); 2654 mdha = rd_reg32(&priv->ctrl->vreg.mdha); 2655 2656 aes_vid = (aesa & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT; 2657 md_vid = (mdha & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT; 2658 2659 des_inst = rd_reg32(&priv->ctrl->vreg.desa) & CHA_VER_NUM_MASK; 2660 aes_inst = aesa & CHA_VER_NUM_MASK; 2661 md_inst = mdha & CHA_VER_NUM_MASK; 2662 } 2663 2664 /* If MD is present, limit digest size based on LP256 */ 2665 if (md_inst && md_vid == CHA_VER_VID_MD_LP256) 2666 md_limit = SHA256_DIGEST_SIZE; 2667 2668 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 2669 struct caam_skcipher_alg *t_alg = driver_algs + i; 2670 u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK; 2671 2672 /* Skip DES algorithms if not supported by device */ 2673 if (!des_inst && 2674 ((alg_sel == OP_ALG_ALGSEL_3DES) || 2675 (alg_sel == OP_ALG_ALGSEL_DES))) 2676 continue; 2677 2678 /* Skip AES algorithms if not supported by device */ 2679 if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES)) 2680 continue; 2681 2682 caam_skcipher_alg_init(t_alg); 2683 2684 err = crypto_register_skcipher(&t_alg->skcipher); 2685 if (err) { 2686 dev_warn(ctrldev, "%s alg registration failed\n", 2687 t_alg->skcipher.base.cra_driver_name); 2688 continue; 2689 } 2690 2691 t_alg->registered = true; 2692 registered = true; 2693 } 2694 2695 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 2696 struct caam_aead_alg *t_alg = driver_aeads + i; 2697 u32 c1_alg_sel = t_alg->caam.class1_alg_type & 2698 OP_ALG_ALGSEL_MASK; 2699 u32 c2_alg_sel = t_alg->caam.class2_alg_type & 2700 OP_ALG_ALGSEL_MASK; 2701 u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; 2702 2703 /* Skip DES algorithms if not supported by device */ 2704 if (!des_inst && 2705 ((c1_alg_sel == OP_ALG_ALGSEL_3DES) || 2706 (c1_alg_sel == OP_ALG_ALGSEL_DES))) 2707 continue; 2708 2709 /* Skip AES algorithms if not supported by device */ 2710 if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES)) 2711 continue; 2712 2713 /* 2714 * Check support for AES algorithms not available 2715 * on LP devices. 2716 */ 2717 if (aes_vid == CHA_VER_VID_AES_LP && alg_aai == OP_ALG_AAI_GCM) 2718 continue; 2719 2720 /* 2721 * Skip algorithms requiring message digests 2722 * if MD or MD size is not supported by device. 2723 */ 2724 if (c2_alg_sel && 2725 (!md_inst || (t_alg->aead.maxauthsize > md_limit))) 2726 continue; 2727 2728 caam_aead_alg_init(t_alg); 2729 2730 err = crypto_register_aead(&t_alg->aead); 2731 if (err) { 2732 pr_warn("%s alg registration failed\n", 2733 t_alg->aead.base.cra_driver_name); 2734 continue; 2735 } 2736 2737 t_alg->registered = true; 2738 registered = true; 2739 } 2740 2741 if (registered) 2742 dev_info(ctrldev, "algorithms registered in /proc/crypto\n"); 2743 2744 return err; 2745 } 2746