1 /* 2 * Freescale FSL CAAM support for crypto API over QI backend. 3 * Based on caamalg.c 4 * 5 * Copyright 2013-2016 Freescale Semiconductor, Inc. 6 * Copyright 2016-2017 NXP 7 */ 8 9 #include "compat.h" 10 #include "ctrl.h" 11 #include "regs.h" 12 #include "intern.h" 13 #include "desc_constr.h" 14 #include "error.h" 15 #include "sg_sw_qm.h" 16 #include "key_gen.h" 17 #include "qi.h" 18 #include "jr.h" 19 #include "caamalg_desc.h" 20 21 /* 22 * crypto alg 23 */ 24 #define CAAM_CRA_PRIORITY 2000 25 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ 26 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ 27 SHA512_DIGEST_SIZE * 2) 28 29 #define DESC_MAX_USED_BYTES (DESC_QI_AEAD_GIVENC_LEN + \ 30 CAAM_MAX_KEY_SIZE) 31 #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) 32 33 struct caam_alg_entry { 34 int class1_alg_type; 35 int class2_alg_type; 36 bool rfc3686; 37 bool geniv; 38 }; 39 40 struct caam_aead_alg { 41 struct aead_alg aead; 42 struct caam_alg_entry caam; 43 bool registered; 44 }; 45 46 /* 47 * per-session context 48 */ 49 struct caam_ctx { 50 struct device *jrdev; 51 u32 sh_desc_enc[DESC_MAX_USED_LEN]; 52 u32 sh_desc_dec[DESC_MAX_USED_LEN]; 53 u32 sh_desc_givenc[DESC_MAX_USED_LEN]; 54 u8 key[CAAM_MAX_KEY_SIZE]; 55 dma_addr_t key_dma; 56 enum dma_data_direction dir; 57 struct alginfo adata; 58 struct alginfo cdata; 59 unsigned int authsize; 60 struct device *qidev; 61 spinlock_t lock; /* Protects multiple init of driver context */ 62 struct caam_drv_ctx *drv_ctx[NUM_OP]; 63 }; 64 65 static int aead_set_sh_desc(struct crypto_aead *aead) 66 { 67 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 68 typeof(*alg), aead); 69 struct caam_ctx *ctx = crypto_aead_ctx(aead); 70 unsigned int ivsize = crypto_aead_ivsize(aead); 71 u32 ctx1_iv_off = 0; 72 u32 *nonce = NULL; 73 unsigned int data_len[2]; 74 u32 inl_mask; 75 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 76 OP_ALG_AAI_CTR_MOD128); 77 const bool is_rfc3686 = alg->caam.rfc3686; 78 struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); 79 80 if (!ctx->cdata.keylen || !ctx->authsize) 81 return 0; 82 83 /* 84 * AES-CTR needs to load IV in CONTEXT1 reg 85 * at an offset of 128bits (16bytes) 86 * CONTEXT1[255:128] = IV 87 */ 88 if (ctr_mode) 89 ctx1_iv_off = 16; 90 91 /* 92 * RFC3686 specific: 93 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 94 */ 95 if (is_rfc3686) { 96 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 97 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + 98 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); 99 } 100 101 data_len[0] = ctx->adata.keylen_pad; 102 data_len[1] = ctx->cdata.keylen; 103 104 if (alg->caam.geniv) 105 goto skip_enc; 106 107 /* aead_encrypt shared descriptor */ 108 if (desc_inline_query(DESC_QI_AEAD_ENC_LEN + 109 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 110 DESC_JOB_IO_LEN, data_len, &inl_mask, 111 ARRAY_SIZE(data_len)) < 0) 112 return -EINVAL; 113 114 if (inl_mask & 1) 115 ctx->adata.key_virt = ctx->key; 116 else 117 ctx->adata.key_dma = ctx->key_dma; 118 119 if (inl_mask & 2) 120 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 121 else 122 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 123 124 ctx->adata.key_inline = !!(inl_mask & 1); 125 ctx->cdata.key_inline = !!(inl_mask & 2); 126 127 cnstr_shdsc_aead_encap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata, 128 ivsize, ctx->authsize, is_rfc3686, nonce, 129 ctx1_iv_off, true, ctrlpriv->era); 130 131 skip_enc: 132 /* aead_decrypt shared descriptor */ 133 if (desc_inline_query(DESC_QI_AEAD_DEC_LEN + 134 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 135 DESC_JOB_IO_LEN, data_len, &inl_mask, 136 ARRAY_SIZE(data_len)) < 0) 137 return -EINVAL; 138 139 if (inl_mask & 1) 140 ctx->adata.key_virt = ctx->key; 141 else 142 ctx->adata.key_dma = ctx->key_dma; 143 144 if (inl_mask & 2) 145 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 146 else 147 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 148 149 ctx->adata.key_inline = !!(inl_mask & 1); 150 ctx->cdata.key_inline = !!(inl_mask & 2); 151 152 cnstr_shdsc_aead_decap(ctx->sh_desc_dec, &ctx->cdata, &ctx->adata, 153 ivsize, ctx->authsize, alg->caam.geniv, 154 is_rfc3686, nonce, ctx1_iv_off, true, 155 ctrlpriv->era); 156 157 if (!alg->caam.geniv) 158 goto skip_givenc; 159 160 /* aead_givencrypt shared descriptor */ 161 if (desc_inline_query(DESC_QI_AEAD_GIVENC_LEN + 162 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 163 DESC_JOB_IO_LEN, data_len, &inl_mask, 164 ARRAY_SIZE(data_len)) < 0) 165 return -EINVAL; 166 167 if (inl_mask & 1) 168 ctx->adata.key_virt = ctx->key; 169 else 170 ctx->adata.key_dma = ctx->key_dma; 171 172 if (inl_mask & 2) 173 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 174 else 175 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 176 177 ctx->adata.key_inline = !!(inl_mask & 1); 178 ctx->cdata.key_inline = !!(inl_mask & 2); 179 180 cnstr_shdsc_aead_givencap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata, 181 ivsize, ctx->authsize, is_rfc3686, nonce, 182 ctx1_iv_off, true, ctrlpriv->era); 183 184 skip_givenc: 185 return 0; 186 } 187 188 static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 189 { 190 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 191 192 ctx->authsize = authsize; 193 aead_set_sh_desc(authenc); 194 195 return 0; 196 } 197 198 static int aead_setkey(struct crypto_aead *aead, const u8 *key, 199 unsigned int keylen) 200 { 201 struct caam_ctx *ctx = crypto_aead_ctx(aead); 202 struct device *jrdev = ctx->jrdev; 203 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); 204 struct crypto_authenc_keys keys; 205 int ret = 0; 206 207 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 208 goto badkey; 209 210 #ifdef DEBUG 211 dev_err(jrdev, "keylen %d enckeylen %d authkeylen %d\n", 212 keys.authkeylen + keys.enckeylen, keys.enckeylen, 213 keys.authkeylen); 214 print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", 215 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 216 #endif 217 218 /* 219 * If DKP is supported, use it in the shared descriptor to generate 220 * the split key. 221 */ 222 if (ctrlpriv->era >= 6) { 223 ctx->adata.keylen = keys.authkeylen; 224 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & 225 OP_ALG_ALGSEL_MASK); 226 227 if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) 228 goto badkey; 229 230 memcpy(ctx->key, keys.authkey, keys.authkeylen); 231 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, 232 keys.enckeylen); 233 dma_sync_single_for_device(jrdev, ctx->key_dma, 234 ctx->adata.keylen_pad + 235 keys.enckeylen, ctx->dir); 236 goto skip_split_key; 237 } 238 239 ret = gen_split_key(jrdev, ctx->key, &ctx->adata, keys.authkey, 240 keys.authkeylen, CAAM_MAX_KEY_SIZE - 241 keys.enckeylen); 242 if (ret) 243 goto badkey; 244 245 /* postpend encryption key to auth split key */ 246 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); 247 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad + 248 keys.enckeylen, ctx->dir); 249 #ifdef DEBUG 250 print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ", 251 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 252 ctx->adata.keylen_pad + keys.enckeylen, 1); 253 #endif 254 255 skip_split_key: 256 ctx->cdata.keylen = keys.enckeylen; 257 258 ret = aead_set_sh_desc(aead); 259 if (ret) 260 goto badkey; 261 262 /* Now update the driver contexts with the new shared descriptor */ 263 if (ctx->drv_ctx[ENCRYPT]) { 264 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 265 ctx->sh_desc_enc); 266 if (ret) { 267 dev_err(jrdev, "driver enc context update failed\n"); 268 goto badkey; 269 } 270 } 271 272 if (ctx->drv_ctx[DECRYPT]) { 273 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 274 ctx->sh_desc_dec); 275 if (ret) { 276 dev_err(jrdev, "driver dec context update failed\n"); 277 goto badkey; 278 } 279 } 280 281 memzero_explicit(&keys, sizeof(keys)); 282 return ret; 283 badkey: 284 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); 285 memzero_explicit(&keys, sizeof(keys)); 286 return -EINVAL; 287 } 288 289 static int gcm_set_sh_desc(struct crypto_aead *aead) 290 { 291 struct caam_ctx *ctx = crypto_aead_ctx(aead); 292 unsigned int ivsize = crypto_aead_ivsize(aead); 293 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 294 ctx->cdata.keylen; 295 296 if (!ctx->cdata.keylen || !ctx->authsize) 297 return 0; 298 299 /* 300 * Job Descriptor and Shared Descriptor 301 * must fit into the 64-word Descriptor h/w Buffer 302 */ 303 if (rem_bytes >= DESC_QI_GCM_ENC_LEN) { 304 ctx->cdata.key_inline = true; 305 ctx->cdata.key_virt = ctx->key; 306 } else { 307 ctx->cdata.key_inline = false; 308 ctx->cdata.key_dma = ctx->key_dma; 309 } 310 311 cnstr_shdsc_gcm_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, 312 ctx->authsize, true); 313 314 /* 315 * Job Descriptor and Shared Descriptor 316 * must fit into the 64-word Descriptor h/w Buffer 317 */ 318 if (rem_bytes >= DESC_QI_GCM_DEC_LEN) { 319 ctx->cdata.key_inline = true; 320 ctx->cdata.key_virt = ctx->key; 321 } else { 322 ctx->cdata.key_inline = false; 323 ctx->cdata.key_dma = ctx->key_dma; 324 } 325 326 cnstr_shdsc_gcm_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, 327 ctx->authsize, true); 328 329 return 0; 330 } 331 332 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 333 { 334 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 335 336 ctx->authsize = authsize; 337 gcm_set_sh_desc(authenc); 338 339 return 0; 340 } 341 342 static int gcm_setkey(struct crypto_aead *aead, 343 const u8 *key, unsigned int keylen) 344 { 345 struct caam_ctx *ctx = crypto_aead_ctx(aead); 346 struct device *jrdev = ctx->jrdev; 347 int ret; 348 349 #ifdef DEBUG 350 print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", 351 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 352 #endif 353 354 memcpy(ctx->key, key, keylen); 355 dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir); 356 ctx->cdata.keylen = keylen; 357 358 ret = gcm_set_sh_desc(aead); 359 if (ret) 360 return ret; 361 362 /* Now update the driver contexts with the new shared descriptor */ 363 if (ctx->drv_ctx[ENCRYPT]) { 364 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 365 ctx->sh_desc_enc); 366 if (ret) { 367 dev_err(jrdev, "driver enc context update failed\n"); 368 return ret; 369 } 370 } 371 372 if (ctx->drv_ctx[DECRYPT]) { 373 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 374 ctx->sh_desc_dec); 375 if (ret) { 376 dev_err(jrdev, "driver dec context update failed\n"); 377 return ret; 378 } 379 } 380 381 return 0; 382 } 383 384 static int rfc4106_set_sh_desc(struct crypto_aead *aead) 385 { 386 struct caam_ctx *ctx = crypto_aead_ctx(aead); 387 unsigned int ivsize = crypto_aead_ivsize(aead); 388 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 389 ctx->cdata.keylen; 390 391 if (!ctx->cdata.keylen || !ctx->authsize) 392 return 0; 393 394 ctx->cdata.key_virt = ctx->key; 395 396 /* 397 * Job Descriptor and Shared Descriptor 398 * must fit into the 64-word Descriptor h/w Buffer 399 */ 400 if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) { 401 ctx->cdata.key_inline = true; 402 } else { 403 ctx->cdata.key_inline = false; 404 ctx->cdata.key_dma = ctx->key_dma; 405 } 406 407 cnstr_shdsc_rfc4106_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, 408 ctx->authsize, true); 409 410 /* 411 * Job Descriptor and Shared Descriptor 412 * must fit into the 64-word Descriptor h/w Buffer 413 */ 414 if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) { 415 ctx->cdata.key_inline = true; 416 } else { 417 ctx->cdata.key_inline = false; 418 ctx->cdata.key_dma = ctx->key_dma; 419 } 420 421 cnstr_shdsc_rfc4106_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, 422 ctx->authsize, true); 423 424 return 0; 425 } 426 427 static int rfc4106_setauthsize(struct crypto_aead *authenc, 428 unsigned int authsize) 429 { 430 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 431 432 ctx->authsize = authsize; 433 rfc4106_set_sh_desc(authenc); 434 435 return 0; 436 } 437 438 static int rfc4106_setkey(struct crypto_aead *aead, 439 const u8 *key, unsigned int keylen) 440 { 441 struct caam_ctx *ctx = crypto_aead_ctx(aead); 442 struct device *jrdev = ctx->jrdev; 443 int ret; 444 445 if (keylen < 4) 446 return -EINVAL; 447 448 #ifdef DEBUG 449 print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", 450 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 451 #endif 452 453 memcpy(ctx->key, key, keylen); 454 /* 455 * The last four bytes of the key material are used as the salt value 456 * in the nonce. Update the AES key length. 457 */ 458 ctx->cdata.keylen = keylen - 4; 459 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, 460 ctx->dir); 461 462 ret = rfc4106_set_sh_desc(aead); 463 if (ret) 464 return ret; 465 466 /* Now update the driver contexts with the new shared descriptor */ 467 if (ctx->drv_ctx[ENCRYPT]) { 468 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 469 ctx->sh_desc_enc); 470 if (ret) { 471 dev_err(jrdev, "driver enc context update failed\n"); 472 return ret; 473 } 474 } 475 476 if (ctx->drv_ctx[DECRYPT]) { 477 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 478 ctx->sh_desc_dec); 479 if (ret) { 480 dev_err(jrdev, "driver dec context update failed\n"); 481 return ret; 482 } 483 } 484 485 return 0; 486 } 487 488 static int rfc4543_set_sh_desc(struct crypto_aead *aead) 489 { 490 struct caam_ctx *ctx = crypto_aead_ctx(aead); 491 unsigned int ivsize = crypto_aead_ivsize(aead); 492 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 493 ctx->cdata.keylen; 494 495 if (!ctx->cdata.keylen || !ctx->authsize) 496 return 0; 497 498 ctx->cdata.key_virt = ctx->key; 499 500 /* 501 * Job Descriptor and Shared Descriptor 502 * must fit into the 64-word Descriptor h/w Buffer 503 */ 504 if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) { 505 ctx->cdata.key_inline = true; 506 } else { 507 ctx->cdata.key_inline = false; 508 ctx->cdata.key_dma = ctx->key_dma; 509 } 510 511 cnstr_shdsc_rfc4543_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, 512 ctx->authsize, true); 513 514 /* 515 * Job Descriptor and Shared Descriptor 516 * must fit into the 64-word Descriptor h/w Buffer 517 */ 518 if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) { 519 ctx->cdata.key_inline = true; 520 } else { 521 ctx->cdata.key_inline = false; 522 ctx->cdata.key_dma = ctx->key_dma; 523 } 524 525 cnstr_shdsc_rfc4543_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, 526 ctx->authsize, true); 527 528 return 0; 529 } 530 531 static int rfc4543_setauthsize(struct crypto_aead *authenc, 532 unsigned int authsize) 533 { 534 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 535 536 ctx->authsize = authsize; 537 rfc4543_set_sh_desc(authenc); 538 539 return 0; 540 } 541 542 static int rfc4543_setkey(struct crypto_aead *aead, 543 const u8 *key, unsigned int keylen) 544 { 545 struct caam_ctx *ctx = crypto_aead_ctx(aead); 546 struct device *jrdev = ctx->jrdev; 547 int ret; 548 549 if (keylen < 4) 550 return -EINVAL; 551 552 #ifdef DEBUG 553 print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", 554 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 555 #endif 556 557 memcpy(ctx->key, key, keylen); 558 /* 559 * The last four bytes of the key material are used as the salt value 560 * in the nonce. Update the AES key length. 561 */ 562 ctx->cdata.keylen = keylen - 4; 563 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, 564 ctx->dir); 565 566 ret = rfc4543_set_sh_desc(aead); 567 if (ret) 568 return ret; 569 570 /* Now update the driver contexts with the new shared descriptor */ 571 if (ctx->drv_ctx[ENCRYPT]) { 572 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 573 ctx->sh_desc_enc); 574 if (ret) { 575 dev_err(jrdev, "driver enc context update failed\n"); 576 return ret; 577 } 578 } 579 580 if (ctx->drv_ctx[DECRYPT]) { 581 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 582 ctx->sh_desc_dec); 583 if (ret) { 584 dev_err(jrdev, "driver dec context update failed\n"); 585 return ret; 586 } 587 } 588 589 return 0; 590 } 591 592 static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, 593 const u8 *key, unsigned int keylen) 594 { 595 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 596 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(ablkcipher); 597 const char *alg_name = crypto_tfm_alg_name(tfm); 598 struct device *jrdev = ctx->jrdev; 599 unsigned int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 600 u32 ctx1_iv_off = 0; 601 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 602 OP_ALG_AAI_CTR_MOD128); 603 const bool is_rfc3686 = (ctr_mode && strstr(alg_name, "rfc3686")); 604 int ret = 0; 605 606 #ifdef DEBUG 607 print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", 608 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 609 #endif 610 /* 611 * AES-CTR needs to load IV in CONTEXT1 reg 612 * at an offset of 128bits (16bytes) 613 * CONTEXT1[255:128] = IV 614 */ 615 if (ctr_mode) 616 ctx1_iv_off = 16; 617 618 /* 619 * RFC3686 specific: 620 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} 621 * | *key = {KEY, NONCE} 622 */ 623 if (is_rfc3686) { 624 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 625 keylen -= CTR_RFC3686_NONCE_SIZE; 626 } 627 628 ctx->cdata.keylen = keylen; 629 ctx->cdata.key_virt = key; 630 ctx->cdata.key_inline = true; 631 632 /* ablkcipher encrypt, decrypt, givencrypt shared descriptors */ 633 cnstr_shdsc_ablkcipher_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, 634 is_rfc3686, ctx1_iv_off); 635 cnstr_shdsc_ablkcipher_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, 636 is_rfc3686, ctx1_iv_off); 637 cnstr_shdsc_ablkcipher_givencap(ctx->sh_desc_givenc, &ctx->cdata, 638 ivsize, is_rfc3686, ctx1_iv_off); 639 640 /* Now update the driver contexts with the new shared descriptor */ 641 if (ctx->drv_ctx[ENCRYPT]) { 642 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 643 ctx->sh_desc_enc); 644 if (ret) { 645 dev_err(jrdev, "driver enc context update failed\n"); 646 goto badkey; 647 } 648 } 649 650 if (ctx->drv_ctx[DECRYPT]) { 651 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 652 ctx->sh_desc_dec); 653 if (ret) { 654 dev_err(jrdev, "driver dec context update failed\n"); 655 goto badkey; 656 } 657 } 658 659 if (ctx->drv_ctx[GIVENCRYPT]) { 660 ret = caam_drv_ctx_update(ctx->drv_ctx[GIVENCRYPT], 661 ctx->sh_desc_givenc); 662 if (ret) { 663 dev_err(jrdev, "driver givenc context update failed\n"); 664 goto badkey; 665 } 666 } 667 668 return ret; 669 badkey: 670 crypto_ablkcipher_set_flags(ablkcipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 671 return -EINVAL; 672 } 673 674 static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, 675 const u8 *key, unsigned int keylen) 676 { 677 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 678 struct device *jrdev = ctx->jrdev; 679 int ret = 0; 680 681 if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) { 682 crypto_ablkcipher_set_flags(ablkcipher, 683 CRYPTO_TFM_RES_BAD_KEY_LEN); 684 dev_err(jrdev, "key size mismatch\n"); 685 return -EINVAL; 686 } 687 688 ctx->cdata.keylen = keylen; 689 ctx->cdata.key_virt = key; 690 ctx->cdata.key_inline = true; 691 692 /* xts ablkcipher encrypt, decrypt shared descriptors */ 693 cnstr_shdsc_xts_ablkcipher_encap(ctx->sh_desc_enc, &ctx->cdata); 694 cnstr_shdsc_xts_ablkcipher_decap(ctx->sh_desc_dec, &ctx->cdata); 695 696 /* Now update the driver contexts with the new shared descriptor */ 697 if (ctx->drv_ctx[ENCRYPT]) { 698 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 699 ctx->sh_desc_enc); 700 if (ret) { 701 dev_err(jrdev, "driver enc context update failed\n"); 702 goto badkey; 703 } 704 } 705 706 if (ctx->drv_ctx[DECRYPT]) { 707 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 708 ctx->sh_desc_dec); 709 if (ret) { 710 dev_err(jrdev, "driver dec context update failed\n"); 711 goto badkey; 712 } 713 } 714 715 return ret; 716 badkey: 717 crypto_ablkcipher_set_flags(ablkcipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 718 return 0; 719 } 720 721 /* 722 * aead_edesc - s/w-extended aead descriptor 723 * @src_nents: number of segments in input scatterlist 724 * @dst_nents: number of segments in output scatterlist 725 * @iv_dma: dma address of iv for checking continuity and link table 726 * @qm_sg_bytes: length of dma mapped h/w link table 727 * @qm_sg_dma: bus physical mapped address of h/w link table 728 * @assoclen: associated data length, in CAAM endianness 729 * @assoclen_dma: bus physical mapped address of req->assoclen 730 * @drv_req: driver-specific request structure 731 * @sgt: the h/w link table, followed by IV 732 */ 733 struct aead_edesc { 734 int src_nents; 735 int dst_nents; 736 dma_addr_t iv_dma; 737 int qm_sg_bytes; 738 dma_addr_t qm_sg_dma; 739 unsigned int assoclen; 740 dma_addr_t assoclen_dma; 741 struct caam_drv_req drv_req; 742 struct qm_sg_entry sgt[0]; 743 }; 744 745 /* 746 * ablkcipher_edesc - s/w-extended ablkcipher descriptor 747 * @src_nents: number of segments in input scatterlist 748 * @dst_nents: number of segments in output scatterlist 749 * @iv_dma: dma address of iv for checking continuity and link table 750 * @qm_sg_bytes: length of dma mapped h/w link table 751 * @qm_sg_dma: bus physical mapped address of h/w link table 752 * @drv_req: driver-specific request structure 753 * @sgt: the h/w link table, followed by IV 754 */ 755 struct ablkcipher_edesc { 756 int src_nents; 757 int dst_nents; 758 dma_addr_t iv_dma; 759 int qm_sg_bytes; 760 dma_addr_t qm_sg_dma; 761 struct caam_drv_req drv_req; 762 struct qm_sg_entry sgt[0]; 763 }; 764 765 static struct caam_drv_ctx *get_drv_ctx(struct caam_ctx *ctx, 766 enum optype type) 767 { 768 /* 769 * This function is called on the fast path with values of 'type' 770 * known at compile time. Invalid arguments are not expected and 771 * thus no checks are made. 772 */ 773 struct caam_drv_ctx *drv_ctx = ctx->drv_ctx[type]; 774 u32 *desc; 775 776 if (unlikely(!drv_ctx)) { 777 spin_lock(&ctx->lock); 778 779 /* Read again to check if some other core init drv_ctx */ 780 drv_ctx = ctx->drv_ctx[type]; 781 if (!drv_ctx) { 782 int cpu; 783 784 if (type == ENCRYPT) 785 desc = ctx->sh_desc_enc; 786 else if (type == DECRYPT) 787 desc = ctx->sh_desc_dec; 788 else /* (type == GIVENCRYPT) */ 789 desc = ctx->sh_desc_givenc; 790 791 cpu = smp_processor_id(); 792 drv_ctx = caam_drv_ctx_init(ctx->qidev, &cpu, desc); 793 if (likely(!IS_ERR_OR_NULL(drv_ctx))) 794 drv_ctx->op_type = type; 795 796 ctx->drv_ctx[type] = drv_ctx; 797 } 798 799 spin_unlock(&ctx->lock); 800 } 801 802 return drv_ctx; 803 } 804 805 static void caam_unmap(struct device *dev, struct scatterlist *src, 806 struct scatterlist *dst, int src_nents, 807 int dst_nents, dma_addr_t iv_dma, int ivsize, 808 enum optype op_type, dma_addr_t qm_sg_dma, 809 int qm_sg_bytes) 810 { 811 if (dst != src) { 812 if (src_nents) 813 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); 814 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); 815 } else { 816 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); 817 } 818 819 if (iv_dma) 820 dma_unmap_single(dev, iv_dma, ivsize, 821 op_type == GIVENCRYPT ? DMA_FROM_DEVICE : 822 DMA_TO_DEVICE); 823 if (qm_sg_bytes) 824 dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE); 825 } 826 827 static void aead_unmap(struct device *dev, 828 struct aead_edesc *edesc, 829 struct aead_request *req) 830 { 831 struct crypto_aead *aead = crypto_aead_reqtfm(req); 832 int ivsize = crypto_aead_ivsize(aead); 833 834 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 835 edesc->iv_dma, ivsize, edesc->drv_req.drv_ctx->op_type, 836 edesc->qm_sg_dma, edesc->qm_sg_bytes); 837 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 838 } 839 840 static void ablkcipher_unmap(struct device *dev, 841 struct ablkcipher_edesc *edesc, 842 struct ablkcipher_request *req) 843 { 844 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 845 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 846 847 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 848 edesc->iv_dma, ivsize, edesc->drv_req.drv_ctx->op_type, 849 edesc->qm_sg_dma, edesc->qm_sg_bytes); 850 } 851 852 static void aead_done(struct caam_drv_req *drv_req, u32 status) 853 { 854 struct device *qidev; 855 struct aead_edesc *edesc; 856 struct aead_request *aead_req = drv_req->app_ctx; 857 struct crypto_aead *aead = crypto_aead_reqtfm(aead_req); 858 struct caam_ctx *caam_ctx = crypto_aead_ctx(aead); 859 int ecode = 0; 860 861 qidev = caam_ctx->qidev; 862 863 if (unlikely(status)) { 864 u32 ssrc = status & JRSTA_SSRC_MASK; 865 u8 err_id = status & JRSTA_CCBERR_ERRID_MASK; 866 867 caam_jr_strstatus(qidev, status); 868 /* 869 * verify hw auth check passed else return -EBADMSG 870 */ 871 if (ssrc == JRSTA_SSRC_CCB_ERROR && 872 err_id == JRSTA_CCBERR_ERRID_ICVCHK) 873 ecode = -EBADMSG; 874 else 875 ecode = -EIO; 876 } 877 878 edesc = container_of(drv_req, typeof(*edesc), drv_req); 879 aead_unmap(qidev, edesc, aead_req); 880 881 aead_request_complete(aead_req, ecode); 882 qi_cache_free(edesc); 883 } 884 885 /* 886 * allocate and map the aead extended descriptor 887 */ 888 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 889 bool encrypt) 890 { 891 struct crypto_aead *aead = crypto_aead_reqtfm(req); 892 struct caam_ctx *ctx = crypto_aead_ctx(aead); 893 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 894 typeof(*alg), aead); 895 struct device *qidev = ctx->qidev; 896 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 897 GFP_KERNEL : GFP_ATOMIC; 898 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 899 struct aead_edesc *edesc; 900 dma_addr_t qm_sg_dma, iv_dma = 0; 901 int ivsize = 0; 902 unsigned int authsize = ctx->authsize; 903 int qm_sg_index = 0, qm_sg_ents = 0, qm_sg_bytes; 904 int in_len, out_len; 905 struct qm_sg_entry *sg_table, *fd_sgt; 906 struct caam_drv_ctx *drv_ctx; 907 enum optype op_type = encrypt ? ENCRYPT : DECRYPT; 908 909 drv_ctx = get_drv_ctx(ctx, op_type); 910 if (unlikely(IS_ERR_OR_NULL(drv_ctx))) 911 return (struct aead_edesc *)drv_ctx; 912 913 /* allocate space for base edesc and hw desc commands, link tables */ 914 edesc = qi_cache_alloc(GFP_DMA | flags); 915 if (unlikely(!edesc)) { 916 dev_err(qidev, "could not allocate extended descriptor\n"); 917 return ERR_PTR(-ENOMEM); 918 } 919 920 if (likely(req->src == req->dst)) { 921 src_nents = sg_nents_for_len(req->src, req->assoclen + 922 req->cryptlen + 923 (encrypt ? authsize : 0)); 924 if (unlikely(src_nents < 0)) { 925 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", 926 req->assoclen + req->cryptlen + 927 (encrypt ? authsize : 0)); 928 qi_cache_free(edesc); 929 return ERR_PTR(src_nents); 930 } 931 932 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, 933 DMA_BIDIRECTIONAL); 934 if (unlikely(!mapped_src_nents)) { 935 dev_err(qidev, "unable to map source\n"); 936 qi_cache_free(edesc); 937 return ERR_PTR(-ENOMEM); 938 } 939 } else { 940 src_nents = sg_nents_for_len(req->src, req->assoclen + 941 req->cryptlen); 942 if (unlikely(src_nents < 0)) { 943 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", 944 req->assoclen + req->cryptlen); 945 qi_cache_free(edesc); 946 return ERR_PTR(src_nents); 947 } 948 949 dst_nents = sg_nents_for_len(req->dst, req->assoclen + 950 req->cryptlen + 951 (encrypt ? authsize : 952 (-authsize))); 953 if (unlikely(dst_nents < 0)) { 954 dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n", 955 req->assoclen + req->cryptlen + 956 (encrypt ? authsize : (-authsize))); 957 qi_cache_free(edesc); 958 return ERR_PTR(dst_nents); 959 } 960 961 if (src_nents) { 962 mapped_src_nents = dma_map_sg(qidev, req->src, 963 src_nents, DMA_TO_DEVICE); 964 if (unlikely(!mapped_src_nents)) { 965 dev_err(qidev, "unable to map source\n"); 966 qi_cache_free(edesc); 967 return ERR_PTR(-ENOMEM); 968 } 969 } else { 970 mapped_src_nents = 0; 971 } 972 973 mapped_dst_nents = dma_map_sg(qidev, req->dst, dst_nents, 974 DMA_FROM_DEVICE); 975 if (unlikely(!mapped_dst_nents)) { 976 dev_err(qidev, "unable to map destination\n"); 977 dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE); 978 qi_cache_free(edesc); 979 return ERR_PTR(-ENOMEM); 980 } 981 } 982 983 if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv) 984 ivsize = crypto_aead_ivsize(aead); 985 986 /* 987 * Create S/G table: req->assoclen, [IV,] req->src [, req->dst]. 988 * Input is not contiguous. 989 */ 990 qm_sg_ents = 1 + !!ivsize + mapped_src_nents + 991 (mapped_dst_nents > 1 ? mapped_dst_nents : 0); 992 sg_table = &edesc->sgt[0]; 993 qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); 994 if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize > 995 CAAM_QI_MEMCACHE_SIZE)) { 996 dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n", 997 qm_sg_ents, ivsize); 998 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, 999 0, 0, 0, 0); 1000 qi_cache_free(edesc); 1001 return ERR_PTR(-ENOMEM); 1002 } 1003 1004 if (ivsize) { 1005 u8 *iv = (u8 *)(sg_table + qm_sg_ents); 1006 1007 /* Make sure IV is located in a DMAable area */ 1008 memcpy(iv, req->iv, ivsize); 1009 1010 iv_dma = dma_map_single(qidev, iv, ivsize, DMA_TO_DEVICE); 1011 if (dma_mapping_error(qidev, iv_dma)) { 1012 dev_err(qidev, "unable to map IV\n"); 1013 caam_unmap(qidev, req->src, req->dst, src_nents, 1014 dst_nents, 0, 0, 0, 0, 0); 1015 qi_cache_free(edesc); 1016 return ERR_PTR(-ENOMEM); 1017 } 1018 } 1019 1020 edesc->src_nents = src_nents; 1021 edesc->dst_nents = dst_nents; 1022 edesc->iv_dma = iv_dma; 1023 edesc->drv_req.app_ctx = req; 1024 edesc->drv_req.cbk = aead_done; 1025 edesc->drv_req.drv_ctx = drv_ctx; 1026 1027 edesc->assoclen = cpu_to_caam32(req->assoclen); 1028 edesc->assoclen_dma = dma_map_single(qidev, &edesc->assoclen, 4, 1029 DMA_TO_DEVICE); 1030 if (dma_mapping_error(qidev, edesc->assoclen_dma)) { 1031 dev_err(qidev, "unable to map assoclen\n"); 1032 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 1033 iv_dma, ivsize, op_type, 0, 0); 1034 qi_cache_free(edesc); 1035 return ERR_PTR(-ENOMEM); 1036 } 1037 1038 dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0); 1039 qm_sg_index++; 1040 if (ivsize) { 1041 dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0); 1042 qm_sg_index++; 1043 } 1044 sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + qm_sg_index, 0); 1045 qm_sg_index += mapped_src_nents; 1046 1047 if (mapped_dst_nents > 1) 1048 sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + 1049 qm_sg_index, 0); 1050 1051 qm_sg_dma = dma_map_single(qidev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); 1052 if (dma_mapping_error(qidev, qm_sg_dma)) { 1053 dev_err(qidev, "unable to map S/G table\n"); 1054 dma_unmap_single(qidev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 1055 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 1056 iv_dma, ivsize, op_type, 0, 0); 1057 qi_cache_free(edesc); 1058 return ERR_PTR(-ENOMEM); 1059 } 1060 1061 edesc->qm_sg_dma = qm_sg_dma; 1062 edesc->qm_sg_bytes = qm_sg_bytes; 1063 1064 out_len = req->assoclen + req->cryptlen + 1065 (encrypt ? ctx->authsize : (-ctx->authsize)); 1066 in_len = 4 + ivsize + req->assoclen + req->cryptlen; 1067 1068 fd_sgt = &edesc->drv_req.fd_sgt[0]; 1069 dma_to_qm_sg_one_last_ext(&fd_sgt[1], qm_sg_dma, in_len, 0); 1070 1071 if (req->dst == req->src) { 1072 if (mapped_src_nents == 1) 1073 dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->src), 1074 out_len, 0); 1075 else 1076 dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + 1077 (1 + !!ivsize) * sizeof(*sg_table), 1078 out_len, 0); 1079 } else if (mapped_dst_nents == 1) { 1080 dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), out_len, 1081 0); 1082 } else { 1083 dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + sizeof(*sg_table) * 1084 qm_sg_index, out_len, 0); 1085 } 1086 1087 return edesc; 1088 } 1089 1090 static inline int aead_crypt(struct aead_request *req, bool encrypt) 1091 { 1092 struct aead_edesc *edesc; 1093 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1094 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1095 int ret; 1096 1097 if (unlikely(caam_congested)) 1098 return -EAGAIN; 1099 1100 /* allocate extended descriptor */ 1101 edesc = aead_edesc_alloc(req, encrypt); 1102 if (IS_ERR_OR_NULL(edesc)) 1103 return PTR_ERR(edesc); 1104 1105 /* Create and submit job descriptor */ 1106 ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req); 1107 if (!ret) { 1108 ret = -EINPROGRESS; 1109 } else { 1110 aead_unmap(ctx->qidev, edesc, req); 1111 qi_cache_free(edesc); 1112 } 1113 1114 return ret; 1115 } 1116 1117 static int aead_encrypt(struct aead_request *req) 1118 { 1119 return aead_crypt(req, true); 1120 } 1121 1122 static int aead_decrypt(struct aead_request *req) 1123 { 1124 return aead_crypt(req, false); 1125 } 1126 1127 static int ipsec_gcm_encrypt(struct aead_request *req) 1128 { 1129 if (req->assoclen < 8) 1130 return -EINVAL; 1131 1132 return aead_crypt(req, true); 1133 } 1134 1135 static int ipsec_gcm_decrypt(struct aead_request *req) 1136 { 1137 if (req->assoclen < 8) 1138 return -EINVAL; 1139 1140 return aead_crypt(req, false); 1141 } 1142 1143 static void ablkcipher_done(struct caam_drv_req *drv_req, u32 status) 1144 { 1145 struct ablkcipher_edesc *edesc; 1146 struct ablkcipher_request *req = drv_req->app_ctx; 1147 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1148 struct caam_ctx *caam_ctx = crypto_ablkcipher_ctx(ablkcipher); 1149 struct device *qidev = caam_ctx->qidev; 1150 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1151 1152 #ifdef DEBUG 1153 dev_err(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status); 1154 #endif 1155 1156 edesc = container_of(drv_req, typeof(*edesc), drv_req); 1157 1158 if (status) 1159 caam_jr_strstatus(qidev, status); 1160 1161 #ifdef DEBUG 1162 print_hex_dump(KERN_ERR, "dstiv @" __stringify(__LINE__)": ", 1163 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 1164 edesc->src_nents > 1 ? 100 : ivsize, 1); 1165 caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", 1166 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 1167 edesc->dst_nents > 1 ? 100 : req->nbytes, 1); 1168 #endif 1169 1170 ablkcipher_unmap(qidev, edesc, req); 1171 1172 /* In case initial IV was generated, copy it in GIVCIPHER request */ 1173 if (edesc->drv_req.drv_ctx->op_type == GIVENCRYPT) { 1174 u8 *iv; 1175 struct skcipher_givcrypt_request *greq; 1176 1177 greq = container_of(req, struct skcipher_givcrypt_request, 1178 creq); 1179 iv = (u8 *)edesc->sgt + edesc->qm_sg_bytes; 1180 memcpy(greq->giv, iv, ivsize); 1181 } 1182 1183 /* 1184 * The crypto API expects us to set the IV (req->info) to the last 1185 * ciphertext block. This is used e.g. by the CTS mode. 1186 */ 1187 if (edesc->drv_req.drv_ctx->op_type != DECRYPT) 1188 scatterwalk_map_and_copy(req->info, req->dst, req->nbytes - 1189 ivsize, ivsize, 0); 1190 1191 qi_cache_free(edesc); 1192 ablkcipher_request_complete(req, status); 1193 } 1194 1195 static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request 1196 *req, bool encrypt) 1197 { 1198 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1199 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1200 struct device *qidev = ctx->qidev; 1201 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1202 GFP_KERNEL : GFP_ATOMIC; 1203 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 1204 struct ablkcipher_edesc *edesc; 1205 dma_addr_t iv_dma; 1206 u8 *iv; 1207 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1208 int dst_sg_idx, qm_sg_ents, qm_sg_bytes; 1209 struct qm_sg_entry *sg_table, *fd_sgt; 1210 struct caam_drv_ctx *drv_ctx; 1211 enum optype op_type = encrypt ? ENCRYPT : DECRYPT; 1212 1213 drv_ctx = get_drv_ctx(ctx, op_type); 1214 if (unlikely(IS_ERR_OR_NULL(drv_ctx))) 1215 return (struct ablkcipher_edesc *)drv_ctx; 1216 1217 src_nents = sg_nents_for_len(req->src, req->nbytes); 1218 if (unlikely(src_nents < 0)) { 1219 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", 1220 req->nbytes); 1221 return ERR_PTR(src_nents); 1222 } 1223 1224 if (unlikely(req->src != req->dst)) { 1225 dst_nents = sg_nents_for_len(req->dst, req->nbytes); 1226 if (unlikely(dst_nents < 0)) { 1227 dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n", 1228 req->nbytes); 1229 return ERR_PTR(dst_nents); 1230 } 1231 1232 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, 1233 DMA_TO_DEVICE); 1234 if (unlikely(!mapped_src_nents)) { 1235 dev_err(qidev, "unable to map source\n"); 1236 return ERR_PTR(-ENOMEM); 1237 } 1238 1239 mapped_dst_nents = dma_map_sg(qidev, req->dst, dst_nents, 1240 DMA_FROM_DEVICE); 1241 if (unlikely(!mapped_dst_nents)) { 1242 dev_err(qidev, "unable to map destination\n"); 1243 dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE); 1244 return ERR_PTR(-ENOMEM); 1245 } 1246 } else { 1247 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, 1248 DMA_BIDIRECTIONAL); 1249 if (unlikely(!mapped_src_nents)) { 1250 dev_err(qidev, "unable to map source\n"); 1251 return ERR_PTR(-ENOMEM); 1252 } 1253 } 1254 1255 qm_sg_ents = 1 + mapped_src_nents; 1256 dst_sg_idx = qm_sg_ents; 1257 1258 qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0; 1259 qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry); 1260 if (unlikely(offsetof(struct ablkcipher_edesc, sgt) + qm_sg_bytes + 1261 ivsize > CAAM_QI_MEMCACHE_SIZE)) { 1262 dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n", 1263 qm_sg_ents, ivsize); 1264 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, 1265 0, 0, 0, 0); 1266 return ERR_PTR(-ENOMEM); 1267 } 1268 1269 /* allocate space for base edesc, link tables and IV */ 1270 edesc = qi_cache_alloc(GFP_DMA | flags); 1271 if (unlikely(!edesc)) { 1272 dev_err(qidev, "could not allocate extended descriptor\n"); 1273 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, 1274 0, 0, 0, 0); 1275 return ERR_PTR(-ENOMEM); 1276 } 1277 1278 /* Make sure IV is located in a DMAable area */ 1279 sg_table = &edesc->sgt[0]; 1280 iv = (u8 *)(sg_table + qm_sg_ents); 1281 memcpy(iv, req->info, ivsize); 1282 1283 iv_dma = dma_map_single(qidev, iv, ivsize, DMA_TO_DEVICE); 1284 if (dma_mapping_error(qidev, iv_dma)) { 1285 dev_err(qidev, "unable to map IV\n"); 1286 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, 1287 0, 0, 0, 0); 1288 qi_cache_free(edesc); 1289 return ERR_PTR(-ENOMEM); 1290 } 1291 1292 edesc->src_nents = src_nents; 1293 edesc->dst_nents = dst_nents; 1294 edesc->iv_dma = iv_dma; 1295 edesc->qm_sg_bytes = qm_sg_bytes; 1296 edesc->drv_req.app_ctx = req; 1297 edesc->drv_req.cbk = ablkcipher_done; 1298 edesc->drv_req.drv_ctx = drv_ctx; 1299 1300 dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); 1301 sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + 1, 0); 1302 1303 if (mapped_dst_nents > 1) 1304 sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + 1305 dst_sg_idx, 0); 1306 1307 edesc->qm_sg_dma = dma_map_single(qidev, sg_table, edesc->qm_sg_bytes, 1308 DMA_TO_DEVICE); 1309 if (dma_mapping_error(qidev, edesc->qm_sg_dma)) { 1310 dev_err(qidev, "unable to map S/G table\n"); 1311 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 1312 iv_dma, ivsize, op_type, 0, 0); 1313 qi_cache_free(edesc); 1314 return ERR_PTR(-ENOMEM); 1315 } 1316 1317 fd_sgt = &edesc->drv_req.fd_sgt[0]; 1318 1319 dma_to_qm_sg_one_last_ext(&fd_sgt[1], edesc->qm_sg_dma, 1320 ivsize + req->nbytes, 0); 1321 1322 if (req->src == req->dst) { 1323 dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + 1324 sizeof(*sg_table), req->nbytes, 0); 1325 } else if (mapped_dst_nents > 1) { 1326 dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx * 1327 sizeof(*sg_table), req->nbytes, 0); 1328 } else { 1329 dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), 1330 req->nbytes, 0); 1331 } 1332 1333 return edesc; 1334 } 1335 1336 static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc( 1337 struct skcipher_givcrypt_request *creq) 1338 { 1339 struct ablkcipher_request *req = &creq->creq; 1340 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1341 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1342 struct device *qidev = ctx->qidev; 1343 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1344 GFP_KERNEL : GFP_ATOMIC; 1345 int src_nents, mapped_src_nents, dst_nents, mapped_dst_nents; 1346 struct ablkcipher_edesc *edesc; 1347 dma_addr_t iv_dma; 1348 u8 *iv; 1349 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1350 struct qm_sg_entry *sg_table, *fd_sgt; 1351 int dst_sg_idx, qm_sg_ents, qm_sg_bytes; 1352 struct caam_drv_ctx *drv_ctx; 1353 1354 drv_ctx = get_drv_ctx(ctx, GIVENCRYPT); 1355 if (unlikely(IS_ERR_OR_NULL(drv_ctx))) 1356 return (struct ablkcipher_edesc *)drv_ctx; 1357 1358 src_nents = sg_nents_for_len(req->src, req->nbytes); 1359 if (unlikely(src_nents < 0)) { 1360 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", 1361 req->nbytes); 1362 return ERR_PTR(src_nents); 1363 } 1364 1365 if (unlikely(req->src != req->dst)) { 1366 dst_nents = sg_nents_for_len(req->dst, req->nbytes); 1367 if (unlikely(dst_nents < 0)) { 1368 dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n", 1369 req->nbytes); 1370 return ERR_PTR(dst_nents); 1371 } 1372 1373 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, 1374 DMA_TO_DEVICE); 1375 if (unlikely(!mapped_src_nents)) { 1376 dev_err(qidev, "unable to map source\n"); 1377 return ERR_PTR(-ENOMEM); 1378 } 1379 1380 mapped_dst_nents = dma_map_sg(qidev, req->dst, dst_nents, 1381 DMA_FROM_DEVICE); 1382 if (unlikely(!mapped_dst_nents)) { 1383 dev_err(qidev, "unable to map destination\n"); 1384 dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE); 1385 return ERR_PTR(-ENOMEM); 1386 } 1387 } else { 1388 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, 1389 DMA_BIDIRECTIONAL); 1390 if (unlikely(!mapped_src_nents)) { 1391 dev_err(qidev, "unable to map source\n"); 1392 return ERR_PTR(-ENOMEM); 1393 } 1394 1395 dst_nents = src_nents; 1396 mapped_dst_nents = src_nents; 1397 } 1398 1399 qm_sg_ents = mapped_src_nents > 1 ? mapped_src_nents : 0; 1400 dst_sg_idx = qm_sg_ents; 1401 1402 qm_sg_ents += 1 + mapped_dst_nents; 1403 qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry); 1404 if (unlikely(offsetof(struct ablkcipher_edesc, sgt) + qm_sg_bytes + 1405 ivsize > CAAM_QI_MEMCACHE_SIZE)) { 1406 dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n", 1407 qm_sg_ents, ivsize); 1408 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, 1409 0, 0, 0, 0); 1410 return ERR_PTR(-ENOMEM); 1411 } 1412 1413 /* allocate space for base edesc, link tables and IV */ 1414 edesc = qi_cache_alloc(GFP_DMA | flags); 1415 if (!edesc) { 1416 dev_err(qidev, "could not allocate extended descriptor\n"); 1417 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, 1418 0, 0, 0, 0); 1419 return ERR_PTR(-ENOMEM); 1420 } 1421 1422 /* Make sure IV is located in a DMAable area */ 1423 sg_table = &edesc->sgt[0]; 1424 iv = (u8 *)(sg_table + qm_sg_ents); 1425 iv_dma = dma_map_single(qidev, iv, ivsize, DMA_FROM_DEVICE); 1426 if (dma_mapping_error(qidev, iv_dma)) { 1427 dev_err(qidev, "unable to map IV\n"); 1428 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, 1429 0, 0, 0, 0); 1430 qi_cache_free(edesc); 1431 return ERR_PTR(-ENOMEM); 1432 } 1433 1434 edesc->src_nents = src_nents; 1435 edesc->dst_nents = dst_nents; 1436 edesc->iv_dma = iv_dma; 1437 edesc->qm_sg_bytes = qm_sg_bytes; 1438 edesc->drv_req.app_ctx = req; 1439 edesc->drv_req.cbk = ablkcipher_done; 1440 edesc->drv_req.drv_ctx = drv_ctx; 1441 1442 if (mapped_src_nents > 1) 1443 sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table, 0); 1444 1445 dma_to_qm_sg_one(sg_table + dst_sg_idx, iv_dma, ivsize, 0); 1446 sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + dst_sg_idx + 1, 1447 0); 1448 1449 edesc->qm_sg_dma = dma_map_single(qidev, sg_table, edesc->qm_sg_bytes, 1450 DMA_TO_DEVICE); 1451 if (dma_mapping_error(qidev, edesc->qm_sg_dma)) { 1452 dev_err(qidev, "unable to map S/G table\n"); 1453 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 1454 iv_dma, ivsize, GIVENCRYPT, 0, 0); 1455 qi_cache_free(edesc); 1456 return ERR_PTR(-ENOMEM); 1457 } 1458 1459 fd_sgt = &edesc->drv_req.fd_sgt[0]; 1460 1461 if (mapped_src_nents > 1) 1462 dma_to_qm_sg_one_ext(&fd_sgt[1], edesc->qm_sg_dma, req->nbytes, 1463 0); 1464 else 1465 dma_to_qm_sg_one(&fd_sgt[1], sg_dma_address(req->src), 1466 req->nbytes, 0); 1467 1468 dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx * 1469 sizeof(*sg_table), ivsize + req->nbytes, 0); 1470 1471 return edesc; 1472 } 1473 1474 static inline int ablkcipher_crypt(struct ablkcipher_request *req, bool encrypt) 1475 { 1476 struct ablkcipher_edesc *edesc; 1477 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1478 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1479 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1480 int ret; 1481 1482 if (unlikely(caam_congested)) 1483 return -EAGAIN; 1484 1485 /* allocate extended descriptor */ 1486 edesc = ablkcipher_edesc_alloc(req, encrypt); 1487 if (IS_ERR(edesc)) 1488 return PTR_ERR(edesc); 1489 1490 /* 1491 * The crypto API expects us to set the IV (req->info) to the last 1492 * ciphertext block. 1493 */ 1494 if (!encrypt) 1495 scatterwalk_map_and_copy(req->info, req->src, req->nbytes - 1496 ivsize, ivsize, 0); 1497 1498 ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req); 1499 if (!ret) { 1500 ret = -EINPROGRESS; 1501 } else { 1502 ablkcipher_unmap(ctx->qidev, edesc, req); 1503 qi_cache_free(edesc); 1504 } 1505 1506 return ret; 1507 } 1508 1509 static int ablkcipher_encrypt(struct ablkcipher_request *req) 1510 { 1511 return ablkcipher_crypt(req, true); 1512 } 1513 1514 static int ablkcipher_decrypt(struct ablkcipher_request *req) 1515 { 1516 return ablkcipher_crypt(req, false); 1517 } 1518 1519 static int ablkcipher_givencrypt(struct skcipher_givcrypt_request *creq) 1520 { 1521 struct ablkcipher_request *req = &creq->creq; 1522 struct ablkcipher_edesc *edesc; 1523 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1524 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1525 int ret; 1526 1527 if (unlikely(caam_congested)) 1528 return -EAGAIN; 1529 1530 /* allocate extended descriptor */ 1531 edesc = ablkcipher_giv_edesc_alloc(creq); 1532 if (IS_ERR(edesc)) 1533 return PTR_ERR(edesc); 1534 1535 ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req); 1536 if (!ret) { 1537 ret = -EINPROGRESS; 1538 } else { 1539 ablkcipher_unmap(ctx->qidev, edesc, req); 1540 qi_cache_free(edesc); 1541 } 1542 1543 return ret; 1544 } 1545 1546 #define template_ablkcipher template_u.ablkcipher 1547 struct caam_alg_template { 1548 char name[CRYPTO_MAX_ALG_NAME]; 1549 char driver_name[CRYPTO_MAX_ALG_NAME]; 1550 unsigned int blocksize; 1551 u32 type; 1552 union { 1553 struct ablkcipher_alg ablkcipher; 1554 } template_u; 1555 u32 class1_alg_type; 1556 u32 class2_alg_type; 1557 }; 1558 1559 static struct caam_alg_template driver_algs[] = { 1560 /* ablkcipher descriptor */ 1561 { 1562 .name = "cbc(aes)", 1563 .driver_name = "cbc-aes-caam-qi", 1564 .blocksize = AES_BLOCK_SIZE, 1565 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 1566 .template_ablkcipher = { 1567 .setkey = ablkcipher_setkey, 1568 .encrypt = ablkcipher_encrypt, 1569 .decrypt = ablkcipher_decrypt, 1570 .givencrypt = ablkcipher_givencrypt, 1571 .geniv = "<built-in>", 1572 .min_keysize = AES_MIN_KEY_SIZE, 1573 .max_keysize = AES_MAX_KEY_SIZE, 1574 .ivsize = AES_BLOCK_SIZE, 1575 }, 1576 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1577 }, 1578 { 1579 .name = "cbc(des3_ede)", 1580 .driver_name = "cbc-3des-caam-qi", 1581 .blocksize = DES3_EDE_BLOCK_SIZE, 1582 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 1583 .template_ablkcipher = { 1584 .setkey = ablkcipher_setkey, 1585 .encrypt = ablkcipher_encrypt, 1586 .decrypt = ablkcipher_decrypt, 1587 .givencrypt = ablkcipher_givencrypt, 1588 .geniv = "<built-in>", 1589 .min_keysize = DES3_EDE_KEY_SIZE, 1590 .max_keysize = DES3_EDE_KEY_SIZE, 1591 .ivsize = DES3_EDE_BLOCK_SIZE, 1592 }, 1593 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1594 }, 1595 { 1596 .name = "cbc(des)", 1597 .driver_name = "cbc-des-caam-qi", 1598 .blocksize = DES_BLOCK_SIZE, 1599 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 1600 .template_ablkcipher = { 1601 .setkey = ablkcipher_setkey, 1602 .encrypt = ablkcipher_encrypt, 1603 .decrypt = ablkcipher_decrypt, 1604 .givencrypt = ablkcipher_givencrypt, 1605 .geniv = "<built-in>", 1606 .min_keysize = DES_KEY_SIZE, 1607 .max_keysize = DES_KEY_SIZE, 1608 .ivsize = DES_BLOCK_SIZE, 1609 }, 1610 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1611 }, 1612 { 1613 .name = "ctr(aes)", 1614 .driver_name = "ctr-aes-caam-qi", 1615 .blocksize = 1, 1616 .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 1617 .template_ablkcipher = { 1618 .setkey = ablkcipher_setkey, 1619 .encrypt = ablkcipher_encrypt, 1620 .decrypt = ablkcipher_decrypt, 1621 .geniv = "chainiv", 1622 .min_keysize = AES_MIN_KEY_SIZE, 1623 .max_keysize = AES_MAX_KEY_SIZE, 1624 .ivsize = AES_BLOCK_SIZE, 1625 }, 1626 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, 1627 }, 1628 { 1629 .name = "rfc3686(ctr(aes))", 1630 .driver_name = "rfc3686-ctr-aes-caam-qi", 1631 .blocksize = 1, 1632 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 1633 .template_ablkcipher = { 1634 .setkey = ablkcipher_setkey, 1635 .encrypt = ablkcipher_encrypt, 1636 .decrypt = ablkcipher_decrypt, 1637 .givencrypt = ablkcipher_givencrypt, 1638 .geniv = "<built-in>", 1639 .min_keysize = AES_MIN_KEY_SIZE + 1640 CTR_RFC3686_NONCE_SIZE, 1641 .max_keysize = AES_MAX_KEY_SIZE + 1642 CTR_RFC3686_NONCE_SIZE, 1643 .ivsize = CTR_RFC3686_IV_SIZE, 1644 }, 1645 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, 1646 }, 1647 { 1648 .name = "xts(aes)", 1649 .driver_name = "xts-aes-caam-qi", 1650 .blocksize = AES_BLOCK_SIZE, 1651 .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 1652 .template_ablkcipher = { 1653 .setkey = xts_ablkcipher_setkey, 1654 .encrypt = ablkcipher_encrypt, 1655 .decrypt = ablkcipher_decrypt, 1656 .geniv = "eseqiv", 1657 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1658 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1659 .ivsize = AES_BLOCK_SIZE, 1660 }, 1661 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, 1662 }, 1663 }; 1664 1665 static struct caam_aead_alg driver_aeads[] = { 1666 { 1667 .aead = { 1668 .base = { 1669 .cra_name = "rfc4106(gcm(aes))", 1670 .cra_driver_name = "rfc4106-gcm-aes-caam-qi", 1671 .cra_blocksize = 1, 1672 }, 1673 .setkey = rfc4106_setkey, 1674 .setauthsize = rfc4106_setauthsize, 1675 .encrypt = ipsec_gcm_encrypt, 1676 .decrypt = ipsec_gcm_decrypt, 1677 .ivsize = 8, 1678 .maxauthsize = AES_BLOCK_SIZE, 1679 }, 1680 .caam = { 1681 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1682 }, 1683 }, 1684 { 1685 .aead = { 1686 .base = { 1687 .cra_name = "rfc4543(gcm(aes))", 1688 .cra_driver_name = "rfc4543-gcm-aes-caam-qi", 1689 .cra_blocksize = 1, 1690 }, 1691 .setkey = rfc4543_setkey, 1692 .setauthsize = rfc4543_setauthsize, 1693 .encrypt = ipsec_gcm_encrypt, 1694 .decrypt = ipsec_gcm_decrypt, 1695 .ivsize = 8, 1696 .maxauthsize = AES_BLOCK_SIZE, 1697 }, 1698 .caam = { 1699 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1700 }, 1701 }, 1702 /* Galois Counter Mode */ 1703 { 1704 .aead = { 1705 .base = { 1706 .cra_name = "gcm(aes)", 1707 .cra_driver_name = "gcm-aes-caam-qi", 1708 .cra_blocksize = 1, 1709 }, 1710 .setkey = gcm_setkey, 1711 .setauthsize = gcm_setauthsize, 1712 .encrypt = aead_encrypt, 1713 .decrypt = aead_decrypt, 1714 .ivsize = 12, 1715 .maxauthsize = AES_BLOCK_SIZE, 1716 }, 1717 .caam = { 1718 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1719 } 1720 }, 1721 /* single-pass ipsec_esp descriptor */ 1722 { 1723 .aead = { 1724 .base = { 1725 .cra_name = "authenc(hmac(md5),cbc(aes))", 1726 .cra_driver_name = "authenc-hmac-md5-" 1727 "cbc-aes-caam-qi", 1728 .cra_blocksize = AES_BLOCK_SIZE, 1729 }, 1730 .setkey = aead_setkey, 1731 .setauthsize = aead_setauthsize, 1732 .encrypt = aead_encrypt, 1733 .decrypt = aead_decrypt, 1734 .ivsize = AES_BLOCK_SIZE, 1735 .maxauthsize = MD5_DIGEST_SIZE, 1736 }, 1737 .caam = { 1738 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1739 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1740 OP_ALG_AAI_HMAC_PRECOMP, 1741 } 1742 }, 1743 { 1744 .aead = { 1745 .base = { 1746 .cra_name = "echainiv(authenc(hmac(md5)," 1747 "cbc(aes)))", 1748 .cra_driver_name = "echainiv-authenc-hmac-md5-" 1749 "cbc-aes-caam-qi", 1750 .cra_blocksize = AES_BLOCK_SIZE, 1751 }, 1752 .setkey = aead_setkey, 1753 .setauthsize = aead_setauthsize, 1754 .encrypt = aead_encrypt, 1755 .decrypt = aead_decrypt, 1756 .ivsize = AES_BLOCK_SIZE, 1757 .maxauthsize = MD5_DIGEST_SIZE, 1758 }, 1759 .caam = { 1760 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1761 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1762 OP_ALG_AAI_HMAC_PRECOMP, 1763 .geniv = true, 1764 } 1765 }, 1766 { 1767 .aead = { 1768 .base = { 1769 .cra_name = "authenc(hmac(sha1),cbc(aes))", 1770 .cra_driver_name = "authenc-hmac-sha1-" 1771 "cbc-aes-caam-qi", 1772 .cra_blocksize = AES_BLOCK_SIZE, 1773 }, 1774 .setkey = aead_setkey, 1775 .setauthsize = aead_setauthsize, 1776 .encrypt = aead_encrypt, 1777 .decrypt = aead_decrypt, 1778 .ivsize = AES_BLOCK_SIZE, 1779 .maxauthsize = SHA1_DIGEST_SIZE, 1780 }, 1781 .caam = { 1782 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1783 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1784 OP_ALG_AAI_HMAC_PRECOMP, 1785 } 1786 }, 1787 { 1788 .aead = { 1789 .base = { 1790 .cra_name = "echainiv(authenc(hmac(sha1)," 1791 "cbc(aes)))", 1792 .cra_driver_name = "echainiv-authenc-" 1793 "hmac-sha1-cbc-aes-caam-qi", 1794 .cra_blocksize = AES_BLOCK_SIZE, 1795 }, 1796 .setkey = aead_setkey, 1797 .setauthsize = aead_setauthsize, 1798 .encrypt = aead_encrypt, 1799 .decrypt = aead_decrypt, 1800 .ivsize = AES_BLOCK_SIZE, 1801 .maxauthsize = SHA1_DIGEST_SIZE, 1802 }, 1803 .caam = { 1804 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1805 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1806 OP_ALG_AAI_HMAC_PRECOMP, 1807 .geniv = true, 1808 }, 1809 }, 1810 { 1811 .aead = { 1812 .base = { 1813 .cra_name = "authenc(hmac(sha224),cbc(aes))", 1814 .cra_driver_name = "authenc-hmac-sha224-" 1815 "cbc-aes-caam-qi", 1816 .cra_blocksize = AES_BLOCK_SIZE, 1817 }, 1818 .setkey = aead_setkey, 1819 .setauthsize = aead_setauthsize, 1820 .encrypt = aead_encrypt, 1821 .decrypt = aead_decrypt, 1822 .ivsize = AES_BLOCK_SIZE, 1823 .maxauthsize = SHA224_DIGEST_SIZE, 1824 }, 1825 .caam = { 1826 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1827 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1828 OP_ALG_AAI_HMAC_PRECOMP, 1829 } 1830 }, 1831 { 1832 .aead = { 1833 .base = { 1834 .cra_name = "echainiv(authenc(hmac(sha224)," 1835 "cbc(aes)))", 1836 .cra_driver_name = "echainiv-authenc-" 1837 "hmac-sha224-cbc-aes-caam-qi", 1838 .cra_blocksize = AES_BLOCK_SIZE, 1839 }, 1840 .setkey = aead_setkey, 1841 .setauthsize = aead_setauthsize, 1842 .encrypt = aead_encrypt, 1843 .decrypt = aead_decrypt, 1844 .ivsize = AES_BLOCK_SIZE, 1845 .maxauthsize = SHA224_DIGEST_SIZE, 1846 }, 1847 .caam = { 1848 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1849 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1850 OP_ALG_AAI_HMAC_PRECOMP, 1851 .geniv = true, 1852 } 1853 }, 1854 { 1855 .aead = { 1856 .base = { 1857 .cra_name = "authenc(hmac(sha256),cbc(aes))", 1858 .cra_driver_name = "authenc-hmac-sha256-" 1859 "cbc-aes-caam-qi", 1860 .cra_blocksize = AES_BLOCK_SIZE, 1861 }, 1862 .setkey = aead_setkey, 1863 .setauthsize = aead_setauthsize, 1864 .encrypt = aead_encrypt, 1865 .decrypt = aead_decrypt, 1866 .ivsize = AES_BLOCK_SIZE, 1867 .maxauthsize = SHA256_DIGEST_SIZE, 1868 }, 1869 .caam = { 1870 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1871 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1872 OP_ALG_AAI_HMAC_PRECOMP, 1873 } 1874 }, 1875 { 1876 .aead = { 1877 .base = { 1878 .cra_name = "echainiv(authenc(hmac(sha256)," 1879 "cbc(aes)))", 1880 .cra_driver_name = "echainiv-authenc-" 1881 "hmac-sha256-cbc-aes-" 1882 "caam-qi", 1883 .cra_blocksize = AES_BLOCK_SIZE, 1884 }, 1885 .setkey = aead_setkey, 1886 .setauthsize = aead_setauthsize, 1887 .encrypt = aead_encrypt, 1888 .decrypt = aead_decrypt, 1889 .ivsize = AES_BLOCK_SIZE, 1890 .maxauthsize = SHA256_DIGEST_SIZE, 1891 }, 1892 .caam = { 1893 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1894 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1895 OP_ALG_AAI_HMAC_PRECOMP, 1896 .geniv = true, 1897 } 1898 }, 1899 { 1900 .aead = { 1901 .base = { 1902 .cra_name = "authenc(hmac(sha384),cbc(aes))", 1903 .cra_driver_name = "authenc-hmac-sha384-" 1904 "cbc-aes-caam-qi", 1905 .cra_blocksize = AES_BLOCK_SIZE, 1906 }, 1907 .setkey = aead_setkey, 1908 .setauthsize = aead_setauthsize, 1909 .encrypt = aead_encrypt, 1910 .decrypt = aead_decrypt, 1911 .ivsize = AES_BLOCK_SIZE, 1912 .maxauthsize = SHA384_DIGEST_SIZE, 1913 }, 1914 .caam = { 1915 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1916 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 1917 OP_ALG_AAI_HMAC_PRECOMP, 1918 } 1919 }, 1920 { 1921 .aead = { 1922 .base = { 1923 .cra_name = "echainiv(authenc(hmac(sha384)," 1924 "cbc(aes)))", 1925 .cra_driver_name = "echainiv-authenc-" 1926 "hmac-sha384-cbc-aes-" 1927 "caam-qi", 1928 .cra_blocksize = AES_BLOCK_SIZE, 1929 }, 1930 .setkey = aead_setkey, 1931 .setauthsize = aead_setauthsize, 1932 .encrypt = aead_encrypt, 1933 .decrypt = aead_decrypt, 1934 .ivsize = AES_BLOCK_SIZE, 1935 .maxauthsize = SHA384_DIGEST_SIZE, 1936 }, 1937 .caam = { 1938 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1939 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 1940 OP_ALG_AAI_HMAC_PRECOMP, 1941 .geniv = true, 1942 } 1943 }, 1944 { 1945 .aead = { 1946 .base = { 1947 .cra_name = "authenc(hmac(sha512),cbc(aes))", 1948 .cra_driver_name = "authenc-hmac-sha512-" 1949 "cbc-aes-caam-qi", 1950 .cra_blocksize = AES_BLOCK_SIZE, 1951 }, 1952 .setkey = aead_setkey, 1953 .setauthsize = aead_setauthsize, 1954 .encrypt = aead_encrypt, 1955 .decrypt = aead_decrypt, 1956 .ivsize = AES_BLOCK_SIZE, 1957 .maxauthsize = SHA512_DIGEST_SIZE, 1958 }, 1959 .caam = { 1960 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1961 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 1962 OP_ALG_AAI_HMAC_PRECOMP, 1963 } 1964 }, 1965 { 1966 .aead = { 1967 .base = { 1968 .cra_name = "echainiv(authenc(hmac(sha512)," 1969 "cbc(aes)))", 1970 .cra_driver_name = "echainiv-authenc-" 1971 "hmac-sha512-cbc-aes-" 1972 "caam-qi", 1973 .cra_blocksize = AES_BLOCK_SIZE, 1974 }, 1975 .setkey = aead_setkey, 1976 .setauthsize = aead_setauthsize, 1977 .encrypt = aead_encrypt, 1978 .decrypt = aead_decrypt, 1979 .ivsize = AES_BLOCK_SIZE, 1980 .maxauthsize = SHA512_DIGEST_SIZE, 1981 }, 1982 .caam = { 1983 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1984 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 1985 OP_ALG_AAI_HMAC_PRECOMP, 1986 .geniv = true, 1987 } 1988 }, 1989 { 1990 .aead = { 1991 .base = { 1992 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 1993 .cra_driver_name = "authenc-hmac-md5-" 1994 "cbc-des3_ede-caam-qi", 1995 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1996 }, 1997 .setkey = aead_setkey, 1998 .setauthsize = aead_setauthsize, 1999 .encrypt = aead_encrypt, 2000 .decrypt = aead_decrypt, 2001 .ivsize = DES3_EDE_BLOCK_SIZE, 2002 .maxauthsize = MD5_DIGEST_SIZE, 2003 }, 2004 .caam = { 2005 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2006 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2007 OP_ALG_AAI_HMAC_PRECOMP, 2008 } 2009 }, 2010 { 2011 .aead = { 2012 .base = { 2013 .cra_name = "echainiv(authenc(hmac(md5)," 2014 "cbc(des3_ede)))", 2015 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2016 "cbc-des3_ede-caam-qi", 2017 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2018 }, 2019 .setkey = aead_setkey, 2020 .setauthsize = aead_setauthsize, 2021 .encrypt = aead_encrypt, 2022 .decrypt = aead_decrypt, 2023 .ivsize = DES3_EDE_BLOCK_SIZE, 2024 .maxauthsize = MD5_DIGEST_SIZE, 2025 }, 2026 .caam = { 2027 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2028 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2029 OP_ALG_AAI_HMAC_PRECOMP, 2030 .geniv = true, 2031 } 2032 }, 2033 { 2034 .aead = { 2035 .base = { 2036 .cra_name = "authenc(hmac(sha1)," 2037 "cbc(des3_ede))", 2038 .cra_driver_name = "authenc-hmac-sha1-" 2039 "cbc-des3_ede-caam-qi", 2040 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2041 }, 2042 .setkey = aead_setkey, 2043 .setauthsize = aead_setauthsize, 2044 .encrypt = aead_encrypt, 2045 .decrypt = aead_decrypt, 2046 .ivsize = DES3_EDE_BLOCK_SIZE, 2047 .maxauthsize = SHA1_DIGEST_SIZE, 2048 }, 2049 .caam = { 2050 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2051 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2052 OP_ALG_AAI_HMAC_PRECOMP, 2053 }, 2054 }, 2055 { 2056 .aead = { 2057 .base = { 2058 .cra_name = "echainiv(authenc(hmac(sha1)," 2059 "cbc(des3_ede)))", 2060 .cra_driver_name = "echainiv-authenc-" 2061 "hmac-sha1-" 2062 "cbc-des3_ede-caam-qi", 2063 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2064 }, 2065 .setkey = aead_setkey, 2066 .setauthsize = aead_setauthsize, 2067 .encrypt = aead_encrypt, 2068 .decrypt = aead_decrypt, 2069 .ivsize = DES3_EDE_BLOCK_SIZE, 2070 .maxauthsize = SHA1_DIGEST_SIZE, 2071 }, 2072 .caam = { 2073 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2074 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2075 OP_ALG_AAI_HMAC_PRECOMP, 2076 .geniv = true, 2077 } 2078 }, 2079 { 2080 .aead = { 2081 .base = { 2082 .cra_name = "authenc(hmac(sha224)," 2083 "cbc(des3_ede))", 2084 .cra_driver_name = "authenc-hmac-sha224-" 2085 "cbc-des3_ede-caam-qi", 2086 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2087 }, 2088 .setkey = aead_setkey, 2089 .setauthsize = aead_setauthsize, 2090 .encrypt = aead_encrypt, 2091 .decrypt = aead_decrypt, 2092 .ivsize = DES3_EDE_BLOCK_SIZE, 2093 .maxauthsize = SHA224_DIGEST_SIZE, 2094 }, 2095 .caam = { 2096 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2097 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2098 OP_ALG_AAI_HMAC_PRECOMP, 2099 }, 2100 }, 2101 { 2102 .aead = { 2103 .base = { 2104 .cra_name = "echainiv(authenc(hmac(sha224)," 2105 "cbc(des3_ede)))", 2106 .cra_driver_name = "echainiv-authenc-" 2107 "hmac-sha224-" 2108 "cbc-des3_ede-caam-qi", 2109 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2110 }, 2111 .setkey = aead_setkey, 2112 .setauthsize = aead_setauthsize, 2113 .encrypt = aead_encrypt, 2114 .decrypt = aead_decrypt, 2115 .ivsize = DES3_EDE_BLOCK_SIZE, 2116 .maxauthsize = SHA224_DIGEST_SIZE, 2117 }, 2118 .caam = { 2119 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2120 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2121 OP_ALG_AAI_HMAC_PRECOMP, 2122 .geniv = true, 2123 } 2124 }, 2125 { 2126 .aead = { 2127 .base = { 2128 .cra_name = "authenc(hmac(sha256)," 2129 "cbc(des3_ede))", 2130 .cra_driver_name = "authenc-hmac-sha256-" 2131 "cbc-des3_ede-caam-qi", 2132 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2133 }, 2134 .setkey = aead_setkey, 2135 .setauthsize = aead_setauthsize, 2136 .encrypt = aead_encrypt, 2137 .decrypt = aead_decrypt, 2138 .ivsize = DES3_EDE_BLOCK_SIZE, 2139 .maxauthsize = SHA256_DIGEST_SIZE, 2140 }, 2141 .caam = { 2142 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2143 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2144 OP_ALG_AAI_HMAC_PRECOMP, 2145 }, 2146 }, 2147 { 2148 .aead = { 2149 .base = { 2150 .cra_name = "echainiv(authenc(hmac(sha256)," 2151 "cbc(des3_ede)))", 2152 .cra_driver_name = "echainiv-authenc-" 2153 "hmac-sha256-" 2154 "cbc-des3_ede-caam-qi", 2155 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2156 }, 2157 .setkey = aead_setkey, 2158 .setauthsize = aead_setauthsize, 2159 .encrypt = aead_encrypt, 2160 .decrypt = aead_decrypt, 2161 .ivsize = DES3_EDE_BLOCK_SIZE, 2162 .maxauthsize = SHA256_DIGEST_SIZE, 2163 }, 2164 .caam = { 2165 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2166 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2167 OP_ALG_AAI_HMAC_PRECOMP, 2168 .geniv = true, 2169 } 2170 }, 2171 { 2172 .aead = { 2173 .base = { 2174 .cra_name = "authenc(hmac(sha384)," 2175 "cbc(des3_ede))", 2176 .cra_driver_name = "authenc-hmac-sha384-" 2177 "cbc-des3_ede-caam-qi", 2178 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2179 }, 2180 .setkey = aead_setkey, 2181 .setauthsize = aead_setauthsize, 2182 .encrypt = aead_encrypt, 2183 .decrypt = aead_decrypt, 2184 .ivsize = DES3_EDE_BLOCK_SIZE, 2185 .maxauthsize = SHA384_DIGEST_SIZE, 2186 }, 2187 .caam = { 2188 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2189 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2190 OP_ALG_AAI_HMAC_PRECOMP, 2191 }, 2192 }, 2193 { 2194 .aead = { 2195 .base = { 2196 .cra_name = "echainiv(authenc(hmac(sha384)," 2197 "cbc(des3_ede)))", 2198 .cra_driver_name = "echainiv-authenc-" 2199 "hmac-sha384-" 2200 "cbc-des3_ede-caam-qi", 2201 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2202 }, 2203 .setkey = aead_setkey, 2204 .setauthsize = aead_setauthsize, 2205 .encrypt = aead_encrypt, 2206 .decrypt = aead_decrypt, 2207 .ivsize = DES3_EDE_BLOCK_SIZE, 2208 .maxauthsize = SHA384_DIGEST_SIZE, 2209 }, 2210 .caam = { 2211 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2212 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2213 OP_ALG_AAI_HMAC_PRECOMP, 2214 .geniv = true, 2215 } 2216 }, 2217 { 2218 .aead = { 2219 .base = { 2220 .cra_name = "authenc(hmac(sha512)," 2221 "cbc(des3_ede))", 2222 .cra_driver_name = "authenc-hmac-sha512-" 2223 "cbc-des3_ede-caam-qi", 2224 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2225 }, 2226 .setkey = aead_setkey, 2227 .setauthsize = aead_setauthsize, 2228 .encrypt = aead_encrypt, 2229 .decrypt = aead_decrypt, 2230 .ivsize = DES3_EDE_BLOCK_SIZE, 2231 .maxauthsize = SHA512_DIGEST_SIZE, 2232 }, 2233 .caam = { 2234 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2235 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2236 OP_ALG_AAI_HMAC_PRECOMP, 2237 }, 2238 }, 2239 { 2240 .aead = { 2241 .base = { 2242 .cra_name = "echainiv(authenc(hmac(sha512)," 2243 "cbc(des3_ede)))", 2244 .cra_driver_name = "echainiv-authenc-" 2245 "hmac-sha512-" 2246 "cbc-des3_ede-caam-qi", 2247 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2248 }, 2249 .setkey = aead_setkey, 2250 .setauthsize = aead_setauthsize, 2251 .encrypt = aead_encrypt, 2252 .decrypt = aead_decrypt, 2253 .ivsize = DES3_EDE_BLOCK_SIZE, 2254 .maxauthsize = SHA512_DIGEST_SIZE, 2255 }, 2256 .caam = { 2257 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2258 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2259 OP_ALG_AAI_HMAC_PRECOMP, 2260 .geniv = true, 2261 } 2262 }, 2263 { 2264 .aead = { 2265 .base = { 2266 .cra_name = "authenc(hmac(md5),cbc(des))", 2267 .cra_driver_name = "authenc-hmac-md5-" 2268 "cbc-des-caam-qi", 2269 .cra_blocksize = DES_BLOCK_SIZE, 2270 }, 2271 .setkey = aead_setkey, 2272 .setauthsize = aead_setauthsize, 2273 .encrypt = aead_encrypt, 2274 .decrypt = aead_decrypt, 2275 .ivsize = DES_BLOCK_SIZE, 2276 .maxauthsize = MD5_DIGEST_SIZE, 2277 }, 2278 .caam = { 2279 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2280 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2281 OP_ALG_AAI_HMAC_PRECOMP, 2282 }, 2283 }, 2284 { 2285 .aead = { 2286 .base = { 2287 .cra_name = "echainiv(authenc(hmac(md5)," 2288 "cbc(des)))", 2289 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2290 "cbc-des-caam-qi", 2291 .cra_blocksize = DES_BLOCK_SIZE, 2292 }, 2293 .setkey = aead_setkey, 2294 .setauthsize = aead_setauthsize, 2295 .encrypt = aead_encrypt, 2296 .decrypt = aead_decrypt, 2297 .ivsize = DES_BLOCK_SIZE, 2298 .maxauthsize = MD5_DIGEST_SIZE, 2299 }, 2300 .caam = { 2301 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2302 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2303 OP_ALG_AAI_HMAC_PRECOMP, 2304 .geniv = true, 2305 } 2306 }, 2307 { 2308 .aead = { 2309 .base = { 2310 .cra_name = "authenc(hmac(sha1),cbc(des))", 2311 .cra_driver_name = "authenc-hmac-sha1-" 2312 "cbc-des-caam-qi", 2313 .cra_blocksize = DES_BLOCK_SIZE, 2314 }, 2315 .setkey = aead_setkey, 2316 .setauthsize = aead_setauthsize, 2317 .encrypt = aead_encrypt, 2318 .decrypt = aead_decrypt, 2319 .ivsize = DES_BLOCK_SIZE, 2320 .maxauthsize = SHA1_DIGEST_SIZE, 2321 }, 2322 .caam = { 2323 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2324 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2325 OP_ALG_AAI_HMAC_PRECOMP, 2326 }, 2327 }, 2328 { 2329 .aead = { 2330 .base = { 2331 .cra_name = "echainiv(authenc(hmac(sha1)," 2332 "cbc(des)))", 2333 .cra_driver_name = "echainiv-authenc-" 2334 "hmac-sha1-cbc-des-caam-qi", 2335 .cra_blocksize = DES_BLOCK_SIZE, 2336 }, 2337 .setkey = aead_setkey, 2338 .setauthsize = aead_setauthsize, 2339 .encrypt = aead_encrypt, 2340 .decrypt = aead_decrypt, 2341 .ivsize = DES_BLOCK_SIZE, 2342 .maxauthsize = SHA1_DIGEST_SIZE, 2343 }, 2344 .caam = { 2345 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2346 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2347 OP_ALG_AAI_HMAC_PRECOMP, 2348 .geniv = true, 2349 } 2350 }, 2351 { 2352 .aead = { 2353 .base = { 2354 .cra_name = "authenc(hmac(sha224),cbc(des))", 2355 .cra_driver_name = "authenc-hmac-sha224-" 2356 "cbc-des-caam-qi", 2357 .cra_blocksize = DES_BLOCK_SIZE, 2358 }, 2359 .setkey = aead_setkey, 2360 .setauthsize = aead_setauthsize, 2361 .encrypt = aead_encrypt, 2362 .decrypt = aead_decrypt, 2363 .ivsize = DES_BLOCK_SIZE, 2364 .maxauthsize = SHA224_DIGEST_SIZE, 2365 }, 2366 .caam = { 2367 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2368 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2369 OP_ALG_AAI_HMAC_PRECOMP, 2370 }, 2371 }, 2372 { 2373 .aead = { 2374 .base = { 2375 .cra_name = "echainiv(authenc(hmac(sha224)," 2376 "cbc(des)))", 2377 .cra_driver_name = "echainiv-authenc-" 2378 "hmac-sha224-cbc-des-" 2379 "caam-qi", 2380 .cra_blocksize = DES_BLOCK_SIZE, 2381 }, 2382 .setkey = aead_setkey, 2383 .setauthsize = aead_setauthsize, 2384 .encrypt = aead_encrypt, 2385 .decrypt = aead_decrypt, 2386 .ivsize = DES_BLOCK_SIZE, 2387 .maxauthsize = SHA224_DIGEST_SIZE, 2388 }, 2389 .caam = { 2390 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2391 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2392 OP_ALG_AAI_HMAC_PRECOMP, 2393 .geniv = true, 2394 } 2395 }, 2396 { 2397 .aead = { 2398 .base = { 2399 .cra_name = "authenc(hmac(sha256),cbc(des))", 2400 .cra_driver_name = "authenc-hmac-sha256-" 2401 "cbc-des-caam-qi", 2402 .cra_blocksize = DES_BLOCK_SIZE, 2403 }, 2404 .setkey = aead_setkey, 2405 .setauthsize = aead_setauthsize, 2406 .encrypt = aead_encrypt, 2407 .decrypt = aead_decrypt, 2408 .ivsize = DES_BLOCK_SIZE, 2409 .maxauthsize = SHA256_DIGEST_SIZE, 2410 }, 2411 .caam = { 2412 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2413 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2414 OP_ALG_AAI_HMAC_PRECOMP, 2415 }, 2416 }, 2417 { 2418 .aead = { 2419 .base = { 2420 .cra_name = "echainiv(authenc(hmac(sha256)," 2421 "cbc(des)))", 2422 .cra_driver_name = "echainiv-authenc-" 2423 "hmac-sha256-cbc-des-" 2424 "caam-qi", 2425 .cra_blocksize = DES_BLOCK_SIZE, 2426 }, 2427 .setkey = aead_setkey, 2428 .setauthsize = aead_setauthsize, 2429 .encrypt = aead_encrypt, 2430 .decrypt = aead_decrypt, 2431 .ivsize = DES_BLOCK_SIZE, 2432 .maxauthsize = SHA256_DIGEST_SIZE, 2433 }, 2434 .caam = { 2435 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2436 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2437 OP_ALG_AAI_HMAC_PRECOMP, 2438 .geniv = true, 2439 }, 2440 }, 2441 { 2442 .aead = { 2443 .base = { 2444 .cra_name = "authenc(hmac(sha384),cbc(des))", 2445 .cra_driver_name = "authenc-hmac-sha384-" 2446 "cbc-des-caam-qi", 2447 .cra_blocksize = DES_BLOCK_SIZE, 2448 }, 2449 .setkey = aead_setkey, 2450 .setauthsize = aead_setauthsize, 2451 .encrypt = aead_encrypt, 2452 .decrypt = aead_decrypt, 2453 .ivsize = DES_BLOCK_SIZE, 2454 .maxauthsize = SHA384_DIGEST_SIZE, 2455 }, 2456 .caam = { 2457 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2458 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2459 OP_ALG_AAI_HMAC_PRECOMP, 2460 }, 2461 }, 2462 { 2463 .aead = { 2464 .base = { 2465 .cra_name = "echainiv(authenc(hmac(sha384)," 2466 "cbc(des)))", 2467 .cra_driver_name = "echainiv-authenc-" 2468 "hmac-sha384-cbc-des-" 2469 "caam-qi", 2470 .cra_blocksize = DES_BLOCK_SIZE, 2471 }, 2472 .setkey = aead_setkey, 2473 .setauthsize = aead_setauthsize, 2474 .encrypt = aead_encrypt, 2475 .decrypt = aead_decrypt, 2476 .ivsize = DES_BLOCK_SIZE, 2477 .maxauthsize = SHA384_DIGEST_SIZE, 2478 }, 2479 .caam = { 2480 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2481 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2482 OP_ALG_AAI_HMAC_PRECOMP, 2483 .geniv = true, 2484 } 2485 }, 2486 { 2487 .aead = { 2488 .base = { 2489 .cra_name = "authenc(hmac(sha512),cbc(des))", 2490 .cra_driver_name = "authenc-hmac-sha512-" 2491 "cbc-des-caam-qi", 2492 .cra_blocksize = DES_BLOCK_SIZE, 2493 }, 2494 .setkey = aead_setkey, 2495 .setauthsize = aead_setauthsize, 2496 .encrypt = aead_encrypt, 2497 .decrypt = aead_decrypt, 2498 .ivsize = DES_BLOCK_SIZE, 2499 .maxauthsize = SHA512_DIGEST_SIZE, 2500 }, 2501 .caam = { 2502 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2503 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2504 OP_ALG_AAI_HMAC_PRECOMP, 2505 } 2506 }, 2507 { 2508 .aead = { 2509 .base = { 2510 .cra_name = "echainiv(authenc(hmac(sha512)," 2511 "cbc(des)))", 2512 .cra_driver_name = "echainiv-authenc-" 2513 "hmac-sha512-cbc-des-" 2514 "caam-qi", 2515 .cra_blocksize = DES_BLOCK_SIZE, 2516 }, 2517 .setkey = aead_setkey, 2518 .setauthsize = aead_setauthsize, 2519 .encrypt = aead_encrypt, 2520 .decrypt = aead_decrypt, 2521 .ivsize = DES_BLOCK_SIZE, 2522 .maxauthsize = SHA512_DIGEST_SIZE, 2523 }, 2524 .caam = { 2525 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2526 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2527 OP_ALG_AAI_HMAC_PRECOMP, 2528 .geniv = true, 2529 } 2530 }, 2531 }; 2532 2533 struct caam_crypto_alg { 2534 struct list_head entry; 2535 struct crypto_alg crypto_alg; 2536 struct caam_alg_entry caam; 2537 }; 2538 2539 static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam, 2540 bool uses_dkp) 2541 { 2542 struct caam_drv_private *priv; 2543 2544 /* 2545 * distribute tfms across job rings to ensure in-order 2546 * crypto request processing per tfm 2547 */ 2548 ctx->jrdev = caam_jr_alloc(); 2549 if (IS_ERR(ctx->jrdev)) { 2550 pr_err("Job Ring Device allocation for transform failed\n"); 2551 return PTR_ERR(ctx->jrdev); 2552 } 2553 2554 priv = dev_get_drvdata(ctx->jrdev->parent); 2555 if (priv->era >= 6 && uses_dkp) 2556 ctx->dir = DMA_BIDIRECTIONAL; 2557 else 2558 ctx->dir = DMA_TO_DEVICE; 2559 2560 ctx->key_dma = dma_map_single(ctx->jrdev, ctx->key, sizeof(ctx->key), 2561 ctx->dir); 2562 if (dma_mapping_error(ctx->jrdev, ctx->key_dma)) { 2563 dev_err(ctx->jrdev, "unable to map key\n"); 2564 caam_jr_free(ctx->jrdev); 2565 return -ENOMEM; 2566 } 2567 2568 /* copy descriptor header template value */ 2569 ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; 2570 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; 2571 2572 ctx->qidev = priv->qidev; 2573 2574 spin_lock_init(&ctx->lock); 2575 ctx->drv_ctx[ENCRYPT] = NULL; 2576 ctx->drv_ctx[DECRYPT] = NULL; 2577 ctx->drv_ctx[GIVENCRYPT] = NULL; 2578 2579 return 0; 2580 } 2581 2582 static int caam_cra_init(struct crypto_tfm *tfm) 2583 { 2584 struct crypto_alg *alg = tfm->__crt_alg; 2585 struct caam_crypto_alg *caam_alg = container_of(alg, typeof(*caam_alg), 2586 crypto_alg); 2587 struct caam_ctx *ctx = crypto_tfm_ctx(tfm); 2588 2589 return caam_init_common(ctx, &caam_alg->caam, false); 2590 } 2591 2592 static int caam_aead_init(struct crypto_aead *tfm) 2593 { 2594 struct aead_alg *alg = crypto_aead_alg(tfm); 2595 struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg), 2596 aead); 2597 struct caam_ctx *ctx = crypto_aead_ctx(tfm); 2598 2599 return caam_init_common(ctx, &caam_alg->caam, 2600 alg->setkey == aead_setkey); 2601 } 2602 2603 static void caam_exit_common(struct caam_ctx *ctx) 2604 { 2605 caam_drv_ctx_rel(ctx->drv_ctx[ENCRYPT]); 2606 caam_drv_ctx_rel(ctx->drv_ctx[DECRYPT]); 2607 caam_drv_ctx_rel(ctx->drv_ctx[GIVENCRYPT]); 2608 2609 dma_unmap_single(ctx->jrdev, ctx->key_dma, sizeof(ctx->key), ctx->dir); 2610 2611 caam_jr_free(ctx->jrdev); 2612 } 2613 2614 static void caam_cra_exit(struct crypto_tfm *tfm) 2615 { 2616 caam_exit_common(crypto_tfm_ctx(tfm)); 2617 } 2618 2619 static void caam_aead_exit(struct crypto_aead *tfm) 2620 { 2621 caam_exit_common(crypto_aead_ctx(tfm)); 2622 } 2623 2624 static struct list_head alg_list; 2625 static void __exit caam_qi_algapi_exit(void) 2626 { 2627 struct caam_crypto_alg *t_alg, *n; 2628 int i; 2629 2630 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 2631 struct caam_aead_alg *t_alg = driver_aeads + i; 2632 2633 if (t_alg->registered) 2634 crypto_unregister_aead(&t_alg->aead); 2635 } 2636 2637 if (!alg_list.next) 2638 return; 2639 2640 list_for_each_entry_safe(t_alg, n, &alg_list, entry) { 2641 crypto_unregister_alg(&t_alg->crypto_alg); 2642 list_del(&t_alg->entry); 2643 kfree(t_alg); 2644 } 2645 } 2646 2647 static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template 2648 *template) 2649 { 2650 struct caam_crypto_alg *t_alg; 2651 struct crypto_alg *alg; 2652 2653 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); 2654 if (!t_alg) 2655 return ERR_PTR(-ENOMEM); 2656 2657 alg = &t_alg->crypto_alg; 2658 2659 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name); 2660 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 2661 template->driver_name); 2662 alg->cra_module = THIS_MODULE; 2663 alg->cra_init = caam_cra_init; 2664 alg->cra_exit = caam_cra_exit; 2665 alg->cra_priority = CAAM_CRA_PRIORITY; 2666 alg->cra_blocksize = template->blocksize; 2667 alg->cra_alignmask = 0; 2668 alg->cra_ctxsize = sizeof(struct caam_ctx); 2669 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY | 2670 template->type; 2671 switch (template->type) { 2672 case CRYPTO_ALG_TYPE_GIVCIPHER: 2673 alg->cra_type = &crypto_givcipher_type; 2674 alg->cra_ablkcipher = template->template_ablkcipher; 2675 break; 2676 case CRYPTO_ALG_TYPE_ABLKCIPHER: 2677 alg->cra_type = &crypto_ablkcipher_type; 2678 alg->cra_ablkcipher = template->template_ablkcipher; 2679 break; 2680 } 2681 2682 t_alg->caam.class1_alg_type = template->class1_alg_type; 2683 t_alg->caam.class2_alg_type = template->class2_alg_type; 2684 2685 return t_alg; 2686 } 2687 2688 static void caam_aead_alg_init(struct caam_aead_alg *t_alg) 2689 { 2690 struct aead_alg *alg = &t_alg->aead; 2691 2692 alg->base.cra_module = THIS_MODULE; 2693 alg->base.cra_priority = CAAM_CRA_PRIORITY; 2694 alg->base.cra_ctxsize = sizeof(struct caam_ctx); 2695 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 2696 2697 alg->init = caam_aead_init; 2698 alg->exit = caam_aead_exit; 2699 } 2700 2701 static int __init caam_qi_algapi_init(void) 2702 { 2703 struct device_node *dev_node; 2704 struct platform_device *pdev; 2705 struct device *ctrldev; 2706 struct caam_drv_private *priv; 2707 int i = 0, err = 0; 2708 u32 cha_vid, cha_inst, des_inst, aes_inst, md_inst; 2709 unsigned int md_limit = SHA512_DIGEST_SIZE; 2710 bool registered = false; 2711 2712 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); 2713 if (!dev_node) { 2714 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); 2715 if (!dev_node) 2716 return -ENODEV; 2717 } 2718 2719 pdev = of_find_device_by_node(dev_node); 2720 of_node_put(dev_node); 2721 if (!pdev) 2722 return -ENODEV; 2723 2724 ctrldev = &pdev->dev; 2725 priv = dev_get_drvdata(ctrldev); 2726 2727 /* 2728 * If priv is NULL, it's probably because the caam driver wasn't 2729 * properly initialized (e.g. RNG4 init failed). Thus, bail out here. 2730 */ 2731 if (!priv || !priv->qi_present) 2732 return -ENODEV; 2733 2734 if (caam_dpaa2) { 2735 dev_info(ctrldev, "caam/qi frontend driver not suitable for DPAA 2.x, aborting...\n"); 2736 return -ENODEV; 2737 } 2738 2739 INIT_LIST_HEAD(&alg_list); 2740 2741 /* 2742 * Register crypto algorithms the device supports. 2743 * First, detect presence and attributes of DES, AES, and MD blocks. 2744 */ 2745 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls); 2746 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls); 2747 des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >> CHA_ID_LS_DES_SHIFT; 2748 aes_inst = (cha_inst & CHA_ID_LS_AES_MASK) >> CHA_ID_LS_AES_SHIFT; 2749 md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; 2750 2751 /* If MD is present, limit digest size based on LP256 */ 2752 if (md_inst && ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256)) 2753 md_limit = SHA256_DIGEST_SIZE; 2754 2755 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 2756 struct caam_crypto_alg *t_alg; 2757 struct caam_alg_template *alg = driver_algs + i; 2758 u32 alg_sel = alg->class1_alg_type & OP_ALG_ALGSEL_MASK; 2759 2760 /* Skip DES algorithms if not supported by device */ 2761 if (!des_inst && 2762 ((alg_sel == OP_ALG_ALGSEL_3DES) || 2763 (alg_sel == OP_ALG_ALGSEL_DES))) 2764 continue; 2765 2766 /* Skip AES algorithms if not supported by device */ 2767 if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES)) 2768 continue; 2769 2770 t_alg = caam_alg_alloc(alg); 2771 if (IS_ERR(t_alg)) { 2772 err = PTR_ERR(t_alg); 2773 dev_warn(priv->qidev, "%s alg allocation failed\n", 2774 alg->driver_name); 2775 continue; 2776 } 2777 2778 err = crypto_register_alg(&t_alg->crypto_alg); 2779 if (err) { 2780 dev_warn(priv->qidev, "%s alg registration failed\n", 2781 t_alg->crypto_alg.cra_driver_name); 2782 kfree(t_alg); 2783 continue; 2784 } 2785 2786 list_add_tail(&t_alg->entry, &alg_list); 2787 registered = true; 2788 } 2789 2790 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 2791 struct caam_aead_alg *t_alg = driver_aeads + i; 2792 u32 c1_alg_sel = t_alg->caam.class1_alg_type & 2793 OP_ALG_ALGSEL_MASK; 2794 u32 c2_alg_sel = t_alg->caam.class2_alg_type & 2795 OP_ALG_ALGSEL_MASK; 2796 u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; 2797 2798 /* Skip DES algorithms if not supported by device */ 2799 if (!des_inst && 2800 ((c1_alg_sel == OP_ALG_ALGSEL_3DES) || 2801 (c1_alg_sel == OP_ALG_ALGSEL_DES))) 2802 continue; 2803 2804 /* Skip AES algorithms if not supported by device */ 2805 if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES)) 2806 continue; 2807 2808 /* 2809 * Check support for AES algorithms not available 2810 * on LP devices. 2811 */ 2812 if (((cha_vid & CHA_ID_LS_AES_MASK) == CHA_ID_LS_AES_LP) && 2813 (alg_aai == OP_ALG_AAI_GCM)) 2814 continue; 2815 2816 /* 2817 * Skip algorithms requiring message digests 2818 * if MD or MD size is not supported by device. 2819 */ 2820 if (c2_alg_sel && 2821 (!md_inst || (t_alg->aead.maxauthsize > md_limit))) 2822 continue; 2823 2824 caam_aead_alg_init(t_alg); 2825 2826 err = crypto_register_aead(&t_alg->aead); 2827 if (err) { 2828 pr_warn("%s alg registration failed\n", 2829 t_alg->aead.base.cra_driver_name); 2830 continue; 2831 } 2832 2833 t_alg->registered = true; 2834 registered = true; 2835 } 2836 2837 if (registered) 2838 dev_info(priv->qidev, "algorithms registered in /proc/crypto\n"); 2839 2840 return err; 2841 } 2842 2843 module_init(caam_qi_algapi_init); 2844 module_exit(caam_qi_algapi_exit); 2845 2846 MODULE_LICENSE("GPL"); 2847 MODULE_DESCRIPTION("Support for crypto API using CAAM-QI backend"); 2848 MODULE_AUTHOR("Freescale Semiconductor"); 2849