1 /* 2 * Freescale FSL CAAM support for crypto API over QI backend. 3 * Based on caamalg.c 4 * 5 * Copyright 2013-2016 Freescale Semiconductor, Inc. 6 * Copyright 2016-2017 NXP 7 */ 8 9 #include "compat.h" 10 #include "ctrl.h" 11 #include "regs.h" 12 #include "intern.h" 13 #include "desc_constr.h" 14 #include "error.h" 15 #include "sg_sw_qm.h" 16 #include "key_gen.h" 17 #include "qi.h" 18 #include "jr.h" 19 #include "caamalg_desc.h" 20 21 /* 22 * crypto alg 23 */ 24 #define CAAM_CRA_PRIORITY 2000 25 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ 26 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ 27 SHA512_DIGEST_SIZE * 2) 28 29 #define DESC_MAX_USED_BYTES (DESC_QI_AEAD_GIVENC_LEN + \ 30 CAAM_MAX_KEY_SIZE) 31 #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) 32 33 struct caam_alg_entry { 34 int class1_alg_type; 35 int class2_alg_type; 36 bool rfc3686; 37 bool geniv; 38 }; 39 40 struct caam_aead_alg { 41 struct aead_alg aead; 42 struct caam_alg_entry caam; 43 bool registered; 44 }; 45 46 /* 47 * per-session context 48 */ 49 struct caam_ctx { 50 struct device *jrdev; 51 u32 sh_desc_enc[DESC_MAX_USED_LEN]; 52 u32 sh_desc_dec[DESC_MAX_USED_LEN]; 53 u32 sh_desc_givenc[DESC_MAX_USED_LEN]; 54 u8 key[CAAM_MAX_KEY_SIZE]; 55 dma_addr_t key_dma; 56 enum dma_data_direction dir; 57 struct alginfo adata; 58 struct alginfo cdata; 59 unsigned int authsize; 60 struct device *qidev; 61 spinlock_t lock; /* Protects multiple init of driver context */ 62 struct caam_drv_ctx *drv_ctx[NUM_OP]; 63 }; 64 65 static int aead_set_sh_desc(struct crypto_aead *aead) 66 { 67 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 68 typeof(*alg), aead); 69 struct caam_ctx *ctx = crypto_aead_ctx(aead); 70 unsigned int ivsize = crypto_aead_ivsize(aead); 71 u32 ctx1_iv_off = 0; 72 u32 *nonce = NULL; 73 unsigned int data_len[2]; 74 u32 inl_mask; 75 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 76 OP_ALG_AAI_CTR_MOD128); 77 const bool is_rfc3686 = alg->caam.rfc3686; 78 struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); 79 80 if (!ctx->cdata.keylen || !ctx->authsize) 81 return 0; 82 83 /* 84 * AES-CTR needs to load IV in CONTEXT1 reg 85 * at an offset of 128bits (16bytes) 86 * CONTEXT1[255:128] = IV 87 */ 88 if (ctr_mode) 89 ctx1_iv_off = 16; 90 91 /* 92 * RFC3686 specific: 93 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 94 */ 95 if (is_rfc3686) { 96 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 97 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + 98 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); 99 } 100 101 data_len[0] = ctx->adata.keylen_pad; 102 data_len[1] = ctx->cdata.keylen; 103 104 if (alg->caam.geniv) 105 goto skip_enc; 106 107 /* aead_encrypt shared descriptor */ 108 if (desc_inline_query(DESC_QI_AEAD_ENC_LEN + 109 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 110 DESC_JOB_IO_LEN, data_len, &inl_mask, 111 ARRAY_SIZE(data_len)) < 0) 112 return -EINVAL; 113 114 if (inl_mask & 1) 115 ctx->adata.key_virt = ctx->key; 116 else 117 ctx->adata.key_dma = ctx->key_dma; 118 119 if (inl_mask & 2) 120 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 121 else 122 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 123 124 ctx->adata.key_inline = !!(inl_mask & 1); 125 ctx->cdata.key_inline = !!(inl_mask & 2); 126 127 cnstr_shdsc_aead_encap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata, 128 ivsize, ctx->authsize, is_rfc3686, nonce, 129 ctx1_iv_off, true, ctrlpriv->era); 130 131 skip_enc: 132 /* aead_decrypt shared descriptor */ 133 if (desc_inline_query(DESC_QI_AEAD_DEC_LEN + 134 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 135 DESC_JOB_IO_LEN, data_len, &inl_mask, 136 ARRAY_SIZE(data_len)) < 0) 137 return -EINVAL; 138 139 if (inl_mask & 1) 140 ctx->adata.key_virt = ctx->key; 141 else 142 ctx->adata.key_dma = ctx->key_dma; 143 144 if (inl_mask & 2) 145 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 146 else 147 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 148 149 ctx->adata.key_inline = !!(inl_mask & 1); 150 ctx->cdata.key_inline = !!(inl_mask & 2); 151 152 cnstr_shdsc_aead_decap(ctx->sh_desc_dec, &ctx->cdata, &ctx->adata, 153 ivsize, ctx->authsize, alg->caam.geniv, 154 is_rfc3686, nonce, ctx1_iv_off, true, 155 ctrlpriv->era); 156 157 if (!alg->caam.geniv) 158 goto skip_givenc; 159 160 /* aead_givencrypt shared descriptor */ 161 if (desc_inline_query(DESC_QI_AEAD_GIVENC_LEN + 162 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 163 DESC_JOB_IO_LEN, data_len, &inl_mask, 164 ARRAY_SIZE(data_len)) < 0) 165 return -EINVAL; 166 167 if (inl_mask & 1) 168 ctx->adata.key_virt = ctx->key; 169 else 170 ctx->adata.key_dma = ctx->key_dma; 171 172 if (inl_mask & 2) 173 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 174 else 175 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 176 177 ctx->adata.key_inline = !!(inl_mask & 1); 178 ctx->cdata.key_inline = !!(inl_mask & 2); 179 180 cnstr_shdsc_aead_givencap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata, 181 ivsize, ctx->authsize, is_rfc3686, nonce, 182 ctx1_iv_off, true, ctrlpriv->era); 183 184 skip_givenc: 185 return 0; 186 } 187 188 static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 189 { 190 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 191 192 ctx->authsize = authsize; 193 aead_set_sh_desc(authenc); 194 195 return 0; 196 } 197 198 static int aead_setkey(struct crypto_aead *aead, const u8 *key, 199 unsigned int keylen) 200 { 201 struct caam_ctx *ctx = crypto_aead_ctx(aead); 202 struct device *jrdev = ctx->jrdev; 203 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); 204 struct crypto_authenc_keys keys; 205 int ret = 0; 206 207 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 208 goto badkey; 209 210 #ifdef DEBUG 211 dev_err(jrdev, "keylen %d enckeylen %d authkeylen %d\n", 212 keys.authkeylen + keys.enckeylen, keys.enckeylen, 213 keys.authkeylen); 214 print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", 215 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 216 #endif 217 218 /* 219 * If DKP is supported, use it in the shared descriptor to generate 220 * the split key. 221 */ 222 if (ctrlpriv->era >= 6) { 223 ctx->adata.keylen = keys.authkeylen; 224 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & 225 OP_ALG_ALGSEL_MASK); 226 227 if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) 228 goto badkey; 229 230 memcpy(ctx->key, keys.authkey, keys.authkeylen); 231 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, 232 keys.enckeylen); 233 dma_sync_single_for_device(jrdev, ctx->key_dma, 234 ctx->adata.keylen_pad + 235 keys.enckeylen, ctx->dir); 236 goto skip_split_key; 237 } 238 239 ret = gen_split_key(jrdev, ctx->key, &ctx->adata, keys.authkey, 240 keys.authkeylen, CAAM_MAX_KEY_SIZE - 241 keys.enckeylen); 242 if (ret) 243 goto badkey; 244 245 /* postpend encryption key to auth split key */ 246 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); 247 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad + 248 keys.enckeylen, ctx->dir); 249 #ifdef DEBUG 250 print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ", 251 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 252 ctx->adata.keylen_pad + keys.enckeylen, 1); 253 #endif 254 255 skip_split_key: 256 ctx->cdata.keylen = keys.enckeylen; 257 258 ret = aead_set_sh_desc(aead); 259 if (ret) 260 goto badkey; 261 262 /* Now update the driver contexts with the new shared descriptor */ 263 if (ctx->drv_ctx[ENCRYPT]) { 264 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 265 ctx->sh_desc_enc); 266 if (ret) { 267 dev_err(jrdev, "driver enc context update failed\n"); 268 goto badkey; 269 } 270 } 271 272 if (ctx->drv_ctx[DECRYPT]) { 273 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 274 ctx->sh_desc_dec); 275 if (ret) { 276 dev_err(jrdev, "driver dec context update failed\n"); 277 goto badkey; 278 } 279 } 280 281 memzero_explicit(&keys, sizeof(keys)); 282 return ret; 283 badkey: 284 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); 285 memzero_explicit(&keys, sizeof(keys)); 286 return -EINVAL; 287 } 288 289 static int gcm_set_sh_desc(struct crypto_aead *aead) 290 { 291 struct caam_ctx *ctx = crypto_aead_ctx(aead); 292 unsigned int ivsize = crypto_aead_ivsize(aead); 293 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 294 ctx->cdata.keylen; 295 296 if (!ctx->cdata.keylen || !ctx->authsize) 297 return 0; 298 299 /* 300 * Job Descriptor and Shared Descriptor 301 * must fit into the 64-word Descriptor h/w Buffer 302 */ 303 if (rem_bytes >= DESC_QI_GCM_ENC_LEN) { 304 ctx->cdata.key_inline = true; 305 ctx->cdata.key_virt = ctx->key; 306 } else { 307 ctx->cdata.key_inline = false; 308 ctx->cdata.key_dma = ctx->key_dma; 309 } 310 311 cnstr_shdsc_gcm_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, 312 ctx->authsize, true); 313 314 /* 315 * Job Descriptor and Shared Descriptor 316 * must fit into the 64-word Descriptor h/w Buffer 317 */ 318 if (rem_bytes >= DESC_QI_GCM_DEC_LEN) { 319 ctx->cdata.key_inline = true; 320 ctx->cdata.key_virt = ctx->key; 321 } else { 322 ctx->cdata.key_inline = false; 323 ctx->cdata.key_dma = ctx->key_dma; 324 } 325 326 cnstr_shdsc_gcm_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, 327 ctx->authsize, true); 328 329 return 0; 330 } 331 332 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 333 { 334 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 335 336 ctx->authsize = authsize; 337 gcm_set_sh_desc(authenc); 338 339 return 0; 340 } 341 342 static int gcm_setkey(struct crypto_aead *aead, 343 const u8 *key, unsigned int keylen) 344 { 345 struct caam_ctx *ctx = crypto_aead_ctx(aead); 346 struct device *jrdev = ctx->jrdev; 347 int ret; 348 349 #ifdef DEBUG 350 print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", 351 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 352 #endif 353 354 memcpy(ctx->key, key, keylen); 355 dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir); 356 ctx->cdata.keylen = keylen; 357 358 ret = gcm_set_sh_desc(aead); 359 if (ret) 360 return ret; 361 362 /* Now update the driver contexts with the new shared descriptor */ 363 if (ctx->drv_ctx[ENCRYPT]) { 364 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 365 ctx->sh_desc_enc); 366 if (ret) { 367 dev_err(jrdev, "driver enc context update failed\n"); 368 return ret; 369 } 370 } 371 372 if (ctx->drv_ctx[DECRYPT]) { 373 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 374 ctx->sh_desc_dec); 375 if (ret) { 376 dev_err(jrdev, "driver dec context update failed\n"); 377 return ret; 378 } 379 } 380 381 return 0; 382 } 383 384 static int rfc4106_set_sh_desc(struct crypto_aead *aead) 385 { 386 struct caam_ctx *ctx = crypto_aead_ctx(aead); 387 unsigned int ivsize = crypto_aead_ivsize(aead); 388 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 389 ctx->cdata.keylen; 390 391 if (!ctx->cdata.keylen || !ctx->authsize) 392 return 0; 393 394 ctx->cdata.key_virt = ctx->key; 395 396 /* 397 * Job Descriptor and Shared Descriptor 398 * must fit into the 64-word Descriptor h/w Buffer 399 */ 400 if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) { 401 ctx->cdata.key_inline = true; 402 } else { 403 ctx->cdata.key_inline = false; 404 ctx->cdata.key_dma = ctx->key_dma; 405 } 406 407 cnstr_shdsc_rfc4106_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, 408 ctx->authsize, true); 409 410 /* 411 * Job Descriptor and Shared Descriptor 412 * must fit into the 64-word Descriptor h/w Buffer 413 */ 414 if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) { 415 ctx->cdata.key_inline = true; 416 } else { 417 ctx->cdata.key_inline = false; 418 ctx->cdata.key_dma = ctx->key_dma; 419 } 420 421 cnstr_shdsc_rfc4106_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, 422 ctx->authsize, true); 423 424 return 0; 425 } 426 427 static int rfc4106_setauthsize(struct crypto_aead *authenc, 428 unsigned int authsize) 429 { 430 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 431 432 ctx->authsize = authsize; 433 rfc4106_set_sh_desc(authenc); 434 435 return 0; 436 } 437 438 static int rfc4106_setkey(struct crypto_aead *aead, 439 const u8 *key, unsigned int keylen) 440 { 441 struct caam_ctx *ctx = crypto_aead_ctx(aead); 442 struct device *jrdev = ctx->jrdev; 443 int ret; 444 445 if (keylen < 4) 446 return -EINVAL; 447 448 #ifdef DEBUG 449 print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", 450 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 451 #endif 452 453 memcpy(ctx->key, key, keylen); 454 /* 455 * The last four bytes of the key material are used as the salt value 456 * in the nonce. Update the AES key length. 457 */ 458 ctx->cdata.keylen = keylen - 4; 459 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, 460 ctx->dir); 461 462 ret = rfc4106_set_sh_desc(aead); 463 if (ret) 464 return ret; 465 466 /* Now update the driver contexts with the new shared descriptor */ 467 if (ctx->drv_ctx[ENCRYPT]) { 468 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 469 ctx->sh_desc_enc); 470 if (ret) { 471 dev_err(jrdev, "driver enc context update failed\n"); 472 return ret; 473 } 474 } 475 476 if (ctx->drv_ctx[DECRYPT]) { 477 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 478 ctx->sh_desc_dec); 479 if (ret) { 480 dev_err(jrdev, "driver dec context update failed\n"); 481 return ret; 482 } 483 } 484 485 return 0; 486 } 487 488 static int rfc4543_set_sh_desc(struct crypto_aead *aead) 489 { 490 struct caam_ctx *ctx = crypto_aead_ctx(aead); 491 unsigned int ivsize = crypto_aead_ivsize(aead); 492 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 493 ctx->cdata.keylen; 494 495 if (!ctx->cdata.keylen || !ctx->authsize) 496 return 0; 497 498 ctx->cdata.key_virt = ctx->key; 499 500 /* 501 * Job Descriptor and Shared Descriptor 502 * must fit into the 64-word Descriptor h/w Buffer 503 */ 504 if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) { 505 ctx->cdata.key_inline = true; 506 } else { 507 ctx->cdata.key_inline = false; 508 ctx->cdata.key_dma = ctx->key_dma; 509 } 510 511 cnstr_shdsc_rfc4543_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, 512 ctx->authsize, true); 513 514 /* 515 * Job Descriptor and Shared Descriptor 516 * must fit into the 64-word Descriptor h/w Buffer 517 */ 518 if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) { 519 ctx->cdata.key_inline = true; 520 } else { 521 ctx->cdata.key_inline = false; 522 ctx->cdata.key_dma = ctx->key_dma; 523 } 524 525 cnstr_shdsc_rfc4543_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, 526 ctx->authsize, true); 527 528 return 0; 529 } 530 531 static int rfc4543_setauthsize(struct crypto_aead *authenc, 532 unsigned int authsize) 533 { 534 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 535 536 ctx->authsize = authsize; 537 rfc4543_set_sh_desc(authenc); 538 539 return 0; 540 } 541 542 static int rfc4543_setkey(struct crypto_aead *aead, 543 const u8 *key, unsigned int keylen) 544 { 545 struct caam_ctx *ctx = crypto_aead_ctx(aead); 546 struct device *jrdev = ctx->jrdev; 547 int ret; 548 549 if (keylen < 4) 550 return -EINVAL; 551 552 #ifdef DEBUG 553 print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", 554 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 555 #endif 556 557 memcpy(ctx->key, key, keylen); 558 /* 559 * The last four bytes of the key material are used as the salt value 560 * in the nonce. Update the AES key length. 561 */ 562 ctx->cdata.keylen = keylen - 4; 563 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, 564 ctx->dir); 565 566 ret = rfc4543_set_sh_desc(aead); 567 if (ret) 568 return ret; 569 570 /* Now update the driver contexts with the new shared descriptor */ 571 if (ctx->drv_ctx[ENCRYPT]) { 572 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 573 ctx->sh_desc_enc); 574 if (ret) { 575 dev_err(jrdev, "driver enc context update failed\n"); 576 return ret; 577 } 578 } 579 580 if (ctx->drv_ctx[DECRYPT]) { 581 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 582 ctx->sh_desc_dec); 583 if (ret) { 584 dev_err(jrdev, "driver dec context update failed\n"); 585 return ret; 586 } 587 } 588 589 return 0; 590 } 591 592 static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, 593 const u8 *key, unsigned int keylen) 594 { 595 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 596 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(ablkcipher); 597 const char *alg_name = crypto_tfm_alg_name(tfm); 598 struct device *jrdev = ctx->jrdev; 599 unsigned int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 600 u32 ctx1_iv_off = 0; 601 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 602 OP_ALG_AAI_CTR_MOD128); 603 const bool is_rfc3686 = (ctr_mode && strstr(alg_name, "rfc3686")); 604 int ret = 0; 605 606 #ifdef DEBUG 607 print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", 608 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 609 #endif 610 /* 611 * AES-CTR needs to load IV in CONTEXT1 reg 612 * at an offset of 128bits (16bytes) 613 * CONTEXT1[255:128] = IV 614 */ 615 if (ctr_mode) 616 ctx1_iv_off = 16; 617 618 /* 619 * RFC3686 specific: 620 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} 621 * | *key = {KEY, NONCE} 622 */ 623 if (is_rfc3686) { 624 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 625 keylen -= CTR_RFC3686_NONCE_SIZE; 626 } 627 628 ctx->cdata.keylen = keylen; 629 ctx->cdata.key_virt = key; 630 ctx->cdata.key_inline = true; 631 632 /* ablkcipher encrypt, decrypt, givencrypt shared descriptors */ 633 cnstr_shdsc_ablkcipher_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, 634 is_rfc3686, ctx1_iv_off); 635 cnstr_shdsc_ablkcipher_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, 636 is_rfc3686, ctx1_iv_off); 637 cnstr_shdsc_ablkcipher_givencap(ctx->sh_desc_givenc, &ctx->cdata, 638 ivsize, is_rfc3686, ctx1_iv_off); 639 640 /* Now update the driver contexts with the new shared descriptor */ 641 if (ctx->drv_ctx[ENCRYPT]) { 642 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 643 ctx->sh_desc_enc); 644 if (ret) { 645 dev_err(jrdev, "driver enc context update failed\n"); 646 goto badkey; 647 } 648 } 649 650 if (ctx->drv_ctx[DECRYPT]) { 651 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 652 ctx->sh_desc_dec); 653 if (ret) { 654 dev_err(jrdev, "driver dec context update failed\n"); 655 goto badkey; 656 } 657 } 658 659 if (ctx->drv_ctx[GIVENCRYPT]) { 660 ret = caam_drv_ctx_update(ctx->drv_ctx[GIVENCRYPT], 661 ctx->sh_desc_givenc); 662 if (ret) { 663 dev_err(jrdev, "driver givenc context update failed\n"); 664 goto badkey; 665 } 666 } 667 668 return ret; 669 badkey: 670 crypto_ablkcipher_set_flags(ablkcipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 671 return -EINVAL; 672 } 673 674 static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, 675 const u8 *key, unsigned int keylen) 676 { 677 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 678 struct device *jrdev = ctx->jrdev; 679 int ret = 0; 680 681 if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) { 682 dev_err(jrdev, "key size mismatch\n"); 683 goto badkey; 684 } 685 686 ctx->cdata.keylen = keylen; 687 ctx->cdata.key_virt = key; 688 ctx->cdata.key_inline = true; 689 690 /* xts ablkcipher encrypt, decrypt shared descriptors */ 691 cnstr_shdsc_xts_ablkcipher_encap(ctx->sh_desc_enc, &ctx->cdata); 692 cnstr_shdsc_xts_ablkcipher_decap(ctx->sh_desc_dec, &ctx->cdata); 693 694 /* Now update the driver contexts with the new shared descriptor */ 695 if (ctx->drv_ctx[ENCRYPT]) { 696 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 697 ctx->sh_desc_enc); 698 if (ret) { 699 dev_err(jrdev, "driver enc context update failed\n"); 700 goto badkey; 701 } 702 } 703 704 if (ctx->drv_ctx[DECRYPT]) { 705 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 706 ctx->sh_desc_dec); 707 if (ret) { 708 dev_err(jrdev, "driver dec context update failed\n"); 709 goto badkey; 710 } 711 } 712 713 return ret; 714 badkey: 715 crypto_ablkcipher_set_flags(ablkcipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 716 return -EINVAL; 717 } 718 719 /* 720 * aead_edesc - s/w-extended aead descriptor 721 * @src_nents: number of segments in input scatterlist 722 * @dst_nents: number of segments in output scatterlist 723 * @iv_dma: dma address of iv for checking continuity and link table 724 * @qm_sg_bytes: length of dma mapped h/w link table 725 * @qm_sg_dma: bus physical mapped address of h/w link table 726 * @assoclen: associated data length, in CAAM endianness 727 * @assoclen_dma: bus physical mapped address of req->assoclen 728 * @drv_req: driver-specific request structure 729 * @sgt: the h/w link table, followed by IV 730 */ 731 struct aead_edesc { 732 int src_nents; 733 int dst_nents; 734 dma_addr_t iv_dma; 735 int qm_sg_bytes; 736 dma_addr_t qm_sg_dma; 737 unsigned int assoclen; 738 dma_addr_t assoclen_dma; 739 struct caam_drv_req drv_req; 740 struct qm_sg_entry sgt[0]; 741 }; 742 743 /* 744 * ablkcipher_edesc - s/w-extended ablkcipher descriptor 745 * @src_nents: number of segments in input scatterlist 746 * @dst_nents: number of segments in output scatterlist 747 * @iv_dma: dma address of iv for checking continuity and link table 748 * @qm_sg_bytes: length of dma mapped h/w link table 749 * @qm_sg_dma: bus physical mapped address of h/w link table 750 * @drv_req: driver-specific request structure 751 * @sgt: the h/w link table, followed by IV 752 */ 753 struct ablkcipher_edesc { 754 int src_nents; 755 int dst_nents; 756 dma_addr_t iv_dma; 757 int qm_sg_bytes; 758 dma_addr_t qm_sg_dma; 759 struct caam_drv_req drv_req; 760 struct qm_sg_entry sgt[0]; 761 }; 762 763 static struct caam_drv_ctx *get_drv_ctx(struct caam_ctx *ctx, 764 enum optype type) 765 { 766 /* 767 * This function is called on the fast path with values of 'type' 768 * known at compile time. Invalid arguments are not expected and 769 * thus no checks are made. 770 */ 771 struct caam_drv_ctx *drv_ctx = ctx->drv_ctx[type]; 772 u32 *desc; 773 774 if (unlikely(!drv_ctx)) { 775 spin_lock(&ctx->lock); 776 777 /* Read again to check if some other core init drv_ctx */ 778 drv_ctx = ctx->drv_ctx[type]; 779 if (!drv_ctx) { 780 int cpu; 781 782 if (type == ENCRYPT) 783 desc = ctx->sh_desc_enc; 784 else if (type == DECRYPT) 785 desc = ctx->sh_desc_dec; 786 else /* (type == GIVENCRYPT) */ 787 desc = ctx->sh_desc_givenc; 788 789 cpu = smp_processor_id(); 790 drv_ctx = caam_drv_ctx_init(ctx->qidev, &cpu, desc); 791 if (likely(!IS_ERR_OR_NULL(drv_ctx))) 792 drv_ctx->op_type = type; 793 794 ctx->drv_ctx[type] = drv_ctx; 795 } 796 797 spin_unlock(&ctx->lock); 798 } 799 800 return drv_ctx; 801 } 802 803 static void caam_unmap(struct device *dev, struct scatterlist *src, 804 struct scatterlist *dst, int src_nents, 805 int dst_nents, dma_addr_t iv_dma, int ivsize, 806 enum optype op_type, dma_addr_t qm_sg_dma, 807 int qm_sg_bytes) 808 { 809 if (dst != src) { 810 if (src_nents) 811 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); 812 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); 813 } else { 814 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); 815 } 816 817 if (iv_dma) 818 dma_unmap_single(dev, iv_dma, ivsize, 819 op_type == GIVENCRYPT ? DMA_FROM_DEVICE : 820 DMA_TO_DEVICE); 821 if (qm_sg_bytes) 822 dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE); 823 } 824 825 static void aead_unmap(struct device *dev, 826 struct aead_edesc *edesc, 827 struct aead_request *req) 828 { 829 struct crypto_aead *aead = crypto_aead_reqtfm(req); 830 int ivsize = crypto_aead_ivsize(aead); 831 832 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 833 edesc->iv_dma, ivsize, edesc->drv_req.drv_ctx->op_type, 834 edesc->qm_sg_dma, edesc->qm_sg_bytes); 835 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 836 } 837 838 static void ablkcipher_unmap(struct device *dev, 839 struct ablkcipher_edesc *edesc, 840 struct ablkcipher_request *req) 841 { 842 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 843 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 844 845 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 846 edesc->iv_dma, ivsize, edesc->drv_req.drv_ctx->op_type, 847 edesc->qm_sg_dma, edesc->qm_sg_bytes); 848 } 849 850 static void aead_done(struct caam_drv_req *drv_req, u32 status) 851 { 852 struct device *qidev; 853 struct aead_edesc *edesc; 854 struct aead_request *aead_req = drv_req->app_ctx; 855 struct crypto_aead *aead = crypto_aead_reqtfm(aead_req); 856 struct caam_ctx *caam_ctx = crypto_aead_ctx(aead); 857 int ecode = 0; 858 859 qidev = caam_ctx->qidev; 860 861 if (unlikely(status)) { 862 u32 ssrc = status & JRSTA_SSRC_MASK; 863 u8 err_id = status & JRSTA_CCBERR_ERRID_MASK; 864 865 caam_jr_strstatus(qidev, status); 866 /* 867 * verify hw auth check passed else return -EBADMSG 868 */ 869 if (ssrc == JRSTA_SSRC_CCB_ERROR && 870 err_id == JRSTA_CCBERR_ERRID_ICVCHK) 871 ecode = -EBADMSG; 872 else 873 ecode = -EIO; 874 } 875 876 edesc = container_of(drv_req, typeof(*edesc), drv_req); 877 aead_unmap(qidev, edesc, aead_req); 878 879 aead_request_complete(aead_req, ecode); 880 qi_cache_free(edesc); 881 } 882 883 /* 884 * allocate and map the aead extended descriptor 885 */ 886 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 887 bool encrypt) 888 { 889 struct crypto_aead *aead = crypto_aead_reqtfm(req); 890 struct caam_ctx *ctx = crypto_aead_ctx(aead); 891 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 892 typeof(*alg), aead); 893 struct device *qidev = ctx->qidev; 894 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 895 GFP_KERNEL : GFP_ATOMIC; 896 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 897 struct aead_edesc *edesc; 898 dma_addr_t qm_sg_dma, iv_dma = 0; 899 int ivsize = 0; 900 unsigned int authsize = ctx->authsize; 901 int qm_sg_index = 0, qm_sg_ents = 0, qm_sg_bytes; 902 int in_len, out_len; 903 struct qm_sg_entry *sg_table, *fd_sgt; 904 struct caam_drv_ctx *drv_ctx; 905 enum optype op_type = encrypt ? ENCRYPT : DECRYPT; 906 907 drv_ctx = get_drv_ctx(ctx, op_type); 908 if (unlikely(IS_ERR_OR_NULL(drv_ctx))) 909 return (struct aead_edesc *)drv_ctx; 910 911 /* allocate space for base edesc and hw desc commands, link tables */ 912 edesc = qi_cache_alloc(GFP_DMA | flags); 913 if (unlikely(!edesc)) { 914 dev_err(qidev, "could not allocate extended descriptor\n"); 915 return ERR_PTR(-ENOMEM); 916 } 917 918 if (likely(req->src == req->dst)) { 919 src_nents = sg_nents_for_len(req->src, req->assoclen + 920 req->cryptlen + 921 (encrypt ? authsize : 0)); 922 if (unlikely(src_nents < 0)) { 923 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", 924 req->assoclen + req->cryptlen + 925 (encrypt ? authsize : 0)); 926 qi_cache_free(edesc); 927 return ERR_PTR(src_nents); 928 } 929 930 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, 931 DMA_BIDIRECTIONAL); 932 if (unlikely(!mapped_src_nents)) { 933 dev_err(qidev, "unable to map source\n"); 934 qi_cache_free(edesc); 935 return ERR_PTR(-ENOMEM); 936 } 937 } else { 938 src_nents = sg_nents_for_len(req->src, req->assoclen + 939 req->cryptlen); 940 if (unlikely(src_nents < 0)) { 941 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", 942 req->assoclen + req->cryptlen); 943 qi_cache_free(edesc); 944 return ERR_PTR(src_nents); 945 } 946 947 dst_nents = sg_nents_for_len(req->dst, req->assoclen + 948 req->cryptlen + 949 (encrypt ? authsize : 950 (-authsize))); 951 if (unlikely(dst_nents < 0)) { 952 dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n", 953 req->assoclen + req->cryptlen + 954 (encrypt ? authsize : (-authsize))); 955 qi_cache_free(edesc); 956 return ERR_PTR(dst_nents); 957 } 958 959 if (src_nents) { 960 mapped_src_nents = dma_map_sg(qidev, req->src, 961 src_nents, DMA_TO_DEVICE); 962 if (unlikely(!mapped_src_nents)) { 963 dev_err(qidev, "unable to map source\n"); 964 qi_cache_free(edesc); 965 return ERR_PTR(-ENOMEM); 966 } 967 } else { 968 mapped_src_nents = 0; 969 } 970 971 mapped_dst_nents = dma_map_sg(qidev, req->dst, dst_nents, 972 DMA_FROM_DEVICE); 973 if (unlikely(!mapped_dst_nents)) { 974 dev_err(qidev, "unable to map destination\n"); 975 dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE); 976 qi_cache_free(edesc); 977 return ERR_PTR(-ENOMEM); 978 } 979 } 980 981 if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv) 982 ivsize = crypto_aead_ivsize(aead); 983 984 /* 985 * Create S/G table: req->assoclen, [IV,] req->src [, req->dst]. 986 * Input is not contiguous. 987 */ 988 qm_sg_ents = 1 + !!ivsize + mapped_src_nents + 989 (mapped_dst_nents > 1 ? mapped_dst_nents : 0); 990 sg_table = &edesc->sgt[0]; 991 qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); 992 if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize > 993 CAAM_QI_MEMCACHE_SIZE)) { 994 dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n", 995 qm_sg_ents, ivsize); 996 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, 997 0, 0, 0, 0); 998 qi_cache_free(edesc); 999 return ERR_PTR(-ENOMEM); 1000 } 1001 1002 if (ivsize) { 1003 u8 *iv = (u8 *)(sg_table + qm_sg_ents); 1004 1005 /* Make sure IV is located in a DMAable area */ 1006 memcpy(iv, req->iv, ivsize); 1007 1008 iv_dma = dma_map_single(qidev, iv, ivsize, DMA_TO_DEVICE); 1009 if (dma_mapping_error(qidev, iv_dma)) { 1010 dev_err(qidev, "unable to map IV\n"); 1011 caam_unmap(qidev, req->src, req->dst, src_nents, 1012 dst_nents, 0, 0, 0, 0, 0); 1013 qi_cache_free(edesc); 1014 return ERR_PTR(-ENOMEM); 1015 } 1016 } 1017 1018 edesc->src_nents = src_nents; 1019 edesc->dst_nents = dst_nents; 1020 edesc->iv_dma = iv_dma; 1021 edesc->drv_req.app_ctx = req; 1022 edesc->drv_req.cbk = aead_done; 1023 edesc->drv_req.drv_ctx = drv_ctx; 1024 1025 edesc->assoclen = cpu_to_caam32(req->assoclen); 1026 edesc->assoclen_dma = dma_map_single(qidev, &edesc->assoclen, 4, 1027 DMA_TO_DEVICE); 1028 if (dma_mapping_error(qidev, edesc->assoclen_dma)) { 1029 dev_err(qidev, "unable to map assoclen\n"); 1030 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 1031 iv_dma, ivsize, op_type, 0, 0); 1032 qi_cache_free(edesc); 1033 return ERR_PTR(-ENOMEM); 1034 } 1035 1036 dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0); 1037 qm_sg_index++; 1038 if (ivsize) { 1039 dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0); 1040 qm_sg_index++; 1041 } 1042 sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + qm_sg_index, 0); 1043 qm_sg_index += mapped_src_nents; 1044 1045 if (mapped_dst_nents > 1) 1046 sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + 1047 qm_sg_index, 0); 1048 1049 qm_sg_dma = dma_map_single(qidev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); 1050 if (dma_mapping_error(qidev, qm_sg_dma)) { 1051 dev_err(qidev, "unable to map S/G table\n"); 1052 dma_unmap_single(qidev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 1053 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 1054 iv_dma, ivsize, op_type, 0, 0); 1055 qi_cache_free(edesc); 1056 return ERR_PTR(-ENOMEM); 1057 } 1058 1059 edesc->qm_sg_dma = qm_sg_dma; 1060 edesc->qm_sg_bytes = qm_sg_bytes; 1061 1062 out_len = req->assoclen + req->cryptlen + 1063 (encrypt ? ctx->authsize : (-ctx->authsize)); 1064 in_len = 4 + ivsize + req->assoclen + req->cryptlen; 1065 1066 fd_sgt = &edesc->drv_req.fd_sgt[0]; 1067 dma_to_qm_sg_one_last_ext(&fd_sgt[1], qm_sg_dma, in_len, 0); 1068 1069 if (req->dst == req->src) { 1070 if (mapped_src_nents == 1) 1071 dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->src), 1072 out_len, 0); 1073 else 1074 dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + 1075 (1 + !!ivsize) * sizeof(*sg_table), 1076 out_len, 0); 1077 } else if (mapped_dst_nents == 1) { 1078 dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), out_len, 1079 0); 1080 } else { 1081 dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + sizeof(*sg_table) * 1082 qm_sg_index, out_len, 0); 1083 } 1084 1085 return edesc; 1086 } 1087 1088 static inline int aead_crypt(struct aead_request *req, bool encrypt) 1089 { 1090 struct aead_edesc *edesc; 1091 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1092 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1093 int ret; 1094 1095 if (unlikely(caam_congested)) 1096 return -EAGAIN; 1097 1098 /* allocate extended descriptor */ 1099 edesc = aead_edesc_alloc(req, encrypt); 1100 if (IS_ERR_OR_NULL(edesc)) 1101 return PTR_ERR(edesc); 1102 1103 /* Create and submit job descriptor */ 1104 ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req); 1105 if (!ret) { 1106 ret = -EINPROGRESS; 1107 } else { 1108 aead_unmap(ctx->qidev, edesc, req); 1109 qi_cache_free(edesc); 1110 } 1111 1112 return ret; 1113 } 1114 1115 static int aead_encrypt(struct aead_request *req) 1116 { 1117 return aead_crypt(req, true); 1118 } 1119 1120 static int aead_decrypt(struct aead_request *req) 1121 { 1122 return aead_crypt(req, false); 1123 } 1124 1125 static int ipsec_gcm_encrypt(struct aead_request *req) 1126 { 1127 if (req->assoclen < 8) 1128 return -EINVAL; 1129 1130 return aead_crypt(req, true); 1131 } 1132 1133 static int ipsec_gcm_decrypt(struct aead_request *req) 1134 { 1135 if (req->assoclen < 8) 1136 return -EINVAL; 1137 1138 return aead_crypt(req, false); 1139 } 1140 1141 static void ablkcipher_done(struct caam_drv_req *drv_req, u32 status) 1142 { 1143 struct ablkcipher_edesc *edesc; 1144 struct ablkcipher_request *req = drv_req->app_ctx; 1145 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1146 struct caam_ctx *caam_ctx = crypto_ablkcipher_ctx(ablkcipher); 1147 struct device *qidev = caam_ctx->qidev; 1148 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1149 1150 #ifdef DEBUG 1151 dev_err(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status); 1152 #endif 1153 1154 edesc = container_of(drv_req, typeof(*edesc), drv_req); 1155 1156 if (status) 1157 caam_jr_strstatus(qidev, status); 1158 1159 #ifdef DEBUG 1160 print_hex_dump(KERN_ERR, "dstiv @" __stringify(__LINE__)": ", 1161 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 1162 edesc->src_nents > 1 ? 100 : ivsize, 1); 1163 caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", 1164 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 1165 edesc->dst_nents > 1 ? 100 : req->nbytes, 1); 1166 #endif 1167 1168 ablkcipher_unmap(qidev, edesc, req); 1169 1170 /* In case initial IV was generated, copy it in GIVCIPHER request */ 1171 if (edesc->drv_req.drv_ctx->op_type == GIVENCRYPT) { 1172 u8 *iv; 1173 struct skcipher_givcrypt_request *greq; 1174 1175 greq = container_of(req, struct skcipher_givcrypt_request, 1176 creq); 1177 iv = (u8 *)edesc->sgt + edesc->qm_sg_bytes; 1178 memcpy(greq->giv, iv, ivsize); 1179 } 1180 1181 /* 1182 * The crypto API expects us to set the IV (req->info) to the last 1183 * ciphertext block. This is used e.g. by the CTS mode. 1184 */ 1185 if (edesc->drv_req.drv_ctx->op_type != DECRYPT) 1186 scatterwalk_map_and_copy(req->info, req->dst, req->nbytes - 1187 ivsize, ivsize, 0); 1188 1189 qi_cache_free(edesc); 1190 ablkcipher_request_complete(req, status); 1191 } 1192 1193 static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request 1194 *req, bool encrypt) 1195 { 1196 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1197 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1198 struct device *qidev = ctx->qidev; 1199 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1200 GFP_KERNEL : GFP_ATOMIC; 1201 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 1202 struct ablkcipher_edesc *edesc; 1203 dma_addr_t iv_dma; 1204 u8 *iv; 1205 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1206 int dst_sg_idx, qm_sg_ents, qm_sg_bytes; 1207 struct qm_sg_entry *sg_table, *fd_sgt; 1208 struct caam_drv_ctx *drv_ctx; 1209 enum optype op_type = encrypt ? ENCRYPT : DECRYPT; 1210 1211 drv_ctx = get_drv_ctx(ctx, op_type); 1212 if (unlikely(IS_ERR_OR_NULL(drv_ctx))) 1213 return (struct ablkcipher_edesc *)drv_ctx; 1214 1215 src_nents = sg_nents_for_len(req->src, req->nbytes); 1216 if (unlikely(src_nents < 0)) { 1217 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", 1218 req->nbytes); 1219 return ERR_PTR(src_nents); 1220 } 1221 1222 if (unlikely(req->src != req->dst)) { 1223 dst_nents = sg_nents_for_len(req->dst, req->nbytes); 1224 if (unlikely(dst_nents < 0)) { 1225 dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n", 1226 req->nbytes); 1227 return ERR_PTR(dst_nents); 1228 } 1229 1230 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, 1231 DMA_TO_DEVICE); 1232 if (unlikely(!mapped_src_nents)) { 1233 dev_err(qidev, "unable to map source\n"); 1234 return ERR_PTR(-ENOMEM); 1235 } 1236 1237 mapped_dst_nents = dma_map_sg(qidev, req->dst, dst_nents, 1238 DMA_FROM_DEVICE); 1239 if (unlikely(!mapped_dst_nents)) { 1240 dev_err(qidev, "unable to map destination\n"); 1241 dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE); 1242 return ERR_PTR(-ENOMEM); 1243 } 1244 } else { 1245 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, 1246 DMA_BIDIRECTIONAL); 1247 if (unlikely(!mapped_src_nents)) { 1248 dev_err(qidev, "unable to map source\n"); 1249 return ERR_PTR(-ENOMEM); 1250 } 1251 } 1252 1253 qm_sg_ents = 1 + mapped_src_nents; 1254 dst_sg_idx = qm_sg_ents; 1255 1256 qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0; 1257 qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry); 1258 if (unlikely(offsetof(struct ablkcipher_edesc, sgt) + qm_sg_bytes + 1259 ivsize > CAAM_QI_MEMCACHE_SIZE)) { 1260 dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n", 1261 qm_sg_ents, ivsize); 1262 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, 1263 0, 0, 0, 0); 1264 return ERR_PTR(-ENOMEM); 1265 } 1266 1267 /* allocate space for base edesc, link tables and IV */ 1268 edesc = qi_cache_alloc(GFP_DMA | flags); 1269 if (unlikely(!edesc)) { 1270 dev_err(qidev, "could not allocate extended descriptor\n"); 1271 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, 1272 0, 0, 0, 0); 1273 return ERR_PTR(-ENOMEM); 1274 } 1275 1276 /* Make sure IV is located in a DMAable area */ 1277 sg_table = &edesc->sgt[0]; 1278 iv = (u8 *)(sg_table + qm_sg_ents); 1279 memcpy(iv, req->info, ivsize); 1280 1281 iv_dma = dma_map_single(qidev, iv, ivsize, DMA_TO_DEVICE); 1282 if (dma_mapping_error(qidev, iv_dma)) { 1283 dev_err(qidev, "unable to map IV\n"); 1284 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, 1285 0, 0, 0, 0); 1286 qi_cache_free(edesc); 1287 return ERR_PTR(-ENOMEM); 1288 } 1289 1290 edesc->src_nents = src_nents; 1291 edesc->dst_nents = dst_nents; 1292 edesc->iv_dma = iv_dma; 1293 edesc->qm_sg_bytes = qm_sg_bytes; 1294 edesc->drv_req.app_ctx = req; 1295 edesc->drv_req.cbk = ablkcipher_done; 1296 edesc->drv_req.drv_ctx = drv_ctx; 1297 1298 dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); 1299 sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + 1, 0); 1300 1301 if (mapped_dst_nents > 1) 1302 sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + 1303 dst_sg_idx, 0); 1304 1305 edesc->qm_sg_dma = dma_map_single(qidev, sg_table, edesc->qm_sg_bytes, 1306 DMA_TO_DEVICE); 1307 if (dma_mapping_error(qidev, edesc->qm_sg_dma)) { 1308 dev_err(qidev, "unable to map S/G table\n"); 1309 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 1310 iv_dma, ivsize, op_type, 0, 0); 1311 qi_cache_free(edesc); 1312 return ERR_PTR(-ENOMEM); 1313 } 1314 1315 fd_sgt = &edesc->drv_req.fd_sgt[0]; 1316 1317 dma_to_qm_sg_one_last_ext(&fd_sgt[1], edesc->qm_sg_dma, 1318 ivsize + req->nbytes, 0); 1319 1320 if (req->src == req->dst) { 1321 dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + 1322 sizeof(*sg_table), req->nbytes, 0); 1323 } else if (mapped_dst_nents > 1) { 1324 dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx * 1325 sizeof(*sg_table), req->nbytes, 0); 1326 } else { 1327 dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), 1328 req->nbytes, 0); 1329 } 1330 1331 return edesc; 1332 } 1333 1334 static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc( 1335 struct skcipher_givcrypt_request *creq) 1336 { 1337 struct ablkcipher_request *req = &creq->creq; 1338 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1339 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1340 struct device *qidev = ctx->qidev; 1341 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1342 GFP_KERNEL : GFP_ATOMIC; 1343 int src_nents, mapped_src_nents, dst_nents, mapped_dst_nents; 1344 struct ablkcipher_edesc *edesc; 1345 dma_addr_t iv_dma; 1346 u8 *iv; 1347 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1348 struct qm_sg_entry *sg_table, *fd_sgt; 1349 int dst_sg_idx, qm_sg_ents, qm_sg_bytes; 1350 struct caam_drv_ctx *drv_ctx; 1351 1352 drv_ctx = get_drv_ctx(ctx, GIVENCRYPT); 1353 if (unlikely(IS_ERR_OR_NULL(drv_ctx))) 1354 return (struct ablkcipher_edesc *)drv_ctx; 1355 1356 src_nents = sg_nents_for_len(req->src, req->nbytes); 1357 if (unlikely(src_nents < 0)) { 1358 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", 1359 req->nbytes); 1360 return ERR_PTR(src_nents); 1361 } 1362 1363 if (unlikely(req->src != req->dst)) { 1364 dst_nents = sg_nents_for_len(req->dst, req->nbytes); 1365 if (unlikely(dst_nents < 0)) { 1366 dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n", 1367 req->nbytes); 1368 return ERR_PTR(dst_nents); 1369 } 1370 1371 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, 1372 DMA_TO_DEVICE); 1373 if (unlikely(!mapped_src_nents)) { 1374 dev_err(qidev, "unable to map source\n"); 1375 return ERR_PTR(-ENOMEM); 1376 } 1377 1378 mapped_dst_nents = dma_map_sg(qidev, req->dst, dst_nents, 1379 DMA_FROM_DEVICE); 1380 if (unlikely(!mapped_dst_nents)) { 1381 dev_err(qidev, "unable to map destination\n"); 1382 dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE); 1383 return ERR_PTR(-ENOMEM); 1384 } 1385 } else { 1386 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, 1387 DMA_BIDIRECTIONAL); 1388 if (unlikely(!mapped_src_nents)) { 1389 dev_err(qidev, "unable to map source\n"); 1390 return ERR_PTR(-ENOMEM); 1391 } 1392 1393 dst_nents = src_nents; 1394 mapped_dst_nents = src_nents; 1395 } 1396 1397 qm_sg_ents = mapped_src_nents > 1 ? mapped_src_nents : 0; 1398 dst_sg_idx = qm_sg_ents; 1399 1400 qm_sg_ents += 1 + mapped_dst_nents; 1401 qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry); 1402 if (unlikely(offsetof(struct ablkcipher_edesc, sgt) + qm_sg_bytes + 1403 ivsize > CAAM_QI_MEMCACHE_SIZE)) { 1404 dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n", 1405 qm_sg_ents, ivsize); 1406 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, 1407 0, 0, 0, 0); 1408 return ERR_PTR(-ENOMEM); 1409 } 1410 1411 /* allocate space for base edesc, link tables and IV */ 1412 edesc = qi_cache_alloc(GFP_DMA | flags); 1413 if (!edesc) { 1414 dev_err(qidev, "could not allocate extended descriptor\n"); 1415 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, 1416 0, 0, 0, 0); 1417 return ERR_PTR(-ENOMEM); 1418 } 1419 1420 /* Make sure IV is located in a DMAable area */ 1421 sg_table = &edesc->sgt[0]; 1422 iv = (u8 *)(sg_table + qm_sg_ents); 1423 iv_dma = dma_map_single(qidev, iv, ivsize, DMA_FROM_DEVICE); 1424 if (dma_mapping_error(qidev, iv_dma)) { 1425 dev_err(qidev, "unable to map IV\n"); 1426 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, 1427 0, 0, 0, 0); 1428 qi_cache_free(edesc); 1429 return ERR_PTR(-ENOMEM); 1430 } 1431 1432 edesc->src_nents = src_nents; 1433 edesc->dst_nents = dst_nents; 1434 edesc->iv_dma = iv_dma; 1435 edesc->qm_sg_bytes = qm_sg_bytes; 1436 edesc->drv_req.app_ctx = req; 1437 edesc->drv_req.cbk = ablkcipher_done; 1438 edesc->drv_req.drv_ctx = drv_ctx; 1439 1440 if (mapped_src_nents > 1) 1441 sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table, 0); 1442 1443 dma_to_qm_sg_one(sg_table + dst_sg_idx, iv_dma, ivsize, 0); 1444 sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + dst_sg_idx + 1, 1445 0); 1446 1447 edesc->qm_sg_dma = dma_map_single(qidev, sg_table, edesc->qm_sg_bytes, 1448 DMA_TO_DEVICE); 1449 if (dma_mapping_error(qidev, edesc->qm_sg_dma)) { 1450 dev_err(qidev, "unable to map S/G table\n"); 1451 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 1452 iv_dma, ivsize, GIVENCRYPT, 0, 0); 1453 qi_cache_free(edesc); 1454 return ERR_PTR(-ENOMEM); 1455 } 1456 1457 fd_sgt = &edesc->drv_req.fd_sgt[0]; 1458 1459 if (mapped_src_nents > 1) 1460 dma_to_qm_sg_one_ext(&fd_sgt[1], edesc->qm_sg_dma, req->nbytes, 1461 0); 1462 else 1463 dma_to_qm_sg_one(&fd_sgt[1], sg_dma_address(req->src), 1464 req->nbytes, 0); 1465 1466 dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx * 1467 sizeof(*sg_table), ivsize + req->nbytes, 0); 1468 1469 return edesc; 1470 } 1471 1472 static inline int ablkcipher_crypt(struct ablkcipher_request *req, bool encrypt) 1473 { 1474 struct ablkcipher_edesc *edesc; 1475 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1476 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1477 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1478 int ret; 1479 1480 if (unlikely(caam_congested)) 1481 return -EAGAIN; 1482 1483 /* allocate extended descriptor */ 1484 edesc = ablkcipher_edesc_alloc(req, encrypt); 1485 if (IS_ERR(edesc)) 1486 return PTR_ERR(edesc); 1487 1488 /* 1489 * The crypto API expects us to set the IV (req->info) to the last 1490 * ciphertext block. 1491 */ 1492 if (!encrypt) 1493 scatterwalk_map_and_copy(req->info, req->src, req->nbytes - 1494 ivsize, ivsize, 0); 1495 1496 ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req); 1497 if (!ret) { 1498 ret = -EINPROGRESS; 1499 } else { 1500 ablkcipher_unmap(ctx->qidev, edesc, req); 1501 qi_cache_free(edesc); 1502 } 1503 1504 return ret; 1505 } 1506 1507 static int ablkcipher_encrypt(struct ablkcipher_request *req) 1508 { 1509 return ablkcipher_crypt(req, true); 1510 } 1511 1512 static int ablkcipher_decrypt(struct ablkcipher_request *req) 1513 { 1514 return ablkcipher_crypt(req, false); 1515 } 1516 1517 static int ablkcipher_givencrypt(struct skcipher_givcrypt_request *creq) 1518 { 1519 struct ablkcipher_request *req = &creq->creq; 1520 struct ablkcipher_edesc *edesc; 1521 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1522 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1523 int ret; 1524 1525 if (unlikely(caam_congested)) 1526 return -EAGAIN; 1527 1528 /* allocate extended descriptor */ 1529 edesc = ablkcipher_giv_edesc_alloc(creq); 1530 if (IS_ERR(edesc)) 1531 return PTR_ERR(edesc); 1532 1533 ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req); 1534 if (!ret) { 1535 ret = -EINPROGRESS; 1536 } else { 1537 ablkcipher_unmap(ctx->qidev, edesc, req); 1538 qi_cache_free(edesc); 1539 } 1540 1541 return ret; 1542 } 1543 1544 #define template_ablkcipher template_u.ablkcipher 1545 struct caam_alg_template { 1546 char name[CRYPTO_MAX_ALG_NAME]; 1547 char driver_name[CRYPTO_MAX_ALG_NAME]; 1548 unsigned int blocksize; 1549 u32 type; 1550 union { 1551 struct ablkcipher_alg ablkcipher; 1552 } template_u; 1553 u32 class1_alg_type; 1554 u32 class2_alg_type; 1555 }; 1556 1557 static struct caam_alg_template driver_algs[] = { 1558 /* ablkcipher descriptor */ 1559 { 1560 .name = "cbc(aes)", 1561 .driver_name = "cbc-aes-caam-qi", 1562 .blocksize = AES_BLOCK_SIZE, 1563 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 1564 .template_ablkcipher = { 1565 .setkey = ablkcipher_setkey, 1566 .encrypt = ablkcipher_encrypt, 1567 .decrypt = ablkcipher_decrypt, 1568 .givencrypt = ablkcipher_givencrypt, 1569 .geniv = "<built-in>", 1570 .min_keysize = AES_MIN_KEY_SIZE, 1571 .max_keysize = AES_MAX_KEY_SIZE, 1572 .ivsize = AES_BLOCK_SIZE, 1573 }, 1574 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1575 }, 1576 { 1577 .name = "cbc(des3_ede)", 1578 .driver_name = "cbc-3des-caam-qi", 1579 .blocksize = DES3_EDE_BLOCK_SIZE, 1580 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 1581 .template_ablkcipher = { 1582 .setkey = ablkcipher_setkey, 1583 .encrypt = ablkcipher_encrypt, 1584 .decrypt = ablkcipher_decrypt, 1585 .givencrypt = ablkcipher_givencrypt, 1586 .geniv = "<built-in>", 1587 .min_keysize = DES3_EDE_KEY_SIZE, 1588 .max_keysize = DES3_EDE_KEY_SIZE, 1589 .ivsize = DES3_EDE_BLOCK_SIZE, 1590 }, 1591 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1592 }, 1593 { 1594 .name = "cbc(des)", 1595 .driver_name = "cbc-des-caam-qi", 1596 .blocksize = DES_BLOCK_SIZE, 1597 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 1598 .template_ablkcipher = { 1599 .setkey = ablkcipher_setkey, 1600 .encrypt = ablkcipher_encrypt, 1601 .decrypt = ablkcipher_decrypt, 1602 .givencrypt = ablkcipher_givencrypt, 1603 .geniv = "<built-in>", 1604 .min_keysize = DES_KEY_SIZE, 1605 .max_keysize = DES_KEY_SIZE, 1606 .ivsize = DES_BLOCK_SIZE, 1607 }, 1608 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1609 }, 1610 { 1611 .name = "ctr(aes)", 1612 .driver_name = "ctr-aes-caam-qi", 1613 .blocksize = 1, 1614 .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 1615 .template_ablkcipher = { 1616 .setkey = ablkcipher_setkey, 1617 .encrypt = ablkcipher_encrypt, 1618 .decrypt = ablkcipher_decrypt, 1619 .geniv = "chainiv", 1620 .min_keysize = AES_MIN_KEY_SIZE, 1621 .max_keysize = AES_MAX_KEY_SIZE, 1622 .ivsize = AES_BLOCK_SIZE, 1623 }, 1624 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, 1625 }, 1626 { 1627 .name = "rfc3686(ctr(aes))", 1628 .driver_name = "rfc3686-ctr-aes-caam-qi", 1629 .blocksize = 1, 1630 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 1631 .template_ablkcipher = { 1632 .setkey = ablkcipher_setkey, 1633 .encrypt = ablkcipher_encrypt, 1634 .decrypt = ablkcipher_decrypt, 1635 .givencrypt = ablkcipher_givencrypt, 1636 .geniv = "<built-in>", 1637 .min_keysize = AES_MIN_KEY_SIZE + 1638 CTR_RFC3686_NONCE_SIZE, 1639 .max_keysize = AES_MAX_KEY_SIZE + 1640 CTR_RFC3686_NONCE_SIZE, 1641 .ivsize = CTR_RFC3686_IV_SIZE, 1642 }, 1643 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, 1644 }, 1645 { 1646 .name = "xts(aes)", 1647 .driver_name = "xts-aes-caam-qi", 1648 .blocksize = AES_BLOCK_SIZE, 1649 .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 1650 .template_ablkcipher = { 1651 .setkey = xts_ablkcipher_setkey, 1652 .encrypt = ablkcipher_encrypt, 1653 .decrypt = ablkcipher_decrypt, 1654 .geniv = "eseqiv", 1655 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1656 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1657 .ivsize = AES_BLOCK_SIZE, 1658 }, 1659 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, 1660 }, 1661 }; 1662 1663 static struct caam_aead_alg driver_aeads[] = { 1664 { 1665 .aead = { 1666 .base = { 1667 .cra_name = "rfc4106(gcm(aes))", 1668 .cra_driver_name = "rfc4106-gcm-aes-caam-qi", 1669 .cra_blocksize = 1, 1670 }, 1671 .setkey = rfc4106_setkey, 1672 .setauthsize = rfc4106_setauthsize, 1673 .encrypt = ipsec_gcm_encrypt, 1674 .decrypt = ipsec_gcm_decrypt, 1675 .ivsize = 8, 1676 .maxauthsize = AES_BLOCK_SIZE, 1677 }, 1678 .caam = { 1679 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1680 }, 1681 }, 1682 { 1683 .aead = { 1684 .base = { 1685 .cra_name = "rfc4543(gcm(aes))", 1686 .cra_driver_name = "rfc4543-gcm-aes-caam-qi", 1687 .cra_blocksize = 1, 1688 }, 1689 .setkey = rfc4543_setkey, 1690 .setauthsize = rfc4543_setauthsize, 1691 .encrypt = ipsec_gcm_encrypt, 1692 .decrypt = ipsec_gcm_decrypt, 1693 .ivsize = 8, 1694 .maxauthsize = AES_BLOCK_SIZE, 1695 }, 1696 .caam = { 1697 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1698 }, 1699 }, 1700 /* Galois Counter Mode */ 1701 { 1702 .aead = { 1703 .base = { 1704 .cra_name = "gcm(aes)", 1705 .cra_driver_name = "gcm-aes-caam-qi", 1706 .cra_blocksize = 1, 1707 }, 1708 .setkey = gcm_setkey, 1709 .setauthsize = gcm_setauthsize, 1710 .encrypt = aead_encrypt, 1711 .decrypt = aead_decrypt, 1712 .ivsize = 12, 1713 .maxauthsize = AES_BLOCK_SIZE, 1714 }, 1715 .caam = { 1716 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1717 } 1718 }, 1719 /* single-pass ipsec_esp descriptor */ 1720 { 1721 .aead = { 1722 .base = { 1723 .cra_name = "authenc(hmac(md5),cbc(aes))", 1724 .cra_driver_name = "authenc-hmac-md5-" 1725 "cbc-aes-caam-qi", 1726 .cra_blocksize = AES_BLOCK_SIZE, 1727 }, 1728 .setkey = aead_setkey, 1729 .setauthsize = aead_setauthsize, 1730 .encrypt = aead_encrypt, 1731 .decrypt = aead_decrypt, 1732 .ivsize = AES_BLOCK_SIZE, 1733 .maxauthsize = MD5_DIGEST_SIZE, 1734 }, 1735 .caam = { 1736 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1737 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1738 OP_ALG_AAI_HMAC_PRECOMP, 1739 } 1740 }, 1741 { 1742 .aead = { 1743 .base = { 1744 .cra_name = "echainiv(authenc(hmac(md5)," 1745 "cbc(aes)))", 1746 .cra_driver_name = "echainiv-authenc-hmac-md5-" 1747 "cbc-aes-caam-qi", 1748 .cra_blocksize = AES_BLOCK_SIZE, 1749 }, 1750 .setkey = aead_setkey, 1751 .setauthsize = aead_setauthsize, 1752 .encrypt = aead_encrypt, 1753 .decrypt = aead_decrypt, 1754 .ivsize = AES_BLOCK_SIZE, 1755 .maxauthsize = MD5_DIGEST_SIZE, 1756 }, 1757 .caam = { 1758 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1759 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1760 OP_ALG_AAI_HMAC_PRECOMP, 1761 .geniv = true, 1762 } 1763 }, 1764 { 1765 .aead = { 1766 .base = { 1767 .cra_name = "authenc(hmac(sha1),cbc(aes))", 1768 .cra_driver_name = "authenc-hmac-sha1-" 1769 "cbc-aes-caam-qi", 1770 .cra_blocksize = AES_BLOCK_SIZE, 1771 }, 1772 .setkey = aead_setkey, 1773 .setauthsize = aead_setauthsize, 1774 .encrypt = aead_encrypt, 1775 .decrypt = aead_decrypt, 1776 .ivsize = AES_BLOCK_SIZE, 1777 .maxauthsize = SHA1_DIGEST_SIZE, 1778 }, 1779 .caam = { 1780 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1781 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1782 OP_ALG_AAI_HMAC_PRECOMP, 1783 } 1784 }, 1785 { 1786 .aead = { 1787 .base = { 1788 .cra_name = "echainiv(authenc(hmac(sha1)," 1789 "cbc(aes)))", 1790 .cra_driver_name = "echainiv-authenc-" 1791 "hmac-sha1-cbc-aes-caam-qi", 1792 .cra_blocksize = AES_BLOCK_SIZE, 1793 }, 1794 .setkey = aead_setkey, 1795 .setauthsize = aead_setauthsize, 1796 .encrypt = aead_encrypt, 1797 .decrypt = aead_decrypt, 1798 .ivsize = AES_BLOCK_SIZE, 1799 .maxauthsize = SHA1_DIGEST_SIZE, 1800 }, 1801 .caam = { 1802 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1803 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1804 OP_ALG_AAI_HMAC_PRECOMP, 1805 .geniv = true, 1806 }, 1807 }, 1808 { 1809 .aead = { 1810 .base = { 1811 .cra_name = "authenc(hmac(sha224),cbc(aes))", 1812 .cra_driver_name = "authenc-hmac-sha224-" 1813 "cbc-aes-caam-qi", 1814 .cra_blocksize = AES_BLOCK_SIZE, 1815 }, 1816 .setkey = aead_setkey, 1817 .setauthsize = aead_setauthsize, 1818 .encrypt = aead_encrypt, 1819 .decrypt = aead_decrypt, 1820 .ivsize = AES_BLOCK_SIZE, 1821 .maxauthsize = SHA224_DIGEST_SIZE, 1822 }, 1823 .caam = { 1824 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1825 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1826 OP_ALG_AAI_HMAC_PRECOMP, 1827 } 1828 }, 1829 { 1830 .aead = { 1831 .base = { 1832 .cra_name = "echainiv(authenc(hmac(sha224)," 1833 "cbc(aes)))", 1834 .cra_driver_name = "echainiv-authenc-" 1835 "hmac-sha224-cbc-aes-caam-qi", 1836 .cra_blocksize = AES_BLOCK_SIZE, 1837 }, 1838 .setkey = aead_setkey, 1839 .setauthsize = aead_setauthsize, 1840 .encrypt = aead_encrypt, 1841 .decrypt = aead_decrypt, 1842 .ivsize = AES_BLOCK_SIZE, 1843 .maxauthsize = SHA224_DIGEST_SIZE, 1844 }, 1845 .caam = { 1846 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1847 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1848 OP_ALG_AAI_HMAC_PRECOMP, 1849 .geniv = true, 1850 } 1851 }, 1852 { 1853 .aead = { 1854 .base = { 1855 .cra_name = "authenc(hmac(sha256),cbc(aes))", 1856 .cra_driver_name = "authenc-hmac-sha256-" 1857 "cbc-aes-caam-qi", 1858 .cra_blocksize = AES_BLOCK_SIZE, 1859 }, 1860 .setkey = aead_setkey, 1861 .setauthsize = aead_setauthsize, 1862 .encrypt = aead_encrypt, 1863 .decrypt = aead_decrypt, 1864 .ivsize = AES_BLOCK_SIZE, 1865 .maxauthsize = SHA256_DIGEST_SIZE, 1866 }, 1867 .caam = { 1868 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1869 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1870 OP_ALG_AAI_HMAC_PRECOMP, 1871 } 1872 }, 1873 { 1874 .aead = { 1875 .base = { 1876 .cra_name = "echainiv(authenc(hmac(sha256)," 1877 "cbc(aes)))", 1878 .cra_driver_name = "echainiv-authenc-" 1879 "hmac-sha256-cbc-aes-" 1880 "caam-qi", 1881 .cra_blocksize = AES_BLOCK_SIZE, 1882 }, 1883 .setkey = aead_setkey, 1884 .setauthsize = aead_setauthsize, 1885 .encrypt = aead_encrypt, 1886 .decrypt = aead_decrypt, 1887 .ivsize = AES_BLOCK_SIZE, 1888 .maxauthsize = SHA256_DIGEST_SIZE, 1889 }, 1890 .caam = { 1891 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1892 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1893 OP_ALG_AAI_HMAC_PRECOMP, 1894 .geniv = true, 1895 } 1896 }, 1897 { 1898 .aead = { 1899 .base = { 1900 .cra_name = "authenc(hmac(sha384),cbc(aes))", 1901 .cra_driver_name = "authenc-hmac-sha384-" 1902 "cbc-aes-caam-qi", 1903 .cra_blocksize = AES_BLOCK_SIZE, 1904 }, 1905 .setkey = aead_setkey, 1906 .setauthsize = aead_setauthsize, 1907 .encrypt = aead_encrypt, 1908 .decrypt = aead_decrypt, 1909 .ivsize = AES_BLOCK_SIZE, 1910 .maxauthsize = SHA384_DIGEST_SIZE, 1911 }, 1912 .caam = { 1913 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1914 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 1915 OP_ALG_AAI_HMAC_PRECOMP, 1916 } 1917 }, 1918 { 1919 .aead = { 1920 .base = { 1921 .cra_name = "echainiv(authenc(hmac(sha384)," 1922 "cbc(aes)))", 1923 .cra_driver_name = "echainiv-authenc-" 1924 "hmac-sha384-cbc-aes-" 1925 "caam-qi", 1926 .cra_blocksize = AES_BLOCK_SIZE, 1927 }, 1928 .setkey = aead_setkey, 1929 .setauthsize = aead_setauthsize, 1930 .encrypt = aead_encrypt, 1931 .decrypt = aead_decrypt, 1932 .ivsize = AES_BLOCK_SIZE, 1933 .maxauthsize = SHA384_DIGEST_SIZE, 1934 }, 1935 .caam = { 1936 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1937 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 1938 OP_ALG_AAI_HMAC_PRECOMP, 1939 .geniv = true, 1940 } 1941 }, 1942 { 1943 .aead = { 1944 .base = { 1945 .cra_name = "authenc(hmac(sha512),cbc(aes))", 1946 .cra_driver_name = "authenc-hmac-sha512-" 1947 "cbc-aes-caam-qi", 1948 .cra_blocksize = AES_BLOCK_SIZE, 1949 }, 1950 .setkey = aead_setkey, 1951 .setauthsize = aead_setauthsize, 1952 .encrypt = aead_encrypt, 1953 .decrypt = aead_decrypt, 1954 .ivsize = AES_BLOCK_SIZE, 1955 .maxauthsize = SHA512_DIGEST_SIZE, 1956 }, 1957 .caam = { 1958 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1959 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 1960 OP_ALG_AAI_HMAC_PRECOMP, 1961 } 1962 }, 1963 { 1964 .aead = { 1965 .base = { 1966 .cra_name = "echainiv(authenc(hmac(sha512)," 1967 "cbc(aes)))", 1968 .cra_driver_name = "echainiv-authenc-" 1969 "hmac-sha512-cbc-aes-" 1970 "caam-qi", 1971 .cra_blocksize = AES_BLOCK_SIZE, 1972 }, 1973 .setkey = aead_setkey, 1974 .setauthsize = aead_setauthsize, 1975 .encrypt = aead_encrypt, 1976 .decrypt = aead_decrypt, 1977 .ivsize = AES_BLOCK_SIZE, 1978 .maxauthsize = SHA512_DIGEST_SIZE, 1979 }, 1980 .caam = { 1981 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1982 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 1983 OP_ALG_AAI_HMAC_PRECOMP, 1984 .geniv = true, 1985 } 1986 }, 1987 { 1988 .aead = { 1989 .base = { 1990 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 1991 .cra_driver_name = "authenc-hmac-md5-" 1992 "cbc-des3_ede-caam-qi", 1993 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1994 }, 1995 .setkey = aead_setkey, 1996 .setauthsize = aead_setauthsize, 1997 .encrypt = aead_encrypt, 1998 .decrypt = aead_decrypt, 1999 .ivsize = DES3_EDE_BLOCK_SIZE, 2000 .maxauthsize = MD5_DIGEST_SIZE, 2001 }, 2002 .caam = { 2003 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2004 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2005 OP_ALG_AAI_HMAC_PRECOMP, 2006 } 2007 }, 2008 { 2009 .aead = { 2010 .base = { 2011 .cra_name = "echainiv(authenc(hmac(md5)," 2012 "cbc(des3_ede)))", 2013 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2014 "cbc-des3_ede-caam-qi", 2015 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2016 }, 2017 .setkey = aead_setkey, 2018 .setauthsize = aead_setauthsize, 2019 .encrypt = aead_encrypt, 2020 .decrypt = aead_decrypt, 2021 .ivsize = DES3_EDE_BLOCK_SIZE, 2022 .maxauthsize = MD5_DIGEST_SIZE, 2023 }, 2024 .caam = { 2025 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2026 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2027 OP_ALG_AAI_HMAC_PRECOMP, 2028 .geniv = true, 2029 } 2030 }, 2031 { 2032 .aead = { 2033 .base = { 2034 .cra_name = "authenc(hmac(sha1)," 2035 "cbc(des3_ede))", 2036 .cra_driver_name = "authenc-hmac-sha1-" 2037 "cbc-des3_ede-caam-qi", 2038 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2039 }, 2040 .setkey = aead_setkey, 2041 .setauthsize = aead_setauthsize, 2042 .encrypt = aead_encrypt, 2043 .decrypt = aead_decrypt, 2044 .ivsize = DES3_EDE_BLOCK_SIZE, 2045 .maxauthsize = SHA1_DIGEST_SIZE, 2046 }, 2047 .caam = { 2048 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2049 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2050 OP_ALG_AAI_HMAC_PRECOMP, 2051 }, 2052 }, 2053 { 2054 .aead = { 2055 .base = { 2056 .cra_name = "echainiv(authenc(hmac(sha1)," 2057 "cbc(des3_ede)))", 2058 .cra_driver_name = "echainiv-authenc-" 2059 "hmac-sha1-" 2060 "cbc-des3_ede-caam-qi", 2061 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2062 }, 2063 .setkey = aead_setkey, 2064 .setauthsize = aead_setauthsize, 2065 .encrypt = aead_encrypt, 2066 .decrypt = aead_decrypt, 2067 .ivsize = DES3_EDE_BLOCK_SIZE, 2068 .maxauthsize = SHA1_DIGEST_SIZE, 2069 }, 2070 .caam = { 2071 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2072 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2073 OP_ALG_AAI_HMAC_PRECOMP, 2074 .geniv = true, 2075 } 2076 }, 2077 { 2078 .aead = { 2079 .base = { 2080 .cra_name = "authenc(hmac(sha224)," 2081 "cbc(des3_ede))", 2082 .cra_driver_name = "authenc-hmac-sha224-" 2083 "cbc-des3_ede-caam-qi", 2084 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2085 }, 2086 .setkey = aead_setkey, 2087 .setauthsize = aead_setauthsize, 2088 .encrypt = aead_encrypt, 2089 .decrypt = aead_decrypt, 2090 .ivsize = DES3_EDE_BLOCK_SIZE, 2091 .maxauthsize = SHA224_DIGEST_SIZE, 2092 }, 2093 .caam = { 2094 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2095 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2096 OP_ALG_AAI_HMAC_PRECOMP, 2097 }, 2098 }, 2099 { 2100 .aead = { 2101 .base = { 2102 .cra_name = "echainiv(authenc(hmac(sha224)," 2103 "cbc(des3_ede)))", 2104 .cra_driver_name = "echainiv-authenc-" 2105 "hmac-sha224-" 2106 "cbc-des3_ede-caam-qi", 2107 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2108 }, 2109 .setkey = aead_setkey, 2110 .setauthsize = aead_setauthsize, 2111 .encrypt = aead_encrypt, 2112 .decrypt = aead_decrypt, 2113 .ivsize = DES3_EDE_BLOCK_SIZE, 2114 .maxauthsize = SHA224_DIGEST_SIZE, 2115 }, 2116 .caam = { 2117 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2118 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2119 OP_ALG_AAI_HMAC_PRECOMP, 2120 .geniv = true, 2121 } 2122 }, 2123 { 2124 .aead = { 2125 .base = { 2126 .cra_name = "authenc(hmac(sha256)," 2127 "cbc(des3_ede))", 2128 .cra_driver_name = "authenc-hmac-sha256-" 2129 "cbc-des3_ede-caam-qi", 2130 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2131 }, 2132 .setkey = aead_setkey, 2133 .setauthsize = aead_setauthsize, 2134 .encrypt = aead_encrypt, 2135 .decrypt = aead_decrypt, 2136 .ivsize = DES3_EDE_BLOCK_SIZE, 2137 .maxauthsize = SHA256_DIGEST_SIZE, 2138 }, 2139 .caam = { 2140 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2141 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2142 OP_ALG_AAI_HMAC_PRECOMP, 2143 }, 2144 }, 2145 { 2146 .aead = { 2147 .base = { 2148 .cra_name = "echainiv(authenc(hmac(sha256)," 2149 "cbc(des3_ede)))", 2150 .cra_driver_name = "echainiv-authenc-" 2151 "hmac-sha256-" 2152 "cbc-des3_ede-caam-qi", 2153 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2154 }, 2155 .setkey = aead_setkey, 2156 .setauthsize = aead_setauthsize, 2157 .encrypt = aead_encrypt, 2158 .decrypt = aead_decrypt, 2159 .ivsize = DES3_EDE_BLOCK_SIZE, 2160 .maxauthsize = SHA256_DIGEST_SIZE, 2161 }, 2162 .caam = { 2163 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2164 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2165 OP_ALG_AAI_HMAC_PRECOMP, 2166 .geniv = true, 2167 } 2168 }, 2169 { 2170 .aead = { 2171 .base = { 2172 .cra_name = "authenc(hmac(sha384)," 2173 "cbc(des3_ede))", 2174 .cra_driver_name = "authenc-hmac-sha384-" 2175 "cbc-des3_ede-caam-qi", 2176 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2177 }, 2178 .setkey = aead_setkey, 2179 .setauthsize = aead_setauthsize, 2180 .encrypt = aead_encrypt, 2181 .decrypt = aead_decrypt, 2182 .ivsize = DES3_EDE_BLOCK_SIZE, 2183 .maxauthsize = SHA384_DIGEST_SIZE, 2184 }, 2185 .caam = { 2186 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2187 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2188 OP_ALG_AAI_HMAC_PRECOMP, 2189 }, 2190 }, 2191 { 2192 .aead = { 2193 .base = { 2194 .cra_name = "echainiv(authenc(hmac(sha384)," 2195 "cbc(des3_ede)))", 2196 .cra_driver_name = "echainiv-authenc-" 2197 "hmac-sha384-" 2198 "cbc-des3_ede-caam-qi", 2199 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2200 }, 2201 .setkey = aead_setkey, 2202 .setauthsize = aead_setauthsize, 2203 .encrypt = aead_encrypt, 2204 .decrypt = aead_decrypt, 2205 .ivsize = DES3_EDE_BLOCK_SIZE, 2206 .maxauthsize = SHA384_DIGEST_SIZE, 2207 }, 2208 .caam = { 2209 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2210 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2211 OP_ALG_AAI_HMAC_PRECOMP, 2212 .geniv = true, 2213 } 2214 }, 2215 { 2216 .aead = { 2217 .base = { 2218 .cra_name = "authenc(hmac(sha512)," 2219 "cbc(des3_ede))", 2220 .cra_driver_name = "authenc-hmac-sha512-" 2221 "cbc-des3_ede-caam-qi", 2222 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2223 }, 2224 .setkey = aead_setkey, 2225 .setauthsize = aead_setauthsize, 2226 .encrypt = aead_encrypt, 2227 .decrypt = aead_decrypt, 2228 .ivsize = DES3_EDE_BLOCK_SIZE, 2229 .maxauthsize = SHA512_DIGEST_SIZE, 2230 }, 2231 .caam = { 2232 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2233 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2234 OP_ALG_AAI_HMAC_PRECOMP, 2235 }, 2236 }, 2237 { 2238 .aead = { 2239 .base = { 2240 .cra_name = "echainiv(authenc(hmac(sha512)," 2241 "cbc(des3_ede)))", 2242 .cra_driver_name = "echainiv-authenc-" 2243 "hmac-sha512-" 2244 "cbc-des3_ede-caam-qi", 2245 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2246 }, 2247 .setkey = aead_setkey, 2248 .setauthsize = aead_setauthsize, 2249 .encrypt = aead_encrypt, 2250 .decrypt = aead_decrypt, 2251 .ivsize = DES3_EDE_BLOCK_SIZE, 2252 .maxauthsize = SHA512_DIGEST_SIZE, 2253 }, 2254 .caam = { 2255 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2256 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2257 OP_ALG_AAI_HMAC_PRECOMP, 2258 .geniv = true, 2259 } 2260 }, 2261 { 2262 .aead = { 2263 .base = { 2264 .cra_name = "authenc(hmac(md5),cbc(des))", 2265 .cra_driver_name = "authenc-hmac-md5-" 2266 "cbc-des-caam-qi", 2267 .cra_blocksize = DES_BLOCK_SIZE, 2268 }, 2269 .setkey = aead_setkey, 2270 .setauthsize = aead_setauthsize, 2271 .encrypt = aead_encrypt, 2272 .decrypt = aead_decrypt, 2273 .ivsize = DES_BLOCK_SIZE, 2274 .maxauthsize = MD5_DIGEST_SIZE, 2275 }, 2276 .caam = { 2277 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2278 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2279 OP_ALG_AAI_HMAC_PRECOMP, 2280 }, 2281 }, 2282 { 2283 .aead = { 2284 .base = { 2285 .cra_name = "echainiv(authenc(hmac(md5)," 2286 "cbc(des)))", 2287 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2288 "cbc-des-caam-qi", 2289 .cra_blocksize = DES_BLOCK_SIZE, 2290 }, 2291 .setkey = aead_setkey, 2292 .setauthsize = aead_setauthsize, 2293 .encrypt = aead_encrypt, 2294 .decrypt = aead_decrypt, 2295 .ivsize = DES_BLOCK_SIZE, 2296 .maxauthsize = MD5_DIGEST_SIZE, 2297 }, 2298 .caam = { 2299 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2300 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2301 OP_ALG_AAI_HMAC_PRECOMP, 2302 .geniv = true, 2303 } 2304 }, 2305 { 2306 .aead = { 2307 .base = { 2308 .cra_name = "authenc(hmac(sha1),cbc(des))", 2309 .cra_driver_name = "authenc-hmac-sha1-" 2310 "cbc-des-caam-qi", 2311 .cra_blocksize = DES_BLOCK_SIZE, 2312 }, 2313 .setkey = aead_setkey, 2314 .setauthsize = aead_setauthsize, 2315 .encrypt = aead_encrypt, 2316 .decrypt = aead_decrypt, 2317 .ivsize = DES_BLOCK_SIZE, 2318 .maxauthsize = SHA1_DIGEST_SIZE, 2319 }, 2320 .caam = { 2321 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2322 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2323 OP_ALG_AAI_HMAC_PRECOMP, 2324 }, 2325 }, 2326 { 2327 .aead = { 2328 .base = { 2329 .cra_name = "echainiv(authenc(hmac(sha1)," 2330 "cbc(des)))", 2331 .cra_driver_name = "echainiv-authenc-" 2332 "hmac-sha1-cbc-des-caam-qi", 2333 .cra_blocksize = DES_BLOCK_SIZE, 2334 }, 2335 .setkey = aead_setkey, 2336 .setauthsize = aead_setauthsize, 2337 .encrypt = aead_encrypt, 2338 .decrypt = aead_decrypt, 2339 .ivsize = DES_BLOCK_SIZE, 2340 .maxauthsize = SHA1_DIGEST_SIZE, 2341 }, 2342 .caam = { 2343 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2344 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2345 OP_ALG_AAI_HMAC_PRECOMP, 2346 .geniv = true, 2347 } 2348 }, 2349 { 2350 .aead = { 2351 .base = { 2352 .cra_name = "authenc(hmac(sha224),cbc(des))", 2353 .cra_driver_name = "authenc-hmac-sha224-" 2354 "cbc-des-caam-qi", 2355 .cra_blocksize = DES_BLOCK_SIZE, 2356 }, 2357 .setkey = aead_setkey, 2358 .setauthsize = aead_setauthsize, 2359 .encrypt = aead_encrypt, 2360 .decrypt = aead_decrypt, 2361 .ivsize = DES_BLOCK_SIZE, 2362 .maxauthsize = SHA224_DIGEST_SIZE, 2363 }, 2364 .caam = { 2365 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2366 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2367 OP_ALG_AAI_HMAC_PRECOMP, 2368 }, 2369 }, 2370 { 2371 .aead = { 2372 .base = { 2373 .cra_name = "echainiv(authenc(hmac(sha224)," 2374 "cbc(des)))", 2375 .cra_driver_name = "echainiv-authenc-" 2376 "hmac-sha224-cbc-des-" 2377 "caam-qi", 2378 .cra_blocksize = DES_BLOCK_SIZE, 2379 }, 2380 .setkey = aead_setkey, 2381 .setauthsize = aead_setauthsize, 2382 .encrypt = aead_encrypt, 2383 .decrypt = aead_decrypt, 2384 .ivsize = DES_BLOCK_SIZE, 2385 .maxauthsize = SHA224_DIGEST_SIZE, 2386 }, 2387 .caam = { 2388 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2389 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2390 OP_ALG_AAI_HMAC_PRECOMP, 2391 .geniv = true, 2392 } 2393 }, 2394 { 2395 .aead = { 2396 .base = { 2397 .cra_name = "authenc(hmac(sha256),cbc(des))", 2398 .cra_driver_name = "authenc-hmac-sha256-" 2399 "cbc-des-caam-qi", 2400 .cra_blocksize = DES_BLOCK_SIZE, 2401 }, 2402 .setkey = aead_setkey, 2403 .setauthsize = aead_setauthsize, 2404 .encrypt = aead_encrypt, 2405 .decrypt = aead_decrypt, 2406 .ivsize = DES_BLOCK_SIZE, 2407 .maxauthsize = SHA256_DIGEST_SIZE, 2408 }, 2409 .caam = { 2410 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2411 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2412 OP_ALG_AAI_HMAC_PRECOMP, 2413 }, 2414 }, 2415 { 2416 .aead = { 2417 .base = { 2418 .cra_name = "echainiv(authenc(hmac(sha256)," 2419 "cbc(des)))", 2420 .cra_driver_name = "echainiv-authenc-" 2421 "hmac-sha256-cbc-des-" 2422 "caam-qi", 2423 .cra_blocksize = DES_BLOCK_SIZE, 2424 }, 2425 .setkey = aead_setkey, 2426 .setauthsize = aead_setauthsize, 2427 .encrypt = aead_encrypt, 2428 .decrypt = aead_decrypt, 2429 .ivsize = DES_BLOCK_SIZE, 2430 .maxauthsize = SHA256_DIGEST_SIZE, 2431 }, 2432 .caam = { 2433 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2434 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2435 OP_ALG_AAI_HMAC_PRECOMP, 2436 .geniv = true, 2437 }, 2438 }, 2439 { 2440 .aead = { 2441 .base = { 2442 .cra_name = "authenc(hmac(sha384),cbc(des))", 2443 .cra_driver_name = "authenc-hmac-sha384-" 2444 "cbc-des-caam-qi", 2445 .cra_blocksize = DES_BLOCK_SIZE, 2446 }, 2447 .setkey = aead_setkey, 2448 .setauthsize = aead_setauthsize, 2449 .encrypt = aead_encrypt, 2450 .decrypt = aead_decrypt, 2451 .ivsize = DES_BLOCK_SIZE, 2452 .maxauthsize = SHA384_DIGEST_SIZE, 2453 }, 2454 .caam = { 2455 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2456 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2457 OP_ALG_AAI_HMAC_PRECOMP, 2458 }, 2459 }, 2460 { 2461 .aead = { 2462 .base = { 2463 .cra_name = "echainiv(authenc(hmac(sha384)," 2464 "cbc(des)))", 2465 .cra_driver_name = "echainiv-authenc-" 2466 "hmac-sha384-cbc-des-" 2467 "caam-qi", 2468 .cra_blocksize = DES_BLOCK_SIZE, 2469 }, 2470 .setkey = aead_setkey, 2471 .setauthsize = aead_setauthsize, 2472 .encrypt = aead_encrypt, 2473 .decrypt = aead_decrypt, 2474 .ivsize = DES_BLOCK_SIZE, 2475 .maxauthsize = SHA384_DIGEST_SIZE, 2476 }, 2477 .caam = { 2478 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2479 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2480 OP_ALG_AAI_HMAC_PRECOMP, 2481 .geniv = true, 2482 } 2483 }, 2484 { 2485 .aead = { 2486 .base = { 2487 .cra_name = "authenc(hmac(sha512),cbc(des))", 2488 .cra_driver_name = "authenc-hmac-sha512-" 2489 "cbc-des-caam-qi", 2490 .cra_blocksize = DES_BLOCK_SIZE, 2491 }, 2492 .setkey = aead_setkey, 2493 .setauthsize = aead_setauthsize, 2494 .encrypt = aead_encrypt, 2495 .decrypt = aead_decrypt, 2496 .ivsize = DES_BLOCK_SIZE, 2497 .maxauthsize = SHA512_DIGEST_SIZE, 2498 }, 2499 .caam = { 2500 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2501 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2502 OP_ALG_AAI_HMAC_PRECOMP, 2503 } 2504 }, 2505 { 2506 .aead = { 2507 .base = { 2508 .cra_name = "echainiv(authenc(hmac(sha512)," 2509 "cbc(des)))", 2510 .cra_driver_name = "echainiv-authenc-" 2511 "hmac-sha512-cbc-des-" 2512 "caam-qi", 2513 .cra_blocksize = DES_BLOCK_SIZE, 2514 }, 2515 .setkey = aead_setkey, 2516 .setauthsize = aead_setauthsize, 2517 .encrypt = aead_encrypt, 2518 .decrypt = aead_decrypt, 2519 .ivsize = DES_BLOCK_SIZE, 2520 .maxauthsize = SHA512_DIGEST_SIZE, 2521 }, 2522 .caam = { 2523 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2524 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2525 OP_ALG_AAI_HMAC_PRECOMP, 2526 .geniv = true, 2527 } 2528 }, 2529 }; 2530 2531 struct caam_crypto_alg { 2532 struct list_head entry; 2533 struct crypto_alg crypto_alg; 2534 struct caam_alg_entry caam; 2535 }; 2536 2537 static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam, 2538 bool uses_dkp) 2539 { 2540 struct caam_drv_private *priv; 2541 2542 /* 2543 * distribute tfms across job rings to ensure in-order 2544 * crypto request processing per tfm 2545 */ 2546 ctx->jrdev = caam_jr_alloc(); 2547 if (IS_ERR(ctx->jrdev)) { 2548 pr_err("Job Ring Device allocation for transform failed\n"); 2549 return PTR_ERR(ctx->jrdev); 2550 } 2551 2552 priv = dev_get_drvdata(ctx->jrdev->parent); 2553 if (priv->era >= 6 && uses_dkp) 2554 ctx->dir = DMA_BIDIRECTIONAL; 2555 else 2556 ctx->dir = DMA_TO_DEVICE; 2557 2558 ctx->key_dma = dma_map_single(ctx->jrdev, ctx->key, sizeof(ctx->key), 2559 ctx->dir); 2560 if (dma_mapping_error(ctx->jrdev, ctx->key_dma)) { 2561 dev_err(ctx->jrdev, "unable to map key\n"); 2562 caam_jr_free(ctx->jrdev); 2563 return -ENOMEM; 2564 } 2565 2566 /* copy descriptor header template value */ 2567 ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; 2568 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; 2569 2570 ctx->qidev = priv->qidev; 2571 2572 spin_lock_init(&ctx->lock); 2573 ctx->drv_ctx[ENCRYPT] = NULL; 2574 ctx->drv_ctx[DECRYPT] = NULL; 2575 ctx->drv_ctx[GIVENCRYPT] = NULL; 2576 2577 return 0; 2578 } 2579 2580 static int caam_cra_init(struct crypto_tfm *tfm) 2581 { 2582 struct crypto_alg *alg = tfm->__crt_alg; 2583 struct caam_crypto_alg *caam_alg = container_of(alg, typeof(*caam_alg), 2584 crypto_alg); 2585 struct caam_ctx *ctx = crypto_tfm_ctx(tfm); 2586 2587 return caam_init_common(ctx, &caam_alg->caam, false); 2588 } 2589 2590 static int caam_aead_init(struct crypto_aead *tfm) 2591 { 2592 struct aead_alg *alg = crypto_aead_alg(tfm); 2593 struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg), 2594 aead); 2595 struct caam_ctx *ctx = crypto_aead_ctx(tfm); 2596 2597 return caam_init_common(ctx, &caam_alg->caam, 2598 alg->setkey == aead_setkey); 2599 } 2600 2601 static void caam_exit_common(struct caam_ctx *ctx) 2602 { 2603 caam_drv_ctx_rel(ctx->drv_ctx[ENCRYPT]); 2604 caam_drv_ctx_rel(ctx->drv_ctx[DECRYPT]); 2605 caam_drv_ctx_rel(ctx->drv_ctx[GIVENCRYPT]); 2606 2607 dma_unmap_single(ctx->jrdev, ctx->key_dma, sizeof(ctx->key), ctx->dir); 2608 2609 caam_jr_free(ctx->jrdev); 2610 } 2611 2612 static void caam_cra_exit(struct crypto_tfm *tfm) 2613 { 2614 caam_exit_common(crypto_tfm_ctx(tfm)); 2615 } 2616 2617 static void caam_aead_exit(struct crypto_aead *tfm) 2618 { 2619 caam_exit_common(crypto_aead_ctx(tfm)); 2620 } 2621 2622 static struct list_head alg_list; 2623 static void __exit caam_qi_algapi_exit(void) 2624 { 2625 struct caam_crypto_alg *t_alg, *n; 2626 int i; 2627 2628 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 2629 struct caam_aead_alg *t_alg = driver_aeads + i; 2630 2631 if (t_alg->registered) 2632 crypto_unregister_aead(&t_alg->aead); 2633 } 2634 2635 if (!alg_list.next) 2636 return; 2637 2638 list_for_each_entry_safe(t_alg, n, &alg_list, entry) { 2639 crypto_unregister_alg(&t_alg->crypto_alg); 2640 list_del(&t_alg->entry); 2641 kfree(t_alg); 2642 } 2643 } 2644 2645 static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template 2646 *template) 2647 { 2648 struct caam_crypto_alg *t_alg; 2649 struct crypto_alg *alg; 2650 2651 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); 2652 if (!t_alg) 2653 return ERR_PTR(-ENOMEM); 2654 2655 alg = &t_alg->crypto_alg; 2656 2657 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name); 2658 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 2659 template->driver_name); 2660 alg->cra_module = THIS_MODULE; 2661 alg->cra_init = caam_cra_init; 2662 alg->cra_exit = caam_cra_exit; 2663 alg->cra_priority = CAAM_CRA_PRIORITY; 2664 alg->cra_blocksize = template->blocksize; 2665 alg->cra_alignmask = 0; 2666 alg->cra_ctxsize = sizeof(struct caam_ctx); 2667 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY | 2668 template->type; 2669 switch (template->type) { 2670 case CRYPTO_ALG_TYPE_GIVCIPHER: 2671 alg->cra_type = &crypto_givcipher_type; 2672 alg->cra_ablkcipher = template->template_ablkcipher; 2673 break; 2674 case CRYPTO_ALG_TYPE_ABLKCIPHER: 2675 alg->cra_type = &crypto_ablkcipher_type; 2676 alg->cra_ablkcipher = template->template_ablkcipher; 2677 break; 2678 } 2679 2680 t_alg->caam.class1_alg_type = template->class1_alg_type; 2681 t_alg->caam.class2_alg_type = template->class2_alg_type; 2682 2683 return t_alg; 2684 } 2685 2686 static void caam_aead_alg_init(struct caam_aead_alg *t_alg) 2687 { 2688 struct aead_alg *alg = &t_alg->aead; 2689 2690 alg->base.cra_module = THIS_MODULE; 2691 alg->base.cra_priority = CAAM_CRA_PRIORITY; 2692 alg->base.cra_ctxsize = sizeof(struct caam_ctx); 2693 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 2694 2695 alg->init = caam_aead_init; 2696 alg->exit = caam_aead_exit; 2697 } 2698 2699 static int __init caam_qi_algapi_init(void) 2700 { 2701 struct device_node *dev_node; 2702 struct platform_device *pdev; 2703 struct device *ctrldev; 2704 struct caam_drv_private *priv; 2705 int i = 0, err = 0; 2706 u32 cha_vid, cha_inst, des_inst, aes_inst, md_inst; 2707 unsigned int md_limit = SHA512_DIGEST_SIZE; 2708 bool registered = false; 2709 2710 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); 2711 if (!dev_node) { 2712 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); 2713 if (!dev_node) 2714 return -ENODEV; 2715 } 2716 2717 pdev = of_find_device_by_node(dev_node); 2718 of_node_put(dev_node); 2719 if (!pdev) 2720 return -ENODEV; 2721 2722 ctrldev = &pdev->dev; 2723 priv = dev_get_drvdata(ctrldev); 2724 2725 /* 2726 * If priv is NULL, it's probably because the caam driver wasn't 2727 * properly initialized (e.g. RNG4 init failed). Thus, bail out here. 2728 */ 2729 if (!priv || !priv->qi_present) 2730 return -ENODEV; 2731 2732 if (caam_dpaa2) { 2733 dev_info(ctrldev, "caam/qi frontend driver not suitable for DPAA 2.x, aborting...\n"); 2734 return -ENODEV; 2735 } 2736 2737 INIT_LIST_HEAD(&alg_list); 2738 2739 /* 2740 * Register crypto algorithms the device supports. 2741 * First, detect presence and attributes of DES, AES, and MD blocks. 2742 */ 2743 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls); 2744 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls); 2745 des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >> CHA_ID_LS_DES_SHIFT; 2746 aes_inst = (cha_inst & CHA_ID_LS_AES_MASK) >> CHA_ID_LS_AES_SHIFT; 2747 md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; 2748 2749 /* If MD is present, limit digest size based on LP256 */ 2750 if (md_inst && ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256)) 2751 md_limit = SHA256_DIGEST_SIZE; 2752 2753 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 2754 struct caam_crypto_alg *t_alg; 2755 struct caam_alg_template *alg = driver_algs + i; 2756 u32 alg_sel = alg->class1_alg_type & OP_ALG_ALGSEL_MASK; 2757 2758 /* Skip DES algorithms if not supported by device */ 2759 if (!des_inst && 2760 ((alg_sel == OP_ALG_ALGSEL_3DES) || 2761 (alg_sel == OP_ALG_ALGSEL_DES))) 2762 continue; 2763 2764 /* Skip AES algorithms if not supported by device */ 2765 if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES)) 2766 continue; 2767 2768 t_alg = caam_alg_alloc(alg); 2769 if (IS_ERR(t_alg)) { 2770 err = PTR_ERR(t_alg); 2771 dev_warn(priv->qidev, "%s alg allocation failed\n", 2772 alg->driver_name); 2773 continue; 2774 } 2775 2776 err = crypto_register_alg(&t_alg->crypto_alg); 2777 if (err) { 2778 dev_warn(priv->qidev, "%s alg registration failed\n", 2779 t_alg->crypto_alg.cra_driver_name); 2780 kfree(t_alg); 2781 continue; 2782 } 2783 2784 list_add_tail(&t_alg->entry, &alg_list); 2785 registered = true; 2786 } 2787 2788 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 2789 struct caam_aead_alg *t_alg = driver_aeads + i; 2790 u32 c1_alg_sel = t_alg->caam.class1_alg_type & 2791 OP_ALG_ALGSEL_MASK; 2792 u32 c2_alg_sel = t_alg->caam.class2_alg_type & 2793 OP_ALG_ALGSEL_MASK; 2794 u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; 2795 2796 /* Skip DES algorithms if not supported by device */ 2797 if (!des_inst && 2798 ((c1_alg_sel == OP_ALG_ALGSEL_3DES) || 2799 (c1_alg_sel == OP_ALG_ALGSEL_DES))) 2800 continue; 2801 2802 /* Skip AES algorithms if not supported by device */ 2803 if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES)) 2804 continue; 2805 2806 /* 2807 * Check support for AES algorithms not available 2808 * on LP devices. 2809 */ 2810 if (((cha_vid & CHA_ID_LS_AES_MASK) == CHA_ID_LS_AES_LP) && 2811 (alg_aai == OP_ALG_AAI_GCM)) 2812 continue; 2813 2814 /* 2815 * Skip algorithms requiring message digests 2816 * if MD or MD size is not supported by device. 2817 */ 2818 if (c2_alg_sel && 2819 (!md_inst || (t_alg->aead.maxauthsize > md_limit))) 2820 continue; 2821 2822 caam_aead_alg_init(t_alg); 2823 2824 err = crypto_register_aead(&t_alg->aead); 2825 if (err) { 2826 pr_warn("%s alg registration failed\n", 2827 t_alg->aead.base.cra_driver_name); 2828 continue; 2829 } 2830 2831 t_alg->registered = true; 2832 registered = true; 2833 } 2834 2835 if (registered) 2836 dev_info(priv->qidev, "algorithms registered in /proc/crypto\n"); 2837 2838 return err; 2839 } 2840 2841 module_init(caam_qi_algapi_init); 2842 module_exit(caam_qi_algapi_exit); 2843 2844 MODULE_LICENSE("GPL"); 2845 MODULE_DESCRIPTION("Support for crypto API using CAAM-QI backend"); 2846 MODULE_AUTHOR("Freescale Semiconductor"); 2847