1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2017 Marvell 4 * 5 * Antoine Tenart <antoine.tenart@free-electrons.com> 6 */ 7 8 #include <asm/unaligned.h> 9 #include <linux/device.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/dmapool.h> 12 #include <crypto/aead.h> 13 #include <crypto/aes.h> 14 #include <crypto/authenc.h> 15 #include <crypto/chacha.h> 16 #include <crypto/ctr.h> 17 #include <crypto/internal/des.h> 18 #include <crypto/gcm.h> 19 #include <crypto/ghash.h> 20 #include <crypto/poly1305.h> 21 #include <crypto/sha1.h> 22 #include <crypto/sha2.h> 23 #include <crypto/sm3.h> 24 #include <crypto/sm4.h> 25 #include <crypto/xts.h> 26 #include <crypto/skcipher.h> 27 #include <crypto/internal/aead.h> 28 #include <crypto/internal/skcipher.h> 29 30 #include "safexcel.h" 31 32 enum safexcel_cipher_direction { 33 SAFEXCEL_ENCRYPT, 34 SAFEXCEL_DECRYPT, 35 }; 36 37 enum safexcel_cipher_alg { 38 SAFEXCEL_DES, 39 SAFEXCEL_3DES, 40 SAFEXCEL_AES, 41 SAFEXCEL_CHACHA20, 42 SAFEXCEL_SM4, 43 }; 44 45 struct safexcel_cipher_ctx { 46 struct safexcel_context base; 47 struct safexcel_crypto_priv *priv; 48 49 u32 mode; 50 enum safexcel_cipher_alg alg; 51 u8 aead; /* !=0=AEAD, 2=IPSec ESP AEAD, 3=IPsec ESP GMAC */ 52 u8 xcm; /* 0=authenc, 1=GCM, 2 reserved for CCM */ 53 u8 aadskip; 54 u8 blocksz; 55 u32 ivmask; 56 u32 ctrinit; 57 58 __le32 key[16]; 59 u32 nonce; 60 unsigned int key_len, xts; 61 62 /* All the below is AEAD specific */ 63 u32 hash_alg; 64 u32 state_sz; 65 66 struct crypto_aead *fback; 67 }; 68 69 struct safexcel_cipher_req { 70 enum safexcel_cipher_direction direction; 71 /* Number of result descriptors associated to the request */ 72 unsigned int rdescs; 73 bool needs_inv; 74 int nr_src, nr_dst; 75 }; 76 77 static int safexcel_skcipher_iv(struct safexcel_cipher_ctx *ctx, u8 *iv, 78 struct safexcel_command_desc *cdesc) 79 { 80 if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD) { 81 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; 82 /* 32 bit nonce */ 83 cdesc->control_data.token[0] = ctx->nonce; 84 /* 64 bit IV part */ 85 memcpy(&cdesc->control_data.token[1], iv, 8); 86 /* 32 bit counter, start at 0 or 1 (big endian!) */ 87 cdesc->control_data.token[3] = 88 (__force u32)cpu_to_be32(ctx->ctrinit); 89 return 4; 90 } 91 if (ctx->alg == SAFEXCEL_CHACHA20) { 92 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; 93 /* 96 bit nonce part */ 94 memcpy(&cdesc->control_data.token[0], &iv[4], 12); 95 /* 32 bit counter */ 96 cdesc->control_data.token[3] = *(u32 *)iv; 97 return 4; 98 } 99 100 cdesc->control_data.options |= ctx->ivmask; 101 memcpy(cdesc->control_data.token, iv, ctx->blocksz); 102 return ctx->blocksz / sizeof(u32); 103 } 104 105 static void safexcel_skcipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv, 106 struct safexcel_command_desc *cdesc, 107 struct safexcel_token *atoken, 108 u32 length) 109 { 110 struct safexcel_token *token; 111 int ivlen; 112 113 ivlen = safexcel_skcipher_iv(ctx, iv, cdesc); 114 if (ivlen == 4) { 115 /* No space in cdesc, instruction moves to atoken */ 116 cdesc->additional_cdata_size = 1; 117 token = atoken; 118 } else { 119 /* Everything fits in cdesc */ 120 token = (struct safexcel_token *)(cdesc->control_data.token + 2); 121 /* Need to pad with NOP */ 122 eip197_noop_token(&token[1]); 123 } 124 125 token->opcode = EIP197_TOKEN_OPCODE_DIRECTION; 126 token->packet_length = length; 127 token->stat = EIP197_TOKEN_STAT_LAST_PACKET | 128 EIP197_TOKEN_STAT_LAST_HASH; 129 token->instructions = EIP197_TOKEN_INS_LAST | 130 EIP197_TOKEN_INS_TYPE_CRYPTO | 131 EIP197_TOKEN_INS_TYPE_OUTPUT; 132 } 133 134 static void safexcel_aead_iv(struct safexcel_cipher_ctx *ctx, u8 *iv, 135 struct safexcel_command_desc *cdesc) 136 { 137 if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD || 138 ctx->aead & EIP197_AEAD_TYPE_IPSEC_ESP) { /* _ESP and _ESP_GMAC */ 139 /* 32 bit nonce */ 140 cdesc->control_data.token[0] = ctx->nonce; 141 /* 64 bit IV part */ 142 memcpy(&cdesc->control_data.token[1], iv, 8); 143 /* 32 bit counter, start at 0 or 1 (big endian!) */ 144 cdesc->control_data.token[3] = 145 (__force u32)cpu_to_be32(ctx->ctrinit); 146 return; 147 } 148 if (ctx->xcm == EIP197_XCM_MODE_GCM || ctx->alg == SAFEXCEL_CHACHA20) { 149 /* 96 bit IV part */ 150 memcpy(&cdesc->control_data.token[0], iv, 12); 151 /* 32 bit counter, start at 0 or 1 (big endian!) */ 152 cdesc->control_data.token[3] = 153 (__force u32)cpu_to_be32(ctx->ctrinit); 154 return; 155 } 156 /* CBC */ 157 memcpy(cdesc->control_data.token, iv, ctx->blocksz); 158 } 159 160 static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv, 161 struct safexcel_command_desc *cdesc, 162 struct safexcel_token *atoken, 163 enum safexcel_cipher_direction direction, 164 u32 cryptlen, u32 assoclen, u32 digestsize) 165 { 166 struct safexcel_token *aadref; 167 int atoksize = 2; /* Start with minimum size */ 168 int assocadj = assoclen - ctx->aadskip, aadalign; 169 170 /* Always 4 dwords of embedded IV for AEAD modes */ 171 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; 172 173 if (direction == SAFEXCEL_DECRYPT) 174 cryptlen -= digestsize; 175 176 if (unlikely(ctx->xcm == EIP197_XCM_MODE_CCM)) { 177 /* Construct IV block B0 for the CBC-MAC */ 178 u8 *final_iv = (u8 *)cdesc->control_data.token; 179 u8 *cbcmaciv = (u8 *)&atoken[1]; 180 __le32 *aadlen = (__le32 *)&atoken[5]; 181 182 if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { 183 /* Length + nonce */ 184 cdesc->control_data.token[0] = ctx->nonce; 185 /* Fixup flags byte */ 186 *(__le32 *)cbcmaciv = 187 cpu_to_le32(ctx->nonce | 188 ((assocadj > 0) << 6) | 189 ((digestsize - 2) << 2)); 190 /* 64 bit IV part */ 191 memcpy(&cdesc->control_data.token[1], iv, 8); 192 memcpy(cbcmaciv + 4, iv, 8); 193 /* Start counter at 0 */ 194 cdesc->control_data.token[3] = 0; 195 /* Message length */ 196 *(__be32 *)(cbcmaciv + 12) = cpu_to_be32(cryptlen); 197 } else { 198 /* Variable length IV part */ 199 memcpy(final_iv, iv, 15 - iv[0]); 200 memcpy(cbcmaciv, iv, 15 - iv[0]); 201 /* Start variable length counter at 0 */ 202 memset(final_iv + 15 - iv[0], 0, iv[0] + 1); 203 memset(cbcmaciv + 15 - iv[0], 0, iv[0] - 1); 204 /* fixup flags byte */ 205 cbcmaciv[0] |= ((assocadj > 0) << 6) | 206 ((digestsize - 2) << 2); 207 /* insert lower 2 bytes of message length */ 208 cbcmaciv[14] = cryptlen >> 8; 209 cbcmaciv[15] = cryptlen & 255; 210 } 211 212 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 213 atoken->packet_length = AES_BLOCK_SIZE + 214 ((assocadj > 0) << 1); 215 atoken->stat = 0; 216 atoken->instructions = EIP197_TOKEN_INS_ORIGIN_TOKEN | 217 EIP197_TOKEN_INS_TYPE_HASH; 218 219 if (likely(assocadj)) { 220 *aadlen = cpu_to_le32((assocadj >> 8) | 221 (assocadj & 255) << 8); 222 atoken += 6; 223 atoksize += 7; 224 } else { 225 atoken += 5; 226 atoksize += 6; 227 } 228 229 /* Process AAD data */ 230 aadref = atoken; 231 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION; 232 atoken->packet_length = assocadj; 233 atoken->stat = 0; 234 atoken->instructions = EIP197_TOKEN_INS_TYPE_HASH; 235 atoken++; 236 237 /* For CCM only, align AAD data towards hash engine */ 238 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 239 aadalign = (assocadj + 2) & 15; 240 atoken->packet_length = assocadj && aadalign ? 241 16 - aadalign : 242 0; 243 if (likely(cryptlen)) { 244 atoken->stat = 0; 245 atoken->instructions = EIP197_TOKEN_INS_TYPE_HASH; 246 } else { 247 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH; 248 atoken->instructions = EIP197_TOKEN_INS_LAST | 249 EIP197_TOKEN_INS_TYPE_HASH; 250 } 251 } else { 252 safexcel_aead_iv(ctx, iv, cdesc); 253 254 /* Process AAD data */ 255 aadref = atoken; 256 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION; 257 atoken->packet_length = assocadj; 258 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH; 259 atoken->instructions = EIP197_TOKEN_INS_LAST | 260 EIP197_TOKEN_INS_TYPE_HASH; 261 } 262 atoken++; 263 264 if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { 265 /* For ESP mode (and not GMAC), skip over the IV */ 266 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION; 267 atoken->packet_length = EIP197_AEAD_IPSEC_IV_SIZE; 268 atoken->stat = 0; 269 atoken->instructions = 0; 270 atoken++; 271 atoksize++; 272 } else if (unlikely(ctx->alg == SAFEXCEL_CHACHA20 && 273 direction == SAFEXCEL_DECRYPT)) { 274 /* Poly-chacha decryption needs a dummy NOP here ... */ 275 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 276 atoken->packet_length = 16; /* According to Op Manual */ 277 atoken->stat = 0; 278 atoken->instructions = 0; 279 atoken++; 280 atoksize++; 281 } 282 283 if (ctx->xcm) { 284 /* For GCM and CCM, obtain enc(Y0) */ 285 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT_REMRES; 286 atoken->packet_length = 0; 287 atoken->stat = 0; 288 atoken->instructions = AES_BLOCK_SIZE; 289 atoken++; 290 291 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 292 atoken->packet_length = AES_BLOCK_SIZE; 293 atoken->stat = 0; 294 atoken->instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | 295 EIP197_TOKEN_INS_TYPE_CRYPTO; 296 atoken++; 297 atoksize += 2; 298 } 299 300 if (likely(cryptlen || ctx->alg == SAFEXCEL_CHACHA20)) { 301 /* Fixup stat field for AAD direction instruction */ 302 aadref->stat = 0; 303 304 /* Process crypto data */ 305 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION; 306 atoken->packet_length = cryptlen; 307 308 if (unlikely(ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP_GMAC)) { 309 /* Fixup instruction field for AAD dir instruction */ 310 aadref->instructions = EIP197_TOKEN_INS_TYPE_HASH; 311 312 /* Do not send to crypt engine in case of GMAC */ 313 atoken->instructions = EIP197_TOKEN_INS_LAST | 314 EIP197_TOKEN_INS_TYPE_HASH | 315 EIP197_TOKEN_INS_TYPE_OUTPUT; 316 } else { 317 atoken->instructions = EIP197_TOKEN_INS_LAST | 318 EIP197_TOKEN_INS_TYPE_CRYPTO | 319 EIP197_TOKEN_INS_TYPE_HASH | 320 EIP197_TOKEN_INS_TYPE_OUTPUT; 321 } 322 323 cryptlen &= 15; 324 if (unlikely(ctx->xcm == EIP197_XCM_MODE_CCM && cryptlen)) { 325 atoken->stat = 0; 326 /* For CCM only, pad crypto data to the hash engine */ 327 atoken++; 328 atoksize++; 329 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 330 atoken->packet_length = 16 - cryptlen; 331 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH; 332 atoken->instructions = EIP197_TOKEN_INS_TYPE_HASH; 333 } else { 334 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH; 335 } 336 atoken++; 337 atoksize++; 338 } 339 340 if (direction == SAFEXCEL_ENCRYPT) { 341 /* Append ICV */ 342 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 343 atoken->packet_length = digestsize; 344 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH | 345 EIP197_TOKEN_STAT_LAST_PACKET; 346 atoken->instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | 347 EIP197_TOKEN_INS_INSERT_HASH_DIGEST; 348 } else { 349 /* Extract ICV */ 350 atoken->opcode = EIP197_TOKEN_OPCODE_RETRIEVE; 351 atoken->packet_length = digestsize; 352 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH | 353 EIP197_TOKEN_STAT_LAST_PACKET; 354 atoken->instructions = EIP197_TOKEN_INS_INSERT_HASH_DIGEST; 355 atoken++; 356 atoksize++; 357 358 /* Verify ICV */ 359 atoken->opcode = EIP197_TOKEN_OPCODE_VERIFY; 360 atoken->packet_length = digestsize | 361 EIP197_TOKEN_HASH_RESULT_VERIFY; 362 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH | 363 EIP197_TOKEN_STAT_LAST_PACKET; 364 atoken->instructions = EIP197_TOKEN_INS_TYPE_OUTPUT; 365 } 366 367 /* Fixup length of the token in the command descriptor */ 368 cdesc->additional_cdata_size = atoksize; 369 } 370 371 static int safexcel_skcipher_aes_setkey(struct crypto_skcipher *ctfm, 372 const u8 *key, unsigned int len) 373 { 374 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 375 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 376 struct safexcel_crypto_priv *priv = ctx->base.priv; 377 struct crypto_aes_ctx aes; 378 int ret, i; 379 380 ret = aes_expandkey(&aes, key, len); 381 if (ret) 382 return ret; 383 384 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 385 for (i = 0; i < len / sizeof(u32); i++) { 386 if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { 387 ctx->base.needs_inv = true; 388 break; 389 } 390 } 391 } 392 393 for (i = 0; i < len / sizeof(u32); i++) 394 ctx->key[i] = cpu_to_le32(aes.key_enc[i]); 395 396 ctx->key_len = len; 397 398 memzero_explicit(&aes, sizeof(aes)); 399 return 0; 400 } 401 402 static int safexcel_aead_setkey(struct crypto_aead *ctfm, const u8 *key, 403 unsigned int len) 404 { 405 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 406 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 407 struct safexcel_crypto_priv *priv = ctx->base.priv; 408 struct crypto_authenc_keys keys; 409 struct crypto_aes_ctx aes; 410 int err = -EINVAL, i; 411 const char *alg; 412 413 if (unlikely(crypto_authenc_extractkeys(&keys, key, len))) 414 goto badkey; 415 416 if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD) { 417 /* Must have at least space for the nonce here */ 418 if (unlikely(keys.enckeylen < CTR_RFC3686_NONCE_SIZE)) 419 goto badkey; 420 /* last 4 bytes of key are the nonce! */ 421 ctx->nonce = *(u32 *)(keys.enckey + keys.enckeylen - 422 CTR_RFC3686_NONCE_SIZE); 423 /* exclude the nonce here */ 424 keys.enckeylen -= CTR_RFC3686_NONCE_SIZE; 425 } 426 427 /* Encryption key */ 428 switch (ctx->alg) { 429 case SAFEXCEL_DES: 430 err = verify_aead_des_key(ctfm, keys.enckey, keys.enckeylen); 431 if (unlikely(err)) 432 goto badkey; 433 break; 434 case SAFEXCEL_3DES: 435 err = verify_aead_des3_key(ctfm, keys.enckey, keys.enckeylen); 436 if (unlikely(err)) 437 goto badkey; 438 break; 439 case SAFEXCEL_AES: 440 err = aes_expandkey(&aes, keys.enckey, keys.enckeylen); 441 if (unlikely(err)) 442 goto badkey; 443 break; 444 case SAFEXCEL_SM4: 445 if (unlikely(keys.enckeylen != SM4_KEY_SIZE)) 446 goto badkey; 447 break; 448 default: 449 dev_err(priv->dev, "aead: unsupported cipher algorithm\n"); 450 goto badkey; 451 } 452 453 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 454 for (i = 0; i < keys.enckeylen / sizeof(u32); i++) { 455 if (le32_to_cpu(ctx->key[i]) != 456 ((u32 *)keys.enckey)[i]) { 457 ctx->base.needs_inv = true; 458 break; 459 } 460 } 461 } 462 463 /* Auth key */ 464 switch (ctx->hash_alg) { 465 case CONTEXT_CONTROL_CRYPTO_ALG_SHA1: 466 alg = "safexcel-sha1"; 467 break; 468 case CONTEXT_CONTROL_CRYPTO_ALG_SHA224: 469 alg = "safexcel-sha224"; 470 break; 471 case CONTEXT_CONTROL_CRYPTO_ALG_SHA256: 472 alg = "safexcel-sha256"; 473 break; 474 case CONTEXT_CONTROL_CRYPTO_ALG_SHA384: 475 alg = "safexcel-sha384"; 476 break; 477 case CONTEXT_CONTROL_CRYPTO_ALG_SHA512: 478 alg = "safexcel-sha512"; 479 break; 480 case CONTEXT_CONTROL_CRYPTO_ALG_SM3: 481 alg = "safexcel-sm3"; 482 break; 483 default: 484 dev_err(priv->dev, "aead: unsupported hash algorithm\n"); 485 goto badkey; 486 } 487 488 if (safexcel_hmac_setkey(&ctx->base, keys.authkey, keys.authkeylen, 489 alg, ctx->state_sz)) 490 goto badkey; 491 492 /* Now copy the keys into the context */ 493 for (i = 0; i < keys.enckeylen / sizeof(u32); i++) 494 ctx->key[i] = cpu_to_le32(((u32 *)keys.enckey)[i]); 495 ctx->key_len = keys.enckeylen; 496 497 memzero_explicit(&keys, sizeof(keys)); 498 return 0; 499 500 badkey: 501 memzero_explicit(&keys, sizeof(keys)); 502 return err; 503 } 504 505 static int safexcel_context_control(struct safexcel_cipher_ctx *ctx, 506 struct crypto_async_request *async, 507 struct safexcel_cipher_req *sreq, 508 struct safexcel_command_desc *cdesc) 509 { 510 struct safexcel_crypto_priv *priv = ctx->base.priv; 511 int ctrl_size = ctx->key_len / sizeof(u32); 512 513 cdesc->control_data.control1 = ctx->mode; 514 515 if (ctx->aead) { 516 /* Take in account the ipad+opad digests */ 517 if (ctx->xcm) { 518 ctrl_size += ctx->state_sz / sizeof(u32); 519 cdesc->control_data.control0 = 520 CONTEXT_CONTROL_KEY_EN | 521 CONTEXT_CONTROL_DIGEST_XCM | 522 ctx->hash_alg | 523 CONTEXT_CONTROL_SIZE(ctrl_size); 524 } else if (ctx->alg == SAFEXCEL_CHACHA20) { 525 /* Chacha20-Poly1305 */ 526 cdesc->control_data.control0 = 527 CONTEXT_CONTROL_KEY_EN | 528 CONTEXT_CONTROL_CRYPTO_ALG_CHACHA20 | 529 (sreq->direction == SAFEXCEL_ENCRYPT ? 530 CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT : 531 CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN) | 532 ctx->hash_alg | 533 CONTEXT_CONTROL_SIZE(ctrl_size); 534 return 0; 535 } else { 536 ctrl_size += ctx->state_sz / sizeof(u32) * 2; 537 cdesc->control_data.control0 = 538 CONTEXT_CONTROL_KEY_EN | 539 CONTEXT_CONTROL_DIGEST_HMAC | 540 ctx->hash_alg | 541 CONTEXT_CONTROL_SIZE(ctrl_size); 542 } 543 544 if (sreq->direction == SAFEXCEL_ENCRYPT && 545 (ctx->xcm == EIP197_XCM_MODE_CCM || 546 ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP_GMAC)) 547 cdesc->control_data.control0 |= 548 CONTEXT_CONTROL_TYPE_HASH_ENCRYPT_OUT; 549 else if (sreq->direction == SAFEXCEL_ENCRYPT) 550 cdesc->control_data.control0 |= 551 CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT; 552 else if (ctx->xcm == EIP197_XCM_MODE_CCM) 553 cdesc->control_data.control0 |= 554 CONTEXT_CONTROL_TYPE_DECRYPT_HASH_IN; 555 else 556 cdesc->control_data.control0 |= 557 CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN; 558 } else { 559 if (sreq->direction == SAFEXCEL_ENCRYPT) 560 cdesc->control_data.control0 = 561 CONTEXT_CONTROL_TYPE_CRYPTO_OUT | 562 CONTEXT_CONTROL_KEY_EN | 563 CONTEXT_CONTROL_SIZE(ctrl_size); 564 else 565 cdesc->control_data.control0 = 566 CONTEXT_CONTROL_TYPE_CRYPTO_IN | 567 CONTEXT_CONTROL_KEY_EN | 568 CONTEXT_CONTROL_SIZE(ctrl_size); 569 } 570 571 if (ctx->alg == SAFEXCEL_DES) { 572 cdesc->control_data.control0 |= 573 CONTEXT_CONTROL_CRYPTO_ALG_DES; 574 } else if (ctx->alg == SAFEXCEL_3DES) { 575 cdesc->control_data.control0 |= 576 CONTEXT_CONTROL_CRYPTO_ALG_3DES; 577 } else if (ctx->alg == SAFEXCEL_AES) { 578 switch (ctx->key_len >> ctx->xts) { 579 case AES_KEYSIZE_128: 580 cdesc->control_data.control0 |= 581 CONTEXT_CONTROL_CRYPTO_ALG_AES128; 582 break; 583 case AES_KEYSIZE_192: 584 cdesc->control_data.control0 |= 585 CONTEXT_CONTROL_CRYPTO_ALG_AES192; 586 break; 587 case AES_KEYSIZE_256: 588 cdesc->control_data.control0 |= 589 CONTEXT_CONTROL_CRYPTO_ALG_AES256; 590 break; 591 default: 592 dev_err(priv->dev, "aes keysize not supported: %u\n", 593 ctx->key_len >> ctx->xts); 594 return -EINVAL; 595 } 596 } else if (ctx->alg == SAFEXCEL_CHACHA20) { 597 cdesc->control_data.control0 |= 598 CONTEXT_CONTROL_CRYPTO_ALG_CHACHA20; 599 } else if (ctx->alg == SAFEXCEL_SM4) { 600 cdesc->control_data.control0 |= 601 CONTEXT_CONTROL_CRYPTO_ALG_SM4; 602 } 603 604 return 0; 605 } 606 607 static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int ring, 608 struct crypto_async_request *async, 609 struct scatterlist *src, 610 struct scatterlist *dst, 611 unsigned int cryptlen, 612 struct safexcel_cipher_req *sreq, 613 bool *should_complete, int *ret) 614 { 615 struct skcipher_request *areq = skcipher_request_cast(async); 616 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(areq); 617 struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(skcipher); 618 struct safexcel_result_desc *rdesc; 619 int ndesc = 0; 620 621 *ret = 0; 622 623 if (unlikely(!sreq->rdescs)) 624 return 0; 625 626 while (sreq->rdescs--) { 627 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); 628 if (IS_ERR(rdesc)) { 629 dev_err(priv->dev, 630 "cipher: result: could not retrieve the result descriptor\n"); 631 *ret = PTR_ERR(rdesc); 632 break; 633 } 634 635 if (likely(!*ret)) 636 *ret = safexcel_rdesc_check_errors(priv, rdesc); 637 638 ndesc++; 639 } 640 641 safexcel_complete(priv, ring); 642 643 if (src == dst) { 644 if (sreq->nr_src > 0) 645 dma_unmap_sg(priv->dev, src, sreq->nr_src, 646 DMA_BIDIRECTIONAL); 647 } else { 648 if (sreq->nr_src > 0) 649 dma_unmap_sg(priv->dev, src, sreq->nr_src, 650 DMA_TO_DEVICE); 651 if (sreq->nr_dst > 0) 652 dma_unmap_sg(priv->dev, dst, sreq->nr_dst, 653 DMA_FROM_DEVICE); 654 } 655 656 /* 657 * Update IV in req from last crypto output word for CBC modes 658 */ 659 if ((!ctx->aead) && (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) && 660 (sreq->direction == SAFEXCEL_ENCRYPT)) { 661 /* For encrypt take the last output word */ 662 sg_pcopy_to_buffer(dst, sreq->nr_dst, areq->iv, 663 crypto_skcipher_ivsize(skcipher), 664 (cryptlen - 665 crypto_skcipher_ivsize(skcipher))); 666 } 667 668 *should_complete = true; 669 670 return ndesc; 671 } 672 673 static int safexcel_send_req(struct crypto_async_request *base, int ring, 674 struct safexcel_cipher_req *sreq, 675 struct scatterlist *src, struct scatterlist *dst, 676 unsigned int cryptlen, unsigned int assoclen, 677 unsigned int digestsize, u8 *iv, int *commands, 678 int *results) 679 { 680 struct skcipher_request *areq = skcipher_request_cast(base); 681 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(areq); 682 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); 683 struct safexcel_crypto_priv *priv = ctx->base.priv; 684 struct safexcel_command_desc *cdesc; 685 struct safexcel_command_desc *first_cdesc = NULL; 686 struct safexcel_result_desc *rdesc, *first_rdesc = NULL; 687 struct scatterlist *sg; 688 unsigned int totlen; 689 unsigned int totlen_src = cryptlen + assoclen; 690 unsigned int totlen_dst = totlen_src; 691 struct safexcel_token *atoken; 692 int n_cdesc = 0, n_rdesc = 0; 693 int queued, i, ret = 0; 694 bool first = true; 695 696 sreq->nr_src = sg_nents_for_len(src, totlen_src); 697 698 if (ctx->aead) { 699 /* 700 * AEAD has auth tag appended to output for encrypt and 701 * removed from the output for decrypt! 702 */ 703 if (sreq->direction == SAFEXCEL_DECRYPT) 704 totlen_dst -= digestsize; 705 else 706 totlen_dst += digestsize; 707 708 memcpy(ctx->base.ctxr->data + ctx->key_len / sizeof(u32), 709 &ctx->base.ipad, ctx->state_sz); 710 if (!ctx->xcm) 711 memcpy(ctx->base.ctxr->data + (ctx->key_len + 712 ctx->state_sz) / sizeof(u32), &ctx->base.opad, 713 ctx->state_sz); 714 } else if ((ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) && 715 (sreq->direction == SAFEXCEL_DECRYPT)) { 716 /* 717 * Save IV from last crypto input word for CBC modes in decrypt 718 * direction. Need to do this first in case of inplace operation 719 * as it will be overwritten. 720 */ 721 sg_pcopy_to_buffer(src, sreq->nr_src, areq->iv, 722 crypto_skcipher_ivsize(skcipher), 723 (totlen_src - 724 crypto_skcipher_ivsize(skcipher))); 725 } 726 727 sreq->nr_dst = sg_nents_for_len(dst, totlen_dst); 728 729 /* 730 * Remember actual input length, source buffer length may be 731 * updated in case of inline operation below. 732 */ 733 totlen = totlen_src; 734 queued = totlen_src; 735 736 if (src == dst) { 737 sreq->nr_src = max(sreq->nr_src, sreq->nr_dst); 738 sreq->nr_dst = sreq->nr_src; 739 if (unlikely((totlen_src || totlen_dst) && 740 (sreq->nr_src <= 0))) { 741 dev_err(priv->dev, "In-place buffer not large enough (need %d bytes)!", 742 max(totlen_src, totlen_dst)); 743 return -EINVAL; 744 } 745 if (sreq->nr_src > 0 && 746 !dma_map_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL)) 747 return -EIO; 748 } else { 749 if (unlikely(totlen_src && (sreq->nr_src <= 0))) { 750 dev_err(priv->dev, "Source buffer not large enough (need %d bytes)!", 751 totlen_src); 752 return -EINVAL; 753 } 754 755 if (sreq->nr_src > 0 && 756 !dma_map_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE)) 757 return -EIO; 758 759 if (unlikely(totlen_dst && (sreq->nr_dst <= 0))) { 760 dev_err(priv->dev, "Dest buffer not large enough (need %d bytes)!", 761 totlen_dst); 762 ret = -EINVAL; 763 goto unmap; 764 } 765 766 if (sreq->nr_dst > 0 && 767 !dma_map_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE)) { 768 ret = -EIO; 769 goto unmap; 770 } 771 } 772 773 memcpy(ctx->base.ctxr->data, ctx->key, ctx->key_len); 774 775 if (!totlen) { 776 /* 777 * The EIP97 cannot deal with zero length input packets! 778 * So stuff a dummy command descriptor indicating a 1 byte 779 * (dummy) input packet, using the context record as source. 780 */ 781 first_cdesc = safexcel_add_cdesc(priv, ring, 782 1, 1, ctx->base.ctxr_dma, 783 1, 1, ctx->base.ctxr_dma, 784 &atoken); 785 if (IS_ERR(first_cdesc)) { 786 /* No space left in the command descriptor ring */ 787 ret = PTR_ERR(first_cdesc); 788 goto cdesc_rollback; 789 } 790 n_cdesc = 1; 791 goto skip_cdesc; 792 } 793 794 /* command descriptors */ 795 for_each_sg(src, sg, sreq->nr_src, i) { 796 int len = sg_dma_len(sg); 797 798 /* Do not overflow the request */ 799 if (queued < len) 800 len = queued; 801 802 cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc, 803 !(queued - len), 804 sg_dma_address(sg), len, totlen, 805 ctx->base.ctxr_dma, &atoken); 806 if (IS_ERR(cdesc)) { 807 /* No space left in the command descriptor ring */ 808 ret = PTR_ERR(cdesc); 809 goto cdesc_rollback; 810 } 811 812 if (!n_cdesc) 813 first_cdesc = cdesc; 814 815 n_cdesc++; 816 queued -= len; 817 if (!queued) 818 break; 819 } 820 skip_cdesc: 821 /* Add context control words and token to first command descriptor */ 822 safexcel_context_control(ctx, base, sreq, first_cdesc); 823 if (ctx->aead) 824 safexcel_aead_token(ctx, iv, first_cdesc, atoken, 825 sreq->direction, cryptlen, 826 assoclen, digestsize); 827 else 828 safexcel_skcipher_token(ctx, iv, first_cdesc, atoken, 829 cryptlen); 830 831 /* result descriptors */ 832 for_each_sg(dst, sg, sreq->nr_dst, i) { 833 bool last = (i == sreq->nr_dst - 1); 834 u32 len = sg_dma_len(sg); 835 836 /* only allow the part of the buffer we know we need */ 837 if (len > totlen_dst) 838 len = totlen_dst; 839 if (unlikely(!len)) 840 break; 841 totlen_dst -= len; 842 843 /* skip over AAD space in buffer - not written */ 844 if (assoclen) { 845 if (assoclen >= len) { 846 assoclen -= len; 847 continue; 848 } 849 rdesc = safexcel_add_rdesc(priv, ring, first, last, 850 sg_dma_address(sg) + 851 assoclen, 852 len - assoclen); 853 assoclen = 0; 854 } else { 855 rdesc = safexcel_add_rdesc(priv, ring, first, last, 856 sg_dma_address(sg), 857 len); 858 } 859 if (IS_ERR(rdesc)) { 860 /* No space left in the result descriptor ring */ 861 ret = PTR_ERR(rdesc); 862 goto rdesc_rollback; 863 } 864 if (first) { 865 first_rdesc = rdesc; 866 first = false; 867 } 868 n_rdesc++; 869 } 870 871 if (unlikely(first)) { 872 /* 873 * Special case: AEAD decrypt with only AAD data. 874 * In this case there is NO output data from the engine, 875 * but the engine still needs a result descriptor! 876 * Create a dummy one just for catching the result token. 877 */ 878 rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0); 879 if (IS_ERR(rdesc)) { 880 /* No space left in the result descriptor ring */ 881 ret = PTR_ERR(rdesc); 882 goto rdesc_rollback; 883 } 884 first_rdesc = rdesc; 885 n_rdesc = 1; 886 } 887 888 safexcel_rdr_req_set(priv, ring, first_rdesc, base); 889 890 *commands = n_cdesc; 891 *results = n_rdesc; 892 return 0; 893 894 rdesc_rollback: 895 for (i = 0; i < n_rdesc; i++) 896 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].rdr); 897 cdesc_rollback: 898 for (i = 0; i < n_cdesc; i++) 899 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr); 900 unmap: 901 if (src == dst) { 902 if (sreq->nr_src > 0) 903 dma_unmap_sg(priv->dev, src, sreq->nr_src, 904 DMA_BIDIRECTIONAL); 905 } else { 906 if (sreq->nr_src > 0) 907 dma_unmap_sg(priv->dev, src, sreq->nr_src, 908 DMA_TO_DEVICE); 909 if (sreq->nr_dst > 0) 910 dma_unmap_sg(priv->dev, dst, sreq->nr_dst, 911 DMA_FROM_DEVICE); 912 } 913 914 return ret; 915 } 916 917 static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv, 918 int ring, 919 struct crypto_async_request *base, 920 struct safexcel_cipher_req *sreq, 921 bool *should_complete, int *ret) 922 { 923 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); 924 struct safexcel_result_desc *rdesc; 925 int ndesc = 0, enq_ret; 926 927 *ret = 0; 928 929 if (unlikely(!sreq->rdescs)) 930 return 0; 931 932 while (sreq->rdescs--) { 933 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); 934 if (IS_ERR(rdesc)) { 935 dev_err(priv->dev, 936 "cipher: invalidate: could not retrieve the result descriptor\n"); 937 *ret = PTR_ERR(rdesc); 938 break; 939 } 940 941 if (likely(!*ret)) 942 *ret = safexcel_rdesc_check_errors(priv, rdesc); 943 944 ndesc++; 945 } 946 947 safexcel_complete(priv, ring); 948 949 if (ctx->base.exit_inv) { 950 dma_pool_free(priv->context_pool, ctx->base.ctxr, 951 ctx->base.ctxr_dma); 952 953 *should_complete = true; 954 955 return ndesc; 956 } 957 958 ring = safexcel_select_ring(priv); 959 ctx->base.ring = ring; 960 961 spin_lock_bh(&priv->ring[ring].queue_lock); 962 enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, base); 963 spin_unlock_bh(&priv->ring[ring].queue_lock); 964 965 if (enq_ret != -EINPROGRESS) 966 *ret = enq_ret; 967 968 queue_work(priv->ring[ring].workqueue, 969 &priv->ring[ring].work_data.work); 970 971 *should_complete = false; 972 973 return ndesc; 974 } 975 976 static int safexcel_skcipher_handle_result(struct safexcel_crypto_priv *priv, 977 int ring, 978 struct crypto_async_request *async, 979 bool *should_complete, int *ret) 980 { 981 struct skcipher_request *req = skcipher_request_cast(async); 982 struct safexcel_cipher_req *sreq = skcipher_request_ctx(req); 983 int err; 984 985 if (sreq->needs_inv) { 986 sreq->needs_inv = false; 987 err = safexcel_handle_inv_result(priv, ring, async, sreq, 988 should_complete, ret); 989 } else { 990 err = safexcel_handle_req_result(priv, ring, async, req->src, 991 req->dst, req->cryptlen, sreq, 992 should_complete, ret); 993 } 994 995 return err; 996 } 997 998 static int safexcel_aead_handle_result(struct safexcel_crypto_priv *priv, 999 int ring, 1000 struct crypto_async_request *async, 1001 bool *should_complete, int *ret) 1002 { 1003 struct aead_request *req = aead_request_cast(async); 1004 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1005 struct safexcel_cipher_req *sreq = aead_request_ctx(req); 1006 int err; 1007 1008 if (sreq->needs_inv) { 1009 sreq->needs_inv = false; 1010 err = safexcel_handle_inv_result(priv, ring, async, sreq, 1011 should_complete, ret); 1012 } else { 1013 err = safexcel_handle_req_result(priv, ring, async, req->src, 1014 req->dst, 1015 req->cryptlen + crypto_aead_authsize(tfm), 1016 sreq, should_complete, ret); 1017 } 1018 1019 return err; 1020 } 1021 1022 static int safexcel_cipher_send_inv(struct crypto_async_request *base, 1023 int ring, int *commands, int *results) 1024 { 1025 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); 1026 struct safexcel_crypto_priv *priv = ctx->base.priv; 1027 int ret; 1028 1029 ret = safexcel_invalidate_cache(base, priv, ctx->base.ctxr_dma, ring); 1030 if (unlikely(ret)) 1031 return ret; 1032 1033 *commands = 1; 1034 *results = 1; 1035 1036 return 0; 1037 } 1038 1039 static int safexcel_skcipher_send(struct crypto_async_request *async, int ring, 1040 int *commands, int *results) 1041 { 1042 struct skcipher_request *req = skcipher_request_cast(async); 1043 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); 1044 struct safexcel_cipher_req *sreq = skcipher_request_ctx(req); 1045 struct safexcel_crypto_priv *priv = ctx->base.priv; 1046 int ret; 1047 1048 BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv); 1049 1050 if (sreq->needs_inv) { 1051 ret = safexcel_cipher_send_inv(async, ring, commands, results); 1052 } else { 1053 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1054 u8 input_iv[AES_BLOCK_SIZE]; 1055 1056 /* 1057 * Save input IV in case of CBC decrypt mode 1058 * Will be overwritten with output IV prior to use! 1059 */ 1060 memcpy(input_iv, req->iv, crypto_skcipher_ivsize(skcipher)); 1061 1062 ret = safexcel_send_req(async, ring, sreq, req->src, 1063 req->dst, req->cryptlen, 0, 0, input_iv, 1064 commands, results); 1065 } 1066 1067 sreq->rdescs = *results; 1068 return ret; 1069 } 1070 1071 static int safexcel_aead_send(struct crypto_async_request *async, int ring, 1072 int *commands, int *results) 1073 { 1074 struct aead_request *req = aead_request_cast(async); 1075 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1076 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); 1077 struct safexcel_cipher_req *sreq = aead_request_ctx(req); 1078 struct safexcel_crypto_priv *priv = ctx->base.priv; 1079 int ret; 1080 1081 BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv); 1082 1083 if (sreq->needs_inv) 1084 ret = safexcel_cipher_send_inv(async, ring, commands, results); 1085 else 1086 ret = safexcel_send_req(async, ring, sreq, req->src, req->dst, 1087 req->cryptlen, req->assoclen, 1088 crypto_aead_authsize(tfm), req->iv, 1089 commands, results); 1090 sreq->rdescs = *results; 1091 return ret; 1092 } 1093 1094 static int safexcel_cipher_exit_inv(struct crypto_tfm *tfm, 1095 struct crypto_async_request *base, 1096 struct safexcel_cipher_req *sreq, 1097 struct crypto_wait *result) 1098 { 1099 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1100 struct safexcel_crypto_priv *priv = ctx->base.priv; 1101 int ring = ctx->base.ring; 1102 int err; 1103 1104 ctx = crypto_tfm_ctx(base->tfm); 1105 ctx->base.exit_inv = true; 1106 sreq->needs_inv = true; 1107 1108 spin_lock_bh(&priv->ring[ring].queue_lock); 1109 crypto_enqueue_request(&priv->ring[ring].queue, base); 1110 spin_unlock_bh(&priv->ring[ring].queue_lock); 1111 1112 queue_work(priv->ring[ring].workqueue, 1113 &priv->ring[ring].work_data.work); 1114 1115 err = crypto_wait_req(-EINPROGRESS, result); 1116 1117 if (err) { 1118 dev_warn(priv->dev, 1119 "cipher: sync: invalidate: completion error %d\n", 1120 err); 1121 return err; 1122 } 1123 1124 return 0; 1125 } 1126 1127 static int safexcel_skcipher_exit_inv(struct crypto_tfm *tfm) 1128 { 1129 EIP197_REQUEST_ON_STACK(req, skcipher, EIP197_SKCIPHER_REQ_SIZE); 1130 struct safexcel_cipher_req *sreq = skcipher_request_ctx(req); 1131 DECLARE_CRYPTO_WAIT(result); 1132 1133 memset(req, 0, sizeof(struct skcipher_request)); 1134 1135 skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 1136 crypto_req_done, &result); 1137 skcipher_request_set_tfm(req, __crypto_skcipher_cast(tfm)); 1138 1139 return safexcel_cipher_exit_inv(tfm, &req->base, sreq, &result); 1140 } 1141 1142 static int safexcel_aead_exit_inv(struct crypto_tfm *tfm) 1143 { 1144 EIP197_REQUEST_ON_STACK(req, aead, EIP197_AEAD_REQ_SIZE); 1145 struct safexcel_cipher_req *sreq = aead_request_ctx(req); 1146 DECLARE_CRYPTO_WAIT(result); 1147 1148 memset(req, 0, sizeof(struct aead_request)); 1149 1150 aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 1151 crypto_req_done, &result); 1152 aead_request_set_tfm(req, __crypto_aead_cast(tfm)); 1153 1154 return safexcel_cipher_exit_inv(tfm, &req->base, sreq, &result); 1155 } 1156 1157 static int safexcel_queue_req(struct crypto_async_request *base, 1158 struct safexcel_cipher_req *sreq, 1159 enum safexcel_cipher_direction dir) 1160 { 1161 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); 1162 struct safexcel_crypto_priv *priv = ctx->base.priv; 1163 int ret, ring; 1164 1165 sreq->needs_inv = false; 1166 sreq->direction = dir; 1167 1168 if (ctx->base.ctxr) { 1169 if (priv->flags & EIP197_TRC_CACHE && ctx->base.needs_inv) { 1170 sreq->needs_inv = true; 1171 ctx->base.needs_inv = false; 1172 } 1173 } else { 1174 ctx->base.ring = safexcel_select_ring(priv); 1175 ctx->base.ctxr = dma_pool_zalloc(priv->context_pool, 1176 EIP197_GFP_FLAGS(*base), 1177 &ctx->base.ctxr_dma); 1178 if (!ctx->base.ctxr) 1179 return -ENOMEM; 1180 } 1181 1182 ring = ctx->base.ring; 1183 1184 spin_lock_bh(&priv->ring[ring].queue_lock); 1185 ret = crypto_enqueue_request(&priv->ring[ring].queue, base); 1186 spin_unlock_bh(&priv->ring[ring].queue_lock); 1187 1188 queue_work(priv->ring[ring].workqueue, 1189 &priv->ring[ring].work_data.work); 1190 1191 return ret; 1192 } 1193 1194 static int safexcel_encrypt(struct skcipher_request *req) 1195 { 1196 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 1197 SAFEXCEL_ENCRYPT); 1198 } 1199 1200 static int safexcel_decrypt(struct skcipher_request *req) 1201 { 1202 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 1203 SAFEXCEL_DECRYPT); 1204 } 1205 1206 static int safexcel_skcipher_cra_init(struct crypto_tfm *tfm) 1207 { 1208 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1209 struct safexcel_alg_template *tmpl = 1210 container_of(tfm->__crt_alg, struct safexcel_alg_template, 1211 alg.skcipher.base); 1212 1213 crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm), 1214 sizeof(struct safexcel_cipher_req)); 1215 1216 ctx->base.priv = tmpl->priv; 1217 1218 ctx->base.send = safexcel_skcipher_send; 1219 ctx->base.handle_result = safexcel_skcipher_handle_result; 1220 ctx->ivmask = EIP197_OPTION_4_TOKEN_IV_CMD; 1221 ctx->ctrinit = 1; 1222 return 0; 1223 } 1224 1225 static int safexcel_cipher_cra_exit(struct crypto_tfm *tfm) 1226 { 1227 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1228 1229 memzero_explicit(ctx->key, sizeof(ctx->key)); 1230 1231 /* context not allocated, skip invalidation */ 1232 if (!ctx->base.ctxr) 1233 return -ENOMEM; 1234 1235 memzero_explicit(ctx->base.ctxr->data, sizeof(ctx->base.ctxr->data)); 1236 return 0; 1237 } 1238 1239 static void safexcel_skcipher_cra_exit(struct crypto_tfm *tfm) 1240 { 1241 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1242 struct safexcel_crypto_priv *priv = ctx->base.priv; 1243 int ret; 1244 1245 if (safexcel_cipher_cra_exit(tfm)) 1246 return; 1247 1248 if (priv->flags & EIP197_TRC_CACHE) { 1249 ret = safexcel_skcipher_exit_inv(tfm); 1250 if (ret) 1251 dev_warn(priv->dev, "skcipher: invalidation error %d\n", 1252 ret); 1253 } else { 1254 dma_pool_free(priv->context_pool, ctx->base.ctxr, 1255 ctx->base.ctxr_dma); 1256 } 1257 } 1258 1259 static void safexcel_aead_cra_exit(struct crypto_tfm *tfm) 1260 { 1261 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1262 struct safexcel_crypto_priv *priv = ctx->base.priv; 1263 int ret; 1264 1265 if (safexcel_cipher_cra_exit(tfm)) 1266 return; 1267 1268 if (priv->flags & EIP197_TRC_CACHE) { 1269 ret = safexcel_aead_exit_inv(tfm); 1270 if (ret) 1271 dev_warn(priv->dev, "aead: invalidation error %d\n", 1272 ret); 1273 } else { 1274 dma_pool_free(priv->context_pool, ctx->base.ctxr, 1275 ctx->base.ctxr_dma); 1276 } 1277 } 1278 1279 static int safexcel_skcipher_aes_ecb_cra_init(struct crypto_tfm *tfm) 1280 { 1281 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1282 1283 safexcel_skcipher_cra_init(tfm); 1284 ctx->alg = SAFEXCEL_AES; 1285 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; 1286 ctx->blocksz = 0; 1287 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1288 return 0; 1289 } 1290 1291 struct safexcel_alg_template safexcel_alg_ecb_aes = { 1292 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1293 .algo_mask = SAFEXCEL_ALG_AES, 1294 .alg.skcipher = { 1295 .setkey = safexcel_skcipher_aes_setkey, 1296 .encrypt = safexcel_encrypt, 1297 .decrypt = safexcel_decrypt, 1298 .min_keysize = AES_MIN_KEY_SIZE, 1299 .max_keysize = AES_MAX_KEY_SIZE, 1300 .base = { 1301 .cra_name = "ecb(aes)", 1302 .cra_driver_name = "safexcel-ecb-aes", 1303 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1304 .cra_flags = CRYPTO_ALG_ASYNC | 1305 CRYPTO_ALG_ALLOCATES_MEMORY | 1306 CRYPTO_ALG_KERN_DRIVER_ONLY, 1307 .cra_blocksize = AES_BLOCK_SIZE, 1308 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1309 .cra_alignmask = 0, 1310 .cra_init = safexcel_skcipher_aes_ecb_cra_init, 1311 .cra_exit = safexcel_skcipher_cra_exit, 1312 .cra_module = THIS_MODULE, 1313 }, 1314 }, 1315 }; 1316 1317 static int safexcel_skcipher_aes_cbc_cra_init(struct crypto_tfm *tfm) 1318 { 1319 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1320 1321 safexcel_skcipher_cra_init(tfm); 1322 ctx->alg = SAFEXCEL_AES; 1323 ctx->blocksz = AES_BLOCK_SIZE; 1324 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; 1325 return 0; 1326 } 1327 1328 struct safexcel_alg_template safexcel_alg_cbc_aes = { 1329 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1330 .algo_mask = SAFEXCEL_ALG_AES, 1331 .alg.skcipher = { 1332 .setkey = safexcel_skcipher_aes_setkey, 1333 .encrypt = safexcel_encrypt, 1334 .decrypt = safexcel_decrypt, 1335 .min_keysize = AES_MIN_KEY_SIZE, 1336 .max_keysize = AES_MAX_KEY_SIZE, 1337 .ivsize = AES_BLOCK_SIZE, 1338 .base = { 1339 .cra_name = "cbc(aes)", 1340 .cra_driver_name = "safexcel-cbc-aes", 1341 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1342 .cra_flags = CRYPTO_ALG_ASYNC | 1343 CRYPTO_ALG_ALLOCATES_MEMORY | 1344 CRYPTO_ALG_KERN_DRIVER_ONLY, 1345 .cra_blocksize = AES_BLOCK_SIZE, 1346 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1347 .cra_alignmask = 0, 1348 .cra_init = safexcel_skcipher_aes_cbc_cra_init, 1349 .cra_exit = safexcel_skcipher_cra_exit, 1350 .cra_module = THIS_MODULE, 1351 }, 1352 }, 1353 }; 1354 1355 static int safexcel_skcipher_aesctr_setkey(struct crypto_skcipher *ctfm, 1356 const u8 *key, unsigned int len) 1357 { 1358 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 1359 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1360 struct safexcel_crypto_priv *priv = ctx->base.priv; 1361 struct crypto_aes_ctx aes; 1362 int ret, i; 1363 unsigned int keylen; 1364 1365 /* last 4 bytes of key are the nonce! */ 1366 ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE); 1367 /* exclude the nonce here */ 1368 keylen = len - CTR_RFC3686_NONCE_SIZE; 1369 ret = aes_expandkey(&aes, key, keylen); 1370 if (ret) 1371 return ret; 1372 1373 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 1374 for (i = 0; i < keylen / sizeof(u32); i++) { 1375 if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { 1376 ctx->base.needs_inv = true; 1377 break; 1378 } 1379 } 1380 } 1381 1382 for (i = 0; i < keylen / sizeof(u32); i++) 1383 ctx->key[i] = cpu_to_le32(aes.key_enc[i]); 1384 1385 ctx->key_len = keylen; 1386 1387 memzero_explicit(&aes, sizeof(aes)); 1388 return 0; 1389 } 1390 1391 static int safexcel_skcipher_aes_ctr_cra_init(struct crypto_tfm *tfm) 1392 { 1393 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1394 1395 safexcel_skcipher_cra_init(tfm); 1396 ctx->alg = SAFEXCEL_AES; 1397 ctx->blocksz = AES_BLOCK_SIZE; 1398 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; 1399 return 0; 1400 } 1401 1402 struct safexcel_alg_template safexcel_alg_ctr_aes = { 1403 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1404 .algo_mask = SAFEXCEL_ALG_AES, 1405 .alg.skcipher = { 1406 .setkey = safexcel_skcipher_aesctr_setkey, 1407 .encrypt = safexcel_encrypt, 1408 .decrypt = safexcel_decrypt, 1409 /* Add nonce size */ 1410 .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 1411 .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 1412 .ivsize = CTR_RFC3686_IV_SIZE, 1413 .base = { 1414 .cra_name = "rfc3686(ctr(aes))", 1415 .cra_driver_name = "safexcel-ctr-aes", 1416 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1417 .cra_flags = CRYPTO_ALG_ASYNC | 1418 CRYPTO_ALG_ALLOCATES_MEMORY | 1419 CRYPTO_ALG_KERN_DRIVER_ONLY, 1420 .cra_blocksize = 1, 1421 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1422 .cra_alignmask = 0, 1423 .cra_init = safexcel_skcipher_aes_ctr_cra_init, 1424 .cra_exit = safexcel_skcipher_cra_exit, 1425 .cra_module = THIS_MODULE, 1426 }, 1427 }, 1428 }; 1429 1430 static int safexcel_des_setkey(struct crypto_skcipher *ctfm, const u8 *key, 1431 unsigned int len) 1432 { 1433 struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm); 1434 struct safexcel_crypto_priv *priv = ctx->base.priv; 1435 int ret; 1436 1437 ret = verify_skcipher_des_key(ctfm, key); 1438 if (ret) 1439 return ret; 1440 1441 /* if context exits and key changed, need to invalidate it */ 1442 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) 1443 if (memcmp(ctx->key, key, len)) 1444 ctx->base.needs_inv = true; 1445 1446 memcpy(ctx->key, key, len); 1447 ctx->key_len = len; 1448 1449 return 0; 1450 } 1451 1452 static int safexcel_skcipher_des_cbc_cra_init(struct crypto_tfm *tfm) 1453 { 1454 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1455 1456 safexcel_skcipher_cra_init(tfm); 1457 ctx->alg = SAFEXCEL_DES; 1458 ctx->blocksz = DES_BLOCK_SIZE; 1459 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1460 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; 1461 return 0; 1462 } 1463 1464 struct safexcel_alg_template safexcel_alg_cbc_des = { 1465 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1466 .algo_mask = SAFEXCEL_ALG_DES, 1467 .alg.skcipher = { 1468 .setkey = safexcel_des_setkey, 1469 .encrypt = safexcel_encrypt, 1470 .decrypt = safexcel_decrypt, 1471 .min_keysize = DES_KEY_SIZE, 1472 .max_keysize = DES_KEY_SIZE, 1473 .ivsize = DES_BLOCK_SIZE, 1474 .base = { 1475 .cra_name = "cbc(des)", 1476 .cra_driver_name = "safexcel-cbc-des", 1477 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1478 .cra_flags = CRYPTO_ALG_ASYNC | 1479 CRYPTO_ALG_ALLOCATES_MEMORY | 1480 CRYPTO_ALG_KERN_DRIVER_ONLY, 1481 .cra_blocksize = DES_BLOCK_SIZE, 1482 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1483 .cra_alignmask = 0, 1484 .cra_init = safexcel_skcipher_des_cbc_cra_init, 1485 .cra_exit = safexcel_skcipher_cra_exit, 1486 .cra_module = THIS_MODULE, 1487 }, 1488 }, 1489 }; 1490 1491 static int safexcel_skcipher_des_ecb_cra_init(struct crypto_tfm *tfm) 1492 { 1493 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1494 1495 safexcel_skcipher_cra_init(tfm); 1496 ctx->alg = SAFEXCEL_DES; 1497 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; 1498 ctx->blocksz = 0; 1499 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1500 return 0; 1501 } 1502 1503 struct safexcel_alg_template safexcel_alg_ecb_des = { 1504 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1505 .algo_mask = SAFEXCEL_ALG_DES, 1506 .alg.skcipher = { 1507 .setkey = safexcel_des_setkey, 1508 .encrypt = safexcel_encrypt, 1509 .decrypt = safexcel_decrypt, 1510 .min_keysize = DES_KEY_SIZE, 1511 .max_keysize = DES_KEY_SIZE, 1512 .base = { 1513 .cra_name = "ecb(des)", 1514 .cra_driver_name = "safexcel-ecb-des", 1515 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1516 .cra_flags = CRYPTO_ALG_ASYNC | 1517 CRYPTO_ALG_ALLOCATES_MEMORY | 1518 CRYPTO_ALG_KERN_DRIVER_ONLY, 1519 .cra_blocksize = DES_BLOCK_SIZE, 1520 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1521 .cra_alignmask = 0, 1522 .cra_init = safexcel_skcipher_des_ecb_cra_init, 1523 .cra_exit = safexcel_skcipher_cra_exit, 1524 .cra_module = THIS_MODULE, 1525 }, 1526 }, 1527 }; 1528 1529 static int safexcel_des3_ede_setkey(struct crypto_skcipher *ctfm, 1530 const u8 *key, unsigned int len) 1531 { 1532 struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm); 1533 struct safexcel_crypto_priv *priv = ctx->base.priv; 1534 int err; 1535 1536 err = verify_skcipher_des3_key(ctfm, key); 1537 if (err) 1538 return err; 1539 1540 /* if context exits and key changed, need to invalidate it */ 1541 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) 1542 if (memcmp(ctx->key, key, len)) 1543 ctx->base.needs_inv = true; 1544 1545 memcpy(ctx->key, key, len); 1546 ctx->key_len = len; 1547 1548 return 0; 1549 } 1550 1551 static int safexcel_skcipher_des3_cbc_cra_init(struct crypto_tfm *tfm) 1552 { 1553 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1554 1555 safexcel_skcipher_cra_init(tfm); 1556 ctx->alg = SAFEXCEL_3DES; 1557 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 1558 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1559 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; 1560 return 0; 1561 } 1562 1563 struct safexcel_alg_template safexcel_alg_cbc_des3_ede = { 1564 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1565 .algo_mask = SAFEXCEL_ALG_DES, 1566 .alg.skcipher = { 1567 .setkey = safexcel_des3_ede_setkey, 1568 .encrypt = safexcel_encrypt, 1569 .decrypt = safexcel_decrypt, 1570 .min_keysize = DES3_EDE_KEY_SIZE, 1571 .max_keysize = DES3_EDE_KEY_SIZE, 1572 .ivsize = DES3_EDE_BLOCK_SIZE, 1573 .base = { 1574 .cra_name = "cbc(des3_ede)", 1575 .cra_driver_name = "safexcel-cbc-des3_ede", 1576 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1577 .cra_flags = CRYPTO_ALG_ASYNC | 1578 CRYPTO_ALG_ALLOCATES_MEMORY | 1579 CRYPTO_ALG_KERN_DRIVER_ONLY, 1580 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1581 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1582 .cra_alignmask = 0, 1583 .cra_init = safexcel_skcipher_des3_cbc_cra_init, 1584 .cra_exit = safexcel_skcipher_cra_exit, 1585 .cra_module = THIS_MODULE, 1586 }, 1587 }, 1588 }; 1589 1590 static int safexcel_skcipher_des3_ecb_cra_init(struct crypto_tfm *tfm) 1591 { 1592 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1593 1594 safexcel_skcipher_cra_init(tfm); 1595 ctx->alg = SAFEXCEL_3DES; 1596 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; 1597 ctx->blocksz = 0; 1598 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1599 return 0; 1600 } 1601 1602 struct safexcel_alg_template safexcel_alg_ecb_des3_ede = { 1603 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1604 .algo_mask = SAFEXCEL_ALG_DES, 1605 .alg.skcipher = { 1606 .setkey = safexcel_des3_ede_setkey, 1607 .encrypt = safexcel_encrypt, 1608 .decrypt = safexcel_decrypt, 1609 .min_keysize = DES3_EDE_KEY_SIZE, 1610 .max_keysize = DES3_EDE_KEY_SIZE, 1611 .base = { 1612 .cra_name = "ecb(des3_ede)", 1613 .cra_driver_name = "safexcel-ecb-des3_ede", 1614 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1615 .cra_flags = CRYPTO_ALG_ASYNC | 1616 CRYPTO_ALG_ALLOCATES_MEMORY | 1617 CRYPTO_ALG_KERN_DRIVER_ONLY, 1618 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1619 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1620 .cra_alignmask = 0, 1621 .cra_init = safexcel_skcipher_des3_ecb_cra_init, 1622 .cra_exit = safexcel_skcipher_cra_exit, 1623 .cra_module = THIS_MODULE, 1624 }, 1625 }, 1626 }; 1627 1628 static int safexcel_aead_encrypt(struct aead_request *req) 1629 { 1630 struct safexcel_cipher_req *creq = aead_request_ctx(req); 1631 1632 return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); 1633 } 1634 1635 static int safexcel_aead_decrypt(struct aead_request *req) 1636 { 1637 struct safexcel_cipher_req *creq = aead_request_ctx(req); 1638 1639 return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); 1640 } 1641 1642 static int safexcel_aead_cra_init(struct crypto_tfm *tfm) 1643 { 1644 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1645 struct safexcel_alg_template *tmpl = 1646 container_of(tfm->__crt_alg, struct safexcel_alg_template, 1647 alg.aead.base); 1648 1649 crypto_aead_set_reqsize(__crypto_aead_cast(tfm), 1650 sizeof(struct safexcel_cipher_req)); 1651 1652 ctx->base.priv = tmpl->priv; 1653 1654 ctx->alg = SAFEXCEL_AES; /* default */ 1655 ctx->blocksz = AES_BLOCK_SIZE; 1656 ctx->ivmask = EIP197_OPTION_4_TOKEN_IV_CMD; 1657 ctx->ctrinit = 1; 1658 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; /* default */ 1659 ctx->aead = true; 1660 ctx->base.send = safexcel_aead_send; 1661 ctx->base.handle_result = safexcel_aead_handle_result; 1662 return 0; 1663 } 1664 1665 static int safexcel_aead_sha1_cra_init(struct crypto_tfm *tfm) 1666 { 1667 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1668 1669 safexcel_aead_cra_init(tfm); 1670 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1; 1671 ctx->state_sz = SHA1_DIGEST_SIZE; 1672 return 0; 1673 } 1674 1675 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_aes = { 1676 .type = SAFEXCEL_ALG_TYPE_AEAD, 1677 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA1, 1678 .alg.aead = { 1679 .setkey = safexcel_aead_setkey, 1680 .encrypt = safexcel_aead_encrypt, 1681 .decrypt = safexcel_aead_decrypt, 1682 .ivsize = AES_BLOCK_SIZE, 1683 .maxauthsize = SHA1_DIGEST_SIZE, 1684 .base = { 1685 .cra_name = "authenc(hmac(sha1),cbc(aes))", 1686 .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-aes", 1687 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1688 .cra_flags = CRYPTO_ALG_ASYNC | 1689 CRYPTO_ALG_ALLOCATES_MEMORY | 1690 CRYPTO_ALG_KERN_DRIVER_ONLY, 1691 .cra_blocksize = AES_BLOCK_SIZE, 1692 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1693 .cra_alignmask = 0, 1694 .cra_init = safexcel_aead_sha1_cra_init, 1695 .cra_exit = safexcel_aead_cra_exit, 1696 .cra_module = THIS_MODULE, 1697 }, 1698 }, 1699 }; 1700 1701 static int safexcel_aead_sha256_cra_init(struct crypto_tfm *tfm) 1702 { 1703 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1704 1705 safexcel_aead_cra_init(tfm); 1706 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256; 1707 ctx->state_sz = SHA256_DIGEST_SIZE; 1708 return 0; 1709 } 1710 1711 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_aes = { 1712 .type = SAFEXCEL_ALG_TYPE_AEAD, 1713 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256, 1714 .alg.aead = { 1715 .setkey = safexcel_aead_setkey, 1716 .encrypt = safexcel_aead_encrypt, 1717 .decrypt = safexcel_aead_decrypt, 1718 .ivsize = AES_BLOCK_SIZE, 1719 .maxauthsize = SHA256_DIGEST_SIZE, 1720 .base = { 1721 .cra_name = "authenc(hmac(sha256),cbc(aes))", 1722 .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-aes", 1723 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1724 .cra_flags = CRYPTO_ALG_ASYNC | 1725 CRYPTO_ALG_ALLOCATES_MEMORY | 1726 CRYPTO_ALG_KERN_DRIVER_ONLY, 1727 .cra_blocksize = AES_BLOCK_SIZE, 1728 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1729 .cra_alignmask = 0, 1730 .cra_init = safexcel_aead_sha256_cra_init, 1731 .cra_exit = safexcel_aead_cra_exit, 1732 .cra_module = THIS_MODULE, 1733 }, 1734 }, 1735 }; 1736 1737 static int safexcel_aead_sha224_cra_init(struct crypto_tfm *tfm) 1738 { 1739 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1740 1741 safexcel_aead_cra_init(tfm); 1742 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224; 1743 ctx->state_sz = SHA256_DIGEST_SIZE; 1744 return 0; 1745 } 1746 1747 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_aes = { 1748 .type = SAFEXCEL_ALG_TYPE_AEAD, 1749 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256, 1750 .alg.aead = { 1751 .setkey = safexcel_aead_setkey, 1752 .encrypt = safexcel_aead_encrypt, 1753 .decrypt = safexcel_aead_decrypt, 1754 .ivsize = AES_BLOCK_SIZE, 1755 .maxauthsize = SHA224_DIGEST_SIZE, 1756 .base = { 1757 .cra_name = "authenc(hmac(sha224),cbc(aes))", 1758 .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-aes", 1759 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1760 .cra_flags = CRYPTO_ALG_ASYNC | 1761 CRYPTO_ALG_ALLOCATES_MEMORY | 1762 CRYPTO_ALG_KERN_DRIVER_ONLY, 1763 .cra_blocksize = AES_BLOCK_SIZE, 1764 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1765 .cra_alignmask = 0, 1766 .cra_init = safexcel_aead_sha224_cra_init, 1767 .cra_exit = safexcel_aead_cra_exit, 1768 .cra_module = THIS_MODULE, 1769 }, 1770 }, 1771 }; 1772 1773 static int safexcel_aead_sha512_cra_init(struct crypto_tfm *tfm) 1774 { 1775 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1776 1777 safexcel_aead_cra_init(tfm); 1778 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512; 1779 ctx->state_sz = SHA512_DIGEST_SIZE; 1780 return 0; 1781 } 1782 1783 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_aes = { 1784 .type = SAFEXCEL_ALG_TYPE_AEAD, 1785 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512, 1786 .alg.aead = { 1787 .setkey = safexcel_aead_setkey, 1788 .encrypt = safexcel_aead_encrypt, 1789 .decrypt = safexcel_aead_decrypt, 1790 .ivsize = AES_BLOCK_SIZE, 1791 .maxauthsize = SHA512_DIGEST_SIZE, 1792 .base = { 1793 .cra_name = "authenc(hmac(sha512),cbc(aes))", 1794 .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-aes", 1795 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1796 .cra_flags = CRYPTO_ALG_ASYNC | 1797 CRYPTO_ALG_ALLOCATES_MEMORY | 1798 CRYPTO_ALG_KERN_DRIVER_ONLY, 1799 .cra_blocksize = AES_BLOCK_SIZE, 1800 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1801 .cra_alignmask = 0, 1802 .cra_init = safexcel_aead_sha512_cra_init, 1803 .cra_exit = safexcel_aead_cra_exit, 1804 .cra_module = THIS_MODULE, 1805 }, 1806 }, 1807 }; 1808 1809 static int safexcel_aead_sha384_cra_init(struct crypto_tfm *tfm) 1810 { 1811 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1812 1813 safexcel_aead_cra_init(tfm); 1814 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384; 1815 ctx->state_sz = SHA512_DIGEST_SIZE; 1816 return 0; 1817 } 1818 1819 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_aes = { 1820 .type = SAFEXCEL_ALG_TYPE_AEAD, 1821 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512, 1822 .alg.aead = { 1823 .setkey = safexcel_aead_setkey, 1824 .encrypt = safexcel_aead_encrypt, 1825 .decrypt = safexcel_aead_decrypt, 1826 .ivsize = AES_BLOCK_SIZE, 1827 .maxauthsize = SHA384_DIGEST_SIZE, 1828 .base = { 1829 .cra_name = "authenc(hmac(sha384),cbc(aes))", 1830 .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-aes", 1831 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1832 .cra_flags = CRYPTO_ALG_ASYNC | 1833 CRYPTO_ALG_ALLOCATES_MEMORY | 1834 CRYPTO_ALG_KERN_DRIVER_ONLY, 1835 .cra_blocksize = AES_BLOCK_SIZE, 1836 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1837 .cra_alignmask = 0, 1838 .cra_init = safexcel_aead_sha384_cra_init, 1839 .cra_exit = safexcel_aead_cra_exit, 1840 .cra_module = THIS_MODULE, 1841 }, 1842 }, 1843 }; 1844 1845 static int safexcel_aead_sha1_des3_cra_init(struct crypto_tfm *tfm) 1846 { 1847 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1848 1849 safexcel_aead_sha1_cra_init(tfm); 1850 ctx->alg = SAFEXCEL_3DES; /* override default */ 1851 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 1852 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1853 return 0; 1854 } 1855 1856 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des3_ede = { 1857 .type = SAFEXCEL_ALG_TYPE_AEAD, 1858 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA1, 1859 .alg.aead = { 1860 .setkey = safexcel_aead_setkey, 1861 .encrypt = safexcel_aead_encrypt, 1862 .decrypt = safexcel_aead_decrypt, 1863 .ivsize = DES3_EDE_BLOCK_SIZE, 1864 .maxauthsize = SHA1_DIGEST_SIZE, 1865 .base = { 1866 .cra_name = "authenc(hmac(sha1),cbc(des3_ede))", 1867 .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-des3_ede", 1868 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1869 .cra_flags = CRYPTO_ALG_ASYNC | 1870 CRYPTO_ALG_ALLOCATES_MEMORY | 1871 CRYPTO_ALG_KERN_DRIVER_ONLY, 1872 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1873 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1874 .cra_alignmask = 0, 1875 .cra_init = safexcel_aead_sha1_des3_cra_init, 1876 .cra_exit = safexcel_aead_cra_exit, 1877 .cra_module = THIS_MODULE, 1878 }, 1879 }, 1880 }; 1881 1882 static int safexcel_aead_sha256_des3_cra_init(struct crypto_tfm *tfm) 1883 { 1884 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1885 1886 safexcel_aead_sha256_cra_init(tfm); 1887 ctx->alg = SAFEXCEL_3DES; /* override default */ 1888 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 1889 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1890 return 0; 1891 } 1892 1893 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des3_ede = { 1894 .type = SAFEXCEL_ALG_TYPE_AEAD, 1895 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, 1896 .alg.aead = { 1897 .setkey = safexcel_aead_setkey, 1898 .encrypt = safexcel_aead_encrypt, 1899 .decrypt = safexcel_aead_decrypt, 1900 .ivsize = DES3_EDE_BLOCK_SIZE, 1901 .maxauthsize = SHA256_DIGEST_SIZE, 1902 .base = { 1903 .cra_name = "authenc(hmac(sha256),cbc(des3_ede))", 1904 .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-des3_ede", 1905 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1906 .cra_flags = CRYPTO_ALG_ASYNC | 1907 CRYPTO_ALG_ALLOCATES_MEMORY | 1908 CRYPTO_ALG_KERN_DRIVER_ONLY, 1909 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1910 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1911 .cra_alignmask = 0, 1912 .cra_init = safexcel_aead_sha256_des3_cra_init, 1913 .cra_exit = safexcel_aead_cra_exit, 1914 .cra_module = THIS_MODULE, 1915 }, 1916 }, 1917 }; 1918 1919 static int safexcel_aead_sha224_des3_cra_init(struct crypto_tfm *tfm) 1920 { 1921 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1922 1923 safexcel_aead_sha224_cra_init(tfm); 1924 ctx->alg = SAFEXCEL_3DES; /* override default */ 1925 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 1926 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1927 return 0; 1928 } 1929 1930 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des3_ede = { 1931 .type = SAFEXCEL_ALG_TYPE_AEAD, 1932 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, 1933 .alg.aead = { 1934 .setkey = safexcel_aead_setkey, 1935 .encrypt = safexcel_aead_encrypt, 1936 .decrypt = safexcel_aead_decrypt, 1937 .ivsize = DES3_EDE_BLOCK_SIZE, 1938 .maxauthsize = SHA224_DIGEST_SIZE, 1939 .base = { 1940 .cra_name = "authenc(hmac(sha224),cbc(des3_ede))", 1941 .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-des3_ede", 1942 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1943 .cra_flags = CRYPTO_ALG_ASYNC | 1944 CRYPTO_ALG_ALLOCATES_MEMORY | 1945 CRYPTO_ALG_KERN_DRIVER_ONLY, 1946 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1947 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1948 .cra_alignmask = 0, 1949 .cra_init = safexcel_aead_sha224_des3_cra_init, 1950 .cra_exit = safexcel_aead_cra_exit, 1951 .cra_module = THIS_MODULE, 1952 }, 1953 }, 1954 }; 1955 1956 static int safexcel_aead_sha512_des3_cra_init(struct crypto_tfm *tfm) 1957 { 1958 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1959 1960 safexcel_aead_sha512_cra_init(tfm); 1961 ctx->alg = SAFEXCEL_3DES; /* override default */ 1962 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 1963 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1964 return 0; 1965 } 1966 1967 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des3_ede = { 1968 .type = SAFEXCEL_ALG_TYPE_AEAD, 1969 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, 1970 .alg.aead = { 1971 .setkey = safexcel_aead_setkey, 1972 .encrypt = safexcel_aead_encrypt, 1973 .decrypt = safexcel_aead_decrypt, 1974 .ivsize = DES3_EDE_BLOCK_SIZE, 1975 .maxauthsize = SHA512_DIGEST_SIZE, 1976 .base = { 1977 .cra_name = "authenc(hmac(sha512),cbc(des3_ede))", 1978 .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-des3_ede", 1979 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1980 .cra_flags = CRYPTO_ALG_ASYNC | 1981 CRYPTO_ALG_ALLOCATES_MEMORY | 1982 CRYPTO_ALG_KERN_DRIVER_ONLY, 1983 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1984 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1985 .cra_alignmask = 0, 1986 .cra_init = safexcel_aead_sha512_des3_cra_init, 1987 .cra_exit = safexcel_aead_cra_exit, 1988 .cra_module = THIS_MODULE, 1989 }, 1990 }, 1991 }; 1992 1993 static int safexcel_aead_sha384_des3_cra_init(struct crypto_tfm *tfm) 1994 { 1995 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1996 1997 safexcel_aead_sha384_cra_init(tfm); 1998 ctx->alg = SAFEXCEL_3DES; /* override default */ 1999 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 2000 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2001 return 0; 2002 } 2003 2004 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des3_ede = { 2005 .type = SAFEXCEL_ALG_TYPE_AEAD, 2006 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, 2007 .alg.aead = { 2008 .setkey = safexcel_aead_setkey, 2009 .encrypt = safexcel_aead_encrypt, 2010 .decrypt = safexcel_aead_decrypt, 2011 .ivsize = DES3_EDE_BLOCK_SIZE, 2012 .maxauthsize = SHA384_DIGEST_SIZE, 2013 .base = { 2014 .cra_name = "authenc(hmac(sha384),cbc(des3_ede))", 2015 .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-des3_ede", 2016 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2017 .cra_flags = CRYPTO_ALG_ASYNC | 2018 CRYPTO_ALG_ALLOCATES_MEMORY | 2019 CRYPTO_ALG_KERN_DRIVER_ONLY, 2020 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2021 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2022 .cra_alignmask = 0, 2023 .cra_init = safexcel_aead_sha384_des3_cra_init, 2024 .cra_exit = safexcel_aead_cra_exit, 2025 .cra_module = THIS_MODULE, 2026 }, 2027 }, 2028 }; 2029 2030 static int safexcel_aead_sha1_des_cra_init(struct crypto_tfm *tfm) 2031 { 2032 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2033 2034 safexcel_aead_sha1_cra_init(tfm); 2035 ctx->alg = SAFEXCEL_DES; /* override default */ 2036 ctx->blocksz = DES_BLOCK_SIZE; 2037 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2038 return 0; 2039 } 2040 2041 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des = { 2042 .type = SAFEXCEL_ALG_TYPE_AEAD, 2043 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA1, 2044 .alg.aead = { 2045 .setkey = safexcel_aead_setkey, 2046 .encrypt = safexcel_aead_encrypt, 2047 .decrypt = safexcel_aead_decrypt, 2048 .ivsize = DES_BLOCK_SIZE, 2049 .maxauthsize = SHA1_DIGEST_SIZE, 2050 .base = { 2051 .cra_name = "authenc(hmac(sha1),cbc(des))", 2052 .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-des", 2053 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2054 .cra_flags = CRYPTO_ALG_ASYNC | 2055 CRYPTO_ALG_ALLOCATES_MEMORY | 2056 CRYPTO_ALG_KERN_DRIVER_ONLY, 2057 .cra_blocksize = DES_BLOCK_SIZE, 2058 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2059 .cra_alignmask = 0, 2060 .cra_init = safexcel_aead_sha1_des_cra_init, 2061 .cra_exit = safexcel_aead_cra_exit, 2062 .cra_module = THIS_MODULE, 2063 }, 2064 }, 2065 }; 2066 2067 static int safexcel_aead_sha256_des_cra_init(struct crypto_tfm *tfm) 2068 { 2069 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2070 2071 safexcel_aead_sha256_cra_init(tfm); 2072 ctx->alg = SAFEXCEL_DES; /* override default */ 2073 ctx->blocksz = DES_BLOCK_SIZE; 2074 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2075 return 0; 2076 } 2077 2078 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des = { 2079 .type = SAFEXCEL_ALG_TYPE_AEAD, 2080 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, 2081 .alg.aead = { 2082 .setkey = safexcel_aead_setkey, 2083 .encrypt = safexcel_aead_encrypt, 2084 .decrypt = safexcel_aead_decrypt, 2085 .ivsize = DES_BLOCK_SIZE, 2086 .maxauthsize = SHA256_DIGEST_SIZE, 2087 .base = { 2088 .cra_name = "authenc(hmac(sha256),cbc(des))", 2089 .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-des", 2090 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2091 .cra_flags = CRYPTO_ALG_ASYNC | 2092 CRYPTO_ALG_ALLOCATES_MEMORY | 2093 CRYPTO_ALG_KERN_DRIVER_ONLY, 2094 .cra_blocksize = DES_BLOCK_SIZE, 2095 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2096 .cra_alignmask = 0, 2097 .cra_init = safexcel_aead_sha256_des_cra_init, 2098 .cra_exit = safexcel_aead_cra_exit, 2099 .cra_module = THIS_MODULE, 2100 }, 2101 }, 2102 }; 2103 2104 static int safexcel_aead_sha224_des_cra_init(struct crypto_tfm *tfm) 2105 { 2106 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2107 2108 safexcel_aead_sha224_cra_init(tfm); 2109 ctx->alg = SAFEXCEL_DES; /* override default */ 2110 ctx->blocksz = DES_BLOCK_SIZE; 2111 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2112 return 0; 2113 } 2114 2115 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des = { 2116 .type = SAFEXCEL_ALG_TYPE_AEAD, 2117 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, 2118 .alg.aead = { 2119 .setkey = safexcel_aead_setkey, 2120 .encrypt = safexcel_aead_encrypt, 2121 .decrypt = safexcel_aead_decrypt, 2122 .ivsize = DES_BLOCK_SIZE, 2123 .maxauthsize = SHA224_DIGEST_SIZE, 2124 .base = { 2125 .cra_name = "authenc(hmac(sha224),cbc(des))", 2126 .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-des", 2127 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2128 .cra_flags = CRYPTO_ALG_ASYNC | 2129 CRYPTO_ALG_ALLOCATES_MEMORY | 2130 CRYPTO_ALG_KERN_DRIVER_ONLY, 2131 .cra_blocksize = DES_BLOCK_SIZE, 2132 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2133 .cra_alignmask = 0, 2134 .cra_init = safexcel_aead_sha224_des_cra_init, 2135 .cra_exit = safexcel_aead_cra_exit, 2136 .cra_module = THIS_MODULE, 2137 }, 2138 }, 2139 }; 2140 2141 static int safexcel_aead_sha512_des_cra_init(struct crypto_tfm *tfm) 2142 { 2143 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2144 2145 safexcel_aead_sha512_cra_init(tfm); 2146 ctx->alg = SAFEXCEL_DES; /* override default */ 2147 ctx->blocksz = DES_BLOCK_SIZE; 2148 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2149 return 0; 2150 } 2151 2152 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des = { 2153 .type = SAFEXCEL_ALG_TYPE_AEAD, 2154 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, 2155 .alg.aead = { 2156 .setkey = safexcel_aead_setkey, 2157 .encrypt = safexcel_aead_encrypt, 2158 .decrypt = safexcel_aead_decrypt, 2159 .ivsize = DES_BLOCK_SIZE, 2160 .maxauthsize = SHA512_DIGEST_SIZE, 2161 .base = { 2162 .cra_name = "authenc(hmac(sha512),cbc(des))", 2163 .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-des", 2164 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2165 .cra_flags = CRYPTO_ALG_ASYNC | 2166 CRYPTO_ALG_ALLOCATES_MEMORY | 2167 CRYPTO_ALG_KERN_DRIVER_ONLY, 2168 .cra_blocksize = DES_BLOCK_SIZE, 2169 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2170 .cra_alignmask = 0, 2171 .cra_init = safexcel_aead_sha512_des_cra_init, 2172 .cra_exit = safexcel_aead_cra_exit, 2173 .cra_module = THIS_MODULE, 2174 }, 2175 }, 2176 }; 2177 2178 static int safexcel_aead_sha384_des_cra_init(struct crypto_tfm *tfm) 2179 { 2180 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2181 2182 safexcel_aead_sha384_cra_init(tfm); 2183 ctx->alg = SAFEXCEL_DES; /* override default */ 2184 ctx->blocksz = DES_BLOCK_SIZE; 2185 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2186 return 0; 2187 } 2188 2189 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des = { 2190 .type = SAFEXCEL_ALG_TYPE_AEAD, 2191 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, 2192 .alg.aead = { 2193 .setkey = safexcel_aead_setkey, 2194 .encrypt = safexcel_aead_encrypt, 2195 .decrypt = safexcel_aead_decrypt, 2196 .ivsize = DES_BLOCK_SIZE, 2197 .maxauthsize = SHA384_DIGEST_SIZE, 2198 .base = { 2199 .cra_name = "authenc(hmac(sha384),cbc(des))", 2200 .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-des", 2201 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2202 .cra_flags = CRYPTO_ALG_ASYNC | 2203 CRYPTO_ALG_ALLOCATES_MEMORY | 2204 CRYPTO_ALG_KERN_DRIVER_ONLY, 2205 .cra_blocksize = DES_BLOCK_SIZE, 2206 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2207 .cra_alignmask = 0, 2208 .cra_init = safexcel_aead_sha384_des_cra_init, 2209 .cra_exit = safexcel_aead_cra_exit, 2210 .cra_module = THIS_MODULE, 2211 }, 2212 }, 2213 }; 2214 2215 static int safexcel_aead_sha1_ctr_cra_init(struct crypto_tfm *tfm) 2216 { 2217 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2218 2219 safexcel_aead_sha1_cra_init(tfm); 2220 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2221 return 0; 2222 } 2223 2224 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_ctr_aes = { 2225 .type = SAFEXCEL_ALG_TYPE_AEAD, 2226 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA1, 2227 .alg.aead = { 2228 .setkey = safexcel_aead_setkey, 2229 .encrypt = safexcel_aead_encrypt, 2230 .decrypt = safexcel_aead_decrypt, 2231 .ivsize = CTR_RFC3686_IV_SIZE, 2232 .maxauthsize = SHA1_DIGEST_SIZE, 2233 .base = { 2234 .cra_name = "authenc(hmac(sha1),rfc3686(ctr(aes)))", 2235 .cra_driver_name = "safexcel-authenc-hmac-sha1-ctr-aes", 2236 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2237 .cra_flags = CRYPTO_ALG_ASYNC | 2238 CRYPTO_ALG_ALLOCATES_MEMORY | 2239 CRYPTO_ALG_KERN_DRIVER_ONLY, 2240 .cra_blocksize = 1, 2241 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2242 .cra_alignmask = 0, 2243 .cra_init = safexcel_aead_sha1_ctr_cra_init, 2244 .cra_exit = safexcel_aead_cra_exit, 2245 .cra_module = THIS_MODULE, 2246 }, 2247 }, 2248 }; 2249 2250 static int safexcel_aead_sha256_ctr_cra_init(struct crypto_tfm *tfm) 2251 { 2252 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2253 2254 safexcel_aead_sha256_cra_init(tfm); 2255 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2256 return 0; 2257 } 2258 2259 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_ctr_aes = { 2260 .type = SAFEXCEL_ALG_TYPE_AEAD, 2261 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256, 2262 .alg.aead = { 2263 .setkey = safexcel_aead_setkey, 2264 .encrypt = safexcel_aead_encrypt, 2265 .decrypt = safexcel_aead_decrypt, 2266 .ivsize = CTR_RFC3686_IV_SIZE, 2267 .maxauthsize = SHA256_DIGEST_SIZE, 2268 .base = { 2269 .cra_name = "authenc(hmac(sha256),rfc3686(ctr(aes)))", 2270 .cra_driver_name = "safexcel-authenc-hmac-sha256-ctr-aes", 2271 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2272 .cra_flags = CRYPTO_ALG_ASYNC | 2273 CRYPTO_ALG_ALLOCATES_MEMORY | 2274 CRYPTO_ALG_KERN_DRIVER_ONLY, 2275 .cra_blocksize = 1, 2276 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2277 .cra_alignmask = 0, 2278 .cra_init = safexcel_aead_sha256_ctr_cra_init, 2279 .cra_exit = safexcel_aead_cra_exit, 2280 .cra_module = THIS_MODULE, 2281 }, 2282 }, 2283 }; 2284 2285 static int safexcel_aead_sha224_ctr_cra_init(struct crypto_tfm *tfm) 2286 { 2287 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2288 2289 safexcel_aead_sha224_cra_init(tfm); 2290 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2291 return 0; 2292 } 2293 2294 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_ctr_aes = { 2295 .type = SAFEXCEL_ALG_TYPE_AEAD, 2296 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256, 2297 .alg.aead = { 2298 .setkey = safexcel_aead_setkey, 2299 .encrypt = safexcel_aead_encrypt, 2300 .decrypt = safexcel_aead_decrypt, 2301 .ivsize = CTR_RFC3686_IV_SIZE, 2302 .maxauthsize = SHA224_DIGEST_SIZE, 2303 .base = { 2304 .cra_name = "authenc(hmac(sha224),rfc3686(ctr(aes)))", 2305 .cra_driver_name = "safexcel-authenc-hmac-sha224-ctr-aes", 2306 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2307 .cra_flags = CRYPTO_ALG_ASYNC | 2308 CRYPTO_ALG_ALLOCATES_MEMORY | 2309 CRYPTO_ALG_KERN_DRIVER_ONLY, 2310 .cra_blocksize = 1, 2311 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2312 .cra_alignmask = 0, 2313 .cra_init = safexcel_aead_sha224_ctr_cra_init, 2314 .cra_exit = safexcel_aead_cra_exit, 2315 .cra_module = THIS_MODULE, 2316 }, 2317 }, 2318 }; 2319 2320 static int safexcel_aead_sha512_ctr_cra_init(struct crypto_tfm *tfm) 2321 { 2322 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2323 2324 safexcel_aead_sha512_cra_init(tfm); 2325 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2326 return 0; 2327 } 2328 2329 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_ctr_aes = { 2330 .type = SAFEXCEL_ALG_TYPE_AEAD, 2331 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512, 2332 .alg.aead = { 2333 .setkey = safexcel_aead_setkey, 2334 .encrypt = safexcel_aead_encrypt, 2335 .decrypt = safexcel_aead_decrypt, 2336 .ivsize = CTR_RFC3686_IV_SIZE, 2337 .maxauthsize = SHA512_DIGEST_SIZE, 2338 .base = { 2339 .cra_name = "authenc(hmac(sha512),rfc3686(ctr(aes)))", 2340 .cra_driver_name = "safexcel-authenc-hmac-sha512-ctr-aes", 2341 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2342 .cra_flags = CRYPTO_ALG_ASYNC | 2343 CRYPTO_ALG_ALLOCATES_MEMORY | 2344 CRYPTO_ALG_KERN_DRIVER_ONLY, 2345 .cra_blocksize = 1, 2346 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2347 .cra_alignmask = 0, 2348 .cra_init = safexcel_aead_sha512_ctr_cra_init, 2349 .cra_exit = safexcel_aead_cra_exit, 2350 .cra_module = THIS_MODULE, 2351 }, 2352 }, 2353 }; 2354 2355 static int safexcel_aead_sha384_ctr_cra_init(struct crypto_tfm *tfm) 2356 { 2357 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2358 2359 safexcel_aead_sha384_cra_init(tfm); 2360 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2361 return 0; 2362 } 2363 2364 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_ctr_aes = { 2365 .type = SAFEXCEL_ALG_TYPE_AEAD, 2366 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512, 2367 .alg.aead = { 2368 .setkey = safexcel_aead_setkey, 2369 .encrypt = safexcel_aead_encrypt, 2370 .decrypt = safexcel_aead_decrypt, 2371 .ivsize = CTR_RFC3686_IV_SIZE, 2372 .maxauthsize = SHA384_DIGEST_SIZE, 2373 .base = { 2374 .cra_name = "authenc(hmac(sha384),rfc3686(ctr(aes)))", 2375 .cra_driver_name = "safexcel-authenc-hmac-sha384-ctr-aes", 2376 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2377 .cra_flags = CRYPTO_ALG_ASYNC | 2378 CRYPTO_ALG_ALLOCATES_MEMORY | 2379 CRYPTO_ALG_KERN_DRIVER_ONLY, 2380 .cra_blocksize = 1, 2381 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2382 .cra_alignmask = 0, 2383 .cra_init = safexcel_aead_sha384_ctr_cra_init, 2384 .cra_exit = safexcel_aead_cra_exit, 2385 .cra_module = THIS_MODULE, 2386 }, 2387 }, 2388 }; 2389 2390 static int safexcel_skcipher_aesxts_setkey(struct crypto_skcipher *ctfm, 2391 const u8 *key, unsigned int len) 2392 { 2393 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 2394 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2395 struct safexcel_crypto_priv *priv = ctx->base.priv; 2396 struct crypto_aes_ctx aes; 2397 int ret, i; 2398 unsigned int keylen; 2399 2400 /* Check for illegal XTS keys */ 2401 ret = xts_verify_key(ctfm, key, len); 2402 if (ret) 2403 return ret; 2404 2405 /* Only half of the key data is cipher key */ 2406 keylen = (len >> 1); 2407 ret = aes_expandkey(&aes, key, keylen); 2408 if (ret) 2409 return ret; 2410 2411 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2412 for (i = 0; i < keylen / sizeof(u32); i++) { 2413 if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { 2414 ctx->base.needs_inv = true; 2415 break; 2416 } 2417 } 2418 } 2419 2420 for (i = 0; i < keylen / sizeof(u32); i++) 2421 ctx->key[i] = cpu_to_le32(aes.key_enc[i]); 2422 2423 /* The other half is the tweak key */ 2424 ret = aes_expandkey(&aes, (u8 *)(key + keylen), keylen); 2425 if (ret) 2426 return ret; 2427 2428 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2429 for (i = 0; i < keylen / sizeof(u32); i++) { 2430 if (le32_to_cpu(ctx->key[i + keylen / sizeof(u32)]) != 2431 aes.key_enc[i]) { 2432 ctx->base.needs_inv = true; 2433 break; 2434 } 2435 } 2436 } 2437 2438 for (i = 0; i < keylen / sizeof(u32); i++) 2439 ctx->key[i + keylen / sizeof(u32)] = 2440 cpu_to_le32(aes.key_enc[i]); 2441 2442 ctx->key_len = keylen << 1; 2443 2444 memzero_explicit(&aes, sizeof(aes)); 2445 return 0; 2446 } 2447 2448 static int safexcel_skcipher_aes_xts_cra_init(struct crypto_tfm *tfm) 2449 { 2450 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2451 2452 safexcel_skcipher_cra_init(tfm); 2453 ctx->alg = SAFEXCEL_AES; 2454 ctx->blocksz = AES_BLOCK_SIZE; 2455 ctx->xts = 1; 2456 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XTS; 2457 return 0; 2458 } 2459 2460 static int safexcel_encrypt_xts(struct skcipher_request *req) 2461 { 2462 if (req->cryptlen < XTS_BLOCK_SIZE) 2463 return -EINVAL; 2464 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 2465 SAFEXCEL_ENCRYPT); 2466 } 2467 2468 static int safexcel_decrypt_xts(struct skcipher_request *req) 2469 { 2470 if (req->cryptlen < XTS_BLOCK_SIZE) 2471 return -EINVAL; 2472 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 2473 SAFEXCEL_DECRYPT); 2474 } 2475 2476 struct safexcel_alg_template safexcel_alg_xts_aes = { 2477 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 2478 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_AES_XTS, 2479 .alg.skcipher = { 2480 .setkey = safexcel_skcipher_aesxts_setkey, 2481 .encrypt = safexcel_encrypt_xts, 2482 .decrypt = safexcel_decrypt_xts, 2483 /* XTS actually uses 2 AES keys glued together */ 2484 .min_keysize = AES_MIN_KEY_SIZE * 2, 2485 .max_keysize = AES_MAX_KEY_SIZE * 2, 2486 .ivsize = XTS_BLOCK_SIZE, 2487 .base = { 2488 .cra_name = "xts(aes)", 2489 .cra_driver_name = "safexcel-xts-aes", 2490 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2491 .cra_flags = CRYPTO_ALG_ASYNC | 2492 CRYPTO_ALG_ALLOCATES_MEMORY | 2493 CRYPTO_ALG_KERN_DRIVER_ONLY, 2494 .cra_blocksize = XTS_BLOCK_SIZE, 2495 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2496 .cra_alignmask = 0, 2497 .cra_init = safexcel_skcipher_aes_xts_cra_init, 2498 .cra_exit = safexcel_skcipher_cra_exit, 2499 .cra_module = THIS_MODULE, 2500 }, 2501 }, 2502 }; 2503 2504 static int safexcel_aead_gcm_setkey(struct crypto_aead *ctfm, const u8 *key, 2505 unsigned int len) 2506 { 2507 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 2508 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2509 struct safexcel_crypto_priv *priv = ctx->base.priv; 2510 struct crypto_aes_ctx aes; 2511 u32 hashkey[AES_BLOCK_SIZE >> 2]; 2512 int ret, i; 2513 2514 ret = aes_expandkey(&aes, key, len); 2515 if (ret) { 2516 memzero_explicit(&aes, sizeof(aes)); 2517 return ret; 2518 } 2519 2520 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2521 for (i = 0; i < len / sizeof(u32); i++) { 2522 if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { 2523 ctx->base.needs_inv = true; 2524 break; 2525 } 2526 } 2527 } 2528 2529 for (i = 0; i < len / sizeof(u32); i++) 2530 ctx->key[i] = cpu_to_le32(aes.key_enc[i]); 2531 2532 ctx->key_len = len; 2533 2534 /* Compute hash key by encrypting zeroes with cipher key */ 2535 memset(hashkey, 0, AES_BLOCK_SIZE); 2536 aes_encrypt(&aes, (u8 *)hashkey, (u8 *)hashkey); 2537 2538 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2539 for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) { 2540 if (be32_to_cpu(ctx->base.ipad.be[i]) != hashkey[i]) { 2541 ctx->base.needs_inv = true; 2542 break; 2543 } 2544 } 2545 } 2546 2547 for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) 2548 ctx->base.ipad.be[i] = cpu_to_be32(hashkey[i]); 2549 2550 memzero_explicit(hashkey, AES_BLOCK_SIZE); 2551 memzero_explicit(&aes, sizeof(aes)); 2552 return 0; 2553 } 2554 2555 static int safexcel_aead_gcm_cra_init(struct crypto_tfm *tfm) 2556 { 2557 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2558 2559 safexcel_aead_cra_init(tfm); 2560 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_GHASH; 2561 ctx->state_sz = GHASH_BLOCK_SIZE; 2562 ctx->xcm = EIP197_XCM_MODE_GCM; 2563 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XCM; /* override default */ 2564 2565 return 0; 2566 } 2567 2568 static void safexcel_aead_gcm_cra_exit(struct crypto_tfm *tfm) 2569 { 2570 safexcel_aead_cra_exit(tfm); 2571 } 2572 2573 static int safexcel_aead_gcm_setauthsize(struct crypto_aead *tfm, 2574 unsigned int authsize) 2575 { 2576 return crypto_gcm_check_authsize(authsize); 2577 } 2578 2579 struct safexcel_alg_template safexcel_alg_gcm = { 2580 .type = SAFEXCEL_ALG_TYPE_AEAD, 2581 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH, 2582 .alg.aead = { 2583 .setkey = safexcel_aead_gcm_setkey, 2584 .setauthsize = safexcel_aead_gcm_setauthsize, 2585 .encrypt = safexcel_aead_encrypt, 2586 .decrypt = safexcel_aead_decrypt, 2587 .ivsize = GCM_AES_IV_SIZE, 2588 .maxauthsize = GHASH_DIGEST_SIZE, 2589 .base = { 2590 .cra_name = "gcm(aes)", 2591 .cra_driver_name = "safexcel-gcm-aes", 2592 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2593 .cra_flags = CRYPTO_ALG_ASYNC | 2594 CRYPTO_ALG_ALLOCATES_MEMORY | 2595 CRYPTO_ALG_KERN_DRIVER_ONLY, 2596 .cra_blocksize = 1, 2597 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2598 .cra_alignmask = 0, 2599 .cra_init = safexcel_aead_gcm_cra_init, 2600 .cra_exit = safexcel_aead_gcm_cra_exit, 2601 .cra_module = THIS_MODULE, 2602 }, 2603 }, 2604 }; 2605 2606 static int safexcel_aead_ccm_setkey(struct crypto_aead *ctfm, const u8 *key, 2607 unsigned int len) 2608 { 2609 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 2610 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2611 struct safexcel_crypto_priv *priv = ctx->base.priv; 2612 struct crypto_aes_ctx aes; 2613 int ret, i; 2614 2615 ret = aes_expandkey(&aes, key, len); 2616 if (ret) { 2617 memzero_explicit(&aes, sizeof(aes)); 2618 return ret; 2619 } 2620 2621 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2622 for (i = 0; i < len / sizeof(u32); i++) { 2623 if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { 2624 ctx->base.needs_inv = true; 2625 break; 2626 } 2627 } 2628 } 2629 2630 for (i = 0; i < len / sizeof(u32); i++) { 2631 ctx->key[i] = cpu_to_le32(aes.key_enc[i]); 2632 ctx->base.ipad.be[i + 2 * AES_BLOCK_SIZE / sizeof(u32)] = 2633 cpu_to_be32(aes.key_enc[i]); 2634 } 2635 2636 ctx->key_len = len; 2637 ctx->state_sz = 2 * AES_BLOCK_SIZE + len; 2638 2639 if (len == AES_KEYSIZE_192) 2640 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192; 2641 else if (len == AES_KEYSIZE_256) 2642 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256; 2643 else 2644 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128; 2645 2646 memzero_explicit(&aes, sizeof(aes)); 2647 return 0; 2648 } 2649 2650 static int safexcel_aead_ccm_cra_init(struct crypto_tfm *tfm) 2651 { 2652 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2653 2654 safexcel_aead_cra_init(tfm); 2655 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128; 2656 ctx->state_sz = 3 * AES_BLOCK_SIZE; 2657 ctx->xcm = EIP197_XCM_MODE_CCM; 2658 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XCM; /* override default */ 2659 ctx->ctrinit = 0; 2660 return 0; 2661 } 2662 2663 static int safexcel_aead_ccm_setauthsize(struct crypto_aead *tfm, 2664 unsigned int authsize) 2665 { 2666 /* Borrowed from crypto/ccm.c */ 2667 switch (authsize) { 2668 case 4: 2669 case 6: 2670 case 8: 2671 case 10: 2672 case 12: 2673 case 14: 2674 case 16: 2675 break; 2676 default: 2677 return -EINVAL; 2678 } 2679 2680 return 0; 2681 } 2682 2683 static int safexcel_ccm_encrypt(struct aead_request *req) 2684 { 2685 struct safexcel_cipher_req *creq = aead_request_ctx(req); 2686 2687 if (req->iv[0] < 1 || req->iv[0] > 7) 2688 return -EINVAL; 2689 2690 return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); 2691 } 2692 2693 static int safexcel_ccm_decrypt(struct aead_request *req) 2694 { 2695 struct safexcel_cipher_req *creq = aead_request_ctx(req); 2696 2697 if (req->iv[0] < 1 || req->iv[0] > 7) 2698 return -EINVAL; 2699 2700 return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); 2701 } 2702 2703 struct safexcel_alg_template safexcel_alg_ccm = { 2704 .type = SAFEXCEL_ALG_TYPE_AEAD, 2705 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_CBC_MAC_ALL, 2706 .alg.aead = { 2707 .setkey = safexcel_aead_ccm_setkey, 2708 .setauthsize = safexcel_aead_ccm_setauthsize, 2709 .encrypt = safexcel_ccm_encrypt, 2710 .decrypt = safexcel_ccm_decrypt, 2711 .ivsize = AES_BLOCK_SIZE, 2712 .maxauthsize = AES_BLOCK_SIZE, 2713 .base = { 2714 .cra_name = "ccm(aes)", 2715 .cra_driver_name = "safexcel-ccm-aes", 2716 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2717 .cra_flags = CRYPTO_ALG_ASYNC | 2718 CRYPTO_ALG_ALLOCATES_MEMORY | 2719 CRYPTO_ALG_KERN_DRIVER_ONLY, 2720 .cra_blocksize = 1, 2721 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2722 .cra_alignmask = 0, 2723 .cra_init = safexcel_aead_ccm_cra_init, 2724 .cra_exit = safexcel_aead_cra_exit, 2725 .cra_module = THIS_MODULE, 2726 }, 2727 }, 2728 }; 2729 2730 static void safexcel_chacha20_setkey(struct safexcel_cipher_ctx *ctx, 2731 const u8 *key) 2732 { 2733 struct safexcel_crypto_priv *priv = ctx->base.priv; 2734 2735 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) 2736 if (memcmp(ctx->key, key, CHACHA_KEY_SIZE)) 2737 ctx->base.needs_inv = true; 2738 2739 memcpy(ctx->key, key, CHACHA_KEY_SIZE); 2740 ctx->key_len = CHACHA_KEY_SIZE; 2741 } 2742 2743 static int safexcel_skcipher_chacha20_setkey(struct crypto_skcipher *ctfm, 2744 const u8 *key, unsigned int len) 2745 { 2746 struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm); 2747 2748 if (len != CHACHA_KEY_SIZE) 2749 return -EINVAL; 2750 2751 safexcel_chacha20_setkey(ctx, key); 2752 2753 return 0; 2754 } 2755 2756 static int safexcel_skcipher_chacha20_cra_init(struct crypto_tfm *tfm) 2757 { 2758 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2759 2760 safexcel_skcipher_cra_init(tfm); 2761 ctx->alg = SAFEXCEL_CHACHA20; 2762 ctx->ctrinit = 0; 2763 ctx->mode = CONTEXT_CONTROL_CHACHA20_MODE_256_32; 2764 return 0; 2765 } 2766 2767 struct safexcel_alg_template safexcel_alg_chacha20 = { 2768 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 2769 .algo_mask = SAFEXCEL_ALG_CHACHA20, 2770 .alg.skcipher = { 2771 .setkey = safexcel_skcipher_chacha20_setkey, 2772 .encrypt = safexcel_encrypt, 2773 .decrypt = safexcel_decrypt, 2774 .min_keysize = CHACHA_KEY_SIZE, 2775 .max_keysize = CHACHA_KEY_SIZE, 2776 .ivsize = CHACHA_IV_SIZE, 2777 .base = { 2778 .cra_name = "chacha20", 2779 .cra_driver_name = "safexcel-chacha20", 2780 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2781 .cra_flags = CRYPTO_ALG_ASYNC | 2782 CRYPTO_ALG_ALLOCATES_MEMORY | 2783 CRYPTO_ALG_KERN_DRIVER_ONLY, 2784 .cra_blocksize = 1, 2785 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2786 .cra_alignmask = 0, 2787 .cra_init = safexcel_skcipher_chacha20_cra_init, 2788 .cra_exit = safexcel_skcipher_cra_exit, 2789 .cra_module = THIS_MODULE, 2790 }, 2791 }, 2792 }; 2793 2794 static int safexcel_aead_chachapoly_setkey(struct crypto_aead *ctfm, 2795 const u8 *key, unsigned int len) 2796 { 2797 struct safexcel_cipher_ctx *ctx = crypto_aead_ctx(ctfm); 2798 2799 if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP && 2800 len > EIP197_AEAD_IPSEC_NONCE_SIZE) { 2801 /* ESP variant has nonce appended to key */ 2802 len -= EIP197_AEAD_IPSEC_NONCE_SIZE; 2803 ctx->nonce = *(u32 *)(key + len); 2804 } 2805 if (len != CHACHA_KEY_SIZE) 2806 return -EINVAL; 2807 2808 safexcel_chacha20_setkey(ctx, key); 2809 2810 return 0; 2811 } 2812 2813 static int safexcel_aead_chachapoly_setauthsize(struct crypto_aead *tfm, 2814 unsigned int authsize) 2815 { 2816 if (authsize != POLY1305_DIGEST_SIZE) 2817 return -EINVAL; 2818 return 0; 2819 } 2820 2821 static int safexcel_aead_chachapoly_crypt(struct aead_request *req, 2822 enum safexcel_cipher_direction dir) 2823 { 2824 struct safexcel_cipher_req *creq = aead_request_ctx(req); 2825 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2826 struct crypto_tfm *tfm = crypto_aead_tfm(aead); 2827 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2828 struct aead_request *subreq = aead_request_ctx(req); 2829 u32 key[CHACHA_KEY_SIZE / sizeof(u32) + 1]; 2830 int ret = 0; 2831 2832 /* 2833 * Instead of wasting time detecting umpteen silly corner cases, 2834 * just dump all "small" requests to the fallback implementation. 2835 * HW would not be faster on such small requests anyway. 2836 */ 2837 if (likely((ctx->aead != EIP197_AEAD_TYPE_IPSEC_ESP || 2838 req->assoclen >= EIP197_AEAD_IPSEC_IV_SIZE) && 2839 req->cryptlen > POLY1305_DIGEST_SIZE)) { 2840 return safexcel_queue_req(&req->base, creq, dir); 2841 } 2842 2843 /* HW cannot do full (AAD+payload) zero length, use fallback */ 2844 memcpy(key, ctx->key, CHACHA_KEY_SIZE); 2845 if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { 2846 /* ESP variant has nonce appended to the key */ 2847 key[CHACHA_KEY_SIZE / sizeof(u32)] = ctx->nonce; 2848 ret = crypto_aead_setkey(ctx->fback, (u8 *)key, 2849 CHACHA_KEY_SIZE + 2850 EIP197_AEAD_IPSEC_NONCE_SIZE); 2851 } else { 2852 ret = crypto_aead_setkey(ctx->fback, (u8 *)key, 2853 CHACHA_KEY_SIZE); 2854 } 2855 if (ret) { 2856 crypto_aead_clear_flags(aead, CRYPTO_TFM_REQ_MASK); 2857 crypto_aead_set_flags(aead, crypto_aead_get_flags(ctx->fback) & 2858 CRYPTO_TFM_REQ_MASK); 2859 return ret; 2860 } 2861 2862 aead_request_set_tfm(subreq, ctx->fback); 2863 aead_request_set_callback(subreq, req->base.flags, req->base.complete, 2864 req->base.data); 2865 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, 2866 req->iv); 2867 aead_request_set_ad(subreq, req->assoclen); 2868 2869 return (dir == SAFEXCEL_ENCRYPT) ? 2870 crypto_aead_encrypt(subreq) : 2871 crypto_aead_decrypt(subreq); 2872 } 2873 2874 static int safexcel_aead_chachapoly_encrypt(struct aead_request *req) 2875 { 2876 return safexcel_aead_chachapoly_crypt(req, SAFEXCEL_ENCRYPT); 2877 } 2878 2879 static int safexcel_aead_chachapoly_decrypt(struct aead_request *req) 2880 { 2881 return safexcel_aead_chachapoly_crypt(req, SAFEXCEL_DECRYPT); 2882 } 2883 2884 static int safexcel_aead_fallback_cra_init(struct crypto_tfm *tfm) 2885 { 2886 struct crypto_aead *aead = __crypto_aead_cast(tfm); 2887 struct aead_alg *alg = crypto_aead_alg(aead); 2888 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2889 2890 safexcel_aead_cra_init(tfm); 2891 2892 /* Allocate fallback implementation */ 2893 ctx->fback = crypto_alloc_aead(alg->base.cra_name, 0, 2894 CRYPTO_ALG_ASYNC | 2895 CRYPTO_ALG_NEED_FALLBACK); 2896 if (IS_ERR(ctx->fback)) 2897 return PTR_ERR(ctx->fback); 2898 2899 crypto_aead_set_reqsize(aead, max(sizeof(struct safexcel_cipher_req), 2900 sizeof(struct aead_request) + 2901 crypto_aead_reqsize(ctx->fback))); 2902 2903 return 0; 2904 } 2905 2906 static int safexcel_aead_chachapoly_cra_init(struct crypto_tfm *tfm) 2907 { 2908 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2909 2910 safexcel_aead_fallback_cra_init(tfm); 2911 ctx->alg = SAFEXCEL_CHACHA20; 2912 ctx->mode = CONTEXT_CONTROL_CHACHA20_MODE_256_32 | 2913 CONTEXT_CONTROL_CHACHA20_MODE_CALC_OTK; 2914 ctx->ctrinit = 0; 2915 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_POLY1305; 2916 ctx->state_sz = 0; /* Precomputed by HW */ 2917 return 0; 2918 } 2919 2920 static void safexcel_aead_fallback_cra_exit(struct crypto_tfm *tfm) 2921 { 2922 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2923 2924 crypto_free_aead(ctx->fback); 2925 safexcel_aead_cra_exit(tfm); 2926 } 2927 2928 struct safexcel_alg_template safexcel_alg_chachapoly = { 2929 .type = SAFEXCEL_ALG_TYPE_AEAD, 2930 .algo_mask = SAFEXCEL_ALG_CHACHA20 | SAFEXCEL_ALG_POLY1305, 2931 .alg.aead = { 2932 .setkey = safexcel_aead_chachapoly_setkey, 2933 .setauthsize = safexcel_aead_chachapoly_setauthsize, 2934 .encrypt = safexcel_aead_chachapoly_encrypt, 2935 .decrypt = safexcel_aead_chachapoly_decrypt, 2936 .ivsize = CHACHAPOLY_IV_SIZE, 2937 .maxauthsize = POLY1305_DIGEST_SIZE, 2938 .base = { 2939 .cra_name = "rfc7539(chacha20,poly1305)", 2940 .cra_driver_name = "safexcel-chacha20-poly1305", 2941 /* +1 to put it above HW chacha + SW poly */ 2942 .cra_priority = SAFEXCEL_CRA_PRIORITY + 1, 2943 .cra_flags = CRYPTO_ALG_ASYNC | 2944 CRYPTO_ALG_ALLOCATES_MEMORY | 2945 CRYPTO_ALG_KERN_DRIVER_ONLY | 2946 CRYPTO_ALG_NEED_FALLBACK, 2947 .cra_blocksize = 1, 2948 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2949 .cra_alignmask = 0, 2950 .cra_init = safexcel_aead_chachapoly_cra_init, 2951 .cra_exit = safexcel_aead_fallback_cra_exit, 2952 .cra_module = THIS_MODULE, 2953 }, 2954 }, 2955 }; 2956 2957 static int safexcel_aead_chachapolyesp_cra_init(struct crypto_tfm *tfm) 2958 { 2959 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2960 int ret; 2961 2962 ret = safexcel_aead_chachapoly_cra_init(tfm); 2963 ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP; 2964 ctx->aadskip = EIP197_AEAD_IPSEC_IV_SIZE; 2965 return ret; 2966 } 2967 2968 struct safexcel_alg_template safexcel_alg_chachapoly_esp = { 2969 .type = SAFEXCEL_ALG_TYPE_AEAD, 2970 .algo_mask = SAFEXCEL_ALG_CHACHA20 | SAFEXCEL_ALG_POLY1305, 2971 .alg.aead = { 2972 .setkey = safexcel_aead_chachapoly_setkey, 2973 .setauthsize = safexcel_aead_chachapoly_setauthsize, 2974 .encrypt = safexcel_aead_chachapoly_encrypt, 2975 .decrypt = safexcel_aead_chachapoly_decrypt, 2976 .ivsize = CHACHAPOLY_IV_SIZE - EIP197_AEAD_IPSEC_NONCE_SIZE, 2977 .maxauthsize = POLY1305_DIGEST_SIZE, 2978 .base = { 2979 .cra_name = "rfc7539esp(chacha20,poly1305)", 2980 .cra_driver_name = "safexcel-chacha20-poly1305-esp", 2981 /* +1 to put it above HW chacha + SW poly */ 2982 .cra_priority = SAFEXCEL_CRA_PRIORITY + 1, 2983 .cra_flags = CRYPTO_ALG_ASYNC | 2984 CRYPTO_ALG_ALLOCATES_MEMORY | 2985 CRYPTO_ALG_KERN_DRIVER_ONLY | 2986 CRYPTO_ALG_NEED_FALLBACK, 2987 .cra_blocksize = 1, 2988 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2989 .cra_alignmask = 0, 2990 .cra_init = safexcel_aead_chachapolyesp_cra_init, 2991 .cra_exit = safexcel_aead_fallback_cra_exit, 2992 .cra_module = THIS_MODULE, 2993 }, 2994 }, 2995 }; 2996 2997 static int safexcel_skcipher_sm4_setkey(struct crypto_skcipher *ctfm, 2998 const u8 *key, unsigned int len) 2999 { 3000 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 3001 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3002 struct safexcel_crypto_priv *priv = ctx->base.priv; 3003 3004 if (len != SM4_KEY_SIZE) 3005 return -EINVAL; 3006 3007 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) 3008 if (memcmp(ctx->key, key, SM4_KEY_SIZE)) 3009 ctx->base.needs_inv = true; 3010 3011 memcpy(ctx->key, key, SM4_KEY_SIZE); 3012 ctx->key_len = SM4_KEY_SIZE; 3013 3014 return 0; 3015 } 3016 3017 static int safexcel_sm4_blk_encrypt(struct skcipher_request *req) 3018 { 3019 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3020 if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) 3021 return -EINVAL; 3022 else 3023 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 3024 SAFEXCEL_ENCRYPT); 3025 } 3026 3027 static int safexcel_sm4_blk_decrypt(struct skcipher_request *req) 3028 { 3029 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3030 if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) 3031 return -EINVAL; 3032 else 3033 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 3034 SAFEXCEL_DECRYPT); 3035 } 3036 3037 static int safexcel_skcipher_sm4_ecb_cra_init(struct crypto_tfm *tfm) 3038 { 3039 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3040 3041 safexcel_skcipher_cra_init(tfm); 3042 ctx->alg = SAFEXCEL_SM4; 3043 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; 3044 ctx->blocksz = 0; 3045 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 3046 return 0; 3047 } 3048 3049 struct safexcel_alg_template safexcel_alg_ecb_sm4 = { 3050 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 3051 .algo_mask = SAFEXCEL_ALG_SM4, 3052 .alg.skcipher = { 3053 .setkey = safexcel_skcipher_sm4_setkey, 3054 .encrypt = safexcel_sm4_blk_encrypt, 3055 .decrypt = safexcel_sm4_blk_decrypt, 3056 .min_keysize = SM4_KEY_SIZE, 3057 .max_keysize = SM4_KEY_SIZE, 3058 .base = { 3059 .cra_name = "ecb(sm4)", 3060 .cra_driver_name = "safexcel-ecb-sm4", 3061 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3062 .cra_flags = CRYPTO_ALG_ASYNC | 3063 CRYPTO_ALG_ALLOCATES_MEMORY | 3064 CRYPTO_ALG_KERN_DRIVER_ONLY, 3065 .cra_blocksize = SM4_BLOCK_SIZE, 3066 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3067 .cra_alignmask = 0, 3068 .cra_init = safexcel_skcipher_sm4_ecb_cra_init, 3069 .cra_exit = safexcel_skcipher_cra_exit, 3070 .cra_module = THIS_MODULE, 3071 }, 3072 }, 3073 }; 3074 3075 static int safexcel_skcipher_sm4_cbc_cra_init(struct crypto_tfm *tfm) 3076 { 3077 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3078 3079 safexcel_skcipher_cra_init(tfm); 3080 ctx->alg = SAFEXCEL_SM4; 3081 ctx->blocksz = SM4_BLOCK_SIZE; 3082 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; 3083 return 0; 3084 } 3085 3086 struct safexcel_alg_template safexcel_alg_cbc_sm4 = { 3087 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 3088 .algo_mask = SAFEXCEL_ALG_SM4, 3089 .alg.skcipher = { 3090 .setkey = safexcel_skcipher_sm4_setkey, 3091 .encrypt = safexcel_sm4_blk_encrypt, 3092 .decrypt = safexcel_sm4_blk_decrypt, 3093 .min_keysize = SM4_KEY_SIZE, 3094 .max_keysize = SM4_KEY_SIZE, 3095 .ivsize = SM4_BLOCK_SIZE, 3096 .base = { 3097 .cra_name = "cbc(sm4)", 3098 .cra_driver_name = "safexcel-cbc-sm4", 3099 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3100 .cra_flags = CRYPTO_ALG_ASYNC | 3101 CRYPTO_ALG_ALLOCATES_MEMORY | 3102 CRYPTO_ALG_KERN_DRIVER_ONLY, 3103 .cra_blocksize = SM4_BLOCK_SIZE, 3104 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3105 .cra_alignmask = 0, 3106 .cra_init = safexcel_skcipher_sm4_cbc_cra_init, 3107 .cra_exit = safexcel_skcipher_cra_exit, 3108 .cra_module = THIS_MODULE, 3109 }, 3110 }, 3111 }; 3112 3113 static int safexcel_skcipher_sm4ctr_setkey(struct crypto_skcipher *ctfm, 3114 const u8 *key, unsigned int len) 3115 { 3116 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 3117 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3118 3119 /* last 4 bytes of key are the nonce! */ 3120 ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE); 3121 /* exclude the nonce here */ 3122 len -= CTR_RFC3686_NONCE_SIZE; 3123 3124 return safexcel_skcipher_sm4_setkey(ctfm, key, len); 3125 } 3126 3127 static int safexcel_skcipher_sm4_ctr_cra_init(struct crypto_tfm *tfm) 3128 { 3129 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3130 3131 safexcel_skcipher_cra_init(tfm); 3132 ctx->alg = SAFEXCEL_SM4; 3133 ctx->blocksz = SM4_BLOCK_SIZE; 3134 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; 3135 return 0; 3136 } 3137 3138 struct safexcel_alg_template safexcel_alg_ctr_sm4 = { 3139 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 3140 .algo_mask = SAFEXCEL_ALG_SM4, 3141 .alg.skcipher = { 3142 .setkey = safexcel_skcipher_sm4ctr_setkey, 3143 .encrypt = safexcel_encrypt, 3144 .decrypt = safexcel_decrypt, 3145 /* Add nonce size */ 3146 .min_keysize = SM4_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 3147 .max_keysize = SM4_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 3148 .ivsize = CTR_RFC3686_IV_SIZE, 3149 .base = { 3150 .cra_name = "rfc3686(ctr(sm4))", 3151 .cra_driver_name = "safexcel-ctr-sm4", 3152 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3153 .cra_flags = CRYPTO_ALG_ASYNC | 3154 CRYPTO_ALG_ALLOCATES_MEMORY | 3155 CRYPTO_ALG_KERN_DRIVER_ONLY, 3156 .cra_blocksize = 1, 3157 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3158 .cra_alignmask = 0, 3159 .cra_init = safexcel_skcipher_sm4_ctr_cra_init, 3160 .cra_exit = safexcel_skcipher_cra_exit, 3161 .cra_module = THIS_MODULE, 3162 }, 3163 }, 3164 }; 3165 3166 static int safexcel_aead_sm4_blk_encrypt(struct aead_request *req) 3167 { 3168 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3169 if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) 3170 return -EINVAL; 3171 3172 return safexcel_queue_req(&req->base, aead_request_ctx(req), 3173 SAFEXCEL_ENCRYPT); 3174 } 3175 3176 static int safexcel_aead_sm4_blk_decrypt(struct aead_request *req) 3177 { 3178 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3179 3180 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3181 if ((req->cryptlen - crypto_aead_authsize(tfm)) & (SM4_BLOCK_SIZE - 1)) 3182 return -EINVAL; 3183 3184 return safexcel_queue_req(&req->base, aead_request_ctx(req), 3185 SAFEXCEL_DECRYPT); 3186 } 3187 3188 static int safexcel_aead_sm4cbc_sha1_cra_init(struct crypto_tfm *tfm) 3189 { 3190 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3191 3192 safexcel_aead_cra_init(tfm); 3193 ctx->alg = SAFEXCEL_SM4; 3194 ctx->blocksz = SM4_BLOCK_SIZE; 3195 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1; 3196 ctx->state_sz = SHA1_DIGEST_SIZE; 3197 return 0; 3198 } 3199 3200 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_sm4 = { 3201 .type = SAFEXCEL_ALG_TYPE_AEAD, 3202 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SHA1, 3203 .alg.aead = { 3204 .setkey = safexcel_aead_setkey, 3205 .encrypt = safexcel_aead_sm4_blk_encrypt, 3206 .decrypt = safexcel_aead_sm4_blk_decrypt, 3207 .ivsize = SM4_BLOCK_SIZE, 3208 .maxauthsize = SHA1_DIGEST_SIZE, 3209 .base = { 3210 .cra_name = "authenc(hmac(sha1),cbc(sm4))", 3211 .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-sm4", 3212 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3213 .cra_flags = CRYPTO_ALG_ASYNC | 3214 CRYPTO_ALG_ALLOCATES_MEMORY | 3215 CRYPTO_ALG_KERN_DRIVER_ONLY, 3216 .cra_blocksize = SM4_BLOCK_SIZE, 3217 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3218 .cra_alignmask = 0, 3219 .cra_init = safexcel_aead_sm4cbc_sha1_cra_init, 3220 .cra_exit = safexcel_aead_cra_exit, 3221 .cra_module = THIS_MODULE, 3222 }, 3223 }, 3224 }; 3225 3226 static int safexcel_aead_fallback_setkey(struct crypto_aead *ctfm, 3227 const u8 *key, unsigned int len) 3228 { 3229 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 3230 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3231 3232 /* Keep fallback cipher synchronized */ 3233 return crypto_aead_setkey(ctx->fback, (u8 *)key, len) ?: 3234 safexcel_aead_setkey(ctfm, key, len); 3235 } 3236 3237 static int safexcel_aead_fallback_setauthsize(struct crypto_aead *ctfm, 3238 unsigned int authsize) 3239 { 3240 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 3241 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3242 3243 /* Keep fallback cipher synchronized */ 3244 return crypto_aead_setauthsize(ctx->fback, authsize); 3245 } 3246 3247 static int safexcel_aead_fallback_crypt(struct aead_request *req, 3248 enum safexcel_cipher_direction dir) 3249 { 3250 struct crypto_aead *aead = crypto_aead_reqtfm(req); 3251 struct crypto_tfm *tfm = crypto_aead_tfm(aead); 3252 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3253 struct aead_request *subreq = aead_request_ctx(req); 3254 3255 aead_request_set_tfm(subreq, ctx->fback); 3256 aead_request_set_callback(subreq, req->base.flags, req->base.complete, 3257 req->base.data); 3258 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, 3259 req->iv); 3260 aead_request_set_ad(subreq, req->assoclen); 3261 3262 return (dir == SAFEXCEL_ENCRYPT) ? 3263 crypto_aead_encrypt(subreq) : 3264 crypto_aead_decrypt(subreq); 3265 } 3266 3267 static int safexcel_aead_sm4cbc_sm3_encrypt(struct aead_request *req) 3268 { 3269 struct safexcel_cipher_req *creq = aead_request_ctx(req); 3270 3271 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3272 if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) 3273 return -EINVAL; 3274 else if (req->cryptlen || req->assoclen) /* If input length > 0 only */ 3275 return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); 3276 3277 /* HW cannot do full (AAD+payload) zero length, use fallback */ 3278 return safexcel_aead_fallback_crypt(req, SAFEXCEL_ENCRYPT); 3279 } 3280 3281 static int safexcel_aead_sm4cbc_sm3_decrypt(struct aead_request *req) 3282 { 3283 struct safexcel_cipher_req *creq = aead_request_ctx(req); 3284 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3285 3286 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3287 if ((req->cryptlen - crypto_aead_authsize(tfm)) & (SM4_BLOCK_SIZE - 1)) 3288 return -EINVAL; 3289 else if (req->cryptlen > crypto_aead_authsize(tfm) || req->assoclen) 3290 /* If input length > 0 only */ 3291 return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); 3292 3293 /* HW cannot do full (AAD+payload) zero length, use fallback */ 3294 return safexcel_aead_fallback_crypt(req, SAFEXCEL_DECRYPT); 3295 } 3296 3297 static int safexcel_aead_sm4cbc_sm3_cra_init(struct crypto_tfm *tfm) 3298 { 3299 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3300 3301 safexcel_aead_fallback_cra_init(tfm); 3302 ctx->alg = SAFEXCEL_SM4; 3303 ctx->blocksz = SM4_BLOCK_SIZE; 3304 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3; 3305 ctx->state_sz = SM3_DIGEST_SIZE; 3306 return 0; 3307 } 3308 3309 struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_cbc_sm4 = { 3310 .type = SAFEXCEL_ALG_TYPE_AEAD, 3311 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SM3, 3312 .alg.aead = { 3313 .setkey = safexcel_aead_fallback_setkey, 3314 .setauthsize = safexcel_aead_fallback_setauthsize, 3315 .encrypt = safexcel_aead_sm4cbc_sm3_encrypt, 3316 .decrypt = safexcel_aead_sm4cbc_sm3_decrypt, 3317 .ivsize = SM4_BLOCK_SIZE, 3318 .maxauthsize = SM3_DIGEST_SIZE, 3319 .base = { 3320 .cra_name = "authenc(hmac(sm3),cbc(sm4))", 3321 .cra_driver_name = "safexcel-authenc-hmac-sm3-cbc-sm4", 3322 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3323 .cra_flags = CRYPTO_ALG_ASYNC | 3324 CRYPTO_ALG_ALLOCATES_MEMORY | 3325 CRYPTO_ALG_KERN_DRIVER_ONLY | 3326 CRYPTO_ALG_NEED_FALLBACK, 3327 .cra_blocksize = SM4_BLOCK_SIZE, 3328 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3329 .cra_alignmask = 0, 3330 .cra_init = safexcel_aead_sm4cbc_sm3_cra_init, 3331 .cra_exit = safexcel_aead_fallback_cra_exit, 3332 .cra_module = THIS_MODULE, 3333 }, 3334 }, 3335 }; 3336 3337 static int safexcel_aead_sm4ctr_sha1_cra_init(struct crypto_tfm *tfm) 3338 { 3339 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3340 3341 safexcel_aead_sm4cbc_sha1_cra_init(tfm); 3342 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; 3343 return 0; 3344 } 3345 3346 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_ctr_sm4 = { 3347 .type = SAFEXCEL_ALG_TYPE_AEAD, 3348 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SHA1, 3349 .alg.aead = { 3350 .setkey = safexcel_aead_setkey, 3351 .encrypt = safexcel_aead_encrypt, 3352 .decrypt = safexcel_aead_decrypt, 3353 .ivsize = CTR_RFC3686_IV_SIZE, 3354 .maxauthsize = SHA1_DIGEST_SIZE, 3355 .base = { 3356 .cra_name = "authenc(hmac(sha1),rfc3686(ctr(sm4)))", 3357 .cra_driver_name = "safexcel-authenc-hmac-sha1-ctr-sm4", 3358 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3359 .cra_flags = CRYPTO_ALG_ASYNC | 3360 CRYPTO_ALG_ALLOCATES_MEMORY | 3361 CRYPTO_ALG_KERN_DRIVER_ONLY, 3362 .cra_blocksize = 1, 3363 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3364 .cra_alignmask = 0, 3365 .cra_init = safexcel_aead_sm4ctr_sha1_cra_init, 3366 .cra_exit = safexcel_aead_cra_exit, 3367 .cra_module = THIS_MODULE, 3368 }, 3369 }, 3370 }; 3371 3372 static int safexcel_aead_sm4ctr_sm3_cra_init(struct crypto_tfm *tfm) 3373 { 3374 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3375 3376 safexcel_aead_sm4cbc_sm3_cra_init(tfm); 3377 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; 3378 return 0; 3379 } 3380 3381 struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_ctr_sm4 = { 3382 .type = SAFEXCEL_ALG_TYPE_AEAD, 3383 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SM3, 3384 .alg.aead = { 3385 .setkey = safexcel_aead_setkey, 3386 .encrypt = safexcel_aead_encrypt, 3387 .decrypt = safexcel_aead_decrypt, 3388 .ivsize = CTR_RFC3686_IV_SIZE, 3389 .maxauthsize = SM3_DIGEST_SIZE, 3390 .base = { 3391 .cra_name = "authenc(hmac(sm3),rfc3686(ctr(sm4)))", 3392 .cra_driver_name = "safexcel-authenc-hmac-sm3-ctr-sm4", 3393 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3394 .cra_flags = CRYPTO_ALG_ASYNC | 3395 CRYPTO_ALG_ALLOCATES_MEMORY | 3396 CRYPTO_ALG_KERN_DRIVER_ONLY, 3397 .cra_blocksize = 1, 3398 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3399 .cra_alignmask = 0, 3400 .cra_init = safexcel_aead_sm4ctr_sm3_cra_init, 3401 .cra_exit = safexcel_aead_cra_exit, 3402 .cra_module = THIS_MODULE, 3403 }, 3404 }, 3405 }; 3406 3407 static int safexcel_rfc4106_gcm_setkey(struct crypto_aead *ctfm, const u8 *key, 3408 unsigned int len) 3409 { 3410 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 3411 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3412 3413 /* last 4 bytes of key are the nonce! */ 3414 ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE); 3415 3416 len -= CTR_RFC3686_NONCE_SIZE; 3417 return safexcel_aead_gcm_setkey(ctfm, key, len); 3418 } 3419 3420 static int safexcel_rfc4106_gcm_setauthsize(struct crypto_aead *tfm, 3421 unsigned int authsize) 3422 { 3423 return crypto_rfc4106_check_authsize(authsize); 3424 } 3425 3426 static int safexcel_rfc4106_encrypt(struct aead_request *req) 3427 { 3428 return crypto_ipsec_check_assoclen(req->assoclen) ?: 3429 safexcel_aead_encrypt(req); 3430 } 3431 3432 static int safexcel_rfc4106_decrypt(struct aead_request *req) 3433 { 3434 return crypto_ipsec_check_assoclen(req->assoclen) ?: 3435 safexcel_aead_decrypt(req); 3436 } 3437 3438 static int safexcel_rfc4106_gcm_cra_init(struct crypto_tfm *tfm) 3439 { 3440 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3441 int ret; 3442 3443 ret = safexcel_aead_gcm_cra_init(tfm); 3444 ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP; 3445 ctx->aadskip = EIP197_AEAD_IPSEC_IV_SIZE; 3446 return ret; 3447 } 3448 3449 struct safexcel_alg_template safexcel_alg_rfc4106_gcm = { 3450 .type = SAFEXCEL_ALG_TYPE_AEAD, 3451 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH, 3452 .alg.aead = { 3453 .setkey = safexcel_rfc4106_gcm_setkey, 3454 .setauthsize = safexcel_rfc4106_gcm_setauthsize, 3455 .encrypt = safexcel_rfc4106_encrypt, 3456 .decrypt = safexcel_rfc4106_decrypt, 3457 .ivsize = GCM_RFC4106_IV_SIZE, 3458 .maxauthsize = GHASH_DIGEST_SIZE, 3459 .base = { 3460 .cra_name = "rfc4106(gcm(aes))", 3461 .cra_driver_name = "safexcel-rfc4106-gcm-aes", 3462 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3463 .cra_flags = CRYPTO_ALG_ASYNC | 3464 CRYPTO_ALG_ALLOCATES_MEMORY | 3465 CRYPTO_ALG_KERN_DRIVER_ONLY, 3466 .cra_blocksize = 1, 3467 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3468 .cra_alignmask = 0, 3469 .cra_init = safexcel_rfc4106_gcm_cra_init, 3470 .cra_exit = safexcel_aead_gcm_cra_exit, 3471 }, 3472 }, 3473 }; 3474 3475 static int safexcel_rfc4543_gcm_setauthsize(struct crypto_aead *tfm, 3476 unsigned int authsize) 3477 { 3478 if (authsize != GHASH_DIGEST_SIZE) 3479 return -EINVAL; 3480 3481 return 0; 3482 } 3483 3484 static int safexcel_rfc4543_gcm_cra_init(struct crypto_tfm *tfm) 3485 { 3486 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3487 int ret; 3488 3489 ret = safexcel_aead_gcm_cra_init(tfm); 3490 ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP_GMAC; 3491 return ret; 3492 } 3493 3494 struct safexcel_alg_template safexcel_alg_rfc4543_gcm = { 3495 .type = SAFEXCEL_ALG_TYPE_AEAD, 3496 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH, 3497 .alg.aead = { 3498 .setkey = safexcel_rfc4106_gcm_setkey, 3499 .setauthsize = safexcel_rfc4543_gcm_setauthsize, 3500 .encrypt = safexcel_rfc4106_encrypt, 3501 .decrypt = safexcel_rfc4106_decrypt, 3502 .ivsize = GCM_RFC4543_IV_SIZE, 3503 .maxauthsize = GHASH_DIGEST_SIZE, 3504 .base = { 3505 .cra_name = "rfc4543(gcm(aes))", 3506 .cra_driver_name = "safexcel-rfc4543-gcm-aes", 3507 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3508 .cra_flags = CRYPTO_ALG_ASYNC | 3509 CRYPTO_ALG_ALLOCATES_MEMORY | 3510 CRYPTO_ALG_KERN_DRIVER_ONLY, 3511 .cra_blocksize = 1, 3512 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3513 .cra_alignmask = 0, 3514 .cra_init = safexcel_rfc4543_gcm_cra_init, 3515 .cra_exit = safexcel_aead_gcm_cra_exit, 3516 }, 3517 }, 3518 }; 3519 3520 static int safexcel_rfc4309_ccm_setkey(struct crypto_aead *ctfm, const u8 *key, 3521 unsigned int len) 3522 { 3523 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 3524 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3525 3526 /* First byte of the nonce = L = always 3 for RFC4309 (4 byte ctr) */ 3527 *(u8 *)&ctx->nonce = EIP197_AEAD_IPSEC_COUNTER_SIZE - 1; 3528 /* last 3 bytes of key are the nonce! */ 3529 memcpy((u8 *)&ctx->nonce + 1, key + len - 3530 EIP197_AEAD_IPSEC_CCM_NONCE_SIZE, 3531 EIP197_AEAD_IPSEC_CCM_NONCE_SIZE); 3532 3533 len -= EIP197_AEAD_IPSEC_CCM_NONCE_SIZE; 3534 return safexcel_aead_ccm_setkey(ctfm, key, len); 3535 } 3536 3537 static int safexcel_rfc4309_ccm_setauthsize(struct crypto_aead *tfm, 3538 unsigned int authsize) 3539 { 3540 /* Borrowed from crypto/ccm.c */ 3541 switch (authsize) { 3542 case 8: 3543 case 12: 3544 case 16: 3545 break; 3546 default: 3547 return -EINVAL; 3548 } 3549 3550 return 0; 3551 } 3552 3553 static int safexcel_rfc4309_ccm_encrypt(struct aead_request *req) 3554 { 3555 struct safexcel_cipher_req *creq = aead_request_ctx(req); 3556 3557 /* Borrowed from crypto/ccm.c */ 3558 if (req->assoclen != 16 && req->assoclen != 20) 3559 return -EINVAL; 3560 3561 return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); 3562 } 3563 3564 static int safexcel_rfc4309_ccm_decrypt(struct aead_request *req) 3565 { 3566 struct safexcel_cipher_req *creq = aead_request_ctx(req); 3567 3568 /* Borrowed from crypto/ccm.c */ 3569 if (req->assoclen != 16 && req->assoclen != 20) 3570 return -EINVAL; 3571 3572 return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); 3573 } 3574 3575 static int safexcel_rfc4309_ccm_cra_init(struct crypto_tfm *tfm) 3576 { 3577 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3578 int ret; 3579 3580 ret = safexcel_aead_ccm_cra_init(tfm); 3581 ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP; 3582 ctx->aadskip = EIP197_AEAD_IPSEC_IV_SIZE; 3583 return ret; 3584 } 3585 3586 struct safexcel_alg_template safexcel_alg_rfc4309_ccm = { 3587 .type = SAFEXCEL_ALG_TYPE_AEAD, 3588 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_CBC_MAC_ALL, 3589 .alg.aead = { 3590 .setkey = safexcel_rfc4309_ccm_setkey, 3591 .setauthsize = safexcel_rfc4309_ccm_setauthsize, 3592 .encrypt = safexcel_rfc4309_ccm_encrypt, 3593 .decrypt = safexcel_rfc4309_ccm_decrypt, 3594 .ivsize = EIP197_AEAD_IPSEC_IV_SIZE, 3595 .maxauthsize = AES_BLOCK_SIZE, 3596 .base = { 3597 .cra_name = "rfc4309(ccm(aes))", 3598 .cra_driver_name = "safexcel-rfc4309-ccm-aes", 3599 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3600 .cra_flags = CRYPTO_ALG_ASYNC | 3601 CRYPTO_ALG_ALLOCATES_MEMORY | 3602 CRYPTO_ALG_KERN_DRIVER_ONLY, 3603 .cra_blocksize = 1, 3604 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3605 .cra_alignmask = 0, 3606 .cra_init = safexcel_rfc4309_ccm_cra_init, 3607 .cra_exit = safexcel_aead_cra_exit, 3608 .cra_module = THIS_MODULE, 3609 }, 3610 }, 3611 }; 3612