1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2017 Marvell 4 * 5 * Antoine Tenart <antoine.tenart@free-electrons.com> 6 */ 7 8 #include <linux/unaligned.h> 9 #include <linux/device.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/dmapool.h> 12 #include <crypto/aead.h> 13 #include <crypto/aes.h> 14 #include <crypto/authenc.h> 15 #include <crypto/chacha.h> 16 #include <crypto/ctr.h> 17 #include <crypto/internal/des.h> 18 #include <crypto/gcm.h> 19 #include <crypto/ghash.h> 20 #include <crypto/poly1305.h> 21 #include <crypto/sha1.h> 22 #include <crypto/sha2.h> 23 #include <crypto/sm3.h> 24 #include <crypto/sm4.h> 25 #include <crypto/xts.h> 26 #include <crypto/skcipher.h> 27 #include <crypto/internal/aead.h> 28 #include <crypto/internal/skcipher.h> 29 30 #include "safexcel.h" 31 32 enum safexcel_cipher_direction { 33 SAFEXCEL_ENCRYPT, 34 SAFEXCEL_DECRYPT, 35 }; 36 37 enum safexcel_cipher_alg { 38 SAFEXCEL_DES, 39 SAFEXCEL_3DES, 40 SAFEXCEL_AES, 41 SAFEXCEL_CHACHA20, 42 SAFEXCEL_SM4, 43 }; 44 45 struct safexcel_cipher_ctx { 46 struct safexcel_context base; 47 struct safexcel_crypto_priv *priv; 48 49 u32 mode; 50 enum safexcel_cipher_alg alg; 51 u8 aead; /* !=0=AEAD, 2=IPSec ESP AEAD, 3=IPsec ESP GMAC */ 52 u8 xcm; /* 0=authenc, 1=GCM, 2 reserved for CCM */ 53 u8 aadskip; 54 u8 blocksz; 55 u32 ivmask; 56 u32 ctrinit; 57 58 __le32 key[16]; 59 u32 nonce; 60 unsigned int key_len, xts; 61 62 /* All the below is AEAD specific */ 63 u32 hash_alg; 64 u32 state_sz; 65 66 struct crypto_aead *fback; 67 }; 68 69 struct safexcel_cipher_req { 70 enum safexcel_cipher_direction direction; 71 /* Number of result descriptors associated to the request */ 72 unsigned int rdescs; 73 bool needs_inv; 74 int nr_src, nr_dst; 75 }; 76 77 static int safexcel_skcipher_iv(struct safexcel_cipher_ctx *ctx, u8 *iv, 78 struct safexcel_command_desc *cdesc) 79 { 80 if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD) { 81 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; 82 /* 32 bit nonce */ 83 cdesc->control_data.token[0] = ctx->nonce; 84 /* 64 bit IV part */ 85 memcpy(&cdesc->control_data.token[1], iv, 8); 86 /* 32 bit counter, start at 0 or 1 (big endian!) */ 87 cdesc->control_data.token[3] = 88 (__force u32)cpu_to_be32(ctx->ctrinit); 89 return 4; 90 } 91 if (ctx->alg == SAFEXCEL_CHACHA20) { 92 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; 93 /* 96 bit nonce part */ 94 memcpy(&cdesc->control_data.token[0], &iv[4], 12); 95 /* 32 bit counter */ 96 cdesc->control_data.token[3] = *(u32 *)iv; 97 return 4; 98 } 99 100 cdesc->control_data.options |= ctx->ivmask; 101 memcpy(cdesc->control_data.token, iv, ctx->blocksz); 102 return ctx->blocksz / sizeof(u32); 103 } 104 105 static void safexcel_skcipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv, 106 struct safexcel_command_desc *cdesc, 107 struct safexcel_token *atoken, 108 u32 length) 109 { 110 struct safexcel_token *token; 111 int ivlen; 112 113 ivlen = safexcel_skcipher_iv(ctx, iv, cdesc); 114 if (ivlen == 4) { 115 /* No space in cdesc, instruction moves to atoken */ 116 cdesc->additional_cdata_size = 1; 117 token = atoken; 118 } else { 119 /* Everything fits in cdesc */ 120 token = (struct safexcel_token *)(cdesc->control_data.token + 2); 121 /* Need to pad with NOP */ 122 eip197_noop_token(&token[1]); 123 } 124 125 token->opcode = EIP197_TOKEN_OPCODE_DIRECTION; 126 token->packet_length = length; 127 token->stat = EIP197_TOKEN_STAT_LAST_PACKET | 128 EIP197_TOKEN_STAT_LAST_HASH; 129 token->instructions = EIP197_TOKEN_INS_LAST | 130 EIP197_TOKEN_INS_TYPE_CRYPTO | 131 EIP197_TOKEN_INS_TYPE_OUTPUT; 132 } 133 134 static void safexcel_aead_iv(struct safexcel_cipher_ctx *ctx, u8 *iv, 135 struct safexcel_command_desc *cdesc) 136 { 137 if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD || 138 ctx->aead & EIP197_AEAD_TYPE_IPSEC_ESP) { /* _ESP and _ESP_GMAC */ 139 /* 32 bit nonce */ 140 cdesc->control_data.token[0] = ctx->nonce; 141 /* 64 bit IV part */ 142 memcpy(&cdesc->control_data.token[1], iv, 8); 143 /* 32 bit counter, start at 0 or 1 (big endian!) */ 144 cdesc->control_data.token[3] = 145 (__force u32)cpu_to_be32(ctx->ctrinit); 146 return; 147 } 148 if (ctx->xcm == EIP197_XCM_MODE_GCM || ctx->alg == SAFEXCEL_CHACHA20) { 149 /* 96 bit IV part */ 150 memcpy(&cdesc->control_data.token[0], iv, 12); 151 /* 32 bit counter, start at 0 or 1 (big endian!) */ 152 cdesc->control_data.token[3] = 153 (__force u32)cpu_to_be32(ctx->ctrinit); 154 return; 155 } 156 /* CBC */ 157 memcpy(cdesc->control_data.token, iv, ctx->blocksz); 158 } 159 160 static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv, 161 struct safexcel_command_desc *cdesc, 162 struct safexcel_token *atoken, 163 enum safexcel_cipher_direction direction, 164 u32 cryptlen, u32 assoclen, u32 digestsize) 165 { 166 struct safexcel_token *aadref; 167 int atoksize = 2; /* Start with minimum size */ 168 int assocadj = assoclen - ctx->aadskip, aadalign; 169 170 /* Always 4 dwords of embedded IV for AEAD modes */ 171 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; 172 173 if (direction == SAFEXCEL_DECRYPT) 174 cryptlen -= digestsize; 175 176 if (unlikely(ctx->xcm == EIP197_XCM_MODE_CCM)) { 177 /* Construct IV block B0 for the CBC-MAC */ 178 u8 *final_iv = (u8 *)cdesc->control_data.token; 179 u8 *cbcmaciv = (u8 *)&atoken[1]; 180 __le32 *aadlen = (__le32 *)&atoken[5]; 181 182 if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { 183 /* Length + nonce */ 184 cdesc->control_data.token[0] = ctx->nonce; 185 /* Fixup flags byte */ 186 *(__le32 *)cbcmaciv = 187 cpu_to_le32(ctx->nonce | 188 ((assocadj > 0) << 6) | 189 ((digestsize - 2) << 2)); 190 /* 64 bit IV part */ 191 memcpy(&cdesc->control_data.token[1], iv, 8); 192 memcpy(cbcmaciv + 4, iv, 8); 193 /* Start counter at 0 */ 194 cdesc->control_data.token[3] = 0; 195 /* Message length */ 196 *(__be32 *)(cbcmaciv + 12) = cpu_to_be32(cryptlen); 197 } else { 198 /* Variable length IV part */ 199 memcpy(final_iv, iv, 15 - iv[0]); 200 memcpy(cbcmaciv, iv, 15 - iv[0]); 201 /* Start variable length counter at 0 */ 202 memset(final_iv + 15 - iv[0], 0, iv[0] + 1); 203 memset(cbcmaciv + 15 - iv[0], 0, iv[0] - 1); 204 /* fixup flags byte */ 205 cbcmaciv[0] |= ((assocadj > 0) << 6) | 206 ((digestsize - 2) << 2); 207 /* insert lower 2 bytes of message length */ 208 cbcmaciv[14] = cryptlen >> 8; 209 cbcmaciv[15] = cryptlen & 255; 210 } 211 212 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 213 atoken->packet_length = AES_BLOCK_SIZE + 214 ((assocadj > 0) << 1); 215 atoken->stat = 0; 216 atoken->instructions = EIP197_TOKEN_INS_ORIGIN_TOKEN | 217 EIP197_TOKEN_INS_TYPE_HASH; 218 219 if (likely(assocadj)) { 220 *aadlen = cpu_to_le32((assocadj >> 8) | 221 (assocadj & 255) << 8); 222 atoken += 6; 223 atoksize += 7; 224 } else { 225 atoken += 5; 226 atoksize += 6; 227 } 228 229 /* Process AAD data */ 230 aadref = atoken; 231 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION; 232 atoken->packet_length = assocadj; 233 atoken->stat = 0; 234 atoken->instructions = EIP197_TOKEN_INS_TYPE_HASH; 235 atoken++; 236 237 /* For CCM only, align AAD data towards hash engine */ 238 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 239 aadalign = (assocadj + 2) & 15; 240 atoken->packet_length = assocadj && aadalign ? 241 16 - aadalign : 242 0; 243 if (likely(cryptlen)) { 244 atoken->stat = 0; 245 atoken->instructions = EIP197_TOKEN_INS_TYPE_HASH; 246 } else { 247 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH; 248 atoken->instructions = EIP197_TOKEN_INS_LAST | 249 EIP197_TOKEN_INS_TYPE_HASH; 250 } 251 } else { 252 safexcel_aead_iv(ctx, iv, cdesc); 253 254 /* Process AAD data */ 255 aadref = atoken; 256 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION; 257 atoken->packet_length = assocadj; 258 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH; 259 atoken->instructions = EIP197_TOKEN_INS_LAST | 260 EIP197_TOKEN_INS_TYPE_HASH; 261 } 262 atoken++; 263 264 if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { 265 /* For ESP mode (and not GMAC), skip over the IV */ 266 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION; 267 atoken->packet_length = EIP197_AEAD_IPSEC_IV_SIZE; 268 atoken->stat = 0; 269 atoken->instructions = 0; 270 atoken++; 271 atoksize++; 272 } else if (unlikely(ctx->alg == SAFEXCEL_CHACHA20 && 273 direction == SAFEXCEL_DECRYPT)) { 274 /* Poly-chacha decryption needs a dummy NOP here ... */ 275 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 276 atoken->packet_length = 16; /* According to Op Manual */ 277 atoken->stat = 0; 278 atoken->instructions = 0; 279 atoken++; 280 atoksize++; 281 } 282 283 if (ctx->xcm) { 284 /* For GCM and CCM, obtain enc(Y0) */ 285 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT_REMRES; 286 atoken->packet_length = 0; 287 atoken->stat = 0; 288 atoken->instructions = AES_BLOCK_SIZE; 289 atoken++; 290 291 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 292 atoken->packet_length = AES_BLOCK_SIZE; 293 atoken->stat = 0; 294 atoken->instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | 295 EIP197_TOKEN_INS_TYPE_CRYPTO; 296 atoken++; 297 atoksize += 2; 298 } 299 300 if (likely(cryptlen || ctx->alg == SAFEXCEL_CHACHA20)) { 301 /* Fixup stat field for AAD direction instruction */ 302 aadref->stat = 0; 303 304 /* Process crypto data */ 305 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION; 306 atoken->packet_length = cryptlen; 307 308 if (unlikely(ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP_GMAC)) { 309 /* Fixup instruction field for AAD dir instruction */ 310 aadref->instructions = EIP197_TOKEN_INS_TYPE_HASH; 311 312 /* Do not send to crypt engine in case of GMAC */ 313 atoken->instructions = EIP197_TOKEN_INS_LAST | 314 EIP197_TOKEN_INS_TYPE_HASH | 315 EIP197_TOKEN_INS_TYPE_OUTPUT; 316 } else { 317 atoken->instructions = EIP197_TOKEN_INS_LAST | 318 EIP197_TOKEN_INS_TYPE_CRYPTO | 319 EIP197_TOKEN_INS_TYPE_HASH | 320 EIP197_TOKEN_INS_TYPE_OUTPUT; 321 } 322 323 cryptlen &= 15; 324 if (unlikely(ctx->xcm == EIP197_XCM_MODE_CCM && cryptlen)) { 325 atoken->stat = 0; 326 /* For CCM only, pad crypto data to the hash engine */ 327 atoken++; 328 atoksize++; 329 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 330 atoken->packet_length = 16 - cryptlen; 331 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH; 332 atoken->instructions = EIP197_TOKEN_INS_TYPE_HASH; 333 } else { 334 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH; 335 } 336 atoken++; 337 atoksize++; 338 } 339 340 if (direction == SAFEXCEL_ENCRYPT) { 341 /* Append ICV */ 342 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 343 atoken->packet_length = digestsize; 344 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH | 345 EIP197_TOKEN_STAT_LAST_PACKET; 346 atoken->instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | 347 EIP197_TOKEN_INS_INSERT_HASH_DIGEST; 348 } else { 349 /* Extract ICV */ 350 atoken->opcode = EIP197_TOKEN_OPCODE_RETRIEVE; 351 atoken->packet_length = digestsize; 352 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH | 353 EIP197_TOKEN_STAT_LAST_PACKET; 354 atoken->instructions = EIP197_TOKEN_INS_INSERT_HASH_DIGEST; 355 atoken++; 356 atoksize++; 357 358 /* Verify ICV */ 359 atoken->opcode = EIP197_TOKEN_OPCODE_VERIFY; 360 atoken->packet_length = digestsize | 361 EIP197_TOKEN_HASH_RESULT_VERIFY; 362 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH | 363 EIP197_TOKEN_STAT_LAST_PACKET; 364 atoken->instructions = EIP197_TOKEN_INS_TYPE_OUTPUT; 365 } 366 367 /* Fixup length of the token in the command descriptor */ 368 cdesc->additional_cdata_size = atoksize; 369 } 370 371 static int safexcel_skcipher_aes_setkey(struct crypto_skcipher *ctfm, 372 const u8 *key, unsigned int len) 373 { 374 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 375 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 376 struct safexcel_crypto_priv *priv = ctx->base.priv; 377 struct crypto_aes_ctx aes; 378 int ret, i; 379 380 ret = aes_expandkey(&aes, key, len); 381 if (ret) 382 return ret; 383 384 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 385 for (i = 0; i < len / sizeof(u32); i++) { 386 if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { 387 ctx->base.needs_inv = true; 388 break; 389 } 390 } 391 } 392 393 for (i = 0; i < len / sizeof(u32); i++) 394 ctx->key[i] = cpu_to_le32(aes.key_enc[i]); 395 396 ctx->key_len = len; 397 398 memzero_explicit(&aes, sizeof(aes)); 399 return 0; 400 } 401 402 static int safexcel_aead_setkey(struct crypto_aead *ctfm, const u8 *key, 403 unsigned int len) 404 { 405 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 406 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 407 struct safexcel_crypto_priv *priv = ctx->base.priv; 408 struct crypto_authenc_keys keys; 409 struct crypto_aes_ctx aes; 410 int err = -EINVAL, i; 411 const char *alg; 412 413 if (unlikely(crypto_authenc_extractkeys(&keys, key, len))) 414 goto badkey; 415 416 if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD) { 417 /* Must have at least space for the nonce here */ 418 if (unlikely(keys.enckeylen < CTR_RFC3686_NONCE_SIZE)) 419 goto badkey; 420 /* last 4 bytes of key are the nonce! */ 421 ctx->nonce = *(u32 *)(keys.enckey + keys.enckeylen - 422 CTR_RFC3686_NONCE_SIZE); 423 /* exclude the nonce here */ 424 keys.enckeylen -= CTR_RFC3686_NONCE_SIZE; 425 } 426 427 /* Encryption key */ 428 switch (ctx->alg) { 429 case SAFEXCEL_DES: 430 err = verify_aead_des_key(ctfm, keys.enckey, keys.enckeylen); 431 if (unlikely(err)) 432 goto badkey; 433 break; 434 case SAFEXCEL_3DES: 435 err = verify_aead_des3_key(ctfm, keys.enckey, keys.enckeylen); 436 if (unlikely(err)) 437 goto badkey; 438 break; 439 case SAFEXCEL_AES: 440 err = aes_expandkey(&aes, keys.enckey, keys.enckeylen); 441 if (unlikely(err)) 442 goto badkey; 443 break; 444 case SAFEXCEL_SM4: 445 if (unlikely(keys.enckeylen != SM4_KEY_SIZE)) 446 goto badkey; 447 break; 448 default: 449 dev_err(priv->dev, "aead: unsupported cipher algorithm\n"); 450 goto badkey; 451 } 452 453 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 454 for (i = 0; i < keys.enckeylen / sizeof(u32); i++) { 455 if (le32_to_cpu(ctx->key[i]) != 456 ((u32 *)keys.enckey)[i]) { 457 ctx->base.needs_inv = true; 458 break; 459 } 460 } 461 } 462 463 /* Auth key */ 464 switch (ctx->hash_alg) { 465 case CONTEXT_CONTROL_CRYPTO_ALG_SHA1: 466 alg = "safexcel-sha1"; 467 break; 468 case CONTEXT_CONTROL_CRYPTO_ALG_SHA224: 469 alg = "safexcel-sha224"; 470 break; 471 case CONTEXT_CONTROL_CRYPTO_ALG_SHA256: 472 alg = "safexcel-sha256"; 473 break; 474 case CONTEXT_CONTROL_CRYPTO_ALG_SHA384: 475 alg = "safexcel-sha384"; 476 break; 477 case CONTEXT_CONTROL_CRYPTO_ALG_SHA512: 478 alg = "safexcel-sha512"; 479 break; 480 case CONTEXT_CONTROL_CRYPTO_ALG_SM3: 481 alg = "safexcel-sm3"; 482 break; 483 default: 484 dev_err(priv->dev, "aead: unsupported hash algorithm\n"); 485 goto badkey; 486 } 487 488 if (safexcel_hmac_setkey(&ctx->base, keys.authkey, keys.authkeylen, 489 alg, ctx->state_sz)) 490 goto badkey; 491 492 /* Now copy the keys into the context */ 493 for (i = 0; i < keys.enckeylen / sizeof(u32); i++) 494 ctx->key[i] = cpu_to_le32(((u32 *)keys.enckey)[i]); 495 ctx->key_len = keys.enckeylen; 496 497 memzero_explicit(&keys, sizeof(keys)); 498 return 0; 499 500 badkey: 501 memzero_explicit(&keys, sizeof(keys)); 502 return err; 503 } 504 505 static int safexcel_context_control(struct safexcel_cipher_ctx *ctx, 506 struct crypto_async_request *async, 507 struct safexcel_cipher_req *sreq, 508 struct safexcel_command_desc *cdesc) 509 { 510 struct safexcel_crypto_priv *priv = ctx->base.priv; 511 int ctrl_size = ctx->key_len / sizeof(u32); 512 513 cdesc->control_data.control1 = ctx->mode; 514 515 if (ctx->aead) { 516 /* Take in account the ipad+opad digests */ 517 if (ctx->xcm) { 518 ctrl_size += ctx->state_sz / sizeof(u32); 519 cdesc->control_data.control0 = 520 CONTEXT_CONTROL_KEY_EN | 521 CONTEXT_CONTROL_DIGEST_XCM | 522 ctx->hash_alg | 523 CONTEXT_CONTROL_SIZE(ctrl_size); 524 } else if (ctx->alg == SAFEXCEL_CHACHA20) { 525 /* Chacha20-Poly1305 */ 526 cdesc->control_data.control0 = 527 CONTEXT_CONTROL_KEY_EN | 528 CONTEXT_CONTROL_CRYPTO_ALG_CHACHA20 | 529 (sreq->direction == SAFEXCEL_ENCRYPT ? 530 CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT : 531 CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN) | 532 ctx->hash_alg | 533 CONTEXT_CONTROL_SIZE(ctrl_size); 534 return 0; 535 } else { 536 ctrl_size += ctx->state_sz / sizeof(u32) * 2; 537 cdesc->control_data.control0 = 538 CONTEXT_CONTROL_KEY_EN | 539 CONTEXT_CONTROL_DIGEST_HMAC | 540 ctx->hash_alg | 541 CONTEXT_CONTROL_SIZE(ctrl_size); 542 } 543 544 if (sreq->direction == SAFEXCEL_ENCRYPT && 545 (ctx->xcm == EIP197_XCM_MODE_CCM || 546 ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP_GMAC)) 547 cdesc->control_data.control0 |= 548 CONTEXT_CONTROL_TYPE_HASH_ENCRYPT_OUT; 549 else if (sreq->direction == SAFEXCEL_ENCRYPT) 550 cdesc->control_data.control0 |= 551 CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT; 552 else if (ctx->xcm == EIP197_XCM_MODE_CCM) 553 cdesc->control_data.control0 |= 554 CONTEXT_CONTROL_TYPE_DECRYPT_HASH_IN; 555 else 556 cdesc->control_data.control0 |= 557 CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN; 558 } else { 559 if (sreq->direction == SAFEXCEL_ENCRYPT) 560 cdesc->control_data.control0 = 561 CONTEXT_CONTROL_TYPE_CRYPTO_OUT | 562 CONTEXT_CONTROL_KEY_EN | 563 CONTEXT_CONTROL_SIZE(ctrl_size); 564 else 565 cdesc->control_data.control0 = 566 CONTEXT_CONTROL_TYPE_CRYPTO_IN | 567 CONTEXT_CONTROL_KEY_EN | 568 CONTEXT_CONTROL_SIZE(ctrl_size); 569 } 570 571 if (ctx->alg == SAFEXCEL_DES) { 572 cdesc->control_data.control0 |= 573 CONTEXT_CONTROL_CRYPTO_ALG_DES; 574 } else if (ctx->alg == SAFEXCEL_3DES) { 575 cdesc->control_data.control0 |= 576 CONTEXT_CONTROL_CRYPTO_ALG_3DES; 577 } else if (ctx->alg == SAFEXCEL_AES) { 578 switch (ctx->key_len >> ctx->xts) { 579 case AES_KEYSIZE_128: 580 cdesc->control_data.control0 |= 581 CONTEXT_CONTROL_CRYPTO_ALG_AES128; 582 break; 583 case AES_KEYSIZE_192: 584 cdesc->control_data.control0 |= 585 CONTEXT_CONTROL_CRYPTO_ALG_AES192; 586 break; 587 case AES_KEYSIZE_256: 588 cdesc->control_data.control0 |= 589 CONTEXT_CONTROL_CRYPTO_ALG_AES256; 590 break; 591 default: 592 dev_err(priv->dev, "aes keysize not supported: %u\n", 593 ctx->key_len >> ctx->xts); 594 return -EINVAL; 595 } 596 } else if (ctx->alg == SAFEXCEL_CHACHA20) { 597 cdesc->control_data.control0 |= 598 CONTEXT_CONTROL_CRYPTO_ALG_CHACHA20; 599 } else if (ctx->alg == SAFEXCEL_SM4) { 600 cdesc->control_data.control0 |= 601 CONTEXT_CONTROL_CRYPTO_ALG_SM4; 602 } 603 604 return 0; 605 } 606 607 static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int ring, 608 struct crypto_async_request *async, 609 struct scatterlist *src, 610 struct scatterlist *dst, 611 unsigned int cryptlen, 612 struct safexcel_cipher_req *sreq, 613 bool *should_complete, int *ret) 614 { 615 struct skcipher_request *areq = skcipher_request_cast(async); 616 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(areq); 617 struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(skcipher); 618 struct safexcel_result_desc *rdesc; 619 int ndesc = 0; 620 621 *ret = 0; 622 623 if (unlikely(!sreq->rdescs)) 624 return 0; 625 626 while (sreq->rdescs--) { 627 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); 628 if (IS_ERR(rdesc)) { 629 dev_err(priv->dev, 630 "cipher: result: could not retrieve the result descriptor\n"); 631 *ret = PTR_ERR(rdesc); 632 break; 633 } 634 635 if (likely(!*ret)) 636 *ret = safexcel_rdesc_check_errors(priv, rdesc); 637 638 ndesc++; 639 } 640 641 safexcel_complete(priv, ring); 642 643 if (src == dst) { 644 if (sreq->nr_src > 0) 645 dma_unmap_sg(priv->dev, src, sreq->nr_src, 646 DMA_BIDIRECTIONAL); 647 } else { 648 if (sreq->nr_src > 0) 649 dma_unmap_sg(priv->dev, src, sreq->nr_src, 650 DMA_TO_DEVICE); 651 if (sreq->nr_dst > 0) 652 dma_unmap_sg(priv->dev, dst, sreq->nr_dst, 653 DMA_FROM_DEVICE); 654 } 655 656 /* 657 * Update IV in req from last crypto output word for CBC modes 658 */ 659 if ((!ctx->aead) && (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) && 660 (sreq->direction == SAFEXCEL_ENCRYPT)) { 661 /* For encrypt take the last output word */ 662 sg_pcopy_to_buffer(dst, sreq->nr_dst, areq->iv, 663 crypto_skcipher_ivsize(skcipher), 664 (cryptlen - 665 crypto_skcipher_ivsize(skcipher))); 666 } 667 668 *should_complete = true; 669 670 return ndesc; 671 } 672 673 static int safexcel_send_req(struct crypto_async_request *base, int ring, 674 struct safexcel_cipher_req *sreq, 675 struct scatterlist *src, struct scatterlist *dst, 676 unsigned int cryptlen, unsigned int assoclen, 677 unsigned int digestsize, u8 *iv, int *commands, 678 int *results) 679 { 680 struct skcipher_request *areq = skcipher_request_cast(base); 681 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(areq); 682 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); 683 struct safexcel_crypto_priv *priv = ctx->base.priv; 684 struct safexcel_command_desc *cdesc; 685 struct safexcel_command_desc *first_cdesc = NULL; 686 struct safexcel_result_desc *rdesc, *first_rdesc = NULL; 687 struct scatterlist *sg; 688 unsigned int totlen; 689 unsigned int totlen_src = cryptlen + assoclen; 690 unsigned int totlen_dst = totlen_src; 691 struct safexcel_token *atoken; 692 int n_cdesc = 0, n_rdesc = 0; 693 int queued, i, ret = 0; 694 bool first = true; 695 696 sreq->nr_src = sg_nents_for_len(src, totlen_src); 697 698 if (ctx->aead) { 699 /* 700 * AEAD has auth tag appended to output for encrypt and 701 * removed from the output for decrypt! 702 */ 703 if (sreq->direction == SAFEXCEL_DECRYPT) 704 totlen_dst -= digestsize; 705 else 706 totlen_dst += digestsize; 707 708 memcpy(ctx->base.ctxr->data + ctx->key_len / sizeof(u32), 709 &ctx->base.ipad, ctx->state_sz); 710 if (!ctx->xcm) 711 memcpy(ctx->base.ctxr->data + (ctx->key_len + 712 ctx->state_sz) / sizeof(u32), &ctx->base.opad, 713 ctx->state_sz); 714 } else if ((ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) && 715 (sreq->direction == SAFEXCEL_DECRYPT)) { 716 /* 717 * Save IV from last crypto input word for CBC modes in decrypt 718 * direction. Need to do this first in case of inplace operation 719 * as it will be overwritten. 720 */ 721 sg_pcopy_to_buffer(src, sreq->nr_src, areq->iv, 722 crypto_skcipher_ivsize(skcipher), 723 (totlen_src - 724 crypto_skcipher_ivsize(skcipher))); 725 } 726 727 sreq->nr_dst = sg_nents_for_len(dst, totlen_dst); 728 729 /* 730 * Remember actual input length, source buffer length may be 731 * updated in case of inline operation below. 732 */ 733 totlen = totlen_src; 734 queued = totlen_src; 735 736 if (src == dst) { 737 sreq->nr_src = max(sreq->nr_src, sreq->nr_dst); 738 sreq->nr_dst = sreq->nr_src; 739 if (unlikely((totlen_src || totlen_dst) && 740 (sreq->nr_src <= 0))) { 741 dev_err(priv->dev, "In-place buffer not large enough (need %d bytes)!", 742 max(totlen_src, totlen_dst)); 743 return -EINVAL; 744 } 745 if (sreq->nr_src > 0 && 746 !dma_map_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL)) 747 return -EIO; 748 } else { 749 if (unlikely(totlen_src && (sreq->nr_src <= 0))) { 750 dev_err(priv->dev, "Source buffer not large enough (need %d bytes)!", 751 totlen_src); 752 return -EINVAL; 753 } 754 755 if (sreq->nr_src > 0 && 756 !dma_map_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE)) 757 return -EIO; 758 759 if (unlikely(totlen_dst && (sreq->nr_dst <= 0))) { 760 dev_err(priv->dev, "Dest buffer not large enough (need %d bytes)!", 761 totlen_dst); 762 ret = -EINVAL; 763 goto unmap; 764 } 765 766 if (sreq->nr_dst > 0 && 767 !dma_map_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE)) { 768 ret = -EIO; 769 goto unmap; 770 } 771 } 772 773 memcpy(ctx->base.ctxr->data, ctx->key, ctx->key_len); 774 775 if (!totlen) { 776 /* 777 * The EIP97 cannot deal with zero length input packets! 778 * So stuff a dummy command descriptor indicating a 1 byte 779 * (dummy) input packet, using the context record as source. 780 */ 781 first_cdesc = safexcel_add_cdesc(priv, ring, 782 1, 1, ctx->base.ctxr_dma, 783 1, 1, ctx->base.ctxr_dma, 784 &atoken); 785 if (IS_ERR(first_cdesc)) { 786 /* No space left in the command descriptor ring */ 787 ret = PTR_ERR(first_cdesc); 788 goto cdesc_rollback; 789 } 790 n_cdesc = 1; 791 goto skip_cdesc; 792 } 793 794 /* command descriptors */ 795 for_each_sg(src, sg, sreq->nr_src, i) { 796 int len = sg_dma_len(sg); 797 798 /* Do not overflow the request */ 799 if (queued < len) 800 len = queued; 801 802 cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc, 803 !(queued - len), 804 sg_dma_address(sg), len, totlen, 805 ctx->base.ctxr_dma, &atoken); 806 if (IS_ERR(cdesc)) { 807 /* No space left in the command descriptor ring */ 808 ret = PTR_ERR(cdesc); 809 goto cdesc_rollback; 810 } 811 812 if (!n_cdesc) 813 first_cdesc = cdesc; 814 815 n_cdesc++; 816 queued -= len; 817 if (!queued) 818 break; 819 } 820 skip_cdesc: 821 /* Add context control words and token to first command descriptor */ 822 safexcel_context_control(ctx, base, sreq, first_cdesc); 823 if (ctx->aead) 824 safexcel_aead_token(ctx, iv, first_cdesc, atoken, 825 sreq->direction, cryptlen, 826 assoclen, digestsize); 827 else 828 safexcel_skcipher_token(ctx, iv, first_cdesc, atoken, 829 cryptlen); 830 831 /* result descriptors */ 832 for_each_sg(dst, sg, sreq->nr_dst, i) { 833 bool last = (i == sreq->nr_dst - 1); 834 u32 len = sg_dma_len(sg); 835 836 /* only allow the part of the buffer we know we need */ 837 if (len > totlen_dst) 838 len = totlen_dst; 839 if (unlikely(!len)) 840 break; 841 totlen_dst -= len; 842 843 /* skip over AAD space in buffer - not written */ 844 if (assoclen) { 845 if (assoclen >= len) { 846 assoclen -= len; 847 continue; 848 } 849 rdesc = safexcel_add_rdesc(priv, ring, first, last, 850 sg_dma_address(sg) + 851 assoclen, 852 len - assoclen); 853 assoclen = 0; 854 } else { 855 rdesc = safexcel_add_rdesc(priv, ring, first, last, 856 sg_dma_address(sg), 857 len); 858 } 859 if (IS_ERR(rdesc)) { 860 /* No space left in the result descriptor ring */ 861 ret = PTR_ERR(rdesc); 862 goto rdesc_rollback; 863 } 864 if (first) { 865 first_rdesc = rdesc; 866 first = false; 867 } 868 n_rdesc++; 869 } 870 871 if (unlikely(first)) { 872 /* 873 * Special case: AEAD decrypt with only AAD data. 874 * In this case there is NO output data from the engine, 875 * but the engine still needs a result descriptor! 876 * Create a dummy one just for catching the result token. 877 */ 878 rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0); 879 if (IS_ERR(rdesc)) { 880 /* No space left in the result descriptor ring */ 881 ret = PTR_ERR(rdesc); 882 goto rdesc_rollback; 883 } 884 first_rdesc = rdesc; 885 n_rdesc = 1; 886 } 887 888 safexcel_rdr_req_set(priv, ring, first_rdesc, base); 889 890 *commands = n_cdesc; 891 *results = n_rdesc; 892 return 0; 893 894 rdesc_rollback: 895 for (i = 0; i < n_rdesc; i++) 896 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].rdr); 897 cdesc_rollback: 898 for (i = 0; i < n_cdesc; i++) 899 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr); 900 unmap: 901 if (src == dst) { 902 if (sreq->nr_src > 0) 903 dma_unmap_sg(priv->dev, src, sreq->nr_src, 904 DMA_BIDIRECTIONAL); 905 } else { 906 if (sreq->nr_src > 0) 907 dma_unmap_sg(priv->dev, src, sreq->nr_src, 908 DMA_TO_DEVICE); 909 if (sreq->nr_dst > 0) 910 dma_unmap_sg(priv->dev, dst, sreq->nr_dst, 911 DMA_FROM_DEVICE); 912 } 913 914 return ret; 915 } 916 917 static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv, 918 int ring, 919 struct crypto_async_request *base, 920 struct safexcel_cipher_req *sreq, 921 bool *should_complete, int *ret) 922 { 923 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); 924 struct safexcel_result_desc *rdesc; 925 int ndesc = 0, enq_ret; 926 927 *ret = 0; 928 929 if (unlikely(!sreq->rdescs)) 930 return 0; 931 932 while (sreq->rdescs--) { 933 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); 934 if (IS_ERR(rdesc)) { 935 dev_err(priv->dev, 936 "cipher: invalidate: could not retrieve the result descriptor\n"); 937 *ret = PTR_ERR(rdesc); 938 break; 939 } 940 941 if (likely(!*ret)) 942 *ret = safexcel_rdesc_check_errors(priv, rdesc); 943 944 ndesc++; 945 } 946 947 safexcel_complete(priv, ring); 948 949 if (ctx->base.exit_inv) { 950 dma_pool_free(priv->context_pool, ctx->base.ctxr, 951 ctx->base.ctxr_dma); 952 953 *should_complete = true; 954 955 return ndesc; 956 } 957 958 ring = safexcel_select_ring(priv); 959 ctx->base.ring = ring; 960 961 spin_lock_bh(&priv->ring[ring].queue_lock); 962 enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, base); 963 spin_unlock_bh(&priv->ring[ring].queue_lock); 964 965 if (enq_ret != -EINPROGRESS) 966 *ret = enq_ret; 967 968 queue_work(priv->ring[ring].workqueue, 969 &priv->ring[ring].work_data.work); 970 971 *should_complete = false; 972 973 return ndesc; 974 } 975 976 static int safexcel_skcipher_handle_result(struct safexcel_crypto_priv *priv, 977 int ring, 978 struct crypto_async_request *async, 979 bool *should_complete, int *ret) 980 { 981 struct skcipher_request *req = skcipher_request_cast(async); 982 struct safexcel_cipher_req *sreq = skcipher_request_ctx(req); 983 int err; 984 985 if (sreq->needs_inv) { 986 sreq->needs_inv = false; 987 err = safexcel_handle_inv_result(priv, ring, async, sreq, 988 should_complete, ret); 989 } else { 990 err = safexcel_handle_req_result(priv, ring, async, req->src, 991 req->dst, req->cryptlen, sreq, 992 should_complete, ret); 993 } 994 995 return err; 996 } 997 998 static int safexcel_aead_handle_result(struct safexcel_crypto_priv *priv, 999 int ring, 1000 struct crypto_async_request *async, 1001 bool *should_complete, int *ret) 1002 { 1003 struct aead_request *req = aead_request_cast(async); 1004 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1005 struct safexcel_cipher_req *sreq = aead_request_ctx(req); 1006 int err; 1007 1008 if (sreq->needs_inv) { 1009 sreq->needs_inv = false; 1010 err = safexcel_handle_inv_result(priv, ring, async, sreq, 1011 should_complete, ret); 1012 } else { 1013 err = safexcel_handle_req_result(priv, ring, async, req->src, 1014 req->dst, 1015 req->cryptlen + crypto_aead_authsize(tfm), 1016 sreq, should_complete, ret); 1017 } 1018 1019 return err; 1020 } 1021 1022 static int safexcel_cipher_send_inv(struct crypto_async_request *base, 1023 int ring, int *commands, int *results) 1024 { 1025 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); 1026 struct safexcel_crypto_priv *priv = ctx->base.priv; 1027 int ret; 1028 1029 ret = safexcel_invalidate_cache(base, priv, ctx->base.ctxr_dma, ring); 1030 if (unlikely(ret)) 1031 return ret; 1032 1033 *commands = 1; 1034 *results = 1; 1035 1036 return 0; 1037 } 1038 1039 static int safexcel_skcipher_send(struct crypto_async_request *async, int ring, 1040 int *commands, int *results) 1041 { 1042 struct skcipher_request *req = skcipher_request_cast(async); 1043 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); 1044 struct safexcel_cipher_req *sreq = skcipher_request_ctx(req); 1045 struct safexcel_crypto_priv *priv = ctx->base.priv; 1046 int ret; 1047 1048 BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv); 1049 1050 if (sreq->needs_inv) { 1051 ret = safexcel_cipher_send_inv(async, ring, commands, results); 1052 } else { 1053 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1054 u8 input_iv[AES_BLOCK_SIZE]; 1055 1056 /* 1057 * Save input IV in case of CBC decrypt mode 1058 * Will be overwritten with output IV prior to use! 1059 */ 1060 memcpy(input_iv, req->iv, crypto_skcipher_ivsize(skcipher)); 1061 1062 ret = safexcel_send_req(async, ring, sreq, req->src, 1063 req->dst, req->cryptlen, 0, 0, input_iv, 1064 commands, results); 1065 } 1066 1067 sreq->rdescs = *results; 1068 return ret; 1069 } 1070 1071 static int safexcel_aead_send(struct crypto_async_request *async, int ring, 1072 int *commands, int *results) 1073 { 1074 struct aead_request *req = aead_request_cast(async); 1075 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1076 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); 1077 struct safexcel_cipher_req *sreq = aead_request_ctx(req); 1078 struct safexcel_crypto_priv *priv = ctx->base.priv; 1079 int ret; 1080 1081 BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv); 1082 1083 if (sreq->needs_inv) 1084 ret = safexcel_cipher_send_inv(async, ring, commands, results); 1085 else 1086 ret = safexcel_send_req(async, ring, sreq, req->src, req->dst, 1087 req->cryptlen, req->assoclen, 1088 crypto_aead_authsize(tfm), req->iv, 1089 commands, results); 1090 sreq->rdescs = *results; 1091 return ret; 1092 } 1093 1094 static int safexcel_cipher_exit_inv(struct crypto_tfm *tfm, 1095 struct crypto_async_request *base, 1096 struct safexcel_cipher_req *sreq, 1097 struct crypto_wait *result) 1098 { 1099 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1100 struct safexcel_crypto_priv *priv = ctx->base.priv; 1101 int ring = ctx->base.ring; 1102 int err; 1103 1104 ctx = crypto_tfm_ctx(base->tfm); 1105 ctx->base.exit_inv = true; 1106 sreq->needs_inv = true; 1107 1108 spin_lock_bh(&priv->ring[ring].queue_lock); 1109 crypto_enqueue_request(&priv->ring[ring].queue, base); 1110 spin_unlock_bh(&priv->ring[ring].queue_lock); 1111 1112 queue_work(priv->ring[ring].workqueue, 1113 &priv->ring[ring].work_data.work); 1114 1115 err = crypto_wait_req(-EINPROGRESS, result); 1116 1117 if (err) { 1118 dev_warn(priv->dev, 1119 "cipher: sync: invalidate: completion error %d\n", 1120 err); 1121 return err; 1122 } 1123 1124 return 0; 1125 } 1126 1127 static int safexcel_skcipher_exit_inv(struct crypto_tfm *tfm) 1128 { 1129 EIP197_REQUEST_ON_STACK(req, skcipher, EIP197_SKCIPHER_REQ_SIZE); 1130 struct safexcel_cipher_req *sreq = skcipher_request_ctx(req); 1131 DECLARE_CRYPTO_WAIT(result); 1132 1133 memset(req, 0, sizeof(struct skcipher_request)); 1134 1135 skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 1136 crypto_req_done, &result); 1137 skcipher_request_set_tfm(req, __crypto_skcipher_cast(tfm)); 1138 1139 return safexcel_cipher_exit_inv(tfm, &req->base, sreq, &result); 1140 } 1141 1142 static int safexcel_aead_exit_inv(struct crypto_tfm *tfm) 1143 { 1144 EIP197_REQUEST_ON_STACK(req, aead, EIP197_AEAD_REQ_SIZE); 1145 struct safexcel_cipher_req *sreq = aead_request_ctx(req); 1146 DECLARE_CRYPTO_WAIT(result); 1147 1148 memset(req, 0, sizeof(struct aead_request)); 1149 1150 aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 1151 crypto_req_done, &result); 1152 aead_request_set_tfm(req, __crypto_aead_cast(tfm)); 1153 1154 return safexcel_cipher_exit_inv(tfm, &req->base, sreq, &result); 1155 } 1156 1157 static int safexcel_queue_req(struct crypto_async_request *base, 1158 struct safexcel_cipher_req *sreq, 1159 enum safexcel_cipher_direction dir) 1160 { 1161 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); 1162 struct safexcel_crypto_priv *priv = ctx->base.priv; 1163 int ret, ring; 1164 1165 sreq->needs_inv = false; 1166 sreq->direction = dir; 1167 1168 if (ctx->base.ctxr) { 1169 if (priv->flags & EIP197_TRC_CACHE && ctx->base.needs_inv) { 1170 sreq->needs_inv = true; 1171 ctx->base.needs_inv = false; 1172 } 1173 } else { 1174 ctx->base.ring = safexcel_select_ring(priv); 1175 ctx->base.ctxr = dma_pool_zalloc(priv->context_pool, 1176 EIP197_GFP_FLAGS(*base), 1177 &ctx->base.ctxr_dma); 1178 if (!ctx->base.ctxr) 1179 return -ENOMEM; 1180 } 1181 1182 ring = ctx->base.ring; 1183 1184 spin_lock_bh(&priv->ring[ring].queue_lock); 1185 ret = crypto_enqueue_request(&priv->ring[ring].queue, base); 1186 spin_unlock_bh(&priv->ring[ring].queue_lock); 1187 1188 queue_work(priv->ring[ring].workqueue, 1189 &priv->ring[ring].work_data.work); 1190 1191 return ret; 1192 } 1193 1194 static int safexcel_encrypt(struct skcipher_request *req) 1195 { 1196 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 1197 SAFEXCEL_ENCRYPT); 1198 } 1199 1200 static int safexcel_decrypt(struct skcipher_request *req) 1201 { 1202 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 1203 SAFEXCEL_DECRYPT); 1204 } 1205 1206 static int safexcel_skcipher_cra_init(struct crypto_tfm *tfm) 1207 { 1208 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1209 struct safexcel_alg_template *tmpl = 1210 container_of(tfm->__crt_alg, struct safexcel_alg_template, 1211 alg.skcipher.base); 1212 1213 crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm), 1214 sizeof(struct safexcel_cipher_req)); 1215 1216 ctx->base.priv = tmpl->priv; 1217 1218 ctx->base.send = safexcel_skcipher_send; 1219 ctx->base.handle_result = safexcel_skcipher_handle_result; 1220 ctx->ivmask = EIP197_OPTION_4_TOKEN_IV_CMD; 1221 ctx->ctrinit = 1; 1222 return 0; 1223 } 1224 1225 static int safexcel_cipher_cra_exit(struct crypto_tfm *tfm) 1226 { 1227 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1228 1229 memzero_explicit(ctx->key, sizeof(ctx->key)); 1230 1231 /* context not allocated, skip invalidation */ 1232 if (!ctx->base.ctxr) 1233 return -ENOMEM; 1234 1235 memzero_explicit(ctx->base.ctxr->data, sizeof(ctx->base.ctxr->data)); 1236 return 0; 1237 } 1238 1239 static void safexcel_skcipher_cra_exit(struct crypto_tfm *tfm) 1240 { 1241 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1242 struct safexcel_crypto_priv *priv = ctx->base.priv; 1243 int ret; 1244 1245 if (safexcel_cipher_cra_exit(tfm)) 1246 return; 1247 1248 if (priv->flags & EIP197_TRC_CACHE) { 1249 ret = safexcel_skcipher_exit_inv(tfm); 1250 if (ret) 1251 dev_warn(priv->dev, "skcipher: invalidation error %d\n", 1252 ret); 1253 } else { 1254 dma_pool_free(priv->context_pool, ctx->base.ctxr, 1255 ctx->base.ctxr_dma); 1256 } 1257 } 1258 1259 static void safexcel_aead_cra_exit(struct crypto_tfm *tfm) 1260 { 1261 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1262 struct safexcel_crypto_priv *priv = ctx->base.priv; 1263 int ret; 1264 1265 if (safexcel_cipher_cra_exit(tfm)) 1266 return; 1267 1268 if (priv->flags & EIP197_TRC_CACHE) { 1269 ret = safexcel_aead_exit_inv(tfm); 1270 if (ret) 1271 dev_warn(priv->dev, "aead: invalidation error %d\n", 1272 ret); 1273 } else { 1274 dma_pool_free(priv->context_pool, ctx->base.ctxr, 1275 ctx->base.ctxr_dma); 1276 } 1277 } 1278 1279 static int safexcel_skcipher_aes_ecb_cra_init(struct crypto_tfm *tfm) 1280 { 1281 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1282 1283 safexcel_skcipher_cra_init(tfm); 1284 ctx->alg = SAFEXCEL_AES; 1285 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; 1286 ctx->blocksz = 0; 1287 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1288 return 0; 1289 } 1290 1291 struct safexcel_alg_template safexcel_alg_ecb_aes = { 1292 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1293 .algo_mask = SAFEXCEL_ALG_AES, 1294 .alg.skcipher = { 1295 .setkey = safexcel_skcipher_aes_setkey, 1296 .encrypt = safexcel_encrypt, 1297 .decrypt = safexcel_decrypt, 1298 .min_keysize = AES_MIN_KEY_SIZE, 1299 .max_keysize = AES_MAX_KEY_SIZE, 1300 .base = { 1301 .cra_name = "ecb(aes)", 1302 .cra_driver_name = "safexcel-ecb-aes", 1303 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1304 .cra_flags = CRYPTO_ALG_ASYNC | 1305 CRYPTO_ALG_ALLOCATES_MEMORY | 1306 CRYPTO_ALG_KERN_DRIVER_ONLY, 1307 .cra_blocksize = AES_BLOCK_SIZE, 1308 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1309 .cra_alignmask = 0, 1310 .cra_init = safexcel_skcipher_aes_ecb_cra_init, 1311 .cra_exit = safexcel_skcipher_cra_exit, 1312 .cra_module = THIS_MODULE, 1313 }, 1314 }, 1315 }; 1316 1317 static int safexcel_skcipher_aes_cbc_cra_init(struct crypto_tfm *tfm) 1318 { 1319 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1320 1321 safexcel_skcipher_cra_init(tfm); 1322 ctx->alg = SAFEXCEL_AES; 1323 ctx->blocksz = AES_BLOCK_SIZE; 1324 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; 1325 return 0; 1326 } 1327 1328 struct safexcel_alg_template safexcel_alg_cbc_aes = { 1329 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1330 .algo_mask = SAFEXCEL_ALG_AES, 1331 .alg.skcipher = { 1332 .setkey = safexcel_skcipher_aes_setkey, 1333 .encrypt = safexcel_encrypt, 1334 .decrypt = safexcel_decrypt, 1335 .min_keysize = AES_MIN_KEY_SIZE, 1336 .max_keysize = AES_MAX_KEY_SIZE, 1337 .ivsize = AES_BLOCK_SIZE, 1338 .base = { 1339 .cra_name = "cbc(aes)", 1340 .cra_driver_name = "safexcel-cbc-aes", 1341 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1342 .cra_flags = CRYPTO_ALG_ASYNC | 1343 CRYPTO_ALG_ALLOCATES_MEMORY | 1344 CRYPTO_ALG_KERN_DRIVER_ONLY, 1345 .cra_blocksize = AES_BLOCK_SIZE, 1346 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1347 .cra_alignmask = 0, 1348 .cra_init = safexcel_skcipher_aes_cbc_cra_init, 1349 .cra_exit = safexcel_skcipher_cra_exit, 1350 .cra_module = THIS_MODULE, 1351 }, 1352 }, 1353 }; 1354 1355 static int safexcel_skcipher_aesctr_setkey(struct crypto_skcipher *ctfm, 1356 const u8 *key, unsigned int len) 1357 { 1358 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 1359 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1360 struct safexcel_crypto_priv *priv = ctx->base.priv; 1361 struct crypto_aes_ctx aes; 1362 int ret, i; 1363 unsigned int keylen; 1364 1365 /* last 4 bytes of key are the nonce! */ 1366 ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE); 1367 /* exclude the nonce here */ 1368 keylen = len - CTR_RFC3686_NONCE_SIZE; 1369 ret = aes_expandkey(&aes, key, keylen); 1370 if (ret) 1371 return ret; 1372 1373 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 1374 for (i = 0; i < keylen / sizeof(u32); i++) { 1375 if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { 1376 ctx->base.needs_inv = true; 1377 break; 1378 } 1379 } 1380 } 1381 1382 for (i = 0; i < keylen / sizeof(u32); i++) 1383 ctx->key[i] = cpu_to_le32(aes.key_enc[i]); 1384 1385 ctx->key_len = keylen; 1386 1387 memzero_explicit(&aes, sizeof(aes)); 1388 return 0; 1389 } 1390 1391 static int safexcel_skcipher_aes_ctr_cra_init(struct crypto_tfm *tfm) 1392 { 1393 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1394 1395 safexcel_skcipher_cra_init(tfm); 1396 ctx->alg = SAFEXCEL_AES; 1397 ctx->blocksz = AES_BLOCK_SIZE; 1398 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; 1399 return 0; 1400 } 1401 1402 struct safexcel_alg_template safexcel_alg_ctr_aes = { 1403 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1404 .algo_mask = SAFEXCEL_ALG_AES, 1405 .alg.skcipher = { 1406 .setkey = safexcel_skcipher_aesctr_setkey, 1407 .encrypt = safexcel_encrypt, 1408 .decrypt = safexcel_decrypt, 1409 /* Add nonce size */ 1410 .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 1411 .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 1412 .ivsize = CTR_RFC3686_IV_SIZE, 1413 .base = { 1414 .cra_name = "rfc3686(ctr(aes))", 1415 .cra_driver_name = "safexcel-ctr-aes", 1416 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1417 .cra_flags = CRYPTO_ALG_ASYNC | 1418 CRYPTO_ALG_ALLOCATES_MEMORY | 1419 CRYPTO_ALG_KERN_DRIVER_ONLY, 1420 .cra_blocksize = 1, 1421 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1422 .cra_alignmask = 0, 1423 .cra_init = safexcel_skcipher_aes_ctr_cra_init, 1424 .cra_exit = safexcel_skcipher_cra_exit, 1425 .cra_module = THIS_MODULE, 1426 }, 1427 }, 1428 }; 1429 1430 static int safexcel_des_setkey(struct crypto_skcipher *ctfm, const u8 *key, 1431 unsigned int len) 1432 { 1433 struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm); 1434 struct safexcel_crypto_priv *priv = ctx->base.priv; 1435 int ret; 1436 1437 ret = verify_skcipher_des_key(ctfm, key); 1438 if (ret) 1439 return ret; 1440 1441 /* if context exits and key changed, need to invalidate it */ 1442 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) 1443 if (memcmp(ctx->key, key, len)) 1444 ctx->base.needs_inv = true; 1445 1446 memcpy(ctx->key, key, len); 1447 ctx->key_len = len; 1448 1449 return 0; 1450 } 1451 1452 static int safexcel_skcipher_des_cbc_cra_init(struct crypto_tfm *tfm) 1453 { 1454 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1455 1456 safexcel_skcipher_cra_init(tfm); 1457 ctx->alg = SAFEXCEL_DES; 1458 ctx->blocksz = DES_BLOCK_SIZE; 1459 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1460 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; 1461 return 0; 1462 } 1463 1464 struct safexcel_alg_template safexcel_alg_cbc_des = { 1465 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1466 .algo_mask = SAFEXCEL_ALG_DES, 1467 .alg.skcipher = { 1468 .setkey = safexcel_des_setkey, 1469 .encrypt = safexcel_encrypt, 1470 .decrypt = safexcel_decrypt, 1471 .min_keysize = DES_KEY_SIZE, 1472 .max_keysize = DES_KEY_SIZE, 1473 .ivsize = DES_BLOCK_SIZE, 1474 .base = { 1475 .cra_name = "cbc(des)", 1476 .cra_driver_name = "safexcel-cbc-des", 1477 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1478 .cra_flags = CRYPTO_ALG_ASYNC | 1479 CRYPTO_ALG_ALLOCATES_MEMORY | 1480 CRYPTO_ALG_KERN_DRIVER_ONLY, 1481 .cra_blocksize = DES_BLOCK_SIZE, 1482 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1483 .cra_alignmask = 0, 1484 .cra_init = safexcel_skcipher_des_cbc_cra_init, 1485 .cra_exit = safexcel_skcipher_cra_exit, 1486 .cra_module = THIS_MODULE, 1487 }, 1488 }, 1489 }; 1490 1491 static int safexcel_skcipher_des_ecb_cra_init(struct crypto_tfm *tfm) 1492 { 1493 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1494 1495 safexcel_skcipher_cra_init(tfm); 1496 ctx->alg = SAFEXCEL_DES; 1497 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; 1498 ctx->blocksz = 0; 1499 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1500 return 0; 1501 } 1502 1503 struct safexcel_alg_template safexcel_alg_ecb_des = { 1504 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1505 .algo_mask = SAFEXCEL_ALG_DES, 1506 .alg.skcipher = { 1507 .setkey = safexcel_des_setkey, 1508 .encrypt = safexcel_encrypt, 1509 .decrypt = safexcel_decrypt, 1510 .min_keysize = DES_KEY_SIZE, 1511 .max_keysize = DES_KEY_SIZE, 1512 .base = { 1513 .cra_name = "ecb(des)", 1514 .cra_driver_name = "safexcel-ecb-des", 1515 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1516 .cra_flags = CRYPTO_ALG_ASYNC | 1517 CRYPTO_ALG_ALLOCATES_MEMORY | 1518 CRYPTO_ALG_KERN_DRIVER_ONLY, 1519 .cra_blocksize = DES_BLOCK_SIZE, 1520 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1521 .cra_alignmask = 0, 1522 .cra_init = safexcel_skcipher_des_ecb_cra_init, 1523 .cra_exit = safexcel_skcipher_cra_exit, 1524 .cra_module = THIS_MODULE, 1525 }, 1526 }, 1527 }; 1528 1529 static int safexcel_des3_ede_setkey(struct crypto_skcipher *ctfm, 1530 const u8 *key, unsigned int len) 1531 { 1532 struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm); 1533 struct safexcel_crypto_priv *priv = ctx->base.priv; 1534 int err; 1535 1536 err = verify_skcipher_des3_key(ctfm, key); 1537 if (err) 1538 return err; 1539 1540 /* if context exits and key changed, need to invalidate it */ 1541 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) 1542 if (memcmp(ctx->key, key, len)) 1543 ctx->base.needs_inv = true; 1544 1545 memcpy(ctx->key, key, len); 1546 ctx->key_len = len; 1547 1548 return 0; 1549 } 1550 1551 static int safexcel_skcipher_des3_cbc_cra_init(struct crypto_tfm *tfm) 1552 { 1553 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1554 1555 safexcel_skcipher_cra_init(tfm); 1556 ctx->alg = SAFEXCEL_3DES; 1557 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 1558 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1559 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; 1560 return 0; 1561 } 1562 1563 struct safexcel_alg_template safexcel_alg_cbc_des3_ede = { 1564 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1565 .algo_mask = SAFEXCEL_ALG_DES, 1566 .alg.skcipher = { 1567 .setkey = safexcel_des3_ede_setkey, 1568 .encrypt = safexcel_encrypt, 1569 .decrypt = safexcel_decrypt, 1570 .min_keysize = DES3_EDE_KEY_SIZE, 1571 .max_keysize = DES3_EDE_KEY_SIZE, 1572 .ivsize = DES3_EDE_BLOCK_SIZE, 1573 .base = { 1574 .cra_name = "cbc(des3_ede)", 1575 .cra_driver_name = "safexcel-cbc-des3_ede", 1576 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1577 .cra_flags = CRYPTO_ALG_ASYNC | 1578 CRYPTO_ALG_ALLOCATES_MEMORY | 1579 CRYPTO_ALG_KERN_DRIVER_ONLY, 1580 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1581 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1582 .cra_alignmask = 0, 1583 .cra_init = safexcel_skcipher_des3_cbc_cra_init, 1584 .cra_exit = safexcel_skcipher_cra_exit, 1585 .cra_module = THIS_MODULE, 1586 }, 1587 }, 1588 }; 1589 1590 static int safexcel_skcipher_des3_ecb_cra_init(struct crypto_tfm *tfm) 1591 { 1592 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1593 1594 safexcel_skcipher_cra_init(tfm); 1595 ctx->alg = SAFEXCEL_3DES; 1596 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; 1597 ctx->blocksz = 0; 1598 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1599 return 0; 1600 } 1601 1602 struct safexcel_alg_template safexcel_alg_ecb_des3_ede = { 1603 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1604 .algo_mask = SAFEXCEL_ALG_DES, 1605 .alg.skcipher = { 1606 .setkey = safexcel_des3_ede_setkey, 1607 .encrypt = safexcel_encrypt, 1608 .decrypt = safexcel_decrypt, 1609 .min_keysize = DES3_EDE_KEY_SIZE, 1610 .max_keysize = DES3_EDE_KEY_SIZE, 1611 .base = { 1612 .cra_name = "ecb(des3_ede)", 1613 .cra_driver_name = "safexcel-ecb-des3_ede", 1614 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1615 .cra_flags = CRYPTO_ALG_ASYNC | 1616 CRYPTO_ALG_ALLOCATES_MEMORY | 1617 CRYPTO_ALG_KERN_DRIVER_ONLY, 1618 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1619 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1620 .cra_alignmask = 0, 1621 .cra_init = safexcel_skcipher_des3_ecb_cra_init, 1622 .cra_exit = safexcel_skcipher_cra_exit, 1623 .cra_module = THIS_MODULE, 1624 }, 1625 }, 1626 }; 1627 1628 static int safexcel_aead_encrypt(struct aead_request *req) 1629 { 1630 struct safexcel_cipher_req *creq = aead_request_ctx(req); 1631 1632 return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); 1633 } 1634 1635 static int safexcel_aead_decrypt(struct aead_request *req) 1636 { 1637 struct safexcel_cipher_req *creq = aead_request_ctx(req); 1638 1639 return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); 1640 } 1641 1642 static int safexcel_aead_cra_init(struct crypto_tfm *tfm) 1643 { 1644 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1645 struct safexcel_alg_template *tmpl = 1646 container_of(tfm->__crt_alg, struct safexcel_alg_template, 1647 alg.aead.base); 1648 1649 crypto_aead_set_reqsize(__crypto_aead_cast(tfm), 1650 sizeof(struct safexcel_cipher_req)); 1651 1652 ctx->base.priv = tmpl->priv; 1653 1654 ctx->alg = SAFEXCEL_AES; /* default */ 1655 ctx->blocksz = AES_BLOCK_SIZE; 1656 ctx->ivmask = EIP197_OPTION_4_TOKEN_IV_CMD; 1657 ctx->ctrinit = 1; 1658 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; /* default */ 1659 ctx->aead = true; 1660 ctx->base.send = safexcel_aead_send; 1661 ctx->base.handle_result = safexcel_aead_handle_result; 1662 return 0; 1663 } 1664 1665 static int safexcel_aead_sha1_cra_init(struct crypto_tfm *tfm) 1666 { 1667 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1668 1669 safexcel_aead_cra_init(tfm); 1670 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1; 1671 ctx->state_sz = SHA1_DIGEST_SIZE; 1672 return 0; 1673 } 1674 1675 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_aes = { 1676 .type = SAFEXCEL_ALG_TYPE_AEAD, 1677 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA1, 1678 .alg.aead = { 1679 .setkey = safexcel_aead_setkey, 1680 .encrypt = safexcel_aead_encrypt, 1681 .decrypt = safexcel_aead_decrypt, 1682 .ivsize = AES_BLOCK_SIZE, 1683 .maxauthsize = SHA1_DIGEST_SIZE, 1684 .base = { 1685 .cra_name = "authenc(hmac(sha1),cbc(aes))", 1686 .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-aes", 1687 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1688 .cra_flags = CRYPTO_ALG_ASYNC | 1689 CRYPTO_ALG_ALLOCATES_MEMORY | 1690 CRYPTO_ALG_KERN_DRIVER_ONLY, 1691 .cra_blocksize = AES_BLOCK_SIZE, 1692 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1693 .cra_alignmask = 0, 1694 .cra_init = safexcel_aead_sha1_cra_init, 1695 .cra_exit = safexcel_aead_cra_exit, 1696 .cra_module = THIS_MODULE, 1697 }, 1698 }, 1699 }; 1700 1701 static int safexcel_aead_sha256_cra_init(struct crypto_tfm *tfm) 1702 { 1703 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1704 1705 safexcel_aead_cra_init(tfm); 1706 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256; 1707 ctx->state_sz = SHA256_DIGEST_SIZE; 1708 return 0; 1709 } 1710 1711 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_aes = { 1712 .type = SAFEXCEL_ALG_TYPE_AEAD, 1713 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256, 1714 .alg.aead = { 1715 .setkey = safexcel_aead_setkey, 1716 .encrypt = safexcel_aead_encrypt, 1717 .decrypt = safexcel_aead_decrypt, 1718 .ivsize = AES_BLOCK_SIZE, 1719 .maxauthsize = SHA256_DIGEST_SIZE, 1720 .base = { 1721 .cra_name = "authenc(hmac(sha256),cbc(aes))", 1722 .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-aes", 1723 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1724 .cra_flags = CRYPTO_ALG_ASYNC | 1725 CRYPTO_ALG_ALLOCATES_MEMORY | 1726 CRYPTO_ALG_KERN_DRIVER_ONLY, 1727 .cra_blocksize = AES_BLOCK_SIZE, 1728 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1729 .cra_alignmask = 0, 1730 .cra_init = safexcel_aead_sha256_cra_init, 1731 .cra_exit = safexcel_aead_cra_exit, 1732 .cra_module = THIS_MODULE, 1733 }, 1734 }, 1735 }; 1736 1737 static int safexcel_aead_sha224_cra_init(struct crypto_tfm *tfm) 1738 { 1739 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1740 1741 safexcel_aead_cra_init(tfm); 1742 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224; 1743 ctx->state_sz = SHA256_DIGEST_SIZE; 1744 return 0; 1745 } 1746 1747 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_aes = { 1748 .type = SAFEXCEL_ALG_TYPE_AEAD, 1749 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256, 1750 .alg.aead = { 1751 .setkey = safexcel_aead_setkey, 1752 .encrypt = safexcel_aead_encrypt, 1753 .decrypt = safexcel_aead_decrypt, 1754 .ivsize = AES_BLOCK_SIZE, 1755 .maxauthsize = SHA224_DIGEST_SIZE, 1756 .base = { 1757 .cra_name = "authenc(hmac(sha224),cbc(aes))", 1758 .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-aes", 1759 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1760 .cra_flags = CRYPTO_ALG_ASYNC | 1761 CRYPTO_ALG_ALLOCATES_MEMORY | 1762 CRYPTO_ALG_KERN_DRIVER_ONLY, 1763 .cra_blocksize = AES_BLOCK_SIZE, 1764 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1765 .cra_alignmask = 0, 1766 .cra_init = safexcel_aead_sha224_cra_init, 1767 .cra_exit = safexcel_aead_cra_exit, 1768 .cra_module = THIS_MODULE, 1769 }, 1770 }, 1771 }; 1772 1773 static int safexcel_aead_sha512_cra_init(struct crypto_tfm *tfm) 1774 { 1775 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1776 1777 safexcel_aead_cra_init(tfm); 1778 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512; 1779 ctx->state_sz = SHA512_DIGEST_SIZE; 1780 return 0; 1781 } 1782 1783 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_aes = { 1784 .type = SAFEXCEL_ALG_TYPE_AEAD, 1785 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512, 1786 .alg.aead = { 1787 .setkey = safexcel_aead_setkey, 1788 .encrypt = safexcel_aead_encrypt, 1789 .decrypt = safexcel_aead_decrypt, 1790 .ivsize = AES_BLOCK_SIZE, 1791 .maxauthsize = SHA512_DIGEST_SIZE, 1792 .base = { 1793 .cra_name = "authenc(hmac(sha512),cbc(aes))", 1794 .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-aes", 1795 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1796 .cra_flags = CRYPTO_ALG_ASYNC | 1797 CRYPTO_ALG_ALLOCATES_MEMORY | 1798 CRYPTO_ALG_KERN_DRIVER_ONLY, 1799 .cra_blocksize = AES_BLOCK_SIZE, 1800 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1801 .cra_alignmask = 0, 1802 .cra_init = safexcel_aead_sha512_cra_init, 1803 .cra_exit = safexcel_aead_cra_exit, 1804 .cra_module = THIS_MODULE, 1805 }, 1806 }, 1807 }; 1808 1809 static int safexcel_aead_sha384_cra_init(struct crypto_tfm *tfm) 1810 { 1811 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1812 1813 safexcel_aead_cra_init(tfm); 1814 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384; 1815 ctx->state_sz = SHA512_DIGEST_SIZE; 1816 return 0; 1817 } 1818 1819 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_aes = { 1820 .type = SAFEXCEL_ALG_TYPE_AEAD, 1821 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512, 1822 .alg.aead = { 1823 .setkey = safexcel_aead_setkey, 1824 .encrypt = safexcel_aead_encrypt, 1825 .decrypt = safexcel_aead_decrypt, 1826 .ivsize = AES_BLOCK_SIZE, 1827 .maxauthsize = SHA384_DIGEST_SIZE, 1828 .base = { 1829 .cra_name = "authenc(hmac(sha384),cbc(aes))", 1830 .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-aes", 1831 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1832 .cra_flags = CRYPTO_ALG_ASYNC | 1833 CRYPTO_ALG_ALLOCATES_MEMORY | 1834 CRYPTO_ALG_KERN_DRIVER_ONLY, 1835 .cra_blocksize = AES_BLOCK_SIZE, 1836 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1837 .cra_alignmask = 0, 1838 .cra_init = safexcel_aead_sha384_cra_init, 1839 .cra_exit = safexcel_aead_cra_exit, 1840 .cra_module = THIS_MODULE, 1841 }, 1842 }, 1843 }; 1844 1845 static int safexcel_aead_sha1_des3_cra_init(struct crypto_tfm *tfm) 1846 { 1847 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1848 1849 safexcel_aead_sha1_cra_init(tfm); 1850 ctx->alg = SAFEXCEL_3DES; /* override default */ 1851 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 1852 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1853 return 0; 1854 } 1855 1856 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des3_ede = { 1857 .type = SAFEXCEL_ALG_TYPE_AEAD, 1858 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA1, 1859 .alg.aead = { 1860 .setkey = safexcel_aead_setkey, 1861 .encrypt = safexcel_aead_encrypt, 1862 .decrypt = safexcel_aead_decrypt, 1863 .ivsize = DES3_EDE_BLOCK_SIZE, 1864 .maxauthsize = SHA1_DIGEST_SIZE, 1865 .base = { 1866 .cra_name = "authenc(hmac(sha1),cbc(des3_ede))", 1867 .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-des3_ede", 1868 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1869 .cra_flags = CRYPTO_ALG_ASYNC | 1870 CRYPTO_ALG_ALLOCATES_MEMORY | 1871 CRYPTO_ALG_KERN_DRIVER_ONLY, 1872 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1873 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1874 .cra_alignmask = 0, 1875 .cra_init = safexcel_aead_sha1_des3_cra_init, 1876 .cra_exit = safexcel_aead_cra_exit, 1877 .cra_module = THIS_MODULE, 1878 }, 1879 }, 1880 }; 1881 1882 static int safexcel_aead_sha256_des3_cra_init(struct crypto_tfm *tfm) 1883 { 1884 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1885 1886 safexcel_aead_sha256_cra_init(tfm); 1887 ctx->alg = SAFEXCEL_3DES; /* override default */ 1888 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 1889 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1890 return 0; 1891 } 1892 1893 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des3_ede = { 1894 .type = SAFEXCEL_ALG_TYPE_AEAD, 1895 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, 1896 .alg.aead = { 1897 .setkey = safexcel_aead_setkey, 1898 .encrypt = safexcel_aead_encrypt, 1899 .decrypt = safexcel_aead_decrypt, 1900 .ivsize = DES3_EDE_BLOCK_SIZE, 1901 .maxauthsize = SHA256_DIGEST_SIZE, 1902 .base = { 1903 .cra_name = "authenc(hmac(sha256),cbc(des3_ede))", 1904 .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-des3_ede", 1905 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1906 .cra_flags = CRYPTO_ALG_ASYNC | 1907 CRYPTO_ALG_ALLOCATES_MEMORY | 1908 CRYPTO_ALG_KERN_DRIVER_ONLY, 1909 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1910 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1911 .cra_alignmask = 0, 1912 .cra_init = safexcel_aead_sha256_des3_cra_init, 1913 .cra_exit = safexcel_aead_cra_exit, 1914 .cra_module = THIS_MODULE, 1915 }, 1916 }, 1917 }; 1918 1919 static int safexcel_aead_sha224_des3_cra_init(struct crypto_tfm *tfm) 1920 { 1921 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1922 1923 safexcel_aead_sha224_cra_init(tfm); 1924 ctx->alg = SAFEXCEL_3DES; /* override default */ 1925 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 1926 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1927 return 0; 1928 } 1929 1930 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des3_ede = { 1931 .type = SAFEXCEL_ALG_TYPE_AEAD, 1932 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, 1933 .alg.aead = { 1934 .setkey = safexcel_aead_setkey, 1935 .encrypt = safexcel_aead_encrypt, 1936 .decrypt = safexcel_aead_decrypt, 1937 .ivsize = DES3_EDE_BLOCK_SIZE, 1938 .maxauthsize = SHA224_DIGEST_SIZE, 1939 .base = { 1940 .cra_name = "authenc(hmac(sha224),cbc(des3_ede))", 1941 .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-des3_ede", 1942 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1943 .cra_flags = CRYPTO_ALG_ASYNC | 1944 CRYPTO_ALG_ALLOCATES_MEMORY | 1945 CRYPTO_ALG_KERN_DRIVER_ONLY, 1946 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1947 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1948 .cra_alignmask = 0, 1949 .cra_init = safexcel_aead_sha224_des3_cra_init, 1950 .cra_exit = safexcel_aead_cra_exit, 1951 .cra_module = THIS_MODULE, 1952 }, 1953 }, 1954 }; 1955 1956 static int safexcel_aead_sha512_des3_cra_init(struct crypto_tfm *tfm) 1957 { 1958 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1959 1960 safexcel_aead_sha512_cra_init(tfm); 1961 ctx->alg = SAFEXCEL_3DES; /* override default */ 1962 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 1963 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1964 return 0; 1965 } 1966 1967 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des3_ede = { 1968 .type = SAFEXCEL_ALG_TYPE_AEAD, 1969 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, 1970 .alg.aead = { 1971 .setkey = safexcel_aead_setkey, 1972 .encrypt = safexcel_aead_encrypt, 1973 .decrypt = safexcel_aead_decrypt, 1974 .ivsize = DES3_EDE_BLOCK_SIZE, 1975 .maxauthsize = SHA512_DIGEST_SIZE, 1976 .base = { 1977 .cra_name = "authenc(hmac(sha512),cbc(des3_ede))", 1978 .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-des3_ede", 1979 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1980 .cra_flags = CRYPTO_ALG_ASYNC | 1981 CRYPTO_ALG_ALLOCATES_MEMORY | 1982 CRYPTO_ALG_KERN_DRIVER_ONLY, 1983 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1984 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1985 .cra_alignmask = 0, 1986 .cra_init = safexcel_aead_sha512_des3_cra_init, 1987 .cra_exit = safexcel_aead_cra_exit, 1988 .cra_module = THIS_MODULE, 1989 }, 1990 }, 1991 }; 1992 1993 static int safexcel_aead_sha384_des3_cra_init(struct crypto_tfm *tfm) 1994 { 1995 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1996 1997 safexcel_aead_sha384_cra_init(tfm); 1998 ctx->alg = SAFEXCEL_3DES; /* override default */ 1999 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 2000 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2001 return 0; 2002 } 2003 2004 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des3_ede = { 2005 .type = SAFEXCEL_ALG_TYPE_AEAD, 2006 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, 2007 .alg.aead = { 2008 .setkey = safexcel_aead_setkey, 2009 .encrypt = safexcel_aead_encrypt, 2010 .decrypt = safexcel_aead_decrypt, 2011 .ivsize = DES3_EDE_BLOCK_SIZE, 2012 .maxauthsize = SHA384_DIGEST_SIZE, 2013 .base = { 2014 .cra_name = "authenc(hmac(sha384),cbc(des3_ede))", 2015 .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-des3_ede", 2016 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2017 .cra_flags = CRYPTO_ALG_ASYNC | 2018 CRYPTO_ALG_ALLOCATES_MEMORY | 2019 CRYPTO_ALG_KERN_DRIVER_ONLY, 2020 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2021 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2022 .cra_alignmask = 0, 2023 .cra_init = safexcel_aead_sha384_des3_cra_init, 2024 .cra_exit = safexcel_aead_cra_exit, 2025 .cra_module = THIS_MODULE, 2026 }, 2027 }, 2028 }; 2029 2030 static int safexcel_aead_sha1_des_cra_init(struct crypto_tfm *tfm) 2031 { 2032 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2033 2034 safexcel_aead_sha1_cra_init(tfm); 2035 ctx->alg = SAFEXCEL_DES; /* override default */ 2036 ctx->blocksz = DES_BLOCK_SIZE; 2037 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2038 return 0; 2039 } 2040 2041 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des = { 2042 .type = SAFEXCEL_ALG_TYPE_AEAD, 2043 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA1, 2044 .alg.aead = { 2045 .setkey = safexcel_aead_setkey, 2046 .encrypt = safexcel_aead_encrypt, 2047 .decrypt = safexcel_aead_decrypt, 2048 .ivsize = DES_BLOCK_SIZE, 2049 .maxauthsize = SHA1_DIGEST_SIZE, 2050 .base = { 2051 .cra_name = "authenc(hmac(sha1),cbc(des))", 2052 .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-des", 2053 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2054 .cra_flags = CRYPTO_ALG_ASYNC | 2055 CRYPTO_ALG_ALLOCATES_MEMORY | 2056 CRYPTO_ALG_KERN_DRIVER_ONLY, 2057 .cra_blocksize = DES_BLOCK_SIZE, 2058 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2059 .cra_alignmask = 0, 2060 .cra_init = safexcel_aead_sha1_des_cra_init, 2061 .cra_exit = safexcel_aead_cra_exit, 2062 .cra_module = THIS_MODULE, 2063 }, 2064 }, 2065 }; 2066 2067 static int safexcel_aead_sha256_des_cra_init(struct crypto_tfm *tfm) 2068 { 2069 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2070 2071 safexcel_aead_sha256_cra_init(tfm); 2072 ctx->alg = SAFEXCEL_DES; /* override default */ 2073 ctx->blocksz = DES_BLOCK_SIZE; 2074 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2075 return 0; 2076 } 2077 2078 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des = { 2079 .type = SAFEXCEL_ALG_TYPE_AEAD, 2080 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, 2081 .alg.aead = { 2082 .setkey = safexcel_aead_setkey, 2083 .encrypt = safexcel_aead_encrypt, 2084 .decrypt = safexcel_aead_decrypt, 2085 .ivsize = DES_BLOCK_SIZE, 2086 .maxauthsize = SHA256_DIGEST_SIZE, 2087 .base = { 2088 .cra_name = "authenc(hmac(sha256),cbc(des))", 2089 .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-des", 2090 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2091 .cra_flags = CRYPTO_ALG_ASYNC | 2092 CRYPTO_ALG_ALLOCATES_MEMORY | 2093 CRYPTO_ALG_KERN_DRIVER_ONLY, 2094 .cra_blocksize = DES_BLOCK_SIZE, 2095 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2096 .cra_alignmask = 0, 2097 .cra_init = safexcel_aead_sha256_des_cra_init, 2098 .cra_exit = safexcel_aead_cra_exit, 2099 .cra_module = THIS_MODULE, 2100 }, 2101 }, 2102 }; 2103 2104 static int safexcel_aead_sha224_des_cra_init(struct crypto_tfm *tfm) 2105 { 2106 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2107 2108 safexcel_aead_sha224_cra_init(tfm); 2109 ctx->alg = SAFEXCEL_DES; /* override default */ 2110 ctx->blocksz = DES_BLOCK_SIZE; 2111 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2112 return 0; 2113 } 2114 2115 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des = { 2116 .type = SAFEXCEL_ALG_TYPE_AEAD, 2117 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, 2118 .alg.aead = { 2119 .setkey = safexcel_aead_setkey, 2120 .encrypt = safexcel_aead_encrypt, 2121 .decrypt = safexcel_aead_decrypt, 2122 .ivsize = DES_BLOCK_SIZE, 2123 .maxauthsize = SHA224_DIGEST_SIZE, 2124 .base = { 2125 .cra_name = "authenc(hmac(sha224),cbc(des))", 2126 .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-des", 2127 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2128 .cra_flags = CRYPTO_ALG_ASYNC | 2129 CRYPTO_ALG_ALLOCATES_MEMORY | 2130 CRYPTO_ALG_KERN_DRIVER_ONLY, 2131 .cra_blocksize = DES_BLOCK_SIZE, 2132 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2133 .cra_alignmask = 0, 2134 .cra_init = safexcel_aead_sha224_des_cra_init, 2135 .cra_exit = safexcel_aead_cra_exit, 2136 .cra_module = THIS_MODULE, 2137 }, 2138 }, 2139 }; 2140 2141 static int safexcel_aead_sha512_des_cra_init(struct crypto_tfm *tfm) 2142 { 2143 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2144 2145 safexcel_aead_sha512_cra_init(tfm); 2146 ctx->alg = SAFEXCEL_DES; /* override default */ 2147 ctx->blocksz = DES_BLOCK_SIZE; 2148 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2149 return 0; 2150 } 2151 2152 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des = { 2153 .type = SAFEXCEL_ALG_TYPE_AEAD, 2154 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, 2155 .alg.aead = { 2156 .setkey = safexcel_aead_setkey, 2157 .encrypt = safexcel_aead_encrypt, 2158 .decrypt = safexcel_aead_decrypt, 2159 .ivsize = DES_BLOCK_SIZE, 2160 .maxauthsize = SHA512_DIGEST_SIZE, 2161 .base = { 2162 .cra_name = "authenc(hmac(sha512),cbc(des))", 2163 .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-des", 2164 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2165 .cra_flags = CRYPTO_ALG_ASYNC | 2166 CRYPTO_ALG_ALLOCATES_MEMORY | 2167 CRYPTO_ALG_KERN_DRIVER_ONLY, 2168 .cra_blocksize = DES_BLOCK_SIZE, 2169 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2170 .cra_alignmask = 0, 2171 .cra_init = safexcel_aead_sha512_des_cra_init, 2172 .cra_exit = safexcel_aead_cra_exit, 2173 .cra_module = THIS_MODULE, 2174 }, 2175 }, 2176 }; 2177 2178 static int safexcel_aead_sha384_des_cra_init(struct crypto_tfm *tfm) 2179 { 2180 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2181 2182 safexcel_aead_sha384_cra_init(tfm); 2183 ctx->alg = SAFEXCEL_DES; /* override default */ 2184 ctx->blocksz = DES_BLOCK_SIZE; 2185 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2186 return 0; 2187 } 2188 2189 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des = { 2190 .type = SAFEXCEL_ALG_TYPE_AEAD, 2191 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, 2192 .alg.aead = { 2193 .setkey = safexcel_aead_setkey, 2194 .encrypt = safexcel_aead_encrypt, 2195 .decrypt = safexcel_aead_decrypt, 2196 .ivsize = DES_BLOCK_SIZE, 2197 .maxauthsize = SHA384_DIGEST_SIZE, 2198 .base = { 2199 .cra_name = "authenc(hmac(sha384),cbc(des))", 2200 .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-des", 2201 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2202 .cra_flags = CRYPTO_ALG_ASYNC | 2203 CRYPTO_ALG_ALLOCATES_MEMORY | 2204 CRYPTO_ALG_KERN_DRIVER_ONLY, 2205 .cra_blocksize = DES_BLOCK_SIZE, 2206 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2207 .cra_alignmask = 0, 2208 .cra_init = safexcel_aead_sha384_des_cra_init, 2209 .cra_exit = safexcel_aead_cra_exit, 2210 .cra_module = THIS_MODULE, 2211 }, 2212 }, 2213 }; 2214 2215 static int safexcel_aead_sha1_ctr_cra_init(struct crypto_tfm *tfm) 2216 { 2217 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2218 2219 safexcel_aead_sha1_cra_init(tfm); 2220 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2221 return 0; 2222 } 2223 2224 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_ctr_aes = { 2225 .type = SAFEXCEL_ALG_TYPE_AEAD, 2226 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA1, 2227 .alg.aead = { 2228 .setkey = safexcel_aead_setkey, 2229 .encrypt = safexcel_aead_encrypt, 2230 .decrypt = safexcel_aead_decrypt, 2231 .ivsize = CTR_RFC3686_IV_SIZE, 2232 .maxauthsize = SHA1_DIGEST_SIZE, 2233 .base = { 2234 .cra_name = "authenc(hmac(sha1),rfc3686(ctr(aes)))", 2235 .cra_driver_name = "safexcel-authenc-hmac-sha1-ctr-aes", 2236 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2237 .cra_flags = CRYPTO_ALG_ASYNC | 2238 CRYPTO_ALG_ALLOCATES_MEMORY | 2239 CRYPTO_ALG_KERN_DRIVER_ONLY, 2240 .cra_blocksize = 1, 2241 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2242 .cra_alignmask = 0, 2243 .cra_init = safexcel_aead_sha1_ctr_cra_init, 2244 .cra_exit = safexcel_aead_cra_exit, 2245 .cra_module = THIS_MODULE, 2246 }, 2247 }, 2248 }; 2249 2250 static int safexcel_aead_sha256_ctr_cra_init(struct crypto_tfm *tfm) 2251 { 2252 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2253 2254 safexcel_aead_sha256_cra_init(tfm); 2255 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2256 return 0; 2257 } 2258 2259 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_ctr_aes = { 2260 .type = SAFEXCEL_ALG_TYPE_AEAD, 2261 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256, 2262 .alg.aead = { 2263 .setkey = safexcel_aead_setkey, 2264 .encrypt = safexcel_aead_encrypt, 2265 .decrypt = safexcel_aead_decrypt, 2266 .ivsize = CTR_RFC3686_IV_SIZE, 2267 .maxauthsize = SHA256_DIGEST_SIZE, 2268 .base = { 2269 .cra_name = "authenc(hmac(sha256),rfc3686(ctr(aes)))", 2270 .cra_driver_name = "safexcel-authenc-hmac-sha256-ctr-aes", 2271 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2272 .cra_flags = CRYPTO_ALG_ASYNC | 2273 CRYPTO_ALG_ALLOCATES_MEMORY | 2274 CRYPTO_ALG_KERN_DRIVER_ONLY, 2275 .cra_blocksize = 1, 2276 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2277 .cra_alignmask = 0, 2278 .cra_init = safexcel_aead_sha256_ctr_cra_init, 2279 .cra_exit = safexcel_aead_cra_exit, 2280 .cra_module = THIS_MODULE, 2281 }, 2282 }, 2283 }; 2284 2285 static int safexcel_aead_sha224_ctr_cra_init(struct crypto_tfm *tfm) 2286 { 2287 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2288 2289 safexcel_aead_sha224_cra_init(tfm); 2290 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2291 return 0; 2292 } 2293 2294 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_ctr_aes = { 2295 .type = SAFEXCEL_ALG_TYPE_AEAD, 2296 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256, 2297 .alg.aead = { 2298 .setkey = safexcel_aead_setkey, 2299 .encrypt = safexcel_aead_encrypt, 2300 .decrypt = safexcel_aead_decrypt, 2301 .ivsize = CTR_RFC3686_IV_SIZE, 2302 .maxauthsize = SHA224_DIGEST_SIZE, 2303 .base = { 2304 .cra_name = "authenc(hmac(sha224),rfc3686(ctr(aes)))", 2305 .cra_driver_name = "safexcel-authenc-hmac-sha224-ctr-aes", 2306 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2307 .cra_flags = CRYPTO_ALG_ASYNC | 2308 CRYPTO_ALG_ALLOCATES_MEMORY | 2309 CRYPTO_ALG_KERN_DRIVER_ONLY, 2310 .cra_blocksize = 1, 2311 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2312 .cra_alignmask = 0, 2313 .cra_init = safexcel_aead_sha224_ctr_cra_init, 2314 .cra_exit = safexcel_aead_cra_exit, 2315 .cra_module = THIS_MODULE, 2316 }, 2317 }, 2318 }; 2319 2320 static int safexcel_aead_sha512_ctr_cra_init(struct crypto_tfm *tfm) 2321 { 2322 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2323 2324 safexcel_aead_sha512_cra_init(tfm); 2325 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2326 return 0; 2327 } 2328 2329 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_ctr_aes = { 2330 .type = SAFEXCEL_ALG_TYPE_AEAD, 2331 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512, 2332 .alg.aead = { 2333 .setkey = safexcel_aead_setkey, 2334 .encrypt = safexcel_aead_encrypt, 2335 .decrypt = safexcel_aead_decrypt, 2336 .ivsize = CTR_RFC3686_IV_SIZE, 2337 .maxauthsize = SHA512_DIGEST_SIZE, 2338 .base = { 2339 .cra_name = "authenc(hmac(sha512),rfc3686(ctr(aes)))", 2340 .cra_driver_name = "safexcel-authenc-hmac-sha512-ctr-aes", 2341 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2342 .cra_flags = CRYPTO_ALG_ASYNC | 2343 CRYPTO_ALG_ALLOCATES_MEMORY | 2344 CRYPTO_ALG_KERN_DRIVER_ONLY, 2345 .cra_blocksize = 1, 2346 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2347 .cra_alignmask = 0, 2348 .cra_init = safexcel_aead_sha512_ctr_cra_init, 2349 .cra_exit = safexcel_aead_cra_exit, 2350 .cra_module = THIS_MODULE, 2351 }, 2352 }, 2353 }; 2354 2355 static int safexcel_aead_sha384_ctr_cra_init(struct crypto_tfm *tfm) 2356 { 2357 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2358 2359 safexcel_aead_sha384_cra_init(tfm); 2360 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2361 return 0; 2362 } 2363 2364 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_ctr_aes = { 2365 .type = SAFEXCEL_ALG_TYPE_AEAD, 2366 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512, 2367 .alg.aead = { 2368 .setkey = safexcel_aead_setkey, 2369 .encrypt = safexcel_aead_encrypt, 2370 .decrypt = safexcel_aead_decrypt, 2371 .ivsize = CTR_RFC3686_IV_SIZE, 2372 .maxauthsize = SHA384_DIGEST_SIZE, 2373 .base = { 2374 .cra_name = "authenc(hmac(sha384),rfc3686(ctr(aes)))", 2375 .cra_driver_name = "safexcel-authenc-hmac-sha384-ctr-aes", 2376 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2377 .cra_flags = CRYPTO_ALG_ASYNC | 2378 CRYPTO_ALG_ALLOCATES_MEMORY | 2379 CRYPTO_ALG_KERN_DRIVER_ONLY, 2380 .cra_blocksize = 1, 2381 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2382 .cra_alignmask = 0, 2383 .cra_init = safexcel_aead_sha384_ctr_cra_init, 2384 .cra_exit = safexcel_aead_cra_exit, 2385 .cra_module = THIS_MODULE, 2386 }, 2387 }, 2388 }; 2389 2390 static int safexcel_skcipher_aesxts_setkey(struct crypto_skcipher *ctfm, 2391 const u8 *key, unsigned int len) 2392 { 2393 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 2394 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2395 struct safexcel_crypto_priv *priv = ctx->base.priv; 2396 struct crypto_aes_ctx aes; 2397 int ret, i; 2398 unsigned int keylen; 2399 2400 /* Check for illegal XTS keys */ 2401 ret = xts_verify_key(ctfm, key, len); 2402 if (ret) 2403 return ret; 2404 2405 /* Only half of the key data is cipher key */ 2406 keylen = (len >> 1); 2407 ret = aes_expandkey(&aes, key, keylen); 2408 if (ret) 2409 return ret; 2410 2411 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2412 for (i = 0; i < keylen / sizeof(u32); i++) { 2413 if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { 2414 ctx->base.needs_inv = true; 2415 break; 2416 } 2417 } 2418 } 2419 2420 for (i = 0; i < keylen / sizeof(u32); i++) 2421 ctx->key[i] = cpu_to_le32(aes.key_enc[i]); 2422 2423 /* The other half is the tweak key */ 2424 ret = aes_expandkey(&aes, (u8 *)(key + keylen), keylen); 2425 if (ret) 2426 return ret; 2427 2428 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2429 for (i = 0; i < keylen / sizeof(u32); i++) { 2430 if (le32_to_cpu(ctx->key[i + keylen / sizeof(u32)]) != 2431 aes.key_enc[i]) { 2432 ctx->base.needs_inv = true; 2433 break; 2434 } 2435 } 2436 } 2437 2438 for (i = 0; i < keylen / sizeof(u32); i++) 2439 ctx->key[i + keylen / sizeof(u32)] = 2440 cpu_to_le32(aes.key_enc[i]); 2441 2442 ctx->key_len = keylen << 1; 2443 2444 memzero_explicit(&aes, sizeof(aes)); 2445 return 0; 2446 } 2447 2448 static int safexcel_skcipher_aes_xts_cra_init(struct crypto_tfm *tfm) 2449 { 2450 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2451 2452 safexcel_skcipher_cra_init(tfm); 2453 ctx->alg = SAFEXCEL_AES; 2454 ctx->blocksz = AES_BLOCK_SIZE; 2455 ctx->xts = 1; 2456 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XTS; 2457 return 0; 2458 } 2459 2460 static int safexcel_encrypt_xts(struct skcipher_request *req) 2461 { 2462 if (req->cryptlen < XTS_BLOCK_SIZE) 2463 return -EINVAL; 2464 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 2465 SAFEXCEL_ENCRYPT); 2466 } 2467 2468 static int safexcel_decrypt_xts(struct skcipher_request *req) 2469 { 2470 if (req->cryptlen < XTS_BLOCK_SIZE) 2471 return -EINVAL; 2472 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 2473 SAFEXCEL_DECRYPT); 2474 } 2475 2476 struct safexcel_alg_template safexcel_alg_xts_aes = { 2477 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 2478 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_AES_XTS, 2479 .alg.skcipher = { 2480 .setkey = safexcel_skcipher_aesxts_setkey, 2481 .encrypt = safexcel_encrypt_xts, 2482 .decrypt = safexcel_decrypt_xts, 2483 /* XTS actually uses 2 AES keys glued together */ 2484 .min_keysize = AES_MIN_KEY_SIZE * 2, 2485 .max_keysize = AES_MAX_KEY_SIZE * 2, 2486 .ivsize = XTS_BLOCK_SIZE, 2487 .base = { 2488 .cra_name = "xts(aes)", 2489 .cra_driver_name = "safexcel-xts-aes", 2490 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2491 .cra_flags = CRYPTO_ALG_ASYNC | 2492 CRYPTO_ALG_ALLOCATES_MEMORY | 2493 CRYPTO_ALG_KERN_DRIVER_ONLY, 2494 .cra_blocksize = XTS_BLOCK_SIZE, 2495 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2496 .cra_alignmask = 0, 2497 .cra_init = safexcel_skcipher_aes_xts_cra_init, 2498 .cra_exit = safexcel_skcipher_cra_exit, 2499 .cra_module = THIS_MODULE, 2500 }, 2501 }, 2502 }; 2503 2504 static int safexcel_aead_gcm_setkey(struct crypto_aead *ctfm, const u8 *key, 2505 unsigned int len) 2506 { 2507 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 2508 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2509 struct safexcel_crypto_priv *priv = ctx->base.priv; 2510 struct aes_enckey aes; 2511 u32 hashkey[AES_BLOCK_SIZE >> 2]; 2512 int ret, i; 2513 2514 ret = aes_prepareenckey(&aes, key, len); 2515 if (ret) 2516 return ret; 2517 2518 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2519 for (i = 0; i < len / sizeof(u32); i++) { 2520 if (ctx->key[i] != get_unaligned((__le32 *)key + i)) { 2521 ctx->base.needs_inv = true; 2522 break; 2523 } 2524 } 2525 } 2526 2527 for (i = 0; i < len / sizeof(u32); i++) 2528 ctx->key[i] = get_unaligned((__le32 *)key + i); 2529 2530 ctx->key_len = len; 2531 2532 /* Compute hash key by encrypting zeroes with cipher key */ 2533 memset(hashkey, 0, AES_BLOCK_SIZE); 2534 aes_encrypt(&aes, (u8 *)hashkey, (u8 *)hashkey); 2535 2536 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2537 for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) { 2538 if (be32_to_cpu(ctx->base.ipad.be[i]) != hashkey[i]) { 2539 ctx->base.needs_inv = true; 2540 break; 2541 } 2542 } 2543 } 2544 2545 for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) 2546 ctx->base.ipad.be[i] = cpu_to_be32(hashkey[i]); 2547 2548 memzero_explicit(hashkey, AES_BLOCK_SIZE); 2549 memzero_explicit(&aes, sizeof(aes)); 2550 return 0; 2551 } 2552 2553 static int safexcel_aead_gcm_cra_init(struct crypto_tfm *tfm) 2554 { 2555 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2556 2557 safexcel_aead_cra_init(tfm); 2558 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_GHASH; 2559 ctx->state_sz = GHASH_BLOCK_SIZE; 2560 ctx->xcm = EIP197_XCM_MODE_GCM; 2561 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XCM; /* override default */ 2562 2563 return 0; 2564 } 2565 2566 static void safexcel_aead_gcm_cra_exit(struct crypto_tfm *tfm) 2567 { 2568 safexcel_aead_cra_exit(tfm); 2569 } 2570 2571 static int safexcel_aead_gcm_setauthsize(struct crypto_aead *tfm, 2572 unsigned int authsize) 2573 { 2574 return crypto_gcm_check_authsize(authsize); 2575 } 2576 2577 struct safexcel_alg_template safexcel_alg_gcm = { 2578 .type = SAFEXCEL_ALG_TYPE_AEAD, 2579 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH, 2580 .alg.aead = { 2581 .setkey = safexcel_aead_gcm_setkey, 2582 .setauthsize = safexcel_aead_gcm_setauthsize, 2583 .encrypt = safexcel_aead_encrypt, 2584 .decrypt = safexcel_aead_decrypt, 2585 .ivsize = GCM_AES_IV_SIZE, 2586 .maxauthsize = GHASH_DIGEST_SIZE, 2587 .base = { 2588 .cra_name = "gcm(aes)", 2589 .cra_driver_name = "safexcel-gcm-aes", 2590 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2591 .cra_flags = CRYPTO_ALG_ASYNC | 2592 CRYPTO_ALG_ALLOCATES_MEMORY | 2593 CRYPTO_ALG_KERN_DRIVER_ONLY, 2594 .cra_blocksize = 1, 2595 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2596 .cra_alignmask = 0, 2597 .cra_init = safexcel_aead_gcm_cra_init, 2598 .cra_exit = safexcel_aead_gcm_cra_exit, 2599 .cra_module = THIS_MODULE, 2600 }, 2601 }, 2602 }; 2603 2604 static int safexcel_aead_ccm_setkey(struct crypto_aead *ctfm, const u8 *key, 2605 unsigned int len) 2606 { 2607 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 2608 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2609 struct safexcel_crypto_priv *priv = ctx->base.priv; 2610 struct crypto_aes_ctx aes; 2611 int ret, i; 2612 2613 ret = aes_expandkey(&aes, key, len); 2614 if (ret) { 2615 memzero_explicit(&aes, sizeof(aes)); 2616 return ret; 2617 } 2618 2619 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2620 for (i = 0; i < len / sizeof(u32); i++) { 2621 if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { 2622 ctx->base.needs_inv = true; 2623 break; 2624 } 2625 } 2626 } 2627 2628 for (i = 0; i < len / sizeof(u32); i++) { 2629 ctx->key[i] = cpu_to_le32(aes.key_enc[i]); 2630 ctx->base.ipad.be[i + 2 * AES_BLOCK_SIZE / sizeof(u32)] = 2631 cpu_to_be32(aes.key_enc[i]); 2632 } 2633 2634 ctx->key_len = len; 2635 ctx->state_sz = 2 * AES_BLOCK_SIZE + len; 2636 2637 if (len == AES_KEYSIZE_192) 2638 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192; 2639 else if (len == AES_KEYSIZE_256) 2640 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256; 2641 else 2642 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128; 2643 2644 memzero_explicit(&aes, sizeof(aes)); 2645 return 0; 2646 } 2647 2648 static int safexcel_aead_ccm_cra_init(struct crypto_tfm *tfm) 2649 { 2650 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2651 2652 safexcel_aead_cra_init(tfm); 2653 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128; 2654 ctx->state_sz = 3 * AES_BLOCK_SIZE; 2655 ctx->xcm = EIP197_XCM_MODE_CCM; 2656 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XCM; /* override default */ 2657 ctx->ctrinit = 0; 2658 return 0; 2659 } 2660 2661 static int safexcel_aead_ccm_setauthsize(struct crypto_aead *tfm, 2662 unsigned int authsize) 2663 { 2664 /* Borrowed from crypto/ccm.c */ 2665 switch (authsize) { 2666 case 4: 2667 case 6: 2668 case 8: 2669 case 10: 2670 case 12: 2671 case 14: 2672 case 16: 2673 break; 2674 default: 2675 return -EINVAL; 2676 } 2677 2678 return 0; 2679 } 2680 2681 static int safexcel_ccm_encrypt(struct aead_request *req) 2682 { 2683 struct safexcel_cipher_req *creq = aead_request_ctx(req); 2684 2685 if (req->iv[0] < 1 || req->iv[0] > 7) 2686 return -EINVAL; 2687 2688 return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); 2689 } 2690 2691 static int safexcel_ccm_decrypt(struct aead_request *req) 2692 { 2693 struct safexcel_cipher_req *creq = aead_request_ctx(req); 2694 2695 if (req->iv[0] < 1 || req->iv[0] > 7) 2696 return -EINVAL; 2697 2698 return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); 2699 } 2700 2701 struct safexcel_alg_template safexcel_alg_ccm = { 2702 .type = SAFEXCEL_ALG_TYPE_AEAD, 2703 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_CBC_MAC_ALL, 2704 .alg.aead = { 2705 .setkey = safexcel_aead_ccm_setkey, 2706 .setauthsize = safexcel_aead_ccm_setauthsize, 2707 .encrypt = safexcel_ccm_encrypt, 2708 .decrypt = safexcel_ccm_decrypt, 2709 .ivsize = AES_BLOCK_SIZE, 2710 .maxauthsize = AES_BLOCK_SIZE, 2711 .base = { 2712 .cra_name = "ccm(aes)", 2713 .cra_driver_name = "safexcel-ccm-aes", 2714 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2715 .cra_flags = CRYPTO_ALG_ASYNC | 2716 CRYPTO_ALG_ALLOCATES_MEMORY | 2717 CRYPTO_ALG_KERN_DRIVER_ONLY, 2718 .cra_blocksize = 1, 2719 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2720 .cra_alignmask = 0, 2721 .cra_init = safexcel_aead_ccm_cra_init, 2722 .cra_exit = safexcel_aead_cra_exit, 2723 .cra_module = THIS_MODULE, 2724 }, 2725 }, 2726 }; 2727 2728 static void safexcel_chacha20_setkey(struct safexcel_cipher_ctx *ctx, 2729 const u8 *key) 2730 { 2731 struct safexcel_crypto_priv *priv = ctx->base.priv; 2732 2733 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) 2734 if (memcmp(ctx->key, key, CHACHA_KEY_SIZE)) 2735 ctx->base.needs_inv = true; 2736 2737 memcpy(ctx->key, key, CHACHA_KEY_SIZE); 2738 ctx->key_len = CHACHA_KEY_SIZE; 2739 } 2740 2741 static int safexcel_skcipher_chacha20_setkey(struct crypto_skcipher *ctfm, 2742 const u8 *key, unsigned int len) 2743 { 2744 struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm); 2745 2746 if (len != CHACHA_KEY_SIZE) 2747 return -EINVAL; 2748 2749 safexcel_chacha20_setkey(ctx, key); 2750 2751 return 0; 2752 } 2753 2754 static int safexcel_skcipher_chacha20_cra_init(struct crypto_tfm *tfm) 2755 { 2756 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2757 2758 safexcel_skcipher_cra_init(tfm); 2759 ctx->alg = SAFEXCEL_CHACHA20; 2760 ctx->ctrinit = 0; 2761 ctx->mode = CONTEXT_CONTROL_CHACHA20_MODE_256_32; 2762 return 0; 2763 } 2764 2765 struct safexcel_alg_template safexcel_alg_chacha20 = { 2766 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 2767 .algo_mask = SAFEXCEL_ALG_CHACHA20, 2768 .alg.skcipher = { 2769 .setkey = safexcel_skcipher_chacha20_setkey, 2770 .encrypt = safexcel_encrypt, 2771 .decrypt = safexcel_decrypt, 2772 .min_keysize = CHACHA_KEY_SIZE, 2773 .max_keysize = CHACHA_KEY_SIZE, 2774 .ivsize = CHACHA_IV_SIZE, 2775 .base = { 2776 .cra_name = "chacha20", 2777 .cra_driver_name = "safexcel-chacha20", 2778 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2779 .cra_flags = CRYPTO_ALG_ASYNC | 2780 CRYPTO_ALG_ALLOCATES_MEMORY | 2781 CRYPTO_ALG_KERN_DRIVER_ONLY, 2782 .cra_blocksize = 1, 2783 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2784 .cra_alignmask = 0, 2785 .cra_init = safexcel_skcipher_chacha20_cra_init, 2786 .cra_exit = safexcel_skcipher_cra_exit, 2787 .cra_module = THIS_MODULE, 2788 }, 2789 }, 2790 }; 2791 2792 static int safexcel_aead_chachapoly_setkey(struct crypto_aead *ctfm, 2793 const u8 *key, unsigned int len) 2794 { 2795 struct safexcel_cipher_ctx *ctx = crypto_aead_ctx(ctfm); 2796 2797 if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP && 2798 len > EIP197_AEAD_IPSEC_NONCE_SIZE) { 2799 /* ESP variant has nonce appended to key */ 2800 len -= EIP197_AEAD_IPSEC_NONCE_SIZE; 2801 ctx->nonce = *(u32 *)(key + len); 2802 } 2803 if (len != CHACHA_KEY_SIZE) 2804 return -EINVAL; 2805 2806 safexcel_chacha20_setkey(ctx, key); 2807 2808 return 0; 2809 } 2810 2811 static int safexcel_aead_chachapoly_setauthsize(struct crypto_aead *tfm, 2812 unsigned int authsize) 2813 { 2814 if (authsize != POLY1305_DIGEST_SIZE) 2815 return -EINVAL; 2816 return 0; 2817 } 2818 2819 static int safexcel_aead_chachapoly_crypt(struct aead_request *req, 2820 enum safexcel_cipher_direction dir) 2821 { 2822 struct safexcel_cipher_req *creq = aead_request_ctx(req); 2823 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2824 struct crypto_tfm *tfm = crypto_aead_tfm(aead); 2825 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2826 struct aead_request *subreq = aead_request_ctx(req); 2827 u32 key[CHACHA_KEY_SIZE / sizeof(u32) + 1]; 2828 int ret = 0; 2829 2830 /* 2831 * Instead of wasting time detecting umpteen silly corner cases, 2832 * just dump all "small" requests to the fallback implementation. 2833 * HW would not be faster on such small requests anyway. 2834 */ 2835 if (likely((ctx->aead != EIP197_AEAD_TYPE_IPSEC_ESP || 2836 req->assoclen >= EIP197_AEAD_IPSEC_IV_SIZE) && 2837 req->cryptlen > POLY1305_DIGEST_SIZE)) { 2838 return safexcel_queue_req(&req->base, creq, dir); 2839 } 2840 2841 /* HW cannot do full (AAD+payload) zero length, use fallback */ 2842 memcpy(key, ctx->key, CHACHA_KEY_SIZE); 2843 if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { 2844 /* ESP variant has nonce appended to the key */ 2845 key[CHACHA_KEY_SIZE / sizeof(u32)] = ctx->nonce; 2846 ret = crypto_aead_setkey(ctx->fback, (u8 *)key, 2847 CHACHA_KEY_SIZE + 2848 EIP197_AEAD_IPSEC_NONCE_SIZE); 2849 } else { 2850 ret = crypto_aead_setkey(ctx->fback, (u8 *)key, 2851 CHACHA_KEY_SIZE); 2852 } 2853 if (ret) { 2854 crypto_aead_clear_flags(aead, CRYPTO_TFM_REQ_MASK); 2855 crypto_aead_set_flags(aead, crypto_aead_get_flags(ctx->fback) & 2856 CRYPTO_TFM_REQ_MASK); 2857 return ret; 2858 } 2859 2860 aead_request_set_tfm(subreq, ctx->fback); 2861 aead_request_set_callback(subreq, req->base.flags, req->base.complete, 2862 req->base.data); 2863 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, 2864 req->iv); 2865 aead_request_set_ad(subreq, req->assoclen); 2866 2867 return (dir == SAFEXCEL_ENCRYPT) ? 2868 crypto_aead_encrypt(subreq) : 2869 crypto_aead_decrypt(subreq); 2870 } 2871 2872 static int safexcel_aead_chachapoly_encrypt(struct aead_request *req) 2873 { 2874 return safexcel_aead_chachapoly_crypt(req, SAFEXCEL_ENCRYPT); 2875 } 2876 2877 static int safexcel_aead_chachapoly_decrypt(struct aead_request *req) 2878 { 2879 return safexcel_aead_chachapoly_crypt(req, SAFEXCEL_DECRYPT); 2880 } 2881 2882 static int safexcel_aead_fallback_cra_init(struct crypto_tfm *tfm) 2883 { 2884 struct crypto_aead *aead = __crypto_aead_cast(tfm); 2885 struct aead_alg *alg = crypto_aead_alg(aead); 2886 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2887 2888 safexcel_aead_cra_init(tfm); 2889 2890 /* Allocate fallback implementation */ 2891 ctx->fback = crypto_alloc_aead(alg->base.cra_name, 0, 2892 CRYPTO_ALG_ASYNC | 2893 CRYPTO_ALG_NEED_FALLBACK); 2894 if (IS_ERR(ctx->fback)) 2895 return PTR_ERR(ctx->fback); 2896 2897 crypto_aead_set_reqsize(aead, max(sizeof(struct safexcel_cipher_req), 2898 sizeof(struct aead_request) + 2899 crypto_aead_reqsize(ctx->fback))); 2900 2901 return 0; 2902 } 2903 2904 static int safexcel_aead_chachapoly_cra_init(struct crypto_tfm *tfm) 2905 { 2906 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2907 2908 safexcel_aead_fallback_cra_init(tfm); 2909 ctx->alg = SAFEXCEL_CHACHA20; 2910 ctx->mode = CONTEXT_CONTROL_CHACHA20_MODE_256_32 | 2911 CONTEXT_CONTROL_CHACHA20_MODE_CALC_OTK; 2912 ctx->ctrinit = 0; 2913 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_POLY1305; 2914 ctx->state_sz = 0; /* Precomputed by HW */ 2915 return 0; 2916 } 2917 2918 static void safexcel_aead_fallback_cra_exit(struct crypto_tfm *tfm) 2919 { 2920 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2921 2922 crypto_free_aead(ctx->fback); 2923 safexcel_aead_cra_exit(tfm); 2924 } 2925 2926 struct safexcel_alg_template safexcel_alg_chachapoly = { 2927 .type = SAFEXCEL_ALG_TYPE_AEAD, 2928 .algo_mask = SAFEXCEL_ALG_CHACHA20 | SAFEXCEL_ALG_POLY1305, 2929 .alg.aead = { 2930 .setkey = safexcel_aead_chachapoly_setkey, 2931 .setauthsize = safexcel_aead_chachapoly_setauthsize, 2932 .encrypt = safexcel_aead_chachapoly_encrypt, 2933 .decrypt = safexcel_aead_chachapoly_decrypt, 2934 .ivsize = CHACHAPOLY_IV_SIZE, 2935 .maxauthsize = POLY1305_DIGEST_SIZE, 2936 .base = { 2937 .cra_name = "rfc7539(chacha20,poly1305)", 2938 .cra_driver_name = "safexcel-chacha20-poly1305", 2939 /* +1 to put it above HW chacha + SW poly */ 2940 .cra_priority = SAFEXCEL_CRA_PRIORITY + 1, 2941 .cra_flags = CRYPTO_ALG_ASYNC | 2942 CRYPTO_ALG_ALLOCATES_MEMORY | 2943 CRYPTO_ALG_KERN_DRIVER_ONLY | 2944 CRYPTO_ALG_NEED_FALLBACK, 2945 .cra_blocksize = 1, 2946 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2947 .cra_alignmask = 0, 2948 .cra_init = safexcel_aead_chachapoly_cra_init, 2949 .cra_exit = safexcel_aead_fallback_cra_exit, 2950 .cra_module = THIS_MODULE, 2951 }, 2952 }, 2953 }; 2954 2955 static int safexcel_aead_chachapolyesp_cra_init(struct crypto_tfm *tfm) 2956 { 2957 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2958 int ret; 2959 2960 ret = safexcel_aead_chachapoly_cra_init(tfm); 2961 ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP; 2962 ctx->aadskip = EIP197_AEAD_IPSEC_IV_SIZE; 2963 return ret; 2964 } 2965 2966 struct safexcel_alg_template safexcel_alg_chachapoly_esp = { 2967 .type = SAFEXCEL_ALG_TYPE_AEAD, 2968 .algo_mask = SAFEXCEL_ALG_CHACHA20 | SAFEXCEL_ALG_POLY1305, 2969 .alg.aead = { 2970 .setkey = safexcel_aead_chachapoly_setkey, 2971 .setauthsize = safexcel_aead_chachapoly_setauthsize, 2972 .encrypt = safexcel_aead_chachapoly_encrypt, 2973 .decrypt = safexcel_aead_chachapoly_decrypt, 2974 .ivsize = CHACHAPOLY_IV_SIZE - EIP197_AEAD_IPSEC_NONCE_SIZE, 2975 .maxauthsize = POLY1305_DIGEST_SIZE, 2976 .base = { 2977 .cra_name = "rfc7539esp(chacha20,poly1305)", 2978 .cra_driver_name = "safexcel-chacha20-poly1305-esp", 2979 /* +1 to put it above HW chacha + SW poly */ 2980 .cra_priority = SAFEXCEL_CRA_PRIORITY + 1, 2981 .cra_flags = CRYPTO_ALG_ASYNC | 2982 CRYPTO_ALG_ALLOCATES_MEMORY | 2983 CRYPTO_ALG_KERN_DRIVER_ONLY | 2984 CRYPTO_ALG_NEED_FALLBACK, 2985 .cra_blocksize = 1, 2986 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2987 .cra_alignmask = 0, 2988 .cra_init = safexcel_aead_chachapolyesp_cra_init, 2989 .cra_exit = safexcel_aead_fallback_cra_exit, 2990 .cra_module = THIS_MODULE, 2991 }, 2992 }, 2993 }; 2994 2995 static int safexcel_skcipher_sm4_setkey(struct crypto_skcipher *ctfm, 2996 const u8 *key, unsigned int len) 2997 { 2998 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 2999 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3000 struct safexcel_crypto_priv *priv = ctx->base.priv; 3001 3002 if (len != SM4_KEY_SIZE) 3003 return -EINVAL; 3004 3005 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) 3006 if (memcmp(ctx->key, key, SM4_KEY_SIZE)) 3007 ctx->base.needs_inv = true; 3008 3009 memcpy(ctx->key, key, SM4_KEY_SIZE); 3010 ctx->key_len = SM4_KEY_SIZE; 3011 3012 return 0; 3013 } 3014 3015 static int safexcel_sm4_blk_encrypt(struct skcipher_request *req) 3016 { 3017 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3018 if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) 3019 return -EINVAL; 3020 else 3021 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 3022 SAFEXCEL_ENCRYPT); 3023 } 3024 3025 static int safexcel_sm4_blk_decrypt(struct skcipher_request *req) 3026 { 3027 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3028 if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) 3029 return -EINVAL; 3030 else 3031 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 3032 SAFEXCEL_DECRYPT); 3033 } 3034 3035 static int safexcel_skcipher_sm4_ecb_cra_init(struct crypto_tfm *tfm) 3036 { 3037 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3038 3039 safexcel_skcipher_cra_init(tfm); 3040 ctx->alg = SAFEXCEL_SM4; 3041 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; 3042 ctx->blocksz = 0; 3043 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 3044 return 0; 3045 } 3046 3047 struct safexcel_alg_template safexcel_alg_ecb_sm4 = { 3048 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 3049 .algo_mask = SAFEXCEL_ALG_SM4, 3050 .alg.skcipher = { 3051 .setkey = safexcel_skcipher_sm4_setkey, 3052 .encrypt = safexcel_sm4_blk_encrypt, 3053 .decrypt = safexcel_sm4_blk_decrypt, 3054 .min_keysize = SM4_KEY_SIZE, 3055 .max_keysize = SM4_KEY_SIZE, 3056 .base = { 3057 .cra_name = "ecb(sm4)", 3058 .cra_driver_name = "safexcel-ecb-sm4", 3059 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3060 .cra_flags = CRYPTO_ALG_ASYNC | 3061 CRYPTO_ALG_ALLOCATES_MEMORY | 3062 CRYPTO_ALG_KERN_DRIVER_ONLY, 3063 .cra_blocksize = SM4_BLOCK_SIZE, 3064 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3065 .cra_alignmask = 0, 3066 .cra_init = safexcel_skcipher_sm4_ecb_cra_init, 3067 .cra_exit = safexcel_skcipher_cra_exit, 3068 .cra_module = THIS_MODULE, 3069 }, 3070 }, 3071 }; 3072 3073 static int safexcel_skcipher_sm4_cbc_cra_init(struct crypto_tfm *tfm) 3074 { 3075 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3076 3077 safexcel_skcipher_cra_init(tfm); 3078 ctx->alg = SAFEXCEL_SM4; 3079 ctx->blocksz = SM4_BLOCK_SIZE; 3080 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; 3081 return 0; 3082 } 3083 3084 struct safexcel_alg_template safexcel_alg_cbc_sm4 = { 3085 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 3086 .algo_mask = SAFEXCEL_ALG_SM4, 3087 .alg.skcipher = { 3088 .setkey = safexcel_skcipher_sm4_setkey, 3089 .encrypt = safexcel_sm4_blk_encrypt, 3090 .decrypt = safexcel_sm4_blk_decrypt, 3091 .min_keysize = SM4_KEY_SIZE, 3092 .max_keysize = SM4_KEY_SIZE, 3093 .ivsize = SM4_BLOCK_SIZE, 3094 .base = { 3095 .cra_name = "cbc(sm4)", 3096 .cra_driver_name = "safexcel-cbc-sm4", 3097 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3098 .cra_flags = CRYPTO_ALG_ASYNC | 3099 CRYPTO_ALG_ALLOCATES_MEMORY | 3100 CRYPTO_ALG_KERN_DRIVER_ONLY, 3101 .cra_blocksize = SM4_BLOCK_SIZE, 3102 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3103 .cra_alignmask = 0, 3104 .cra_init = safexcel_skcipher_sm4_cbc_cra_init, 3105 .cra_exit = safexcel_skcipher_cra_exit, 3106 .cra_module = THIS_MODULE, 3107 }, 3108 }, 3109 }; 3110 3111 static int safexcel_skcipher_sm4ctr_setkey(struct crypto_skcipher *ctfm, 3112 const u8 *key, unsigned int len) 3113 { 3114 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 3115 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3116 3117 /* last 4 bytes of key are the nonce! */ 3118 ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE); 3119 /* exclude the nonce here */ 3120 len -= CTR_RFC3686_NONCE_SIZE; 3121 3122 return safexcel_skcipher_sm4_setkey(ctfm, key, len); 3123 } 3124 3125 static int safexcel_skcipher_sm4_ctr_cra_init(struct crypto_tfm *tfm) 3126 { 3127 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3128 3129 safexcel_skcipher_cra_init(tfm); 3130 ctx->alg = SAFEXCEL_SM4; 3131 ctx->blocksz = SM4_BLOCK_SIZE; 3132 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; 3133 return 0; 3134 } 3135 3136 struct safexcel_alg_template safexcel_alg_ctr_sm4 = { 3137 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 3138 .algo_mask = SAFEXCEL_ALG_SM4, 3139 .alg.skcipher = { 3140 .setkey = safexcel_skcipher_sm4ctr_setkey, 3141 .encrypt = safexcel_encrypt, 3142 .decrypt = safexcel_decrypt, 3143 /* Add nonce size */ 3144 .min_keysize = SM4_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 3145 .max_keysize = SM4_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 3146 .ivsize = CTR_RFC3686_IV_SIZE, 3147 .base = { 3148 .cra_name = "rfc3686(ctr(sm4))", 3149 .cra_driver_name = "safexcel-ctr-sm4", 3150 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3151 .cra_flags = CRYPTO_ALG_ASYNC | 3152 CRYPTO_ALG_ALLOCATES_MEMORY | 3153 CRYPTO_ALG_KERN_DRIVER_ONLY, 3154 .cra_blocksize = 1, 3155 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3156 .cra_alignmask = 0, 3157 .cra_init = safexcel_skcipher_sm4_ctr_cra_init, 3158 .cra_exit = safexcel_skcipher_cra_exit, 3159 .cra_module = THIS_MODULE, 3160 }, 3161 }, 3162 }; 3163 3164 static int safexcel_aead_sm4_blk_encrypt(struct aead_request *req) 3165 { 3166 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3167 if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) 3168 return -EINVAL; 3169 3170 return safexcel_queue_req(&req->base, aead_request_ctx(req), 3171 SAFEXCEL_ENCRYPT); 3172 } 3173 3174 static int safexcel_aead_sm4_blk_decrypt(struct aead_request *req) 3175 { 3176 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3177 3178 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3179 if ((req->cryptlen - crypto_aead_authsize(tfm)) & (SM4_BLOCK_SIZE - 1)) 3180 return -EINVAL; 3181 3182 return safexcel_queue_req(&req->base, aead_request_ctx(req), 3183 SAFEXCEL_DECRYPT); 3184 } 3185 3186 static int safexcel_aead_sm4cbc_sha1_cra_init(struct crypto_tfm *tfm) 3187 { 3188 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3189 3190 safexcel_aead_cra_init(tfm); 3191 ctx->alg = SAFEXCEL_SM4; 3192 ctx->blocksz = SM4_BLOCK_SIZE; 3193 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1; 3194 ctx->state_sz = SHA1_DIGEST_SIZE; 3195 return 0; 3196 } 3197 3198 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_sm4 = { 3199 .type = SAFEXCEL_ALG_TYPE_AEAD, 3200 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SHA1, 3201 .alg.aead = { 3202 .setkey = safexcel_aead_setkey, 3203 .encrypt = safexcel_aead_sm4_blk_encrypt, 3204 .decrypt = safexcel_aead_sm4_blk_decrypt, 3205 .ivsize = SM4_BLOCK_SIZE, 3206 .maxauthsize = SHA1_DIGEST_SIZE, 3207 .base = { 3208 .cra_name = "authenc(hmac(sha1),cbc(sm4))", 3209 .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-sm4", 3210 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3211 .cra_flags = CRYPTO_ALG_ASYNC | 3212 CRYPTO_ALG_ALLOCATES_MEMORY | 3213 CRYPTO_ALG_KERN_DRIVER_ONLY, 3214 .cra_blocksize = SM4_BLOCK_SIZE, 3215 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3216 .cra_alignmask = 0, 3217 .cra_init = safexcel_aead_sm4cbc_sha1_cra_init, 3218 .cra_exit = safexcel_aead_cra_exit, 3219 .cra_module = THIS_MODULE, 3220 }, 3221 }, 3222 }; 3223 3224 static int safexcel_aead_fallback_setkey(struct crypto_aead *ctfm, 3225 const u8 *key, unsigned int len) 3226 { 3227 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 3228 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3229 3230 /* Keep fallback cipher synchronized */ 3231 return crypto_aead_setkey(ctx->fback, (u8 *)key, len) ?: 3232 safexcel_aead_setkey(ctfm, key, len); 3233 } 3234 3235 static int safexcel_aead_fallback_setauthsize(struct crypto_aead *ctfm, 3236 unsigned int authsize) 3237 { 3238 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 3239 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3240 3241 /* Keep fallback cipher synchronized */ 3242 return crypto_aead_setauthsize(ctx->fback, authsize); 3243 } 3244 3245 static int safexcel_aead_fallback_crypt(struct aead_request *req, 3246 enum safexcel_cipher_direction dir) 3247 { 3248 struct crypto_aead *aead = crypto_aead_reqtfm(req); 3249 struct crypto_tfm *tfm = crypto_aead_tfm(aead); 3250 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3251 struct aead_request *subreq = aead_request_ctx(req); 3252 3253 aead_request_set_tfm(subreq, ctx->fback); 3254 aead_request_set_callback(subreq, req->base.flags, req->base.complete, 3255 req->base.data); 3256 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, 3257 req->iv); 3258 aead_request_set_ad(subreq, req->assoclen); 3259 3260 return (dir == SAFEXCEL_ENCRYPT) ? 3261 crypto_aead_encrypt(subreq) : 3262 crypto_aead_decrypt(subreq); 3263 } 3264 3265 static int safexcel_aead_sm4cbc_sm3_encrypt(struct aead_request *req) 3266 { 3267 struct safexcel_cipher_req *creq = aead_request_ctx(req); 3268 3269 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3270 if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) 3271 return -EINVAL; 3272 else if (req->cryptlen || req->assoclen) /* If input length > 0 only */ 3273 return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); 3274 3275 /* HW cannot do full (AAD+payload) zero length, use fallback */ 3276 return safexcel_aead_fallback_crypt(req, SAFEXCEL_ENCRYPT); 3277 } 3278 3279 static int safexcel_aead_sm4cbc_sm3_decrypt(struct aead_request *req) 3280 { 3281 struct safexcel_cipher_req *creq = aead_request_ctx(req); 3282 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3283 3284 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3285 if ((req->cryptlen - crypto_aead_authsize(tfm)) & (SM4_BLOCK_SIZE - 1)) 3286 return -EINVAL; 3287 else if (req->cryptlen > crypto_aead_authsize(tfm) || req->assoclen) 3288 /* If input length > 0 only */ 3289 return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); 3290 3291 /* HW cannot do full (AAD+payload) zero length, use fallback */ 3292 return safexcel_aead_fallback_crypt(req, SAFEXCEL_DECRYPT); 3293 } 3294 3295 static int safexcel_aead_sm4cbc_sm3_cra_init(struct crypto_tfm *tfm) 3296 { 3297 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3298 3299 safexcel_aead_fallback_cra_init(tfm); 3300 ctx->alg = SAFEXCEL_SM4; 3301 ctx->blocksz = SM4_BLOCK_SIZE; 3302 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3; 3303 ctx->state_sz = SM3_DIGEST_SIZE; 3304 return 0; 3305 } 3306 3307 struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_cbc_sm4 = { 3308 .type = SAFEXCEL_ALG_TYPE_AEAD, 3309 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SM3, 3310 .alg.aead = { 3311 .setkey = safexcel_aead_fallback_setkey, 3312 .setauthsize = safexcel_aead_fallback_setauthsize, 3313 .encrypt = safexcel_aead_sm4cbc_sm3_encrypt, 3314 .decrypt = safexcel_aead_sm4cbc_sm3_decrypt, 3315 .ivsize = SM4_BLOCK_SIZE, 3316 .maxauthsize = SM3_DIGEST_SIZE, 3317 .base = { 3318 .cra_name = "authenc(hmac(sm3),cbc(sm4))", 3319 .cra_driver_name = "safexcel-authenc-hmac-sm3-cbc-sm4", 3320 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3321 .cra_flags = CRYPTO_ALG_ASYNC | 3322 CRYPTO_ALG_ALLOCATES_MEMORY | 3323 CRYPTO_ALG_KERN_DRIVER_ONLY | 3324 CRYPTO_ALG_NEED_FALLBACK, 3325 .cra_blocksize = SM4_BLOCK_SIZE, 3326 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3327 .cra_alignmask = 0, 3328 .cra_init = safexcel_aead_sm4cbc_sm3_cra_init, 3329 .cra_exit = safexcel_aead_fallback_cra_exit, 3330 .cra_module = THIS_MODULE, 3331 }, 3332 }, 3333 }; 3334 3335 static int safexcel_aead_sm4ctr_sha1_cra_init(struct crypto_tfm *tfm) 3336 { 3337 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3338 3339 safexcel_aead_sm4cbc_sha1_cra_init(tfm); 3340 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; 3341 return 0; 3342 } 3343 3344 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_ctr_sm4 = { 3345 .type = SAFEXCEL_ALG_TYPE_AEAD, 3346 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SHA1, 3347 .alg.aead = { 3348 .setkey = safexcel_aead_setkey, 3349 .encrypt = safexcel_aead_encrypt, 3350 .decrypt = safexcel_aead_decrypt, 3351 .ivsize = CTR_RFC3686_IV_SIZE, 3352 .maxauthsize = SHA1_DIGEST_SIZE, 3353 .base = { 3354 .cra_name = "authenc(hmac(sha1),rfc3686(ctr(sm4)))", 3355 .cra_driver_name = "safexcel-authenc-hmac-sha1-ctr-sm4", 3356 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3357 .cra_flags = CRYPTO_ALG_ASYNC | 3358 CRYPTO_ALG_ALLOCATES_MEMORY | 3359 CRYPTO_ALG_KERN_DRIVER_ONLY, 3360 .cra_blocksize = 1, 3361 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3362 .cra_alignmask = 0, 3363 .cra_init = safexcel_aead_sm4ctr_sha1_cra_init, 3364 .cra_exit = safexcel_aead_cra_exit, 3365 .cra_module = THIS_MODULE, 3366 }, 3367 }, 3368 }; 3369 3370 static int safexcel_aead_sm4ctr_sm3_cra_init(struct crypto_tfm *tfm) 3371 { 3372 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3373 3374 safexcel_aead_sm4cbc_sm3_cra_init(tfm); 3375 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; 3376 return 0; 3377 } 3378 3379 struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_ctr_sm4 = { 3380 .type = SAFEXCEL_ALG_TYPE_AEAD, 3381 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SM3, 3382 .alg.aead = { 3383 .setkey = safexcel_aead_setkey, 3384 .encrypt = safexcel_aead_encrypt, 3385 .decrypt = safexcel_aead_decrypt, 3386 .ivsize = CTR_RFC3686_IV_SIZE, 3387 .maxauthsize = SM3_DIGEST_SIZE, 3388 .base = { 3389 .cra_name = "authenc(hmac(sm3),rfc3686(ctr(sm4)))", 3390 .cra_driver_name = "safexcel-authenc-hmac-sm3-ctr-sm4", 3391 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3392 .cra_flags = CRYPTO_ALG_ASYNC | 3393 CRYPTO_ALG_ALLOCATES_MEMORY | 3394 CRYPTO_ALG_KERN_DRIVER_ONLY, 3395 .cra_blocksize = 1, 3396 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3397 .cra_alignmask = 0, 3398 .cra_init = safexcel_aead_sm4ctr_sm3_cra_init, 3399 .cra_exit = safexcel_aead_cra_exit, 3400 .cra_module = THIS_MODULE, 3401 }, 3402 }, 3403 }; 3404 3405 static int safexcel_rfc4106_gcm_setkey(struct crypto_aead *ctfm, const u8 *key, 3406 unsigned int len) 3407 { 3408 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 3409 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3410 3411 /* last 4 bytes of key are the nonce! */ 3412 ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE); 3413 3414 len -= CTR_RFC3686_NONCE_SIZE; 3415 return safexcel_aead_gcm_setkey(ctfm, key, len); 3416 } 3417 3418 static int safexcel_rfc4106_gcm_setauthsize(struct crypto_aead *tfm, 3419 unsigned int authsize) 3420 { 3421 return crypto_rfc4106_check_authsize(authsize); 3422 } 3423 3424 static int safexcel_rfc4106_encrypt(struct aead_request *req) 3425 { 3426 return crypto_ipsec_check_assoclen(req->assoclen) ?: 3427 safexcel_aead_encrypt(req); 3428 } 3429 3430 static int safexcel_rfc4106_decrypt(struct aead_request *req) 3431 { 3432 return crypto_ipsec_check_assoclen(req->assoclen) ?: 3433 safexcel_aead_decrypt(req); 3434 } 3435 3436 static int safexcel_rfc4106_gcm_cra_init(struct crypto_tfm *tfm) 3437 { 3438 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3439 int ret; 3440 3441 ret = safexcel_aead_gcm_cra_init(tfm); 3442 ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP; 3443 ctx->aadskip = EIP197_AEAD_IPSEC_IV_SIZE; 3444 return ret; 3445 } 3446 3447 struct safexcel_alg_template safexcel_alg_rfc4106_gcm = { 3448 .type = SAFEXCEL_ALG_TYPE_AEAD, 3449 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH, 3450 .alg.aead = { 3451 .setkey = safexcel_rfc4106_gcm_setkey, 3452 .setauthsize = safexcel_rfc4106_gcm_setauthsize, 3453 .encrypt = safexcel_rfc4106_encrypt, 3454 .decrypt = safexcel_rfc4106_decrypt, 3455 .ivsize = GCM_RFC4106_IV_SIZE, 3456 .maxauthsize = GHASH_DIGEST_SIZE, 3457 .base = { 3458 .cra_name = "rfc4106(gcm(aes))", 3459 .cra_driver_name = "safexcel-rfc4106-gcm-aes", 3460 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3461 .cra_flags = CRYPTO_ALG_ASYNC | 3462 CRYPTO_ALG_ALLOCATES_MEMORY | 3463 CRYPTO_ALG_KERN_DRIVER_ONLY, 3464 .cra_blocksize = 1, 3465 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3466 .cra_alignmask = 0, 3467 .cra_init = safexcel_rfc4106_gcm_cra_init, 3468 .cra_exit = safexcel_aead_gcm_cra_exit, 3469 }, 3470 }, 3471 }; 3472 3473 static int safexcel_rfc4543_gcm_setauthsize(struct crypto_aead *tfm, 3474 unsigned int authsize) 3475 { 3476 if (authsize != GHASH_DIGEST_SIZE) 3477 return -EINVAL; 3478 3479 return 0; 3480 } 3481 3482 static int safexcel_rfc4543_gcm_cra_init(struct crypto_tfm *tfm) 3483 { 3484 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3485 int ret; 3486 3487 ret = safexcel_aead_gcm_cra_init(tfm); 3488 ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP_GMAC; 3489 return ret; 3490 } 3491 3492 struct safexcel_alg_template safexcel_alg_rfc4543_gcm = { 3493 .type = SAFEXCEL_ALG_TYPE_AEAD, 3494 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH, 3495 .alg.aead = { 3496 .setkey = safexcel_rfc4106_gcm_setkey, 3497 .setauthsize = safexcel_rfc4543_gcm_setauthsize, 3498 .encrypt = safexcel_rfc4106_encrypt, 3499 .decrypt = safexcel_rfc4106_decrypt, 3500 .ivsize = GCM_RFC4543_IV_SIZE, 3501 .maxauthsize = GHASH_DIGEST_SIZE, 3502 .base = { 3503 .cra_name = "rfc4543(gcm(aes))", 3504 .cra_driver_name = "safexcel-rfc4543-gcm-aes", 3505 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3506 .cra_flags = CRYPTO_ALG_ASYNC | 3507 CRYPTO_ALG_ALLOCATES_MEMORY | 3508 CRYPTO_ALG_KERN_DRIVER_ONLY, 3509 .cra_blocksize = 1, 3510 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3511 .cra_alignmask = 0, 3512 .cra_init = safexcel_rfc4543_gcm_cra_init, 3513 .cra_exit = safexcel_aead_gcm_cra_exit, 3514 }, 3515 }, 3516 }; 3517 3518 static int safexcel_rfc4309_ccm_setkey(struct crypto_aead *ctfm, const u8 *key, 3519 unsigned int len) 3520 { 3521 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 3522 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3523 3524 /* First byte of the nonce = L = always 3 for RFC4309 (4 byte ctr) */ 3525 *(u8 *)&ctx->nonce = EIP197_AEAD_IPSEC_COUNTER_SIZE - 1; 3526 /* last 3 bytes of key are the nonce! */ 3527 memcpy((u8 *)&ctx->nonce + 1, key + len - 3528 EIP197_AEAD_IPSEC_CCM_NONCE_SIZE, 3529 EIP197_AEAD_IPSEC_CCM_NONCE_SIZE); 3530 3531 len -= EIP197_AEAD_IPSEC_CCM_NONCE_SIZE; 3532 return safexcel_aead_ccm_setkey(ctfm, key, len); 3533 } 3534 3535 static int safexcel_rfc4309_ccm_setauthsize(struct crypto_aead *tfm, 3536 unsigned int authsize) 3537 { 3538 /* Borrowed from crypto/ccm.c */ 3539 switch (authsize) { 3540 case 8: 3541 case 12: 3542 case 16: 3543 break; 3544 default: 3545 return -EINVAL; 3546 } 3547 3548 return 0; 3549 } 3550 3551 static int safexcel_rfc4309_ccm_encrypt(struct aead_request *req) 3552 { 3553 struct safexcel_cipher_req *creq = aead_request_ctx(req); 3554 3555 /* Borrowed from crypto/ccm.c */ 3556 if (req->assoclen != 16 && req->assoclen != 20) 3557 return -EINVAL; 3558 3559 return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); 3560 } 3561 3562 static int safexcel_rfc4309_ccm_decrypt(struct aead_request *req) 3563 { 3564 struct safexcel_cipher_req *creq = aead_request_ctx(req); 3565 3566 /* Borrowed from crypto/ccm.c */ 3567 if (req->assoclen != 16 && req->assoclen != 20) 3568 return -EINVAL; 3569 3570 return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); 3571 } 3572 3573 static int safexcel_rfc4309_ccm_cra_init(struct crypto_tfm *tfm) 3574 { 3575 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3576 int ret; 3577 3578 ret = safexcel_aead_ccm_cra_init(tfm); 3579 ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP; 3580 ctx->aadskip = EIP197_AEAD_IPSEC_IV_SIZE; 3581 return ret; 3582 } 3583 3584 struct safexcel_alg_template safexcel_alg_rfc4309_ccm = { 3585 .type = SAFEXCEL_ALG_TYPE_AEAD, 3586 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_CBC_MAC_ALL, 3587 .alg.aead = { 3588 .setkey = safexcel_rfc4309_ccm_setkey, 3589 .setauthsize = safexcel_rfc4309_ccm_setauthsize, 3590 .encrypt = safexcel_rfc4309_ccm_encrypt, 3591 .decrypt = safexcel_rfc4309_ccm_decrypt, 3592 .ivsize = EIP197_AEAD_IPSEC_IV_SIZE, 3593 .maxauthsize = AES_BLOCK_SIZE, 3594 .base = { 3595 .cra_name = "rfc4309(ccm(aes))", 3596 .cra_driver_name = "safexcel-rfc4309-ccm-aes", 3597 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3598 .cra_flags = CRYPTO_ALG_ASYNC | 3599 CRYPTO_ALG_ALLOCATES_MEMORY | 3600 CRYPTO_ALG_KERN_DRIVER_ONLY, 3601 .cra_blocksize = 1, 3602 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3603 .cra_alignmask = 0, 3604 .cra_init = safexcel_rfc4309_ccm_cra_init, 3605 .cra_exit = safexcel_aead_cra_exit, 3606 .cra_module = THIS_MODULE, 3607 }, 3608 }, 3609 }; 3610