1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2017 Marvell 4 * 5 * Antoine Tenart <antoine.tenart@free-electrons.com> 6 */ 7 8 #include <asm/unaligned.h> 9 #include <linux/device.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/dmapool.h> 12 #include <crypto/aead.h> 13 #include <crypto/aes.h> 14 #include <crypto/authenc.h> 15 #include <crypto/chacha.h> 16 #include <crypto/ctr.h> 17 #include <crypto/internal/des.h> 18 #include <crypto/gcm.h> 19 #include <crypto/ghash.h> 20 #include <crypto/poly1305.h> 21 #include <crypto/sha1.h> 22 #include <crypto/sha2.h> 23 #include <crypto/sm3.h> 24 #include <crypto/sm4.h> 25 #include <crypto/xts.h> 26 #include <crypto/skcipher.h> 27 #include <crypto/internal/aead.h> 28 #include <crypto/internal/skcipher.h> 29 30 #include "safexcel.h" 31 32 enum safexcel_cipher_direction { 33 SAFEXCEL_ENCRYPT, 34 SAFEXCEL_DECRYPT, 35 }; 36 37 enum safexcel_cipher_alg { 38 SAFEXCEL_DES, 39 SAFEXCEL_3DES, 40 SAFEXCEL_AES, 41 SAFEXCEL_CHACHA20, 42 SAFEXCEL_SM4, 43 }; 44 45 struct safexcel_cipher_ctx { 46 struct safexcel_context base; 47 struct safexcel_crypto_priv *priv; 48 49 u32 mode; 50 enum safexcel_cipher_alg alg; 51 u8 aead; /* !=0=AEAD, 2=IPSec ESP AEAD, 3=IPsec ESP GMAC */ 52 u8 xcm; /* 0=authenc, 1=GCM, 2 reserved for CCM */ 53 u8 aadskip; 54 u8 blocksz; 55 u32 ivmask; 56 u32 ctrinit; 57 58 __le32 key[16]; 59 u32 nonce; 60 unsigned int key_len, xts; 61 62 /* All the below is AEAD specific */ 63 u32 hash_alg; 64 u32 state_sz; 65 66 struct crypto_cipher *hkaes; 67 struct crypto_aead *fback; 68 }; 69 70 struct safexcel_cipher_req { 71 enum safexcel_cipher_direction direction; 72 /* Number of result descriptors associated to the request */ 73 unsigned int rdescs; 74 bool needs_inv; 75 int nr_src, nr_dst; 76 }; 77 78 static int safexcel_skcipher_iv(struct safexcel_cipher_ctx *ctx, u8 *iv, 79 struct safexcel_command_desc *cdesc) 80 { 81 if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD) { 82 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; 83 /* 32 bit nonce */ 84 cdesc->control_data.token[0] = ctx->nonce; 85 /* 64 bit IV part */ 86 memcpy(&cdesc->control_data.token[1], iv, 8); 87 /* 32 bit counter, start at 0 or 1 (big endian!) */ 88 cdesc->control_data.token[3] = 89 (__force u32)cpu_to_be32(ctx->ctrinit); 90 return 4; 91 } 92 if (ctx->alg == SAFEXCEL_CHACHA20) { 93 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; 94 /* 96 bit nonce part */ 95 memcpy(&cdesc->control_data.token[0], &iv[4], 12); 96 /* 32 bit counter */ 97 cdesc->control_data.token[3] = *(u32 *)iv; 98 return 4; 99 } 100 101 cdesc->control_data.options |= ctx->ivmask; 102 memcpy(cdesc->control_data.token, iv, ctx->blocksz); 103 return ctx->blocksz / sizeof(u32); 104 } 105 106 static void safexcel_skcipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv, 107 struct safexcel_command_desc *cdesc, 108 struct safexcel_token *atoken, 109 u32 length) 110 { 111 struct safexcel_token *token; 112 int ivlen; 113 114 ivlen = safexcel_skcipher_iv(ctx, iv, cdesc); 115 if (ivlen == 4) { 116 /* No space in cdesc, instruction moves to atoken */ 117 cdesc->additional_cdata_size = 1; 118 token = atoken; 119 } else { 120 /* Everything fits in cdesc */ 121 token = (struct safexcel_token *)(cdesc->control_data.token + 2); 122 /* Need to pad with NOP */ 123 eip197_noop_token(&token[1]); 124 } 125 126 token->opcode = EIP197_TOKEN_OPCODE_DIRECTION; 127 token->packet_length = length; 128 token->stat = EIP197_TOKEN_STAT_LAST_PACKET | 129 EIP197_TOKEN_STAT_LAST_HASH; 130 token->instructions = EIP197_TOKEN_INS_LAST | 131 EIP197_TOKEN_INS_TYPE_CRYPTO | 132 EIP197_TOKEN_INS_TYPE_OUTPUT; 133 } 134 135 static void safexcel_aead_iv(struct safexcel_cipher_ctx *ctx, u8 *iv, 136 struct safexcel_command_desc *cdesc) 137 { 138 if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD || 139 ctx->aead & EIP197_AEAD_TYPE_IPSEC_ESP) { /* _ESP and _ESP_GMAC */ 140 /* 32 bit nonce */ 141 cdesc->control_data.token[0] = ctx->nonce; 142 /* 64 bit IV part */ 143 memcpy(&cdesc->control_data.token[1], iv, 8); 144 /* 32 bit counter, start at 0 or 1 (big endian!) */ 145 cdesc->control_data.token[3] = 146 (__force u32)cpu_to_be32(ctx->ctrinit); 147 return; 148 } 149 if (ctx->xcm == EIP197_XCM_MODE_GCM || ctx->alg == SAFEXCEL_CHACHA20) { 150 /* 96 bit IV part */ 151 memcpy(&cdesc->control_data.token[0], iv, 12); 152 /* 32 bit counter, start at 0 or 1 (big endian!) */ 153 cdesc->control_data.token[3] = 154 (__force u32)cpu_to_be32(ctx->ctrinit); 155 return; 156 } 157 /* CBC */ 158 memcpy(cdesc->control_data.token, iv, ctx->blocksz); 159 } 160 161 static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv, 162 struct safexcel_command_desc *cdesc, 163 struct safexcel_token *atoken, 164 enum safexcel_cipher_direction direction, 165 u32 cryptlen, u32 assoclen, u32 digestsize) 166 { 167 struct safexcel_token *aadref; 168 int atoksize = 2; /* Start with minimum size */ 169 int assocadj = assoclen - ctx->aadskip, aadalign; 170 171 /* Always 4 dwords of embedded IV for AEAD modes */ 172 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; 173 174 if (direction == SAFEXCEL_DECRYPT) 175 cryptlen -= digestsize; 176 177 if (unlikely(ctx->xcm == EIP197_XCM_MODE_CCM)) { 178 /* Construct IV block B0 for the CBC-MAC */ 179 u8 *final_iv = (u8 *)cdesc->control_data.token; 180 u8 *cbcmaciv = (u8 *)&atoken[1]; 181 __le32 *aadlen = (__le32 *)&atoken[5]; 182 183 if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { 184 /* Length + nonce */ 185 cdesc->control_data.token[0] = ctx->nonce; 186 /* Fixup flags byte */ 187 *(__le32 *)cbcmaciv = 188 cpu_to_le32(ctx->nonce | 189 ((assocadj > 0) << 6) | 190 ((digestsize - 2) << 2)); 191 /* 64 bit IV part */ 192 memcpy(&cdesc->control_data.token[1], iv, 8); 193 memcpy(cbcmaciv + 4, iv, 8); 194 /* Start counter at 0 */ 195 cdesc->control_data.token[3] = 0; 196 /* Message length */ 197 *(__be32 *)(cbcmaciv + 12) = cpu_to_be32(cryptlen); 198 } else { 199 /* Variable length IV part */ 200 memcpy(final_iv, iv, 15 - iv[0]); 201 memcpy(cbcmaciv, iv, 15 - iv[0]); 202 /* Start variable length counter at 0 */ 203 memset(final_iv + 15 - iv[0], 0, iv[0] + 1); 204 memset(cbcmaciv + 15 - iv[0], 0, iv[0] - 1); 205 /* fixup flags byte */ 206 cbcmaciv[0] |= ((assocadj > 0) << 6) | 207 ((digestsize - 2) << 2); 208 /* insert lower 2 bytes of message length */ 209 cbcmaciv[14] = cryptlen >> 8; 210 cbcmaciv[15] = cryptlen & 255; 211 } 212 213 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 214 atoken->packet_length = AES_BLOCK_SIZE + 215 ((assocadj > 0) << 1); 216 atoken->stat = 0; 217 atoken->instructions = EIP197_TOKEN_INS_ORIGIN_TOKEN | 218 EIP197_TOKEN_INS_TYPE_HASH; 219 220 if (likely(assocadj)) { 221 *aadlen = cpu_to_le32((assocadj >> 8) | 222 (assocadj & 255) << 8); 223 atoken += 6; 224 atoksize += 7; 225 } else { 226 atoken += 5; 227 atoksize += 6; 228 } 229 230 /* Process AAD data */ 231 aadref = atoken; 232 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION; 233 atoken->packet_length = assocadj; 234 atoken->stat = 0; 235 atoken->instructions = EIP197_TOKEN_INS_TYPE_HASH; 236 atoken++; 237 238 /* For CCM only, align AAD data towards hash engine */ 239 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 240 aadalign = (assocadj + 2) & 15; 241 atoken->packet_length = assocadj && aadalign ? 242 16 - aadalign : 243 0; 244 if (likely(cryptlen)) { 245 atoken->stat = 0; 246 atoken->instructions = EIP197_TOKEN_INS_TYPE_HASH; 247 } else { 248 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH; 249 atoken->instructions = EIP197_TOKEN_INS_LAST | 250 EIP197_TOKEN_INS_TYPE_HASH; 251 } 252 } else { 253 safexcel_aead_iv(ctx, iv, cdesc); 254 255 /* Process AAD data */ 256 aadref = atoken; 257 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION; 258 atoken->packet_length = assocadj; 259 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH; 260 atoken->instructions = EIP197_TOKEN_INS_LAST | 261 EIP197_TOKEN_INS_TYPE_HASH; 262 } 263 atoken++; 264 265 if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { 266 /* For ESP mode (and not GMAC), skip over the IV */ 267 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION; 268 atoken->packet_length = EIP197_AEAD_IPSEC_IV_SIZE; 269 atoken->stat = 0; 270 atoken->instructions = 0; 271 atoken++; 272 atoksize++; 273 } else if (unlikely(ctx->alg == SAFEXCEL_CHACHA20 && 274 direction == SAFEXCEL_DECRYPT)) { 275 /* Poly-chacha decryption needs a dummy NOP here ... */ 276 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 277 atoken->packet_length = 16; /* According to Op Manual */ 278 atoken->stat = 0; 279 atoken->instructions = 0; 280 atoken++; 281 atoksize++; 282 } 283 284 if (ctx->xcm) { 285 /* For GCM and CCM, obtain enc(Y0) */ 286 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT_REMRES; 287 atoken->packet_length = 0; 288 atoken->stat = 0; 289 atoken->instructions = AES_BLOCK_SIZE; 290 atoken++; 291 292 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 293 atoken->packet_length = AES_BLOCK_SIZE; 294 atoken->stat = 0; 295 atoken->instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | 296 EIP197_TOKEN_INS_TYPE_CRYPTO; 297 atoken++; 298 atoksize += 2; 299 } 300 301 if (likely(cryptlen || ctx->alg == SAFEXCEL_CHACHA20)) { 302 /* Fixup stat field for AAD direction instruction */ 303 aadref->stat = 0; 304 305 /* Process crypto data */ 306 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION; 307 atoken->packet_length = cryptlen; 308 309 if (unlikely(ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP_GMAC)) { 310 /* Fixup instruction field for AAD dir instruction */ 311 aadref->instructions = EIP197_TOKEN_INS_TYPE_HASH; 312 313 /* Do not send to crypt engine in case of GMAC */ 314 atoken->instructions = EIP197_TOKEN_INS_LAST | 315 EIP197_TOKEN_INS_TYPE_HASH | 316 EIP197_TOKEN_INS_TYPE_OUTPUT; 317 } else { 318 atoken->instructions = EIP197_TOKEN_INS_LAST | 319 EIP197_TOKEN_INS_TYPE_CRYPTO | 320 EIP197_TOKEN_INS_TYPE_HASH | 321 EIP197_TOKEN_INS_TYPE_OUTPUT; 322 } 323 324 cryptlen &= 15; 325 if (unlikely(ctx->xcm == EIP197_XCM_MODE_CCM && cryptlen)) { 326 atoken->stat = 0; 327 /* For CCM only, pad crypto data to the hash engine */ 328 atoken++; 329 atoksize++; 330 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 331 atoken->packet_length = 16 - cryptlen; 332 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH; 333 atoken->instructions = EIP197_TOKEN_INS_TYPE_HASH; 334 } else { 335 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH; 336 } 337 atoken++; 338 atoksize++; 339 } 340 341 if (direction == SAFEXCEL_ENCRYPT) { 342 /* Append ICV */ 343 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 344 atoken->packet_length = digestsize; 345 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH | 346 EIP197_TOKEN_STAT_LAST_PACKET; 347 atoken->instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | 348 EIP197_TOKEN_INS_INSERT_HASH_DIGEST; 349 } else { 350 /* Extract ICV */ 351 atoken->opcode = EIP197_TOKEN_OPCODE_RETRIEVE; 352 atoken->packet_length = digestsize; 353 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH | 354 EIP197_TOKEN_STAT_LAST_PACKET; 355 atoken->instructions = EIP197_TOKEN_INS_INSERT_HASH_DIGEST; 356 atoken++; 357 atoksize++; 358 359 /* Verify ICV */ 360 atoken->opcode = EIP197_TOKEN_OPCODE_VERIFY; 361 atoken->packet_length = digestsize | 362 EIP197_TOKEN_HASH_RESULT_VERIFY; 363 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH | 364 EIP197_TOKEN_STAT_LAST_PACKET; 365 atoken->instructions = EIP197_TOKEN_INS_TYPE_OUTPUT; 366 } 367 368 /* Fixup length of the token in the command descriptor */ 369 cdesc->additional_cdata_size = atoksize; 370 } 371 372 static int safexcel_skcipher_aes_setkey(struct crypto_skcipher *ctfm, 373 const u8 *key, unsigned int len) 374 { 375 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 376 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 377 struct safexcel_crypto_priv *priv = ctx->base.priv; 378 struct crypto_aes_ctx aes; 379 int ret, i; 380 381 ret = aes_expandkey(&aes, key, len); 382 if (ret) 383 return ret; 384 385 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 386 for (i = 0; i < len / sizeof(u32); i++) { 387 if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { 388 ctx->base.needs_inv = true; 389 break; 390 } 391 } 392 } 393 394 for (i = 0; i < len / sizeof(u32); i++) 395 ctx->key[i] = cpu_to_le32(aes.key_enc[i]); 396 397 ctx->key_len = len; 398 399 memzero_explicit(&aes, sizeof(aes)); 400 return 0; 401 } 402 403 static int safexcel_aead_setkey(struct crypto_aead *ctfm, const u8 *key, 404 unsigned int len) 405 { 406 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 407 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 408 struct safexcel_crypto_priv *priv = ctx->base.priv; 409 struct crypto_authenc_keys keys; 410 struct crypto_aes_ctx aes; 411 int err = -EINVAL, i; 412 const char *alg; 413 414 if (unlikely(crypto_authenc_extractkeys(&keys, key, len))) 415 goto badkey; 416 417 if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD) { 418 /* Must have at least space for the nonce here */ 419 if (unlikely(keys.enckeylen < CTR_RFC3686_NONCE_SIZE)) 420 goto badkey; 421 /* last 4 bytes of key are the nonce! */ 422 ctx->nonce = *(u32 *)(keys.enckey + keys.enckeylen - 423 CTR_RFC3686_NONCE_SIZE); 424 /* exclude the nonce here */ 425 keys.enckeylen -= CTR_RFC3686_NONCE_SIZE; 426 } 427 428 /* Encryption key */ 429 switch (ctx->alg) { 430 case SAFEXCEL_DES: 431 err = verify_aead_des_key(ctfm, keys.enckey, keys.enckeylen); 432 if (unlikely(err)) 433 goto badkey; 434 break; 435 case SAFEXCEL_3DES: 436 err = verify_aead_des3_key(ctfm, keys.enckey, keys.enckeylen); 437 if (unlikely(err)) 438 goto badkey; 439 break; 440 case SAFEXCEL_AES: 441 err = aes_expandkey(&aes, keys.enckey, keys.enckeylen); 442 if (unlikely(err)) 443 goto badkey; 444 break; 445 case SAFEXCEL_SM4: 446 if (unlikely(keys.enckeylen != SM4_KEY_SIZE)) 447 goto badkey; 448 break; 449 default: 450 dev_err(priv->dev, "aead: unsupported cipher algorithm\n"); 451 goto badkey; 452 } 453 454 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 455 for (i = 0; i < keys.enckeylen / sizeof(u32); i++) { 456 if (le32_to_cpu(ctx->key[i]) != 457 ((u32 *)keys.enckey)[i]) { 458 ctx->base.needs_inv = true; 459 break; 460 } 461 } 462 } 463 464 /* Auth key */ 465 switch (ctx->hash_alg) { 466 case CONTEXT_CONTROL_CRYPTO_ALG_SHA1: 467 alg = "safexcel-sha1"; 468 break; 469 case CONTEXT_CONTROL_CRYPTO_ALG_SHA224: 470 alg = "safexcel-sha224"; 471 break; 472 case CONTEXT_CONTROL_CRYPTO_ALG_SHA256: 473 alg = "safexcel-sha256"; 474 break; 475 case CONTEXT_CONTROL_CRYPTO_ALG_SHA384: 476 alg = "safexcel-sha384"; 477 break; 478 case CONTEXT_CONTROL_CRYPTO_ALG_SHA512: 479 alg = "safexcel-sha512"; 480 break; 481 case CONTEXT_CONTROL_CRYPTO_ALG_SM3: 482 alg = "safexcel-sm3"; 483 break; 484 default: 485 dev_err(priv->dev, "aead: unsupported hash algorithm\n"); 486 goto badkey; 487 } 488 489 if (safexcel_hmac_setkey(&ctx->base, keys.authkey, keys.authkeylen, 490 alg, ctx->state_sz)) 491 goto badkey; 492 493 /* Now copy the keys into the context */ 494 for (i = 0; i < keys.enckeylen / sizeof(u32); i++) 495 ctx->key[i] = cpu_to_le32(((u32 *)keys.enckey)[i]); 496 ctx->key_len = keys.enckeylen; 497 498 memzero_explicit(&keys, sizeof(keys)); 499 return 0; 500 501 badkey: 502 memzero_explicit(&keys, sizeof(keys)); 503 return err; 504 } 505 506 static int safexcel_context_control(struct safexcel_cipher_ctx *ctx, 507 struct crypto_async_request *async, 508 struct safexcel_cipher_req *sreq, 509 struct safexcel_command_desc *cdesc) 510 { 511 struct safexcel_crypto_priv *priv = ctx->base.priv; 512 int ctrl_size = ctx->key_len / sizeof(u32); 513 514 cdesc->control_data.control1 = ctx->mode; 515 516 if (ctx->aead) { 517 /* Take in account the ipad+opad digests */ 518 if (ctx->xcm) { 519 ctrl_size += ctx->state_sz / sizeof(u32); 520 cdesc->control_data.control0 = 521 CONTEXT_CONTROL_KEY_EN | 522 CONTEXT_CONTROL_DIGEST_XCM | 523 ctx->hash_alg | 524 CONTEXT_CONTROL_SIZE(ctrl_size); 525 } else if (ctx->alg == SAFEXCEL_CHACHA20) { 526 /* Chacha20-Poly1305 */ 527 cdesc->control_data.control0 = 528 CONTEXT_CONTROL_KEY_EN | 529 CONTEXT_CONTROL_CRYPTO_ALG_CHACHA20 | 530 (sreq->direction == SAFEXCEL_ENCRYPT ? 531 CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT : 532 CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN) | 533 ctx->hash_alg | 534 CONTEXT_CONTROL_SIZE(ctrl_size); 535 return 0; 536 } else { 537 ctrl_size += ctx->state_sz / sizeof(u32) * 2; 538 cdesc->control_data.control0 = 539 CONTEXT_CONTROL_KEY_EN | 540 CONTEXT_CONTROL_DIGEST_HMAC | 541 ctx->hash_alg | 542 CONTEXT_CONTROL_SIZE(ctrl_size); 543 } 544 545 if (sreq->direction == SAFEXCEL_ENCRYPT && 546 (ctx->xcm == EIP197_XCM_MODE_CCM || 547 ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP_GMAC)) 548 cdesc->control_data.control0 |= 549 CONTEXT_CONTROL_TYPE_HASH_ENCRYPT_OUT; 550 else if (sreq->direction == SAFEXCEL_ENCRYPT) 551 cdesc->control_data.control0 |= 552 CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT; 553 else if (ctx->xcm == EIP197_XCM_MODE_CCM) 554 cdesc->control_data.control0 |= 555 CONTEXT_CONTROL_TYPE_DECRYPT_HASH_IN; 556 else 557 cdesc->control_data.control0 |= 558 CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN; 559 } else { 560 if (sreq->direction == SAFEXCEL_ENCRYPT) 561 cdesc->control_data.control0 = 562 CONTEXT_CONTROL_TYPE_CRYPTO_OUT | 563 CONTEXT_CONTROL_KEY_EN | 564 CONTEXT_CONTROL_SIZE(ctrl_size); 565 else 566 cdesc->control_data.control0 = 567 CONTEXT_CONTROL_TYPE_CRYPTO_IN | 568 CONTEXT_CONTROL_KEY_EN | 569 CONTEXT_CONTROL_SIZE(ctrl_size); 570 } 571 572 if (ctx->alg == SAFEXCEL_DES) { 573 cdesc->control_data.control0 |= 574 CONTEXT_CONTROL_CRYPTO_ALG_DES; 575 } else if (ctx->alg == SAFEXCEL_3DES) { 576 cdesc->control_data.control0 |= 577 CONTEXT_CONTROL_CRYPTO_ALG_3DES; 578 } else if (ctx->alg == SAFEXCEL_AES) { 579 switch (ctx->key_len >> ctx->xts) { 580 case AES_KEYSIZE_128: 581 cdesc->control_data.control0 |= 582 CONTEXT_CONTROL_CRYPTO_ALG_AES128; 583 break; 584 case AES_KEYSIZE_192: 585 cdesc->control_data.control0 |= 586 CONTEXT_CONTROL_CRYPTO_ALG_AES192; 587 break; 588 case AES_KEYSIZE_256: 589 cdesc->control_data.control0 |= 590 CONTEXT_CONTROL_CRYPTO_ALG_AES256; 591 break; 592 default: 593 dev_err(priv->dev, "aes keysize not supported: %u\n", 594 ctx->key_len >> ctx->xts); 595 return -EINVAL; 596 } 597 } else if (ctx->alg == SAFEXCEL_CHACHA20) { 598 cdesc->control_data.control0 |= 599 CONTEXT_CONTROL_CRYPTO_ALG_CHACHA20; 600 } else if (ctx->alg == SAFEXCEL_SM4) { 601 cdesc->control_data.control0 |= 602 CONTEXT_CONTROL_CRYPTO_ALG_SM4; 603 } 604 605 return 0; 606 } 607 608 static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int ring, 609 struct crypto_async_request *async, 610 struct scatterlist *src, 611 struct scatterlist *dst, 612 unsigned int cryptlen, 613 struct safexcel_cipher_req *sreq, 614 bool *should_complete, int *ret) 615 { 616 struct skcipher_request *areq = skcipher_request_cast(async); 617 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(areq); 618 struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(skcipher); 619 struct safexcel_result_desc *rdesc; 620 int ndesc = 0; 621 622 *ret = 0; 623 624 if (unlikely(!sreq->rdescs)) 625 return 0; 626 627 while (sreq->rdescs--) { 628 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); 629 if (IS_ERR(rdesc)) { 630 dev_err(priv->dev, 631 "cipher: result: could not retrieve the result descriptor\n"); 632 *ret = PTR_ERR(rdesc); 633 break; 634 } 635 636 if (likely(!*ret)) 637 *ret = safexcel_rdesc_check_errors(priv, rdesc); 638 639 ndesc++; 640 } 641 642 safexcel_complete(priv, ring); 643 644 if (src == dst) { 645 dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL); 646 } else { 647 dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE); 648 dma_unmap_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE); 649 } 650 651 /* 652 * Update IV in req from last crypto output word for CBC modes 653 */ 654 if ((!ctx->aead) && (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) && 655 (sreq->direction == SAFEXCEL_ENCRYPT)) { 656 /* For encrypt take the last output word */ 657 sg_pcopy_to_buffer(dst, sreq->nr_dst, areq->iv, 658 crypto_skcipher_ivsize(skcipher), 659 (cryptlen - 660 crypto_skcipher_ivsize(skcipher))); 661 } 662 663 *should_complete = true; 664 665 return ndesc; 666 } 667 668 static int safexcel_send_req(struct crypto_async_request *base, int ring, 669 struct safexcel_cipher_req *sreq, 670 struct scatterlist *src, struct scatterlist *dst, 671 unsigned int cryptlen, unsigned int assoclen, 672 unsigned int digestsize, u8 *iv, int *commands, 673 int *results) 674 { 675 struct skcipher_request *areq = skcipher_request_cast(base); 676 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(areq); 677 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); 678 struct safexcel_crypto_priv *priv = ctx->base.priv; 679 struct safexcel_command_desc *cdesc; 680 struct safexcel_command_desc *first_cdesc = NULL; 681 struct safexcel_result_desc *rdesc, *first_rdesc = NULL; 682 struct scatterlist *sg; 683 unsigned int totlen; 684 unsigned int totlen_src = cryptlen + assoclen; 685 unsigned int totlen_dst = totlen_src; 686 struct safexcel_token *atoken; 687 int n_cdesc = 0, n_rdesc = 0; 688 int queued, i, ret = 0; 689 bool first = true; 690 691 sreq->nr_src = sg_nents_for_len(src, totlen_src); 692 693 if (ctx->aead) { 694 /* 695 * AEAD has auth tag appended to output for encrypt and 696 * removed from the output for decrypt! 697 */ 698 if (sreq->direction == SAFEXCEL_DECRYPT) 699 totlen_dst -= digestsize; 700 else 701 totlen_dst += digestsize; 702 703 memcpy(ctx->base.ctxr->data + ctx->key_len / sizeof(u32), 704 &ctx->base.ipad, ctx->state_sz); 705 if (!ctx->xcm) 706 memcpy(ctx->base.ctxr->data + (ctx->key_len + 707 ctx->state_sz) / sizeof(u32), &ctx->base.opad, 708 ctx->state_sz); 709 } else if ((ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) && 710 (sreq->direction == SAFEXCEL_DECRYPT)) { 711 /* 712 * Save IV from last crypto input word for CBC modes in decrypt 713 * direction. Need to do this first in case of inplace operation 714 * as it will be overwritten. 715 */ 716 sg_pcopy_to_buffer(src, sreq->nr_src, areq->iv, 717 crypto_skcipher_ivsize(skcipher), 718 (totlen_src - 719 crypto_skcipher_ivsize(skcipher))); 720 } 721 722 sreq->nr_dst = sg_nents_for_len(dst, totlen_dst); 723 724 /* 725 * Remember actual input length, source buffer length may be 726 * updated in case of inline operation below. 727 */ 728 totlen = totlen_src; 729 queued = totlen_src; 730 731 if (src == dst) { 732 sreq->nr_src = max(sreq->nr_src, sreq->nr_dst); 733 sreq->nr_dst = sreq->nr_src; 734 if (unlikely((totlen_src || totlen_dst) && 735 (sreq->nr_src <= 0))) { 736 dev_err(priv->dev, "In-place buffer not large enough (need %d bytes)!", 737 max(totlen_src, totlen_dst)); 738 return -EINVAL; 739 } 740 dma_map_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL); 741 } else { 742 if (unlikely(totlen_src && (sreq->nr_src <= 0))) { 743 dev_err(priv->dev, "Source buffer not large enough (need %d bytes)!", 744 totlen_src); 745 return -EINVAL; 746 } 747 dma_map_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE); 748 749 if (unlikely(totlen_dst && (sreq->nr_dst <= 0))) { 750 dev_err(priv->dev, "Dest buffer not large enough (need %d bytes)!", 751 totlen_dst); 752 dma_unmap_sg(priv->dev, src, sreq->nr_src, 753 DMA_TO_DEVICE); 754 return -EINVAL; 755 } 756 dma_map_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE); 757 } 758 759 memcpy(ctx->base.ctxr->data, ctx->key, ctx->key_len); 760 761 if (!totlen) { 762 /* 763 * The EIP97 cannot deal with zero length input packets! 764 * So stuff a dummy command descriptor indicating a 1 byte 765 * (dummy) input packet, using the context record as source. 766 */ 767 first_cdesc = safexcel_add_cdesc(priv, ring, 768 1, 1, ctx->base.ctxr_dma, 769 1, 1, ctx->base.ctxr_dma, 770 &atoken); 771 if (IS_ERR(first_cdesc)) { 772 /* No space left in the command descriptor ring */ 773 ret = PTR_ERR(first_cdesc); 774 goto cdesc_rollback; 775 } 776 n_cdesc = 1; 777 goto skip_cdesc; 778 } 779 780 /* command descriptors */ 781 for_each_sg(src, sg, sreq->nr_src, i) { 782 int len = sg_dma_len(sg); 783 784 /* Do not overflow the request */ 785 if (queued < len) 786 len = queued; 787 788 cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc, 789 !(queued - len), 790 sg_dma_address(sg), len, totlen, 791 ctx->base.ctxr_dma, &atoken); 792 if (IS_ERR(cdesc)) { 793 /* No space left in the command descriptor ring */ 794 ret = PTR_ERR(cdesc); 795 goto cdesc_rollback; 796 } 797 798 if (!n_cdesc) 799 first_cdesc = cdesc; 800 801 n_cdesc++; 802 queued -= len; 803 if (!queued) 804 break; 805 } 806 skip_cdesc: 807 /* Add context control words and token to first command descriptor */ 808 safexcel_context_control(ctx, base, sreq, first_cdesc); 809 if (ctx->aead) 810 safexcel_aead_token(ctx, iv, first_cdesc, atoken, 811 sreq->direction, cryptlen, 812 assoclen, digestsize); 813 else 814 safexcel_skcipher_token(ctx, iv, first_cdesc, atoken, 815 cryptlen); 816 817 /* result descriptors */ 818 for_each_sg(dst, sg, sreq->nr_dst, i) { 819 bool last = (i == sreq->nr_dst - 1); 820 u32 len = sg_dma_len(sg); 821 822 /* only allow the part of the buffer we know we need */ 823 if (len > totlen_dst) 824 len = totlen_dst; 825 if (unlikely(!len)) 826 break; 827 totlen_dst -= len; 828 829 /* skip over AAD space in buffer - not written */ 830 if (assoclen) { 831 if (assoclen >= len) { 832 assoclen -= len; 833 continue; 834 } 835 rdesc = safexcel_add_rdesc(priv, ring, first, last, 836 sg_dma_address(sg) + 837 assoclen, 838 len - assoclen); 839 assoclen = 0; 840 } else { 841 rdesc = safexcel_add_rdesc(priv, ring, first, last, 842 sg_dma_address(sg), 843 len); 844 } 845 if (IS_ERR(rdesc)) { 846 /* No space left in the result descriptor ring */ 847 ret = PTR_ERR(rdesc); 848 goto rdesc_rollback; 849 } 850 if (first) { 851 first_rdesc = rdesc; 852 first = false; 853 } 854 n_rdesc++; 855 } 856 857 if (unlikely(first)) { 858 /* 859 * Special case: AEAD decrypt with only AAD data. 860 * In this case there is NO output data from the engine, 861 * but the engine still needs a result descriptor! 862 * Create a dummy one just for catching the result token. 863 */ 864 rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0); 865 if (IS_ERR(rdesc)) { 866 /* No space left in the result descriptor ring */ 867 ret = PTR_ERR(rdesc); 868 goto rdesc_rollback; 869 } 870 first_rdesc = rdesc; 871 n_rdesc = 1; 872 } 873 874 safexcel_rdr_req_set(priv, ring, first_rdesc, base); 875 876 *commands = n_cdesc; 877 *results = n_rdesc; 878 return 0; 879 880 rdesc_rollback: 881 for (i = 0; i < n_rdesc; i++) 882 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].rdr); 883 cdesc_rollback: 884 for (i = 0; i < n_cdesc; i++) 885 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr); 886 887 if (src == dst) { 888 dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL); 889 } else { 890 dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE); 891 dma_unmap_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE); 892 } 893 894 return ret; 895 } 896 897 static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv, 898 int ring, 899 struct crypto_async_request *base, 900 struct safexcel_cipher_req *sreq, 901 bool *should_complete, int *ret) 902 { 903 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); 904 struct safexcel_result_desc *rdesc; 905 int ndesc = 0, enq_ret; 906 907 *ret = 0; 908 909 if (unlikely(!sreq->rdescs)) 910 return 0; 911 912 while (sreq->rdescs--) { 913 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); 914 if (IS_ERR(rdesc)) { 915 dev_err(priv->dev, 916 "cipher: invalidate: could not retrieve the result descriptor\n"); 917 *ret = PTR_ERR(rdesc); 918 break; 919 } 920 921 if (likely(!*ret)) 922 *ret = safexcel_rdesc_check_errors(priv, rdesc); 923 924 ndesc++; 925 } 926 927 safexcel_complete(priv, ring); 928 929 if (ctx->base.exit_inv) { 930 dma_pool_free(priv->context_pool, ctx->base.ctxr, 931 ctx->base.ctxr_dma); 932 933 *should_complete = true; 934 935 return ndesc; 936 } 937 938 ring = safexcel_select_ring(priv); 939 ctx->base.ring = ring; 940 941 spin_lock_bh(&priv->ring[ring].queue_lock); 942 enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, base); 943 spin_unlock_bh(&priv->ring[ring].queue_lock); 944 945 if (enq_ret != -EINPROGRESS) 946 *ret = enq_ret; 947 948 queue_work(priv->ring[ring].workqueue, 949 &priv->ring[ring].work_data.work); 950 951 *should_complete = false; 952 953 return ndesc; 954 } 955 956 static int safexcel_skcipher_handle_result(struct safexcel_crypto_priv *priv, 957 int ring, 958 struct crypto_async_request *async, 959 bool *should_complete, int *ret) 960 { 961 struct skcipher_request *req = skcipher_request_cast(async); 962 struct safexcel_cipher_req *sreq = skcipher_request_ctx(req); 963 int err; 964 965 if (sreq->needs_inv) { 966 sreq->needs_inv = false; 967 err = safexcel_handle_inv_result(priv, ring, async, sreq, 968 should_complete, ret); 969 } else { 970 err = safexcel_handle_req_result(priv, ring, async, req->src, 971 req->dst, req->cryptlen, sreq, 972 should_complete, ret); 973 } 974 975 return err; 976 } 977 978 static int safexcel_aead_handle_result(struct safexcel_crypto_priv *priv, 979 int ring, 980 struct crypto_async_request *async, 981 bool *should_complete, int *ret) 982 { 983 struct aead_request *req = aead_request_cast(async); 984 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 985 struct safexcel_cipher_req *sreq = aead_request_ctx(req); 986 int err; 987 988 if (sreq->needs_inv) { 989 sreq->needs_inv = false; 990 err = safexcel_handle_inv_result(priv, ring, async, sreq, 991 should_complete, ret); 992 } else { 993 err = safexcel_handle_req_result(priv, ring, async, req->src, 994 req->dst, 995 req->cryptlen + crypto_aead_authsize(tfm), 996 sreq, should_complete, ret); 997 } 998 999 return err; 1000 } 1001 1002 static int safexcel_cipher_send_inv(struct crypto_async_request *base, 1003 int ring, int *commands, int *results) 1004 { 1005 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); 1006 struct safexcel_crypto_priv *priv = ctx->base.priv; 1007 int ret; 1008 1009 ret = safexcel_invalidate_cache(base, priv, ctx->base.ctxr_dma, ring); 1010 if (unlikely(ret)) 1011 return ret; 1012 1013 *commands = 1; 1014 *results = 1; 1015 1016 return 0; 1017 } 1018 1019 static int safexcel_skcipher_send(struct crypto_async_request *async, int ring, 1020 int *commands, int *results) 1021 { 1022 struct skcipher_request *req = skcipher_request_cast(async); 1023 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); 1024 struct safexcel_cipher_req *sreq = skcipher_request_ctx(req); 1025 struct safexcel_crypto_priv *priv = ctx->base.priv; 1026 int ret; 1027 1028 BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv); 1029 1030 if (sreq->needs_inv) { 1031 ret = safexcel_cipher_send_inv(async, ring, commands, results); 1032 } else { 1033 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1034 u8 input_iv[AES_BLOCK_SIZE]; 1035 1036 /* 1037 * Save input IV in case of CBC decrypt mode 1038 * Will be overwritten with output IV prior to use! 1039 */ 1040 memcpy(input_iv, req->iv, crypto_skcipher_ivsize(skcipher)); 1041 1042 ret = safexcel_send_req(async, ring, sreq, req->src, 1043 req->dst, req->cryptlen, 0, 0, input_iv, 1044 commands, results); 1045 } 1046 1047 sreq->rdescs = *results; 1048 return ret; 1049 } 1050 1051 static int safexcel_aead_send(struct crypto_async_request *async, int ring, 1052 int *commands, int *results) 1053 { 1054 struct aead_request *req = aead_request_cast(async); 1055 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1056 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); 1057 struct safexcel_cipher_req *sreq = aead_request_ctx(req); 1058 struct safexcel_crypto_priv *priv = ctx->base.priv; 1059 int ret; 1060 1061 BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv); 1062 1063 if (sreq->needs_inv) 1064 ret = safexcel_cipher_send_inv(async, ring, commands, results); 1065 else 1066 ret = safexcel_send_req(async, ring, sreq, req->src, req->dst, 1067 req->cryptlen, req->assoclen, 1068 crypto_aead_authsize(tfm), req->iv, 1069 commands, results); 1070 sreq->rdescs = *results; 1071 return ret; 1072 } 1073 1074 static int safexcel_cipher_exit_inv(struct crypto_tfm *tfm, 1075 struct crypto_async_request *base, 1076 struct safexcel_cipher_req *sreq, 1077 struct safexcel_inv_result *result) 1078 { 1079 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1080 struct safexcel_crypto_priv *priv = ctx->base.priv; 1081 int ring = ctx->base.ring; 1082 1083 init_completion(&result->completion); 1084 1085 ctx = crypto_tfm_ctx(base->tfm); 1086 ctx->base.exit_inv = true; 1087 sreq->needs_inv = true; 1088 1089 spin_lock_bh(&priv->ring[ring].queue_lock); 1090 crypto_enqueue_request(&priv->ring[ring].queue, base); 1091 spin_unlock_bh(&priv->ring[ring].queue_lock); 1092 1093 queue_work(priv->ring[ring].workqueue, 1094 &priv->ring[ring].work_data.work); 1095 1096 wait_for_completion(&result->completion); 1097 1098 if (result->error) { 1099 dev_warn(priv->dev, 1100 "cipher: sync: invalidate: completion error %d\n", 1101 result->error); 1102 return result->error; 1103 } 1104 1105 return 0; 1106 } 1107 1108 static int safexcel_skcipher_exit_inv(struct crypto_tfm *tfm) 1109 { 1110 EIP197_REQUEST_ON_STACK(req, skcipher, EIP197_SKCIPHER_REQ_SIZE); 1111 struct safexcel_cipher_req *sreq = skcipher_request_ctx(req); 1112 struct safexcel_inv_result result = {}; 1113 1114 memset(req, 0, sizeof(struct skcipher_request)); 1115 1116 skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 1117 safexcel_inv_complete, &result); 1118 skcipher_request_set_tfm(req, __crypto_skcipher_cast(tfm)); 1119 1120 return safexcel_cipher_exit_inv(tfm, &req->base, sreq, &result); 1121 } 1122 1123 static int safexcel_aead_exit_inv(struct crypto_tfm *tfm) 1124 { 1125 EIP197_REQUEST_ON_STACK(req, aead, EIP197_AEAD_REQ_SIZE); 1126 struct safexcel_cipher_req *sreq = aead_request_ctx(req); 1127 struct safexcel_inv_result result = {}; 1128 1129 memset(req, 0, sizeof(struct aead_request)); 1130 1131 aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 1132 safexcel_inv_complete, &result); 1133 aead_request_set_tfm(req, __crypto_aead_cast(tfm)); 1134 1135 return safexcel_cipher_exit_inv(tfm, &req->base, sreq, &result); 1136 } 1137 1138 static int safexcel_queue_req(struct crypto_async_request *base, 1139 struct safexcel_cipher_req *sreq, 1140 enum safexcel_cipher_direction dir) 1141 { 1142 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); 1143 struct safexcel_crypto_priv *priv = ctx->base.priv; 1144 int ret, ring; 1145 1146 sreq->needs_inv = false; 1147 sreq->direction = dir; 1148 1149 if (ctx->base.ctxr) { 1150 if (priv->flags & EIP197_TRC_CACHE && ctx->base.needs_inv) { 1151 sreq->needs_inv = true; 1152 ctx->base.needs_inv = false; 1153 } 1154 } else { 1155 ctx->base.ring = safexcel_select_ring(priv); 1156 ctx->base.ctxr = dma_pool_zalloc(priv->context_pool, 1157 EIP197_GFP_FLAGS(*base), 1158 &ctx->base.ctxr_dma); 1159 if (!ctx->base.ctxr) 1160 return -ENOMEM; 1161 } 1162 1163 ring = ctx->base.ring; 1164 1165 spin_lock_bh(&priv->ring[ring].queue_lock); 1166 ret = crypto_enqueue_request(&priv->ring[ring].queue, base); 1167 spin_unlock_bh(&priv->ring[ring].queue_lock); 1168 1169 queue_work(priv->ring[ring].workqueue, 1170 &priv->ring[ring].work_data.work); 1171 1172 return ret; 1173 } 1174 1175 static int safexcel_encrypt(struct skcipher_request *req) 1176 { 1177 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 1178 SAFEXCEL_ENCRYPT); 1179 } 1180 1181 static int safexcel_decrypt(struct skcipher_request *req) 1182 { 1183 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 1184 SAFEXCEL_DECRYPT); 1185 } 1186 1187 static int safexcel_skcipher_cra_init(struct crypto_tfm *tfm) 1188 { 1189 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1190 struct safexcel_alg_template *tmpl = 1191 container_of(tfm->__crt_alg, struct safexcel_alg_template, 1192 alg.skcipher.base); 1193 1194 crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm), 1195 sizeof(struct safexcel_cipher_req)); 1196 1197 ctx->base.priv = tmpl->priv; 1198 1199 ctx->base.send = safexcel_skcipher_send; 1200 ctx->base.handle_result = safexcel_skcipher_handle_result; 1201 ctx->ivmask = EIP197_OPTION_4_TOKEN_IV_CMD; 1202 ctx->ctrinit = 1; 1203 return 0; 1204 } 1205 1206 static int safexcel_cipher_cra_exit(struct crypto_tfm *tfm) 1207 { 1208 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1209 1210 memzero_explicit(ctx->key, sizeof(ctx->key)); 1211 1212 /* context not allocated, skip invalidation */ 1213 if (!ctx->base.ctxr) 1214 return -ENOMEM; 1215 1216 memzero_explicit(ctx->base.ctxr->data, sizeof(ctx->base.ctxr->data)); 1217 return 0; 1218 } 1219 1220 static void safexcel_skcipher_cra_exit(struct crypto_tfm *tfm) 1221 { 1222 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1223 struct safexcel_crypto_priv *priv = ctx->base.priv; 1224 int ret; 1225 1226 if (safexcel_cipher_cra_exit(tfm)) 1227 return; 1228 1229 if (priv->flags & EIP197_TRC_CACHE) { 1230 ret = safexcel_skcipher_exit_inv(tfm); 1231 if (ret) 1232 dev_warn(priv->dev, "skcipher: invalidation error %d\n", 1233 ret); 1234 } else { 1235 dma_pool_free(priv->context_pool, ctx->base.ctxr, 1236 ctx->base.ctxr_dma); 1237 } 1238 } 1239 1240 static void safexcel_aead_cra_exit(struct crypto_tfm *tfm) 1241 { 1242 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1243 struct safexcel_crypto_priv *priv = ctx->base.priv; 1244 int ret; 1245 1246 if (safexcel_cipher_cra_exit(tfm)) 1247 return; 1248 1249 if (priv->flags & EIP197_TRC_CACHE) { 1250 ret = safexcel_aead_exit_inv(tfm); 1251 if (ret) 1252 dev_warn(priv->dev, "aead: invalidation error %d\n", 1253 ret); 1254 } else { 1255 dma_pool_free(priv->context_pool, ctx->base.ctxr, 1256 ctx->base.ctxr_dma); 1257 } 1258 } 1259 1260 static int safexcel_skcipher_aes_ecb_cra_init(struct crypto_tfm *tfm) 1261 { 1262 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1263 1264 safexcel_skcipher_cra_init(tfm); 1265 ctx->alg = SAFEXCEL_AES; 1266 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; 1267 ctx->blocksz = 0; 1268 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1269 return 0; 1270 } 1271 1272 struct safexcel_alg_template safexcel_alg_ecb_aes = { 1273 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1274 .algo_mask = SAFEXCEL_ALG_AES, 1275 .alg.skcipher = { 1276 .setkey = safexcel_skcipher_aes_setkey, 1277 .encrypt = safexcel_encrypt, 1278 .decrypt = safexcel_decrypt, 1279 .min_keysize = AES_MIN_KEY_SIZE, 1280 .max_keysize = AES_MAX_KEY_SIZE, 1281 .base = { 1282 .cra_name = "ecb(aes)", 1283 .cra_driver_name = "safexcel-ecb-aes", 1284 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1285 .cra_flags = CRYPTO_ALG_ASYNC | 1286 CRYPTO_ALG_ALLOCATES_MEMORY | 1287 CRYPTO_ALG_KERN_DRIVER_ONLY, 1288 .cra_blocksize = AES_BLOCK_SIZE, 1289 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1290 .cra_alignmask = 0, 1291 .cra_init = safexcel_skcipher_aes_ecb_cra_init, 1292 .cra_exit = safexcel_skcipher_cra_exit, 1293 .cra_module = THIS_MODULE, 1294 }, 1295 }, 1296 }; 1297 1298 static int safexcel_skcipher_aes_cbc_cra_init(struct crypto_tfm *tfm) 1299 { 1300 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1301 1302 safexcel_skcipher_cra_init(tfm); 1303 ctx->alg = SAFEXCEL_AES; 1304 ctx->blocksz = AES_BLOCK_SIZE; 1305 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; 1306 return 0; 1307 } 1308 1309 struct safexcel_alg_template safexcel_alg_cbc_aes = { 1310 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1311 .algo_mask = SAFEXCEL_ALG_AES, 1312 .alg.skcipher = { 1313 .setkey = safexcel_skcipher_aes_setkey, 1314 .encrypt = safexcel_encrypt, 1315 .decrypt = safexcel_decrypt, 1316 .min_keysize = AES_MIN_KEY_SIZE, 1317 .max_keysize = AES_MAX_KEY_SIZE, 1318 .ivsize = AES_BLOCK_SIZE, 1319 .base = { 1320 .cra_name = "cbc(aes)", 1321 .cra_driver_name = "safexcel-cbc-aes", 1322 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1323 .cra_flags = CRYPTO_ALG_ASYNC | 1324 CRYPTO_ALG_ALLOCATES_MEMORY | 1325 CRYPTO_ALG_KERN_DRIVER_ONLY, 1326 .cra_blocksize = AES_BLOCK_SIZE, 1327 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1328 .cra_alignmask = 0, 1329 .cra_init = safexcel_skcipher_aes_cbc_cra_init, 1330 .cra_exit = safexcel_skcipher_cra_exit, 1331 .cra_module = THIS_MODULE, 1332 }, 1333 }, 1334 }; 1335 1336 static int safexcel_skcipher_aes_cfb_cra_init(struct crypto_tfm *tfm) 1337 { 1338 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1339 1340 safexcel_skcipher_cra_init(tfm); 1341 ctx->alg = SAFEXCEL_AES; 1342 ctx->blocksz = AES_BLOCK_SIZE; 1343 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CFB; 1344 return 0; 1345 } 1346 1347 struct safexcel_alg_template safexcel_alg_cfb_aes = { 1348 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1349 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_AES_XFB, 1350 .alg.skcipher = { 1351 .setkey = safexcel_skcipher_aes_setkey, 1352 .encrypt = safexcel_encrypt, 1353 .decrypt = safexcel_decrypt, 1354 .min_keysize = AES_MIN_KEY_SIZE, 1355 .max_keysize = AES_MAX_KEY_SIZE, 1356 .ivsize = AES_BLOCK_SIZE, 1357 .base = { 1358 .cra_name = "cfb(aes)", 1359 .cra_driver_name = "safexcel-cfb-aes", 1360 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1361 .cra_flags = CRYPTO_ALG_ASYNC | 1362 CRYPTO_ALG_ALLOCATES_MEMORY | 1363 CRYPTO_ALG_KERN_DRIVER_ONLY, 1364 .cra_blocksize = 1, 1365 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1366 .cra_alignmask = 0, 1367 .cra_init = safexcel_skcipher_aes_cfb_cra_init, 1368 .cra_exit = safexcel_skcipher_cra_exit, 1369 .cra_module = THIS_MODULE, 1370 }, 1371 }, 1372 }; 1373 1374 static int safexcel_skcipher_aes_ofb_cra_init(struct crypto_tfm *tfm) 1375 { 1376 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1377 1378 safexcel_skcipher_cra_init(tfm); 1379 ctx->alg = SAFEXCEL_AES; 1380 ctx->blocksz = AES_BLOCK_SIZE; 1381 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_OFB; 1382 return 0; 1383 } 1384 1385 struct safexcel_alg_template safexcel_alg_ofb_aes = { 1386 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1387 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_AES_XFB, 1388 .alg.skcipher = { 1389 .setkey = safexcel_skcipher_aes_setkey, 1390 .encrypt = safexcel_encrypt, 1391 .decrypt = safexcel_decrypt, 1392 .min_keysize = AES_MIN_KEY_SIZE, 1393 .max_keysize = AES_MAX_KEY_SIZE, 1394 .ivsize = AES_BLOCK_SIZE, 1395 .base = { 1396 .cra_name = "ofb(aes)", 1397 .cra_driver_name = "safexcel-ofb-aes", 1398 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1399 .cra_flags = CRYPTO_ALG_ASYNC | 1400 CRYPTO_ALG_ALLOCATES_MEMORY | 1401 CRYPTO_ALG_KERN_DRIVER_ONLY, 1402 .cra_blocksize = 1, 1403 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1404 .cra_alignmask = 0, 1405 .cra_init = safexcel_skcipher_aes_ofb_cra_init, 1406 .cra_exit = safexcel_skcipher_cra_exit, 1407 .cra_module = THIS_MODULE, 1408 }, 1409 }, 1410 }; 1411 1412 static int safexcel_skcipher_aesctr_setkey(struct crypto_skcipher *ctfm, 1413 const u8 *key, unsigned int len) 1414 { 1415 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 1416 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1417 struct safexcel_crypto_priv *priv = ctx->base.priv; 1418 struct crypto_aes_ctx aes; 1419 int ret, i; 1420 unsigned int keylen; 1421 1422 /* last 4 bytes of key are the nonce! */ 1423 ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE); 1424 /* exclude the nonce here */ 1425 keylen = len - CTR_RFC3686_NONCE_SIZE; 1426 ret = aes_expandkey(&aes, key, keylen); 1427 if (ret) 1428 return ret; 1429 1430 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 1431 for (i = 0; i < keylen / sizeof(u32); i++) { 1432 if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { 1433 ctx->base.needs_inv = true; 1434 break; 1435 } 1436 } 1437 } 1438 1439 for (i = 0; i < keylen / sizeof(u32); i++) 1440 ctx->key[i] = cpu_to_le32(aes.key_enc[i]); 1441 1442 ctx->key_len = keylen; 1443 1444 memzero_explicit(&aes, sizeof(aes)); 1445 return 0; 1446 } 1447 1448 static int safexcel_skcipher_aes_ctr_cra_init(struct crypto_tfm *tfm) 1449 { 1450 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1451 1452 safexcel_skcipher_cra_init(tfm); 1453 ctx->alg = SAFEXCEL_AES; 1454 ctx->blocksz = AES_BLOCK_SIZE; 1455 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; 1456 return 0; 1457 } 1458 1459 struct safexcel_alg_template safexcel_alg_ctr_aes = { 1460 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1461 .algo_mask = SAFEXCEL_ALG_AES, 1462 .alg.skcipher = { 1463 .setkey = safexcel_skcipher_aesctr_setkey, 1464 .encrypt = safexcel_encrypt, 1465 .decrypt = safexcel_decrypt, 1466 /* Add nonce size */ 1467 .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 1468 .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 1469 .ivsize = CTR_RFC3686_IV_SIZE, 1470 .base = { 1471 .cra_name = "rfc3686(ctr(aes))", 1472 .cra_driver_name = "safexcel-ctr-aes", 1473 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1474 .cra_flags = CRYPTO_ALG_ASYNC | 1475 CRYPTO_ALG_ALLOCATES_MEMORY | 1476 CRYPTO_ALG_KERN_DRIVER_ONLY, 1477 .cra_blocksize = 1, 1478 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1479 .cra_alignmask = 0, 1480 .cra_init = safexcel_skcipher_aes_ctr_cra_init, 1481 .cra_exit = safexcel_skcipher_cra_exit, 1482 .cra_module = THIS_MODULE, 1483 }, 1484 }, 1485 }; 1486 1487 static int safexcel_des_setkey(struct crypto_skcipher *ctfm, const u8 *key, 1488 unsigned int len) 1489 { 1490 struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm); 1491 struct safexcel_crypto_priv *priv = ctx->base.priv; 1492 int ret; 1493 1494 ret = verify_skcipher_des_key(ctfm, key); 1495 if (ret) 1496 return ret; 1497 1498 /* if context exits and key changed, need to invalidate it */ 1499 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) 1500 if (memcmp(ctx->key, key, len)) 1501 ctx->base.needs_inv = true; 1502 1503 memcpy(ctx->key, key, len); 1504 ctx->key_len = len; 1505 1506 return 0; 1507 } 1508 1509 static int safexcel_skcipher_des_cbc_cra_init(struct crypto_tfm *tfm) 1510 { 1511 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1512 1513 safexcel_skcipher_cra_init(tfm); 1514 ctx->alg = SAFEXCEL_DES; 1515 ctx->blocksz = DES_BLOCK_SIZE; 1516 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1517 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; 1518 return 0; 1519 } 1520 1521 struct safexcel_alg_template safexcel_alg_cbc_des = { 1522 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1523 .algo_mask = SAFEXCEL_ALG_DES, 1524 .alg.skcipher = { 1525 .setkey = safexcel_des_setkey, 1526 .encrypt = safexcel_encrypt, 1527 .decrypt = safexcel_decrypt, 1528 .min_keysize = DES_KEY_SIZE, 1529 .max_keysize = DES_KEY_SIZE, 1530 .ivsize = DES_BLOCK_SIZE, 1531 .base = { 1532 .cra_name = "cbc(des)", 1533 .cra_driver_name = "safexcel-cbc-des", 1534 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1535 .cra_flags = CRYPTO_ALG_ASYNC | 1536 CRYPTO_ALG_ALLOCATES_MEMORY | 1537 CRYPTO_ALG_KERN_DRIVER_ONLY, 1538 .cra_blocksize = DES_BLOCK_SIZE, 1539 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1540 .cra_alignmask = 0, 1541 .cra_init = safexcel_skcipher_des_cbc_cra_init, 1542 .cra_exit = safexcel_skcipher_cra_exit, 1543 .cra_module = THIS_MODULE, 1544 }, 1545 }, 1546 }; 1547 1548 static int safexcel_skcipher_des_ecb_cra_init(struct crypto_tfm *tfm) 1549 { 1550 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1551 1552 safexcel_skcipher_cra_init(tfm); 1553 ctx->alg = SAFEXCEL_DES; 1554 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; 1555 ctx->blocksz = 0; 1556 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1557 return 0; 1558 } 1559 1560 struct safexcel_alg_template safexcel_alg_ecb_des = { 1561 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1562 .algo_mask = SAFEXCEL_ALG_DES, 1563 .alg.skcipher = { 1564 .setkey = safexcel_des_setkey, 1565 .encrypt = safexcel_encrypt, 1566 .decrypt = safexcel_decrypt, 1567 .min_keysize = DES_KEY_SIZE, 1568 .max_keysize = DES_KEY_SIZE, 1569 .base = { 1570 .cra_name = "ecb(des)", 1571 .cra_driver_name = "safexcel-ecb-des", 1572 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1573 .cra_flags = CRYPTO_ALG_ASYNC | 1574 CRYPTO_ALG_ALLOCATES_MEMORY | 1575 CRYPTO_ALG_KERN_DRIVER_ONLY, 1576 .cra_blocksize = DES_BLOCK_SIZE, 1577 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1578 .cra_alignmask = 0, 1579 .cra_init = safexcel_skcipher_des_ecb_cra_init, 1580 .cra_exit = safexcel_skcipher_cra_exit, 1581 .cra_module = THIS_MODULE, 1582 }, 1583 }, 1584 }; 1585 1586 static int safexcel_des3_ede_setkey(struct crypto_skcipher *ctfm, 1587 const u8 *key, unsigned int len) 1588 { 1589 struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm); 1590 struct safexcel_crypto_priv *priv = ctx->base.priv; 1591 int err; 1592 1593 err = verify_skcipher_des3_key(ctfm, key); 1594 if (err) 1595 return err; 1596 1597 /* if context exits and key changed, need to invalidate it */ 1598 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) 1599 if (memcmp(ctx->key, key, len)) 1600 ctx->base.needs_inv = true; 1601 1602 memcpy(ctx->key, key, len); 1603 ctx->key_len = len; 1604 1605 return 0; 1606 } 1607 1608 static int safexcel_skcipher_des3_cbc_cra_init(struct crypto_tfm *tfm) 1609 { 1610 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1611 1612 safexcel_skcipher_cra_init(tfm); 1613 ctx->alg = SAFEXCEL_3DES; 1614 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 1615 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1616 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; 1617 return 0; 1618 } 1619 1620 struct safexcel_alg_template safexcel_alg_cbc_des3_ede = { 1621 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1622 .algo_mask = SAFEXCEL_ALG_DES, 1623 .alg.skcipher = { 1624 .setkey = safexcel_des3_ede_setkey, 1625 .encrypt = safexcel_encrypt, 1626 .decrypt = safexcel_decrypt, 1627 .min_keysize = DES3_EDE_KEY_SIZE, 1628 .max_keysize = DES3_EDE_KEY_SIZE, 1629 .ivsize = DES3_EDE_BLOCK_SIZE, 1630 .base = { 1631 .cra_name = "cbc(des3_ede)", 1632 .cra_driver_name = "safexcel-cbc-des3_ede", 1633 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1634 .cra_flags = CRYPTO_ALG_ASYNC | 1635 CRYPTO_ALG_ALLOCATES_MEMORY | 1636 CRYPTO_ALG_KERN_DRIVER_ONLY, 1637 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1638 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1639 .cra_alignmask = 0, 1640 .cra_init = safexcel_skcipher_des3_cbc_cra_init, 1641 .cra_exit = safexcel_skcipher_cra_exit, 1642 .cra_module = THIS_MODULE, 1643 }, 1644 }, 1645 }; 1646 1647 static int safexcel_skcipher_des3_ecb_cra_init(struct crypto_tfm *tfm) 1648 { 1649 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1650 1651 safexcel_skcipher_cra_init(tfm); 1652 ctx->alg = SAFEXCEL_3DES; 1653 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; 1654 ctx->blocksz = 0; 1655 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1656 return 0; 1657 } 1658 1659 struct safexcel_alg_template safexcel_alg_ecb_des3_ede = { 1660 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1661 .algo_mask = SAFEXCEL_ALG_DES, 1662 .alg.skcipher = { 1663 .setkey = safexcel_des3_ede_setkey, 1664 .encrypt = safexcel_encrypt, 1665 .decrypt = safexcel_decrypt, 1666 .min_keysize = DES3_EDE_KEY_SIZE, 1667 .max_keysize = DES3_EDE_KEY_SIZE, 1668 .base = { 1669 .cra_name = "ecb(des3_ede)", 1670 .cra_driver_name = "safexcel-ecb-des3_ede", 1671 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1672 .cra_flags = CRYPTO_ALG_ASYNC | 1673 CRYPTO_ALG_ALLOCATES_MEMORY | 1674 CRYPTO_ALG_KERN_DRIVER_ONLY, 1675 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1676 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1677 .cra_alignmask = 0, 1678 .cra_init = safexcel_skcipher_des3_ecb_cra_init, 1679 .cra_exit = safexcel_skcipher_cra_exit, 1680 .cra_module = THIS_MODULE, 1681 }, 1682 }, 1683 }; 1684 1685 static int safexcel_aead_encrypt(struct aead_request *req) 1686 { 1687 struct safexcel_cipher_req *creq = aead_request_ctx(req); 1688 1689 return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); 1690 } 1691 1692 static int safexcel_aead_decrypt(struct aead_request *req) 1693 { 1694 struct safexcel_cipher_req *creq = aead_request_ctx(req); 1695 1696 return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); 1697 } 1698 1699 static int safexcel_aead_cra_init(struct crypto_tfm *tfm) 1700 { 1701 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1702 struct safexcel_alg_template *tmpl = 1703 container_of(tfm->__crt_alg, struct safexcel_alg_template, 1704 alg.aead.base); 1705 1706 crypto_aead_set_reqsize(__crypto_aead_cast(tfm), 1707 sizeof(struct safexcel_cipher_req)); 1708 1709 ctx->base.priv = tmpl->priv; 1710 1711 ctx->alg = SAFEXCEL_AES; /* default */ 1712 ctx->blocksz = AES_BLOCK_SIZE; 1713 ctx->ivmask = EIP197_OPTION_4_TOKEN_IV_CMD; 1714 ctx->ctrinit = 1; 1715 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; /* default */ 1716 ctx->aead = true; 1717 ctx->base.send = safexcel_aead_send; 1718 ctx->base.handle_result = safexcel_aead_handle_result; 1719 return 0; 1720 } 1721 1722 static int safexcel_aead_sha1_cra_init(struct crypto_tfm *tfm) 1723 { 1724 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1725 1726 safexcel_aead_cra_init(tfm); 1727 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1; 1728 ctx->state_sz = SHA1_DIGEST_SIZE; 1729 return 0; 1730 } 1731 1732 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_aes = { 1733 .type = SAFEXCEL_ALG_TYPE_AEAD, 1734 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA1, 1735 .alg.aead = { 1736 .setkey = safexcel_aead_setkey, 1737 .encrypt = safexcel_aead_encrypt, 1738 .decrypt = safexcel_aead_decrypt, 1739 .ivsize = AES_BLOCK_SIZE, 1740 .maxauthsize = SHA1_DIGEST_SIZE, 1741 .base = { 1742 .cra_name = "authenc(hmac(sha1),cbc(aes))", 1743 .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-aes", 1744 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1745 .cra_flags = CRYPTO_ALG_ASYNC | 1746 CRYPTO_ALG_ALLOCATES_MEMORY | 1747 CRYPTO_ALG_KERN_DRIVER_ONLY, 1748 .cra_blocksize = AES_BLOCK_SIZE, 1749 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1750 .cra_alignmask = 0, 1751 .cra_init = safexcel_aead_sha1_cra_init, 1752 .cra_exit = safexcel_aead_cra_exit, 1753 .cra_module = THIS_MODULE, 1754 }, 1755 }, 1756 }; 1757 1758 static int safexcel_aead_sha256_cra_init(struct crypto_tfm *tfm) 1759 { 1760 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1761 1762 safexcel_aead_cra_init(tfm); 1763 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256; 1764 ctx->state_sz = SHA256_DIGEST_SIZE; 1765 return 0; 1766 } 1767 1768 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_aes = { 1769 .type = SAFEXCEL_ALG_TYPE_AEAD, 1770 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256, 1771 .alg.aead = { 1772 .setkey = safexcel_aead_setkey, 1773 .encrypt = safexcel_aead_encrypt, 1774 .decrypt = safexcel_aead_decrypt, 1775 .ivsize = AES_BLOCK_SIZE, 1776 .maxauthsize = SHA256_DIGEST_SIZE, 1777 .base = { 1778 .cra_name = "authenc(hmac(sha256),cbc(aes))", 1779 .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-aes", 1780 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1781 .cra_flags = CRYPTO_ALG_ASYNC | 1782 CRYPTO_ALG_ALLOCATES_MEMORY | 1783 CRYPTO_ALG_KERN_DRIVER_ONLY, 1784 .cra_blocksize = AES_BLOCK_SIZE, 1785 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1786 .cra_alignmask = 0, 1787 .cra_init = safexcel_aead_sha256_cra_init, 1788 .cra_exit = safexcel_aead_cra_exit, 1789 .cra_module = THIS_MODULE, 1790 }, 1791 }, 1792 }; 1793 1794 static int safexcel_aead_sha224_cra_init(struct crypto_tfm *tfm) 1795 { 1796 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1797 1798 safexcel_aead_cra_init(tfm); 1799 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224; 1800 ctx->state_sz = SHA256_DIGEST_SIZE; 1801 return 0; 1802 } 1803 1804 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_aes = { 1805 .type = SAFEXCEL_ALG_TYPE_AEAD, 1806 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256, 1807 .alg.aead = { 1808 .setkey = safexcel_aead_setkey, 1809 .encrypt = safexcel_aead_encrypt, 1810 .decrypt = safexcel_aead_decrypt, 1811 .ivsize = AES_BLOCK_SIZE, 1812 .maxauthsize = SHA224_DIGEST_SIZE, 1813 .base = { 1814 .cra_name = "authenc(hmac(sha224),cbc(aes))", 1815 .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-aes", 1816 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1817 .cra_flags = CRYPTO_ALG_ASYNC | 1818 CRYPTO_ALG_ALLOCATES_MEMORY | 1819 CRYPTO_ALG_KERN_DRIVER_ONLY, 1820 .cra_blocksize = AES_BLOCK_SIZE, 1821 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1822 .cra_alignmask = 0, 1823 .cra_init = safexcel_aead_sha224_cra_init, 1824 .cra_exit = safexcel_aead_cra_exit, 1825 .cra_module = THIS_MODULE, 1826 }, 1827 }, 1828 }; 1829 1830 static int safexcel_aead_sha512_cra_init(struct crypto_tfm *tfm) 1831 { 1832 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1833 1834 safexcel_aead_cra_init(tfm); 1835 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512; 1836 ctx->state_sz = SHA512_DIGEST_SIZE; 1837 return 0; 1838 } 1839 1840 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_aes = { 1841 .type = SAFEXCEL_ALG_TYPE_AEAD, 1842 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512, 1843 .alg.aead = { 1844 .setkey = safexcel_aead_setkey, 1845 .encrypt = safexcel_aead_encrypt, 1846 .decrypt = safexcel_aead_decrypt, 1847 .ivsize = AES_BLOCK_SIZE, 1848 .maxauthsize = SHA512_DIGEST_SIZE, 1849 .base = { 1850 .cra_name = "authenc(hmac(sha512),cbc(aes))", 1851 .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-aes", 1852 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1853 .cra_flags = CRYPTO_ALG_ASYNC | 1854 CRYPTO_ALG_ALLOCATES_MEMORY | 1855 CRYPTO_ALG_KERN_DRIVER_ONLY, 1856 .cra_blocksize = AES_BLOCK_SIZE, 1857 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1858 .cra_alignmask = 0, 1859 .cra_init = safexcel_aead_sha512_cra_init, 1860 .cra_exit = safexcel_aead_cra_exit, 1861 .cra_module = THIS_MODULE, 1862 }, 1863 }, 1864 }; 1865 1866 static int safexcel_aead_sha384_cra_init(struct crypto_tfm *tfm) 1867 { 1868 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1869 1870 safexcel_aead_cra_init(tfm); 1871 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384; 1872 ctx->state_sz = SHA512_DIGEST_SIZE; 1873 return 0; 1874 } 1875 1876 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_aes = { 1877 .type = SAFEXCEL_ALG_TYPE_AEAD, 1878 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512, 1879 .alg.aead = { 1880 .setkey = safexcel_aead_setkey, 1881 .encrypt = safexcel_aead_encrypt, 1882 .decrypt = safexcel_aead_decrypt, 1883 .ivsize = AES_BLOCK_SIZE, 1884 .maxauthsize = SHA384_DIGEST_SIZE, 1885 .base = { 1886 .cra_name = "authenc(hmac(sha384),cbc(aes))", 1887 .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-aes", 1888 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1889 .cra_flags = CRYPTO_ALG_ASYNC | 1890 CRYPTO_ALG_ALLOCATES_MEMORY | 1891 CRYPTO_ALG_KERN_DRIVER_ONLY, 1892 .cra_blocksize = AES_BLOCK_SIZE, 1893 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1894 .cra_alignmask = 0, 1895 .cra_init = safexcel_aead_sha384_cra_init, 1896 .cra_exit = safexcel_aead_cra_exit, 1897 .cra_module = THIS_MODULE, 1898 }, 1899 }, 1900 }; 1901 1902 static int safexcel_aead_sha1_des3_cra_init(struct crypto_tfm *tfm) 1903 { 1904 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1905 1906 safexcel_aead_sha1_cra_init(tfm); 1907 ctx->alg = SAFEXCEL_3DES; /* override default */ 1908 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 1909 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1910 return 0; 1911 } 1912 1913 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des3_ede = { 1914 .type = SAFEXCEL_ALG_TYPE_AEAD, 1915 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA1, 1916 .alg.aead = { 1917 .setkey = safexcel_aead_setkey, 1918 .encrypt = safexcel_aead_encrypt, 1919 .decrypt = safexcel_aead_decrypt, 1920 .ivsize = DES3_EDE_BLOCK_SIZE, 1921 .maxauthsize = SHA1_DIGEST_SIZE, 1922 .base = { 1923 .cra_name = "authenc(hmac(sha1),cbc(des3_ede))", 1924 .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-des3_ede", 1925 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1926 .cra_flags = CRYPTO_ALG_ASYNC | 1927 CRYPTO_ALG_ALLOCATES_MEMORY | 1928 CRYPTO_ALG_KERN_DRIVER_ONLY, 1929 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1930 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1931 .cra_alignmask = 0, 1932 .cra_init = safexcel_aead_sha1_des3_cra_init, 1933 .cra_exit = safexcel_aead_cra_exit, 1934 .cra_module = THIS_MODULE, 1935 }, 1936 }, 1937 }; 1938 1939 static int safexcel_aead_sha256_des3_cra_init(struct crypto_tfm *tfm) 1940 { 1941 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1942 1943 safexcel_aead_sha256_cra_init(tfm); 1944 ctx->alg = SAFEXCEL_3DES; /* override default */ 1945 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 1946 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1947 return 0; 1948 } 1949 1950 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des3_ede = { 1951 .type = SAFEXCEL_ALG_TYPE_AEAD, 1952 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, 1953 .alg.aead = { 1954 .setkey = safexcel_aead_setkey, 1955 .encrypt = safexcel_aead_encrypt, 1956 .decrypt = safexcel_aead_decrypt, 1957 .ivsize = DES3_EDE_BLOCK_SIZE, 1958 .maxauthsize = SHA256_DIGEST_SIZE, 1959 .base = { 1960 .cra_name = "authenc(hmac(sha256),cbc(des3_ede))", 1961 .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-des3_ede", 1962 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1963 .cra_flags = CRYPTO_ALG_ASYNC | 1964 CRYPTO_ALG_ALLOCATES_MEMORY | 1965 CRYPTO_ALG_KERN_DRIVER_ONLY, 1966 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1967 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1968 .cra_alignmask = 0, 1969 .cra_init = safexcel_aead_sha256_des3_cra_init, 1970 .cra_exit = safexcel_aead_cra_exit, 1971 .cra_module = THIS_MODULE, 1972 }, 1973 }, 1974 }; 1975 1976 static int safexcel_aead_sha224_des3_cra_init(struct crypto_tfm *tfm) 1977 { 1978 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1979 1980 safexcel_aead_sha224_cra_init(tfm); 1981 ctx->alg = SAFEXCEL_3DES; /* override default */ 1982 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 1983 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1984 return 0; 1985 } 1986 1987 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des3_ede = { 1988 .type = SAFEXCEL_ALG_TYPE_AEAD, 1989 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, 1990 .alg.aead = { 1991 .setkey = safexcel_aead_setkey, 1992 .encrypt = safexcel_aead_encrypt, 1993 .decrypt = safexcel_aead_decrypt, 1994 .ivsize = DES3_EDE_BLOCK_SIZE, 1995 .maxauthsize = SHA224_DIGEST_SIZE, 1996 .base = { 1997 .cra_name = "authenc(hmac(sha224),cbc(des3_ede))", 1998 .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-des3_ede", 1999 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2000 .cra_flags = CRYPTO_ALG_ASYNC | 2001 CRYPTO_ALG_ALLOCATES_MEMORY | 2002 CRYPTO_ALG_KERN_DRIVER_ONLY, 2003 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2004 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2005 .cra_alignmask = 0, 2006 .cra_init = safexcel_aead_sha224_des3_cra_init, 2007 .cra_exit = safexcel_aead_cra_exit, 2008 .cra_module = THIS_MODULE, 2009 }, 2010 }, 2011 }; 2012 2013 static int safexcel_aead_sha512_des3_cra_init(struct crypto_tfm *tfm) 2014 { 2015 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2016 2017 safexcel_aead_sha512_cra_init(tfm); 2018 ctx->alg = SAFEXCEL_3DES; /* override default */ 2019 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 2020 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2021 return 0; 2022 } 2023 2024 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des3_ede = { 2025 .type = SAFEXCEL_ALG_TYPE_AEAD, 2026 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, 2027 .alg.aead = { 2028 .setkey = safexcel_aead_setkey, 2029 .encrypt = safexcel_aead_encrypt, 2030 .decrypt = safexcel_aead_decrypt, 2031 .ivsize = DES3_EDE_BLOCK_SIZE, 2032 .maxauthsize = SHA512_DIGEST_SIZE, 2033 .base = { 2034 .cra_name = "authenc(hmac(sha512),cbc(des3_ede))", 2035 .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-des3_ede", 2036 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2037 .cra_flags = CRYPTO_ALG_ASYNC | 2038 CRYPTO_ALG_ALLOCATES_MEMORY | 2039 CRYPTO_ALG_KERN_DRIVER_ONLY, 2040 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2041 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2042 .cra_alignmask = 0, 2043 .cra_init = safexcel_aead_sha512_des3_cra_init, 2044 .cra_exit = safexcel_aead_cra_exit, 2045 .cra_module = THIS_MODULE, 2046 }, 2047 }, 2048 }; 2049 2050 static int safexcel_aead_sha384_des3_cra_init(struct crypto_tfm *tfm) 2051 { 2052 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2053 2054 safexcel_aead_sha384_cra_init(tfm); 2055 ctx->alg = SAFEXCEL_3DES; /* override default */ 2056 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 2057 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2058 return 0; 2059 } 2060 2061 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des3_ede = { 2062 .type = SAFEXCEL_ALG_TYPE_AEAD, 2063 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, 2064 .alg.aead = { 2065 .setkey = safexcel_aead_setkey, 2066 .encrypt = safexcel_aead_encrypt, 2067 .decrypt = safexcel_aead_decrypt, 2068 .ivsize = DES3_EDE_BLOCK_SIZE, 2069 .maxauthsize = SHA384_DIGEST_SIZE, 2070 .base = { 2071 .cra_name = "authenc(hmac(sha384),cbc(des3_ede))", 2072 .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-des3_ede", 2073 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2074 .cra_flags = CRYPTO_ALG_ASYNC | 2075 CRYPTO_ALG_ALLOCATES_MEMORY | 2076 CRYPTO_ALG_KERN_DRIVER_ONLY, 2077 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2078 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2079 .cra_alignmask = 0, 2080 .cra_init = safexcel_aead_sha384_des3_cra_init, 2081 .cra_exit = safexcel_aead_cra_exit, 2082 .cra_module = THIS_MODULE, 2083 }, 2084 }, 2085 }; 2086 2087 static int safexcel_aead_sha1_des_cra_init(struct crypto_tfm *tfm) 2088 { 2089 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2090 2091 safexcel_aead_sha1_cra_init(tfm); 2092 ctx->alg = SAFEXCEL_DES; /* override default */ 2093 ctx->blocksz = DES_BLOCK_SIZE; 2094 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2095 return 0; 2096 } 2097 2098 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des = { 2099 .type = SAFEXCEL_ALG_TYPE_AEAD, 2100 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA1, 2101 .alg.aead = { 2102 .setkey = safexcel_aead_setkey, 2103 .encrypt = safexcel_aead_encrypt, 2104 .decrypt = safexcel_aead_decrypt, 2105 .ivsize = DES_BLOCK_SIZE, 2106 .maxauthsize = SHA1_DIGEST_SIZE, 2107 .base = { 2108 .cra_name = "authenc(hmac(sha1),cbc(des))", 2109 .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-des", 2110 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2111 .cra_flags = CRYPTO_ALG_ASYNC | 2112 CRYPTO_ALG_ALLOCATES_MEMORY | 2113 CRYPTO_ALG_KERN_DRIVER_ONLY, 2114 .cra_blocksize = DES_BLOCK_SIZE, 2115 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2116 .cra_alignmask = 0, 2117 .cra_init = safexcel_aead_sha1_des_cra_init, 2118 .cra_exit = safexcel_aead_cra_exit, 2119 .cra_module = THIS_MODULE, 2120 }, 2121 }, 2122 }; 2123 2124 static int safexcel_aead_sha256_des_cra_init(struct crypto_tfm *tfm) 2125 { 2126 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2127 2128 safexcel_aead_sha256_cra_init(tfm); 2129 ctx->alg = SAFEXCEL_DES; /* override default */ 2130 ctx->blocksz = DES_BLOCK_SIZE; 2131 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2132 return 0; 2133 } 2134 2135 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des = { 2136 .type = SAFEXCEL_ALG_TYPE_AEAD, 2137 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, 2138 .alg.aead = { 2139 .setkey = safexcel_aead_setkey, 2140 .encrypt = safexcel_aead_encrypt, 2141 .decrypt = safexcel_aead_decrypt, 2142 .ivsize = DES_BLOCK_SIZE, 2143 .maxauthsize = SHA256_DIGEST_SIZE, 2144 .base = { 2145 .cra_name = "authenc(hmac(sha256),cbc(des))", 2146 .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-des", 2147 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2148 .cra_flags = CRYPTO_ALG_ASYNC | 2149 CRYPTO_ALG_ALLOCATES_MEMORY | 2150 CRYPTO_ALG_KERN_DRIVER_ONLY, 2151 .cra_blocksize = DES_BLOCK_SIZE, 2152 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2153 .cra_alignmask = 0, 2154 .cra_init = safexcel_aead_sha256_des_cra_init, 2155 .cra_exit = safexcel_aead_cra_exit, 2156 .cra_module = THIS_MODULE, 2157 }, 2158 }, 2159 }; 2160 2161 static int safexcel_aead_sha224_des_cra_init(struct crypto_tfm *tfm) 2162 { 2163 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2164 2165 safexcel_aead_sha224_cra_init(tfm); 2166 ctx->alg = SAFEXCEL_DES; /* override default */ 2167 ctx->blocksz = DES_BLOCK_SIZE; 2168 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2169 return 0; 2170 } 2171 2172 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des = { 2173 .type = SAFEXCEL_ALG_TYPE_AEAD, 2174 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, 2175 .alg.aead = { 2176 .setkey = safexcel_aead_setkey, 2177 .encrypt = safexcel_aead_encrypt, 2178 .decrypt = safexcel_aead_decrypt, 2179 .ivsize = DES_BLOCK_SIZE, 2180 .maxauthsize = SHA224_DIGEST_SIZE, 2181 .base = { 2182 .cra_name = "authenc(hmac(sha224),cbc(des))", 2183 .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-des", 2184 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2185 .cra_flags = CRYPTO_ALG_ASYNC | 2186 CRYPTO_ALG_ALLOCATES_MEMORY | 2187 CRYPTO_ALG_KERN_DRIVER_ONLY, 2188 .cra_blocksize = DES_BLOCK_SIZE, 2189 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2190 .cra_alignmask = 0, 2191 .cra_init = safexcel_aead_sha224_des_cra_init, 2192 .cra_exit = safexcel_aead_cra_exit, 2193 .cra_module = THIS_MODULE, 2194 }, 2195 }, 2196 }; 2197 2198 static int safexcel_aead_sha512_des_cra_init(struct crypto_tfm *tfm) 2199 { 2200 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2201 2202 safexcel_aead_sha512_cra_init(tfm); 2203 ctx->alg = SAFEXCEL_DES; /* override default */ 2204 ctx->blocksz = DES_BLOCK_SIZE; 2205 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2206 return 0; 2207 } 2208 2209 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des = { 2210 .type = SAFEXCEL_ALG_TYPE_AEAD, 2211 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, 2212 .alg.aead = { 2213 .setkey = safexcel_aead_setkey, 2214 .encrypt = safexcel_aead_encrypt, 2215 .decrypt = safexcel_aead_decrypt, 2216 .ivsize = DES_BLOCK_SIZE, 2217 .maxauthsize = SHA512_DIGEST_SIZE, 2218 .base = { 2219 .cra_name = "authenc(hmac(sha512),cbc(des))", 2220 .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-des", 2221 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2222 .cra_flags = CRYPTO_ALG_ASYNC | 2223 CRYPTO_ALG_ALLOCATES_MEMORY | 2224 CRYPTO_ALG_KERN_DRIVER_ONLY, 2225 .cra_blocksize = DES_BLOCK_SIZE, 2226 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2227 .cra_alignmask = 0, 2228 .cra_init = safexcel_aead_sha512_des_cra_init, 2229 .cra_exit = safexcel_aead_cra_exit, 2230 .cra_module = THIS_MODULE, 2231 }, 2232 }, 2233 }; 2234 2235 static int safexcel_aead_sha384_des_cra_init(struct crypto_tfm *tfm) 2236 { 2237 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2238 2239 safexcel_aead_sha384_cra_init(tfm); 2240 ctx->alg = SAFEXCEL_DES; /* override default */ 2241 ctx->blocksz = DES_BLOCK_SIZE; 2242 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2243 return 0; 2244 } 2245 2246 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des = { 2247 .type = SAFEXCEL_ALG_TYPE_AEAD, 2248 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, 2249 .alg.aead = { 2250 .setkey = safexcel_aead_setkey, 2251 .encrypt = safexcel_aead_encrypt, 2252 .decrypt = safexcel_aead_decrypt, 2253 .ivsize = DES_BLOCK_SIZE, 2254 .maxauthsize = SHA384_DIGEST_SIZE, 2255 .base = { 2256 .cra_name = "authenc(hmac(sha384),cbc(des))", 2257 .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-des", 2258 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2259 .cra_flags = CRYPTO_ALG_ASYNC | 2260 CRYPTO_ALG_ALLOCATES_MEMORY | 2261 CRYPTO_ALG_KERN_DRIVER_ONLY, 2262 .cra_blocksize = DES_BLOCK_SIZE, 2263 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2264 .cra_alignmask = 0, 2265 .cra_init = safexcel_aead_sha384_des_cra_init, 2266 .cra_exit = safexcel_aead_cra_exit, 2267 .cra_module = THIS_MODULE, 2268 }, 2269 }, 2270 }; 2271 2272 static int safexcel_aead_sha1_ctr_cra_init(struct crypto_tfm *tfm) 2273 { 2274 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2275 2276 safexcel_aead_sha1_cra_init(tfm); 2277 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2278 return 0; 2279 } 2280 2281 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_ctr_aes = { 2282 .type = SAFEXCEL_ALG_TYPE_AEAD, 2283 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA1, 2284 .alg.aead = { 2285 .setkey = safexcel_aead_setkey, 2286 .encrypt = safexcel_aead_encrypt, 2287 .decrypt = safexcel_aead_decrypt, 2288 .ivsize = CTR_RFC3686_IV_SIZE, 2289 .maxauthsize = SHA1_DIGEST_SIZE, 2290 .base = { 2291 .cra_name = "authenc(hmac(sha1),rfc3686(ctr(aes)))", 2292 .cra_driver_name = "safexcel-authenc-hmac-sha1-ctr-aes", 2293 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2294 .cra_flags = CRYPTO_ALG_ASYNC | 2295 CRYPTO_ALG_ALLOCATES_MEMORY | 2296 CRYPTO_ALG_KERN_DRIVER_ONLY, 2297 .cra_blocksize = 1, 2298 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2299 .cra_alignmask = 0, 2300 .cra_init = safexcel_aead_sha1_ctr_cra_init, 2301 .cra_exit = safexcel_aead_cra_exit, 2302 .cra_module = THIS_MODULE, 2303 }, 2304 }, 2305 }; 2306 2307 static int safexcel_aead_sha256_ctr_cra_init(struct crypto_tfm *tfm) 2308 { 2309 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2310 2311 safexcel_aead_sha256_cra_init(tfm); 2312 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2313 return 0; 2314 } 2315 2316 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_ctr_aes = { 2317 .type = SAFEXCEL_ALG_TYPE_AEAD, 2318 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256, 2319 .alg.aead = { 2320 .setkey = safexcel_aead_setkey, 2321 .encrypt = safexcel_aead_encrypt, 2322 .decrypt = safexcel_aead_decrypt, 2323 .ivsize = CTR_RFC3686_IV_SIZE, 2324 .maxauthsize = SHA256_DIGEST_SIZE, 2325 .base = { 2326 .cra_name = "authenc(hmac(sha256),rfc3686(ctr(aes)))", 2327 .cra_driver_name = "safexcel-authenc-hmac-sha256-ctr-aes", 2328 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2329 .cra_flags = CRYPTO_ALG_ASYNC | 2330 CRYPTO_ALG_ALLOCATES_MEMORY | 2331 CRYPTO_ALG_KERN_DRIVER_ONLY, 2332 .cra_blocksize = 1, 2333 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2334 .cra_alignmask = 0, 2335 .cra_init = safexcel_aead_sha256_ctr_cra_init, 2336 .cra_exit = safexcel_aead_cra_exit, 2337 .cra_module = THIS_MODULE, 2338 }, 2339 }, 2340 }; 2341 2342 static int safexcel_aead_sha224_ctr_cra_init(struct crypto_tfm *tfm) 2343 { 2344 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2345 2346 safexcel_aead_sha224_cra_init(tfm); 2347 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2348 return 0; 2349 } 2350 2351 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_ctr_aes = { 2352 .type = SAFEXCEL_ALG_TYPE_AEAD, 2353 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256, 2354 .alg.aead = { 2355 .setkey = safexcel_aead_setkey, 2356 .encrypt = safexcel_aead_encrypt, 2357 .decrypt = safexcel_aead_decrypt, 2358 .ivsize = CTR_RFC3686_IV_SIZE, 2359 .maxauthsize = SHA224_DIGEST_SIZE, 2360 .base = { 2361 .cra_name = "authenc(hmac(sha224),rfc3686(ctr(aes)))", 2362 .cra_driver_name = "safexcel-authenc-hmac-sha224-ctr-aes", 2363 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2364 .cra_flags = CRYPTO_ALG_ASYNC | 2365 CRYPTO_ALG_ALLOCATES_MEMORY | 2366 CRYPTO_ALG_KERN_DRIVER_ONLY, 2367 .cra_blocksize = 1, 2368 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2369 .cra_alignmask = 0, 2370 .cra_init = safexcel_aead_sha224_ctr_cra_init, 2371 .cra_exit = safexcel_aead_cra_exit, 2372 .cra_module = THIS_MODULE, 2373 }, 2374 }, 2375 }; 2376 2377 static int safexcel_aead_sha512_ctr_cra_init(struct crypto_tfm *tfm) 2378 { 2379 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2380 2381 safexcel_aead_sha512_cra_init(tfm); 2382 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2383 return 0; 2384 } 2385 2386 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_ctr_aes = { 2387 .type = SAFEXCEL_ALG_TYPE_AEAD, 2388 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512, 2389 .alg.aead = { 2390 .setkey = safexcel_aead_setkey, 2391 .encrypt = safexcel_aead_encrypt, 2392 .decrypt = safexcel_aead_decrypt, 2393 .ivsize = CTR_RFC3686_IV_SIZE, 2394 .maxauthsize = SHA512_DIGEST_SIZE, 2395 .base = { 2396 .cra_name = "authenc(hmac(sha512),rfc3686(ctr(aes)))", 2397 .cra_driver_name = "safexcel-authenc-hmac-sha512-ctr-aes", 2398 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2399 .cra_flags = CRYPTO_ALG_ASYNC | 2400 CRYPTO_ALG_ALLOCATES_MEMORY | 2401 CRYPTO_ALG_KERN_DRIVER_ONLY, 2402 .cra_blocksize = 1, 2403 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2404 .cra_alignmask = 0, 2405 .cra_init = safexcel_aead_sha512_ctr_cra_init, 2406 .cra_exit = safexcel_aead_cra_exit, 2407 .cra_module = THIS_MODULE, 2408 }, 2409 }, 2410 }; 2411 2412 static int safexcel_aead_sha384_ctr_cra_init(struct crypto_tfm *tfm) 2413 { 2414 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2415 2416 safexcel_aead_sha384_cra_init(tfm); 2417 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2418 return 0; 2419 } 2420 2421 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_ctr_aes = { 2422 .type = SAFEXCEL_ALG_TYPE_AEAD, 2423 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512, 2424 .alg.aead = { 2425 .setkey = safexcel_aead_setkey, 2426 .encrypt = safexcel_aead_encrypt, 2427 .decrypt = safexcel_aead_decrypt, 2428 .ivsize = CTR_RFC3686_IV_SIZE, 2429 .maxauthsize = SHA384_DIGEST_SIZE, 2430 .base = { 2431 .cra_name = "authenc(hmac(sha384),rfc3686(ctr(aes)))", 2432 .cra_driver_name = "safexcel-authenc-hmac-sha384-ctr-aes", 2433 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2434 .cra_flags = CRYPTO_ALG_ASYNC | 2435 CRYPTO_ALG_ALLOCATES_MEMORY | 2436 CRYPTO_ALG_KERN_DRIVER_ONLY, 2437 .cra_blocksize = 1, 2438 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2439 .cra_alignmask = 0, 2440 .cra_init = safexcel_aead_sha384_ctr_cra_init, 2441 .cra_exit = safexcel_aead_cra_exit, 2442 .cra_module = THIS_MODULE, 2443 }, 2444 }, 2445 }; 2446 2447 static int safexcel_skcipher_aesxts_setkey(struct crypto_skcipher *ctfm, 2448 const u8 *key, unsigned int len) 2449 { 2450 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 2451 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2452 struct safexcel_crypto_priv *priv = ctx->base.priv; 2453 struct crypto_aes_ctx aes; 2454 int ret, i; 2455 unsigned int keylen; 2456 2457 /* Check for illegal XTS keys */ 2458 ret = xts_verify_key(ctfm, key, len); 2459 if (ret) 2460 return ret; 2461 2462 /* Only half of the key data is cipher key */ 2463 keylen = (len >> 1); 2464 ret = aes_expandkey(&aes, key, keylen); 2465 if (ret) 2466 return ret; 2467 2468 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2469 for (i = 0; i < keylen / sizeof(u32); i++) { 2470 if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { 2471 ctx->base.needs_inv = true; 2472 break; 2473 } 2474 } 2475 } 2476 2477 for (i = 0; i < keylen / sizeof(u32); i++) 2478 ctx->key[i] = cpu_to_le32(aes.key_enc[i]); 2479 2480 /* The other half is the tweak key */ 2481 ret = aes_expandkey(&aes, (u8 *)(key + keylen), keylen); 2482 if (ret) 2483 return ret; 2484 2485 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2486 for (i = 0; i < keylen / sizeof(u32); i++) { 2487 if (le32_to_cpu(ctx->key[i + keylen / sizeof(u32)]) != 2488 aes.key_enc[i]) { 2489 ctx->base.needs_inv = true; 2490 break; 2491 } 2492 } 2493 } 2494 2495 for (i = 0; i < keylen / sizeof(u32); i++) 2496 ctx->key[i + keylen / sizeof(u32)] = 2497 cpu_to_le32(aes.key_enc[i]); 2498 2499 ctx->key_len = keylen << 1; 2500 2501 memzero_explicit(&aes, sizeof(aes)); 2502 return 0; 2503 } 2504 2505 static int safexcel_skcipher_aes_xts_cra_init(struct crypto_tfm *tfm) 2506 { 2507 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2508 2509 safexcel_skcipher_cra_init(tfm); 2510 ctx->alg = SAFEXCEL_AES; 2511 ctx->blocksz = AES_BLOCK_SIZE; 2512 ctx->xts = 1; 2513 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XTS; 2514 return 0; 2515 } 2516 2517 static int safexcel_encrypt_xts(struct skcipher_request *req) 2518 { 2519 if (req->cryptlen < XTS_BLOCK_SIZE) 2520 return -EINVAL; 2521 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 2522 SAFEXCEL_ENCRYPT); 2523 } 2524 2525 static int safexcel_decrypt_xts(struct skcipher_request *req) 2526 { 2527 if (req->cryptlen < XTS_BLOCK_SIZE) 2528 return -EINVAL; 2529 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 2530 SAFEXCEL_DECRYPT); 2531 } 2532 2533 struct safexcel_alg_template safexcel_alg_xts_aes = { 2534 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 2535 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_AES_XTS, 2536 .alg.skcipher = { 2537 .setkey = safexcel_skcipher_aesxts_setkey, 2538 .encrypt = safexcel_encrypt_xts, 2539 .decrypt = safexcel_decrypt_xts, 2540 /* XTS actually uses 2 AES keys glued together */ 2541 .min_keysize = AES_MIN_KEY_SIZE * 2, 2542 .max_keysize = AES_MAX_KEY_SIZE * 2, 2543 .ivsize = XTS_BLOCK_SIZE, 2544 .base = { 2545 .cra_name = "xts(aes)", 2546 .cra_driver_name = "safexcel-xts-aes", 2547 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2548 .cra_flags = CRYPTO_ALG_ASYNC | 2549 CRYPTO_ALG_ALLOCATES_MEMORY | 2550 CRYPTO_ALG_KERN_DRIVER_ONLY, 2551 .cra_blocksize = XTS_BLOCK_SIZE, 2552 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2553 .cra_alignmask = 0, 2554 .cra_init = safexcel_skcipher_aes_xts_cra_init, 2555 .cra_exit = safexcel_skcipher_cra_exit, 2556 .cra_module = THIS_MODULE, 2557 }, 2558 }, 2559 }; 2560 2561 static int safexcel_aead_gcm_setkey(struct crypto_aead *ctfm, const u8 *key, 2562 unsigned int len) 2563 { 2564 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 2565 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2566 struct safexcel_crypto_priv *priv = ctx->base.priv; 2567 struct crypto_aes_ctx aes; 2568 u32 hashkey[AES_BLOCK_SIZE >> 2]; 2569 int ret, i; 2570 2571 ret = aes_expandkey(&aes, key, len); 2572 if (ret) { 2573 memzero_explicit(&aes, sizeof(aes)); 2574 return ret; 2575 } 2576 2577 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2578 for (i = 0; i < len / sizeof(u32); i++) { 2579 if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { 2580 ctx->base.needs_inv = true; 2581 break; 2582 } 2583 } 2584 } 2585 2586 for (i = 0; i < len / sizeof(u32); i++) 2587 ctx->key[i] = cpu_to_le32(aes.key_enc[i]); 2588 2589 ctx->key_len = len; 2590 2591 /* Compute hash key by encrypting zeroes with cipher key */ 2592 crypto_cipher_clear_flags(ctx->hkaes, CRYPTO_TFM_REQ_MASK); 2593 crypto_cipher_set_flags(ctx->hkaes, crypto_aead_get_flags(ctfm) & 2594 CRYPTO_TFM_REQ_MASK); 2595 ret = crypto_cipher_setkey(ctx->hkaes, key, len); 2596 if (ret) 2597 return ret; 2598 2599 memset(hashkey, 0, AES_BLOCK_SIZE); 2600 crypto_cipher_encrypt_one(ctx->hkaes, (u8 *)hashkey, (u8 *)hashkey); 2601 2602 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2603 for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) { 2604 if (be32_to_cpu(ctx->base.ipad.be[i]) != hashkey[i]) { 2605 ctx->base.needs_inv = true; 2606 break; 2607 } 2608 } 2609 } 2610 2611 for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) 2612 ctx->base.ipad.be[i] = cpu_to_be32(hashkey[i]); 2613 2614 memzero_explicit(hashkey, AES_BLOCK_SIZE); 2615 memzero_explicit(&aes, sizeof(aes)); 2616 return 0; 2617 } 2618 2619 static int safexcel_aead_gcm_cra_init(struct crypto_tfm *tfm) 2620 { 2621 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2622 2623 safexcel_aead_cra_init(tfm); 2624 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_GHASH; 2625 ctx->state_sz = GHASH_BLOCK_SIZE; 2626 ctx->xcm = EIP197_XCM_MODE_GCM; 2627 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XCM; /* override default */ 2628 2629 ctx->hkaes = crypto_alloc_cipher("aes", 0, 0); 2630 return PTR_ERR_OR_ZERO(ctx->hkaes); 2631 } 2632 2633 static void safexcel_aead_gcm_cra_exit(struct crypto_tfm *tfm) 2634 { 2635 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2636 2637 crypto_free_cipher(ctx->hkaes); 2638 safexcel_aead_cra_exit(tfm); 2639 } 2640 2641 static int safexcel_aead_gcm_setauthsize(struct crypto_aead *tfm, 2642 unsigned int authsize) 2643 { 2644 return crypto_gcm_check_authsize(authsize); 2645 } 2646 2647 struct safexcel_alg_template safexcel_alg_gcm = { 2648 .type = SAFEXCEL_ALG_TYPE_AEAD, 2649 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH, 2650 .alg.aead = { 2651 .setkey = safexcel_aead_gcm_setkey, 2652 .setauthsize = safexcel_aead_gcm_setauthsize, 2653 .encrypt = safexcel_aead_encrypt, 2654 .decrypt = safexcel_aead_decrypt, 2655 .ivsize = GCM_AES_IV_SIZE, 2656 .maxauthsize = GHASH_DIGEST_SIZE, 2657 .base = { 2658 .cra_name = "gcm(aes)", 2659 .cra_driver_name = "safexcel-gcm-aes", 2660 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2661 .cra_flags = CRYPTO_ALG_ASYNC | 2662 CRYPTO_ALG_ALLOCATES_MEMORY | 2663 CRYPTO_ALG_KERN_DRIVER_ONLY, 2664 .cra_blocksize = 1, 2665 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2666 .cra_alignmask = 0, 2667 .cra_init = safexcel_aead_gcm_cra_init, 2668 .cra_exit = safexcel_aead_gcm_cra_exit, 2669 .cra_module = THIS_MODULE, 2670 }, 2671 }, 2672 }; 2673 2674 static int safexcel_aead_ccm_setkey(struct crypto_aead *ctfm, const u8 *key, 2675 unsigned int len) 2676 { 2677 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 2678 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2679 struct safexcel_crypto_priv *priv = ctx->base.priv; 2680 struct crypto_aes_ctx aes; 2681 int ret, i; 2682 2683 ret = aes_expandkey(&aes, key, len); 2684 if (ret) { 2685 memzero_explicit(&aes, sizeof(aes)); 2686 return ret; 2687 } 2688 2689 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2690 for (i = 0; i < len / sizeof(u32); i++) { 2691 if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { 2692 ctx->base.needs_inv = true; 2693 break; 2694 } 2695 } 2696 } 2697 2698 for (i = 0; i < len / sizeof(u32); i++) { 2699 ctx->key[i] = cpu_to_le32(aes.key_enc[i]); 2700 ctx->base.ipad.be[i + 2 * AES_BLOCK_SIZE / sizeof(u32)] = 2701 cpu_to_be32(aes.key_enc[i]); 2702 } 2703 2704 ctx->key_len = len; 2705 ctx->state_sz = 2 * AES_BLOCK_SIZE + len; 2706 2707 if (len == AES_KEYSIZE_192) 2708 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192; 2709 else if (len == AES_KEYSIZE_256) 2710 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256; 2711 else 2712 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128; 2713 2714 memzero_explicit(&aes, sizeof(aes)); 2715 return 0; 2716 } 2717 2718 static int safexcel_aead_ccm_cra_init(struct crypto_tfm *tfm) 2719 { 2720 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2721 2722 safexcel_aead_cra_init(tfm); 2723 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128; 2724 ctx->state_sz = 3 * AES_BLOCK_SIZE; 2725 ctx->xcm = EIP197_XCM_MODE_CCM; 2726 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XCM; /* override default */ 2727 ctx->ctrinit = 0; 2728 return 0; 2729 } 2730 2731 static int safexcel_aead_ccm_setauthsize(struct crypto_aead *tfm, 2732 unsigned int authsize) 2733 { 2734 /* Borrowed from crypto/ccm.c */ 2735 switch (authsize) { 2736 case 4: 2737 case 6: 2738 case 8: 2739 case 10: 2740 case 12: 2741 case 14: 2742 case 16: 2743 break; 2744 default: 2745 return -EINVAL; 2746 } 2747 2748 return 0; 2749 } 2750 2751 static int safexcel_ccm_encrypt(struct aead_request *req) 2752 { 2753 struct safexcel_cipher_req *creq = aead_request_ctx(req); 2754 2755 if (req->iv[0] < 1 || req->iv[0] > 7) 2756 return -EINVAL; 2757 2758 return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); 2759 } 2760 2761 static int safexcel_ccm_decrypt(struct aead_request *req) 2762 { 2763 struct safexcel_cipher_req *creq = aead_request_ctx(req); 2764 2765 if (req->iv[0] < 1 || req->iv[0] > 7) 2766 return -EINVAL; 2767 2768 return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); 2769 } 2770 2771 struct safexcel_alg_template safexcel_alg_ccm = { 2772 .type = SAFEXCEL_ALG_TYPE_AEAD, 2773 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_CBC_MAC_ALL, 2774 .alg.aead = { 2775 .setkey = safexcel_aead_ccm_setkey, 2776 .setauthsize = safexcel_aead_ccm_setauthsize, 2777 .encrypt = safexcel_ccm_encrypt, 2778 .decrypt = safexcel_ccm_decrypt, 2779 .ivsize = AES_BLOCK_SIZE, 2780 .maxauthsize = AES_BLOCK_SIZE, 2781 .base = { 2782 .cra_name = "ccm(aes)", 2783 .cra_driver_name = "safexcel-ccm-aes", 2784 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2785 .cra_flags = CRYPTO_ALG_ASYNC | 2786 CRYPTO_ALG_ALLOCATES_MEMORY | 2787 CRYPTO_ALG_KERN_DRIVER_ONLY, 2788 .cra_blocksize = 1, 2789 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2790 .cra_alignmask = 0, 2791 .cra_init = safexcel_aead_ccm_cra_init, 2792 .cra_exit = safexcel_aead_cra_exit, 2793 .cra_module = THIS_MODULE, 2794 }, 2795 }, 2796 }; 2797 2798 static void safexcel_chacha20_setkey(struct safexcel_cipher_ctx *ctx, 2799 const u8 *key) 2800 { 2801 struct safexcel_crypto_priv *priv = ctx->base.priv; 2802 2803 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) 2804 if (memcmp(ctx->key, key, CHACHA_KEY_SIZE)) 2805 ctx->base.needs_inv = true; 2806 2807 memcpy(ctx->key, key, CHACHA_KEY_SIZE); 2808 ctx->key_len = CHACHA_KEY_SIZE; 2809 } 2810 2811 static int safexcel_skcipher_chacha20_setkey(struct crypto_skcipher *ctfm, 2812 const u8 *key, unsigned int len) 2813 { 2814 struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm); 2815 2816 if (len != CHACHA_KEY_SIZE) 2817 return -EINVAL; 2818 2819 safexcel_chacha20_setkey(ctx, key); 2820 2821 return 0; 2822 } 2823 2824 static int safexcel_skcipher_chacha20_cra_init(struct crypto_tfm *tfm) 2825 { 2826 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2827 2828 safexcel_skcipher_cra_init(tfm); 2829 ctx->alg = SAFEXCEL_CHACHA20; 2830 ctx->ctrinit = 0; 2831 ctx->mode = CONTEXT_CONTROL_CHACHA20_MODE_256_32; 2832 return 0; 2833 } 2834 2835 struct safexcel_alg_template safexcel_alg_chacha20 = { 2836 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 2837 .algo_mask = SAFEXCEL_ALG_CHACHA20, 2838 .alg.skcipher = { 2839 .setkey = safexcel_skcipher_chacha20_setkey, 2840 .encrypt = safexcel_encrypt, 2841 .decrypt = safexcel_decrypt, 2842 .min_keysize = CHACHA_KEY_SIZE, 2843 .max_keysize = CHACHA_KEY_SIZE, 2844 .ivsize = CHACHA_IV_SIZE, 2845 .base = { 2846 .cra_name = "chacha20", 2847 .cra_driver_name = "safexcel-chacha20", 2848 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2849 .cra_flags = CRYPTO_ALG_ASYNC | 2850 CRYPTO_ALG_ALLOCATES_MEMORY | 2851 CRYPTO_ALG_KERN_DRIVER_ONLY, 2852 .cra_blocksize = 1, 2853 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2854 .cra_alignmask = 0, 2855 .cra_init = safexcel_skcipher_chacha20_cra_init, 2856 .cra_exit = safexcel_skcipher_cra_exit, 2857 .cra_module = THIS_MODULE, 2858 }, 2859 }, 2860 }; 2861 2862 static int safexcel_aead_chachapoly_setkey(struct crypto_aead *ctfm, 2863 const u8 *key, unsigned int len) 2864 { 2865 struct safexcel_cipher_ctx *ctx = crypto_aead_ctx(ctfm); 2866 2867 if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP && 2868 len > EIP197_AEAD_IPSEC_NONCE_SIZE) { 2869 /* ESP variant has nonce appended to key */ 2870 len -= EIP197_AEAD_IPSEC_NONCE_SIZE; 2871 ctx->nonce = *(u32 *)(key + len); 2872 } 2873 if (len != CHACHA_KEY_SIZE) 2874 return -EINVAL; 2875 2876 safexcel_chacha20_setkey(ctx, key); 2877 2878 return 0; 2879 } 2880 2881 static int safexcel_aead_chachapoly_setauthsize(struct crypto_aead *tfm, 2882 unsigned int authsize) 2883 { 2884 if (authsize != POLY1305_DIGEST_SIZE) 2885 return -EINVAL; 2886 return 0; 2887 } 2888 2889 static int safexcel_aead_chachapoly_crypt(struct aead_request *req, 2890 enum safexcel_cipher_direction dir) 2891 { 2892 struct safexcel_cipher_req *creq = aead_request_ctx(req); 2893 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2894 struct crypto_tfm *tfm = crypto_aead_tfm(aead); 2895 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2896 struct aead_request *subreq = aead_request_ctx(req); 2897 u32 key[CHACHA_KEY_SIZE / sizeof(u32) + 1]; 2898 int ret = 0; 2899 2900 /* 2901 * Instead of wasting time detecting umpteen silly corner cases, 2902 * just dump all "small" requests to the fallback implementation. 2903 * HW would not be faster on such small requests anyway. 2904 */ 2905 if (likely((ctx->aead != EIP197_AEAD_TYPE_IPSEC_ESP || 2906 req->assoclen >= EIP197_AEAD_IPSEC_IV_SIZE) && 2907 req->cryptlen > POLY1305_DIGEST_SIZE)) { 2908 return safexcel_queue_req(&req->base, creq, dir); 2909 } 2910 2911 /* HW cannot do full (AAD+payload) zero length, use fallback */ 2912 memcpy(key, ctx->key, CHACHA_KEY_SIZE); 2913 if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { 2914 /* ESP variant has nonce appended to the key */ 2915 key[CHACHA_KEY_SIZE / sizeof(u32)] = ctx->nonce; 2916 ret = crypto_aead_setkey(ctx->fback, (u8 *)key, 2917 CHACHA_KEY_SIZE + 2918 EIP197_AEAD_IPSEC_NONCE_SIZE); 2919 } else { 2920 ret = crypto_aead_setkey(ctx->fback, (u8 *)key, 2921 CHACHA_KEY_SIZE); 2922 } 2923 if (ret) { 2924 crypto_aead_clear_flags(aead, CRYPTO_TFM_REQ_MASK); 2925 crypto_aead_set_flags(aead, crypto_aead_get_flags(ctx->fback) & 2926 CRYPTO_TFM_REQ_MASK); 2927 return ret; 2928 } 2929 2930 aead_request_set_tfm(subreq, ctx->fback); 2931 aead_request_set_callback(subreq, req->base.flags, req->base.complete, 2932 req->base.data); 2933 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, 2934 req->iv); 2935 aead_request_set_ad(subreq, req->assoclen); 2936 2937 return (dir == SAFEXCEL_ENCRYPT) ? 2938 crypto_aead_encrypt(subreq) : 2939 crypto_aead_decrypt(subreq); 2940 } 2941 2942 static int safexcel_aead_chachapoly_encrypt(struct aead_request *req) 2943 { 2944 return safexcel_aead_chachapoly_crypt(req, SAFEXCEL_ENCRYPT); 2945 } 2946 2947 static int safexcel_aead_chachapoly_decrypt(struct aead_request *req) 2948 { 2949 return safexcel_aead_chachapoly_crypt(req, SAFEXCEL_DECRYPT); 2950 } 2951 2952 static int safexcel_aead_fallback_cra_init(struct crypto_tfm *tfm) 2953 { 2954 struct crypto_aead *aead = __crypto_aead_cast(tfm); 2955 struct aead_alg *alg = crypto_aead_alg(aead); 2956 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2957 2958 safexcel_aead_cra_init(tfm); 2959 2960 /* Allocate fallback implementation */ 2961 ctx->fback = crypto_alloc_aead(alg->base.cra_name, 0, 2962 CRYPTO_ALG_ASYNC | 2963 CRYPTO_ALG_NEED_FALLBACK); 2964 if (IS_ERR(ctx->fback)) 2965 return PTR_ERR(ctx->fback); 2966 2967 crypto_aead_set_reqsize(aead, max(sizeof(struct safexcel_cipher_req), 2968 sizeof(struct aead_request) + 2969 crypto_aead_reqsize(ctx->fback))); 2970 2971 return 0; 2972 } 2973 2974 static int safexcel_aead_chachapoly_cra_init(struct crypto_tfm *tfm) 2975 { 2976 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2977 2978 safexcel_aead_fallback_cra_init(tfm); 2979 ctx->alg = SAFEXCEL_CHACHA20; 2980 ctx->mode = CONTEXT_CONTROL_CHACHA20_MODE_256_32 | 2981 CONTEXT_CONTROL_CHACHA20_MODE_CALC_OTK; 2982 ctx->ctrinit = 0; 2983 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_POLY1305; 2984 ctx->state_sz = 0; /* Precomputed by HW */ 2985 return 0; 2986 } 2987 2988 static void safexcel_aead_fallback_cra_exit(struct crypto_tfm *tfm) 2989 { 2990 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2991 2992 crypto_free_aead(ctx->fback); 2993 safexcel_aead_cra_exit(tfm); 2994 } 2995 2996 struct safexcel_alg_template safexcel_alg_chachapoly = { 2997 .type = SAFEXCEL_ALG_TYPE_AEAD, 2998 .algo_mask = SAFEXCEL_ALG_CHACHA20 | SAFEXCEL_ALG_POLY1305, 2999 .alg.aead = { 3000 .setkey = safexcel_aead_chachapoly_setkey, 3001 .setauthsize = safexcel_aead_chachapoly_setauthsize, 3002 .encrypt = safexcel_aead_chachapoly_encrypt, 3003 .decrypt = safexcel_aead_chachapoly_decrypt, 3004 .ivsize = CHACHAPOLY_IV_SIZE, 3005 .maxauthsize = POLY1305_DIGEST_SIZE, 3006 .base = { 3007 .cra_name = "rfc7539(chacha20,poly1305)", 3008 .cra_driver_name = "safexcel-chacha20-poly1305", 3009 /* +1 to put it above HW chacha + SW poly */ 3010 .cra_priority = SAFEXCEL_CRA_PRIORITY + 1, 3011 .cra_flags = CRYPTO_ALG_ASYNC | 3012 CRYPTO_ALG_ALLOCATES_MEMORY | 3013 CRYPTO_ALG_KERN_DRIVER_ONLY | 3014 CRYPTO_ALG_NEED_FALLBACK, 3015 .cra_blocksize = 1, 3016 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3017 .cra_alignmask = 0, 3018 .cra_init = safexcel_aead_chachapoly_cra_init, 3019 .cra_exit = safexcel_aead_fallback_cra_exit, 3020 .cra_module = THIS_MODULE, 3021 }, 3022 }, 3023 }; 3024 3025 static int safexcel_aead_chachapolyesp_cra_init(struct crypto_tfm *tfm) 3026 { 3027 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3028 int ret; 3029 3030 ret = safexcel_aead_chachapoly_cra_init(tfm); 3031 ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP; 3032 ctx->aadskip = EIP197_AEAD_IPSEC_IV_SIZE; 3033 return ret; 3034 } 3035 3036 struct safexcel_alg_template safexcel_alg_chachapoly_esp = { 3037 .type = SAFEXCEL_ALG_TYPE_AEAD, 3038 .algo_mask = SAFEXCEL_ALG_CHACHA20 | SAFEXCEL_ALG_POLY1305, 3039 .alg.aead = { 3040 .setkey = safexcel_aead_chachapoly_setkey, 3041 .setauthsize = safexcel_aead_chachapoly_setauthsize, 3042 .encrypt = safexcel_aead_chachapoly_encrypt, 3043 .decrypt = safexcel_aead_chachapoly_decrypt, 3044 .ivsize = CHACHAPOLY_IV_SIZE - EIP197_AEAD_IPSEC_NONCE_SIZE, 3045 .maxauthsize = POLY1305_DIGEST_SIZE, 3046 .base = { 3047 .cra_name = "rfc7539esp(chacha20,poly1305)", 3048 .cra_driver_name = "safexcel-chacha20-poly1305-esp", 3049 /* +1 to put it above HW chacha + SW poly */ 3050 .cra_priority = SAFEXCEL_CRA_PRIORITY + 1, 3051 .cra_flags = CRYPTO_ALG_ASYNC | 3052 CRYPTO_ALG_ALLOCATES_MEMORY | 3053 CRYPTO_ALG_KERN_DRIVER_ONLY | 3054 CRYPTO_ALG_NEED_FALLBACK, 3055 .cra_blocksize = 1, 3056 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3057 .cra_alignmask = 0, 3058 .cra_init = safexcel_aead_chachapolyesp_cra_init, 3059 .cra_exit = safexcel_aead_fallback_cra_exit, 3060 .cra_module = THIS_MODULE, 3061 }, 3062 }, 3063 }; 3064 3065 static int safexcel_skcipher_sm4_setkey(struct crypto_skcipher *ctfm, 3066 const u8 *key, unsigned int len) 3067 { 3068 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 3069 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3070 struct safexcel_crypto_priv *priv = ctx->base.priv; 3071 3072 if (len != SM4_KEY_SIZE) 3073 return -EINVAL; 3074 3075 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) 3076 if (memcmp(ctx->key, key, SM4_KEY_SIZE)) 3077 ctx->base.needs_inv = true; 3078 3079 memcpy(ctx->key, key, SM4_KEY_SIZE); 3080 ctx->key_len = SM4_KEY_SIZE; 3081 3082 return 0; 3083 } 3084 3085 static int safexcel_sm4_blk_encrypt(struct skcipher_request *req) 3086 { 3087 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3088 if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) 3089 return -EINVAL; 3090 else 3091 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 3092 SAFEXCEL_ENCRYPT); 3093 } 3094 3095 static int safexcel_sm4_blk_decrypt(struct skcipher_request *req) 3096 { 3097 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3098 if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) 3099 return -EINVAL; 3100 else 3101 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 3102 SAFEXCEL_DECRYPT); 3103 } 3104 3105 static int safexcel_skcipher_sm4_ecb_cra_init(struct crypto_tfm *tfm) 3106 { 3107 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3108 3109 safexcel_skcipher_cra_init(tfm); 3110 ctx->alg = SAFEXCEL_SM4; 3111 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; 3112 ctx->blocksz = 0; 3113 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 3114 return 0; 3115 } 3116 3117 struct safexcel_alg_template safexcel_alg_ecb_sm4 = { 3118 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 3119 .algo_mask = SAFEXCEL_ALG_SM4, 3120 .alg.skcipher = { 3121 .setkey = safexcel_skcipher_sm4_setkey, 3122 .encrypt = safexcel_sm4_blk_encrypt, 3123 .decrypt = safexcel_sm4_blk_decrypt, 3124 .min_keysize = SM4_KEY_SIZE, 3125 .max_keysize = SM4_KEY_SIZE, 3126 .base = { 3127 .cra_name = "ecb(sm4)", 3128 .cra_driver_name = "safexcel-ecb-sm4", 3129 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3130 .cra_flags = CRYPTO_ALG_ASYNC | 3131 CRYPTO_ALG_ALLOCATES_MEMORY | 3132 CRYPTO_ALG_KERN_DRIVER_ONLY, 3133 .cra_blocksize = SM4_BLOCK_SIZE, 3134 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3135 .cra_alignmask = 0, 3136 .cra_init = safexcel_skcipher_sm4_ecb_cra_init, 3137 .cra_exit = safexcel_skcipher_cra_exit, 3138 .cra_module = THIS_MODULE, 3139 }, 3140 }, 3141 }; 3142 3143 static int safexcel_skcipher_sm4_cbc_cra_init(struct crypto_tfm *tfm) 3144 { 3145 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3146 3147 safexcel_skcipher_cra_init(tfm); 3148 ctx->alg = SAFEXCEL_SM4; 3149 ctx->blocksz = SM4_BLOCK_SIZE; 3150 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; 3151 return 0; 3152 } 3153 3154 struct safexcel_alg_template safexcel_alg_cbc_sm4 = { 3155 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 3156 .algo_mask = SAFEXCEL_ALG_SM4, 3157 .alg.skcipher = { 3158 .setkey = safexcel_skcipher_sm4_setkey, 3159 .encrypt = safexcel_sm4_blk_encrypt, 3160 .decrypt = safexcel_sm4_blk_decrypt, 3161 .min_keysize = SM4_KEY_SIZE, 3162 .max_keysize = SM4_KEY_SIZE, 3163 .ivsize = SM4_BLOCK_SIZE, 3164 .base = { 3165 .cra_name = "cbc(sm4)", 3166 .cra_driver_name = "safexcel-cbc-sm4", 3167 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3168 .cra_flags = CRYPTO_ALG_ASYNC | 3169 CRYPTO_ALG_ALLOCATES_MEMORY | 3170 CRYPTO_ALG_KERN_DRIVER_ONLY, 3171 .cra_blocksize = SM4_BLOCK_SIZE, 3172 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3173 .cra_alignmask = 0, 3174 .cra_init = safexcel_skcipher_sm4_cbc_cra_init, 3175 .cra_exit = safexcel_skcipher_cra_exit, 3176 .cra_module = THIS_MODULE, 3177 }, 3178 }, 3179 }; 3180 3181 static int safexcel_skcipher_sm4_ofb_cra_init(struct crypto_tfm *tfm) 3182 { 3183 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3184 3185 safexcel_skcipher_cra_init(tfm); 3186 ctx->alg = SAFEXCEL_SM4; 3187 ctx->blocksz = SM4_BLOCK_SIZE; 3188 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_OFB; 3189 return 0; 3190 } 3191 3192 struct safexcel_alg_template safexcel_alg_ofb_sm4 = { 3193 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 3194 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_AES_XFB, 3195 .alg.skcipher = { 3196 .setkey = safexcel_skcipher_sm4_setkey, 3197 .encrypt = safexcel_encrypt, 3198 .decrypt = safexcel_decrypt, 3199 .min_keysize = SM4_KEY_SIZE, 3200 .max_keysize = SM4_KEY_SIZE, 3201 .ivsize = SM4_BLOCK_SIZE, 3202 .base = { 3203 .cra_name = "ofb(sm4)", 3204 .cra_driver_name = "safexcel-ofb-sm4", 3205 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3206 .cra_flags = CRYPTO_ALG_ASYNC | 3207 CRYPTO_ALG_ALLOCATES_MEMORY | 3208 CRYPTO_ALG_KERN_DRIVER_ONLY, 3209 .cra_blocksize = 1, 3210 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3211 .cra_alignmask = 0, 3212 .cra_init = safexcel_skcipher_sm4_ofb_cra_init, 3213 .cra_exit = safexcel_skcipher_cra_exit, 3214 .cra_module = THIS_MODULE, 3215 }, 3216 }, 3217 }; 3218 3219 static int safexcel_skcipher_sm4_cfb_cra_init(struct crypto_tfm *tfm) 3220 { 3221 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3222 3223 safexcel_skcipher_cra_init(tfm); 3224 ctx->alg = SAFEXCEL_SM4; 3225 ctx->blocksz = SM4_BLOCK_SIZE; 3226 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CFB; 3227 return 0; 3228 } 3229 3230 struct safexcel_alg_template safexcel_alg_cfb_sm4 = { 3231 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 3232 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_AES_XFB, 3233 .alg.skcipher = { 3234 .setkey = safexcel_skcipher_sm4_setkey, 3235 .encrypt = safexcel_encrypt, 3236 .decrypt = safexcel_decrypt, 3237 .min_keysize = SM4_KEY_SIZE, 3238 .max_keysize = SM4_KEY_SIZE, 3239 .ivsize = SM4_BLOCK_SIZE, 3240 .base = { 3241 .cra_name = "cfb(sm4)", 3242 .cra_driver_name = "safexcel-cfb-sm4", 3243 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3244 .cra_flags = CRYPTO_ALG_ASYNC | 3245 CRYPTO_ALG_ALLOCATES_MEMORY | 3246 CRYPTO_ALG_KERN_DRIVER_ONLY, 3247 .cra_blocksize = 1, 3248 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3249 .cra_alignmask = 0, 3250 .cra_init = safexcel_skcipher_sm4_cfb_cra_init, 3251 .cra_exit = safexcel_skcipher_cra_exit, 3252 .cra_module = THIS_MODULE, 3253 }, 3254 }, 3255 }; 3256 3257 static int safexcel_skcipher_sm4ctr_setkey(struct crypto_skcipher *ctfm, 3258 const u8 *key, unsigned int len) 3259 { 3260 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 3261 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3262 3263 /* last 4 bytes of key are the nonce! */ 3264 ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE); 3265 /* exclude the nonce here */ 3266 len -= CTR_RFC3686_NONCE_SIZE; 3267 3268 return safexcel_skcipher_sm4_setkey(ctfm, key, len); 3269 } 3270 3271 static int safexcel_skcipher_sm4_ctr_cra_init(struct crypto_tfm *tfm) 3272 { 3273 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3274 3275 safexcel_skcipher_cra_init(tfm); 3276 ctx->alg = SAFEXCEL_SM4; 3277 ctx->blocksz = SM4_BLOCK_SIZE; 3278 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; 3279 return 0; 3280 } 3281 3282 struct safexcel_alg_template safexcel_alg_ctr_sm4 = { 3283 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 3284 .algo_mask = SAFEXCEL_ALG_SM4, 3285 .alg.skcipher = { 3286 .setkey = safexcel_skcipher_sm4ctr_setkey, 3287 .encrypt = safexcel_encrypt, 3288 .decrypt = safexcel_decrypt, 3289 /* Add nonce size */ 3290 .min_keysize = SM4_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 3291 .max_keysize = SM4_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 3292 .ivsize = CTR_RFC3686_IV_SIZE, 3293 .base = { 3294 .cra_name = "rfc3686(ctr(sm4))", 3295 .cra_driver_name = "safexcel-ctr-sm4", 3296 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3297 .cra_flags = CRYPTO_ALG_ASYNC | 3298 CRYPTO_ALG_ALLOCATES_MEMORY | 3299 CRYPTO_ALG_KERN_DRIVER_ONLY, 3300 .cra_blocksize = 1, 3301 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3302 .cra_alignmask = 0, 3303 .cra_init = safexcel_skcipher_sm4_ctr_cra_init, 3304 .cra_exit = safexcel_skcipher_cra_exit, 3305 .cra_module = THIS_MODULE, 3306 }, 3307 }, 3308 }; 3309 3310 static int safexcel_aead_sm4_blk_encrypt(struct aead_request *req) 3311 { 3312 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3313 if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) 3314 return -EINVAL; 3315 3316 return safexcel_queue_req(&req->base, aead_request_ctx(req), 3317 SAFEXCEL_ENCRYPT); 3318 } 3319 3320 static int safexcel_aead_sm4_blk_decrypt(struct aead_request *req) 3321 { 3322 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3323 3324 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3325 if ((req->cryptlen - crypto_aead_authsize(tfm)) & (SM4_BLOCK_SIZE - 1)) 3326 return -EINVAL; 3327 3328 return safexcel_queue_req(&req->base, aead_request_ctx(req), 3329 SAFEXCEL_DECRYPT); 3330 } 3331 3332 static int safexcel_aead_sm4cbc_sha1_cra_init(struct crypto_tfm *tfm) 3333 { 3334 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3335 3336 safexcel_aead_cra_init(tfm); 3337 ctx->alg = SAFEXCEL_SM4; 3338 ctx->blocksz = SM4_BLOCK_SIZE; 3339 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1; 3340 ctx->state_sz = SHA1_DIGEST_SIZE; 3341 return 0; 3342 } 3343 3344 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_sm4 = { 3345 .type = SAFEXCEL_ALG_TYPE_AEAD, 3346 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SHA1, 3347 .alg.aead = { 3348 .setkey = safexcel_aead_setkey, 3349 .encrypt = safexcel_aead_sm4_blk_encrypt, 3350 .decrypt = safexcel_aead_sm4_blk_decrypt, 3351 .ivsize = SM4_BLOCK_SIZE, 3352 .maxauthsize = SHA1_DIGEST_SIZE, 3353 .base = { 3354 .cra_name = "authenc(hmac(sha1),cbc(sm4))", 3355 .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-sm4", 3356 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3357 .cra_flags = CRYPTO_ALG_ASYNC | 3358 CRYPTO_ALG_ALLOCATES_MEMORY | 3359 CRYPTO_ALG_KERN_DRIVER_ONLY, 3360 .cra_blocksize = SM4_BLOCK_SIZE, 3361 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3362 .cra_alignmask = 0, 3363 .cra_init = safexcel_aead_sm4cbc_sha1_cra_init, 3364 .cra_exit = safexcel_aead_cra_exit, 3365 .cra_module = THIS_MODULE, 3366 }, 3367 }, 3368 }; 3369 3370 static int safexcel_aead_fallback_setkey(struct crypto_aead *ctfm, 3371 const u8 *key, unsigned int len) 3372 { 3373 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 3374 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3375 3376 /* Keep fallback cipher synchronized */ 3377 return crypto_aead_setkey(ctx->fback, (u8 *)key, len) ?: 3378 safexcel_aead_setkey(ctfm, key, len); 3379 } 3380 3381 static int safexcel_aead_fallback_setauthsize(struct crypto_aead *ctfm, 3382 unsigned int authsize) 3383 { 3384 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 3385 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3386 3387 /* Keep fallback cipher synchronized */ 3388 return crypto_aead_setauthsize(ctx->fback, authsize); 3389 } 3390 3391 static int safexcel_aead_fallback_crypt(struct aead_request *req, 3392 enum safexcel_cipher_direction dir) 3393 { 3394 struct crypto_aead *aead = crypto_aead_reqtfm(req); 3395 struct crypto_tfm *tfm = crypto_aead_tfm(aead); 3396 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3397 struct aead_request *subreq = aead_request_ctx(req); 3398 3399 aead_request_set_tfm(subreq, ctx->fback); 3400 aead_request_set_callback(subreq, req->base.flags, req->base.complete, 3401 req->base.data); 3402 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, 3403 req->iv); 3404 aead_request_set_ad(subreq, req->assoclen); 3405 3406 return (dir == SAFEXCEL_ENCRYPT) ? 3407 crypto_aead_encrypt(subreq) : 3408 crypto_aead_decrypt(subreq); 3409 } 3410 3411 static int safexcel_aead_sm4cbc_sm3_encrypt(struct aead_request *req) 3412 { 3413 struct safexcel_cipher_req *creq = aead_request_ctx(req); 3414 3415 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3416 if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) 3417 return -EINVAL; 3418 else if (req->cryptlen || req->assoclen) /* If input length > 0 only */ 3419 return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); 3420 3421 /* HW cannot do full (AAD+payload) zero length, use fallback */ 3422 return safexcel_aead_fallback_crypt(req, SAFEXCEL_ENCRYPT); 3423 } 3424 3425 static int safexcel_aead_sm4cbc_sm3_decrypt(struct aead_request *req) 3426 { 3427 struct safexcel_cipher_req *creq = aead_request_ctx(req); 3428 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3429 3430 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3431 if ((req->cryptlen - crypto_aead_authsize(tfm)) & (SM4_BLOCK_SIZE - 1)) 3432 return -EINVAL; 3433 else if (req->cryptlen > crypto_aead_authsize(tfm) || req->assoclen) 3434 /* If input length > 0 only */ 3435 return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); 3436 3437 /* HW cannot do full (AAD+payload) zero length, use fallback */ 3438 return safexcel_aead_fallback_crypt(req, SAFEXCEL_DECRYPT); 3439 } 3440 3441 static int safexcel_aead_sm4cbc_sm3_cra_init(struct crypto_tfm *tfm) 3442 { 3443 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3444 3445 safexcel_aead_fallback_cra_init(tfm); 3446 ctx->alg = SAFEXCEL_SM4; 3447 ctx->blocksz = SM4_BLOCK_SIZE; 3448 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3; 3449 ctx->state_sz = SM3_DIGEST_SIZE; 3450 return 0; 3451 } 3452 3453 struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_cbc_sm4 = { 3454 .type = SAFEXCEL_ALG_TYPE_AEAD, 3455 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SM3, 3456 .alg.aead = { 3457 .setkey = safexcel_aead_fallback_setkey, 3458 .setauthsize = safexcel_aead_fallback_setauthsize, 3459 .encrypt = safexcel_aead_sm4cbc_sm3_encrypt, 3460 .decrypt = safexcel_aead_sm4cbc_sm3_decrypt, 3461 .ivsize = SM4_BLOCK_SIZE, 3462 .maxauthsize = SM3_DIGEST_SIZE, 3463 .base = { 3464 .cra_name = "authenc(hmac(sm3),cbc(sm4))", 3465 .cra_driver_name = "safexcel-authenc-hmac-sm3-cbc-sm4", 3466 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3467 .cra_flags = CRYPTO_ALG_ASYNC | 3468 CRYPTO_ALG_ALLOCATES_MEMORY | 3469 CRYPTO_ALG_KERN_DRIVER_ONLY | 3470 CRYPTO_ALG_NEED_FALLBACK, 3471 .cra_blocksize = SM4_BLOCK_SIZE, 3472 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3473 .cra_alignmask = 0, 3474 .cra_init = safexcel_aead_sm4cbc_sm3_cra_init, 3475 .cra_exit = safexcel_aead_fallback_cra_exit, 3476 .cra_module = THIS_MODULE, 3477 }, 3478 }, 3479 }; 3480 3481 static int safexcel_aead_sm4ctr_sha1_cra_init(struct crypto_tfm *tfm) 3482 { 3483 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3484 3485 safexcel_aead_sm4cbc_sha1_cra_init(tfm); 3486 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; 3487 return 0; 3488 } 3489 3490 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_ctr_sm4 = { 3491 .type = SAFEXCEL_ALG_TYPE_AEAD, 3492 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SHA1, 3493 .alg.aead = { 3494 .setkey = safexcel_aead_setkey, 3495 .encrypt = safexcel_aead_encrypt, 3496 .decrypt = safexcel_aead_decrypt, 3497 .ivsize = CTR_RFC3686_IV_SIZE, 3498 .maxauthsize = SHA1_DIGEST_SIZE, 3499 .base = { 3500 .cra_name = "authenc(hmac(sha1),rfc3686(ctr(sm4)))", 3501 .cra_driver_name = "safexcel-authenc-hmac-sha1-ctr-sm4", 3502 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3503 .cra_flags = CRYPTO_ALG_ASYNC | 3504 CRYPTO_ALG_ALLOCATES_MEMORY | 3505 CRYPTO_ALG_KERN_DRIVER_ONLY, 3506 .cra_blocksize = 1, 3507 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3508 .cra_alignmask = 0, 3509 .cra_init = safexcel_aead_sm4ctr_sha1_cra_init, 3510 .cra_exit = safexcel_aead_cra_exit, 3511 .cra_module = THIS_MODULE, 3512 }, 3513 }, 3514 }; 3515 3516 static int safexcel_aead_sm4ctr_sm3_cra_init(struct crypto_tfm *tfm) 3517 { 3518 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3519 3520 safexcel_aead_sm4cbc_sm3_cra_init(tfm); 3521 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; 3522 return 0; 3523 } 3524 3525 struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_ctr_sm4 = { 3526 .type = SAFEXCEL_ALG_TYPE_AEAD, 3527 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SM3, 3528 .alg.aead = { 3529 .setkey = safexcel_aead_setkey, 3530 .encrypt = safexcel_aead_encrypt, 3531 .decrypt = safexcel_aead_decrypt, 3532 .ivsize = CTR_RFC3686_IV_SIZE, 3533 .maxauthsize = SM3_DIGEST_SIZE, 3534 .base = { 3535 .cra_name = "authenc(hmac(sm3),rfc3686(ctr(sm4)))", 3536 .cra_driver_name = "safexcel-authenc-hmac-sm3-ctr-sm4", 3537 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3538 .cra_flags = CRYPTO_ALG_ASYNC | 3539 CRYPTO_ALG_ALLOCATES_MEMORY | 3540 CRYPTO_ALG_KERN_DRIVER_ONLY, 3541 .cra_blocksize = 1, 3542 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3543 .cra_alignmask = 0, 3544 .cra_init = safexcel_aead_sm4ctr_sm3_cra_init, 3545 .cra_exit = safexcel_aead_cra_exit, 3546 .cra_module = THIS_MODULE, 3547 }, 3548 }, 3549 }; 3550 3551 static int safexcel_rfc4106_gcm_setkey(struct crypto_aead *ctfm, const u8 *key, 3552 unsigned int len) 3553 { 3554 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 3555 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3556 3557 /* last 4 bytes of key are the nonce! */ 3558 ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE); 3559 3560 len -= CTR_RFC3686_NONCE_SIZE; 3561 return safexcel_aead_gcm_setkey(ctfm, key, len); 3562 } 3563 3564 static int safexcel_rfc4106_gcm_setauthsize(struct crypto_aead *tfm, 3565 unsigned int authsize) 3566 { 3567 return crypto_rfc4106_check_authsize(authsize); 3568 } 3569 3570 static int safexcel_rfc4106_encrypt(struct aead_request *req) 3571 { 3572 return crypto_ipsec_check_assoclen(req->assoclen) ?: 3573 safexcel_aead_encrypt(req); 3574 } 3575 3576 static int safexcel_rfc4106_decrypt(struct aead_request *req) 3577 { 3578 return crypto_ipsec_check_assoclen(req->assoclen) ?: 3579 safexcel_aead_decrypt(req); 3580 } 3581 3582 static int safexcel_rfc4106_gcm_cra_init(struct crypto_tfm *tfm) 3583 { 3584 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3585 int ret; 3586 3587 ret = safexcel_aead_gcm_cra_init(tfm); 3588 ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP; 3589 ctx->aadskip = EIP197_AEAD_IPSEC_IV_SIZE; 3590 return ret; 3591 } 3592 3593 struct safexcel_alg_template safexcel_alg_rfc4106_gcm = { 3594 .type = SAFEXCEL_ALG_TYPE_AEAD, 3595 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH, 3596 .alg.aead = { 3597 .setkey = safexcel_rfc4106_gcm_setkey, 3598 .setauthsize = safexcel_rfc4106_gcm_setauthsize, 3599 .encrypt = safexcel_rfc4106_encrypt, 3600 .decrypt = safexcel_rfc4106_decrypt, 3601 .ivsize = GCM_RFC4106_IV_SIZE, 3602 .maxauthsize = GHASH_DIGEST_SIZE, 3603 .base = { 3604 .cra_name = "rfc4106(gcm(aes))", 3605 .cra_driver_name = "safexcel-rfc4106-gcm-aes", 3606 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3607 .cra_flags = CRYPTO_ALG_ASYNC | 3608 CRYPTO_ALG_ALLOCATES_MEMORY | 3609 CRYPTO_ALG_KERN_DRIVER_ONLY, 3610 .cra_blocksize = 1, 3611 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3612 .cra_alignmask = 0, 3613 .cra_init = safexcel_rfc4106_gcm_cra_init, 3614 .cra_exit = safexcel_aead_gcm_cra_exit, 3615 }, 3616 }, 3617 }; 3618 3619 static int safexcel_rfc4543_gcm_setauthsize(struct crypto_aead *tfm, 3620 unsigned int authsize) 3621 { 3622 if (authsize != GHASH_DIGEST_SIZE) 3623 return -EINVAL; 3624 3625 return 0; 3626 } 3627 3628 static int safexcel_rfc4543_gcm_cra_init(struct crypto_tfm *tfm) 3629 { 3630 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3631 int ret; 3632 3633 ret = safexcel_aead_gcm_cra_init(tfm); 3634 ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP_GMAC; 3635 return ret; 3636 } 3637 3638 struct safexcel_alg_template safexcel_alg_rfc4543_gcm = { 3639 .type = SAFEXCEL_ALG_TYPE_AEAD, 3640 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH, 3641 .alg.aead = { 3642 .setkey = safexcel_rfc4106_gcm_setkey, 3643 .setauthsize = safexcel_rfc4543_gcm_setauthsize, 3644 .encrypt = safexcel_rfc4106_encrypt, 3645 .decrypt = safexcel_rfc4106_decrypt, 3646 .ivsize = GCM_RFC4543_IV_SIZE, 3647 .maxauthsize = GHASH_DIGEST_SIZE, 3648 .base = { 3649 .cra_name = "rfc4543(gcm(aes))", 3650 .cra_driver_name = "safexcel-rfc4543-gcm-aes", 3651 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3652 .cra_flags = CRYPTO_ALG_ASYNC | 3653 CRYPTO_ALG_ALLOCATES_MEMORY | 3654 CRYPTO_ALG_KERN_DRIVER_ONLY, 3655 .cra_blocksize = 1, 3656 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3657 .cra_alignmask = 0, 3658 .cra_init = safexcel_rfc4543_gcm_cra_init, 3659 .cra_exit = safexcel_aead_gcm_cra_exit, 3660 }, 3661 }, 3662 }; 3663 3664 static int safexcel_rfc4309_ccm_setkey(struct crypto_aead *ctfm, const u8 *key, 3665 unsigned int len) 3666 { 3667 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 3668 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3669 3670 /* First byte of the nonce = L = always 3 for RFC4309 (4 byte ctr) */ 3671 *(u8 *)&ctx->nonce = EIP197_AEAD_IPSEC_COUNTER_SIZE - 1; 3672 /* last 3 bytes of key are the nonce! */ 3673 memcpy((u8 *)&ctx->nonce + 1, key + len - 3674 EIP197_AEAD_IPSEC_CCM_NONCE_SIZE, 3675 EIP197_AEAD_IPSEC_CCM_NONCE_SIZE); 3676 3677 len -= EIP197_AEAD_IPSEC_CCM_NONCE_SIZE; 3678 return safexcel_aead_ccm_setkey(ctfm, key, len); 3679 } 3680 3681 static int safexcel_rfc4309_ccm_setauthsize(struct crypto_aead *tfm, 3682 unsigned int authsize) 3683 { 3684 /* Borrowed from crypto/ccm.c */ 3685 switch (authsize) { 3686 case 8: 3687 case 12: 3688 case 16: 3689 break; 3690 default: 3691 return -EINVAL; 3692 } 3693 3694 return 0; 3695 } 3696 3697 static int safexcel_rfc4309_ccm_encrypt(struct aead_request *req) 3698 { 3699 struct safexcel_cipher_req *creq = aead_request_ctx(req); 3700 3701 /* Borrowed from crypto/ccm.c */ 3702 if (req->assoclen != 16 && req->assoclen != 20) 3703 return -EINVAL; 3704 3705 return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); 3706 } 3707 3708 static int safexcel_rfc4309_ccm_decrypt(struct aead_request *req) 3709 { 3710 struct safexcel_cipher_req *creq = aead_request_ctx(req); 3711 3712 /* Borrowed from crypto/ccm.c */ 3713 if (req->assoclen != 16 && req->assoclen != 20) 3714 return -EINVAL; 3715 3716 return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); 3717 } 3718 3719 static int safexcel_rfc4309_ccm_cra_init(struct crypto_tfm *tfm) 3720 { 3721 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3722 int ret; 3723 3724 ret = safexcel_aead_ccm_cra_init(tfm); 3725 ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP; 3726 ctx->aadskip = EIP197_AEAD_IPSEC_IV_SIZE; 3727 return ret; 3728 } 3729 3730 struct safexcel_alg_template safexcel_alg_rfc4309_ccm = { 3731 .type = SAFEXCEL_ALG_TYPE_AEAD, 3732 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_CBC_MAC_ALL, 3733 .alg.aead = { 3734 .setkey = safexcel_rfc4309_ccm_setkey, 3735 .setauthsize = safexcel_rfc4309_ccm_setauthsize, 3736 .encrypt = safexcel_rfc4309_ccm_encrypt, 3737 .decrypt = safexcel_rfc4309_ccm_decrypt, 3738 .ivsize = EIP197_AEAD_IPSEC_IV_SIZE, 3739 .maxauthsize = AES_BLOCK_SIZE, 3740 .base = { 3741 .cra_name = "rfc4309(ccm(aes))", 3742 .cra_driver_name = "safexcel-rfc4309-ccm-aes", 3743 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3744 .cra_flags = CRYPTO_ALG_ASYNC | 3745 CRYPTO_ALG_ALLOCATES_MEMORY | 3746 CRYPTO_ALG_KERN_DRIVER_ONLY, 3747 .cra_blocksize = 1, 3748 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3749 .cra_alignmask = 0, 3750 .cra_init = safexcel_rfc4309_ccm_cra_init, 3751 .cra_exit = safexcel_aead_cra_exit, 3752 .cra_module = THIS_MODULE, 3753 }, 3754 }, 3755 }; 3756