1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2017 Marvell 4 * 5 * Antoine Tenart <antoine.tenart@free-electrons.com> 6 */ 7 8 #include <linux/unaligned.h> 9 #include <linux/device.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/dmapool.h> 12 #include <crypto/aead.h> 13 #include <crypto/aes.h> 14 #include <crypto/authenc.h> 15 #include <crypto/chacha.h> 16 #include <crypto/ctr.h> 17 #include <crypto/internal/des.h> 18 #include <crypto/gcm.h> 19 #include <crypto/ghash.h> 20 #include <crypto/md5.h> 21 #include <crypto/poly1305.h> 22 #include <crypto/sha1.h> 23 #include <crypto/sha2.h> 24 #include <crypto/sm3.h> 25 #include <crypto/sm4.h> 26 #include <crypto/xts.h> 27 #include <crypto/skcipher.h> 28 #include <crypto/internal/aead.h> 29 #include <crypto/internal/skcipher.h> 30 31 #include "safexcel.h" 32 33 enum safexcel_cipher_direction { 34 SAFEXCEL_ENCRYPT, 35 SAFEXCEL_DECRYPT, 36 }; 37 38 enum safexcel_cipher_alg { 39 SAFEXCEL_DES, 40 SAFEXCEL_3DES, 41 SAFEXCEL_AES, 42 SAFEXCEL_CHACHA20, 43 SAFEXCEL_SM4, 44 }; 45 46 struct safexcel_cipher_ctx { 47 struct safexcel_context base; 48 struct safexcel_crypto_priv *priv; 49 50 u32 mode; 51 enum safexcel_cipher_alg alg; 52 u8 aead; /* !=0=AEAD, 2=IPSec ESP AEAD, 3=IPsec ESP GMAC */ 53 u8 xcm; /* 0=authenc, 1=GCM, 2 reserved for CCM */ 54 u8 aadskip; 55 u8 blocksz; 56 u32 ivmask; 57 u32 ctrinit; 58 59 __le32 key[16]; 60 u32 nonce; 61 unsigned int key_len, xts; 62 63 /* All the below is AEAD specific */ 64 u32 hash_alg; 65 u32 state_sz; 66 67 struct crypto_aead *fback; 68 }; 69 70 struct safexcel_cipher_req { 71 enum safexcel_cipher_direction direction; 72 /* Number of result descriptors associated to the request */ 73 unsigned int rdescs; 74 bool needs_inv; 75 int nr_src, nr_dst; 76 }; 77 78 static int safexcel_skcipher_iv(struct safexcel_cipher_ctx *ctx, u8 *iv, 79 struct safexcel_command_desc *cdesc) 80 { 81 if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD) { 82 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; 83 /* 32 bit nonce */ 84 cdesc->control_data.token[0] = ctx->nonce; 85 /* 64 bit IV part */ 86 memcpy(&cdesc->control_data.token[1], iv, 8); 87 /* 32 bit counter, start at 0 or 1 (big endian!) */ 88 cdesc->control_data.token[3] = 89 (__force u32)cpu_to_be32(ctx->ctrinit); 90 return 4; 91 } 92 if (ctx->alg == SAFEXCEL_CHACHA20) { 93 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; 94 /* 96 bit nonce part */ 95 memcpy(&cdesc->control_data.token[0], &iv[4], 12); 96 /* 32 bit counter */ 97 cdesc->control_data.token[3] = *(u32 *)iv; 98 return 4; 99 } 100 101 cdesc->control_data.options |= ctx->ivmask; 102 memcpy(cdesc->control_data.token, iv, ctx->blocksz); 103 return ctx->blocksz / sizeof(u32); 104 } 105 106 static void safexcel_skcipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv, 107 struct safexcel_command_desc *cdesc, 108 struct safexcel_token *atoken, 109 u32 length) 110 { 111 struct safexcel_token *token; 112 int ivlen; 113 114 ivlen = safexcel_skcipher_iv(ctx, iv, cdesc); 115 if (ivlen == 4) { 116 /* No space in cdesc, instruction moves to atoken */ 117 cdesc->additional_cdata_size = 1; 118 token = atoken; 119 } else { 120 /* Everything fits in cdesc */ 121 token = (struct safexcel_token *)(cdesc->control_data.token + 2); 122 /* Need to pad with NOP */ 123 eip197_noop_token(&token[1]); 124 } 125 126 token->opcode = EIP197_TOKEN_OPCODE_DIRECTION; 127 token->packet_length = length; 128 token->stat = EIP197_TOKEN_STAT_LAST_PACKET | 129 EIP197_TOKEN_STAT_LAST_HASH; 130 token->instructions = EIP197_TOKEN_INS_LAST | 131 EIP197_TOKEN_INS_TYPE_CRYPTO | 132 EIP197_TOKEN_INS_TYPE_OUTPUT; 133 } 134 135 static void safexcel_aead_iv(struct safexcel_cipher_ctx *ctx, u8 *iv, 136 struct safexcel_command_desc *cdesc) 137 { 138 if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD || 139 ctx->aead & EIP197_AEAD_TYPE_IPSEC_ESP) { /* _ESP and _ESP_GMAC */ 140 /* 32 bit nonce */ 141 cdesc->control_data.token[0] = ctx->nonce; 142 /* 64 bit IV part */ 143 memcpy(&cdesc->control_data.token[1], iv, 8); 144 /* 32 bit counter, start at 0 or 1 (big endian!) */ 145 cdesc->control_data.token[3] = 146 (__force u32)cpu_to_be32(ctx->ctrinit); 147 return; 148 } 149 if (ctx->xcm == EIP197_XCM_MODE_GCM || ctx->alg == SAFEXCEL_CHACHA20) { 150 /* 96 bit IV part */ 151 memcpy(&cdesc->control_data.token[0], iv, 12); 152 /* 32 bit counter, start at 0 or 1 (big endian!) */ 153 cdesc->control_data.token[3] = 154 (__force u32)cpu_to_be32(ctx->ctrinit); 155 return; 156 } 157 /* CBC */ 158 memcpy(cdesc->control_data.token, iv, ctx->blocksz); 159 } 160 161 static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv, 162 struct safexcel_command_desc *cdesc, 163 struct safexcel_token *atoken, 164 enum safexcel_cipher_direction direction, 165 u32 cryptlen, u32 assoclen, u32 digestsize) 166 { 167 struct safexcel_token *aadref; 168 int atoksize = 2; /* Start with minimum size */ 169 int assocadj = assoclen - ctx->aadskip, aadalign; 170 171 /* Always 4 dwords of embedded IV for AEAD modes */ 172 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; 173 174 if (direction == SAFEXCEL_DECRYPT) 175 cryptlen -= digestsize; 176 177 if (unlikely(ctx->xcm == EIP197_XCM_MODE_CCM)) { 178 /* Construct IV block B0 for the CBC-MAC */ 179 u8 *final_iv = (u8 *)cdesc->control_data.token; 180 u8 *cbcmaciv = (u8 *)&atoken[1]; 181 __le32 *aadlen = (__le32 *)&atoken[5]; 182 183 if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { 184 /* Length + nonce */ 185 cdesc->control_data.token[0] = ctx->nonce; 186 /* Fixup flags byte */ 187 *(__le32 *)cbcmaciv = 188 cpu_to_le32(ctx->nonce | 189 ((assocadj > 0) << 6) | 190 ((digestsize - 2) << 2)); 191 /* 64 bit IV part */ 192 memcpy(&cdesc->control_data.token[1], iv, 8); 193 memcpy(cbcmaciv + 4, iv, 8); 194 /* Start counter at 0 */ 195 cdesc->control_data.token[3] = 0; 196 /* Message length */ 197 *(__be32 *)(cbcmaciv + 12) = cpu_to_be32(cryptlen); 198 } else { 199 /* Variable length IV part */ 200 memcpy(final_iv, iv, 15 - iv[0]); 201 memcpy(cbcmaciv, iv, 15 - iv[0]); 202 /* Start variable length counter at 0 */ 203 memset(final_iv + 15 - iv[0], 0, iv[0] + 1); 204 memset(cbcmaciv + 15 - iv[0], 0, iv[0] - 1); 205 /* fixup flags byte */ 206 cbcmaciv[0] |= ((assocadj > 0) << 6) | 207 ((digestsize - 2) << 2); 208 /* insert lower 2 bytes of message length */ 209 cbcmaciv[14] = cryptlen >> 8; 210 cbcmaciv[15] = cryptlen & 255; 211 } 212 213 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 214 atoken->packet_length = AES_BLOCK_SIZE + 215 ((assocadj > 0) << 1); 216 atoken->stat = 0; 217 atoken->instructions = EIP197_TOKEN_INS_ORIGIN_TOKEN | 218 EIP197_TOKEN_INS_TYPE_HASH; 219 220 if (likely(assocadj)) { 221 *aadlen = cpu_to_le32((assocadj >> 8) | 222 (assocadj & 255) << 8); 223 atoken += 6; 224 atoksize += 7; 225 } else { 226 atoken += 5; 227 atoksize += 6; 228 } 229 230 /* Process AAD data */ 231 aadref = atoken; 232 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION; 233 atoken->packet_length = assocadj; 234 atoken->stat = 0; 235 atoken->instructions = EIP197_TOKEN_INS_TYPE_HASH; 236 atoken++; 237 238 /* For CCM only, align AAD data towards hash engine */ 239 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 240 aadalign = (assocadj + 2) & 15; 241 atoken->packet_length = assocadj && aadalign ? 242 16 - aadalign : 243 0; 244 if (likely(cryptlen)) { 245 atoken->stat = 0; 246 atoken->instructions = EIP197_TOKEN_INS_TYPE_HASH; 247 } else { 248 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH; 249 atoken->instructions = EIP197_TOKEN_INS_LAST | 250 EIP197_TOKEN_INS_TYPE_HASH; 251 } 252 } else { 253 safexcel_aead_iv(ctx, iv, cdesc); 254 255 /* Process AAD data */ 256 aadref = atoken; 257 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION; 258 atoken->packet_length = assocadj; 259 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH; 260 atoken->instructions = EIP197_TOKEN_INS_LAST | 261 EIP197_TOKEN_INS_TYPE_HASH; 262 } 263 atoken++; 264 265 if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { 266 /* For ESP mode (and not GMAC), skip over the IV */ 267 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION; 268 atoken->packet_length = EIP197_AEAD_IPSEC_IV_SIZE; 269 atoken->stat = 0; 270 atoken->instructions = 0; 271 atoken++; 272 atoksize++; 273 } else if (unlikely(ctx->alg == SAFEXCEL_CHACHA20 && 274 direction == SAFEXCEL_DECRYPT)) { 275 /* Poly-chacha decryption needs a dummy NOP here ... */ 276 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 277 atoken->packet_length = 16; /* According to Op Manual */ 278 atoken->stat = 0; 279 atoken->instructions = 0; 280 atoken++; 281 atoksize++; 282 } 283 284 if (ctx->xcm) { 285 /* For GCM and CCM, obtain enc(Y0) */ 286 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT_REMRES; 287 atoken->packet_length = 0; 288 atoken->stat = 0; 289 atoken->instructions = AES_BLOCK_SIZE; 290 atoken++; 291 292 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 293 atoken->packet_length = AES_BLOCK_SIZE; 294 atoken->stat = 0; 295 atoken->instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | 296 EIP197_TOKEN_INS_TYPE_CRYPTO; 297 atoken++; 298 atoksize += 2; 299 } 300 301 if (likely(cryptlen || ctx->alg == SAFEXCEL_CHACHA20)) { 302 /* Fixup stat field for AAD direction instruction */ 303 aadref->stat = 0; 304 305 /* Process crypto data */ 306 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION; 307 atoken->packet_length = cryptlen; 308 309 if (unlikely(ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP_GMAC)) { 310 /* Fixup instruction field for AAD dir instruction */ 311 aadref->instructions = EIP197_TOKEN_INS_TYPE_HASH; 312 313 /* Do not send to crypt engine in case of GMAC */ 314 atoken->instructions = EIP197_TOKEN_INS_LAST | 315 EIP197_TOKEN_INS_TYPE_HASH | 316 EIP197_TOKEN_INS_TYPE_OUTPUT; 317 } else { 318 atoken->instructions = EIP197_TOKEN_INS_LAST | 319 EIP197_TOKEN_INS_TYPE_CRYPTO | 320 EIP197_TOKEN_INS_TYPE_HASH | 321 EIP197_TOKEN_INS_TYPE_OUTPUT; 322 } 323 324 cryptlen &= 15; 325 if (unlikely(ctx->xcm == EIP197_XCM_MODE_CCM && cryptlen)) { 326 atoken->stat = 0; 327 /* For CCM only, pad crypto data to the hash engine */ 328 atoken++; 329 atoksize++; 330 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 331 atoken->packet_length = 16 - cryptlen; 332 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH; 333 atoken->instructions = EIP197_TOKEN_INS_TYPE_HASH; 334 } else { 335 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH; 336 } 337 atoken++; 338 atoksize++; 339 } 340 341 if (direction == SAFEXCEL_ENCRYPT) { 342 /* Append ICV */ 343 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 344 atoken->packet_length = digestsize; 345 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH | 346 EIP197_TOKEN_STAT_LAST_PACKET; 347 atoken->instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | 348 EIP197_TOKEN_INS_INSERT_HASH_DIGEST; 349 } else { 350 /* Extract ICV */ 351 atoken->opcode = EIP197_TOKEN_OPCODE_RETRIEVE; 352 atoken->packet_length = digestsize; 353 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH | 354 EIP197_TOKEN_STAT_LAST_PACKET; 355 atoken->instructions = EIP197_TOKEN_INS_INSERT_HASH_DIGEST; 356 atoken++; 357 atoksize++; 358 359 /* Verify ICV */ 360 atoken->opcode = EIP197_TOKEN_OPCODE_VERIFY; 361 atoken->packet_length = digestsize | 362 EIP197_TOKEN_HASH_RESULT_VERIFY; 363 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH | 364 EIP197_TOKEN_STAT_LAST_PACKET; 365 atoken->instructions = EIP197_TOKEN_INS_TYPE_OUTPUT; 366 } 367 368 /* Fixup length of the token in the command descriptor */ 369 cdesc->additional_cdata_size = atoksize; 370 } 371 372 static int safexcel_skcipher_aes_setkey(struct crypto_skcipher *ctfm, 373 const u8 *key, unsigned int len) 374 { 375 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 376 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 377 struct safexcel_crypto_priv *priv = ctx->base.priv; 378 struct crypto_aes_ctx aes __cleanup(aes_zeroize_ctx); 379 int ret, i; 380 381 ret = aes_expandkey(&aes, key, len); 382 if (ret) 383 return ret; 384 385 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 386 for (i = 0; i < len / sizeof(u32); i++) { 387 if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { 388 ctx->base.needs_inv = true; 389 break; 390 } 391 } 392 } 393 394 for (i = 0; i < len / sizeof(u32); i++) 395 ctx->key[i] = cpu_to_le32(aes.key_enc[i]); 396 397 ctx->key_len = len; 398 399 return 0; 400 } 401 402 static int safexcel_aead_setkey(struct crypto_aead *ctfm, const u8 *key, 403 unsigned int len) 404 { 405 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 406 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 407 struct safexcel_crypto_priv *priv = ctx->base.priv; 408 struct crypto_authenc_keys keys; 409 int err = -EINVAL, i; 410 const char *alg; 411 412 if (unlikely(crypto_authenc_extractkeys(&keys, key, len))) 413 goto badkey; 414 415 if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD) { 416 /* Must have at least space for the nonce here */ 417 if (unlikely(keys.enckeylen < CTR_RFC3686_NONCE_SIZE)) 418 goto badkey; 419 /* last 4 bytes of key are the nonce! */ 420 ctx->nonce = *(u32 *)(keys.enckey + keys.enckeylen - 421 CTR_RFC3686_NONCE_SIZE); 422 /* exclude the nonce here */ 423 keys.enckeylen -= CTR_RFC3686_NONCE_SIZE; 424 } 425 426 /* Encryption key */ 427 switch (ctx->alg) { 428 case SAFEXCEL_DES: 429 err = verify_aead_des_key(ctfm, keys.enckey, keys.enckeylen); 430 if (unlikely(err)) 431 goto badkey; 432 break; 433 case SAFEXCEL_3DES: 434 err = verify_aead_des3_key(ctfm, keys.enckey, keys.enckeylen); 435 if (unlikely(err)) 436 goto badkey; 437 break; 438 case SAFEXCEL_AES: 439 err = aes_check_keylen(keys.enckeylen); 440 if (unlikely(err)) 441 goto badkey; 442 break; 443 case SAFEXCEL_SM4: 444 if (unlikely(keys.enckeylen != SM4_KEY_SIZE)) 445 goto badkey; 446 break; 447 default: 448 dev_err(priv->dev, "aead: unsupported cipher algorithm\n"); 449 goto badkey; 450 } 451 452 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 453 for (i = 0; i < keys.enckeylen / sizeof(u32); i++) { 454 if (le32_to_cpu(ctx->key[i]) != 455 ((u32 *)keys.enckey)[i]) { 456 ctx->base.needs_inv = true; 457 break; 458 } 459 } 460 } 461 462 /* Auth key */ 463 switch (ctx->hash_alg) { 464 case CONTEXT_CONTROL_CRYPTO_ALG_MD5: 465 alg = "safexcel-md5"; 466 break; 467 case CONTEXT_CONTROL_CRYPTO_ALG_SHA1: 468 alg = "safexcel-sha1"; 469 break; 470 case CONTEXT_CONTROL_CRYPTO_ALG_SHA224: 471 alg = "safexcel-sha224"; 472 break; 473 case CONTEXT_CONTROL_CRYPTO_ALG_SHA256: 474 alg = "safexcel-sha256"; 475 break; 476 case CONTEXT_CONTROL_CRYPTO_ALG_SHA384: 477 alg = "safexcel-sha384"; 478 break; 479 case CONTEXT_CONTROL_CRYPTO_ALG_SHA512: 480 alg = "safexcel-sha512"; 481 break; 482 case CONTEXT_CONTROL_CRYPTO_ALG_SM3: 483 alg = "safexcel-sm3"; 484 break; 485 default: 486 dev_err(priv->dev, "aead: unsupported hash algorithm\n"); 487 goto badkey; 488 } 489 490 if (safexcel_hmac_setkey(&ctx->base, keys.authkey, keys.authkeylen, 491 alg, ctx->state_sz)) 492 goto badkey; 493 494 /* Now copy the keys into the context */ 495 for (i = 0; i < keys.enckeylen / sizeof(u32); i++) 496 ctx->key[i] = cpu_to_le32(((u32 *)keys.enckey)[i]); 497 ctx->key_len = keys.enckeylen; 498 499 memzero_explicit(&keys, sizeof(keys)); 500 return 0; 501 502 badkey: 503 memzero_explicit(&keys, sizeof(keys)); 504 return err; 505 } 506 507 static int safexcel_context_control(struct safexcel_cipher_ctx *ctx, 508 struct crypto_async_request *async, 509 struct safexcel_cipher_req *sreq, 510 struct safexcel_command_desc *cdesc) 511 { 512 struct safexcel_crypto_priv *priv = ctx->base.priv; 513 int ctrl_size = ctx->key_len / sizeof(u32); 514 515 cdesc->control_data.control1 = ctx->mode; 516 517 if (ctx->aead) { 518 /* Take in account the ipad+opad digests */ 519 if (ctx->xcm) { 520 ctrl_size += ctx->state_sz / sizeof(u32); 521 cdesc->control_data.control0 = 522 CONTEXT_CONTROL_KEY_EN | 523 CONTEXT_CONTROL_DIGEST_XCM | 524 ctx->hash_alg | 525 CONTEXT_CONTROL_SIZE(ctrl_size); 526 } else if (ctx->alg == SAFEXCEL_CHACHA20) { 527 /* Chacha20-Poly1305 */ 528 cdesc->control_data.control0 = 529 CONTEXT_CONTROL_KEY_EN | 530 CONTEXT_CONTROL_CRYPTO_ALG_CHACHA20 | 531 (sreq->direction == SAFEXCEL_ENCRYPT ? 532 CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT : 533 CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN) | 534 ctx->hash_alg | 535 CONTEXT_CONTROL_SIZE(ctrl_size); 536 return 0; 537 } else { 538 ctrl_size += ctx->state_sz / sizeof(u32) * 2; 539 cdesc->control_data.control0 = 540 CONTEXT_CONTROL_KEY_EN | 541 CONTEXT_CONTROL_DIGEST_HMAC | 542 ctx->hash_alg | 543 CONTEXT_CONTROL_SIZE(ctrl_size); 544 } 545 546 if (sreq->direction == SAFEXCEL_ENCRYPT && 547 (ctx->xcm == EIP197_XCM_MODE_CCM || 548 ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP_GMAC)) 549 cdesc->control_data.control0 |= 550 CONTEXT_CONTROL_TYPE_HASH_ENCRYPT_OUT; 551 else if (sreq->direction == SAFEXCEL_ENCRYPT) 552 cdesc->control_data.control0 |= 553 CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT; 554 else if (ctx->xcm == EIP197_XCM_MODE_CCM) 555 cdesc->control_data.control0 |= 556 CONTEXT_CONTROL_TYPE_DECRYPT_HASH_IN; 557 else 558 cdesc->control_data.control0 |= 559 CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN; 560 } else { 561 if (sreq->direction == SAFEXCEL_ENCRYPT) 562 cdesc->control_data.control0 = 563 CONTEXT_CONTROL_TYPE_CRYPTO_OUT | 564 CONTEXT_CONTROL_KEY_EN | 565 CONTEXT_CONTROL_SIZE(ctrl_size); 566 else 567 cdesc->control_data.control0 = 568 CONTEXT_CONTROL_TYPE_CRYPTO_IN | 569 CONTEXT_CONTROL_KEY_EN | 570 CONTEXT_CONTROL_SIZE(ctrl_size); 571 } 572 573 if (ctx->alg == SAFEXCEL_DES) { 574 cdesc->control_data.control0 |= 575 CONTEXT_CONTROL_CRYPTO_ALG_DES; 576 } else if (ctx->alg == SAFEXCEL_3DES) { 577 cdesc->control_data.control0 |= 578 CONTEXT_CONTROL_CRYPTO_ALG_3DES; 579 } else if (ctx->alg == SAFEXCEL_AES) { 580 switch (ctx->key_len >> ctx->xts) { 581 case AES_KEYSIZE_128: 582 cdesc->control_data.control0 |= 583 CONTEXT_CONTROL_CRYPTO_ALG_AES128; 584 break; 585 case AES_KEYSIZE_192: 586 cdesc->control_data.control0 |= 587 CONTEXT_CONTROL_CRYPTO_ALG_AES192; 588 break; 589 case AES_KEYSIZE_256: 590 cdesc->control_data.control0 |= 591 CONTEXT_CONTROL_CRYPTO_ALG_AES256; 592 break; 593 default: 594 dev_err(priv->dev, "aes keysize not supported: %u\n", 595 ctx->key_len >> ctx->xts); 596 return -EINVAL; 597 } 598 } else if (ctx->alg == SAFEXCEL_CHACHA20) { 599 cdesc->control_data.control0 |= 600 CONTEXT_CONTROL_CRYPTO_ALG_CHACHA20; 601 } else if (ctx->alg == SAFEXCEL_SM4) { 602 cdesc->control_data.control0 |= 603 CONTEXT_CONTROL_CRYPTO_ALG_SM4; 604 } 605 606 return 0; 607 } 608 609 static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int ring, 610 struct crypto_async_request *async, 611 struct scatterlist *src, 612 struct scatterlist *dst, 613 unsigned int cryptlen, 614 struct safexcel_cipher_req *sreq, 615 bool *should_complete, int *ret) 616 { 617 struct skcipher_request *areq = skcipher_request_cast(async); 618 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(areq); 619 struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(skcipher); 620 struct safexcel_result_desc *rdesc; 621 int ndesc = 0; 622 623 *ret = 0; 624 625 if (unlikely(!sreq->rdescs)) 626 return 0; 627 628 while (sreq->rdescs--) { 629 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); 630 if (IS_ERR(rdesc)) { 631 dev_err(priv->dev, 632 "cipher: result: could not retrieve the result descriptor\n"); 633 *ret = PTR_ERR(rdesc); 634 break; 635 } 636 637 if (likely(!*ret)) 638 *ret = safexcel_rdesc_check_errors(priv, rdesc); 639 640 ndesc++; 641 } 642 643 safexcel_complete(priv, ring); 644 645 if (src == dst) { 646 if (sreq->nr_src > 0) 647 dma_unmap_sg(priv->dev, src, sreq->nr_src, 648 DMA_BIDIRECTIONAL); 649 } else { 650 if (sreq->nr_src > 0) 651 dma_unmap_sg(priv->dev, src, sreq->nr_src, 652 DMA_TO_DEVICE); 653 if (sreq->nr_dst > 0) 654 dma_unmap_sg(priv->dev, dst, sreq->nr_dst, 655 DMA_FROM_DEVICE); 656 } 657 658 /* 659 * Update IV in req from last crypto output word for CBC modes 660 */ 661 if ((!ctx->aead) && (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) && 662 (sreq->direction == SAFEXCEL_ENCRYPT)) { 663 /* For encrypt take the last output word */ 664 sg_pcopy_to_buffer(dst, sreq->nr_dst, areq->iv, 665 crypto_skcipher_ivsize(skcipher), 666 (cryptlen - 667 crypto_skcipher_ivsize(skcipher))); 668 } 669 670 *should_complete = true; 671 672 return ndesc; 673 } 674 675 static int safexcel_send_req(struct crypto_async_request *base, int ring, 676 struct safexcel_cipher_req *sreq, 677 struct scatterlist *src, struct scatterlist *dst, 678 unsigned int cryptlen, unsigned int assoclen, 679 unsigned int digestsize, u8 *iv, int *commands, 680 int *results) 681 { 682 struct skcipher_request *areq = skcipher_request_cast(base); 683 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(areq); 684 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); 685 struct safexcel_crypto_priv *priv = ctx->base.priv; 686 struct safexcel_command_desc *cdesc; 687 struct safexcel_command_desc *first_cdesc = NULL; 688 struct safexcel_result_desc *rdesc, *first_rdesc = NULL; 689 struct scatterlist *sg; 690 unsigned int totlen; 691 unsigned int totlen_src = cryptlen + assoclen; 692 unsigned int totlen_dst = totlen_src; 693 struct safexcel_token *atoken; 694 int n_cdesc = 0, n_rdesc = 0; 695 int queued, i, ret = 0; 696 bool first = true; 697 698 sreq->nr_src = sg_nents_for_len(src, totlen_src); 699 700 if (ctx->aead) { 701 /* 702 * AEAD has auth tag appended to output for encrypt and 703 * removed from the output for decrypt! 704 */ 705 if (sreq->direction == SAFEXCEL_DECRYPT) 706 totlen_dst -= digestsize; 707 else 708 totlen_dst += digestsize; 709 710 memcpy(ctx->base.ctxr->data + ctx->key_len / sizeof(u32), 711 &ctx->base.ipad, ctx->state_sz); 712 if (!ctx->xcm) 713 memcpy(ctx->base.ctxr->data + (ctx->key_len + 714 ctx->state_sz) / sizeof(u32), &ctx->base.opad, 715 ctx->state_sz); 716 } else if ((ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) && 717 (sreq->direction == SAFEXCEL_DECRYPT)) { 718 /* 719 * Save IV from last crypto input word for CBC modes in decrypt 720 * direction. Need to do this first in case of inplace operation 721 * as it will be overwritten. 722 */ 723 sg_pcopy_to_buffer(src, sreq->nr_src, areq->iv, 724 crypto_skcipher_ivsize(skcipher), 725 (totlen_src - 726 crypto_skcipher_ivsize(skcipher))); 727 } 728 729 sreq->nr_dst = sg_nents_for_len(dst, totlen_dst); 730 731 /* 732 * Remember actual input length, source buffer length may be 733 * updated in case of inline operation below. 734 */ 735 totlen = totlen_src; 736 queued = totlen_src; 737 738 if (src == dst) { 739 sreq->nr_src = max(sreq->nr_src, sreq->nr_dst); 740 sreq->nr_dst = sreq->nr_src; 741 if (unlikely((totlen_src || totlen_dst) && 742 (sreq->nr_src <= 0))) { 743 dev_err(priv->dev, "In-place buffer not large enough (need %d bytes)!", 744 max(totlen_src, totlen_dst)); 745 return -EINVAL; 746 } 747 if (sreq->nr_src > 0 && 748 !dma_map_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL)) 749 return -EIO; 750 } else { 751 if (unlikely(totlen_src && (sreq->nr_src <= 0))) { 752 dev_err(priv->dev, "Source buffer not large enough (need %d bytes)!", 753 totlen_src); 754 return -EINVAL; 755 } 756 757 if (sreq->nr_src > 0 && 758 !dma_map_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE)) 759 return -EIO; 760 761 if (unlikely(totlen_dst && (sreq->nr_dst <= 0))) { 762 dev_err(priv->dev, "Dest buffer not large enough (need %d bytes)!", 763 totlen_dst); 764 ret = -EINVAL; 765 goto unmap; 766 } 767 768 if (sreq->nr_dst > 0 && 769 !dma_map_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE)) { 770 ret = -EIO; 771 goto unmap; 772 } 773 } 774 775 memcpy(ctx->base.ctxr->data, ctx->key, ctx->key_len); 776 777 if (!totlen) { 778 /* 779 * The EIP97 cannot deal with zero length input packets! 780 * So stuff a dummy command descriptor indicating a 1 byte 781 * (dummy) input packet, using the context record as source. 782 */ 783 first_cdesc = safexcel_add_cdesc(priv, ring, 784 1, 1, ctx->base.ctxr_dma, 785 1, 1, ctx->base.ctxr_dma, 786 &atoken); 787 if (IS_ERR(first_cdesc)) { 788 /* No space left in the command descriptor ring */ 789 ret = PTR_ERR(first_cdesc); 790 goto cdesc_rollback; 791 } 792 n_cdesc = 1; 793 goto skip_cdesc; 794 } 795 796 /* command descriptors */ 797 for_each_sg(src, sg, sreq->nr_src, i) { 798 int len = sg_dma_len(sg); 799 800 /* Do not overflow the request */ 801 if (queued < len) 802 len = queued; 803 804 cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc, 805 !(queued - len), 806 sg_dma_address(sg), len, totlen, 807 ctx->base.ctxr_dma, &atoken); 808 if (IS_ERR(cdesc)) { 809 /* No space left in the command descriptor ring */ 810 ret = PTR_ERR(cdesc); 811 goto cdesc_rollback; 812 } 813 814 if (!n_cdesc) 815 first_cdesc = cdesc; 816 817 n_cdesc++; 818 queued -= len; 819 if (!queued) 820 break; 821 } 822 skip_cdesc: 823 /* Add context control words and token to first command descriptor */ 824 safexcel_context_control(ctx, base, sreq, first_cdesc); 825 if (ctx->aead) 826 safexcel_aead_token(ctx, iv, first_cdesc, atoken, 827 sreq->direction, cryptlen, 828 assoclen, digestsize); 829 else 830 safexcel_skcipher_token(ctx, iv, first_cdesc, atoken, 831 cryptlen); 832 833 /* result descriptors */ 834 for_each_sg(dst, sg, sreq->nr_dst, i) { 835 bool last = (i == sreq->nr_dst - 1); 836 u32 len = sg_dma_len(sg); 837 838 /* only allow the part of the buffer we know we need */ 839 if (len > totlen_dst) 840 len = totlen_dst; 841 if (unlikely(!len)) 842 break; 843 totlen_dst -= len; 844 845 /* skip over AAD space in buffer - not written */ 846 if (assoclen) { 847 if (assoclen >= len) { 848 assoclen -= len; 849 continue; 850 } 851 rdesc = safexcel_add_rdesc(priv, ring, first, last, 852 sg_dma_address(sg) + 853 assoclen, 854 len - assoclen); 855 assoclen = 0; 856 } else { 857 rdesc = safexcel_add_rdesc(priv, ring, first, last, 858 sg_dma_address(sg), 859 len); 860 } 861 if (IS_ERR(rdesc)) { 862 /* No space left in the result descriptor ring */ 863 ret = PTR_ERR(rdesc); 864 goto rdesc_rollback; 865 } 866 if (first) { 867 first_rdesc = rdesc; 868 first = false; 869 } 870 n_rdesc++; 871 } 872 873 if (unlikely(first)) { 874 /* 875 * Special case: AEAD decrypt with only AAD data. 876 * In this case there is NO output data from the engine, 877 * but the engine still needs a result descriptor! 878 * Create a dummy one just for catching the result token. 879 */ 880 rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0); 881 if (IS_ERR(rdesc)) { 882 /* No space left in the result descriptor ring */ 883 ret = PTR_ERR(rdesc); 884 goto rdesc_rollback; 885 } 886 first_rdesc = rdesc; 887 n_rdesc = 1; 888 } 889 890 safexcel_rdr_req_set(priv, ring, first_rdesc, base); 891 892 *commands = n_cdesc; 893 *results = n_rdesc; 894 return 0; 895 896 rdesc_rollback: 897 for (i = 0; i < n_rdesc; i++) 898 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].rdr); 899 cdesc_rollback: 900 for (i = 0; i < n_cdesc; i++) 901 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr); 902 unmap: 903 if (src == dst) { 904 if (sreq->nr_src > 0) 905 dma_unmap_sg(priv->dev, src, sreq->nr_src, 906 DMA_BIDIRECTIONAL); 907 } else { 908 if (sreq->nr_src > 0) 909 dma_unmap_sg(priv->dev, src, sreq->nr_src, 910 DMA_TO_DEVICE); 911 if (sreq->nr_dst > 0) 912 dma_unmap_sg(priv->dev, dst, sreq->nr_dst, 913 DMA_FROM_DEVICE); 914 } 915 916 return ret; 917 } 918 919 static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv, 920 int ring, 921 struct crypto_async_request *base, 922 struct safexcel_cipher_req *sreq, 923 bool *should_complete, int *ret) 924 { 925 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); 926 struct safexcel_result_desc *rdesc; 927 int ndesc = 0, enq_ret; 928 929 *ret = 0; 930 931 if (unlikely(!sreq->rdescs)) 932 return 0; 933 934 while (sreq->rdescs--) { 935 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); 936 if (IS_ERR(rdesc)) { 937 dev_err(priv->dev, 938 "cipher: invalidate: could not retrieve the result descriptor\n"); 939 *ret = PTR_ERR(rdesc); 940 break; 941 } 942 943 if (likely(!*ret)) 944 *ret = safexcel_rdesc_check_errors(priv, rdesc); 945 946 ndesc++; 947 } 948 949 safexcel_complete(priv, ring); 950 951 if (ctx->base.exit_inv) { 952 dma_pool_free(priv->context_pool, ctx->base.ctxr, 953 ctx->base.ctxr_dma); 954 955 *should_complete = true; 956 957 return ndesc; 958 } 959 960 ring = safexcel_select_ring(priv); 961 ctx->base.ring = ring; 962 963 spin_lock_bh(&priv->ring[ring].queue_lock); 964 enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, base); 965 spin_unlock_bh(&priv->ring[ring].queue_lock); 966 967 if (enq_ret != -EINPROGRESS) 968 *ret = enq_ret; 969 970 queue_work(priv->ring[ring].workqueue, 971 &priv->ring[ring].work_data.work); 972 973 *should_complete = false; 974 975 return ndesc; 976 } 977 978 static int safexcel_skcipher_handle_result(struct safexcel_crypto_priv *priv, 979 int ring, 980 struct crypto_async_request *async, 981 bool *should_complete, int *ret) 982 { 983 struct skcipher_request *req = skcipher_request_cast(async); 984 struct safexcel_cipher_req *sreq = skcipher_request_ctx(req); 985 int err; 986 987 if (sreq->needs_inv) { 988 sreq->needs_inv = false; 989 err = safexcel_handle_inv_result(priv, ring, async, sreq, 990 should_complete, ret); 991 } else { 992 err = safexcel_handle_req_result(priv, ring, async, req->src, 993 req->dst, req->cryptlen, sreq, 994 should_complete, ret); 995 } 996 997 return err; 998 } 999 1000 static int safexcel_aead_handle_result(struct safexcel_crypto_priv *priv, 1001 int ring, 1002 struct crypto_async_request *async, 1003 bool *should_complete, int *ret) 1004 { 1005 struct aead_request *req = aead_request_cast(async); 1006 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1007 struct safexcel_cipher_req *sreq = aead_request_ctx(req); 1008 int err; 1009 1010 if (sreq->needs_inv) { 1011 sreq->needs_inv = false; 1012 err = safexcel_handle_inv_result(priv, ring, async, sreq, 1013 should_complete, ret); 1014 } else { 1015 err = safexcel_handle_req_result(priv, ring, async, req->src, 1016 req->dst, 1017 req->cryptlen + crypto_aead_authsize(tfm), 1018 sreq, should_complete, ret); 1019 } 1020 1021 return err; 1022 } 1023 1024 static int safexcel_cipher_send_inv(struct crypto_async_request *base, 1025 int ring, int *commands, int *results) 1026 { 1027 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); 1028 struct safexcel_crypto_priv *priv = ctx->base.priv; 1029 int ret; 1030 1031 ret = safexcel_invalidate_cache(base, priv, ctx->base.ctxr_dma, ring); 1032 if (unlikely(ret)) 1033 return ret; 1034 1035 *commands = 1; 1036 *results = 1; 1037 1038 return 0; 1039 } 1040 1041 static int safexcel_skcipher_send(struct crypto_async_request *async, int ring, 1042 int *commands, int *results) 1043 { 1044 struct skcipher_request *req = skcipher_request_cast(async); 1045 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); 1046 struct safexcel_cipher_req *sreq = skcipher_request_ctx(req); 1047 struct safexcel_crypto_priv *priv = ctx->base.priv; 1048 int ret; 1049 1050 BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv); 1051 1052 if (sreq->needs_inv) { 1053 ret = safexcel_cipher_send_inv(async, ring, commands, results); 1054 } else { 1055 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1056 u8 input_iv[AES_BLOCK_SIZE]; 1057 1058 /* 1059 * Save input IV in case of CBC decrypt mode 1060 * Will be overwritten with output IV prior to use! 1061 */ 1062 memcpy(input_iv, req->iv, crypto_skcipher_ivsize(skcipher)); 1063 1064 ret = safexcel_send_req(async, ring, sreq, req->src, 1065 req->dst, req->cryptlen, 0, 0, input_iv, 1066 commands, results); 1067 } 1068 1069 sreq->rdescs = *results; 1070 return ret; 1071 } 1072 1073 static int safexcel_aead_send(struct crypto_async_request *async, int ring, 1074 int *commands, int *results) 1075 { 1076 struct aead_request *req = aead_request_cast(async); 1077 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1078 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); 1079 struct safexcel_cipher_req *sreq = aead_request_ctx(req); 1080 struct safexcel_crypto_priv *priv = ctx->base.priv; 1081 int ret; 1082 1083 BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv); 1084 1085 if (sreq->needs_inv) 1086 ret = safexcel_cipher_send_inv(async, ring, commands, results); 1087 else 1088 ret = safexcel_send_req(async, ring, sreq, req->src, req->dst, 1089 req->cryptlen, req->assoclen, 1090 crypto_aead_authsize(tfm), req->iv, 1091 commands, results); 1092 sreq->rdescs = *results; 1093 return ret; 1094 } 1095 1096 static int safexcel_cipher_exit_inv(struct crypto_tfm *tfm, 1097 struct crypto_async_request *base, 1098 struct safexcel_cipher_req *sreq, 1099 struct crypto_wait *result) 1100 { 1101 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1102 struct safexcel_crypto_priv *priv = ctx->base.priv; 1103 int ring = ctx->base.ring; 1104 int err; 1105 1106 ctx = crypto_tfm_ctx(base->tfm); 1107 ctx->base.exit_inv = true; 1108 sreq->needs_inv = true; 1109 1110 spin_lock_bh(&priv->ring[ring].queue_lock); 1111 crypto_enqueue_request(&priv->ring[ring].queue, base); 1112 spin_unlock_bh(&priv->ring[ring].queue_lock); 1113 1114 queue_work(priv->ring[ring].workqueue, 1115 &priv->ring[ring].work_data.work); 1116 1117 err = crypto_wait_req(-EINPROGRESS, result); 1118 1119 if (err) { 1120 dev_warn(priv->dev, 1121 "cipher: sync: invalidate: completion error %d\n", 1122 err); 1123 return err; 1124 } 1125 1126 return 0; 1127 } 1128 1129 static int safexcel_skcipher_exit_inv(struct crypto_tfm *tfm) 1130 { 1131 EIP197_REQUEST_ON_STACK(req, skcipher, EIP197_SKCIPHER_REQ_SIZE); 1132 struct safexcel_cipher_req *sreq = skcipher_request_ctx(req); 1133 DECLARE_CRYPTO_WAIT(result); 1134 1135 memset(req, 0, sizeof(struct skcipher_request)); 1136 1137 skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 1138 crypto_req_done, &result); 1139 skcipher_request_set_tfm(req, __crypto_skcipher_cast(tfm)); 1140 1141 return safexcel_cipher_exit_inv(tfm, &req->base, sreq, &result); 1142 } 1143 1144 static int safexcel_aead_exit_inv(struct crypto_tfm *tfm) 1145 { 1146 EIP197_REQUEST_ON_STACK(req, aead, EIP197_AEAD_REQ_SIZE); 1147 struct safexcel_cipher_req *sreq = aead_request_ctx(req); 1148 DECLARE_CRYPTO_WAIT(result); 1149 1150 memset(req, 0, sizeof(struct aead_request)); 1151 1152 aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 1153 crypto_req_done, &result); 1154 aead_request_set_tfm(req, __crypto_aead_cast(tfm)); 1155 1156 return safexcel_cipher_exit_inv(tfm, &req->base, sreq, &result); 1157 } 1158 1159 static int safexcel_queue_req(struct crypto_async_request *base, 1160 struct safexcel_cipher_req *sreq, 1161 enum safexcel_cipher_direction dir) 1162 { 1163 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); 1164 struct safexcel_crypto_priv *priv = ctx->base.priv; 1165 int ret, ring; 1166 1167 sreq->needs_inv = false; 1168 sreq->direction = dir; 1169 1170 if (ctx->base.ctxr) { 1171 if (priv->flags & EIP197_TRC_CACHE && ctx->base.needs_inv) { 1172 sreq->needs_inv = true; 1173 ctx->base.needs_inv = false; 1174 } 1175 } else { 1176 ctx->base.ring = safexcel_select_ring(priv); 1177 ctx->base.ctxr = dma_pool_zalloc(priv->context_pool, 1178 EIP197_GFP_FLAGS(*base), 1179 &ctx->base.ctxr_dma); 1180 if (!ctx->base.ctxr) 1181 return -ENOMEM; 1182 } 1183 1184 ring = ctx->base.ring; 1185 1186 spin_lock_bh(&priv->ring[ring].queue_lock); 1187 ret = crypto_enqueue_request(&priv->ring[ring].queue, base); 1188 spin_unlock_bh(&priv->ring[ring].queue_lock); 1189 1190 queue_work(priv->ring[ring].workqueue, 1191 &priv->ring[ring].work_data.work); 1192 1193 return ret; 1194 } 1195 1196 static int safexcel_encrypt(struct skcipher_request *req) 1197 { 1198 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 1199 SAFEXCEL_ENCRYPT); 1200 } 1201 1202 static int safexcel_decrypt(struct skcipher_request *req) 1203 { 1204 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 1205 SAFEXCEL_DECRYPT); 1206 } 1207 1208 static int safexcel_skcipher_cra_init(struct crypto_tfm *tfm) 1209 { 1210 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1211 struct safexcel_alg_template *tmpl = 1212 container_of(tfm->__crt_alg, struct safexcel_alg_template, 1213 alg.skcipher.base); 1214 1215 crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm), 1216 sizeof(struct safexcel_cipher_req)); 1217 1218 ctx->base.priv = tmpl->priv; 1219 1220 ctx->base.send = safexcel_skcipher_send; 1221 ctx->base.handle_result = safexcel_skcipher_handle_result; 1222 ctx->ivmask = EIP197_OPTION_4_TOKEN_IV_CMD; 1223 ctx->ctrinit = 1; 1224 return 0; 1225 } 1226 1227 static int safexcel_cipher_cra_exit(struct crypto_tfm *tfm) 1228 { 1229 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1230 1231 memzero_explicit(ctx->key, sizeof(ctx->key)); 1232 1233 /* context not allocated, skip invalidation */ 1234 if (!ctx->base.ctxr) 1235 return -ENOMEM; 1236 1237 memzero_explicit(ctx->base.ctxr->data, sizeof(ctx->base.ctxr->data)); 1238 return 0; 1239 } 1240 1241 static void safexcel_skcipher_cra_exit(struct crypto_tfm *tfm) 1242 { 1243 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1244 struct safexcel_crypto_priv *priv = ctx->base.priv; 1245 int ret; 1246 1247 if (safexcel_cipher_cra_exit(tfm)) 1248 return; 1249 1250 if (priv->flags & EIP197_TRC_CACHE) { 1251 ret = safexcel_skcipher_exit_inv(tfm); 1252 if (ret) 1253 dev_warn(priv->dev, "skcipher: invalidation error %d\n", 1254 ret); 1255 } else { 1256 dma_pool_free(priv->context_pool, ctx->base.ctxr, 1257 ctx->base.ctxr_dma); 1258 } 1259 } 1260 1261 static void safexcel_aead_cra_exit(struct crypto_tfm *tfm) 1262 { 1263 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1264 struct safexcel_crypto_priv *priv = ctx->base.priv; 1265 int ret; 1266 1267 if (safexcel_cipher_cra_exit(tfm)) 1268 return; 1269 1270 if (priv->flags & EIP197_TRC_CACHE) { 1271 ret = safexcel_aead_exit_inv(tfm); 1272 if (ret) 1273 dev_warn(priv->dev, "aead: invalidation error %d\n", 1274 ret); 1275 } else { 1276 dma_pool_free(priv->context_pool, ctx->base.ctxr, 1277 ctx->base.ctxr_dma); 1278 } 1279 } 1280 1281 static int safexcel_skcipher_aes_ecb_cra_init(struct crypto_tfm *tfm) 1282 { 1283 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1284 1285 safexcel_skcipher_cra_init(tfm); 1286 ctx->alg = SAFEXCEL_AES; 1287 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; 1288 ctx->blocksz = 0; 1289 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1290 return 0; 1291 } 1292 1293 struct safexcel_alg_template safexcel_alg_ecb_aes = { 1294 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1295 .algo_mask = SAFEXCEL_ALG_AES, 1296 .alg.skcipher = { 1297 .setkey = safexcel_skcipher_aes_setkey, 1298 .encrypt = safexcel_encrypt, 1299 .decrypt = safexcel_decrypt, 1300 .min_keysize = AES_MIN_KEY_SIZE, 1301 .max_keysize = AES_MAX_KEY_SIZE, 1302 .base = { 1303 .cra_name = "ecb(aes)", 1304 .cra_driver_name = "safexcel-ecb-aes", 1305 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1306 .cra_flags = CRYPTO_ALG_ASYNC | 1307 CRYPTO_ALG_ALLOCATES_MEMORY | 1308 CRYPTO_ALG_KERN_DRIVER_ONLY, 1309 .cra_blocksize = AES_BLOCK_SIZE, 1310 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1311 .cra_alignmask = 0, 1312 .cra_init = safexcel_skcipher_aes_ecb_cra_init, 1313 .cra_exit = safexcel_skcipher_cra_exit, 1314 .cra_module = THIS_MODULE, 1315 }, 1316 }, 1317 }; 1318 1319 static int safexcel_skcipher_aes_cbc_cra_init(struct crypto_tfm *tfm) 1320 { 1321 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1322 1323 safexcel_skcipher_cra_init(tfm); 1324 ctx->alg = SAFEXCEL_AES; 1325 ctx->blocksz = AES_BLOCK_SIZE; 1326 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; 1327 return 0; 1328 } 1329 1330 struct safexcel_alg_template safexcel_alg_cbc_aes = { 1331 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1332 .algo_mask = SAFEXCEL_ALG_AES, 1333 .alg.skcipher = { 1334 .setkey = safexcel_skcipher_aes_setkey, 1335 .encrypt = safexcel_encrypt, 1336 .decrypt = safexcel_decrypt, 1337 .min_keysize = AES_MIN_KEY_SIZE, 1338 .max_keysize = AES_MAX_KEY_SIZE, 1339 .ivsize = AES_BLOCK_SIZE, 1340 .base = { 1341 .cra_name = "cbc(aes)", 1342 .cra_driver_name = "safexcel-cbc-aes", 1343 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1344 .cra_flags = CRYPTO_ALG_ASYNC | 1345 CRYPTO_ALG_ALLOCATES_MEMORY | 1346 CRYPTO_ALG_KERN_DRIVER_ONLY, 1347 .cra_blocksize = AES_BLOCK_SIZE, 1348 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1349 .cra_alignmask = 0, 1350 .cra_init = safexcel_skcipher_aes_cbc_cra_init, 1351 .cra_exit = safexcel_skcipher_cra_exit, 1352 .cra_module = THIS_MODULE, 1353 }, 1354 }, 1355 }; 1356 1357 static int safexcel_skcipher_aesctr_setkey(struct crypto_skcipher *ctfm, 1358 const u8 *key, unsigned int len) 1359 { 1360 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 1361 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1362 struct safexcel_crypto_priv *priv = ctx->base.priv; 1363 struct crypto_aes_ctx aes __cleanup(aes_zeroize_ctx); 1364 int ret, i; 1365 unsigned int keylen; 1366 1367 /* last 4 bytes of key are the nonce! */ 1368 ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE); 1369 /* exclude the nonce here */ 1370 keylen = len - CTR_RFC3686_NONCE_SIZE; 1371 ret = aes_expandkey(&aes, key, keylen); 1372 if (ret) 1373 return ret; 1374 1375 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 1376 for (i = 0; i < keylen / sizeof(u32); i++) { 1377 if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { 1378 ctx->base.needs_inv = true; 1379 break; 1380 } 1381 } 1382 } 1383 1384 for (i = 0; i < keylen / sizeof(u32); i++) 1385 ctx->key[i] = cpu_to_le32(aes.key_enc[i]); 1386 1387 ctx->key_len = keylen; 1388 1389 return 0; 1390 } 1391 1392 static int safexcel_skcipher_aes_ctr_cra_init(struct crypto_tfm *tfm) 1393 { 1394 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1395 1396 safexcel_skcipher_cra_init(tfm); 1397 ctx->alg = SAFEXCEL_AES; 1398 ctx->blocksz = AES_BLOCK_SIZE; 1399 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; 1400 return 0; 1401 } 1402 1403 struct safexcel_alg_template safexcel_alg_ctr_aes = { 1404 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1405 .algo_mask = SAFEXCEL_ALG_AES, 1406 .alg.skcipher = { 1407 .setkey = safexcel_skcipher_aesctr_setkey, 1408 .encrypt = safexcel_encrypt, 1409 .decrypt = safexcel_decrypt, 1410 /* Add nonce size */ 1411 .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 1412 .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 1413 .ivsize = CTR_RFC3686_IV_SIZE, 1414 .base = { 1415 .cra_name = "rfc3686(ctr(aes))", 1416 .cra_driver_name = "safexcel-ctr-aes", 1417 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1418 .cra_flags = CRYPTO_ALG_ASYNC | 1419 CRYPTO_ALG_ALLOCATES_MEMORY | 1420 CRYPTO_ALG_KERN_DRIVER_ONLY, 1421 .cra_blocksize = 1, 1422 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1423 .cra_alignmask = 0, 1424 .cra_init = safexcel_skcipher_aes_ctr_cra_init, 1425 .cra_exit = safexcel_skcipher_cra_exit, 1426 .cra_module = THIS_MODULE, 1427 }, 1428 }, 1429 }; 1430 1431 static int safexcel_des_setkey(struct crypto_skcipher *ctfm, const u8 *key, 1432 unsigned int len) 1433 { 1434 struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm); 1435 struct safexcel_crypto_priv *priv = ctx->base.priv; 1436 int ret; 1437 1438 ret = verify_skcipher_des_key(ctfm, key); 1439 if (ret) 1440 return ret; 1441 1442 /* if context exits and key changed, need to invalidate it */ 1443 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) 1444 if (memcmp(ctx->key, key, len)) 1445 ctx->base.needs_inv = true; 1446 1447 memcpy(ctx->key, key, len); 1448 ctx->key_len = len; 1449 1450 return 0; 1451 } 1452 1453 static int safexcel_skcipher_des_cbc_cra_init(struct crypto_tfm *tfm) 1454 { 1455 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1456 1457 safexcel_skcipher_cra_init(tfm); 1458 ctx->alg = SAFEXCEL_DES; 1459 ctx->blocksz = DES_BLOCK_SIZE; 1460 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1461 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; 1462 return 0; 1463 } 1464 1465 struct safexcel_alg_template safexcel_alg_cbc_des = { 1466 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1467 .algo_mask = SAFEXCEL_ALG_DES, 1468 .alg.skcipher = { 1469 .setkey = safexcel_des_setkey, 1470 .encrypt = safexcel_encrypt, 1471 .decrypt = safexcel_decrypt, 1472 .min_keysize = DES_KEY_SIZE, 1473 .max_keysize = DES_KEY_SIZE, 1474 .ivsize = DES_BLOCK_SIZE, 1475 .base = { 1476 .cra_name = "cbc(des)", 1477 .cra_driver_name = "safexcel-cbc-des", 1478 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1479 .cra_flags = CRYPTO_ALG_ASYNC | 1480 CRYPTO_ALG_ALLOCATES_MEMORY | 1481 CRYPTO_ALG_KERN_DRIVER_ONLY, 1482 .cra_blocksize = DES_BLOCK_SIZE, 1483 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1484 .cra_alignmask = 0, 1485 .cra_init = safexcel_skcipher_des_cbc_cra_init, 1486 .cra_exit = safexcel_skcipher_cra_exit, 1487 .cra_module = THIS_MODULE, 1488 }, 1489 }, 1490 }; 1491 1492 static int safexcel_skcipher_des_ecb_cra_init(struct crypto_tfm *tfm) 1493 { 1494 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1495 1496 safexcel_skcipher_cra_init(tfm); 1497 ctx->alg = SAFEXCEL_DES; 1498 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; 1499 ctx->blocksz = 0; 1500 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1501 return 0; 1502 } 1503 1504 struct safexcel_alg_template safexcel_alg_ecb_des = { 1505 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1506 .algo_mask = SAFEXCEL_ALG_DES, 1507 .alg.skcipher = { 1508 .setkey = safexcel_des_setkey, 1509 .encrypt = safexcel_encrypt, 1510 .decrypt = safexcel_decrypt, 1511 .min_keysize = DES_KEY_SIZE, 1512 .max_keysize = DES_KEY_SIZE, 1513 .base = { 1514 .cra_name = "ecb(des)", 1515 .cra_driver_name = "safexcel-ecb-des", 1516 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1517 .cra_flags = CRYPTO_ALG_ASYNC | 1518 CRYPTO_ALG_ALLOCATES_MEMORY | 1519 CRYPTO_ALG_KERN_DRIVER_ONLY, 1520 .cra_blocksize = DES_BLOCK_SIZE, 1521 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1522 .cra_alignmask = 0, 1523 .cra_init = safexcel_skcipher_des_ecb_cra_init, 1524 .cra_exit = safexcel_skcipher_cra_exit, 1525 .cra_module = THIS_MODULE, 1526 }, 1527 }, 1528 }; 1529 1530 static int safexcel_des3_ede_setkey(struct crypto_skcipher *ctfm, 1531 const u8 *key, unsigned int len) 1532 { 1533 struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm); 1534 struct safexcel_crypto_priv *priv = ctx->base.priv; 1535 int err; 1536 1537 err = verify_skcipher_des3_key(ctfm, key); 1538 if (err) 1539 return err; 1540 1541 /* if context exits and key changed, need to invalidate it */ 1542 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) 1543 if (memcmp(ctx->key, key, len)) 1544 ctx->base.needs_inv = true; 1545 1546 memcpy(ctx->key, key, len); 1547 ctx->key_len = len; 1548 1549 return 0; 1550 } 1551 1552 static int safexcel_skcipher_des3_cbc_cra_init(struct crypto_tfm *tfm) 1553 { 1554 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1555 1556 safexcel_skcipher_cra_init(tfm); 1557 ctx->alg = SAFEXCEL_3DES; 1558 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 1559 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1560 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; 1561 return 0; 1562 } 1563 1564 struct safexcel_alg_template safexcel_alg_cbc_des3_ede = { 1565 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1566 .algo_mask = SAFEXCEL_ALG_DES, 1567 .alg.skcipher = { 1568 .setkey = safexcel_des3_ede_setkey, 1569 .encrypt = safexcel_encrypt, 1570 .decrypt = safexcel_decrypt, 1571 .min_keysize = DES3_EDE_KEY_SIZE, 1572 .max_keysize = DES3_EDE_KEY_SIZE, 1573 .ivsize = DES3_EDE_BLOCK_SIZE, 1574 .base = { 1575 .cra_name = "cbc(des3_ede)", 1576 .cra_driver_name = "safexcel-cbc-des3_ede", 1577 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1578 .cra_flags = CRYPTO_ALG_ASYNC | 1579 CRYPTO_ALG_ALLOCATES_MEMORY | 1580 CRYPTO_ALG_KERN_DRIVER_ONLY, 1581 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1582 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1583 .cra_alignmask = 0, 1584 .cra_init = safexcel_skcipher_des3_cbc_cra_init, 1585 .cra_exit = safexcel_skcipher_cra_exit, 1586 .cra_module = THIS_MODULE, 1587 }, 1588 }, 1589 }; 1590 1591 static int safexcel_skcipher_des3_ecb_cra_init(struct crypto_tfm *tfm) 1592 { 1593 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1594 1595 safexcel_skcipher_cra_init(tfm); 1596 ctx->alg = SAFEXCEL_3DES; 1597 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; 1598 ctx->blocksz = 0; 1599 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1600 return 0; 1601 } 1602 1603 struct safexcel_alg_template safexcel_alg_ecb_des3_ede = { 1604 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1605 .algo_mask = SAFEXCEL_ALG_DES, 1606 .alg.skcipher = { 1607 .setkey = safexcel_des3_ede_setkey, 1608 .encrypt = safexcel_encrypt, 1609 .decrypt = safexcel_decrypt, 1610 .min_keysize = DES3_EDE_KEY_SIZE, 1611 .max_keysize = DES3_EDE_KEY_SIZE, 1612 .base = { 1613 .cra_name = "ecb(des3_ede)", 1614 .cra_driver_name = "safexcel-ecb-des3_ede", 1615 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1616 .cra_flags = CRYPTO_ALG_ASYNC | 1617 CRYPTO_ALG_ALLOCATES_MEMORY | 1618 CRYPTO_ALG_KERN_DRIVER_ONLY, 1619 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1620 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1621 .cra_alignmask = 0, 1622 .cra_init = safexcel_skcipher_des3_ecb_cra_init, 1623 .cra_exit = safexcel_skcipher_cra_exit, 1624 .cra_module = THIS_MODULE, 1625 }, 1626 }, 1627 }; 1628 1629 static int safexcel_aead_encrypt(struct aead_request *req) 1630 { 1631 struct safexcel_cipher_req *creq = aead_request_ctx(req); 1632 1633 return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); 1634 } 1635 1636 static int safexcel_aead_decrypt(struct aead_request *req) 1637 { 1638 struct safexcel_cipher_req *creq = aead_request_ctx(req); 1639 1640 return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); 1641 } 1642 1643 static int safexcel_aead_cra_init(struct crypto_tfm *tfm) 1644 { 1645 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1646 struct safexcel_alg_template *tmpl = 1647 container_of(tfm->__crt_alg, struct safexcel_alg_template, 1648 alg.aead.base); 1649 1650 crypto_aead_set_reqsize(__crypto_aead_cast(tfm), 1651 sizeof(struct safexcel_cipher_req)); 1652 1653 ctx->base.priv = tmpl->priv; 1654 1655 ctx->alg = SAFEXCEL_AES; /* default */ 1656 ctx->blocksz = AES_BLOCK_SIZE; 1657 ctx->ivmask = EIP197_OPTION_4_TOKEN_IV_CMD; 1658 ctx->ctrinit = 1; 1659 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; /* default */ 1660 ctx->aead = true; 1661 ctx->base.send = safexcel_aead_send; 1662 ctx->base.handle_result = safexcel_aead_handle_result; 1663 return 0; 1664 } 1665 1666 static int safexcel_aead_md5_cra_init(struct crypto_tfm *tfm) 1667 { 1668 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1669 1670 safexcel_aead_cra_init(tfm); 1671 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5; 1672 ctx->state_sz = MD5_DIGEST_SIZE; 1673 return 0; 1674 } 1675 1676 struct safexcel_alg_template safexcel_alg_authenc_hmac_md5_cbc_aes = { 1677 .type = SAFEXCEL_ALG_TYPE_AEAD, 1678 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_MD5, 1679 .alg.aead = { 1680 .setkey = safexcel_aead_setkey, 1681 .encrypt = safexcel_aead_encrypt, 1682 .decrypt = safexcel_aead_decrypt, 1683 .ivsize = AES_BLOCK_SIZE, 1684 .maxauthsize = MD5_DIGEST_SIZE, 1685 .base = { 1686 .cra_name = "authenc(hmac(md5),cbc(aes))", 1687 .cra_driver_name = "safexcel-authenc-hmac-md5-cbc-aes", 1688 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1689 .cra_flags = CRYPTO_ALG_ASYNC | 1690 CRYPTO_ALG_ALLOCATES_MEMORY | 1691 CRYPTO_ALG_KERN_DRIVER_ONLY, 1692 .cra_blocksize = AES_BLOCK_SIZE, 1693 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1694 .cra_alignmask = 0, 1695 .cra_init = safexcel_aead_md5_cra_init, 1696 .cra_exit = safexcel_aead_cra_exit, 1697 .cra_module = THIS_MODULE, 1698 }, 1699 }, 1700 }; 1701 1702 static int safexcel_aead_sha1_cra_init(struct crypto_tfm *tfm) 1703 { 1704 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1705 1706 safexcel_aead_cra_init(tfm); 1707 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1; 1708 ctx->state_sz = SHA1_DIGEST_SIZE; 1709 return 0; 1710 } 1711 1712 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_aes = { 1713 .type = SAFEXCEL_ALG_TYPE_AEAD, 1714 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA1, 1715 .alg.aead = { 1716 .setkey = safexcel_aead_setkey, 1717 .encrypt = safexcel_aead_encrypt, 1718 .decrypt = safexcel_aead_decrypt, 1719 .ivsize = AES_BLOCK_SIZE, 1720 .maxauthsize = SHA1_DIGEST_SIZE, 1721 .base = { 1722 .cra_name = "authenc(hmac(sha1),cbc(aes))", 1723 .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-aes", 1724 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1725 .cra_flags = CRYPTO_ALG_ASYNC | 1726 CRYPTO_ALG_ALLOCATES_MEMORY | 1727 CRYPTO_ALG_KERN_DRIVER_ONLY, 1728 .cra_blocksize = AES_BLOCK_SIZE, 1729 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1730 .cra_alignmask = 0, 1731 .cra_init = safexcel_aead_sha1_cra_init, 1732 .cra_exit = safexcel_aead_cra_exit, 1733 .cra_module = THIS_MODULE, 1734 }, 1735 }, 1736 }; 1737 1738 static int safexcel_aead_sha256_cra_init(struct crypto_tfm *tfm) 1739 { 1740 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1741 1742 safexcel_aead_cra_init(tfm); 1743 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256; 1744 ctx->state_sz = SHA256_DIGEST_SIZE; 1745 return 0; 1746 } 1747 1748 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_aes = { 1749 .type = SAFEXCEL_ALG_TYPE_AEAD, 1750 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256, 1751 .alg.aead = { 1752 .setkey = safexcel_aead_setkey, 1753 .encrypt = safexcel_aead_encrypt, 1754 .decrypt = safexcel_aead_decrypt, 1755 .ivsize = AES_BLOCK_SIZE, 1756 .maxauthsize = SHA256_DIGEST_SIZE, 1757 .base = { 1758 .cra_name = "authenc(hmac(sha256),cbc(aes))", 1759 .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-aes", 1760 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1761 .cra_flags = CRYPTO_ALG_ASYNC | 1762 CRYPTO_ALG_ALLOCATES_MEMORY | 1763 CRYPTO_ALG_KERN_DRIVER_ONLY, 1764 .cra_blocksize = AES_BLOCK_SIZE, 1765 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1766 .cra_alignmask = 0, 1767 .cra_init = safexcel_aead_sha256_cra_init, 1768 .cra_exit = safexcel_aead_cra_exit, 1769 .cra_module = THIS_MODULE, 1770 }, 1771 }, 1772 }; 1773 1774 static int safexcel_aead_sha224_cra_init(struct crypto_tfm *tfm) 1775 { 1776 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1777 1778 safexcel_aead_cra_init(tfm); 1779 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224; 1780 ctx->state_sz = SHA256_DIGEST_SIZE; 1781 return 0; 1782 } 1783 1784 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_aes = { 1785 .type = SAFEXCEL_ALG_TYPE_AEAD, 1786 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256, 1787 .alg.aead = { 1788 .setkey = safexcel_aead_setkey, 1789 .encrypt = safexcel_aead_encrypt, 1790 .decrypt = safexcel_aead_decrypt, 1791 .ivsize = AES_BLOCK_SIZE, 1792 .maxauthsize = SHA224_DIGEST_SIZE, 1793 .base = { 1794 .cra_name = "authenc(hmac(sha224),cbc(aes))", 1795 .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-aes", 1796 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1797 .cra_flags = CRYPTO_ALG_ASYNC | 1798 CRYPTO_ALG_ALLOCATES_MEMORY | 1799 CRYPTO_ALG_KERN_DRIVER_ONLY, 1800 .cra_blocksize = AES_BLOCK_SIZE, 1801 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1802 .cra_alignmask = 0, 1803 .cra_init = safexcel_aead_sha224_cra_init, 1804 .cra_exit = safexcel_aead_cra_exit, 1805 .cra_module = THIS_MODULE, 1806 }, 1807 }, 1808 }; 1809 1810 static int safexcel_aead_sha512_cra_init(struct crypto_tfm *tfm) 1811 { 1812 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1813 1814 safexcel_aead_cra_init(tfm); 1815 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512; 1816 ctx->state_sz = SHA512_DIGEST_SIZE; 1817 return 0; 1818 } 1819 1820 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_aes = { 1821 .type = SAFEXCEL_ALG_TYPE_AEAD, 1822 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512, 1823 .alg.aead = { 1824 .setkey = safexcel_aead_setkey, 1825 .encrypt = safexcel_aead_encrypt, 1826 .decrypt = safexcel_aead_decrypt, 1827 .ivsize = AES_BLOCK_SIZE, 1828 .maxauthsize = SHA512_DIGEST_SIZE, 1829 .base = { 1830 .cra_name = "authenc(hmac(sha512),cbc(aes))", 1831 .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-aes", 1832 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1833 .cra_flags = CRYPTO_ALG_ASYNC | 1834 CRYPTO_ALG_ALLOCATES_MEMORY | 1835 CRYPTO_ALG_KERN_DRIVER_ONLY, 1836 .cra_blocksize = AES_BLOCK_SIZE, 1837 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1838 .cra_alignmask = 0, 1839 .cra_init = safexcel_aead_sha512_cra_init, 1840 .cra_exit = safexcel_aead_cra_exit, 1841 .cra_module = THIS_MODULE, 1842 }, 1843 }, 1844 }; 1845 1846 static int safexcel_aead_sha384_cra_init(struct crypto_tfm *tfm) 1847 { 1848 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1849 1850 safexcel_aead_cra_init(tfm); 1851 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384; 1852 ctx->state_sz = SHA512_DIGEST_SIZE; 1853 return 0; 1854 } 1855 1856 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_aes = { 1857 .type = SAFEXCEL_ALG_TYPE_AEAD, 1858 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512, 1859 .alg.aead = { 1860 .setkey = safexcel_aead_setkey, 1861 .encrypt = safexcel_aead_encrypt, 1862 .decrypt = safexcel_aead_decrypt, 1863 .ivsize = AES_BLOCK_SIZE, 1864 .maxauthsize = SHA384_DIGEST_SIZE, 1865 .base = { 1866 .cra_name = "authenc(hmac(sha384),cbc(aes))", 1867 .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-aes", 1868 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1869 .cra_flags = CRYPTO_ALG_ASYNC | 1870 CRYPTO_ALG_ALLOCATES_MEMORY | 1871 CRYPTO_ALG_KERN_DRIVER_ONLY, 1872 .cra_blocksize = AES_BLOCK_SIZE, 1873 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1874 .cra_alignmask = 0, 1875 .cra_init = safexcel_aead_sha384_cra_init, 1876 .cra_exit = safexcel_aead_cra_exit, 1877 .cra_module = THIS_MODULE, 1878 }, 1879 }, 1880 }; 1881 1882 static int safexcel_aead_md5_des3_cra_init(struct crypto_tfm *tfm) 1883 { 1884 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1885 1886 safexcel_aead_md5_cra_init(tfm); 1887 ctx->alg = SAFEXCEL_3DES; /* override default */ 1888 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 1889 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1890 return 0; 1891 } 1892 1893 struct safexcel_alg_template safexcel_alg_authenc_hmac_md5_cbc_des3_ede = { 1894 .type = SAFEXCEL_ALG_TYPE_AEAD, 1895 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_MD5, 1896 .alg.aead = { 1897 .setkey = safexcel_aead_setkey, 1898 .encrypt = safexcel_aead_encrypt, 1899 .decrypt = safexcel_aead_decrypt, 1900 .ivsize = DES3_EDE_BLOCK_SIZE, 1901 .maxauthsize = MD5_DIGEST_SIZE, 1902 .base = { 1903 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 1904 .cra_driver_name = "safexcel-authenc-hmac-md5-cbc-des3_ede", 1905 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1906 .cra_flags = CRYPTO_ALG_ASYNC | 1907 CRYPTO_ALG_ALLOCATES_MEMORY | 1908 CRYPTO_ALG_KERN_DRIVER_ONLY, 1909 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1910 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1911 .cra_alignmask = 0, 1912 .cra_init = safexcel_aead_md5_des3_cra_init, 1913 .cra_exit = safexcel_aead_cra_exit, 1914 .cra_module = THIS_MODULE, 1915 }, 1916 }, 1917 }; 1918 1919 static int safexcel_aead_sha1_des3_cra_init(struct crypto_tfm *tfm) 1920 { 1921 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1922 1923 safexcel_aead_sha1_cra_init(tfm); 1924 ctx->alg = SAFEXCEL_3DES; /* override default */ 1925 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 1926 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1927 return 0; 1928 } 1929 1930 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des3_ede = { 1931 .type = SAFEXCEL_ALG_TYPE_AEAD, 1932 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA1, 1933 .alg.aead = { 1934 .setkey = safexcel_aead_setkey, 1935 .encrypt = safexcel_aead_encrypt, 1936 .decrypt = safexcel_aead_decrypt, 1937 .ivsize = DES3_EDE_BLOCK_SIZE, 1938 .maxauthsize = SHA1_DIGEST_SIZE, 1939 .base = { 1940 .cra_name = "authenc(hmac(sha1),cbc(des3_ede))", 1941 .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-des3_ede", 1942 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1943 .cra_flags = CRYPTO_ALG_ASYNC | 1944 CRYPTO_ALG_ALLOCATES_MEMORY | 1945 CRYPTO_ALG_KERN_DRIVER_ONLY, 1946 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1947 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1948 .cra_alignmask = 0, 1949 .cra_init = safexcel_aead_sha1_des3_cra_init, 1950 .cra_exit = safexcel_aead_cra_exit, 1951 .cra_module = THIS_MODULE, 1952 }, 1953 }, 1954 }; 1955 1956 static int safexcel_aead_sha256_des3_cra_init(struct crypto_tfm *tfm) 1957 { 1958 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1959 1960 safexcel_aead_sha256_cra_init(tfm); 1961 ctx->alg = SAFEXCEL_3DES; /* override default */ 1962 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 1963 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1964 return 0; 1965 } 1966 1967 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des3_ede = { 1968 .type = SAFEXCEL_ALG_TYPE_AEAD, 1969 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, 1970 .alg.aead = { 1971 .setkey = safexcel_aead_setkey, 1972 .encrypt = safexcel_aead_encrypt, 1973 .decrypt = safexcel_aead_decrypt, 1974 .ivsize = DES3_EDE_BLOCK_SIZE, 1975 .maxauthsize = SHA256_DIGEST_SIZE, 1976 .base = { 1977 .cra_name = "authenc(hmac(sha256),cbc(des3_ede))", 1978 .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-des3_ede", 1979 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1980 .cra_flags = CRYPTO_ALG_ASYNC | 1981 CRYPTO_ALG_ALLOCATES_MEMORY | 1982 CRYPTO_ALG_KERN_DRIVER_ONLY, 1983 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1984 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1985 .cra_alignmask = 0, 1986 .cra_init = safexcel_aead_sha256_des3_cra_init, 1987 .cra_exit = safexcel_aead_cra_exit, 1988 .cra_module = THIS_MODULE, 1989 }, 1990 }, 1991 }; 1992 1993 static int safexcel_aead_sha224_des3_cra_init(struct crypto_tfm *tfm) 1994 { 1995 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1996 1997 safexcel_aead_sha224_cra_init(tfm); 1998 ctx->alg = SAFEXCEL_3DES; /* override default */ 1999 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 2000 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2001 return 0; 2002 } 2003 2004 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des3_ede = { 2005 .type = SAFEXCEL_ALG_TYPE_AEAD, 2006 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, 2007 .alg.aead = { 2008 .setkey = safexcel_aead_setkey, 2009 .encrypt = safexcel_aead_encrypt, 2010 .decrypt = safexcel_aead_decrypt, 2011 .ivsize = DES3_EDE_BLOCK_SIZE, 2012 .maxauthsize = SHA224_DIGEST_SIZE, 2013 .base = { 2014 .cra_name = "authenc(hmac(sha224),cbc(des3_ede))", 2015 .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-des3_ede", 2016 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2017 .cra_flags = CRYPTO_ALG_ASYNC | 2018 CRYPTO_ALG_ALLOCATES_MEMORY | 2019 CRYPTO_ALG_KERN_DRIVER_ONLY, 2020 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2021 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2022 .cra_alignmask = 0, 2023 .cra_init = safexcel_aead_sha224_des3_cra_init, 2024 .cra_exit = safexcel_aead_cra_exit, 2025 .cra_module = THIS_MODULE, 2026 }, 2027 }, 2028 }; 2029 2030 static int safexcel_aead_sha512_des3_cra_init(struct crypto_tfm *tfm) 2031 { 2032 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2033 2034 safexcel_aead_sha512_cra_init(tfm); 2035 ctx->alg = SAFEXCEL_3DES; /* override default */ 2036 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 2037 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2038 return 0; 2039 } 2040 2041 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des3_ede = { 2042 .type = SAFEXCEL_ALG_TYPE_AEAD, 2043 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, 2044 .alg.aead = { 2045 .setkey = safexcel_aead_setkey, 2046 .encrypt = safexcel_aead_encrypt, 2047 .decrypt = safexcel_aead_decrypt, 2048 .ivsize = DES3_EDE_BLOCK_SIZE, 2049 .maxauthsize = SHA512_DIGEST_SIZE, 2050 .base = { 2051 .cra_name = "authenc(hmac(sha512),cbc(des3_ede))", 2052 .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-des3_ede", 2053 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2054 .cra_flags = CRYPTO_ALG_ASYNC | 2055 CRYPTO_ALG_ALLOCATES_MEMORY | 2056 CRYPTO_ALG_KERN_DRIVER_ONLY, 2057 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2058 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2059 .cra_alignmask = 0, 2060 .cra_init = safexcel_aead_sha512_des3_cra_init, 2061 .cra_exit = safexcel_aead_cra_exit, 2062 .cra_module = THIS_MODULE, 2063 }, 2064 }, 2065 }; 2066 2067 static int safexcel_aead_sha384_des3_cra_init(struct crypto_tfm *tfm) 2068 { 2069 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2070 2071 safexcel_aead_sha384_cra_init(tfm); 2072 ctx->alg = SAFEXCEL_3DES; /* override default */ 2073 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 2074 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2075 return 0; 2076 } 2077 2078 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des3_ede = { 2079 .type = SAFEXCEL_ALG_TYPE_AEAD, 2080 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, 2081 .alg.aead = { 2082 .setkey = safexcel_aead_setkey, 2083 .encrypt = safexcel_aead_encrypt, 2084 .decrypt = safexcel_aead_decrypt, 2085 .ivsize = DES3_EDE_BLOCK_SIZE, 2086 .maxauthsize = SHA384_DIGEST_SIZE, 2087 .base = { 2088 .cra_name = "authenc(hmac(sha384),cbc(des3_ede))", 2089 .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-des3_ede", 2090 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2091 .cra_flags = CRYPTO_ALG_ASYNC | 2092 CRYPTO_ALG_ALLOCATES_MEMORY | 2093 CRYPTO_ALG_KERN_DRIVER_ONLY, 2094 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2095 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2096 .cra_alignmask = 0, 2097 .cra_init = safexcel_aead_sha384_des3_cra_init, 2098 .cra_exit = safexcel_aead_cra_exit, 2099 .cra_module = THIS_MODULE, 2100 }, 2101 }, 2102 }; 2103 2104 static int safexcel_aead_md5_des_cra_init(struct crypto_tfm *tfm) 2105 { 2106 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2107 2108 safexcel_aead_md5_cra_init(tfm); 2109 ctx->alg = SAFEXCEL_DES; /* override default */ 2110 ctx->blocksz = DES_BLOCK_SIZE; 2111 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2112 return 0; 2113 } 2114 2115 struct safexcel_alg_template safexcel_alg_authenc_hmac_md5_cbc_des = { 2116 .type = SAFEXCEL_ALG_TYPE_AEAD, 2117 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_MD5, 2118 .alg.aead = { 2119 .setkey = safexcel_aead_setkey, 2120 .encrypt = safexcel_aead_encrypt, 2121 .decrypt = safexcel_aead_decrypt, 2122 .ivsize = DES_BLOCK_SIZE, 2123 .maxauthsize = MD5_DIGEST_SIZE, 2124 .base = { 2125 .cra_name = "authenc(hmac(md5),cbc(des))", 2126 .cra_driver_name = "safexcel-authenc-hmac-md5-cbc-des", 2127 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2128 .cra_flags = CRYPTO_ALG_ASYNC | 2129 CRYPTO_ALG_ALLOCATES_MEMORY | 2130 CRYPTO_ALG_KERN_DRIVER_ONLY, 2131 .cra_blocksize = DES_BLOCK_SIZE, 2132 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2133 .cra_alignmask = 0, 2134 .cra_init = safexcel_aead_md5_des_cra_init, 2135 .cra_exit = safexcel_aead_cra_exit, 2136 .cra_module = THIS_MODULE, 2137 }, 2138 }, 2139 }; 2140 2141 static int safexcel_aead_sha1_des_cra_init(struct crypto_tfm *tfm) 2142 { 2143 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2144 2145 safexcel_aead_sha1_cra_init(tfm); 2146 ctx->alg = SAFEXCEL_DES; /* override default */ 2147 ctx->blocksz = DES_BLOCK_SIZE; 2148 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2149 return 0; 2150 } 2151 2152 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des = { 2153 .type = SAFEXCEL_ALG_TYPE_AEAD, 2154 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA1, 2155 .alg.aead = { 2156 .setkey = safexcel_aead_setkey, 2157 .encrypt = safexcel_aead_encrypt, 2158 .decrypt = safexcel_aead_decrypt, 2159 .ivsize = DES_BLOCK_SIZE, 2160 .maxauthsize = SHA1_DIGEST_SIZE, 2161 .base = { 2162 .cra_name = "authenc(hmac(sha1),cbc(des))", 2163 .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-des", 2164 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2165 .cra_flags = CRYPTO_ALG_ASYNC | 2166 CRYPTO_ALG_ALLOCATES_MEMORY | 2167 CRYPTO_ALG_KERN_DRIVER_ONLY, 2168 .cra_blocksize = DES_BLOCK_SIZE, 2169 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2170 .cra_alignmask = 0, 2171 .cra_init = safexcel_aead_sha1_des_cra_init, 2172 .cra_exit = safexcel_aead_cra_exit, 2173 .cra_module = THIS_MODULE, 2174 }, 2175 }, 2176 }; 2177 2178 static int safexcel_aead_sha256_des_cra_init(struct crypto_tfm *tfm) 2179 { 2180 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2181 2182 safexcel_aead_sha256_cra_init(tfm); 2183 ctx->alg = SAFEXCEL_DES; /* override default */ 2184 ctx->blocksz = DES_BLOCK_SIZE; 2185 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2186 return 0; 2187 } 2188 2189 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des = { 2190 .type = SAFEXCEL_ALG_TYPE_AEAD, 2191 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, 2192 .alg.aead = { 2193 .setkey = safexcel_aead_setkey, 2194 .encrypt = safexcel_aead_encrypt, 2195 .decrypt = safexcel_aead_decrypt, 2196 .ivsize = DES_BLOCK_SIZE, 2197 .maxauthsize = SHA256_DIGEST_SIZE, 2198 .base = { 2199 .cra_name = "authenc(hmac(sha256),cbc(des))", 2200 .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-des", 2201 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2202 .cra_flags = CRYPTO_ALG_ASYNC | 2203 CRYPTO_ALG_ALLOCATES_MEMORY | 2204 CRYPTO_ALG_KERN_DRIVER_ONLY, 2205 .cra_blocksize = DES_BLOCK_SIZE, 2206 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2207 .cra_alignmask = 0, 2208 .cra_init = safexcel_aead_sha256_des_cra_init, 2209 .cra_exit = safexcel_aead_cra_exit, 2210 .cra_module = THIS_MODULE, 2211 }, 2212 }, 2213 }; 2214 2215 static int safexcel_aead_sha224_des_cra_init(struct crypto_tfm *tfm) 2216 { 2217 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2218 2219 safexcel_aead_sha224_cra_init(tfm); 2220 ctx->alg = SAFEXCEL_DES; /* override default */ 2221 ctx->blocksz = DES_BLOCK_SIZE; 2222 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2223 return 0; 2224 } 2225 2226 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des = { 2227 .type = SAFEXCEL_ALG_TYPE_AEAD, 2228 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, 2229 .alg.aead = { 2230 .setkey = safexcel_aead_setkey, 2231 .encrypt = safexcel_aead_encrypt, 2232 .decrypt = safexcel_aead_decrypt, 2233 .ivsize = DES_BLOCK_SIZE, 2234 .maxauthsize = SHA224_DIGEST_SIZE, 2235 .base = { 2236 .cra_name = "authenc(hmac(sha224),cbc(des))", 2237 .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-des", 2238 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2239 .cra_flags = CRYPTO_ALG_ASYNC | 2240 CRYPTO_ALG_ALLOCATES_MEMORY | 2241 CRYPTO_ALG_KERN_DRIVER_ONLY, 2242 .cra_blocksize = DES_BLOCK_SIZE, 2243 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2244 .cra_alignmask = 0, 2245 .cra_init = safexcel_aead_sha224_des_cra_init, 2246 .cra_exit = safexcel_aead_cra_exit, 2247 .cra_module = THIS_MODULE, 2248 }, 2249 }, 2250 }; 2251 2252 static int safexcel_aead_sha512_des_cra_init(struct crypto_tfm *tfm) 2253 { 2254 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2255 2256 safexcel_aead_sha512_cra_init(tfm); 2257 ctx->alg = SAFEXCEL_DES; /* override default */ 2258 ctx->blocksz = DES_BLOCK_SIZE; 2259 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2260 return 0; 2261 } 2262 2263 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des = { 2264 .type = SAFEXCEL_ALG_TYPE_AEAD, 2265 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, 2266 .alg.aead = { 2267 .setkey = safexcel_aead_setkey, 2268 .encrypt = safexcel_aead_encrypt, 2269 .decrypt = safexcel_aead_decrypt, 2270 .ivsize = DES_BLOCK_SIZE, 2271 .maxauthsize = SHA512_DIGEST_SIZE, 2272 .base = { 2273 .cra_name = "authenc(hmac(sha512),cbc(des))", 2274 .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-des", 2275 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2276 .cra_flags = CRYPTO_ALG_ASYNC | 2277 CRYPTO_ALG_ALLOCATES_MEMORY | 2278 CRYPTO_ALG_KERN_DRIVER_ONLY, 2279 .cra_blocksize = DES_BLOCK_SIZE, 2280 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2281 .cra_alignmask = 0, 2282 .cra_init = safexcel_aead_sha512_des_cra_init, 2283 .cra_exit = safexcel_aead_cra_exit, 2284 .cra_module = THIS_MODULE, 2285 }, 2286 }, 2287 }; 2288 2289 static int safexcel_aead_sha384_des_cra_init(struct crypto_tfm *tfm) 2290 { 2291 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2292 2293 safexcel_aead_sha384_cra_init(tfm); 2294 ctx->alg = SAFEXCEL_DES; /* override default */ 2295 ctx->blocksz = DES_BLOCK_SIZE; 2296 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2297 return 0; 2298 } 2299 2300 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des = { 2301 .type = SAFEXCEL_ALG_TYPE_AEAD, 2302 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, 2303 .alg.aead = { 2304 .setkey = safexcel_aead_setkey, 2305 .encrypt = safexcel_aead_encrypt, 2306 .decrypt = safexcel_aead_decrypt, 2307 .ivsize = DES_BLOCK_SIZE, 2308 .maxauthsize = SHA384_DIGEST_SIZE, 2309 .base = { 2310 .cra_name = "authenc(hmac(sha384),cbc(des))", 2311 .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-des", 2312 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2313 .cra_flags = CRYPTO_ALG_ASYNC | 2314 CRYPTO_ALG_ALLOCATES_MEMORY | 2315 CRYPTO_ALG_KERN_DRIVER_ONLY, 2316 .cra_blocksize = DES_BLOCK_SIZE, 2317 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2318 .cra_alignmask = 0, 2319 .cra_init = safexcel_aead_sha384_des_cra_init, 2320 .cra_exit = safexcel_aead_cra_exit, 2321 .cra_module = THIS_MODULE, 2322 }, 2323 }, 2324 }; 2325 2326 static int safexcel_aead_md5_ctr_cra_init(struct crypto_tfm *tfm) 2327 { 2328 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2329 2330 safexcel_aead_md5_cra_init(tfm); 2331 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2332 return 0; 2333 } 2334 2335 struct safexcel_alg_template safexcel_alg_authenc_hmac_md5_ctr_aes = { 2336 .type = SAFEXCEL_ALG_TYPE_AEAD, 2337 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_MD5, 2338 .alg.aead = { 2339 .setkey = safexcel_aead_setkey, 2340 .encrypt = safexcel_aead_encrypt, 2341 .decrypt = safexcel_aead_decrypt, 2342 .ivsize = CTR_RFC3686_IV_SIZE, 2343 .maxauthsize = MD5_DIGEST_SIZE, 2344 .base = { 2345 .cra_name = "authenc(hmac(md5),rfc3686(ctr(aes)))", 2346 .cra_driver_name = "safexcel-authenc-hmac-md5-ctr-aes", 2347 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2348 .cra_flags = CRYPTO_ALG_ASYNC | 2349 CRYPTO_ALG_ALLOCATES_MEMORY | 2350 CRYPTO_ALG_KERN_DRIVER_ONLY, 2351 .cra_blocksize = 1, 2352 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2353 .cra_alignmask = 0, 2354 .cra_init = safexcel_aead_md5_ctr_cra_init, 2355 .cra_exit = safexcel_aead_cra_exit, 2356 .cra_module = THIS_MODULE, 2357 }, 2358 }, 2359 }; 2360 2361 static int safexcel_aead_sha1_ctr_cra_init(struct crypto_tfm *tfm) 2362 { 2363 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2364 2365 safexcel_aead_sha1_cra_init(tfm); 2366 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2367 return 0; 2368 } 2369 2370 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_ctr_aes = { 2371 .type = SAFEXCEL_ALG_TYPE_AEAD, 2372 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA1, 2373 .alg.aead = { 2374 .setkey = safexcel_aead_setkey, 2375 .encrypt = safexcel_aead_encrypt, 2376 .decrypt = safexcel_aead_decrypt, 2377 .ivsize = CTR_RFC3686_IV_SIZE, 2378 .maxauthsize = SHA1_DIGEST_SIZE, 2379 .base = { 2380 .cra_name = "authenc(hmac(sha1),rfc3686(ctr(aes)))", 2381 .cra_driver_name = "safexcel-authenc-hmac-sha1-ctr-aes", 2382 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2383 .cra_flags = CRYPTO_ALG_ASYNC | 2384 CRYPTO_ALG_ALLOCATES_MEMORY | 2385 CRYPTO_ALG_KERN_DRIVER_ONLY, 2386 .cra_blocksize = 1, 2387 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2388 .cra_alignmask = 0, 2389 .cra_init = safexcel_aead_sha1_ctr_cra_init, 2390 .cra_exit = safexcel_aead_cra_exit, 2391 .cra_module = THIS_MODULE, 2392 }, 2393 }, 2394 }; 2395 2396 static int safexcel_aead_sha256_ctr_cra_init(struct crypto_tfm *tfm) 2397 { 2398 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2399 2400 safexcel_aead_sha256_cra_init(tfm); 2401 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2402 return 0; 2403 } 2404 2405 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_ctr_aes = { 2406 .type = SAFEXCEL_ALG_TYPE_AEAD, 2407 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256, 2408 .alg.aead = { 2409 .setkey = safexcel_aead_setkey, 2410 .encrypt = safexcel_aead_encrypt, 2411 .decrypt = safexcel_aead_decrypt, 2412 .ivsize = CTR_RFC3686_IV_SIZE, 2413 .maxauthsize = SHA256_DIGEST_SIZE, 2414 .base = { 2415 .cra_name = "authenc(hmac(sha256),rfc3686(ctr(aes)))", 2416 .cra_driver_name = "safexcel-authenc-hmac-sha256-ctr-aes", 2417 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2418 .cra_flags = CRYPTO_ALG_ASYNC | 2419 CRYPTO_ALG_ALLOCATES_MEMORY | 2420 CRYPTO_ALG_KERN_DRIVER_ONLY, 2421 .cra_blocksize = 1, 2422 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2423 .cra_alignmask = 0, 2424 .cra_init = safexcel_aead_sha256_ctr_cra_init, 2425 .cra_exit = safexcel_aead_cra_exit, 2426 .cra_module = THIS_MODULE, 2427 }, 2428 }, 2429 }; 2430 2431 static int safexcel_aead_sha224_ctr_cra_init(struct crypto_tfm *tfm) 2432 { 2433 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2434 2435 safexcel_aead_sha224_cra_init(tfm); 2436 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2437 return 0; 2438 } 2439 2440 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_ctr_aes = { 2441 .type = SAFEXCEL_ALG_TYPE_AEAD, 2442 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256, 2443 .alg.aead = { 2444 .setkey = safexcel_aead_setkey, 2445 .encrypt = safexcel_aead_encrypt, 2446 .decrypt = safexcel_aead_decrypt, 2447 .ivsize = CTR_RFC3686_IV_SIZE, 2448 .maxauthsize = SHA224_DIGEST_SIZE, 2449 .base = { 2450 .cra_name = "authenc(hmac(sha224),rfc3686(ctr(aes)))", 2451 .cra_driver_name = "safexcel-authenc-hmac-sha224-ctr-aes", 2452 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2453 .cra_flags = CRYPTO_ALG_ASYNC | 2454 CRYPTO_ALG_ALLOCATES_MEMORY | 2455 CRYPTO_ALG_KERN_DRIVER_ONLY, 2456 .cra_blocksize = 1, 2457 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2458 .cra_alignmask = 0, 2459 .cra_init = safexcel_aead_sha224_ctr_cra_init, 2460 .cra_exit = safexcel_aead_cra_exit, 2461 .cra_module = THIS_MODULE, 2462 }, 2463 }, 2464 }; 2465 2466 static int safexcel_aead_sha512_ctr_cra_init(struct crypto_tfm *tfm) 2467 { 2468 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2469 2470 safexcel_aead_sha512_cra_init(tfm); 2471 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2472 return 0; 2473 } 2474 2475 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_ctr_aes = { 2476 .type = SAFEXCEL_ALG_TYPE_AEAD, 2477 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512, 2478 .alg.aead = { 2479 .setkey = safexcel_aead_setkey, 2480 .encrypt = safexcel_aead_encrypt, 2481 .decrypt = safexcel_aead_decrypt, 2482 .ivsize = CTR_RFC3686_IV_SIZE, 2483 .maxauthsize = SHA512_DIGEST_SIZE, 2484 .base = { 2485 .cra_name = "authenc(hmac(sha512),rfc3686(ctr(aes)))", 2486 .cra_driver_name = "safexcel-authenc-hmac-sha512-ctr-aes", 2487 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2488 .cra_flags = CRYPTO_ALG_ASYNC | 2489 CRYPTO_ALG_ALLOCATES_MEMORY | 2490 CRYPTO_ALG_KERN_DRIVER_ONLY, 2491 .cra_blocksize = 1, 2492 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2493 .cra_alignmask = 0, 2494 .cra_init = safexcel_aead_sha512_ctr_cra_init, 2495 .cra_exit = safexcel_aead_cra_exit, 2496 .cra_module = THIS_MODULE, 2497 }, 2498 }, 2499 }; 2500 2501 static int safexcel_aead_sha384_ctr_cra_init(struct crypto_tfm *tfm) 2502 { 2503 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2504 2505 safexcel_aead_sha384_cra_init(tfm); 2506 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2507 return 0; 2508 } 2509 2510 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_ctr_aes = { 2511 .type = SAFEXCEL_ALG_TYPE_AEAD, 2512 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512, 2513 .alg.aead = { 2514 .setkey = safexcel_aead_setkey, 2515 .encrypt = safexcel_aead_encrypt, 2516 .decrypt = safexcel_aead_decrypt, 2517 .ivsize = CTR_RFC3686_IV_SIZE, 2518 .maxauthsize = SHA384_DIGEST_SIZE, 2519 .base = { 2520 .cra_name = "authenc(hmac(sha384),rfc3686(ctr(aes)))", 2521 .cra_driver_name = "safexcel-authenc-hmac-sha384-ctr-aes", 2522 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2523 .cra_flags = CRYPTO_ALG_ASYNC | 2524 CRYPTO_ALG_ALLOCATES_MEMORY | 2525 CRYPTO_ALG_KERN_DRIVER_ONLY, 2526 .cra_blocksize = 1, 2527 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2528 .cra_alignmask = 0, 2529 .cra_init = safexcel_aead_sha384_ctr_cra_init, 2530 .cra_exit = safexcel_aead_cra_exit, 2531 .cra_module = THIS_MODULE, 2532 }, 2533 }, 2534 }; 2535 2536 static int safexcel_skcipher_aesxts_setkey(struct crypto_skcipher *ctfm, 2537 const u8 *key, unsigned int len) 2538 { 2539 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 2540 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2541 struct safexcel_crypto_priv *priv = ctx->base.priv; 2542 struct crypto_aes_ctx aes __cleanup(aes_zeroize_ctx); 2543 int ret, i; 2544 unsigned int keylen; 2545 2546 /* Check for illegal XTS keys */ 2547 ret = xts_verify_key(ctfm, key, len); 2548 if (ret) 2549 return ret; 2550 2551 /* Only half of the key data is cipher key */ 2552 keylen = (len >> 1); 2553 ret = aes_expandkey(&aes, key, keylen); 2554 if (ret) 2555 return ret; 2556 2557 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2558 for (i = 0; i < keylen / sizeof(u32); i++) { 2559 if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { 2560 ctx->base.needs_inv = true; 2561 break; 2562 } 2563 } 2564 } 2565 2566 for (i = 0; i < keylen / sizeof(u32); i++) 2567 ctx->key[i] = cpu_to_le32(aes.key_enc[i]); 2568 2569 /* The other half is the tweak key */ 2570 ret = aes_expandkey(&aes, (u8 *)(key + keylen), keylen); 2571 if (ret) 2572 return ret; 2573 2574 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2575 for (i = 0; i < keylen / sizeof(u32); i++) { 2576 if (le32_to_cpu(ctx->key[i + keylen / sizeof(u32)]) != 2577 aes.key_enc[i]) { 2578 ctx->base.needs_inv = true; 2579 break; 2580 } 2581 } 2582 } 2583 2584 for (i = 0; i < keylen / sizeof(u32); i++) 2585 ctx->key[i + keylen / sizeof(u32)] = 2586 cpu_to_le32(aes.key_enc[i]); 2587 2588 ctx->key_len = keylen << 1; 2589 2590 return 0; 2591 } 2592 2593 static int safexcel_skcipher_aes_xts_cra_init(struct crypto_tfm *tfm) 2594 { 2595 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2596 2597 safexcel_skcipher_cra_init(tfm); 2598 ctx->alg = SAFEXCEL_AES; 2599 ctx->blocksz = AES_BLOCK_SIZE; 2600 ctx->xts = 1; 2601 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XTS; 2602 return 0; 2603 } 2604 2605 static int safexcel_encrypt_xts(struct skcipher_request *req) 2606 { 2607 if (req->cryptlen < XTS_BLOCK_SIZE) 2608 return -EINVAL; 2609 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 2610 SAFEXCEL_ENCRYPT); 2611 } 2612 2613 static int safexcel_decrypt_xts(struct skcipher_request *req) 2614 { 2615 if (req->cryptlen < XTS_BLOCK_SIZE) 2616 return -EINVAL; 2617 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 2618 SAFEXCEL_DECRYPT); 2619 } 2620 2621 struct safexcel_alg_template safexcel_alg_xts_aes = { 2622 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 2623 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_AES_XTS, 2624 .alg.skcipher = { 2625 .setkey = safexcel_skcipher_aesxts_setkey, 2626 .encrypt = safexcel_encrypt_xts, 2627 .decrypt = safexcel_decrypt_xts, 2628 /* XTS actually uses 2 AES keys glued together */ 2629 .min_keysize = AES_MIN_KEY_SIZE * 2, 2630 .max_keysize = AES_MAX_KEY_SIZE * 2, 2631 .ivsize = XTS_BLOCK_SIZE, 2632 .base = { 2633 .cra_name = "xts(aes)", 2634 .cra_driver_name = "safexcel-xts-aes", 2635 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2636 .cra_flags = CRYPTO_ALG_ASYNC | 2637 CRYPTO_ALG_ALLOCATES_MEMORY | 2638 CRYPTO_ALG_KERN_DRIVER_ONLY, 2639 .cra_blocksize = XTS_BLOCK_SIZE, 2640 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2641 .cra_alignmask = 0, 2642 .cra_init = safexcel_skcipher_aes_xts_cra_init, 2643 .cra_exit = safexcel_skcipher_cra_exit, 2644 .cra_module = THIS_MODULE, 2645 }, 2646 }, 2647 }; 2648 2649 static int safexcel_aead_gcm_setkey(struct crypto_aead *ctfm, const u8 *key, 2650 unsigned int len) 2651 { 2652 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 2653 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2654 struct safexcel_crypto_priv *priv = ctx->base.priv; 2655 struct aes_enckey aes; 2656 u32 hashkey[AES_BLOCK_SIZE >> 2]; 2657 int ret, i; 2658 2659 ret = aes_prepareenckey(&aes, key, len); 2660 if (ret) 2661 return ret; 2662 2663 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2664 for (i = 0; i < len / sizeof(u32); i++) { 2665 if (ctx->key[i] != get_unaligned((__le32 *)key + i)) { 2666 ctx->base.needs_inv = true; 2667 break; 2668 } 2669 } 2670 } 2671 2672 for (i = 0; i < len / sizeof(u32); i++) 2673 ctx->key[i] = get_unaligned((__le32 *)key + i); 2674 2675 ctx->key_len = len; 2676 2677 /* Compute hash key by encrypting zeroes with cipher key */ 2678 memset(hashkey, 0, AES_BLOCK_SIZE); 2679 aes_encrypt(&aes, (u8 *)hashkey, (u8 *)hashkey); 2680 2681 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2682 for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) { 2683 if (be32_to_cpu(ctx->base.ipad.be[i]) != hashkey[i]) { 2684 ctx->base.needs_inv = true; 2685 break; 2686 } 2687 } 2688 } 2689 2690 for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) 2691 ctx->base.ipad.be[i] = cpu_to_be32(hashkey[i]); 2692 2693 memzero_explicit(hashkey, AES_BLOCK_SIZE); 2694 memzero_explicit(&aes, sizeof(aes)); 2695 return 0; 2696 } 2697 2698 static int safexcel_aead_gcm_cra_init(struct crypto_tfm *tfm) 2699 { 2700 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2701 2702 safexcel_aead_cra_init(tfm); 2703 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_GHASH; 2704 ctx->state_sz = GHASH_BLOCK_SIZE; 2705 ctx->xcm = EIP197_XCM_MODE_GCM; 2706 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XCM; /* override default */ 2707 2708 return 0; 2709 } 2710 2711 static void safexcel_aead_gcm_cra_exit(struct crypto_tfm *tfm) 2712 { 2713 safexcel_aead_cra_exit(tfm); 2714 } 2715 2716 static int safexcel_aead_gcm_setauthsize(struct crypto_aead *tfm, 2717 unsigned int authsize) 2718 { 2719 return crypto_gcm_check_authsize(authsize); 2720 } 2721 2722 struct safexcel_alg_template safexcel_alg_gcm = { 2723 .type = SAFEXCEL_ALG_TYPE_AEAD, 2724 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH, 2725 .alg.aead = { 2726 .setkey = safexcel_aead_gcm_setkey, 2727 .setauthsize = safexcel_aead_gcm_setauthsize, 2728 .encrypt = safexcel_aead_encrypt, 2729 .decrypt = safexcel_aead_decrypt, 2730 .ivsize = GCM_AES_IV_SIZE, 2731 .maxauthsize = GHASH_DIGEST_SIZE, 2732 .base = { 2733 .cra_name = "gcm(aes)", 2734 .cra_driver_name = "safexcel-gcm-aes", 2735 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2736 .cra_flags = CRYPTO_ALG_ASYNC | 2737 CRYPTO_ALG_ALLOCATES_MEMORY | 2738 CRYPTO_ALG_KERN_DRIVER_ONLY, 2739 .cra_blocksize = 1, 2740 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2741 .cra_alignmask = 0, 2742 .cra_init = safexcel_aead_gcm_cra_init, 2743 .cra_exit = safexcel_aead_gcm_cra_exit, 2744 .cra_module = THIS_MODULE, 2745 }, 2746 }, 2747 }; 2748 2749 static int safexcel_aead_ccm_setkey(struct crypto_aead *ctfm, const u8 *key, 2750 unsigned int len) 2751 { 2752 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 2753 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2754 struct safexcel_crypto_priv *priv = ctx->base.priv; 2755 struct crypto_aes_ctx aes __cleanup(aes_zeroize_ctx); 2756 int ret, i; 2757 2758 ret = aes_expandkey(&aes, key, len); 2759 if (ret) { 2760 return ret; 2761 } 2762 2763 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2764 for (i = 0; i < len / sizeof(u32); i++) { 2765 if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { 2766 ctx->base.needs_inv = true; 2767 break; 2768 } 2769 } 2770 } 2771 2772 for (i = 0; i < len / sizeof(u32); i++) { 2773 ctx->key[i] = cpu_to_le32(aes.key_enc[i]); 2774 ctx->base.ipad.be[i + 2 * AES_BLOCK_SIZE / sizeof(u32)] = 2775 cpu_to_be32(aes.key_enc[i]); 2776 } 2777 2778 ctx->key_len = len; 2779 ctx->state_sz = 2 * AES_BLOCK_SIZE + len; 2780 2781 if (len == AES_KEYSIZE_192) 2782 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192; 2783 else if (len == AES_KEYSIZE_256) 2784 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256; 2785 else 2786 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128; 2787 2788 return 0; 2789 } 2790 2791 static int safexcel_aead_ccm_cra_init(struct crypto_tfm *tfm) 2792 { 2793 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2794 2795 safexcel_aead_cra_init(tfm); 2796 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128; 2797 ctx->state_sz = 3 * AES_BLOCK_SIZE; 2798 ctx->xcm = EIP197_XCM_MODE_CCM; 2799 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XCM; /* override default */ 2800 ctx->ctrinit = 0; 2801 return 0; 2802 } 2803 2804 static int safexcel_aead_ccm_setauthsize(struct crypto_aead *tfm, 2805 unsigned int authsize) 2806 { 2807 /* Borrowed from crypto/ccm.c */ 2808 switch (authsize) { 2809 case 4: 2810 case 6: 2811 case 8: 2812 case 10: 2813 case 12: 2814 case 14: 2815 case 16: 2816 break; 2817 default: 2818 return -EINVAL; 2819 } 2820 2821 return 0; 2822 } 2823 2824 static int safexcel_ccm_encrypt(struct aead_request *req) 2825 { 2826 struct safexcel_cipher_req *creq = aead_request_ctx(req); 2827 2828 if (req->iv[0] < 1 || req->iv[0] > 7) 2829 return -EINVAL; 2830 2831 return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); 2832 } 2833 2834 static int safexcel_ccm_decrypt(struct aead_request *req) 2835 { 2836 struct safexcel_cipher_req *creq = aead_request_ctx(req); 2837 2838 if (req->iv[0] < 1 || req->iv[0] > 7) 2839 return -EINVAL; 2840 2841 return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); 2842 } 2843 2844 struct safexcel_alg_template safexcel_alg_ccm = { 2845 .type = SAFEXCEL_ALG_TYPE_AEAD, 2846 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_CBC_MAC_ALL, 2847 .alg.aead = { 2848 .setkey = safexcel_aead_ccm_setkey, 2849 .setauthsize = safexcel_aead_ccm_setauthsize, 2850 .encrypt = safexcel_ccm_encrypt, 2851 .decrypt = safexcel_ccm_decrypt, 2852 .ivsize = AES_BLOCK_SIZE, 2853 .maxauthsize = AES_BLOCK_SIZE, 2854 .base = { 2855 .cra_name = "ccm(aes)", 2856 .cra_driver_name = "safexcel-ccm-aes", 2857 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2858 .cra_flags = CRYPTO_ALG_ASYNC | 2859 CRYPTO_ALG_ALLOCATES_MEMORY | 2860 CRYPTO_ALG_KERN_DRIVER_ONLY, 2861 .cra_blocksize = 1, 2862 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2863 .cra_alignmask = 0, 2864 .cra_init = safexcel_aead_ccm_cra_init, 2865 .cra_exit = safexcel_aead_cra_exit, 2866 .cra_module = THIS_MODULE, 2867 }, 2868 }, 2869 }; 2870 2871 static void safexcel_chacha20_setkey(struct safexcel_cipher_ctx *ctx, 2872 const u8 *key) 2873 { 2874 struct safexcel_crypto_priv *priv = ctx->base.priv; 2875 2876 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) 2877 if (memcmp(ctx->key, key, CHACHA_KEY_SIZE)) 2878 ctx->base.needs_inv = true; 2879 2880 memcpy(ctx->key, key, CHACHA_KEY_SIZE); 2881 ctx->key_len = CHACHA_KEY_SIZE; 2882 } 2883 2884 static int safexcel_skcipher_chacha20_setkey(struct crypto_skcipher *ctfm, 2885 const u8 *key, unsigned int len) 2886 { 2887 struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm); 2888 2889 if (len != CHACHA_KEY_SIZE) 2890 return -EINVAL; 2891 2892 safexcel_chacha20_setkey(ctx, key); 2893 2894 return 0; 2895 } 2896 2897 static int safexcel_skcipher_chacha20_cra_init(struct crypto_tfm *tfm) 2898 { 2899 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2900 2901 safexcel_skcipher_cra_init(tfm); 2902 ctx->alg = SAFEXCEL_CHACHA20; 2903 ctx->ctrinit = 0; 2904 ctx->mode = CONTEXT_CONTROL_CHACHA20_MODE_256_32; 2905 return 0; 2906 } 2907 2908 struct safexcel_alg_template safexcel_alg_chacha20 = { 2909 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 2910 .algo_mask = SAFEXCEL_ALG_CHACHA20, 2911 .alg.skcipher = { 2912 .setkey = safexcel_skcipher_chacha20_setkey, 2913 .encrypt = safexcel_encrypt, 2914 .decrypt = safexcel_decrypt, 2915 .min_keysize = CHACHA_KEY_SIZE, 2916 .max_keysize = CHACHA_KEY_SIZE, 2917 .ivsize = CHACHA_IV_SIZE, 2918 .base = { 2919 .cra_name = "chacha20", 2920 .cra_driver_name = "safexcel-chacha20", 2921 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2922 .cra_flags = CRYPTO_ALG_ASYNC | 2923 CRYPTO_ALG_ALLOCATES_MEMORY | 2924 CRYPTO_ALG_KERN_DRIVER_ONLY, 2925 .cra_blocksize = 1, 2926 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2927 .cra_alignmask = 0, 2928 .cra_init = safexcel_skcipher_chacha20_cra_init, 2929 .cra_exit = safexcel_skcipher_cra_exit, 2930 .cra_module = THIS_MODULE, 2931 }, 2932 }, 2933 }; 2934 2935 static int safexcel_aead_chachapoly_setkey(struct crypto_aead *ctfm, 2936 const u8 *key, unsigned int len) 2937 { 2938 struct safexcel_cipher_ctx *ctx = crypto_aead_ctx(ctfm); 2939 2940 if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP && 2941 len > EIP197_AEAD_IPSEC_NONCE_SIZE) { 2942 /* ESP variant has nonce appended to key */ 2943 len -= EIP197_AEAD_IPSEC_NONCE_SIZE; 2944 ctx->nonce = *(u32 *)(key + len); 2945 } 2946 if (len != CHACHA_KEY_SIZE) 2947 return -EINVAL; 2948 2949 safexcel_chacha20_setkey(ctx, key); 2950 2951 return 0; 2952 } 2953 2954 static int safexcel_aead_chachapoly_setauthsize(struct crypto_aead *tfm, 2955 unsigned int authsize) 2956 { 2957 if (authsize != POLY1305_DIGEST_SIZE) 2958 return -EINVAL; 2959 return 0; 2960 } 2961 2962 static int safexcel_aead_chachapoly_crypt(struct aead_request *req, 2963 enum safexcel_cipher_direction dir) 2964 { 2965 struct safexcel_cipher_req *creq = aead_request_ctx(req); 2966 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2967 struct crypto_tfm *tfm = crypto_aead_tfm(aead); 2968 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2969 struct aead_request *subreq = aead_request_ctx(req); 2970 u32 key[CHACHA_KEY_SIZE / sizeof(u32) + 1]; 2971 int ret = 0; 2972 2973 /* 2974 * Instead of wasting time detecting umpteen silly corner cases, 2975 * just dump all "small" requests to the fallback implementation. 2976 * HW would not be faster on such small requests anyway. 2977 */ 2978 if (likely((ctx->aead != EIP197_AEAD_TYPE_IPSEC_ESP || 2979 req->assoclen >= EIP197_AEAD_IPSEC_IV_SIZE) && 2980 req->cryptlen > POLY1305_DIGEST_SIZE)) { 2981 return safexcel_queue_req(&req->base, creq, dir); 2982 } 2983 2984 /* HW cannot do full (AAD+payload) zero length, use fallback */ 2985 memcpy(key, ctx->key, CHACHA_KEY_SIZE); 2986 if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { 2987 /* ESP variant has nonce appended to the key */ 2988 key[CHACHA_KEY_SIZE / sizeof(u32)] = ctx->nonce; 2989 ret = crypto_aead_setkey(ctx->fback, (u8 *)key, 2990 CHACHA_KEY_SIZE + 2991 EIP197_AEAD_IPSEC_NONCE_SIZE); 2992 } else { 2993 ret = crypto_aead_setkey(ctx->fback, (u8 *)key, 2994 CHACHA_KEY_SIZE); 2995 } 2996 if (ret) { 2997 crypto_aead_clear_flags(aead, CRYPTO_TFM_REQ_MASK); 2998 crypto_aead_set_flags(aead, crypto_aead_get_flags(ctx->fback) & 2999 CRYPTO_TFM_REQ_MASK); 3000 return ret; 3001 } 3002 3003 aead_request_set_tfm(subreq, ctx->fback); 3004 aead_request_set_callback(subreq, req->base.flags, req->base.complete, 3005 req->base.data); 3006 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, 3007 req->iv); 3008 aead_request_set_ad(subreq, req->assoclen); 3009 3010 return (dir == SAFEXCEL_ENCRYPT) ? 3011 crypto_aead_encrypt(subreq) : 3012 crypto_aead_decrypt(subreq); 3013 } 3014 3015 static int safexcel_aead_chachapoly_encrypt(struct aead_request *req) 3016 { 3017 return safexcel_aead_chachapoly_crypt(req, SAFEXCEL_ENCRYPT); 3018 } 3019 3020 static int safexcel_aead_chachapoly_decrypt(struct aead_request *req) 3021 { 3022 return safexcel_aead_chachapoly_crypt(req, SAFEXCEL_DECRYPT); 3023 } 3024 3025 static int safexcel_aead_fallback_cra_init(struct crypto_tfm *tfm) 3026 { 3027 struct crypto_aead *aead = __crypto_aead_cast(tfm); 3028 struct aead_alg *alg = crypto_aead_alg(aead); 3029 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3030 3031 safexcel_aead_cra_init(tfm); 3032 3033 /* Allocate fallback implementation */ 3034 ctx->fback = crypto_alloc_aead(alg->base.cra_name, 0, 3035 CRYPTO_ALG_ASYNC | 3036 CRYPTO_ALG_NEED_FALLBACK); 3037 if (IS_ERR(ctx->fback)) 3038 return PTR_ERR(ctx->fback); 3039 3040 crypto_aead_set_reqsize(aead, max(sizeof(struct safexcel_cipher_req), 3041 sizeof(struct aead_request) + 3042 crypto_aead_reqsize(ctx->fback))); 3043 3044 return 0; 3045 } 3046 3047 static int safexcel_aead_chachapoly_cra_init(struct crypto_tfm *tfm) 3048 { 3049 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3050 3051 safexcel_aead_fallback_cra_init(tfm); 3052 ctx->alg = SAFEXCEL_CHACHA20; 3053 ctx->mode = CONTEXT_CONTROL_CHACHA20_MODE_256_32 | 3054 CONTEXT_CONTROL_CHACHA20_MODE_CALC_OTK; 3055 ctx->ctrinit = 0; 3056 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_POLY1305; 3057 ctx->state_sz = 0; /* Precomputed by HW */ 3058 return 0; 3059 } 3060 3061 static void safexcel_aead_fallback_cra_exit(struct crypto_tfm *tfm) 3062 { 3063 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3064 3065 crypto_free_aead(ctx->fback); 3066 safexcel_aead_cra_exit(tfm); 3067 } 3068 3069 struct safexcel_alg_template safexcel_alg_chachapoly = { 3070 .type = SAFEXCEL_ALG_TYPE_AEAD, 3071 .algo_mask = SAFEXCEL_ALG_CHACHA20 | SAFEXCEL_ALG_POLY1305, 3072 .alg.aead = { 3073 .setkey = safexcel_aead_chachapoly_setkey, 3074 .setauthsize = safexcel_aead_chachapoly_setauthsize, 3075 .encrypt = safexcel_aead_chachapoly_encrypt, 3076 .decrypt = safexcel_aead_chachapoly_decrypt, 3077 .ivsize = CHACHAPOLY_IV_SIZE, 3078 .maxauthsize = POLY1305_DIGEST_SIZE, 3079 .base = { 3080 .cra_name = "rfc7539(chacha20,poly1305)", 3081 .cra_driver_name = "safexcel-chacha20-poly1305", 3082 /* +1 to put it above HW chacha + SW poly */ 3083 .cra_priority = SAFEXCEL_CRA_PRIORITY + 1, 3084 .cra_flags = CRYPTO_ALG_ASYNC | 3085 CRYPTO_ALG_ALLOCATES_MEMORY | 3086 CRYPTO_ALG_KERN_DRIVER_ONLY | 3087 CRYPTO_ALG_NEED_FALLBACK, 3088 .cra_blocksize = 1, 3089 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3090 .cra_alignmask = 0, 3091 .cra_init = safexcel_aead_chachapoly_cra_init, 3092 .cra_exit = safexcel_aead_fallback_cra_exit, 3093 .cra_module = THIS_MODULE, 3094 }, 3095 }, 3096 }; 3097 3098 static int safexcel_aead_chachapolyesp_cra_init(struct crypto_tfm *tfm) 3099 { 3100 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3101 int ret; 3102 3103 ret = safexcel_aead_chachapoly_cra_init(tfm); 3104 ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP; 3105 ctx->aadskip = EIP197_AEAD_IPSEC_IV_SIZE; 3106 return ret; 3107 } 3108 3109 struct safexcel_alg_template safexcel_alg_chachapoly_esp = { 3110 .type = SAFEXCEL_ALG_TYPE_AEAD, 3111 .algo_mask = SAFEXCEL_ALG_CHACHA20 | SAFEXCEL_ALG_POLY1305, 3112 .alg.aead = { 3113 .setkey = safexcel_aead_chachapoly_setkey, 3114 .setauthsize = safexcel_aead_chachapoly_setauthsize, 3115 .encrypt = safexcel_aead_chachapoly_encrypt, 3116 .decrypt = safexcel_aead_chachapoly_decrypt, 3117 .ivsize = CHACHAPOLY_IV_SIZE - EIP197_AEAD_IPSEC_NONCE_SIZE, 3118 .maxauthsize = POLY1305_DIGEST_SIZE, 3119 .base = { 3120 .cra_name = "rfc7539esp(chacha20,poly1305)", 3121 .cra_driver_name = "safexcel-chacha20-poly1305-esp", 3122 /* +1 to put it above HW chacha + SW poly */ 3123 .cra_priority = SAFEXCEL_CRA_PRIORITY + 1, 3124 .cra_flags = CRYPTO_ALG_ASYNC | 3125 CRYPTO_ALG_ALLOCATES_MEMORY | 3126 CRYPTO_ALG_KERN_DRIVER_ONLY | 3127 CRYPTO_ALG_NEED_FALLBACK, 3128 .cra_blocksize = 1, 3129 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3130 .cra_alignmask = 0, 3131 .cra_init = safexcel_aead_chachapolyesp_cra_init, 3132 .cra_exit = safexcel_aead_fallback_cra_exit, 3133 .cra_module = THIS_MODULE, 3134 }, 3135 }, 3136 }; 3137 3138 static int safexcel_skcipher_sm4_setkey(struct crypto_skcipher *ctfm, 3139 const u8 *key, unsigned int len) 3140 { 3141 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 3142 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3143 struct safexcel_crypto_priv *priv = ctx->base.priv; 3144 3145 if (len != SM4_KEY_SIZE) 3146 return -EINVAL; 3147 3148 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) 3149 if (memcmp(ctx->key, key, SM4_KEY_SIZE)) 3150 ctx->base.needs_inv = true; 3151 3152 memcpy(ctx->key, key, SM4_KEY_SIZE); 3153 ctx->key_len = SM4_KEY_SIZE; 3154 3155 return 0; 3156 } 3157 3158 static int safexcel_sm4_blk_encrypt(struct skcipher_request *req) 3159 { 3160 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3161 if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) 3162 return -EINVAL; 3163 else 3164 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 3165 SAFEXCEL_ENCRYPT); 3166 } 3167 3168 static int safexcel_sm4_blk_decrypt(struct skcipher_request *req) 3169 { 3170 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3171 if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) 3172 return -EINVAL; 3173 else 3174 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 3175 SAFEXCEL_DECRYPT); 3176 } 3177 3178 static int safexcel_skcipher_sm4_ecb_cra_init(struct crypto_tfm *tfm) 3179 { 3180 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3181 3182 safexcel_skcipher_cra_init(tfm); 3183 ctx->alg = SAFEXCEL_SM4; 3184 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; 3185 ctx->blocksz = 0; 3186 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 3187 return 0; 3188 } 3189 3190 struct safexcel_alg_template safexcel_alg_ecb_sm4 = { 3191 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 3192 .algo_mask = SAFEXCEL_ALG_SM4, 3193 .alg.skcipher = { 3194 .setkey = safexcel_skcipher_sm4_setkey, 3195 .encrypt = safexcel_sm4_blk_encrypt, 3196 .decrypt = safexcel_sm4_blk_decrypt, 3197 .min_keysize = SM4_KEY_SIZE, 3198 .max_keysize = SM4_KEY_SIZE, 3199 .base = { 3200 .cra_name = "ecb(sm4)", 3201 .cra_driver_name = "safexcel-ecb-sm4", 3202 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3203 .cra_flags = CRYPTO_ALG_ASYNC | 3204 CRYPTO_ALG_ALLOCATES_MEMORY | 3205 CRYPTO_ALG_KERN_DRIVER_ONLY, 3206 .cra_blocksize = SM4_BLOCK_SIZE, 3207 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3208 .cra_alignmask = 0, 3209 .cra_init = safexcel_skcipher_sm4_ecb_cra_init, 3210 .cra_exit = safexcel_skcipher_cra_exit, 3211 .cra_module = THIS_MODULE, 3212 }, 3213 }, 3214 }; 3215 3216 static int safexcel_skcipher_sm4_cbc_cra_init(struct crypto_tfm *tfm) 3217 { 3218 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3219 3220 safexcel_skcipher_cra_init(tfm); 3221 ctx->alg = SAFEXCEL_SM4; 3222 ctx->blocksz = SM4_BLOCK_SIZE; 3223 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; 3224 return 0; 3225 } 3226 3227 struct safexcel_alg_template safexcel_alg_cbc_sm4 = { 3228 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 3229 .algo_mask = SAFEXCEL_ALG_SM4, 3230 .alg.skcipher = { 3231 .setkey = safexcel_skcipher_sm4_setkey, 3232 .encrypt = safexcel_sm4_blk_encrypt, 3233 .decrypt = safexcel_sm4_blk_decrypt, 3234 .min_keysize = SM4_KEY_SIZE, 3235 .max_keysize = SM4_KEY_SIZE, 3236 .ivsize = SM4_BLOCK_SIZE, 3237 .base = { 3238 .cra_name = "cbc(sm4)", 3239 .cra_driver_name = "safexcel-cbc-sm4", 3240 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3241 .cra_flags = CRYPTO_ALG_ASYNC | 3242 CRYPTO_ALG_ALLOCATES_MEMORY | 3243 CRYPTO_ALG_KERN_DRIVER_ONLY, 3244 .cra_blocksize = SM4_BLOCK_SIZE, 3245 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3246 .cra_alignmask = 0, 3247 .cra_init = safexcel_skcipher_sm4_cbc_cra_init, 3248 .cra_exit = safexcel_skcipher_cra_exit, 3249 .cra_module = THIS_MODULE, 3250 }, 3251 }, 3252 }; 3253 3254 static int safexcel_skcipher_sm4ctr_setkey(struct crypto_skcipher *ctfm, 3255 const u8 *key, unsigned int len) 3256 { 3257 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 3258 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3259 3260 /* last 4 bytes of key are the nonce! */ 3261 ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE); 3262 /* exclude the nonce here */ 3263 len -= CTR_RFC3686_NONCE_SIZE; 3264 3265 return safexcel_skcipher_sm4_setkey(ctfm, key, len); 3266 } 3267 3268 static int safexcel_skcipher_sm4_ctr_cra_init(struct crypto_tfm *tfm) 3269 { 3270 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3271 3272 safexcel_skcipher_cra_init(tfm); 3273 ctx->alg = SAFEXCEL_SM4; 3274 ctx->blocksz = SM4_BLOCK_SIZE; 3275 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; 3276 return 0; 3277 } 3278 3279 struct safexcel_alg_template safexcel_alg_ctr_sm4 = { 3280 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 3281 .algo_mask = SAFEXCEL_ALG_SM4, 3282 .alg.skcipher = { 3283 .setkey = safexcel_skcipher_sm4ctr_setkey, 3284 .encrypt = safexcel_encrypt, 3285 .decrypt = safexcel_decrypt, 3286 /* Add nonce size */ 3287 .min_keysize = SM4_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 3288 .max_keysize = SM4_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 3289 .ivsize = CTR_RFC3686_IV_SIZE, 3290 .base = { 3291 .cra_name = "rfc3686(ctr(sm4))", 3292 .cra_driver_name = "safexcel-ctr-sm4", 3293 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3294 .cra_flags = CRYPTO_ALG_ASYNC | 3295 CRYPTO_ALG_ALLOCATES_MEMORY | 3296 CRYPTO_ALG_KERN_DRIVER_ONLY, 3297 .cra_blocksize = 1, 3298 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3299 .cra_alignmask = 0, 3300 .cra_init = safexcel_skcipher_sm4_ctr_cra_init, 3301 .cra_exit = safexcel_skcipher_cra_exit, 3302 .cra_module = THIS_MODULE, 3303 }, 3304 }, 3305 }; 3306 3307 static int safexcel_aead_sm4_blk_encrypt(struct aead_request *req) 3308 { 3309 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3310 if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) 3311 return -EINVAL; 3312 3313 return safexcel_queue_req(&req->base, aead_request_ctx(req), 3314 SAFEXCEL_ENCRYPT); 3315 } 3316 3317 static int safexcel_aead_sm4_blk_decrypt(struct aead_request *req) 3318 { 3319 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3320 3321 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3322 if ((req->cryptlen - crypto_aead_authsize(tfm)) & (SM4_BLOCK_SIZE - 1)) 3323 return -EINVAL; 3324 3325 return safexcel_queue_req(&req->base, aead_request_ctx(req), 3326 SAFEXCEL_DECRYPT); 3327 } 3328 3329 static int safexcel_aead_sm4cbc_sha1_cra_init(struct crypto_tfm *tfm) 3330 { 3331 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3332 3333 safexcel_aead_cra_init(tfm); 3334 ctx->alg = SAFEXCEL_SM4; 3335 ctx->blocksz = SM4_BLOCK_SIZE; 3336 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1; 3337 ctx->state_sz = SHA1_DIGEST_SIZE; 3338 return 0; 3339 } 3340 3341 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_sm4 = { 3342 .type = SAFEXCEL_ALG_TYPE_AEAD, 3343 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SHA1, 3344 .alg.aead = { 3345 .setkey = safexcel_aead_setkey, 3346 .encrypt = safexcel_aead_sm4_blk_encrypt, 3347 .decrypt = safexcel_aead_sm4_blk_decrypt, 3348 .ivsize = SM4_BLOCK_SIZE, 3349 .maxauthsize = SHA1_DIGEST_SIZE, 3350 .base = { 3351 .cra_name = "authenc(hmac(sha1),cbc(sm4))", 3352 .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-sm4", 3353 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3354 .cra_flags = CRYPTO_ALG_ASYNC | 3355 CRYPTO_ALG_ALLOCATES_MEMORY | 3356 CRYPTO_ALG_KERN_DRIVER_ONLY, 3357 .cra_blocksize = SM4_BLOCK_SIZE, 3358 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3359 .cra_alignmask = 0, 3360 .cra_init = safexcel_aead_sm4cbc_sha1_cra_init, 3361 .cra_exit = safexcel_aead_cra_exit, 3362 .cra_module = THIS_MODULE, 3363 }, 3364 }, 3365 }; 3366 3367 static int safexcel_aead_fallback_setkey(struct crypto_aead *ctfm, 3368 const u8 *key, unsigned int len) 3369 { 3370 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 3371 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3372 3373 /* Keep fallback cipher synchronized */ 3374 return crypto_aead_setkey(ctx->fback, (u8 *)key, len) ?: 3375 safexcel_aead_setkey(ctfm, key, len); 3376 } 3377 3378 static int safexcel_aead_fallback_setauthsize(struct crypto_aead *ctfm, 3379 unsigned int authsize) 3380 { 3381 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 3382 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3383 3384 /* Keep fallback cipher synchronized */ 3385 return crypto_aead_setauthsize(ctx->fback, authsize); 3386 } 3387 3388 static int safexcel_aead_fallback_crypt(struct aead_request *req, 3389 enum safexcel_cipher_direction dir) 3390 { 3391 struct crypto_aead *aead = crypto_aead_reqtfm(req); 3392 struct crypto_tfm *tfm = crypto_aead_tfm(aead); 3393 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3394 struct aead_request *subreq = aead_request_ctx(req); 3395 3396 aead_request_set_tfm(subreq, ctx->fback); 3397 aead_request_set_callback(subreq, req->base.flags, req->base.complete, 3398 req->base.data); 3399 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, 3400 req->iv); 3401 aead_request_set_ad(subreq, req->assoclen); 3402 3403 return (dir == SAFEXCEL_ENCRYPT) ? 3404 crypto_aead_encrypt(subreq) : 3405 crypto_aead_decrypt(subreq); 3406 } 3407 3408 static int safexcel_aead_sm4cbc_sm3_encrypt(struct aead_request *req) 3409 { 3410 struct safexcel_cipher_req *creq = aead_request_ctx(req); 3411 3412 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3413 if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) 3414 return -EINVAL; 3415 else if (req->cryptlen || req->assoclen) /* If input length > 0 only */ 3416 return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); 3417 3418 /* HW cannot do full (AAD+payload) zero length, use fallback */ 3419 return safexcel_aead_fallback_crypt(req, SAFEXCEL_ENCRYPT); 3420 } 3421 3422 static int safexcel_aead_sm4cbc_sm3_decrypt(struct aead_request *req) 3423 { 3424 struct safexcel_cipher_req *creq = aead_request_ctx(req); 3425 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3426 3427 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3428 if ((req->cryptlen - crypto_aead_authsize(tfm)) & (SM4_BLOCK_SIZE - 1)) 3429 return -EINVAL; 3430 else if (req->cryptlen > crypto_aead_authsize(tfm) || req->assoclen) 3431 /* If input length > 0 only */ 3432 return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); 3433 3434 /* HW cannot do full (AAD+payload) zero length, use fallback */ 3435 return safexcel_aead_fallback_crypt(req, SAFEXCEL_DECRYPT); 3436 } 3437 3438 static int safexcel_aead_sm4cbc_sm3_cra_init(struct crypto_tfm *tfm) 3439 { 3440 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3441 3442 safexcel_aead_fallback_cra_init(tfm); 3443 ctx->alg = SAFEXCEL_SM4; 3444 ctx->blocksz = SM4_BLOCK_SIZE; 3445 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3; 3446 ctx->state_sz = SM3_DIGEST_SIZE; 3447 return 0; 3448 } 3449 3450 struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_cbc_sm4 = { 3451 .type = SAFEXCEL_ALG_TYPE_AEAD, 3452 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SM3, 3453 .alg.aead = { 3454 .setkey = safexcel_aead_fallback_setkey, 3455 .setauthsize = safexcel_aead_fallback_setauthsize, 3456 .encrypt = safexcel_aead_sm4cbc_sm3_encrypt, 3457 .decrypt = safexcel_aead_sm4cbc_sm3_decrypt, 3458 .ivsize = SM4_BLOCK_SIZE, 3459 .maxauthsize = SM3_DIGEST_SIZE, 3460 .base = { 3461 .cra_name = "authenc(hmac(sm3),cbc(sm4))", 3462 .cra_driver_name = "safexcel-authenc-hmac-sm3-cbc-sm4", 3463 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3464 .cra_flags = CRYPTO_ALG_ASYNC | 3465 CRYPTO_ALG_ALLOCATES_MEMORY | 3466 CRYPTO_ALG_KERN_DRIVER_ONLY | 3467 CRYPTO_ALG_NEED_FALLBACK, 3468 .cra_blocksize = SM4_BLOCK_SIZE, 3469 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3470 .cra_alignmask = 0, 3471 .cra_init = safexcel_aead_sm4cbc_sm3_cra_init, 3472 .cra_exit = safexcel_aead_fallback_cra_exit, 3473 .cra_module = THIS_MODULE, 3474 }, 3475 }, 3476 }; 3477 3478 static int safexcel_aead_sm4ctr_sha1_cra_init(struct crypto_tfm *tfm) 3479 { 3480 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3481 3482 safexcel_aead_sm4cbc_sha1_cra_init(tfm); 3483 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; 3484 return 0; 3485 } 3486 3487 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_ctr_sm4 = { 3488 .type = SAFEXCEL_ALG_TYPE_AEAD, 3489 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SHA1, 3490 .alg.aead = { 3491 .setkey = safexcel_aead_setkey, 3492 .encrypt = safexcel_aead_encrypt, 3493 .decrypt = safexcel_aead_decrypt, 3494 .ivsize = CTR_RFC3686_IV_SIZE, 3495 .maxauthsize = SHA1_DIGEST_SIZE, 3496 .base = { 3497 .cra_name = "authenc(hmac(sha1),rfc3686(ctr(sm4)))", 3498 .cra_driver_name = "safexcel-authenc-hmac-sha1-ctr-sm4", 3499 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3500 .cra_flags = CRYPTO_ALG_ASYNC | 3501 CRYPTO_ALG_ALLOCATES_MEMORY | 3502 CRYPTO_ALG_KERN_DRIVER_ONLY, 3503 .cra_blocksize = 1, 3504 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3505 .cra_alignmask = 0, 3506 .cra_init = safexcel_aead_sm4ctr_sha1_cra_init, 3507 .cra_exit = safexcel_aead_cra_exit, 3508 .cra_module = THIS_MODULE, 3509 }, 3510 }, 3511 }; 3512 3513 static int safexcel_aead_sm4ctr_sm3_cra_init(struct crypto_tfm *tfm) 3514 { 3515 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3516 3517 safexcel_aead_sm4cbc_sm3_cra_init(tfm); 3518 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; 3519 return 0; 3520 } 3521 3522 struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_ctr_sm4 = { 3523 .type = SAFEXCEL_ALG_TYPE_AEAD, 3524 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SM3, 3525 .alg.aead = { 3526 .setkey = safexcel_aead_setkey, 3527 .encrypt = safexcel_aead_encrypt, 3528 .decrypt = safexcel_aead_decrypt, 3529 .ivsize = CTR_RFC3686_IV_SIZE, 3530 .maxauthsize = SM3_DIGEST_SIZE, 3531 .base = { 3532 .cra_name = "authenc(hmac(sm3),rfc3686(ctr(sm4)))", 3533 .cra_driver_name = "safexcel-authenc-hmac-sm3-ctr-sm4", 3534 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3535 .cra_flags = CRYPTO_ALG_ASYNC | 3536 CRYPTO_ALG_ALLOCATES_MEMORY | 3537 CRYPTO_ALG_KERN_DRIVER_ONLY, 3538 .cra_blocksize = 1, 3539 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3540 .cra_alignmask = 0, 3541 .cra_init = safexcel_aead_sm4ctr_sm3_cra_init, 3542 .cra_exit = safexcel_aead_cra_exit, 3543 .cra_module = THIS_MODULE, 3544 }, 3545 }, 3546 }; 3547 3548 static int safexcel_rfc4106_gcm_setkey(struct crypto_aead *ctfm, const u8 *key, 3549 unsigned int len) 3550 { 3551 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 3552 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3553 3554 /* last 4 bytes of key are the nonce! */ 3555 ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE); 3556 3557 len -= CTR_RFC3686_NONCE_SIZE; 3558 return safexcel_aead_gcm_setkey(ctfm, key, len); 3559 } 3560 3561 static int safexcel_rfc4106_gcm_setauthsize(struct crypto_aead *tfm, 3562 unsigned int authsize) 3563 { 3564 return crypto_rfc4106_check_authsize(authsize); 3565 } 3566 3567 static int safexcel_rfc4106_encrypt(struct aead_request *req) 3568 { 3569 return crypto_ipsec_check_assoclen(req->assoclen) ?: 3570 safexcel_aead_encrypt(req); 3571 } 3572 3573 static int safexcel_rfc4106_decrypt(struct aead_request *req) 3574 { 3575 return crypto_ipsec_check_assoclen(req->assoclen) ?: 3576 safexcel_aead_decrypt(req); 3577 } 3578 3579 static int safexcel_rfc4106_gcm_cra_init(struct crypto_tfm *tfm) 3580 { 3581 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3582 int ret; 3583 3584 ret = safexcel_aead_gcm_cra_init(tfm); 3585 ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP; 3586 ctx->aadskip = EIP197_AEAD_IPSEC_IV_SIZE; 3587 return ret; 3588 } 3589 3590 struct safexcel_alg_template safexcel_alg_rfc4106_gcm = { 3591 .type = SAFEXCEL_ALG_TYPE_AEAD, 3592 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH, 3593 .alg.aead = { 3594 .setkey = safexcel_rfc4106_gcm_setkey, 3595 .setauthsize = safexcel_rfc4106_gcm_setauthsize, 3596 .encrypt = safexcel_rfc4106_encrypt, 3597 .decrypt = safexcel_rfc4106_decrypt, 3598 .ivsize = GCM_RFC4106_IV_SIZE, 3599 .maxauthsize = GHASH_DIGEST_SIZE, 3600 .base = { 3601 .cra_name = "rfc4106(gcm(aes))", 3602 .cra_driver_name = "safexcel-rfc4106-gcm-aes", 3603 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3604 .cra_flags = CRYPTO_ALG_ASYNC | 3605 CRYPTO_ALG_ALLOCATES_MEMORY | 3606 CRYPTO_ALG_KERN_DRIVER_ONLY, 3607 .cra_blocksize = 1, 3608 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3609 .cra_alignmask = 0, 3610 .cra_init = safexcel_rfc4106_gcm_cra_init, 3611 .cra_exit = safexcel_aead_gcm_cra_exit, 3612 }, 3613 }, 3614 }; 3615 3616 static int safexcel_rfc4543_gcm_setauthsize(struct crypto_aead *tfm, 3617 unsigned int authsize) 3618 { 3619 if (authsize != GHASH_DIGEST_SIZE) 3620 return -EINVAL; 3621 3622 return 0; 3623 } 3624 3625 static int safexcel_rfc4543_gcm_cra_init(struct crypto_tfm *tfm) 3626 { 3627 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3628 int ret; 3629 3630 ret = safexcel_aead_gcm_cra_init(tfm); 3631 ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP_GMAC; 3632 return ret; 3633 } 3634 3635 struct safexcel_alg_template safexcel_alg_rfc4543_gcm = { 3636 .type = SAFEXCEL_ALG_TYPE_AEAD, 3637 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH, 3638 .alg.aead = { 3639 .setkey = safexcel_rfc4106_gcm_setkey, 3640 .setauthsize = safexcel_rfc4543_gcm_setauthsize, 3641 .encrypt = safexcel_rfc4106_encrypt, 3642 .decrypt = safexcel_rfc4106_decrypt, 3643 .ivsize = GCM_RFC4543_IV_SIZE, 3644 .maxauthsize = GHASH_DIGEST_SIZE, 3645 .base = { 3646 .cra_name = "rfc4543(gcm(aes))", 3647 .cra_driver_name = "safexcel-rfc4543-gcm-aes", 3648 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3649 .cra_flags = CRYPTO_ALG_ASYNC | 3650 CRYPTO_ALG_ALLOCATES_MEMORY | 3651 CRYPTO_ALG_KERN_DRIVER_ONLY, 3652 .cra_blocksize = 1, 3653 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3654 .cra_alignmask = 0, 3655 .cra_init = safexcel_rfc4543_gcm_cra_init, 3656 .cra_exit = safexcel_aead_gcm_cra_exit, 3657 }, 3658 }, 3659 }; 3660 3661 static int safexcel_rfc4309_ccm_setkey(struct crypto_aead *ctfm, const u8 *key, 3662 unsigned int len) 3663 { 3664 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 3665 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3666 3667 /* First byte of the nonce = L = always 3 for RFC4309 (4 byte ctr) */ 3668 *(u8 *)&ctx->nonce = EIP197_AEAD_IPSEC_COUNTER_SIZE - 1; 3669 /* last 3 bytes of key are the nonce! */ 3670 memcpy((u8 *)&ctx->nonce + 1, key + len - 3671 EIP197_AEAD_IPSEC_CCM_NONCE_SIZE, 3672 EIP197_AEAD_IPSEC_CCM_NONCE_SIZE); 3673 3674 len -= EIP197_AEAD_IPSEC_CCM_NONCE_SIZE; 3675 return safexcel_aead_ccm_setkey(ctfm, key, len); 3676 } 3677 3678 static int safexcel_rfc4309_ccm_setauthsize(struct crypto_aead *tfm, 3679 unsigned int authsize) 3680 { 3681 /* Borrowed from crypto/ccm.c */ 3682 switch (authsize) { 3683 case 8: 3684 case 12: 3685 case 16: 3686 break; 3687 default: 3688 return -EINVAL; 3689 } 3690 3691 return 0; 3692 } 3693 3694 static int safexcel_rfc4309_ccm_encrypt(struct aead_request *req) 3695 { 3696 struct safexcel_cipher_req *creq = aead_request_ctx(req); 3697 3698 /* Borrowed from crypto/ccm.c */ 3699 if (req->assoclen != 16 && req->assoclen != 20) 3700 return -EINVAL; 3701 3702 return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); 3703 } 3704 3705 static int safexcel_rfc4309_ccm_decrypt(struct aead_request *req) 3706 { 3707 struct safexcel_cipher_req *creq = aead_request_ctx(req); 3708 3709 /* Borrowed from crypto/ccm.c */ 3710 if (req->assoclen != 16 && req->assoclen != 20) 3711 return -EINVAL; 3712 3713 return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); 3714 } 3715 3716 static int safexcel_rfc4309_ccm_cra_init(struct crypto_tfm *tfm) 3717 { 3718 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3719 int ret; 3720 3721 ret = safexcel_aead_ccm_cra_init(tfm); 3722 ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP; 3723 ctx->aadskip = EIP197_AEAD_IPSEC_IV_SIZE; 3724 return ret; 3725 } 3726 3727 struct safexcel_alg_template safexcel_alg_rfc4309_ccm = { 3728 .type = SAFEXCEL_ALG_TYPE_AEAD, 3729 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_CBC_MAC_ALL, 3730 .alg.aead = { 3731 .setkey = safexcel_rfc4309_ccm_setkey, 3732 .setauthsize = safexcel_rfc4309_ccm_setauthsize, 3733 .encrypt = safexcel_rfc4309_ccm_encrypt, 3734 .decrypt = safexcel_rfc4309_ccm_decrypt, 3735 .ivsize = EIP197_AEAD_IPSEC_IV_SIZE, 3736 .maxauthsize = AES_BLOCK_SIZE, 3737 .base = { 3738 .cra_name = "rfc4309(ccm(aes))", 3739 .cra_driver_name = "safexcel-rfc4309-ccm-aes", 3740 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3741 .cra_flags = CRYPTO_ALG_ASYNC | 3742 CRYPTO_ALG_ALLOCATES_MEMORY | 3743 CRYPTO_ALG_KERN_DRIVER_ONLY, 3744 .cra_blocksize = 1, 3745 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3746 .cra_alignmask = 0, 3747 .cra_init = safexcel_rfc4309_ccm_cra_init, 3748 .cra_exit = safexcel_aead_cra_exit, 3749 .cra_module = THIS_MODULE, 3750 }, 3751 }, 3752 }; 3753