1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2017 Marvell 4 * 5 * Antoine Tenart <antoine.tenart@free-electrons.com> 6 */ 7 8 #include <asm/unaligned.h> 9 #include <linux/device.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/dmapool.h> 12 #include <crypto/aead.h> 13 #include <crypto/aes.h> 14 #include <crypto/authenc.h> 15 #include <crypto/chacha.h> 16 #include <crypto/ctr.h> 17 #include <crypto/internal/des.h> 18 #include <crypto/gcm.h> 19 #include <crypto/ghash.h> 20 #include <crypto/poly1305.h> 21 #include <crypto/sha.h> 22 #include <crypto/sm3.h> 23 #include <crypto/sm4.h> 24 #include <crypto/xts.h> 25 #include <crypto/skcipher.h> 26 #include <crypto/internal/aead.h> 27 #include <crypto/internal/skcipher.h> 28 29 #include "safexcel.h" 30 31 enum safexcel_cipher_direction { 32 SAFEXCEL_ENCRYPT, 33 SAFEXCEL_DECRYPT, 34 }; 35 36 enum safexcel_cipher_alg { 37 SAFEXCEL_DES, 38 SAFEXCEL_3DES, 39 SAFEXCEL_AES, 40 SAFEXCEL_CHACHA20, 41 SAFEXCEL_SM4, 42 }; 43 44 struct safexcel_cipher_ctx { 45 struct safexcel_context base; 46 struct safexcel_crypto_priv *priv; 47 48 u32 mode; 49 enum safexcel_cipher_alg alg; 50 char aead; /* !=0=AEAD, 2=IPSec ESP AEAD, 3=IPsec ESP GMAC */ 51 char xcm; /* 0=authenc, 1=GCM, 2 reserved for CCM */ 52 53 __le32 key[16]; 54 u32 nonce; 55 unsigned int key_len, xts; 56 57 /* All the below is AEAD specific */ 58 u32 hash_alg; 59 u32 state_sz; 60 __be32 ipad[SHA512_DIGEST_SIZE / sizeof(u32)]; 61 __be32 opad[SHA512_DIGEST_SIZE / sizeof(u32)]; 62 63 struct crypto_cipher *hkaes; 64 struct crypto_aead *fback; 65 }; 66 67 struct safexcel_cipher_req { 68 enum safexcel_cipher_direction direction; 69 /* Number of result descriptors associated to the request */ 70 unsigned int rdescs; 71 bool needs_inv; 72 int nr_src, nr_dst; 73 }; 74 75 static void safexcel_cipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv, 76 struct safexcel_command_desc *cdesc) 77 { 78 u32 block_sz = 0; 79 80 if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD || 81 ctx->aead & EIP197_AEAD_TYPE_IPSEC_ESP) { /* _ESP and _ESP_GMAC */ 82 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; 83 84 /* 32 bit nonce */ 85 cdesc->control_data.token[0] = ctx->nonce; 86 /* 64 bit IV part */ 87 memcpy(&cdesc->control_data.token[1], iv, 8); 88 89 if (ctx->alg == SAFEXCEL_CHACHA20 || 90 ctx->xcm == EIP197_XCM_MODE_CCM) { 91 /* 32 bit counter, starting at 0 */ 92 cdesc->control_data.token[3] = 0; 93 } else { 94 /* 32 bit counter, start at 1 (big endian!) */ 95 cdesc->control_data.token[3] = 96 (__force u32)cpu_to_be32(1); 97 } 98 99 return; 100 } else if (ctx->xcm == EIP197_XCM_MODE_GCM || 101 (ctx->aead && ctx->alg == SAFEXCEL_CHACHA20)) { 102 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; 103 104 /* 96 bit IV part */ 105 memcpy(&cdesc->control_data.token[0], iv, 12); 106 107 if (ctx->alg == SAFEXCEL_CHACHA20) { 108 /* 32 bit counter, starting at 0 */ 109 cdesc->control_data.token[3] = 0; 110 } else { 111 /* 32 bit counter, start at 1 (big endian!) */ 112 *(__be32 *)&cdesc->control_data.token[3] = 113 cpu_to_be32(1); 114 } 115 116 return; 117 } else if (ctx->alg == SAFEXCEL_CHACHA20) { 118 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; 119 120 /* 96 bit nonce part */ 121 memcpy(&cdesc->control_data.token[0], &iv[4], 12); 122 /* 32 bit counter */ 123 cdesc->control_data.token[3] = *(u32 *)iv; 124 125 return; 126 } else if (ctx->xcm == EIP197_XCM_MODE_CCM) { 127 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; 128 129 /* Variable length IV part */ 130 memcpy(&cdesc->control_data.token[0], iv, 15 - iv[0]); 131 /* Start variable length counter at 0 */ 132 memset((u8 *)&cdesc->control_data.token[0] + 15 - iv[0], 133 0, iv[0] + 1); 134 135 return; 136 } 137 138 if (ctx->mode != CONTEXT_CONTROL_CRYPTO_MODE_ECB) { 139 switch (ctx->alg) { 140 case SAFEXCEL_DES: 141 block_sz = DES_BLOCK_SIZE; 142 cdesc->control_data.options |= EIP197_OPTION_2_TOKEN_IV_CMD; 143 break; 144 case SAFEXCEL_3DES: 145 block_sz = DES3_EDE_BLOCK_SIZE; 146 cdesc->control_data.options |= EIP197_OPTION_2_TOKEN_IV_CMD; 147 break; 148 case SAFEXCEL_SM4: 149 block_sz = SM4_BLOCK_SIZE; 150 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; 151 break; 152 case SAFEXCEL_AES: 153 block_sz = AES_BLOCK_SIZE; 154 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; 155 break; 156 default: 157 break; 158 } 159 memcpy(cdesc->control_data.token, iv, block_sz); 160 } 161 } 162 163 static void safexcel_skcipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv, 164 struct safexcel_command_desc *cdesc, 165 u32 length) 166 { 167 struct safexcel_token *token; 168 169 safexcel_cipher_token(ctx, iv, cdesc); 170 171 /* skip over worst case IV of 4 dwords, no need to be exact */ 172 token = (struct safexcel_token *)(cdesc->control_data.token + 4); 173 174 token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION; 175 token[0].packet_length = length; 176 token[0].stat = EIP197_TOKEN_STAT_LAST_PACKET | 177 EIP197_TOKEN_STAT_LAST_HASH; 178 token[0].instructions = EIP197_TOKEN_INS_LAST | 179 EIP197_TOKEN_INS_TYPE_CRYPTO | 180 EIP197_TOKEN_INS_TYPE_OUTPUT; 181 } 182 183 static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv, 184 struct safexcel_command_desc *cdesc, 185 enum safexcel_cipher_direction direction, 186 u32 cryptlen, u32 assoclen, u32 digestsize) 187 { 188 struct safexcel_token *token; 189 190 safexcel_cipher_token(ctx, iv, cdesc); 191 192 if (direction == SAFEXCEL_ENCRYPT) { 193 /* align end of instruction sequence to end of token */ 194 token = (struct safexcel_token *)(cdesc->control_data.token + 195 EIP197_MAX_TOKENS - 14); 196 197 token[13].opcode = EIP197_TOKEN_OPCODE_INSERT; 198 token[13].packet_length = digestsize; 199 token[13].stat = EIP197_TOKEN_STAT_LAST_HASH | 200 EIP197_TOKEN_STAT_LAST_PACKET; 201 token[13].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | 202 EIP197_TOKEN_INS_INSERT_HASH_DIGEST; 203 } else { 204 cryptlen -= digestsize; 205 206 /* align end of instruction sequence to end of token */ 207 token = (struct safexcel_token *)(cdesc->control_data.token + 208 EIP197_MAX_TOKENS - 15); 209 210 token[13].opcode = EIP197_TOKEN_OPCODE_RETRIEVE; 211 token[13].packet_length = digestsize; 212 token[13].stat = EIP197_TOKEN_STAT_LAST_HASH | 213 EIP197_TOKEN_STAT_LAST_PACKET; 214 token[13].instructions = EIP197_TOKEN_INS_INSERT_HASH_DIGEST; 215 216 token[14].opcode = EIP197_TOKEN_OPCODE_VERIFY; 217 token[14].packet_length = digestsize | 218 EIP197_TOKEN_HASH_RESULT_VERIFY; 219 token[14].stat = EIP197_TOKEN_STAT_LAST_HASH | 220 EIP197_TOKEN_STAT_LAST_PACKET; 221 token[14].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT; 222 } 223 224 if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { 225 /* For ESP mode (and not GMAC), skip over the IV */ 226 token[8].opcode = EIP197_TOKEN_OPCODE_DIRECTION; 227 token[8].packet_length = EIP197_AEAD_IPSEC_IV_SIZE; 228 229 assoclen -= EIP197_AEAD_IPSEC_IV_SIZE; 230 } 231 232 token[6].opcode = EIP197_TOKEN_OPCODE_DIRECTION; 233 token[6].packet_length = assoclen; 234 token[6].instructions = EIP197_TOKEN_INS_LAST | 235 EIP197_TOKEN_INS_TYPE_HASH; 236 237 if (likely(cryptlen || ctx->alg == SAFEXCEL_CHACHA20)) { 238 token[11].opcode = EIP197_TOKEN_OPCODE_DIRECTION; 239 token[11].packet_length = cryptlen; 240 token[11].stat = EIP197_TOKEN_STAT_LAST_HASH; 241 if (unlikely(ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP_GMAC)) { 242 token[6].instructions = EIP197_TOKEN_INS_TYPE_HASH; 243 /* Do not send to crypt engine in case of GMAC */ 244 token[11].instructions = EIP197_TOKEN_INS_LAST | 245 EIP197_TOKEN_INS_TYPE_HASH | 246 EIP197_TOKEN_INS_TYPE_OUTPUT; 247 } else { 248 token[11].instructions = EIP197_TOKEN_INS_LAST | 249 EIP197_TOKEN_INS_TYPE_CRYPTO | 250 EIP197_TOKEN_INS_TYPE_HASH | 251 EIP197_TOKEN_INS_TYPE_OUTPUT; 252 } 253 } else if (ctx->xcm != EIP197_XCM_MODE_CCM) { 254 token[6].stat = EIP197_TOKEN_STAT_LAST_HASH; 255 } 256 257 if (!ctx->xcm) 258 return; 259 260 token[9].opcode = EIP197_TOKEN_OPCODE_INSERT_REMRES; 261 token[9].packet_length = 0; 262 token[9].instructions = AES_BLOCK_SIZE; 263 264 token[10].opcode = EIP197_TOKEN_OPCODE_INSERT; 265 token[10].packet_length = AES_BLOCK_SIZE; 266 token[10].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | 267 EIP197_TOKEN_INS_TYPE_CRYPTO; 268 269 if (ctx->xcm != EIP197_XCM_MODE_GCM) { 270 u8 *final_iv = (u8 *)cdesc->control_data.token; 271 u8 *cbcmaciv = (u8 *)&token[1]; 272 __le32 *aadlen = (__le32 *)&token[5]; 273 274 /* Construct IV block B0 for the CBC-MAC */ 275 token[0].opcode = EIP197_TOKEN_OPCODE_INSERT; 276 token[0].packet_length = AES_BLOCK_SIZE + 277 ((assoclen > 0) << 1); 278 token[0].instructions = EIP197_TOKEN_INS_ORIGIN_TOKEN | 279 EIP197_TOKEN_INS_TYPE_HASH; 280 /* Variable length IV part */ 281 memcpy(cbcmaciv, final_iv, 15 - final_iv[0]); 282 /* fixup flags byte */ 283 cbcmaciv[0] |= ((assoclen > 0) << 6) | ((digestsize - 2) << 2); 284 /* Clear upper bytes of variable message length to 0 */ 285 memset(cbcmaciv + 15 - final_iv[0], 0, final_iv[0] - 1); 286 /* insert lower 2 bytes of message length */ 287 cbcmaciv[14] = cryptlen >> 8; 288 cbcmaciv[15] = cryptlen & 255; 289 290 if (assoclen) { 291 *aadlen = cpu_to_le32((assoclen >> 8) | 292 ((assoclen & 0xff) << 8)); 293 assoclen += 2; 294 } 295 296 token[6].instructions = EIP197_TOKEN_INS_TYPE_HASH; 297 298 /* Align AAD data towards hash engine */ 299 token[7].opcode = EIP197_TOKEN_OPCODE_INSERT; 300 assoclen &= 15; 301 token[7].packet_length = assoclen ? 16 - assoclen : 0; 302 303 if (likely(cryptlen)) { 304 token[7].instructions = EIP197_TOKEN_INS_TYPE_HASH; 305 306 /* Align crypto data towards hash engine */ 307 token[11].stat = 0; 308 309 token[12].opcode = EIP197_TOKEN_OPCODE_INSERT; 310 cryptlen &= 15; 311 token[12].packet_length = cryptlen ? 16 - cryptlen : 0; 312 token[12].stat = EIP197_TOKEN_STAT_LAST_HASH; 313 token[12].instructions = EIP197_TOKEN_INS_TYPE_HASH; 314 } else { 315 token[7].stat = EIP197_TOKEN_STAT_LAST_HASH; 316 token[7].instructions = EIP197_TOKEN_INS_LAST | 317 EIP197_TOKEN_INS_TYPE_HASH; 318 } 319 } 320 } 321 322 static int safexcel_skcipher_aes_setkey(struct crypto_skcipher *ctfm, 323 const u8 *key, unsigned int len) 324 { 325 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 326 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 327 struct safexcel_crypto_priv *priv = ctx->priv; 328 struct crypto_aes_ctx aes; 329 int ret, i; 330 331 ret = aes_expandkey(&aes, key, len); 332 if (ret) { 333 crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 334 return ret; 335 } 336 337 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 338 for (i = 0; i < len / sizeof(u32); i++) { 339 if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { 340 ctx->base.needs_inv = true; 341 break; 342 } 343 } 344 } 345 346 for (i = 0; i < len / sizeof(u32); i++) 347 ctx->key[i] = cpu_to_le32(aes.key_enc[i]); 348 349 ctx->key_len = len; 350 351 memzero_explicit(&aes, sizeof(aes)); 352 return 0; 353 } 354 355 static int safexcel_aead_setkey(struct crypto_aead *ctfm, const u8 *key, 356 unsigned int len) 357 { 358 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 359 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 360 struct safexcel_ahash_export_state istate, ostate; 361 struct safexcel_crypto_priv *priv = ctx->priv; 362 struct crypto_authenc_keys keys; 363 struct crypto_aes_ctx aes; 364 int err = -EINVAL, i; 365 366 if (unlikely(crypto_authenc_extractkeys(&keys, key, len))) 367 goto badkey; 368 369 if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD) { 370 /* Must have at least space for the nonce here */ 371 if (unlikely(keys.enckeylen < CTR_RFC3686_NONCE_SIZE)) 372 goto badkey; 373 /* last 4 bytes of key are the nonce! */ 374 ctx->nonce = *(u32 *)(keys.enckey + keys.enckeylen - 375 CTR_RFC3686_NONCE_SIZE); 376 /* exclude the nonce here */ 377 keys.enckeylen -= CTR_RFC3686_NONCE_SIZE; 378 } 379 380 /* Encryption key */ 381 switch (ctx->alg) { 382 case SAFEXCEL_DES: 383 err = verify_aead_des_key(ctfm, keys.enckey, keys.enckeylen); 384 if (unlikely(err)) 385 goto badkey_expflags; 386 break; 387 case SAFEXCEL_3DES: 388 err = verify_aead_des3_key(ctfm, keys.enckey, keys.enckeylen); 389 if (unlikely(err)) 390 goto badkey_expflags; 391 break; 392 case SAFEXCEL_AES: 393 err = aes_expandkey(&aes, keys.enckey, keys.enckeylen); 394 if (unlikely(err)) 395 goto badkey; 396 break; 397 case SAFEXCEL_SM4: 398 if (unlikely(keys.enckeylen != SM4_KEY_SIZE)) 399 goto badkey; 400 break; 401 default: 402 dev_err(priv->dev, "aead: unsupported cipher algorithm\n"); 403 goto badkey; 404 } 405 406 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 407 for (i = 0; i < keys.enckeylen / sizeof(u32); i++) { 408 if (le32_to_cpu(ctx->key[i]) != 409 ((u32 *)keys.enckey)[i]) { 410 ctx->base.needs_inv = true; 411 break; 412 } 413 } 414 } 415 416 /* Auth key */ 417 switch (ctx->hash_alg) { 418 case CONTEXT_CONTROL_CRYPTO_ALG_SHA1: 419 if (safexcel_hmac_setkey("safexcel-sha1", keys.authkey, 420 keys.authkeylen, &istate, &ostate)) 421 goto badkey; 422 break; 423 case CONTEXT_CONTROL_CRYPTO_ALG_SHA224: 424 if (safexcel_hmac_setkey("safexcel-sha224", keys.authkey, 425 keys.authkeylen, &istate, &ostate)) 426 goto badkey; 427 break; 428 case CONTEXT_CONTROL_CRYPTO_ALG_SHA256: 429 if (safexcel_hmac_setkey("safexcel-sha256", keys.authkey, 430 keys.authkeylen, &istate, &ostate)) 431 goto badkey; 432 break; 433 case CONTEXT_CONTROL_CRYPTO_ALG_SHA384: 434 if (safexcel_hmac_setkey("safexcel-sha384", keys.authkey, 435 keys.authkeylen, &istate, &ostate)) 436 goto badkey; 437 break; 438 case CONTEXT_CONTROL_CRYPTO_ALG_SHA512: 439 if (safexcel_hmac_setkey("safexcel-sha512", keys.authkey, 440 keys.authkeylen, &istate, &ostate)) 441 goto badkey; 442 break; 443 case CONTEXT_CONTROL_CRYPTO_ALG_SM3: 444 if (safexcel_hmac_setkey("safexcel-sm3", keys.authkey, 445 keys.authkeylen, &istate, &ostate)) 446 goto badkey; 447 break; 448 default: 449 dev_err(priv->dev, "aead: unsupported hash algorithm\n"); 450 goto badkey; 451 } 452 453 crypto_aead_set_flags(ctfm, crypto_aead_get_flags(ctfm) & 454 CRYPTO_TFM_RES_MASK); 455 456 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma && 457 (memcmp(ctx->ipad, istate.state, ctx->state_sz) || 458 memcmp(ctx->opad, ostate.state, ctx->state_sz))) 459 ctx->base.needs_inv = true; 460 461 /* Now copy the keys into the context */ 462 for (i = 0; i < keys.enckeylen / sizeof(u32); i++) 463 ctx->key[i] = cpu_to_le32(((u32 *)keys.enckey)[i]); 464 ctx->key_len = keys.enckeylen; 465 466 memcpy(ctx->ipad, &istate.state, ctx->state_sz); 467 memcpy(ctx->opad, &ostate.state, ctx->state_sz); 468 469 memzero_explicit(&keys, sizeof(keys)); 470 return 0; 471 472 badkey: 473 crypto_aead_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 474 badkey_expflags: 475 memzero_explicit(&keys, sizeof(keys)); 476 return err; 477 } 478 479 static int safexcel_context_control(struct safexcel_cipher_ctx *ctx, 480 struct crypto_async_request *async, 481 struct safexcel_cipher_req *sreq, 482 struct safexcel_command_desc *cdesc) 483 { 484 struct safexcel_crypto_priv *priv = ctx->priv; 485 int ctrl_size = ctx->key_len / sizeof(u32); 486 487 cdesc->control_data.control1 = ctx->mode; 488 489 if (ctx->aead) { 490 /* Take in account the ipad+opad digests */ 491 if (ctx->xcm) { 492 ctrl_size += ctx->state_sz / sizeof(u32); 493 cdesc->control_data.control0 = 494 CONTEXT_CONTROL_KEY_EN | 495 CONTEXT_CONTROL_DIGEST_XCM | 496 ctx->hash_alg | 497 CONTEXT_CONTROL_SIZE(ctrl_size); 498 } else if (ctx->alg == SAFEXCEL_CHACHA20) { 499 /* Chacha20-Poly1305 */ 500 cdesc->control_data.control0 = 501 CONTEXT_CONTROL_KEY_EN | 502 CONTEXT_CONTROL_CRYPTO_ALG_CHACHA20 | 503 (sreq->direction == SAFEXCEL_ENCRYPT ? 504 CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT : 505 CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN) | 506 ctx->hash_alg | 507 CONTEXT_CONTROL_SIZE(ctrl_size); 508 return 0; 509 } else { 510 ctrl_size += ctx->state_sz / sizeof(u32) * 2; 511 cdesc->control_data.control0 = 512 CONTEXT_CONTROL_KEY_EN | 513 CONTEXT_CONTROL_DIGEST_HMAC | 514 ctx->hash_alg | 515 CONTEXT_CONTROL_SIZE(ctrl_size); 516 } 517 518 if (sreq->direction == SAFEXCEL_ENCRYPT && 519 (ctx->xcm == EIP197_XCM_MODE_CCM || 520 ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP_GMAC)) 521 cdesc->control_data.control0 |= 522 CONTEXT_CONTROL_TYPE_HASH_ENCRYPT_OUT; 523 else if (sreq->direction == SAFEXCEL_ENCRYPT) 524 cdesc->control_data.control0 |= 525 CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT; 526 else if (ctx->xcm == EIP197_XCM_MODE_CCM) 527 cdesc->control_data.control0 |= 528 CONTEXT_CONTROL_TYPE_DECRYPT_HASH_IN; 529 else 530 cdesc->control_data.control0 |= 531 CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN; 532 } else { 533 if (sreq->direction == SAFEXCEL_ENCRYPT) 534 cdesc->control_data.control0 = 535 CONTEXT_CONTROL_TYPE_CRYPTO_OUT | 536 CONTEXT_CONTROL_KEY_EN | 537 CONTEXT_CONTROL_SIZE(ctrl_size); 538 else 539 cdesc->control_data.control0 = 540 CONTEXT_CONTROL_TYPE_CRYPTO_IN | 541 CONTEXT_CONTROL_KEY_EN | 542 CONTEXT_CONTROL_SIZE(ctrl_size); 543 } 544 545 if (ctx->alg == SAFEXCEL_DES) { 546 cdesc->control_data.control0 |= 547 CONTEXT_CONTROL_CRYPTO_ALG_DES; 548 } else if (ctx->alg == SAFEXCEL_3DES) { 549 cdesc->control_data.control0 |= 550 CONTEXT_CONTROL_CRYPTO_ALG_3DES; 551 } else if (ctx->alg == SAFEXCEL_AES) { 552 switch (ctx->key_len >> ctx->xts) { 553 case AES_KEYSIZE_128: 554 cdesc->control_data.control0 |= 555 CONTEXT_CONTROL_CRYPTO_ALG_AES128; 556 break; 557 case AES_KEYSIZE_192: 558 cdesc->control_data.control0 |= 559 CONTEXT_CONTROL_CRYPTO_ALG_AES192; 560 break; 561 case AES_KEYSIZE_256: 562 cdesc->control_data.control0 |= 563 CONTEXT_CONTROL_CRYPTO_ALG_AES256; 564 break; 565 default: 566 dev_err(priv->dev, "aes keysize not supported: %u\n", 567 ctx->key_len >> ctx->xts); 568 return -EINVAL; 569 } 570 } else if (ctx->alg == SAFEXCEL_CHACHA20) { 571 cdesc->control_data.control0 |= 572 CONTEXT_CONTROL_CRYPTO_ALG_CHACHA20; 573 } else if (ctx->alg == SAFEXCEL_SM4) { 574 cdesc->control_data.control0 |= 575 CONTEXT_CONTROL_CRYPTO_ALG_SM4; 576 } 577 578 return 0; 579 } 580 581 static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int ring, 582 struct crypto_async_request *async, 583 struct scatterlist *src, 584 struct scatterlist *dst, 585 unsigned int cryptlen, 586 struct safexcel_cipher_req *sreq, 587 bool *should_complete, int *ret) 588 { 589 struct skcipher_request *areq = skcipher_request_cast(async); 590 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(areq); 591 struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(skcipher); 592 struct safexcel_result_desc *rdesc; 593 int ndesc = 0; 594 595 *ret = 0; 596 597 if (unlikely(!sreq->rdescs)) 598 return 0; 599 600 while (sreq->rdescs--) { 601 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); 602 if (IS_ERR(rdesc)) { 603 dev_err(priv->dev, 604 "cipher: result: could not retrieve the result descriptor\n"); 605 *ret = PTR_ERR(rdesc); 606 break; 607 } 608 609 if (likely(!*ret)) 610 *ret = safexcel_rdesc_check_errors(priv, rdesc); 611 612 ndesc++; 613 } 614 615 safexcel_complete(priv, ring); 616 617 if (src == dst) { 618 dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL); 619 } else { 620 dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE); 621 dma_unmap_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE); 622 } 623 624 /* 625 * Update IV in req from last crypto output word for CBC modes 626 */ 627 if ((!ctx->aead) && (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) && 628 (sreq->direction == SAFEXCEL_ENCRYPT)) { 629 /* For encrypt take the last output word */ 630 sg_pcopy_to_buffer(dst, sreq->nr_dst, areq->iv, 631 crypto_skcipher_ivsize(skcipher), 632 (cryptlen - 633 crypto_skcipher_ivsize(skcipher))); 634 } 635 636 *should_complete = true; 637 638 return ndesc; 639 } 640 641 static int safexcel_send_req(struct crypto_async_request *base, int ring, 642 struct safexcel_cipher_req *sreq, 643 struct scatterlist *src, struct scatterlist *dst, 644 unsigned int cryptlen, unsigned int assoclen, 645 unsigned int digestsize, u8 *iv, int *commands, 646 int *results) 647 { 648 struct skcipher_request *areq = skcipher_request_cast(base); 649 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(areq); 650 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); 651 struct safexcel_crypto_priv *priv = ctx->priv; 652 struct safexcel_command_desc *cdesc; 653 struct safexcel_command_desc *first_cdesc = NULL; 654 struct safexcel_result_desc *rdesc, *first_rdesc = NULL; 655 struct scatterlist *sg; 656 unsigned int totlen; 657 unsigned int totlen_src = cryptlen + assoclen; 658 unsigned int totlen_dst = totlen_src; 659 int n_cdesc = 0, n_rdesc = 0; 660 int queued, i, ret = 0; 661 bool first = true; 662 663 sreq->nr_src = sg_nents_for_len(src, totlen_src); 664 665 if (ctx->aead) { 666 /* 667 * AEAD has auth tag appended to output for encrypt and 668 * removed from the output for decrypt! 669 */ 670 if (sreq->direction == SAFEXCEL_DECRYPT) 671 totlen_dst -= digestsize; 672 else 673 totlen_dst += digestsize; 674 675 memcpy(ctx->base.ctxr->data + ctx->key_len / sizeof(u32), 676 ctx->ipad, ctx->state_sz); 677 if (!ctx->xcm) 678 memcpy(ctx->base.ctxr->data + (ctx->key_len + 679 ctx->state_sz) / sizeof(u32), ctx->opad, 680 ctx->state_sz); 681 } else if ((ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) && 682 (sreq->direction == SAFEXCEL_DECRYPT)) { 683 /* 684 * Save IV from last crypto input word for CBC modes in decrypt 685 * direction. Need to do this first in case of inplace operation 686 * as it will be overwritten. 687 */ 688 sg_pcopy_to_buffer(src, sreq->nr_src, areq->iv, 689 crypto_skcipher_ivsize(skcipher), 690 (totlen_src - 691 crypto_skcipher_ivsize(skcipher))); 692 } 693 694 sreq->nr_dst = sg_nents_for_len(dst, totlen_dst); 695 696 /* 697 * Remember actual input length, source buffer length may be 698 * updated in case of inline operation below. 699 */ 700 totlen = totlen_src; 701 queued = totlen_src; 702 703 if (src == dst) { 704 sreq->nr_src = max(sreq->nr_src, sreq->nr_dst); 705 sreq->nr_dst = sreq->nr_src; 706 if (unlikely((totlen_src || totlen_dst) && 707 (sreq->nr_src <= 0))) { 708 dev_err(priv->dev, "In-place buffer not large enough (need %d bytes)!", 709 max(totlen_src, totlen_dst)); 710 return -EINVAL; 711 } 712 dma_map_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL); 713 } else { 714 if (unlikely(totlen_src && (sreq->nr_src <= 0))) { 715 dev_err(priv->dev, "Source buffer not large enough (need %d bytes)!", 716 totlen_src); 717 return -EINVAL; 718 } 719 dma_map_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE); 720 721 if (unlikely(totlen_dst && (sreq->nr_dst <= 0))) { 722 dev_err(priv->dev, "Dest buffer not large enough (need %d bytes)!", 723 totlen_dst); 724 dma_unmap_sg(priv->dev, src, sreq->nr_src, 725 DMA_TO_DEVICE); 726 return -EINVAL; 727 } 728 dma_map_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE); 729 } 730 731 memcpy(ctx->base.ctxr->data, ctx->key, ctx->key_len); 732 733 /* The EIP cannot deal with zero length input packets! */ 734 if (totlen == 0) 735 totlen = 1; 736 737 /* command descriptors */ 738 for_each_sg(src, sg, sreq->nr_src, i) { 739 int len = sg_dma_len(sg); 740 741 /* Do not overflow the request */ 742 if (queued - len < 0) 743 len = queued; 744 745 cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc, 746 !(queued - len), 747 sg_dma_address(sg), len, totlen, 748 ctx->base.ctxr_dma); 749 if (IS_ERR(cdesc)) { 750 /* No space left in the command descriptor ring */ 751 ret = PTR_ERR(cdesc); 752 goto cdesc_rollback; 753 } 754 n_cdesc++; 755 756 if (n_cdesc == 1) { 757 first_cdesc = cdesc; 758 } 759 760 queued -= len; 761 if (!queued) 762 break; 763 } 764 765 if (unlikely(!n_cdesc)) { 766 /* 767 * Special case: zero length input buffer. 768 * The engine always needs the 1st command descriptor, however! 769 */ 770 first_cdesc = safexcel_add_cdesc(priv, ring, 1, 1, 0, 0, totlen, 771 ctx->base.ctxr_dma); 772 n_cdesc = 1; 773 } 774 775 /* Add context control words and token to first command descriptor */ 776 safexcel_context_control(ctx, base, sreq, first_cdesc); 777 if (ctx->aead) 778 safexcel_aead_token(ctx, iv, first_cdesc, 779 sreq->direction, cryptlen, 780 assoclen, digestsize); 781 else 782 safexcel_skcipher_token(ctx, iv, first_cdesc, 783 cryptlen); 784 785 /* result descriptors */ 786 for_each_sg(dst, sg, sreq->nr_dst, i) { 787 bool last = (i == sreq->nr_dst - 1); 788 u32 len = sg_dma_len(sg); 789 790 /* only allow the part of the buffer we know we need */ 791 if (len > totlen_dst) 792 len = totlen_dst; 793 if (unlikely(!len)) 794 break; 795 totlen_dst -= len; 796 797 /* skip over AAD space in buffer - not written */ 798 if (assoclen) { 799 if (assoclen >= len) { 800 assoclen -= len; 801 continue; 802 } 803 rdesc = safexcel_add_rdesc(priv, ring, first, last, 804 sg_dma_address(sg) + 805 assoclen, 806 len - assoclen); 807 assoclen = 0; 808 } else { 809 rdesc = safexcel_add_rdesc(priv, ring, first, last, 810 sg_dma_address(sg), 811 len); 812 } 813 if (IS_ERR(rdesc)) { 814 /* No space left in the result descriptor ring */ 815 ret = PTR_ERR(rdesc); 816 goto rdesc_rollback; 817 } 818 if (first) { 819 first_rdesc = rdesc; 820 first = false; 821 } 822 n_rdesc++; 823 } 824 825 if (unlikely(first)) { 826 /* 827 * Special case: AEAD decrypt with only AAD data. 828 * In this case there is NO output data from the engine, 829 * but the engine still needs a result descriptor! 830 * Create a dummy one just for catching the result token. 831 */ 832 rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0); 833 if (IS_ERR(rdesc)) { 834 /* No space left in the result descriptor ring */ 835 ret = PTR_ERR(rdesc); 836 goto rdesc_rollback; 837 } 838 first_rdesc = rdesc; 839 n_rdesc = 1; 840 } 841 842 safexcel_rdr_req_set(priv, ring, first_rdesc, base); 843 844 *commands = n_cdesc; 845 *results = n_rdesc; 846 return 0; 847 848 rdesc_rollback: 849 for (i = 0; i < n_rdesc; i++) 850 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].rdr); 851 cdesc_rollback: 852 for (i = 0; i < n_cdesc; i++) 853 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr); 854 855 if (src == dst) { 856 dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL); 857 } else { 858 dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE); 859 dma_unmap_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE); 860 } 861 862 return ret; 863 } 864 865 static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv, 866 int ring, 867 struct crypto_async_request *base, 868 struct safexcel_cipher_req *sreq, 869 bool *should_complete, int *ret) 870 { 871 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); 872 struct safexcel_result_desc *rdesc; 873 int ndesc = 0, enq_ret; 874 875 *ret = 0; 876 877 if (unlikely(!sreq->rdescs)) 878 return 0; 879 880 while (sreq->rdescs--) { 881 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); 882 if (IS_ERR(rdesc)) { 883 dev_err(priv->dev, 884 "cipher: invalidate: could not retrieve the result descriptor\n"); 885 *ret = PTR_ERR(rdesc); 886 break; 887 } 888 889 if (likely(!*ret)) 890 *ret = safexcel_rdesc_check_errors(priv, rdesc); 891 892 ndesc++; 893 } 894 895 safexcel_complete(priv, ring); 896 897 if (ctx->base.exit_inv) { 898 dma_pool_free(priv->context_pool, ctx->base.ctxr, 899 ctx->base.ctxr_dma); 900 901 *should_complete = true; 902 903 return ndesc; 904 } 905 906 ring = safexcel_select_ring(priv); 907 ctx->base.ring = ring; 908 909 spin_lock_bh(&priv->ring[ring].queue_lock); 910 enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, base); 911 spin_unlock_bh(&priv->ring[ring].queue_lock); 912 913 if (enq_ret != -EINPROGRESS) 914 *ret = enq_ret; 915 916 queue_work(priv->ring[ring].workqueue, 917 &priv->ring[ring].work_data.work); 918 919 *should_complete = false; 920 921 return ndesc; 922 } 923 924 static int safexcel_skcipher_handle_result(struct safexcel_crypto_priv *priv, 925 int ring, 926 struct crypto_async_request *async, 927 bool *should_complete, int *ret) 928 { 929 struct skcipher_request *req = skcipher_request_cast(async); 930 struct safexcel_cipher_req *sreq = skcipher_request_ctx(req); 931 int err; 932 933 if (sreq->needs_inv) { 934 sreq->needs_inv = false; 935 err = safexcel_handle_inv_result(priv, ring, async, sreq, 936 should_complete, ret); 937 } else { 938 err = safexcel_handle_req_result(priv, ring, async, req->src, 939 req->dst, req->cryptlen, sreq, 940 should_complete, ret); 941 } 942 943 return err; 944 } 945 946 static int safexcel_aead_handle_result(struct safexcel_crypto_priv *priv, 947 int ring, 948 struct crypto_async_request *async, 949 bool *should_complete, int *ret) 950 { 951 struct aead_request *req = aead_request_cast(async); 952 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 953 struct safexcel_cipher_req *sreq = aead_request_ctx(req); 954 int err; 955 956 if (sreq->needs_inv) { 957 sreq->needs_inv = false; 958 err = safexcel_handle_inv_result(priv, ring, async, sreq, 959 should_complete, ret); 960 } else { 961 err = safexcel_handle_req_result(priv, ring, async, req->src, 962 req->dst, 963 req->cryptlen + crypto_aead_authsize(tfm), 964 sreq, should_complete, ret); 965 } 966 967 return err; 968 } 969 970 static int safexcel_cipher_send_inv(struct crypto_async_request *base, 971 int ring, int *commands, int *results) 972 { 973 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); 974 struct safexcel_crypto_priv *priv = ctx->priv; 975 int ret; 976 977 ret = safexcel_invalidate_cache(base, priv, ctx->base.ctxr_dma, ring); 978 if (unlikely(ret)) 979 return ret; 980 981 *commands = 1; 982 *results = 1; 983 984 return 0; 985 } 986 987 static int safexcel_skcipher_send(struct crypto_async_request *async, int ring, 988 int *commands, int *results) 989 { 990 struct skcipher_request *req = skcipher_request_cast(async); 991 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); 992 struct safexcel_cipher_req *sreq = skcipher_request_ctx(req); 993 struct safexcel_crypto_priv *priv = ctx->priv; 994 int ret; 995 996 BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv); 997 998 if (sreq->needs_inv) { 999 ret = safexcel_cipher_send_inv(async, ring, commands, results); 1000 } else { 1001 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1002 u8 input_iv[AES_BLOCK_SIZE]; 1003 1004 /* 1005 * Save input IV in case of CBC decrypt mode 1006 * Will be overwritten with output IV prior to use! 1007 */ 1008 memcpy(input_iv, req->iv, crypto_skcipher_ivsize(skcipher)); 1009 1010 ret = safexcel_send_req(async, ring, sreq, req->src, 1011 req->dst, req->cryptlen, 0, 0, input_iv, 1012 commands, results); 1013 } 1014 1015 sreq->rdescs = *results; 1016 return ret; 1017 } 1018 1019 static int safexcel_aead_send(struct crypto_async_request *async, int ring, 1020 int *commands, int *results) 1021 { 1022 struct aead_request *req = aead_request_cast(async); 1023 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1024 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); 1025 struct safexcel_cipher_req *sreq = aead_request_ctx(req); 1026 struct safexcel_crypto_priv *priv = ctx->priv; 1027 int ret; 1028 1029 BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv); 1030 1031 if (sreq->needs_inv) 1032 ret = safexcel_cipher_send_inv(async, ring, commands, results); 1033 else 1034 ret = safexcel_send_req(async, ring, sreq, req->src, req->dst, 1035 req->cryptlen, req->assoclen, 1036 crypto_aead_authsize(tfm), req->iv, 1037 commands, results); 1038 sreq->rdescs = *results; 1039 return ret; 1040 } 1041 1042 static int safexcel_cipher_exit_inv(struct crypto_tfm *tfm, 1043 struct crypto_async_request *base, 1044 struct safexcel_cipher_req *sreq, 1045 struct safexcel_inv_result *result) 1046 { 1047 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1048 struct safexcel_crypto_priv *priv = ctx->priv; 1049 int ring = ctx->base.ring; 1050 1051 init_completion(&result->completion); 1052 1053 ctx = crypto_tfm_ctx(base->tfm); 1054 ctx->base.exit_inv = true; 1055 sreq->needs_inv = true; 1056 1057 spin_lock_bh(&priv->ring[ring].queue_lock); 1058 crypto_enqueue_request(&priv->ring[ring].queue, base); 1059 spin_unlock_bh(&priv->ring[ring].queue_lock); 1060 1061 queue_work(priv->ring[ring].workqueue, 1062 &priv->ring[ring].work_data.work); 1063 1064 wait_for_completion(&result->completion); 1065 1066 if (result->error) { 1067 dev_warn(priv->dev, 1068 "cipher: sync: invalidate: completion error %d\n", 1069 result->error); 1070 return result->error; 1071 } 1072 1073 return 0; 1074 } 1075 1076 static int safexcel_skcipher_exit_inv(struct crypto_tfm *tfm) 1077 { 1078 EIP197_REQUEST_ON_STACK(req, skcipher, EIP197_SKCIPHER_REQ_SIZE); 1079 struct safexcel_cipher_req *sreq = skcipher_request_ctx(req); 1080 struct safexcel_inv_result result = {}; 1081 1082 memset(req, 0, sizeof(struct skcipher_request)); 1083 1084 skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 1085 safexcel_inv_complete, &result); 1086 skcipher_request_set_tfm(req, __crypto_skcipher_cast(tfm)); 1087 1088 return safexcel_cipher_exit_inv(tfm, &req->base, sreq, &result); 1089 } 1090 1091 static int safexcel_aead_exit_inv(struct crypto_tfm *tfm) 1092 { 1093 EIP197_REQUEST_ON_STACK(req, aead, EIP197_AEAD_REQ_SIZE); 1094 struct safexcel_cipher_req *sreq = aead_request_ctx(req); 1095 struct safexcel_inv_result result = {}; 1096 1097 memset(req, 0, sizeof(struct aead_request)); 1098 1099 aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 1100 safexcel_inv_complete, &result); 1101 aead_request_set_tfm(req, __crypto_aead_cast(tfm)); 1102 1103 return safexcel_cipher_exit_inv(tfm, &req->base, sreq, &result); 1104 } 1105 1106 static int safexcel_queue_req(struct crypto_async_request *base, 1107 struct safexcel_cipher_req *sreq, 1108 enum safexcel_cipher_direction dir) 1109 { 1110 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); 1111 struct safexcel_crypto_priv *priv = ctx->priv; 1112 int ret, ring; 1113 1114 sreq->needs_inv = false; 1115 sreq->direction = dir; 1116 1117 if (ctx->base.ctxr) { 1118 if (priv->flags & EIP197_TRC_CACHE && ctx->base.needs_inv) { 1119 sreq->needs_inv = true; 1120 ctx->base.needs_inv = false; 1121 } 1122 } else { 1123 ctx->base.ring = safexcel_select_ring(priv); 1124 ctx->base.ctxr = dma_pool_zalloc(priv->context_pool, 1125 EIP197_GFP_FLAGS(*base), 1126 &ctx->base.ctxr_dma); 1127 if (!ctx->base.ctxr) 1128 return -ENOMEM; 1129 } 1130 1131 ring = ctx->base.ring; 1132 1133 spin_lock_bh(&priv->ring[ring].queue_lock); 1134 ret = crypto_enqueue_request(&priv->ring[ring].queue, base); 1135 spin_unlock_bh(&priv->ring[ring].queue_lock); 1136 1137 queue_work(priv->ring[ring].workqueue, 1138 &priv->ring[ring].work_data.work); 1139 1140 return ret; 1141 } 1142 1143 static int safexcel_encrypt(struct skcipher_request *req) 1144 { 1145 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 1146 SAFEXCEL_ENCRYPT); 1147 } 1148 1149 static int safexcel_decrypt(struct skcipher_request *req) 1150 { 1151 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 1152 SAFEXCEL_DECRYPT); 1153 } 1154 1155 static int safexcel_skcipher_cra_init(struct crypto_tfm *tfm) 1156 { 1157 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1158 struct safexcel_alg_template *tmpl = 1159 container_of(tfm->__crt_alg, struct safexcel_alg_template, 1160 alg.skcipher.base); 1161 1162 crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm), 1163 sizeof(struct safexcel_cipher_req)); 1164 1165 ctx->priv = tmpl->priv; 1166 1167 ctx->base.send = safexcel_skcipher_send; 1168 ctx->base.handle_result = safexcel_skcipher_handle_result; 1169 return 0; 1170 } 1171 1172 static int safexcel_cipher_cra_exit(struct crypto_tfm *tfm) 1173 { 1174 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1175 1176 memzero_explicit(ctx->key, sizeof(ctx->key)); 1177 1178 /* context not allocated, skip invalidation */ 1179 if (!ctx->base.ctxr) 1180 return -ENOMEM; 1181 1182 memzero_explicit(ctx->base.ctxr->data, sizeof(ctx->base.ctxr->data)); 1183 return 0; 1184 } 1185 1186 static void safexcel_skcipher_cra_exit(struct crypto_tfm *tfm) 1187 { 1188 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1189 struct safexcel_crypto_priv *priv = ctx->priv; 1190 int ret; 1191 1192 if (safexcel_cipher_cra_exit(tfm)) 1193 return; 1194 1195 if (priv->flags & EIP197_TRC_CACHE) { 1196 ret = safexcel_skcipher_exit_inv(tfm); 1197 if (ret) 1198 dev_warn(priv->dev, "skcipher: invalidation error %d\n", 1199 ret); 1200 } else { 1201 dma_pool_free(priv->context_pool, ctx->base.ctxr, 1202 ctx->base.ctxr_dma); 1203 } 1204 } 1205 1206 static void safexcel_aead_cra_exit(struct crypto_tfm *tfm) 1207 { 1208 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1209 struct safexcel_crypto_priv *priv = ctx->priv; 1210 int ret; 1211 1212 if (safexcel_cipher_cra_exit(tfm)) 1213 return; 1214 1215 if (priv->flags & EIP197_TRC_CACHE) { 1216 ret = safexcel_aead_exit_inv(tfm); 1217 if (ret) 1218 dev_warn(priv->dev, "aead: invalidation error %d\n", 1219 ret); 1220 } else { 1221 dma_pool_free(priv->context_pool, ctx->base.ctxr, 1222 ctx->base.ctxr_dma); 1223 } 1224 } 1225 1226 static int safexcel_skcipher_aes_ecb_cra_init(struct crypto_tfm *tfm) 1227 { 1228 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1229 1230 safexcel_skcipher_cra_init(tfm); 1231 ctx->alg = SAFEXCEL_AES; 1232 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; 1233 return 0; 1234 } 1235 1236 struct safexcel_alg_template safexcel_alg_ecb_aes = { 1237 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1238 .algo_mask = SAFEXCEL_ALG_AES, 1239 .alg.skcipher = { 1240 .setkey = safexcel_skcipher_aes_setkey, 1241 .encrypt = safexcel_encrypt, 1242 .decrypt = safexcel_decrypt, 1243 .min_keysize = AES_MIN_KEY_SIZE, 1244 .max_keysize = AES_MAX_KEY_SIZE, 1245 .base = { 1246 .cra_name = "ecb(aes)", 1247 .cra_driver_name = "safexcel-ecb-aes", 1248 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1249 .cra_flags = CRYPTO_ALG_ASYNC | 1250 CRYPTO_ALG_KERN_DRIVER_ONLY, 1251 .cra_blocksize = AES_BLOCK_SIZE, 1252 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1253 .cra_alignmask = 0, 1254 .cra_init = safexcel_skcipher_aes_ecb_cra_init, 1255 .cra_exit = safexcel_skcipher_cra_exit, 1256 .cra_module = THIS_MODULE, 1257 }, 1258 }, 1259 }; 1260 1261 static int safexcel_skcipher_aes_cbc_cra_init(struct crypto_tfm *tfm) 1262 { 1263 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1264 1265 safexcel_skcipher_cra_init(tfm); 1266 ctx->alg = SAFEXCEL_AES; 1267 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; 1268 return 0; 1269 } 1270 1271 struct safexcel_alg_template safexcel_alg_cbc_aes = { 1272 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1273 .algo_mask = SAFEXCEL_ALG_AES, 1274 .alg.skcipher = { 1275 .setkey = safexcel_skcipher_aes_setkey, 1276 .encrypt = safexcel_encrypt, 1277 .decrypt = safexcel_decrypt, 1278 .min_keysize = AES_MIN_KEY_SIZE, 1279 .max_keysize = AES_MAX_KEY_SIZE, 1280 .ivsize = AES_BLOCK_SIZE, 1281 .base = { 1282 .cra_name = "cbc(aes)", 1283 .cra_driver_name = "safexcel-cbc-aes", 1284 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1285 .cra_flags = CRYPTO_ALG_ASYNC | 1286 CRYPTO_ALG_KERN_DRIVER_ONLY, 1287 .cra_blocksize = AES_BLOCK_SIZE, 1288 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1289 .cra_alignmask = 0, 1290 .cra_init = safexcel_skcipher_aes_cbc_cra_init, 1291 .cra_exit = safexcel_skcipher_cra_exit, 1292 .cra_module = THIS_MODULE, 1293 }, 1294 }, 1295 }; 1296 1297 static int safexcel_skcipher_aes_cfb_cra_init(struct crypto_tfm *tfm) 1298 { 1299 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1300 1301 safexcel_skcipher_cra_init(tfm); 1302 ctx->alg = SAFEXCEL_AES; 1303 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CFB; 1304 return 0; 1305 } 1306 1307 struct safexcel_alg_template safexcel_alg_cfb_aes = { 1308 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1309 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_AES_XFB, 1310 .alg.skcipher = { 1311 .setkey = safexcel_skcipher_aes_setkey, 1312 .encrypt = safexcel_encrypt, 1313 .decrypt = safexcel_decrypt, 1314 .min_keysize = AES_MIN_KEY_SIZE, 1315 .max_keysize = AES_MAX_KEY_SIZE, 1316 .ivsize = AES_BLOCK_SIZE, 1317 .base = { 1318 .cra_name = "cfb(aes)", 1319 .cra_driver_name = "safexcel-cfb-aes", 1320 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1321 .cra_flags = CRYPTO_ALG_ASYNC | 1322 CRYPTO_ALG_KERN_DRIVER_ONLY, 1323 .cra_blocksize = 1, 1324 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1325 .cra_alignmask = 0, 1326 .cra_init = safexcel_skcipher_aes_cfb_cra_init, 1327 .cra_exit = safexcel_skcipher_cra_exit, 1328 .cra_module = THIS_MODULE, 1329 }, 1330 }, 1331 }; 1332 1333 static int safexcel_skcipher_aes_ofb_cra_init(struct crypto_tfm *tfm) 1334 { 1335 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1336 1337 safexcel_skcipher_cra_init(tfm); 1338 ctx->alg = SAFEXCEL_AES; 1339 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_OFB; 1340 return 0; 1341 } 1342 1343 struct safexcel_alg_template safexcel_alg_ofb_aes = { 1344 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1345 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_AES_XFB, 1346 .alg.skcipher = { 1347 .setkey = safexcel_skcipher_aes_setkey, 1348 .encrypt = safexcel_encrypt, 1349 .decrypt = safexcel_decrypt, 1350 .min_keysize = AES_MIN_KEY_SIZE, 1351 .max_keysize = AES_MAX_KEY_SIZE, 1352 .ivsize = AES_BLOCK_SIZE, 1353 .base = { 1354 .cra_name = "ofb(aes)", 1355 .cra_driver_name = "safexcel-ofb-aes", 1356 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1357 .cra_flags = CRYPTO_ALG_ASYNC | 1358 CRYPTO_ALG_KERN_DRIVER_ONLY, 1359 .cra_blocksize = 1, 1360 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1361 .cra_alignmask = 0, 1362 .cra_init = safexcel_skcipher_aes_ofb_cra_init, 1363 .cra_exit = safexcel_skcipher_cra_exit, 1364 .cra_module = THIS_MODULE, 1365 }, 1366 }, 1367 }; 1368 1369 static int safexcel_skcipher_aesctr_setkey(struct crypto_skcipher *ctfm, 1370 const u8 *key, unsigned int len) 1371 { 1372 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 1373 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1374 struct safexcel_crypto_priv *priv = ctx->priv; 1375 struct crypto_aes_ctx aes; 1376 int ret, i; 1377 unsigned int keylen; 1378 1379 /* last 4 bytes of key are the nonce! */ 1380 ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE); 1381 /* exclude the nonce here */ 1382 keylen = len - CTR_RFC3686_NONCE_SIZE; 1383 ret = aes_expandkey(&aes, key, keylen); 1384 if (ret) { 1385 crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 1386 return ret; 1387 } 1388 1389 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 1390 for (i = 0; i < keylen / sizeof(u32); i++) { 1391 if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { 1392 ctx->base.needs_inv = true; 1393 break; 1394 } 1395 } 1396 } 1397 1398 for (i = 0; i < keylen / sizeof(u32); i++) 1399 ctx->key[i] = cpu_to_le32(aes.key_enc[i]); 1400 1401 ctx->key_len = keylen; 1402 1403 memzero_explicit(&aes, sizeof(aes)); 1404 return 0; 1405 } 1406 1407 static int safexcel_skcipher_aes_ctr_cra_init(struct crypto_tfm *tfm) 1408 { 1409 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1410 1411 safexcel_skcipher_cra_init(tfm); 1412 ctx->alg = SAFEXCEL_AES; 1413 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; 1414 return 0; 1415 } 1416 1417 struct safexcel_alg_template safexcel_alg_ctr_aes = { 1418 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1419 .algo_mask = SAFEXCEL_ALG_AES, 1420 .alg.skcipher = { 1421 .setkey = safexcel_skcipher_aesctr_setkey, 1422 .encrypt = safexcel_encrypt, 1423 .decrypt = safexcel_decrypt, 1424 /* Add nonce size */ 1425 .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 1426 .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 1427 .ivsize = CTR_RFC3686_IV_SIZE, 1428 .base = { 1429 .cra_name = "rfc3686(ctr(aes))", 1430 .cra_driver_name = "safexcel-ctr-aes", 1431 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1432 .cra_flags = CRYPTO_ALG_ASYNC | 1433 CRYPTO_ALG_KERN_DRIVER_ONLY, 1434 .cra_blocksize = 1, 1435 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1436 .cra_alignmask = 0, 1437 .cra_init = safexcel_skcipher_aes_ctr_cra_init, 1438 .cra_exit = safexcel_skcipher_cra_exit, 1439 .cra_module = THIS_MODULE, 1440 }, 1441 }, 1442 }; 1443 1444 static int safexcel_des_setkey(struct crypto_skcipher *ctfm, const u8 *key, 1445 unsigned int len) 1446 { 1447 struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm); 1448 int ret; 1449 1450 ret = verify_skcipher_des_key(ctfm, key); 1451 if (ret) 1452 return ret; 1453 1454 /* if context exits and key changed, need to invalidate it */ 1455 if (ctx->base.ctxr_dma) 1456 if (memcmp(ctx->key, key, len)) 1457 ctx->base.needs_inv = true; 1458 1459 memcpy(ctx->key, key, len); 1460 ctx->key_len = len; 1461 1462 return 0; 1463 } 1464 1465 static int safexcel_skcipher_des_cbc_cra_init(struct crypto_tfm *tfm) 1466 { 1467 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1468 1469 safexcel_skcipher_cra_init(tfm); 1470 ctx->alg = SAFEXCEL_DES; 1471 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; 1472 return 0; 1473 } 1474 1475 struct safexcel_alg_template safexcel_alg_cbc_des = { 1476 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1477 .algo_mask = SAFEXCEL_ALG_DES, 1478 .alg.skcipher = { 1479 .setkey = safexcel_des_setkey, 1480 .encrypt = safexcel_encrypt, 1481 .decrypt = safexcel_decrypt, 1482 .min_keysize = DES_KEY_SIZE, 1483 .max_keysize = DES_KEY_SIZE, 1484 .ivsize = DES_BLOCK_SIZE, 1485 .base = { 1486 .cra_name = "cbc(des)", 1487 .cra_driver_name = "safexcel-cbc-des", 1488 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1489 .cra_flags = CRYPTO_ALG_ASYNC | 1490 CRYPTO_ALG_KERN_DRIVER_ONLY, 1491 .cra_blocksize = DES_BLOCK_SIZE, 1492 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1493 .cra_alignmask = 0, 1494 .cra_init = safexcel_skcipher_des_cbc_cra_init, 1495 .cra_exit = safexcel_skcipher_cra_exit, 1496 .cra_module = THIS_MODULE, 1497 }, 1498 }, 1499 }; 1500 1501 static int safexcel_skcipher_des_ecb_cra_init(struct crypto_tfm *tfm) 1502 { 1503 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1504 1505 safexcel_skcipher_cra_init(tfm); 1506 ctx->alg = SAFEXCEL_DES; 1507 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; 1508 return 0; 1509 } 1510 1511 struct safexcel_alg_template safexcel_alg_ecb_des = { 1512 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1513 .algo_mask = SAFEXCEL_ALG_DES, 1514 .alg.skcipher = { 1515 .setkey = safexcel_des_setkey, 1516 .encrypt = safexcel_encrypt, 1517 .decrypt = safexcel_decrypt, 1518 .min_keysize = DES_KEY_SIZE, 1519 .max_keysize = DES_KEY_SIZE, 1520 .base = { 1521 .cra_name = "ecb(des)", 1522 .cra_driver_name = "safexcel-ecb-des", 1523 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1524 .cra_flags = CRYPTO_ALG_ASYNC | 1525 CRYPTO_ALG_KERN_DRIVER_ONLY, 1526 .cra_blocksize = DES_BLOCK_SIZE, 1527 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1528 .cra_alignmask = 0, 1529 .cra_init = safexcel_skcipher_des_ecb_cra_init, 1530 .cra_exit = safexcel_skcipher_cra_exit, 1531 .cra_module = THIS_MODULE, 1532 }, 1533 }, 1534 }; 1535 1536 static int safexcel_des3_ede_setkey(struct crypto_skcipher *ctfm, 1537 const u8 *key, unsigned int len) 1538 { 1539 struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm); 1540 int err; 1541 1542 err = verify_skcipher_des3_key(ctfm, key); 1543 if (err) 1544 return err; 1545 1546 /* if context exits and key changed, need to invalidate it */ 1547 if (ctx->base.ctxr_dma) 1548 if (memcmp(ctx->key, key, len)) 1549 ctx->base.needs_inv = true; 1550 1551 memcpy(ctx->key, key, len); 1552 ctx->key_len = len; 1553 1554 return 0; 1555 } 1556 1557 static int safexcel_skcipher_des3_cbc_cra_init(struct crypto_tfm *tfm) 1558 { 1559 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1560 1561 safexcel_skcipher_cra_init(tfm); 1562 ctx->alg = SAFEXCEL_3DES; 1563 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; 1564 return 0; 1565 } 1566 1567 struct safexcel_alg_template safexcel_alg_cbc_des3_ede = { 1568 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1569 .algo_mask = SAFEXCEL_ALG_DES, 1570 .alg.skcipher = { 1571 .setkey = safexcel_des3_ede_setkey, 1572 .encrypt = safexcel_encrypt, 1573 .decrypt = safexcel_decrypt, 1574 .min_keysize = DES3_EDE_KEY_SIZE, 1575 .max_keysize = DES3_EDE_KEY_SIZE, 1576 .ivsize = DES3_EDE_BLOCK_SIZE, 1577 .base = { 1578 .cra_name = "cbc(des3_ede)", 1579 .cra_driver_name = "safexcel-cbc-des3_ede", 1580 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1581 .cra_flags = CRYPTO_ALG_ASYNC | 1582 CRYPTO_ALG_KERN_DRIVER_ONLY, 1583 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1584 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1585 .cra_alignmask = 0, 1586 .cra_init = safexcel_skcipher_des3_cbc_cra_init, 1587 .cra_exit = safexcel_skcipher_cra_exit, 1588 .cra_module = THIS_MODULE, 1589 }, 1590 }, 1591 }; 1592 1593 static int safexcel_skcipher_des3_ecb_cra_init(struct crypto_tfm *tfm) 1594 { 1595 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1596 1597 safexcel_skcipher_cra_init(tfm); 1598 ctx->alg = SAFEXCEL_3DES; 1599 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; 1600 return 0; 1601 } 1602 1603 struct safexcel_alg_template safexcel_alg_ecb_des3_ede = { 1604 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1605 .algo_mask = SAFEXCEL_ALG_DES, 1606 .alg.skcipher = { 1607 .setkey = safexcel_des3_ede_setkey, 1608 .encrypt = safexcel_encrypt, 1609 .decrypt = safexcel_decrypt, 1610 .min_keysize = DES3_EDE_KEY_SIZE, 1611 .max_keysize = DES3_EDE_KEY_SIZE, 1612 .base = { 1613 .cra_name = "ecb(des3_ede)", 1614 .cra_driver_name = "safexcel-ecb-des3_ede", 1615 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1616 .cra_flags = CRYPTO_ALG_ASYNC | 1617 CRYPTO_ALG_KERN_DRIVER_ONLY, 1618 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1619 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1620 .cra_alignmask = 0, 1621 .cra_init = safexcel_skcipher_des3_ecb_cra_init, 1622 .cra_exit = safexcel_skcipher_cra_exit, 1623 .cra_module = THIS_MODULE, 1624 }, 1625 }, 1626 }; 1627 1628 static int safexcel_aead_encrypt(struct aead_request *req) 1629 { 1630 struct safexcel_cipher_req *creq = aead_request_ctx(req); 1631 1632 return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); 1633 } 1634 1635 static int safexcel_aead_decrypt(struct aead_request *req) 1636 { 1637 struct safexcel_cipher_req *creq = aead_request_ctx(req); 1638 1639 return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); 1640 } 1641 1642 static int safexcel_aead_cra_init(struct crypto_tfm *tfm) 1643 { 1644 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1645 struct safexcel_alg_template *tmpl = 1646 container_of(tfm->__crt_alg, struct safexcel_alg_template, 1647 alg.aead.base); 1648 1649 crypto_aead_set_reqsize(__crypto_aead_cast(tfm), 1650 sizeof(struct safexcel_cipher_req)); 1651 1652 ctx->priv = tmpl->priv; 1653 1654 ctx->alg = SAFEXCEL_AES; /* default */ 1655 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; /* default */ 1656 ctx->aead = true; 1657 ctx->base.send = safexcel_aead_send; 1658 ctx->base.handle_result = safexcel_aead_handle_result; 1659 return 0; 1660 } 1661 1662 static int safexcel_aead_sha1_cra_init(struct crypto_tfm *tfm) 1663 { 1664 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1665 1666 safexcel_aead_cra_init(tfm); 1667 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1; 1668 ctx->state_sz = SHA1_DIGEST_SIZE; 1669 return 0; 1670 } 1671 1672 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_aes = { 1673 .type = SAFEXCEL_ALG_TYPE_AEAD, 1674 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA1, 1675 .alg.aead = { 1676 .setkey = safexcel_aead_setkey, 1677 .encrypt = safexcel_aead_encrypt, 1678 .decrypt = safexcel_aead_decrypt, 1679 .ivsize = AES_BLOCK_SIZE, 1680 .maxauthsize = SHA1_DIGEST_SIZE, 1681 .base = { 1682 .cra_name = "authenc(hmac(sha1),cbc(aes))", 1683 .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-aes", 1684 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1685 .cra_flags = CRYPTO_ALG_ASYNC | 1686 CRYPTO_ALG_KERN_DRIVER_ONLY, 1687 .cra_blocksize = AES_BLOCK_SIZE, 1688 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1689 .cra_alignmask = 0, 1690 .cra_init = safexcel_aead_sha1_cra_init, 1691 .cra_exit = safexcel_aead_cra_exit, 1692 .cra_module = THIS_MODULE, 1693 }, 1694 }, 1695 }; 1696 1697 static int safexcel_aead_sha256_cra_init(struct crypto_tfm *tfm) 1698 { 1699 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1700 1701 safexcel_aead_cra_init(tfm); 1702 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256; 1703 ctx->state_sz = SHA256_DIGEST_SIZE; 1704 return 0; 1705 } 1706 1707 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_aes = { 1708 .type = SAFEXCEL_ALG_TYPE_AEAD, 1709 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256, 1710 .alg.aead = { 1711 .setkey = safexcel_aead_setkey, 1712 .encrypt = safexcel_aead_encrypt, 1713 .decrypt = safexcel_aead_decrypt, 1714 .ivsize = AES_BLOCK_SIZE, 1715 .maxauthsize = SHA256_DIGEST_SIZE, 1716 .base = { 1717 .cra_name = "authenc(hmac(sha256),cbc(aes))", 1718 .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-aes", 1719 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1720 .cra_flags = CRYPTO_ALG_ASYNC | 1721 CRYPTO_ALG_KERN_DRIVER_ONLY, 1722 .cra_blocksize = AES_BLOCK_SIZE, 1723 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1724 .cra_alignmask = 0, 1725 .cra_init = safexcel_aead_sha256_cra_init, 1726 .cra_exit = safexcel_aead_cra_exit, 1727 .cra_module = THIS_MODULE, 1728 }, 1729 }, 1730 }; 1731 1732 static int safexcel_aead_sha224_cra_init(struct crypto_tfm *tfm) 1733 { 1734 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1735 1736 safexcel_aead_cra_init(tfm); 1737 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224; 1738 ctx->state_sz = SHA256_DIGEST_SIZE; 1739 return 0; 1740 } 1741 1742 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_aes = { 1743 .type = SAFEXCEL_ALG_TYPE_AEAD, 1744 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256, 1745 .alg.aead = { 1746 .setkey = safexcel_aead_setkey, 1747 .encrypt = safexcel_aead_encrypt, 1748 .decrypt = safexcel_aead_decrypt, 1749 .ivsize = AES_BLOCK_SIZE, 1750 .maxauthsize = SHA224_DIGEST_SIZE, 1751 .base = { 1752 .cra_name = "authenc(hmac(sha224),cbc(aes))", 1753 .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-aes", 1754 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1755 .cra_flags = CRYPTO_ALG_ASYNC | 1756 CRYPTO_ALG_KERN_DRIVER_ONLY, 1757 .cra_blocksize = AES_BLOCK_SIZE, 1758 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1759 .cra_alignmask = 0, 1760 .cra_init = safexcel_aead_sha224_cra_init, 1761 .cra_exit = safexcel_aead_cra_exit, 1762 .cra_module = THIS_MODULE, 1763 }, 1764 }, 1765 }; 1766 1767 static int safexcel_aead_sha512_cra_init(struct crypto_tfm *tfm) 1768 { 1769 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1770 1771 safexcel_aead_cra_init(tfm); 1772 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512; 1773 ctx->state_sz = SHA512_DIGEST_SIZE; 1774 return 0; 1775 } 1776 1777 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_aes = { 1778 .type = SAFEXCEL_ALG_TYPE_AEAD, 1779 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512, 1780 .alg.aead = { 1781 .setkey = safexcel_aead_setkey, 1782 .encrypt = safexcel_aead_encrypt, 1783 .decrypt = safexcel_aead_decrypt, 1784 .ivsize = AES_BLOCK_SIZE, 1785 .maxauthsize = SHA512_DIGEST_SIZE, 1786 .base = { 1787 .cra_name = "authenc(hmac(sha512),cbc(aes))", 1788 .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-aes", 1789 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1790 .cra_flags = CRYPTO_ALG_ASYNC | 1791 CRYPTO_ALG_KERN_DRIVER_ONLY, 1792 .cra_blocksize = AES_BLOCK_SIZE, 1793 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1794 .cra_alignmask = 0, 1795 .cra_init = safexcel_aead_sha512_cra_init, 1796 .cra_exit = safexcel_aead_cra_exit, 1797 .cra_module = THIS_MODULE, 1798 }, 1799 }, 1800 }; 1801 1802 static int safexcel_aead_sha384_cra_init(struct crypto_tfm *tfm) 1803 { 1804 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1805 1806 safexcel_aead_cra_init(tfm); 1807 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384; 1808 ctx->state_sz = SHA512_DIGEST_SIZE; 1809 return 0; 1810 } 1811 1812 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_aes = { 1813 .type = SAFEXCEL_ALG_TYPE_AEAD, 1814 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512, 1815 .alg.aead = { 1816 .setkey = safexcel_aead_setkey, 1817 .encrypt = safexcel_aead_encrypt, 1818 .decrypt = safexcel_aead_decrypt, 1819 .ivsize = AES_BLOCK_SIZE, 1820 .maxauthsize = SHA384_DIGEST_SIZE, 1821 .base = { 1822 .cra_name = "authenc(hmac(sha384),cbc(aes))", 1823 .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-aes", 1824 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1825 .cra_flags = CRYPTO_ALG_ASYNC | 1826 CRYPTO_ALG_KERN_DRIVER_ONLY, 1827 .cra_blocksize = AES_BLOCK_SIZE, 1828 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1829 .cra_alignmask = 0, 1830 .cra_init = safexcel_aead_sha384_cra_init, 1831 .cra_exit = safexcel_aead_cra_exit, 1832 .cra_module = THIS_MODULE, 1833 }, 1834 }, 1835 }; 1836 1837 static int safexcel_aead_sha1_des3_cra_init(struct crypto_tfm *tfm) 1838 { 1839 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1840 1841 safexcel_aead_sha1_cra_init(tfm); 1842 ctx->alg = SAFEXCEL_3DES; /* override default */ 1843 return 0; 1844 } 1845 1846 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des3_ede = { 1847 .type = SAFEXCEL_ALG_TYPE_AEAD, 1848 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA1, 1849 .alg.aead = { 1850 .setkey = safexcel_aead_setkey, 1851 .encrypt = safexcel_aead_encrypt, 1852 .decrypt = safexcel_aead_decrypt, 1853 .ivsize = DES3_EDE_BLOCK_SIZE, 1854 .maxauthsize = SHA1_DIGEST_SIZE, 1855 .base = { 1856 .cra_name = "authenc(hmac(sha1),cbc(des3_ede))", 1857 .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-des3_ede", 1858 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1859 .cra_flags = CRYPTO_ALG_ASYNC | 1860 CRYPTO_ALG_KERN_DRIVER_ONLY, 1861 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1862 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1863 .cra_alignmask = 0, 1864 .cra_init = safexcel_aead_sha1_des3_cra_init, 1865 .cra_exit = safexcel_aead_cra_exit, 1866 .cra_module = THIS_MODULE, 1867 }, 1868 }, 1869 }; 1870 1871 static int safexcel_aead_sha256_des3_cra_init(struct crypto_tfm *tfm) 1872 { 1873 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1874 1875 safexcel_aead_sha256_cra_init(tfm); 1876 ctx->alg = SAFEXCEL_3DES; /* override default */ 1877 return 0; 1878 } 1879 1880 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des3_ede = { 1881 .type = SAFEXCEL_ALG_TYPE_AEAD, 1882 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, 1883 .alg.aead = { 1884 .setkey = safexcel_aead_setkey, 1885 .encrypt = safexcel_aead_encrypt, 1886 .decrypt = safexcel_aead_decrypt, 1887 .ivsize = DES3_EDE_BLOCK_SIZE, 1888 .maxauthsize = SHA256_DIGEST_SIZE, 1889 .base = { 1890 .cra_name = "authenc(hmac(sha256),cbc(des3_ede))", 1891 .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-des3_ede", 1892 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1893 .cra_flags = CRYPTO_ALG_ASYNC | 1894 CRYPTO_ALG_KERN_DRIVER_ONLY, 1895 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1896 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1897 .cra_alignmask = 0, 1898 .cra_init = safexcel_aead_sha256_des3_cra_init, 1899 .cra_exit = safexcel_aead_cra_exit, 1900 .cra_module = THIS_MODULE, 1901 }, 1902 }, 1903 }; 1904 1905 static int safexcel_aead_sha224_des3_cra_init(struct crypto_tfm *tfm) 1906 { 1907 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1908 1909 safexcel_aead_sha224_cra_init(tfm); 1910 ctx->alg = SAFEXCEL_3DES; /* override default */ 1911 return 0; 1912 } 1913 1914 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des3_ede = { 1915 .type = SAFEXCEL_ALG_TYPE_AEAD, 1916 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, 1917 .alg.aead = { 1918 .setkey = safexcel_aead_setkey, 1919 .encrypt = safexcel_aead_encrypt, 1920 .decrypt = safexcel_aead_decrypt, 1921 .ivsize = DES3_EDE_BLOCK_SIZE, 1922 .maxauthsize = SHA224_DIGEST_SIZE, 1923 .base = { 1924 .cra_name = "authenc(hmac(sha224),cbc(des3_ede))", 1925 .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-des3_ede", 1926 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1927 .cra_flags = CRYPTO_ALG_ASYNC | 1928 CRYPTO_ALG_KERN_DRIVER_ONLY, 1929 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1930 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1931 .cra_alignmask = 0, 1932 .cra_init = safexcel_aead_sha224_des3_cra_init, 1933 .cra_exit = safexcel_aead_cra_exit, 1934 .cra_module = THIS_MODULE, 1935 }, 1936 }, 1937 }; 1938 1939 static int safexcel_aead_sha512_des3_cra_init(struct crypto_tfm *tfm) 1940 { 1941 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1942 1943 safexcel_aead_sha512_cra_init(tfm); 1944 ctx->alg = SAFEXCEL_3DES; /* override default */ 1945 return 0; 1946 } 1947 1948 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des3_ede = { 1949 .type = SAFEXCEL_ALG_TYPE_AEAD, 1950 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, 1951 .alg.aead = { 1952 .setkey = safexcel_aead_setkey, 1953 .encrypt = safexcel_aead_encrypt, 1954 .decrypt = safexcel_aead_decrypt, 1955 .ivsize = DES3_EDE_BLOCK_SIZE, 1956 .maxauthsize = SHA512_DIGEST_SIZE, 1957 .base = { 1958 .cra_name = "authenc(hmac(sha512),cbc(des3_ede))", 1959 .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-des3_ede", 1960 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1961 .cra_flags = CRYPTO_ALG_ASYNC | 1962 CRYPTO_ALG_KERN_DRIVER_ONLY, 1963 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1964 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1965 .cra_alignmask = 0, 1966 .cra_init = safexcel_aead_sha512_des3_cra_init, 1967 .cra_exit = safexcel_aead_cra_exit, 1968 .cra_module = THIS_MODULE, 1969 }, 1970 }, 1971 }; 1972 1973 static int safexcel_aead_sha384_des3_cra_init(struct crypto_tfm *tfm) 1974 { 1975 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1976 1977 safexcel_aead_sha384_cra_init(tfm); 1978 ctx->alg = SAFEXCEL_3DES; /* override default */ 1979 return 0; 1980 } 1981 1982 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des3_ede = { 1983 .type = SAFEXCEL_ALG_TYPE_AEAD, 1984 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, 1985 .alg.aead = { 1986 .setkey = safexcel_aead_setkey, 1987 .encrypt = safexcel_aead_encrypt, 1988 .decrypt = safexcel_aead_decrypt, 1989 .ivsize = DES3_EDE_BLOCK_SIZE, 1990 .maxauthsize = SHA384_DIGEST_SIZE, 1991 .base = { 1992 .cra_name = "authenc(hmac(sha384),cbc(des3_ede))", 1993 .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-des3_ede", 1994 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1995 .cra_flags = CRYPTO_ALG_ASYNC | 1996 CRYPTO_ALG_KERN_DRIVER_ONLY, 1997 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1998 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1999 .cra_alignmask = 0, 2000 .cra_init = safexcel_aead_sha384_des3_cra_init, 2001 .cra_exit = safexcel_aead_cra_exit, 2002 .cra_module = THIS_MODULE, 2003 }, 2004 }, 2005 }; 2006 2007 static int safexcel_aead_sha1_des_cra_init(struct crypto_tfm *tfm) 2008 { 2009 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2010 2011 safexcel_aead_sha1_cra_init(tfm); 2012 ctx->alg = SAFEXCEL_DES; /* override default */ 2013 return 0; 2014 } 2015 2016 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des = { 2017 .type = SAFEXCEL_ALG_TYPE_AEAD, 2018 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA1, 2019 .alg.aead = { 2020 .setkey = safexcel_aead_setkey, 2021 .encrypt = safexcel_aead_encrypt, 2022 .decrypt = safexcel_aead_decrypt, 2023 .ivsize = DES_BLOCK_SIZE, 2024 .maxauthsize = SHA1_DIGEST_SIZE, 2025 .base = { 2026 .cra_name = "authenc(hmac(sha1),cbc(des))", 2027 .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-des", 2028 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2029 .cra_flags = CRYPTO_ALG_ASYNC | 2030 CRYPTO_ALG_KERN_DRIVER_ONLY, 2031 .cra_blocksize = DES_BLOCK_SIZE, 2032 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2033 .cra_alignmask = 0, 2034 .cra_init = safexcel_aead_sha1_des_cra_init, 2035 .cra_exit = safexcel_aead_cra_exit, 2036 .cra_module = THIS_MODULE, 2037 }, 2038 }, 2039 }; 2040 2041 static int safexcel_aead_sha256_des_cra_init(struct crypto_tfm *tfm) 2042 { 2043 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2044 2045 safexcel_aead_sha256_cra_init(tfm); 2046 ctx->alg = SAFEXCEL_DES; /* override default */ 2047 return 0; 2048 } 2049 2050 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des = { 2051 .type = SAFEXCEL_ALG_TYPE_AEAD, 2052 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, 2053 .alg.aead = { 2054 .setkey = safexcel_aead_setkey, 2055 .encrypt = safexcel_aead_encrypt, 2056 .decrypt = safexcel_aead_decrypt, 2057 .ivsize = DES_BLOCK_SIZE, 2058 .maxauthsize = SHA256_DIGEST_SIZE, 2059 .base = { 2060 .cra_name = "authenc(hmac(sha256),cbc(des))", 2061 .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-des", 2062 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2063 .cra_flags = CRYPTO_ALG_ASYNC | 2064 CRYPTO_ALG_KERN_DRIVER_ONLY, 2065 .cra_blocksize = DES_BLOCK_SIZE, 2066 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2067 .cra_alignmask = 0, 2068 .cra_init = safexcel_aead_sha256_des_cra_init, 2069 .cra_exit = safexcel_aead_cra_exit, 2070 .cra_module = THIS_MODULE, 2071 }, 2072 }, 2073 }; 2074 2075 static int safexcel_aead_sha224_des_cra_init(struct crypto_tfm *tfm) 2076 { 2077 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2078 2079 safexcel_aead_sha224_cra_init(tfm); 2080 ctx->alg = SAFEXCEL_DES; /* override default */ 2081 return 0; 2082 } 2083 2084 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des = { 2085 .type = SAFEXCEL_ALG_TYPE_AEAD, 2086 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, 2087 .alg.aead = { 2088 .setkey = safexcel_aead_setkey, 2089 .encrypt = safexcel_aead_encrypt, 2090 .decrypt = safexcel_aead_decrypt, 2091 .ivsize = DES_BLOCK_SIZE, 2092 .maxauthsize = SHA224_DIGEST_SIZE, 2093 .base = { 2094 .cra_name = "authenc(hmac(sha224),cbc(des))", 2095 .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-des", 2096 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2097 .cra_flags = CRYPTO_ALG_ASYNC | 2098 CRYPTO_ALG_KERN_DRIVER_ONLY, 2099 .cra_blocksize = DES_BLOCK_SIZE, 2100 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2101 .cra_alignmask = 0, 2102 .cra_init = safexcel_aead_sha224_des_cra_init, 2103 .cra_exit = safexcel_aead_cra_exit, 2104 .cra_module = THIS_MODULE, 2105 }, 2106 }, 2107 }; 2108 2109 static int safexcel_aead_sha512_des_cra_init(struct crypto_tfm *tfm) 2110 { 2111 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2112 2113 safexcel_aead_sha512_cra_init(tfm); 2114 ctx->alg = SAFEXCEL_DES; /* override default */ 2115 return 0; 2116 } 2117 2118 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des = { 2119 .type = SAFEXCEL_ALG_TYPE_AEAD, 2120 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, 2121 .alg.aead = { 2122 .setkey = safexcel_aead_setkey, 2123 .encrypt = safexcel_aead_encrypt, 2124 .decrypt = safexcel_aead_decrypt, 2125 .ivsize = DES_BLOCK_SIZE, 2126 .maxauthsize = SHA512_DIGEST_SIZE, 2127 .base = { 2128 .cra_name = "authenc(hmac(sha512),cbc(des))", 2129 .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-des", 2130 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2131 .cra_flags = CRYPTO_ALG_ASYNC | 2132 CRYPTO_ALG_KERN_DRIVER_ONLY, 2133 .cra_blocksize = DES_BLOCK_SIZE, 2134 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2135 .cra_alignmask = 0, 2136 .cra_init = safexcel_aead_sha512_des_cra_init, 2137 .cra_exit = safexcel_aead_cra_exit, 2138 .cra_module = THIS_MODULE, 2139 }, 2140 }, 2141 }; 2142 2143 static int safexcel_aead_sha384_des_cra_init(struct crypto_tfm *tfm) 2144 { 2145 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2146 2147 safexcel_aead_sha384_cra_init(tfm); 2148 ctx->alg = SAFEXCEL_DES; /* override default */ 2149 return 0; 2150 } 2151 2152 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des = { 2153 .type = SAFEXCEL_ALG_TYPE_AEAD, 2154 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, 2155 .alg.aead = { 2156 .setkey = safexcel_aead_setkey, 2157 .encrypt = safexcel_aead_encrypt, 2158 .decrypt = safexcel_aead_decrypt, 2159 .ivsize = DES_BLOCK_SIZE, 2160 .maxauthsize = SHA384_DIGEST_SIZE, 2161 .base = { 2162 .cra_name = "authenc(hmac(sha384),cbc(des))", 2163 .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-des", 2164 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2165 .cra_flags = CRYPTO_ALG_ASYNC | 2166 CRYPTO_ALG_KERN_DRIVER_ONLY, 2167 .cra_blocksize = DES_BLOCK_SIZE, 2168 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2169 .cra_alignmask = 0, 2170 .cra_init = safexcel_aead_sha384_des_cra_init, 2171 .cra_exit = safexcel_aead_cra_exit, 2172 .cra_module = THIS_MODULE, 2173 }, 2174 }, 2175 }; 2176 2177 static int safexcel_aead_sha1_ctr_cra_init(struct crypto_tfm *tfm) 2178 { 2179 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2180 2181 safexcel_aead_sha1_cra_init(tfm); 2182 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2183 return 0; 2184 } 2185 2186 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_ctr_aes = { 2187 .type = SAFEXCEL_ALG_TYPE_AEAD, 2188 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA1, 2189 .alg.aead = { 2190 .setkey = safexcel_aead_setkey, 2191 .encrypt = safexcel_aead_encrypt, 2192 .decrypt = safexcel_aead_decrypt, 2193 .ivsize = CTR_RFC3686_IV_SIZE, 2194 .maxauthsize = SHA1_DIGEST_SIZE, 2195 .base = { 2196 .cra_name = "authenc(hmac(sha1),rfc3686(ctr(aes)))", 2197 .cra_driver_name = "safexcel-authenc-hmac-sha1-ctr-aes", 2198 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2199 .cra_flags = CRYPTO_ALG_ASYNC | 2200 CRYPTO_ALG_KERN_DRIVER_ONLY, 2201 .cra_blocksize = 1, 2202 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2203 .cra_alignmask = 0, 2204 .cra_init = safexcel_aead_sha1_ctr_cra_init, 2205 .cra_exit = safexcel_aead_cra_exit, 2206 .cra_module = THIS_MODULE, 2207 }, 2208 }, 2209 }; 2210 2211 static int safexcel_aead_sha256_ctr_cra_init(struct crypto_tfm *tfm) 2212 { 2213 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2214 2215 safexcel_aead_sha256_cra_init(tfm); 2216 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2217 return 0; 2218 } 2219 2220 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_ctr_aes = { 2221 .type = SAFEXCEL_ALG_TYPE_AEAD, 2222 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256, 2223 .alg.aead = { 2224 .setkey = safexcel_aead_setkey, 2225 .encrypt = safexcel_aead_encrypt, 2226 .decrypt = safexcel_aead_decrypt, 2227 .ivsize = CTR_RFC3686_IV_SIZE, 2228 .maxauthsize = SHA256_DIGEST_SIZE, 2229 .base = { 2230 .cra_name = "authenc(hmac(sha256),rfc3686(ctr(aes)))", 2231 .cra_driver_name = "safexcel-authenc-hmac-sha256-ctr-aes", 2232 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2233 .cra_flags = CRYPTO_ALG_ASYNC | 2234 CRYPTO_ALG_KERN_DRIVER_ONLY, 2235 .cra_blocksize = 1, 2236 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2237 .cra_alignmask = 0, 2238 .cra_init = safexcel_aead_sha256_ctr_cra_init, 2239 .cra_exit = safexcel_aead_cra_exit, 2240 .cra_module = THIS_MODULE, 2241 }, 2242 }, 2243 }; 2244 2245 static int safexcel_aead_sha224_ctr_cra_init(struct crypto_tfm *tfm) 2246 { 2247 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2248 2249 safexcel_aead_sha224_cra_init(tfm); 2250 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2251 return 0; 2252 } 2253 2254 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_ctr_aes = { 2255 .type = SAFEXCEL_ALG_TYPE_AEAD, 2256 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256, 2257 .alg.aead = { 2258 .setkey = safexcel_aead_setkey, 2259 .encrypt = safexcel_aead_encrypt, 2260 .decrypt = safexcel_aead_decrypt, 2261 .ivsize = CTR_RFC3686_IV_SIZE, 2262 .maxauthsize = SHA224_DIGEST_SIZE, 2263 .base = { 2264 .cra_name = "authenc(hmac(sha224),rfc3686(ctr(aes)))", 2265 .cra_driver_name = "safexcel-authenc-hmac-sha224-ctr-aes", 2266 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2267 .cra_flags = CRYPTO_ALG_ASYNC | 2268 CRYPTO_ALG_KERN_DRIVER_ONLY, 2269 .cra_blocksize = 1, 2270 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2271 .cra_alignmask = 0, 2272 .cra_init = safexcel_aead_sha224_ctr_cra_init, 2273 .cra_exit = safexcel_aead_cra_exit, 2274 .cra_module = THIS_MODULE, 2275 }, 2276 }, 2277 }; 2278 2279 static int safexcel_aead_sha512_ctr_cra_init(struct crypto_tfm *tfm) 2280 { 2281 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2282 2283 safexcel_aead_sha512_cra_init(tfm); 2284 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2285 return 0; 2286 } 2287 2288 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_ctr_aes = { 2289 .type = SAFEXCEL_ALG_TYPE_AEAD, 2290 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512, 2291 .alg.aead = { 2292 .setkey = safexcel_aead_setkey, 2293 .encrypt = safexcel_aead_encrypt, 2294 .decrypt = safexcel_aead_decrypt, 2295 .ivsize = CTR_RFC3686_IV_SIZE, 2296 .maxauthsize = SHA512_DIGEST_SIZE, 2297 .base = { 2298 .cra_name = "authenc(hmac(sha512),rfc3686(ctr(aes)))", 2299 .cra_driver_name = "safexcel-authenc-hmac-sha512-ctr-aes", 2300 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2301 .cra_flags = CRYPTO_ALG_ASYNC | 2302 CRYPTO_ALG_KERN_DRIVER_ONLY, 2303 .cra_blocksize = 1, 2304 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2305 .cra_alignmask = 0, 2306 .cra_init = safexcel_aead_sha512_ctr_cra_init, 2307 .cra_exit = safexcel_aead_cra_exit, 2308 .cra_module = THIS_MODULE, 2309 }, 2310 }, 2311 }; 2312 2313 static int safexcel_aead_sha384_ctr_cra_init(struct crypto_tfm *tfm) 2314 { 2315 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2316 2317 safexcel_aead_sha384_cra_init(tfm); 2318 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2319 return 0; 2320 } 2321 2322 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_ctr_aes = { 2323 .type = SAFEXCEL_ALG_TYPE_AEAD, 2324 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512, 2325 .alg.aead = { 2326 .setkey = safexcel_aead_setkey, 2327 .encrypt = safexcel_aead_encrypt, 2328 .decrypt = safexcel_aead_decrypt, 2329 .ivsize = CTR_RFC3686_IV_SIZE, 2330 .maxauthsize = SHA384_DIGEST_SIZE, 2331 .base = { 2332 .cra_name = "authenc(hmac(sha384),rfc3686(ctr(aes)))", 2333 .cra_driver_name = "safexcel-authenc-hmac-sha384-ctr-aes", 2334 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2335 .cra_flags = CRYPTO_ALG_ASYNC | 2336 CRYPTO_ALG_KERN_DRIVER_ONLY, 2337 .cra_blocksize = 1, 2338 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2339 .cra_alignmask = 0, 2340 .cra_init = safexcel_aead_sha384_ctr_cra_init, 2341 .cra_exit = safexcel_aead_cra_exit, 2342 .cra_module = THIS_MODULE, 2343 }, 2344 }, 2345 }; 2346 2347 static int safexcel_skcipher_aesxts_setkey(struct crypto_skcipher *ctfm, 2348 const u8 *key, unsigned int len) 2349 { 2350 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 2351 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2352 struct safexcel_crypto_priv *priv = ctx->priv; 2353 struct crypto_aes_ctx aes; 2354 int ret, i; 2355 unsigned int keylen; 2356 2357 /* Check for illegal XTS keys */ 2358 ret = xts_verify_key(ctfm, key, len); 2359 if (ret) 2360 return ret; 2361 2362 /* Only half of the key data is cipher key */ 2363 keylen = (len >> 1); 2364 ret = aes_expandkey(&aes, key, keylen); 2365 if (ret) { 2366 crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 2367 return ret; 2368 } 2369 2370 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2371 for (i = 0; i < keylen / sizeof(u32); i++) { 2372 if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { 2373 ctx->base.needs_inv = true; 2374 break; 2375 } 2376 } 2377 } 2378 2379 for (i = 0; i < keylen / sizeof(u32); i++) 2380 ctx->key[i] = cpu_to_le32(aes.key_enc[i]); 2381 2382 /* The other half is the tweak key */ 2383 ret = aes_expandkey(&aes, (u8 *)(key + keylen), keylen); 2384 if (ret) { 2385 crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 2386 return ret; 2387 } 2388 2389 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2390 for (i = 0; i < keylen / sizeof(u32); i++) { 2391 if (le32_to_cpu(ctx->key[i + keylen / sizeof(u32)]) != 2392 aes.key_enc[i]) { 2393 ctx->base.needs_inv = true; 2394 break; 2395 } 2396 } 2397 } 2398 2399 for (i = 0; i < keylen / sizeof(u32); i++) 2400 ctx->key[i + keylen / sizeof(u32)] = 2401 cpu_to_le32(aes.key_enc[i]); 2402 2403 ctx->key_len = keylen << 1; 2404 2405 memzero_explicit(&aes, sizeof(aes)); 2406 return 0; 2407 } 2408 2409 static int safexcel_skcipher_aes_xts_cra_init(struct crypto_tfm *tfm) 2410 { 2411 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2412 2413 safexcel_skcipher_cra_init(tfm); 2414 ctx->alg = SAFEXCEL_AES; 2415 ctx->xts = 1; 2416 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XTS; 2417 return 0; 2418 } 2419 2420 static int safexcel_encrypt_xts(struct skcipher_request *req) 2421 { 2422 if (req->cryptlen < XTS_BLOCK_SIZE) 2423 return -EINVAL; 2424 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 2425 SAFEXCEL_ENCRYPT); 2426 } 2427 2428 static int safexcel_decrypt_xts(struct skcipher_request *req) 2429 { 2430 if (req->cryptlen < XTS_BLOCK_SIZE) 2431 return -EINVAL; 2432 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 2433 SAFEXCEL_DECRYPT); 2434 } 2435 2436 struct safexcel_alg_template safexcel_alg_xts_aes = { 2437 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 2438 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_AES_XTS, 2439 .alg.skcipher = { 2440 .setkey = safexcel_skcipher_aesxts_setkey, 2441 .encrypt = safexcel_encrypt_xts, 2442 .decrypt = safexcel_decrypt_xts, 2443 /* XTS actually uses 2 AES keys glued together */ 2444 .min_keysize = AES_MIN_KEY_SIZE * 2, 2445 .max_keysize = AES_MAX_KEY_SIZE * 2, 2446 .ivsize = XTS_BLOCK_SIZE, 2447 .base = { 2448 .cra_name = "xts(aes)", 2449 .cra_driver_name = "safexcel-xts-aes", 2450 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2451 .cra_flags = CRYPTO_ALG_ASYNC | 2452 CRYPTO_ALG_KERN_DRIVER_ONLY, 2453 .cra_blocksize = XTS_BLOCK_SIZE, 2454 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2455 .cra_alignmask = 0, 2456 .cra_init = safexcel_skcipher_aes_xts_cra_init, 2457 .cra_exit = safexcel_skcipher_cra_exit, 2458 .cra_module = THIS_MODULE, 2459 }, 2460 }, 2461 }; 2462 2463 static int safexcel_aead_gcm_setkey(struct crypto_aead *ctfm, const u8 *key, 2464 unsigned int len) 2465 { 2466 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 2467 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2468 struct safexcel_crypto_priv *priv = ctx->priv; 2469 struct crypto_aes_ctx aes; 2470 u32 hashkey[AES_BLOCK_SIZE >> 2]; 2471 int ret, i; 2472 2473 ret = aes_expandkey(&aes, key, len); 2474 if (ret) { 2475 crypto_aead_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 2476 memzero_explicit(&aes, sizeof(aes)); 2477 return ret; 2478 } 2479 2480 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2481 for (i = 0; i < len / sizeof(u32); i++) { 2482 if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { 2483 ctx->base.needs_inv = true; 2484 break; 2485 } 2486 } 2487 } 2488 2489 for (i = 0; i < len / sizeof(u32); i++) 2490 ctx->key[i] = cpu_to_le32(aes.key_enc[i]); 2491 2492 ctx->key_len = len; 2493 2494 /* Compute hash key by encrypting zeroes with cipher key */ 2495 crypto_cipher_clear_flags(ctx->hkaes, CRYPTO_TFM_REQ_MASK); 2496 crypto_cipher_set_flags(ctx->hkaes, crypto_aead_get_flags(ctfm) & 2497 CRYPTO_TFM_REQ_MASK); 2498 ret = crypto_cipher_setkey(ctx->hkaes, key, len); 2499 crypto_aead_set_flags(ctfm, crypto_cipher_get_flags(ctx->hkaes) & 2500 CRYPTO_TFM_RES_MASK); 2501 if (ret) 2502 return ret; 2503 2504 memset(hashkey, 0, AES_BLOCK_SIZE); 2505 crypto_cipher_encrypt_one(ctx->hkaes, (u8 *)hashkey, (u8 *)hashkey); 2506 2507 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2508 for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) { 2509 if (be32_to_cpu(ctx->ipad[i]) != hashkey[i]) { 2510 ctx->base.needs_inv = true; 2511 break; 2512 } 2513 } 2514 } 2515 2516 for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) 2517 ctx->ipad[i] = cpu_to_be32(hashkey[i]); 2518 2519 memzero_explicit(hashkey, AES_BLOCK_SIZE); 2520 memzero_explicit(&aes, sizeof(aes)); 2521 return 0; 2522 } 2523 2524 static int safexcel_aead_gcm_cra_init(struct crypto_tfm *tfm) 2525 { 2526 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2527 2528 safexcel_aead_cra_init(tfm); 2529 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_GHASH; 2530 ctx->state_sz = GHASH_BLOCK_SIZE; 2531 ctx->xcm = EIP197_XCM_MODE_GCM; 2532 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XCM; /* override default */ 2533 2534 ctx->hkaes = crypto_alloc_cipher("aes", 0, 0); 2535 if (IS_ERR(ctx->hkaes)) 2536 return PTR_ERR(ctx->hkaes); 2537 2538 return 0; 2539 } 2540 2541 static void safexcel_aead_gcm_cra_exit(struct crypto_tfm *tfm) 2542 { 2543 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2544 2545 crypto_free_cipher(ctx->hkaes); 2546 safexcel_aead_cra_exit(tfm); 2547 } 2548 2549 static int safexcel_aead_gcm_setauthsize(struct crypto_aead *tfm, 2550 unsigned int authsize) 2551 { 2552 return crypto_gcm_check_authsize(authsize); 2553 } 2554 2555 struct safexcel_alg_template safexcel_alg_gcm = { 2556 .type = SAFEXCEL_ALG_TYPE_AEAD, 2557 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH, 2558 .alg.aead = { 2559 .setkey = safexcel_aead_gcm_setkey, 2560 .setauthsize = safexcel_aead_gcm_setauthsize, 2561 .encrypt = safexcel_aead_encrypt, 2562 .decrypt = safexcel_aead_decrypt, 2563 .ivsize = GCM_AES_IV_SIZE, 2564 .maxauthsize = GHASH_DIGEST_SIZE, 2565 .base = { 2566 .cra_name = "gcm(aes)", 2567 .cra_driver_name = "safexcel-gcm-aes", 2568 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2569 .cra_flags = CRYPTO_ALG_ASYNC | 2570 CRYPTO_ALG_KERN_DRIVER_ONLY, 2571 .cra_blocksize = 1, 2572 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2573 .cra_alignmask = 0, 2574 .cra_init = safexcel_aead_gcm_cra_init, 2575 .cra_exit = safexcel_aead_gcm_cra_exit, 2576 .cra_module = THIS_MODULE, 2577 }, 2578 }, 2579 }; 2580 2581 static int safexcel_aead_ccm_setkey(struct crypto_aead *ctfm, const u8 *key, 2582 unsigned int len) 2583 { 2584 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 2585 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2586 struct safexcel_crypto_priv *priv = ctx->priv; 2587 struct crypto_aes_ctx aes; 2588 int ret, i; 2589 2590 ret = aes_expandkey(&aes, key, len); 2591 if (ret) { 2592 crypto_aead_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 2593 memzero_explicit(&aes, sizeof(aes)); 2594 return ret; 2595 } 2596 2597 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2598 for (i = 0; i < len / sizeof(u32); i++) { 2599 if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { 2600 ctx->base.needs_inv = true; 2601 break; 2602 } 2603 } 2604 } 2605 2606 for (i = 0; i < len / sizeof(u32); i++) { 2607 ctx->key[i] = cpu_to_le32(aes.key_enc[i]); 2608 ctx->ipad[i + 2 * AES_BLOCK_SIZE / sizeof(u32)] = 2609 cpu_to_be32(aes.key_enc[i]); 2610 } 2611 2612 ctx->key_len = len; 2613 ctx->state_sz = 2 * AES_BLOCK_SIZE + len; 2614 2615 if (len == AES_KEYSIZE_192) 2616 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192; 2617 else if (len == AES_KEYSIZE_256) 2618 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256; 2619 else 2620 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128; 2621 2622 memzero_explicit(&aes, sizeof(aes)); 2623 return 0; 2624 } 2625 2626 static int safexcel_aead_ccm_cra_init(struct crypto_tfm *tfm) 2627 { 2628 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2629 2630 safexcel_aead_cra_init(tfm); 2631 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128; 2632 ctx->state_sz = 3 * AES_BLOCK_SIZE; 2633 ctx->xcm = EIP197_XCM_MODE_CCM; 2634 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XCM; /* override default */ 2635 return 0; 2636 } 2637 2638 static int safexcel_aead_ccm_setauthsize(struct crypto_aead *tfm, 2639 unsigned int authsize) 2640 { 2641 /* Borrowed from crypto/ccm.c */ 2642 switch (authsize) { 2643 case 4: 2644 case 6: 2645 case 8: 2646 case 10: 2647 case 12: 2648 case 14: 2649 case 16: 2650 break; 2651 default: 2652 return -EINVAL; 2653 } 2654 2655 return 0; 2656 } 2657 2658 static int safexcel_ccm_encrypt(struct aead_request *req) 2659 { 2660 struct safexcel_cipher_req *creq = aead_request_ctx(req); 2661 2662 if (req->iv[0] < 1 || req->iv[0] > 7) 2663 return -EINVAL; 2664 2665 return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); 2666 } 2667 2668 static int safexcel_ccm_decrypt(struct aead_request *req) 2669 { 2670 struct safexcel_cipher_req *creq = aead_request_ctx(req); 2671 2672 if (req->iv[0] < 1 || req->iv[0] > 7) 2673 return -EINVAL; 2674 2675 return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); 2676 } 2677 2678 struct safexcel_alg_template safexcel_alg_ccm = { 2679 .type = SAFEXCEL_ALG_TYPE_AEAD, 2680 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_CBC_MAC_ALL, 2681 .alg.aead = { 2682 .setkey = safexcel_aead_ccm_setkey, 2683 .setauthsize = safexcel_aead_ccm_setauthsize, 2684 .encrypt = safexcel_ccm_encrypt, 2685 .decrypt = safexcel_ccm_decrypt, 2686 .ivsize = AES_BLOCK_SIZE, 2687 .maxauthsize = AES_BLOCK_SIZE, 2688 .base = { 2689 .cra_name = "ccm(aes)", 2690 .cra_driver_name = "safexcel-ccm-aes", 2691 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2692 .cra_flags = CRYPTO_ALG_ASYNC | 2693 CRYPTO_ALG_KERN_DRIVER_ONLY, 2694 .cra_blocksize = 1, 2695 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2696 .cra_alignmask = 0, 2697 .cra_init = safexcel_aead_ccm_cra_init, 2698 .cra_exit = safexcel_aead_cra_exit, 2699 .cra_module = THIS_MODULE, 2700 }, 2701 }, 2702 }; 2703 2704 static void safexcel_chacha20_setkey(struct safexcel_cipher_ctx *ctx, 2705 const u8 *key) 2706 { 2707 struct safexcel_crypto_priv *priv = ctx->priv; 2708 2709 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) 2710 if (memcmp(ctx->key, key, CHACHA_KEY_SIZE)) 2711 ctx->base.needs_inv = true; 2712 2713 memcpy(ctx->key, key, CHACHA_KEY_SIZE); 2714 ctx->key_len = CHACHA_KEY_SIZE; 2715 } 2716 2717 static int safexcel_skcipher_chacha20_setkey(struct crypto_skcipher *ctfm, 2718 const u8 *key, unsigned int len) 2719 { 2720 struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm); 2721 2722 if (len != CHACHA_KEY_SIZE) { 2723 crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 2724 return -EINVAL; 2725 } 2726 safexcel_chacha20_setkey(ctx, key); 2727 2728 return 0; 2729 } 2730 2731 static int safexcel_skcipher_chacha20_cra_init(struct crypto_tfm *tfm) 2732 { 2733 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2734 2735 safexcel_skcipher_cra_init(tfm); 2736 ctx->alg = SAFEXCEL_CHACHA20; 2737 ctx->mode = CONTEXT_CONTROL_CHACHA20_MODE_256_32; 2738 return 0; 2739 } 2740 2741 struct safexcel_alg_template safexcel_alg_chacha20 = { 2742 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 2743 .algo_mask = SAFEXCEL_ALG_CHACHA20, 2744 .alg.skcipher = { 2745 .setkey = safexcel_skcipher_chacha20_setkey, 2746 .encrypt = safexcel_encrypt, 2747 .decrypt = safexcel_decrypt, 2748 .min_keysize = CHACHA_KEY_SIZE, 2749 .max_keysize = CHACHA_KEY_SIZE, 2750 .ivsize = CHACHA_IV_SIZE, 2751 .base = { 2752 .cra_name = "chacha20", 2753 .cra_driver_name = "safexcel-chacha20", 2754 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2755 .cra_flags = CRYPTO_ALG_ASYNC | 2756 CRYPTO_ALG_KERN_DRIVER_ONLY, 2757 .cra_blocksize = 1, 2758 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2759 .cra_alignmask = 0, 2760 .cra_init = safexcel_skcipher_chacha20_cra_init, 2761 .cra_exit = safexcel_skcipher_cra_exit, 2762 .cra_module = THIS_MODULE, 2763 }, 2764 }, 2765 }; 2766 2767 static int safexcel_aead_chachapoly_setkey(struct crypto_aead *ctfm, 2768 const u8 *key, unsigned int len) 2769 { 2770 struct safexcel_cipher_ctx *ctx = crypto_aead_ctx(ctfm); 2771 2772 if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP && 2773 len > EIP197_AEAD_IPSEC_NONCE_SIZE) { 2774 /* ESP variant has nonce appended to key */ 2775 len -= EIP197_AEAD_IPSEC_NONCE_SIZE; 2776 ctx->nonce = *(u32 *)(key + len); 2777 } 2778 if (len != CHACHA_KEY_SIZE) { 2779 crypto_aead_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 2780 return -EINVAL; 2781 } 2782 safexcel_chacha20_setkey(ctx, key); 2783 2784 return 0; 2785 } 2786 2787 static int safexcel_aead_chachapoly_setauthsize(struct crypto_aead *tfm, 2788 unsigned int authsize) 2789 { 2790 if (authsize != POLY1305_DIGEST_SIZE) 2791 return -EINVAL; 2792 return 0; 2793 } 2794 2795 static int safexcel_aead_chachapoly_crypt(struct aead_request *req, 2796 enum safexcel_cipher_direction dir) 2797 { 2798 struct safexcel_cipher_req *creq = aead_request_ctx(req); 2799 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2800 struct crypto_tfm *tfm = crypto_aead_tfm(aead); 2801 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2802 struct aead_request *subreq = aead_request_ctx(req); 2803 u32 key[CHACHA_KEY_SIZE / sizeof(u32) + 1]; 2804 int ret = 0; 2805 2806 /* 2807 * Instead of wasting time detecting umpteen silly corner cases, 2808 * just dump all "small" requests to the fallback implementation. 2809 * HW would not be faster on such small requests anyway. 2810 */ 2811 if (likely((ctx->aead != EIP197_AEAD_TYPE_IPSEC_ESP || 2812 req->assoclen >= EIP197_AEAD_IPSEC_IV_SIZE) && 2813 req->cryptlen > POLY1305_DIGEST_SIZE)) { 2814 return safexcel_queue_req(&req->base, creq, dir); 2815 } 2816 2817 /* HW cannot do full (AAD+payload) zero length, use fallback */ 2818 memcpy(key, ctx->key, CHACHA_KEY_SIZE); 2819 if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { 2820 /* ESP variant has nonce appended to the key */ 2821 key[CHACHA_KEY_SIZE / sizeof(u32)] = ctx->nonce; 2822 ret = crypto_aead_setkey(ctx->fback, (u8 *)key, 2823 CHACHA_KEY_SIZE + 2824 EIP197_AEAD_IPSEC_NONCE_SIZE); 2825 } else { 2826 ret = crypto_aead_setkey(ctx->fback, (u8 *)key, 2827 CHACHA_KEY_SIZE); 2828 } 2829 if (ret) { 2830 crypto_aead_clear_flags(aead, CRYPTO_TFM_REQ_MASK); 2831 crypto_aead_set_flags(aead, crypto_aead_get_flags(ctx->fback) & 2832 CRYPTO_TFM_REQ_MASK); 2833 return ret; 2834 } 2835 2836 aead_request_set_tfm(subreq, ctx->fback); 2837 aead_request_set_callback(subreq, req->base.flags, req->base.complete, 2838 req->base.data); 2839 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, 2840 req->iv); 2841 aead_request_set_ad(subreq, req->assoclen); 2842 2843 return (dir == SAFEXCEL_ENCRYPT) ? 2844 crypto_aead_encrypt(subreq) : 2845 crypto_aead_decrypt(subreq); 2846 } 2847 2848 static int safexcel_aead_chachapoly_encrypt(struct aead_request *req) 2849 { 2850 return safexcel_aead_chachapoly_crypt(req, SAFEXCEL_ENCRYPT); 2851 } 2852 2853 static int safexcel_aead_chachapoly_decrypt(struct aead_request *req) 2854 { 2855 return safexcel_aead_chachapoly_crypt(req, SAFEXCEL_DECRYPT); 2856 } 2857 2858 static int safexcel_aead_fallback_cra_init(struct crypto_tfm *tfm) 2859 { 2860 struct crypto_aead *aead = __crypto_aead_cast(tfm); 2861 struct aead_alg *alg = crypto_aead_alg(aead); 2862 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2863 2864 safexcel_aead_cra_init(tfm); 2865 2866 /* Allocate fallback implementation */ 2867 ctx->fback = crypto_alloc_aead(alg->base.cra_name, 0, 2868 CRYPTO_ALG_ASYNC | 2869 CRYPTO_ALG_NEED_FALLBACK); 2870 if (IS_ERR(ctx->fback)) 2871 return PTR_ERR(ctx->fback); 2872 2873 crypto_aead_set_reqsize(aead, max(sizeof(struct safexcel_cipher_req), 2874 sizeof(struct aead_request) + 2875 crypto_aead_reqsize(ctx->fback))); 2876 2877 return 0; 2878 } 2879 2880 static int safexcel_aead_chachapoly_cra_init(struct crypto_tfm *tfm) 2881 { 2882 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2883 2884 safexcel_aead_fallback_cra_init(tfm); 2885 ctx->alg = SAFEXCEL_CHACHA20; 2886 ctx->mode = CONTEXT_CONTROL_CHACHA20_MODE_256_32 | 2887 CONTEXT_CONTROL_CHACHA20_MODE_CALC_OTK; 2888 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_POLY1305; 2889 ctx->state_sz = 0; /* Precomputed by HW */ 2890 return 0; 2891 } 2892 2893 static void safexcel_aead_fallback_cra_exit(struct crypto_tfm *tfm) 2894 { 2895 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2896 2897 crypto_free_aead(ctx->fback); 2898 safexcel_aead_cra_exit(tfm); 2899 } 2900 2901 struct safexcel_alg_template safexcel_alg_chachapoly = { 2902 .type = SAFEXCEL_ALG_TYPE_AEAD, 2903 .algo_mask = SAFEXCEL_ALG_CHACHA20 | SAFEXCEL_ALG_POLY1305, 2904 .alg.aead = { 2905 .setkey = safexcel_aead_chachapoly_setkey, 2906 .setauthsize = safexcel_aead_chachapoly_setauthsize, 2907 .encrypt = safexcel_aead_chachapoly_encrypt, 2908 .decrypt = safexcel_aead_chachapoly_decrypt, 2909 .ivsize = CHACHAPOLY_IV_SIZE, 2910 .maxauthsize = POLY1305_DIGEST_SIZE, 2911 .base = { 2912 .cra_name = "rfc7539(chacha20,poly1305)", 2913 .cra_driver_name = "safexcel-chacha20-poly1305", 2914 /* +1 to put it above HW chacha + SW poly */ 2915 .cra_priority = SAFEXCEL_CRA_PRIORITY + 1, 2916 .cra_flags = CRYPTO_ALG_ASYNC | 2917 CRYPTO_ALG_KERN_DRIVER_ONLY | 2918 CRYPTO_ALG_NEED_FALLBACK, 2919 .cra_blocksize = 1, 2920 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2921 .cra_alignmask = 0, 2922 .cra_init = safexcel_aead_chachapoly_cra_init, 2923 .cra_exit = safexcel_aead_fallback_cra_exit, 2924 .cra_module = THIS_MODULE, 2925 }, 2926 }, 2927 }; 2928 2929 static int safexcel_aead_chachapolyesp_cra_init(struct crypto_tfm *tfm) 2930 { 2931 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2932 int ret; 2933 2934 ret = safexcel_aead_chachapoly_cra_init(tfm); 2935 ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP; 2936 return ret; 2937 } 2938 2939 struct safexcel_alg_template safexcel_alg_chachapoly_esp = { 2940 .type = SAFEXCEL_ALG_TYPE_AEAD, 2941 .algo_mask = SAFEXCEL_ALG_CHACHA20 | SAFEXCEL_ALG_POLY1305, 2942 .alg.aead = { 2943 .setkey = safexcel_aead_chachapoly_setkey, 2944 .setauthsize = safexcel_aead_chachapoly_setauthsize, 2945 .encrypt = safexcel_aead_chachapoly_encrypt, 2946 .decrypt = safexcel_aead_chachapoly_decrypt, 2947 .ivsize = CHACHAPOLY_IV_SIZE - EIP197_AEAD_IPSEC_NONCE_SIZE, 2948 .maxauthsize = POLY1305_DIGEST_SIZE, 2949 .base = { 2950 .cra_name = "rfc7539esp(chacha20,poly1305)", 2951 .cra_driver_name = "safexcel-chacha20-poly1305-esp", 2952 /* +1 to put it above HW chacha + SW poly */ 2953 .cra_priority = SAFEXCEL_CRA_PRIORITY + 1, 2954 .cra_flags = CRYPTO_ALG_ASYNC | 2955 CRYPTO_ALG_KERN_DRIVER_ONLY | 2956 CRYPTO_ALG_NEED_FALLBACK, 2957 .cra_blocksize = 1, 2958 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2959 .cra_alignmask = 0, 2960 .cra_init = safexcel_aead_chachapolyesp_cra_init, 2961 .cra_exit = safexcel_aead_fallback_cra_exit, 2962 .cra_module = THIS_MODULE, 2963 }, 2964 }, 2965 }; 2966 2967 static int safexcel_skcipher_sm4_setkey(struct crypto_skcipher *ctfm, 2968 const u8 *key, unsigned int len) 2969 { 2970 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 2971 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2972 struct safexcel_crypto_priv *priv = ctx->priv; 2973 2974 if (len != SM4_KEY_SIZE) { 2975 crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 2976 return -EINVAL; 2977 } 2978 2979 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) 2980 if (memcmp(ctx->key, key, SM4_KEY_SIZE)) 2981 ctx->base.needs_inv = true; 2982 2983 memcpy(ctx->key, key, SM4_KEY_SIZE); 2984 ctx->key_len = SM4_KEY_SIZE; 2985 2986 return 0; 2987 } 2988 2989 static int safexcel_sm4_blk_encrypt(struct skcipher_request *req) 2990 { 2991 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 2992 if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) 2993 return -EINVAL; 2994 else 2995 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 2996 SAFEXCEL_ENCRYPT); 2997 } 2998 2999 static int safexcel_sm4_blk_decrypt(struct skcipher_request *req) 3000 { 3001 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3002 if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) 3003 return -EINVAL; 3004 else 3005 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 3006 SAFEXCEL_DECRYPT); 3007 } 3008 3009 static int safexcel_skcipher_sm4_ecb_cra_init(struct crypto_tfm *tfm) 3010 { 3011 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3012 3013 safexcel_skcipher_cra_init(tfm); 3014 ctx->alg = SAFEXCEL_SM4; 3015 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; 3016 return 0; 3017 } 3018 3019 struct safexcel_alg_template safexcel_alg_ecb_sm4 = { 3020 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 3021 .algo_mask = SAFEXCEL_ALG_SM4, 3022 .alg.skcipher = { 3023 .setkey = safexcel_skcipher_sm4_setkey, 3024 .encrypt = safexcel_sm4_blk_encrypt, 3025 .decrypt = safexcel_sm4_blk_decrypt, 3026 .min_keysize = SM4_KEY_SIZE, 3027 .max_keysize = SM4_KEY_SIZE, 3028 .base = { 3029 .cra_name = "ecb(sm4)", 3030 .cra_driver_name = "safexcel-ecb-sm4", 3031 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3032 .cra_flags = CRYPTO_ALG_ASYNC | 3033 CRYPTO_ALG_KERN_DRIVER_ONLY, 3034 .cra_blocksize = SM4_BLOCK_SIZE, 3035 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3036 .cra_alignmask = 0, 3037 .cra_init = safexcel_skcipher_sm4_ecb_cra_init, 3038 .cra_exit = safexcel_skcipher_cra_exit, 3039 .cra_module = THIS_MODULE, 3040 }, 3041 }, 3042 }; 3043 3044 static int safexcel_skcipher_sm4_cbc_cra_init(struct crypto_tfm *tfm) 3045 { 3046 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3047 3048 safexcel_skcipher_cra_init(tfm); 3049 ctx->alg = SAFEXCEL_SM4; 3050 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; 3051 return 0; 3052 } 3053 3054 struct safexcel_alg_template safexcel_alg_cbc_sm4 = { 3055 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 3056 .algo_mask = SAFEXCEL_ALG_SM4, 3057 .alg.skcipher = { 3058 .setkey = safexcel_skcipher_sm4_setkey, 3059 .encrypt = safexcel_sm4_blk_encrypt, 3060 .decrypt = safexcel_sm4_blk_decrypt, 3061 .min_keysize = SM4_KEY_SIZE, 3062 .max_keysize = SM4_KEY_SIZE, 3063 .ivsize = SM4_BLOCK_SIZE, 3064 .base = { 3065 .cra_name = "cbc(sm4)", 3066 .cra_driver_name = "safexcel-cbc-sm4", 3067 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3068 .cra_flags = CRYPTO_ALG_ASYNC | 3069 CRYPTO_ALG_KERN_DRIVER_ONLY, 3070 .cra_blocksize = SM4_BLOCK_SIZE, 3071 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3072 .cra_alignmask = 0, 3073 .cra_init = safexcel_skcipher_sm4_cbc_cra_init, 3074 .cra_exit = safexcel_skcipher_cra_exit, 3075 .cra_module = THIS_MODULE, 3076 }, 3077 }, 3078 }; 3079 3080 static int safexcel_skcipher_sm4_ofb_cra_init(struct crypto_tfm *tfm) 3081 { 3082 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3083 3084 safexcel_skcipher_cra_init(tfm); 3085 ctx->alg = SAFEXCEL_SM4; 3086 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_OFB; 3087 return 0; 3088 } 3089 3090 struct safexcel_alg_template safexcel_alg_ofb_sm4 = { 3091 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 3092 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_AES_XFB, 3093 .alg.skcipher = { 3094 .setkey = safexcel_skcipher_sm4_setkey, 3095 .encrypt = safexcel_encrypt, 3096 .decrypt = safexcel_decrypt, 3097 .min_keysize = SM4_KEY_SIZE, 3098 .max_keysize = SM4_KEY_SIZE, 3099 .ivsize = SM4_BLOCK_SIZE, 3100 .base = { 3101 .cra_name = "ofb(sm4)", 3102 .cra_driver_name = "safexcel-ofb-sm4", 3103 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3104 .cra_flags = CRYPTO_ALG_ASYNC | 3105 CRYPTO_ALG_KERN_DRIVER_ONLY, 3106 .cra_blocksize = 1, 3107 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3108 .cra_alignmask = 0, 3109 .cra_init = safexcel_skcipher_sm4_ofb_cra_init, 3110 .cra_exit = safexcel_skcipher_cra_exit, 3111 .cra_module = THIS_MODULE, 3112 }, 3113 }, 3114 }; 3115 3116 static int safexcel_skcipher_sm4_cfb_cra_init(struct crypto_tfm *tfm) 3117 { 3118 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3119 3120 safexcel_skcipher_cra_init(tfm); 3121 ctx->alg = SAFEXCEL_SM4; 3122 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CFB; 3123 return 0; 3124 } 3125 3126 struct safexcel_alg_template safexcel_alg_cfb_sm4 = { 3127 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 3128 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_AES_XFB, 3129 .alg.skcipher = { 3130 .setkey = safexcel_skcipher_sm4_setkey, 3131 .encrypt = safexcel_encrypt, 3132 .decrypt = safexcel_decrypt, 3133 .min_keysize = SM4_KEY_SIZE, 3134 .max_keysize = SM4_KEY_SIZE, 3135 .ivsize = SM4_BLOCK_SIZE, 3136 .base = { 3137 .cra_name = "cfb(sm4)", 3138 .cra_driver_name = "safexcel-cfb-sm4", 3139 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3140 .cra_flags = CRYPTO_ALG_ASYNC | 3141 CRYPTO_ALG_KERN_DRIVER_ONLY, 3142 .cra_blocksize = 1, 3143 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3144 .cra_alignmask = 0, 3145 .cra_init = safexcel_skcipher_sm4_cfb_cra_init, 3146 .cra_exit = safexcel_skcipher_cra_exit, 3147 .cra_module = THIS_MODULE, 3148 }, 3149 }, 3150 }; 3151 3152 static int safexcel_skcipher_sm4ctr_setkey(struct crypto_skcipher *ctfm, 3153 const u8 *key, unsigned int len) 3154 { 3155 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 3156 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3157 3158 /* last 4 bytes of key are the nonce! */ 3159 ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE); 3160 /* exclude the nonce here */ 3161 len -= CTR_RFC3686_NONCE_SIZE; 3162 3163 return safexcel_skcipher_sm4_setkey(ctfm, key, len); 3164 } 3165 3166 static int safexcel_skcipher_sm4_ctr_cra_init(struct crypto_tfm *tfm) 3167 { 3168 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3169 3170 safexcel_skcipher_cra_init(tfm); 3171 ctx->alg = SAFEXCEL_SM4; 3172 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; 3173 return 0; 3174 } 3175 3176 struct safexcel_alg_template safexcel_alg_ctr_sm4 = { 3177 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 3178 .algo_mask = SAFEXCEL_ALG_SM4, 3179 .alg.skcipher = { 3180 .setkey = safexcel_skcipher_sm4ctr_setkey, 3181 .encrypt = safexcel_encrypt, 3182 .decrypt = safexcel_decrypt, 3183 /* Add nonce size */ 3184 .min_keysize = SM4_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 3185 .max_keysize = SM4_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 3186 .ivsize = CTR_RFC3686_IV_SIZE, 3187 .base = { 3188 .cra_name = "rfc3686(ctr(sm4))", 3189 .cra_driver_name = "safexcel-ctr-sm4", 3190 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3191 .cra_flags = CRYPTO_ALG_ASYNC | 3192 CRYPTO_ALG_KERN_DRIVER_ONLY, 3193 .cra_blocksize = 1, 3194 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3195 .cra_alignmask = 0, 3196 .cra_init = safexcel_skcipher_sm4_ctr_cra_init, 3197 .cra_exit = safexcel_skcipher_cra_exit, 3198 .cra_module = THIS_MODULE, 3199 }, 3200 }, 3201 }; 3202 3203 static int safexcel_aead_sm4_blk_encrypt(struct aead_request *req) 3204 { 3205 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3206 if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) 3207 return -EINVAL; 3208 3209 return safexcel_queue_req(&req->base, aead_request_ctx(req), 3210 SAFEXCEL_ENCRYPT); 3211 } 3212 3213 static int safexcel_aead_sm4_blk_decrypt(struct aead_request *req) 3214 { 3215 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3216 3217 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3218 if ((req->cryptlen - crypto_aead_authsize(tfm)) & (SM4_BLOCK_SIZE - 1)) 3219 return -EINVAL; 3220 3221 return safexcel_queue_req(&req->base, aead_request_ctx(req), 3222 SAFEXCEL_DECRYPT); 3223 } 3224 3225 static int safexcel_aead_sm4cbc_sha1_cra_init(struct crypto_tfm *tfm) 3226 { 3227 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3228 3229 safexcel_aead_cra_init(tfm); 3230 ctx->alg = SAFEXCEL_SM4; 3231 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1; 3232 ctx->state_sz = SHA1_DIGEST_SIZE; 3233 return 0; 3234 } 3235 3236 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_sm4 = { 3237 .type = SAFEXCEL_ALG_TYPE_AEAD, 3238 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SHA1, 3239 .alg.aead = { 3240 .setkey = safexcel_aead_setkey, 3241 .encrypt = safexcel_aead_sm4_blk_encrypt, 3242 .decrypt = safexcel_aead_sm4_blk_decrypt, 3243 .ivsize = SM4_BLOCK_SIZE, 3244 .maxauthsize = SHA1_DIGEST_SIZE, 3245 .base = { 3246 .cra_name = "authenc(hmac(sha1),cbc(sm4))", 3247 .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-sm4", 3248 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3249 .cra_flags = CRYPTO_ALG_ASYNC | 3250 CRYPTO_ALG_KERN_DRIVER_ONLY, 3251 .cra_blocksize = SM4_BLOCK_SIZE, 3252 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3253 .cra_alignmask = 0, 3254 .cra_init = safexcel_aead_sm4cbc_sha1_cra_init, 3255 .cra_exit = safexcel_aead_cra_exit, 3256 .cra_module = THIS_MODULE, 3257 }, 3258 }, 3259 }; 3260 3261 static int safexcel_aead_fallback_setkey(struct crypto_aead *ctfm, 3262 const u8 *key, unsigned int len) 3263 { 3264 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 3265 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3266 3267 /* Keep fallback cipher synchronized */ 3268 return crypto_aead_setkey(ctx->fback, (u8 *)key, len) ?: 3269 safexcel_aead_setkey(ctfm, key, len); 3270 } 3271 3272 static int safexcel_aead_fallback_setauthsize(struct crypto_aead *ctfm, 3273 unsigned int authsize) 3274 { 3275 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 3276 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3277 3278 /* Keep fallback cipher synchronized */ 3279 return crypto_aead_setauthsize(ctx->fback, authsize); 3280 } 3281 3282 static int safexcel_aead_fallback_crypt(struct aead_request *req, 3283 enum safexcel_cipher_direction dir) 3284 { 3285 struct crypto_aead *aead = crypto_aead_reqtfm(req); 3286 struct crypto_tfm *tfm = crypto_aead_tfm(aead); 3287 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3288 struct aead_request *subreq = aead_request_ctx(req); 3289 3290 aead_request_set_tfm(subreq, ctx->fback); 3291 aead_request_set_callback(subreq, req->base.flags, req->base.complete, 3292 req->base.data); 3293 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, 3294 req->iv); 3295 aead_request_set_ad(subreq, req->assoclen); 3296 3297 return (dir == SAFEXCEL_ENCRYPT) ? 3298 crypto_aead_encrypt(subreq) : 3299 crypto_aead_decrypt(subreq); 3300 } 3301 3302 static int safexcel_aead_sm4cbc_sm3_encrypt(struct aead_request *req) 3303 { 3304 struct safexcel_cipher_req *creq = aead_request_ctx(req); 3305 3306 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3307 if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) 3308 return -EINVAL; 3309 else if (req->cryptlen || req->assoclen) /* If input length > 0 only */ 3310 return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); 3311 3312 /* HW cannot do full (AAD+payload) zero length, use fallback */ 3313 return safexcel_aead_fallback_crypt(req, SAFEXCEL_ENCRYPT); 3314 } 3315 3316 static int safexcel_aead_sm4cbc_sm3_decrypt(struct aead_request *req) 3317 { 3318 struct safexcel_cipher_req *creq = aead_request_ctx(req); 3319 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3320 3321 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3322 if ((req->cryptlen - crypto_aead_authsize(tfm)) & (SM4_BLOCK_SIZE - 1)) 3323 return -EINVAL; 3324 else if (req->cryptlen > crypto_aead_authsize(tfm) || req->assoclen) 3325 /* If input length > 0 only */ 3326 return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); 3327 3328 /* HW cannot do full (AAD+payload) zero length, use fallback */ 3329 return safexcel_aead_fallback_crypt(req, SAFEXCEL_DECRYPT); 3330 } 3331 3332 static int safexcel_aead_sm4cbc_sm3_cra_init(struct crypto_tfm *tfm) 3333 { 3334 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3335 3336 safexcel_aead_fallback_cra_init(tfm); 3337 ctx->alg = SAFEXCEL_SM4; 3338 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3; 3339 ctx->state_sz = SM3_DIGEST_SIZE; 3340 return 0; 3341 } 3342 3343 struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_cbc_sm4 = { 3344 .type = SAFEXCEL_ALG_TYPE_AEAD, 3345 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SM3, 3346 .alg.aead = { 3347 .setkey = safexcel_aead_fallback_setkey, 3348 .setauthsize = safexcel_aead_fallback_setauthsize, 3349 .encrypt = safexcel_aead_sm4cbc_sm3_encrypt, 3350 .decrypt = safexcel_aead_sm4cbc_sm3_decrypt, 3351 .ivsize = SM4_BLOCK_SIZE, 3352 .maxauthsize = SM3_DIGEST_SIZE, 3353 .base = { 3354 .cra_name = "authenc(hmac(sm3),cbc(sm4))", 3355 .cra_driver_name = "safexcel-authenc-hmac-sm3-cbc-sm4", 3356 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3357 .cra_flags = CRYPTO_ALG_ASYNC | 3358 CRYPTO_ALG_KERN_DRIVER_ONLY | 3359 CRYPTO_ALG_NEED_FALLBACK, 3360 .cra_blocksize = SM4_BLOCK_SIZE, 3361 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3362 .cra_alignmask = 0, 3363 .cra_init = safexcel_aead_sm4cbc_sm3_cra_init, 3364 .cra_exit = safexcel_aead_fallback_cra_exit, 3365 .cra_module = THIS_MODULE, 3366 }, 3367 }, 3368 }; 3369 3370 static int safexcel_aead_sm4ctr_sha1_cra_init(struct crypto_tfm *tfm) 3371 { 3372 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3373 3374 safexcel_aead_sm4cbc_sha1_cra_init(tfm); 3375 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; 3376 return 0; 3377 } 3378 3379 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_ctr_sm4 = { 3380 .type = SAFEXCEL_ALG_TYPE_AEAD, 3381 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SHA1, 3382 .alg.aead = { 3383 .setkey = safexcel_aead_setkey, 3384 .encrypt = safexcel_aead_encrypt, 3385 .decrypt = safexcel_aead_decrypt, 3386 .ivsize = CTR_RFC3686_IV_SIZE, 3387 .maxauthsize = SHA1_DIGEST_SIZE, 3388 .base = { 3389 .cra_name = "authenc(hmac(sha1),rfc3686(ctr(sm4)))", 3390 .cra_driver_name = "safexcel-authenc-hmac-sha1-ctr-sm4", 3391 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3392 .cra_flags = CRYPTO_ALG_ASYNC | 3393 CRYPTO_ALG_KERN_DRIVER_ONLY, 3394 .cra_blocksize = 1, 3395 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3396 .cra_alignmask = 0, 3397 .cra_init = safexcel_aead_sm4ctr_sha1_cra_init, 3398 .cra_exit = safexcel_aead_cra_exit, 3399 .cra_module = THIS_MODULE, 3400 }, 3401 }, 3402 }; 3403 3404 static int safexcel_aead_sm4ctr_sm3_cra_init(struct crypto_tfm *tfm) 3405 { 3406 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3407 3408 safexcel_aead_sm4cbc_sm3_cra_init(tfm); 3409 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; 3410 return 0; 3411 } 3412 3413 struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_ctr_sm4 = { 3414 .type = SAFEXCEL_ALG_TYPE_AEAD, 3415 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SM3, 3416 .alg.aead = { 3417 .setkey = safexcel_aead_setkey, 3418 .encrypt = safexcel_aead_encrypt, 3419 .decrypt = safexcel_aead_decrypt, 3420 .ivsize = CTR_RFC3686_IV_SIZE, 3421 .maxauthsize = SM3_DIGEST_SIZE, 3422 .base = { 3423 .cra_name = "authenc(hmac(sm3),rfc3686(ctr(sm4)))", 3424 .cra_driver_name = "safexcel-authenc-hmac-sm3-ctr-sm4", 3425 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3426 .cra_flags = CRYPTO_ALG_ASYNC | 3427 CRYPTO_ALG_KERN_DRIVER_ONLY, 3428 .cra_blocksize = 1, 3429 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3430 .cra_alignmask = 0, 3431 .cra_init = safexcel_aead_sm4ctr_sm3_cra_init, 3432 .cra_exit = safexcel_aead_cra_exit, 3433 .cra_module = THIS_MODULE, 3434 }, 3435 }, 3436 }; 3437 3438 static int safexcel_rfc4106_gcm_setkey(struct crypto_aead *ctfm, const u8 *key, 3439 unsigned int len) 3440 { 3441 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 3442 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3443 3444 /* last 4 bytes of key are the nonce! */ 3445 ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE); 3446 3447 len -= CTR_RFC3686_NONCE_SIZE; 3448 return safexcel_aead_gcm_setkey(ctfm, key, len); 3449 } 3450 3451 static int safexcel_rfc4106_gcm_setauthsize(struct crypto_aead *tfm, 3452 unsigned int authsize) 3453 { 3454 return crypto_rfc4106_check_authsize(authsize); 3455 } 3456 3457 static int safexcel_rfc4106_encrypt(struct aead_request *req) 3458 { 3459 return crypto_ipsec_check_assoclen(req->assoclen) ?: 3460 safexcel_aead_encrypt(req); 3461 } 3462 3463 static int safexcel_rfc4106_decrypt(struct aead_request *req) 3464 { 3465 return crypto_ipsec_check_assoclen(req->assoclen) ?: 3466 safexcel_aead_decrypt(req); 3467 } 3468 3469 static int safexcel_rfc4106_gcm_cra_init(struct crypto_tfm *tfm) 3470 { 3471 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3472 int ret; 3473 3474 ret = safexcel_aead_gcm_cra_init(tfm); 3475 ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP; 3476 return ret; 3477 } 3478 3479 struct safexcel_alg_template safexcel_alg_rfc4106_gcm = { 3480 .type = SAFEXCEL_ALG_TYPE_AEAD, 3481 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH, 3482 .alg.aead = { 3483 .setkey = safexcel_rfc4106_gcm_setkey, 3484 .setauthsize = safexcel_rfc4106_gcm_setauthsize, 3485 .encrypt = safexcel_rfc4106_encrypt, 3486 .decrypt = safexcel_rfc4106_decrypt, 3487 .ivsize = GCM_RFC4106_IV_SIZE, 3488 .maxauthsize = GHASH_DIGEST_SIZE, 3489 .base = { 3490 .cra_name = "rfc4106(gcm(aes))", 3491 .cra_driver_name = "safexcel-rfc4106-gcm-aes", 3492 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3493 .cra_flags = CRYPTO_ALG_ASYNC | 3494 CRYPTO_ALG_KERN_DRIVER_ONLY, 3495 .cra_blocksize = 1, 3496 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3497 .cra_alignmask = 0, 3498 .cra_init = safexcel_rfc4106_gcm_cra_init, 3499 .cra_exit = safexcel_aead_gcm_cra_exit, 3500 }, 3501 }, 3502 }; 3503 3504 static int safexcel_rfc4543_gcm_setauthsize(struct crypto_aead *tfm, 3505 unsigned int authsize) 3506 { 3507 if (authsize != GHASH_DIGEST_SIZE) 3508 return -EINVAL; 3509 3510 return 0; 3511 } 3512 3513 static int safexcel_rfc4543_gcm_cra_init(struct crypto_tfm *tfm) 3514 { 3515 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3516 int ret; 3517 3518 ret = safexcel_aead_gcm_cra_init(tfm); 3519 ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP_GMAC; 3520 return ret; 3521 } 3522 3523 struct safexcel_alg_template safexcel_alg_rfc4543_gcm = { 3524 .type = SAFEXCEL_ALG_TYPE_AEAD, 3525 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH, 3526 .alg.aead = { 3527 .setkey = safexcel_rfc4106_gcm_setkey, 3528 .setauthsize = safexcel_rfc4543_gcm_setauthsize, 3529 .encrypt = safexcel_rfc4106_encrypt, 3530 .decrypt = safexcel_rfc4106_decrypt, 3531 .ivsize = GCM_RFC4543_IV_SIZE, 3532 .maxauthsize = GHASH_DIGEST_SIZE, 3533 .base = { 3534 .cra_name = "rfc4543(gcm(aes))", 3535 .cra_driver_name = "safexcel-rfc4543-gcm-aes", 3536 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3537 .cra_flags = CRYPTO_ALG_ASYNC | 3538 CRYPTO_ALG_KERN_DRIVER_ONLY, 3539 .cra_blocksize = 1, 3540 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3541 .cra_alignmask = 0, 3542 .cra_init = safexcel_rfc4543_gcm_cra_init, 3543 .cra_exit = safexcel_aead_gcm_cra_exit, 3544 }, 3545 }, 3546 }; 3547 3548 static int safexcel_rfc4309_ccm_setkey(struct crypto_aead *ctfm, const u8 *key, 3549 unsigned int len) 3550 { 3551 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 3552 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3553 3554 /* First byte of the nonce = L = always 3 for RFC4309 (4 byte ctr) */ 3555 *(u8 *)&ctx->nonce = EIP197_AEAD_IPSEC_COUNTER_SIZE - 1; 3556 /* last 3 bytes of key are the nonce! */ 3557 memcpy((u8 *)&ctx->nonce + 1, key + len - 3558 EIP197_AEAD_IPSEC_CCM_NONCE_SIZE, 3559 EIP197_AEAD_IPSEC_CCM_NONCE_SIZE); 3560 3561 len -= EIP197_AEAD_IPSEC_CCM_NONCE_SIZE; 3562 return safexcel_aead_ccm_setkey(ctfm, key, len); 3563 } 3564 3565 static int safexcel_rfc4309_ccm_setauthsize(struct crypto_aead *tfm, 3566 unsigned int authsize) 3567 { 3568 /* Borrowed from crypto/ccm.c */ 3569 switch (authsize) { 3570 case 8: 3571 case 12: 3572 case 16: 3573 break; 3574 default: 3575 return -EINVAL; 3576 } 3577 3578 return 0; 3579 } 3580 3581 static int safexcel_rfc4309_ccm_encrypt(struct aead_request *req) 3582 { 3583 struct safexcel_cipher_req *creq = aead_request_ctx(req); 3584 3585 /* Borrowed from crypto/ccm.c */ 3586 if (req->assoclen != 16 && req->assoclen != 20) 3587 return -EINVAL; 3588 3589 return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); 3590 } 3591 3592 static int safexcel_rfc4309_ccm_decrypt(struct aead_request *req) 3593 { 3594 struct safexcel_cipher_req *creq = aead_request_ctx(req); 3595 3596 /* Borrowed from crypto/ccm.c */ 3597 if (req->assoclen != 16 && req->assoclen != 20) 3598 return -EINVAL; 3599 3600 return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); 3601 } 3602 3603 static int safexcel_rfc4309_ccm_cra_init(struct crypto_tfm *tfm) 3604 { 3605 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3606 int ret; 3607 3608 ret = safexcel_aead_ccm_cra_init(tfm); 3609 ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP; 3610 return ret; 3611 } 3612 3613 struct safexcel_alg_template safexcel_alg_rfc4309_ccm = { 3614 .type = SAFEXCEL_ALG_TYPE_AEAD, 3615 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_CBC_MAC_ALL, 3616 .alg.aead = { 3617 .setkey = safexcel_rfc4309_ccm_setkey, 3618 .setauthsize = safexcel_rfc4309_ccm_setauthsize, 3619 .encrypt = safexcel_rfc4309_ccm_encrypt, 3620 .decrypt = safexcel_rfc4309_ccm_decrypt, 3621 .ivsize = EIP197_AEAD_IPSEC_IV_SIZE, 3622 .maxauthsize = AES_BLOCK_SIZE, 3623 .base = { 3624 .cra_name = "rfc4309(ccm(aes))", 3625 .cra_driver_name = "safexcel-rfc4309-ccm-aes", 3626 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3627 .cra_flags = CRYPTO_ALG_ASYNC | 3628 CRYPTO_ALG_KERN_DRIVER_ONLY, 3629 .cra_blocksize = 1, 3630 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3631 .cra_alignmask = 0, 3632 .cra_init = safexcel_rfc4309_ccm_cra_init, 3633 .cra_exit = safexcel_aead_cra_exit, 3634 .cra_module = THIS_MODULE, 3635 }, 3636 }, 3637 }; 3638