1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/arch/arm64/crypto/aes-glue.c - wrapper code for ARMv8 AES 4 * 5 * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> 6 */ 7 8 #include <asm/neon.h> 9 #include <asm/hwcap.h> 10 #include <asm/simd.h> 11 #include <crypto/aes.h> 12 #include <crypto/ctr.h> 13 #include <crypto/sha2.h> 14 #include <crypto/internal/hash.h> 15 #include <crypto/internal/simd.h> 16 #include <crypto/internal/skcipher.h> 17 #include <crypto/scatterwalk.h> 18 #include <linux/module.h> 19 #include <linux/cpufeature.h> 20 #include <crypto/xts.h> 21 22 #include "aes-ce-setkey.h" 23 24 #ifdef USE_V8_CRYPTO_EXTENSIONS 25 #define MODE "ce" 26 #define PRIO 300 27 #define STRIDE 5 28 #define aes_expandkey ce_aes_expandkey 29 #define aes_ecb_encrypt ce_aes_ecb_encrypt 30 #define aes_ecb_decrypt ce_aes_ecb_decrypt 31 #define aes_cbc_encrypt ce_aes_cbc_encrypt 32 #define aes_cbc_decrypt ce_aes_cbc_decrypt 33 #define aes_cbc_cts_encrypt ce_aes_cbc_cts_encrypt 34 #define aes_cbc_cts_decrypt ce_aes_cbc_cts_decrypt 35 #define aes_essiv_cbc_encrypt ce_aes_essiv_cbc_encrypt 36 #define aes_essiv_cbc_decrypt ce_aes_essiv_cbc_decrypt 37 #define aes_ctr_encrypt ce_aes_ctr_encrypt 38 #define aes_xts_encrypt ce_aes_xts_encrypt 39 #define aes_xts_decrypt ce_aes_xts_decrypt 40 #define aes_mac_update ce_aes_mac_update 41 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions"); 42 #else 43 #define MODE "neon" 44 #define PRIO 200 45 #define STRIDE 4 46 #define aes_ecb_encrypt neon_aes_ecb_encrypt 47 #define aes_ecb_decrypt neon_aes_ecb_decrypt 48 #define aes_cbc_encrypt neon_aes_cbc_encrypt 49 #define aes_cbc_decrypt neon_aes_cbc_decrypt 50 #define aes_cbc_cts_encrypt neon_aes_cbc_cts_encrypt 51 #define aes_cbc_cts_decrypt neon_aes_cbc_cts_decrypt 52 #define aes_essiv_cbc_encrypt neon_aes_essiv_cbc_encrypt 53 #define aes_essiv_cbc_decrypt neon_aes_essiv_cbc_decrypt 54 #define aes_ctr_encrypt neon_aes_ctr_encrypt 55 #define aes_xts_encrypt neon_aes_xts_encrypt 56 #define aes_xts_decrypt neon_aes_xts_decrypt 57 #define aes_mac_update neon_aes_mac_update 58 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 NEON"); 59 #endif 60 #if defined(USE_V8_CRYPTO_EXTENSIONS) || !IS_ENABLED(CONFIG_CRYPTO_AES_ARM64_BS) 61 MODULE_ALIAS_CRYPTO("ecb(aes)"); 62 MODULE_ALIAS_CRYPTO("cbc(aes)"); 63 MODULE_ALIAS_CRYPTO("ctr(aes)"); 64 MODULE_ALIAS_CRYPTO("xts(aes)"); 65 #endif 66 MODULE_ALIAS_CRYPTO("cts(cbc(aes))"); 67 MODULE_ALIAS_CRYPTO("essiv(cbc(aes),sha256)"); 68 MODULE_ALIAS_CRYPTO("cmac(aes)"); 69 MODULE_ALIAS_CRYPTO("xcbc(aes)"); 70 MODULE_ALIAS_CRYPTO("cbcmac(aes)"); 71 72 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 73 MODULE_LICENSE("GPL v2"); 74 75 /* defined in aes-modes.S */ 76 asmlinkage void aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[], 77 int rounds, int blocks); 78 asmlinkage void aes_ecb_decrypt(u8 out[], u8 const in[], u32 const rk[], 79 int rounds, int blocks); 80 81 asmlinkage void aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[], 82 int rounds, int blocks, u8 iv[]); 83 asmlinkage void aes_cbc_decrypt(u8 out[], u8 const in[], u32 const rk[], 84 int rounds, int blocks, u8 iv[]); 85 86 asmlinkage void aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[], 87 int rounds, int bytes, u8 const iv[]); 88 asmlinkage void aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[], 89 int rounds, int bytes, u8 const iv[]); 90 91 asmlinkage void aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[], 92 int rounds, int bytes, u8 ctr[], u8 finalbuf[]); 93 94 asmlinkage void aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[], 95 int rounds, int bytes, u32 const rk2[], u8 iv[], 96 int first); 97 asmlinkage void aes_xts_decrypt(u8 out[], u8 const in[], u32 const rk1[], 98 int rounds, int bytes, u32 const rk2[], u8 iv[], 99 int first); 100 101 asmlinkage void aes_essiv_cbc_encrypt(u8 out[], u8 const in[], u32 const rk1[], 102 int rounds, int blocks, u8 iv[], 103 u32 const rk2[]); 104 asmlinkage void aes_essiv_cbc_decrypt(u8 out[], u8 const in[], u32 const rk1[], 105 int rounds, int blocks, u8 iv[], 106 u32 const rk2[]); 107 108 asmlinkage int aes_mac_update(u8 const in[], u32 const rk[], int rounds, 109 int blocks, u8 dg[], int enc_before, 110 int enc_after); 111 112 struct crypto_aes_xts_ctx { 113 struct crypto_aes_ctx key1; 114 struct crypto_aes_ctx __aligned(8) key2; 115 }; 116 117 struct crypto_aes_essiv_cbc_ctx { 118 struct crypto_aes_ctx key1; 119 struct crypto_aes_ctx __aligned(8) key2; 120 struct crypto_shash *hash; 121 }; 122 123 struct mac_tfm_ctx { 124 struct crypto_aes_ctx key; 125 u8 __aligned(8) consts[]; 126 }; 127 128 struct mac_desc_ctx { 129 unsigned int len; 130 u8 dg[AES_BLOCK_SIZE]; 131 }; 132 133 static int skcipher_aes_setkey(struct crypto_skcipher *tfm, const u8 *in_key, 134 unsigned int key_len) 135 { 136 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 137 138 return aes_expandkey(ctx, in_key, key_len); 139 } 140 141 static int __maybe_unused xts_set_key(struct crypto_skcipher *tfm, 142 const u8 *in_key, unsigned int key_len) 143 { 144 struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 145 int ret; 146 147 ret = xts_verify_key(tfm, in_key, key_len); 148 if (ret) 149 return ret; 150 151 ret = aes_expandkey(&ctx->key1, in_key, key_len / 2); 152 if (!ret) 153 ret = aes_expandkey(&ctx->key2, &in_key[key_len / 2], 154 key_len / 2); 155 return ret; 156 } 157 158 static int __maybe_unused essiv_cbc_set_key(struct crypto_skcipher *tfm, 159 const u8 *in_key, 160 unsigned int key_len) 161 { 162 struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); 163 u8 digest[SHA256_DIGEST_SIZE]; 164 int ret; 165 166 ret = aes_expandkey(&ctx->key1, in_key, key_len); 167 if (ret) 168 return ret; 169 170 crypto_shash_tfm_digest(ctx->hash, in_key, key_len, digest); 171 172 return aes_expandkey(&ctx->key2, digest, sizeof(digest)); 173 } 174 175 static int __maybe_unused ecb_encrypt(struct skcipher_request *req) 176 { 177 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 178 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 179 int err, rounds = 6 + ctx->key_length / 4; 180 struct skcipher_walk walk; 181 unsigned int blocks; 182 183 err = skcipher_walk_virt(&walk, req, false); 184 185 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { 186 kernel_neon_begin(); 187 aes_ecb_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 188 ctx->key_enc, rounds, blocks); 189 kernel_neon_end(); 190 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 191 } 192 return err; 193 } 194 195 static int __maybe_unused ecb_decrypt(struct skcipher_request *req) 196 { 197 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 198 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 199 int err, rounds = 6 + ctx->key_length / 4; 200 struct skcipher_walk walk; 201 unsigned int blocks; 202 203 err = skcipher_walk_virt(&walk, req, false); 204 205 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { 206 kernel_neon_begin(); 207 aes_ecb_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 208 ctx->key_dec, rounds, blocks); 209 kernel_neon_end(); 210 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 211 } 212 return err; 213 } 214 215 static int cbc_encrypt_walk(struct skcipher_request *req, 216 struct skcipher_walk *walk) 217 { 218 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 219 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 220 int err = 0, rounds = 6 + ctx->key_length / 4; 221 unsigned int blocks; 222 223 while ((blocks = (walk->nbytes / AES_BLOCK_SIZE))) { 224 kernel_neon_begin(); 225 aes_cbc_encrypt(walk->dst.virt.addr, walk->src.virt.addr, 226 ctx->key_enc, rounds, blocks, walk->iv); 227 kernel_neon_end(); 228 err = skcipher_walk_done(walk, walk->nbytes % AES_BLOCK_SIZE); 229 } 230 return err; 231 } 232 233 static int __maybe_unused cbc_encrypt(struct skcipher_request *req) 234 { 235 struct skcipher_walk walk; 236 int err; 237 238 err = skcipher_walk_virt(&walk, req, false); 239 if (err) 240 return err; 241 return cbc_encrypt_walk(req, &walk); 242 } 243 244 static int cbc_decrypt_walk(struct skcipher_request *req, 245 struct skcipher_walk *walk) 246 { 247 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 248 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 249 int err = 0, rounds = 6 + ctx->key_length / 4; 250 unsigned int blocks; 251 252 while ((blocks = (walk->nbytes / AES_BLOCK_SIZE))) { 253 kernel_neon_begin(); 254 aes_cbc_decrypt(walk->dst.virt.addr, walk->src.virt.addr, 255 ctx->key_dec, rounds, blocks, walk->iv); 256 kernel_neon_end(); 257 err = skcipher_walk_done(walk, walk->nbytes % AES_BLOCK_SIZE); 258 } 259 return err; 260 } 261 262 static int __maybe_unused cbc_decrypt(struct skcipher_request *req) 263 { 264 struct skcipher_walk walk; 265 int err; 266 267 err = skcipher_walk_virt(&walk, req, false); 268 if (err) 269 return err; 270 return cbc_decrypt_walk(req, &walk); 271 } 272 273 static int cts_cbc_encrypt(struct skcipher_request *req) 274 { 275 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 276 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 277 int err, rounds = 6 + ctx->key_length / 4; 278 int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2; 279 struct scatterlist *src = req->src, *dst = req->dst; 280 struct scatterlist sg_src[2], sg_dst[2]; 281 struct skcipher_request subreq; 282 struct skcipher_walk walk; 283 284 skcipher_request_set_tfm(&subreq, tfm); 285 skcipher_request_set_callback(&subreq, skcipher_request_flags(req), 286 NULL, NULL); 287 288 if (req->cryptlen <= AES_BLOCK_SIZE) { 289 if (req->cryptlen < AES_BLOCK_SIZE) 290 return -EINVAL; 291 cbc_blocks = 1; 292 } 293 294 if (cbc_blocks > 0) { 295 skcipher_request_set_crypt(&subreq, req->src, req->dst, 296 cbc_blocks * AES_BLOCK_SIZE, 297 req->iv); 298 299 err = skcipher_walk_virt(&walk, &subreq, false) ?: 300 cbc_encrypt_walk(&subreq, &walk); 301 if (err) 302 return err; 303 304 if (req->cryptlen == AES_BLOCK_SIZE) 305 return 0; 306 307 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen); 308 if (req->dst != req->src) 309 dst = scatterwalk_ffwd(sg_dst, req->dst, 310 subreq.cryptlen); 311 } 312 313 /* handle ciphertext stealing */ 314 skcipher_request_set_crypt(&subreq, src, dst, 315 req->cryptlen - cbc_blocks * AES_BLOCK_SIZE, 316 req->iv); 317 318 err = skcipher_walk_virt(&walk, &subreq, false); 319 if (err) 320 return err; 321 322 kernel_neon_begin(); 323 aes_cbc_cts_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 324 ctx->key_enc, rounds, walk.nbytes, walk.iv); 325 kernel_neon_end(); 326 327 return skcipher_walk_done(&walk, 0); 328 } 329 330 static int cts_cbc_decrypt(struct skcipher_request *req) 331 { 332 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 333 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 334 int err, rounds = 6 + ctx->key_length / 4; 335 int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2; 336 struct scatterlist *src = req->src, *dst = req->dst; 337 struct scatterlist sg_src[2], sg_dst[2]; 338 struct skcipher_request subreq; 339 struct skcipher_walk walk; 340 341 skcipher_request_set_tfm(&subreq, tfm); 342 skcipher_request_set_callback(&subreq, skcipher_request_flags(req), 343 NULL, NULL); 344 345 if (req->cryptlen <= AES_BLOCK_SIZE) { 346 if (req->cryptlen < AES_BLOCK_SIZE) 347 return -EINVAL; 348 cbc_blocks = 1; 349 } 350 351 if (cbc_blocks > 0) { 352 skcipher_request_set_crypt(&subreq, req->src, req->dst, 353 cbc_blocks * AES_BLOCK_SIZE, 354 req->iv); 355 356 err = skcipher_walk_virt(&walk, &subreq, false) ?: 357 cbc_decrypt_walk(&subreq, &walk); 358 if (err) 359 return err; 360 361 if (req->cryptlen == AES_BLOCK_SIZE) 362 return 0; 363 364 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen); 365 if (req->dst != req->src) 366 dst = scatterwalk_ffwd(sg_dst, req->dst, 367 subreq.cryptlen); 368 } 369 370 /* handle ciphertext stealing */ 371 skcipher_request_set_crypt(&subreq, src, dst, 372 req->cryptlen - cbc_blocks * AES_BLOCK_SIZE, 373 req->iv); 374 375 err = skcipher_walk_virt(&walk, &subreq, false); 376 if (err) 377 return err; 378 379 kernel_neon_begin(); 380 aes_cbc_cts_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 381 ctx->key_dec, rounds, walk.nbytes, walk.iv); 382 kernel_neon_end(); 383 384 return skcipher_walk_done(&walk, 0); 385 } 386 387 static int __maybe_unused essiv_cbc_init_tfm(struct crypto_skcipher *tfm) 388 { 389 struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); 390 391 ctx->hash = crypto_alloc_shash("sha256", 0, 0); 392 393 return PTR_ERR_OR_ZERO(ctx->hash); 394 } 395 396 static void __maybe_unused essiv_cbc_exit_tfm(struct crypto_skcipher *tfm) 397 { 398 struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); 399 400 crypto_free_shash(ctx->hash); 401 } 402 403 static int __maybe_unused essiv_cbc_encrypt(struct skcipher_request *req) 404 { 405 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 406 struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); 407 int err, rounds = 6 + ctx->key1.key_length / 4; 408 struct skcipher_walk walk; 409 unsigned int blocks; 410 411 err = skcipher_walk_virt(&walk, req, false); 412 413 blocks = walk.nbytes / AES_BLOCK_SIZE; 414 if (blocks) { 415 kernel_neon_begin(); 416 aes_essiv_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 417 ctx->key1.key_enc, rounds, blocks, 418 req->iv, ctx->key2.key_enc); 419 kernel_neon_end(); 420 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 421 } 422 return err ?: cbc_encrypt_walk(req, &walk); 423 } 424 425 static int __maybe_unused essiv_cbc_decrypt(struct skcipher_request *req) 426 { 427 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 428 struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); 429 int err, rounds = 6 + ctx->key1.key_length / 4; 430 struct skcipher_walk walk; 431 unsigned int blocks; 432 433 err = skcipher_walk_virt(&walk, req, false); 434 435 blocks = walk.nbytes / AES_BLOCK_SIZE; 436 if (blocks) { 437 kernel_neon_begin(); 438 aes_essiv_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 439 ctx->key1.key_dec, rounds, blocks, 440 req->iv, ctx->key2.key_enc); 441 kernel_neon_end(); 442 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 443 } 444 return err ?: cbc_decrypt_walk(req, &walk); 445 } 446 447 static int ctr_encrypt(struct skcipher_request *req) 448 { 449 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 450 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 451 int err, rounds = 6 + ctx->key_length / 4; 452 struct skcipher_walk walk; 453 454 err = skcipher_walk_virt(&walk, req, false); 455 456 while (walk.nbytes > 0) { 457 const u8 *src = walk.src.virt.addr; 458 unsigned int nbytes = walk.nbytes; 459 u8 *dst = walk.dst.virt.addr; 460 u8 buf[AES_BLOCK_SIZE]; 461 unsigned int tail; 462 463 if (unlikely(nbytes < AES_BLOCK_SIZE)) 464 src = memcpy(buf, src, nbytes); 465 else if (nbytes < walk.total) 466 nbytes &= ~(AES_BLOCK_SIZE - 1); 467 468 kernel_neon_begin(); 469 aes_ctr_encrypt(dst, src, ctx->key_enc, rounds, nbytes, 470 walk.iv, buf); 471 kernel_neon_end(); 472 473 tail = nbytes % (STRIDE * AES_BLOCK_SIZE); 474 if (tail > 0 && tail < AES_BLOCK_SIZE) 475 /* 476 * The final partial block could not be returned using 477 * an overlapping store, so it was passed via buf[] 478 * instead. 479 */ 480 memcpy(dst + nbytes - tail, buf, tail); 481 482 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 483 } 484 485 return err; 486 } 487 488 static void ctr_encrypt_one(struct crypto_skcipher *tfm, const u8 *src, u8 *dst) 489 { 490 const struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 491 unsigned long flags; 492 493 /* 494 * Temporarily disable interrupts to avoid races where 495 * cachelines are evicted when the CPU is interrupted 496 * to do something else. 497 */ 498 local_irq_save(flags); 499 aes_encrypt(ctx, dst, src); 500 local_irq_restore(flags); 501 } 502 503 static int __maybe_unused ctr_encrypt_sync(struct skcipher_request *req) 504 { 505 if (!crypto_simd_usable()) 506 return crypto_ctr_encrypt_walk(req, ctr_encrypt_one); 507 508 return ctr_encrypt(req); 509 } 510 511 static int __maybe_unused xts_encrypt(struct skcipher_request *req) 512 { 513 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 514 struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 515 int err, first, rounds = 6 + ctx->key1.key_length / 4; 516 int tail = req->cryptlen % AES_BLOCK_SIZE; 517 struct scatterlist sg_src[2], sg_dst[2]; 518 struct skcipher_request subreq; 519 struct scatterlist *src, *dst; 520 struct skcipher_walk walk; 521 522 if (req->cryptlen < AES_BLOCK_SIZE) 523 return -EINVAL; 524 525 err = skcipher_walk_virt(&walk, req, false); 526 527 if (unlikely(tail > 0 && walk.nbytes < walk.total)) { 528 int xts_blocks = DIV_ROUND_UP(req->cryptlen, 529 AES_BLOCK_SIZE) - 2; 530 531 skcipher_walk_abort(&walk); 532 533 skcipher_request_set_tfm(&subreq, tfm); 534 skcipher_request_set_callback(&subreq, 535 skcipher_request_flags(req), 536 NULL, NULL); 537 skcipher_request_set_crypt(&subreq, req->src, req->dst, 538 xts_blocks * AES_BLOCK_SIZE, 539 req->iv); 540 req = &subreq; 541 err = skcipher_walk_virt(&walk, req, false); 542 } else { 543 tail = 0; 544 } 545 546 for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) { 547 int nbytes = walk.nbytes; 548 549 if (walk.nbytes < walk.total) 550 nbytes &= ~(AES_BLOCK_SIZE - 1); 551 552 kernel_neon_begin(); 553 aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 554 ctx->key1.key_enc, rounds, nbytes, 555 ctx->key2.key_enc, walk.iv, first); 556 kernel_neon_end(); 557 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 558 } 559 560 if (err || likely(!tail)) 561 return err; 562 563 dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen); 564 if (req->dst != req->src) 565 dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen); 566 567 skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail, 568 req->iv); 569 570 err = skcipher_walk_virt(&walk, &subreq, false); 571 if (err) 572 return err; 573 574 kernel_neon_begin(); 575 aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 576 ctx->key1.key_enc, rounds, walk.nbytes, 577 ctx->key2.key_enc, walk.iv, first); 578 kernel_neon_end(); 579 580 return skcipher_walk_done(&walk, 0); 581 } 582 583 static int __maybe_unused xts_decrypt(struct skcipher_request *req) 584 { 585 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 586 struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 587 int err, first, rounds = 6 + ctx->key1.key_length / 4; 588 int tail = req->cryptlen % AES_BLOCK_SIZE; 589 struct scatterlist sg_src[2], sg_dst[2]; 590 struct skcipher_request subreq; 591 struct scatterlist *src, *dst; 592 struct skcipher_walk walk; 593 594 if (req->cryptlen < AES_BLOCK_SIZE) 595 return -EINVAL; 596 597 err = skcipher_walk_virt(&walk, req, false); 598 599 if (unlikely(tail > 0 && walk.nbytes < walk.total)) { 600 int xts_blocks = DIV_ROUND_UP(req->cryptlen, 601 AES_BLOCK_SIZE) - 2; 602 603 skcipher_walk_abort(&walk); 604 605 skcipher_request_set_tfm(&subreq, tfm); 606 skcipher_request_set_callback(&subreq, 607 skcipher_request_flags(req), 608 NULL, NULL); 609 skcipher_request_set_crypt(&subreq, req->src, req->dst, 610 xts_blocks * AES_BLOCK_SIZE, 611 req->iv); 612 req = &subreq; 613 err = skcipher_walk_virt(&walk, req, false); 614 } else { 615 tail = 0; 616 } 617 618 for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) { 619 int nbytes = walk.nbytes; 620 621 if (walk.nbytes < walk.total) 622 nbytes &= ~(AES_BLOCK_SIZE - 1); 623 624 kernel_neon_begin(); 625 aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 626 ctx->key1.key_dec, rounds, nbytes, 627 ctx->key2.key_enc, walk.iv, first); 628 kernel_neon_end(); 629 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 630 } 631 632 if (err || likely(!tail)) 633 return err; 634 635 dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen); 636 if (req->dst != req->src) 637 dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen); 638 639 skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail, 640 req->iv); 641 642 err = skcipher_walk_virt(&walk, &subreq, false); 643 if (err) 644 return err; 645 646 647 kernel_neon_begin(); 648 aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 649 ctx->key1.key_dec, rounds, walk.nbytes, 650 ctx->key2.key_enc, walk.iv, first); 651 kernel_neon_end(); 652 653 return skcipher_walk_done(&walk, 0); 654 } 655 656 static struct skcipher_alg aes_algs[] = { { 657 #if defined(USE_V8_CRYPTO_EXTENSIONS) || !IS_ENABLED(CONFIG_CRYPTO_AES_ARM64_BS) 658 .base = { 659 .cra_name = "__ecb(aes)", 660 .cra_driver_name = "__ecb-aes-" MODE, 661 .cra_priority = PRIO, 662 .cra_flags = CRYPTO_ALG_INTERNAL, 663 .cra_blocksize = AES_BLOCK_SIZE, 664 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 665 .cra_module = THIS_MODULE, 666 }, 667 .min_keysize = AES_MIN_KEY_SIZE, 668 .max_keysize = AES_MAX_KEY_SIZE, 669 .setkey = skcipher_aes_setkey, 670 .encrypt = ecb_encrypt, 671 .decrypt = ecb_decrypt, 672 }, { 673 .base = { 674 .cra_name = "__cbc(aes)", 675 .cra_driver_name = "__cbc-aes-" MODE, 676 .cra_priority = PRIO, 677 .cra_flags = CRYPTO_ALG_INTERNAL, 678 .cra_blocksize = AES_BLOCK_SIZE, 679 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 680 .cra_module = THIS_MODULE, 681 }, 682 .min_keysize = AES_MIN_KEY_SIZE, 683 .max_keysize = AES_MAX_KEY_SIZE, 684 .ivsize = AES_BLOCK_SIZE, 685 .setkey = skcipher_aes_setkey, 686 .encrypt = cbc_encrypt, 687 .decrypt = cbc_decrypt, 688 }, { 689 .base = { 690 .cra_name = "__ctr(aes)", 691 .cra_driver_name = "__ctr-aes-" MODE, 692 .cra_priority = PRIO, 693 .cra_flags = CRYPTO_ALG_INTERNAL, 694 .cra_blocksize = 1, 695 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 696 .cra_module = THIS_MODULE, 697 }, 698 .min_keysize = AES_MIN_KEY_SIZE, 699 .max_keysize = AES_MAX_KEY_SIZE, 700 .ivsize = AES_BLOCK_SIZE, 701 .chunksize = AES_BLOCK_SIZE, 702 .setkey = skcipher_aes_setkey, 703 .encrypt = ctr_encrypt, 704 .decrypt = ctr_encrypt, 705 }, { 706 .base = { 707 .cra_name = "ctr(aes)", 708 .cra_driver_name = "ctr-aes-" MODE, 709 .cra_priority = PRIO - 1, 710 .cra_blocksize = 1, 711 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 712 .cra_module = THIS_MODULE, 713 }, 714 .min_keysize = AES_MIN_KEY_SIZE, 715 .max_keysize = AES_MAX_KEY_SIZE, 716 .ivsize = AES_BLOCK_SIZE, 717 .chunksize = AES_BLOCK_SIZE, 718 .setkey = skcipher_aes_setkey, 719 .encrypt = ctr_encrypt_sync, 720 .decrypt = ctr_encrypt_sync, 721 }, { 722 .base = { 723 .cra_name = "__xts(aes)", 724 .cra_driver_name = "__xts-aes-" MODE, 725 .cra_priority = PRIO, 726 .cra_flags = CRYPTO_ALG_INTERNAL, 727 .cra_blocksize = AES_BLOCK_SIZE, 728 .cra_ctxsize = sizeof(struct crypto_aes_xts_ctx), 729 .cra_module = THIS_MODULE, 730 }, 731 .min_keysize = 2 * AES_MIN_KEY_SIZE, 732 .max_keysize = 2 * AES_MAX_KEY_SIZE, 733 .ivsize = AES_BLOCK_SIZE, 734 .walksize = 2 * AES_BLOCK_SIZE, 735 .setkey = xts_set_key, 736 .encrypt = xts_encrypt, 737 .decrypt = xts_decrypt, 738 }, { 739 #endif 740 .base = { 741 .cra_name = "__cts(cbc(aes))", 742 .cra_driver_name = "__cts-cbc-aes-" MODE, 743 .cra_priority = PRIO, 744 .cra_flags = CRYPTO_ALG_INTERNAL, 745 .cra_blocksize = AES_BLOCK_SIZE, 746 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 747 .cra_module = THIS_MODULE, 748 }, 749 .min_keysize = AES_MIN_KEY_SIZE, 750 .max_keysize = AES_MAX_KEY_SIZE, 751 .ivsize = AES_BLOCK_SIZE, 752 .walksize = 2 * AES_BLOCK_SIZE, 753 .setkey = skcipher_aes_setkey, 754 .encrypt = cts_cbc_encrypt, 755 .decrypt = cts_cbc_decrypt, 756 }, { 757 .base = { 758 .cra_name = "__essiv(cbc(aes),sha256)", 759 .cra_driver_name = "__essiv-cbc-aes-sha256-" MODE, 760 .cra_priority = PRIO + 1, 761 .cra_flags = CRYPTO_ALG_INTERNAL, 762 .cra_blocksize = AES_BLOCK_SIZE, 763 .cra_ctxsize = sizeof(struct crypto_aes_essiv_cbc_ctx), 764 .cra_module = THIS_MODULE, 765 }, 766 .min_keysize = AES_MIN_KEY_SIZE, 767 .max_keysize = AES_MAX_KEY_SIZE, 768 .ivsize = AES_BLOCK_SIZE, 769 .setkey = essiv_cbc_set_key, 770 .encrypt = essiv_cbc_encrypt, 771 .decrypt = essiv_cbc_decrypt, 772 .init = essiv_cbc_init_tfm, 773 .exit = essiv_cbc_exit_tfm, 774 } }; 775 776 static int cbcmac_setkey(struct crypto_shash *tfm, const u8 *in_key, 777 unsigned int key_len) 778 { 779 struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm); 780 781 return aes_expandkey(&ctx->key, in_key, key_len); 782 } 783 784 static void cmac_gf128_mul_by_x(be128 *y, const be128 *x) 785 { 786 u64 a = be64_to_cpu(x->a); 787 u64 b = be64_to_cpu(x->b); 788 789 y->a = cpu_to_be64((a << 1) | (b >> 63)); 790 y->b = cpu_to_be64((b << 1) ^ ((a >> 63) ? 0x87 : 0)); 791 } 792 793 static int cmac_setkey(struct crypto_shash *tfm, const u8 *in_key, 794 unsigned int key_len) 795 { 796 struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm); 797 be128 *consts = (be128 *)ctx->consts; 798 int rounds = 6 + key_len / 4; 799 int err; 800 801 err = cbcmac_setkey(tfm, in_key, key_len); 802 if (err) 803 return err; 804 805 /* encrypt the zero vector */ 806 kernel_neon_begin(); 807 aes_ecb_encrypt(ctx->consts, (u8[AES_BLOCK_SIZE]){}, ctx->key.key_enc, 808 rounds, 1); 809 kernel_neon_end(); 810 811 cmac_gf128_mul_by_x(consts, consts); 812 cmac_gf128_mul_by_x(consts + 1, consts); 813 814 return 0; 815 } 816 817 static int xcbc_setkey(struct crypto_shash *tfm, const u8 *in_key, 818 unsigned int key_len) 819 { 820 static u8 const ks[3][AES_BLOCK_SIZE] = { 821 { [0 ... AES_BLOCK_SIZE - 1] = 0x1 }, 822 { [0 ... AES_BLOCK_SIZE - 1] = 0x2 }, 823 { [0 ... AES_BLOCK_SIZE - 1] = 0x3 }, 824 }; 825 826 struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm); 827 int rounds = 6 + key_len / 4; 828 u8 key[AES_BLOCK_SIZE]; 829 int err; 830 831 err = cbcmac_setkey(tfm, in_key, key_len); 832 if (err) 833 return err; 834 835 kernel_neon_begin(); 836 aes_ecb_encrypt(key, ks[0], ctx->key.key_enc, rounds, 1); 837 aes_ecb_encrypt(ctx->consts, ks[1], ctx->key.key_enc, rounds, 2); 838 kernel_neon_end(); 839 840 return cbcmac_setkey(tfm, key, sizeof(key)); 841 } 842 843 static int mac_init(struct shash_desc *desc) 844 { 845 struct mac_desc_ctx *ctx = shash_desc_ctx(desc); 846 847 memset(ctx->dg, 0, AES_BLOCK_SIZE); 848 ctx->len = 0; 849 850 return 0; 851 } 852 853 static void mac_do_update(struct crypto_aes_ctx *ctx, u8 const in[], int blocks, 854 u8 dg[], int enc_before, int enc_after) 855 { 856 int rounds = 6 + ctx->key_length / 4; 857 858 if (crypto_simd_usable()) { 859 int rem; 860 861 do { 862 kernel_neon_begin(); 863 rem = aes_mac_update(in, ctx->key_enc, rounds, blocks, 864 dg, enc_before, enc_after); 865 kernel_neon_end(); 866 in += (blocks - rem) * AES_BLOCK_SIZE; 867 blocks = rem; 868 enc_before = 0; 869 } while (blocks); 870 } else { 871 if (enc_before) 872 aes_encrypt(ctx, dg, dg); 873 874 while (blocks--) { 875 crypto_xor(dg, in, AES_BLOCK_SIZE); 876 in += AES_BLOCK_SIZE; 877 878 if (blocks || enc_after) 879 aes_encrypt(ctx, dg, dg); 880 } 881 } 882 } 883 884 static int mac_update(struct shash_desc *desc, const u8 *p, unsigned int len) 885 { 886 struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); 887 struct mac_desc_ctx *ctx = shash_desc_ctx(desc); 888 889 while (len > 0) { 890 unsigned int l; 891 892 if ((ctx->len % AES_BLOCK_SIZE) == 0 && 893 (ctx->len + len) > AES_BLOCK_SIZE) { 894 895 int blocks = len / AES_BLOCK_SIZE; 896 897 len %= AES_BLOCK_SIZE; 898 899 mac_do_update(&tctx->key, p, blocks, ctx->dg, 900 (ctx->len != 0), (len != 0)); 901 902 p += blocks * AES_BLOCK_SIZE; 903 904 if (!len) { 905 ctx->len = AES_BLOCK_SIZE; 906 break; 907 } 908 ctx->len = 0; 909 } 910 911 l = min(len, AES_BLOCK_SIZE - ctx->len); 912 913 if (l <= AES_BLOCK_SIZE) { 914 crypto_xor(ctx->dg + ctx->len, p, l); 915 ctx->len += l; 916 len -= l; 917 p += l; 918 } 919 } 920 921 return 0; 922 } 923 924 static int cbcmac_final(struct shash_desc *desc, u8 *out) 925 { 926 struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); 927 struct mac_desc_ctx *ctx = shash_desc_ctx(desc); 928 929 mac_do_update(&tctx->key, NULL, 0, ctx->dg, (ctx->len != 0), 0); 930 931 memcpy(out, ctx->dg, AES_BLOCK_SIZE); 932 933 return 0; 934 } 935 936 static int cmac_final(struct shash_desc *desc, u8 *out) 937 { 938 struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); 939 struct mac_desc_ctx *ctx = shash_desc_ctx(desc); 940 u8 *consts = tctx->consts; 941 942 if (ctx->len != AES_BLOCK_SIZE) { 943 ctx->dg[ctx->len] ^= 0x80; 944 consts += AES_BLOCK_SIZE; 945 } 946 947 mac_do_update(&tctx->key, consts, 1, ctx->dg, 0, 1); 948 949 memcpy(out, ctx->dg, AES_BLOCK_SIZE); 950 951 return 0; 952 } 953 954 static struct shash_alg mac_algs[] = { { 955 .base.cra_name = "cmac(aes)", 956 .base.cra_driver_name = "cmac-aes-" MODE, 957 .base.cra_priority = PRIO, 958 .base.cra_blocksize = AES_BLOCK_SIZE, 959 .base.cra_ctxsize = sizeof(struct mac_tfm_ctx) + 960 2 * AES_BLOCK_SIZE, 961 .base.cra_module = THIS_MODULE, 962 963 .digestsize = AES_BLOCK_SIZE, 964 .init = mac_init, 965 .update = mac_update, 966 .final = cmac_final, 967 .setkey = cmac_setkey, 968 .descsize = sizeof(struct mac_desc_ctx), 969 }, { 970 .base.cra_name = "xcbc(aes)", 971 .base.cra_driver_name = "xcbc-aes-" MODE, 972 .base.cra_priority = PRIO, 973 .base.cra_blocksize = AES_BLOCK_SIZE, 974 .base.cra_ctxsize = sizeof(struct mac_tfm_ctx) + 975 2 * AES_BLOCK_SIZE, 976 .base.cra_module = THIS_MODULE, 977 978 .digestsize = AES_BLOCK_SIZE, 979 .init = mac_init, 980 .update = mac_update, 981 .final = cmac_final, 982 .setkey = xcbc_setkey, 983 .descsize = sizeof(struct mac_desc_ctx), 984 }, { 985 .base.cra_name = "cbcmac(aes)", 986 .base.cra_driver_name = "cbcmac-aes-" MODE, 987 .base.cra_priority = PRIO, 988 .base.cra_blocksize = 1, 989 .base.cra_ctxsize = sizeof(struct mac_tfm_ctx), 990 .base.cra_module = THIS_MODULE, 991 992 .digestsize = AES_BLOCK_SIZE, 993 .init = mac_init, 994 .update = mac_update, 995 .final = cbcmac_final, 996 .setkey = cbcmac_setkey, 997 .descsize = sizeof(struct mac_desc_ctx), 998 } }; 999 1000 static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)]; 1001 1002 static void aes_exit(void) 1003 { 1004 int i; 1005 1006 for (i = 0; i < ARRAY_SIZE(aes_simd_algs); i++) 1007 if (aes_simd_algs[i]) 1008 simd_skcipher_free(aes_simd_algs[i]); 1009 1010 crypto_unregister_shashes(mac_algs, ARRAY_SIZE(mac_algs)); 1011 crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); 1012 } 1013 1014 static int __init aes_init(void) 1015 { 1016 struct simd_skcipher_alg *simd; 1017 const char *basename; 1018 const char *algname; 1019 const char *drvname; 1020 int err; 1021 int i; 1022 1023 err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); 1024 if (err) 1025 return err; 1026 1027 err = crypto_register_shashes(mac_algs, ARRAY_SIZE(mac_algs)); 1028 if (err) 1029 goto unregister_ciphers; 1030 1031 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { 1032 if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL)) 1033 continue; 1034 1035 algname = aes_algs[i].base.cra_name + 2; 1036 drvname = aes_algs[i].base.cra_driver_name + 2; 1037 basename = aes_algs[i].base.cra_driver_name; 1038 simd = simd_skcipher_create_compat(algname, drvname, basename); 1039 err = PTR_ERR(simd); 1040 if (IS_ERR(simd)) 1041 goto unregister_simds; 1042 1043 aes_simd_algs[i] = simd; 1044 } 1045 1046 return 0; 1047 1048 unregister_simds: 1049 aes_exit(); 1050 return err; 1051 unregister_ciphers: 1052 crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); 1053 return err; 1054 } 1055 1056 #ifdef USE_V8_CRYPTO_EXTENSIONS 1057 module_cpu_feature_match(AES, aes_init); 1058 #else 1059 module_init(aes_init); 1060 EXPORT_SYMBOL(neon_aes_ecb_encrypt); 1061 EXPORT_SYMBOL(neon_aes_cbc_encrypt); 1062 EXPORT_SYMBOL(neon_aes_xts_encrypt); 1063 EXPORT_SYMBOL(neon_aes_xts_decrypt); 1064 #endif 1065 module_exit(aes_exit); 1066