1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/arch/arm64/crypto/aes-glue.c - wrapper code for ARMv8 AES 4 * 5 * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> 6 */ 7 8 #include <asm/hwcap.h> 9 #include <asm/neon.h> 10 #include <crypto/aes.h> 11 #include <crypto/ctr.h> 12 #include <crypto/internal/hash.h> 13 #include <crypto/internal/skcipher.h> 14 #include <crypto/scatterwalk.h> 15 #include <crypto/sha2.h> 16 #include <crypto/utils.h> 17 #include <crypto/xts.h> 18 #include <linux/cpufeature.h> 19 #include <linux/kernel.h> 20 #include <linux/module.h> 21 #include <linux/string.h> 22 23 #include "aes-ce-setkey.h" 24 25 #ifdef USE_V8_CRYPTO_EXTENSIONS 26 #define MODE "ce" 27 #define PRIO 300 28 #define aes_expandkey ce_aes_expandkey 29 #define aes_ecb_encrypt ce_aes_ecb_encrypt 30 #define aes_ecb_decrypt ce_aes_ecb_decrypt 31 #define aes_cbc_encrypt ce_aes_cbc_encrypt 32 #define aes_cbc_decrypt ce_aes_cbc_decrypt 33 #define aes_cbc_cts_encrypt ce_aes_cbc_cts_encrypt 34 #define aes_cbc_cts_decrypt ce_aes_cbc_cts_decrypt 35 #define aes_essiv_cbc_encrypt ce_aes_essiv_cbc_encrypt 36 #define aes_essiv_cbc_decrypt ce_aes_essiv_cbc_decrypt 37 #define aes_ctr_encrypt ce_aes_ctr_encrypt 38 #define aes_xctr_encrypt ce_aes_xctr_encrypt 39 #define aes_xts_encrypt ce_aes_xts_encrypt 40 #define aes_xts_decrypt ce_aes_xts_decrypt 41 #define aes_mac_update ce_aes_mac_update 42 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS/XCTR using ARMv8 Crypto Extensions"); 43 #else 44 #define MODE "neon" 45 #define PRIO 200 46 #define aes_ecb_encrypt neon_aes_ecb_encrypt 47 #define aes_ecb_decrypt neon_aes_ecb_decrypt 48 #define aes_cbc_encrypt neon_aes_cbc_encrypt 49 #define aes_cbc_decrypt neon_aes_cbc_decrypt 50 #define aes_cbc_cts_encrypt neon_aes_cbc_cts_encrypt 51 #define aes_cbc_cts_decrypt neon_aes_cbc_cts_decrypt 52 #define aes_essiv_cbc_encrypt neon_aes_essiv_cbc_encrypt 53 #define aes_essiv_cbc_decrypt neon_aes_essiv_cbc_decrypt 54 #define aes_ctr_encrypt neon_aes_ctr_encrypt 55 #define aes_xctr_encrypt neon_aes_xctr_encrypt 56 #define aes_xts_encrypt neon_aes_xts_encrypt 57 #define aes_xts_decrypt neon_aes_xts_decrypt 58 #define aes_mac_update neon_aes_mac_update 59 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS/XCTR using ARMv8 NEON"); 60 #endif 61 #if defined(USE_V8_CRYPTO_EXTENSIONS) || !IS_ENABLED(CONFIG_CRYPTO_AES_ARM64_BS) 62 MODULE_ALIAS_CRYPTO("ecb(aes)"); 63 MODULE_ALIAS_CRYPTO("cbc(aes)"); 64 MODULE_ALIAS_CRYPTO("ctr(aes)"); 65 MODULE_ALIAS_CRYPTO("xts(aes)"); 66 MODULE_ALIAS_CRYPTO("xctr(aes)"); 67 #endif 68 MODULE_ALIAS_CRYPTO("cts(cbc(aes))"); 69 MODULE_ALIAS_CRYPTO("essiv(cbc(aes),sha256)"); 70 MODULE_ALIAS_CRYPTO("cmac(aes)"); 71 MODULE_ALIAS_CRYPTO("xcbc(aes)"); 72 MODULE_ALIAS_CRYPTO("cbcmac(aes)"); 73 74 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 75 MODULE_LICENSE("GPL v2"); 76 77 /* defined in aes-modes.S */ 78 asmlinkage void aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[], 79 int rounds, int blocks); 80 asmlinkage void aes_ecb_decrypt(u8 out[], u8 const in[], u32 const rk[], 81 int rounds, int blocks); 82 83 asmlinkage void aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[], 84 int rounds, int blocks, u8 iv[]); 85 asmlinkage void aes_cbc_decrypt(u8 out[], u8 const in[], u32 const rk[], 86 int rounds, int blocks, u8 iv[]); 87 88 asmlinkage void aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[], 89 int rounds, int bytes, u8 const iv[]); 90 asmlinkage void aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[], 91 int rounds, int bytes, u8 const iv[]); 92 93 asmlinkage void aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[], 94 int rounds, int bytes, u8 ctr[]); 95 96 asmlinkage void aes_xctr_encrypt(u8 out[], u8 const in[], u32 const rk[], 97 int rounds, int bytes, u8 ctr[], int byte_ctr); 98 99 asmlinkage void aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[], 100 int rounds, int bytes, u32 const rk2[], u8 iv[], 101 int first); 102 asmlinkage void aes_xts_decrypt(u8 out[], u8 const in[], u32 const rk1[], 103 int rounds, int bytes, u32 const rk2[], u8 iv[], 104 int first); 105 106 asmlinkage void aes_essiv_cbc_encrypt(u8 out[], u8 const in[], u32 const rk1[], 107 int rounds, int blocks, u8 iv[], 108 u32 const rk2[]); 109 asmlinkage void aes_essiv_cbc_decrypt(u8 out[], u8 const in[], u32 const rk1[], 110 int rounds, int blocks, u8 iv[], 111 u32 const rk2[]); 112 113 asmlinkage int aes_mac_update(u8 const in[], u32 const rk[], int rounds, 114 int blocks, u8 dg[], int enc_before, 115 int enc_after); 116 117 struct crypto_aes_xts_ctx { 118 struct crypto_aes_ctx key1; 119 struct crypto_aes_ctx __aligned(8) key2; 120 }; 121 122 struct crypto_aes_essiv_cbc_ctx { 123 struct crypto_aes_ctx key1; 124 struct crypto_aes_ctx __aligned(8) key2; 125 }; 126 127 struct mac_tfm_ctx { 128 struct crypto_aes_ctx key; 129 u8 __aligned(8) consts[]; 130 }; 131 132 struct mac_desc_ctx { 133 u8 dg[AES_BLOCK_SIZE]; 134 }; 135 136 static int skcipher_aes_setkey(struct crypto_skcipher *tfm, const u8 *in_key, 137 unsigned int key_len) 138 { 139 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 140 141 return aes_expandkey(ctx, in_key, key_len); 142 } 143 144 static int __maybe_unused xts_set_key(struct crypto_skcipher *tfm, 145 const u8 *in_key, unsigned int key_len) 146 { 147 struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 148 int ret; 149 150 ret = xts_verify_key(tfm, in_key, key_len); 151 if (ret) 152 return ret; 153 154 ret = aes_expandkey(&ctx->key1, in_key, key_len / 2); 155 if (!ret) 156 ret = aes_expandkey(&ctx->key2, &in_key[key_len / 2], 157 key_len / 2); 158 return ret; 159 } 160 161 static int __maybe_unused essiv_cbc_set_key(struct crypto_skcipher *tfm, 162 const u8 *in_key, 163 unsigned int key_len) 164 { 165 struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); 166 u8 digest[SHA256_DIGEST_SIZE]; 167 int ret; 168 169 ret = aes_expandkey(&ctx->key1, in_key, key_len); 170 if (ret) 171 return ret; 172 173 sha256(in_key, key_len, digest); 174 175 return aes_expandkey(&ctx->key2, digest, sizeof(digest)); 176 } 177 178 static int __maybe_unused ecb_encrypt(struct skcipher_request *req) 179 { 180 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 181 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 182 int err, rounds = 6 + ctx->key_length / 4; 183 struct skcipher_walk walk; 184 unsigned int blocks; 185 186 err = skcipher_walk_virt(&walk, req, false); 187 188 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { 189 kernel_neon_begin(); 190 aes_ecb_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 191 ctx->key_enc, rounds, blocks); 192 kernel_neon_end(); 193 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 194 } 195 return err; 196 } 197 198 static int __maybe_unused ecb_decrypt(struct skcipher_request *req) 199 { 200 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 201 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 202 int err, rounds = 6 + ctx->key_length / 4; 203 struct skcipher_walk walk; 204 unsigned int blocks; 205 206 err = skcipher_walk_virt(&walk, req, false); 207 208 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { 209 kernel_neon_begin(); 210 aes_ecb_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 211 ctx->key_dec, rounds, blocks); 212 kernel_neon_end(); 213 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 214 } 215 return err; 216 } 217 218 static int cbc_encrypt_walk(struct skcipher_request *req, 219 struct skcipher_walk *walk) 220 { 221 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 222 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 223 int err = 0, rounds = 6 + ctx->key_length / 4; 224 unsigned int blocks; 225 226 while ((blocks = (walk->nbytes / AES_BLOCK_SIZE))) { 227 kernel_neon_begin(); 228 aes_cbc_encrypt(walk->dst.virt.addr, walk->src.virt.addr, 229 ctx->key_enc, rounds, blocks, walk->iv); 230 kernel_neon_end(); 231 err = skcipher_walk_done(walk, walk->nbytes % AES_BLOCK_SIZE); 232 } 233 return err; 234 } 235 236 static int __maybe_unused cbc_encrypt(struct skcipher_request *req) 237 { 238 struct skcipher_walk walk; 239 int err; 240 241 err = skcipher_walk_virt(&walk, req, false); 242 if (err) 243 return err; 244 return cbc_encrypt_walk(req, &walk); 245 } 246 247 static int cbc_decrypt_walk(struct skcipher_request *req, 248 struct skcipher_walk *walk) 249 { 250 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 251 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 252 int err = 0, rounds = 6 + ctx->key_length / 4; 253 unsigned int blocks; 254 255 while ((blocks = (walk->nbytes / AES_BLOCK_SIZE))) { 256 kernel_neon_begin(); 257 aes_cbc_decrypt(walk->dst.virt.addr, walk->src.virt.addr, 258 ctx->key_dec, rounds, blocks, walk->iv); 259 kernel_neon_end(); 260 err = skcipher_walk_done(walk, walk->nbytes % AES_BLOCK_SIZE); 261 } 262 return err; 263 } 264 265 static int __maybe_unused cbc_decrypt(struct skcipher_request *req) 266 { 267 struct skcipher_walk walk; 268 int err; 269 270 err = skcipher_walk_virt(&walk, req, false); 271 if (err) 272 return err; 273 return cbc_decrypt_walk(req, &walk); 274 } 275 276 static int cts_cbc_encrypt(struct skcipher_request *req) 277 { 278 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 279 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 280 int err, rounds = 6 + ctx->key_length / 4; 281 int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2; 282 struct scatterlist *src = req->src, *dst = req->dst; 283 struct scatterlist sg_src[2], sg_dst[2]; 284 struct skcipher_request subreq; 285 struct skcipher_walk walk; 286 287 skcipher_request_set_tfm(&subreq, tfm); 288 skcipher_request_set_callback(&subreq, skcipher_request_flags(req), 289 NULL, NULL); 290 291 if (req->cryptlen <= AES_BLOCK_SIZE) { 292 if (req->cryptlen < AES_BLOCK_SIZE) 293 return -EINVAL; 294 cbc_blocks = 1; 295 } 296 297 if (cbc_blocks > 0) { 298 skcipher_request_set_crypt(&subreq, req->src, req->dst, 299 cbc_blocks * AES_BLOCK_SIZE, 300 req->iv); 301 302 err = skcipher_walk_virt(&walk, &subreq, false) ?: 303 cbc_encrypt_walk(&subreq, &walk); 304 if (err) 305 return err; 306 307 if (req->cryptlen == AES_BLOCK_SIZE) 308 return 0; 309 310 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen); 311 if (req->dst != req->src) 312 dst = scatterwalk_ffwd(sg_dst, req->dst, 313 subreq.cryptlen); 314 } 315 316 /* handle ciphertext stealing */ 317 skcipher_request_set_crypt(&subreq, src, dst, 318 req->cryptlen - cbc_blocks * AES_BLOCK_SIZE, 319 req->iv); 320 321 err = skcipher_walk_virt(&walk, &subreq, false); 322 if (err) 323 return err; 324 325 kernel_neon_begin(); 326 aes_cbc_cts_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 327 ctx->key_enc, rounds, walk.nbytes, walk.iv); 328 kernel_neon_end(); 329 330 return skcipher_walk_done(&walk, 0); 331 } 332 333 static int cts_cbc_decrypt(struct skcipher_request *req) 334 { 335 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 336 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 337 int err, rounds = 6 + ctx->key_length / 4; 338 int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2; 339 struct scatterlist *src = req->src, *dst = req->dst; 340 struct scatterlist sg_src[2], sg_dst[2]; 341 struct skcipher_request subreq; 342 struct skcipher_walk walk; 343 344 skcipher_request_set_tfm(&subreq, tfm); 345 skcipher_request_set_callback(&subreq, skcipher_request_flags(req), 346 NULL, NULL); 347 348 if (req->cryptlen <= AES_BLOCK_SIZE) { 349 if (req->cryptlen < AES_BLOCK_SIZE) 350 return -EINVAL; 351 cbc_blocks = 1; 352 } 353 354 if (cbc_blocks > 0) { 355 skcipher_request_set_crypt(&subreq, req->src, req->dst, 356 cbc_blocks * AES_BLOCK_SIZE, 357 req->iv); 358 359 err = skcipher_walk_virt(&walk, &subreq, false) ?: 360 cbc_decrypt_walk(&subreq, &walk); 361 if (err) 362 return err; 363 364 if (req->cryptlen == AES_BLOCK_SIZE) 365 return 0; 366 367 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen); 368 if (req->dst != req->src) 369 dst = scatterwalk_ffwd(sg_dst, req->dst, 370 subreq.cryptlen); 371 } 372 373 /* handle ciphertext stealing */ 374 skcipher_request_set_crypt(&subreq, src, dst, 375 req->cryptlen - cbc_blocks * AES_BLOCK_SIZE, 376 req->iv); 377 378 err = skcipher_walk_virt(&walk, &subreq, false); 379 if (err) 380 return err; 381 382 kernel_neon_begin(); 383 aes_cbc_cts_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 384 ctx->key_dec, rounds, walk.nbytes, walk.iv); 385 kernel_neon_end(); 386 387 return skcipher_walk_done(&walk, 0); 388 } 389 390 static int __maybe_unused essiv_cbc_encrypt(struct skcipher_request *req) 391 { 392 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 393 struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); 394 int err, rounds = 6 + ctx->key1.key_length / 4; 395 struct skcipher_walk walk; 396 unsigned int blocks; 397 398 err = skcipher_walk_virt(&walk, req, false); 399 400 blocks = walk.nbytes / AES_BLOCK_SIZE; 401 if (blocks) { 402 kernel_neon_begin(); 403 aes_essiv_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 404 ctx->key1.key_enc, rounds, blocks, 405 req->iv, ctx->key2.key_enc); 406 kernel_neon_end(); 407 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 408 } 409 return err ?: cbc_encrypt_walk(req, &walk); 410 } 411 412 static int __maybe_unused essiv_cbc_decrypt(struct skcipher_request *req) 413 { 414 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 415 struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); 416 int err, rounds = 6 + ctx->key1.key_length / 4; 417 struct skcipher_walk walk; 418 unsigned int blocks; 419 420 err = skcipher_walk_virt(&walk, req, false); 421 422 blocks = walk.nbytes / AES_BLOCK_SIZE; 423 if (blocks) { 424 kernel_neon_begin(); 425 aes_essiv_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 426 ctx->key1.key_dec, rounds, blocks, 427 req->iv, ctx->key2.key_enc); 428 kernel_neon_end(); 429 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 430 } 431 return err ?: cbc_decrypt_walk(req, &walk); 432 } 433 434 static int __maybe_unused xctr_encrypt(struct skcipher_request *req) 435 { 436 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 437 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 438 int err, rounds = 6 + ctx->key_length / 4; 439 struct skcipher_walk walk; 440 unsigned int byte_ctr = 0; 441 442 err = skcipher_walk_virt(&walk, req, false); 443 444 while (walk.nbytes > 0) { 445 const u8 *src = walk.src.virt.addr; 446 unsigned int nbytes = walk.nbytes; 447 u8 *dst = walk.dst.virt.addr; 448 u8 buf[AES_BLOCK_SIZE]; 449 450 /* 451 * If given less than 16 bytes, we must copy the partial block 452 * into a temporary buffer of 16 bytes to avoid out of bounds 453 * reads and writes. Furthermore, this code is somewhat unusual 454 * in that it expects the end of the data to be at the end of 455 * the temporary buffer, rather than the start of the data at 456 * the start of the temporary buffer. 457 */ 458 if (unlikely(nbytes < AES_BLOCK_SIZE)) 459 src = dst = memcpy(buf + sizeof(buf) - nbytes, 460 src, nbytes); 461 else if (nbytes < walk.total) 462 nbytes &= ~(AES_BLOCK_SIZE - 1); 463 464 kernel_neon_begin(); 465 aes_xctr_encrypt(dst, src, ctx->key_enc, rounds, nbytes, 466 walk.iv, byte_ctr); 467 kernel_neon_end(); 468 469 if (unlikely(nbytes < AES_BLOCK_SIZE)) 470 memcpy(walk.dst.virt.addr, 471 buf + sizeof(buf) - nbytes, nbytes); 472 byte_ctr += nbytes; 473 474 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 475 } 476 477 return err; 478 } 479 480 static int __maybe_unused ctr_encrypt(struct skcipher_request *req) 481 { 482 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 483 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 484 int err, rounds = 6 + ctx->key_length / 4; 485 struct skcipher_walk walk; 486 487 err = skcipher_walk_virt(&walk, req, false); 488 489 while (walk.nbytes > 0) { 490 const u8 *src = walk.src.virt.addr; 491 unsigned int nbytes = walk.nbytes; 492 u8 *dst = walk.dst.virt.addr; 493 u8 buf[AES_BLOCK_SIZE]; 494 495 /* 496 * If given less than 16 bytes, we must copy the partial block 497 * into a temporary buffer of 16 bytes to avoid out of bounds 498 * reads and writes. Furthermore, this code is somewhat unusual 499 * in that it expects the end of the data to be at the end of 500 * the temporary buffer, rather than the start of the data at 501 * the start of the temporary buffer. 502 */ 503 if (unlikely(nbytes < AES_BLOCK_SIZE)) 504 src = dst = memcpy(buf + sizeof(buf) - nbytes, 505 src, nbytes); 506 else if (nbytes < walk.total) 507 nbytes &= ~(AES_BLOCK_SIZE - 1); 508 509 kernel_neon_begin(); 510 aes_ctr_encrypt(dst, src, ctx->key_enc, rounds, nbytes, 511 walk.iv); 512 kernel_neon_end(); 513 514 if (unlikely(nbytes < AES_BLOCK_SIZE)) 515 memcpy(walk.dst.virt.addr, 516 buf + sizeof(buf) - nbytes, nbytes); 517 518 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 519 } 520 521 return err; 522 } 523 524 static int __maybe_unused xts_encrypt(struct skcipher_request *req) 525 { 526 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 527 struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 528 int err, first, rounds = 6 + ctx->key1.key_length / 4; 529 int tail = req->cryptlen % AES_BLOCK_SIZE; 530 struct scatterlist sg_src[2], sg_dst[2]; 531 struct skcipher_request subreq; 532 struct scatterlist *src, *dst; 533 struct skcipher_walk walk; 534 535 if (req->cryptlen < AES_BLOCK_SIZE) 536 return -EINVAL; 537 538 err = skcipher_walk_virt(&walk, req, false); 539 540 if (unlikely(tail > 0 && walk.nbytes < walk.total)) { 541 int xts_blocks = DIV_ROUND_UP(req->cryptlen, 542 AES_BLOCK_SIZE) - 2; 543 544 skcipher_walk_abort(&walk); 545 546 skcipher_request_set_tfm(&subreq, tfm); 547 skcipher_request_set_callback(&subreq, 548 skcipher_request_flags(req), 549 NULL, NULL); 550 skcipher_request_set_crypt(&subreq, req->src, req->dst, 551 xts_blocks * AES_BLOCK_SIZE, 552 req->iv); 553 req = &subreq; 554 err = skcipher_walk_virt(&walk, req, false); 555 } else { 556 tail = 0; 557 } 558 559 for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) { 560 int nbytes = walk.nbytes; 561 562 if (walk.nbytes < walk.total) 563 nbytes &= ~(AES_BLOCK_SIZE - 1); 564 565 kernel_neon_begin(); 566 aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 567 ctx->key1.key_enc, rounds, nbytes, 568 ctx->key2.key_enc, walk.iv, first); 569 kernel_neon_end(); 570 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 571 } 572 573 if (err || likely(!tail)) 574 return err; 575 576 dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen); 577 if (req->dst != req->src) 578 dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen); 579 580 skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail, 581 req->iv); 582 583 err = skcipher_walk_virt(&walk, &subreq, false); 584 if (err) 585 return err; 586 587 kernel_neon_begin(); 588 aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 589 ctx->key1.key_enc, rounds, walk.nbytes, 590 ctx->key2.key_enc, walk.iv, first); 591 kernel_neon_end(); 592 593 return skcipher_walk_done(&walk, 0); 594 } 595 596 static int __maybe_unused xts_decrypt(struct skcipher_request *req) 597 { 598 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 599 struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 600 int err, first, rounds = 6 + ctx->key1.key_length / 4; 601 int tail = req->cryptlen % AES_BLOCK_SIZE; 602 struct scatterlist sg_src[2], sg_dst[2]; 603 struct skcipher_request subreq; 604 struct scatterlist *src, *dst; 605 struct skcipher_walk walk; 606 607 if (req->cryptlen < AES_BLOCK_SIZE) 608 return -EINVAL; 609 610 err = skcipher_walk_virt(&walk, req, false); 611 612 if (unlikely(tail > 0 && walk.nbytes < walk.total)) { 613 int xts_blocks = DIV_ROUND_UP(req->cryptlen, 614 AES_BLOCK_SIZE) - 2; 615 616 skcipher_walk_abort(&walk); 617 618 skcipher_request_set_tfm(&subreq, tfm); 619 skcipher_request_set_callback(&subreq, 620 skcipher_request_flags(req), 621 NULL, NULL); 622 skcipher_request_set_crypt(&subreq, req->src, req->dst, 623 xts_blocks * AES_BLOCK_SIZE, 624 req->iv); 625 req = &subreq; 626 err = skcipher_walk_virt(&walk, req, false); 627 } else { 628 tail = 0; 629 } 630 631 for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) { 632 int nbytes = walk.nbytes; 633 634 if (walk.nbytes < walk.total) 635 nbytes &= ~(AES_BLOCK_SIZE - 1); 636 637 kernel_neon_begin(); 638 aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 639 ctx->key1.key_dec, rounds, nbytes, 640 ctx->key2.key_enc, walk.iv, first); 641 kernel_neon_end(); 642 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 643 } 644 645 if (err || likely(!tail)) 646 return err; 647 648 dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen); 649 if (req->dst != req->src) 650 dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen); 651 652 skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail, 653 req->iv); 654 655 err = skcipher_walk_virt(&walk, &subreq, false); 656 if (err) 657 return err; 658 659 660 kernel_neon_begin(); 661 aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 662 ctx->key1.key_dec, rounds, walk.nbytes, 663 ctx->key2.key_enc, walk.iv, first); 664 kernel_neon_end(); 665 666 return skcipher_walk_done(&walk, 0); 667 } 668 669 static struct skcipher_alg aes_algs[] = { { 670 #if defined(USE_V8_CRYPTO_EXTENSIONS) || !IS_ENABLED(CONFIG_CRYPTO_AES_ARM64_BS) 671 .base = { 672 .cra_name = "ecb(aes)", 673 .cra_driver_name = "ecb-aes-" MODE, 674 .cra_priority = PRIO, 675 .cra_blocksize = AES_BLOCK_SIZE, 676 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 677 .cra_module = THIS_MODULE, 678 }, 679 .min_keysize = AES_MIN_KEY_SIZE, 680 .max_keysize = AES_MAX_KEY_SIZE, 681 .setkey = skcipher_aes_setkey, 682 .encrypt = ecb_encrypt, 683 .decrypt = ecb_decrypt, 684 }, { 685 .base = { 686 .cra_name = "cbc(aes)", 687 .cra_driver_name = "cbc-aes-" MODE, 688 .cra_priority = PRIO, 689 .cra_blocksize = AES_BLOCK_SIZE, 690 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 691 .cra_module = THIS_MODULE, 692 }, 693 .min_keysize = AES_MIN_KEY_SIZE, 694 .max_keysize = AES_MAX_KEY_SIZE, 695 .ivsize = AES_BLOCK_SIZE, 696 .setkey = skcipher_aes_setkey, 697 .encrypt = cbc_encrypt, 698 .decrypt = cbc_decrypt, 699 }, { 700 .base = { 701 .cra_name = "ctr(aes)", 702 .cra_driver_name = "ctr-aes-" MODE, 703 .cra_priority = PRIO, 704 .cra_blocksize = 1, 705 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 706 .cra_module = THIS_MODULE, 707 }, 708 .min_keysize = AES_MIN_KEY_SIZE, 709 .max_keysize = AES_MAX_KEY_SIZE, 710 .ivsize = AES_BLOCK_SIZE, 711 .chunksize = AES_BLOCK_SIZE, 712 .setkey = skcipher_aes_setkey, 713 .encrypt = ctr_encrypt, 714 .decrypt = ctr_encrypt, 715 }, { 716 .base = { 717 .cra_name = "xctr(aes)", 718 .cra_driver_name = "xctr-aes-" MODE, 719 .cra_priority = PRIO, 720 .cra_blocksize = 1, 721 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 722 .cra_module = THIS_MODULE, 723 }, 724 .min_keysize = AES_MIN_KEY_SIZE, 725 .max_keysize = AES_MAX_KEY_SIZE, 726 .ivsize = AES_BLOCK_SIZE, 727 .chunksize = AES_BLOCK_SIZE, 728 .setkey = skcipher_aes_setkey, 729 .encrypt = xctr_encrypt, 730 .decrypt = xctr_encrypt, 731 }, { 732 .base = { 733 .cra_name = "xts(aes)", 734 .cra_driver_name = "xts-aes-" MODE, 735 .cra_priority = PRIO, 736 .cra_blocksize = AES_BLOCK_SIZE, 737 .cra_ctxsize = sizeof(struct crypto_aes_xts_ctx), 738 .cra_module = THIS_MODULE, 739 }, 740 .min_keysize = 2 * AES_MIN_KEY_SIZE, 741 .max_keysize = 2 * AES_MAX_KEY_SIZE, 742 .ivsize = AES_BLOCK_SIZE, 743 .walksize = 2 * AES_BLOCK_SIZE, 744 .setkey = xts_set_key, 745 .encrypt = xts_encrypt, 746 .decrypt = xts_decrypt, 747 }, { 748 #endif 749 .base = { 750 .cra_name = "cts(cbc(aes))", 751 .cra_driver_name = "cts-cbc-aes-" MODE, 752 .cra_priority = PRIO, 753 .cra_blocksize = AES_BLOCK_SIZE, 754 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 755 .cra_module = THIS_MODULE, 756 }, 757 .min_keysize = AES_MIN_KEY_SIZE, 758 .max_keysize = AES_MAX_KEY_SIZE, 759 .ivsize = AES_BLOCK_SIZE, 760 .walksize = 2 * AES_BLOCK_SIZE, 761 .setkey = skcipher_aes_setkey, 762 .encrypt = cts_cbc_encrypt, 763 .decrypt = cts_cbc_decrypt, 764 }, { 765 .base = { 766 .cra_name = "essiv(cbc(aes),sha256)", 767 .cra_driver_name = "essiv-cbc-aes-sha256-" MODE, 768 .cra_priority = PRIO + 1, 769 .cra_blocksize = AES_BLOCK_SIZE, 770 .cra_ctxsize = sizeof(struct crypto_aes_essiv_cbc_ctx), 771 .cra_module = THIS_MODULE, 772 }, 773 .min_keysize = AES_MIN_KEY_SIZE, 774 .max_keysize = AES_MAX_KEY_SIZE, 775 .ivsize = AES_BLOCK_SIZE, 776 .setkey = essiv_cbc_set_key, 777 .encrypt = essiv_cbc_encrypt, 778 .decrypt = essiv_cbc_decrypt, 779 } }; 780 781 static int cbcmac_setkey(struct crypto_shash *tfm, const u8 *in_key, 782 unsigned int key_len) 783 { 784 struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm); 785 786 return aes_expandkey(&ctx->key, in_key, key_len); 787 } 788 789 static void cmac_gf128_mul_by_x(be128 *y, const be128 *x) 790 { 791 u64 a = be64_to_cpu(x->a); 792 u64 b = be64_to_cpu(x->b); 793 794 y->a = cpu_to_be64((a << 1) | (b >> 63)); 795 y->b = cpu_to_be64((b << 1) ^ ((a >> 63) ? 0x87 : 0)); 796 } 797 798 static int cmac_setkey(struct crypto_shash *tfm, const u8 *in_key, 799 unsigned int key_len) 800 { 801 struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm); 802 be128 *consts = (be128 *)ctx->consts; 803 int rounds = 6 + key_len / 4; 804 int err; 805 806 err = cbcmac_setkey(tfm, in_key, key_len); 807 if (err) 808 return err; 809 810 /* encrypt the zero vector */ 811 kernel_neon_begin(); 812 aes_ecb_encrypt(ctx->consts, (u8[AES_BLOCK_SIZE]){}, ctx->key.key_enc, 813 rounds, 1); 814 kernel_neon_end(); 815 816 cmac_gf128_mul_by_x(consts, consts); 817 cmac_gf128_mul_by_x(consts + 1, consts); 818 819 return 0; 820 } 821 822 static int xcbc_setkey(struct crypto_shash *tfm, const u8 *in_key, 823 unsigned int key_len) 824 { 825 static u8 const ks[3][AES_BLOCK_SIZE] = { 826 { [0 ... AES_BLOCK_SIZE - 1] = 0x1 }, 827 { [0 ... AES_BLOCK_SIZE - 1] = 0x2 }, 828 { [0 ... AES_BLOCK_SIZE - 1] = 0x3 }, 829 }; 830 831 struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm); 832 int rounds = 6 + key_len / 4; 833 u8 key[AES_BLOCK_SIZE]; 834 int err; 835 836 err = cbcmac_setkey(tfm, in_key, key_len); 837 if (err) 838 return err; 839 840 kernel_neon_begin(); 841 aes_ecb_encrypt(key, ks[0], ctx->key.key_enc, rounds, 1); 842 aes_ecb_encrypt(ctx->consts, ks[1], ctx->key.key_enc, rounds, 2); 843 kernel_neon_end(); 844 845 return cbcmac_setkey(tfm, key, sizeof(key)); 846 } 847 848 static int mac_init(struct shash_desc *desc) 849 { 850 struct mac_desc_ctx *ctx = shash_desc_ctx(desc); 851 852 memset(ctx->dg, 0, AES_BLOCK_SIZE); 853 return 0; 854 } 855 856 static void mac_do_update(struct crypto_aes_ctx *ctx, u8 const in[], int blocks, 857 u8 dg[], int enc_before) 858 { 859 int rounds = 6 + ctx->key_length / 4; 860 int rem; 861 862 do { 863 kernel_neon_begin(); 864 rem = aes_mac_update(in, ctx->key_enc, rounds, blocks, 865 dg, enc_before, !enc_before); 866 kernel_neon_end(); 867 in += (blocks - rem) * AES_BLOCK_SIZE; 868 blocks = rem; 869 } while (blocks); 870 } 871 872 static int mac_update(struct shash_desc *desc, const u8 *p, unsigned int len) 873 { 874 struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); 875 struct mac_desc_ctx *ctx = shash_desc_ctx(desc); 876 int blocks = len / AES_BLOCK_SIZE; 877 878 len %= AES_BLOCK_SIZE; 879 mac_do_update(&tctx->key, p, blocks, ctx->dg, 0); 880 return len; 881 } 882 883 static int cbcmac_finup(struct shash_desc *desc, const u8 *src, 884 unsigned int len, u8 *out) 885 { 886 struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); 887 struct mac_desc_ctx *ctx = shash_desc_ctx(desc); 888 889 if (len) { 890 crypto_xor(ctx->dg, src, len); 891 mac_do_update(&tctx->key, NULL, 0, ctx->dg, 1); 892 } 893 memcpy(out, ctx->dg, AES_BLOCK_SIZE); 894 return 0; 895 } 896 897 static int cmac_finup(struct shash_desc *desc, const u8 *src, unsigned int len, 898 u8 *out) 899 { 900 struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); 901 struct mac_desc_ctx *ctx = shash_desc_ctx(desc); 902 u8 *consts = tctx->consts; 903 904 crypto_xor(ctx->dg, src, len); 905 if (len != AES_BLOCK_SIZE) { 906 ctx->dg[len] ^= 0x80; 907 consts += AES_BLOCK_SIZE; 908 } 909 mac_do_update(&tctx->key, consts, 1, ctx->dg, 0); 910 memcpy(out, ctx->dg, AES_BLOCK_SIZE); 911 return 0; 912 } 913 914 static struct shash_alg mac_algs[] = { { 915 .base.cra_name = "cmac(aes)", 916 .base.cra_driver_name = "cmac-aes-" MODE, 917 .base.cra_priority = PRIO, 918 .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | 919 CRYPTO_AHASH_ALG_FINAL_NONZERO, 920 .base.cra_blocksize = AES_BLOCK_SIZE, 921 .base.cra_ctxsize = sizeof(struct mac_tfm_ctx) + 922 2 * AES_BLOCK_SIZE, 923 .base.cra_module = THIS_MODULE, 924 925 .digestsize = AES_BLOCK_SIZE, 926 .init = mac_init, 927 .update = mac_update, 928 .finup = cmac_finup, 929 .setkey = cmac_setkey, 930 .descsize = sizeof(struct mac_desc_ctx), 931 }, { 932 .base.cra_name = "xcbc(aes)", 933 .base.cra_driver_name = "xcbc-aes-" MODE, 934 .base.cra_priority = PRIO, 935 .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | 936 CRYPTO_AHASH_ALG_FINAL_NONZERO, 937 .base.cra_blocksize = AES_BLOCK_SIZE, 938 .base.cra_ctxsize = sizeof(struct mac_tfm_ctx) + 939 2 * AES_BLOCK_SIZE, 940 .base.cra_module = THIS_MODULE, 941 942 .digestsize = AES_BLOCK_SIZE, 943 .init = mac_init, 944 .update = mac_update, 945 .finup = cmac_finup, 946 .setkey = xcbc_setkey, 947 .descsize = sizeof(struct mac_desc_ctx), 948 }, { 949 .base.cra_name = "cbcmac(aes)", 950 .base.cra_driver_name = "cbcmac-aes-" MODE, 951 .base.cra_priority = PRIO, 952 .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, 953 .base.cra_blocksize = AES_BLOCK_SIZE, 954 .base.cra_ctxsize = sizeof(struct mac_tfm_ctx), 955 .base.cra_module = THIS_MODULE, 956 957 .digestsize = AES_BLOCK_SIZE, 958 .init = mac_init, 959 .update = mac_update, 960 .finup = cbcmac_finup, 961 .setkey = cbcmac_setkey, 962 .descsize = sizeof(struct mac_desc_ctx), 963 } }; 964 965 static void aes_exit(void) 966 { 967 crypto_unregister_shashes(mac_algs, ARRAY_SIZE(mac_algs)); 968 crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); 969 } 970 971 static int __init aes_init(void) 972 { 973 int err; 974 975 err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); 976 if (err) 977 return err; 978 979 err = crypto_register_shashes(mac_algs, ARRAY_SIZE(mac_algs)); 980 if (err) 981 goto unregister_ciphers; 982 983 return 0; 984 985 unregister_ciphers: 986 crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); 987 return err; 988 } 989 990 #ifdef USE_V8_CRYPTO_EXTENSIONS 991 module_cpu_feature_match(AES, aes_init); 992 EXPORT_SYMBOL_NS(ce_aes_mac_update, "CRYPTO_INTERNAL"); 993 #else 994 module_init(aes_init); 995 EXPORT_SYMBOL(neon_aes_ecb_encrypt); 996 EXPORT_SYMBOL(neon_aes_cbc_encrypt); 997 EXPORT_SYMBOL(neon_aes_ctr_encrypt); 998 EXPORT_SYMBOL(neon_aes_xts_encrypt); 999 EXPORT_SYMBOL(neon_aes_xts_decrypt); 1000 #endif 1001 module_exit(aes_exit); 1002