1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Crypto API support for AES block cipher 4 * 5 * Copyright 2026 Google LLC 6 */ 7 8 #include <crypto/aes-cbc-macs.h> 9 #include <crypto/aes-cbc.h> 10 #include <crypto/aes-ccm.h> 11 #include <crypto/aes-ctr.h> 12 #include <crypto/aes-ecb.h> 13 #include <crypto/aes-gcm.h> 14 #include <crypto/aes-xts.h> 15 #include <crypto/aes.h> 16 #include <crypto/algapi.h> 17 #include <crypto/internal/aead.h> 18 #include <crypto/internal/hash.h> 19 #include <crypto/internal/skcipher.h> 20 #include <crypto/scatterwalk.h> 21 #include <linux/module.h> 22 23 static_assert(__alignof__(struct aes_key) <= CRYPTO_MINALIGN); 24 static_assert(__alignof__(struct aes_enckey) <= CRYPTO_MINALIGN); 25 26 static int crypto_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key, 27 unsigned int key_len) 28 { 29 struct aes_key *key = crypto_tfm_ctx(tfm); 30 31 return aes_preparekey(key, in_key, key_len); 32 } 33 34 static void crypto_aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) 35 { 36 const struct aes_key *key = crypto_tfm_ctx(tfm); 37 38 aes_encrypt(key, out, in); 39 } 40 41 static void crypto_aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) 42 { 43 const struct aes_key *key = crypto_tfm_ctx(tfm); 44 45 aes_decrypt(key, out, in); 46 } 47 48 static_assert(__alignof__(struct aes_cmac_key) <= CRYPTO_MINALIGN); 49 #define AES_CMAC_KEY(tfm) ((struct aes_cmac_key *)crypto_shash_ctx(tfm)) 50 #define AES_CMAC_CTX(desc) ((struct aes_cmac_ctx *)shash_desc_ctx(desc)) 51 52 static int __maybe_unused crypto_aes_cmac_setkey(struct crypto_shash *tfm, 53 const u8 *in_key, 54 unsigned int key_len) 55 { 56 return aes_cmac_preparekey(AES_CMAC_KEY(tfm), in_key, key_len); 57 } 58 59 static int __maybe_unused crypto_aes_xcbc_setkey(struct crypto_shash *tfm, 60 const u8 *in_key, 61 unsigned int key_len) 62 { 63 if (key_len != AES_KEYSIZE_128) 64 return -EINVAL; 65 aes_xcbcmac_preparekey(AES_CMAC_KEY(tfm), in_key); 66 return 0; 67 } 68 69 static int __maybe_unused crypto_aes_cmac_init(struct shash_desc *desc) 70 { 71 aes_cmac_init(AES_CMAC_CTX(desc), AES_CMAC_KEY(desc->tfm)); 72 return 0; 73 } 74 75 static int __maybe_unused crypto_aes_cmac_update(struct shash_desc *desc, 76 const u8 *data, 77 unsigned int len) 78 { 79 aes_cmac_update(AES_CMAC_CTX(desc), data, len); 80 return 0; 81 } 82 83 static int __maybe_unused crypto_aes_cmac_final(struct shash_desc *desc, 84 u8 *out) 85 { 86 aes_cmac_final(AES_CMAC_CTX(desc), out); 87 return 0; 88 } 89 90 static int __maybe_unused crypto_aes_cmac_digest(struct shash_desc *desc, 91 const u8 *data, 92 unsigned int len, u8 *out) 93 { 94 aes_cmac(AES_CMAC_KEY(desc->tfm), data, len, out); 95 return 0; 96 } 97 98 #define AES_CBCMAC_KEY(tfm) ((struct aes_enckey *)crypto_shash_ctx(tfm)) 99 #define AES_CBCMAC_CTX(desc) ((struct aes_cbcmac_ctx *)shash_desc_ctx(desc)) 100 101 static int __maybe_unused crypto_aes_cbcmac_setkey(struct crypto_shash *tfm, 102 const u8 *in_key, 103 unsigned int key_len) 104 { 105 return aes_prepareenckey(AES_CBCMAC_KEY(tfm), in_key, key_len); 106 } 107 108 static int __maybe_unused crypto_aes_cbcmac_init(struct shash_desc *desc) 109 { 110 aes_cbcmac_init(AES_CBCMAC_CTX(desc), AES_CBCMAC_KEY(desc->tfm)); 111 return 0; 112 } 113 114 static int __maybe_unused crypto_aes_cbcmac_update(struct shash_desc *desc, 115 const u8 *data, 116 unsigned int len) 117 { 118 aes_cbcmac_update(AES_CBCMAC_CTX(desc), data, len); 119 return 0; 120 } 121 122 static int __maybe_unused crypto_aes_cbcmac_final(struct shash_desc *desc, 123 u8 *out) 124 { 125 aes_cbcmac_final(AES_CBCMAC_CTX(desc), out); 126 return 0; 127 } 128 129 static int __maybe_unused crypto_aes_cbcmac_digest(struct shash_desc *desc, 130 const u8 *data, 131 unsigned int len, u8 *out) 132 { 133 aes_cbcmac_init(AES_CBCMAC_CTX(desc), AES_CBCMAC_KEY(desc->tfm)); 134 aes_cbcmac_update(AES_CBCMAC_CTX(desc), data, len); 135 aes_cbcmac_final(AES_CBCMAC_CTX(desc), out); 136 return 0; 137 } 138 139 static struct crypto_alg alg = { 140 .cra_name = "aes", 141 .cra_driver_name = "aes-lib", 142 .cra_priority = 100, 143 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 144 .cra_blocksize = AES_BLOCK_SIZE, 145 .cra_ctxsize = sizeof(struct aes_key), 146 .cra_module = THIS_MODULE, 147 .cra_u = { .cipher = { .cia_min_keysize = AES_MIN_KEY_SIZE, 148 .cia_max_keysize = AES_MAX_KEY_SIZE, 149 .cia_setkey = crypto_aes_setkey, 150 .cia_encrypt = crypto_aes_encrypt, 151 .cia_decrypt = crypto_aes_decrypt } } 152 }; 153 154 static struct shash_alg mac_algs[] = { 155 #if IS_ENABLED(CONFIG_CRYPTO_CMAC) 156 { 157 .base.cra_name = "cmac(aes)", 158 .base.cra_driver_name = "cmac-aes-lib", 159 .base.cra_priority = 300, 160 .base.cra_blocksize = AES_BLOCK_SIZE, 161 .base.cra_ctxsize = sizeof(struct aes_cmac_key), 162 .base.cra_module = THIS_MODULE, 163 .digestsize = AES_BLOCK_SIZE, 164 .setkey = crypto_aes_cmac_setkey, 165 .init = crypto_aes_cmac_init, 166 .update = crypto_aes_cmac_update, 167 .final = crypto_aes_cmac_final, 168 .digest = crypto_aes_cmac_digest, 169 .descsize = sizeof(struct aes_cmac_ctx), 170 }, 171 #endif 172 #if IS_ENABLED(CONFIG_CRYPTO_XCBC) 173 { 174 /* 175 * Note that the only difference between xcbc(aes) and cmac(aes) 176 * is the preparekey function. 177 */ 178 .base.cra_name = "xcbc(aes)", 179 .base.cra_driver_name = "xcbc-aes-lib", 180 .base.cra_priority = 300, 181 .base.cra_blocksize = AES_BLOCK_SIZE, 182 .base.cra_ctxsize = sizeof(struct aes_cmac_key), 183 .base.cra_module = THIS_MODULE, 184 .digestsize = AES_BLOCK_SIZE, 185 .setkey = crypto_aes_xcbc_setkey, 186 .init = crypto_aes_cmac_init, 187 .update = crypto_aes_cmac_update, 188 .final = crypto_aes_cmac_final, 189 .digest = crypto_aes_cmac_digest, 190 .descsize = sizeof(struct aes_cmac_ctx), 191 }, 192 #endif 193 #if IS_ENABLED(CONFIG_CRYPTO_CCM) 194 { 195 .base.cra_name = "cbcmac(aes)", 196 .base.cra_driver_name = "cbcmac-aes-lib", 197 .base.cra_priority = 300, 198 .base.cra_blocksize = AES_BLOCK_SIZE, 199 .base.cra_ctxsize = sizeof(struct aes_enckey), 200 .base.cra_module = THIS_MODULE, 201 .digestsize = AES_BLOCK_SIZE, 202 .setkey = crypto_aes_cbcmac_setkey, 203 .init = crypto_aes_cbcmac_init, 204 .update = crypto_aes_cbcmac_update, 205 .final = crypto_aes_cbcmac_final, 206 .digest = crypto_aes_cbcmac_digest, 207 .descsize = sizeof(struct aes_cbcmac_ctx), 208 }, 209 #endif 210 }; 211 212 static __maybe_unused int 213 crypto_aes_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *in_key, 214 unsigned int key_len) 215 { 216 struct aes_key *key = crypto_skcipher_ctx(tfm); 217 218 return aes_preparekey(key, in_key, key_len); 219 } 220 221 static __maybe_unused int 222 crypto_aes_skcipher_setenckey(struct crypto_skcipher *tfm, const u8 *in_key, 223 unsigned int key_len) 224 { 225 struct aes_enckey *key = crypto_skcipher_ctx(tfm); 226 227 return aes_prepareenckey(key, in_key, key_len); 228 } 229 230 /* 231 * Return true if the request uses only a single scatterlist element and high 232 * memory isn't enabled. This assumes that both scatterlists are non-NULL, i.e. 233 * the caller must have handled the cryptlen == 0 case already. 234 */ 235 static inline bool 236 skcipher_request_is_linear_lowmem(const struct skcipher_request *req) 237 { 238 return !IS_ENABLED(CONFIG_HIGHMEM) && 239 req->dst->length >= req->cryptlen && 240 req->src->length >= req->cryptlen; 241 } 242 243 /* 244 * Call crypt_func() (a function that operates on simple virtual addresses) zero 245 * or more times to en/decrypt 'cryptlen' bytes of data from the source 246 * scatterlist 'src' and write it into the destination scatterlist 'dst', 247 * starting at 'start_pos' bytes into both. 248 * 249 * This always calls crypt_func() with a length that's a multiple of 250 * AES_BLOCK_SIZE, except the last call which includes any remainder. This is 251 * implemented by using an on-stack bounce buffer when necessary. The current 252 * implementation also tries to prefer passing at least 4 blocks, so e.g. 253 * scatterlist entries [16,16,16,16] result in a single 64-byte call. 254 * 255 * The scatterlists must describe either entirely different memory 256 * (out-of-place) or entirely the same memory (in-place). In the latter case, 257 * crypt_func() is always called with the source and dest pointers the same. 258 */ 259 #define AES_CRYPT_SG(crypt_func, dst, src, cryptlen, start_pos, ...) \ 260 ({ \ 261 unsigned int remaining = (cryptlen); \ 262 unsigned int spos = (start_pos); \ 263 \ 264 if (remaining != 0) { \ 265 struct scatter_walk dst_walk, src_walk; \ 266 u8 tmp[4 * AES_BLOCK_SIZE] __aligned( \ 267 __alignof__(long)); \ 268 \ 269 scatterwalk_start_at_pos(&dst_walk, (dst), spos); \ 270 scatterwalk_start_at_pos(&src_walk, (src), spos); \ 271 do { \ 272 unsigned int dst_avail = scatterwalk_clamp( \ 273 &dst_walk, remaining); \ 274 unsigned int src_avail = scatterwalk_clamp( \ 275 &src_walk, remaining); \ 276 unsigned int n = min(dst_avail, src_avail); \ 277 u8 *dst_virt; \ 278 const u8 *src_virt; \ 279 \ 280 if (n < remaining) { \ 281 if (n < sizeof(tmp)) { \ 282 n = min(remaining, \ 283 sizeof(tmp)); \ 284 memcpy_from_scatterwalk( \ 285 tmp, &src_walk, n); \ 286 crypt_func(tmp, tmp, n, \ 287 ##__VA_ARGS__); \ 288 memcpy_to_scatterwalk( \ 289 &dst_walk, tmp, n); \ 290 remaining -= n; \ 291 continue; \ 292 } \ 293 n = round_down(n, AES_BLOCK_SIZE); \ 294 } \ 295 \ 296 scatterwalk_map(&dst_walk); \ 297 dst_virt = dst_walk.addr; \ 298 if (IS_ENABLED(CONFIG_HIGHMEM) && \ 299 offset_in_page(src_walk.offset) == \ 300 offset_in_page(dst_walk.offset) && \ 301 sg_page(src_walk.sg) + (src_walk.offset / \ 302 PAGE_SIZE) == \ 303 sg_page(dst_walk.sg) + \ 304 (dst_walk.offset / \ 305 PAGE_SIZE)) { \ 306 src_virt = dst_virt; \ 307 } else { \ 308 scatterwalk_map(&src_walk); \ 309 src_virt = src_walk.addr; \ 310 } \ 311 crypt_func(dst_virt, src_virt, n, \ 312 ##__VA_ARGS__); \ 313 if (src_virt != dst_virt) \ 314 scatterwalk_unmap(&src_walk); \ 315 scatterwalk_advance(&src_walk, n); \ 316 scatterwalk_done_dst(&dst_walk, n); \ 317 remaining -= n; \ 318 } while (remaining); \ 319 memzero_explicit(tmp, sizeof(tmp)); \ 320 } \ 321 }) 322 323 /* 324 * Call ad_func() as needed to process the associated data in the first 325 * 'assoclen' bytes of the scatterlist 'src'. 326 */ 327 #define AES_PROCESS_ASSOC_DATA(ad_func, src, assoclen, ctx) \ 328 ({ \ 329 unsigned int remaining = (assoclen); \ 330 \ 331 if (remaining != 0) { \ 332 struct scatter_walk walk; \ 333 \ 334 scatterwalk_start(&walk, (src)); \ 335 do { \ 336 unsigned int n = \ 337 scatterwalk_next(&walk, remaining); \ 338 \ 339 ad_func((ctx), walk.addr, n); \ 340 scatterwalk_done_src(&walk, n); \ 341 remaining -= n; \ 342 } while (remaining); \ 343 } \ 344 }) 345 346 /* AES-ECB */ 347 348 static __maybe_unused int crypto_aes_ecb_encrypt(struct skcipher_request *req) 349 { 350 const struct aes_key *key = 351 crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); 352 353 if (unlikely(req->cryptlen % AES_BLOCK_SIZE)) 354 return -EINVAL; 355 AES_CRYPT_SG(aes_ecb_encrypt, req->dst, req->src, req->cryptlen, 0, 356 key); 357 return 0; 358 } 359 360 static __maybe_unused int crypto_aes_ecb_decrypt(struct skcipher_request *req) 361 { 362 const struct aes_key *key = 363 crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); 364 365 if (unlikely(req->cryptlen % AES_BLOCK_SIZE)) 366 return -EINVAL; 367 AES_CRYPT_SG(aes_ecb_decrypt, req->dst, req->src, req->cryptlen, 0, 368 key); 369 return 0; 370 } 371 372 /* AES-CBC */ 373 374 static void crypto_aes_cbc_encrypt_sg(struct skcipher_request *req, 375 unsigned int cryptlen, 376 const struct aes_key *key) 377 { 378 AES_CRYPT_SG(aes_cbc_encrypt, req->dst, req->src, cryptlen, 0, req->iv, 379 key); 380 } 381 382 static void crypto_aes_cbc_decrypt_sg(struct skcipher_request *req, 383 unsigned int cryptlen, 384 const struct aes_key *key) 385 { 386 AES_CRYPT_SG(aes_cbc_decrypt, req->dst, req->src, cryptlen, 0, req->iv, 387 key); 388 } 389 390 static __maybe_unused int crypto_aes_cbc_encrypt(struct skcipher_request *req) 391 { 392 const struct aes_key *key = 393 crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); 394 395 if (unlikely(req->cryptlen % AES_BLOCK_SIZE)) 396 return -EINVAL; 397 crypto_aes_cbc_encrypt_sg(req, req->cryptlen, key); 398 return 0; 399 } 400 401 static __maybe_unused int crypto_aes_cbc_decrypt(struct skcipher_request *req) 402 { 403 const struct aes_key *key = 404 crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); 405 406 if (unlikely(req->cryptlen % AES_BLOCK_SIZE)) 407 return -EINVAL; 408 crypto_aes_cbc_decrypt_sg(req, req->cryptlen, key); 409 return 0; 410 } 411 412 /* AES-CBC-CTS */ 413 414 /* 415 * This handles AES-CBC-CTS en/decryption requests that use a nonlinear 416 * scatterlist layout or where HIGHMEM is enabled. It is explicitly 'noinline' 417 * to keep the temporary buffer out of the stack frame of the fast path. 418 */ 419 static noinline int 420 crypto_aes_cbc_cts_crypt_nonlinear(struct skcipher_request *req, bool enc) 421 { 422 const struct aes_key *key = 423 crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); 424 unsigned int main_len = req->cryptlen; 425 unsigned int tail_len; 426 u8 tmp[2 * AES_BLOCK_SIZE] __aligned(__alignof__(long)); 427 428 if (main_len == AES_BLOCK_SIZE) { 429 /* Single block is a special case that just does CBC. */ 430 if (enc) 431 crypto_aes_cbc_encrypt_sg(req, main_len, key); 432 else 433 crypto_aes_cbc_decrypt_sg(req, main_len, key); 434 return 0; 435 } 436 /* Just do the last two blocks separately. */ 437 tail_len = AES_BLOCK_SIZE + ((main_len - 1) % AES_BLOCK_SIZE) + 1; 438 main_len -= tail_len; 439 if (enc) 440 crypto_aes_cbc_encrypt_sg(req, main_len, key); 441 else 442 crypto_aes_cbc_decrypt_sg(req, main_len, key); 443 memcpy_from_sglist(tmp, req->src, main_len, tail_len); 444 if (enc) 445 aes_cbc_cts_encrypt(tmp, tmp, tail_len, req->iv, key); 446 else 447 aes_cbc_cts_decrypt(tmp, tmp, tail_len, req->iv, key); 448 memcpy_to_sglist(req->dst, main_len, tmp, tail_len); 449 memzero_explicit(tmp, sizeof(tmp)); 450 return 0; 451 } 452 453 static __maybe_unused int 454 crypto_aes_cbc_cts_encrypt(struct skcipher_request *req) 455 { 456 const struct aes_key *key = 457 crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); 458 459 if (unlikely(req->cryptlen < AES_BLOCK_SIZE)) 460 return -EINVAL; 461 if (likely(skcipher_request_is_linear_lowmem(req))) { 462 /* Fast path */ 463 aes_cbc_cts_encrypt(sg_virt(req->dst), sg_virt(req->src), 464 req->cryptlen, req->iv, key); 465 return 0; 466 } 467 return crypto_aes_cbc_cts_crypt_nonlinear(req, /* enc= */ true); 468 } 469 470 static __maybe_unused int 471 crypto_aes_cbc_cts_decrypt(struct skcipher_request *req) 472 { 473 const struct aes_key *key = 474 crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); 475 476 if (unlikely(req->cryptlen < AES_BLOCK_SIZE)) 477 return -EINVAL; 478 if (likely(skcipher_request_is_linear_lowmem(req))) { 479 /* Fast path */ 480 aes_cbc_cts_decrypt(sg_virt(req->dst), sg_virt(req->src), 481 req->cryptlen, req->iv, key); 482 return 0; 483 } 484 return crypto_aes_cbc_cts_crypt_nonlinear(req, /* enc= */ false); 485 } 486 487 /* AES-CTR */ 488 489 static __maybe_unused int crypto_aes_ctr_crypt(struct skcipher_request *req) 490 { 491 const struct aes_enckey *key = 492 crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); 493 494 AES_CRYPT_SG(aes_ctr, req->dst, req->src, req->cryptlen, 0, req->iv, 495 key); 496 return 0; 497 } 498 499 /* AES-XCTR */ 500 501 static __maybe_unused int crypto_aes_xctr_crypt(struct skcipher_request *req) 502 { 503 const struct aes_enckey *key = 504 crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); 505 u64 ctr = 1; 506 507 AES_CRYPT_SG(aes_xctr, req->dst, req->src, req->cryptlen, 0, &ctr, 508 req->iv, key); 509 return 0; 510 } 511 512 /* AES-XTS */ 513 514 static __maybe_unused int crypto_aes_xts_setkey(struct crypto_skcipher *tfm, 515 const u8 *in_key, 516 unsigned int key_len) 517 { 518 struct aes_xts_key *key = crypto_skcipher_ctx(tfm); 519 int flags = (crypto_skcipher_get_flags(tfm) & 520 CRYPTO_TFM_REQ_FORBID_WEAK_KEYS) ? 521 XTS_FORBID_WEAK_KEYS : 522 0; 523 524 return aes_xts_preparekey(key, in_key, key_len, flags); 525 } 526 527 static void aes_xts_crypt_wrapper(u8 *dst, const u8 *src, size_t len, 528 u8 iv[AES_BLOCK_SIZE], 529 const struct aes_xts_key *key, bool enc, 530 bool *cont) 531 { 532 if (enc) 533 aes_xts_encrypt(dst, src, len, iv, key, *cont); 534 else 535 aes_xts_decrypt(dst, src, len, iv, key, *cont); 536 *cont = true; 537 } 538 539 /* 540 * This handles AES-XTS en/decryption requests that use a nonlinear scatterlist 541 * layout or where HIGHMEM is enabled. It is explicitly 'noinline' to keep the 542 * temporary buffer out of the stack frame of the fast path. 543 */ 544 static noinline int crypto_aes_xts_crypt_nonlinear(struct skcipher_request *req, 545 bool enc) 546 { 547 const struct aes_xts_key *key = 548 crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); 549 u8 tmp[2 * AES_BLOCK_SIZE] __aligned(__alignof__(long)); 550 unsigned int main_len = req->cryptlen; 551 unsigned int tail_len = main_len % AES_BLOCK_SIZE; 552 bool cont = false; 553 554 if (unlikely(tail_len)) { 555 /* 556 * Ciphertext stealing is needed. 557 * Just do the last two blocks separately. 558 */ 559 tail_len += AES_BLOCK_SIZE; 560 main_len -= tail_len; 561 } 562 563 AES_CRYPT_SG(aes_xts_crypt_wrapper, req->dst, req->src, main_len, 0, 564 req->iv, key, enc, &cont); 565 566 if (unlikely(tail_len)) { 567 memcpy_from_sglist(tmp, req->src, main_len, tail_len); 568 aes_xts_crypt_wrapper(tmp, tmp, tail_len, req->iv, key, enc, 569 &cont); 570 memcpy_to_sglist(req->dst, main_len, tmp, tail_len); 571 memzero_explicit(tmp, sizeof(tmp)); 572 } 573 return 0; 574 } 575 576 static __maybe_unused int crypto_aes_xts_encrypt(struct skcipher_request *req) 577 { 578 const struct aes_xts_key *key = 579 crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); 580 581 if (unlikely(req->cryptlen < AES_BLOCK_SIZE)) 582 return -EINVAL; 583 if (likely(skcipher_request_is_linear_lowmem(req))) { 584 /* Fast path */ 585 aes_xts_encrypt(sg_virt(req->dst), sg_virt(req->src), 586 req->cryptlen, req->iv, key, /* cont= */ false); 587 return 0; 588 } 589 return crypto_aes_xts_crypt_nonlinear(req, /* enc= */ true); 590 } 591 592 static __maybe_unused int crypto_aes_xts_decrypt(struct skcipher_request *req) 593 { 594 const struct aes_xts_key *key = 595 crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); 596 597 if (unlikely(req->cryptlen < AES_BLOCK_SIZE)) 598 return -EINVAL; 599 if (likely(skcipher_request_is_linear_lowmem(req))) { 600 /* Fast path */ 601 aes_xts_decrypt(sg_virt(req->dst), sg_virt(req->src), 602 req->cryptlen, req->iv, key, /* cont= */ false); 603 return 0; 604 } 605 return crypto_aes_xts_crypt_nonlinear(req, /* enc= */ false); 606 } 607 608 static struct skcipher_alg skcipher_algs[] = { 609 #if IS_ENABLED(CONFIG_CRYPTO_ECB) 610 { 611 .base.cra_name = "ecb(aes)", 612 .base.cra_driver_name = "ecb-aes-lib", 613 .base.cra_priority = 110, 614 .base.cra_blocksize = AES_BLOCK_SIZE, 615 .base.cra_ctxsize = sizeof(struct aes_key), 616 .base.cra_module = THIS_MODULE, 617 .min_keysize = AES_MIN_KEY_SIZE, 618 .max_keysize = AES_MAX_KEY_SIZE, 619 .setkey = crypto_aes_skcipher_setkey, 620 .encrypt = crypto_aes_ecb_encrypt, 621 .decrypt = crypto_aes_ecb_decrypt, 622 }, 623 #endif 624 #if IS_ENABLED(CONFIG_CRYPTO_CBC) 625 { 626 .base.cra_name = "cbc(aes)", 627 .base.cra_driver_name = "cbc-aes-lib", 628 .base.cra_priority = 110, 629 .base.cra_blocksize = AES_BLOCK_SIZE, 630 .base.cra_ctxsize = sizeof(struct aes_key), 631 .base.cra_module = THIS_MODULE, 632 .min_keysize = AES_MIN_KEY_SIZE, 633 .max_keysize = AES_MAX_KEY_SIZE, 634 .ivsize = AES_BLOCK_SIZE, 635 .setkey = crypto_aes_skcipher_setkey, 636 .encrypt = crypto_aes_cbc_encrypt, 637 .decrypt = crypto_aes_cbc_decrypt, 638 }, 639 #endif 640 #if IS_ENABLED(CONFIG_CRYPTO_CTS) 641 { 642 .base.cra_name = "cts(cbc(aes))", 643 .base.cra_driver_name = "cts-cbc-aes-lib", 644 .base.cra_priority = 110, 645 .base.cra_blocksize = AES_BLOCK_SIZE, 646 .base.cra_ctxsize = sizeof(struct aes_key), 647 .base.cra_module = THIS_MODULE, 648 .min_keysize = AES_MIN_KEY_SIZE, 649 .max_keysize = AES_MAX_KEY_SIZE, 650 .ivsize = AES_BLOCK_SIZE, 651 .setkey = crypto_aes_skcipher_setkey, 652 .encrypt = crypto_aes_cbc_cts_encrypt, 653 .decrypt = crypto_aes_cbc_cts_decrypt, 654 }, 655 #endif 656 #if IS_ENABLED(CONFIG_CRYPTO_CTR) 657 { 658 .base.cra_name = "ctr(aes)", 659 .base.cra_driver_name = "ctr-aes-lib", 660 .base.cra_priority = 110, 661 .base.cra_blocksize = 1, 662 .base.cra_ctxsize = sizeof(struct aes_enckey), 663 .base.cra_module = THIS_MODULE, 664 .min_keysize = AES_MIN_KEY_SIZE, 665 .max_keysize = AES_MAX_KEY_SIZE, 666 .ivsize = AES_BLOCK_SIZE, 667 .chunksize = AES_BLOCK_SIZE, 668 .setkey = crypto_aes_skcipher_setenckey, 669 .encrypt = crypto_aes_ctr_crypt, 670 .decrypt = crypto_aes_ctr_crypt, 671 }, 672 #endif 673 #if IS_ENABLED(CONFIG_CRYPTO_XCTR) 674 { 675 .base.cra_name = "xctr(aes)", 676 .base.cra_driver_name = "xctr-aes-lib", 677 .base.cra_priority = 110, 678 .base.cra_blocksize = 1, 679 .base.cra_ctxsize = sizeof(struct aes_enckey), 680 .base.cra_module = THIS_MODULE, 681 .min_keysize = AES_MIN_KEY_SIZE, 682 .max_keysize = AES_MAX_KEY_SIZE, 683 .ivsize = AES_BLOCK_SIZE, 684 .chunksize = AES_BLOCK_SIZE, 685 .setkey = crypto_aes_skcipher_setenckey, 686 .encrypt = crypto_aes_xctr_crypt, 687 .decrypt = crypto_aes_xctr_crypt, 688 }, 689 #endif 690 #if IS_ENABLED(CONFIG_CRYPTO_XTS) 691 { 692 .base.cra_name = "xts(aes)", 693 .base.cra_driver_name = "xts-aes-lib", 694 .base.cra_priority = 110, 695 .base.cra_blocksize = AES_BLOCK_SIZE, 696 .base.cra_ctxsize = sizeof(struct aes_xts_key), 697 .base.cra_module = THIS_MODULE, 698 .min_keysize = 2 * AES_MIN_KEY_SIZE, 699 .max_keysize = 2 * AES_MAX_KEY_SIZE, 700 .ivsize = AES_BLOCK_SIZE, 701 .setkey = crypto_aes_xts_setkey, 702 .encrypt = crypto_aes_xts_encrypt, 703 .decrypt = crypto_aes_xts_decrypt, 704 }, 705 #endif 706 }; 707 708 /* AES-GCM */ 709 710 static __maybe_unused int crypto_aes_gcm_setkey(struct crypto_aead *tfm, 711 const u8 *in_key, 712 unsigned int key_len) 713 { 714 struct aes_gcm_key *key = crypto_aead_ctx(tfm); 715 716 return aes_gcm_preparekey(key, in_key, key_len, 717 crypto_aead_authsize(tfm)); 718 } 719 720 static __maybe_unused int crypto_aes_gcm_setauthsize(struct crypto_aead *tfm, 721 unsigned int authsize) 722 { 723 struct aes_gcm_key *key = crypto_aead_ctx(tfm); 724 725 if (crypto_gcm_check_authsize(authsize) != 0) 726 return -EINVAL; 727 /* Synchronize the tag length to the struct aes_gcm_key. */ 728 key->authtag_len = authsize; 729 return 0; 730 } 731 732 static void crypto_aes_gcm_auth_update(struct aes_gcm_ctx *ctx, 733 struct scatterlist *src, 734 unsigned int assoclen) 735 { 736 AES_PROCESS_ASSOC_DATA(aes_gcm_auth_update, src, assoclen, ctx); 737 } 738 739 static void aes_gcm_encrypt_update_helper(u8 *dst, const u8 *src, 740 unsigned int len, 741 struct aes_gcm_ctx *ctx) 742 { 743 aes_gcm_encrypt_update(ctx, dst, src, len); 744 } 745 746 static void aes_gcm_decrypt_update_helper(u8 *dst, const u8 *src, 747 unsigned int len, 748 struct aes_gcm_ctx *ctx) 749 { 750 aes_gcm_decrypt_update(ctx, dst, src, len); 751 } 752 753 static int crypto_aes_gcm_encrypt_common(struct aead_request *req, 754 const struct aes_gcm_key *key, 755 u8 iv[12], unsigned int assoclen) 756 { 757 struct aes_gcm_ctx ctx; 758 u8 authtag[16]; 759 760 aes_gcm_init(&ctx, iv, key); 761 crypto_aes_gcm_auth_update(&ctx, req->src, assoclen); 762 AES_CRYPT_SG(aes_gcm_encrypt_update_helper, req->dst, req->src, 763 req->cryptlen, req->assoclen, &ctx); 764 aes_gcm_encrypt_final(&ctx, authtag); 765 memcpy_to_sglist(req->dst, req->assoclen + req->cryptlen, authtag, 766 key->authtag_len); 767 memzero_explicit(authtag, sizeof(authtag)); 768 return 0; 769 } 770 771 static int crypto_aes_gcm_decrypt_common(struct aead_request *req, 772 const struct aes_gcm_key *key, 773 u8 iv[12], unsigned int assoclen) 774 { 775 struct aes_gcm_ctx ctx; 776 unsigned int data_len; 777 u8 authtag[16]; 778 int err; 779 780 aes_gcm_init(&ctx, iv, key); 781 crypto_aes_gcm_auth_update(&ctx, req->src, assoclen); 782 783 /* crypto_aead_decrypt() already checked cryptlen >= authtag_len. */ 784 data_len = req->cryptlen - key->authtag_len; 785 AES_CRYPT_SG(aes_gcm_decrypt_update_helper, req->dst, req->src, 786 data_len, req->assoclen, &ctx); 787 788 memcpy_from_sglist(authtag, req->src, req->assoclen + data_len, 789 key->authtag_len); 790 err = aes_gcm_decrypt_final(&ctx, authtag); 791 memzero_explicit(authtag, sizeof(authtag)); 792 return err; 793 } 794 795 static __maybe_unused int crypto_aes_gcm_encrypt(struct aead_request *req) 796 { 797 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 798 const struct aes_gcm_key *key = crypto_aead_ctx(tfm); 799 800 return crypto_aes_gcm_encrypt_common(req, key, req->iv, req->assoclen); 801 } 802 803 static __maybe_unused int crypto_aes_gcm_decrypt(struct aead_request *req) 804 { 805 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 806 const struct aes_gcm_key *key = crypto_aead_ctx(tfm); 807 808 return crypto_aes_gcm_decrypt_common(req, key, req->iv, req->assoclen); 809 } 810 811 struct aes_rfc4106_key { 812 struct aes_gcm_key gcm; 813 u8 nonce[4]; 814 }; 815 816 static __maybe_unused int crypto_aes_rfc4106_setkey(struct crypto_aead *tfm, 817 const u8 *in_key, 818 unsigned int key_len) 819 { 820 struct aes_rfc4106_key *key = crypto_aead_ctx(tfm); 821 822 if (key_len < 4) 823 return -EINVAL; 824 825 key_len -= 4; 826 memcpy(key->nonce, in_key + key_len, 4); 827 828 return aes_gcm_preparekey(&key->gcm, in_key, key_len, 829 crypto_aead_authsize(tfm)); 830 } 831 832 static __maybe_unused int 833 crypto_aes_rfc4106_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 834 { 835 struct aes_rfc4106_key *key = crypto_aead_ctx(tfm); 836 837 if (crypto_rfc4106_check_authsize(authsize) != 0) 838 return -EINVAL; 839 840 /* Synchronize the tag length to the struct aes_gcm_key. */ 841 key->gcm.authtag_len = authsize; 842 return 0; 843 } 844 845 static __maybe_unused int crypto_aes_rfc4106_encrypt(struct aead_request *req) 846 { 847 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 848 const struct aes_rfc4106_key *key = crypto_aead_ctx(tfm); 849 u8 iv[12]; 850 851 if (crypto_ipsec_check_assoclen(req->assoclen) != 0) 852 return -EINVAL; 853 memcpy(iv, key->nonce, 4); 854 memcpy(&iv[4], req->iv, 8); 855 856 return crypto_aes_gcm_encrypt_common(req, &key->gcm, iv, 857 req->assoclen - 8); 858 } 859 860 static __maybe_unused int crypto_aes_rfc4106_decrypt(struct aead_request *req) 861 { 862 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 863 const struct aes_rfc4106_key *key = crypto_aead_ctx(tfm); 864 u8 iv[12]; 865 866 if (crypto_ipsec_check_assoclen(req->assoclen) != 0) 867 return -EINVAL; 868 memcpy(iv, key->nonce, 4); 869 memcpy(&iv[4], req->iv, 8); 870 871 return crypto_aes_gcm_decrypt_common(req, &key->gcm, iv, 872 req->assoclen - 8); 873 } 874 875 /* AES-CCM */ 876 877 static __maybe_unused int crypto_aes_ccm_setkey(struct crypto_aead *tfm, 878 const u8 *in_key, 879 unsigned int key_len) 880 { 881 struct aes_ccm_key *key = crypto_aead_ctx(tfm); 882 883 return aes_ccm_preparekey(key, in_key, key_len, 884 crypto_aead_authsize(tfm)); 885 } 886 887 static __maybe_unused int crypto_aes_ccm_setauthsize(struct crypto_aead *tfm, 888 unsigned int authsize) 889 { 890 struct aes_ccm_key *key = crypto_aead_ctx(tfm); 891 892 if (authsize < 4 || authsize > 16 || authsize % 2) 893 return -EINVAL; 894 /* Synchronize the tag length to the struct aes_ccm_key. */ 895 key->authtag_len = authsize; 896 return 0; 897 } 898 899 static int crypto_aes_ccm_init(struct aes_ccm_ctx *ctx, 900 struct aead_request *req, unsigned int data_len, 901 const struct aes_ccm_key *key) 902 { 903 int nonce_len; 904 const u8 *nonce; 905 int err; 906 907 /* 908 * CCM accepts a variable-length nonce between 7 and 13 bytes 909 * inclusively, while crypto_aead assumes a fixed-length IV. This is 910 * worked around by requiring that iv[0] contain '14 - nonce_len' and 911 * iv[1..] contain the actual nonce. Extra bytes at the end are unused. 912 */ 913 nonce_len = 14 - (int)req->iv[0]; 914 if (unlikely(nonce_len < 7 || nonce_len > 13)) 915 return -EINVAL; 916 nonce = &req->iv[1]; 917 err = aes_ccm_init(ctx, data_len, req->assoclen, nonce, nonce_len, key); 918 if (unlikely(err)) 919 return err; 920 AES_PROCESS_ASSOC_DATA(aes_ccm_auth_update, req->src, req->assoclen, 921 ctx); 922 return 0; 923 } 924 925 static void aes_ccm_encrypt_update_helper(u8 *dst, const u8 *src, 926 unsigned int len, 927 struct aes_ccm_ctx *ctx) 928 { 929 aes_ccm_encrypt_update(ctx, dst, src, len); 930 } 931 932 static void aes_ccm_decrypt_update_helper(u8 *dst, const u8 *src, 933 unsigned int len, 934 struct aes_ccm_ctx *ctx) 935 { 936 aes_ccm_decrypt_update(ctx, dst, src, len); 937 } 938 939 static __maybe_unused int crypto_aes_ccm_encrypt(struct aead_request *req) 940 { 941 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 942 const struct aes_ccm_key *key = crypto_aead_ctx(tfm); 943 struct aes_ccm_ctx ctx; 944 u8 authtag[16]; 945 int err; 946 947 err = crypto_aes_ccm_init(&ctx, req, req->cryptlen, key); 948 if (unlikely(err)) 949 return err; 950 AES_CRYPT_SG(aes_ccm_encrypt_update_helper, req->dst, req->src, 951 req->cryptlen, req->assoclen, &ctx); 952 aes_ccm_encrypt_final(&ctx, authtag); 953 memcpy_to_sglist(req->dst, req->assoclen + req->cryptlen, authtag, 954 key->authtag_len); 955 memzero_explicit(authtag, sizeof(authtag)); 956 return 0; 957 } 958 959 static __maybe_unused int crypto_aes_ccm_decrypt(struct aead_request *req) 960 { 961 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 962 const struct aes_ccm_key *key = crypto_aead_ctx(tfm); 963 unsigned int data_len; 964 struct aes_ccm_ctx ctx; 965 u8 authtag[16]; 966 int err; 967 968 /* crypto_aead_decrypt() already checked cryptlen >= authtag_len. */ 969 data_len = req->cryptlen - key->authtag_len; 970 err = crypto_aes_ccm_init(&ctx, req, data_len, key); 971 if (unlikely(err)) 972 return err; 973 AES_CRYPT_SG(aes_ccm_decrypt_update_helper, req->dst, req->src, 974 data_len, req->assoclen, &ctx); 975 memcpy_from_sglist(authtag, req->src, req->assoclen + data_len, 976 key->authtag_len); 977 err = aes_ccm_decrypt_final(&ctx, authtag); 978 memzero_explicit(authtag, sizeof(authtag)); 979 return err; 980 } 981 982 static struct aead_alg aead_algs[] = { 983 #if IS_ENABLED(CONFIG_CRYPTO_GCM) 984 { 985 .base.cra_name = "gcm(aes)", 986 .base.cra_driver_name = "gcm-aes-lib", 987 .base.cra_priority = 110, 988 .base.cra_blocksize = 1, 989 .base.cra_ctxsize = sizeof(struct aes_gcm_key), 990 .base.cra_module = THIS_MODULE, 991 .setkey = crypto_aes_gcm_setkey, 992 .setauthsize = crypto_aes_gcm_setauthsize, 993 .encrypt = crypto_aes_gcm_encrypt, 994 .decrypt = crypto_aes_gcm_decrypt, 995 .ivsize = GCM_AES_IV_SIZE, 996 .maxauthsize = AES_BLOCK_SIZE, 997 .chunksize = AES_BLOCK_SIZE, 998 }, 999 { 1000 .base.cra_name = "rfc4106(gcm(aes))", 1001 .base.cra_driver_name = "rfc4106-gcm-aes-lib", 1002 .base.cra_priority = 110, 1003 .base.cra_blocksize = 1, 1004 .base.cra_ctxsize = sizeof(struct aes_rfc4106_key), 1005 .base.cra_module = THIS_MODULE, 1006 .setkey = crypto_aes_rfc4106_setkey, 1007 .setauthsize = crypto_aes_rfc4106_setauthsize, 1008 .encrypt = crypto_aes_rfc4106_encrypt, 1009 .decrypt = crypto_aes_rfc4106_decrypt, 1010 .ivsize = GCM_RFC4106_IV_SIZE, 1011 .maxauthsize = AES_BLOCK_SIZE, 1012 .chunksize = AES_BLOCK_SIZE, 1013 }, 1014 #endif /* CONFIG_CRYPTO_GCM */ 1015 #if IS_ENABLED(CONFIG_CRYPTO_CCM) 1016 { 1017 .base.cra_name = "ccm(aes)", 1018 .base.cra_driver_name = "ccm-aes-lib", 1019 .base.cra_priority = 110, 1020 .base.cra_blocksize = 1, 1021 .base.cra_ctxsize = sizeof(struct aes_ccm_key), 1022 .base.cra_module = THIS_MODULE, 1023 .setkey = crypto_aes_ccm_setkey, 1024 .setauthsize = crypto_aes_ccm_setauthsize, 1025 .encrypt = crypto_aes_ccm_encrypt, 1026 .decrypt = crypto_aes_ccm_decrypt, 1027 .ivsize = 16, 1028 .maxauthsize = 16, 1029 .chunksize = AES_BLOCK_SIZE, 1030 }, 1031 #endif /* CONFIG_CRYPTO_CCM */ 1032 }; 1033 1034 static int __init crypto_aes_mod_init(void) 1035 { 1036 int err = crypto_register_alg(&alg); 1037 1038 if (err) 1039 return err; 1040 1041 if (ARRAY_SIZE(mac_algs) > 0) { 1042 err = crypto_register_shashes(mac_algs, ARRAY_SIZE(mac_algs)); 1043 if (err) 1044 goto err_unregister_alg; 1045 } /* Else, CONFIG_CRYPTO_HASH might not be enabled. */ 1046 1047 if (ARRAY_SIZE(skcipher_algs) > 0) { 1048 err = crypto_register_skciphers(skcipher_algs, 1049 ARRAY_SIZE(skcipher_algs)); 1050 if (err) 1051 goto err_unregister_macs; 1052 } 1053 1054 if (ARRAY_SIZE(aead_algs) > 0) { 1055 err = crypto_register_aeads(aead_algs, ARRAY_SIZE(aead_algs)); 1056 if (err) 1057 goto err_unregister_skciphers; 1058 } /* Else, CONFIG_CRYPTO_AEAD might not be enabled. */ 1059 return 0; 1060 1061 err_unregister_skciphers: 1062 if (ARRAY_SIZE(skcipher_algs) > 0) 1063 crypto_unregister_skciphers(skcipher_algs, 1064 ARRAY_SIZE(skcipher_algs)); 1065 err_unregister_macs: 1066 if (ARRAY_SIZE(mac_algs) > 0) 1067 crypto_unregister_shashes(mac_algs, ARRAY_SIZE(mac_algs)); 1068 err_unregister_alg: 1069 crypto_unregister_alg(&alg); 1070 return err; 1071 } 1072 module_init(crypto_aes_mod_init); 1073 1074 static void __exit crypto_aes_mod_exit(void) 1075 { 1076 if (ARRAY_SIZE(aead_algs) > 0) 1077 crypto_unregister_aeads(aead_algs, ARRAY_SIZE(aead_algs)); 1078 if (ARRAY_SIZE(skcipher_algs) > 0) 1079 crypto_unregister_skciphers(skcipher_algs, 1080 ARRAY_SIZE(skcipher_algs)); 1081 if (ARRAY_SIZE(mac_algs) > 0) 1082 crypto_unregister_shashes(mac_algs, ARRAY_SIZE(mac_algs)); 1083 crypto_unregister_alg(&alg); 1084 } 1085 module_exit(crypto_aes_mod_exit); 1086 1087 MODULE_DESCRIPTION("Crypto API support for AES block cipher"); 1088 MODULE_IMPORT_NS("CRYPTO_INTERNAL"); 1089 MODULE_LICENSE("GPL"); 1090 MODULE_ALIAS_CRYPTO("aes"); 1091 MODULE_ALIAS_CRYPTO("aes-lib"); 1092 #if IS_ENABLED(CONFIG_CRYPTO_CMAC) 1093 MODULE_ALIAS_CRYPTO("cmac(aes)"); 1094 MODULE_ALIAS_CRYPTO("cmac-aes-lib"); 1095 #endif 1096 #if IS_ENABLED(CONFIG_CRYPTO_XCBC) 1097 MODULE_ALIAS_CRYPTO("xcbc(aes)"); 1098 MODULE_ALIAS_CRYPTO("xcbc-aes-lib"); 1099 #endif 1100 #if IS_ENABLED(CONFIG_CRYPTO_CCM) 1101 MODULE_ALIAS_CRYPTO("cbcmac(aes)"); 1102 MODULE_ALIAS_CRYPTO("cbcmac-aes-lib"); 1103 #endif 1104 #if IS_ENABLED(CONFIG_CRYPTO_ECB) 1105 MODULE_ALIAS_CRYPTO("ecb(aes)"); 1106 MODULE_ALIAS_CRYPTO("ecb-aes-lib"); 1107 #endif 1108 #if IS_ENABLED(CONFIG_CRYPTO_CBC) 1109 MODULE_ALIAS_CRYPTO("cbc(aes)"); 1110 MODULE_ALIAS_CRYPTO("cbc-aes-lib"); 1111 #endif 1112 #if IS_ENABLED(CONFIG_CRYPTO_CTS) 1113 MODULE_ALIAS_CRYPTO("cts(cbc(aes))"); 1114 MODULE_ALIAS_CRYPTO("cts-cbc-aes-lib"); 1115 #endif 1116 #if IS_ENABLED(CONFIG_CRYPTO_CTR) 1117 MODULE_ALIAS_CRYPTO("ctr(aes)"); 1118 MODULE_ALIAS_CRYPTO("ctr-aes-lib"); 1119 #endif 1120 #if IS_ENABLED(CONFIG_CRYPTO_XCTR) 1121 MODULE_ALIAS_CRYPTO("xctr(aes)"); 1122 MODULE_ALIAS_CRYPTO("xctr-aes-lib"); 1123 #endif 1124 #if IS_ENABLED(CONFIG_CRYPTO_XTS) 1125 MODULE_ALIAS_CRYPTO("xts(aes)"); 1126 MODULE_ALIAS_CRYPTO("xts-aes-lib"); 1127 #endif 1128 #if IS_ENABLED(CONFIG_CRYPTO_GCM) 1129 MODULE_ALIAS_CRYPTO("gcm(aes)"); 1130 MODULE_ALIAS_CRYPTO("gcm-aes-lib"); 1131 MODULE_ALIAS_CRYPTO("rfc4106(gcm(aes))"); 1132 MODULE_ALIAS_CRYPTO("rfc4106-gcm-aes-lib"); 1133 #endif 1134 #if IS_ENABLED(CONFIG_CRYPTO_CCM) 1135 MODULE_ALIAS_CRYPTO("ccm(aes)"); 1136 MODULE_ALIAS_CRYPTO("ccm-aes-lib"); 1137 #endif 1138