1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Accelerated GHASH implementation with ARMv8 vmull.p64 instructions. 4 * 5 * Copyright (C) 2015 - 2018 Linaro Ltd. 6 * Copyright (C) 2023 Google LLC. 7 */ 8 9 #include <asm/hwcap.h> 10 #include <asm/neon.h> 11 #include <asm/simd.h> 12 #include <linux/unaligned.h> 13 #include <crypto/aes.h> 14 #include <crypto/gcm.h> 15 #include <crypto/b128ops.h> 16 #include <crypto/cryptd.h> 17 #include <crypto/internal/aead.h> 18 #include <crypto/internal/hash.h> 19 #include <crypto/internal/simd.h> 20 #include <crypto/internal/skcipher.h> 21 #include <crypto/gf128mul.h> 22 #include <crypto/scatterwalk.h> 23 #include <linux/cpufeature.h> 24 #include <linux/crypto.h> 25 #include <linux/jump_label.h> 26 #include <linux/module.h> 27 28 MODULE_DESCRIPTION("GHASH hash function using ARMv8 Crypto Extensions"); 29 MODULE_AUTHOR("Ard Biesheuvel <ardb@kernel.org>"); 30 MODULE_LICENSE("GPL"); 31 MODULE_ALIAS_CRYPTO("ghash"); 32 MODULE_ALIAS_CRYPTO("gcm(aes)"); 33 MODULE_ALIAS_CRYPTO("rfc4106(gcm(aes))"); 34 35 #define GHASH_BLOCK_SIZE 16 36 #define GHASH_DIGEST_SIZE 16 37 38 #define RFC4106_NONCE_SIZE 4 39 40 struct ghash_key { 41 be128 k; 42 u64 h[][2]; 43 }; 44 45 struct gcm_key { 46 u64 h[4][2]; 47 u32 rk[AES_MAX_KEYLENGTH_U32]; 48 int rounds; 49 u8 nonce[]; // for RFC4106 nonce 50 }; 51 52 struct ghash_desc_ctx { 53 u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)]; 54 u8 buf[GHASH_BLOCK_SIZE]; 55 u32 count; 56 }; 57 58 struct ghash_async_ctx { 59 struct cryptd_ahash *cryptd_tfm; 60 }; 61 62 asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src, 63 u64 const h[][2], const char *head); 64 65 asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src, 66 u64 const h[][2], const char *head); 67 68 static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_p64); 69 70 static int ghash_init(struct shash_desc *desc) 71 { 72 struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); 73 74 *ctx = (struct ghash_desc_ctx){}; 75 return 0; 76 } 77 78 static void ghash_do_update(int blocks, u64 dg[], const char *src, 79 struct ghash_key *key, const char *head) 80 { 81 if (likely(crypto_simd_usable())) { 82 kernel_neon_begin(); 83 if (static_branch_likely(&use_p64)) 84 pmull_ghash_update_p64(blocks, dg, src, key->h, head); 85 else 86 pmull_ghash_update_p8(blocks, dg, src, key->h, head); 87 kernel_neon_end(); 88 } else { 89 be128 dst = { cpu_to_be64(dg[1]), cpu_to_be64(dg[0]) }; 90 91 do { 92 const u8 *in = src; 93 94 if (head) { 95 in = head; 96 blocks++; 97 head = NULL; 98 } else { 99 src += GHASH_BLOCK_SIZE; 100 } 101 102 crypto_xor((u8 *)&dst, in, GHASH_BLOCK_SIZE); 103 gf128mul_lle(&dst, &key->k); 104 } while (--blocks); 105 106 dg[0] = be64_to_cpu(dst.b); 107 dg[1] = be64_to_cpu(dst.a); 108 } 109 } 110 111 static int ghash_update(struct shash_desc *desc, const u8 *src, 112 unsigned int len) 113 { 114 struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); 115 unsigned int partial = ctx->count % GHASH_BLOCK_SIZE; 116 117 ctx->count += len; 118 119 if ((partial + len) >= GHASH_BLOCK_SIZE) { 120 struct ghash_key *key = crypto_shash_ctx(desc->tfm); 121 int blocks; 122 123 if (partial) { 124 int p = GHASH_BLOCK_SIZE - partial; 125 126 memcpy(ctx->buf + partial, src, p); 127 src += p; 128 len -= p; 129 } 130 131 blocks = len / GHASH_BLOCK_SIZE; 132 len %= GHASH_BLOCK_SIZE; 133 134 ghash_do_update(blocks, ctx->digest, src, key, 135 partial ? ctx->buf : NULL); 136 src += blocks * GHASH_BLOCK_SIZE; 137 partial = 0; 138 } 139 if (len) 140 memcpy(ctx->buf + partial, src, len); 141 return 0; 142 } 143 144 static int ghash_final(struct shash_desc *desc, u8 *dst) 145 { 146 struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); 147 unsigned int partial = ctx->count % GHASH_BLOCK_SIZE; 148 149 if (partial) { 150 struct ghash_key *key = crypto_shash_ctx(desc->tfm); 151 152 memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial); 153 ghash_do_update(1, ctx->digest, ctx->buf, key, NULL); 154 } 155 put_unaligned_be64(ctx->digest[1], dst); 156 put_unaligned_be64(ctx->digest[0], dst + 8); 157 158 *ctx = (struct ghash_desc_ctx){}; 159 return 0; 160 } 161 162 static void ghash_reflect(u64 h[], const be128 *k) 163 { 164 u64 carry = be64_to_cpu(k->a) >> 63; 165 166 h[0] = (be64_to_cpu(k->b) << 1) | carry; 167 h[1] = (be64_to_cpu(k->a) << 1) | (be64_to_cpu(k->b) >> 63); 168 169 if (carry) 170 h[1] ^= 0xc200000000000000UL; 171 } 172 173 static int ghash_setkey(struct crypto_shash *tfm, 174 const u8 *inkey, unsigned int keylen) 175 { 176 struct ghash_key *key = crypto_shash_ctx(tfm); 177 178 if (keylen != GHASH_BLOCK_SIZE) 179 return -EINVAL; 180 181 /* needed for the fallback */ 182 memcpy(&key->k, inkey, GHASH_BLOCK_SIZE); 183 ghash_reflect(key->h[0], &key->k); 184 185 if (static_branch_likely(&use_p64)) { 186 be128 h = key->k; 187 188 gf128mul_lle(&h, &key->k); 189 ghash_reflect(key->h[1], &h); 190 191 gf128mul_lle(&h, &key->k); 192 ghash_reflect(key->h[2], &h); 193 194 gf128mul_lle(&h, &key->k); 195 ghash_reflect(key->h[3], &h); 196 } 197 return 0; 198 } 199 200 static struct shash_alg ghash_alg = { 201 .digestsize = GHASH_DIGEST_SIZE, 202 .init = ghash_init, 203 .update = ghash_update, 204 .final = ghash_final, 205 .setkey = ghash_setkey, 206 .descsize = sizeof(struct ghash_desc_ctx), 207 208 .base.cra_name = "ghash", 209 .base.cra_driver_name = "ghash-ce-sync", 210 .base.cra_priority = 300 - 1, 211 .base.cra_blocksize = GHASH_BLOCK_SIZE, 212 .base.cra_ctxsize = sizeof(struct ghash_key) + sizeof(u64[2]), 213 .base.cra_module = THIS_MODULE, 214 }; 215 216 static int ghash_async_init(struct ahash_request *req) 217 { 218 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 219 struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); 220 struct ahash_request *cryptd_req = ahash_request_ctx(req); 221 struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; 222 struct shash_desc *desc = cryptd_shash_desc(cryptd_req); 223 struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm); 224 225 desc->tfm = child; 226 return crypto_shash_init(desc); 227 } 228 229 static int ghash_async_update(struct ahash_request *req) 230 { 231 struct ahash_request *cryptd_req = ahash_request_ctx(req); 232 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 233 struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); 234 struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; 235 236 if (!crypto_simd_usable() || 237 (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { 238 memcpy(cryptd_req, req, sizeof(*req)); 239 ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); 240 return crypto_ahash_update(cryptd_req); 241 } else { 242 struct shash_desc *desc = cryptd_shash_desc(cryptd_req); 243 return shash_ahash_update(req, desc); 244 } 245 } 246 247 static int ghash_async_final(struct ahash_request *req) 248 { 249 struct ahash_request *cryptd_req = ahash_request_ctx(req); 250 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 251 struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); 252 struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; 253 254 if (!crypto_simd_usable() || 255 (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { 256 memcpy(cryptd_req, req, sizeof(*req)); 257 ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); 258 return crypto_ahash_final(cryptd_req); 259 } else { 260 struct shash_desc *desc = cryptd_shash_desc(cryptd_req); 261 return crypto_shash_final(desc, req->result); 262 } 263 } 264 265 static int ghash_async_digest(struct ahash_request *req) 266 { 267 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 268 struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); 269 struct ahash_request *cryptd_req = ahash_request_ctx(req); 270 struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; 271 272 if (!crypto_simd_usable() || 273 (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { 274 memcpy(cryptd_req, req, sizeof(*req)); 275 ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); 276 return crypto_ahash_digest(cryptd_req); 277 } else { 278 struct shash_desc *desc = cryptd_shash_desc(cryptd_req); 279 struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm); 280 281 desc->tfm = child; 282 return shash_ahash_digest(req, desc); 283 } 284 } 285 286 static int ghash_async_import(struct ahash_request *req, const void *in) 287 { 288 struct ahash_request *cryptd_req = ahash_request_ctx(req); 289 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 290 struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); 291 struct shash_desc *desc = cryptd_shash_desc(cryptd_req); 292 293 desc->tfm = cryptd_ahash_child(ctx->cryptd_tfm); 294 295 return crypto_shash_import(desc, in); 296 } 297 298 static int ghash_async_export(struct ahash_request *req, void *out) 299 { 300 struct ahash_request *cryptd_req = ahash_request_ctx(req); 301 struct shash_desc *desc = cryptd_shash_desc(cryptd_req); 302 303 return crypto_shash_export(desc, out); 304 } 305 306 static int ghash_async_setkey(struct crypto_ahash *tfm, const u8 *key, 307 unsigned int keylen) 308 { 309 struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); 310 struct crypto_ahash *child = &ctx->cryptd_tfm->base; 311 312 crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK); 313 crypto_ahash_set_flags(child, crypto_ahash_get_flags(tfm) 314 & CRYPTO_TFM_REQ_MASK); 315 return crypto_ahash_setkey(child, key, keylen); 316 } 317 318 static int ghash_async_init_tfm(struct crypto_tfm *tfm) 319 { 320 struct cryptd_ahash *cryptd_tfm; 321 struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm); 322 323 cryptd_tfm = cryptd_alloc_ahash("ghash-ce-sync", 0, 0); 324 if (IS_ERR(cryptd_tfm)) 325 return PTR_ERR(cryptd_tfm); 326 ctx->cryptd_tfm = cryptd_tfm; 327 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 328 sizeof(struct ahash_request) + 329 crypto_ahash_reqsize(&cryptd_tfm->base)); 330 331 return 0; 332 } 333 334 static void ghash_async_exit_tfm(struct crypto_tfm *tfm) 335 { 336 struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm); 337 338 cryptd_free_ahash(ctx->cryptd_tfm); 339 } 340 341 static struct ahash_alg ghash_async_alg = { 342 .init = ghash_async_init, 343 .update = ghash_async_update, 344 .final = ghash_async_final, 345 .setkey = ghash_async_setkey, 346 .digest = ghash_async_digest, 347 .import = ghash_async_import, 348 .export = ghash_async_export, 349 .halg.digestsize = GHASH_DIGEST_SIZE, 350 .halg.statesize = sizeof(struct ghash_desc_ctx), 351 .halg.base = { 352 .cra_name = "ghash", 353 .cra_driver_name = "ghash-ce", 354 .cra_priority = 300, 355 .cra_flags = CRYPTO_ALG_ASYNC, 356 .cra_blocksize = GHASH_BLOCK_SIZE, 357 .cra_ctxsize = sizeof(struct ghash_async_ctx), 358 .cra_module = THIS_MODULE, 359 .cra_init = ghash_async_init_tfm, 360 .cra_exit = ghash_async_exit_tfm, 361 }, 362 }; 363 364 365 void pmull_gcm_encrypt(int blocks, u64 dg[], const char *src, 366 struct gcm_key const *k, char *dst, 367 const char *iv, int rounds, u32 counter); 368 369 void pmull_gcm_enc_final(int blocks, u64 dg[], char *tag, 370 struct gcm_key const *k, char *head, 371 const char *iv, int rounds, u32 counter); 372 373 void pmull_gcm_decrypt(int bytes, u64 dg[], const char *src, 374 struct gcm_key const *k, char *dst, 375 const char *iv, int rounds, u32 counter); 376 377 int pmull_gcm_dec_final(int bytes, u64 dg[], char *tag, 378 struct gcm_key const *k, char *head, 379 const char *iv, int rounds, u32 counter, 380 const char *otag, int authsize); 381 382 static int gcm_aes_setkey(struct crypto_aead *tfm, const u8 *inkey, 383 unsigned int keylen) 384 { 385 struct gcm_key *ctx = crypto_aead_ctx(tfm); 386 struct crypto_aes_ctx aes_ctx; 387 be128 h, k; 388 int ret; 389 390 ret = aes_expandkey(&aes_ctx, inkey, keylen); 391 if (ret) 392 return -EINVAL; 393 394 aes_encrypt(&aes_ctx, (u8 *)&k, (u8[AES_BLOCK_SIZE]){}); 395 396 memcpy(ctx->rk, aes_ctx.key_enc, sizeof(ctx->rk)); 397 ctx->rounds = 6 + keylen / 4; 398 399 memzero_explicit(&aes_ctx, sizeof(aes_ctx)); 400 401 ghash_reflect(ctx->h[0], &k); 402 403 h = k; 404 gf128mul_lle(&h, &k); 405 ghash_reflect(ctx->h[1], &h); 406 407 gf128mul_lle(&h, &k); 408 ghash_reflect(ctx->h[2], &h); 409 410 gf128mul_lle(&h, &k); 411 ghash_reflect(ctx->h[3], &h); 412 413 return 0; 414 } 415 416 static int gcm_aes_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 417 { 418 return crypto_gcm_check_authsize(authsize); 419 } 420 421 static void gcm_update_mac(u64 dg[], const u8 *src, int count, u8 buf[], 422 int *buf_count, struct gcm_key *ctx) 423 { 424 if (*buf_count > 0) { 425 int buf_added = min(count, GHASH_BLOCK_SIZE - *buf_count); 426 427 memcpy(&buf[*buf_count], src, buf_added); 428 429 *buf_count += buf_added; 430 src += buf_added; 431 count -= buf_added; 432 } 433 434 if (count >= GHASH_BLOCK_SIZE || *buf_count == GHASH_BLOCK_SIZE) { 435 int blocks = count / GHASH_BLOCK_SIZE; 436 437 pmull_ghash_update_p64(blocks, dg, src, ctx->h, 438 *buf_count ? buf : NULL); 439 440 src += blocks * GHASH_BLOCK_SIZE; 441 count %= GHASH_BLOCK_SIZE; 442 *buf_count = 0; 443 } 444 445 if (count > 0) { 446 memcpy(buf, src, count); 447 *buf_count = count; 448 } 449 } 450 451 static void gcm_calculate_auth_mac(struct aead_request *req, u64 dg[], u32 len) 452 { 453 struct crypto_aead *aead = crypto_aead_reqtfm(req); 454 struct gcm_key *ctx = crypto_aead_ctx(aead); 455 u8 buf[GHASH_BLOCK_SIZE]; 456 struct scatter_walk walk; 457 int buf_count = 0; 458 459 scatterwalk_start(&walk, req->src); 460 461 do { 462 unsigned int n; 463 const u8 *p; 464 465 p = scatterwalk_next(&walk, len, &n); 466 gcm_update_mac(dg, p, n, buf, &buf_count, ctx); 467 scatterwalk_done_src(&walk, p, n); 468 469 if (unlikely(len / SZ_4K > (len - n) / SZ_4K)) { 470 kernel_neon_end(); 471 kernel_neon_begin(); 472 } 473 474 len -= n; 475 } while (len); 476 477 if (buf_count) { 478 memset(&buf[buf_count], 0, GHASH_BLOCK_SIZE - buf_count); 479 pmull_ghash_update_p64(1, dg, buf, ctx->h, NULL); 480 } 481 } 482 483 static int gcm_encrypt(struct aead_request *req, const u8 *iv, u32 assoclen) 484 { 485 struct crypto_aead *aead = crypto_aead_reqtfm(req); 486 struct gcm_key *ctx = crypto_aead_ctx(aead); 487 struct skcipher_walk walk; 488 u8 buf[AES_BLOCK_SIZE]; 489 u32 counter = 2; 490 u64 dg[2] = {}; 491 be128 lengths; 492 const u8 *src; 493 u8 *tag, *dst; 494 int tail, err; 495 496 if (WARN_ON_ONCE(!may_use_simd())) 497 return -EBUSY; 498 499 err = skcipher_walk_aead_encrypt(&walk, req, false); 500 501 kernel_neon_begin(); 502 503 if (assoclen) 504 gcm_calculate_auth_mac(req, dg, assoclen); 505 506 src = walk.src.virt.addr; 507 dst = walk.dst.virt.addr; 508 509 while (walk.nbytes >= AES_BLOCK_SIZE) { 510 int nblocks = walk.nbytes / AES_BLOCK_SIZE; 511 512 pmull_gcm_encrypt(nblocks, dg, src, ctx, dst, iv, 513 ctx->rounds, counter); 514 counter += nblocks; 515 516 if (walk.nbytes == walk.total) { 517 src += nblocks * AES_BLOCK_SIZE; 518 dst += nblocks * AES_BLOCK_SIZE; 519 break; 520 } 521 522 kernel_neon_end(); 523 524 err = skcipher_walk_done(&walk, 525 walk.nbytes % AES_BLOCK_SIZE); 526 if (err) 527 return err; 528 529 src = walk.src.virt.addr; 530 dst = walk.dst.virt.addr; 531 532 kernel_neon_begin(); 533 } 534 535 536 lengths.a = cpu_to_be64(assoclen * 8); 537 lengths.b = cpu_to_be64(req->cryptlen * 8); 538 539 tag = (u8 *)&lengths; 540 tail = walk.nbytes % AES_BLOCK_SIZE; 541 542 /* 543 * Bounce via a buffer unless we are encrypting in place and src/dst 544 * are not pointing to the start of the walk buffer. In that case, we 545 * can do a NEON load/xor/store sequence in place as long as we move 546 * the plain/ciphertext and keystream to the start of the register. If 547 * not, do a memcpy() to the end of the buffer so we can reuse the same 548 * logic. 549 */ 550 if (unlikely(tail && (tail == walk.nbytes || src != dst))) 551 src = memcpy(buf + sizeof(buf) - tail, src, tail); 552 553 pmull_gcm_enc_final(tail, dg, tag, ctx, (u8 *)src, iv, 554 ctx->rounds, counter); 555 kernel_neon_end(); 556 557 if (unlikely(tail && src != dst)) 558 memcpy(dst, src, tail); 559 560 if (walk.nbytes) { 561 err = skcipher_walk_done(&walk, 0); 562 if (err) 563 return err; 564 } 565 566 /* copy authtag to end of dst */ 567 scatterwalk_map_and_copy(tag, req->dst, req->assoclen + req->cryptlen, 568 crypto_aead_authsize(aead), 1); 569 570 return 0; 571 } 572 573 static int gcm_decrypt(struct aead_request *req, const u8 *iv, u32 assoclen) 574 { 575 struct crypto_aead *aead = crypto_aead_reqtfm(req); 576 struct gcm_key *ctx = crypto_aead_ctx(aead); 577 int authsize = crypto_aead_authsize(aead); 578 struct skcipher_walk walk; 579 u8 otag[AES_BLOCK_SIZE]; 580 u8 buf[AES_BLOCK_SIZE]; 581 u32 counter = 2; 582 u64 dg[2] = {}; 583 be128 lengths; 584 const u8 *src; 585 u8 *tag, *dst; 586 int tail, err, ret; 587 588 if (WARN_ON_ONCE(!may_use_simd())) 589 return -EBUSY; 590 591 scatterwalk_map_and_copy(otag, req->src, 592 req->assoclen + req->cryptlen - authsize, 593 authsize, 0); 594 595 err = skcipher_walk_aead_decrypt(&walk, req, false); 596 597 kernel_neon_begin(); 598 599 if (assoclen) 600 gcm_calculate_auth_mac(req, dg, assoclen); 601 602 src = walk.src.virt.addr; 603 dst = walk.dst.virt.addr; 604 605 while (walk.nbytes >= AES_BLOCK_SIZE) { 606 int nblocks = walk.nbytes / AES_BLOCK_SIZE; 607 608 pmull_gcm_decrypt(nblocks, dg, src, ctx, dst, iv, 609 ctx->rounds, counter); 610 counter += nblocks; 611 612 if (walk.nbytes == walk.total) { 613 src += nblocks * AES_BLOCK_SIZE; 614 dst += nblocks * AES_BLOCK_SIZE; 615 break; 616 } 617 618 kernel_neon_end(); 619 620 err = skcipher_walk_done(&walk, 621 walk.nbytes % AES_BLOCK_SIZE); 622 if (err) 623 return err; 624 625 src = walk.src.virt.addr; 626 dst = walk.dst.virt.addr; 627 628 kernel_neon_begin(); 629 } 630 631 lengths.a = cpu_to_be64(assoclen * 8); 632 lengths.b = cpu_to_be64((req->cryptlen - authsize) * 8); 633 634 tag = (u8 *)&lengths; 635 tail = walk.nbytes % AES_BLOCK_SIZE; 636 637 if (unlikely(tail && (tail == walk.nbytes || src != dst))) 638 src = memcpy(buf + sizeof(buf) - tail, src, tail); 639 640 ret = pmull_gcm_dec_final(tail, dg, tag, ctx, (u8 *)src, iv, 641 ctx->rounds, counter, otag, authsize); 642 kernel_neon_end(); 643 644 if (unlikely(tail && src != dst)) 645 memcpy(dst, src, tail); 646 647 if (walk.nbytes) { 648 err = skcipher_walk_done(&walk, 0); 649 if (err) 650 return err; 651 } 652 653 return ret ? -EBADMSG : 0; 654 } 655 656 static int gcm_aes_encrypt(struct aead_request *req) 657 { 658 return gcm_encrypt(req, req->iv, req->assoclen); 659 } 660 661 static int gcm_aes_decrypt(struct aead_request *req) 662 { 663 return gcm_decrypt(req, req->iv, req->assoclen); 664 } 665 666 static int rfc4106_setkey(struct crypto_aead *tfm, const u8 *inkey, 667 unsigned int keylen) 668 { 669 struct gcm_key *ctx = crypto_aead_ctx(tfm); 670 int err; 671 672 keylen -= RFC4106_NONCE_SIZE; 673 err = gcm_aes_setkey(tfm, inkey, keylen); 674 if (err) 675 return err; 676 677 memcpy(ctx->nonce, inkey + keylen, RFC4106_NONCE_SIZE); 678 return 0; 679 } 680 681 static int rfc4106_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 682 { 683 return crypto_rfc4106_check_authsize(authsize); 684 } 685 686 static int rfc4106_encrypt(struct aead_request *req) 687 { 688 struct crypto_aead *aead = crypto_aead_reqtfm(req); 689 struct gcm_key *ctx = crypto_aead_ctx(aead); 690 u8 iv[GCM_AES_IV_SIZE]; 691 692 memcpy(iv, ctx->nonce, RFC4106_NONCE_SIZE); 693 memcpy(iv + RFC4106_NONCE_SIZE, req->iv, GCM_RFC4106_IV_SIZE); 694 695 return crypto_ipsec_check_assoclen(req->assoclen) ?: 696 gcm_encrypt(req, iv, req->assoclen - GCM_RFC4106_IV_SIZE); 697 } 698 699 static int rfc4106_decrypt(struct aead_request *req) 700 { 701 struct crypto_aead *aead = crypto_aead_reqtfm(req); 702 struct gcm_key *ctx = crypto_aead_ctx(aead); 703 u8 iv[GCM_AES_IV_SIZE]; 704 705 memcpy(iv, ctx->nonce, RFC4106_NONCE_SIZE); 706 memcpy(iv + RFC4106_NONCE_SIZE, req->iv, GCM_RFC4106_IV_SIZE); 707 708 return crypto_ipsec_check_assoclen(req->assoclen) ?: 709 gcm_decrypt(req, iv, req->assoclen - GCM_RFC4106_IV_SIZE); 710 } 711 712 static struct aead_alg gcm_aes_algs[] = {{ 713 .ivsize = GCM_AES_IV_SIZE, 714 .chunksize = AES_BLOCK_SIZE, 715 .maxauthsize = AES_BLOCK_SIZE, 716 .setkey = gcm_aes_setkey, 717 .setauthsize = gcm_aes_setauthsize, 718 .encrypt = gcm_aes_encrypt, 719 .decrypt = gcm_aes_decrypt, 720 721 .base.cra_name = "gcm(aes)", 722 .base.cra_driver_name = "gcm-aes-ce", 723 .base.cra_priority = 400, 724 .base.cra_blocksize = 1, 725 .base.cra_ctxsize = sizeof(struct gcm_key), 726 .base.cra_module = THIS_MODULE, 727 }, { 728 .ivsize = GCM_RFC4106_IV_SIZE, 729 .chunksize = AES_BLOCK_SIZE, 730 .maxauthsize = AES_BLOCK_SIZE, 731 .setkey = rfc4106_setkey, 732 .setauthsize = rfc4106_setauthsize, 733 .encrypt = rfc4106_encrypt, 734 .decrypt = rfc4106_decrypt, 735 736 .base.cra_name = "rfc4106(gcm(aes))", 737 .base.cra_driver_name = "rfc4106-gcm-aes-ce", 738 .base.cra_priority = 400, 739 .base.cra_blocksize = 1, 740 .base.cra_ctxsize = sizeof(struct gcm_key) + RFC4106_NONCE_SIZE, 741 .base.cra_module = THIS_MODULE, 742 }}; 743 744 static int __init ghash_ce_mod_init(void) 745 { 746 int err; 747 748 if (!(elf_hwcap & HWCAP_NEON)) 749 return -ENODEV; 750 751 if (elf_hwcap2 & HWCAP2_PMULL) { 752 err = crypto_register_aeads(gcm_aes_algs, 753 ARRAY_SIZE(gcm_aes_algs)); 754 if (err) 755 return err; 756 ghash_alg.base.cra_ctxsize += 3 * sizeof(u64[2]); 757 static_branch_enable(&use_p64); 758 } 759 760 err = crypto_register_shash(&ghash_alg); 761 if (err) 762 goto err_aead; 763 err = crypto_register_ahash(&ghash_async_alg); 764 if (err) 765 goto err_shash; 766 767 return 0; 768 769 err_shash: 770 crypto_unregister_shash(&ghash_alg); 771 err_aead: 772 if (elf_hwcap2 & HWCAP2_PMULL) 773 crypto_unregister_aeads(gcm_aes_algs, 774 ARRAY_SIZE(gcm_aes_algs)); 775 return err; 776 } 777 778 static void __exit ghash_ce_mod_exit(void) 779 { 780 crypto_unregister_ahash(&ghash_async_alg); 781 crypto_unregister_shash(&ghash_alg); 782 if (elf_hwcap2 & HWCAP2_PMULL) 783 crypto_unregister_aeads(gcm_aes_algs, 784 ARRAY_SIZE(gcm_aes_algs)); 785 } 786 787 module_init(ghash_ce_mod_init); 788 module_exit(ghash_ce_mod_exit); 789