1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Accelerated GHASH implementation with ARMv8 vmull.p64 instructions. 4 * 5 * Copyright (C) 2015 - 2018 Linaro Ltd. 6 * Copyright (C) 2023 Google LLC. 7 */ 8 9 #include <asm/hwcap.h> 10 #include <asm/neon.h> 11 #include <asm/simd.h> 12 #include <linux/unaligned.h> 13 #include <crypto/aes.h> 14 #include <crypto/gcm.h> 15 #include <crypto/b128ops.h> 16 #include <crypto/cryptd.h> 17 #include <crypto/internal/aead.h> 18 #include <crypto/internal/hash.h> 19 #include <crypto/internal/simd.h> 20 #include <crypto/internal/skcipher.h> 21 #include <crypto/gf128mul.h> 22 #include <crypto/scatterwalk.h> 23 #include <linux/cpufeature.h> 24 #include <linux/crypto.h> 25 #include <linux/jump_label.h> 26 #include <linux/module.h> 27 28 MODULE_DESCRIPTION("GHASH hash function using ARMv8 Crypto Extensions"); 29 MODULE_AUTHOR("Ard Biesheuvel <ardb@kernel.org>"); 30 MODULE_LICENSE("GPL"); 31 MODULE_ALIAS_CRYPTO("ghash"); 32 MODULE_ALIAS_CRYPTO("gcm(aes)"); 33 MODULE_ALIAS_CRYPTO("rfc4106(gcm(aes))"); 34 35 #define GHASH_BLOCK_SIZE 16 36 #define GHASH_DIGEST_SIZE 16 37 38 #define RFC4106_NONCE_SIZE 4 39 40 struct ghash_key { 41 be128 k; 42 u64 h[][2]; 43 }; 44 45 struct gcm_key { 46 u64 h[4][2]; 47 u32 rk[AES_MAX_KEYLENGTH_U32]; 48 int rounds; 49 u8 nonce[]; // for RFC4106 nonce 50 }; 51 52 struct ghash_desc_ctx { 53 u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)]; 54 u8 buf[GHASH_BLOCK_SIZE]; 55 u32 count; 56 }; 57 58 struct ghash_async_ctx { 59 struct cryptd_ahash *cryptd_tfm; 60 }; 61 62 asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src, 63 u64 const h[][2], const char *head); 64 65 asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src, 66 u64 const h[][2], const char *head); 67 68 static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_p64); 69 70 static int ghash_init(struct shash_desc *desc) 71 { 72 struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); 73 74 *ctx = (struct ghash_desc_ctx){}; 75 return 0; 76 } 77 78 static void ghash_do_update(int blocks, u64 dg[], const char *src, 79 struct ghash_key *key, const char *head) 80 { 81 if (likely(crypto_simd_usable())) { 82 kernel_neon_begin(); 83 if (static_branch_likely(&use_p64)) 84 pmull_ghash_update_p64(blocks, dg, src, key->h, head); 85 else 86 pmull_ghash_update_p8(blocks, dg, src, key->h, head); 87 kernel_neon_end(); 88 } else { 89 be128 dst = { cpu_to_be64(dg[1]), cpu_to_be64(dg[0]) }; 90 91 do { 92 const u8 *in = src; 93 94 if (head) { 95 in = head; 96 blocks++; 97 head = NULL; 98 } else { 99 src += GHASH_BLOCK_SIZE; 100 } 101 102 crypto_xor((u8 *)&dst, in, GHASH_BLOCK_SIZE); 103 gf128mul_lle(&dst, &key->k); 104 } while (--blocks); 105 106 dg[0] = be64_to_cpu(dst.b); 107 dg[1] = be64_to_cpu(dst.a); 108 } 109 } 110 111 static int ghash_update(struct shash_desc *desc, const u8 *src, 112 unsigned int len) 113 { 114 struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); 115 unsigned int partial = ctx->count % GHASH_BLOCK_SIZE; 116 117 ctx->count += len; 118 119 if ((partial + len) >= GHASH_BLOCK_SIZE) { 120 struct ghash_key *key = crypto_shash_ctx(desc->tfm); 121 int blocks; 122 123 if (partial) { 124 int p = GHASH_BLOCK_SIZE - partial; 125 126 memcpy(ctx->buf + partial, src, p); 127 src += p; 128 len -= p; 129 } 130 131 blocks = len / GHASH_BLOCK_SIZE; 132 len %= GHASH_BLOCK_SIZE; 133 134 ghash_do_update(blocks, ctx->digest, src, key, 135 partial ? ctx->buf : NULL); 136 src += blocks * GHASH_BLOCK_SIZE; 137 partial = 0; 138 } 139 if (len) 140 memcpy(ctx->buf + partial, src, len); 141 return 0; 142 } 143 144 static int ghash_final(struct shash_desc *desc, u8 *dst) 145 { 146 struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); 147 unsigned int partial = ctx->count % GHASH_BLOCK_SIZE; 148 149 if (partial) { 150 struct ghash_key *key = crypto_shash_ctx(desc->tfm); 151 152 memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial); 153 ghash_do_update(1, ctx->digest, ctx->buf, key, NULL); 154 } 155 put_unaligned_be64(ctx->digest[1], dst); 156 put_unaligned_be64(ctx->digest[0], dst + 8); 157 158 *ctx = (struct ghash_desc_ctx){}; 159 return 0; 160 } 161 162 static void ghash_reflect(u64 h[], const be128 *k) 163 { 164 u64 carry = be64_to_cpu(k->a) >> 63; 165 166 h[0] = (be64_to_cpu(k->b) << 1) | carry; 167 h[1] = (be64_to_cpu(k->a) << 1) | (be64_to_cpu(k->b) >> 63); 168 169 if (carry) 170 h[1] ^= 0xc200000000000000UL; 171 } 172 173 static int ghash_setkey(struct crypto_shash *tfm, 174 const u8 *inkey, unsigned int keylen) 175 { 176 struct ghash_key *key = crypto_shash_ctx(tfm); 177 178 if (keylen != GHASH_BLOCK_SIZE) 179 return -EINVAL; 180 181 /* needed for the fallback */ 182 memcpy(&key->k, inkey, GHASH_BLOCK_SIZE); 183 ghash_reflect(key->h[0], &key->k); 184 185 if (static_branch_likely(&use_p64)) { 186 be128 h = key->k; 187 188 gf128mul_lle(&h, &key->k); 189 ghash_reflect(key->h[1], &h); 190 191 gf128mul_lle(&h, &key->k); 192 ghash_reflect(key->h[2], &h); 193 194 gf128mul_lle(&h, &key->k); 195 ghash_reflect(key->h[3], &h); 196 } 197 return 0; 198 } 199 200 static struct shash_alg ghash_alg = { 201 .digestsize = GHASH_DIGEST_SIZE, 202 .init = ghash_init, 203 .update = ghash_update, 204 .final = ghash_final, 205 .setkey = ghash_setkey, 206 .descsize = sizeof(struct ghash_desc_ctx), 207 208 .base.cra_name = "ghash", 209 .base.cra_driver_name = "ghash-ce-sync", 210 .base.cra_priority = 300 - 1, 211 .base.cra_blocksize = GHASH_BLOCK_SIZE, 212 .base.cra_ctxsize = sizeof(struct ghash_key) + sizeof(u64[2]), 213 .base.cra_module = THIS_MODULE, 214 }; 215 216 static int ghash_async_init(struct ahash_request *req) 217 { 218 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 219 struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); 220 struct ahash_request *cryptd_req = ahash_request_ctx(req); 221 struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; 222 struct shash_desc *desc = cryptd_shash_desc(cryptd_req); 223 struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm); 224 225 desc->tfm = child; 226 return crypto_shash_init(desc); 227 } 228 229 static int ghash_async_update(struct ahash_request *req) 230 { 231 struct ahash_request *cryptd_req = ahash_request_ctx(req); 232 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 233 struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); 234 struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; 235 236 if (!crypto_simd_usable() || 237 (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { 238 memcpy(cryptd_req, req, sizeof(*req)); 239 ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); 240 return crypto_ahash_update(cryptd_req); 241 } else { 242 struct shash_desc *desc = cryptd_shash_desc(cryptd_req); 243 return shash_ahash_update(req, desc); 244 } 245 } 246 247 static int ghash_async_final(struct ahash_request *req) 248 { 249 struct ahash_request *cryptd_req = ahash_request_ctx(req); 250 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 251 struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); 252 struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; 253 254 if (!crypto_simd_usable() || 255 (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { 256 memcpy(cryptd_req, req, sizeof(*req)); 257 ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); 258 return crypto_ahash_final(cryptd_req); 259 } else { 260 struct shash_desc *desc = cryptd_shash_desc(cryptd_req); 261 return crypto_shash_final(desc, req->result); 262 } 263 } 264 265 static int ghash_async_digest(struct ahash_request *req) 266 { 267 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 268 struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); 269 struct ahash_request *cryptd_req = ahash_request_ctx(req); 270 struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; 271 272 if (!crypto_simd_usable() || 273 (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { 274 memcpy(cryptd_req, req, sizeof(*req)); 275 ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); 276 return crypto_ahash_digest(cryptd_req); 277 } else { 278 struct shash_desc *desc = cryptd_shash_desc(cryptd_req); 279 struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm); 280 281 desc->tfm = child; 282 return shash_ahash_digest(req, desc); 283 } 284 } 285 286 static int ghash_async_import(struct ahash_request *req, const void *in) 287 { 288 struct ahash_request *cryptd_req = ahash_request_ctx(req); 289 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 290 struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); 291 struct shash_desc *desc = cryptd_shash_desc(cryptd_req); 292 293 desc->tfm = cryptd_ahash_child(ctx->cryptd_tfm); 294 295 return crypto_shash_import(desc, in); 296 } 297 298 static int ghash_async_export(struct ahash_request *req, void *out) 299 { 300 struct ahash_request *cryptd_req = ahash_request_ctx(req); 301 struct shash_desc *desc = cryptd_shash_desc(cryptd_req); 302 303 return crypto_shash_export(desc, out); 304 } 305 306 static int ghash_async_setkey(struct crypto_ahash *tfm, const u8 *key, 307 unsigned int keylen) 308 { 309 struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); 310 struct crypto_ahash *child = &ctx->cryptd_tfm->base; 311 312 crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK); 313 crypto_ahash_set_flags(child, crypto_ahash_get_flags(tfm) 314 & CRYPTO_TFM_REQ_MASK); 315 return crypto_ahash_setkey(child, key, keylen); 316 } 317 318 static int ghash_async_init_tfm(struct crypto_tfm *tfm) 319 { 320 struct cryptd_ahash *cryptd_tfm; 321 struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm); 322 323 cryptd_tfm = cryptd_alloc_ahash("ghash-ce-sync", 0, 0); 324 if (IS_ERR(cryptd_tfm)) 325 return PTR_ERR(cryptd_tfm); 326 ctx->cryptd_tfm = cryptd_tfm; 327 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 328 sizeof(struct ahash_request) + 329 crypto_ahash_reqsize(&cryptd_tfm->base)); 330 331 return 0; 332 } 333 334 static void ghash_async_exit_tfm(struct crypto_tfm *tfm) 335 { 336 struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm); 337 338 cryptd_free_ahash(ctx->cryptd_tfm); 339 } 340 341 static struct ahash_alg ghash_async_alg = { 342 .init = ghash_async_init, 343 .update = ghash_async_update, 344 .final = ghash_async_final, 345 .setkey = ghash_async_setkey, 346 .digest = ghash_async_digest, 347 .import = ghash_async_import, 348 .export = ghash_async_export, 349 .halg.digestsize = GHASH_DIGEST_SIZE, 350 .halg.statesize = sizeof(struct ghash_desc_ctx), 351 .halg.base = { 352 .cra_name = "ghash", 353 .cra_driver_name = "ghash-ce", 354 .cra_priority = 300, 355 .cra_flags = CRYPTO_ALG_ASYNC, 356 .cra_blocksize = GHASH_BLOCK_SIZE, 357 .cra_ctxsize = sizeof(struct ghash_async_ctx), 358 .cra_module = THIS_MODULE, 359 .cra_init = ghash_async_init_tfm, 360 .cra_exit = ghash_async_exit_tfm, 361 }, 362 }; 363 364 365 void pmull_gcm_encrypt(int blocks, u64 dg[], const char *src, 366 struct gcm_key const *k, char *dst, 367 const char *iv, int rounds, u32 counter); 368 369 void pmull_gcm_enc_final(int blocks, u64 dg[], char *tag, 370 struct gcm_key const *k, char *head, 371 const char *iv, int rounds, u32 counter); 372 373 void pmull_gcm_decrypt(int bytes, u64 dg[], const char *src, 374 struct gcm_key const *k, char *dst, 375 const char *iv, int rounds, u32 counter); 376 377 int pmull_gcm_dec_final(int bytes, u64 dg[], char *tag, 378 struct gcm_key const *k, char *head, 379 const char *iv, int rounds, u32 counter, 380 const char *otag, int authsize); 381 382 static int gcm_aes_setkey(struct crypto_aead *tfm, const u8 *inkey, 383 unsigned int keylen) 384 { 385 struct gcm_key *ctx = crypto_aead_ctx(tfm); 386 struct crypto_aes_ctx aes_ctx; 387 be128 h, k; 388 int ret; 389 390 ret = aes_expandkey(&aes_ctx, inkey, keylen); 391 if (ret) 392 return -EINVAL; 393 394 aes_encrypt(&aes_ctx, (u8 *)&k, (u8[AES_BLOCK_SIZE]){}); 395 396 memcpy(ctx->rk, aes_ctx.key_enc, sizeof(ctx->rk)); 397 ctx->rounds = 6 + keylen / 4; 398 399 memzero_explicit(&aes_ctx, sizeof(aes_ctx)); 400 401 ghash_reflect(ctx->h[0], &k); 402 403 h = k; 404 gf128mul_lle(&h, &k); 405 ghash_reflect(ctx->h[1], &h); 406 407 gf128mul_lle(&h, &k); 408 ghash_reflect(ctx->h[2], &h); 409 410 gf128mul_lle(&h, &k); 411 ghash_reflect(ctx->h[3], &h); 412 413 return 0; 414 } 415 416 static int gcm_aes_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 417 { 418 return crypto_gcm_check_authsize(authsize); 419 } 420 421 static void gcm_update_mac(u64 dg[], const u8 *src, int count, u8 buf[], 422 int *buf_count, struct gcm_key *ctx) 423 { 424 if (*buf_count > 0) { 425 int buf_added = min(count, GHASH_BLOCK_SIZE - *buf_count); 426 427 memcpy(&buf[*buf_count], src, buf_added); 428 429 *buf_count += buf_added; 430 src += buf_added; 431 count -= buf_added; 432 } 433 434 if (count >= GHASH_BLOCK_SIZE || *buf_count == GHASH_BLOCK_SIZE) { 435 int blocks = count / GHASH_BLOCK_SIZE; 436 437 pmull_ghash_update_p64(blocks, dg, src, ctx->h, 438 *buf_count ? buf : NULL); 439 440 src += blocks * GHASH_BLOCK_SIZE; 441 count %= GHASH_BLOCK_SIZE; 442 *buf_count = 0; 443 } 444 445 if (count > 0) { 446 memcpy(buf, src, count); 447 *buf_count = count; 448 } 449 } 450 451 static void gcm_calculate_auth_mac(struct aead_request *req, u64 dg[], u32 len) 452 { 453 struct crypto_aead *aead = crypto_aead_reqtfm(req); 454 struct gcm_key *ctx = crypto_aead_ctx(aead); 455 u8 buf[GHASH_BLOCK_SIZE]; 456 struct scatter_walk walk; 457 int buf_count = 0; 458 459 scatterwalk_start(&walk, req->src); 460 461 do { 462 unsigned int n; 463 464 n = scatterwalk_next(&walk, len); 465 gcm_update_mac(dg, walk.addr, n, buf, &buf_count, ctx); 466 scatterwalk_done_src(&walk, n); 467 468 if (unlikely(len / SZ_4K > (len - n) / SZ_4K)) { 469 kernel_neon_end(); 470 kernel_neon_begin(); 471 } 472 473 len -= n; 474 } while (len); 475 476 if (buf_count) { 477 memset(&buf[buf_count], 0, GHASH_BLOCK_SIZE - buf_count); 478 pmull_ghash_update_p64(1, dg, buf, ctx->h, NULL); 479 } 480 } 481 482 static int gcm_encrypt(struct aead_request *req, const u8 *iv, u32 assoclen) 483 { 484 struct crypto_aead *aead = crypto_aead_reqtfm(req); 485 struct gcm_key *ctx = crypto_aead_ctx(aead); 486 struct skcipher_walk walk; 487 u8 buf[AES_BLOCK_SIZE]; 488 u32 counter = 2; 489 u64 dg[2] = {}; 490 be128 lengths; 491 const u8 *src; 492 u8 *tag, *dst; 493 int tail, err; 494 495 if (WARN_ON_ONCE(!may_use_simd())) 496 return -EBUSY; 497 498 err = skcipher_walk_aead_encrypt(&walk, req, false); 499 500 kernel_neon_begin(); 501 502 if (assoclen) 503 gcm_calculate_auth_mac(req, dg, assoclen); 504 505 src = walk.src.virt.addr; 506 dst = walk.dst.virt.addr; 507 508 while (walk.nbytes >= AES_BLOCK_SIZE) { 509 int nblocks = walk.nbytes / AES_BLOCK_SIZE; 510 511 pmull_gcm_encrypt(nblocks, dg, src, ctx, dst, iv, 512 ctx->rounds, counter); 513 counter += nblocks; 514 515 if (walk.nbytes == walk.total) { 516 src += nblocks * AES_BLOCK_SIZE; 517 dst += nblocks * AES_BLOCK_SIZE; 518 break; 519 } 520 521 kernel_neon_end(); 522 523 err = skcipher_walk_done(&walk, 524 walk.nbytes % AES_BLOCK_SIZE); 525 if (err) 526 return err; 527 528 src = walk.src.virt.addr; 529 dst = walk.dst.virt.addr; 530 531 kernel_neon_begin(); 532 } 533 534 535 lengths.a = cpu_to_be64(assoclen * 8); 536 lengths.b = cpu_to_be64(req->cryptlen * 8); 537 538 tag = (u8 *)&lengths; 539 tail = walk.nbytes % AES_BLOCK_SIZE; 540 541 /* 542 * Bounce via a buffer unless we are encrypting in place and src/dst 543 * are not pointing to the start of the walk buffer. In that case, we 544 * can do a NEON load/xor/store sequence in place as long as we move 545 * the plain/ciphertext and keystream to the start of the register. If 546 * not, do a memcpy() to the end of the buffer so we can reuse the same 547 * logic. 548 */ 549 if (unlikely(tail && (tail == walk.nbytes || src != dst))) 550 src = memcpy(buf + sizeof(buf) - tail, src, tail); 551 552 pmull_gcm_enc_final(tail, dg, tag, ctx, (u8 *)src, iv, 553 ctx->rounds, counter); 554 kernel_neon_end(); 555 556 if (unlikely(tail && src != dst)) 557 memcpy(dst, src, tail); 558 559 if (walk.nbytes) { 560 err = skcipher_walk_done(&walk, 0); 561 if (err) 562 return err; 563 } 564 565 /* copy authtag to end of dst */ 566 scatterwalk_map_and_copy(tag, req->dst, req->assoclen + req->cryptlen, 567 crypto_aead_authsize(aead), 1); 568 569 return 0; 570 } 571 572 static int gcm_decrypt(struct aead_request *req, const u8 *iv, u32 assoclen) 573 { 574 struct crypto_aead *aead = crypto_aead_reqtfm(req); 575 struct gcm_key *ctx = crypto_aead_ctx(aead); 576 int authsize = crypto_aead_authsize(aead); 577 struct skcipher_walk walk; 578 u8 otag[AES_BLOCK_SIZE]; 579 u8 buf[AES_BLOCK_SIZE]; 580 u32 counter = 2; 581 u64 dg[2] = {}; 582 be128 lengths; 583 const u8 *src; 584 u8 *tag, *dst; 585 int tail, err, ret; 586 587 if (WARN_ON_ONCE(!may_use_simd())) 588 return -EBUSY; 589 590 scatterwalk_map_and_copy(otag, req->src, 591 req->assoclen + req->cryptlen - authsize, 592 authsize, 0); 593 594 err = skcipher_walk_aead_decrypt(&walk, req, false); 595 596 kernel_neon_begin(); 597 598 if (assoclen) 599 gcm_calculate_auth_mac(req, dg, assoclen); 600 601 src = walk.src.virt.addr; 602 dst = walk.dst.virt.addr; 603 604 while (walk.nbytes >= AES_BLOCK_SIZE) { 605 int nblocks = walk.nbytes / AES_BLOCK_SIZE; 606 607 pmull_gcm_decrypt(nblocks, dg, src, ctx, dst, iv, 608 ctx->rounds, counter); 609 counter += nblocks; 610 611 if (walk.nbytes == walk.total) { 612 src += nblocks * AES_BLOCK_SIZE; 613 dst += nblocks * AES_BLOCK_SIZE; 614 break; 615 } 616 617 kernel_neon_end(); 618 619 err = skcipher_walk_done(&walk, 620 walk.nbytes % AES_BLOCK_SIZE); 621 if (err) 622 return err; 623 624 src = walk.src.virt.addr; 625 dst = walk.dst.virt.addr; 626 627 kernel_neon_begin(); 628 } 629 630 lengths.a = cpu_to_be64(assoclen * 8); 631 lengths.b = cpu_to_be64((req->cryptlen - authsize) * 8); 632 633 tag = (u8 *)&lengths; 634 tail = walk.nbytes % AES_BLOCK_SIZE; 635 636 if (unlikely(tail && (tail == walk.nbytes || src != dst))) 637 src = memcpy(buf + sizeof(buf) - tail, src, tail); 638 639 ret = pmull_gcm_dec_final(tail, dg, tag, ctx, (u8 *)src, iv, 640 ctx->rounds, counter, otag, authsize); 641 kernel_neon_end(); 642 643 if (unlikely(tail && src != dst)) 644 memcpy(dst, src, tail); 645 646 if (walk.nbytes) { 647 err = skcipher_walk_done(&walk, 0); 648 if (err) 649 return err; 650 } 651 652 return ret ? -EBADMSG : 0; 653 } 654 655 static int gcm_aes_encrypt(struct aead_request *req) 656 { 657 return gcm_encrypt(req, req->iv, req->assoclen); 658 } 659 660 static int gcm_aes_decrypt(struct aead_request *req) 661 { 662 return gcm_decrypt(req, req->iv, req->assoclen); 663 } 664 665 static int rfc4106_setkey(struct crypto_aead *tfm, const u8 *inkey, 666 unsigned int keylen) 667 { 668 struct gcm_key *ctx = crypto_aead_ctx(tfm); 669 int err; 670 671 keylen -= RFC4106_NONCE_SIZE; 672 err = gcm_aes_setkey(tfm, inkey, keylen); 673 if (err) 674 return err; 675 676 memcpy(ctx->nonce, inkey + keylen, RFC4106_NONCE_SIZE); 677 return 0; 678 } 679 680 static int rfc4106_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 681 { 682 return crypto_rfc4106_check_authsize(authsize); 683 } 684 685 static int rfc4106_encrypt(struct aead_request *req) 686 { 687 struct crypto_aead *aead = crypto_aead_reqtfm(req); 688 struct gcm_key *ctx = crypto_aead_ctx(aead); 689 u8 iv[GCM_AES_IV_SIZE]; 690 691 memcpy(iv, ctx->nonce, RFC4106_NONCE_SIZE); 692 memcpy(iv + RFC4106_NONCE_SIZE, req->iv, GCM_RFC4106_IV_SIZE); 693 694 return crypto_ipsec_check_assoclen(req->assoclen) ?: 695 gcm_encrypt(req, iv, req->assoclen - GCM_RFC4106_IV_SIZE); 696 } 697 698 static int rfc4106_decrypt(struct aead_request *req) 699 { 700 struct crypto_aead *aead = crypto_aead_reqtfm(req); 701 struct gcm_key *ctx = crypto_aead_ctx(aead); 702 u8 iv[GCM_AES_IV_SIZE]; 703 704 memcpy(iv, ctx->nonce, RFC4106_NONCE_SIZE); 705 memcpy(iv + RFC4106_NONCE_SIZE, req->iv, GCM_RFC4106_IV_SIZE); 706 707 return crypto_ipsec_check_assoclen(req->assoclen) ?: 708 gcm_decrypt(req, iv, req->assoclen - GCM_RFC4106_IV_SIZE); 709 } 710 711 static struct aead_alg gcm_aes_algs[] = {{ 712 .ivsize = GCM_AES_IV_SIZE, 713 .chunksize = AES_BLOCK_SIZE, 714 .maxauthsize = AES_BLOCK_SIZE, 715 .setkey = gcm_aes_setkey, 716 .setauthsize = gcm_aes_setauthsize, 717 .encrypt = gcm_aes_encrypt, 718 .decrypt = gcm_aes_decrypt, 719 720 .base.cra_name = "gcm(aes)", 721 .base.cra_driver_name = "gcm-aes-ce", 722 .base.cra_priority = 400, 723 .base.cra_blocksize = 1, 724 .base.cra_ctxsize = sizeof(struct gcm_key), 725 .base.cra_module = THIS_MODULE, 726 }, { 727 .ivsize = GCM_RFC4106_IV_SIZE, 728 .chunksize = AES_BLOCK_SIZE, 729 .maxauthsize = AES_BLOCK_SIZE, 730 .setkey = rfc4106_setkey, 731 .setauthsize = rfc4106_setauthsize, 732 .encrypt = rfc4106_encrypt, 733 .decrypt = rfc4106_decrypt, 734 735 .base.cra_name = "rfc4106(gcm(aes))", 736 .base.cra_driver_name = "rfc4106-gcm-aes-ce", 737 .base.cra_priority = 400, 738 .base.cra_blocksize = 1, 739 .base.cra_ctxsize = sizeof(struct gcm_key) + RFC4106_NONCE_SIZE, 740 .base.cra_module = THIS_MODULE, 741 }}; 742 743 static int __init ghash_ce_mod_init(void) 744 { 745 int err; 746 747 if (!(elf_hwcap & HWCAP_NEON)) 748 return -ENODEV; 749 750 if (elf_hwcap2 & HWCAP2_PMULL) { 751 err = crypto_register_aeads(gcm_aes_algs, 752 ARRAY_SIZE(gcm_aes_algs)); 753 if (err) 754 return err; 755 ghash_alg.base.cra_ctxsize += 3 * sizeof(u64[2]); 756 static_branch_enable(&use_p64); 757 } 758 759 err = crypto_register_shash(&ghash_alg); 760 if (err) 761 goto err_aead; 762 err = crypto_register_ahash(&ghash_async_alg); 763 if (err) 764 goto err_shash; 765 766 return 0; 767 768 err_shash: 769 crypto_unregister_shash(&ghash_alg); 770 err_aead: 771 if (elf_hwcap2 & HWCAP2_PMULL) 772 crypto_unregister_aeads(gcm_aes_algs, 773 ARRAY_SIZE(gcm_aes_algs)); 774 return err; 775 } 776 777 static void __exit ghash_ce_mod_exit(void) 778 { 779 crypto_unregister_ahash(&ghash_async_alg); 780 crypto_unregister_shash(&ghash_alg); 781 if (elf_hwcap2 & HWCAP2_PMULL) 782 crypto_unregister_aeads(gcm_aes_algs, 783 ARRAY_SIZE(gcm_aes_algs)); 784 } 785 786 module_init(ghash_ce_mod_init); 787 module_exit(ghash_ce_mod_exit); 788