1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Accelerated GHASH implementation with ARMv8 PMULL instructions. 4 * 5 * Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org> 6 */ 7 8 #include <asm/neon.h> 9 #include <crypto/aes.h> 10 #include <crypto/b128ops.h> 11 #include <crypto/gcm.h> 12 #include <crypto/ghash.h> 13 #include <crypto/gf128mul.h> 14 #include <crypto/internal/aead.h> 15 #include <crypto/internal/hash.h> 16 #include <crypto/internal/skcipher.h> 17 #include <crypto/scatterwalk.h> 18 #include <linux/cpufeature.h> 19 #include <linux/errno.h> 20 #include <linux/kernel.h> 21 #include <linux/module.h> 22 #include <linux/string.h> 23 #include <linux/unaligned.h> 24 25 MODULE_DESCRIPTION("GHASH and AES-GCM using ARMv8 Crypto Extensions"); 26 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 27 MODULE_LICENSE("GPL v2"); 28 MODULE_ALIAS_CRYPTO("ghash"); 29 30 #define RFC4106_NONCE_SIZE 4 31 32 struct ghash_key { 33 be128 k; 34 u64 h[][2]; 35 }; 36 37 struct arm_ghash_desc_ctx { 38 u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)]; 39 }; 40 41 struct gcm_aes_ctx { 42 struct crypto_aes_ctx aes_key; 43 u8 nonce[RFC4106_NONCE_SIZE]; 44 struct ghash_key ghash_key; 45 }; 46 47 asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src, 48 u64 const h[][2], const char *head); 49 50 asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src, 51 u64 const h[][2], const char *head); 52 53 asmlinkage void pmull_gcm_encrypt(int bytes, u8 dst[], const u8 src[], 54 u64 const h[][2], u64 dg[], u8 ctr[], 55 u32 const rk[], int rounds, u8 tag[]); 56 asmlinkage int pmull_gcm_decrypt(int bytes, u8 dst[], const u8 src[], 57 u64 const h[][2], u64 dg[], u8 ctr[], 58 u32 const rk[], int rounds, const u8 l[], 59 const u8 tag[], u64 authsize); 60 61 static int ghash_init(struct shash_desc *desc) 62 { 63 struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); 64 65 *ctx = (struct arm_ghash_desc_ctx){}; 66 return 0; 67 } 68 69 static __always_inline 70 void ghash_do_simd_update(int blocks, u64 dg[], const char *src, 71 struct ghash_key *key, const char *head, 72 void (*simd_update)(int blocks, u64 dg[], 73 const char *src, 74 u64 const h[][2], 75 const char *head)) 76 { 77 kernel_neon_begin(); 78 simd_update(blocks, dg, src, key->h, head); 79 kernel_neon_end(); 80 } 81 82 /* avoid hogging the CPU for too long */ 83 #define MAX_BLOCKS (SZ_64K / GHASH_BLOCK_SIZE) 84 85 static int ghash_update(struct shash_desc *desc, const u8 *src, 86 unsigned int len) 87 { 88 struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); 89 struct ghash_key *key = crypto_shash_ctx(desc->tfm); 90 int blocks; 91 92 blocks = len / GHASH_BLOCK_SIZE; 93 len -= blocks * GHASH_BLOCK_SIZE; 94 95 do { 96 int chunk = min(blocks, MAX_BLOCKS); 97 98 ghash_do_simd_update(chunk, ctx->digest, src, key, NULL, 99 pmull_ghash_update_p8); 100 blocks -= chunk; 101 src += chunk * GHASH_BLOCK_SIZE; 102 } while (unlikely(blocks > 0)); 103 return len; 104 } 105 106 static int ghash_export(struct shash_desc *desc, void *out) 107 { 108 struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); 109 u8 *dst = out; 110 111 put_unaligned_be64(ctx->digest[1], dst); 112 put_unaligned_be64(ctx->digest[0], dst + 8); 113 return 0; 114 } 115 116 static int ghash_import(struct shash_desc *desc, const void *in) 117 { 118 struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); 119 const u8 *src = in; 120 121 ctx->digest[1] = get_unaligned_be64(src); 122 ctx->digest[0] = get_unaligned_be64(src + 8); 123 return 0; 124 } 125 126 static int ghash_finup(struct shash_desc *desc, const u8 *src, 127 unsigned int len, u8 *dst) 128 { 129 struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); 130 struct ghash_key *key = crypto_shash_ctx(desc->tfm); 131 132 if (len) { 133 u8 buf[GHASH_BLOCK_SIZE] = {}; 134 135 memcpy(buf, src, len); 136 ghash_do_simd_update(1, ctx->digest, src, key, NULL, 137 pmull_ghash_update_p8); 138 memzero_explicit(buf, sizeof(buf)); 139 } 140 return ghash_export(desc, dst); 141 } 142 143 static void ghash_reflect(u64 h[], const be128 *k) 144 { 145 u64 carry = be64_to_cpu(k->a) & BIT(63) ? 1 : 0; 146 147 h[0] = (be64_to_cpu(k->b) << 1) | carry; 148 h[1] = (be64_to_cpu(k->a) << 1) | (be64_to_cpu(k->b) >> 63); 149 150 if (carry) 151 h[1] ^= 0xc200000000000000UL; 152 } 153 154 static int ghash_setkey(struct crypto_shash *tfm, 155 const u8 *inkey, unsigned int keylen) 156 { 157 struct ghash_key *key = crypto_shash_ctx(tfm); 158 159 if (keylen != GHASH_BLOCK_SIZE) 160 return -EINVAL; 161 162 /* needed for the fallback */ 163 memcpy(&key->k, inkey, GHASH_BLOCK_SIZE); 164 165 ghash_reflect(key->h[0], &key->k); 166 return 0; 167 } 168 169 static struct shash_alg ghash_alg = { 170 .base.cra_name = "ghash", 171 .base.cra_driver_name = "ghash-neon", 172 .base.cra_priority = 150, 173 .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, 174 .base.cra_blocksize = GHASH_BLOCK_SIZE, 175 .base.cra_ctxsize = sizeof(struct ghash_key) + sizeof(u64[2]), 176 .base.cra_module = THIS_MODULE, 177 178 .digestsize = GHASH_DIGEST_SIZE, 179 .init = ghash_init, 180 .update = ghash_update, 181 .finup = ghash_finup, 182 .setkey = ghash_setkey, 183 .export = ghash_export, 184 .import = ghash_import, 185 .descsize = sizeof(struct arm_ghash_desc_ctx), 186 .statesize = sizeof(struct ghash_desc_ctx), 187 }; 188 189 static int num_rounds(struct crypto_aes_ctx *ctx) 190 { 191 /* 192 * # of rounds specified by AES: 193 * 128 bit key 10 rounds 194 * 192 bit key 12 rounds 195 * 256 bit key 14 rounds 196 * => n byte key => 6 + (n/4) rounds 197 */ 198 return 6 + ctx->key_length / 4; 199 } 200 201 static int gcm_aes_setkey(struct crypto_aead *tfm, const u8 *inkey, 202 unsigned int keylen) 203 { 204 struct gcm_aes_ctx *ctx = crypto_aead_ctx(tfm); 205 u8 key[GHASH_BLOCK_SIZE]; 206 be128 h; 207 int ret; 208 209 ret = aes_expandkey(&ctx->aes_key, inkey, keylen); 210 if (ret) 211 return -EINVAL; 212 213 aes_encrypt(&ctx->aes_key, key, (u8[AES_BLOCK_SIZE]){}); 214 215 /* needed for the fallback */ 216 memcpy(&ctx->ghash_key.k, key, GHASH_BLOCK_SIZE); 217 218 ghash_reflect(ctx->ghash_key.h[0], &ctx->ghash_key.k); 219 220 h = ctx->ghash_key.k; 221 gf128mul_lle(&h, &ctx->ghash_key.k); 222 ghash_reflect(ctx->ghash_key.h[1], &h); 223 224 gf128mul_lle(&h, &ctx->ghash_key.k); 225 ghash_reflect(ctx->ghash_key.h[2], &h); 226 227 gf128mul_lle(&h, &ctx->ghash_key.k); 228 ghash_reflect(ctx->ghash_key.h[3], &h); 229 230 return 0; 231 } 232 233 static int gcm_aes_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 234 { 235 return crypto_gcm_check_authsize(authsize); 236 } 237 238 static void gcm_update_mac(u64 dg[], const u8 *src, int count, u8 buf[], 239 int *buf_count, struct gcm_aes_ctx *ctx) 240 { 241 if (*buf_count > 0) { 242 int buf_added = min(count, GHASH_BLOCK_SIZE - *buf_count); 243 244 memcpy(&buf[*buf_count], src, buf_added); 245 246 *buf_count += buf_added; 247 src += buf_added; 248 count -= buf_added; 249 } 250 251 if (count >= GHASH_BLOCK_SIZE || *buf_count == GHASH_BLOCK_SIZE) { 252 int blocks = count / GHASH_BLOCK_SIZE; 253 254 ghash_do_simd_update(blocks, dg, src, &ctx->ghash_key, 255 *buf_count ? buf : NULL, 256 pmull_ghash_update_p64); 257 258 src += blocks * GHASH_BLOCK_SIZE; 259 count %= GHASH_BLOCK_SIZE; 260 *buf_count = 0; 261 } 262 263 if (count > 0) { 264 memcpy(buf, src, count); 265 *buf_count = count; 266 } 267 } 268 269 static void gcm_calculate_auth_mac(struct aead_request *req, u64 dg[], u32 len) 270 { 271 struct crypto_aead *aead = crypto_aead_reqtfm(req); 272 struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead); 273 u8 buf[GHASH_BLOCK_SIZE]; 274 struct scatter_walk walk; 275 int buf_count = 0; 276 277 scatterwalk_start(&walk, req->src); 278 279 do { 280 unsigned int n; 281 282 n = scatterwalk_next(&walk, len); 283 gcm_update_mac(dg, walk.addr, n, buf, &buf_count, ctx); 284 scatterwalk_done_src(&walk, n); 285 len -= n; 286 } while (len); 287 288 if (buf_count) { 289 memset(&buf[buf_count], 0, GHASH_BLOCK_SIZE - buf_count); 290 ghash_do_simd_update(1, dg, buf, &ctx->ghash_key, NULL, 291 pmull_ghash_update_p64); 292 } 293 } 294 295 static int gcm_encrypt(struct aead_request *req, char *iv, int assoclen) 296 { 297 struct crypto_aead *aead = crypto_aead_reqtfm(req); 298 struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead); 299 int nrounds = num_rounds(&ctx->aes_key); 300 struct skcipher_walk walk; 301 u8 buf[AES_BLOCK_SIZE]; 302 u64 dg[2] = {}; 303 be128 lengths; 304 u8 *tag; 305 int err; 306 307 lengths.a = cpu_to_be64(assoclen * 8); 308 lengths.b = cpu_to_be64(req->cryptlen * 8); 309 310 if (assoclen) 311 gcm_calculate_auth_mac(req, dg, assoclen); 312 313 put_unaligned_be32(2, iv + GCM_AES_IV_SIZE); 314 315 err = skcipher_walk_aead_encrypt(&walk, req, false); 316 317 do { 318 const u8 *src = walk.src.virt.addr; 319 u8 *dst = walk.dst.virt.addr; 320 int nbytes = walk.nbytes; 321 322 tag = (u8 *)&lengths; 323 324 if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) { 325 src = dst = memcpy(buf + sizeof(buf) - nbytes, 326 src, nbytes); 327 } else if (nbytes < walk.total) { 328 nbytes &= ~(AES_BLOCK_SIZE - 1); 329 tag = NULL; 330 } 331 332 kernel_neon_begin(); 333 pmull_gcm_encrypt(nbytes, dst, src, ctx->ghash_key.h, 334 dg, iv, ctx->aes_key.key_enc, nrounds, 335 tag); 336 kernel_neon_end(); 337 338 if (unlikely(!nbytes)) 339 break; 340 341 if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) 342 memcpy(walk.dst.virt.addr, 343 buf + sizeof(buf) - nbytes, nbytes); 344 345 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 346 } while (walk.nbytes); 347 348 if (err) 349 return err; 350 351 /* copy authtag to end of dst */ 352 scatterwalk_map_and_copy(tag, req->dst, req->assoclen + req->cryptlen, 353 crypto_aead_authsize(aead), 1); 354 355 return 0; 356 } 357 358 static int gcm_decrypt(struct aead_request *req, char *iv, int assoclen) 359 { 360 struct crypto_aead *aead = crypto_aead_reqtfm(req); 361 struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead); 362 unsigned int authsize = crypto_aead_authsize(aead); 363 int nrounds = num_rounds(&ctx->aes_key); 364 struct skcipher_walk walk; 365 u8 otag[AES_BLOCK_SIZE]; 366 u8 buf[AES_BLOCK_SIZE]; 367 u64 dg[2] = {}; 368 be128 lengths; 369 u8 *tag; 370 int ret; 371 int err; 372 373 lengths.a = cpu_to_be64(assoclen * 8); 374 lengths.b = cpu_to_be64((req->cryptlen - authsize) * 8); 375 376 if (assoclen) 377 gcm_calculate_auth_mac(req, dg, assoclen); 378 379 put_unaligned_be32(2, iv + GCM_AES_IV_SIZE); 380 381 scatterwalk_map_and_copy(otag, req->src, 382 req->assoclen + req->cryptlen - authsize, 383 authsize, 0); 384 385 err = skcipher_walk_aead_decrypt(&walk, req, false); 386 387 do { 388 const u8 *src = walk.src.virt.addr; 389 u8 *dst = walk.dst.virt.addr; 390 int nbytes = walk.nbytes; 391 392 tag = (u8 *)&lengths; 393 394 if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) { 395 src = dst = memcpy(buf + sizeof(buf) - nbytes, 396 src, nbytes); 397 } else if (nbytes < walk.total) { 398 nbytes &= ~(AES_BLOCK_SIZE - 1); 399 tag = NULL; 400 } 401 402 kernel_neon_begin(); 403 ret = pmull_gcm_decrypt(nbytes, dst, src, ctx->ghash_key.h, 404 dg, iv, ctx->aes_key.key_enc, 405 nrounds, tag, otag, authsize); 406 kernel_neon_end(); 407 408 if (unlikely(!nbytes)) 409 break; 410 411 if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) 412 memcpy(walk.dst.virt.addr, 413 buf + sizeof(buf) - nbytes, nbytes); 414 415 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 416 } while (walk.nbytes); 417 418 if (err) 419 return err; 420 421 return ret ? -EBADMSG : 0; 422 } 423 424 static int gcm_aes_encrypt(struct aead_request *req) 425 { 426 u8 iv[AES_BLOCK_SIZE]; 427 428 memcpy(iv, req->iv, GCM_AES_IV_SIZE); 429 return gcm_encrypt(req, iv, req->assoclen); 430 } 431 432 static int gcm_aes_decrypt(struct aead_request *req) 433 { 434 u8 iv[AES_BLOCK_SIZE]; 435 436 memcpy(iv, req->iv, GCM_AES_IV_SIZE); 437 return gcm_decrypt(req, iv, req->assoclen); 438 } 439 440 static int rfc4106_setkey(struct crypto_aead *tfm, const u8 *inkey, 441 unsigned int keylen) 442 { 443 struct gcm_aes_ctx *ctx = crypto_aead_ctx(tfm); 444 int err; 445 446 keylen -= RFC4106_NONCE_SIZE; 447 err = gcm_aes_setkey(tfm, inkey, keylen); 448 if (err) 449 return err; 450 451 memcpy(ctx->nonce, inkey + keylen, RFC4106_NONCE_SIZE); 452 return 0; 453 } 454 455 static int rfc4106_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 456 { 457 return crypto_rfc4106_check_authsize(authsize); 458 } 459 460 static int rfc4106_encrypt(struct aead_request *req) 461 { 462 struct crypto_aead *aead = crypto_aead_reqtfm(req); 463 struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead); 464 u8 iv[AES_BLOCK_SIZE]; 465 466 memcpy(iv, ctx->nonce, RFC4106_NONCE_SIZE); 467 memcpy(iv + RFC4106_NONCE_SIZE, req->iv, GCM_RFC4106_IV_SIZE); 468 469 return crypto_ipsec_check_assoclen(req->assoclen) ?: 470 gcm_encrypt(req, iv, req->assoclen - GCM_RFC4106_IV_SIZE); 471 } 472 473 static int rfc4106_decrypt(struct aead_request *req) 474 { 475 struct crypto_aead *aead = crypto_aead_reqtfm(req); 476 struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead); 477 u8 iv[AES_BLOCK_SIZE]; 478 479 memcpy(iv, ctx->nonce, RFC4106_NONCE_SIZE); 480 memcpy(iv + RFC4106_NONCE_SIZE, req->iv, GCM_RFC4106_IV_SIZE); 481 482 return crypto_ipsec_check_assoclen(req->assoclen) ?: 483 gcm_decrypt(req, iv, req->assoclen - GCM_RFC4106_IV_SIZE); 484 } 485 486 static struct aead_alg gcm_aes_algs[] = {{ 487 .ivsize = GCM_AES_IV_SIZE, 488 .chunksize = AES_BLOCK_SIZE, 489 .maxauthsize = AES_BLOCK_SIZE, 490 .setkey = gcm_aes_setkey, 491 .setauthsize = gcm_aes_setauthsize, 492 .encrypt = gcm_aes_encrypt, 493 .decrypt = gcm_aes_decrypt, 494 495 .base.cra_name = "gcm(aes)", 496 .base.cra_driver_name = "gcm-aes-ce", 497 .base.cra_priority = 300, 498 .base.cra_blocksize = 1, 499 .base.cra_ctxsize = sizeof(struct gcm_aes_ctx) + 500 4 * sizeof(u64[2]), 501 .base.cra_module = THIS_MODULE, 502 }, { 503 .ivsize = GCM_RFC4106_IV_SIZE, 504 .chunksize = AES_BLOCK_SIZE, 505 .maxauthsize = AES_BLOCK_SIZE, 506 .setkey = rfc4106_setkey, 507 .setauthsize = rfc4106_setauthsize, 508 .encrypt = rfc4106_encrypt, 509 .decrypt = rfc4106_decrypt, 510 511 .base.cra_name = "rfc4106(gcm(aes))", 512 .base.cra_driver_name = "rfc4106-gcm-aes-ce", 513 .base.cra_priority = 300, 514 .base.cra_blocksize = 1, 515 .base.cra_ctxsize = sizeof(struct gcm_aes_ctx) + 516 4 * sizeof(u64[2]), 517 .base.cra_module = THIS_MODULE, 518 }}; 519 520 static int __init ghash_ce_mod_init(void) 521 { 522 if (!cpu_have_named_feature(ASIMD)) 523 return -ENODEV; 524 525 if (cpu_have_named_feature(PMULL)) 526 return crypto_register_aeads(gcm_aes_algs, 527 ARRAY_SIZE(gcm_aes_algs)); 528 529 return crypto_register_shash(&ghash_alg); 530 } 531 532 static void __exit ghash_ce_mod_exit(void) 533 { 534 if (cpu_have_named_feature(PMULL)) 535 crypto_unregister_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs)); 536 else 537 crypto_unregister_shash(&ghash_alg); 538 } 539 540 static const struct cpu_feature __maybe_unused ghash_cpu_feature[] = { 541 { cpu_feature(PMULL) }, { } 542 }; 543 MODULE_DEVICE_TABLE(cpu, ghash_cpu_feature); 544 545 module_init(ghash_ce_mod_init); 546 module_exit(ghash_ce_mod_exit); 547