1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Accelerated GHASH implementation with ARMv8 PMULL instructions. 4 * 5 * Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org> 6 */ 7 8 #include <crypto/aes.h> 9 #include <crypto/b128ops.h> 10 #include <crypto/gcm.h> 11 #include <crypto/ghash.h> 12 #include <crypto/gf128mul.h> 13 #include <crypto/internal/aead.h> 14 #include <crypto/internal/hash.h> 15 #include <crypto/internal/skcipher.h> 16 #include <crypto/scatterwalk.h> 17 #include <linux/cpufeature.h> 18 #include <linux/errno.h> 19 #include <linux/kernel.h> 20 #include <linux/module.h> 21 #include <linux/string.h> 22 #include <linux/unaligned.h> 23 24 #include <asm/simd.h> 25 26 MODULE_DESCRIPTION("GHASH and AES-GCM using ARMv8 Crypto Extensions"); 27 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 28 MODULE_LICENSE("GPL v2"); 29 MODULE_ALIAS_CRYPTO("ghash"); 30 31 #define RFC4106_NONCE_SIZE 4 32 33 struct ghash_key { 34 be128 k; 35 u64 h[][2]; 36 }; 37 38 struct arm_ghash_desc_ctx { 39 u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)]; 40 }; 41 42 struct gcm_aes_ctx { 43 struct crypto_aes_ctx aes_key; 44 u8 nonce[RFC4106_NONCE_SIZE]; 45 struct ghash_key ghash_key; 46 }; 47 48 asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src, 49 u64 const h[][2], const char *head); 50 51 asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src, 52 u64 const h[][2], const char *head); 53 54 asmlinkage void pmull_gcm_encrypt(int bytes, u8 dst[], const u8 src[], 55 u64 const h[][2], u64 dg[], u8 ctr[], 56 u32 const rk[], int rounds, u8 tag[]); 57 asmlinkage int pmull_gcm_decrypt(int bytes, u8 dst[], const u8 src[], 58 u64 const h[][2], u64 dg[], u8 ctr[], 59 u32 const rk[], int rounds, const u8 l[], 60 const u8 tag[], u64 authsize); 61 62 static int ghash_init(struct shash_desc *desc) 63 { 64 struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); 65 66 *ctx = (struct arm_ghash_desc_ctx){}; 67 return 0; 68 } 69 70 static __always_inline 71 void ghash_do_simd_update(int blocks, u64 dg[], const char *src, 72 struct ghash_key *key, const char *head, 73 void (*simd_update)(int blocks, u64 dg[], 74 const char *src, 75 u64 const h[][2], 76 const char *head)) 77 { 78 scoped_ksimd() 79 simd_update(blocks, dg, src, key->h, head); 80 } 81 82 /* avoid hogging the CPU for too long */ 83 #define MAX_BLOCKS (SZ_64K / GHASH_BLOCK_SIZE) 84 85 static int ghash_update(struct shash_desc *desc, const u8 *src, 86 unsigned int len) 87 { 88 struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); 89 struct ghash_key *key = crypto_shash_ctx(desc->tfm); 90 int blocks; 91 92 blocks = len / GHASH_BLOCK_SIZE; 93 len -= blocks * GHASH_BLOCK_SIZE; 94 95 do { 96 int chunk = min(blocks, MAX_BLOCKS); 97 98 ghash_do_simd_update(chunk, ctx->digest, src, key, NULL, 99 pmull_ghash_update_p8); 100 blocks -= chunk; 101 src += chunk * GHASH_BLOCK_SIZE; 102 } while (unlikely(blocks > 0)); 103 return len; 104 } 105 106 static int ghash_export(struct shash_desc *desc, void *out) 107 { 108 struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); 109 u8 *dst = out; 110 111 put_unaligned_be64(ctx->digest[1], dst); 112 put_unaligned_be64(ctx->digest[0], dst + 8); 113 return 0; 114 } 115 116 static int ghash_import(struct shash_desc *desc, const void *in) 117 { 118 struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); 119 const u8 *src = in; 120 121 ctx->digest[1] = get_unaligned_be64(src); 122 ctx->digest[0] = get_unaligned_be64(src + 8); 123 return 0; 124 } 125 126 static int ghash_finup(struct shash_desc *desc, const u8 *src, 127 unsigned int len, u8 *dst) 128 { 129 struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); 130 struct ghash_key *key = crypto_shash_ctx(desc->tfm); 131 132 if (len) { 133 u8 buf[GHASH_BLOCK_SIZE] = {}; 134 135 memcpy(buf, src, len); 136 ghash_do_simd_update(1, ctx->digest, src, key, NULL, 137 pmull_ghash_update_p8); 138 memzero_explicit(buf, sizeof(buf)); 139 } 140 return ghash_export(desc, dst); 141 } 142 143 static void ghash_reflect(u64 h[], const be128 *k) 144 { 145 u64 carry = be64_to_cpu(k->a) & BIT(63) ? 1 : 0; 146 147 h[0] = (be64_to_cpu(k->b) << 1) | carry; 148 h[1] = (be64_to_cpu(k->a) << 1) | (be64_to_cpu(k->b) >> 63); 149 150 if (carry) 151 h[1] ^= 0xc200000000000000UL; 152 } 153 154 static int ghash_setkey(struct crypto_shash *tfm, 155 const u8 *inkey, unsigned int keylen) 156 { 157 struct ghash_key *key = crypto_shash_ctx(tfm); 158 159 if (keylen != GHASH_BLOCK_SIZE) 160 return -EINVAL; 161 162 /* needed for the fallback */ 163 memcpy(&key->k, inkey, GHASH_BLOCK_SIZE); 164 165 ghash_reflect(key->h[0], &key->k); 166 return 0; 167 } 168 169 static struct shash_alg ghash_alg = { 170 .base.cra_name = "ghash", 171 .base.cra_driver_name = "ghash-neon", 172 .base.cra_priority = 150, 173 .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, 174 .base.cra_blocksize = GHASH_BLOCK_SIZE, 175 .base.cra_ctxsize = sizeof(struct ghash_key) + sizeof(u64[2]), 176 .base.cra_module = THIS_MODULE, 177 178 .digestsize = GHASH_DIGEST_SIZE, 179 .init = ghash_init, 180 .update = ghash_update, 181 .finup = ghash_finup, 182 .setkey = ghash_setkey, 183 .export = ghash_export, 184 .import = ghash_import, 185 .descsize = sizeof(struct arm_ghash_desc_ctx), 186 .statesize = sizeof(struct ghash_desc_ctx), 187 }; 188 189 static int num_rounds(struct crypto_aes_ctx *ctx) 190 { 191 /* 192 * # of rounds specified by AES: 193 * 128 bit key 10 rounds 194 * 192 bit key 12 rounds 195 * 256 bit key 14 rounds 196 * => n byte key => 6 + (n/4) rounds 197 */ 198 return 6 + ctx->key_length / 4; 199 } 200 201 static int gcm_aes_setkey(struct crypto_aead *tfm, const u8 *inkey, 202 unsigned int keylen) 203 { 204 struct gcm_aes_ctx *ctx = crypto_aead_ctx(tfm); 205 u8 key[GHASH_BLOCK_SIZE]; 206 be128 h; 207 int ret; 208 209 ret = aes_expandkey(&ctx->aes_key, inkey, keylen); 210 if (ret) 211 return -EINVAL; 212 213 aes_encrypt(&ctx->aes_key, key, (u8[AES_BLOCK_SIZE]){}); 214 215 /* needed for the fallback */ 216 memcpy(&ctx->ghash_key.k, key, GHASH_BLOCK_SIZE); 217 218 ghash_reflect(ctx->ghash_key.h[0], &ctx->ghash_key.k); 219 220 h = ctx->ghash_key.k; 221 gf128mul_lle(&h, &ctx->ghash_key.k); 222 ghash_reflect(ctx->ghash_key.h[1], &h); 223 224 gf128mul_lle(&h, &ctx->ghash_key.k); 225 ghash_reflect(ctx->ghash_key.h[2], &h); 226 227 gf128mul_lle(&h, &ctx->ghash_key.k); 228 ghash_reflect(ctx->ghash_key.h[3], &h); 229 230 return 0; 231 } 232 233 static int gcm_aes_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 234 { 235 return crypto_gcm_check_authsize(authsize); 236 } 237 238 static void gcm_update_mac(u64 dg[], const u8 *src, int count, u8 buf[], 239 int *buf_count, struct gcm_aes_ctx *ctx) 240 { 241 if (*buf_count > 0) { 242 int buf_added = min(count, GHASH_BLOCK_SIZE - *buf_count); 243 244 memcpy(&buf[*buf_count], src, buf_added); 245 246 *buf_count += buf_added; 247 src += buf_added; 248 count -= buf_added; 249 } 250 251 if (count >= GHASH_BLOCK_SIZE || *buf_count == GHASH_BLOCK_SIZE) { 252 int blocks = count / GHASH_BLOCK_SIZE; 253 254 ghash_do_simd_update(blocks, dg, src, &ctx->ghash_key, 255 *buf_count ? buf : NULL, 256 pmull_ghash_update_p64); 257 258 src += blocks * GHASH_BLOCK_SIZE; 259 count %= GHASH_BLOCK_SIZE; 260 *buf_count = 0; 261 } 262 263 if (count > 0) { 264 memcpy(buf, src, count); 265 *buf_count = count; 266 } 267 } 268 269 static void gcm_calculate_auth_mac(struct aead_request *req, u64 dg[], u32 len) 270 { 271 struct crypto_aead *aead = crypto_aead_reqtfm(req); 272 struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead); 273 u8 buf[GHASH_BLOCK_SIZE]; 274 struct scatter_walk walk; 275 int buf_count = 0; 276 277 scatterwalk_start(&walk, req->src); 278 279 do { 280 unsigned int n; 281 282 n = scatterwalk_next(&walk, len); 283 gcm_update_mac(dg, walk.addr, n, buf, &buf_count, ctx); 284 scatterwalk_done_src(&walk, n); 285 len -= n; 286 } while (len); 287 288 if (buf_count) { 289 memset(&buf[buf_count], 0, GHASH_BLOCK_SIZE - buf_count); 290 ghash_do_simd_update(1, dg, buf, &ctx->ghash_key, NULL, 291 pmull_ghash_update_p64); 292 } 293 } 294 295 static int gcm_encrypt(struct aead_request *req, char *iv, int assoclen) 296 { 297 struct crypto_aead *aead = crypto_aead_reqtfm(req); 298 struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead); 299 int nrounds = num_rounds(&ctx->aes_key); 300 struct skcipher_walk walk; 301 u8 buf[AES_BLOCK_SIZE]; 302 u64 dg[2] = {}; 303 be128 lengths; 304 u8 *tag; 305 int err; 306 307 lengths.a = cpu_to_be64(assoclen * 8); 308 lengths.b = cpu_to_be64(req->cryptlen * 8); 309 310 if (assoclen) 311 gcm_calculate_auth_mac(req, dg, assoclen); 312 313 put_unaligned_be32(2, iv + GCM_AES_IV_SIZE); 314 315 err = skcipher_walk_aead_encrypt(&walk, req, false); 316 317 do { 318 const u8 *src = walk.src.virt.addr; 319 u8 *dst = walk.dst.virt.addr; 320 int nbytes = walk.nbytes; 321 322 tag = (u8 *)&lengths; 323 324 if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) { 325 src = dst = memcpy(buf + sizeof(buf) - nbytes, 326 src, nbytes); 327 } else if (nbytes < walk.total) { 328 nbytes &= ~(AES_BLOCK_SIZE - 1); 329 tag = NULL; 330 } 331 332 scoped_ksimd() 333 pmull_gcm_encrypt(nbytes, dst, src, ctx->ghash_key.h, 334 dg, iv, ctx->aes_key.key_enc, nrounds, 335 tag); 336 337 if (unlikely(!nbytes)) 338 break; 339 340 if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) 341 memcpy(walk.dst.virt.addr, 342 buf + sizeof(buf) - nbytes, nbytes); 343 344 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 345 } while (walk.nbytes); 346 347 if (err) 348 return err; 349 350 /* copy authtag to end of dst */ 351 scatterwalk_map_and_copy(tag, req->dst, req->assoclen + req->cryptlen, 352 crypto_aead_authsize(aead), 1); 353 354 return 0; 355 } 356 357 static int gcm_decrypt(struct aead_request *req, char *iv, int assoclen) 358 { 359 struct crypto_aead *aead = crypto_aead_reqtfm(req); 360 struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead); 361 unsigned int authsize = crypto_aead_authsize(aead); 362 int nrounds = num_rounds(&ctx->aes_key); 363 struct skcipher_walk walk; 364 u8 otag[AES_BLOCK_SIZE]; 365 u8 buf[AES_BLOCK_SIZE]; 366 u64 dg[2] = {}; 367 be128 lengths; 368 u8 *tag; 369 int ret; 370 int err; 371 372 lengths.a = cpu_to_be64(assoclen * 8); 373 lengths.b = cpu_to_be64((req->cryptlen - authsize) * 8); 374 375 if (assoclen) 376 gcm_calculate_auth_mac(req, dg, assoclen); 377 378 put_unaligned_be32(2, iv + GCM_AES_IV_SIZE); 379 380 scatterwalk_map_and_copy(otag, req->src, 381 req->assoclen + req->cryptlen - authsize, 382 authsize, 0); 383 384 err = skcipher_walk_aead_decrypt(&walk, req, false); 385 386 do { 387 const u8 *src = walk.src.virt.addr; 388 u8 *dst = walk.dst.virt.addr; 389 int nbytes = walk.nbytes; 390 391 tag = (u8 *)&lengths; 392 393 if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) { 394 src = dst = memcpy(buf + sizeof(buf) - nbytes, 395 src, nbytes); 396 } else if (nbytes < walk.total) { 397 nbytes &= ~(AES_BLOCK_SIZE - 1); 398 tag = NULL; 399 } 400 401 scoped_ksimd() 402 ret = pmull_gcm_decrypt(nbytes, dst, src, 403 ctx->ghash_key.h, 404 dg, iv, ctx->aes_key.key_enc, 405 nrounds, tag, otag, authsize); 406 407 if (unlikely(!nbytes)) 408 break; 409 410 if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) 411 memcpy(walk.dst.virt.addr, 412 buf + sizeof(buf) - nbytes, nbytes); 413 414 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 415 } while (walk.nbytes); 416 417 if (err) 418 return err; 419 420 return ret ? -EBADMSG : 0; 421 } 422 423 static int gcm_aes_encrypt(struct aead_request *req) 424 { 425 u8 iv[AES_BLOCK_SIZE]; 426 427 memcpy(iv, req->iv, GCM_AES_IV_SIZE); 428 return gcm_encrypt(req, iv, req->assoclen); 429 } 430 431 static int gcm_aes_decrypt(struct aead_request *req) 432 { 433 u8 iv[AES_BLOCK_SIZE]; 434 435 memcpy(iv, req->iv, GCM_AES_IV_SIZE); 436 return gcm_decrypt(req, iv, req->assoclen); 437 } 438 439 static int rfc4106_setkey(struct crypto_aead *tfm, const u8 *inkey, 440 unsigned int keylen) 441 { 442 struct gcm_aes_ctx *ctx = crypto_aead_ctx(tfm); 443 int err; 444 445 keylen -= RFC4106_NONCE_SIZE; 446 err = gcm_aes_setkey(tfm, inkey, keylen); 447 if (err) 448 return err; 449 450 memcpy(ctx->nonce, inkey + keylen, RFC4106_NONCE_SIZE); 451 return 0; 452 } 453 454 static int rfc4106_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 455 { 456 return crypto_rfc4106_check_authsize(authsize); 457 } 458 459 static int rfc4106_encrypt(struct aead_request *req) 460 { 461 struct crypto_aead *aead = crypto_aead_reqtfm(req); 462 struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead); 463 u8 iv[AES_BLOCK_SIZE]; 464 465 memcpy(iv, ctx->nonce, RFC4106_NONCE_SIZE); 466 memcpy(iv + RFC4106_NONCE_SIZE, req->iv, GCM_RFC4106_IV_SIZE); 467 468 return crypto_ipsec_check_assoclen(req->assoclen) ?: 469 gcm_encrypt(req, iv, req->assoclen - GCM_RFC4106_IV_SIZE); 470 } 471 472 static int rfc4106_decrypt(struct aead_request *req) 473 { 474 struct crypto_aead *aead = crypto_aead_reqtfm(req); 475 struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead); 476 u8 iv[AES_BLOCK_SIZE]; 477 478 memcpy(iv, ctx->nonce, RFC4106_NONCE_SIZE); 479 memcpy(iv + RFC4106_NONCE_SIZE, req->iv, GCM_RFC4106_IV_SIZE); 480 481 return crypto_ipsec_check_assoclen(req->assoclen) ?: 482 gcm_decrypt(req, iv, req->assoclen - GCM_RFC4106_IV_SIZE); 483 } 484 485 static struct aead_alg gcm_aes_algs[] = {{ 486 .ivsize = GCM_AES_IV_SIZE, 487 .chunksize = AES_BLOCK_SIZE, 488 .maxauthsize = AES_BLOCK_SIZE, 489 .setkey = gcm_aes_setkey, 490 .setauthsize = gcm_aes_setauthsize, 491 .encrypt = gcm_aes_encrypt, 492 .decrypt = gcm_aes_decrypt, 493 494 .base.cra_name = "gcm(aes)", 495 .base.cra_driver_name = "gcm-aes-ce", 496 .base.cra_priority = 300, 497 .base.cra_blocksize = 1, 498 .base.cra_ctxsize = sizeof(struct gcm_aes_ctx) + 499 4 * sizeof(u64[2]), 500 .base.cra_module = THIS_MODULE, 501 }, { 502 .ivsize = GCM_RFC4106_IV_SIZE, 503 .chunksize = AES_BLOCK_SIZE, 504 .maxauthsize = AES_BLOCK_SIZE, 505 .setkey = rfc4106_setkey, 506 .setauthsize = rfc4106_setauthsize, 507 .encrypt = rfc4106_encrypt, 508 .decrypt = rfc4106_decrypt, 509 510 .base.cra_name = "rfc4106(gcm(aes))", 511 .base.cra_driver_name = "rfc4106-gcm-aes-ce", 512 .base.cra_priority = 300, 513 .base.cra_blocksize = 1, 514 .base.cra_ctxsize = sizeof(struct gcm_aes_ctx) + 515 4 * sizeof(u64[2]), 516 .base.cra_module = THIS_MODULE, 517 }}; 518 519 static int __init ghash_ce_mod_init(void) 520 { 521 if (!cpu_have_named_feature(ASIMD)) 522 return -ENODEV; 523 524 if (cpu_have_named_feature(PMULL)) 525 return crypto_register_aeads(gcm_aes_algs, 526 ARRAY_SIZE(gcm_aes_algs)); 527 528 return crypto_register_shash(&ghash_alg); 529 } 530 531 static void __exit ghash_ce_mod_exit(void) 532 { 533 if (cpu_have_named_feature(PMULL)) 534 crypto_unregister_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs)); 535 else 536 crypto_unregister_shash(&ghash_alg); 537 } 538 539 static const struct cpu_feature __maybe_unused ghash_cpu_feature[] = { 540 { cpu_feature(PMULL) }, { } 541 }; 542 MODULE_DEVICE_TABLE(cpu, ghash_cpu_feature); 543 544 module_init(ghash_ce_mod_init); 545 module_exit(ghash_ce_mod_exit); 546