1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * AES-GCM using ARMv8 Crypto Extensions 4 * 5 * Copyright (C) 2015 - 2018 Linaro Ltd. 6 * Copyright (C) 2023 Google LLC. 7 */ 8 9 #include <asm/hwcap.h> 10 #include <asm/neon.h> 11 #include <crypto/aes.h> 12 #include <crypto/b128ops.h> 13 #include <crypto/gcm.h> 14 #include <crypto/gf128mul.h> 15 #include <crypto/ghash.h> 16 #include <crypto/internal/aead.h> 17 #include <crypto/internal/skcipher.h> 18 #include <crypto/scatterwalk.h> 19 #include <linux/cpufeature.h> 20 #include <linux/errno.h> 21 #include <linux/jump_label.h> 22 #include <linux/kernel.h> 23 #include <linux/module.h> 24 #include <linux/string.h> 25 #include <linux/unaligned.h> 26 27 MODULE_DESCRIPTION("AES-GCM using ARMv8 Crypto Extensions"); 28 MODULE_AUTHOR("Ard Biesheuvel <ardb@kernel.org>"); 29 MODULE_LICENSE("GPL"); 30 MODULE_ALIAS_CRYPTO("gcm(aes)"); 31 MODULE_ALIAS_CRYPTO("rfc4106(gcm(aes))"); 32 33 #define RFC4106_NONCE_SIZE 4 34 35 struct gcm_key { 36 u64 h[4][2]; 37 u32 rk[AES_MAX_KEYLENGTH_U32]; 38 int rounds; 39 u8 nonce[]; // for RFC4106 nonce 40 }; 41 42 asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src, 43 u64 const h[4][2], const char *head); 44 45 static void ghash_reflect(u64 h[], const be128 *k) 46 { 47 u64 carry = be64_to_cpu(k->a) >> 63; 48 49 h[0] = (be64_to_cpu(k->b) << 1) | carry; 50 h[1] = (be64_to_cpu(k->a) << 1) | (be64_to_cpu(k->b) >> 63); 51 52 if (carry) 53 h[1] ^= 0xc200000000000000UL; 54 } 55 56 void pmull_gcm_encrypt(int blocks, u64 dg[], const char *src, 57 struct gcm_key const *k, char *dst, 58 const char *iv, int rounds, u32 counter); 59 60 void pmull_gcm_enc_final(int blocks, u64 dg[], char *tag, 61 struct gcm_key const *k, char *head, 62 const char *iv, int rounds, u32 counter); 63 64 void pmull_gcm_decrypt(int bytes, u64 dg[], const char *src, 65 struct gcm_key const *k, char *dst, 66 const char *iv, int rounds, u32 counter); 67 68 int pmull_gcm_dec_final(int bytes, u64 dg[], char *tag, 69 struct gcm_key const *k, char *head, 70 const char *iv, int rounds, u32 counter, 71 const char *otag, int authsize); 72 73 static int gcm_aes_setkey(struct crypto_aead *tfm, const u8 *inkey, 74 unsigned int keylen) 75 { 76 struct gcm_key *ctx = crypto_aead_ctx(tfm); 77 struct aes_enckey aes_key; 78 be128 h, k; 79 int ret; 80 81 ret = aes_prepareenckey(&aes_key, inkey, keylen); 82 if (ret) 83 return -EINVAL; 84 85 aes_encrypt(&aes_key, (u8 *)&k, (u8[AES_BLOCK_SIZE]){}); 86 87 /* 88 * Note: this assumes that the arm implementation of the AES library 89 * stores the standard round keys in k.rndkeys. 90 */ 91 memcpy(ctx->rk, aes_key.k.rndkeys, sizeof(ctx->rk)); 92 ctx->rounds = 6 + keylen / 4; 93 94 memzero_explicit(&aes_key, sizeof(aes_key)); 95 96 ghash_reflect(ctx->h[0], &k); 97 98 h = k; 99 gf128mul_lle(&h, &k); 100 ghash_reflect(ctx->h[1], &h); 101 102 gf128mul_lle(&h, &k); 103 ghash_reflect(ctx->h[2], &h); 104 105 gf128mul_lle(&h, &k); 106 ghash_reflect(ctx->h[3], &h); 107 108 return 0; 109 } 110 111 static int gcm_aes_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 112 { 113 return crypto_gcm_check_authsize(authsize); 114 } 115 116 static void gcm_update_mac(u64 dg[], const u8 *src, int count, u8 buf[], 117 int *buf_count, struct gcm_key *ctx) 118 { 119 if (*buf_count > 0) { 120 int buf_added = min(count, GHASH_BLOCK_SIZE - *buf_count); 121 122 memcpy(&buf[*buf_count], src, buf_added); 123 124 *buf_count += buf_added; 125 src += buf_added; 126 count -= buf_added; 127 } 128 129 if (count >= GHASH_BLOCK_SIZE || *buf_count == GHASH_BLOCK_SIZE) { 130 int blocks = count / GHASH_BLOCK_SIZE; 131 132 pmull_ghash_update_p64(blocks, dg, src, ctx->h, 133 *buf_count ? buf : NULL); 134 135 src += blocks * GHASH_BLOCK_SIZE; 136 count %= GHASH_BLOCK_SIZE; 137 *buf_count = 0; 138 } 139 140 if (count > 0) { 141 memcpy(buf, src, count); 142 *buf_count = count; 143 } 144 } 145 146 static void gcm_calculate_auth_mac(struct aead_request *req, u64 dg[], u32 len) 147 { 148 struct crypto_aead *aead = crypto_aead_reqtfm(req); 149 struct gcm_key *ctx = crypto_aead_ctx(aead); 150 u8 buf[GHASH_BLOCK_SIZE]; 151 struct scatter_walk walk; 152 int buf_count = 0; 153 154 scatterwalk_start(&walk, req->src); 155 156 do { 157 unsigned int n; 158 159 n = scatterwalk_next(&walk, len); 160 gcm_update_mac(dg, walk.addr, n, buf, &buf_count, ctx); 161 scatterwalk_done_src(&walk, n); 162 163 if (unlikely(len / SZ_4K > (len - n) / SZ_4K)) { 164 kernel_neon_end(); 165 kernel_neon_begin(); 166 } 167 168 len -= n; 169 } while (len); 170 171 if (buf_count) { 172 memset(&buf[buf_count], 0, GHASH_BLOCK_SIZE - buf_count); 173 pmull_ghash_update_p64(1, dg, buf, ctx->h, NULL); 174 } 175 } 176 177 static int gcm_encrypt(struct aead_request *req, const u8 *iv, u32 assoclen) 178 { 179 struct crypto_aead *aead = crypto_aead_reqtfm(req); 180 struct gcm_key *ctx = crypto_aead_ctx(aead); 181 struct skcipher_walk walk; 182 u8 buf[AES_BLOCK_SIZE]; 183 u32 counter = 2; 184 u64 dg[2] = {}; 185 be128 lengths; 186 const u8 *src; 187 u8 *tag, *dst; 188 int tail, err; 189 190 err = skcipher_walk_aead_encrypt(&walk, req, false); 191 192 kernel_neon_begin(); 193 194 if (assoclen) 195 gcm_calculate_auth_mac(req, dg, assoclen); 196 197 src = walk.src.virt.addr; 198 dst = walk.dst.virt.addr; 199 200 while (walk.nbytes >= AES_BLOCK_SIZE) { 201 int nblocks = walk.nbytes / AES_BLOCK_SIZE; 202 203 pmull_gcm_encrypt(nblocks, dg, src, ctx, dst, iv, 204 ctx->rounds, counter); 205 counter += nblocks; 206 207 if (walk.nbytes == walk.total) { 208 src += nblocks * AES_BLOCK_SIZE; 209 dst += nblocks * AES_BLOCK_SIZE; 210 break; 211 } 212 213 kernel_neon_end(); 214 215 err = skcipher_walk_done(&walk, 216 walk.nbytes % AES_BLOCK_SIZE); 217 if (err) 218 return err; 219 220 src = walk.src.virt.addr; 221 dst = walk.dst.virt.addr; 222 223 kernel_neon_begin(); 224 } 225 226 227 lengths.a = cpu_to_be64(assoclen * 8); 228 lengths.b = cpu_to_be64(req->cryptlen * 8); 229 230 tag = (u8 *)&lengths; 231 tail = walk.nbytes % AES_BLOCK_SIZE; 232 233 /* 234 * Bounce via a buffer unless we are encrypting in place and src/dst 235 * are not pointing to the start of the walk buffer. In that case, we 236 * can do a NEON load/xor/store sequence in place as long as we move 237 * the plain/ciphertext and keystream to the start of the register. If 238 * not, do a memcpy() to the end of the buffer so we can reuse the same 239 * logic. 240 */ 241 if (unlikely(tail && (tail == walk.nbytes || src != dst))) 242 src = memcpy(buf + sizeof(buf) - tail, src, tail); 243 244 pmull_gcm_enc_final(tail, dg, tag, ctx, (u8 *)src, iv, 245 ctx->rounds, counter); 246 kernel_neon_end(); 247 248 if (unlikely(tail && src != dst)) 249 memcpy(dst, src, tail); 250 251 if (walk.nbytes) { 252 err = skcipher_walk_done(&walk, 0); 253 if (err) 254 return err; 255 } 256 257 /* copy authtag to end of dst */ 258 scatterwalk_map_and_copy(tag, req->dst, req->assoclen + req->cryptlen, 259 crypto_aead_authsize(aead), 1); 260 261 return 0; 262 } 263 264 static int gcm_decrypt(struct aead_request *req, const u8 *iv, u32 assoclen) 265 { 266 struct crypto_aead *aead = crypto_aead_reqtfm(req); 267 struct gcm_key *ctx = crypto_aead_ctx(aead); 268 int authsize = crypto_aead_authsize(aead); 269 struct skcipher_walk walk; 270 u8 otag[AES_BLOCK_SIZE]; 271 u8 buf[AES_BLOCK_SIZE]; 272 u32 counter = 2; 273 u64 dg[2] = {}; 274 be128 lengths; 275 const u8 *src; 276 u8 *tag, *dst; 277 int tail, err, ret; 278 279 scatterwalk_map_and_copy(otag, req->src, 280 req->assoclen + req->cryptlen - authsize, 281 authsize, 0); 282 283 err = skcipher_walk_aead_decrypt(&walk, req, false); 284 285 kernel_neon_begin(); 286 287 if (assoclen) 288 gcm_calculate_auth_mac(req, dg, assoclen); 289 290 src = walk.src.virt.addr; 291 dst = walk.dst.virt.addr; 292 293 while (walk.nbytes >= AES_BLOCK_SIZE) { 294 int nblocks = walk.nbytes / AES_BLOCK_SIZE; 295 296 pmull_gcm_decrypt(nblocks, dg, src, ctx, dst, iv, 297 ctx->rounds, counter); 298 counter += nblocks; 299 300 if (walk.nbytes == walk.total) { 301 src += nblocks * AES_BLOCK_SIZE; 302 dst += nblocks * AES_BLOCK_SIZE; 303 break; 304 } 305 306 kernel_neon_end(); 307 308 err = skcipher_walk_done(&walk, 309 walk.nbytes % AES_BLOCK_SIZE); 310 if (err) 311 return err; 312 313 src = walk.src.virt.addr; 314 dst = walk.dst.virt.addr; 315 316 kernel_neon_begin(); 317 } 318 319 lengths.a = cpu_to_be64(assoclen * 8); 320 lengths.b = cpu_to_be64((req->cryptlen - authsize) * 8); 321 322 tag = (u8 *)&lengths; 323 tail = walk.nbytes % AES_BLOCK_SIZE; 324 325 if (unlikely(tail && (tail == walk.nbytes || src != dst))) 326 src = memcpy(buf + sizeof(buf) - tail, src, tail); 327 328 ret = pmull_gcm_dec_final(tail, dg, tag, ctx, (u8 *)src, iv, 329 ctx->rounds, counter, otag, authsize); 330 kernel_neon_end(); 331 332 if (unlikely(tail && src != dst)) 333 memcpy(dst, src, tail); 334 335 if (walk.nbytes) { 336 err = skcipher_walk_done(&walk, 0); 337 if (err) 338 return err; 339 } 340 341 return ret ? -EBADMSG : 0; 342 } 343 344 static int gcm_aes_encrypt(struct aead_request *req) 345 { 346 return gcm_encrypt(req, req->iv, req->assoclen); 347 } 348 349 static int gcm_aes_decrypt(struct aead_request *req) 350 { 351 return gcm_decrypt(req, req->iv, req->assoclen); 352 } 353 354 static int rfc4106_setkey(struct crypto_aead *tfm, const u8 *inkey, 355 unsigned int keylen) 356 { 357 struct gcm_key *ctx = crypto_aead_ctx(tfm); 358 int err; 359 360 keylen -= RFC4106_NONCE_SIZE; 361 err = gcm_aes_setkey(tfm, inkey, keylen); 362 if (err) 363 return err; 364 365 memcpy(ctx->nonce, inkey + keylen, RFC4106_NONCE_SIZE); 366 return 0; 367 } 368 369 static int rfc4106_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 370 { 371 return crypto_rfc4106_check_authsize(authsize); 372 } 373 374 static int rfc4106_encrypt(struct aead_request *req) 375 { 376 struct crypto_aead *aead = crypto_aead_reqtfm(req); 377 struct gcm_key *ctx = crypto_aead_ctx(aead); 378 u8 iv[GCM_AES_IV_SIZE]; 379 380 memcpy(iv, ctx->nonce, RFC4106_NONCE_SIZE); 381 memcpy(iv + RFC4106_NONCE_SIZE, req->iv, GCM_RFC4106_IV_SIZE); 382 383 return crypto_ipsec_check_assoclen(req->assoclen) ?: 384 gcm_encrypt(req, iv, req->assoclen - GCM_RFC4106_IV_SIZE); 385 } 386 387 static int rfc4106_decrypt(struct aead_request *req) 388 { 389 struct crypto_aead *aead = crypto_aead_reqtfm(req); 390 struct gcm_key *ctx = crypto_aead_ctx(aead); 391 u8 iv[GCM_AES_IV_SIZE]; 392 393 memcpy(iv, ctx->nonce, RFC4106_NONCE_SIZE); 394 memcpy(iv + RFC4106_NONCE_SIZE, req->iv, GCM_RFC4106_IV_SIZE); 395 396 return crypto_ipsec_check_assoclen(req->assoclen) ?: 397 gcm_decrypt(req, iv, req->assoclen - GCM_RFC4106_IV_SIZE); 398 } 399 400 static struct aead_alg gcm_aes_algs[] = {{ 401 .ivsize = GCM_AES_IV_SIZE, 402 .chunksize = AES_BLOCK_SIZE, 403 .maxauthsize = AES_BLOCK_SIZE, 404 .setkey = gcm_aes_setkey, 405 .setauthsize = gcm_aes_setauthsize, 406 .encrypt = gcm_aes_encrypt, 407 .decrypt = gcm_aes_decrypt, 408 409 .base.cra_name = "gcm(aes)", 410 .base.cra_driver_name = "gcm-aes-ce", 411 .base.cra_priority = 400, 412 .base.cra_blocksize = 1, 413 .base.cra_ctxsize = sizeof(struct gcm_key), 414 .base.cra_module = THIS_MODULE, 415 }, { 416 .ivsize = GCM_RFC4106_IV_SIZE, 417 .chunksize = AES_BLOCK_SIZE, 418 .maxauthsize = AES_BLOCK_SIZE, 419 .setkey = rfc4106_setkey, 420 .setauthsize = rfc4106_setauthsize, 421 .encrypt = rfc4106_encrypt, 422 .decrypt = rfc4106_decrypt, 423 424 .base.cra_name = "rfc4106(gcm(aes))", 425 .base.cra_driver_name = "rfc4106-gcm-aes-ce", 426 .base.cra_priority = 400, 427 .base.cra_blocksize = 1, 428 .base.cra_ctxsize = sizeof(struct gcm_key) + RFC4106_NONCE_SIZE, 429 .base.cra_module = THIS_MODULE, 430 }}; 431 432 static int __init ghash_ce_mod_init(void) 433 { 434 if (!(elf_hwcap & HWCAP_NEON) || !(elf_hwcap2 & HWCAP2_PMULL)) 435 return -ENODEV; 436 437 return crypto_register_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs)); 438 } 439 440 static void __exit ghash_ce_mod_exit(void) 441 { 442 crypto_unregister_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs)); 443 } 444 445 module_init(ghash_ce_mod_init); 446 module_exit(ghash_ce_mod_exit); 447