1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * aes-ce-ccm-glue.c - AES-CCM transform for ARMv8 with Crypto Extensions 4 * 5 * Copyright (C) 2013 - 2017 Linaro Ltd. 6 * Copyright (C) 2024 Google LLC 7 * 8 * Author: Ard Biesheuvel <ardb@kernel.org> 9 */ 10 11 #include <asm/neon.h> 12 #include <linux/unaligned.h> 13 #include <crypto/aes.h> 14 #include <crypto/scatterwalk.h> 15 #include <crypto/internal/aead.h> 16 #include <crypto/internal/skcipher.h> 17 #include <linux/module.h> 18 19 #include "aes-ce-setkey.h" 20 21 MODULE_IMPORT_NS(CRYPTO_INTERNAL); 22 23 static int num_rounds(struct crypto_aes_ctx *ctx) 24 { 25 /* 26 * # of rounds specified by AES: 27 * 128 bit key 10 rounds 28 * 192 bit key 12 rounds 29 * 256 bit key 14 rounds 30 * => n byte key => 6 + (n/4) rounds 31 */ 32 return 6 + ctx->key_length / 4; 33 } 34 35 asmlinkage u32 ce_aes_mac_update(u8 const in[], u32 const rk[], int rounds, 36 int blocks, u8 dg[], int enc_before, 37 int enc_after); 38 39 asmlinkage void ce_aes_ccm_encrypt(u8 out[], u8 const in[], u32 cbytes, 40 u32 const rk[], u32 rounds, u8 mac[], 41 u8 ctr[], u8 const final_iv[]); 42 43 asmlinkage void ce_aes_ccm_decrypt(u8 out[], u8 const in[], u32 cbytes, 44 u32 const rk[], u32 rounds, u8 mac[], 45 u8 ctr[], u8 const final_iv[]); 46 47 static int ccm_setkey(struct crypto_aead *tfm, const u8 *in_key, 48 unsigned int key_len) 49 { 50 struct crypto_aes_ctx *ctx = crypto_aead_ctx(tfm); 51 52 return ce_aes_expandkey(ctx, in_key, key_len); 53 } 54 55 static int ccm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 56 { 57 if ((authsize & 1) || authsize < 4) 58 return -EINVAL; 59 return 0; 60 } 61 62 static int ccm_init_mac(struct aead_request *req, u8 maciv[], u32 msglen) 63 { 64 struct crypto_aead *aead = crypto_aead_reqtfm(req); 65 __be32 *n = (__be32 *)&maciv[AES_BLOCK_SIZE - 8]; 66 u32 l = req->iv[0] + 1; 67 68 /* verify that CCM dimension 'L' is set correctly in the IV */ 69 if (l < 2 || l > 8) 70 return -EINVAL; 71 72 /* verify that msglen can in fact be represented in L bytes */ 73 if (l < 4 && msglen >> (8 * l)) 74 return -EOVERFLOW; 75 76 /* 77 * Even if the CCM spec allows L values of up to 8, the Linux cryptoapi 78 * uses a u32 type to represent msglen so the top 4 bytes are always 0. 79 */ 80 n[0] = 0; 81 n[1] = cpu_to_be32(msglen); 82 83 memcpy(maciv, req->iv, AES_BLOCK_SIZE - l); 84 85 /* 86 * Meaning of byte 0 according to CCM spec (RFC 3610/NIST 800-38C) 87 * - bits 0..2 : max # of bytes required to represent msglen, minus 1 88 * (already set by caller) 89 * - bits 3..5 : size of auth tag (1 => 4 bytes, 2 => 6 bytes, etc) 90 * - bit 6 : indicates presence of authenticate-only data 91 */ 92 maciv[0] |= (crypto_aead_authsize(aead) - 2) << 2; 93 if (req->assoclen) 94 maciv[0] |= 0x40; 95 96 memset(&req->iv[AES_BLOCK_SIZE - l], 0, l); 97 return 0; 98 } 99 100 static u32 ce_aes_ccm_auth_data(u8 mac[], u8 const in[], u32 abytes, 101 u32 macp, u32 const rk[], u32 rounds) 102 { 103 int enc_after = (macp + abytes) % AES_BLOCK_SIZE; 104 105 do { 106 u32 blocks = abytes / AES_BLOCK_SIZE; 107 108 if (macp == AES_BLOCK_SIZE || (!macp && blocks > 0)) { 109 u32 rem = ce_aes_mac_update(in, rk, rounds, blocks, mac, 110 macp, enc_after); 111 u32 adv = (blocks - rem) * AES_BLOCK_SIZE; 112 113 macp = enc_after ? 0 : AES_BLOCK_SIZE; 114 in += adv; 115 abytes -= adv; 116 117 if (unlikely(rem)) { 118 kernel_neon_end(); 119 kernel_neon_begin(); 120 macp = 0; 121 } 122 } else { 123 u32 l = min(AES_BLOCK_SIZE - macp, abytes); 124 125 crypto_xor(&mac[macp], in, l); 126 in += l; 127 macp += l; 128 abytes -= l; 129 } 130 } while (abytes > 0); 131 132 return macp; 133 } 134 135 static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[]) 136 { 137 struct crypto_aead *aead = crypto_aead_reqtfm(req); 138 struct crypto_aes_ctx *ctx = crypto_aead_ctx(aead); 139 struct __packed { __be16 l; __be32 h; u16 len; } ltag; 140 struct scatter_walk walk; 141 u32 len = req->assoclen; 142 u32 macp = AES_BLOCK_SIZE; 143 144 /* prepend the AAD with a length tag */ 145 if (len < 0xff00) { 146 ltag.l = cpu_to_be16(len); 147 ltag.len = 2; 148 } else { 149 ltag.l = cpu_to_be16(0xfffe); 150 put_unaligned_be32(len, <ag.h); 151 ltag.len = 6; 152 } 153 154 macp = ce_aes_ccm_auth_data(mac, (u8 *)<ag, ltag.len, macp, 155 ctx->key_enc, num_rounds(ctx)); 156 scatterwalk_start(&walk, req->src); 157 158 do { 159 u32 n = scatterwalk_clamp(&walk, len); 160 u8 *p; 161 162 if (!n) { 163 scatterwalk_start(&walk, sg_next(walk.sg)); 164 n = scatterwalk_clamp(&walk, len); 165 } 166 p = scatterwalk_map(&walk); 167 168 macp = ce_aes_ccm_auth_data(mac, p, n, macp, ctx->key_enc, 169 num_rounds(ctx)); 170 171 len -= n; 172 173 scatterwalk_unmap(p); 174 scatterwalk_advance(&walk, n); 175 scatterwalk_done(&walk, 0, len); 176 } while (len); 177 } 178 179 static int ccm_encrypt(struct aead_request *req) 180 { 181 struct crypto_aead *aead = crypto_aead_reqtfm(req); 182 struct crypto_aes_ctx *ctx = crypto_aead_ctx(aead); 183 struct skcipher_walk walk; 184 u8 __aligned(8) mac[AES_BLOCK_SIZE]; 185 u8 orig_iv[AES_BLOCK_SIZE]; 186 u32 len = req->cryptlen; 187 int err; 188 189 err = ccm_init_mac(req, mac, len); 190 if (err) 191 return err; 192 193 /* preserve the original iv for the final round */ 194 memcpy(orig_iv, req->iv, AES_BLOCK_SIZE); 195 196 err = skcipher_walk_aead_encrypt(&walk, req, false); 197 if (unlikely(err)) 198 return err; 199 200 kernel_neon_begin(); 201 202 if (req->assoclen) 203 ccm_calculate_auth_mac(req, mac); 204 205 do { 206 u32 tail = walk.nbytes % AES_BLOCK_SIZE; 207 const u8 *src = walk.src.virt.addr; 208 u8 *dst = walk.dst.virt.addr; 209 u8 buf[AES_BLOCK_SIZE]; 210 u8 *final_iv = NULL; 211 212 if (walk.nbytes == walk.total) { 213 tail = 0; 214 final_iv = orig_iv; 215 } 216 217 if (unlikely(walk.nbytes < AES_BLOCK_SIZE)) 218 src = dst = memcpy(&buf[sizeof(buf) - walk.nbytes], 219 src, walk.nbytes); 220 221 ce_aes_ccm_encrypt(dst, src, walk.nbytes - tail, 222 ctx->key_enc, num_rounds(ctx), 223 mac, walk.iv, final_iv); 224 225 if (unlikely(walk.nbytes < AES_BLOCK_SIZE)) 226 memcpy(walk.dst.virt.addr, dst, walk.nbytes); 227 228 if (walk.nbytes) { 229 err = skcipher_walk_done(&walk, tail); 230 } 231 } while (walk.nbytes); 232 233 kernel_neon_end(); 234 235 if (unlikely(err)) 236 return err; 237 238 /* copy authtag to end of dst */ 239 scatterwalk_map_and_copy(mac, req->dst, req->assoclen + req->cryptlen, 240 crypto_aead_authsize(aead), 1); 241 242 return 0; 243 } 244 245 static int ccm_decrypt(struct aead_request *req) 246 { 247 struct crypto_aead *aead = crypto_aead_reqtfm(req); 248 struct crypto_aes_ctx *ctx = crypto_aead_ctx(aead); 249 unsigned int authsize = crypto_aead_authsize(aead); 250 struct skcipher_walk walk; 251 u8 __aligned(8) mac[AES_BLOCK_SIZE]; 252 u8 orig_iv[AES_BLOCK_SIZE]; 253 u32 len = req->cryptlen - authsize; 254 int err; 255 256 err = ccm_init_mac(req, mac, len); 257 if (err) 258 return err; 259 260 /* preserve the original iv for the final round */ 261 memcpy(orig_iv, req->iv, AES_BLOCK_SIZE); 262 263 err = skcipher_walk_aead_decrypt(&walk, req, false); 264 if (unlikely(err)) 265 return err; 266 267 kernel_neon_begin(); 268 269 if (req->assoclen) 270 ccm_calculate_auth_mac(req, mac); 271 272 do { 273 u32 tail = walk.nbytes % AES_BLOCK_SIZE; 274 const u8 *src = walk.src.virt.addr; 275 u8 *dst = walk.dst.virt.addr; 276 u8 buf[AES_BLOCK_SIZE]; 277 u8 *final_iv = NULL; 278 279 if (walk.nbytes == walk.total) { 280 tail = 0; 281 final_iv = orig_iv; 282 } 283 284 if (unlikely(walk.nbytes < AES_BLOCK_SIZE)) 285 src = dst = memcpy(&buf[sizeof(buf) - walk.nbytes], 286 src, walk.nbytes); 287 288 ce_aes_ccm_decrypt(dst, src, walk.nbytes - tail, 289 ctx->key_enc, num_rounds(ctx), 290 mac, walk.iv, final_iv); 291 292 if (unlikely(walk.nbytes < AES_BLOCK_SIZE)) 293 memcpy(walk.dst.virt.addr, dst, walk.nbytes); 294 295 if (walk.nbytes) { 296 err = skcipher_walk_done(&walk, tail); 297 } 298 } while (walk.nbytes); 299 300 kernel_neon_end(); 301 302 if (unlikely(err)) 303 return err; 304 305 /* compare calculated auth tag with the stored one */ 306 scatterwalk_map_and_copy(orig_iv, req->src, 307 req->assoclen + req->cryptlen - authsize, 308 authsize, 0); 309 310 if (crypto_memneq(mac, orig_iv, authsize)) 311 return -EBADMSG; 312 return 0; 313 } 314 315 static struct aead_alg ccm_aes_alg = { 316 .base = { 317 .cra_name = "ccm(aes)", 318 .cra_driver_name = "ccm-aes-ce", 319 .cra_priority = 300, 320 .cra_blocksize = 1, 321 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 322 .cra_module = THIS_MODULE, 323 }, 324 .ivsize = AES_BLOCK_SIZE, 325 .chunksize = AES_BLOCK_SIZE, 326 .maxauthsize = AES_BLOCK_SIZE, 327 .setkey = ccm_setkey, 328 .setauthsize = ccm_setauthsize, 329 .encrypt = ccm_encrypt, 330 .decrypt = ccm_decrypt, 331 }; 332 333 static int __init aes_mod_init(void) 334 { 335 if (!cpu_have_named_feature(AES)) 336 return -ENODEV; 337 return crypto_register_aead(&ccm_aes_alg); 338 } 339 340 static void __exit aes_mod_exit(void) 341 { 342 crypto_unregister_aead(&ccm_aes_alg); 343 } 344 345 module_init(aes_mod_init); 346 module_exit(aes_mod_exit); 347 348 MODULE_DESCRIPTION("Synchronous AES in CCM mode using ARMv8 Crypto Extensions"); 349 MODULE_AUTHOR("Ard Biesheuvel <ardb@kernel.org>"); 350 MODULE_LICENSE("GPL v2"); 351 MODULE_ALIAS_CRYPTO("ccm(aes)"); 352