1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * aes-ce-ccm-glue.c - AES-CCM transform for ARMv8 with Crypto Extensions 4 * 5 * Copyright (C) 2013 - 2017 Linaro Ltd. 6 * Copyright (C) 2024 Google LLC 7 * 8 * Author: Ard Biesheuvel <ardb@kernel.org> 9 */ 10 11 #include <linux/unaligned.h> 12 #include <crypto/aes.h> 13 #include <crypto/scatterwalk.h> 14 #include <crypto/internal/aead.h> 15 #include <crypto/internal/skcipher.h> 16 #include <linux/module.h> 17 18 #include <asm/simd.h> 19 20 MODULE_IMPORT_NS("CRYPTO_INTERNAL"); 21 22 static int num_rounds(struct crypto_aes_ctx *ctx) 23 { 24 /* 25 * # of rounds specified by AES: 26 * 128 bit key 10 rounds 27 * 192 bit key 12 rounds 28 * 256 bit key 14 rounds 29 * => n byte key => 6 + (n/4) rounds 30 */ 31 return 6 + ctx->key_length / 4; 32 } 33 34 asmlinkage u32 ce_aes_mac_update(u8 const in[], u32 const rk[], int rounds, 35 int blocks, u8 dg[], int enc_before, 36 int enc_after); 37 38 asmlinkage void ce_aes_ccm_encrypt(u8 out[], u8 const in[], u32 cbytes, 39 u32 const rk[], u32 rounds, u8 mac[], 40 u8 ctr[], u8 const final_iv[]); 41 42 asmlinkage void ce_aes_ccm_decrypt(u8 out[], u8 const in[], u32 cbytes, 43 u32 const rk[], u32 rounds, u8 mac[], 44 u8 ctr[], u8 const final_iv[]); 45 46 static int ccm_setkey(struct crypto_aead *tfm, const u8 *in_key, 47 unsigned int key_len) 48 { 49 struct crypto_aes_ctx *ctx = crypto_aead_ctx(tfm); 50 51 return ce_aes_expandkey(ctx, in_key, key_len); 52 } 53 54 static int ccm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 55 { 56 if ((authsize & 1) || authsize < 4) 57 return -EINVAL; 58 return 0; 59 } 60 61 static int ccm_init_mac(struct aead_request *req, u8 maciv[], u32 msglen) 62 { 63 struct crypto_aead *aead = crypto_aead_reqtfm(req); 64 __be32 *n = (__be32 *)&maciv[AES_BLOCK_SIZE - 8]; 65 u32 l = req->iv[0] + 1; 66 67 /* verify that CCM dimension 'L' is set correctly in the IV */ 68 if (l < 2 || l > 8) 69 return -EINVAL; 70 71 /* verify that msglen can in fact be represented in L bytes */ 72 if (l < 4 && msglen >> (8 * l)) 73 return -EOVERFLOW; 74 75 /* 76 * Even if the CCM spec allows L values of up to 8, the Linux cryptoapi 77 * uses a u32 type to represent msglen so the top 4 bytes are always 0. 78 */ 79 n[0] = 0; 80 n[1] = cpu_to_be32(msglen); 81 82 memcpy(maciv, req->iv, AES_BLOCK_SIZE - l); 83 84 /* 85 * Meaning of byte 0 according to CCM spec (RFC 3610/NIST 800-38C) 86 * - bits 0..2 : max # of bytes required to represent msglen, minus 1 87 * (already set by caller) 88 * - bits 3..5 : size of auth tag (1 => 4 bytes, 2 => 6 bytes, etc) 89 * - bit 6 : indicates presence of authenticate-only data 90 */ 91 maciv[0] |= (crypto_aead_authsize(aead) - 2) << 2; 92 if (req->assoclen) 93 maciv[0] |= 0x40; 94 95 memset(&req->iv[AES_BLOCK_SIZE - l], 0, l); 96 return 0; 97 } 98 99 static u32 ce_aes_ccm_auth_data(u8 mac[], u8 const in[], u32 abytes, 100 u32 macp, u32 const rk[], u32 rounds) 101 { 102 int enc_after = (macp + abytes) % AES_BLOCK_SIZE; 103 104 do { 105 u32 blocks = abytes / AES_BLOCK_SIZE; 106 107 if (macp == AES_BLOCK_SIZE || (!macp && blocks > 0)) { 108 u32 rem = ce_aes_mac_update(in, rk, rounds, blocks, mac, 109 macp, enc_after); 110 u32 adv = (blocks - rem) * AES_BLOCK_SIZE; 111 112 macp = enc_after ? 0 : AES_BLOCK_SIZE; 113 in += adv; 114 abytes -= adv; 115 116 if (unlikely(rem)) 117 macp = 0; 118 } else { 119 u32 l = min(AES_BLOCK_SIZE - macp, abytes); 120 121 crypto_xor(&mac[macp], in, l); 122 in += l; 123 macp += l; 124 abytes -= l; 125 } 126 } while (abytes > 0); 127 128 return macp; 129 } 130 131 static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[]) 132 { 133 struct crypto_aead *aead = crypto_aead_reqtfm(req); 134 struct crypto_aes_ctx *ctx = crypto_aead_ctx(aead); 135 struct __packed { __be16 l; __be32 h; u16 len; } ltag; 136 struct scatter_walk walk; 137 u32 len = req->assoclen; 138 u32 macp = AES_BLOCK_SIZE; 139 140 /* prepend the AAD with a length tag */ 141 if (len < 0xff00) { 142 ltag.l = cpu_to_be16(len); 143 ltag.len = 2; 144 } else { 145 ltag.l = cpu_to_be16(0xfffe); 146 put_unaligned_be32(len, <ag.h); 147 ltag.len = 6; 148 } 149 150 macp = ce_aes_ccm_auth_data(mac, (u8 *)<ag, ltag.len, macp, 151 ctx->key_enc, num_rounds(ctx)); 152 scatterwalk_start(&walk, req->src); 153 154 do { 155 unsigned int n; 156 157 n = scatterwalk_next(&walk, len); 158 macp = ce_aes_ccm_auth_data(mac, walk.addr, n, macp, 159 ctx->key_enc, num_rounds(ctx)); 160 scatterwalk_done_src(&walk, n); 161 len -= n; 162 } while (len); 163 } 164 165 static int ccm_encrypt(struct aead_request *req) 166 { 167 struct crypto_aead *aead = crypto_aead_reqtfm(req); 168 struct crypto_aes_ctx *ctx = crypto_aead_ctx(aead); 169 struct skcipher_walk walk; 170 u8 __aligned(8) mac[AES_BLOCK_SIZE]; 171 u8 orig_iv[AES_BLOCK_SIZE]; 172 u32 len = req->cryptlen; 173 int err; 174 175 err = ccm_init_mac(req, mac, len); 176 if (err) 177 return err; 178 179 /* preserve the original iv for the final round */ 180 memcpy(orig_iv, req->iv, AES_BLOCK_SIZE); 181 182 err = skcipher_walk_aead_encrypt(&walk, req, false); 183 if (unlikely(err)) 184 return err; 185 186 scoped_ksimd() { 187 if (req->assoclen) 188 ccm_calculate_auth_mac(req, mac); 189 190 do { 191 u32 tail = walk.nbytes % AES_BLOCK_SIZE; 192 const u8 *src = walk.src.virt.addr; 193 u8 *dst = walk.dst.virt.addr; 194 u8 buf[AES_BLOCK_SIZE]; 195 u8 *final_iv = NULL; 196 197 if (walk.nbytes == walk.total) { 198 tail = 0; 199 final_iv = orig_iv; 200 } 201 202 if (unlikely(walk.nbytes < AES_BLOCK_SIZE)) 203 src = dst = memcpy(&buf[sizeof(buf) - walk.nbytes], 204 src, walk.nbytes); 205 206 ce_aes_ccm_encrypt(dst, src, walk.nbytes - tail, 207 ctx->key_enc, num_rounds(ctx), 208 mac, walk.iv, final_iv); 209 210 if (unlikely(walk.nbytes < AES_BLOCK_SIZE)) 211 memcpy(walk.dst.virt.addr, dst, walk.nbytes); 212 213 if (walk.nbytes) { 214 err = skcipher_walk_done(&walk, tail); 215 } 216 } while (walk.nbytes); 217 } 218 219 if (unlikely(err)) 220 return err; 221 222 /* copy authtag to end of dst */ 223 scatterwalk_map_and_copy(mac, req->dst, req->assoclen + req->cryptlen, 224 crypto_aead_authsize(aead), 1); 225 226 return 0; 227 } 228 229 static int ccm_decrypt(struct aead_request *req) 230 { 231 struct crypto_aead *aead = crypto_aead_reqtfm(req); 232 struct crypto_aes_ctx *ctx = crypto_aead_ctx(aead); 233 unsigned int authsize = crypto_aead_authsize(aead); 234 struct skcipher_walk walk; 235 u8 __aligned(8) mac[AES_BLOCK_SIZE]; 236 u8 orig_iv[AES_BLOCK_SIZE]; 237 u32 len = req->cryptlen - authsize; 238 int err; 239 240 err = ccm_init_mac(req, mac, len); 241 if (err) 242 return err; 243 244 /* preserve the original iv for the final round */ 245 memcpy(orig_iv, req->iv, AES_BLOCK_SIZE); 246 247 err = skcipher_walk_aead_decrypt(&walk, req, false); 248 if (unlikely(err)) 249 return err; 250 251 scoped_ksimd() { 252 if (req->assoclen) 253 ccm_calculate_auth_mac(req, mac); 254 255 do { 256 u32 tail = walk.nbytes % AES_BLOCK_SIZE; 257 const u8 *src = walk.src.virt.addr; 258 u8 *dst = walk.dst.virt.addr; 259 u8 buf[AES_BLOCK_SIZE]; 260 u8 *final_iv = NULL; 261 262 if (walk.nbytes == walk.total) { 263 tail = 0; 264 final_iv = orig_iv; 265 } 266 267 if (unlikely(walk.nbytes < AES_BLOCK_SIZE)) 268 src = dst = memcpy(&buf[sizeof(buf) - walk.nbytes], 269 src, walk.nbytes); 270 271 ce_aes_ccm_decrypt(dst, src, walk.nbytes - tail, 272 ctx->key_enc, num_rounds(ctx), 273 mac, walk.iv, final_iv); 274 275 if (unlikely(walk.nbytes < AES_BLOCK_SIZE)) 276 memcpy(walk.dst.virt.addr, dst, walk.nbytes); 277 278 if (walk.nbytes) { 279 err = skcipher_walk_done(&walk, tail); 280 } 281 } while (walk.nbytes); 282 } 283 284 if (unlikely(err)) 285 return err; 286 287 /* compare calculated auth tag with the stored one */ 288 scatterwalk_map_and_copy(orig_iv, req->src, 289 req->assoclen + req->cryptlen - authsize, 290 authsize, 0); 291 292 if (crypto_memneq(mac, orig_iv, authsize)) 293 return -EBADMSG; 294 return 0; 295 } 296 297 static struct aead_alg ccm_aes_alg = { 298 .base = { 299 .cra_name = "ccm(aes)", 300 .cra_driver_name = "ccm-aes-ce", 301 .cra_priority = 300, 302 .cra_blocksize = 1, 303 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 304 .cra_module = THIS_MODULE, 305 }, 306 .ivsize = AES_BLOCK_SIZE, 307 .chunksize = AES_BLOCK_SIZE, 308 .maxauthsize = AES_BLOCK_SIZE, 309 .setkey = ccm_setkey, 310 .setauthsize = ccm_setauthsize, 311 .encrypt = ccm_encrypt, 312 .decrypt = ccm_decrypt, 313 }; 314 315 static int __init aes_mod_init(void) 316 { 317 if (!cpu_have_named_feature(AES)) 318 return -ENODEV; 319 return crypto_register_aead(&ccm_aes_alg); 320 } 321 322 static void __exit aes_mod_exit(void) 323 { 324 crypto_unregister_aead(&ccm_aes_alg); 325 } 326 327 module_init(aes_mod_init); 328 module_exit(aes_mod_exit); 329 330 MODULE_DESCRIPTION("Synchronous AES in CCM mode using ARMv8 Crypto Extensions"); 331 MODULE_AUTHOR("Ard Biesheuvel <ardb@kernel.org>"); 332 MODULE_LICENSE("GPL v2"); 333 MODULE_ALIAS_CRYPTO("ccm(aes)"); 334