1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* 3 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. 4 * 5 * This is an implementation of the ChaCha20Poly1305 AEAD construction. 6 * 7 * Information: https://tools.ietf.org/html/rfc8439 8 */ 9 10 #include <crypto/chacha20poly1305.h> 11 #include <crypto/chacha.h> 12 #include <crypto/poly1305.h> 13 #include <crypto/utils.h> 14 15 #include <linux/unaligned.h> 16 #include <linux/kernel.h> 17 #include <linux/init.h> 18 #include <linux/mm.h> 19 #include <linux/module.h> 20 21 #define CHACHA_KEY_WORDS (CHACHA_KEY_SIZE / sizeof(u32)) 22 23 static void chacha_load_key(u32 *k, const u8 *in) 24 { 25 k[0] = get_unaligned_le32(in); 26 k[1] = get_unaligned_le32(in + 4); 27 k[2] = get_unaligned_le32(in + 8); 28 k[3] = get_unaligned_le32(in + 12); 29 k[4] = get_unaligned_le32(in + 16); 30 k[5] = get_unaligned_le32(in + 20); 31 k[6] = get_unaligned_le32(in + 24); 32 k[7] = get_unaligned_le32(in + 28); 33 } 34 35 static void xchacha_init(u32 *chacha_state, const u8 *key, const u8 *nonce) 36 { 37 u32 k[CHACHA_KEY_WORDS]; 38 u8 iv[CHACHA_IV_SIZE]; 39 40 memset(iv, 0, 8); 41 memcpy(iv + 8, nonce + 16, 8); 42 43 chacha_load_key(k, key); 44 45 /* Compute the subkey given the original key and first 128 nonce bits */ 46 chacha_init(chacha_state, k, nonce); 47 hchacha_block(chacha_state, k, 20); 48 49 chacha_init(chacha_state, k, iv); 50 51 memzero_explicit(k, sizeof(k)); 52 memzero_explicit(iv, sizeof(iv)); 53 } 54 55 static void 56 __chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, 57 const u8 *ad, const size_t ad_len, u32 *chacha_state) 58 { 59 const u8 *pad0 = page_address(ZERO_PAGE(0)); 60 struct poly1305_desc_ctx poly1305_state; 61 union { 62 u8 block0[POLY1305_KEY_SIZE]; 63 __le64 lens[2]; 64 } b; 65 66 chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0)); 67 poly1305_init(&poly1305_state, b.block0); 68 69 poly1305_update(&poly1305_state, ad, ad_len); 70 if (ad_len & 0xf) 71 poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf)); 72 73 chacha20_crypt(chacha_state, dst, src, src_len); 74 75 poly1305_update(&poly1305_state, dst, src_len); 76 if (src_len & 0xf) 77 poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf)); 78 79 b.lens[0] = cpu_to_le64(ad_len); 80 b.lens[1] = cpu_to_le64(src_len); 81 poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens)); 82 83 poly1305_final(&poly1305_state, dst + src_len); 84 85 memzero_explicit(chacha_state, CHACHA_STATE_WORDS * sizeof(u32)); 86 memzero_explicit(&b, sizeof(b)); 87 } 88 89 void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, 90 const u8 *ad, const size_t ad_len, 91 const u64 nonce, 92 const u8 key[CHACHA20POLY1305_KEY_SIZE]) 93 { 94 u32 chacha_state[CHACHA_STATE_WORDS]; 95 u32 k[CHACHA_KEY_WORDS]; 96 __le64 iv[2]; 97 98 chacha_load_key(k, key); 99 100 iv[0] = 0; 101 iv[1] = cpu_to_le64(nonce); 102 103 chacha_init(chacha_state, k, (u8 *)iv); 104 __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state); 105 106 memzero_explicit(iv, sizeof(iv)); 107 memzero_explicit(k, sizeof(k)); 108 } 109 EXPORT_SYMBOL(chacha20poly1305_encrypt); 110 111 void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, 112 const u8 *ad, const size_t ad_len, 113 const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], 114 const u8 key[CHACHA20POLY1305_KEY_SIZE]) 115 { 116 u32 chacha_state[CHACHA_STATE_WORDS]; 117 118 xchacha_init(chacha_state, key, nonce); 119 __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state); 120 } 121 EXPORT_SYMBOL(xchacha20poly1305_encrypt); 122 123 static bool 124 __chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, 125 const u8 *ad, const size_t ad_len, u32 *chacha_state) 126 { 127 const u8 *pad0 = page_address(ZERO_PAGE(0)); 128 struct poly1305_desc_ctx poly1305_state; 129 size_t dst_len; 130 int ret; 131 union { 132 u8 block0[POLY1305_KEY_SIZE]; 133 u8 mac[POLY1305_DIGEST_SIZE]; 134 __le64 lens[2]; 135 } b; 136 137 if (unlikely(src_len < POLY1305_DIGEST_SIZE)) 138 return false; 139 140 chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0)); 141 poly1305_init(&poly1305_state, b.block0); 142 143 poly1305_update(&poly1305_state, ad, ad_len); 144 if (ad_len & 0xf) 145 poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf)); 146 147 dst_len = src_len - POLY1305_DIGEST_SIZE; 148 poly1305_update(&poly1305_state, src, dst_len); 149 if (dst_len & 0xf) 150 poly1305_update(&poly1305_state, pad0, 0x10 - (dst_len & 0xf)); 151 152 b.lens[0] = cpu_to_le64(ad_len); 153 b.lens[1] = cpu_to_le64(dst_len); 154 poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens)); 155 156 poly1305_final(&poly1305_state, b.mac); 157 158 ret = crypto_memneq(b.mac, src + dst_len, POLY1305_DIGEST_SIZE); 159 if (likely(!ret)) 160 chacha20_crypt(chacha_state, dst, src, dst_len); 161 162 memzero_explicit(&b, sizeof(b)); 163 164 return !ret; 165 } 166 167 bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, 168 const u8 *ad, const size_t ad_len, 169 const u64 nonce, 170 const u8 key[CHACHA20POLY1305_KEY_SIZE]) 171 { 172 u32 chacha_state[CHACHA_STATE_WORDS]; 173 u32 k[CHACHA_KEY_WORDS]; 174 __le64 iv[2]; 175 bool ret; 176 177 chacha_load_key(k, key); 178 179 iv[0] = 0; 180 iv[1] = cpu_to_le64(nonce); 181 182 chacha_init(chacha_state, k, (u8 *)iv); 183 ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len, 184 chacha_state); 185 186 memzero_explicit(chacha_state, sizeof(chacha_state)); 187 memzero_explicit(iv, sizeof(iv)); 188 memzero_explicit(k, sizeof(k)); 189 return ret; 190 } 191 EXPORT_SYMBOL(chacha20poly1305_decrypt); 192 193 bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, 194 const u8 *ad, const size_t ad_len, 195 const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], 196 const u8 key[CHACHA20POLY1305_KEY_SIZE]) 197 { 198 u32 chacha_state[CHACHA_STATE_WORDS]; 199 200 xchacha_init(chacha_state, key, nonce); 201 return __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len, 202 chacha_state); 203 } 204 EXPORT_SYMBOL(xchacha20poly1305_decrypt); 205 206 static 207 bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src, 208 const size_t src_len, 209 const u8 *ad, const size_t ad_len, 210 const u64 nonce, 211 const u8 key[CHACHA20POLY1305_KEY_SIZE], 212 int encrypt) 213 { 214 const u8 *pad0 = page_address(ZERO_PAGE(0)); 215 struct poly1305_desc_ctx poly1305_state; 216 u32 chacha_state[CHACHA_STATE_WORDS]; 217 struct sg_mapping_iter miter; 218 size_t partial = 0; 219 unsigned int flags; 220 bool ret = true; 221 int sl; 222 union { 223 struct { 224 u32 k[CHACHA_KEY_WORDS]; 225 __le64 iv[2]; 226 }; 227 u8 block0[POLY1305_KEY_SIZE]; 228 u8 chacha_stream[CHACHA_BLOCK_SIZE]; 229 struct { 230 u8 mac[2][POLY1305_DIGEST_SIZE]; 231 }; 232 __le64 lens[2]; 233 } b __aligned(16); 234 235 if (WARN_ON(src_len > INT_MAX)) 236 return false; 237 238 chacha_load_key(b.k, key); 239 240 b.iv[0] = 0; 241 b.iv[1] = cpu_to_le64(nonce); 242 243 chacha_init(chacha_state, b.k, (u8 *)b.iv); 244 chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0)); 245 poly1305_init(&poly1305_state, b.block0); 246 247 if (unlikely(ad_len)) { 248 poly1305_update(&poly1305_state, ad, ad_len); 249 if (ad_len & 0xf) 250 poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf)); 251 } 252 253 flags = SG_MITER_TO_SG | SG_MITER_ATOMIC; 254 255 sg_miter_start(&miter, src, sg_nents(src), flags); 256 257 for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) { 258 u8 *addr = miter.addr; 259 size_t length = min_t(size_t, sl, miter.length); 260 261 if (!encrypt) 262 poly1305_update(&poly1305_state, addr, length); 263 264 if (unlikely(partial)) { 265 size_t l = min(length, CHACHA_BLOCK_SIZE - partial); 266 267 crypto_xor(addr, b.chacha_stream + partial, l); 268 partial = (partial + l) & (CHACHA_BLOCK_SIZE - 1); 269 270 addr += l; 271 length -= l; 272 } 273 274 if (likely(length >= CHACHA_BLOCK_SIZE || length == sl)) { 275 size_t l = length; 276 277 if (unlikely(length < sl)) 278 l &= ~(CHACHA_BLOCK_SIZE - 1); 279 chacha20_crypt(chacha_state, addr, addr, l); 280 addr += l; 281 length -= l; 282 } 283 284 if (unlikely(length > 0)) { 285 chacha20_crypt(chacha_state, b.chacha_stream, pad0, 286 CHACHA_BLOCK_SIZE); 287 crypto_xor(addr, b.chacha_stream, length); 288 partial = length; 289 } 290 291 if (encrypt) 292 poly1305_update(&poly1305_state, miter.addr, 293 min_t(size_t, sl, miter.length)); 294 } 295 296 if (src_len & 0xf) 297 poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf)); 298 299 b.lens[0] = cpu_to_le64(ad_len); 300 b.lens[1] = cpu_to_le64(src_len); 301 poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens)); 302 303 if (likely(sl <= -POLY1305_DIGEST_SIZE)) { 304 if (encrypt) { 305 poly1305_final(&poly1305_state, 306 miter.addr + miter.length + sl); 307 ret = true; 308 } else { 309 poly1305_final(&poly1305_state, b.mac[0]); 310 ret = !crypto_memneq(b.mac[0], 311 miter.addr + miter.length + sl, 312 POLY1305_DIGEST_SIZE); 313 } 314 } 315 316 sg_miter_stop(&miter); 317 318 if (unlikely(sl > -POLY1305_DIGEST_SIZE)) { 319 poly1305_final(&poly1305_state, b.mac[1]); 320 sg_copy_buffer(src, sg_nents(src), b.mac[encrypt], 321 sizeof(b.mac[1]), src_len, !encrypt); 322 ret = encrypt || 323 !crypto_memneq(b.mac[0], b.mac[1], POLY1305_DIGEST_SIZE); 324 } 325 326 memzero_explicit(chacha_state, sizeof(chacha_state)); 327 memzero_explicit(&b, sizeof(b)); 328 329 return ret; 330 } 331 332 bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src, size_t src_len, 333 const u8 *ad, const size_t ad_len, 334 const u64 nonce, 335 const u8 key[CHACHA20POLY1305_KEY_SIZE]) 336 { 337 return chacha20poly1305_crypt_sg_inplace(src, src_len, ad, ad_len, 338 nonce, key, 1); 339 } 340 EXPORT_SYMBOL(chacha20poly1305_encrypt_sg_inplace); 341 342 bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len, 343 const u8 *ad, const size_t ad_len, 344 const u64 nonce, 345 const u8 key[CHACHA20POLY1305_KEY_SIZE]) 346 { 347 if (unlikely(src_len < POLY1305_DIGEST_SIZE)) 348 return false; 349 350 return chacha20poly1305_crypt_sg_inplace(src, 351 src_len - POLY1305_DIGEST_SIZE, 352 ad, ad_len, nonce, key, 0); 353 } 354 EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace); 355 356 static int __init chacha20poly1305_init(void) 357 { 358 if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) && 359 WARN_ON(!chacha20poly1305_selftest())) 360 return -ENODEV; 361 return 0; 362 } 363 364 static void __exit chacha20poly1305_exit(void) 365 { 366 } 367 368 module_init(chacha20poly1305_init); 369 module_exit(chacha20poly1305_exit); 370 MODULE_LICENSE("GPL v2"); 371 MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction"); 372 MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>"); 373