1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* 3 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. 4 * 5 * This is an implementation of the ChaCha20Poly1305 AEAD construction. 6 * 7 * Information: https://tools.ietf.org/html/rfc8439 8 */ 9 10 #include <crypto/chacha.h> 11 #include <crypto/chacha20poly1305.h> 12 #include <crypto/poly1305.h> 13 #include <crypto/utils.h> 14 #include <linux/export.h> 15 #include <linux/init.h> 16 #include <linux/kernel.h> 17 #include <linux/mm.h> 18 #include <linux/module.h> 19 #include <linux/unaligned.h> 20 21 static void chacha_load_key(u32 *k, const u8 *in) 22 { 23 k[0] = get_unaligned_le32(in); 24 k[1] = get_unaligned_le32(in + 4); 25 k[2] = get_unaligned_le32(in + 8); 26 k[3] = get_unaligned_le32(in + 12); 27 k[4] = get_unaligned_le32(in + 16); 28 k[5] = get_unaligned_le32(in + 20); 29 k[6] = get_unaligned_le32(in + 24); 30 k[7] = get_unaligned_le32(in + 28); 31 } 32 33 static void xchacha_init(struct chacha_state *chacha_state, 34 const u8 *key, const u8 *nonce) 35 { 36 u32 k[CHACHA_KEY_WORDS]; 37 u8 iv[CHACHA_IV_SIZE]; 38 39 memset(iv, 0, 8); 40 memcpy(iv + 8, nonce + 16, 8); 41 42 chacha_load_key(k, key); 43 44 /* Compute the subkey given the original key and first 128 nonce bits */ 45 chacha_init(chacha_state, k, nonce); 46 hchacha_block(chacha_state, k, 20); 47 48 chacha_init(chacha_state, k, iv); 49 50 memzero_explicit(k, sizeof(k)); 51 memzero_explicit(iv, sizeof(iv)); 52 } 53 54 static void 55 __chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, 56 const u8 *ad, const size_t ad_len, 57 struct chacha_state *chacha_state) 58 { 59 const u8 *pad0 = page_address(ZERO_PAGE(0)); 60 struct poly1305_desc_ctx poly1305_state; 61 union { 62 u8 block0[POLY1305_KEY_SIZE]; 63 __le64 lens[2]; 64 } b; 65 66 chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0)); 67 poly1305_init(&poly1305_state, b.block0); 68 69 poly1305_update(&poly1305_state, ad, ad_len); 70 if (ad_len & 0xf) 71 poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf)); 72 73 chacha20_crypt(chacha_state, dst, src, src_len); 74 75 poly1305_update(&poly1305_state, dst, src_len); 76 if (src_len & 0xf) 77 poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf)); 78 79 b.lens[0] = cpu_to_le64(ad_len); 80 b.lens[1] = cpu_to_le64(src_len); 81 poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens)); 82 83 poly1305_final(&poly1305_state, dst + src_len); 84 85 chacha_zeroize_state(chacha_state); 86 memzero_explicit(&b, sizeof(b)); 87 } 88 89 void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, 90 const u8 *ad, const size_t ad_len, 91 const u64 nonce, 92 const u8 key[at_least CHACHA20POLY1305_KEY_SIZE]) 93 { 94 struct chacha_state chacha_state; 95 u32 k[CHACHA_KEY_WORDS]; 96 __le64 iv[2]; 97 98 chacha_load_key(k, key); 99 100 iv[0] = 0; 101 iv[1] = cpu_to_le64(nonce); 102 103 chacha_init(&chacha_state, k, (u8 *)iv); 104 __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, 105 &chacha_state); 106 107 memzero_explicit(iv, sizeof(iv)); 108 memzero_explicit(k, sizeof(k)); 109 } 110 EXPORT_SYMBOL(chacha20poly1305_encrypt); 111 112 void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, 113 const u8 *ad, const size_t ad_len, 114 const u8 nonce[at_least XCHACHA20POLY1305_NONCE_SIZE], 115 const u8 key[at_least CHACHA20POLY1305_KEY_SIZE]) 116 { 117 struct chacha_state chacha_state; 118 119 xchacha_init(&chacha_state, key, nonce); 120 __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, 121 &chacha_state); 122 } 123 EXPORT_SYMBOL(xchacha20poly1305_encrypt); 124 125 static bool 126 __chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, 127 const u8 *ad, const size_t ad_len, 128 struct chacha_state *chacha_state) 129 { 130 const u8 *pad0 = page_address(ZERO_PAGE(0)); 131 struct poly1305_desc_ctx poly1305_state; 132 size_t dst_len; 133 int ret; 134 union { 135 u8 block0[POLY1305_KEY_SIZE]; 136 u8 mac[POLY1305_DIGEST_SIZE]; 137 __le64 lens[2]; 138 } b; 139 140 chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0)); 141 poly1305_init(&poly1305_state, b.block0); 142 143 poly1305_update(&poly1305_state, ad, ad_len); 144 if (ad_len & 0xf) 145 poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf)); 146 147 dst_len = src_len - POLY1305_DIGEST_SIZE; 148 poly1305_update(&poly1305_state, src, dst_len); 149 if (dst_len & 0xf) 150 poly1305_update(&poly1305_state, pad0, 0x10 - (dst_len & 0xf)); 151 152 b.lens[0] = cpu_to_le64(ad_len); 153 b.lens[1] = cpu_to_le64(dst_len); 154 poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens)); 155 156 poly1305_final(&poly1305_state, b.mac); 157 158 ret = crypto_memneq(b.mac, src + dst_len, POLY1305_DIGEST_SIZE); 159 if (likely(!ret)) 160 chacha20_crypt(chacha_state, dst, src, dst_len); 161 162 chacha_zeroize_state(chacha_state); 163 memzero_explicit(&b, sizeof(b)); 164 165 return !ret; 166 } 167 168 bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, 169 const u8 *ad, const size_t ad_len, 170 const u64 nonce, 171 const u8 key[at_least CHACHA20POLY1305_KEY_SIZE]) 172 { 173 struct chacha_state chacha_state; 174 u32 k[CHACHA_KEY_WORDS]; 175 __le64 iv[2]; 176 bool ret; 177 178 if (unlikely(src_len < POLY1305_DIGEST_SIZE)) 179 return false; 180 181 chacha_load_key(k, key); 182 183 iv[0] = 0; 184 iv[1] = cpu_to_le64(nonce); 185 186 chacha_init(&chacha_state, k, (u8 *)iv); 187 ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len, 188 &chacha_state); 189 190 memzero_explicit(iv, sizeof(iv)); 191 memzero_explicit(k, sizeof(k)); 192 return ret; 193 } 194 EXPORT_SYMBOL(chacha20poly1305_decrypt); 195 196 bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, 197 const u8 *ad, const size_t ad_len, 198 const u8 nonce[at_least XCHACHA20POLY1305_NONCE_SIZE], 199 const u8 key[at_least CHACHA20POLY1305_KEY_SIZE]) 200 { 201 struct chacha_state chacha_state; 202 203 if (unlikely(src_len < POLY1305_DIGEST_SIZE)) 204 return false; 205 206 xchacha_init(&chacha_state, key, nonce); 207 return __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len, 208 &chacha_state); 209 } 210 EXPORT_SYMBOL(xchacha20poly1305_decrypt); 211 212 static 213 bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src, 214 const size_t src_len, 215 const u8 *ad, const size_t ad_len, 216 const u64 nonce, 217 const u8 key[at_least CHACHA20POLY1305_KEY_SIZE], 218 int encrypt) 219 { 220 const u8 *pad0 = page_address(ZERO_PAGE(0)); 221 struct poly1305_desc_ctx poly1305_state; 222 struct chacha_state chacha_state; 223 struct sg_mapping_iter miter; 224 size_t partial = 0; 225 unsigned int flags; 226 bool ret = true; 227 int sl; 228 union { 229 struct { 230 u32 k[CHACHA_KEY_WORDS]; 231 __le64 iv[2]; 232 }; 233 u8 block0[POLY1305_KEY_SIZE]; 234 u8 chacha_stream[CHACHA_BLOCK_SIZE]; 235 struct { 236 u8 mac[2][POLY1305_DIGEST_SIZE]; 237 }; 238 __le64 lens[2]; 239 } b __aligned(16); 240 241 if (WARN_ON(src_len > INT_MAX)) 242 return false; 243 244 chacha_load_key(b.k, key); 245 246 b.iv[0] = 0; 247 b.iv[1] = cpu_to_le64(nonce); 248 249 chacha_init(&chacha_state, b.k, (u8 *)b.iv); 250 chacha20_crypt(&chacha_state, b.block0, pad0, sizeof(b.block0)); 251 poly1305_init(&poly1305_state, b.block0); 252 253 if (unlikely(ad_len)) { 254 poly1305_update(&poly1305_state, ad, ad_len); 255 if (ad_len & 0xf) 256 poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf)); 257 } 258 259 flags = SG_MITER_TO_SG | SG_MITER_ATOMIC; 260 261 sg_miter_start(&miter, src, sg_nents(src), flags); 262 263 for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) { 264 u8 *addr = miter.addr; 265 size_t length = min_t(size_t, sl, miter.length); 266 267 if (!encrypt) 268 poly1305_update(&poly1305_state, addr, length); 269 270 if (unlikely(partial)) { 271 size_t l = min(length, CHACHA_BLOCK_SIZE - partial); 272 273 crypto_xor(addr, b.chacha_stream + partial, l); 274 partial = (partial + l) & (CHACHA_BLOCK_SIZE - 1); 275 276 addr += l; 277 length -= l; 278 } 279 280 if (likely(length >= CHACHA_BLOCK_SIZE || length == sl)) { 281 size_t l = length; 282 283 if (unlikely(length < sl)) 284 l &= ~(CHACHA_BLOCK_SIZE - 1); 285 chacha20_crypt(&chacha_state, addr, addr, l); 286 addr += l; 287 length -= l; 288 } 289 290 if (unlikely(length > 0)) { 291 chacha20_crypt(&chacha_state, b.chacha_stream, pad0, 292 CHACHA_BLOCK_SIZE); 293 crypto_xor(addr, b.chacha_stream, length); 294 partial = length; 295 } 296 297 if (encrypt) 298 poly1305_update(&poly1305_state, miter.addr, 299 min_t(size_t, sl, miter.length)); 300 } 301 302 if (src_len & 0xf) 303 poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf)); 304 305 b.lens[0] = cpu_to_le64(ad_len); 306 b.lens[1] = cpu_to_le64(src_len); 307 poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens)); 308 309 if (likely(sl <= -POLY1305_DIGEST_SIZE)) { 310 if (encrypt) { 311 poly1305_final(&poly1305_state, 312 miter.addr + miter.length + sl); 313 ret = true; 314 } else { 315 poly1305_final(&poly1305_state, b.mac[0]); 316 ret = !crypto_memneq(b.mac[0], 317 miter.addr + miter.length + sl, 318 POLY1305_DIGEST_SIZE); 319 } 320 } 321 322 sg_miter_stop(&miter); 323 324 if (unlikely(sl > -POLY1305_DIGEST_SIZE)) { 325 poly1305_final(&poly1305_state, b.mac[1]); 326 sg_copy_buffer(src, sg_nents(src), b.mac[encrypt], 327 sizeof(b.mac[1]), src_len, !encrypt); 328 ret = encrypt || 329 !crypto_memneq(b.mac[0], b.mac[1], POLY1305_DIGEST_SIZE); 330 } 331 332 chacha_zeroize_state(&chacha_state); 333 memzero_explicit(&b, sizeof(b)); 334 335 return ret; 336 } 337 338 bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src, size_t src_len, 339 const u8 *ad, const size_t ad_len, 340 const u64 nonce, 341 const u8 key[at_least CHACHA20POLY1305_KEY_SIZE]) 342 { 343 return chacha20poly1305_crypt_sg_inplace(src, src_len, ad, ad_len, 344 nonce, key, 1); 345 } 346 EXPORT_SYMBOL(chacha20poly1305_encrypt_sg_inplace); 347 348 bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len, 349 const u8 *ad, const size_t ad_len, 350 const u64 nonce, 351 const u8 key[at_least CHACHA20POLY1305_KEY_SIZE]) 352 { 353 if (unlikely(src_len < POLY1305_DIGEST_SIZE)) 354 return false; 355 356 return chacha20poly1305_crypt_sg_inplace(src, 357 src_len - POLY1305_DIGEST_SIZE, 358 ad, ad_len, nonce, key, 0); 359 } 360 EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace); 361 362 MODULE_LICENSE("GPL v2"); 363 MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction"); 364 MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>"); 365