1 /*- 2 * Copyright (C) 2008 Damien Miller <djm@mindrot.org> 3 * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org> 4 * Copyright (c) 2010-2011 Pawel Jakub Dawidek <pawel@dawidek.net> 5 * Copyright 2012-2013 John-Mark Gurney <jmg@FreeBSD.org> 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include <sys/param.h> 34 #include <sys/libkern.h> 35 #include <sys/malloc.h> 36 #include <sys/proc.h> 37 #include <sys/systm.h> 38 #include <crypto/aesni/aesni.h> 39 40 #include "aesencdec.h" 41 42 MALLOC_DECLARE(M_AESNI); 43 44 struct blocks8 { 45 __m128i blk[8]; 46 } __packed; 47 48 void 49 aesni_encrypt_cbc(int rounds, const void *key_schedule, size_t len, 50 const uint8_t *from, uint8_t *to, const uint8_t iv[AES_BLOCK_LEN]) 51 { 52 __m128i tot, ivreg; 53 size_t i; 54 55 len /= AES_BLOCK_LEN; 56 ivreg = _mm_loadu_si128((const __m128i *)iv); 57 for (i = 0; i < len; i++) { 58 tot = aesni_enc(rounds - 1, key_schedule, 59 _mm_loadu_si128((const __m128i *)from) ^ ivreg); 60 ivreg = tot; 61 _mm_storeu_si128((__m128i *)to, tot); 62 from += AES_BLOCK_LEN; 63 to += AES_BLOCK_LEN; 64 } 65 } 66 67 void 68 aesni_decrypt_cbc(int rounds, const void *key_schedule, size_t len, 69 uint8_t *buf, const uint8_t iv[AES_BLOCK_LEN]) 70 { 71 __m128i blocks[8]; 72 struct blocks8 *blks; 73 __m128i ivreg, nextiv; 74 size_t i, j, cnt; 75 76 ivreg = _mm_loadu_si128((const __m128i *)iv); 77 cnt = len / AES_BLOCK_LEN / 8; 78 for (i = 0; i < cnt; i++) { 79 blks = (struct blocks8 *)buf; 80 aesni_dec8(rounds - 1, key_schedule, blks->blk[0], blks->blk[1], 81 blks->blk[2], blks->blk[3], blks->blk[4], blks->blk[5], 82 blks->blk[6], blks->blk[7], &blocks[0]); 83 for (j = 0; j < 8; j++) { 84 nextiv = blks->blk[j]; 85 blks->blk[j] = blocks[j] ^ ivreg; 86 ivreg = nextiv; 87 } 88 buf += AES_BLOCK_LEN * 8; 89 } 90 i *= 8; 91 cnt = len / AES_BLOCK_LEN; 92 for (; i < cnt; i++) { 93 nextiv = _mm_loadu_si128((void *)buf); 94 _mm_storeu_si128((void *)buf, 95 aesni_dec(rounds - 1, key_schedule, nextiv) ^ ivreg); 96 ivreg = nextiv; 97 buf += AES_BLOCK_LEN; 98 } 99 } 100 101 void 102 aesni_encrypt_ecb(int rounds, const void *key_schedule, size_t len, 103 const uint8_t *from, uint8_t *to) 104 { 105 __m128i tot; 106 __m128i tout[8]; 107 struct blocks8 *top; 108 const struct blocks8 *blks; 109 size_t i, cnt; 110 111 cnt = len / AES_BLOCK_LEN / 8; 112 for (i = 0; i < cnt; i++) { 113 blks = (const struct blocks8 *)from; 114 top = (struct blocks8 *)to; 115 aesni_enc8(rounds - 1, key_schedule, blks->blk[0], blks->blk[1], 116 blks->blk[2], blks->blk[3], blks->blk[4], blks->blk[5], 117 blks->blk[6], blks->blk[7], tout); 118 top->blk[0] = tout[0]; 119 top->blk[1] = tout[1]; 120 top->blk[2] = tout[2]; 121 top->blk[3] = tout[3]; 122 top->blk[4] = tout[4]; 123 top->blk[5] = tout[5]; 124 top->blk[6] = tout[6]; 125 top->blk[7] = tout[7]; 126 from += AES_BLOCK_LEN * 8; 127 to += AES_BLOCK_LEN * 8; 128 } 129 i *= 8; 130 cnt = len / AES_BLOCK_LEN; 131 for (; i < cnt; i++) { 132 tot = aesni_enc(rounds - 1, key_schedule, 133 _mm_loadu_si128((const __m128i *)from)); 134 _mm_storeu_si128((__m128i *)to, tot); 135 from += AES_BLOCK_LEN; 136 to += AES_BLOCK_LEN; 137 } 138 } 139 140 void 141 aesni_decrypt_ecb(int rounds, const void *key_schedule, size_t len, 142 const uint8_t from[AES_BLOCK_LEN], uint8_t to[AES_BLOCK_LEN]) 143 { 144 __m128i tot; 145 __m128i tout[8]; 146 const struct blocks8 *blks; 147 struct blocks8 *top; 148 size_t i, cnt; 149 150 cnt = len / AES_BLOCK_LEN / 8; 151 for (i = 0; i < cnt; i++) { 152 blks = (const struct blocks8 *)from; 153 top = (struct blocks8 *)to; 154 aesni_dec8(rounds - 1, key_schedule, blks->blk[0], blks->blk[1], 155 blks->blk[2], blks->blk[3], blks->blk[4], blks->blk[5], 156 blks->blk[6], blks->blk[7], tout); 157 top->blk[0] = tout[0]; 158 top->blk[1] = tout[1]; 159 top->blk[2] = tout[2]; 160 top->blk[3] = tout[3]; 161 top->blk[4] = tout[4]; 162 top->blk[5] = tout[5]; 163 top->blk[6] = tout[6]; 164 top->blk[7] = tout[7]; 165 from += AES_BLOCK_LEN * 8; 166 to += AES_BLOCK_LEN * 8; 167 } 168 i *= 8; 169 cnt = len / AES_BLOCK_LEN; 170 for (; i < cnt; i++) { 171 tot = aesni_dec(rounds - 1, key_schedule, 172 _mm_loadu_si128((const __m128i *)from)); 173 _mm_storeu_si128((__m128i *)to, tot); 174 from += AES_BLOCK_LEN; 175 to += AES_BLOCK_LEN; 176 } 177 } 178 179 #define AES_XTS_BLOCKSIZE 16 180 #define AES_XTS_IVSIZE 8 181 #define AES_XTS_ALPHA 0x87 /* GF(2^128) generator polynomial */ 182 183 static inline __m128i 184 xts_crank_lfsr(__m128i inp) 185 { 186 const __m128i alphamask = _mm_set_epi32(1, 1, 1, AES_XTS_ALPHA); 187 __m128i xtweak, ret; 188 189 /* set up xor mask */ 190 xtweak = _mm_shuffle_epi32(inp, 0x93); 191 xtweak = _mm_srai_epi32(xtweak, 31); 192 xtweak &= alphamask; 193 194 /* next term */ 195 ret = _mm_slli_epi32(inp, 1); 196 ret ^= xtweak; 197 198 return ret; 199 } 200 201 static void 202 aesni_crypt_xts_block(int rounds, const __m128i *key_schedule, __m128i *tweak, 203 const uint8_t *from, uint8_t *to, int do_encrypt) 204 { 205 __m128i block; 206 207 block = _mm_loadu_si128((const __m128i *)from) ^ *tweak; 208 209 if (do_encrypt) 210 block = aesni_enc(rounds - 1, key_schedule, block); 211 else 212 block = aesni_dec(rounds - 1, key_schedule, block); 213 214 _mm_storeu_si128((__m128i *)to, block ^ *tweak); 215 216 *tweak = xts_crank_lfsr(*tweak); 217 } 218 219 static void 220 aesni_crypt_xts_block8(int rounds, const __m128i *key_schedule, __m128i *tweak, 221 const uint8_t *from, uint8_t *to, int do_encrypt) 222 { 223 __m128i tmptweak; 224 __m128i a, b, c, d, e, f, g, h; 225 __m128i tweaks[8]; 226 __m128i tmp[8]; 227 __m128i *top; 228 const __m128i *fromp; 229 230 tmptweak = *tweak; 231 232 /* 233 * unroll the loop. This lets gcc put values directly in the 234 * register and saves memory accesses. 235 */ 236 fromp = (const __m128i *)from; 237 #define PREPINP(v, pos) \ 238 do { \ 239 tweaks[(pos)] = tmptweak; \ 240 (v) = _mm_loadu_si128(&fromp[pos]) ^ \ 241 tmptweak; \ 242 tmptweak = xts_crank_lfsr(tmptweak); \ 243 } while (0) 244 PREPINP(a, 0); 245 PREPINP(b, 1); 246 PREPINP(c, 2); 247 PREPINP(d, 3); 248 PREPINP(e, 4); 249 PREPINP(f, 5); 250 PREPINP(g, 6); 251 PREPINP(h, 7); 252 *tweak = tmptweak; 253 254 if (do_encrypt) 255 aesni_enc8(rounds - 1, key_schedule, a, b, c, d, e, f, g, h, 256 tmp); 257 else 258 aesni_dec8(rounds - 1, key_schedule, a, b, c, d, e, f, g, h, 259 tmp); 260 261 top = (__m128i *)to; 262 _mm_storeu_si128(&top[0], tmp[0] ^ tweaks[0]); 263 _mm_storeu_si128(&top[1], tmp[1] ^ tweaks[1]); 264 _mm_storeu_si128(&top[2], tmp[2] ^ tweaks[2]); 265 _mm_storeu_si128(&top[3], tmp[3] ^ tweaks[3]); 266 _mm_storeu_si128(&top[4], tmp[4] ^ tweaks[4]); 267 _mm_storeu_si128(&top[5], tmp[5] ^ tweaks[5]); 268 _mm_storeu_si128(&top[6], tmp[6] ^ tweaks[6]); 269 _mm_storeu_si128(&top[7], tmp[7] ^ tweaks[7]); 270 } 271 272 static void 273 aesni_crypt_xts(int rounds, const __m128i *data_schedule, 274 const __m128i *tweak_schedule, size_t len, const uint8_t *from, 275 uint8_t *to, const uint8_t iv[AES_BLOCK_LEN], int do_encrypt) 276 { 277 __m128i tweakreg; 278 uint8_t tweak[AES_XTS_BLOCKSIZE] __aligned(16); 279 size_t i, cnt; 280 281 /* 282 * Prepare tweak as E_k2(IV). IV is specified as LE representation 283 * of a 64-bit block number which we allow to be passed in directly. 284 */ 285 #if BYTE_ORDER == LITTLE_ENDIAN 286 bcopy(iv, tweak, AES_XTS_IVSIZE); 287 /* Last 64 bits of IV are always zero. */ 288 bzero(tweak + AES_XTS_IVSIZE, AES_XTS_IVSIZE); 289 #else 290 #error Only LITTLE_ENDIAN architectures are supported. 291 #endif 292 tweakreg = _mm_loadu_si128((__m128i *)&tweak[0]); 293 tweakreg = aesni_enc(rounds - 1, tweak_schedule, tweakreg); 294 295 cnt = len / AES_XTS_BLOCKSIZE / 8; 296 for (i = 0; i < cnt; i++) { 297 aesni_crypt_xts_block8(rounds, data_schedule, &tweakreg, 298 from, to, do_encrypt); 299 from += AES_XTS_BLOCKSIZE * 8; 300 to += AES_XTS_BLOCKSIZE * 8; 301 } 302 i *= 8; 303 cnt = len / AES_XTS_BLOCKSIZE; 304 for (; i < cnt; i++) { 305 aesni_crypt_xts_block(rounds, data_schedule, &tweakreg, 306 from, to, do_encrypt); 307 from += AES_XTS_BLOCKSIZE; 308 to += AES_XTS_BLOCKSIZE; 309 } 310 } 311 312 void 313 aesni_encrypt_xts(int rounds, const void *data_schedule, 314 const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to, 315 const uint8_t iv[AES_BLOCK_LEN]) 316 { 317 318 aesni_crypt_xts(rounds, data_schedule, tweak_schedule, len, from, to, 319 iv, 1); 320 } 321 322 void 323 aesni_decrypt_xts(int rounds, const void *data_schedule, 324 const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to, 325 const uint8_t iv[AES_BLOCK_LEN]) 326 { 327 328 aesni_crypt_xts(rounds, data_schedule, tweak_schedule, len, from, to, 329 iv, 0); 330 } 331 332 int 333 aesni_cipher_setup_common(struct aesni_session *ses, const uint8_t *key, 334 int keylen) 335 { 336 337 switch (ses->algo) { 338 case CRYPTO_AES_CBC: 339 switch (keylen) { 340 case 128: 341 ses->rounds = AES128_ROUNDS; 342 break; 343 case 192: 344 ses->rounds = AES192_ROUNDS; 345 break; 346 case 256: 347 ses->rounds = AES256_ROUNDS; 348 break; 349 default: 350 return (EINVAL); 351 } 352 break; 353 case CRYPTO_AES_XTS: 354 switch (keylen) { 355 case 256: 356 ses->rounds = AES128_ROUNDS; 357 break; 358 case 512: 359 ses->rounds = AES256_ROUNDS; 360 break; 361 default: 362 return (EINVAL); 363 } 364 break; 365 default: 366 return (EINVAL); 367 } 368 369 aesni_set_enckey(key, ses->enc_schedule, ses->rounds); 370 aesni_set_deckey(ses->enc_schedule, ses->dec_schedule, ses->rounds); 371 if (ses->algo == CRYPTO_AES_CBC) 372 arc4rand(ses->iv, sizeof(ses->iv), 0); 373 else /* if (ses->algo == CRYPTO_AES_XTS) */ { 374 aesni_set_enckey(key + keylen / 16, ses->xts_schedule, 375 ses->rounds); 376 } 377 378 return (0); 379 } 380