1 /*- 2 * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org> 3 * Copyright (c) 2010-2011 Pawel Jakub Dawidek <pawel@dawidek.net> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include <sys/param.h> 32 #include <sys/libkern.h> 33 #include <sys/malloc.h> 34 #include <sys/proc.h> 35 #include <sys/systm.h> 36 #include <crypto/aesni/aesni.h> 37 38 MALLOC_DECLARE(M_AESNI); 39 40 void 41 aesni_encrypt_cbc(int rounds, const void *key_schedule, size_t len, 42 const uint8_t *from, uint8_t *to, const uint8_t iv[AES_BLOCK_LEN]) 43 { 44 const uint8_t *ivp; 45 size_t i; 46 47 len /= AES_BLOCK_LEN; 48 ivp = iv; 49 for (i = 0; i < len; i++) { 50 aesni_enc(rounds - 1, key_schedule, from, to, ivp); 51 ivp = to; 52 from += AES_BLOCK_LEN; 53 to += AES_BLOCK_LEN; 54 } 55 } 56 57 void 58 aesni_encrypt_ecb(int rounds, const void *key_schedule, size_t len, 59 const uint8_t from[AES_BLOCK_LEN], uint8_t to[AES_BLOCK_LEN]) 60 { 61 size_t i; 62 63 len /= AES_BLOCK_LEN; 64 for (i = 0; i < len; i++) { 65 aesni_enc(rounds - 1, key_schedule, from, to, NULL); 66 from += AES_BLOCK_LEN; 67 to += AES_BLOCK_LEN; 68 } 69 } 70 71 void 72 aesni_decrypt_ecb(int rounds, const void *key_schedule, size_t len, 73 const uint8_t from[AES_BLOCK_LEN], uint8_t to[AES_BLOCK_LEN]) 74 { 75 size_t i; 76 77 len /= AES_BLOCK_LEN; 78 for (i = 0; i < len; i++) { 79 aesni_dec(rounds - 1, key_schedule, from, to, NULL); 80 from += AES_BLOCK_LEN; 81 to += AES_BLOCK_LEN; 82 } 83 } 84 85 #define AES_XTS_BLOCKSIZE 16 86 #define AES_XTS_IVSIZE 8 87 #define AES_XTS_ALPHA 0x87 /* GF(2^128) generator polynomial */ 88 89 static void 90 aesni_crypt_xts_block(int rounds, const void *key_schedule, uint64_t *tweak, 91 const uint64_t *from, uint64_t *to, uint64_t *block, int do_encrypt) 92 { 93 int carry; 94 95 block[0] = from[0] ^ tweak[0]; 96 block[1] = from[1] ^ tweak[1]; 97 98 if (do_encrypt) 99 aesni_enc(rounds - 1, key_schedule, (uint8_t *)block, (uint8_t *)to, NULL); 100 else 101 aesni_dec(rounds - 1, key_schedule, (uint8_t *)block, (uint8_t *)to, NULL); 102 103 to[0] ^= tweak[0]; 104 to[1] ^= tweak[1]; 105 106 /* Exponentiate tweak. */ 107 carry = ((tweak[0] & 0x8000000000000000ULL) > 0); 108 tweak[0] <<= 1; 109 if (tweak[1] & 0x8000000000000000ULL) { 110 uint8_t *twk = (uint8_t *)tweak; 111 112 twk[0] ^= AES_XTS_ALPHA; 113 } 114 tweak[1] <<= 1; 115 if (carry) 116 tweak[1] |= 1; 117 } 118 119 static void 120 aesni_crypt_xts(int rounds, const void *data_schedule, 121 const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to, 122 const uint8_t iv[AES_BLOCK_LEN], int do_encrypt) 123 { 124 uint64_t block[AES_XTS_BLOCKSIZE / 8]; 125 uint8_t tweak[AES_XTS_BLOCKSIZE]; 126 size_t i; 127 128 /* 129 * Prepare tweak as E_k2(IV). IV is specified as LE representation 130 * of a 64-bit block number which we allow to be passed in directly. 131 */ 132 #if BYTE_ORDER == LITTLE_ENDIAN 133 bcopy(iv, tweak, AES_XTS_IVSIZE); 134 /* Last 64 bits of IV are always zero. */ 135 bzero(tweak + AES_XTS_IVSIZE, AES_XTS_IVSIZE); 136 #else 137 #error Only LITTLE_ENDIAN architectures are supported. 138 #endif 139 aesni_enc(rounds - 1, tweak_schedule, tweak, tweak, NULL); 140 141 len /= AES_XTS_BLOCKSIZE; 142 for (i = 0; i < len; i++) { 143 aesni_crypt_xts_block(rounds, data_schedule, (uint64_t *)tweak, 144 (const uint64_t *)from, (uint64_t *)to, block, do_encrypt); 145 from += AES_XTS_BLOCKSIZE; 146 to += AES_XTS_BLOCKSIZE; 147 } 148 149 bzero(tweak, sizeof(tweak)); 150 bzero(block, sizeof(block)); 151 } 152 153 static void 154 aesni_encrypt_xts(int rounds, const void *data_schedule, 155 const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to, 156 const uint8_t iv[AES_BLOCK_LEN]) 157 { 158 159 aesni_crypt_xts(rounds, data_schedule, tweak_schedule, len, from, to, 160 iv, 1); 161 } 162 163 static void 164 aesni_decrypt_xts(int rounds, const void *data_schedule, 165 const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to, 166 const uint8_t iv[AES_BLOCK_LEN]) 167 { 168 169 aesni_crypt_xts(rounds, data_schedule, tweak_schedule, len, from, to, 170 iv, 0); 171 } 172 173 static int 174 aesni_cipher_setup_common(struct aesni_session *ses, const uint8_t *key, 175 int keylen) 176 { 177 178 switch (ses->algo) { 179 case CRYPTO_AES_CBC: 180 switch (keylen) { 181 case 128: 182 ses->rounds = AES128_ROUNDS; 183 break; 184 case 192: 185 ses->rounds = AES192_ROUNDS; 186 break; 187 case 256: 188 ses->rounds = AES256_ROUNDS; 189 break; 190 default: 191 return (EINVAL); 192 } 193 break; 194 case CRYPTO_AES_XTS: 195 switch (keylen) { 196 case 256: 197 ses->rounds = AES128_ROUNDS; 198 break; 199 case 512: 200 ses->rounds = AES256_ROUNDS; 201 break; 202 default: 203 return (EINVAL); 204 } 205 break; 206 default: 207 return (EINVAL); 208 } 209 210 aesni_set_enckey(key, ses->enc_schedule, ses->rounds); 211 aesni_set_deckey(ses->enc_schedule, ses->dec_schedule, ses->rounds); 212 if (ses->algo == CRYPTO_AES_CBC) 213 arc4rand(ses->iv, sizeof(ses->iv), 0); 214 else /* if (ses->algo == CRYPTO_AES_XTS) */ { 215 aesni_set_enckey(key + keylen / 16, ses->xts_schedule, 216 ses->rounds); 217 } 218 219 return (0); 220 } 221 222 int 223 aesni_cipher_setup(struct aesni_session *ses, struct cryptoini *encini) 224 { 225 struct thread *td; 226 int error, saved_ctx; 227 228 td = curthread; 229 if (!is_fpu_kern_thread(0)) { 230 error = fpu_kern_enter(td, &ses->fpu_ctx, FPU_KERN_NORMAL); 231 saved_ctx = 1; 232 } else { 233 error = 0; 234 saved_ctx = 0; 235 } 236 if (error == 0) { 237 error = aesni_cipher_setup_common(ses, encini->cri_key, 238 encini->cri_klen); 239 if (saved_ctx) 240 fpu_kern_leave(td, &ses->fpu_ctx); 241 } 242 return (error); 243 } 244 245 int 246 aesni_cipher_process(struct aesni_session *ses, struct cryptodesc *enccrd, 247 struct cryptop *crp) 248 { 249 struct thread *td; 250 uint8_t *buf; 251 int error, allocated, saved_ctx; 252 253 buf = aesni_cipher_alloc(enccrd, crp, &allocated); 254 if (buf == NULL) 255 return (ENOMEM); 256 257 td = curthread; 258 if (!is_fpu_kern_thread(0)) { 259 error = fpu_kern_enter(td, &ses->fpu_ctx, FPU_KERN_NORMAL); 260 if (error != 0) 261 goto out; 262 saved_ctx = 1; 263 } else { 264 saved_ctx = 0; 265 error = 0; 266 } 267 268 if ((enccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) { 269 error = aesni_cipher_setup_common(ses, enccrd->crd_key, 270 enccrd->crd_klen); 271 if (error != 0) 272 goto out; 273 } 274 275 if ((enccrd->crd_flags & CRD_F_ENCRYPT) != 0) { 276 if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0) 277 bcopy(enccrd->crd_iv, ses->iv, AES_BLOCK_LEN); 278 if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) 279 crypto_copyback(crp->crp_flags, crp->crp_buf, 280 enccrd->crd_inject, AES_BLOCK_LEN, ses->iv); 281 if (ses->algo == CRYPTO_AES_CBC) { 282 aesni_encrypt_cbc(ses->rounds, ses->enc_schedule, 283 enccrd->crd_len, buf, buf, ses->iv); 284 } else /* if (ses->algo == CRYPTO_AES_XTS) */ { 285 aesni_encrypt_xts(ses->rounds, ses->enc_schedule, 286 ses->xts_schedule, enccrd->crd_len, buf, buf, 287 ses->iv); 288 } 289 } else { 290 if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0) 291 bcopy(enccrd->crd_iv, ses->iv, AES_BLOCK_LEN); 292 else 293 crypto_copydata(crp->crp_flags, crp->crp_buf, 294 enccrd->crd_inject, AES_BLOCK_LEN, ses->iv); 295 if (ses->algo == CRYPTO_AES_CBC) { 296 aesni_decrypt_cbc(ses->rounds, ses->dec_schedule, 297 enccrd->crd_len, buf, ses->iv); 298 } else /* if (ses->algo == CRYPTO_AES_XTS) */ { 299 aesni_decrypt_xts(ses->rounds, ses->dec_schedule, 300 ses->xts_schedule, enccrd->crd_len, buf, buf, 301 ses->iv); 302 } 303 } 304 if (saved_ctx) 305 fpu_kern_leave(td, &ses->fpu_ctx); 306 if (allocated) 307 crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip, 308 enccrd->crd_len, buf); 309 if ((enccrd->crd_flags & CRD_F_ENCRYPT) != 0) 310 crypto_copydata(crp->crp_flags, crp->crp_buf, 311 enccrd->crd_skip + enccrd->crd_len - AES_BLOCK_LEN, 312 AES_BLOCK_LEN, ses->iv); 313 out: 314 if (allocated) { 315 bzero(buf, enccrd->crd_len); 316 free(buf, M_AESNI); 317 } 318 return (error); 319 } 320