xref: /freebsd/sys/crypto/aesni/aesni_wrap.c (revision bb15ca603fa442c72dde3f3cb8b46db6970e3950)
1 /*-
2  * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org>
3  * Copyright (c) 2010-2011 Pawel Jakub Dawidek <pawel@dawidek.net>
4  * All rights reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25  * SUCH DAMAGE.
26  */
27 
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
30 
31 #include <sys/param.h>
32 #include <sys/libkern.h>
33 #include <sys/malloc.h>
34 #include <sys/proc.h>
35 #include <sys/systm.h>
36 #include <crypto/aesni/aesni.h>
37 
38 MALLOC_DECLARE(M_AESNI);
39 
40 void
41 aesni_encrypt_cbc(int rounds, const void *key_schedule, size_t len,
42     const uint8_t *from, uint8_t *to, const uint8_t iv[AES_BLOCK_LEN])
43 {
44 	const uint8_t *ivp;
45 	size_t i;
46 
47 	len /= AES_BLOCK_LEN;
48 	ivp = iv;
49 	for (i = 0; i < len; i++) {
50 		aesni_enc(rounds - 1, key_schedule, from, to, ivp);
51 		ivp = to;
52 		from += AES_BLOCK_LEN;
53 		to += AES_BLOCK_LEN;
54 	}
55 }
56 
57 void
58 aesni_encrypt_ecb(int rounds, const void *key_schedule, size_t len,
59     const uint8_t from[AES_BLOCK_LEN], uint8_t to[AES_BLOCK_LEN])
60 {
61 	size_t i;
62 
63 	len /= AES_BLOCK_LEN;
64 	for (i = 0; i < len; i++) {
65 		aesni_enc(rounds - 1, key_schedule, from, to, NULL);
66 		from += AES_BLOCK_LEN;
67 		to += AES_BLOCK_LEN;
68 	}
69 }
70 
71 void
72 aesni_decrypt_ecb(int rounds, const void *key_schedule, size_t len,
73     const uint8_t from[AES_BLOCK_LEN], uint8_t to[AES_BLOCK_LEN])
74 {
75 	size_t i;
76 
77 	len /= AES_BLOCK_LEN;
78 	for (i = 0; i < len; i++) {
79 		aesni_dec(rounds - 1, key_schedule, from, to, NULL);
80 		from += AES_BLOCK_LEN;
81 		to += AES_BLOCK_LEN;
82 	}
83 }
84 
85 #define	AES_XTS_BLOCKSIZE	16
86 #define	AES_XTS_IVSIZE		8
87 #define	AES_XTS_ALPHA		0x87	/* GF(2^128) generator polynomial */
88 
89 static void
90 aesni_crypt_xts_block(int rounds, const void *key_schedule, uint64_t *tweak,
91     const uint64_t *from, uint64_t *to, uint64_t *block, int do_encrypt)
92 {
93 	int carry;
94 
95 	block[0] = from[0] ^ tweak[0];
96 	block[1] = from[1] ^ tweak[1];
97 
98 	if (do_encrypt)
99 		aesni_enc(rounds - 1, key_schedule, (uint8_t *)block, (uint8_t *)to, NULL);
100 	else
101 		aesni_dec(rounds - 1, key_schedule, (uint8_t *)block, (uint8_t *)to, NULL);
102 
103 	to[0] ^= tweak[0];
104 	to[1] ^= tweak[1];
105 
106 	/* Exponentiate tweak. */
107 	carry = ((tweak[0] & 0x8000000000000000ULL) > 0);
108 	tweak[0] <<= 1;
109 	if (tweak[1] & 0x8000000000000000ULL) {
110 		uint8_t *twk = (uint8_t *)tweak;
111 
112 		twk[0] ^= AES_XTS_ALPHA;
113 	}
114 	tweak[1] <<= 1;
115 	if (carry)
116 		tweak[1] |= 1;
117 }
118 
119 static void
120 aesni_crypt_xts(int rounds, const void *data_schedule,
121     const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to,
122     const uint8_t iv[AES_BLOCK_LEN], int do_encrypt)
123 {
124 	uint64_t block[AES_XTS_BLOCKSIZE / 8];
125 	uint8_t tweak[AES_XTS_BLOCKSIZE];
126 	size_t i;
127 
128 	/*
129 	 * Prepare tweak as E_k2(IV). IV is specified as LE representation
130 	 * of a 64-bit block number which we allow to be passed in directly.
131 	 */
132 #if BYTE_ORDER == LITTLE_ENDIAN
133 	bcopy(iv, tweak, AES_XTS_IVSIZE);
134 	/* Last 64 bits of IV are always zero. */
135 	bzero(tweak + AES_XTS_IVSIZE, AES_XTS_IVSIZE);
136 #else
137 #error Only LITTLE_ENDIAN architectures are supported.
138 #endif
139 	aesni_enc(rounds - 1, tweak_schedule, tweak, tweak, NULL);
140 
141 	len /= AES_XTS_BLOCKSIZE;
142 	for (i = 0; i < len; i++) {
143 		aesni_crypt_xts_block(rounds, data_schedule, (uint64_t *)tweak,
144 		    (const uint64_t *)from, (uint64_t *)to, block, do_encrypt);
145 		from += AES_XTS_BLOCKSIZE;
146 		to += AES_XTS_BLOCKSIZE;
147 	}
148 
149 	bzero(tweak, sizeof(tweak));
150 	bzero(block, sizeof(block));
151 }
152 
153 static void
154 aesni_encrypt_xts(int rounds, const void *data_schedule,
155     const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to,
156     const uint8_t iv[AES_BLOCK_LEN])
157 {
158 
159 	aesni_crypt_xts(rounds, data_schedule, tweak_schedule, len, from, to,
160 	    iv, 1);
161 }
162 
163 static void
164 aesni_decrypt_xts(int rounds, const void *data_schedule,
165     const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to,
166     const uint8_t iv[AES_BLOCK_LEN])
167 {
168 
169 	aesni_crypt_xts(rounds, data_schedule, tweak_schedule, len, from, to,
170 	    iv, 0);
171 }
172 
173 static int
174 aesni_cipher_setup_common(struct aesni_session *ses, const uint8_t *key,
175     int keylen)
176 {
177 
178 	switch (ses->algo) {
179 	case CRYPTO_AES_CBC:
180 		switch (keylen) {
181 		case 128:
182 			ses->rounds = AES128_ROUNDS;
183 			break;
184 		case 192:
185 			ses->rounds = AES192_ROUNDS;
186 			break;
187 		case 256:
188 			ses->rounds = AES256_ROUNDS;
189 			break;
190 		default:
191 			return (EINVAL);
192 		}
193 		break;
194 	case CRYPTO_AES_XTS:
195 		switch (keylen) {
196 		case 256:
197 			ses->rounds = AES128_ROUNDS;
198 			break;
199 		case 512:
200 			ses->rounds = AES256_ROUNDS;
201 			break;
202 		default:
203 			return (EINVAL);
204 		}
205 		break;
206 	default:
207 		return (EINVAL);
208 	}
209 
210 	aesni_set_enckey(key, ses->enc_schedule, ses->rounds);
211 	aesni_set_deckey(ses->enc_schedule, ses->dec_schedule, ses->rounds);
212 	if (ses->algo == CRYPTO_AES_CBC)
213 		arc4rand(ses->iv, sizeof(ses->iv), 0);
214 	else /* if (ses->algo == CRYPTO_AES_XTS) */ {
215 		aesni_set_enckey(key + keylen / 16, ses->xts_schedule,
216 		    ses->rounds);
217 	}
218 
219 	return (0);
220 }
221 
222 int
223 aesni_cipher_setup(struct aesni_session *ses, struct cryptoini *encini)
224 {
225 	struct thread *td;
226 	int error, saved_ctx;
227 
228 	td = curthread;
229 	if (!is_fpu_kern_thread(0)) {
230 		error = fpu_kern_enter(td, &ses->fpu_ctx, FPU_KERN_NORMAL);
231 		saved_ctx = 1;
232 	} else {
233 		error = 0;
234 		saved_ctx = 0;
235 	}
236 	if (error == 0) {
237 		error = aesni_cipher_setup_common(ses, encini->cri_key,
238 		    encini->cri_klen);
239 		if (saved_ctx)
240 			fpu_kern_leave(td, &ses->fpu_ctx);
241 	}
242 	return (error);
243 }
244 
245 int
246 aesni_cipher_process(struct aesni_session *ses, struct cryptodesc *enccrd,
247     struct cryptop *crp)
248 {
249 	struct thread *td;
250 	uint8_t *buf;
251 	int error, allocated, saved_ctx;
252 
253 	buf = aesni_cipher_alloc(enccrd, crp, &allocated);
254 	if (buf == NULL)
255 		return (ENOMEM);
256 
257 	td = curthread;
258 	if (!is_fpu_kern_thread(0)) {
259 		error = fpu_kern_enter(td, &ses->fpu_ctx, FPU_KERN_NORMAL);
260 		if (error != 0)
261 			goto out;
262 		saved_ctx = 1;
263 	} else {
264 		saved_ctx = 0;
265 		error = 0;
266 	}
267 
268 	if ((enccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) {
269 		error = aesni_cipher_setup_common(ses, enccrd->crd_key,
270 		    enccrd->crd_klen);
271 		if (error != 0)
272 			goto out;
273 	}
274 
275 	if ((enccrd->crd_flags & CRD_F_ENCRYPT) != 0) {
276 		if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
277 			bcopy(enccrd->crd_iv, ses->iv, AES_BLOCK_LEN);
278 		if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0)
279 			crypto_copyback(crp->crp_flags, crp->crp_buf,
280 			    enccrd->crd_inject, AES_BLOCK_LEN, ses->iv);
281 		if (ses->algo == CRYPTO_AES_CBC) {
282 			aesni_encrypt_cbc(ses->rounds, ses->enc_schedule,
283 			    enccrd->crd_len, buf, buf, ses->iv);
284 		} else /* if (ses->algo == CRYPTO_AES_XTS) */ {
285 			aesni_encrypt_xts(ses->rounds, ses->enc_schedule,
286 			    ses->xts_schedule, enccrd->crd_len, buf, buf,
287 			    ses->iv);
288 		}
289 	} else {
290 		if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
291 			bcopy(enccrd->crd_iv, ses->iv, AES_BLOCK_LEN);
292 		else
293 			crypto_copydata(crp->crp_flags, crp->crp_buf,
294 			    enccrd->crd_inject, AES_BLOCK_LEN, ses->iv);
295 		if (ses->algo == CRYPTO_AES_CBC) {
296 			aesni_decrypt_cbc(ses->rounds, ses->dec_schedule,
297 			    enccrd->crd_len, buf, ses->iv);
298 		} else /* if (ses->algo == CRYPTO_AES_XTS) */ {
299 			aesni_decrypt_xts(ses->rounds, ses->dec_schedule,
300 			    ses->xts_schedule, enccrd->crd_len, buf, buf,
301 			    ses->iv);
302 		}
303 	}
304 	if (saved_ctx)
305 		fpu_kern_leave(td, &ses->fpu_ctx);
306 	if (allocated)
307 		crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
308 		    enccrd->crd_len, buf);
309 	if ((enccrd->crd_flags & CRD_F_ENCRYPT) != 0)
310 		crypto_copydata(crp->crp_flags, crp->crp_buf,
311 		    enccrd->crd_skip + enccrd->crd_len - AES_BLOCK_LEN,
312 		    AES_BLOCK_LEN, ses->iv);
313  out:
314 	if (allocated) {
315 		bzero(buf, enccrd->crd_len);
316 		free(buf, M_AESNI);
317 	}
318 	return (error);
319 }
320