1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2020 Netflix, Inc
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer,
11 * without modification.
12 * 2. Redistributions in binary form must reproduce at minimum a disclaimer
13 * similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any
14 * redistribution must be conditioned upon including a substantially
15 * similar Disclaimer requirement for further binary redistribution.
16 *
17 * NO WARRANTY
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY
21 * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
22 * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY,
23 * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
26 * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
28 * THE POSSIBILITY OF SUCH DAMAGES.
29 */
30
31 /*
32 * A driver for the OpenCrypto framework which uses assembly routines
33 * from OpenSSL.
34 */
35
36 #include <sys/types.h>
37 #include <sys/bus.h>
38 #include <sys/kernel.h>
39 #include <sys/malloc.h>
40 #include <sys/module.h>
41
42 #include <machine/fpu.h>
43
44 #include <opencrypto/cryptodev.h>
45 #include <opencrypto/xform_auth.h>
46
47 #include <crypto/openssl/ossl.h>
48 #include <crypto/openssl/ossl_chacha.h>
49 #include <crypto/openssl/ossl_cipher.h>
50
51 #include "cryptodev_if.h"
52
53 static MALLOC_DEFINE(M_OSSL, "ossl", "OpenSSL crypto");
54
55 static void
ossl_identify(driver_t * driver,device_t parent)56 ossl_identify(driver_t *driver, device_t parent)
57 {
58
59 if (device_find_child(parent, "ossl", DEVICE_UNIT_ANY) == NULL)
60 BUS_ADD_CHILD(parent, 10, "ossl", DEVICE_UNIT_ANY);
61 }
62
63 static int
ossl_probe(device_t dev)64 ossl_probe(device_t dev)
65 {
66
67 device_set_desc(dev, "OpenSSL crypto");
68 return (BUS_PROBE_DEFAULT);
69 }
70
71 static int
ossl_attach(device_t dev)72 ossl_attach(device_t dev)
73 {
74 struct ossl_softc *sc;
75
76 sc = device_get_softc(dev);
77
78 sc->has_aes = sc->has_aes_gcm = false;
79
80 ossl_cpuid(sc);
81 sc->sc_cid = crypto_get_driverid(dev, sizeof(struct ossl_session),
82 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC |
83 CRYPTOCAP_F_ACCEL_SOFTWARE);
84 if (sc->sc_cid < 0) {
85 device_printf(dev, "failed to allocate crypto driver id\n");
86 return (ENXIO);
87 }
88
89 return (0);
90 }
91
92 static int
ossl_detach(device_t dev)93 ossl_detach(device_t dev)
94 {
95 struct ossl_softc *sc;
96
97 sc = device_get_softc(dev);
98
99 crypto_unregister_all(sc->sc_cid);
100
101 return (0);
102 }
103
104 static struct auth_hash *
ossl_lookup_hash(const struct crypto_session_params * csp)105 ossl_lookup_hash(const struct crypto_session_params *csp)
106 {
107
108 switch (csp->csp_auth_alg) {
109 case CRYPTO_SHA1:
110 case CRYPTO_SHA1_HMAC:
111 return (&ossl_hash_sha1);
112 case CRYPTO_SHA2_224:
113 case CRYPTO_SHA2_224_HMAC:
114 return (&ossl_hash_sha224);
115 case CRYPTO_SHA2_256:
116 case CRYPTO_SHA2_256_HMAC:
117 return (&ossl_hash_sha256);
118 case CRYPTO_SHA2_384:
119 case CRYPTO_SHA2_384_HMAC:
120 return (&ossl_hash_sha384);
121 case CRYPTO_SHA2_512:
122 case CRYPTO_SHA2_512_HMAC:
123 return (&ossl_hash_sha512);
124 case CRYPTO_POLY1305:
125 return (&ossl_hash_poly1305);
126 default:
127 return (NULL);
128 }
129 }
130
131 static struct ossl_cipher*
ossl_lookup_cipher(const struct crypto_session_params * csp)132 ossl_lookup_cipher(const struct crypto_session_params *csp)
133 {
134
135 switch (csp->csp_cipher_alg) {
136 case CRYPTO_AES_CBC:
137 switch (csp->csp_cipher_klen * 8) {
138 case 128:
139 case 192:
140 case 256:
141 break;
142 default:
143 return (NULL);
144 }
145 return (&ossl_cipher_aes_cbc);
146 case CRYPTO_AES_NIST_GCM_16:
147 switch (csp->csp_cipher_klen * 8) {
148 case 128:
149 case 192:
150 case 256:
151 break;
152 default:
153 return (NULL);
154 }
155 return (&ossl_cipher_aes_gcm);
156 case CRYPTO_CHACHA20:
157 if (csp->csp_cipher_klen != CHACHA_KEY_SIZE)
158 return (NULL);
159 return (&ossl_cipher_chacha20);
160 default:
161 return (NULL);
162 }
163 }
164
165 static int
ossl_probesession(device_t dev,const struct crypto_session_params * csp)166 ossl_probesession(device_t dev, const struct crypto_session_params *csp)
167 {
168 struct ossl_softc *sc = device_get_softc(dev);
169
170 if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)) !=
171 0)
172 return (EINVAL);
173 switch (csp->csp_mode) {
174 case CSP_MODE_DIGEST:
175 if (ossl_lookup_hash(csp) == NULL)
176 return (EINVAL);
177 break;
178 case CSP_MODE_CIPHER:
179 if (csp->csp_cipher_alg != CRYPTO_CHACHA20 && !sc->has_aes)
180 return (EINVAL);
181 if (ossl_lookup_cipher(csp) == NULL)
182 return (EINVAL);
183 break;
184 case CSP_MODE_ETA:
185 if (!sc->has_aes ||
186 csp->csp_cipher_alg == CRYPTO_CHACHA20 ||
187 ossl_lookup_hash(csp) == NULL ||
188 ossl_lookup_cipher(csp) == NULL)
189 return (EINVAL);
190 break;
191 case CSP_MODE_AEAD:
192 switch (csp->csp_cipher_alg) {
193 case CRYPTO_CHACHA20_POLY1305:
194 break;
195 case CRYPTO_AES_NIST_GCM_16:
196 if (!sc->has_aes_gcm || ossl_lookup_cipher(csp) == NULL)
197 return (EINVAL);
198 if (csp->csp_ivlen != AES_GCM_IV_LEN)
199 return (EINVAL);
200 if (csp->csp_auth_mlen != 0 &&
201 csp->csp_auth_mlen != GMAC_DIGEST_LEN)
202 return (EINVAL);
203 break;
204 default:
205 return (EINVAL);
206 }
207 break;
208 default:
209 return (EINVAL);
210 }
211
212 return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
213 }
214
215 static void
ossl_newsession_hash(struct ossl_session * s,const struct crypto_session_params * csp)216 ossl_newsession_hash(struct ossl_session *s,
217 const struct crypto_session_params *csp)
218 {
219 struct auth_hash *axf;
220
221 axf = ossl_lookup_hash(csp);
222 s->hash.axf = axf;
223 if (csp->csp_auth_mlen == 0)
224 s->hash.mlen = axf->hashsize;
225 else
226 s->hash.mlen = csp->csp_auth_mlen;
227
228 if (csp->csp_auth_klen == 0) {
229 axf->Init(&s->hash.ictx);
230 } else {
231 if (csp->csp_auth_key != NULL) {
232 fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX);
233 if (axf->Setkey != NULL) {
234 axf->Init(&s->hash.ictx);
235 axf->Setkey(&s->hash.ictx, csp->csp_auth_key,
236 csp->csp_auth_klen);
237 } else {
238 hmac_init_ipad(axf, csp->csp_auth_key,
239 csp->csp_auth_klen, &s->hash.ictx);
240 hmac_init_opad(axf, csp->csp_auth_key,
241 csp->csp_auth_klen, &s->hash.octx);
242 }
243 fpu_kern_leave(curthread, NULL);
244 }
245 }
246 }
247
248 static int
ossl_newsession_cipher(struct ossl_session * s,const struct crypto_session_params * csp)249 ossl_newsession_cipher(struct ossl_session *s,
250 const struct crypto_session_params *csp)
251 {
252 struct ossl_cipher *cipher;
253 int error = 0;
254
255 cipher = ossl_lookup_cipher(csp);
256 if (cipher == NULL)
257 return (EINVAL);
258
259 s->cipher.cipher = cipher;
260
261 if (csp->csp_cipher_key == NULL)
262 return (0);
263
264 fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX);
265 if (cipher->set_encrypt_key != NULL) {
266 error = cipher->set_encrypt_key(csp->csp_cipher_key,
267 8 * csp->csp_cipher_klen, &s->cipher.enc_ctx);
268 if (error != 0) {
269 fpu_kern_leave(curthread, NULL);
270 return (error);
271 }
272 }
273 if (cipher->set_decrypt_key != NULL)
274 error = cipher->set_decrypt_key(csp->csp_cipher_key,
275 8 * csp->csp_cipher_klen, &s->cipher.dec_ctx);
276 fpu_kern_leave(curthread, NULL);
277
278 return (error);
279 }
280
281 static int
ossl_newsession(device_t dev,crypto_session_t cses,const struct crypto_session_params * csp)282 ossl_newsession(device_t dev, crypto_session_t cses,
283 const struct crypto_session_params *csp)
284 {
285 struct ossl_session *s;
286 int error = 0;
287
288 s = crypto_get_driver_session(cses);
289 switch (csp->csp_mode) {
290 case CSP_MODE_DIGEST:
291 ossl_newsession_hash(s, csp);
292 break;
293 case CSP_MODE_CIPHER:
294 error = ossl_newsession_cipher(s, csp);
295 break;
296 case CSP_MODE_ETA:
297 ossl_newsession_hash(s, csp);
298 error = ossl_newsession_cipher(s, csp);
299 break;
300 case CSP_MODE_AEAD:
301 if (csp->csp_cipher_alg != CRYPTO_CHACHA20_POLY1305)
302 error = ossl_newsession_cipher(s, csp);
303 break;
304 default:
305 __assert_unreachable();
306 }
307
308 return (error);
309 }
310
311 static int
ossl_process_hash(struct ossl_session * s,struct cryptop * crp,const struct crypto_session_params * csp)312 ossl_process_hash(struct ossl_session *s, struct cryptop *crp,
313 const struct crypto_session_params *csp)
314 {
315 struct ossl_hash_context ctx;
316 char digest[HASH_MAX_LEN];
317 struct auth_hash *axf;
318 int error;
319
320 axf = s->hash.axf;
321
322 if (crp->crp_auth_key == NULL) {
323 ctx = s->hash.ictx;
324 } else {
325 if (axf->Setkey != NULL) {
326 axf->Init(&ctx);
327 axf->Setkey(&ctx, crp->crp_auth_key,
328 csp->csp_auth_klen);
329 } else {
330 hmac_init_ipad(axf, crp->crp_auth_key,
331 csp->csp_auth_klen, &ctx);
332 }
333 }
334
335 if (crp->crp_aad != NULL)
336 error = axf->Update(&ctx, crp->crp_aad, crp->crp_aad_length);
337 else
338 error = crypto_apply(crp, crp->crp_aad_start,
339 crp->crp_aad_length, axf->Update, &ctx);
340 if (error)
341 goto out;
342
343 error = crypto_apply(crp, crp->crp_payload_start,
344 crp->crp_payload_length, axf->Update, &ctx);
345 if (error)
346 goto out;
347
348 axf->Final(digest, &ctx);
349
350 if (csp->csp_auth_klen != 0 && axf->Setkey == NULL) {
351 if (crp->crp_auth_key == NULL)
352 ctx = s->hash.octx;
353 else
354 hmac_init_opad(axf, crp->crp_auth_key,
355 csp->csp_auth_klen, &ctx);
356 axf->Update(&ctx, digest, axf->hashsize);
357 axf->Final(digest, &ctx);
358 }
359
360 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
361 char digest2[HASH_MAX_LEN];
362
363 crypto_copydata(crp, crp->crp_digest_start, s->hash.mlen,
364 digest2);
365 if (timingsafe_bcmp(digest, digest2, s->hash.mlen) != 0)
366 error = EBADMSG;
367 explicit_bzero(digest2, sizeof(digest2));
368 } else {
369 crypto_copyback(crp, crp->crp_digest_start, s->hash.mlen,
370 digest);
371 }
372 explicit_bzero(digest, sizeof(digest));
373
374 out:
375 explicit_bzero(&ctx, sizeof(ctx));
376 return (error);
377 }
378
379 static int
ossl_process_cipher(struct ossl_session * s,struct cryptop * crp,const struct crypto_session_params * csp)380 ossl_process_cipher(struct ossl_session *s, struct cryptop *crp,
381 const struct crypto_session_params *csp)
382 {
383 return (s->cipher.cipher->process(&s->cipher, crp, csp));
384 }
385
386 static int
ossl_process_eta(struct ossl_session * s,struct cryptop * crp,const struct crypto_session_params * csp)387 ossl_process_eta(struct ossl_session *s, struct cryptop *crp,
388 const struct crypto_session_params *csp)
389 {
390 int error;
391
392 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
393 error = s->cipher.cipher->process(&s->cipher, crp, csp);
394 if (error == 0)
395 error = ossl_process_hash(s, crp, csp);
396 } else {
397 error = ossl_process_hash(s, crp, csp);
398 if (error == 0)
399 error = s->cipher.cipher->process(&s->cipher, crp, csp);
400 }
401
402 return (error);
403 }
404
405 static int
ossl_process_aead(struct ossl_session * s,struct cryptop * crp,const struct crypto_session_params * csp)406 ossl_process_aead(struct ossl_session *s, struct cryptop *crp,
407 const struct crypto_session_params *csp)
408 {
409 if (csp->csp_cipher_alg == CRYPTO_CHACHA20_POLY1305) {
410 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
411 return (ossl_chacha20_poly1305_encrypt(crp, csp));
412 else
413 return (ossl_chacha20_poly1305_decrypt(crp, csp));
414 } else {
415 return (s->cipher.cipher->process(&s->cipher, crp, csp));
416 }
417 }
418
419 static int
ossl_process(device_t dev,struct cryptop * crp,int hint)420 ossl_process(device_t dev, struct cryptop *crp, int hint)
421 {
422 const struct crypto_session_params *csp;
423 struct ossl_session *s;
424 int error;
425 bool fpu_entered;
426
427 s = crypto_get_driver_session(crp->crp_session);
428 csp = crypto_get_params(crp->crp_session);
429
430 if (is_fpu_kern_thread(0)) {
431 fpu_entered = false;
432 } else {
433 fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX);
434 fpu_entered = true;
435 }
436
437 switch (csp->csp_mode) {
438 case CSP_MODE_DIGEST:
439 error = ossl_process_hash(s, crp, csp);
440 break;
441 case CSP_MODE_CIPHER:
442 error = ossl_process_cipher(s, crp, csp);
443 break;
444 case CSP_MODE_ETA:
445 error = ossl_process_eta(s, crp, csp);
446 break;
447 case CSP_MODE_AEAD:
448 error = ossl_process_aead(s, crp, csp);
449 break;
450 default:
451 __assert_unreachable();
452 }
453
454 if (fpu_entered)
455 fpu_kern_leave(curthread, NULL);
456
457 crp->crp_etype = error;
458 crypto_done(crp);
459
460 return (0);
461 }
462
463 static device_method_t ossl_methods[] = {
464 DEVMETHOD(device_identify, ossl_identify),
465 DEVMETHOD(device_probe, ossl_probe),
466 DEVMETHOD(device_attach, ossl_attach),
467 DEVMETHOD(device_detach, ossl_detach),
468
469 DEVMETHOD(cryptodev_probesession, ossl_probesession),
470 DEVMETHOD(cryptodev_newsession, ossl_newsession),
471 DEVMETHOD(cryptodev_process, ossl_process),
472
473 DEVMETHOD_END
474 };
475
476 static driver_t ossl_driver = {
477 "ossl",
478 ossl_methods,
479 sizeof(struct ossl_softc)
480 };
481
482 DRIVER_MODULE(ossl, nexus, ossl_driver, NULL, NULL);
483 MODULE_VERSION(ossl, 1);
484 MODULE_DEPEND(ossl, crypto, 1, 1, 1);
485