1 /*- 2 * Copyright (c) 2018 Conrad Meyer <cem@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 #include <sys/param.h> 29 #include <sys/bus.h> 30 #include <sys/kernel.h> 31 #include <sys/kobj.h> 32 #include <sys/lock.h> 33 #include <sys/malloc.h> 34 #include <sys/module.h> 35 #include <sys/mutex.h> 36 #include <sys/rwlock.h> 37 #include <sys/smp.h> 38 39 #include <blake2.h> 40 41 #include <opencrypto/cryptodev.h> 42 #include <cryptodev_if.h> 43 44 #include <machine/fpu.h> 45 46 struct blake2_session { 47 size_t mlen; 48 }; 49 CTASSERT((size_t)BLAKE2B_KEYBYTES > (size_t)BLAKE2S_KEYBYTES); 50 51 struct blake2_softc { 52 bool dying; 53 int32_t cid; 54 struct rwlock lock; 55 }; 56 57 static struct mtx_padalign *ctx_mtx; 58 static struct fpu_kern_ctx **ctx_fpu; 59 60 #define ACQUIRE_CTX(i, ctx) \ 61 do { \ 62 (i) = PCPU_GET(cpuid); \ 63 mtx_lock(&ctx_mtx[(i)]); \ 64 (ctx) = ctx_fpu[(i)]; \ 65 } while (0) 66 #define RELEASE_CTX(i, ctx) \ 67 do { \ 68 mtx_unlock(&ctx_mtx[(i)]); \ 69 (i) = -1; \ 70 (ctx) = NULL; \ 71 } while (0) 72 73 static int blake2_cipher_setup(struct blake2_session *ses, 74 const struct crypto_session_params *csp); 75 static int blake2_cipher_process(struct blake2_session *ses, 76 struct cryptop *crp); 77 78 MALLOC_DEFINE(M_BLAKE2, "blake2_data", "Blake2 Data"); 79 80 static void 81 blake2_identify(driver_t *drv, device_t parent) 82 { 83 84 /* NB: order 10 is so we get attached after h/w devices */ 85 if (device_find_child(parent, "blaketwo", -1) == NULL && 86 BUS_ADD_CHILD(parent, 10, "blaketwo", -1) == 0) 87 panic("blaketwo: could not attach"); 88 } 89 90 static int 91 blake2_probe(device_t dev) 92 { 93 device_set_desc(dev, "Blake2"); 94 return (0); 95 } 96 97 static void 98 blake2_cleanctx(void) 99 { 100 int i; 101 102 /* XXX - no way to return driverid */ 103 CPU_FOREACH(i) { 104 if (ctx_fpu[i] != NULL) { 105 mtx_destroy(&ctx_mtx[i]); 106 fpu_kern_free_ctx(ctx_fpu[i]); 107 } 108 ctx_fpu[i] = NULL; 109 } 110 free(ctx_mtx, M_BLAKE2); 111 ctx_mtx = NULL; 112 free(ctx_fpu, M_BLAKE2); 113 ctx_fpu = NULL; 114 } 115 116 static int 117 blake2_attach(device_t dev) 118 { 119 struct blake2_softc *sc; 120 int i; 121 122 sc = device_get_softc(dev); 123 sc->dying = false; 124 125 sc->cid = crypto_get_driverid(dev, sizeof(struct blake2_session), 126 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC | 127 CRYPTOCAP_F_ACCEL_SOFTWARE); 128 if (sc->cid < 0) { 129 device_printf(dev, "Could not get crypto driver id.\n"); 130 return (ENOMEM); 131 } 132 133 ctx_mtx = malloc(sizeof(*ctx_mtx) * (mp_maxid + 1), M_BLAKE2, 134 M_WAITOK | M_ZERO); 135 ctx_fpu = malloc(sizeof(*ctx_fpu) * (mp_maxid + 1), M_BLAKE2, 136 M_WAITOK | M_ZERO); 137 138 CPU_FOREACH(i) { 139 #ifdef __amd64__ 140 ctx_fpu[i] = fpu_kern_alloc_ctx_domain( 141 pcpu_find(i)->pc_domain, FPU_KERN_NORMAL); 142 #else 143 ctx_fpu[i] = fpu_kern_alloc_ctx(FPU_KERN_NORMAL); 144 #endif 145 mtx_init(&ctx_mtx[i], "bl2fpumtx", NULL, MTX_DEF | MTX_NEW); 146 } 147 148 rw_init(&sc->lock, "blake2_lock"); 149 150 return (0); 151 } 152 153 static int 154 blake2_detach(device_t dev) 155 { 156 struct blake2_softc *sc; 157 158 sc = device_get_softc(dev); 159 160 rw_wlock(&sc->lock); 161 sc->dying = true; 162 rw_wunlock(&sc->lock); 163 crypto_unregister_all(sc->cid); 164 165 rw_destroy(&sc->lock); 166 167 blake2_cleanctx(); 168 169 return (0); 170 } 171 172 static int 173 blake2_probesession(device_t dev, const struct crypto_session_params *csp) 174 { 175 176 if (csp->csp_flags != 0) 177 return (EINVAL); 178 switch (csp->csp_mode) { 179 case CSP_MODE_DIGEST: 180 switch (csp->csp_auth_alg) { 181 case CRYPTO_BLAKE2B: 182 case CRYPTO_BLAKE2S: 183 break; 184 default: 185 return (EINVAL); 186 } 187 break; 188 default: 189 return (EINVAL); 190 } 191 return (CRYPTODEV_PROBE_ACCEL_SOFTWARE); 192 } 193 194 static int 195 blake2_newsession(device_t dev, crypto_session_t cses, 196 const struct crypto_session_params *csp) 197 { 198 struct blake2_softc *sc; 199 struct blake2_session *ses; 200 int error; 201 202 sc = device_get_softc(dev); 203 204 ses = crypto_get_driver_session(cses); 205 206 rw_rlock(&sc->lock); 207 if (sc->dying) { 208 rw_runlock(&sc->lock); 209 return (EINVAL); 210 } 211 rw_runlock(&sc->lock); 212 213 error = blake2_cipher_setup(ses, csp); 214 if (error != 0) { 215 CRYPTDEB("setup failed"); 216 return (error); 217 } 218 219 return (0); 220 } 221 222 static int 223 blake2_process(device_t dev, struct cryptop *crp, int hint __unused) 224 { 225 struct blake2_session *ses; 226 int error; 227 228 ses = crypto_get_driver_session(crp->crp_session); 229 error = blake2_cipher_process(ses, crp); 230 231 crp->crp_etype = error; 232 crypto_done(crp); 233 return (0); 234 } 235 236 static device_method_t blake2_methods[] = { 237 DEVMETHOD(device_identify, blake2_identify), 238 DEVMETHOD(device_probe, blake2_probe), 239 DEVMETHOD(device_attach, blake2_attach), 240 DEVMETHOD(device_detach, blake2_detach), 241 242 DEVMETHOD(cryptodev_probesession, blake2_probesession), 243 DEVMETHOD(cryptodev_newsession, blake2_newsession), 244 DEVMETHOD(cryptodev_process, blake2_process), 245 246 DEVMETHOD_END 247 }; 248 249 static driver_t blake2_driver = { 250 "blaketwo", 251 blake2_methods, 252 sizeof(struct blake2_softc), 253 }; 254 255 DRIVER_MODULE(blake2, nexus, blake2_driver, 0, 0); 256 MODULE_VERSION(blake2, 1); 257 MODULE_DEPEND(blake2, crypto, 1, 1, 1); 258 259 static bool 260 blake2_check_klen(const struct crypto_session_params *csp, unsigned klen) 261 { 262 263 if (csp->csp_auth_alg == CRYPTO_BLAKE2S) 264 return (klen <= BLAKE2S_KEYBYTES); 265 else 266 return (klen <= BLAKE2B_KEYBYTES); 267 } 268 269 static int 270 blake2_cipher_setup(struct blake2_session *ses, 271 const struct crypto_session_params *csp) 272 { 273 int hashlen; 274 275 CTASSERT((size_t)BLAKE2S_OUTBYTES <= (size_t)BLAKE2B_OUTBYTES); 276 277 if (!blake2_check_klen(csp, csp->csp_auth_klen)) 278 return (EINVAL); 279 280 if (csp->csp_auth_mlen < 0) 281 return (EINVAL); 282 283 switch (csp->csp_auth_alg) { 284 case CRYPTO_BLAKE2S: 285 hashlen = BLAKE2S_OUTBYTES; 286 break; 287 case CRYPTO_BLAKE2B: 288 hashlen = BLAKE2B_OUTBYTES; 289 break; 290 default: 291 return (EINVAL); 292 } 293 294 if (csp->csp_auth_mlen > hashlen) 295 return (EINVAL); 296 297 if (csp->csp_auth_mlen == 0) 298 ses->mlen = hashlen; 299 else 300 ses->mlen = csp->csp_auth_mlen; 301 return (0); 302 } 303 304 static int 305 blake2b_applicator(void *state, const void *buf, u_int len) 306 { 307 int rc; 308 309 rc = blake2b_update(state, buf, len); 310 if (rc != 0) 311 return (EINVAL); 312 return (0); 313 } 314 315 static int 316 blake2s_applicator(void *state, const void *buf, u_int len) 317 { 318 int rc; 319 320 rc = blake2s_update(state, buf, len); 321 if (rc != 0) 322 return (EINVAL); 323 return (0); 324 } 325 326 static int 327 blake2_cipher_process(struct blake2_session *ses, struct cryptop *crp) 328 { 329 union { 330 blake2b_state sb; 331 blake2s_state ss; 332 } bctx; 333 char res[BLAKE2B_OUTBYTES], res2[BLAKE2B_OUTBYTES]; 334 const struct crypto_session_params *csp; 335 struct fpu_kern_ctx *ctx; 336 const void *key; 337 int ctxidx; 338 bool kt; 339 int error, rc; 340 unsigned klen; 341 342 ctx = NULL; 343 ctxidx = 0; 344 error = EINVAL; 345 346 kt = is_fpu_kern_thread(0); 347 if (!kt) { 348 ACQUIRE_CTX(ctxidx, ctx); 349 fpu_kern_enter(curthread, ctx, 350 FPU_KERN_NORMAL | FPU_KERN_KTHR); 351 } 352 353 csp = crypto_get_params(crp->crp_session); 354 if (crp->crp_auth_key != NULL) 355 key = crp->crp_auth_key; 356 else 357 key = csp->csp_auth_key; 358 klen = csp->csp_auth_klen; 359 switch (csp->csp_auth_alg) { 360 case CRYPTO_BLAKE2B: 361 if (klen > 0) 362 rc = blake2b_init_key(&bctx.sb, ses->mlen, key, klen); 363 else 364 rc = blake2b_init(&bctx.sb, ses->mlen); 365 if (rc != 0) 366 goto out; 367 error = crypto_apply(crp, crp->crp_payload_start, 368 crp->crp_payload_length, blake2b_applicator, &bctx.sb); 369 if (error != 0) 370 goto out; 371 rc = blake2b_final(&bctx.sb, res, ses->mlen); 372 if (rc != 0) { 373 error = EINVAL; 374 goto out; 375 } 376 break; 377 case CRYPTO_BLAKE2S: 378 if (klen > 0) 379 rc = blake2s_init_key(&bctx.ss, ses->mlen, key, klen); 380 else 381 rc = blake2s_init(&bctx.ss, ses->mlen); 382 if (rc != 0) 383 goto out; 384 error = crypto_apply(crp, crp->crp_payload_start, 385 crp->crp_payload_length, blake2s_applicator, &bctx.ss); 386 if (error != 0) 387 goto out; 388 rc = blake2s_final(&bctx.ss, res, ses->mlen); 389 if (rc != 0) { 390 error = EINVAL; 391 goto out; 392 } 393 break; 394 default: 395 panic("unreachable"); 396 } 397 398 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 399 crypto_copydata(crp, crp->crp_digest_start, ses->mlen, res2); 400 if (timingsafe_bcmp(res, res2, ses->mlen) != 0) 401 return (EBADMSG); 402 } else 403 crypto_copyback(crp, crp->crp_digest_start, ses->mlen, res); 404 405 out: 406 if (!kt) { 407 fpu_kern_leave(curthread, ctx); 408 RELEASE_CTX(ctxidx, ctx); 409 } 410 return (error); 411 } 412