1 /* 2 * Copyright (c) 2005-2010 Pawel Jakub Dawidek <pjd@FreeBSD.org> 3 * Copyright (c) 2018 Sean Eric Fagan <sef@ixsystems.com> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 * Portions of this file are derived from sys/geom/eli/g_eli_hmac.c 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include <sys/types.h> 34 #include <sys/errno.h> 35 36 #ifdef _KERNEL 37 #include <sys/libkern.h> 38 #include <sys/malloc.h> 39 #include <sys/sysctl.h> 40 #include <opencrypto/cryptodev.h> 41 #include <opencrypto/xform.h> 42 #else 43 #include <strings.h> 44 #endif 45 46 #include <sys/zio_crypt.h> 47 #include <sys/fs/zfs.h> 48 #include <sys/zio.h> 49 50 #include <sys/freebsd_crypto.h> 51 52 #define SHA512_HMAC_BLOCK_SIZE 128 53 54 static int crypt_sessions = 0; 55 SYSCTL_DECL(_vfs_zfs); 56 SYSCTL_INT(_vfs_zfs, OID_AUTO, crypt_sessions, CTLFLAG_RD, 57 &crypt_sessions, 0, "Number of cryptographic sessions created"); 58 59 void 60 crypto_mac_init(struct hmac_ctx *ctx, const crypto_key_t *c_key) 61 { 62 uint8_t k_ipad[SHA512_HMAC_BLOCK_SIZE], 63 k_opad[SHA512_HMAC_BLOCK_SIZE], 64 key[SHA512_HMAC_BLOCK_SIZE]; 65 SHA512_CTX lctx; 66 int i; 67 size_t cl_bytes = CRYPTO_BITS2BYTES(c_key->ck_length); 68 69 /* 70 * This code is based on the similar code in geom/eli/g_eli_hmac.c 71 */ 72 explicit_bzero(key, sizeof (key)); 73 if (c_key->ck_length == 0) 74 /* do nothing */; 75 else if (cl_bytes <= SHA512_HMAC_BLOCK_SIZE) 76 bcopy(c_key->ck_data, key, cl_bytes); 77 else { 78 /* 79 * If key is longer than 128 bytes reset it to 80 * key = SHA512(key). 81 */ 82 SHA512_Init(&lctx); 83 SHA512_Update(&lctx, c_key->ck_data, cl_bytes); 84 SHA512_Final(key, &lctx); 85 } 86 87 /* XOR key with ipad and opad values. */ 88 for (i = 0; i < sizeof (key); i++) { 89 k_ipad[i] = key[i] ^ 0x36; 90 k_opad[i] = key[i] ^ 0x5c; 91 } 92 explicit_bzero(key, sizeof (key)); 93 94 /* Start inner SHA512. */ 95 SHA512_Init(&ctx->innerctx); 96 SHA512_Update(&ctx->innerctx, k_ipad, sizeof (k_ipad)); 97 explicit_bzero(k_ipad, sizeof (k_ipad)); 98 /* Start outer SHA512. */ 99 SHA512_Init(&ctx->outerctx); 100 SHA512_Update(&ctx->outerctx, k_opad, sizeof (k_opad)); 101 explicit_bzero(k_opad, sizeof (k_opad)); 102 } 103 104 void 105 crypto_mac_update(struct hmac_ctx *ctx, const void *data, size_t datasize) 106 { 107 SHA512_Update(&ctx->innerctx, data, datasize); 108 } 109 110 void 111 crypto_mac_final(struct hmac_ctx *ctx, void *md, size_t mdsize) 112 { 113 uint8_t digest[SHA512_DIGEST_LENGTH]; 114 115 /* Complete inner hash */ 116 SHA512_Final(digest, &ctx->innerctx); 117 118 /* Complete outer hash */ 119 SHA512_Update(&ctx->outerctx, digest, sizeof (digest)); 120 SHA512_Final(digest, &ctx->outerctx); 121 122 explicit_bzero(ctx, sizeof (*ctx)); 123 /* mdsize == 0 means "Give me the whole hash!" */ 124 if (mdsize == 0) 125 mdsize = SHA512_DIGEST_LENGTH; 126 bcopy(digest, md, mdsize); 127 explicit_bzero(digest, sizeof (digest)); 128 } 129 130 void 131 crypto_mac(const crypto_key_t *key, const void *in_data, size_t in_data_size, 132 void *out_data, size_t out_data_size) 133 { 134 struct hmac_ctx ctx; 135 136 crypto_mac_init(&ctx, key); 137 crypto_mac_update(&ctx, in_data, in_data_size); 138 crypto_mac_final(&ctx, out_data, out_data_size); 139 } 140 141 static int 142 freebsd_zfs_crypt_done(struct cryptop *crp) 143 { 144 freebsd_crypt_session_t *ses; 145 146 ses = crp->crp_opaque; 147 mtx_lock(&ses->fs_lock); 148 ses->fs_done = true; 149 mtx_unlock(&ses->fs_lock); 150 wakeup(crp); 151 return (0); 152 } 153 154 void 155 freebsd_crypt_freesession(freebsd_crypt_session_t *sess) 156 { 157 mtx_destroy(&sess->fs_lock); 158 crypto_freesession(sess->fs_sid); 159 explicit_bzero(sess, sizeof (*sess)); 160 } 161 162 static int 163 zfs_crypto_dispatch(freebsd_crypt_session_t *session, struct cryptop *crp) 164 { 165 int error; 166 167 crp->crp_opaque = session; 168 crp->crp_callback = freebsd_zfs_crypt_done; 169 for (;;) { 170 error = crypto_dispatch(crp); 171 if (error) 172 break; 173 mtx_lock(&session->fs_lock); 174 while (session->fs_done == false) 175 msleep(crp, &session->fs_lock, PRIBIO, 176 "zfs_crypto", hz/5); 177 mtx_unlock(&session->fs_lock); 178 179 if (crp->crp_etype != EAGAIN) { 180 error = crp->crp_etype; 181 break; 182 } 183 crp->crp_etype = 0; 184 crp->crp_flags &= ~CRYPTO_F_DONE; 185 session->fs_done = false; 186 #if __FreeBSD_version < 1300087 187 /* 188 * Session ID changed, so we should record that, 189 * and try again 190 */ 191 session->fs_sid = crp->crp_session; 192 #endif 193 } 194 return (error); 195 } 196 static void 197 freebsd_crypt_uio_debug_log(boolean_t encrypt, 198 freebsd_crypt_session_t *input_sessionp, 199 struct zio_crypt_info *c_info, 200 uio_t *data_uio, 201 crypto_key_t *key, 202 uint8_t *ivbuf, 203 size_t datalen, 204 size_t auth_len) 205 { 206 #ifdef FCRYPTO_DEBUG 207 struct cryptodesc *crd; 208 uint8_t *p = NULL; 209 size_t total = 0; 210 211 printf("%s(%s, %p, { %s, %d, %d, %s }, %p, { %d, %p, %u }, " 212 "%p, %u, %u)\n", 213 __FUNCTION__, encrypt ? "encrypt" : "decrypt", input_sessionp, 214 c_info->ci_algname, c_info->ci_crypt_type, 215 (unsigned int)c_info->ci_keylen, c_info->ci_name, 216 data_uio, key->ck_format, key->ck_data, 217 (unsigned int)key->ck_length, 218 ivbuf, (unsigned int)datalen, (unsigned int)auth_len); 219 printf("\tkey = { "); 220 for (int i = 0; i < key->ck_length / 8; i++) { 221 uint8_t *b = (uint8_t *)key->ck_data; 222 printf("%02x ", b[i]); 223 } 224 printf("}\n"); 225 for (int i = 0; i < data_uio->uio_iovcnt; i++) { 226 printf("\tiovec #%d: <%p, %u>\n", i, 227 data_uio->uio_iov[i].iov_base, 228 (unsigned int)data_uio->uio_iov[i].iov_len); 229 total += data_uio->uio_iov[i].iov_len; 230 } 231 data_uio->uio_resid = total; 232 #endif 233 } 234 /* 235 * Create a new cryptographic session. This should 236 * happen every time the key changes (including when 237 * it's first loaded). 238 */ 239 #if __FreeBSD_version >= 1300087 240 int 241 freebsd_crypt_newsession(freebsd_crypt_session_t *sessp, 242 struct zio_crypt_info *c_info, crypto_key_t *key) 243 { 244 struct crypto_session_params csp; 245 int error = 0; 246 247 #ifdef FCRYPTO_DEBUG 248 printf("%s(%p, { %s, %d, %d, %s }, { %d, %p, %u })\n", 249 __FUNCTION__, sessp, 250 c_info->ci_algname, c_info->ci_crypt_type, 251 (unsigned int)c_info->ci_keylen, c_info->ci_name, 252 key->ck_format, key->ck_data, (unsigned int)key->ck_length); 253 printf("\tkey = { "); 254 for (int i = 0; i < key->ck_length / 8; i++) { 255 uint8_t *b = (uint8_t *)key->ck_data; 256 printf("%02x ", b[i]); 257 } 258 printf("}\n"); 259 #endif 260 bzero(&csp, sizeof (csp)); 261 csp.csp_mode = CSP_MODE_AEAD; 262 csp.csp_cipher_key = key->ck_data; 263 csp.csp_cipher_klen = key->ck_length / 8; 264 switch (c_info->ci_crypt_type) { 265 case ZC_TYPE_GCM: 266 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16; 267 csp.csp_ivlen = AES_GCM_IV_LEN; 268 switch (key->ck_length/8) { 269 case AES_128_GMAC_KEY_LEN: 270 case AES_192_GMAC_KEY_LEN: 271 case AES_256_GMAC_KEY_LEN: 272 break; 273 default: 274 error = EINVAL; 275 goto bad; 276 } 277 break; 278 case ZC_TYPE_CCM: 279 csp.csp_cipher_alg = CRYPTO_AES_CCM_16; 280 csp.csp_ivlen = AES_CCM_IV_LEN; 281 switch (key->ck_length/8) { 282 case AES_128_CBC_MAC_KEY_LEN: 283 case AES_192_CBC_MAC_KEY_LEN: 284 case AES_256_CBC_MAC_KEY_LEN: 285 break; 286 default: 287 error = EINVAL; 288 goto bad; 289 break; 290 } 291 break; 292 default: 293 error = ENOTSUP; 294 goto bad; 295 } 296 error = crypto_newsession(&sessp->fs_sid, &csp, 297 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE); 298 mtx_init(&sessp->fs_lock, "FreeBSD Cryptographic Session Lock", 299 NULL, MTX_DEF); 300 crypt_sessions++; 301 bad: 302 #ifdef FCRYPTO_DEBUG 303 if (error) 304 printf("%s: returning error %d\n", __FUNCTION__, error); 305 #endif 306 return (error); 307 } 308 309 int 310 freebsd_crypt_uio(boolean_t encrypt, 311 freebsd_crypt_session_t *input_sessionp, 312 struct zio_crypt_info *c_info, 313 uio_t *data_uio, 314 crypto_key_t *key, 315 uint8_t *ivbuf, 316 size_t datalen, 317 size_t auth_len) 318 { 319 struct cryptop *crp; 320 freebsd_crypt_session_t *session = NULL; 321 int error = 0; 322 size_t total = 0; 323 324 freebsd_crypt_uio_debug_log(encrypt, input_sessionp, c_info, data_uio, 325 key, ivbuf, datalen, auth_len); 326 for (int i = 0; i < data_uio->uio_iovcnt; i++) 327 total += data_uio->uio_iov[i].iov_len; 328 data_uio->uio_resid = total; 329 if (input_sessionp == NULL) { 330 session = kmem_zalloc(sizeof (*session), KM_SLEEP); 331 error = freebsd_crypt_newsession(session, c_info, key); 332 if (error) 333 goto out; 334 } else 335 session = input_sessionp; 336 337 crp = crypto_getreq(session->fs_sid, M_WAITOK); 338 if (encrypt) { 339 crp->crp_op = CRYPTO_OP_ENCRYPT | 340 CRYPTO_OP_COMPUTE_DIGEST; 341 } else { 342 crp->crp_op = CRYPTO_OP_DECRYPT | 343 CRYPTO_OP_VERIFY_DIGEST; 344 } 345 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_IV_SEPARATE; 346 crypto_use_uio(crp, data_uio); 347 348 crp->crp_aad_start = 0; 349 crp->crp_aad_length = auth_len; 350 crp->crp_payload_start = auth_len; 351 crp->crp_payload_length = datalen; 352 crp->crp_digest_start = auth_len + datalen; 353 354 bcopy(ivbuf, crp->crp_iv, ZIO_DATA_IV_LEN); 355 error = zfs_crypto_dispatch(session, crp); 356 crypto_freereq(crp); 357 out: 358 #ifdef FCRYPTO_DEBUG 359 if (error) 360 printf("%s: returning error %d\n", __FUNCTION__, error); 361 #endif 362 if (input_sessionp == NULL) { 363 freebsd_crypt_freesession(session); 364 kmem_free(session, sizeof (*session)); 365 } 366 return (error); 367 } 368 369 #else 370 int 371 freebsd_crypt_newsession(freebsd_crypt_session_t *sessp, 372 struct zio_crypt_info *c_info, crypto_key_t *key) 373 { 374 struct cryptoini cria, crie, *crip; 375 struct enc_xform *xform; 376 struct auth_hash *xauth; 377 int error = 0; 378 crypto_session_t sid; 379 380 #ifdef FCRYPTO_DEBUG 381 printf("%s(%p, { %s, %d, %d, %s }, { %d, %p, %u })\n", 382 __FUNCTION__, sessp, 383 c_info->ci_algname, c_info->ci_crypt_type, 384 (unsigned int)c_info->ci_keylen, c_info->ci_name, 385 key->ck_format, key->ck_data, (unsigned int)key->ck_length); 386 printf("\tkey = { "); 387 for (int i = 0; i < key->ck_length / 8; i++) { 388 uint8_t *b = (uint8_t *)key->ck_data; 389 printf("%02x ", b[i]); 390 } 391 printf("}\n"); 392 #endif 393 switch (c_info->ci_crypt_type) { 394 case ZC_TYPE_GCM: 395 xform = &enc_xform_aes_nist_gcm; 396 switch (key->ck_length/8) { 397 case AES_128_GMAC_KEY_LEN: 398 xauth = &auth_hash_nist_gmac_aes_128; 399 break; 400 case AES_192_GMAC_KEY_LEN: 401 xauth = &auth_hash_nist_gmac_aes_192; 402 break; 403 case AES_256_GMAC_KEY_LEN: 404 xauth = &auth_hash_nist_gmac_aes_256; 405 break; 406 default: 407 error = EINVAL; 408 goto bad; 409 } 410 break; 411 case ZC_TYPE_CCM: 412 xform = &enc_xform_ccm; 413 switch (key->ck_length/8) { 414 case AES_128_CBC_MAC_KEY_LEN: 415 xauth = &auth_hash_ccm_cbc_mac_128; 416 break; 417 case AES_192_CBC_MAC_KEY_LEN: 418 xauth = &auth_hash_ccm_cbc_mac_192; 419 break; 420 case AES_256_CBC_MAC_KEY_LEN: 421 xauth = &auth_hash_ccm_cbc_mac_256; 422 break; 423 default: 424 error = EINVAL; 425 goto bad; 426 break; 427 } 428 break; 429 default: 430 error = ENOTSUP; 431 goto bad; 432 } 433 #ifdef FCRYPTO_DEBUG 434 printf("%s(%d): Using crypt %s (key length %u [%u bytes]), " 435 "auth %s (key length %d)\n", 436 __FUNCTION__, __LINE__, 437 xform->name, (unsigned int)key->ck_length, 438 (unsigned int)key->ck_length/8, 439 xauth->name, xauth->keysize); 440 #endif 441 442 bzero(&crie, sizeof (crie)); 443 bzero(&cria, sizeof (cria)); 444 445 crie.cri_alg = xform->type; 446 crie.cri_key = key->ck_data; 447 crie.cri_klen = key->ck_length; 448 449 cria.cri_alg = xauth->type; 450 cria.cri_key = key->ck_data; 451 cria.cri_klen = key->ck_length; 452 453 cria.cri_next = &crie; 454 crie.cri_next = NULL; 455 crip = &cria; 456 // Everything else is bzero'd 457 458 error = crypto_newsession(&sid, crip, 459 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE); 460 if (error != 0) { 461 printf("%s(%d): crypto_newsession failed with %d\n", 462 __FUNCTION__, __LINE__, error); 463 goto bad; 464 } 465 sessp->fs_sid = sid; 466 mtx_init(&sessp->fs_lock, "FreeBSD Cryptographic Session Lock", 467 NULL, MTX_DEF); 468 crypt_sessions++; 469 bad: 470 return (error); 471 } 472 473 /* 474 * The meat of encryption/decryption. 475 * If sessp is NULL, then it will create a 476 * temporary cryptographic session, and release 477 * it when done. 478 */ 479 int 480 freebsd_crypt_uio(boolean_t encrypt, 481 freebsd_crypt_session_t *input_sessionp, 482 struct zio_crypt_info *c_info, 483 uio_t *data_uio, 484 crypto_key_t *key, 485 uint8_t *ivbuf, 486 size_t datalen, 487 size_t auth_len) 488 { 489 struct cryptop *crp; 490 struct cryptodesc *enc_desc, *auth_desc; 491 struct enc_xform *xform; 492 struct auth_hash *xauth; 493 freebsd_crypt_session_t *session = NULL; 494 int error; 495 496 freebsd_crypt_uio_debug_log(encrypt, input_sessionp, c_info, data_uio, 497 key, ivbuf, datalen, auth_len); 498 switch (c_info->ci_crypt_type) { 499 case ZC_TYPE_GCM: 500 xform = &enc_xform_aes_nist_gcm; 501 switch (key->ck_length/8) { 502 case AES_128_GMAC_KEY_LEN: 503 xauth = &auth_hash_nist_gmac_aes_128; 504 break; 505 case AES_192_GMAC_KEY_LEN: 506 xauth = &auth_hash_nist_gmac_aes_192; 507 break; 508 case AES_256_GMAC_KEY_LEN: 509 xauth = &auth_hash_nist_gmac_aes_256; 510 break; 511 default: 512 error = EINVAL; 513 goto bad; 514 } 515 break; 516 case ZC_TYPE_CCM: 517 xform = &enc_xform_ccm; 518 switch (key->ck_length/8) { 519 case AES_128_CBC_MAC_KEY_LEN: 520 xauth = &auth_hash_ccm_cbc_mac_128; 521 break; 522 case AES_192_CBC_MAC_KEY_LEN: 523 xauth = &auth_hash_ccm_cbc_mac_192; 524 break; 525 case AES_256_CBC_MAC_KEY_LEN: 526 xauth = &auth_hash_ccm_cbc_mac_256; 527 break; 528 default: 529 error = EINVAL; 530 goto bad; 531 break; 532 } 533 break; 534 default: 535 error = ENOTSUP; 536 goto bad; 537 } 538 539 #ifdef FCRYPTO_DEBUG 540 printf("%s(%d): Using crypt %s (key length %u [%u bytes]), " 541 "auth %s (key length %d)\n", 542 __FUNCTION__, __LINE__, 543 xform->name, (unsigned int)key->ck_length, 544 (unsigned int)key->ck_length/8, 545 xauth->name, xauth->keysize); 546 #endif 547 548 if (input_sessionp == NULL) { 549 session = kmem_zalloc(sizeof (*session), KM_SLEEP); 550 error = freebsd_crypt_newsession(session, c_info, key); 551 if (error) 552 goto out; 553 } else 554 session = input_sessionp; 555 556 crp = crypto_getreq(2); 557 if (crp == NULL) { 558 error = ENOMEM; 559 goto bad; 560 } 561 562 auth_desc = crp->crp_desc; 563 enc_desc = auth_desc->crd_next; 564 565 crp->crp_session = session->fs_sid; 566 crp->crp_ilen = auth_len + datalen; 567 crp->crp_buf = (void*)data_uio; 568 crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIFSYNC; 569 570 auth_desc->crd_skip = 0; 571 auth_desc->crd_len = auth_len; 572 auth_desc->crd_inject = auth_len + datalen; 573 auth_desc->crd_alg = xauth->type; 574 #ifdef FCRYPTO_DEBUG 575 printf("%s: auth: skip = %u, len = %u, inject = %u\n", 576 __FUNCTION__, auth_desc->crd_skip, auth_desc->crd_len, 577 auth_desc->crd_inject); 578 #endif 579 580 enc_desc->crd_skip = auth_len; 581 enc_desc->crd_len = datalen; 582 enc_desc->crd_inject = auth_len; 583 enc_desc->crd_alg = xform->type; 584 enc_desc->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT; 585 bcopy(ivbuf, enc_desc->crd_iv, ZIO_DATA_IV_LEN); 586 enc_desc->crd_next = NULL; 587 588 #ifdef FCRYPTO_DEBUG 589 printf("%s: enc: skip = %u, len = %u, inject = %u\n", 590 __FUNCTION__, enc_desc->crd_skip, enc_desc->crd_len, 591 enc_desc->crd_inject); 592 #endif 593 594 if (encrypt) 595 enc_desc->crd_flags |= CRD_F_ENCRYPT; 596 597 error = zfs_crypto_dispatch(session, crp); 598 crypto_freereq(crp); 599 out: 600 if (input_sessionp == NULL) { 601 freebsd_crypt_freesession(session); 602 kmem_free(session, sizeof (*session)); 603 } 604 bad: 605 #ifdef FCRYPTO_DEBUG 606 if (error) 607 printf("%s: returning error %d\n", __FUNCTION__, error); 608 #endif 609 return (error); 610 } 611 #endif 612