1 /* 2 * Copyright (c) 2005-2010 Pawel Jakub Dawidek <pjd@FreeBSD.org> 3 * Copyright (c) 2018 Sean Eric Fagan <sef@ixsystems.com> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 * Portions of this file are derived from sys/geom/eli/g_eli_hmac.c 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include <sys/types.h> 34 #include <sys/errno.h> 35 36 #ifdef _KERNEL 37 #include <sys/libkern.h> 38 #include <sys/malloc.h> 39 #include <sys/sysctl.h> 40 #include <opencrypto/cryptodev.h> 41 #include <opencrypto/xform.h> 42 #else 43 #include <strings.h> 44 #endif 45 46 #include <sys/zio_crypt.h> 47 #include <sys/fs/zfs.h> 48 #include <sys/zio.h> 49 50 #include <sys/freebsd_crypto.h> 51 52 #define SHA512_HMAC_BLOCK_SIZE 128 53 54 static int crypt_sessions = 0; 55 SYSCTL_DECL(_vfs_zfs); 56 SYSCTL_INT(_vfs_zfs, OID_AUTO, crypt_sessions, CTLFLAG_RD, 57 &crypt_sessions, 0, "Number of cryptographic sessions created"); 58 59 void 60 crypto_mac_init(struct hmac_ctx *ctx, const crypto_key_t *c_key) 61 { 62 uint8_t k_ipad[SHA512_HMAC_BLOCK_SIZE], 63 k_opad[SHA512_HMAC_BLOCK_SIZE], 64 key[SHA512_HMAC_BLOCK_SIZE]; 65 SHA512_CTX lctx; 66 int i; 67 size_t cl_bytes = CRYPTO_BITS2BYTES(c_key->ck_length); 68 69 /* 70 * This code is based on the similar code in geom/eli/g_eli_hmac.c 71 */ 72 explicit_bzero(key, sizeof (key)); 73 if (c_key->ck_length == 0) 74 /* do nothing */; 75 else if (cl_bytes <= SHA512_HMAC_BLOCK_SIZE) 76 bcopy(c_key->ck_data, key, cl_bytes); 77 else { 78 /* 79 * If key is longer than 128 bytes reset it to 80 * key = SHA512(key). 81 */ 82 SHA512_Init(&lctx); 83 SHA512_Update(&lctx, c_key->ck_data, cl_bytes); 84 SHA512_Final(key, &lctx); 85 } 86 87 /* XOR key with ipad and opad values. */ 88 for (i = 0; i < sizeof (key); i++) { 89 k_ipad[i] = key[i] ^ 0x36; 90 k_opad[i] = key[i] ^ 0x5c; 91 } 92 explicit_bzero(key, sizeof (key)); 93 94 /* Start inner SHA512. */ 95 SHA512_Init(&ctx->innerctx); 96 SHA512_Update(&ctx->innerctx, k_ipad, sizeof (k_ipad)); 97 explicit_bzero(k_ipad, sizeof (k_ipad)); 98 /* Start outer SHA512. */ 99 SHA512_Init(&ctx->outerctx); 100 SHA512_Update(&ctx->outerctx, k_opad, sizeof (k_opad)); 101 explicit_bzero(k_opad, sizeof (k_opad)); 102 } 103 104 void 105 crypto_mac_update(struct hmac_ctx *ctx, const void *data, size_t datasize) 106 { 107 SHA512_Update(&ctx->innerctx, data, datasize); 108 } 109 110 void 111 crypto_mac_final(struct hmac_ctx *ctx, void *md, size_t mdsize) 112 { 113 uint8_t digest[SHA512_DIGEST_LENGTH]; 114 115 /* Complete inner hash */ 116 SHA512_Final(digest, &ctx->innerctx); 117 118 /* Complete outer hash */ 119 SHA512_Update(&ctx->outerctx, digest, sizeof (digest)); 120 SHA512_Final(digest, &ctx->outerctx); 121 122 explicit_bzero(ctx, sizeof (*ctx)); 123 /* mdsize == 0 means "Give me the whole hash!" */ 124 if (mdsize == 0) 125 mdsize = SHA512_DIGEST_LENGTH; 126 bcopy(digest, md, mdsize); 127 explicit_bzero(digest, sizeof (digest)); 128 } 129 130 void 131 crypto_mac(const crypto_key_t *key, const void *in_data, size_t in_data_size, 132 void *out_data, size_t out_data_size) 133 { 134 struct hmac_ctx ctx; 135 136 crypto_mac_init(&ctx, key); 137 crypto_mac_update(&ctx, in_data, in_data_size); 138 crypto_mac_final(&ctx, out_data, out_data_size); 139 } 140 141 static int 142 freebsd_zfs_crypt_done(struct cryptop *crp) 143 { 144 freebsd_crypt_session_t *ses; 145 146 ses = crp->crp_opaque; 147 mtx_lock(&ses->fs_lock); 148 ses->fs_done = true; 149 mtx_unlock(&ses->fs_lock); 150 wakeup(crp); 151 return (0); 152 } 153 154 void 155 freebsd_crypt_freesession(freebsd_crypt_session_t *sess) 156 { 157 mtx_destroy(&sess->fs_lock); 158 crypto_freesession(sess->fs_sid); 159 explicit_bzero(sess, sizeof (*sess)); 160 } 161 162 static int 163 zfs_crypto_dispatch(freebsd_crypt_session_t *session, struct cryptop *crp) 164 { 165 int error; 166 167 crp->crp_opaque = session; 168 crp->crp_callback = freebsd_zfs_crypt_done; 169 for (;;) { 170 error = crypto_dispatch(crp); 171 if (error) 172 break; 173 mtx_lock(&session->fs_lock); 174 while (session->fs_done == false) 175 msleep(crp, &session->fs_lock, PRIBIO, 176 "zfs_crypto", hz/5); 177 mtx_unlock(&session->fs_lock); 178 179 if (crp->crp_etype != EAGAIN) { 180 error = crp->crp_etype; 181 break; 182 } 183 crp->crp_etype = 0; 184 crp->crp_flags &= ~CRYPTO_F_DONE; 185 session->fs_done = false; 186 #if __FreeBSD_version < 1300087 187 /* 188 * Session ID changed, so we should record that, 189 * and try again 190 */ 191 session->fs_sid = crp->crp_session; 192 #endif 193 } 194 return (error); 195 } 196 static void 197 freebsd_crypt_uio_debug_log(boolean_t encrypt, 198 freebsd_crypt_session_t *input_sessionp, 199 struct zio_crypt_info *c_info, 200 zfs_uio_t *data_uio, 201 crypto_key_t *key, 202 uint8_t *ivbuf, 203 size_t datalen, 204 size_t auth_len) 205 { 206 #ifdef FCRYPTO_DEBUG 207 struct cryptodesc *crd; 208 uint8_t *p = NULL; 209 size_t total = 0; 210 211 printf("%s(%s, %p, { %s, %d, %d, %s }, %p, { %d, %p, %u }, " 212 "%p, %u, %u)\n", 213 __FUNCTION__, encrypt ? "encrypt" : "decrypt", input_sessionp, 214 c_info->ci_algname, c_info->ci_crypt_type, 215 (unsigned int)c_info->ci_keylen, c_info->ci_name, 216 data_uio, key->ck_format, key->ck_data, 217 (unsigned int)key->ck_length, 218 ivbuf, (unsigned int)datalen, (unsigned int)auth_len); 219 printf("\tkey = { "); 220 for (int i = 0; i < key->ck_length / 8; i++) { 221 uint8_t *b = (uint8_t *)key->ck_data; 222 printf("%02x ", b[i]); 223 } 224 printf("}\n"); 225 for (int i = 0; i < zfs_uio_iovcnt(data_uio); i++) { 226 printf("\tiovec #%d: <%p, %u>\n", i, 227 zfs_uio_iovbase(data_uio, i), 228 (unsigned int)zfs_uio_iovlen(data_uio, i)); 229 total += zfs_uio_iovlen(data_uio, i); 230 } 231 zfs_uio_resid(data_uio) = total; 232 #endif 233 } 234 /* 235 * Create a new cryptographic session. This should 236 * happen every time the key changes (including when 237 * it's first loaded). 238 */ 239 #if __FreeBSD_version >= 1300087 240 int 241 freebsd_crypt_newsession(freebsd_crypt_session_t *sessp, 242 struct zio_crypt_info *c_info, crypto_key_t *key) 243 { 244 struct crypto_session_params csp; 245 int error = 0; 246 247 #ifdef FCRYPTO_DEBUG 248 printf("%s(%p, { %s, %d, %d, %s }, { %d, %p, %u })\n", 249 __FUNCTION__, sessp, 250 c_info->ci_algname, c_info->ci_crypt_type, 251 (unsigned int)c_info->ci_keylen, c_info->ci_name, 252 key->ck_format, key->ck_data, (unsigned int)key->ck_length); 253 printf("\tkey = { "); 254 for (int i = 0; i < key->ck_length / 8; i++) { 255 uint8_t *b = (uint8_t *)key->ck_data; 256 printf("%02x ", b[i]); 257 } 258 printf("}\n"); 259 #endif 260 bzero(&csp, sizeof (csp)); 261 csp.csp_mode = CSP_MODE_AEAD; 262 csp.csp_cipher_key = key->ck_data; 263 csp.csp_cipher_klen = key->ck_length / 8; 264 switch (c_info->ci_crypt_type) { 265 case ZC_TYPE_GCM: 266 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16; 267 csp.csp_ivlen = AES_GCM_IV_LEN; 268 switch (key->ck_length/8) { 269 case AES_128_GMAC_KEY_LEN: 270 case AES_192_GMAC_KEY_LEN: 271 case AES_256_GMAC_KEY_LEN: 272 break; 273 default: 274 error = EINVAL; 275 goto bad; 276 } 277 break; 278 case ZC_TYPE_CCM: 279 csp.csp_cipher_alg = CRYPTO_AES_CCM_16; 280 csp.csp_ivlen = AES_CCM_IV_LEN; 281 switch (key->ck_length/8) { 282 case AES_128_CBC_MAC_KEY_LEN: 283 case AES_192_CBC_MAC_KEY_LEN: 284 case AES_256_CBC_MAC_KEY_LEN: 285 break; 286 default: 287 error = EINVAL; 288 goto bad; 289 break; 290 } 291 break; 292 default: 293 error = ENOTSUP; 294 goto bad; 295 } 296 297 /* 298 * Disable the use of hardware drivers on FreeBSD 13 and later since 299 * common crypto offload drivers impose constraints on AES-GCM AAD 300 * lengths that make them unusable for ZFS, and we currently do not have 301 * a mechanism to fall back to a software driver for requests not 302 * handled by a hardware driver. 303 * 304 * On 12 we continue to permit the use of hardware drivers since 305 * CPU-accelerated drivers such as aesni(4) register themselves as 306 * hardware drivers. 307 */ 308 error = crypto_newsession(&sessp->fs_sid, &csp, CRYPTOCAP_F_SOFTWARE); 309 mtx_init(&sessp->fs_lock, "FreeBSD Cryptographic Session Lock", 310 NULL, MTX_DEF); 311 crypt_sessions++; 312 bad: 313 #ifdef FCRYPTO_DEBUG 314 if (error) 315 printf("%s: returning error %d\n", __FUNCTION__, error); 316 #endif 317 return (error); 318 } 319 320 int 321 freebsd_crypt_uio(boolean_t encrypt, 322 freebsd_crypt_session_t *input_sessionp, 323 struct zio_crypt_info *c_info, 324 zfs_uio_t *data_uio, 325 crypto_key_t *key, 326 uint8_t *ivbuf, 327 size_t datalen, 328 size_t auth_len) 329 { 330 struct cryptop *crp; 331 freebsd_crypt_session_t *session = NULL; 332 int error = 0; 333 size_t total = 0; 334 335 freebsd_crypt_uio_debug_log(encrypt, input_sessionp, c_info, data_uio, 336 key, ivbuf, datalen, auth_len); 337 for (int i = 0; i < zfs_uio_iovcnt(data_uio); i++) 338 total += zfs_uio_iovlen(data_uio, i); 339 zfs_uio_resid(data_uio) = total; 340 if (input_sessionp == NULL) { 341 session = kmem_zalloc(sizeof (*session), KM_SLEEP); 342 error = freebsd_crypt_newsession(session, c_info, key); 343 if (error) 344 goto out; 345 } else 346 session = input_sessionp; 347 348 crp = crypto_getreq(session->fs_sid, M_WAITOK); 349 if (encrypt) { 350 crp->crp_op = CRYPTO_OP_ENCRYPT | 351 CRYPTO_OP_COMPUTE_DIGEST; 352 } else { 353 crp->crp_op = CRYPTO_OP_DECRYPT | 354 CRYPTO_OP_VERIFY_DIGEST; 355 } 356 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_IV_SEPARATE; 357 crypto_use_uio(crp, GET_UIO_STRUCT(data_uio)); 358 359 crp->crp_aad_start = 0; 360 crp->crp_aad_length = auth_len; 361 crp->crp_payload_start = auth_len; 362 crp->crp_payload_length = datalen; 363 crp->crp_digest_start = auth_len + datalen; 364 365 bcopy(ivbuf, crp->crp_iv, ZIO_DATA_IV_LEN); 366 error = zfs_crypto_dispatch(session, crp); 367 crypto_freereq(crp); 368 out: 369 #ifdef FCRYPTO_DEBUG 370 if (error) 371 printf("%s: returning error %d\n", __FUNCTION__, error); 372 #endif 373 if (input_sessionp == NULL) { 374 freebsd_crypt_freesession(session); 375 kmem_free(session, sizeof (*session)); 376 } 377 return (error); 378 } 379 380 #else 381 int 382 freebsd_crypt_newsession(freebsd_crypt_session_t *sessp, 383 struct zio_crypt_info *c_info, crypto_key_t *key) 384 { 385 struct cryptoini cria, crie, *crip; 386 struct enc_xform *xform; 387 struct auth_hash *xauth; 388 int error = 0; 389 crypto_session_t sid; 390 391 #ifdef FCRYPTO_DEBUG 392 printf("%s(%p, { %s, %d, %d, %s }, { %d, %p, %u })\n", 393 __FUNCTION__, sessp, 394 c_info->ci_algname, c_info->ci_crypt_type, 395 (unsigned int)c_info->ci_keylen, c_info->ci_name, 396 key->ck_format, key->ck_data, (unsigned int)key->ck_length); 397 printf("\tkey = { "); 398 for (int i = 0; i < key->ck_length / 8; i++) { 399 uint8_t *b = (uint8_t *)key->ck_data; 400 printf("%02x ", b[i]); 401 } 402 printf("}\n"); 403 #endif 404 switch (c_info->ci_crypt_type) { 405 case ZC_TYPE_GCM: 406 xform = &enc_xform_aes_nist_gcm; 407 switch (key->ck_length/8) { 408 case AES_128_GMAC_KEY_LEN: 409 xauth = &auth_hash_nist_gmac_aes_128; 410 break; 411 case AES_192_GMAC_KEY_LEN: 412 xauth = &auth_hash_nist_gmac_aes_192; 413 break; 414 case AES_256_GMAC_KEY_LEN: 415 xauth = &auth_hash_nist_gmac_aes_256; 416 break; 417 default: 418 error = EINVAL; 419 goto bad; 420 } 421 break; 422 case ZC_TYPE_CCM: 423 xform = &enc_xform_ccm; 424 switch (key->ck_length/8) { 425 case AES_128_CBC_MAC_KEY_LEN: 426 xauth = &auth_hash_ccm_cbc_mac_128; 427 break; 428 case AES_192_CBC_MAC_KEY_LEN: 429 xauth = &auth_hash_ccm_cbc_mac_192; 430 break; 431 case AES_256_CBC_MAC_KEY_LEN: 432 xauth = &auth_hash_ccm_cbc_mac_256; 433 break; 434 default: 435 error = EINVAL; 436 goto bad; 437 break; 438 } 439 break; 440 default: 441 error = ENOTSUP; 442 goto bad; 443 } 444 #ifdef FCRYPTO_DEBUG 445 printf("%s(%d): Using crypt %s (key length %u [%u bytes]), " 446 "auth %s (key length %d)\n", 447 __FUNCTION__, __LINE__, 448 xform->name, (unsigned int)key->ck_length, 449 (unsigned int)key->ck_length/8, 450 xauth->name, xauth->keysize); 451 #endif 452 453 bzero(&crie, sizeof (crie)); 454 bzero(&cria, sizeof (cria)); 455 456 crie.cri_alg = xform->type; 457 crie.cri_key = key->ck_data; 458 crie.cri_klen = key->ck_length; 459 460 cria.cri_alg = xauth->type; 461 cria.cri_key = key->ck_data; 462 cria.cri_klen = key->ck_length; 463 464 cria.cri_next = &crie; 465 crie.cri_next = NULL; 466 crip = &cria; 467 // Everything else is bzero'd 468 469 error = crypto_newsession(&sid, crip, 470 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE); 471 if (error != 0) { 472 printf("%s(%d): crypto_newsession failed with %d\n", 473 __FUNCTION__, __LINE__, error); 474 goto bad; 475 } 476 sessp->fs_sid = sid; 477 mtx_init(&sessp->fs_lock, "FreeBSD Cryptographic Session Lock", 478 NULL, MTX_DEF); 479 crypt_sessions++; 480 bad: 481 return (error); 482 } 483 484 /* 485 * The meat of encryption/decryption. 486 * If sessp is NULL, then it will create a 487 * temporary cryptographic session, and release 488 * it when done. 489 */ 490 int 491 freebsd_crypt_uio(boolean_t encrypt, 492 freebsd_crypt_session_t *input_sessionp, 493 struct zio_crypt_info *c_info, 494 zfs_uio_t *data_uio, 495 crypto_key_t *key, 496 uint8_t *ivbuf, 497 size_t datalen, 498 size_t auth_len) 499 { 500 struct cryptop *crp; 501 struct cryptodesc *enc_desc, *auth_desc; 502 struct enc_xform *xform; 503 struct auth_hash *xauth; 504 freebsd_crypt_session_t *session = NULL; 505 int error; 506 507 freebsd_crypt_uio_debug_log(encrypt, input_sessionp, c_info, data_uio, 508 key, ivbuf, datalen, auth_len); 509 switch (c_info->ci_crypt_type) { 510 case ZC_TYPE_GCM: 511 xform = &enc_xform_aes_nist_gcm; 512 switch (key->ck_length/8) { 513 case AES_128_GMAC_KEY_LEN: 514 xauth = &auth_hash_nist_gmac_aes_128; 515 break; 516 case AES_192_GMAC_KEY_LEN: 517 xauth = &auth_hash_nist_gmac_aes_192; 518 break; 519 case AES_256_GMAC_KEY_LEN: 520 xauth = &auth_hash_nist_gmac_aes_256; 521 break; 522 default: 523 error = EINVAL; 524 goto bad; 525 } 526 break; 527 case ZC_TYPE_CCM: 528 xform = &enc_xform_ccm; 529 switch (key->ck_length/8) { 530 case AES_128_CBC_MAC_KEY_LEN: 531 xauth = &auth_hash_ccm_cbc_mac_128; 532 break; 533 case AES_192_CBC_MAC_KEY_LEN: 534 xauth = &auth_hash_ccm_cbc_mac_192; 535 break; 536 case AES_256_CBC_MAC_KEY_LEN: 537 xauth = &auth_hash_ccm_cbc_mac_256; 538 break; 539 default: 540 error = EINVAL; 541 goto bad; 542 break; 543 } 544 break; 545 default: 546 error = ENOTSUP; 547 goto bad; 548 } 549 550 #ifdef FCRYPTO_DEBUG 551 printf("%s(%d): Using crypt %s (key length %u [%u bytes]), " 552 "auth %s (key length %d)\n", 553 __FUNCTION__, __LINE__, 554 xform->name, (unsigned int)key->ck_length, 555 (unsigned int)key->ck_length/8, 556 xauth->name, xauth->keysize); 557 #endif 558 559 if (input_sessionp == NULL) { 560 session = kmem_zalloc(sizeof (*session), KM_SLEEP); 561 error = freebsd_crypt_newsession(session, c_info, key); 562 if (error) 563 goto out; 564 } else 565 session = input_sessionp; 566 567 crp = crypto_getreq(2); 568 if (crp == NULL) { 569 error = ENOMEM; 570 goto bad; 571 } 572 573 auth_desc = crp->crp_desc; 574 enc_desc = auth_desc->crd_next; 575 576 crp->crp_session = session->fs_sid; 577 crp->crp_ilen = auth_len + datalen; 578 crp->crp_buf = (void*)GET_UIO_STRUCT(data_uio); 579 crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIFSYNC; 580 581 auth_desc->crd_skip = 0; 582 auth_desc->crd_len = auth_len; 583 auth_desc->crd_inject = auth_len + datalen; 584 auth_desc->crd_alg = xauth->type; 585 #ifdef FCRYPTO_DEBUG 586 printf("%s: auth: skip = %u, len = %u, inject = %u\n", 587 __FUNCTION__, auth_desc->crd_skip, auth_desc->crd_len, 588 auth_desc->crd_inject); 589 #endif 590 591 enc_desc->crd_skip = auth_len; 592 enc_desc->crd_len = datalen; 593 enc_desc->crd_inject = auth_len; 594 enc_desc->crd_alg = xform->type; 595 enc_desc->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT; 596 bcopy(ivbuf, enc_desc->crd_iv, ZIO_DATA_IV_LEN); 597 enc_desc->crd_next = NULL; 598 599 #ifdef FCRYPTO_DEBUG 600 printf("%s: enc: skip = %u, len = %u, inject = %u\n", 601 __FUNCTION__, enc_desc->crd_skip, enc_desc->crd_len, 602 enc_desc->crd_inject); 603 #endif 604 605 if (encrypt) 606 enc_desc->crd_flags |= CRD_F_ENCRYPT; 607 608 error = zfs_crypto_dispatch(session, crp); 609 crypto_freereq(crp); 610 out: 611 if (input_sessionp == NULL) { 612 freebsd_crypt_freesession(session); 613 kmem_free(session, sizeof (*session)); 614 } 615 bad: 616 #ifdef FCRYPTO_DEBUG 617 if (error) 618 printf("%s: returning error %d\n", __FUNCTION__, error); 619 #endif 620 return (error); 621 } 622 #endif 623