1 /* 2 * Copyright (c) 2005-2010 Pawel Jakub Dawidek <pjd@FreeBSD.org> 3 * Copyright (c) 2018 Sean Eric Fagan <sef@ixsystems.com> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 * Portions of this file are derived from sys/geom/eli/g_eli_hmac.c 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include <sys/types.h> 34 #include <sys/errno.h> 35 36 #ifdef _KERNEL 37 #include <sys/libkern.h> 38 #include <sys/malloc.h> 39 #include <sys/sysctl.h> 40 #include <opencrypto/cryptodev.h> 41 #include <opencrypto/xform.h> 42 #else 43 #include <strings.h> 44 #endif 45 46 #include <sys/zio_crypt.h> 47 #include <sys/fs/zfs.h> 48 #include <sys/zio.h> 49 50 #include <sys/freebsd_crypto.h> 51 52 #define SHA512_HMAC_BLOCK_SIZE 128 53 54 static int crypt_sessions = 0; 55 SYSCTL_DECL(_vfs_zfs); 56 SYSCTL_INT(_vfs_zfs, OID_AUTO, crypt_sessions, CTLFLAG_RD, 57 &crypt_sessions, 0, "Number of cryptographic sessions created"); 58 59 void 60 crypto_mac_init(struct hmac_ctx *ctx, const crypto_key_t *c_key) 61 { 62 uint8_t k_ipad[SHA512_HMAC_BLOCK_SIZE], 63 k_opad[SHA512_HMAC_BLOCK_SIZE], 64 key[SHA512_HMAC_BLOCK_SIZE]; 65 SHA512_CTX lctx; 66 int i; 67 size_t cl_bytes = CRYPTO_BITS2BYTES(c_key->ck_length); 68 69 /* 70 * This code is based on the similar code in geom/eli/g_eli_hmac.c 71 */ 72 explicit_bzero(key, sizeof (key)); 73 if (c_key->ck_length == 0) 74 /* do nothing */; 75 else if (cl_bytes <= SHA512_HMAC_BLOCK_SIZE) 76 bcopy(c_key->ck_data, key, cl_bytes); 77 else { 78 /* 79 * If key is longer than 128 bytes reset it to 80 * key = SHA512(key). 81 */ 82 SHA512_Init(&lctx); 83 SHA512_Update(&lctx, c_key->ck_data, cl_bytes); 84 SHA512_Final(key, &lctx); 85 } 86 87 /* XOR key with ipad and opad values. */ 88 for (i = 0; i < sizeof (key); i++) { 89 k_ipad[i] = key[i] ^ 0x36; 90 k_opad[i] = key[i] ^ 0x5c; 91 } 92 explicit_bzero(key, sizeof (key)); 93 94 /* Start inner SHA512. */ 95 SHA512_Init(&ctx->innerctx); 96 SHA512_Update(&ctx->innerctx, k_ipad, sizeof (k_ipad)); 97 explicit_bzero(k_ipad, sizeof (k_ipad)); 98 /* Start outer SHA512. */ 99 SHA512_Init(&ctx->outerctx); 100 SHA512_Update(&ctx->outerctx, k_opad, sizeof (k_opad)); 101 explicit_bzero(k_opad, sizeof (k_opad)); 102 } 103 104 void 105 crypto_mac_update(struct hmac_ctx *ctx, const void *data, size_t datasize) 106 { 107 SHA512_Update(&ctx->innerctx, data, datasize); 108 } 109 110 void 111 crypto_mac_final(struct hmac_ctx *ctx, void *md, size_t mdsize) 112 { 113 uint8_t digest[SHA512_DIGEST_LENGTH]; 114 115 /* Complete inner hash */ 116 SHA512_Final(digest, &ctx->innerctx); 117 118 /* Complete outer hash */ 119 SHA512_Update(&ctx->outerctx, digest, sizeof (digest)); 120 SHA512_Final(digest, &ctx->outerctx); 121 122 explicit_bzero(ctx, sizeof (*ctx)); 123 /* mdsize == 0 means "Give me the whole hash!" */ 124 if (mdsize == 0) 125 mdsize = SHA512_DIGEST_LENGTH; 126 bcopy(digest, md, mdsize); 127 explicit_bzero(digest, sizeof (digest)); 128 } 129 130 void 131 crypto_mac(const crypto_key_t *key, const void *in_data, size_t in_data_size, 132 void *out_data, size_t out_data_size) 133 { 134 struct hmac_ctx ctx; 135 136 crypto_mac_init(&ctx, key); 137 crypto_mac_update(&ctx, in_data, in_data_size); 138 crypto_mac_final(&ctx, out_data, out_data_size); 139 } 140 141 static int 142 freebsd_zfs_crypt_done(struct cryptop *crp) 143 { 144 freebsd_crypt_session_t *ses; 145 146 ses = crp->crp_opaque; 147 mtx_lock(&ses->fs_lock); 148 ses->fs_done = true; 149 mtx_unlock(&ses->fs_lock); 150 wakeup(crp); 151 return (0); 152 } 153 154 void 155 freebsd_crypt_freesession(freebsd_crypt_session_t *sess) 156 { 157 mtx_destroy(&sess->fs_lock); 158 crypto_freesession(sess->fs_sid); 159 explicit_bzero(sess, sizeof (*sess)); 160 } 161 162 static int 163 zfs_crypto_dispatch(freebsd_crypt_session_t *session, struct cryptop *crp) 164 { 165 int error; 166 167 crp->crp_opaque = session; 168 crp->crp_callback = freebsd_zfs_crypt_done; 169 for (;;) { 170 error = crypto_dispatch(crp); 171 if (error) 172 break; 173 mtx_lock(&session->fs_lock); 174 while (session->fs_done == false) 175 msleep(crp, &session->fs_lock, 0, 176 "zfs_crypto", 0); 177 mtx_unlock(&session->fs_lock); 178 179 if (crp->crp_etype == ENOMEM) { 180 pause("zcrnomem", 1); 181 } else if (crp->crp_etype != EAGAIN) { 182 error = crp->crp_etype; 183 break; 184 } 185 crp->crp_etype = 0; 186 crp->crp_flags &= ~CRYPTO_F_DONE; 187 session->fs_done = false; 188 #if __FreeBSD_version < 1300087 189 /* 190 * Session ID changed, so we should record that, 191 * and try again 192 */ 193 session->fs_sid = crp->crp_session; 194 #endif 195 } 196 return (error); 197 } 198 static void 199 freebsd_crypt_uio_debug_log(boolean_t encrypt, 200 freebsd_crypt_session_t *input_sessionp, 201 struct zio_crypt_info *c_info, 202 zfs_uio_t *data_uio, 203 crypto_key_t *key, 204 uint8_t *ivbuf, 205 size_t datalen, 206 size_t auth_len) 207 { 208 #ifdef FCRYPTO_DEBUG 209 struct cryptodesc *crd; 210 uint8_t *p = NULL; 211 size_t total = 0; 212 213 printf("%s(%s, %p, { %s, %d, %d, %s }, %p, { %d, %p, %u }, " 214 "%p, %u, %u)\n", 215 __FUNCTION__, encrypt ? "encrypt" : "decrypt", input_sessionp, 216 c_info->ci_algname, c_info->ci_crypt_type, 217 (unsigned int)c_info->ci_keylen, c_info->ci_name, 218 data_uio, key->ck_format, key->ck_data, 219 (unsigned int)key->ck_length, 220 ivbuf, (unsigned int)datalen, (unsigned int)auth_len); 221 printf("\tkey = { "); 222 for (int i = 0; i < key->ck_length / 8; i++) { 223 uint8_t *b = (uint8_t *)key->ck_data; 224 printf("%02x ", b[i]); 225 } 226 printf("}\n"); 227 for (int i = 0; i < zfs_uio_iovcnt(data_uio); i++) { 228 printf("\tiovec #%d: <%p, %u>\n", i, 229 zfs_uio_iovbase(data_uio, i), 230 (unsigned int)zfs_uio_iovlen(data_uio, i)); 231 total += zfs_uio_iovlen(data_uio, i); 232 } 233 zfs_uio_resid(data_uio) = total; 234 #endif 235 } 236 /* 237 * Create a new cryptographic session. This should 238 * happen every time the key changes (including when 239 * it's first loaded). 240 */ 241 #if __FreeBSD_version >= 1300087 242 int 243 freebsd_crypt_newsession(freebsd_crypt_session_t *sessp, 244 struct zio_crypt_info *c_info, crypto_key_t *key) 245 { 246 struct crypto_session_params csp; 247 int error = 0; 248 249 #ifdef FCRYPTO_DEBUG 250 printf("%s(%p, { %s, %d, %d, %s }, { %d, %p, %u })\n", 251 __FUNCTION__, sessp, 252 c_info->ci_algname, c_info->ci_crypt_type, 253 (unsigned int)c_info->ci_keylen, c_info->ci_name, 254 key->ck_format, key->ck_data, (unsigned int)key->ck_length); 255 printf("\tkey = { "); 256 for (int i = 0; i < key->ck_length / 8; i++) { 257 uint8_t *b = (uint8_t *)key->ck_data; 258 printf("%02x ", b[i]); 259 } 260 printf("}\n"); 261 #endif 262 bzero(&csp, sizeof (csp)); 263 csp.csp_mode = CSP_MODE_AEAD; 264 csp.csp_cipher_key = key->ck_data; 265 csp.csp_cipher_klen = key->ck_length / 8; 266 switch (c_info->ci_crypt_type) { 267 case ZC_TYPE_GCM: 268 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16; 269 csp.csp_ivlen = AES_GCM_IV_LEN; 270 switch (key->ck_length/8) { 271 case AES_128_GMAC_KEY_LEN: 272 case AES_192_GMAC_KEY_LEN: 273 case AES_256_GMAC_KEY_LEN: 274 break; 275 default: 276 error = EINVAL; 277 goto bad; 278 } 279 break; 280 case ZC_TYPE_CCM: 281 csp.csp_cipher_alg = CRYPTO_AES_CCM_16; 282 csp.csp_ivlen = AES_CCM_IV_LEN; 283 switch (key->ck_length/8) { 284 case AES_128_CBC_MAC_KEY_LEN: 285 case AES_192_CBC_MAC_KEY_LEN: 286 case AES_256_CBC_MAC_KEY_LEN: 287 break; 288 default: 289 error = EINVAL; 290 goto bad; 291 break; 292 } 293 break; 294 default: 295 error = ENOTSUP; 296 goto bad; 297 } 298 299 /* 300 * Disable the use of hardware drivers on FreeBSD 13 and later since 301 * common crypto offload drivers impose constraints on AES-GCM AAD 302 * lengths that make them unusable for ZFS, and we currently do not have 303 * a mechanism to fall back to a software driver for requests not 304 * handled by a hardware driver. 305 * 306 * On 12 we continue to permit the use of hardware drivers since 307 * CPU-accelerated drivers such as aesni(4) register themselves as 308 * hardware drivers. 309 */ 310 error = crypto_newsession(&sessp->fs_sid, &csp, CRYPTOCAP_F_SOFTWARE); 311 mtx_init(&sessp->fs_lock, "FreeBSD Cryptographic Session Lock", 312 NULL, MTX_DEF); 313 crypt_sessions++; 314 bad: 315 #ifdef FCRYPTO_DEBUG 316 if (error) 317 printf("%s: returning error %d\n", __FUNCTION__, error); 318 #endif 319 return (error); 320 } 321 322 int 323 freebsd_crypt_uio(boolean_t encrypt, 324 freebsd_crypt_session_t *input_sessionp, 325 struct zio_crypt_info *c_info, 326 zfs_uio_t *data_uio, 327 crypto_key_t *key, 328 uint8_t *ivbuf, 329 size_t datalen, 330 size_t auth_len) 331 { 332 struct cryptop *crp; 333 freebsd_crypt_session_t *session = NULL; 334 int error = 0; 335 size_t total = 0; 336 337 freebsd_crypt_uio_debug_log(encrypt, input_sessionp, c_info, data_uio, 338 key, ivbuf, datalen, auth_len); 339 for (int i = 0; i < zfs_uio_iovcnt(data_uio); i++) 340 total += zfs_uio_iovlen(data_uio, i); 341 zfs_uio_resid(data_uio) = total; 342 if (input_sessionp == NULL) { 343 session = kmem_zalloc(sizeof (*session), KM_SLEEP); 344 error = freebsd_crypt_newsession(session, c_info, key); 345 if (error) 346 goto out; 347 } else 348 session = input_sessionp; 349 350 crp = crypto_getreq(session->fs_sid, M_WAITOK); 351 if (encrypt) { 352 crp->crp_op = CRYPTO_OP_ENCRYPT | 353 CRYPTO_OP_COMPUTE_DIGEST; 354 } else { 355 crp->crp_op = CRYPTO_OP_DECRYPT | 356 CRYPTO_OP_VERIFY_DIGEST; 357 } 358 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_IV_SEPARATE; 359 crypto_use_uio(crp, GET_UIO_STRUCT(data_uio)); 360 361 crp->crp_aad_start = 0; 362 crp->crp_aad_length = auth_len; 363 crp->crp_payload_start = auth_len; 364 crp->crp_payload_length = datalen; 365 crp->crp_digest_start = auth_len + datalen; 366 367 bcopy(ivbuf, crp->crp_iv, ZIO_DATA_IV_LEN); 368 error = zfs_crypto_dispatch(session, crp); 369 crypto_freereq(crp); 370 out: 371 #ifdef FCRYPTO_DEBUG 372 if (error) 373 printf("%s: returning error %d\n", __FUNCTION__, error); 374 #endif 375 if (input_sessionp == NULL) { 376 freebsd_crypt_freesession(session); 377 kmem_free(session, sizeof (*session)); 378 } 379 return (error); 380 } 381 382 #else 383 int 384 freebsd_crypt_newsession(freebsd_crypt_session_t *sessp, 385 struct zio_crypt_info *c_info, crypto_key_t *key) 386 { 387 struct cryptoini cria, crie, *crip; 388 struct enc_xform *xform; 389 struct auth_hash *xauth; 390 int error = 0; 391 crypto_session_t sid; 392 393 #ifdef FCRYPTO_DEBUG 394 printf("%s(%p, { %s, %d, %d, %s }, { %d, %p, %u })\n", 395 __FUNCTION__, sessp, 396 c_info->ci_algname, c_info->ci_crypt_type, 397 (unsigned int)c_info->ci_keylen, c_info->ci_name, 398 key->ck_format, key->ck_data, (unsigned int)key->ck_length); 399 printf("\tkey = { "); 400 for (int i = 0; i < key->ck_length / 8; i++) { 401 uint8_t *b = (uint8_t *)key->ck_data; 402 printf("%02x ", b[i]); 403 } 404 printf("}\n"); 405 #endif 406 switch (c_info->ci_crypt_type) { 407 case ZC_TYPE_GCM: 408 xform = &enc_xform_aes_nist_gcm; 409 switch (key->ck_length/8) { 410 case AES_128_GMAC_KEY_LEN: 411 xauth = &auth_hash_nist_gmac_aes_128; 412 break; 413 case AES_192_GMAC_KEY_LEN: 414 xauth = &auth_hash_nist_gmac_aes_192; 415 break; 416 case AES_256_GMAC_KEY_LEN: 417 xauth = &auth_hash_nist_gmac_aes_256; 418 break; 419 default: 420 error = EINVAL; 421 goto bad; 422 } 423 break; 424 case ZC_TYPE_CCM: 425 xform = &enc_xform_ccm; 426 switch (key->ck_length/8) { 427 case AES_128_CBC_MAC_KEY_LEN: 428 xauth = &auth_hash_ccm_cbc_mac_128; 429 break; 430 case AES_192_CBC_MAC_KEY_LEN: 431 xauth = &auth_hash_ccm_cbc_mac_192; 432 break; 433 case AES_256_CBC_MAC_KEY_LEN: 434 xauth = &auth_hash_ccm_cbc_mac_256; 435 break; 436 default: 437 error = EINVAL; 438 goto bad; 439 break; 440 } 441 break; 442 default: 443 error = ENOTSUP; 444 goto bad; 445 } 446 #ifdef FCRYPTO_DEBUG 447 printf("%s(%d): Using crypt %s (key length %u [%u bytes]), " 448 "auth %s (key length %d)\n", 449 __FUNCTION__, __LINE__, 450 xform->name, (unsigned int)key->ck_length, 451 (unsigned int)key->ck_length/8, 452 xauth->name, xauth->keysize); 453 #endif 454 455 bzero(&crie, sizeof (crie)); 456 bzero(&cria, sizeof (cria)); 457 458 crie.cri_alg = xform->type; 459 crie.cri_key = key->ck_data; 460 crie.cri_klen = key->ck_length; 461 462 cria.cri_alg = xauth->type; 463 cria.cri_key = key->ck_data; 464 cria.cri_klen = key->ck_length; 465 466 cria.cri_next = &crie; 467 crie.cri_next = NULL; 468 crip = &cria; 469 // Everything else is bzero'd 470 471 error = crypto_newsession(&sid, crip, 472 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE); 473 if (error != 0) { 474 printf("%s(%d): crypto_newsession failed with %d\n", 475 __FUNCTION__, __LINE__, error); 476 goto bad; 477 } 478 sessp->fs_sid = sid; 479 mtx_init(&sessp->fs_lock, "FreeBSD Cryptographic Session Lock", 480 NULL, MTX_DEF); 481 crypt_sessions++; 482 bad: 483 return (error); 484 } 485 486 /* 487 * The meat of encryption/decryption. 488 * If sessp is NULL, then it will create a 489 * temporary cryptographic session, and release 490 * it when done. 491 */ 492 int 493 freebsd_crypt_uio(boolean_t encrypt, 494 freebsd_crypt_session_t *input_sessionp, 495 struct zio_crypt_info *c_info, 496 zfs_uio_t *data_uio, 497 crypto_key_t *key, 498 uint8_t *ivbuf, 499 size_t datalen, 500 size_t auth_len) 501 { 502 struct cryptop *crp; 503 struct cryptodesc *enc_desc, *auth_desc; 504 struct enc_xform *xform; 505 struct auth_hash *xauth; 506 freebsd_crypt_session_t *session = NULL; 507 int error; 508 509 freebsd_crypt_uio_debug_log(encrypt, input_sessionp, c_info, data_uio, 510 key, ivbuf, datalen, auth_len); 511 switch (c_info->ci_crypt_type) { 512 case ZC_TYPE_GCM: 513 xform = &enc_xform_aes_nist_gcm; 514 switch (key->ck_length/8) { 515 case AES_128_GMAC_KEY_LEN: 516 xauth = &auth_hash_nist_gmac_aes_128; 517 break; 518 case AES_192_GMAC_KEY_LEN: 519 xauth = &auth_hash_nist_gmac_aes_192; 520 break; 521 case AES_256_GMAC_KEY_LEN: 522 xauth = &auth_hash_nist_gmac_aes_256; 523 break; 524 default: 525 error = EINVAL; 526 goto bad; 527 } 528 break; 529 case ZC_TYPE_CCM: 530 xform = &enc_xform_ccm; 531 switch (key->ck_length/8) { 532 case AES_128_CBC_MAC_KEY_LEN: 533 xauth = &auth_hash_ccm_cbc_mac_128; 534 break; 535 case AES_192_CBC_MAC_KEY_LEN: 536 xauth = &auth_hash_ccm_cbc_mac_192; 537 break; 538 case AES_256_CBC_MAC_KEY_LEN: 539 xauth = &auth_hash_ccm_cbc_mac_256; 540 break; 541 default: 542 error = EINVAL; 543 goto bad; 544 break; 545 } 546 break; 547 default: 548 error = ENOTSUP; 549 goto bad; 550 } 551 552 #ifdef FCRYPTO_DEBUG 553 printf("%s(%d): Using crypt %s (key length %u [%u bytes]), " 554 "auth %s (key length %d)\n", 555 __FUNCTION__, __LINE__, 556 xform->name, (unsigned int)key->ck_length, 557 (unsigned int)key->ck_length/8, 558 xauth->name, xauth->keysize); 559 #endif 560 561 if (input_sessionp == NULL) { 562 session = kmem_zalloc(sizeof (*session), KM_SLEEP); 563 error = freebsd_crypt_newsession(session, c_info, key); 564 if (error) 565 goto out; 566 } else 567 session = input_sessionp; 568 569 crp = crypto_getreq(2); 570 if (crp == NULL) { 571 error = ENOMEM; 572 goto bad; 573 } 574 575 auth_desc = crp->crp_desc; 576 enc_desc = auth_desc->crd_next; 577 578 crp->crp_session = session->fs_sid; 579 crp->crp_ilen = auth_len + datalen; 580 crp->crp_buf = (void*)GET_UIO_STRUCT(data_uio); 581 crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIFSYNC; 582 583 auth_desc->crd_skip = 0; 584 auth_desc->crd_len = auth_len; 585 auth_desc->crd_inject = auth_len + datalen; 586 auth_desc->crd_alg = xauth->type; 587 #ifdef FCRYPTO_DEBUG 588 printf("%s: auth: skip = %u, len = %u, inject = %u\n", 589 __FUNCTION__, auth_desc->crd_skip, auth_desc->crd_len, 590 auth_desc->crd_inject); 591 #endif 592 593 enc_desc->crd_skip = auth_len; 594 enc_desc->crd_len = datalen; 595 enc_desc->crd_inject = auth_len; 596 enc_desc->crd_alg = xform->type; 597 enc_desc->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT; 598 bcopy(ivbuf, enc_desc->crd_iv, ZIO_DATA_IV_LEN); 599 enc_desc->crd_next = NULL; 600 601 #ifdef FCRYPTO_DEBUG 602 printf("%s: enc: skip = %u, len = %u, inject = %u\n", 603 __FUNCTION__, enc_desc->crd_skip, enc_desc->crd_len, 604 enc_desc->crd_inject); 605 #endif 606 607 if (encrypt) 608 enc_desc->crd_flags |= CRD_F_ENCRYPT; 609 610 error = zfs_crypto_dispatch(session, crp); 611 crypto_freereq(crp); 612 out: 613 if (input_sessionp == NULL) { 614 freebsd_crypt_freesession(session); 615 kmem_free(session, sizeof (*session)); 616 } 617 bad: 618 #ifdef FCRYPTO_DEBUG 619 if (error) 620 printf("%s: returning error %d\n", __FUNCTION__, error); 621 #endif 622 return (error); 623 } 624 #endif 625