1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or https://opensource.org/licenses/CDDL-1.0. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. 24 * Copyright (c) 2012, 2016 by Delphix. All rights reserved. 25 * Copyright (c) 2022 by Pawel Jakub Dawidek 26 */ 27 28 #include <sys/zfs_context.h> 29 #include <sys/spa.h> 30 #include <sys/spa_impl.h> 31 #include <sys/zio.h> 32 #include <sys/ddt.h> 33 #include <sys/zap.h> 34 #include <sys/dmu_tx.h> 35 #include <sys/arc.h> 36 #include <sys/dsl_pool.h> 37 #include <sys/zio_checksum.h> 38 #include <sys/zio_compress.h> 39 #include <sys/dsl_scan.h> 40 #include <sys/abd.h> 41 42 static kmem_cache_t *ddt_cache; 43 static kmem_cache_t *ddt_entry_cache; 44 45 /* 46 * Enable/disable prefetching of dedup-ed blocks which are going to be freed. 47 */ 48 int zfs_dedup_prefetch = 0; 49 50 static const ddt_ops_t *const ddt_ops[DDT_TYPES] = { 51 &ddt_zap_ops, 52 }; 53 54 static const char *const ddt_class_name[DDT_CLASSES] = { 55 "ditto", 56 "duplicate", 57 "unique", 58 }; 59 60 static void 61 ddt_object_create(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 62 dmu_tx_t *tx) 63 { 64 spa_t *spa = ddt->ddt_spa; 65 objset_t *os = ddt->ddt_os; 66 uint64_t *objectp = &ddt->ddt_object[type][class]; 67 boolean_t prehash = zio_checksum_table[ddt->ddt_checksum].ci_flags & 68 ZCHECKSUM_FLAG_DEDUP; 69 char name[DDT_NAMELEN]; 70 71 ddt_object_name(ddt, type, class, name); 72 73 ASSERT(*objectp == 0); 74 VERIFY(ddt_ops[type]->ddt_op_create(os, objectp, tx, prehash) == 0); 75 ASSERT(*objectp != 0); 76 77 VERIFY(zap_add(os, DMU_POOL_DIRECTORY_OBJECT, name, 78 sizeof (uint64_t), 1, objectp, tx) == 0); 79 80 VERIFY(zap_add(os, spa->spa_ddt_stat_object, name, 81 sizeof (uint64_t), sizeof (ddt_histogram_t) / sizeof (uint64_t), 82 &ddt->ddt_histogram[type][class], tx) == 0); 83 } 84 85 static void 86 ddt_object_destroy(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 87 dmu_tx_t *tx) 88 { 89 spa_t *spa = ddt->ddt_spa; 90 objset_t *os = ddt->ddt_os; 91 uint64_t *objectp = &ddt->ddt_object[type][class]; 92 uint64_t count; 93 char name[DDT_NAMELEN]; 94 95 ddt_object_name(ddt, type, class, name); 96 97 ASSERT(*objectp != 0); 98 ASSERT(ddt_histogram_empty(&ddt->ddt_histogram[type][class])); 99 VERIFY(ddt_object_count(ddt, type, class, &count) == 0 && count == 0); 100 VERIFY(zap_remove(os, DMU_POOL_DIRECTORY_OBJECT, name, tx) == 0); 101 VERIFY(zap_remove(os, spa->spa_ddt_stat_object, name, tx) == 0); 102 VERIFY(ddt_ops[type]->ddt_op_destroy(os, *objectp, tx) == 0); 103 memset(&ddt->ddt_object_stats[type][class], 0, sizeof (ddt_object_t)); 104 105 *objectp = 0; 106 } 107 108 static int 109 ddt_object_load(ddt_t *ddt, enum ddt_type type, enum ddt_class class) 110 { 111 ddt_object_t *ddo = &ddt->ddt_object_stats[type][class]; 112 dmu_object_info_t doi; 113 uint64_t count; 114 char name[DDT_NAMELEN]; 115 int error; 116 117 ddt_object_name(ddt, type, class, name); 118 119 error = zap_lookup(ddt->ddt_os, DMU_POOL_DIRECTORY_OBJECT, name, 120 sizeof (uint64_t), 1, &ddt->ddt_object[type][class]); 121 if (error != 0) 122 return (error); 123 124 error = zap_lookup(ddt->ddt_os, ddt->ddt_spa->spa_ddt_stat_object, name, 125 sizeof (uint64_t), sizeof (ddt_histogram_t) / sizeof (uint64_t), 126 &ddt->ddt_histogram[type][class]); 127 if (error != 0) 128 return (error); 129 130 /* 131 * Seed the cached statistics. 132 */ 133 error = ddt_object_info(ddt, type, class, &doi); 134 if (error) 135 return (error); 136 137 error = ddt_object_count(ddt, type, class, &count); 138 if (error) 139 return (error); 140 141 ddo->ddo_count = count; 142 ddo->ddo_dspace = doi.doi_physical_blocks_512 << 9; 143 ddo->ddo_mspace = doi.doi_fill_count * doi.doi_data_block_size; 144 145 return (0); 146 } 147 148 static void 149 ddt_object_sync(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 150 dmu_tx_t *tx) 151 { 152 ddt_object_t *ddo = &ddt->ddt_object_stats[type][class]; 153 dmu_object_info_t doi; 154 uint64_t count; 155 char name[DDT_NAMELEN]; 156 157 ddt_object_name(ddt, type, class, name); 158 159 VERIFY(zap_update(ddt->ddt_os, ddt->ddt_spa->spa_ddt_stat_object, name, 160 sizeof (uint64_t), sizeof (ddt_histogram_t) / sizeof (uint64_t), 161 &ddt->ddt_histogram[type][class], tx) == 0); 162 163 /* 164 * Cache DDT statistics; this is the only time they'll change. 165 */ 166 VERIFY(ddt_object_info(ddt, type, class, &doi) == 0); 167 VERIFY(ddt_object_count(ddt, type, class, &count) == 0); 168 169 ddo->ddo_count = count; 170 ddo->ddo_dspace = doi.doi_physical_blocks_512 << 9; 171 ddo->ddo_mspace = doi.doi_fill_count * doi.doi_data_block_size; 172 } 173 174 static int 175 ddt_object_lookup(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 176 ddt_entry_t *dde) 177 { 178 if (!ddt_object_exists(ddt, type, class)) 179 return (SET_ERROR(ENOENT)); 180 181 return (ddt_ops[type]->ddt_op_lookup(ddt->ddt_os, 182 ddt->ddt_object[type][class], dde)); 183 } 184 185 static void 186 ddt_object_prefetch(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 187 ddt_entry_t *dde) 188 { 189 if (!ddt_object_exists(ddt, type, class)) 190 return; 191 192 ddt_ops[type]->ddt_op_prefetch(ddt->ddt_os, 193 ddt->ddt_object[type][class], dde); 194 } 195 196 int 197 ddt_object_update(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 198 ddt_entry_t *dde, dmu_tx_t *tx) 199 { 200 ASSERT(ddt_object_exists(ddt, type, class)); 201 202 return (ddt_ops[type]->ddt_op_update(ddt->ddt_os, 203 ddt->ddt_object[type][class], dde, tx)); 204 } 205 206 static int 207 ddt_object_remove(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 208 ddt_entry_t *dde, dmu_tx_t *tx) 209 { 210 ASSERT(ddt_object_exists(ddt, type, class)); 211 212 return (ddt_ops[type]->ddt_op_remove(ddt->ddt_os, 213 ddt->ddt_object[type][class], dde, tx)); 214 } 215 216 int 217 ddt_object_walk(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 218 uint64_t *walk, ddt_entry_t *dde) 219 { 220 ASSERT(ddt_object_exists(ddt, type, class)); 221 222 return (ddt_ops[type]->ddt_op_walk(ddt->ddt_os, 223 ddt->ddt_object[type][class], dde, walk)); 224 } 225 226 int 227 ddt_object_count(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 228 uint64_t *count) 229 { 230 ASSERT(ddt_object_exists(ddt, type, class)); 231 232 return (ddt_ops[type]->ddt_op_count(ddt->ddt_os, 233 ddt->ddt_object[type][class], count)); 234 } 235 236 int 237 ddt_object_info(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 238 dmu_object_info_t *doi) 239 { 240 if (!ddt_object_exists(ddt, type, class)) 241 return (SET_ERROR(ENOENT)); 242 243 return (dmu_object_info(ddt->ddt_os, ddt->ddt_object[type][class], 244 doi)); 245 } 246 247 boolean_t 248 ddt_object_exists(ddt_t *ddt, enum ddt_type type, enum ddt_class class) 249 { 250 return (!!ddt->ddt_object[type][class]); 251 } 252 253 void 254 ddt_object_name(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 255 char *name) 256 { 257 (void) snprintf(name, DDT_NAMELEN, DMU_POOL_DDT, 258 zio_checksum_table[ddt->ddt_checksum].ci_name, 259 ddt_ops[type]->ddt_op_name, ddt_class_name[class]); 260 } 261 262 void 263 ddt_bp_fill(const ddt_phys_t *ddp, blkptr_t *bp, uint64_t txg) 264 { 265 ASSERT(txg != 0); 266 267 for (int d = 0; d < SPA_DVAS_PER_BP; d++) 268 bp->blk_dva[d] = ddp->ddp_dva[d]; 269 BP_SET_BIRTH(bp, txg, ddp->ddp_phys_birth); 270 } 271 272 /* 273 * The bp created via this function may be used for repairs and scrub, but it 274 * will be missing the salt / IV required to do a full decrypting read. 275 */ 276 void 277 ddt_bp_create(enum zio_checksum checksum, 278 const ddt_key_t *ddk, const ddt_phys_t *ddp, blkptr_t *bp) 279 { 280 BP_ZERO(bp); 281 282 if (ddp != NULL) 283 ddt_bp_fill(ddp, bp, ddp->ddp_phys_birth); 284 285 bp->blk_cksum = ddk->ddk_cksum; 286 287 BP_SET_LSIZE(bp, DDK_GET_LSIZE(ddk)); 288 BP_SET_PSIZE(bp, DDK_GET_PSIZE(ddk)); 289 BP_SET_COMPRESS(bp, DDK_GET_COMPRESS(ddk)); 290 BP_SET_CRYPT(bp, DDK_GET_CRYPT(ddk)); 291 BP_SET_FILL(bp, 1); 292 BP_SET_CHECKSUM(bp, checksum); 293 BP_SET_TYPE(bp, DMU_OT_DEDUP); 294 BP_SET_LEVEL(bp, 0); 295 BP_SET_DEDUP(bp, 1); 296 BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER); 297 } 298 299 void 300 ddt_key_fill(ddt_key_t *ddk, const blkptr_t *bp) 301 { 302 ddk->ddk_cksum = bp->blk_cksum; 303 ddk->ddk_prop = 0; 304 305 ASSERT(BP_IS_ENCRYPTED(bp) || !BP_USES_CRYPT(bp)); 306 307 DDK_SET_LSIZE(ddk, BP_GET_LSIZE(bp)); 308 DDK_SET_PSIZE(ddk, BP_GET_PSIZE(bp)); 309 DDK_SET_COMPRESS(ddk, BP_GET_COMPRESS(bp)); 310 DDK_SET_CRYPT(ddk, BP_USES_CRYPT(bp)); 311 } 312 313 void 314 ddt_phys_fill(ddt_phys_t *ddp, const blkptr_t *bp) 315 { 316 ASSERT(ddp->ddp_phys_birth == 0); 317 318 for (int d = 0; d < SPA_DVAS_PER_BP; d++) 319 ddp->ddp_dva[d] = bp->blk_dva[d]; 320 ddp->ddp_phys_birth = BP_PHYSICAL_BIRTH(bp); 321 } 322 323 void 324 ddt_phys_clear(ddt_phys_t *ddp) 325 { 326 memset(ddp, 0, sizeof (*ddp)); 327 } 328 329 void 330 ddt_phys_addref(ddt_phys_t *ddp) 331 { 332 ddp->ddp_refcnt++; 333 } 334 335 void 336 ddt_phys_decref(ddt_phys_t *ddp) 337 { 338 if (ddp) { 339 ASSERT(ddp->ddp_refcnt > 0); 340 ddp->ddp_refcnt--; 341 } 342 } 343 344 void 345 ddt_phys_free(ddt_t *ddt, ddt_key_t *ddk, ddt_phys_t *ddp, uint64_t txg) 346 { 347 blkptr_t blk; 348 349 ddt_bp_create(ddt->ddt_checksum, ddk, ddp, &blk); 350 351 /* 352 * We clear the dedup bit so that zio_free() will actually free the 353 * space, rather than just decrementing the refcount in the DDT. 354 */ 355 BP_SET_DEDUP(&blk, 0); 356 357 ddt_phys_clear(ddp); 358 zio_free(ddt->ddt_spa, txg, &blk); 359 } 360 361 ddt_phys_t * 362 ddt_phys_select(const ddt_entry_t *dde, const blkptr_t *bp) 363 { 364 ddt_phys_t *ddp = (ddt_phys_t *)dde->dde_phys; 365 366 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { 367 if (DVA_EQUAL(BP_IDENTITY(bp), &ddp->ddp_dva[0]) && 368 BP_PHYSICAL_BIRTH(bp) == ddp->ddp_phys_birth) 369 return (ddp); 370 } 371 return (NULL); 372 } 373 374 uint64_t 375 ddt_phys_total_refcnt(const ddt_entry_t *dde) 376 { 377 uint64_t refcnt = 0; 378 379 for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) 380 refcnt += dde->dde_phys[p].ddp_refcnt; 381 382 return (refcnt); 383 } 384 385 static void 386 ddt_stat_generate(ddt_t *ddt, ddt_entry_t *dde, ddt_stat_t *dds) 387 { 388 spa_t *spa = ddt->ddt_spa; 389 ddt_phys_t *ddp = dde->dde_phys; 390 ddt_key_t *ddk = &dde->dde_key; 391 uint64_t lsize = DDK_GET_LSIZE(ddk); 392 uint64_t psize = DDK_GET_PSIZE(ddk); 393 394 memset(dds, 0, sizeof (*dds)); 395 396 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { 397 uint64_t dsize = 0; 398 uint64_t refcnt = ddp->ddp_refcnt; 399 400 if (ddp->ddp_phys_birth == 0) 401 continue; 402 403 for (int d = 0; d < DDE_GET_NDVAS(dde); d++) 404 dsize += dva_get_dsize_sync(spa, &ddp->ddp_dva[d]); 405 406 dds->dds_blocks += 1; 407 dds->dds_lsize += lsize; 408 dds->dds_psize += psize; 409 dds->dds_dsize += dsize; 410 411 dds->dds_ref_blocks += refcnt; 412 dds->dds_ref_lsize += lsize * refcnt; 413 dds->dds_ref_psize += psize * refcnt; 414 dds->dds_ref_dsize += dsize * refcnt; 415 } 416 } 417 418 void 419 ddt_stat_add(ddt_stat_t *dst, const ddt_stat_t *src, uint64_t neg) 420 { 421 const uint64_t *s = (const uint64_t *)src; 422 uint64_t *d = (uint64_t *)dst; 423 uint64_t *d_end = (uint64_t *)(dst + 1); 424 425 ASSERT(neg == 0 || neg == -1ULL); /* add or subtract */ 426 427 for (int i = 0; i < d_end - d; i++) 428 d[i] += (s[i] ^ neg) - neg; 429 } 430 431 static void 432 ddt_stat_update(ddt_t *ddt, ddt_entry_t *dde, uint64_t neg) 433 { 434 ddt_stat_t dds; 435 ddt_histogram_t *ddh; 436 int bucket; 437 438 ddt_stat_generate(ddt, dde, &dds); 439 440 bucket = highbit64(dds.dds_ref_blocks) - 1; 441 ASSERT(bucket >= 0); 442 443 ddh = &ddt->ddt_histogram[dde->dde_type][dde->dde_class]; 444 445 ddt_stat_add(&ddh->ddh_stat[bucket], &dds, neg); 446 } 447 448 void 449 ddt_histogram_add(ddt_histogram_t *dst, const ddt_histogram_t *src) 450 { 451 for (int h = 0; h < 64; h++) 452 ddt_stat_add(&dst->ddh_stat[h], &src->ddh_stat[h], 0); 453 } 454 455 void 456 ddt_histogram_stat(ddt_stat_t *dds, const ddt_histogram_t *ddh) 457 { 458 memset(dds, 0, sizeof (*dds)); 459 460 for (int h = 0; h < 64; h++) 461 ddt_stat_add(dds, &ddh->ddh_stat[h], 0); 462 } 463 464 boolean_t 465 ddt_histogram_empty(const ddt_histogram_t *ddh) 466 { 467 const uint64_t *s = (const uint64_t *)ddh; 468 const uint64_t *s_end = (const uint64_t *)(ddh + 1); 469 470 while (s < s_end) 471 if (*s++ != 0) 472 return (B_FALSE); 473 474 return (B_TRUE); 475 } 476 477 void 478 ddt_get_dedup_object_stats(spa_t *spa, ddt_object_t *ddo_total) 479 { 480 /* Sum the statistics we cached in ddt_object_sync(). */ 481 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { 482 ddt_t *ddt = spa->spa_ddt[c]; 483 for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 484 for (enum ddt_class class = 0; class < DDT_CLASSES; 485 class++) { 486 ddt_object_t *ddo = 487 &ddt->ddt_object_stats[type][class]; 488 ddo_total->ddo_count += ddo->ddo_count; 489 ddo_total->ddo_dspace += ddo->ddo_dspace; 490 ddo_total->ddo_mspace += ddo->ddo_mspace; 491 } 492 } 493 } 494 495 /* ... and compute the averages. */ 496 if (ddo_total->ddo_count != 0) { 497 ddo_total->ddo_dspace /= ddo_total->ddo_count; 498 ddo_total->ddo_mspace /= ddo_total->ddo_count; 499 } 500 } 501 502 void 503 ddt_get_dedup_histogram(spa_t *spa, ddt_histogram_t *ddh) 504 { 505 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { 506 ddt_t *ddt = spa->spa_ddt[c]; 507 for (enum ddt_type type = 0; type < DDT_TYPES && ddt; type++) { 508 for (enum ddt_class class = 0; class < DDT_CLASSES; 509 class++) { 510 ddt_histogram_add(ddh, 511 &ddt->ddt_histogram_cache[type][class]); 512 } 513 } 514 } 515 } 516 517 void 518 ddt_get_dedup_stats(spa_t *spa, ddt_stat_t *dds_total) 519 { 520 ddt_histogram_t *ddh_total; 521 522 ddh_total = kmem_zalloc(sizeof (ddt_histogram_t), KM_SLEEP); 523 ddt_get_dedup_histogram(spa, ddh_total); 524 ddt_histogram_stat(dds_total, ddh_total); 525 kmem_free(ddh_total, sizeof (ddt_histogram_t)); 526 } 527 528 uint64_t 529 ddt_get_dedup_dspace(spa_t *spa) 530 { 531 ddt_stat_t dds_total; 532 533 if (spa->spa_dedup_dspace != ~0ULL) 534 return (spa->spa_dedup_dspace); 535 536 memset(&dds_total, 0, sizeof (ddt_stat_t)); 537 538 /* Calculate and cache the stats */ 539 ddt_get_dedup_stats(spa, &dds_total); 540 spa->spa_dedup_dspace = dds_total.dds_ref_dsize - dds_total.dds_dsize; 541 return (spa->spa_dedup_dspace); 542 } 543 544 uint64_t 545 ddt_get_pool_dedup_ratio(spa_t *spa) 546 { 547 ddt_stat_t dds_total = { 0 }; 548 549 ddt_get_dedup_stats(spa, &dds_total); 550 if (dds_total.dds_dsize == 0) 551 return (100); 552 553 return (dds_total.dds_ref_dsize * 100 / dds_total.dds_dsize); 554 } 555 556 size_t 557 ddt_compress(void *src, uchar_t *dst, size_t s_len, size_t d_len) 558 { 559 uchar_t *version = dst++; 560 int cpfunc = ZIO_COMPRESS_ZLE; 561 zio_compress_info_t *ci = &zio_compress_table[cpfunc]; 562 size_t c_len; 563 564 ASSERT(d_len >= s_len + 1); /* no compression plus version byte */ 565 566 c_len = ci->ci_compress(src, dst, s_len, d_len - 1, ci->ci_level); 567 568 if (c_len == s_len) { 569 cpfunc = ZIO_COMPRESS_OFF; 570 memcpy(dst, src, s_len); 571 } 572 573 *version = cpfunc; 574 if (ZFS_HOST_BYTEORDER) 575 *version |= DDT_COMPRESS_BYTEORDER_MASK; 576 577 return (c_len + 1); 578 } 579 580 void 581 ddt_decompress(uchar_t *src, void *dst, size_t s_len, size_t d_len) 582 { 583 uchar_t version = *src++; 584 int cpfunc = version & DDT_COMPRESS_FUNCTION_MASK; 585 zio_compress_info_t *ci = &zio_compress_table[cpfunc]; 586 587 if (ci->ci_decompress != NULL) 588 (void) ci->ci_decompress(src, dst, s_len, d_len, ci->ci_level); 589 else 590 memcpy(dst, src, d_len); 591 592 if (((version & DDT_COMPRESS_BYTEORDER_MASK) != 0) != 593 (ZFS_HOST_BYTEORDER != 0)) 594 byteswap_uint64_array(dst, d_len); 595 } 596 597 ddt_t * 598 ddt_select(spa_t *spa, const blkptr_t *bp) 599 { 600 return (spa->spa_ddt[BP_GET_CHECKSUM(bp)]); 601 } 602 603 void 604 ddt_enter(ddt_t *ddt) 605 { 606 mutex_enter(&ddt->ddt_lock); 607 } 608 609 void 610 ddt_exit(ddt_t *ddt) 611 { 612 mutex_exit(&ddt->ddt_lock); 613 } 614 615 void 616 ddt_init(void) 617 { 618 ddt_cache = kmem_cache_create("ddt_cache", 619 sizeof (ddt_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 620 ddt_entry_cache = kmem_cache_create("ddt_entry_cache", 621 sizeof (ddt_entry_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 622 } 623 624 void 625 ddt_fini(void) 626 { 627 kmem_cache_destroy(ddt_entry_cache); 628 kmem_cache_destroy(ddt_cache); 629 } 630 631 static ddt_entry_t * 632 ddt_alloc(const ddt_key_t *ddk) 633 { 634 ddt_entry_t *dde; 635 636 dde = kmem_cache_alloc(ddt_entry_cache, KM_SLEEP); 637 memset(dde, 0, sizeof (ddt_entry_t)); 638 cv_init(&dde->dde_cv, NULL, CV_DEFAULT, NULL); 639 640 dde->dde_key = *ddk; 641 642 return (dde); 643 } 644 645 static void 646 ddt_free(ddt_entry_t *dde) 647 { 648 ASSERT(!dde->dde_loading); 649 650 for (int p = 0; p < DDT_PHYS_TYPES; p++) 651 ASSERT(dde->dde_lead_zio[p] == NULL); 652 653 if (dde->dde_repair_abd != NULL) 654 abd_free(dde->dde_repair_abd); 655 656 cv_destroy(&dde->dde_cv); 657 kmem_cache_free(ddt_entry_cache, dde); 658 } 659 660 void 661 ddt_remove(ddt_t *ddt, ddt_entry_t *dde) 662 { 663 ASSERT(MUTEX_HELD(&ddt->ddt_lock)); 664 665 avl_remove(&ddt->ddt_tree, dde); 666 ddt_free(dde); 667 } 668 669 ddt_entry_t * 670 ddt_lookup(ddt_t *ddt, const blkptr_t *bp, boolean_t add) 671 { 672 ddt_entry_t *dde, dde_search; 673 enum ddt_type type; 674 enum ddt_class class; 675 avl_index_t where; 676 int error; 677 678 ASSERT(MUTEX_HELD(&ddt->ddt_lock)); 679 680 ddt_key_fill(&dde_search.dde_key, bp); 681 682 dde = avl_find(&ddt->ddt_tree, &dde_search, &where); 683 if (dde == NULL) { 684 if (!add) 685 return (NULL); 686 dde = ddt_alloc(&dde_search.dde_key); 687 avl_insert(&ddt->ddt_tree, dde, where); 688 } 689 690 while (dde->dde_loading) 691 cv_wait(&dde->dde_cv, &ddt->ddt_lock); 692 693 if (dde->dde_loaded) 694 return (dde); 695 696 dde->dde_loading = B_TRUE; 697 698 ddt_exit(ddt); 699 700 error = ENOENT; 701 702 for (type = 0; type < DDT_TYPES; type++) { 703 for (class = 0; class < DDT_CLASSES; class++) { 704 error = ddt_object_lookup(ddt, type, class, dde); 705 if (error != ENOENT) { 706 ASSERT0(error); 707 break; 708 } 709 } 710 if (error != ENOENT) 711 break; 712 } 713 714 ddt_enter(ddt); 715 716 ASSERT(dde->dde_loaded == B_FALSE); 717 ASSERT(dde->dde_loading == B_TRUE); 718 719 dde->dde_type = type; /* will be DDT_TYPES if no entry found */ 720 dde->dde_class = class; /* will be DDT_CLASSES if no entry found */ 721 dde->dde_loaded = B_TRUE; 722 dde->dde_loading = B_FALSE; 723 724 if (error == 0) 725 ddt_stat_update(ddt, dde, -1ULL); 726 727 cv_broadcast(&dde->dde_cv); 728 729 return (dde); 730 } 731 732 void 733 ddt_prefetch(spa_t *spa, const blkptr_t *bp) 734 { 735 ddt_t *ddt; 736 ddt_entry_t dde; 737 738 if (!zfs_dedup_prefetch || bp == NULL || !BP_GET_DEDUP(bp)) 739 return; 740 741 /* 742 * We only remove the DDT once all tables are empty and only 743 * prefetch dedup blocks when there are entries in the DDT. 744 * Thus no locking is required as the DDT can't disappear on us. 745 */ 746 ddt = ddt_select(spa, bp); 747 ddt_key_fill(&dde.dde_key, bp); 748 749 for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 750 for (enum ddt_class class = 0; class < DDT_CLASSES; class++) { 751 ddt_object_prefetch(ddt, type, class, &dde); 752 } 753 } 754 } 755 756 /* 757 * Opaque struct used for ddt_key comparison 758 */ 759 #define DDT_KEY_CMP_LEN (sizeof (ddt_key_t) / sizeof (uint16_t)) 760 761 typedef struct ddt_key_cmp { 762 uint16_t u16[DDT_KEY_CMP_LEN]; 763 } ddt_key_cmp_t; 764 765 int 766 ddt_entry_compare(const void *x1, const void *x2) 767 { 768 const ddt_entry_t *dde1 = x1; 769 const ddt_entry_t *dde2 = x2; 770 const ddt_key_cmp_t *k1 = (const ddt_key_cmp_t *)&dde1->dde_key; 771 const ddt_key_cmp_t *k2 = (const ddt_key_cmp_t *)&dde2->dde_key; 772 int32_t cmp = 0; 773 774 for (int i = 0; i < DDT_KEY_CMP_LEN; i++) { 775 cmp = (int32_t)k1->u16[i] - (int32_t)k2->u16[i]; 776 if (likely(cmp)) 777 break; 778 } 779 780 return (TREE_ISIGN(cmp)); 781 } 782 783 static ddt_t * 784 ddt_table_alloc(spa_t *spa, enum zio_checksum c) 785 { 786 ddt_t *ddt; 787 788 ddt = kmem_cache_alloc(ddt_cache, KM_SLEEP); 789 memset(ddt, 0, sizeof (ddt_t)); 790 791 mutex_init(&ddt->ddt_lock, NULL, MUTEX_DEFAULT, NULL); 792 avl_create(&ddt->ddt_tree, ddt_entry_compare, 793 sizeof (ddt_entry_t), offsetof(ddt_entry_t, dde_node)); 794 avl_create(&ddt->ddt_repair_tree, ddt_entry_compare, 795 sizeof (ddt_entry_t), offsetof(ddt_entry_t, dde_node)); 796 ddt->ddt_checksum = c; 797 ddt->ddt_spa = spa; 798 ddt->ddt_os = spa->spa_meta_objset; 799 800 return (ddt); 801 } 802 803 static void 804 ddt_table_free(ddt_t *ddt) 805 { 806 ASSERT(avl_numnodes(&ddt->ddt_tree) == 0); 807 ASSERT(avl_numnodes(&ddt->ddt_repair_tree) == 0); 808 avl_destroy(&ddt->ddt_tree); 809 avl_destroy(&ddt->ddt_repair_tree); 810 mutex_destroy(&ddt->ddt_lock); 811 kmem_cache_free(ddt_cache, ddt); 812 } 813 814 void 815 ddt_create(spa_t *spa) 816 { 817 spa->spa_dedup_checksum = ZIO_DEDUPCHECKSUM; 818 819 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) 820 spa->spa_ddt[c] = ddt_table_alloc(spa, c); 821 } 822 823 int 824 ddt_load(spa_t *spa) 825 { 826 int error; 827 828 ddt_create(spa); 829 830 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 831 DMU_POOL_DDT_STATS, sizeof (uint64_t), 1, 832 &spa->spa_ddt_stat_object); 833 834 if (error) 835 return (error == ENOENT ? 0 : error); 836 837 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { 838 ddt_t *ddt = spa->spa_ddt[c]; 839 for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 840 for (enum ddt_class class = 0; class < DDT_CLASSES; 841 class++) { 842 error = ddt_object_load(ddt, type, class); 843 if (error != 0 && error != ENOENT) 844 return (error); 845 } 846 } 847 848 /* 849 * Seed the cached histograms. 850 */ 851 memcpy(&ddt->ddt_histogram_cache, ddt->ddt_histogram, 852 sizeof (ddt->ddt_histogram)); 853 spa->spa_dedup_dspace = ~0ULL; 854 } 855 856 return (0); 857 } 858 859 void 860 ddt_unload(spa_t *spa) 861 { 862 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { 863 if (spa->spa_ddt[c]) { 864 ddt_table_free(spa->spa_ddt[c]); 865 spa->spa_ddt[c] = NULL; 866 } 867 } 868 } 869 870 boolean_t 871 ddt_class_contains(spa_t *spa, enum ddt_class max_class, const blkptr_t *bp) 872 { 873 ddt_t *ddt; 874 ddt_entry_t *dde; 875 876 if (!BP_GET_DEDUP(bp)) 877 return (B_FALSE); 878 879 if (max_class == DDT_CLASS_UNIQUE) 880 return (B_TRUE); 881 882 ddt = spa->spa_ddt[BP_GET_CHECKSUM(bp)]; 883 dde = kmem_cache_alloc(ddt_entry_cache, KM_SLEEP); 884 885 ddt_key_fill(&(dde->dde_key), bp); 886 887 for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 888 for (enum ddt_class class = 0; class <= max_class; class++) { 889 if (ddt_object_lookup(ddt, type, class, dde) == 0) { 890 kmem_cache_free(ddt_entry_cache, dde); 891 return (B_TRUE); 892 } 893 } 894 } 895 896 kmem_cache_free(ddt_entry_cache, dde); 897 return (B_FALSE); 898 } 899 900 ddt_entry_t * 901 ddt_repair_start(ddt_t *ddt, const blkptr_t *bp) 902 { 903 ddt_key_t ddk; 904 ddt_entry_t *dde; 905 906 ddt_key_fill(&ddk, bp); 907 908 dde = ddt_alloc(&ddk); 909 910 for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 911 for (enum ddt_class class = 0; class < DDT_CLASSES; class++) { 912 /* 913 * We can only do repair if there are multiple copies 914 * of the block. For anything in the UNIQUE class, 915 * there's definitely only one copy, so don't even try. 916 */ 917 if (class != DDT_CLASS_UNIQUE && 918 ddt_object_lookup(ddt, type, class, dde) == 0) 919 return (dde); 920 } 921 } 922 923 memset(dde->dde_phys, 0, sizeof (dde->dde_phys)); 924 925 return (dde); 926 } 927 928 void 929 ddt_repair_done(ddt_t *ddt, ddt_entry_t *dde) 930 { 931 avl_index_t where; 932 933 ddt_enter(ddt); 934 935 if (dde->dde_repair_abd != NULL && spa_writeable(ddt->ddt_spa) && 936 avl_find(&ddt->ddt_repair_tree, dde, &where) == NULL) 937 avl_insert(&ddt->ddt_repair_tree, dde, where); 938 else 939 ddt_free(dde); 940 941 ddt_exit(ddt); 942 } 943 944 static void 945 ddt_repair_entry_done(zio_t *zio) 946 { 947 ddt_entry_t *rdde = zio->io_private; 948 949 ddt_free(rdde); 950 } 951 952 static void 953 ddt_repair_entry(ddt_t *ddt, ddt_entry_t *dde, ddt_entry_t *rdde, zio_t *rio) 954 { 955 ddt_phys_t *ddp = dde->dde_phys; 956 ddt_phys_t *rddp = rdde->dde_phys; 957 ddt_key_t *ddk = &dde->dde_key; 958 ddt_key_t *rddk = &rdde->dde_key; 959 zio_t *zio; 960 blkptr_t blk; 961 962 zio = zio_null(rio, rio->io_spa, NULL, 963 ddt_repair_entry_done, rdde, rio->io_flags); 964 965 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++, rddp++) { 966 if (ddp->ddp_phys_birth == 0 || 967 ddp->ddp_phys_birth != rddp->ddp_phys_birth || 968 memcmp(ddp->ddp_dva, rddp->ddp_dva, sizeof (ddp->ddp_dva))) 969 continue; 970 ddt_bp_create(ddt->ddt_checksum, ddk, ddp, &blk); 971 zio_nowait(zio_rewrite(zio, zio->io_spa, 0, &blk, 972 rdde->dde_repair_abd, DDK_GET_PSIZE(rddk), NULL, NULL, 973 ZIO_PRIORITY_SYNC_WRITE, ZIO_DDT_CHILD_FLAGS(zio), NULL)); 974 } 975 976 zio_nowait(zio); 977 } 978 979 static void 980 ddt_repair_table(ddt_t *ddt, zio_t *rio) 981 { 982 spa_t *spa = ddt->ddt_spa; 983 ddt_entry_t *dde, *rdde_next, *rdde; 984 avl_tree_t *t = &ddt->ddt_repair_tree; 985 blkptr_t blk; 986 987 if (spa_sync_pass(spa) > 1) 988 return; 989 990 ddt_enter(ddt); 991 for (rdde = avl_first(t); rdde != NULL; rdde = rdde_next) { 992 rdde_next = AVL_NEXT(t, rdde); 993 avl_remove(&ddt->ddt_repair_tree, rdde); 994 ddt_exit(ddt); 995 ddt_bp_create(ddt->ddt_checksum, &rdde->dde_key, NULL, &blk); 996 dde = ddt_repair_start(ddt, &blk); 997 ddt_repair_entry(ddt, dde, rdde, rio); 998 ddt_repair_done(ddt, dde); 999 ddt_enter(ddt); 1000 } 1001 ddt_exit(ddt); 1002 } 1003 1004 static void 1005 ddt_sync_entry(ddt_t *ddt, ddt_entry_t *dde, dmu_tx_t *tx, uint64_t txg) 1006 { 1007 dsl_pool_t *dp = ddt->ddt_spa->spa_dsl_pool; 1008 ddt_phys_t *ddp = dde->dde_phys; 1009 ddt_key_t *ddk = &dde->dde_key; 1010 enum ddt_type otype = dde->dde_type; 1011 enum ddt_type ntype = DDT_TYPE_CURRENT; 1012 enum ddt_class oclass = dde->dde_class; 1013 enum ddt_class nclass; 1014 uint64_t total_refcnt = 0; 1015 1016 ASSERT(dde->dde_loaded); 1017 ASSERT(!dde->dde_loading); 1018 1019 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { 1020 ASSERT(dde->dde_lead_zio[p] == NULL); 1021 if (ddp->ddp_phys_birth == 0) { 1022 ASSERT(ddp->ddp_refcnt == 0); 1023 continue; 1024 } 1025 if (p == DDT_PHYS_DITTO) { 1026 /* 1027 * Note, we no longer create DDT-DITTO blocks, but we 1028 * don't want to leak any written by older software. 1029 */ 1030 ddt_phys_free(ddt, ddk, ddp, txg); 1031 continue; 1032 } 1033 if (ddp->ddp_refcnt == 0) 1034 ddt_phys_free(ddt, ddk, ddp, txg); 1035 total_refcnt += ddp->ddp_refcnt; 1036 } 1037 1038 /* We do not create new DDT-DITTO blocks. */ 1039 ASSERT0(dde->dde_phys[DDT_PHYS_DITTO].ddp_phys_birth); 1040 if (total_refcnt > 1) 1041 nclass = DDT_CLASS_DUPLICATE; 1042 else 1043 nclass = DDT_CLASS_UNIQUE; 1044 1045 if (otype != DDT_TYPES && 1046 (otype != ntype || oclass != nclass || total_refcnt == 0)) { 1047 VERIFY(ddt_object_remove(ddt, otype, oclass, dde, tx) == 0); 1048 ASSERT(ddt_object_lookup(ddt, otype, oclass, dde) == ENOENT); 1049 } 1050 1051 if (total_refcnt != 0) { 1052 dde->dde_type = ntype; 1053 dde->dde_class = nclass; 1054 ddt_stat_update(ddt, dde, 0); 1055 if (!ddt_object_exists(ddt, ntype, nclass)) 1056 ddt_object_create(ddt, ntype, nclass, tx); 1057 VERIFY(ddt_object_update(ddt, ntype, nclass, dde, tx) == 0); 1058 1059 /* 1060 * If the class changes, the order that we scan this bp 1061 * changes. If it decreases, we could miss it, so 1062 * scan it right now. (This covers both class changing 1063 * while we are doing ddt_walk(), and when we are 1064 * traversing.) 1065 */ 1066 if (nclass < oclass) { 1067 dsl_scan_ddt_entry(dp->dp_scan, 1068 ddt->ddt_checksum, dde, tx); 1069 } 1070 } 1071 } 1072 1073 static void 1074 ddt_sync_table(ddt_t *ddt, dmu_tx_t *tx, uint64_t txg) 1075 { 1076 spa_t *spa = ddt->ddt_spa; 1077 ddt_entry_t *dde; 1078 void *cookie = NULL; 1079 1080 if (avl_numnodes(&ddt->ddt_tree) == 0) 1081 return; 1082 1083 ASSERT(spa->spa_uberblock.ub_version >= SPA_VERSION_DEDUP); 1084 1085 if (spa->spa_ddt_stat_object == 0) { 1086 spa->spa_ddt_stat_object = zap_create_link(ddt->ddt_os, 1087 DMU_OT_DDT_STATS, DMU_POOL_DIRECTORY_OBJECT, 1088 DMU_POOL_DDT_STATS, tx); 1089 } 1090 1091 while ((dde = avl_destroy_nodes(&ddt->ddt_tree, &cookie)) != NULL) { 1092 ddt_sync_entry(ddt, dde, tx, txg); 1093 ddt_free(dde); 1094 } 1095 1096 for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 1097 uint64_t add, count = 0; 1098 for (enum ddt_class class = 0; class < DDT_CLASSES; class++) { 1099 if (ddt_object_exists(ddt, type, class)) { 1100 ddt_object_sync(ddt, type, class, tx); 1101 VERIFY(ddt_object_count(ddt, type, class, 1102 &add) == 0); 1103 count += add; 1104 } 1105 } 1106 for (enum ddt_class class = 0; class < DDT_CLASSES; class++) { 1107 if (count == 0 && ddt_object_exists(ddt, type, class)) 1108 ddt_object_destroy(ddt, type, class, tx); 1109 } 1110 } 1111 1112 memcpy(&ddt->ddt_histogram_cache, ddt->ddt_histogram, 1113 sizeof (ddt->ddt_histogram)); 1114 spa->spa_dedup_dspace = ~0ULL; 1115 } 1116 1117 void 1118 ddt_sync(spa_t *spa, uint64_t txg) 1119 { 1120 dsl_scan_t *scn = spa->spa_dsl_pool->dp_scan; 1121 dmu_tx_t *tx; 1122 zio_t *rio; 1123 1124 ASSERT(spa_syncing_txg(spa) == txg); 1125 1126 tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg); 1127 1128 rio = zio_root(spa, NULL, NULL, 1129 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | ZIO_FLAG_SELF_HEAL); 1130 1131 /* 1132 * This function may cause an immediate scan of ddt blocks (see 1133 * the comment above dsl_scan_ddt() for details). We set the 1134 * scan's root zio here so that we can wait for any scan IOs in 1135 * addition to the regular ddt IOs. 1136 */ 1137 ASSERT3P(scn->scn_zio_root, ==, NULL); 1138 scn->scn_zio_root = rio; 1139 1140 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { 1141 ddt_t *ddt = spa->spa_ddt[c]; 1142 if (ddt == NULL) 1143 continue; 1144 ddt_sync_table(ddt, tx, txg); 1145 ddt_repair_table(ddt, rio); 1146 } 1147 1148 (void) zio_wait(rio); 1149 scn->scn_zio_root = NULL; 1150 1151 dmu_tx_commit(tx); 1152 } 1153 1154 int 1155 ddt_walk(spa_t *spa, ddt_bookmark_t *ddb, ddt_entry_t *dde) 1156 { 1157 do { 1158 do { 1159 do { 1160 ddt_t *ddt = spa->spa_ddt[ddb->ddb_checksum]; 1161 int error = ENOENT; 1162 if (ddt_object_exists(ddt, ddb->ddb_type, 1163 ddb->ddb_class)) { 1164 error = ddt_object_walk(ddt, 1165 ddb->ddb_type, ddb->ddb_class, 1166 &ddb->ddb_cursor, dde); 1167 } 1168 dde->dde_type = ddb->ddb_type; 1169 dde->dde_class = ddb->ddb_class; 1170 if (error == 0) 1171 return (0); 1172 if (error != ENOENT) 1173 return (error); 1174 ddb->ddb_cursor = 0; 1175 } while (++ddb->ddb_checksum < ZIO_CHECKSUM_FUNCTIONS); 1176 ddb->ddb_checksum = 0; 1177 } while (++ddb->ddb_type < DDT_TYPES); 1178 ddb->ddb_type = 0; 1179 } while (++ddb->ddb_class < DDT_CLASSES); 1180 1181 return (SET_ERROR(ENOENT)); 1182 } 1183 1184 /* 1185 * This function is used by Block Cloning (brt.c) to increase reference 1186 * counter for the DDT entry if the block is already in DDT. 1187 * 1188 * Return false if the block, despite having the D bit set, is not present 1189 * in the DDT. Currently this is not possible but might be in the future. 1190 * See the comment below. 1191 */ 1192 boolean_t 1193 ddt_addref(spa_t *spa, const blkptr_t *bp) 1194 { 1195 ddt_t *ddt; 1196 ddt_entry_t *dde; 1197 boolean_t result; 1198 1199 spa_config_enter(spa, SCL_ZIO, FTAG, RW_READER); 1200 ddt = ddt_select(spa, bp); 1201 ddt_enter(ddt); 1202 1203 dde = ddt_lookup(ddt, bp, B_TRUE); 1204 ASSERT(dde != NULL); 1205 1206 if (dde->dde_type < DDT_TYPES) { 1207 ddt_phys_t *ddp; 1208 1209 ASSERT3S(dde->dde_class, <, DDT_CLASSES); 1210 1211 ddp = &dde->dde_phys[BP_GET_NDVAS(bp)]; 1212 if (ddp->ddp_refcnt == 0) { 1213 /* This should never happen? */ 1214 ddt_phys_fill(ddp, bp); 1215 } 1216 ddt_phys_addref(ddp); 1217 result = B_TRUE; 1218 } else { 1219 /* 1220 * At the time of implementating this if the block has the 1221 * DEDUP flag set it must exist in the DEDUP table, but 1222 * there are many advocates that want ability to remove 1223 * entries from DDT with refcnt=1. If this will happen, 1224 * we may have a block with the DEDUP set, but which doesn't 1225 * have a corresponding entry in the DDT. Be ready. 1226 */ 1227 ASSERT3S(dde->dde_class, ==, DDT_CLASSES); 1228 ddt_remove(ddt, dde); 1229 result = B_FALSE; 1230 } 1231 1232 ddt_exit(ddt); 1233 spa_config_exit(spa, SCL_ZIO, FTAG); 1234 1235 return (result); 1236 } 1237 1238 ZFS_MODULE_PARAM(zfs_dedup, zfs_dedup_, prefetch, INT, ZMOD_RW, 1239 "Enable prefetching dedup-ed blks"); 1240