1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #include <sys/zfs_context.h> 28 #include <sys/spa.h> 29 #include <sys/spa_impl.h> 30 #include <sys/zio.h> 31 #include <sys/ddt.h> 32 #include <sys/zap.h> 33 #include <sys/dmu_tx.h> 34 #include <sys/arc.h> 35 #include <sys/dsl_pool.h> 36 #include <sys/zio_checksum.h> 37 #include <sys/zio_compress.h> 38 39 static const ddt_ops_t *ddt_ops[DDT_TYPES] = { 40 &ddt_zap_ops, 41 }; 42 43 static const char *ddt_class_name[DDT_CLASSES] = { 44 "ditto", 45 "duplicate", 46 "unique", 47 }; 48 49 static void 50 ddt_object_create(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 51 dmu_tx_t *tx) 52 { 53 spa_t *spa = ddt->ddt_spa; 54 objset_t *os = ddt->ddt_os; 55 uint64_t *objectp = &ddt->ddt_object[type][class]; 56 boolean_t prehash = zio_checksum_table[ddt->ddt_checksum].ci_dedup; 57 char name[DDT_NAMELEN]; 58 59 ddt_object_name(ddt, type, class, name); 60 61 ASSERT(*objectp == 0); 62 VERIFY(ddt_ops[type]->ddt_op_create(os, objectp, tx, prehash) == 0); 63 ASSERT(*objectp != 0); 64 65 VERIFY(zap_add(os, DMU_POOL_DIRECTORY_OBJECT, name, 66 sizeof (uint64_t), 1, objectp, tx) == 0); 67 68 VERIFY(zap_add(os, spa->spa_ddt_stat_object, name, 69 sizeof (uint64_t), sizeof (ddt_histogram_t) / sizeof (uint64_t), 70 &ddt->ddt_histogram[type][class], tx) == 0); 71 } 72 73 static void 74 ddt_object_destroy(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 75 dmu_tx_t *tx) 76 { 77 spa_t *spa = ddt->ddt_spa; 78 objset_t *os = ddt->ddt_os; 79 uint64_t *objectp = &ddt->ddt_object[type][class]; 80 char name[DDT_NAMELEN]; 81 82 ddt_object_name(ddt, type, class, name); 83 84 ASSERT(*objectp != 0); 85 ASSERT(ddt_object_count(ddt, type, class) == 0); 86 ASSERT(ddt_histogram_empty(&ddt->ddt_histogram[type][class])); 87 VERIFY(zap_remove(os, DMU_POOL_DIRECTORY_OBJECT, name, tx) == 0); 88 VERIFY(zap_remove(os, spa->spa_ddt_stat_object, name, tx) == 0); 89 VERIFY(ddt_ops[type]->ddt_op_destroy(os, *objectp, tx) == 0); 90 91 *objectp = 0; 92 } 93 94 static int 95 ddt_object_load(ddt_t *ddt, enum ddt_type type, enum ddt_class class) 96 { 97 char name[DDT_NAMELEN]; 98 int error; 99 100 ddt_object_name(ddt, type, class, name); 101 102 error = zap_lookup(ddt->ddt_os, DMU_POOL_DIRECTORY_OBJECT, name, 103 sizeof (uint64_t), 1, &ddt->ddt_object[type][class]); 104 105 if (error) 106 return (error); 107 108 error = zap_lookup(ddt->ddt_os, ddt->ddt_spa->spa_ddt_stat_object, name, 109 sizeof (uint64_t), sizeof (ddt_histogram_t) / sizeof (uint64_t), 110 &ddt->ddt_histogram[type][class]); 111 112 ASSERT(error == 0); 113 return (error); 114 } 115 116 static void 117 ddt_object_sync(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 118 dmu_tx_t *tx) 119 { 120 char name[DDT_NAMELEN]; 121 122 ddt_object_name(ddt, type, class, name); 123 124 VERIFY(zap_update(ddt->ddt_os, ddt->ddt_spa->spa_ddt_stat_object, name, 125 sizeof (uint64_t), sizeof (ddt_histogram_t) / sizeof (uint64_t), 126 &ddt->ddt_histogram[type][class], tx) == 0); 127 } 128 129 static int 130 ddt_object_lookup(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 131 ddt_entry_t *dde) 132 { 133 if (!ddt_object_exists(ddt, type, class)) 134 return (ENOENT); 135 136 return (ddt_ops[type]->ddt_op_lookup(ddt->ddt_os, 137 ddt->ddt_object[type][class], dde)); 138 } 139 140 static int 141 ddt_object_update(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 142 ddt_entry_t *dde, dmu_tx_t *tx) 143 { 144 ASSERT(ddt_object_exists(ddt, type, class)); 145 146 return (ddt_ops[type]->ddt_op_update(ddt->ddt_os, 147 ddt->ddt_object[type][class], dde, tx)); 148 } 149 150 static int 151 ddt_object_remove(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 152 ddt_entry_t *dde, dmu_tx_t *tx) 153 { 154 ASSERT(ddt_object_exists(ddt, type, class)); 155 156 return (ddt_ops[type]->ddt_op_remove(ddt->ddt_os, 157 ddt->ddt_object[type][class], dde, tx)); 158 } 159 160 int 161 ddt_object_walk(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 162 uint64_t *walk, ddt_entry_t *dde) 163 { 164 ASSERT(ddt_object_exists(ddt, type, class)); 165 166 return (ddt_ops[type]->ddt_op_walk(ddt->ddt_os, 167 ddt->ddt_object[type][class], dde, walk)); 168 } 169 170 uint64_t 171 ddt_object_count(ddt_t *ddt, enum ddt_type type, enum ddt_class class) 172 { 173 ASSERT(ddt_object_exists(ddt, type, class)); 174 175 return (ddt_ops[type]->ddt_op_count(ddt->ddt_os, 176 ddt->ddt_object[type][class])); 177 } 178 179 int 180 ddt_object_info(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 181 dmu_object_info_t *doi) 182 { 183 if (!ddt_object_exists(ddt, type, class)) 184 return (ENOENT); 185 186 return (dmu_object_info(ddt->ddt_os, ddt->ddt_object[type][class], 187 doi)); 188 } 189 190 boolean_t 191 ddt_object_exists(ddt_t *ddt, enum ddt_type type, enum ddt_class class) 192 { 193 return (!!ddt->ddt_object[type][class]); 194 } 195 196 void 197 ddt_object_name(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 198 char *name) 199 { 200 (void) sprintf(name, DMU_POOL_DDT, 201 zio_checksum_table[ddt->ddt_checksum].ci_name, 202 ddt_ops[type]->ddt_op_name, ddt_class_name[class]); 203 } 204 205 void 206 ddt_bp_fill(const ddt_phys_t *ddp, blkptr_t *bp, uint64_t txg) 207 { 208 ASSERT(txg != 0); 209 210 for (int d = 0; d < SPA_DVAS_PER_BP; d++) 211 bp->blk_dva[d] = ddp->ddp_dva[d]; 212 BP_SET_BIRTH(bp, txg, ddp->ddp_phys_birth); 213 } 214 215 void 216 ddt_bp_create(enum zio_checksum checksum, 217 const ddt_key_t *ddk, const ddt_phys_t *ddp, blkptr_t *bp) 218 { 219 BP_ZERO(bp); 220 221 if (ddp != NULL) 222 ddt_bp_fill(ddp, bp, ddp->ddp_phys_birth); 223 224 bp->blk_cksum = ddk->ddk_cksum; 225 226 BP_SET_LSIZE(bp, DDK_GET_LSIZE(ddk)); 227 BP_SET_PSIZE(bp, DDK_GET_PSIZE(ddk)); 228 BP_SET_COMPRESS(bp, DDK_GET_COMPRESS(ddk)); 229 BP_SET_CHECKSUM(bp, checksum); 230 BP_SET_TYPE(bp, DMU_OT_NONE); 231 BP_SET_LEVEL(bp, 0); 232 BP_SET_DEDUP(bp, 0); 233 BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER); 234 } 235 236 void 237 ddt_key_fill(ddt_key_t *ddk, const blkptr_t *bp) 238 { 239 ddk->ddk_cksum = bp->blk_cksum; 240 ddk->ddk_prop = 0; 241 242 DDK_SET_LSIZE(ddk, BP_GET_LSIZE(bp)); 243 DDK_SET_PSIZE(ddk, BP_GET_PSIZE(bp)); 244 DDK_SET_COMPRESS(ddk, BP_GET_COMPRESS(bp)); 245 } 246 247 void 248 ddt_phys_fill(ddt_phys_t *ddp, const blkptr_t *bp) 249 { 250 ASSERT(ddp->ddp_phys_birth == 0); 251 252 for (int d = 0; d < SPA_DVAS_PER_BP; d++) 253 ddp->ddp_dva[d] = bp->blk_dva[d]; 254 ddp->ddp_phys_birth = BP_PHYSICAL_BIRTH(bp); 255 } 256 257 void 258 ddt_phys_clear(ddt_phys_t *ddp) 259 { 260 bzero(ddp, sizeof (*ddp)); 261 } 262 263 void 264 ddt_phys_addref(ddt_phys_t *ddp) 265 { 266 ddp->ddp_refcnt++; 267 } 268 269 void 270 ddt_phys_decref(ddt_phys_t *ddp) 271 { 272 ASSERT((int64_t)ddp->ddp_refcnt > 0); 273 ddp->ddp_refcnt--; 274 } 275 276 void 277 ddt_phys_free(ddt_t *ddt, ddt_key_t *ddk, ddt_phys_t *ddp, uint64_t txg) 278 { 279 blkptr_t blk; 280 281 ddt_bp_create(ddt->ddt_checksum, ddk, ddp, &blk); 282 ddt_phys_clear(ddp); 283 zio_free(ddt->ddt_spa, txg, &blk); 284 } 285 286 ddt_phys_t * 287 ddt_phys_select(const ddt_entry_t *dde, const blkptr_t *bp) 288 { 289 ddt_phys_t *ddp = (ddt_phys_t *)dde->dde_phys; 290 291 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { 292 if (DVA_EQUAL(BP_IDENTITY(bp), &ddp->ddp_dva[0]) && 293 BP_PHYSICAL_BIRTH(bp) == ddp->ddp_phys_birth) 294 return (ddp); 295 } 296 return (NULL); 297 } 298 299 uint64_t 300 ddt_phys_total_refcnt(const ddt_entry_t *dde) 301 { 302 uint64_t refcnt = 0; 303 304 for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) 305 refcnt += dde->dde_phys[p].ddp_refcnt; 306 307 return (refcnt); 308 } 309 310 static void 311 ddt_stat_generate(ddt_t *ddt, ddt_entry_t *dde, ddt_stat_t *dds) 312 { 313 spa_t *spa = ddt->ddt_spa; 314 ddt_phys_t *ddp = dde->dde_phys; 315 ddt_key_t *ddk = &dde->dde_key; 316 uint64_t lsize = DDK_GET_LSIZE(ddk); 317 uint64_t psize = DDK_GET_PSIZE(ddk); 318 319 bzero(dds, sizeof (*dds)); 320 321 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { 322 uint64_t dsize = 0; 323 uint64_t refcnt = ddp->ddp_refcnt; 324 325 if (ddp->ddp_phys_birth == 0) 326 continue; 327 328 for (int d = 0; d < SPA_DVAS_PER_BP; d++) 329 dsize += dva_get_dsize_sync(spa, &ddp->ddp_dva[d]); 330 331 dds->dds_blocks += 1; 332 dds->dds_lsize += lsize; 333 dds->dds_psize += psize; 334 dds->dds_dsize += dsize; 335 336 dds->dds_ref_blocks += refcnt; 337 dds->dds_ref_lsize += lsize * refcnt; 338 dds->dds_ref_psize += psize * refcnt; 339 dds->dds_ref_dsize += dsize * refcnt; 340 } 341 } 342 343 void 344 ddt_stat_add(ddt_stat_t *dst, const ddt_stat_t *src, uint64_t neg) 345 { 346 const uint64_t *s = (const uint64_t *)src; 347 uint64_t *d = (uint64_t *)dst; 348 uint64_t *d_end = (uint64_t *)(dst + 1); 349 350 ASSERT(neg == 0 || neg == -1ULL); /* add or subtract */ 351 352 while (d < d_end) 353 *d++ += (*s++ ^ neg) - neg; 354 } 355 356 static void 357 ddt_stat_update(ddt_t *ddt, ddt_entry_t *dde, uint64_t neg) 358 { 359 ddt_stat_t dds; 360 ddt_histogram_t *ddh; 361 int bucket; 362 363 ddt_stat_generate(ddt, dde, &dds); 364 365 bucket = highbit(dds.dds_ref_blocks) - 1; 366 ASSERT(bucket >= 0); 367 368 ddh = &ddt->ddt_histogram[dde->dde_type][dde->dde_class]; 369 370 ddt_stat_add(&ddh->ddh_stat[bucket], &dds, neg); 371 } 372 373 void 374 ddt_histogram_add(ddt_histogram_t *dst, const ddt_histogram_t *src) 375 { 376 for (int h = 0; h < 64; h++) 377 ddt_stat_add(&dst->ddh_stat[h], &src->ddh_stat[h], 0); 378 } 379 380 void 381 ddt_histogram_stat(ddt_stat_t *dds, const ddt_histogram_t *ddh) 382 { 383 bzero(dds, sizeof (*dds)); 384 385 for (int h = 0; h < 64; h++) 386 ddt_stat_add(dds, &ddh->ddh_stat[h], 0); 387 } 388 389 boolean_t 390 ddt_histogram_empty(const ddt_histogram_t *ddh) 391 { 392 const uint64_t *s = (const uint64_t *)ddh; 393 const uint64_t *s_end = (const uint64_t *)(ddh + 1); 394 395 while (s < s_end) 396 if (*s++ != 0) 397 return (B_FALSE); 398 399 return (B_TRUE); 400 } 401 402 void 403 ddt_get_dedup_object_stats(spa_t *spa, ddt_object_t *ddo) 404 { 405 dmu_object_info_t doi; 406 uint64_t count; 407 int error; 408 409 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { 410 ddt_t *ddt = spa->spa_ddt[c]; 411 for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 412 for (enum ddt_class class = 0; class < DDT_CLASSES; 413 class++) { 414 error = ddt_object_info(ddt, type, class, &doi); 415 if (error == ENOENT) 416 continue; 417 ASSERT3U(error, ==, 0); 418 419 count = ddt_object_count(ddt, type, class); 420 ddo->ddo_count += count; 421 ddo->ddo_dspace += 422 (doi.doi_physical_blocks_512 << 9) / count; 423 ddo->ddo_mspace += doi.doi_fill_count * 424 doi.doi_data_block_size / count; 425 } 426 } 427 } 428 } 429 430 void 431 ddt_get_dedup_histogram(spa_t *spa, ddt_histogram_t *ddh) 432 { 433 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { 434 ddt_t *ddt = spa->spa_ddt[c]; 435 for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 436 for (enum ddt_class class = 0; class < DDT_CLASSES; 437 class++) { 438 ddt_histogram_add(ddh, 439 &ddt->ddt_histogram[type][class]); 440 } 441 } 442 } 443 } 444 445 void 446 ddt_get_dedup_stats(spa_t *spa, ddt_stat_t *dds_total) 447 { 448 ddt_histogram_t *ddh_total; 449 450 ddh_total = kmem_zalloc(sizeof (ddt_histogram_t), KM_SLEEP); 451 ddt_get_dedup_histogram(spa, ddh_total); 452 ddt_histogram_stat(dds_total, ddh_total); 453 kmem_free(ddh_total, sizeof (ddt_histogram_t)); 454 } 455 456 uint64_t 457 ddt_get_dedup_dspace(spa_t *spa) 458 { 459 ddt_stat_t dds_total = { 0 }; 460 461 ddt_get_dedup_stats(spa, &dds_total); 462 return (dds_total.dds_ref_dsize - dds_total.dds_dsize); 463 } 464 465 uint64_t 466 ddt_get_pool_dedup_ratio(spa_t *spa) 467 { 468 ddt_stat_t dds_total = { 0 }; 469 470 ddt_get_dedup_stats(spa, &dds_total); 471 if (dds_total.dds_dsize == 0) 472 return (100); 473 474 return (dds_total.dds_ref_dsize * 100 / dds_total.dds_dsize); 475 } 476 477 int 478 ddt_ditto_copies_needed(ddt_t *ddt, ddt_entry_t *dde, ddt_phys_t *ddp_willref) 479 { 480 spa_t *spa = ddt->ddt_spa; 481 uint64_t total_refcnt = 0; 482 uint64_t ditto = spa->spa_dedup_ditto; 483 int total_copies = 0; 484 int desired_copies = 0; 485 486 for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) { 487 ddt_phys_t *ddp = &dde->dde_phys[p]; 488 zio_t *zio = dde->dde_lead_zio[p]; 489 uint64_t refcnt = ddp->ddp_refcnt; /* committed refs */ 490 if (zio != NULL) 491 refcnt += zio->io_parent_count; /* pending refs */ 492 if (ddp == ddp_willref) 493 refcnt++; /* caller's ref */ 494 if (refcnt != 0) { 495 total_refcnt += refcnt; 496 total_copies += p; 497 } 498 } 499 500 if (ditto == 0 || ditto > UINT32_MAX) 501 ditto = UINT32_MAX; 502 503 if (total_refcnt >= 1) 504 desired_copies++; 505 if (total_refcnt >= ditto) 506 desired_copies++; 507 if (total_refcnt >= ditto * ditto) 508 desired_copies++; 509 510 return (MAX(desired_copies, total_copies) - total_copies); 511 } 512 513 int 514 ddt_ditto_copies_present(ddt_entry_t *dde) 515 { 516 ddt_phys_t *ddp = &dde->dde_phys[DDT_PHYS_DITTO]; 517 dva_t *dva = ddp->ddp_dva; 518 int copies = 0 - DVA_GET_GANG(dva); 519 520 for (int d = 0; d < SPA_DVAS_PER_BP; d++, dva++) 521 if (DVA_IS_VALID(dva)) 522 copies++; 523 524 ASSERT(copies >= 0 && copies < SPA_DVAS_PER_BP); 525 526 return (copies); 527 } 528 529 size_t 530 ddt_compress(void *src, uchar_t *dst, size_t s_len, size_t d_len) 531 { 532 uchar_t *version = dst++; 533 int cpfunc = ZIO_COMPRESS_ZLE; 534 zio_compress_info_t *ci = &zio_compress_table[cpfunc]; 535 size_t c_len; 536 537 ASSERT(d_len >= s_len + 1); /* no compression plus version byte */ 538 539 c_len = ci->ci_compress(src, dst, s_len, d_len - 1, ci->ci_level); 540 541 if (c_len == s_len) { 542 cpfunc = ZIO_COMPRESS_OFF; 543 bcopy(src, dst, s_len); 544 } 545 546 *version = (ZFS_HOST_BYTEORDER & DDT_COMPRESS_BYTEORDER_MASK) | cpfunc; 547 548 return (c_len + 1); 549 } 550 551 void 552 ddt_decompress(uchar_t *src, void *dst, size_t s_len, size_t d_len) 553 { 554 uchar_t version = *src++; 555 int cpfunc = version & DDT_COMPRESS_FUNCTION_MASK; 556 zio_compress_info_t *ci = &zio_compress_table[cpfunc]; 557 558 if (ci->ci_decompress != NULL) 559 (void) ci->ci_decompress(src, dst, s_len, d_len, ci->ci_level); 560 else 561 bcopy(src, dst, d_len); 562 563 if ((version ^ ZFS_HOST_BYTEORDER) & DDT_COMPRESS_BYTEORDER_MASK) 564 byteswap_uint64_array(dst, d_len); 565 } 566 567 ddt_t * 568 ddt_select_by_checksum(spa_t *spa, enum zio_checksum c) 569 { 570 return (spa->spa_ddt[c]); 571 } 572 573 ddt_t * 574 ddt_select(spa_t *spa, const blkptr_t *bp) 575 { 576 return (spa->spa_ddt[BP_GET_CHECKSUM(bp)]); 577 } 578 579 void 580 ddt_enter(ddt_t *ddt) 581 { 582 mutex_enter(&ddt->ddt_lock); 583 } 584 585 void 586 ddt_exit(ddt_t *ddt) 587 { 588 mutex_exit(&ddt->ddt_lock); 589 } 590 591 static ddt_entry_t * 592 ddt_alloc(const ddt_key_t *ddk) 593 { 594 ddt_entry_t *dde; 595 596 dde = kmem_zalloc(sizeof (ddt_entry_t), KM_SLEEP); 597 cv_init(&dde->dde_cv, NULL, CV_DEFAULT, NULL); 598 599 dde->dde_key = *ddk; 600 601 return (dde); 602 } 603 604 static void 605 ddt_free(ddt_entry_t *dde) 606 { 607 ASSERT(!dde->dde_loading); 608 609 for (int p = 0; p < DDT_PHYS_TYPES; p++) 610 ASSERT(dde->dde_lead_zio[p] == NULL); 611 612 if (dde->dde_repair_data != NULL) 613 zio_buf_free(dde->dde_repair_data, 614 DDK_GET_PSIZE(&dde->dde_key)); 615 616 cv_destroy(&dde->dde_cv); 617 kmem_free(dde, sizeof (*dde)); 618 } 619 620 void 621 ddt_remove(ddt_t *ddt, ddt_entry_t *dde) 622 { 623 ASSERT(MUTEX_HELD(&ddt->ddt_lock)); 624 625 avl_remove(&ddt->ddt_tree, dde); 626 ddt_free(dde); 627 } 628 629 ddt_entry_t * 630 ddt_lookup(ddt_t *ddt, const blkptr_t *bp, boolean_t add) 631 { 632 ddt_entry_t *dde, dde_search; 633 enum ddt_type type; 634 enum ddt_class class; 635 avl_index_t where; 636 int error; 637 638 ASSERT(MUTEX_HELD(&ddt->ddt_lock)); 639 640 ddt_key_fill(&dde_search.dde_key, bp); 641 642 dde = avl_find(&ddt->ddt_tree, &dde_search, &where); 643 if (dde == NULL) { 644 if (!add) 645 return (NULL); 646 dde = ddt_alloc(&dde_search.dde_key); 647 avl_insert(&ddt->ddt_tree, dde, where); 648 } 649 650 while (dde->dde_loading) 651 cv_wait(&dde->dde_cv, &ddt->ddt_lock); 652 653 if (dde->dde_loaded) 654 return (dde); 655 656 dde->dde_loading = B_TRUE; 657 658 ddt_exit(ddt); 659 660 error = ENOENT; 661 662 for (type = 0; type < DDT_TYPES; type++) { 663 for (class = 0; class < DDT_CLASSES; class++) { 664 error = ddt_object_lookup(ddt, type, class, dde); 665 if (error != ENOENT) 666 break; 667 } 668 if (error != ENOENT) 669 break; 670 } 671 672 ASSERT(error == 0 || error == ENOENT); 673 674 ddt_enter(ddt); 675 676 ASSERT(dde->dde_loaded == B_FALSE); 677 ASSERT(dde->dde_loading == B_TRUE); 678 679 dde->dde_type = type; /* will be DDT_TYPES if no entry found */ 680 dde->dde_class = class; /* will be DDT_CLASSES if no entry found */ 681 dde->dde_loaded = B_TRUE; 682 dde->dde_loading = B_FALSE; 683 684 if (error == 0) 685 ddt_stat_update(ddt, dde, -1ULL); 686 687 cv_broadcast(&dde->dde_cv); 688 689 return (dde); 690 } 691 692 int 693 ddt_entry_compare(const void *x1, const void *x2) 694 { 695 const ddt_entry_t *dde1 = x1; 696 const ddt_entry_t *dde2 = x2; 697 const uint64_t *u1 = (const uint64_t *)&dde1->dde_key; 698 const uint64_t *u2 = (const uint64_t *)&dde2->dde_key; 699 700 for (int i = 0; i < DDT_KEY_WORDS; i++) { 701 if (u1[i] < u2[i]) 702 return (-1); 703 if (u1[i] > u2[i]) 704 return (1); 705 } 706 707 return (0); 708 } 709 710 static ddt_t * 711 ddt_table_alloc(spa_t *spa, enum zio_checksum c) 712 { 713 ddt_t *ddt; 714 715 ddt = kmem_zalloc(sizeof (*ddt), KM_SLEEP); 716 717 mutex_init(&ddt->ddt_lock, NULL, MUTEX_DEFAULT, NULL); 718 avl_create(&ddt->ddt_tree, ddt_entry_compare, 719 sizeof (ddt_entry_t), offsetof(ddt_entry_t, dde_node)); 720 avl_create(&ddt->ddt_repair_tree, ddt_entry_compare, 721 sizeof (ddt_entry_t), offsetof(ddt_entry_t, dde_node)); 722 ddt->ddt_checksum = c; 723 ddt->ddt_spa = spa; 724 ddt->ddt_os = spa->spa_meta_objset; 725 726 return (ddt); 727 } 728 729 static void 730 ddt_table_free(ddt_t *ddt) 731 { 732 ASSERT(avl_numnodes(&ddt->ddt_tree) == 0); 733 ASSERT(avl_numnodes(&ddt->ddt_repair_tree) == 0); 734 avl_destroy(&ddt->ddt_tree); 735 avl_destroy(&ddt->ddt_repair_tree); 736 mutex_destroy(&ddt->ddt_lock); 737 kmem_free(ddt, sizeof (*ddt)); 738 } 739 740 void 741 ddt_create(spa_t *spa) 742 { 743 spa->spa_dedup_checksum = ZIO_DEDUPCHECKSUM; 744 745 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) 746 spa->spa_ddt[c] = ddt_table_alloc(spa, c); 747 } 748 749 int 750 ddt_load(spa_t *spa) 751 { 752 int error; 753 754 ddt_create(spa); 755 756 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 757 DMU_POOL_DDT_STATS, sizeof (uint64_t), 1, 758 &spa->spa_ddt_stat_object); 759 760 if (error) 761 return (error == ENOENT ? 0 : error); 762 763 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { 764 for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 765 for (enum ddt_class class = 0; class < DDT_CLASSES; 766 class++) { 767 ddt_t *ddt = spa->spa_ddt[c]; 768 error = ddt_object_load(ddt, type, class); 769 if (error != 0 && error != ENOENT) 770 return (error); 771 } 772 } 773 } 774 775 return (0); 776 } 777 778 void 779 ddt_unload(spa_t *spa) 780 { 781 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { 782 if (spa->spa_ddt[c]) { 783 ddt_table_free(spa->spa_ddt[c]); 784 spa->spa_ddt[c] = NULL; 785 } 786 } 787 } 788 789 boolean_t 790 ddt_class_contains(spa_t *spa, enum ddt_class max_class, const blkptr_t *bp) 791 { 792 ddt_t *ddt; 793 ddt_entry_t dde; 794 795 if (!BP_GET_DEDUP(bp)) 796 return (B_FALSE); 797 798 if (max_class == DDT_CLASS_UNIQUE) 799 return (B_TRUE); 800 801 ddt = spa->spa_ddt[BP_GET_CHECKSUM(bp)]; 802 803 ddt_key_fill(&dde.dde_key, bp); 804 805 for (enum ddt_type type = 0; type < DDT_TYPES; type++) 806 for (enum ddt_class class = 0; class <= max_class; class++) 807 if (ddt_object_lookup(ddt, type, class, &dde) == 0) 808 return (B_TRUE); 809 810 return (B_FALSE); 811 } 812 813 ddt_entry_t * 814 ddt_repair_start(ddt_t *ddt, const blkptr_t *bp) 815 { 816 ddt_key_t ddk; 817 ddt_entry_t *dde; 818 819 ddt_key_fill(&ddk, bp); 820 821 dde = ddt_alloc(&ddk); 822 823 for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 824 for (enum ddt_class class = 0; class < DDT_CLASSES; class++) { 825 /* 826 * We can only do repair if there are multiple copies 827 * of the block. For anything in the UNIQUE class, 828 * there's definitely only one copy, so don't even try. 829 */ 830 if (class != DDT_CLASS_UNIQUE && 831 ddt_object_lookup(ddt, type, class, dde) == 0) 832 return (dde); 833 } 834 } 835 836 bzero(dde->dde_phys, sizeof (dde->dde_phys)); 837 838 return (dde); 839 } 840 841 void 842 ddt_repair_done(ddt_t *ddt, ddt_entry_t *dde) 843 { 844 avl_index_t where; 845 846 ddt_enter(ddt); 847 848 if (dde->dde_repair_data != NULL && spa_writeable(ddt->ddt_spa) && 849 avl_find(&ddt->ddt_repair_tree, dde, &where) == NULL) 850 avl_insert(&ddt->ddt_repair_tree, dde, where); 851 else 852 ddt_free(dde); 853 854 ddt_exit(ddt); 855 } 856 857 static void 858 ddt_repair_entry_done(zio_t *zio) 859 { 860 ddt_entry_t *rdde = zio->io_private; 861 862 ddt_free(rdde); 863 } 864 865 static void 866 ddt_repair_entry(ddt_t *ddt, ddt_entry_t *dde, ddt_entry_t *rdde, zio_t *rio) 867 { 868 ddt_phys_t *ddp = dde->dde_phys; 869 ddt_phys_t *rddp = rdde->dde_phys; 870 ddt_key_t *ddk = &dde->dde_key; 871 ddt_key_t *rddk = &rdde->dde_key; 872 zio_t *zio; 873 blkptr_t blk; 874 875 zio = zio_null(rio, rio->io_spa, NULL, 876 ddt_repair_entry_done, rdde, rio->io_flags); 877 878 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++, rddp++) { 879 if (ddp->ddp_phys_birth == 0 || 880 ddp->ddp_phys_birth != rddp->ddp_phys_birth || 881 bcmp(ddp->ddp_dva, rddp->ddp_dva, sizeof (ddp->ddp_dva))) 882 continue; 883 ddt_bp_create(ddt->ddt_checksum, ddk, ddp, &blk); 884 zio_nowait(zio_rewrite(zio, zio->io_spa, 0, &blk, 885 rdde->dde_repair_data, DDK_GET_PSIZE(rddk), NULL, NULL, 886 ZIO_PRIORITY_SYNC_WRITE, ZIO_DDT_CHILD_FLAGS(zio), NULL)); 887 } 888 889 zio_nowait(zio); 890 } 891 892 static void 893 ddt_repair_table(ddt_t *ddt, zio_t *rio) 894 { 895 spa_t *spa = ddt->ddt_spa; 896 ddt_entry_t *dde, *rdde_next, *rdde; 897 avl_tree_t *t = &ddt->ddt_repair_tree; 898 blkptr_t blk; 899 900 if (spa_sync_pass(spa) > 1) 901 return; 902 903 ddt_enter(ddt); 904 for (rdde = avl_first(t); rdde != NULL; rdde = rdde_next) { 905 rdde_next = AVL_NEXT(t, rdde); 906 avl_remove(&ddt->ddt_repair_tree, rdde); 907 ddt_exit(ddt); 908 ddt_bp_create(ddt->ddt_checksum, &rdde->dde_key, NULL, &blk); 909 dde = ddt_repair_start(ddt, &blk); 910 ddt_repair_entry(ddt, dde, rdde, rio); 911 ddt_repair_done(ddt, dde); 912 ddt_enter(ddt); 913 } 914 ddt_exit(ddt); 915 } 916 917 static void 918 ddt_sync_entry(ddt_t *ddt, ddt_entry_t *dde, dmu_tx_t *tx, uint64_t txg) 919 { 920 dsl_pool_t *dp = ddt->ddt_spa->spa_dsl_pool; 921 ddt_phys_t *ddp = dde->dde_phys; 922 ddt_key_t *ddk = &dde->dde_key; 923 enum ddt_type otype = dde->dde_type; 924 enum ddt_type ntype = DDT_TYPE_CURRENT; 925 enum ddt_class oclass = dde->dde_class; 926 enum ddt_class nclass; 927 uint64_t total_refcnt = 0; 928 929 ASSERT(dde->dde_loaded); 930 ASSERT(!dde->dde_loading); 931 932 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { 933 ASSERT(dde->dde_lead_zio[p] == NULL); 934 ASSERT((int64_t)ddp->ddp_refcnt >= 0); 935 if (ddp->ddp_phys_birth == 0) { 936 ASSERT(ddp->ddp_refcnt == 0); 937 continue; 938 } 939 if (p == DDT_PHYS_DITTO) { 940 if (ddt_ditto_copies_needed(ddt, dde, NULL) == 0) 941 ddt_phys_free(ddt, ddk, ddp, txg); 942 continue; 943 } 944 if (ddp->ddp_refcnt == 0) 945 ddt_phys_free(ddt, ddk, ddp, txg); 946 total_refcnt += ddp->ddp_refcnt; 947 } 948 949 if (dde->dde_phys[DDT_PHYS_DITTO].ddp_phys_birth != 0) 950 nclass = DDT_CLASS_DITTO; 951 else if (total_refcnt > 1) 952 nclass = DDT_CLASS_DUPLICATE; 953 else 954 nclass = DDT_CLASS_UNIQUE; 955 956 if (otype != DDT_TYPES && 957 (otype != ntype || oclass != nclass || total_refcnt == 0)) { 958 VERIFY(ddt_object_remove(ddt, otype, oclass, dde, tx) == 0); 959 ASSERT(ddt_object_lookup(ddt, otype, oclass, dde) == ENOENT); 960 } 961 962 if (total_refcnt != 0) { 963 dde->dde_type = ntype; 964 dde->dde_class = nclass; 965 ddt_stat_update(ddt, dde, 0); 966 if (!ddt_object_exists(ddt, ntype, nclass)) 967 ddt_object_create(ddt, ntype, nclass, tx); 968 VERIFY(ddt_object_update(ddt, ntype, nclass, dde, tx) == 0); 969 970 if (dp->dp_scrub_func != SCRUB_FUNC_NONE && 971 oclass > nclass && 972 nclass <= dp->dp_scrub_ddt_class_max) 973 dsl_pool_scrub_ddt_entry(dp, ddt->ddt_checksum, dde); 974 } 975 } 976 977 static void 978 ddt_sync_table(ddt_t *ddt, dmu_tx_t *tx, uint64_t txg) 979 { 980 spa_t *spa = ddt->ddt_spa; 981 ddt_entry_t *dde; 982 void *cookie = NULL; 983 984 if (avl_numnodes(&ddt->ddt_tree) == 0) 985 return; 986 987 ASSERT(spa_sync_pass(spa) == 1); 988 ASSERT(spa->spa_uberblock.ub_version >= SPA_VERSION_DEDUP); 989 990 if (spa->spa_ddt_stat_object == 0) { 991 spa->spa_ddt_stat_object = zap_create(ddt->ddt_os, 992 DMU_OT_DDT_STATS, DMU_OT_NONE, 0, tx); 993 VERIFY(zap_add(ddt->ddt_os, DMU_POOL_DIRECTORY_OBJECT, 994 DMU_POOL_DDT_STATS, sizeof (uint64_t), 1, 995 &spa->spa_ddt_stat_object, tx) == 0); 996 } 997 998 while ((dde = avl_destroy_nodes(&ddt->ddt_tree, &cookie)) != NULL) { 999 ddt_sync_entry(ddt, dde, tx, txg); 1000 ddt_free(dde); 1001 } 1002 1003 for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 1004 for (enum ddt_class class = 0; class < DDT_CLASSES; class++) { 1005 if (!ddt_object_exists(ddt, type, class)) 1006 continue; 1007 ddt_object_sync(ddt, type, class, tx); 1008 if (ddt_object_count(ddt, type, class) == 0) 1009 ddt_object_destroy(ddt, type, class, tx); 1010 } 1011 } 1012 } 1013 1014 void 1015 ddt_sync(spa_t *spa, uint64_t txg) 1016 { 1017 dmu_tx_t *tx; 1018 zio_t *rio = zio_root(spa, NULL, NULL, 1019 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE); 1020 1021 ASSERT(spa_syncing_txg(spa) == txg); 1022 1023 tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg); 1024 1025 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { 1026 ddt_t *ddt = spa->spa_ddt[c]; 1027 if (ddt == NULL) 1028 continue; 1029 ddt_sync_table(ddt, tx, txg); 1030 ddt_repair_table(ddt, rio); 1031 } 1032 1033 (void) zio_wait(rio); 1034 1035 dmu_tx_commit(tx); 1036 } 1037 1038 int 1039 ddt_walk(spa_t *spa, ddt_bookmark_t *ddb, ddt_entry_t *dde) 1040 { 1041 do { 1042 do { 1043 do { 1044 ddt_t *ddt = spa->spa_ddt[ddb->ddb_checksum]; 1045 int error = ENOENT; 1046 if (ddt_object_exists(ddt, ddb->ddb_type, 1047 ddb->ddb_class)) { 1048 error = ddt_object_walk(ddt, 1049 ddb->ddb_type, ddb->ddb_class, 1050 &ddb->ddb_cursor, dde); 1051 } 1052 if (error == 0) 1053 return (0); 1054 if (error != ENOENT) 1055 return (error); 1056 ddb->ddb_cursor = 0; 1057 } while (++ddb->ddb_checksum < ZIO_CHECKSUM_FUNCTIONS); 1058 ddb->ddb_checksum = 0; 1059 } while (++ddb->ddb_type < DDT_TYPES); 1060 ddb->ddb_type = 0; 1061 } while (++ddb->ddb_class < DDT_CLASSES); 1062 1063 return (ENOENT); 1064 } 1065