1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. 24 */ 25 26 #include <sys/zfs_context.h> 27 #include <sys/spa.h> 28 #include <sys/spa_impl.h> 29 #include <sys/zio.h> 30 #include <sys/ddt.h> 31 #include <sys/zap.h> 32 #include <sys/dmu_tx.h> 33 #include <sys/arc.h> 34 #include <sys/dsl_pool.h> 35 #include <sys/zio_checksum.h> 36 #include <sys/zio_compress.h> 37 #include <sys/dsl_scan.h> 38 39 static const ddt_ops_t *ddt_ops[DDT_TYPES] = { 40 &ddt_zap_ops, 41 }; 42 43 static const char *ddt_class_name[DDT_CLASSES] = { 44 "ditto", 45 "duplicate", 46 "unique", 47 }; 48 49 static void 50 ddt_object_create(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 51 dmu_tx_t *tx) 52 { 53 spa_t *spa = ddt->ddt_spa; 54 objset_t *os = ddt->ddt_os; 55 uint64_t *objectp = &ddt->ddt_object[type][class]; 56 boolean_t prehash = zio_checksum_table[ddt->ddt_checksum].ci_dedup; 57 char name[DDT_NAMELEN]; 58 59 ddt_object_name(ddt, type, class, name); 60 61 ASSERT(*objectp == 0); 62 VERIFY(ddt_ops[type]->ddt_op_create(os, objectp, tx, prehash) == 0); 63 ASSERT(*objectp != 0); 64 65 VERIFY(zap_add(os, DMU_POOL_DIRECTORY_OBJECT, name, 66 sizeof (uint64_t), 1, objectp, tx) == 0); 67 68 VERIFY(zap_add(os, spa->spa_ddt_stat_object, name, 69 sizeof (uint64_t), sizeof (ddt_histogram_t) / sizeof (uint64_t), 70 &ddt->ddt_histogram[type][class], tx) == 0); 71 } 72 73 static void 74 ddt_object_destroy(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 75 dmu_tx_t *tx) 76 { 77 spa_t *spa = ddt->ddt_spa; 78 objset_t *os = ddt->ddt_os; 79 uint64_t *objectp = &ddt->ddt_object[type][class]; 80 char name[DDT_NAMELEN]; 81 82 ddt_object_name(ddt, type, class, name); 83 84 ASSERT(*objectp != 0); 85 ASSERT(ddt_object_count(ddt, type, class) == 0); 86 ASSERT(ddt_histogram_empty(&ddt->ddt_histogram[type][class])); 87 VERIFY(zap_remove(os, DMU_POOL_DIRECTORY_OBJECT, name, tx) == 0); 88 VERIFY(zap_remove(os, spa->spa_ddt_stat_object, name, tx) == 0); 89 VERIFY(ddt_ops[type]->ddt_op_destroy(os, *objectp, tx) == 0); 90 bzero(&ddt->ddt_object_stats[type][class], sizeof (ddt_object_t)); 91 92 *objectp = 0; 93 } 94 95 static int 96 ddt_object_load(ddt_t *ddt, enum ddt_type type, enum ddt_class class) 97 { 98 ddt_object_t *ddo = &ddt->ddt_object_stats[type][class]; 99 dmu_object_info_t doi; 100 char name[DDT_NAMELEN]; 101 int error; 102 103 ddt_object_name(ddt, type, class, name); 104 105 error = zap_lookup(ddt->ddt_os, DMU_POOL_DIRECTORY_OBJECT, name, 106 sizeof (uint64_t), 1, &ddt->ddt_object[type][class]); 107 108 if (error) 109 return (error); 110 111 error = zap_lookup(ddt->ddt_os, ddt->ddt_spa->spa_ddt_stat_object, name, 112 sizeof (uint64_t), sizeof (ddt_histogram_t) / sizeof (uint64_t), 113 &ddt->ddt_histogram[type][class]); 114 115 /* 116 * Seed the cached statistics. 117 */ 118 VERIFY(ddt_object_info(ddt, type, class, &doi) == 0); 119 120 ddo->ddo_count = ddt_object_count(ddt, type, class); 121 ddo->ddo_dspace = doi.doi_physical_blocks_512 << 9; 122 ddo->ddo_mspace = doi.doi_fill_count * doi.doi_data_block_size; 123 124 ASSERT(error == 0); 125 return (error); 126 } 127 128 static void 129 ddt_object_sync(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 130 dmu_tx_t *tx) 131 { 132 ddt_object_t *ddo = &ddt->ddt_object_stats[type][class]; 133 dmu_object_info_t doi; 134 char name[DDT_NAMELEN]; 135 136 ddt_object_name(ddt, type, class, name); 137 138 VERIFY(zap_update(ddt->ddt_os, ddt->ddt_spa->spa_ddt_stat_object, name, 139 sizeof (uint64_t), sizeof (ddt_histogram_t) / sizeof (uint64_t), 140 &ddt->ddt_histogram[type][class], tx) == 0); 141 142 /* 143 * Cache DDT statistics; this is the only time they'll change. 144 */ 145 VERIFY(ddt_object_info(ddt, type, class, &doi) == 0); 146 147 ddo->ddo_count = ddt_object_count(ddt, type, class); 148 ddo->ddo_dspace = doi.doi_physical_blocks_512 << 9; 149 ddo->ddo_mspace = doi.doi_fill_count * doi.doi_data_block_size; 150 } 151 152 static int 153 ddt_object_lookup(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 154 ddt_entry_t *dde) 155 { 156 if (!ddt_object_exists(ddt, type, class)) 157 return (ENOENT); 158 159 return (ddt_ops[type]->ddt_op_lookup(ddt->ddt_os, 160 ddt->ddt_object[type][class], dde)); 161 } 162 163 int 164 ddt_object_update(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 165 ddt_entry_t *dde, dmu_tx_t *tx) 166 { 167 ASSERT(ddt_object_exists(ddt, type, class)); 168 169 return (ddt_ops[type]->ddt_op_update(ddt->ddt_os, 170 ddt->ddt_object[type][class], dde, tx)); 171 } 172 173 static int 174 ddt_object_remove(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 175 ddt_entry_t *dde, dmu_tx_t *tx) 176 { 177 ASSERT(ddt_object_exists(ddt, type, class)); 178 179 return (ddt_ops[type]->ddt_op_remove(ddt->ddt_os, 180 ddt->ddt_object[type][class], dde, tx)); 181 } 182 183 int 184 ddt_object_walk(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 185 uint64_t *walk, ddt_entry_t *dde) 186 { 187 ASSERT(ddt_object_exists(ddt, type, class)); 188 189 return (ddt_ops[type]->ddt_op_walk(ddt->ddt_os, 190 ddt->ddt_object[type][class], dde, walk)); 191 } 192 193 uint64_t 194 ddt_object_count(ddt_t *ddt, enum ddt_type type, enum ddt_class class) 195 { 196 ASSERT(ddt_object_exists(ddt, type, class)); 197 198 return (ddt_ops[type]->ddt_op_count(ddt->ddt_os, 199 ddt->ddt_object[type][class])); 200 } 201 202 int 203 ddt_object_info(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 204 dmu_object_info_t *doi) 205 { 206 if (!ddt_object_exists(ddt, type, class)) 207 return (ENOENT); 208 209 return (dmu_object_info(ddt->ddt_os, ddt->ddt_object[type][class], 210 doi)); 211 } 212 213 boolean_t 214 ddt_object_exists(ddt_t *ddt, enum ddt_type type, enum ddt_class class) 215 { 216 return (!!ddt->ddt_object[type][class]); 217 } 218 219 void 220 ddt_object_name(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 221 char *name) 222 { 223 (void) sprintf(name, DMU_POOL_DDT, 224 zio_checksum_table[ddt->ddt_checksum].ci_name, 225 ddt_ops[type]->ddt_op_name, ddt_class_name[class]); 226 } 227 228 void 229 ddt_bp_fill(const ddt_phys_t *ddp, blkptr_t *bp, uint64_t txg) 230 { 231 ASSERT(txg != 0); 232 233 for (int d = 0; d < SPA_DVAS_PER_BP; d++) 234 bp->blk_dva[d] = ddp->ddp_dva[d]; 235 BP_SET_BIRTH(bp, txg, ddp->ddp_phys_birth); 236 } 237 238 void 239 ddt_bp_create(enum zio_checksum checksum, 240 const ddt_key_t *ddk, const ddt_phys_t *ddp, blkptr_t *bp) 241 { 242 BP_ZERO(bp); 243 244 if (ddp != NULL) 245 ddt_bp_fill(ddp, bp, ddp->ddp_phys_birth); 246 247 bp->blk_cksum = ddk->ddk_cksum; 248 bp->blk_fill = 1; 249 250 BP_SET_LSIZE(bp, DDK_GET_LSIZE(ddk)); 251 BP_SET_PSIZE(bp, DDK_GET_PSIZE(ddk)); 252 BP_SET_COMPRESS(bp, DDK_GET_COMPRESS(ddk)); 253 BP_SET_CHECKSUM(bp, checksum); 254 BP_SET_TYPE(bp, DMU_OT_DEDUP); 255 BP_SET_LEVEL(bp, 0); 256 BP_SET_DEDUP(bp, 0); 257 BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER); 258 } 259 260 void 261 ddt_key_fill(ddt_key_t *ddk, const blkptr_t *bp) 262 { 263 ddk->ddk_cksum = bp->blk_cksum; 264 ddk->ddk_prop = 0; 265 266 DDK_SET_LSIZE(ddk, BP_GET_LSIZE(bp)); 267 DDK_SET_PSIZE(ddk, BP_GET_PSIZE(bp)); 268 DDK_SET_COMPRESS(ddk, BP_GET_COMPRESS(bp)); 269 } 270 271 void 272 ddt_phys_fill(ddt_phys_t *ddp, const blkptr_t *bp) 273 { 274 ASSERT(ddp->ddp_phys_birth == 0); 275 276 for (int d = 0; d < SPA_DVAS_PER_BP; d++) 277 ddp->ddp_dva[d] = bp->blk_dva[d]; 278 ddp->ddp_phys_birth = BP_PHYSICAL_BIRTH(bp); 279 } 280 281 void 282 ddt_phys_clear(ddt_phys_t *ddp) 283 { 284 bzero(ddp, sizeof (*ddp)); 285 } 286 287 void 288 ddt_phys_addref(ddt_phys_t *ddp) 289 { 290 ddp->ddp_refcnt++; 291 } 292 293 void 294 ddt_phys_decref(ddt_phys_t *ddp) 295 { 296 ASSERT((int64_t)ddp->ddp_refcnt > 0); 297 ddp->ddp_refcnt--; 298 } 299 300 void 301 ddt_phys_free(ddt_t *ddt, ddt_key_t *ddk, ddt_phys_t *ddp, uint64_t txg) 302 { 303 blkptr_t blk; 304 305 ddt_bp_create(ddt->ddt_checksum, ddk, ddp, &blk); 306 ddt_phys_clear(ddp); 307 zio_free(ddt->ddt_spa, txg, &blk); 308 } 309 310 ddt_phys_t * 311 ddt_phys_select(const ddt_entry_t *dde, const blkptr_t *bp) 312 { 313 ddt_phys_t *ddp = (ddt_phys_t *)dde->dde_phys; 314 315 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { 316 if (DVA_EQUAL(BP_IDENTITY(bp), &ddp->ddp_dva[0]) && 317 BP_PHYSICAL_BIRTH(bp) == ddp->ddp_phys_birth) 318 return (ddp); 319 } 320 return (NULL); 321 } 322 323 uint64_t 324 ddt_phys_total_refcnt(const ddt_entry_t *dde) 325 { 326 uint64_t refcnt = 0; 327 328 for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) 329 refcnt += dde->dde_phys[p].ddp_refcnt; 330 331 return (refcnt); 332 } 333 334 static void 335 ddt_stat_generate(ddt_t *ddt, ddt_entry_t *dde, ddt_stat_t *dds) 336 { 337 spa_t *spa = ddt->ddt_spa; 338 ddt_phys_t *ddp = dde->dde_phys; 339 ddt_key_t *ddk = &dde->dde_key; 340 uint64_t lsize = DDK_GET_LSIZE(ddk); 341 uint64_t psize = DDK_GET_PSIZE(ddk); 342 343 bzero(dds, sizeof (*dds)); 344 345 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { 346 uint64_t dsize = 0; 347 uint64_t refcnt = ddp->ddp_refcnt; 348 349 if (ddp->ddp_phys_birth == 0) 350 continue; 351 352 for (int d = 0; d < SPA_DVAS_PER_BP; d++) 353 dsize += dva_get_dsize_sync(spa, &ddp->ddp_dva[d]); 354 355 dds->dds_blocks += 1; 356 dds->dds_lsize += lsize; 357 dds->dds_psize += psize; 358 dds->dds_dsize += dsize; 359 360 dds->dds_ref_blocks += refcnt; 361 dds->dds_ref_lsize += lsize * refcnt; 362 dds->dds_ref_psize += psize * refcnt; 363 dds->dds_ref_dsize += dsize * refcnt; 364 } 365 } 366 367 void 368 ddt_stat_add(ddt_stat_t *dst, const ddt_stat_t *src, uint64_t neg) 369 { 370 const uint64_t *s = (const uint64_t *)src; 371 uint64_t *d = (uint64_t *)dst; 372 uint64_t *d_end = (uint64_t *)(dst + 1); 373 374 ASSERT(neg == 0 || neg == -1ULL); /* add or subtract */ 375 376 while (d < d_end) 377 *d++ += (*s++ ^ neg) - neg; 378 } 379 380 static void 381 ddt_stat_update(ddt_t *ddt, ddt_entry_t *dde, uint64_t neg) 382 { 383 ddt_stat_t dds; 384 ddt_histogram_t *ddh; 385 int bucket; 386 387 ddt_stat_generate(ddt, dde, &dds); 388 389 bucket = highbit(dds.dds_ref_blocks) - 1; 390 ASSERT(bucket >= 0); 391 392 ddh = &ddt->ddt_histogram[dde->dde_type][dde->dde_class]; 393 394 ddt_stat_add(&ddh->ddh_stat[bucket], &dds, neg); 395 } 396 397 void 398 ddt_histogram_add(ddt_histogram_t *dst, const ddt_histogram_t *src) 399 { 400 for (int h = 0; h < 64; h++) 401 ddt_stat_add(&dst->ddh_stat[h], &src->ddh_stat[h], 0); 402 } 403 404 void 405 ddt_histogram_stat(ddt_stat_t *dds, const ddt_histogram_t *ddh) 406 { 407 bzero(dds, sizeof (*dds)); 408 409 for (int h = 0; h < 64; h++) 410 ddt_stat_add(dds, &ddh->ddh_stat[h], 0); 411 } 412 413 boolean_t 414 ddt_histogram_empty(const ddt_histogram_t *ddh) 415 { 416 const uint64_t *s = (const uint64_t *)ddh; 417 const uint64_t *s_end = (const uint64_t *)(ddh + 1); 418 419 while (s < s_end) 420 if (*s++ != 0) 421 return (B_FALSE); 422 423 return (B_TRUE); 424 } 425 426 void 427 ddt_get_dedup_object_stats(spa_t *spa, ddt_object_t *ddo_total) 428 { 429 /* Sum the statistics we cached in ddt_object_sync(). */ 430 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { 431 ddt_t *ddt = spa->spa_ddt[c]; 432 for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 433 for (enum ddt_class class = 0; class < DDT_CLASSES; 434 class++) { 435 ddt_object_t *ddo = 436 &ddt->ddt_object_stats[type][class]; 437 ddo_total->ddo_count += ddo->ddo_count; 438 ddo_total->ddo_dspace += ddo->ddo_dspace; 439 ddo_total->ddo_mspace += ddo->ddo_mspace; 440 } 441 } 442 } 443 444 /* ... and compute the averages. */ 445 if (ddo_total->ddo_count != 0) { 446 ddo_total->ddo_dspace /= ddo_total->ddo_count; 447 ddo_total->ddo_mspace /= ddo_total->ddo_count; 448 } else { 449 ASSERT(ddo_total->ddo_dspace == 0); 450 ASSERT(ddo_total->ddo_mspace == 0); 451 } 452 } 453 454 void 455 ddt_get_dedup_histogram(spa_t *spa, ddt_histogram_t *ddh) 456 { 457 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { 458 ddt_t *ddt = spa->spa_ddt[c]; 459 for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 460 for (enum ddt_class class = 0; class < DDT_CLASSES; 461 class++) { 462 ddt_histogram_add(ddh, 463 &ddt->ddt_histogram_cache[type][class]); 464 } 465 } 466 } 467 } 468 469 void 470 ddt_get_dedup_stats(spa_t *spa, ddt_stat_t *dds_total) 471 { 472 ddt_histogram_t *ddh_total; 473 474 ddh_total = kmem_zalloc(sizeof (ddt_histogram_t), KM_SLEEP); 475 ddt_get_dedup_histogram(spa, ddh_total); 476 ddt_histogram_stat(dds_total, ddh_total); 477 kmem_free(ddh_total, sizeof (ddt_histogram_t)); 478 } 479 480 uint64_t 481 ddt_get_dedup_dspace(spa_t *spa) 482 { 483 ddt_stat_t dds_total = { 0 }; 484 485 ddt_get_dedup_stats(spa, &dds_total); 486 return (dds_total.dds_ref_dsize - dds_total.dds_dsize); 487 } 488 489 uint64_t 490 ddt_get_pool_dedup_ratio(spa_t *spa) 491 { 492 ddt_stat_t dds_total = { 0 }; 493 494 ddt_get_dedup_stats(spa, &dds_total); 495 if (dds_total.dds_dsize == 0) 496 return (100); 497 498 return (dds_total.dds_ref_dsize * 100 / dds_total.dds_dsize); 499 } 500 501 int 502 ddt_ditto_copies_needed(ddt_t *ddt, ddt_entry_t *dde, ddt_phys_t *ddp_willref) 503 { 504 spa_t *spa = ddt->ddt_spa; 505 uint64_t total_refcnt = 0; 506 uint64_t ditto = spa->spa_dedup_ditto; 507 int total_copies = 0; 508 int desired_copies = 0; 509 510 for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) { 511 ddt_phys_t *ddp = &dde->dde_phys[p]; 512 zio_t *zio = dde->dde_lead_zio[p]; 513 uint64_t refcnt = ddp->ddp_refcnt; /* committed refs */ 514 if (zio != NULL) 515 refcnt += zio->io_parent_count; /* pending refs */ 516 if (ddp == ddp_willref) 517 refcnt++; /* caller's ref */ 518 if (refcnt != 0) { 519 total_refcnt += refcnt; 520 total_copies += p; 521 } 522 } 523 524 if (ditto == 0 || ditto > UINT32_MAX) 525 ditto = UINT32_MAX; 526 527 if (total_refcnt >= 1) 528 desired_copies++; 529 if (total_refcnt >= ditto) 530 desired_copies++; 531 if (total_refcnt >= ditto * ditto) 532 desired_copies++; 533 534 return (MAX(desired_copies, total_copies) - total_copies); 535 } 536 537 int 538 ddt_ditto_copies_present(ddt_entry_t *dde) 539 { 540 ddt_phys_t *ddp = &dde->dde_phys[DDT_PHYS_DITTO]; 541 dva_t *dva = ddp->ddp_dva; 542 int copies = 0 - DVA_GET_GANG(dva); 543 544 for (int d = 0; d < SPA_DVAS_PER_BP; d++, dva++) 545 if (DVA_IS_VALID(dva)) 546 copies++; 547 548 ASSERT(copies >= 0 && copies < SPA_DVAS_PER_BP); 549 550 return (copies); 551 } 552 553 size_t 554 ddt_compress(void *src, uchar_t *dst, size_t s_len, size_t d_len) 555 { 556 uchar_t *version = dst++; 557 int cpfunc = ZIO_COMPRESS_ZLE; 558 zio_compress_info_t *ci = &zio_compress_table[cpfunc]; 559 size_t c_len; 560 561 ASSERT(d_len >= s_len + 1); /* no compression plus version byte */ 562 563 c_len = ci->ci_compress(src, dst, s_len, d_len - 1, ci->ci_level); 564 565 if (c_len == s_len) { 566 cpfunc = ZIO_COMPRESS_OFF; 567 bcopy(src, dst, s_len); 568 } 569 570 *version = (ZFS_HOST_BYTEORDER & DDT_COMPRESS_BYTEORDER_MASK) | cpfunc; 571 572 return (c_len + 1); 573 } 574 575 void 576 ddt_decompress(uchar_t *src, void *dst, size_t s_len, size_t d_len) 577 { 578 uchar_t version = *src++; 579 int cpfunc = version & DDT_COMPRESS_FUNCTION_MASK; 580 zio_compress_info_t *ci = &zio_compress_table[cpfunc]; 581 582 if (ci->ci_decompress != NULL) 583 (void) ci->ci_decompress(src, dst, s_len, d_len, ci->ci_level); 584 else 585 bcopy(src, dst, d_len); 586 587 if ((version ^ ZFS_HOST_BYTEORDER) & DDT_COMPRESS_BYTEORDER_MASK) 588 byteswap_uint64_array(dst, d_len); 589 } 590 591 ddt_t * 592 ddt_select_by_checksum(spa_t *spa, enum zio_checksum c) 593 { 594 return (spa->spa_ddt[c]); 595 } 596 597 ddt_t * 598 ddt_select(spa_t *spa, const blkptr_t *bp) 599 { 600 return (spa->spa_ddt[BP_GET_CHECKSUM(bp)]); 601 } 602 603 void 604 ddt_enter(ddt_t *ddt) 605 { 606 mutex_enter(&ddt->ddt_lock); 607 } 608 609 void 610 ddt_exit(ddt_t *ddt) 611 { 612 mutex_exit(&ddt->ddt_lock); 613 } 614 615 static ddt_entry_t * 616 ddt_alloc(const ddt_key_t *ddk) 617 { 618 ddt_entry_t *dde; 619 620 dde = kmem_zalloc(sizeof (ddt_entry_t), KM_SLEEP); 621 cv_init(&dde->dde_cv, NULL, CV_DEFAULT, NULL); 622 623 dde->dde_key = *ddk; 624 625 return (dde); 626 } 627 628 static void 629 ddt_free(ddt_entry_t *dde) 630 { 631 ASSERT(!dde->dde_loading); 632 633 for (int p = 0; p < DDT_PHYS_TYPES; p++) 634 ASSERT(dde->dde_lead_zio[p] == NULL); 635 636 if (dde->dde_repair_data != NULL) 637 zio_buf_free(dde->dde_repair_data, 638 DDK_GET_PSIZE(&dde->dde_key)); 639 640 cv_destroy(&dde->dde_cv); 641 kmem_free(dde, sizeof (*dde)); 642 } 643 644 void 645 ddt_remove(ddt_t *ddt, ddt_entry_t *dde) 646 { 647 ASSERT(MUTEX_HELD(&ddt->ddt_lock)); 648 649 avl_remove(&ddt->ddt_tree, dde); 650 ddt_free(dde); 651 } 652 653 ddt_entry_t * 654 ddt_lookup(ddt_t *ddt, const blkptr_t *bp, boolean_t add) 655 { 656 ddt_entry_t *dde, dde_search; 657 enum ddt_type type; 658 enum ddt_class class; 659 avl_index_t where; 660 int error; 661 662 ASSERT(MUTEX_HELD(&ddt->ddt_lock)); 663 664 ddt_key_fill(&dde_search.dde_key, bp); 665 666 dde = avl_find(&ddt->ddt_tree, &dde_search, &where); 667 if (dde == NULL) { 668 if (!add) 669 return (NULL); 670 dde = ddt_alloc(&dde_search.dde_key); 671 avl_insert(&ddt->ddt_tree, dde, where); 672 } 673 674 while (dde->dde_loading) 675 cv_wait(&dde->dde_cv, &ddt->ddt_lock); 676 677 if (dde->dde_loaded) 678 return (dde); 679 680 dde->dde_loading = B_TRUE; 681 682 ddt_exit(ddt); 683 684 error = ENOENT; 685 686 for (type = 0; type < DDT_TYPES; type++) { 687 for (class = 0; class < DDT_CLASSES; class++) { 688 error = ddt_object_lookup(ddt, type, class, dde); 689 if (error != ENOENT) 690 break; 691 } 692 if (error != ENOENT) 693 break; 694 } 695 696 ASSERT(error == 0 || error == ENOENT); 697 698 ddt_enter(ddt); 699 700 ASSERT(dde->dde_loaded == B_FALSE); 701 ASSERT(dde->dde_loading == B_TRUE); 702 703 dde->dde_type = type; /* will be DDT_TYPES if no entry found */ 704 dde->dde_class = class; /* will be DDT_CLASSES if no entry found */ 705 dde->dde_loaded = B_TRUE; 706 dde->dde_loading = B_FALSE; 707 708 if (error == 0) 709 ddt_stat_update(ddt, dde, -1ULL); 710 711 cv_broadcast(&dde->dde_cv); 712 713 return (dde); 714 } 715 716 int 717 ddt_entry_compare(const void *x1, const void *x2) 718 { 719 const ddt_entry_t *dde1 = x1; 720 const ddt_entry_t *dde2 = x2; 721 const uint64_t *u1 = (const uint64_t *)&dde1->dde_key; 722 const uint64_t *u2 = (const uint64_t *)&dde2->dde_key; 723 724 for (int i = 0; i < DDT_KEY_WORDS; i++) { 725 if (u1[i] < u2[i]) 726 return (-1); 727 if (u1[i] > u2[i]) 728 return (1); 729 } 730 731 return (0); 732 } 733 734 static ddt_t * 735 ddt_table_alloc(spa_t *spa, enum zio_checksum c) 736 { 737 ddt_t *ddt; 738 739 ddt = kmem_zalloc(sizeof (*ddt), KM_SLEEP); 740 741 mutex_init(&ddt->ddt_lock, NULL, MUTEX_DEFAULT, NULL); 742 avl_create(&ddt->ddt_tree, ddt_entry_compare, 743 sizeof (ddt_entry_t), offsetof(ddt_entry_t, dde_node)); 744 avl_create(&ddt->ddt_repair_tree, ddt_entry_compare, 745 sizeof (ddt_entry_t), offsetof(ddt_entry_t, dde_node)); 746 ddt->ddt_checksum = c; 747 ddt->ddt_spa = spa; 748 ddt->ddt_os = spa->spa_meta_objset; 749 750 return (ddt); 751 } 752 753 static void 754 ddt_table_free(ddt_t *ddt) 755 { 756 ASSERT(avl_numnodes(&ddt->ddt_tree) == 0); 757 ASSERT(avl_numnodes(&ddt->ddt_repair_tree) == 0); 758 avl_destroy(&ddt->ddt_tree); 759 avl_destroy(&ddt->ddt_repair_tree); 760 mutex_destroy(&ddt->ddt_lock); 761 kmem_free(ddt, sizeof (*ddt)); 762 } 763 764 void 765 ddt_create(spa_t *spa) 766 { 767 spa->spa_dedup_checksum = ZIO_DEDUPCHECKSUM; 768 769 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) 770 spa->spa_ddt[c] = ddt_table_alloc(spa, c); 771 } 772 773 int 774 ddt_load(spa_t *spa) 775 { 776 int error; 777 778 ddt_create(spa); 779 780 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 781 DMU_POOL_DDT_STATS, sizeof (uint64_t), 1, 782 &spa->spa_ddt_stat_object); 783 784 if (error) 785 return (error == ENOENT ? 0 : error); 786 787 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { 788 ddt_t *ddt = spa->spa_ddt[c]; 789 for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 790 for (enum ddt_class class = 0; class < DDT_CLASSES; 791 class++) { 792 error = ddt_object_load(ddt, type, class); 793 if (error != 0 && error != ENOENT) 794 return (error); 795 } 796 } 797 798 /* 799 * Seed the cached histograms. 800 */ 801 bcopy(ddt->ddt_histogram, &ddt->ddt_histogram_cache, 802 sizeof (ddt->ddt_histogram)); 803 } 804 805 return (0); 806 } 807 808 void 809 ddt_unload(spa_t *spa) 810 { 811 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { 812 if (spa->spa_ddt[c]) { 813 ddt_table_free(spa->spa_ddt[c]); 814 spa->spa_ddt[c] = NULL; 815 } 816 } 817 } 818 819 boolean_t 820 ddt_class_contains(spa_t *spa, enum ddt_class max_class, const blkptr_t *bp) 821 { 822 ddt_t *ddt; 823 ddt_entry_t dde; 824 825 if (!BP_GET_DEDUP(bp)) 826 return (B_FALSE); 827 828 if (max_class == DDT_CLASS_UNIQUE) 829 return (B_TRUE); 830 831 ddt = spa->spa_ddt[BP_GET_CHECKSUM(bp)]; 832 833 ddt_key_fill(&dde.dde_key, bp); 834 835 for (enum ddt_type type = 0; type < DDT_TYPES; type++) 836 for (enum ddt_class class = 0; class <= max_class; class++) 837 if (ddt_object_lookup(ddt, type, class, &dde) == 0) 838 return (B_TRUE); 839 840 return (B_FALSE); 841 } 842 843 ddt_entry_t * 844 ddt_repair_start(ddt_t *ddt, const blkptr_t *bp) 845 { 846 ddt_key_t ddk; 847 ddt_entry_t *dde; 848 849 ddt_key_fill(&ddk, bp); 850 851 dde = ddt_alloc(&ddk); 852 853 for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 854 for (enum ddt_class class = 0; class < DDT_CLASSES; class++) { 855 /* 856 * We can only do repair if there are multiple copies 857 * of the block. For anything in the UNIQUE class, 858 * there's definitely only one copy, so don't even try. 859 */ 860 if (class != DDT_CLASS_UNIQUE && 861 ddt_object_lookup(ddt, type, class, dde) == 0) 862 return (dde); 863 } 864 } 865 866 bzero(dde->dde_phys, sizeof (dde->dde_phys)); 867 868 return (dde); 869 } 870 871 void 872 ddt_repair_done(ddt_t *ddt, ddt_entry_t *dde) 873 { 874 avl_index_t where; 875 876 ddt_enter(ddt); 877 878 if (dde->dde_repair_data != NULL && spa_writeable(ddt->ddt_spa) && 879 avl_find(&ddt->ddt_repair_tree, dde, &where) == NULL) 880 avl_insert(&ddt->ddt_repair_tree, dde, where); 881 else 882 ddt_free(dde); 883 884 ddt_exit(ddt); 885 } 886 887 static void 888 ddt_repair_entry_done(zio_t *zio) 889 { 890 ddt_entry_t *rdde = zio->io_private; 891 892 ddt_free(rdde); 893 } 894 895 static void 896 ddt_repair_entry(ddt_t *ddt, ddt_entry_t *dde, ddt_entry_t *rdde, zio_t *rio) 897 { 898 ddt_phys_t *ddp = dde->dde_phys; 899 ddt_phys_t *rddp = rdde->dde_phys; 900 ddt_key_t *ddk = &dde->dde_key; 901 ddt_key_t *rddk = &rdde->dde_key; 902 zio_t *zio; 903 blkptr_t blk; 904 905 zio = zio_null(rio, rio->io_spa, NULL, 906 ddt_repair_entry_done, rdde, rio->io_flags); 907 908 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++, rddp++) { 909 if (ddp->ddp_phys_birth == 0 || 910 ddp->ddp_phys_birth != rddp->ddp_phys_birth || 911 bcmp(ddp->ddp_dva, rddp->ddp_dva, sizeof (ddp->ddp_dva))) 912 continue; 913 ddt_bp_create(ddt->ddt_checksum, ddk, ddp, &blk); 914 zio_nowait(zio_rewrite(zio, zio->io_spa, 0, &blk, 915 rdde->dde_repair_data, DDK_GET_PSIZE(rddk), NULL, NULL, 916 ZIO_PRIORITY_SYNC_WRITE, ZIO_DDT_CHILD_FLAGS(zio), NULL)); 917 } 918 919 zio_nowait(zio); 920 } 921 922 static void 923 ddt_repair_table(ddt_t *ddt, zio_t *rio) 924 { 925 spa_t *spa = ddt->ddt_spa; 926 ddt_entry_t *dde, *rdde_next, *rdde; 927 avl_tree_t *t = &ddt->ddt_repair_tree; 928 blkptr_t blk; 929 930 if (spa_sync_pass(spa) > 1) 931 return; 932 933 ddt_enter(ddt); 934 for (rdde = avl_first(t); rdde != NULL; rdde = rdde_next) { 935 rdde_next = AVL_NEXT(t, rdde); 936 avl_remove(&ddt->ddt_repair_tree, rdde); 937 ddt_exit(ddt); 938 ddt_bp_create(ddt->ddt_checksum, &rdde->dde_key, NULL, &blk); 939 dde = ddt_repair_start(ddt, &blk); 940 ddt_repair_entry(ddt, dde, rdde, rio); 941 ddt_repair_done(ddt, dde); 942 ddt_enter(ddt); 943 } 944 ddt_exit(ddt); 945 } 946 947 static void 948 ddt_sync_entry(ddt_t *ddt, ddt_entry_t *dde, dmu_tx_t *tx, uint64_t txg) 949 { 950 dsl_pool_t *dp = ddt->ddt_spa->spa_dsl_pool; 951 ddt_phys_t *ddp = dde->dde_phys; 952 ddt_key_t *ddk = &dde->dde_key; 953 enum ddt_type otype = dde->dde_type; 954 enum ddt_type ntype = DDT_TYPE_CURRENT; 955 enum ddt_class oclass = dde->dde_class; 956 enum ddt_class nclass; 957 uint64_t total_refcnt = 0; 958 959 ASSERT(dde->dde_loaded); 960 ASSERT(!dde->dde_loading); 961 962 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { 963 ASSERT(dde->dde_lead_zio[p] == NULL); 964 ASSERT((int64_t)ddp->ddp_refcnt >= 0); 965 if (ddp->ddp_phys_birth == 0) { 966 ASSERT(ddp->ddp_refcnt == 0); 967 continue; 968 } 969 if (p == DDT_PHYS_DITTO) { 970 if (ddt_ditto_copies_needed(ddt, dde, NULL) == 0) 971 ddt_phys_free(ddt, ddk, ddp, txg); 972 continue; 973 } 974 if (ddp->ddp_refcnt == 0) 975 ddt_phys_free(ddt, ddk, ddp, txg); 976 total_refcnt += ddp->ddp_refcnt; 977 } 978 979 if (dde->dde_phys[DDT_PHYS_DITTO].ddp_phys_birth != 0) 980 nclass = DDT_CLASS_DITTO; 981 else if (total_refcnt > 1) 982 nclass = DDT_CLASS_DUPLICATE; 983 else 984 nclass = DDT_CLASS_UNIQUE; 985 986 if (otype != DDT_TYPES && 987 (otype != ntype || oclass != nclass || total_refcnt == 0)) { 988 VERIFY(ddt_object_remove(ddt, otype, oclass, dde, tx) == 0); 989 ASSERT(ddt_object_lookup(ddt, otype, oclass, dde) == ENOENT); 990 } 991 992 if (total_refcnt != 0) { 993 dde->dde_type = ntype; 994 dde->dde_class = nclass; 995 ddt_stat_update(ddt, dde, 0); 996 if (!ddt_object_exists(ddt, ntype, nclass)) 997 ddt_object_create(ddt, ntype, nclass, tx); 998 VERIFY(ddt_object_update(ddt, ntype, nclass, dde, tx) == 0); 999 1000 /* 1001 * If the class changes, the order that we scan this bp 1002 * changes. If it decreases, we could miss it, so 1003 * scan it right now. (This covers both class changing 1004 * while we are doing ddt_walk(), and when we are 1005 * traversing.) 1006 */ 1007 if (nclass < oclass) { 1008 dsl_scan_ddt_entry(dp->dp_scan, 1009 ddt->ddt_checksum, dde, tx); 1010 } 1011 } 1012 } 1013 1014 static void 1015 ddt_sync_table(ddt_t *ddt, dmu_tx_t *tx, uint64_t txg) 1016 { 1017 spa_t *spa = ddt->ddt_spa; 1018 ddt_entry_t *dde; 1019 void *cookie = NULL; 1020 1021 if (avl_numnodes(&ddt->ddt_tree) == 0) 1022 return; 1023 1024 ASSERT(spa->spa_uberblock.ub_version >= SPA_VERSION_DEDUP); 1025 1026 if (spa->spa_ddt_stat_object == 0) { 1027 spa->spa_ddt_stat_object = zap_create(ddt->ddt_os, 1028 DMU_OT_DDT_STATS, DMU_OT_NONE, 0, tx); 1029 VERIFY(zap_add(ddt->ddt_os, DMU_POOL_DIRECTORY_OBJECT, 1030 DMU_POOL_DDT_STATS, sizeof (uint64_t), 1, 1031 &spa->spa_ddt_stat_object, tx) == 0); 1032 } 1033 1034 while ((dde = avl_destroy_nodes(&ddt->ddt_tree, &cookie)) != NULL) { 1035 ddt_sync_entry(ddt, dde, tx, txg); 1036 ddt_free(dde); 1037 } 1038 1039 for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 1040 for (enum ddt_class class = 0; class < DDT_CLASSES; class++) { 1041 if (!ddt_object_exists(ddt, type, class)) 1042 continue; 1043 ddt_object_sync(ddt, type, class, tx); 1044 if (ddt_object_count(ddt, type, class) == 0) 1045 ddt_object_destroy(ddt, type, class, tx); 1046 } 1047 } 1048 1049 bcopy(ddt->ddt_histogram, &ddt->ddt_histogram_cache, 1050 sizeof (ddt->ddt_histogram)); 1051 } 1052 1053 void 1054 ddt_sync(spa_t *spa, uint64_t txg) 1055 { 1056 dmu_tx_t *tx; 1057 zio_t *rio = zio_root(spa, NULL, NULL, 1058 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE); 1059 1060 ASSERT(spa_syncing_txg(spa) == txg); 1061 1062 tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg); 1063 1064 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { 1065 ddt_t *ddt = spa->spa_ddt[c]; 1066 if (ddt == NULL) 1067 continue; 1068 ddt_sync_table(ddt, tx, txg); 1069 ddt_repair_table(ddt, rio); 1070 } 1071 1072 (void) zio_wait(rio); 1073 1074 dmu_tx_commit(tx); 1075 } 1076 1077 int 1078 ddt_walk(spa_t *spa, ddt_bookmark_t *ddb, ddt_entry_t *dde) 1079 { 1080 do { 1081 do { 1082 do { 1083 ddt_t *ddt = spa->spa_ddt[ddb->ddb_checksum]; 1084 int error = ENOENT; 1085 if (ddt_object_exists(ddt, ddb->ddb_type, 1086 ddb->ddb_class)) { 1087 error = ddt_object_walk(ddt, 1088 ddb->ddb_type, ddb->ddb_class, 1089 &ddb->ddb_cursor, dde); 1090 } 1091 dde->dde_type = ddb->ddb_type; 1092 dde->dde_class = ddb->ddb_class; 1093 if (error == 0) 1094 return (0); 1095 if (error != ENOENT) 1096 return (error); 1097 ddb->ddb_cursor = 0; 1098 } while (++ddb->ddb_checksum < ZIO_CHECKSUM_FUNCTIONS); 1099 ddb->ddb_checksum = 0; 1100 } while (++ddb->ddb_type < DDT_TYPES); 1101 ddb->ddb_type = 0; 1102 } while (++ddb->ddb_class < DDT_CLASSES); 1103 1104 return (ENOENT); 1105 } 1106