1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. 24 * Copyright (c) 2012, 2014 by Delphix. All rights reserved. 25 */ 26 27 #include <sys/zfs_context.h> 28 #include <sys/spa.h> 29 #include <sys/spa_impl.h> 30 #include <sys/zio.h> 31 #include <sys/ddt.h> 32 #include <sys/zap.h> 33 #include <sys/dmu_tx.h> 34 #include <sys/arc.h> 35 #include <sys/dsl_pool.h> 36 #include <sys/zio_checksum.h> 37 #include <sys/zio_compress.h> 38 #include <sys/dsl_scan.h> 39 40 /* 41 * Enable/disable prefetching of dedup-ed blocks which are going to be freed. 42 */ 43 int zfs_dedup_prefetch = 1; 44 45 static const ddt_ops_t *ddt_ops[DDT_TYPES] = { 46 &ddt_zap_ops, 47 }; 48 49 static const char *ddt_class_name[DDT_CLASSES] = { 50 "ditto", 51 "duplicate", 52 "unique", 53 }; 54 55 static void 56 ddt_object_create(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 57 dmu_tx_t *tx) 58 { 59 spa_t *spa = ddt->ddt_spa; 60 objset_t *os = ddt->ddt_os; 61 uint64_t *objectp = &ddt->ddt_object[type][class]; 62 boolean_t prehash = zio_checksum_table[ddt->ddt_checksum].ci_dedup; 63 char name[DDT_NAMELEN]; 64 65 ddt_object_name(ddt, type, class, name); 66 67 ASSERT(*objectp == 0); 68 VERIFY(ddt_ops[type]->ddt_op_create(os, objectp, tx, prehash) == 0); 69 ASSERT(*objectp != 0); 70 71 VERIFY(zap_add(os, DMU_POOL_DIRECTORY_OBJECT, name, 72 sizeof (uint64_t), 1, objectp, tx) == 0); 73 74 VERIFY(zap_add(os, spa->spa_ddt_stat_object, name, 75 sizeof (uint64_t), sizeof (ddt_histogram_t) / sizeof (uint64_t), 76 &ddt->ddt_histogram[type][class], tx) == 0); 77 } 78 79 static void 80 ddt_object_destroy(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 81 dmu_tx_t *tx) 82 { 83 spa_t *spa = ddt->ddt_spa; 84 objset_t *os = ddt->ddt_os; 85 uint64_t *objectp = &ddt->ddt_object[type][class]; 86 char name[DDT_NAMELEN]; 87 88 ddt_object_name(ddt, type, class, name); 89 90 ASSERT(*objectp != 0); 91 ASSERT(ddt_object_count(ddt, type, class) == 0); 92 ASSERT(ddt_histogram_empty(&ddt->ddt_histogram[type][class])); 93 VERIFY(zap_remove(os, DMU_POOL_DIRECTORY_OBJECT, name, tx) == 0); 94 VERIFY(zap_remove(os, spa->spa_ddt_stat_object, name, tx) == 0); 95 VERIFY(ddt_ops[type]->ddt_op_destroy(os, *objectp, tx) == 0); 96 bzero(&ddt->ddt_object_stats[type][class], sizeof (ddt_object_t)); 97 98 *objectp = 0; 99 } 100 101 static int 102 ddt_object_load(ddt_t *ddt, enum ddt_type type, enum ddt_class class) 103 { 104 ddt_object_t *ddo = &ddt->ddt_object_stats[type][class]; 105 dmu_object_info_t doi; 106 char name[DDT_NAMELEN]; 107 int error; 108 109 ddt_object_name(ddt, type, class, name); 110 111 error = zap_lookup(ddt->ddt_os, DMU_POOL_DIRECTORY_OBJECT, name, 112 sizeof (uint64_t), 1, &ddt->ddt_object[type][class]); 113 114 if (error != 0) 115 return (error); 116 117 VERIFY0(zap_lookup(ddt->ddt_os, ddt->ddt_spa->spa_ddt_stat_object, name, 118 sizeof (uint64_t), sizeof (ddt_histogram_t) / sizeof (uint64_t), 119 &ddt->ddt_histogram[type][class])); 120 121 /* 122 * Seed the cached statistics. 123 */ 124 VERIFY(ddt_object_info(ddt, type, class, &doi) == 0); 125 126 ddo->ddo_count = ddt_object_count(ddt, type, class); 127 ddo->ddo_dspace = doi.doi_physical_blocks_512 << 9; 128 ddo->ddo_mspace = doi.doi_fill_count * doi.doi_data_block_size; 129 130 return (0); 131 } 132 133 static void 134 ddt_object_sync(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 135 dmu_tx_t *tx) 136 { 137 ddt_object_t *ddo = &ddt->ddt_object_stats[type][class]; 138 dmu_object_info_t doi; 139 char name[DDT_NAMELEN]; 140 141 ddt_object_name(ddt, type, class, name); 142 143 VERIFY(zap_update(ddt->ddt_os, ddt->ddt_spa->spa_ddt_stat_object, name, 144 sizeof (uint64_t), sizeof (ddt_histogram_t) / sizeof (uint64_t), 145 &ddt->ddt_histogram[type][class], tx) == 0); 146 147 /* 148 * Cache DDT statistics; this is the only time they'll change. 149 */ 150 VERIFY(ddt_object_info(ddt, type, class, &doi) == 0); 151 152 ddo->ddo_count = ddt_object_count(ddt, type, class); 153 ddo->ddo_dspace = doi.doi_physical_blocks_512 << 9; 154 ddo->ddo_mspace = doi.doi_fill_count * doi.doi_data_block_size; 155 } 156 157 static int 158 ddt_object_lookup(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 159 ddt_entry_t *dde) 160 { 161 if (!ddt_object_exists(ddt, type, class)) 162 return (SET_ERROR(ENOENT)); 163 164 return (ddt_ops[type]->ddt_op_lookup(ddt->ddt_os, 165 ddt->ddt_object[type][class], dde)); 166 } 167 168 static void 169 ddt_object_prefetch(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 170 ddt_entry_t *dde) 171 { 172 if (!ddt_object_exists(ddt, type, class)) 173 return; 174 175 ddt_ops[type]->ddt_op_prefetch(ddt->ddt_os, 176 ddt->ddt_object[type][class], dde); 177 } 178 179 int 180 ddt_object_update(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 181 ddt_entry_t *dde, dmu_tx_t *tx) 182 { 183 ASSERT(ddt_object_exists(ddt, type, class)); 184 185 return (ddt_ops[type]->ddt_op_update(ddt->ddt_os, 186 ddt->ddt_object[type][class], dde, tx)); 187 } 188 189 static int 190 ddt_object_remove(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 191 ddt_entry_t *dde, dmu_tx_t *tx) 192 { 193 ASSERT(ddt_object_exists(ddt, type, class)); 194 195 return (ddt_ops[type]->ddt_op_remove(ddt->ddt_os, 196 ddt->ddt_object[type][class], dde, tx)); 197 } 198 199 int 200 ddt_object_walk(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 201 uint64_t *walk, ddt_entry_t *dde) 202 { 203 ASSERT(ddt_object_exists(ddt, type, class)); 204 205 return (ddt_ops[type]->ddt_op_walk(ddt->ddt_os, 206 ddt->ddt_object[type][class], dde, walk)); 207 } 208 209 uint64_t 210 ddt_object_count(ddt_t *ddt, enum ddt_type type, enum ddt_class class) 211 { 212 ASSERT(ddt_object_exists(ddt, type, class)); 213 214 return (ddt_ops[type]->ddt_op_count(ddt->ddt_os, 215 ddt->ddt_object[type][class])); 216 } 217 218 int 219 ddt_object_info(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 220 dmu_object_info_t *doi) 221 { 222 if (!ddt_object_exists(ddt, type, class)) 223 return (SET_ERROR(ENOENT)); 224 225 return (dmu_object_info(ddt->ddt_os, ddt->ddt_object[type][class], 226 doi)); 227 } 228 229 boolean_t 230 ddt_object_exists(ddt_t *ddt, enum ddt_type type, enum ddt_class class) 231 { 232 return (!!ddt->ddt_object[type][class]); 233 } 234 235 void 236 ddt_object_name(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 237 char *name) 238 { 239 (void) sprintf(name, DMU_POOL_DDT, 240 zio_checksum_table[ddt->ddt_checksum].ci_name, 241 ddt_ops[type]->ddt_op_name, ddt_class_name[class]); 242 } 243 244 void 245 ddt_bp_fill(const ddt_phys_t *ddp, blkptr_t *bp, uint64_t txg) 246 { 247 ASSERT(txg != 0); 248 249 for (int d = 0; d < SPA_DVAS_PER_BP; d++) 250 bp->blk_dva[d] = ddp->ddp_dva[d]; 251 BP_SET_BIRTH(bp, txg, ddp->ddp_phys_birth); 252 } 253 254 void 255 ddt_bp_create(enum zio_checksum checksum, 256 const ddt_key_t *ddk, const ddt_phys_t *ddp, blkptr_t *bp) 257 { 258 BP_ZERO(bp); 259 260 if (ddp != NULL) 261 ddt_bp_fill(ddp, bp, ddp->ddp_phys_birth); 262 263 bp->blk_cksum = ddk->ddk_cksum; 264 bp->blk_fill = 1; 265 266 BP_SET_LSIZE(bp, DDK_GET_LSIZE(ddk)); 267 BP_SET_PSIZE(bp, DDK_GET_PSIZE(ddk)); 268 BP_SET_COMPRESS(bp, DDK_GET_COMPRESS(ddk)); 269 BP_SET_CHECKSUM(bp, checksum); 270 BP_SET_TYPE(bp, DMU_OT_DEDUP); 271 BP_SET_LEVEL(bp, 0); 272 BP_SET_DEDUP(bp, 0); 273 BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER); 274 } 275 276 void 277 ddt_key_fill(ddt_key_t *ddk, const blkptr_t *bp) 278 { 279 ddk->ddk_cksum = bp->blk_cksum; 280 ddk->ddk_prop = 0; 281 282 DDK_SET_LSIZE(ddk, BP_GET_LSIZE(bp)); 283 DDK_SET_PSIZE(ddk, BP_GET_PSIZE(bp)); 284 DDK_SET_COMPRESS(ddk, BP_GET_COMPRESS(bp)); 285 } 286 287 void 288 ddt_phys_fill(ddt_phys_t *ddp, const blkptr_t *bp) 289 { 290 ASSERT(ddp->ddp_phys_birth == 0); 291 292 for (int d = 0; d < SPA_DVAS_PER_BP; d++) 293 ddp->ddp_dva[d] = bp->blk_dva[d]; 294 ddp->ddp_phys_birth = BP_PHYSICAL_BIRTH(bp); 295 } 296 297 void 298 ddt_phys_clear(ddt_phys_t *ddp) 299 { 300 bzero(ddp, sizeof (*ddp)); 301 } 302 303 void 304 ddt_phys_addref(ddt_phys_t *ddp) 305 { 306 ddp->ddp_refcnt++; 307 } 308 309 void 310 ddt_phys_decref(ddt_phys_t *ddp) 311 { 312 ASSERT((int64_t)ddp->ddp_refcnt > 0); 313 ddp->ddp_refcnt--; 314 } 315 316 void 317 ddt_phys_free(ddt_t *ddt, ddt_key_t *ddk, ddt_phys_t *ddp, uint64_t txg) 318 { 319 blkptr_t blk; 320 321 ddt_bp_create(ddt->ddt_checksum, ddk, ddp, &blk); 322 ddt_phys_clear(ddp); 323 zio_free(ddt->ddt_spa, txg, &blk); 324 } 325 326 ddt_phys_t * 327 ddt_phys_select(const ddt_entry_t *dde, const blkptr_t *bp) 328 { 329 ddt_phys_t *ddp = (ddt_phys_t *)dde->dde_phys; 330 331 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { 332 if (DVA_EQUAL(BP_IDENTITY(bp), &ddp->ddp_dva[0]) && 333 BP_PHYSICAL_BIRTH(bp) == ddp->ddp_phys_birth) 334 return (ddp); 335 } 336 return (NULL); 337 } 338 339 uint64_t 340 ddt_phys_total_refcnt(const ddt_entry_t *dde) 341 { 342 uint64_t refcnt = 0; 343 344 for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) 345 refcnt += dde->dde_phys[p].ddp_refcnt; 346 347 return (refcnt); 348 } 349 350 static void 351 ddt_stat_generate(ddt_t *ddt, ddt_entry_t *dde, ddt_stat_t *dds) 352 { 353 spa_t *spa = ddt->ddt_spa; 354 ddt_phys_t *ddp = dde->dde_phys; 355 ddt_key_t *ddk = &dde->dde_key; 356 uint64_t lsize = DDK_GET_LSIZE(ddk); 357 uint64_t psize = DDK_GET_PSIZE(ddk); 358 359 bzero(dds, sizeof (*dds)); 360 361 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { 362 uint64_t dsize = 0; 363 uint64_t refcnt = ddp->ddp_refcnt; 364 365 if (ddp->ddp_phys_birth == 0) 366 continue; 367 368 for (int d = 0; d < SPA_DVAS_PER_BP; d++) 369 dsize += dva_get_dsize_sync(spa, &ddp->ddp_dva[d]); 370 371 dds->dds_blocks += 1; 372 dds->dds_lsize += lsize; 373 dds->dds_psize += psize; 374 dds->dds_dsize += dsize; 375 376 dds->dds_ref_blocks += refcnt; 377 dds->dds_ref_lsize += lsize * refcnt; 378 dds->dds_ref_psize += psize * refcnt; 379 dds->dds_ref_dsize += dsize * refcnt; 380 } 381 } 382 383 void 384 ddt_stat_add(ddt_stat_t *dst, const ddt_stat_t *src, uint64_t neg) 385 { 386 const uint64_t *s = (const uint64_t *)src; 387 uint64_t *d = (uint64_t *)dst; 388 uint64_t *d_end = (uint64_t *)(dst + 1); 389 390 ASSERT(neg == 0 || neg == -1ULL); /* add or subtract */ 391 392 while (d < d_end) 393 *d++ += (*s++ ^ neg) - neg; 394 } 395 396 static void 397 ddt_stat_update(ddt_t *ddt, ddt_entry_t *dde, uint64_t neg) 398 { 399 ddt_stat_t dds; 400 ddt_histogram_t *ddh; 401 int bucket; 402 403 ddt_stat_generate(ddt, dde, &dds); 404 405 bucket = highbit64(dds.dds_ref_blocks) - 1; 406 ASSERT(bucket >= 0); 407 408 ddh = &ddt->ddt_histogram[dde->dde_type][dde->dde_class]; 409 410 ddt_stat_add(&ddh->ddh_stat[bucket], &dds, neg); 411 } 412 413 void 414 ddt_histogram_add(ddt_histogram_t *dst, const ddt_histogram_t *src) 415 { 416 for (int h = 0; h < 64; h++) 417 ddt_stat_add(&dst->ddh_stat[h], &src->ddh_stat[h], 0); 418 } 419 420 void 421 ddt_histogram_stat(ddt_stat_t *dds, const ddt_histogram_t *ddh) 422 { 423 bzero(dds, sizeof (*dds)); 424 425 for (int h = 0; h < 64; h++) 426 ddt_stat_add(dds, &ddh->ddh_stat[h], 0); 427 } 428 429 boolean_t 430 ddt_histogram_empty(const ddt_histogram_t *ddh) 431 { 432 const uint64_t *s = (const uint64_t *)ddh; 433 const uint64_t *s_end = (const uint64_t *)(ddh + 1); 434 435 while (s < s_end) 436 if (*s++ != 0) 437 return (B_FALSE); 438 439 return (B_TRUE); 440 } 441 442 void 443 ddt_get_dedup_object_stats(spa_t *spa, ddt_object_t *ddo_total) 444 { 445 /* Sum the statistics we cached in ddt_object_sync(). */ 446 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { 447 ddt_t *ddt = spa->spa_ddt[c]; 448 for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 449 for (enum ddt_class class = 0; class < DDT_CLASSES; 450 class++) { 451 ddt_object_t *ddo = 452 &ddt->ddt_object_stats[type][class]; 453 ddo_total->ddo_count += ddo->ddo_count; 454 ddo_total->ddo_dspace += ddo->ddo_dspace; 455 ddo_total->ddo_mspace += ddo->ddo_mspace; 456 } 457 } 458 } 459 460 /* ... and compute the averages. */ 461 if (ddo_total->ddo_count != 0) { 462 ddo_total->ddo_dspace /= ddo_total->ddo_count; 463 ddo_total->ddo_mspace /= ddo_total->ddo_count; 464 } 465 } 466 467 void 468 ddt_get_dedup_histogram(spa_t *spa, ddt_histogram_t *ddh) 469 { 470 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { 471 ddt_t *ddt = spa->spa_ddt[c]; 472 for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 473 for (enum ddt_class class = 0; class < DDT_CLASSES; 474 class++) { 475 ddt_histogram_add(ddh, 476 &ddt->ddt_histogram_cache[type][class]); 477 } 478 } 479 } 480 } 481 482 void 483 ddt_get_dedup_stats(spa_t *spa, ddt_stat_t *dds_total) 484 { 485 ddt_histogram_t *ddh_total; 486 487 ddh_total = kmem_zalloc(sizeof (ddt_histogram_t), KM_SLEEP); 488 ddt_get_dedup_histogram(spa, ddh_total); 489 ddt_histogram_stat(dds_total, ddh_total); 490 kmem_free(ddh_total, sizeof (ddt_histogram_t)); 491 } 492 493 uint64_t 494 ddt_get_dedup_dspace(spa_t *spa) 495 { 496 ddt_stat_t dds_total = { 0 }; 497 498 ddt_get_dedup_stats(spa, &dds_total); 499 return (dds_total.dds_ref_dsize - dds_total.dds_dsize); 500 } 501 502 uint64_t 503 ddt_get_pool_dedup_ratio(spa_t *spa) 504 { 505 ddt_stat_t dds_total = { 0 }; 506 507 ddt_get_dedup_stats(spa, &dds_total); 508 if (dds_total.dds_dsize == 0) 509 return (100); 510 511 return (dds_total.dds_ref_dsize * 100 / dds_total.dds_dsize); 512 } 513 514 int 515 ddt_ditto_copies_needed(ddt_t *ddt, ddt_entry_t *dde, ddt_phys_t *ddp_willref) 516 { 517 spa_t *spa = ddt->ddt_spa; 518 uint64_t total_refcnt = 0; 519 uint64_t ditto = spa->spa_dedup_ditto; 520 int total_copies = 0; 521 int desired_copies = 0; 522 523 for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) { 524 ddt_phys_t *ddp = &dde->dde_phys[p]; 525 zio_t *zio = dde->dde_lead_zio[p]; 526 uint64_t refcnt = ddp->ddp_refcnt; /* committed refs */ 527 if (zio != NULL) 528 refcnt += zio->io_parent_count; /* pending refs */ 529 if (ddp == ddp_willref) 530 refcnt++; /* caller's ref */ 531 if (refcnt != 0) { 532 total_refcnt += refcnt; 533 total_copies += p; 534 } 535 } 536 537 if (ditto == 0 || ditto > UINT32_MAX) 538 ditto = UINT32_MAX; 539 540 if (total_refcnt >= 1) 541 desired_copies++; 542 if (total_refcnt >= ditto) 543 desired_copies++; 544 if (total_refcnt >= ditto * ditto) 545 desired_copies++; 546 547 return (MAX(desired_copies, total_copies) - total_copies); 548 } 549 550 int 551 ddt_ditto_copies_present(ddt_entry_t *dde) 552 { 553 ddt_phys_t *ddp = &dde->dde_phys[DDT_PHYS_DITTO]; 554 dva_t *dva = ddp->ddp_dva; 555 int copies = 0 - DVA_GET_GANG(dva); 556 557 for (int d = 0; d < SPA_DVAS_PER_BP; d++, dva++) 558 if (DVA_IS_VALID(dva)) 559 copies++; 560 561 ASSERT(copies >= 0 && copies < SPA_DVAS_PER_BP); 562 563 return (copies); 564 } 565 566 size_t 567 ddt_compress(void *src, uchar_t *dst, size_t s_len, size_t d_len) 568 { 569 uchar_t *version = dst++; 570 int cpfunc = ZIO_COMPRESS_ZLE; 571 zio_compress_info_t *ci = &zio_compress_table[cpfunc]; 572 size_t c_len; 573 574 ASSERT(d_len >= s_len + 1); /* no compression plus version byte */ 575 576 c_len = ci->ci_compress(src, dst, s_len, d_len - 1, ci->ci_level); 577 578 if (c_len == s_len) { 579 cpfunc = ZIO_COMPRESS_OFF; 580 bcopy(src, dst, s_len); 581 } 582 583 *version = cpfunc; 584 /* CONSTCOND */ 585 if (ZFS_HOST_BYTEORDER) 586 *version |= DDT_COMPRESS_BYTEORDER_MASK; 587 588 return (c_len + 1); 589 } 590 591 void 592 ddt_decompress(uchar_t *src, void *dst, size_t s_len, size_t d_len) 593 { 594 uchar_t version = *src++; 595 int cpfunc = version & DDT_COMPRESS_FUNCTION_MASK; 596 zio_compress_info_t *ci = &zio_compress_table[cpfunc]; 597 598 if (ci->ci_decompress != NULL) 599 (void) ci->ci_decompress(src, dst, s_len, d_len, ci->ci_level); 600 else 601 bcopy(src, dst, d_len); 602 603 if (((version & DDT_COMPRESS_BYTEORDER_MASK) != 0) != 604 (ZFS_HOST_BYTEORDER != 0)) 605 byteswap_uint64_array(dst, d_len); 606 } 607 608 ddt_t * 609 ddt_select_by_checksum(spa_t *spa, enum zio_checksum c) 610 { 611 return (spa->spa_ddt[c]); 612 } 613 614 ddt_t * 615 ddt_select(spa_t *spa, const blkptr_t *bp) 616 { 617 return (spa->spa_ddt[BP_GET_CHECKSUM(bp)]); 618 } 619 620 void 621 ddt_enter(ddt_t *ddt) 622 { 623 mutex_enter(&ddt->ddt_lock); 624 } 625 626 void 627 ddt_exit(ddt_t *ddt) 628 { 629 mutex_exit(&ddt->ddt_lock); 630 } 631 632 static ddt_entry_t * 633 ddt_alloc(const ddt_key_t *ddk) 634 { 635 ddt_entry_t *dde; 636 637 dde = kmem_zalloc(sizeof (ddt_entry_t), KM_SLEEP); 638 cv_init(&dde->dde_cv, NULL, CV_DEFAULT, NULL); 639 640 dde->dde_key = *ddk; 641 642 return (dde); 643 } 644 645 static void 646 ddt_free(ddt_entry_t *dde) 647 { 648 ASSERT(!dde->dde_loading); 649 650 for (int p = 0; p < DDT_PHYS_TYPES; p++) 651 ASSERT(dde->dde_lead_zio[p] == NULL); 652 653 if (dde->dde_repair_data != NULL) 654 zio_buf_free(dde->dde_repair_data, 655 DDK_GET_PSIZE(&dde->dde_key)); 656 657 cv_destroy(&dde->dde_cv); 658 kmem_free(dde, sizeof (*dde)); 659 } 660 661 void 662 ddt_remove(ddt_t *ddt, ddt_entry_t *dde) 663 { 664 ASSERT(MUTEX_HELD(&ddt->ddt_lock)); 665 666 avl_remove(&ddt->ddt_tree, dde); 667 ddt_free(dde); 668 } 669 670 ddt_entry_t * 671 ddt_lookup(ddt_t *ddt, const blkptr_t *bp, boolean_t add) 672 { 673 ddt_entry_t *dde, dde_search; 674 enum ddt_type type; 675 enum ddt_class class; 676 avl_index_t where; 677 int error; 678 679 ASSERT(MUTEX_HELD(&ddt->ddt_lock)); 680 681 ddt_key_fill(&dde_search.dde_key, bp); 682 683 dde = avl_find(&ddt->ddt_tree, &dde_search, &where); 684 if (dde == NULL) { 685 if (!add) 686 return (NULL); 687 dde = ddt_alloc(&dde_search.dde_key); 688 avl_insert(&ddt->ddt_tree, dde, where); 689 } 690 691 while (dde->dde_loading) 692 cv_wait(&dde->dde_cv, &ddt->ddt_lock); 693 694 if (dde->dde_loaded) 695 return (dde); 696 697 dde->dde_loading = B_TRUE; 698 699 ddt_exit(ddt); 700 701 error = ENOENT; 702 703 for (type = 0; type < DDT_TYPES; type++) { 704 for (class = 0; class < DDT_CLASSES; class++) { 705 error = ddt_object_lookup(ddt, type, class, dde); 706 if (error != ENOENT) 707 break; 708 } 709 if (error != ENOENT) 710 break; 711 } 712 713 ASSERT(error == 0 || error == ENOENT); 714 715 ddt_enter(ddt); 716 717 ASSERT(dde->dde_loaded == B_FALSE); 718 ASSERT(dde->dde_loading == B_TRUE); 719 720 dde->dde_type = type; /* will be DDT_TYPES if no entry found */ 721 dde->dde_class = class; /* will be DDT_CLASSES if no entry found */ 722 dde->dde_loaded = B_TRUE; 723 dde->dde_loading = B_FALSE; 724 725 if (error == 0) 726 ddt_stat_update(ddt, dde, -1ULL); 727 728 cv_broadcast(&dde->dde_cv); 729 730 return (dde); 731 } 732 733 void 734 ddt_prefetch(spa_t *spa, const blkptr_t *bp) 735 { 736 ddt_t *ddt; 737 ddt_entry_t dde; 738 739 if (!zfs_dedup_prefetch || bp == NULL || !BP_GET_DEDUP(bp)) 740 return; 741 742 /* 743 * We only remove the DDT once all tables are empty and only 744 * prefetch dedup blocks when there are entries in the DDT. 745 * Thus no locking is required as the DDT can't disappear on us. 746 */ 747 ddt = ddt_select(spa, bp); 748 ddt_key_fill(&dde.dde_key, bp); 749 750 for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 751 for (enum ddt_class class = 0; class < DDT_CLASSES; class++) { 752 ddt_object_prefetch(ddt, type, class, &dde); 753 } 754 } 755 } 756 757 int 758 ddt_entry_compare(const void *x1, const void *x2) 759 { 760 const ddt_entry_t *dde1 = x1; 761 const ddt_entry_t *dde2 = x2; 762 const uint64_t *u1 = (const uint64_t *)&dde1->dde_key; 763 const uint64_t *u2 = (const uint64_t *)&dde2->dde_key; 764 765 for (int i = 0; i < DDT_KEY_WORDS; i++) { 766 if (u1[i] < u2[i]) 767 return (-1); 768 if (u1[i] > u2[i]) 769 return (1); 770 } 771 772 return (0); 773 } 774 775 static ddt_t * 776 ddt_table_alloc(spa_t *spa, enum zio_checksum c) 777 { 778 ddt_t *ddt; 779 780 ddt = kmem_zalloc(sizeof (*ddt), KM_SLEEP); 781 782 mutex_init(&ddt->ddt_lock, NULL, MUTEX_DEFAULT, NULL); 783 avl_create(&ddt->ddt_tree, ddt_entry_compare, 784 sizeof (ddt_entry_t), offsetof(ddt_entry_t, dde_node)); 785 avl_create(&ddt->ddt_repair_tree, ddt_entry_compare, 786 sizeof (ddt_entry_t), offsetof(ddt_entry_t, dde_node)); 787 ddt->ddt_checksum = c; 788 ddt->ddt_spa = spa; 789 ddt->ddt_os = spa->spa_meta_objset; 790 791 return (ddt); 792 } 793 794 static void 795 ddt_table_free(ddt_t *ddt) 796 { 797 ASSERT(avl_numnodes(&ddt->ddt_tree) == 0); 798 ASSERT(avl_numnodes(&ddt->ddt_repair_tree) == 0); 799 avl_destroy(&ddt->ddt_tree); 800 avl_destroy(&ddt->ddt_repair_tree); 801 mutex_destroy(&ddt->ddt_lock); 802 kmem_free(ddt, sizeof (*ddt)); 803 } 804 805 void 806 ddt_create(spa_t *spa) 807 { 808 spa->spa_dedup_checksum = ZIO_DEDUPCHECKSUM; 809 810 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) 811 spa->spa_ddt[c] = ddt_table_alloc(spa, c); 812 } 813 814 int 815 ddt_load(spa_t *spa) 816 { 817 int error; 818 819 ddt_create(spa); 820 821 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 822 DMU_POOL_DDT_STATS, sizeof (uint64_t), 1, 823 &spa->spa_ddt_stat_object); 824 825 if (error) 826 return (error == ENOENT ? 0 : error); 827 828 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { 829 ddt_t *ddt = spa->spa_ddt[c]; 830 for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 831 for (enum ddt_class class = 0; class < DDT_CLASSES; 832 class++) { 833 error = ddt_object_load(ddt, type, class); 834 if (error != 0 && error != ENOENT) 835 return (error); 836 } 837 } 838 839 /* 840 * Seed the cached histograms. 841 */ 842 bcopy(ddt->ddt_histogram, &ddt->ddt_histogram_cache, 843 sizeof (ddt->ddt_histogram)); 844 } 845 846 return (0); 847 } 848 849 void 850 ddt_unload(spa_t *spa) 851 { 852 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { 853 if (spa->spa_ddt[c]) { 854 ddt_table_free(spa->spa_ddt[c]); 855 spa->spa_ddt[c] = NULL; 856 } 857 } 858 } 859 860 boolean_t 861 ddt_class_contains(spa_t *spa, enum ddt_class max_class, const blkptr_t *bp) 862 { 863 ddt_t *ddt; 864 ddt_entry_t dde; 865 866 if (!BP_GET_DEDUP(bp)) 867 return (B_FALSE); 868 869 if (max_class == DDT_CLASS_UNIQUE) 870 return (B_TRUE); 871 872 ddt = spa->spa_ddt[BP_GET_CHECKSUM(bp)]; 873 874 ddt_key_fill(&dde.dde_key, bp); 875 876 for (enum ddt_type type = 0; type < DDT_TYPES; type++) 877 for (enum ddt_class class = 0; class <= max_class; class++) 878 if (ddt_object_lookup(ddt, type, class, &dde) == 0) 879 return (B_TRUE); 880 881 return (B_FALSE); 882 } 883 884 ddt_entry_t * 885 ddt_repair_start(ddt_t *ddt, const blkptr_t *bp) 886 { 887 ddt_key_t ddk; 888 ddt_entry_t *dde; 889 890 ddt_key_fill(&ddk, bp); 891 892 dde = ddt_alloc(&ddk); 893 894 for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 895 for (enum ddt_class class = 0; class < DDT_CLASSES; class++) { 896 /* 897 * We can only do repair if there are multiple copies 898 * of the block. For anything in the UNIQUE class, 899 * there's definitely only one copy, so don't even try. 900 */ 901 if (class != DDT_CLASS_UNIQUE && 902 ddt_object_lookup(ddt, type, class, dde) == 0) 903 return (dde); 904 } 905 } 906 907 bzero(dde->dde_phys, sizeof (dde->dde_phys)); 908 909 return (dde); 910 } 911 912 void 913 ddt_repair_done(ddt_t *ddt, ddt_entry_t *dde) 914 { 915 avl_index_t where; 916 917 ddt_enter(ddt); 918 919 if (dde->dde_repair_data != NULL && spa_writeable(ddt->ddt_spa) && 920 avl_find(&ddt->ddt_repair_tree, dde, &where) == NULL) 921 avl_insert(&ddt->ddt_repair_tree, dde, where); 922 else 923 ddt_free(dde); 924 925 ddt_exit(ddt); 926 } 927 928 static void 929 ddt_repair_entry_done(zio_t *zio) 930 { 931 ddt_entry_t *rdde = zio->io_private; 932 933 ddt_free(rdde); 934 } 935 936 static void 937 ddt_repair_entry(ddt_t *ddt, ddt_entry_t *dde, ddt_entry_t *rdde, zio_t *rio) 938 { 939 ddt_phys_t *ddp = dde->dde_phys; 940 ddt_phys_t *rddp = rdde->dde_phys; 941 ddt_key_t *ddk = &dde->dde_key; 942 ddt_key_t *rddk = &rdde->dde_key; 943 zio_t *zio; 944 blkptr_t blk; 945 946 zio = zio_null(rio, rio->io_spa, NULL, 947 ddt_repair_entry_done, rdde, rio->io_flags); 948 949 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++, rddp++) { 950 if (ddp->ddp_phys_birth == 0 || 951 ddp->ddp_phys_birth != rddp->ddp_phys_birth || 952 bcmp(ddp->ddp_dva, rddp->ddp_dva, sizeof (ddp->ddp_dva))) 953 continue; 954 ddt_bp_create(ddt->ddt_checksum, ddk, ddp, &blk); 955 zio_nowait(zio_rewrite(zio, zio->io_spa, 0, &blk, 956 rdde->dde_repair_data, DDK_GET_PSIZE(rddk), NULL, NULL, 957 ZIO_PRIORITY_SYNC_WRITE, ZIO_DDT_CHILD_FLAGS(zio), NULL)); 958 } 959 960 zio_nowait(zio); 961 } 962 963 static void 964 ddt_repair_table(ddt_t *ddt, zio_t *rio) 965 { 966 spa_t *spa = ddt->ddt_spa; 967 ddt_entry_t *dde, *rdde_next, *rdde; 968 avl_tree_t *t = &ddt->ddt_repair_tree; 969 blkptr_t blk; 970 971 if (spa_sync_pass(spa) > 1) 972 return; 973 974 ddt_enter(ddt); 975 for (rdde = avl_first(t); rdde != NULL; rdde = rdde_next) { 976 rdde_next = AVL_NEXT(t, rdde); 977 avl_remove(&ddt->ddt_repair_tree, rdde); 978 ddt_exit(ddt); 979 ddt_bp_create(ddt->ddt_checksum, &rdde->dde_key, NULL, &blk); 980 dde = ddt_repair_start(ddt, &blk); 981 ddt_repair_entry(ddt, dde, rdde, rio); 982 ddt_repair_done(ddt, dde); 983 ddt_enter(ddt); 984 } 985 ddt_exit(ddt); 986 } 987 988 static void 989 ddt_sync_entry(ddt_t *ddt, ddt_entry_t *dde, dmu_tx_t *tx, uint64_t txg) 990 { 991 dsl_pool_t *dp = ddt->ddt_spa->spa_dsl_pool; 992 ddt_phys_t *ddp = dde->dde_phys; 993 ddt_key_t *ddk = &dde->dde_key; 994 enum ddt_type otype = dde->dde_type; 995 enum ddt_type ntype = DDT_TYPE_CURRENT; 996 enum ddt_class oclass = dde->dde_class; 997 enum ddt_class nclass; 998 uint64_t total_refcnt = 0; 999 1000 ASSERT(dde->dde_loaded); 1001 ASSERT(!dde->dde_loading); 1002 1003 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { 1004 ASSERT(dde->dde_lead_zio[p] == NULL); 1005 ASSERT((int64_t)ddp->ddp_refcnt >= 0); 1006 if (ddp->ddp_phys_birth == 0) { 1007 ASSERT(ddp->ddp_refcnt == 0); 1008 continue; 1009 } 1010 if (p == DDT_PHYS_DITTO) { 1011 if (ddt_ditto_copies_needed(ddt, dde, NULL) == 0) 1012 ddt_phys_free(ddt, ddk, ddp, txg); 1013 continue; 1014 } 1015 if (ddp->ddp_refcnt == 0) 1016 ddt_phys_free(ddt, ddk, ddp, txg); 1017 total_refcnt += ddp->ddp_refcnt; 1018 } 1019 1020 if (dde->dde_phys[DDT_PHYS_DITTO].ddp_phys_birth != 0) 1021 nclass = DDT_CLASS_DITTO; 1022 else if (total_refcnt > 1) 1023 nclass = DDT_CLASS_DUPLICATE; 1024 else 1025 nclass = DDT_CLASS_UNIQUE; 1026 1027 if (otype != DDT_TYPES && 1028 (otype != ntype || oclass != nclass || total_refcnt == 0)) { 1029 VERIFY(ddt_object_remove(ddt, otype, oclass, dde, tx) == 0); 1030 ASSERT(ddt_object_lookup(ddt, otype, oclass, dde) == ENOENT); 1031 } 1032 1033 if (total_refcnt != 0) { 1034 dde->dde_type = ntype; 1035 dde->dde_class = nclass; 1036 ddt_stat_update(ddt, dde, 0); 1037 if (!ddt_object_exists(ddt, ntype, nclass)) 1038 ddt_object_create(ddt, ntype, nclass, tx); 1039 VERIFY(ddt_object_update(ddt, ntype, nclass, dde, tx) == 0); 1040 1041 /* 1042 * If the class changes, the order that we scan this bp 1043 * changes. If it decreases, we could miss it, so 1044 * scan it right now. (This covers both class changing 1045 * while we are doing ddt_walk(), and when we are 1046 * traversing.) 1047 */ 1048 if (nclass < oclass) { 1049 dsl_scan_ddt_entry(dp->dp_scan, 1050 ddt->ddt_checksum, dde, tx); 1051 } 1052 } 1053 } 1054 1055 static void 1056 ddt_sync_table(ddt_t *ddt, dmu_tx_t *tx, uint64_t txg) 1057 { 1058 spa_t *spa = ddt->ddt_spa; 1059 ddt_entry_t *dde; 1060 void *cookie = NULL; 1061 1062 if (avl_numnodes(&ddt->ddt_tree) == 0) 1063 return; 1064 1065 ASSERT(spa->spa_uberblock.ub_version >= SPA_VERSION_DEDUP); 1066 1067 if (spa->spa_ddt_stat_object == 0) { 1068 spa->spa_ddt_stat_object = zap_create_link(ddt->ddt_os, 1069 DMU_OT_DDT_STATS, DMU_POOL_DIRECTORY_OBJECT, 1070 DMU_POOL_DDT_STATS, tx); 1071 } 1072 1073 while ((dde = avl_destroy_nodes(&ddt->ddt_tree, &cookie)) != NULL) { 1074 ddt_sync_entry(ddt, dde, tx, txg); 1075 ddt_free(dde); 1076 } 1077 1078 for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 1079 uint64_t count = 0; 1080 for (enum ddt_class class = 0; class < DDT_CLASSES; class++) { 1081 if (ddt_object_exists(ddt, type, class)) { 1082 ddt_object_sync(ddt, type, class, tx); 1083 count += ddt_object_count(ddt, type, class); 1084 } 1085 } 1086 for (enum ddt_class class = 0; class < DDT_CLASSES; class++) { 1087 if (count == 0 && ddt_object_exists(ddt, type, class)) 1088 ddt_object_destroy(ddt, type, class, tx); 1089 } 1090 } 1091 1092 bcopy(ddt->ddt_histogram, &ddt->ddt_histogram_cache, 1093 sizeof (ddt->ddt_histogram)); 1094 } 1095 1096 void 1097 ddt_sync(spa_t *spa, uint64_t txg) 1098 { 1099 dmu_tx_t *tx; 1100 zio_t *rio = zio_root(spa, NULL, NULL, 1101 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE); 1102 1103 ASSERT(spa_syncing_txg(spa) == txg); 1104 1105 tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg); 1106 1107 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { 1108 ddt_t *ddt = spa->spa_ddt[c]; 1109 if (ddt == NULL) 1110 continue; 1111 ddt_sync_table(ddt, tx, txg); 1112 ddt_repair_table(ddt, rio); 1113 } 1114 1115 (void) zio_wait(rio); 1116 1117 dmu_tx_commit(tx); 1118 } 1119 1120 int 1121 ddt_walk(spa_t *spa, ddt_bookmark_t *ddb, ddt_entry_t *dde) 1122 { 1123 do { 1124 do { 1125 do { 1126 ddt_t *ddt = spa->spa_ddt[ddb->ddb_checksum]; 1127 int error = ENOENT; 1128 if (ddt_object_exists(ddt, ddb->ddb_type, 1129 ddb->ddb_class)) { 1130 error = ddt_object_walk(ddt, 1131 ddb->ddb_type, ddb->ddb_class, 1132 &ddb->ddb_cursor, dde); 1133 } 1134 dde->dde_type = ddb->ddb_type; 1135 dde->dde_class = ddb->ddb_class; 1136 if (error == 0) 1137 return (0); 1138 if (error != ENOENT) 1139 return (error); 1140 ddb->ddb_cursor = 0; 1141 } while (++ddb->ddb_checksum < ZIO_CHECKSUM_FUNCTIONS); 1142 ddb->ddb_checksum = 0; 1143 } while (++ddb->ddb_type < DDT_TYPES); 1144 ddb->ddb_type = 0; 1145 } while (++ddb->ddb_class < DDT_CLASSES); 1146 1147 return (SET_ERROR(ENOENT)); 1148 } 1149