56 ddt_object_create(ddt_t *ddt, enum ddt_type type, enum ddt_class class,  in ddt_object_create()  argument
59 	spa_t *spa = ddt->ddt_spa;  in ddt_object_create()
60 	objset_t *os = ddt->ddt_os;  in ddt_object_create()
61 	uint64_t *objectp = &ddt->ddt_object[type][class];  in ddt_object_create()
62 	boolean_t prehash = zio_checksum_table[ddt->ddt_checksum].ci_dedup;  in ddt_object_create()
65 	ddt_object_name(ddt, type, class, name);  in ddt_object_create()
76 	    &ddt->ddt_histogram[type][class], tx) == 0);  in ddt_object_create()
80 ddt_object_destroy(ddt_t *ddt, enum ddt_type type, enum ddt_class class,  in ddt_object_destroy()  argument
83 	spa_t *spa = ddt->ddt_spa;  in ddt_object_destroy()
84 	objset_t *os = ddt->ddt_os;  in ddt_object_destroy()
85 	uint64_t *objectp = &ddt->ddt_object[type][class];  in ddt_object_destroy()
88 	ddt_object_name(ddt, type, class, name);  in ddt_object_destroy()
91 	ASSERT(ddt_object_count(ddt, type, class) == 0);  in ddt_object_destroy()
92 	ASSERT(ddt_histogram_empty(&ddt->ddt_histogram[type][class]));  in ddt_object_destroy()
96 	bzero(&ddt->ddt_object_stats[type][class], sizeof (ddt_object_t));  in ddt_object_destroy()
102 ddt_object_load(ddt_t *ddt, enum ddt_type type, enum ddt_class class)  in ddt_object_load()  argument
104 	ddt_object_t *ddo = &ddt->ddt_object_stats[type][class];  in ddt_object_load()
109 	ddt_object_name(ddt, type, class, name);  in ddt_object_load()
111 	error = zap_lookup(ddt->ddt_os, DMU_POOL_DIRECTORY_OBJECT, name,  in ddt_object_load()
112 	    sizeof (uint64_t), 1, &ddt->ddt_object[type][class]);  in ddt_object_load()
117 	VERIFY0(zap_lookup(ddt->ddt_os, ddt->ddt_spa->spa_ddt_stat_object, name,  in ddt_object_load()
119 	    &ddt->ddt_histogram[type][class]));  in ddt_object_load()
124 	VERIFY(ddt_object_info(ddt, type, class, &doi) == 0);  in ddt_object_load()
126 	ddo->ddo_count = ddt_object_count(ddt, type, class);  in ddt_object_load()
134 ddt_object_sync(ddt_t *ddt, enum ddt_type type, enum ddt_class class,  in ddt_object_sync()  argument
137 	ddt_object_t *ddo = &ddt->ddt_object_stats[type][class];  in ddt_object_sync()
141 	ddt_object_name(ddt, type, class, name);  in ddt_object_sync()
143 	VERIFY(zap_update(ddt->ddt_os, ddt->ddt_spa->spa_ddt_stat_object, name,  in ddt_object_sync()
145 	    &ddt->ddt_histogram[type][class], tx) == 0);  in ddt_object_sync()
150 	VERIFY(ddt_object_info(ddt, type, class, &doi) == 0);  in ddt_object_sync()
152 	ddo->ddo_count = ddt_object_count(ddt, type, class);  in ddt_object_sync()
158 ddt_object_lookup(ddt_t *ddt, enum ddt_type type, enum ddt_class class,  in ddt_object_lookup()  argument
161 	if (!ddt_object_exists(ddt, type, class))  in ddt_object_lookup()
164 	return (ddt_ops[type]->ddt_op_lookup(ddt->ddt_os,  in ddt_object_lookup()
165 	    ddt->ddt_object[type][class], dde));  in ddt_object_lookup()
169 ddt_object_prefetch(ddt_t *ddt, enum ddt_type type, enum ddt_class class,  in ddt_object_prefetch()  argument
172 	if (!ddt_object_exists(ddt, type, class))  in ddt_object_prefetch()
175 	ddt_ops[type]->ddt_op_prefetch(ddt->ddt_os,  in ddt_object_prefetch()
176 	    ddt->ddt_object[type][class], dde);  in ddt_object_prefetch()
180 ddt_object_update(ddt_t *ddt, enum ddt_type type, enum ddt_class class,  in ddt_object_update()  argument
183 	ASSERT(ddt_object_exists(ddt, type, class));  in ddt_object_update()
185 	return (ddt_ops[type]->ddt_op_update(ddt->ddt_os,  in ddt_object_update()
186 	    ddt->ddt_object[type][class], dde, tx));  in ddt_object_update()
190 ddt_object_remove(ddt_t *ddt, enum ddt_type type, enum ddt_class class,  in ddt_object_remove()  argument
193 	ASSERT(ddt_object_exists(ddt, type, class));  in ddt_object_remove()
195 	return (ddt_ops[type]->ddt_op_remove(ddt->ddt_os,  in ddt_object_remove()
196 	    ddt->ddt_object[type][class], dde, tx));  in ddt_object_remove()
200 ddt_object_walk(ddt_t *ddt, enum ddt_type type, enum ddt_class class,  in ddt_object_walk()  argument
203 	ASSERT(ddt_object_exists(ddt, type, class));  in ddt_object_walk()
205 	return (ddt_ops[type]->ddt_op_walk(ddt->ddt_os,  in ddt_object_walk()
206 	    ddt->ddt_object[type][class], dde, walk));  in ddt_object_walk()
210 ddt_object_count(ddt_t *ddt, enum ddt_type type, enum ddt_class class)  in ddt_object_count()  argument
212 	ASSERT(ddt_object_exists(ddt, type, class));  in ddt_object_count()
214 	return (ddt_ops[type]->ddt_op_count(ddt->ddt_os,  in ddt_object_count()
215 	    ddt->ddt_object[type][class]));  in ddt_object_count()
219 ddt_object_info(ddt_t *ddt, enum ddt_type type, enum ddt_class class,  in ddt_object_info()  argument
222 	if (!ddt_object_exists(ddt, type, class))  in ddt_object_info()
225 	return (dmu_object_info(ddt->ddt_os, ddt->ddt_object[type][class],  in ddt_object_info()
230 ddt_object_exists(ddt_t *ddt, enum ddt_type type, enum ddt_class class)  in ddt_object_exists()  argument
232 	return (!!ddt->ddt_object[type][class]);  in ddt_object_exists()
236 ddt_object_name(ddt_t *ddt, enum ddt_type type, enum ddt_class class,  in ddt_object_name()  argument
240 	    zio_checksum_table[ddt->ddt_checksum].ci_name,  in ddt_object_name()
317 ddt_phys_free(ddt_t *ddt, ddt_key_t *ddk, ddt_phys_t *ddp, uint64_t txg)  in ddt_phys_free()  argument
321 	ddt_bp_create(ddt->ddt_checksum, ddk, ddp, &blk);  in ddt_phys_free()
323 	zio_free(ddt->ddt_spa, txg, &blk);  in ddt_phys_free()
351 ddt_stat_generate(ddt_t *ddt, ddt_entry_t *dde, ddt_stat_t *dds)  in ddt_stat_generate()  argument
353 	spa_t *spa = ddt->ddt_spa;  in ddt_stat_generate()
397 ddt_stat_update(ddt_t *ddt, ddt_entry_t *dde, uint64_t neg)  in ddt_stat_update()  argument
403 	ddt_stat_generate(ddt, dde, &dds);  in ddt_stat_update()
408 	ddh = &ddt->ddt_histogram[dde->dde_type][dde->dde_class];  in ddt_stat_update()
447 		ddt_t *ddt = spa->spa_ddt[c];  in ddt_get_dedup_object_stats()  local
452 				    &ddt->ddt_object_stats[type][class];  in ddt_get_dedup_object_stats()
471 		ddt_t *ddt = spa->spa_ddt[c];  in ddt_get_dedup_histogram()  local
476 				    &ddt->ddt_histogram_cache[type][class]);  in ddt_get_dedup_histogram()
515 ddt_ditto_copies_needed(ddt_t *ddt, ddt_entry_t *dde, ddt_phys_t *ddp_willref)  in ddt_ditto_copies_needed()  argument
517 	spa_t *spa = ddt->ddt_spa;  in ddt_ditto_copies_needed()
621 ddt_enter(ddt_t *ddt)  in ddt_enter()  argument
623 	mutex_enter(&ddt->ddt_lock);  in ddt_enter()
627 ddt_exit(ddt_t *ddt)  in ddt_exit()  argument
629 	mutex_exit(&ddt->ddt_lock);  in ddt_exit()
662 ddt_remove(ddt_t *ddt, ddt_entry_t *dde)  in ddt_remove()  argument
664 	ASSERT(MUTEX_HELD(&ddt->ddt_lock));  in ddt_remove()
666 	avl_remove(&ddt->ddt_tree, dde);  in ddt_remove()
671 ddt_lookup(ddt_t *ddt, const blkptr_t *bp, boolean_t add)  in ddt_lookup()  argument
679 	ASSERT(MUTEX_HELD(&ddt->ddt_lock));  in ddt_lookup()
683 	dde = avl_find(&ddt->ddt_tree, &dde_search, &where);  in ddt_lookup()
688 		avl_insert(&ddt->ddt_tree, dde, where);  in ddt_lookup()
692 		cv_wait(&dde->dde_cv, &ddt->ddt_lock);  in ddt_lookup()
699 	ddt_exit(ddt);  in ddt_lookup()
705 			error = ddt_object_lookup(ddt, type, class, dde);  in ddt_lookup()
715 	ddt_enter(ddt);  in ddt_lookup()
726 		ddt_stat_update(ddt, dde, -1ULL);  in ddt_lookup()
736 	ddt_t *ddt;  in ddt_prefetch()  local
747 	ddt = ddt_select(spa, bp);  in ddt_prefetch()
752 			ddt_object_prefetch(ddt, type, class, &dde);  in ddt_prefetch()
778 	ddt_t *ddt;  in ddt_table_alloc()  local
780 	ddt = kmem_zalloc(sizeof (*ddt), KM_SLEEP);  in ddt_table_alloc()
782 	mutex_init(&ddt->ddt_lock, NULL, MUTEX_DEFAULT, NULL);  in ddt_table_alloc()
783 	avl_create(&ddt->ddt_tree, ddt_entry_compare,  in ddt_table_alloc()
785 	avl_create(&ddt->ddt_repair_tree, ddt_entry_compare,  in ddt_table_alloc()
787 	ddt->ddt_checksum = c;  in ddt_table_alloc()
788 	ddt->ddt_spa = spa;  in ddt_table_alloc()
789 	ddt->ddt_os = spa->spa_meta_objset;  in ddt_table_alloc()
791 	return (ddt);  in ddt_table_alloc()
795 ddt_table_free(ddt_t *ddt)  in ddt_table_free()  argument
797 	ASSERT(avl_numnodes(&ddt->ddt_tree) == 0);  in ddt_table_free()
798 	ASSERT(avl_numnodes(&ddt->ddt_repair_tree) == 0);  in ddt_table_free()
799 	avl_destroy(&ddt->ddt_tree);  in ddt_table_free()
800 	avl_destroy(&ddt->ddt_repair_tree);  in ddt_table_free()
801 	mutex_destroy(&ddt->ddt_lock);  in ddt_table_free()
802 	kmem_free(ddt, sizeof (*ddt));  in ddt_table_free()
829 		ddt_t *ddt = spa->spa_ddt[c];  in ddt_load()  local
833 				error = ddt_object_load(ddt, type, class);  in ddt_load()
842 		bcopy(ddt->ddt_histogram, &ddt->ddt_histogram_cache,  in ddt_load()
843 		    sizeof (ddt->ddt_histogram));  in ddt_load()
863 	ddt_t *ddt;  in ddt_class_contains()  local
872 	ddt = spa->spa_ddt[BP_GET_CHECKSUM(bp)];  in ddt_class_contains()
878 			if (ddt_object_lookup(ddt, type, class, &dde) == 0)  in ddt_class_contains()
885 ddt_repair_start(ddt_t *ddt, const blkptr_t *bp)  in ddt_repair_start()  argument
902 			    ddt_object_lookup(ddt, type, class, dde) == 0)  in ddt_repair_start()
913 ddt_repair_done(ddt_t *ddt, ddt_entry_t *dde)  in ddt_repair_done()  argument
917 	ddt_enter(ddt);  in ddt_repair_done()
919 	if (dde->dde_repair_data != NULL && spa_writeable(ddt->ddt_spa) &&  in ddt_repair_done()
920 	    avl_find(&ddt->ddt_repair_tree, dde, &where) == NULL)  in ddt_repair_done()
921 		avl_insert(&ddt->ddt_repair_tree, dde, where);  in ddt_repair_done()
925 	ddt_exit(ddt);  in ddt_repair_done()
937 ddt_repair_entry(ddt_t *ddt, ddt_entry_t *dde, ddt_entry_t *rdde, zio_t *rio)  in ddt_repair_entry()  argument
954 		ddt_bp_create(ddt->ddt_checksum, ddk, ddp, &blk);  in ddt_repair_entry()
964 ddt_repair_table(ddt_t *ddt, zio_t *rio)  in ddt_repair_table()  argument
966 	spa_t *spa = ddt->ddt_spa;  in ddt_repair_table()
968 	avl_tree_t *t = &ddt->ddt_repair_tree;  in ddt_repair_table()
974 	ddt_enter(ddt);  in ddt_repair_table()
977 		avl_remove(&ddt->ddt_repair_tree, rdde);  in ddt_repair_table()
978 		ddt_exit(ddt);  in ddt_repair_table()
979 		ddt_bp_create(ddt->ddt_checksum, &rdde->dde_key, NULL, &blk);  in ddt_repair_table()
980 		dde = ddt_repair_start(ddt, &blk);  in ddt_repair_table()
981 		ddt_repair_entry(ddt, dde, rdde, rio);  in ddt_repair_table()
982 		ddt_repair_done(ddt, dde);  in ddt_repair_table()
983 		ddt_enter(ddt);  in ddt_repair_table()
985 	ddt_exit(ddt);  in ddt_repair_table()
989 ddt_sync_entry(ddt_t *ddt, ddt_entry_t *dde, dmu_tx_t *tx, uint64_t txg)  in ddt_sync_entry()  argument
991 	dsl_pool_t *dp = ddt->ddt_spa->spa_dsl_pool;  in ddt_sync_entry()
1011 			if (ddt_ditto_copies_needed(ddt, dde, NULL) == 0)  in ddt_sync_entry()
1012 				ddt_phys_free(ddt, ddk, ddp, txg);  in ddt_sync_entry()
1016 			ddt_phys_free(ddt, ddk, ddp, txg);  in ddt_sync_entry()
1029 		VERIFY(ddt_object_remove(ddt, otype, oclass, dde, tx) == 0);  in ddt_sync_entry()
1030 		ASSERT(ddt_object_lookup(ddt, otype, oclass, dde) == ENOENT);  in ddt_sync_entry()
1036 		ddt_stat_update(ddt, dde, 0);  in ddt_sync_entry()
1037 		if (!ddt_object_exists(ddt, ntype, nclass))  in ddt_sync_entry()
1038 			ddt_object_create(ddt, ntype, nclass, tx);  in ddt_sync_entry()
1039 		VERIFY(ddt_object_update(ddt, ntype, nclass, dde, tx) == 0);  in ddt_sync_entry()
1050 			    ddt->ddt_checksum, dde, tx);  in ddt_sync_entry()
1056 ddt_sync_table(ddt_t *ddt, dmu_tx_t *tx, uint64_t txg)  in ddt_sync_table()  argument
1058 	spa_t *spa = ddt->ddt_spa;  in ddt_sync_table()
1062 	if (avl_numnodes(&ddt->ddt_tree) == 0)  in ddt_sync_table()
1068 		spa->spa_ddt_stat_object = zap_create_link(ddt->ddt_os,  in ddt_sync_table()
1073 	while ((dde = avl_destroy_nodes(&ddt->ddt_tree, &cookie)) != NULL) {  in ddt_sync_table()
1074 		ddt_sync_entry(ddt, dde, tx, txg);  in ddt_sync_table()
1081 			if (ddt_object_exists(ddt, type, class)) {  in ddt_sync_table()
1082 				ddt_object_sync(ddt, type, class, tx);  in ddt_sync_table()
1083 				count += ddt_object_count(ddt, type, class);  in ddt_sync_table()
1087 			if (count == 0 && ddt_object_exists(ddt, type, class))  in ddt_sync_table()
1088 				ddt_object_destroy(ddt, type, class, tx);  in ddt_sync_table()
1092 	bcopy(ddt->ddt_histogram, &ddt->ddt_histogram_cache,  in ddt_sync_table()
1093 	    sizeof (ddt->ddt_histogram));  in ddt_sync_table()
1108 		ddt_t *ddt = spa->spa_ddt[c];  in ddt_sync()  local
1109 		if (ddt == NULL)  in ddt_sync()
1111 		ddt_sync_table(ddt, tx, txg);  in ddt_sync()
1112 		ddt_repair_table(ddt, rio);  in ddt_sync()
1126 				ddt_t *ddt = spa->spa_ddt[ddb->ddb_checksum];  in ddt_walk()  local
1128 				if (ddt_object_exists(ddt, ddb->ddb_type,  in ddt_walk()
1130 					error = ddt_object_walk(ddt,  in ddt_walk()