56 ddt_object_create(ddt_t *ddt, enum ddt_type type, enum ddt_class class,  in ddt_object_create()  argument
59 	spa_t *spa = ddt->ddt_spa;  in ddt_object_create()
60 	objset_t *os = ddt->ddt_os;  in ddt_object_create()
61 	uint64_t *objectp = &ddt->ddt_object[type][class];  in ddt_object_create()
62 	boolean_t prehash = zio_checksum_table[ddt->ddt_checksum].ci_flags &  in ddt_object_create()
66 	ddt_object_name(ddt, type, class, name);  in ddt_object_create()
77 	    &ddt->ddt_histogram[type][class], tx) == 0);  in ddt_object_create()
81 ddt_object_destroy(ddt_t *ddt, enum ddt_type type, enum ddt_class class,  in ddt_object_destroy()  argument
84 	spa_t *spa = ddt->ddt_spa;  in ddt_object_destroy()
85 	objset_t *os = ddt->ddt_os;  in ddt_object_destroy()
86 	uint64_t *objectp = &ddt->ddt_object[type][class];  in ddt_object_destroy()
89 	ddt_object_name(ddt, type, class, name);  in ddt_object_destroy()
92 	ASSERT(ddt_object_count(ddt, type, class) == 0);  in ddt_object_destroy()
93 	ASSERT(ddt_histogram_empty(&ddt->ddt_histogram[type][class]));  in ddt_object_destroy()
97 	bzero(&ddt->ddt_object_stats[type][class], sizeof (ddt_object_t));  in ddt_object_destroy()
103 ddt_object_load(ddt_t *ddt, enum ddt_type type, enum ddt_class class)  in ddt_object_load()  argument
105 	ddt_object_t *ddo = &ddt->ddt_object_stats[type][class];  in ddt_object_load()
110 	ddt_object_name(ddt, type, class, name);  in ddt_object_load()
112 	error = zap_lookup(ddt->ddt_os, DMU_POOL_DIRECTORY_OBJECT, name,  in ddt_object_load()
113 	    sizeof (uint64_t), 1, &ddt->ddt_object[type][class]);  in ddt_object_load()
118 	VERIFY0(zap_lookup(ddt->ddt_os, ddt->ddt_spa->spa_ddt_stat_object, name,  in ddt_object_load()
120 	    &ddt->ddt_histogram[type][class]));  in ddt_object_load()
125 	VERIFY(ddt_object_info(ddt, type, class, &doi) == 0);  in ddt_object_load()
127 	ddo->ddo_count = ddt_object_count(ddt, type, class);  in ddt_object_load()
135 ddt_object_sync(ddt_t *ddt, enum ddt_type type, enum ddt_class class,  in ddt_object_sync()  argument
138 	ddt_object_t *ddo = &ddt->ddt_object_stats[type][class];  in ddt_object_sync()
142 	ddt_object_name(ddt, type, class, name);  in ddt_object_sync()
144 	VERIFY(zap_update(ddt->ddt_os, ddt->ddt_spa->spa_ddt_stat_object, name,  in ddt_object_sync()
146 	    &ddt->ddt_histogram[type][class], tx) == 0);  in ddt_object_sync()
151 	VERIFY(ddt_object_info(ddt, type, class, &doi) == 0);  in ddt_object_sync()
153 	ddo->ddo_count = ddt_object_count(ddt, type, class);  in ddt_object_sync()
159 ddt_object_lookup(ddt_t *ddt, enum ddt_type type, enum ddt_class class,  in ddt_object_lookup()  argument
162 	if (!ddt_object_exists(ddt, type, class))  in ddt_object_lookup()
165 	return (ddt_ops[type]->ddt_op_lookup(ddt->ddt_os,  in ddt_object_lookup()
166 	    ddt->ddt_object[type][class], dde));  in ddt_object_lookup()
170 ddt_object_prefetch(ddt_t *ddt, enum ddt_type type, enum ddt_class class,  in ddt_object_prefetch()  argument
173 	if (!ddt_object_exists(ddt, type, class))  in ddt_object_prefetch()
176 	ddt_ops[type]->ddt_op_prefetch(ddt->ddt_os,  in ddt_object_prefetch()
177 	    ddt->ddt_object[type][class], dde);  in ddt_object_prefetch()
181 ddt_object_update(ddt_t *ddt, enum ddt_type type, enum ddt_class class,  in ddt_object_update()  argument
184 	ASSERT(ddt_object_exists(ddt, type, class));  in ddt_object_update()
186 	return (ddt_ops[type]->ddt_op_update(ddt->ddt_os,  in ddt_object_update()
187 	    ddt->ddt_object[type][class], dde, tx));  in ddt_object_update()
191 ddt_object_remove(ddt_t *ddt, enum ddt_type type, enum ddt_class class,  in ddt_object_remove()  argument
194 	ASSERT(ddt_object_exists(ddt, type, class));  in ddt_object_remove()
196 	return (ddt_ops[type]->ddt_op_remove(ddt->ddt_os,  in ddt_object_remove()
197 	    ddt->ddt_object[type][class], dde, tx));  in ddt_object_remove()
201 ddt_object_walk(ddt_t *ddt, enum ddt_type type, enum ddt_class class,  in ddt_object_walk()  argument
204 	ASSERT(ddt_object_exists(ddt, type, class));  in ddt_object_walk()
206 	return (ddt_ops[type]->ddt_op_walk(ddt->ddt_os,  in ddt_object_walk()
207 	    ddt->ddt_object[type][class], dde, walk));  in ddt_object_walk()
211 ddt_object_count(ddt_t *ddt, enum ddt_type type, enum ddt_class class)  in ddt_object_count()  argument
213 	ASSERT(ddt_object_exists(ddt, type, class));  in ddt_object_count()
215 	return (ddt_ops[type]->ddt_op_count(ddt->ddt_os,  in ddt_object_count()
216 	    ddt->ddt_object[type][class]));  in ddt_object_count()
220 ddt_object_info(ddt_t *ddt, enum ddt_type type, enum ddt_class class,  in ddt_object_info()  argument
223 	if (!ddt_object_exists(ddt, type, class))  in ddt_object_info()
226 	return (dmu_object_info(ddt->ddt_os, ddt->ddt_object[type][class],  in ddt_object_info()
231 ddt_object_exists(ddt_t *ddt, enum ddt_type type, enum ddt_class class)  in ddt_object_exists()  argument
233 	return (!!ddt->ddt_object[type][class]);  in ddt_object_exists()
237 ddt_object_name(ddt_t *ddt, enum ddt_type type, enum ddt_class class,  in ddt_object_name()  argument
241 	    zio_checksum_table[ddt->ddt_checksum].ci_name,  in ddt_object_name()
318 ddt_phys_free(ddt_t *ddt, ddt_key_t *ddk, ddt_phys_t *ddp, uint64_t txg)  in ddt_phys_free()  argument
322 	ddt_bp_create(ddt->ddt_checksum, ddk, ddp, &blk);  in ddt_phys_free()
324 	zio_free(ddt->ddt_spa, txg, &blk);  in ddt_phys_free()
352 ddt_stat_generate(ddt_t *ddt, ddt_entry_t *dde, ddt_stat_t *dds)  in ddt_stat_generate()  argument
354 	spa_t *spa = ddt->ddt_spa;  in ddt_stat_generate()
398 ddt_stat_update(ddt_t *ddt, ddt_entry_t *dde, uint64_t neg)  in ddt_stat_update()  argument
404 	ddt_stat_generate(ddt, dde, &dds);  in ddt_stat_update()
409 	ddh = &ddt->ddt_histogram[dde->dde_type][dde->dde_class];  in ddt_stat_update()
448 		ddt_t *ddt = spa->spa_ddt[c];  in ddt_get_dedup_object_stats()  local
453 				    &ddt->ddt_object_stats[type][class];  in ddt_get_dedup_object_stats()
472 		ddt_t *ddt = spa->spa_ddt[c];  in ddt_get_dedup_histogram()  local
477 				    &ddt->ddt_histogram_cache[type][class]);  in ddt_get_dedup_histogram()
516 ddt_ditto_copies_needed(ddt_t *ddt, ddt_entry_t *dde, ddt_phys_t *ddp_willref)  in ddt_ditto_copies_needed()  argument
518 	spa_t *spa = ddt->ddt_spa;  in ddt_ditto_copies_needed()
622 ddt_enter(ddt_t *ddt)  in ddt_enter()  argument
624 	mutex_enter(&ddt->ddt_lock);  in ddt_enter()
628 ddt_exit(ddt_t *ddt)  in ddt_exit()  argument
630 	mutex_exit(&ddt->ddt_lock);  in ddt_exit()
663 ddt_remove(ddt_t *ddt, ddt_entry_t *dde)  in ddt_remove()  argument
665 	ASSERT(MUTEX_HELD(&ddt->ddt_lock));  in ddt_remove()
667 	avl_remove(&ddt->ddt_tree, dde);  in ddt_remove()
672 ddt_lookup(ddt_t *ddt, const blkptr_t *bp, boolean_t add)  in ddt_lookup()  argument
680 	ASSERT(MUTEX_HELD(&ddt->ddt_lock));  in ddt_lookup()
684 	dde = avl_find(&ddt->ddt_tree, &dde_search, &where);  in ddt_lookup()
689 		avl_insert(&ddt->ddt_tree, dde, where);  in ddt_lookup()
693 		cv_wait(&dde->dde_cv, &ddt->ddt_lock);  in ddt_lookup()
700 	ddt_exit(ddt);  in ddt_lookup()
706 			error = ddt_object_lookup(ddt, type, class, dde);  in ddt_lookup()
716 	ddt_enter(ddt);  in ddt_lookup()
727 		ddt_stat_update(ddt, dde, -1ULL);  in ddt_lookup()
737 	ddt_t *ddt;  in ddt_prefetch()  local
748 	ddt = ddt_select(spa, bp);  in ddt_prefetch()
753 			ddt_object_prefetch(ddt, type, class, &dde);  in ddt_prefetch()
779 	ddt_t *ddt;  in ddt_table_alloc()  local
781 	ddt = kmem_zalloc(sizeof (*ddt), KM_SLEEP);  in ddt_table_alloc()
783 	mutex_init(&ddt->ddt_lock, NULL, MUTEX_DEFAULT, NULL);  in ddt_table_alloc()
784 	avl_create(&ddt->ddt_tree, ddt_entry_compare,  in ddt_table_alloc()
786 	avl_create(&ddt->ddt_repair_tree, ddt_entry_compare,  in ddt_table_alloc()
788 	ddt->ddt_checksum = c;  in ddt_table_alloc()
789 	ddt->ddt_spa = spa;  in ddt_table_alloc()
790 	ddt->ddt_os = spa->spa_meta_objset;  in ddt_table_alloc()
792 	return (ddt);  in ddt_table_alloc()
796 ddt_table_free(ddt_t *ddt)  in ddt_table_free()  argument
798 	ASSERT(avl_numnodes(&ddt->ddt_tree) == 0);  in ddt_table_free()
799 	ASSERT(avl_numnodes(&ddt->ddt_repair_tree) == 0);  in ddt_table_free()
800 	avl_destroy(&ddt->ddt_tree);  in ddt_table_free()
801 	avl_destroy(&ddt->ddt_repair_tree);  in ddt_table_free()
802 	mutex_destroy(&ddt->ddt_lock);  in ddt_table_free()
803 	kmem_free(ddt, sizeof (*ddt));  in ddt_table_free()
830 		ddt_t *ddt = spa->spa_ddt[c];  in ddt_load()  local
834 				error = ddt_object_load(ddt, type, class);  in ddt_load()
843 		bcopy(ddt->ddt_histogram, &ddt->ddt_histogram_cache,  in ddt_load()
844 		    sizeof (ddt->ddt_histogram));  in ddt_load()
864 	ddt_t *ddt;  in ddt_class_contains()  local
873 	ddt = spa->spa_ddt[BP_GET_CHECKSUM(bp)];  in ddt_class_contains()
879 			if (ddt_object_lookup(ddt, type, class, &dde) == 0)  in ddt_class_contains()
886 ddt_repair_start(ddt_t *ddt, const blkptr_t *bp)  in ddt_repair_start()  argument
903 			    ddt_object_lookup(ddt, type, class, dde) == 0)  in ddt_repair_start()
914 ddt_repair_done(ddt_t *ddt, ddt_entry_t *dde)  in ddt_repair_done()  argument
918 	ddt_enter(ddt);  in ddt_repair_done()
920 	if (dde->dde_repair_data != NULL && spa_writeable(ddt->ddt_spa) &&  in ddt_repair_done()
921 	    avl_find(&ddt->ddt_repair_tree, dde, &where) == NULL)  in ddt_repair_done()
922 		avl_insert(&ddt->ddt_repair_tree, dde, where);  in ddt_repair_done()
926 	ddt_exit(ddt);  in ddt_repair_done()
938 ddt_repair_entry(ddt_t *ddt, ddt_entry_t *dde, ddt_entry_t *rdde, zio_t *rio)  in ddt_repair_entry()  argument
955 		ddt_bp_create(ddt->ddt_checksum, ddk, ddp, &blk);  in ddt_repair_entry()
965 ddt_repair_table(ddt_t *ddt, zio_t *rio)  in ddt_repair_table()  argument
967 	spa_t *spa = ddt->ddt_spa;  in ddt_repair_table()
969 	avl_tree_t *t = &ddt->ddt_repair_tree;  in ddt_repair_table()
975 	ddt_enter(ddt);  in ddt_repair_table()
978 		avl_remove(&ddt->ddt_repair_tree, rdde);  in ddt_repair_table()
979 		ddt_exit(ddt);  in ddt_repair_table()
980 		ddt_bp_create(ddt->ddt_checksum, &rdde->dde_key, NULL, &blk);  in ddt_repair_table()
981 		dde = ddt_repair_start(ddt, &blk);  in ddt_repair_table()
982 		ddt_repair_entry(ddt, dde, rdde, rio);  in ddt_repair_table()
983 		ddt_repair_done(ddt, dde);  in ddt_repair_table()
984 		ddt_enter(ddt);  in ddt_repair_table()
986 	ddt_exit(ddt);  in ddt_repair_table()
990 ddt_sync_entry(ddt_t *ddt, ddt_entry_t *dde, dmu_tx_t *tx, uint64_t txg)  in ddt_sync_entry()  argument
992 	dsl_pool_t *dp = ddt->ddt_spa->spa_dsl_pool;  in ddt_sync_entry()
1012 			if (ddt_ditto_copies_needed(ddt, dde, NULL) == 0)  in ddt_sync_entry()
1013 				ddt_phys_free(ddt, ddk, ddp, txg);  in ddt_sync_entry()
1017 			ddt_phys_free(ddt, ddk, ddp, txg);  in ddt_sync_entry()
1030 		VERIFY(ddt_object_remove(ddt, otype, oclass, dde, tx) == 0);  in ddt_sync_entry()
1031 		ASSERT(ddt_object_lookup(ddt, otype, oclass, dde) == ENOENT);  in ddt_sync_entry()
1037 		ddt_stat_update(ddt, dde, 0);  in ddt_sync_entry()
1038 		if (!ddt_object_exists(ddt, ntype, nclass))  in ddt_sync_entry()
1039 			ddt_object_create(ddt, ntype, nclass, tx);  in ddt_sync_entry()
1040 		VERIFY(ddt_object_update(ddt, ntype, nclass, dde, tx) == 0);  in ddt_sync_entry()
1051 			    ddt->ddt_checksum, dde, tx);  in ddt_sync_entry()
1057 ddt_sync_table(ddt_t *ddt, dmu_tx_t *tx, uint64_t txg)  in ddt_sync_table()  argument
1059 	spa_t *spa = ddt->ddt_spa;  in ddt_sync_table()
1063 	if (avl_numnodes(&ddt->ddt_tree) == 0)  in ddt_sync_table()
1069 		spa->spa_ddt_stat_object = zap_create_link(ddt->ddt_os,  in ddt_sync_table()
1074 	while ((dde = avl_destroy_nodes(&ddt->ddt_tree, &cookie)) != NULL) {  in ddt_sync_table()
1075 		ddt_sync_entry(ddt, dde, tx, txg);  in ddt_sync_table()
1082 			if (ddt_object_exists(ddt, type, class)) {  in ddt_sync_table()
1083 				ddt_object_sync(ddt, type, class, tx);  in ddt_sync_table()
1084 				count += ddt_object_count(ddt, type, class);  in ddt_sync_table()
1088 			if (count == 0 && ddt_object_exists(ddt, type, class))  in ddt_sync_table()
1089 				ddt_object_destroy(ddt, type, class, tx);  in ddt_sync_table()
1093 	bcopy(ddt->ddt_histogram, &ddt->ddt_histogram_cache,  in ddt_sync_table()
1094 	    sizeof (ddt->ddt_histogram));  in ddt_sync_table()
1109 		ddt_t *ddt = spa->spa_ddt[c];  in ddt_sync()  local
1110 		if (ddt == NULL)  in ddt_sync()
1112 		ddt_sync_table(ddt, tx, txg);  in ddt_sync()
1113 		ddt_repair_table(ddt, rio);  in ddt_sync()
1127 				ddt_t *ddt = spa->spa_ddt[ddb->ddb_checksum];  in ddt_walk()  local
1129 				if (ddt_object_exists(ddt, ddb->ddb_type,  in ddt_walk()
1131 					error = ddt_object_walk(ddt,  in ddt_walk()