89 static void dmu_objset_upgrade(objset_t *os, dmu_objset_upgrade_cb_t cb);
90 static void dmu_objset_upgrade_stop(objset_t *os);
105 dmu_objset_spa(objset_t *os)  in dmu_objset_spa()  argument
107 	return (os->os_spa);  in dmu_objset_spa()
111 dmu_objset_zil(objset_t *os)  in dmu_objset_zil()  argument
113 	return (os->os_zil);  in dmu_objset_zil()
117 dmu_objset_pool(objset_t *os)  in dmu_objset_pool()  argument
121 	if ((ds = os->os_dsl_dataset) != NULL && ds->ds_dir)  in dmu_objset_pool()
124 		return (spa_get_dsl(os->os_spa));  in dmu_objset_pool()
128 dmu_objset_ds(objset_t *os)  in dmu_objset_ds()  argument
130 	return (os->os_dsl_dataset);  in dmu_objset_ds()
134 dmu_objset_type(objset_t *os)  in dmu_objset_type()  argument
136 	return (os->os_phys->os_type);  in dmu_objset_type()
140 dmu_objset_name(objset_t *os, char *buf)  in dmu_objset_name()  argument
142 	dsl_dataset_name(os->os_dsl_dataset, buf);  in dmu_objset_name()
146 dmu_objset_id(objset_t *os)  in dmu_objset_id()  argument
148 	dsl_dataset_t *ds = os->os_dsl_dataset;  in dmu_objset_id()
154 dmu_objset_dnodesize(objset_t *os)  in dmu_objset_dnodesize()  argument
156 	return (os->os_dnodesize);  in dmu_objset_dnodesize()
160 dmu_objset_syncprop(objset_t *os)  in dmu_objset_syncprop()  argument
162 	return (os->os_sync);  in dmu_objset_syncprop()
166 dmu_objset_logbias(objset_t *os)  in dmu_objset_logbias()  argument
168 	return (os->os_logbias);  in dmu_objset_logbias()
174 	objset_t *os = arg;  in checksum_changed_cb()  local
181 	os->os_checksum = zio_checksum_select(newval, ZIO_CHECKSUM_ON_VALUE);  in checksum_changed_cb()
187 	objset_t *os = arg;  in compression_changed_cb()  local
194 	os->os_compress = zio_compress_select(os->os_spa, newval,  in compression_changed_cb()
201 	objset_t *os = arg;  in copies_changed_cb()  local
207 	ASSERT(newval <= spa_max_replication(os->os_spa));  in copies_changed_cb()
209 	os->os_copies = newval;  in copies_changed_cb()
215 	objset_t *os = arg;  in dedup_changed_cb()  local
216 	spa_t *spa = os->os_spa;  in dedup_changed_cb()
226 	os->os_dedup_checksum = checksum & ZIO_CHECKSUM_MASK;  in dedup_changed_cb()
227 	os->os_dedup_verify = !!(checksum & ZIO_CHECKSUM_VERIFY);  in dedup_changed_cb()
233 	objset_t *os = arg;  in primary_cache_changed_cb()  local
241 	os->os_primary_cache = newval;  in primary_cache_changed_cb()
247 	objset_t *os = arg;  in secondary_cache_changed_cb()  local
255 	os->os_secondary_cache = newval;  in secondary_cache_changed_cb()
261 	objset_t *os = arg;  in sync_changed_cb()  local
269 	os->os_sync = newval;  in sync_changed_cb()
270 	if (os->os_zil)  in sync_changed_cb()
271 		zil_set_sync(os->os_zil, newval);  in sync_changed_cb()
277 	objset_t *os = arg;  in redundant_metadata_changed_cb()  local
285 	os->os_redundant_metadata = newval;  in redundant_metadata_changed_cb()
291 	objset_t *os = arg;  in dnodesize_changed_cb()  local
295 		os->os_dnodesize = DNODE_MIN_SIZE;  in dnodesize_changed_cb()
304 		os->os_dnodesize = DNODE_MIN_SIZE * 2;  in dnodesize_changed_cb()
311 		os->os_dnodesize = newval;  in dnodesize_changed_cb()
319 	objset_t *os = arg;  in smallblk_changed_cb()  local
327 	os->os_zpl_special_smallblock = newval;  in smallblk_changed_cb()
333 	objset_t *os = arg;  in logbias_changed_cb()  local
337 	os->os_logbias = newval;  in logbias_changed_cb()
338 	if (os->os_zil)  in logbias_changed_cb()
339 		zil_set_logbias(os->os_zil, newval);  in logbias_changed_cb()
345 	objset_t *os = arg;  in recordsize_changed_cb()  local
347 	os->os_recordsize = newval;  in recordsize_changed_cb()
373 dnode_hash(const objset_t *os, uint64_t obj)  in dnode_hash()  argument
375 	uintptr_t osv = (uintptr_t)os;  in dnode_hash()
409 	objset_t *os;  in dmu_objset_open_impl()  local
427 	os = kmem_zalloc(sizeof (objset_t), KM_SLEEP);  in dmu_objset_open_impl()
428 	os->os_dsl_dataset = ds;  in dmu_objset_open_impl()
429 	os->os_spa = spa;  in dmu_objset_open_impl()
430 	os->os_rootbp = bp;  in dmu_objset_open_impl()
431 	if (!BP_IS_HOLE(os->os_rootbp)) {  in dmu_objset_open_impl()
439 		if (DMU_OS_IS_L2CACHEABLE(os))  in dmu_objset_open_impl()
448 		dprintf_bp(os->os_rootbp, "reading %s", "");  in dmu_objset_open_impl()
449 		err = arc_read(NULL, spa, os->os_rootbp,  in dmu_objset_open_impl()
450 		    arc_getbuf_func, &os->os_phys_buf,  in dmu_objset_open_impl()
453 			kmem_free(os, sizeof (objset_t));  in dmu_objset_open_impl()
469 		if (arc_buf_size(os->os_phys_buf) < size) {  in dmu_objset_open_impl()
470 			arc_buf_t *buf = arc_alloc_buf(spa, &os->os_phys_buf,  in dmu_objset_open_impl()
473 			bcopy(os->os_phys_buf->b_data, buf->b_data,  in dmu_objset_open_impl()
474 			    arc_buf_size(os->os_phys_buf));  in dmu_objset_open_impl()
475 			arc_buf_destroy(os->os_phys_buf, &os->os_phys_buf);  in dmu_objset_open_impl()
476 			os->os_phys_buf = buf;  in dmu_objset_open_impl()
479 		os->os_phys = os->os_phys_buf->b_data;  in dmu_objset_open_impl()
480 		os->os_flags = os->os_phys->os_flags;  in dmu_objset_open_impl()
484 		os->os_phys_buf = arc_alloc_buf(spa, &os->os_phys_buf,  in dmu_objset_open_impl()
486 		os->os_phys = os->os_phys_buf->b_data;  in dmu_objset_open_impl()
487 		bzero(os->os_phys, size);  in dmu_objset_open_impl()
499 		os->os_encrypted = (ds->ds_dir->dd_crypto_obj != 0);  in dmu_objset_open_impl()
506 		if (!dsl_pool_config_held(dmu_objset_pool(os))) {  in dmu_objset_open_impl()
508 			dsl_pool_config_enter(dmu_objset_pool(os), FTAG);  in dmu_objset_open_impl()
513 		    primary_cache_changed_cb, os);  in dmu_objset_open_impl()
517 			    secondary_cache_changed_cb, os);  in dmu_objset_open_impl()
523 				    checksum_changed_cb, os);  in dmu_objset_open_impl()
528 				    compression_changed_cb, os);  in dmu_objset_open_impl()
533 				    copies_changed_cb, os);  in dmu_objset_open_impl()
538 				    dedup_changed_cb, os);  in dmu_objset_open_impl()
543 				    logbias_changed_cb, os);  in dmu_objset_open_impl()
548 				    sync_changed_cb, os);  in dmu_objset_open_impl()
554 				    redundant_metadata_changed_cb, os);  in dmu_objset_open_impl()
559 				    recordsize_changed_cb, os);  in dmu_objset_open_impl()
564 				    dnodesize_changed_cb, os);  in dmu_objset_open_impl()
570 				    smallblk_changed_cb, os);  in dmu_objset_open_impl()
574 			dsl_pool_config_exit(dmu_objset_pool(os), FTAG);  in dmu_objset_open_impl()
576 			arc_buf_destroy(os->os_phys_buf, &os->os_phys_buf);  in dmu_objset_open_impl()
577 			kmem_free(os, sizeof (objset_t));  in dmu_objset_open_impl()
582 		os->os_checksum = ZIO_CHECKSUM_FLETCHER_4;  in dmu_objset_open_impl()
583 		os->os_compress = ZIO_COMPRESS_ON;  in dmu_objset_open_impl()
584 		os->os_encrypted = B_FALSE;  in dmu_objset_open_impl()
585 		os->os_copies = spa_max_replication(spa);  in dmu_objset_open_impl()
586 		os->os_dedup_checksum = ZIO_CHECKSUM_OFF;  in dmu_objset_open_impl()
587 		os->os_dedup_verify = B_FALSE;  in dmu_objset_open_impl()
588 		os->os_logbias = ZFS_LOGBIAS_LATENCY;  in dmu_objset_open_impl()
589 		os->os_sync = ZFS_SYNC_STANDARD;  in dmu_objset_open_impl()
590 		os->os_primary_cache = ZFS_CACHE_ALL;  in dmu_objset_open_impl()
591 		os->os_secondary_cache = ZFS_CACHE_ALL;  in dmu_objset_open_impl()
592 		os->os_dnodesize = DNODE_MIN_SIZE;  in dmu_objset_open_impl()
598 	os->os_version = OBJSET_PROP_UNINITIALIZED;  in dmu_objset_open_impl()
599 	os->os_normalization = OBJSET_PROP_UNINITIALIZED;  in dmu_objset_open_impl()
600 	os->os_utf8only = OBJSET_PROP_UNINITIALIZED;  in dmu_objset_open_impl()
601 	os->os_casesensitivity = OBJSET_PROP_UNINITIALIZED;  in dmu_objset_open_impl()
604 		os->os_zil_header = os->os_phys->os_zil_header;  in dmu_objset_open_impl()
605 	os->os_zil = zil_alloc(os, &os->os_zil_header);  in dmu_objset_open_impl()
608 		os->os_dirty_dnodes[i] = multilist_create(sizeof (dnode_t),  in dmu_objset_open_impl()
612 	list_create(&os->os_dnodes, sizeof (dnode_t),  in dmu_objset_open_impl()
614 	list_create(&os->os_downgraded_dbufs, sizeof (dmu_buf_impl_t),  in dmu_objset_open_impl()
617 	mutex_init(&os->os_lock, NULL, MUTEX_DEFAULT, NULL);  in dmu_objset_open_impl()
618 	mutex_init(&os->os_userused_lock, NULL, MUTEX_DEFAULT, NULL);  in dmu_objset_open_impl()
619 	mutex_init(&os->os_obj_lock, NULL, MUTEX_DEFAULT, NULL);  in dmu_objset_open_impl()
620 	mutex_init(&os->os_user_ptr_lock, NULL, MUTEX_DEFAULT, NULL);  in dmu_objset_open_impl()
621 	os->os_obj_next_percpu_len = boot_ncpus;  in dmu_objset_open_impl()
622 	os->os_obj_next_percpu = kmem_zalloc(os->os_obj_next_percpu_len *  in dmu_objset_open_impl()
623 	    sizeof (os->os_obj_next_percpu[0]), KM_SLEEP);  in dmu_objset_open_impl()
625 	dnode_special_open(os, &os->os_phys->os_meta_dnode,  in dmu_objset_open_impl()
626 	    DMU_META_DNODE_OBJECT, &os->os_meta_dnode);  in dmu_objset_open_impl()
627 	if (OBJSET_BUF_HAS_USERUSED(os->os_phys_buf)) {  in dmu_objset_open_impl()
628 		dnode_special_open(os, &os->os_phys->os_userused_dnode,  in dmu_objset_open_impl()
629 		    DMU_USERUSED_OBJECT, &os->os_userused_dnode);  in dmu_objset_open_impl()
630 		dnode_special_open(os, &os->os_phys->os_groupused_dnode,  in dmu_objset_open_impl()
631 		    DMU_GROUPUSED_OBJECT, &os->os_groupused_dnode);  in dmu_objset_open_impl()
632 		if (OBJSET_BUF_HAS_PROJECTUSED(os->os_phys_buf))  in dmu_objset_open_impl()
633 			dnode_special_open(os,  in dmu_objset_open_impl()
634 			    &os->os_phys->os_projectused_dnode,  in dmu_objset_open_impl()
635 			    DMU_PROJECTUSED_OBJECT, &os->os_projectused_dnode);  in dmu_objset_open_impl()
638 	mutex_init(&os->os_upgrade_lock, NULL, MUTEX_DEFAULT, NULL);  in dmu_objset_open_impl()
640 	*osp = os;  in dmu_objset_open_impl()
658 		objset_t *os;  in dmu_objset_from_ds()  local
661 		    ds, dsl_dataset_get_blkptr(ds), &os);  in dmu_objset_from_ds()
667 			ds->ds_objset = os;  in dmu_objset_from_ds()
819 dmu_objset_rele_flags(objset_t *os, boolean_t decrypt, void *tag)  in dmu_objset_rele_flags()  argument
822 	dsl_pool_t *dp = dmu_objset_pool(os);  in dmu_objset_rele_flags()
825 	dsl_dataset_rele_flags(os->os_dsl_dataset, flags, tag);  in dmu_objset_rele_flags()
830 dmu_objset_rele(objset_t *os, void *tag)  in dmu_objset_rele()  argument
832 	dmu_objset_rele_flags(os, B_FALSE, tag);  in dmu_objset_rele()
869 dmu_objset_disown(objset_t *os, boolean_t decrypt, void *tag)  in dmu_objset_disown()  argument
877 	dmu_objset_upgrade_stop(os);  in dmu_objset_disown()
878 	dsl_dataset_disown(os->os_dsl_dataset, flags, tag);  in dmu_objset_disown()
882 dmu_objset_evict_dbufs(objset_t *os)  in dmu_objset_evict_dbufs()  argument
887 	mutex_enter(&os->os_lock);  in dmu_objset_evict_dbufs()
888 	dn = list_head(&os->os_dnodes);  in dmu_objset_evict_dbufs()
896 			list_insert_after(&os->os_dnodes, dn, &dn_marker);  in dmu_objset_evict_dbufs()
897 			mutex_exit(&os->os_lock);  in dmu_objset_evict_dbufs()
902 			mutex_enter(&os->os_lock);  in dmu_objset_evict_dbufs()
903 			dn = list_next(&os->os_dnodes, &dn_marker);  in dmu_objset_evict_dbufs()
904 			list_remove(&os->os_dnodes, &dn_marker);  in dmu_objset_evict_dbufs()
906 			dn = list_next(&os->os_dnodes, dn);  in dmu_objset_evict_dbufs()
909 	mutex_exit(&os->os_lock);  in dmu_objset_evict_dbufs()
911 	if (DMU_USERUSED_DNODE(os) != NULL) {  in dmu_objset_evict_dbufs()
912 		if (DMU_PROJECTUSED_DNODE(os) != NULL)  in dmu_objset_evict_dbufs()
913 			dnode_evict_dbufs(DMU_PROJECTUSED_DNODE(os));  in dmu_objset_evict_dbufs()
914 		dnode_evict_dbufs(DMU_GROUPUSED_DNODE(os));  in dmu_objset_evict_dbufs()
915 		dnode_evict_dbufs(DMU_USERUSED_DNODE(os));  in dmu_objset_evict_dbufs()
917 	dnode_evict_dbufs(DMU_META_DNODE(os));  in dmu_objset_evict_dbufs()
934 dmu_objset_evict(objset_t *os)  in dmu_objset_evict()  argument
936 	dsl_dataset_t *ds = os->os_dsl_dataset;  in dmu_objset_evict()
939 		ASSERT(!dmu_objset_is_dirty(os, t));  in dmu_objset_evict()
942 		dsl_prop_unregister_all(ds, os);  in dmu_objset_evict()
944 	if (os->os_sa)  in dmu_objset_evict()
945 		sa_tear_down(os);  in dmu_objset_evict()
947 	dmu_objset_evict_dbufs(os);  in dmu_objset_evict()
949 	mutex_enter(&os->os_lock);  in dmu_objset_evict()
950 	spa_evicting_os_register(os->os_spa, os);  in dmu_objset_evict()
951 	if (list_is_empty(&os->os_dnodes)) {  in dmu_objset_evict()
952 		mutex_exit(&os->os_lock);  in dmu_objset_evict()
953 		dmu_objset_evict_done(os);  in dmu_objset_evict()
955 		mutex_exit(&os->os_lock);  in dmu_objset_evict()
962 dmu_objset_evict_done(objset_t *os)  in dmu_objset_evict_done()  argument
964 	ASSERT3P(list_head(&os->os_dnodes), ==, NULL);  in dmu_objset_evict_done()
966 	dnode_special_close(&os->os_meta_dnode);  in dmu_objset_evict_done()
967 	if (DMU_USERUSED_DNODE(os)) {  in dmu_objset_evict_done()
968 		if (DMU_PROJECTUSED_DNODE(os))  in dmu_objset_evict_done()
969 			dnode_special_close(&os->os_projectused_dnode);  in dmu_objset_evict_done()
970 		dnode_special_close(&os->os_userused_dnode);  in dmu_objset_evict_done()
971 		dnode_special_close(&os->os_groupused_dnode);  in dmu_objset_evict_done()
973 	zil_free(os->os_zil);  in dmu_objset_evict_done()
975 	arc_buf_destroy(os->os_phys_buf, &os->os_phys_buf);  in dmu_objset_evict_done()
986 	kmem_free(os->os_obj_next_percpu,  in dmu_objset_evict_done()
987 	    os->os_obj_next_percpu_len * sizeof (os->os_obj_next_percpu[0]));  in dmu_objset_evict_done()
989 	mutex_destroy(&os->os_lock);  in dmu_objset_evict_done()
990 	mutex_destroy(&os->os_userused_lock);  in dmu_objset_evict_done()
991 	mutex_destroy(&os->os_obj_lock);  in dmu_objset_evict_done()
992 	mutex_destroy(&os->os_user_ptr_lock);  in dmu_objset_evict_done()
993 	mutex_destroy(&os->os_upgrade_lock);  in dmu_objset_evict_done()
995 		multilist_destroy(os->os_dirty_dnodes[i]);  in dmu_objset_evict_done()
997 	spa_evicting_os_deregister(os->os_spa, os);  in dmu_objset_evict_done()
998 	kmem_free(os, sizeof (objset_t));  in dmu_objset_evict_done()
1002 dmu_objset_snap_cmtime(objset_t *os)  in dmu_objset_snap_cmtime()  argument
1004 	return (dsl_dir_snap_cmtime(os->os_dsl_dataset->ds_dir));  in dmu_objset_snap_cmtime()
1012 	objset_t *os;  in dmu_objset_create_impl_dnstats()  local
1023 		VERIFY0(dmu_objset_from_ds(ds, &os));  in dmu_objset_create_impl_dnstats()
1025 		VERIFY0(dmu_objset_open_impl(spa, NULL, bp, &os));  in dmu_objset_create_impl_dnstats()
1027 	mdn = DMU_META_DNODE(os);  in dmu_objset_create_impl_dnstats()
1071 	os->os_phys->os_type = type;  in dmu_objset_create_impl_dnstats()
1077 	if (dmu_objset_userused_enabled(os) &&  in dmu_objset_create_impl_dnstats()
1078 	    (!os->os_encrypted || !dmu_objset_is_receiving(os))) {  in dmu_objset_create_impl_dnstats()
1079 		os->os_phys->os_flags |= OBJSET_FLAG_USERACCOUNTING_COMPLETE;  in dmu_objset_create_impl_dnstats()
1080 		if (dmu_objset_userobjused_enabled(os)) {  in dmu_objset_create_impl_dnstats()
1084 			os->os_phys->os_flags |=  in dmu_objset_create_impl_dnstats()
1087 		if (dmu_objset_projectquota_enabled(os)) {  in dmu_objset_create_impl_dnstats()
1091 			os->os_phys->os_flags |=  in dmu_objset_create_impl_dnstats()
1094 		os->os_flags = os->os_phys->os_flags;  in dmu_objset_create_impl_dnstats()
1099 	return (os);  in dmu_objset_create_impl_dnstats()
1113 	void (*doca_userfunc)(objset_t *os, void *arg,
1173 	objset_t *os;  in dmu_objset_create_sync()  local
1185 	os = dmu_objset_create_impl(spa, ds, bp, doca->doca_type, tx);  in dmu_objset_create_sync()
1189 		doca->doca_userfunc(os, doca->doca_userarg,  in dmu_objset_create_sync()
1201 	if (os->os_encrypted) {  in dmu_objset_create_sync()
1217 		dmu_objset_do_userquota_updates(os, tx);  in dmu_objset_create_sync()
1376 dmu_objset_remap_indirects_impl(objset_t *os, uint64_t last_removed_txg)  in dmu_objset_remap_indirects_impl()  argument
1380 	while ((error = dmu_object_next(os, &object, B_FALSE, 0)) == 0) {  in dmu_objset_remap_indirects_impl()
1381 		error = dmu_object_remap_indirects(os, object,  in dmu_objset_remap_indirects_impl()
1405 	objset_t *os = NULL;  in dmu_objset_remap_indirects()  local
1410 	error = dmu_objset_hold(fsname, FTAG, &os);  in dmu_objset_remap_indirects()
1414 	dd = dmu_objset_ds(os)->ds_dir;  in dmu_objset_remap_indirects()
1416 	if (!spa_feature_is_enabled(dmu_objset_spa(os),  in dmu_objset_remap_indirects()
1418 		dmu_objset_rele(os, FTAG);  in dmu_objset_remap_indirects()
1422 	if (dsl_dataset_is_snapshot(dmu_objset_ds(os))) {  in dmu_objset_remap_indirects()
1423 		dmu_objset_rele(os, FTAG);  in dmu_objset_remap_indirects()
1430 	last_removed_txg = spa_get_last_removal_txg(dmu_objset_spa(os));  in dmu_objset_remap_indirects()
1432 		dmu_objset_rele(os, FTAG);  in dmu_objset_remap_indirects()
1441 		if (zap_lookup(spa_meta_objset(dmu_objset_spa(os)),  in dmu_objset_remap_indirects()
1445 			dmu_objset_rele(os, FTAG);  in dmu_objset_remap_indirects()
1450 	dsl_dataset_long_hold(dmu_objset_ds(os), FTAG);  in dmu_objset_remap_indirects()
1451 	dsl_pool_rele(dmu_objset_pool(os), FTAG);  in dmu_objset_remap_indirects()
1453 	remap_start_txg = spa_last_synced_txg(dmu_objset_spa(os));  in dmu_objset_remap_indirects()
1454 	error = dmu_objset_remap_indirects_impl(os, last_removed_txg);  in dmu_objset_remap_indirects()
1464 	dsl_dataset_long_rele(dmu_objset_ds(os), FTAG);  in dmu_objset_remap_indirects()
1465 	dsl_dataset_rele(dmu_objset_ds(os), FTAG);  in dmu_objset_remap_indirects()
1487 	objset_t *os = data;  in dmu_objset_upgrade_task_cb()  local
1489 	mutex_enter(&os->os_upgrade_lock);  in dmu_objset_upgrade_task_cb()
1490 	os->os_upgrade_status = EINTR;  in dmu_objset_upgrade_task_cb()
1491 	if (!os->os_upgrade_exit) {  in dmu_objset_upgrade_task_cb()
1494 		mutex_exit(&os->os_upgrade_lock);  in dmu_objset_upgrade_task_cb()
1496 		status = os->os_upgrade_cb(os);  in dmu_objset_upgrade_task_cb()
1498 		mutex_enter(&os->os_upgrade_lock);  in dmu_objset_upgrade_task_cb()
1500 		os->os_upgrade_status = status;  in dmu_objset_upgrade_task_cb()
1502 	os->os_upgrade_exit = B_TRUE;  in dmu_objset_upgrade_task_cb()
1503 	os->os_upgrade_id = 0;  in dmu_objset_upgrade_task_cb()
1504 	mutex_exit(&os->os_upgrade_lock);  in dmu_objset_upgrade_task_cb()
1505 	dsl_dataset_long_rele(dmu_objset_ds(os), upgrade_tag);  in dmu_objset_upgrade_task_cb()
1509 dmu_objset_upgrade(objset_t *os, dmu_objset_upgrade_cb_t cb)  in dmu_objset_upgrade()  argument
1511 	if (os->os_upgrade_id != 0)  in dmu_objset_upgrade()
1514 	ASSERT(dsl_pool_config_held(dmu_objset_pool(os)));  in dmu_objset_upgrade()
1515 	dsl_dataset_long_hold(dmu_objset_ds(os), upgrade_tag);  in dmu_objset_upgrade()
1517 	mutex_enter(&os->os_upgrade_lock);  in dmu_objset_upgrade()
1518 	if (os->os_upgrade_id == 0 && os->os_upgrade_status == 0) {  in dmu_objset_upgrade()
1519 		os->os_upgrade_exit = B_FALSE;  in dmu_objset_upgrade()
1520 		os->os_upgrade_cb = cb;  in dmu_objset_upgrade()
1521 		os->os_upgrade_id = taskq_dispatch(  in dmu_objset_upgrade()
1522 		    os->os_spa->spa_upgrade_taskq,  in dmu_objset_upgrade()
1523 		    dmu_objset_upgrade_task_cb, os, TQ_SLEEP);  in dmu_objset_upgrade()
1524 		if (os->os_upgrade_id == TASKQID_INVALID) {  in dmu_objset_upgrade()
1525 			dsl_dataset_long_rele(dmu_objset_ds(os), upgrade_tag);  in dmu_objset_upgrade()
1526 			os->os_upgrade_status = ENOMEM;  in dmu_objset_upgrade()
1529 		dsl_dataset_long_rele(dmu_objset_ds(os), upgrade_tag);  in dmu_objset_upgrade()
1531 	mutex_exit(&os->os_upgrade_lock);  in dmu_objset_upgrade()
1535 dmu_objset_upgrade_stop(objset_t *os)  in dmu_objset_upgrade_stop()  argument
1537 	mutex_enter(&os->os_upgrade_lock);  in dmu_objset_upgrade_stop()
1538 	os->os_upgrade_exit = B_TRUE;  in dmu_objset_upgrade_stop()
1539 	if (os->os_upgrade_id != 0) {  in dmu_objset_upgrade_stop()
1540 		taskqid_t tid = os->os_upgrade_id;  in dmu_objset_upgrade_stop()
1542 		mutex_exit(&os->os_upgrade_lock);  in dmu_objset_upgrade_stop()
1544 		taskq_wait_id(os->os_spa->spa_upgrade_taskq, tid);  in dmu_objset_upgrade_stop()
1545 		txg_wait_synced(os->os_spa->spa_dsl_pool, 0);  in dmu_objset_upgrade_stop()
1547 		mutex_exit(&os->os_upgrade_lock);  in dmu_objset_upgrade_stop()
1597 	objset_t *os = arg;  in dmu_objset_write_ready()  local
1598 	dnode_phys_t *dnp = &os->os_phys->os_meta_dnode;  in dmu_objset_write_ready()
1615 	if (os->os_dsl_dataset != NULL)  in dmu_objset_write_ready()
1616 		rrw_enter(&os->os_dsl_dataset->ds_bp_rwlock, RW_WRITER, FTAG);  in dmu_objset_write_ready()
1617 	*os->os_rootbp = *bp;  in dmu_objset_write_ready()
1618 	if (os->os_dsl_dataset != NULL)  in dmu_objset_write_ready()
1619 		rrw_exit(&os->os_dsl_dataset->ds_bp_rwlock, FTAG);  in dmu_objset_write_ready()
1628 	objset_t *os = arg;  in dmu_objset_write_done()  local
1633 		dsl_dataset_t *ds = os->os_dsl_dataset;  in dmu_objset_write_done()
1634 		dmu_tx_t *tx = os->os_synctx;  in dmu_objset_write_done()
1667 dmu_objset_sync(objset_t *os, zio_t *pio, dmu_tx_t *tx)  in dmu_objset_sync()  argument
1675 	blkptr_t *blkptr_copy = kmem_alloc(sizeof (*os->os_rootbp), KM_SLEEP);  in dmu_objset_sync()
1676 	*blkptr_copy = *os->os_rootbp;  in dmu_objset_sync()
1678 	dprintf_ds(os->os_dsl_dataset, "txg=%llu\n", tx->tx_txg);  in dmu_objset_sync()
1682 	os->os_synctx = tx;  in dmu_objset_sync()
1684 	if (os->os_dsl_dataset == NULL) {  in dmu_objset_sync()
1690 		os->os_copies = spa_max_replication(os->os_spa);  in dmu_objset_sync()
1696 	SET_BOOKMARK(&zb, os->os_dsl_dataset ?  in dmu_objset_sync()
1697 	    os->os_dsl_dataset->ds_object : DMU_META_OBJSET,  in dmu_objset_sync()
1699 	arc_release(os->os_phys_buf, &os->os_phys_buf);  in dmu_objset_sync()
1701 	dmu_write_policy(os, NULL, 0, 0, &zp);  in dmu_objset_sync()
1708 	if (os->os_raw_receive ||  in dmu_objset_sync()
1709 	    os->os_next_write_raw[tx->tx_txg & TXG_MASK]) {  in dmu_objset_sync()
1710 		ASSERT(os->os_encrypted);  in dmu_objset_sync()
1711 		arc_convert_to_raw(os->os_phys_buf,  in dmu_objset_sync()
1712 		    os->os_dsl_dataset->ds_object, ZFS_HOST_BYTEORDER,  in dmu_objset_sync()
1716 	zio = arc_write(pio, os->os_spa, tx->tx_txg,  in dmu_objset_sync()
1717 	    blkptr_copy, os->os_phys_buf, DMU_OS_IS_L2CACHEABLE(os),  in dmu_objset_sync()
1719 	    os, ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb);  in dmu_objset_sync()
1724 	DMU_META_DNODE(os)->dn_zio = zio;  in dmu_objset_sync()
1725 	dnode_sync(DMU_META_DNODE(os), tx);  in dmu_objset_sync()
1727 	os->os_phys->os_flags = os->os_flags;  in dmu_objset_sync()
1729 	if (DMU_USERUSED_DNODE(os) &&  in dmu_objset_sync()
1730 	    DMU_USERUSED_DNODE(os)->dn_type != DMU_OT_NONE) {  in dmu_objset_sync()
1731 		DMU_USERUSED_DNODE(os)->dn_zio = zio;  in dmu_objset_sync()
1732 		dnode_sync(DMU_USERUSED_DNODE(os), tx);  in dmu_objset_sync()
1733 		DMU_GROUPUSED_DNODE(os)->dn_zio = zio;  in dmu_objset_sync()
1734 		dnode_sync(DMU_GROUPUSED_DNODE(os), tx);  in dmu_objset_sync()
1737 	if (DMU_PROJECTUSED_DNODE(os) &&  in dmu_objset_sync()
1738 	    DMU_PROJECTUSED_DNODE(os)->dn_type != DMU_OT_NONE) {  in dmu_objset_sync()
1739 		DMU_PROJECTUSED_DNODE(os)->dn_zio = zio;  in dmu_objset_sync()
1740 		dnode_sync(DMU_PROJECTUSED_DNODE(os), tx);  in dmu_objset_sync()
1745 	if (dmu_objset_userused_enabled(os) &&  in dmu_objset_sync()
1746 	    (!os->os_encrypted || !dmu_objset_is_receiving(os))) {  in dmu_objset_sync()
1752 		if (os->os_synced_dnodes == NULL) {  in dmu_objset_sync()
1753 			os->os_synced_dnodes =  in dmu_objset_sync()
1758 			ASSERT3U(os->os_synced_dnodes->ml_offset, ==,  in dmu_objset_sync()
1764 	    i < multilist_get_num_sublists(os->os_dirty_dnodes[txgoff]); i++) {  in dmu_objset_sync()
1766 		sda->sda_list = os->os_dirty_dnodes[txgoff];  in dmu_objset_sync()
1769 		(void) taskq_dispatch(dmu_objset_pool(os)->dp_sync_taskq,  in dmu_objset_sync()
1773 	taskq_wait(dmu_objset_pool(os)->dp_sync_taskq);  in dmu_objset_sync()
1775 	list = &DMU_META_DNODE(os)->dn_dirty_records[txgoff];  in dmu_objset_sync()
1784 	if (os->os_freed_dnodes >= dmu_rescan_dnode_threshold) {  in dmu_objset_sync()
1785 		os->os_rescan_dnodes = B_TRUE;  in dmu_objset_sync()
1786 		os->os_freed_dnodes = 0;  in dmu_objset_sync()
1792 	zil_sync(os->os_zil, tx);  in dmu_objset_sync()
1793 	os->os_phys->os_zil_header = os->os_zil_header;  in dmu_objset_sync()
1798 dmu_objset_is_dirty(objset_t *os, uint64_t txg)  in dmu_objset_is_dirty()  argument
1800 	return (!multilist_is_empty(os->os_dirty_dnodes[txg & TXG_MASK]));  in dmu_objset_is_dirty()
1812 dmu_objset_userused_enabled(objset_t *os)  in dmu_objset_userused_enabled()  argument
1814 	return (spa_version(os->os_spa) >= SPA_VERSION_USERSPACE &&  in dmu_objset_userused_enabled()
1815 	    used_cbs[os->os_phys->os_type] != NULL &&  in dmu_objset_userused_enabled()
1816 	    DMU_USERUSED_DNODE(os) != NULL);  in dmu_objset_userused_enabled()
1820 dmu_objset_userobjused_enabled(objset_t *os)  in dmu_objset_userobjused_enabled()  argument
1822 	return (dmu_objset_userused_enabled(os) &&  in dmu_objset_userobjused_enabled()
1823 	    spa_feature_is_enabled(os->os_spa, SPA_FEATURE_USEROBJ_ACCOUNTING));  in dmu_objset_userobjused_enabled()
1827 dmu_objset_projectquota_enabled(objset_t *os)  in dmu_objset_projectquota_enabled()  argument
1829 	return (used_cbs[os->os_phys->os_type] != NULL &&  in dmu_objset_projectquota_enabled()
1830 	    DMU_PROJECTUSED_DNODE(os) != NULL &&  in dmu_objset_projectquota_enabled()
1831 	    spa_feature_is_enabled(os->os_spa, SPA_FEATURE_PROJECT_QUOTA));  in dmu_objset_projectquota_enabled()
1864 do_userquota_cacheflush(objset_t *os, userquota_cache_t *cache, dmu_tx_t *tx)  in do_userquota_cacheflush()  argument
1879 		mutex_enter(&os->os_userused_lock);  in do_userquota_cacheflush()
1880 		VERIFY0(zap_increment(os, DMU_USERUSED_OBJECT,  in do_userquota_cacheflush()
1882 		mutex_exit(&os->os_userused_lock);  in do_userquota_cacheflush()
1890 		mutex_enter(&os->os_userused_lock);  in do_userquota_cacheflush()
1891 		VERIFY0(zap_increment(os, DMU_GROUPUSED_OBJECT,  in do_userquota_cacheflush()
1893 		mutex_exit(&os->os_userused_lock);  in do_userquota_cacheflush()
1898 	if (dmu_objset_projectquota_enabled(os)) {  in do_userquota_cacheflush()
1902 			mutex_enter(&os->os_userused_lock);  in do_userquota_cacheflush()
1903 			VERIFY0(zap_increment(os, DMU_PROJECTUSED_OBJECT,  in do_userquota_cacheflush()
1905 			mutex_exit(&os->os_userused_lock);  in do_userquota_cacheflush()
1934 do_userquota_update(objset_t *os, userquota_cache_t *cache, uint64_t used,  in do_userquota_update()  argument
1951 		if (dmu_objset_projectquota_enabled(os)) {  in do_userquota_update()
1960 do_userobjquota_update(objset_t *os, userquota_cache_t *cache, uint64_t flags,  in do_userobjquota_update()  argument
1975 		if (dmu_objset_projectquota_enabled(os)) {  in do_userobjquota_update()
1994 	objset_t *os = uua->uua_os;  in userquota_updates_task()  local
2000 	    multilist_sublist_lock(os->os_synced_dnodes, uua->uua_sublist_idx);  in userquota_updates_task()
2003 	    dmu_objset_userused_enabled(os));  in userquota_updates_task()
2008 	if (dmu_objset_projectquota_enabled(os))  in userquota_updates_task()
2023 			do_userquota_update(os, &cache, dn->dn_oldused,  in userquota_updates_task()
2026 			do_userobjquota_update(os, &cache, dn->dn_oldflags,  in userquota_updates_task()
2031 			do_userquota_update(os, &cache,  in userquota_updates_task()
2035 			do_userobjquota_update(os, &cache,  in userquota_updates_task()
2054 		if (dn->dn_dirty_txg == spa_syncing_txg(os->os_spa))  in userquota_updates_task()
2059 		dnode_rele(dn, os->os_synced_dnodes);  in userquota_updates_task()
2061 	do_userquota_cacheflush(os, &cache, tx);  in userquota_updates_task()
2067 dmu_objset_do_userquota_updates(objset_t *os, dmu_tx_t *tx)  in dmu_objset_do_userquota_updates()  argument
2069 	if (!dmu_objset_userused_enabled(os))  in dmu_objset_do_userquota_updates()
2079 	if (os->os_encrypted && dmu_objset_is_receiving(os))  in dmu_objset_do_userquota_updates()
2082 	if (tx->tx_txg <= os->os_spa->spa_claim_max_txg)  in dmu_objset_do_userquota_updates()
2086 	if (DMU_USERUSED_DNODE(os)->dn_type == DMU_OT_NONE) {  in dmu_objset_do_userquota_updates()
2087 		VERIFY0(zap_create_claim(os,  in dmu_objset_do_userquota_updates()
2090 		VERIFY0(zap_create_claim(os,  in dmu_objset_do_userquota_updates()
2095 	if (dmu_objset_projectquota_enabled(os) &&  in dmu_objset_do_userquota_updates()
2096 	    DMU_PROJECTUSED_DNODE(os)->dn_type == DMU_OT_NONE) {  in dmu_objset_do_userquota_updates()
2097 		VERIFY0(zap_create_claim(os, DMU_PROJECTUSED_OBJECT,  in dmu_objset_do_userquota_updates()
2102 	    i < multilist_get_num_sublists(os->os_synced_dnodes); i++) {  in dmu_objset_do_userquota_updates()
2105 		uua->uua_os = os;  in dmu_objset_do_userquota_updates()
2109 		(void) taskq_dispatch(dmu_objset_pool(os)->dp_sync_taskq,  in dmu_objset_do_userquota_updates()
2158 	objset_t *os = dn->dn_objset;  in dmu_objset_userquota_get_ids()  local
2180 	if (os->os_encrypted && dmu_objset_is_receiving(os))  in dmu_objset_userquota_get_ids()
2232 	error = used_cbs[os->os_phys->os_type](dn->dn_bonustype, data,  in dmu_objset_userquota_get_ids()
2275 dmu_objset_userspace_present(objset_t *os)  in dmu_objset_userspace_present()  argument
2277 	return (os->os_phys->os_flags &  in dmu_objset_userspace_present()
2282 dmu_objset_userobjspace_present(objset_t *os)  in dmu_objset_userobjspace_present()  argument
2284 	return (os->os_phys->os_flags &  in dmu_objset_userobjspace_present()
2289 dmu_objset_projectquota_present(objset_t *os)  in dmu_objset_projectquota_present()  argument
2291 	return (os->os_phys->os_flags &  in dmu_objset_projectquota_present()
2296 dmu_objset_space_upgrade(objset_t *os)  in dmu_objset_space_upgrade()  argument
2309 	for (obj = 0; err == 0; err = dmu_object_next(os, &obj, FALSE, 0)) {  in dmu_objset_space_upgrade()
2314 		mutex_enter(&os->os_upgrade_lock);  in dmu_objset_space_upgrade()
2315 		if (os->os_upgrade_exit)  in dmu_objset_space_upgrade()
2317 		mutex_exit(&os->os_upgrade_lock);  in dmu_objset_space_upgrade()
2330 		objerr = dmu_bonus_hold(os, obj, FTAG, &db);  in dmu_objset_space_upgrade()
2333 		tx = dmu_tx_create(os);  in dmu_objset_space_upgrade()
2349 dmu_objset_userspace_upgrade(objset_t *os)  in dmu_objset_userspace_upgrade()  argument
2353 	if (dmu_objset_userspace_present(os))  in dmu_objset_userspace_upgrade()
2355 	if (dmu_objset_is_snapshot(os))  in dmu_objset_userspace_upgrade()
2357 	if (!dmu_objset_userused_enabled(os))  in dmu_objset_userspace_upgrade()
2360 	err = dmu_objset_space_upgrade(os);  in dmu_objset_userspace_upgrade()
2364 	os->os_flags |= OBJSET_FLAG_USERACCOUNTING_COMPLETE;  in dmu_objset_userspace_upgrade()
2365 	txg_wait_synced(dmu_objset_pool(os), 0);  in dmu_objset_userspace_upgrade()
2370 dmu_objset_id_quota_upgrade_cb(objset_t *os)  in dmu_objset_id_quota_upgrade_cb()  argument
2374 	if (dmu_objset_userobjspace_present(os) &&  in dmu_objset_id_quota_upgrade_cb()
2375 	    dmu_objset_projectquota_present(os))  in dmu_objset_id_quota_upgrade_cb()
2377 	if (dmu_objset_is_snapshot(os))  in dmu_objset_id_quota_upgrade_cb()
2379 	if (!dmu_objset_userobjused_enabled(os))  in dmu_objset_id_quota_upgrade_cb()
2381 	if (!dmu_objset_projectquota_enabled(os) &&  in dmu_objset_id_quota_upgrade_cb()
2382 	    dmu_objset_userobjspace_present(os))  in dmu_objset_id_quota_upgrade_cb()
2385 	dmu_objset_ds(os)->ds_feature_activation[  in dmu_objset_id_quota_upgrade_cb()
2387 	if (dmu_objset_projectquota_enabled(os))  in dmu_objset_id_quota_upgrade_cb()
2388 		dmu_objset_ds(os)->ds_feature_activation[  in dmu_objset_id_quota_upgrade_cb()
2391 	err = dmu_objset_space_upgrade(os);  in dmu_objset_id_quota_upgrade_cb()
2395 	os->os_flags |= OBJSET_FLAG_USEROBJACCOUNTING_COMPLETE;  in dmu_objset_id_quota_upgrade_cb()
2396 	if (dmu_objset_projectquota_enabled(os))  in dmu_objset_id_quota_upgrade_cb()
2397 		os->os_flags |= OBJSET_FLAG_PROJECTQUOTA_COMPLETE;  in dmu_objset_id_quota_upgrade_cb()
2399 	txg_wait_synced(dmu_objset_pool(os), 0);  in dmu_objset_id_quota_upgrade_cb()
2404 dmu_objset_id_quota_upgrade(objset_t *os)  in dmu_objset_id_quota_upgrade()  argument
2406 	dmu_objset_upgrade(os, dmu_objset_id_quota_upgrade_cb);  in dmu_objset_id_quota_upgrade()
2410 dmu_objset_userobjspace_upgradable(objset_t *os)  in dmu_objset_userobjspace_upgradable()  argument
2412 	return (dmu_objset_type(os) == DMU_OST_ZFS &&  in dmu_objset_userobjspace_upgradable()
2413 	    !dmu_objset_is_snapshot(os) &&  in dmu_objset_userobjspace_upgradable()
2414 	    dmu_objset_userobjused_enabled(os) &&  in dmu_objset_userobjspace_upgradable()
2415 	    !dmu_objset_userobjspace_present(os) &&  in dmu_objset_userobjspace_upgradable()
2416 	    spa_writeable(dmu_objset_spa(os)));  in dmu_objset_userobjspace_upgradable()
2420 dmu_objset_projectquota_upgradable(objset_t *os)  in dmu_objset_projectquota_upgradable()  argument
2422 	return (dmu_objset_type(os) == DMU_OST_ZFS &&  in dmu_objset_projectquota_upgradable()
2423 	    !dmu_objset_is_snapshot(os) &&  in dmu_objset_projectquota_upgradable()
2424 	    dmu_objset_projectquota_enabled(os) &&  in dmu_objset_projectquota_upgradable()
2425 	    !dmu_objset_projectquota_present(os) &&  in dmu_objset_projectquota_upgradable()
2426 	    spa_writeable(dmu_objset_spa(os)));  in dmu_objset_projectquota_upgradable()
2430 dmu_objset_space(objset_t *os, uint64_t *refdbytesp, uint64_t *availbytesp,  in dmu_objset_space()  argument
2433 	dsl_dataset_space(os->os_dsl_dataset, refdbytesp, availbytesp,  in dmu_objset_space()
2438 dmu_objset_fsid_guid(objset_t *os)  in dmu_objset_fsid_guid()  argument
2440 	return (dsl_dataset_fsid_guid(os->os_dsl_dataset));  in dmu_objset_fsid_guid()
2444 dmu_objset_fast_stat(objset_t *os, dmu_objset_stats_t *stat)  in dmu_objset_fast_stat()  argument
2446 	stat->dds_type = os->os_phys->os_type;  in dmu_objset_fast_stat()
2447 	if (os->os_dsl_dataset)  in dmu_objset_fast_stat()
2448 		dsl_dataset_fast_stat(os->os_dsl_dataset, stat);  in dmu_objset_fast_stat()
2452 dmu_objset_stats(objset_t *os, nvlist_t *nv)  in dmu_objset_stats()  argument
2454 	ASSERT(os->os_dsl_dataset ||  in dmu_objset_stats()
2455 	    os->os_phys->os_type == DMU_OST_META);  in dmu_objset_stats()
2457 	if (os->os_dsl_dataset != NULL)  in dmu_objset_stats()
2458 		dsl_dataset_stats(os->os_dsl_dataset, nv);  in dmu_objset_stats()
2461 	    os->os_phys->os_type);  in dmu_objset_stats()
2463 	    dmu_objset_userspace_present(os));  in dmu_objset_stats()
2467 dmu_objset_is_snapshot(objset_t *os)  in dmu_objset_is_snapshot()  argument
2469 	if (os->os_dsl_dataset != NULL)  in dmu_objset_is_snapshot()
2470 		return (os->os_dsl_dataset->ds_is_snapshot);  in dmu_objset_is_snapshot()
2476 dmu_snapshot_realname(objset_t *os, char *name, char *real, int maxlen,  in dmu_snapshot_realname()  argument
2479 	dsl_dataset_t *ds = os->os_dsl_dataset;  in dmu_snapshot_realname()
2491 dmu_snapshot_list_next(objset_t *os, int namelen, char *name,  in dmu_snapshot_list_next()  argument
2494 	dsl_dataset_t *ds = os->os_dsl_dataset;  in dmu_snapshot_list_next()
2498 	ASSERT(dsl_pool_config_held(dmu_objset_pool(os)));  in dmu_snapshot_list_next()
2530 dmu_dir_list_next(objset_t *os, int namelen, char *name,  in dmu_dir_list_next()  argument
2533 	dsl_dir_t *dd = os->os_dsl_dataset->ds_dir;  in dmu_dir_list_next()
2538 	if (os->os_dsl_dataset->ds_object !=  in dmu_dir_list_next()
2939 dmu_objset_incompatible_encryption_version(objset_t *os)  in dmu_objset_incompatible_encryption_version()  argument
2942 	    os->os_dsl_dataset->ds_dir));  in dmu_objset_incompatible_encryption_version()
2946 dmu_objset_set_user(objset_t *os, void *user_ptr)  in dmu_objset_set_user()  argument
2948 	ASSERT(MUTEX_HELD(&os->os_user_ptr_lock));  in dmu_objset_set_user()
2949 	os->os_user_ptr = user_ptr;  in dmu_objset_set_user()
2953 dmu_objset_get_user(objset_t *os)  in dmu_objset_get_user()  argument
2955 	ASSERT(MUTEX_HELD(&os->os_user_ptr_lock));  in dmu_objset_get_user()
2956 	return (os->os_user_ptr);  in dmu_objset_get_user()
2981 dmu_objset_willuse_space(objset_t *os, int64_t space, dmu_tx_t *tx)  in dmu_objset_willuse_space()  argument
2983 	dsl_dataset_t *ds = os->os_dsl_dataset;  in dmu_objset_willuse_space()
2984 	int64_t aspace = spa_get_worst_case_asize(os->os_spa, space);  in dmu_objset_willuse_space()