135 static int sa_build_index(sa_handle_t *hdl, sa_buf_type_t buftype);
142 static int sa_modify_attrs(sa_handle_t *hdl, sa_attr_type_t newattr,
221 	sa_handle_t *hdl = buf;  in sa_cache_constructor()  local
223 	mutex_init(&hdl->sa_lock, NULL, MUTEX_DEFAULT, NULL);  in sa_cache_constructor()
231 	sa_handle_t *hdl = buf;  in sa_cache_destructor()  local
232 	mutex_destroy(&hdl->sa_lock);  in sa_cache_destructor()
301 sa_get_spill(sa_handle_t *hdl)  in sa_get_spill()  argument
304 	if (hdl->sa_spill == NULL) {  in sa_get_spill()
305 		if ((rc = dmu_spill_hold_existing(hdl->sa_bonus, NULL,  in sa_get_spill()
306 		    &hdl->sa_spill)) == 0)  in sa_get_spill()
307 			VERIFY0(sa_build_index(hdl, SA_SPILL));  in sa_get_spill()
322 sa_attr_op(sa_handle_t *hdl, sa_bulk_attr_t *bulk, int count,  in sa_attr_op()  argument
325 	sa_os_t *sa = hdl->sa_os->os_sa;  in sa_attr_op()
334 		ASSERT(bulk[i].sa_attr <= hdl->sa_os->os_sa->sa_num_attrs);  in sa_attr_op()
339 		if (hdl->sa_bonus_tab && TOC_ATTR_PRESENT(  in sa_attr_op()
340 		    hdl->sa_bonus_tab->sa_idx_tab[bulk[i].sa_attr])) {  in sa_attr_op()
341 			SA_ATTR_INFO(sa, hdl->sa_bonus_tab,  in sa_attr_op()
342 			    SA_GET_HDR(hdl, SA_BONUS),  in sa_attr_op()
343 			    bulk[i].sa_attr, bulk[i], SA_BONUS, hdl);  in sa_attr_op()
345 				dmu_buf_will_dirty(hdl->sa_bonus, tx);  in sa_attr_op()
350 		    ((error = sa_get_spill(hdl)) == 0)) {  in sa_attr_op()
352 			    hdl->sa_spill_tab->sa_idx_tab[bulk[i].sa_attr])) {  in sa_attr_op()
353 				SA_ATTR_INFO(sa, hdl->sa_spill_tab,  in sa_attr_op()
354 				    SA_GET_HDR(hdl, SA_SPILL),  in sa_attr_op()
355 				    bulk[i].sa_attr, bulk[i], SA_SPILL, hdl);  in sa_attr_op()
358 					dmu_buf_will_dirty(hdl->sa_spill, tx);  in sa_attr_op()
387 				error = sa_modify_attrs(hdl, bulk[i].sa_attr,  in sa_attr_op()
391 				error = sa_modify_attrs(hdl, bulk[i].sa_attr,  in sa_attr_op()
493 sa_resize_spill(sa_handle_t *hdl, uint32_t size, dmu_tx_t *tx)  in sa_resize_spill()  argument
507 	error = dbuf_spill_set_blksz(hdl->sa_spill, blocksize, tx);  in sa_resize_spill()
660 sa_build_layouts(sa_handle_t *hdl, sa_bulk_attr_t *attr_desc, int attr_count,  in sa_build_layouts()  argument
663 	sa_os_t *sa = hdl->sa_os->os_sa;  in sa_build_layouts()
682 	dmu_buf_will_dirty(hdl->sa_bonus, tx);  in sa_build_layouts()
683 	bonustype = SA_BONUSTYPE_FROM_DB(hdl->sa_bonus);  in sa_build_layouts()
684 	dmu_object_dnsize_from_db(hdl->sa_bonus, &dnodesize);  in sa_build_layouts()
688 	hdrsize = sa_find_sizes(sa, attr_desc, attr_count, hdl->sa_bonus,  in sa_build_layouts()
694 	VERIFY0(dmu_set_bonus(hdl->sa_bonus, spilling ?  in sa_build_layouts()
705 		if (hdl->sa_spill == NULL) {  in sa_build_layouts()
706 			VERIFY0(dmu_spill_hold_by_bonus(hdl->sa_bonus,  in sa_build_layouts()
707 			    DB_RF_MUST_SUCCEED, NULL, &hdl->sa_spill));  in sa_build_layouts()
709 		dmu_buf_will_dirty(hdl->sa_spill, tx);  in sa_build_layouts()
712 		    attr_count - spill_idx, hdl->sa_spill, SA_SPILL,  in sa_build_layouts()
713 		    hdl->sa_spill->db_size, &i, &spill_used, &dummy);  in sa_build_layouts()
719 		    hdl->sa_spill->db_size)  in sa_build_layouts()
720 			VERIFY0(sa_resize_spill(hdl,  in sa_build_layouts()
725 	data_start = (void *)((uintptr_t)hdl->sa_bonus->db_data + hdrsize);  in sa_build_layouts()
726 	sahdr = (sa_hdr_phys_t *)hdl->sa_bonus->db_data;  in sa_build_layouts()
745 				sa_find_layout(hdl->sa_os, hash, attrs_start,  in sa_build_layouts()
754 			sahdr = (sa_hdr_phys_t *)hdl->sa_spill->db_data;  in sa_build_layouts()
774 	sa_find_layout(hdl->sa_os, hash, attrs_start, lot_count, tx, &lot);  in sa_build_layouts()
789 	if (hdl->sa_bonus_tab) {  in sa_build_layouts()
790 		sa_idx_tab_rele(hdl->sa_os, hdl->sa_bonus_tab);  in sa_build_layouts()
791 		hdl->sa_bonus_tab = NULL;  in sa_build_layouts()
794 		VERIFY0(sa_build_index(hdl, SA_BONUS));  in sa_build_layouts()
795 	if (hdl->sa_spill) {  in sa_build_layouts()
796 		sa_idx_tab_rele(hdl->sa_os, hdl->sa_spill_tab);  in sa_build_layouts()
801 			dmu_buf_rele(hdl->sa_spill, NULL);  in sa_build_layouts()
802 			hdl->sa_spill = NULL;  in sa_build_layouts()
803 			hdl->sa_spill_tab = NULL;  in sa_build_layouts()
804 			VERIFY0(dmu_rm_spill(hdl->sa_os,  in sa_build_layouts()
805 			    sa_handle_object(hdl), tx));  in sa_build_layouts()
807 			VERIFY0(sa_build_index(hdl, SA_SPILL));  in sa_build_layouts()
1232 	sa_handle_t *hdl = userp;  in sa_byteswap_cb()  local
1233 	sa_os_t *sa = hdl->sa_os->os_sa;  in sa_byteswap_cb()
1239 sa_byteswap(sa_handle_t *hdl, sa_buf_type_t buftype)  in sa_byteswap()  argument
1241 	sa_hdr_phys_t *sa_hdr_phys = SA_GET_HDR(hdl, buftype);  in sa_byteswap()
1245 	sa_os_t *sa __maybe_unused = hdl->sa_os->os_sa;  in sa_byteswap()
1251 	db = SA_GET_DB(hdl, buftype);  in sa_byteswap()
1272 	sa_attr_iter(hdl->sa_os, sa_hdr_phys, DMU_OT_SA,  in sa_byteswap()
1273 	    sa_byteswap_cb, NULL, hdl);  in sa_byteswap()
1276 		arc_buf_freeze(((dmu_buf_impl_t *)hdl->sa_spill)->db_buf);  in sa_byteswap()
1280 sa_build_index(sa_handle_t *hdl, sa_buf_type_t buftype)  in sa_build_index()  argument
1283 	dmu_buf_impl_t *db = SA_GET_DB(hdl, buftype);  in sa_build_index()
1285 	sa_os_t *sa = hdl->sa_os->os_sa;  in sa_build_index()
1288 	sa_hdr_phys = SA_GET_HDR(hdl, buftype);  in sa_build_index()
1304 		sa_byteswap(hdl, buftype);  in sa_build_index()
1307 	idx_tab = sa_find_idx_tab(hdl->sa_os, bonustype, sa_hdr_phys);  in sa_build_index()
1310 		hdl->sa_bonus_tab = idx_tab;  in sa_build_index()
1312 		hdl->sa_spill_tab = idx_tab;  in sa_build_index()
1359 sa_spill_rele(sa_handle_t *hdl)  in sa_spill_rele()  argument
1361 	mutex_enter(&hdl->sa_lock);  in sa_spill_rele()
1362 	if (hdl->sa_spill) {  in sa_spill_rele()
1363 		sa_idx_tab_rele(hdl->sa_os, hdl->sa_spill_tab);  in sa_spill_rele()
1364 		dmu_buf_rele(hdl->sa_spill, NULL);  in sa_spill_rele()
1365 		hdl->sa_spill = NULL;  in sa_spill_rele()
1366 		hdl->sa_spill_tab = NULL;  in sa_spill_rele()
1368 	mutex_exit(&hdl->sa_lock);  in sa_spill_rele()
1372 sa_handle_destroy(sa_handle_t *hdl)  in sa_handle_destroy()  argument
1374 	dmu_buf_t *db = hdl->sa_bonus;  in sa_handle_destroy()
1376 	mutex_enter(&hdl->sa_lock);  in sa_handle_destroy()
1377 	(void) dmu_buf_remove_user(db, &hdl->sa_dbu);  in sa_handle_destroy()
1379 	if (hdl->sa_bonus_tab)  in sa_handle_destroy()
1380 		sa_idx_tab_rele(hdl->sa_os, hdl->sa_bonus_tab);  in sa_handle_destroy()
1382 	if (hdl->sa_spill_tab)  in sa_handle_destroy()
1383 		sa_idx_tab_rele(hdl->sa_os, hdl->sa_spill_tab);  in sa_handle_destroy()
1385 	dmu_buf_rele(hdl->sa_bonus, NULL);  in sa_handle_destroy()
1387 	if (hdl->sa_spill)  in sa_handle_destroy()
1388 		dmu_buf_rele(hdl->sa_spill, NULL);  in sa_handle_destroy()
1389 	mutex_exit(&hdl->sa_lock);  in sa_handle_destroy()
1391 	kmem_cache_free(sa_cache, hdl);  in sa_handle_destroy()
1471 sa_lookup_impl(sa_handle_t *hdl, sa_bulk_attr_t *bulk, int count)  in sa_lookup_impl()  argument
1473 	ASSERT(hdl);  in sa_lookup_impl()
1474 	ASSERT(MUTEX_HELD(&hdl->sa_lock));  in sa_lookup_impl()
1475 	return (sa_attr_op(hdl, bulk, count, SA_LOOKUP, NULL));  in sa_lookup_impl()
1479 sa_lookup_locked(sa_handle_t *hdl, sa_attr_type_t attr, void *buf,  in sa_lookup_locked()  argument
1492 	ASSERT(hdl);  in sa_lookup_locked()
1493 	error = sa_lookup_impl(hdl, &bulk, 1);  in sa_lookup_locked()
1498 sa_lookup(sa_handle_t *hdl, sa_attr_type_t attr, void *buf, uint32_t buflen)  in sa_lookup()  argument
1502 	mutex_enter(&hdl->sa_lock);  in sa_lookup()
1503 	error = sa_lookup_locked(hdl, attr, buf, buflen);  in sa_lookup()
1504 	mutex_exit(&hdl->sa_lock);  in sa_lookup()
1514 sa_size_locked(sa_handle_t *hdl, sa_attr_type_t attr, int *size)  in sa_size_locked()  argument
1523 	ASSERT(hdl);  in sa_size_locked()
1524 	ASSERT(MUTEX_HELD(&hdl->sa_lock));  in sa_size_locked()
1525 	if ((error = sa_attr_op(hdl, &bulk, 1, SA_LOOKUP, NULL)) != 0) {  in sa_size_locked()
1534 sa_size(sa_handle_t *hdl, sa_attr_type_t attr, int *size)  in sa_size()  argument
1538 	mutex_enter(&hdl->sa_lock);  in sa_size()
1539 	error = sa_size_locked(hdl, attr, size);  in sa_size()
1540 	mutex_exit(&hdl->sa_lock);  in sa_size()
1547 sa_lookup_uio(sa_handle_t *hdl, sa_attr_type_t attr, zfs_uio_t *uio)  in sa_lookup_uio()  argument
1556 	ASSERT(hdl);  in sa_lookup_uio()
1558 	mutex_enter(&hdl->sa_lock);  in sa_lookup_uio()
1559 	if ((error = sa_attr_op(hdl, &bulk, 1, SA_LOOKUP, NULL)) == 0) {  in sa_lookup_uio()
1563 	mutex_exit(&hdl->sa_lock);  in sa_lookup_uio()
1574 sa_add_projid(sa_handle_t *hdl, dmu_tx_t *tx, uint64_t projid)  in sa_add_projid()  argument
1576 	znode_t *zp = sa_get_userdata(hdl);  in sa_add_projid()
1577 	dmu_buf_t *db = sa_get_db(hdl);  in sa_add_projid()
1601 	mutex_enter(&hdl->sa_lock);  in sa_add_projid()
1604 	err = sa_lookup_locked(hdl, SA_ZPL_PROJID(zfsvfs), &projid,  in sa_add_projid()
1661 	err = sa_bulk_lookup_locked(hdl, bulk, count);  in sa_add_projid()
1665 	err = sa_lookup_locked(hdl, SA_ZPL_XATTR(zfsvfs), &xattr, 8);  in sa_add_projid()
1669 	err = sa_size_locked(hdl, SA_ZPL_DXATTR(zfsvfs), &dxattr_size);  in sa_add_projid()
1674 		err = sa_lookup_locked(hdl, SA_ZPL_DXATTR(zfsvfs), dxattr_obj,  in sa_add_projid()
1737 	VERIFY0(sa_replace_all_by_template_locked(hdl, attrs, count, tx));  in sa_add_projid()
1747 	mutex_exit(&hdl->sa_lock);  in sa_add_projid()
1837 sa_attr_register_sync(sa_handle_t *hdl, dmu_tx_t *tx)  in sa_attr_register_sync()  argument
1840 	sa_os_t *sa = hdl->sa_os->os_sa;  in sa_attr_register_sync()
1852 		sa->sa_reg_attr_obj = zap_create_link(hdl->sa_os,  in sa_attr_register_sync()
1861 		VERIFY0(zap_update(hdl->sa_os, sa->sa_reg_attr_obj,  in sa_attr_register_sync()
1879 sa_replace_all_by_template_locked(sa_handle_t *hdl, sa_bulk_attr_t *attr_desc,  in sa_replace_all_by_template_locked()  argument
1882 	sa_os_t *sa = hdl->sa_os->os_sa;  in sa_replace_all_by_template_locked()
1885 		sa_attr_register_sync(hdl, tx);  in sa_replace_all_by_template_locked()
1886 	return (sa_build_layouts(hdl, attr_desc, attr_count, tx));  in sa_replace_all_by_template_locked()
1890 sa_replace_all_by_template(sa_handle_t *hdl, sa_bulk_attr_t *attr_desc,  in sa_replace_all_by_template()  argument
1895 	mutex_enter(&hdl->sa_lock);  in sa_replace_all_by_template()
1896 	error = sa_replace_all_by_template_locked(hdl, attr_desc,  in sa_replace_all_by_template()
1898 	mutex_exit(&hdl->sa_lock);  in sa_replace_all_by_template()
1910 sa_modify_attrs(sa_handle_t *hdl, sa_attr_type_t newattr,  in sa_modify_attrs()  argument
1914 	sa_os_t *sa = hdl->sa_os->os_sa;  in sa_modify_attrs()
1915 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)hdl->sa_bonus;  in sa_modify_attrs()
1930 	ASSERT(MUTEX_HELD(&hdl->sa_lock));  in sa_modify_attrs()
1936 		bonus_data_size = hdl->sa_bonus->db_size;  in sa_modify_attrs()
1938 		memcpy(old_data[0], hdl->sa_bonus->db_data,  in sa_modify_attrs()
1939 		    hdl->sa_bonus->db_size);  in sa_modify_attrs()
1940 		bonus_attr_count = hdl->sa_bonus_tab->sa_layout->lot_attr_count;  in sa_modify_attrs()
1948 	if ((error = sa_get_spill(hdl)) == 0) {  in sa_modify_attrs()
1949 		spill_data_size = hdl->sa_spill->db_size;  in sa_modify_attrs()
1951 		memcpy(old_data[1], hdl->sa_spill->db_data,  in sa_modify_attrs()
1952 		    hdl->sa_spill->db_size);  in sa_modify_attrs()
1954 		    hdl->sa_spill_tab->sa_layout->lot_attr_count;  in sa_modify_attrs()
1979 	hdr = SA_GET_HDR(hdl, SA_BONUS);  in sa_modify_attrs()
1980 	idx_tab = SA_IDX_TAB_GET(hdl, SA_BONUS);  in sa_modify_attrs()
2027 		if (k == 0 && hdl->sa_spill) {  in sa_modify_attrs()
2028 			hdr = SA_GET_HDR(hdl, SA_SPILL);  in sa_modify_attrs()
2029 			idx_tab = SA_IDX_TAB_GET(hdl, SA_SPILL);  in sa_modify_attrs()
2043 	error = sa_build_layouts(hdl, attr_desc, attr_count, tx);  in sa_modify_attrs()
2055 sa_bulk_update_impl(sa_handle_t *hdl, sa_bulk_attr_t *bulk, int count,  in sa_bulk_update_impl()  argument
2059 	sa_os_t *sa = hdl->sa_os->os_sa;  in sa_bulk_update_impl()
2063 	ASSERT(hdl);  in sa_bulk_update_impl()
2064 	ASSERT(MUTEX_HELD(&hdl->sa_lock));  in sa_bulk_update_impl()
2066 	bonustype = SA_BONUSTYPE_FROM_DB(SA_GET_DB(hdl, SA_BONUS));  in sa_bulk_update_impl()
2067 	saved_spill = hdl->sa_spill;  in sa_bulk_update_impl()
2071 		sa_attr_register_sync(hdl, tx);  in sa_bulk_update_impl()
2073 	error = sa_attr_op(hdl, bulk, count, SA_UPDATE, tx);  in sa_bulk_update_impl()
2075 		sa->sa_update_cb(hdl, tx);  in sa_bulk_update_impl()
2085 	if (!saved_spill && hdl->sa_spill) {  in sa_bulk_update_impl()
2086 		if (hdl->sa_spill_tab) {  in sa_bulk_update_impl()
2087 			sa_idx_tab_rele(hdl->sa_os, hdl->sa_spill_tab);  in sa_bulk_update_impl()
2088 			hdl->sa_spill_tab = NULL;  in sa_bulk_update_impl()
2091 		dmu_buf_rele(hdl->sa_spill, NULL);  in sa_bulk_update_impl()
2092 		hdl->sa_spill = NULL;  in sa_bulk_update_impl()
2102 sa_update(sa_handle_t *hdl, sa_attr_type_t type,  in sa_update()  argument
2115 	mutex_enter(&hdl->sa_lock);  in sa_update()
2116 	error = sa_bulk_update_impl(hdl, &bulk, 1, tx);  in sa_update()
2117 	mutex_exit(&hdl->sa_lock);  in sa_update()
2122 sa_bulk_lookup_locked(sa_handle_t *hdl, sa_bulk_attr_t *attrs, int count)  in sa_bulk_lookup_locked()  argument
2124 	ASSERT(hdl);  in sa_bulk_lookup_locked()
2125 	ASSERT(MUTEX_HELD(&hdl->sa_lock));  in sa_bulk_lookup_locked()
2126 	return (sa_lookup_impl(hdl, attrs, count));  in sa_bulk_lookup_locked()
2130 sa_bulk_lookup(sa_handle_t *hdl, sa_bulk_attr_t *attrs, int count)  in sa_bulk_lookup()  argument
2134 	ASSERT(hdl);  in sa_bulk_lookup()
2135 	mutex_enter(&hdl->sa_lock);  in sa_bulk_lookup()
2136 	error = sa_bulk_lookup_locked(hdl, attrs, count);  in sa_bulk_lookup()
2137 	mutex_exit(&hdl->sa_lock);  in sa_bulk_lookup()
2142 sa_bulk_update(sa_handle_t *hdl, sa_bulk_attr_t *attrs, int count, dmu_tx_t *tx)  in sa_bulk_update()  argument
2146 	ASSERT(hdl);  in sa_bulk_update()
2147 	mutex_enter(&hdl->sa_lock);  in sa_bulk_update()
2148 	error = sa_bulk_update_impl(hdl, attrs, count, tx);  in sa_bulk_update()
2149 	mutex_exit(&hdl->sa_lock);  in sa_bulk_update()
2154 sa_remove(sa_handle_t *hdl, sa_attr_type_t attr, dmu_tx_t *tx)  in sa_remove()  argument
2158 	mutex_enter(&hdl->sa_lock);  in sa_remove()
2159 	error = sa_modify_attrs(hdl, attr, SA_REMOVE, NULL,  in sa_remove()
2161 	mutex_exit(&hdl->sa_lock);  in sa_remove()
2166 sa_object_info(sa_handle_t *hdl, dmu_object_info_t *doi)  in sa_object_info()  argument
2168 	dmu_object_info_from_db(hdl->sa_bonus, doi);  in sa_object_info()
2172 sa_object_size(sa_handle_t *hdl, uint32_t *blksize, u_longlong_t *nblocks)  in sa_object_size()  argument
2174 	dmu_object_size_from_db(hdl->sa_bonus,  in sa_object_size()
2179 sa_set_userp(sa_handle_t *hdl, void *ptr)  in sa_set_userp()  argument
2181 	hdl->sa_userp = ptr;  in sa_set_userp()
2185 sa_get_db(sa_handle_t *hdl)  in sa_get_db()  argument
2187 	return (hdl->sa_bonus);  in sa_get_db()
2191 sa_get_userdata(sa_handle_t *hdl)  in sa_get_userdata()  argument
2193 	return (hdl->sa_userp);  in sa_get_userdata()
2213 sa_handle_object(sa_handle_t *hdl)  in sa_handle_object()  argument
2215 	return (hdl->sa_bonus->db_object);  in sa_handle_object()
2246 sa_handle_lock(sa_handle_t *hdl)  in sa_handle_lock()  argument
2248 	ASSERT(hdl);  in sa_handle_lock()
2249 	mutex_enter(&hdl->sa_lock);  in sa_handle_lock()
2253 sa_handle_unlock(sa_handle_t *hdl)  in sa_handle_unlock()  argument
2255 	ASSERT(hdl);  in sa_handle_unlock()
2256 	mutex_exit(&hdl->sa_lock);  in sa_handle_unlock()