xref: /freebsd/sys/contrib/openzfs/module/zfs/sa.c (revision 7cd22ac43418da08448d0bab1009ff3cbda85120)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright (c) 2013, 2017 by Delphix. All rights reserved.
25  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
26  */
27 
28 #include <sys/zfs_context.h>
29 #include <sys/types.h>
30 #include <sys/param.h>
31 #include <sys/sysmacros.h>
32 #include <sys/dmu.h>
33 #include <sys/dmu_impl.h>
34 #include <sys/dmu_objset.h>
35 #include <sys/dmu_tx.h>
36 #include <sys/dbuf.h>
37 #include <sys/dnode.h>
38 #include <sys/zap.h>
39 #include <sys/sa.h>
40 #include <sys/sunddi.h>
41 #include <sys/sa_impl.h>
42 #include <sys/errno.h>
43 #include <sys/zfs_context.h>
44 
45 #ifdef _KERNEL
46 #include <sys/zfs_znode.h>
47 #endif
48 
49 /*
50  * ZFS System attributes:
51  *
52  * A generic mechanism to allow for arbitrary attributes
53  * to be stored in a dnode.  The data will be stored in the bonus buffer of
54  * the dnode and if necessary a special "spill" block will be used to handle
55  * overflow situations.  The spill block will be sized to fit the data
56  * from 512 - 128K.  When a spill block is used the BP (blkptr_t) for the
57  * spill block is stored at the end of the current bonus buffer.  Any
58  * attributes that would be in the way of the blkptr_t will be relocated
59  * into the spill block.
60  *
61  * Attribute registration:
62  *
63  * Stored persistently on a per dataset basis
64  * a mapping between attribute "string" names and their actual attribute
65  * numeric values, length, and byteswap function.  The names are only used
66  * during registration.  All  attributes are known by their unique attribute
67  * id value.  If an attribute can have a variable size then the value
68  * 0 will be used to indicate this.
69  *
70  * Attribute Layout:
71  *
72  * Attribute layouts are a way to compactly store multiple attributes, but
73  * without taking the overhead associated with managing each attribute
74  * individually.  Since you will typically have the same set of attributes
75  * stored in the same order a single table will be used to represent that
76  * layout.  The ZPL for example will usually have only about 10 different
77  * layouts (regular files, device files, symlinks,
78  * regular files + scanstamp, files/dir with extended attributes, and then
79  * you have the possibility of all of those minus ACL, because it would
80  * be kicked out into the spill block)
81  *
82  * Layouts are simply an array of the attributes and their
83  * ordering i.e. [0, 1, 4, 5, 2]
84  *
85  * Each distinct layout is given a unique layout number and that is what's
86  * stored in the header at the beginning of the SA data buffer.
87  *
88  * A layout only covers a single dbuf (bonus or spill).  If a set of
89  * attributes is split up between the bonus buffer and a spill buffer then
90  * two different layouts will be used.  This allows us to byteswap the
91  * spill without looking at the bonus buffer and keeps the on disk format of
92  * the bonus and spill buffer the same.
93  *
94  * Adding a single attribute will cause the entire set of attributes to
95  * be rewritten and could result in a new layout number being constructed
96  * as part of the rewrite if no such layout exists for the new set of
97  * attributes.  The new attribute will be appended to the end of the already
98  * existing attributes.
99  *
100  * Both the attribute registration and attribute layout information are
101  * stored in normal ZAP attributes.  Their should be a small number of
102  * known layouts and the set of attributes is assumed to typically be quite
103  * small.
104  *
105  * The registered attributes and layout "table" information is maintained
106  * in core and a special "sa_os_t" is attached to the objset_t.
107  *
108  * A special interface is provided to allow for quickly applying
109  * a large set of attributes at once.  sa_replace_all_by_template() is
110  * used to set an array of attributes.  This is used by the ZPL when
111  * creating a brand new file.  The template that is passed into the function
112  * specifies the attribute, size for variable length attributes, location of
113  * data and special "data locator" function if the data isn't in a contiguous
114  * location.
115  *
116  * Byteswap implications:
117  *
118  * Since the SA attributes are not entirely self describing we can't do
119  * the normal byteswap processing.  The special ZAP layout attribute and
120  * attribute registration attributes define the byteswap function and the
121  * size of the attributes, unless it is variable sized.
122  * The normal ZFS byteswapping infrastructure assumes you don't need
123  * to read any objects in order to do the necessary byteswapping.  Whereas
124  * SA attributes can only be properly byteswapped if the dataset is opened
125  * and the layout/attribute ZAP attributes are available.  Because of this
126  * the SA attributes will be byteswapped when they are first accessed by
127  * the SA code that will read the SA data.
128  */
129 
130 typedef void (sa_iterfunc_t)(void *hdr, void *addr, sa_attr_type_t,
131     uint16_t length, int length_idx, boolean_t, void *userp);
132 
133 static int sa_build_index(sa_handle_t *hdl, sa_buf_type_t buftype);
134 static void sa_idx_tab_hold(objset_t *os, sa_idx_tab_t *idx_tab);
135 static sa_idx_tab_t *sa_find_idx_tab(objset_t *os, dmu_object_type_t bonustype,
136     sa_hdr_phys_t *hdr);
137 static void sa_idx_tab_rele(objset_t *os, void *arg);
138 static void sa_copy_data(sa_data_locator_t *func, void *start, void *target,
139     int buflen);
140 static int sa_modify_attrs(sa_handle_t *hdl, sa_attr_type_t newattr,
141     sa_data_op_t action, sa_data_locator_t *locator, void *datastart,
142     uint16_t buflen, dmu_tx_t *tx);
143 
144 arc_byteswap_func_t sa_bswap_table[] = {
145 	byteswap_uint64_array,
146 	byteswap_uint32_array,
147 	byteswap_uint16_array,
148 	byteswap_uint8_array,
149 	zfs_acl_byteswap,
150 };
151 
152 #ifdef HAVE_EFFICIENT_UNALIGNED_ACCESS
153 #define	SA_COPY_DATA(f, s, t, l)				\
154 do {								\
155 	if (f == NULL) {					\
156 		if (l == 8) {					\
157 			*(uint64_t *)t = *(uint64_t *)s;	\
158 		} else if (l == 16) {				\
159 			*(uint64_t *)t = *(uint64_t *)s;	\
160 			*(uint64_t *)((uintptr_t)t + 8) =	\
161 			    *(uint64_t *)((uintptr_t)s + 8);	\
162 		} else {					\
163 			bcopy(s, t, l);				\
164 		}						\
165 	} else {						\
166 		sa_copy_data(f, s, t, l);			\
167 	}							\
168 } while (0)
169 #else
170 #define	SA_COPY_DATA(f, s, t, l)	sa_copy_data(f, s, t, l)
171 #endif
172 
173 /*
174  * This table is fixed and cannot be changed.  Its purpose is to
175  * allow the SA code to work with both old/new ZPL file systems.
176  * It contains the list of legacy attributes.  These attributes aren't
177  * stored in the "attribute" registry zap objects, since older ZPL file systems
178  * won't have the registry.  Only objsets of type ZFS_TYPE_FILESYSTEM will
179  * use this static table.
180  */
181 sa_attr_reg_t sa_legacy_attrs[] = {
182 	{"ZPL_ATIME", sizeof (uint64_t) * 2, SA_UINT64_ARRAY, 0},
183 	{"ZPL_MTIME", sizeof (uint64_t) * 2, SA_UINT64_ARRAY, 1},
184 	{"ZPL_CTIME", sizeof (uint64_t) * 2, SA_UINT64_ARRAY, 2},
185 	{"ZPL_CRTIME", sizeof (uint64_t) * 2, SA_UINT64_ARRAY, 3},
186 	{"ZPL_GEN", sizeof (uint64_t), SA_UINT64_ARRAY, 4},
187 	{"ZPL_MODE", sizeof (uint64_t), SA_UINT64_ARRAY, 5},
188 	{"ZPL_SIZE", sizeof (uint64_t), SA_UINT64_ARRAY, 6},
189 	{"ZPL_PARENT", sizeof (uint64_t), SA_UINT64_ARRAY, 7},
190 	{"ZPL_LINKS", sizeof (uint64_t), SA_UINT64_ARRAY, 8},
191 	{"ZPL_XATTR", sizeof (uint64_t), SA_UINT64_ARRAY, 9},
192 	{"ZPL_RDEV", sizeof (uint64_t), SA_UINT64_ARRAY, 10},
193 	{"ZPL_FLAGS", sizeof (uint64_t), SA_UINT64_ARRAY, 11},
194 	{"ZPL_UID", sizeof (uint64_t), SA_UINT64_ARRAY, 12},
195 	{"ZPL_GID", sizeof (uint64_t), SA_UINT64_ARRAY, 13},
196 	{"ZPL_PAD", sizeof (uint64_t) * 4, SA_UINT64_ARRAY, 14},
197 	{"ZPL_ZNODE_ACL", 88, SA_UINT8_ARRAY, 15},
198 };
199 
200 /*
201  * This is only used for objects of type DMU_OT_ZNODE
202  */
203 sa_attr_type_t sa_legacy_zpl_layout[] = {
204     0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
205 };
206 
207 /*
208  * Special dummy layout used for buffers with no attributes.
209  */
210 sa_attr_type_t sa_dummy_zpl_layout[] = { 0 };
211 
212 static int sa_legacy_attr_count = ARRAY_SIZE(sa_legacy_attrs);
213 static kmem_cache_t *sa_cache = NULL;
214 
215 /*ARGSUSED*/
216 static int
217 sa_cache_constructor(void *buf, void *unused, int kmflag)
218 {
219 	sa_handle_t *hdl = buf;
220 
221 	mutex_init(&hdl->sa_lock, NULL, MUTEX_DEFAULT, NULL);
222 	return (0);
223 }
224 
225 /*ARGSUSED*/
226 static void
227 sa_cache_destructor(void *buf, void *unused)
228 {
229 	sa_handle_t *hdl = buf;
230 	mutex_destroy(&hdl->sa_lock);
231 }
232 
233 void
234 sa_cache_init(void)
235 {
236 	sa_cache = kmem_cache_create("sa_cache",
237 	    sizeof (sa_handle_t), 0, sa_cache_constructor,
238 	    sa_cache_destructor, NULL, NULL, NULL, 0);
239 }
240 
241 void
242 sa_cache_fini(void)
243 {
244 	if (sa_cache)
245 		kmem_cache_destroy(sa_cache);
246 }
247 
248 static int
249 layout_num_compare(const void *arg1, const void *arg2)
250 {
251 	const sa_lot_t *node1 = (const sa_lot_t *)arg1;
252 	const sa_lot_t *node2 = (const sa_lot_t *)arg2;
253 
254 	return (TREE_CMP(node1->lot_num, node2->lot_num));
255 }
256 
257 static int
258 layout_hash_compare(const void *arg1, const void *arg2)
259 {
260 	const sa_lot_t *node1 = (const sa_lot_t *)arg1;
261 	const sa_lot_t *node2 = (const sa_lot_t *)arg2;
262 
263 	int cmp = TREE_CMP(node1->lot_hash, node2->lot_hash);
264 	if (likely(cmp))
265 		return (cmp);
266 
267 	return (TREE_CMP(node1->lot_instance, node2->lot_instance));
268 }
269 
270 static boolean_t
271 sa_layout_equal(sa_lot_t *tbf, sa_attr_type_t *attrs, int count)
272 {
273 	int i;
274 
275 	if (count != tbf->lot_attr_count)
276 		return (1);
277 
278 	for (i = 0; i != count; i++) {
279 		if (attrs[i] != tbf->lot_attrs[i])
280 			return (1);
281 	}
282 	return (0);
283 }
284 
285 #define	SA_ATTR_HASH(attr) (zfs_crc64_table[(-1ULL ^ attr) & 0xFF])
286 
287 static uint64_t
288 sa_layout_info_hash(sa_attr_type_t *attrs, int attr_count)
289 {
290 	int i;
291 	uint64_t crc = -1ULL;
292 
293 	for (i = 0; i != attr_count; i++)
294 		crc ^= SA_ATTR_HASH(attrs[i]);
295 
296 	return (crc);
297 }
298 
299 static int
300 sa_get_spill(sa_handle_t *hdl)
301 {
302 	int rc;
303 	if (hdl->sa_spill == NULL) {
304 		if ((rc = dmu_spill_hold_existing(hdl->sa_bonus, NULL,
305 		    &hdl->sa_spill)) == 0)
306 			VERIFY(0 == sa_build_index(hdl, SA_SPILL));
307 	} else {
308 		rc = 0;
309 	}
310 
311 	return (rc);
312 }
313 
314 /*
315  * Main attribute lookup/update function
316  * returns 0 for success or non zero for failures
317  *
318  * Operates on bulk array, first failure will abort further processing
319  */
320 static int
321 sa_attr_op(sa_handle_t *hdl, sa_bulk_attr_t *bulk, int count,
322     sa_data_op_t data_op, dmu_tx_t *tx)
323 {
324 	sa_os_t *sa = hdl->sa_os->os_sa;
325 	int i;
326 	int error = 0;
327 	sa_buf_type_t buftypes;
328 
329 	buftypes = 0;
330 
331 	ASSERT(count > 0);
332 	for (i = 0; i != count; i++) {
333 		ASSERT(bulk[i].sa_attr <= hdl->sa_os->os_sa->sa_num_attrs);
334 
335 		bulk[i].sa_addr = NULL;
336 		/* First check the bonus buffer */
337 
338 		if (hdl->sa_bonus_tab && TOC_ATTR_PRESENT(
339 		    hdl->sa_bonus_tab->sa_idx_tab[bulk[i].sa_attr])) {
340 			SA_ATTR_INFO(sa, hdl->sa_bonus_tab,
341 			    SA_GET_HDR(hdl, SA_BONUS),
342 			    bulk[i].sa_attr, bulk[i], SA_BONUS, hdl);
343 			if (tx && !(buftypes & SA_BONUS)) {
344 				dmu_buf_will_dirty(hdl->sa_bonus, tx);
345 				buftypes |= SA_BONUS;
346 			}
347 		}
348 		if (bulk[i].sa_addr == NULL &&
349 		    ((error = sa_get_spill(hdl)) == 0)) {
350 			if (TOC_ATTR_PRESENT(
351 			    hdl->sa_spill_tab->sa_idx_tab[bulk[i].sa_attr])) {
352 				SA_ATTR_INFO(sa, hdl->sa_spill_tab,
353 				    SA_GET_HDR(hdl, SA_SPILL),
354 				    bulk[i].sa_attr, bulk[i], SA_SPILL, hdl);
355 				if (tx && !(buftypes & SA_SPILL) &&
356 				    bulk[i].sa_size == bulk[i].sa_length) {
357 					dmu_buf_will_dirty(hdl->sa_spill, tx);
358 					buftypes |= SA_SPILL;
359 				}
360 			}
361 		}
362 		if (error && error != ENOENT) {
363 			return ((error == ECKSUM) ? EIO : error);
364 		}
365 
366 		switch (data_op) {
367 		case SA_LOOKUP:
368 			if (bulk[i].sa_addr == NULL)
369 				return (SET_ERROR(ENOENT));
370 			if (bulk[i].sa_data) {
371 				SA_COPY_DATA(bulk[i].sa_data_func,
372 				    bulk[i].sa_addr, bulk[i].sa_data,
373 				    bulk[i].sa_size);
374 			}
375 			continue;
376 
377 		case SA_UPDATE:
378 			/* existing rewrite of attr */
379 			if (bulk[i].sa_addr &&
380 			    bulk[i].sa_size == bulk[i].sa_length) {
381 				SA_COPY_DATA(bulk[i].sa_data_func,
382 				    bulk[i].sa_data, bulk[i].sa_addr,
383 				    bulk[i].sa_length);
384 				continue;
385 			} else if (bulk[i].sa_addr) { /* attr size change */
386 				error = sa_modify_attrs(hdl, bulk[i].sa_attr,
387 				    SA_REPLACE, bulk[i].sa_data_func,
388 				    bulk[i].sa_data, bulk[i].sa_length, tx);
389 			} else { /* adding new attribute */
390 				error = sa_modify_attrs(hdl, bulk[i].sa_attr,
391 				    SA_ADD, bulk[i].sa_data_func,
392 				    bulk[i].sa_data, bulk[i].sa_length, tx);
393 			}
394 			if (error)
395 				return (error);
396 			break;
397 		default:
398 			break;
399 		}
400 	}
401 	return (error);
402 }
403 
404 static sa_lot_t *
405 sa_add_layout_entry(objset_t *os, sa_attr_type_t *attrs, int attr_count,
406     uint64_t lot_num, uint64_t hash, boolean_t zapadd, dmu_tx_t *tx)
407 {
408 	sa_os_t *sa = os->os_sa;
409 	sa_lot_t *tb, *findtb;
410 	int i;
411 	avl_index_t loc;
412 
413 	ASSERT(MUTEX_HELD(&sa->sa_lock));
414 	tb = kmem_zalloc(sizeof (sa_lot_t), KM_SLEEP);
415 	tb->lot_attr_count = attr_count;
416 	tb->lot_attrs = kmem_alloc(sizeof (sa_attr_type_t) * attr_count,
417 	    KM_SLEEP);
418 	bcopy(attrs, tb->lot_attrs, sizeof (sa_attr_type_t) * attr_count);
419 	tb->lot_num = lot_num;
420 	tb->lot_hash = hash;
421 	tb->lot_instance = 0;
422 
423 	if (zapadd) {
424 		char attr_name[8];
425 
426 		if (sa->sa_layout_attr_obj == 0) {
427 			sa->sa_layout_attr_obj = zap_create_link(os,
428 			    DMU_OT_SA_ATTR_LAYOUTS,
429 			    sa->sa_master_obj, SA_LAYOUTS, tx);
430 		}
431 
432 		(void) snprintf(attr_name, sizeof (attr_name),
433 		    "%d", (int)lot_num);
434 		VERIFY(0 == zap_update(os, os->os_sa->sa_layout_attr_obj,
435 		    attr_name, 2, attr_count, attrs, tx));
436 	}
437 
438 	list_create(&tb->lot_idx_tab, sizeof (sa_idx_tab_t),
439 	    offsetof(sa_idx_tab_t, sa_next));
440 
441 	for (i = 0; i != attr_count; i++) {
442 		if (sa->sa_attr_table[tb->lot_attrs[i]].sa_length == 0)
443 			tb->lot_var_sizes++;
444 	}
445 
446 	avl_add(&sa->sa_layout_num_tree, tb);
447 
448 	/* verify we don't have a hash collision */
449 	if ((findtb = avl_find(&sa->sa_layout_hash_tree, tb, &loc)) != NULL) {
450 		for (; findtb && findtb->lot_hash == hash;
451 		    findtb = AVL_NEXT(&sa->sa_layout_hash_tree, findtb)) {
452 			if (findtb->lot_instance != tb->lot_instance)
453 				break;
454 			tb->lot_instance++;
455 		}
456 	}
457 	avl_add(&sa->sa_layout_hash_tree, tb);
458 	return (tb);
459 }
460 
461 static void
462 sa_find_layout(objset_t *os, uint64_t hash, sa_attr_type_t *attrs,
463     int count, dmu_tx_t *tx, sa_lot_t **lot)
464 {
465 	sa_lot_t *tb, tbsearch;
466 	avl_index_t loc;
467 	sa_os_t *sa = os->os_sa;
468 	boolean_t found = B_FALSE;
469 
470 	mutex_enter(&sa->sa_lock);
471 	tbsearch.lot_hash = hash;
472 	tbsearch.lot_instance = 0;
473 	tb = avl_find(&sa->sa_layout_hash_tree, &tbsearch, &loc);
474 	if (tb) {
475 		for (; tb && tb->lot_hash == hash;
476 		    tb = AVL_NEXT(&sa->sa_layout_hash_tree, tb)) {
477 			if (sa_layout_equal(tb, attrs, count) == 0) {
478 				found = B_TRUE;
479 				break;
480 			}
481 		}
482 	}
483 	if (!found) {
484 		tb = sa_add_layout_entry(os, attrs, count,
485 		    avl_numnodes(&sa->sa_layout_num_tree), hash, B_TRUE, tx);
486 	}
487 	mutex_exit(&sa->sa_lock);
488 	*lot = tb;
489 }
490 
491 static int
492 sa_resize_spill(sa_handle_t *hdl, uint32_t size, dmu_tx_t *tx)
493 {
494 	int error;
495 	uint32_t blocksize;
496 
497 	if (size == 0) {
498 		blocksize = SPA_MINBLOCKSIZE;
499 	} else if (size > SPA_OLD_MAXBLOCKSIZE) {
500 		ASSERT(0);
501 		return (SET_ERROR(EFBIG));
502 	} else {
503 		blocksize = P2ROUNDUP_TYPED(size, SPA_MINBLOCKSIZE, uint32_t);
504 	}
505 
506 	error = dbuf_spill_set_blksz(hdl->sa_spill, blocksize, tx);
507 	ASSERT(error == 0);
508 	return (error);
509 }
510 
511 static void
512 sa_copy_data(sa_data_locator_t *func, void *datastart, void *target, int buflen)
513 {
514 	if (func == NULL) {
515 		bcopy(datastart, target, buflen);
516 	} else {
517 		boolean_t start;
518 		int bytes;
519 		void *dataptr;
520 		void *saptr = target;
521 		uint32_t length;
522 
523 		start = B_TRUE;
524 		bytes = 0;
525 		while (bytes < buflen) {
526 			func(&dataptr, &length, buflen, start, datastart);
527 			bcopy(dataptr, saptr, length);
528 			saptr = (void *)((caddr_t)saptr + length);
529 			bytes += length;
530 			start = B_FALSE;
531 		}
532 	}
533 }
534 
535 /*
536  * Determine several different values pertaining to system attribute
537  * buffers.
538  *
539  * Return the size of the sa_hdr_phys_t header for the buffer. Each
540  * variable length attribute except the first contributes two bytes to
541  * the header size, which is then rounded up to an 8-byte boundary.
542  *
543  * The following output parameters are also computed.
544  *
545  *  index - The index of the first attribute in attr_desc that will
546  *  spill over. Only valid if will_spill is set.
547  *
548  *  total - The total number of bytes of all system attributes described
549  *  in attr_desc.
550  *
551  *  will_spill - Set when spilling is necessary. It is only set when
552  *  the buftype is SA_BONUS.
553  */
554 static int
555 sa_find_sizes(sa_os_t *sa, sa_bulk_attr_t *attr_desc, int attr_count,
556     dmu_buf_t *db, sa_buf_type_t buftype, int full_space, int *index,
557     int *total, boolean_t *will_spill)
558 {
559 	int var_size_count = 0;
560 	int i;
561 	int hdrsize;
562 	int extra_hdrsize;
563 
564 	if (buftype == SA_BONUS && sa->sa_force_spill) {
565 		*total = 0;
566 		*index = 0;
567 		*will_spill = B_TRUE;
568 		return (0);
569 	}
570 
571 	*index = -1;
572 	*total = 0;
573 	*will_spill = B_FALSE;
574 
575 	extra_hdrsize = 0;
576 	hdrsize = (SA_BONUSTYPE_FROM_DB(db) == DMU_OT_ZNODE) ? 0 :
577 	    sizeof (sa_hdr_phys_t);
578 
579 	ASSERT(IS_P2ALIGNED(full_space, 8));
580 
581 	for (i = 0; i != attr_count; i++) {
582 		boolean_t is_var_sz, might_spill_here;
583 		int tmp_hdrsize;
584 
585 		*total = P2ROUNDUP(*total, 8);
586 		*total += attr_desc[i].sa_length;
587 		if (*will_spill)
588 			continue;
589 
590 		is_var_sz = (SA_REGISTERED_LEN(sa, attr_desc[i].sa_attr) == 0);
591 		if (is_var_sz)
592 			var_size_count++;
593 
594 		/*
595 		 * Calculate what the SA header size would be if this
596 		 * attribute doesn't spill.
597 		 */
598 		tmp_hdrsize = hdrsize + ((is_var_sz && var_size_count > 1) ?
599 		    sizeof (uint16_t) : 0);
600 
601 		/*
602 		 * Check whether this attribute spans into the space
603 		 * that would be used by the spill block pointer should
604 		 * a spill block be needed.
605 		 */
606 		might_spill_here =
607 		    buftype == SA_BONUS && *index == -1 &&
608 		    (*total + P2ROUNDUP(tmp_hdrsize, 8)) >
609 		    (full_space - sizeof (blkptr_t));
610 
611 		if (is_var_sz && var_size_count > 1) {
612 			if (buftype == SA_SPILL ||
613 			    tmp_hdrsize + *total < full_space) {
614 				/*
615 				 * Record the extra header size in case this
616 				 * increase needs to be reversed due to
617 				 * spill-over.
618 				 */
619 				hdrsize = tmp_hdrsize;
620 				if (*index != -1 || might_spill_here)
621 					extra_hdrsize += sizeof (uint16_t);
622 			} else {
623 				ASSERT(buftype == SA_BONUS);
624 				if (*index == -1)
625 					*index = i;
626 				*will_spill = B_TRUE;
627 				continue;
628 			}
629 		}
630 
631 		/*
632 		 * Store index of where spill *could* occur. Then
633 		 * continue to count the remaining attribute sizes. The
634 		 * sum is used later for sizing bonus and spill buffer.
635 		 */
636 		if (might_spill_here)
637 			*index = i;
638 
639 		if ((*total + P2ROUNDUP(hdrsize, 8)) > full_space &&
640 		    buftype == SA_BONUS)
641 			*will_spill = B_TRUE;
642 	}
643 
644 	if (*will_spill)
645 		hdrsize -= extra_hdrsize;
646 
647 	hdrsize = P2ROUNDUP(hdrsize, 8);
648 	return (hdrsize);
649 }
650 
651 #define	BUF_SPACE_NEEDED(total, header) (total + header)
652 
653 /*
654  * Find layout that corresponds to ordering of attributes
655  * If not found a new layout number is created and added to
656  * persistent layout tables.
657  */
658 static int
659 sa_build_layouts(sa_handle_t *hdl, sa_bulk_attr_t *attr_desc, int attr_count,
660     dmu_tx_t *tx)
661 {
662 	sa_os_t *sa = hdl->sa_os->os_sa;
663 	uint64_t hash;
664 	sa_buf_type_t buftype;
665 	sa_hdr_phys_t *sahdr;
666 	void *data_start;
667 	sa_attr_type_t *attrs, *attrs_start;
668 	int i, lot_count;
669 	int dnodesize;
670 	int spill_idx;
671 	int hdrsize;
672 	int spillhdrsize = 0;
673 	int used;
674 	dmu_object_type_t bonustype;
675 	sa_lot_t *lot;
676 	int len_idx;
677 	int spill_used;
678 	int bonuslen;
679 	boolean_t spilling;
680 
681 	dmu_buf_will_dirty(hdl->sa_bonus, tx);
682 	bonustype = SA_BONUSTYPE_FROM_DB(hdl->sa_bonus);
683 	dmu_object_dnsize_from_db(hdl->sa_bonus, &dnodesize);
684 	bonuslen = DN_BONUS_SIZE(dnodesize);
685 
686 	/* first determine bonus header size and sum of all attributes */
687 	hdrsize = sa_find_sizes(sa, attr_desc, attr_count, hdl->sa_bonus,
688 	    SA_BONUS, bonuslen, &spill_idx, &used, &spilling);
689 
690 	if (used > SPA_OLD_MAXBLOCKSIZE)
691 		return (SET_ERROR(EFBIG));
692 
693 	VERIFY0(dmu_set_bonus(hdl->sa_bonus, spilling ?
694 	    MIN(bonuslen - sizeof (blkptr_t), used + hdrsize) :
695 	    used + hdrsize, tx));
696 
697 	ASSERT((bonustype == DMU_OT_ZNODE && spilling == 0) ||
698 	    bonustype == DMU_OT_SA);
699 
700 	/* setup and size spill buffer when needed */
701 	if (spilling) {
702 		boolean_t dummy;
703 
704 		if (hdl->sa_spill == NULL) {
705 			VERIFY(dmu_spill_hold_by_bonus(hdl->sa_bonus, 0, NULL,
706 			    &hdl->sa_spill) == 0);
707 		}
708 		dmu_buf_will_dirty(hdl->sa_spill, tx);
709 
710 		spillhdrsize = sa_find_sizes(sa, &attr_desc[spill_idx],
711 		    attr_count - spill_idx, hdl->sa_spill, SA_SPILL,
712 		    hdl->sa_spill->db_size, &i, &spill_used, &dummy);
713 
714 		if (spill_used > SPA_OLD_MAXBLOCKSIZE)
715 			return (SET_ERROR(EFBIG));
716 
717 		if (BUF_SPACE_NEEDED(spill_used, spillhdrsize) >
718 		    hdl->sa_spill->db_size)
719 			VERIFY(0 == sa_resize_spill(hdl,
720 			    BUF_SPACE_NEEDED(spill_used, spillhdrsize), tx));
721 	}
722 
723 	/* setup starting pointers to lay down data */
724 	data_start = (void *)((uintptr_t)hdl->sa_bonus->db_data + hdrsize);
725 	sahdr = (sa_hdr_phys_t *)hdl->sa_bonus->db_data;
726 	buftype = SA_BONUS;
727 
728 	attrs_start = attrs = kmem_alloc(sizeof (sa_attr_type_t) * attr_count,
729 	    KM_SLEEP);
730 	lot_count = 0;
731 
732 	for (i = 0, len_idx = 0, hash = -1ULL; i != attr_count; i++) {
733 		uint16_t length;
734 
735 		ASSERT(IS_P2ALIGNED(data_start, 8));
736 		attrs[i] = attr_desc[i].sa_attr;
737 		length = SA_REGISTERED_LEN(sa, attrs[i]);
738 		if (length == 0)
739 			length = attr_desc[i].sa_length;
740 
741 		if (spilling && i == spill_idx) { /* switch to spill buffer */
742 			VERIFY(bonustype == DMU_OT_SA);
743 			if (buftype == SA_BONUS && !sa->sa_force_spill) {
744 				sa_find_layout(hdl->sa_os, hash, attrs_start,
745 				    lot_count, tx, &lot);
746 				SA_SET_HDR(sahdr, lot->lot_num, hdrsize);
747 			}
748 
749 			buftype = SA_SPILL;
750 			hash = -1ULL;
751 			len_idx = 0;
752 
753 			sahdr = (sa_hdr_phys_t *)hdl->sa_spill->db_data;
754 			sahdr->sa_magic = SA_MAGIC;
755 			data_start = (void *)((uintptr_t)sahdr +
756 			    spillhdrsize);
757 			attrs_start = &attrs[i];
758 			lot_count = 0;
759 		}
760 		hash ^= SA_ATTR_HASH(attrs[i]);
761 		attr_desc[i].sa_addr = data_start;
762 		attr_desc[i].sa_size = length;
763 		SA_COPY_DATA(attr_desc[i].sa_data_func, attr_desc[i].sa_data,
764 		    data_start, length);
765 		if (sa->sa_attr_table[attrs[i]].sa_length == 0) {
766 			sahdr->sa_lengths[len_idx++] = length;
767 		}
768 		data_start = (void *)P2ROUNDUP(((uintptr_t)data_start +
769 		    length), 8);
770 		lot_count++;
771 	}
772 
773 	sa_find_layout(hdl->sa_os, hash, attrs_start, lot_count, tx, &lot);
774 
775 	/*
776 	 * Verify that old znodes always have layout number 0.
777 	 * Must be DMU_OT_SA for arbitrary layouts
778 	 */
779 	VERIFY((bonustype == DMU_OT_ZNODE && lot->lot_num == 0) ||
780 	    (bonustype == DMU_OT_SA && lot->lot_num > 1));
781 
782 	if (bonustype == DMU_OT_SA) {
783 		SA_SET_HDR(sahdr, lot->lot_num,
784 		    buftype == SA_BONUS ? hdrsize : spillhdrsize);
785 	}
786 
787 	kmem_free(attrs, sizeof (sa_attr_type_t) * attr_count);
788 	if (hdl->sa_bonus_tab) {
789 		sa_idx_tab_rele(hdl->sa_os, hdl->sa_bonus_tab);
790 		hdl->sa_bonus_tab = NULL;
791 	}
792 	if (!sa->sa_force_spill)
793 		VERIFY(0 == sa_build_index(hdl, SA_BONUS));
794 	if (hdl->sa_spill) {
795 		sa_idx_tab_rele(hdl->sa_os, hdl->sa_spill_tab);
796 		if (!spilling) {
797 			/*
798 			 * remove spill block that is no longer needed.
799 			 */
800 			dmu_buf_rele(hdl->sa_spill, NULL);
801 			hdl->sa_spill = NULL;
802 			hdl->sa_spill_tab = NULL;
803 			VERIFY(0 == dmu_rm_spill(hdl->sa_os,
804 			    sa_handle_object(hdl), tx));
805 		} else {
806 			VERIFY(0 == sa_build_index(hdl, SA_SPILL));
807 		}
808 	}
809 
810 	return (0);
811 }
812 
813 static void
814 sa_free_attr_table(sa_os_t *sa)
815 {
816 	int i;
817 
818 	if (sa->sa_attr_table == NULL)
819 		return;
820 
821 	for (i = 0; i != sa->sa_num_attrs; i++) {
822 		if (sa->sa_attr_table[i].sa_name)
823 			kmem_free(sa->sa_attr_table[i].sa_name,
824 			    strlen(sa->sa_attr_table[i].sa_name) + 1);
825 	}
826 
827 	kmem_free(sa->sa_attr_table,
828 	    sizeof (sa_attr_table_t) * sa->sa_num_attrs);
829 
830 	sa->sa_attr_table = NULL;
831 }
832 
833 static int
834 sa_attr_table_setup(objset_t *os, sa_attr_reg_t *reg_attrs, int count)
835 {
836 	sa_os_t *sa = os->os_sa;
837 	uint64_t sa_attr_count = 0;
838 	uint64_t sa_reg_count = 0;
839 	int error = 0;
840 	uint64_t attr_value;
841 	sa_attr_table_t *tb;
842 	zap_cursor_t zc;
843 	zap_attribute_t za;
844 	int registered_count = 0;
845 	int i;
846 	dmu_objset_type_t ostype = dmu_objset_type(os);
847 
848 	sa->sa_user_table =
849 	    kmem_zalloc(count * sizeof (sa_attr_type_t), KM_SLEEP);
850 	sa->sa_user_table_sz = count * sizeof (sa_attr_type_t);
851 
852 	if (sa->sa_reg_attr_obj != 0) {
853 		error = zap_count(os, sa->sa_reg_attr_obj,
854 		    &sa_attr_count);
855 
856 		/*
857 		 * Make sure we retrieved a count and that it isn't zero
858 		 */
859 		if (error || (error == 0 && sa_attr_count == 0)) {
860 			if (error == 0)
861 				error = SET_ERROR(EINVAL);
862 			goto bail;
863 		}
864 		sa_reg_count = sa_attr_count;
865 	}
866 
867 	if (ostype == DMU_OST_ZFS && sa_attr_count == 0)
868 		sa_attr_count += sa_legacy_attr_count;
869 
870 	/* Allocate attribute numbers for attributes that aren't registered */
871 	for (i = 0; i != count; i++) {
872 		boolean_t found = B_FALSE;
873 		int j;
874 
875 		if (ostype == DMU_OST_ZFS) {
876 			for (j = 0; j != sa_legacy_attr_count; j++) {
877 				if (strcmp(reg_attrs[i].sa_name,
878 				    sa_legacy_attrs[j].sa_name) == 0) {
879 					sa->sa_user_table[i] =
880 					    sa_legacy_attrs[j].sa_attr;
881 					found = B_TRUE;
882 				}
883 			}
884 		}
885 		if (found)
886 			continue;
887 
888 		if (sa->sa_reg_attr_obj)
889 			error = zap_lookup(os, sa->sa_reg_attr_obj,
890 			    reg_attrs[i].sa_name, 8, 1, &attr_value);
891 		else
892 			error = SET_ERROR(ENOENT);
893 		switch (error) {
894 		case ENOENT:
895 			sa->sa_user_table[i] = (sa_attr_type_t)sa_attr_count;
896 			sa_attr_count++;
897 			break;
898 		case 0:
899 			sa->sa_user_table[i] = ATTR_NUM(attr_value);
900 			break;
901 		default:
902 			goto bail;
903 		}
904 	}
905 
906 	sa->sa_num_attrs = sa_attr_count;
907 	tb = sa->sa_attr_table =
908 	    kmem_zalloc(sizeof (sa_attr_table_t) * sa_attr_count, KM_SLEEP);
909 
910 	/*
911 	 * Attribute table is constructed from requested attribute list,
912 	 * previously foreign registered attributes, and also the legacy
913 	 * ZPL set of attributes.
914 	 */
915 
916 	if (sa->sa_reg_attr_obj) {
917 		for (zap_cursor_init(&zc, os, sa->sa_reg_attr_obj);
918 		    (error = zap_cursor_retrieve(&zc, &za)) == 0;
919 		    zap_cursor_advance(&zc)) {
920 			uint64_t value;
921 			value  = za.za_first_integer;
922 
923 			registered_count++;
924 			tb[ATTR_NUM(value)].sa_attr = ATTR_NUM(value);
925 			tb[ATTR_NUM(value)].sa_length = ATTR_LENGTH(value);
926 			tb[ATTR_NUM(value)].sa_byteswap = ATTR_BSWAP(value);
927 			tb[ATTR_NUM(value)].sa_registered = B_TRUE;
928 
929 			if (tb[ATTR_NUM(value)].sa_name) {
930 				continue;
931 			}
932 			tb[ATTR_NUM(value)].sa_name =
933 			    kmem_zalloc(strlen(za.za_name) +1, KM_SLEEP);
934 			(void) strlcpy(tb[ATTR_NUM(value)].sa_name, za.za_name,
935 			    strlen(za.za_name) +1);
936 		}
937 		zap_cursor_fini(&zc);
938 		/*
939 		 * Make sure we processed the correct number of registered
940 		 * attributes
941 		 */
942 		if (registered_count != sa_reg_count) {
943 			ASSERT(error != 0);
944 			goto bail;
945 		}
946 
947 	}
948 
949 	if (ostype == DMU_OST_ZFS) {
950 		for (i = 0; i != sa_legacy_attr_count; i++) {
951 			if (tb[i].sa_name)
952 				continue;
953 			tb[i].sa_attr = sa_legacy_attrs[i].sa_attr;
954 			tb[i].sa_length = sa_legacy_attrs[i].sa_length;
955 			tb[i].sa_byteswap = sa_legacy_attrs[i].sa_byteswap;
956 			tb[i].sa_registered = B_FALSE;
957 			tb[i].sa_name =
958 			    kmem_zalloc(strlen(sa_legacy_attrs[i].sa_name) +1,
959 			    KM_SLEEP);
960 			(void) strlcpy(tb[i].sa_name,
961 			    sa_legacy_attrs[i].sa_name,
962 			    strlen(sa_legacy_attrs[i].sa_name) + 1);
963 		}
964 	}
965 
966 	for (i = 0; i != count; i++) {
967 		sa_attr_type_t attr_id;
968 
969 		attr_id = sa->sa_user_table[i];
970 		if (tb[attr_id].sa_name)
971 			continue;
972 
973 		tb[attr_id].sa_length = reg_attrs[i].sa_length;
974 		tb[attr_id].sa_byteswap = reg_attrs[i].sa_byteswap;
975 		tb[attr_id].sa_attr = attr_id;
976 		tb[attr_id].sa_name =
977 		    kmem_zalloc(strlen(reg_attrs[i].sa_name) + 1, KM_SLEEP);
978 		(void) strlcpy(tb[attr_id].sa_name, reg_attrs[i].sa_name,
979 		    strlen(reg_attrs[i].sa_name) + 1);
980 	}
981 
982 	sa->sa_need_attr_registration =
983 	    (sa_attr_count != registered_count);
984 
985 	return (0);
986 bail:
987 	kmem_free(sa->sa_user_table, count * sizeof (sa_attr_type_t));
988 	sa->sa_user_table = NULL;
989 	sa_free_attr_table(sa);
990 	ASSERT(error != 0);
991 	return (error);
992 }
993 
994 int
995 sa_setup(objset_t *os, uint64_t sa_obj, sa_attr_reg_t *reg_attrs, int count,
996     sa_attr_type_t **user_table)
997 {
998 	zap_cursor_t zc;
999 	zap_attribute_t za;
1000 	sa_os_t *sa;
1001 	dmu_objset_type_t ostype = dmu_objset_type(os);
1002 	sa_attr_type_t *tb;
1003 	int error;
1004 
1005 	mutex_enter(&os->os_user_ptr_lock);
1006 	if (os->os_sa) {
1007 		mutex_enter(&os->os_sa->sa_lock);
1008 		mutex_exit(&os->os_user_ptr_lock);
1009 		tb = os->os_sa->sa_user_table;
1010 		mutex_exit(&os->os_sa->sa_lock);
1011 		*user_table = tb;
1012 		return (0);
1013 	}
1014 
1015 	sa = kmem_zalloc(sizeof (sa_os_t), KM_SLEEP);
1016 	mutex_init(&sa->sa_lock, NULL, MUTEX_NOLOCKDEP, NULL);
1017 	sa->sa_master_obj = sa_obj;
1018 
1019 	os->os_sa = sa;
1020 	mutex_enter(&sa->sa_lock);
1021 	mutex_exit(&os->os_user_ptr_lock);
1022 	avl_create(&sa->sa_layout_num_tree, layout_num_compare,
1023 	    sizeof (sa_lot_t), offsetof(sa_lot_t, lot_num_node));
1024 	avl_create(&sa->sa_layout_hash_tree, layout_hash_compare,
1025 	    sizeof (sa_lot_t), offsetof(sa_lot_t, lot_hash_node));
1026 
1027 	if (sa_obj) {
1028 		error = zap_lookup(os, sa_obj, SA_LAYOUTS,
1029 		    8, 1, &sa->sa_layout_attr_obj);
1030 		if (error != 0 && error != ENOENT)
1031 			goto fail;
1032 		error = zap_lookup(os, sa_obj, SA_REGISTRY,
1033 		    8, 1, &sa->sa_reg_attr_obj);
1034 		if (error != 0 && error != ENOENT)
1035 			goto fail;
1036 	}
1037 
1038 	if ((error = sa_attr_table_setup(os, reg_attrs, count)) != 0)
1039 		goto fail;
1040 
1041 	if (sa->sa_layout_attr_obj != 0) {
1042 		uint64_t layout_count;
1043 
1044 		error = zap_count(os, sa->sa_layout_attr_obj,
1045 		    &layout_count);
1046 
1047 		/*
1048 		 * Layout number count should be > 0
1049 		 */
1050 		if (error || (error == 0 && layout_count == 0)) {
1051 			if (error == 0)
1052 				error = SET_ERROR(EINVAL);
1053 			goto fail;
1054 		}
1055 
1056 		for (zap_cursor_init(&zc, os, sa->sa_layout_attr_obj);
1057 		    (error = zap_cursor_retrieve(&zc, &za)) == 0;
1058 		    zap_cursor_advance(&zc)) {
1059 			sa_attr_type_t *lot_attrs;
1060 			uint64_t lot_num;
1061 
1062 			lot_attrs = kmem_zalloc(sizeof (sa_attr_type_t) *
1063 			    za.za_num_integers, KM_SLEEP);
1064 
1065 			if ((error = (zap_lookup(os, sa->sa_layout_attr_obj,
1066 			    za.za_name, 2, za.za_num_integers,
1067 			    lot_attrs))) != 0) {
1068 				kmem_free(lot_attrs, sizeof (sa_attr_type_t) *
1069 				    za.za_num_integers);
1070 				break;
1071 			}
1072 			VERIFY(ddi_strtoull(za.za_name, NULL, 10,
1073 			    (unsigned long long *)&lot_num) == 0);
1074 
1075 			(void) sa_add_layout_entry(os, lot_attrs,
1076 			    za.za_num_integers, lot_num,
1077 			    sa_layout_info_hash(lot_attrs,
1078 			    za.za_num_integers), B_FALSE, NULL);
1079 			kmem_free(lot_attrs, sizeof (sa_attr_type_t) *
1080 			    za.za_num_integers);
1081 		}
1082 		zap_cursor_fini(&zc);
1083 
1084 		/*
1085 		 * Make sure layout count matches number of entries added
1086 		 * to AVL tree
1087 		 */
1088 		if (avl_numnodes(&sa->sa_layout_num_tree) != layout_count) {
1089 			ASSERT(error != 0);
1090 			goto fail;
1091 		}
1092 	}
1093 
1094 	/* Add special layout number for old ZNODES */
1095 	if (ostype == DMU_OST_ZFS) {
1096 		(void) sa_add_layout_entry(os, sa_legacy_zpl_layout,
1097 		    sa_legacy_attr_count, 0,
1098 		    sa_layout_info_hash(sa_legacy_zpl_layout,
1099 		    sa_legacy_attr_count), B_FALSE, NULL);
1100 
1101 		(void) sa_add_layout_entry(os, sa_dummy_zpl_layout, 0, 1,
1102 		    0, B_FALSE, NULL);
1103 	}
1104 	*user_table = os->os_sa->sa_user_table;
1105 	mutex_exit(&sa->sa_lock);
1106 	return (0);
1107 fail:
1108 	os->os_sa = NULL;
1109 	sa_free_attr_table(sa);
1110 	if (sa->sa_user_table)
1111 		kmem_free(sa->sa_user_table, sa->sa_user_table_sz);
1112 	mutex_exit(&sa->sa_lock);
1113 	avl_destroy(&sa->sa_layout_hash_tree);
1114 	avl_destroy(&sa->sa_layout_num_tree);
1115 	mutex_destroy(&sa->sa_lock);
1116 	kmem_free(sa, sizeof (sa_os_t));
1117 	return ((error == ECKSUM) ? EIO : error);
1118 }
1119 
1120 void
1121 sa_tear_down(objset_t *os)
1122 {
1123 	sa_os_t *sa = os->os_sa;
1124 	sa_lot_t *layout;
1125 	void *cookie;
1126 
1127 	kmem_free(sa->sa_user_table, sa->sa_user_table_sz);
1128 
1129 	/* Free up attr table */
1130 
1131 	sa_free_attr_table(sa);
1132 
1133 	cookie = NULL;
1134 	while ((layout =
1135 	    avl_destroy_nodes(&sa->sa_layout_hash_tree, &cookie))) {
1136 		sa_idx_tab_t *tab;
1137 		while ((tab = list_head(&layout->lot_idx_tab))) {
1138 			ASSERT(zfs_refcount_count(&tab->sa_refcount));
1139 			sa_idx_tab_rele(os, tab);
1140 		}
1141 	}
1142 
1143 	cookie = NULL;
1144 	while ((layout = avl_destroy_nodes(&sa->sa_layout_num_tree, &cookie))) {
1145 		kmem_free(layout->lot_attrs,
1146 		    sizeof (sa_attr_type_t) * layout->lot_attr_count);
1147 		kmem_free(layout, sizeof (sa_lot_t));
1148 	}
1149 
1150 	avl_destroy(&sa->sa_layout_hash_tree);
1151 	avl_destroy(&sa->sa_layout_num_tree);
1152 	mutex_destroy(&sa->sa_lock);
1153 
1154 	kmem_free(sa, sizeof (sa_os_t));
1155 	os->os_sa = NULL;
1156 }
1157 
1158 static void
1159 sa_build_idx_tab(void *hdr, void *attr_addr, sa_attr_type_t attr,
1160     uint16_t length, int length_idx, boolean_t var_length, void *userp)
1161 {
1162 	sa_idx_tab_t *idx_tab = userp;
1163 
1164 	if (var_length) {
1165 		ASSERT(idx_tab->sa_variable_lengths);
1166 		idx_tab->sa_variable_lengths[length_idx] = length;
1167 	}
1168 	TOC_ATTR_ENCODE(idx_tab->sa_idx_tab[attr], length_idx,
1169 	    (uint32_t)((uintptr_t)attr_addr - (uintptr_t)hdr));
1170 }
1171 
1172 static void
1173 sa_attr_iter(objset_t *os, sa_hdr_phys_t *hdr, dmu_object_type_t type,
1174     sa_iterfunc_t func, sa_lot_t *tab, void *userp)
1175 {
1176 	void *data_start;
1177 	sa_lot_t *tb = tab;
1178 	sa_lot_t search;
1179 	avl_index_t loc;
1180 	sa_os_t *sa = os->os_sa;
1181 	int i;
1182 	uint16_t *length_start = NULL;
1183 	uint8_t length_idx = 0;
1184 
1185 	if (tab == NULL) {
1186 		search.lot_num = SA_LAYOUT_NUM(hdr, type);
1187 		tb = avl_find(&sa->sa_layout_num_tree, &search, &loc);
1188 		ASSERT(tb);
1189 	}
1190 
1191 	if (IS_SA_BONUSTYPE(type)) {
1192 		data_start = (void *)P2ROUNDUP(((uintptr_t)hdr +
1193 		    offsetof(sa_hdr_phys_t, sa_lengths) +
1194 		    (sizeof (uint16_t) * tb->lot_var_sizes)), 8);
1195 		length_start = hdr->sa_lengths;
1196 	} else {
1197 		data_start = hdr;
1198 	}
1199 
1200 	for (i = 0; i != tb->lot_attr_count; i++) {
1201 		int attr_length, reg_length;
1202 		uint8_t idx_len;
1203 
1204 		reg_length = sa->sa_attr_table[tb->lot_attrs[i]].sa_length;
1205 		if (reg_length) {
1206 			attr_length = reg_length;
1207 			idx_len = 0;
1208 		} else {
1209 			attr_length = length_start[length_idx];
1210 			idx_len = length_idx++;
1211 		}
1212 
1213 		func(hdr, data_start, tb->lot_attrs[i], attr_length,
1214 		    idx_len, reg_length == 0 ? B_TRUE : B_FALSE, userp);
1215 
1216 		data_start = (void *)P2ROUNDUP(((uintptr_t)data_start +
1217 		    attr_length), 8);
1218 	}
1219 }
1220 
1221 /*ARGSUSED*/
1222 static void
1223 sa_byteswap_cb(void *hdr, void *attr_addr, sa_attr_type_t attr,
1224     uint16_t length, int length_idx, boolean_t variable_length, void *userp)
1225 {
1226 	sa_handle_t *hdl = userp;
1227 	sa_os_t *sa = hdl->sa_os->os_sa;
1228 
1229 	sa_bswap_table[sa->sa_attr_table[attr].sa_byteswap](attr_addr, length);
1230 }
1231 
1232 static void
1233 sa_byteswap(sa_handle_t *hdl, sa_buf_type_t buftype)
1234 {
1235 	sa_hdr_phys_t *sa_hdr_phys = SA_GET_HDR(hdl, buftype);
1236 	dmu_buf_impl_t *db;
1237 	int num_lengths = 1;
1238 	int i;
1239 	sa_os_t *sa __maybe_unused = hdl->sa_os->os_sa;
1240 
1241 	ASSERT(MUTEX_HELD(&sa->sa_lock));
1242 	if (sa_hdr_phys->sa_magic == SA_MAGIC)
1243 		return;
1244 
1245 	db = SA_GET_DB(hdl, buftype);
1246 
1247 	if (buftype == SA_SPILL) {
1248 		arc_release(db->db_buf, NULL);
1249 		arc_buf_thaw(db->db_buf);
1250 	}
1251 
1252 	sa_hdr_phys->sa_magic = BSWAP_32(sa_hdr_phys->sa_magic);
1253 	sa_hdr_phys->sa_layout_info = BSWAP_16(sa_hdr_phys->sa_layout_info);
1254 
1255 	/*
1256 	 * Determine number of variable lengths in header
1257 	 * The standard 8 byte header has one for free and a
1258 	 * 16 byte header would have 4 + 1;
1259 	 */
1260 	if (SA_HDR_SIZE(sa_hdr_phys) > 8)
1261 		num_lengths += (SA_HDR_SIZE(sa_hdr_phys) - 8) >> 1;
1262 	for (i = 0; i != num_lengths; i++)
1263 		sa_hdr_phys->sa_lengths[i] =
1264 		    BSWAP_16(sa_hdr_phys->sa_lengths[i]);
1265 
1266 	sa_attr_iter(hdl->sa_os, sa_hdr_phys, DMU_OT_SA,
1267 	    sa_byteswap_cb, NULL, hdl);
1268 
1269 	if (buftype == SA_SPILL)
1270 		arc_buf_freeze(((dmu_buf_impl_t *)hdl->sa_spill)->db_buf);
1271 }
1272 
1273 static int
1274 sa_build_index(sa_handle_t *hdl, sa_buf_type_t buftype)
1275 {
1276 	sa_hdr_phys_t *sa_hdr_phys;
1277 	dmu_buf_impl_t *db = SA_GET_DB(hdl, buftype);
1278 	dmu_object_type_t bonustype = SA_BONUSTYPE_FROM_DB(db);
1279 	sa_os_t *sa = hdl->sa_os->os_sa;
1280 	sa_idx_tab_t *idx_tab;
1281 
1282 	sa_hdr_phys = SA_GET_HDR(hdl, buftype);
1283 
1284 	mutex_enter(&sa->sa_lock);
1285 
1286 	/* Do we need to byteswap? */
1287 
1288 	/* only check if not old znode */
1289 	if (IS_SA_BONUSTYPE(bonustype) && sa_hdr_phys->sa_magic != SA_MAGIC &&
1290 	    sa_hdr_phys->sa_magic != 0) {
1291 		if (BSWAP_32(sa_hdr_phys->sa_magic) != SA_MAGIC) {
1292 			mutex_exit(&sa->sa_lock);
1293 			zfs_dbgmsg("Buffer Header: %x != SA_MAGIC:%x "
1294 			    "object=%#llx\n", sa_hdr_phys->sa_magic, SA_MAGIC,
1295 			    (u_longlong_t)db->db.db_object);
1296 			return (SET_ERROR(EIO));
1297 		}
1298 		sa_byteswap(hdl, buftype);
1299 	}
1300 
1301 	idx_tab = sa_find_idx_tab(hdl->sa_os, bonustype, sa_hdr_phys);
1302 
1303 	if (buftype == SA_BONUS)
1304 		hdl->sa_bonus_tab = idx_tab;
1305 	else
1306 		hdl->sa_spill_tab = idx_tab;
1307 
1308 	mutex_exit(&sa->sa_lock);
1309 	return (0);
1310 }
1311 
1312 /*ARGSUSED*/
1313 static void
1314 sa_evict_sync(void *dbu)
1315 {
1316 	panic("evicting sa dbuf\n");
1317 }
1318 
1319 static void
1320 sa_idx_tab_rele(objset_t *os, void *arg)
1321 {
1322 	sa_os_t *sa = os->os_sa;
1323 	sa_idx_tab_t *idx_tab = arg;
1324 
1325 	if (idx_tab == NULL)
1326 		return;
1327 
1328 	mutex_enter(&sa->sa_lock);
1329 	if (zfs_refcount_remove(&idx_tab->sa_refcount, NULL) == 0) {
1330 		list_remove(&idx_tab->sa_layout->lot_idx_tab, idx_tab);
1331 		if (idx_tab->sa_variable_lengths)
1332 			kmem_free(idx_tab->sa_variable_lengths,
1333 			    sizeof (uint16_t) *
1334 			    idx_tab->sa_layout->lot_var_sizes);
1335 		zfs_refcount_destroy(&idx_tab->sa_refcount);
1336 		kmem_free(idx_tab->sa_idx_tab,
1337 		    sizeof (uint32_t) * sa->sa_num_attrs);
1338 		kmem_free(idx_tab, sizeof (sa_idx_tab_t));
1339 	}
1340 	mutex_exit(&sa->sa_lock);
1341 }
1342 
1343 static void
1344 sa_idx_tab_hold(objset_t *os, sa_idx_tab_t *idx_tab)
1345 {
1346 	sa_os_t *sa __maybe_unused = os->os_sa;
1347 
1348 	ASSERT(MUTEX_HELD(&sa->sa_lock));
1349 	(void) zfs_refcount_add(&idx_tab->sa_refcount, NULL);
1350 }
1351 
1352 void
1353 sa_spill_rele(sa_handle_t *hdl)
1354 {
1355 	mutex_enter(&hdl->sa_lock);
1356 	if (hdl->sa_spill) {
1357 		sa_idx_tab_rele(hdl->sa_os, hdl->sa_spill_tab);
1358 		dmu_buf_rele(hdl->sa_spill, NULL);
1359 		hdl->sa_spill = NULL;
1360 		hdl->sa_spill_tab = NULL;
1361 	}
1362 	mutex_exit(&hdl->sa_lock);
1363 }
1364 
1365 void
1366 sa_handle_destroy(sa_handle_t *hdl)
1367 {
1368 	dmu_buf_t *db = hdl->sa_bonus;
1369 
1370 	mutex_enter(&hdl->sa_lock);
1371 	(void) dmu_buf_remove_user(db, &hdl->sa_dbu);
1372 
1373 	if (hdl->sa_bonus_tab)
1374 		sa_idx_tab_rele(hdl->sa_os, hdl->sa_bonus_tab);
1375 
1376 	if (hdl->sa_spill_tab)
1377 		sa_idx_tab_rele(hdl->sa_os, hdl->sa_spill_tab);
1378 
1379 	dmu_buf_rele(hdl->sa_bonus, NULL);
1380 
1381 	if (hdl->sa_spill)
1382 		dmu_buf_rele(hdl->sa_spill, NULL);
1383 	mutex_exit(&hdl->sa_lock);
1384 
1385 	kmem_cache_free(sa_cache, hdl);
1386 }
1387 
1388 int
1389 sa_handle_get_from_db(objset_t *os, dmu_buf_t *db, void *userp,
1390     sa_handle_type_t hdl_type, sa_handle_t **handlepp)
1391 {
1392 	int error = 0;
1393 	sa_handle_t *handle = NULL;
1394 #ifdef ZFS_DEBUG
1395 	dmu_object_info_t doi;
1396 
1397 	dmu_object_info_from_db(db, &doi);
1398 	ASSERT(doi.doi_bonus_type == DMU_OT_SA ||
1399 	    doi.doi_bonus_type == DMU_OT_ZNODE);
1400 #endif
1401 	/* find handle, if it exists */
1402 	/* if one doesn't exist then create a new one, and initialize it */
1403 
1404 	if (hdl_type == SA_HDL_SHARED)
1405 		handle = dmu_buf_get_user(db);
1406 
1407 	if (handle == NULL) {
1408 		sa_handle_t *winner = NULL;
1409 
1410 		handle = kmem_cache_alloc(sa_cache, KM_SLEEP);
1411 		handle->sa_dbu.dbu_evict_func_sync = NULL;
1412 		handle->sa_dbu.dbu_evict_func_async = NULL;
1413 		handle->sa_userp = userp;
1414 		handle->sa_bonus = db;
1415 		handle->sa_os = os;
1416 		handle->sa_spill = NULL;
1417 		handle->sa_bonus_tab = NULL;
1418 		handle->sa_spill_tab = NULL;
1419 
1420 		error = sa_build_index(handle, SA_BONUS);
1421 
1422 		if (hdl_type == SA_HDL_SHARED) {
1423 			dmu_buf_init_user(&handle->sa_dbu, sa_evict_sync, NULL,
1424 			    NULL);
1425 			winner = dmu_buf_set_user_ie(db, &handle->sa_dbu);
1426 		}
1427 
1428 		if (winner != NULL) {
1429 			kmem_cache_free(sa_cache, handle);
1430 			handle = winner;
1431 		}
1432 	}
1433 	*handlepp = handle;
1434 
1435 	return (error);
1436 }
1437 
1438 int
1439 sa_handle_get(objset_t *objset, uint64_t objid, void *userp,
1440     sa_handle_type_t hdl_type, sa_handle_t **handlepp)
1441 {
1442 	dmu_buf_t *db;
1443 	int error;
1444 
1445 	if ((error = dmu_bonus_hold(objset, objid, NULL, &db)))
1446 		return (error);
1447 
1448 	return (sa_handle_get_from_db(objset, db, userp, hdl_type,
1449 	    handlepp));
1450 }
1451 
1452 int
1453 sa_buf_hold(objset_t *objset, uint64_t obj_num, void *tag, dmu_buf_t **db)
1454 {
1455 	return (dmu_bonus_hold(objset, obj_num, tag, db));
1456 }
1457 
1458 void
1459 sa_buf_rele(dmu_buf_t *db, void *tag)
1460 {
1461 	dmu_buf_rele(db, tag);
1462 }
1463 
1464 static int
1465 sa_lookup_impl(sa_handle_t *hdl, sa_bulk_attr_t *bulk, int count)
1466 {
1467 	ASSERT(hdl);
1468 	ASSERT(MUTEX_HELD(&hdl->sa_lock));
1469 	return (sa_attr_op(hdl, bulk, count, SA_LOOKUP, NULL));
1470 }
1471 
1472 static int
1473 sa_lookup_locked(sa_handle_t *hdl, sa_attr_type_t attr, void *buf,
1474     uint32_t buflen)
1475 {
1476 	int error;
1477 	sa_bulk_attr_t bulk;
1478 
1479 	VERIFY3U(buflen, <=, SA_ATTR_MAX_LEN);
1480 
1481 	bulk.sa_attr = attr;
1482 	bulk.sa_data = buf;
1483 	bulk.sa_length = buflen;
1484 	bulk.sa_data_func = NULL;
1485 
1486 	ASSERT(hdl);
1487 	error = sa_lookup_impl(hdl, &bulk, 1);
1488 	return (error);
1489 }
1490 
1491 int
1492 sa_lookup(sa_handle_t *hdl, sa_attr_type_t attr, void *buf, uint32_t buflen)
1493 {
1494 	int error;
1495 
1496 	mutex_enter(&hdl->sa_lock);
1497 	error = sa_lookup_locked(hdl, attr, buf, buflen);
1498 	mutex_exit(&hdl->sa_lock);
1499 
1500 	return (error);
1501 }
1502 
1503 #ifdef _KERNEL
1504 int
1505 sa_lookup_uio(sa_handle_t *hdl, sa_attr_type_t attr, zfs_uio_t *uio)
1506 {
1507 	int error;
1508 	sa_bulk_attr_t bulk;
1509 
1510 	bulk.sa_data = NULL;
1511 	bulk.sa_attr = attr;
1512 	bulk.sa_data_func = NULL;
1513 
1514 	ASSERT(hdl);
1515 
1516 	mutex_enter(&hdl->sa_lock);
1517 	if ((error = sa_attr_op(hdl, &bulk, 1, SA_LOOKUP, NULL)) == 0) {
1518 		error = zfs_uiomove((void *)bulk.sa_addr, MIN(bulk.sa_size,
1519 		    zfs_uio_resid(uio)), UIO_READ, uio);
1520 	}
1521 	mutex_exit(&hdl->sa_lock);
1522 	return (error);
1523 }
1524 
1525 /*
1526  * For the existed object that is upgraded from old system, its ondisk layout
1527  * has no slot for the project ID attribute. But quota accounting logic needs
1528  * to access related slots by offset directly. So we need to adjust these old
1529  * objects' layout to make the project ID to some unified and fixed offset.
1530  */
1531 int
1532 sa_add_projid(sa_handle_t *hdl, dmu_tx_t *tx, uint64_t projid)
1533 {
1534 	znode_t *zp = sa_get_userdata(hdl);
1535 	dmu_buf_t *db = sa_get_db(hdl);
1536 	zfsvfs_t *zfsvfs = ZTOZSB(zp);
1537 	int count = 0, err = 0;
1538 	sa_bulk_attr_t *bulk, *attrs;
1539 	zfs_acl_locator_cb_t locate = { 0 };
1540 	uint64_t uid, gid, mode, rdev, xattr = 0, parent, gen, links;
1541 	uint64_t crtime[2], mtime[2], ctime[2], atime[2];
1542 	zfs_acl_phys_t znode_acl = { 0 };
1543 	char scanstamp[AV_SCANSTAMP_SZ];
1544 
1545 	if (zp->z_acl_cached == NULL) {
1546 		zfs_acl_t *aclp;
1547 
1548 		mutex_enter(&zp->z_acl_lock);
1549 		err = zfs_acl_node_read(zp, B_FALSE, &aclp, B_FALSE);
1550 		mutex_exit(&zp->z_acl_lock);
1551 		if (err != 0 && err != ENOENT)
1552 			return (err);
1553 	}
1554 
1555 	bulk = kmem_zalloc(sizeof (sa_bulk_attr_t) * ZPL_END, KM_SLEEP);
1556 	attrs = kmem_zalloc(sizeof (sa_bulk_attr_t) * ZPL_END, KM_SLEEP);
1557 	mutex_enter(&hdl->sa_lock);
1558 	mutex_enter(&zp->z_lock);
1559 
1560 	err = sa_lookup_locked(hdl, SA_ZPL_PROJID(zfsvfs), &projid,
1561 	    sizeof (uint64_t));
1562 	if (unlikely(err == 0))
1563 		/* Someone has added project ID attr by race. */
1564 		err = EEXIST;
1565 	if (err != ENOENT)
1566 		goto out;
1567 
1568 	/* First do a bulk query of the attributes that aren't cached */
1569 	if (zp->z_is_sa) {
1570 		SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MODE(zfsvfs), NULL,
1571 		    &mode, 8);
1572 		SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_GEN(zfsvfs), NULL,
1573 		    &gen, 8);
1574 		SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_UID(zfsvfs), NULL,
1575 		    &uid, 8);
1576 		SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_GID(zfsvfs), NULL,
1577 		    &gid, 8);
1578 		SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_PARENT(zfsvfs), NULL,
1579 		    &parent, 8);
1580 		SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_ATIME(zfsvfs), NULL,
1581 		    &atime, 16);
1582 		SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL,
1583 		    &mtime, 16);
1584 		SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL,
1585 		    &ctime, 16);
1586 		SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CRTIME(zfsvfs), NULL,
1587 		    &crtime, 16);
1588 		if (Z_ISBLK(ZTOTYPE(zp)) || Z_ISCHR(ZTOTYPE(zp)))
1589 			SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_RDEV(zfsvfs), NULL,
1590 			    &rdev, 8);
1591 	} else {
1592 		SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_ATIME(zfsvfs), NULL,
1593 		    &atime, 16);
1594 		SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL,
1595 		    &mtime, 16);
1596 		SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL,
1597 		    &ctime, 16);
1598 		SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CRTIME(zfsvfs), NULL,
1599 		    &crtime, 16);
1600 		SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_GEN(zfsvfs), NULL,
1601 		    &gen, 8);
1602 		SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MODE(zfsvfs), NULL,
1603 		    &mode, 8);
1604 		SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_PARENT(zfsvfs), NULL,
1605 		    &parent, 8);
1606 		SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_XATTR(zfsvfs), NULL,
1607 		    &xattr, 8);
1608 		SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_RDEV(zfsvfs), NULL,
1609 		    &rdev, 8);
1610 		SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_UID(zfsvfs), NULL,
1611 		    &uid, 8);
1612 		SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_GID(zfsvfs), NULL,
1613 		    &gid, 8);
1614 		SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_ZNODE_ACL(zfsvfs), NULL,
1615 		    &znode_acl, 88);
1616 	}
1617 	err = sa_bulk_lookup_locked(hdl, bulk, count);
1618 	if (err != 0)
1619 		goto out;
1620 
1621 	err = sa_lookup_locked(hdl, SA_ZPL_XATTR(zfsvfs), &xattr, 8);
1622 	if (err != 0 && err != ENOENT)
1623 		goto out;
1624 
1625 	zp->z_projid = projid;
1626 	zp->z_pflags |= ZFS_PROJID;
1627 	links = ZTONLNK(zp);
1628 	count = 0;
1629 	err = 0;
1630 
1631 	SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_MODE(zfsvfs), NULL, &mode, 8);
1632 	SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_SIZE(zfsvfs), NULL,
1633 	    &zp->z_size, 8);
1634 	SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_GEN(zfsvfs), NULL, &gen, 8);
1635 	SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_UID(zfsvfs), NULL, &uid, 8);
1636 	SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_GID(zfsvfs), NULL, &gid, 8);
1637 	SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_PARENT(zfsvfs), NULL, &parent, 8);
1638 	SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_FLAGS(zfsvfs), NULL,
1639 	    &zp->z_pflags, 8);
1640 	SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_ATIME(zfsvfs), NULL, &atime, 16);
1641 	SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_MTIME(zfsvfs), NULL, &mtime, 16);
1642 	SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, 16);
1643 	SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_CRTIME(zfsvfs), NULL,
1644 	    &crtime, 16);
1645 	SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_LINKS(zfsvfs), NULL, &links, 8);
1646 	SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_PROJID(zfsvfs), NULL, &projid, 8);
1647 
1648 	if (Z_ISBLK(ZTOTYPE(zp)) || Z_ISCHR(ZTOTYPE(zp)))
1649 		SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_RDEV(zfsvfs), NULL,
1650 		    &rdev, 8);
1651 
1652 	if (zp->z_acl_cached != NULL) {
1653 		SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_DACL_COUNT(zfsvfs), NULL,
1654 		    &zp->z_acl_cached->z_acl_count, 8);
1655 		if (zp->z_acl_cached->z_version < ZFS_ACL_VERSION_FUID)
1656 			zfs_acl_xform(zp, zp->z_acl_cached, CRED());
1657 		locate.cb_aclp = zp->z_acl_cached;
1658 		SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_DACL_ACES(zfsvfs),
1659 		    zfs_acl_data_locator, &locate,
1660 		    zp->z_acl_cached->z_acl_bytes);
1661 	}
1662 
1663 	if (xattr)
1664 		SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_XATTR(zfsvfs), NULL,
1665 		    &xattr, 8);
1666 
1667 	if (zp->z_pflags & ZFS_BONUS_SCANSTAMP) {
1668 		bcopy((caddr_t)db->db_data + ZFS_OLD_ZNODE_PHYS_SIZE,
1669 		    scanstamp, AV_SCANSTAMP_SZ);
1670 		SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_SCANSTAMP(zfsvfs), NULL,
1671 		    scanstamp, AV_SCANSTAMP_SZ);
1672 		zp->z_pflags &= ~ZFS_BONUS_SCANSTAMP;
1673 	}
1674 
1675 	VERIFY(dmu_set_bonustype(db, DMU_OT_SA, tx) == 0);
1676 	VERIFY(sa_replace_all_by_template_locked(hdl, attrs, count, tx) == 0);
1677 	if (znode_acl.z_acl_extern_obj) {
1678 		VERIFY(0 == dmu_object_free(zfsvfs->z_os,
1679 		    znode_acl.z_acl_extern_obj, tx));
1680 	}
1681 
1682 	zp->z_is_sa = B_TRUE;
1683 
1684 out:
1685 	mutex_exit(&zp->z_lock);
1686 	mutex_exit(&hdl->sa_lock);
1687 	kmem_free(attrs, sizeof (sa_bulk_attr_t) * ZPL_END);
1688 	kmem_free(bulk, sizeof (sa_bulk_attr_t) * ZPL_END);
1689 	return (err);
1690 }
1691 #endif
1692 
1693 static sa_idx_tab_t *
1694 sa_find_idx_tab(objset_t *os, dmu_object_type_t bonustype, sa_hdr_phys_t *hdr)
1695 {
1696 	sa_idx_tab_t *idx_tab;
1697 	sa_os_t *sa = os->os_sa;
1698 	sa_lot_t *tb, search;
1699 	avl_index_t loc;
1700 
1701 	/*
1702 	 * Deterimine layout number.  If SA node and header == 0 then
1703 	 * force the index table to the dummy "1" empty layout.
1704 	 *
1705 	 * The layout number would only be zero for a newly created file
1706 	 * that has not added any attributes yet, or with crypto enabled which
1707 	 * doesn't write any attributes to the bonus buffer.
1708 	 */
1709 
1710 	search.lot_num = SA_LAYOUT_NUM(hdr, bonustype);
1711 
1712 	tb = avl_find(&sa->sa_layout_num_tree, &search, &loc);
1713 
1714 	/* Verify header size is consistent with layout information */
1715 	ASSERT(tb);
1716 	ASSERT((IS_SA_BONUSTYPE(bonustype) &&
1717 	    SA_HDR_SIZE_MATCH_LAYOUT(hdr, tb)) || !IS_SA_BONUSTYPE(bonustype) ||
1718 	    (IS_SA_BONUSTYPE(bonustype) && hdr->sa_layout_info == 0));
1719 
1720 	/*
1721 	 * See if any of the already existing TOC entries can be reused?
1722 	 */
1723 
1724 	for (idx_tab = list_head(&tb->lot_idx_tab); idx_tab;
1725 	    idx_tab = list_next(&tb->lot_idx_tab, idx_tab)) {
1726 		boolean_t valid_idx = B_TRUE;
1727 		int i;
1728 
1729 		if (tb->lot_var_sizes != 0 &&
1730 		    idx_tab->sa_variable_lengths != NULL) {
1731 			for (i = 0; i != tb->lot_var_sizes; i++) {
1732 				if (hdr->sa_lengths[i] !=
1733 				    idx_tab->sa_variable_lengths[i]) {
1734 					valid_idx = B_FALSE;
1735 					break;
1736 				}
1737 			}
1738 		}
1739 		if (valid_idx) {
1740 			sa_idx_tab_hold(os, idx_tab);
1741 			return (idx_tab);
1742 		}
1743 	}
1744 
1745 	/* No such luck, create a new entry */
1746 	idx_tab = kmem_zalloc(sizeof (sa_idx_tab_t), KM_SLEEP);
1747 	idx_tab->sa_idx_tab =
1748 	    kmem_zalloc(sizeof (uint32_t) * sa->sa_num_attrs, KM_SLEEP);
1749 	idx_tab->sa_layout = tb;
1750 	zfs_refcount_create(&idx_tab->sa_refcount);
1751 	if (tb->lot_var_sizes)
1752 		idx_tab->sa_variable_lengths = kmem_alloc(sizeof (uint16_t) *
1753 		    tb->lot_var_sizes, KM_SLEEP);
1754 
1755 	sa_attr_iter(os, hdr, bonustype, sa_build_idx_tab,
1756 	    tb, idx_tab);
1757 	sa_idx_tab_hold(os, idx_tab);   /* one hold for consumer */
1758 	sa_idx_tab_hold(os, idx_tab);	/* one for layout */
1759 	list_insert_tail(&tb->lot_idx_tab, idx_tab);
1760 	return (idx_tab);
1761 }
1762 
1763 void
1764 sa_default_locator(void **dataptr, uint32_t *len, uint32_t total_len,
1765     boolean_t start, void *userdata)
1766 {
1767 	ASSERT(start);
1768 
1769 	*dataptr = userdata;
1770 	*len = total_len;
1771 }
1772 
1773 static void
1774 sa_attr_register_sync(sa_handle_t *hdl, dmu_tx_t *tx)
1775 {
1776 	uint64_t attr_value = 0;
1777 	sa_os_t *sa = hdl->sa_os->os_sa;
1778 	sa_attr_table_t *tb = sa->sa_attr_table;
1779 	int i;
1780 
1781 	mutex_enter(&sa->sa_lock);
1782 
1783 	if (!sa->sa_need_attr_registration || sa->sa_master_obj == 0) {
1784 		mutex_exit(&sa->sa_lock);
1785 		return;
1786 	}
1787 
1788 	if (sa->sa_reg_attr_obj == 0) {
1789 		sa->sa_reg_attr_obj = zap_create_link(hdl->sa_os,
1790 		    DMU_OT_SA_ATTR_REGISTRATION,
1791 		    sa->sa_master_obj, SA_REGISTRY, tx);
1792 	}
1793 	for (i = 0; i != sa->sa_num_attrs; i++) {
1794 		if (sa->sa_attr_table[i].sa_registered)
1795 			continue;
1796 		ATTR_ENCODE(attr_value, tb[i].sa_attr, tb[i].sa_length,
1797 		    tb[i].sa_byteswap);
1798 		VERIFY(0 == zap_update(hdl->sa_os, sa->sa_reg_attr_obj,
1799 		    tb[i].sa_name, 8, 1, &attr_value, tx));
1800 		tb[i].sa_registered = B_TRUE;
1801 	}
1802 	sa->sa_need_attr_registration = B_FALSE;
1803 	mutex_exit(&sa->sa_lock);
1804 }
1805 
1806 /*
1807  * Replace all attributes with attributes specified in template.
1808  * If dnode had a spill buffer then those attributes will be
1809  * also be replaced, possibly with just an empty spill block
1810  *
1811  * This interface is intended to only be used for bulk adding of
1812  * attributes for a new file.  It will also be used by the ZPL
1813  * when converting and old formatted znode to native SA support.
1814  */
1815 int
1816 sa_replace_all_by_template_locked(sa_handle_t *hdl, sa_bulk_attr_t *attr_desc,
1817     int attr_count, dmu_tx_t *tx)
1818 {
1819 	sa_os_t *sa = hdl->sa_os->os_sa;
1820 
1821 	if (sa->sa_need_attr_registration)
1822 		sa_attr_register_sync(hdl, tx);
1823 	return (sa_build_layouts(hdl, attr_desc, attr_count, tx));
1824 }
1825 
1826 int
1827 sa_replace_all_by_template(sa_handle_t *hdl, sa_bulk_attr_t *attr_desc,
1828     int attr_count, dmu_tx_t *tx)
1829 {
1830 	int error;
1831 
1832 	mutex_enter(&hdl->sa_lock);
1833 	error = sa_replace_all_by_template_locked(hdl, attr_desc,
1834 	    attr_count, tx);
1835 	mutex_exit(&hdl->sa_lock);
1836 	return (error);
1837 }
1838 
1839 /*
1840  * Add/remove a single attribute or replace a variable-sized attribute value
1841  * with a value of a different size, and then rewrite the entire set
1842  * of attributes.
1843  * Same-length attribute value replacement (including fixed-length attributes)
1844  * is handled more efficiently by the upper layers.
1845  */
1846 static int
1847 sa_modify_attrs(sa_handle_t *hdl, sa_attr_type_t newattr,
1848     sa_data_op_t action, sa_data_locator_t *locator, void *datastart,
1849     uint16_t buflen, dmu_tx_t *tx)
1850 {
1851 	sa_os_t *sa = hdl->sa_os->os_sa;
1852 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)hdl->sa_bonus;
1853 	dnode_t *dn;
1854 	sa_bulk_attr_t *attr_desc;
1855 	void *old_data[2];
1856 	int bonus_attr_count = 0;
1857 	int bonus_data_size = 0;
1858 	int spill_data_size = 0;
1859 	int spill_attr_count = 0;
1860 	int error;
1861 	uint16_t length, reg_length;
1862 	int i, j, k, length_idx;
1863 	sa_hdr_phys_t *hdr;
1864 	sa_idx_tab_t *idx_tab;
1865 	int attr_count;
1866 	int count;
1867 
1868 	ASSERT(MUTEX_HELD(&hdl->sa_lock));
1869 
1870 	/* First make of copy of the old data */
1871 
1872 	DB_DNODE_ENTER(db);
1873 	dn = DB_DNODE(db);
1874 	if (dn->dn_bonuslen != 0) {
1875 		bonus_data_size = hdl->sa_bonus->db_size;
1876 		old_data[0] = kmem_alloc(bonus_data_size, KM_SLEEP);
1877 		bcopy(hdl->sa_bonus->db_data, old_data[0],
1878 		    hdl->sa_bonus->db_size);
1879 		bonus_attr_count = hdl->sa_bonus_tab->sa_layout->lot_attr_count;
1880 	} else {
1881 		old_data[0] = NULL;
1882 	}
1883 	DB_DNODE_EXIT(db);
1884 
1885 	/* Bring spill buffer online if it isn't currently */
1886 
1887 	if ((error = sa_get_spill(hdl)) == 0) {
1888 		spill_data_size = hdl->sa_spill->db_size;
1889 		old_data[1] = vmem_alloc(spill_data_size, KM_SLEEP);
1890 		bcopy(hdl->sa_spill->db_data, old_data[1],
1891 		    hdl->sa_spill->db_size);
1892 		spill_attr_count =
1893 		    hdl->sa_spill_tab->sa_layout->lot_attr_count;
1894 	} else if (error && error != ENOENT) {
1895 		if (old_data[0])
1896 			kmem_free(old_data[0], bonus_data_size);
1897 		return (error);
1898 	} else {
1899 		old_data[1] = NULL;
1900 	}
1901 
1902 	/* build descriptor of all attributes */
1903 
1904 	attr_count = bonus_attr_count + spill_attr_count;
1905 	if (action == SA_ADD)
1906 		attr_count++;
1907 	else if (action == SA_REMOVE)
1908 		attr_count--;
1909 
1910 	attr_desc = kmem_zalloc(sizeof (sa_bulk_attr_t) * attr_count, KM_SLEEP);
1911 
1912 	/*
1913 	 * loop through bonus and spill buffer if it exists, and
1914 	 * build up new attr_descriptor to reset the attributes
1915 	 */
1916 	k = j = 0;
1917 	count = bonus_attr_count;
1918 	hdr = SA_GET_HDR(hdl, SA_BONUS);
1919 	idx_tab = SA_IDX_TAB_GET(hdl, SA_BONUS);
1920 	for (; k != 2; k++) {
1921 		/*
1922 		 * Iterate over each attribute in layout.  Fetch the
1923 		 * size of variable-length attributes needing rewrite
1924 		 * from sa_lengths[].
1925 		 */
1926 		for (i = 0, length_idx = 0; i != count; i++) {
1927 			sa_attr_type_t attr;
1928 
1929 			attr = idx_tab->sa_layout->lot_attrs[i];
1930 			reg_length = SA_REGISTERED_LEN(sa, attr);
1931 			if (reg_length == 0) {
1932 				length = hdr->sa_lengths[length_idx];
1933 				length_idx++;
1934 			} else {
1935 				length = reg_length;
1936 			}
1937 			if (attr == newattr) {
1938 				/*
1939 				 * There is nothing to do for SA_REMOVE,
1940 				 * so it is just skipped.
1941 				 */
1942 				if (action == SA_REMOVE)
1943 					continue;
1944 
1945 				/*
1946 				 * Duplicate attributes are not allowed, so the
1947 				 * action can not be SA_ADD here.
1948 				 */
1949 				ASSERT3S(action, ==, SA_REPLACE);
1950 
1951 				/*
1952 				 * Only a variable-sized attribute can be
1953 				 * replaced here, and its size must be changing.
1954 				 */
1955 				ASSERT3U(reg_length, ==, 0);
1956 				ASSERT3U(length, !=, buflen);
1957 				SA_ADD_BULK_ATTR(attr_desc, j, attr,
1958 				    locator, datastart, buflen);
1959 			} else {
1960 				SA_ADD_BULK_ATTR(attr_desc, j, attr,
1961 				    NULL, (void *)
1962 				    (TOC_OFF(idx_tab->sa_idx_tab[attr]) +
1963 				    (uintptr_t)old_data[k]), length);
1964 			}
1965 		}
1966 		if (k == 0 && hdl->sa_spill) {
1967 			hdr = SA_GET_HDR(hdl, SA_SPILL);
1968 			idx_tab = SA_IDX_TAB_GET(hdl, SA_SPILL);
1969 			count = spill_attr_count;
1970 		} else {
1971 			break;
1972 		}
1973 	}
1974 	if (action == SA_ADD) {
1975 		reg_length = SA_REGISTERED_LEN(sa, newattr);
1976 		IMPLY(reg_length != 0, reg_length == buflen);
1977 		SA_ADD_BULK_ATTR(attr_desc, j, newattr, locator,
1978 		    datastart, buflen);
1979 	}
1980 	ASSERT3U(j, ==, attr_count);
1981 
1982 	error = sa_build_layouts(hdl, attr_desc, attr_count, tx);
1983 
1984 	if (old_data[0])
1985 		kmem_free(old_data[0], bonus_data_size);
1986 	if (old_data[1])
1987 		vmem_free(old_data[1], spill_data_size);
1988 	kmem_free(attr_desc, sizeof (sa_bulk_attr_t) * attr_count);
1989 
1990 	return (error);
1991 }
1992 
1993 static int
1994 sa_bulk_update_impl(sa_handle_t *hdl, sa_bulk_attr_t *bulk, int count,
1995     dmu_tx_t *tx)
1996 {
1997 	int error;
1998 	sa_os_t *sa = hdl->sa_os->os_sa;
1999 	dmu_object_type_t bonustype;
2000 	dmu_buf_t *saved_spill;
2001 
2002 	ASSERT(hdl);
2003 	ASSERT(MUTEX_HELD(&hdl->sa_lock));
2004 
2005 	bonustype = SA_BONUSTYPE_FROM_DB(SA_GET_DB(hdl, SA_BONUS));
2006 	saved_spill = hdl->sa_spill;
2007 
2008 	/* sync out registration table if necessary */
2009 	if (sa->sa_need_attr_registration)
2010 		sa_attr_register_sync(hdl, tx);
2011 
2012 	error = sa_attr_op(hdl, bulk, count, SA_UPDATE, tx);
2013 	if (error == 0 && !IS_SA_BONUSTYPE(bonustype) && sa->sa_update_cb)
2014 		sa->sa_update_cb(hdl, tx);
2015 
2016 	/*
2017 	 * If saved_spill is NULL and current sa_spill is not NULL that
2018 	 * means we increased the refcount of the spill buffer through
2019 	 * sa_get_spill() or dmu_spill_hold_by_dnode().  Therefore we
2020 	 * must release the hold before calling dmu_tx_commit() to avoid
2021 	 * making a copy of this buffer in dbuf_sync_leaf() due to the
2022 	 * reference count now being greater than 1.
2023 	 */
2024 	if (!saved_spill && hdl->sa_spill) {
2025 		if (hdl->sa_spill_tab) {
2026 			sa_idx_tab_rele(hdl->sa_os, hdl->sa_spill_tab);
2027 			hdl->sa_spill_tab = NULL;
2028 		}
2029 
2030 		dmu_buf_rele(hdl->sa_spill, NULL);
2031 		hdl->sa_spill = NULL;
2032 	}
2033 
2034 	return (error);
2035 }
2036 
2037 /*
2038  * update or add new attribute
2039  */
2040 int
2041 sa_update(sa_handle_t *hdl, sa_attr_type_t type,
2042     void *buf, uint32_t buflen, dmu_tx_t *tx)
2043 {
2044 	int error;
2045 	sa_bulk_attr_t bulk;
2046 
2047 	VERIFY3U(buflen, <=, SA_ATTR_MAX_LEN);
2048 
2049 	bulk.sa_attr = type;
2050 	bulk.sa_data_func = NULL;
2051 	bulk.sa_length = buflen;
2052 	bulk.sa_data = buf;
2053 
2054 	mutex_enter(&hdl->sa_lock);
2055 	error = sa_bulk_update_impl(hdl, &bulk, 1, tx);
2056 	mutex_exit(&hdl->sa_lock);
2057 	return (error);
2058 }
2059 
2060 /*
2061  * Return size of an attribute
2062  */
2063 
2064 int
2065 sa_size(sa_handle_t *hdl, sa_attr_type_t attr, int *size)
2066 {
2067 	sa_bulk_attr_t bulk;
2068 	int error;
2069 
2070 	bulk.sa_data = NULL;
2071 	bulk.sa_attr = attr;
2072 	bulk.sa_data_func = NULL;
2073 
2074 	ASSERT(hdl);
2075 	mutex_enter(&hdl->sa_lock);
2076 	if ((error = sa_attr_op(hdl, &bulk, 1, SA_LOOKUP, NULL)) != 0) {
2077 		mutex_exit(&hdl->sa_lock);
2078 		return (error);
2079 	}
2080 	*size = bulk.sa_size;
2081 
2082 	mutex_exit(&hdl->sa_lock);
2083 	return (0);
2084 }
2085 
2086 int
2087 sa_bulk_lookup_locked(sa_handle_t *hdl, sa_bulk_attr_t *attrs, int count)
2088 {
2089 	ASSERT(hdl);
2090 	ASSERT(MUTEX_HELD(&hdl->sa_lock));
2091 	return (sa_lookup_impl(hdl, attrs, count));
2092 }
2093 
2094 int
2095 sa_bulk_lookup(sa_handle_t *hdl, sa_bulk_attr_t *attrs, int count)
2096 {
2097 	int error;
2098 
2099 	ASSERT(hdl);
2100 	mutex_enter(&hdl->sa_lock);
2101 	error = sa_bulk_lookup_locked(hdl, attrs, count);
2102 	mutex_exit(&hdl->sa_lock);
2103 	return (error);
2104 }
2105 
2106 int
2107 sa_bulk_update(sa_handle_t *hdl, sa_bulk_attr_t *attrs, int count, dmu_tx_t *tx)
2108 {
2109 	int error;
2110 
2111 	ASSERT(hdl);
2112 	mutex_enter(&hdl->sa_lock);
2113 	error = sa_bulk_update_impl(hdl, attrs, count, tx);
2114 	mutex_exit(&hdl->sa_lock);
2115 	return (error);
2116 }
2117 
2118 int
2119 sa_remove(sa_handle_t *hdl, sa_attr_type_t attr, dmu_tx_t *tx)
2120 {
2121 	int error;
2122 
2123 	mutex_enter(&hdl->sa_lock);
2124 	error = sa_modify_attrs(hdl, attr, SA_REMOVE, NULL,
2125 	    NULL, 0, tx);
2126 	mutex_exit(&hdl->sa_lock);
2127 	return (error);
2128 }
2129 
2130 void
2131 sa_object_info(sa_handle_t *hdl, dmu_object_info_t *doi)
2132 {
2133 	dmu_object_info_from_db(hdl->sa_bonus, doi);
2134 }
2135 
2136 void
2137 sa_object_size(sa_handle_t *hdl, uint32_t *blksize, u_longlong_t *nblocks)
2138 {
2139 	dmu_object_size_from_db(hdl->sa_bonus,
2140 	    blksize, nblocks);
2141 }
2142 
2143 void
2144 sa_set_userp(sa_handle_t *hdl, void *ptr)
2145 {
2146 	hdl->sa_userp = ptr;
2147 }
2148 
2149 dmu_buf_t *
2150 sa_get_db(sa_handle_t *hdl)
2151 {
2152 	return (hdl->sa_bonus);
2153 }
2154 
2155 void *
2156 sa_get_userdata(sa_handle_t *hdl)
2157 {
2158 	return (hdl->sa_userp);
2159 }
2160 
2161 void
2162 sa_register_update_callback_locked(objset_t *os, sa_update_cb_t *func)
2163 {
2164 	ASSERT(MUTEX_HELD(&os->os_sa->sa_lock));
2165 	os->os_sa->sa_update_cb = func;
2166 }
2167 
2168 void
2169 sa_register_update_callback(objset_t *os, sa_update_cb_t *func)
2170 {
2171 
2172 	mutex_enter(&os->os_sa->sa_lock);
2173 	sa_register_update_callback_locked(os, func);
2174 	mutex_exit(&os->os_sa->sa_lock);
2175 }
2176 
2177 uint64_t
2178 sa_handle_object(sa_handle_t *hdl)
2179 {
2180 	return (hdl->sa_bonus->db_object);
2181 }
2182 
2183 boolean_t
2184 sa_enabled(objset_t *os)
2185 {
2186 	return (os->os_sa == NULL);
2187 }
2188 
2189 int
2190 sa_set_sa_object(objset_t *os, uint64_t sa_object)
2191 {
2192 	sa_os_t *sa = os->os_sa;
2193 
2194 	if (sa->sa_master_obj)
2195 		return (1);
2196 
2197 	sa->sa_master_obj = sa_object;
2198 
2199 	return (0);
2200 }
2201 
2202 int
2203 sa_hdrsize(void *arg)
2204 {
2205 	sa_hdr_phys_t *hdr = arg;
2206 
2207 	return (SA_HDR_SIZE(hdr));
2208 }
2209 
2210 void
2211 sa_handle_lock(sa_handle_t *hdl)
2212 {
2213 	ASSERT(hdl);
2214 	mutex_enter(&hdl->sa_lock);
2215 }
2216 
2217 void
2218 sa_handle_unlock(sa_handle_t *hdl)
2219 {
2220 	ASSERT(hdl);
2221 	mutex_exit(&hdl->sa_lock);
2222 }
2223 
2224 #ifdef _KERNEL
2225 EXPORT_SYMBOL(sa_handle_get);
2226 EXPORT_SYMBOL(sa_handle_get_from_db);
2227 EXPORT_SYMBOL(sa_handle_destroy);
2228 EXPORT_SYMBOL(sa_buf_hold);
2229 EXPORT_SYMBOL(sa_buf_rele);
2230 EXPORT_SYMBOL(sa_spill_rele);
2231 EXPORT_SYMBOL(sa_lookup);
2232 EXPORT_SYMBOL(sa_update);
2233 EXPORT_SYMBOL(sa_remove);
2234 EXPORT_SYMBOL(sa_bulk_lookup);
2235 EXPORT_SYMBOL(sa_bulk_lookup_locked);
2236 EXPORT_SYMBOL(sa_bulk_update);
2237 EXPORT_SYMBOL(sa_size);
2238 EXPORT_SYMBOL(sa_object_info);
2239 EXPORT_SYMBOL(sa_object_size);
2240 EXPORT_SYMBOL(sa_get_userdata);
2241 EXPORT_SYMBOL(sa_set_userp);
2242 EXPORT_SYMBOL(sa_get_db);
2243 EXPORT_SYMBOL(sa_handle_object);
2244 EXPORT_SYMBOL(sa_register_update_callback);
2245 EXPORT_SYMBOL(sa_setup);
2246 EXPORT_SYMBOL(sa_replace_all_by_template);
2247 EXPORT_SYMBOL(sa_replace_all_by_template_locked);
2248 EXPORT_SYMBOL(sa_enabled);
2249 EXPORT_SYMBOL(sa_cache_init);
2250 EXPORT_SYMBOL(sa_cache_fini);
2251 EXPORT_SYMBOL(sa_set_sa_object);
2252 EXPORT_SYMBOL(sa_hdrsize);
2253 EXPORT_SYMBOL(sa_handle_lock);
2254 EXPORT_SYMBOL(sa_handle_unlock);
2255 EXPORT_SYMBOL(sa_lookup_uio);
2256 EXPORT_SYMBOL(sa_add_projid);
2257 #endif /* _KERNEL */
2258