1 // SPDX-License-Identifier: CDDL-1.0
10  * or https://opensource.org/licenses/CDDL-1.0.
33  * (a) Linear buffer. In this case, all the data in the ABD is stored in one
36  *         +-------------------+
37  *         | ABD (linear)      |
39  *         |   abd_size = ...  |     +--------------------------------+
40  *         |   abd_buf ------------->| raw buffer of size abd_size    |
41  *         +-------------------+     +--------------------------------+
45  *     equal-sized chunks (from the abd_chunk_cache kmem_cache), with pointers
48  *         +-------------------+
52  *         |   abd_offset = 0  |                           +-----------+
53  *         |   abd_chunks[0] ----------------------------->| chunk 0   |
54  *         |   abd_chunks[1] ---------------------+        +-----------+
55  *         |   ...             |                  |        +-----------+
56  *         |   abd_chunks[N-1] ---------+         +------->| chunk 1   |
57  *         +-------------------+        |                  +-----------+
59  *                                      |                  +-----------+
60  *                                      +----------------->| chunk N-1 |
61  *                                                         +-----------+
63  * In addition to directly allocating a linear or scattered ABD, it is also
64  * possible to create an ABD by requesting the "sub-ABD" starting at an offset
65  * within an existing ABD. In linear buffers this is simple (set abd_buf of
69  * provide arbitrary rather than only chunk-aligned starting offsets, it also
71  * within the first chunk in abd_chunks. For both linear and scattered ABDs,
76  * Most consumers should never need to know what type of ABD they're using --
78  * using a linear ABD to a scattered one when doing so would be beneficial.
80  * If you need to use the data within an ABD directly, if you know it's linear
92  * As an additional feature, linear and scatter ABD's can be stitched together
96  * It is possible to make all ABDs linear by setting zfs_abd_scatter_enabled to
114 		ASSERT3U(abd->abd_size, <=, DMU_MAX_ACCESS);  in abd_verify()
116 		ASSERT3U(abd->abd_size, <=, SPA_MAXBLOCKSIZE);  in abd_verify()
118 	ASSERT3U(abd->abd_flags, ==, abd->abd_flags & (ABD_FLAG_LINEAR |  in abd_verify()
122 	IMPLY(abd->abd_parent != NULL, !(abd->abd_flags & ABD_FLAG_OWNER));  in abd_verify()
123 	IMPLY(abd->abd_flags & ABD_FLAG_META, abd->abd_flags & ABD_FLAG_OWNER);  in abd_verify()
125 		ASSERT3U(abd->abd_size, >, 0);  in abd_verify()
132 			ASSERT(list_link_active(&cabd->abd_gang_link));  in abd_verify()
133 			child_sizes += cabd->abd_size;  in abd_verify()
136 		ASSERT3U(abd->abd_size, ==, child_sizes);  in abd_verify()
138 		ASSERT3U(abd->abd_size, >, 0);  in abd_verify()
147 	list_link_init(&abd->abd_gang_link);  in abd_init_struct()
148 	mutex_init(&abd->abd_mtx, NULL, MUTEX_DEFAULT, NULL);  in abd_init_struct()
149 	abd->abd_flags = 0;  in abd_init_struct()
151 	zfs_refcount_create(&abd->abd_children);  in abd_init_struct()
152 	abd->abd_parent = NULL;  in abd_init_struct()
154 	abd->abd_size = 0;  in abd_init_struct()
160 	mutex_destroy(&abd->abd_mtx);  in abd_fini_struct()
161 	ASSERT(!list_link_active(&abd->abd_gang_link));  in abd_fini_struct()
163 	zfs_refcount_destroy(&abd->abd_children);  in abd_fini_struct()
172 	abd->abd_flags |= ABD_FLAG_ALLOCD;  in abd_alloc_struct()
185  * don't care whether the ABD is linear or not.
196 	abd->abd_flags |= ABD_FLAG_OWNER;  in abd_alloc()
197 	abd->abd_u.abd_scatter.abd_offset = 0;  in abd_alloc()
201 		abd->abd_flags |= ABD_FLAG_META;  in abd_alloc()
203 	abd->abd_size = size;  in abd_alloc()
211  * Allocate an ABD that must be linear, along with its own underlying data
222 	abd->abd_flags |= ABD_FLAG_LINEAR | ABD_FLAG_OWNER;  in abd_alloc_linear()
224 		abd->abd_flags |= ABD_FLAG_META;  in abd_alloc_linear()
226 	abd->abd_size = size;  in abd_alloc_linear()
247 	if (abd->abd_flags & ABD_FLAG_META) {  in abd_free_linear()
248 		zio_buf_free(ABD_LINEAR_BUF(abd), abd->abd_size);  in abd_free_linear()
250 		zio_data_buf_free(ABD_LINEAR_BUF(abd), abd->abd_size);  in abd_free_linear()
269 		mutex_enter(&cabd->abd_mtx);  in abd_free_gang()
270 		ASSERT(list_link_active(&cabd->abd_gang_link));  in abd_free_gang()
272 		mutex_exit(&cabd->abd_mtx);  in abd_free_gang()
273 		if (cabd->abd_flags & ABD_FLAG_GANG_FREE)  in abd_free_gang()
291  * (scatterlist or linear buffer) will also be freed.  (Subject to ownership
305 	IMPLY(abd->abd_flags & ABD_FLAG_OWNER, abd->abd_parent == NULL);  in abd_free()
311 		if (abd->abd_flags & ABD_FLAG_OWNER)  in abd_free()
314 		if (abd->abd_flags & ABD_FLAG_OWNER)  in abd_free()
319 	if (abd->abd_parent != NULL) {  in abd_free()
320 		(void) zfs_refcount_remove_many(&abd->abd_parent->abd_children,  in abd_free()
321 		    abd->abd_size, abd);  in abd_free()
326 	if (abd->abd_flags & ABD_FLAG_ALLOCD)  in abd_free()
337 	boolean_t is_metadata = (sabd->abd_flags & ABD_FLAG_META) != 0;  in abd_alloc_sametype()
355 	abd->abd_flags |= ABD_FLAG_GANG | ABD_FLAG_OWNER;  in abd_alloc_gang()
384 		if (cabd->abd_parent != NULL) {  in abd_gang_add_gang()
386 			    &cabd->abd_parent->abd_children,  in abd_gang_add_gang()
387 			    cabd->abd_size, cabd);  in abd_gang_add_gang()
388 			cabd->abd_parent = NULL;  in abd_gang_add_gang()
391 		pabd->abd_size += cabd->abd_size;  in abd_gang_add_gang()
392 		cabd->abd_size = 0;  in abd_gang_add_gang()
429 		ASSERT(!list_link_active(&cabd->abd_gang_link));  in abd_gang_add()
442 	mutex_enter(&cabd->abd_mtx);  in abd_gang_add()
443 	if (list_link_active(&cabd->abd_gang_link)) {  in abd_gang_add()
472 		child_abd->abd_flags |= ABD_FLAG_GANG_FREE;  in abd_gang_add()
476 			child_abd->abd_flags |= ABD_FLAG_GANG_FREE;  in abd_gang_add()
481 	mutex_exit(&cabd->abd_mtx);  in abd_gang_add()
482 	pabd->abd_size += child_abd->abd_size;  in abd_gang_add()
495 	ASSERT3U(*off, <, abd->abd_size);  in abd_gang_get_offset()
498 		if (*off >= cabd->abd_size)  in abd_gang_get_offset()
499 			*off -= cabd->abd_size;  in abd_gang_get_offset()
508  * Allocate a new ABD, using the provided struct (if non-NULL, and if
509  * circumstances allow - otherwise allocate the struct).  The returned ABD will
517 	ASSERT3U(off + size, <=, sabd->abd_size);  in abd_get_offset_impl()
527 		abd->abd_flags |= ABD_FLAG_LINEAR;  in abd_get_offset_impl()
540 			abd->abd_flags |= ABD_FLAG_FROM_PAGES |  in abd_get_offset_impl()
550 			abd->abd_flags |= ABD_FLAG_GANG;  in abd_get_offset_impl()
555 		abd->abd_flags &= ~ABD_FLAG_OWNER;  in abd_get_offset_impl()
559 			int csize = MIN(left, cabd->abd_size - off);  in abd_get_offset_impl()
563 			left -= csize;  in abd_get_offset_impl()
572 	abd->abd_size = size;  in abd_get_offset_impl()
574 	abd->abd_parent = sabd;  in abd_get_offset_impl()
575 	(void) zfs_refcount_add_many(&sabd->abd_children, abd->abd_size, abd);  in abd_get_offset_impl()
603 	size_t size = sabd->abd_size > off ? sabd->abd_size - off : 0;  in abd_get_offset()
611 	ASSERT3U(off + size, <=, sabd->abd_size);  in abd_get_offset_size()
627  * Create a linear ABD for an existing buf.
639 	abd->abd_flags |= ABD_FLAG_LINEAR;  in abd_get_from_buf_impl()
640 	abd->abd_size = size;  in abd_get_from_buf_impl()
662  * Get the raw buffer associated with a linear ABD.
676 	ASSERT(abd->abd_flags & ABD_FLAG_OWNER);  in abd_release_ownership_of_buf()
681 	 * abd_release_ownership_of_buf() -> abd_get_from_buf() ->  in abd_release_ownership_of_buf()
683 	 * these "linear but not zio_[data_]buf_alloc()'ed" ABD's.  in abd_release_ownership_of_buf()
689 	abd->abd_flags &= ~ABD_FLAG_OWNER;  in abd_release_ownership_of_buf()
691 	abd->abd_flags &= ~ABD_FLAG_META;  in abd_release_ownership_of_buf()
699  * linear ABDs which were allocated via abd_get_from_buf(), or ones allocated
707 	ASSERT(!(abd->abd_flags & ABD_FLAG_OWNER));  in abd_take_ownership_of_buf()
710 	abd->abd_flags |= ABD_FLAG_OWNER;  in abd_take_ownership_of_buf()
712 		abd->abd_flags |= ABD_FLAG_META;  in abd_take_ownership_of_buf()
772 	ASSERT3U(off + size, <=, abd->abd_size);  in abd_iterate_func()
791 		size -= len;  in abd_iterate_func()
810 	ASSERT3U(off + size, <=, abd->abd_size);  in abd_iterate_page_func()
832 		size -= len;  in abd_iterate_page_func()
849 	(void) memcpy(ba_ptr->arg_buf, buf, size);  in abd_copy_to_buf_off_cb()
850 	ba_ptr->arg_buf = (char *)ba_ptr->arg_buf + size;  in abd_copy_to_buf_off_cb()
873 	ret = memcmp(buf, ba_ptr->arg_buf, size);  in abd_cmp_buf_off_cb()
874 	ba_ptr->arg_buf = (char *)ba_ptr->arg_buf + size;  in abd_cmp_buf_off_cb()
895 	(void) memcpy(buf, ba_ptr->arg_buf, size);  in abd_copy_from_buf_off_cb()
896 	ba_ptr->arg_buf = (char *)ba_ptr->arg_buf + size;  in abd_copy_from_buf_off_cb()
932  * equal-sized chunks (passed to func as raw buffers). func could be called many
949 	ASSERT3U(doff + size, <=, dabd->abd_size);  in abd_iterate_func2()
950 	ASSERT3U(soff + size, <=, sabd->abd_size);  in abd_iterate_func2()
976 		size -= len;  in abd_iterate_func2()
1017 	ASSERT3U(dabd->abd_size, ==, sabd->abd_size);  in abd_cmp()
1018 	return (abd_iterate_func2(dabd, sabd, 0, 0, dabd->abd_size,  in abd_cmp()
1023  * Check if ABD content is all-zeroes.
1053  *                 is the same when taking linear and when taking scatter
1072 		ASSERT3U(off + csize, <=, cabds[i]->abd_size);  in abd_raidz_gen_iterate()
1079 		ASSERT3U(off + dsize, <=, dabd->abd_size);  in abd_raidz_gen_iterate()
1114 		for (i = parity-1; i >= 0; i--) {  in abd_raidz_gen_iterate()
1126 			dsize -= dlen;  in abd_raidz_gen_iterate()
1129 		csize -= len;  in abd_raidz_gen_iterate()
1165 		ASSERT3U(tsize, <=, cabds[i]->abd_size);  in abd_raidz_rec_iterate()
1166 		ASSERT3U(tsize, <=, tabds[i]->abd_size);  in abd_raidz_rec_iterate()
1197 		for (i = parity-1; i >= 0; i--) {  in abd_raidz_rec_iterate()
1208 		tsize -= len;  in abd_raidz_rec_iterate()