1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * This file and its contents are supplied under the terms of the
4 * Common Development and Distribution License ("CDDL"), version 1.0.
5 * You may only use this file in accordance with the terms of version
6 * 1.0 of the CDDL.
7 *
8 * A full copy of the text of the CDDL should have accompanied this
9 * source. A copy of the CDDL is also available via the Internet at
10 * https://opensource.org/license/CDDL-1.0.
11 */
12 /*
13 * Copyright (c) 2014 by Chunwei Chen. All rights reserved.
14 * Copyright (c) 2016, 2019 by Delphix. All rights reserved.
15 */
16
17 #ifndef _ABD_H
18 #define _ABD_H
19
20 #include <sys/isa_defs.h>
21 #include <sys/debug.h>
22 #include <sys/zfs_refcount.h>
23 #include <sys/uio.h>
24 #include <sys/abd_os.h>
25
26 #ifdef __cplusplus
27 extern "C" {
28 #endif
29
30 typedef enum abd_flags {
31 ABD_FLAG_LINEAR = 1 << 0, /* is buffer linear (or scattered)? */
32 ABD_FLAG_OWNER = 1 << 1, /* does it own its data buffers? */
33 ABD_FLAG_META = 1 << 2, /* does this represent FS metadata? */
34 ABD_FLAG_MULTI_ZONE = 1 << 3, /* pages split over memory zones */
35 ABD_FLAG_MULTI_CHUNK = 1 << 4, /* pages split over multiple chunks */
36 ABD_FLAG_LINEAR_PAGE = 1 << 5, /* linear but allocd from page */
37 ABD_FLAG_GANG = 1 << 6, /* mult ABDs chained together */
38 ABD_FLAG_GANG_FREE = 1 << 7, /* gang ABD is responsible for mem */
39 ABD_FLAG_ALLOCD = 1 << 8, /* we allocated the abd_t */
40 ABD_FLAG_FROM_PAGES = 1 << 9, /* does not own pages */
41 } abd_flags_t;
42
43 typedef struct abd {
44 abd_flags_t abd_flags;
45 uint_t abd_size; /* excludes scattered abd_offset */
46 list_node_t abd_gang_link;
47 #ifdef ZFS_DEBUG
48 struct abd *abd_parent;
49 zfs_refcount_t abd_children;
50 #endif
51 kmutex_t abd_mtx;
52 union {
53 struct abd_scatter abd_scatter;
54 struct abd_linear abd_linear;
55 struct abd_gang {
56 list_t abd_gang_chain;
57 } abd_gang;
58 } abd_u;
59 } abd_t;
60
61 typedef int abd_iter_func_t(void *buf, size_t len, void *priv);
62 typedef int abd_iter_func2_t(void *bufa, void *bufb, size_t len, void *priv);
63
64 extern int zfs_abd_scatter_enabled;
65
66 /*
67 * Allocations and deallocations
68 */
69
70 __attribute__((malloc))
71 abd_t *abd_alloc(size_t, boolean_t);
72 __attribute__((malloc))
73 abd_t *abd_alloc_linear(size_t, boolean_t);
74 abd_t *abd_alloc_linear_struct(abd_t *, size_t, boolean_t);
75 __attribute__((malloc))
76 abd_t *abd_alloc_gang(void);
77 abd_t *abd_alloc_gang_struct(abd_t *);
78 __attribute__((malloc))
79 abd_t *abd_alloc_for_io(size_t, boolean_t);
80 __attribute__((malloc))
81 abd_t *abd_alloc_sametype(abd_t *, size_t);
82 boolean_t abd_size_alloc_linear(size_t);
83 void abd_gang_add(abd_t *, abd_t *, boolean_t);
84 void abd_free(abd_t *);
85 abd_t *abd_get_offset(abd_t *, size_t);
86 abd_t *abd_get_offset_size(abd_t *, size_t, size_t);
87 abd_t *abd_get_offset_struct(abd_t *, abd_t *, size_t, size_t);
88 abd_t *abd_get_zeros(size_t);
89 abd_t *abd_get_zeros_struct(abd_t *, size_t);
90 abd_t *abd_get_from_buf(void *, size_t);
91 abd_t *abd_get_from_buf_struct(abd_t *, void *, size_t);
92 void abd_cache_reap_now(void);
93
94 /*
95 * Conversion to and from a normal buffer
96 */
97
98 void *abd_to_buf(abd_t *);
99 void *abd_borrow_buf(abd_t *, size_t);
100 void *abd_borrow_buf_copy(abd_t *, size_t);
101 void abd_return_buf(abd_t *, void *, size_t);
102 void abd_return_buf_copy(abd_t *, void *, size_t);
103 void abd_take_ownership_of_buf(abd_t *, boolean_t);
104 void abd_release_ownership_of_buf(abd_t *);
105
106 /*
107 * ABD operations
108 */
109
110 int abd_iterate_func(abd_t *, size_t, size_t, abd_iter_func_t *, void *);
111 int abd_iterate_func2(abd_t *, abd_t *, size_t, size_t, size_t,
112 abd_iter_func2_t *, void *);
113 void abd_copy_off(abd_t *, abd_t *, size_t, size_t, size_t);
114 void abd_copy_from_buf_off(abd_t *, const void *, size_t, size_t);
115 void abd_copy_to_buf_off(void *, abd_t *, size_t, size_t);
116 int abd_cmp(abd_t *, abd_t *);
117 int abd_cmp_buf_off(abd_t *, const void *, size_t, size_t);
118 int abd_cmp_zero_off(abd_t *, size_t, size_t);
119 void abd_zero_off(abd_t *, size_t, size_t);
120 void abd_verify(abd_t *);
121
122 void abd_raidz_gen_iterate(abd_t **cabds, abd_t *dabd, size_t off,
123 size_t csize, size_t dsize, const unsigned parity,
124 void (*func_raidz_gen)(void **, const void *, size_t, size_t));
125 void abd_raidz_rec_iterate(abd_t **cabds, abd_t **tabds,
126 size_t tsize, const unsigned parity,
127 void (*func_raidz_rec)(void **t, const size_t tsize, void **c,
128 const unsigned *mul),
129 const unsigned *mul);
130
131 /*
132 * Wrappers for calls with offsets of 0
133 */
134
135 static inline void
abd_copy(abd_t * dabd,abd_t * sabd,size_t size)136 abd_copy(abd_t *dabd, abd_t *sabd, size_t size)
137 {
138 abd_copy_off(dabd, sabd, 0, 0, size);
139 }
140
141 static inline void
abd_copy_from_buf(abd_t * abd,const void * buf,size_t size)142 abd_copy_from_buf(abd_t *abd, const void *buf, size_t size)
143 {
144 abd_copy_from_buf_off(abd, buf, 0, size);
145 }
146
147 static inline void
abd_copy_to_buf(void * buf,abd_t * abd,size_t size)148 abd_copy_to_buf(void* buf, abd_t *abd, size_t size)
149 {
150 abd_copy_to_buf_off(buf, abd, 0, size);
151 }
152
153 static inline int
abd_cmp_buf(abd_t * abd,const void * buf,size_t size)154 abd_cmp_buf(abd_t *abd, const void *buf, size_t size)
155 {
156 return (abd_cmp_buf_off(abd, buf, 0, size));
157 }
158
159 static inline void
abd_zero(abd_t * abd,size_t size)160 abd_zero(abd_t *abd, size_t size)
161 {
162 abd_zero_off(abd, 0, size);
163 }
164
165 static inline int
abd_cmp_zero(abd_t * abd,size_t size)166 abd_cmp_zero(abd_t *abd, size_t size)
167 {
168 return (abd_cmp_zero_off(abd, 0, size));
169 }
170
171 /*
172 * ABD type check functions
173 */
174 static inline boolean_t
abd_is_linear(abd_t * abd)175 abd_is_linear(abd_t *abd)
176 {
177 return ((abd->abd_flags & ABD_FLAG_LINEAR) ? B_TRUE : B_FALSE);
178 }
179
180 static inline boolean_t
abd_is_linear_page(abd_t * abd)181 abd_is_linear_page(abd_t *abd)
182 {
183 return ((abd->abd_flags & ABD_FLAG_LINEAR_PAGE) ? B_TRUE : B_FALSE);
184 }
185
186 static inline boolean_t
abd_is_gang(abd_t * abd)187 abd_is_gang(abd_t *abd)
188 {
189 return ((abd->abd_flags & ABD_FLAG_GANG) ? B_TRUE : B_FALSE);
190 }
191
192 static inline uint_t
abd_get_size(abd_t * abd)193 abd_get_size(abd_t *abd)
194 {
195 return (abd->abd_size);
196 }
197
198 static inline boolean_t
abd_is_from_pages(abd_t * abd)199 abd_is_from_pages(abd_t *abd)
200 {
201 return ((abd->abd_flags & ABD_FLAG_FROM_PAGES) ? B_TRUE : B_FALSE);
202 }
203
204 /*
205 * Module lifecycle
206 * Defined in each specific OS's abd_os.c
207 */
208
209 void abd_init(void);
210 void abd_fini(void);
211
212 #ifdef __cplusplus
213 }
214 #endif
215
216 #endif /* _ABD_H */
217