1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * This file and its contents are supplied under the terms of the
4 * Common Development and Distribution License ("CDDL"), version 1.0.
5 * You may only use this file in accordance with the terms of version
6 * 1.0 of the CDDL.
7 *
8 * A full copy of the text of the CDDL should have accompanied this
9 * source. A copy of the CDDL is also available via the Internet at
10 * https://opensource.org/license/CDDL-1.0.
11 */
12 /*
13 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
14 * Use is subject to license terms.
15 */
16
17 /*
18 * Copyright (c) 2013, 2019 by Delphix. All rights reserved.
19 */
20
21 #ifndef _SYS_RANGE_TREE_H
22 #define _SYS_RANGE_TREE_H
23
24 #include <sys/btree.h>
25 #include <sys/dmu.h>
26
27 #ifdef __cplusplus
28 extern "C" {
29 #endif
30
31 #define ZFS_RANGE_TREE_HISTOGRAM_SIZE 64
32
33 typedef struct zfs_range_tree_ops zfs_range_tree_ops_t;
34
35 typedef enum zfs_range_seg_type {
36 ZFS_RANGE_SEG32,
37 ZFS_RANGE_SEG64,
38 ZFS_RANGE_SEG_GAP,
39 ZFS_RANGE_SEG_NUM_TYPES,
40 } zfs_range_seg_type_t;
41
42 #define ZFS_RT_NAME(rt) (((rt)->rt_name != NULL) ? (rt)->rt_name : "")
43 #define ZFS_RT_F_DYN_NAME (1ULL << 0) /* if rt_name must be freed */
44
45 /*
46 * Note: the range_tree may not be accessed concurrently; consumers
47 * must provide external locking if required.
48 */
49 typedef struct zfs_range_tree {
50 zfs_btree_t rt_root; /* offset-ordered segment b-tree */
51 uint64_t rt_space; /* sum of all segments in the map */
52 zfs_range_seg_type_t rt_type; /* type of zfs_range_seg_t in use */
53 /*
54 * All data that is stored in the range tree must have a start higher
55 * than or equal to rt_start, and all sizes and offsets must be
56 * multiples of 1 << rt_shift.
57 */
58 uint8_t rt_shift;
59 uint64_t rt_start;
60 const zfs_range_tree_ops_t *rt_ops;
61 void *rt_arg;
62 uint64_t rt_gap; /* allowable inter-segment gap */
63
64 uint64_t rt_flags;
65 const char *rt_name; /* details for debugging */
66
67 /*
68 * The rt_histogram maintains a histogram of ranges. Each bucket,
69 * rt_histogram[i], contains the number of ranges whose size is:
70 * 2^i <= size of range in bytes < 2^(i+1)
71 */
72 uint64_t rt_histogram[ZFS_RANGE_TREE_HISTOGRAM_SIZE];
73 } zfs_range_tree_t;
74
75 typedef struct zfs_range_seg32 {
76 uint32_t rs_start; /* starting offset of this segment */
77 uint32_t rs_end; /* ending offset (non-inclusive) */
78 } zfs_range_seg32_t;
79
80 /*
81 * Extremely large metaslabs, vdev-wide trees, and dnode-wide trees may
82 * require 64-bit integers for ranges.
83 */
84 typedef struct zfs_range_seg64 {
85 uint64_t rs_start; /* starting offset of this segment */
86 uint64_t rs_end; /* ending offset (non-inclusive) */
87 } zfs_range_seg64_t;
88
89 typedef struct zfs_range_seg_gap {
90 uint64_t rs_start; /* starting offset of this segment */
91 uint64_t rs_end; /* ending offset (non-inclusive) */
92 uint64_t rs_fill; /* actual fill if gap mode is on */
93 } zfs_range_seg_gap_t;
94
95 /*
96 * This type needs to be the largest of the range segs, since it will be stack
97 * allocated and then cast the actual type to do tree operations.
98 */
99 typedef zfs_range_seg_gap_t zfs_range_seg_max_t;
100
101 /*
102 * This is just for clarity of code purposes, so we can make it clear that a
103 * pointer is to a range seg of some type; when we need to do the actual math,
104 * we'll figure out the real type.
105 */
106 typedef void zfs_range_seg_t;
107
108 struct zfs_range_tree_ops {
109 void (*rtop_create)(zfs_range_tree_t *rt, void *arg);
110 void (*rtop_destroy)(zfs_range_tree_t *rt, void *arg);
111 void (*rtop_add)(zfs_range_tree_t *rt, void *rs, void *arg);
112 void (*rtop_remove)(zfs_range_tree_t *rt, void *rs, void *arg);
113 void (*rtop_vacate)(zfs_range_tree_t *rt, void *arg);
114 };
115
116 static inline uint64_t
zfs_rs_get_start_raw(const zfs_range_seg_t * rs,const zfs_range_tree_t * rt)117 zfs_rs_get_start_raw(const zfs_range_seg_t *rs, const zfs_range_tree_t *rt)
118 {
119 ASSERT3U(rt->rt_type, <=, ZFS_RANGE_SEG_NUM_TYPES);
120 switch (rt->rt_type) {
121 case ZFS_RANGE_SEG32:
122 return (((const zfs_range_seg32_t *)rs)->rs_start);
123 case ZFS_RANGE_SEG64:
124 return (((const zfs_range_seg64_t *)rs)->rs_start);
125 case ZFS_RANGE_SEG_GAP:
126 return (((const zfs_range_seg_gap_t *)rs)->rs_start);
127 default:
128 VERIFY(0);
129 return (0);
130 }
131 }
132
133 static inline uint64_t
zfs_rs_get_end_raw(const zfs_range_seg_t * rs,const zfs_range_tree_t * rt)134 zfs_rs_get_end_raw(const zfs_range_seg_t *rs, const zfs_range_tree_t *rt)
135 {
136 ASSERT3U(rt->rt_type, <=, ZFS_RANGE_SEG_NUM_TYPES);
137 switch (rt->rt_type) {
138 case ZFS_RANGE_SEG32:
139 return (((const zfs_range_seg32_t *)rs)->rs_end);
140 case ZFS_RANGE_SEG64:
141 return (((const zfs_range_seg64_t *)rs)->rs_end);
142 case ZFS_RANGE_SEG_GAP:
143 return (((const zfs_range_seg_gap_t *)rs)->rs_end);
144 default:
145 VERIFY(0);
146 return (0);
147 }
148 }
149
150 static inline uint64_t
zfs_rs_get_fill_raw(const zfs_range_seg_t * rs,const zfs_range_tree_t * rt)151 zfs_rs_get_fill_raw(const zfs_range_seg_t *rs, const zfs_range_tree_t *rt)
152 {
153 ASSERT3U(rt->rt_type, <=, ZFS_RANGE_SEG_NUM_TYPES);
154 switch (rt->rt_type) {
155 case ZFS_RANGE_SEG32: {
156 const zfs_range_seg32_t *r32 = (const zfs_range_seg32_t *)rs;
157 return (r32->rs_end - r32->rs_start);
158 }
159 case ZFS_RANGE_SEG64: {
160 const zfs_range_seg64_t *r64 = (const zfs_range_seg64_t *)rs;
161 return (r64->rs_end - r64->rs_start);
162 }
163 case ZFS_RANGE_SEG_GAP:
164 return (((const zfs_range_seg_gap_t *)rs)->rs_fill);
165 default:
166 VERIFY(0);
167 return (0);
168 }
169
170 }
171
172 static inline uint64_t
zfs_rs_get_start(const zfs_range_seg_t * rs,const zfs_range_tree_t * rt)173 zfs_rs_get_start(const zfs_range_seg_t *rs, const zfs_range_tree_t *rt)
174 {
175 return ((zfs_rs_get_start_raw(rs, rt) << rt->rt_shift) + rt->rt_start);
176 }
177
178 static inline uint64_t
zfs_rs_get_end(const zfs_range_seg_t * rs,const zfs_range_tree_t * rt)179 zfs_rs_get_end(const zfs_range_seg_t *rs, const zfs_range_tree_t *rt)
180 {
181 return ((zfs_rs_get_end_raw(rs, rt) << rt->rt_shift) + rt->rt_start);
182 }
183
184 static inline uint64_t
zfs_rs_get_fill(const zfs_range_seg_t * rs,const zfs_range_tree_t * rt)185 zfs_rs_get_fill(const zfs_range_seg_t *rs, const zfs_range_tree_t *rt)
186 {
187 return (zfs_rs_get_fill_raw(rs, rt) << rt->rt_shift);
188 }
189
190 static inline void
zfs_rs_set_start_raw(zfs_range_seg_t * rs,zfs_range_tree_t * rt,uint64_t start)191 zfs_rs_set_start_raw(zfs_range_seg_t *rs, zfs_range_tree_t *rt, uint64_t start)
192 {
193 ASSERT3U(rt->rt_type, <=, ZFS_RANGE_SEG_NUM_TYPES);
194 switch (rt->rt_type) {
195 case ZFS_RANGE_SEG32:
196 ASSERT3U(start, <=, UINT32_MAX);
197 ((zfs_range_seg32_t *)rs)->rs_start = (uint32_t)start;
198 break;
199 case ZFS_RANGE_SEG64:
200 ((zfs_range_seg64_t *)rs)->rs_start = start;
201 break;
202 case ZFS_RANGE_SEG_GAP:
203 ((zfs_range_seg_gap_t *)rs)->rs_start = start;
204 break;
205 default:
206 VERIFY(0);
207 }
208 }
209
210 static inline void
zfs_rs_set_end_raw(zfs_range_seg_t * rs,zfs_range_tree_t * rt,uint64_t end)211 zfs_rs_set_end_raw(zfs_range_seg_t *rs, zfs_range_tree_t *rt, uint64_t end)
212 {
213 ASSERT3U(rt->rt_type, <=, ZFS_RANGE_SEG_NUM_TYPES);
214 switch (rt->rt_type) {
215 case ZFS_RANGE_SEG32:
216 ASSERT3U(end, <=, UINT32_MAX);
217 ((zfs_range_seg32_t *)rs)->rs_end = (uint32_t)end;
218 break;
219 case ZFS_RANGE_SEG64:
220 ((zfs_range_seg64_t *)rs)->rs_end = end;
221 break;
222 case ZFS_RANGE_SEG_GAP:
223 ((zfs_range_seg_gap_t *)rs)->rs_end = end;
224 break;
225 default:
226 VERIFY(0);
227 }
228 }
229
230 static inline void
zfs_rs_set_fill_raw(zfs_range_seg_t * rs,zfs_range_tree_t * rt,uint64_t fill)231 zfs_rs_set_fill_raw(zfs_range_seg_t *rs, zfs_range_tree_t *rt, uint64_t fill)
232 {
233 ASSERT3U(rt->rt_type, <=, ZFS_RANGE_SEG_NUM_TYPES);
234 switch (rt->rt_type) {
235 case ZFS_RANGE_SEG32:
236 /* fall through */
237 case ZFS_RANGE_SEG64:
238 ASSERT3U(fill, ==, zfs_rs_get_end_raw(rs, rt) -
239 zfs_rs_get_start_raw(rs, rt));
240 break;
241 case ZFS_RANGE_SEG_GAP:
242 ((zfs_range_seg_gap_t *)rs)->rs_fill = fill;
243 break;
244 default:
245 VERIFY(0);
246 }
247 }
248
249 static inline void
zfs_rs_set_start(zfs_range_seg_t * rs,zfs_range_tree_t * rt,uint64_t start)250 zfs_rs_set_start(zfs_range_seg_t *rs, zfs_range_tree_t *rt, uint64_t start)
251 {
252 ASSERT3U(start, >=, rt->rt_start);
253 ASSERT(IS_P2ALIGNED(start, 1ULL << rt->rt_shift));
254 zfs_rs_set_start_raw(rs, rt, (start - rt->rt_start) >> rt->rt_shift);
255 }
256
257 static inline void
zfs_rs_set_end(zfs_range_seg_t * rs,zfs_range_tree_t * rt,uint64_t end)258 zfs_rs_set_end(zfs_range_seg_t *rs, zfs_range_tree_t *rt, uint64_t end)
259 {
260 ASSERT3U(end, >=, rt->rt_start);
261 ASSERT(IS_P2ALIGNED(end, 1ULL << rt->rt_shift));
262 zfs_rs_set_end_raw(rs, rt, (end - rt->rt_start) >> rt->rt_shift);
263 }
264
265 static inline void
zfs_rs_set_fill(zfs_range_seg_t * rs,zfs_range_tree_t * rt,uint64_t fill)266 zfs_rs_set_fill(zfs_range_seg_t *rs, zfs_range_tree_t *rt, uint64_t fill)
267 {
268 ASSERT(IS_P2ALIGNED(fill, 1ULL << rt->rt_shift));
269 zfs_rs_set_fill_raw(rs, rt, fill >> rt->rt_shift);
270 }
271
272 typedef void zfs_range_tree_func_t(void *arg, uint64_t start, uint64_t size);
273
274 zfs_range_tree_t *zfs_range_tree_create_gap(const zfs_range_tree_ops_t *ops,
275 zfs_range_seg_type_t type, void *arg, uint64_t start, uint64_t shift,
276 uint64_t gap);
277 zfs_range_tree_t *zfs_range_tree_create(const zfs_range_tree_ops_t *ops,
278 zfs_range_seg_type_t type, void *arg, uint64_t start, uint64_t shift);
279 zfs_range_tree_t *zfs_range_tree_create_flags(const zfs_range_tree_ops_t *ops,
280 zfs_range_seg_type_t type, void *arg, uint64_t start, uint64_t shift,
281 uint64_t flags, const char *name);
282 void zfs_range_tree_destroy(zfs_range_tree_t *rt);
283 boolean_t zfs_range_tree_contains(zfs_range_tree_t *rt, uint64_t start,
284 uint64_t size);
285 zfs_range_seg_t *zfs_range_tree_find(zfs_range_tree_t *rt, uint64_t start,
286 uint64_t size);
287 boolean_t zfs_range_tree_find_in(zfs_range_tree_t *rt, uint64_t start,
288 uint64_t size, uint64_t *ostart, uint64_t *osize);
289 void zfs_range_tree_verify_not_present(zfs_range_tree_t *rt,
290 uint64_t start, uint64_t size);
291 void zfs_range_tree_resize_segment(zfs_range_tree_t *rt, zfs_range_seg_t *rs,
292 uint64_t newstart, uint64_t newsize);
293 uint64_t zfs_range_tree_space(zfs_range_tree_t *rt);
294 uint64_t zfs_range_tree_numsegs(zfs_range_tree_t *rt);
295 boolean_t zfs_range_tree_is_empty(zfs_range_tree_t *rt);
296 void zfs_range_tree_swap(zfs_range_tree_t **rtsrc, zfs_range_tree_t **rtdst);
297 void zfs_range_tree_stat_verify(zfs_range_tree_t *rt);
298 uint64_t zfs_range_tree_min(zfs_range_tree_t *rt);
299 uint64_t zfs_range_tree_max(zfs_range_tree_t *rt);
300 uint64_t zfs_range_tree_span(zfs_range_tree_t *rt);
301
302 void zfs_range_tree_add(void *arg, uint64_t start, uint64_t size);
303 void zfs_range_tree_remove(void *arg, uint64_t start, uint64_t size);
304 void zfs_range_tree_remove_fill(zfs_range_tree_t *rt, uint64_t start,
305 uint64_t size);
306 void zfs_range_tree_adjust_fill(zfs_range_tree_t *rt, zfs_range_seg_t *rs,
307 int64_t delta);
308 void zfs_range_tree_clear(zfs_range_tree_t *rt, uint64_t start, uint64_t size);
309
310 void zfs_range_tree_vacate(zfs_range_tree_t *rt, zfs_range_tree_func_t *func,
311 void *arg);
312 void zfs_range_tree_walk(zfs_range_tree_t *rt, zfs_range_tree_func_t *func,
313 void *arg);
314 zfs_range_seg_t *zfs_range_tree_first(zfs_range_tree_t *rt);
315
316 void zfs_range_tree_remove_xor_add_segment(uint64_t start, uint64_t end,
317 zfs_range_tree_t *removefrom, zfs_range_tree_t *addto);
318 void zfs_range_tree_remove_xor_add(zfs_range_tree_t *rt,
319 zfs_range_tree_t *removefrom, zfs_range_tree_t *addto);
320
321 #ifdef __cplusplus
322 }
323 #endif
324
325 #endif /* _SYS_RANGE_TREE_H */
326