1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or https://opensource.org/licenses/CDDL-1.0.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2012, 2020 by Delphix. All rights reserved.
24 * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
25 */
26
27 #include <sys/zfs_context.h>
28 #include <sys/dbuf.h>
29 #include <sys/dnode.h>
30 #include <sys/dmu.h>
31 #include <sys/dmu_impl.h>
32 #include <sys/dmu_tx.h>
33 #include <sys/dmu_objset.h>
34 #include <sys/dsl_dir.h>
35 #include <sys/dsl_dataset.h>
36 #include <sys/spa.h>
37 #include <sys/zio.h>
38 #include <sys/dmu_zfetch.h>
39 #include <sys/range_tree.h>
40 #include <sys/trace_zfs.h>
41 #include <sys/zfs_project.h>
42
43 dnode_stats_t dnode_stats = {
44 { "dnode_hold_dbuf_hold", KSTAT_DATA_UINT64 },
45 { "dnode_hold_dbuf_read", KSTAT_DATA_UINT64 },
46 { "dnode_hold_alloc_hits", KSTAT_DATA_UINT64 },
47 { "dnode_hold_alloc_misses", KSTAT_DATA_UINT64 },
48 { "dnode_hold_alloc_interior", KSTAT_DATA_UINT64 },
49 { "dnode_hold_alloc_lock_retry", KSTAT_DATA_UINT64 },
50 { "dnode_hold_alloc_lock_misses", KSTAT_DATA_UINT64 },
51 { "dnode_hold_alloc_type_none", KSTAT_DATA_UINT64 },
52 { "dnode_hold_free_hits", KSTAT_DATA_UINT64 },
53 { "dnode_hold_free_misses", KSTAT_DATA_UINT64 },
54 { "dnode_hold_free_lock_misses", KSTAT_DATA_UINT64 },
55 { "dnode_hold_free_lock_retry", KSTAT_DATA_UINT64 },
56 { "dnode_hold_free_overflow", KSTAT_DATA_UINT64 },
57 { "dnode_hold_free_refcount", KSTAT_DATA_UINT64 },
58 { "dnode_free_interior_lock_retry", KSTAT_DATA_UINT64 },
59 { "dnode_allocate", KSTAT_DATA_UINT64 },
60 { "dnode_reallocate", KSTAT_DATA_UINT64 },
61 { "dnode_buf_evict", KSTAT_DATA_UINT64 },
62 { "dnode_alloc_next_chunk", KSTAT_DATA_UINT64 },
63 { "dnode_alloc_race", KSTAT_DATA_UINT64 },
64 { "dnode_alloc_next_block", KSTAT_DATA_UINT64 },
65 { "dnode_move_invalid", KSTAT_DATA_UINT64 },
66 { "dnode_move_recheck1", KSTAT_DATA_UINT64 },
67 { "dnode_move_recheck2", KSTAT_DATA_UINT64 },
68 { "dnode_move_special", KSTAT_DATA_UINT64 },
69 { "dnode_move_handle", KSTAT_DATA_UINT64 },
70 { "dnode_move_rwlock", KSTAT_DATA_UINT64 },
71 { "dnode_move_active", KSTAT_DATA_UINT64 },
72 };
73
74 dnode_sums_t dnode_sums;
75
76 static kstat_t *dnode_ksp;
77 static kmem_cache_t *dnode_cache;
78
79 static dnode_phys_t dnode_phys_zero __maybe_unused;
80
81 int zfs_default_bs = SPA_MINBLOCKSHIFT;
82 int zfs_default_ibs = DN_MAX_INDBLKSHIFT;
83
84 #ifdef _KERNEL
85 static kmem_cbrc_t dnode_move(void *, void *, size_t, void *);
86 #endif /* _KERNEL */
87
88 static int
dbuf_compare(const void * x1,const void * x2)89 dbuf_compare(const void *x1, const void *x2)
90 {
91 const dmu_buf_impl_t *d1 = x1;
92 const dmu_buf_impl_t *d2 = x2;
93
94 int cmp = TREE_CMP(d1->db_level, d2->db_level);
95 if (likely(cmp))
96 return (cmp);
97
98 cmp = TREE_CMP(d1->db_blkid, d2->db_blkid);
99 if (likely(cmp))
100 return (cmp);
101
102 if (d1->db_state == DB_MARKER) {
103 ASSERT3S(d2->db_state, !=, DB_MARKER);
104 return (TREE_PCMP(d1->db_parent, d2));
105 } else if (d2->db_state == DB_MARKER) {
106 ASSERT3S(d1->db_state, !=, DB_MARKER);
107 return (TREE_PCMP(d1, d2->db_parent));
108 }
109
110 if (d1->db_state == DB_SEARCH) {
111 ASSERT3S(d2->db_state, !=, DB_SEARCH);
112 return (-1);
113 } else if (d2->db_state == DB_SEARCH) {
114 ASSERT3S(d1->db_state, !=, DB_SEARCH);
115 return (1);
116 }
117
118 return (TREE_PCMP(d1, d2));
119 }
120
121 static int
dnode_cons(void * arg,void * unused,int kmflag)122 dnode_cons(void *arg, void *unused, int kmflag)
123 {
124 (void) unused, (void) kmflag;
125 dnode_t *dn = arg;
126
127 rw_init(&dn->dn_struct_rwlock, NULL, RW_NOLOCKDEP, NULL);
128 mutex_init(&dn->dn_mtx, NULL, MUTEX_DEFAULT, NULL);
129 mutex_init(&dn->dn_dbufs_mtx, NULL, MUTEX_DEFAULT, NULL);
130 cv_init(&dn->dn_notxholds, NULL, CV_DEFAULT, NULL);
131 cv_init(&dn->dn_nodnholds, NULL, CV_DEFAULT, NULL);
132
133 /*
134 * Every dbuf has a reference, and dropping a tracked reference is
135 * O(number of references), so don't track dn_holds.
136 */
137 zfs_refcount_create_untracked(&dn->dn_holds);
138 zfs_refcount_create(&dn->dn_tx_holds);
139 list_link_init(&dn->dn_link);
140
141 memset(dn->dn_next_type, 0, sizeof (dn->dn_next_type));
142 memset(dn->dn_next_nblkptr, 0, sizeof (dn->dn_next_nblkptr));
143 memset(dn->dn_next_nlevels, 0, sizeof (dn->dn_next_nlevels));
144 memset(dn->dn_next_indblkshift, 0, sizeof (dn->dn_next_indblkshift));
145 memset(dn->dn_next_bonustype, 0, sizeof (dn->dn_next_bonustype));
146 memset(dn->dn_rm_spillblk, 0, sizeof (dn->dn_rm_spillblk));
147 memset(dn->dn_next_bonuslen, 0, sizeof (dn->dn_next_bonuslen));
148 memset(dn->dn_next_blksz, 0, sizeof (dn->dn_next_blksz));
149 memset(dn->dn_next_maxblkid, 0, sizeof (dn->dn_next_maxblkid));
150
151 for (int i = 0; i < TXG_SIZE; i++) {
152 multilist_link_init(&dn->dn_dirty_link[i]);
153 dn->dn_free_ranges[i] = NULL;
154 list_create(&dn->dn_dirty_records[i],
155 sizeof (dbuf_dirty_record_t),
156 offsetof(dbuf_dirty_record_t, dr_dirty_node));
157 }
158
159 dn->dn_allocated_txg = 0;
160 dn->dn_free_txg = 0;
161 dn->dn_assigned_txg = 0;
162 dn->dn_dirty_txg = 0;
163 dn->dn_dirtyctx = 0;
164 dn->dn_dirtyctx_firstset = NULL;
165 dn->dn_bonus = NULL;
166 dn->dn_have_spill = B_FALSE;
167 dn->dn_zio = NULL;
168 dn->dn_oldused = 0;
169 dn->dn_oldflags = 0;
170 dn->dn_olduid = 0;
171 dn->dn_oldgid = 0;
172 dn->dn_oldprojid = ZFS_DEFAULT_PROJID;
173 dn->dn_newuid = 0;
174 dn->dn_newgid = 0;
175 dn->dn_newprojid = ZFS_DEFAULT_PROJID;
176 dn->dn_id_flags = 0;
177
178 dn->dn_dbufs_count = 0;
179 avl_create(&dn->dn_dbufs, dbuf_compare, sizeof (dmu_buf_impl_t),
180 offsetof(dmu_buf_impl_t, db_link));
181
182 dn->dn_moved = 0;
183 return (0);
184 }
185
186 static void
dnode_dest(void * arg,void * unused)187 dnode_dest(void *arg, void *unused)
188 {
189 (void) unused;
190 dnode_t *dn = arg;
191
192 rw_destroy(&dn->dn_struct_rwlock);
193 mutex_destroy(&dn->dn_mtx);
194 mutex_destroy(&dn->dn_dbufs_mtx);
195 cv_destroy(&dn->dn_notxholds);
196 cv_destroy(&dn->dn_nodnholds);
197 zfs_refcount_destroy(&dn->dn_holds);
198 zfs_refcount_destroy(&dn->dn_tx_holds);
199 ASSERT(!list_link_active(&dn->dn_link));
200
201 for (int i = 0; i < TXG_SIZE; i++) {
202 ASSERT(!multilist_link_active(&dn->dn_dirty_link[i]));
203 ASSERT3P(dn->dn_free_ranges[i], ==, NULL);
204 list_destroy(&dn->dn_dirty_records[i]);
205 ASSERT0(dn->dn_next_nblkptr[i]);
206 ASSERT0(dn->dn_next_nlevels[i]);
207 ASSERT0(dn->dn_next_indblkshift[i]);
208 ASSERT0(dn->dn_next_bonustype[i]);
209 ASSERT0(dn->dn_rm_spillblk[i]);
210 ASSERT0(dn->dn_next_bonuslen[i]);
211 ASSERT0(dn->dn_next_blksz[i]);
212 ASSERT0(dn->dn_next_maxblkid[i]);
213 }
214
215 ASSERT0(dn->dn_allocated_txg);
216 ASSERT0(dn->dn_free_txg);
217 ASSERT0(dn->dn_assigned_txg);
218 ASSERT0(dn->dn_dirty_txg);
219 ASSERT0(dn->dn_dirtyctx);
220 ASSERT3P(dn->dn_dirtyctx_firstset, ==, NULL);
221 ASSERT3P(dn->dn_bonus, ==, NULL);
222 ASSERT(!dn->dn_have_spill);
223 ASSERT3P(dn->dn_zio, ==, NULL);
224 ASSERT0(dn->dn_oldused);
225 ASSERT0(dn->dn_oldflags);
226 ASSERT0(dn->dn_olduid);
227 ASSERT0(dn->dn_oldgid);
228 ASSERT0(dn->dn_oldprojid);
229 ASSERT0(dn->dn_newuid);
230 ASSERT0(dn->dn_newgid);
231 ASSERT0(dn->dn_newprojid);
232 ASSERT0(dn->dn_id_flags);
233
234 ASSERT0(dn->dn_dbufs_count);
235 avl_destroy(&dn->dn_dbufs);
236 }
237
238 static int
dnode_kstats_update(kstat_t * ksp,int rw)239 dnode_kstats_update(kstat_t *ksp, int rw)
240 {
241 dnode_stats_t *ds = ksp->ks_data;
242
243 if (rw == KSTAT_WRITE)
244 return (EACCES);
245 ds->dnode_hold_dbuf_hold.value.ui64 =
246 wmsum_value(&dnode_sums.dnode_hold_dbuf_hold);
247 ds->dnode_hold_dbuf_read.value.ui64 =
248 wmsum_value(&dnode_sums.dnode_hold_dbuf_read);
249 ds->dnode_hold_alloc_hits.value.ui64 =
250 wmsum_value(&dnode_sums.dnode_hold_alloc_hits);
251 ds->dnode_hold_alloc_misses.value.ui64 =
252 wmsum_value(&dnode_sums.dnode_hold_alloc_misses);
253 ds->dnode_hold_alloc_interior.value.ui64 =
254 wmsum_value(&dnode_sums.dnode_hold_alloc_interior);
255 ds->dnode_hold_alloc_lock_retry.value.ui64 =
256 wmsum_value(&dnode_sums.dnode_hold_alloc_lock_retry);
257 ds->dnode_hold_alloc_lock_misses.value.ui64 =
258 wmsum_value(&dnode_sums.dnode_hold_alloc_lock_misses);
259 ds->dnode_hold_alloc_type_none.value.ui64 =
260 wmsum_value(&dnode_sums.dnode_hold_alloc_type_none);
261 ds->dnode_hold_free_hits.value.ui64 =
262 wmsum_value(&dnode_sums.dnode_hold_free_hits);
263 ds->dnode_hold_free_misses.value.ui64 =
264 wmsum_value(&dnode_sums.dnode_hold_free_misses);
265 ds->dnode_hold_free_lock_misses.value.ui64 =
266 wmsum_value(&dnode_sums.dnode_hold_free_lock_misses);
267 ds->dnode_hold_free_lock_retry.value.ui64 =
268 wmsum_value(&dnode_sums.dnode_hold_free_lock_retry);
269 ds->dnode_hold_free_refcount.value.ui64 =
270 wmsum_value(&dnode_sums.dnode_hold_free_refcount);
271 ds->dnode_hold_free_overflow.value.ui64 =
272 wmsum_value(&dnode_sums.dnode_hold_free_overflow);
273 ds->dnode_free_interior_lock_retry.value.ui64 =
274 wmsum_value(&dnode_sums.dnode_free_interior_lock_retry);
275 ds->dnode_allocate.value.ui64 =
276 wmsum_value(&dnode_sums.dnode_allocate);
277 ds->dnode_reallocate.value.ui64 =
278 wmsum_value(&dnode_sums.dnode_reallocate);
279 ds->dnode_buf_evict.value.ui64 =
280 wmsum_value(&dnode_sums.dnode_buf_evict);
281 ds->dnode_alloc_next_chunk.value.ui64 =
282 wmsum_value(&dnode_sums.dnode_alloc_next_chunk);
283 ds->dnode_alloc_race.value.ui64 =
284 wmsum_value(&dnode_sums.dnode_alloc_race);
285 ds->dnode_alloc_next_block.value.ui64 =
286 wmsum_value(&dnode_sums.dnode_alloc_next_block);
287 ds->dnode_move_invalid.value.ui64 =
288 wmsum_value(&dnode_sums.dnode_move_invalid);
289 ds->dnode_move_recheck1.value.ui64 =
290 wmsum_value(&dnode_sums.dnode_move_recheck1);
291 ds->dnode_move_recheck2.value.ui64 =
292 wmsum_value(&dnode_sums.dnode_move_recheck2);
293 ds->dnode_move_special.value.ui64 =
294 wmsum_value(&dnode_sums.dnode_move_special);
295 ds->dnode_move_handle.value.ui64 =
296 wmsum_value(&dnode_sums.dnode_move_handle);
297 ds->dnode_move_rwlock.value.ui64 =
298 wmsum_value(&dnode_sums.dnode_move_rwlock);
299 ds->dnode_move_active.value.ui64 =
300 wmsum_value(&dnode_sums.dnode_move_active);
301 return (0);
302 }
303
304 void
dnode_init(void)305 dnode_init(void)
306 {
307 ASSERT(dnode_cache == NULL);
308 dnode_cache = kmem_cache_create("dnode_t", sizeof (dnode_t),
309 0, dnode_cons, dnode_dest, NULL, NULL, NULL, KMC_RECLAIMABLE);
310 kmem_cache_set_move(dnode_cache, dnode_move);
311
312 wmsum_init(&dnode_sums.dnode_hold_dbuf_hold, 0);
313 wmsum_init(&dnode_sums.dnode_hold_dbuf_read, 0);
314 wmsum_init(&dnode_sums.dnode_hold_alloc_hits, 0);
315 wmsum_init(&dnode_sums.dnode_hold_alloc_misses, 0);
316 wmsum_init(&dnode_sums.dnode_hold_alloc_interior, 0);
317 wmsum_init(&dnode_sums.dnode_hold_alloc_lock_retry, 0);
318 wmsum_init(&dnode_sums.dnode_hold_alloc_lock_misses, 0);
319 wmsum_init(&dnode_sums.dnode_hold_alloc_type_none, 0);
320 wmsum_init(&dnode_sums.dnode_hold_free_hits, 0);
321 wmsum_init(&dnode_sums.dnode_hold_free_misses, 0);
322 wmsum_init(&dnode_sums.dnode_hold_free_lock_misses, 0);
323 wmsum_init(&dnode_sums.dnode_hold_free_lock_retry, 0);
324 wmsum_init(&dnode_sums.dnode_hold_free_refcount, 0);
325 wmsum_init(&dnode_sums.dnode_hold_free_overflow, 0);
326 wmsum_init(&dnode_sums.dnode_free_interior_lock_retry, 0);
327 wmsum_init(&dnode_sums.dnode_allocate, 0);
328 wmsum_init(&dnode_sums.dnode_reallocate, 0);
329 wmsum_init(&dnode_sums.dnode_buf_evict, 0);
330 wmsum_init(&dnode_sums.dnode_alloc_next_chunk, 0);
331 wmsum_init(&dnode_sums.dnode_alloc_race, 0);
332 wmsum_init(&dnode_sums.dnode_alloc_next_block, 0);
333 wmsum_init(&dnode_sums.dnode_move_invalid, 0);
334 wmsum_init(&dnode_sums.dnode_move_recheck1, 0);
335 wmsum_init(&dnode_sums.dnode_move_recheck2, 0);
336 wmsum_init(&dnode_sums.dnode_move_special, 0);
337 wmsum_init(&dnode_sums.dnode_move_handle, 0);
338 wmsum_init(&dnode_sums.dnode_move_rwlock, 0);
339 wmsum_init(&dnode_sums.dnode_move_active, 0);
340
341 dnode_ksp = kstat_create("zfs", 0, "dnodestats", "misc",
342 KSTAT_TYPE_NAMED, sizeof (dnode_stats) / sizeof (kstat_named_t),
343 KSTAT_FLAG_VIRTUAL);
344 if (dnode_ksp != NULL) {
345 dnode_ksp->ks_data = &dnode_stats;
346 dnode_ksp->ks_update = dnode_kstats_update;
347 kstat_install(dnode_ksp);
348 }
349 }
350
351 void
dnode_fini(void)352 dnode_fini(void)
353 {
354 if (dnode_ksp != NULL) {
355 kstat_delete(dnode_ksp);
356 dnode_ksp = NULL;
357 }
358
359 wmsum_fini(&dnode_sums.dnode_hold_dbuf_hold);
360 wmsum_fini(&dnode_sums.dnode_hold_dbuf_read);
361 wmsum_fini(&dnode_sums.dnode_hold_alloc_hits);
362 wmsum_fini(&dnode_sums.dnode_hold_alloc_misses);
363 wmsum_fini(&dnode_sums.dnode_hold_alloc_interior);
364 wmsum_fini(&dnode_sums.dnode_hold_alloc_lock_retry);
365 wmsum_fini(&dnode_sums.dnode_hold_alloc_lock_misses);
366 wmsum_fini(&dnode_sums.dnode_hold_alloc_type_none);
367 wmsum_fini(&dnode_sums.dnode_hold_free_hits);
368 wmsum_fini(&dnode_sums.dnode_hold_free_misses);
369 wmsum_fini(&dnode_sums.dnode_hold_free_lock_misses);
370 wmsum_fini(&dnode_sums.dnode_hold_free_lock_retry);
371 wmsum_fini(&dnode_sums.dnode_hold_free_refcount);
372 wmsum_fini(&dnode_sums.dnode_hold_free_overflow);
373 wmsum_fini(&dnode_sums.dnode_free_interior_lock_retry);
374 wmsum_fini(&dnode_sums.dnode_allocate);
375 wmsum_fini(&dnode_sums.dnode_reallocate);
376 wmsum_fini(&dnode_sums.dnode_buf_evict);
377 wmsum_fini(&dnode_sums.dnode_alloc_next_chunk);
378 wmsum_fini(&dnode_sums.dnode_alloc_race);
379 wmsum_fini(&dnode_sums.dnode_alloc_next_block);
380 wmsum_fini(&dnode_sums.dnode_move_invalid);
381 wmsum_fini(&dnode_sums.dnode_move_recheck1);
382 wmsum_fini(&dnode_sums.dnode_move_recheck2);
383 wmsum_fini(&dnode_sums.dnode_move_special);
384 wmsum_fini(&dnode_sums.dnode_move_handle);
385 wmsum_fini(&dnode_sums.dnode_move_rwlock);
386 wmsum_fini(&dnode_sums.dnode_move_active);
387
388 kmem_cache_destroy(dnode_cache);
389 dnode_cache = NULL;
390 }
391
392
393 #ifdef ZFS_DEBUG
394 void
dnode_verify(dnode_t * dn)395 dnode_verify(dnode_t *dn)
396 {
397 int drop_struct_lock = FALSE;
398
399 ASSERT(dn->dn_phys);
400 ASSERT(dn->dn_objset);
401 ASSERT(dn->dn_handle->dnh_dnode == dn);
402
403 ASSERT(DMU_OT_IS_VALID(dn->dn_phys->dn_type));
404
405 if (!(zfs_flags & ZFS_DEBUG_DNODE_VERIFY))
406 return;
407
408 if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) {
409 rw_enter(&dn->dn_struct_rwlock, RW_READER);
410 drop_struct_lock = TRUE;
411 }
412 if (dn->dn_phys->dn_type != DMU_OT_NONE || dn->dn_allocated_txg != 0) {
413 int i;
414 int max_bonuslen = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots);
415 ASSERT3U(dn->dn_indblkshift, <=, SPA_MAXBLOCKSHIFT);
416 if (dn->dn_datablkshift) {
417 ASSERT3U(dn->dn_datablkshift, >=, SPA_MINBLOCKSHIFT);
418 ASSERT3U(dn->dn_datablkshift, <=, SPA_MAXBLOCKSHIFT);
419 ASSERT3U(1<<dn->dn_datablkshift, ==, dn->dn_datablksz);
420 }
421 ASSERT3U(dn->dn_nlevels, <=, 30);
422 ASSERT(DMU_OT_IS_VALID(dn->dn_type));
423 ASSERT3U(dn->dn_nblkptr, >=, 1);
424 ASSERT3U(dn->dn_nblkptr, <=, DN_MAX_NBLKPTR);
425 ASSERT3U(dn->dn_bonuslen, <=, max_bonuslen);
426 ASSERT3U(dn->dn_datablksz, ==,
427 dn->dn_datablkszsec << SPA_MINBLOCKSHIFT);
428 ASSERT3U(ISP2(dn->dn_datablksz), ==, dn->dn_datablkshift != 0);
429 ASSERT3U((dn->dn_nblkptr - 1) * sizeof (blkptr_t) +
430 dn->dn_bonuslen, <=, max_bonuslen);
431 for (i = 0; i < TXG_SIZE; i++) {
432 ASSERT3U(dn->dn_next_nlevels[i], <=, dn->dn_nlevels);
433 }
434 }
435 if (dn->dn_phys->dn_type != DMU_OT_NONE)
436 ASSERT3U(dn->dn_phys->dn_nlevels, <=, dn->dn_nlevels);
437 ASSERT(DMU_OBJECT_IS_SPECIAL(dn->dn_object) || dn->dn_dbuf != NULL);
438 if (dn->dn_dbuf != NULL) {
439 ASSERT3P(dn->dn_phys, ==,
440 (dnode_phys_t *)dn->dn_dbuf->db.db_data +
441 (dn->dn_object % (dn->dn_dbuf->db.db_size >> DNODE_SHIFT)));
442 }
443 if (drop_struct_lock)
444 rw_exit(&dn->dn_struct_rwlock);
445 }
446 #endif
447
448 void
dnode_byteswap(dnode_phys_t * dnp)449 dnode_byteswap(dnode_phys_t *dnp)
450 {
451 uint64_t *buf64 = (void*)&dnp->dn_blkptr;
452 int i;
453
454 if (dnp->dn_type == DMU_OT_NONE) {
455 memset(dnp, 0, sizeof (dnode_phys_t));
456 return;
457 }
458
459 dnp->dn_datablkszsec = BSWAP_16(dnp->dn_datablkszsec);
460 dnp->dn_bonuslen = BSWAP_16(dnp->dn_bonuslen);
461 dnp->dn_extra_slots = BSWAP_8(dnp->dn_extra_slots);
462 dnp->dn_maxblkid = BSWAP_64(dnp->dn_maxblkid);
463 dnp->dn_used = BSWAP_64(dnp->dn_used);
464
465 /*
466 * dn_nblkptr is only one byte, so it's OK to read it in either
467 * byte order. We can't read dn_bouslen.
468 */
469 ASSERT(dnp->dn_indblkshift <= SPA_MAXBLOCKSHIFT);
470 ASSERT(dnp->dn_nblkptr <= DN_MAX_NBLKPTR);
471 for (i = 0; i < dnp->dn_nblkptr * sizeof (blkptr_t)/8; i++)
472 buf64[i] = BSWAP_64(buf64[i]);
473
474 /*
475 * OK to check dn_bonuslen for zero, because it won't matter if
476 * we have the wrong byte order. This is necessary because the
477 * dnode dnode is smaller than a regular dnode.
478 */
479 if (dnp->dn_bonuslen != 0) {
480 dmu_object_byteswap_t byteswap;
481 ASSERT(DMU_OT_IS_VALID(dnp->dn_bonustype));
482 byteswap = DMU_OT_BYTESWAP(dnp->dn_bonustype);
483 dmu_ot_byteswap[byteswap].ob_func(DN_BONUS(dnp),
484 DN_MAX_BONUS_LEN(dnp));
485 }
486
487 /* Swap SPILL block if we have one */
488 if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR)
489 byteswap_uint64_array(DN_SPILL_BLKPTR(dnp), sizeof (blkptr_t));
490 }
491
492 void
dnode_buf_byteswap(void * vbuf,size_t size)493 dnode_buf_byteswap(void *vbuf, size_t size)
494 {
495 int i = 0;
496
497 ASSERT3U(sizeof (dnode_phys_t), ==, (1<<DNODE_SHIFT));
498 ASSERT((size & (sizeof (dnode_phys_t)-1)) == 0);
499
500 while (i < size) {
501 dnode_phys_t *dnp = (void *)(((char *)vbuf) + i);
502 dnode_byteswap(dnp);
503
504 i += DNODE_MIN_SIZE;
505 if (dnp->dn_type != DMU_OT_NONE)
506 i += dnp->dn_extra_slots * DNODE_MIN_SIZE;
507 }
508 }
509
510 void
dnode_setbonuslen(dnode_t * dn,int newsize,dmu_tx_t * tx)511 dnode_setbonuslen(dnode_t *dn, int newsize, dmu_tx_t *tx)
512 {
513 ASSERT3U(zfs_refcount_count(&dn->dn_holds), >=, 1);
514
515 dnode_setdirty(dn, tx);
516 rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
517 ASSERT3U(newsize, <=, DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots) -
518 (dn->dn_nblkptr-1) * sizeof (blkptr_t));
519
520 if (newsize < dn->dn_bonuslen) {
521 /* clear any data after the end of the new size */
522 size_t diff = dn->dn_bonuslen - newsize;
523 char *data_end = ((char *)dn->dn_bonus->db.db_data) + newsize;
524 memset(data_end, 0, diff);
525 }
526
527 dn->dn_bonuslen = newsize;
528 if (newsize == 0)
529 dn->dn_next_bonuslen[tx->tx_txg & TXG_MASK] = DN_ZERO_BONUSLEN;
530 else
531 dn->dn_next_bonuslen[tx->tx_txg & TXG_MASK] = dn->dn_bonuslen;
532 rw_exit(&dn->dn_struct_rwlock);
533 }
534
535 void
dnode_setbonus_type(dnode_t * dn,dmu_object_type_t newtype,dmu_tx_t * tx)536 dnode_setbonus_type(dnode_t *dn, dmu_object_type_t newtype, dmu_tx_t *tx)
537 {
538 ASSERT3U(zfs_refcount_count(&dn->dn_holds), >=, 1);
539 dnode_setdirty(dn, tx);
540 rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
541 dn->dn_bonustype = newtype;
542 dn->dn_next_bonustype[tx->tx_txg & TXG_MASK] = dn->dn_bonustype;
543 rw_exit(&dn->dn_struct_rwlock);
544 }
545
546 void
dnode_set_storage_type(dnode_t * dn,dmu_object_type_t newtype)547 dnode_set_storage_type(dnode_t *dn, dmu_object_type_t newtype)
548 {
549 /*
550 * This is not in the dnode_phys, but it should be, and perhaps one day
551 * will. For now we require it be set after taking a hold.
552 */
553 ASSERT3U(zfs_refcount_count(&dn->dn_holds), >=, 1);
554 dn->dn_storage_type = newtype;
555 }
556
557 void
dnode_rm_spill(dnode_t * dn,dmu_tx_t * tx)558 dnode_rm_spill(dnode_t *dn, dmu_tx_t *tx)
559 {
560 ASSERT3U(zfs_refcount_count(&dn->dn_holds), >=, 1);
561 ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock));
562 dnode_setdirty(dn, tx);
563 dn->dn_rm_spillblk[tx->tx_txg & TXG_MASK] = DN_KILL_SPILLBLK;
564 dn->dn_have_spill = B_FALSE;
565 }
566
567 static void
dnode_setdblksz(dnode_t * dn,int size)568 dnode_setdblksz(dnode_t *dn, int size)
569 {
570 ASSERT0(P2PHASE(size, SPA_MINBLOCKSIZE));
571 ASSERT3U(size, <=, SPA_MAXBLOCKSIZE);
572 ASSERT3U(size, >=, SPA_MINBLOCKSIZE);
573 ASSERT3U(size >> SPA_MINBLOCKSHIFT, <,
574 1<<(sizeof (dn->dn_phys->dn_datablkszsec) * 8));
575 dn->dn_datablksz = size;
576 dn->dn_datablkszsec = size >> SPA_MINBLOCKSHIFT;
577 dn->dn_datablkshift = ISP2(size) ? highbit64(size - 1) : 0;
578 }
579
580 static dnode_t *
dnode_create(objset_t * os,dnode_phys_t * dnp,dmu_buf_impl_t * db,uint64_t object,dnode_handle_t * dnh)581 dnode_create(objset_t *os, dnode_phys_t *dnp, dmu_buf_impl_t *db,
582 uint64_t object, dnode_handle_t *dnh)
583 {
584 dnode_t *dn;
585
586 dn = kmem_cache_alloc(dnode_cache, KM_SLEEP);
587 dn->dn_moved = 0;
588
589 /*
590 * Defer setting dn_objset until the dnode is ready to be a candidate
591 * for the dnode_move() callback.
592 */
593 dn->dn_object = object;
594 dn->dn_dbuf = db;
595 dn->dn_handle = dnh;
596 dn->dn_phys = dnp;
597
598 if (dnp->dn_datablkszsec) {
599 dnode_setdblksz(dn, dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT);
600 } else {
601 dn->dn_datablksz = 0;
602 dn->dn_datablkszsec = 0;
603 dn->dn_datablkshift = 0;
604 }
605 dn->dn_indblkshift = dnp->dn_indblkshift;
606 dn->dn_nlevels = dnp->dn_nlevels;
607 dn->dn_type = dnp->dn_type;
608 dn->dn_nblkptr = dnp->dn_nblkptr;
609 dn->dn_checksum = dnp->dn_checksum;
610 dn->dn_compress = dnp->dn_compress;
611 dn->dn_bonustype = dnp->dn_bonustype;
612 dn->dn_bonuslen = dnp->dn_bonuslen;
613 dn->dn_num_slots = dnp->dn_extra_slots + 1;
614 dn->dn_maxblkid = dnp->dn_maxblkid;
615 dn->dn_have_spill = ((dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) != 0);
616 dn->dn_id_flags = 0;
617
618 dn->dn_storage_type = DMU_OT_NONE;
619
620 dmu_zfetch_init(&dn->dn_zfetch, dn);
621
622 ASSERT(DMU_OT_IS_VALID(dn->dn_phys->dn_type));
623 ASSERT(zrl_is_locked(&dnh->dnh_zrlock));
624 ASSERT(!DN_SLOT_IS_PTR(dnh->dnh_dnode));
625
626 mutex_enter(&os->os_lock);
627
628 /*
629 * Exclude special dnodes from os_dnodes so an empty os_dnodes
630 * signifies that the special dnodes have no references from
631 * their children (the entries in os_dnodes). This allows
632 * dnode_destroy() to easily determine if the last child has
633 * been removed and then complete eviction of the objset.
634 */
635 if (!DMU_OBJECT_IS_SPECIAL(object))
636 list_insert_head(&os->os_dnodes, dn);
637 membar_producer();
638
639 /*
640 * Everything else must be valid before assigning dn_objset
641 * makes the dnode eligible for dnode_move().
642 */
643 dn->dn_objset = os;
644
645 dnh->dnh_dnode = dn;
646 mutex_exit(&os->os_lock);
647
648 arc_space_consume(sizeof (dnode_t), ARC_SPACE_DNODE);
649
650 return (dn);
651 }
652
653 /*
654 * Caller must be holding the dnode handle, which is released upon return.
655 */
656 static void
dnode_destroy(dnode_t * dn)657 dnode_destroy(dnode_t *dn)
658 {
659 objset_t *os = dn->dn_objset;
660 boolean_t complete_os_eviction = B_FALSE;
661
662 ASSERT((dn->dn_id_flags & DN_ID_NEW_EXIST) == 0);
663
664 mutex_enter(&os->os_lock);
665 POINTER_INVALIDATE(&dn->dn_objset);
666 if (!DMU_OBJECT_IS_SPECIAL(dn->dn_object)) {
667 list_remove(&os->os_dnodes, dn);
668 complete_os_eviction =
669 list_is_empty(&os->os_dnodes) &&
670 list_link_active(&os->os_evicting_node);
671 }
672 mutex_exit(&os->os_lock);
673
674 /* the dnode can no longer move, so we can release the handle */
675 if (!zrl_is_locked(&dn->dn_handle->dnh_zrlock))
676 zrl_remove(&dn->dn_handle->dnh_zrlock);
677
678 dn->dn_allocated_txg = 0;
679 dn->dn_free_txg = 0;
680 dn->dn_assigned_txg = 0;
681 dn->dn_dirty_txg = 0;
682
683 dn->dn_dirtyctx = 0;
684 dn->dn_dirtyctx_firstset = NULL;
685 if (dn->dn_bonus != NULL) {
686 mutex_enter(&dn->dn_bonus->db_mtx);
687 dbuf_destroy(dn->dn_bonus);
688 dn->dn_bonus = NULL;
689 }
690 dn->dn_zio = NULL;
691
692 dn->dn_have_spill = B_FALSE;
693 dn->dn_oldused = 0;
694 dn->dn_oldflags = 0;
695 dn->dn_olduid = 0;
696 dn->dn_oldgid = 0;
697 dn->dn_oldprojid = ZFS_DEFAULT_PROJID;
698 dn->dn_newuid = 0;
699 dn->dn_newgid = 0;
700 dn->dn_newprojid = ZFS_DEFAULT_PROJID;
701 dn->dn_id_flags = 0;
702
703 dn->dn_storage_type = DMU_OT_NONE;
704
705 dmu_zfetch_fini(&dn->dn_zfetch);
706 kmem_cache_free(dnode_cache, dn);
707 arc_space_return(sizeof (dnode_t), ARC_SPACE_DNODE);
708
709 if (complete_os_eviction)
710 dmu_objset_evict_done(os);
711 }
712
713 void
dnode_allocate(dnode_t * dn,dmu_object_type_t ot,int blocksize,int ibs,dmu_object_type_t bonustype,int bonuslen,int dn_slots,dmu_tx_t * tx)714 dnode_allocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, int ibs,
715 dmu_object_type_t bonustype, int bonuslen, int dn_slots, dmu_tx_t *tx)
716 {
717 int i;
718
719 ASSERT3U(dn_slots, >, 0);
720 ASSERT3U(dn_slots << DNODE_SHIFT, <=,
721 spa_maxdnodesize(dmu_objset_spa(dn->dn_objset)));
722 ASSERT3U(blocksize, <=,
723 spa_maxblocksize(dmu_objset_spa(dn->dn_objset)));
724 if (blocksize == 0)
725 blocksize = 1 << zfs_default_bs;
726 else
727 blocksize = P2ROUNDUP(blocksize, SPA_MINBLOCKSIZE);
728
729 if (ibs == 0)
730 ibs = zfs_default_ibs;
731
732 ibs = MIN(MAX(ibs, DN_MIN_INDBLKSHIFT), DN_MAX_INDBLKSHIFT);
733
734 dprintf("os=%p obj=%llu txg=%llu blocksize=%d ibs=%d dn_slots=%d\n",
735 dn->dn_objset, (u_longlong_t)dn->dn_object,
736 (u_longlong_t)tx->tx_txg, blocksize, ibs, dn_slots);
737 DNODE_STAT_BUMP(dnode_allocate);
738
739 ASSERT(dn->dn_type == DMU_OT_NONE);
740 ASSERT0(memcmp(dn->dn_phys, &dnode_phys_zero, sizeof (dnode_phys_t)));
741 ASSERT(dn->dn_phys->dn_type == DMU_OT_NONE);
742 ASSERT(ot != DMU_OT_NONE);
743 ASSERT(DMU_OT_IS_VALID(ot));
744 ASSERT((bonustype == DMU_OT_NONE && bonuslen == 0) ||
745 (bonustype == DMU_OT_SA && bonuslen == 0) ||
746 (bonustype == DMU_OTN_UINT64_METADATA && bonuslen == 0) ||
747 (bonustype != DMU_OT_NONE && bonuslen != 0));
748 ASSERT(DMU_OT_IS_VALID(bonustype));
749 ASSERT3U(bonuslen, <=, DN_SLOTS_TO_BONUSLEN(dn_slots));
750 ASSERT(dn->dn_type == DMU_OT_NONE);
751 ASSERT0(dn->dn_maxblkid);
752 ASSERT0(dn->dn_allocated_txg);
753 ASSERT0(dn->dn_assigned_txg);
754 ASSERT(zfs_refcount_is_zero(&dn->dn_tx_holds));
755 ASSERT3U(zfs_refcount_count(&dn->dn_holds), <=, 1);
756 ASSERT(avl_is_empty(&dn->dn_dbufs));
757
758 for (i = 0; i < TXG_SIZE; i++) {
759 ASSERT0(dn->dn_next_nblkptr[i]);
760 ASSERT0(dn->dn_next_nlevels[i]);
761 ASSERT0(dn->dn_next_indblkshift[i]);
762 ASSERT0(dn->dn_next_bonuslen[i]);
763 ASSERT0(dn->dn_next_bonustype[i]);
764 ASSERT0(dn->dn_rm_spillblk[i]);
765 ASSERT0(dn->dn_next_blksz[i]);
766 ASSERT0(dn->dn_next_maxblkid[i]);
767 ASSERT(!multilist_link_active(&dn->dn_dirty_link[i]));
768 ASSERT3P(list_head(&dn->dn_dirty_records[i]), ==, NULL);
769 ASSERT3P(dn->dn_free_ranges[i], ==, NULL);
770 }
771
772 dn->dn_type = ot;
773 dnode_setdblksz(dn, blocksize);
774 dn->dn_indblkshift = ibs;
775 dn->dn_nlevels = 1;
776 dn->dn_num_slots = dn_slots;
777 if (bonustype == DMU_OT_SA) /* Maximize bonus space for SA */
778 dn->dn_nblkptr = 1;
779 else {
780 dn->dn_nblkptr = MIN(DN_MAX_NBLKPTR,
781 1 + ((DN_SLOTS_TO_BONUSLEN(dn_slots) - bonuslen) >>
782 SPA_BLKPTRSHIFT));
783 }
784
785 dn->dn_bonustype = bonustype;
786 dn->dn_bonuslen = bonuslen;
787 dn->dn_checksum = ZIO_CHECKSUM_INHERIT;
788 dn->dn_compress = ZIO_COMPRESS_INHERIT;
789 dn->dn_dirtyctx = 0;
790
791 dn->dn_free_txg = 0;
792 dn->dn_dirtyctx_firstset = NULL;
793 dn->dn_dirty_txg = 0;
794
795 dn->dn_allocated_txg = tx->tx_txg;
796 dn->dn_id_flags = 0;
797
798 dnode_setdirty(dn, tx);
799 dn->dn_next_indblkshift[tx->tx_txg & TXG_MASK] = ibs;
800 dn->dn_next_bonuslen[tx->tx_txg & TXG_MASK] = dn->dn_bonuslen;
801 dn->dn_next_bonustype[tx->tx_txg & TXG_MASK] = dn->dn_bonustype;
802 dn->dn_next_blksz[tx->tx_txg & TXG_MASK] = dn->dn_datablksz;
803 }
804
805 void
dnode_reallocate(dnode_t * dn,dmu_object_type_t ot,int blocksize,dmu_object_type_t bonustype,int bonuslen,int dn_slots,boolean_t keep_spill,dmu_tx_t * tx)806 dnode_reallocate(dnode_t *dn, dmu_object_type_t ot, int blocksize,
807 dmu_object_type_t bonustype, int bonuslen, int dn_slots,
808 boolean_t keep_spill, dmu_tx_t *tx)
809 {
810 int nblkptr;
811
812 ASSERT3U(blocksize, >=, SPA_MINBLOCKSIZE);
813 ASSERT3U(blocksize, <=,
814 spa_maxblocksize(dmu_objset_spa(dn->dn_objset)));
815 ASSERT0(blocksize % SPA_MINBLOCKSIZE);
816 ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT || dmu_tx_private_ok(tx));
817 ASSERT(tx->tx_txg != 0);
818 ASSERT((bonustype == DMU_OT_NONE && bonuslen == 0) ||
819 (bonustype != DMU_OT_NONE && bonuslen != 0) ||
820 (bonustype == DMU_OT_SA && bonuslen == 0));
821 ASSERT(DMU_OT_IS_VALID(bonustype));
822 ASSERT3U(bonuslen, <=,
823 DN_BONUS_SIZE(spa_maxdnodesize(dmu_objset_spa(dn->dn_objset))));
824 ASSERT3U(bonuslen, <=, DN_BONUS_SIZE(dn_slots << DNODE_SHIFT));
825
826 dnode_free_interior_slots(dn);
827 DNODE_STAT_BUMP(dnode_reallocate);
828
829 /* clean up any unreferenced dbufs */
830 dnode_evict_dbufs(dn);
831
832 dn->dn_id_flags = 0;
833
834 rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
835 dnode_setdirty(dn, tx);
836 if (dn->dn_datablksz != blocksize) {
837 /* change blocksize */
838 ASSERT0(dn->dn_maxblkid);
839 ASSERT(BP_IS_HOLE(&dn->dn_phys->dn_blkptr[0]) ||
840 dnode_block_freed(dn, 0));
841
842 dnode_setdblksz(dn, blocksize);
843 dn->dn_next_blksz[tx->tx_txg & TXG_MASK] = blocksize;
844 }
845 if (dn->dn_bonuslen != bonuslen)
846 dn->dn_next_bonuslen[tx->tx_txg & TXG_MASK] = bonuslen;
847
848 if (bonustype == DMU_OT_SA) /* Maximize bonus space for SA */
849 nblkptr = 1;
850 else
851 nblkptr = MIN(DN_MAX_NBLKPTR,
852 1 + ((DN_SLOTS_TO_BONUSLEN(dn_slots) - bonuslen) >>
853 SPA_BLKPTRSHIFT));
854 if (dn->dn_bonustype != bonustype)
855 dn->dn_next_bonustype[tx->tx_txg & TXG_MASK] = bonustype;
856 if (dn->dn_nblkptr != nblkptr)
857 dn->dn_next_nblkptr[tx->tx_txg & TXG_MASK] = nblkptr;
858 if (dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR && !keep_spill) {
859 dbuf_rm_spill(dn, tx);
860 dnode_rm_spill(dn, tx);
861 }
862
863 rw_exit(&dn->dn_struct_rwlock);
864
865 /* change type */
866 dn->dn_type = ot;
867
868 /* change bonus size and type */
869 mutex_enter(&dn->dn_mtx);
870 dn->dn_bonustype = bonustype;
871 dn->dn_bonuslen = bonuslen;
872 dn->dn_num_slots = dn_slots;
873 dn->dn_nblkptr = nblkptr;
874 dn->dn_checksum = ZIO_CHECKSUM_INHERIT;
875 dn->dn_compress = ZIO_COMPRESS_INHERIT;
876 ASSERT3U(dn->dn_nblkptr, <=, DN_MAX_NBLKPTR);
877
878 /* fix up the bonus db_size */
879 if (dn->dn_bonus) {
880 dn->dn_bonus->db.db_size =
881 DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots) -
882 (dn->dn_nblkptr-1) * sizeof (blkptr_t);
883 ASSERT(dn->dn_bonuslen <= dn->dn_bonus->db.db_size);
884 }
885
886 dn->dn_allocated_txg = tx->tx_txg;
887 mutex_exit(&dn->dn_mtx);
888 }
889
890 #ifdef _KERNEL
891 static void
dnode_move_impl(dnode_t * odn,dnode_t * ndn)892 dnode_move_impl(dnode_t *odn, dnode_t *ndn)
893 {
894 ASSERT(!RW_LOCK_HELD(&odn->dn_struct_rwlock));
895 ASSERT(MUTEX_NOT_HELD(&odn->dn_mtx));
896 ASSERT(MUTEX_NOT_HELD(&odn->dn_dbufs_mtx));
897
898 /* Copy fields. */
899 ndn->dn_objset = odn->dn_objset;
900 ndn->dn_object = odn->dn_object;
901 ndn->dn_dbuf = odn->dn_dbuf;
902 ndn->dn_handle = odn->dn_handle;
903 ndn->dn_phys = odn->dn_phys;
904 ndn->dn_type = odn->dn_type;
905 ndn->dn_bonuslen = odn->dn_bonuslen;
906 ndn->dn_bonustype = odn->dn_bonustype;
907 ndn->dn_nblkptr = odn->dn_nblkptr;
908 ndn->dn_checksum = odn->dn_checksum;
909 ndn->dn_compress = odn->dn_compress;
910 ndn->dn_nlevels = odn->dn_nlevels;
911 ndn->dn_indblkshift = odn->dn_indblkshift;
912 ndn->dn_datablkshift = odn->dn_datablkshift;
913 ndn->dn_datablkszsec = odn->dn_datablkszsec;
914 ndn->dn_datablksz = odn->dn_datablksz;
915 ndn->dn_maxblkid = odn->dn_maxblkid;
916 ndn->dn_num_slots = odn->dn_num_slots;
917 memcpy(ndn->dn_next_type, odn->dn_next_type,
918 sizeof (odn->dn_next_type));
919 memcpy(ndn->dn_next_nblkptr, odn->dn_next_nblkptr,
920 sizeof (odn->dn_next_nblkptr));
921 memcpy(ndn->dn_next_nlevels, odn->dn_next_nlevels,
922 sizeof (odn->dn_next_nlevels));
923 memcpy(ndn->dn_next_indblkshift, odn->dn_next_indblkshift,
924 sizeof (odn->dn_next_indblkshift));
925 memcpy(ndn->dn_next_bonustype, odn->dn_next_bonustype,
926 sizeof (odn->dn_next_bonustype));
927 memcpy(ndn->dn_rm_spillblk, odn->dn_rm_spillblk,
928 sizeof (odn->dn_rm_spillblk));
929 memcpy(ndn->dn_next_bonuslen, odn->dn_next_bonuslen,
930 sizeof (odn->dn_next_bonuslen));
931 memcpy(ndn->dn_next_blksz, odn->dn_next_blksz,
932 sizeof (odn->dn_next_blksz));
933 memcpy(ndn->dn_next_maxblkid, odn->dn_next_maxblkid,
934 sizeof (odn->dn_next_maxblkid));
935 for (int i = 0; i < TXG_SIZE; i++) {
936 list_move_tail(&ndn->dn_dirty_records[i],
937 &odn->dn_dirty_records[i]);
938 }
939 memcpy(ndn->dn_free_ranges, odn->dn_free_ranges,
940 sizeof (odn->dn_free_ranges));
941 ndn->dn_allocated_txg = odn->dn_allocated_txg;
942 ndn->dn_free_txg = odn->dn_free_txg;
943 ndn->dn_assigned_txg = odn->dn_assigned_txg;
944 ndn->dn_dirty_txg = odn->dn_dirty_txg;
945 ndn->dn_dirtyctx = odn->dn_dirtyctx;
946 ndn->dn_dirtyctx_firstset = odn->dn_dirtyctx_firstset;
947 ASSERT(zfs_refcount_count(&odn->dn_tx_holds) == 0);
948 zfs_refcount_transfer(&ndn->dn_holds, &odn->dn_holds);
949 ASSERT(avl_is_empty(&ndn->dn_dbufs));
950 avl_swap(&ndn->dn_dbufs, &odn->dn_dbufs);
951 ndn->dn_dbufs_count = odn->dn_dbufs_count;
952 ndn->dn_bonus = odn->dn_bonus;
953 ndn->dn_have_spill = odn->dn_have_spill;
954 ndn->dn_zio = odn->dn_zio;
955 ndn->dn_oldused = odn->dn_oldused;
956 ndn->dn_oldflags = odn->dn_oldflags;
957 ndn->dn_olduid = odn->dn_olduid;
958 ndn->dn_oldgid = odn->dn_oldgid;
959 ndn->dn_oldprojid = odn->dn_oldprojid;
960 ndn->dn_newuid = odn->dn_newuid;
961 ndn->dn_newgid = odn->dn_newgid;
962 ndn->dn_newprojid = odn->dn_newprojid;
963 ndn->dn_id_flags = odn->dn_id_flags;
964 ndn->dn_storage_type = odn->dn_storage_type;
965 dmu_zfetch_init(&ndn->dn_zfetch, ndn);
966
967 /*
968 * Update back pointers. Updating the handle fixes the back pointer of
969 * every descendant dbuf as well as the bonus dbuf.
970 */
971 ASSERT(ndn->dn_handle->dnh_dnode == odn);
972 ndn->dn_handle->dnh_dnode = ndn;
973
974 /*
975 * Invalidate the original dnode by clearing all of its back pointers.
976 */
977 odn->dn_dbuf = NULL;
978 odn->dn_handle = NULL;
979 avl_create(&odn->dn_dbufs, dbuf_compare, sizeof (dmu_buf_impl_t),
980 offsetof(dmu_buf_impl_t, db_link));
981 odn->dn_dbufs_count = 0;
982 odn->dn_bonus = NULL;
983 dmu_zfetch_fini(&odn->dn_zfetch);
984
985 /*
986 * Set the low bit of the objset pointer to ensure that dnode_move()
987 * recognizes the dnode as invalid in any subsequent callback.
988 */
989 POINTER_INVALIDATE(&odn->dn_objset);
990
991 /*
992 * Satisfy the destructor.
993 */
994 for (int i = 0; i < TXG_SIZE; i++) {
995 list_create(&odn->dn_dirty_records[i],
996 sizeof (dbuf_dirty_record_t),
997 offsetof(dbuf_dirty_record_t, dr_dirty_node));
998 odn->dn_free_ranges[i] = NULL;
999 odn->dn_next_nlevels[i] = 0;
1000 odn->dn_next_indblkshift[i] = 0;
1001 odn->dn_next_bonustype[i] = 0;
1002 odn->dn_rm_spillblk[i] = 0;
1003 odn->dn_next_bonuslen[i] = 0;
1004 odn->dn_next_blksz[i] = 0;
1005 }
1006 odn->dn_allocated_txg = 0;
1007 odn->dn_free_txg = 0;
1008 odn->dn_assigned_txg = 0;
1009 odn->dn_dirty_txg = 0;
1010 odn->dn_dirtyctx = 0;
1011 odn->dn_dirtyctx_firstset = NULL;
1012 odn->dn_have_spill = B_FALSE;
1013 odn->dn_zio = NULL;
1014 odn->dn_oldused = 0;
1015 odn->dn_oldflags = 0;
1016 odn->dn_olduid = 0;
1017 odn->dn_oldgid = 0;
1018 odn->dn_oldprojid = ZFS_DEFAULT_PROJID;
1019 odn->dn_newuid = 0;
1020 odn->dn_newgid = 0;
1021 odn->dn_newprojid = ZFS_DEFAULT_PROJID;
1022 odn->dn_id_flags = 0;
1023 odn->dn_storage_type = DMU_OT_NONE;
1024
1025 /*
1026 * Mark the dnode.
1027 */
1028 ndn->dn_moved = 1;
1029 odn->dn_moved = (uint8_t)-1;
1030 }
1031
1032 static kmem_cbrc_t
dnode_move(void * buf,void * newbuf,size_t size,void * arg)1033 dnode_move(void *buf, void *newbuf, size_t size, void *arg)
1034 {
1035 dnode_t *odn = buf, *ndn = newbuf;
1036 objset_t *os;
1037 int64_t refcount;
1038 uint32_t dbufs;
1039
1040 #ifndef USE_DNODE_HANDLE
1041 /*
1042 * We can't move dnodes if dbufs reference them directly without
1043 * using handles and respecitve locking. Unless USE_DNODE_HANDLE
1044 * is defined the code below is only to make sure it still builds,
1045 * but it should never be used, since it is unsafe.
1046 */
1047 #ifdef ZFS_DEBUG
1048 PANIC("dnode_move() called without USE_DNODE_HANDLE");
1049 #endif
1050 return (KMEM_CBRC_NO);
1051 #endif
1052
1053 /*
1054 * The dnode is on the objset's list of known dnodes if the objset
1055 * pointer is valid. We set the low bit of the objset pointer when
1056 * freeing the dnode to invalidate it, and the memory patterns written
1057 * by kmem (baddcafe and deadbeef) set at least one of the two low bits.
1058 * A newly created dnode sets the objset pointer last of all to indicate
1059 * that the dnode is known and in a valid state to be moved by this
1060 * function.
1061 */
1062 os = odn->dn_objset;
1063 if (!POINTER_IS_VALID(os)) {
1064 DNODE_STAT_BUMP(dnode_move_invalid);
1065 return (KMEM_CBRC_DONT_KNOW);
1066 }
1067
1068 /*
1069 * Ensure that the objset does not go away during the move.
1070 */
1071 rw_enter(&os_lock, RW_WRITER);
1072 if (os != odn->dn_objset) {
1073 rw_exit(&os_lock);
1074 DNODE_STAT_BUMP(dnode_move_recheck1);
1075 return (KMEM_CBRC_DONT_KNOW);
1076 }
1077
1078 /*
1079 * If the dnode is still valid, then so is the objset. We know that no
1080 * valid objset can be freed while we hold os_lock, so we can safely
1081 * ensure that the objset remains in use.
1082 */
1083 mutex_enter(&os->os_lock);
1084
1085 /*
1086 * Recheck the objset pointer in case the dnode was removed just before
1087 * acquiring the lock.
1088 */
1089 if (os != odn->dn_objset) {
1090 mutex_exit(&os->os_lock);
1091 rw_exit(&os_lock);
1092 DNODE_STAT_BUMP(dnode_move_recheck2);
1093 return (KMEM_CBRC_DONT_KNOW);
1094 }
1095
1096 /*
1097 * At this point we know that as long as we hold os->os_lock, the dnode
1098 * cannot be freed and fields within the dnode can be safely accessed.
1099 * The objset listing this dnode cannot go away as long as this dnode is
1100 * on its list.
1101 */
1102 rw_exit(&os_lock);
1103 if (DMU_OBJECT_IS_SPECIAL(odn->dn_object)) {
1104 mutex_exit(&os->os_lock);
1105 DNODE_STAT_BUMP(dnode_move_special);
1106 return (KMEM_CBRC_NO);
1107 }
1108 ASSERT(odn->dn_dbuf != NULL); /* only "special" dnodes have no parent */
1109
1110 /*
1111 * Lock the dnode handle to prevent the dnode from obtaining any new
1112 * holds. This also prevents the descendant dbufs and the bonus dbuf
1113 * from accessing the dnode, so that we can discount their holds. The
1114 * handle is safe to access because we know that while the dnode cannot
1115 * go away, neither can its handle. Once we hold dnh_zrlock, we can
1116 * safely move any dnode referenced only by dbufs.
1117 */
1118 if (!zrl_tryenter(&odn->dn_handle->dnh_zrlock)) {
1119 mutex_exit(&os->os_lock);
1120 DNODE_STAT_BUMP(dnode_move_handle);
1121 return (KMEM_CBRC_LATER);
1122 }
1123
1124 /*
1125 * Ensure a consistent view of the dnode's holds and the dnode's dbufs.
1126 * We need to guarantee that there is a hold for every dbuf in order to
1127 * determine whether the dnode is actively referenced. Falsely matching
1128 * a dbuf to an active hold would lead to an unsafe move. It's possible
1129 * that a thread already having an active dnode hold is about to add a
1130 * dbuf, and we can't compare hold and dbuf counts while the add is in
1131 * progress.
1132 */
1133 if (!rw_tryenter(&odn->dn_struct_rwlock, RW_WRITER)) {
1134 zrl_exit(&odn->dn_handle->dnh_zrlock);
1135 mutex_exit(&os->os_lock);
1136 DNODE_STAT_BUMP(dnode_move_rwlock);
1137 return (KMEM_CBRC_LATER);
1138 }
1139
1140 /*
1141 * A dbuf may be removed (evicted) without an active dnode hold. In that
1142 * case, the dbuf count is decremented under the handle lock before the
1143 * dbuf's hold is released. This order ensures that if we count the hold
1144 * after the dbuf is removed but before its hold is released, we will
1145 * treat the unmatched hold as active and exit safely. If we count the
1146 * hold before the dbuf is removed, the hold is discounted, and the
1147 * removal is blocked until the move completes.
1148 */
1149 refcount = zfs_refcount_count(&odn->dn_holds);
1150 ASSERT(refcount >= 0);
1151 dbufs = DN_DBUFS_COUNT(odn);
1152
1153 /* We can't have more dbufs than dnode holds. */
1154 ASSERT3U(dbufs, <=, refcount);
1155 DTRACE_PROBE3(dnode__move, dnode_t *, odn, int64_t, refcount,
1156 uint32_t, dbufs);
1157
1158 if (refcount > dbufs) {
1159 rw_exit(&odn->dn_struct_rwlock);
1160 zrl_exit(&odn->dn_handle->dnh_zrlock);
1161 mutex_exit(&os->os_lock);
1162 DNODE_STAT_BUMP(dnode_move_active);
1163 return (KMEM_CBRC_LATER);
1164 }
1165
1166 rw_exit(&odn->dn_struct_rwlock);
1167
1168 /*
1169 * At this point we know that anyone with a hold on the dnode is not
1170 * actively referencing it. The dnode is known and in a valid state to
1171 * move. We're holding the locks needed to execute the critical section.
1172 */
1173 dnode_move_impl(odn, ndn);
1174
1175 list_link_replace(&odn->dn_link, &ndn->dn_link);
1176 /* If the dnode was safe to move, the refcount cannot have changed. */
1177 ASSERT(refcount == zfs_refcount_count(&ndn->dn_holds));
1178 ASSERT(dbufs == DN_DBUFS_COUNT(ndn));
1179 zrl_exit(&ndn->dn_handle->dnh_zrlock); /* handle has moved */
1180 mutex_exit(&os->os_lock);
1181
1182 return (KMEM_CBRC_YES);
1183 }
1184 #endif /* _KERNEL */
1185
1186 static void
dnode_slots_hold(dnode_children_t * children,int idx,int slots)1187 dnode_slots_hold(dnode_children_t *children, int idx, int slots)
1188 {
1189 ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK);
1190
1191 for (int i = idx; i < idx + slots; i++) {
1192 dnode_handle_t *dnh = &children->dnc_children[i];
1193 zrl_add(&dnh->dnh_zrlock);
1194 }
1195 }
1196
1197 static void
dnode_slots_rele(dnode_children_t * children,int idx,int slots)1198 dnode_slots_rele(dnode_children_t *children, int idx, int slots)
1199 {
1200 ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK);
1201
1202 for (int i = idx; i < idx + slots; i++) {
1203 dnode_handle_t *dnh = &children->dnc_children[i];
1204
1205 if (zrl_is_locked(&dnh->dnh_zrlock))
1206 zrl_exit(&dnh->dnh_zrlock);
1207 else
1208 zrl_remove(&dnh->dnh_zrlock);
1209 }
1210 }
1211
1212 static int
dnode_slots_tryenter(dnode_children_t * children,int idx,int slots)1213 dnode_slots_tryenter(dnode_children_t *children, int idx, int slots)
1214 {
1215 ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK);
1216
1217 for (int i = idx; i < idx + slots; i++) {
1218 dnode_handle_t *dnh = &children->dnc_children[i];
1219
1220 if (!zrl_tryenter(&dnh->dnh_zrlock)) {
1221 for (int j = idx; j < i; j++) {
1222 dnh = &children->dnc_children[j];
1223 zrl_exit(&dnh->dnh_zrlock);
1224 }
1225
1226 return (0);
1227 }
1228 }
1229
1230 return (1);
1231 }
1232
1233 static void
dnode_set_slots(dnode_children_t * children,int idx,int slots,void * ptr)1234 dnode_set_slots(dnode_children_t *children, int idx, int slots, void *ptr)
1235 {
1236 ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK);
1237
1238 for (int i = idx; i < idx + slots; i++) {
1239 dnode_handle_t *dnh = &children->dnc_children[i];
1240 dnh->dnh_dnode = ptr;
1241 }
1242 }
1243
1244 static boolean_t
dnode_check_slots_free(dnode_children_t * children,int idx,int slots)1245 dnode_check_slots_free(dnode_children_t *children, int idx, int slots)
1246 {
1247 ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK);
1248
1249 /*
1250 * If all dnode slots are either already free or
1251 * evictable return B_TRUE.
1252 */
1253 for (int i = idx; i < idx + slots; i++) {
1254 dnode_handle_t *dnh = &children->dnc_children[i];
1255 dnode_t *dn = dnh->dnh_dnode;
1256
1257 if (dn == DN_SLOT_FREE) {
1258 continue;
1259 } else if (DN_SLOT_IS_PTR(dn)) {
1260 mutex_enter(&dn->dn_mtx);
1261 boolean_t can_free = (dn->dn_type == DMU_OT_NONE &&
1262 zfs_refcount_is_zero(&dn->dn_holds) &&
1263 !DNODE_IS_DIRTY(dn));
1264 mutex_exit(&dn->dn_mtx);
1265
1266 if (!can_free)
1267 return (B_FALSE);
1268 else
1269 continue;
1270 } else {
1271 return (B_FALSE);
1272 }
1273 }
1274
1275 return (B_TRUE);
1276 }
1277
1278 static uint_t
dnode_reclaim_slots(dnode_children_t * children,int idx,int slots)1279 dnode_reclaim_slots(dnode_children_t *children, int idx, int slots)
1280 {
1281 uint_t reclaimed = 0;
1282
1283 ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK);
1284
1285 for (int i = idx; i < idx + slots; i++) {
1286 dnode_handle_t *dnh = &children->dnc_children[i];
1287
1288 ASSERT(zrl_is_locked(&dnh->dnh_zrlock));
1289
1290 if (DN_SLOT_IS_PTR(dnh->dnh_dnode)) {
1291 ASSERT3S(dnh->dnh_dnode->dn_type, ==, DMU_OT_NONE);
1292 dnode_destroy(dnh->dnh_dnode);
1293 dnh->dnh_dnode = DN_SLOT_FREE;
1294 reclaimed++;
1295 }
1296 }
1297
1298 return (reclaimed);
1299 }
1300
1301 void
dnode_free_interior_slots(dnode_t * dn)1302 dnode_free_interior_slots(dnode_t *dn)
1303 {
1304 dnode_children_t *children = dmu_buf_get_user(&dn->dn_dbuf->db);
1305 int epb = dn->dn_dbuf->db.db_size >> DNODE_SHIFT;
1306 int idx = (dn->dn_object & (epb - 1)) + 1;
1307 int slots = dn->dn_num_slots - 1;
1308
1309 if (slots == 0)
1310 return;
1311
1312 ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK);
1313
1314 while (!dnode_slots_tryenter(children, idx, slots)) {
1315 DNODE_STAT_BUMP(dnode_free_interior_lock_retry);
1316 kpreempt(KPREEMPT_SYNC);
1317 }
1318
1319 dnode_set_slots(children, idx, slots, DN_SLOT_FREE);
1320 dnode_slots_rele(children, idx, slots);
1321 }
1322
1323 void
dnode_special_close(dnode_handle_t * dnh)1324 dnode_special_close(dnode_handle_t *dnh)
1325 {
1326 dnode_t *dn = dnh->dnh_dnode;
1327
1328 /*
1329 * Ensure dnode_rele_and_unlock() has released dn_mtx, after final
1330 * zfs_refcount_remove()
1331 */
1332 mutex_enter(&dn->dn_mtx);
1333 if (zfs_refcount_count(&dn->dn_holds) > 0)
1334 cv_wait(&dn->dn_nodnholds, &dn->dn_mtx);
1335 mutex_exit(&dn->dn_mtx);
1336 ASSERT3U(zfs_refcount_count(&dn->dn_holds), ==, 0);
1337
1338 ASSERT(dn->dn_dbuf == NULL ||
1339 dmu_buf_get_user(&dn->dn_dbuf->db) == NULL);
1340 zrl_add(&dnh->dnh_zrlock);
1341 dnode_destroy(dn); /* implicit zrl_remove() */
1342 zrl_destroy(&dnh->dnh_zrlock);
1343 dnh->dnh_dnode = NULL;
1344 }
1345
1346 void
dnode_special_open(objset_t * os,dnode_phys_t * dnp,uint64_t object,dnode_handle_t * dnh)1347 dnode_special_open(objset_t *os, dnode_phys_t *dnp, uint64_t object,
1348 dnode_handle_t *dnh)
1349 {
1350 dnode_t *dn;
1351
1352 zrl_init(&dnh->dnh_zrlock);
1353 VERIFY3U(1, ==, zrl_tryenter(&dnh->dnh_zrlock));
1354
1355 dn = dnode_create(os, dnp, NULL, object, dnh);
1356 DNODE_VERIFY(dn);
1357
1358 zrl_exit(&dnh->dnh_zrlock);
1359 }
1360
1361 static void
dnode_buf_evict_async(void * dbu)1362 dnode_buf_evict_async(void *dbu)
1363 {
1364 dnode_children_t *dnc = dbu;
1365
1366 DNODE_STAT_BUMP(dnode_buf_evict);
1367
1368 for (int i = 0; i < dnc->dnc_count; i++) {
1369 dnode_handle_t *dnh = &dnc->dnc_children[i];
1370 dnode_t *dn;
1371
1372 /*
1373 * The dnode handle lock guards against the dnode moving to
1374 * another valid address, so there is no need here to guard
1375 * against changes to or from NULL.
1376 */
1377 if (!DN_SLOT_IS_PTR(dnh->dnh_dnode)) {
1378 zrl_destroy(&dnh->dnh_zrlock);
1379 dnh->dnh_dnode = DN_SLOT_UNINIT;
1380 continue;
1381 }
1382
1383 zrl_add(&dnh->dnh_zrlock);
1384 dn = dnh->dnh_dnode;
1385 /*
1386 * If there are holds on this dnode, then there should
1387 * be holds on the dnode's containing dbuf as well; thus
1388 * it wouldn't be eligible for eviction and this function
1389 * would not have been called.
1390 */
1391 ASSERT(zfs_refcount_is_zero(&dn->dn_holds));
1392 ASSERT(zfs_refcount_is_zero(&dn->dn_tx_holds));
1393
1394 dnode_destroy(dn); /* implicit zrl_remove() for first slot */
1395 zrl_destroy(&dnh->dnh_zrlock);
1396 dnh->dnh_dnode = DN_SLOT_UNINIT;
1397 }
1398 kmem_free(dnc, sizeof (dnode_children_t) +
1399 dnc->dnc_count * sizeof (dnode_handle_t));
1400 }
1401
1402 /*
1403 * When the DNODE_MUST_BE_FREE flag is set, the "slots" parameter is used
1404 * to ensure the hole at the specified object offset is large enough to
1405 * hold the dnode being created. The slots parameter is also used to ensure
1406 * a dnode does not span multiple dnode blocks. In both of these cases, if
1407 * a failure occurs, ENOSPC is returned. Keep in mind, these failure cases
1408 * are only possible when using DNODE_MUST_BE_FREE.
1409 *
1410 * If the DNODE_MUST_BE_ALLOCATED flag is set, "slots" must be 0.
1411 * dnode_hold_impl() will check if the requested dnode is already consumed
1412 * as an extra dnode slot by an large dnode, in which case it returns
1413 * ENOENT.
1414 *
1415 * If the DNODE_DRY_RUN flag is set, we don't actually hold the dnode, just
1416 * return whether the hold would succeed or not. tag and dnp should set to
1417 * NULL in this case.
1418 *
1419 * errors:
1420 * EINVAL - Invalid object number or flags.
1421 * ENOSPC - Hole too small to fulfill "slots" request (DNODE_MUST_BE_FREE)
1422 * EEXIST - Refers to an allocated dnode (DNODE_MUST_BE_FREE)
1423 * - Refers to a freeing dnode (DNODE_MUST_BE_FREE)
1424 * - Refers to an interior dnode slot (DNODE_MUST_BE_ALLOCATED)
1425 * ENOENT - The requested dnode is not allocated (DNODE_MUST_BE_ALLOCATED)
1426 * - The requested dnode is being freed (DNODE_MUST_BE_ALLOCATED)
1427 * EIO - I/O error when reading the meta dnode dbuf.
1428 *
1429 * succeeds even for free dnodes.
1430 */
1431 int
dnode_hold_impl(objset_t * os,uint64_t object,int flag,int slots,const void * tag,dnode_t ** dnp)1432 dnode_hold_impl(objset_t *os, uint64_t object, int flag, int slots,
1433 const void *tag, dnode_t **dnp)
1434 {
1435 int epb, idx, err;
1436 int drop_struct_lock = FALSE;
1437 int type;
1438 uint64_t blk;
1439 dnode_t *mdn, *dn;
1440 dmu_buf_impl_t *db;
1441 dnode_children_t *dnc;
1442 dnode_phys_t *dn_block;
1443 dnode_handle_t *dnh;
1444
1445 ASSERT(!(flag & DNODE_MUST_BE_ALLOCATED) || (slots == 0));
1446 ASSERT(!(flag & DNODE_MUST_BE_FREE) || (slots > 0));
1447 IMPLY(flag & DNODE_DRY_RUN, (tag == NULL) && (dnp == NULL));
1448
1449 /*
1450 * If you are holding the spa config lock as writer, you shouldn't
1451 * be asking the DMU to do *anything* unless it's the root pool
1452 * which may require us to read from the root filesystem while
1453 * holding some (not all) of the locks as writer.
1454 */
1455 ASSERT(spa_config_held(os->os_spa, SCL_ALL, RW_WRITER) == 0 ||
1456 (spa_is_root(os->os_spa) &&
1457 spa_config_held(os->os_spa, SCL_STATE, RW_WRITER)));
1458
1459 ASSERT((flag & DNODE_MUST_BE_ALLOCATED) || (flag & DNODE_MUST_BE_FREE));
1460
1461 if (object == DMU_USERUSED_OBJECT || object == DMU_GROUPUSED_OBJECT ||
1462 object == DMU_PROJECTUSED_OBJECT) {
1463 if (object == DMU_USERUSED_OBJECT)
1464 dn = DMU_USERUSED_DNODE(os);
1465 else if (object == DMU_GROUPUSED_OBJECT)
1466 dn = DMU_GROUPUSED_DNODE(os);
1467 else
1468 dn = DMU_PROJECTUSED_DNODE(os);
1469 if (dn == NULL)
1470 return (SET_ERROR(ENOENT));
1471 type = dn->dn_type;
1472 if ((flag & DNODE_MUST_BE_ALLOCATED) && type == DMU_OT_NONE)
1473 return (SET_ERROR(ENOENT));
1474 if ((flag & DNODE_MUST_BE_FREE) && type != DMU_OT_NONE)
1475 return (SET_ERROR(EEXIST));
1476 DNODE_VERIFY(dn);
1477 /* Don't actually hold if dry run, just return 0 */
1478 if (!(flag & DNODE_DRY_RUN)) {
1479 (void) zfs_refcount_add(&dn->dn_holds, tag);
1480 *dnp = dn;
1481 }
1482 return (0);
1483 }
1484
1485 if (object == 0 || object >= DN_MAX_OBJECT)
1486 return (SET_ERROR(EINVAL));
1487
1488 mdn = DMU_META_DNODE(os);
1489 ASSERT(mdn->dn_object == DMU_META_DNODE_OBJECT);
1490
1491 DNODE_VERIFY(mdn);
1492
1493 if (!RW_WRITE_HELD(&mdn->dn_struct_rwlock)) {
1494 rw_enter(&mdn->dn_struct_rwlock, RW_READER);
1495 drop_struct_lock = TRUE;
1496 }
1497
1498 blk = dbuf_whichblock(mdn, 0, object * sizeof (dnode_phys_t));
1499 db = dbuf_hold(mdn, blk, FTAG);
1500 if (drop_struct_lock)
1501 rw_exit(&mdn->dn_struct_rwlock);
1502 if (db == NULL) {
1503 DNODE_STAT_BUMP(dnode_hold_dbuf_hold);
1504 return (SET_ERROR(EIO));
1505 }
1506
1507 /*
1508 * We do not need to decrypt to read the dnode so it doesn't matter
1509 * if we get the encrypted or decrypted version.
1510 */
1511 err = dbuf_read(db, NULL, DB_RF_CANFAIL |
1512 DB_RF_NO_DECRYPT | DB_RF_NOPREFETCH);
1513 if (err) {
1514 DNODE_STAT_BUMP(dnode_hold_dbuf_read);
1515 dbuf_rele(db, FTAG);
1516 return (err);
1517 }
1518
1519 ASSERT3U(db->db.db_size, >=, 1<<DNODE_SHIFT);
1520 epb = db->db.db_size >> DNODE_SHIFT;
1521
1522 idx = object & (epb - 1);
1523 dn_block = (dnode_phys_t *)db->db.db_data;
1524
1525 ASSERT(DB_DNODE(db)->dn_type == DMU_OT_DNODE);
1526 dnc = dmu_buf_get_user(&db->db);
1527 dnh = NULL;
1528 if (dnc == NULL) {
1529 dnode_children_t *winner;
1530 int skip = 0;
1531
1532 dnc = kmem_zalloc(sizeof (dnode_children_t) +
1533 epb * sizeof (dnode_handle_t), KM_SLEEP);
1534 dnc->dnc_count = epb;
1535 dnh = &dnc->dnc_children[0];
1536
1537 /* Initialize dnode slot status from dnode_phys_t */
1538 for (int i = 0; i < epb; i++) {
1539 zrl_init(&dnh[i].dnh_zrlock);
1540
1541 if (skip) {
1542 skip--;
1543 continue;
1544 }
1545
1546 if (dn_block[i].dn_type != DMU_OT_NONE) {
1547 int interior = dn_block[i].dn_extra_slots;
1548
1549 dnode_set_slots(dnc, i, 1, DN_SLOT_ALLOCATED);
1550 dnode_set_slots(dnc, i + 1, interior,
1551 DN_SLOT_INTERIOR);
1552 skip = interior;
1553 } else {
1554 dnh[i].dnh_dnode = DN_SLOT_FREE;
1555 skip = 0;
1556 }
1557 }
1558
1559 dmu_buf_init_user(&dnc->dnc_dbu, NULL,
1560 dnode_buf_evict_async, NULL);
1561 winner = dmu_buf_set_user(&db->db, &dnc->dnc_dbu);
1562 if (winner != NULL) {
1563
1564 for (int i = 0; i < epb; i++)
1565 zrl_destroy(&dnh[i].dnh_zrlock);
1566
1567 kmem_free(dnc, sizeof (dnode_children_t) +
1568 epb * sizeof (dnode_handle_t));
1569 dnc = winner;
1570 }
1571 }
1572
1573 ASSERT(dnc->dnc_count == epb);
1574
1575 if (flag & DNODE_MUST_BE_ALLOCATED) {
1576 slots = 1;
1577
1578 dnode_slots_hold(dnc, idx, slots);
1579 dnh = &dnc->dnc_children[idx];
1580
1581 if (DN_SLOT_IS_PTR(dnh->dnh_dnode)) {
1582 dn = dnh->dnh_dnode;
1583 } else if (dnh->dnh_dnode == DN_SLOT_INTERIOR) {
1584 DNODE_STAT_BUMP(dnode_hold_alloc_interior);
1585 dnode_slots_rele(dnc, idx, slots);
1586 dbuf_rele(db, FTAG);
1587 return (SET_ERROR(EEXIST));
1588 } else if (dnh->dnh_dnode != DN_SLOT_ALLOCATED) {
1589 DNODE_STAT_BUMP(dnode_hold_alloc_misses);
1590 dnode_slots_rele(dnc, idx, slots);
1591 dbuf_rele(db, FTAG);
1592 return (SET_ERROR(ENOENT));
1593 } else {
1594 dnode_slots_rele(dnc, idx, slots);
1595 while (!dnode_slots_tryenter(dnc, idx, slots)) {
1596 DNODE_STAT_BUMP(dnode_hold_alloc_lock_retry);
1597 kpreempt(KPREEMPT_SYNC);
1598 }
1599
1600 /*
1601 * Someone else won the race and called dnode_create()
1602 * after we checked DN_SLOT_IS_PTR() above but before
1603 * we acquired the lock.
1604 */
1605 if (DN_SLOT_IS_PTR(dnh->dnh_dnode)) {
1606 DNODE_STAT_BUMP(dnode_hold_alloc_lock_misses);
1607 dn = dnh->dnh_dnode;
1608 } else {
1609 dn = dnode_create(os, dn_block + idx, db,
1610 object, dnh);
1611 dmu_buf_add_user_size(&db->db,
1612 sizeof (dnode_t));
1613 }
1614 }
1615
1616 mutex_enter(&dn->dn_mtx);
1617 if (dn->dn_type == DMU_OT_NONE || dn->dn_free_txg != 0) {
1618 DNODE_STAT_BUMP(dnode_hold_alloc_type_none);
1619 mutex_exit(&dn->dn_mtx);
1620 dnode_slots_rele(dnc, idx, slots);
1621 dbuf_rele(db, FTAG);
1622 return (SET_ERROR(ENOENT));
1623 }
1624
1625 /* Don't actually hold if dry run, just return 0 */
1626 if (flag & DNODE_DRY_RUN) {
1627 mutex_exit(&dn->dn_mtx);
1628 dnode_slots_rele(dnc, idx, slots);
1629 dbuf_rele(db, FTAG);
1630 return (0);
1631 }
1632
1633 DNODE_STAT_BUMP(dnode_hold_alloc_hits);
1634 } else if (flag & DNODE_MUST_BE_FREE) {
1635
1636 if (idx + slots - 1 >= DNODES_PER_BLOCK) {
1637 DNODE_STAT_BUMP(dnode_hold_free_overflow);
1638 dbuf_rele(db, FTAG);
1639 return (SET_ERROR(ENOSPC));
1640 }
1641
1642 dnode_slots_hold(dnc, idx, slots);
1643
1644 if (!dnode_check_slots_free(dnc, idx, slots)) {
1645 DNODE_STAT_BUMP(dnode_hold_free_misses);
1646 dnode_slots_rele(dnc, idx, slots);
1647 dbuf_rele(db, FTAG);
1648 return (SET_ERROR(ENOSPC));
1649 }
1650
1651 dnode_slots_rele(dnc, idx, slots);
1652 while (!dnode_slots_tryenter(dnc, idx, slots)) {
1653 DNODE_STAT_BUMP(dnode_hold_free_lock_retry);
1654 kpreempt(KPREEMPT_SYNC);
1655 }
1656
1657 if (!dnode_check_slots_free(dnc, idx, slots)) {
1658 DNODE_STAT_BUMP(dnode_hold_free_lock_misses);
1659 dnode_slots_rele(dnc, idx, slots);
1660 dbuf_rele(db, FTAG);
1661 return (SET_ERROR(ENOSPC));
1662 }
1663
1664 /*
1665 * Allocated but otherwise free dnodes which would
1666 * be in the interior of a multi-slot dnodes need
1667 * to be freed. Single slot dnodes can be safely
1668 * re-purposed as a performance optimization.
1669 */
1670 if (slots > 1) {
1671 uint_t reclaimed =
1672 dnode_reclaim_slots(dnc, idx + 1, slots - 1);
1673 if (reclaimed > 0)
1674 dmu_buf_sub_user_size(&db->db,
1675 reclaimed * sizeof (dnode_t));
1676 }
1677
1678 dnh = &dnc->dnc_children[idx];
1679 if (DN_SLOT_IS_PTR(dnh->dnh_dnode)) {
1680 dn = dnh->dnh_dnode;
1681 } else {
1682 dn = dnode_create(os, dn_block + idx, db,
1683 object, dnh);
1684 dmu_buf_add_user_size(&db->db, sizeof (dnode_t));
1685 }
1686
1687 mutex_enter(&dn->dn_mtx);
1688 if (!zfs_refcount_is_zero(&dn->dn_holds) || dn->dn_free_txg) {
1689 DNODE_STAT_BUMP(dnode_hold_free_refcount);
1690 mutex_exit(&dn->dn_mtx);
1691 dnode_slots_rele(dnc, idx, slots);
1692 dbuf_rele(db, FTAG);
1693 return (SET_ERROR(EEXIST));
1694 }
1695
1696 /* Don't actually hold if dry run, just return 0 */
1697 if (flag & DNODE_DRY_RUN) {
1698 mutex_exit(&dn->dn_mtx);
1699 dnode_slots_rele(dnc, idx, slots);
1700 dbuf_rele(db, FTAG);
1701 return (0);
1702 }
1703
1704 dnode_set_slots(dnc, idx + 1, slots - 1, DN_SLOT_INTERIOR);
1705 DNODE_STAT_BUMP(dnode_hold_free_hits);
1706 } else {
1707 dbuf_rele(db, FTAG);
1708 return (SET_ERROR(EINVAL));
1709 }
1710
1711 ASSERT0(dn->dn_free_txg);
1712
1713 if (zfs_refcount_add(&dn->dn_holds, tag) == 1)
1714 dbuf_add_ref(db, dnh);
1715
1716 mutex_exit(&dn->dn_mtx);
1717
1718 /* Now we can rely on the hold to prevent the dnode from moving. */
1719 dnode_slots_rele(dnc, idx, slots);
1720
1721 DNODE_VERIFY(dn);
1722 ASSERT3P(dnp, !=, NULL);
1723 ASSERT3P(dn->dn_dbuf, ==, db);
1724 ASSERT3U(dn->dn_object, ==, object);
1725 dbuf_rele(db, FTAG);
1726
1727 *dnp = dn;
1728 return (0);
1729 }
1730
1731 /*
1732 * Return held dnode if the object is allocated, NULL if not.
1733 */
1734 int
dnode_hold(objset_t * os,uint64_t object,const void * tag,dnode_t ** dnp)1735 dnode_hold(objset_t *os, uint64_t object, const void *tag, dnode_t **dnp)
1736 {
1737 return (dnode_hold_impl(os, object, DNODE_MUST_BE_ALLOCATED, 0, tag,
1738 dnp));
1739 }
1740
1741 /*
1742 * Can only add a reference if there is already at least one
1743 * reference on the dnode. Returns FALSE if unable to add a
1744 * new reference.
1745 */
1746 boolean_t
dnode_add_ref(dnode_t * dn,const void * tag)1747 dnode_add_ref(dnode_t *dn, const void *tag)
1748 {
1749 mutex_enter(&dn->dn_mtx);
1750 if (zfs_refcount_is_zero(&dn->dn_holds)) {
1751 mutex_exit(&dn->dn_mtx);
1752 return (FALSE);
1753 }
1754 VERIFY(1 < zfs_refcount_add(&dn->dn_holds, tag));
1755 mutex_exit(&dn->dn_mtx);
1756 return (TRUE);
1757 }
1758
1759 void
dnode_rele(dnode_t * dn,const void * tag)1760 dnode_rele(dnode_t *dn, const void *tag)
1761 {
1762 mutex_enter(&dn->dn_mtx);
1763 dnode_rele_and_unlock(dn, tag, B_FALSE);
1764 }
1765
1766 void
dnode_rele_and_unlock(dnode_t * dn,const void * tag,boolean_t evicting)1767 dnode_rele_and_unlock(dnode_t *dn, const void *tag, boolean_t evicting)
1768 {
1769 uint64_t refs;
1770 /* Get while the hold prevents the dnode from moving. */
1771 dmu_buf_impl_t *db = dn->dn_dbuf;
1772 dnode_handle_t *dnh = dn->dn_handle;
1773
1774 refs = zfs_refcount_remove(&dn->dn_holds, tag);
1775 if (refs == 0)
1776 cv_broadcast(&dn->dn_nodnholds);
1777 mutex_exit(&dn->dn_mtx);
1778 /* dnode could get destroyed at this point, so don't use it anymore */
1779
1780 /*
1781 * It's unsafe to release the last hold on a dnode by dnode_rele() or
1782 * indirectly by dbuf_rele() while relying on the dnode handle to
1783 * prevent the dnode from moving, since releasing the last hold could
1784 * result in the dnode's parent dbuf evicting its dnode handles. For
1785 * that reason anyone calling dnode_rele() or dbuf_rele() without some
1786 * other direct or indirect hold on the dnode must first drop the dnode
1787 * handle.
1788 */
1789 #ifdef ZFS_DEBUG
1790 ASSERT(refs > 0 || zrl_owner(&dnh->dnh_zrlock) != curthread);
1791 #endif
1792
1793 /* NOTE: the DNODE_DNODE does not have a dn_dbuf */
1794 if (refs == 0 && db != NULL) {
1795 /*
1796 * Another thread could add a hold to the dnode handle in
1797 * dnode_hold_impl() while holding the parent dbuf. Since the
1798 * hold on the parent dbuf prevents the handle from being
1799 * destroyed, the hold on the handle is OK. We can't yet assert
1800 * that the handle has zero references, but that will be
1801 * asserted anyway when the handle gets destroyed.
1802 */
1803 mutex_enter(&db->db_mtx);
1804 dbuf_rele_and_unlock(db, dnh, evicting);
1805 }
1806 }
1807
1808 /*
1809 * Test whether we can create a dnode at the specified location.
1810 */
1811 int
dnode_try_claim(objset_t * os,uint64_t object,int slots)1812 dnode_try_claim(objset_t *os, uint64_t object, int slots)
1813 {
1814 return (dnode_hold_impl(os, object, DNODE_MUST_BE_FREE | DNODE_DRY_RUN,
1815 slots, NULL, NULL));
1816 }
1817
1818 /*
1819 * Checks if the dnode itself is dirty, or is carrying any uncommitted records.
1820 * It is important to check both conditions, as some operations (eg appending
1821 * to a file) can dirty both as a single logical unit, but they are not synced
1822 * out atomically, so checking one and not the other can result in an object
1823 * appearing to be clean mid-way through a commit.
1824 *
1825 * Do not change this lightly! If you get it wrong, dmu_offset_next() can
1826 * detect a hole where there is really data, leading to silent corruption.
1827 */
1828 boolean_t
dnode_is_dirty(dnode_t * dn)1829 dnode_is_dirty(dnode_t *dn)
1830 {
1831 mutex_enter(&dn->dn_mtx);
1832
1833 for (int i = 0; i < TXG_SIZE; i++) {
1834 if (multilist_link_active(&dn->dn_dirty_link[i]) ||
1835 !list_is_empty(&dn->dn_dirty_records[i])) {
1836 mutex_exit(&dn->dn_mtx);
1837 return (B_TRUE);
1838 }
1839 }
1840
1841 mutex_exit(&dn->dn_mtx);
1842
1843 return (B_FALSE);
1844 }
1845
1846 void
dnode_setdirty(dnode_t * dn,dmu_tx_t * tx)1847 dnode_setdirty(dnode_t *dn, dmu_tx_t *tx)
1848 {
1849 objset_t *os = dn->dn_objset;
1850 uint64_t txg = tx->tx_txg;
1851
1852 if (DMU_OBJECT_IS_SPECIAL(dn->dn_object)) {
1853 dsl_dataset_dirty(os->os_dsl_dataset, tx);
1854 return;
1855 }
1856
1857 DNODE_VERIFY(dn);
1858
1859 #ifdef ZFS_DEBUG
1860 mutex_enter(&dn->dn_mtx);
1861 ASSERT(dn->dn_phys->dn_type || dn->dn_allocated_txg);
1862 ASSERT(dn->dn_free_txg == 0 || dn->dn_free_txg >= txg);
1863 mutex_exit(&dn->dn_mtx);
1864 #endif
1865
1866 /*
1867 * Determine old uid/gid when necessary
1868 */
1869 dmu_objset_userquota_get_ids(dn, B_TRUE, tx);
1870
1871 multilist_t *dirtylist = &os->os_dirty_dnodes[txg & TXG_MASK];
1872 multilist_sublist_t *mls = multilist_sublist_lock_obj(dirtylist, dn);
1873
1874 /*
1875 * If we are already marked dirty, we're done.
1876 */
1877 if (multilist_link_active(&dn->dn_dirty_link[txg & TXG_MASK])) {
1878 multilist_sublist_unlock(mls);
1879 return;
1880 }
1881
1882 ASSERT(!zfs_refcount_is_zero(&dn->dn_holds) ||
1883 !avl_is_empty(&dn->dn_dbufs));
1884 ASSERT(dn->dn_datablksz != 0);
1885 ASSERT0(dn->dn_next_bonuslen[txg & TXG_MASK]);
1886 ASSERT0(dn->dn_next_blksz[txg & TXG_MASK]);
1887 ASSERT0(dn->dn_next_bonustype[txg & TXG_MASK]);
1888
1889 dprintf_ds(os->os_dsl_dataset, "obj=%llu txg=%llu\n",
1890 (u_longlong_t)dn->dn_object, (u_longlong_t)txg);
1891
1892 multilist_sublist_insert_head(mls, dn);
1893
1894 multilist_sublist_unlock(mls);
1895
1896 /*
1897 * The dnode maintains a hold on its containing dbuf as
1898 * long as there are holds on it. Each instantiated child
1899 * dbuf maintains a hold on the dnode. When the last child
1900 * drops its hold, the dnode will drop its hold on the
1901 * containing dbuf. We add a "dirty hold" here so that the
1902 * dnode will hang around after we finish processing its
1903 * children.
1904 */
1905 VERIFY(dnode_add_ref(dn, (void *)(uintptr_t)tx->tx_txg));
1906
1907 (void) dbuf_dirty(dn->dn_dbuf, tx);
1908
1909 dsl_dataset_dirty(os->os_dsl_dataset, tx);
1910 }
1911
1912 void
dnode_free(dnode_t * dn,dmu_tx_t * tx)1913 dnode_free(dnode_t *dn, dmu_tx_t *tx)
1914 {
1915 mutex_enter(&dn->dn_mtx);
1916 if (dn->dn_type == DMU_OT_NONE || dn->dn_free_txg) {
1917 mutex_exit(&dn->dn_mtx);
1918 return;
1919 }
1920 dn->dn_free_txg = tx->tx_txg;
1921 mutex_exit(&dn->dn_mtx);
1922
1923 dnode_setdirty(dn, tx);
1924 }
1925
1926 /*
1927 * Try to change the block size for the indicated dnode. This can only
1928 * succeed if there are no blocks allocated or dirty beyond first block
1929 */
1930 int
dnode_set_blksz(dnode_t * dn,uint64_t size,int ibs,dmu_tx_t * tx)1931 dnode_set_blksz(dnode_t *dn, uint64_t size, int ibs, dmu_tx_t *tx)
1932 {
1933 dmu_buf_impl_t *db;
1934 int err;
1935
1936 ASSERT3U(size, <=, spa_maxblocksize(dmu_objset_spa(dn->dn_objset)));
1937 if (size == 0)
1938 size = SPA_MINBLOCKSIZE;
1939 else
1940 size = P2ROUNDUP(size, SPA_MINBLOCKSIZE);
1941
1942 if (ibs == dn->dn_indblkshift)
1943 ibs = 0;
1944
1945 if (size == dn->dn_datablksz && ibs == 0)
1946 return (0);
1947
1948 rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
1949
1950 /* Check for any allocated blocks beyond the first */
1951 if (dn->dn_maxblkid != 0)
1952 goto fail;
1953
1954 mutex_enter(&dn->dn_dbufs_mtx);
1955 for (db = avl_first(&dn->dn_dbufs); db != NULL;
1956 db = AVL_NEXT(&dn->dn_dbufs, db)) {
1957 if (db->db_blkid != 0 && db->db_blkid != DMU_BONUS_BLKID &&
1958 db->db_blkid != DMU_SPILL_BLKID) {
1959 mutex_exit(&dn->dn_dbufs_mtx);
1960 goto fail;
1961 }
1962 }
1963 mutex_exit(&dn->dn_dbufs_mtx);
1964
1965 if (ibs && dn->dn_nlevels != 1)
1966 goto fail;
1967
1968 dnode_setdirty(dn, tx);
1969 if (size != dn->dn_datablksz) {
1970 /* resize the old block */
1971 err = dbuf_hold_impl(dn, 0, 0, TRUE, FALSE, FTAG, &db);
1972 if (err == 0) {
1973 dbuf_new_size(db, size, tx);
1974 } else if (err != ENOENT) {
1975 goto fail;
1976 }
1977
1978 dnode_setdblksz(dn, size);
1979 dn->dn_next_blksz[tx->tx_txg & TXG_MASK] = size;
1980 if (db)
1981 dbuf_rele(db, FTAG);
1982 }
1983 if (ibs) {
1984 dn->dn_indblkshift = ibs;
1985 dn->dn_next_indblkshift[tx->tx_txg & TXG_MASK] = ibs;
1986 }
1987
1988 rw_exit(&dn->dn_struct_rwlock);
1989 return (0);
1990
1991 fail:
1992 rw_exit(&dn->dn_struct_rwlock);
1993 return (SET_ERROR(ENOTSUP));
1994 }
1995
1996 static void
dnode_set_nlevels_impl(dnode_t * dn,int new_nlevels,dmu_tx_t * tx)1997 dnode_set_nlevels_impl(dnode_t *dn, int new_nlevels, dmu_tx_t *tx)
1998 {
1999 uint64_t txgoff = tx->tx_txg & TXG_MASK;
2000 int old_nlevels = dn->dn_nlevels;
2001 dmu_buf_impl_t *db;
2002 list_t *list;
2003 dbuf_dirty_record_t *new, *dr, *dr_next;
2004
2005 ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock));
2006
2007 ASSERT3U(new_nlevels, >, dn->dn_nlevels);
2008 dn->dn_nlevels = new_nlevels;
2009
2010 ASSERT3U(new_nlevels, >, dn->dn_next_nlevels[txgoff]);
2011 dn->dn_next_nlevels[txgoff] = new_nlevels;
2012
2013 /* dirty the left indirects */
2014 db = dbuf_hold_level(dn, old_nlevels, 0, FTAG);
2015 ASSERT(db != NULL);
2016 new = dbuf_dirty(db, tx);
2017 dbuf_rele(db, FTAG);
2018
2019 /* transfer the dirty records to the new indirect */
2020 mutex_enter(&dn->dn_mtx);
2021 mutex_enter(&new->dt.di.dr_mtx);
2022 list = &dn->dn_dirty_records[txgoff];
2023 for (dr = list_head(list); dr; dr = dr_next) {
2024 dr_next = list_next(&dn->dn_dirty_records[txgoff], dr);
2025
2026 IMPLY(dr->dr_dbuf == NULL, old_nlevels == 1);
2027 if (dr->dr_dbuf == NULL ||
2028 (dr->dr_dbuf->db_level == old_nlevels - 1 &&
2029 dr->dr_dbuf->db_blkid != DMU_BONUS_BLKID &&
2030 dr->dr_dbuf->db_blkid != DMU_SPILL_BLKID)) {
2031 list_remove(&dn->dn_dirty_records[txgoff], dr);
2032 list_insert_tail(&new->dt.di.dr_children, dr);
2033 dr->dr_parent = new;
2034 }
2035 }
2036 mutex_exit(&new->dt.di.dr_mtx);
2037 mutex_exit(&dn->dn_mtx);
2038 }
2039
2040 int
dnode_set_nlevels(dnode_t * dn,int nlevels,dmu_tx_t * tx)2041 dnode_set_nlevels(dnode_t *dn, int nlevels, dmu_tx_t *tx)
2042 {
2043 int ret = 0;
2044
2045 rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
2046
2047 if (dn->dn_nlevels == nlevels) {
2048 ret = 0;
2049 goto out;
2050 } else if (nlevels < dn->dn_nlevels) {
2051 ret = SET_ERROR(EINVAL);
2052 goto out;
2053 }
2054
2055 dnode_set_nlevels_impl(dn, nlevels, tx);
2056
2057 out:
2058 rw_exit(&dn->dn_struct_rwlock);
2059 return (ret);
2060 }
2061
2062 /* read-holding callers must not rely on the lock being continuously held */
2063 void
dnode_new_blkid(dnode_t * dn,uint64_t blkid,dmu_tx_t * tx,boolean_t have_read,boolean_t force)2064 dnode_new_blkid(dnode_t *dn, uint64_t blkid, dmu_tx_t *tx, boolean_t have_read,
2065 boolean_t force)
2066 {
2067 int epbs, new_nlevels;
2068 uint64_t sz;
2069
2070 ASSERT(blkid != DMU_BONUS_BLKID);
2071
2072 ASSERT(have_read ?
2073 RW_READ_HELD(&dn->dn_struct_rwlock) :
2074 RW_WRITE_HELD(&dn->dn_struct_rwlock));
2075
2076 /*
2077 * if we have a read-lock, check to see if we need to do any work
2078 * before upgrading to a write-lock.
2079 */
2080 if (have_read) {
2081 if (blkid <= dn->dn_maxblkid)
2082 return;
2083
2084 if (!rw_tryupgrade(&dn->dn_struct_rwlock)) {
2085 rw_exit(&dn->dn_struct_rwlock);
2086 rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
2087 }
2088 }
2089
2090 /*
2091 * Raw sends (indicated by the force flag) require that we take the
2092 * given blkid even if the value is lower than the current value.
2093 */
2094 if (!force && blkid <= dn->dn_maxblkid)
2095 goto out;
2096
2097 /*
2098 * We use the (otherwise unused) top bit of dn_next_maxblkid[txgoff]
2099 * to indicate that this field is set. This allows us to set the
2100 * maxblkid to 0 on an existing object in dnode_sync().
2101 */
2102 dn->dn_maxblkid = blkid;
2103 dn->dn_next_maxblkid[tx->tx_txg & TXG_MASK] =
2104 blkid | DMU_NEXT_MAXBLKID_SET;
2105
2106 /*
2107 * Compute the number of levels necessary to support the new maxblkid.
2108 * Raw sends will ensure nlevels is set correctly for us.
2109 */
2110 new_nlevels = 1;
2111 epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
2112 for (sz = dn->dn_nblkptr;
2113 sz <= blkid && sz >= dn->dn_nblkptr; sz <<= epbs)
2114 new_nlevels++;
2115
2116 ASSERT3U(new_nlevels, <=, DN_MAX_LEVELS);
2117
2118 if (!force) {
2119 if (new_nlevels > dn->dn_nlevels)
2120 dnode_set_nlevels_impl(dn, new_nlevels, tx);
2121 } else {
2122 ASSERT3U(dn->dn_nlevels, >=, new_nlevels);
2123 }
2124
2125 out:
2126 if (have_read)
2127 rw_downgrade(&dn->dn_struct_rwlock);
2128 }
2129
2130 static void
dnode_dirty_l1(dnode_t * dn,uint64_t l1blkid,dmu_tx_t * tx)2131 dnode_dirty_l1(dnode_t *dn, uint64_t l1blkid, dmu_tx_t *tx)
2132 {
2133 dmu_buf_impl_t *db = dbuf_hold_level(dn, 1, l1blkid, FTAG);
2134 if (db != NULL) {
2135 dmu_buf_will_dirty(&db->db, tx);
2136 dbuf_rele(db, FTAG);
2137 }
2138 }
2139
2140 /*
2141 * Dirty all the in-core level-1 dbufs in the range specified by start_blkid
2142 * and end_blkid.
2143 */
2144 static void
dnode_dirty_l1range(dnode_t * dn,uint64_t start_blkid,uint64_t end_blkid,dmu_tx_t * tx)2145 dnode_dirty_l1range(dnode_t *dn, uint64_t start_blkid, uint64_t end_blkid,
2146 dmu_tx_t *tx)
2147 {
2148 dmu_buf_impl_t *db_search;
2149 dmu_buf_impl_t *db;
2150 avl_index_t where;
2151
2152 db_search = kmem_zalloc(sizeof (dmu_buf_impl_t), KM_SLEEP);
2153
2154 mutex_enter(&dn->dn_dbufs_mtx);
2155
2156 db_search->db_level = 1;
2157 db_search->db_blkid = start_blkid + 1;
2158 db_search->db_state = DB_SEARCH;
2159 for (;;) {
2160
2161 db = avl_find(&dn->dn_dbufs, db_search, &where);
2162 if (db == NULL)
2163 db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER);
2164
2165 if (db == NULL || db->db_level != 1 ||
2166 db->db_blkid >= end_blkid) {
2167 break;
2168 }
2169
2170 /*
2171 * Setup the next blkid we want to search for.
2172 */
2173 db_search->db_blkid = db->db_blkid + 1;
2174 ASSERT3U(db->db_blkid, >=, start_blkid);
2175
2176 /*
2177 * If the dbuf transitions to DB_EVICTING while we're trying
2178 * to dirty it, then we will be unable to discover it in
2179 * the dbuf hash table. This will result in a call to
2180 * dbuf_create() which needs to acquire the dn_dbufs_mtx
2181 * lock. To avoid a deadlock, we drop the lock before
2182 * dirtying the level-1 dbuf.
2183 */
2184 mutex_exit(&dn->dn_dbufs_mtx);
2185 dnode_dirty_l1(dn, db->db_blkid, tx);
2186 mutex_enter(&dn->dn_dbufs_mtx);
2187 }
2188
2189 #ifdef ZFS_DEBUG
2190 /*
2191 * Walk all the in-core level-1 dbufs and verify they have been dirtied.
2192 */
2193 db_search->db_level = 1;
2194 db_search->db_blkid = start_blkid + 1;
2195 db_search->db_state = DB_SEARCH;
2196 db = avl_find(&dn->dn_dbufs, db_search, &where);
2197 if (db == NULL)
2198 db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER);
2199 for (; db != NULL; db = AVL_NEXT(&dn->dn_dbufs, db)) {
2200 if (db->db_level != 1 || db->db_blkid >= end_blkid)
2201 break;
2202 if (db->db_state != DB_EVICTING)
2203 ASSERT(db->db_dirtycnt > 0);
2204 }
2205 #endif
2206 kmem_free(db_search, sizeof (dmu_buf_impl_t));
2207 mutex_exit(&dn->dn_dbufs_mtx);
2208 }
2209
2210 void
dnode_set_dirtyctx(dnode_t * dn,dmu_tx_t * tx,const void * tag)2211 dnode_set_dirtyctx(dnode_t *dn, dmu_tx_t *tx, const void *tag)
2212 {
2213 /*
2214 * Don't set dirtyctx to SYNC if we're just modifying this as we
2215 * initialize the objset.
2216 */
2217 if (dn->dn_dirtyctx == DN_UNDIRTIED) {
2218 dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset;
2219
2220 if (ds != NULL) {
2221 rrw_enter(&ds->ds_bp_rwlock, RW_READER, tag);
2222 }
2223 if (!BP_IS_HOLE(dn->dn_objset->os_rootbp)) {
2224 if (dmu_tx_is_syncing(tx))
2225 dn->dn_dirtyctx = DN_DIRTY_SYNC;
2226 else
2227 dn->dn_dirtyctx = DN_DIRTY_OPEN;
2228 dn->dn_dirtyctx_firstset = tag;
2229 }
2230 if (ds != NULL) {
2231 rrw_exit(&ds->ds_bp_rwlock, tag);
2232 }
2233 }
2234 }
2235
2236 static void
dnode_partial_zero(dnode_t * dn,uint64_t off,uint64_t blkoff,uint64_t len,dmu_tx_t * tx)2237 dnode_partial_zero(dnode_t *dn, uint64_t off, uint64_t blkoff, uint64_t len,
2238 dmu_tx_t *tx)
2239 {
2240 dmu_buf_impl_t *db;
2241 int res;
2242
2243 rw_enter(&dn->dn_struct_rwlock, RW_READER);
2244 res = dbuf_hold_impl(dn, 0, dbuf_whichblock(dn, 0, off), TRUE, FALSE,
2245 FTAG, &db);
2246 rw_exit(&dn->dn_struct_rwlock);
2247 if (res == 0) {
2248 db_lock_type_t dblt;
2249 boolean_t dirty;
2250
2251 dblt = dmu_buf_lock_parent(db, RW_READER, FTAG);
2252 /* don't dirty if not on disk and not dirty */
2253 dirty = !list_is_empty(&db->db_dirty_records) ||
2254 (db->db_blkptr && !BP_IS_HOLE(db->db_blkptr));
2255 dmu_buf_unlock_parent(db, dblt, FTAG);
2256 if (dirty) {
2257 caddr_t data;
2258
2259 dmu_buf_will_dirty(&db->db, tx);
2260 data = db->db.db_data;
2261 memset(data + blkoff, 0, len);
2262 }
2263 dbuf_rele(db, FTAG);
2264 }
2265 }
2266
2267 void
dnode_free_range(dnode_t * dn,uint64_t off,uint64_t len,dmu_tx_t * tx)2268 dnode_free_range(dnode_t *dn, uint64_t off, uint64_t len, dmu_tx_t *tx)
2269 {
2270 uint64_t blkoff, blkid, nblks;
2271 int blksz, blkshift, head, tail;
2272 int trunc = FALSE;
2273 int epbs;
2274
2275 blksz = dn->dn_datablksz;
2276 blkshift = dn->dn_datablkshift;
2277 epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
2278
2279 if (len == DMU_OBJECT_END) {
2280 len = UINT64_MAX - off;
2281 trunc = TRUE;
2282 }
2283
2284 /*
2285 * First, block align the region to free:
2286 */
2287 if (ISP2(blksz)) {
2288 head = P2NPHASE(off, blksz);
2289 blkoff = P2PHASE(off, blksz);
2290 if ((off >> blkshift) > dn->dn_maxblkid)
2291 return;
2292 } else {
2293 ASSERT(dn->dn_maxblkid == 0);
2294 if (off == 0 && len >= blksz) {
2295 /*
2296 * Freeing the whole block; fast-track this request.
2297 */
2298 blkid = 0;
2299 nblks = 1;
2300 if (dn->dn_nlevels > 1) {
2301 rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
2302 dnode_dirty_l1(dn, 0, tx);
2303 rw_exit(&dn->dn_struct_rwlock);
2304 }
2305 goto done;
2306 } else if (off >= blksz) {
2307 /* Freeing past end-of-data */
2308 return;
2309 } else {
2310 /* Freeing part of the block. */
2311 head = blksz - off;
2312 ASSERT3U(head, >, 0);
2313 }
2314 blkoff = off;
2315 }
2316 /* zero out any partial block data at the start of the range */
2317 if (head) {
2318 ASSERT3U(blkoff + head, ==, blksz);
2319 if (len < head)
2320 head = len;
2321 dnode_partial_zero(dn, off, blkoff, head, tx);
2322 off += head;
2323 len -= head;
2324 }
2325
2326 /* If the range was less than one block, we're done */
2327 if (len == 0)
2328 return;
2329
2330 /* If the remaining range is past end of file, we're done */
2331 if ((off >> blkshift) > dn->dn_maxblkid)
2332 return;
2333
2334 ASSERT(ISP2(blksz));
2335 if (trunc)
2336 tail = 0;
2337 else
2338 tail = P2PHASE(len, blksz);
2339
2340 ASSERT0(P2PHASE(off, blksz));
2341 /* zero out any partial block data at the end of the range */
2342 if (tail) {
2343 if (len < tail)
2344 tail = len;
2345 dnode_partial_zero(dn, off + len, 0, tail, tx);
2346 len -= tail;
2347 }
2348
2349 /* If the range did not include a full block, we are done */
2350 if (len == 0)
2351 return;
2352
2353 ASSERT(IS_P2ALIGNED(off, blksz));
2354 ASSERT(trunc || IS_P2ALIGNED(len, blksz));
2355 blkid = off >> blkshift;
2356 nblks = len >> blkshift;
2357 if (trunc)
2358 nblks += 1;
2359
2360 /*
2361 * Dirty all the indirect blocks in this range. Note that only
2362 * the first and last indirect blocks can actually be written
2363 * (if they were partially freed) -- they must be dirtied, even if
2364 * they do not exist on disk yet. The interior blocks will
2365 * be freed by free_children(), so they will not actually be written.
2366 * Even though these interior blocks will not be written, we
2367 * dirty them for two reasons:
2368 *
2369 * - It ensures that the indirect blocks remain in memory until
2370 * syncing context. (They have already been prefetched by
2371 * dmu_tx_hold_free(), so we don't have to worry about reading
2372 * them serially here.)
2373 *
2374 * - The dirty space accounting will put pressure on the txg sync
2375 * mechanism to begin syncing, and to delay transactions if there
2376 * is a large amount of freeing. Even though these indirect
2377 * blocks will not be written, we could need to write the same
2378 * amount of space if we copy the freed BPs into deadlists.
2379 */
2380 if (dn->dn_nlevels > 1) {
2381 rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
2382 uint64_t first, last;
2383
2384 first = blkid >> epbs;
2385 dnode_dirty_l1(dn, first, tx);
2386 if (trunc)
2387 last = dn->dn_maxblkid >> epbs;
2388 else
2389 last = (blkid + nblks - 1) >> epbs;
2390 if (last != first)
2391 dnode_dirty_l1(dn, last, tx);
2392
2393 dnode_dirty_l1range(dn, first, last, tx);
2394
2395 int shift = dn->dn_datablkshift + dn->dn_indblkshift -
2396 SPA_BLKPTRSHIFT;
2397 for (uint64_t i = first + 1; i < last; i++) {
2398 /*
2399 * Set i to the blockid of the next non-hole
2400 * level-1 indirect block at or after i. Note
2401 * that dnode_next_offset() operates in terms of
2402 * level-0-equivalent bytes.
2403 */
2404 uint64_t ibyte = i << shift;
2405 int err = dnode_next_offset(dn, DNODE_FIND_HAVELOCK,
2406 &ibyte, 2, 1, 0);
2407 i = ibyte >> shift;
2408 if (i >= last)
2409 break;
2410
2411 /*
2412 * Normally we should not see an error, either
2413 * from dnode_next_offset() or dbuf_hold_level()
2414 * (except for ESRCH from dnode_next_offset).
2415 * If there is an i/o error, then when we read
2416 * this block in syncing context, it will use
2417 * ZIO_FLAG_MUSTSUCCEED, and thus hang/panic according
2418 * to the "failmode" property. dnode_next_offset()
2419 * doesn't have a flag to indicate MUSTSUCCEED.
2420 */
2421 if (err != 0)
2422 break;
2423
2424 dnode_dirty_l1(dn, i, tx);
2425 }
2426 rw_exit(&dn->dn_struct_rwlock);
2427 }
2428
2429 done:
2430 /*
2431 * Add this range to the dnode range list.
2432 * We will finish up this free operation in the syncing phase.
2433 */
2434 mutex_enter(&dn->dn_mtx);
2435 {
2436 int txgoff = tx->tx_txg & TXG_MASK;
2437 if (dn->dn_free_ranges[txgoff] == NULL) {
2438 dn->dn_free_ranges[txgoff] = range_tree_create(NULL,
2439 RANGE_SEG64, NULL, 0, 0);
2440 }
2441 range_tree_clear(dn->dn_free_ranges[txgoff], blkid, nblks);
2442 range_tree_add(dn->dn_free_ranges[txgoff], blkid, nblks);
2443 }
2444 dprintf_dnode(dn, "blkid=%llu nblks=%llu txg=%llu\n",
2445 (u_longlong_t)blkid, (u_longlong_t)nblks,
2446 (u_longlong_t)tx->tx_txg);
2447 mutex_exit(&dn->dn_mtx);
2448
2449 dbuf_free_range(dn, blkid, blkid + nblks - 1, tx);
2450 dnode_setdirty(dn, tx);
2451 }
2452
2453 static boolean_t
dnode_spill_freed(dnode_t * dn)2454 dnode_spill_freed(dnode_t *dn)
2455 {
2456 int i;
2457
2458 mutex_enter(&dn->dn_mtx);
2459 for (i = 0; i < TXG_SIZE; i++) {
2460 if (dn->dn_rm_spillblk[i] == DN_KILL_SPILLBLK)
2461 break;
2462 }
2463 mutex_exit(&dn->dn_mtx);
2464 return (i < TXG_SIZE);
2465 }
2466
2467 /* return TRUE if this blkid was freed in a recent txg, or FALSE if it wasn't */
2468 uint64_t
dnode_block_freed(dnode_t * dn,uint64_t blkid)2469 dnode_block_freed(dnode_t *dn, uint64_t blkid)
2470 {
2471 int i;
2472
2473 if (blkid == DMU_BONUS_BLKID)
2474 return (FALSE);
2475
2476 if (dn->dn_free_txg)
2477 return (TRUE);
2478
2479 if (blkid == DMU_SPILL_BLKID)
2480 return (dnode_spill_freed(dn));
2481
2482 mutex_enter(&dn->dn_mtx);
2483 for (i = 0; i < TXG_SIZE; i++) {
2484 if (dn->dn_free_ranges[i] != NULL &&
2485 range_tree_contains(dn->dn_free_ranges[i], blkid, 1))
2486 break;
2487 }
2488 mutex_exit(&dn->dn_mtx);
2489 return (i < TXG_SIZE);
2490 }
2491
2492 /* call from syncing context when we actually write/free space for this dnode */
2493 void
dnode_diduse_space(dnode_t * dn,int64_t delta)2494 dnode_diduse_space(dnode_t *dn, int64_t delta)
2495 {
2496 uint64_t space;
2497 dprintf_dnode(dn, "dn=%p dnp=%p used=%llu delta=%lld\n",
2498 dn, dn->dn_phys,
2499 (u_longlong_t)dn->dn_phys->dn_used,
2500 (longlong_t)delta);
2501
2502 mutex_enter(&dn->dn_mtx);
2503 space = DN_USED_BYTES(dn->dn_phys);
2504 if (delta > 0) {
2505 ASSERT3U(space + delta, >=, space); /* no overflow */
2506 } else {
2507 ASSERT3U(space, >=, -delta); /* no underflow */
2508 }
2509 space += delta;
2510 if (spa_version(dn->dn_objset->os_spa) < SPA_VERSION_DNODE_BYTES) {
2511 ASSERT((dn->dn_phys->dn_flags & DNODE_FLAG_USED_BYTES) == 0);
2512 ASSERT0(P2PHASE(space, 1<<DEV_BSHIFT));
2513 dn->dn_phys->dn_used = space >> DEV_BSHIFT;
2514 } else {
2515 dn->dn_phys->dn_used = space;
2516 dn->dn_phys->dn_flags |= DNODE_FLAG_USED_BYTES;
2517 }
2518 mutex_exit(&dn->dn_mtx);
2519 }
2520
2521 /*
2522 * Scans a block at the indicated "level" looking for a hole or data,
2523 * depending on 'flags'.
2524 *
2525 * If level > 0, then we are scanning an indirect block looking at its
2526 * pointers. If level == 0, then we are looking at a block of dnodes.
2527 *
2528 * If we don't find what we are looking for in the block, we return ESRCH.
2529 * Otherwise, return with *offset pointing to the beginning (if searching
2530 * forwards) or end (if searching backwards) of the range covered by the
2531 * block pointer we matched on (or dnode).
2532 *
2533 * The basic search algorithm used below by dnode_next_offset() is to
2534 * use this function to search up the block tree (widen the search) until
2535 * we find something (i.e., we don't return ESRCH) and then search back
2536 * down the tree (narrow the search) until we reach our original search
2537 * level.
2538 */
2539 static int
dnode_next_offset_level(dnode_t * dn,int flags,uint64_t * offset,int lvl,uint64_t blkfill,uint64_t txg)2540 dnode_next_offset_level(dnode_t *dn, int flags, uint64_t *offset,
2541 int lvl, uint64_t blkfill, uint64_t txg)
2542 {
2543 dmu_buf_impl_t *db = NULL;
2544 void *data = NULL;
2545 uint64_t epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;
2546 uint64_t epb = 1ULL << epbs;
2547 uint64_t minfill, maxfill;
2548 boolean_t hole;
2549 int i, inc, error, span;
2550
2551 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
2552
2553 hole = ((flags & DNODE_FIND_HOLE) != 0);
2554 inc = (flags & DNODE_FIND_BACKWARDS) ? -1 : 1;
2555 ASSERT(txg == 0 || !hole);
2556
2557 if (lvl == dn->dn_phys->dn_nlevels) {
2558 error = 0;
2559 epb = dn->dn_phys->dn_nblkptr;
2560 data = dn->dn_phys->dn_blkptr;
2561 } else {
2562 uint64_t blkid = dbuf_whichblock(dn, lvl, *offset);
2563 error = dbuf_hold_impl(dn, lvl, blkid, TRUE, FALSE, FTAG, &db);
2564 if (error) {
2565 if (error != ENOENT)
2566 return (error);
2567 if (hole)
2568 return (0);
2569 /*
2570 * This can only happen when we are searching up
2571 * the block tree for data. We don't really need to
2572 * adjust the offset, as we will just end up looking
2573 * at the pointer to this block in its parent, and its
2574 * going to be unallocated, so we will skip over it.
2575 */
2576 return (SET_ERROR(ESRCH));
2577 }
2578 error = dbuf_read(db, NULL,
2579 DB_RF_CANFAIL | DB_RF_HAVESTRUCT |
2580 DB_RF_NO_DECRYPT | DB_RF_NOPREFETCH);
2581 if (error) {
2582 dbuf_rele(db, FTAG);
2583 return (error);
2584 }
2585 data = db->db.db_data;
2586 rw_enter(&db->db_rwlock, RW_READER);
2587 }
2588
2589 if (db != NULL && txg != 0 && (db->db_blkptr == NULL ||
2590 BP_GET_LOGICAL_BIRTH(db->db_blkptr) <= txg ||
2591 BP_IS_HOLE(db->db_blkptr))) {
2592 /*
2593 * This can only happen when we are searching up the tree
2594 * and these conditions mean that we need to keep climbing.
2595 */
2596 error = SET_ERROR(ESRCH);
2597 } else if (lvl == 0) {
2598 dnode_phys_t *dnp = data;
2599
2600 ASSERT(dn->dn_type == DMU_OT_DNODE);
2601 ASSERT(!(flags & DNODE_FIND_BACKWARDS));
2602
2603 for (i = (*offset >> DNODE_SHIFT) & (blkfill - 1);
2604 i < blkfill; i += dnp[i].dn_extra_slots + 1) {
2605 if ((dnp[i].dn_type == DMU_OT_NONE) == hole)
2606 break;
2607 }
2608
2609 if (i == blkfill)
2610 error = SET_ERROR(ESRCH);
2611
2612 *offset = (*offset & ~(DNODE_BLOCK_SIZE - 1)) +
2613 (i << DNODE_SHIFT);
2614 } else {
2615 blkptr_t *bp = data;
2616 uint64_t start = *offset;
2617 span = (lvl - 1) * epbs + dn->dn_datablkshift;
2618 minfill = 0;
2619 maxfill = blkfill << ((lvl - 1) * epbs);
2620
2621 if (hole)
2622 maxfill--;
2623 else
2624 minfill++;
2625
2626 if (span >= 8 * sizeof (*offset)) {
2627 /* This only happens on the highest indirection level */
2628 ASSERT3U((lvl - 1), ==, dn->dn_phys->dn_nlevels - 1);
2629 *offset = 0;
2630 } else {
2631 *offset = *offset >> span;
2632 }
2633
2634 for (i = BF64_GET(*offset, 0, epbs);
2635 i >= 0 && i < epb; i += inc) {
2636 if (BP_GET_FILL(&bp[i]) >= minfill &&
2637 BP_GET_FILL(&bp[i]) <= maxfill &&
2638 (hole || BP_GET_LOGICAL_BIRTH(&bp[i]) > txg))
2639 break;
2640 if (inc > 0 || *offset > 0)
2641 *offset += inc;
2642 }
2643
2644 if (span >= 8 * sizeof (*offset)) {
2645 *offset = start;
2646 } else {
2647 *offset = *offset << span;
2648 }
2649
2650 if (inc < 0) {
2651 /* traversing backwards; position offset at the end */
2652 if (span < 8 * sizeof (*offset))
2653 *offset = MIN(*offset + (1ULL << span) - 1,
2654 start);
2655 } else if (*offset < start) {
2656 *offset = start;
2657 }
2658 if (i < 0 || i >= epb)
2659 error = SET_ERROR(ESRCH);
2660 }
2661
2662 if (db != NULL) {
2663 rw_exit(&db->db_rwlock);
2664 dbuf_rele(db, FTAG);
2665 }
2666
2667 return (error);
2668 }
2669
2670 /*
2671 * Find the next hole, data, or sparse region at or after *offset.
2672 * The value 'blkfill' tells us how many items we expect to find
2673 * in an L0 data block; this value is 1 for normal objects,
2674 * DNODES_PER_BLOCK for the meta dnode, and some fraction of
2675 * DNODES_PER_BLOCK when searching for sparse regions thereof.
2676 *
2677 * Examples:
2678 *
2679 * dnode_next_offset(dn, flags, offset, 1, 1, 0);
2680 * Finds the next/previous hole/data in a file.
2681 * Used in dmu_offset_next().
2682 *
2683 * dnode_next_offset(mdn, flags, offset, 0, DNODES_PER_BLOCK, txg);
2684 * Finds the next free/allocated dnode an objset's meta-dnode.
2685 * Only finds objects that have new contents since txg (ie.
2686 * bonus buffer changes and content removal are ignored).
2687 * Used in dmu_object_next().
2688 *
2689 * dnode_next_offset(mdn, DNODE_FIND_HOLE, offset, 2, DNODES_PER_BLOCK >> 2, 0);
2690 * Finds the next L2 meta-dnode bp that's at most 1/4 full.
2691 * Used in dmu_object_alloc().
2692 */
2693 int
dnode_next_offset(dnode_t * dn,int flags,uint64_t * offset,int minlvl,uint64_t blkfill,uint64_t txg)2694 dnode_next_offset(dnode_t *dn, int flags, uint64_t *offset,
2695 int minlvl, uint64_t blkfill, uint64_t txg)
2696 {
2697 uint64_t initial_offset = *offset;
2698 int lvl, maxlvl;
2699 int error = 0;
2700
2701 if (!(flags & DNODE_FIND_HAVELOCK))
2702 rw_enter(&dn->dn_struct_rwlock, RW_READER);
2703
2704 if (dn->dn_phys->dn_nlevels == 0) {
2705 error = SET_ERROR(ESRCH);
2706 goto out;
2707 }
2708
2709 if (dn->dn_datablkshift == 0) {
2710 if (*offset < dn->dn_datablksz) {
2711 if (flags & DNODE_FIND_HOLE)
2712 *offset = dn->dn_datablksz;
2713 } else {
2714 error = SET_ERROR(ESRCH);
2715 }
2716 goto out;
2717 }
2718
2719 maxlvl = dn->dn_phys->dn_nlevels;
2720
2721 for (lvl = minlvl; lvl <= maxlvl; lvl++) {
2722 error = dnode_next_offset_level(dn,
2723 flags, offset, lvl, blkfill, txg);
2724 if (error != ESRCH)
2725 break;
2726 }
2727
2728 while (error == 0 && --lvl >= minlvl) {
2729 error = dnode_next_offset_level(dn,
2730 flags, offset, lvl, blkfill, txg);
2731 }
2732
2733 /*
2734 * There's always a "virtual hole" at the end of the object, even
2735 * if all BP's which physically exist are non-holes.
2736 */
2737 if ((flags & DNODE_FIND_HOLE) && error == ESRCH && txg == 0 &&
2738 minlvl == 1 && blkfill == 1 && !(flags & DNODE_FIND_BACKWARDS)) {
2739 error = 0;
2740 }
2741
2742 if (error == 0 && (flags & DNODE_FIND_BACKWARDS ?
2743 initial_offset < *offset : initial_offset > *offset))
2744 error = SET_ERROR(ESRCH);
2745 out:
2746 if (!(flags & DNODE_FIND_HAVELOCK))
2747 rw_exit(&dn->dn_struct_rwlock);
2748
2749 return (error);
2750 }
2751
2752 #if defined(_KERNEL)
2753 EXPORT_SYMBOL(dnode_hold);
2754 EXPORT_SYMBOL(dnode_rele);
2755 EXPORT_SYMBOL(dnode_set_nlevels);
2756 EXPORT_SYMBOL(dnode_set_blksz);
2757 EXPORT_SYMBOL(dnode_free_range);
2758 EXPORT_SYMBOL(dnode_evict_dbufs);
2759 EXPORT_SYMBOL(dnode_evict_bonus);
2760 #endif
2761
2762 ZFS_MODULE_PARAM(zfs, zfs_, default_bs, INT, ZMOD_RW,
2763 "Default dnode block shift");
2764 ZFS_MODULE_PARAM(zfs, zfs_, default_ibs, INT, ZMOD_RW,
2765 "Default dnode indirect block shift");
2766