xref: /titanic_52/usr/src/uts/common/fs/zfs/dbuf.c (revision dfbb6957eafddc352416799f71513a7be3d0c52c)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
24  * Copyright (c) 2012, 2015 by Delphix. All rights reserved.
25  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
26  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
27  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
28  * Copyright (c) 2014 Integros [integros.com]
29  */
30 
31 #include <sys/zfs_context.h>
32 #include <sys/dmu.h>
33 #include <sys/dmu_send.h>
34 #include <sys/dmu_impl.h>
35 #include <sys/dbuf.h>
36 #include <sys/dmu_objset.h>
37 #include <sys/dsl_dataset.h>
38 #include <sys/dsl_dir.h>
39 #include <sys/dmu_tx.h>
40 #include <sys/spa.h>
41 #include <sys/zio.h>
42 #include <sys/dmu_zfetch.h>
43 #include <sys/sa.h>
44 #include <sys/sa_impl.h>
45 #include <sys/zfeature.h>
46 #include <sys/blkptr.h>
47 #include <sys/range_tree.h>
48 
49 /*
50  * Number of times that zfs_free_range() took the slow path while doing
51  * a zfs receive.  A nonzero value indicates a potential performance problem.
52  */
53 uint64_t zfs_free_range_recv_miss;
54 
55 static void dbuf_destroy(dmu_buf_impl_t *db);
56 static boolean_t dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx);
57 static void dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx);
58 
59 #ifndef __lint
60 extern inline void dmu_buf_init_user(dmu_buf_user_t *dbu,
61     dmu_buf_evict_func_t *evict_func_sync,
62     dmu_buf_evict_func_t *evict_func_async,
63     dmu_buf_t **clear_on_evict_dbufp);
64 #endif /* ! __lint */
65 
66 /*
67  * Global data structures and functions for the dbuf cache.
68  */
69 static kmem_cache_t *dbuf_cache;
70 static taskq_t *dbu_evict_taskq;
71 
72 /* ARGSUSED */
73 static int
74 dbuf_cons(void *vdb, void *unused, int kmflag)
75 {
76 	dmu_buf_impl_t *db = vdb;
77 	bzero(db, sizeof (dmu_buf_impl_t));
78 
79 	mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL);
80 	cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL);
81 	refcount_create(&db->db_holds);
82 
83 	return (0);
84 }
85 
86 /* ARGSUSED */
87 static void
88 dbuf_dest(void *vdb, void *unused)
89 {
90 	dmu_buf_impl_t *db = vdb;
91 	mutex_destroy(&db->db_mtx);
92 	cv_destroy(&db->db_changed);
93 	refcount_destroy(&db->db_holds);
94 }
95 
96 /*
97  * dbuf hash table routines
98  */
99 static dbuf_hash_table_t dbuf_hash_table;
100 
101 static uint64_t dbuf_hash_count;
102 
103 static uint64_t
104 dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid)
105 {
106 	uintptr_t osv = (uintptr_t)os;
107 	uint64_t crc = -1ULL;
108 
109 	ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY);
110 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (lvl)) & 0xFF];
111 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF];
112 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF];
113 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF];
114 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 0)) & 0xFF];
115 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 8)) & 0xFF];
116 
117 	crc ^= (osv>>14) ^ (obj>>16) ^ (blkid>>16);
118 
119 	return (crc);
120 }
121 
122 #define	DBUF_HASH(os, obj, level, blkid) dbuf_hash(os, obj, level, blkid);
123 
124 #define	DBUF_EQUAL(dbuf, os, obj, level, blkid)		\
125 	((dbuf)->db.db_object == (obj) &&		\
126 	(dbuf)->db_objset == (os) &&			\
127 	(dbuf)->db_level == (level) &&			\
128 	(dbuf)->db_blkid == (blkid))
129 
130 dmu_buf_impl_t *
131 dbuf_find(objset_t *os, uint64_t obj, uint8_t level, uint64_t blkid)
132 {
133 	dbuf_hash_table_t *h = &dbuf_hash_table;
134 	uint64_t hv = DBUF_HASH(os, obj, level, blkid);
135 	uint64_t idx = hv & h->hash_table_mask;
136 	dmu_buf_impl_t *db;
137 
138 	mutex_enter(DBUF_HASH_MUTEX(h, idx));
139 	for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) {
140 		if (DBUF_EQUAL(db, os, obj, level, blkid)) {
141 			mutex_enter(&db->db_mtx);
142 			if (db->db_state != DB_EVICTING) {
143 				mutex_exit(DBUF_HASH_MUTEX(h, idx));
144 				return (db);
145 			}
146 			mutex_exit(&db->db_mtx);
147 		}
148 	}
149 	mutex_exit(DBUF_HASH_MUTEX(h, idx));
150 	return (NULL);
151 }
152 
153 static dmu_buf_impl_t *
154 dbuf_find_bonus(objset_t *os, uint64_t object)
155 {
156 	dnode_t *dn;
157 	dmu_buf_impl_t *db = NULL;
158 
159 	if (dnode_hold(os, object, FTAG, &dn) == 0) {
160 		rw_enter(&dn->dn_struct_rwlock, RW_READER);
161 		if (dn->dn_bonus != NULL) {
162 			db = dn->dn_bonus;
163 			mutex_enter(&db->db_mtx);
164 		}
165 		rw_exit(&dn->dn_struct_rwlock);
166 		dnode_rele(dn, FTAG);
167 	}
168 	return (db);
169 }
170 
171 /*
172  * Insert an entry into the hash table.  If there is already an element
173  * equal to elem in the hash table, then the already existing element
174  * will be returned and the new element will not be inserted.
175  * Otherwise returns NULL.
176  */
177 static dmu_buf_impl_t *
178 dbuf_hash_insert(dmu_buf_impl_t *db)
179 {
180 	dbuf_hash_table_t *h = &dbuf_hash_table;
181 	objset_t *os = db->db_objset;
182 	uint64_t obj = db->db.db_object;
183 	int level = db->db_level;
184 	uint64_t blkid = db->db_blkid;
185 	uint64_t hv = DBUF_HASH(os, obj, level, blkid);
186 	uint64_t idx = hv & h->hash_table_mask;
187 	dmu_buf_impl_t *dbf;
188 
189 	mutex_enter(DBUF_HASH_MUTEX(h, idx));
190 	for (dbf = h->hash_table[idx]; dbf != NULL; dbf = dbf->db_hash_next) {
191 		if (DBUF_EQUAL(dbf, os, obj, level, blkid)) {
192 			mutex_enter(&dbf->db_mtx);
193 			if (dbf->db_state != DB_EVICTING) {
194 				mutex_exit(DBUF_HASH_MUTEX(h, idx));
195 				return (dbf);
196 			}
197 			mutex_exit(&dbf->db_mtx);
198 		}
199 	}
200 
201 	mutex_enter(&db->db_mtx);
202 	db->db_hash_next = h->hash_table[idx];
203 	h->hash_table[idx] = db;
204 	mutex_exit(DBUF_HASH_MUTEX(h, idx));
205 	atomic_inc_64(&dbuf_hash_count);
206 
207 	return (NULL);
208 }
209 
210 /*
211  * Remove an entry from the hash table.  It must be in the EVICTING state.
212  */
213 static void
214 dbuf_hash_remove(dmu_buf_impl_t *db)
215 {
216 	dbuf_hash_table_t *h = &dbuf_hash_table;
217 	uint64_t hv = DBUF_HASH(db->db_objset, db->db.db_object,
218 	    db->db_level, db->db_blkid);
219 	uint64_t idx = hv & h->hash_table_mask;
220 	dmu_buf_impl_t *dbf, **dbp;
221 
222 	/*
223 	 * We musn't hold db_mtx to maintain lock ordering:
224 	 * DBUF_HASH_MUTEX > db_mtx.
225 	 */
226 	ASSERT(refcount_is_zero(&db->db_holds));
227 	ASSERT(db->db_state == DB_EVICTING);
228 	ASSERT(!MUTEX_HELD(&db->db_mtx));
229 
230 	mutex_enter(DBUF_HASH_MUTEX(h, idx));
231 	dbp = &h->hash_table[idx];
232 	while ((dbf = *dbp) != db) {
233 		dbp = &dbf->db_hash_next;
234 		ASSERT(dbf != NULL);
235 	}
236 	*dbp = db->db_hash_next;
237 	db->db_hash_next = NULL;
238 	mutex_exit(DBUF_HASH_MUTEX(h, idx));
239 	atomic_dec_64(&dbuf_hash_count);
240 }
241 
242 static arc_evict_func_t dbuf_do_evict;
243 
244 typedef enum {
245 	DBVU_EVICTING,
246 	DBVU_NOT_EVICTING
247 } dbvu_verify_type_t;
248 
249 static void
250 dbuf_verify_user(dmu_buf_impl_t *db, dbvu_verify_type_t verify_type)
251 {
252 #ifdef ZFS_DEBUG
253 	int64_t holds;
254 
255 	if (db->db_user == NULL)
256 		return;
257 
258 	/* Only data blocks support the attachment of user data. */
259 	ASSERT(db->db_level == 0);
260 
261 	/* Clients must resolve a dbuf before attaching user data. */
262 	ASSERT(db->db.db_data != NULL);
263 	ASSERT3U(db->db_state, ==, DB_CACHED);
264 
265 	holds = refcount_count(&db->db_holds);
266 	if (verify_type == DBVU_EVICTING) {
267 		/*
268 		 * Immediate eviction occurs when holds == dirtycnt.
269 		 * For normal eviction buffers, holds is zero on
270 		 * eviction, except when dbuf_fix_old_data() calls
271 		 * dbuf_clear_data().  However, the hold count can grow
272 		 * during eviction even though db_mtx is held (see
273 		 * dmu_bonus_hold() for an example), so we can only
274 		 * test the generic invariant that holds >= dirtycnt.
275 		 */
276 		ASSERT3U(holds, >=, db->db_dirtycnt);
277 	} else {
278 		if (db->db_user_immediate_evict == TRUE)
279 			ASSERT3U(holds, >=, db->db_dirtycnt);
280 		else
281 			ASSERT3U(holds, >, 0);
282 	}
283 #endif
284 }
285 
286 static void
287 dbuf_evict_user(dmu_buf_impl_t *db)
288 {
289 	dmu_buf_user_t *dbu = db->db_user;
290 
291 	ASSERT(MUTEX_HELD(&db->db_mtx));
292 
293 	if (dbu == NULL)
294 		return;
295 
296 	dbuf_verify_user(db, DBVU_EVICTING);
297 	db->db_user = NULL;
298 
299 #ifdef ZFS_DEBUG
300 	if (dbu->dbu_clear_on_evict_dbufp != NULL)
301 		*dbu->dbu_clear_on_evict_dbufp = NULL;
302 #endif
303 
304 	/*
305 	 * There are two eviction callbacks - one that we call synchronously
306 	 * and one that we invoke via a taskq.  The async one is useful for
307 	 * avoiding lock order reversals and limiting stack depth.
308 	 *
309 	 * Note that if we have a sync callback but no async callback,
310 	 * it's likely that the sync callback will free the structure
311 	 * containing the dbu.  In that case we need to take care to not
312 	 * dereference dbu after calling the sync evict func.
313 	 */
314 	boolean_t has_async = (dbu->dbu_evict_func_async != NULL);
315 
316 	if (dbu->dbu_evict_func_sync != NULL)
317 		dbu->dbu_evict_func_sync(dbu);
318 
319 	if (has_async) {
320 		taskq_dispatch_ent(dbu_evict_taskq, dbu->dbu_evict_func_async,
321 		    dbu, 0, &dbu->dbu_tqent);
322 	}
323 }
324 
325 boolean_t
326 dbuf_is_metadata(dmu_buf_impl_t *db)
327 {
328 	if (db->db_level > 0) {
329 		return (B_TRUE);
330 	} else {
331 		boolean_t is_metadata;
332 
333 		DB_DNODE_ENTER(db);
334 		is_metadata = DMU_OT_IS_METADATA(DB_DNODE(db)->dn_type);
335 		DB_DNODE_EXIT(db);
336 
337 		return (is_metadata);
338 	}
339 }
340 
341 void
342 dbuf_evict(dmu_buf_impl_t *db)
343 {
344 	ASSERT(MUTEX_HELD(&db->db_mtx));
345 	ASSERT(db->db_buf == NULL);
346 	ASSERT(db->db_data_pending == NULL);
347 
348 	dbuf_clear(db);
349 	dbuf_destroy(db);
350 }
351 
352 void
353 dbuf_init(void)
354 {
355 	uint64_t hsize = 1ULL << 16;
356 	dbuf_hash_table_t *h = &dbuf_hash_table;
357 	int i;
358 
359 	/*
360 	 * The hash table is big enough to fill all of physical memory
361 	 * with an average 4K block size.  The table will take up
362 	 * totalmem*sizeof(void*)/4K (i.e. 2MB/GB with 8-byte pointers).
363 	 */
364 	while (hsize * 4096 < physmem * PAGESIZE)
365 		hsize <<= 1;
366 
367 retry:
368 	h->hash_table_mask = hsize - 1;
369 	h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP);
370 	if (h->hash_table == NULL) {
371 		/* XXX - we should really return an error instead of assert */
372 		ASSERT(hsize > (1ULL << 10));
373 		hsize >>= 1;
374 		goto retry;
375 	}
376 
377 	dbuf_cache = kmem_cache_create("dmu_buf_impl_t",
378 	    sizeof (dmu_buf_impl_t),
379 	    0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0);
380 
381 	for (i = 0; i < DBUF_MUTEXES; i++)
382 		mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL);
383 
384 	/*
385 	 * All entries are queued via taskq_dispatch_ent(), so min/maxalloc
386 	 * configuration is not required.
387 	 */
388 	dbu_evict_taskq = taskq_create("dbu_evict", 1, minclsyspri, 0, 0, 0);
389 }
390 
391 void
392 dbuf_fini(void)
393 {
394 	dbuf_hash_table_t *h = &dbuf_hash_table;
395 	int i;
396 
397 	for (i = 0; i < DBUF_MUTEXES; i++)
398 		mutex_destroy(&h->hash_mutexes[i]);
399 	kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *));
400 	kmem_cache_destroy(dbuf_cache);
401 	taskq_destroy(dbu_evict_taskq);
402 }
403 
404 /*
405  * Other stuff.
406  */
407 
408 #ifdef ZFS_DEBUG
409 static void
410 dbuf_verify(dmu_buf_impl_t *db)
411 {
412 	dnode_t *dn;
413 	dbuf_dirty_record_t *dr;
414 
415 	ASSERT(MUTEX_HELD(&db->db_mtx));
416 
417 	if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY))
418 		return;
419 
420 	ASSERT(db->db_objset != NULL);
421 	DB_DNODE_ENTER(db);
422 	dn = DB_DNODE(db);
423 	if (dn == NULL) {
424 		ASSERT(db->db_parent == NULL);
425 		ASSERT(db->db_blkptr == NULL);
426 	} else {
427 		ASSERT3U(db->db.db_object, ==, dn->dn_object);
428 		ASSERT3P(db->db_objset, ==, dn->dn_objset);
429 		ASSERT3U(db->db_level, <, dn->dn_nlevels);
430 		ASSERT(db->db_blkid == DMU_BONUS_BLKID ||
431 		    db->db_blkid == DMU_SPILL_BLKID ||
432 		    !avl_is_empty(&dn->dn_dbufs));
433 	}
434 	if (db->db_blkid == DMU_BONUS_BLKID) {
435 		ASSERT(dn != NULL);
436 		ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen);
437 		ASSERT3U(db->db.db_offset, ==, DMU_BONUS_BLKID);
438 	} else if (db->db_blkid == DMU_SPILL_BLKID) {
439 		ASSERT(dn != NULL);
440 		ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen);
441 		ASSERT0(db->db.db_offset);
442 	} else {
443 		ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size);
444 	}
445 
446 	for (dr = db->db_data_pending; dr != NULL; dr = dr->dr_next)
447 		ASSERT(dr->dr_dbuf == db);
448 
449 	for (dr = db->db_last_dirty; dr != NULL; dr = dr->dr_next)
450 		ASSERT(dr->dr_dbuf == db);
451 
452 	/*
453 	 * We can't assert that db_size matches dn_datablksz because it
454 	 * can be momentarily different when another thread is doing
455 	 * dnode_set_blksz().
456 	 */
457 	if (db->db_level == 0 && db->db.db_object == DMU_META_DNODE_OBJECT) {
458 		dr = db->db_data_pending;
459 		/*
460 		 * It should only be modified in syncing context, so
461 		 * make sure we only have one copy of the data.
462 		 */
463 		ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf);
464 	}
465 
466 	/* verify db->db_blkptr */
467 	if (db->db_blkptr) {
468 		if (db->db_parent == dn->dn_dbuf) {
469 			/* db is pointed to by the dnode */
470 			/* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */
471 			if (DMU_OBJECT_IS_SPECIAL(db->db.db_object))
472 				ASSERT(db->db_parent == NULL);
473 			else
474 				ASSERT(db->db_parent != NULL);
475 			if (db->db_blkid != DMU_SPILL_BLKID)
476 				ASSERT3P(db->db_blkptr, ==,
477 				    &dn->dn_phys->dn_blkptr[db->db_blkid]);
478 		} else {
479 			/* db is pointed to by an indirect block */
480 			int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT;
481 			ASSERT3U(db->db_parent->db_level, ==, db->db_level+1);
482 			ASSERT3U(db->db_parent->db.db_object, ==,
483 			    db->db.db_object);
484 			/*
485 			 * dnode_grow_indblksz() can make this fail if we don't
486 			 * have the struct_rwlock.  XXX indblksz no longer
487 			 * grows.  safe to do this now?
488 			 */
489 			if (RW_WRITE_HELD(&dn->dn_struct_rwlock)) {
490 				ASSERT3P(db->db_blkptr, ==,
491 				    ((blkptr_t *)db->db_parent->db.db_data +
492 				    db->db_blkid % epb));
493 			}
494 		}
495 	}
496 	if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) &&
497 	    (db->db_buf == NULL || db->db_buf->b_data) &&
498 	    db->db.db_data && db->db_blkid != DMU_BONUS_BLKID &&
499 	    db->db_state != DB_FILL && !dn->dn_free_txg) {
500 		/*
501 		 * If the blkptr isn't set but they have nonzero data,
502 		 * it had better be dirty, otherwise we'll lose that
503 		 * data when we evict this buffer.
504 		 */
505 		if (db->db_dirtycnt == 0) {
506 			uint64_t *buf = db->db.db_data;
507 			int i;
508 
509 			for (i = 0; i < db->db.db_size >> 3; i++) {
510 				ASSERT(buf[i] == 0);
511 			}
512 		}
513 	}
514 	DB_DNODE_EXIT(db);
515 }
516 #endif
517 
518 static void
519 dbuf_clear_data(dmu_buf_impl_t *db)
520 {
521 	ASSERT(MUTEX_HELD(&db->db_mtx));
522 	dbuf_evict_user(db);
523 	db->db_buf = NULL;
524 	db->db.db_data = NULL;
525 	if (db->db_state != DB_NOFILL)
526 		db->db_state = DB_UNCACHED;
527 }
528 
529 static void
530 dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf)
531 {
532 	ASSERT(MUTEX_HELD(&db->db_mtx));
533 	ASSERT(buf != NULL);
534 
535 	db->db_buf = buf;
536 	ASSERT(buf->b_data != NULL);
537 	db->db.db_data = buf->b_data;
538 	if (!arc_released(buf))
539 		arc_set_callback(buf, dbuf_do_evict, db);
540 }
541 
542 /*
543  * Loan out an arc_buf for read.  Return the loaned arc_buf.
544  */
545 arc_buf_t *
546 dbuf_loan_arcbuf(dmu_buf_impl_t *db)
547 {
548 	arc_buf_t *abuf;
549 
550 	mutex_enter(&db->db_mtx);
551 	if (arc_released(db->db_buf) || refcount_count(&db->db_holds) > 1) {
552 		int blksz = db->db.db_size;
553 		spa_t *spa = db->db_objset->os_spa;
554 
555 		mutex_exit(&db->db_mtx);
556 		abuf = arc_loan_buf(spa, blksz);
557 		bcopy(db->db.db_data, abuf->b_data, blksz);
558 	} else {
559 		abuf = db->db_buf;
560 		arc_loan_inuse_buf(abuf, db);
561 		dbuf_clear_data(db);
562 		mutex_exit(&db->db_mtx);
563 	}
564 	return (abuf);
565 }
566 
567 /*
568  * Calculate which level n block references the data at the level 0 offset
569  * provided.
570  */
571 uint64_t
572 dbuf_whichblock(dnode_t *dn, int64_t level, uint64_t offset)
573 {
574 	if (dn->dn_datablkshift != 0 && dn->dn_indblkshift != 0) {
575 		/*
576 		 * The level n blkid is equal to the level 0 blkid divided by
577 		 * the number of level 0s in a level n block.
578 		 *
579 		 * The level 0 blkid is offset >> datablkshift =
580 		 * offset / 2^datablkshift.
581 		 *
582 		 * The number of level 0s in a level n is the number of block
583 		 * pointers in an indirect block, raised to the power of level.
584 		 * This is 2^(indblkshift - SPA_BLKPTRSHIFT)^level =
585 		 * 2^(level*(indblkshift - SPA_BLKPTRSHIFT)).
586 		 *
587 		 * Thus, the level n blkid is: offset /
588 		 * ((2^datablkshift)*(2^(level*(indblkshift - SPA_BLKPTRSHIFT)))
589 		 * = offset / 2^(datablkshift + level *
590 		 *   (indblkshift - SPA_BLKPTRSHIFT))
591 		 * = offset >> (datablkshift + level *
592 		 *   (indblkshift - SPA_BLKPTRSHIFT))
593 		 */
594 		return (offset >> (dn->dn_datablkshift + level *
595 		    (dn->dn_indblkshift - SPA_BLKPTRSHIFT)));
596 	} else {
597 		ASSERT3U(offset, <, dn->dn_datablksz);
598 		return (0);
599 	}
600 }
601 
602 static void
603 dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb)
604 {
605 	dmu_buf_impl_t *db = vdb;
606 
607 	mutex_enter(&db->db_mtx);
608 	ASSERT3U(db->db_state, ==, DB_READ);
609 	/*
610 	 * All reads are synchronous, so we must have a hold on the dbuf
611 	 */
612 	ASSERT(refcount_count(&db->db_holds) > 0);
613 	ASSERT(db->db_buf == NULL);
614 	ASSERT(db->db.db_data == NULL);
615 	if (db->db_level == 0 && db->db_freed_in_flight) {
616 		/* we were freed in flight; disregard any error */
617 		arc_release(buf, db);
618 		bzero(buf->b_data, db->db.db_size);
619 		arc_buf_freeze(buf);
620 		db->db_freed_in_flight = FALSE;
621 		dbuf_set_data(db, buf);
622 		db->db_state = DB_CACHED;
623 	} else if (zio == NULL || zio->io_error == 0) {
624 		dbuf_set_data(db, buf);
625 		db->db_state = DB_CACHED;
626 	} else {
627 		ASSERT(db->db_blkid != DMU_BONUS_BLKID);
628 		ASSERT3P(db->db_buf, ==, NULL);
629 		VERIFY(arc_buf_remove_ref(buf, db));
630 		db->db_state = DB_UNCACHED;
631 	}
632 	cv_broadcast(&db->db_changed);
633 	dbuf_rele_and_unlock(db, NULL);
634 }
635 
636 static void
637 dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags)
638 {
639 	dnode_t *dn;
640 	zbookmark_phys_t zb;
641 	arc_flags_t aflags = ARC_FLAG_NOWAIT;
642 
643 	DB_DNODE_ENTER(db);
644 	dn = DB_DNODE(db);
645 	ASSERT(!refcount_is_zero(&db->db_holds));
646 	/* We need the struct_rwlock to prevent db_blkptr from changing. */
647 	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
648 	ASSERT(MUTEX_HELD(&db->db_mtx));
649 	ASSERT(db->db_state == DB_UNCACHED);
650 	ASSERT(db->db_buf == NULL);
651 
652 	if (db->db_blkid == DMU_BONUS_BLKID) {
653 		int bonuslen = MIN(dn->dn_bonuslen, dn->dn_phys->dn_bonuslen);
654 
655 		ASSERT3U(bonuslen, <=, db->db.db_size);
656 		db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN);
657 		arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER);
658 		if (bonuslen < DN_MAX_BONUSLEN)
659 			bzero(db->db.db_data, DN_MAX_BONUSLEN);
660 		if (bonuslen)
661 			bcopy(DN_BONUS(dn->dn_phys), db->db.db_data, bonuslen);
662 		DB_DNODE_EXIT(db);
663 		db->db_state = DB_CACHED;
664 		mutex_exit(&db->db_mtx);
665 		return;
666 	}
667 
668 	/*
669 	 * Recheck BP_IS_HOLE() after dnode_block_freed() in case dnode_sync()
670 	 * processes the delete record and clears the bp while we are waiting
671 	 * for the dn_mtx (resulting in a "no" from block_freed).
672 	 */
673 	if (db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr) ||
674 	    (db->db_level == 0 && (dnode_block_freed(dn, db->db_blkid) ||
675 	    BP_IS_HOLE(db->db_blkptr)))) {
676 		arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
677 
678 		DB_DNODE_EXIT(db);
679 		dbuf_set_data(db, arc_buf_alloc(db->db_objset->os_spa,
680 		    db->db.db_size, db, type));
681 		bzero(db->db.db_data, db->db.db_size);
682 		db->db_state = DB_CACHED;
683 		mutex_exit(&db->db_mtx);
684 		return;
685 	}
686 
687 	DB_DNODE_EXIT(db);
688 
689 	db->db_state = DB_READ;
690 	mutex_exit(&db->db_mtx);
691 
692 	if (DBUF_IS_L2CACHEABLE(db))
693 		aflags |= ARC_FLAG_L2CACHE;
694 	if (DBUF_IS_L2COMPRESSIBLE(db))
695 		aflags |= ARC_FLAG_L2COMPRESS;
696 
697 	SET_BOOKMARK(&zb, db->db_objset->os_dsl_dataset ?
698 	    db->db_objset->os_dsl_dataset->ds_object : DMU_META_OBJSET,
699 	    db->db.db_object, db->db_level, db->db_blkid);
700 
701 	dbuf_add_ref(db, NULL);
702 
703 	(void) arc_read(zio, db->db_objset->os_spa, db->db_blkptr,
704 	    dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ,
705 	    (flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED,
706 	    &aflags, &zb);
707 }
708 
709 int
710 dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags)
711 {
712 	int err = 0;
713 	boolean_t havepzio = (zio != NULL);
714 	boolean_t prefetch;
715 	dnode_t *dn;
716 
717 	/*
718 	 * We don't have to hold the mutex to check db_state because it
719 	 * can't be freed while we have a hold on the buffer.
720 	 */
721 	ASSERT(!refcount_is_zero(&db->db_holds));
722 
723 	if (db->db_state == DB_NOFILL)
724 		return (SET_ERROR(EIO));
725 
726 	DB_DNODE_ENTER(db);
727 	dn = DB_DNODE(db);
728 	if ((flags & DB_RF_HAVESTRUCT) == 0)
729 		rw_enter(&dn->dn_struct_rwlock, RW_READER);
730 
731 	prefetch = db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID &&
732 	    (flags & DB_RF_NOPREFETCH) == 0 && dn != NULL &&
733 	    DBUF_IS_CACHEABLE(db);
734 
735 	mutex_enter(&db->db_mtx);
736 	if (db->db_state == DB_CACHED) {
737 		mutex_exit(&db->db_mtx);
738 		if (prefetch)
739 			dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1);
740 		if ((flags & DB_RF_HAVESTRUCT) == 0)
741 			rw_exit(&dn->dn_struct_rwlock);
742 		DB_DNODE_EXIT(db);
743 	} else if (db->db_state == DB_UNCACHED) {
744 		spa_t *spa = dn->dn_objset->os_spa;
745 
746 		if (zio == NULL)
747 			zio = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL);
748 		dbuf_read_impl(db, zio, flags);
749 
750 		/* dbuf_read_impl has dropped db_mtx for us */
751 
752 		if (prefetch)
753 			dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1);
754 
755 		if ((flags & DB_RF_HAVESTRUCT) == 0)
756 			rw_exit(&dn->dn_struct_rwlock);
757 		DB_DNODE_EXIT(db);
758 
759 		if (!havepzio)
760 			err = zio_wait(zio);
761 	} else {
762 		/*
763 		 * Another reader came in while the dbuf was in flight
764 		 * between UNCACHED and CACHED.  Either a writer will finish
765 		 * writing the buffer (sending the dbuf to CACHED) or the
766 		 * first reader's request will reach the read_done callback
767 		 * and send the dbuf to CACHED.  Otherwise, a failure
768 		 * occurred and the dbuf went to UNCACHED.
769 		 */
770 		mutex_exit(&db->db_mtx);
771 		if (prefetch)
772 			dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1);
773 		if ((flags & DB_RF_HAVESTRUCT) == 0)
774 			rw_exit(&dn->dn_struct_rwlock);
775 		DB_DNODE_EXIT(db);
776 
777 		/* Skip the wait per the caller's request. */
778 		mutex_enter(&db->db_mtx);
779 		if ((flags & DB_RF_NEVERWAIT) == 0) {
780 			while (db->db_state == DB_READ ||
781 			    db->db_state == DB_FILL) {
782 				ASSERT(db->db_state == DB_READ ||
783 				    (flags & DB_RF_HAVESTRUCT) == 0);
784 				DTRACE_PROBE2(blocked__read, dmu_buf_impl_t *,
785 				    db, zio_t *, zio);
786 				cv_wait(&db->db_changed, &db->db_mtx);
787 			}
788 			if (db->db_state == DB_UNCACHED)
789 				err = SET_ERROR(EIO);
790 		}
791 		mutex_exit(&db->db_mtx);
792 	}
793 
794 	ASSERT(err || havepzio || db->db_state == DB_CACHED);
795 	return (err);
796 }
797 
798 static void
799 dbuf_noread(dmu_buf_impl_t *db)
800 {
801 	ASSERT(!refcount_is_zero(&db->db_holds));
802 	ASSERT(db->db_blkid != DMU_BONUS_BLKID);
803 	mutex_enter(&db->db_mtx);
804 	while (db->db_state == DB_READ || db->db_state == DB_FILL)
805 		cv_wait(&db->db_changed, &db->db_mtx);
806 	if (db->db_state == DB_UNCACHED) {
807 		arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
808 		spa_t *spa = db->db_objset->os_spa;
809 
810 		ASSERT(db->db_buf == NULL);
811 		ASSERT(db->db.db_data == NULL);
812 		dbuf_set_data(db, arc_buf_alloc(spa, db->db.db_size, db, type));
813 		db->db_state = DB_FILL;
814 	} else if (db->db_state == DB_NOFILL) {
815 		dbuf_clear_data(db);
816 	} else {
817 		ASSERT3U(db->db_state, ==, DB_CACHED);
818 	}
819 	mutex_exit(&db->db_mtx);
820 }
821 
822 /*
823  * This is our just-in-time copy function.  It makes a copy of
824  * buffers, that have been modified in a previous transaction
825  * group, before we modify them in the current active group.
826  *
827  * This function is used in two places: when we are dirtying a
828  * buffer for the first time in a txg, and when we are freeing
829  * a range in a dnode that includes this buffer.
830  *
831  * Note that when we are called from dbuf_free_range() we do
832  * not put a hold on the buffer, we just traverse the active
833  * dbuf list for the dnode.
834  */
835 static void
836 dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg)
837 {
838 	dbuf_dirty_record_t *dr = db->db_last_dirty;
839 
840 	ASSERT(MUTEX_HELD(&db->db_mtx));
841 	ASSERT(db->db.db_data != NULL);
842 	ASSERT(db->db_level == 0);
843 	ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT);
844 
845 	if (dr == NULL ||
846 	    (dr->dt.dl.dr_data !=
847 	    ((db->db_blkid  == DMU_BONUS_BLKID) ? db->db.db_data : db->db_buf)))
848 		return;
849 
850 	/*
851 	 * If the last dirty record for this dbuf has not yet synced
852 	 * and its referencing the dbuf data, either:
853 	 *	reset the reference to point to a new copy,
854 	 * or (if there a no active holders)
855 	 *	just null out the current db_data pointer.
856 	 */
857 	ASSERT(dr->dr_txg >= txg - 2);
858 	if (db->db_blkid == DMU_BONUS_BLKID) {
859 		/* Note that the data bufs here are zio_bufs */
860 		dr->dt.dl.dr_data = zio_buf_alloc(DN_MAX_BONUSLEN);
861 		arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER);
862 		bcopy(db->db.db_data, dr->dt.dl.dr_data, DN_MAX_BONUSLEN);
863 	} else if (refcount_count(&db->db_holds) > db->db_dirtycnt) {
864 		int size = db->db.db_size;
865 		arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
866 		spa_t *spa = db->db_objset->os_spa;
867 
868 		dr->dt.dl.dr_data = arc_buf_alloc(spa, size, db, type);
869 		bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size);
870 	} else {
871 		dbuf_clear_data(db);
872 	}
873 }
874 
875 void
876 dbuf_unoverride(dbuf_dirty_record_t *dr)
877 {
878 	dmu_buf_impl_t *db = dr->dr_dbuf;
879 	blkptr_t *bp = &dr->dt.dl.dr_overridden_by;
880 	uint64_t txg = dr->dr_txg;
881 
882 	ASSERT(MUTEX_HELD(&db->db_mtx));
883 	ASSERT(dr->dt.dl.dr_override_state != DR_IN_DMU_SYNC);
884 	ASSERT(db->db_level == 0);
885 
886 	if (db->db_blkid == DMU_BONUS_BLKID ||
887 	    dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN)
888 		return;
889 
890 	ASSERT(db->db_data_pending != dr);
891 
892 	/* free this block */
893 	if (!BP_IS_HOLE(bp) && !dr->dt.dl.dr_nopwrite)
894 		zio_free(db->db_objset->os_spa, txg, bp);
895 
896 	dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN;
897 	dr->dt.dl.dr_nopwrite = B_FALSE;
898 
899 	/*
900 	 * Release the already-written buffer, so we leave it in
901 	 * a consistent dirty state.  Note that all callers are
902 	 * modifying the buffer, so they will immediately do
903 	 * another (redundant) arc_release().  Therefore, leave
904 	 * the buf thawed to save the effort of freezing &
905 	 * immediately re-thawing it.
906 	 */
907 	arc_release(dr->dt.dl.dr_data, db);
908 }
909 
910 /*
911  * Evict (if its unreferenced) or clear (if its referenced) any level-0
912  * data blocks in the free range, so that any future readers will find
913  * empty blocks.
914  *
915  * This is a no-op if the dataset is in the middle of an incremental
916  * receive; see comment below for details.
917  */
918 void
919 dbuf_free_range(dnode_t *dn, uint64_t start_blkid, uint64_t end_blkid,
920     dmu_tx_t *tx)
921 {
922 	dmu_buf_impl_t db_search;
923 	dmu_buf_impl_t *db, *db_next;
924 	uint64_t txg = tx->tx_txg;
925 	avl_index_t where;
926 
927 	if (end_blkid > dn->dn_maxblkid && (end_blkid != DMU_SPILL_BLKID))
928 		end_blkid = dn->dn_maxblkid;
929 	dprintf_dnode(dn, "start=%llu end=%llu\n", start_blkid, end_blkid);
930 
931 	db_search.db_level = 0;
932 	db_search.db_blkid = start_blkid;
933 	db_search.db_state = DB_SEARCH;
934 
935 	mutex_enter(&dn->dn_dbufs_mtx);
936 	if (start_blkid >= dn->dn_unlisted_l0_blkid) {
937 		/* There can't be any dbufs in this range; no need to search. */
938 #ifdef DEBUG
939 		db = avl_find(&dn->dn_dbufs, &db_search, &where);
940 		ASSERT3P(db, ==, NULL);
941 		db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER);
942 		ASSERT(db == NULL || db->db_level > 0);
943 #endif
944 		mutex_exit(&dn->dn_dbufs_mtx);
945 		return;
946 	} else if (dmu_objset_is_receiving(dn->dn_objset)) {
947 		/*
948 		 * If we are receiving, we expect there to be no dbufs in
949 		 * the range to be freed, because receive modifies each
950 		 * block at most once, and in offset order.  If this is
951 		 * not the case, it can lead to performance problems,
952 		 * so note that we unexpectedly took the slow path.
953 		 */
954 		atomic_inc_64(&zfs_free_range_recv_miss);
955 	}
956 
957 	db = avl_find(&dn->dn_dbufs, &db_search, &where);
958 	ASSERT3P(db, ==, NULL);
959 	db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER);
960 
961 	for (; db != NULL; db = db_next) {
962 		db_next = AVL_NEXT(&dn->dn_dbufs, db);
963 		ASSERT(db->db_blkid != DMU_BONUS_BLKID);
964 
965 		if (db->db_level != 0 || db->db_blkid > end_blkid) {
966 			break;
967 		}
968 		ASSERT3U(db->db_blkid, >=, start_blkid);
969 
970 		/* found a level 0 buffer in the range */
971 		mutex_enter(&db->db_mtx);
972 		if (dbuf_undirty(db, tx)) {
973 			/* mutex has been dropped and dbuf destroyed */
974 			continue;
975 		}
976 
977 		if (db->db_state == DB_UNCACHED ||
978 		    db->db_state == DB_NOFILL ||
979 		    db->db_state == DB_EVICTING) {
980 			ASSERT(db->db.db_data == NULL);
981 			mutex_exit(&db->db_mtx);
982 			continue;
983 		}
984 		if (db->db_state == DB_READ || db->db_state == DB_FILL) {
985 			/* will be handled in dbuf_read_done or dbuf_rele */
986 			db->db_freed_in_flight = TRUE;
987 			mutex_exit(&db->db_mtx);
988 			continue;
989 		}
990 		if (refcount_count(&db->db_holds) == 0) {
991 			ASSERT(db->db_buf);
992 			dbuf_clear(db);
993 			continue;
994 		}
995 		/* The dbuf is referenced */
996 
997 		if (db->db_last_dirty != NULL) {
998 			dbuf_dirty_record_t *dr = db->db_last_dirty;
999 
1000 			if (dr->dr_txg == txg) {
1001 				/*
1002 				 * This buffer is "in-use", re-adjust the file
1003 				 * size to reflect that this buffer may
1004 				 * contain new data when we sync.
1005 				 */
1006 				if (db->db_blkid != DMU_SPILL_BLKID &&
1007 				    db->db_blkid > dn->dn_maxblkid)
1008 					dn->dn_maxblkid = db->db_blkid;
1009 				dbuf_unoverride(dr);
1010 			} else {
1011 				/*
1012 				 * This dbuf is not dirty in the open context.
1013 				 * Either uncache it (if its not referenced in
1014 				 * the open context) or reset its contents to
1015 				 * empty.
1016 				 */
1017 				dbuf_fix_old_data(db, txg);
1018 			}
1019 		}
1020 		/* clear the contents if its cached */
1021 		if (db->db_state == DB_CACHED) {
1022 			ASSERT(db->db.db_data != NULL);
1023 			arc_release(db->db_buf, db);
1024 			bzero(db->db.db_data, db->db.db_size);
1025 			arc_buf_freeze(db->db_buf);
1026 		}
1027 
1028 		mutex_exit(&db->db_mtx);
1029 	}
1030 	mutex_exit(&dn->dn_dbufs_mtx);
1031 }
1032 
1033 static int
1034 dbuf_block_freeable(dmu_buf_impl_t *db)
1035 {
1036 	dsl_dataset_t *ds = db->db_objset->os_dsl_dataset;
1037 	uint64_t birth_txg = 0;
1038 
1039 	/*
1040 	 * We don't need any locking to protect db_blkptr:
1041 	 * If it's syncing, then db_last_dirty will be set
1042 	 * so we'll ignore db_blkptr.
1043 	 *
1044 	 * This logic ensures that only block births for
1045 	 * filled blocks are considered.
1046 	 */
1047 	ASSERT(MUTEX_HELD(&db->db_mtx));
1048 	if (db->db_last_dirty && (db->db_blkptr == NULL ||
1049 	    !BP_IS_HOLE(db->db_blkptr))) {
1050 		birth_txg = db->db_last_dirty->dr_txg;
1051 	} else if (db->db_blkptr != NULL && !BP_IS_HOLE(db->db_blkptr)) {
1052 		birth_txg = db->db_blkptr->blk_birth;
1053 	}
1054 
1055 	/*
1056 	 * If this block don't exist or is in a snapshot, it can't be freed.
1057 	 * Don't pass the bp to dsl_dataset_block_freeable() since we
1058 	 * are holding the db_mtx lock and might deadlock if we are
1059 	 * prefetching a dedup-ed block.
1060 	 */
1061 	if (birth_txg != 0)
1062 		return (ds == NULL ||
1063 		    dsl_dataset_block_freeable(ds, NULL, birth_txg));
1064 	else
1065 		return (B_FALSE);
1066 }
1067 
1068 void
1069 dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx)
1070 {
1071 	arc_buf_t *buf, *obuf;
1072 	int osize = db->db.db_size;
1073 	arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
1074 	dnode_t *dn;
1075 
1076 	ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1077 
1078 	DB_DNODE_ENTER(db);
1079 	dn = DB_DNODE(db);
1080 
1081 	/* XXX does *this* func really need the lock? */
1082 	ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock));
1083 
1084 	/*
1085 	 * This call to dmu_buf_will_dirty() with the dn_struct_rwlock held
1086 	 * is OK, because there can be no other references to the db
1087 	 * when we are changing its size, so no concurrent DB_FILL can
1088 	 * be happening.
1089 	 */
1090 	/*
1091 	 * XXX we should be doing a dbuf_read, checking the return
1092 	 * value and returning that up to our callers
1093 	 */
1094 	dmu_buf_will_dirty(&db->db, tx);
1095 
1096 	/* create the data buffer for the new block */
1097 	buf = arc_buf_alloc(dn->dn_objset->os_spa, size, db, type);
1098 
1099 	/* copy old block data to the new block */
1100 	obuf = db->db_buf;
1101 	bcopy(obuf->b_data, buf->b_data, MIN(osize, size));
1102 	/* zero the remainder */
1103 	if (size > osize)
1104 		bzero((uint8_t *)buf->b_data + osize, size - osize);
1105 
1106 	mutex_enter(&db->db_mtx);
1107 	dbuf_set_data(db, buf);
1108 	VERIFY(arc_buf_remove_ref(obuf, db));
1109 	db->db.db_size = size;
1110 
1111 	if (db->db_level == 0) {
1112 		ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg);
1113 		db->db_last_dirty->dt.dl.dr_data = buf;
1114 	}
1115 	mutex_exit(&db->db_mtx);
1116 
1117 	dnode_willuse_space(dn, size-osize, tx);
1118 	DB_DNODE_EXIT(db);
1119 }
1120 
1121 void
1122 dbuf_release_bp(dmu_buf_impl_t *db)
1123 {
1124 	objset_t *os = db->db_objset;
1125 
1126 	ASSERT(dsl_pool_sync_context(dmu_objset_pool(os)));
1127 	ASSERT(arc_released(os->os_phys_buf) ||
1128 	    list_link_active(&os->os_dsl_dataset->ds_synced_link));
1129 	ASSERT(db->db_parent == NULL || arc_released(db->db_parent->db_buf));
1130 
1131 	(void) arc_release(db->db_buf, db);
1132 }
1133 
1134 /*
1135  * We already have a dirty record for this TXG, and we are being
1136  * dirtied again.
1137  */
1138 static void
1139 dbuf_redirty(dbuf_dirty_record_t *dr)
1140 {
1141 	dmu_buf_impl_t *db = dr->dr_dbuf;
1142 
1143 	ASSERT(MUTEX_HELD(&db->db_mtx));
1144 
1145 	if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID) {
1146 		/*
1147 		 * If this buffer has already been written out,
1148 		 * we now need to reset its state.
1149 		 */
1150 		dbuf_unoverride(dr);
1151 		if (db->db.db_object != DMU_META_DNODE_OBJECT &&
1152 		    db->db_state != DB_NOFILL) {
1153 			/* Already released on initial dirty, so just thaw. */
1154 			ASSERT(arc_released(db->db_buf));
1155 			arc_buf_thaw(db->db_buf);
1156 		}
1157 	}
1158 }
1159 
1160 dbuf_dirty_record_t *
1161 dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
1162 {
1163 	dnode_t *dn;
1164 	objset_t *os;
1165 	dbuf_dirty_record_t **drp, *dr;
1166 	int drop_struct_lock = FALSE;
1167 	boolean_t do_free_accounting = B_FALSE;
1168 	int txgoff = tx->tx_txg & TXG_MASK;
1169 
1170 	ASSERT(tx->tx_txg != 0);
1171 	ASSERT(!refcount_is_zero(&db->db_holds));
1172 	DMU_TX_DIRTY_BUF(tx, db);
1173 
1174 	DB_DNODE_ENTER(db);
1175 	dn = DB_DNODE(db);
1176 	/*
1177 	 * Shouldn't dirty a regular buffer in syncing context.  Private
1178 	 * objects may be dirtied in syncing context, but only if they
1179 	 * were already pre-dirtied in open context.
1180 	 */
1181 	ASSERT(!dmu_tx_is_syncing(tx) ||
1182 	    BP_IS_HOLE(dn->dn_objset->os_rootbp) ||
1183 	    DMU_OBJECT_IS_SPECIAL(dn->dn_object) ||
1184 	    dn->dn_objset->os_dsl_dataset == NULL);
1185 	/*
1186 	 * We make this assert for private objects as well, but after we
1187 	 * check if we're already dirty.  They are allowed to re-dirty
1188 	 * in syncing context.
1189 	 */
1190 	ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT ||
1191 	    dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx ==
1192 	    (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN));
1193 
1194 	mutex_enter(&db->db_mtx);
1195 	/*
1196 	 * XXX make this true for indirects too?  The problem is that
1197 	 * transactions created with dmu_tx_create_assigned() from
1198 	 * syncing context don't bother holding ahead.
1199 	 */
1200 	ASSERT(db->db_level != 0 ||
1201 	    db->db_state == DB_CACHED || db->db_state == DB_FILL ||
1202 	    db->db_state == DB_NOFILL);
1203 
1204 	mutex_enter(&dn->dn_mtx);
1205 	/*
1206 	 * Don't set dirtyctx to SYNC if we're just modifying this as we
1207 	 * initialize the objset.
1208 	 */
1209 	if (dn->dn_dirtyctx == DN_UNDIRTIED &&
1210 	    !BP_IS_HOLE(dn->dn_objset->os_rootbp)) {
1211 		dn->dn_dirtyctx =
1212 		    (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN);
1213 		ASSERT(dn->dn_dirtyctx_firstset == NULL);
1214 		dn->dn_dirtyctx_firstset = kmem_alloc(1, KM_SLEEP);
1215 	}
1216 	mutex_exit(&dn->dn_mtx);
1217 
1218 	if (db->db_blkid == DMU_SPILL_BLKID)
1219 		dn->dn_have_spill = B_TRUE;
1220 
1221 	/*
1222 	 * If this buffer is already dirty, we're done.
1223 	 */
1224 	drp = &db->db_last_dirty;
1225 	ASSERT(*drp == NULL || (*drp)->dr_txg <= tx->tx_txg ||
1226 	    db->db.db_object == DMU_META_DNODE_OBJECT);
1227 	while ((dr = *drp) != NULL && dr->dr_txg > tx->tx_txg)
1228 		drp = &dr->dr_next;
1229 	if (dr && dr->dr_txg == tx->tx_txg) {
1230 		DB_DNODE_EXIT(db);
1231 
1232 		dbuf_redirty(dr);
1233 		mutex_exit(&db->db_mtx);
1234 		return (dr);
1235 	}
1236 
1237 	/*
1238 	 * Only valid if not already dirty.
1239 	 */
1240 	ASSERT(dn->dn_object == 0 ||
1241 	    dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx ==
1242 	    (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN));
1243 
1244 	ASSERT3U(dn->dn_nlevels, >, db->db_level);
1245 	ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) ||
1246 	    dn->dn_phys->dn_nlevels > db->db_level ||
1247 	    dn->dn_next_nlevels[txgoff] > db->db_level ||
1248 	    dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level ||
1249 	    dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level);
1250 
1251 	/*
1252 	 * We should only be dirtying in syncing context if it's the
1253 	 * mos or we're initializing the os or it's a special object.
1254 	 * However, we are allowed to dirty in syncing context provided
1255 	 * we already dirtied it in open context.  Hence we must make
1256 	 * this assertion only if we're not already dirty.
1257 	 */
1258 	os = dn->dn_objset;
1259 	ASSERT(!dmu_tx_is_syncing(tx) || DMU_OBJECT_IS_SPECIAL(dn->dn_object) ||
1260 	    os->os_dsl_dataset == NULL || BP_IS_HOLE(os->os_rootbp));
1261 	ASSERT(db->db.db_size != 0);
1262 
1263 	dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size);
1264 
1265 	if (db->db_blkid != DMU_BONUS_BLKID) {
1266 		/*
1267 		 * Update the accounting.
1268 		 * Note: we delay "free accounting" until after we drop
1269 		 * the db_mtx.  This keeps us from grabbing other locks
1270 		 * (and possibly deadlocking) in bp_get_dsize() while
1271 		 * also holding the db_mtx.
1272 		 */
1273 		dnode_willuse_space(dn, db->db.db_size, tx);
1274 		do_free_accounting = dbuf_block_freeable(db);
1275 	}
1276 
1277 	/*
1278 	 * If this buffer is dirty in an old transaction group we need
1279 	 * to make a copy of it so that the changes we make in this
1280 	 * transaction group won't leak out when we sync the older txg.
1281 	 */
1282 	dr = kmem_zalloc(sizeof (dbuf_dirty_record_t), KM_SLEEP);
1283 	if (db->db_level == 0) {
1284 		void *data_old = db->db_buf;
1285 
1286 		if (db->db_state != DB_NOFILL) {
1287 			if (db->db_blkid == DMU_BONUS_BLKID) {
1288 				dbuf_fix_old_data(db, tx->tx_txg);
1289 				data_old = db->db.db_data;
1290 			} else if (db->db.db_object != DMU_META_DNODE_OBJECT) {
1291 				/*
1292 				 * Release the data buffer from the cache so
1293 				 * that we can modify it without impacting
1294 				 * possible other users of this cached data
1295 				 * block.  Note that indirect blocks and
1296 				 * private objects are not released until the
1297 				 * syncing state (since they are only modified
1298 				 * then).
1299 				 */
1300 				arc_release(db->db_buf, db);
1301 				dbuf_fix_old_data(db, tx->tx_txg);
1302 				data_old = db->db_buf;
1303 			}
1304 			ASSERT(data_old != NULL);
1305 		}
1306 		dr->dt.dl.dr_data = data_old;
1307 	} else {
1308 		mutex_init(&dr->dt.di.dr_mtx, NULL, MUTEX_DEFAULT, NULL);
1309 		list_create(&dr->dt.di.dr_children,
1310 		    sizeof (dbuf_dirty_record_t),
1311 		    offsetof(dbuf_dirty_record_t, dr_dirty_node));
1312 	}
1313 	if (db->db_blkid != DMU_BONUS_BLKID && os->os_dsl_dataset != NULL)
1314 		dr->dr_accounted = db->db.db_size;
1315 	dr->dr_dbuf = db;
1316 	dr->dr_txg = tx->tx_txg;
1317 	dr->dr_next = *drp;
1318 	*drp = dr;
1319 
1320 	/*
1321 	 * We could have been freed_in_flight between the dbuf_noread
1322 	 * and dbuf_dirty.  We win, as though the dbuf_noread() had
1323 	 * happened after the free.
1324 	 */
1325 	if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID &&
1326 	    db->db_blkid != DMU_SPILL_BLKID) {
1327 		mutex_enter(&dn->dn_mtx);
1328 		if (dn->dn_free_ranges[txgoff] != NULL) {
1329 			range_tree_clear(dn->dn_free_ranges[txgoff],
1330 			    db->db_blkid, 1);
1331 		}
1332 		mutex_exit(&dn->dn_mtx);
1333 		db->db_freed_in_flight = FALSE;
1334 	}
1335 
1336 	/*
1337 	 * This buffer is now part of this txg
1338 	 */
1339 	dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg);
1340 	db->db_dirtycnt += 1;
1341 	ASSERT3U(db->db_dirtycnt, <=, 3);
1342 
1343 	mutex_exit(&db->db_mtx);
1344 
1345 	if (db->db_blkid == DMU_BONUS_BLKID ||
1346 	    db->db_blkid == DMU_SPILL_BLKID) {
1347 		mutex_enter(&dn->dn_mtx);
1348 		ASSERT(!list_link_active(&dr->dr_dirty_node));
1349 		list_insert_tail(&dn->dn_dirty_records[txgoff], dr);
1350 		mutex_exit(&dn->dn_mtx);
1351 		dnode_setdirty(dn, tx);
1352 		DB_DNODE_EXIT(db);
1353 		return (dr);
1354 	} else if (do_free_accounting) {
1355 		blkptr_t *bp = db->db_blkptr;
1356 		int64_t willfree = (bp && !BP_IS_HOLE(bp)) ?
1357 		    bp_get_dsize(os->os_spa, bp) : db->db.db_size;
1358 		/*
1359 		 * This is only a guess -- if the dbuf is dirty
1360 		 * in a previous txg, we don't know how much
1361 		 * space it will use on disk yet.  We should
1362 		 * really have the struct_rwlock to access
1363 		 * db_blkptr, but since this is just a guess,
1364 		 * it's OK if we get an odd answer.
1365 		 */
1366 		ddt_prefetch(os->os_spa, bp);
1367 		dnode_willuse_space(dn, -willfree, tx);
1368 	}
1369 
1370 	if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) {
1371 		rw_enter(&dn->dn_struct_rwlock, RW_READER);
1372 		drop_struct_lock = TRUE;
1373 	}
1374 
1375 	if (db->db_level == 0) {
1376 		dnode_new_blkid(dn, db->db_blkid, tx, drop_struct_lock);
1377 		ASSERT(dn->dn_maxblkid >= db->db_blkid);
1378 	}
1379 
1380 	if (db->db_level+1 < dn->dn_nlevels) {
1381 		dmu_buf_impl_t *parent = db->db_parent;
1382 		dbuf_dirty_record_t *di;
1383 		int parent_held = FALSE;
1384 
1385 		if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) {
1386 			int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
1387 
1388 			parent = dbuf_hold_level(dn, db->db_level+1,
1389 			    db->db_blkid >> epbs, FTAG);
1390 			ASSERT(parent != NULL);
1391 			parent_held = TRUE;
1392 		}
1393 		if (drop_struct_lock)
1394 			rw_exit(&dn->dn_struct_rwlock);
1395 		ASSERT3U(db->db_level+1, ==, parent->db_level);
1396 		di = dbuf_dirty(parent, tx);
1397 		if (parent_held)
1398 			dbuf_rele(parent, FTAG);
1399 
1400 		mutex_enter(&db->db_mtx);
1401 		/*
1402 		 * Since we've dropped the mutex, it's possible that
1403 		 * dbuf_undirty() might have changed this out from under us.
1404 		 */
1405 		if (db->db_last_dirty == dr ||
1406 		    dn->dn_object == DMU_META_DNODE_OBJECT) {
1407 			mutex_enter(&di->dt.di.dr_mtx);
1408 			ASSERT3U(di->dr_txg, ==, tx->tx_txg);
1409 			ASSERT(!list_link_active(&dr->dr_dirty_node));
1410 			list_insert_tail(&di->dt.di.dr_children, dr);
1411 			mutex_exit(&di->dt.di.dr_mtx);
1412 			dr->dr_parent = di;
1413 		}
1414 		mutex_exit(&db->db_mtx);
1415 	} else {
1416 		ASSERT(db->db_level+1 == dn->dn_nlevels);
1417 		ASSERT(db->db_blkid < dn->dn_nblkptr);
1418 		ASSERT(db->db_parent == NULL || db->db_parent == dn->dn_dbuf);
1419 		mutex_enter(&dn->dn_mtx);
1420 		ASSERT(!list_link_active(&dr->dr_dirty_node));
1421 		list_insert_tail(&dn->dn_dirty_records[txgoff], dr);
1422 		mutex_exit(&dn->dn_mtx);
1423 		if (drop_struct_lock)
1424 			rw_exit(&dn->dn_struct_rwlock);
1425 	}
1426 
1427 	dnode_setdirty(dn, tx);
1428 	DB_DNODE_EXIT(db);
1429 	return (dr);
1430 }
1431 
1432 /*
1433  * Undirty a buffer in the transaction group referenced by the given
1434  * transaction.  Return whether this evicted the dbuf.
1435  */
1436 static boolean_t
1437 dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
1438 {
1439 	dnode_t *dn;
1440 	uint64_t txg = tx->tx_txg;
1441 	dbuf_dirty_record_t *dr, **drp;
1442 
1443 	ASSERT(txg != 0);
1444 
1445 	/*
1446 	 * Due to our use of dn_nlevels below, this can only be called
1447 	 * in open context, unless we are operating on the MOS.
1448 	 * From syncing context, dn_nlevels may be different from the
1449 	 * dn_nlevels used when dbuf was dirtied.
1450 	 */
1451 	ASSERT(db->db_objset ==
1452 	    dmu_objset_pool(db->db_objset)->dp_meta_objset ||
1453 	    txg != spa_syncing_txg(dmu_objset_spa(db->db_objset)));
1454 	ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1455 	ASSERT0(db->db_level);
1456 	ASSERT(MUTEX_HELD(&db->db_mtx));
1457 
1458 	/*
1459 	 * If this buffer is not dirty, we're done.
1460 	 */
1461 	for (drp = &db->db_last_dirty; (dr = *drp) != NULL; drp = &dr->dr_next)
1462 		if (dr->dr_txg <= txg)
1463 			break;
1464 	if (dr == NULL || dr->dr_txg < txg)
1465 		return (B_FALSE);
1466 	ASSERT(dr->dr_txg == txg);
1467 	ASSERT(dr->dr_dbuf == db);
1468 
1469 	DB_DNODE_ENTER(db);
1470 	dn = DB_DNODE(db);
1471 
1472 	dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size);
1473 
1474 	ASSERT(db->db.db_size != 0);
1475 
1476 	dsl_pool_undirty_space(dmu_objset_pool(dn->dn_objset),
1477 	    dr->dr_accounted, txg);
1478 
1479 	*drp = dr->dr_next;
1480 
1481 	/*
1482 	 * Note that there are three places in dbuf_dirty()
1483 	 * where this dirty record may be put on a list.
1484 	 * Make sure to do a list_remove corresponding to
1485 	 * every one of those list_insert calls.
1486 	 */
1487 	if (dr->dr_parent) {
1488 		mutex_enter(&dr->dr_parent->dt.di.dr_mtx);
1489 		list_remove(&dr->dr_parent->dt.di.dr_children, dr);
1490 		mutex_exit(&dr->dr_parent->dt.di.dr_mtx);
1491 	} else if (db->db_blkid == DMU_SPILL_BLKID ||
1492 	    db->db_level + 1 == dn->dn_nlevels) {
1493 		ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf);
1494 		mutex_enter(&dn->dn_mtx);
1495 		list_remove(&dn->dn_dirty_records[txg & TXG_MASK], dr);
1496 		mutex_exit(&dn->dn_mtx);
1497 	}
1498 	DB_DNODE_EXIT(db);
1499 
1500 	if (db->db_state != DB_NOFILL) {
1501 		dbuf_unoverride(dr);
1502 
1503 		ASSERT(db->db_buf != NULL);
1504 		ASSERT(dr->dt.dl.dr_data != NULL);
1505 		if (dr->dt.dl.dr_data != db->db_buf)
1506 			VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, db));
1507 	}
1508 
1509 	kmem_free(dr, sizeof (dbuf_dirty_record_t));
1510 
1511 	ASSERT(db->db_dirtycnt > 0);
1512 	db->db_dirtycnt -= 1;
1513 
1514 	if (refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) {
1515 		arc_buf_t *buf = db->db_buf;
1516 
1517 		ASSERT(db->db_state == DB_NOFILL || arc_released(buf));
1518 		dbuf_clear_data(db);
1519 		VERIFY(arc_buf_remove_ref(buf, db));
1520 		dbuf_evict(db);
1521 		return (B_TRUE);
1522 	}
1523 
1524 	return (B_FALSE);
1525 }
1526 
1527 void
1528 dmu_buf_will_dirty(dmu_buf_t *db_fake, dmu_tx_t *tx)
1529 {
1530 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1531 	int rf = DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH;
1532 
1533 	ASSERT(tx->tx_txg != 0);
1534 	ASSERT(!refcount_is_zero(&db->db_holds));
1535 
1536 	/*
1537 	 * Quick check for dirtyness.  For already dirty blocks, this
1538 	 * reduces runtime of this function by >90%, and overall performance
1539 	 * by 50% for some workloads (e.g. file deletion with indirect blocks
1540 	 * cached).
1541 	 */
1542 	mutex_enter(&db->db_mtx);
1543 	dbuf_dirty_record_t *dr;
1544 	for (dr = db->db_last_dirty;
1545 	    dr != NULL && dr->dr_txg >= tx->tx_txg; dr = dr->dr_next) {
1546 		/*
1547 		 * It's possible that it is already dirty but not cached,
1548 		 * because there are some calls to dbuf_dirty() that don't
1549 		 * go through dmu_buf_will_dirty().
1550 		 */
1551 		if (dr->dr_txg == tx->tx_txg && db->db_state == DB_CACHED) {
1552 			/* This dbuf is already dirty and cached. */
1553 			dbuf_redirty(dr);
1554 			mutex_exit(&db->db_mtx);
1555 			return;
1556 		}
1557 	}
1558 	mutex_exit(&db->db_mtx);
1559 
1560 	DB_DNODE_ENTER(db);
1561 	if (RW_WRITE_HELD(&DB_DNODE(db)->dn_struct_rwlock))
1562 		rf |= DB_RF_HAVESTRUCT;
1563 	DB_DNODE_EXIT(db);
1564 	(void) dbuf_read(db, NULL, rf);
1565 	(void) dbuf_dirty(db, tx);
1566 }
1567 
1568 void
1569 dmu_buf_will_not_fill(dmu_buf_t *db_fake, dmu_tx_t *tx)
1570 {
1571 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1572 
1573 	db->db_state = DB_NOFILL;
1574 
1575 	dmu_buf_will_fill(db_fake, tx);
1576 }
1577 
1578 void
1579 dmu_buf_will_fill(dmu_buf_t *db_fake, dmu_tx_t *tx)
1580 {
1581 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1582 
1583 	ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1584 	ASSERT(tx->tx_txg != 0);
1585 	ASSERT(db->db_level == 0);
1586 	ASSERT(!refcount_is_zero(&db->db_holds));
1587 
1588 	ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT ||
1589 	    dmu_tx_private_ok(tx));
1590 
1591 	dbuf_noread(db);
1592 	(void) dbuf_dirty(db, tx);
1593 }
1594 
1595 #pragma weak dmu_buf_fill_done = dbuf_fill_done
1596 /* ARGSUSED */
1597 void
1598 dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx)
1599 {
1600 	mutex_enter(&db->db_mtx);
1601 	DBUF_VERIFY(db);
1602 
1603 	if (db->db_state == DB_FILL) {
1604 		if (db->db_level == 0 && db->db_freed_in_flight) {
1605 			ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1606 			/* we were freed while filling */
1607 			/* XXX dbuf_undirty? */
1608 			bzero(db->db.db_data, db->db.db_size);
1609 			db->db_freed_in_flight = FALSE;
1610 		}
1611 		db->db_state = DB_CACHED;
1612 		cv_broadcast(&db->db_changed);
1613 	}
1614 	mutex_exit(&db->db_mtx);
1615 }
1616 
1617 void
1618 dmu_buf_write_embedded(dmu_buf_t *dbuf, void *data,
1619     bp_embedded_type_t etype, enum zio_compress comp,
1620     int uncompressed_size, int compressed_size, int byteorder,
1621     dmu_tx_t *tx)
1622 {
1623 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf;
1624 	struct dirty_leaf *dl;
1625 	dmu_object_type_t type;
1626 
1627 	if (etype == BP_EMBEDDED_TYPE_DATA) {
1628 		ASSERT(spa_feature_is_active(dmu_objset_spa(db->db_objset),
1629 		    SPA_FEATURE_EMBEDDED_DATA));
1630 	}
1631 
1632 	DB_DNODE_ENTER(db);
1633 	type = DB_DNODE(db)->dn_type;
1634 	DB_DNODE_EXIT(db);
1635 
1636 	ASSERT0(db->db_level);
1637 	ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1638 
1639 	dmu_buf_will_not_fill(dbuf, tx);
1640 
1641 	ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg);
1642 	dl = &db->db_last_dirty->dt.dl;
1643 	encode_embedded_bp_compressed(&dl->dr_overridden_by,
1644 	    data, comp, uncompressed_size, compressed_size);
1645 	BPE_SET_ETYPE(&dl->dr_overridden_by, etype);
1646 	BP_SET_TYPE(&dl->dr_overridden_by, type);
1647 	BP_SET_LEVEL(&dl->dr_overridden_by, 0);
1648 	BP_SET_BYTEORDER(&dl->dr_overridden_by, byteorder);
1649 
1650 	dl->dr_override_state = DR_OVERRIDDEN;
1651 	dl->dr_overridden_by.blk_birth = db->db_last_dirty->dr_txg;
1652 }
1653 
1654 /*
1655  * Directly assign a provided arc buf to a given dbuf if it's not referenced
1656  * by anybody except our caller. Otherwise copy arcbuf's contents to dbuf.
1657  */
1658 void
1659 dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx)
1660 {
1661 	ASSERT(!refcount_is_zero(&db->db_holds));
1662 	ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1663 	ASSERT(db->db_level == 0);
1664 	ASSERT(DBUF_GET_BUFC_TYPE(db) == ARC_BUFC_DATA);
1665 	ASSERT(buf != NULL);
1666 	ASSERT(arc_buf_size(buf) == db->db.db_size);
1667 	ASSERT(tx->tx_txg != 0);
1668 
1669 	arc_return_buf(buf, db);
1670 	ASSERT(arc_released(buf));
1671 
1672 	mutex_enter(&db->db_mtx);
1673 
1674 	while (db->db_state == DB_READ || db->db_state == DB_FILL)
1675 		cv_wait(&db->db_changed, &db->db_mtx);
1676 
1677 	ASSERT(db->db_state == DB_CACHED || db->db_state == DB_UNCACHED);
1678 
1679 	if (db->db_state == DB_CACHED &&
1680 	    refcount_count(&db->db_holds) - 1 > db->db_dirtycnt) {
1681 		mutex_exit(&db->db_mtx);
1682 		(void) dbuf_dirty(db, tx);
1683 		bcopy(buf->b_data, db->db.db_data, db->db.db_size);
1684 		VERIFY(arc_buf_remove_ref(buf, db));
1685 		xuio_stat_wbuf_copied();
1686 		return;
1687 	}
1688 
1689 	xuio_stat_wbuf_nocopy();
1690 	if (db->db_state == DB_CACHED) {
1691 		dbuf_dirty_record_t *dr = db->db_last_dirty;
1692 
1693 		ASSERT(db->db_buf != NULL);
1694 		if (dr != NULL && dr->dr_txg == tx->tx_txg) {
1695 			ASSERT(dr->dt.dl.dr_data == db->db_buf);
1696 			if (!arc_released(db->db_buf)) {
1697 				ASSERT(dr->dt.dl.dr_override_state ==
1698 				    DR_OVERRIDDEN);
1699 				arc_release(db->db_buf, db);
1700 			}
1701 			dr->dt.dl.dr_data = buf;
1702 			VERIFY(arc_buf_remove_ref(db->db_buf, db));
1703 		} else if (dr == NULL || dr->dt.dl.dr_data != db->db_buf) {
1704 			arc_release(db->db_buf, db);
1705 			VERIFY(arc_buf_remove_ref(db->db_buf, db));
1706 		}
1707 		db->db_buf = NULL;
1708 	}
1709 	ASSERT(db->db_buf == NULL);
1710 	dbuf_set_data(db, buf);
1711 	db->db_state = DB_FILL;
1712 	mutex_exit(&db->db_mtx);
1713 	(void) dbuf_dirty(db, tx);
1714 	dmu_buf_fill_done(&db->db, tx);
1715 }
1716 
1717 /*
1718  * "Clear" the contents of this dbuf.  This will mark the dbuf
1719  * EVICTING and clear *most* of its references.  Unfortunately,
1720  * when we are not holding the dn_dbufs_mtx, we can't clear the
1721  * entry in the dn_dbufs list.  We have to wait until dbuf_destroy()
1722  * in this case.  For callers from the DMU we will usually see:
1723  *	dbuf_clear()->arc_clear_callback()->dbuf_do_evict()->dbuf_destroy()
1724  * For the arc callback, we will usually see:
1725  *	dbuf_do_evict()->dbuf_clear();dbuf_destroy()
1726  * Sometimes, though, we will get a mix of these two:
1727  *	DMU: dbuf_clear()->arc_clear_callback()
1728  *	ARC: dbuf_do_evict()->dbuf_destroy()
1729  *
1730  * This routine will dissociate the dbuf from the arc, by calling
1731  * arc_clear_callback(), but will not evict the data from the ARC.
1732  */
1733 void
1734 dbuf_clear(dmu_buf_impl_t *db)
1735 {
1736 	dnode_t *dn;
1737 	dmu_buf_impl_t *parent = db->db_parent;
1738 	dmu_buf_impl_t *dndb;
1739 	boolean_t dbuf_gone = B_FALSE;
1740 
1741 	ASSERT(MUTEX_HELD(&db->db_mtx));
1742 	ASSERT(refcount_is_zero(&db->db_holds));
1743 
1744 	dbuf_evict_user(db);
1745 
1746 	if (db->db_state == DB_CACHED) {
1747 		ASSERT(db->db.db_data != NULL);
1748 		if (db->db_blkid == DMU_BONUS_BLKID) {
1749 			zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN);
1750 			arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER);
1751 		}
1752 		db->db.db_data = NULL;
1753 		db->db_state = DB_UNCACHED;
1754 	}
1755 
1756 	ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL);
1757 	ASSERT(db->db_data_pending == NULL);
1758 
1759 	db->db_state = DB_EVICTING;
1760 	db->db_blkptr = NULL;
1761 
1762 	DB_DNODE_ENTER(db);
1763 	dn = DB_DNODE(db);
1764 	dndb = dn->dn_dbuf;
1765 	if (db->db_blkid != DMU_BONUS_BLKID && MUTEX_HELD(&dn->dn_dbufs_mtx)) {
1766 		avl_remove(&dn->dn_dbufs, db);
1767 		atomic_dec_32(&dn->dn_dbufs_count);
1768 		membar_producer();
1769 		DB_DNODE_EXIT(db);
1770 		/*
1771 		 * Decrementing the dbuf count means that the hold corresponding
1772 		 * to the removed dbuf is no longer discounted in dnode_move(),
1773 		 * so the dnode cannot be moved until after we release the hold.
1774 		 * The membar_producer() ensures visibility of the decremented
1775 		 * value in dnode_move(), since DB_DNODE_EXIT doesn't actually
1776 		 * release any lock.
1777 		 */
1778 		dnode_rele(dn, db);
1779 		db->db_dnode_handle = NULL;
1780 	} else {
1781 		DB_DNODE_EXIT(db);
1782 	}
1783 
1784 	if (db->db_buf)
1785 		dbuf_gone = arc_clear_callback(db->db_buf);
1786 
1787 	if (!dbuf_gone)
1788 		mutex_exit(&db->db_mtx);
1789 
1790 	/*
1791 	 * If this dbuf is referenced from an indirect dbuf,
1792 	 * decrement the ref count on the indirect dbuf.
1793 	 */
1794 	if (parent && parent != dndb)
1795 		dbuf_rele(parent, db);
1796 }
1797 
1798 /*
1799  * Note: While bpp will always be updated if the function returns success,
1800  * parentp will not be updated if the dnode does not have dn_dbuf filled in;
1801  * this happens when the dnode is the meta-dnode, or a userused or groupused
1802  * object.
1803  */
1804 static int
1805 dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse,
1806     dmu_buf_impl_t **parentp, blkptr_t **bpp)
1807 {
1808 	int nlevels, epbs;
1809 
1810 	*parentp = NULL;
1811 	*bpp = NULL;
1812 
1813 	ASSERT(blkid != DMU_BONUS_BLKID);
1814 
1815 	if (blkid == DMU_SPILL_BLKID) {
1816 		mutex_enter(&dn->dn_mtx);
1817 		if (dn->dn_have_spill &&
1818 		    (dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR))
1819 			*bpp = &dn->dn_phys->dn_spill;
1820 		else
1821 			*bpp = NULL;
1822 		dbuf_add_ref(dn->dn_dbuf, NULL);
1823 		*parentp = dn->dn_dbuf;
1824 		mutex_exit(&dn->dn_mtx);
1825 		return (0);
1826 	}
1827 
1828 	if (dn->dn_phys->dn_nlevels == 0)
1829 		nlevels = 1;
1830 	else
1831 		nlevels = dn->dn_phys->dn_nlevels;
1832 
1833 	epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
1834 
1835 	ASSERT3U(level * epbs, <, 64);
1836 	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
1837 	if (level >= nlevels ||
1838 	    (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) {
1839 		/* the buffer has no parent yet */
1840 		return (SET_ERROR(ENOENT));
1841 	} else if (level < nlevels-1) {
1842 		/* this block is referenced from an indirect block */
1843 		int err = dbuf_hold_impl(dn, level+1,
1844 		    blkid >> epbs, fail_sparse, FALSE, NULL, parentp);
1845 		if (err)
1846 			return (err);
1847 		err = dbuf_read(*parentp, NULL,
1848 		    (DB_RF_HAVESTRUCT | DB_RF_NOPREFETCH | DB_RF_CANFAIL));
1849 		if (err) {
1850 			dbuf_rele(*parentp, NULL);
1851 			*parentp = NULL;
1852 			return (err);
1853 		}
1854 		*bpp = ((blkptr_t *)(*parentp)->db.db_data) +
1855 		    (blkid & ((1ULL << epbs) - 1));
1856 		return (0);
1857 	} else {
1858 		/* the block is referenced from the dnode */
1859 		ASSERT3U(level, ==, nlevels-1);
1860 		ASSERT(dn->dn_phys->dn_nblkptr == 0 ||
1861 		    blkid < dn->dn_phys->dn_nblkptr);
1862 		if (dn->dn_dbuf) {
1863 			dbuf_add_ref(dn->dn_dbuf, NULL);
1864 			*parentp = dn->dn_dbuf;
1865 		}
1866 		*bpp = &dn->dn_phys->dn_blkptr[blkid];
1867 		return (0);
1868 	}
1869 }
1870 
1871 static dmu_buf_impl_t *
1872 dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid,
1873     dmu_buf_impl_t *parent, blkptr_t *blkptr)
1874 {
1875 	objset_t *os = dn->dn_objset;
1876 	dmu_buf_impl_t *db, *odb;
1877 
1878 	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
1879 	ASSERT(dn->dn_type != DMU_OT_NONE);
1880 
1881 	db = kmem_cache_alloc(dbuf_cache, KM_SLEEP);
1882 
1883 	db->db_objset = os;
1884 	db->db.db_object = dn->dn_object;
1885 	db->db_level = level;
1886 	db->db_blkid = blkid;
1887 	db->db_last_dirty = NULL;
1888 	db->db_dirtycnt = 0;
1889 	db->db_dnode_handle = dn->dn_handle;
1890 	db->db_parent = parent;
1891 	db->db_blkptr = blkptr;
1892 
1893 	db->db_user = NULL;
1894 	db->db_user_immediate_evict = FALSE;
1895 	db->db_freed_in_flight = FALSE;
1896 	db->db_pending_evict = FALSE;
1897 
1898 	if (blkid == DMU_BONUS_BLKID) {
1899 		ASSERT3P(parent, ==, dn->dn_dbuf);
1900 		db->db.db_size = DN_MAX_BONUSLEN -
1901 		    (dn->dn_nblkptr-1) * sizeof (blkptr_t);
1902 		ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen);
1903 		db->db.db_offset = DMU_BONUS_BLKID;
1904 		db->db_state = DB_UNCACHED;
1905 		/* the bonus dbuf is not placed in the hash table */
1906 		arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER);
1907 		return (db);
1908 	} else if (blkid == DMU_SPILL_BLKID) {
1909 		db->db.db_size = (blkptr != NULL) ?
1910 		    BP_GET_LSIZE(blkptr) : SPA_MINBLOCKSIZE;
1911 		db->db.db_offset = 0;
1912 	} else {
1913 		int blocksize =
1914 		    db->db_level ? 1 << dn->dn_indblkshift : dn->dn_datablksz;
1915 		db->db.db_size = blocksize;
1916 		db->db.db_offset = db->db_blkid * blocksize;
1917 	}
1918 
1919 	/*
1920 	 * Hold the dn_dbufs_mtx while we get the new dbuf
1921 	 * in the hash table *and* added to the dbufs list.
1922 	 * This prevents a possible deadlock with someone
1923 	 * trying to look up this dbuf before its added to the
1924 	 * dn_dbufs list.
1925 	 */
1926 	mutex_enter(&dn->dn_dbufs_mtx);
1927 	db->db_state = DB_EVICTING;
1928 	if ((odb = dbuf_hash_insert(db)) != NULL) {
1929 		/* someone else inserted it first */
1930 		kmem_cache_free(dbuf_cache, db);
1931 		mutex_exit(&dn->dn_dbufs_mtx);
1932 		return (odb);
1933 	}
1934 	avl_add(&dn->dn_dbufs, db);
1935 	if (db->db_level == 0 && db->db_blkid >=
1936 	    dn->dn_unlisted_l0_blkid)
1937 		dn->dn_unlisted_l0_blkid = db->db_blkid + 1;
1938 	db->db_state = DB_UNCACHED;
1939 	mutex_exit(&dn->dn_dbufs_mtx);
1940 	arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER);
1941 
1942 	if (parent && parent != dn->dn_dbuf)
1943 		dbuf_add_ref(parent, db);
1944 
1945 	ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT ||
1946 	    refcount_count(&dn->dn_holds) > 0);
1947 	(void) refcount_add(&dn->dn_holds, db);
1948 	atomic_inc_32(&dn->dn_dbufs_count);
1949 
1950 	dprintf_dbuf(db, "db=%p\n", db);
1951 
1952 	return (db);
1953 }
1954 
1955 static int
1956 dbuf_do_evict(void *private)
1957 {
1958 	dmu_buf_impl_t *db = private;
1959 
1960 	if (!MUTEX_HELD(&db->db_mtx))
1961 		mutex_enter(&db->db_mtx);
1962 
1963 	ASSERT(refcount_is_zero(&db->db_holds));
1964 
1965 	if (db->db_state != DB_EVICTING) {
1966 		ASSERT(db->db_state == DB_CACHED);
1967 		DBUF_VERIFY(db);
1968 		db->db_buf = NULL;
1969 		dbuf_evict(db);
1970 	} else {
1971 		mutex_exit(&db->db_mtx);
1972 		dbuf_destroy(db);
1973 	}
1974 	return (0);
1975 }
1976 
1977 static void
1978 dbuf_destroy(dmu_buf_impl_t *db)
1979 {
1980 	ASSERT(refcount_is_zero(&db->db_holds));
1981 
1982 	if (db->db_blkid != DMU_BONUS_BLKID) {
1983 		/*
1984 		 * If this dbuf is still on the dn_dbufs list,
1985 		 * remove it from that list.
1986 		 */
1987 		if (db->db_dnode_handle != NULL) {
1988 			dnode_t *dn;
1989 
1990 			DB_DNODE_ENTER(db);
1991 			dn = DB_DNODE(db);
1992 			mutex_enter(&dn->dn_dbufs_mtx);
1993 			avl_remove(&dn->dn_dbufs, db);
1994 			atomic_dec_32(&dn->dn_dbufs_count);
1995 			mutex_exit(&dn->dn_dbufs_mtx);
1996 			DB_DNODE_EXIT(db);
1997 			/*
1998 			 * Decrementing the dbuf count means that the hold
1999 			 * corresponding to the removed dbuf is no longer
2000 			 * discounted in dnode_move(), so the dnode cannot be
2001 			 * moved until after we release the hold.
2002 			 */
2003 			dnode_rele(dn, db);
2004 			db->db_dnode_handle = NULL;
2005 		}
2006 		dbuf_hash_remove(db);
2007 	}
2008 	db->db_parent = NULL;
2009 	db->db_buf = NULL;
2010 
2011 	ASSERT(db->db.db_data == NULL);
2012 	ASSERT(db->db_hash_next == NULL);
2013 	ASSERT(db->db_blkptr == NULL);
2014 	ASSERT(db->db_data_pending == NULL);
2015 
2016 	kmem_cache_free(dbuf_cache, db);
2017 	arc_space_return(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER);
2018 }
2019 
2020 typedef struct dbuf_prefetch_arg {
2021 	spa_t *dpa_spa;	/* The spa to issue the prefetch in. */
2022 	zbookmark_phys_t dpa_zb; /* The target block to prefetch. */
2023 	int dpa_epbs; /* Entries (blkptr_t's) Per Block Shift. */
2024 	int dpa_curlevel; /* The current level that we're reading */
2025 	zio_priority_t dpa_prio; /* The priority I/Os should be issued at. */
2026 	zio_t *dpa_zio; /* The parent zio_t for all prefetches. */
2027 	arc_flags_t dpa_aflags; /* Flags to pass to the final prefetch. */
2028 } dbuf_prefetch_arg_t;
2029 
2030 /*
2031  * Actually issue the prefetch read for the block given.
2032  */
2033 static void
2034 dbuf_issue_final_prefetch(dbuf_prefetch_arg_t *dpa, blkptr_t *bp)
2035 {
2036 	if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp))
2037 		return;
2038 
2039 	arc_flags_t aflags =
2040 	    dpa->dpa_aflags | ARC_FLAG_NOWAIT | ARC_FLAG_PREFETCH;
2041 
2042 	ASSERT3U(dpa->dpa_curlevel, ==, BP_GET_LEVEL(bp));
2043 	ASSERT3U(dpa->dpa_curlevel, ==, dpa->dpa_zb.zb_level);
2044 	ASSERT(dpa->dpa_zio != NULL);
2045 	(void) arc_read(dpa->dpa_zio, dpa->dpa_spa, bp, NULL, NULL,
2046 	    dpa->dpa_prio, ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE,
2047 	    &aflags, &dpa->dpa_zb);
2048 }
2049 
2050 /*
2051  * Called when an indirect block above our prefetch target is read in.  This
2052  * will either read in the next indirect block down the tree or issue the actual
2053  * prefetch if the next block down is our target.
2054  */
2055 static void
2056 dbuf_prefetch_indirect_done(zio_t *zio, arc_buf_t *abuf, void *private)
2057 {
2058 	dbuf_prefetch_arg_t *dpa = private;
2059 
2060 	ASSERT3S(dpa->dpa_zb.zb_level, <, dpa->dpa_curlevel);
2061 	ASSERT3S(dpa->dpa_curlevel, >, 0);
2062 	if (zio != NULL) {
2063 		ASSERT3S(BP_GET_LEVEL(zio->io_bp), ==, dpa->dpa_curlevel);
2064 		ASSERT3U(BP_GET_LSIZE(zio->io_bp), ==, zio->io_size);
2065 		ASSERT3P(zio->io_spa, ==, dpa->dpa_spa);
2066 	}
2067 
2068 	dpa->dpa_curlevel--;
2069 
2070 	uint64_t nextblkid = dpa->dpa_zb.zb_blkid >>
2071 	    (dpa->dpa_epbs * (dpa->dpa_curlevel - dpa->dpa_zb.zb_level));
2072 	blkptr_t *bp = ((blkptr_t *)abuf->b_data) +
2073 	    P2PHASE(nextblkid, 1ULL << dpa->dpa_epbs);
2074 	if (BP_IS_HOLE(bp) || (zio != NULL && zio->io_error != 0)) {
2075 		kmem_free(dpa, sizeof (*dpa));
2076 	} else if (dpa->dpa_curlevel == dpa->dpa_zb.zb_level) {
2077 		ASSERT3U(nextblkid, ==, dpa->dpa_zb.zb_blkid);
2078 		dbuf_issue_final_prefetch(dpa, bp);
2079 		kmem_free(dpa, sizeof (*dpa));
2080 	} else {
2081 		arc_flags_t iter_aflags = ARC_FLAG_NOWAIT;
2082 		zbookmark_phys_t zb;
2083 
2084 		ASSERT3U(dpa->dpa_curlevel, ==, BP_GET_LEVEL(bp));
2085 
2086 		SET_BOOKMARK(&zb, dpa->dpa_zb.zb_objset,
2087 		    dpa->dpa_zb.zb_object, dpa->dpa_curlevel, nextblkid);
2088 
2089 		(void) arc_read(dpa->dpa_zio, dpa->dpa_spa,
2090 		    bp, dbuf_prefetch_indirect_done, dpa, dpa->dpa_prio,
2091 		    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE,
2092 		    &iter_aflags, &zb);
2093 	}
2094 	(void) arc_buf_remove_ref(abuf, private);
2095 }
2096 
2097 /*
2098  * Issue prefetch reads for the given block on the given level.  If the indirect
2099  * blocks above that block are not in memory, we will read them in
2100  * asynchronously.  As a result, this call never blocks waiting for a read to
2101  * complete.
2102  */
2103 void
2104 dbuf_prefetch(dnode_t *dn, int64_t level, uint64_t blkid, zio_priority_t prio,
2105     arc_flags_t aflags)
2106 {
2107 	blkptr_t bp;
2108 	int epbs, nlevels, curlevel;
2109 	uint64_t curblkid;
2110 
2111 	ASSERT(blkid != DMU_BONUS_BLKID);
2112 	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
2113 
2114 	if (blkid > dn->dn_maxblkid)
2115 		return;
2116 
2117 	if (dnode_block_freed(dn, blkid))
2118 		return;
2119 
2120 	/*
2121 	 * This dnode hasn't been written to disk yet, so there's nothing to
2122 	 * prefetch.
2123 	 */
2124 	nlevels = dn->dn_phys->dn_nlevels;
2125 	if (level >= nlevels || dn->dn_phys->dn_nblkptr == 0)
2126 		return;
2127 
2128 	epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;
2129 	if (dn->dn_phys->dn_maxblkid < blkid << (epbs * level))
2130 		return;
2131 
2132 	dmu_buf_impl_t *db = dbuf_find(dn->dn_objset, dn->dn_object,
2133 	    level, blkid);
2134 	if (db != NULL) {
2135 		mutex_exit(&db->db_mtx);
2136 		/*
2137 		 * This dbuf already exists.  It is either CACHED, or
2138 		 * (we assume) about to be read or filled.
2139 		 */
2140 		return;
2141 	}
2142 
2143 	/*
2144 	 * Find the closest ancestor (indirect block) of the target block
2145 	 * that is present in the cache.  In this indirect block, we will
2146 	 * find the bp that is at curlevel, curblkid.
2147 	 */
2148 	curlevel = level;
2149 	curblkid = blkid;
2150 	while (curlevel < nlevels - 1) {
2151 		int parent_level = curlevel + 1;
2152 		uint64_t parent_blkid = curblkid >> epbs;
2153 		dmu_buf_impl_t *db;
2154 
2155 		if (dbuf_hold_impl(dn, parent_level, parent_blkid,
2156 		    FALSE, TRUE, FTAG, &db) == 0) {
2157 			blkptr_t *bpp = db->db_buf->b_data;
2158 			bp = bpp[P2PHASE(curblkid, 1 << epbs)];
2159 			dbuf_rele(db, FTAG);
2160 			break;
2161 		}
2162 
2163 		curlevel = parent_level;
2164 		curblkid = parent_blkid;
2165 	}
2166 
2167 	if (curlevel == nlevels - 1) {
2168 		/* No cached indirect blocks found. */
2169 		ASSERT3U(curblkid, <, dn->dn_phys->dn_nblkptr);
2170 		bp = dn->dn_phys->dn_blkptr[curblkid];
2171 	}
2172 	if (BP_IS_HOLE(&bp))
2173 		return;
2174 
2175 	ASSERT3U(curlevel, ==, BP_GET_LEVEL(&bp));
2176 
2177 	zio_t *pio = zio_root(dmu_objset_spa(dn->dn_objset), NULL, NULL,
2178 	    ZIO_FLAG_CANFAIL);
2179 
2180 	dbuf_prefetch_arg_t *dpa = kmem_zalloc(sizeof (*dpa), KM_SLEEP);
2181 	dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset;
2182 	SET_BOOKMARK(&dpa->dpa_zb, ds != NULL ? ds->ds_object : DMU_META_OBJSET,
2183 	    dn->dn_object, level, blkid);
2184 	dpa->dpa_curlevel = curlevel;
2185 	dpa->dpa_prio = prio;
2186 	dpa->dpa_aflags = aflags;
2187 	dpa->dpa_spa = dn->dn_objset->os_spa;
2188 	dpa->dpa_epbs = epbs;
2189 	dpa->dpa_zio = pio;
2190 
2191 	/*
2192 	 * If we have the indirect just above us, no need to do the asynchronous
2193 	 * prefetch chain; we'll just run the last step ourselves.  If we're at
2194 	 * a higher level, though, we want to issue the prefetches for all the
2195 	 * indirect blocks asynchronously, so we can go on with whatever we were
2196 	 * doing.
2197 	 */
2198 	if (curlevel == level) {
2199 		ASSERT3U(curblkid, ==, blkid);
2200 		dbuf_issue_final_prefetch(dpa, &bp);
2201 		kmem_free(dpa, sizeof (*dpa));
2202 	} else {
2203 		arc_flags_t iter_aflags = ARC_FLAG_NOWAIT;
2204 		zbookmark_phys_t zb;
2205 
2206 		SET_BOOKMARK(&zb, ds != NULL ? ds->ds_object : DMU_META_OBJSET,
2207 		    dn->dn_object, curlevel, curblkid);
2208 		(void) arc_read(dpa->dpa_zio, dpa->dpa_spa,
2209 		    &bp, dbuf_prefetch_indirect_done, dpa, prio,
2210 		    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE,
2211 		    &iter_aflags, &zb);
2212 	}
2213 	/*
2214 	 * We use pio here instead of dpa_zio since it's possible that
2215 	 * dpa may have already been freed.
2216 	 */
2217 	zio_nowait(pio);
2218 }
2219 
2220 /*
2221  * Returns with db_holds incremented, and db_mtx not held.
2222  * Note: dn_struct_rwlock must be held.
2223  */
2224 int
2225 dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid,
2226     boolean_t fail_sparse, boolean_t fail_uncached,
2227     void *tag, dmu_buf_impl_t **dbp)
2228 {
2229 	dmu_buf_impl_t *db, *parent = NULL;
2230 
2231 	ASSERT(blkid != DMU_BONUS_BLKID);
2232 	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
2233 	ASSERT3U(dn->dn_nlevels, >, level);
2234 
2235 	*dbp = NULL;
2236 top:
2237 	/* dbuf_find() returns with db_mtx held */
2238 	db = dbuf_find(dn->dn_objset, dn->dn_object, level, blkid);
2239 
2240 	if (db == NULL) {
2241 		blkptr_t *bp = NULL;
2242 		int err;
2243 
2244 		if (fail_uncached)
2245 			return (SET_ERROR(ENOENT));
2246 
2247 		ASSERT3P(parent, ==, NULL);
2248 		err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp);
2249 		if (fail_sparse) {
2250 			if (err == 0 && bp && BP_IS_HOLE(bp))
2251 				err = SET_ERROR(ENOENT);
2252 			if (err) {
2253 				if (parent)
2254 					dbuf_rele(parent, NULL);
2255 				return (err);
2256 			}
2257 		}
2258 		if (err && err != ENOENT)
2259 			return (err);
2260 		db = dbuf_create(dn, level, blkid, parent, bp);
2261 	}
2262 
2263 	if (fail_uncached && db->db_state != DB_CACHED) {
2264 		mutex_exit(&db->db_mtx);
2265 		return (SET_ERROR(ENOENT));
2266 	}
2267 
2268 	if (db->db_buf && refcount_is_zero(&db->db_holds)) {
2269 		arc_buf_add_ref(db->db_buf, db);
2270 		if (db->db_buf->b_data == NULL) {
2271 			dbuf_clear(db);
2272 			if (parent) {
2273 				dbuf_rele(parent, NULL);
2274 				parent = NULL;
2275 			}
2276 			goto top;
2277 		}
2278 		ASSERT3P(db->db.db_data, ==, db->db_buf->b_data);
2279 	}
2280 
2281 	ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf));
2282 
2283 	/*
2284 	 * If this buffer is currently syncing out, and we are are
2285 	 * still referencing it from db_data, we need to make a copy
2286 	 * of it in case we decide we want to dirty it again in this txg.
2287 	 */
2288 	if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID &&
2289 	    dn->dn_object != DMU_META_DNODE_OBJECT &&
2290 	    db->db_state == DB_CACHED && db->db_data_pending) {
2291 		dbuf_dirty_record_t *dr = db->db_data_pending;
2292 
2293 		if (dr->dt.dl.dr_data == db->db_buf) {
2294 			arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
2295 
2296 			dbuf_set_data(db,
2297 			    arc_buf_alloc(dn->dn_objset->os_spa,
2298 			    db->db.db_size, db, type));
2299 			bcopy(dr->dt.dl.dr_data->b_data, db->db.db_data,
2300 			    db->db.db_size);
2301 		}
2302 	}
2303 
2304 	(void) refcount_add(&db->db_holds, tag);
2305 	DBUF_VERIFY(db);
2306 	mutex_exit(&db->db_mtx);
2307 
2308 	/* NOTE: we can't rele the parent until after we drop the db_mtx */
2309 	if (parent)
2310 		dbuf_rele(parent, NULL);
2311 
2312 	ASSERT3P(DB_DNODE(db), ==, dn);
2313 	ASSERT3U(db->db_blkid, ==, blkid);
2314 	ASSERT3U(db->db_level, ==, level);
2315 	*dbp = db;
2316 
2317 	return (0);
2318 }
2319 
2320 dmu_buf_impl_t *
2321 dbuf_hold(dnode_t *dn, uint64_t blkid, void *tag)
2322 {
2323 	return (dbuf_hold_level(dn, 0, blkid, tag));
2324 }
2325 
2326 dmu_buf_impl_t *
2327 dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, void *tag)
2328 {
2329 	dmu_buf_impl_t *db;
2330 	int err = dbuf_hold_impl(dn, level, blkid, FALSE, FALSE, tag, &db);
2331 	return (err ? NULL : db);
2332 }
2333 
2334 void
2335 dbuf_create_bonus(dnode_t *dn)
2336 {
2337 	ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock));
2338 
2339 	ASSERT(dn->dn_bonus == NULL);
2340 	dn->dn_bonus = dbuf_create(dn, 0, DMU_BONUS_BLKID, dn->dn_dbuf, NULL);
2341 }
2342 
2343 int
2344 dbuf_spill_set_blksz(dmu_buf_t *db_fake, uint64_t blksz, dmu_tx_t *tx)
2345 {
2346 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2347 	dnode_t *dn;
2348 
2349 	if (db->db_blkid != DMU_SPILL_BLKID)
2350 		return (SET_ERROR(ENOTSUP));
2351 	if (blksz == 0)
2352 		blksz = SPA_MINBLOCKSIZE;
2353 	ASSERT3U(blksz, <=, spa_maxblocksize(dmu_objset_spa(db->db_objset)));
2354 	blksz = P2ROUNDUP(blksz, SPA_MINBLOCKSIZE);
2355 
2356 	DB_DNODE_ENTER(db);
2357 	dn = DB_DNODE(db);
2358 	rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
2359 	dbuf_new_size(db, blksz, tx);
2360 	rw_exit(&dn->dn_struct_rwlock);
2361 	DB_DNODE_EXIT(db);
2362 
2363 	return (0);
2364 }
2365 
2366 void
2367 dbuf_rm_spill(dnode_t *dn, dmu_tx_t *tx)
2368 {
2369 	dbuf_free_range(dn, DMU_SPILL_BLKID, DMU_SPILL_BLKID, tx);
2370 }
2371 
2372 #pragma weak dmu_buf_add_ref = dbuf_add_ref
2373 void
2374 dbuf_add_ref(dmu_buf_impl_t *db, void *tag)
2375 {
2376 	int64_t holds = refcount_add(&db->db_holds, tag);
2377 	ASSERT(holds > 1);
2378 }
2379 
2380 #pragma weak dmu_buf_try_add_ref = dbuf_try_add_ref
2381 boolean_t
2382 dbuf_try_add_ref(dmu_buf_t *db_fake, objset_t *os, uint64_t obj, uint64_t blkid,
2383     void *tag)
2384 {
2385 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2386 	dmu_buf_impl_t *found_db;
2387 	boolean_t result = B_FALSE;
2388 
2389 	if (blkid == DMU_BONUS_BLKID)
2390 		found_db = dbuf_find_bonus(os, obj);
2391 	else
2392 		found_db = dbuf_find(os, obj, 0, blkid);
2393 
2394 	if (found_db != NULL) {
2395 		if (db == found_db && dbuf_refcount(db) > db->db_dirtycnt) {
2396 			(void) refcount_add(&db->db_holds, tag);
2397 			result = B_TRUE;
2398 		}
2399 		mutex_exit(&found_db->db_mtx);
2400 	}
2401 	return (result);
2402 }
2403 
2404 /*
2405  * If you call dbuf_rele() you had better not be referencing the dnode handle
2406  * unless you have some other direct or indirect hold on the dnode. (An indirect
2407  * hold is a hold on one of the dnode's dbufs, including the bonus buffer.)
2408  * Without that, the dbuf_rele() could lead to a dnode_rele() followed by the
2409  * dnode's parent dbuf evicting its dnode handles.
2410  */
2411 void
2412 dbuf_rele(dmu_buf_impl_t *db, void *tag)
2413 {
2414 	mutex_enter(&db->db_mtx);
2415 	dbuf_rele_and_unlock(db, tag);
2416 }
2417 
2418 void
2419 dmu_buf_rele(dmu_buf_t *db, void *tag)
2420 {
2421 	dbuf_rele((dmu_buf_impl_t *)db, tag);
2422 }
2423 
2424 /*
2425  * dbuf_rele() for an already-locked dbuf.  This is necessary to allow
2426  * db_dirtycnt and db_holds to be updated atomically.
2427  */
2428 void
2429 dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag)
2430 {
2431 	int64_t holds;
2432 
2433 	ASSERT(MUTEX_HELD(&db->db_mtx));
2434 	DBUF_VERIFY(db);
2435 
2436 	/*
2437 	 * Remove the reference to the dbuf before removing its hold on the
2438 	 * dnode so we can guarantee in dnode_move() that a referenced bonus
2439 	 * buffer has a corresponding dnode hold.
2440 	 */
2441 	holds = refcount_remove(&db->db_holds, tag);
2442 	ASSERT(holds >= 0);
2443 
2444 	/*
2445 	 * We can't freeze indirects if there is a possibility that they
2446 	 * may be modified in the current syncing context.
2447 	 */
2448 	if (db->db_buf && holds == (db->db_level == 0 ? db->db_dirtycnt : 0))
2449 		arc_buf_freeze(db->db_buf);
2450 
2451 	if (holds == db->db_dirtycnt &&
2452 	    db->db_level == 0 && db->db_user_immediate_evict)
2453 		dbuf_evict_user(db);
2454 
2455 	if (holds == 0) {
2456 		if (db->db_blkid == DMU_BONUS_BLKID) {
2457 			dnode_t *dn;
2458 			boolean_t evict_dbuf = db->db_pending_evict;
2459 
2460 			/*
2461 			 * If the dnode moves here, we cannot cross this
2462 			 * barrier until the move completes.
2463 			 */
2464 			DB_DNODE_ENTER(db);
2465 
2466 			dn = DB_DNODE(db);
2467 			atomic_dec_32(&dn->dn_dbufs_count);
2468 
2469 			/*
2470 			 * Decrementing the dbuf count means that the bonus
2471 			 * buffer's dnode hold is no longer discounted in
2472 			 * dnode_move(). The dnode cannot move until after
2473 			 * the dnode_rele() below.
2474 			 */
2475 			DB_DNODE_EXIT(db);
2476 
2477 			/*
2478 			 * Do not reference db after its lock is dropped.
2479 			 * Another thread may evict it.
2480 			 */
2481 			mutex_exit(&db->db_mtx);
2482 
2483 			if (evict_dbuf)
2484 				dnode_evict_bonus(dn);
2485 
2486 			dnode_rele(dn, db);
2487 		} else if (db->db_buf == NULL) {
2488 			/*
2489 			 * This is a special case: we never associated this
2490 			 * dbuf with any data allocated from the ARC.
2491 			 */
2492 			ASSERT(db->db_state == DB_UNCACHED ||
2493 			    db->db_state == DB_NOFILL);
2494 			dbuf_evict(db);
2495 		} else if (arc_released(db->db_buf)) {
2496 			arc_buf_t *buf = db->db_buf;
2497 			/*
2498 			 * This dbuf has anonymous data associated with it.
2499 			 */
2500 			dbuf_clear_data(db);
2501 			VERIFY(arc_buf_remove_ref(buf, db));
2502 			dbuf_evict(db);
2503 		} else {
2504 			VERIFY(!arc_buf_remove_ref(db->db_buf, db));
2505 
2506 			/*
2507 			 * A dbuf will be eligible for eviction if either the
2508 			 * 'primarycache' property is set or a duplicate
2509 			 * copy of this buffer is already cached in the arc.
2510 			 *
2511 			 * In the case of the 'primarycache' a buffer
2512 			 * is considered for eviction if it matches the
2513 			 * criteria set in the property.
2514 			 *
2515 			 * To decide if our buffer is considered a
2516 			 * duplicate, we must call into the arc to determine
2517 			 * if multiple buffers are referencing the same
2518 			 * block on-disk. If so, then we simply evict
2519 			 * ourselves.
2520 			 */
2521 			if (!DBUF_IS_CACHEABLE(db)) {
2522 				if (db->db_blkptr != NULL &&
2523 				    !BP_IS_HOLE(db->db_blkptr) &&
2524 				    !BP_IS_EMBEDDED(db->db_blkptr)) {
2525 					spa_t *spa =
2526 					    dmu_objset_spa(db->db_objset);
2527 					blkptr_t bp = *db->db_blkptr;
2528 					dbuf_clear(db);
2529 					arc_freed(spa, &bp);
2530 				} else {
2531 					dbuf_clear(db);
2532 				}
2533 			} else if (db->db_pending_evict ||
2534 			    arc_buf_eviction_needed(db->db_buf)) {
2535 				dbuf_clear(db);
2536 			} else {
2537 				mutex_exit(&db->db_mtx);
2538 			}
2539 		}
2540 	} else {
2541 		mutex_exit(&db->db_mtx);
2542 	}
2543 }
2544 
2545 #pragma weak dmu_buf_refcount = dbuf_refcount
2546 uint64_t
2547 dbuf_refcount(dmu_buf_impl_t *db)
2548 {
2549 	return (refcount_count(&db->db_holds));
2550 }
2551 
2552 void *
2553 dmu_buf_replace_user(dmu_buf_t *db_fake, dmu_buf_user_t *old_user,
2554     dmu_buf_user_t *new_user)
2555 {
2556 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2557 
2558 	mutex_enter(&db->db_mtx);
2559 	dbuf_verify_user(db, DBVU_NOT_EVICTING);
2560 	if (db->db_user == old_user)
2561 		db->db_user = new_user;
2562 	else
2563 		old_user = db->db_user;
2564 	dbuf_verify_user(db, DBVU_NOT_EVICTING);
2565 	mutex_exit(&db->db_mtx);
2566 
2567 	return (old_user);
2568 }
2569 
2570 void *
2571 dmu_buf_set_user(dmu_buf_t *db_fake, dmu_buf_user_t *user)
2572 {
2573 	return (dmu_buf_replace_user(db_fake, NULL, user));
2574 }
2575 
2576 void *
2577 dmu_buf_set_user_ie(dmu_buf_t *db_fake, dmu_buf_user_t *user)
2578 {
2579 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2580 
2581 	db->db_user_immediate_evict = TRUE;
2582 	return (dmu_buf_set_user(db_fake, user));
2583 }
2584 
2585 void *
2586 dmu_buf_remove_user(dmu_buf_t *db_fake, dmu_buf_user_t *user)
2587 {
2588 	return (dmu_buf_replace_user(db_fake, user, NULL));
2589 }
2590 
2591 void *
2592 dmu_buf_get_user(dmu_buf_t *db_fake)
2593 {
2594 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2595 
2596 	dbuf_verify_user(db, DBVU_NOT_EVICTING);
2597 	return (db->db_user);
2598 }
2599 
2600 void
2601 dmu_buf_user_evict_wait()
2602 {
2603 	taskq_wait(dbu_evict_taskq);
2604 }
2605 
2606 boolean_t
2607 dmu_buf_freeable(dmu_buf_t *dbuf)
2608 {
2609 	boolean_t res = B_FALSE;
2610 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf;
2611 
2612 	if (db->db_blkptr)
2613 		res = dsl_dataset_block_freeable(db->db_objset->os_dsl_dataset,
2614 		    db->db_blkptr, db->db_blkptr->blk_birth);
2615 
2616 	return (res);
2617 }
2618 
2619 blkptr_t *
2620 dmu_buf_get_blkptr(dmu_buf_t *db)
2621 {
2622 	dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db;
2623 	return (dbi->db_blkptr);
2624 }
2625 
2626 static void
2627 dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db)
2628 {
2629 	/* ASSERT(dmu_tx_is_syncing(tx) */
2630 	ASSERT(MUTEX_HELD(&db->db_mtx));
2631 
2632 	if (db->db_blkptr != NULL)
2633 		return;
2634 
2635 	if (db->db_blkid == DMU_SPILL_BLKID) {
2636 		db->db_blkptr = &dn->dn_phys->dn_spill;
2637 		BP_ZERO(db->db_blkptr);
2638 		return;
2639 	}
2640 	if (db->db_level == dn->dn_phys->dn_nlevels-1) {
2641 		/*
2642 		 * This buffer was allocated at a time when there was
2643 		 * no available blkptrs from the dnode, or it was
2644 		 * inappropriate to hook it in (i.e., nlevels mis-match).
2645 		 */
2646 		ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr);
2647 		ASSERT(db->db_parent == NULL);
2648 		db->db_parent = dn->dn_dbuf;
2649 		db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid];
2650 		DBUF_VERIFY(db);
2651 	} else {
2652 		dmu_buf_impl_t *parent = db->db_parent;
2653 		int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;
2654 
2655 		ASSERT(dn->dn_phys->dn_nlevels > 1);
2656 		if (parent == NULL) {
2657 			mutex_exit(&db->db_mtx);
2658 			rw_enter(&dn->dn_struct_rwlock, RW_READER);
2659 			parent = dbuf_hold_level(dn, db->db_level + 1,
2660 			    db->db_blkid >> epbs, db);
2661 			rw_exit(&dn->dn_struct_rwlock);
2662 			mutex_enter(&db->db_mtx);
2663 			db->db_parent = parent;
2664 		}
2665 		db->db_blkptr = (blkptr_t *)parent->db.db_data +
2666 		    (db->db_blkid & ((1ULL << epbs) - 1));
2667 		DBUF_VERIFY(db);
2668 	}
2669 }
2670 
2671 static void
2672 dbuf_sync_indirect(dbuf_dirty_record_t *dr, dmu_tx_t *tx)
2673 {
2674 	dmu_buf_impl_t *db = dr->dr_dbuf;
2675 	dnode_t *dn;
2676 	zio_t *zio;
2677 
2678 	ASSERT(dmu_tx_is_syncing(tx));
2679 
2680 	dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr);
2681 
2682 	mutex_enter(&db->db_mtx);
2683 
2684 	ASSERT(db->db_level > 0);
2685 	DBUF_VERIFY(db);
2686 
2687 	/* Read the block if it hasn't been read yet. */
2688 	if (db->db_buf == NULL) {
2689 		mutex_exit(&db->db_mtx);
2690 		(void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED);
2691 		mutex_enter(&db->db_mtx);
2692 	}
2693 	ASSERT3U(db->db_state, ==, DB_CACHED);
2694 	ASSERT(db->db_buf != NULL);
2695 
2696 	DB_DNODE_ENTER(db);
2697 	dn = DB_DNODE(db);
2698 	/* Indirect block size must match what the dnode thinks it is. */
2699 	ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift);
2700 	dbuf_check_blkptr(dn, db);
2701 	DB_DNODE_EXIT(db);
2702 
2703 	/* Provide the pending dirty record to child dbufs */
2704 	db->db_data_pending = dr;
2705 
2706 	mutex_exit(&db->db_mtx);
2707 	dbuf_write(dr, db->db_buf, tx);
2708 
2709 	zio = dr->dr_zio;
2710 	mutex_enter(&dr->dt.di.dr_mtx);
2711 	dbuf_sync_list(&dr->dt.di.dr_children, db->db_level - 1, tx);
2712 	ASSERT(list_head(&dr->dt.di.dr_children) == NULL);
2713 	mutex_exit(&dr->dt.di.dr_mtx);
2714 	zio_nowait(zio);
2715 }
2716 
2717 static void
2718 dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx)
2719 {
2720 	arc_buf_t **datap = &dr->dt.dl.dr_data;
2721 	dmu_buf_impl_t *db = dr->dr_dbuf;
2722 	dnode_t *dn;
2723 	objset_t *os;
2724 	uint64_t txg = tx->tx_txg;
2725 
2726 	ASSERT(dmu_tx_is_syncing(tx));
2727 
2728 	dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr);
2729 
2730 	mutex_enter(&db->db_mtx);
2731 	/*
2732 	 * To be synced, we must be dirtied.  But we
2733 	 * might have been freed after the dirty.
2734 	 */
2735 	if (db->db_state == DB_UNCACHED) {
2736 		/* This buffer has been freed since it was dirtied */
2737 		ASSERT(db->db.db_data == NULL);
2738 	} else if (db->db_state == DB_FILL) {
2739 		/* This buffer was freed and is now being re-filled */
2740 		ASSERT(db->db.db_data != dr->dt.dl.dr_data);
2741 	} else {
2742 		ASSERT(db->db_state == DB_CACHED || db->db_state == DB_NOFILL);
2743 	}
2744 	DBUF_VERIFY(db);
2745 
2746 	DB_DNODE_ENTER(db);
2747 	dn = DB_DNODE(db);
2748 
2749 	if (db->db_blkid == DMU_SPILL_BLKID) {
2750 		mutex_enter(&dn->dn_mtx);
2751 		dn->dn_phys->dn_flags |= DNODE_FLAG_SPILL_BLKPTR;
2752 		mutex_exit(&dn->dn_mtx);
2753 	}
2754 
2755 	/*
2756 	 * If this is a bonus buffer, simply copy the bonus data into the
2757 	 * dnode.  It will be written out when the dnode is synced (and it
2758 	 * will be synced, since it must have been dirty for dbuf_sync to
2759 	 * be called).
2760 	 */
2761 	if (db->db_blkid == DMU_BONUS_BLKID) {
2762 		dbuf_dirty_record_t **drp;
2763 
2764 		ASSERT(*datap != NULL);
2765 		ASSERT0(db->db_level);
2766 		ASSERT3U(dn->dn_phys->dn_bonuslen, <=, DN_MAX_BONUSLEN);
2767 		bcopy(*datap, DN_BONUS(dn->dn_phys), dn->dn_phys->dn_bonuslen);
2768 		DB_DNODE_EXIT(db);
2769 
2770 		if (*datap != db->db.db_data) {
2771 			zio_buf_free(*datap, DN_MAX_BONUSLEN);
2772 			arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER);
2773 		}
2774 		db->db_data_pending = NULL;
2775 		drp = &db->db_last_dirty;
2776 		while (*drp != dr)
2777 			drp = &(*drp)->dr_next;
2778 		ASSERT(dr->dr_next == NULL);
2779 		ASSERT(dr->dr_dbuf == db);
2780 		*drp = dr->dr_next;
2781 		kmem_free(dr, sizeof (dbuf_dirty_record_t));
2782 		ASSERT(db->db_dirtycnt > 0);
2783 		db->db_dirtycnt -= 1;
2784 		dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg);
2785 		return;
2786 	}
2787 
2788 	os = dn->dn_objset;
2789 
2790 	/*
2791 	 * This function may have dropped the db_mtx lock allowing a dmu_sync
2792 	 * operation to sneak in. As a result, we need to ensure that we
2793 	 * don't check the dr_override_state until we have returned from
2794 	 * dbuf_check_blkptr.
2795 	 */
2796 	dbuf_check_blkptr(dn, db);
2797 
2798 	/*
2799 	 * If this buffer is in the middle of an immediate write,
2800 	 * wait for the synchronous IO to complete.
2801 	 */
2802 	while (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC) {
2803 		ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT);
2804 		cv_wait(&db->db_changed, &db->db_mtx);
2805 		ASSERT(dr->dt.dl.dr_override_state != DR_NOT_OVERRIDDEN);
2806 	}
2807 
2808 	if (db->db_state != DB_NOFILL &&
2809 	    dn->dn_object != DMU_META_DNODE_OBJECT &&
2810 	    refcount_count(&db->db_holds) > 1 &&
2811 	    dr->dt.dl.dr_override_state != DR_OVERRIDDEN &&
2812 	    *datap == db->db_buf) {
2813 		/*
2814 		 * If this buffer is currently "in use" (i.e., there
2815 		 * are active holds and db_data still references it),
2816 		 * then make a copy before we start the write so that
2817 		 * any modifications from the open txg will not leak
2818 		 * into this write.
2819 		 *
2820 		 * NOTE: this copy does not need to be made for
2821 		 * objects only modified in the syncing context (e.g.
2822 		 * DNONE_DNODE blocks).
2823 		 */
2824 		int blksz = arc_buf_size(*datap);
2825 		arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
2826 		*datap = arc_buf_alloc(os->os_spa, blksz, db, type);
2827 		bcopy(db->db.db_data, (*datap)->b_data, blksz);
2828 	}
2829 	db->db_data_pending = dr;
2830 
2831 	mutex_exit(&db->db_mtx);
2832 
2833 	dbuf_write(dr, *datap, tx);
2834 
2835 	ASSERT(!list_link_active(&dr->dr_dirty_node));
2836 	if (dn->dn_object == DMU_META_DNODE_OBJECT) {
2837 		list_insert_tail(&dn->dn_dirty_records[txg&TXG_MASK], dr);
2838 		DB_DNODE_EXIT(db);
2839 	} else {
2840 		/*
2841 		 * Although zio_nowait() does not "wait for an IO", it does
2842 		 * initiate the IO. If this is an empty write it seems plausible
2843 		 * that the IO could actually be completed before the nowait
2844 		 * returns. We need to DB_DNODE_EXIT() first in case
2845 		 * zio_nowait() invalidates the dbuf.
2846 		 */
2847 		DB_DNODE_EXIT(db);
2848 		zio_nowait(dr->dr_zio);
2849 	}
2850 }
2851 
2852 void
2853 dbuf_sync_list(list_t *list, int level, dmu_tx_t *tx)
2854 {
2855 	dbuf_dirty_record_t *dr;
2856 
2857 	while (dr = list_head(list)) {
2858 		if (dr->dr_zio != NULL) {
2859 			/*
2860 			 * If we find an already initialized zio then we
2861 			 * are processing the meta-dnode, and we have finished.
2862 			 * The dbufs for all dnodes are put back on the list
2863 			 * during processing, so that we can zio_wait()
2864 			 * these IOs after initiating all child IOs.
2865 			 */
2866 			ASSERT3U(dr->dr_dbuf->db.db_object, ==,
2867 			    DMU_META_DNODE_OBJECT);
2868 			break;
2869 		}
2870 		if (dr->dr_dbuf->db_blkid != DMU_BONUS_BLKID &&
2871 		    dr->dr_dbuf->db_blkid != DMU_SPILL_BLKID) {
2872 			VERIFY3U(dr->dr_dbuf->db_level, ==, level);
2873 		}
2874 		list_remove(list, dr);
2875 		if (dr->dr_dbuf->db_level > 0)
2876 			dbuf_sync_indirect(dr, tx);
2877 		else
2878 			dbuf_sync_leaf(dr, tx);
2879 	}
2880 }
2881 
2882 /* ARGSUSED */
2883 static void
2884 dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb)
2885 {
2886 	dmu_buf_impl_t *db = vdb;
2887 	dnode_t *dn;
2888 	blkptr_t *bp = zio->io_bp;
2889 	blkptr_t *bp_orig = &zio->io_bp_orig;
2890 	spa_t *spa = zio->io_spa;
2891 	int64_t delta;
2892 	uint64_t fill = 0;
2893 	int i;
2894 
2895 	ASSERT3P(db->db_blkptr, ==, bp);
2896 
2897 	DB_DNODE_ENTER(db);
2898 	dn = DB_DNODE(db);
2899 	delta = bp_get_dsize_sync(spa, bp) - bp_get_dsize_sync(spa, bp_orig);
2900 	dnode_diduse_space(dn, delta - zio->io_prev_space_delta);
2901 	zio->io_prev_space_delta = delta;
2902 
2903 	if (bp->blk_birth != 0) {
2904 		ASSERT((db->db_blkid != DMU_SPILL_BLKID &&
2905 		    BP_GET_TYPE(bp) == dn->dn_type) ||
2906 		    (db->db_blkid == DMU_SPILL_BLKID &&
2907 		    BP_GET_TYPE(bp) == dn->dn_bonustype) ||
2908 		    BP_IS_EMBEDDED(bp));
2909 		ASSERT(BP_GET_LEVEL(bp) == db->db_level);
2910 	}
2911 
2912 	mutex_enter(&db->db_mtx);
2913 
2914 #ifdef ZFS_DEBUG
2915 	if (db->db_blkid == DMU_SPILL_BLKID) {
2916 		ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR);
2917 		ASSERT(!(BP_IS_HOLE(db->db_blkptr)) &&
2918 		    db->db_blkptr == &dn->dn_phys->dn_spill);
2919 	}
2920 #endif
2921 
2922 	if (db->db_level == 0) {
2923 		mutex_enter(&dn->dn_mtx);
2924 		if (db->db_blkid > dn->dn_phys->dn_maxblkid &&
2925 		    db->db_blkid != DMU_SPILL_BLKID)
2926 			dn->dn_phys->dn_maxblkid = db->db_blkid;
2927 		mutex_exit(&dn->dn_mtx);
2928 
2929 		if (dn->dn_type == DMU_OT_DNODE) {
2930 			dnode_phys_t *dnp = db->db.db_data;
2931 			for (i = db->db.db_size >> DNODE_SHIFT; i > 0;
2932 			    i--, dnp++) {
2933 				if (dnp->dn_type != DMU_OT_NONE)
2934 					fill++;
2935 			}
2936 		} else {
2937 			if (BP_IS_HOLE(bp)) {
2938 				fill = 0;
2939 			} else {
2940 				fill = 1;
2941 			}
2942 		}
2943 	} else {
2944 		blkptr_t *ibp = db->db.db_data;
2945 		ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift);
2946 		for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, ibp++) {
2947 			if (BP_IS_HOLE(ibp))
2948 				continue;
2949 			fill += BP_GET_FILL(ibp);
2950 		}
2951 	}
2952 	DB_DNODE_EXIT(db);
2953 
2954 	if (!BP_IS_EMBEDDED(bp))
2955 		bp->blk_fill = fill;
2956 
2957 	mutex_exit(&db->db_mtx);
2958 }
2959 
2960 /*
2961  * The SPA will call this callback several times for each zio - once
2962  * for every physical child i/o (zio->io_phys_children times).  This
2963  * allows the DMU to monitor the progress of each logical i/o.  For example,
2964  * there may be 2 copies of an indirect block, or many fragments of a RAID-Z
2965  * block.  There may be a long delay before all copies/fragments are completed,
2966  * so this callback allows us to retire dirty space gradually, as the physical
2967  * i/os complete.
2968  */
2969 /* ARGSUSED */
2970 static void
2971 dbuf_write_physdone(zio_t *zio, arc_buf_t *buf, void *arg)
2972 {
2973 	dmu_buf_impl_t *db = arg;
2974 	objset_t *os = db->db_objset;
2975 	dsl_pool_t *dp = dmu_objset_pool(os);
2976 	dbuf_dirty_record_t *dr;
2977 	int delta = 0;
2978 
2979 	dr = db->db_data_pending;
2980 	ASSERT3U(dr->dr_txg, ==, zio->io_txg);
2981 
2982 	/*
2983 	 * The callback will be called io_phys_children times.  Retire one
2984 	 * portion of our dirty space each time we are called.  Any rounding
2985 	 * error will be cleaned up by dsl_pool_sync()'s call to
2986 	 * dsl_pool_undirty_space().
2987 	 */
2988 	delta = dr->dr_accounted / zio->io_phys_children;
2989 	dsl_pool_undirty_space(dp, delta, zio->io_txg);
2990 }
2991 
2992 /* ARGSUSED */
2993 static void
2994 dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb)
2995 {
2996 	dmu_buf_impl_t *db = vdb;
2997 	blkptr_t *bp_orig = &zio->io_bp_orig;
2998 	blkptr_t *bp = db->db_blkptr;
2999 	objset_t *os = db->db_objset;
3000 	dmu_tx_t *tx = os->os_synctx;
3001 	dbuf_dirty_record_t **drp, *dr;
3002 
3003 	ASSERT0(zio->io_error);
3004 	ASSERT(db->db_blkptr == bp);
3005 
3006 	/*
3007 	 * For nopwrites and rewrites we ensure that the bp matches our
3008 	 * original and bypass all the accounting.
3009 	 */
3010 	if (zio->io_flags & (ZIO_FLAG_IO_REWRITE | ZIO_FLAG_NOPWRITE)) {
3011 		ASSERT(BP_EQUAL(bp, bp_orig));
3012 	} else {
3013 		dsl_dataset_t *ds = os->os_dsl_dataset;
3014 		(void) dsl_dataset_block_kill(ds, bp_orig, tx, B_TRUE);
3015 		dsl_dataset_block_born(ds, bp, tx);
3016 	}
3017 
3018 	mutex_enter(&db->db_mtx);
3019 
3020 	DBUF_VERIFY(db);
3021 
3022 	drp = &db->db_last_dirty;
3023 	while ((dr = *drp) != db->db_data_pending)
3024 		drp = &dr->dr_next;
3025 	ASSERT(!list_link_active(&dr->dr_dirty_node));
3026 	ASSERT(dr->dr_dbuf == db);
3027 	ASSERT(dr->dr_next == NULL);
3028 	*drp = dr->dr_next;
3029 
3030 #ifdef ZFS_DEBUG
3031 	if (db->db_blkid == DMU_SPILL_BLKID) {
3032 		dnode_t *dn;
3033 
3034 		DB_DNODE_ENTER(db);
3035 		dn = DB_DNODE(db);
3036 		ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR);
3037 		ASSERT(!(BP_IS_HOLE(db->db_blkptr)) &&
3038 		    db->db_blkptr == &dn->dn_phys->dn_spill);
3039 		DB_DNODE_EXIT(db);
3040 	}
3041 #endif
3042 
3043 	if (db->db_level == 0) {
3044 		ASSERT(db->db_blkid != DMU_BONUS_BLKID);
3045 		ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN);
3046 		if (db->db_state != DB_NOFILL) {
3047 			if (dr->dt.dl.dr_data != db->db_buf)
3048 				VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data,
3049 				    db));
3050 			else if (!arc_released(db->db_buf))
3051 				arc_set_callback(db->db_buf, dbuf_do_evict, db);
3052 		}
3053 	} else {
3054 		dnode_t *dn;
3055 
3056 		DB_DNODE_ENTER(db);
3057 		dn = DB_DNODE(db);
3058 		ASSERT(list_head(&dr->dt.di.dr_children) == NULL);
3059 		ASSERT3U(db->db.db_size, ==, 1 << dn->dn_phys->dn_indblkshift);
3060 		if (!BP_IS_HOLE(db->db_blkptr)) {
3061 			int epbs =
3062 			    dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;
3063 			ASSERT3U(db->db_blkid, <=,
3064 			    dn->dn_phys->dn_maxblkid >> (db->db_level * epbs));
3065 			ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==,
3066 			    db->db.db_size);
3067 			if (!arc_released(db->db_buf))
3068 				arc_set_callback(db->db_buf, dbuf_do_evict, db);
3069 		}
3070 		DB_DNODE_EXIT(db);
3071 		mutex_destroy(&dr->dt.di.dr_mtx);
3072 		list_destroy(&dr->dt.di.dr_children);
3073 	}
3074 	kmem_free(dr, sizeof (dbuf_dirty_record_t));
3075 
3076 	cv_broadcast(&db->db_changed);
3077 	ASSERT(db->db_dirtycnt > 0);
3078 	db->db_dirtycnt -= 1;
3079 	db->db_data_pending = NULL;
3080 	dbuf_rele_and_unlock(db, (void *)(uintptr_t)tx->tx_txg);
3081 }
3082 
3083 static void
3084 dbuf_write_nofill_ready(zio_t *zio)
3085 {
3086 	dbuf_write_ready(zio, NULL, zio->io_private);
3087 }
3088 
3089 static void
3090 dbuf_write_nofill_done(zio_t *zio)
3091 {
3092 	dbuf_write_done(zio, NULL, zio->io_private);
3093 }
3094 
3095 static void
3096 dbuf_write_override_ready(zio_t *zio)
3097 {
3098 	dbuf_dirty_record_t *dr = zio->io_private;
3099 	dmu_buf_impl_t *db = dr->dr_dbuf;
3100 
3101 	dbuf_write_ready(zio, NULL, db);
3102 }
3103 
3104 static void
3105 dbuf_write_override_done(zio_t *zio)
3106 {
3107 	dbuf_dirty_record_t *dr = zio->io_private;
3108 	dmu_buf_impl_t *db = dr->dr_dbuf;
3109 	blkptr_t *obp = &dr->dt.dl.dr_overridden_by;
3110 
3111 	mutex_enter(&db->db_mtx);
3112 	if (!BP_EQUAL(zio->io_bp, obp)) {
3113 		if (!BP_IS_HOLE(obp))
3114 			dsl_free(spa_get_dsl(zio->io_spa), zio->io_txg, obp);
3115 		arc_release(dr->dt.dl.dr_data, db);
3116 	}
3117 	mutex_exit(&db->db_mtx);
3118 
3119 	dbuf_write_done(zio, NULL, db);
3120 }
3121 
3122 /* Issue I/O to commit a dirty buffer to disk. */
3123 static void
3124 dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx)
3125 {
3126 	dmu_buf_impl_t *db = dr->dr_dbuf;
3127 	dnode_t *dn;
3128 	objset_t *os;
3129 	dmu_buf_impl_t *parent = db->db_parent;
3130 	uint64_t txg = tx->tx_txg;
3131 	zbookmark_phys_t zb;
3132 	zio_prop_t zp;
3133 	zio_t *zio;
3134 	int wp_flag = 0;
3135 
3136 	DB_DNODE_ENTER(db);
3137 	dn = DB_DNODE(db);
3138 	os = dn->dn_objset;
3139 
3140 	if (db->db_state != DB_NOFILL) {
3141 		if (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE) {
3142 			/*
3143 			 * Private object buffers are released here rather
3144 			 * than in dbuf_dirty() since they are only modified
3145 			 * in the syncing context and we don't want the
3146 			 * overhead of making multiple copies of the data.
3147 			 */
3148 			if (BP_IS_HOLE(db->db_blkptr)) {
3149 				arc_buf_thaw(data);
3150 			} else {
3151 				dbuf_release_bp(db);
3152 			}
3153 		}
3154 	}
3155 
3156 	if (parent != dn->dn_dbuf) {
3157 		/* Our parent is an indirect block. */
3158 		/* We have a dirty parent that has been scheduled for write. */
3159 		ASSERT(parent && parent->db_data_pending);
3160 		/* Our parent's buffer is one level closer to the dnode. */
3161 		ASSERT(db->db_level == parent->db_level-1);
3162 		/*
3163 		 * We're about to modify our parent's db_data by modifying
3164 		 * our block pointer, so the parent must be released.
3165 		 */
3166 		ASSERT(arc_released(parent->db_buf));
3167 		zio = parent->db_data_pending->dr_zio;
3168 	} else {
3169 		/* Our parent is the dnode itself. */
3170 		ASSERT((db->db_level == dn->dn_phys->dn_nlevels-1 &&
3171 		    db->db_blkid != DMU_SPILL_BLKID) ||
3172 		    (db->db_blkid == DMU_SPILL_BLKID && db->db_level == 0));
3173 		if (db->db_blkid != DMU_SPILL_BLKID)
3174 			ASSERT3P(db->db_blkptr, ==,
3175 			    &dn->dn_phys->dn_blkptr[db->db_blkid]);
3176 		zio = dn->dn_zio;
3177 	}
3178 
3179 	ASSERT(db->db_level == 0 || data == db->db_buf);
3180 	ASSERT3U(db->db_blkptr->blk_birth, <=, txg);
3181 	ASSERT(zio);
3182 
3183 	SET_BOOKMARK(&zb, os->os_dsl_dataset ?
3184 	    os->os_dsl_dataset->ds_object : DMU_META_OBJSET,
3185 	    db->db.db_object, db->db_level, db->db_blkid);
3186 
3187 	if (db->db_blkid == DMU_SPILL_BLKID)
3188 		wp_flag = WP_SPILL;
3189 	wp_flag |= (db->db_state == DB_NOFILL) ? WP_NOFILL : 0;
3190 
3191 	dmu_write_policy(os, dn, db->db_level, wp_flag, &zp);
3192 	DB_DNODE_EXIT(db);
3193 
3194 	if (db->db_level == 0 &&
3195 	    dr->dt.dl.dr_override_state == DR_OVERRIDDEN) {
3196 		/*
3197 		 * The BP for this block has been provided by open context
3198 		 * (by dmu_sync() or dmu_buf_write_embedded()).
3199 		 */
3200 		void *contents = (data != NULL) ? data->b_data : NULL;
3201 
3202 		dr->dr_zio = zio_write(zio, os->os_spa, txg,
3203 		    db->db_blkptr, contents, db->db.db_size, &zp,
3204 		    dbuf_write_override_ready, NULL, dbuf_write_override_done,
3205 		    dr, ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb);
3206 		mutex_enter(&db->db_mtx);
3207 		dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN;
3208 		zio_write_override(dr->dr_zio, &dr->dt.dl.dr_overridden_by,
3209 		    dr->dt.dl.dr_copies, dr->dt.dl.dr_nopwrite);
3210 		mutex_exit(&db->db_mtx);
3211 	} else if (db->db_state == DB_NOFILL) {
3212 		ASSERT(zp.zp_checksum == ZIO_CHECKSUM_OFF ||
3213 		    zp.zp_checksum == ZIO_CHECKSUM_NOPARITY);
3214 		dr->dr_zio = zio_write(zio, os->os_spa, txg,
3215 		    db->db_blkptr, NULL, db->db.db_size, &zp,
3216 		    dbuf_write_nofill_ready, NULL, dbuf_write_nofill_done, db,
3217 		    ZIO_PRIORITY_ASYNC_WRITE,
3218 		    ZIO_FLAG_MUSTSUCCEED | ZIO_FLAG_NODATA, &zb);
3219 	} else {
3220 		ASSERT(arc_released(data));
3221 		dr->dr_zio = arc_write(zio, os->os_spa, txg,
3222 		    db->db_blkptr, data, DBUF_IS_L2CACHEABLE(db),
3223 		    DBUF_IS_L2COMPRESSIBLE(db), &zp, dbuf_write_ready,
3224 		    dbuf_write_physdone, dbuf_write_done, db,
3225 		    ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb);
3226 	}
3227 }
3228