Lines Matching defs:db

55 static void dbuf_destroy(dmu_buf_impl_t *db);
56 static boolean_t dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx);
76 dmu_buf_impl_t *db = vdb;
77 bzero(db, sizeof (dmu_buf_impl_t));
79 mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL);
80 cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL);
81 refcount_create(&db->db_holds);
90 dmu_buf_impl_t *db = vdb;
91 mutex_destroy(&db->db_mtx);
92 cv_destroy(&db->db_changed);
93 refcount_destroy(&db->db_holds);
125 ((dbuf)->db.db_object == (obj) && \
136 dmu_buf_impl_t *db;
139 for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) {
140 if (DBUF_EQUAL(db, os, obj, level, blkid)) {
141 mutex_enter(&db->db_mtx);
142 if (db->db_state != DB_EVICTING) {
144 return (db);
146 mutex_exit(&db->db_mtx);
157 dmu_buf_impl_t *db = NULL;
162 db = dn->dn_bonus;
163 mutex_enter(&db->db_mtx);
168 return (db);
178 dbuf_hash_insert(dmu_buf_impl_t *db)
181 objset_t *os = db->db_objset;
182 uint64_t obj = db->db.db_object;
183 int level = db->db_level;
184 uint64_t blkid = db->db_blkid;
201 mutex_enter(&db->db_mtx);
202 db->db_hash_next = h->hash_table[idx];
203 h->hash_table[idx] = db;
214 dbuf_hash_remove(dmu_buf_impl_t *db)
217 uint64_t hv = DBUF_HASH(db->db_objset, db->db.db_object,
218 db->db_level, db->db_blkid);
226 ASSERT(refcount_is_zero(&db->db_holds));
227 ASSERT(db->db_state == DB_EVICTING);
228 ASSERT(!MUTEX_HELD(&db->db_mtx));
232 while ((dbf = *dbp) != db) {
236 *dbp = db->db_hash_next;
237 db->db_hash_next = NULL;
250 dbuf_verify_user(dmu_buf_impl_t *db, dbvu_verify_type_t verify_type)
255 if (db->db_user == NULL)
259 ASSERT(db->db_level == 0);
262 ASSERT(db->db.db_data != NULL);
263 ASSERT3U(db->db_state, ==, DB_CACHED);
265 holds = refcount_count(&db->db_holds);
276 ASSERT3U(holds, >=, db->db_dirtycnt);
278 if (db->db_user_immediate_evict == TRUE)
279 ASSERT3U(holds, >=, db->db_dirtycnt);
287 dbuf_evict_user(dmu_buf_impl_t *db)
289 dmu_buf_user_t *dbu = db->db_user;
291 ASSERT(MUTEX_HELD(&db->db_mtx));
296 dbuf_verify_user(db, DBVU_EVICTING);
297 db->db_user = NULL;
326 dbuf_is_metadata(dmu_buf_impl_t *db)
328 if (db->db_level > 0) {
333 DB_DNODE_ENTER(db);
334 is_metadata = DMU_OT_IS_METADATA(DB_DNODE(db)->dn_type);
335 DB_DNODE_EXIT(db);
342 dbuf_evict(dmu_buf_impl_t *db)
344 ASSERT(MUTEX_HELD(&db->db_mtx));
345 ASSERT(db->db_buf == NULL);
346 ASSERT(db->db_data_pending == NULL);
348 dbuf_clear(db);
349 dbuf_destroy(db);
410 dbuf_verify(dmu_buf_impl_t *db)
415 ASSERT(MUTEX_HELD(&db->db_mtx));
420 ASSERT(db->db_objset != NULL);
421 DB_DNODE_ENTER(db);
422 dn = DB_DNODE(db);
424 ASSERT(db->db_parent == NULL);
425 ASSERT(db->db_blkptr == NULL);
427 ASSERT3U(db->db.db_object, ==, dn->dn_object);
428 ASSERT3P(db->db_objset, ==, dn->dn_objset);
429 ASSERT3U(db->db_level, <, dn->dn_nlevels);
430 ASSERT(db->db_blkid == DMU_BONUS_BLKID ||
431 db->db_blkid == DMU_SPILL_BLKID ||
434 if (db->db_blkid == DMU_BONUS_BLKID) {
436 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen);
437 ASSERT3U(db->db.db_offset, ==, DMU_BONUS_BLKID);
438 } else if (db->db_blkid == DMU_SPILL_BLKID) {
440 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen);
441 ASSERT0(db->db.db_offset);
443 ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size);
446 for (dr = db->db_data_pending; dr != NULL; dr = dr->dr_next)
447 ASSERT(dr->dr_dbuf == db);
449 for (dr = db->db_last_dirty; dr != NULL; dr = dr->dr_next)
450 ASSERT(dr->dr_dbuf == db);
457 if (db->db_level == 0 && db->db.db_object == DMU_META_DNODE_OBJECT) {
458 dr = db->db_data_pending;
463 ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf);
466 /* verify db->db_blkptr */
467 if (db->db_blkptr) {
468 if (db->db_parent == dn->dn_dbuf) {
469 /* db is pointed to by the dnode */
470 /* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */
471 if (DMU_OBJECT_IS_SPECIAL(db->db.db_object))
472 ASSERT(db->db_parent == NULL);
474 ASSERT(db->db_parent != NULL);
475 if (db->db_blkid != DMU_SPILL_BLKID)
476 ASSERT3P(db->db_blkptr, ==,
477 &dn->dn_phys->dn_blkptr[db->db_blkid]);
479 /* db is pointed to by an indirect block */
480 int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT;
481 ASSERT3U(db->db_parent->db_level, ==, db->db_level+1);
482 ASSERT3U(db->db_parent->db.db_object, ==,
483 db->db.db_object);
490 ASSERT3P(db->db_blkptr, ==,
491 ((blkptr_t *)db->db_parent->db.db_data +
492 db->db_blkid % epb));
496 if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) &&
497 (db->db_buf == NULL || db->db_buf->b_data) &&
498 db->db.db_data && db->db_blkid != DMU_BONUS_BLKID &&
499 db->db_state != DB_FILL && !dn->dn_free_txg) {
505 if (db->db_dirtycnt == 0) {
506 uint64_t *buf = db->db.db_data;
509 for (i = 0; i < db->db.db_size >> 3; i++) {
514 DB_DNODE_EXIT(db);
519 dbuf_clear_data(dmu_buf_impl_t *db)
521 ASSERT(MUTEX_HELD(&db->db_mtx));
522 dbuf_evict_user(db);
523 db->db_buf = NULL;
524 db->db.db_data = NULL;
525 if (db->db_state != DB_NOFILL)
526 db->db_state = DB_UNCACHED;
530 dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf)
532 ASSERT(MUTEX_HELD(&db->db_mtx));
535 db->db_buf = buf;
537 db->db.db_data = buf->b_data;
539 arc_set_callback(buf, dbuf_do_evict, db);
546 dbuf_loan_arcbuf(dmu_buf_impl_t *db)
550 mutex_enter(&db->db_mtx);
551 if (arc_released(db->db_buf) || refcount_count(&db->db_holds) > 1) {
552 int blksz = db->db.db_size;
553 spa_t *spa = db->db_objset->os_spa;
555 mutex_exit(&db->db_mtx);
557 bcopy(db->db.db_data, abuf->b_data, blksz);
559 abuf = db->db_buf;
560 arc_loan_inuse_buf(abuf, db);
561 dbuf_clear_data(db);
562 mutex_exit(&db->db_mtx);
605 dmu_buf_impl_t *db = vdb;
607 mutex_enter(&db->db_mtx);
608 ASSERT3U(db->db_state, ==, DB_READ);
612 ASSERT(refcount_count(&db->db_holds) > 0);
613 ASSERT(db->db_buf == NULL);
614 ASSERT(db->db.db_data == NULL);
615 if (db->db_level == 0 && db->db_freed_in_flight) {
617 arc_release(buf, db);
618 bzero(buf->b_data, db->db.db_size);
620 db->db_freed_in_flight = FALSE;
621 dbuf_set_data(db, buf);
622 db->db_state = DB_CACHED;
624 dbuf_set_data(db, buf);
625 db->db_state = DB_CACHED;
627 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
628 ASSERT3P(db->db_buf, ==, NULL);
629 VERIFY(arc_buf_remove_ref(buf, db));
630 db->db_state = DB_UNCACHED;
632 cv_broadcast(&db->db_changed);
633 dbuf_rele_and_unlock(db, NULL);
637 dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags)
643 DB_DNODE_ENTER(db);
644 dn = DB_DNODE(db);
645 ASSERT(!refcount_is_zero(&db->db_holds));
648 ASSERT(MUTEX_HELD(&db->db_mtx));
649 ASSERT(db->db_state == DB_UNCACHED);
650 ASSERT(db->db_buf == NULL);
652 if (db->db_blkid == DMU_BONUS_BLKID) {
655 ASSERT3U(bonuslen, <=, db->db.db_size);
656 db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN);
659 bzero(db->db.db_data, DN_MAX_BONUSLEN);
661 bcopy(DN_BONUS(dn->dn_phys), db->db.db_data, bonuslen);
662 DB_DNODE_EXIT(db);
663 db->db_state = DB_CACHED;
664 mutex_exit(&db->db_mtx);
673 if (db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr) ||
674 (db->db_level == 0 && (dnode_block_freed(dn, db->db_blkid) ||
675 BP_IS_HOLE(db->db_blkptr)))) {
676 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
678 DB_DNODE_EXIT(db);
679 dbuf_set_data(db, arc_buf_alloc(db->db_objset->os_spa,
680 db->db.db_size, db, type));
681 bzero(db->db.db_data, db->db.db_size);
682 db->db_state = DB_CACHED;
683 mutex_exit(&db->db_mtx);
687 DB_DNODE_EXIT(db);
689 db->db_state = DB_READ;
690 mutex_exit(&db->db_mtx);
692 if (DBUF_IS_L2CACHEABLE(db))
694 if (DBUF_IS_L2COMPRESSIBLE(db))
697 SET_BOOKMARK(&zb, db->db_objset->os_dsl_dataset ?
698 db->db_objset->os_dsl_dataset->ds_object : DMU_META_OBJSET,
699 db->db.db_object, db->db_level, db->db_blkid);
701 dbuf_add_ref(db, NULL);
703 (void) arc_read(zio, db->db_objset->os_spa, db->db_blkptr,
704 dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ,
710 dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags)
721 ASSERT(!refcount_is_zero(&db->db_holds));
723 if (db->db_state == DB_NOFILL)
726 DB_DNODE_ENTER(db);
727 dn = DB_DNODE(db);
731 prefetch = db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID &&
733 DBUF_IS_CACHEABLE(db);
735 mutex_enter(&db->db_mtx);
736 if (db->db_state == DB_CACHED) {
737 mutex_exit(&db->db_mtx);
739 dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1);
742 DB_DNODE_EXIT(db);
743 } else if (db->db_state == DB_UNCACHED) {
748 dbuf_read_impl(db, zio, flags);
753 dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1);
757 DB_DNODE_EXIT(db);
770 mutex_exit(&db->db_mtx);
772 dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1);
775 DB_DNODE_EXIT(db);
778 mutex_enter(&db->db_mtx);
780 while (db->db_state == DB_READ ||
781 db->db_state == DB_FILL) {
782 ASSERT(db->db_state == DB_READ ||
785 db, zio_t *, zio);
786 cv_wait(&db->db_changed, &db->db_mtx);
788 if (db->db_state == DB_UNCACHED)
791 mutex_exit(&db->db_mtx);
794 ASSERT(err || havepzio || db->db_state == DB_CACHED);
799 dbuf_noread(dmu_buf_impl_t *db)
801 ASSERT(!refcount_is_zero(&db->db_holds));
802 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
803 mutex_enter(&db->db_mtx);
804 while (db->db_state == DB_READ || db->db_state == DB_FILL)
805 cv_wait(&db->db_changed, &db->db_mtx);
806 if (db->db_state == DB_UNCACHED) {
807 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
808 spa_t *spa = db->db_objset->os_spa;
810 ASSERT(db->db_buf == NULL);
811 ASSERT(db->db.db_data == NULL);
812 dbuf_set_data(db, arc_buf_alloc(spa, db->db.db_size, db, type));
813 db->db_state = DB_FILL;
814 } else if (db->db_state == DB_NOFILL) {
815 dbuf_clear_data(db);
817 ASSERT3U(db->db_state, ==, DB_CACHED);
819 mutex_exit(&db->db_mtx);
836 dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg)
838 dbuf_dirty_record_t *dr = db->db_last_dirty;
840 ASSERT(MUTEX_HELD(&db->db_mtx));
841 ASSERT(db->db.db_data != NULL);
842 ASSERT(db->db_level == 0);
843 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT);
847 ((db->db_blkid == DMU_BONUS_BLKID) ? db->db.db_data : db->db_buf)))
858 if (db->db_blkid == DMU_BONUS_BLKID) {
862 bcopy(db->db.db_data, dr->dt.dl.dr_data, DN_MAX_BONUSLEN);
863 } else if (refcount_count(&db->db_holds) > db->db_dirtycnt) {
864 int size = db->db.db_size;
865 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
866 spa_t *spa = db->db_objset->os_spa;
868 dr->dt.dl.dr_data = arc_buf_alloc(spa, size, db, type);
869 bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size);
871 dbuf_clear_data(db);
878 dmu_buf_impl_t *db = dr->dr_dbuf;
882 ASSERT(MUTEX_HELD(&db->db_mtx));
884 ASSERT(db->db_level == 0);
886 if (db->db_blkid == DMU_BONUS_BLKID ||
890 ASSERT(db->db_data_pending != dr);
894 zio_free(db->db_objset->os_spa, txg, bp);
907 arc_release(dr->dt.dl.dr_data, db);
923 dmu_buf_impl_t *db, *db_next;
939 db = avl_find(&dn->dn_dbufs, &db_search, &where);
940 ASSERT3P(db, ==, NULL);
941 db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER);
942 ASSERT(db == NULL || db->db_level > 0);
957 db = avl_find(&dn->dn_dbufs, &db_search, &where);
958 ASSERT3P(db, ==, NULL);
959 db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER);
961 for (; db != NULL; db = db_next) {
962 db_next = AVL_NEXT(&dn->dn_dbufs, db);
963 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
965 if (db->db_level != 0 || db->db_blkid > end_blkid) {
968 ASSERT3U(db->db_blkid, >=, start_blkid);
971 mutex_enter(&db->db_mtx);
972 if (dbuf_undirty(db, tx)) {
977 if (db->db_state == DB_UNCACHED ||
978 db->db_state == DB_NOFILL ||
979 db->db_state == DB_EVICTING) {
980 ASSERT(db->db.db_data == NULL);
981 mutex_exit(&db->db_mtx);
984 if (db->db_state == DB_READ || db->db_state == DB_FILL) {
986 db->db_freed_in_flight = TRUE;
987 mutex_exit(&db->db_mtx);
990 if (refcount_count(&db->db_holds) == 0) {
991 ASSERT(db->db_buf);
992 dbuf_clear(db);
997 if (db->db_last_dirty != NULL) {
998 dbuf_dirty_record_t *dr = db->db_last_dirty;
1006 if (db->db_blkid != DMU_SPILL_BLKID &&
1007 db->db_blkid > dn->dn_maxblkid)
1008 dn->dn_maxblkid = db->db_blkid;
1017 dbuf_fix_old_data(db, txg);
1021 if (db->db_state == DB_CACHED) {
1022 ASSERT(db->db.db_data != NULL);
1023 arc_release(db->db_buf, db);
1024 bzero(db->db.db_data, db->db.db_size);
1025 arc_buf_freeze(db->db_buf);
1028 mutex_exit(&db->db_mtx);
1034 dbuf_block_freeable(dmu_buf_impl_t *db)
1036 dsl_dataset_t *ds = db->db_objset->os_dsl_dataset;
1047 ASSERT(MUTEX_HELD(&db->db_mtx));
1048 if (db->db_last_dirty && (db->db_blkptr == NULL ||
1049 !BP_IS_HOLE(db->db_blkptr))) {
1050 birth_txg = db->db_last_dirty->dr_txg;
1051 } else if (db->db_blkptr != NULL && !BP_IS_HOLE(db->db_blkptr)) {
1052 birth_txg = db->db_blkptr->blk_birth;
1069 dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx)
1072 int osize = db->db.db_size;
1073 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
1076 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1078 DB_DNODE_ENTER(db);
1079 dn = DB_DNODE(db);
1086 * is OK, because there can be no other references to the db
1094 dmu_buf_will_dirty(&db->db, tx);
1097 buf = arc_buf_alloc(dn->dn_objset->os_spa, size, db, type);
1100 obuf = db->db_buf;
1106 mutex_enter(&db->db_mtx);
1107 dbuf_set_data(db, buf);
1108 VERIFY(arc_buf_remove_ref(obuf, db));
1109 db->db.db_size = size;
1111 if (db->db_level == 0) {
1112 ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg);
1113 db->db_last_dirty->dt.dl.dr_data = buf;
1115 mutex_exit(&db->db_mtx);
1118 DB_DNODE_EXIT(db);
1122 dbuf_release_bp(dmu_buf_impl_t *db)
1124 objset_t *os = db->db_objset;
1129 ASSERT(db->db_parent == NULL || arc_released(db->db_parent->db_buf));
1131 (void) arc_release(db->db_buf, db);
1141 dmu_buf_impl_t *db = dr->dr_dbuf;
1143 ASSERT(MUTEX_HELD(&db->db_mtx));
1145 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID) {
1151 if (db->db.db_object != DMU_META_DNODE_OBJECT &&
1152 db->db_state != DB_NOFILL) {
1154 ASSERT(arc_released(db->db_buf));
1155 arc_buf_thaw(db->db_buf);
1161 dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
1171 ASSERT(!refcount_is_zero(&db->db_holds));
1172 DMU_TX_DIRTY_BUF(tx, db);
1174 DB_DNODE_ENTER(db);
1175 dn = DB_DNODE(db);
1194 mutex_enter(&db->db_mtx);
1200 ASSERT(db->db_level != 0 ||
1201 db->db_state == DB_CACHED || db->db_state == DB_FILL ||
1202 db->db_state == DB_NOFILL);
1218 if (db->db_blkid == DMU_SPILL_BLKID)
1224 drp = &db->db_last_dirty;
1226 db->db.db_object == DMU_META_DNODE_OBJECT);
1230 DB_DNODE_EXIT(db);
1233 mutex_exit(&db->db_mtx);
1244 ASSERT3U(dn->dn_nlevels, >, db->db_level);
1245 ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) ||
1246 dn->dn_phys->dn_nlevels > db->db_level ||
1247 dn->dn_next_nlevels[txgoff] > db->db_level ||
1248 dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level ||
1249 dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level);
1261 ASSERT(db->db.db_size != 0);
1263 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size);
1265 if (db->db_blkid != DMU_BONUS_BLKID) {
1273 dnode_willuse_space(dn, db->db.db_size, tx);
1274 do_free_accounting = dbuf_block_freeable(db);
1283 if (db->db_level == 0) {
1284 void *data_old = db->db_buf;
1286 if (db->db_state != DB_NOFILL) {
1287 if (db->db_blkid == DMU_BONUS_BLKID) {
1288 dbuf_fix_old_data(db, tx->tx_txg);
1289 data_old = db->db.db_data;
1290 } else if (db->db.db_object != DMU_META_DNODE_OBJECT) {
1300 arc_release(db->db_buf, db);
1301 dbuf_fix_old_data(db, tx->tx_txg);
1302 data_old = db->db_buf;
1313 if (db->db_blkid != DMU_BONUS_BLKID && os->os_dsl_dataset != NULL)
1314 dr->dr_accounted = db->db.db_size;
1315 dr->dr_dbuf = db;
1325 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID &&
1326 db->db_blkid != DMU_SPILL_BLKID) {
1330 db->db_blkid, 1);
1333 db->db_freed_in_flight = FALSE;
1339 dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg);
1340 db->db_dirtycnt += 1;
1341 ASSERT3U(db->db_dirtycnt, <=, 3);
1343 mutex_exit(&db->db_mtx);
1345 if (db->db_blkid == DMU_BONUS_BLKID ||
1346 db->db_blkid == DMU_SPILL_BLKID) {
1352 DB_DNODE_EXIT(db);
1355 blkptr_t *bp = db->db_blkptr;
1357 bp_get_dsize(os->os_spa, bp) : db->db.db_size;
1375 if (db->db_level == 0) {
1376 dnode_new_blkid(dn, db->db_blkid, tx, drop_struct_lock);
1377 ASSERT(dn->dn_maxblkid >= db->db_blkid);
1380 if (db->db_level+1 < dn->dn_nlevels) {
1381 dmu_buf_impl_t *parent = db->db_parent;
1385 if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) {
1388 parent = dbuf_hold_level(dn, db->db_level+1,
1389 db->db_blkid >> epbs, FTAG);
1395 ASSERT3U(db->db_level+1, ==, parent->db_level);
1400 mutex_enter(&db->db_mtx);
1405 if (db->db_last_dirty == dr ||
1414 mutex_exit(&db->db_mtx);
1416 ASSERT(db->db_level+1 == dn->dn_nlevels);
1417 ASSERT(db->db_blkid < dn->dn_nblkptr);
1418 ASSERT(db->db_parent == NULL || db->db_parent == dn->dn_dbuf);
1428 DB_DNODE_EXIT(db);
1437 dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
1451 ASSERT(db->db_objset ==
1452 dmu_objset_pool(db->db_objset)->dp_meta_objset ||
1453 txg != spa_syncing_txg(dmu_objset_spa(db->db_objset)));
1454 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1455 ASSERT0(db->db_level);
1456 ASSERT(MUTEX_HELD(&db->db_mtx));
1461 for (drp = &db->db_last_dirty; (dr = *drp) != NULL; drp = &dr->dr_next)
1467 ASSERT(dr->dr_dbuf == db);
1469 DB_DNODE_ENTER(db);
1470 dn = DB_DNODE(db);
1472 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size);
1474 ASSERT(db->db.db_size != 0);
1491 } else if (db->db_blkid == DMU_SPILL_BLKID ||
1492 db->db_level + 1 == dn->dn_nlevels) {
1493 ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf);
1498 DB_DNODE_EXIT(db);
1500 if (db->db_state != DB_NOFILL) {
1503 ASSERT(db->db_buf != NULL);
1505 if (dr->dt.dl.dr_data != db->db_buf)
1506 VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, db));
1511 ASSERT(db->db_dirtycnt > 0);
1512 db->db_dirtycnt -= 1;
1514 if (refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) {
1515 arc_buf_t *buf = db->db_buf;
1517 ASSERT(db->db_state == DB_NOFILL || arc_released(buf));
1518 dbuf_clear_data(db);
1519 VERIFY(arc_buf_remove_ref(buf, db));
1520 dbuf_evict(db);
1530 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1534 ASSERT(!refcount_is_zero(&db->db_holds));
1542 mutex_enter(&db->db_mtx);
1544 for (dr = db->db_last_dirty;
1551 if (dr->dr_txg == tx->tx_txg && db->db_state == DB_CACHED) {
1554 mutex_exit(&db->db_mtx);
1558 mutex_exit(&db->db_mtx);
1560 DB_DNODE_ENTER(db);
1561 if (RW_WRITE_HELD(&DB_DNODE(db)->dn_struct_rwlock))
1563 DB_DNODE_EXIT(db);
1564 (void) dbuf_read(db, NULL, rf);
1565 (void) dbuf_dirty(db, tx);
1571 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1573 db->db_state = DB_NOFILL;
1581 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1583 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1585 ASSERT(db->db_level == 0);
1586 ASSERT(!refcount_is_zero(&db->db_holds));
1588 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT ||
1591 dbuf_noread(db);
1592 (void) dbuf_dirty(db, tx);
1598 dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx)
1600 mutex_enter(&db->db_mtx);
1601 DBUF_VERIFY(db);
1603 if (db->db_state == DB_FILL) {
1604 if (db->db_level == 0 && db->db_freed_in_flight) {
1605 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1608 bzero(db->db.db_data, db->db.db_size);
1609 db->db_freed_in_flight = FALSE;
1611 db->db_state = DB_CACHED;
1612 cv_broadcast(&db->db_changed);
1614 mutex_exit(&db->db_mtx);
1623 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf;
1628 ASSERT(spa_feature_is_active(dmu_objset_spa(db->db_objset),
1632 DB_DNODE_ENTER(db);
1633 type = DB_DNODE(db)->dn_type;
1634 DB_DNODE_EXIT(db);
1636 ASSERT0(db->db_level);
1637 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1641 ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg);
1642 dl = &db->db_last_dirty->dt.dl;
1651 dl->dr_overridden_by.blk_birth = db->db_last_dirty->dr_txg;
1659 dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx)
1661 ASSERT(!refcount_is_zero(&db->db_holds));
1662 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1663 ASSERT(db->db_level == 0);
1664 ASSERT(DBUF_GET_BUFC_TYPE(db) == ARC_BUFC_DATA);
1666 ASSERT(arc_buf_size(buf) == db->db.db_size);
1669 arc_return_buf(buf, db);
1672 mutex_enter(&db->db_mtx);
1674 while (db->db_state == DB_READ || db->db_state == DB_FILL)
1675 cv_wait(&db->db_changed, &db->db_mtx);
1677 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_UNCACHED);
1679 if (db->db_state == DB_CACHED &&
1680 refcount_count(&db->db_holds) - 1 > db->db_dirtycnt) {
1681 mutex_exit(&db->db_mtx);
1682 (void) dbuf_dirty(db, tx);
1683 bcopy(buf->b_data, db->db.db_data, db->db.db_size);
1684 VERIFY(arc_buf_remove_ref(buf, db));
1690 if (db->db_state == DB_CACHED) {
1691 dbuf_dirty_record_t *dr = db->db_last_dirty;
1693 ASSERT(db->db_buf != NULL);
1695 ASSERT(dr->dt.dl.dr_data == db->db_buf);
1696 if (!arc_released(db->db_buf)) {
1699 arc_release(db->db_buf, db);
1702 VERIFY(arc_buf_remove_ref(db->db_buf, db));
1703 } else if (dr == NULL || dr->dt.dl.dr_data != db->db_buf) {
1704 arc_release(db->db_buf, db);
1705 VERIFY(arc_buf_remove_ref(db->db_buf, db));
1707 db->db_buf = NULL;
1709 ASSERT(db->db_buf == NULL);
1710 dbuf_set_data(db, buf);
1711 db->db_state = DB_FILL;
1712 mutex_exit(&db->db_mtx);
1713 (void) dbuf_dirty(db, tx);
1714 dmu_buf_fill_done(&db->db, tx);
1734 dbuf_clear(dmu_buf_impl_t *db)
1737 dmu_buf_impl_t *parent = db->db_parent;
1741 ASSERT(MUTEX_HELD(&db->db_mtx));
1742 ASSERT(refcount_is_zero(&db->db_holds));
1744 dbuf_evict_user(db);
1746 if (db->db_state == DB_CACHED) {
1747 ASSERT(db->db.db_data != NULL);
1748 if (db->db_blkid == DMU_BONUS_BLKID) {
1749 zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN);
1752 db->db.db_data = NULL;
1753 db->db_state = DB_UNCACHED;
1756 ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL);
1757 ASSERT(db->db_data_pending == NULL);
1759 db->db_state = DB_EVICTING;
1760 db->db_blkptr = NULL;
1762 DB_DNODE_ENTER(db);
1763 dn = DB_DNODE(db);
1765 if (db->db_blkid != DMU_BONUS_BLKID && MUTEX_HELD(&dn->dn_dbufs_mtx)) {
1766 avl_remove(&dn->dn_dbufs, db);
1769 DB_DNODE_EXIT(db);
1778 dnode_rele(dn, db);
1779 db->db_dnode_handle = NULL;
1781 DB_DNODE_EXIT(db);
1784 if (db->db_buf)
1785 dbuf_gone = arc_clear_callback(db->db_buf);
1788 mutex_exit(&db->db_mtx);
1795 dbuf_rele(parent, db);
1854 *bpp = ((blkptr_t *)(*parentp)->db.db_data) +
1876 dmu_buf_impl_t *db, *odb;
1881 db = kmem_cache_alloc(dbuf_cache, KM_SLEEP);
1883 db->db_objset = os;
1884 db->db.db_object = dn->dn_object;
1885 db->db_level = level;
1886 db->db_blkid = blkid;
1887 db->db_last_dirty = NULL;
1888 db->db_dirtycnt = 0;
1889 db->db_dnode_handle = dn->dn_handle;
1890 db->db_parent = parent;
1891 db->db_blkptr = blkptr;
1893 db->db_user = NULL;
1894 db->db_user_immediate_evict = FALSE;
1895 db->db_freed_in_flight = FALSE;
1896 db->db_pending_evict = FALSE;
1900 db->db.db_size = DN_MAX_BONUSLEN -
1902 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen);
1903 db->db.db_offset = DMU_BONUS_BLKID;
1904 db->db_state = DB_UNCACHED;
1907 return (db);
1909 db->db.db_size = (blkptr != NULL) ?
1911 db->db.db_offset = 0;
1914 db->db_level ? 1 << dn->dn_indblkshift : dn->dn_datablksz;
1915 db->db.db_size = blocksize;
1916 db->db.db_offset = db->db_blkid * blocksize;
1927 db->db_state = DB_EVICTING;
1928 if ((odb = dbuf_hash_insert(db)) != NULL) {
1930 kmem_cache_free(dbuf_cache, db);
1934 avl_add(&dn->dn_dbufs, db);
1935 if (db->db_level == 0 && db->db_blkid >=
1937 dn->dn_unlisted_l0_blkid = db->db_blkid + 1;
1938 db->db_state = DB_UNCACHED;
1943 dbuf_add_ref(parent, db);
1947 (void) refcount_add(&dn->dn_holds, db);
1950 dprintf_dbuf(db, "db=%p\n", db);
1952 return (db);
1958 dmu_buf_impl_t *db = private;
1960 if (!MUTEX_HELD(&db->db_mtx))
1961 mutex_enter(&db->db_mtx);
1963 ASSERT(refcount_is_zero(&db->db_holds));
1965 if (db->db_state != DB_EVICTING) {
1966 ASSERT(db->db_state == DB_CACHED);
1967 DBUF_VERIFY(db);
1968 db->db_buf = NULL;
1969 dbuf_evict(db);
1971 mutex_exit(&db->db_mtx);
1972 dbuf_destroy(db);
1978 dbuf_destroy(dmu_buf_impl_t *db)
1980 ASSERT(refcount_is_zero(&db->db_holds));
1982 if (db->db_blkid != DMU_BONUS_BLKID) {
1987 if (db->db_dnode_handle != NULL) {
1990 DB_DNODE_ENTER(db);
1991 dn = DB_DNODE(db);
1993 avl_remove(&dn->dn_dbufs, db);
1996 DB_DNODE_EXIT(db);
2003 dnode_rele(dn, db);
2004 db->db_dnode_handle = NULL;
2006 dbuf_hash_remove(db);
2008 db->db_parent = NULL;
2009 db->db_buf = NULL;
2011 ASSERT(db->db.db_data == NULL);
2012 ASSERT(db->db_hash_next == NULL);
2013 ASSERT(db->db_blkptr == NULL);
2014 ASSERT(db->db_data_pending == NULL);
2016 kmem_cache_free(dbuf_cache, db);
2132 dmu_buf_impl_t *db = dbuf_find(dn->dn_objset, dn->dn_object,
2134 if (db != NULL) {
2135 mutex_exit(&db->db_mtx);
2153 dmu_buf_impl_t *db;
2156 FALSE, TRUE, FTAG, &db) == 0) {
2157 blkptr_t *bpp = db->db_buf->b_data;
2159 dbuf_rele(db, FTAG);
2229 dmu_buf_impl_t *db, *parent = NULL;
2238 db = dbuf_find(dn->dn_objset, dn->dn_object, level, blkid);
2240 if (db == NULL) {
2260 db = dbuf_create(dn, level, blkid, parent, bp);
2263 if (fail_uncached && db->db_state != DB_CACHED) {
2264 mutex_exit(&db->db_mtx);
2268 if (db->db_buf && refcount_is_zero(&db->db_holds)) {
2269 arc_buf_add_ref(db->db_buf, db);
2270 if (db->db_buf->b_data == NULL) {
2271 dbuf_clear(db);
2278 ASSERT3P(db->db.db_data, ==, db->db_buf->b_data);
2281 ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf));
2288 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID &&
2290 db->db_state == DB_CACHED && db->db_data_pending) {
2291 dbuf_dirty_record_t *dr = db->db_data_pending;
2293 if (dr->dt.dl.dr_data == db->db_buf) {
2294 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
2296 dbuf_set_data(db,
2298 db->db.db_size, db, type));
2299 bcopy(dr->dt.dl.dr_data->b_data, db->db.db_data,
2300 db->db.db_size);
2304 (void) refcount_add(&db->db_holds, tag);
2305 DBUF_VERIFY(db);
2306 mutex_exit(&db->db_mtx);
2312 ASSERT3P(DB_DNODE(db), ==, dn);
2313 ASSERT3U(db->db_blkid, ==, blkid);
2314 ASSERT3U(db->db_level, ==, level);
2315 *dbp = db;
2329 dmu_buf_impl_t *db;
2330 int err = dbuf_hold_impl(dn, level, blkid, FALSE, FALSE, tag, &db);
2331 return (err ? NULL : db);
2346 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2349 if (db->db_blkid != DMU_SPILL_BLKID)
2353 ASSERT3U(blksz, <=, spa_maxblocksize(dmu_objset_spa(db->db_objset)));
2356 DB_DNODE_ENTER(db);
2357 dn = DB_DNODE(db);
2359 dbuf_new_size(db, blksz, tx);
2361 DB_DNODE_EXIT(db);
2374 dbuf_add_ref(dmu_buf_impl_t *db, void *tag)
2376 int64_t holds = refcount_add(&db->db_holds, tag);
2385 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2395 if (db == found_db && dbuf_refcount(db) > db->db_dirtycnt) {
2396 (void) refcount_add(&db->db_holds, tag);
2412 dbuf_rele(dmu_buf_impl_t *db, void *tag)
2414 mutex_enter(&db->db_mtx);
2415 dbuf_rele_and_unlock(db, tag);
2419 dmu_buf_rele(dmu_buf_t *db, void *tag)
2421 dbuf_rele((dmu_buf_impl_t *)db, tag);
2429 dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag)
2433 ASSERT(MUTEX_HELD(&db->db_mtx));
2434 DBUF_VERIFY(db);
2441 holds = refcount_remove(&db->db_holds, tag);
2448 if (db->db_buf && holds == (db->db_level == 0 ? db->db_dirtycnt : 0))
2449 arc_buf_freeze(db->db_buf);
2451 if (holds == db->db_dirtycnt &&
2452 db->db_level == 0 && db->db_user_immediate_evict)
2453 dbuf_evict_user(db);
2456 if (db->db_blkid == DMU_BONUS_BLKID) {
2458 boolean_t evict_dbuf = db->db_pending_evict;
2464 DB_DNODE_ENTER(db);
2466 dn = DB_DNODE(db);
2475 DB_DNODE_EXIT(db);
2478 * Do not reference db after its lock is dropped.
2481 mutex_exit(&db->db_mtx);
2486 dnode_rele(dn, db);
2487 } else if (db->db_buf == NULL) {
2492 ASSERT(db->db_state == DB_UNCACHED ||
2493 db->db_state == DB_NOFILL);
2494 dbuf_evict(db);
2495 } else if (arc_released(db->db_buf)) {
2496 arc_buf_t *buf = db->db_buf;
2500 dbuf_clear_data(db);
2501 VERIFY(arc_buf_remove_ref(buf, db));
2502 dbuf_evict(db);
2504 VERIFY(!arc_buf_remove_ref(db->db_buf, db));
2521 if (!DBUF_IS_CACHEABLE(db)) {
2522 if (db->db_blkptr != NULL &&
2523 !BP_IS_HOLE(db->db_blkptr) &&
2524 !BP_IS_EMBEDDED(db->db_blkptr)) {
2526 dmu_objset_spa(db->db_objset);
2527 blkptr_t bp = *db->db_blkptr;
2528 dbuf_clear(db);
2531 dbuf_clear(db);
2533 } else if (db->db_pending_evict ||
2534 arc_buf_eviction_needed(db->db_buf)) {
2535 dbuf_clear(db);
2537 mutex_exit(&db->db_mtx);
2541 mutex_exit(&db->db_mtx);
2547 dbuf_refcount(dmu_buf_impl_t *db)
2549 return (refcount_count(&db->db_holds));
2556 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2558 mutex_enter(&db->db_mtx);
2559 dbuf_verify_user(db, DBVU_NOT_EVICTING);
2560 if (db->db_user == old_user)
2561 db->db_user = new_user;
2563 old_user = db->db_user;
2564 dbuf_verify_user(db, DBVU_NOT_EVICTING);
2565 mutex_exit(&db->db_mtx);
2579 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2581 db->db_user_immediate_evict = TRUE;
2594 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2596 dbuf_verify_user(db, DBVU_NOT_EVICTING);
2597 return (db->db_user);
2610 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf;
2612 if (db->db_blkptr)
2613 res = dsl_dataset_block_freeable(db->db_objset->os_dsl_dataset,
2614 db->db_blkptr, db->db_blkptr->blk_birth);
2620 dmu_buf_get_blkptr(dmu_buf_t *db)
2622 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db;
2627 dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db)
2630 ASSERT(MUTEX_HELD(&db->db_mtx));
2632 if (db->db_blkptr != NULL)
2635 if (db->db_blkid == DMU_SPILL_BLKID) {
2636 db->db_blkptr = &dn->dn_phys->dn_spill;
2637 BP_ZERO(db->db_blkptr);
2640 if (db->db_level == dn->dn_phys->dn_nlevels-1) {
2646 ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr);
2647 ASSERT(db->db_parent == NULL);
2648 db->db_parent = dn->dn_dbuf;
2649 db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid];
2650 DBUF_VERIFY(db);
2652 dmu_buf_impl_t *parent = db->db_parent;
2657 mutex_exit(&db->db_mtx);
2659 parent = dbuf_hold_level(dn, db->db_level + 1,
2660 db->db_blkid >> epbs, db);
2662 mutex_enter(&db->db_mtx);
2663 db->db_parent = parent;
2665 db->db_blkptr = (blkptr_t *)parent->db.db_data +
2666 (db->db_blkid & ((1ULL << epbs) - 1));
2667 DBUF_VERIFY(db);
2674 dmu_buf_impl_t *db = dr->dr_dbuf;
2680 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr);
2682 mutex_enter(&db->db_mtx);
2684 ASSERT(db->db_level > 0);
2685 DBUF_VERIFY(db);
2688 if (db->db_buf == NULL) {
2689 mutex_exit(&db->db_mtx);
2690 (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED);
2691 mutex_enter(&db->db_mtx);
2693 ASSERT3U(db->db_state, ==, DB_CACHED);
2694 ASSERT(db->db_buf != NULL);
2696 DB_DNODE_ENTER(db);
2697 dn = DB_DNODE(db);
2699 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift);
2700 dbuf_check_blkptr(dn, db);
2701 DB_DNODE_EXIT(db);
2704 db->db_data_pending = dr;
2706 mutex_exit(&db->db_mtx);
2707 dbuf_write(dr, db->db_buf, tx);
2711 dbuf_sync_list(&dr->dt.di.dr_children, db->db_level - 1, tx);
2721 dmu_buf_impl_t *db = dr->dr_dbuf;
2728 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr);
2730 mutex_enter(&db->db_mtx);
2735 if (db->db_state == DB_UNCACHED) {
2737 ASSERT(db->db.db_data == NULL);
2738 } else if (db->db_state == DB_FILL) {
2740 ASSERT(db->db.db_data != dr->dt.dl.dr_data);
2742 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_NOFILL);
2744 DBUF_VERIFY(db);
2746 DB_DNODE_ENTER(db);
2747 dn = DB_DNODE(db);
2749 if (db->db_blkid == DMU_SPILL_BLKID) {
2761 if (db->db_blkid == DMU_BONUS_BLKID) {
2765 ASSERT0(db->db_level);
2768 DB_DNODE_EXIT(db);
2770 if (*datap != db->db.db_data) {
2774 db->db_data_pending = NULL;
2775 drp = &db->db_last_dirty;
2779 ASSERT(dr->dr_dbuf == db);
2782 ASSERT(db->db_dirtycnt > 0);
2783 db->db_dirtycnt -= 1;
2784 dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg);
2796 dbuf_check_blkptr(dn, db);
2804 cv_wait(&db->db_changed, &db->db_mtx);
2808 if (db->db_state != DB_NOFILL &&
2810 refcount_count(&db->db_holds) > 1 &&
2812 *datap == db->db_buf) {
2825 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
2826 *datap = arc_buf_alloc(os->os_spa, blksz, db, type);
2827 bcopy(db->db.db_data, (*datap)->b_data, blksz);
2829 db->db_data_pending = dr;
2831 mutex_exit(&db->db_mtx);
2838 DB_DNODE_EXIT(db);
2847 DB_DNODE_EXIT(db);
2866 ASSERT3U(dr->dr_dbuf->db.db_object, ==,
2886 dmu_buf_impl_t *db = vdb;
2895 ASSERT3P(db->db_blkptr, ==, bp);
2897 DB_DNODE_ENTER(db);
2898 dn = DB_DNODE(db);
2904 ASSERT((db->db_blkid != DMU_SPILL_BLKID &&
2906 (db->db_blkid == DMU_SPILL_BLKID &&
2909 ASSERT(BP_GET_LEVEL(bp) == db->db_level);
2912 mutex_enter(&db->db_mtx);
2915 if (db->db_blkid == DMU_SPILL_BLKID) {
2917 ASSERT(!(BP_IS_HOLE(db->db_blkptr)) &&
2918 db->db_blkptr == &dn->dn_phys->dn_spill);
2922 if (db->db_level == 0) {
2924 if (db->db_blkid > dn->dn_phys->dn_maxblkid &&
2925 db->db_blkid != DMU_SPILL_BLKID)
2926 dn->dn_phys->dn_maxblkid = db->db_blkid;
2930 dnode_phys_t *dnp = db->db.db_data;
2931 for (i = db->db.db_size >> DNODE_SHIFT; i > 0;
2944 blkptr_t *ibp = db->db.db_data;
2945 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift);
2946 for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, ibp++) {
2952 DB_DNODE_EXIT(db);
2957 mutex_exit(&db->db_mtx);
2973 dmu_buf_impl_t *db = arg;
2974 objset_t *os = db->db_objset;
2979 dr = db->db_data_pending;
2996 dmu_buf_impl_t *db = vdb;
2998 blkptr_t *bp = db->db_blkptr;
2999 objset_t *os = db->db_objset;
3004 ASSERT(db->db_blkptr == bp);
3018 mutex_enter(&db->db_mtx);
3020 DBUF_VERIFY(db);
3022 drp = &db->db_last_dirty;
3023 while ((dr = *drp) != db->db_data_pending)
3026 ASSERT(dr->dr_dbuf == db);
3031 if (db->db_blkid == DMU_SPILL_BLKID) {
3034 DB_DNODE_ENTER(db);
3035 dn = DB_DNODE(db);
3037 ASSERT(!(BP_IS_HOLE(db->db_blkptr)) &&
3038 db->db_blkptr == &dn->dn_phys->dn_spill);
3039 DB_DNODE_EXIT(db);
3043 if (db->db_level == 0) {
3044 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
3046 if (db->db_state != DB_NOFILL) {
3047 if (dr->dt.dl.dr_data != db->db_buf)
3049 db));
3050 else if (!arc_released(db->db_buf))
3051 arc_set_callback(db->db_buf, dbuf_do_evict, db);
3056 DB_DNODE_ENTER(db);
3057 dn = DB_DNODE(db);
3059 ASSERT3U(db->db.db_size, ==, 1 << dn->dn_phys->dn_indblkshift);
3060 if (!BP_IS_HOLE(db->db_blkptr)) {
3063 ASSERT3U(db->db_blkid, <=,
3064 dn->dn_phys->dn_maxblkid >> (db->db_level * epbs));
3065 ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==,
3066 db->db.db_size);
3067 if (!arc_released(db->db_buf))
3068 arc_set_callback(db->db_buf, dbuf_do_evict, db);
3070 DB_DNODE_EXIT(db);
3076 cv_broadcast(&db->db_changed);
3077 ASSERT(db->db_dirtycnt > 0);
3078 db->db_dirtycnt -= 1;
3079 db->db_data_pending = NULL;
3080 dbuf_rele_and_unlock(db, (void *)(uintptr_t)tx->tx_txg);
3099 dmu_buf_impl_t *db = dr->dr_dbuf;
3101 dbuf_write_ready(zio, NULL, db);
3108 dmu_buf_impl_t *db = dr->dr_dbuf;
3111 mutex_enter(&db->db_mtx);
3115 arc_release(dr->dt.dl.dr_data, db);
3117 mutex_exit(&db->db_mtx);
3119 dbuf_write_done(zio, NULL, db);
3126 dmu_buf_impl_t *db = dr->dr_dbuf;
3129 dmu_buf_impl_t *parent = db->db_parent;
3136 DB_DNODE_ENTER(db);
3137 dn = DB_DNODE(db);
3140 if (db->db_state != DB_NOFILL) {
3141 if (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE) {
3148 if (BP_IS_HOLE(db->db_blkptr)) {
3151 dbuf_release_bp(db);
3161 ASSERT(db->db_level == parent->db_level-1);
3170 ASSERT((db->db_level == dn->dn_phys->dn_nlevels-1 &&
3171 db->db_blkid != DMU_SPILL_BLKID) ||
3172 (db->db_blkid == DMU_SPILL_BLKID && db->db_level == 0));
3173 if (db->db_blkid != DMU_SPILL_BLKID)
3174 ASSERT3P(db->db_blkptr, ==,
3175 &dn->dn_phys->dn_blkptr[db->db_blkid]);
3179 ASSERT(db->db_level == 0 || data == db->db_buf);
3180 ASSERT3U(db->db_blkptr->blk_birth, <=, txg);
3185 db->db.db_object, db->db_level, db->db_blkid);
3187 if (db->db_blkid == DMU_SPILL_BLKID)
3189 wp_flag |= (db->db_state == DB_NOFILL) ? WP_NOFILL : 0;
3191 dmu_write_policy(os, dn, db->db_level, wp_flag, &zp);
3192 DB_DNODE_EXIT(db);
3194 if (db->db_level == 0 &&
3203 db->db_blkptr, contents, db->db.db_size, &zp,
3206 mutex_enter(&db->db_mtx);
3210 mutex_exit(&db->db_mtx);
3211 } else if (db->db_state == DB_NOFILL) {
3215 db->db_blkptr, NULL, db->db.db_size, &zp,
3216 dbuf_write_nofill_ready, NULL, dbuf_write_nofill_done, db,
3222 db->db_blkptr, data, DBUF_IS_L2CACHEABLE(db),
3223 DBUF_IS_L2COMPRESSIBLE(db), &zp, dbuf_write_ready,
3224 dbuf_write_physdone, dbuf_write_done, db,