1fa9e4066Sahrens /* 2fa9e4066Sahrens * CDDL HEADER START 3fa9e4066Sahrens * 4fa9e4066Sahrens * The contents of this file are subject to the terms of the 5f65e61c0Sahrens * Common Development and Distribution License (the "License"). 6f65e61c0Sahrens * You may not use this file except in compliance with the License. 7fa9e4066Sahrens * 8fa9e4066Sahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9fa9e4066Sahrens * or http://www.opensolaris.org/os/licensing. 10fa9e4066Sahrens * See the License for the specific language governing permissions 11fa9e4066Sahrens * and limitations under the License. 12fa9e4066Sahrens * 13fa9e4066Sahrens * When distributing Covered Code, include this CDDL HEADER in each 14fa9e4066Sahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15fa9e4066Sahrens * If applicable, add the following below this CDDL HEADER, with the 16fa9e4066Sahrens * fields enclosed by brackets "[]" replaced with your own identifying 17fa9e4066Sahrens * information: Portions Copyright [yyyy] [name of copyright owner] 18fa9e4066Sahrens * 19fa9e4066Sahrens * CDDL HEADER END 20fa9e4066Sahrens */ 21fa9e4066Sahrens /* 2206e0070dSMark Shellenbaum * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23fa9e4066Sahrens */ 24fa9e4066Sahrens 25fa9e4066Sahrens #include <sys/zfs_context.h> 26fa9e4066Sahrens #include <sys/dmu.h> 27fa9e4066Sahrens #include <sys/dmu_impl.h> 28fa9e4066Sahrens #include <sys/dbuf.h> 29fa9e4066Sahrens #include <sys/dmu_objset.h> 30fa9e4066Sahrens #include <sys/dsl_dataset.h> 31fa9e4066Sahrens #include <sys/dsl_dir.h> 32fa9e4066Sahrens #include <sys/dmu_tx.h> 33fa9e4066Sahrens #include <sys/spa.h> 34fa9e4066Sahrens #include <sys/zio.h> 35fa9e4066Sahrens #include <sys/dmu_zfetch.h> 360a586ceaSMark Shellenbaum #include <sys/sa.h> 370a586ceaSMark Shellenbaum #include <sys/sa_impl.h> 38fa9e4066Sahrens 39fa9e4066Sahrens static void dbuf_destroy(dmu_buf_impl_t *db); 40fa9e4066Sahrens static int dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx); 41088f3894Sahrens static void dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx); 42fa9e4066Sahrens 43fa9e4066Sahrens /* 44fa9e4066Sahrens * Global data structures and functions for the dbuf cache. 45fa9e4066Sahrens */ 46fa9e4066Sahrens static kmem_cache_t *dbuf_cache; 47fa9e4066Sahrens 48fa9e4066Sahrens /* ARGSUSED */ 49fa9e4066Sahrens static int 50fa9e4066Sahrens dbuf_cons(void *vdb, void *unused, int kmflag) 51fa9e4066Sahrens { 52fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 53fa9e4066Sahrens bzero(db, sizeof (dmu_buf_impl_t)); 54fa9e4066Sahrens 55fa9e4066Sahrens mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL); 56fa9e4066Sahrens cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL); 57fa9e4066Sahrens refcount_create(&db->db_holds); 58fa9e4066Sahrens return (0); 59fa9e4066Sahrens } 60fa9e4066Sahrens 61fa9e4066Sahrens /* ARGSUSED */ 62fa9e4066Sahrens static void 63fa9e4066Sahrens dbuf_dest(void *vdb, void *unused) 64fa9e4066Sahrens { 65fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 66fa9e4066Sahrens mutex_destroy(&db->db_mtx); 67fa9e4066Sahrens cv_destroy(&db->db_changed); 68fa9e4066Sahrens refcount_destroy(&db->db_holds); 69fa9e4066Sahrens } 70fa9e4066Sahrens 71fa9e4066Sahrens /* 72fa9e4066Sahrens * dbuf hash table routines 73fa9e4066Sahrens */ 74fa9e4066Sahrens static dbuf_hash_table_t dbuf_hash_table; 75fa9e4066Sahrens 76fa9e4066Sahrens static uint64_t dbuf_hash_count; 77fa9e4066Sahrens 78fa9e4066Sahrens static uint64_t 79fa9e4066Sahrens dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid) 80fa9e4066Sahrens { 81fa9e4066Sahrens uintptr_t osv = (uintptr_t)os; 82fa9e4066Sahrens uint64_t crc = -1ULL; 83fa9e4066Sahrens 84fa9e4066Sahrens ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 85fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (lvl)) & 0xFF]; 86fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF]; 87fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF]; 88fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF]; 89fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 0)) & 0xFF]; 90fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 8)) & 0xFF]; 91fa9e4066Sahrens 92fa9e4066Sahrens crc ^= (osv>>14) ^ (obj>>16) ^ (blkid>>16); 93fa9e4066Sahrens 94fa9e4066Sahrens return (crc); 95fa9e4066Sahrens } 96fa9e4066Sahrens 97fa9e4066Sahrens #define DBUF_HASH(os, obj, level, blkid) dbuf_hash(os, obj, level, blkid); 98fa9e4066Sahrens 99fa9e4066Sahrens #define DBUF_EQUAL(dbuf, os, obj, level, blkid) \ 100fa9e4066Sahrens ((dbuf)->db.db_object == (obj) && \ 101fa9e4066Sahrens (dbuf)->db_objset == (os) && \ 102fa9e4066Sahrens (dbuf)->db_level == (level) && \ 103fa9e4066Sahrens (dbuf)->db_blkid == (blkid)) 104fa9e4066Sahrens 105fa9e4066Sahrens dmu_buf_impl_t * 106fa9e4066Sahrens dbuf_find(dnode_t *dn, uint8_t level, uint64_t blkid) 107fa9e4066Sahrens { 108fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 109503ad85cSMatthew Ahrens objset_t *os = dn->dn_objset; 110fa9e4066Sahrens uint64_t obj = dn->dn_object; 111fa9e4066Sahrens uint64_t hv = DBUF_HASH(os, obj, level, blkid); 112fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 113fa9e4066Sahrens dmu_buf_impl_t *db; 114fa9e4066Sahrens 115fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 116fa9e4066Sahrens for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) { 117fa9e4066Sahrens if (DBUF_EQUAL(db, os, obj, level, blkid)) { 118fa9e4066Sahrens mutex_enter(&db->db_mtx); 119ea8dc4b6Seschrock if (db->db_state != DB_EVICTING) { 120fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 121fa9e4066Sahrens return (db); 122fa9e4066Sahrens } 123fa9e4066Sahrens mutex_exit(&db->db_mtx); 124fa9e4066Sahrens } 125fa9e4066Sahrens } 126fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 127fa9e4066Sahrens return (NULL); 128fa9e4066Sahrens } 129fa9e4066Sahrens 130fa9e4066Sahrens /* 131fa9e4066Sahrens * Insert an entry into the hash table. If there is already an element 132fa9e4066Sahrens * equal to elem in the hash table, then the already existing element 133fa9e4066Sahrens * will be returned and the new element will not be inserted. 134fa9e4066Sahrens * Otherwise returns NULL. 135fa9e4066Sahrens */ 136fa9e4066Sahrens static dmu_buf_impl_t * 137fa9e4066Sahrens dbuf_hash_insert(dmu_buf_impl_t *db) 138fa9e4066Sahrens { 139fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 140503ad85cSMatthew Ahrens objset_t *os = db->db_objset; 141fa9e4066Sahrens uint64_t obj = db->db.db_object; 142fa9e4066Sahrens int level = db->db_level; 143fa9e4066Sahrens uint64_t blkid = db->db_blkid; 144fa9e4066Sahrens uint64_t hv = DBUF_HASH(os, obj, level, blkid); 145fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 146fa9e4066Sahrens dmu_buf_impl_t *dbf; 147fa9e4066Sahrens 148fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 149fa9e4066Sahrens for (dbf = h->hash_table[idx]; dbf != NULL; dbf = dbf->db_hash_next) { 150fa9e4066Sahrens if (DBUF_EQUAL(dbf, os, obj, level, blkid)) { 151fa9e4066Sahrens mutex_enter(&dbf->db_mtx); 152ea8dc4b6Seschrock if (dbf->db_state != DB_EVICTING) { 153fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 154fa9e4066Sahrens return (dbf); 155fa9e4066Sahrens } 156fa9e4066Sahrens mutex_exit(&dbf->db_mtx); 157fa9e4066Sahrens } 158fa9e4066Sahrens } 159fa9e4066Sahrens 160fa9e4066Sahrens mutex_enter(&db->db_mtx); 161fa9e4066Sahrens db->db_hash_next = h->hash_table[idx]; 162fa9e4066Sahrens h->hash_table[idx] = db; 163fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 164fa9e4066Sahrens atomic_add_64(&dbuf_hash_count, 1); 165fa9e4066Sahrens 166fa9e4066Sahrens return (NULL); 167fa9e4066Sahrens } 168fa9e4066Sahrens 169fa9e4066Sahrens /* 170fa9e4066Sahrens * Remove an entry from the hash table. This operation will 171fa9e4066Sahrens * fail if there are any existing holds on the db. 172fa9e4066Sahrens */ 173fa9e4066Sahrens static void 174fa9e4066Sahrens dbuf_hash_remove(dmu_buf_impl_t *db) 175fa9e4066Sahrens { 176fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 177fa9e4066Sahrens uint64_t hv = DBUF_HASH(db->db_objset, db->db.db_object, 178fa9e4066Sahrens db->db_level, db->db_blkid); 179fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 180fa9e4066Sahrens dmu_buf_impl_t *dbf, **dbp; 181fa9e4066Sahrens 182fa9e4066Sahrens /* 183fa9e4066Sahrens * We musn't hold db_mtx to maintin lock ordering: 184fa9e4066Sahrens * DBUF_HASH_MUTEX > db_mtx. 185fa9e4066Sahrens */ 186fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 187ea8dc4b6Seschrock ASSERT(db->db_state == DB_EVICTING); 188fa9e4066Sahrens ASSERT(!MUTEX_HELD(&db->db_mtx)); 189fa9e4066Sahrens 190fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 191fa9e4066Sahrens dbp = &h->hash_table[idx]; 192fa9e4066Sahrens while ((dbf = *dbp) != db) { 193fa9e4066Sahrens dbp = &dbf->db_hash_next; 194fa9e4066Sahrens ASSERT(dbf != NULL); 195fa9e4066Sahrens } 196fa9e4066Sahrens *dbp = db->db_hash_next; 197fa9e4066Sahrens db->db_hash_next = NULL; 198fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 199fa9e4066Sahrens atomic_add_64(&dbuf_hash_count, -1); 200fa9e4066Sahrens } 201fa9e4066Sahrens 202ea8dc4b6Seschrock static arc_evict_func_t dbuf_do_evict; 203fa9e4066Sahrens 204fa9e4066Sahrens static void 205fa9e4066Sahrens dbuf_evict_user(dmu_buf_impl_t *db) 206fa9e4066Sahrens { 207fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 208fa9e4066Sahrens 209c717a561Smaybee if (db->db_level != 0 || db->db_evict_func == NULL) 210fa9e4066Sahrens return; 211fa9e4066Sahrens 212c717a561Smaybee if (db->db_user_data_ptr_ptr) 213c717a561Smaybee *db->db_user_data_ptr_ptr = db->db.db_data; 214c717a561Smaybee db->db_evict_func(&db->db, db->db_user_ptr); 215c717a561Smaybee db->db_user_ptr = NULL; 216c717a561Smaybee db->db_user_data_ptr_ptr = NULL; 217c717a561Smaybee db->db_evict_func = NULL; 218fa9e4066Sahrens } 219fa9e4066Sahrens 220fa9e4066Sahrens void 221ea8dc4b6Seschrock dbuf_evict(dmu_buf_impl_t *db) 222ea8dc4b6Seschrock { 223ea8dc4b6Seschrock ASSERT(MUTEX_HELD(&db->db_mtx)); 224ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 225c717a561Smaybee ASSERT(db->db_data_pending == NULL); 226ea8dc4b6Seschrock 227ea8dc4b6Seschrock dbuf_clear(db); 228ea8dc4b6Seschrock dbuf_destroy(db); 229ea8dc4b6Seschrock } 230ea8dc4b6Seschrock 231ea8dc4b6Seschrock void 232fa9e4066Sahrens dbuf_init(void) 233fa9e4066Sahrens { 234ea8dc4b6Seschrock uint64_t hsize = 1ULL << 16; 235fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 236fa9e4066Sahrens int i; 237fa9e4066Sahrens 238fa9e4066Sahrens /* 239fa9e4066Sahrens * The hash table is big enough to fill all of physical memory 240ea8dc4b6Seschrock * with an average 4K block size. The table will take up 241ea8dc4b6Seschrock * totalmem*sizeof(void*)/4K (i.e. 2MB/GB with 8-byte pointers). 242fa9e4066Sahrens */ 243ea8dc4b6Seschrock while (hsize * 4096 < physmem * PAGESIZE) 244fa9e4066Sahrens hsize <<= 1; 245fa9e4066Sahrens 246ea8dc4b6Seschrock retry: 247fa9e4066Sahrens h->hash_table_mask = hsize - 1; 248ea8dc4b6Seschrock h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP); 249ea8dc4b6Seschrock if (h->hash_table == NULL) { 250ea8dc4b6Seschrock /* XXX - we should really return an error instead of assert */ 251ea8dc4b6Seschrock ASSERT(hsize > (1ULL << 10)); 252ea8dc4b6Seschrock hsize >>= 1; 253ea8dc4b6Seschrock goto retry; 254ea8dc4b6Seschrock } 255fa9e4066Sahrens 256fa9e4066Sahrens dbuf_cache = kmem_cache_create("dmu_buf_impl_t", 257fa9e4066Sahrens sizeof (dmu_buf_impl_t), 258fa9e4066Sahrens 0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0); 259fa9e4066Sahrens 260fa9e4066Sahrens for (i = 0; i < DBUF_MUTEXES; i++) 261fa9e4066Sahrens mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL); 262fa9e4066Sahrens } 263fa9e4066Sahrens 264fa9e4066Sahrens void 265fa9e4066Sahrens dbuf_fini(void) 266fa9e4066Sahrens { 267fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 268fa9e4066Sahrens int i; 269fa9e4066Sahrens 270fa9e4066Sahrens for (i = 0; i < DBUF_MUTEXES; i++) 271fa9e4066Sahrens mutex_destroy(&h->hash_mutexes[i]); 272fa9e4066Sahrens kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *)); 273fa9e4066Sahrens kmem_cache_destroy(dbuf_cache); 274fa9e4066Sahrens } 275fa9e4066Sahrens 276fa9e4066Sahrens /* 277fa9e4066Sahrens * Other stuff. 278fa9e4066Sahrens */ 279fa9e4066Sahrens 2809c9dc39aSek110237 #ifdef ZFS_DEBUG 281fa9e4066Sahrens static void 282fa9e4066Sahrens dbuf_verify(dmu_buf_impl_t *db) 283fa9e4066Sahrens { 284fa9e4066Sahrens dnode_t *dn = db->db_dnode; 285b24ab676SJeff Bonwick dbuf_dirty_record_t *dr; 286fa9e4066Sahrens 287fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 288fa9e4066Sahrens 289fa9e4066Sahrens if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY)) 290fa9e4066Sahrens return; 291fa9e4066Sahrens 292fa9e4066Sahrens ASSERT(db->db_objset != NULL); 293fa9e4066Sahrens if (dn == NULL) { 294fa9e4066Sahrens ASSERT(db->db_parent == NULL); 295fa9e4066Sahrens ASSERT(db->db_blkptr == NULL); 296fa9e4066Sahrens } else { 297fa9e4066Sahrens ASSERT3U(db->db.db_object, ==, dn->dn_object); 298fa9e4066Sahrens ASSERT3P(db->db_objset, ==, dn->dn_objset); 299fa9e4066Sahrens ASSERT3U(db->db_level, <, dn->dn_nlevels); 3000a586ceaSMark Shellenbaum ASSERT(db->db_blkid == DMU_BONUS_BLKID || db->db_blkid == 3010a586ceaSMark Shellenbaum DMU_SPILL_BLKID || list_head(&dn->dn_dbufs)); 302fa9e4066Sahrens } 3030a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 304fa9e4066Sahrens ASSERT(dn != NULL); 3051934e92fSmaybee ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 3060a586ceaSMark Shellenbaum ASSERT3U(db->db.db_offset, ==, DMU_BONUS_BLKID); 3070a586ceaSMark Shellenbaum } else if (db->db_blkid == DMU_SPILL_BLKID) { 3080a586ceaSMark Shellenbaum ASSERT(dn != NULL); 3090a586ceaSMark Shellenbaum ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 3100a586ceaSMark Shellenbaum ASSERT3U(db->db.db_offset, ==, 0); 311fa9e4066Sahrens } else { 312fa9e4066Sahrens ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size); 313fa9e4066Sahrens } 314fa9e4066Sahrens 315b24ab676SJeff Bonwick for (dr = db->db_data_pending; dr != NULL; dr = dr->dr_next) 316b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 317b24ab676SJeff Bonwick 318b24ab676SJeff Bonwick for (dr = db->db_last_dirty; dr != NULL; dr = dr->dr_next) 319b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 320b24ab676SJeff Bonwick 32188b7b0f2SMatthew Ahrens /* 32288b7b0f2SMatthew Ahrens * We can't assert that db_size matches dn_datablksz because it 32388b7b0f2SMatthew Ahrens * can be momentarily different when another thread is doing 32488b7b0f2SMatthew Ahrens * dnode_set_blksz(). 32588b7b0f2SMatthew Ahrens */ 32688b7b0f2SMatthew Ahrens if (db->db_level == 0 && db->db.db_object == DMU_META_DNODE_OBJECT) { 327b24ab676SJeff Bonwick dr = db->db_data_pending; 328fa9e4066Sahrens /* 32988b7b0f2SMatthew Ahrens * It should only be modified in syncing context, so 33088b7b0f2SMatthew Ahrens * make sure we only have one copy of the data. 331fa9e4066Sahrens */ 332c717a561Smaybee ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf); 333fa9e4066Sahrens } 334fa9e4066Sahrens 335fa9e4066Sahrens /* verify db->db_blkptr */ 336fa9e4066Sahrens if (db->db_blkptr) { 337fa9e4066Sahrens if (db->db_parent == dn->dn_dbuf) { 338fa9e4066Sahrens /* db is pointed to by the dnode */ 339fa9e4066Sahrens /* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */ 34014843421SMatthew Ahrens if (DMU_OBJECT_IS_SPECIAL(db->db.db_object)) 341fa9e4066Sahrens ASSERT(db->db_parent == NULL); 342fa9e4066Sahrens else 343fa9e4066Sahrens ASSERT(db->db_parent != NULL); 3440a586ceaSMark Shellenbaum if (db->db_blkid != DMU_SPILL_BLKID) 345fa9e4066Sahrens ASSERT3P(db->db_blkptr, ==, 346fa9e4066Sahrens &dn->dn_phys->dn_blkptr[db->db_blkid]); 347fa9e4066Sahrens } else { 348fa9e4066Sahrens /* db is pointed to by an indirect block */ 349fa9e4066Sahrens int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT; 350fa9e4066Sahrens ASSERT3U(db->db_parent->db_level, ==, db->db_level+1); 351fa9e4066Sahrens ASSERT3U(db->db_parent->db.db_object, ==, 352fa9e4066Sahrens db->db.db_object); 353fa9e4066Sahrens /* 354fa9e4066Sahrens * dnode_grow_indblksz() can make this fail if we don't 355fa9e4066Sahrens * have the struct_rwlock. XXX indblksz no longer 356fa9e4066Sahrens * grows. safe to do this now? 357fa9e4066Sahrens */ 358fa9e4066Sahrens if (RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)) { 359fa9e4066Sahrens ASSERT3P(db->db_blkptr, ==, 360fa9e4066Sahrens ((blkptr_t *)db->db_parent->db.db_data + 361fa9e4066Sahrens db->db_blkid % epb)); 362fa9e4066Sahrens } 363fa9e4066Sahrens } 364fa9e4066Sahrens } 365fa9e4066Sahrens if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) && 366*3f9d6ad7SLin Ling (db->db_buf == NULL || db->db_buf->b_data) && 3670a586ceaSMark Shellenbaum db->db.db_data && db->db_blkid != DMU_BONUS_BLKID && 368fa9e4066Sahrens db->db_state != DB_FILL && !dn->dn_free_txg) { 369fa9e4066Sahrens /* 370fa9e4066Sahrens * If the blkptr isn't set but they have nonzero data, 371fa9e4066Sahrens * it had better be dirty, otherwise we'll lose that 372fa9e4066Sahrens * data when we evict this buffer. 373fa9e4066Sahrens */ 374fa9e4066Sahrens if (db->db_dirtycnt == 0) { 375fa9e4066Sahrens uint64_t *buf = db->db.db_data; 376fa9e4066Sahrens int i; 377fa9e4066Sahrens 378fa9e4066Sahrens for (i = 0; i < db->db.db_size >> 3; i++) { 379fa9e4066Sahrens ASSERT(buf[i] == 0); 380fa9e4066Sahrens } 381fa9e4066Sahrens } 382fa9e4066Sahrens } 383fa9e4066Sahrens } 3849c9dc39aSek110237 #endif 385fa9e4066Sahrens 386fa9e4066Sahrens static void 387fa9e4066Sahrens dbuf_update_data(dmu_buf_impl_t *db) 388fa9e4066Sahrens { 389fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 390c717a561Smaybee if (db->db_level == 0 && db->db_user_data_ptr_ptr) { 391fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 392c717a561Smaybee *db->db_user_data_ptr_ptr = db->db.db_data; 393fa9e4066Sahrens } 394fa9e4066Sahrens } 395fa9e4066Sahrens 396fa9e4066Sahrens static void 397fa9e4066Sahrens dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf) 398fa9e4066Sahrens { 399fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 400ea8dc4b6Seschrock ASSERT(db->db_buf == NULL || !arc_has_callback(db->db_buf)); 401fa9e4066Sahrens db->db_buf = buf; 402ea8dc4b6Seschrock if (buf != NULL) { 403ea8dc4b6Seschrock ASSERT(buf->b_data != NULL); 404fa9e4066Sahrens db->db.db_data = buf->b_data; 405ea8dc4b6Seschrock if (!arc_released(buf)) 406ea8dc4b6Seschrock arc_set_callback(buf, dbuf_do_evict, db); 407fa9e4066Sahrens dbuf_update_data(db); 408ea8dc4b6Seschrock } else { 409ea8dc4b6Seschrock dbuf_evict_user(db); 410ea8dc4b6Seschrock db->db.db_data = NULL; 41182c9918fSTim Haley if (db->db_state != DB_NOFILL) 412ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 413ea8dc4b6Seschrock } 414fa9e4066Sahrens } 415fa9e4066Sahrens 416c242f9a0Schunli zhang - Sun Microsystems - Irvine United States /* 417c242f9a0Schunli zhang - Sun Microsystems - Irvine United States * Loan out an arc_buf for read. Return the loaned arc_buf. 418c242f9a0Schunli zhang - Sun Microsystems - Irvine United States */ 419c242f9a0Schunli zhang - Sun Microsystems - Irvine United States arc_buf_t * 420c242f9a0Schunli zhang - Sun Microsystems - Irvine United States dbuf_loan_arcbuf(dmu_buf_impl_t *db) 421c242f9a0Schunli zhang - Sun Microsystems - Irvine United States { 422c242f9a0Schunli zhang - Sun Microsystems - Irvine United States arc_buf_t *abuf; 423c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 424c242f9a0Schunli zhang - Sun Microsystems - Irvine United States mutex_enter(&db->db_mtx); 425c242f9a0Schunli zhang - Sun Microsystems - Irvine United States if (arc_released(db->db_buf) || refcount_count(&db->db_holds) > 1) { 426c242f9a0Schunli zhang - Sun Microsystems - Irvine United States int blksz = db->db.db_size; 427c242f9a0Schunli zhang - Sun Microsystems - Irvine United States mutex_exit(&db->db_mtx); 428c242f9a0Schunli zhang - Sun Microsystems - Irvine United States abuf = arc_loan_buf(db->db_dnode->dn_objset->os_spa, blksz); 429c242f9a0Schunli zhang - Sun Microsystems - Irvine United States bcopy(db->db.db_data, abuf->b_data, blksz); 430c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } else { 431c242f9a0Schunli zhang - Sun Microsystems - Irvine United States abuf = db->db_buf; 432c242f9a0Schunli zhang - Sun Microsystems - Irvine United States arc_loan_inuse_buf(abuf, db); 433c242f9a0Schunli zhang - Sun Microsystems - Irvine United States dbuf_set_data(db, NULL); 434c242f9a0Schunli zhang - Sun Microsystems - Irvine United States mutex_exit(&db->db_mtx); 435c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } 436c242f9a0Schunli zhang - Sun Microsystems - Irvine United States return (abuf); 437c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } 438c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 439fa9e4066Sahrens uint64_t 440fa9e4066Sahrens dbuf_whichblock(dnode_t *dn, uint64_t offset) 441fa9e4066Sahrens { 442fa9e4066Sahrens if (dn->dn_datablkshift) { 443fa9e4066Sahrens return (offset >> dn->dn_datablkshift); 444fa9e4066Sahrens } else { 445fa9e4066Sahrens ASSERT3U(offset, <, dn->dn_datablksz); 446fa9e4066Sahrens return (0); 447fa9e4066Sahrens } 448fa9e4066Sahrens } 449fa9e4066Sahrens 450fa9e4066Sahrens static void 451fa9e4066Sahrens dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb) 452fa9e4066Sahrens { 453fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 454fa9e4066Sahrens 455fa9e4066Sahrens mutex_enter(&db->db_mtx); 456fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_READ); 457fa9e4066Sahrens /* 458fa9e4066Sahrens * All reads are synchronous, so we must have a hold on the dbuf 459fa9e4066Sahrens */ 460fa9e4066Sahrens ASSERT(refcount_count(&db->db_holds) > 0); 461ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 462fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 463c717a561Smaybee if (db->db_level == 0 && db->db_freed_in_flight) { 464fa9e4066Sahrens /* we were freed in flight; disregard any error */ 465fa9e4066Sahrens arc_release(buf, db); 466fa9e4066Sahrens bzero(buf->b_data, db->db.db_size); 4676b4acc8bSahrens arc_buf_freeze(buf); 468c717a561Smaybee db->db_freed_in_flight = FALSE; 469fa9e4066Sahrens dbuf_set_data(db, buf); 470fa9e4066Sahrens db->db_state = DB_CACHED; 471fa9e4066Sahrens } else if (zio == NULL || zio->io_error == 0) { 472fa9e4066Sahrens dbuf_set_data(db, buf); 473fa9e4066Sahrens db->db_state = DB_CACHED; 474fa9e4066Sahrens } else { 4750a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 476fa9e4066Sahrens ASSERT3P(db->db_buf, ==, NULL); 477ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(buf, db) == 1); 478ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 479fa9e4066Sahrens } 480fa9e4066Sahrens cv_broadcast(&db->db_changed); 481*3f9d6ad7SLin Ling dbuf_rele_and_unlock(db, NULL); 482fa9e4066Sahrens } 483fa9e4066Sahrens 484ea8dc4b6Seschrock static void 48513506d1eSmaybee dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t *flags) 486fa9e4066Sahrens { 487088f3894Sahrens dnode_t *dn = db->db_dnode; 488ea8dc4b6Seschrock zbookmark_t zb; 48913506d1eSmaybee uint32_t aflags = ARC_NOWAIT; 490088f3894Sahrens arc_buf_t *pbuf; 491fa9e4066Sahrens 492fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 493fa9e4066Sahrens /* We need the struct_rwlock to prevent db_blkptr from changing. */ 494088f3894Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 495ea8dc4b6Seschrock ASSERT(MUTEX_HELD(&db->db_mtx)); 496ea8dc4b6Seschrock ASSERT(db->db_state == DB_UNCACHED); 497ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 498fa9e4066Sahrens 4990a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 500cf04dda1SMark Maybee int bonuslen = MIN(dn->dn_bonuslen, dn->dn_phys->dn_bonuslen); 5011934e92fSmaybee 5021934e92fSmaybee ASSERT3U(bonuslen, <=, db->db.db_size); 503ea8dc4b6Seschrock db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN); 5045a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 5051934e92fSmaybee if (bonuslen < DN_MAX_BONUSLEN) 506ea8dc4b6Seschrock bzero(db->db.db_data, DN_MAX_BONUSLEN); 507cf04dda1SMark Maybee if (bonuslen) 508cf04dda1SMark Maybee bcopy(DN_BONUS(dn->dn_phys), db->db.db_data, bonuslen); 509ea8dc4b6Seschrock dbuf_update_data(db); 510fa9e4066Sahrens db->db_state = DB_CACHED; 511fa9e4066Sahrens mutex_exit(&db->db_mtx); 512fa9e4066Sahrens return; 513fa9e4066Sahrens } 514fa9e4066Sahrens 5151c8564a7SMark Maybee /* 5161c8564a7SMark Maybee * Recheck BP_IS_HOLE() after dnode_block_freed() in case dnode_sync() 5171c8564a7SMark Maybee * processes the delete record and clears the bp while we are waiting 5181c8564a7SMark Maybee * for the dn_mtx (resulting in a "no" from block_freed). 5191c8564a7SMark Maybee */ 520088f3894Sahrens if (db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr) || 5211c8564a7SMark Maybee (db->db_level == 0 && (dnode_block_freed(dn, db->db_blkid) || 5221c8564a7SMark Maybee BP_IS_HOLE(db->db_blkptr)))) { 523ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 524ad23a2dbSjohansen 525088f3894Sahrens dbuf_set_data(db, arc_buf_alloc(dn->dn_objset->os_spa, 526ad23a2dbSjohansen db->db.db_size, db, type)); 527fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 528fa9e4066Sahrens db->db_state = DB_CACHED; 52913506d1eSmaybee *flags |= DB_RF_CACHED; 530fa9e4066Sahrens mutex_exit(&db->db_mtx); 531fa9e4066Sahrens return; 532fa9e4066Sahrens } 533fa9e4066Sahrens 534fa9e4066Sahrens db->db_state = DB_READ; 535fa9e4066Sahrens mutex_exit(&db->db_mtx); 536fa9e4066Sahrens 5373baa08fcSek110237 if (DBUF_IS_L2CACHEABLE(db)) 5383baa08fcSek110237 aflags |= ARC_L2CACHE; 5393baa08fcSek110237 540b24ab676SJeff Bonwick SET_BOOKMARK(&zb, db->db_objset->os_dsl_dataset ? 541b24ab676SJeff Bonwick db->db_objset->os_dsl_dataset->ds_object : DMU_META_OBJSET, 542b24ab676SJeff Bonwick db->db.db_object, db->db_level, db->db_blkid); 543ea8dc4b6Seschrock 544ea8dc4b6Seschrock dbuf_add_ref(db, NULL); 545fa9e4066Sahrens /* ZIO_FLAG_CANFAIL callers have to check the parent zio's error */ 546088f3894Sahrens 547088f3894Sahrens if (db->db_parent) 548088f3894Sahrens pbuf = db->db_parent->db_buf; 549088f3894Sahrens else 550088f3894Sahrens pbuf = db->db_objset->os_phys_buf; 551088f3894Sahrens 552*3f9d6ad7SLin Ling (void) dsl_read(zio, dn->dn_objset->os_spa, db->db_blkptr, pbuf, 553fa9e4066Sahrens dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ, 55413506d1eSmaybee (*flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED, 55513506d1eSmaybee &aflags, &zb); 55613506d1eSmaybee if (aflags & ARC_CACHED) 55713506d1eSmaybee *flags |= DB_RF_CACHED; 558fa9e4066Sahrens } 559fa9e4066Sahrens 560ea8dc4b6Seschrock int 561ea8dc4b6Seschrock dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags) 562fa9e4066Sahrens { 563ea8dc4b6Seschrock int err = 0; 564ea8dc4b6Seschrock int havepzio = (zio != NULL); 56513506d1eSmaybee int prefetch; 566fa9e4066Sahrens 567fa9e4066Sahrens /* 568fa9e4066Sahrens * We don't have to hold the mutex to check db_state because it 569fa9e4066Sahrens * can't be freed while we have a hold on the buffer. 570fa9e4066Sahrens */ 571fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 572fa9e4066Sahrens 57382c9918fSTim Haley if (db->db_state == DB_NOFILL) 57482c9918fSTim Haley return (EIO); 57582c9918fSTim Haley 576fa9e4066Sahrens if ((flags & DB_RF_HAVESTRUCT) == 0) 577fa9e4066Sahrens rw_enter(&db->db_dnode->dn_struct_rwlock, RW_READER); 578fa9e4066Sahrens 5790a586ceaSMark Shellenbaum prefetch = db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 5803baa08fcSek110237 (flags & DB_RF_NOPREFETCH) == 0 && db->db_dnode != NULL && 5813baa08fcSek110237 DBUF_IS_CACHEABLE(db); 58213506d1eSmaybee 583fa9e4066Sahrens mutex_enter(&db->db_mtx); 584ea8dc4b6Seschrock if (db->db_state == DB_CACHED) { 585ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 58613506d1eSmaybee if (prefetch) 58713506d1eSmaybee dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset, 58813506d1eSmaybee db->db.db_size, TRUE); 589ea8dc4b6Seschrock if ((flags & DB_RF_HAVESTRUCT) == 0) 590ea8dc4b6Seschrock rw_exit(&db->db_dnode->dn_struct_rwlock); 591ea8dc4b6Seschrock } else if (db->db_state == DB_UNCACHED) { 592ea8dc4b6Seschrock if (zio == NULL) { 593ea8dc4b6Seschrock zio = zio_root(db->db_dnode->dn_objset->os_spa, 594ea8dc4b6Seschrock NULL, NULL, ZIO_FLAG_CANFAIL); 595ea8dc4b6Seschrock } 59613506d1eSmaybee dbuf_read_impl(db, zio, &flags); 59713506d1eSmaybee 598ea8dc4b6Seschrock /* dbuf_read_impl has dropped db_mtx for us */ 599ea8dc4b6Seschrock 60013506d1eSmaybee if (prefetch) 601ea8dc4b6Seschrock dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset, 60213506d1eSmaybee db->db.db_size, flags & DB_RF_CACHED); 603ea8dc4b6Seschrock 604ea8dc4b6Seschrock if ((flags & DB_RF_HAVESTRUCT) == 0) 605ea8dc4b6Seschrock rw_exit(&db->db_dnode->dn_struct_rwlock); 606ea8dc4b6Seschrock 607ea8dc4b6Seschrock if (!havepzio) 608ea8dc4b6Seschrock err = zio_wait(zio); 609ea8dc4b6Seschrock } else { 61013506d1eSmaybee mutex_exit(&db->db_mtx); 61113506d1eSmaybee if (prefetch) 61213506d1eSmaybee dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset, 61313506d1eSmaybee db->db.db_size, TRUE); 614ea8dc4b6Seschrock if ((flags & DB_RF_HAVESTRUCT) == 0) 615ea8dc4b6Seschrock rw_exit(&db->db_dnode->dn_struct_rwlock); 61613506d1eSmaybee 61713506d1eSmaybee mutex_enter(&db->db_mtx); 618ea8dc4b6Seschrock if ((flags & DB_RF_NEVERWAIT) == 0) { 619ea8dc4b6Seschrock while (db->db_state == DB_READ || 620ea8dc4b6Seschrock db->db_state == DB_FILL) { 621fa9e4066Sahrens ASSERT(db->db_state == DB_READ || 622fa9e4066Sahrens (flags & DB_RF_HAVESTRUCT) == 0); 623fa9e4066Sahrens cv_wait(&db->db_changed, &db->db_mtx); 624fa9e4066Sahrens } 625ea8dc4b6Seschrock if (db->db_state == DB_UNCACHED) 626ea8dc4b6Seschrock err = EIO; 627ea8dc4b6Seschrock } 628fa9e4066Sahrens mutex_exit(&db->db_mtx); 629fa9e4066Sahrens } 630fa9e4066Sahrens 631ea8dc4b6Seschrock ASSERT(err || havepzio || db->db_state == DB_CACHED); 632ea8dc4b6Seschrock return (err); 633fa9e4066Sahrens } 634fa9e4066Sahrens 635fa9e4066Sahrens static void 636fa9e4066Sahrens dbuf_noread(dmu_buf_impl_t *db) 637fa9e4066Sahrens { 638fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 6390a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 640fa9e4066Sahrens mutex_enter(&db->db_mtx); 641fa9e4066Sahrens while (db->db_state == DB_READ || db->db_state == DB_FILL) 642fa9e4066Sahrens cv_wait(&db->db_changed, &db->db_mtx); 643fa9e4066Sahrens if (db->db_state == DB_UNCACHED) { 644ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 645ad23a2dbSjohansen 646ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 647fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 648fa9e4066Sahrens dbuf_set_data(db, arc_buf_alloc(db->db_dnode->dn_objset->os_spa, 649ad23a2dbSjohansen db->db.db_size, db, type)); 650fa9e4066Sahrens db->db_state = DB_FILL; 65182c9918fSTim Haley } else if (db->db_state == DB_NOFILL) { 65282c9918fSTim Haley dbuf_set_data(db, NULL); 653fa9e4066Sahrens } else { 654fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_CACHED); 655fa9e4066Sahrens } 656fa9e4066Sahrens mutex_exit(&db->db_mtx); 657fa9e4066Sahrens } 658fa9e4066Sahrens 659fa9e4066Sahrens /* 660fa9e4066Sahrens * This is our just-in-time copy function. It makes a copy of 661fa9e4066Sahrens * buffers, that have been modified in a previous transaction 662fa9e4066Sahrens * group, before we modify them in the current active group. 663fa9e4066Sahrens * 664fa9e4066Sahrens * This function is used in two places: when we are dirtying a 665fa9e4066Sahrens * buffer for the first time in a txg, and when we are freeing 666fa9e4066Sahrens * a range in a dnode that includes this buffer. 667fa9e4066Sahrens * 668fa9e4066Sahrens * Note that when we are called from dbuf_free_range() we do 669fa9e4066Sahrens * not put a hold on the buffer, we just traverse the active 670fa9e4066Sahrens * dbuf list for the dnode. 671fa9e4066Sahrens */ 672fa9e4066Sahrens static void 673fa9e4066Sahrens dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg) 674fa9e4066Sahrens { 675c717a561Smaybee dbuf_dirty_record_t *dr = db->db_last_dirty; 676fa9e4066Sahrens 677fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 678fa9e4066Sahrens ASSERT(db->db.db_data != NULL); 679c717a561Smaybee ASSERT(db->db_level == 0); 680c717a561Smaybee ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT); 681fa9e4066Sahrens 6824d31c452Smaybee if (dr == NULL || 6834d31c452Smaybee (dr->dt.dl.dr_data != 6840a586ceaSMark Shellenbaum ((db->db_blkid == DMU_BONUS_BLKID) ? db->db.db_data : db->db_buf))) 685fa9e4066Sahrens return; 686fa9e4066Sahrens 687fa9e4066Sahrens /* 688c717a561Smaybee * If the last dirty record for this dbuf has not yet synced 689c717a561Smaybee * and its referencing the dbuf data, either: 690c717a561Smaybee * reset the reference to point to a new copy, 691c717a561Smaybee * or (if there a no active holders) 692c717a561Smaybee * just null out the current db_data pointer. 693fa9e4066Sahrens */ 694c717a561Smaybee ASSERT(dr->dr_txg >= txg - 2); 6950a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 696c717a561Smaybee /* Note that the data bufs here are zio_bufs */ 697c717a561Smaybee dr->dt.dl.dr_data = zio_buf_alloc(DN_MAX_BONUSLEN); 6985a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 699c717a561Smaybee bcopy(db->db.db_data, dr->dt.dl.dr_data, DN_MAX_BONUSLEN); 700c717a561Smaybee } else if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 701ea8dc4b6Seschrock int size = db->db.db_size; 702c717a561Smaybee arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 703c717a561Smaybee dr->dt.dl.dr_data = arc_buf_alloc( 704ad23a2dbSjohansen db->db_dnode->dn_objset->os_spa, size, db, type); 705c717a561Smaybee bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size); 706fa9e4066Sahrens } else { 707ea8dc4b6Seschrock dbuf_set_data(db, NULL); 708fa9e4066Sahrens } 709fa9e4066Sahrens } 710ea8dc4b6Seschrock 711fa9e4066Sahrens void 712c717a561Smaybee dbuf_unoverride(dbuf_dirty_record_t *dr) 713fa9e4066Sahrens { 714c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 715b24ab676SJeff Bonwick blkptr_t *bp = &dr->dt.dl.dr_overridden_by; 716c717a561Smaybee uint64_t txg = dr->dr_txg; 717c5c6ffa0Smaybee 718c717a561Smaybee ASSERT(MUTEX_HELD(&db->db_mtx)); 719c717a561Smaybee ASSERT(dr->dt.dl.dr_override_state != DR_IN_DMU_SYNC); 720c717a561Smaybee ASSERT(db->db_level == 0); 721c717a561Smaybee 7220a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID || 723c717a561Smaybee dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN) 724c717a561Smaybee return; 725c717a561Smaybee 726b24ab676SJeff Bonwick ASSERT(db->db_data_pending != dr); 727b24ab676SJeff Bonwick 728fa9e4066Sahrens /* free this block */ 729b24ab676SJeff Bonwick if (!BP_IS_HOLE(bp)) 730*3f9d6ad7SLin Ling zio_free(db->db_dnode->dn_objset->os_spa, txg, bp); 731b24ab676SJeff Bonwick 732c717a561Smaybee dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 7336b4acc8bSahrens /* 7346b4acc8bSahrens * Release the already-written buffer, so we leave it in 7356b4acc8bSahrens * a consistent dirty state. Note that all callers are 7366b4acc8bSahrens * modifying the buffer, so they will immediately do 7376b4acc8bSahrens * another (redundant) arc_release(). Therefore, leave 7386b4acc8bSahrens * the buf thawed to save the effort of freezing & 7396b4acc8bSahrens * immediately re-thawing it. 7406b4acc8bSahrens */ 741c717a561Smaybee arc_release(dr->dt.dl.dr_data, db); 742fa9e4066Sahrens } 743fa9e4066Sahrens 744cdb0ab79Smaybee /* 745cdb0ab79Smaybee * Evict (if its unreferenced) or clear (if its referenced) any level-0 746cdb0ab79Smaybee * data blocks in the free range, so that any future readers will find 747cdb0ab79Smaybee * empty blocks. Also, if we happen accross any level-1 dbufs in the 748cdb0ab79Smaybee * range that have not already been marked dirty, mark them dirty so 749cdb0ab79Smaybee * they stay in memory. 750cdb0ab79Smaybee */ 751fa9e4066Sahrens void 752cdb0ab79Smaybee dbuf_free_range(dnode_t *dn, uint64_t start, uint64_t end, dmu_tx_t *tx) 753fa9e4066Sahrens { 754fa9e4066Sahrens dmu_buf_impl_t *db, *db_next; 755fa9e4066Sahrens uint64_t txg = tx->tx_txg; 756cdb0ab79Smaybee int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 757cdb0ab79Smaybee uint64_t first_l1 = start >> epbs; 758cdb0ab79Smaybee uint64_t last_l1 = end >> epbs; 759fa9e4066Sahrens 7600a586ceaSMark Shellenbaum if (end > dn->dn_maxblkid && (end != DMU_SPILL_BLKID)) { 761cdb0ab79Smaybee end = dn->dn_maxblkid; 762cdb0ab79Smaybee last_l1 = end >> epbs; 763cdb0ab79Smaybee } 764cdb0ab79Smaybee dprintf_dnode(dn, "start=%llu end=%llu\n", start, end); 765fa9e4066Sahrens mutex_enter(&dn->dn_dbufs_mtx); 766fa9e4066Sahrens for (db = list_head(&dn->dn_dbufs); db; db = db_next) { 767fa9e4066Sahrens db_next = list_next(&dn->dn_dbufs, db); 7680a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 769cdb0ab79Smaybee 770cdb0ab79Smaybee if (db->db_level == 1 && 771cdb0ab79Smaybee db->db_blkid >= first_l1 && db->db_blkid <= last_l1) { 772cdb0ab79Smaybee mutex_enter(&db->db_mtx); 773cdb0ab79Smaybee if (db->db_last_dirty && 774cdb0ab79Smaybee db->db_last_dirty->dr_txg < txg) { 775cdb0ab79Smaybee dbuf_add_ref(db, FTAG); 776cdb0ab79Smaybee mutex_exit(&db->db_mtx); 777cdb0ab79Smaybee dbuf_will_dirty(db, tx); 778cdb0ab79Smaybee dbuf_rele(db, FTAG); 779cdb0ab79Smaybee } else { 780cdb0ab79Smaybee mutex_exit(&db->db_mtx); 781cdb0ab79Smaybee } 782cdb0ab79Smaybee } 783cdb0ab79Smaybee 784ea8dc4b6Seschrock if (db->db_level != 0) 785fa9e4066Sahrens continue; 786fa9e4066Sahrens dprintf_dbuf(db, "found buf %s\n", ""); 787cdb0ab79Smaybee if (db->db_blkid < start || db->db_blkid > end) 788fa9e4066Sahrens continue; 789fa9e4066Sahrens 790fa9e4066Sahrens /* found a level 0 buffer in the range */ 791fa9e4066Sahrens if (dbuf_undirty(db, tx)) 792fa9e4066Sahrens continue; 793fa9e4066Sahrens 794fa9e4066Sahrens mutex_enter(&db->db_mtx); 795ea8dc4b6Seschrock if (db->db_state == DB_UNCACHED || 79682c9918fSTim Haley db->db_state == DB_NOFILL || 797ea8dc4b6Seschrock db->db_state == DB_EVICTING) { 798fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 799fa9e4066Sahrens mutex_exit(&db->db_mtx); 800fa9e4066Sahrens continue; 801fa9e4066Sahrens } 802c543ec06Sahrens if (db->db_state == DB_READ || db->db_state == DB_FILL) { 803c543ec06Sahrens /* will be handled in dbuf_read_done or dbuf_rele */ 804c717a561Smaybee db->db_freed_in_flight = TRUE; 805fa9e4066Sahrens mutex_exit(&db->db_mtx); 806fa9e4066Sahrens continue; 807fa9e4066Sahrens } 808ea8dc4b6Seschrock if (refcount_count(&db->db_holds) == 0) { 809ea8dc4b6Seschrock ASSERT(db->db_buf); 810ea8dc4b6Seschrock dbuf_clear(db); 811ea8dc4b6Seschrock continue; 812ea8dc4b6Seschrock } 813c717a561Smaybee /* The dbuf is referenced */ 814fa9e4066Sahrens 815c717a561Smaybee if (db->db_last_dirty != NULL) { 816c717a561Smaybee dbuf_dirty_record_t *dr = db->db_last_dirty; 817c717a561Smaybee 818c717a561Smaybee if (dr->dr_txg == txg) { 819ea8dc4b6Seschrock /* 820c717a561Smaybee * This buffer is "in-use", re-adjust the file 821c717a561Smaybee * size to reflect that this buffer may 822c717a561Smaybee * contain new data when we sync. 823ea8dc4b6Seschrock */ 82406e0070dSMark Shellenbaum if (db->db_blkid != DMU_SPILL_BLKID && 82506e0070dSMark Shellenbaum db->db_blkid > dn->dn_maxblkid) 82644eda4d7Smaybee dn->dn_maxblkid = db->db_blkid; 827c717a561Smaybee dbuf_unoverride(dr); 828c717a561Smaybee } else { 829c717a561Smaybee /* 830c717a561Smaybee * This dbuf is not dirty in the open context. 831c717a561Smaybee * Either uncache it (if its not referenced in 832c717a561Smaybee * the open context) or reset its contents to 833c717a561Smaybee * empty. 834c717a561Smaybee */ 835c717a561Smaybee dbuf_fix_old_data(db, txg); 83644eda4d7Smaybee } 837c717a561Smaybee } 838c717a561Smaybee /* clear the contents if its cached */ 839ea8dc4b6Seschrock if (db->db_state == DB_CACHED) { 840ea8dc4b6Seschrock ASSERT(db->db.db_data != NULL); 841fa9e4066Sahrens arc_release(db->db_buf, db); 842fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 8436b4acc8bSahrens arc_buf_freeze(db->db_buf); 844fa9e4066Sahrens } 845ea8dc4b6Seschrock 846fa9e4066Sahrens mutex_exit(&db->db_mtx); 847fa9e4066Sahrens } 848fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 849fa9e4066Sahrens } 850fa9e4066Sahrens 851fa9e4066Sahrens static int 8521934e92fSmaybee dbuf_block_freeable(dmu_buf_impl_t *db) 853fa9e4066Sahrens { 854fa9e4066Sahrens dsl_dataset_t *ds = db->db_objset->os_dsl_dataset; 855fa9e4066Sahrens uint64_t birth_txg = 0; 856fa9e4066Sahrens 857fa9e4066Sahrens /* 858fa9e4066Sahrens * We don't need any locking to protect db_blkptr: 859c717a561Smaybee * If it's syncing, then db_last_dirty will be set 860c717a561Smaybee * so we'll ignore db_blkptr. 861fa9e4066Sahrens */ 862c717a561Smaybee ASSERT(MUTEX_HELD(&db->db_mtx)); 863c717a561Smaybee if (db->db_last_dirty) 864c717a561Smaybee birth_txg = db->db_last_dirty->dr_txg; 865fa9e4066Sahrens else if (db->db_blkptr) 866fa9e4066Sahrens birth_txg = db->db_blkptr->blk_birth; 867fa9e4066Sahrens 8681934e92fSmaybee /* If we don't exist or are in a snapshot, we can't be freed */ 869fa9e4066Sahrens if (birth_txg) 8701934e92fSmaybee return (ds == NULL || 8711934e92fSmaybee dsl_dataset_block_freeable(ds, birth_txg)); 872fa9e4066Sahrens else 8731934e92fSmaybee return (FALSE); 874fa9e4066Sahrens } 875fa9e4066Sahrens 876fa9e4066Sahrens void 877fa9e4066Sahrens dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx) 878fa9e4066Sahrens { 879fa9e4066Sahrens arc_buf_t *buf, *obuf; 880fa9e4066Sahrens int osize = db->db.db_size; 881ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 882fa9e4066Sahrens 8830a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 884ea8dc4b6Seschrock 885fa9e4066Sahrens /* XXX does *this* func really need the lock? */ 886fa9e4066Sahrens ASSERT(RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)); 887fa9e4066Sahrens 888fa9e4066Sahrens /* 889fa9e4066Sahrens * This call to dbuf_will_dirty() with the dn_struct_rwlock held 890fa9e4066Sahrens * is OK, because there can be no other references to the db 891fa9e4066Sahrens * when we are changing its size, so no concurrent DB_FILL can 892fa9e4066Sahrens * be happening. 893fa9e4066Sahrens */ 894ea8dc4b6Seschrock /* 895ea8dc4b6Seschrock * XXX we should be doing a dbuf_read, checking the return 896ea8dc4b6Seschrock * value and returning that up to our callers 897ea8dc4b6Seschrock */ 898fa9e4066Sahrens dbuf_will_dirty(db, tx); 899fa9e4066Sahrens 900fa9e4066Sahrens /* create the data buffer for the new block */ 901ad23a2dbSjohansen buf = arc_buf_alloc(db->db_dnode->dn_objset->os_spa, size, db, type); 902fa9e4066Sahrens 903fa9e4066Sahrens /* copy old block data to the new block */ 904fa9e4066Sahrens obuf = db->db_buf; 905f65e61c0Sahrens bcopy(obuf->b_data, buf->b_data, MIN(osize, size)); 906fa9e4066Sahrens /* zero the remainder */ 907f65e61c0Sahrens if (size > osize) 908fa9e4066Sahrens bzero((uint8_t *)buf->b_data + osize, size - osize); 909fa9e4066Sahrens 910fa9e4066Sahrens mutex_enter(&db->db_mtx); 911fa9e4066Sahrens dbuf_set_data(db, buf); 912ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(obuf, db) == 1); 913fa9e4066Sahrens db->db.db_size = size; 914fa9e4066Sahrens 915c717a561Smaybee if (db->db_level == 0) { 916c717a561Smaybee ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg); 917c717a561Smaybee db->db_last_dirty->dt.dl.dr_data = buf; 918c717a561Smaybee } 919fa9e4066Sahrens mutex_exit(&db->db_mtx); 920fa9e4066Sahrens 921fa9e4066Sahrens dnode_willuse_space(db->db_dnode, size-osize, tx); 922fa9e4066Sahrens } 923fa9e4066Sahrens 924*3f9d6ad7SLin Ling void 925*3f9d6ad7SLin Ling dbuf_release_bp(dmu_buf_impl_t *db) 926*3f9d6ad7SLin Ling { 927*3f9d6ad7SLin Ling objset_t *os = db->db_dnode->dn_objset; 928*3f9d6ad7SLin Ling zbookmark_t zb; 929*3f9d6ad7SLin Ling 930*3f9d6ad7SLin Ling ASSERT(dsl_pool_sync_context(dmu_objset_pool(os))); 931*3f9d6ad7SLin Ling ASSERT(arc_released(os->os_phys_buf) || 932*3f9d6ad7SLin Ling list_link_active(&os->os_dsl_dataset->ds_synced_link)); 933*3f9d6ad7SLin Ling ASSERT(db->db_parent == NULL || arc_released(db->db_parent->db_buf)); 934*3f9d6ad7SLin Ling 935*3f9d6ad7SLin Ling zb.zb_objset = os->os_dsl_dataset ? 936*3f9d6ad7SLin Ling os->os_dsl_dataset->ds_object : 0; 937*3f9d6ad7SLin Ling zb.zb_object = db->db.db_object; 938*3f9d6ad7SLin Ling zb.zb_level = db->db_level; 939*3f9d6ad7SLin Ling zb.zb_blkid = db->db_blkid; 940*3f9d6ad7SLin Ling (void) arc_release_bp(db->db_buf, db, 941*3f9d6ad7SLin Ling db->db_blkptr, os->os_spa, &zb); 942*3f9d6ad7SLin Ling } 943*3f9d6ad7SLin Ling 944c717a561Smaybee dbuf_dirty_record_t * 945fa9e4066Sahrens dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 946fa9e4066Sahrens { 947fa9e4066Sahrens dnode_t *dn = db->db_dnode; 948503ad85cSMatthew Ahrens objset_t *os = dn->dn_objset; 949c717a561Smaybee dbuf_dirty_record_t **drp, *dr; 950fa9e4066Sahrens int drop_struct_lock = FALSE; 951d3469faaSMark Maybee boolean_t do_free_accounting = B_FALSE; 952fa9e4066Sahrens int txgoff = tx->tx_txg & TXG_MASK; 953fa9e4066Sahrens 954fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 955fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 9569c9dc39aSek110237 DMU_TX_DIRTY_BUF(tx, db); 957fa9e4066Sahrens 958fa9e4066Sahrens /* 959fa9e4066Sahrens * Shouldn't dirty a regular buffer in syncing context. Private 960fa9e4066Sahrens * objects may be dirtied in syncing context, but only if they 961fa9e4066Sahrens * were already pre-dirtied in open context. 962fa9e4066Sahrens */ 963c717a561Smaybee ASSERT(!dmu_tx_is_syncing(tx) || 964c717a561Smaybee BP_IS_HOLE(dn->dn_objset->os_rootbp) || 96514843421SMatthew Ahrens DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 96614843421SMatthew Ahrens dn->dn_objset->os_dsl_dataset == NULL); 967fa9e4066Sahrens /* 968fa9e4066Sahrens * We make this assert for private objects as well, but after we 969fa9e4066Sahrens * check if we're already dirty. They are allowed to re-dirty 970fa9e4066Sahrens * in syncing context. 971fa9e4066Sahrens */ 972ea8dc4b6Seschrock ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 973c717a561Smaybee dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 974fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 975fa9e4066Sahrens 976fa9e4066Sahrens mutex_enter(&db->db_mtx); 977fa9e4066Sahrens /* 978c717a561Smaybee * XXX make this true for indirects too? The problem is that 979c717a561Smaybee * transactions created with dmu_tx_create_assigned() from 980c717a561Smaybee * syncing context don't bother holding ahead. 981fa9e4066Sahrens */ 982c717a561Smaybee ASSERT(db->db_level != 0 || 98382c9918fSTim Haley db->db_state == DB_CACHED || db->db_state == DB_FILL || 98482c9918fSTim Haley db->db_state == DB_NOFILL); 985fa9e4066Sahrens 986fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 987fa9e4066Sahrens /* 988fa9e4066Sahrens * Don't set dirtyctx to SYNC if we're just modifying this as we 989fa9e4066Sahrens * initialize the objset. 990fa9e4066Sahrens */ 991fa9e4066Sahrens if (dn->dn_dirtyctx == DN_UNDIRTIED && 992c717a561Smaybee !BP_IS_HOLE(dn->dn_objset->os_rootbp)) { 993fa9e4066Sahrens dn->dn_dirtyctx = 994fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN); 995fa9e4066Sahrens ASSERT(dn->dn_dirtyctx_firstset == NULL); 996fa9e4066Sahrens dn->dn_dirtyctx_firstset = kmem_alloc(1, KM_SLEEP); 997fa9e4066Sahrens } 998fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 999fa9e4066Sahrens 10000a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) 10010a586ceaSMark Shellenbaum dn->dn_have_spill = B_TRUE; 10020a586ceaSMark Shellenbaum 1003fa9e4066Sahrens /* 1004fa9e4066Sahrens * If this buffer is already dirty, we're done. 1005fa9e4066Sahrens */ 1006c717a561Smaybee drp = &db->db_last_dirty; 1007c717a561Smaybee ASSERT(*drp == NULL || (*drp)->dr_txg <= tx->tx_txg || 1008c717a561Smaybee db->db.db_object == DMU_META_DNODE_OBJECT); 10097e2186e3Sbonwick while ((dr = *drp) != NULL && dr->dr_txg > tx->tx_txg) 10107e2186e3Sbonwick drp = &dr->dr_next; 10117e2186e3Sbonwick if (dr && dr->dr_txg == tx->tx_txg) { 10120a586ceaSMark Shellenbaum if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID) { 1013c717a561Smaybee /* 1014c717a561Smaybee * If this buffer has already been written out, 1015c717a561Smaybee * we now need to reset its state. 1016c717a561Smaybee */ 10177e2186e3Sbonwick dbuf_unoverride(dr); 1018b24ab676SJeff Bonwick if (db->db.db_object != DMU_META_DNODE_OBJECT && 1019b24ab676SJeff Bonwick db->db_state != DB_NOFILL) 10206b4acc8bSahrens arc_buf_thaw(db->db_buf); 1021c717a561Smaybee } 1022fa9e4066Sahrens mutex_exit(&db->db_mtx); 10237e2186e3Sbonwick return (dr); 1024fa9e4066Sahrens } 1025fa9e4066Sahrens 1026fa9e4066Sahrens /* 1027fa9e4066Sahrens * Only valid if not already dirty. 1028fa9e4066Sahrens */ 102914843421SMatthew Ahrens ASSERT(dn->dn_object == 0 || 103014843421SMatthew Ahrens dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 1031fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 1032fa9e4066Sahrens 1033fa9e4066Sahrens ASSERT3U(dn->dn_nlevels, >, db->db_level); 1034fa9e4066Sahrens ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) || 1035fa9e4066Sahrens dn->dn_phys->dn_nlevels > db->db_level || 1036fa9e4066Sahrens dn->dn_next_nlevels[txgoff] > db->db_level || 1037fa9e4066Sahrens dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level || 1038fa9e4066Sahrens dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level); 1039fa9e4066Sahrens 1040fa9e4066Sahrens /* 1041fa9e4066Sahrens * We should only be dirtying in syncing context if it's the 104214843421SMatthew Ahrens * mos or we're initializing the os or it's a special object. 104314843421SMatthew Ahrens * However, we are allowed to dirty in syncing context provided 104414843421SMatthew Ahrens * we already dirtied it in open context. Hence we must make 104514843421SMatthew Ahrens * this assertion only if we're not already dirty. 1046fa9e4066Sahrens */ 104714843421SMatthew Ahrens ASSERT(!dmu_tx_is_syncing(tx) || DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 104814843421SMatthew Ahrens os->os_dsl_dataset == NULL || BP_IS_HOLE(os->os_rootbp)); 1049fa9e4066Sahrens ASSERT(db->db.db_size != 0); 1050fa9e4066Sahrens 1051fa9e4066Sahrens dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1052fa9e4066Sahrens 10530a586ceaSMark Shellenbaum if (db->db_blkid != DMU_BONUS_BLKID) { 10541934e92fSmaybee /* 10551934e92fSmaybee * Update the accounting. 1056d3469faaSMark Maybee * Note: we delay "free accounting" until after we drop 1057d3469faaSMark Maybee * the db_mtx. This keeps us from grabbing other locks 1058b24ab676SJeff Bonwick * (and possibly deadlocking) in bp_get_dsize() while 1059d3469faaSMark Maybee * also holding the db_mtx. 10601934e92fSmaybee */ 10611934e92fSmaybee dnode_willuse_space(dn, db->db.db_size, tx); 1062d3469faaSMark Maybee do_free_accounting = dbuf_block_freeable(db); 10631934e92fSmaybee } 10641934e92fSmaybee 1065ea8dc4b6Seschrock /* 1066ea8dc4b6Seschrock * If this buffer is dirty in an old transaction group we need 1067ea8dc4b6Seschrock * to make a copy of it so that the changes we make in this 1068ea8dc4b6Seschrock * transaction group won't leak out when we sync the older txg. 1069ea8dc4b6Seschrock */ 1070c717a561Smaybee dr = kmem_zalloc(sizeof (dbuf_dirty_record_t), KM_SLEEP); 1071c717a561Smaybee if (db->db_level == 0) { 1072c717a561Smaybee void *data_old = db->db_buf; 1073c717a561Smaybee 107482c9918fSTim Haley if (db->db_state != DB_NOFILL) { 10750a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 1076c717a561Smaybee dbuf_fix_old_data(db, tx->tx_txg); 1077c717a561Smaybee data_old = db->db.db_data; 1078c717a561Smaybee } else if (db->db.db_object != DMU_META_DNODE_OBJECT) { 1079fa9e4066Sahrens /* 108082c9918fSTim Haley * Release the data buffer from the cache so 108182c9918fSTim Haley * that we can modify it without impacting 108282c9918fSTim Haley * possible other users of this cached data 108382c9918fSTim Haley * block. Note that indirect blocks and 108482c9918fSTim Haley * private objects are not released until the 108582c9918fSTim Haley * syncing state (since they are only modified 108682c9918fSTim Haley * then). 1087fa9e4066Sahrens */ 1088fa9e4066Sahrens arc_release(db->db_buf, db); 1089fa9e4066Sahrens dbuf_fix_old_data(db, tx->tx_txg); 1090c717a561Smaybee data_old = db->db_buf; 1091fa9e4066Sahrens } 1092c717a561Smaybee ASSERT(data_old != NULL); 109382c9918fSTim Haley } 1094c717a561Smaybee dr->dt.dl.dr_data = data_old; 1095c717a561Smaybee } else { 1096c717a561Smaybee mutex_init(&dr->dt.di.dr_mtx, NULL, MUTEX_DEFAULT, NULL); 1097c717a561Smaybee list_create(&dr->dt.di.dr_children, 1098c717a561Smaybee sizeof (dbuf_dirty_record_t), 1099c717a561Smaybee offsetof(dbuf_dirty_record_t, dr_dirty_node)); 1100fa9e4066Sahrens } 1101c717a561Smaybee dr->dr_dbuf = db; 1102c717a561Smaybee dr->dr_txg = tx->tx_txg; 1103c717a561Smaybee dr->dr_next = *drp; 1104c717a561Smaybee *drp = dr; 1105fa9e4066Sahrens 1106fa9e4066Sahrens /* 1107fa9e4066Sahrens * We could have been freed_in_flight between the dbuf_noread 1108fa9e4066Sahrens * and dbuf_dirty. We win, as though the dbuf_noread() had 1109fa9e4066Sahrens * happened after the free. 1110fa9e4066Sahrens */ 11110a586ceaSMark Shellenbaum if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 11120a586ceaSMark Shellenbaum db->db_blkid != DMU_SPILL_BLKID) { 1113c717a561Smaybee mutex_enter(&dn->dn_mtx); 1114fa9e4066Sahrens dnode_clear_range(dn, db->db_blkid, 1, tx); 1115fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1116c717a561Smaybee db->db_freed_in_flight = FALSE; 1117c717a561Smaybee } 1118fa9e4066Sahrens 1119fa9e4066Sahrens /* 1120fa9e4066Sahrens * This buffer is now part of this txg 1121fa9e4066Sahrens */ 1122fa9e4066Sahrens dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg); 1123fa9e4066Sahrens db->db_dirtycnt += 1; 1124fa9e4066Sahrens ASSERT3U(db->db_dirtycnt, <=, 3); 1125fa9e4066Sahrens 1126fa9e4066Sahrens mutex_exit(&db->db_mtx); 1127fa9e4066Sahrens 11280a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID || 11290a586ceaSMark Shellenbaum db->db_blkid == DMU_SPILL_BLKID) { 1130c717a561Smaybee mutex_enter(&dn->dn_mtx); 1131c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 1132c717a561Smaybee list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1133c717a561Smaybee mutex_exit(&dn->dn_mtx); 1134fa9e4066Sahrens dnode_setdirty(dn, tx); 1135c717a561Smaybee return (dr); 1136d3469faaSMark Maybee } else if (do_free_accounting) { 1137d3469faaSMark Maybee blkptr_t *bp = db->db_blkptr; 1138d3469faaSMark Maybee int64_t willfree = (bp && !BP_IS_HOLE(bp)) ? 1139b24ab676SJeff Bonwick bp_get_dsize(os->os_spa, bp) : db->db.db_size; 1140d3469faaSMark Maybee /* 1141d3469faaSMark Maybee * This is only a guess -- if the dbuf is dirty 1142d3469faaSMark Maybee * in a previous txg, we don't know how much 1143d3469faaSMark Maybee * space it will use on disk yet. We should 1144d3469faaSMark Maybee * really have the struct_rwlock to access 1145d3469faaSMark Maybee * db_blkptr, but since this is just a guess, 1146d3469faaSMark Maybee * it's OK if we get an odd answer. 1147d3469faaSMark Maybee */ 1148d3469faaSMark Maybee dnode_willuse_space(dn, -willfree, tx); 1149fa9e4066Sahrens } 1150fa9e4066Sahrens 1151fa9e4066Sahrens if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 1152fa9e4066Sahrens rw_enter(&dn->dn_struct_rwlock, RW_READER); 1153fa9e4066Sahrens drop_struct_lock = TRUE; 1154fa9e4066Sahrens } 1155fa9e4066Sahrens 11568346f03fSJonathan W Adams if (db->db_level == 0) { 11578346f03fSJonathan W Adams dnode_new_blkid(dn, db->db_blkid, tx, drop_struct_lock); 11588346f03fSJonathan W Adams ASSERT(dn->dn_maxblkid >= db->db_blkid); 11598346f03fSJonathan W Adams } 11608346f03fSJonathan W Adams 116144eda4d7Smaybee if (db->db_level+1 < dn->dn_nlevels) { 1162c717a561Smaybee dmu_buf_impl_t *parent = db->db_parent; 1163c717a561Smaybee dbuf_dirty_record_t *di; 1164c717a561Smaybee int parent_held = FALSE; 1165c717a561Smaybee 1166c717a561Smaybee if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) { 1167fa9e4066Sahrens int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1168c717a561Smaybee 1169fa9e4066Sahrens parent = dbuf_hold_level(dn, db->db_level+1, 1170fa9e4066Sahrens db->db_blkid >> epbs, FTAG); 1171c717a561Smaybee parent_held = TRUE; 1172c717a561Smaybee } 1173fa9e4066Sahrens if (drop_struct_lock) 1174fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 1175c717a561Smaybee ASSERT3U(db->db_level+1, ==, parent->db_level); 1176c717a561Smaybee di = dbuf_dirty(parent, tx); 1177c717a561Smaybee if (parent_held) 1178ea8dc4b6Seschrock dbuf_rele(parent, FTAG); 1179c717a561Smaybee 1180c717a561Smaybee mutex_enter(&db->db_mtx); 1181c717a561Smaybee /* possible race with dbuf_undirty() */ 1182c717a561Smaybee if (db->db_last_dirty == dr || 1183c717a561Smaybee dn->dn_object == DMU_META_DNODE_OBJECT) { 1184c717a561Smaybee mutex_enter(&di->dt.di.dr_mtx); 1185c717a561Smaybee ASSERT3U(di->dr_txg, ==, tx->tx_txg); 1186c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 1187c717a561Smaybee list_insert_tail(&di->dt.di.dr_children, dr); 1188c717a561Smaybee mutex_exit(&di->dt.di.dr_mtx); 1189c717a561Smaybee dr->dr_parent = di; 1190c717a561Smaybee } 1191c717a561Smaybee mutex_exit(&db->db_mtx); 1192fa9e4066Sahrens } else { 1193c717a561Smaybee ASSERT(db->db_level+1 == dn->dn_nlevels); 1194c717a561Smaybee ASSERT(db->db_blkid < dn->dn_nblkptr); 1195c717a561Smaybee ASSERT(db->db_parent == NULL || 1196c717a561Smaybee db->db_parent == db->db_dnode->dn_dbuf); 1197c717a561Smaybee mutex_enter(&dn->dn_mtx); 1198c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 1199c717a561Smaybee list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1200c717a561Smaybee mutex_exit(&dn->dn_mtx); 1201fa9e4066Sahrens if (drop_struct_lock) 1202fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 1203fa9e4066Sahrens } 1204fa9e4066Sahrens 1205fa9e4066Sahrens dnode_setdirty(dn, tx); 1206c717a561Smaybee return (dr); 1207fa9e4066Sahrens } 1208fa9e4066Sahrens 1209fa9e4066Sahrens static int 1210fa9e4066Sahrens dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1211fa9e4066Sahrens { 1212fa9e4066Sahrens dnode_t *dn = db->db_dnode; 1213c717a561Smaybee uint64_t txg = tx->tx_txg; 121417f17c2dSbonwick dbuf_dirty_record_t *dr, **drp; 1215fa9e4066Sahrens 1216c717a561Smaybee ASSERT(txg != 0); 12170a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1218fa9e4066Sahrens 1219fa9e4066Sahrens mutex_enter(&db->db_mtx); 1220fa9e4066Sahrens /* 1221fa9e4066Sahrens * If this buffer is not dirty, we're done. 1222fa9e4066Sahrens */ 122317f17c2dSbonwick for (drp = &db->db_last_dirty; (dr = *drp) != NULL; drp = &dr->dr_next) 1224c717a561Smaybee if (dr->dr_txg <= txg) 1225c717a561Smaybee break; 1226c717a561Smaybee if (dr == NULL || dr->dr_txg < txg) { 1227fa9e4066Sahrens mutex_exit(&db->db_mtx); 1228fa9e4066Sahrens return (0); 1229fa9e4066Sahrens } 1230c717a561Smaybee ASSERT(dr->dr_txg == txg); 1231b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 1232fa9e4066Sahrens 1233fa9e4066Sahrens /* 1234fa9e4066Sahrens * If this buffer is currently held, we cannot undirty 1235fa9e4066Sahrens * it, since one of the current holders may be in the 1236fa9e4066Sahrens * middle of an update. Note that users of dbuf_undirty() 1237fa9e4066Sahrens * should not place a hold on the dbuf before the call. 1238fa9e4066Sahrens */ 1239fa9e4066Sahrens if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 1240fa9e4066Sahrens mutex_exit(&db->db_mtx); 124144eda4d7Smaybee /* Make sure we don't toss this buffer at sync phase */ 1242fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 1243fa9e4066Sahrens dnode_clear_range(dn, db->db_blkid, 1, tx); 1244fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1245fa9e4066Sahrens return (0); 1246fa9e4066Sahrens } 1247fa9e4066Sahrens 1248fa9e4066Sahrens dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1249fa9e4066Sahrens 1250fa9e4066Sahrens ASSERT(db->db.db_size != 0); 1251fa9e4066Sahrens 1252fa9e4066Sahrens /* XXX would be nice to fix up dn_towrite_space[] */ 1253fa9e4066Sahrens 125417f17c2dSbonwick *drp = dr->dr_next; 1255c717a561Smaybee 1256c717a561Smaybee if (dr->dr_parent) { 1257c717a561Smaybee mutex_enter(&dr->dr_parent->dt.di.dr_mtx); 1258c717a561Smaybee list_remove(&dr->dr_parent->dt.di.dr_children, dr); 1259c717a561Smaybee mutex_exit(&dr->dr_parent->dt.di.dr_mtx); 1260c717a561Smaybee } else if (db->db_level+1 == dn->dn_nlevels) { 1261cdb0ab79Smaybee ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf); 1262fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 1263c717a561Smaybee list_remove(&dn->dn_dirty_records[txg & TXG_MASK], dr); 1264fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1265c717a561Smaybee } 1266c717a561Smaybee 1267c717a561Smaybee if (db->db_level == 0) { 126882c9918fSTim Haley if (db->db_state != DB_NOFILL) { 1269c717a561Smaybee dbuf_unoverride(dr); 1270c717a561Smaybee 1271c717a561Smaybee ASSERT(db->db_buf != NULL); 1272c717a561Smaybee ASSERT(dr->dt.dl.dr_data != NULL); 1273c717a561Smaybee if (dr->dt.dl.dr_data != db->db_buf) 127482c9918fSTim Haley VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, 127582c9918fSTim Haley db) == 1); 127682c9918fSTim Haley } 1277c717a561Smaybee } else { 1278c717a561Smaybee ASSERT(db->db_buf != NULL); 1279c717a561Smaybee ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 1280c25056deSgw25295 mutex_destroy(&dr->dt.di.dr_mtx); 1281c25056deSgw25295 list_destroy(&dr->dt.di.dr_children); 1282c717a561Smaybee } 1283c717a561Smaybee kmem_free(dr, sizeof (dbuf_dirty_record_t)); 1284fa9e4066Sahrens 1285fa9e4066Sahrens ASSERT(db->db_dirtycnt > 0); 1286fa9e4066Sahrens db->db_dirtycnt -= 1; 1287fa9e4066Sahrens 1288c717a561Smaybee if (refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) { 1289ea8dc4b6Seschrock arc_buf_t *buf = db->db_buf; 1290fa9e4066Sahrens 1291b24ab676SJeff Bonwick ASSERT(db->db_state == DB_NOFILL || arc_released(buf)); 1292ea8dc4b6Seschrock dbuf_set_data(db, NULL); 1293ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(buf, db) == 1); 1294fa9e4066Sahrens dbuf_evict(db); 1295fa9e4066Sahrens return (1); 1296fa9e4066Sahrens } 1297fa9e4066Sahrens 1298fa9e4066Sahrens mutex_exit(&db->db_mtx); 1299fa9e4066Sahrens return (0); 1300fa9e4066Sahrens } 1301fa9e4066Sahrens 1302fa9e4066Sahrens #pragma weak dmu_buf_will_dirty = dbuf_will_dirty 1303fa9e4066Sahrens void 1304fa9e4066Sahrens dbuf_will_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1305fa9e4066Sahrens { 13061ab7f2deSmaybee int rf = DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH; 1307fa9e4066Sahrens 1308fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 1309fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 1310fa9e4066Sahrens 1311fa9e4066Sahrens if (RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)) 1312fa9e4066Sahrens rf |= DB_RF_HAVESTRUCT; 1313ea8dc4b6Seschrock (void) dbuf_read(db, NULL, rf); 1314c717a561Smaybee (void) dbuf_dirty(db, tx); 1315fa9e4066Sahrens } 1316fa9e4066Sahrens 1317fa9e4066Sahrens void 131882c9918fSTim Haley dmu_buf_will_not_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 131982c9918fSTim Haley { 132082c9918fSTim Haley dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 132182c9918fSTim Haley 132282c9918fSTim Haley db->db_state = DB_NOFILL; 132382c9918fSTim Haley 132482c9918fSTim Haley dmu_buf_will_fill(db_fake, tx); 132582c9918fSTim Haley } 132682c9918fSTim Haley 132782c9918fSTim Haley void 1328ea8dc4b6Seschrock dmu_buf_will_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 1329fa9e4066Sahrens { 1330ea8dc4b6Seschrock dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1331ea8dc4b6Seschrock 13320a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1333fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 1334fa9e4066Sahrens ASSERT(db->db_level == 0); 1335fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 1336fa9e4066Sahrens 1337ea8dc4b6Seschrock ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT || 1338fa9e4066Sahrens dmu_tx_private_ok(tx)); 1339fa9e4066Sahrens 1340fa9e4066Sahrens dbuf_noread(db); 1341c717a561Smaybee (void) dbuf_dirty(db, tx); 1342fa9e4066Sahrens } 1343fa9e4066Sahrens 1344fa9e4066Sahrens #pragma weak dmu_buf_fill_done = dbuf_fill_done 1345fa9e4066Sahrens /* ARGSUSED */ 1346fa9e4066Sahrens void 1347fa9e4066Sahrens dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx) 1348fa9e4066Sahrens { 1349fa9e4066Sahrens mutex_enter(&db->db_mtx); 13509c9dc39aSek110237 DBUF_VERIFY(db); 1351fa9e4066Sahrens 1352fa9e4066Sahrens if (db->db_state == DB_FILL) { 1353c717a561Smaybee if (db->db_level == 0 && db->db_freed_in_flight) { 13540a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1355fa9e4066Sahrens /* we were freed while filling */ 1356fa9e4066Sahrens /* XXX dbuf_undirty? */ 1357fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 1358c717a561Smaybee db->db_freed_in_flight = FALSE; 1359fa9e4066Sahrens } 1360fa9e4066Sahrens db->db_state = DB_CACHED; 1361fa9e4066Sahrens cv_broadcast(&db->db_changed); 1362fa9e4066Sahrens } 1363fa9e4066Sahrens mutex_exit(&db->db_mtx); 1364fa9e4066Sahrens } 1365fa9e4066Sahrens 1366ea8dc4b6Seschrock /* 13672fdbea25SAleksandr Guzovskiy * Directly assign a provided arc buf to a given dbuf if it's not referenced 13682fdbea25SAleksandr Guzovskiy * by anybody except our caller. Otherwise copy arcbuf's contents to dbuf. 13692fdbea25SAleksandr Guzovskiy */ 13702fdbea25SAleksandr Guzovskiy void 13712fdbea25SAleksandr Guzovskiy dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx) 13722fdbea25SAleksandr Guzovskiy { 13732fdbea25SAleksandr Guzovskiy ASSERT(!refcount_is_zero(&db->db_holds)); 13742fdbea25SAleksandr Guzovskiy ASSERT(db->db_dnode->dn_object != DMU_META_DNODE_OBJECT); 13750a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 13762fdbea25SAleksandr Guzovskiy ASSERT(db->db_level == 0); 13772fdbea25SAleksandr Guzovskiy ASSERT(DBUF_GET_BUFC_TYPE(db) == ARC_BUFC_DATA); 13782fdbea25SAleksandr Guzovskiy ASSERT(buf != NULL); 13792fdbea25SAleksandr Guzovskiy ASSERT(arc_buf_size(buf) == db->db.db_size); 13802fdbea25SAleksandr Guzovskiy ASSERT(tx->tx_txg != 0); 13812fdbea25SAleksandr Guzovskiy 13822fdbea25SAleksandr Guzovskiy arc_return_buf(buf, db); 13832fdbea25SAleksandr Guzovskiy ASSERT(arc_released(buf)); 13842fdbea25SAleksandr Guzovskiy 13852fdbea25SAleksandr Guzovskiy mutex_enter(&db->db_mtx); 13862fdbea25SAleksandr Guzovskiy 13872fdbea25SAleksandr Guzovskiy while (db->db_state == DB_READ || db->db_state == DB_FILL) 13882fdbea25SAleksandr Guzovskiy cv_wait(&db->db_changed, &db->db_mtx); 13892fdbea25SAleksandr Guzovskiy 13902fdbea25SAleksandr Guzovskiy ASSERT(db->db_state == DB_CACHED || db->db_state == DB_UNCACHED); 13912fdbea25SAleksandr Guzovskiy 13922fdbea25SAleksandr Guzovskiy if (db->db_state == DB_CACHED && 13932fdbea25SAleksandr Guzovskiy refcount_count(&db->db_holds) - 1 > db->db_dirtycnt) { 13942fdbea25SAleksandr Guzovskiy mutex_exit(&db->db_mtx); 13952fdbea25SAleksandr Guzovskiy (void) dbuf_dirty(db, tx); 13962fdbea25SAleksandr Guzovskiy bcopy(buf->b_data, db->db.db_data, db->db.db_size); 13972fdbea25SAleksandr Guzovskiy VERIFY(arc_buf_remove_ref(buf, db) == 1); 1398c242f9a0Schunli zhang - Sun Microsystems - Irvine United States xuio_stat_wbuf_copied(); 13992fdbea25SAleksandr Guzovskiy return; 14002fdbea25SAleksandr Guzovskiy } 14012fdbea25SAleksandr Guzovskiy 1402c242f9a0Schunli zhang - Sun Microsystems - Irvine United States xuio_stat_wbuf_nocopy(); 14032fdbea25SAleksandr Guzovskiy if (db->db_state == DB_CACHED) { 14042fdbea25SAleksandr Guzovskiy dbuf_dirty_record_t *dr = db->db_last_dirty; 14052fdbea25SAleksandr Guzovskiy 14062fdbea25SAleksandr Guzovskiy ASSERT(db->db_buf != NULL); 14072fdbea25SAleksandr Guzovskiy if (dr != NULL && dr->dr_txg == tx->tx_txg) { 14082fdbea25SAleksandr Guzovskiy ASSERT(dr->dt.dl.dr_data == db->db_buf); 14092fdbea25SAleksandr Guzovskiy if (!arc_released(db->db_buf)) { 14102fdbea25SAleksandr Guzovskiy ASSERT(dr->dt.dl.dr_override_state == 14112fdbea25SAleksandr Guzovskiy DR_OVERRIDDEN); 14122fdbea25SAleksandr Guzovskiy arc_release(db->db_buf, db); 14132fdbea25SAleksandr Guzovskiy } 14142fdbea25SAleksandr Guzovskiy dr->dt.dl.dr_data = buf; 14152fdbea25SAleksandr Guzovskiy VERIFY(arc_buf_remove_ref(db->db_buf, db) == 1); 14162fdbea25SAleksandr Guzovskiy } else if (dr == NULL || dr->dt.dl.dr_data != db->db_buf) { 14172fdbea25SAleksandr Guzovskiy arc_release(db->db_buf, db); 14182fdbea25SAleksandr Guzovskiy VERIFY(arc_buf_remove_ref(db->db_buf, db) == 1); 14192fdbea25SAleksandr Guzovskiy } 14202fdbea25SAleksandr Guzovskiy db->db_buf = NULL; 14212fdbea25SAleksandr Guzovskiy } 14222fdbea25SAleksandr Guzovskiy ASSERT(db->db_buf == NULL); 14232fdbea25SAleksandr Guzovskiy dbuf_set_data(db, buf); 14242fdbea25SAleksandr Guzovskiy db->db_state = DB_FILL; 14252fdbea25SAleksandr Guzovskiy mutex_exit(&db->db_mtx); 14262fdbea25SAleksandr Guzovskiy (void) dbuf_dirty(db, tx); 14272fdbea25SAleksandr Guzovskiy dbuf_fill_done(db, tx); 14282fdbea25SAleksandr Guzovskiy } 14292fdbea25SAleksandr Guzovskiy 14302fdbea25SAleksandr Guzovskiy /* 1431ea8dc4b6Seschrock * "Clear" the contents of this dbuf. This will mark the dbuf 1432ea8dc4b6Seschrock * EVICTING and clear *most* of its references. Unfortunetely, 1433ea8dc4b6Seschrock * when we are not holding the dn_dbufs_mtx, we can't clear the 1434ea8dc4b6Seschrock * entry in the dn_dbufs list. We have to wait until dbuf_destroy() 1435ea8dc4b6Seschrock * in this case. For callers from the DMU we will usually see: 1436ea8dc4b6Seschrock * dbuf_clear()->arc_buf_evict()->dbuf_do_evict()->dbuf_destroy() 1437ea8dc4b6Seschrock * For the arc callback, we will usually see: 1438ea8dc4b6Seschrock * dbuf_do_evict()->dbuf_clear();dbuf_destroy() 1439ea8dc4b6Seschrock * Sometimes, though, we will get a mix of these two: 1440ea8dc4b6Seschrock * DMU: dbuf_clear()->arc_buf_evict() 1441ea8dc4b6Seschrock * ARC: dbuf_do_evict()->dbuf_destroy() 1442ea8dc4b6Seschrock */ 1443ea8dc4b6Seschrock void 1444fa9e4066Sahrens dbuf_clear(dmu_buf_impl_t *db) 1445fa9e4066Sahrens { 1446fa9e4066Sahrens dnode_t *dn = db->db_dnode; 1447ea8dc4b6Seschrock dmu_buf_impl_t *parent = db->db_parent; 1448c543ec06Sahrens dmu_buf_impl_t *dndb = dn->dn_dbuf; 1449ea8dc4b6Seschrock int dbuf_gone = FALSE; 1450fa9e4066Sahrens 1451fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 1452fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 1453fa9e4066Sahrens 1454ea8dc4b6Seschrock dbuf_evict_user(db); 1455ea8dc4b6Seschrock 1456fa9e4066Sahrens if (db->db_state == DB_CACHED) { 1457ea8dc4b6Seschrock ASSERT(db->db.db_data != NULL); 14580a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 1459ea8dc4b6Seschrock zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN); 14605a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 14610e8c6158Smaybee } 1462fa9e4066Sahrens db->db.db_data = NULL; 1463fa9e4066Sahrens db->db_state = DB_UNCACHED; 1464fa9e4066Sahrens } 1465fa9e4066Sahrens 146682c9918fSTim Haley ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL); 1467fa9e4066Sahrens ASSERT(db->db_data_pending == NULL); 1468fa9e4066Sahrens 1469ea8dc4b6Seschrock db->db_state = DB_EVICTING; 1470ea8dc4b6Seschrock db->db_blkptr = NULL; 1471ea8dc4b6Seschrock 14720a586ceaSMark Shellenbaum if (db->db_blkid != DMU_BONUS_BLKID && MUTEX_HELD(&dn->dn_dbufs_mtx)) { 1473ea8dc4b6Seschrock list_remove(&dn->dn_dbufs, db); 1474ea8dc4b6Seschrock dnode_rele(dn, db); 14751934e92fSmaybee db->db_dnode = NULL; 1476ea8dc4b6Seschrock } 1477ea8dc4b6Seschrock 1478ea8dc4b6Seschrock if (db->db_buf) 1479ea8dc4b6Seschrock dbuf_gone = arc_buf_evict(db->db_buf); 1480ea8dc4b6Seschrock 1481ea8dc4b6Seschrock if (!dbuf_gone) 1482fa9e4066Sahrens mutex_exit(&db->db_mtx); 1483fa9e4066Sahrens 1484fa9e4066Sahrens /* 1485fa9e4066Sahrens * If this dbuf is referened from an indirect dbuf, 1486fa9e4066Sahrens * decrement the ref count on the indirect dbuf. 1487fa9e4066Sahrens */ 1488c543ec06Sahrens if (parent && parent != dndb) 1489ea8dc4b6Seschrock dbuf_rele(parent, db); 1490fa9e4066Sahrens } 1491fa9e4066Sahrens 1492fa9e4066Sahrens static int 1493fa9e4066Sahrens dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse, 1494fa9e4066Sahrens dmu_buf_impl_t **parentp, blkptr_t **bpp) 1495fa9e4066Sahrens { 1496fa9e4066Sahrens int nlevels, epbs; 1497fa9e4066Sahrens 14980b69c2f0Sahrens *parentp = NULL; 14990b69c2f0Sahrens *bpp = NULL; 15000b69c2f0Sahrens 15010a586ceaSMark Shellenbaum ASSERT(blkid != DMU_BONUS_BLKID); 15020a586ceaSMark Shellenbaum 15030a586ceaSMark Shellenbaum if (blkid == DMU_SPILL_BLKID) { 15040a586ceaSMark Shellenbaum mutex_enter(&dn->dn_mtx); 150506e0070dSMark Shellenbaum if (dn->dn_have_spill && 150606e0070dSMark Shellenbaum (dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR)) 15070a586ceaSMark Shellenbaum *bpp = &dn->dn_phys->dn_spill; 15080a586ceaSMark Shellenbaum else 15090a586ceaSMark Shellenbaum *bpp = NULL; 15100a586ceaSMark Shellenbaum dbuf_add_ref(dn->dn_dbuf, NULL); 15110a586ceaSMark Shellenbaum *parentp = dn->dn_dbuf; 15120a586ceaSMark Shellenbaum mutex_exit(&dn->dn_mtx); 15130a586ceaSMark Shellenbaum return (0); 15140a586ceaSMark Shellenbaum } 1515ea8dc4b6Seschrock 1516fa9e4066Sahrens if (dn->dn_phys->dn_nlevels == 0) 1517fa9e4066Sahrens nlevels = 1; 1518fa9e4066Sahrens else 1519fa9e4066Sahrens nlevels = dn->dn_phys->dn_nlevels; 1520fa9e4066Sahrens 1521fa9e4066Sahrens epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1522fa9e4066Sahrens 1523fa9e4066Sahrens ASSERT3U(level * epbs, <, 64); 1524fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1525ea8dc4b6Seschrock if (level >= nlevels || 1526fa9e4066Sahrens (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) { 1527fa9e4066Sahrens /* the buffer has no parent yet */ 1528fa9e4066Sahrens return (ENOENT); 1529fa9e4066Sahrens } else if (level < nlevels-1) { 1530fa9e4066Sahrens /* this block is referenced from an indirect block */ 1531fa9e4066Sahrens int err = dbuf_hold_impl(dn, level+1, 1532fa9e4066Sahrens blkid >> epbs, fail_sparse, NULL, parentp); 1533fa9e4066Sahrens if (err) 1534fa9e4066Sahrens return (err); 1535ea8dc4b6Seschrock err = dbuf_read(*parentp, NULL, 1536ea8dc4b6Seschrock (DB_RF_HAVESTRUCT | DB_RF_NOPREFETCH | DB_RF_CANFAIL)); 1537c543ec06Sahrens if (err) { 1538c543ec06Sahrens dbuf_rele(*parentp, NULL); 1539c543ec06Sahrens *parentp = NULL; 1540c543ec06Sahrens return (err); 1541c543ec06Sahrens } 1542fa9e4066Sahrens *bpp = ((blkptr_t *)(*parentp)->db.db_data) + 1543fa9e4066Sahrens (blkid & ((1ULL << epbs) - 1)); 1544c543ec06Sahrens return (0); 1545fa9e4066Sahrens } else { 1546fa9e4066Sahrens /* the block is referenced from the dnode */ 1547fa9e4066Sahrens ASSERT3U(level, ==, nlevels-1); 1548fa9e4066Sahrens ASSERT(dn->dn_phys->dn_nblkptr == 0 || 1549fa9e4066Sahrens blkid < dn->dn_phys->dn_nblkptr); 1550c543ec06Sahrens if (dn->dn_dbuf) { 1551c543ec06Sahrens dbuf_add_ref(dn->dn_dbuf, NULL); 1552fa9e4066Sahrens *parentp = dn->dn_dbuf; 1553c543ec06Sahrens } 1554fa9e4066Sahrens *bpp = &dn->dn_phys->dn_blkptr[blkid]; 1555fa9e4066Sahrens return (0); 1556fa9e4066Sahrens } 1557fa9e4066Sahrens } 1558fa9e4066Sahrens 1559fa9e4066Sahrens static dmu_buf_impl_t * 1560fa9e4066Sahrens dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid, 1561fa9e4066Sahrens dmu_buf_impl_t *parent, blkptr_t *blkptr) 1562fa9e4066Sahrens { 1563503ad85cSMatthew Ahrens objset_t *os = dn->dn_objset; 1564fa9e4066Sahrens dmu_buf_impl_t *db, *odb; 1565fa9e4066Sahrens 1566fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1567fa9e4066Sahrens ASSERT(dn->dn_type != DMU_OT_NONE); 1568fa9e4066Sahrens 1569fa9e4066Sahrens db = kmem_cache_alloc(dbuf_cache, KM_SLEEP); 1570fa9e4066Sahrens 1571fa9e4066Sahrens db->db_objset = os; 1572fa9e4066Sahrens db->db.db_object = dn->dn_object; 1573fa9e4066Sahrens db->db_level = level; 1574fa9e4066Sahrens db->db_blkid = blkid; 1575c717a561Smaybee db->db_last_dirty = NULL; 1576ea8dc4b6Seschrock db->db_dirtycnt = 0; 1577ea8dc4b6Seschrock db->db_dnode = dn; 1578ea8dc4b6Seschrock db->db_parent = parent; 1579ea8dc4b6Seschrock db->db_blkptr = blkptr; 1580fa9e4066Sahrens 1581c717a561Smaybee db->db_user_ptr = NULL; 1582c717a561Smaybee db->db_user_data_ptr_ptr = NULL; 1583c717a561Smaybee db->db_evict_func = NULL; 1584c717a561Smaybee db->db_immediate_evict = 0; 1585c717a561Smaybee db->db_freed_in_flight = 0; 1586ea8dc4b6Seschrock 15870a586ceaSMark Shellenbaum if (blkid == DMU_BONUS_BLKID) { 1588ea8dc4b6Seschrock ASSERT3P(parent, ==, dn->dn_dbuf); 15891934e92fSmaybee db->db.db_size = DN_MAX_BONUSLEN - 15901934e92fSmaybee (dn->dn_nblkptr-1) * sizeof (blkptr_t); 15911934e92fSmaybee ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 15920a586ceaSMark Shellenbaum db->db.db_offset = DMU_BONUS_BLKID; 1593ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 1594ea8dc4b6Seschrock /* the bonus dbuf is not placed in the hash table */ 15955a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1596ea8dc4b6Seschrock return (db); 15970a586ceaSMark Shellenbaum } else if (blkid == DMU_SPILL_BLKID) { 15980a586ceaSMark Shellenbaum db->db.db_size = (blkptr != NULL) ? 15990a586ceaSMark Shellenbaum BP_GET_LSIZE(blkptr) : SPA_MINBLOCKSIZE; 16000a586ceaSMark Shellenbaum db->db.db_offset = 0; 1601fa9e4066Sahrens } else { 1602fa9e4066Sahrens int blocksize = 1603fa9e4066Sahrens db->db_level ? 1<<dn->dn_indblkshift : dn->dn_datablksz; 1604fa9e4066Sahrens db->db.db_size = blocksize; 1605fa9e4066Sahrens db->db.db_offset = db->db_blkid * blocksize; 1606fa9e4066Sahrens } 1607fa9e4066Sahrens 1608fa9e4066Sahrens /* 1609fa9e4066Sahrens * Hold the dn_dbufs_mtx while we get the new dbuf 1610fa9e4066Sahrens * in the hash table *and* added to the dbufs list. 1611fa9e4066Sahrens * This prevents a possible deadlock with someone 1612fa9e4066Sahrens * trying to look up this dbuf before its added to the 1613fa9e4066Sahrens * dn_dbufs list. 1614fa9e4066Sahrens */ 1615fa9e4066Sahrens mutex_enter(&dn->dn_dbufs_mtx); 1616ea8dc4b6Seschrock db->db_state = DB_EVICTING; 1617fa9e4066Sahrens if ((odb = dbuf_hash_insert(db)) != NULL) { 1618fa9e4066Sahrens /* someone else inserted it first */ 1619fa9e4066Sahrens kmem_cache_free(dbuf_cache, db); 1620fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 1621fa9e4066Sahrens return (odb); 1622fa9e4066Sahrens } 1623fa9e4066Sahrens list_insert_head(&dn->dn_dbufs, db); 1624ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 1625fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 16265a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1627fa9e4066Sahrens 1628fa9e4066Sahrens if (parent && parent != dn->dn_dbuf) 1629fa9e4066Sahrens dbuf_add_ref(parent, db); 1630fa9e4066Sahrens 1631ea8dc4b6Seschrock ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 1632ea8dc4b6Seschrock refcount_count(&dn->dn_holds) > 0); 1633fa9e4066Sahrens (void) refcount_add(&dn->dn_holds, db); 1634fa9e4066Sahrens 1635fa9e4066Sahrens dprintf_dbuf(db, "db=%p\n", db); 1636fa9e4066Sahrens 1637fa9e4066Sahrens return (db); 1638fa9e4066Sahrens } 1639fa9e4066Sahrens 1640fa9e4066Sahrens static int 1641ea8dc4b6Seschrock dbuf_do_evict(void *private) 1642fa9e4066Sahrens { 1643ea8dc4b6Seschrock arc_buf_t *buf = private; 1644ea8dc4b6Seschrock dmu_buf_impl_t *db = buf->b_private; 1645fa9e4066Sahrens 1646ea8dc4b6Seschrock if (!MUTEX_HELD(&db->db_mtx)) 1647ea8dc4b6Seschrock mutex_enter(&db->db_mtx); 1648ea8dc4b6Seschrock 1649ea8dc4b6Seschrock ASSERT(refcount_is_zero(&db->db_holds)); 1650ea8dc4b6Seschrock 1651ea8dc4b6Seschrock if (db->db_state != DB_EVICTING) { 1652ea8dc4b6Seschrock ASSERT(db->db_state == DB_CACHED); 16539c9dc39aSek110237 DBUF_VERIFY(db); 1654ea8dc4b6Seschrock db->db_buf = NULL; 1655ea8dc4b6Seschrock dbuf_evict(db); 1656ea8dc4b6Seschrock } else { 1657ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 1658ea8dc4b6Seschrock dbuf_destroy(db); 1659fa9e4066Sahrens } 1660ea8dc4b6Seschrock return (0); 1661fa9e4066Sahrens } 1662fa9e4066Sahrens 1663fa9e4066Sahrens static void 1664fa9e4066Sahrens dbuf_destroy(dmu_buf_impl_t *db) 1665fa9e4066Sahrens { 1666fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 1667fa9e4066Sahrens 16680a586ceaSMark Shellenbaum if (db->db_blkid != DMU_BONUS_BLKID) { 1669ea8dc4b6Seschrock /* 1670ea8dc4b6Seschrock * If this dbuf is still on the dn_dbufs list, 1671ea8dc4b6Seschrock * remove it from that list. 1672ea8dc4b6Seschrock */ 16731934e92fSmaybee if (db->db_dnode) { 16741934e92fSmaybee dnode_t *dn = db->db_dnode; 16751934e92fSmaybee 16761934e92fSmaybee mutex_enter(&dn->dn_dbufs_mtx); 1677ea8dc4b6Seschrock list_remove(&dn->dn_dbufs, db); 1678ea8dc4b6Seschrock mutex_exit(&dn->dn_dbufs_mtx); 1679ea8dc4b6Seschrock 1680ea8dc4b6Seschrock dnode_rele(dn, db); 16811934e92fSmaybee db->db_dnode = NULL; 1682ea8dc4b6Seschrock } 1683ea8dc4b6Seschrock dbuf_hash_remove(db); 1684ea8dc4b6Seschrock } 1685ea8dc4b6Seschrock db->db_parent = NULL; 1686ea8dc4b6Seschrock db->db_buf = NULL; 1687ea8dc4b6Seschrock 1688f82bfe17Sgw25295 ASSERT(!list_link_active(&db->db_link)); 1689fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 1690fa9e4066Sahrens ASSERT(db->db_hash_next == NULL); 1691fa9e4066Sahrens ASSERT(db->db_blkptr == NULL); 1692fa9e4066Sahrens ASSERT(db->db_data_pending == NULL); 1693fa9e4066Sahrens 1694fa9e4066Sahrens kmem_cache_free(dbuf_cache, db); 16955a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1696fa9e4066Sahrens } 1697fa9e4066Sahrens 1698fa9e4066Sahrens void 1699fa9e4066Sahrens dbuf_prefetch(dnode_t *dn, uint64_t blkid) 1700fa9e4066Sahrens { 170113506d1eSmaybee dmu_buf_impl_t *db = NULL; 1702fa9e4066Sahrens blkptr_t *bp = NULL; 1703fa9e4066Sahrens 17040a586ceaSMark Shellenbaum ASSERT(blkid != DMU_BONUS_BLKID); 1705fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1706fa9e4066Sahrens 1707fa9e4066Sahrens if (dnode_block_freed(dn, blkid)) 1708fa9e4066Sahrens return; 1709fa9e4066Sahrens 1710fa9e4066Sahrens /* dbuf_find() returns with db_mtx held */ 1711fa9e4066Sahrens if (db = dbuf_find(dn, 0, blkid)) { 171213506d1eSmaybee if (refcount_count(&db->db_holds) > 0) { 1713fa9e4066Sahrens /* 171413506d1eSmaybee * This dbuf is active. We assume that it is 171513506d1eSmaybee * already CACHED, or else about to be either 1716fa9e4066Sahrens * read or filled. 1717fa9e4066Sahrens */ 1718fa9e4066Sahrens mutex_exit(&db->db_mtx); 1719fa9e4066Sahrens return; 1720fa9e4066Sahrens } 172113506d1eSmaybee mutex_exit(&db->db_mtx); 17220b69c2f0Sahrens db = NULL; 172313506d1eSmaybee } 1724fa9e4066Sahrens 172513506d1eSmaybee if (dbuf_findbp(dn, 0, blkid, TRUE, &db, &bp) == 0) { 1726fa9e4066Sahrens if (bp && !BP_IS_HOLE(bp)) { 1727088f3894Sahrens arc_buf_t *pbuf; 1728b24ab676SJeff Bonwick dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset; 172913506d1eSmaybee uint32_t aflags = ARC_NOWAIT | ARC_PREFETCH; 1730ea8dc4b6Seschrock zbookmark_t zb; 1731b24ab676SJeff Bonwick 1732b24ab676SJeff Bonwick SET_BOOKMARK(&zb, ds ? ds->ds_object : DMU_META_OBJSET, 1733b24ab676SJeff Bonwick dn->dn_object, 0, blkid); 1734ea8dc4b6Seschrock 1735088f3894Sahrens if (db) 1736088f3894Sahrens pbuf = db->db_buf; 1737088f3894Sahrens else 1738088f3894Sahrens pbuf = dn->dn_objset->os_phys_buf; 1739088f3894Sahrens 1740*3f9d6ad7SLin Ling (void) dsl_read(NULL, dn->dn_objset->os_spa, 1741088f3894Sahrens bp, pbuf, NULL, NULL, ZIO_PRIORITY_ASYNC_READ, 1742fa9e4066Sahrens ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 174313506d1eSmaybee &aflags, &zb); 1744fa9e4066Sahrens } 174513506d1eSmaybee if (db) 174613506d1eSmaybee dbuf_rele(db, NULL); 1747fa9e4066Sahrens } 1748fa9e4066Sahrens } 1749fa9e4066Sahrens 1750fa9e4066Sahrens /* 1751fa9e4066Sahrens * Returns with db_holds incremented, and db_mtx not held. 1752fa9e4066Sahrens * Note: dn_struct_rwlock must be held. 1753fa9e4066Sahrens */ 1754fa9e4066Sahrens int 1755fa9e4066Sahrens dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, int fail_sparse, 1756fa9e4066Sahrens void *tag, dmu_buf_impl_t **dbp) 1757fa9e4066Sahrens { 1758fa9e4066Sahrens dmu_buf_impl_t *db, *parent = NULL; 1759fa9e4066Sahrens 17600a586ceaSMark Shellenbaum ASSERT(blkid != DMU_BONUS_BLKID); 1761fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1762fa9e4066Sahrens ASSERT3U(dn->dn_nlevels, >, level); 1763fa9e4066Sahrens 1764fa9e4066Sahrens *dbp = NULL; 1765ea8dc4b6Seschrock top: 1766fa9e4066Sahrens /* dbuf_find() returns with db_mtx held */ 1767fa9e4066Sahrens db = dbuf_find(dn, level, blkid); 1768fa9e4066Sahrens 1769fa9e4066Sahrens if (db == NULL) { 1770fa9e4066Sahrens blkptr_t *bp = NULL; 1771fa9e4066Sahrens int err; 1772fa9e4066Sahrens 1773c543ec06Sahrens ASSERT3P(parent, ==, NULL); 1774fa9e4066Sahrens err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp); 1775fa9e4066Sahrens if (fail_sparse) { 1776fa9e4066Sahrens if (err == 0 && bp && BP_IS_HOLE(bp)) 1777fa9e4066Sahrens err = ENOENT; 1778fa9e4066Sahrens if (err) { 1779c543ec06Sahrens if (parent) 1780ea8dc4b6Seschrock dbuf_rele(parent, NULL); 1781fa9e4066Sahrens return (err); 1782fa9e4066Sahrens } 1783fa9e4066Sahrens } 1784ea8dc4b6Seschrock if (err && err != ENOENT) 1785ea8dc4b6Seschrock return (err); 1786fa9e4066Sahrens db = dbuf_create(dn, level, blkid, parent, bp); 1787fa9e4066Sahrens } 1788fa9e4066Sahrens 1789ea8dc4b6Seschrock if (db->db_buf && refcount_is_zero(&db->db_holds)) { 1790ea8dc4b6Seschrock arc_buf_add_ref(db->db_buf, db); 1791ea8dc4b6Seschrock if (db->db_buf->b_data == NULL) { 1792ea8dc4b6Seschrock dbuf_clear(db); 1793c543ec06Sahrens if (parent) { 1794c543ec06Sahrens dbuf_rele(parent, NULL); 1795c543ec06Sahrens parent = NULL; 1796c543ec06Sahrens } 1797ea8dc4b6Seschrock goto top; 1798ea8dc4b6Seschrock } 1799ea8dc4b6Seschrock ASSERT3P(db->db.db_data, ==, db->db_buf->b_data); 1800ea8dc4b6Seschrock } 1801ea8dc4b6Seschrock 1802ea8dc4b6Seschrock ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf)); 1803ea8dc4b6Seschrock 1804fa9e4066Sahrens /* 1805c717a561Smaybee * If this buffer is currently syncing out, and we are are 1806c717a561Smaybee * still referencing it from db_data, we need to make a copy 1807c717a561Smaybee * of it in case we decide we want to dirty it again in this txg. 1808fa9e4066Sahrens */ 18090a586ceaSMark Shellenbaum if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 1810ea8dc4b6Seschrock dn->dn_object != DMU_META_DNODE_OBJECT && 1811c717a561Smaybee db->db_state == DB_CACHED && db->db_data_pending) { 1812c717a561Smaybee dbuf_dirty_record_t *dr = db->db_data_pending; 1813c717a561Smaybee 1814c717a561Smaybee if (dr->dt.dl.dr_data == db->db_buf) { 1815ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 1816fa9e4066Sahrens 1817c717a561Smaybee dbuf_set_data(db, 1818c717a561Smaybee arc_buf_alloc(db->db_dnode->dn_objset->os_spa, 1819c717a561Smaybee db->db.db_size, db, type)); 1820c717a561Smaybee bcopy(dr->dt.dl.dr_data->b_data, db->db.db_data, 1821fa9e4066Sahrens db->db.db_size); 1822fa9e4066Sahrens } 1823c717a561Smaybee } 1824fa9e4066Sahrens 1825ea8dc4b6Seschrock (void) refcount_add(&db->db_holds, tag); 1826fa9e4066Sahrens dbuf_update_data(db); 18279c9dc39aSek110237 DBUF_VERIFY(db); 1828fa9e4066Sahrens mutex_exit(&db->db_mtx); 1829fa9e4066Sahrens 1830fa9e4066Sahrens /* NOTE: we can't rele the parent until after we drop the db_mtx */ 1831c543ec06Sahrens if (parent) 1832ea8dc4b6Seschrock dbuf_rele(parent, NULL); 1833fa9e4066Sahrens 1834fa9e4066Sahrens ASSERT3P(db->db_dnode, ==, dn); 1835fa9e4066Sahrens ASSERT3U(db->db_blkid, ==, blkid); 1836fa9e4066Sahrens ASSERT3U(db->db_level, ==, level); 1837fa9e4066Sahrens *dbp = db; 1838fa9e4066Sahrens 1839fa9e4066Sahrens return (0); 1840fa9e4066Sahrens } 1841fa9e4066Sahrens 1842fa9e4066Sahrens dmu_buf_impl_t * 1843ea8dc4b6Seschrock dbuf_hold(dnode_t *dn, uint64_t blkid, void *tag) 1844fa9e4066Sahrens { 1845fa9e4066Sahrens dmu_buf_impl_t *db; 1846ea8dc4b6Seschrock int err = dbuf_hold_impl(dn, 0, blkid, FALSE, tag, &db); 1847ea8dc4b6Seschrock return (err ? NULL : db); 1848fa9e4066Sahrens } 1849fa9e4066Sahrens 1850fa9e4066Sahrens dmu_buf_impl_t * 1851fa9e4066Sahrens dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, void *tag) 1852fa9e4066Sahrens { 1853fa9e4066Sahrens dmu_buf_impl_t *db; 1854ea8dc4b6Seschrock int err = dbuf_hold_impl(dn, level, blkid, FALSE, tag, &db); 1855ea8dc4b6Seschrock return (err ? NULL : db); 1856fa9e4066Sahrens } 1857fa9e4066Sahrens 18581934e92fSmaybee void 1859ea8dc4b6Seschrock dbuf_create_bonus(dnode_t *dn) 1860fa9e4066Sahrens { 1861ea8dc4b6Seschrock ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 1862ea8dc4b6Seschrock 1863ea8dc4b6Seschrock ASSERT(dn->dn_bonus == NULL); 18640a586ceaSMark Shellenbaum dn->dn_bonus = dbuf_create(dn, 0, DMU_BONUS_BLKID, dn->dn_dbuf, NULL); 18650a586ceaSMark Shellenbaum } 18660a586ceaSMark Shellenbaum 18670a586ceaSMark Shellenbaum int 18680a586ceaSMark Shellenbaum dbuf_spill_set_blksz(dmu_buf_t *db_fake, uint64_t blksz, dmu_tx_t *tx) 18690a586ceaSMark Shellenbaum { 18700a586ceaSMark Shellenbaum dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 18710a586ceaSMark Shellenbaum if (db->db_blkid != DMU_SPILL_BLKID) 18720a586ceaSMark Shellenbaum return (ENOTSUP); 18730a586ceaSMark Shellenbaum if (blksz == 0) 18740a586ceaSMark Shellenbaum blksz = SPA_MINBLOCKSIZE; 18750a586ceaSMark Shellenbaum if (blksz > SPA_MAXBLOCKSIZE) 18760a586ceaSMark Shellenbaum blksz = SPA_MAXBLOCKSIZE; 18770a586ceaSMark Shellenbaum else 18780a586ceaSMark Shellenbaum blksz = P2ROUNDUP(blksz, SPA_MINBLOCKSIZE); 18790a586ceaSMark Shellenbaum 18800a586ceaSMark Shellenbaum rw_enter(&db->db_dnode->dn_struct_rwlock, RW_WRITER); 18810a586ceaSMark Shellenbaum dbuf_new_size(db, blksz, tx); 18820a586ceaSMark Shellenbaum rw_exit(&db->db_dnode->dn_struct_rwlock); 18830a586ceaSMark Shellenbaum 18840a586ceaSMark Shellenbaum return (0); 18850a586ceaSMark Shellenbaum } 18860a586ceaSMark Shellenbaum 18870a586ceaSMark Shellenbaum void 18880a586ceaSMark Shellenbaum dbuf_rm_spill(dnode_t *dn, dmu_tx_t *tx) 18890a586ceaSMark Shellenbaum { 18900a586ceaSMark Shellenbaum dbuf_free_range(dn, DMU_SPILL_BLKID, DMU_SPILL_BLKID, tx); 1891fa9e4066Sahrens } 1892fa9e4066Sahrens 1893ea8dc4b6Seschrock #pragma weak dmu_buf_add_ref = dbuf_add_ref 1894fa9e4066Sahrens void 1895fa9e4066Sahrens dbuf_add_ref(dmu_buf_impl_t *db, void *tag) 1896fa9e4066Sahrens { 1897ea8dc4b6Seschrock int64_t holds = refcount_add(&db->db_holds, tag); 1898ea8dc4b6Seschrock ASSERT(holds > 1); 1899fa9e4066Sahrens } 1900fa9e4066Sahrens 1901ea8dc4b6Seschrock #pragma weak dmu_buf_rele = dbuf_rele 1902fa9e4066Sahrens void 1903ea8dc4b6Seschrock dbuf_rele(dmu_buf_impl_t *db, void *tag) 1904fa9e4066Sahrens { 1905b24ab676SJeff Bonwick mutex_enter(&db->db_mtx); 1906b24ab676SJeff Bonwick dbuf_rele_and_unlock(db, tag); 1907b24ab676SJeff Bonwick } 1908b24ab676SJeff Bonwick 1909b24ab676SJeff Bonwick /* 1910b24ab676SJeff Bonwick * dbuf_rele() for an already-locked dbuf. This is necessary to allow 1911b24ab676SJeff Bonwick * db_dirtycnt and db_holds to be updated atomically. 1912b24ab676SJeff Bonwick */ 1913b24ab676SJeff Bonwick void 1914b24ab676SJeff Bonwick dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag) 1915b24ab676SJeff Bonwick { 1916fa9e4066Sahrens int64_t holds; 1917fa9e4066Sahrens 1918b24ab676SJeff Bonwick ASSERT(MUTEX_HELD(&db->db_mtx)); 19199c9dc39aSek110237 DBUF_VERIFY(db); 1920fa9e4066Sahrens 1921fa9e4066Sahrens holds = refcount_remove(&db->db_holds, tag); 1922ea8dc4b6Seschrock ASSERT(holds >= 0); 1923fa9e4066Sahrens 1924c717a561Smaybee /* 1925c717a561Smaybee * We can't freeze indirects if there is a possibility that they 1926c717a561Smaybee * may be modified in the current syncing context. 1927c717a561Smaybee */ 1928c717a561Smaybee if (db->db_buf && holds == (db->db_level == 0 ? db->db_dirtycnt : 0)) 19296b4acc8bSahrens arc_buf_freeze(db->db_buf); 19306b4acc8bSahrens 1931fa9e4066Sahrens if (holds == db->db_dirtycnt && 1932c717a561Smaybee db->db_level == 0 && db->db_immediate_evict) 1933fa9e4066Sahrens dbuf_evict_user(db); 1934ea8dc4b6Seschrock 1935ea8dc4b6Seschrock if (holds == 0) { 19360a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 1937ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 1938ea8dc4b6Seschrock dnode_rele(db->db_dnode, db); 1939ea8dc4b6Seschrock } else if (db->db_buf == NULL) { 1940ea8dc4b6Seschrock /* 1941ea8dc4b6Seschrock * This is a special case: we never associated this 1942ea8dc4b6Seschrock * dbuf with any data allocated from the ARC. 1943ea8dc4b6Seschrock */ 194482c9918fSTim Haley ASSERT(db->db_state == DB_UNCACHED || 194582c9918fSTim Haley db->db_state == DB_NOFILL); 1946ea8dc4b6Seschrock dbuf_evict(db); 1947ea8dc4b6Seschrock } else if (arc_released(db->db_buf)) { 1948ea8dc4b6Seschrock arc_buf_t *buf = db->db_buf; 1949ea8dc4b6Seschrock /* 1950ea8dc4b6Seschrock * This dbuf has anonymous data associated with it. 1951ea8dc4b6Seschrock */ 1952ea8dc4b6Seschrock dbuf_set_data(db, NULL); 1953ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(buf, db) == 1); 1954ea8dc4b6Seschrock dbuf_evict(db); 1955ea8dc4b6Seschrock } else { 1956ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(db->db_buf, db) == 0); 19573baa08fcSek110237 if (!DBUF_IS_CACHEABLE(db)) 19583baa08fcSek110237 dbuf_clear(db); 19593baa08fcSek110237 else 1960fa9e4066Sahrens mutex_exit(&db->db_mtx); 1961fa9e4066Sahrens } 1962ea8dc4b6Seschrock } else { 1963ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 1964fa9e4066Sahrens } 1965fa9e4066Sahrens } 1966fa9e4066Sahrens 1967fa9e4066Sahrens #pragma weak dmu_buf_refcount = dbuf_refcount 1968fa9e4066Sahrens uint64_t 1969fa9e4066Sahrens dbuf_refcount(dmu_buf_impl_t *db) 1970fa9e4066Sahrens { 1971fa9e4066Sahrens return (refcount_count(&db->db_holds)); 1972fa9e4066Sahrens } 1973fa9e4066Sahrens 1974fa9e4066Sahrens void * 1975fa9e4066Sahrens dmu_buf_set_user(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 1976fa9e4066Sahrens dmu_buf_evict_func_t *evict_func) 1977fa9e4066Sahrens { 1978fa9e4066Sahrens return (dmu_buf_update_user(db_fake, NULL, user_ptr, 1979fa9e4066Sahrens user_data_ptr_ptr, evict_func)); 1980fa9e4066Sahrens } 1981fa9e4066Sahrens 1982fa9e4066Sahrens void * 1983fa9e4066Sahrens dmu_buf_set_user_ie(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 1984fa9e4066Sahrens dmu_buf_evict_func_t *evict_func) 1985fa9e4066Sahrens { 1986fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1987fa9e4066Sahrens 1988c717a561Smaybee db->db_immediate_evict = TRUE; 1989fa9e4066Sahrens return (dmu_buf_update_user(db_fake, NULL, user_ptr, 1990fa9e4066Sahrens user_data_ptr_ptr, evict_func)); 1991fa9e4066Sahrens } 1992fa9e4066Sahrens 1993fa9e4066Sahrens void * 1994fa9e4066Sahrens dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr, void *user_ptr, 1995fa9e4066Sahrens void *user_data_ptr_ptr, dmu_buf_evict_func_t *evict_func) 1996fa9e4066Sahrens { 1997fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1998fa9e4066Sahrens ASSERT(db->db_level == 0); 1999fa9e4066Sahrens 2000fa9e4066Sahrens ASSERT((user_ptr == NULL) == (evict_func == NULL)); 2001fa9e4066Sahrens 2002fa9e4066Sahrens mutex_enter(&db->db_mtx); 2003fa9e4066Sahrens 2004c717a561Smaybee if (db->db_user_ptr == old_user_ptr) { 2005c717a561Smaybee db->db_user_ptr = user_ptr; 2006c717a561Smaybee db->db_user_data_ptr_ptr = user_data_ptr_ptr; 2007c717a561Smaybee db->db_evict_func = evict_func; 2008fa9e4066Sahrens 2009fa9e4066Sahrens dbuf_update_data(db); 2010fa9e4066Sahrens } else { 2011c717a561Smaybee old_user_ptr = db->db_user_ptr; 2012fa9e4066Sahrens } 2013fa9e4066Sahrens 2014fa9e4066Sahrens mutex_exit(&db->db_mtx); 2015fa9e4066Sahrens return (old_user_ptr); 2016fa9e4066Sahrens } 2017fa9e4066Sahrens 2018fa9e4066Sahrens void * 2019fa9e4066Sahrens dmu_buf_get_user(dmu_buf_t *db_fake) 2020fa9e4066Sahrens { 2021fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2022fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 2023fa9e4066Sahrens 2024c717a561Smaybee return (db->db_user_ptr); 2025fa9e4066Sahrens } 2026fa9e4066Sahrens 20273d692628SSanjeev Bagewadi boolean_t 20283d692628SSanjeev Bagewadi dmu_buf_freeable(dmu_buf_t *dbuf) 20293d692628SSanjeev Bagewadi { 20303d692628SSanjeev Bagewadi boolean_t res = B_FALSE; 20313d692628SSanjeev Bagewadi dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf; 20323d692628SSanjeev Bagewadi 20333d692628SSanjeev Bagewadi if (db->db_blkptr) 20343d692628SSanjeev Bagewadi res = dsl_dataset_block_freeable(db->db_objset->os_dsl_dataset, 20353d692628SSanjeev Bagewadi db->db_blkptr->blk_birth); 20363d692628SSanjeev Bagewadi 20373d692628SSanjeev Bagewadi return (res); 20383d692628SSanjeev Bagewadi } 20393d692628SSanjeev Bagewadi 2040c717a561Smaybee static void 2041c717a561Smaybee dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db) 2042fa9e4066Sahrens { 2043c717a561Smaybee /* ASSERT(dmu_tx_is_syncing(tx) */ 2044c717a561Smaybee ASSERT(MUTEX_HELD(&db->db_mtx)); 2045c717a561Smaybee 2046c717a561Smaybee if (db->db_blkptr != NULL) 2047c717a561Smaybee return; 2048c717a561Smaybee 20490a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) { 20500a586ceaSMark Shellenbaum db->db_blkptr = &dn->dn_phys->dn_spill; 20510a586ceaSMark Shellenbaum BP_ZERO(db->db_blkptr); 20520a586ceaSMark Shellenbaum return; 20530a586ceaSMark Shellenbaum } 2054c717a561Smaybee if (db->db_level == dn->dn_phys->dn_nlevels-1) { 2055c717a561Smaybee /* 2056c717a561Smaybee * This buffer was allocated at a time when there was 2057c717a561Smaybee * no available blkptrs from the dnode, or it was 2058c717a561Smaybee * inappropriate to hook it in (i.e., nlevels mis-match). 2059c717a561Smaybee */ 2060c717a561Smaybee ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr); 2061c717a561Smaybee ASSERT(db->db_parent == NULL); 2062c717a561Smaybee db->db_parent = dn->dn_dbuf; 2063c717a561Smaybee db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid]; 2064c717a561Smaybee DBUF_VERIFY(db); 2065c717a561Smaybee } else { 2066c717a561Smaybee dmu_buf_impl_t *parent = db->db_parent; 2067c717a561Smaybee int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2068c717a561Smaybee 2069c717a561Smaybee ASSERT(dn->dn_phys->dn_nlevels > 1); 2070c717a561Smaybee if (parent == NULL) { 2071c717a561Smaybee mutex_exit(&db->db_mtx); 2072c717a561Smaybee rw_enter(&dn->dn_struct_rwlock, RW_READER); 2073c717a561Smaybee (void) dbuf_hold_impl(dn, db->db_level+1, 2074c717a561Smaybee db->db_blkid >> epbs, FALSE, db, &parent); 2075c717a561Smaybee rw_exit(&dn->dn_struct_rwlock); 2076c717a561Smaybee mutex_enter(&db->db_mtx); 2077c717a561Smaybee db->db_parent = parent; 2078c717a561Smaybee } 2079c717a561Smaybee db->db_blkptr = (blkptr_t *)parent->db.db_data + 2080c717a561Smaybee (db->db_blkid & ((1ULL << epbs) - 1)); 2081c717a561Smaybee DBUF_VERIFY(db); 2082c717a561Smaybee } 2083c717a561Smaybee } 2084c717a561Smaybee 2085c717a561Smaybee static void 2086c717a561Smaybee dbuf_sync_indirect(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 2087c717a561Smaybee { 2088c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 2089c717a561Smaybee dnode_t *dn = db->db_dnode; 2090c717a561Smaybee zio_t *zio; 2091c717a561Smaybee 2092c717a561Smaybee ASSERT(dmu_tx_is_syncing(tx)); 2093c717a561Smaybee 2094c717a561Smaybee dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 2095c717a561Smaybee 2096c717a561Smaybee mutex_enter(&db->db_mtx); 2097c717a561Smaybee 2098c717a561Smaybee ASSERT(db->db_level > 0); 2099c717a561Smaybee DBUF_VERIFY(db); 2100c717a561Smaybee 2101c717a561Smaybee if (db->db_buf == NULL) { 2102c717a561Smaybee mutex_exit(&db->db_mtx); 2103c717a561Smaybee (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED); 2104c717a561Smaybee mutex_enter(&db->db_mtx); 2105c717a561Smaybee } 2106c717a561Smaybee ASSERT3U(db->db_state, ==, DB_CACHED); 2107c717a561Smaybee ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2108c717a561Smaybee ASSERT(db->db_buf != NULL); 2109c717a561Smaybee 2110c717a561Smaybee dbuf_check_blkptr(dn, db); 2111c717a561Smaybee 2112c717a561Smaybee db->db_data_pending = dr; 2113c717a561Smaybee 2114af2c4821Smaybee mutex_exit(&db->db_mtx); 2115088f3894Sahrens dbuf_write(dr, db->db_buf, tx); 2116c717a561Smaybee 2117c717a561Smaybee zio = dr->dr_zio; 2118c717a561Smaybee mutex_enter(&dr->dt.di.dr_mtx); 2119c717a561Smaybee dbuf_sync_list(&dr->dt.di.dr_children, tx); 2120c717a561Smaybee ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 2121c717a561Smaybee mutex_exit(&dr->dt.di.dr_mtx); 2122c717a561Smaybee zio_nowait(zio); 2123c717a561Smaybee } 2124c717a561Smaybee 2125c717a561Smaybee static void 2126c717a561Smaybee dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 2127c717a561Smaybee { 2128c717a561Smaybee arc_buf_t **datap = &dr->dt.dl.dr_data; 2129c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 2130fa9e4066Sahrens dnode_t *dn = db->db_dnode; 2131503ad85cSMatthew Ahrens objset_t *os = dn->dn_objset; 2132c717a561Smaybee uint64_t txg = tx->tx_txg; 2133fa9e4066Sahrens 2134fa9e4066Sahrens ASSERT(dmu_tx_is_syncing(tx)); 2135fa9e4066Sahrens 2136fa9e4066Sahrens dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 2137fa9e4066Sahrens 2138fa9e4066Sahrens mutex_enter(&db->db_mtx); 2139fa9e4066Sahrens /* 2140fa9e4066Sahrens * To be synced, we must be dirtied. But we 2141fa9e4066Sahrens * might have been freed after the dirty. 2142fa9e4066Sahrens */ 2143fa9e4066Sahrens if (db->db_state == DB_UNCACHED) { 2144fa9e4066Sahrens /* This buffer has been freed since it was dirtied */ 2145fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 2146fa9e4066Sahrens } else if (db->db_state == DB_FILL) { 2147fa9e4066Sahrens /* This buffer was freed and is now being re-filled */ 2148c717a561Smaybee ASSERT(db->db.db_data != dr->dt.dl.dr_data); 2149fa9e4066Sahrens } else { 215082c9918fSTim Haley ASSERT(db->db_state == DB_CACHED || db->db_state == DB_NOFILL); 2151fa9e4066Sahrens } 21529c9dc39aSek110237 DBUF_VERIFY(db); 2153fa9e4066Sahrens 21540a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) { 21550a586ceaSMark Shellenbaum mutex_enter(&dn->dn_mtx); 21560a586ceaSMark Shellenbaum dn->dn_phys->dn_flags |= DNODE_FLAG_SPILL_BLKPTR; 21570a586ceaSMark Shellenbaum mutex_exit(&dn->dn_mtx); 21580a586ceaSMark Shellenbaum } 21590a586ceaSMark Shellenbaum 2160fa9e4066Sahrens /* 2161c717a561Smaybee * If this is a bonus buffer, simply copy the bonus data into the 2162c717a561Smaybee * dnode. It will be written out when the dnode is synced (and it 2163c717a561Smaybee * will be synced, since it must have been dirty for dbuf_sync to 2164c717a561Smaybee * be called). 2165fa9e4066Sahrens */ 21660a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 2167c717a561Smaybee dbuf_dirty_record_t **drp; 21681934e92fSmaybee 2169ea8dc4b6Seschrock ASSERT(*datap != NULL); 2170ea8dc4b6Seschrock ASSERT3U(db->db_level, ==, 0); 2171ea8dc4b6Seschrock ASSERT3U(dn->dn_phys->dn_bonuslen, <=, DN_MAX_BONUSLEN); 2172ea8dc4b6Seschrock bcopy(*datap, DN_BONUS(dn->dn_phys), dn->dn_phys->dn_bonuslen); 21730e8c6158Smaybee if (*datap != db->db.db_data) { 2174ea8dc4b6Seschrock zio_buf_free(*datap, DN_MAX_BONUSLEN); 21755a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 21760e8c6158Smaybee } 2177ea8dc4b6Seschrock db->db_data_pending = NULL; 2178c717a561Smaybee drp = &db->db_last_dirty; 2179c717a561Smaybee while (*drp != dr) 2180c717a561Smaybee drp = &(*drp)->dr_next; 218117f17c2dSbonwick ASSERT(dr->dr_next == NULL); 2182b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 218317f17c2dSbonwick *drp = dr->dr_next; 2184c717a561Smaybee kmem_free(dr, sizeof (dbuf_dirty_record_t)); 2185ea8dc4b6Seschrock ASSERT(db->db_dirtycnt > 0); 2186ea8dc4b6Seschrock db->db_dirtycnt -= 1; 2187b24ab676SJeff Bonwick dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg); 2188ea8dc4b6Seschrock return; 2189ea8dc4b6Seschrock } 2190ea8dc4b6Seschrock 2191c5c6ffa0Smaybee /* 2192f82bfe17Sgw25295 * This function may have dropped the db_mtx lock allowing a dmu_sync 2193f82bfe17Sgw25295 * operation to sneak in. As a result, we need to ensure that we 2194f82bfe17Sgw25295 * don't check the dr_override_state until we have returned from 2195f82bfe17Sgw25295 * dbuf_check_blkptr. 2196f82bfe17Sgw25295 */ 2197f82bfe17Sgw25295 dbuf_check_blkptr(dn, db); 2198f82bfe17Sgw25295 2199f82bfe17Sgw25295 /* 2200c717a561Smaybee * If this buffer is in the middle of an immdiate write, 2201c717a561Smaybee * wait for the synchronous IO to complete. 2202c5c6ffa0Smaybee */ 2203c717a561Smaybee while (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC) { 2204c5c6ffa0Smaybee ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT); 2205c5c6ffa0Smaybee cv_wait(&db->db_changed, &db->db_mtx); 2206c717a561Smaybee ASSERT(dr->dt.dl.dr_override_state != DR_NOT_OVERRIDDEN); 2207c5c6ffa0Smaybee } 2208c717a561Smaybee 2209ab69d62fSMatthew Ahrens if (db->db_state != DB_NOFILL && 2210ab69d62fSMatthew Ahrens dn->dn_object != DMU_META_DNODE_OBJECT && 2211ab69d62fSMatthew Ahrens refcount_count(&db->db_holds) > 1 && 2212b24ab676SJeff Bonwick dr->dt.dl.dr_override_state != DR_OVERRIDDEN && 2213ab69d62fSMatthew Ahrens *datap == db->db_buf) { 2214fa9e4066Sahrens /* 221582c9918fSTim Haley * If this buffer is currently "in use" (i.e., there 221682c9918fSTim Haley * are active holds and db_data still references it), 221782c9918fSTim Haley * then make a copy before we start the write so that 221882c9918fSTim Haley * any modifications from the open txg will not leak 221982c9918fSTim Haley * into this write. 2220fa9e4066Sahrens * 222182c9918fSTim Haley * NOTE: this copy does not need to be made for 222282c9918fSTim Haley * objects only modified in the syncing context (e.g. 222382c9918fSTim Haley * DNONE_DNODE blocks). 2224fa9e4066Sahrens */ 2225ab69d62fSMatthew Ahrens int blksz = arc_buf_size(*datap); 2226ab69d62fSMatthew Ahrens arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 2227ab69d62fSMatthew Ahrens *datap = arc_buf_alloc(os->os_spa, blksz, db, type); 2228c717a561Smaybee bcopy(db->db.db_data, (*datap)->b_data, blksz); 2229fa9e4066Sahrens } 2230c717a561Smaybee db->db_data_pending = dr; 2231fa9e4066Sahrens 2232fa9e4066Sahrens mutex_exit(&db->db_mtx); 2233fa9e4066Sahrens 2234088f3894Sahrens dbuf_write(dr, *datap, tx); 2235c717a561Smaybee 2236c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 2237c717a561Smaybee if (dn->dn_object == DMU_META_DNODE_OBJECT) 2238c717a561Smaybee list_insert_tail(&dn->dn_dirty_records[txg&TXG_MASK], dr); 2239c717a561Smaybee else 2240c717a561Smaybee zio_nowait(dr->dr_zio); 2241fa9e4066Sahrens } 2242c717a561Smaybee 2243c717a561Smaybee void 2244c717a561Smaybee dbuf_sync_list(list_t *list, dmu_tx_t *tx) 2245c717a561Smaybee { 2246c717a561Smaybee dbuf_dirty_record_t *dr; 2247c717a561Smaybee 2248c717a561Smaybee while (dr = list_head(list)) { 2249c717a561Smaybee if (dr->dr_zio != NULL) { 2250c717a561Smaybee /* 2251c717a561Smaybee * If we find an already initialized zio then we 2252c717a561Smaybee * are processing the meta-dnode, and we have finished. 2253c717a561Smaybee * The dbufs for all dnodes are put back on the list 2254c717a561Smaybee * during processing, so that we can zio_wait() 2255c717a561Smaybee * these IOs after initiating all child IOs. 2256c717a561Smaybee */ 2257c717a561Smaybee ASSERT3U(dr->dr_dbuf->db.db_object, ==, 2258c717a561Smaybee DMU_META_DNODE_OBJECT); 2259c717a561Smaybee break; 2260fa9e4066Sahrens } 2261c717a561Smaybee list_remove(list, dr); 2262c717a561Smaybee if (dr->dr_dbuf->db_level > 0) 2263c717a561Smaybee dbuf_sync_indirect(dr, tx); 2264c717a561Smaybee else 2265c717a561Smaybee dbuf_sync_leaf(dr, tx); 2266c717a561Smaybee } 2267c717a561Smaybee } 2268c717a561Smaybee 2269fa9e4066Sahrens /* ARGSUSED */ 2270fa9e4066Sahrens static void 2271c717a561Smaybee dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb) 2272fa9e4066Sahrens { 2273fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 2274e14bb325SJeff Bonwick blkptr_t *bp = zio->io_bp; 2275c717a561Smaybee blkptr_t *bp_orig = &zio->io_bp_orig; 2276b24ab676SJeff Bonwick dnode_t *dn = db->db_dnode; 2277b24ab676SJeff Bonwick spa_t *spa = zio->io_spa; 2278b24ab676SJeff Bonwick int64_t delta; 2279fa9e4066Sahrens uint64_t fill = 0; 2280b24ab676SJeff Bonwick int i; 2281fa9e4066Sahrens 2282e14bb325SJeff Bonwick ASSERT(db->db_blkptr == bp); 2283e14bb325SJeff Bonwick 2284b24ab676SJeff Bonwick delta = bp_get_dsize_sync(spa, bp) - bp_get_dsize_sync(spa, bp_orig); 2285b24ab676SJeff Bonwick dnode_diduse_space(dn, delta - zio->io_prev_space_delta); 2286b24ab676SJeff Bonwick zio->io_prev_space_delta = delta; 2287fa9e4066Sahrens 2288e14bb325SJeff Bonwick if (BP_IS_HOLE(bp)) { 2289b24ab676SJeff Bonwick ASSERT(bp->blk_fill == 0); 2290c717a561Smaybee return; 2291c717a561Smaybee } 2292c717a561Smaybee 22930a586ceaSMark Shellenbaum ASSERT((db->db_blkid != DMU_SPILL_BLKID && 22940a586ceaSMark Shellenbaum BP_GET_TYPE(bp) == dn->dn_type) || 22950a586ceaSMark Shellenbaum (db->db_blkid == DMU_SPILL_BLKID && 22960a586ceaSMark Shellenbaum BP_GET_TYPE(bp) == dn->dn_bonustype)); 2297e14bb325SJeff Bonwick ASSERT(BP_GET_LEVEL(bp) == db->db_level); 2298e14bb325SJeff Bonwick 2299fa9e4066Sahrens mutex_enter(&db->db_mtx); 2300fa9e4066Sahrens 23010a586ceaSMark Shellenbaum #ifdef ZFS_DEBUG 23020a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) { 23030a586ceaSMark Shellenbaum dnode_t *dn = db->db_dnode; 23040a586ceaSMark Shellenbaum ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); 23050a586ceaSMark Shellenbaum ASSERT(!(BP_IS_HOLE(db->db_blkptr)) && 23060a586ceaSMark Shellenbaum db->db_blkptr == &dn->dn_phys->dn_spill); 23070a586ceaSMark Shellenbaum } 23080a586ceaSMark Shellenbaum #endif 23090a586ceaSMark Shellenbaum 2310fa9e4066Sahrens if (db->db_level == 0) { 2311fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 23120a586ceaSMark Shellenbaum if (db->db_blkid > dn->dn_phys->dn_maxblkid && 23130a586ceaSMark Shellenbaum db->db_blkid != DMU_SPILL_BLKID) 2314fa9e4066Sahrens dn->dn_phys->dn_maxblkid = db->db_blkid; 2315fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 2316fa9e4066Sahrens 2317fa9e4066Sahrens if (dn->dn_type == DMU_OT_DNODE) { 2318fa9e4066Sahrens dnode_phys_t *dnp = db->db.db_data; 2319fa9e4066Sahrens for (i = db->db.db_size >> DNODE_SHIFT; i > 0; 2320fa9e4066Sahrens i--, dnp++) { 2321fa9e4066Sahrens if (dnp->dn_type != DMU_OT_NONE) 2322fa9e4066Sahrens fill++; 2323fa9e4066Sahrens } 2324fa9e4066Sahrens } else { 2325fa9e4066Sahrens fill = 1; 2326fa9e4066Sahrens } 2327fa9e4066Sahrens } else { 2328e14bb325SJeff Bonwick blkptr_t *ibp = db->db.db_data; 2329fa9e4066Sahrens ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2330e14bb325SJeff Bonwick for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, ibp++) { 2331e14bb325SJeff Bonwick if (BP_IS_HOLE(ibp)) 2332fa9e4066Sahrens continue; 2333e14bb325SJeff Bonwick fill += ibp->blk_fill; 2334fa9e4066Sahrens } 2335fa9e4066Sahrens } 2336fa9e4066Sahrens 2337e14bb325SJeff Bonwick bp->blk_fill = fill; 2338fa9e4066Sahrens 2339fa9e4066Sahrens mutex_exit(&db->db_mtx); 2340fa9e4066Sahrens } 2341fa9e4066Sahrens 2342c717a561Smaybee /* ARGSUSED */ 2343c717a561Smaybee static void 2344c717a561Smaybee dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb) 2345c717a561Smaybee { 2346c717a561Smaybee dmu_buf_impl_t *db = vdb; 2347b24ab676SJeff Bonwick blkptr_t *bp = zio->io_bp; 2348b24ab676SJeff Bonwick blkptr_t *bp_orig = &zio->io_bp_orig; 2349b24ab676SJeff Bonwick dnode_t *dn = db->db_dnode; 2350b24ab676SJeff Bonwick objset_t *os = dn->dn_objset; 2351c717a561Smaybee uint64_t txg = zio->io_txg; 2352c717a561Smaybee dbuf_dirty_record_t **drp, *dr; 2353c717a561Smaybee 2354c717a561Smaybee ASSERT3U(zio->io_error, ==, 0); 2355b24ab676SJeff Bonwick ASSERT(db->db_blkptr == bp); 2356b24ab676SJeff Bonwick 2357b24ab676SJeff Bonwick if (zio->io_flags & ZIO_FLAG_IO_REWRITE) { 2358b24ab676SJeff Bonwick ASSERT(BP_EQUAL(bp, bp_orig)); 2359b24ab676SJeff Bonwick } else { 2360b24ab676SJeff Bonwick dsl_dataset_t *ds = os->os_dsl_dataset; 2361b24ab676SJeff Bonwick dmu_tx_t *tx = os->os_synctx; 2362b24ab676SJeff Bonwick 2363b24ab676SJeff Bonwick (void) dsl_dataset_block_kill(ds, bp_orig, tx, B_TRUE); 2364b24ab676SJeff Bonwick dsl_dataset_block_born(ds, bp, tx); 2365b24ab676SJeff Bonwick } 2366c717a561Smaybee 2367c717a561Smaybee mutex_enter(&db->db_mtx); 2368c717a561Smaybee 2369b24ab676SJeff Bonwick DBUF_VERIFY(db); 2370b24ab676SJeff Bonwick 2371c717a561Smaybee drp = &db->db_last_dirty; 237217f17c2dSbonwick while ((dr = *drp) != db->db_data_pending) 237317f17c2dSbonwick drp = &dr->dr_next; 237417f17c2dSbonwick ASSERT(!list_link_active(&dr->dr_dirty_node)); 237517f17c2dSbonwick ASSERT(dr->dr_txg == txg); 2376b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 237717f17c2dSbonwick ASSERT(dr->dr_next == NULL); 237817f17c2dSbonwick *drp = dr->dr_next; 2379c717a561Smaybee 23800a586ceaSMark Shellenbaum #ifdef ZFS_DEBUG 23810a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) { 23820a586ceaSMark Shellenbaum dnode_t *dn = db->db_dnode; 23830a586ceaSMark Shellenbaum ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); 23840a586ceaSMark Shellenbaum ASSERT(!(BP_IS_HOLE(db->db_blkptr)) && 23850a586ceaSMark Shellenbaum db->db_blkptr == &dn->dn_phys->dn_spill); 23860a586ceaSMark Shellenbaum } 23870a586ceaSMark Shellenbaum #endif 23880a586ceaSMark Shellenbaum 2389c717a561Smaybee if (db->db_level == 0) { 23900a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 2391c717a561Smaybee ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN); 239282c9918fSTim Haley if (db->db_state != DB_NOFILL) { 2393c717a561Smaybee if (dr->dt.dl.dr_data != db->db_buf) 239482c9918fSTim Haley VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, 239582c9918fSTim Haley db) == 1); 2396b24ab676SJeff Bonwick else if (!arc_released(db->db_buf)) 2397c717a561Smaybee arc_set_callback(db->db_buf, dbuf_do_evict, db); 239882c9918fSTim Haley } 2399c717a561Smaybee } else { 2400c717a561Smaybee ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 2401c717a561Smaybee ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2402c717a561Smaybee if (!BP_IS_HOLE(db->db_blkptr)) { 2403c717a561Smaybee int epbs = 2404c717a561Smaybee dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2405c717a561Smaybee ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, 2406c717a561Smaybee db->db.db_size); 2407c717a561Smaybee ASSERT3U(dn->dn_phys->dn_maxblkid 2408c717a561Smaybee >> (db->db_level * epbs), >=, db->db_blkid); 2409c717a561Smaybee arc_set_callback(db->db_buf, dbuf_do_evict, db); 2410c717a561Smaybee } 2411c25056deSgw25295 mutex_destroy(&dr->dt.di.dr_mtx); 2412c25056deSgw25295 list_destroy(&dr->dt.di.dr_children); 2413c717a561Smaybee } 2414c717a561Smaybee kmem_free(dr, sizeof (dbuf_dirty_record_t)); 2415c717a561Smaybee 2416c717a561Smaybee cv_broadcast(&db->db_changed); 2417c717a561Smaybee ASSERT(db->db_dirtycnt > 0); 2418c717a561Smaybee db->db_dirtycnt -= 1; 2419c717a561Smaybee db->db_data_pending = NULL; 2420b24ab676SJeff Bonwick dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg); 2421b24ab676SJeff Bonwick } 2422b24ab676SJeff Bonwick 2423b24ab676SJeff Bonwick static void 2424b24ab676SJeff Bonwick dbuf_write_nofill_ready(zio_t *zio) 2425b24ab676SJeff Bonwick { 2426b24ab676SJeff Bonwick dbuf_write_ready(zio, NULL, zio->io_private); 2427b24ab676SJeff Bonwick } 2428b24ab676SJeff Bonwick 2429b24ab676SJeff Bonwick static void 2430b24ab676SJeff Bonwick dbuf_write_nofill_done(zio_t *zio) 2431b24ab676SJeff Bonwick { 2432b24ab676SJeff Bonwick dbuf_write_done(zio, NULL, zio->io_private); 2433b24ab676SJeff Bonwick } 2434b24ab676SJeff Bonwick 2435b24ab676SJeff Bonwick static void 2436b24ab676SJeff Bonwick dbuf_write_override_ready(zio_t *zio) 2437b24ab676SJeff Bonwick { 2438b24ab676SJeff Bonwick dbuf_dirty_record_t *dr = zio->io_private; 2439b24ab676SJeff Bonwick dmu_buf_impl_t *db = dr->dr_dbuf; 2440b24ab676SJeff Bonwick 2441b24ab676SJeff Bonwick dbuf_write_ready(zio, NULL, db); 2442b24ab676SJeff Bonwick } 2443b24ab676SJeff Bonwick 2444b24ab676SJeff Bonwick static void 2445b24ab676SJeff Bonwick dbuf_write_override_done(zio_t *zio) 2446b24ab676SJeff Bonwick { 2447b24ab676SJeff Bonwick dbuf_dirty_record_t *dr = zio->io_private; 2448b24ab676SJeff Bonwick dmu_buf_impl_t *db = dr->dr_dbuf; 2449b24ab676SJeff Bonwick blkptr_t *obp = &dr->dt.dl.dr_overridden_by; 2450b24ab676SJeff Bonwick 2451b24ab676SJeff Bonwick mutex_enter(&db->db_mtx); 2452b24ab676SJeff Bonwick if (!BP_EQUAL(zio->io_bp, obp)) { 2453b24ab676SJeff Bonwick if (!BP_IS_HOLE(obp)) 2454b24ab676SJeff Bonwick dsl_free(spa_get_dsl(zio->io_spa), zio->io_txg, obp); 2455b24ab676SJeff Bonwick arc_release(dr->dt.dl.dr_data, db); 2456b24ab676SJeff Bonwick } 2457c717a561Smaybee mutex_exit(&db->db_mtx); 2458c717a561Smaybee 2459b24ab676SJeff Bonwick dbuf_write_done(zio, NULL, db); 2460b24ab676SJeff Bonwick } 2461c717a561Smaybee 2462b24ab676SJeff Bonwick static void 2463b24ab676SJeff Bonwick dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx) 2464b24ab676SJeff Bonwick { 2465b24ab676SJeff Bonwick dmu_buf_impl_t *db = dr->dr_dbuf; 2466b24ab676SJeff Bonwick dnode_t *dn = db->db_dnode; 2467b24ab676SJeff Bonwick objset_t *os = dn->dn_objset; 2468b24ab676SJeff Bonwick dmu_buf_impl_t *parent = db->db_parent; 2469b24ab676SJeff Bonwick uint64_t txg = tx->tx_txg; 2470b24ab676SJeff Bonwick zbookmark_t zb; 2471b24ab676SJeff Bonwick zio_prop_t zp; 2472b24ab676SJeff Bonwick zio_t *zio; 24730a586ceaSMark Shellenbaum int wp_flag = 0; 2474b24ab676SJeff Bonwick 2475b24ab676SJeff Bonwick if (db->db_state != DB_NOFILL) { 2476b24ab676SJeff Bonwick if (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE) { 2477b24ab676SJeff Bonwick /* 2478b24ab676SJeff Bonwick * Private object buffers are released here rather 2479b24ab676SJeff Bonwick * than in dbuf_dirty() since they are only modified 2480b24ab676SJeff Bonwick * in the syncing context and we don't want the 2481b24ab676SJeff Bonwick * overhead of making multiple copies of the data. 2482b24ab676SJeff Bonwick */ 2483b24ab676SJeff Bonwick if (BP_IS_HOLE(db->db_blkptr)) { 2484b24ab676SJeff Bonwick arc_buf_thaw(data); 2485b24ab676SJeff Bonwick } else { 2486*3f9d6ad7SLin Ling dbuf_release_bp(db); 2487b24ab676SJeff Bonwick } 2488b24ab676SJeff Bonwick } 2489b24ab676SJeff Bonwick } 2490b24ab676SJeff Bonwick 2491b24ab676SJeff Bonwick if (parent != dn->dn_dbuf) { 2492b24ab676SJeff Bonwick ASSERT(parent && parent->db_data_pending); 2493b24ab676SJeff Bonwick ASSERT(db->db_level == parent->db_level-1); 2494b24ab676SJeff Bonwick ASSERT(arc_released(parent->db_buf)); 2495b24ab676SJeff Bonwick zio = parent->db_data_pending->dr_zio; 2496b24ab676SJeff Bonwick } else { 24970a586ceaSMark Shellenbaum ASSERT((db->db_level == dn->dn_phys->dn_nlevels-1 && 24980a586ceaSMark Shellenbaum db->db_blkid != DMU_SPILL_BLKID) || 24990a586ceaSMark Shellenbaum (db->db_blkid == DMU_SPILL_BLKID && db->db_level == 0)); 25000a586ceaSMark Shellenbaum if (db->db_blkid != DMU_SPILL_BLKID) 2501b24ab676SJeff Bonwick ASSERT3P(db->db_blkptr, ==, 2502b24ab676SJeff Bonwick &dn->dn_phys->dn_blkptr[db->db_blkid]); 2503b24ab676SJeff Bonwick zio = dn->dn_zio; 2504b24ab676SJeff Bonwick } 2505b24ab676SJeff Bonwick 2506b24ab676SJeff Bonwick ASSERT(db->db_level == 0 || data == db->db_buf); 2507b24ab676SJeff Bonwick ASSERT3U(db->db_blkptr->blk_birth, <=, txg); 2508b24ab676SJeff Bonwick ASSERT(zio); 2509b24ab676SJeff Bonwick 2510b24ab676SJeff Bonwick SET_BOOKMARK(&zb, os->os_dsl_dataset ? 2511b24ab676SJeff Bonwick os->os_dsl_dataset->ds_object : DMU_META_OBJSET, 2512b24ab676SJeff Bonwick db->db.db_object, db->db_level, db->db_blkid); 2513b24ab676SJeff Bonwick 25140a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) 25150a586ceaSMark Shellenbaum wp_flag = WP_SPILL; 25160a586ceaSMark Shellenbaum wp_flag |= (db->db_state == DB_NOFILL) ? WP_NOFILL : 0; 25170a586ceaSMark Shellenbaum 25180a586ceaSMark Shellenbaum dmu_write_policy(os, dn, db->db_level, wp_flag, &zp); 2519b24ab676SJeff Bonwick 2520b24ab676SJeff Bonwick if (db->db_level == 0 && dr->dt.dl.dr_override_state == DR_OVERRIDDEN) { 2521b24ab676SJeff Bonwick ASSERT(db->db_state != DB_NOFILL); 2522b24ab676SJeff Bonwick dr->dr_zio = zio_write(zio, os->os_spa, txg, 2523b24ab676SJeff Bonwick db->db_blkptr, data->b_data, arc_buf_size(data), &zp, 2524b24ab676SJeff Bonwick dbuf_write_override_ready, dbuf_write_override_done, dr, 2525b24ab676SJeff Bonwick ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); 2526b24ab676SJeff Bonwick mutex_enter(&db->db_mtx); 2527b24ab676SJeff Bonwick dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 2528b24ab676SJeff Bonwick zio_write_override(dr->dr_zio, &dr->dt.dl.dr_overridden_by, 2529b24ab676SJeff Bonwick dr->dt.dl.dr_copies); 2530b24ab676SJeff Bonwick mutex_exit(&db->db_mtx); 2531b24ab676SJeff Bonwick } else if (db->db_state == DB_NOFILL) { 2532b24ab676SJeff Bonwick ASSERT(zp.zp_checksum == ZIO_CHECKSUM_OFF); 2533b24ab676SJeff Bonwick dr->dr_zio = zio_write(zio, os->os_spa, txg, 2534b24ab676SJeff Bonwick db->db_blkptr, NULL, db->db.db_size, &zp, 2535b24ab676SJeff Bonwick dbuf_write_nofill_ready, dbuf_write_nofill_done, db, 2536b24ab676SJeff Bonwick ZIO_PRIORITY_ASYNC_WRITE, 2537b24ab676SJeff Bonwick ZIO_FLAG_MUSTSUCCEED | ZIO_FLAG_NODATA, &zb); 2538b24ab676SJeff Bonwick } else { 2539b24ab676SJeff Bonwick ASSERT(arc_released(data)); 2540b24ab676SJeff Bonwick dr->dr_zio = arc_write(zio, os->os_spa, txg, 2541b24ab676SJeff Bonwick db->db_blkptr, data, DBUF_IS_L2CACHEABLE(db), &zp, 2542b24ab676SJeff Bonwick dbuf_write_ready, dbuf_write_done, db, 2543b24ab676SJeff Bonwick ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); 2544b24ab676SJeff Bonwick } 2545fa9e4066Sahrens } 2546