1fa9e4066Sahrens /* 2fa9e4066Sahrens * CDDL HEADER START 3fa9e4066Sahrens * 4fa9e4066Sahrens * The contents of this file are subject to the terms of the 5f65e61c0Sahrens * Common Development and Distribution License (the "License"). 6f65e61c0Sahrens * You may not use this file except in compliance with the License. 7fa9e4066Sahrens * 8fa9e4066Sahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9fa9e4066Sahrens * or http://www.opensolaris.org/os/licensing. 10fa9e4066Sahrens * See the License for the specific language governing permissions 11fa9e4066Sahrens * and limitations under the License. 12fa9e4066Sahrens * 13fa9e4066Sahrens * When distributing Covered Code, include this CDDL HEADER in each 14fa9e4066Sahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15fa9e4066Sahrens * If applicable, add the following below this CDDL HEADER, with the 16fa9e4066Sahrens * fields enclosed by brackets "[]" replaced with your own identifying 17fa9e4066Sahrens * information: Portions Copyright [yyyy] [name of copyright owner] 18fa9e4066Sahrens * 19fa9e4066Sahrens * CDDL HEADER END 20fa9e4066Sahrens */ 21fa9e4066Sahrens /* 22*c242f9a0Schunli zhang - Sun Microsystems - Irvine United States * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 23fa9e4066Sahrens * Use is subject to license terms. 24fa9e4066Sahrens */ 25fa9e4066Sahrens 26fa9e4066Sahrens #include <sys/zfs_context.h> 27fa9e4066Sahrens #include <sys/dmu.h> 28fa9e4066Sahrens #include <sys/dmu_impl.h> 29fa9e4066Sahrens #include <sys/dbuf.h> 30fa9e4066Sahrens #include <sys/dmu_objset.h> 31fa9e4066Sahrens #include <sys/dsl_dataset.h> 32fa9e4066Sahrens #include <sys/dsl_dir.h> 33fa9e4066Sahrens #include <sys/dmu_tx.h> 34fa9e4066Sahrens #include <sys/spa.h> 35fa9e4066Sahrens #include <sys/zio.h> 36fa9e4066Sahrens #include <sys/dmu_zfetch.h> 37fa9e4066Sahrens 38fa9e4066Sahrens static void dbuf_destroy(dmu_buf_impl_t *db); 39fa9e4066Sahrens static int dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx); 40088f3894Sahrens static void dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx); 41fa9e4066Sahrens 42fa9e4066Sahrens /* 43fa9e4066Sahrens * Global data structures and functions for the dbuf cache. 44fa9e4066Sahrens */ 45fa9e4066Sahrens static kmem_cache_t *dbuf_cache; 46fa9e4066Sahrens 47fa9e4066Sahrens /* ARGSUSED */ 48fa9e4066Sahrens static int 49fa9e4066Sahrens dbuf_cons(void *vdb, void *unused, int kmflag) 50fa9e4066Sahrens { 51fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 52fa9e4066Sahrens bzero(db, sizeof (dmu_buf_impl_t)); 53fa9e4066Sahrens 54fa9e4066Sahrens mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL); 55fa9e4066Sahrens cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL); 56fa9e4066Sahrens refcount_create(&db->db_holds); 57fa9e4066Sahrens return (0); 58fa9e4066Sahrens } 59fa9e4066Sahrens 60fa9e4066Sahrens /* ARGSUSED */ 61fa9e4066Sahrens static void 62fa9e4066Sahrens dbuf_dest(void *vdb, void *unused) 63fa9e4066Sahrens { 64fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 65fa9e4066Sahrens mutex_destroy(&db->db_mtx); 66fa9e4066Sahrens cv_destroy(&db->db_changed); 67fa9e4066Sahrens refcount_destroy(&db->db_holds); 68fa9e4066Sahrens } 69fa9e4066Sahrens 70fa9e4066Sahrens /* 71fa9e4066Sahrens * dbuf hash table routines 72fa9e4066Sahrens */ 73fa9e4066Sahrens static dbuf_hash_table_t dbuf_hash_table; 74fa9e4066Sahrens 75fa9e4066Sahrens static uint64_t dbuf_hash_count; 76fa9e4066Sahrens 77fa9e4066Sahrens static uint64_t 78fa9e4066Sahrens dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid) 79fa9e4066Sahrens { 80fa9e4066Sahrens uintptr_t osv = (uintptr_t)os; 81fa9e4066Sahrens uint64_t crc = -1ULL; 82fa9e4066Sahrens 83fa9e4066Sahrens ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 84fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (lvl)) & 0xFF]; 85fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF]; 86fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF]; 87fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF]; 88fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 0)) & 0xFF]; 89fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 8)) & 0xFF]; 90fa9e4066Sahrens 91fa9e4066Sahrens crc ^= (osv>>14) ^ (obj>>16) ^ (blkid>>16); 92fa9e4066Sahrens 93fa9e4066Sahrens return (crc); 94fa9e4066Sahrens } 95fa9e4066Sahrens 96fa9e4066Sahrens #define DBUF_HASH(os, obj, level, blkid) dbuf_hash(os, obj, level, blkid); 97fa9e4066Sahrens 98fa9e4066Sahrens #define DBUF_EQUAL(dbuf, os, obj, level, blkid) \ 99fa9e4066Sahrens ((dbuf)->db.db_object == (obj) && \ 100fa9e4066Sahrens (dbuf)->db_objset == (os) && \ 101fa9e4066Sahrens (dbuf)->db_level == (level) && \ 102fa9e4066Sahrens (dbuf)->db_blkid == (blkid)) 103fa9e4066Sahrens 104fa9e4066Sahrens dmu_buf_impl_t * 105fa9e4066Sahrens dbuf_find(dnode_t *dn, uint8_t level, uint64_t blkid) 106fa9e4066Sahrens { 107fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 108503ad85cSMatthew Ahrens objset_t *os = dn->dn_objset; 109fa9e4066Sahrens uint64_t obj = dn->dn_object; 110fa9e4066Sahrens uint64_t hv = DBUF_HASH(os, obj, level, blkid); 111fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 112fa9e4066Sahrens dmu_buf_impl_t *db; 113fa9e4066Sahrens 114fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 115fa9e4066Sahrens for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) { 116fa9e4066Sahrens if (DBUF_EQUAL(db, os, obj, level, blkid)) { 117fa9e4066Sahrens mutex_enter(&db->db_mtx); 118ea8dc4b6Seschrock if (db->db_state != DB_EVICTING) { 119fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 120fa9e4066Sahrens return (db); 121fa9e4066Sahrens } 122fa9e4066Sahrens mutex_exit(&db->db_mtx); 123fa9e4066Sahrens } 124fa9e4066Sahrens } 125fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 126fa9e4066Sahrens return (NULL); 127fa9e4066Sahrens } 128fa9e4066Sahrens 129fa9e4066Sahrens /* 130fa9e4066Sahrens * Insert an entry into the hash table. If there is already an element 131fa9e4066Sahrens * equal to elem in the hash table, then the already existing element 132fa9e4066Sahrens * will be returned and the new element will not be inserted. 133fa9e4066Sahrens * Otherwise returns NULL. 134fa9e4066Sahrens */ 135fa9e4066Sahrens static dmu_buf_impl_t * 136fa9e4066Sahrens dbuf_hash_insert(dmu_buf_impl_t *db) 137fa9e4066Sahrens { 138fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 139503ad85cSMatthew Ahrens objset_t *os = db->db_objset; 140fa9e4066Sahrens uint64_t obj = db->db.db_object; 141fa9e4066Sahrens int level = db->db_level; 142fa9e4066Sahrens uint64_t blkid = db->db_blkid; 143fa9e4066Sahrens uint64_t hv = DBUF_HASH(os, obj, level, blkid); 144fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 145fa9e4066Sahrens dmu_buf_impl_t *dbf; 146fa9e4066Sahrens 147fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 148fa9e4066Sahrens for (dbf = h->hash_table[idx]; dbf != NULL; dbf = dbf->db_hash_next) { 149fa9e4066Sahrens if (DBUF_EQUAL(dbf, os, obj, level, blkid)) { 150fa9e4066Sahrens mutex_enter(&dbf->db_mtx); 151ea8dc4b6Seschrock if (dbf->db_state != DB_EVICTING) { 152fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 153fa9e4066Sahrens return (dbf); 154fa9e4066Sahrens } 155fa9e4066Sahrens mutex_exit(&dbf->db_mtx); 156fa9e4066Sahrens } 157fa9e4066Sahrens } 158fa9e4066Sahrens 159fa9e4066Sahrens mutex_enter(&db->db_mtx); 160fa9e4066Sahrens db->db_hash_next = h->hash_table[idx]; 161fa9e4066Sahrens h->hash_table[idx] = db; 162fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 163fa9e4066Sahrens atomic_add_64(&dbuf_hash_count, 1); 164fa9e4066Sahrens 165fa9e4066Sahrens return (NULL); 166fa9e4066Sahrens } 167fa9e4066Sahrens 168fa9e4066Sahrens /* 169fa9e4066Sahrens * Remove an entry from the hash table. This operation will 170fa9e4066Sahrens * fail if there are any existing holds on the db. 171fa9e4066Sahrens */ 172fa9e4066Sahrens static void 173fa9e4066Sahrens dbuf_hash_remove(dmu_buf_impl_t *db) 174fa9e4066Sahrens { 175fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 176fa9e4066Sahrens uint64_t hv = DBUF_HASH(db->db_objset, db->db.db_object, 177fa9e4066Sahrens db->db_level, db->db_blkid); 178fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 179fa9e4066Sahrens dmu_buf_impl_t *dbf, **dbp; 180fa9e4066Sahrens 181fa9e4066Sahrens /* 182fa9e4066Sahrens * We musn't hold db_mtx to maintin lock ordering: 183fa9e4066Sahrens * DBUF_HASH_MUTEX > db_mtx. 184fa9e4066Sahrens */ 185fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 186ea8dc4b6Seschrock ASSERT(db->db_state == DB_EVICTING); 187fa9e4066Sahrens ASSERT(!MUTEX_HELD(&db->db_mtx)); 188fa9e4066Sahrens 189fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 190fa9e4066Sahrens dbp = &h->hash_table[idx]; 191fa9e4066Sahrens while ((dbf = *dbp) != db) { 192fa9e4066Sahrens dbp = &dbf->db_hash_next; 193fa9e4066Sahrens ASSERT(dbf != NULL); 194fa9e4066Sahrens } 195fa9e4066Sahrens *dbp = db->db_hash_next; 196fa9e4066Sahrens db->db_hash_next = NULL; 197fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 198fa9e4066Sahrens atomic_add_64(&dbuf_hash_count, -1); 199fa9e4066Sahrens } 200fa9e4066Sahrens 201ea8dc4b6Seschrock static arc_evict_func_t dbuf_do_evict; 202fa9e4066Sahrens 203fa9e4066Sahrens static void 204fa9e4066Sahrens dbuf_evict_user(dmu_buf_impl_t *db) 205fa9e4066Sahrens { 206fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 207fa9e4066Sahrens 208c717a561Smaybee if (db->db_level != 0 || db->db_evict_func == NULL) 209fa9e4066Sahrens return; 210fa9e4066Sahrens 211c717a561Smaybee if (db->db_user_data_ptr_ptr) 212c717a561Smaybee *db->db_user_data_ptr_ptr = db->db.db_data; 213c717a561Smaybee db->db_evict_func(&db->db, db->db_user_ptr); 214c717a561Smaybee db->db_user_ptr = NULL; 215c717a561Smaybee db->db_user_data_ptr_ptr = NULL; 216c717a561Smaybee db->db_evict_func = NULL; 217fa9e4066Sahrens } 218fa9e4066Sahrens 219fa9e4066Sahrens void 220ea8dc4b6Seschrock dbuf_evict(dmu_buf_impl_t *db) 221ea8dc4b6Seschrock { 222ea8dc4b6Seschrock ASSERT(MUTEX_HELD(&db->db_mtx)); 223ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 224c717a561Smaybee ASSERT(db->db_data_pending == NULL); 225ea8dc4b6Seschrock 226ea8dc4b6Seschrock dbuf_clear(db); 227ea8dc4b6Seschrock dbuf_destroy(db); 228ea8dc4b6Seschrock } 229ea8dc4b6Seschrock 230ea8dc4b6Seschrock void 231fa9e4066Sahrens dbuf_init(void) 232fa9e4066Sahrens { 233ea8dc4b6Seschrock uint64_t hsize = 1ULL << 16; 234fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 235fa9e4066Sahrens int i; 236fa9e4066Sahrens 237fa9e4066Sahrens /* 238fa9e4066Sahrens * The hash table is big enough to fill all of physical memory 239ea8dc4b6Seschrock * with an average 4K block size. The table will take up 240ea8dc4b6Seschrock * totalmem*sizeof(void*)/4K (i.e. 2MB/GB with 8-byte pointers). 241fa9e4066Sahrens */ 242ea8dc4b6Seschrock while (hsize * 4096 < physmem * PAGESIZE) 243fa9e4066Sahrens hsize <<= 1; 244fa9e4066Sahrens 245ea8dc4b6Seschrock retry: 246fa9e4066Sahrens h->hash_table_mask = hsize - 1; 247ea8dc4b6Seschrock h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP); 248ea8dc4b6Seschrock if (h->hash_table == NULL) { 249ea8dc4b6Seschrock /* XXX - we should really return an error instead of assert */ 250ea8dc4b6Seschrock ASSERT(hsize > (1ULL << 10)); 251ea8dc4b6Seschrock hsize >>= 1; 252ea8dc4b6Seschrock goto retry; 253ea8dc4b6Seschrock } 254fa9e4066Sahrens 255fa9e4066Sahrens dbuf_cache = kmem_cache_create("dmu_buf_impl_t", 256fa9e4066Sahrens sizeof (dmu_buf_impl_t), 257fa9e4066Sahrens 0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0); 258fa9e4066Sahrens 259fa9e4066Sahrens for (i = 0; i < DBUF_MUTEXES; i++) 260fa9e4066Sahrens mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL); 261fa9e4066Sahrens } 262fa9e4066Sahrens 263fa9e4066Sahrens void 264fa9e4066Sahrens dbuf_fini(void) 265fa9e4066Sahrens { 266fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 267fa9e4066Sahrens int i; 268fa9e4066Sahrens 269fa9e4066Sahrens for (i = 0; i < DBUF_MUTEXES; i++) 270fa9e4066Sahrens mutex_destroy(&h->hash_mutexes[i]); 271fa9e4066Sahrens kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *)); 272fa9e4066Sahrens kmem_cache_destroy(dbuf_cache); 273fa9e4066Sahrens } 274fa9e4066Sahrens 275fa9e4066Sahrens /* 276fa9e4066Sahrens * Other stuff. 277fa9e4066Sahrens */ 278fa9e4066Sahrens 2799c9dc39aSek110237 #ifdef ZFS_DEBUG 280fa9e4066Sahrens static void 281fa9e4066Sahrens dbuf_verify(dmu_buf_impl_t *db) 282fa9e4066Sahrens { 283fa9e4066Sahrens dnode_t *dn = db->db_dnode; 284b24ab676SJeff Bonwick dbuf_dirty_record_t *dr; 285fa9e4066Sahrens 286fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 287fa9e4066Sahrens 288fa9e4066Sahrens if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY)) 289fa9e4066Sahrens return; 290fa9e4066Sahrens 291fa9e4066Sahrens ASSERT(db->db_objset != NULL); 292fa9e4066Sahrens if (dn == NULL) { 293fa9e4066Sahrens ASSERT(db->db_parent == NULL); 294fa9e4066Sahrens ASSERT(db->db_blkptr == NULL); 295fa9e4066Sahrens } else { 296fa9e4066Sahrens ASSERT3U(db->db.db_object, ==, dn->dn_object); 297fa9e4066Sahrens ASSERT3P(db->db_objset, ==, dn->dn_objset); 298fa9e4066Sahrens ASSERT3U(db->db_level, <, dn->dn_nlevels); 299ea8dc4b6Seschrock ASSERT(db->db_blkid == DB_BONUS_BLKID || 300ea8dc4b6Seschrock list_head(&dn->dn_dbufs)); 301fa9e4066Sahrens } 302fa9e4066Sahrens if (db->db_blkid == DB_BONUS_BLKID) { 303fa9e4066Sahrens ASSERT(dn != NULL); 3041934e92fSmaybee ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 305fa9e4066Sahrens ASSERT3U(db->db.db_offset, ==, DB_BONUS_BLKID); 306fa9e4066Sahrens } else { 307fa9e4066Sahrens ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size); 308fa9e4066Sahrens } 309fa9e4066Sahrens 310b24ab676SJeff Bonwick for (dr = db->db_data_pending; dr != NULL; dr = dr->dr_next) 311b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 312b24ab676SJeff Bonwick 313b24ab676SJeff Bonwick for (dr = db->db_last_dirty; dr != NULL; dr = dr->dr_next) 314b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 315b24ab676SJeff Bonwick 31688b7b0f2SMatthew Ahrens /* 31788b7b0f2SMatthew Ahrens * We can't assert that db_size matches dn_datablksz because it 31888b7b0f2SMatthew Ahrens * can be momentarily different when another thread is doing 31988b7b0f2SMatthew Ahrens * dnode_set_blksz(). 32088b7b0f2SMatthew Ahrens */ 32188b7b0f2SMatthew Ahrens if (db->db_level == 0 && db->db.db_object == DMU_META_DNODE_OBJECT) { 322b24ab676SJeff Bonwick dr = db->db_data_pending; 323fa9e4066Sahrens /* 32488b7b0f2SMatthew Ahrens * It should only be modified in syncing context, so 32588b7b0f2SMatthew Ahrens * make sure we only have one copy of the data. 326fa9e4066Sahrens */ 327c717a561Smaybee ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf); 328fa9e4066Sahrens } 329fa9e4066Sahrens 330fa9e4066Sahrens /* verify db->db_blkptr */ 331fa9e4066Sahrens if (db->db_blkptr) { 332fa9e4066Sahrens if (db->db_parent == dn->dn_dbuf) { 333fa9e4066Sahrens /* db is pointed to by the dnode */ 334fa9e4066Sahrens /* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */ 33514843421SMatthew Ahrens if (DMU_OBJECT_IS_SPECIAL(db->db.db_object)) 336fa9e4066Sahrens ASSERT(db->db_parent == NULL); 337fa9e4066Sahrens else 338fa9e4066Sahrens ASSERT(db->db_parent != NULL); 339fa9e4066Sahrens ASSERT3P(db->db_blkptr, ==, 340fa9e4066Sahrens &dn->dn_phys->dn_blkptr[db->db_blkid]); 341fa9e4066Sahrens } else { 342fa9e4066Sahrens /* db is pointed to by an indirect block */ 343fa9e4066Sahrens int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT; 344fa9e4066Sahrens ASSERT3U(db->db_parent->db_level, ==, db->db_level+1); 345fa9e4066Sahrens ASSERT3U(db->db_parent->db.db_object, ==, 346fa9e4066Sahrens db->db.db_object); 347fa9e4066Sahrens /* 348fa9e4066Sahrens * dnode_grow_indblksz() can make this fail if we don't 349fa9e4066Sahrens * have the struct_rwlock. XXX indblksz no longer 350fa9e4066Sahrens * grows. safe to do this now? 351fa9e4066Sahrens */ 352fa9e4066Sahrens if (RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)) { 353fa9e4066Sahrens ASSERT3P(db->db_blkptr, ==, 354fa9e4066Sahrens ((blkptr_t *)db->db_parent->db.db_data + 355fa9e4066Sahrens db->db_blkid % epb)); 356fa9e4066Sahrens } 357fa9e4066Sahrens } 358fa9e4066Sahrens } 359fa9e4066Sahrens if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) && 360fa9e4066Sahrens db->db.db_data && db->db_blkid != DB_BONUS_BLKID && 361fa9e4066Sahrens db->db_state != DB_FILL && !dn->dn_free_txg) { 362fa9e4066Sahrens /* 363fa9e4066Sahrens * If the blkptr isn't set but they have nonzero data, 364fa9e4066Sahrens * it had better be dirty, otherwise we'll lose that 365fa9e4066Sahrens * data when we evict this buffer. 366fa9e4066Sahrens */ 367fa9e4066Sahrens if (db->db_dirtycnt == 0) { 368fa9e4066Sahrens uint64_t *buf = db->db.db_data; 369fa9e4066Sahrens int i; 370fa9e4066Sahrens 371fa9e4066Sahrens for (i = 0; i < db->db.db_size >> 3; i++) { 372fa9e4066Sahrens ASSERT(buf[i] == 0); 373fa9e4066Sahrens } 374fa9e4066Sahrens } 375fa9e4066Sahrens } 376fa9e4066Sahrens } 3779c9dc39aSek110237 #endif 378fa9e4066Sahrens 379fa9e4066Sahrens static void 380fa9e4066Sahrens dbuf_update_data(dmu_buf_impl_t *db) 381fa9e4066Sahrens { 382fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 383c717a561Smaybee if (db->db_level == 0 && db->db_user_data_ptr_ptr) { 384fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 385c717a561Smaybee *db->db_user_data_ptr_ptr = db->db.db_data; 386fa9e4066Sahrens } 387fa9e4066Sahrens } 388fa9e4066Sahrens 389fa9e4066Sahrens static void 390fa9e4066Sahrens dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf) 391fa9e4066Sahrens { 392fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 393ea8dc4b6Seschrock ASSERT(db->db_buf == NULL || !arc_has_callback(db->db_buf)); 394fa9e4066Sahrens db->db_buf = buf; 395ea8dc4b6Seschrock if (buf != NULL) { 396ea8dc4b6Seschrock ASSERT(buf->b_data != NULL); 397fa9e4066Sahrens db->db.db_data = buf->b_data; 398ea8dc4b6Seschrock if (!arc_released(buf)) 399ea8dc4b6Seschrock arc_set_callback(buf, dbuf_do_evict, db); 400fa9e4066Sahrens dbuf_update_data(db); 401ea8dc4b6Seschrock } else { 402ea8dc4b6Seschrock dbuf_evict_user(db); 403ea8dc4b6Seschrock db->db.db_data = NULL; 40482c9918fSTim Haley if (db->db_state != DB_NOFILL) 405ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 406ea8dc4b6Seschrock } 407fa9e4066Sahrens } 408fa9e4066Sahrens 409*c242f9a0Schunli zhang - Sun Microsystems - Irvine United States /* 410*c242f9a0Schunli zhang - Sun Microsystems - Irvine United States * Loan out an arc_buf for read. Return the loaned arc_buf. 411*c242f9a0Schunli zhang - Sun Microsystems - Irvine United States */ 412*c242f9a0Schunli zhang - Sun Microsystems - Irvine United States arc_buf_t * 413*c242f9a0Schunli zhang - Sun Microsystems - Irvine United States dbuf_loan_arcbuf(dmu_buf_impl_t *db) 414*c242f9a0Schunli zhang - Sun Microsystems - Irvine United States { 415*c242f9a0Schunli zhang - Sun Microsystems - Irvine United States arc_buf_t *abuf; 416*c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 417*c242f9a0Schunli zhang - Sun Microsystems - Irvine United States mutex_enter(&db->db_mtx); 418*c242f9a0Schunli zhang - Sun Microsystems - Irvine United States if (arc_released(db->db_buf) || refcount_count(&db->db_holds) > 1) { 419*c242f9a0Schunli zhang - Sun Microsystems - Irvine United States int blksz = db->db.db_size; 420*c242f9a0Schunli zhang - Sun Microsystems - Irvine United States mutex_exit(&db->db_mtx); 421*c242f9a0Schunli zhang - Sun Microsystems - Irvine United States abuf = arc_loan_buf(db->db_dnode->dn_objset->os_spa, blksz); 422*c242f9a0Schunli zhang - Sun Microsystems - Irvine United States bcopy(db->db.db_data, abuf->b_data, blksz); 423*c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } else { 424*c242f9a0Schunli zhang - Sun Microsystems - Irvine United States abuf = db->db_buf; 425*c242f9a0Schunli zhang - Sun Microsystems - Irvine United States arc_loan_inuse_buf(abuf, db); 426*c242f9a0Schunli zhang - Sun Microsystems - Irvine United States dbuf_set_data(db, NULL); 427*c242f9a0Schunli zhang - Sun Microsystems - Irvine United States mutex_exit(&db->db_mtx); 428*c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } 429*c242f9a0Schunli zhang - Sun Microsystems - Irvine United States return (abuf); 430*c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } 431*c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 432fa9e4066Sahrens uint64_t 433fa9e4066Sahrens dbuf_whichblock(dnode_t *dn, uint64_t offset) 434fa9e4066Sahrens { 435fa9e4066Sahrens if (dn->dn_datablkshift) { 436fa9e4066Sahrens return (offset >> dn->dn_datablkshift); 437fa9e4066Sahrens } else { 438fa9e4066Sahrens ASSERT3U(offset, <, dn->dn_datablksz); 439fa9e4066Sahrens return (0); 440fa9e4066Sahrens } 441fa9e4066Sahrens } 442fa9e4066Sahrens 443fa9e4066Sahrens static void 444fa9e4066Sahrens dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb) 445fa9e4066Sahrens { 446fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 447fa9e4066Sahrens 448fa9e4066Sahrens mutex_enter(&db->db_mtx); 449fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_READ); 450fa9e4066Sahrens /* 451fa9e4066Sahrens * All reads are synchronous, so we must have a hold on the dbuf 452fa9e4066Sahrens */ 453fa9e4066Sahrens ASSERT(refcount_count(&db->db_holds) > 0); 454ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 455fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 456c717a561Smaybee if (db->db_level == 0 && db->db_freed_in_flight) { 457fa9e4066Sahrens /* we were freed in flight; disregard any error */ 458fa9e4066Sahrens arc_release(buf, db); 459fa9e4066Sahrens bzero(buf->b_data, db->db.db_size); 4606b4acc8bSahrens arc_buf_freeze(buf); 461c717a561Smaybee db->db_freed_in_flight = FALSE; 462fa9e4066Sahrens dbuf_set_data(db, buf); 463fa9e4066Sahrens db->db_state = DB_CACHED; 464fa9e4066Sahrens } else if (zio == NULL || zio->io_error == 0) { 465fa9e4066Sahrens dbuf_set_data(db, buf); 466fa9e4066Sahrens db->db_state = DB_CACHED; 467fa9e4066Sahrens } else { 468fa9e4066Sahrens ASSERT(db->db_blkid != DB_BONUS_BLKID); 469fa9e4066Sahrens ASSERT3P(db->db_buf, ==, NULL); 470ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(buf, db) == 1); 471ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 472fa9e4066Sahrens } 473fa9e4066Sahrens cv_broadcast(&db->db_changed); 474fa9e4066Sahrens mutex_exit(&db->db_mtx); 475ea8dc4b6Seschrock dbuf_rele(db, NULL); 476fa9e4066Sahrens } 477fa9e4066Sahrens 478ea8dc4b6Seschrock static void 47913506d1eSmaybee dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t *flags) 480fa9e4066Sahrens { 481088f3894Sahrens dnode_t *dn = db->db_dnode; 482ea8dc4b6Seschrock zbookmark_t zb; 48313506d1eSmaybee uint32_t aflags = ARC_NOWAIT; 484088f3894Sahrens arc_buf_t *pbuf; 485fa9e4066Sahrens 486fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 487fa9e4066Sahrens /* We need the struct_rwlock to prevent db_blkptr from changing. */ 488088f3894Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 489ea8dc4b6Seschrock ASSERT(MUTEX_HELD(&db->db_mtx)); 490ea8dc4b6Seschrock ASSERT(db->db_state == DB_UNCACHED); 491ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 492fa9e4066Sahrens 493fa9e4066Sahrens if (db->db_blkid == DB_BONUS_BLKID) { 494cf04dda1SMark Maybee int bonuslen = MIN(dn->dn_bonuslen, dn->dn_phys->dn_bonuslen); 4951934e92fSmaybee 4961934e92fSmaybee ASSERT3U(bonuslen, <=, db->db.db_size); 497ea8dc4b6Seschrock db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN); 4985a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 4991934e92fSmaybee if (bonuslen < DN_MAX_BONUSLEN) 500ea8dc4b6Seschrock bzero(db->db.db_data, DN_MAX_BONUSLEN); 501cf04dda1SMark Maybee if (bonuslen) 502cf04dda1SMark Maybee bcopy(DN_BONUS(dn->dn_phys), db->db.db_data, bonuslen); 503ea8dc4b6Seschrock dbuf_update_data(db); 504fa9e4066Sahrens db->db_state = DB_CACHED; 505fa9e4066Sahrens mutex_exit(&db->db_mtx); 506fa9e4066Sahrens return; 507fa9e4066Sahrens } 508fa9e4066Sahrens 5091c8564a7SMark Maybee /* 5101c8564a7SMark Maybee * Recheck BP_IS_HOLE() after dnode_block_freed() in case dnode_sync() 5111c8564a7SMark Maybee * processes the delete record and clears the bp while we are waiting 5121c8564a7SMark Maybee * for the dn_mtx (resulting in a "no" from block_freed). 5131c8564a7SMark Maybee */ 514088f3894Sahrens if (db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr) || 5151c8564a7SMark Maybee (db->db_level == 0 && (dnode_block_freed(dn, db->db_blkid) || 5161c8564a7SMark Maybee BP_IS_HOLE(db->db_blkptr)))) { 517ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 518ad23a2dbSjohansen 519088f3894Sahrens dbuf_set_data(db, arc_buf_alloc(dn->dn_objset->os_spa, 520ad23a2dbSjohansen db->db.db_size, db, type)); 521fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 522fa9e4066Sahrens db->db_state = DB_CACHED; 52313506d1eSmaybee *flags |= DB_RF_CACHED; 524fa9e4066Sahrens mutex_exit(&db->db_mtx); 525fa9e4066Sahrens return; 526fa9e4066Sahrens } 527fa9e4066Sahrens 528fa9e4066Sahrens db->db_state = DB_READ; 529fa9e4066Sahrens mutex_exit(&db->db_mtx); 530fa9e4066Sahrens 5313baa08fcSek110237 if (DBUF_IS_L2CACHEABLE(db)) 5323baa08fcSek110237 aflags |= ARC_L2CACHE; 5333baa08fcSek110237 534b24ab676SJeff Bonwick SET_BOOKMARK(&zb, db->db_objset->os_dsl_dataset ? 535b24ab676SJeff Bonwick db->db_objset->os_dsl_dataset->ds_object : DMU_META_OBJSET, 536b24ab676SJeff Bonwick db->db.db_object, db->db_level, db->db_blkid); 537ea8dc4b6Seschrock 538ea8dc4b6Seschrock dbuf_add_ref(db, NULL); 539fa9e4066Sahrens /* ZIO_FLAG_CANFAIL callers have to check the parent zio's error */ 540088f3894Sahrens 541088f3894Sahrens if (db->db_parent) 542088f3894Sahrens pbuf = db->db_parent->db_buf; 543088f3894Sahrens else 544088f3894Sahrens pbuf = db->db_objset->os_phys_buf; 545088f3894Sahrens 546088f3894Sahrens (void) arc_read(zio, dn->dn_objset->os_spa, db->db_blkptr, pbuf, 547fa9e4066Sahrens dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ, 54813506d1eSmaybee (*flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED, 54913506d1eSmaybee &aflags, &zb); 55013506d1eSmaybee if (aflags & ARC_CACHED) 55113506d1eSmaybee *flags |= DB_RF_CACHED; 552fa9e4066Sahrens } 553fa9e4066Sahrens 554ea8dc4b6Seschrock int 555ea8dc4b6Seschrock dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags) 556fa9e4066Sahrens { 557ea8dc4b6Seschrock int err = 0; 558ea8dc4b6Seschrock int havepzio = (zio != NULL); 55913506d1eSmaybee int prefetch; 560fa9e4066Sahrens 561fa9e4066Sahrens /* 562fa9e4066Sahrens * We don't have to hold the mutex to check db_state because it 563fa9e4066Sahrens * can't be freed while we have a hold on the buffer. 564fa9e4066Sahrens */ 565fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 566fa9e4066Sahrens 56782c9918fSTim Haley if (db->db_state == DB_NOFILL) 56882c9918fSTim Haley return (EIO); 56982c9918fSTim Haley 570fa9e4066Sahrens if ((flags & DB_RF_HAVESTRUCT) == 0) 571fa9e4066Sahrens rw_enter(&db->db_dnode->dn_struct_rwlock, RW_READER); 572fa9e4066Sahrens 57313506d1eSmaybee prefetch = db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID && 5743baa08fcSek110237 (flags & DB_RF_NOPREFETCH) == 0 && db->db_dnode != NULL && 5753baa08fcSek110237 DBUF_IS_CACHEABLE(db); 57613506d1eSmaybee 577fa9e4066Sahrens mutex_enter(&db->db_mtx); 578ea8dc4b6Seschrock if (db->db_state == DB_CACHED) { 579ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 58013506d1eSmaybee if (prefetch) 58113506d1eSmaybee dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset, 58213506d1eSmaybee db->db.db_size, TRUE); 583ea8dc4b6Seschrock if ((flags & DB_RF_HAVESTRUCT) == 0) 584ea8dc4b6Seschrock rw_exit(&db->db_dnode->dn_struct_rwlock); 585ea8dc4b6Seschrock } else if (db->db_state == DB_UNCACHED) { 586ea8dc4b6Seschrock if (zio == NULL) { 587ea8dc4b6Seschrock zio = zio_root(db->db_dnode->dn_objset->os_spa, 588ea8dc4b6Seschrock NULL, NULL, ZIO_FLAG_CANFAIL); 589ea8dc4b6Seschrock } 59013506d1eSmaybee dbuf_read_impl(db, zio, &flags); 59113506d1eSmaybee 592ea8dc4b6Seschrock /* dbuf_read_impl has dropped db_mtx for us */ 593ea8dc4b6Seschrock 59413506d1eSmaybee if (prefetch) 595ea8dc4b6Seschrock dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset, 59613506d1eSmaybee db->db.db_size, flags & DB_RF_CACHED); 597ea8dc4b6Seschrock 598ea8dc4b6Seschrock if ((flags & DB_RF_HAVESTRUCT) == 0) 599ea8dc4b6Seschrock rw_exit(&db->db_dnode->dn_struct_rwlock); 600ea8dc4b6Seschrock 601ea8dc4b6Seschrock if (!havepzio) 602ea8dc4b6Seschrock err = zio_wait(zio); 603ea8dc4b6Seschrock } else { 60413506d1eSmaybee mutex_exit(&db->db_mtx); 60513506d1eSmaybee if (prefetch) 60613506d1eSmaybee dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset, 60713506d1eSmaybee db->db.db_size, TRUE); 608ea8dc4b6Seschrock if ((flags & DB_RF_HAVESTRUCT) == 0) 609ea8dc4b6Seschrock rw_exit(&db->db_dnode->dn_struct_rwlock); 61013506d1eSmaybee 61113506d1eSmaybee mutex_enter(&db->db_mtx); 612ea8dc4b6Seschrock if ((flags & DB_RF_NEVERWAIT) == 0) { 613ea8dc4b6Seschrock while (db->db_state == DB_READ || 614ea8dc4b6Seschrock db->db_state == DB_FILL) { 615fa9e4066Sahrens ASSERT(db->db_state == DB_READ || 616fa9e4066Sahrens (flags & DB_RF_HAVESTRUCT) == 0); 617fa9e4066Sahrens cv_wait(&db->db_changed, &db->db_mtx); 618fa9e4066Sahrens } 619ea8dc4b6Seschrock if (db->db_state == DB_UNCACHED) 620ea8dc4b6Seschrock err = EIO; 621ea8dc4b6Seschrock } 622fa9e4066Sahrens mutex_exit(&db->db_mtx); 623fa9e4066Sahrens } 624fa9e4066Sahrens 625ea8dc4b6Seschrock ASSERT(err || havepzio || db->db_state == DB_CACHED); 626ea8dc4b6Seschrock return (err); 627fa9e4066Sahrens } 628fa9e4066Sahrens 629fa9e4066Sahrens static void 630fa9e4066Sahrens dbuf_noread(dmu_buf_impl_t *db) 631fa9e4066Sahrens { 632fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 633ea8dc4b6Seschrock ASSERT(db->db_blkid != DB_BONUS_BLKID); 634fa9e4066Sahrens mutex_enter(&db->db_mtx); 635fa9e4066Sahrens while (db->db_state == DB_READ || db->db_state == DB_FILL) 636fa9e4066Sahrens cv_wait(&db->db_changed, &db->db_mtx); 637fa9e4066Sahrens if (db->db_state == DB_UNCACHED) { 638ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 639ad23a2dbSjohansen 640ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 641fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 642fa9e4066Sahrens dbuf_set_data(db, arc_buf_alloc(db->db_dnode->dn_objset->os_spa, 643ad23a2dbSjohansen db->db.db_size, db, type)); 644fa9e4066Sahrens db->db_state = DB_FILL; 64582c9918fSTim Haley } else if (db->db_state == DB_NOFILL) { 64682c9918fSTim Haley dbuf_set_data(db, NULL); 647fa9e4066Sahrens } else { 648fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_CACHED); 649fa9e4066Sahrens } 650fa9e4066Sahrens mutex_exit(&db->db_mtx); 651fa9e4066Sahrens } 652fa9e4066Sahrens 653fa9e4066Sahrens /* 654fa9e4066Sahrens * This is our just-in-time copy function. It makes a copy of 655fa9e4066Sahrens * buffers, that have been modified in a previous transaction 656fa9e4066Sahrens * group, before we modify them in the current active group. 657fa9e4066Sahrens * 658fa9e4066Sahrens * This function is used in two places: when we are dirtying a 659fa9e4066Sahrens * buffer for the first time in a txg, and when we are freeing 660fa9e4066Sahrens * a range in a dnode that includes this buffer. 661fa9e4066Sahrens * 662fa9e4066Sahrens * Note that when we are called from dbuf_free_range() we do 663fa9e4066Sahrens * not put a hold on the buffer, we just traverse the active 664fa9e4066Sahrens * dbuf list for the dnode. 665fa9e4066Sahrens */ 666fa9e4066Sahrens static void 667fa9e4066Sahrens dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg) 668fa9e4066Sahrens { 669c717a561Smaybee dbuf_dirty_record_t *dr = db->db_last_dirty; 670fa9e4066Sahrens 671fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 672fa9e4066Sahrens ASSERT(db->db.db_data != NULL); 673c717a561Smaybee ASSERT(db->db_level == 0); 674c717a561Smaybee ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT); 675fa9e4066Sahrens 6764d31c452Smaybee if (dr == NULL || 6774d31c452Smaybee (dr->dt.dl.dr_data != 6784d31c452Smaybee ((db->db_blkid == DB_BONUS_BLKID) ? db->db.db_data : db->db_buf))) 679fa9e4066Sahrens return; 680fa9e4066Sahrens 681fa9e4066Sahrens /* 682c717a561Smaybee * If the last dirty record for this dbuf has not yet synced 683c717a561Smaybee * and its referencing the dbuf data, either: 684c717a561Smaybee * reset the reference to point to a new copy, 685c717a561Smaybee * or (if there a no active holders) 686c717a561Smaybee * just null out the current db_data pointer. 687fa9e4066Sahrens */ 688c717a561Smaybee ASSERT(dr->dr_txg >= txg - 2); 689c717a561Smaybee if (db->db_blkid == DB_BONUS_BLKID) { 690c717a561Smaybee /* Note that the data bufs here are zio_bufs */ 691c717a561Smaybee dr->dt.dl.dr_data = zio_buf_alloc(DN_MAX_BONUSLEN); 6925a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 693c717a561Smaybee bcopy(db->db.db_data, dr->dt.dl.dr_data, DN_MAX_BONUSLEN); 694c717a561Smaybee } else if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 695ea8dc4b6Seschrock int size = db->db.db_size; 696c717a561Smaybee arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 697c717a561Smaybee dr->dt.dl.dr_data = arc_buf_alloc( 698ad23a2dbSjohansen db->db_dnode->dn_objset->os_spa, size, db, type); 699c717a561Smaybee bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size); 700fa9e4066Sahrens } else { 701ea8dc4b6Seschrock dbuf_set_data(db, NULL); 702fa9e4066Sahrens } 703fa9e4066Sahrens } 704ea8dc4b6Seschrock 705fa9e4066Sahrens void 706c717a561Smaybee dbuf_unoverride(dbuf_dirty_record_t *dr) 707fa9e4066Sahrens { 708c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 709b24ab676SJeff Bonwick blkptr_t *bp = &dr->dt.dl.dr_overridden_by; 710c717a561Smaybee uint64_t txg = dr->dr_txg; 711c5c6ffa0Smaybee 712c717a561Smaybee ASSERT(MUTEX_HELD(&db->db_mtx)); 713c717a561Smaybee ASSERT(dr->dt.dl.dr_override_state != DR_IN_DMU_SYNC); 714c717a561Smaybee ASSERT(db->db_level == 0); 715c717a561Smaybee 716c717a561Smaybee if (db->db_blkid == DB_BONUS_BLKID || 717c717a561Smaybee dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN) 718c717a561Smaybee return; 719c717a561Smaybee 720b24ab676SJeff Bonwick ASSERT(db->db_data_pending != dr); 721b24ab676SJeff Bonwick 722fa9e4066Sahrens /* free this block */ 723b24ab676SJeff Bonwick if (!BP_IS_HOLE(bp)) 724b24ab676SJeff Bonwick dsl_free(spa_get_dsl(db->db_dnode->dn_objset->os_spa), txg, bp); 725b24ab676SJeff Bonwick 726c717a561Smaybee dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 7276b4acc8bSahrens /* 7286b4acc8bSahrens * Release the already-written buffer, so we leave it in 7296b4acc8bSahrens * a consistent dirty state. Note that all callers are 7306b4acc8bSahrens * modifying the buffer, so they will immediately do 7316b4acc8bSahrens * another (redundant) arc_release(). Therefore, leave 7326b4acc8bSahrens * the buf thawed to save the effort of freezing & 7336b4acc8bSahrens * immediately re-thawing it. 7346b4acc8bSahrens */ 735c717a561Smaybee arc_release(dr->dt.dl.dr_data, db); 736fa9e4066Sahrens } 737fa9e4066Sahrens 738cdb0ab79Smaybee /* 739cdb0ab79Smaybee * Evict (if its unreferenced) or clear (if its referenced) any level-0 740cdb0ab79Smaybee * data blocks in the free range, so that any future readers will find 741cdb0ab79Smaybee * empty blocks. Also, if we happen accross any level-1 dbufs in the 742cdb0ab79Smaybee * range that have not already been marked dirty, mark them dirty so 743cdb0ab79Smaybee * they stay in memory. 744cdb0ab79Smaybee */ 745fa9e4066Sahrens void 746cdb0ab79Smaybee dbuf_free_range(dnode_t *dn, uint64_t start, uint64_t end, dmu_tx_t *tx) 747fa9e4066Sahrens { 748fa9e4066Sahrens dmu_buf_impl_t *db, *db_next; 749fa9e4066Sahrens uint64_t txg = tx->tx_txg; 750cdb0ab79Smaybee int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 751cdb0ab79Smaybee uint64_t first_l1 = start >> epbs; 752cdb0ab79Smaybee uint64_t last_l1 = end >> epbs; 753fa9e4066Sahrens 754cdb0ab79Smaybee if (end > dn->dn_maxblkid) { 755cdb0ab79Smaybee end = dn->dn_maxblkid; 756cdb0ab79Smaybee last_l1 = end >> epbs; 757cdb0ab79Smaybee } 758cdb0ab79Smaybee dprintf_dnode(dn, "start=%llu end=%llu\n", start, end); 759fa9e4066Sahrens mutex_enter(&dn->dn_dbufs_mtx); 760fa9e4066Sahrens for (db = list_head(&dn->dn_dbufs); db; db = db_next) { 761fa9e4066Sahrens db_next = list_next(&dn->dn_dbufs, db); 762ea8dc4b6Seschrock ASSERT(db->db_blkid != DB_BONUS_BLKID); 763cdb0ab79Smaybee 764cdb0ab79Smaybee if (db->db_level == 1 && 765cdb0ab79Smaybee db->db_blkid >= first_l1 && db->db_blkid <= last_l1) { 766cdb0ab79Smaybee mutex_enter(&db->db_mtx); 767cdb0ab79Smaybee if (db->db_last_dirty && 768cdb0ab79Smaybee db->db_last_dirty->dr_txg < txg) { 769cdb0ab79Smaybee dbuf_add_ref(db, FTAG); 770cdb0ab79Smaybee mutex_exit(&db->db_mtx); 771cdb0ab79Smaybee dbuf_will_dirty(db, tx); 772cdb0ab79Smaybee dbuf_rele(db, FTAG); 773cdb0ab79Smaybee } else { 774cdb0ab79Smaybee mutex_exit(&db->db_mtx); 775cdb0ab79Smaybee } 776cdb0ab79Smaybee } 777cdb0ab79Smaybee 778ea8dc4b6Seschrock if (db->db_level != 0) 779fa9e4066Sahrens continue; 780fa9e4066Sahrens dprintf_dbuf(db, "found buf %s\n", ""); 781cdb0ab79Smaybee if (db->db_blkid < start || db->db_blkid > end) 782fa9e4066Sahrens continue; 783fa9e4066Sahrens 784fa9e4066Sahrens /* found a level 0 buffer in the range */ 785fa9e4066Sahrens if (dbuf_undirty(db, tx)) 786fa9e4066Sahrens continue; 787fa9e4066Sahrens 788fa9e4066Sahrens mutex_enter(&db->db_mtx); 789ea8dc4b6Seschrock if (db->db_state == DB_UNCACHED || 79082c9918fSTim Haley db->db_state == DB_NOFILL || 791ea8dc4b6Seschrock db->db_state == DB_EVICTING) { 792fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 793fa9e4066Sahrens mutex_exit(&db->db_mtx); 794fa9e4066Sahrens continue; 795fa9e4066Sahrens } 796c543ec06Sahrens if (db->db_state == DB_READ || db->db_state == DB_FILL) { 797c543ec06Sahrens /* will be handled in dbuf_read_done or dbuf_rele */ 798c717a561Smaybee db->db_freed_in_flight = TRUE; 799fa9e4066Sahrens mutex_exit(&db->db_mtx); 800fa9e4066Sahrens continue; 801fa9e4066Sahrens } 802ea8dc4b6Seschrock if (refcount_count(&db->db_holds) == 0) { 803ea8dc4b6Seschrock ASSERT(db->db_buf); 804ea8dc4b6Seschrock dbuf_clear(db); 805ea8dc4b6Seschrock continue; 806ea8dc4b6Seschrock } 807c717a561Smaybee /* The dbuf is referenced */ 808fa9e4066Sahrens 809c717a561Smaybee if (db->db_last_dirty != NULL) { 810c717a561Smaybee dbuf_dirty_record_t *dr = db->db_last_dirty; 811c717a561Smaybee 812c717a561Smaybee if (dr->dr_txg == txg) { 813ea8dc4b6Seschrock /* 814c717a561Smaybee * This buffer is "in-use", re-adjust the file 815c717a561Smaybee * size to reflect that this buffer may 816c717a561Smaybee * contain new data when we sync. 817ea8dc4b6Seschrock */ 81844eda4d7Smaybee if (db->db_blkid > dn->dn_maxblkid) 81944eda4d7Smaybee dn->dn_maxblkid = db->db_blkid; 820c717a561Smaybee dbuf_unoverride(dr); 821c717a561Smaybee } else { 822c717a561Smaybee /* 823c717a561Smaybee * This dbuf is not dirty in the open context. 824c717a561Smaybee * Either uncache it (if its not referenced in 825c717a561Smaybee * the open context) or reset its contents to 826c717a561Smaybee * empty. 827c717a561Smaybee */ 828c717a561Smaybee dbuf_fix_old_data(db, txg); 82944eda4d7Smaybee } 830c717a561Smaybee } 831c717a561Smaybee /* clear the contents if its cached */ 832ea8dc4b6Seschrock if (db->db_state == DB_CACHED) { 833ea8dc4b6Seschrock ASSERT(db->db.db_data != NULL); 834fa9e4066Sahrens arc_release(db->db_buf, db); 835fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 8366b4acc8bSahrens arc_buf_freeze(db->db_buf); 837fa9e4066Sahrens } 838ea8dc4b6Seschrock 839fa9e4066Sahrens mutex_exit(&db->db_mtx); 840fa9e4066Sahrens } 841fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 842fa9e4066Sahrens } 843fa9e4066Sahrens 844fa9e4066Sahrens static int 8451934e92fSmaybee dbuf_block_freeable(dmu_buf_impl_t *db) 846fa9e4066Sahrens { 847fa9e4066Sahrens dsl_dataset_t *ds = db->db_objset->os_dsl_dataset; 848fa9e4066Sahrens uint64_t birth_txg = 0; 849fa9e4066Sahrens 850fa9e4066Sahrens /* 851fa9e4066Sahrens * We don't need any locking to protect db_blkptr: 852c717a561Smaybee * If it's syncing, then db_last_dirty will be set 853c717a561Smaybee * so we'll ignore db_blkptr. 854fa9e4066Sahrens */ 855c717a561Smaybee ASSERT(MUTEX_HELD(&db->db_mtx)); 856c717a561Smaybee if (db->db_last_dirty) 857c717a561Smaybee birth_txg = db->db_last_dirty->dr_txg; 858fa9e4066Sahrens else if (db->db_blkptr) 859fa9e4066Sahrens birth_txg = db->db_blkptr->blk_birth; 860fa9e4066Sahrens 8611934e92fSmaybee /* If we don't exist or are in a snapshot, we can't be freed */ 862fa9e4066Sahrens if (birth_txg) 8631934e92fSmaybee return (ds == NULL || 8641934e92fSmaybee dsl_dataset_block_freeable(ds, birth_txg)); 865fa9e4066Sahrens else 8661934e92fSmaybee return (FALSE); 867fa9e4066Sahrens } 868fa9e4066Sahrens 869fa9e4066Sahrens void 870fa9e4066Sahrens dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx) 871fa9e4066Sahrens { 872fa9e4066Sahrens arc_buf_t *buf, *obuf; 873fa9e4066Sahrens int osize = db->db.db_size; 874ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 875fa9e4066Sahrens 876ea8dc4b6Seschrock ASSERT(db->db_blkid != DB_BONUS_BLKID); 877ea8dc4b6Seschrock 878fa9e4066Sahrens /* XXX does *this* func really need the lock? */ 879fa9e4066Sahrens ASSERT(RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)); 880fa9e4066Sahrens 881fa9e4066Sahrens /* 882fa9e4066Sahrens * This call to dbuf_will_dirty() with the dn_struct_rwlock held 883fa9e4066Sahrens * is OK, because there can be no other references to the db 884fa9e4066Sahrens * when we are changing its size, so no concurrent DB_FILL can 885fa9e4066Sahrens * be happening. 886fa9e4066Sahrens */ 887ea8dc4b6Seschrock /* 888ea8dc4b6Seschrock * XXX we should be doing a dbuf_read, checking the return 889ea8dc4b6Seschrock * value and returning that up to our callers 890ea8dc4b6Seschrock */ 891fa9e4066Sahrens dbuf_will_dirty(db, tx); 892fa9e4066Sahrens 893fa9e4066Sahrens /* create the data buffer for the new block */ 894ad23a2dbSjohansen buf = arc_buf_alloc(db->db_dnode->dn_objset->os_spa, size, db, type); 895fa9e4066Sahrens 896fa9e4066Sahrens /* copy old block data to the new block */ 897fa9e4066Sahrens obuf = db->db_buf; 898f65e61c0Sahrens bcopy(obuf->b_data, buf->b_data, MIN(osize, size)); 899fa9e4066Sahrens /* zero the remainder */ 900f65e61c0Sahrens if (size > osize) 901fa9e4066Sahrens bzero((uint8_t *)buf->b_data + osize, size - osize); 902fa9e4066Sahrens 903fa9e4066Sahrens mutex_enter(&db->db_mtx); 904fa9e4066Sahrens dbuf_set_data(db, buf); 905ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(obuf, db) == 1); 906fa9e4066Sahrens db->db.db_size = size; 907fa9e4066Sahrens 908c717a561Smaybee if (db->db_level == 0) { 909c717a561Smaybee ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg); 910c717a561Smaybee db->db_last_dirty->dt.dl.dr_data = buf; 911c717a561Smaybee } 912fa9e4066Sahrens mutex_exit(&db->db_mtx); 913fa9e4066Sahrens 914fa9e4066Sahrens dnode_willuse_space(db->db_dnode, size-osize, tx); 915fa9e4066Sahrens } 916fa9e4066Sahrens 917c717a561Smaybee dbuf_dirty_record_t * 918fa9e4066Sahrens dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 919fa9e4066Sahrens { 920fa9e4066Sahrens dnode_t *dn = db->db_dnode; 921503ad85cSMatthew Ahrens objset_t *os = dn->dn_objset; 922c717a561Smaybee dbuf_dirty_record_t **drp, *dr; 923fa9e4066Sahrens int drop_struct_lock = FALSE; 924d3469faaSMark Maybee boolean_t do_free_accounting = B_FALSE; 925fa9e4066Sahrens int txgoff = tx->tx_txg & TXG_MASK; 926fa9e4066Sahrens 927fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 928fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 9299c9dc39aSek110237 DMU_TX_DIRTY_BUF(tx, db); 930fa9e4066Sahrens 931fa9e4066Sahrens /* 932fa9e4066Sahrens * Shouldn't dirty a regular buffer in syncing context. Private 933fa9e4066Sahrens * objects may be dirtied in syncing context, but only if they 934fa9e4066Sahrens * were already pre-dirtied in open context. 935fa9e4066Sahrens */ 936c717a561Smaybee ASSERT(!dmu_tx_is_syncing(tx) || 937c717a561Smaybee BP_IS_HOLE(dn->dn_objset->os_rootbp) || 93814843421SMatthew Ahrens DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 93914843421SMatthew Ahrens dn->dn_objset->os_dsl_dataset == NULL); 940fa9e4066Sahrens /* 941fa9e4066Sahrens * We make this assert for private objects as well, but after we 942fa9e4066Sahrens * check if we're already dirty. They are allowed to re-dirty 943fa9e4066Sahrens * in syncing context. 944fa9e4066Sahrens */ 945ea8dc4b6Seschrock ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 946c717a561Smaybee dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 947fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 948fa9e4066Sahrens 949fa9e4066Sahrens mutex_enter(&db->db_mtx); 950fa9e4066Sahrens /* 951c717a561Smaybee * XXX make this true for indirects too? The problem is that 952c717a561Smaybee * transactions created with dmu_tx_create_assigned() from 953c717a561Smaybee * syncing context don't bother holding ahead. 954fa9e4066Sahrens */ 955c717a561Smaybee ASSERT(db->db_level != 0 || 95682c9918fSTim Haley db->db_state == DB_CACHED || db->db_state == DB_FILL || 95782c9918fSTim Haley db->db_state == DB_NOFILL); 958fa9e4066Sahrens 959fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 960fa9e4066Sahrens /* 961fa9e4066Sahrens * Don't set dirtyctx to SYNC if we're just modifying this as we 962fa9e4066Sahrens * initialize the objset. 963fa9e4066Sahrens */ 964fa9e4066Sahrens if (dn->dn_dirtyctx == DN_UNDIRTIED && 965c717a561Smaybee !BP_IS_HOLE(dn->dn_objset->os_rootbp)) { 966fa9e4066Sahrens dn->dn_dirtyctx = 967fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN); 968fa9e4066Sahrens ASSERT(dn->dn_dirtyctx_firstset == NULL); 969fa9e4066Sahrens dn->dn_dirtyctx_firstset = kmem_alloc(1, KM_SLEEP); 970fa9e4066Sahrens } 971fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 972fa9e4066Sahrens 973fa9e4066Sahrens /* 974fa9e4066Sahrens * If this buffer is already dirty, we're done. 975fa9e4066Sahrens */ 976c717a561Smaybee drp = &db->db_last_dirty; 977c717a561Smaybee ASSERT(*drp == NULL || (*drp)->dr_txg <= tx->tx_txg || 978c717a561Smaybee db->db.db_object == DMU_META_DNODE_OBJECT); 9797e2186e3Sbonwick while ((dr = *drp) != NULL && dr->dr_txg > tx->tx_txg) 9807e2186e3Sbonwick drp = &dr->dr_next; 9817e2186e3Sbonwick if (dr && dr->dr_txg == tx->tx_txg) { 982c717a561Smaybee if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID) { 983c717a561Smaybee /* 984c717a561Smaybee * If this buffer has already been written out, 985c717a561Smaybee * we now need to reset its state. 986c717a561Smaybee */ 9877e2186e3Sbonwick dbuf_unoverride(dr); 988b24ab676SJeff Bonwick if (db->db.db_object != DMU_META_DNODE_OBJECT && 989b24ab676SJeff Bonwick db->db_state != DB_NOFILL) 9906b4acc8bSahrens arc_buf_thaw(db->db_buf); 991c717a561Smaybee } 992fa9e4066Sahrens mutex_exit(&db->db_mtx); 9937e2186e3Sbonwick return (dr); 994fa9e4066Sahrens } 995fa9e4066Sahrens 996fa9e4066Sahrens /* 997fa9e4066Sahrens * Only valid if not already dirty. 998fa9e4066Sahrens */ 99914843421SMatthew Ahrens ASSERT(dn->dn_object == 0 || 100014843421SMatthew Ahrens dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 1001fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 1002fa9e4066Sahrens 1003fa9e4066Sahrens ASSERT3U(dn->dn_nlevels, >, db->db_level); 1004fa9e4066Sahrens ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) || 1005fa9e4066Sahrens dn->dn_phys->dn_nlevels > db->db_level || 1006fa9e4066Sahrens dn->dn_next_nlevels[txgoff] > db->db_level || 1007fa9e4066Sahrens dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level || 1008fa9e4066Sahrens dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level); 1009fa9e4066Sahrens 1010fa9e4066Sahrens /* 1011fa9e4066Sahrens * We should only be dirtying in syncing context if it's the 101214843421SMatthew Ahrens * mos or we're initializing the os or it's a special object. 101314843421SMatthew Ahrens * However, we are allowed to dirty in syncing context provided 101414843421SMatthew Ahrens * we already dirtied it in open context. Hence we must make 101514843421SMatthew Ahrens * this assertion only if we're not already dirty. 1016fa9e4066Sahrens */ 101714843421SMatthew Ahrens ASSERT(!dmu_tx_is_syncing(tx) || DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 101814843421SMatthew Ahrens os->os_dsl_dataset == NULL || BP_IS_HOLE(os->os_rootbp)); 1019fa9e4066Sahrens ASSERT(db->db.db_size != 0); 1020fa9e4066Sahrens 1021fa9e4066Sahrens dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1022fa9e4066Sahrens 10231934e92fSmaybee if (db->db_blkid != DB_BONUS_BLKID) { 10241934e92fSmaybee /* 10251934e92fSmaybee * Update the accounting. 1026d3469faaSMark Maybee * Note: we delay "free accounting" until after we drop 1027d3469faaSMark Maybee * the db_mtx. This keeps us from grabbing other locks 1028b24ab676SJeff Bonwick * (and possibly deadlocking) in bp_get_dsize() while 1029d3469faaSMark Maybee * also holding the db_mtx. 10301934e92fSmaybee */ 10311934e92fSmaybee dnode_willuse_space(dn, db->db.db_size, tx); 1032d3469faaSMark Maybee do_free_accounting = dbuf_block_freeable(db); 10331934e92fSmaybee } 10341934e92fSmaybee 1035ea8dc4b6Seschrock /* 1036ea8dc4b6Seschrock * If this buffer is dirty in an old transaction group we need 1037ea8dc4b6Seschrock * to make a copy of it so that the changes we make in this 1038ea8dc4b6Seschrock * transaction group won't leak out when we sync the older txg. 1039ea8dc4b6Seschrock */ 1040c717a561Smaybee dr = kmem_zalloc(sizeof (dbuf_dirty_record_t), KM_SLEEP); 1041c717a561Smaybee if (db->db_level == 0) { 1042c717a561Smaybee void *data_old = db->db_buf; 1043c717a561Smaybee 104482c9918fSTim Haley if (db->db_state != DB_NOFILL) { 1045ea8dc4b6Seschrock if (db->db_blkid == DB_BONUS_BLKID) { 1046c717a561Smaybee dbuf_fix_old_data(db, tx->tx_txg); 1047c717a561Smaybee data_old = db->db.db_data; 1048c717a561Smaybee } else if (db->db.db_object != DMU_META_DNODE_OBJECT) { 1049fa9e4066Sahrens /* 105082c9918fSTim Haley * Release the data buffer from the cache so 105182c9918fSTim Haley * that we can modify it without impacting 105282c9918fSTim Haley * possible other users of this cached data 105382c9918fSTim Haley * block. Note that indirect blocks and 105482c9918fSTim Haley * private objects are not released until the 105582c9918fSTim Haley * syncing state (since they are only modified 105682c9918fSTim Haley * then). 1057fa9e4066Sahrens */ 1058fa9e4066Sahrens arc_release(db->db_buf, db); 1059fa9e4066Sahrens dbuf_fix_old_data(db, tx->tx_txg); 1060c717a561Smaybee data_old = db->db_buf; 1061fa9e4066Sahrens } 1062c717a561Smaybee ASSERT(data_old != NULL); 106382c9918fSTim Haley } 1064c717a561Smaybee dr->dt.dl.dr_data = data_old; 1065c717a561Smaybee } else { 1066c717a561Smaybee mutex_init(&dr->dt.di.dr_mtx, NULL, MUTEX_DEFAULT, NULL); 1067c717a561Smaybee list_create(&dr->dt.di.dr_children, 1068c717a561Smaybee sizeof (dbuf_dirty_record_t), 1069c717a561Smaybee offsetof(dbuf_dirty_record_t, dr_dirty_node)); 1070fa9e4066Sahrens } 1071c717a561Smaybee dr->dr_dbuf = db; 1072c717a561Smaybee dr->dr_txg = tx->tx_txg; 1073c717a561Smaybee dr->dr_next = *drp; 1074c717a561Smaybee *drp = dr; 1075fa9e4066Sahrens 1076fa9e4066Sahrens /* 1077fa9e4066Sahrens * We could have been freed_in_flight between the dbuf_noread 1078fa9e4066Sahrens * and dbuf_dirty. We win, as though the dbuf_noread() had 1079fa9e4066Sahrens * happened after the free. 1080fa9e4066Sahrens */ 1081fa9e4066Sahrens if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID) { 1082c717a561Smaybee mutex_enter(&dn->dn_mtx); 1083fa9e4066Sahrens dnode_clear_range(dn, db->db_blkid, 1, tx); 1084fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1085c717a561Smaybee db->db_freed_in_flight = FALSE; 1086c717a561Smaybee } 1087fa9e4066Sahrens 1088fa9e4066Sahrens /* 1089fa9e4066Sahrens * This buffer is now part of this txg 1090fa9e4066Sahrens */ 1091fa9e4066Sahrens dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg); 1092fa9e4066Sahrens db->db_dirtycnt += 1; 1093fa9e4066Sahrens ASSERT3U(db->db_dirtycnt, <=, 3); 1094fa9e4066Sahrens 1095fa9e4066Sahrens mutex_exit(&db->db_mtx); 1096fa9e4066Sahrens 1097fa9e4066Sahrens if (db->db_blkid == DB_BONUS_BLKID) { 1098c717a561Smaybee mutex_enter(&dn->dn_mtx); 1099c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 1100c717a561Smaybee list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1101c717a561Smaybee mutex_exit(&dn->dn_mtx); 1102fa9e4066Sahrens dnode_setdirty(dn, tx); 1103c717a561Smaybee return (dr); 1104d3469faaSMark Maybee } else if (do_free_accounting) { 1105d3469faaSMark Maybee blkptr_t *bp = db->db_blkptr; 1106d3469faaSMark Maybee int64_t willfree = (bp && !BP_IS_HOLE(bp)) ? 1107b24ab676SJeff Bonwick bp_get_dsize(os->os_spa, bp) : db->db.db_size; 1108d3469faaSMark Maybee /* 1109d3469faaSMark Maybee * This is only a guess -- if the dbuf is dirty 1110d3469faaSMark Maybee * in a previous txg, we don't know how much 1111d3469faaSMark Maybee * space it will use on disk yet. We should 1112d3469faaSMark Maybee * really have the struct_rwlock to access 1113d3469faaSMark Maybee * db_blkptr, but since this is just a guess, 1114d3469faaSMark Maybee * it's OK if we get an odd answer. 1115d3469faaSMark Maybee */ 1116d3469faaSMark Maybee dnode_willuse_space(dn, -willfree, tx); 1117fa9e4066Sahrens } 1118fa9e4066Sahrens 1119fa9e4066Sahrens if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 1120fa9e4066Sahrens rw_enter(&dn->dn_struct_rwlock, RW_READER); 1121fa9e4066Sahrens drop_struct_lock = TRUE; 1122fa9e4066Sahrens } 1123fa9e4066Sahrens 11248346f03fSJonathan W Adams if (db->db_level == 0) { 11258346f03fSJonathan W Adams dnode_new_blkid(dn, db->db_blkid, tx, drop_struct_lock); 11268346f03fSJonathan W Adams ASSERT(dn->dn_maxblkid >= db->db_blkid); 11278346f03fSJonathan W Adams } 11288346f03fSJonathan W Adams 112944eda4d7Smaybee if (db->db_level+1 < dn->dn_nlevels) { 1130c717a561Smaybee dmu_buf_impl_t *parent = db->db_parent; 1131c717a561Smaybee dbuf_dirty_record_t *di; 1132c717a561Smaybee int parent_held = FALSE; 1133c717a561Smaybee 1134c717a561Smaybee if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) { 1135fa9e4066Sahrens int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1136c717a561Smaybee 1137fa9e4066Sahrens parent = dbuf_hold_level(dn, db->db_level+1, 1138fa9e4066Sahrens db->db_blkid >> epbs, FTAG); 1139c717a561Smaybee parent_held = TRUE; 1140c717a561Smaybee } 1141fa9e4066Sahrens if (drop_struct_lock) 1142fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 1143c717a561Smaybee ASSERT3U(db->db_level+1, ==, parent->db_level); 1144c717a561Smaybee di = dbuf_dirty(parent, tx); 1145c717a561Smaybee if (parent_held) 1146ea8dc4b6Seschrock dbuf_rele(parent, FTAG); 1147c717a561Smaybee 1148c717a561Smaybee mutex_enter(&db->db_mtx); 1149c717a561Smaybee /* possible race with dbuf_undirty() */ 1150c717a561Smaybee if (db->db_last_dirty == dr || 1151c717a561Smaybee dn->dn_object == DMU_META_DNODE_OBJECT) { 1152c717a561Smaybee mutex_enter(&di->dt.di.dr_mtx); 1153c717a561Smaybee ASSERT3U(di->dr_txg, ==, tx->tx_txg); 1154c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 1155c717a561Smaybee list_insert_tail(&di->dt.di.dr_children, dr); 1156c717a561Smaybee mutex_exit(&di->dt.di.dr_mtx); 1157c717a561Smaybee dr->dr_parent = di; 1158c717a561Smaybee } 1159c717a561Smaybee mutex_exit(&db->db_mtx); 1160fa9e4066Sahrens } else { 1161c717a561Smaybee ASSERT(db->db_level+1 == dn->dn_nlevels); 1162c717a561Smaybee ASSERT(db->db_blkid < dn->dn_nblkptr); 1163c717a561Smaybee ASSERT(db->db_parent == NULL || 1164c717a561Smaybee db->db_parent == db->db_dnode->dn_dbuf); 1165c717a561Smaybee mutex_enter(&dn->dn_mtx); 1166c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 1167c717a561Smaybee list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1168c717a561Smaybee mutex_exit(&dn->dn_mtx); 1169fa9e4066Sahrens if (drop_struct_lock) 1170fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 1171fa9e4066Sahrens } 1172fa9e4066Sahrens 1173fa9e4066Sahrens dnode_setdirty(dn, tx); 1174c717a561Smaybee return (dr); 1175fa9e4066Sahrens } 1176fa9e4066Sahrens 1177fa9e4066Sahrens static int 1178fa9e4066Sahrens dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1179fa9e4066Sahrens { 1180fa9e4066Sahrens dnode_t *dn = db->db_dnode; 1181c717a561Smaybee uint64_t txg = tx->tx_txg; 118217f17c2dSbonwick dbuf_dirty_record_t *dr, **drp; 1183fa9e4066Sahrens 1184c717a561Smaybee ASSERT(txg != 0); 1185ea8dc4b6Seschrock ASSERT(db->db_blkid != DB_BONUS_BLKID); 1186fa9e4066Sahrens 1187fa9e4066Sahrens mutex_enter(&db->db_mtx); 1188fa9e4066Sahrens /* 1189fa9e4066Sahrens * If this buffer is not dirty, we're done. 1190fa9e4066Sahrens */ 119117f17c2dSbonwick for (drp = &db->db_last_dirty; (dr = *drp) != NULL; drp = &dr->dr_next) 1192c717a561Smaybee if (dr->dr_txg <= txg) 1193c717a561Smaybee break; 1194c717a561Smaybee if (dr == NULL || dr->dr_txg < txg) { 1195fa9e4066Sahrens mutex_exit(&db->db_mtx); 1196fa9e4066Sahrens return (0); 1197fa9e4066Sahrens } 1198c717a561Smaybee ASSERT(dr->dr_txg == txg); 1199b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 1200fa9e4066Sahrens 1201fa9e4066Sahrens /* 1202fa9e4066Sahrens * If this buffer is currently held, we cannot undirty 1203fa9e4066Sahrens * it, since one of the current holders may be in the 1204fa9e4066Sahrens * middle of an update. Note that users of dbuf_undirty() 1205fa9e4066Sahrens * should not place a hold on the dbuf before the call. 1206fa9e4066Sahrens */ 1207fa9e4066Sahrens if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 1208fa9e4066Sahrens mutex_exit(&db->db_mtx); 120944eda4d7Smaybee /* Make sure we don't toss this buffer at sync phase */ 1210fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 1211fa9e4066Sahrens dnode_clear_range(dn, db->db_blkid, 1, tx); 1212fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1213fa9e4066Sahrens return (0); 1214fa9e4066Sahrens } 1215fa9e4066Sahrens 1216fa9e4066Sahrens dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1217fa9e4066Sahrens 1218fa9e4066Sahrens ASSERT(db->db.db_size != 0); 1219fa9e4066Sahrens 1220fa9e4066Sahrens /* XXX would be nice to fix up dn_towrite_space[] */ 1221fa9e4066Sahrens 122217f17c2dSbonwick *drp = dr->dr_next; 1223c717a561Smaybee 1224c717a561Smaybee if (dr->dr_parent) { 1225c717a561Smaybee mutex_enter(&dr->dr_parent->dt.di.dr_mtx); 1226c717a561Smaybee list_remove(&dr->dr_parent->dt.di.dr_children, dr); 1227c717a561Smaybee mutex_exit(&dr->dr_parent->dt.di.dr_mtx); 1228c717a561Smaybee } else if (db->db_level+1 == dn->dn_nlevels) { 1229cdb0ab79Smaybee ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf); 1230fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 1231c717a561Smaybee list_remove(&dn->dn_dirty_records[txg & TXG_MASK], dr); 1232fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1233c717a561Smaybee } 1234c717a561Smaybee 1235c717a561Smaybee if (db->db_level == 0) { 123682c9918fSTim Haley if (db->db_state != DB_NOFILL) { 1237c717a561Smaybee dbuf_unoverride(dr); 1238c717a561Smaybee 1239c717a561Smaybee ASSERT(db->db_buf != NULL); 1240c717a561Smaybee ASSERT(dr->dt.dl.dr_data != NULL); 1241c717a561Smaybee if (dr->dt.dl.dr_data != db->db_buf) 124282c9918fSTim Haley VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, 124382c9918fSTim Haley db) == 1); 124482c9918fSTim Haley } 1245c717a561Smaybee } else { 1246c717a561Smaybee ASSERT(db->db_buf != NULL); 1247c717a561Smaybee ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 1248c25056deSgw25295 mutex_destroy(&dr->dt.di.dr_mtx); 1249c25056deSgw25295 list_destroy(&dr->dt.di.dr_children); 1250c717a561Smaybee } 1251c717a561Smaybee kmem_free(dr, sizeof (dbuf_dirty_record_t)); 1252fa9e4066Sahrens 1253fa9e4066Sahrens ASSERT(db->db_dirtycnt > 0); 1254fa9e4066Sahrens db->db_dirtycnt -= 1; 1255fa9e4066Sahrens 1256c717a561Smaybee if (refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) { 1257ea8dc4b6Seschrock arc_buf_t *buf = db->db_buf; 1258fa9e4066Sahrens 1259b24ab676SJeff Bonwick ASSERT(db->db_state == DB_NOFILL || arc_released(buf)); 1260ea8dc4b6Seschrock dbuf_set_data(db, NULL); 1261ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(buf, db) == 1); 1262fa9e4066Sahrens dbuf_evict(db); 1263fa9e4066Sahrens return (1); 1264fa9e4066Sahrens } 1265fa9e4066Sahrens 1266fa9e4066Sahrens mutex_exit(&db->db_mtx); 1267fa9e4066Sahrens return (0); 1268fa9e4066Sahrens } 1269fa9e4066Sahrens 1270fa9e4066Sahrens #pragma weak dmu_buf_will_dirty = dbuf_will_dirty 1271fa9e4066Sahrens void 1272fa9e4066Sahrens dbuf_will_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1273fa9e4066Sahrens { 12741ab7f2deSmaybee int rf = DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH; 1275fa9e4066Sahrens 1276fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 1277fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 1278fa9e4066Sahrens 1279fa9e4066Sahrens if (RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)) 1280fa9e4066Sahrens rf |= DB_RF_HAVESTRUCT; 1281ea8dc4b6Seschrock (void) dbuf_read(db, NULL, rf); 1282c717a561Smaybee (void) dbuf_dirty(db, tx); 1283fa9e4066Sahrens } 1284fa9e4066Sahrens 1285fa9e4066Sahrens void 128682c9918fSTim Haley dmu_buf_will_not_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 128782c9918fSTim Haley { 128882c9918fSTim Haley dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 128982c9918fSTim Haley 129082c9918fSTim Haley db->db_state = DB_NOFILL; 129182c9918fSTim Haley 129282c9918fSTim Haley dmu_buf_will_fill(db_fake, tx); 129382c9918fSTim Haley } 129482c9918fSTim Haley 129582c9918fSTim Haley void 1296ea8dc4b6Seschrock dmu_buf_will_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 1297fa9e4066Sahrens { 1298ea8dc4b6Seschrock dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1299ea8dc4b6Seschrock 1300ea8dc4b6Seschrock ASSERT(db->db_blkid != DB_BONUS_BLKID); 1301fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 1302fa9e4066Sahrens ASSERT(db->db_level == 0); 1303fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 1304fa9e4066Sahrens 1305ea8dc4b6Seschrock ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT || 1306fa9e4066Sahrens dmu_tx_private_ok(tx)); 1307fa9e4066Sahrens 1308fa9e4066Sahrens dbuf_noread(db); 1309c717a561Smaybee (void) dbuf_dirty(db, tx); 1310fa9e4066Sahrens } 1311fa9e4066Sahrens 1312fa9e4066Sahrens #pragma weak dmu_buf_fill_done = dbuf_fill_done 1313fa9e4066Sahrens /* ARGSUSED */ 1314fa9e4066Sahrens void 1315fa9e4066Sahrens dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx) 1316fa9e4066Sahrens { 1317fa9e4066Sahrens mutex_enter(&db->db_mtx); 13189c9dc39aSek110237 DBUF_VERIFY(db); 1319fa9e4066Sahrens 1320fa9e4066Sahrens if (db->db_state == DB_FILL) { 1321c717a561Smaybee if (db->db_level == 0 && db->db_freed_in_flight) { 1322ea8dc4b6Seschrock ASSERT(db->db_blkid != DB_BONUS_BLKID); 1323fa9e4066Sahrens /* we were freed while filling */ 1324fa9e4066Sahrens /* XXX dbuf_undirty? */ 1325fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 1326c717a561Smaybee db->db_freed_in_flight = FALSE; 1327fa9e4066Sahrens } 1328fa9e4066Sahrens db->db_state = DB_CACHED; 1329fa9e4066Sahrens cv_broadcast(&db->db_changed); 1330fa9e4066Sahrens } 1331fa9e4066Sahrens mutex_exit(&db->db_mtx); 1332fa9e4066Sahrens } 1333fa9e4066Sahrens 1334ea8dc4b6Seschrock /* 13352fdbea25SAleksandr Guzovskiy * Directly assign a provided arc buf to a given dbuf if it's not referenced 13362fdbea25SAleksandr Guzovskiy * by anybody except our caller. Otherwise copy arcbuf's contents to dbuf. 13372fdbea25SAleksandr Guzovskiy */ 13382fdbea25SAleksandr Guzovskiy void 13392fdbea25SAleksandr Guzovskiy dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx) 13402fdbea25SAleksandr Guzovskiy { 13412fdbea25SAleksandr Guzovskiy ASSERT(!refcount_is_zero(&db->db_holds)); 13422fdbea25SAleksandr Guzovskiy ASSERT(db->db_dnode->dn_object != DMU_META_DNODE_OBJECT); 13432fdbea25SAleksandr Guzovskiy ASSERT(db->db_blkid != DB_BONUS_BLKID); 13442fdbea25SAleksandr Guzovskiy ASSERT(db->db_level == 0); 13452fdbea25SAleksandr Guzovskiy ASSERT(DBUF_GET_BUFC_TYPE(db) == ARC_BUFC_DATA); 13462fdbea25SAleksandr Guzovskiy ASSERT(buf != NULL); 13472fdbea25SAleksandr Guzovskiy ASSERT(arc_buf_size(buf) == db->db.db_size); 13482fdbea25SAleksandr Guzovskiy ASSERT(tx->tx_txg != 0); 13492fdbea25SAleksandr Guzovskiy 13502fdbea25SAleksandr Guzovskiy arc_return_buf(buf, db); 13512fdbea25SAleksandr Guzovskiy ASSERT(arc_released(buf)); 13522fdbea25SAleksandr Guzovskiy 13532fdbea25SAleksandr Guzovskiy mutex_enter(&db->db_mtx); 13542fdbea25SAleksandr Guzovskiy 13552fdbea25SAleksandr Guzovskiy while (db->db_state == DB_READ || db->db_state == DB_FILL) 13562fdbea25SAleksandr Guzovskiy cv_wait(&db->db_changed, &db->db_mtx); 13572fdbea25SAleksandr Guzovskiy 13582fdbea25SAleksandr Guzovskiy ASSERT(db->db_state == DB_CACHED || db->db_state == DB_UNCACHED); 13592fdbea25SAleksandr Guzovskiy 13602fdbea25SAleksandr Guzovskiy if (db->db_state == DB_CACHED && 13612fdbea25SAleksandr Guzovskiy refcount_count(&db->db_holds) - 1 > db->db_dirtycnt) { 13622fdbea25SAleksandr Guzovskiy mutex_exit(&db->db_mtx); 13632fdbea25SAleksandr Guzovskiy (void) dbuf_dirty(db, tx); 13642fdbea25SAleksandr Guzovskiy bcopy(buf->b_data, db->db.db_data, db->db.db_size); 13652fdbea25SAleksandr Guzovskiy VERIFY(arc_buf_remove_ref(buf, db) == 1); 1366*c242f9a0Schunli zhang - Sun Microsystems - Irvine United States xuio_stat_wbuf_copied(); 13672fdbea25SAleksandr Guzovskiy return; 13682fdbea25SAleksandr Guzovskiy } 13692fdbea25SAleksandr Guzovskiy 1370*c242f9a0Schunli zhang - Sun Microsystems - Irvine United States xuio_stat_wbuf_nocopy(); 13712fdbea25SAleksandr Guzovskiy if (db->db_state == DB_CACHED) { 13722fdbea25SAleksandr Guzovskiy dbuf_dirty_record_t *dr = db->db_last_dirty; 13732fdbea25SAleksandr Guzovskiy 13742fdbea25SAleksandr Guzovskiy ASSERT(db->db_buf != NULL); 13752fdbea25SAleksandr Guzovskiy if (dr != NULL && dr->dr_txg == tx->tx_txg) { 13762fdbea25SAleksandr Guzovskiy ASSERT(dr->dt.dl.dr_data == db->db_buf); 13772fdbea25SAleksandr Guzovskiy if (!arc_released(db->db_buf)) { 13782fdbea25SAleksandr Guzovskiy ASSERT(dr->dt.dl.dr_override_state == 13792fdbea25SAleksandr Guzovskiy DR_OVERRIDDEN); 13802fdbea25SAleksandr Guzovskiy arc_release(db->db_buf, db); 13812fdbea25SAleksandr Guzovskiy } 13822fdbea25SAleksandr Guzovskiy dr->dt.dl.dr_data = buf; 13832fdbea25SAleksandr Guzovskiy VERIFY(arc_buf_remove_ref(db->db_buf, db) == 1); 13842fdbea25SAleksandr Guzovskiy } else if (dr == NULL || dr->dt.dl.dr_data != db->db_buf) { 13852fdbea25SAleksandr Guzovskiy arc_release(db->db_buf, db); 13862fdbea25SAleksandr Guzovskiy VERIFY(arc_buf_remove_ref(db->db_buf, db) == 1); 13872fdbea25SAleksandr Guzovskiy } 13882fdbea25SAleksandr Guzovskiy db->db_buf = NULL; 13892fdbea25SAleksandr Guzovskiy } 13902fdbea25SAleksandr Guzovskiy ASSERT(db->db_buf == NULL); 13912fdbea25SAleksandr Guzovskiy dbuf_set_data(db, buf); 13922fdbea25SAleksandr Guzovskiy db->db_state = DB_FILL; 13932fdbea25SAleksandr Guzovskiy mutex_exit(&db->db_mtx); 13942fdbea25SAleksandr Guzovskiy (void) dbuf_dirty(db, tx); 13952fdbea25SAleksandr Guzovskiy dbuf_fill_done(db, tx); 13962fdbea25SAleksandr Guzovskiy } 13972fdbea25SAleksandr Guzovskiy 13982fdbea25SAleksandr Guzovskiy /* 1399ea8dc4b6Seschrock * "Clear" the contents of this dbuf. This will mark the dbuf 1400ea8dc4b6Seschrock * EVICTING and clear *most* of its references. Unfortunetely, 1401ea8dc4b6Seschrock * when we are not holding the dn_dbufs_mtx, we can't clear the 1402ea8dc4b6Seschrock * entry in the dn_dbufs list. We have to wait until dbuf_destroy() 1403ea8dc4b6Seschrock * in this case. For callers from the DMU we will usually see: 1404ea8dc4b6Seschrock * dbuf_clear()->arc_buf_evict()->dbuf_do_evict()->dbuf_destroy() 1405ea8dc4b6Seschrock * For the arc callback, we will usually see: 1406ea8dc4b6Seschrock * dbuf_do_evict()->dbuf_clear();dbuf_destroy() 1407ea8dc4b6Seschrock * Sometimes, though, we will get a mix of these two: 1408ea8dc4b6Seschrock * DMU: dbuf_clear()->arc_buf_evict() 1409ea8dc4b6Seschrock * ARC: dbuf_do_evict()->dbuf_destroy() 1410ea8dc4b6Seschrock */ 1411ea8dc4b6Seschrock void 1412fa9e4066Sahrens dbuf_clear(dmu_buf_impl_t *db) 1413fa9e4066Sahrens { 1414fa9e4066Sahrens dnode_t *dn = db->db_dnode; 1415ea8dc4b6Seschrock dmu_buf_impl_t *parent = db->db_parent; 1416c543ec06Sahrens dmu_buf_impl_t *dndb = dn->dn_dbuf; 1417ea8dc4b6Seschrock int dbuf_gone = FALSE; 1418fa9e4066Sahrens 1419fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 1420fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 1421fa9e4066Sahrens 1422ea8dc4b6Seschrock dbuf_evict_user(db); 1423ea8dc4b6Seschrock 1424fa9e4066Sahrens if (db->db_state == DB_CACHED) { 1425ea8dc4b6Seschrock ASSERT(db->db.db_data != NULL); 14260e8c6158Smaybee if (db->db_blkid == DB_BONUS_BLKID) { 1427ea8dc4b6Seschrock zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN); 14285a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 14290e8c6158Smaybee } 1430fa9e4066Sahrens db->db.db_data = NULL; 1431fa9e4066Sahrens db->db_state = DB_UNCACHED; 1432fa9e4066Sahrens } 1433fa9e4066Sahrens 143482c9918fSTim Haley ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL); 1435fa9e4066Sahrens ASSERT(db->db_data_pending == NULL); 1436fa9e4066Sahrens 1437ea8dc4b6Seschrock db->db_state = DB_EVICTING; 1438ea8dc4b6Seschrock db->db_blkptr = NULL; 1439ea8dc4b6Seschrock 1440ea8dc4b6Seschrock if (db->db_blkid != DB_BONUS_BLKID && MUTEX_HELD(&dn->dn_dbufs_mtx)) { 1441ea8dc4b6Seschrock list_remove(&dn->dn_dbufs, db); 1442ea8dc4b6Seschrock dnode_rele(dn, db); 14431934e92fSmaybee db->db_dnode = NULL; 1444ea8dc4b6Seschrock } 1445ea8dc4b6Seschrock 1446ea8dc4b6Seschrock if (db->db_buf) 1447ea8dc4b6Seschrock dbuf_gone = arc_buf_evict(db->db_buf); 1448ea8dc4b6Seschrock 1449ea8dc4b6Seschrock if (!dbuf_gone) 1450fa9e4066Sahrens mutex_exit(&db->db_mtx); 1451fa9e4066Sahrens 1452fa9e4066Sahrens /* 1453fa9e4066Sahrens * If this dbuf is referened from an indirect dbuf, 1454fa9e4066Sahrens * decrement the ref count on the indirect dbuf. 1455fa9e4066Sahrens */ 1456c543ec06Sahrens if (parent && parent != dndb) 1457ea8dc4b6Seschrock dbuf_rele(parent, db); 1458fa9e4066Sahrens } 1459fa9e4066Sahrens 1460fa9e4066Sahrens static int 1461fa9e4066Sahrens dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse, 1462fa9e4066Sahrens dmu_buf_impl_t **parentp, blkptr_t **bpp) 1463fa9e4066Sahrens { 1464fa9e4066Sahrens int nlevels, epbs; 1465fa9e4066Sahrens 14660b69c2f0Sahrens *parentp = NULL; 14670b69c2f0Sahrens *bpp = NULL; 14680b69c2f0Sahrens 1469ea8dc4b6Seschrock ASSERT(blkid != DB_BONUS_BLKID); 1470ea8dc4b6Seschrock 1471fa9e4066Sahrens if (dn->dn_phys->dn_nlevels == 0) 1472fa9e4066Sahrens nlevels = 1; 1473fa9e4066Sahrens else 1474fa9e4066Sahrens nlevels = dn->dn_phys->dn_nlevels; 1475fa9e4066Sahrens 1476fa9e4066Sahrens epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1477fa9e4066Sahrens 1478fa9e4066Sahrens ASSERT3U(level * epbs, <, 64); 1479fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1480ea8dc4b6Seschrock if (level >= nlevels || 1481fa9e4066Sahrens (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) { 1482fa9e4066Sahrens /* the buffer has no parent yet */ 1483fa9e4066Sahrens return (ENOENT); 1484fa9e4066Sahrens } else if (level < nlevels-1) { 1485fa9e4066Sahrens /* this block is referenced from an indirect block */ 1486fa9e4066Sahrens int err = dbuf_hold_impl(dn, level+1, 1487fa9e4066Sahrens blkid >> epbs, fail_sparse, NULL, parentp); 1488fa9e4066Sahrens if (err) 1489fa9e4066Sahrens return (err); 1490ea8dc4b6Seschrock err = dbuf_read(*parentp, NULL, 1491ea8dc4b6Seschrock (DB_RF_HAVESTRUCT | DB_RF_NOPREFETCH | DB_RF_CANFAIL)); 1492c543ec06Sahrens if (err) { 1493c543ec06Sahrens dbuf_rele(*parentp, NULL); 1494c543ec06Sahrens *parentp = NULL; 1495c543ec06Sahrens return (err); 1496c543ec06Sahrens } 1497fa9e4066Sahrens *bpp = ((blkptr_t *)(*parentp)->db.db_data) + 1498fa9e4066Sahrens (blkid & ((1ULL << epbs) - 1)); 1499c543ec06Sahrens return (0); 1500fa9e4066Sahrens } else { 1501fa9e4066Sahrens /* the block is referenced from the dnode */ 1502fa9e4066Sahrens ASSERT3U(level, ==, nlevels-1); 1503fa9e4066Sahrens ASSERT(dn->dn_phys->dn_nblkptr == 0 || 1504fa9e4066Sahrens blkid < dn->dn_phys->dn_nblkptr); 1505c543ec06Sahrens if (dn->dn_dbuf) { 1506c543ec06Sahrens dbuf_add_ref(dn->dn_dbuf, NULL); 1507fa9e4066Sahrens *parentp = dn->dn_dbuf; 1508c543ec06Sahrens } 1509fa9e4066Sahrens *bpp = &dn->dn_phys->dn_blkptr[blkid]; 1510fa9e4066Sahrens return (0); 1511fa9e4066Sahrens } 1512fa9e4066Sahrens } 1513fa9e4066Sahrens 1514fa9e4066Sahrens static dmu_buf_impl_t * 1515fa9e4066Sahrens dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid, 1516fa9e4066Sahrens dmu_buf_impl_t *parent, blkptr_t *blkptr) 1517fa9e4066Sahrens { 1518503ad85cSMatthew Ahrens objset_t *os = dn->dn_objset; 1519fa9e4066Sahrens dmu_buf_impl_t *db, *odb; 1520fa9e4066Sahrens 1521fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1522fa9e4066Sahrens ASSERT(dn->dn_type != DMU_OT_NONE); 1523fa9e4066Sahrens 1524fa9e4066Sahrens db = kmem_cache_alloc(dbuf_cache, KM_SLEEP); 1525fa9e4066Sahrens 1526fa9e4066Sahrens db->db_objset = os; 1527fa9e4066Sahrens db->db.db_object = dn->dn_object; 1528fa9e4066Sahrens db->db_level = level; 1529fa9e4066Sahrens db->db_blkid = blkid; 1530c717a561Smaybee db->db_last_dirty = NULL; 1531ea8dc4b6Seschrock db->db_dirtycnt = 0; 1532ea8dc4b6Seschrock db->db_dnode = dn; 1533ea8dc4b6Seschrock db->db_parent = parent; 1534ea8dc4b6Seschrock db->db_blkptr = blkptr; 1535fa9e4066Sahrens 1536c717a561Smaybee db->db_user_ptr = NULL; 1537c717a561Smaybee db->db_user_data_ptr_ptr = NULL; 1538c717a561Smaybee db->db_evict_func = NULL; 1539c717a561Smaybee db->db_immediate_evict = 0; 1540c717a561Smaybee db->db_freed_in_flight = 0; 1541ea8dc4b6Seschrock 1542ea8dc4b6Seschrock if (blkid == DB_BONUS_BLKID) { 1543ea8dc4b6Seschrock ASSERT3P(parent, ==, dn->dn_dbuf); 15441934e92fSmaybee db->db.db_size = DN_MAX_BONUSLEN - 15451934e92fSmaybee (dn->dn_nblkptr-1) * sizeof (blkptr_t); 15461934e92fSmaybee ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 1547fa9e4066Sahrens db->db.db_offset = DB_BONUS_BLKID; 1548ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 1549ea8dc4b6Seschrock /* the bonus dbuf is not placed in the hash table */ 15505a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1551ea8dc4b6Seschrock return (db); 1552fa9e4066Sahrens } else { 1553fa9e4066Sahrens int blocksize = 1554fa9e4066Sahrens db->db_level ? 1<<dn->dn_indblkshift : dn->dn_datablksz; 1555fa9e4066Sahrens db->db.db_size = blocksize; 1556fa9e4066Sahrens db->db.db_offset = db->db_blkid * blocksize; 1557fa9e4066Sahrens } 1558fa9e4066Sahrens 1559fa9e4066Sahrens /* 1560fa9e4066Sahrens * Hold the dn_dbufs_mtx while we get the new dbuf 1561fa9e4066Sahrens * in the hash table *and* added to the dbufs list. 1562fa9e4066Sahrens * This prevents a possible deadlock with someone 1563fa9e4066Sahrens * trying to look up this dbuf before its added to the 1564fa9e4066Sahrens * dn_dbufs list. 1565fa9e4066Sahrens */ 1566fa9e4066Sahrens mutex_enter(&dn->dn_dbufs_mtx); 1567ea8dc4b6Seschrock db->db_state = DB_EVICTING; 1568fa9e4066Sahrens if ((odb = dbuf_hash_insert(db)) != NULL) { 1569fa9e4066Sahrens /* someone else inserted it first */ 1570fa9e4066Sahrens kmem_cache_free(dbuf_cache, db); 1571fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 1572fa9e4066Sahrens return (odb); 1573fa9e4066Sahrens } 1574fa9e4066Sahrens list_insert_head(&dn->dn_dbufs, db); 1575ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 1576fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 15775a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1578fa9e4066Sahrens 1579fa9e4066Sahrens if (parent && parent != dn->dn_dbuf) 1580fa9e4066Sahrens dbuf_add_ref(parent, db); 1581fa9e4066Sahrens 1582ea8dc4b6Seschrock ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 1583ea8dc4b6Seschrock refcount_count(&dn->dn_holds) > 0); 1584fa9e4066Sahrens (void) refcount_add(&dn->dn_holds, db); 1585fa9e4066Sahrens 1586fa9e4066Sahrens dprintf_dbuf(db, "db=%p\n", db); 1587fa9e4066Sahrens 1588fa9e4066Sahrens return (db); 1589fa9e4066Sahrens } 1590fa9e4066Sahrens 1591fa9e4066Sahrens static int 1592ea8dc4b6Seschrock dbuf_do_evict(void *private) 1593fa9e4066Sahrens { 1594ea8dc4b6Seschrock arc_buf_t *buf = private; 1595ea8dc4b6Seschrock dmu_buf_impl_t *db = buf->b_private; 1596fa9e4066Sahrens 1597ea8dc4b6Seschrock if (!MUTEX_HELD(&db->db_mtx)) 1598ea8dc4b6Seschrock mutex_enter(&db->db_mtx); 1599ea8dc4b6Seschrock 1600ea8dc4b6Seschrock ASSERT(refcount_is_zero(&db->db_holds)); 1601ea8dc4b6Seschrock 1602ea8dc4b6Seschrock if (db->db_state != DB_EVICTING) { 1603ea8dc4b6Seschrock ASSERT(db->db_state == DB_CACHED); 16049c9dc39aSek110237 DBUF_VERIFY(db); 1605ea8dc4b6Seschrock db->db_buf = NULL; 1606ea8dc4b6Seschrock dbuf_evict(db); 1607ea8dc4b6Seschrock } else { 1608ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 1609ea8dc4b6Seschrock dbuf_destroy(db); 1610fa9e4066Sahrens } 1611ea8dc4b6Seschrock return (0); 1612fa9e4066Sahrens } 1613fa9e4066Sahrens 1614fa9e4066Sahrens static void 1615fa9e4066Sahrens dbuf_destroy(dmu_buf_impl_t *db) 1616fa9e4066Sahrens { 1617fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 1618fa9e4066Sahrens 1619ea8dc4b6Seschrock if (db->db_blkid != DB_BONUS_BLKID) { 1620ea8dc4b6Seschrock /* 1621ea8dc4b6Seschrock * If this dbuf is still on the dn_dbufs list, 1622ea8dc4b6Seschrock * remove it from that list. 1623ea8dc4b6Seschrock */ 16241934e92fSmaybee if (db->db_dnode) { 16251934e92fSmaybee dnode_t *dn = db->db_dnode; 16261934e92fSmaybee 16271934e92fSmaybee mutex_enter(&dn->dn_dbufs_mtx); 1628ea8dc4b6Seschrock list_remove(&dn->dn_dbufs, db); 1629ea8dc4b6Seschrock mutex_exit(&dn->dn_dbufs_mtx); 1630ea8dc4b6Seschrock 1631ea8dc4b6Seschrock dnode_rele(dn, db); 16321934e92fSmaybee db->db_dnode = NULL; 1633ea8dc4b6Seschrock } 1634ea8dc4b6Seschrock dbuf_hash_remove(db); 1635ea8dc4b6Seschrock } 1636ea8dc4b6Seschrock db->db_parent = NULL; 1637ea8dc4b6Seschrock db->db_buf = NULL; 1638ea8dc4b6Seschrock 1639f82bfe17Sgw25295 ASSERT(!list_link_active(&db->db_link)); 1640fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 1641fa9e4066Sahrens ASSERT(db->db_hash_next == NULL); 1642fa9e4066Sahrens ASSERT(db->db_blkptr == NULL); 1643fa9e4066Sahrens ASSERT(db->db_data_pending == NULL); 1644fa9e4066Sahrens 1645fa9e4066Sahrens kmem_cache_free(dbuf_cache, db); 16465a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1647fa9e4066Sahrens } 1648fa9e4066Sahrens 1649fa9e4066Sahrens void 1650fa9e4066Sahrens dbuf_prefetch(dnode_t *dn, uint64_t blkid) 1651fa9e4066Sahrens { 165213506d1eSmaybee dmu_buf_impl_t *db = NULL; 1653fa9e4066Sahrens blkptr_t *bp = NULL; 1654fa9e4066Sahrens 1655fa9e4066Sahrens ASSERT(blkid != DB_BONUS_BLKID); 1656fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1657fa9e4066Sahrens 1658fa9e4066Sahrens if (dnode_block_freed(dn, blkid)) 1659fa9e4066Sahrens return; 1660fa9e4066Sahrens 1661fa9e4066Sahrens /* dbuf_find() returns with db_mtx held */ 1662fa9e4066Sahrens if (db = dbuf_find(dn, 0, blkid)) { 166313506d1eSmaybee if (refcount_count(&db->db_holds) > 0) { 1664fa9e4066Sahrens /* 166513506d1eSmaybee * This dbuf is active. We assume that it is 166613506d1eSmaybee * already CACHED, or else about to be either 1667fa9e4066Sahrens * read or filled. 1668fa9e4066Sahrens */ 1669fa9e4066Sahrens mutex_exit(&db->db_mtx); 1670fa9e4066Sahrens return; 1671fa9e4066Sahrens } 167213506d1eSmaybee mutex_exit(&db->db_mtx); 16730b69c2f0Sahrens db = NULL; 167413506d1eSmaybee } 1675fa9e4066Sahrens 167613506d1eSmaybee if (dbuf_findbp(dn, 0, blkid, TRUE, &db, &bp) == 0) { 1677fa9e4066Sahrens if (bp && !BP_IS_HOLE(bp)) { 1678088f3894Sahrens arc_buf_t *pbuf; 1679b24ab676SJeff Bonwick dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset; 168013506d1eSmaybee uint32_t aflags = ARC_NOWAIT | ARC_PREFETCH; 1681ea8dc4b6Seschrock zbookmark_t zb; 1682b24ab676SJeff Bonwick 1683b24ab676SJeff Bonwick SET_BOOKMARK(&zb, ds ? ds->ds_object : DMU_META_OBJSET, 1684b24ab676SJeff Bonwick dn->dn_object, 0, blkid); 1685ea8dc4b6Seschrock 1686088f3894Sahrens if (db) 1687088f3894Sahrens pbuf = db->db_buf; 1688088f3894Sahrens else 1689088f3894Sahrens pbuf = dn->dn_objset->os_phys_buf; 1690088f3894Sahrens 1691088f3894Sahrens (void) arc_read(NULL, dn->dn_objset->os_spa, 1692088f3894Sahrens bp, pbuf, NULL, NULL, ZIO_PRIORITY_ASYNC_READ, 1693fa9e4066Sahrens ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 169413506d1eSmaybee &aflags, &zb); 1695fa9e4066Sahrens } 169613506d1eSmaybee if (db) 169713506d1eSmaybee dbuf_rele(db, NULL); 1698fa9e4066Sahrens } 1699fa9e4066Sahrens } 1700fa9e4066Sahrens 1701fa9e4066Sahrens /* 1702fa9e4066Sahrens * Returns with db_holds incremented, and db_mtx not held. 1703fa9e4066Sahrens * Note: dn_struct_rwlock must be held. 1704fa9e4066Sahrens */ 1705fa9e4066Sahrens int 1706fa9e4066Sahrens dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, int fail_sparse, 1707fa9e4066Sahrens void *tag, dmu_buf_impl_t **dbp) 1708fa9e4066Sahrens { 1709fa9e4066Sahrens dmu_buf_impl_t *db, *parent = NULL; 1710fa9e4066Sahrens 1711ea8dc4b6Seschrock ASSERT(blkid != DB_BONUS_BLKID); 1712fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1713fa9e4066Sahrens ASSERT3U(dn->dn_nlevels, >, level); 1714fa9e4066Sahrens 1715fa9e4066Sahrens *dbp = NULL; 1716ea8dc4b6Seschrock top: 1717fa9e4066Sahrens /* dbuf_find() returns with db_mtx held */ 1718fa9e4066Sahrens db = dbuf_find(dn, level, blkid); 1719fa9e4066Sahrens 1720fa9e4066Sahrens if (db == NULL) { 1721fa9e4066Sahrens blkptr_t *bp = NULL; 1722fa9e4066Sahrens int err; 1723fa9e4066Sahrens 1724c543ec06Sahrens ASSERT3P(parent, ==, NULL); 1725fa9e4066Sahrens err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp); 1726fa9e4066Sahrens if (fail_sparse) { 1727fa9e4066Sahrens if (err == 0 && bp && BP_IS_HOLE(bp)) 1728fa9e4066Sahrens err = ENOENT; 1729fa9e4066Sahrens if (err) { 1730c543ec06Sahrens if (parent) 1731ea8dc4b6Seschrock dbuf_rele(parent, NULL); 1732fa9e4066Sahrens return (err); 1733fa9e4066Sahrens } 1734fa9e4066Sahrens } 1735ea8dc4b6Seschrock if (err && err != ENOENT) 1736ea8dc4b6Seschrock return (err); 1737fa9e4066Sahrens db = dbuf_create(dn, level, blkid, parent, bp); 1738fa9e4066Sahrens } 1739fa9e4066Sahrens 1740ea8dc4b6Seschrock if (db->db_buf && refcount_is_zero(&db->db_holds)) { 1741ea8dc4b6Seschrock arc_buf_add_ref(db->db_buf, db); 1742ea8dc4b6Seschrock if (db->db_buf->b_data == NULL) { 1743ea8dc4b6Seschrock dbuf_clear(db); 1744c543ec06Sahrens if (parent) { 1745c543ec06Sahrens dbuf_rele(parent, NULL); 1746c543ec06Sahrens parent = NULL; 1747c543ec06Sahrens } 1748ea8dc4b6Seschrock goto top; 1749ea8dc4b6Seschrock } 1750ea8dc4b6Seschrock ASSERT3P(db->db.db_data, ==, db->db_buf->b_data); 1751ea8dc4b6Seschrock } 1752ea8dc4b6Seschrock 1753ea8dc4b6Seschrock ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf)); 1754ea8dc4b6Seschrock 1755fa9e4066Sahrens /* 1756c717a561Smaybee * If this buffer is currently syncing out, and we are are 1757c717a561Smaybee * still referencing it from db_data, we need to make a copy 1758c717a561Smaybee * of it in case we decide we want to dirty it again in this txg. 1759fa9e4066Sahrens */ 1760c717a561Smaybee if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID && 1761ea8dc4b6Seschrock dn->dn_object != DMU_META_DNODE_OBJECT && 1762c717a561Smaybee db->db_state == DB_CACHED && db->db_data_pending) { 1763c717a561Smaybee dbuf_dirty_record_t *dr = db->db_data_pending; 1764c717a561Smaybee 1765c717a561Smaybee if (dr->dt.dl.dr_data == db->db_buf) { 1766ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 1767fa9e4066Sahrens 1768c717a561Smaybee dbuf_set_data(db, 1769c717a561Smaybee arc_buf_alloc(db->db_dnode->dn_objset->os_spa, 1770c717a561Smaybee db->db.db_size, db, type)); 1771c717a561Smaybee bcopy(dr->dt.dl.dr_data->b_data, db->db.db_data, 1772fa9e4066Sahrens db->db.db_size); 1773fa9e4066Sahrens } 1774c717a561Smaybee } 1775fa9e4066Sahrens 1776ea8dc4b6Seschrock (void) refcount_add(&db->db_holds, tag); 1777fa9e4066Sahrens dbuf_update_data(db); 17789c9dc39aSek110237 DBUF_VERIFY(db); 1779fa9e4066Sahrens mutex_exit(&db->db_mtx); 1780fa9e4066Sahrens 1781fa9e4066Sahrens /* NOTE: we can't rele the parent until after we drop the db_mtx */ 1782c543ec06Sahrens if (parent) 1783ea8dc4b6Seschrock dbuf_rele(parent, NULL); 1784fa9e4066Sahrens 1785fa9e4066Sahrens ASSERT3P(db->db_dnode, ==, dn); 1786fa9e4066Sahrens ASSERT3U(db->db_blkid, ==, blkid); 1787fa9e4066Sahrens ASSERT3U(db->db_level, ==, level); 1788fa9e4066Sahrens *dbp = db; 1789fa9e4066Sahrens 1790fa9e4066Sahrens return (0); 1791fa9e4066Sahrens } 1792fa9e4066Sahrens 1793fa9e4066Sahrens dmu_buf_impl_t * 1794ea8dc4b6Seschrock dbuf_hold(dnode_t *dn, uint64_t blkid, void *tag) 1795fa9e4066Sahrens { 1796fa9e4066Sahrens dmu_buf_impl_t *db; 1797ea8dc4b6Seschrock int err = dbuf_hold_impl(dn, 0, blkid, FALSE, tag, &db); 1798ea8dc4b6Seschrock return (err ? NULL : db); 1799fa9e4066Sahrens } 1800fa9e4066Sahrens 1801fa9e4066Sahrens dmu_buf_impl_t * 1802fa9e4066Sahrens dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, void *tag) 1803fa9e4066Sahrens { 1804fa9e4066Sahrens dmu_buf_impl_t *db; 1805ea8dc4b6Seschrock int err = dbuf_hold_impl(dn, level, blkid, FALSE, tag, &db); 1806ea8dc4b6Seschrock return (err ? NULL : db); 1807fa9e4066Sahrens } 1808fa9e4066Sahrens 18091934e92fSmaybee void 1810ea8dc4b6Seschrock dbuf_create_bonus(dnode_t *dn) 1811fa9e4066Sahrens { 1812ea8dc4b6Seschrock ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 1813ea8dc4b6Seschrock 1814ea8dc4b6Seschrock ASSERT(dn->dn_bonus == NULL); 18151934e92fSmaybee dn->dn_bonus = dbuf_create(dn, 0, DB_BONUS_BLKID, dn->dn_dbuf, NULL); 1816fa9e4066Sahrens } 1817fa9e4066Sahrens 1818ea8dc4b6Seschrock #pragma weak dmu_buf_add_ref = dbuf_add_ref 1819fa9e4066Sahrens void 1820fa9e4066Sahrens dbuf_add_ref(dmu_buf_impl_t *db, void *tag) 1821fa9e4066Sahrens { 1822ea8dc4b6Seschrock int64_t holds = refcount_add(&db->db_holds, tag); 1823ea8dc4b6Seschrock ASSERT(holds > 1); 1824fa9e4066Sahrens } 1825fa9e4066Sahrens 1826ea8dc4b6Seschrock #pragma weak dmu_buf_rele = dbuf_rele 1827fa9e4066Sahrens void 1828ea8dc4b6Seschrock dbuf_rele(dmu_buf_impl_t *db, void *tag) 1829fa9e4066Sahrens { 1830b24ab676SJeff Bonwick mutex_enter(&db->db_mtx); 1831b24ab676SJeff Bonwick dbuf_rele_and_unlock(db, tag); 1832b24ab676SJeff Bonwick } 1833b24ab676SJeff Bonwick 1834b24ab676SJeff Bonwick /* 1835b24ab676SJeff Bonwick * dbuf_rele() for an already-locked dbuf. This is necessary to allow 1836b24ab676SJeff Bonwick * db_dirtycnt and db_holds to be updated atomically. 1837b24ab676SJeff Bonwick */ 1838b24ab676SJeff Bonwick void 1839b24ab676SJeff Bonwick dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag) 1840b24ab676SJeff Bonwick { 1841fa9e4066Sahrens int64_t holds; 1842fa9e4066Sahrens 1843b24ab676SJeff Bonwick ASSERT(MUTEX_HELD(&db->db_mtx)); 18449c9dc39aSek110237 DBUF_VERIFY(db); 1845fa9e4066Sahrens 1846fa9e4066Sahrens holds = refcount_remove(&db->db_holds, tag); 1847ea8dc4b6Seschrock ASSERT(holds >= 0); 1848fa9e4066Sahrens 1849c717a561Smaybee /* 1850c717a561Smaybee * We can't freeze indirects if there is a possibility that they 1851c717a561Smaybee * may be modified in the current syncing context. 1852c717a561Smaybee */ 1853c717a561Smaybee if (db->db_buf && holds == (db->db_level == 0 ? db->db_dirtycnt : 0)) 18546b4acc8bSahrens arc_buf_freeze(db->db_buf); 18556b4acc8bSahrens 1856fa9e4066Sahrens if (holds == db->db_dirtycnt && 1857c717a561Smaybee db->db_level == 0 && db->db_immediate_evict) 1858fa9e4066Sahrens dbuf_evict_user(db); 1859ea8dc4b6Seschrock 1860ea8dc4b6Seschrock if (holds == 0) { 1861ea8dc4b6Seschrock if (db->db_blkid == DB_BONUS_BLKID) { 1862ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 1863ea8dc4b6Seschrock dnode_rele(db->db_dnode, db); 1864ea8dc4b6Seschrock } else if (db->db_buf == NULL) { 1865ea8dc4b6Seschrock /* 1866ea8dc4b6Seschrock * This is a special case: we never associated this 1867ea8dc4b6Seschrock * dbuf with any data allocated from the ARC. 1868ea8dc4b6Seschrock */ 186982c9918fSTim Haley ASSERT(db->db_state == DB_UNCACHED || 187082c9918fSTim Haley db->db_state == DB_NOFILL); 1871ea8dc4b6Seschrock dbuf_evict(db); 1872ea8dc4b6Seschrock } else if (arc_released(db->db_buf)) { 1873ea8dc4b6Seschrock arc_buf_t *buf = db->db_buf; 1874ea8dc4b6Seschrock /* 1875ea8dc4b6Seschrock * This dbuf has anonymous data associated with it. 1876ea8dc4b6Seschrock */ 1877ea8dc4b6Seschrock dbuf_set_data(db, NULL); 1878ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(buf, db) == 1); 1879ea8dc4b6Seschrock dbuf_evict(db); 1880ea8dc4b6Seschrock } else { 1881ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(db->db_buf, db) == 0); 18823baa08fcSek110237 if (!DBUF_IS_CACHEABLE(db)) 18833baa08fcSek110237 dbuf_clear(db); 18843baa08fcSek110237 else 1885fa9e4066Sahrens mutex_exit(&db->db_mtx); 1886fa9e4066Sahrens } 1887ea8dc4b6Seschrock } else { 1888ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 1889fa9e4066Sahrens } 1890fa9e4066Sahrens } 1891fa9e4066Sahrens 1892fa9e4066Sahrens #pragma weak dmu_buf_refcount = dbuf_refcount 1893fa9e4066Sahrens uint64_t 1894fa9e4066Sahrens dbuf_refcount(dmu_buf_impl_t *db) 1895fa9e4066Sahrens { 1896fa9e4066Sahrens return (refcount_count(&db->db_holds)); 1897fa9e4066Sahrens } 1898fa9e4066Sahrens 1899fa9e4066Sahrens void * 1900fa9e4066Sahrens dmu_buf_set_user(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 1901fa9e4066Sahrens dmu_buf_evict_func_t *evict_func) 1902fa9e4066Sahrens { 1903fa9e4066Sahrens return (dmu_buf_update_user(db_fake, NULL, user_ptr, 1904fa9e4066Sahrens user_data_ptr_ptr, evict_func)); 1905fa9e4066Sahrens } 1906fa9e4066Sahrens 1907fa9e4066Sahrens void * 1908fa9e4066Sahrens dmu_buf_set_user_ie(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 1909fa9e4066Sahrens dmu_buf_evict_func_t *evict_func) 1910fa9e4066Sahrens { 1911fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1912fa9e4066Sahrens 1913c717a561Smaybee db->db_immediate_evict = TRUE; 1914fa9e4066Sahrens return (dmu_buf_update_user(db_fake, NULL, user_ptr, 1915fa9e4066Sahrens user_data_ptr_ptr, evict_func)); 1916fa9e4066Sahrens } 1917fa9e4066Sahrens 1918fa9e4066Sahrens void * 1919fa9e4066Sahrens dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr, void *user_ptr, 1920fa9e4066Sahrens void *user_data_ptr_ptr, dmu_buf_evict_func_t *evict_func) 1921fa9e4066Sahrens { 1922fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1923fa9e4066Sahrens ASSERT(db->db_level == 0); 1924fa9e4066Sahrens 1925fa9e4066Sahrens ASSERT((user_ptr == NULL) == (evict_func == NULL)); 1926fa9e4066Sahrens 1927fa9e4066Sahrens mutex_enter(&db->db_mtx); 1928fa9e4066Sahrens 1929c717a561Smaybee if (db->db_user_ptr == old_user_ptr) { 1930c717a561Smaybee db->db_user_ptr = user_ptr; 1931c717a561Smaybee db->db_user_data_ptr_ptr = user_data_ptr_ptr; 1932c717a561Smaybee db->db_evict_func = evict_func; 1933fa9e4066Sahrens 1934fa9e4066Sahrens dbuf_update_data(db); 1935fa9e4066Sahrens } else { 1936c717a561Smaybee old_user_ptr = db->db_user_ptr; 1937fa9e4066Sahrens } 1938fa9e4066Sahrens 1939fa9e4066Sahrens mutex_exit(&db->db_mtx); 1940fa9e4066Sahrens return (old_user_ptr); 1941fa9e4066Sahrens } 1942fa9e4066Sahrens 1943fa9e4066Sahrens void * 1944fa9e4066Sahrens dmu_buf_get_user(dmu_buf_t *db_fake) 1945fa9e4066Sahrens { 1946fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1947fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 1948fa9e4066Sahrens 1949c717a561Smaybee return (db->db_user_ptr); 1950fa9e4066Sahrens } 1951fa9e4066Sahrens 19523d692628SSanjeev Bagewadi boolean_t 19533d692628SSanjeev Bagewadi dmu_buf_freeable(dmu_buf_t *dbuf) 19543d692628SSanjeev Bagewadi { 19553d692628SSanjeev Bagewadi boolean_t res = B_FALSE; 19563d692628SSanjeev Bagewadi dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf; 19573d692628SSanjeev Bagewadi 19583d692628SSanjeev Bagewadi if (db->db_blkptr) 19593d692628SSanjeev Bagewadi res = dsl_dataset_block_freeable(db->db_objset->os_dsl_dataset, 19603d692628SSanjeev Bagewadi db->db_blkptr->blk_birth); 19613d692628SSanjeev Bagewadi 19623d692628SSanjeev Bagewadi return (res); 19633d692628SSanjeev Bagewadi } 19643d692628SSanjeev Bagewadi 1965c717a561Smaybee static void 1966c717a561Smaybee dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db) 1967fa9e4066Sahrens { 1968c717a561Smaybee /* ASSERT(dmu_tx_is_syncing(tx) */ 1969c717a561Smaybee ASSERT(MUTEX_HELD(&db->db_mtx)); 1970c717a561Smaybee 1971c717a561Smaybee if (db->db_blkptr != NULL) 1972c717a561Smaybee return; 1973c717a561Smaybee 1974c717a561Smaybee if (db->db_level == dn->dn_phys->dn_nlevels-1) { 1975c717a561Smaybee /* 1976c717a561Smaybee * This buffer was allocated at a time when there was 1977c717a561Smaybee * no available blkptrs from the dnode, or it was 1978c717a561Smaybee * inappropriate to hook it in (i.e., nlevels mis-match). 1979c717a561Smaybee */ 1980c717a561Smaybee ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr); 1981c717a561Smaybee ASSERT(db->db_parent == NULL); 1982c717a561Smaybee db->db_parent = dn->dn_dbuf; 1983c717a561Smaybee db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid]; 1984c717a561Smaybee DBUF_VERIFY(db); 1985c717a561Smaybee } else { 1986c717a561Smaybee dmu_buf_impl_t *parent = db->db_parent; 1987c717a561Smaybee int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 1988c717a561Smaybee 1989c717a561Smaybee ASSERT(dn->dn_phys->dn_nlevels > 1); 1990c717a561Smaybee if (parent == NULL) { 1991c717a561Smaybee mutex_exit(&db->db_mtx); 1992c717a561Smaybee rw_enter(&dn->dn_struct_rwlock, RW_READER); 1993c717a561Smaybee (void) dbuf_hold_impl(dn, db->db_level+1, 1994c717a561Smaybee db->db_blkid >> epbs, FALSE, db, &parent); 1995c717a561Smaybee rw_exit(&dn->dn_struct_rwlock); 1996c717a561Smaybee mutex_enter(&db->db_mtx); 1997c717a561Smaybee db->db_parent = parent; 1998c717a561Smaybee } 1999c717a561Smaybee db->db_blkptr = (blkptr_t *)parent->db.db_data + 2000c717a561Smaybee (db->db_blkid & ((1ULL << epbs) - 1)); 2001c717a561Smaybee DBUF_VERIFY(db); 2002c717a561Smaybee } 2003c717a561Smaybee } 2004c717a561Smaybee 2005c717a561Smaybee static void 2006c717a561Smaybee dbuf_sync_indirect(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 2007c717a561Smaybee { 2008c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 2009c717a561Smaybee dnode_t *dn = db->db_dnode; 2010c717a561Smaybee zio_t *zio; 2011c717a561Smaybee 2012c717a561Smaybee ASSERT(dmu_tx_is_syncing(tx)); 2013c717a561Smaybee 2014c717a561Smaybee dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 2015c717a561Smaybee 2016c717a561Smaybee mutex_enter(&db->db_mtx); 2017c717a561Smaybee 2018c717a561Smaybee ASSERT(db->db_level > 0); 2019c717a561Smaybee DBUF_VERIFY(db); 2020c717a561Smaybee 2021c717a561Smaybee if (db->db_buf == NULL) { 2022c717a561Smaybee mutex_exit(&db->db_mtx); 2023c717a561Smaybee (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED); 2024c717a561Smaybee mutex_enter(&db->db_mtx); 2025c717a561Smaybee } 2026c717a561Smaybee ASSERT3U(db->db_state, ==, DB_CACHED); 2027c717a561Smaybee ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2028c717a561Smaybee ASSERT(db->db_buf != NULL); 2029c717a561Smaybee 2030c717a561Smaybee dbuf_check_blkptr(dn, db); 2031c717a561Smaybee 2032c717a561Smaybee db->db_data_pending = dr; 2033c717a561Smaybee 2034af2c4821Smaybee mutex_exit(&db->db_mtx); 2035088f3894Sahrens dbuf_write(dr, db->db_buf, tx); 2036c717a561Smaybee 2037c717a561Smaybee zio = dr->dr_zio; 2038c717a561Smaybee mutex_enter(&dr->dt.di.dr_mtx); 2039c717a561Smaybee dbuf_sync_list(&dr->dt.di.dr_children, tx); 2040c717a561Smaybee ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 2041c717a561Smaybee mutex_exit(&dr->dt.di.dr_mtx); 2042c717a561Smaybee zio_nowait(zio); 2043c717a561Smaybee } 2044c717a561Smaybee 2045c717a561Smaybee static void 2046c717a561Smaybee dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 2047c717a561Smaybee { 2048c717a561Smaybee arc_buf_t **datap = &dr->dt.dl.dr_data; 2049c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 2050fa9e4066Sahrens dnode_t *dn = db->db_dnode; 2051503ad85cSMatthew Ahrens objset_t *os = dn->dn_objset; 2052c717a561Smaybee uint64_t txg = tx->tx_txg; 2053fa9e4066Sahrens 2054fa9e4066Sahrens ASSERT(dmu_tx_is_syncing(tx)); 2055fa9e4066Sahrens 2056fa9e4066Sahrens dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 2057fa9e4066Sahrens 2058fa9e4066Sahrens mutex_enter(&db->db_mtx); 2059fa9e4066Sahrens /* 2060fa9e4066Sahrens * To be synced, we must be dirtied. But we 2061fa9e4066Sahrens * might have been freed after the dirty. 2062fa9e4066Sahrens */ 2063fa9e4066Sahrens if (db->db_state == DB_UNCACHED) { 2064fa9e4066Sahrens /* This buffer has been freed since it was dirtied */ 2065fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 2066fa9e4066Sahrens } else if (db->db_state == DB_FILL) { 2067fa9e4066Sahrens /* This buffer was freed and is now being re-filled */ 2068c717a561Smaybee ASSERT(db->db.db_data != dr->dt.dl.dr_data); 2069fa9e4066Sahrens } else { 207082c9918fSTim Haley ASSERT(db->db_state == DB_CACHED || db->db_state == DB_NOFILL); 2071fa9e4066Sahrens } 20729c9dc39aSek110237 DBUF_VERIFY(db); 2073fa9e4066Sahrens 2074fa9e4066Sahrens /* 2075c717a561Smaybee * If this is a bonus buffer, simply copy the bonus data into the 2076c717a561Smaybee * dnode. It will be written out when the dnode is synced (and it 2077c717a561Smaybee * will be synced, since it must have been dirty for dbuf_sync to 2078c717a561Smaybee * be called). 2079fa9e4066Sahrens */ 2080ea8dc4b6Seschrock if (db->db_blkid == DB_BONUS_BLKID) { 2081c717a561Smaybee dbuf_dirty_record_t **drp; 20821934e92fSmaybee 2083ea8dc4b6Seschrock ASSERT(*datap != NULL); 2084ea8dc4b6Seschrock ASSERT3U(db->db_level, ==, 0); 2085ea8dc4b6Seschrock ASSERT3U(dn->dn_phys->dn_bonuslen, <=, DN_MAX_BONUSLEN); 2086ea8dc4b6Seschrock bcopy(*datap, DN_BONUS(dn->dn_phys), dn->dn_phys->dn_bonuslen); 20870e8c6158Smaybee if (*datap != db->db.db_data) { 2088ea8dc4b6Seschrock zio_buf_free(*datap, DN_MAX_BONUSLEN); 20895a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 20900e8c6158Smaybee } 2091ea8dc4b6Seschrock db->db_data_pending = NULL; 2092c717a561Smaybee drp = &db->db_last_dirty; 2093c717a561Smaybee while (*drp != dr) 2094c717a561Smaybee drp = &(*drp)->dr_next; 209517f17c2dSbonwick ASSERT(dr->dr_next == NULL); 2096b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 209717f17c2dSbonwick *drp = dr->dr_next; 2098c717a561Smaybee kmem_free(dr, sizeof (dbuf_dirty_record_t)); 2099ea8dc4b6Seschrock ASSERT(db->db_dirtycnt > 0); 2100ea8dc4b6Seschrock db->db_dirtycnt -= 1; 2101b24ab676SJeff Bonwick dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg); 2102ea8dc4b6Seschrock return; 2103ea8dc4b6Seschrock } 2104ea8dc4b6Seschrock 2105c5c6ffa0Smaybee /* 2106f82bfe17Sgw25295 * This function may have dropped the db_mtx lock allowing a dmu_sync 2107f82bfe17Sgw25295 * operation to sneak in. As a result, we need to ensure that we 2108f82bfe17Sgw25295 * don't check the dr_override_state until we have returned from 2109f82bfe17Sgw25295 * dbuf_check_blkptr. 2110f82bfe17Sgw25295 */ 2111f82bfe17Sgw25295 dbuf_check_blkptr(dn, db); 2112f82bfe17Sgw25295 2113f82bfe17Sgw25295 /* 2114c717a561Smaybee * If this buffer is in the middle of an immdiate write, 2115c717a561Smaybee * wait for the synchronous IO to complete. 2116c5c6ffa0Smaybee */ 2117c717a561Smaybee while (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC) { 2118c5c6ffa0Smaybee ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT); 2119c5c6ffa0Smaybee cv_wait(&db->db_changed, &db->db_mtx); 2120c717a561Smaybee ASSERT(dr->dt.dl.dr_override_state != DR_NOT_OVERRIDDEN); 2121c5c6ffa0Smaybee } 2122c717a561Smaybee 2123ab69d62fSMatthew Ahrens if (db->db_state != DB_NOFILL && 2124ab69d62fSMatthew Ahrens dn->dn_object != DMU_META_DNODE_OBJECT && 2125ab69d62fSMatthew Ahrens refcount_count(&db->db_holds) > 1 && 2126b24ab676SJeff Bonwick dr->dt.dl.dr_override_state != DR_OVERRIDDEN && 2127ab69d62fSMatthew Ahrens *datap == db->db_buf) { 2128fa9e4066Sahrens /* 212982c9918fSTim Haley * If this buffer is currently "in use" (i.e., there 213082c9918fSTim Haley * are active holds and db_data still references it), 213182c9918fSTim Haley * then make a copy before we start the write so that 213282c9918fSTim Haley * any modifications from the open txg will not leak 213382c9918fSTim Haley * into this write. 2134fa9e4066Sahrens * 213582c9918fSTim Haley * NOTE: this copy does not need to be made for 213682c9918fSTim Haley * objects only modified in the syncing context (e.g. 213782c9918fSTim Haley * DNONE_DNODE blocks). 2138fa9e4066Sahrens */ 2139ab69d62fSMatthew Ahrens int blksz = arc_buf_size(*datap); 2140ab69d62fSMatthew Ahrens arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 2141ab69d62fSMatthew Ahrens *datap = arc_buf_alloc(os->os_spa, blksz, db, type); 2142c717a561Smaybee bcopy(db->db.db_data, (*datap)->b_data, blksz); 2143fa9e4066Sahrens } 2144c717a561Smaybee db->db_data_pending = dr; 2145fa9e4066Sahrens 2146fa9e4066Sahrens mutex_exit(&db->db_mtx); 2147fa9e4066Sahrens 2148088f3894Sahrens dbuf_write(dr, *datap, tx); 2149c717a561Smaybee 2150c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 2151c717a561Smaybee if (dn->dn_object == DMU_META_DNODE_OBJECT) 2152c717a561Smaybee list_insert_tail(&dn->dn_dirty_records[txg&TXG_MASK], dr); 2153c717a561Smaybee else 2154c717a561Smaybee zio_nowait(dr->dr_zio); 2155fa9e4066Sahrens } 2156c717a561Smaybee 2157c717a561Smaybee void 2158c717a561Smaybee dbuf_sync_list(list_t *list, dmu_tx_t *tx) 2159c717a561Smaybee { 2160c717a561Smaybee dbuf_dirty_record_t *dr; 2161c717a561Smaybee 2162c717a561Smaybee while (dr = list_head(list)) { 2163c717a561Smaybee if (dr->dr_zio != NULL) { 2164c717a561Smaybee /* 2165c717a561Smaybee * If we find an already initialized zio then we 2166c717a561Smaybee * are processing the meta-dnode, and we have finished. 2167c717a561Smaybee * The dbufs for all dnodes are put back on the list 2168c717a561Smaybee * during processing, so that we can zio_wait() 2169c717a561Smaybee * these IOs after initiating all child IOs. 2170c717a561Smaybee */ 2171c717a561Smaybee ASSERT3U(dr->dr_dbuf->db.db_object, ==, 2172c717a561Smaybee DMU_META_DNODE_OBJECT); 2173c717a561Smaybee break; 2174fa9e4066Sahrens } 2175c717a561Smaybee list_remove(list, dr); 2176c717a561Smaybee if (dr->dr_dbuf->db_level > 0) 2177c717a561Smaybee dbuf_sync_indirect(dr, tx); 2178c717a561Smaybee else 2179c717a561Smaybee dbuf_sync_leaf(dr, tx); 2180c717a561Smaybee } 2181c717a561Smaybee } 2182c717a561Smaybee 2183fa9e4066Sahrens /* ARGSUSED */ 2184fa9e4066Sahrens static void 2185c717a561Smaybee dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb) 2186fa9e4066Sahrens { 2187fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 2188e14bb325SJeff Bonwick blkptr_t *bp = zio->io_bp; 2189c717a561Smaybee blkptr_t *bp_orig = &zio->io_bp_orig; 2190b24ab676SJeff Bonwick dnode_t *dn = db->db_dnode; 2191b24ab676SJeff Bonwick spa_t *spa = zio->io_spa; 2192b24ab676SJeff Bonwick int64_t delta; 2193fa9e4066Sahrens uint64_t fill = 0; 2194b24ab676SJeff Bonwick int i; 2195fa9e4066Sahrens 2196e14bb325SJeff Bonwick ASSERT(db->db_blkptr == bp); 2197e14bb325SJeff Bonwick 2198b24ab676SJeff Bonwick delta = bp_get_dsize_sync(spa, bp) - bp_get_dsize_sync(spa, bp_orig); 2199b24ab676SJeff Bonwick dnode_diduse_space(dn, delta - zio->io_prev_space_delta); 2200b24ab676SJeff Bonwick zio->io_prev_space_delta = delta; 2201fa9e4066Sahrens 2202e14bb325SJeff Bonwick if (BP_IS_HOLE(bp)) { 2203b24ab676SJeff Bonwick ASSERT(bp->blk_fill == 0); 2204c717a561Smaybee return; 2205c717a561Smaybee } 2206c717a561Smaybee 2207e14bb325SJeff Bonwick ASSERT(BP_GET_TYPE(bp) == dn->dn_type); 2208e14bb325SJeff Bonwick ASSERT(BP_GET_LEVEL(bp) == db->db_level); 2209e14bb325SJeff Bonwick 2210fa9e4066Sahrens mutex_enter(&db->db_mtx); 2211fa9e4066Sahrens 2212fa9e4066Sahrens if (db->db_level == 0) { 2213fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 2214c717a561Smaybee if (db->db_blkid > dn->dn_phys->dn_maxblkid) 2215fa9e4066Sahrens dn->dn_phys->dn_maxblkid = db->db_blkid; 2216fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 2217fa9e4066Sahrens 2218fa9e4066Sahrens if (dn->dn_type == DMU_OT_DNODE) { 2219fa9e4066Sahrens dnode_phys_t *dnp = db->db.db_data; 2220fa9e4066Sahrens for (i = db->db.db_size >> DNODE_SHIFT; i > 0; 2221fa9e4066Sahrens i--, dnp++) { 2222fa9e4066Sahrens if (dnp->dn_type != DMU_OT_NONE) 2223fa9e4066Sahrens fill++; 2224fa9e4066Sahrens } 2225fa9e4066Sahrens } else { 2226fa9e4066Sahrens fill = 1; 2227fa9e4066Sahrens } 2228fa9e4066Sahrens } else { 2229e14bb325SJeff Bonwick blkptr_t *ibp = db->db.db_data; 2230fa9e4066Sahrens ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2231e14bb325SJeff Bonwick for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, ibp++) { 2232e14bb325SJeff Bonwick if (BP_IS_HOLE(ibp)) 2233fa9e4066Sahrens continue; 2234e14bb325SJeff Bonwick ASSERT3U(BP_GET_LSIZE(ibp), ==, 2235fa9e4066Sahrens db->db_level == 1 ? dn->dn_datablksz : 2236fa9e4066Sahrens (1<<dn->dn_phys->dn_indblkshift)); 2237e14bb325SJeff Bonwick fill += ibp->blk_fill; 2238fa9e4066Sahrens } 2239fa9e4066Sahrens } 2240fa9e4066Sahrens 2241e14bb325SJeff Bonwick bp->blk_fill = fill; 2242fa9e4066Sahrens 2243fa9e4066Sahrens mutex_exit(&db->db_mtx); 2244fa9e4066Sahrens } 2245fa9e4066Sahrens 2246c717a561Smaybee /* ARGSUSED */ 2247c717a561Smaybee static void 2248c717a561Smaybee dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb) 2249c717a561Smaybee { 2250c717a561Smaybee dmu_buf_impl_t *db = vdb; 2251b24ab676SJeff Bonwick blkptr_t *bp = zio->io_bp; 2252b24ab676SJeff Bonwick blkptr_t *bp_orig = &zio->io_bp_orig; 2253b24ab676SJeff Bonwick dnode_t *dn = db->db_dnode; 2254b24ab676SJeff Bonwick objset_t *os = dn->dn_objset; 2255c717a561Smaybee uint64_t txg = zio->io_txg; 2256c717a561Smaybee dbuf_dirty_record_t **drp, *dr; 2257c717a561Smaybee 2258c717a561Smaybee ASSERT3U(zio->io_error, ==, 0); 2259b24ab676SJeff Bonwick ASSERT(db->db_blkptr == bp); 2260b24ab676SJeff Bonwick 2261b24ab676SJeff Bonwick if (zio->io_flags & ZIO_FLAG_IO_REWRITE) { 2262b24ab676SJeff Bonwick ASSERT(BP_EQUAL(bp, bp_orig)); 2263b24ab676SJeff Bonwick } else { 2264b24ab676SJeff Bonwick dsl_dataset_t *ds = os->os_dsl_dataset; 2265b24ab676SJeff Bonwick dmu_tx_t *tx = os->os_synctx; 2266b24ab676SJeff Bonwick 2267b24ab676SJeff Bonwick (void) dsl_dataset_block_kill(ds, bp_orig, tx, B_TRUE); 2268b24ab676SJeff Bonwick dsl_dataset_block_born(ds, bp, tx); 2269b24ab676SJeff Bonwick } 2270c717a561Smaybee 2271c717a561Smaybee mutex_enter(&db->db_mtx); 2272c717a561Smaybee 2273b24ab676SJeff Bonwick DBUF_VERIFY(db); 2274b24ab676SJeff Bonwick 2275c717a561Smaybee drp = &db->db_last_dirty; 227617f17c2dSbonwick while ((dr = *drp) != db->db_data_pending) 227717f17c2dSbonwick drp = &dr->dr_next; 227817f17c2dSbonwick ASSERT(!list_link_active(&dr->dr_dirty_node)); 227917f17c2dSbonwick ASSERT(dr->dr_txg == txg); 2280b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 228117f17c2dSbonwick ASSERT(dr->dr_next == NULL); 228217f17c2dSbonwick *drp = dr->dr_next; 2283c717a561Smaybee 2284c717a561Smaybee if (db->db_level == 0) { 2285c717a561Smaybee ASSERT(db->db_blkid != DB_BONUS_BLKID); 2286c717a561Smaybee ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN); 228782c9918fSTim Haley if (db->db_state != DB_NOFILL) { 2288c717a561Smaybee if (dr->dt.dl.dr_data != db->db_buf) 228982c9918fSTim Haley VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, 229082c9918fSTim Haley db) == 1); 2291b24ab676SJeff Bonwick else if (!arc_released(db->db_buf)) 2292c717a561Smaybee arc_set_callback(db->db_buf, dbuf_do_evict, db); 229382c9918fSTim Haley } 2294c717a561Smaybee } else { 2295c717a561Smaybee ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 2296c717a561Smaybee ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2297c717a561Smaybee if (!BP_IS_HOLE(db->db_blkptr)) { 2298c717a561Smaybee int epbs = 2299c717a561Smaybee dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2300c717a561Smaybee ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, 2301c717a561Smaybee db->db.db_size); 2302c717a561Smaybee ASSERT3U(dn->dn_phys->dn_maxblkid 2303c717a561Smaybee >> (db->db_level * epbs), >=, db->db_blkid); 2304c717a561Smaybee arc_set_callback(db->db_buf, dbuf_do_evict, db); 2305c717a561Smaybee } 2306c25056deSgw25295 mutex_destroy(&dr->dt.di.dr_mtx); 2307c25056deSgw25295 list_destroy(&dr->dt.di.dr_children); 2308c717a561Smaybee } 2309c717a561Smaybee kmem_free(dr, sizeof (dbuf_dirty_record_t)); 2310c717a561Smaybee 2311c717a561Smaybee cv_broadcast(&db->db_changed); 2312c717a561Smaybee ASSERT(db->db_dirtycnt > 0); 2313c717a561Smaybee db->db_dirtycnt -= 1; 2314c717a561Smaybee db->db_data_pending = NULL; 2315b24ab676SJeff Bonwick dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg); 2316b24ab676SJeff Bonwick } 2317b24ab676SJeff Bonwick 2318b24ab676SJeff Bonwick static void 2319b24ab676SJeff Bonwick dbuf_write_nofill_ready(zio_t *zio) 2320b24ab676SJeff Bonwick { 2321b24ab676SJeff Bonwick dbuf_write_ready(zio, NULL, zio->io_private); 2322b24ab676SJeff Bonwick } 2323b24ab676SJeff Bonwick 2324b24ab676SJeff Bonwick static void 2325b24ab676SJeff Bonwick dbuf_write_nofill_done(zio_t *zio) 2326b24ab676SJeff Bonwick { 2327b24ab676SJeff Bonwick dbuf_write_done(zio, NULL, zio->io_private); 2328b24ab676SJeff Bonwick } 2329b24ab676SJeff Bonwick 2330b24ab676SJeff Bonwick static void 2331b24ab676SJeff Bonwick dbuf_write_override_ready(zio_t *zio) 2332b24ab676SJeff Bonwick { 2333b24ab676SJeff Bonwick dbuf_dirty_record_t *dr = zio->io_private; 2334b24ab676SJeff Bonwick dmu_buf_impl_t *db = dr->dr_dbuf; 2335b24ab676SJeff Bonwick 2336b24ab676SJeff Bonwick dbuf_write_ready(zio, NULL, db); 2337b24ab676SJeff Bonwick } 2338b24ab676SJeff Bonwick 2339b24ab676SJeff Bonwick static void 2340b24ab676SJeff Bonwick dbuf_write_override_done(zio_t *zio) 2341b24ab676SJeff Bonwick { 2342b24ab676SJeff Bonwick dbuf_dirty_record_t *dr = zio->io_private; 2343b24ab676SJeff Bonwick dmu_buf_impl_t *db = dr->dr_dbuf; 2344b24ab676SJeff Bonwick blkptr_t *obp = &dr->dt.dl.dr_overridden_by; 2345b24ab676SJeff Bonwick 2346b24ab676SJeff Bonwick mutex_enter(&db->db_mtx); 2347b24ab676SJeff Bonwick if (!BP_EQUAL(zio->io_bp, obp)) { 2348b24ab676SJeff Bonwick if (!BP_IS_HOLE(obp)) 2349b24ab676SJeff Bonwick dsl_free(spa_get_dsl(zio->io_spa), zio->io_txg, obp); 2350b24ab676SJeff Bonwick arc_release(dr->dt.dl.dr_data, db); 2351b24ab676SJeff Bonwick } 2352c717a561Smaybee mutex_exit(&db->db_mtx); 2353c717a561Smaybee 2354b24ab676SJeff Bonwick dbuf_write_done(zio, NULL, db); 2355b24ab676SJeff Bonwick } 2356c717a561Smaybee 2357b24ab676SJeff Bonwick static void 2358b24ab676SJeff Bonwick dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx) 2359b24ab676SJeff Bonwick { 2360b24ab676SJeff Bonwick dmu_buf_impl_t *db = dr->dr_dbuf; 2361b24ab676SJeff Bonwick dnode_t *dn = db->db_dnode; 2362b24ab676SJeff Bonwick objset_t *os = dn->dn_objset; 2363b24ab676SJeff Bonwick dmu_buf_impl_t *parent = db->db_parent; 2364b24ab676SJeff Bonwick uint64_t txg = tx->tx_txg; 2365b24ab676SJeff Bonwick zbookmark_t zb; 2366b24ab676SJeff Bonwick zio_prop_t zp; 2367b24ab676SJeff Bonwick zio_t *zio; 2368b24ab676SJeff Bonwick 2369b24ab676SJeff Bonwick if (db->db_state != DB_NOFILL) { 2370b24ab676SJeff Bonwick if (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE) { 2371b24ab676SJeff Bonwick /* 2372b24ab676SJeff Bonwick * Private object buffers are released here rather 2373b24ab676SJeff Bonwick * than in dbuf_dirty() since they are only modified 2374b24ab676SJeff Bonwick * in the syncing context and we don't want the 2375b24ab676SJeff Bonwick * overhead of making multiple copies of the data. 2376b24ab676SJeff Bonwick */ 2377b24ab676SJeff Bonwick if (BP_IS_HOLE(db->db_blkptr)) { 2378b24ab676SJeff Bonwick arc_buf_thaw(data); 2379b24ab676SJeff Bonwick } else { 2380b24ab676SJeff Bonwick arc_release(data, db); 2381b24ab676SJeff Bonwick } 2382b24ab676SJeff Bonwick } 2383b24ab676SJeff Bonwick } 2384b24ab676SJeff Bonwick 2385b24ab676SJeff Bonwick if (parent != dn->dn_dbuf) { 2386b24ab676SJeff Bonwick ASSERT(parent && parent->db_data_pending); 2387b24ab676SJeff Bonwick ASSERT(db->db_level == parent->db_level-1); 2388b24ab676SJeff Bonwick ASSERT(arc_released(parent->db_buf)); 2389b24ab676SJeff Bonwick zio = parent->db_data_pending->dr_zio; 2390b24ab676SJeff Bonwick } else { 2391b24ab676SJeff Bonwick ASSERT(db->db_level == dn->dn_phys->dn_nlevels-1); 2392b24ab676SJeff Bonwick ASSERT3P(db->db_blkptr, ==, 2393b24ab676SJeff Bonwick &dn->dn_phys->dn_blkptr[db->db_blkid]); 2394b24ab676SJeff Bonwick zio = dn->dn_zio; 2395b24ab676SJeff Bonwick } 2396b24ab676SJeff Bonwick 2397b24ab676SJeff Bonwick ASSERT(db->db_level == 0 || data == db->db_buf); 2398b24ab676SJeff Bonwick ASSERT3U(db->db_blkptr->blk_birth, <=, txg); 2399b24ab676SJeff Bonwick ASSERT(zio); 2400b24ab676SJeff Bonwick 2401b24ab676SJeff Bonwick SET_BOOKMARK(&zb, os->os_dsl_dataset ? 2402b24ab676SJeff Bonwick os->os_dsl_dataset->ds_object : DMU_META_OBJSET, 2403b24ab676SJeff Bonwick db->db.db_object, db->db_level, db->db_blkid); 2404b24ab676SJeff Bonwick 2405b24ab676SJeff Bonwick dmu_write_policy(os, dn, db->db_level, 2406b24ab676SJeff Bonwick db->db_state == DB_NOFILL ? WP_NOFILL : 0, &zp); 2407b24ab676SJeff Bonwick 2408b24ab676SJeff Bonwick if (db->db_level == 0 && dr->dt.dl.dr_override_state == DR_OVERRIDDEN) { 2409b24ab676SJeff Bonwick ASSERT(db->db_state != DB_NOFILL); 2410b24ab676SJeff Bonwick dr->dr_zio = zio_write(zio, os->os_spa, txg, 2411b24ab676SJeff Bonwick db->db_blkptr, data->b_data, arc_buf_size(data), &zp, 2412b24ab676SJeff Bonwick dbuf_write_override_ready, dbuf_write_override_done, dr, 2413b24ab676SJeff Bonwick ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); 2414b24ab676SJeff Bonwick mutex_enter(&db->db_mtx); 2415b24ab676SJeff Bonwick dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 2416b24ab676SJeff Bonwick zio_write_override(dr->dr_zio, &dr->dt.dl.dr_overridden_by, 2417b24ab676SJeff Bonwick dr->dt.dl.dr_copies); 2418b24ab676SJeff Bonwick mutex_exit(&db->db_mtx); 2419b24ab676SJeff Bonwick } else if (db->db_state == DB_NOFILL) { 2420b24ab676SJeff Bonwick ASSERT(zp.zp_checksum == ZIO_CHECKSUM_OFF); 2421b24ab676SJeff Bonwick dr->dr_zio = zio_write(zio, os->os_spa, txg, 2422b24ab676SJeff Bonwick db->db_blkptr, NULL, db->db.db_size, &zp, 2423b24ab676SJeff Bonwick dbuf_write_nofill_ready, dbuf_write_nofill_done, db, 2424b24ab676SJeff Bonwick ZIO_PRIORITY_ASYNC_WRITE, 2425b24ab676SJeff Bonwick ZIO_FLAG_MUSTSUCCEED | ZIO_FLAG_NODATA, &zb); 2426b24ab676SJeff Bonwick } else { 2427b24ab676SJeff Bonwick ASSERT(arc_released(data)); 2428b24ab676SJeff Bonwick dr->dr_zio = arc_write(zio, os->os_spa, txg, 2429b24ab676SJeff Bonwick db->db_blkptr, data, DBUF_IS_L2CACHEABLE(db), &zp, 2430b24ab676SJeff Bonwick dbuf_write_ready, dbuf_write_done, db, 2431b24ab676SJeff Bonwick ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); 2432b24ab676SJeff Bonwick } 2433fa9e4066Sahrens } 2434