1fa9e4066Sahrens /* 2fa9e4066Sahrens * CDDL HEADER START 3fa9e4066Sahrens * 4fa9e4066Sahrens * The contents of this file are subject to the terms of the 5f65e61c0Sahrens * Common Development and Distribution License (the "License"). 6f65e61c0Sahrens * You may not use this file except in compliance with the License. 7fa9e4066Sahrens * 8fa9e4066Sahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9fa9e4066Sahrens * or http://www.opensolaris.org/os/licensing. 10fa9e4066Sahrens * See the License for the specific language governing permissions 11fa9e4066Sahrens * and limitations under the License. 12fa9e4066Sahrens * 13fa9e4066Sahrens * When distributing Covered Code, include this CDDL HEADER in each 14fa9e4066Sahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15fa9e4066Sahrens * If applicable, add the following below this CDDL HEADER, with the 16fa9e4066Sahrens * fields enclosed by brackets "[]" replaced with your own identifying 17fa9e4066Sahrens * information: Portions Copyright [yyyy] [name of copyright owner] 18fa9e4066Sahrens * 19fa9e4066Sahrens * CDDL HEADER END 20fa9e4066Sahrens */ 21fa9e4066Sahrens /* 225a98e54bSBrendan Gregg - Sun Microsystems * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23fa9e4066Sahrens * Use is subject to license terms. 24fa9e4066Sahrens */ 25fa9e4066Sahrens 26fa9e4066Sahrens #include <sys/zfs_context.h> 27fa9e4066Sahrens #include <sys/dmu.h> 28fa9e4066Sahrens #include <sys/dmu_impl.h> 29fa9e4066Sahrens #include <sys/dbuf.h> 30fa9e4066Sahrens #include <sys/dmu_objset.h> 31fa9e4066Sahrens #include <sys/dsl_dataset.h> 32fa9e4066Sahrens #include <sys/dsl_dir.h> 33fa9e4066Sahrens #include <sys/dmu_tx.h> 34fa9e4066Sahrens #include <sys/spa.h> 35fa9e4066Sahrens #include <sys/zio.h> 36fa9e4066Sahrens #include <sys/dmu_zfetch.h> 37fa9e4066Sahrens 38fa9e4066Sahrens static void dbuf_destroy(dmu_buf_impl_t *db); 39fa9e4066Sahrens static int dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx); 40088f3894Sahrens static void dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx); 41c717a561Smaybee static arc_done_func_t dbuf_write_ready; 42fa9e4066Sahrens static arc_done_func_t dbuf_write_done; 4382c9918fSTim Haley static zio_done_func_t dbuf_skip_write_ready; 4482c9918fSTim Haley static zio_done_func_t dbuf_skip_write_done; 45fa9e4066Sahrens 46fa9e4066Sahrens /* 47fa9e4066Sahrens * Global data structures and functions for the dbuf cache. 48fa9e4066Sahrens */ 49fa9e4066Sahrens static kmem_cache_t *dbuf_cache; 50fa9e4066Sahrens 51fa9e4066Sahrens /* ARGSUSED */ 52fa9e4066Sahrens static int 53fa9e4066Sahrens dbuf_cons(void *vdb, void *unused, int kmflag) 54fa9e4066Sahrens { 55fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 56fa9e4066Sahrens bzero(db, sizeof (dmu_buf_impl_t)); 57fa9e4066Sahrens 58fa9e4066Sahrens mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL); 59fa9e4066Sahrens cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL); 60fa9e4066Sahrens refcount_create(&db->db_holds); 61fa9e4066Sahrens return (0); 62fa9e4066Sahrens } 63fa9e4066Sahrens 64fa9e4066Sahrens /* ARGSUSED */ 65fa9e4066Sahrens static void 66fa9e4066Sahrens dbuf_dest(void *vdb, void *unused) 67fa9e4066Sahrens { 68fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 69fa9e4066Sahrens mutex_destroy(&db->db_mtx); 70fa9e4066Sahrens cv_destroy(&db->db_changed); 71fa9e4066Sahrens refcount_destroy(&db->db_holds); 72fa9e4066Sahrens } 73fa9e4066Sahrens 74fa9e4066Sahrens /* 75fa9e4066Sahrens * dbuf hash table routines 76fa9e4066Sahrens */ 77fa9e4066Sahrens static dbuf_hash_table_t dbuf_hash_table; 78fa9e4066Sahrens 79fa9e4066Sahrens static uint64_t dbuf_hash_count; 80fa9e4066Sahrens 81fa9e4066Sahrens static uint64_t 82fa9e4066Sahrens dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid) 83fa9e4066Sahrens { 84fa9e4066Sahrens uintptr_t osv = (uintptr_t)os; 85fa9e4066Sahrens uint64_t crc = -1ULL; 86fa9e4066Sahrens 87fa9e4066Sahrens ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 88fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (lvl)) & 0xFF]; 89fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF]; 90fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF]; 91fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF]; 92fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 0)) & 0xFF]; 93fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 8)) & 0xFF]; 94fa9e4066Sahrens 95fa9e4066Sahrens crc ^= (osv>>14) ^ (obj>>16) ^ (blkid>>16); 96fa9e4066Sahrens 97fa9e4066Sahrens return (crc); 98fa9e4066Sahrens } 99fa9e4066Sahrens 100fa9e4066Sahrens #define DBUF_HASH(os, obj, level, blkid) dbuf_hash(os, obj, level, blkid); 101fa9e4066Sahrens 102fa9e4066Sahrens #define DBUF_EQUAL(dbuf, os, obj, level, blkid) \ 103fa9e4066Sahrens ((dbuf)->db.db_object == (obj) && \ 104fa9e4066Sahrens (dbuf)->db_objset == (os) && \ 105fa9e4066Sahrens (dbuf)->db_level == (level) && \ 106fa9e4066Sahrens (dbuf)->db_blkid == (blkid)) 107fa9e4066Sahrens 108fa9e4066Sahrens dmu_buf_impl_t * 109fa9e4066Sahrens dbuf_find(dnode_t *dn, uint8_t level, uint64_t blkid) 110fa9e4066Sahrens { 111fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 112fa9e4066Sahrens objset_impl_t *os = dn->dn_objset; 113fa9e4066Sahrens uint64_t obj = dn->dn_object; 114fa9e4066Sahrens uint64_t hv = DBUF_HASH(os, obj, level, blkid); 115fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 116fa9e4066Sahrens dmu_buf_impl_t *db; 117fa9e4066Sahrens 118fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 119fa9e4066Sahrens for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) { 120fa9e4066Sahrens if (DBUF_EQUAL(db, os, obj, level, blkid)) { 121fa9e4066Sahrens mutex_enter(&db->db_mtx); 122ea8dc4b6Seschrock if (db->db_state != DB_EVICTING) { 123fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 124fa9e4066Sahrens return (db); 125fa9e4066Sahrens } 126fa9e4066Sahrens mutex_exit(&db->db_mtx); 127fa9e4066Sahrens } 128fa9e4066Sahrens } 129fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 130fa9e4066Sahrens return (NULL); 131fa9e4066Sahrens } 132fa9e4066Sahrens 133fa9e4066Sahrens /* 134fa9e4066Sahrens * Insert an entry into the hash table. If there is already an element 135fa9e4066Sahrens * equal to elem in the hash table, then the already existing element 136fa9e4066Sahrens * will be returned and the new element will not be inserted. 137fa9e4066Sahrens * Otherwise returns NULL. 138fa9e4066Sahrens */ 139fa9e4066Sahrens static dmu_buf_impl_t * 140fa9e4066Sahrens dbuf_hash_insert(dmu_buf_impl_t *db) 141fa9e4066Sahrens { 142fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 143fa9e4066Sahrens objset_impl_t *os = db->db_objset; 144fa9e4066Sahrens uint64_t obj = db->db.db_object; 145fa9e4066Sahrens int level = db->db_level; 146fa9e4066Sahrens uint64_t blkid = db->db_blkid; 147fa9e4066Sahrens uint64_t hv = DBUF_HASH(os, obj, level, blkid); 148fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 149fa9e4066Sahrens dmu_buf_impl_t *dbf; 150fa9e4066Sahrens 151fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 152fa9e4066Sahrens for (dbf = h->hash_table[idx]; dbf != NULL; dbf = dbf->db_hash_next) { 153fa9e4066Sahrens if (DBUF_EQUAL(dbf, os, obj, level, blkid)) { 154fa9e4066Sahrens mutex_enter(&dbf->db_mtx); 155ea8dc4b6Seschrock if (dbf->db_state != DB_EVICTING) { 156fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 157fa9e4066Sahrens return (dbf); 158fa9e4066Sahrens } 159fa9e4066Sahrens mutex_exit(&dbf->db_mtx); 160fa9e4066Sahrens } 161fa9e4066Sahrens } 162fa9e4066Sahrens 163fa9e4066Sahrens mutex_enter(&db->db_mtx); 164fa9e4066Sahrens db->db_hash_next = h->hash_table[idx]; 165fa9e4066Sahrens h->hash_table[idx] = db; 166fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 167fa9e4066Sahrens atomic_add_64(&dbuf_hash_count, 1); 168fa9e4066Sahrens 169fa9e4066Sahrens return (NULL); 170fa9e4066Sahrens } 171fa9e4066Sahrens 172fa9e4066Sahrens /* 173fa9e4066Sahrens * Remove an entry from the hash table. This operation will 174fa9e4066Sahrens * fail if there are any existing holds on the db. 175fa9e4066Sahrens */ 176fa9e4066Sahrens static void 177fa9e4066Sahrens dbuf_hash_remove(dmu_buf_impl_t *db) 178fa9e4066Sahrens { 179fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 180fa9e4066Sahrens uint64_t hv = DBUF_HASH(db->db_objset, db->db.db_object, 181fa9e4066Sahrens db->db_level, db->db_blkid); 182fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 183fa9e4066Sahrens dmu_buf_impl_t *dbf, **dbp; 184fa9e4066Sahrens 185fa9e4066Sahrens /* 186fa9e4066Sahrens * We musn't hold db_mtx to maintin lock ordering: 187fa9e4066Sahrens * DBUF_HASH_MUTEX > db_mtx. 188fa9e4066Sahrens */ 189fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 190ea8dc4b6Seschrock ASSERT(db->db_state == DB_EVICTING); 191fa9e4066Sahrens ASSERT(!MUTEX_HELD(&db->db_mtx)); 192fa9e4066Sahrens 193fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 194fa9e4066Sahrens dbp = &h->hash_table[idx]; 195fa9e4066Sahrens while ((dbf = *dbp) != db) { 196fa9e4066Sahrens dbp = &dbf->db_hash_next; 197fa9e4066Sahrens ASSERT(dbf != NULL); 198fa9e4066Sahrens } 199fa9e4066Sahrens *dbp = db->db_hash_next; 200fa9e4066Sahrens db->db_hash_next = NULL; 201fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 202fa9e4066Sahrens atomic_add_64(&dbuf_hash_count, -1); 203fa9e4066Sahrens } 204fa9e4066Sahrens 205ea8dc4b6Seschrock static arc_evict_func_t dbuf_do_evict; 206fa9e4066Sahrens 207fa9e4066Sahrens static void 208fa9e4066Sahrens dbuf_evict_user(dmu_buf_impl_t *db) 209fa9e4066Sahrens { 210fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 211fa9e4066Sahrens 212c717a561Smaybee if (db->db_level != 0 || db->db_evict_func == NULL) 213fa9e4066Sahrens return; 214fa9e4066Sahrens 215c717a561Smaybee if (db->db_user_data_ptr_ptr) 216c717a561Smaybee *db->db_user_data_ptr_ptr = db->db.db_data; 217c717a561Smaybee db->db_evict_func(&db->db, db->db_user_ptr); 218c717a561Smaybee db->db_user_ptr = NULL; 219c717a561Smaybee db->db_user_data_ptr_ptr = NULL; 220c717a561Smaybee db->db_evict_func = NULL; 221fa9e4066Sahrens } 222fa9e4066Sahrens 223fa9e4066Sahrens void 224ea8dc4b6Seschrock dbuf_evict(dmu_buf_impl_t *db) 225ea8dc4b6Seschrock { 226ea8dc4b6Seschrock ASSERT(MUTEX_HELD(&db->db_mtx)); 227ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 228c717a561Smaybee ASSERT(db->db_data_pending == NULL); 229ea8dc4b6Seschrock 230ea8dc4b6Seschrock dbuf_clear(db); 231ea8dc4b6Seschrock dbuf_destroy(db); 232ea8dc4b6Seschrock } 233ea8dc4b6Seschrock 234ea8dc4b6Seschrock void 235fa9e4066Sahrens dbuf_init(void) 236fa9e4066Sahrens { 237ea8dc4b6Seschrock uint64_t hsize = 1ULL << 16; 238fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 239fa9e4066Sahrens int i; 240fa9e4066Sahrens 241fa9e4066Sahrens /* 242fa9e4066Sahrens * The hash table is big enough to fill all of physical memory 243ea8dc4b6Seschrock * with an average 4K block size. The table will take up 244ea8dc4b6Seschrock * totalmem*sizeof(void*)/4K (i.e. 2MB/GB with 8-byte pointers). 245fa9e4066Sahrens */ 246ea8dc4b6Seschrock while (hsize * 4096 < physmem * PAGESIZE) 247fa9e4066Sahrens hsize <<= 1; 248fa9e4066Sahrens 249ea8dc4b6Seschrock retry: 250fa9e4066Sahrens h->hash_table_mask = hsize - 1; 251ea8dc4b6Seschrock h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP); 252ea8dc4b6Seschrock if (h->hash_table == NULL) { 253ea8dc4b6Seschrock /* XXX - we should really return an error instead of assert */ 254ea8dc4b6Seschrock ASSERT(hsize > (1ULL << 10)); 255ea8dc4b6Seschrock hsize >>= 1; 256ea8dc4b6Seschrock goto retry; 257ea8dc4b6Seschrock } 258fa9e4066Sahrens 259fa9e4066Sahrens dbuf_cache = kmem_cache_create("dmu_buf_impl_t", 260fa9e4066Sahrens sizeof (dmu_buf_impl_t), 261fa9e4066Sahrens 0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0); 262fa9e4066Sahrens 263fa9e4066Sahrens for (i = 0; i < DBUF_MUTEXES; i++) 264fa9e4066Sahrens mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL); 265fa9e4066Sahrens } 266fa9e4066Sahrens 267fa9e4066Sahrens void 268fa9e4066Sahrens dbuf_fini(void) 269fa9e4066Sahrens { 270fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 271fa9e4066Sahrens int i; 272fa9e4066Sahrens 273fa9e4066Sahrens for (i = 0; i < DBUF_MUTEXES; i++) 274fa9e4066Sahrens mutex_destroy(&h->hash_mutexes[i]); 275fa9e4066Sahrens kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *)); 276fa9e4066Sahrens kmem_cache_destroy(dbuf_cache); 277fa9e4066Sahrens } 278fa9e4066Sahrens 279fa9e4066Sahrens /* 280fa9e4066Sahrens * Other stuff. 281fa9e4066Sahrens */ 282fa9e4066Sahrens 2839c9dc39aSek110237 #ifdef ZFS_DEBUG 284fa9e4066Sahrens static void 285fa9e4066Sahrens dbuf_verify(dmu_buf_impl_t *db) 286fa9e4066Sahrens { 287fa9e4066Sahrens dnode_t *dn = db->db_dnode; 288fa9e4066Sahrens 289fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 290fa9e4066Sahrens 291fa9e4066Sahrens if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY)) 292fa9e4066Sahrens return; 293fa9e4066Sahrens 294fa9e4066Sahrens ASSERT(db->db_objset != NULL); 295fa9e4066Sahrens if (dn == NULL) { 296fa9e4066Sahrens ASSERT(db->db_parent == NULL); 297fa9e4066Sahrens ASSERT(db->db_blkptr == NULL); 298fa9e4066Sahrens } else { 299fa9e4066Sahrens ASSERT3U(db->db.db_object, ==, dn->dn_object); 300fa9e4066Sahrens ASSERT3P(db->db_objset, ==, dn->dn_objset); 301fa9e4066Sahrens ASSERT3U(db->db_level, <, dn->dn_nlevels); 302ea8dc4b6Seschrock ASSERT(db->db_blkid == DB_BONUS_BLKID || 303ea8dc4b6Seschrock list_head(&dn->dn_dbufs)); 304fa9e4066Sahrens } 305fa9e4066Sahrens if (db->db_blkid == DB_BONUS_BLKID) { 306fa9e4066Sahrens ASSERT(dn != NULL); 3071934e92fSmaybee ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 308fa9e4066Sahrens ASSERT3U(db->db.db_offset, ==, DB_BONUS_BLKID); 309fa9e4066Sahrens } else { 310fa9e4066Sahrens ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size); 311fa9e4066Sahrens } 312fa9e4066Sahrens 31388b7b0f2SMatthew Ahrens /* 31488b7b0f2SMatthew Ahrens * We can't assert that db_size matches dn_datablksz because it 31588b7b0f2SMatthew Ahrens * can be momentarily different when another thread is doing 31688b7b0f2SMatthew Ahrens * dnode_set_blksz(). 31788b7b0f2SMatthew Ahrens */ 31888b7b0f2SMatthew Ahrens if (db->db_level == 0 && db->db.db_object == DMU_META_DNODE_OBJECT) { 319c717a561Smaybee dbuf_dirty_record_t *dr = db->db_data_pending; 320fa9e4066Sahrens /* 32188b7b0f2SMatthew Ahrens * It should only be modified in syncing context, so 32288b7b0f2SMatthew Ahrens * make sure we only have one copy of the data. 323fa9e4066Sahrens */ 324c717a561Smaybee ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf); 325fa9e4066Sahrens } 326fa9e4066Sahrens 327fa9e4066Sahrens /* verify db->db_blkptr */ 328fa9e4066Sahrens if (db->db_blkptr) { 329fa9e4066Sahrens if (db->db_parent == dn->dn_dbuf) { 330fa9e4066Sahrens /* db is pointed to by the dnode */ 331fa9e4066Sahrens /* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */ 33214843421SMatthew Ahrens if (DMU_OBJECT_IS_SPECIAL(db->db.db_object)) 333fa9e4066Sahrens ASSERT(db->db_parent == NULL); 334fa9e4066Sahrens else 335fa9e4066Sahrens ASSERT(db->db_parent != NULL); 336fa9e4066Sahrens ASSERT3P(db->db_blkptr, ==, 337fa9e4066Sahrens &dn->dn_phys->dn_blkptr[db->db_blkid]); 338fa9e4066Sahrens } else { 339fa9e4066Sahrens /* db is pointed to by an indirect block */ 340fa9e4066Sahrens int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT; 341fa9e4066Sahrens ASSERT3U(db->db_parent->db_level, ==, db->db_level+1); 342fa9e4066Sahrens ASSERT3U(db->db_parent->db.db_object, ==, 343fa9e4066Sahrens db->db.db_object); 344fa9e4066Sahrens /* 345fa9e4066Sahrens * dnode_grow_indblksz() can make this fail if we don't 346fa9e4066Sahrens * have the struct_rwlock. XXX indblksz no longer 347fa9e4066Sahrens * grows. safe to do this now? 348fa9e4066Sahrens */ 349fa9e4066Sahrens if (RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)) { 350fa9e4066Sahrens ASSERT3P(db->db_blkptr, ==, 351fa9e4066Sahrens ((blkptr_t *)db->db_parent->db.db_data + 352fa9e4066Sahrens db->db_blkid % epb)); 353fa9e4066Sahrens } 354fa9e4066Sahrens } 355fa9e4066Sahrens } 356fa9e4066Sahrens if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) && 357fa9e4066Sahrens db->db.db_data && db->db_blkid != DB_BONUS_BLKID && 358fa9e4066Sahrens db->db_state != DB_FILL && !dn->dn_free_txg) { 359fa9e4066Sahrens /* 360fa9e4066Sahrens * If the blkptr isn't set but they have nonzero data, 361fa9e4066Sahrens * it had better be dirty, otherwise we'll lose that 362fa9e4066Sahrens * data when we evict this buffer. 363fa9e4066Sahrens */ 364fa9e4066Sahrens if (db->db_dirtycnt == 0) { 365fa9e4066Sahrens uint64_t *buf = db->db.db_data; 366fa9e4066Sahrens int i; 367fa9e4066Sahrens 368fa9e4066Sahrens for (i = 0; i < db->db.db_size >> 3; i++) { 369fa9e4066Sahrens ASSERT(buf[i] == 0); 370fa9e4066Sahrens } 371fa9e4066Sahrens } 372fa9e4066Sahrens } 373fa9e4066Sahrens } 3749c9dc39aSek110237 #endif 375fa9e4066Sahrens 376fa9e4066Sahrens static void 377fa9e4066Sahrens dbuf_update_data(dmu_buf_impl_t *db) 378fa9e4066Sahrens { 379fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 380c717a561Smaybee if (db->db_level == 0 && db->db_user_data_ptr_ptr) { 381fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 382c717a561Smaybee *db->db_user_data_ptr_ptr = db->db.db_data; 383fa9e4066Sahrens } 384fa9e4066Sahrens } 385fa9e4066Sahrens 386fa9e4066Sahrens static void 387fa9e4066Sahrens dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf) 388fa9e4066Sahrens { 389fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 390ea8dc4b6Seschrock ASSERT(db->db_buf == NULL || !arc_has_callback(db->db_buf)); 391fa9e4066Sahrens db->db_buf = buf; 392ea8dc4b6Seschrock if (buf != NULL) { 393ea8dc4b6Seschrock ASSERT(buf->b_data != NULL); 394fa9e4066Sahrens db->db.db_data = buf->b_data; 395ea8dc4b6Seschrock if (!arc_released(buf)) 396ea8dc4b6Seschrock arc_set_callback(buf, dbuf_do_evict, db); 397fa9e4066Sahrens dbuf_update_data(db); 398ea8dc4b6Seschrock } else { 399ea8dc4b6Seschrock dbuf_evict_user(db); 400ea8dc4b6Seschrock db->db.db_data = NULL; 40182c9918fSTim Haley if (db->db_state != DB_NOFILL) 402ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 403ea8dc4b6Seschrock } 404fa9e4066Sahrens } 405fa9e4066Sahrens 406fa9e4066Sahrens uint64_t 407fa9e4066Sahrens dbuf_whichblock(dnode_t *dn, uint64_t offset) 408fa9e4066Sahrens { 409fa9e4066Sahrens if (dn->dn_datablkshift) { 410fa9e4066Sahrens return (offset >> dn->dn_datablkshift); 411fa9e4066Sahrens } else { 412fa9e4066Sahrens ASSERT3U(offset, <, dn->dn_datablksz); 413fa9e4066Sahrens return (0); 414fa9e4066Sahrens } 415fa9e4066Sahrens } 416fa9e4066Sahrens 417fa9e4066Sahrens static void 418fa9e4066Sahrens dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb) 419fa9e4066Sahrens { 420fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 421fa9e4066Sahrens 422fa9e4066Sahrens mutex_enter(&db->db_mtx); 423fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_READ); 424fa9e4066Sahrens /* 425fa9e4066Sahrens * All reads are synchronous, so we must have a hold on the dbuf 426fa9e4066Sahrens */ 427fa9e4066Sahrens ASSERT(refcount_count(&db->db_holds) > 0); 428ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 429fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 430c717a561Smaybee if (db->db_level == 0 && db->db_freed_in_flight) { 431fa9e4066Sahrens /* we were freed in flight; disregard any error */ 432fa9e4066Sahrens arc_release(buf, db); 433fa9e4066Sahrens bzero(buf->b_data, db->db.db_size); 4346b4acc8bSahrens arc_buf_freeze(buf); 435c717a561Smaybee db->db_freed_in_flight = FALSE; 436fa9e4066Sahrens dbuf_set_data(db, buf); 437fa9e4066Sahrens db->db_state = DB_CACHED; 438fa9e4066Sahrens } else if (zio == NULL || zio->io_error == 0) { 439fa9e4066Sahrens dbuf_set_data(db, buf); 440fa9e4066Sahrens db->db_state = DB_CACHED; 441fa9e4066Sahrens } else { 442fa9e4066Sahrens ASSERT(db->db_blkid != DB_BONUS_BLKID); 443fa9e4066Sahrens ASSERT3P(db->db_buf, ==, NULL); 444ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(buf, db) == 1); 445ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 446fa9e4066Sahrens } 447fa9e4066Sahrens cv_broadcast(&db->db_changed); 448fa9e4066Sahrens mutex_exit(&db->db_mtx); 449ea8dc4b6Seschrock dbuf_rele(db, NULL); 450fa9e4066Sahrens } 451fa9e4066Sahrens 452ea8dc4b6Seschrock static void 45313506d1eSmaybee dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t *flags) 454fa9e4066Sahrens { 455088f3894Sahrens dnode_t *dn = db->db_dnode; 456ea8dc4b6Seschrock zbookmark_t zb; 45713506d1eSmaybee uint32_t aflags = ARC_NOWAIT; 458088f3894Sahrens arc_buf_t *pbuf; 459fa9e4066Sahrens 460fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 461fa9e4066Sahrens /* We need the struct_rwlock to prevent db_blkptr from changing. */ 462088f3894Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 463ea8dc4b6Seschrock ASSERT(MUTEX_HELD(&db->db_mtx)); 464ea8dc4b6Seschrock ASSERT(db->db_state == DB_UNCACHED); 465ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 466fa9e4066Sahrens 467fa9e4066Sahrens if (db->db_blkid == DB_BONUS_BLKID) { 468cf04dda1SMark Maybee int bonuslen = MIN(dn->dn_bonuslen, dn->dn_phys->dn_bonuslen); 4691934e92fSmaybee 4701934e92fSmaybee ASSERT3U(bonuslen, <=, db->db.db_size); 471ea8dc4b6Seschrock db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN); 4725a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 4731934e92fSmaybee if (bonuslen < DN_MAX_BONUSLEN) 474ea8dc4b6Seschrock bzero(db->db.db_data, DN_MAX_BONUSLEN); 475cf04dda1SMark Maybee if (bonuslen) 476cf04dda1SMark Maybee bcopy(DN_BONUS(dn->dn_phys), db->db.db_data, bonuslen); 477ea8dc4b6Seschrock dbuf_update_data(db); 478fa9e4066Sahrens db->db_state = DB_CACHED; 479fa9e4066Sahrens mutex_exit(&db->db_mtx); 480fa9e4066Sahrens return; 481fa9e4066Sahrens } 482fa9e4066Sahrens 4831c8564a7SMark Maybee /* 4841c8564a7SMark Maybee * Recheck BP_IS_HOLE() after dnode_block_freed() in case dnode_sync() 4851c8564a7SMark Maybee * processes the delete record and clears the bp while we are waiting 4861c8564a7SMark Maybee * for the dn_mtx (resulting in a "no" from block_freed). 4871c8564a7SMark Maybee */ 488088f3894Sahrens if (db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr) || 4891c8564a7SMark Maybee (db->db_level == 0 && (dnode_block_freed(dn, db->db_blkid) || 4901c8564a7SMark Maybee BP_IS_HOLE(db->db_blkptr)))) { 491ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 492ad23a2dbSjohansen 493088f3894Sahrens dbuf_set_data(db, arc_buf_alloc(dn->dn_objset->os_spa, 494ad23a2dbSjohansen db->db.db_size, db, type)); 495fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 496fa9e4066Sahrens db->db_state = DB_CACHED; 49713506d1eSmaybee *flags |= DB_RF_CACHED; 498fa9e4066Sahrens mutex_exit(&db->db_mtx); 499fa9e4066Sahrens return; 500fa9e4066Sahrens } 501fa9e4066Sahrens 502fa9e4066Sahrens db->db_state = DB_READ; 503fa9e4066Sahrens mutex_exit(&db->db_mtx); 504fa9e4066Sahrens 5053baa08fcSek110237 if (DBUF_IS_L2CACHEABLE(db)) 5063baa08fcSek110237 aflags |= ARC_L2CACHE; 5073baa08fcSek110237 508ea8dc4b6Seschrock zb.zb_objset = db->db_objset->os_dsl_dataset ? 509ea8dc4b6Seschrock db->db_objset->os_dsl_dataset->ds_object : 0; 510ea8dc4b6Seschrock zb.zb_object = db->db.db_object; 511ea8dc4b6Seschrock zb.zb_level = db->db_level; 512ea8dc4b6Seschrock zb.zb_blkid = db->db_blkid; 513ea8dc4b6Seschrock 514ea8dc4b6Seschrock dbuf_add_ref(db, NULL); 515fa9e4066Sahrens /* ZIO_FLAG_CANFAIL callers have to check the parent zio's error */ 516088f3894Sahrens 517088f3894Sahrens if (db->db_parent) 518088f3894Sahrens pbuf = db->db_parent->db_buf; 519088f3894Sahrens else 520088f3894Sahrens pbuf = db->db_objset->os_phys_buf; 521088f3894Sahrens 522088f3894Sahrens (void) arc_read(zio, dn->dn_objset->os_spa, db->db_blkptr, pbuf, 523fa9e4066Sahrens dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ, 52413506d1eSmaybee (*flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED, 52513506d1eSmaybee &aflags, &zb); 52613506d1eSmaybee if (aflags & ARC_CACHED) 52713506d1eSmaybee *flags |= DB_RF_CACHED; 528fa9e4066Sahrens } 529fa9e4066Sahrens 530ea8dc4b6Seschrock int 531ea8dc4b6Seschrock dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags) 532fa9e4066Sahrens { 533ea8dc4b6Seschrock int err = 0; 534ea8dc4b6Seschrock int havepzio = (zio != NULL); 53513506d1eSmaybee int prefetch; 536fa9e4066Sahrens 537fa9e4066Sahrens /* 538fa9e4066Sahrens * We don't have to hold the mutex to check db_state because it 539fa9e4066Sahrens * can't be freed while we have a hold on the buffer. 540fa9e4066Sahrens */ 541fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 542fa9e4066Sahrens 54382c9918fSTim Haley if (db->db_state == DB_NOFILL) 54482c9918fSTim Haley return (EIO); 54582c9918fSTim Haley 546fa9e4066Sahrens if ((flags & DB_RF_HAVESTRUCT) == 0) 547fa9e4066Sahrens rw_enter(&db->db_dnode->dn_struct_rwlock, RW_READER); 548fa9e4066Sahrens 54913506d1eSmaybee prefetch = db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID && 5503baa08fcSek110237 (flags & DB_RF_NOPREFETCH) == 0 && db->db_dnode != NULL && 5513baa08fcSek110237 DBUF_IS_CACHEABLE(db); 55213506d1eSmaybee 553fa9e4066Sahrens mutex_enter(&db->db_mtx); 554ea8dc4b6Seschrock if (db->db_state == DB_CACHED) { 555ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 55613506d1eSmaybee if (prefetch) 55713506d1eSmaybee dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset, 55813506d1eSmaybee db->db.db_size, TRUE); 559ea8dc4b6Seschrock if ((flags & DB_RF_HAVESTRUCT) == 0) 560ea8dc4b6Seschrock rw_exit(&db->db_dnode->dn_struct_rwlock); 561ea8dc4b6Seschrock } else if (db->db_state == DB_UNCACHED) { 562ea8dc4b6Seschrock if (zio == NULL) { 563ea8dc4b6Seschrock zio = zio_root(db->db_dnode->dn_objset->os_spa, 564ea8dc4b6Seschrock NULL, NULL, ZIO_FLAG_CANFAIL); 565ea8dc4b6Seschrock } 56613506d1eSmaybee dbuf_read_impl(db, zio, &flags); 56713506d1eSmaybee 568ea8dc4b6Seschrock /* dbuf_read_impl has dropped db_mtx for us */ 569ea8dc4b6Seschrock 57013506d1eSmaybee if (prefetch) 571ea8dc4b6Seschrock dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset, 57213506d1eSmaybee db->db.db_size, flags & DB_RF_CACHED); 573ea8dc4b6Seschrock 574ea8dc4b6Seschrock if ((flags & DB_RF_HAVESTRUCT) == 0) 575ea8dc4b6Seschrock rw_exit(&db->db_dnode->dn_struct_rwlock); 576ea8dc4b6Seschrock 577ea8dc4b6Seschrock if (!havepzio) 578ea8dc4b6Seschrock err = zio_wait(zio); 579ea8dc4b6Seschrock } else { 58013506d1eSmaybee mutex_exit(&db->db_mtx); 58113506d1eSmaybee if (prefetch) 58213506d1eSmaybee dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset, 58313506d1eSmaybee db->db.db_size, TRUE); 584ea8dc4b6Seschrock if ((flags & DB_RF_HAVESTRUCT) == 0) 585ea8dc4b6Seschrock rw_exit(&db->db_dnode->dn_struct_rwlock); 58613506d1eSmaybee 58713506d1eSmaybee mutex_enter(&db->db_mtx); 588ea8dc4b6Seschrock if ((flags & DB_RF_NEVERWAIT) == 0) { 589ea8dc4b6Seschrock while (db->db_state == DB_READ || 590ea8dc4b6Seschrock db->db_state == DB_FILL) { 591fa9e4066Sahrens ASSERT(db->db_state == DB_READ || 592fa9e4066Sahrens (flags & DB_RF_HAVESTRUCT) == 0); 593fa9e4066Sahrens cv_wait(&db->db_changed, &db->db_mtx); 594fa9e4066Sahrens } 595ea8dc4b6Seschrock if (db->db_state == DB_UNCACHED) 596ea8dc4b6Seschrock err = EIO; 597ea8dc4b6Seschrock } 598fa9e4066Sahrens mutex_exit(&db->db_mtx); 599fa9e4066Sahrens } 600fa9e4066Sahrens 601ea8dc4b6Seschrock ASSERT(err || havepzio || db->db_state == DB_CACHED); 602ea8dc4b6Seschrock return (err); 603fa9e4066Sahrens } 604fa9e4066Sahrens 605fa9e4066Sahrens static void 606fa9e4066Sahrens dbuf_noread(dmu_buf_impl_t *db) 607fa9e4066Sahrens { 608fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 609ea8dc4b6Seschrock ASSERT(db->db_blkid != DB_BONUS_BLKID); 610fa9e4066Sahrens mutex_enter(&db->db_mtx); 611fa9e4066Sahrens while (db->db_state == DB_READ || db->db_state == DB_FILL) 612fa9e4066Sahrens cv_wait(&db->db_changed, &db->db_mtx); 613fa9e4066Sahrens if (db->db_state == DB_UNCACHED) { 614ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 615ad23a2dbSjohansen 616ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 617fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 618fa9e4066Sahrens dbuf_set_data(db, arc_buf_alloc(db->db_dnode->dn_objset->os_spa, 619ad23a2dbSjohansen db->db.db_size, db, type)); 620fa9e4066Sahrens db->db_state = DB_FILL; 62182c9918fSTim Haley } else if (db->db_state == DB_NOFILL) { 62282c9918fSTim Haley dbuf_set_data(db, NULL); 623fa9e4066Sahrens } else { 624fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_CACHED); 625fa9e4066Sahrens } 626fa9e4066Sahrens mutex_exit(&db->db_mtx); 627fa9e4066Sahrens } 628fa9e4066Sahrens 629fa9e4066Sahrens /* 630fa9e4066Sahrens * This is our just-in-time copy function. It makes a copy of 631fa9e4066Sahrens * buffers, that have been modified in a previous transaction 632fa9e4066Sahrens * group, before we modify them in the current active group. 633fa9e4066Sahrens * 634fa9e4066Sahrens * This function is used in two places: when we are dirtying a 635fa9e4066Sahrens * buffer for the first time in a txg, and when we are freeing 636fa9e4066Sahrens * a range in a dnode that includes this buffer. 637fa9e4066Sahrens * 638fa9e4066Sahrens * Note that when we are called from dbuf_free_range() we do 639fa9e4066Sahrens * not put a hold on the buffer, we just traverse the active 640fa9e4066Sahrens * dbuf list for the dnode. 641fa9e4066Sahrens */ 642fa9e4066Sahrens static void 643fa9e4066Sahrens dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg) 644fa9e4066Sahrens { 645c717a561Smaybee dbuf_dirty_record_t *dr = db->db_last_dirty; 646fa9e4066Sahrens 647fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 648fa9e4066Sahrens ASSERT(db->db.db_data != NULL); 649c717a561Smaybee ASSERT(db->db_level == 0); 650c717a561Smaybee ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT); 651fa9e4066Sahrens 6524d31c452Smaybee if (dr == NULL || 6534d31c452Smaybee (dr->dt.dl.dr_data != 6544d31c452Smaybee ((db->db_blkid == DB_BONUS_BLKID) ? db->db.db_data : db->db_buf))) 655fa9e4066Sahrens return; 656fa9e4066Sahrens 657fa9e4066Sahrens /* 658c717a561Smaybee * If the last dirty record for this dbuf has not yet synced 659c717a561Smaybee * and its referencing the dbuf data, either: 660c717a561Smaybee * reset the reference to point to a new copy, 661c717a561Smaybee * or (if there a no active holders) 662c717a561Smaybee * just null out the current db_data pointer. 663fa9e4066Sahrens */ 664c717a561Smaybee ASSERT(dr->dr_txg >= txg - 2); 665c717a561Smaybee if (db->db_blkid == DB_BONUS_BLKID) { 666c717a561Smaybee /* Note that the data bufs here are zio_bufs */ 667c717a561Smaybee dr->dt.dl.dr_data = zio_buf_alloc(DN_MAX_BONUSLEN); 6685a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 669c717a561Smaybee bcopy(db->db.db_data, dr->dt.dl.dr_data, DN_MAX_BONUSLEN); 670c717a561Smaybee } else if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 671ea8dc4b6Seschrock int size = db->db.db_size; 672c717a561Smaybee arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 673c717a561Smaybee dr->dt.dl.dr_data = arc_buf_alloc( 674ad23a2dbSjohansen db->db_dnode->dn_objset->os_spa, size, db, type); 675c717a561Smaybee bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size); 676fa9e4066Sahrens } else { 677ea8dc4b6Seschrock dbuf_set_data(db, NULL); 678fa9e4066Sahrens } 679fa9e4066Sahrens } 680ea8dc4b6Seschrock 681fa9e4066Sahrens void 682c717a561Smaybee dbuf_unoverride(dbuf_dirty_record_t *dr) 683fa9e4066Sahrens { 684c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 685c717a561Smaybee uint64_t txg = dr->dr_txg; 686c5c6ffa0Smaybee 687c717a561Smaybee ASSERT(MUTEX_HELD(&db->db_mtx)); 688c717a561Smaybee ASSERT(dr->dt.dl.dr_override_state != DR_IN_DMU_SYNC); 689c717a561Smaybee ASSERT(db->db_level == 0); 690c717a561Smaybee 691c717a561Smaybee if (db->db_blkid == DB_BONUS_BLKID || 692c717a561Smaybee dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN) 693c717a561Smaybee return; 694c717a561Smaybee 695fa9e4066Sahrens /* free this block */ 696c717a561Smaybee if (!BP_IS_HOLE(&dr->dt.dl.dr_overridden_by)) { 697fa9e4066Sahrens /* XXX can get silent EIO here */ 698088f3894Sahrens (void) dsl_free(NULL, 699088f3894Sahrens spa_get_dsl(db->db_dnode->dn_objset->os_spa), 700c717a561Smaybee txg, &dr->dt.dl.dr_overridden_by, NULL, NULL, ARC_WAIT); 701fa9e4066Sahrens } 702c717a561Smaybee dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 7036b4acc8bSahrens /* 7046b4acc8bSahrens * Release the already-written buffer, so we leave it in 7056b4acc8bSahrens * a consistent dirty state. Note that all callers are 7066b4acc8bSahrens * modifying the buffer, so they will immediately do 7076b4acc8bSahrens * another (redundant) arc_release(). Therefore, leave 7086b4acc8bSahrens * the buf thawed to save the effort of freezing & 7096b4acc8bSahrens * immediately re-thawing it. 7106b4acc8bSahrens */ 711c717a561Smaybee arc_release(dr->dt.dl.dr_data, db); 712fa9e4066Sahrens } 713fa9e4066Sahrens 714cdb0ab79Smaybee /* 715cdb0ab79Smaybee * Evict (if its unreferenced) or clear (if its referenced) any level-0 716cdb0ab79Smaybee * data blocks in the free range, so that any future readers will find 717cdb0ab79Smaybee * empty blocks. Also, if we happen accross any level-1 dbufs in the 718cdb0ab79Smaybee * range that have not already been marked dirty, mark them dirty so 719cdb0ab79Smaybee * they stay in memory. 720cdb0ab79Smaybee */ 721fa9e4066Sahrens void 722cdb0ab79Smaybee dbuf_free_range(dnode_t *dn, uint64_t start, uint64_t end, dmu_tx_t *tx) 723fa9e4066Sahrens { 724fa9e4066Sahrens dmu_buf_impl_t *db, *db_next; 725fa9e4066Sahrens uint64_t txg = tx->tx_txg; 726cdb0ab79Smaybee int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 727cdb0ab79Smaybee uint64_t first_l1 = start >> epbs; 728cdb0ab79Smaybee uint64_t last_l1 = end >> epbs; 729fa9e4066Sahrens 730cdb0ab79Smaybee if (end > dn->dn_maxblkid) { 731cdb0ab79Smaybee end = dn->dn_maxblkid; 732cdb0ab79Smaybee last_l1 = end >> epbs; 733cdb0ab79Smaybee } 734cdb0ab79Smaybee dprintf_dnode(dn, "start=%llu end=%llu\n", start, end); 735fa9e4066Sahrens mutex_enter(&dn->dn_dbufs_mtx); 736fa9e4066Sahrens for (db = list_head(&dn->dn_dbufs); db; db = db_next) { 737fa9e4066Sahrens db_next = list_next(&dn->dn_dbufs, db); 738ea8dc4b6Seschrock ASSERT(db->db_blkid != DB_BONUS_BLKID); 739cdb0ab79Smaybee 740cdb0ab79Smaybee if (db->db_level == 1 && 741cdb0ab79Smaybee db->db_blkid >= first_l1 && db->db_blkid <= last_l1) { 742cdb0ab79Smaybee mutex_enter(&db->db_mtx); 743cdb0ab79Smaybee if (db->db_last_dirty && 744cdb0ab79Smaybee db->db_last_dirty->dr_txg < txg) { 745cdb0ab79Smaybee dbuf_add_ref(db, FTAG); 746cdb0ab79Smaybee mutex_exit(&db->db_mtx); 747cdb0ab79Smaybee dbuf_will_dirty(db, tx); 748cdb0ab79Smaybee dbuf_rele(db, FTAG); 749cdb0ab79Smaybee } else { 750cdb0ab79Smaybee mutex_exit(&db->db_mtx); 751cdb0ab79Smaybee } 752cdb0ab79Smaybee } 753cdb0ab79Smaybee 754ea8dc4b6Seschrock if (db->db_level != 0) 755fa9e4066Sahrens continue; 756fa9e4066Sahrens dprintf_dbuf(db, "found buf %s\n", ""); 757cdb0ab79Smaybee if (db->db_blkid < start || db->db_blkid > end) 758fa9e4066Sahrens continue; 759fa9e4066Sahrens 760fa9e4066Sahrens /* found a level 0 buffer in the range */ 761fa9e4066Sahrens if (dbuf_undirty(db, tx)) 762fa9e4066Sahrens continue; 763fa9e4066Sahrens 764fa9e4066Sahrens mutex_enter(&db->db_mtx); 765ea8dc4b6Seschrock if (db->db_state == DB_UNCACHED || 76682c9918fSTim Haley db->db_state == DB_NOFILL || 767ea8dc4b6Seschrock db->db_state == DB_EVICTING) { 768fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 769fa9e4066Sahrens mutex_exit(&db->db_mtx); 770fa9e4066Sahrens continue; 771fa9e4066Sahrens } 772c543ec06Sahrens if (db->db_state == DB_READ || db->db_state == DB_FILL) { 773c543ec06Sahrens /* will be handled in dbuf_read_done or dbuf_rele */ 774c717a561Smaybee db->db_freed_in_flight = TRUE; 775fa9e4066Sahrens mutex_exit(&db->db_mtx); 776fa9e4066Sahrens continue; 777fa9e4066Sahrens } 778ea8dc4b6Seschrock if (refcount_count(&db->db_holds) == 0) { 779ea8dc4b6Seschrock ASSERT(db->db_buf); 780ea8dc4b6Seschrock dbuf_clear(db); 781ea8dc4b6Seschrock continue; 782ea8dc4b6Seschrock } 783c717a561Smaybee /* The dbuf is referenced */ 784fa9e4066Sahrens 785c717a561Smaybee if (db->db_last_dirty != NULL) { 786c717a561Smaybee dbuf_dirty_record_t *dr = db->db_last_dirty; 787c717a561Smaybee 788c717a561Smaybee if (dr->dr_txg == txg) { 789ea8dc4b6Seschrock /* 790c717a561Smaybee * This buffer is "in-use", re-adjust the file 791c717a561Smaybee * size to reflect that this buffer may 792c717a561Smaybee * contain new data when we sync. 793ea8dc4b6Seschrock */ 79444eda4d7Smaybee if (db->db_blkid > dn->dn_maxblkid) 79544eda4d7Smaybee dn->dn_maxblkid = db->db_blkid; 796c717a561Smaybee dbuf_unoverride(dr); 797c717a561Smaybee } else { 798c717a561Smaybee /* 799c717a561Smaybee * This dbuf is not dirty in the open context. 800c717a561Smaybee * Either uncache it (if its not referenced in 801c717a561Smaybee * the open context) or reset its contents to 802c717a561Smaybee * empty. 803c717a561Smaybee */ 804c717a561Smaybee dbuf_fix_old_data(db, txg); 80544eda4d7Smaybee } 806c717a561Smaybee } 807c717a561Smaybee /* clear the contents if its cached */ 808ea8dc4b6Seschrock if (db->db_state == DB_CACHED) { 809ea8dc4b6Seschrock ASSERT(db->db.db_data != NULL); 810fa9e4066Sahrens arc_release(db->db_buf, db); 811fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 8126b4acc8bSahrens arc_buf_freeze(db->db_buf); 813fa9e4066Sahrens } 814ea8dc4b6Seschrock 815fa9e4066Sahrens mutex_exit(&db->db_mtx); 816fa9e4066Sahrens } 817fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 818fa9e4066Sahrens } 819fa9e4066Sahrens 820fa9e4066Sahrens static int 8211934e92fSmaybee dbuf_block_freeable(dmu_buf_impl_t *db) 822fa9e4066Sahrens { 823fa9e4066Sahrens dsl_dataset_t *ds = db->db_objset->os_dsl_dataset; 824fa9e4066Sahrens uint64_t birth_txg = 0; 825fa9e4066Sahrens 826fa9e4066Sahrens /* 827fa9e4066Sahrens * We don't need any locking to protect db_blkptr: 828c717a561Smaybee * If it's syncing, then db_last_dirty will be set 829c717a561Smaybee * so we'll ignore db_blkptr. 830fa9e4066Sahrens */ 831c717a561Smaybee ASSERT(MUTEX_HELD(&db->db_mtx)); 832c717a561Smaybee if (db->db_last_dirty) 833c717a561Smaybee birth_txg = db->db_last_dirty->dr_txg; 834fa9e4066Sahrens else if (db->db_blkptr) 835fa9e4066Sahrens birth_txg = db->db_blkptr->blk_birth; 836fa9e4066Sahrens 8371934e92fSmaybee /* If we don't exist or are in a snapshot, we can't be freed */ 838fa9e4066Sahrens if (birth_txg) 8391934e92fSmaybee return (ds == NULL || 8401934e92fSmaybee dsl_dataset_block_freeable(ds, birth_txg)); 841fa9e4066Sahrens else 8421934e92fSmaybee return (FALSE); 843fa9e4066Sahrens } 844fa9e4066Sahrens 845fa9e4066Sahrens void 846fa9e4066Sahrens dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx) 847fa9e4066Sahrens { 848fa9e4066Sahrens arc_buf_t *buf, *obuf; 849fa9e4066Sahrens int osize = db->db.db_size; 850ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 851fa9e4066Sahrens 852ea8dc4b6Seschrock ASSERT(db->db_blkid != DB_BONUS_BLKID); 853ea8dc4b6Seschrock 854fa9e4066Sahrens /* XXX does *this* func really need the lock? */ 855fa9e4066Sahrens ASSERT(RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)); 856fa9e4066Sahrens 857fa9e4066Sahrens /* 858fa9e4066Sahrens * This call to dbuf_will_dirty() with the dn_struct_rwlock held 859fa9e4066Sahrens * is OK, because there can be no other references to the db 860fa9e4066Sahrens * when we are changing its size, so no concurrent DB_FILL can 861fa9e4066Sahrens * be happening. 862fa9e4066Sahrens */ 863ea8dc4b6Seschrock /* 864ea8dc4b6Seschrock * XXX we should be doing a dbuf_read, checking the return 865ea8dc4b6Seschrock * value and returning that up to our callers 866ea8dc4b6Seschrock */ 867fa9e4066Sahrens dbuf_will_dirty(db, tx); 868fa9e4066Sahrens 869fa9e4066Sahrens /* create the data buffer for the new block */ 870ad23a2dbSjohansen buf = arc_buf_alloc(db->db_dnode->dn_objset->os_spa, size, db, type); 871fa9e4066Sahrens 872fa9e4066Sahrens /* copy old block data to the new block */ 873fa9e4066Sahrens obuf = db->db_buf; 874f65e61c0Sahrens bcopy(obuf->b_data, buf->b_data, MIN(osize, size)); 875fa9e4066Sahrens /* zero the remainder */ 876f65e61c0Sahrens if (size > osize) 877fa9e4066Sahrens bzero((uint8_t *)buf->b_data + osize, size - osize); 878fa9e4066Sahrens 879fa9e4066Sahrens mutex_enter(&db->db_mtx); 880fa9e4066Sahrens dbuf_set_data(db, buf); 881ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(obuf, db) == 1); 882fa9e4066Sahrens db->db.db_size = size; 883fa9e4066Sahrens 884c717a561Smaybee if (db->db_level == 0) { 885c717a561Smaybee ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg); 886c717a561Smaybee db->db_last_dirty->dt.dl.dr_data = buf; 887c717a561Smaybee } 888fa9e4066Sahrens mutex_exit(&db->db_mtx); 889fa9e4066Sahrens 890fa9e4066Sahrens dnode_willuse_space(db->db_dnode, size-osize, tx); 891fa9e4066Sahrens } 892fa9e4066Sahrens 893c717a561Smaybee dbuf_dirty_record_t * 894fa9e4066Sahrens dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 895fa9e4066Sahrens { 896fa9e4066Sahrens dnode_t *dn = db->db_dnode; 897fa9e4066Sahrens objset_impl_t *os = dn->dn_objset; 898c717a561Smaybee dbuf_dirty_record_t **drp, *dr; 899fa9e4066Sahrens int drop_struct_lock = FALSE; 900d3469faaSMark Maybee boolean_t do_free_accounting = B_FALSE; 901fa9e4066Sahrens int txgoff = tx->tx_txg & TXG_MASK; 902fa9e4066Sahrens 903fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 904fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 9059c9dc39aSek110237 DMU_TX_DIRTY_BUF(tx, db); 906fa9e4066Sahrens 907fa9e4066Sahrens /* 908fa9e4066Sahrens * Shouldn't dirty a regular buffer in syncing context. Private 909fa9e4066Sahrens * objects may be dirtied in syncing context, but only if they 910fa9e4066Sahrens * were already pre-dirtied in open context. 911fa9e4066Sahrens */ 912c717a561Smaybee ASSERT(!dmu_tx_is_syncing(tx) || 913c717a561Smaybee BP_IS_HOLE(dn->dn_objset->os_rootbp) || 91414843421SMatthew Ahrens DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 91514843421SMatthew Ahrens dn->dn_objset->os_dsl_dataset == NULL); 916fa9e4066Sahrens /* 917fa9e4066Sahrens * We make this assert for private objects as well, but after we 918fa9e4066Sahrens * check if we're already dirty. They are allowed to re-dirty 919fa9e4066Sahrens * in syncing context. 920fa9e4066Sahrens */ 921ea8dc4b6Seschrock ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 922c717a561Smaybee dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 923fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 924fa9e4066Sahrens 925fa9e4066Sahrens mutex_enter(&db->db_mtx); 926fa9e4066Sahrens /* 927c717a561Smaybee * XXX make this true for indirects too? The problem is that 928c717a561Smaybee * transactions created with dmu_tx_create_assigned() from 929c717a561Smaybee * syncing context don't bother holding ahead. 930fa9e4066Sahrens */ 931c717a561Smaybee ASSERT(db->db_level != 0 || 93282c9918fSTim Haley db->db_state == DB_CACHED || db->db_state == DB_FILL || 93382c9918fSTim Haley db->db_state == DB_NOFILL); 934fa9e4066Sahrens 935fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 936fa9e4066Sahrens /* 937fa9e4066Sahrens * Don't set dirtyctx to SYNC if we're just modifying this as we 938fa9e4066Sahrens * initialize the objset. 939fa9e4066Sahrens */ 940fa9e4066Sahrens if (dn->dn_dirtyctx == DN_UNDIRTIED && 941c717a561Smaybee !BP_IS_HOLE(dn->dn_objset->os_rootbp)) { 942fa9e4066Sahrens dn->dn_dirtyctx = 943fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN); 944fa9e4066Sahrens ASSERT(dn->dn_dirtyctx_firstset == NULL); 945fa9e4066Sahrens dn->dn_dirtyctx_firstset = kmem_alloc(1, KM_SLEEP); 946fa9e4066Sahrens } 947fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 948fa9e4066Sahrens 949fa9e4066Sahrens /* 950fa9e4066Sahrens * If this buffer is already dirty, we're done. 951fa9e4066Sahrens */ 952c717a561Smaybee drp = &db->db_last_dirty; 953c717a561Smaybee ASSERT(*drp == NULL || (*drp)->dr_txg <= tx->tx_txg || 954c717a561Smaybee db->db.db_object == DMU_META_DNODE_OBJECT); 9557e2186e3Sbonwick while ((dr = *drp) != NULL && dr->dr_txg > tx->tx_txg) 9567e2186e3Sbonwick drp = &dr->dr_next; 9577e2186e3Sbonwick if (dr && dr->dr_txg == tx->tx_txg) { 958c717a561Smaybee if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID) { 959c717a561Smaybee /* 960c717a561Smaybee * If this buffer has already been written out, 961c717a561Smaybee * we now need to reset its state. 962c717a561Smaybee */ 9637e2186e3Sbonwick dbuf_unoverride(dr); 964c717a561Smaybee if (db->db.db_object != DMU_META_DNODE_OBJECT) 9656b4acc8bSahrens arc_buf_thaw(db->db_buf); 966c717a561Smaybee } 967fa9e4066Sahrens mutex_exit(&db->db_mtx); 9687e2186e3Sbonwick return (dr); 969fa9e4066Sahrens } 970fa9e4066Sahrens 971fa9e4066Sahrens /* 972fa9e4066Sahrens * Only valid if not already dirty. 973fa9e4066Sahrens */ 97414843421SMatthew Ahrens ASSERT(dn->dn_object == 0 || 97514843421SMatthew Ahrens dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 976fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 977fa9e4066Sahrens 978fa9e4066Sahrens ASSERT3U(dn->dn_nlevels, >, db->db_level); 979fa9e4066Sahrens ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) || 980fa9e4066Sahrens dn->dn_phys->dn_nlevels > db->db_level || 981fa9e4066Sahrens dn->dn_next_nlevels[txgoff] > db->db_level || 982fa9e4066Sahrens dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level || 983fa9e4066Sahrens dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level); 984fa9e4066Sahrens 985fa9e4066Sahrens /* 986fa9e4066Sahrens * We should only be dirtying in syncing context if it's the 98714843421SMatthew Ahrens * mos or we're initializing the os or it's a special object. 98814843421SMatthew Ahrens * However, we are allowed to dirty in syncing context provided 98914843421SMatthew Ahrens * we already dirtied it in open context. Hence we must make 99014843421SMatthew Ahrens * this assertion only if we're not already dirty. 991fa9e4066Sahrens */ 99214843421SMatthew Ahrens ASSERT(!dmu_tx_is_syncing(tx) || DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 99314843421SMatthew Ahrens os->os_dsl_dataset == NULL || BP_IS_HOLE(os->os_rootbp)); 994fa9e4066Sahrens ASSERT(db->db.db_size != 0); 995fa9e4066Sahrens 996fa9e4066Sahrens dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 997fa9e4066Sahrens 9981934e92fSmaybee if (db->db_blkid != DB_BONUS_BLKID) { 9991934e92fSmaybee /* 10001934e92fSmaybee * Update the accounting. 1001d3469faaSMark Maybee * Note: we delay "free accounting" until after we drop 1002d3469faaSMark Maybee * the db_mtx. This keeps us from grabbing other locks 1003d3469faaSMark Maybee * (and possibly deadlocking) in bp_get_dasize() while 1004d3469faaSMark Maybee * also holding the db_mtx. 10051934e92fSmaybee */ 10061934e92fSmaybee dnode_willuse_space(dn, db->db.db_size, tx); 1007d3469faaSMark Maybee do_free_accounting = dbuf_block_freeable(db); 10081934e92fSmaybee } 10091934e92fSmaybee 1010ea8dc4b6Seschrock /* 1011ea8dc4b6Seschrock * If this buffer is dirty in an old transaction group we need 1012ea8dc4b6Seschrock * to make a copy of it so that the changes we make in this 1013ea8dc4b6Seschrock * transaction group won't leak out when we sync the older txg. 1014ea8dc4b6Seschrock */ 1015c717a561Smaybee dr = kmem_zalloc(sizeof (dbuf_dirty_record_t), KM_SLEEP); 1016c717a561Smaybee if (db->db_level == 0) { 1017c717a561Smaybee void *data_old = db->db_buf; 1018c717a561Smaybee 101982c9918fSTim Haley if (db->db_state != DB_NOFILL) { 1020ea8dc4b6Seschrock if (db->db_blkid == DB_BONUS_BLKID) { 1021c717a561Smaybee dbuf_fix_old_data(db, tx->tx_txg); 1022c717a561Smaybee data_old = db->db.db_data; 1023c717a561Smaybee } else if (db->db.db_object != DMU_META_DNODE_OBJECT) { 1024fa9e4066Sahrens /* 102582c9918fSTim Haley * Release the data buffer from the cache so 102682c9918fSTim Haley * that we can modify it without impacting 102782c9918fSTim Haley * possible other users of this cached data 102882c9918fSTim Haley * block. Note that indirect blocks and 102982c9918fSTim Haley * private objects are not released until the 103082c9918fSTim Haley * syncing state (since they are only modified 103182c9918fSTim Haley * then). 1032fa9e4066Sahrens */ 1033fa9e4066Sahrens arc_release(db->db_buf, db); 1034fa9e4066Sahrens dbuf_fix_old_data(db, tx->tx_txg); 1035c717a561Smaybee data_old = db->db_buf; 1036fa9e4066Sahrens } 1037c717a561Smaybee ASSERT(data_old != NULL); 103882c9918fSTim Haley } 1039c717a561Smaybee dr->dt.dl.dr_data = data_old; 1040c717a561Smaybee } else { 1041c717a561Smaybee mutex_init(&dr->dt.di.dr_mtx, NULL, MUTEX_DEFAULT, NULL); 1042c717a561Smaybee list_create(&dr->dt.di.dr_children, 1043c717a561Smaybee sizeof (dbuf_dirty_record_t), 1044c717a561Smaybee offsetof(dbuf_dirty_record_t, dr_dirty_node)); 1045fa9e4066Sahrens } 1046c717a561Smaybee dr->dr_dbuf = db; 1047c717a561Smaybee dr->dr_txg = tx->tx_txg; 1048c717a561Smaybee dr->dr_next = *drp; 1049c717a561Smaybee *drp = dr; 1050fa9e4066Sahrens 1051fa9e4066Sahrens /* 1052fa9e4066Sahrens * We could have been freed_in_flight between the dbuf_noread 1053fa9e4066Sahrens * and dbuf_dirty. We win, as though the dbuf_noread() had 1054fa9e4066Sahrens * happened after the free. 1055fa9e4066Sahrens */ 1056fa9e4066Sahrens if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID) { 1057c717a561Smaybee mutex_enter(&dn->dn_mtx); 1058fa9e4066Sahrens dnode_clear_range(dn, db->db_blkid, 1, tx); 1059fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1060c717a561Smaybee db->db_freed_in_flight = FALSE; 1061c717a561Smaybee } 1062fa9e4066Sahrens 1063fa9e4066Sahrens /* 1064fa9e4066Sahrens * This buffer is now part of this txg 1065fa9e4066Sahrens */ 1066fa9e4066Sahrens dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg); 1067fa9e4066Sahrens db->db_dirtycnt += 1; 1068fa9e4066Sahrens ASSERT3U(db->db_dirtycnt, <=, 3); 1069fa9e4066Sahrens 1070fa9e4066Sahrens mutex_exit(&db->db_mtx); 1071fa9e4066Sahrens 1072fa9e4066Sahrens if (db->db_blkid == DB_BONUS_BLKID) { 1073c717a561Smaybee mutex_enter(&dn->dn_mtx); 1074c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 1075c717a561Smaybee list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1076c717a561Smaybee mutex_exit(&dn->dn_mtx); 1077fa9e4066Sahrens dnode_setdirty(dn, tx); 1078c717a561Smaybee return (dr); 1079d3469faaSMark Maybee } else if (do_free_accounting) { 1080d3469faaSMark Maybee blkptr_t *bp = db->db_blkptr; 1081d3469faaSMark Maybee int64_t willfree = (bp && !BP_IS_HOLE(bp)) ? 1082d3469faaSMark Maybee bp_get_dasize(os->os_spa, bp) : db->db.db_size; 1083d3469faaSMark Maybee /* 1084d3469faaSMark Maybee * This is only a guess -- if the dbuf is dirty 1085d3469faaSMark Maybee * in a previous txg, we don't know how much 1086d3469faaSMark Maybee * space it will use on disk yet. We should 1087d3469faaSMark Maybee * really have the struct_rwlock to access 1088d3469faaSMark Maybee * db_blkptr, but since this is just a guess, 1089d3469faaSMark Maybee * it's OK if we get an odd answer. 1090d3469faaSMark Maybee */ 1091d3469faaSMark Maybee dnode_willuse_space(dn, -willfree, tx); 1092fa9e4066Sahrens } 1093fa9e4066Sahrens 1094fa9e4066Sahrens if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 1095fa9e4066Sahrens rw_enter(&dn->dn_struct_rwlock, RW_READER); 1096fa9e4066Sahrens drop_struct_lock = TRUE; 1097fa9e4066Sahrens } 1098fa9e4066Sahrens 10998346f03fSJonathan W Adams if (db->db_level == 0) { 11008346f03fSJonathan W Adams dnode_new_blkid(dn, db->db_blkid, tx, drop_struct_lock); 11018346f03fSJonathan W Adams ASSERT(dn->dn_maxblkid >= db->db_blkid); 11028346f03fSJonathan W Adams } 11038346f03fSJonathan W Adams 110444eda4d7Smaybee if (db->db_level+1 < dn->dn_nlevels) { 1105c717a561Smaybee dmu_buf_impl_t *parent = db->db_parent; 1106c717a561Smaybee dbuf_dirty_record_t *di; 1107c717a561Smaybee int parent_held = FALSE; 1108c717a561Smaybee 1109c717a561Smaybee if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) { 1110fa9e4066Sahrens int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1111c717a561Smaybee 1112fa9e4066Sahrens parent = dbuf_hold_level(dn, db->db_level+1, 1113fa9e4066Sahrens db->db_blkid >> epbs, FTAG); 1114c717a561Smaybee parent_held = TRUE; 1115c717a561Smaybee } 1116fa9e4066Sahrens if (drop_struct_lock) 1117fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 1118c717a561Smaybee ASSERT3U(db->db_level+1, ==, parent->db_level); 1119c717a561Smaybee di = dbuf_dirty(parent, tx); 1120c717a561Smaybee if (parent_held) 1121ea8dc4b6Seschrock dbuf_rele(parent, FTAG); 1122c717a561Smaybee 1123c717a561Smaybee mutex_enter(&db->db_mtx); 1124c717a561Smaybee /* possible race with dbuf_undirty() */ 1125c717a561Smaybee if (db->db_last_dirty == dr || 1126c717a561Smaybee dn->dn_object == DMU_META_DNODE_OBJECT) { 1127c717a561Smaybee mutex_enter(&di->dt.di.dr_mtx); 1128c717a561Smaybee ASSERT3U(di->dr_txg, ==, tx->tx_txg); 1129c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 1130c717a561Smaybee list_insert_tail(&di->dt.di.dr_children, dr); 1131c717a561Smaybee mutex_exit(&di->dt.di.dr_mtx); 1132c717a561Smaybee dr->dr_parent = di; 1133c717a561Smaybee } 1134c717a561Smaybee mutex_exit(&db->db_mtx); 1135fa9e4066Sahrens } else { 1136c717a561Smaybee ASSERT(db->db_level+1 == dn->dn_nlevels); 1137c717a561Smaybee ASSERT(db->db_blkid < dn->dn_nblkptr); 1138c717a561Smaybee ASSERT(db->db_parent == NULL || 1139c717a561Smaybee db->db_parent == db->db_dnode->dn_dbuf); 1140c717a561Smaybee mutex_enter(&dn->dn_mtx); 1141c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 1142c717a561Smaybee list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1143c717a561Smaybee mutex_exit(&dn->dn_mtx); 1144fa9e4066Sahrens if (drop_struct_lock) 1145fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 1146fa9e4066Sahrens } 1147fa9e4066Sahrens 1148fa9e4066Sahrens dnode_setdirty(dn, tx); 1149c717a561Smaybee return (dr); 1150fa9e4066Sahrens } 1151fa9e4066Sahrens 1152fa9e4066Sahrens static int 1153fa9e4066Sahrens dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1154fa9e4066Sahrens { 1155fa9e4066Sahrens dnode_t *dn = db->db_dnode; 1156c717a561Smaybee uint64_t txg = tx->tx_txg; 115717f17c2dSbonwick dbuf_dirty_record_t *dr, **drp; 1158fa9e4066Sahrens 1159c717a561Smaybee ASSERT(txg != 0); 1160ea8dc4b6Seschrock ASSERT(db->db_blkid != DB_BONUS_BLKID); 1161fa9e4066Sahrens 1162fa9e4066Sahrens mutex_enter(&db->db_mtx); 1163fa9e4066Sahrens 1164fa9e4066Sahrens /* 1165fa9e4066Sahrens * If this buffer is not dirty, we're done. 1166fa9e4066Sahrens */ 116717f17c2dSbonwick for (drp = &db->db_last_dirty; (dr = *drp) != NULL; drp = &dr->dr_next) 1168c717a561Smaybee if (dr->dr_txg <= txg) 1169c717a561Smaybee break; 1170c717a561Smaybee if (dr == NULL || dr->dr_txg < txg) { 1171fa9e4066Sahrens mutex_exit(&db->db_mtx); 1172fa9e4066Sahrens return (0); 1173fa9e4066Sahrens } 1174c717a561Smaybee ASSERT(dr->dr_txg == txg); 1175fa9e4066Sahrens 1176fa9e4066Sahrens /* 1177fa9e4066Sahrens * If this buffer is currently held, we cannot undirty 1178fa9e4066Sahrens * it, since one of the current holders may be in the 1179fa9e4066Sahrens * middle of an update. Note that users of dbuf_undirty() 1180fa9e4066Sahrens * should not place a hold on the dbuf before the call. 1181fa9e4066Sahrens */ 1182fa9e4066Sahrens if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 1183fa9e4066Sahrens mutex_exit(&db->db_mtx); 118444eda4d7Smaybee /* Make sure we don't toss this buffer at sync phase */ 1185fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 1186fa9e4066Sahrens dnode_clear_range(dn, db->db_blkid, 1, tx); 1187fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1188fa9e4066Sahrens return (0); 1189fa9e4066Sahrens } 1190fa9e4066Sahrens 1191fa9e4066Sahrens dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1192fa9e4066Sahrens 1193fa9e4066Sahrens ASSERT(db->db.db_size != 0); 1194fa9e4066Sahrens 1195fa9e4066Sahrens /* XXX would be nice to fix up dn_towrite_space[] */ 1196fa9e4066Sahrens 119717f17c2dSbonwick *drp = dr->dr_next; 1198c717a561Smaybee 1199c717a561Smaybee if (dr->dr_parent) { 1200c717a561Smaybee mutex_enter(&dr->dr_parent->dt.di.dr_mtx); 1201c717a561Smaybee list_remove(&dr->dr_parent->dt.di.dr_children, dr); 1202c717a561Smaybee mutex_exit(&dr->dr_parent->dt.di.dr_mtx); 1203c717a561Smaybee } else if (db->db_level+1 == dn->dn_nlevels) { 1204cdb0ab79Smaybee ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf); 1205fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 1206c717a561Smaybee list_remove(&dn->dn_dirty_records[txg & TXG_MASK], dr); 1207fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1208c717a561Smaybee } 1209c717a561Smaybee 1210c717a561Smaybee if (db->db_level == 0) { 121182c9918fSTim Haley if (db->db_state != DB_NOFILL) { 1212c717a561Smaybee dbuf_unoverride(dr); 1213c717a561Smaybee 1214c717a561Smaybee ASSERT(db->db_buf != NULL); 1215c717a561Smaybee ASSERT(dr->dt.dl.dr_data != NULL); 1216c717a561Smaybee if (dr->dt.dl.dr_data != db->db_buf) 121782c9918fSTim Haley VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, 121882c9918fSTim Haley db) == 1); 121982c9918fSTim Haley } 1220c717a561Smaybee } else { 1221c717a561Smaybee ASSERT(db->db_buf != NULL); 1222c717a561Smaybee ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 1223c25056deSgw25295 mutex_destroy(&dr->dt.di.dr_mtx); 1224c25056deSgw25295 list_destroy(&dr->dt.di.dr_children); 1225c717a561Smaybee } 1226c717a561Smaybee kmem_free(dr, sizeof (dbuf_dirty_record_t)); 1227fa9e4066Sahrens 1228fa9e4066Sahrens ASSERT(db->db_dirtycnt > 0); 1229fa9e4066Sahrens db->db_dirtycnt -= 1; 1230fa9e4066Sahrens 1231c717a561Smaybee if (refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) { 1232ea8dc4b6Seschrock arc_buf_t *buf = db->db_buf; 1233fa9e4066Sahrens 1234ea8dc4b6Seschrock ASSERT(arc_released(buf)); 1235ea8dc4b6Seschrock dbuf_set_data(db, NULL); 1236ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(buf, db) == 1); 1237fa9e4066Sahrens dbuf_evict(db); 1238fa9e4066Sahrens return (1); 1239fa9e4066Sahrens } 1240fa9e4066Sahrens 1241fa9e4066Sahrens mutex_exit(&db->db_mtx); 1242fa9e4066Sahrens return (0); 1243fa9e4066Sahrens } 1244fa9e4066Sahrens 1245fa9e4066Sahrens #pragma weak dmu_buf_will_dirty = dbuf_will_dirty 1246fa9e4066Sahrens void 1247fa9e4066Sahrens dbuf_will_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1248fa9e4066Sahrens { 12491ab7f2deSmaybee int rf = DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH; 1250fa9e4066Sahrens 1251fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 1252fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 1253fa9e4066Sahrens 1254fa9e4066Sahrens if (RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)) 1255fa9e4066Sahrens rf |= DB_RF_HAVESTRUCT; 1256ea8dc4b6Seschrock (void) dbuf_read(db, NULL, rf); 1257c717a561Smaybee (void) dbuf_dirty(db, tx); 1258fa9e4066Sahrens } 1259fa9e4066Sahrens 1260fa9e4066Sahrens void 126182c9918fSTim Haley dmu_buf_will_not_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 126282c9918fSTim Haley { 126382c9918fSTim Haley dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 126482c9918fSTim Haley 126582c9918fSTim Haley db->db_state = DB_NOFILL; 126682c9918fSTim Haley 126782c9918fSTim Haley dmu_buf_will_fill(db_fake, tx); 126882c9918fSTim Haley } 126982c9918fSTim Haley 127082c9918fSTim Haley void 1271ea8dc4b6Seschrock dmu_buf_will_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 1272fa9e4066Sahrens { 1273ea8dc4b6Seschrock dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1274ea8dc4b6Seschrock 1275ea8dc4b6Seschrock ASSERT(db->db_blkid != DB_BONUS_BLKID); 1276fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 1277fa9e4066Sahrens ASSERT(db->db_level == 0); 1278fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 1279fa9e4066Sahrens 1280ea8dc4b6Seschrock ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT || 1281fa9e4066Sahrens dmu_tx_private_ok(tx)); 1282fa9e4066Sahrens 1283fa9e4066Sahrens dbuf_noread(db); 1284c717a561Smaybee (void) dbuf_dirty(db, tx); 1285fa9e4066Sahrens } 1286fa9e4066Sahrens 1287fa9e4066Sahrens #pragma weak dmu_buf_fill_done = dbuf_fill_done 1288fa9e4066Sahrens /* ARGSUSED */ 1289fa9e4066Sahrens void 1290fa9e4066Sahrens dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx) 1291fa9e4066Sahrens { 1292fa9e4066Sahrens mutex_enter(&db->db_mtx); 12939c9dc39aSek110237 DBUF_VERIFY(db); 1294fa9e4066Sahrens 1295fa9e4066Sahrens if (db->db_state == DB_FILL) { 1296c717a561Smaybee if (db->db_level == 0 && db->db_freed_in_flight) { 1297ea8dc4b6Seschrock ASSERT(db->db_blkid != DB_BONUS_BLKID); 1298fa9e4066Sahrens /* we were freed while filling */ 1299fa9e4066Sahrens /* XXX dbuf_undirty? */ 1300fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 1301c717a561Smaybee db->db_freed_in_flight = FALSE; 1302fa9e4066Sahrens } 1303fa9e4066Sahrens db->db_state = DB_CACHED; 1304fa9e4066Sahrens cv_broadcast(&db->db_changed); 1305fa9e4066Sahrens } 1306fa9e4066Sahrens mutex_exit(&db->db_mtx); 1307fa9e4066Sahrens } 1308fa9e4066Sahrens 1309ea8dc4b6Seschrock /* 1310*2fdbea25SAleksandr Guzovskiy * Directly assign a provided arc buf to a given dbuf if it's not referenced 1311*2fdbea25SAleksandr Guzovskiy * by anybody except our caller. Otherwise copy arcbuf's contents to dbuf. 1312*2fdbea25SAleksandr Guzovskiy */ 1313*2fdbea25SAleksandr Guzovskiy void 1314*2fdbea25SAleksandr Guzovskiy dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx) 1315*2fdbea25SAleksandr Guzovskiy { 1316*2fdbea25SAleksandr Guzovskiy ASSERT(!refcount_is_zero(&db->db_holds)); 1317*2fdbea25SAleksandr Guzovskiy ASSERT(db->db_dnode->dn_object != DMU_META_DNODE_OBJECT); 1318*2fdbea25SAleksandr Guzovskiy ASSERT(db->db_blkid != DB_BONUS_BLKID); 1319*2fdbea25SAleksandr Guzovskiy ASSERT(db->db_level == 0); 1320*2fdbea25SAleksandr Guzovskiy ASSERT(DBUF_GET_BUFC_TYPE(db) == ARC_BUFC_DATA); 1321*2fdbea25SAleksandr Guzovskiy ASSERT(buf != NULL); 1322*2fdbea25SAleksandr Guzovskiy ASSERT(arc_buf_size(buf) == db->db.db_size); 1323*2fdbea25SAleksandr Guzovskiy ASSERT(tx->tx_txg != 0); 1324*2fdbea25SAleksandr Guzovskiy 1325*2fdbea25SAleksandr Guzovskiy arc_return_buf(buf, db); 1326*2fdbea25SAleksandr Guzovskiy ASSERT(arc_released(buf)); 1327*2fdbea25SAleksandr Guzovskiy 1328*2fdbea25SAleksandr Guzovskiy mutex_enter(&db->db_mtx); 1329*2fdbea25SAleksandr Guzovskiy 1330*2fdbea25SAleksandr Guzovskiy while (db->db_state == DB_READ || db->db_state == DB_FILL) 1331*2fdbea25SAleksandr Guzovskiy cv_wait(&db->db_changed, &db->db_mtx); 1332*2fdbea25SAleksandr Guzovskiy 1333*2fdbea25SAleksandr Guzovskiy ASSERT(db->db_state == DB_CACHED || db->db_state == DB_UNCACHED); 1334*2fdbea25SAleksandr Guzovskiy 1335*2fdbea25SAleksandr Guzovskiy if (db->db_state == DB_CACHED && 1336*2fdbea25SAleksandr Guzovskiy refcount_count(&db->db_holds) - 1 > db->db_dirtycnt) { 1337*2fdbea25SAleksandr Guzovskiy mutex_exit(&db->db_mtx); 1338*2fdbea25SAleksandr Guzovskiy (void) dbuf_dirty(db, tx); 1339*2fdbea25SAleksandr Guzovskiy bcopy(buf->b_data, db->db.db_data, db->db.db_size); 1340*2fdbea25SAleksandr Guzovskiy VERIFY(arc_buf_remove_ref(buf, db) == 1); 1341*2fdbea25SAleksandr Guzovskiy return; 1342*2fdbea25SAleksandr Guzovskiy } 1343*2fdbea25SAleksandr Guzovskiy 1344*2fdbea25SAleksandr Guzovskiy if (db->db_state == DB_CACHED) { 1345*2fdbea25SAleksandr Guzovskiy dbuf_dirty_record_t *dr = db->db_last_dirty; 1346*2fdbea25SAleksandr Guzovskiy 1347*2fdbea25SAleksandr Guzovskiy ASSERT(db->db_buf != NULL); 1348*2fdbea25SAleksandr Guzovskiy if (dr != NULL && dr->dr_txg == tx->tx_txg) { 1349*2fdbea25SAleksandr Guzovskiy ASSERT(dr->dt.dl.dr_data == db->db_buf); 1350*2fdbea25SAleksandr Guzovskiy if (!arc_released(db->db_buf)) { 1351*2fdbea25SAleksandr Guzovskiy ASSERT(dr->dt.dl.dr_override_state == 1352*2fdbea25SAleksandr Guzovskiy DR_OVERRIDDEN); 1353*2fdbea25SAleksandr Guzovskiy arc_release(db->db_buf, db); 1354*2fdbea25SAleksandr Guzovskiy } 1355*2fdbea25SAleksandr Guzovskiy dr->dt.dl.dr_data = buf; 1356*2fdbea25SAleksandr Guzovskiy VERIFY(arc_buf_remove_ref(db->db_buf, db) == 1); 1357*2fdbea25SAleksandr Guzovskiy } else if (dr == NULL || dr->dt.dl.dr_data != db->db_buf) { 1358*2fdbea25SAleksandr Guzovskiy arc_release(db->db_buf, db); 1359*2fdbea25SAleksandr Guzovskiy VERIFY(arc_buf_remove_ref(db->db_buf, db) == 1); 1360*2fdbea25SAleksandr Guzovskiy } 1361*2fdbea25SAleksandr Guzovskiy db->db_buf = NULL; 1362*2fdbea25SAleksandr Guzovskiy } 1363*2fdbea25SAleksandr Guzovskiy ASSERT(db->db_buf == NULL); 1364*2fdbea25SAleksandr Guzovskiy dbuf_set_data(db, buf); 1365*2fdbea25SAleksandr Guzovskiy db->db_state = DB_FILL; 1366*2fdbea25SAleksandr Guzovskiy mutex_exit(&db->db_mtx); 1367*2fdbea25SAleksandr Guzovskiy (void) dbuf_dirty(db, tx); 1368*2fdbea25SAleksandr Guzovskiy dbuf_fill_done(db, tx); 1369*2fdbea25SAleksandr Guzovskiy } 1370*2fdbea25SAleksandr Guzovskiy 1371*2fdbea25SAleksandr Guzovskiy /* 1372ea8dc4b6Seschrock * "Clear" the contents of this dbuf. This will mark the dbuf 1373ea8dc4b6Seschrock * EVICTING and clear *most* of its references. Unfortunetely, 1374ea8dc4b6Seschrock * when we are not holding the dn_dbufs_mtx, we can't clear the 1375ea8dc4b6Seschrock * entry in the dn_dbufs list. We have to wait until dbuf_destroy() 1376ea8dc4b6Seschrock * in this case. For callers from the DMU we will usually see: 1377ea8dc4b6Seschrock * dbuf_clear()->arc_buf_evict()->dbuf_do_evict()->dbuf_destroy() 1378ea8dc4b6Seschrock * For the arc callback, we will usually see: 1379ea8dc4b6Seschrock * dbuf_do_evict()->dbuf_clear();dbuf_destroy() 1380ea8dc4b6Seschrock * Sometimes, though, we will get a mix of these two: 1381ea8dc4b6Seschrock * DMU: dbuf_clear()->arc_buf_evict() 1382ea8dc4b6Seschrock * ARC: dbuf_do_evict()->dbuf_destroy() 1383ea8dc4b6Seschrock */ 1384ea8dc4b6Seschrock void 1385fa9e4066Sahrens dbuf_clear(dmu_buf_impl_t *db) 1386fa9e4066Sahrens { 1387fa9e4066Sahrens dnode_t *dn = db->db_dnode; 1388ea8dc4b6Seschrock dmu_buf_impl_t *parent = db->db_parent; 1389c543ec06Sahrens dmu_buf_impl_t *dndb = dn->dn_dbuf; 1390ea8dc4b6Seschrock int dbuf_gone = FALSE; 1391fa9e4066Sahrens 1392fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 1393fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 1394fa9e4066Sahrens 1395ea8dc4b6Seschrock dbuf_evict_user(db); 1396ea8dc4b6Seschrock 1397fa9e4066Sahrens if (db->db_state == DB_CACHED) { 1398ea8dc4b6Seschrock ASSERT(db->db.db_data != NULL); 13990e8c6158Smaybee if (db->db_blkid == DB_BONUS_BLKID) { 1400ea8dc4b6Seschrock zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN); 14015a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 14020e8c6158Smaybee } 1403fa9e4066Sahrens db->db.db_data = NULL; 1404fa9e4066Sahrens db->db_state = DB_UNCACHED; 1405fa9e4066Sahrens } 1406fa9e4066Sahrens 140782c9918fSTim Haley ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL); 1408fa9e4066Sahrens ASSERT(db->db_data_pending == NULL); 1409fa9e4066Sahrens 1410ea8dc4b6Seschrock db->db_state = DB_EVICTING; 1411ea8dc4b6Seschrock db->db_blkptr = NULL; 1412ea8dc4b6Seschrock 1413ea8dc4b6Seschrock if (db->db_blkid != DB_BONUS_BLKID && MUTEX_HELD(&dn->dn_dbufs_mtx)) { 1414ea8dc4b6Seschrock list_remove(&dn->dn_dbufs, db); 1415ea8dc4b6Seschrock dnode_rele(dn, db); 14161934e92fSmaybee db->db_dnode = NULL; 1417ea8dc4b6Seschrock } 1418ea8dc4b6Seschrock 1419ea8dc4b6Seschrock if (db->db_buf) 1420ea8dc4b6Seschrock dbuf_gone = arc_buf_evict(db->db_buf); 1421ea8dc4b6Seschrock 1422ea8dc4b6Seschrock if (!dbuf_gone) 1423fa9e4066Sahrens mutex_exit(&db->db_mtx); 1424fa9e4066Sahrens 1425fa9e4066Sahrens /* 1426fa9e4066Sahrens * If this dbuf is referened from an indirect dbuf, 1427fa9e4066Sahrens * decrement the ref count on the indirect dbuf. 1428fa9e4066Sahrens */ 1429c543ec06Sahrens if (parent && parent != dndb) 1430ea8dc4b6Seschrock dbuf_rele(parent, db); 1431fa9e4066Sahrens } 1432fa9e4066Sahrens 1433fa9e4066Sahrens static int 1434fa9e4066Sahrens dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse, 1435fa9e4066Sahrens dmu_buf_impl_t **parentp, blkptr_t **bpp) 1436fa9e4066Sahrens { 1437fa9e4066Sahrens int nlevels, epbs; 1438fa9e4066Sahrens 14390b69c2f0Sahrens *parentp = NULL; 14400b69c2f0Sahrens *bpp = NULL; 14410b69c2f0Sahrens 1442ea8dc4b6Seschrock ASSERT(blkid != DB_BONUS_BLKID); 1443ea8dc4b6Seschrock 1444fa9e4066Sahrens if (dn->dn_phys->dn_nlevels == 0) 1445fa9e4066Sahrens nlevels = 1; 1446fa9e4066Sahrens else 1447fa9e4066Sahrens nlevels = dn->dn_phys->dn_nlevels; 1448fa9e4066Sahrens 1449fa9e4066Sahrens epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1450fa9e4066Sahrens 1451fa9e4066Sahrens ASSERT3U(level * epbs, <, 64); 1452fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1453ea8dc4b6Seschrock if (level >= nlevels || 1454fa9e4066Sahrens (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) { 1455fa9e4066Sahrens /* the buffer has no parent yet */ 1456fa9e4066Sahrens return (ENOENT); 1457fa9e4066Sahrens } else if (level < nlevels-1) { 1458fa9e4066Sahrens /* this block is referenced from an indirect block */ 1459fa9e4066Sahrens int err = dbuf_hold_impl(dn, level+1, 1460fa9e4066Sahrens blkid >> epbs, fail_sparse, NULL, parentp); 1461fa9e4066Sahrens if (err) 1462fa9e4066Sahrens return (err); 1463ea8dc4b6Seschrock err = dbuf_read(*parentp, NULL, 1464ea8dc4b6Seschrock (DB_RF_HAVESTRUCT | DB_RF_NOPREFETCH | DB_RF_CANFAIL)); 1465c543ec06Sahrens if (err) { 1466c543ec06Sahrens dbuf_rele(*parentp, NULL); 1467c543ec06Sahrens *parentp = NULL; 1468c543ec06Sahrens return (err); 1469c543ec06Sahrens } 1470fa9e4066Sahrens *bpp = ((blkptr_t *)(*parentp)->db.db_data) + 1471fa9e4066Sahrens (blkid & ((1ULL << epbs) - 1)); 1472c543ec06Sahrens return (0); 1473fa9e4066Sahrens } else { 1474fa9e4066Sahrens /* the block is referenced from the dnode */ 1475fa9e4066Sahrens ASSERT3U(level, ==, nlevels-1); 1476fa9e4066Sahrens ASSERT(dn->dn_phys->dn_nblkptr == 0 || 1477fa9e4066Sahrens blkid < dn->dn_phys->dn_nblkptr); 1478c543ec06Sahrens if (dn->dn_dbuf) { 1479c543ec06Sahrens dbuf_add_ref(dn->dn_dbuf, NULL); 1480fa9e4066Sahrens *parentp = dn->dn_dbuf; 1481c543ec06Sahrens } 1482fa9e4066Sahrens *bpp = &dn->dn_phys->dn_blkptr[blkid]; 1483fa9e4066Sahrens return (0); 1484fa9e4066Sahrens } 1485fa9e4066Sahrens } 1486fa9e4066Sahrens 1487fa9e4066Sahrens static dmu_buf_impl_t * 1488fa9e4066Sahrens dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid, 1489fa9e4066Sahrens dmu_buf_impl_t *parent, blkptr_t *blkptr) 1490fa9e4066Sahrens { 1491fa9e4066Sahrens objset_impl_t *os = dn->dn_objset; 1492fa9e4066Sahrens dmu_buf_impl_t *db, *odb; 1493fa9e4066Sahrens 1494fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1495fa9e4066Sahrens ASSERT(dn->dn_type != DMU_OT_NONE); 1496fa9e4066Sahrens 1497fa9e4066Sahrens db = kmem_cache_alloc(dbuf_cache, KM_SLEEP); 1498fa9e4066Sahrens 1499fa9e4066Sahrens db->db_objset = os; 1500fa9e4066Sahrens db->db.db_object = dn->dn_object; 1501fa9e4066Sahrens db->db_level = level; 1502fa9e4066Sahrens db->db_blkid = blkid; 1503c717a561Smaybee db->db_last_dirty = NULL; 1504ea8dc4b6Seschrock db->db_dirtycnt = 0; 1505ea8dc4b6Seschrock db->db_dnode = dn; 1506ea8dc4b6Seschrock db->db_parent = parent; 1507ea8dc4b6Seschrock db->db_blkptr = blkptr; 1508fa9e4066Sahrens 1509c717a561Smaybee db->db_user_ptr = NULL; 1510c717a561Smaybee db->db_user_data_ptr_ptr = NULL; 1511c717a561Smaybee db->db_evict_func = NULL; 1512c717a561Smaybee db->db_immediate_evict = 0; 1513c717a561Smaybee db->db_freed_in_flight = 0; 1514ea8dc4b6Seschrock 1515ea8dc4b6Seschrock if (blkid == DB_BONUS_BLKID) { 1516ea8dc4b6Seschrock ASSERT3P(parent, ==, dn->dn_dbuf); 15171934e92fSmaybee db->db.db_size = DN_MAX_BONUSLEN - 15181934e92fSmaybee (dn->dn_nblkptr-1) * sizeof (blkptr_t); 15191934e92fSmaybee ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 1520fa9e4066Sahrens db->db.db_offset = DB_BONUS_BLKID; 1521ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 1522ea8dc4b6Seschrock /* the bonus dbuf is not placed in the hash table */ 15235a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1524ea8dc4b6Seschrock return (db); 1525fa9e4066Sahrens } else { 1526fa9e4066Sahrens int blocksize = 1527fa9e4066Sahrens db->db_level ? 1<<dn->dn_indblkshift : dn->dn_datablksz; 1528fa9e4066Sahrens db->db.db_size = blocksize; 1529fa9e4066Sahrens db->db.db_offset = db->db_blkid * blocksize; 1530fa9e4066Sahrens } 1531fa9e4066Sahrens 1532fa9e4066Sahrens /* 1533fa9e4066Sahrens * Hold the dn_dbufs_mtx while we get the new dbuf 1534fa9e4066Sahrens * in the hash table *and* added to the dbufs list. 1535fa9e4066Sahrens * This prevents a possible deadlock with someone 1536fa9e4066Sahrens * trying to look up this dbuf before its added to the 1537fa9e4066Sahrens * dn_dbufs list. 1538fa9e4066Sahrens */ 1539fa9e4066Sahrens mutex_enter(&dn->dn_dbufs_mtx); 1540ea8dc4b6Seschrock db->db_state = DB_EVICTING; 1541fa9e4066Sahrens if ((odb = dbuf_hash_insert(db)) != NULL) { 1542fa9e4066Sahrens /* someone else inserted it first */ 1543fa9e4066Sahrens kmem_cache_free(dbuf_cache, db); 1544fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 1545fa9e4066Sahrens return (odb); 1546fa9e4066Sahrens } 1547fa9e4066Sahrens list_insert_head(&dn->dn_dbufs, db); 1548ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 1549fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 15505a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1551fa9e4066Sahrens 1552fa9e4066Sahrens if (parent && parent != dn->dn_dbuf) 1553fa9e4066Sahrens dbuf_add_ref(parent, db); 1554fa9e4066Sahrens 1555ea8dc4b6Seschrock ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 1556ea8dc4b6Seschrock refcount_count(&dn->dn_holds) > 0); 1557fa9e4066Sahrens (void) refcount_add(&dn->dn_holds, db); 1558fa9e4066Sahrens 1559fa9e4066Sahrens dprintf_dbuf(db, "db=%p\n", db); 1560fa9e4066Sahrens 1561fa9e4066Sahrens return (db); 1562fa9e4066Sahrens } 1563fa9e4066Sahrens 1564fa9e4066Sahrens static int 1565ea8dc4b6Seschrock dbuf_do_evict(void *private) 1566fa9e4066Sahrens { 1567ea8dc4b6Seschrock arc_buf_t *buf = private; 1568ea8dc4b6Seschrock dmu_buf_impl_t *db = buf->b_private; 1569fa9e4066Sahrens 1570ea8dc4b6Seschrock if (!MUTEX_HELD(&db->db_mtx)) 1571ea8dc4b6Seschrock mutex_enter(&db->db_mtx); 1572ea8dc4b6Seschrock 1573ea8dc4b6Seschrock ASSERT(refcount_is_zero(&db->db_holds)); 1574ea8dc4b6Seschrock 1575ea8dc4b6Seschrock if (db->db_state != DB_EVICTING) { 1576ea8dc4b6Seschrock ASSERT(db->db_state == DB_CACHED); 15779c9dc39aSek110237 DBUF_VERIFY(db); 1578ea8dc4b6Seschrock db->db_buf = NULL; 1579ea8dc4b6Seschrock dbuf_evict(db); 1580ea8dc4b6Seschrock } else { 1581ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 1582ea8dc4b6Seschrock dbuf_destroy(db); 1583fa9e4066Sahrens } 1584ea8dc4b6Seschrock return (0); 1585fa9e4066Sahrens } 1586fa9e4066Sahrens 1587fa9e4066Sahrens static void 1588fa9e4066Sahrens dbuf_destroy(dmu_buf_impl_t *db) 1589fa9e4066Sahrens { 1590fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 1591fa9e4066Sahrens 1592ea8dc4b6Seschrock if (db->db_blkid != DB_BONUS_BLKID) { 1593ea8dc4b6Seschrock /* 1594ea8dc4b6Seschrock * If this dbuf is still on the dn_dbufs list, 1595ea8dc4b6Seschrock * remove it from that list. 1596ea8dc4b6Seschrock */ 15971934e92fSmaybee if (db->db_dnode) { 15981934e92fSmaybee dnode_t *dn = db->db_dnode; 15991934e92fSmaybee 16001934e92fSmaybee mutex_enter(&dn->dn_dbufs_mtx); 1601ea8dc4b6Seschrock list_remove(&dn->dn_dbufs, db); 1602ea8dc4b6Seschrock mutex_exit(&dn->dn_dbufs_mtx); 1603ea8dc4b6Seschrock 1604ea8dc4b6Seschrock dnode_rele(dn, db); 16051934e92fSmaybee db->db_dnode = NULL; 1606ea8dc4b6Seschrock } 1607ea8dc4b6Seschrock dbuf_hash_remove(db); 1608ea8dc4b6Seschrock } 1609ea8dc4b6Seschrock db->db_parent = NULL; 1610ea8dc4b6Seschrock db->db_buf = NULL; 1611ea8dc4b6Seschrock 1612f82bfe17Sgw25295 ASSERT(!list_link_active(&db->db_link)); 1613fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 1614fa9e4066Sahrens ASSERT(db->db_hash_next == NULL); 1615fa9e4066Sahrens ASSERT(db->db_blkptr == NULL); 1616fa9e4066Sahrens ASSERT(db->db_data_pending == NULL); 1617fa9e4066Sahrens 1618fa9e4066Sahrens kmem_cache_free(dbuf_cache, db); 16195a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1620fa9e4066Sahrens } 1621fa9e4066Sahrens 1622fa9e4066Sahrens void 1623fa9e4066Sahrens dbuf_prefetch(dnode_t *dn, uint64_t blkid) 1624fa9e4066Sahrens { 162513506d1eSmaybee dmu_buf_impl_t *db = NULL; 1626fa9e4066Sahrens blkptr_t *bp = NULL; 1627fa9e4066Sahrens 1628fa9e4066Sahrens ASSERT(blkid != DB_BONUS_BLKID); 1629fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1630fa9e4066Sahrens 1631fa9e4066Sahrens if (dnode_block_freed(dn, blkid)) 1632fa9e4066Sahrens return; 1633fa9e4066Sahrens 1634fa9e4066Sahrens /* dbuf_find() returns with db_mtx held */ 1635fa9e4066Sahrens if (db = dbuf_find(dn, 0, blkid)) { 163613506d1eSmaybee if (refcount_count(&db->db_holds) > 0) { 1637fa9e4066Sahrens /* 163813506d1eSmaybee * This dbuf is active. We assume that it is 163913506d1eSmaybee * already CACHED, or else about to be either 1640fa9e4066Sahrens * read or filled. 1641fa9e4066Sahrens */ 1642fa9e4066Sahrens mutex_exit(&db->db_mtx); 1643fa9e4066Sahrens return; 1644fa9e4066Sahrens } 164513506d1eSmaybee mutex_exit(&db->db_mtx); 16460b69c2f0Sahrens db = NULL; 164713506d1eSmaybee } 1648fa9e4066Sahrens 164913506d1eSmaybee if (dbuf_findbp(dn, 0, blkid, TRUE, &db, &bp) == 0) { 1650fa9e4066Sahrens if (bp && !BP_IS_HOLE(bp)) { 1651088f3894Sahrens arc_buf_t *pbuf; 165213506d1eSmaybee uint32_t aflags = ARC_NOWAIT | ARC_PREFETCH; 1653ea8dc4b6Seschrock zbookmark_t zb; 1654ea8dc4b6Seschrock zb.zb_objset = dn->dn_objset->os_dsl_dataset ? 1655ea8dc4b6Seschrock dn->dn_objset->os_dsl_dataset->ds_object : 0; 1656ea8dc4b6Seschrock zb.zb_object = dn->dn_object; 1657ea8dc4b6Seschrock zb.zb_level = 0; 1658ea8dc4b6Seschrock zb.zb_blkid = blkid; 1659ea8dc4b6Seschrock 1660088f3894Sahrens if (db) 1661088f3894Sahrens pbuf = db->db_buf; 1662088f3894Sahrens else 1663088f3894Sahrens pbuf = dn->dn_objset->os_phys_buf; 1664088f3894Sahrens 1665088f3894Sahrens (void) arc_read(NULL, dn->dn_objset->os_spa, 1666088f3894Sahrens bp, pbuf, NULL, NULL, ZIO_PRIORITY_ASYNC_READ, 1667fa9e4066Sahrens ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 166813506d1eSmaybee &aflags, &zb); 1669fa9e4066Sahrens } 167013506d1eSmaybee if (db) 167113506d1eSmaybee dbuf_rele(db, NULL); 1672fa9e4066Sahrens } 1673fa9e4066Sahrens } 1674fa9e4066Sahrens 1675fa9e4066Sahrens /* 1676fa9e4066Sahrens * Returns with db_holds incremented, and db_mtx not held. 1677fa9e4066Sahrens * Note: dn_struct_rwlock must be held. 1678fa9e4066Sahrens */ 1679fa9e4066Sahrens int 1680fa9e4066Sahrens dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, int fail_sparse, 1681fa9e4066Sahrens void *tag, dmu_buf_impl_t **dbp) 1682fa9e4066Sahrens { 1683fa9e4066Sahrens dmu_buf_impl_t *db, *parent = NULL; 1684fa9e4066Sahrens 1685ea8dc4b6Seschrock ASSERT(blkid != DB_BONUS_BLKID); 1686fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1687fa9e4066Sahrens ASSERT3U(dn->dn_nlevels, >, level); 1688fa9e4066Sahrens 1689fa9e4066Sahrens *dbp = NULL; 1690ea8dc4b6Seschrock top: 1691fa9e4066Sahrens /* dbuf_find() returns with db_mtx held */ 1692fa9e4066Sahrens db = dbuf_find(dn, level, blkid); 1693fa9e4066Sahrens 1694fa9e4066Sahrens if (db == NULL) { 1695fa9e4066Sahrens blkptr_t *bp = NULL; 1696fa9e4066Sahrens int err; 1697fa9e4066Sahrens 1698c543ec06Sahrens ASSERT3P(parent, ==, NULL); 1699fa9e4066Sahrens err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp); 1700fa9e4066Sahrens if (fail_sparse) { 1701fa9e4066Sahrens if (err == 0 && bp && BP_IS_HOLE(bp)) 1702fa9e4066Sahrens err = ENOENT; 1703fa9e4066Sahrens if (err) { 1704c543ec06Sahrens if (parent) 1705ea8dc4b6Seschrock dbuf_rele(parent, NULL); 1706fa9e4066Sahrens return (err); 1707fa9e4066Sahrens } 1708fa9e4066Sahrens } 1709ea8dc4b6Seschrock if (err && err != ENOENT) 1710ea8dc4b6Seschrock return (err); 1711fa9e4066Sahrens db = dbuf_create(dn, level, blkid, parent, bp); 1712fa9e4066Sahrens } 1713fa9e4066Sahrens 1714ea8dc4b6Seschrock if (db->db_buf && refcount_is_zero(&db->db_holds)) { 1715ea8dc4b6Seschrock arc_buf_add_ref(db->db_buf, db); 1716ea8dc4b6Seschrock if (db->db_buf->b_data == NULL) { 1717ea8dc4b6Seschrock dbuf_clear(db); 1718c543ec06Sahrens if (parent) { 1719c543ec06Sahrens dbuf_rele(parent, NULL); 1720c543ec06Sahrens parent = NULL; 1721c543ec06Sahrens } 1722ea8dc4b6Seschrock goto top; 1723ea8dc4b6Seschrock } 1724ea8dc4b6Seschrock ASSERT3P(db->db.db_data, ==, db->db_buf->b_data); 1725ea8dc4b6Seschrock } 1726ea8dc4b6Seschrock 1727ea8dc4b6Seschrock ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf)); 1728ea8dc4b6Seschrock 1729fa9e4066Sahrens /* 1730c717a561Smaybee * If this buffer is currently syncing out, and we are are 1731c717a561Smaybee * still referencing it from db_data, we need to make a copy 1732c717a561Smaybee * of it in case we decide we want to dirty it again in this txg. 1733fa9e4066Sahrens */ 1734c717a561Smaybee if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID && 1735ea8dc4b6Seschrock dn->dn_object != DMU_META_DNODE_OBJECT && 1736c717a561Smaybee db->db_state == DB_CACHED && db->db_data_pending) { 1737c717a561Smaybee dbuf_dirty_record_t *dr = db->db_data_pending; 1738c717a561Smaybee 1739c717a561Smaybee if (dr->dt.dl.dr_data == db->db_buf) { 1740ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 1741fa9e4066Sahrens 1742c717a561Smaybee dbuf_set_data(db, 1743c717a561Smaybee arc_buf_alloc(db->db_dnode->dn_objset->os_spa, 1744c717a561Smaybee db->db.db_size, db, type)); 1745c717a561Smaybee bcopy(dr->dt.dl.dr_data->b_data, db->db.db_data, 1746fa9e4066Sahrens db->db.db_size); 1747fa9e4066Sahrens } 1748c717a561Smaybee } 1749fa9e4066Sahrens 1750ea8dc4b6Seschrock (void) refcount_add(&db->db_holds, tag); 1751fa9e4066Sahrens dbuf_update_data(db); 17529c9dc39aSek110237 DBUF_VERIFY(db); 1753fa9e4066Sahrens mutex_exit(&db->db_mtx); 1754fa9e4066Sahrens 1755fa9e4066Sahrens /* NOTE: we can't rele the parent until after we drop the db_mtx */ 1756c543ec06Sahrens if (parent) 1757ea8dc4b6Seschrock dbuf_rele(parent, NULL); 1758fa9e4066Sahrens 1759fa9e4066Sahrens ASSERT3P(db->db_dnode, ==, dn); 1760fa9e4066Sahrens ASSERT3U(db->db_blkid, ==, blkid); 1761fa9e4066Sahrens ASSERT3U(db->db_level, ==, level); 1762fa9e4066Sahrens *dbp = db; 1763fa9e4066Sahrens 1764fa9e4066Sahrens return (0); 1765fa9e4066Sahrens } 1766fa9e4066Sahrens 1767fa9e4066Sahrens dmu_buf_impl_t * 1768ea8dc4b6Seschrock dbuf_hold(dnode_t *dn, uint64_t blkid, void *tag) 1769fa9e4066Sahrens { 1770fa9e4066Sahrens dmu_buf_impl_t *db; 1771ea8dc4b6Seschrock int err = dbuf_hold_impl(dn, 0, blkid, FALSE, tag, &db); 1772ea8dc4b6Seschrock return (err ? NULL : db); 1773fa9e4066Sahrens } 1774fa9e4066Sahrens 1775fa9e4066Sahrens dmu_buf_impl_t * 1776fa9e4066Sahrens dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, void *tag) 1777fa9e4066Sahrens { 1778fa9e4066Sahrens dmu_buf_impl_t *db; 1779ea8dc4b6Seschrock int err = dbuf_hold_impl(dn, level, blkid, FALSE, tag, &db); 1780ea8dc4b6Seschrock return (err ? NULL : db); 1781fa9e4066Sahrens } 1782fa9e4066Sahrens 17831934e92fSmaybee void 1784ea8dc4b6Seschrock dbuf_create_bonus(dnode_t *dn) 1785fa9e4066Sahrens { 1786ea8dc4b6Seschrock ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 1787ea8dc4b6Seschrock 1788ea8dc4b6Seschrock ASSERT(dn->dn_bonus == NULL); 17891934e92fSmaybee dn->dn_bonus = dbuf_create(dn, 0, DB_BONUS_BLKID, dn->dn_dbuf, NULL); 1790fa9e4066Sahrens } 1791fa9e4066Sahrens 1792ea8dc4b6Seschrock #pragma weak dmu_buf_add_ref = dbuf_add_ref 1793fa9e4066Sahrens void 1794fa9e4066Sahrens dbuf_add_ref(dmu_buf_impl_t *db, void *tag) 1795fa9e4066Sahrens { 1796ea8dc4b6Seschrock int64_t holds = refcount_add(&db->db_holds, tag); 1797ea8dc4b6Seschrock ASSERT(holds > 1); 1798fa9e4066Sahrens } 1799fa9e4066Sahrens 1800ea8dc4b6Seschrock #pragma weak dmu_buf_rele = dbuf_rele 1801fa9e4066Sahrens void 1802ea8dc4b6Seschrock dbuf_rele(dmu_buf_impl_t *db, void *tag) 1803fa9e4066Sahrens { 1804fa9e4066Sahrens int64_t holds; 1805fa9e4066Sahrens 1806fa9e4066Sahrens mutex_enter(&db->db_mtx); 18079c9dc39aSek110237 DBUF_VERIFY(db); 1808fa9e4066Sahrens 1809fa9e4066Sahrens holds = refcount_remove(&db->db_holds, tag); 1810ea8dc4b6Seschrock ASSERT(holds >= 0); 1811fa9e4066Sahrens 1812c717a561Smaybee /* 1813c717a561Smaybee * We can't freeze indirects if there is a possibility that they 1814c717a561Smaybee * may be modified in the current syncing context. 1815c717a561Smaybee */ 1816c717a561Smaybee if (db->db_buf && holds == (db->db_level == 0 ? db->db_dirtycnt : 0)) 18176b4acc8bSahrens arc_buf_freeze(db->db_buf); 18186b4acc8bSahrens 1819fa9e4066Sahrens if (holds == db->db_dirtycnt && 1820c717a561Smaybee db->db_level == 0 && db->db_immediate_evict) 1821fa9e4066Sahrens dbuf_evict_user(db); 1822ea8dc4b6Seschrock 1823ea8dc4b6Seschrock if (holds == 0) { 1824ea8dc4b6Seschrock if (db->db_blkid == DB_BONUS_BLKID) { 1825ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 1826ea8dc4b6Seschrock dnode_rele(db->db_dnode, db); 1827ea8dc4b6Seschrock } else if (db->db_buf == NULL) { 1828ea8dc4b6Seschrock /* 1829ea8dc4b6Seschrock * This is a special case: we never associated this 1830ea8dc4b6Seschrock * dbuf with any data allocated from the ARC. 1831ea8dc4b6Seschrock */ 183282c9918fSTim Haley ASSERT(db->db_state == DB_UNCACHED || 183382c9918fSTim Haley db->db_state == DB_NOFILL); 1834ea8dc4b6Seschrock dbuf_evict(db); 1835ea8dc4b6Seschrock } else if (arc_released(db->db_buf)) { 1836ea8dc4b6Seschrock arc_buf_t *buf = db->db_buf; 1837ea8dc4b6Seschrock /* 1838ea8dc4b6Seschrock * This dbuf has anonymous data associated with it. 1839ea8dc4b6Seschrock */ 1840ea8dc4b6Seschrock dbuf_set_data(db, NULL); 1841ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(buf, db) == 1); 1842ea8dc4b6Seschrock dbuf_evict(db); 1843ea8dc4b6Seschrock } else { 1844ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(db->db_buf, db) == 0); 18453baa08fcSek110237 if (!DBUF_IS_CACHEABLE(db)) 18463baa08fcSek110237 dbuf_clear(db); 18473baa08fcSek110237 else 1848fa9e4066Sahrens mutex_exit(&db->db_mtx); 1849fa9e4066Sahrens } 1850ea8dc4b6Seschrock } else { 1851ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 1852fa9e4066Sahrens } 1853fa9e4066Sahrens } 1854fa9e4066Sahrens 1855fa9e4066Sahrens #pragma weak dmu_buf_refcount = dbuf_refcount 1856fa9e4066Sahrens uint64_t 1857fa9e4066Sahrens dbuf_refcount(dmu_buf_impl_t *db) 1858fa9e4066Sahrens { 1859fa9e4066Sahrens return (refcount_count(&db->db_holds)); 1860fa9e4066Sahrens } 1861fa9e4066Sahrens 1862fa9e4066Sahrens void * 1863fa9e4066Sahrens dmu_buf_set_user(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 1864fa9e4066Sahrens dmu_buf_evict_func_t *evict_func) 1865fa9e4066Sahrens { 1866fa9e4066Sahrens return (dmu_buf_update_user(db_fake, NULL, user_ptr, 1867fa9e4066Sahrens user_data_ptr_ptr, evict_func)); 1868fa9e4066Sahrens } 1869fa9e4066Sahrens 1870fa9e4066Sahrens void * 1871fa9e4066Sahrens dmu_buf_set_user_ie(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 1872fa9e4066Sahrens dmu_buf_evict_func_t *evict_func) 1873fa9e4066Sahrens { 1874fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1875fa9e4066Sahrens 1876c717a561Smaybee db->db_immediate_evict = TRUE; 1877fa9e4066Sahrens return (dmu_buf_update_user(db_fake, NULL, user_ptr, 1878fa9e4066Sahrens user_data_ptr_ptr, evict_func)); 1879fa9e4066Sahrens } 1880fa9e4066Sahrens 1881fa9e4066Sahrens void * 1882fa9e4066Sahrens dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr, void *user_ptr, 1883fa9e4066Sahrens void *user_data_ptr_ptr, dmu_buf_evict_func_t *evict_func) 1884fa9e4066Sahrens { 1885fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1886fa9e4066Sahrens ASSERT(db->db_level == 0); 1887fa9e4066Sahrens 1888fa9e4066Sahrens ASSERT((user_ptr == NULL) == (evict_func == NULL)); 1889fa9e4066Sahrens 1890fa9e4066Sahrens mutex_enter(&db->db_mtx); 1891fa9e4066Sahrens 1892c717a561Smaybee if (db->db_user_ptr == old_user_ptr) { 1893c717a561Smaybee db->db_user_ptr = user_ptr; 1894c717a561Smaybee db->db_user_data_ptr_ptr = user_data_ptr_ptr; 1895c717a561Smaybee db->db_evict_func = evict_func; 1896fa9e4066Sahrens 1897fa9e4066Sahrens dbuf_update_data(db); 1898fa9e4066Sahrens } else { 1899c717a561Smaybee old_user_ptr = db->db_user_ptr; 1900fa9e4066Sahrens } 1901fa9e4066Sahrens 1902fa9e4066Sahrens mutex_exit(&db->db_mtx); 1903fa9e4066Sahrens return (old_user_ptr); 1904fa9e4066Sahrens } 1905fa9e4066Sahrens 1906fa9e4066Sahrens void * 1907fa9e4066Sahrens dmu_buf_get_user(dmu_buf_t *db_fake) 1908fa9e4066Sahrens { 1909fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1910fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 1911fa9e4066Sahrens 1912c717a561Smaybee return (db->db_user_ptr); 1913fa9e4066Sahrens } 1914fa9e4066Sahrens 1915c717a561Smaybee static void 1916c717a561Smaybee dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db) 1917fa9e4066Sahrens { 1918c717a561Smaybee /* ASSERT(dmu_tx_is_syncing(tx) */ 1919c717a561Smaybee ASSERT(MUTEX_HELD(&db->db_mtx)); 1920c717a561Smaybee 1921c717a561Smaybee if (db->db_blkptr != NULL) 1922c717a561Smaybee return; 1923c717a561Smaybee 1924c717a561Smaybee if (db->db_level == dn->dn_phys->dn_nlevels-1) { 1925c717a561Smaybee /* 1926c717a561Smaybee * This buffer was allocated at a time when there was 1927c717a561Smaybee * no available blkptrs from the dnode, or it was 1928c717a561Smaybee * inappropriate to hook it in (i.e., nlevels mis-match). 1929c717a561Smaybee */ 1930c717a561Smaybee ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr); 1931c717a561Smaybee ASSERT(db->db_parent == NULL); 1932c717a561Smaybee db->db_parent = dn->dn_dbuf; 1933c717a561Smaybee db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid]; 1934c717a561Smaybee DBUF_VERIFY(db); 1935c717a561Smaybee } else { 1936c717a561Smaybee dmu_buf_impl_t *parent = db->db_parent; 1937c717a561Smaybee int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 1938c717a561Smaybee 1939c717a561Smaybee ASSERT(dn->dn_phys->dn_nlevels > 1); 1940c717a561Smaybee if (parent == NULL) { 1941c717a561Smaybee mutex_exit(&db->db_mtx); 1942c717a561Smaybee rw_enter(&dn->dn_struct_rwlock, RW_READER); 1943c717a561Smaybee (void) dbuf_hold_impl(dn, db->db_level+1, 1944c717a561Smaybee db->db_blkid >> epbs, FALSE, db, &parent); 1945c717a561Smaybee rw_exit(&dn->dn_struct_rwlock); 1946c717a561Smaybee mutex_enter(&db->db_mtx); 1947c717a561Smaybee db->db_parent = parent; 1948c717a561Smaybee } 1949c717a561Smaybee db->db_blkptr = (blkptr_t *)parent->db.db_data + 1950c717a561Smaybee (db->db_blkid & ((1ULL << epbs) - 1)); 1951c717a561Smaybee DBUF_VERIFY(db); 1952c717a561Smaybee } 1953c717a561Smaybee } 1954c717a561Smaybee 1955c717a561Smaybee static void 1956c717a561Smaybee dbuf_sync_indirect(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 1957c717a561Smaybee { 1958c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 1959c717a561Smaybee dnode_t *dn = db->db_dnode; 1960c717a561Smaybee zio_t *zio; 1961c717a561Smaybee 1962c717a561Smaybee ASSERT(dmu_tx_is_syncing(tx)); 1963c717a561Smaybee 1964c717a561Smaybee dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 1965c717a561Smaybee 1966c717a561Smaybee mutex_enter(&db->db_mtx); 1967c717a561Smaybee 1968c717a561Smaybee ASSERT(db->db_level > 0); 1969c717a561Smaybee DBUF_VERIFY(db); 1970c717a561Smaybee 1971c717a561Smaybee if (db->db_buf == NULL) { 1972c717a561Smaybee mutex_exit(&db->db_mtx); 1973c717a561Smaybee (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED); 1974c717a561Smaybee mutex_enter(&db->db_mtx); 1975c717a561Smaybee } 1976c717a561Smaybee ASSERT3U(db->db_state, ==, DB_CACHED); 1977c717a561Smaybee ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 1978c717a561Smaybee ASSERT(db->db_buf != NULL); 1979c717a561Smaybee 1980c717a561Smaybee dbuf_check_blkptr(dn, db); 1981c717a561Smaybee 1982c717a561Smaybee db->db_data_pending = dr; 1983c717a561Smaybee 1984af2c4821Smaybee mutex_exit(&db->db_mtx); 1985088f3894Sahrens dbuf_write(dr, db->db_buf, tx); 1986c717a561Smaybee 1987c717a561Smaybee zio = dr->dr_zio; 1988c717a561Smaybee mutex_enter(&dr->dt.di.dr_mtx); 1989c717a561Smaybee dbuf_sync_list(&dr->dt.di.dr_children, tx); 1990c717a561Smaybee ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 1991c717a561Smaybee mutex_exit(&dr->dt.di.dr_mtx); 1992c717a561Smaybee zio_nowait(zio); 1993c717a561Smaybee } 1994c717a561Smaybee 1995c717a561Smaybee static void 1996c717a561Smaybee dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 1997c717a561Smaybee { 1998c717a561Smaybee arc_buf_t **datap = &dr->dt.dl.dr_data; 1999c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 2000fa9e4066Sahrens dnode_t *dn = db->db_dnode; 2001fa9e4066Sahrens objset_impl_t *os = dn->dn_objset; 2002c717a561Smaybee uint64_t txg = tx->tx_txg; 2003fa9e4066Sahrens 2004fa9e4066Sahrens ASSERT(dmu_tx_is_syncing(tx)); 2005fa9e4066Sahrens 2006fa9e4066Sahrens dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 2007fa9e4066Sahrens 2008fa9e4066Sahrens mutex_enter(&db->db_mtx); 2009fa9e4066Sahrens /* 2010fa9e4066Sahrens * To be synced, we must be dirtied. But we 2011fa9e4066Sahrens * might have been freed after the dirty. 2012fa9e4066Sahrens */ 2013fa9e4066Sahrens if (db->db_state == DB_UNCACHED) { 2014fa9e4066Sahrens /* This buffer has been freed since it was dirtied */ 2015fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 2016fa9e4066Sahrens } else if (db->db_state == DB_FILL) { 2017fa9e4066Sahrens /* This buffer was freed and is now being re-filled */ 2018c717a561Smaybee ASSERT(db->db.db_data != dr->dt.dl.dr_data); 2019fa9e4066Sahrens } else { 202082c9918fSTim Haley ASSERT(db->db_state == DB_CACHED || db->db_state == DB_NOFILL); 2021fa9e4066Sahrens } 20229c9dc39aSek110237 DBUF_VERIFY(db); 2023fa9e4066Sahrens 2024fa9e4066Sahrens /* 2025c717a561Smaybee * If this is a bonus buffer, simply copy the bonus data into the 2026c717a561Smaybee * dnode. It will be written out when the dnode is synced (and it 2027c717a561Smaybee * will be synced, since it must have been dirty for dbuf_sync to 2028c717a561Smaybee * be called). 2029fa9e4066Sahrens */ 2030ea8dc4b6Seschrock if (db->db_blkid == DB_BONUS_BLKID) { 2031c717a561Smaybee dbuf_dirty_record_t **drp; 20321934e92fSmaybee 2033ea8dc4b6Seschrock ASSERT(*datap != NULL); 2034ea8dc4b6Seschrock ASSERT3U(db->db_level, ==, 0); 2035ea8dc4b6Seschrock ASSERT3U(dn->dn_phys->dn_bonuslen, <=, DN_MAX_BONUSLEN); 2036ea8dc4b6Seschrock bcopy(*datap, DN_BONUS(dn->dn_phys), dn->dn_phys->dn_bonuslen); 20370e8c6158Smaybee if (*datap != db->db.db_data) { 2038ea8dc4b6Seschrock zio_buf_free(*datap, DN_MAX_BONUSLEN); 20395a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 20400e8c6158Smaybee } 2041ea8dc4b6Seschrock db->db_data_pending = NULL; 2042c717a561Smaybee drp = &db->db_last_dirty; 2043c717a561Smaybee while (*drp != dr) 2044c717a561Smaybee drp = &(*drp)->dr_next; 204517f17c2dSbonwick ASSERT(dr->dr_next == NULL); 204617f17c2dSbonwick *drp = dr->dr_next; 2047c717a561Smaybee kmem_free(dr, sizeof (dbuf_dirty_record_t)); 2048ea8dc4b6Seschrock ASSERT(db->db_dirtycnt > 0); 2049ea8dc4b6Seschrock db->db_dirtycnt -= 1; 2050ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 2051ea8dc4b6Seschrock dbuf_rele(db, (void *)(uintptr_t)txg); 2052ea8dc4b6Seschrock return; 2053ea8dc4b6Seschrock } 2054ea8dc4b6Seschrock 2055c5c6ffa0Smaybee /* 2056f82bfe17Sgw25295 * This function may have dropped the db_mtx lock allowing a dmu_sync 2057f82bfe17Sgw25295 * operation to sneak in. As a result, we need to ensure that we 2058f82bfe17Sgw25295 * don't check the dr_override_state until we have returned from 2059f82bfe17Sgw25295 * dbuf_check_blkptr. 2060f82bfe17Sgw25295 */ 2061f82bfe17Sgw25295 dbuf_check_blkptr(dn, db); 2062f82bfe17Sgw25295 2063f82bfe17Sgw25295 /* 2064c717a561Smaybee * If this buffer is in the middle of an immdiate write, 2065c717a561Smaybee * wait for the synchronous IO to complete. 2066c5c6ffa0Smaybee */ 2067c717a561Smaybee while (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC) { 2068c5c6ffa0Smaybee ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT); 2069c5c6ffa0Smaybee cv_wait(&db->db_changed, &db->db_mtx); 2070c717a561Smaybee ASSERT(dr->dt.dl.dr_override_state != DR_NOT_OVERRIDDEN); 2071c5c6ffa0Smaybee } 2072c717a561Smaybee 2073c717a561Smaybee /* 2074c717a561Smaybee * If this dbuf has already been written out via an immediate write, 2075c717a561Smaybee * just complete the write by copying over the new block pointer and 2076c717a561Smaybee * updating the accounting via the write-completion functions. 2077c717a561Smaybee */ 2078c717a561Smaybee if (dr->dt.dl.dr_override_state == DR_OVERRIDDEN) { 2079c717a561Smaybee zio_t zio_fake; 2080c717a561Smaybee 2081c717a561Smaybee zio_fake.io_private = &db; 2082c717a561Smaybee zio_fake.io_error = 0; 2083c717a561Smaybee zio_fake.io_bp = db->db_blkptr; 2084c717a561Smaybee zio_fake.io_bp_orig = *db->db_blkptr; 2085c717a561Smaybee zio_fake.io_txg = txg; 2086e14bb325SJeff Bonwick zio_fake.io_flags = 0; 2087c717a561Smaybee 2088c717a561Smaybee *db->db_blkptr = dr->dt.dl.dr_overridden_by; 2089c717a561Smaybee dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 2090c717a561Smaybee db->db_data_pending = dr; 2091c717a561Smaybee dr->dr_zio = &zio_fake; 2092c717a561Smaybee mutex_exit(&db->db_mtx); 2093c717a561Smaybee 2094e14bb325SJeff Bonwick ASSERT(!DVA_EQUAL(BP_IDENTITY(zio_fake.io_bp), 2095f4a72450SJeff Bonwick BP_IDENTITY(&zio_fake.io_bp_orig)) || 2096f4a72450SJeff Bonwick BP_IS_HOLE(zio_fake.io_bp)); 2097e14bb325SJeff Bonwick 2098c717a561Smaybee if (BP_IS_OLDER(&zio_fake.io_bp_orig, txg)) 2099cdb0ab79Smaybee (void) dsl_dataset_block_kill(os->os_dsl_dataset, 2100c717a561Smaybee &zio_fake.io_bp_orig, dn->dn_zio, tx); 2101c717a561Smaybee 2102c717a561Smaybee dbuf_write_ready(&zio_fake, db->db_buf, db); 2103c717a561Smaybee dbuf_write_done(&zio_fake, db->db_buf, db); 2104c717a561Smaybee 2105c717a561Smaybee return; 2106c717a561Smaybee } 2107c717a561Smaybee 2108ab69d62fSMatthew Ahrens if (db->db_state != DB_NOFILL && 2109ab69d62fSMatthew Ahrens dn->dn_object != DMU_META_DNODE_OBJECT && 2110ab69d62fSMatthew Ahrens refcount_count(&db->db_holds) > 1 && 2111ab69d62fSMatthew Ahrens *datap == db->db_buf) { 2112fa9e4066Sahrens /* 211382c9918fSTim Haley * If this buffer is currently "in use" (i.e., there 211482c9918fSTim Haley * are active holds and db_data still references it), 211582c9918fSTim Haley * then make a copy before we start the write so that 211682c9918fSTim Haley * any modifications from the open txg will not leak 211782c9918fSTim Haley * into this write. 2118fa9e4066Sahrens * 211982c9918fSTim Haley * NOTE: this copy does not need to be made for 212082c9918fSTim Haley * objects only modified in the syncing context (e.g. 212182c9918fSTim Haley * DNONE_DNODE blocks). 2122fa9e4066Sahrens */ 2123ab69d62fSMatthew Ahrens int blksz = arc_buf_size(*datap); 2124ab69d62fSMatthew Ahrens arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 2125ab69d62fSMatthew Ahrens *datap = arc_buf_alloc(os->os_spa, blksz, db, type); 2126c717a561Smaybee bcopy(db->db.db_data, (*datap)->b_data, blksz); 2127fa9e4066Sahrens } 2128c717a561Smaybee db->db_data_pending = dr; 2129fa9e4066Sahrens 2130fa9e4066Sahrens mutex_exit(&db->db_mtx); 2131fa9e4066Sahrens 2132088f3894Sahrens dbuf_write(dr, *datap, tx); 2133c717a561Smaybee 2134c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 2135c717a561Smaybee if (dn->dn_object == DMU_META_DNODE_OBJECT) 2136c717a561Smaybee list_insert_tail(&dn->dn_dirty_records[txg&TXG_MASK], dr); 2137c717a561Smaybee else 2138c717a561Smaybee zio_nowait(dr->dr_zio); 2139fa9e4066Sahrens } 2140c717a561Smaybee 2141c717a561Smaybee void 2142c717a561Smaybee dbuf_sync_list(list_t *list, dmu_tx_t *tx) 2143c717a561Smaybee { 2144c717a561Smaybee dbuf_dirty_record_t *dr; 2145c717a561Smaybee 2146c717a561Smaybee while (dr = list_head(list)) { 2147c717a561Smaybee if (dr->dr_zio != NULL) { 2148c717a561Smaybee /* 2149c717a561Smaybee * If we find an already initialized zio then we 2150c717a561Smaybee * are processing the meta-dnode, and we have finished. 2151c717a561Smaybee * The dbufs for all dnodes are put back on the list 2152c717a561Smaybee * during processing, so that we can zio_wait() 2153c717a561Smaybee * these IOs after initiating all child IOs. 2154c717a561Smaybee */ 2155c717a561Smaybee ASSERT3U(dr->dr_dbuf->db.db_object, ==, 2156c717a561Smaybee DMU_META_DNODE_OBJECT); 2157c717a561Smaybee break; 2158fa9e4066Sahrens } 2159c717a561Smaybee list_remove(list, dr); 2160c717a561Smaybee if (dr->dr_dbuf->db_level > 0) 2161c717a561Smaybee dbuf_sync_indirect(dr, tx); 2162c717a561Smaybee else 2163c717a561Smaybee dbuf_sync_leaf(dr, tx); 2164c717a561Smaybee } 2165c717a561Smaybee } 2166c717a561Smaybee 2167c717a561Smaybee static void 2168088f3894Sahrens dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx) 2169c717a561Smaybee { 2170c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 2171c717a561Smaybee dnode_t *dn = db->db_dnode; 2172c717a561Smaybee objset_impl_t *os = dn->dn_objset; 2173c717a561Smaybee dmu_buf_impl_t *parent = db->db_parent; 2174c717a561Smaybee uint64_t txg = tx->tx_txg; 2175c717a561Smaybee zbookmark_t zb; 2176088f3894Sahrens writeprops_t wp = { 0 }; 2177c717a561Smaybee zio_t *zio; 2178c717a561Smaybee 2179088f3894Sahrens if (!BP_IS_HOLE(db->db_blkptr) && 2180088f3894Sahrens (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE)) { 2181088f3894Sahrens /* 2182088f3894Sahrens * Private object buffers are released here rather 2183088f3894Sahrens * than in dbuf_dirty() since they are only modified 2184088f3894Sahrens * in the syncing context and we don't want the 2185088f3894Sahrens * overhead of making multiple copies of the data. 2186088f3894Sahrens */ 2187088f3894Sahrens arc_release(data, db); 218882c9918fSTim Haley } else if (db->db_state != DB_NOFILL) { 2189088f3894Sahrens ASSERT(arc_released(data)); 2190088f3894Sahrens /* XXX why do we need to thaw here? */ 2191088f3894Sahrens arc_buf_thaw(data); 2192088f3894Sahrens } 2193088f3894Sahrens 2194c717a561Smaybee if (parent != dn->dn_dbuf) { 2195c717a561Smaybee ASSERT(parent && parent->db_data_pending); 2196c717a561Smaybee ASSERT(db->db_level == parent->db_level-1); 2197c717a561Smaybee ASSERT(arc_released(parent->db_buf)); 2198c717a561Smaybee zio = parent->db_data_pending->dr_zio; 2199c717a561Smaybee } else { 2200c717a561Smaybee ASSERT(db->db_level == dn->dn_phys->dn_nlevels-1); 2201c717a561Smaybee ASSERT3P(db->db_blkptr, ==, 2202c717a561Smaybee &dn->dn_phys->dn_blkptr[db->db_blkid]); 2203c717a561Smaybee zio = dn->dn_zio; 2204c717a561Smaybee } 2205c717a561Smaybee 2206c717a561Smaybee ASSERT(db->db_level == 0 || data == db->db_buf); 2207c717a561Smaybee ASSERT3U(db->db_blkptr->blk_birth, <=, txg); 2208c717a561Smaybee ASSERT(zio); 2209c717a561Smaybee 2210ea8dc4b6Seschrock zb.zb_objset = os->os_dsl_dataset ? os->os_dsl_dataset->ds_object : 0; 2211ea8dc4b6Seschrock zb.zb_object = db->db.db_object; 2212ea8dc4b6Seschrock zb.zb_level = db->db_level; 2213ea8dc4b6Seschrock zb.zb_blkid = db->db_blkid; 221444cd46caSbillm 2215088f3894Sahrens wp.wp_type = dn->dn_type; 2216088f3894Sahrens wp.wp_level = db->db_level; 2217088f3894Sahrens wp.wp_copies = os->os_copies; 2218088f3894Sahrens wp.wp_dncompress = dn->dn_compress; 2219088f3894Sahrens wp.wp_oscompress = os->os_compress; 2220088f3894Sahrens wp.wp_dnchecksum = dn->dn_checksum; 2221088f3894Sahrens wp.wp_oschecksum = os->os_checksum; 2222088f3894Sahrens 2223c717a561Smaybee if (BP_IS_OLDER(db->db_blkptr, txg)) 2224cdb0ab79Smaybee (void) dsl_dataset_block_kill( 2225c717a561Smaybee os->os_dsl_dataset, db->db_blkptr, zio, tx); 2226c717a561Smaybee 222782c9918fSTim Haley if (db->db_state == DB_NOFILL) { 222882c9918fSTim Haley zio_prop_t zp = { 0 }; 222982c9918fSTim Haley 223082c9918fSTim Haley write_policy(os->os_spa, &wp, &zp); 223182c9918fSTim Haley dr->dr_zio = zio_write(zio, os->os_spa, 223282c9918fSTim Haley txg, db->db_blkptr, NULL, 223382c9918fSTim Haley db->db.db_size, &zp, dbuf_skip_write_ready, 223482c9918fSTim Haley dbuf_skip_write_done, db, ZIO_PRIORITY_ASYNC_WRITE, 223582c9918fSTim Haley ZIO_FLAG_MUSTSUCCEED, &zb); 223682c9918fSTim Haley } else { 2237088f3894Sahrens dr->dr_zio = arc_write(zio, os->os_spa, &wp, 22383baa08fcSek110237 DBUF_IS_L2CACHEABLE(db), txg, db->db_blkptr, 22393baa08fcSek110237 data, dbuf_write_ready, dbuf_write_done, db, 2240e14bb325SJeff Bonwick ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); 2241fa9e4066Sahrens } 224282c9918fSTim Haley } 224382c9918fSTim Haley 224482c9918fSTim Haley /* wrapper function for dbuf_write_ready bypassing ARC */ 224582c9918fSTim Haley static void 224682c9918fSTim Haley dbuf_skip_write_ready(zio_t *zio) 224782c9918fSTim Haley { 224882c9918fSTim Haley blkptr_t *bp = zio->io_bp; 224982c9918fSTim Haley 225082c9918fSTim Haley if (!BP_IS_GANG(bp)) 225182c9918fSTim Haley zio_skip_write(zio); 225282c9918fSTim Haley 225382c9918fSTim Haley dbuf_write_ready(zio, NULL, zio->io_private); 225482c9918fSTim Haley } 225582c9918fSTim Haley 225682c9918fSTim Haley /* wrapper function for dbuf_write_done bypassing ARC */ 225782c9918fSTim Haley static void 225882c9918fSTim Haley dbuf_skip_write_done(zio_t *zio) 225982c9918fSTim Haley { 226082c9918fSTim Haley dbuf_write_done(zio, NULL, zio->io_private); 226182c9918fSTim Haley } 2262fa9e4066Sahrens 2263fa9e4066Sahrens /* ARGSUSED */ 2264fa9e4066Sahrens static void 2265c717a561Smaybee dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb) 2266fa9e4066Sahrens { 2267fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 2268fa9e4066Sahrens dnode_t *dn = db->db_dnode; 2269fa9e4066Sahrens objset_impl_t *os = dn->dn_objset; 2270e14bb325SJeff Bonwick blkptr_t *bp = zio->io_bp; 2271c717a561Smaybee blkptr_t *bp_orig = &zio->io_bp_orig; 2272fa9e4066Sahrens uint64_t fill = 0; 2273c717a561Smaybee int old_size, new_size, i; 2274fa9e4066Sahrens 2275e14bb325SJeff Bonwick ASSERT(db->db_blkptr == bp); 2276e14bb325SJeff Bonwick 2277c717a561Smaybee dprintf_dbuf_bp(db, bp_orig, "bp_orig: %s", ""); 2278fa9e4066Sahrens 2279c717a561Smaybee old_size = bp_get_dasize(os->os_spa, bp_orig); 2280e14bb325SJeff Bonwick new_size = bp_get_dasize(os->os_spa, bp); 2281fa9e4066Sahrens 2282fa9e4066Sahrens dnode_diduse_space(dn, new_size - old_size); 2283fa9e4066Sahrens 2284e14bb325SJeff Bonwick if (BP_IS_HOLE(bp)) { 2285c717a561Smaybee dsl_dataset_t *ds = os->os_dsl_dataset; 2286c717a561Smaybee dmu_tx_t *tx = os->os_synctx; 2287c717a561Smaybee 2288c717a561Smaybee if (bp_orig->blk_birth == tx->tx_txg) 2289e14bb325SJeff Bonwick (void) dsl_dataset_block_kill(ds, bp_orig, zio, tx); 2290e14bb325SJeff Bonwick ASSERT3U(bp->blk_fill, ==, 0); 2291c717a561Smaybee return; 2292c717a561Smaybee } 2293c717a561Smaybee 2294e14bb325SJeff Bonwick ASSERT(BP_GET_TYPE(bp) == dn->dn_type); 2295e14bb325SJeff Bonwick ASSERT(BP_GET_LEVEL(bp) == db->db_level); 2296e14bb325SJeff Bonwick 2297fa9e4066Sahrens mutex_enter(&db->db_mtx); 2298fa9e4066Sahrens 2299fa9e4066Sahrens if (db->db_level == 0) { 2300fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 2301c717a561Smaybee if (db->db_blkid > dn->dn_phys->dn_maxblkid) 2302fa9e4066Sahrens dn->dn_phys->dn_maxblkid = db->db_blkid; 2303fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 2304fa9e4066Sahrens 2305fa9e4066Sahrens if (dn->dn_type == DMU_OT_DNODE) { 2306fa9e4066Sahrens dnode_phys_t *dnp = db->db.db_data; 2307fa9e4066Sahrens for (i = db->db.db_size >> DNODE_SHIFT; i > 0; 2308fa9e4066Sahrens i--, dnp++) { 2309fa9e4066Sahrens if (dnp->dn_type != DMU_OT_NONE) 2310fa9e4066Sahrens fill++; 2311fa9e4066Sahrens } 2312fa9e4066Sahrens } else { 2313fa9e4066Sahrens fill = 1; 2314fa9e4066Sahrens } 2315fa9e4066Sahrens } else { 2316e14bb325SJeff Bonwick blkptr_t *ibp = db->db.db_data; 2317fa9e4066Sahrens ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2318e14bb325SJeff Bonwick for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, ibp++) { 2319e14bb325SJeff Bonwick if (BP_IS_HOLE(ibp)) 2320fa9e4066Sahrens continue; 2321e14bb325SJeff Bonwick ASSERT3U(BP_GET_LSIZE(ibp), ==, 2322fa9e4066Sahrens db->db_level == 1 ? dn->dn_datablksz : 2323fa9e4066Sahrens (1<<dn->dn_phys->dn_indblkshift)); 2324e14bb325SJeff Bonwick fill += ibp->blk_fill; 2325fa9e4066Sahrens } 2326fa9e4066Sahrens } 2327fa9e4066Sahrens 2328e14bb325SJeff Bonwick bp->blk_fill = fill; 2329fa9e4066Sahrens 2330fa9e4066Sahrens mutex_exit(&db->db_mtx); 2331fa9e4066Sahrens 2332e14bb325SJeff Bonwick if (zio->io_flags & ZIO_FLAG_IO_REWRITE) { 2333e14bb325SJeff Bonwick ASSERT(DVA_EQUAL(BP_IDENTITY(bp), BP_IDENTITY(bp_orig))); 2334e14bb325SJeff Bonwick } else { 2335c717a561Smaybee dsl_dataset_t *ds = os->os_dsl_dataset; 2336c717a561Smaybee dmu_tx_t *tx = os->os_synctx; 2337c717a561Smaybee 2338c717a561Smaybee if (bp_orig->blk_birth == tx->tx_txg) 2339e14bb325SJeff Bonwick (void) dsl_dataset_block_kill(ds, bp_orig, zio, tx); 2340e14bb325SJeff Bonwick dsl_dataset_block_born(ds, bp, tx); 2341fa9e4066Sahrens } 2342fa9e4066Sahrens } 2343fa9e4066Sahrens 2344c717a561Smaybee /* ARGSUSED */ 2345c717a561Smaybee static void 2346c717a561Smaybee dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb) 2347c717a561Smaybee { 2348c717a561Smaybee dmu_buf_impl_t *db = vdb; 2349c717a561Smaybee uint64_t txg = zio->io_txg; 2350c717a561Smaybee dbuf_dirty_record_t **drp, *dr; 2351c717a561Smaybee 2352c717a561Smaybee ASSERT3U(zio->io_error, ==, 0); 2353c717a561Smaybee 2354c717a561Smaybee mutex_enter(&db->db_mtx); 2355c717a561Smaybee 2356c717a561Smaybee drp = &db->db_last_dirty; 235717f17c2dSbonwick while ((dr = *drp) != db->db_data_pending) 235817f17c2dSbonwick drp = &dr->dr_next; 235917f17c2dSbonwick ASSERT(!list_link_active(&dr->dr_dirty_node)); 236017f17c2dSbonwick ASSERT(dr->dr_txg == txg); 236117f17c2dSbonwick ASSERT(dr->dr_next == NULL); 236217f17c2dSbonwick *drp = dr->dr_next; 2363c717a561Smaybee 2364c717a561Smaybee if (db->db_level == 0) { 2365c717a561Smaybee ASSERT(db->db_blkid != DB_BONUS_BLKID); 2366c717a561Smaybee ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN); 2367c717a561Smaybee 236882c9918fSTim Haley if (db->db_state != DB_NOFILL) { 2369c717a561Smaybee if (dr->dt.dl.dr_data != db->db_buf) 237082c9918fSTim Haley VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, 237182c9918fSTim Haley db) == 1); 2372c717a561Smaybee else if (!BP_IS_HOLE(db->db_blkptr)) 2373c717a561Smaybee arc_set_callback(db->db_buf, dbuf_do_evict, db); 2374c717a561Smaybee else 2375c717a561Smaybee ASSERT(arc_released(db->db_buf)); 237682c9918fSTim Haley } 2377c717a561Smaybee } else { 2378c717a561Smaybee dnode_t *dn = db->db_dnode; 2379c717a561Smaybee 2380c717a561Smaybee ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 2381c717a561Smaybee ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2382c717a561Smaybee if (!BP_IS_HOLE(db->db_blkptr)) { 2383c717a561Smaybee int epbs = 2384c717a561Smaybee dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2385c717a561Smaybee ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, 2386c717a561Smaybee db->db.db_size); 2387c717a561Smaybee ASSERT3U(dn->dn_phys->dn_maxblkid 2388c717a561Smaybee >> (db->db_level * epbs), >=, db->db_blkid); 2389c717a561Smaybee arc_set_callback(db->db_buf, dbuf_do_evict, db); 2390c717a561Smaybee } 2391c25056deSgw25295 mutex_destroy(&dr->dt.di.dr_mtx); 2392c25056deSgw25295 list_destroy(&dr->dt.di.dr_children); 2393c717a561Smaybee } 2394c717a561Smaybee kmem_free(dr, sizeof (dbuf_dirty_record_t)); 2395c717a561Smaybee 2396c717a561Smaybee cv_broadcast(&db->db_changed); 2397c717a561Smaybee ASSERT(db->db_dirtycnt > 0); 2398c717a561Smaybee db->db_dirtycnt -= 1; 2399c717a561Smaybee db->db_data_pending = NULL; 2400c717a561Smaybee mutex_exit(&db->db_mtx); 2401c717a561Smaybee 2402c717a561Smaybee dprintf_dbuf_bp(db, zio->io_bp, "bp: %s", ""); 2403c717a561Smaybee 2404ea8dc4b6Seschrock dbuf_rele(db, (void *)(uintptr_t)txg); 2405fa9e4066Sahrens } 2406