1fa9e4066Sahrens /* 2fa9e4066Sahrens * CDDL HEADER START 3fa9e4066Sahrens * 4fa9e4066Sahrens * The contents of this file are subject to the terms of the 5f65e61c0Sahrens * Common Development and Distribution License (the "License"). 6f65e61c0Sahrens * You may not use this file except in compliance with the License. 7fa9e4066Sahrens * 8fa9e4066Sahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9fa9e4066Sahrens * or http://www.opensolaris.org/os/licensing. 10fa9e4066Sahrens * See the License for the specific language governing permissions 11fa9e4066Sahrens * and limitations under the License. 12fa9e4066Sahrens * 13fa9e4066Sahrens * When distributing Covered Code, include this CDDL HEADER in each 14fa9e4066Sahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15fa9e4066Sahrens * If applicable, add the following below this CDDL HEADER, with the 16fa9e4066Sahrens * fields enclosed by brackets "[]" replaced with your own identifying 17fa9e4066Sahrens * information: Portions Copyright [yyyy] [name of copyright owner] 18fa9e4066Sahrens * 19fa9e4066Sahrens * CDDL HEADER END 20fa9e4066Sahrens */ 21fa9e4066Sahrens /* 22c717a561Smaybee * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23fa9e4066Sahrens * Use is subject to license terms. 24fa9e4066Sahrens */ 25fa9e4066Sahrens 26fa9e4066Sahrens #pragma ident "%Z%%M% %I% %E% SMI" 27fa9e4066Sahrens 28fa9e4066Sahrens #include <sys/zfs_context.h> 29fa9e4066Sahrens #include <sys/dmu.h> 30fa9e4066Sahrens #include <sys/dmu_impl.h> 31fa9e4066Sahrens #include <sys/dbuf.h> 32fa9e4066Sahrens #include <sys/dmu_objset.h> 33fa9e4066Sahrens #include <sys/dsl_dataset.h> 34fa9e4066Sahrens #include <sys/dsl_dir.h> 35fa9e4066Sahrens #include <sys/dmu_tx.h> 36fa9e4066Sahrens #include <sys/spa.h> 37fa9e4066Sahrens #include <sys/zio.h> 38fa9e4066Sahrens #include <sys/dmu_zfetch.h> 39fa9e4066Sahrens 40fa9e4066Sahrens static void dbuf_destroy(dmu_buf_impl_t *db); 41fa9e4066Sahrens static int dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx); 42c717a561Smaybee static void dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, int checksum, 43c717a561Smaybee int compress, dmu_tx_t *tx); 44c717a561Smaybee static arc_done_func_t dbuf_write_ready; 45fa9e4066Sahrens static arc_done_func_t dbuf_write_done; 46fa9e4066Sahrens 47416e0cd8Sek110237 int zfs_mdcomp_disable = 0; 48416e0cd8Sek110237 49fa9e4066Sahrens /* 50fa9e4066Sahrens * Global data structures and functions for the dbuf cache. 51fa9e4066Sahrens */ 52fa9e4066Sahrens static kmem_cache_t *dbuf_cache; 53fa9e4066Sahrens 54fa9e4066Sahrens /* ARGSUSED */ 55fa9e4066Sahrens static int 56fa9e4066Sahrens dbuf_cons(void *vdb, void *unused, int kmflag) 57fa9e4066Sahrens { 58fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 59fa9e4066Sahrens bzero(db, sizeof (dmu_buf_impl_t)); 60fa9e4066Sahrens 61fa9e4066Sahrens mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL); 62fa9e4066Sahrens cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL); 63fa9e4066Sahrens refcount_create(&db->db_holds); 64fa9e4066Sahrens return (0); 65fa9e4066Sahrens } 66fa9e4066Sahrens 67fa9e4066Sahrens /* ARGSUSED */ 68fa9e4066Sahrens static void 69fa9e4066Sahrens dbuf_dest(void *vdb, void *unused) 70fa9e4066Sahrens { 71fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 72fa9e4066Sahrens mutex_destroy(&db->db_mtx); 73fa9e4066Sahrens cv_destroy(&db->db_changed); 74fa9e4066Sahrens refcount_destroy(&db->db_holds); 75fa9e4066Sahrens } 76fa9e4066Sahrens 77fa9e4066Sahrens /* 78fa9e4066Sahrens * dbuf hash table routines 79fa9e4066Sahrens */ 80fa9e4066Sahrens static dbuf_hash_table_t dbuf_hash_table; 81fa9e4066Sahrens 82fa9e4066Sahrens static uint64_t dbuf_hash_count; 83fa9e4066Sahrens 84fa9e4066Sahrens static uint64_t 85fa9e4066Sahrens dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid) 86fa9e4066Sahrens { 87fa9e4066Sahrens uintptr_t osv = (uintptr_t)os; 88fa9e4066Sahrens uint64_t crc = -1ULL; 89fa9e4066Sahrens 90fa9e4066Sahrens ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 91fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (lvl)) & 0xFF]; 92fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF]; 93fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF]; 94fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF]; 95fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 0)) & 0xFF]; 96fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 8)) & 0xFF]; 97fa9e4066Sahrens 98fa9e4066Sahrens crc ^= (osv>>14) ^ (obj>>16) ^ (blkid>>16); 99fa9e4066Sahrens 100fa9e4066Sahrens return (crc); 101fa9e4066Sahrens } 102fa9e4066Sahrens 103fa9e4066Sahrens #define DBUF_HASH(os, obj, level, blkid) dbuf_hash(os, obj, level, blkid); 104fa9e4066Sahrens 105fa9e4066Sahrens #define DBUF_EQUAL(dbuf, os, obj, level, blkid) \ 106fa9e4066Sahrens ((dbuf)->db.db_object == (obj) && \ 107fa9e4066Sahrens (dbuf)->db_objset == (os) && \ 108fa9e4066Sahrens (dbuf)->db_level == (level) && \ 109fa9e4066Sahrens (dbuf)->db_blkid == (blkid)) 110fa9e4066Sahrens 111fa9e4066Sahrens dmu_buf_impl_t * 112fa9e4066Sahrens dbuf_find(dnode_t *dn, uint8_t level, uint64_t blkid) 113fa9e4066Sahrens { 114fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 115fa9e4066Sahrens objset_impl_t *os = dn->dn_objset; 116fa9e4066Sahrens uint64_t obj = dn->dn_object; 117fa9e4066Sahrens uint64_t hv = DBUF_HASH(os, obj, level, blkid); 118fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 119fa9e4066Sahrens dmu_buf_impl_t *db; 120fa9e4066Sahrens 121fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 122fa9e4066Sahrens for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) { 123fa9e4066Sahrens if (DBUF_EQUAL(db, os, obj, level, blkid)) { 124fa9e4066Sahrens mutex_enter(&db->db_mtx); 125ea8dc4b6Seschrock if (db->db_state != DB_EVICTING) { 126fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 127fa9e4066Sahrens return (db); 128fa9e4066Sahrens } 129fa9e4066Sahrens mutex_exit(&db->db_mtx); 130fa9e4066Sahrens } 131fa9e4066Sahrens } 132fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 133fa9e4066Sahrens return (NULL); 134fa9e4066Sahrens } 135fa9e4066Sahrens 136fa9e4066Sahrens /* 137fa9e4066Sahrens * Insert an entry into the hash table. If there is already an element 138fa9e4066Sahrens * equal to elem in the hash table, then the already existing element 139fa9e4066Sahrens * will be returned and the new element will not be inserted. 140fa9e4066Sahrens * Otherwise returns NULL. 141fa9e4066Sahrens */ 142fa9e4066Sahrens static dmu_buf_impl_t * 143fa9e4066Sahrens dbuf_hash_insert(dmu_buf_impl_t *db) 144fa9e4066Sahrens { 145fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 146fa9e4066Sahrens objset_impl_t *os = db->db_objset; 147fa9e4066Sahrens uint64_t obj = db->db.db_object; 148fa9e4066Sahrens int level = db->db_level; 149fa9e4066Sahrens uint64_t blkid = db->db_blkid; 150fa9e4066Sahrens uint64_t hv = DBUF_HASH(os, obj, level, blkid); 151fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 152fa9e4066Sahrens dmu_buf_impl_t *dbf; 153fa9e4066Sahrens 154fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 155fa9e4066Sahrens for (dbf = h->hash_table[idx]; dbf != NULL; dbf = dbf->db_hash_next) { 156fa9e4066Sahrens if (DBUF_EQUAL(dbf, os, obj, level, blkid)) { 157fa9e4066Sahrens mutex_enter(&dbf->db_mtx); 158ea8dc4b6Seschrock if (dbf->db_state != DB_EVICTING) { 159fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 160fa9e4066Sahrens return (dbf); 161fa9e4066Sahrens } 162fa9e4066Sahrens mutex_exit(&dbf->db_mtx); 163fa9e4066Sahrens } 164fa9e4066Sahrens } 165fa9e4066Sahrens 166fa9e4066Sahrens mutex_enter(&db->db_mtx); 167fa9e4066Sahrens db->db_hash_next = h->hash_table[idx]; 168fa9e4066Sahrens h->hash_table[idx] = db; 169fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 170fa9e4066Sahrens atomic_add_64(&dbuf_hash_count, 1); 171fa9e4066Sahrens 172fa9e4066Sahrens return (NULL); 173fa9e4066Sahrens } 174fa9e4066Sahrens 175fa9e4066Sahrens /* 176fa9e4066Sahrens * Remove an entry from the hash table. This operation will 177fa9e4066Sahrens * fail if there are any existing holds on the db. 178fa9e4066Sahrens */ 179fa9e4066Sahrens static void 180fa9e4066Sahrens dbuf_hash_remove(dmu_buf_impl_t *db) 181fa9e4066Sahrens { 182fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 183fa9e4066Sahrens uint64_t hv = DBUF_HASH(db->db_objset, db->db.db_object, 184fa9e4066Sahrens db->db_level, db->db_blkid); 185fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 186fa9e4066Sahrens dmu_buf_impl_t *dbf, **dbp; 187fa9e4066Sahrens 188fa9e4066Sahrens /* 189fa9e4066Sahrens * We musn't hold db_mtx to maintin lock ordering: 190fa9e4066Sahrens * DBUF_HASH_MUTEX > db_mtx. 191fa9e4066Sahrens */ 192fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 193ea8dc4b6Seschrock ASSERT(db->db_state == DB_EVICTING); 194fa9e4066Sahrens ASSERT(!MUTEX_HELD(&db->db_mtx)); 195fa9e4066Sahrens 196fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 197fa9e4066Sahrens dbp = &h->hash_table[idx]; 198fa9e4066Sahrens while ((dbf = *dbp) != db) { 199fa9e4066Sahrens dbp = &dbf->db_hash_next; 200fa9e4066Sahrens ASSERT(dbf != NULL); 201fa9e4066Sahrens } 202fa9e4066Sahrens *dbp = db->db_hash_next; 203fa9e4066Sahrens db->db_hash_next = NULL; 204fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 205fa9e4066Sahrens atomic_add_64(&dbuf_hash_count, -1); 206fa9e4066Sahrens } 207fa9e4066Sahrens 208ea8dc4b6Seschrock static arc_evict_func_t dbuf_do_evict; 209fa9e4066Sahrens 210fa9e4066Sahrens static void 211fa9e4066Sahrens dbuf_evict_user(dmu_buf_impl_t *db) 212fa9e4066Sahrens { 213fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 214fa9e4066Sahrens 215c717a561Smaybee if (db->db_level != 0 || db->db_evict_func == NULL) 216fa9e4066Sahrens return; 217fa9e4066Sahrens 218c717a561Smaybee if (db->db_user_data_ptr_ptr) 219c717a561Smaybee *db->db_user_data_ptr_ptr = db->db.db_data; 220c717a561Smaybee db->db_evict_func(&db->db, db->db_user_ptr); 221c717a561Smaybee db->db_user_ptr = NULL; 222c717a561Smaybee db->db_user_data_ptr_ptr = NULL; 223c717a561Smaybee db->db_evict_func = NULL; 224fa9e4066Sahrens } 225fa9e4066Sahrens 226fa9e4066Sahrens void 227ea8dc4b6Seschrock dbuf_evict(dmu_buf_impl_t *db) 228ea8dc4b6Seschrock { 229ea8dc4b6Seschrock ASSERT(MUTEX_HELD(&db->db_mtx)); 230ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 231c717a561Smaybee ASSERT(db->db_data_pending == NULL); 232ea8dc4b6Seschrock 233ea8dc4b6Seschrock dbuf_clear(db); 234ea8dc4b6Seschrock dbuf_destroy(db); 235ea8dc4b6Seschrock } 236ea8dc4b6Seschrock 237ea8dc4b6Seschrock void 238fa9e4066Sahrens dbuf_init(void) 239fa9e4066Sahrens { 240ea8dc4b6Seschrock uint64_t hsize = 1ULL << 16; 241fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 242fa9e4066Sahrens int i; 243fa9e4066Sahrens 244fa9e4066Sahrens /* 245fa9e4066Sahrens * The hash table is big enough to fill all of physical memory 246ea8dc4b6Seschrock * with an average 4K block size. The table will take up 247ea8dc4b6Seschrock * totalmem*sizeof(void*)/4K (i.e. 2MB/GB with 8-byte pointers). 248fa9e4066Sahrens */ 249ea8dc4b6Seschrock while (hsize * 4096 < physmem * PAGESIZE) 250fa9e4066Sahrens hsize <<= 1; 251fa9e4066Sahrens 252ea8dc4b6Seschrock retry: 253fa9e4066Sahrens h->hash_table_mask = hsize - 1; 254ea8dc4b6Seschrock h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP); 255ea8dc4b6Seschrock if (h->hash_table == NULL) { 256ea8dc4b6Seschrock /* XXX - we should really return an error instead of assert */ 257ea8dc4b6Seschrock ASSERT(hsize > (1ULL << 10)); 258ea8dc4b6Seschrock hsize >>= 1; 259ea8dc4b6Seschrock goto retry; 260ea8dc4b6Seschrock } 261fa9e4066Sahrens 262fa9e4066Sahrens dbuf_cache = kmem_cache_create("dmu_buf_impl_t", 263fa9e4066Sahrens sizeof (dmu_buf_impl_t), 264fa9e4066Sahrens 0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0); 265fa9e4066Sahrens 266fa9e4066Sahrens for (i = 0; i < DBUF_MUTEXES; i++) 267fa9e4066Sahrens mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL); 268fa9e4066Sahrens } 269fa9e4066Sahrens 270fa9e4066Sahrens void 271fa9e4066Sahrens dbuf_fini(void) 272fa9e4066Sahrens { 273fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 274fa9e4066Sahrens int i; 275fa9e4066Sahrens 276fa9e4066Sahrens for (i = 0; i < DBUF_MUTEXES; i++) 277fa9e4066Sahrens mutex_destroy(&h->hash_mutexes[i]); 278fa9e4066Sahrens kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *)); 279fa9e4066Sahrens kmem_cache_destroy(dbuf_cache); 280fa9e4066Sahrens } 281fa9e4066Sahrens 282fa9e4066Sahrens /* 283fa9e4066Sahrens * Other stuff. 284fa9e4066Sahrens */ 285fa9e4066Sahrens 2869c9dc39aSek110237 #ifdef ZFS_DEBUG 287fa9e4066Sahrens static void 288fa9e4066Sahrens dbuf_verify(dmu_buf_impl_t *db) 289fa9e4066Sahrens { 290fa9e4066Sahrens dnode_t *dn = db->db_dnode; 291fa9e4066Sahrens 292fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 293fa9e4066Sahrens 294fa9e4066Sahrens if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY)) 295fa9e4066Sahrens return; 296fa9e4066Sahrens 297fa9e4066Sahrens ASSERT(db->db_objset != NULL); 298fa9e4066Sahrens if (dn == NULL) { 299fa9e4066Sahrens ASSERT(db->db_parent == NULL); 300fa9e4066Sahrens ASSERT(db->db_blkptr == NULL); 301fa9e4066Sahrens } else { 302fa9e4066Sahrens ASSERT3U(db->db.db_object, ==, dn->dn_object); 303fa9e4066Sahrens ASSERT3P(db->db_objset, ==, dn->dn_objset); 304fa9e4066Sahrens ASSERT3U(db->db_level, <, dn->dn_nlevels); 305ea8dc4b6Seschrock ASSERT(db->db_blkid == DB_BONUS_BLKID || 306ea8dc4b6Seschrock list_head(&dn->dn_dbufs)); 307fa9e4066Sahrens } 308fa9e4066Sahrens if (db->db_blkid == DB_BONUS_BLKID) { 309fa9e4066Sahrens ASSERT(dn != NULL); 310fa9e4066Sahrens ASSERT3U(db->db.db_size, ==, dn->dn_bonuslen); 311fa9e4066Sahrens ASSERT3U(db->db.db_offset, ==, DB_BONUS_BLKID); 312fa9e4066Sahrens } else { 313fa9e4066Sahrens ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size); 314fa9e4066Sahrens } 315fa9e4066Sahrens 316fa9e4066Sahrens if (db->db_level == 0) { 317fa9e4066Sahrens /* we can be momentarily larger in dnode_set_blksz() */ 318fa9e4066Sahrens if (db->db_blkid != DB_BONUS_BLKID && dn) { 319fa9e4066Sahrens ASSERT3U(db->db.db_size, >=, dn->dn_datablksz); 320fa9e4066Sahrens } 321ea8dc4b6Seschrock if (db->db.db_object == DMU_META_DNODE_OBJECT) { 322c717a561Smaybee dbuf_dirty_record_t *dr = db->db_data_pending; 323fa9e4066Sahrens /* 324fa9e4066Sahrens * it should only be modified in syncing 325fa9e4066Sahrens * context, so make sure we only have 326fa9e4066Sahrens * one copy of the data. 327fa9e4066Sahrens */ 328c717a561Smaybee ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf); 329fa9e4066Sahrens } 330fa9e4066Sahrens } 331fa9e4066Sahrens 332fa9e4066Sahrens /* verify db->db_blkptr */ 333fa9e4066Sahrens if (db->db_blkptr) { 334fa9e4066Sahrens if (db->db_parent == dn->dn_dbuf) { 335fa9e4066Sahrens /* db is pointed to by the dnode */ 336fa9e4066Sahrens /* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */ 337ea8dc4b6Seschrock if (db->db.db_object == DMU_META_DNODE_OBJECT) 338fa9e4066Sahrens ASSERT(db->db_parent == NULL); 339fa9e4066Sahrens else 340fa9e4066Sahrens ASSERT(db->db_parent != NULL); 341fa9e4066Sahrens ASSERT3P(db->db_blkptr, ==, 342fa9e4066Sahrens &dn->dn_phys->dn_blkptr[db->db_blkid]); 343fa9e4066Sahrens } else { 344fa9e4066Sahrens /* db is pointed to by an indirect block */ 345fa9e4066Sahrens int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT; 346fa9e4066Sahrens ASSERT3U(db->db_parent->db_level, ==, db->db_level+1); 347fa9e4066Sahrens ASSERT3U(db->db_parent->db.db_object, ==, 348fa9e4066Sahrens db->db.db_object); 349fa9e4066Sahrens /* 350fa9e4066Sahrens * dnode_grow_indblksz() can make this fail if we don't 351fa9e4066Sahrens * have the struct_rwlock. XXX indblksz no longer 352fa9e4066Sahrens * grows. safe to do this now? 353fa9e4066Sahrens */ 354fa9e4066Sahrens if (RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)) { 355fa9e4066Sahrens ASSERT3P(db->db_blkptr, ==, 356fa9e4066Sahrens ((blkptr_t *)db->db_parent->db.db_data + 357fa9e4066Sahrens db->db_blkid % epb)); 358fa9e4066Sahrens } 359fa9e4066Sahrens } 360fa9e4066Sahrens } 361fa9e4066Sahrens if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) && 362fa9e4066Sahrens db->db.db_data && db->db_blkid != DB_BONUS_BLKID && 363fa9e4066Sahrens db->db_state != DB_FILL && !dn->dn_free_txg) { 364fa9e4066Sahrens /* 365fa9e4066Sahrens * If the blkptr isn't set but they have nonzero data, 366fa9e4066Sahrens * it had better be dirty, otherwise we'll lose that 367fa9e4066Sahrens * data when we evict this buffer. 368fa9e4066Sahrens */ 369fa9e4066Sahrens if (db->db_dirtycnt == 0) { 370fa9e4066Sahrens uint64_t *buf = db->db.db_data; 371fa9e4066Sahrens int i; 372fa9e4066Sahrens 373fa9e4066Sahrens for (i = 0; i < db->db.db_size >> 3; i++) { 374fa9e4066Sahrens ASSERT(buf[i] == 0); 375fa9e4066Sahrens } 376fa9e4066Sahrens } 377fa9e4066Sahrens } 378fa9e4066Sahrens } 3799c9dc39aSek110237 #endif 380fa9e4066Sahrens 381fa9e4066Sahrens static void 382fa9e4066Sahrens dbuf_update_data(dmu_buf_impl_t *db) 383fa9e4066Sahrens { 384fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 385c717a561Smaybee if (db->db_level == 0 && db->db_user_data_ptr_ptr) { 386fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 387c717a561Smaybee *db->db_user_data_ptr_ptr = db->db.db_data; 388fa9e4066Sahrens } 389fa9e4066Sahrens } 390fa9e4066Sahrens 391fa9e4066Sahrens static void 392fa9e4066Sahrens dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf) 393fa9e4066Sahrens { 394fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 395ea8dc4b6Seschrock ASSERT(db->db_buf == NULL || !arc_has_callback(db->db_buf)); 396fa9e4066Sahrens db->db_buf = buf; 397ea8dc4b6Seschrock if (buf != NULL) { 398ea8dc4b6Seschrock ASSERT(buf->b_data != NULL); 399fa9e4066Sahrens db->db.db_data = buf->b_data; 400ea8dc4b6Seschrock if (!arc_released(buf)) 401ea8dc4b6Seschrock arc_set_callback(buf, dbuf_do_evict, db); 402fa9e4066Sahrens dbuf_update_data(db); 403ea8dc4b6Seschrock } else { 404ea8dc4b6Seschrock dbuf_evict_user(db); 405ea8dc4b6Seschrock db->db.db_data = NULL; 406ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 407ea8dc4b6Seschrock } 408fa9e4066Sahrens } 409fa9e4066Sahrens 410fa9e4066Sahrens uint64_t 411fa9e4066Sahrens dbuf_whichblock(dnode_t *dn, uint64_t offset) 412fa9e4066Sahrens { 413fa9e4066Sahrens if (dn->dn_datablkshift) { 414fa9e4066Sahrens return (offset >> dn->dn_datablkshift); 415fa9e4066Sahrens } else { 416fa9e4066Sahrens ASSERT3U(offset, <, dn->dn_datablksz); 417fa9e4066Sahrens return (0); 418fa9e4066Sahrens } 419fa9e4066Sahrens } 420fa9e4066Sahrens 421fa9e4066Sahrens static void 422fa9e4066Sahrens dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb) 423fa9e4066Sahrens { 424fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 425fa9e4066Sahrens 426fa9e4066Sahrens mutex_enter(&db->db_mtx); 427fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_READ); 428fa9e4066Sahrens /* 429fa9e4066Sahrens * All reads are synchronous, so we must have a hold on the dbuf 430fa9e4066Sahrens */ 431fa9e4066Sahrens ASSERT(refcount_count(&db->db_holds) > 0); 432ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 433fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 434c717a561Smaybee if (db->db_level == 0 && db->db_freed_in_flight) { 435fa9e4066Sahrens /* we were freed in flight; disregard any error */ 436fa9e4066Sahrens arc_release(buf, db); 437fa9e4066Sahrens bzero(buf->b_data, db->db.db_size); 4386b4acc8bSahrens arc_buf_freeze(buf); 439c717a561Smaybee db->db_freed_in_flight = FALSE; 440fa9e4066Sahrens dbuf_set_data(db, buf); 441fa9e4066Sahrens db->db_state = DB_CACHED; 442fa9e4066Sahrens } else if (zio == NULL || zio->io_error == 0) { 443fa9e4066Sahrens dbuf_set_data(db, buf); 444fa9e4066Sahrens db->db_state = DB_CACHED; 445fa9e4066Sahrens } else { 446fa9e4066Sahrens ASSERT(db->db_blkid != DB_BONUS_BLKID); 447fa9e4066Sahrens ASSERT3P(db->db_buf, ==, NULL); 448ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(buf, db) == 1); 449ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 450fa9e4066Sahrens } 451fa9e4066Sahrens cv_broadcast(&db->db_changed); 452fa9e4066Sahrens mutex_exit(&db->db_mtx); 453ea8dc4b6Seschrock dbuf_rele(db, NULL); 454fa9e4066Sahrens } 455fa9e4066Sahrens 456ea8dc4b6Seschrock static void 45713506d1eSmaybee dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t *flags) 458fa9e4066Sahrens { 459fa9e4066Sahrens blkptr_t *bp; 460ea8dc4b6Seschrock zbookmark_t zb; 46113506d1eSmaybee uint32_t aflags = ARC_NOWAIT; 462fa9e4066Sahrens 463fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 464fa9e4066Sahrens /* We need the struct_rwlock to prevent db_blkptr from changing. */ 465fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&db->db_dnode->dn_struct_rwlock)); 466ea8dc4b6Seschrock ASSERT(MUTEX_HELD(&db->db_mtx)); 467ea8dc4b6Seschrock ASSERT(db->db_state == DB_UNCACHED); 468ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 469fa9e4066Sahrens 470fa9e4066Sahrens if (db->db_blkid == DB_BONUS_BLKID) { 471fa9e4066Sahrens ASSERT3U(db->db_dnode->dn_bonuslen, ==, db->db.db_size); 472ea8dc4b6Seschrock db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN); 473fa9e4066Sahrens if (db->db.db_size < DN_MAX_BONUSLEN) 474ea8dc4b6Seschrock bzero(db->db.db_data, DN_MAX_BONUSLEN); 475ea8dc4b6Seschrock bcopy(DN_BONUS(db->db_dnode->dn_phys), db->db.db_data, 476fa9e4066Sahrens db->db.db_size); 477ea8dc4b6Seschrock dbuf_update_data(db); 478fa9e4066Sahrens db->db_state = DB_CACHED; 479fa9e4066Sahrens mutex_exit(&db->db_mtx); 480fa9e4066Sahrens return; 481fa9e4066Sahrens } 482fa9e4066Sahrens 483fa9e4066Sahrens if (db->db_level == 0 && dnode_block_freed(db->db_dnode, db->db_blkid)) 484fa9e4066Sahrens bp = NULL; 485fa9e4066Sahrens else 486fa9e4066Sahrens bp = db->db_blkptr; 487fa9e4066Sahrens 488fa9e4066Sahrens if (bp == NULL) 489fa9e4066Sahrens dprintf_dbuf(db, "blkptr: %s\n", "NULL"); 490fa9e4066Sahrens else 491fa9e4066Sahrens dprintf_dbuf_bp(db, bp, "%s", "blkptr:"); 492fa9e4066Sahrens 493fa9e4066Sahrens if (bp == NULL || BP_IS_HOLE(bp)) { 494ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 495ad23a2dbSjohansen 496fa9e4066Sahrens ASSERT(bp == NULL || BP_IS_HOLE(bp)); 497fa9e4066Sahrens dbuf_set_data(db, arc_buf_alloc(db->db_dnode->dn_objset->os_spa, 498ad23a2dbSjohansen db->db.db_size, db, type)); 499fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 500fa9e4066Sahrens db->db_state = DB_CACHED; 50113506d1eSmaybee *flags |= DB_RF_CACHED; 502fa9e4066Sahrens mutex_exit(&db->db_mtx); 503fa9e4066Sahrens return; 504fa9e4066Sahrens } 505fa9e4066Sahrens 506fa9e4066Sahrens db->db_state = DB_READ; 507fa9e4066Sahrens mutex_exit(&db->db_mtx); 508fa9e4066Sahrens 509ea8dc4b6Seschrock zb.zb_objset = db->db_objset->os_dsl_dataset ? 510ea8dc4b6Seschrock db->db_objset->os_dsl_dataset->ds_object : 0; 511ea8dc4b6Seschrock zb.zb_object = db->db.db_object; 512ea8dc4b6Seschrock zb.zb_level = db->db_level; 513ea8dc4b6Seschrock zb.zb_blkid = db->db_blkid; 514ea8dc4b6Seschrock 515ea8dc4b6Seschrock dbuf_add_ref(db, NULL); 516fa9e4066Sahrens /* ZIO_FLAG_CANFAIL callers have to check the parent zio's error */ 517fa9e4066Sahrens (void) arc_read(zio, db->db_dnode->dn_objset->os_spa, bp, 518fa9e4066Sahrens db->db_level > 0 ? byteswap_uint64_array : 519fa9e4066Sahrens dmu_ot[db->db_dnode->dn_type].ot_byteswap, 520fa9e4066Sahrens dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ, 52113506d1eSmaybee (*flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED, 52213506d1eSmaybee &aflags, &zb); 52313506d1eSmaybee if (aflags & ARC_CACHED) 52413506d1eSmaybee *flags |= DB_RF_CACHED; 525fa9e4066Sahrens } 526fa9e4066Sahrens 527ea8dc4b6Seschrock int 528ea8dc4b6Seschrock dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags) 529fa9e4066Sahrens { 530ea8dc4b6Seschrock int err = 0; 531ea8dc4b6Seschrock int havepzio = (zio != NULL); 53213506d1eSmaybee int prefetch; 533fa9e4066Sahrens 534fa9e4066Sahrens /* 535fa9e4066Sahrens * We don't have to hold the mutex to check db_state because it 536fa9e4066Sahrens * can't be freed while we have a hold on the buffer. 537fa9e4066Sahrens */ 538fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 539fa9e4066Sahrens 540fa9e4066Sahrens if ((flags & DB_RF_HAVESTRUCT) == 0) 541fa9e4066Sahrens rw_enter(&db->db_dnode->dn_struct_rwlock, RW_READER); 542fa9e4066Sahrens 54313506d1eSmaybee prefetch = db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID && 54413506d1eSmaybee (flags & DB_RF_NOPREFETCH) == 0 && db->db_dnode != NULL; 54513506d1eSmaybee 546fa9e4066Sahrens mutex_enter(&db->db_mtx); 547ea8dc4b6Seschrock if (db->db_state == DB_CACHED) { 548ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 54913506d1eSmaybee if (prefetch) 55013506d1eSmaybee dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset, 55113506d1eSmaybee db->db.db_size, TRUE); 552ea8dc4b6Seschrock if ((flags & DB_RF_HAVESTRUCT) == 0) 553ea8dc4b6Seschrock rw_exit(&db->db_dnode->dn_struct_rwlock); 554ea8dc4b6Seschrock } else if (db->db_state == DB_UNCACHED) { 555ea8dc4b6Seschrock if (zio == NULL) { 556ea8dc4b6Seschrock zio = zio_root(db->db_dnode->dn_objset->os_spa, 557ea8dc4b6Seschrock NULL, NULL, ZIO_FLAG_CANFAIL); 558ea8dc4b6Seschrock } 55913506d1eSmaybee dbuf_read_impl(db, zio, &flags); 56013506d1eSmaybee 561ea8dc4b6Seschrock /* dbuf_read_impl has dropped db_mtx for us */ 562ea8dc4b6Seschrock 56313506d1eSmaybee if (prefetch) 564ea8dc4b6Seschrock dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset, 56513506d1eSmaybee db->db.db_size, flags & DB_RF_CACHED); 566ea8dc4b6Seschrock 567ea8dc4b6Seschrock if ((flags & DB_RF_HAVESTRUCT) == 0) 568ea8dc4b6Seschrock rw_exit(&db->db_dnode->dn_struct_rwlock); 569ea8dc4b6Seschrock 570ea8dc4b6Seschrock if (!havepzio) 571ea8dc4b6Seschrock err = zio_wait(zio); 572ea8dc4b6Seschrock } else { 57313506d1eSmaybee mutex_exit(&db->db_mtx); 57413506d1eSmaybee if (prefetch) 57513506d1eSmaybee dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset, 57613506d1eSmaybee db->db.db_size, TRUE); 577ea8dc4b6Seschrock if ((flags & DB_RF_HAVESTRUCT) == 0) 578ea8dc4b6Seschrock rw_exit(&db->db_dnode->dn_struct_rwlock); 57913506d1eSmaybee 58013506d1eSmaybee mutex_enter(&db->db_mtx); 581ea8dc4b6Seschrock if ((flags & DB_RF_NEVERWAIT) == 0) { 582ea8dc4b6Seschrock while (db->db_state == DB_READ || 583ea8dc4b6Seschrock db->db_state == DB_FILL) { 584fa9e4066Sahrens ASSERT(db->db_state == DB_READ || 585fa9e4066Sahrens (flags & DB_RF_HAVESTRUCT) == 0); 586fa9e4066Sahrens cv_wait(&db->db_changed, &db->db_mtx); 587fa9e4066Sahrens } 588ea8dc4b6Seschrock if (db->db_state == DB_UNCACHED) 589ea8dc4b6Seschrock err = EIO; 590ea8dc4b6Seschrock } 591fa9e4066Sahrens mutex_exit(&db->db_mtx); 592fa9e4066Sahrens } 593fa9e4066Sahrens 594ea8dc4b6Seschrock ASSERT(err || havepzio || db->db_state == DB_CACHED); 595ea8dc4b6Seschrock return (err); 596fa9e4066Sahrens } 597fa9e4066Sahrens 598fa9e4066Sahrens static void 599fa9e4066Sahrens dbuf_noread(dmu_buf_impl_t *db) 600fa9e4066Sahrens { 601fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 602ea8dc4b6Seschrock ASSERT(db->db_blkid != DB_BONUS_BLKID); 603fa9e4066Sahrens mutex_enter(&db->db_mtx); 604fa9e4066Sahrens while (db->db_state == DB_READ || db->db_state == DB_FILL) 605fa9e4066Sahrens cv_wait(&db->db_changed, &db->db_mtx); 606fa9e4066Sahrens if (db->db_state == DB_UNCACHED) { 607ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 608ad23a2dbSjohansen 609ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 610fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 611fa9e4066Sahrens dbuf_set_data(db, arc_buf_alloc(db->db_dnode->dn_objset->os_spa, 612ad23a2dbSjohansen db->db.db_size, db, type)); 613fa9e4066Sahrens db->db_state = DB_FILL; 614fa9e4066Sahrens } else { 615fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_CACHED); 616fa9e4066Sahrens } 617fa9e4066Sahrens mutex_exit(&db->db_mtx); 618fa9e4066Sahrens } 619fa9e4066Sahrens 620fa9e4066Sahrens /* 621fa9e4066Sahrens * This is our just-in-time copy function. It makes a copy of 622fa9e4066Sahrens * buffers, that have been modified in a previous transaction 623fa9e4066Sahrens * group, before we modify them in the current active group. 624fa9e4066Sahrens * 625fa9e4066Sahrens * This function is used in two places: when we are dirtying a 626fa9e4066Sahrens * buffer for the first time in a txg, and when we are freeing 627fa9e4066Sahrens * a range in a dnode that includes this buffer. 628fa9e4066Sahrens * 629fa9e4066Sahrens * Note that when we are called from dbuf_free_range() we do 630fa9e4066Sahrens * not put a hold on the buffer, we just traverse the active 631fa9e4066Sahrens * dbuf list for the dnode. 632fa9e4066Sahrens */ 633fa9e4066Sahrens static void 634fa9e4066Sahrens dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg) 635fa9e4066Sahrens { 636c717a561Smaybee dbuf_dirty_record_t *dr = db->db_last_dirty; 637fa9e4066Sahrens 638fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 639fa9e4066Sahrens ASSERT(db->db.db_data != NULL); 640c717a561Smaybee ASSERT(db->db_level == 0); 641c717a561Smaybee ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT); 642fa9e4066Sahrens 6434d31c452Smaybee if (dr == NULL || 6444d31c452Smaybee (dr->dt.dl.dr_data != 6454d31c452Smaybee ((db->db_blkid == DB_BONUS_BLKID) ? db->db.db_data : db->db_buf))) 646fa9e4066Sahrens return; 647fa9e4066Sahrens 648fa9e4066Sahrens /* 649c717a561Smaybee * If the last dirty record for this dbuf has not yet synced 650c717a561Smaybee * and its referencing the dbuf data, either: 651c717a561Smaybee * reset the reference to point to a new copy, 652c717a561Smaybee * or (if there a no active holders) 653c717a561Smaybee * just null out the current db_data pointer. 654fa9e4066Sahrens */ 655c717a561Smaybee ASSERT(dr->dr_txg >= txg - 2); 656c717a561Smaybee if (db->db_blkid == DB_BONUS_BLKID) { 657c717a561Smaybee /* Note that the data bufs here are zio_bufs */ 658c717a561Smaybee dr->dt.dl.dr_data = zio_buf_alloc(DN_MAX_BONUSLEN); 659c717a561Smaybee bcopy(db->db.db_data, dr->dt.dl.dr_data, DN_MAX_BONUSLEN); 660c717a561Smaybee } else if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 661ea8dc4b6Seschrock int size = db->db.db_size; 662c717a561Smaybee arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 663c717a561Smaybee dr->dt.dl.dr_data = arc_buf_alloc( 664ad23a2dbSjohansen db->db_dnode->dn_objset->os_spa, size, db, type); 665c717a561Smaybee bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size); 666fa9e4066Sahrens } else { 667ea8dc4b6Seschrock dbuf_set_data(db, NULL); 668fa9e4066Sahrens } 669fa9e4066Sahrens } 670ea8dc4b6Seschrock 671fa9e4066Sahrens void 672c717a561Smaybee dbuf_unoverride(dbuf_dirty_record_t *dr) 673fa9e4066Sahrens { 674c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 675c717a561Smaybee uint64_t txg = dr->dr_txg; 676c5c6ffa0Smaybee 677c717a561Smaybee ASSERT(MUTEX_HELD(&db->db_mtx)); 678c717a561Smaybee ASSERT(dr->dt.dl.dr_override_state != DR_IN_DMU_SYNC); 679c717a561Smaybee ASSERT(db->db_level == 0); 680c717a561Smaybee 681c717a561Smaybee if (db->db_blkid == DB_BONUS_BLKID || 682c717a561Smaybee dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN) 683c717a561Smaybee return; 684c717a561Smaybee 685fa9e4066Sahrens /* free this block */ 686c717a561Smaybee if (!BP_IS_HOLE(&dr->dt.dl.dr_overridden_by)) { 687fa9e4066Sahrens /* XXX can get silent EIO here */ 688fa9e4066Sahrens (void) arc_free(NULL, db->db_dnode->dn_objset->os_spa, 689c717a561Smaybee txg, &dr->dt.dl.dr_overridden_by, NULL, NULL, ARC_WAIT); 690fa9e4066Sahrens } 691c717a561Smaybee dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 6926b4acc8bSahrens /* 6936b4acc8bSahrens * Release the already-written buffer, so we leave it in 6946b4acc8bSahrens * a consistent dirty state. Note that all callers are 6956b4acc8bSahrens * modifying the buffer, so they will immediately do 6966b4acc8bSahrens * another (redundant) arc_release(). Therefore, leave 6976b4acc8bSahrens * the buf thawed to save the effort of freezing & 6986b4acc8bSahrens * immediately re-thawing it. 6996b4acc8bSahrens */ 700c717a561Smaybee arc_release(dr->dt.dl.dr_data, db); 701fa9e4066Sahrens } 702fa9e4066Sahrens 703fa9e4066Sahrens void 704fa9e4066Sahrens dbuf_free_range(dnode_t *dn, uint64_t blkid, uint64_t nblks, dmu_tx_t *tx) 705fa9e4066Sahrens { 706fa9e4066Sahrens dmu_buf_impl_t *db, *db_next; 707fa9e4066Sahrens uint64_t txg = tx->tx_txg; 708fa9e4066Sahrens 709fa9e4066Sahrens dprintf_dnode(dn, "blkid=%llu nblks=%llu\n", blkid, nblks); 710fa9e4066Sahrens mutex_enter(&dn->dn_dbufs_mtx); 711fa9e4066Sahrens for (db = list_head(&dn->dn_dbufs); db; db = db_next) { 712fa9e4066Sahrens db_next = list_next(&dn->dn_dbufs, db); 713ea8dc4b6Seschrock ASSERT(db->db_blkid != DB_BONUS_BLKID); 714ea8dc4b6Seschrock if (db->db_level != 0) 715fa9e4066Sahrens continue; 716fa9e4066Sahrens dprintf_dbuf(db, "found buf %s\n", ""); 717fa9e4066Sahrens if (db->db_blkid < blkid || 718fa9e4066Sahrens db->db_blkid >= blkid+nblks) 719fa9e4066Sahrens continue; 720fa9e4066Sahrens 721fa9e4066Sahrens /* found a level 0 buffer in the range */ 722fa9e4066Sahrens if (dbuf_undirty(db, tx)) 723fa9e4066Sahrens continue; 724fa9e4066Sahrens 725fa9e4066Sahrens mutex_enter(&db->db_mtx); 726ea8dc4b6Seschrock if (db->db_state == DB_UNCACHED || 727ea8dc4b6Seschrock db->db_state == DB_EVICTING) { 728fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 729fa9e4066Sahrens mutex_exit(&db->db_mtx); 730fa9e4066Sahrens continue; 731fa9e4066Sahrens } 732c543ec06Sahrens if (db->db_state == DB_READ || db->db_state == DB_FILL) { 733c543ec06Sahrens /* will be handled in dbuf_read_done or dbuf_rele */ 734c717a561Smaybee db->db_freed_in_flight = TRUE; 735fa9e4066Sahrens mutex_exit(&db->db_mtx); 736fa9e4066Sahrens continue; 737fa9e4066Sahrens } 738ea8dc4b6Seschrock if (refcount_count(&db->db_holds) == 0) { 739ea8dc4b6Seschrock ASSERT(db->db_buf); 740ea8dc4b6Seschrock dbuf_clear(db); 741ea8dc4b6Seschrock continue; 742ea8dc4b6Seschrock } 743c717a561Smaybee /* The dbuf is referenced */ 744fa9e4066Sahrens 745c717a561Smaybee if (db->db_last_dirty != NULL) { 746c717a561Smaybee dbuf_dirty_record_t *dr = db->db_last_dirty; 747c717a561Smaybee 748c717a561Smaybee if (dr->dr_txg == txg) { 749ea8dc4b6Seschrock /* 750c717a561Smaybee * This buffer is "in-use", re-adjust the file 751c717a561Smaybee * size to reflect that this buffer may 752c717a561Smaybee * contain new data when we sync. 753ea8dc4b6Seschrock */ 75444eda4d7Smaybee if (db->db_blkid > dn->dn_maxblkid) 75544eda4d7Smaybee dn->dn_maxblkid = db->db_blkid; 756c717a561Smaybee dbuf_unoverride(dr); 757c717a561Smaybee } else { 758c717a561Smaybee /* 759c717a561Smaybee * This dbuf is not dirty in the open context. 760c717a561Smaybee * Either uncache it (if its not referenced in 761c717a561Smaybee * the open context) or reset its contents to 762c717a561Smaybee * empty. 763c717a561Smaybee */ 764c717a561Smaybee dbuf_fix_old_data(db, txg); 76544eda4d7Smaybee } 766c717a561Smaybee } 767c717a561Smaybee /* clear the contents if its cached */ 768ea8dc4b6Seschrock if (db->db_state == DB_CACHED) { 769ea8dc4b6Seschrock ASSERT(db->db.db_data != NULL); 770fa9e4066Sahrens arc_release(db->db_buf, db); 771fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 7726b4acc8bSahrens arc_buf_freeze(db->db_buf); 773fa9e4066Sahrens } 774ea8dc4b6Seschrock 775fa9e4066Sahrens mutex_exit(&db->db_mtx); 776fa9e4066Sahrens } 777fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 778fa9e4066Sahrens } 779fa9e4066Sahrens 780fa9e4066Sahrens static int 781ea8dc4b6Seschrock dbuf_new_block(dmu_buf_impl_t *db) 782fa9e4066Sahrens { 783fa9e4066Sahrens dsl_dataset_t *ds = db->db_objset->os_dsl_dataset; 784fa9e4066Sahrens uint64_t birth_txg = 0; 785fa9e4066Sahrens 786fa9e4066Sahrens /* Don't count meta-objects */ 787fa9e4066Sahrens if (ds == NULL) 788fa9e4066Sahrens return (FALSE); 789fa9e4066Sahrens 790fa9e4066Sahrens /* 791fa9e4066Sahrens * We don't need any locking to protect db_blkptr: 792c717a561Smaybee * If it's syncing, then db_last_dirty will be set 793c717a561Smaybee * so we'll ignore db_blkptr. 794fa9e4066Sahrens */ 795c717a561Smaybee ASSERT(MUTEX_HELD(&db->db_mtx)); 796fa9e4066Sahrens /* If we have been dirtied since the last snapshot, its not new */ 797c717a561Smaybee if (db->db_last_dirty) 798c717a561Smaybee birth_txg = db->db_last_dirty->dr_txg; 799fa9e4066Sahrens else if (db->db_blkptr) 800fa9e4066Sahrens birth_txg = db->db_blkptr->blk_birth; 801fa9e4066Sahrens 802fa9e4066Sahrens if (birth_txg) 803ea8dc4b6Seschrock return (!dsl_dataset_block_freeable(ds, birth_txg)); 804fa9e4066Sahrens else 805fa9e4066Sahrens return (TRUE); 806fa9e4066Sahrens } 807fa9e4066Sahrens 808fa9e4066Sahrens void 809fa9e4066Sahrens dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx) 810fa9e4066Sahrens { 811fa9e4066Sahrens arc_buf_t *buf, *obuf; 812fa9e4066Sahrens int osize = db->db.db_size; 813ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 814fa9e4066Sahrens 815ea8dc4b6Seschrock ASSERT(db->db_blkid != DB_BONUS_BLKID); 816ea8dc4b6Seschrock 817fa9e4066Sahrens /* XXX does *this* func really need the lock? */ 818fa9e4066Sahrens ASSERT(RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)); 819fa9e4066Sahrens 820fa9e4066Sahrens /* 821fa9e4066Sahrens * This call to dbuf_will_dirty() with the dn_struct_rwlock held 822fa9e4066Sahrens * is OK, because there can be no other references to the db 823fa9e4066Sahrens * when we are changing its size, so no concurrent DB_FILL can 824fa9e4066Sahrens * be happening. 825fa9e4066Sahrens */ 826ea8dc4b6Seschrock /* 827ea8dc4b6Seschrock * XXX we should be doing a dbuf_read, checking the return 828ea8dc4b6Seschrock * value and returning that up to our callers 829ea8dc4b6Seschrock */ 830fa9e4066Sahrens dbuf_will_dirty(db, tx); 831fa9e4066Sahrens 832fa9e4066Sahrens /* create the data buffer for the new block */ 833ad23a2dbSjohansen buf = arc_buf_alloc(db->db_dnode->dn_objset->os_spa, size, db, type); 834fa9e4066Sahrens 835fa9e4066Sahrens /* copy old block data to the new block */ 836fa9e4066Sahrens obuf = db->db_buf; 837f65e61c0Sahrens bcopy(obuf->b_data, buf->b_data, MIN(osize, size)); 838fa9e4066Sahrens /* zero the remainder */ 839f65e61c0Sahrens if (size > osize) 840fa9e4066Sahrens bzero((uint8_t *)buf->b_data + osize, size - osize); 841fa9e4066Sahrens 842fa9e4066Sahrens mutex_enter(&db->db_mtx); 843fa9e4066Sahrens dbuf_set_data(db, buf); 844ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(obuf, db) == 1); 845fa9e4066Sahrens db->db.db_size = size; 846fa9e4066Sahrens 847c717a561Smaybee if (db->db_level == 0) { 848c717a561Smaybee ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg); 849c717a561Smaybee db->db_last_dirty->dt.dl.dr_data = buf; 850c717a561Smaybee } 851fa9e4066Sahrens mutex_exit(&db->db_mtx); 852fa9e4066Sahrens 853fa9e4066Sahrens dnode_willuse_space(db->db_dnode, size-osize, tx); 854fa9e4066Sahrens } 855fa9e4066Sahrens 856c717a561Smaybee dbuf_dirty_record_t * 857fa9e4066Sahrens dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 858fa9e4066Sahrens { 859fa9e4066Sahrens dnode_t *dn = db->db_dnode; 860fa9e4066Sahrens objset_impl_t *os = dn->dn_objset; 861c717a561Smaybee dbuf_dirty_record_t **drp, *dr; 862fa9e4066Sahrens int drop_struct_lock = FALSE; 863fa9e4066Sahrens int txgoff = tx->tx_txg & TXG_MASK; 864fa9e4066Sahrens 865fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 866fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 8679c9dc39aSek110237 DMU_TX_DIRTY_BUF(tx, db); 868fa9e4066Sahrens 869fa9e4066Sahrens /* 870fa9e4066Sahrens * Shouldn't dirty a regular buffer in syncing context. Private 871fa9e4066Sahrens * objects may be dirtied in syncing context, but only if they 872fa9e4066Sahrens * were already pre-dirtied in open context. 873fa9e4066Sahrens * XXX We may want to prohibit dirtying in syncing context even 874fa9e4066Sahrens * if they did pre-dirty. 875fa9e4066Sahrens */ 876c717a561Smaybee ASSERT(!dmu_tx_is_syncing(tx) || 877c717a561Smaybee BP_IS_HOLE(dn->dn_objset->os_rootbp) || 878c717a561Smaybee dn->dn_object == DMU_META_DNODE_OBJECT || 879c717a561Smaybee dn->dn_objset->os_dsl_dataset == NULL || 880c717a561Smaybee dsl_dir_is_private(dn->dn_objset->os_dsl_dataset->ds_dir)); 881fa9e4066Sahrens 882fa9e4066Sahrens /* 883fa9e4066Sahrens * We make this assert for private objects as well, but after we 884fa9e4066Sahrens * check if we're already dirty. They are allowed to re-dirty 885fa9e4066Sahrens * in syncing context. 886fa9e4066Sahrens */ 887ea8dc4b6Seschrock ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 888c717a561Smaybee dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 889fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 890fa9e4066Sahrens 891fa9e4066Sahrens mutex_enter(&db->db_mtx); 892fa9e4066Sahrens /* 893c717a561Smaybee * XXX make this true for indirects too? The problem is that 894c717a561Smaybee * transactions created with dmu_tx_create_assigned() from 895c717a561Smaybee * syncing context don't bother holding ahead. 896fa9e4066Sahrens */ 897c717a561Smaybee ASSERT(db->db_level != 0 || 898c717a561Smaybee db->db_state == DB_CACHED || db->db_state == DB_FILL); 899fa9e4066Sahrens 900fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 901fa9e4066Sahrens /* 902fa9e4066Sahrens * Don't set dirtyctx to SYNC if we're just modifying this as we 903fa9e4066Sahrens * initialize the objset. 904fa9e4066Sahrens */ 905fa9e4066Sahrens if (dn->dn_dirtyctx == DN_UNDIRTIED && 906c717a561Smaybee !BP_IS_HOLE(dn->dn_objset->os_rootbp)) { 907fa9e4066Sahrens dn->dn_dirtyctx = 908fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN); 909fa9e4066Sahrens ASSERT(dn->dn_dirtyctx_firstset == NULL); 910fa9e4066Sahrens dn->dn_dirtyctx_firstset = kmem_alloc(1, KM_SLEEP); 911fa9e4066Sahrens } 912fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 913fa9e4066Sahrens 914fa9e4066Sahrens /* 915fa9e4066Sahrens * If this buffer is already dirty, we're done. 916fa9e4066Sahrens */ 917c717a561Smaybee drp = &db->db_last_dirty; 918c717a561Smaybee ASSERT(*drp == NULL || (*drp)->dr_txg <= tx->tx_txg || 919c717a561Smaybee db->db.db_object == DMU_META_DNODE_OBJECT); 920c717a561Smaybee while (*drp && (*drp)->dr_txg > tx->tx_txg) 921c717a561Smaybee drp = &(*drp)->dr_next; 922c717a561Smaybee if (*drp && (*drp)->dr_txg == tx->tx_txg) { 923c717a561Smaybee if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID) { 924c717a561Smaybee /* 925c717a561Smaybee * If this buffer has already been written out, 926c717a561Smaybee * we now need to reset its state. 927c717a561Smaybee */ 928c717a561Smaybee dbuf_unoverride(*drp); 929c717a561Smaybee if (db->db.db_object != DMU_META_DNODE_OBJECT) 9306b4acc8bSahrens arc_buf_thaw(db->db_buf); 931c717a561Smaybee } 932fa9e4066Sahrens mutex_exit(&db->db_mtx); 933c717a561Smaybee return (*drp); 934fa9e4066Sahrens } 935fa9e4066Sahrens 936fa9e4066Sahrens /* 937fa9e4066Sahrens * Only valid if not already dirty. 938fa9e4066Sahrens */ 939fa9e4066Sahrens ASSERT(dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 940fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 941fa9e4066Sahrens 942fa9e4066Sahrens ASSERT3U(dn->dn_nlevels, >, db->db_level); 943fa9e4066Sahrens ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) || 944fa9e4066Sahrens dn->dn_phys->dn_nlevels > db->db_level || 945fa9e4066Sahrens dn->dn_next_nlevels[txgoff] > db->db_level || 946fa9e4066Sahrens dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level || 947fa9e4066Sahrens dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level); 948fa9e4066Sahrens 949fa9e4066Sahrens /* 950fa9e4066Sahrens * We should only be dirtying in syncing context if it's the 951fa9e4066Sahrens * mos, a spa os, or we're initializing the os. However, we are 952fa9e4066Sahrens * allowed to dirty in syncing context provided we already 953fa9e4066Sahrens * dirtied it in open context. Hence we must make this 954fa9e4066Sahrens * assertion only if we're not already dirty. 955fa9e4066Sahrens */ 956fa9e4066Sahrens ASSERT(!dmu_tx_is_syncing(tx) || 957fa9e4066Sahrens os->os_dsl_dataset == NULL || 958fa9e4066Sahrens !dsl_dir_is_private(os->os_dsl_dataset->ds_dir) || 959c717a561Smaybee !BP_IS_HOLE(os->os_rootbp)); 960fa9e4066Sahrens ASSERT(db->db.db_size != 0); 961fa9e4066Sahrens 962fa9e4066Sahrens dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 963fa9e4066Sahrens 964ea8dc4b6Seschrock /* 965ea8dc4b6Seschrock * If this buffer is dirty in an old transaction group we need 966ea8dc4b6Seschrock * to make a copy of it so that the changes we make in this 967ea8dc4b6Seschrock * transaction group won't leak out when we sync the older txg. 968ea8dc4b6Seschrock */ 969c717a561Smaybee dr = kmem_zalloc(sizeof (dbuf_dirty_record_t), KM_SLEEP); 970c717a561Smaybee if (db->db_level == 0) { 971c717a561Smaybee void *data_old = db->db_buf; 972c717a561Smaybee 973ea8dc4b6Seschrock if (db->db_blkid == DB_BONUS_BLKID) { 974c717a561Smaybee dbuf_fix_old_data(db, tx->tx_txg); 975c717a561Smaybee data_old = db->db.db_data; 976c717a561Smaybee } else if (db->db.db_object != DMU_META_DNODE_OBJECT) { 977fa9e4066Sahrens /* 978fa9e4066Sahrens * Release the data buffer from the cache so that we 979fa9e4066Sahrens * can modify it without impacting possible other users 980c717a561Smaybee * of this cached data block. Note that indirect 981c717a561Smaybee * blocks and private objects are not released until the 982c717a561Smaybee * syncing state (since they are only modified then). 983fa9e4066Sahrens */ 984fa9e4066Sahrens arc_release(db->db_buf, db); 985fa9e4066Sahrens dbuf_fix_old_data(db, tx->tx_txg); 986c717a561Smaybee data_old = db->db_buf; 987fa9e4066Sahrens } 988c717a561Smaybee ASSERT(data_old != NULL); 989c717a561Smaybee dr->dt.dl.dr_data = data_old; 990c717a561Smaybee } else { 991c717a561Smaybee mutex_init(&dr->dt.di.dr_mtx, NULL, MUTEX_DEFAULT, NULL); 992c717a561Smaybee list_create(&dr->dt.di.dr_children, 993c717a561Smaybee sizeof (dbuf_dirty_record_t), 994c717a561Smaybee offsetof(dbuf_dirty_record_t, dr_dirty_node)); 995fa9e4066Sahrens } 996c717a561Smaybee dr->dr_dbuf = db; 997c717a561Smaybee dr->dr_txg = tx->tx_txg; 998c717a561Smaybee dr->dr_next = *drp; 999c717a561Smaybee *drp = dr; 1000fa9e4066Sahrens 1001fa9e4066Sahrens /* 1002fa9e4066Sahrens * We could have been freed_in_flight between the dbuf_noread 1003fa9e4066Sahrens * and dbuf_dirty. We win, as though the dbuf_noread() had 1004fa9e4066Sahrens * happened after the free. 1005fa9e4066Sahrens */ 1006fa9e4066Sahrens if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID) { 1007c717a561Smaybee mutex_enter(&dn->dn_mtx); 1008fa9e4066Sahrens dnode_clear_range(dn, db->db_blkid, 1, tx); 1009fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1010c717a561Smaybee db->db_freed_in_flight = FALSE; 1011c717a561Smaybee } 1012fa9e4066Sahrens 1013fa9e4066Sahrens if (db->db_blkid != DB_BONUS_BLKID) { 1014ea8dc4b6Seschrock /* 1015ea8dc4b6Seschrock * Update the accounting. 1016ea8dc4b6Seschrock */ 1017ea8dc4b6Seschrock if (!dbuf_new_block(db) && db->db_blkptr) { 1018fa9e4066Sahrens /* 1019fa9e4066Sahrens * This is only a guess -- if the dbuf is dirty 1020fa9e4066Sahrens * in a previous txg, we don't know how much 1021fa9e4066Sahrens * space it will use on disk yet. We should 1022fa9e4066Sahrens * really have the struct_rwlock to access 1023fa9e4066Sahrens * db_blkptr, but since this is just a guess, 1024fa9e4066Sahrens * it's OK if we get an odd answer. 1025fa9e4066Sahrens */ 1026fa9e4066Sahrens dnode_willuse_space(dn, 102799653d4eSeschrock -bp_get_dasize(os->os_spa, db->db_blkptr), tx); 1028fa9e4066Sahrens } 1029fa9e4066Sahrens dnode_willuse_space(dn, db->db.db_size, tx); 1030fa9e4066Sahrens } 1031fa9e4066Sahrens 1032fa9e4066Sahrens /* 1033fa9e4066Sahrens * This buffer is now part of this txg 1034fa9e4066Sahrens */ 1035fa9e4066Sahrens dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg); 1036fa9e4066Sahrens db->db_dirtycnt += 1; 1037fa9e4066Sahrens ASSERT3U(db->db_dirtycnt, <=, 3); 1038fa9e4066Sahrens 1039fa9e4066Sahrens mutex_exit(&db->db_mtx); 1040fa9e4066Sahrens 1041fa9e4066Sahrens if (db->db_blkid == DB_BONUS_BLKID) { 1042c717a561Smaybee mutex_enter(&dn->dn_mtx); 1043c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 1044c717a561Smaybee list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1045c717a561Smaybee mutex_exit(&dn->dn_mtx); 1046fa9e4066Sahrens dnode_setdirty(dn, tx); 1047c717a561Smaybee return (dr); 1048fa9e4066Sahrens } 1049fa9e4066Sahrens 105044eda4d7Smaybee if (db->db_level == 0) { 1051fa9e4066Sahrens dnode_new_blkid(dn, db->db_blkid, tx); 105244eda4d7Smaybee ASSERT(dn->dn_maxblkid >= db->db_blkid); 105344eda4d7Smaybee } 1054fa9e4066Sahrens 1055fa9e4066Sahrens if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 1056fa9e4066Sahrens rw_enter(&dn->dn_struct_rwlock, RW_READER); 1057fa9e4066Sahrens drop_struct_lock = TRUE; 1058fa9e4066Sahrens } 1059fa9e4066Sahrens 106044eda4d7Smaybee if (db->db_level+1 < dn->dn_nlevels) { 1061c717a561Smaybee dmu_buf_impl_t *parent = db->db_parent; 1062c717a561Smaybee dbuf_dirty_record_t *di; 1063c717a561Smaybee int parent_held = FALSE; 1064c717a561Smaybee 1065c717a561Smaybee if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) { 1066fa9e4066Sahrens int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1067c717a561Smaybee 1068fa9e4066Sahrens parent = dbuf_hold_level(dn, db->db_level+1, 1069fa9e4066Sahrens db->db_blkid >> epbs, FTAG); 1070c717a561Smaybee parent_held = TRUE; 1071c717a561Smaybee } 1072fa9e4066Sahrens if (drop_struct_lock) 1073fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 1074c717a561Smaybee ASSERT3U(db->db_level+1, ==, parent->db_level); 1075c717a561Smaybee di = dbuf_dirty(parent, tx); 1076c717a561Smaybee if (parent_held) 1077ea8dc4b6Seschrock dbuf_rele(parent, FTAG); 1078c717a561Smaybee 1079c717a561Smaybee mutex_enter(&db->db_mtx); 1080c717a561Smaybee /* possible race with dbuf_undirty() */ 1081c717a561Smaybee if (db->db_last_dirty == dr || 1082c717a561Smaybee dn->dn_object == DMU_META_DNODE_OBJECT) { 1083c717a561Smaybee mutex_enter(&di->dt.di.dr_mtx); 1084c717a561Smaybee ASSERT3U(di->dr_txg, ==, tx->tx_txg); 1085c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 1086c717a561Smaybee list_insert_tail(&di->dt.di.dr_children, dr); 1087c717a561Smaybee mutex_exit(&di->dt.di.dr_mtx); 1088c717a561Smaybee dr->dr_parent = di; 1089c717a561Smaybee } 1090c717a561Smaybee mutex_exit(&db->db_mtx); 1091fa9e4066Sahrens } else { 1092c717a561Smaybee ASSERT(db->db_level+1 == dn->dn_nlevels); 1093c717a561Smaybee ASSERT(db->db_blkid < dn->dn_nblkptr); 1094c717a561Smaybee ASSERT(db->db_parent == NULL || 1095c717a561Smaybee db->db_parent == db->db_dnode->dn_dbuf); 1096c717a561Smaybee mutex_enter(&dn->dn_mtx); 1097c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 1098c717a561Smaybee list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1099c717a561Smaybee mutex_exit(&dn->dn_mtx); 1100fa9e4066Sahrens if (drop_struct_lock) 1101fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 1102fa9e4066Sahrens } 1103fa9e4066Sahrens 1104fa9e4066Sahrens dnode_setdirty(dn, tx); 1105c717a561Smaybee return (dr); 1106fa9e4066Sahrens } 1107fa9e4066Sahrens 1108fa9e4066Sahrens static int 1109fa9e4066Sahrens dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1110fa9e4066Sahrens { 1111fa9e4066Sahrens dnode_t *dn = db->db_dnode; 1112c717a561Smaybee uint64_t txg = tx->tx_txg; 1113c717a561Smaybee dbuf_dirty_record_t *dr; 1114fa9e4066Sahrens 1115c717a561Smaybee ASSERT(txg != 0); 1116ea8dc4b6Seschrock ASSERT(db->db_blkid != DB_BONUS_BLKID); 1117fa9e4066Sahrens 1118fa9e4066Sahrens mutex_enter(&db->db_mtx); 1119fa9e4066Sahrens 1120fa9e4066Sahrens /* 1121fa9e4066Sahrens * If this buffer is not dirty, we're done. 1122fa9e4066Sahrens */ 1123c717a561Smaybee for (dr = db->db_last_dirty; dr; dr = dr->dr_next) 1124c717a561Smaybee if (dr->dr_txg <= txg) 1125c717a561Smaybee break; 1126c717a561Smaybee if (dr == NULL || dr->dr_txg < txg) { 1127fa9e4066Sahrens mutex_exit(&db->db_mtx); 1128fa9e4066Sahrens return (0); 1129fa9e4066Sahrens } 1130c717a561Smaybee ASSERT(dr->dr_txg == txg); 1131fa9e4066Sahrens 1132fa9e4066Sahrens /* 1133fa9e4066Sahrens * If this buffer is currently held, we cannot undirty 1134fa9e4066Sahrens * it, since one of the current holders may be in the 1135fa9e4066Sahrens * middle of an update. Note that users of dbuf_undirty() 1136fa9e4066Sahrens * should not place a hold on the dbuf before the call. 1137fa9e4066Sahrens */ 1138fa9e4066Sahrens if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 1139fa9e4066Sahrens mutex_exit(&db->db_mtx); 114044eda4d7Smaybee /* Make sure we don't toss this buffer at sync phase */ 1141fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 1142fa9e4066Sahrens dnode_clear_range(dn, db->db_blkid, 1, tx); 1143fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1144fa9e4066Sahrens return (0); 1145fa9e4066Sahrens } 1146fa9e4066Sahrens 1147fa9e4066Sahrens dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1148fa9e4066Sahrens 1149fa9e4066Sahrens ASSERT(db->db.db_size != 0); 1150fa9e4066Sahrens 1151fa9e4066Sahrens /* XXX would be nice to fix up dn_towrite_space[] */ 1152fa9e4066Sahrens 1153c717a561Smaybee db->db_last_dirty = dr->dr_next; 1154c717a561Smaybee 1155c717a561Smaybee if (dr->dr_parent) { 1156c717a561Smaybee mutex_enter(&dr->dr_parent->dt.di.dr_mtx); 1157c717a561Smaybee list_remove(&dr->dr_parent->dt.di.dr_children, dr); 1158c717a561Smaybee mutex_exit(&dr->dr_parent->dt.di.dr_mtx); 1159c717a561Smaybee } else if (db->db_level+1 == dn->dn_nlevels) { 1160c717a561Smaybee ASSERT3P(db->db_parent, ==, dn->dn_dbuf); 1161fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 1162c717a561Smaybee list_remove(&dn->dn_dirty_records[txg & TXG_MASK], dr); 1163fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1164c717a561Smaybee } 1165c717a561Smaybee 1166c717a561Smaybee if (db->db_level == 0) { 1167c717a561Smaybee dbuf_unoverride(dr); 1168c717a561Smaybee 1169c717a561Smaybee ASSERT(db->db_buf != NULL); 1170c717a561Smaybee ASSERT(dr->dt.dl.dr_data != NULL); 1171c717a561Smaybee if (dr->dt.dl.dr_data != db->db_buf) 1172c717a561Smaybee VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, db) == 1); 1173c717a561Smaybee } else { 1174c717a561Smaybee ASSERT(db->db_buf != NULL); 1175c717a561Smaybee ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 1176c717a561Smaybee /* XXX - mutex and list destroy? */ 1177c717a561Smaybee } 1178c717a561Smaybee kmem_free(dr, sizeof (dbuf_dirty_record_t)); 1179fa9e4066Sahrens 1180fa9e4066Sahrens ASSERT(db->db_dirtycnt > 0); 1181fa9e4066Sahrens db->db_dirtycnt -= 1; 1182fa9e4066Sahrens 1183c717a561Smaybee if (refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) { 1184ea8dc4b6Seschrock arc_buf_t *buf = db->db_buf; 1185fa9e4066Sahrens 1186ea8dc4b6Seschrock ASSERT(arc_released(buf)); 1187ea8dc4b6Seschrock dbuf_set_data(db, NULL); 1188ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(buf, db) == 1); 1189fa9e4066Sahrens dbuf_evict(db); 1190fa9e4066Sahrens return (1); 1191fa9e4066Sahrens } 1192fa9e4066Sahrens 1193fa9e4066Sahrens mutex_exit(&db->db_mtx); 1194fa9e4066Sahrens return (0); 1195fa9e4066Sahrens } 1196fa9e4066Sahrens 1197fa9e4066Sahrens #pragma weak dmu_buf_will_dirty = dbuf_will_dirty 1198fa9e4066Sahrens void 1199fa9e4066Sahrens dbuf_will_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1200fa9e4066Sahrens { 1201fa9e4066Sahrens int rf = DB_RF_MUST_SUCCEED; 1202fa9e4066Sahrens 1203fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 1204fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 1205fa9e4066Sahrens 1206fa9e4066Sahrens if (RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)) 1207fa9e4066Sahrens rf |= DB_RF_HAVESTRUCT; 1208ea8dc4b6Seschrock (void) dbuf_read(db, NULL, rf); 1209c717a561Smaybee (void) dbuf_dirty(db, tx); 1210fa9e4066Sahrens } 1211fa9e4066Sahrens 1212fa9e4066Sahrens void 1213ea8dc4b6Seschrock dmu_buf_will_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 1214fa9e4066Sahrens { 1215ea8dc4b6Seschrock dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1216ea8dc4b6Seschrock 1217ea8dc4b6Seschrock ASSERT(db->db_blkid != DB_BONUS_BLKID); 1218fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 1219fa9e4066Sahrens ASSERT(db->db_level == 0); 1220fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 1221fa9e4066Sahrens 1222ea8dc4b6Seschrock ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT || 1223fa9e4066Sahrens dmu_tx_private_ok(tx)); 1224fa9e4066Sahrens 1225fa9e4066Sahrens dbuf_noread(db); 1226c717a561Smaybee (void) dbuf_dirty(db, tx); 1227fa9e4066Sahrens } 1228fa9e4066Sahrens 1229fa9e4066Sahrens #pragma weak dmu_buf_fill_done = dbuf_fill_done 1230fa9e4066Sahrens /* ARGSUSED */ 1231fa9e4066Sahrens void 1232fa9e4066Sahrens dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx) 1233fa9e4066Sahrens { 1234fa9e4066Sahrens mutex_enter(&db->db_mtx); 12359c9dc39aSek110237 DBUF_VERIFY(db); 1236fa9e4066Sahrens 1237fa9e4066Sahrens if (db->db_state == DB_FILL) { 1238c717a561Smaybee if (db->db_level == 0 && db->db_freed_in_flight) { 1239ea8dc4b6Seschrock ASSERT(db->db_blkid != DB_BONUS_BLKID); 1240fa9e4066Sahrens /* we were freed while filling */ 1241fa9e4066Sahrens /* XXX dbuf_undirty? */ 1242fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 1243c717a561Smaybee db->db_freed_in_flight = FALSE; 1244fa9e4066Sahrens } 1245fa9e4066Sahrens db->db_state = DB_CACHED; 1246fa9e4066Sahrens cv_broadcast(&db->db_changed); 1247fa9e4066Sahrens } 1248fa9e4066Sahrens mutex_exit(&db->db_mtx); 1249fa9e4066Sahrens } 1250fa9e4066Sahrens 1251ea8dc4b6Seschrock /* 1252ea8dc4b6Seschrock * "Clear" the contents of this dbuf. This will mark the dbuf 1253ea8dc4b6Seschrock * EVICTING and clear *most* of its references. Unfortunetely, 1254ea8dc4b6Seschrock * when we are not holding the dn_dbufs_mtx, we can't clear the 1255ea8dc4b6Seschrock * entry in the dn_dbufs list. We have to wait until dbuf_destroy() 1256ea8dc4b6Seschrock * in this case. For callers from the DMU we will usually see: 1257ea8dc4b6Seschrock * dbuf_clear()->arc_buf_evict()->dbuf_do_evict()->dbuf_destroy() 1258ea8dc4b6Seschrock * For the arc callback, we will usually see: 1259ea8dc4b6Seschrock * dbuf_do_evict()->dbuf_clear();dbuf_destroy() 1260ea8dc4b6Seschrock * Sometimes, though, we will get a mix of these two: 1261ea8dc4b6Seschrock * DMU: dbuf_clear()->arc_buf_evict() 1262ea8dc4b6Seschrock * ARC: dbuf_do_evict()->dbuf_destroy() 1263ea8dc4b6Seschrock */ 1264ea8dc4b6Seschrock void 1265fa9e4066Sahrens dbuf_clear(dmu_buf_impl_t *db) 1266fa9e4066Sahrens { 1267fa9e4066Sahrens dnode_t *dn = db->db_dnode; 1268ea8dc4b6Seschrock dmu_buf_impl_t *parent = db->db_parent; 1269c543ec06Sahrens dmu_buf_impl_t *dndb = dn->dn_dbuf; 1270ea8dc4b6Seschrock int dbuf_gone = FALSE; 1271fa9e4066Sahrens 1272fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 1273fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 1274fa9e4066Sahrens 1275ea8dc4b6Seschrock dbuf_evict_user(db); 1276ea8dc4b6Seschrock 1277fa9e4066Sahrens if (db->db_state == DB_CACHED) { 1278ea8dc4b6Seschrock ASSERT(db->db.db_data != NULL); 1279ea8dc4b6Seschrock if (db->db_blkid == DB_BONUS_BLKID) 1280ea8dc4b6Seschrock zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN); 1281fa9e4066Sahrens db->db.db_data = NULL; 1282fa9e4066Sahrens db->db_state = DB_UNCACHED; 1283fa9e4066Sahrens } 1284fa9e4066Sahrens 1285fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_UNCACHED); 1286fa9e4066Sahrens ASSERT(db->db_data_pending == NULL); 1287fa9e4066Sahrens 1288ea8dc4b6Seschrock db->db_state = DB_EVICTING; 1289ea8dc4b6Seschrock db->db_blkptr = NULL; 1290ea8dc4b6Seschrock 1291ea8dc4b6Seschrock if (db->db_blkid != DB_BONUS_BLKID && MUTEX_HELD(&dn->dn_dbufs_mtx)) { 1292ea8dc4b6Seschrock list_remove(&dn->dn_dbufs, db); 1293ea8dc4b6Seschrock dnode_rele(dn, db); 1294ea8dc4b6Seschrock } 1295ea8dc4b6Seschrock 1296ea8dc4b6Seschrock if (db->db_buf) 1297ea8dc4b6Seschrock dbuf_gone = arc_buf_evict(db->db_buf); 1298ea8dc4b6Seschrock 1299ea8dc4b6Seschrock if (!dbuf_gone) 1300fa9e4066Sahrens mutex_exit(&db->db_mtx); 1301fa9e4066Sahrens 1302fa9e4066Sahrens /* 1303fa9e4066Sahrens * If this dbuf is referened from an indirect dbuf, 1304fa9e4066Sahrens * decrement the ref count on the indirect dbuf. 1305fa9e4066Sahrens */ 1306c543ec06Sahrens if (parent && parent != dndb) 1307ea8dc4b6Seschrock dbuf_rele(parent, db); 1308fa9e4066Sahrens } 1309fa9e4066Sahrens 1310fa9e4066Sahrens static int 1311fa9e4066Sahrens dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse, 1312fa9e4066Sahrens dmu_buf_impl_t **parentp, blkptr_t **bpp) 1313fa9e4066Sahrens { 1314fa9e4066Sahrens int nlevels, epbs; 1315fa9e4066Sahrens 13160b69c2f0Sahrens *parentp = NULL; 13170b69c2f0Sahrens *bpp = NULL; 13180b69c2f0Sahrens 1319ea8dc4b6Seschrock ASSERT(blkid != DB_BONUS_BLKID); 1320ea8dc4b6Seschrock 1321fa9e4066Sahrens if (dn->dn_phys->dn_nlevels == 0) 1322fa9e4066Sahrens nlevels = 1; 1323fa9e4066Sahrens else 1324fa9e4066Sahrens nlevels = dn->dn_phys->dn_nlevels; 1325fa9e4066Sahrens 1326fa9e4066Sahrens epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1327fa9e4066Sahrens 1328fa9e4066Sahrens ASSERT3U(level * epbs, <, 64); 1329fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1330ea8dc4b6Seschrock if (level >= nlevels || 1331fa9e4066Sahrens (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) { 1332fa9e4066Sahrens /* the buffer has no parent yet */ 1333fa9e4066Sahrens return (ENOENT); 1334fa9e4066Sahrens } else if (level < nlevels-1) { 1335fa9e4066Sahrens /* this block is referenced from an indirect block */ 1336fa9e4066Sahrens int err = dbuf_hold_impl(dn, level+1, 1337fa9e4066Sahrens blkid >> epbs, fail_sparse, NULL, parentp); 1338fa9e4066Sahrens if (err) 1339fa9e4066Sahrens return (err); 1340ea8dc4b6Seschrock err = dbuf_read(*parentp, NULL, 1341ea8dc4b6Seschrock (DB_RF_HAVESTRUCT | DB_RF_NOPREFETCH | DB_RF_CANFAIL)); 1342c543ec06Sahrens if (err) { 1343c543ec06Sahrens dbuf_rele(*parentp, NULL); 1344c543ec06Sahrens *parentp = NULL; 1345c543ec06Sahrens return (err); 1346c543ec06Sahrens } 1347fa9e4066Sahrens *bpp = ((blkptr_t *)(*parentp)->db.db_data) + 1348fa9e4066Sahrens (blkid & ((1ULL << epbs) - 1)); 1349c543ec06Sahrens return (0); 1350fa9e4066Sahrens } else { 1351fa9e4066Sahrens /* the block is referenced from the dnode */ 1352fa9e4066Sahrens ASSERT3U(level, ==, nlevels-1); 1353fa9e4066Sahrens ASSERT(dn->dn_phys->dn_nblkptr == 0 || 1354fa9e4066Sahrens blkid < dn->dn_phys->dn_nblkptr); 1355c543ec06Sahrens if (dn->dn_dbuf) { 1356c543ec06Sahrens dbuf_add_ref(dn->dn_dbuf, NULL); 1357fa9e4066Sahrens *parentp = dn->dn_dbuf; 1358c543ec06Sahrens } 1359fa9e4066Sahrens *bpp = &dn->dn_phys->dn_blkptr[blkid]; 1360fa9e4066Sahrens return (0); 1361fa9e4066Sahrens } 1362fa9e4066Sahrens } 1363fa9e4066Sahrens 1364fa9e4066Sahrens static dmu_buf_impl_t * 1365fa9e4066Sahrens dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid, 1366fa9e4066Sahrens dmu_buf_impl_t *parent, blkptr_t *blkptr) 1367fa9e4066Sahrens { 1368fa9e4066Sahrens objset_impl_t *os = dn->dn_objset; 1369fa9e4066Sahrens dmu_buf_impl_t *db, *odb; 1370fa9e4066Sahrens 1371fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1372fa9e4066Sahrens ASSERT(dn->dn_type != DMU_OT_NONE); 1373fa9e4066Sahrens 1374fa9e4066Sahrens db = kmem_cache_alloc(dbuf_cache, KM_SLEEP); 1375fa9e4066Sahrens 1376fa9e4066Sahrens db->db_objset = os; 1377fa9e4066Sahrens db->db.db_object = dn->dn_object; 1378fa9e4066Sahrens db->db_level = level; 1379fa9e4066Sahrens db->db_blkid = blkid; 1380c717a561Smaybee db->db_last_dirty = NULL; 1381ea8dc4b6Seschrock db->db_dirtycnt = 0; 1382ea8dc4b6Seschrock db->db_dnode = dn; 1383ea8dc4b6Seschrock db->db_parent = parent; 1384ea8dc4b6Seschrock db->db_blkptr = blkptr; 1385fa9e4066Sahrens 1386c717a561Smaybee db->db_user_ptr = NULL; 1387c717a561Smaybee db->db_user_data_ptr_ptr = NULL; 1388c717a561Smaybee db->db_evict_func = NULL; 1389c717a561Smaybee db->db_immediate_evict = 0; 1390c717a561Smaybee db->db_freed_in_flight = 0; 1391ea8dc4b6Seschrock 1392ea8dc4b6Seschrock if (blkid == DB_BONUS_BLKID) { 1393ea8dc4b6Seschrock ASSERT3P(parent, ==, dn->dn_dbuf); 1394fa9e4066Sahrens db->db.db_size = dn->dn_bonuslen; 1395fa9e4066Sahrens db->db.db_offset = DB_BONUS_BLKID; 1396ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 1397ea8dc4b6Seschrock /* the bonus dbuf is not placed in the hash table */ 1398ea8dc4b6Seschrock return (db); 1399fa9e4066Sahrens } else { 1400fa9e4066Sahrens int blocksize = 1401fa9e4066Sahrens db->db_level ? 1<<dn->dn_indblkshift : dn->dn_datablksz; 1402fa9e4066Sahrens db->db.db_size = blocksize; 1403fa9e4066Sahrens db->db.db_offset = db->db_blkid * blocksize; 1404fa9e4066Sahrens } 1405fa9e4066Sahrens 1406fa9e4066Sahrens /* 1407fa9e4066Sahrens * Hold the dn_dbufs_mtx while we get the new dbuf 1408fa9e4066Sahrens * in the hash table *and* added to the dbufs list. 1409fa9e4066Sahrens * This prevents a possible deadlock with someone 1410fa9e4066Sahrens * trying to look up this dbuf before its added to the 1411fa9e4066Sahrens * dn_dbufs list. 1412fa9e4066Sahrens */ 1413fa9e4066Sahrens mutex_enter(&dn->dn_dbufs_mtx); 1414ea8dc4b6Seschrock db->db_state = DB_EVICTING; 1415fa9e4066Sahrens if ((odb = dbuf_hash_insert(db)) != NULL) { 1416fa9e4066Sahrens /* someone else inserted it first */ 1417fa9e4066Sahrens kmem_cache_free(dbuf_cache, db); 1418fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 1419fa9e4066Sahrens return (odb); 1420fa9e4066Sahrens } 1421fa9e4066Sahrens list_insert_head(&dn->dn_dbufs, db); 1422ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 1423fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 1424fa9e4066Sahrens 1425fa9e4066Sahrens if (parent && parent != dn->dn_dbuf) 1426fa9e4066Sahrens dbuf_add_ref(parent, db); 1427fa9e4066Sahrens 1428ea8dc4b6Seschrock ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 1429ea8dc4b6Seschrock refcount_count(&dn->dn_holds) > 0); 1430fa9e4066Sahrens (void) refcount_add(&dn->dn_holds, db); 1431fa9e4066Sahrens 1432fa9e4066Sahrens dprintf_dbuf(db, "db=%p\n", db); 1433fa9e4066Sahrens 1434fa9e4066Sahrens return (db); 1435fa9e4066Sahrens } 1436fa9e4066Sahrens 1437fa9e4066Sahrens static int 1438ea8dc4b6Seschrock dbuf_do_evict(void *private) 1439fa9e4066Sahrens { 1440ea8dc4b6Seschrock arc_buf_t *buf = private; 1441ea8dc4b6Seschrock dmu_buf_impl_t *db = buf->b_private; 1442fa9e4066Sahrens 1443ea8dc4b6Seschrock if (!MUTEX_HELD(&db->db_mtx)) 1444ea8dc4b6Seschrock mutex_enter(&db->db_mtx); 1445ea8dc4b6Seschrock 1446ea8dc4b6Seschrock ASSERT(refcount_is_zero(&db->db_holds)); 1447ea8dc4b6Seschrock 1448ea8dc4b6Seschrock if (db->db_state != DB_EVICTING) { 1449ea8dc4b6Seschrock ASSERT(db->db_state == DB_CACHED); 14509c9dc39aSek110237 DBUF_VERIFY(db); 1451ea8dc4b6Seschrock db->db_buf = NULL; 1452ea8dc4b6Seschrock dbuf_evict(db); 1453ea8dc4b6Seschrock } else { 1454ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 1455ea8dc4b6Seschrock dbuf_destroy(db); 1456fa9e4066Sahrens } 1457ea8dc4b6Seschrock return (0); 1458fa9e4066Sahrens } 1459fa9e4066Sahrens 1460fa9e4066Sahrens static void 1461fa9e4066Sahrens dbuf_destroy(dmu_buf_impl_t *db) 1462fa9e4066Sahrens { 1463fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 1464fa9e4066Sahrens 1465ea8dc4b6Seschrock if (db->db_blkid != DB_BONUS_BLKID) { 1466ea8dc4b6Seschrock dnode_t *dn = db->db_dnode; 1467ea8dc4b6Seschrock 1468ea8dc4b6Seschrock /* 1469ea8dc4b6Seschrock * If this dbuf is still on the dn_dbufs list, 1470ea8dc4b6Seschrock * remove it from that list. 1471ea8dc4b6Seschrock */ 1472ea8dc4b6Seschrock if (list_link_active(&db->db_link)) { 1473ea8dc4b6Seschrock mutex_enter(&dn->dn_dbufs_mtx); 1474ea8dc4b6Seschrock list_remove(&dn->dn_dbufs, db); 1475ea8dc4b6Seschrock mutex_exit(&dn->dn_dbufs_mtx); 1476ea8dc4b6Seschrock 1477ea8dc4b6Seschrock dnode_rele(dn, db); 1478ea8dc4b6Seschrock } 1479ea8dc4b6Seschrock dbuf_hash_remove(db); 1480ea8dc4b6Seschrock } 1481ea8dc4b6Seschrock db->db_parent = NULL; 1482ea8dc4b6Seschrock db->db_dnode = NULL; 1483ea8dc4b6Seschrock db->db_buf = NULL; 1484ea8dc4b6Seschrock 1485fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 1486fa9e4066Sahrens ASSERT(db->db_hash_next == NULL); 1487fa9e4066Sahrens ASSERT(db->db_blkptr == NULL); 1488fa9e4066Sahrens ASSERT(db->db_data_pending == NULL); 1489fa9e4066Sahrens 1490fa9e4066Sahrens kmem_cache_free(dbuf_cache, db); 1491fa9e4066Sahrens } 1492fa9e4066Sahrens 1493fa9e4066Sahrens void 1494fa9e4066Sahrens dbuf_prefetch(dnode_t *dn, uint64_t blkid) 1495fa9e4066Sahrens { 149613506d1eSmaybee dmu_buf_impl_t *db = NULL; 1497fa9e4066Sahrens blkptr_t *bp = NULL; 1498fa9e4066Sahrens 1499fa9e4066Sahrens ASSERT(blkid != DB_BONUS_BLKID); 1500fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1501fa9e4066Sahrens 1502fa9e4066Sahrens if (dnode_block_freed(dn, blkid)) 1503fa9e4066Sahrens return; 1504fa9e4066Sahrens 1505fa9e4066Sahrens /* dbuf_find() returns with db_mtx held */ 1506fa9e4066Sahrens if (db = dbuf_find(dn, 0, blkid)) { 150713506d1eSmaybee if (refcount_count(&db->db_holds) > 0) { 1508fa9e4066Sahrens /* 150913506d1eSmaybee * This dbuf is active. We assume that it is 151013506d1eSmaybee * already CACHED, or else about to be either 1511fa9e4066Sahrens * read or filled. 1512fa9e4066Sahrens */ 1513fa9e4066Sahrens mutex_exit(&db->db_mtx); 1514fa9e4066Sahrens return; 1515fa9e4066Sahrens } 151613506d1eSmaybee mutex_exit(&db->db_mtx); 15170b69c2f0Sahrens db = NULL; 151813506d1eSmaybee } 1519fa9e4066Sahrens 152013506d1eSmaybee if (dbuf_findbp(dn, 0, blkid, TRUE, &db, &bp) == 0) { 1521fa9e4066Sahrens if (bp && !BP_IS_HOLE(bp)) { 152213506d1eSmaybee uint32_t aflags = ARC_NOWAIT | ARC_PREFETCH; 1523ea8dc4b6Seschrock zbookmark_t zb; 1524ea8dc4b6Seschrock zb.zb_objset = dn->dn_objset->os_dsl_dataset ? 1525ea8dc4b6Seschrock dn->dn_objset->os_dsl_dataset->ds_object : 0; 1526ea8dc4b6Seschrock zb.zb_object = dn->dn_object; 1527ea8dc4b6Seschrock zb.zb_level = 0; 1528ea8dc4b6Seschrock zb.zb_blkid = blkid; 1529ea8dc4b6Seschrock 1530fa9e4066Sahrens (void) arc_read(NULL, dn->dn_objset->os_spa, bp, 1531fa9e4066Sahrens dmu_ot[dn->dn_type].ot_byteswap, 1532fa9e4066Sahrens NULL, NULL, ZIO_PRIORITY_ASYNC_READ, 1533fa9e4066Sahrens ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 153413506d1eSmaybee &aflags, &zb); 1535fa9e4066Sahrens } 153613506d1eSmaybee if (db) 153713506d1eSmaybee dbuf_rele(db, NULL); 1538fa9e4066Sahrens } 1539fa9e4066Sahrens } 1540fa9e4066Sahrens 1541fa9e4066Sahrens /* 1542fa9e4066Sahrens * Returns with db_holds incremented, and db_mtx not held. 1543fa9e4066Sahrens * Note: dn_struct_rwlock must be held. 1544fa9e4066Sahrens */ 1545fa9e4066Sahrens int 1546fa9e4066Sahrens dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, int fail_sparse, 1547fa9e4066Sahrens void *tag, dmu_buf_impl_t **dbp) 1548fa9e4066Sahrens { 1549fa9e4066Sahrens dmu_buf_impl_t *db, *parent = NULL; 1550fa9e4066Sahrens 1551ea8dc4b6Seschrock ASSERT(blkid != DB_BONUS_BLKID); 1552fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1553fa9e4066Sahrens ASSERT3U(dn->dn_nlevels, >, level); 1554fa9e4066Sahrens 1555fa9e4066Sahrens *dbp = NULL; 1556ea8dc4b6Seschrock top: 1557fa9e4066Sahrens /* dbuf_find() returns with db_mtx held */ 1558fa9e4066Sahrens db = dbuf_find(dn, level, blkid); 1559fa9e4066Sahrens 1560fa9e4066Sahrens if (db == NULL) { 1561fa9e4066Sahrens blkptr_t *bp = NULL; 1562fa9e4066Sahrens int err; 1563fa9e4066Sahrens 1564c543ec06Sahrens ASSERT3P(parent, ==, NULL); 1565fa9e4066Sahrens err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp); 1566fa9e4066Sahrens if (fail_sparse) { 1567fa9e4066Sahrens if (err == 0 && bp && BP_IS_HOLE(bp)) 1568fa9e4066Sahrens err = ENOENT; 1569fa9e4066Sahrens if (err) { 1570c543ec06Sahrens if (parent) 1571ea8dc4b6Seschrock dbuf_rele(parent, NULL); 1572fa9e4066Sahrens return (err); 1573fa9e4066Sahrens } 1574fa9e4066Sahrens } 1575ea8dc4b6Seschrock if (err && err != ENOENT) 1576ea8dc4b6Seschrock return (err); 1577fa9e4066Sahrens db = dbuf_create(dn, level, blkid, parent, bp); 1578fa9e4066Sahrens } 1579fa9e4066Sahrens 1580ea8dc4b6Seschrock if (db->db_buf && refcount_is_zero(&db->db_holds)) { 1581ea8dc4b6Seschrock arc_buf_add_ref(db->db_buf, db); 1582ea8dc4b6Seschrock if (db->db_buf->b_data == NULL) { 1583ea8dc4b6Seschrock dbuf_clear(db); 1584c543ec06Sahrens if (parent) { 1585c543ec06Sahrens dbuf_rele(parent, NULL); 1586c543ec06Sahrens parent = NULL; 1587c543ec06Sahrens } 1588ea8dc4b6Seschrock goto top; 1589ea8dc4b6Seschrock } 1590ea8dc4b6Seschrock ASSERT3P(db->db.db_data, ==, db->db_buf->b_data); 1591ea8dc4b6Seschrock } 1592ea8dc4b6Seschrock 1593ea8dc4b6Seschrock ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf)); 1594ea8dc4b6Seschrock 1595fa9e4066Sahrens /* 1596c717a561Smaybee * If this buffer is currently syncing out, and we are are 1597c717a561Smaybee * still referencing it from db_data, we need to make a copy 1598c717a561Smaybee * of it in case we decide we want to dirty it again in this txg. 1599fa9e4066Sahrens */ 1600c717a561Smaybee if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID && 1601ea8dc4b6Seschrock dn->dn_object != DMU_META_DNODE_OBJECT && 1602c717a561Smaybee db->db_state == DB_CACHED && db->db_data_pending) { 1603c717a561Smaybee dbuf_dirty_record_t *dr = db->db_data_pending; 1604c717a561Smaybee 1605c717a561Smaybee if (dr->dt.dl.dr_data == db->db_buf) { 1606ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 1607fa9e4066Sahrens 1608c717a561Smaybee dbuf_set_data(db, 1609c717a561Smaybee arc_buf_alloc(db->db_dnode->dn_objset->os_spa, 1610c717a561Smaybee db->db.db_size, db, type)); 1611c717a561Smaybee bcopy(dr->dt.dl.dr_data->b_data, db->db.db_data, 1612fa9e4066Sahrens db->db.db_size); 1613fa9e4066Sahrens } 1614c717a561Smaybee } 1615fa9e4066Sahrens 1616ea8dc4b6Seschrock (void) refcount_add(&db->db_holds, tag); 1617fa9e4066Sahrens dbuf_update_data(db); 16189c9dc39aSek110237 DBUF_VERIFY(db); 1619fa9e4066Sahrens mutex_exit(&db->db_mtx); 1620fa9e4066Sahrens 1621fa9e4066Sahrens /* NOTE: we can't rele the parent until after we drop the db_mtx */ 1622c543ec06Sahrens if (parent) 1623ea8dc4b6Seschrock dbuf_rele(parent, NULL); 1624fa9e4066Sahrens 1625fa9e4066Sahrens ASSERT3P(db->db_dnode, ==, dn); 1626fa9e4066Sahrens ASSERT3U(db->db_blkid, ==, blkid); 1627fa9e4066Sahrens ASSERT3U(db->db_level, ==, level); 1628fa9e4066Sahrens *dbp = db; 1629fa9e4066Sahrens 1630fa9e4066Sahrens return (0); 1631fa9e4066Sahrens } 1632fa9e4066Sahrens 1633fa9e4066Sahrens dmu_buf_impl_t * 1634ea8dc4b6Seschrock dbuf_hold(dnode_t *dn, uint64_t blkid, void *tag) 1635fa9e4066Sahrens { 1636fa9e4066Sahrens dmu_buf_impl_t *db; 1637ea8dc4b6Seschrock int err = dbuf_hold_impl(dn, 0, blkid, FALSE, tag, &db); 1638ea8dc4b6Seschrock return (err ? NULL : db); 1639fa9e4066Sahrens } 1640fa9e4066Sahrens 1641fa9e4066Sahrens dmu_buf_impl_t * 1642fa9e4066Sahrens dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, void *tag) 1643fa9e4066Sahrens { 1644fa9e4066Sahrens dmu_buf_impl_t *db; 1645ea8dc4b6Seschrock int err = dbuf_hold_impl(dn, level, blkid, FALSE, tag, &db); 1646ea8dc4b6Seschrock return (err ? NULL : db); 1647fa9e4066Sahrens } 1648fa9e4066Sahrens 1649fa9e4066Sahrens dmu_buf_impl_t * 1650ea8dc4b6Seschrock dbuf_create_bonus(dnode_t *dn) 1651fa9e4066Sahrens { 1652ea8dc4b6Seschrock dmu_buf_impl_t *db = dn->dn_bonus; 1653ea8dc4b6Seschrock 1654ea8dc4b6Seschrock ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 1655ea8dc4b6Seschrock 1656ea8dc4b6Seschrock ASSERT(dn->dn_bonus == NULL); 1657ea8dc4b6Seschrock db = dbuf_create(dn, 0, DB_BONUS_BLKID, dn->dn_dbuf, NULL); 1658fa9e4066Sahrens return (db); 1659fa9e4066Sahrens } 1660fa9e4066Sahrens 1661ea8dc4b6Seschrock #pragma weak dmu_buf_add_ref = dbuf_add_ref 1662fa9e4066Sahrens void 1663fa9e4066Sahrens dbuf_add_ref(dmu_buf_impl_t *db, void *tag) 1664fa9e4066Sahrens { 1665ea8dc4b6Seschrock int64_t holds = refcount_add(&db->db_holds, tag); 1666ea8dc4b6Seschrock ASSERT(holds > 1); 1667fa9e4066Sahrens } 1668fa9e4066Sahrens 1669ea8dc4b6Seschrock #pragma weak dmu_buf_rele = dbuf_rele 1670fa9e4066Sahrens void 1671ea8dc4b6Seschrock dbuf_rele(dmu_buf_impl_t *db, void *tag) 1672fa9e4066Sahrens { 1673fa9e4066Sahrens int64_t holds; 1674fa9e4066Sahrens 1675fa9e4066Sahrens mutex_enter(&db->db_mtx); 16769c9dc39aSek110237 DBUF_VERIFY(db); 1677fa9e4066Sahrens 1678fa9e4066Sahrens holds = refcount_remove(&db->db_holds, tag); 1679ea8dc4b6Seschrock ASSERT(holds >= 0); 1680fa9e4066Sahrens 1681c717a561Smaybee /* 1682c717a561Smaybee * We can't freeze indirects if there is a possibility that they 1683c717a561Smaybee * may be modified in the current syncing context. 1684c717a561Smaybee */ 1685c717a561Smaybee if (db->db_buf && holds == (db->db_level == 0 ? db->db_dirtycnt : 0)) 16866b4acc8bSahrens arc_buf_freeze(db->db_buf); 16876b4acc8bSahrens 1688fa9e4066Sahrens if (holds == db->db_dirtycnt && 1689c717a561Smaybee db->db_level == 0 && db->db_immediate_evict) 1690fa9e4066Sahrens dbuf_evict_user(db); 1691ea8dc4b6Seschrock 1692ea8dc4b6Seschrock if (holds == 0) { 1693ea8dc4b6Seschrock if (db->db_blkid == DB_BONUS_BLKID) { 1694ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 1695ea8dc4b6Seschrock dnode_rele(db->db_dnode, db); 1696ea8dc4b6Seschrock } else if (db->db_buf == NULL) { 1697ea8dc4b6Seschrock /* 1698ea8dc4b6Seschrock * This is a special case: we never associated this 1699ea8dc4b6Seschrock * dbuf with any data allocated from the ARC. 1700ea8dc4b6Seschrock */ 1701ea8dc4b6Seschrock ASSERT3U(db->db_state, ==, DB_UNCACHED); 1702ea8dc4b6Seschrock dbuf_evict(db); 1703ea8dc4b6Seschrock } else if (arc_released(db->db_buf)) { 1704ea8dc4b6Seschrock arc_buf_t *buf = db->db_buf; 1705ea8dc4b6Seschrock /* 1706ea8dc4b6Seschrock * This dbuf has anonymous data associated with it. 1707ea8dc4b6Seschrock */ 1708ea8dc4b6Seschrock dbuf_set_data(db, NULL); 1709ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(buf, db) == 1); 1710ea8dc4b6Seschrock dbuf_evict(db); 1711ea8dc4b6Seschrock } else { 1712ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(db->db_buf, db) == 0); 1713fa9e4066Sahrens mutex_exit(&db->db_mtx); 1714fa9e4066Sahrens } 1715ea8dc4b6Seschrock } else { 1716ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 1717fa9e4066Sahrens } 1718fa9e4066Sahrens } 1719fa9e4066Sahrens 1720fa9e4066Sahrens #pragma weak dmu_buf_refcount = dbuf_refcount 1721fa9e4066Sahrens uint64_t 1722fa9e4066Sahrens dbuf_refcount(dmu_buf_impl_t *db) 1723fa9e4066Sahrens { 1724fa9e4066Sahrens return (refcount_count(&db->db_holds)); 1725fa9e4066Sahrens } 1726fa9e4066Sahrens 1727fa9e4066Sahrens void * 1728fa9e4066Sahrens dmu_buf_set_user(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 1729fa9e4066Sahrens dmu_buf_evict_func_t *evict_func) 1730fa9e4066Sahrens { 1731fa9e4066Sahrens return (dmu_buf_update_user(db_fake, NULL, user_ptr, 1732fa9e4066Sahrens user_data_ptr_ptr, evict_func)); 1733fa9e4066Sahrens } 1734fa9e4066Sahrens 1735fa9e4066Sahrens void * 1736fa9e4066Sahrens dmu_buf_set_user_ie(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 1737fa9e4066Sahrens dmu_buf_evict_func_t *evict_func) 1738fa9e4066Sahrens { 1739fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1740fa9e4066Sahrens 1741c717a561Smaybee db->db_immediate_evict = TRUE; 1742fa9e4066Sahrens return (dmu_buf_update_user(db_fake, NULL, user_ptr, 1743fa9e4066Sahrens user_data_ptr_ptr, evict_func)); 1744fa9e4066Sahrens } 1745fa9e4066Sahrens 1746fa9e4066Sahrens void * 1747fa9e4066Sahrens dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr, void *user_ptr, 1748fa9e4066Sahrens void *user_data_ptr_ptr, dmu_buf_evict_func_t *evict_func) 1749fa9e4066Sahrens { 1750fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1751fa9e4066Sahrens ASSERT(db->db_level == 0); 1752fa9e4066Sahrens 1753fa9e4066Sahrens ASSERT((user_ptr == NULL) == (evict_func == NULL)); 1754fa9e4066Sahrens 1755fa9e4066Sahrens mutex_enter(&db->db_mtx); 1756fa9e4066Sahrens 1757c717a561Smaybee if (db->db_user_ptr == old_user_ptr) { 1758c717a561Smaybee db->db_user_ptr = user_ptr; 1759c717a561Smaybee db->db_user_data_ptr_ptr = user_data_ptr_ptr; 1760c717a561Smaybee db->db_evict_func = evict_func; 1761fa9e4066Sahrens 1762fa9e4066Sahrens dbuf_update_data(db); 1763fa9e4066Sahrens } else { 1764c717a561Smaybee old_user_ptr = db->db_user_ptr; 1765fa9e4066Sahrens } 1766fa9e4066Sahrens 1767fa9e4066Sahrens mutex_exit(&db->db_mtx); 1768fa9e4066Sahrens return (old_user_ptr); 1769fa9e4066Sahrens } 1770fa9e4066Sahrens 1771fa9e4066Sahrens void * 1772fa9e4066Sahrens dmu_buf_get_user(dmu_buf_t *db_fake) 1773fa9e4066Sahrens { 1774fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1775fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 1776fa9e4066Sahrens 1777c717a561Smaybee return (db->db_user_ptr); 1778fa9e4066Sahrens } 1779fa9e4066Sahrens 1780c717a561Smaybee static void 1781c717a561Smaybee dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db) 1782fa9e4066Sahrens { 1783c717a561Smaybee /* ASSERT(dmu_tx_is_syncing(tx) */ 1784c717a561Smaybee ASSERT(MUTEX_HELD(&db->db_mtx)); 1785c717a561Smaybee 1786c717a561Smaybee if (db->db_blkptr != NULL) 1787c717a561Smaybee return; 1788c717a561Smaybee 1789c717a561Smaybee if (db->db_level == dn->dn_phys->dn_nlevels-1) { 1790c717a561Smaybee /* 1791c717a561Smaybee * This buffer was allocated at a time when there was 1792c717a561Smaybee * no available blkptrs from the dnode, or it was 1793c717a561Smaybee * inappropriate to hook it in (i.e., nlevels mis-match). 1794c717a561Smaybee */ 1795c717a561Smaybee ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr); 1796c717a561Smaybee ASSERT(db->db_parent == NULL); 1797c717a561Smaybee db->db_parent = dn->dn_dbuf; 1798c717a561Smaybee db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid]; 1799c717a561Smaybee DBUF_VERIFY(db); 1800c717a561Smaybee } else { 1801c717a561Smaybee dmu_buf_impl_t *parent = db->db_parent; 1802c717a561Smaybee int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 1803c717a561Smaybee 1804c717a561Smaybee ASSERT(dn->dn_phys->dn_nlevels > 1); 1805c717a561Smaybee if (parent == NULL) { 1806c717a561Smaybee mutex_exit(&db->db_mtx); 1807c717a561Smaybee rw_enter(&dn->dn_struct_rwlock, RW_READER); 1808c717a561Smaybee (void) dbuf_hold_impl(dn, db->db_level+1, 1809c717a561Smaybee db->db_blkid >> epbs, FALSE, db, &parent); 1810c717a561Smaybee rw_exit(&dn->dn_struct_rwlock); 1811c717a561Smaybee mutex_enter(&db->db_mtx); 1812c717a561Smaybee db->db_parent = parent; 1813c717a561Smaybee } 1814c717a561Smaybee db->db_blkptr = (blkptr_t *)parent->db.db_data + 1815c717a561Smaybee (db->db_blkid & ((1ULL << epbs) - 1)); 1816c717a561Smaybee DBUF_VERIFY(db); 1817c717a561Smaybee } 1818c717a561Smaybee } 1819c717a561Smaybee 1820c717a561Smaybee static void 1821c717a561Smaybee dbuf_sync_indirect(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 1822c717a561Smaybee { 1823c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 1824c717a561Smaybee dnode_t *dn = db->db_dnode; 1825c717a561Smaybee zio_t *zio; 1826c717a561Smaybee 1827c717a561Smaybee ASSERT(dmu_tx_is_syncing(tx)); 1828c717a561Smaybee 1829c717a561Smaybee dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 1830c717a561Smaybee 1831c717a561Smaybee mutex_enter(&db->db_mtx); 1832c717a561Smaybee 1833c717a561Smaybee ASSERT(db->db_level > 0); 1834c717a561Smaybee DBUF_VERIFY(db); 1835c717a561Smaybee 1836c717a561Smaybee if (db->db_buf == NULL) { 1837c717a561Smaybee mutex_exit(&db->db_mtx); 1838c717a561Smaybee (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED); 1839c717a561Smaybee mutex_enter(&db->db_mtx); 1840c717a561Smaybee } 1841c717a561Smaybee ASSERT3U(db->db_state, ==, DB_CACHED); 1842c717a561Smaybee ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 1843c717a561Smaybee ASSERT(db->db_buf != NULL); 1844c717a561Smaybee 1845c717a561Smaybee dbuf_check_blkptr(dn, db); 1846c717a561Smaybee 1847c717a561Smaybee db->db_data_pending = dr; 1848c717a561Smaybee mutex_exit(&db->db_mtx); 1849c717a561Smaybee 1850c717a561Smaybee arc_release(db->db_buf, db); 1851c717a561Smaybee 1852c717a561Smaybee /* 1853c717a561Smaybee * XXX -- we should design a compression algorithm 1854c717a561Smaybee * that specializes in arrays of bps. 1855c717a561Smaybee */ 1856c717a561Smaybee dbuf_write(dr, db->db_buf, ZIO_CHECKSUM_FLETCHER_4, 1857c717a561Smaybee zfs_mdcomp_disable ? ZIO_COMPRESS_EMPTY : ZIO_COMPRESS_LZJB, tx); 1858c717a561Smaybee 1859c717a561Smaybee zio = dr->dr_zio; 1860c717a561Smaybee mutex_enter(&dr->dt.di.dr_mtx); 1861c717a561Smaybee dbuf_sync_list(&dr->dt.di.dr_children, tx); 1862c717a561Smaybee ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 1863c717a561Smaybee mutex_exit(&dr->dt.di.dr_mtx); 1864c717a561Smaybee zio_nowait(zio); 1865c717a561Smaybee } 1866c717a561Smaybee 1867c717a561Smaybee static void 1868c717a561Smaybee dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 1869c717a561Smaybee { 1870c717a561Smaybee arc_buf_t **datap = &dr->dt.dl.dr_data; 1871c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 1872fa9e4066Sahrens dnode_t *dn = db->db_dnode; 1873fa9e4066Sahrens objset_impl_t *os = dn->dn_objset; 1874c717a561Smaybee uint64_t txg = tx->tx_txg; 1875ea8dc4b6Seschrock int checksum, compress; 1876fa9e4066Sahrens int blksz; 1877fa9e4066Sahrens 1878fa9e4066Sahrens ASSERT(dmu_tx_is_syncing(tx)); 1879fa9e4066Sahrens 1880fa9e4066Sahrens dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 1881fa9e4066Sahrens 1882fa9e4066Sahrens mutex_enter(&db->db_mtx); 1883fa9e4066Sahrens /* 1884fa9e4066Sahrens * To be synced, we must be dirtied. But we 1885fa9e4066Sahrens * might have been freed after the dirty. 1886fa9e4066Sahrens */ 1887fa9e4066Sahrens if (db->db_state == DB_UNCACHED) { 1888fa9e4066Sahrens /* This buffer has been freed since it was dirtied */ 1889fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 1890fa9e4066Sahrens } else if (db->db_state == DB_FILL) { 1891fa9e4066Sahrens /* This buffer was freed and is now being re-filled */ 1892c717a561Smaybee ASSERT(db->db.db_data != dr->dt.dl.dr_data); 1893fa9e4066Sahrens } else { 1894fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_CACHED); 1895fa9e4066Sahrens } 18969c9dc39aSek110237 DBUF_VERIFY(db); 1897fa9e4066Sahrens 1898fa9e4066Sahrens /* 1899c717a561Smaybee * If this is a bonus buffer, simply copy the bonus data into the 1900c717a561Smaybee * dnode. It will be written out when the dnode is synced (and it 1901c717a561Smaybee * will be synced, since it must have been dirty for dbuf_sync to 1902c717a561Smaybee * be called). 1903fa9e4066Sahrens */ 1904ea8dc4b6Seschrock if (db->db_blkid == DB_BONUS_BLKID) { 1905c717a561Smaybee dbuf_dirty_record_t **drp; 1906ea8dc4b6Seschrock /* 1907ea8dc4b6Seschrock * Use dn_phys->dn_bonuslen since db.db_size is the length 1908ea8dc4b6Seschrock * of the bonus buffer in the open transaction rather than 1909ea8dc4b6Seschrock * the syncing transaction. 1910ea8dc4b6Seschrock */ 1911ea8dc4b6Seschrock ASSERT(*datap != NULL); 1912ea8dc4b6Seschrock ASSERT3U(db->db_level, ==, 0); 1913ea8dc4b6Seschrock ASSERT3U(dn->dn_phys->dn_bonuslen, <=, DN_MAX_BONUSLEN); 1914ea8dc4b6Seschrock bcopy(*datap, DN_BONUS(dn->dn_phys), dn->dn_phys->dn_bonuslen); 1915ea8dc4b6Seschrock if (*datap != db->db.db_data) 1916ea8dc4b6Seschrock zio_buf_free(*datap, DN_MAX_BONUSLEN); 1917ea8dc4b6Seschrock db->db_data_pending = NULL; 1918c717a561Smaybee drp = &db->db_last_dirty; 1919c717a561Smaybee while (*drp != dr) 1920c717a561Smaybee drp = &(*drp)->dr_next; 1921c717a561Smaybee ASSERT((*drp)->dr_next == NULL); 1922c717a561Smaybee *drp = NULL; 1923c717a561Smaybee kmem_free(dr, sizeof (dbuf_dirty_record_t)); 1924ea8dc4b6Seschrock ASSERT(db->db_dirtycnt > 0); 1925ea8dc4b6Seschrock db->db_dirtycnt -= 1; 1926ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 1927ea8dc4b6Seschrock dbuf_rele(db, (void *)(uintptr_t)txg); 1928ea8dc4b6Seschrock return; 1929ea8dc4b6Seschrock } 1930ea8dc4b6Seschrock 1931c5c6ffa0Smaybee /* 1932c717a561Smaybee * If this buffer is in the middle of an immdiate write, 1933c717a561Smaybee * wait for the synchronous IO to complete. 1934c5c6ffa0Smaybee */ 1935c717a561Smaybee while (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC) { 1936c5c6ffa0Smaybee ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT); 1937c5c6ffa0Smaybee cv_wait(&db->db_changed, &db->db_mtx); 1938c717a561Smaybee ASSERT(dr->dt.dl.dr_override_state != DR_NOT_OVERRIDDEN); 1939c5c6ffa0Smaybee } 1940c717a561Smaybee 1941c717a561Smaybee dbuf_check_blkptr(dn, db); 1942c717a561Smaybee 1943c717a561Smaybee /* 1944c717a561Smaybee * If this dbuf has already been written out via an immediate write, 1945c717a561Smaybee * just complete the write by copying over the new block pointer and 1946c717a561Smaybee * updating the accounting via the write-completion functions. 1947c717a561Smaybee */ 1948c717a561Smaybee if (dr->dt.dl.dr_override_state == DR_OVERRIDDEN) { 1949c717a561Smaybee zio_t zio_fake; 1950c717a561Smaybee 1951c717a561Smaybee zio_fake.io_private = &db; 1952c717a561Smaybee zio_fake.io_error = 0; 1953c717a561Smaybee zio_fake.io_bp = db->db_blkptr; 1954c717a561Smaybee zio_fake.io_bp_orig = *db->db_blkptr; 1955c717a561Smaybee zio_fake.io_txg = txg; 1956c717a561Smaybee 1957c717a561Smaybee *db->db_blkptr = dr->dt.dl.dr_overridden_by; 1958c717a561Smaybee dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 1959c717a561Smaybee db->db_data_pending = dr; 1960c717a561Smaybee dr->dr_zio = &zio_fake; 1961c717a561Smaybee mutex_exit(&db->db_mtx); 1962c717a561Smaybee 1963c717a561Smaybee if (BP_IS_OLDER(&zio_fake.io_bp_orig, txg)) 1964c717a561Smaybee dsl_dataset_block_kill(os->os_dsl_dataset, 1965c717a561Smaybee &zio_fake.io_bp_orig, dn->dn_zio, tx); 1966c717a561Smaybee 1967c717a561Smaybee dbuf_write_ready(&zio_fake, db->db_buf, db); 1968c717a561Smaybee dbuf_write_done(&zio_fake, db->db_buf, db); 1969c717a561Smaybee 1970c717a561Smaybee return; 1971c717a561Smaybee } 1972c717a561Smaybee 1973c717a561Smaybee blksz = arc_buf_size(*datap); 1974c717a561Smaybee 1975c717a561Smaybee if (dn->dn_object != DMU_META_DNODE_OBJECT) { 1976fa9e4066Sahrens /* 1977fa9e4066Sahrens * If this buffer is currently "in use" (i.e., there are 1978fa9e4066Sahrens * active holds and db_data still references it), then make 1979fa9e4066Sahrens * a copy before we start the write so that any modifications 1980fa9e4066Sahrens * from the open txg will not leak into this write. 1981fa9e4066Sahrens * 1982fa9e4066Sahrens * NOTE: this copy does not need to be made for objects only 1983c717a561Smaybee * modified in the syncing context (e.g. DNONE_DNODE blocks). 1984fa9e4066Sahrens */ 1985c717a561Smaybee if (refcount_count(&db->db_holds) > 1 && *datap == db->db_buf) { 1986c717a561Smaybee arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 1987c717a561Smaybee *datap = arc_buf_alloc(os->os_spa, blksz, db, type); 1988c717a561Smaybee bcopy(db->db.db_data, (*datap)->b_data, blksz); 1989fa9e4066Sahrens } 1990c717a561Smaybee } else { 1991fa9e4066Sahrens /* 1992fa9e4066Sahrens * Private object buffers are released here rather 1993fa9e4066Sahrens * than in dbuf_dirty() since they are only modified 1994fa9e4066Sahrens * in the syncing context and we don't want the 1995fa9e4066Sahrens * overhead of making multiple copies of the data. 1996fa9e4066Sahrens */ 1997fa9e4066Sahrens arc_release(db->db_buf, db); 1998fa9e4066Sahrens } 1999fa9e4066Sahrens 2000c717a561Smaybee ASSERT(*datap != NULL); 2001c717a561Smaybee db->db_data_pending = dr; 2002fa9e4066Sahrens 2003fa9e4066Sahrens mutex_exit(&db->db_mtx); 2004fa9e4066Sahrens 2005fa9e4066Sahrens /* 2006fa9e4066Sahrens * Allow dnode settings to override objset settings, 2007fa9e4066Sahrens * except for metadata checksums. 2008fa9e4066Sahrens */ 2009fa9e4066Sahrens if (dmu_ot[dn->dn_type].ot_metadata) { 2010fa9e4066Sahrens checksum = os->os_md_checksum; 2011fa9e4066Sahrens compress = zio_compress_select(dn->dn_compress, 2012fa9e4066Sahrens os->os_md_compress); 2013fa9e4066Sahrens } else { 2014fa9e4066Sahrens checksum = zio_checksum_select(dn->dn_checksum, 2015fa9e4066Sahrens os->os_checksum); 2016fa9e4066Sahrens compress = zio_compress_select(dn->dn_compress, 2017fa9e4066Sahrens os->os_compress); 2018fa9e4066Sahrens } 2019c717a561Smaybee 2020c717a561Smaybee dbuf_write(dr, *datap, checksum, compress, tx); 2021c717a561Smaybee 2022c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 2023c717a561Smaybee if (dn->dn_object == DMU_META_DNODE_OBJECT) 2024c717a561Smaybee list_insert_tail(&dn->dn_dirty_records[txg&TXG_MASK], dr); 2025c717a561Smaybee else 2026c717a561Smaybee zio_nowait(dr->dr_zio); 2027fa9e4066Sahrens } 2028c717a561Smaybee 2029c717a561Smaybee void 2030c717a561Smaybee dbuf_sync_list(list_t *list, dmu_tx_t *tx) 2031c717a561Smaybee { 2032c717a561Smaybee dbuf_dirty_record_t *dr; 2033c717a561Smaybee 2034c717a561Smaybee while (dr = list_head(list)) { 2035c717a561Smaybee if (dr->dr_zio != NULL) { 2036c717a561Smaybee /* 2037c717a561Smaybee * If we find an already initialized zio then we 2038c717a561Smaybee * are processing the meta-dnode, and we have finished. 2039c717a561Smaybee * The dbufs for all dnodes are put back on the list 2040c717a561Smaybee * during processing, so that we can zio_wait() 2041c717a561Smaybee * these IOs after initiating all child IOs. 2042c717a561Smaybee */ 2043c717a561Smaybee ASSERT3U(dr->dr_dbuf->db.db_object, ==, 2044c717a561Smaybee DMU_META_DNODE_OBJECT); 2045c717a561Smaybee break; 2046fa9e4066Sahrens } 2047c717a561Smaybee list_remove(list, dr); 2048c717a561Smaybee if (dr->dr_dbuf->db_level > 0) 2049c717a561Smaybee dbuf_sync_indirect(dr, tx); 2050c717a561Smaybee else 2051c717a561Smaybee dbuf_sync_leaf(dr, tx); 2052c717a561Smaybee } 2053c717a561Smaybee } 2054c717a561Smaybee 2055c717a561Smaybee static void 2056c717a561Smaybee dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, int checksum, 2057c717a561Smaybee int compress, dmu_tx_t *tx) 2058c717a561Smaybee { 2059c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 2060c717a561Smaybee dnode_t *dn = db->db_dnode; 2061c717a561Smaybee objset_impl_t *os = dn->dn_objset; 2062c717a561Smaybee dmu_buf_impl_t *parent = db->db_parent; 2063c717a561Smaybee uint64_t txg = tx->tx_txg; 2064c717a561Smaybee zbookmark_t zb; 2065c717a561Smaybee zio_t *zio; 20669bc11082Sek110237 int zio_flags; 2067c717a561Smaybee 2068c717a561Smaybee if (parent != dn->dn_dbuf) { 2069c717a561Smaybee ASSERT(parent && parent->db_data_pending); 2070c717a561Smaybee ASSERT(db->db_level == parent->db_level-1); 2071c717a561Smaybee ASSERT(arc_released(parent->db_buf)); 2072c717a561Smaybee zio = parent->db_data_pending->dr_zio; 2073c717a561Smaybee } else { 2074c717a561Smaybee ASSERT(db->db_level == dn->dn_phys->dn_nlevels-1); 2075c717a561Smaybee ASSERT3P(db->db_blkptr, ==, 2076c717a561Smaybee &dn->dn_phys->dn_blkptr[db->db_blkid]); 2077c717a561Smaybee zio = dn->dn_zio; 2078c717a561Smaybee } 2079c717a561Smaybee 2080c717a561Smaybee ASSERT(db->db_level == 0 || data == db->db_buf); 2081c717a561Smaybee ASSERT3U(db->db_blkptr->blk_birth, <=, txg); 2082c717a561Smaybee ASSERT(zio); 2083c717a561Smaybee 2084ea8dc4b6Seschrock zb.zb_objset = os->os_dsl_dataset ? os->os_dsl_dataset->ds_object : 0; 2085ea8dc4b6Seschrock zb.zb_object = db->db.db_object; 2086ea8dc4b6Seschrock zb.zb_level = db->db_level; 2087ea8dc4b6Seschrock zb.zb_blkid = db->db_blkid; 208844cd46caSbillm 20899bc11082Sek110237 zio_flags = ZIO_FLAG_MUSTSUCCEED; 20909bc11082Sek110237 if (dmu_ot[dn->dn_type].ot_metadata || zb.zb_level != 0) 20919bc11082Sek110237 zio_flags |= ZIO_FLAG_METADATA; 2092c717a561Smaybee if (BP_IS_OLDER(db->db_blkptr, txg)) 2093c717a561Smaybee dsl_dataset_block_kill( 2094c717a561Smaybee os->os_dsl_dataset, db->db_blkptr, zio, tx); 2095c717a561Smaybee 2096c717a561Smaybee dr->dr_zio = arc_write(zio, os->os_spa, checksum, compress, 2097*d0ad202dSahrens dmu_get_replication_level(os, &zb, dn->dn_type), txg, 2098c717a561Smaybee db->db_blkptr, data, dbuf_write_ready, dbuf_write_done, db, 20999bc11082Sek110237 ZIO_PRIORITY_ASYNC_WRITE, zio_flags, &zb); 2100fa9e4066Sahrens } 2101fa9e4066Sahrens 2102fa9e4066Sahrens /* ARGSUSED */ 2103fa9e4066Sahrens static void 2104c717a561Smaybee dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb) 2105fa9e4066Sahrens { 2106fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 2107fa9e4066Sahrens dnode_t *dn = db->db_dnode; 2108fa9e4066Sahrens objset_impl_t *os = dn->dn_objset; 2109c717a561Smaybee blkptr_t *bp_orig = &zio->io_bp_orig; 2110fa9e4066Sahrens uint64_t fill = 0; 2111c717a561Smaybee int old_size, new_size, i; 2112fa9e4066Sahrens 2113c717a561Smaybee dprintf_dbuf_bp(db, bp_orig, "bp_orig: %s", ""); 2114fa9e4066Sahrens 2115c717a561Smaybee old_size = bp_get_dasize(os->os_spa, bp_orig); 211699653d4eSeschrock new_size = bp_get_dasize(os->os_spa, zio->io_bp); 2117fa9e4066Sahrens 2118fa9e4066Sahrens dnode_diduse_space(dn, new_size-old_size); 2119fa9e4066Sahrens 2120c717a561Smaybee if (BP_IS_HOLE(zio->io_bp)) { 2121c717a561Smaybee dsl_dataset_t *ds = os->os_dsl_dataset; 2122c717a561Smaybee dmu_tx_t *tx = os->os_synctx; 2123c717a561Smaybee 2124c717a561Smaybee if (bp_orig->blk_birth == tx->tx_txg) 2125c717a561Smaybee dsl_dataset_block_kill(ds, bp_orig, NULL, tx); 2126c717a561Smaybee ASSERT3U(db->db_blkptr->blk_fill, ==, 0); 2127c717a561Smaybee return; 2128c717a561Smaybee } 2129c717a561Smaybee 2130fa9e4066Sahrens mutex_enter(&db->db_mtx); 2131fa9e4066Sahrens 2132fa9e4066Sahrens if (db->db_level == 0) { 2133fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 2134c717a561Smaybee if (db->db_blkid > dn->dn_phys->dn_maxblkid) 2135fa9e4066Sahrens dn->dn_phys->dn_maxblkid = db->db_blkid; 2136fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 2137fa9e4066Sahrens 2138fa9e4066Sahrens if (dn->dn_type == DMU_OT_DNODE) { 2139fa9e4066Sahrens dnode_phys_t *dnp = db->db.db_data; 2140fa9e4066Sahrens for (i = db->db.db_size >> DNODE_SHIFT; i > 0; 2141fa9e4066Sahrens i--, dnp++) { 2142fa9e4066Sahrens if (dnp->dn_type != DMU_OT_NONE) 2143fa9e4066Sahrens fill++; 2144fa9e4066Sahrens } 2145fa9e4066Sahrens } else { 2146fa9e4066Sahrens fill = 1; 2147fa9e4066Sahrens } 2148fa9e4066Sahrens } else { 2149fa9e4066Sahrens blkptr_t *bp = db->db.db_data; 2150fa9e4066Sahrens ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2151fa9e4066Sahrens for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, bp++) { 2152fa9e4066Sahrens if (BP_IS_HOLE(bp)) 2153fa9e4066Sahrens continue; 2154fa9e4066Sahrens ASSERT3U(BP_GET_LSIZE(bp), ==, 2155fa9e4066Sahrens db->db_level == 1 ? dn->dn_datablksz : 2156fa9e4066Sahrens (1<<dn->dn_phys->dn_indblkshift)); 2157fa9e4066Sahrens fill += bp->blk_fill; 2158fa9e4066Sahrens } 2159fa9e4066Sahrens } 2160fa9e4066Sahrens 2161fa9e4066Sahrens db->db_blkptr->blk_fill = fill; 2162fa9e4066Sahrens BP_SET_TYPE(db->db_blkptr, dn->dn_type); 2163fa9e4066Sahrens BP_SET_LEVEL(db->db_blkptr, db->db_level); 2164fa9e4066Sahrens 2165fa9e4066Sahrens mutex_exit(&db->db_mtx); 2166fa9e4066Sahrens 2167fa9e4066Sahrens /* We must do this after we've set the bp's type and level */ 2168c717a561Smaybee if (!DVA_EQUAL(BP_IDENTITY(zio->io_bp), BP_IDENTITY(bp_orig))) { 2169c717a561Smaybee dsl_dataset_t *ds = os->os_dsl_dataset; 2170c717a561Smaybee dmu_tx_t *tx = os->os_synctx; 2171c717a561Smaybee 2172c717a561Smaybee if (bp_orig->blk_birth == tx->tx_txg) 2173c717a561Smaybee dsl_dataset_block_kill(ds, bp_orig, NULL, tx); 2174c717a561Smaybee dsl_dataset_block_born(ds, zio->io_bp, tx); 2175fa9e4066Sahrens } 2176fa9e4066Sahrens } 2177fa9e4066Sahrens 2178c717a561Smaybee /* ARGSUSED */ 2179c717a561Smaybee static void 2180c717a561Smaybee dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb) 2181c717a561Smaybee { 2182c717a561Smaybee dmu_buf_impl_t *db = vdb; 2183c717a561Smaybee uint64_t txg = zio->io_txg; 2184c717a561Smaybee dbuf_dirty_record_t **drp, *dr; 2185c717a561Smaybee 2186c717a561Smaybee ASSERT3U(zio->io_error, ==, 0); 2187c717a561Smaybee 2188c717a561Smaybee mutex_enter(&db->db_mtx); 2189c717a561Smaybee 2190c717a561Smaybee drp = &db->db_last_dirty; 2191c717a561Smaybee while (*drp != db->db_data_pending) 2192c717a561Smaybee drp = &(*drp)->dr_next; 2193c717a561Smaybee ASSERT(!list_link_active(&(*drp)->dr_dirty_node)); 2194c717a561Smaybee ASSERT((*drp)->dr_txg == txg); 2195c717a561Smaybee ASSERT((*drp)->dr_next == NULL); 2196c717a561Smaybee dr = *drp; 2197c717a561Smaybee *drp = NULL; 2198c717a561Smaybee 2199c717a561Smaybee if (db->db_level == 0) { 2200c717a561Smaybee ASSERT(db->db_blkid != DB_BONUS_BLKID); 2201c717a561Smaybee ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN); 2202c717a561Smaybee 2203c717a561Smaybee if (dr->dt.dl.dr_data != db->db_buf) 2204c717a561Smaybee VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, db) == 1); 2205c717a561Smaybee else if (!BP_IS_HOLE(db->db_blkptr)) 2206c717a561Smaybee arc_set_callback(db->db_buf, dbuf_do_evict, db); 2207c717a561Smaybee else 2208c717a561Smaybee ASSERT(arc_released(db->db_buf)); 2209c717a561Smaybee } else { 2210c717a561Smaybee dnode_t *dn = db->db_dnode; 2211c717a561Smaybee 2212c717a561Smaybee ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 2213c717a561Smaybee ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2214c717a561Smaybee if (!BP_IS_HOLE(db->db_blkptr)) { 2215c717a561Smaybee int epbs = 2216c717a561Smaybee dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2217c717a561Smaybee ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, 2218c717a561Smaybee db->db.db_size); 2219c717a561Smaybee ASSERT3U(dn->dn_phys->dn_maxblkid 2220c717a561Smaybee >> (db->db_level * epbs), >=, db->db_blkid); 2221c717a561Smaybee arc_set_callback(db->db_buf, dbuf_do_evict, db); 2222c717a561Smaybee } 2223c717a561Smaybee } 2224c717a561Smaybee kmem_free(dr, sizeof (dbuf_dirty_record_t)); 2225c717a561Smaybee 2226c717a561Smaybee cv_broadcast(&db->db_changed); 2227c717a561Smaybee ASSERT(db->db_dirtycnt > 0); 2228c717a561Smaybee db->db_dirtycnt -= 1; 2229c717a561Smaybee db->db_data_pending = NULL; 2230c717a561Smaybee mutex_exit(&db->db_mtx); 2231c717a561Smaybee 2232c717a561Smaybee dprintf_dbuf_bp(db, zio->io_bp, "bp: %s", ""); 2233c717a561Smaybee 2234ea8dc4b6Seschrock dbuf_rele(db, (void *)(uintptr_t)txg); 2235fa9e4066Sahrens } 2236