1fa9e4066Sahrens /* 2fa9e4066Sahrens * CDDL HEADER START 3fa9e4066Sahrens * 4fa9e4066Sahrens * The contents of this file are subject to the terms of the 5f65e61c0Sahrens * Common Development and Distribution License (the "License"). 6f65e61c0Sahrens * You may not use this file except in compliance with the License. 7fa9e4066Sahrens * 8fa9e4066Sahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9fa9e4066Sahrens * or http://www.opensolaris.org/os/licensing. 10fa9e4066Sahrens * See the License for the specific language governing permissions 11fa9e4066Sahrens * and limitations under the License. 12fa9e4066Sahrens * 13fa9e4066Sahrens * When distributing Covered Code, include this CDDL HEADER in each 14fa9e4066Sahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15fa9e4066Sahrens * If applicable, add the following below this CDDL HEADER, with the 16fa9e4066Sahrens * fields enclosed by brackets "[]" replaced with your own identifying 17fa9e4066Sahrens * information: Portions Copyright [yyyy] [name of copyright owner] 18fa9e4066Sahrens * 19fa9e4066Sahrens * CDDL HEADER END 20fa9e4066Sahrens */ 21fa9e4066Sahrens /* 2206e0070dSMark Shellenbaum * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 233f2366c2SGordon Ross * Copyright 2011 Nexenta Systems, Inc. All rights reserved. 24be6fd75aSMatthew Ahrens * Copyright (c) 2013 by Delphix. All rights reserved. 25aad02571SSaso Kiselkov * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. 26fa9e4066Sahrens */ 27fa9e4066Sahrens 28fa9e4066Sahrens #include <sys/zfs_context.h> 29fa9e4066Sahrens #include <sys/dmu.h> 302f3d8780SMatthew Ahrens #include <sys/dmu_send.h> 31fa9e4066Sahrens #include <sys/dmu_impl.h> 32fa9e4066Sahrens #include <sys/dbuf.h> 33fa9e4066Sahrens #include <sys/dmu_objset.h> 34fa9e4066Sahrens #include <sys/dsl_dataset.h> 35fa9e4066Sahrens #include <sys/dsl_dir.h> 36fa9e4066Sahrens #include <sys/dmu_tx.h> 37fa9e4066Sahrens #include <sys/spa.h> 38fa9e4066Sahrens #include <sys/zio.h> 39fa9e4066Sahrens #include <sys/dmu_zfetch.h> 400a586ceaSMark Shellenbaum #include <sys/sa.h> 410a586ceaSMark Shellenbaum #include <sys/sa_impl.h> 42fa9e4066Sahrens 43*713d6c20SMatthew Ahrens /* 44*713d6c20SMatthew Ahrens * Number of times that zfs_free_range() took the slow path while doing 45*713d6c20SMatthew Ahrens * a zfs receive. A nonzero value indicates a potential performance problem. 46*713d6c20SMatthew Ahrens */ 47*713d6c20SMatthew Ahrens uint64_t zfs_free_range_recv_miss; 48*713d6c20SMatthew Ahrens 49fa9e4066Sahrens static void dbuf_destroy(dmu_buf_impl_t *db); 503b2aab18SMatthew Ahrens static boolean_t dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx); 51088f3894Sahrens static void dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx); 52fa9e4066Sahrens 53fa9e4066Sahrens /* 54fa9e4066Sahrens * Global data structures and functions for the dbuf cache. 55fa9e4066Sahrens */ 56fa9e4066Sahrens static kmem_cache_t *dbuf_cache; 57fa9e4066Sahrens 58fa9e4066Sahrens /* ARGSUSED */ 59fa9e4066Sahrens static int 60fa9e4066Sahrens dbuf_cons(void *vdb, void *unused, int kmflag) 61fa9e4066Sahrens { 62fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 63fa9e4066Sahrens bzero(db, sizeof (dmu_buf_impl_t)); 64fa9e4066Sahrens 65fa9e4066Sahrens mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL); 66fa9e4066Sahrens cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL); 67fa9e4066Sahrens refcount_create(&db->db_holds); 68fa9e4066Sahrens return (0); 69fa9e4066Sahrens } 70fa9e4066Sahrens 71fa9e4066Sahrens /* ARGSUSED */ 72fa9e4066Sahrens static void 73fa9e4066Sahrens dbuf_dest(void *vdb, void *unused) 74fa9e4066Sahrens { 75fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 76fa9e4066Sahrens mutex_destroy(&db->db_mtx); 77fa9e4066Sahrens cv_destroy(&db->db_changed); 78fa9e4066Sahrens refcount_destroy(&db->db_holds); 79fa9e4066Sahrens } 80fa9e4066Sahrens 81fa9e4066Sahrens /* 82fa9e4066Sahrens * dbuf hash table routines 83fa9e4066Sahrens */ 84fa9e4066Sahrens static dbuf_hash_table_t dbuf_hash_table; 85fa9e4066Sahrens 86fa9e4066Sahrens static uint64_t dbuf_hash_count; 87fa9e4066Sahrens 88fa9e4066Sahrens static uint64_t 89fa9e4066Sahrens dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid) 90fa9e4066Sahrens { 91fa9e4066Sahrens uintptr_t osv = (uintptr_t)os; 92fa9e4066Sahrens uint64_t crc = -1ULL; 93fa9e4066Sahrens 94fa9e4066Sahrens ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 95fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (lvl)) & 0xFF]; 96fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF]; 97fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF]; 98fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF]; 99fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 0)) & 0xFF]; 100fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 8)) & 0xFF]; 101fa9e4066Sahrens 102fa9e4066Sahrens crc ^= (osv>>14) ^ (obj>>16) ^ (blkid>>16); 103fa9e4066Sahrens 104fa9e4066Sahrens return (crc); 105fa9e4066Sahrens } 106fa9e4066Sahrens 107fa9e4066Sahrens #define DBUF_HASH(os, obj, level, blkid) dbuf_hash(os, obj, level, blkid); 108fa9e4066Sahrens 109fa9e4066Sahrens #define DBUF_EQUAL(dbuf, os, obj, level, blkid) \ 110fa9e4066Sahrens ((dbuf)->db.db_object == (obj) && \ 111fa9e4066Sahrens (dbuf)->db_objset == (os) && \ 112fa9e4066Sahrens (dbuf)->db_level == (level) && \ 113fa9e4066Sahrens (dbuf)->db_blkid == (blkid)) 114fa9e4066Sahrens 115fa9e4066Sahrens dmu_buf_impl_t * 116fa9e4066Sahrens dbuf_find(dnode_t *dn, uint8_t level, uint64_t blkid) 117fa9e4066Sahrens { 118fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 119503ad85cSMatthew Ahrens objset_t *os = dn->dn_objset; 120fa9e4066Sahrens uint64_t obj = dn->dn_object; 121fa9e4066Sahrens uint64_t hv = DBUF_HASH(os, obj, level, blkid); 122fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 123fa9e4066Sahrens dmu_buf_impl_t *db; 124fa9e4066Sahrens 125fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 126fa9e4066Sahrens for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) { 127fa9e4066Sahrens if (DBUF_EQUAL(db, os, obj, level, blkid)) { 128fa9e4066Sahrens mutex_enter(&db->db_mtx); 129ea8dc4b6Seschrock if (db->db_state != DB_EVICTING) { 130fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 131fa9e4066Sahrens return (db); 132fa9e4066Sahrens } 133fa9e4066Sahrens mutex_exit(&db->db_mtx); 134fa9e4066Sahrens } 135fa9e4066Sahrens } 136fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 137fa9e4066Sahrens return (NULL); 138fa9e4066Sahrens } 139fa9e4066Sahrens 140fa9e4066Sahrens /* 141fa9e4066Sahrens * Insert an entry into the hash table. If there is already an element 142fa9e4066Sahrens * equal to elem in the hash table, then the already existing element 143fa9e4066Sahrens * will be returned and the new element will not be inserted. 144fa9e4066Sahrens * Otherwise returns NULL. 145fa9e4066Sahrens */ 146fa9e4066Sahrens static dmu_buf_impl_t * 147fa9e4066Sahrens dbuf_hash_insert(dmu_buf_impl_t *db) 148fa9e4066Sahrens { 149fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 150503ad85cSMatthew Ahrens objset_t *os = db->db_objset; 151fa9e4066Sahrens uint64_t obj = db->db.db_object; 152fa9e4066Sahrens int level = db->db_level; 153fa9e4066Sahrens uint64_t blkid = db->db_blkid; 154fa9e4066Sahrens uint64_t hv = DBUF_HASH(os, obj, level, blkid); 155fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 156fa9e4066Sahrens dmu_buf_impl_t *dbf; 157fa9e4066Sahrens 158fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 159fa9e4066Sahrens for (dbf = h->hash_table[idx]; dbf != NULL; dbf = dbf->db_hash_next) { 160fa9e4066Sahrens if (DBUF_EQUAL(dbf, os, obj, level, blkid)) { 161fa9e4066Sahrens mutex_enter(&dbf->db_mtx); 162ea8dc4b6Seschrock if (dbf->db_state != DB_EVICTING) { 163fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 164fa9e4066Sahrens return (dbf); 165fa9e4066Sahrens } 166fa9e4066Sahrens mutex_exit(&dbf->db_mtx); 167fa9e4066Sahrens } 168fa9e4066Sahrens } 169fa9e4066Sahrens 170fa9e4066Sahrens mutex_enter(&db->db_mtx); 171fa9e4066Sahrens db->db_hash_next = h->hash_table[idx]; 172fa9e4066Sahrens h->hash_table[idx] = db; 173fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 174fa9e4066Sahrens atomic_add_64(&dbuf_hash_count, 1); 175fa9e4066Sahrens 176fa9e4066Sahrens return (NULL); 177fa9e4066Sahrens } 178fa9e4066Sahrens 179fa9e4066Sahrens /* 180fa9e4066Sahrens * Remove an entry from the hash table. This operation will 181fa9e4066Sahrens * fail if there are any existing holds on the db. 182fa9e4066Sahrens */ 183fa9e4066Sahrens static void 184fa9e4066Sahrens dbuf_hash_remove(dmu_buf_impl_t *db) 185fa9e4066Sahrens { 186fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 187fa9e4066Sahrens uint64_t hv = DBUF_HASH(db->db_objset, db->db.db_object, 188fa9e4066Sahrens db->db_level, db->db_blkid); 189fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 190fa9e4066Sahrens dmu_buf_impl_t *dbf, **dbp; 191fa9e4066Sahrens 192fa9e4066Sahrens /* 193fa9e4066Sahrens * We musn't hold db_mtx to maintin lock ordering: 194fa9e4066Sahrens * DBUF_HASH_MUTEX > db_mtx. 195fa9e4066Sahrens */ 196fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 197ea8dc4b6Seschrock ASSERT(db->db_state == DB_EVICTING); 198fa9e4066Sahrens ASSERT(!MUTEX_HELD(&db->db_mtx)); 199fa9e4066Sahrens 200fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 201fa9e4066Sahrens dbp = &h->hash_table[idx]; 202fa9e4066Sahrens while ((dbf = *dbp) != db) { 203fa9e4066Sahrens dbp = &dbf->db_hash_next; 204fa9e4066Sahrens ASSERT(dbf != NULL); 205fa9e4066Sahrens } 206fa9e4066Sahrens *dbp = db->db_hash_next; 207fa9e4066Sahrens db->db_hash_next = NULL; 208fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 209fa9e4066Sahrens atomic_add_64(&dbuf_hash_count, -1); 210fa9e4066Sahrens } 211fa9e4066Sahrens 212ea8dc4b6Seschrock static arc_evict_func_t dbuf_do_evict; 213fa9e4066Sahrens 214fa9e4066Sahrens static void 215fa9e4066Sahrens dbuf_evict_user(dmu_buf_impl_t *db) 216fa9e4066Sahrens { 217fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 218fa9e4066Sahrens 219c717a561Smaybee if (db->db_level != 0 || db->db_evict_func == NULL) 220fa9e4066Sahrens return; 221fa9e4066Sahrens 222c717a561Smaybee if (db->db_user_data_ptr_ptr) 223c717a561Smaybee *db->db_user_data_ptr_ptr = db->db.db_data; 224c717a561Smaybee db->db_evict_func(&db->db, db->db_user_ptr); 225c717a561Smaybee db->db_user_ptr = NULL; 226c717a561Smaybee db->db_user_data_ptr_ptr = NULL; 227c717a561Smaybee db->db_evict_func = NULL; 228fa9e4066Sahrens } 229fa9e4066Sahrens 230744947dcSTom Erickson boolean_t 231744947dcSTom Erickson dbuf_is_metadata(dmu_buf_impl_t *db) 232744947dcSTom Erickson { 233744947dcSTom Erickson if (db->db_level > 0) { 234744947dcSTom Erickson return (B_TRUE); 235744947dcSTom Erickson } else { 236744947dcSTom Erickson boolean_t is_metadata; 237744947dcSTom Erickson 238744947dcSTom Erickson DB_DNODE_ENTER(db); 239ad135b5dSChristopher Siden is_metadata = DMU_OT_IS_METADATA(DB_DNODE(db)->dn_type); 240744947dcSTom Erickson DB_DNODE_EXIT(db); 241744947dcSTom Erickson 242744947dcSTom Erickson return (is_metadata); 243744947dcSTom Erickson } 244744947dcSTom Erickson } 245744947dcSTom Erickson 246fa9e4066Sahrens void 247ea8dc4b6Seschrock dbuf_evict(dmu_buf_impl_t *db) 248ea8dc4b6Seschrock { 249ea8dc4b6Seschrock ASSERT(MUTEX_HELD(&db->db_mtx)); 250ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 251c717a561Smaybee ASSERT(db->db_data_pending == NULL); 252ea8dc4b6Seschrock 253ea8dc4b6Seschrock dbuf_clear(db); 254ea8dc4b6Seschrock dbuf_destroy(db); 255ea8dc4b6Seschrock } 256ea8dc4b6Seschrock 257ea8dc4b6Seschrock void 258fa9e4066Sahrens dbuf_init(void) 259fa9e4066Sahrens { 260ea8dc4b6Seschrock uint64_t hsize = 1ULL << 16; 261fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 262fa9e4066Sahrens int i; 263fa9e4066Sahrens 264fa9e4066Sahrens /* 265fa9e4066Sahrens * The hash table is big enough to fill all of physical memory 266ea8dc4b6Seschrock * with an average 4K block size. The table will take up 267ea8dc4b6Seschrock * totalmem*sizeof(void*)/4K (i.e. 2MB/GB with 8-byte pointers). 268fa9e4066Sahrens */ 269ea8dc4b6Seschrock while (hsize * 4096 < physmem * PAGESIZE) 270fa9e4066Sahrens hsize <<= 1; 271fa9e4066Sahrens 272ea8dc4b6Seschrock retry: 273fa9e4066Sahrens h->hash_table_mask = hsize - 1; 274ea8dc4b6Seschrock h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP); 275ea8dc4b6Seschrock if (h->hash_table == NULL) { 276ea8dc4b6Seschrock /* XXX - we should really return an error instead of assert */ 277ea8dc4b6Seschrock ASSERT(hsize > (1ULL << 10)); 278ea8dc4b6Seschrock hsize >>= 1; 279ea8dc4b6Seschrock goto retry; 280ea8dc4b6Seschrock } 281fa9e4066Sahrens 282fa9e4066Sahrens dbuf_cache = kmem_cache_create("dmu_buf_impl_t", 283fa9e4066Sahrens sizeof (dmu_buf_impl_t), 284fa9e4066Sahrens 0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0); 285fa9e4066Sahrens 286fa9e4066Sahrens for (i = 0; i < DBUF_MUTEXES; i++) 287fa9e4066Sahrens mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL); 288fa9e4066Sahrens } 289fa9e4066Sahrens 290fa9e4066Sahrens void 291fa9e4066Sahrens dbuf_fini(void) 292fa9e4066Sahrens { 293fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 294fa9e4066Sahrens int i; 295fa9e4066Sahrens 296fa9e4066Sahrens for (i = 0; i < DBUF_MUTEXES; i++) 297fa9e4066Sahrens mutex_destroy(&h->hash_mutexes[i]); 298fa9e4066Sahrens kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *)); 299fa9e4066Sahrens kmem_cache_destroy(dbuf_cache); 300fa9e4066Sahrens } 301fa9e4066Sahrens 302fa9e4066Sahrens /* 303fa9e4066Sahrens * Other stuff. 304fa9e4066Sahrens */ 305fa9e4066Sahrens 3069c9dc39aSek110237 #ifdef ZFS_DEBUG 307fa9e4066Sahrens static void 308fa9e4066Sahrens dbuf_verify(dmu_buf_impl_t *db) 309fa9e4066Sahrens { 310744947dcSTom Erickson dnode_t *dn; 311b24ab676SJeff Bonwick dbuf_dirty_record_t *dr; 312fa9e4066Sahrens 313fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 314fa9e4066Sahrens 315fa9e4066Sahrens if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY)) 316fa9e4066Sahrens return; 317fa9e4066Sahrens 318fa9e4066Sahrens ASSERT(db->db_objset != NULL); 319744947dcSTom Erickson DB_DNODE_ENTER(db); 320744947dcSTom Erickson dn = DB_DNODE(db); 321fa9e4066Sahrens if (dn == NULL) { 322fa9e4066Sahrens ASSERT(db->db_parent == NULL); 323fa9e4066Sahrens ASSERT(db->db_blkptr == NULL); 324fa9e4066Sahrens } else { 325fa9e4066Sahrens ASSERT3U(db->db.db_object, ==, dn->dn_object); 326fa9e4066Sahrens ASSERT3P(db->db_objset, ==, dn->dn_objset); 327fa9e4066Sahrens ASSERT3U(db->db_level, <, dn->dn_nlevels); 328744947dcSTom Erickson ASSERT(db->db_blkid == DMU_BONUS_BLKID || 329744947dcSTom Erickson db->db_blkid == DMU_SPILL_BLKID || 330744947dcSTom Erickson !list_is_empty(&dn->dn_dbufs)); 331fa9e4066Sahrens } 3320a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 333fa9e4066Sahrens ASSERT(dn != NULL); 3341934e92fSmaybee ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 3350a586ceaSMark Shellenbaum ASSERT3U(db->db.db_offset, ==, DMU_BONUS_BLKID); 3360a586ceaSMark Shellenbaum } else if (db->db_blkid == DMU_SPILL_BLKID) { 3370a586ceaSMark Shellenbaum ASSERT(dn != NULL); 3380a586ceaSMark Shellenbaum ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 339fb09f5aaSMadhav Suresh ASSERT0(db->db.db_offset); 340fa9e4066Sahrens } else { 341fa9e4066Sahrens ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size); 342fa9e4066Sahrens } 343fa9e4066Sahrens 344b24ab676SJeff Bonwick for (dr = db->db_data_pending; dr != NULL; dr = dr->dr_next) 345b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 346b24ab676SJeff Bonwick 347b24ab676SJeff Bonwick for (dr = db->db_last_dirty; dr != NULL; dr = dr->dr_next) 348b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 349b24ab676SJeff Bonwick 35088b7b0f2SMatthew Ahrens /* 35188b7b0f2SMatthew Ahrens * We can't assert that db_size matches dn_datablksz because it 35288b7b0f2SMatthew Ahrens * can be momentarily different when another thread is doing 35388b7b0f2SMatthew Ahrens * dnode_set_blksz(). 35488b7b0f2SMatthew Ahrens */ 35588b7b0f2SMatthew Ahrens if (db->db_level == 0 && db->db.db_object == DMU_META_DNODE_OBJECT) { 356b24ab676SJeff Bonwick dr = db->db_data_pending; 357fa9e4066Sahrens /* 35888b7b0f2SMatthew Ahrens * It should only be modified in syncing context, so 35988b7b0f2SMatthew Ahrens * make sure we only have one copy of the data. 360fa9e4066Sahrens */ 361c717a561Smaybee ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf); 362fa9e4066Sahrens } 363fa9e4066Sahrens 364fa9e4066Sahrens /* verify db->db_blkptr */ 365fa9e4066Sahrens if (db->db_blkptr) { 366fa9e4066Sahrens if (db->db_parent == dn->dn_dbuf) { 367fa9e4066Sahrens /* db is pointed to by the dnode */ 368fa9e4066Sahrens /* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */ 36914843421SMatthew Ahrens if (DMU_OBJECT_IS_SPECIAL(db->db.db_object)) 370fa9e4066Sahrens ASSERT(db->db_parent == NULL); 371fa9e4066Sahrens else 372fa9e4066Sahrens ASSERT(db->db_parent != NULL); 3730a586ceaSMark Shellenbaum if (db->db_blkid != DMU_SPILL_BLKID) 374fa9e4066Sahrens ASSERT3P(db->db_blkptr, ==, 375fa9e4066Sahrens &dn->dn_phys->dn_blkptr[db->db_blkid]); 376fa9e4066Sahrens } else { 377fa9e4066Sahrens /* db is pointed to by an indirect block */ 378fa9e4066Sahrens int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT; 379fa9e4066Sahrens ASSERT3U(db->db_parent->db_level, ==, db->db_level+1); 380fa9e4066Sahrens ASSERT3U(db->db_parent->db.db_object, ==, 381fa9e4066Sahrens db->db.db_object); 382fa9e4066Sahrens /* 383fa9e4066Sahrens * dnode_grow_indblksz() can make this fail if we don't 384fa9e4066Sahrens * have the struct_rwlock. XXX indblksz no longer 385fa9e4066Sahrens * grows. safe to do this now? 386fa9e4066Sahrens */ 387744947dcSTom Erickson if (RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 388fa9e4066Sahrens ASSERT3P(db->db_blkptr, ==, 389fa9e4066Sahrens ((blkptr_t *)db->db_parent->db.db_data + 390fa9e4066Sahrens db->db_blkid % epb)); 391fa9e4066Sahrens } 392fa9e4066Sahrens } 393fa9e4066Sahrens } 394fa9e4066Sahrens if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) && 3953f9d6ad7SLin Ling (db->db_buf == NULL || db->db_buf->b_data) && 3960a586ceaSMark Shellenbaum db->db.db_data && db->db_blkid != DMU_BONUS_BLKID && 397fa9e4066Sahrens db->db_state != DB_FILL && !dn->dn_free_txg) { 398fa9e4066Sahrens /* 399fa9e4066Sahrens * If the blkptr isn't set but they have nonzero data, 400fa9e4066Sahrens * it had better be dirty, otherwise we'll lose that 401fa9e4066Sahrens * data when we evict this buffer. 402fa9e4066Sahrens */ 403fa9e4066Sahrens if (db->db_dirtycnt == 0) { 404fa9e4066Sahrens uint64_t *buf = db->db.db_data; 405fa9e4066Sahrens int i; 406fa9e4066Sahrens 407fa9e4066Sahrens for (i = 0; i < db->db.db_size >> 3; i++) { 408fa9e4066Sahrens ASSERT(buf[i] == 0); 409fa9e4066Sahrens } 410fa9e4066Sahrens } 411fa9e4066Sahrens } 412744947dcSTom Erickson DB_DNODE_EXIT(db); 413fa9e4066Sahrens } 4149c9dc39aSek110237 #endif 415fa9e4066Sahrens 416fa9e4066Sahrens static void 417fa9e4066Sahrens dbuf_update_data(dmu_buf_impl_t *db) 418fa9e4066Sahrens { 419fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 420c717a561Smaybee if (db->db_level == 0 && db->db_user_data_ptr_ptr) { 421fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 422c717a561Smaybee *db->db_user_data_ptr_ptr = db->db.db_data; 423fa9e4066Sahrens } 424fa9e4066Sahrens } 425fa9e4066Sahrens 426fa9e4066Sahrens static void 427fa9e4066Sahrens dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf) 428fa9e4066Sahrens { 429fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 430ea8dc4b6Seschrock ASSERT(db->db_buf == NULL || !arc_has_callback(db->db_buf)); 431fa9e4066Sahrens db->db_buf = buf; 432ea8dc4b6Seschrock if (buf != NULL) { 433ea8dc4b6Seschrock ASSERT(buf->b_data != NULL); 434fa9e4066Sahrens db->db.db_data = buf->b_data; 435ea8dc4b6Seschrock if (!arc_released(buf)) 436ea8dc4b6Seschrock arc_set_callback(buf, dbuf_do_evict, db); 437fa9e4066Sahrens dbuf_update_data(db); 438ea8dc4b6Seschrock } else { 439ea8dc4b6Seschrock dbuf_evict_user(db); 440ea8dc4b6Seschrock db->db.db_data = NULL; 44182c9918fSTim Haley if (db->db_state != DB_NOFILL) 442ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 443ea8dc4b6Seschrock } 444fa9e4066Sahrens } 445fa9e4066Sahrens 446c242f9a0Schunli zhang - Sun Microsystems - Irvine United States /* 447c242f9a0Schunli zhang - Sun Microsystems - Irvine United States * Loan out an arc_buf for read. Return the loaned arc_buf. 448c242f9a0Schunli zhang - Sun Microsystems - Irvine United States */ 449c242f9a0Schunli zhang - Sun Microsystems - Irvine United States arc_buf_t * 450c242f9a0Schunli zhang - Sun Microsystems - Irvine United States dbuf_loan_arcbuf(dmu_buf_impl_t *db) 451c242f9a0Schunli zhang - Sun Microsystems - Irvine United States { 452c242f9a0Schunli zhang - Sun Microsystems - Irvine United States arc_buf_t *abuf; 453c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 454c242f9a0Schunli zhang - Sun Microsystems - Irvine United States mutex_enter(&db->db_mtx); 455c242f9a0Schunli zhang - Sun Microsystems - Irvine United States if (arc_released(db->db_buf) || refcount_count(&db->db_holds) > 1) { 456c242f9a0Schunli zhang - Sun Microsystems - Irvine United States int blksz = db->db.db_size; 457744947dcSTom Erickson spa_t *spa; 458744947dcSTom Erickson 459c242f9a0Schunli zhang - Sun Microsystems - Irvine United States mutex_exit(&db->db_mtx); 460744947dcSTom Erickson DB_GET_SPA(&spa, db); 461744947dcSTom Erickson abuf = arc_loan_buf(spa, blksz); 462c242f9a0Schunli zhang - Sun Microsystems - Irvine United States bcopy(db->db.db_data, abuf->b_data, blksz); 463c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } else { 464c242f9a0Schunli zhang - Sun Microsystems - Irvine United States abuf = db->db_buf; 465c242f9a0Schunli zhang - Sun Microsystems - Irvine United States arc_loan_inuse_buf(abuf, db); 466c242f9a0Schunli zhang - Sun Microsystems - Irvine United States dbuf_set_data(db, NULL); 467c242f9a0Schunli zhang - Sun Microsystems - Irvine United States mutex_exit(&db->db_mtx); 468c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } 469c242f9a0Schunli zhang - Sun Microsystems - Irvine United States return (abuf); 470c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } 471c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 472fa9e4066Sahrens uint64_t 473fa9e4066Sahrens dbuf_whichblock(dnode_t *dn, uint64_t offset) 474fa9e4066Sahrens { 475fa9e4066Sahrens if (dn->dn_datablkshift) { 476fa9e4066Sahrens return (offset >> dn->dn_datablkshift); 477fa9e4066Sahrens } else { 478fa9e4066Sahrens ASSERT3U(offset, <, dn->dn_datablksz); 479fa9e4066Sahrens return (0); 480fa9e4066Sahrens } 481fa9e4066Sahrens } 482fa9e4066Sahrens 483fa9e4066Sahrens static void 484fa9e4066Sahrens dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb) 485fa9e4066Sahrens { 486fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 487fa9e4066Sahrens 488fa9e4066Sahrens mutex_enter(&db->db_mtx); 489fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_READ); 490fa9e4066Sahrens /* 491fa9e4066Sahrens * All reads are synchronous, so we must have a hold on the dbuf 492fa9e4066Sahrens */ 493fa9e4066Sahrens ASSERT(refcount_count(&db->db_holds) > 0); 494ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 495fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 496c717a561Smaybee if (db->db_level == 0 && db->db_freed_in_flight) { 497fa9e4066Sahrens /* we were freed in flight; disregard any error */ 498fa9e4066Sahrens arc_release(buf, db); 499fa9e4066Sahrens bzero(buf->b_data, db->db.db_size); 5006b4acc8bSahrens arc_buf_freeze(buf); 501c717a561Smaybee db->db_freed_in_flight = FALSE; 502fa9e4066Sahrens dbuf_set_data(db, buf); 503fa9e4066Sahrens db->db_state = DB_CACHED; 504fa9e4066Sahrens } else if (zio == NULL || zio->io_error == 0) { 505fa9e4066Sahrens dbuf_set_data(db, buf); 506fa9e4066Sahrens db->db_state = DB_CACHED; 507fa9e4066Sahrens } else { 5080a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 509fa9e4066Sahrens ASSERT3P(db->db_buf, ==, NULL); 5103b2aab18SMatthew Ahrens VERIFY(arc_buf_remove_ref(buf, db)); 511ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 512fa9e4066Sahrens } 513fa9e4066Sahrens cv_broadcast(&db->db_changed); 5143f9d6ad7SLin Ling dbuf_rele_and_unlock(db, NULL); 515fa9e4066Sahrens } 516fa9e4066Sahrens 517ea8dc4b6Seschrock static void 51813506d1eSmaybee dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t *flags) 519fa9e4066Sahrens { 520744947dcSTom Erickson dnode_t *dn; 521744947dcSTom Erickson spa_t *spa; 522ea8dc4b6Seschrock zbookmark_t zb; 52313506d1eSmaybee uint32_t aflags = ARC_NOWAIT; 524fa9e4066Sahrens 525744947dcSTom Erickson DB_DNODE_ENTER(db); 526744947dcSTom Erickson dn = DB_DNODE(db); 527fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 528fa9e4066Sahrens /* We need the struct_rwlock to prevent db_blkptr from changing. */ 529088f3894Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 530ea8dc4b6Seschrock ASSERT(MUTEX_HELD(&db->db_mtx)); 531ea8dc4b6Seschrock ASSERT(db->db_state == DB_UNCACHED); 532ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 533fa9e4066Sahrens 5340a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 535cf04dda1SMark Maybee int bonuslen = MIN(dn->dn_bonuslen, dn->dn_phys->dn_bonuslen); 5361934e92fSmaybee 5371934e92fSmaybee ASSERT3U(bonuslen, <=, db->db.db_size); 538ea8dc4b6Seschrock db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN); 5395a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 5401934e92fSmaybee if (bonuslen < DN_MAX_BONUSLEN) 541ea8dc4b6Seschrock bzero(db->db.db_data, DN_MAX_BONUSLEN); 542cf04dda1SMark Maybee if (bonuslen) 543cf04dda1SMark Maybee bcopy(DN_BONUS(dn->dn_phys), db->db.db_data, bonuslen); 544744947dcSTom Erickson DB_DNODE_EXIT(db); 545ea8dc4b6Seschrock dbuf_update_data(db); 546fa9e4066Sahrens db->db_state = DB_CACHED; 547fa9e4066Sahrens mutex_exit(&db->db_mtx); 548fa9e4066Sahrens return; 549fa9e4066Sahrens } 550fa9e4066Sahrens 5511c8564a7SMark Maybee /* 5521c8564a7SMark Maybee * Recheck BP_IS_HOLE() after dnode_block_freed() in case dnode_sync() 5531c8564a7SMark Maybee * processes the delete record and clears the bp while we are waiting 5541c8564a7SMark Maybee * for the dn_mtx (resulting in a "no" from block_freed). 5551c8564a7SMark Maybee */ 556088f3894Sahrens if (db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr) || 5571c8564a7SMark Maybee (db->db_level == 0 && (dnode_block_freed(dn, db->db_blkid) || 5581c8564a7SMark Maybee BP_IS_HOLE(db->db_blkptr)))) { 559ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 560ad23a2dbSjohansen 561088f3894Sahrens dbuf_set_data(db, arc_buf_alloc(dn->dn_objset->os_spa, 562ad23a2dbSjohansen db->db.db_size, db, type)); 563744947dcSTom Erickson DB_DNODE_EXIT(db); 564fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 565fa9e4066Sahrens db->db_state = DB_CACHED; 56613506d1eSmaybee *flags |= DB_RF_CACHED; 567fa9e4066Sahrens mutex_exit(&db->db_mtx); 568fa9e4066Sahrens return; 569fa9e4066Sahrens } 570fa9e4066Sahrens 571744947dcSTom Erickson spa = dn->dn_objset->os_spa; 572744947dcSTom Erickson DB_DNODE_EXIT(db); 573744947dcSTom Erickson 574fa9e4066Sahrens db->db_state = DB_READ; 575fa9e4066Sahrens mutex_exit(&db->db_mtx); 576fa9e4066Sahrens 5773baa08fcSek110237 if (DBUF_IS_L2CACHEABLE(db)) 5783baa08fcSek110237 aflags |= ARC_L2CACHE; 579aad02571SSaso Kiselkov if (DBUF_IS_L2COMPRESSIBLE(db)) 580aad02571SSaso Kiselkov aflags |= ARC_L2COMPRESS; 5813baa08fcSek110237 582b24ab676SJeff Bonwick SET_BOOKMARK(&zb, db->db_objset->os_dsl_dataset ? 583b24ab676SJeff Bonwick db->db_objset->os_dsl_dataset->ds_object : DMU_META_OBJSET, 584b24ab676SJeff Bonwick db->db.db_object, db->db_level, db->db_blkid); 585ea8dc4b6Seschrock 586ea8dc4b6Seschrock dbuf_add_ref(db, NULL); 587088f3894Sahrens 5881b912ec7SGeorge Wilson (void) arc_read(zio, spa, db->db_blkptr, 589fa9e4066Sahrens dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ, 59013506d1eSmaybee (*flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED, 59113506d1eSmaybee &aflags, &zb); 59213506d1eSmaybee if (aflags & ARC_CACHED) 59313506d1eSmaybee *flags |= DB_RF_CACHED; 594fa9e4066Sahrens } 595fa9e4066Sahrens 596ea8dc4b6Seschrock int 597ea8dc4b6Seschrock dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags) 598fa9e4066Sahrens { 599ea8dc4b6Seschrock int err = 0; 600ea8dc4b6Seschrock int havepzio = (zio != NULL); 60113506d1eSmaybee int prefetch; 602744947dcSTom Erickson dnode_t *dn; 603fa9e4066Sahrens 604fa9e4066Sahrens /* 605fa9e4066Sahrens * We don't have to hold the mutex to check db_state because it 606fa9e4066Sahrens * can't be freed while we have a hold on the buffer. 607fa9e4066Sahrens */ 608fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 609fa9e4066Sahrens 61082c9918fSTim Haley if (db->db_state == DB_NOFILL) 611be6fd75aSMatthew Ahrens return (SET_ERROR(EIO)); 61282c9918fSTim Haley 613744947dcSTom Erickson DB_DNODE_ENTER(db); 614744947dcSTom Erickson dn = DB_DNODE(db); 615fa9e4066Sahrens if ((flags & DB_RF_HAVESTRUCT) == 0) 616744947dcSTom Erickson rw_enter(&dn->dn_struct_rwlock, RW_READER); 617fa9e4066Sahrens 6180a586ceaSMark Shellenbaum prefetch = db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 619744947dcSTom Erickson (flags & DB_RF_NOPREFETCH) == 0 && dn != NULL && 6203baa08fcSek110237 DBUF_IS_CACHEABLE(db); 62113506d1eSmaybee 622fa9e4066Sahrens mutex_enter(&db->db_mtx); 623ea8dc4b6Seschrock if (db->db_state == DB_CACHED) { 624ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 62513506d1eSmaybee if (prefetch) 626744947dcSTom Erickson dmu_zfetch(&dn->dn_zfetch, db->db.db_offset, 62713506d1eSmaybee db->db.db_size, TRUE); 628ea8dc4b6Seschrock if ((flags & DB_RF_HAVESTRUCT) == 0) 629744947dcSTom Erickson rw_exit(&dn->dn_struct_rwlock); 630744947dcSTom Erickson DB_DNODE_EXIT(db); 631ea8dc4b6Seschrock } else if (db->db_state == DB_UNCACHED) { 632744947dcSTom Erickson spa_t *spa = dn->dn_objset->os_spa; 633744947dcSTom Erickson 634744947dcSTom Erickson if (zio == NULL) 635744947dcSTom Erickson zio = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL); 63613506d1eSmaybee dbuf_read_impl(db, zio, &flags); 63713506d1eSmaybee 638ea8dc4b6Seschrock /* dbuf_read_impl has dropped db_mtx for us */ 639ea8dc4b6Seschrock 64013506d1eSmaybee if (prefetch) 641744947dcSTom Erickson dmu_zfetch(&dn->dn_zfetch, db->db.db_offset, 64213506d1eSmaybee db->db.db_size, flags & DB_RF_CACHED); 643ea8dc4b6Seschrock 644ea8dc4b6Seschrock if ((flags & DB_RF_HAVESTRUCT) == 0) 645744947dcSTom Erickson rw_exit(&dn->dn_struct_rwlock); 646744947dcSTom Erickson DB_DNODE_EXIT(db); 647ea8dc4b6Seschrock 648ea8dc4b6Seschrock if (!havepzio) 649ea8dc4b6Seschrock err = zio_wait(zio); 650ea8dc4b6Seschrock } else { 6513e30c24aSWill Andrews /* 6523e30c24aSWill Andrews * Another reader came in while the dbuf was in flight 6533e30c24aSWill Andrews * between UNCACHED and CACHED. Either a writer will finish 6543e30c24aSWill Andrews * writing the buffer (sending the dbuf to CACHED) or the 6553e30c24aSWill Andrews * first reader's request will reach the read_done callback 6563e30c24aSWill Andrews * and send the dbuf to CACHED. Otherwise, a failure 6573e30c24aSWill Andrews * occurred and the dbuf went to UNCACHED. 6583e30c24aSWill Andrews */ 65913506d1eSmaybee mutex_exit(&db->db_mtx); 66013506d1eSmaybee if (prefetch) 661744947dcSTom Erickson dmu_zfetch(&dn->dn_zfetch, db->db.db_offset, 66213506d1eSmaybee db->db.db_size, TRUE); 663ea8dc4b6Seschrock if ((flags & DB_RF_HAVESTRUCT) == 0) 664744947dcSTom Erickson rw_exit(&dn->dn_struct_rwlock); 665744947dcSTom Erickson DB_DNODE_EXIT(db); 66613506d1eSmaybee 6673e30c24aSWill Andrews /* Skip the wait per the caller's request. */ 66813506d1eSmaybee mutex_enter(&db->db_mtx); 669ea8dc4b6Seschrock if ((flags & DB_RF_NEVERWAIT) == 0) { 670ea8dc4b6Seschrock while (db->db_state == DB_READ || 671ea8dc4b6Seschrock db->db_state == DB_FILL) { 672fa9e4066Sahrens ASSERT(db->db_state == DB_READ || 673fa9e4066Sahrens (flags & DB_RF_HAVESTRUCT) == 0); 674fa9e4066Sahrens cv_wait(&db->db_changed, &db->db_mtx); 675fa9e4066Sahrens } 676ea8dc4b6Seschrock if (db->db_state == DB_UNCACHED) 677be6fd75aSMatthew Ahrens err = SET_ERROR(EIO); 678ea8dc4b6Seschrock } 679fa9e4066Sahrens mutex_exit(&db->db_mtx); 680fa9e4066Sahrens } 681fa9e4066Sahrens 682ea8dc4b6Seschrock ASSERT(err || havepzio || db->db_state == DB_CACHED); 683ea8dc4b6Seschrock return (err); 684fa9e4066Sahrens } 685fa9e4066Sahrens 686fa9e4066Sahrens static void 687fa9e4066Sahrens dbuf_noread(dmu_buf_impl_t *db) 688fa9e4066Sahrens { 689fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 6900a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 691fa9e4066Sahrens mutex_enter(&db->db_mtx); 692fa9e4066Sahrens while (db->db_state == DB_READ || db->db_state == DB_FILL) 693fa9e4066Sahrens cv_wait(&db->db_changed, &db->db_mtx); 694fa9e4066Sahrens if (db->db_state == DB_UNCACHED) { 695ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 696744947dcSTom Erickson spa_t *spa; 697ad23a2dbSjohansen 698ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 699fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 700744947dcSTom Erickson DB_GET_SPA(&spa, db); 701744947dcSTom Erickson dbuf_set_data(db, arc_buf_alloc(spa, db->db.db_size, db, type)); 702fa9e4066Sahrens db->db_state = DB_FILL; 70382c9918fSTim Haley } else if (db->db_state == DB_NOFILL) { 70482c9918fSTim Haley dbuf_set_data(db, NULL); 705fa9e4066Sahrens } else { 706fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_CACHED); 707fa9e4066Sahrens } 708fa9e4066Sahrens mutex_exit(&db->db_mtx); 709fa9e4066Sahrens } 710fa9e4066Sahrens 711fa9e4066Sahrens /* 712fa9e4066Sahrens * This is our just-in-time copy function. It makes a copy of 713fa9e4066Sahrens * buffers, that have been modified in a previous transaction 714fa9e4066Sahrens * group, before we modify them in the current active group. 715fa9e4066Sahrens * 716fa9e4066Sahrens * This function is used in two places: when we are dirtying a 717fa9e4066Sahrens * buffer for the first time in a txg, and when we are freeing 718fa9e4066Sahrens * a range in a dnode that includes this buffer. 719fa9e4066Sahrens * 720fa9e4066Sahrens * Note that when we are called from dbuf_free_range() we do 721fa9e4066Sahrens * not put a hold on the buffer, we just traverse the active 722fa9e4066Sahrens * dbuf list for the dnode. 723fa9e4066Sahrens */ 724fa9e4066Sahrens static void 725fa9e4066Sahrens dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg) 726fa9e4066Sahrens { 727c717a561Smaybee dbuf_dirty_record_t *dr = db->db_last_dirty; 728fa9e4066Sahrens 729fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 730fa9e4066Sahrens ASSERT(db->db.db_data != NULL); 731c717a561Smaybee ASSERT(db->db_level == 0); 732c717a561Smaybee ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT); 733fa9e4066Sahrens 7344d31c452Smaybee if (dr == NULL || 7354d31c452Smaybee (dr->dt.dl.dr_data != 7360a586ceaSMark Shellenbaum ((db->db_blkid == DMU_BONUS_BLKID) ? db->db.db_data : db->db_buf))) 737fa9e4066Sahrens return; 738fa9e4066Sahrens 739fa9e4066Sahrens /* 740c717a561Smaybee * If the last dirty record for this dbuf has not yet synced 741c717a561Smaybee * and its referencing the dbuf data, either: 742c717a561Smaybee * reset the reference to point to a new copy, 743c717a561Smaybee * or (if there a no active holders) 744c717a561Smaybee * just null out the current db_data pointer. 745fa9e4066Sahrens */ 746c717a561Smaybee ASSERT(dr->dr_txg >= txg - 2); 7470a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 748c717a561Smaybee /* Note that the data bufs here are zio_bufs */ 749c717a561Smaybee dr->dt.dl.dr_data = zio_buf_alloc(DN_MAX_BONUSLEN); 7505a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 751c717a561Smaybee bcopy(db->db.db_data, dr->dt.dl.dr_data, DN_MAX_BONUSLEN); 752c717a561Smaybee } else if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 753ea8dc4b6Seschrock int size = db->db.db_size; 754c717a561Smaybee arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 755744947dcSTom Erickson spa_t *spa; 756744947dcSTom Erickson 757744947dcSTom Erickson DB_GET_SPA(&spa, db); 758744947dcSTom Erickson dr->dt.dl.dr_data = arc_buf_alloc(spa, size, db, type); 759c717a561Smaybee bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size); 760fa9e4066Sahrens } else { 761ea8dc4b6Seschrock dbuf_set_data(db, NULL); 762fa9e4066Sahrens } 763fa9e4066Sahrens } 764ea8dc4b6Seschrock 765fa9e4066Sahrens void 766c717a561Smaybee dbuf_unoverride(dbuf_dirty_record_t *dr) 767fa9e4066Sahrens { 768c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 769b24ab676SJeff Bonwick blkptr_t *bp = &dr->dt.dl.dr_overridden_by; 770c717a561Smaybee uint64_t txg = dr->dr_txg; 771c5c6ffa0Smaybee 772c717a561Smaybee ASSERT(MUTEX_HELD(&db->db_mtx)); 773c717a561Smaybee ASSERT(dr->dt.dl.dr_override_state != DR_IN_DMU_SYNC); 774c717a561Smaybee ASSERT(db->db_level == 0); 775c717a561Smaybee 7760a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID || 777c717a561Smaybee dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN) 778c717a561Smaybee return; 779c717a561Smaybee 780b24ab676SJeff Bonwick ASSERT(db->db_data_pending != dr); 781b24ab676SJeff Bonwick 782fa9e4066Sahrens /* free this block */ 78380901aeaSGeorge Wilson if (!BP_IS_HOLE(bp) && !dr->dt.dl.dr_nopwrite) { 784744947dcSTom Erickson spa_t *spa; 785b24ab676SJeff Bonwick 786744947dcSTom Erickson DB_GET_SPA(&spa, db); 787744947dcSTom Erickson zio_free(spa, txg, bp); 788744947dcSTom Erickson } 789c717a561Smaybee dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 79080901aeaSGeorge Wilson dr->dt.dl.dr_nopwrite = B_FALSE; 79180901aeaSGeorge Wilson 7926b4acc8bSahrens /* 7936b4acc8bSahrens * Release the already-written buffer, so we leave it in 7946b4acc8bSahrens * a consistent dirty state. Note that all callers are 7956b4acc8bSahrens * modifying the buffer, so they will immediately do 7966b4acc8bSahrens * another (redundant) arc_release(). Therefore, leave 7976b4acc8bSahrens * the buf thawed to save the effort of freezing & 7986b4acc8bSahrens * immediately re-thawing it. 7996b4acc8bSahrens */ 800c717a561Smaybee arc_release(dr->dt.dl.dr_data, db); 801fa9e4066Sahrens } 802fa9e4066Sahrens 803cdb0ab79Smaybee /* 804cdb0ab79Smaybee * Evict (if its unreferenced) or clear (if its referenced) any level-0 805cdb0ab79Smaybee * data blocks in the free range, so that any future readers will find 8062f3d8780SMatthew Ahrens * empty blocks. Also, if we happen across any level-1 dbufs in the 807cdb0ab79Smaybee * range that have not already been marked dirty, mark them dirty so 808cdb0ab79Smaybee * they stay in memory. 8092f3d8780SMatthew Ahrens * 8102f3d8780SMatthew Ahrens * This is a no-op if the dataset is in the middle of an incremental 8112f3d8780SMatthew Ahrens * receive; see comment below for details. 812cdb0ab79Smaybee */ 813fa9e4066Sahrens void 814cdb0ab79Smaybee dbuf_free_range(dnode_t *dn, uint64_t start, uint64_t end, dmu_tx_t *tx) 815fa9e4066Sahrens { 816fa9e4066Sahrens dmu_buf_impl_t *db, *db_next; 817fa9e4066Sahrens uint64_t txg = tx->tx_txg; 818cdb0ab79Smaybee int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 819cdb0ab79Smaybee uint64_t first_l1 = start >> epbs; 820cdb0ab79Smaybee uint64_t last_l1 = end >> epbs; 821fa9e4066Sahrens 8220a586ceaSMark Shellenbaum if (end > dn->dn_maxblkid && (end != DMU_SPILL_BLKID)) { 823cdb0ab79Smaybee end = dn->dn_maxblkid; 824cdb0ab79Smaybee last_l1 = end >> epbs; 825cdb0ab79Smaybee } 826cdb0ab79Smaybee dprintf_dnode(dn, "start=%llu end=%llu\n", start, end); 8272f3d8780SMatthew Ahrens 828*713d6c20SMatthew Ahrens mutex_enter(&dn->dn_dbufs_mtx); 829*713d6c20SMatthew Ahrens if (start >= dn->dn_unlisted_l0_blkid * dn->dn_datablksz) { 830*713d6c20SMatthew Ahrens /* There can't be any dbufs in this range; no need to search. */ 831*713d6c20SMatthew Ahrens mutex_exit(&dn->dn_dbufs_mtx); 8322f3d8780SMatthew Ahrens return; 833*713d6c20SMatthew Ahrens } else if (dmu_objset_is_receiving(dn->dn_objset)) { 834*713d6c20SMatthew Ahrens /* 835*713d6c20SMatthew Ahrens * If we are receiving, we expect there to be no dbufs in 836*713d6c20SMatthew Ahrens * the range to be freed, because receive modifies each 837*713d6c20SMatthew Ahrens * block at most once, and in offset order. If this is 838*713d6c20SMatthew Ahrens * not the case, it can lead to performance problems, 839*713d6c20SMatthew Ahrens * so note that we unexpectedly took the slow path. 840*713d6c20SMatthew Ahrens */ 841*713d6c20SMatthew Ahrens atomic_inc_64(&zfs_free_range_recv_miss); 8422f3d8780SMatthew Ahrens } 8432f3d8780SMatthew Ahrens 844fa9e4066Sahrens for (db = list_head(&dn->dn_dbufs); db; db = db_next) { 845fa9e4066Sahrens db_next = list_next(&dn->dn_dbufs, db); 8460a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 847cdb0ab79Smaybee 848cdb0ab79Smaybee if (db->db_level == 1 && 849cdb0ab79Smaybee db->db_blkid >= first_l1 && db->db_blkid <= last_l1) { 850cdb0ab79Smaybee mutex_enter(&db->db_mtx); 851cdb0ab79Smaybee if (db->db_last_dirty && 852cdb0ab79Smaybee db->db_last_dirty->dr_txg < txg) { 853cdb0ab79Smaybee dbuf_add_ref(db, FTAG); 854cdb0ab79Smaybee mutex_exit(&db->db_mtx); 855cdb0ab79Smaybee dbuf_will_dirty(db, tx); 856cdb0ab79Smaybee dbuf_rele(db, FTAG); 857cdb0ab79Smaybee } else { 858cdb0ab79Smaybee mutex_exit(&db->db_mtx); 859cdb0ab79Smaybee } 860cdb0ab79Smaybee } 861cdb0ab79Smaybee 862ea8dc4b6Seschrock if (db->db_level != 0) 863fa9e4066Sahrens continue; 864fa9e4066Sahrens dprintf_dbuf(db, "found buf %s\n", ""); 865cdb0ab79Smaybee if (db->db_blkid < start || db->db_blkid > end) 866fa9e4066Sahrens continue; 867fa9e4066Sahrens 868fa9e4066Sahrens /* found a level 0 buffer in the range */ 869fa9e4066Sahrens mutex_enter(&db->db_mtx); 8703b2aab18SMatthew Ahrens if (dbuf_undirty(db, tx)) { 8713b2aab18SMatthew Ahrens /* mutex has been dropped and dbuf destroyed */ 8723b2aab18SMatthew Ahrens continue; 8733b2aab18SMatthew Ahrens } 8743b2aab18SMatthew Ahrens 875ea8dc4b6Seschrock if (db->db_state == DB_UNCACHED || 87682c9918fSTim Haley db->db_state == DB_NOFILL || 877ea8dc4b6Seschrock db->db_state == DB_EVICTING) { 878fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 879fa9e4066Sahrens mutex_exit(&db->db_mtx); 880fa9e4066Sahrens continue; 881fa9e4066Sahrens } 882c543ec06Sahrens if (db->db_state == DB_READ || db->db_state == DB_FILL) { 883c543ec06Sahrens /* will be handled in dbuf_read_done or dbuf_rele */ 884c717a561Smaybee db->db_freed_in_flight = TRUE; 885fa9e4066Sahrens mutex_exit(&db->db_mtx); 886fa9e4066Sahrens continue; 887fa9e4066Sahrens } 888ea8dc4b6Seschrock if (refcount_count(&db->db_holds) == 0) { 889ea8dc4b6Seschrock ASSERT(db->db_buf); 890ea8dc4b6Seschrock dbuf_clear(db); 891ea8dc4b6Seschrock continue; 892ea8dc4b6Seschrock } 893c717a561Smaybee /* The dbuf is referenced */ 894fa9e4066Sahrens 895c717a561Smaybee if (db->db_last_dirty != NULL) { 896c717a561Smaybee dbuf_dirty_record_t *dr = db->db_last_dirty; 897c717a561Smaybee 898c717a561Smaybee if (dr->dr_txg == txg) { 899ea8dc4b6Seschrock /* 900c717a561Smaybee * This buffer is "in-use", re-adjust the file 901c717a561Smaybee * size to reflect that this buffer may 902c717a561Smaybee * contain new data when we sync. 903ea8dc4b6Seschrock */ 90406e0070dSMark Shellenbaum if (db->db_blkid != DMU_SPILL_BLKID && 90506e0070dSMark Shellenbaum db->db_blkid > dn->dn_maxblkid) 90644eda4d7Smaybee dn->dn_maxblkid = db->db_blkid; 907c717a561Smaybee dbuf_unoverride(dr); 908c717a561Smaybee } else { 909c717a561Smaybee /* 910c717a561Smaybee * This dbuf is not dirty in the open context. 911c717a561Smaybee * Either uncache it (if its not referenced in 912c717a561Smaybee * the open context) or reset its contents to 913c717a561Smaybee * empty. 914c717a561Smaybee */ 915c717a561Smaybee dbuf_fix_old_data(db, txg); 91644eda4d7Smaybee } 917c717a561Smaybee } 918c717a561Smaybee /* clear the contents if its cached */ 919ea8dc4b6Seschrock if (db->db_state == DB_CACHED) { 920ea8dc4b6Seschrock ASSERT(db->db.db_data != NULL); 921fa9e4066Sahrens arc_release(db->db_buf, db); 922fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 9236b4acc8bSahrens arc_buf_freeze(db->db_buf); 924fa9e4066Sahrens } 925ea8dc4b6Seschrock 926fa9e4066Sahrens mutex_exit(&db->db_mtx); 927fa9e4066Sahrens } 928fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 929fa9e4066Sahrens } 930fa9e4066Sahrens 931fa9e4066Sahrens static int 9321934e92fSmaybee dbuf_block_freeable(dmu_buf_impl_t *db) 933fa9e4066Sahrens { 934fa9e4066Sahrens dsl_dataset_t *ds = db->db_objset->os_dsl_dataset; 935fa9e4066Sahrens uint64_t birth_txg = 0; 936fa9e4066Sahrens 937fa9e4066Sahrens /* 938fa9e4066Sahrens * We don't need any locking to protect db_blkptr: 939c717a561Smaybee * If it's syncing, then db_last_dirty will be set 940c717a561Smaybee * so we'll ignore db_blkptr. 941fa9e4066Sahrens */ 942c717a561Smaybee ASSERT(MUTEX_HELD(&db->db_mtx)); 943c717a561Smaybee if (db->db_last_dirty) 944c717a561Smaybee birth_txg = db->db_last_dirty->dr_txg; 945fa9e4066Sahrens else if (db->db_blkptr) 946fa9e4066Sahrens birth_txg = db->db_blkptr->blk_birth; 947fa9e4066Sahrens 948837b568bSGeorge Wilson /* 949837b568bSGeorge Wilson * If we don't exist or are in a snapshot, we can't be freed. 950837b568bSGeorge Wilson * Don't pass the bp to dsl_dataset_block_freeable() since we 951837b568bSGeorge Wilson * are holding the db_mtx lock and might deadlock if we are 952837b568bSGeorge Wilson * prefetching a dedup-ed block. 953837b568bSGeorge Wilson */ 954fa9e4066Sahrens if (birth_txg) 9551934e92fSmaybee return (ds == NULL || 956837b568bSGeorge Wilson dsl_dataset_block_freeable(ds, NULL, birth_txg)); 957fa9e4066Sahrens else 9581934e92fSmaybee return (FALSE); 959fa9e4066Sahrens } 960fa9e4066Sahrens 961fa9e4066Sahrens void 962fa9e4066Sahrens dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx) 963fa9e4066Sahrens { 964fa9e4066Sahrens arc_buf_t *buf, *obuf; 965fa9e4066Sahrens int osize = db->db.db_size; 966ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 967744947dcSTom Erickson dnode_t *dn; 968fa9e4066Sahrens 9690a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 970ea8dc4b6Seschrock 971744947dcSTom Erickson DB_DNODE_ENTER(db); 972744947dcSTom Erickson dn = DB_DNODE(db); 973744947dcSTom Erickson 974fa9e4066Sahrens /* XXX does *this* func really need the lock? */ 975744947dcSTom Erickson ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 976fa9e4066Sahrens 977fa9e4066Sahrens /* 978fa9e4066Sahrens * This call to dbuf_will_dirty() with the dn_struct_rwlock held 979fa9e4066Sahrens * is OK, because there can be no other references to the db 980fa9e4066Sahrens * when we are changing its size, so no concurrent DB_FILL can 981fa9e4066Sahrens * be happening. 982fa9e4066Sahrens */ 983ea8dc4b6Seschrock /* 984ea8dc4b6Seschrock * XXX we should be doing a dbuf_read, checking the return 985ea8dc4b6Seschrock * value and returning that up to our callers 986ea8dc4b6Seschrock */ 987fa9e4066Sahrens dbuf_will_dirty(db, tx); 988fa9e4066Sahrens 989fa9e4066Sahrens /* create the data buffer for the new block */ 990744947dcSTom Erickson buf = arc_buf_alloc(dn->dn_objset->os_spa, size, db, type); 991fa9e4066Sahrens 992fa9e4066Sahrens /* copy old block data to the new block */ 993fa9e4066Sahrens obuf = db->db_buf; 994f65e61c0Sahrens bcopy(obuf->b_data, buf->b_data, MIN(osize, size)); 995fa9e4066Sahrens /* zero the remainder */ 996f65e61c0Sahrens if (size > osize) 997fa9e4066Sahrens bzero((uint8_t *)buf->b_data + osize, size - osize); 998fa9e4066Sahrens 999fa9e4066Sahrens mutex_enter(&db->db_mtx); 1000fa9e4066Sahrens dbuf_set_data(db, buf); 10013b2aab18SMatthew Ahrens VERIFY(arc_buf_remove_ref(obuf, db)); 1002fa9e4066Sahrens db->db.db_size = size; 1003fa9e4066Sahrens 1004c717a561Smaybee if (db->db_level == 0) { 1005c717a561Smaybee ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg); 1006c717a561Smaybee db->db_last_dirty->dt.dl.dr_data = buf; 1007c717a561Smaybee } 1008fa9e4066Sahrens mutex_exit(&db->db_mtx); 1009fa9e4066Sahrens 1010744947dcSTom Erickson dnode_willuse_space(dn, size-osize, tx); 1011744947dcSTom Erickson DB_DNODE_EXIT(db); 1012fa9e4066Sahrens } 1013fa9e4066Sahrens 10143f9d6ad7SLin Ling void 10153f9d6ad7SLin Ling dbuf_release_bp(dmu_buf_impl_t *db) 10163f9d6ad7SLin Ling { 1017744947dcSTom Erickson objset_t *os; 10183f9d6ad7SLin Ling 1019744947dcSTom Erickson DB_GET_OBJSET(&os, db); 10203f9d6ad7SLin Ling ASSERT(dsl_pool_sync_context(dmu_objset_pool(os))); 10213f9d6ad7SLin Ling ASSERT(arc_released(os->os_phys_buf) || 10223f9d6ad7SLin Ling list_link_active(&os->os_dsl_dataset->ds_synced_link)); 10233f9d6ad7SLin Ling ASSERT(db->db_parent == NULL || arc_released(db->db_parent->db_buf)); 10243f9d6ad7SLin Ling 10251b912ec7SGeorge Wilson (void) arc_release(db->db_buf, db); 10263f9d6ad7SLin Ling } 10273f9d6ad7SLin Ling 1028c717a561Smaybee dbuf_dirty_record_t * 1029fa9e4066Sahrens dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1030fa9e4066Sahrens { 1031744947dcSTom Erickson dnode_t *dn; 1032744947dcSTom Erickson objset_t *os; 1033c717a561Smaybee dbuf_dirty_record_t **drp, *dr; 1034fa9e4066Sahrens int drop_struct_lock = FALSE; 1035d3469faaSMark Maybee boolean_t do_free_accounting = B_FALSE; 1036fa9e4066Sahrens int txgoff = tx->tx_txg & TXG_MASK; 1037fa9e4066Sahrens 1038fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 1039fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 10409c9dc39aSek110237 DMU_TX_DIRTY_BUF(tx, db); 1041fa9e4066Sahrens 1042744947dcSTom Erickson DB_DNODE_ENTER(db); 1043744947dcSTom Erickson dn = DB_DNODE(db); 1044fa9e4066Sahrens /* 1045fa9e4066Sahrens * Shouldn't dirty a regular buffer in syncing context. Private 1046fa9e4066Sahrens * objects may be dirtied in syncing context, but only if they 1047fa9e4066Sahrens * were already pre-dirtied in open context. 1048fa9e4066Sahrens */ 1049c717a561Smaybee ASSERT(!dmu_tx_is_syncing(tx) || 1050c717a561Smaybee BP_IS_HOLE(dn->dn_objset->os_rootbp) || 105114843421SMatthew Ahrens DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 105214843421SMatthew Ahrens dn->dn_objset->os_dsl_dataset == NULL); 1053fa9e4066Sahrens /* 1054fa9e4066Sahrens * We make this assert for private objects as well, but after we 1055fa9e4066Sahrens * check if we're already dirty. They are allowed to re-dirty 1056fa9e4066Sahrens * in syncing context. 1057fa9e4066Sahrens */ 1058ea8dc4b6Seschrock ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 1059c717a561Smaybee dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 1060fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 1061fa9e4066Sahrens 1062fa9e4066Sahrens mutex_enter(&db->db_mtx); 1063fa9e4066Sahrens /* 1064c717a561Smaybee * XXX make this true for indirects too? The problem is that 1065c717a561Smaybee * transactions created with dmu_tx_create_assigned() from 1066c717a561Smaybee * syncing context don't bother holding ahead. 1067fa9e4066Sahrens */ 1068c717a561Smaybee ASSERT(db->db_level != 0 || 106982c9918fSTim Haley db->db_state == DB_CACHED || db->db_state == DB_FILL || 107082c9918fSTim Haley db->db_state == DB_NOFILL); 1071fa9e4066Sahrens 1072fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 1073fa9e4066Sahrens /* 1074fa9e4066Sahrens * Don't set dirtyctx to SYNC if we're just modifying this as we 1075fa9e4066Sahrens * initialize the objset. 1076fa9e4066Sahrens */ 1077fa9e4066Sahrens if (dn->dn_dirtyctx == DN_UNDIRTIED && 1078c717a561Smaybee !BP_IS_HOLE(dn->dn_objset->os_rootbp)) { 1079fa9e4066Sahrens dn->dn_dirtyctx = 1080fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN); 1081fa9e4066Sahrens ASSERT(dn->dn_dirtyctx_firstset == NULL); 1082fa9e4066Sahrens dn->dn_dirtyctx_firstset = kmem_alloc(1, KM_SLEEP); 1083fa9e4066Sahrens } 1084fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1085fa9e4066Sahrens 10860a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) 10870a586ceaSMark Shellenbaum dn->dn_have_spill = B_TRUE; 10880a586ceaSMark Shellenbaum 1089fa9e4066Sahrens /* 1090fa9e4066Sahrens * If this buffer is already dirty, we're done. 1091fa9e4066Sahrens */ 1092c717a561Smaybee drp = &db->db_last_dirty; 1093c717a561Smaybee ASSERT(*drp == NULL || (*drp)->dr_txg <= tx->tx_txg || 1094c717a561Smaybee db->db.db_object == DMU_META_DNODE_OBJECT); 10957e2186e3Sbonwick while ((dr = *drp) != NULL && dr->dr_txg > tx->tx_txg) 10967e2186e3Sbonwick drp = &dr->dr_next; 10977e2186e3Sbonwick if (dr && dr->dr_txg == tx->tx_txg) { 1098744947dcSTom Erickson DB_DNODE_EXIT(db); 1099744947dcSTom Erickson 11000a586ceaSMark Shellenbaum if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID) { 1101c717a561Smaybee /* 1102c717a561Smaybee * If this buffer has already been written out, 1103c717a561Smaybee * we now need to reset its state. 1104c717a561Smaybee */ 11057e2186e3Sbonwick dbuf_unoverride(dr); 1106b24ab676SJeff Bonwick if (db->db.db_object != DMU_META_DNODE_OBJECT && 1107b24ab676SJeff Bonwick db->db_state != DB_NOFILL) 11086b4acc8bSahrens arc_buf_thaw(db->db_buf); 1109c717a561Smaybee } 1110fa9e4066Sahrens mutex_exit(&db->db_mtx); 11117e2186e3Sbonwick return (dr); 1112fa9e4066Sahrens } 1113fa9e4066Sahrens 1114fa9e4066Sahrens /* 1115fa9e4066Sahrens * Only valid if not already dirty. 1116fa9e4066Sahrens */ 111714843421SMatthew Ahrens ASSERT(dn->dn_object == 0 || 111814843421SMatthew Ahrens dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 1119fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 1120fa9e4066Sahrens 1121fa9e4066Sahrens ASSERT3U(dn->dn_nlevels, >, db->db_level); 1122fa9e4066Sahrens ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) || 1123fa9e4066Sahrens dn->dn_phys->dn_nlevels > db->db_level || 1124fa9e4066Sahrens dn->dn_next_nlevels[txgoff] > db->db_level || 1125fa9e4066Sahrens dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level || 1126fa9e4066Sahrens dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level); 1127fa9e4066Sahrens 1128fa9e4066Sahrens /* 1129fa9e4066Sahrens * We should only be dirtying in syncing context if it's the 113014843421SMatthew Ahrens * mos or we're initializing the os or it's a special object. 113114843421SMatthew Ahrens * However, we are allowed to dirty in syncing context provided 113214843421SMatthew Ahrens * we already dirtied it in open context. Hence we must make 113314843421SMatthew Ahrens * this assertion only if we're not already dirty. 1134fa9e4066Sahrens */ 1135744947dcSTom Erickson os = dn->dn_objset; 113614843421SMatthew Ahrens ASSERT(!dmu_tx_is_syncing(tx) || DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 113714843421SMatthew Ahrens os->os_dsl_dataset == NULL || BP_IS_HOLE(os->os_rootbp)); 1138fa9e4066Sahrens ASSERT(db->db.db_size != 0); 1139fa9e4066Sahrens 1140fa9e4066Sahrens dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1141fa9e4066Sahrens 11420a586ceaSMark Shellenbaum if (db->db_blkid != DMU_BONUS_BLKID) { 11431934e92fSmaybee /* 11441934e92fSmaybee * Update the accounting. 1145d3469faaSMark Maybee * Note: we delay "free accounting" until after we drop 1146d3469faaSMark Maybee * the db_mtx. This keeps us from grabbing other locks 1147b24ab676SJeff Bonwick * (and possibly deadlocking) in bp_get_dsize() while 1148d3469faaSMark Maybee * also holding the db_mtx. 11491934e92fSmaybee */ 11501934e92fSmaybee dnode_willuse_space(dn, db->db.db_size, tx); 1151d3469faaSMark Maybee do_free_accounting = dbuf_block_freeable(db); 11521934e92fSmaybee } 11531934e92fSmaybee 1154ea8dc4b6Seschrock /* 1155ea8dc4b6Seschrock * If this buffer is dirty in an old transaction group we need 1156ea8dc4b6Seschrock * to make a copy of it so that the changes we make in this 1157ea8dc4b6Seschrock * transaction group won't leak out when we sync the older txg. 1158ea8dc4b6Seschrock */ 1159c717a561Smaybee dr = kmem_zalloc(sizeof (dbuf_dirty_record_t), KM_SLEEP); 1160c717a561Smaybee if (db->db_level == 0) { 1161c717a561Smaybee void *data_old = db->db_buf; 1162c717a561Smaybee 116382c9918fSTim Haley if (db->db_state != DB_NOFILL) { 11640a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 1165c717a561Smaybee dbuf_fix_old_data(db, tx->tx_txg); 1166c717a561Smaybee data_old = db->db.db_data; 1167c717a561Smaybee } else if (db->db.db_object != DMU_META_DNODE_OBJECT) { 1168fa9e4066Sahrens /* 116982c9918fSTim Haley * Release the data buffer from the cache so 117082c9918fSTim Haley * that we can modify it without impacting 117182c9918fSTim Haley * possible other users of this cached data 117282c9918fSTim Haley * block. Note that indirect blocks and 117382c9918fSTim Haley * private objects are not released until the 117482c9918fSTim Haley * syncing state (since they are only modified 117582c9918fSTim Haley * then). 1176fa9e4066Sahrens */ 1177fa9e4066Sahrens arc_release(db->db_buf, db); 1178fa9e4066Sahrens dbuf_fix_old_data(db, tx->tx_txg); 1179c717a561Smaybee data_old = db->db_buf; 1180fa9e4066Sahrens } 1181c717a561Smaybee ASSERT(data_old != NULL); 118282c9918fSTim Haley } 1183c717a561Smaybee dr->dt.dl.dr_data = data_old; 1184c717a561Smaybee } else { 1185c717a561Smaybee mutex_init(&dr->dt.di.dr_mtx, NULL, MUTEX_DEFAULT, NULL); 1186c717a561Smaybee list_create(&dr->dt.di.dr_children, 1187c717a561Smaybee sizeof (dbuf_dirty_record_t), 1188c717a561Smaybee offsetof(dbuf_dirty_record_t, dr_dirty_node)); 1189fa9e4066Sahrens } 1190c717a561Smaybee dr->dr_dbuf = db; 1191c717a561Smaybee dr->dr_txg = tx->tx_txg; 1192c717a561Smaybee dr->dr_next = *drp; 1193c717a561Smaybee *drp = dr; 1194fa9e4066Sahrens 1195fa9e4066Sahrens /* 1196fa9e4066Sahrens * We could have been freed_in_flight between the dbuf_noread 1197fa9e4066Sahrens * and dbuf_dirty. We win, as though the dbuf_noread() had 1198fa9e4066Sahrens * happened after the free. 1199fa9e4066Sahrens */ 12000a586ceaSMark Shellenbaum if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 12010a586ceaSMark Shellenbaum db->db_blkid != DMU_SPILL_BLKID) { 1202c717a561Smaybee mutex_enter(&dn->dn_mtx); 1203fa9e4066Sahrens dnode_clear_range(dn, db->db_blkid, 1, tx); 1204fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1205c717a561Smaybee db->db_freed_in_flight = FALSE; 1206c717a561Smaybee } 1207fa9e4066Sahrens 1208fa9e4066Sahrens /* 1209fa9e4066Sahrens * This buffer is now part of this txg 1210fa9e4066Sahrens */ 1211fa9e4066Sahrens dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg); 1212fa9e4066Sahrens db->db_dirtycnt += 1; 1213fa9e4066Sahrens ASSERT3U(db->db_dirtycnt, <=, 3); 1214fa9e4066Sahrens 1215fa9e4066Sahrens mutex_exit(&db->db_mtx); 1216fa9e4066Sahrens 12170a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID || 12180a586ceaSMark Shellenbaum db->db_blkid == DMU_SPILL_BLKID) { 1219c717a561Smaybee mutex_enter(&dn->dn_mtx); 1220c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 1221c717a561Smaybee list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1222c717a561Smaybee mutex_exit(&dn->dn_mtx); 1223fa9e4066Sahrens dnode_setdirty(dn, tx); 1224744947dcSTom Erickson DB_DNODE_EXIT(db); 1225c717a561Smaybee return (dr); 1226d3469faaSMark Maybee } else if (do_free_accounting) { 1227d3469faaSMark Maybee blkptr_t *bp = db->db_blkptr; 1228d3469faaSMark Maybee int64_t willfree = (bp && !BP_IS_HOLE(bp)) ? 1229b24ab676SJeff Bonwick bp_get_dsize(os->os_spa, bp) : db->db.db_size; 1230d3469faaSMark Maybee /* 1231d3469faaSMark Maybee * This is only a guess -- if the dbuf is dirty 1232d3469faaSMark Maybee * in a previous txg, we don't know how much 1233d3469faaSMark Maybee * space it will use on disk yet. We should 1234d3469faaSMark Maybee * really have the struct_rwlock to access 1235d3469faaSMark Maybee * db_blkptr, but since this is just a guess, 1236d3469faaSMark Maybee * it's OK if we get an odd answer. 1237d3469faaSMark Maybee */ 1238837b568bSGeorge Wilson ddt_prefetch(os->os_spa, bp); 1239d3469faaSMark Maybee dnode_willuse_space(dn, -willfree, tx); 1240fa9e4066Sahrens } 1241fa9e4066Sahrens 1242fa9e4066Sahrens if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 1243fa9e4066Sahrens rw_enter(&dn->dn_struct_rwlock, RW_READER); 1244fa9e4066Sahrens drop_struct_lock = TRUE; 1245fa9e4066Sahrens } 1246fa9e4066Sahrens 12478346f03fSJonathan W Adams if (db->db_level == 0) { 12488346f03fSJonathan W Adams dnode_new_blkid(dn, db->db_blkid, tx, drop_struct_lock); 12498346f03fSJonathan W Adams ASSERT(dn->dn_maxblkid >= db->db_blkid); 12508346f03fSJonathan W Adams } 12518346f03fSJonathan W Adams 125244eda4d7Smaybee if (db->db_level+1 < dn->dn_nlevels) { 1253c717a561Smaybee dmu_buf_impl_t *parent = db->db_parent; 1254c717a561Smaybee dbuf_dirty_record_t *di; 1255c717a561Smaybee int parent_held = FALSE; 1256c717a561Smaybee 1257c717a561Smaybee if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) { 1258fa9e4066Sahrens int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1259c717a561Smaybee 1260fa9e4066Sahrens parent = dbuf_hold_level(dn, db->db_level+1, 1261fa9e4066Sahrens db->db_blkid >> epbs, FTAG); 126201025c89SJohn Harres ASSERT(parent != NULL); 1263c717a561Smaybee parent_held = TRUE; 1264c717a561Smaybee } 1265fa9e4066Sahrens if (drop_struct_lock) 1266fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 1267c717a561Smaybee ASSERT3U(db->db_level+1, ==, parent->db_level); 1268c717a561Smaybee di = dbuf_dirty(parent, tx); 1269c717a561Smaybee if (parent_held) 1270ea8dc4b6Seschrock dbuf_rele(parent, FTAG); 1271c717a561Smaybee 1272c717a561Smaybee mutex_enter(&db->db_mtx); 1273c717a561Smaybee /* possible race with dbuf_undirty() */ 1274c717a561Smaybee if (db->db_last_dirty == dr || 1275c717a561Smaybee dn->dn_object == DMU_META_DNODE_OBJECT) { 1276c717a561Smaybee mutex_enter(&di->dt.di.dr_mtx); 1277c717a561Smaybee ASSERT3U(di->dr_txg, ==, tx->tx_txg); 1278c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 1279c717a561Smaybee list_insert_tail(&di->dt.di.dr_children, dr); 1280c717a561Smaybee mutex_exit(&di->dt.di.dr_mtx); 1281c717a561Smaybee dr->dr_parent = di; 1282c717a561Smaybee } 1283c717a561Smaybee mutex_exit(&db->db_mtx); 1284fa9e4066Sahrens } else { 1285c717a561Smaybee ASSERT(db->db_level+1 == dn->dn_nlevels); 1286c717a561Smaybee ASSERT(db->db_blkid < dn->dn_nblkptr); 1287744947dcSTom Erickson ASSERT(db->db_parent == NULL || db->db_parent == dn->dn_dbuf); 1288c717a561Smaybee mutex_enter(&dn->dn_mtx); 1289c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 1290c717a561Smaybee list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1291c717a561Smaybee mutex_exit(&dn->dn_mtx); 1292fa9e4066Sahrens if (drop_struct_lock) 1293fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 1294fa9e4066Sahrens } 1295fa9e4066Sahrens 1296fa9e4066Sahrens dnode_setdirty(dn, tx); 1297744947dcSTom Erickson DB_DNODE_EXIT(db); 1298c717a561Smaybee return (dr); 1299fa9e4066Sahrens } 1300fa9e4066Sahrens 13013b2aab18SMatthew Ahrens /* 13023e30c24aSWill Andrews * Undirty a buffer in the transaction group referenced by the given 13033e30c24aSWill Andrews * transaction. Return whether this evicted the dbuf. 13043b2aab18SMatthew Ahrens */ 13053b2aab18SMatthew Ahrens static boolean_t 1306fa9e4066Sahrens dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1307fa9e4066Sahrens { 1308744947dcSTom Erickson dnode_t *dn; 1309c717a561Smaybee uint64_t txg = tx->tx_txg; 131017f17c2dSbonwick dbuf_dirty_record_t *dr, **drp; 1311fa9e4066Sahrens 1312c717a561Smaybee ASSERT(txg != 0); 13130a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 13143b2aab18SMatthew Ahrens ASSERT0(db->db_level); 13153b2aab18SMatthew Ahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 1316fa9e4066Sahrens 1317fa9e4066Sahrens /* 1318fa9e4066Sahrens * If this buffer is not dirty, we're done. 1319fa9e4066Sahrens */ 132017f17c2dSbonwick for (drp = &db->db_last_dirty; (dr = *drp) != NULL; drp = &dr->dr_next) 1321c717a561Smaybee if (dr->dr_txg <= txg) 1322c717a561Smaybee break; 13233b2aab18SMatthew Ahrens if (dr == NULL || dr->dr_txg < txg) 13243b2aab18SMatthew Ahrens return (B_FALSE); 1325c717a561Smaybee ASSERT(dr->dr_txg == txg); 1326b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 1327fa9e4066Sahrens 1328744947dcSTom Erickson DB_DNODE_ENTER(db); 1329744947dcSTom Erickson dn = DB_DNODE(db); 1330744947dcSTom Erickson 1331fa9e4066Sahrens /* 13323b2aab18SMatthew Ahrens * Note: This code will probably work even if there are concurrent 13333b2aab18SMatthew Ahrens * holders, but it is untested in that scenerio, as the ZPL and 13343b2aab18SMatthew Ahrens * ztest have additional locking (the range locks) that prevents 13353b2aab18SMatthew Ahrens * that type of concurrent access. 1336fa9e4066Sahrens */ 13373b2aab18SMatthew Ahrens ASSERT3U(refcount_count(&db->db_holds), ==, db->db_dirtycnt); 1338fa9e4066Sahrens 1339fa9e4066Sahrens dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1340fa9e4066Sahrens 1341fa9e4066Sahrens ASSERT(db->db.db_size != 0); 1342fa9e4066Sahrens 1343fa9e4066Sahrens /* XXX would be nice to fix up dn_towrite_space[] */ 1344fa9e4066Sahrens 134517f17c2dSbonwick *drp = dr->dr_next; 1346c717a561Smaybee 13473f2366c2SGordon Ross /* 13483f2366c2SGordon Ross * Note that there are three places in dbuf_dirty() 13493f2366c2SGordon Ross * where this dirty record may be put on a list. 13503f2366c2SGordon Ross * Make sure to do a list_remove corresponding to 13513f2366c2SGordon Ross * every one of those list_insert calls. 13523f2366c2SGordon Ross */ 1353c717a561Smaybee if (dr->dr_parent) { 1354c717a561Smaybee mutex_enter(&dr->dr_parent->dt.di.dr_mtx); 1355c717a561Smaybee list_remove(&dr->dr_parent->dt.di.dr_children, dr); 1356c717a561Smaybee mutex_exit(&dr->dr_parent->dt.di.dr_mtx); 13573f2366c2SGordon Ross } else if (db->db_blkid == DMU_SPILL_BLKID || 13583f2366c2SGordon Ross db->db_level+1 == dn->dn_nlevels) { 1359cdb0ab79Smaybee ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf); 1360fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 1361c717a561Smaybee list_remove(&dn->dn_dirty_records[txg & TXG_MASK], dr); 1362fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1363c717a561Smaybee } 1364744947dcSTom Erickson DB_DNODE_EXIT(db); 1365c717a561Smaybee 136682c9918fSTim Haley if (db->db_state != DB_NOFILL) { 1367c717a561Smaybee dbuf_unoverride(dr); 1368c717a561Smaybee 1369c717a561Smaybee ASSERT(db->db_buf != NULL); 1370c717a561Smaybee ASSERT(dr->dt.dl.dr_data != NULL); 1371c717a561Smaybee if (dr->dt.dl.dr_data != db->db_buf) 13723b2aab18SMatthew Ahrens VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, db)); 1373c717a561Smaybee } 1374c717a561Smaybee kmem_free(dr, sizeof (dbuf_dirty_record_t)); 1375fa9e4066Sahrens 1376fa9e4066Sahrens ASSERT(db->db_dirtycnt > 0); 1377fa9e4066Sahrens db->db_dirtycnt -= 1; 1378fa9e4066Sahrens 1379c717a561Smaybee if (refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) { 1380ea8dc4b6Seschrock arc_buf_t *buf = db->db_buf; 1381fa9e4066Sahrens 1382b24ab676SJeff Bonwick ASSERT(db->db_state == DB_NOFILL || arc_released(buf)); 1383ea8dc4b6Seschrock dbuf_set_data(db, NULL); 13843b2aab18SMatthew Ahrens VERIFY(arc_buf_remove_ref(buf, db)); 1385fa9e4066Sahrens dbuf_evict(db); 13863b2aab18SMatthew Ahrens return (B_TRUE); 1387fa9e4066Sahrens } 1388fa9e4066Sahrens 13893b2aab18SMatthew Ahrens return (B_FALSE); 1390fa9e4066Sahrens } 1391fa9e4066Sahrens 1392fa9e4066Sahrens #pragma weak dmu_buf_will_dirty = dbuf_will_dirty 1393fa9e4066Sahrens void 1394fa9e4066Sahrens dbuf_will_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1395fa9e4066Sahrens { 13961ab7f2deSmaybee int rf = DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH; 1397fa9e4066Sahrens 1398fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 1399fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 1400fa9e4066Sahrens 1401744947dcSTom Erickson DB_DNODE_ENTER(db); 1402744947dcSTom Erickson if (RW_WRITE_HELD(&DB_DNODE(db)->dn_struct_rwlock)) 1403fa9e4066Sahrens rf |= DB_RF_HAVESTRUCT; 1404744947dcSTom Erickson DB_DNODE_EXIT(db); 1405ea8dc4b6Seschrock (void) dbuf_read(db, NULL, rf); 1406c717a561Smaybee (void) dbuf_dirty(db, tx); 1407fa9e4066Sahrens } 1408fa9e4066Sahrens 1409fa9e4066Sahrens void 141082c9918fSTim Haley dmu_buf_will_not_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 141182c9918fSTim Haley { 141282c9918fSTim Haley dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 141382c9918fSTim Haley 141482c9918fSTim Haley db->db_state = DB_NOFILL; 141582c9918fSTim Haley 141682c9918fSTim Haley dmu_buf_will_fill(db_fake, tx); 141782c9918fSTim Haley } 141882c9918fSTim Haley 141982c9918fSTim Haley void 1420ea8dc4b6Seschrock dmu_buf_will_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 1421fa9e4066Sahrens { 1422ea8dc4b6Seschrock dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1423ea8dc4b6Seschrock 14240a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1425fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 1426fa9e4066Sahrens ASSERT(db->db_level == 0); 1427fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 1428fa9e4066Sahrens 1429ea8dc4b6Seschrock ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT || 1430fa9e4066Sahrens dmu_tx_private_ok(tx)); 1431fa9e4066Sahrens 1432fa9e4066Sahrens dbuf_noread(db); 1433c717a561Smaybee (void) dbuf_dirty(db, tx); 1434fa9e4066Sahrens } 1435fa9e4066Sahrens 1436fa9e4066Sahrens #pragma weak dmu_buf_fill_done = dbuf_fill_done 1437fa9e4066Sahrens /* ARGSUSED */ 1438fa9e4066Sahrens void 1439fa9e4066Sahrens dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx) 1440fa9e4066Sahrens { 1441fa9e4066Sahrens mutex_enter(&db->db_mtx); 14429c9dc39aSek110237 DBUF_VERIFY(db); 1443fa9e4066Sahrens 1444fa9e4066Sahrens if (db->db_state == DB_FILL) { 1445c717a561Smaybee if (db->db_level == 0 && db->db_freed_in_flight) { 14460a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1447fa9e4066Sahrens /* we were freed while filling */ 1448fa9e4066Sahrens /* XXX dbuf_undirty? */ 1449fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 1450c717a561Smaybee db->db_freed_in_flight = FALSE; 1451fa9e4066Sahrens } 1452fa9e4066Sahrens db->db_state = DB_CACHED; 1453fa9e4066Sahrens cv_broadcast(&db->db_changed); 1454fa9e4066Sahrens } 1455fa9e4066Sahrens mutex_exit(&db->db_mtx); 1456fa9e4066Sahrens } 1457fa9e4066Sahrens 1458ea8dc4b6Seschrock /* 14592fdbea25SAleksandr Guzovskiy * Directly assign a provided arc buf to a given dbuf if it's not referenced 14602fdbea25SAleksandr Guzovskiy * by anybody except our caller. Otherwise copy arcbuf's contents to dbuf. 14612fdbea25SAleksandr Guzovskiy */ 14622fdbea25SAleksandr Guzovskiy void 14632fdbea25SAleksandr Guzovskiy dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx) 14642fdbea25SAleksandr Guzovskiy { 14652fdbea25SAleksandr Guzovskiy ASSERT(!refcount_is_zero(&db->db_holds)); 14660a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 14672fdbea25SAleksandr Guzovskiy ASSERT(db->db_level == 0); 14682fdbea25SAleksandr Guzovskiy ASSERT(DBUF_GET_BUFC_TYPE(db) == ARC_BUFC_DATA); 14692fdbea25SAleksandr Guzovskiy ASSERT(buf != NULL); 14702fdbea25SAleksandr Guzovskiy ASSERT(arc_buf_size(buf) == db->db.db_size); 14712fdbea25SAleksandr Guzovskiy ASSERT(tx->tx_txg != 0); 14722fdbea25SAleksandr Guzovskiy 14732fdbea25SAleksandr Guzovskiy arc_return_buf(buf, db); 14742fdbea25SAleksandr Guzovskiy ASSERT(arc_released(buf)); 14752fdbea25SAleksandr Guzovskiy 14762fdbea25SAleksandr Guzovskiy mutex_enter(&db->db_mtx); 14772fdbea25SAleksandr Guzovskiy 14782fdbea25SAleksandr Guzovskiy while (db->db_state == DB_READ || db->db_state == DB_FILL) 14792fdbea25SAleksandr Guzovskiy cv_wait(&db->db_changed, &db->db_mtx); 14802fdbea25SAleksandr Guzovskiy 14812fdbea25SAleksandr Guzovskiy ASSERT(db->db_state == DB_CACHED || db->db_state == DB_UNCACHED); 14822fdbea25SAleksandr Guzovskiy 14832fdbea25SAleksandr Guzovskiy if (db->db_state == DB_CACHED && 14842fdbea25SAleksandr Guzovskiy refcount_count(&db->db_holds) - 1 > db->db_dirtycnt) { 14852fdbea25SAleksandr Guzovskiy mutex_exit(&db->db_mtx); 14862fdbea25SAleksandr Guzovskiy (void) dbuf_dirty(db, tx); 14872fdbea25SAleksandr Guzovskiy bcopy(buf->b_data, db->db.db_data, db->db.db_size); 14883b2aab18SMatthew Ahrens VERIFY(arc_buf_remove_ref(buf, db)); 1489c242f9a0Schunli zhang - Sun Microsystems - Irvine United States xuio_stat_wbuf_copied(); 14902fdbea25SAleksandr Guzovskiy return; 14912fdbea25SAleksandr Guzovskiy } 14922fdbea25SAleksandr Guzovskiy 1493c242f9a0Schunli zhang - Sun Microsystems - Irvine United States xuio_stat_wbuf_nocopy(); 14942fdbea25SAleksandr Guzovskiy if (db->db_state == DB_CACHED) { 14952fdbea25SAleksandr Guzovskiy dbuf_dirty_record_t *dr = db->db_last_dirty; 14962fdbea25SAleksandr Guzovskiy 14972fdbea25SAleksandr Guzovskiy ASSERT(db->db_buf != NULL); 14982fdbea25SAleksandr Guzovskiy if (dr != NULL && dr->dr_txg == tx->tx_txg) { 14992fdbea25SAleksandr Guzovskiy ASSERT(dr->dt.dl.dr_data == db->db_buf); 15002fdbea25SAleksandr Guzovskiy if (!arc_released(db->db_buf)) { 15012fdbea25SAleksandr Guzovskiy ASSERT(dr->dt.dl.dr_override_state == 15022fdbea25SAleksandr Guzovskiy DR_OVERRIDDEN); 15032fdbea25SAleksandr Guzovskiy arc_release(db->db_buf, db); 15042fdbea25SAleksandr Guzovskiy } 15052fdbea25SAleksandr Guzovskiy dr->dt.dl.dr_data = buf; 15063b2aab18SMatthew Ahrens VERIFY(arc_buf_remove_ref(db->db_buf, db)); 15072fdbea25SAleksandr Guzovskiy } else if (dr == NULL || dr->dt.dl.dr_data != db->db_buf) { 15082fdbea25SAleksandr Guzovskiy arc_release(db->db_buf, db); 15093b2aab18SMatthew Ahrens VERIFY(arc_buf_remove_ref(db->db_buf, db)); 15102fdbea25SAleksandr Guzovskiy } 15112fdbea25SAleksandr Guzovskiy db->db_buf = NULL; 15122fdbea25SAleksandr Guzovskiy } 15132fdbea25SAleksandr Guzovskiy ASSERT(db->db_buf == NULL); 15142fdbea25SAleksandr Guzovskiy dbuf_set_data(db, buf); 15152fdbea25SAleksandr Guzovskiy db->db_state = DB_FILL; 15162fdbea25SAleksandr Guzovskiy mutex_exit(&db->db_mtx); 15172fdbea25SAleksandr Guzovskiy (void) dbuf_dirty(db, tx); 15182fdbea25SAleksandr Guzovskiy dbuf_fill_done(db, tx); 15192fdbea25SAleksandr Guzovskiy } 15202fdbea25SAleksandr Guzovskiy 15212fdbea25SAleksandr Guzovskiy /* 1522ea8dc4b6Seschrock * "Clear" the contents of this dbuf. This will mark the dbuf 1523ea8dc4b6Seschrock * EVICTING and clear *most* of its references. Unfortunetely, 1524ea8dc4b6Seschrock * when we are not holding the dn_dbufs_mtx, we can't clear the 1525ea8dc4b6Seschrock * entry in the dn_dbufs list. We have to wait until dbuf_destroy() 1526ea8dc4b6Seschrock * in this case. For callers from the DMU we will usually see: 1527ea8dc4b6Seschrock * dbuf_clear()->arc_buf_evict()->dbuf_do_evict()->dbuf_destroy() 1528ea8dc4b6Seschrock * For the arc callback, we will usually see: 1529ea8dc4b6Seschrock * dbuf_do_evict()->dbuf_clear();dbuf_destroy() 1530ea8dc4b6Seschrock * Sometimes, though, we will get a mix of these two: 1531ea8dc4b6Seschrock * DMU: dbuf_clear()->arc_buf_evict() 1532ea8dc4b6Seschrock * ARC: dbuf_do_evict()->dbuf_destroy() 1533ea8dc4b6Seschrock */ 1534ea8dc4b6Seschrock void 1535fa9e4066Sahrens dbuf_clear(dmu_buf_impl_t *db) 1536fa9e4066Sahrens { 1537744947dcSTom Erickson dnode_t *dn; 1538ea8dc4b6Seschrock dmu_buf_impl_t *parent = db->db_parent; 1539744947dcSTom Erickson dmu_buf_impl_t *dndb; 1540ea8dc4b6Seschrock int dbuf_gone = FALSE; 1541fa9e4066Sahrens 1542fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 1543fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 1544fa9e4066Sahrens 1545ea8dc4b6Seschrock dbuf_evict_user(db); 1546ea8dc4b6Seschrock 1547fa9e4066Sahrens if (db->db_state == DB_CACHED) { 1548ea8dc4b6Seschrock ASSERT(db->db.db_data != NULL); 15490a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 1550ea8dc4b6Seschrock zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN); 15515a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 15520e8c6158Smaybee } 1553fa9e4066Sahrens db->db.db_data = NULL; 1554fa9e4066Sahrens db->db_state = DB_UNCACHED; 1555fa9e4066Sahrens } 1556fa9e4066Sahrens 155782c9918fSTim Haley ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL); 1558fa9e4066Sahrens ASSERT(db->db_data_pending == NULL); 1559fa9e4066Sahrens 1560ea8dc4b6Seschrock db->db_state = DB_EVICTING; 1561ea8dc4b6Seschrock db->db_blkptr = NULL; 1562ea8dc4b6Seschrock 1563744947dcSTom Erickson DB_DNODE_ENTER(db); 1564744947dcSTom Erickson dn = DB_DNODE(db); 1565744947dcSTom Erickson dndb = dn->dn_dbuf; 15660a586ceaSMark Shellenbaum if (db->db_blkid != DMU_BONUS_BLKID && MUTEX_HELD(&dn->dn_dbufs_mtx)) { 1567ea8dc4b6Seschrock list_remove(&dn->dn_dbufs, db); 1568744947dcSTom Erickson (void) atomic_dec_32_nv(&dn->dn_dbufs_count); 1569744947dcSTom Erickson membar_producer(); 1570744947dcSTom Erickson DB_DNODE_EXIT(db); 1571744947dcSTom Erickson /* 1572744947dcSTom Erickson * Decrementing the dbuf count means that the hold corresponding 1573744947dcSTom Erickson * to the removed dbuf is no longer discounted in dnode_move(), 1574744947dcSTom Erickson * so the dnode cannot be moved until after we release the hold. 1575744947dcSTom Erickson * The membar_producer() ensures visibility of the decremented 1576744947dcSTom Erickson * value in dnode_move(), since DB_DNODE_EXIT doesn't actually 1577744947dcSTom Erickson * release any lock. 1578744947dcSTom Erickson */ 1579ea8dc4b6Seschrock dnode_rele(dn, db); 1580744947dcSTom Erickson db->db_dnode_handle = NULL; 1581744947dcSTom Erickson } else { 1582744947dcSTom Erickson DB_DNODE_EXIT(db); 1583ea8dc4b6Seschrock } 1584ea8dc4b6Seschrock 1585ea8dc4b6Seschrock if (db->db_buf) 1586ea8dc4b6Seschrock dbuf_gone = arc_buf_evict(db->db_buf); 1587ea8dc4b6Seschrock 1588ea8dc4b6Seschrock if (!dbuf_gone) 1589fa9e4066Sahrens mutex_exit(&db->db_mtx); 1590fa9e4066Sahrens 1591fa9e4066Sahrens /* 1592744947dcSTom Erickson * If this dbuf is referenced from an indirect dbuf, 1593fa9e4066Sahrens * decrement the ref count on the indirect dbuf. 1594fa9e4066Sahrens */ 1595c543ec06Sahrens if (parent && parent != dndb) 1596ea8dc4b6Seschrock dbuf_rele(parent, db); 1597fa9e4066Sahrens } 1598fa9e4066Sahrens 1599fa9e4066Sahrens static int 1600fa9e4066Sahrens dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse, 1601fa9e4066Sahrens dmu_buf_impl_t **parentp, blkptr_t **bpp) 1602fa9e4066Sahrens { 1603fa9e4066Sahrens int nlevels, epbs; 1604fa9e4066Sahrens 16050b69c2f0Sahrens *parentp = NULL; 16060b69c2f0Sahrens *bpp = NULL; 16070b69c2f0Sahrens 16080a586ceaSMark Shellenbaum ASSERT(blkid != DMU_BONUS_BLKID); 16090a586ceaSMark Shellenbaum 16100a586ceaSMark Shellenbaum if (blkid == DMU_SPILL_BLKID) { 16110a586ceaSMark Shellenbaum mutex_enter(&dn->dn_mtx); 161206e0070dSMark Shellenbaum if (dn->dn_have_spill && 161306e0070dSMark Shellenbaum (dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR)) 16140a586ceaSMark Shellenbaum *bpp = &dn->dn_phys->dn_spill; 16150a586ceaSMark Shellenbaum else 16160a586ceaSMark Shellenbaum *bpp = NULL; 16170a586ceaSMark Shellenbaum dbuf_add_ref(dn->dn_dbuf, NULL); 16180a586ceaSMark Shellenbaum *parentp = dn->dn_dbuf; 16190a586ceaSMark Shellenbaum mutex_exit(&dn->dn_mtx); 16200a586ceaSMark Shellenbaum return (0); 16210a586ceaSMark Shellenbaum } 1622ea8dc4b6Seschrock 1623fa9e4066Sahrens if (dn->dn_phys->dn_nlevels == 0) 1624fa9e4066Sahrens nlevels = 1; 1625fa9e4066Sahrens else 1626fa9e4066Sahrens nlevels = dn->dn_phys->dn_nlevels; 1627fa9e4066Sahrens 1628fa9e4066Sahrens epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1629fa9e4066Sahrens 1630fa9e4066Sahrens ASSERT3U(level * epbs, <, 64); 1631fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1632ea8dc4b6Seschrock if (level >= nlevels || 1633fa9e4066Sahrens (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) { 1634fa9e4066Sahrens /* the buffer has no parent yet */ 1635be6fd75aSMatthew Ahrens return (SET_ERROR(ENOENT)); 1636fa9e4066Sahrens } else if (level < nlevels-1) { 1637fa9e4066Sahrens /* this block is referenced from an indirect block */ 1638fa9e4066Sahrens int err = dbuf_hold_impl(dn, level+1, 1639fa9e4066Sahrens blkid >> epbs, fail_sparse, NULL, parentp); 1640fa9e4066Sahrens if (err) 1641fa9e4066Sahrens return (err); 1642ea8dc4b6Seschrock err = dbuf_read(*parentp, NULL, 1643ea8dc4b6Seschrock (DB_RF_HAVESTRUCT | DB_RF_NOPREFETCH | DB_RF_CANFAIL)); 1644c543ec06Sahrens if (err) { 1645c543ec06Sahrens dbuf_rele(*parentp, NULL); 1646c543ec06Sahrens *parentp = NULL; 1647c543ec06Sahrens return (err); 1648c543ec06Sahrens } 1649fa9e4066Sahrens *bpp = ((blkptr_t *)(*parentp)->db.db_data) + 1650fa9e4066Sahrens (blkid & ((1ULL << epbs) - 1)); 1651c543ec06Sahrens return (0); 1652fa9e4066Sahrens } else { 1653fa9e4066Sahrens /* the block is referenced from the dnode */ 1654fa9e4066Sahrens ASSERT3U(level, ==, nlevels-1); 1655fa9e4066Sahrens ASSERT(dn->dn_phys->dn_nblkptr == 0 || 1656fa9e4066Sahrens blkid < dn->dn_phys->dn_nblkptr); 1657c543ec06Sahrens if (dn->dn_dbuf) { 1658c543ec06Sahrens dbuf_add_ref(dn->dn_dbuf, NULL); 1659fa9e4066Sahrens *parentp = dn->dn_dbuf; 1660c543ec06Sahrens } 1661fa9e4066Sahrens *bpp = &dn->dn_phys->dn_blkptr[blkid]; 1662fa9e4066Sahrens return (0); 1663fa9e4066Sahrens } 1664fa9e4066Sahrens } 1665fa9e4066Sahrens 1666fa9e4066Sahrens static dmu_buf_impl_t * 1667fa9e4066Sahrens dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid, 1668fa9e4066Sahrens dmu_buf_impl_t *parent, blkptr_t *blkptr) 1669fa9e4066Sahrens { 1670503ad85cSMatthew Ahrens objset_t *os = dn->dn_objset; 1671fa9e4066Sahrens dmu_buf_impl_t *db, *odb; 1672fa9e4066Sahrens 1673fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1674fa9e4066Sahrens ASSERT(dn->dn_type != DMU_OT_NONE); 1675fa9e4066Sahrens 1676fa9e4066Sahrens db = kmem_cache_alloc(dbuf_cache, KM_SLEEP); 1677fa9e4066Sahrens 1678fa9e4066Sahrens db->db_objset = os; 1679fa9e4066Sahrens db->db.db_object = dn->dn_object; 1680fa9e4066Sahrens db->db_level = level; 1681fa9e4066Sahrens db->db_blkid = blkid; 1682c717a561Smaybee db->db_last_dirty = NULL; 1683ea8dc4b6Seschrock db->db_dirtycnt = 0; 1684744947dcSTom Erickson db->db_dnode_handle = dn->dn_handle; 1685ea8dc4b6Seschrock db->db_parent = parent; 1686ea8dc4b6Seschrock db->db_blkptr = blkptr; 1687fa9e4066Sahrens 1688c717a561Smaybee db->db_user_ptr = NULL; 1689c717a561Smaybee db->db_user_data_ptr_ptr = NULL; 1690c717a561Smaybee db->db_evict_func = NULL; 1691c717a561Smaybee db->db_immediate_evict = 0; 1692c717a561Smaybee db->db_freed_in_flight = 0; 1693ea8dc4b6Seschrock 16940a586ceaSMark Shellenbaum if (blkid == DMU_BONUS_BLKID) { 1695ea8dc4b6Seschrock ASSERT3P(parent, ==, dn->dn_dbuf); 16961934e92fSmaybee db->db.db_size = DN_MAX_BONUSLEN - 16971934e92fSmaybee (dn->dn_nblkptr-1) * sizeof (blkptr_t); 16981934e92fSmaybee ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 16990a586ceaSMark Shellenbaum db->db.db_offset = DMU_BONUS_BLKID; 1700ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 1701ea8dc4b6Seschrock /* the bonus dbuf is not placed in the hash table */ 17025a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1703ea8dc4b6Seschrock return (db); 17040a586ceaSMark Shellenbaum } else if (blkid == DMU_SPILL_BLKID) { 17050a586ceaSMark Shellenbaum db->db.db_size = (blkptr != NULL) ? 17060a586ceaSMark Shellenbaum BP_GET_LSIZE(blkptr) : SPA_MINBLOCKSIZE; 17070a586ceaSMark Shellenbaum db->db.db_offset = 0; 1708fa9e4066Sahrens } else { 1709fa9e4066Sahrens int blocksize = 1710fa9e4066Sahrens db->db_level ? 1<<dn->dn_indblkshift : dn->dn_datablksz; 1711fa9e4066Sahrens db->db.db_size = blocksize; 1712fa9e4066Sahrens db->db.db_offset = db->db_blkid * blocksize; 1713fa9e4066Sahrens } 1714fa9e4066Sahrens 1715fa9e4066Sahrens /* 1716fa9e4066Sahrens * Hold the dn_dbufs_mtx while we get the new dbuf 1717fa9e4066Sahrens * in the hash table *and* added to the dbufs list. 1718fa9e4066Sahrens * This prevents a possible deadlock with someone 1719fa9e4066Sahrens * trying to look up this dbuf before its added to the 1720fa9e4066Sahrens * dn_dbufs list. 1721fa9e4066Sahrens */ 1722fa9e4066Sahrens mutex_enter(&dn->dn_dbufs_mtx); 1723ea8dc4b6Seschrock db->db_state = DB_EVICTING; 1724fa9e4066Sahrens if ((odb = dbuf_hash_insert(db)) != NULL) { 1725fa9e4066Sahrens /* someone else inserted it first */ 1726fa9e4066Sahrens kmem_cache_free(dbuf_cache, db); 1727fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 1728fa9e4066Sahrens return (odb); 1729fa9e4066Sahrens } 1730fa9e4066Sahrens list_insert_head(&dn->dn_dbufs, db); 1731*713d6c20SMatthew Ahrens if (db->db_level == 0 && db->db_blkid >= 1732*713d6c20SMatthew Ahrens dn->dn_unlisted_l0_blkid) 1733*713d6c20SMatthew Ahrens dn->dn_unlisted_l0_blkid = db->db_blkid + 1; 1734ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 1735fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 17365a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1737fa9e4066Sahrens 1738fa9e4066Sahrens if (parent && parent != dn->dn_dbuf) 1739fa9e4066Sahrens dbuf_add_ref(parent, db); 1740fa9e4066Sahrens 1741ea8dc4b6Seschrock ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 1742ea8dc4b6Seschrock refcount_count(&dn->dn_holds) > 0); 1743fa9e4066Sahrens (void) refcount_add(&dn->dn_holds, db); 1744744947dcSTom Erickson (void) atomic_inc_32_nv(&dn->dn_dbufs_count); 1745fa9e4066Sahrens 1746fa9e4066Sahrens dprintf_dbuf(db, "db=%p\n", db); 1747fa9e4066Sahrens 1748fa9e4066Sahrens return (db); 1749fa9e4066Sahrens } 1750fa9e4066Sahrens 1751fa9e4066Sahrens static int 1752ea8dc4b6Seschrock dbuf_do_evict(void *private) 1753fa9e4066Sahrens { 1754ea8dc4b6Seschrock arc_buf_t *buf = private; 1755ea8dc4b6Seschrock dmu_buf_impl_t *db = buf->b_private; 1756fa9e4066Sahrens 1757ea8dc4b6Seschrock if (!MUTEX_HELD(&db->db_mtx)) 1758ea8dc4b6Seschrock mutex_enter(&db->db_mtx); 1759ea8dc4b6Seschrock 1760ea8dc4b6Seschrock ASSERT(refcount_is_zero(&db->db_holds)); 1761ea8dc4b6Seschrock 1762ea8dc4b6Seschrock if (db->db_state != DB_EVICTING) { 1763ea8dc4b6Seschrock ASSERT(db->db_state == DB_CACHED); 17649c9dc39aSek110237 DBUF_VERIFY(db); 1765ea8dc4b6Seschrock db->db_buf = NULL; 1766ea8dc4b6Seschrock dbuf_evict(db); 1767ea8dc4b6Seschrock } else { 1768ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 1769ea8dc4b6Seschrock dbuf_destroy(db); 1770fa9e4066Sahrens } 1771ea8dc4b6Seschrock return (0); 1772fa9e4066Sahrens } 1773fa9e4066Sahrens 1774fa9e4066Sahrens static void 1775fa9e4066Sahrens dbuf_destroy(dmu_buf_impl_t *db) 1776fa9e4066Sahrens { 1777fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 1778fa9e4066Sahrens 17790a586ceaSMark Shellenbaum if (db->db_blkid != DMU_BONUS_BLKID) { 1780ea8dc4b6Seschrock /* 1781ea8dc4b6Seschrock * If this dbuf is still on the dn_dbufs list, 1782ea8dc4b6Seschrock * remove it from that list. 1783ea8dc4b6Seschrock */ 1784744947dcSTom Erickson if (db->db_dnode_handle != NULL) { 1785744947dcSTom Erickson dnode_t *dn; 17861934e92fSmaybee 1787744947dcSTom Erickson DB_DNODE_ENTER(db); 1788744947dcSTom Erickson dn = DB_DNODE(db); 17891934e92fSmaybee mutex_enter(&dn->dn_dbufs_mtx); 1790ea8dc4b6Seschrock list_remove(&dn->dn_dbufs, db); 1791744947dcSTom Erickson (void) atomic_dec_32_nv(&dn->dn_dbufs_count); 1792ea8dc4b6Seschrock mutex_exit(&dn->dn_dbufs_mtx); 1793744947dcSTom Erickson DB_DNODE_EXIT(db); 1794744947dcSTom Erickson /* 1795744947dcSTom Erickson * Decrementing the dbuf count means that the hold 1796744947dcSTom Erickson * corresponding to the removed dbuf is no longer 1797744947dcSTom Erickson * discounted in dnode_move(), so the dnode cannot be 1798744947dcSTom Erickson * moved until after we release the hold. 1799744947dcSTom Erickson */ 1800ea8dc4b6Seschrock dnode_rele(dn, db); 1801744947dcSTom Erickson db->db_dnode_handle = NULL; 1802ea8dc4b6Seschrock } 1803ea8dc4b6Seschrock dbuf_hash_remove(db); 1804ea8dc4b6Seschrock } 1805ea8dc4b6Seschrock db->db_parent = NULL; 1806ea8dc4b6Seschrock db->db_buf = NULL; 1807ea8dc4b6Seschrock 1808f82bfe17Sgw25295 ASSERT(!list_link_active(&db->db_link)); 1809fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 1810fa9e4066Sahrens ASSERT(db->db_hash_next == NULL); 1811fa9e4066Sahrens ASSERT(db->db_blkptr == NULL); 1812fa9e4066Sahrens ASSERT(db->db_data_pending == NULL); 1813fa9e4066Sahrens 1814fa9e4066Sahrens kmem_cache_free(dbuf_cache, db); 18155a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1816fa9e4066Sahrens } 1817fa9e4066Sahrens 1818fa9e4066Sahrens void 1819fa9e4066Sahrens dbuf_prefetch(dnode_t *dn, uint64_t blkid) 1820fa9e4066Sahrens { 182113506d1eSmaybee dmu_buf_impl_t *db = NULL; 1822fa9e4066Sahrens blkptr_t *bp = NULL; 1823fa9e4066Sahrens 18240a586ceaSMark Shellenbaum ASSERT(blkid != DMU_BONUS_BLKID); 1825fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1826fa9e4066Sahrens 1827fa9e4066Sahrens if (dnode_block_freed(dn, blkid)) 1828fa9e4066Sahrens return; 1829fa9e4066Sahrens 1830fa9e4066Sahrens /* dbuf_find() returns with db_mtx held */ 1831fa9e4066Sahrens if (db = dbuf_find(dn, 0, blkid)) { 1832fa9e4066Sahrens /* 18339e9c486fSGeorge Wilson * This dbuf is already in the cache. We assume that 18349e9c486fSGeorge Wilson * it is already CACHED, or else about to be either 1835fa9e4066Sahrens * read or filled. 1836fa9e4066Sahrens */ 1837fa9e4066Sahrens mutex_exit(&db->db_mtx); 1838fa9e4066Sahrens return; 1839fa9e4066Sahrens } 1840fa9e4066Sahrens 184113506d1eSmaybee if (dbuf_findbp(dn, 0, blkid, TRUE, &db, &bp) == 0) { 1842fa9e4066Sahrens if (bp && !BP_IS_HOLE(bp)) { 1843c7cd2421SGeorge Wilson int priority = dn->dn_type == DMU_OT_DDT_ZAP ? 1844c7cd2421SGeorge Wilson ZIO_PRIORITY_DDT_PREFETCH : ZIO_PRIORITY_ASYNC_READ; 1845b24ab676SJeff Bonwick dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset; 184613506d1eSmaybee uint32_t aflags = ARC_NOWAIT | ARC_PREFETCH; 1847ea8dc4b6Seschrock zbookmark_t zb; 1848b24ab676SJeff Bonwick 1849b24ab676SJeff Bonwick SET_BOOKMARK(&zb, ds ? ds->ds_object : DMU_META_OBJSET, 1850b24ab676SJeff Bonwick dn->dn_object, 0, blkid); 1851ea8dc4b6Seschrock 18521b912ec7SGeorge Wilson (void) arc_read(NULL, dn->dn_objset->os_spa, 18531b912ec7SGeorge Wilson bp, NULL, NULL, priority, 1854fa9e4066Sahrens ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 185513506d1eSmaybee &aflags, &zb); 1856fa9e4066Sahrens } 185713506d1eSmaybee if (db) 185813506d1eSmaybee dbuf_rele(db, NULL); 1859fa9e4066Sahrens } 1860fa9e4066Sahrens } 1861fa9e4066Sahrens 1862fa9e4066Sahrens /* 1863fa9e4066Sahrens * Returns with db_holds incremented, and db_mtx not held. 1864fa9e4066Sahrens * Note: dn_struct_rwlock must be held. 1865fa9e4066Sahrens */ 1866fa9e4066Sahrens int 1867fa9e4066Sahrens dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, int fail_sparse, 1868fa9e4066Sahrens void *tag, dmu_buf_impl_t **dbp) 1869fa9e4066Sahrens { 1870fa9e4066Sahrens dmu_buf_impl_t *db, *parent = NULL; 1871fa9e4066Sahrens 18720a586ceaSMark Shellenbaum ASSERT(blkid != DMU_BONUS_BLKID); 1873fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1874fa9e4066Sahrens ASSERT3U(dn->dn_nlevels, >, level); 1875fa9e4066Sahrens 1876fa9e4066Sahrens *dbp = NULL; 1877ea8dc4b6Seschrock top: 1878fa9e4066Sahrens /* dbuf_find() returns with db_mtx held */ 1879fa9e4066Sahrens db = dbuf_find(dn, level, blkid); 1880fa9e4066Sahrens 1881fa9e4066Sahrens if (db == NULL) { 1882fa9e4066Sahrens blkptr_t *bp = NULL; 1883fa9e4066Sahrens int err; 1884fa9e4066Sahrens 1885c543ec06Sahrens ASSERT3P(parent, ==, NULL); 1886fa9e4066Sahrens err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp); 1887fa9e4066Sahrens if (fail_sparse) { 1888fa9e4066Sahrens if (err == 0 && bp && BP_IS_HOLE(bp)) 1889be6fd75aSMatthew Ahrens err = SET_ERROR(ENOENT); 1890fa9e4066Sahrens if (err) { 1891c543ec06Sahrens if (parent) 1892ea8dc4b6Seschrock dbuf_rele(parent, NULL); 1893fa9e4066Sahrens return (err); 1894fa9e4066Sahrens } 1895fa9e4066Sahrens } 1896ea8dc4b6Seschrock if (err && err != ENOENT) 1897ea8dc4b6Seschrock return (err); 1898fa9e4066Sahrens db = dbuf_create(dn, level, blkid, parent, bp); 1899fa9e4066Sahrens } 1900fa9e4066Sahrens 1901ea8dc4b6Seschrock if (db->db_buf && refcount_is_zero(&db->db_holds)) { 1902ea8dc4b6Seschrock arc_buf_add_ref(db->db_buf, db); 1903ea8dc4b6Seschrock if (db->db_buf->b_data == NULL) { 1904ea8dc4b6Seschrock dbuf_clear(db); 1905c543ec06Sahrens if (parent) { 1906c543ec06Sahrens dbuf_rele(parent, NULL); 1907c543ec06Sahrens parent = NULL; 1908c543ec06Sahrens } 1909ea8dc4b6Seschrock goto top; 1910ea8dc4b6Seschrock } 1911ea8dc4b6Seschrock ASSERT3P(db->db.db_data, ==, db->db_buf->b_data); 1912ea8dc4b6Seschrock } 1913ea8dc4b6Seschrock 1914ea8dc4b6Seschrock ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf)); 1915ea8dc4b6Seschrock 1916fa9e4066Sahrens /* 1917c717a561Smaybee * If this buffer is currently syncing out, and we are are 1918c717a561Smaybee * still referencing it from db_data, we need to make a copy 1919c717a561Smaybee * of it in case we decide we want to dirty it again in this txg. 1920fa9e4066Sahrens */ 19210a586ceaSMark Shellenbaum if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 1922ea8dc4b6Seschrock dn->dn_object != DMU_META_DNODE_OBJECT && 1923c717a561Smaybee db->db_state == DB_CACHED && db->db_data_pending) { 1924c717a561Smaybee dbuf_dirty_record_t *dr = db->db_data_pending; 1925c717a561Smaybee 1926c717a561Smaybee if (dr->dt.dl.dr_data == db->db_buf) { 1927ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 1928fa9e4066Sahrens 1929c717a561Smaybee dbuf_set_data(db, 1930744947dcSTom Erickson arc_buf_alloc(dn->dn_objset->os_spa, 1931c717a561Smaybee db->db.db_size, db, type)); 1932c717a561Smaybee bcopy(dr->dt.dl.dr_data->b_data, db->db.db_data, 1933fa9e4066Sahrens db->db.db_size); 1934fa9e4066Sahrens } 1935c717a561Smaybee } 1936fa9e4066Sahrens 1937ea8dc4b6Seschrock (void) refcount_add(&db->db_holds, tag); 1938fa9e4066Sahrens dbuf_update_data(db); 19399c9dc39aSek110237 DBUF_VERIFY(db); 1940fa9e4066Sahrens mutex_exit(&db->db_mtx); 1941fa9e4066Sahrens 1942fa9e4066Sahrens /* NOTE: we can't rele the parent until after we drop the db_mtx */ 1943c543ec06Sahrens if (parent) 1944ea8dc4b6Seschrock dbuf_rele(parent, NULL); 1945fa9e4066Sahrens 1946744947dcSTom Erickson ASSERT3P(DB_DNODE(db), ==, dn); 1947fa9e4066Sahrens ASSERT3U(db->db_blkid, ==, blkid); 1948fa9e4066Sahrens ASSERT3U(db->db_level, ==, level); 1949fa9e4066Sahrens *dbp = db; 1950fa9e4066Sahrens 1951fa9e4066Sahrens return (0); 1952fa9e4066Sahrens } 1953fa9e4066Sahrens 1954fa9e4066Sahrens dmu_buf_impl_t * 1955ea8dc4b6Seschrock dbuf_hold(dnode_t *dn, uint64_t blkid, void *tag) 1956fa9e4066Sahrens { 1957fa9e4066Sahrens dmu_buf_impl_t *db; 1958ea8dc4b6Seschrock int err = dbuf_hold_impl(dn, 0, blkid, FALSE, tag, &db); 1959ea8dc4b6Seschrock return (err ? NULL : db); 1960fa9e4066Sahrens } 1961fa9e4066Sahrens 1962fa9e4066Sahrens dmu_buf_impl_t * 1963fa9e4066Sahrens dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, void *tag) 1964fa9e4066Sahrens { 1965fa9e4066Sahrens dmu_buf_impl_t *db; 1966ea8dc4b6Seschrock int err = dbuf_hold_impl(dn, level, blkid, FALSE, tag, &db); 1967ea8dc4b6Seschrock return (err ? NULL : db); 1968fa9e4066Sahrens } 1969fa9e4066Sahrens 19701934e92fSmaybee void 1971ea8dc4b6Seschrock dbuf_create_bonus(dnode_t *dn) 1972fa9e4066Sahrens { 1973ea8dc4b6Seschrock ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 1974ea8dc4b6Seschrock 1975ea8dc4b6Seschrock ASSERT(dn->dn_bonus == NULL); 19760a586ceaSMark Shellenbaum dn->dn_bonus = dbuf_create(dn, 0, DMU_BONUS_BLKID, dn->dn_dbuf, NULL); 19770a586ceaSMark Shellenbaum } 19780a586ceaSMark Shellenbaum 19790a586ceaSMark Shellenbaum int 19800a586ceaSMark Shellenbaum dbuf_spill_set_blksz(dmu_buf_t *db_fake, uint64_t blksz, dmu_tx_t *tx) 19810a586ceaSMark Shellenbaum { 19820a586ceaSMark Shellenbaum dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1983744947dcSTom Erickson dnode_t *dn; 1984744947dcSTom Erickson 19850a586ceaSMark Shellenbaum if (db->db_blkid != DMU_SPILL_BLKID) 1986be6fd75aSMatthew Ahrens return (SET_ERROR(ENOTSUP)); 19870a586ceaSMark Shellenbaum if (blksz == 0) 19880a586ceaSMark Shellenbaum blksz = SPA_MINBLOCKSIZE; 19890a586ceaSMark Shellenbaum if (blksz > SPA_MAXBLOCKSIZE) 19900a586ceaSMark Shellenbaum blksz = SPA_MAXBLOCKSIZE; 19910a586ceaSMark Shellenbaum else 19920a586ceaSMark Shellenbaum blksz = P2ROUNDUP(blksz, SPA_MINBLOCKSIZE); 19930a586ceaSMark Shellenbaum 1994744947dcSTom Erickson DB_DNODE_ENTER(db); 1995744947dcSTom Erickson dn = DB_DNODE(db); 1996744947dcSTom Erickson rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 19970a586ceaSMark Shellenbaum dbuf_new_size(db, blksz, tx); 1998744947dcSTom Erickson rw_exit(&dn->dn_struct_rwlock); 1999744947dcSTom Erickson DB_DNODE_EXIT(db); 20000a586ceaSMark Shellenbaum 20010a586ceaSMark Shellenbaum return (0); 20020a586ceaSMark Shellenbaum } 20030a586ceaSMark Shellenbaum 20040a586ceaSMark Shellenbaum void 20050a586ceaSMark Shellenbaum dbuf_rm_spill(dnode_t *dn, dmu_tx_t *tx) 20060a586ceaSMark Shellenbaum { 20070a586ceaSMark Shellenbaum dbuf_free_range(dn, DMU_SPILL_BLKID, DMU_SPILL_BLKID, tx); 2008fa9e4066Sahrens } 2009fa9e4066Sahrens 2010ea8dc4b6Seschrock #pragma weak dmu_buf_add_ref = dbuf_add_ref 2011fa9e4066Sahrens void 2012fa9e4066Sahrens dbuf_add_ref(dmu_buf_impl_t *db, void *tag) 2013fa9e4066Sahrens { 2014ea8dc4b6Seschrock int64_t holds = refcount_add(&db->db_holds, tag); 2015ea8dc4b6Seschrock ASSERT(holds > 1); 2016fa9e4066Sahrens } 2017fa9e4066Sahrens 2018744947dcSTom Erickson /* 2019744947dcSTom Erickson * If you call dbuf_rele() you had better not be referencing the dnode handle 2020744947dcSTom Erickson * unless you have some other direct or indirect hold on the dnode. (An indirect 2021744947dcSTom Erickson * hold is a hold on one of the dnode's dbufs, including the bonus buffer.) 2022744947dcSTom Erickson * Without that, the dbuf_rele() could lead to a dnode_rele() followed by the 2023744947dcSTom Erickson * dnode's parent dbuf evicting its dnode handles. 2024744947dcSTom Erickson */ 2025ea8dc4b6Seschrock #pragma weak dmu_buf_rele = dbuf_rele 2026fa9e4066Sahrens void 2027ea8dc4b6Seschrock dbuf_rele(dmu_buf_impl_t *db, void *tag) 2028fa9e4066Sahrens { 2029b24ab676SJeff Bonwick mutex_enter(&db->db_mtx); 2030b24ab676SJeff Bonwick dbuf_rele_and_unlock(db, tag); 2031b24ab676SJeff Bonwick } 2032b24ab676SJeff Bonwick 2033b24ab676SJeff Bonwick /* 2034b24ab676SJeff Bonwick * dbuf_rele() for an already-locked dbuf. This is necessary to allow 2035b24ab676SJeff Bonwick * db_dirtycnt and db_holds to be updated atomically. 2036b24ab676SJeff Bonwick */ 2037b24ab676SJeff Bonwick void 2038b24ab676SJeff Bonwick dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag) 2039b24ab676SJeff Bonwick { 2040fa9e4066Sahrens int64_t holds; 2041fa9e4066Sahrens 2042b24ab676SJeff Bonwick ASSERT(MUTEX_HELD(&db->db_mtx)); 20439c9dc39aSek110237 DBUF_VERIFY(db); 2044fa9e4066Sahrens 2045744947dcSTom Erickson /* 2046744947dcSTom Erickson * Remove the reference to the dbuf before removing its hold on the 2047744947dcSTom Erickson * dnode so we can guarantee in dnode_move() that a referenced bonus 2048744947dcSTom Erickson * buffer has a corresponding dnode hold. 2049744947dcSTom Erickson */ 2050fa9e4066Sahrens holds = refcount_remove(&db->db_holds, tag); 2051ea8dc4b6Seschrock ASSERT(holds >= 0); 2052fa9e4066Sahrens 2053c717a561Smaybee /* 2054c717a561Smaybee * We can't freeze indirects if there is a possibility that they 2055c717a561Smaybee * may be modified in the current syncing context. 2056c717a561Smaybee */ 2057c717a561Smaybee if (db->db_buf && holds == (db->db_level == 0 ? db->db_dirtycnt : 0)) 20586b4acc8bSahrens arc_buf_freeze(db->db_buf); 20596b4acc8bSahrens 2060fa9e4066Sahrens if (holds == db->db_dirtycnt && 2061c717a561Smaybee db->db_level == 0 && db->db_immediate_evict) 2062fa9e4066Sahrens dbuf_evict_user(db); 2063ea8dc4b6Seschrock 2064ea8dc4b6Seschrock if (holds == 0) { 20650a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 2066ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 2067744947dcSTom Erickson 2068744947dcSTom Erickson /* 2069744947dcSTom Erickson * If the dnode moves here, we cannot cross this barrier 2070744947dcSTom Erickson * until the move completes. 2071744947dcSTom Erickson */ 2072744947dcSTom Erickson DB_DNODE_ENTER(db); 2073744947dcSTom Erickson (void) atomic_dec_32_nv(&DB_DNODE(db)->dn_dbufs_count); 2074744947dcSTom Erickson DB_DNODE_EXIT(db); 2075744947dcSTom Erickson /* 2076744947dcSTom Erickson * The bonus buffer's dnode hold is no longer discounted 2077744947dcSTom Erickson * in dnode_move(). The dnode cannot move until after 2078744947dcSTom Erickson * the dnode_rele(). 2079744947dcSTom Erickson */ 2080744947dcSTom Erickson dnode_rele(DB_DNODE(db), db); 2081ea8dc4b6Seschrock } else if (db->db_buf == NULL) { 2082ea8dc4b6Seschrock /* 2083ea8dc4b6Seschrock * This is a special case: we never associated this 2084ea8dc4b6Seschrock * dbuf with any data allocated from the ARC. 2085ea8dc4b6Seschrock */ 208682c9918fSTim Haley ASSERT(db->db_state == DB_UNCACHED || 208782c9918fSTim Haley db->db_state == DB_NOFILL); 2088ea8dc4b6Seschrock dbuf_evict(db); 2089ea8dc4b6Seschrock } else if (arc_released(db->db_buf)) { 2090ea8dc4b6Seschrock arc_buf_t *buf = db->db_buf; 2091ea8dc4b6Seschrock /* 2092ea8dc4b6Seschrock * This dbuf has anonymous data associated with it. 2093ea8dc4b6Seschrock */ 2094ea8dc4b6Seschrock dbuf_set_data(db, NULL); 20953b2aab18SMatthew Ahrens VERIFY(arc_buf_remove_ref(buf, db)); 2096ea8dc4b6Seschrock dbuf_evict(db); 2097ea8dc4b6Seschrock } else { 20983b2aab18SMatthew Ahrens VERIFY(!arc_buf_remove_ref(db->db_buf, db)); 20999253d63dSGeorge Wilson 21009253d63dSGeorge Wilson /* 21019253d63dSGeorge Wilson * A dbuf will be eligible for eviction if either the 21029253d63dSGeorge Wilson * 'primarycache' property is set or a duplicate 21039253d63dSGeorge Wilson * copy of this buffer is already cached in the arc. 21049253d63dSGeorge Wilson * 21059253d63dSGeorge Wilson * In the case of the 'primarycache' a buffer 21069253d63dSGeorge Wilson * is considered for eviction if it matches the 21079253d63dSGeorge Wilson * criteria set in the property. 21089253d63dSGeorge Wilson * 21099253d63dSGeorge Wilson * To decide if our buffer is considered a 21109253d63dSGeorge Wilson * duplicate, we must call into the arc to determine 21119253d63dSGeorge Wilson * if multiple buffers are referencing the same 21129253d63dSGeorge Wilson * block on-disk. If so, then we simply evict 21139253d63dSGeorge Wilson * ourselves. 21149253d63dSGeorge Wilson */ 21159253d63dSGeorge Wilson if (!DBUF_IS_CACHEABLE(db) || 21169253d63dSGeorge Wilson arc_buf_eviction_needed(db->db_buf)) 21173baa08fcSek110237 dbuf_clear(db); 21183baa08fcSek110237 else 2119fa9e4066Sahrens mutex_exit(&db->db_mtx); 2120fa9e4066Sahrens } 2121ea8dc4b6Seschrock } else { 2122ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 2123fa9e4066Sahrens } 2124fa9e4066Sahrens } 2125fa9e4066Sahrens 2126fa9e4066Sahrens #pragma weak dmu_buf_refcount = dbuf_refcount 2127fa9e4066Sahrens uint64_t 2128fa9e4066Sahrens dbuf_refcount(dmu_buf_impl_t *db) 2129fa9e4066Sahrens { 2130fa9e4066Sahrens return (refcount_count(&db->db_holds)); 2131fa9e4066Sahrens } 2132fa9e4066Sahrens 2133fa9e4066Sahrens void * 2134fa9e4066Sahrens dmu_buf_set_user(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 2135fa9e4066Sahrens dmu_buf_evict_func_t *evict_func) 2136fa9e4066Sahrens { 2137fa9e4066Sahrens return (dmu_buf_update_user(db_fake, NULL, user_ptr, 2138fa9e4066Sahrens user_data_ptr_ptr, evict_func)); 2139fa9e4066Sahrens } 2140fa9e4066Sahrens 2141fa9e4066Sahrens void * 2142fa9e4066Sahrens dmu_buf_set_user_ie(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 2143fa9e4066Sahrens dmu_buf_evict_func_t *evict_func) 2144fa9e4066Sahrens { 2145fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2146fa9e4066Sahrens 2147c717a561Smaybee db->db_immediate_evict = TRUE; 2148fa9e4066Sahrens return (dmu_buf_update_user(db_fake, NULL, user_ptr, 2149fa9e4066Sahrens user_data_ptr_ptr, evict_func)); 2150fa9e4066Sahrens } 2151fa9e4066Sahrens 2152fa9e4066Sahrens void * 2153fa9e4066Sahrens dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr, void *user_ptr, 2154fa9e4066Sahrens void *user_data_ptr_ptr, dmu_buf_evict_func_t *evict_func) 2155fa9e4066Sahrens { 2156fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2157fa9e4066Sahrens ASSERT(db->db_level == 0); 2158fa9e4066Sahrens 2159fa9e4066Sahrens ASSERT((user_ptr == NULL) == (evict_func == NULL)); 2160fa9e4066Sahrens 2161fa9e4066Sahrens mutex_enter(&db->db_mtx); 2162fa9e4066Sahrens 2163c717a561Smaybee if (db->db_user_ptr == old_user_ptr) { 2164c717a561Smaybee db->db_user_ptr = user_ptr; 2165c717a561Smaybee db->db_user_data_ptr_ptr = user_data_ptr_ptr; 2166c717a561Smaybee db->db_evict_func = evict_func; 2167fa9e4066Sahrens 2168fa9e4066Sahrens dbuf_update_data(db); 2169fa9e4066Sahrens } else { 2170c717a561Smaybee old_user_ptr = db->db_user_ptr; 2171fa9e4066Sahrens } 2172fa9e4066Sahrens 2173fa9e4066Sahrens mutex_exit(&db->db_mtx); 2174fa9e4066Sahrens return (old_user_ptr); 2175fa9e4066Sahrens } 2176fa9e4066Sahrens 2177fa9e4066Sahrens void * 2178fa9e4066Sahrens dmu_buf_get_user(dmu_buf_t *db_fake) 2179fa9e4066Sahrens { 2180fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2181fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 2182fa9e4066Sahrens 2183c717a561Smaybee return (db->db_user_ptr); 2184fa9e4066Sahrens } 2185fa9e4066Sahrens 21863d692628SSanjeev Bagewadi boolean_t 21873d692628SSanjeev Bagewadi dmu_buf_freeable(dmu_buf_t *dbuf) 21883d692628SSanjeev Bagewadi { 21893d692628SSanjeev Bagewadi boolean_t res = B_FALSE; 21903d692628SSanjeev Bagewadi dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf; 21913d692628SSanjeev Bagewadi 21923d692628SSanjeev Bagewadi if (db->db_blkptr) 21933d692628SSanjeev Bagewadi res = dsl_dataset_block_freeable(db->db_objset->os_dsl_dataset, 2194c7cd2421SGeorge Wilson db->db_blkptr, db->db_blkptr->blk_birth); 21953d692628SSanjeev Bagewadi 21963d692628SSanjeev Bagewadi return (res); 21973d692628SSanjeev Bagewadi } 21983d692628SSanjeev Bagewadi 219980901aeaSGeorge Wilson blkptr_t * 220080901aeaSGeorge Wilson dmu_buf_get_blkptr(dmu_buf_t *db) 220180901aeaSGeorge Wilson { 220280901aeaSGeorge Wilson dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db; 220380901aeaSGeorge Wilson return (dbi->db_blkptr); 220480901aeaSGeorge Wilson } 220580901aeaSGeorge Wilson 2206c717a561Smaybee static void 2207c717a561Smaybee dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db) 2208fa9e4066Sahrens { 2209c717a561Smaybee /* ASSERT(dmu_tx_is_syncing(tx) */ 2210c717a561Smaybee ASSERT(MUTEX_HELD(&db->db_mtx)); 2211c717a561Smaybee 2212c717a561Smaybee if (db->db_blkptr != NULL) 2213c717a561Smaybee return; 2214c717a561Smaybee 22150a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) { 22160a586ceaSMark Shellenbaum db->db_blkptr = &dn->dn_phys->dn_spill; 22170a586ceaSMark Shellenbaum BP_ZERO(db->db_blkptr); 22180a586ceaSMark Shellenbaum return; 22190a586ceaSMark Shellenbaum } 2220c717a561Smaybee if (db->db_level == dn->dn_phys->dn_nlevels-1) { 2221c717a561Smaybee /* 2222c717a561Smaybee * This buffer was allocated at a time when there was 2223c717a561Smaybee * no available blkptrs from the dnode, or it was 2224c717a561Smaybee * inappropriate to hook it in (i.e., nlevels mis-match). 2225c717a561Smaybee */ 2226c717a561Smaybee ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr); 2227c717a561Smaybee ASSERT(db->db_parent == NULL); 2228c717a561Smaybee db->db_parent = dn->dn_dbuf; 2229c717a561Smaybee db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid]; 2230c717a561Smaybee DBUF_VERIFY(db); 2231c717a561Smaybee } else { 2232c717a561Smaybee dmu_buf_impl_t *parent = db->db_parent; 2233c717a561Smaybee int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2234c717a561Smaybee 2235c717a561Smaybee ASSERT(dn->dn_phys->dn_nlevels > 1); 2236c717a561Smaybee if (parent == NULL) { 2237c717a561Smaybee mutex_exit(&db->db_mtx); 2238c717a561Smaybee rw_enter(&dn->dn_struct_rwlock, RW_READER); 2239c717a561Smaybee (void) dbuf_hold_impl(dn, db->db_level+1, 2240c717a561Smaybee db->db_blkid >> epbs, FALSE, db, &parent); 2241c717a561Smaybee rw_exit(&dn->dn_struct_rwlock); 2242c717a561Smaybee mutex_enter(&db->db_mtx); 2243c717a561Smaybee db->db_parent = parent; 2244c717a561Smaybee } 2245c717a561Smaybee db->db_blkptr = (blkptr_t *)parent->db.db_data + 2246c717a561Smaybee (db->db_blkid & ((1ULL << epbs) - 1)); 2247c717a561Smaybee DBUF_VERIFY(db); 2248c717a561Smaybee } 2249c717a561Smaybee } 2250c717a561Smaybee 2251c717a561Smaybee static void 2252c717a561Smaybee dbuf_sync_indirect(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 2253c717a561Smaybee { 2254c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 2255744947dcSTom Erickson dnode_t *dn; 2256c717a561Smaybee zio_t *zio; 2257c717a561Smaybee 2258c717a561Smaybee ASSERT(dmu_tx_is_syncing(tx)); 2259c717a561Smaybee 2260c717a561Smaybee dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 2261c717a561Smaybee 2262c717a561Smaybee mutex_enter(&db->db_mtx); 2263c717a561Smaybee 2264c717a561Smaybee ASSERT(db->db_level > 0); 2265c717a561Smaybee DBUF_VERIFY(db); 2266c717a561Smaybee 22673e30c24aSWill Andrews /* Read the block if it hasn't been read yet. */ 2268c717a561Smaybee if (db->db_buf == NULL) { 2269c717a561Smaybee mutex_exit(&db->db_mtx); 2270c717a561Smaybee (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED); 2271c717a561Smaybee mutex_enter(&db->db_mtx); 2272c717a561Smaybee } 2273c717a561Smaybee ASSERT3U(db->db_state, ==, DB_CACHED); 2274c717a561Smaybee ASSERT(db->db_buf != NULL); 2275c717a561Smaybee 2276744947dcSTom Erickson DB_DNODE_ENTER(db); 2277744947dcSTom Erickson dn = DB_DNODE(db); 22783e30c24aSWill Andrews /* Indirect block size must match what the dnode thinks it is. */ 2279744947dcSTom Erickson ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2280c717a561Smaybee dbuf_check_blkptr(dn, db); 2281744947dcSTom Erickson DB_DNODE_EXIT(db); 2282c717a561Smaybee 22833e30c24aSWill Andrews /* Provide the pending dirty record to child dbufs */ 2284c717a561Smaybee db->db_data_pending = dr; 2285c717a561Smaybee 2286af2c4821Smaybee mutex_exit(&db->db_mtx); 2287088f3894Sahrens dbuf_write(dr, db->db_buf, tx); 2288c717a561Smaybee 2289c717a561Smaybee zio = dr->dr_zio; 2290c717a561Smaybee mutex_enter(&dr->dt.di.dr_mtx); 2291c717a561Smaybee dbuf_sync_list(&dr->dt.di.dr_children, tx); 2292c717a561Smaybee ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 2293c717a561Smaybee mutex_exit(&dr->dt.di.dr_mtx); 2294c717a561Smaybee zio_nowait(zio); 2295c717a561Smaybee } 2296c717a561Smaybee 2297c717a561Smaybee static void 2298c717a561Smaybee dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 2299c717a561Smaybee { 2300c717a561Smaybee arc_buf_t **datap = &dr->dt.dl.dr_data; 2301c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 2302744947dcSTom Erickson dnode_t *dn; 2303744947dcSTom Erickson objset_t *os; 2304c717a561Smaybee uint64_t txg = tx->tx_txg; 2305fa9e4066Sahrens 2306fa9e4066Sahrens ASSERT(dmu_tx_is_syncing(tx)); 2307fa9e4066Sahrens 2308fa9e4066Sahrens dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 2309fa9e4066Sahrens 2310fa9e4066Sahrens mutex_enter(&db->db_mtx); 2311fa9e4066Sahrens /* 2312fa9e4066Sahrens * To be synced, we must be dirtied. But we 2313fa9e4066Sahrens * might have been freed after the dirty. 2314fa9e4066Sahrens */ 2315fa9e4066Sahrens if (db->db_state == DB_UNCACHED) { 2316fa9e4066Sahrens /* This buffer has been freed since it was dirtied */ 2317fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 2318fa9e4066Sahrens } else if (db->db_state == DB_FILL) { 2319fa9e4066Sahrens /* This buffer was freed and is now being re-filled */ 2320c717a561Smaybee ASSERT(db->db.db_data != dr->dt.dl.dr_data); 2321fa9e4066Sahrens } else { 232282c9918fSTim Haley ASSERT(db->db_state == DB_CACHED || db->db_state == DB_NOFILL); 2323fa9e4066Sahrens } 23249c9dc39aSek110237 DBUF_VERIFY(db); 2325fa9e4066Sahrens 2326744947dcSTom Erickson DB_DNODE_ENTER(db); 2327744947dcSTom Erickson dn = DB_DNODE(db); 2328744947dcSTom Erickson 23290a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) { 23300a586ceaSMark Shellenbaum mutex_enter(&dn->dn_mtx); 23310a586ceaSMark Shellenbaum dn->dn_phys->dn_flags |= DNODE_FLAG_SPILL_BLKPTR; 23320a586ceaSMark Shellenbaum mutex_exit(&dn->dn_mtx); 23330a586ceaSMark Shellenbaum } 23340a586ceaSMark Shellenbaum 2335fa9e4066Sahrens /* 2336c717a561Smaybee * If this is a bonus buffer, simply copy the bonus data into the 2337c717a561Smaybee * dnode. It will be written out when the dnode is synced (and it 2338c717a561Smaybee * will be synced, since it must have been dirty for dbuf_sync to 2339c717a561Smaybee * be called). 2340fa9e4066Sahrens */ 23410a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 2342c717a561Smaybee dbuf_dirty_record_t **drp; 23431934e92fSmaybee 2344ea8dc4b6Seschrock ASSERT(*datap != NULL); 2345fb09f5aaSMadhav Suresh ASSERT0(db->db_level); 2346ea8dc4b6Seschrock ASSERT3U(dn->dn_phys->dn_bonuslen, <=, DN_MAX_BONUSLEN); 2347ea8dc4b6Seschrock bcopy(*datap, DN_BONUS(dn->dn_phys), dn->dn_phys->dn_bonuslen); 2348744947dcSTom Erickson DB_DNODE_EXIT(db); 2349744947dcSTom Erickson 23500e8c6158Smaybee if (*datap != db->db.db_data) { 2351ea8dc4b6Seschrock zio_buf_free(*datap, DN_MAX_BONUSLEN); 23525a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 23530e8c6158Smaybee } 2354ea8dc4b6Seschrock db->db_data_pending = NULL; 2355c717a561Smaybee drp = &db->db_last_dirty; 2356c717a561Smaybee while (*drp != dr) 2357c717a561Smaybee drp = &(*drp)->dr_next; 235817f17c2dSbonwick ASSERT(dr->dr_next == NULL); 2359b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 236017f17c2dSbonwick *drp = dr->dr_next; 2361c717a561Smaybee kmem_free(dr, sizeof (dbuf_dirty_record_t)); 2362ea8dc4b6Seschrock ASSERT(db->db_dirtycnt > 0); 2363ea8dc4b6Seschrock db->db_dirtycnt -= 1; 2364b24ab676SJeff Bonwick dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg); 2365ea8dc4b6Seschrock return; 2366ea8dc4b6Seschrock } 2367ea8dc4b6Seschrock 2368744947dcSTom Erickson os = dn->dn_objset; 2369744947dcSTom Erickson 2370c5c6ffa0Smaybee /* 2371f82bfe17Sgw25295 * This function may have dropped the db_mtx lock allowing a dmu_sync 2372f82bfe17Sgw25295 * operation to sneak in. As a result, we need to ensure that we 2373f82bfe17Sgw25295 * don't check the dr_override_state until we have returned from 2374f82bfe17Sgw25295 * dbuf_check_blkptr. 2375f82bfe17Sgw25295 */ 2376f82bfe17Sgw25295 dbuf_check_blkptr(dn, db); 2377f82bfe17Sgw25295 2378f82bfe17Sgw25295 /* 2379744947dcSTom Erickson * If this buffer is in the middle of an immediate write, 2380c717a561Smaybee * wait for the synchronous IO to complete. 2381c5c6ffa0Smaybee */ 2382c717a561Smaybee while (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC) { 2383c5c6ffa0Smaybee ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT); 2384c5c6ffa0Smaybee cv_wait(&db->db_changed, &db->db_mtx); 2385c717a561Smaybee ASSERT(dr->dt.dl.dr_override_state != DR_NOT_OVERRIDDEN); 2386c5c6ffa0Smaybee } 2387c717a561Smaybee 2388ab69d62fSMatthew Ahrens if (db->db_state != DB_NOFILL && 2389ab69d62fSMatthew Ahrens dn->dn_object != DMU_META_DNODE_OBJECT && 2390ab69d62fSMatthew Ahrens refcount_count(&db->db_holds) > 1 && 2391b24ab676SJeff Bonwick dr->dt.dl.dr_override_state != DR_OVERRIDDEN && 2392ab69d62fSMatthew Ahrens *datap == db->db_buf) { 2393fa9e4066Sahrens /* 239482c9918fSTim Haley * If this buffer is currently "in use" (i.e., there 239582c9918fSTim Haley * are active holds and db_data still references it), 239682c9918fSTim Haley * then make a copy before we start the write so that 239782c9918fSTim Haley * any modifications from the open txg will not leak 239882c9918fSTim Haley * into this write. 2399fa9e4066Sahrens * 240082c9918fSTim Haley * NOTE: this copy does not need to be made for 240182c9918fSTim Haley * objects only modified in the syncing context (e.g. 240282c9918fSTim Haley * DNONE_DNODE blocks). 2403fa9e4066Sahrens */ 2404ab69d62fSMatthew Ahrens int blksz = arc_buf_size(*datap); 2405ab69d62fSMatthew Ahrens arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 2406ab69d62fSMatthew Ahrens *datap = arc_buf_alloc(os->os_spa, blksz, db, type); 2407c717a561Smaybee bcopy(db->db.db_data, (*datap)->b_data, blksz); 2408fa9e4066Sahrens } 2409c717a561Smaybee db->db_data_pending = dr; 2410fa9e4066Sahrens 2411fa9e4066Sahrens mutex_exit(&db->db_mtx); 2412fa9e4066Sahrens 2413088f3894Sahrens dbuf_write(dr, *datap, tx); 2414c717a561Smaybee 2415c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 2416744947dcSTom Erickson if (dn->dn_object == DMU_META_DNODE_OBJECT) { 2417c717a561Smaybee list_insert_tail(&dn->dn_dirty_records[txg&TXG_MASK], dr); 2418744947dcSTom Erickson DB_DNODE_EXIT(db); 2419744947dcSTom Erickson } else { 2420744947dcSTom Erickson /* 2421744947dcSTom Erickson * Although zio_nowait() does not "wait for an IO", it does 2422744947dcSTom Erickson * initiate the IO. If this is an empty write it seems plausible 2423744947dcSTom Erickson * that the IO could actually be completed before the nowait 2424744947dcSTom Erickson * returns. We need to DB_DNODE_EXIT() first in case 2425744947dcSTom Erickson * zio_nowait() invalidates the dbuf. 2426744947dcSTom Erickson */ 2427744947dcSTom Erickson DB_DNODE_EXIT(db); 2428c717a561Smaybee zio_nowait(dr->dr_zio); 2429fa9e4066Sahrens } 2430744947dcSTom Erickson } 2431c717a561Smaybee 2432c717a561Smaybee void 2433c717a561Smaybee dbuf_sync_list(list_t *list, dmu_tx_t *tx) 2434c717a561Smaybee { 2435c717a561Smaybee dbuf_dirty_record_t *dr; 2436c717a561Smaybee 2437c717a561Smaybee while (dr = list_head(list)) { 2438c717a561Smaybee if (dr->dr_zio != NULL) { 2439c717a561Smaybee /* 2440c717a561Smaybee * If we find an already initialized zio then we 2441c717a561Smaybee * are processing the meta-dnode, and we have finished. 2442c717a561Smaybee * The dbufs for all dnodes are put back on the list 2443c717a561Smaybee * during processing, so that we can zio_wait() 2444c717a561Smaybee * these IOs after initiating all child IOs. 2445c717a561Smaybee */ 2446c717a561Smaybee ASSERT3U(dr->dr_dbuf->db.db_object, ==, 2447c717a561Smaybee DMU_META_DNODE_OBJECT); 2448c717a561Smaybee break; 2449fa9e4066Sahrens } 2450c717a561Smaybee list_remove(list, dr); 2451c717a561Smaybee if (dr->dr_dbuf->db_level > 0) 2452c717a561Smaybee dbuf_sync_indirect(dr, tx); 2453c717a561Smaybee else 2454c717a561Smaybee dbuf_sync_leaf(dr, tx); 2455c717a561Smaybee } 2456c717a561Smaybee } 2457c717a561Smaybee 2458fa9e4066Sahrens /* ARGSUSED */ 2459fa9e4066Sahrens static void 2460c717a561Smaybee dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb) 2461fa9e4066Sahrens { 2462fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 2463744947dcSTom Erickson dnode_t *dn; 2464e14bb325SJeff Bonwick blkptr_t *bp = zio->io_bp; 2465c717a561Smaybee blkptr_t *bp_orig = &zio->io_bp_orig; 2466b24ab676SJeff Bonwick spa_t *spa = zio->io_spa; 2467b24ab676SJeff Bonwick int64_t delta; 2468fa9e4066Sahrens uint64_t fill = 0; 2469b24ab676SJeff Bonwick int i; 2470fa9e4066Sahrens 2471e14bb325SJeff Bonwick ASSERT(db->db_blkptr == bp); 2472e14bb325SJeff Bonwick 2473744947dcSTom Erickson DB_DNODE_ENTER(db); 2474744947dcSTom Erickson dn = DB_DNODE(db); 2475b24ab676SJeff Bonwick delta = bp_get_dsize_sync(spa, bp) - bp_get_dsize_sync(spa, bp_orig); 2476b24ab676SJeff Bonwick dnode_diduse_space(dn, delta - zio->io_prev_space_delta); 2477b24ab676SJeff Bonwick zio->io_prev_space_delta = delta; 2478fa9e4066Sahrens 2479e14bb325SJeff Bonwick if (BP_IS_HOLE(bp)) { 2480b24ab676SJeff Bonwick ASSERT(bp->blk_fill == 0); 2481744947dcSTom Erickson DB_DNODE_EXIT(db); 2482c717a561Smaybee return; 2483c717a561Smaybee } 2484c717a561Smaybee 24850a586ceaSMark Shellenbaum ASSERT((db->db_blkid != DMU_SPILL_BLKID && 24860a586ceaSMark Shellenbaum BP_GET_TYPE(bp) == dn->dn_type) || 24870a586ceaSMark Shellenbaum (db->db_blkid == DMU_SPILL_BLKID && 24880a586ceaSMark Shellenbaum BP_GET_TYPE(bp) == dn->dn_bonustype)); 2489e14bb325SJeff Bonwick ASSERT(BP_GET_LEVEL(bp) == db->db_level); 2490e14bb325SJeff Bonwick 2491fa9e4066Sahrens mutex_enter(&db->db_mtx); 2492fa9e4066Sahrens 24930a586ceaSMark Shellenbaum #ifdef ZFS_DEBUG 24940a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) { 24950a586ceaSMark Shellenbaum ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); 24960a586ceaSMark Shellenbaum ASSERT(!(BP_IS_HOLE(db->db_blkptr)) && 24970a586ceaSMark Shellenbaum db->db_blkptr == &dn->dn_phys->dn_spill); 24980a586ceaSMark Shellenbaum } 24990a586ceaSMark Shellenbaum #endif 25000a586ceaSMark Shellenbaum 2501fa9e4066Sahrens if (db->db_level == 0) { 2502fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 25030a586ceaSMark Shellenbaum if (db->db_blkid > dn->dn_phys->dn_maxblkid && 25040a586ceaSMark Shellenbaum db->db_blkid != DMU_SPILL_BLKID) 2505fa9e4066Sahrens dn->dn_phys->dn_maxblkid = db->db_blkid; 2506fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 2507fa9e4066Sahrens 2508fa9e4066Sahrens if (dn->dn_type == DMU_OT_DNODE) { 2509fa9e4066Sahrens dnode_phys_t *dnp = db->db.db_data; 2510fa9e4066Sahrens for (i = db->db.db_size >> DNODE_SHIFT; i > 0; 2511fa9e4066Sahrens i--, dnp++) { 2512fa9e4066Sahrens if (dnp->dn_type != DMU_OT_NONE) 2513fa9e4066Sahrens fill++; 2514fa9e4066Sahrens } 2515fa9e4066Sahrens } else { 2516fa9e4066Sahrens fill = 1; 2517fa9e4066Sahrens } 2518fa9e4066Sahrens } else { 2519e14bb325SJeff Bonwick blkptr_t *ibp = db->db.db_data; 2520fa9e4066Sahrens ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2521e14bb325SJeff Bonwick for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, ibp++) { 2522e14bb325SJeff Bonwick if (BP_IS_HOLE(ibp)) 2523fa9e4066Sahrens continue; 2524e14bb325SJeff Bonwick fill += ibp->blk_fill; 2525fa9e4066Sahrens } 2526fa9e4066Sahrens } 2527744947dcSTom Erickson DB_DNODE_EXIT(db); 2528fa9e4066Sahrens 2529e14bb325SJeff Bonwick bp->blk_fill = fill; 2530fa9e4066Sahrens 2531fa9e4066Sahrens mutex_exit(&db->db_mtx); 2532fa9e4066Sahrens } 2533fa9e4066Sahrens 2534c717a561Smaybee /* ARGSUSED */ 2535c717a561Smaybee static void 2536c717a561Smaybee dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb) 2537c717a561Smaybee { 2538c717a561Smaybee dmu_buf_impl_t *db = vdb; 2539b24ab676SJeff Bonwick blkptr_t *bp = zio->io_bp; 2540b24ab676SJeff Bonwick blkptr_t *bp_orig = &zio->io_bp_orig; 2541c717a561Smaybee uint64_t txg = zio->io_txg; 2542c717a561Smaybee dbuf_dirty_record_t **drp, *dr; 2543c717a561Smaybee 2544fb09f5aaSMadhav Suresh ASSERT0(zio->io_error); 2545b24ab676SJeff Bonwick ASSERT(db->db_blkptr == bp); 2546b24ab676SJeff Bonwick 254780901aeaSGeorge Wilson /* 254880901aeaSGeorge Wilson * For nopwrites and rewrites we ensure that the bp matches our 254980901aeaSGeorge Wilson * original and bypass all the accounting. 255080901aeaSGeorge Wilson */ 255180901aeaSGeorge Wilson if (zio->io_flags & (ZIO_FLAG_IO_REWRITE | ZIO_FLAG_NOPWRITE)) { 2552b24ab676SJeff Bonwick ASSERT(BP_EQUAL(bp, bp_orig)); 2553b24ab676SJeff Bonwick } else { 2554744947dcSTom Erickson objset_t *os; 2555744947dcSTom Erickson dsl_dataset_t *ds; 2556744947dcSTom Erickson dmu_tx_t *tx; 2557744947dcSTom Erickson 2558744947dcSTom Erickson DB_GET_OBJSET(&os, db); 2559744947dcSTom Erickson ds = os->os_dsl_dataset; 2560744947dcSTom Erickson tx = os->os_synctx; 2561b24ab676SJeff Bonwick 2562b24ab676SJeff Bonwick (void) dsl_dataset_block_kill(ds, bp_orig, tx, B_TRUE); 2563b24ab676SJeff Bonwick dsl_dataset_block_born(ds, bp, tx); 2564b24ab676SJeff Bonwick } 2565c717a561Smaybee 2566c717a561Smaybee mutex_enter(&db->db_mtx); 2567c717a561Smaybee 2568b24ab676SJeff Bonwick DBUF_VERIFY(db); 2569b24ab676SJeff Bonwick 2570c717a561Smaybee drp = &db->db_last_dirty; 257117f17c2dSbonwick while ((dr = *drp) != db->db_data_pending) 257217f17c2dSbonwick drp = &dr->dr_next; 257317f17c2dSbonwick ASSERT(!list_link_active(&dr->dr_dirty_node)); 257417f17c2dSbonwick ASSERT(dr->dr_txg == txg); 2575b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 257617f17c2dSbonwick ASSERT(dr->dr_next == NULL); 257717f17c2dSbonwick *drp = dr->dr_next; 2578c717a561Smaybee 25790a586ceaSMark Shellenbaum #ifdef ZFS_DEBUG 25800a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) { 2581744947dcSTom Erickson dnode_t *dn; 2582744947dcSTom Erickson 2583744947dcSTom Erickson DB_DNODE_ENTER(db); 2584744947dcSTom Erickson dn = DB_DNODE(db); 25850a586ceaSMark Shellenbaum ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); 25860a586ceaSMark Shellenbaum ASSERT(!(BP_IS_HOLE(db->db_blkptr)) && 25870a586ceaSMark Shellenbaum db->db_blkptr == &dn->dn_phys->dn_spill); 2588744947dcSTom Erickson DB_DNODE_EXIT(db); 25890a586ceaSMark Shellenbaum } 25900a586ceaSMark Shellenbaum #endif 25910a586ceaSMark Shellenbaum 2592c717a561Smaybee if (db->db_level == 0) { 25930a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 2594c717a561Smaybee ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN); 259582c9918fSTim Haley if (db->db_state != DB_NOFILL) { 2596c717a561Smaybee if (dr->dt.dl.dr_data != db->db_buf) 259782c9918fSTim Haley VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, 25983b2aab18SMatthew Ahrens db)); 2599b24ab676SJeff Bonwick else if (!arc_released(db->db_buf)) 2600c717a561Smaybee arc_set_callback(db->db_buf, dbuf_do_evict, db); 260182c9918fSTim Haley } 2602c717a561Smaybee } else { 2603744947dcSTom Erickson dnode_t *dn; 2604744947dcSTom Erickson 2605744947dcSTom Erickson DB_DNODE_ENTER(db); 2606744947dcSTom Erickson dn = DB_DNODE(db); 2607c717a561Smaybee ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 2608c717a561Smaybee ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2609c717a561Smaybee if (!BP_IS_HOLE(db->db_blkptr)) { 2610c717a561Smaybee int epbs = 2611c717a561Smaybee dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2612c717a561Smaybee ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, 2613c717a561Smaybee db->db.db_size); 2614c717a561Smaybee ASSERT3U(dn->dn_phys->dn_maxblkid 2615c717a561Smaybee >> (db->db_level * epbs), >=, db->db_blkid); 2616c717a561Smaybee arc_set_callback(db->db_buf, dbuf_do_evict, db); 2617c717a561Smaybee } 2618744947dcSTom Erickson DB_DNODE_EXIT(db); 2619c25056deSgw25295 mutex_destroy(&dr->dt.di.dr_mtx); 2620c25056deSgw25295 list_destroy(&dr->dt.di.dr_children); 2621c717a561Smaybee } 2622c717a561Smaybee kmem_free(dr, sizeof (dbuf_dirty_record_t)); 2623c717a561Smaybee 2624c717a561Smaybee cv_broadcast(&db->db_changed); 2625c717a561Smaybee ASSERT(db->db_dirtycnt > 0); 2626c717a561Smaybee db->db_dirtycnt -= 1; 2627c717a561Smaybee db->db_data_pending = NULL; 2628b24ab676SJeff Bonwick dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg); 2629b24ab676SJeff Bonwick } 2630b24ab676SJeff Bonwick 2631b24ab676SJeff Bonwick static void 2632b24ab676SJeff Bonwick dbuf_write_nofill_ready(zio_t *zio) 2633b24ab676SJeff Bonwick { 2634b24ab676SJeff Bonwick dbuf_write_ready(zio, NULL, zio->io_private); 2635b24ab676SJeff Bonwick } 2636b24ab676SJeff Bonwick 2637b24ab676SJeff Bonwick static void 2638b24ab676SJeff Bonwick dbuf_write_nofill_done(zio_t *zio) 2639b24ab676SJeff Bonwick { 2640b24ab676SJeff Bonwick dbuf_write_done(zio, NULL, zio->io_private); 2641b24ab676SJeff Bonwick } 2642b24ab676SJeff Bonwick 2643b24ab676SJeff Bonwick static void 2644b24ab676SJeff Bonwick dbuf_write_override_ready(zio_t *zio) 2645b24ab676SJeff Bonwick { 2646b24ab676SJeff Bonwick dbuf_dirty_record_t *dr = zio->io_private; 2647b24ab676SJeff Bonwick dmu_buf_impl_t *db = dr->dr_dbuf; 2648b24ab676SJeff Bonwick 2649b24ab676SJeff Bonwick dbuf_write_ready(zio, NULL, db); 2650b24ab676SJeff Bonwick } 2651b24ab676SJeff Bonwick 2652b24ab676SJeff Bonwick static void 2653b24ab676SJeff Bonwick dbuf_write_override_done(zio_t *zio) 2654b24ab676SJeff Bonwick { 2655b24ab676SJeff Bonwick dbuf_dirty_record_t *dr = zio->io_private; 2656b24ab676SJeff Bonwick dmu_buf_impl_t *db = dr->dr_dbuf; 2657b24ab676SJeff Bonwick blkptr_t *obp = &dr->dt.dl.dr_overridden_by; 2658b24ab676SJeff Bonwick 2659b24ab676SJeff Bonwick mutex_enter(&db->db_mtx); 2660b24ab676SJeff Bonwick if (!BP_EQUAL(zio->io_bp, obp)) { 2661b24ab676SJeff Bonwick if (!BP_IS_HOLE(obp)) 2662b24ab676SJeff Bonwick dsl_free(spa_get_dsl(zio->io_spa), zio->io_txg, obp); 2663b24ab676SJeff Bonwick arc_release(dr->dt.dl.dr_data, db); 2664b24ab676SJeff Bonwick } 2665c717a561Smaybee mutex_exit(&db->db_mtx); 2666c717a561Smaybee 2667b24ab676SJeff Bonwick dbuf_write_done(zio, NULL, db); 2668b24ab676SJeff Bonwick } 2669c717a561Smaybee 26703e30c24aSWill Andrews /* Issue I/O to commit a dirty buffer to disk. */ 2671b24ab676SJeff Bonwick static void 2672b24ab676SJeff Bonwick dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx) 2673b24ab676SJeff Bonwick { 2674b24ab676SJeff Bonwick dmu_buf_impl_t *db = dr->dr_dbuf; 2675744947dcSTom Erickson dnode_t *dn; 2676744947dcSTom Erickson objset_t *os; 2677b24ab676SJeff Bonwick dmu_buf_impl_t *parent = db->db_parent; 2678b24ab676SJeff Bonwick uint64_t txg = tx->tx_txg; 2679b24ab676SJeff Bonwick zbookmark_t zb; 2680b24ab676SJeff Bonwick zio_prop_t zp; 2681b24ab676SJeff Bonwick zio_t *zio; 26820a586ceaSMark Shellenbaum int wp_flag = 0; 2683b24ab676SJeff Bonwick 2684744947dcSTom Erickson DB_DNODE_ENTER(db); 2685744947dcSTom Erickson dn = DB_DNODE(db); 2686744947dcSTom Erickson os = dn->dn_objset; 2687744947dcSTom Erickson 2688b24ab676SJeff Bonwick if (db->db_state != DB_NOFILL) { 2689b24ab676SJeff Bonwick if (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE) { 2690b24ab676SJeff Bonwick /* 2691b24ab676SJeff Bonwick * Private object buffers are released here rather 2692b24ab676SJeff Bonwick * than in dbuf_dirty() since they are only modified 2693b24ab676SJeff Bonwick * in the syncing context and we don't want the 2694b24ab676SJeff Bonwick * overhead of making multiple copies of the data. 2695b24ab676SJeff Bonwick */ 2696b24ab676SJeff Bonwick if (BP_IS_HOLE(db->db_blkptr)) { 2697b24ab676SJeff Bonwick arc_buf_thaw(data); 2698b24ab676SJeff Bonwick } else { 26993f9d6ad7SLin Ling dbuf_release_bp(db); 2700b24ab676SJeff Bonwick } 2701b24ab676SJeff Bonwick } 2702b24ab676SJeff Bonwick } 2703b24ab676SJeff Bonwick 2704b24ab676SJeff Bonwick if (parent != dn->dn_dbuf) { 27053e30c24aSWill Andrews /* Our parent is an indirect block. */ 27063e30c24aSWill Andrews /* We have a dirty parent that has been scheduled for write. */ 2707b24ab676SJeff Bonwick ASSERT(parent && parent->db_data_pending); 27083e30c24aSWill Andrews /* Our parent's buffer is one level closer to the dnode. */ 2709b24ab676SJeff Bonwick ASSERT(db->db_level == parent->db_level-1); 27103e30c24aSWill Andrews /* 27113e30c24aSWill Andrews * We're about to modify our parent's db_data by modifying 27123e30c24aSWill Andrews * our block pointer, so the parent must be released. 27133e30c24aSWill Andrews */ 2714b24ab676SJeff Bonwick ASSERT(arc_released(parent->db_buf)); 2715b24ab676SJeff Bonwick zio = parent->db_data_pending->dr_zio; 2716b24ab676SJeff Bonwick } else { 27173e30c24aSWill Andrews /* Our parent is the dnode itself. */ 27180a586ceaSMark Shellenbaum ASSERT((db->db_level == dn->dn_phys->dn_nlevels-1 && 27190a586ceaSMark Shellenbaum db->db_blkid != DMU_SPILL_BLKID) || 27200a586ceaSMark Shellenbaum (db->db_blkid == DMU_SPILL_BLKID && db->db_level == 0)); 27210a586ceaSMark Shellenbaum if (db->db_blkid != DMU_SPILL_BLKID) 2722b24ab676SJeff Bonwick ASSERT3P(db->db_blkptr, ==, 2723b24ab676SJeff Bonwick &dn->dn_phys->dn_blkptr[db->db_blkid]); 2724b24ab676SJeff Bonwick zio = dn->dn_zio; 2725b24ab676SJeff Bonwick } 2726b24ab676SJeff Bonwick 2727b24ab676SJeff Bonwick ASSERT(db->db_level == 0 || data == db->db_buf); 2728b24ab676SJeff Bonwick ASSERT3U(db->db_blkptr->blk_birth, <=, txg); 2729b24ab676SJeff Bonwick ASSERT(zio); 2730b24ab676SJeff Bonwick 2731b24ab676SJeff Bonwick SET_BOOKMARK(&zb, os->os_dsl_dataset ? 2732b24ab676SJeff Bonwick os->os_dsl_dataset->ds_object : DMU_META_OBJSET, 2733b24ab676SJeff Bonwick db->db.db_object, db->db_level, db->db_blkid); 2734b24ab676SJeff Bonwick 27350a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) 27360a586ceaSMark Shellenbaum wp_flag = WP_SPILL; 27370a586ceaSMark Shellenbaum wp_flag |= (db->db_state == DB_NOFILL) ? WP_NOFILL : 0; 27380a586ceaSMark Shellenbaum 27390a586ceaSMark Shellenbaum dmu_write_policy(os, dn, db->db_level, wp_flag, &zp); 2740744947dcSTom Erickson DB_DNODE_EXIT(db); 2741b24ab676SJeff Bonwick 2742b24ab676SJeff Bonwick if (db->db_level == 0 && dr->dt.dl.dr_override_state == DR_OVERRIDDEN) { 2743b24ab676SJeff Bonwick ASSERT(db->db_state != DB_NOFILL); 2744b24ab676SJeff Bonwick dr->dr_zio = zio_write(zio, os->os_spa, txg, 2745b24ab676SJeff Bonwick db->db_blkptr, data->b_data, arc_buf_size(data), &zp, 2746b24ab676SJeff Bonwick dbuf_write_override_ready, dbuf_write_override_done, dr, 2747b24ab676SJeff Bonwick ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); 2748b24ab676SJeff Bonwick mutex_enter(&db->db_mtx); 2749b24ab676SJeff Bonwick dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 2750b24ab676SJeff Bonwick zio_write_override(dr->dr_zio, &dr->dt.dl.dr_overridden_by, 275180901aeaSGeorge Wilson dr->dt.dl.dr_copies, dr->dt.dl.dr_nopwrite); 2752b24ab676SJeff Bonwick mutex_exit(&db->db_mtx); 2753b24ab676SJeff Bonwick } else if (db->db_state == DB_NOFILL) { 2754b24ab676SJeff Bonwick ASSERT(zp.zp_checksum == ZIO_CHECKSUM_OFF); 2755b24ab676SJeff Bonwick dr->dr_zio = zio_write(zio, os->os_spa, txg, 2756b24ab676SJeff Bonwick db->db_blkptr, NULL, db->db.db_size, &zp, 2757b24ab676SJeff Bonwick dbuf_write_nofill_ready, dbuf_write_nofill_done, db, 2758b24ab676SJeff Bonwick ZIO_PRIORITY_ASYNC_WRITE, 2759b24ab676SJeff Bonwick ZIO_FLAG_MUSTSUCCEED | ZIO_FLAG_NODATA, &zb); 2760b24ab676SJeff Bonwick } else { 2761b24ab676SJeff Bonwick ASSERT(arc_released(data)); 2762b24ab676SJeff Bonwick dr->dr_zio = arc_write(zio, os->os_spa, txg, 2763aad02571SSaso Kiselkov db->db_blkptr, data, DBUF_IS_L2CACHEABLE(db), 2764aad02571SSaso Kiselkov DBUF_IS_L2COMPRESSIBLE(db), &zp, dbuf_write_ready, 2765aad02571SSaso Kiselkov dbuf_write_done, db, ZIO_PRIORITY_ASYNC_WRITE, 2766aad02571SSaso Kiselkov ZIO_FLAG_MUSTSUCCEED, &zb); 2767b24ab676SJeff Bonwick } 2768fa9e4066Sahrens } 2769