1fa9e4066Sahrens /* 2fa9e4066Sahrens * CDDL HEADER START 3fa9e4066Sahrens * 4fa9e4066Sahrens * The contents of this file are subject to the terms of the 5f65e61c0Sahrens * Common Development and Distribution License (the "License"). 6f65e61c0Sahrens * You may not use this file except in compliance with the License. 7fa9e4066Sahrens * 8fa9e4066Sahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9fa9e4066Sahrens * or http://www.opensolaris.org/os/licensing. 10fa9e4066Sahrens * See the License for the specific language governing permissions 11fa9e4066Sahrens * and limitations under the License. 12fa9e4066Sahrens * 13fa9e4066Sahrens * When distributing Covered Code, include this CDDL HEADER in each 14fa9e4066Sahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15fa9e4066Sahrens * If applicable, add the following below this CDDL HEADER, with the 16fa9e4066Sahrens * fields enclosed by brackets "[]" replaced with your own identifying 17fa9e4066Sahrens * information: Portions Copyright [yyyy] [name of copyright owner] 18fa9e4066Sahrens * 19fa9e4066Sahrens * CDDL HEADER END 20fa9e4066Sahrens */ 21fa9e4066Sahrens /* 2206e0070dSMark Shellenbaum * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 233f2366c2SGordon Ross * Copyright 2011 Nexenta Systems, Inc. All rights reserved. 2446e1baa6SMatthew Ahrens * Copyright (c) 2012, 2015 by Delphix. All rights reserved. 25aad02571SSaso Kiselkov * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. 26810e43b2SBill Pijewski * Copyright (c) 2013, Joyent, Inc. All rights reserved. 27bc9014e6SJustin Gibbs * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. 28fa9e4066Sahrens */ 29fa9e4066Sahrens 30fa9e4066Sahrens #include <sys/zfs_context.h> 31fa9e4066Sahrens #include <sys/dmu.h> 322f3d8780SMatthew Ahrens #include <sys/dmu_send.h> 33fa9e4066Sahrens #include <sys/dmu_impl.h> 34fa9e4066Sahrens #include <sys/dbuf.h> 35fa9e4066Sahrens #include <sys/dmu_objset.h> 36fa9e4066Sahrens #include <sys/dsl_dataset.h> 37fa9e4066Sahrens #include <sys/dsl_dir.h> 38fa9e4066Sahrens #include <sys/dmu_tx.h> 39fa9e4066Sahrens #include <sys/spa.h> 40fa9e4066Sahrens #include <sys/zio.h> 41fa9e4066Sahrens #include <sys/dmu_zfetch.h> 420a586ceaSMark Shellenbaum #include <sys/sa.h> 430a586ceaSMark Shellenbaum #include <sys/sa_impl.h> 445d7b4d43SMatthew Ahrens #include <sys/zfeature.h> 455d7b4d43SMatthew Ahrens #include <sys/blkptr.h> 46bf16b11eSMatthew Ahrens #include <sys/range_tree.h> 47fa9e4066Sahrens 48713d6c20SMatthew Ahrens /* 49713d6c20SMatthew Ahrens * Number of times that zfs_free_range() took the slow path while doing 50713d6c20SMatthew Ahrens * a zfs receive. A nonzero value indicates a potential performance problem. 51713d6c20SMatthew Ahrens */ 52713d6c20SMatthew Ahrens uint64_t zfs_free_range_recv_miss; 53713d6c20SMatthew Ahrens 54fa9e4066Sahrens static void dbuf_destroy(dmu_buf_impl_t *db); 553b2aab18SMatthew Ahrens static boolean_t dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx); 56088f3894Sahrens static void dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx); 57fa9e4066Sahrens 58bc9014e6SJustin Gibbs #ifndef __lint 59bc9014e6SJustin Gibbs extern inline void dmu_buf_init_user(dmu_buf_user_t *dbu, 60bc9014e6SJustin Gibbs dmu_buf_evict_func_t *evict_func, dmu_buf_t **clear_on_evict_dbufp); 61bc9014e6SJustin Gibbs #endif /* ! __lint */ 62bc9014e6SJustin Gibbs 63fa9e4066Sahrens /* 64fa9e4066Sahrens * Global data structures and functions for the dbuf cache. 65fa9e4066Sahrens */ 66fa9e4066Sahrens static kmem_cache_t *dbuf_cache; 67bc9014e6SJustin Gibbs static taskq_t *dbu_evict_taskq; 68fa9e4066Sahrens 69fa9e4066Sahrens /* ARGSUSED */ 70fa9e4066Sahrens static int 71fa9e4066Sahrens dbuf_cons(void *vdb, void *unused, int kmflag) 72fa9e4066Sahrens { 73fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 74fa9e4066Sahrens bzero(db, sizeof (dmu_buf_impl_t)); 75fa9e4066Sahrens 76fa9e4066Sahrens mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL); 77fa9e4066Sahrens cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL); 78fa9e4066Sahrens refcount_create(&db->db_holds); 790f6d88adSAlex Reece 80fa9e4066Sahrens return (0); 81fa9e4066Sahrens } 82fa9e4066Sahrens 83fa9e4066Sahrens /* ARGSUSED */ 84fa9e4066Sahrens static void 85fa9e4066Sahrens dbuf_dest(void *vdb, void *unused) 86fa9e4066Sahrens { 87fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 88fa9e4066Sahrens mutex_destroy(&db->db_mtx); 89fa9e4066Sahrens cv_destroy(&db->db_changed); 90fa9e4066Sahrens refcount_destroy(&db->db_holds); 91fa9e4066Sahrens } 92fa9e4066Sahrens 93fa9e4066Sahrens /* 94fa9e4066Sahrens * dbuf hash table routines 95fa9e4066Sahrens */ 96fa9e4066Sahrens static dbuf_hash_table_t dbuf_hash_table; 97fa9e4066Sahrens 98fa9e4066Sahrens static uint64_t dbuf_hash_count; 99fa9e4066Sahrens 100fa9e4066Sahrens static uint64_t 101fa9e4066Sahrens dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid) 102fa9e4066Sahrens { 103fa9e4066Sahrens uintptr_t osv = (uintptr_t)os; 104fa9e4066Sahrens uint64_t crc = -1ULL; 105fa9e4066Sahrens 106fa9e4066Sahrens ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 107fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (lvl)) & 0xFF]; 108fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF]; 109fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF]; 110fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF]; 111fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 0)) & 0xFF]; 112fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 8)) & 0xFF]; 113fa9e4066Sahrens 114fa9e4066Sahrens crc ^= (osv>>14) ^ (obj>>16) ^ (blkid>>16); 115fa9e4066Sahrens 116fa9e4066Sahrens return (crc); 117fa9e4066Sahrens } 118fa9e4066Sahrens 119fa9e4066Sahrens #define DBUF_HASH(os, obj, level, blkid) dbuf_hash(os, obj, level, blkid); 120fa9e4066Sahrens 121fa9e4066Sahrens #define DBUF_EQUAL(dbuf, os, obj, level, blkid) \ 122fa9e4066Sahrens ((dbuf)->db.db_object == (obj) && \ 123fa9e4066Sahrens (dbuf)->db_objset == (os) && \ 124fa9e4066Sahrens (dbuf)->db_level == (level) && \ 125fa9e4066Sahrens (dbuf)->db_blkid == (blkid)) 126fa9e4066Sahrens 127fa9e4066Sahrens dmu_buf_impl_t * 128e57a022bSJustin T. Gibbs dbuf_find(objset_t *os, uint64_t obj, uint8_t level, uint64_t blkid) 129fa9e4066Sahrens { 130fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 131fa9e4066Sahrens uint64_t hv = DBUF_HASH(os, obj, level, blkid); 132fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 133fa9e4066Sahrens dmu_buf_impl_t *db; 134fa9e4066Sahrens 135fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 136fa9e4066Sahrens for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) { 137fa9e4066Sahrens if (DBUF_EQUAL(db, os, obj, level, blkid)) { 138fa9e4066Sahrens mutex_enter(&db->db_mtx); 139ea8dc4b6Seschrock if (db->db_state != DB_EVICTING) { 140fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 141fa9e4066Sahrens return (db); 142fa9e4066Sahrens } 143fa9e4066Sahrens mutex_exit(&db->db_mtx); 144fa9e4066Sahrens } 145fa9e4066Sahrens } 146fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 147fa9e4066Sahrens return (NULL); 148fa9e4066Sahrens } 149fa9e4066Sahrens 150e57a022bSJustin T. Gibbs static dmu_buf_impl_t * 151e57a022bSJustin T. Gibbs dbuf_find_bonus(objset_t *os, uint64_t object) 152e57a022bSJustin T. Gibbs { 153e57a022bSJustin T. Gibbs dnode_t *dn; 154e57a022bSJustin T. Gibbs dmu_buf_impl_t *db = NULL; 155e57a022bSJustin T. Gibbs 156e57a022bSJustin T. Gibbs if (dnode_hold(os, object, FTAG, &dn) == 0) { 157e57a022bSJustin T. Gibbs rw_enter(&dn->dn_struct_rwlock, RW_READER); 158e57a022bSJustin T. Gibbs if (dn->dn_bonus != NULL) { 159e57a022bSJustin T. Gibbs db = dn->dn_bonus; 160e57a022bSJustin T. Gibbs mutex_enter(&db->db_mtx); 161e57a022bSJustin T. Gibbs } 162e57a022bSJustin T. Gibbs rw_exit(&dn->dn_struct_rwlock); 163e57a022bSJustin T. Gibbs dnode_rele(dn, FTAG); 164e57a022bSJustin T. Gibbs } 165e57a022bSJustin T. Gibbs return (db); 166e57a022bSJustin T. Gibbs } 167e57a022bSJustin T. Gibbs 168fa9e4066Sahrens /* 169fa9e4066Sahrens * Insert an entry into the hash table. If there is already an element 170fa9e4066Sahrens * equal to elem in the hash table, then the already existing element 171fa9e4066Sahrens * will be returned and the new element will not be inserted. 172fa9e4066Sahrens * Otherwise returns NULL. 173fa9e4066Sahrens */ 174fa9e4066Sahrens static dmu_buf_impl_t * 175fa9e4066Sahrens dbuf_hash_insert(dmu_buf_impl_t *db) 176fa9e4066Sahrens { 177fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 178503ad85cSMatthew Ahrens objset_t *os = db->db_objset; 179fa9e4066Sahrens uint64_t obj = db->db.db_object; 180fa9e4066Sahrens int level = db->db_level; 181fa9e4066Sahrens uint64_t blkid = db->db_blkid; 182fa9e4066Sahrens uint64_t hv = DBUF_HASH(os, obj, level, blkid); 183fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 184fa9e4066Sahrens dmu_buf_impl_t *dbf; 185fa9e4066Sahrens 186fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 187fa9e4066Sahrens for (dbf = h->hash_table[idx]; dbf != NULL; dbf = dbf->db_hash_next) { 188fa9e4066Sahrens if (DBUF_EQUAL(dbf, os, obj, level, blkid)) { 189fa9e4066Sahrens mutex_enter(&dbf->db_mtx); 190ea8dc4b6Seschrock if (dbf->db_state != DB_EVICTING) { 191fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 192fa9e4066Sahrens return (dbf); 193fa9e4066Sahrens } 194fa9e4066Sahrens mutex_exit(&dbf->db_mtx); 195fa9e4066Sahrens } 196fa9e4066Sahrens } 197fa9e4066Sahrens 198fa9e4066Sahrens mutex_enter(&db->db_mtx); 199fa9e4066Sahrens db->db_hash_next = h->hash_table[idx]; 200fa9e4066Sahrens h->hash_table[idx] = db; 201fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 2021a5e258fSJosef 'Jeff' Sipek atomic_inc_64(&dbuf_hash_count); 203fa9e4066Sahrens 204fa9e4066Sahrens return (NULL); 205fa9e4066Sahrens } 206fa9e4066Sahrens 207fa9e4066Sahrens /* 208bbfa8ea8SMatthew Ahrens * Remove an entry from the hash table. It must be in the EVICTING state. 209fa9e4066Sahrens */ 210fa9e4066Sahrens static void 211fa9e4066Sahrens dbuf_hash_remove(dmu_buf_impl_t *db) 212fa9e4066Sahrens { 213fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 214fa9e4066Sahrens uint64_t hv = DBUF_HASH(db->db_objset, db->db.db_object, 215fa9e4066Sahrens db->db_level, db->db_blkid); 216fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 217fa9e4066Sahrens dmu_buf_impl_t *dbf, **dbp; 218fa9e4066Sahrens 219fa9e4066Sahrens /* 220bbfa8ea8SMatthew Ahrens * We musn't hold db_mtx to maintain lock ordering: 221fa9e4066Sahrens * DBUF_HASH_MUTEX > db_mtx. 222fa9e4066Sahrens */ 223fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 224ea8dc4b6Seschrock ASSERT(db->db_state == DB_EVICTING); 225fa9e4066Sahrens ASSERT(!MUTEX_HELD(&db->db_mtx)); 226fa9e4066Sahrens 227fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 228fa9e4066Sahrens dbp = &h->hash_table[idx]; 229fa9e4066Sahrens while ((dbf = *dbp) != db) { 230fa9e4066Sahrens dbp = &dbf->db_hash_next; 231fa9e4066Sahrens ASSERT(dbf != NULL); 232fa9e4066Sahrens } 233fa9e4066Sahrens *dbp = db->db_hash_next; 234fa9e4066Sahrens db->db_hash_next = NULL; 235fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 2361a5e258fSJosef 'Jeff' Sipek atomic_dec_64(&dbuf_hash_count); 237fa9e4066Sahrens } 238fa9e4066Sahrens 239ea8dc4b6Seschrock static arc_evict_func_t dbuf_do_evict; 240fa9e4066Sahrens 241bc9014e6SJustin Gibbs typedef enum { 242bc9014e6SJustin Gibbs DBVU_EVICTING, 243bc9014e6SJustin Gibbs DBVU_NOT_EVICTING 244bc9014e6SJustin Gibbs } dbvu_verify_type_t; 245bc9014e6SJustin Gibbs 246bc9014e6SJustin Gibbs static void 247bc9014e6SJustin Gibbs dbuf_verify_user(dmu_buf_impl_t *db, dbvu_verify_type_t verify_type) 248bc9014e6SJustin Gibbs { 249bc9014e6SJustin Gibbs #ifdef ZFS_DEBUG 250bc9014e6SJustin Gibbs int64_t holds; 251bc9014e6SJustin Gibbs 252bc9014e6SJustin Gibbs if (db->db_user == NULL) 253bc9014e6SJustin Gibbs return; 254bc9014e6SJustin Gibbs 255bc9014e6SJustin Gibbs /* Only data blocks support the attachment of user data. */ 256bc9014e6SJustin Gibbs ASSERT(db->db_level == 0); 257bc9014e6SJustin Gibbs 258bc9014e6SJustin Gibbs /* Clients must resolve a dbuf before attaching user data. */ 259bc9014e6SJustin Gibbs ASSERT(db->db.db_data != NULL); 260bc9014e6SJustin Gibbs ASSERT3U(db->db_state, ==, DB_CACHED); 261bc9014e6SJustin Gibbs 262bc9014e6SJustin Gibbs holds = refcount_count(&db->db_holds); 263bc9014e6SJustin Gibbs if (verify_type == DBVU_EVICTING) { 264bc9014e6SJustin Gibbs /* 265bc9014e6SJustin Gibbs * Immediate eviction occurs when holds == dirtycnt. 266bc9014e6SJustin Gibbs * For normal eviction buffers, holds is zero on 267bc9014e6SJustin Gibbs * eviction, except when dbuf_fix_old_data() calls 268bc9014e6SJustin Gibbs * dbuf_clear_data(). However, the hold count can grow 269bc9014e6SJustin Gibbs * during eviction even though db_mtx is held (see 270bc9014e6SJustin Gibbs * dmu_bonus_hold() for an example), so we can only 271bc9014e6SJustin Gibbs * test the generic invariant that holds >= dirtycnt. 272bc9014e6SJustin Gibbs */ 273bc9014e6SJustin Gibbs ASSERT3U(holds, >=, db->db_dirtycnt); 274bc9014e6SJustin Gibbs } else { 275bc9014e6SJustin Gibbs if (db->db_immediate_evict == TRUE) 276bc9014e6SJustin Gibbs ASSERT3U(holds, >=, db->db_dirtycnt); 277bc9014e6SJustin Gibbs else 278bc9014e6SJustin Gibbs ASSERT3U(holds, >, 0); 279bc9014e6SJustin Gibbs } 280bc9014e6SJustin Gibbs #endif 281bc9014e6SJustin Gibbs } 282bc9014e6SJustin Gibbs 283fa9e4066Sahrens static void 284fa9e4066Sahrens dbuf_evict_user(dmu_buf_impl_t *db) 285fa9e4066Sahrens { 286bc9014e6SJustin Gibbs dmu_buf_user_t *dbu = db->db_user; 287bc9014e6SJustin Gibbs 288fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 289fa9e4066Sahrens 290bc9014e6SJustin Gibbs if (dbu == NULL) 291fa9e4066Sahrens return; 292fa9e4066Sahrens 293bc9014e6SJustin Gibbs dbuf_verify_user(db, DBVU_EVICTING); 294bc9014e6SJustin Gibbs db->db_user = NULL; 295bc9014e6SJustin Gibbs 296bc9014e6SJustin Gibbs #ifdef ZFS_DEBUG 297bc9014e6SJustin Gibbs if (dbu->dbu_clear_on_evict_dbufp != NULL) 298bc9014e6SJustin Gibbs *dbu->dbu_clear_on_evict_dbufp = NULL; 299bc9014e6SJustin Gibbs #endif 300bc9014e6SJustin Gibbs 301bc9014e6SJustin Gibbs /* 302bc9014e6SJustin Gibbs * Invoke the callback from a taskq to avoid lock order reversals 303bc9014e6SJustin Gibbs * and limit stack depth. 304bc9014e6SJustin Gibbs */ 305bc9014e6SJustin Gibbs taskq_dispatch_ent(dbu_evict_taskq, dbu->dbu_evict_func, dbu, 0, 306bc9014e6SJustin Gibbs &dbu->dbu_tqent); 307fa9e4066Sahrens } 308fa9e4066Sahrens 309744947dcSTom Erickson boolean_t 310744947dcSTom Erickson dbuf_is_metadata(dmu_buf_impl_t *db) 311744947dcSTom Erickson { 312744947dcSTom Erickson if (db->db_level > 0) { 313744947dcSTom Erickson return (B_TRUE); 314744947dcSTom Erickson } else { 315744947dcSTom Erickson boolean_t is_metadata; 316744947dcSTom Erickson 317744947dcSTom Erickson DB_DNODE_ENTER(db); 318ad135b5dSChristopher Siden is_metadata = DMU_OT_IS_METADATA(DB_DNODE(db)->dn_type); 319744947dcSTom Erickson DB_DNODE_EXIT(db); 320744947dcSTom Erickson 321744947dcSTom Erickson return (is_metadata); 322744947dcSTom Erickson } 323744947dcSTom Erickson } 324744947dcSTom Erickson 325fa9e4066Sahrens void 326ea8dc4b6Seschrock dbuf_evict(dmu_buf_impl_t *db) 327ea8dc4b6Seschrock { 328ea8dc4b6Seschrock ASSERT(MUTEX_HELD(&db->db_mtx)); 329ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 330c717a561Smaybee ASSERT(db->db_data_pending == NULL); 331ea8dc4b6Seschrock 332ea8dc4b6Seschrock dbuf_clear(db); 333ea8dc4b6Seschrock dbuf_destroy(db); 334ea8dc4b6Seschrock } 335ea8dc4b6Seschrock 336ea8dc4b6Seschrock void 337fa9e4066Sahrens dbuf_init(void) 338fa9e4066Sahrens { 339ea8dc4b6Seschrock uint64_t hsize = 1ULL << 16; 340fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 341fa9e4066Sahrens int i; 342fa9e4066Sahrens 343fa9e4066Sahrens /* 344fa9e4066Sahrens * The hash table is big enough to fill all of physical memory 345ea8dc4b6Seschrock * with an average 4K block size. The table will take up 346ea8dc4b6Seschrock * totalmem*sizeof(void*)/4K (i.e. 2MB/GB with 8-byte pointers). 347fa9e4066Sahrens */ 348ea8dc4b6Seschrock while (hsize * 4096 < physmem * PAGESIZE) 349fa9e4066Sahrens hsize <<= 1; 350fa9e4066Sahrens 351ea8dc4b6Seschrock retry: 352fa9e4066Sahrens h->hash_table_mask = hsize - 1; 353ea8dc4b6Seschrock h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP); 354ea8dc4b6Seschrock if (h->hash_table == NULL) { 355ea8dc4b6Seschrock /* XXX - we should really return an error instead of assert */ 356ea8dc4b6Seschrock ASSERT(hsize > (1ULL << 10)); 357ea8dc4b6Seschrock hsize >>= 1; 358ea8dc4b6Seschrock goto retry; 359ea8dc4b6Seschrock } 360fa9e4066Sahrens 361fa9e4066Sahrens dbuf_cache = kmem_cache_create("dmu_buf_impl_t", 362fa9e4066Sahrens sizeof (dmu_buf_impl_t), 363fa9e4066Sahrens 0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0); 364fa9e4066Sahrens 365fa9e4066Sahrens for (i = 0; i < DBUF_MUTEXES; i++) 366fa9e4066Sahrens mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL); 367bc9014e6SJustin Gibbs 368bc9014e6SJustin Gibbs /* 369bc9014e6SJustin Gibbs * All entries are queued via taskq_dispatch_ent(), so min/maxalloc 370bc9014e6SJustin Gibbs * configuration is not required. 371bc9014e6SJustin Gibbs */ 372bc9014e6SJustin Gibbs dbu_evict_taskq = taskq_create("dbu_evict", 1, minclsyspri, 0, 0, 0); 373fa9e4066Sahrens } 374fa9e4066Sahrens 375fa9e4066Sahrens void 376fa9e4066Sahrens dbuf_fini(void) 377fa9e4066Sahrens { 378fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 379fa9e4066Sahrens int i; 380fa9e4066Sahrens 381fa9e4066Sahrens for (i = 0; i < DBUF_MUTEXES; i++) 382fa9e4066Sahrens mutex_destroy(&h->hash_mutexes[i]); 383fa9e4066Sahrens kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *)); 384fa9e4066Sahrens kmem_cache_destroy(dbuf_cache); 385bc9014e6SJustin Gibbs taskq_destroy(dbu_evict_taskq); 386fa9e4066Sahrens } 387fa9e4066Sahrens 388fa9e4066Sahrens /* 389fa9e4066Sahrens * Other stuff. 390fa9e4066Sahrens */ 391fa9e4066Sahrens 3929c9dc39aSek110237 #ifdef ZFS_DEBUG 393fa9e4066Sahrens static void 394fa9e4066Sahrens dbuf_verify(dmu_buf_impl_t *db) 395fa9e4066Sahrens { 396744947dcSTom Erickson dnode_t *dn; 397b24ab676SJeff Bonwick dbuf_dirty_record_t *dr; 398fa9e4066Sahrens 399fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 400fa9e4066Sahrens 401fa9e4066Sahrens if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY)) 402fa9e4066Sahrens return; 403fa9e4066Sahrens 404fa9e4066Sahrens ASSERT(db->db_objset != NULL); 405744947dcSTom Erickson DB_DNODE_ENTER(db); 406744947dcSTom Erickson dn = DB_DNODE(db); 407fa9e4066Sahrens if (dn == NULL) { 408fa9e4066Sahrens ASSERT(db->db_parent == NULL); 409fa9e4066Sahrens ASSERT(db->db_blkptr == NULL); 410fa9e4066Sahrens } else { 411fa9e4066Sahrens ASSERT3U(db->db.db_object, ==, dn->dn_object); 412fa9e4066Sahrens ASSERT3P(db->db_objset, ==, dn->dn_objset); 413fa9e4066Sahrens ASSERT3U(db->db_level, <, dn->dn_nlevels); 414744947dcSTom Erickson ASSERT(db->db_blkid == DMU_BONUS_BLKID || 415744947dcSTom Erickson db->db_blkid == DMU_SPILL_BLKID || 4160f6d88adSAlex Reece !avl_is_empty(&dn->dn_dbufs)); 417fa9e4066Sahrens } 4180a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 419fa9e4066Sahrens ASSERT(dn != NULL); 4201934e92fSmaybee ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 4210a586ceaSMark Shellenbaum ASSERT3U(db->db.db_offset, ==, DMU_BONUS_BLKID); 4220a586ceaSMark Shellenbaum } else if (db->db_blkid == DMU_SPILL_BLKID) { 4230a586ceaSMark Shellenbaum ASSERT(dn != NULL); 4240a586ceaSMark Shellenbaum ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 425fb09f5aaSMadhav Suresh ASSERT0(db->db.db_offset); 426fa9e4066Sahrens } else { 427fa9e4066Sahrens ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size); 428fa9e4066Sahrens } 429fa9e4066Sahrens 430b24ab676SJeff Bonwick for (dr = db->db_data_pending; dr != NULL; dr = dr->dr_next) 431b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 432b24ab676SJeff Bonwick 433b24ab676SJeff Bonwick for (dr = db->db_last_dirty; dr != NULL; dr = dr->dr_next) 434b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 435b24ab676SJeff Bonwick 43688b7b0f2SMatthew Ahrens /* 43788b7b0f2SMatthew Ahrens * We can't assert that db_size matches dn_datablksz because it 43888b7b0f2SMatthew Ahrens * can be momentarily different when another thread is doing 43988b7b0f2SMatthew Ahrens * dnode_set_blksz(). 44088b7b0f2SMatthew Ahrens */ 44188b7b0f2SMatthew Ahrens if (db->db_level == 0 && db->db.db_object == DMU_META_DNODE_OBJECT) { 442b24ab676SJeff Bonwick dr = db->db_data_pending; 443fa9e4066Sahrens /* 44488b7b0f2SMatthew Ahrens * It should only be modified in syncing context, so 44588b7b0f2SMatthew Ahrens * make sure we only have one copy of the data. 446fa9e4066Sahrens */ 447c717a561Smaybee ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf); 448fa9e4066Sahrens } 449fa9e4066Sahrens 450fa9e4066Sahrens /* verify db->db_blkptr */ 451fa9e4066Sahrens if (db->db_blkptr) { 452fa9e4066Sahrens if (db->db_parent == dn->dn_dbuf) { 453fa9e4066Sahrens /* db is pointed to by the dnode */ 454fa9e4066Sahrens /* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */ 45514843421SMatthew Ahrens if (DMU_OBJECT_IS_SPECIAL(db->db.db_object)) 456fa9e4066Sahrens ASSERT(db->db_parent == NULL); 457fa9e4066Sahrens else 458fa9e4066Sahrens ASSERT(db->db_parent != NULL); 4590a586ceaSMark Shellenbaum if (db->db_blkid != DMU_SPILL_BLKID) 460fa9e4066Sahrens ASSERT3P(db->db_blkptr, ==, 461fa9e4066Sahrens &dn->dn_phys->dn_blkptr[db->db_blkid]); 462fa9e4066Sahrens } else { 463fa9e4066Sahrens /* db is pointed to by an indirect block */ 464fa9e4066Sahrens int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT; 465fa9e4066Sahrens ASSERT3U(db->db_parent->db_level, ==, db->db_level+1); 466fa9e4066Sahrens ASSERT3U(db->db_parent->db.db_object, ==, 467fa9e4066Sahrens db->db.db_object); 468fa9e4066Sahrens /* 469fa9e4066Sahrens * dnode_grow_indblksz() can make this fail if we don't 470fa9e4066Sahrens * have the struct_rwlock. XXX indblksz no longer 471fa9e4066Sahrens * grows. safe to do this now? 472fa9e4066Sahrens */ 473744947dcSTom Erickson if (RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 474fa9e4066Sahrens ASSERT3P(db->db_blkptr, ==, 475fa9e4066Sahrens ((blkptr_t *)db->db_parent->db.db_data + 476fa9e4066Sahrens db->db_blkid % epb)); 477fa9e4066Sahrens } 478fa9e4066Sahrens } 479fa9e4066Sahrens } 480fa9e4066Sahrens if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) && 4813f9d6ad7SLin Ling (db->db_buf == NULL || db->db_buf->b_data) && 4820a586ceaSMark Shellenbaum db->db.db_data && db->db_blkid != DMU_BONUS_BLKID && 483fa9e4066Sahrens db->db_state != DB_FILL && !dn->dn_free_txg) { 484fa9e4066Sahrens /* 485fa9e4066Sahrens * If the blkptr isn't set but they have nonzero data, 486fa9e4066Sahrens * it had better be dirty, otherwise we'll lose that 487fa9e4066Sahrens * data when we evict this buffer. 488fa9e4066Sahrens */ 489fa9e4066Sahrens if (db->db_dirtycnt == 0) { 490fa9e4066Sahrens uint64_t *buf = db->db.db_data; 491fa9e4066Sahrens int i; 492fa9e4066Sahrens 493fa9e4066Sahrens for (i = 0; i < db->db.db_size >> 3; i++) { 494fa9e4066Sahrens ASSERT(buf[i] == 0); 495fa9e4066Sahrens } 496fa9e4066Sahrens } 497fa9e4066Sahrens } 498744947dcSTom Erickson DB_DNODE_EXIT(db); 499fa9e4066Sahrens } 5009c9dc39aSek110237 #endif 501fa9e4066Sahrens 502fa9e4066Sahrens static void 503bc9014e6SJustin Gibbs dbuf_clear_data(dmu_buf_impl_t *db) 504fa9e4066Sahrens { 505fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 506ea8dc4b6Seschrock dbuf_evict_user(db); 507bc9014e6SJustin Gibbs db->db_buf = NULL; 508ea8dc4b6Seschrock db->db.db_data = NULL; 50982c9918fSTim Haley if (db->db_state != DB_NOFILL) 510ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 511ea8dc4b6Seschrock } 512bc9014e6SJustin Gibbs 513bc9014e6SJustin Gibbs static void 514bc9014e6SJustin Gibbs dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf) 515bc9014e6SJustin Gibbs { 516bc9014e6SJustin Gibbs ASSERT(MUTEX_HELD(&db->db_mtx)); 517bc9014e6SJustin Gibbs ASSERT(buf != NULL); 518bc9014e6SJustin Gibbs 519bc9014e6SJustin Gibbs db->db_buf = buf; 520bc9014e6SJustin Gibbs ASSERT(buf->b_data != NULL); 521bc9014e6SJustin Gibbs db->db.db_data = buf->b_data; 522bc9014e6SJustin Gibbs if (!arc_released(buf)) 523bc9014e6SJustin Gibbs arc_set_callback(buf, dbuf_do_evict, db); 524fa9e4066Sahrens } 525fa9e4066Sahrens 526c242f9a0Schunli zhang - Sun Microsystems - Irvine United States /* 527c242f9a0Schunli zhang - Sun Microsystems - Irvine United States * Loan out an arc_buf for read. Return the loaned arc_buf. 528c242f9a0Schunli zhang - Sun Microsystems - Irvine United States */ 529c242f9a0Schunli zhang - Sun Microsystems - Irvine United States arc_buf_t * 530c242f9a0Schunli zhang - Sun Microsystems - Irvine United States dbuf_loan_arcbuf(dmu_buf_impl_t *db) 531c242f9a0Schunli zhang - Sun Microsystems - Irvine United States { 532c242f9a0Schunli zhang - Sun Microsystems - Irvine United States arc_buf_t *abuf; 533c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 534c242f9a0Schunli zhang - Sun Microsystems - Irvine United States mutex_enter(&db->db_mtx); 535c242f9a0Schunli zhang - Sun Microsystems - Irvine United States if (arc_released(db->db_buf) || refcount_count(&db->db_holds) > 1) { 536c242f9a0Schunli zhang - Sun Microsystems - Irvine United States int blksz = db->db.db_size; 53743466aaeSMax Grossman spa_t *spa = db->db_objset->os_spa; 538744947dcSTom Erickson 539c242f9a0Schunli zhang - Sun Microsystems - Irvine United States mutex_exit(&db->db_mtx); 540744947dcSTom Erickson abuf = arc_loan_buf(spa, blksz); 541c242f9a0Schunli zhang - Sun Microsystems - Irvine United States bcopy(db->db.db_data, abuf->b_data, blksz); 542c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } else { 543c242f9a0Schunli zhang - Sun Microsystems - Irvine United States abuf = db->db_buf; 544c242f9a0Schunli zhang - Sun Microsystems - Irvine United States arc_loan_inuse_buf(abuf, db); 545bc9014e6SJustin Gibbs dbuf_clear_data(db); 546c242f9a0Schunli zhang - Sun Microsystems - Irvine United States mutex_exit(&db->db_mtx); 547c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } 548c242f9a0Schunli zhang - Sun Microsystems - Irvine United States return (abuf); 549c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } 550c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 551a2cdcdd2SPaul Dagnelie /* 552a2cdcdd2SPaul Dagnelie * Calculate which level n block references the data at the level 0 offset 553a2cdcdd2SPaul Dagnelie * provided. 554a2cdcdd2SPaul Dagnelie */ 555fa9e4066Sahrens uint64_t 556a2cdcdd2SPaul Dagnelie dbuf_whichblock(dnode_t *dn, int64_t level, uint64_t offset) 557fa9e4066Sahrens { 558a2cdcdd2SPaul Dagnelie if (dn->dn_datablkshift != 0 && dn->dn_indblkshift != 0) { 559a2cdcdd2SPaul Dagnelie /* 560a2cdcdd2SPaul Dagnelie * The level n blkid is equal to the level 0 blkid divided by 561a2cdcdd2SPaul Dagnelie * the number of level 0s in a level n block. 562a2cdcdd2SPaul Dagnelie * 563a2cdcdd2SPaul Dagnelie * The level 0 blkid is offset >> datablkshift = 564a2cdcdd2SPaul Dagnelie * offset / 2^datablkshift. 565a2cdcdd2SPaul Dagnelie * 566a2cdcdd2SPaul Dagnelie * The number of level 0s in a level n is the number of block 567a2cdcdd2SPaul Dagnelie * pointers in an indirect block, raised to the power of level. 568a2cdcdd2SPaul Dagnelie * This is 2^(indblkshift - SPA_BLKPTRSHIFT)^level = 569a2cdcdd2SPaul Dagnelie * 2^(level*(indblkshift - SPA_BLKPTRSHIFT)). 570a2cdcdd2SPaul Dagnelie * 571a2cdcdd2SPaul Dagnelie * Thus, the level n blkid is: offset / 572a2cdcdd2SPaul Dagnelie * ((2^datablkshift)*(2^(level*(indblkshift - SPA_BLKPTRSHIFT))) 573a2cdcdd2SPaul Dagnelie * = offset / 2^(datablkshift + level * 574a2cdcdd2SPaul Dagnelie * (indblkshift - SPA_BLKPTRSHIFT)) 575a2cdcdd2SPaul Dagnelie * = offset >> (datablkshift + level * 576a2cdcdd2SPaul Dagnelie * (indblkshift - SPA_BLKPTRSHIFT)) 577a2cdcdd2SPaul Dagnelie */ 578a2cdcdd2SPaul Dagnelie return (offset >> (dn->dn_datablkshift + level * 579a2cdcdd2SPaul Dagnelie (dn->dn_indblkshift - SPA_BLKPTRSHIFT))); 580fa9e4066Sahrens } else { 581fa9e4066Sahrens ASSERT3U(offset, <, dn->dn_datablksz); 582fa9e4066Sahrens return (0); 583fa9e4066Sahrens } 584fa9e4066Sahrens } 585fa9e4066Sahrens 586fa9e4066Sahrens static void 587fa9e4066Sahrens dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb) 588fa9e4066Sahrens { 589fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 590fa9e4066Sahrens 591fa9e4066Sahrens mutex_enter(&db->db_mtx); 592fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_READ); 593fa9e4066Sahrens /* 594fa9e4066Sahrens * All reads are synchronous, so we must have a hold on the dbuf 595fa9e4066Sahrens */ 596fa9e4066Sahrens ASSERT(refcount_count(&db->db_holds) > 0); 597ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 598fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 599c717a561Smaybee if (db->db_level == 0 && db->db_freed_in_flight) { 600fa9e4066Sahrens /* we were freed in flight; disregard any error */ 601fa9e4066Sahrens arc_release(buf, db); 602fa9e4066Sahrens bzero(buf->b_data, db->db.db_size); 6036b4acc8bSahrens arc_buf_freeze(buf); 604c717a561Smaybee db->db_freed_in_flight = FALSE; 605fa9e4066Sahrens dbuf_set_data(db, buf); 606fa9e4066Sahrens db->db_state = DB_CACHED; 607fa9e4066Sahrens } else if (zio == NULL || zio->io_error == 0) { 608fa9e4066Sahrens dbuf_set_data(db, buf); 609fa9e4066Sahrens db->db_state = DB_CACHED; 610fa9e4066Sahrens } else { 6110a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 612fa9e4066Sahrens ASSERT3P(db->db_buf, ==, NULL); 6133b2aab18SMatthew Ahrens VERIFY(arc_buf_remove_ref(buf, db)); 614ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 615fa9e4066Sahrens } 616fa9e4066Sahrens cv_broadcast(&db->db_changed); 6173f9d6ad7SLin Ling dbuf_rele_and_unlock(db, NULL); 618fa9e4066Sahrens } 619fa9e4066Sahrens 620ea8dc4b6Seschrock static void 621cf6106c8SMatthew Ahrens dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags) 622fa9e4066Sahrens { 623744947dcSTom Erickson dnode_t *dn; 6247802d7bfSMatthew Ahrens zbookmark_phys_t zb; 6257adb730bSGeorge Wilson arc_flags_t aflags = ARC_FLAG_NOWAIT; 626fa9e4066Sahrens 627744947dcSTom Erickson DB_DNODE_ENTER(db); 628744947dcSTom Erickson dn = DB_DNODE(db); 629fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 630fa9e4066Sahrens /* We need the struct_rwlock to prevent db_blkptr from changing. */ 631088f3894Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 632ea8dc4b6Seschrock ASSERT(MUTEX_HELD(&db->db_mtx)); 633ea8dc4b6Seschrock ASSERT(db->db_state == DB_UNCACHED); 634ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 635fa9e4066Sahrens 6360a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 637cf04dda1SMark Maybee int bonuslen = MIN(dn->dn_bonuslen, dn->dn_phys->dn_bonuslen); 6381934e92fSmaybee 6391934e92fSmaybee ASSERT3U(bonuslen, <=, db->db.db_size); 640ea8dc4b6Seschrock db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN); 6415a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 6421934e92fSmaybee if (bonuslen < DN_MAX_BONUSLEN) 643ea8dc4b6Seschrock bzero(db->db.db_data, DN_MAX_BONUSLEN); 644cf04dda1SMark Maybee if (bonuslen) 645cf04dda1SMark Maybee bcopy(DN_BONUS(dn->dn_phys), db->db.db_data, bonuslen); 646744947dcSTom Erickson DB_DNODE_EXIT(db); 647fa9e4066Sahrens db->db_state = DB_CACHED; 648fa9e4066Sahrens mutex_exit(&db->db_mtx); 649fa9e4066Sahrens return; 650fa9e4066Sahrens } 651fa9e4066Sahrens 6521c8564a7SMark Maybee /* 6531c8564a7SMark Maybee * Recheck BP_IS_HOLE() after dnode_block_freed() in case dnode_sync() 6541c8564a7SMark Maybee * processes the delete record and clears the bp while we are waiting 6551c8564a7SMark Maybee * for the dn_mtx (resulting in a "no" from block_freed). 6561c8564a7SMark Maybee */ 657088f3894Sahrens if (db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr) || 6581c8564a7SMark Maybee (db->db_level == 0 && (dnode_block_freed(dn, db->db_blkid) || 6591c8564a7SMark Maybee BP_IS_HOLE(db->db_blkptr)))) { 660ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 661ad23a2dbSjohansen 662744947dcSTom Erickson DB_DNODE_EXIT(db); 66343466aaeSMax Grossman dbuf_set_data(db, arc_buf_alloc(db->db_objset->os_spa, 66443466aaeSMax Grossman db->db.db_size, db, type)); 665fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 666fa9e4066Sahrens db->db_state = DB_CACHED; 667fa9e4066Sahrens mutex_exit(&db->db_mtx); 668fa9e4066Sahrens return; 669fa9e4066Sahrens } 670fa9e4066Sahrens 671744947dcSTom Erickson DB_DNODE_EXIT(db); 672744947dcSTom Erickson 673fa9e4066Sahrens db->db_state = DB_READ; 674fa9e4066Sahrens mutex_exit(&db->db_mtx); 675fa9e4066Sahrens 6763baa08fcSek110237 if (DBUF_IS_L2CACHEABLE(db)) 6777adb730bSGeorge Wilson aflags |= ARC_FLAG_L2CACHE; 678aad02571SSaso Kiselkov if (DBUF_IS_L2COMPRESSIBLE(db)) 6797adb730bSGeorge Wilson aflags |= ARC_FLAG_L2COMPRESS; 6803baa08fcSek110237 681b24ab676SJeff Bonwick SET_BOOKMARK(&zb, db->db_objset->os_dsl_dataset ? 682b24ab676SJeff Bonwick db->db_objset->os_dsl_dataset->ds_object : DMU_META_OBJSET, 683b24ab676SJeff Bonwick db->db.db_object, db->db_level, db->db_blkid); 684ea8dc4b6Seschrock 685ea8dc4b6Seschrock dbuf_add_ref(db, NULL); 686088f3894Sahrens 68743466aaeSMax Grossman (void) arc_read(zio, db->db_objset->os_spa, db->db_blkptr, 688fa9e4066Sahrens dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ, 689cf6106c8SMatthew Ahrens (flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED, 69013506d1eSmaybee &aflags, &zb); 691fa9e4066Sahrens } 692fa9e4066Sahrens 693ea8dc4b6Seschrock int 694ea8dc4b6Seschrock dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags) 695fa9e4066Sahrens { 696ea8dc4b6Seschrock int err = 0; 69743466aaeSMax Grossman boolean_t havepzio = (zio != NULL); 69843466aaeSMax Grossman boolean_t prefetch; 699744947dcSTom Erickson dnode_t *dn; 700fa9e4066Sahrens 701fa9e4066Sahrens /* 702fa9e4066Sahrens * We don't have to hold the mutex to check db_state because it 703fa9e4066Sahrens * can't be freed while we have a hold on the buffer. 704fa9e4066Sahrens */ 705fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 706fa9e4066Sahrens 70782c9918fSTim Haley if (db->db_state == DB_NOFILL) 708be6fd75aSMatthew Ahrens return (SET_ERROR(EIO)); 70982c9918fSTim Haley 710744947dcSTom Erickson DB_DNODE_ENTER(db); 711744947dcSTom Erickson dn = DB_DNODE(db); 712fa9e4066Sahrens if ((flags & DB_RF_HAVESTRUCT) == 0) 713744947dcSTom Erickson rw_enter(&dn->dn_struct_rwlock, RW_READER); 714fa9e4066Sahrens 7150a586ceaSMark Shellenbaum prefetch = db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 716744947dcSTom Erickson (flags & DB_RF_NOPREFETCH) == 0 && dn != NULL && 7173baa08fcSek110237 DBUF_IS_CACHEABLE(db); 71813506d1eSmaybee 719fa9e4066Sahrens mutex_enter(&db->db_mtx); 720ea8dc4b6Seschrock if (db->db_state == DB_CACHED) { 721ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 72213506d1eSmaybee if (prefetch) 723cf6106c8SMatthew Ahrens dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1); 724ea8dc4b6Seschrock if ((flags & DB_RF_HAVESTRUCT) == 0) 725744947dcSTom Erickson rw_exit(&dn->dn_struct_rwlock); 726744947dcSTom Erickson DB_DNODE_EXIT(db); 727ea8dc4b6Seschrock } else if (db->db_state == DB_UNCACHED) { 728744947dcSTom Erickson spa_t *spa = dn->dn_objset->os_spa; 729744947dcSTom Erickson 730744947dcSTom Erickson if (zio == NULL) 731744947dcSTom Erickson zio = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL); 732cf6106c8SMatthew Ahrens dbuf_read_impl(db, zio, flags); 73313506d1eSmaybee 734ea8dc4b6Seschrock /* dbuf_read_impl has dropped db_mtx for us */ 735ea8dc4b6Seschrock 73613506d1eSmaybee if (prefetch) 737cf6106c8SMatthew Ahrens dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1); 738ea8dc4b6Seschrock 739ea8dc4b6Seschrock if ((flags & DB_RF_HAVESTRUCT) == 0) 740744947dcSTom Erickson rw_exit(&dn->dn_struct_rwlock); 741744947dcSTom Erickson DB_DNODE_EXIT(db); 742ea8dc4b6Seschrock 743ea8dc4b6Seschrock if (!havepzio) 744ea8dc4b6Seschrock err = zio_wait(zio); 745ea8dc4b6Seschrock } else { 7463e30c24aSWill Andrews /* 7473e30c24aSWill Andrews * Another reader came in while the dbuf was in flight 7483e30c24aSWill Andrews * between UNCACHED and CACHED. Either a writer will finish 7493e30c24aSWill Andrews * writing the buffer (sending the dbuf to CACHED) or the 7503e30c24aSWill Andrews * first reader's request will reach the read_done callback 7513e30c24aSWill Andrews * and send the dbuf to CACHED. Otherwise, a failure 7523e30c24aSWill Andrews * occurred and the dbuf went to UNCACHED. 7533e30c24aSWill Andrews */ 75413506d1eSmaybee mutex_exit(&db->db_mtx); 75513506d1eSmaybee if (prefetch) 756cf6106c8SMatthew Ahrens dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1); 757ea8dc4b6Seschrock if ((flags & DB_RF_HAVESTRUCT) == 0) 758744947dcSTom Erickson rw_exit(&dn->dn_struct_rwlock); 759744947dcSTom Erickson DB_DNODE_EXIT(db); 76013506d1eSmaybee 7613e30c24aSWill Andrews /* Skip the wait per the caller's request. */ 76213506d1eSmaybee mutex_enter(&db->db_mtx); 763ea8dc4b6Seschrock if ((flags & DB_RF_NEVERWAIT) == 0) { 764ea8dc4b6Seschrock while (db->db_state == DB_READ || 765ea8dc4b6Seschrock db->db_state == DB_FILL) { 766fa9e4066Sahrens ASSERT(db->db_state == DB_READ || 767fa9e4066Sahrens (flags & DB_RF_HAVESTRUCT) == 0); 768f6164ad6SAdam H. Leventhal DTRACE_PROBE2(blocked__read, dmu_buf_impl_t *, 769f6164ad6SAdam H. Leventhal db, zio_t *, zio); 770fa9e4066Sahrens cv_wait(&db->db_changed, &db->db_mtx); 771fa9e4066Sahrens } 772ea8dc4b6Seschrock if (db->db_state == DB_UNCACHED) 773be6fd75aSMatthew Ahrens err = SET_ERROR(EIO); 774ea8dc4b6Seschrock } 775fa9e4066Sahrens mutex_exit(&db->db_mtx); 776fa9e4066Sahrens } 777fa9e4066Sahrens 778ea8dc4b6Seschrock ASSERT(err || havepzio || db->db_state == DB_CACHED); 779ea8dc4b6Seschrock return (err); 780fa9e4066Sahrens } 781fa9e4066Sahrens 782fa9e4066Sahrens static void 783fa9e4066Sahrens dbuf_noread(dmu_buf_impl_t *db) 784fa9e4066Sahrens { 785fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 7860a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 787fa9e4066Sahrens mutex_enter(&db->db_mtx); 788fa9e4066Sahrens while (db->db_state == DB_READ || db->db_state == DB_FILL) 789fa9e4066Sahrens cv_wait(&db->db_changed, &db->db_mtx); 790fa9e4066Sahrens if (db->db_state == DB_UNCACHED) { 791ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 79243466aaeSMax Grossman spa_t *spa = db->db_objset->os_spa; 793ad23a2dbSjohansen 794ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 795fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 796744947dcSTom Erickson dbuf_set_data(db, arc_buf_alloc(spa, db->db.db_size, db, type)); 797fa9e4066Sahrens db->db_state = DB_FILL; 79882c9918fSTim Haley } else if (db->db_state == DB_NOFILL) { 799bc9014e6SJustin Gibbs dbuf_clear_data(db); 800fa9e4066Sahrens } else { 801fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_CACHED); 802fa9e4066Sahrens } 803fa9e4066Sahrens mutex_exit(&db->db_mtx); 804fa9e4066Sahrens } 805fa9e4066Sahrens 806fa9e4066Sahrens /* 807fa9e4066Sahrens * This is our just-in-time copy function. It makes a copy of 808fa9e4066Sahrens * buffers, that have been modified in a previous transaction 809fa9e4066Sahrens * group, before we modify them in the current active group. 810fa9e4066Sahrens * 811fa9e4066Sahrens * This function is used in two places: when we are dirtying a 812fa9e4066Sahrens * buffer for the first time in a txg, and when we are freeing 813fa9e4066Sahrens * a range in a dnode that includes this buffer. 814fa9e4066Sahrens * 815fa9e4066Sahrens * Note that when we are called from dbuf_free_range() we do 816fa9e4066Sahrens * not put a hold on the buffer, we just traverse the active 817fa9e4066Sahrens * dbuf list for the dnode. 818fa9e4066Sahrens */ 819fa9e4066Sahrens static void 820fa9e4066Sahrens dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg) 821fa9e4066Sahrens { 822c717a561Smaybee dbuf_dirty_record_t *dr = db->db_last_dirty; 823fa9e4066Sahrens 824fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 825fa9e4066Sahrens ASSERT(db->db.db_data != NULL); 826c717a561Smaybee ASSERT(db->db_level == 0); 827c717a561Smaybee ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT); 828fa9e4066Sahrens 8294d31c452Smaybee if (dr == NULL || 8304d31c452Smaybee (dr->dt.dl.dr_data != 8310a586ceaSMark Shellenbaum ((db->db_blkid == DMU_BONUS_BLKID) ? db->db.db_data : db->db_buf))) 832fa9e4066Sahrens return; 833fa9e4066Sahrens 834fa9e4066Sahrens /* 835c717a561Smaybee * If the last dirty record for this dbuf has not yet synced 836c717a561Smaybee * and its referencing the dbuf data, either: 837c717a561Smaybee * reset the reference to point to a new copy, 838c717a561Smaybee * or (if there a no active holders) 839c717a561Smaybee * just null out the current db_data pointer. 840fa9e4066Sahrens */ 841c717a561Smaybee ASSERT(dr->dr_txg >= txg - 2); 8420a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 843c717a561Smaybee /* Note that the data bufs here are zio_bufs */ 844c717a561Smaybee dr->dt.dl.dr_data = zio_buf_alloc(DN_MAX_BONUSLEN); 8455a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 846c717a561Smaybee bcopy(db->db.db_data, dr->dt.dl.dr_data, DN_MAX_BONUSLEN); 847c717a561Smaybee } else if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 848ea8dc4b6Seschrock int size = db->db.db_size; 849c717a561Smaybee arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 85043466aaeSMax Grossman spa_t *spa = db->db_objset->os_spa; 851744947dcSTom Erickson 852744947dcSTom Erickson dr->dt.dl.dr_data = arc_buf_alloc(spa, size, db, type); 853c717a561Smaybee bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size); 854fa9e4066Sahrens } else { 855bc9014e6SJustin Gibbs dbuf_clear_data(db); 856fa9e4066Sahrens } 857fa9e4066Sahrens } 858ea8dc4b6Seschrock 859fa9e4066Sahrens void 860c717a561Smaybee dbuf_unoverride(dbuf_dirty_record_t *dr) 861fa9e4066Sahrens { 862c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 863b24ab676SJeff Bonwick blkptr_t *bp = &dr->dt.dl.dr_overridden_by; 864c717a561Smaybee uint64_t txg = dr->dr_txg; 865c5c6ffa0Smaybee 866c717a561Smaybee ASSERT(MUTEX_HELD(&db->db_mtx)); 867c717a561Smaybee ASSERT(dr->dt.dl.dr_override_state != DR_IN_DMU_SYNC); 868c717a561Smaybee ASSERT(db->db_level == 0); 869c717a561Smaybee 8700a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID || 871c717a561Smaybee dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN) 872c717a561Smaybee return; 873c717a561Smaybee 874b24ab676SJeff Bonwick ASSERT(db->db_data_pending != dr); 875b24ab676SJeff Bonwick 876fa9e4066Sahrens /* free this block */ 87743466aaeSMax Grossman if (!BP_IS_HOLE(bp) && !dr->dt.dl.dr_nopwrite) 87843466aaeSMax Grossman zio_free(db->db_objset->os_spa, txg, bp); 879b24ab676SJeff Bonwick 880c717a561Smaybee dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 88180901aeaSGeorge Wilson dr->dt.dl.dr_nopwrite = B_FALSE; 88280901aeaSGeorge Wilson 8836b4acc8bSahrens /* 8846b4acc8bSahrens * Release the already-written buffer, so we leave it in 8856b4acc8bSahrens * a consistent dirty state. Note that all callers are 8866b4acc8bSahrens * modifying the buffer, so they will immediately do 8876b4acc8bSahrens * another (redundant) arc_release(). Therefore, leave 8886b4acc8bSahrens * the buf thawed to save the effort of freezing & 8896b4acc8bSahrens * immediately re-thawing it. 8906b4acc8bSahrens */ 891c717a561Smaybee arc_release(dr->dt.dl.dr_data, db); 892fa9e4066Sahrens } 893fa9e4066Sahrens 894cdb0ab79Smaybee /* 895cdb0ab79Smaybee * Evict (if its unreferenced) or clear (if its referenced) any level-0 896cdb0ab79Smaybee * data blocks in the free range, so that any future readers will find 89743466aaeSMax Grossman * empty blocks. 8982f3d8780SMatthew Ahrens * 8992f3d8780SMatthew Ahrens * This is a no-op if the dataset is in the middle of an incremental 9002f3d8780SMatthew Ahrens * receive; see comment below for details. 901cdb0ab79Smaybee */ 902fa9e4066Sahrens void 9030f6d88adSAlex Reece dbuf_free_range(dnode_t *dn, uint64_t start_blkid, uint64_t end_blkid, 9040f6d88adSAlex Reece dmu_tx_t *tx) 905fa9e4066Sahrens { 906bc9014e6SJustin Gibbs dmu_buf_impl_t db_search; 907bc9014e6SJustin Gibbs dmu_buf_impl_t *db, *db_next; 908fa9e4066Sahrens uint64_t txg = tx->tx_txg; 9090f6d88adSAlex Reece avl_index_t where; 910fa9e4066Sahrens 9110f6d88adSAlex Reece if (end_blkid > dn->dn_maxblkid && (end_blkid != DMU_SPILL_BLKID)) 9120f6d88adSAlex Reece end_blkid = dn->dn_maxblkid; 9130f6d88adSAlex Reece dprintf_dnode(dn, "start=%llu end=%llu\n", start_blkid, end_blkid); 9140f6d88adSAlex Reece 9150f6d88adSAlex Reece db_search.db_level = 0; 9160f6d88adSAlex Reece db_search.db_blkid = start_blkid; 91786bb58aeSAlex Reece db_search.db_state = DB_SEARCH; 9182f3d8780SMatthew Ahrens 919713d6c20SMatthew Ahrens mutex_enter(&dn->dn_dbufs_mtx); 9200f6d88adSAlex Reece if (start_blkid >= dn->dn_unlisted_l0_blkid) { 921713d6c20SMatthew Ahrens /* There can't be any dbufs in this range; no need to search. */ 9220f6d88adSAlex Reece #ifdef DEBUG 9230f6d88adSAlex Reece db = avl_find(&dn->dn_dbufs, &db_search, &where); 9240f6d88adSAlex Reece ASSERT3P(db, ==, NULL); 9250f6d88adSAlex Reece db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER); 9260f6d88adSAlex Reece ASSERT(db == NULL || db->db_level > 0); 9270f6d88adSAlex Reece #endif 928713d6c20SMatthew Ahrens mutex_exit(&dn->dn_dbufs_mtx); 9292f3d8780SMatthew Ahrens return; 930713d6c20SMatthew Ahrens } else if (dmu_objset_is_receiving(dn->dn_objset)) { 931713d6c20SMatthew Ahrens /* 932713d6c20SMatthew Ahrens * If we are receiving, we expect there to be no dbufs in 933713d6c20SMatthew Ahrens * the range to be freed, because receive modifies each 934713d6c20SMatthew Ahrens * block at most once, and in offset order. If this is 935713d6c20SMatthew Ahrens * not the case, it can lead to performance problems, 936713d6c20SMatthew Ahrens * so note that we unexpectedly took the slow path. 937713d6c20SMatthew Ahrens */ 938713d6c20SMatthew Ahrens atomic_inc_64(&zfs_free_range_recv_miss); 9392f3d8780SMatthew Ahrens } 9402f3d8780SMatthew Ahrens 9410f6d88adSAlex Reece db = avl_find(&dn->dn_dbufs, &db_search, &where); 9420f6d88adSAlex Reece ASSERT3P(db, ==, NULL); 9430f6d88adSAlex Reece db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER); 9440f6d88adSAlex Reece 9450f6d88adSAlex Reece for (; db != NULL; db = db_next) { 9460f6d88adSAlex Reece db_next = AVL_NEXT(&dn->dn_dbufs, db); 9470a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 948cdb0ab79Smaybee 9490f6d88adSAlex Reece if (db->db_level != 0 || db->db_blkid > end_blkid) { 9500f6d88adSAlex Reece break; 9510f6d88adSAlex Reece } 9520f6d88adSAlex Reece ASSERT3U(db->db_blkid, >=, start_blkid); 953fa9e4066Sahrens 954fa9e4066Sahrens /* found a level 0 buffer in the range */ 955fa9e4066Sahrens mutex_enter(&db->db_mtx); 9563b2aab18SMatthew Ahrens if (dbuf_undirty(db, tx)) { 9573b2aab18SMatthew Ahrens /* mutex has been dropped and dbuf destroyed */ 9583b2aab18SMatthew Ahrens continue; 9593b2aab18SMatthew Ahrens } 9603b2aab18SMatthew Ahrens 961ea8dc4b6Seschrock if (db->db_state == DB_UNCACHED || 96282c9918fSTim Haley db->db_state == DB_NOFILL || 963ea8dc4b6Seschrock db->db_state == DB_EVICTING) { 964fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 965fa9e4066Sahrens mutex_exit(&db->db_mtx); 966fa9e4066Sahrens continue; 967fa9e4066Sahrens } 968c543ec06Sahrens if (db->db_state == DB_READ || db->db_state == DB_FILL) { 969c543ec06Sahrens /* will be handled in dbuf_read_done or dbuf_rele */ 970c717a561Smaybee db->db_freed_in_flight = TRUE; 971fa9e4066Sahrens mutex_exit(&db->db_mtx); 972fa9e4066Sahrens continue; 973fa9e4066Sahrens } 974ea8dc4b6Seschrock if (refcount_count(&db->db_holds) == 0) { 975ea8dc4b6Seschrock ASSERT(db->db_buf); 976ea8dc4b6Seschrock dbuf_clear(db); 977ea8dc4b6Seschrock continue; 978ea8dc4b6Seschrock } 979c717a561Smaybee /* The dbuf is referenced */ 980fa9e4066Sahrens 981c717a561Smaybee if (db->db_last_dirty != NULL) { 982c717a561Smaybee dbuf_dirty_record_t *dr = db->db_last_dirty; 983c717a561Smaybee 984c717a561Smaybee if (dr->dr_txg == txg) { 985ea8dc4b6Seschrock /* 986c717a561Smaybee * This buffer is "in-use", re-adjust the file 987c717a561Smaybee * size to reflect that this buffer may 988c717a561Smaybee * contain new data when we sync. 989ea8dc4b6Seschrock */ 99006e0070dSMark Shellenbaum if (db->db_blkid != DMU_SPILL_BLKID && 99106e0070dSMark Shellenbaum db->db_blkid > dn->dn_maxblkid) 99244eda4d7Smaybee dn->dn_maxblkid = db->db_blkid; 993c717a561Smaybee dbuf_unoverride(dr); 994c717a561Smaybee } else { 995c717a561Smaybee /* 996c717a561Smaybee * This dbuf is not dirty in the open context. 997c717a561Smaybee * Either uncache it (if its not referenced in 998c717a561Smaybee * the open context) or reset its contents to 999c717a561Smaybee * empty. 1000c717a561Smaybee */ 1001c717a561Smaybee dbuf_fix_old_data(db, txg); 100244eda4d7Smaybee } 1003c717a561Smaybee } 1004c717a561Smaybee /* clear the contents if its cached */ 1005ea8dc4b6Seschrock if (db->db_state == DB_CACHED) { 1006ea8dc4b6Seschrock ASSERT(db->db.db_data != NULL); 1007fa9e4066Sahrens arc_release(db->db_buf, db); 1008fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 10096b4acc8bSahrens arc_buf_freeze(db->db_buf); 1010fa9e4066Sahrens } 1011ea8dc4b6Seschrock 1012fa9e4066Sahrens mutex_exit(&db->db_mtx); 1013fa9e4066Sahrens } 1014fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 1015fa9e4066Sahrens } 1016fa9e4066Sahrens 1017fa9e4066Sahrens static int 10181934e92fSmaybee dbuf_block_freeable(dmu_buf_impl_t *db) 1019fa9e4066Sahrens { 1020fa9e4066Sahrens dsl_dataset_t *ds = db->db_objset->os_dsl_dataset; 1021fa9e4066Sahrens uint64_t birth_txg = 0; 1022fa9e4066Sahrens 1023fa9e4066Sahrens /* 1024fa9e4066Sahrens * We don't need any locking to protect db_blkptr: 1025c717a561Smaybee * If it's syncing, then db_last_dirty will be set 1026c717a561Smaybee * so we'll ignore db_blkptr. 102743466aaeSMax Grossman * 102843466aaeSMax Grossman * This logic ensures that only block births for 102943466aaeSMax Grossman * filled blocks are considered. 1030fa9e4066Sahrens */ 1031c717a561Smaybee ASSERT(MUTEX_HELD(&db->db_mtx)); 103243466aaeSMax Grossman if (db->db_last_dirty && (db->db_blkptr == NULL || 103343466aaeSMax Grossman !BP_IS_HOLE(db->db_blkptr))) { 1034c717a561Smaybee birth_txg = db->db_last_dirty->dr_txg; 103543466aaeSMax Grossman } else if (db->db_blkptr != NULL && !BP_IS_HOLE(db->db_blkptr)) { 1036fa9e4066Sahrens birth_txg = db->db_blkptr->blk_birth; 103743466aaeSMax Grossman } 1038fa9e4066Sahrens 1039837b568bSGeorge Wilson /* 104043466aaeSMax Grossman * If this block don't exist or is in a snapshot, it can't be freed. 1041837b568bSGeorge Wilson * Don't pass the bp to dsl_dataset_block_freeable() since we 1042837b568bSGeorge Wilson * are holding the db_mtx lock and might deadlock if we are 1043837b568bSGeorge Wilson * prefetching a dedup-ed block. 1044837b568bSGeorge Wilson */ 104543466aaeSMax Grossman if (birth_txg != 0) 10461934e92fSmaybee return (ds == NULL || 1047837b568bSGeorge Wilson dsl_dataset_block_freeable(ds, NULL, birth_txg)); 1048fa9e4066Sahrens else 104943466aaeSMax Grossman return (B_FALSE); 1050fa9e4066Sahrens } 1051fa9e4066Sahrens 1052fa9e4066Sahrens void 1053fa9e4066Sahrens dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx) 1054fa9e4066Sahrens { 1055fa9e4066Sahrens arc_buf_t *buf, *obuf; 1056fa9e4066Sahrens int osize = db->db.db_size; 1057ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 1058744947dcSTom Erickson dnode_t *dn; 1059fa9e4066Sahrens 10600a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1061ea8dc4b6Seschrock 1062744947dcSTom Erickson DB_DNODE_ENTER(db); 1063744947dcSTom Erickson dn = DB_DNODE(db); 1064744947dcSTom Erickson 1065fa9e4066Sahrens /* XXX does *this* func really need the lock? */ 1066744947dcSTom Erickson ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 1067fa9e4066Sahrens 1068fa9e4066Sahrens /* 106943466aaeSMax Grossman * This call to dmu_buf_will_dirty() with the dn_struct_rwlock held 1070fa9e4066Sahrens * is OK, because there can be no other references to the db 1071fa9e4066Sahrens * when we are changing its size, so no concurrent DB_FILL can 1072fa9e4066Sahrens * be happening. 1073fa9e4066Sahrens */ 1074ea8dc4b6Seschrock /* 1075ea8dc4b6Seschrock * XXX we should be doing a dbuf_read, checking the return 1076ea8dc4b6Seschrock * value and returning that up to our callers 1077ea8dc4b6Seschrock */ 107843466aaeSMax Grossman dmu_buf_will_dirty(&db->db, tx); 1079fa9e4066Sahrens 1080fa9e4066Sahrens /* create the data buffer for the new block */ 1081744947dcSTom Erickson buf = arc_buf_alloc(dn->dn_objset->os_spa, size, db, type); 1082fa9e4066Sahrens 1083fa9e4066Sahrens /* copy old block data to the new block */ 1084fa9e4066Sahrens obuf = db->db_buf; 1085f65e61c0Sahrens bcopy(obuf->b_data, buf->b_data, MIN(osize, size)); 1086fa9e4066Sahrens /* zero the remainder */ 1087f65e61c0Sahrens if (size > osize) 1088fa9e4066Sahrens bzero((uint8_t *)buf->b_data + osize, size - osize); 1089fa9e4066Sahrens 1090fa9e4066Sahrens mutex_enter(&db->db_mtx); 1091fa9e4066Sahrens dbuf_set_data(db, buf); 10923b2aab18SMatthew Ahrens VERIFY(arc_buf_remove_ref(obuf, db)); 1093fa9e4066Sahrens db->db.db_size = size; 1094fa9e4066Sahrens 1095c717a561Smaybee if (db->db_level == 0) { 1096c717a561Smaybee ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg); 1097c717a561Smaybee db->db_last_dirty->dt.dl.dr_data = buf; 1098c717a561Smaybee } 1099fa9e4066Sahrens mutex_exit(&db->db_mtx); 1100fa9e4066Sahrens 1101744947dcSTom Erickson dnode_willuse_space(dn, size-osize, tx); 1102744947dcSTom Erickson DB_DNODE_EXIT(db); 1103fa9e4066Sahrens } 1104fa9e4066Sahrens 11053f9d6ad7SLin Ling void 11063f9d6ad7SLin Ling dbuf_release_bp(dmu_buf_impl_t *db) 11073f9d6ad7SLin Ling { 110843466aaeSMax Grossman objset_t *os = db->db_objset; 11093f9d6ad7SLin Ling 11103f9d6ad7SLin Ling ASSERT(dsl_pool_sync_context(dmu_objset_pool(os))); 11113f9d6ad7SLin Ling ASSERT(arc_released(os->os_phys_buf) || 11123f9d6ad7SLin Ling list_link_active(&os->os_dsl_dataset->ds_synced_link)); 11133f9d6ad7SLin Ling ASSERT(db->db_parent == NULL || arc_released(db->db_parent->db_buf)); 11143f9d6ad7SLin Ling 11151b912ec7SGeorge Wilson (void) arc_release(db->db_buf, db); 11163f9d6ad7SLin Ling } 11173f9d6ad7SLin Ling 1118*0f2e7d03SMatthew Ahrens /* 1119*0f2e7d03SMatthew Ahrens * We already have a dirty record for this TXG, and we are being 1120*0f2e7d03SMatthew Ahrens * dirtied again. 1121*0f2e7d03SMatthew Ahrens */ 1122*0f2e7d03SMatthew Ahrens static void 1123*0f2e7d03SMatthew Ahrens dbuf_redirty(dbuf_dirty_record_t *dr) 1124*0f2e7d03SMatthew Ahrens { 1125*0f2e7d03SMatthew Ahrens dmu_buf_impl_t *db = dr->dr_dbuf; 1126*0f2e7d03SMatthew Ahrens 1127*0f2e7d03SMatthew Ahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 1128*0f2e7d03SMatthew Ahrens 1129*0f2e7d03SMatthew Ahrens if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID) { 1130*0f2e7d03SMatthew Ahrens /* 1131*0f2e7d03SMatthew Ahrens * If this buffer has already been written out, 1132*0f2e7d03SMatthew Ahrens * we now need to reset its state. 1133*0f2e7d03SMatthew Ahrens */ 1134*0f2e7d03SMatthew Ahrens dbuf_unoverride(dr); 1135*0f2e7d03SMatthew Ahrens if (db->db.db_object != DMU_META_DNODE_OBJECT && 1136*0f2e7d03SMatthew Ahrens db->db_state != DB_NOFILL) { 1137*0f2e7d03SMatthew Ahrens /* Already released on initial dirty, so just thaw. */ 1138*0f2e7d03SMatthew Ahrens ASSERT(arc_released(db->db_buf)); 1139*0f2e7d03SMatthew Ahrens arc_buf_thaw(db->db_buf); 1140*0f2e7d03SMatthew Ahrens } 1141*0f2e7d03SMatthew Ahrens } 1142*0f2e7d03SMatthew Ahrens } 1143*0f2e7d03SMatthew Ahrens 1144c717a561Smaybee dbuf_dirty_record_t * 1145fa9e4066Sahrens dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1146fa9e4066Sahrens { 1147744947dcSTom Erickson dnode_t *dn; 1148744947dcSTom Erickson objset_t *os; 1149c717a561Smaybee dbuf_dirty_record_t **drp, *dr; 1150fa9e4066Sahrens int drop_struct_lock = FALSE; 1151d3469faaSMark Maybee boolean_t do_free_accounting = B_FALSE; 1152fa9e4066Sahrens int txgoff = tx->tx_txg & TXG_MASK; 1153fa9e4066Sahrens 1154fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 1155fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 11569c9dc39aSek110237 DMU_TX_DIRTY_BUF(tx, db); 1157fa9e4066Sahrens 1158744947dcSTom Erickson DB_DNODE_ENTER(db); 1159744947dcSTom Erickson dn = DB_DNODE(db); 1160fa9e4066Sahrens /* 1161fa9e4066Sahrens * Shouldn't dirty a regular buffer in syncing context. Private 1162fa9e4066Sahrens * objects may be dirtied in syncing context, but only if they 1163fa9e4066Sahrens * were already pre-dirtied in open context. 1164fa9e4066Sahrens */ 1165c717a561Smaybee ASSERT(!dmu_tx_is_syncing(tx) || 1166c717a561Smaybee BP_IS_HOLE(dn->dn_objset->os_rootbp) || 116714843421SMatthew Ahrens DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 116814843421SMatthew Ahrens dn->dn_objset->os_dsl_dataset == NULL); 1169fa9e4066Sahrens /* 1170fa9e4066Sahrens * We make this assert for private objects as well, but after we 1171fa9e4066Sahrens * check if we're already dirty. They are allowed to re-dirty 1172fa9e4066Sahrens * in syncing context. 1173fa9e4066Sahrens */ 1174ea8dc4b6Seschrock ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 1175c717a561Smaybee dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 1176fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 1177fa9e4066Sahrens 1178fa9e4066Sahrens mutex_enter(&db->db_mtx); 1179fa9e4066Sahrens /* 1180c717a561Smaybee * XXX make this true for indirects too? The problem is that 1181c717a561Smaybee * transactions created with dmu_tx_create_assigned() from 1182c717a561Smaybee * syncing context don't bother holding ahead. 1183fa9e4066Sahrens */ 1184c717a561Smaybee ASSERT(db->db_level != 0 || 118582c9918fSTim Haley db->db_state == DB_CACHED || db->db_state == DB_FILL || 118682c9918fSTim Haley db->db_state == DB_NOFILL); 1187fa9e4066Sahrens 1188fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 1189fa9e4066Sahrens /* 1190fa9e4066Sahrens * Don't set dirtyctx to SYNC if we're just modifying this as we 1191fa9e4066Sahrens * initialize the objset. 1192fa9e4066Sahrens */ 1193fa9e4066Sahrens if (dn->dn_dirtyctx == DN_UNDIRTIED && 1194c717a561Smaybee !BP_IS_HOLE(dn->dn_objset->os_rootbp)) { 1195fa9e4066Sahrens dn->dn_dirtyctx = 1196fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN); 1197fa9e4066Sahrens ASSERT(dn->dn_dirtyctx_firstset == NULL); 1198fa9e4066Sahrens dn->dn_dirtyctx_firstset = kmem_alloc(1, KM_SLEEP); 1199fa9e4066Sahrens } 1200fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1201fa9e4066Sahrens 12020a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) 12030a586ceaSMark Shellenbaum dn->dn_have_spill = B_TRUE; 12040a586ceaSMark Shellenbaum 1205fa9e4066Sahrens /* 1206fa9e4066Sahrens * If this buffer is already dirty, we're done. 1207fa9e4066Sahrens */ 1208c717a561Smaybee drp = &db->db_last_dirty; 1209c717a561Smaybee ASSERT(*drp == NULL || (*drp)->dr_txg <= tx->tx_txg || 1210c717a561Smaybee db->db.db_object == DMU_META_DNODE_OBJECT); 12117e2186e3Sbonwick while ((dr = *drp) != NULL && dr->dr_txg > tx->tx_txg) 12127e2186e3Sbonwick drp = &dr->dr_next; 12137e2186e3Sbonwick if (dr && dr->dr_txg == tx->tx_txg) { 1214744947dcSTom Erickson DB_DNODE_EXIT(db); 1215744947dcSTom Erickson 1216*0f2e7d03SMatthew Ahrens dbuf_redirty(dr); 1217fa9e4066Sahrens mutex_exit(&db->db_mtx); 12187e2186e3Sbonwick return (dr); 1219fa9e4066Sahrens } 1220fa9e4066Sahrens 1221fa9e4066Sahrens /* 1222fa9e4066Sahrens * Only valid if not already dirty. 1223fa9e4066Sahrens */ 122414843421SMatthew Ahrens ASSERT(dn->dn_object == 0 || 122514843421SMatthew Ahrens dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 1226fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 1227fa9e4066Sahrens 1228fa9e4066Sahrens ASSERT3U(dn->dn_nlevels, >, db->db_level); 1229fa9e4066Sahrens ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) || 1230fa9e4066Sahrens dn->dn_phys->dn_nlevels > db->db_level || 1231fa9e4066Sahrens dn->dn_next_nlevels[txgoff] > db->db_level || 1232fa9e4066Sahrens dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level || 1233fa9e4066Sahrens dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level); 1234fa9e4066Sahrens 1235fa9e4066Sahrens /* 1236fa9e4066Sahrens * We should only be dirtying in syncing context if it's the 123714843421SMatthew Ahrens * mos or we're initializing the os or it's a special object. 123814843421SMatthew Ahrens * However, we are allowed to dirty in syncing context provided 123914843421SMatthew Ahrens * we already dirtied it in open context. Hence we must make 124014843421SMatthew Ahrens * this assertion only if we're not already dirty. 1241fa9e4066Sahrens */ 1242744947dcSTom Erickson os = dn->dn_objset; 124314843421SMatthew Ahrens ASSERT(!dmu_tx_is_syncing(tx) || DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 124414843421SMatthew Ahrens os->os_dsl_dataset == NULL || BP_IS_HOLE(os->os_rootbp)); 1245fa9e4066Sahrens ASSERT(db->db.db_size != 0); 1246fa9e4066Sahrens 1247fa9e4066Sahrens dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1248fa9e4066Sahrens 12490a586ceaSMark Shellenbaum if (db->db_blkid != DMU_BONUS_BLKID) { 12501934e92fSmaybee /* 12511934e92fSmaybee * Update the accounting. 1252d3469faaSMark Maybee * Note: we delay "free accounting" until after we drop 1253d3469faaSMark Maybee * the db_mtx. This keeps us from grabbing other locks 1254b24ab676SJeff Bonwick * (and possibly deadlocking) in bp_get_dsize() while 1255d3469faaSMark Maybee * also holding the db_mtx. 12561934e92fSmaybee */ 12571934e92fSmaybee dnode_willuse_space(dn, db->db.db_size, tx); 1258d3469faaSMark Maybee do_free_accounting = dbuf_block_freeable(db); 12591934e92fSmaybee } 12601934e92fSmaybee 1261ea8dc4b6Seschrock /* 1262ea8dc4b6Seschrock * If this buffer is dirty in an old transaction group we need 1263ea8dc4b6Seschrock * to make a copy of it so that the changes we make in this 1264ea8dc4b6Seschrock * transaction group won't leak out when we sync the older txg. 1265ea8dc4b6Seschrock */ 1266c717a561Smaybee dr = kmem_zalloc(sizeof (dbuf_dirty_record_t), KM_SLEEP); 1267c717a561Smaybee if (db->db_level == 0) { 1268c717a561Smaybee void *data_old = db->db_buf; 1269c717a561Smaybee 127082c9918fSTim Haley if (db->db_state != DB_NOFILL) { 12710a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 1272c717a561Smaybee dbuf_fix_old_data(db, tx->tx_txg); 1273c717a561Smaybee data_old = db->db.db_data; 1274c717a561Smaybee } else if (db->db.db_object != DMU_META_DNODE_OBJECT) { 1275fa9e4066Sahrens /* 127682c9918fSTim Haley * Release the data buffer from the cache so 127782c9918fSTim Haley * that we can modify it without impacting 127882c9918fSTim Haley * possible other users of this cached data 127982c9918fSTim Haley * block. Note that indirect blocks and 128082c9918fSTim Haley * private objects are not released until the 128182c9918fSTim Haley * syncing state (since they are only modified 128282c9918fSTim Haley * then). 1283fa9e4066Sahrens */ 1284fa9e4066Sahrens arc_release(db->db_buf, db); 1285fa9e4066Sahrens dbuf_fix_old_data(db, tx->tx_txg); 1286c717a561Smaybee data_old = db->db_buf; 1287fa9e4066Sahrens } 1288c717a561Smaybee ASSERT(data_old != NULL); 128982c9918fSTim Haley } 1290c717a561Smaybee dr->dt.dl.dr_data = data_old; 1291c717a561Smaybee } else { 1292c717a561Smaybee mutex_init(&dr->dt.di.dr_mtx, NULL, MUTEX_DEFAULT, NULL); 1293c717a561Smaybee list_create(&dr->dt.di.dr_children, 1294c717a561Smaybee sizeof (dbuf_dirty_record_t), 1295c717a561Smaybee offsetof(dbuf_dirty_record_t, dr_dirty_node)); 1296fa9e4066Sahrens } 129769962b56SMatthew Ahrens if (db->db_blkid != DMU_BONUS_BLKID && os->os_dsl_dataset != NULL) 129869962b56SMatthew Ahrens dr->dr_accounted = db->db.db_size; 1299c717a561Smaybee dr->dr_dbuf = db; 1300c717a561Smaybee dr->dr_txg = tx->tx_txg; 1301c717a561Smaybee dr->dr_next = *drp; 1302c717a561Smaybee *drp = dr; 1303fa9e4066Sahrens 1304fa9e4066Sahrens /* 1305fa9e4066Sahrens * We could have been freed_in_flight between the dbuf_noread 1306fa9e4066Sahrens * and dbuf_dirty. We win, as though the dbuf_noread() had 1307fa9e4066Sahrens * happened after the free. 1308fa9e4066Sahrens */ 13090a586ceaSMark Shellenbaum if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 13100a586ceaSMark Shellenbaum db->db_blkid != DMU_SPILL_BLKID) { 1311c717a561Smaybee mutex_enter(&dn->dn_mtx); 1312bf16b11eSMatthew Ahrens if (dn->dn_free_ranges[txgoff] != NULL) { 1313bf16b11eSMatthew Ahrens range_tree_clear(dn->dn_free_ranges[txgoff], 1314bf16b11eSMatthew Ahrens db->db_blkid, 1); 1315bf16b11eSMatthew Ahrens } 1316fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1317c717a561Smaybee db->db_freed_in_flight = FALSE; 1318c717a561Smaybee } 1319fa9e4066Sahrens 1320fa9e4066Sahrens /* 1321fa9e4066Sahrens * This buffer is now part of this txg 1322fa9e4066Sahrens */ 1323fa9e4066Sahrens dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg); 1324fa9e4066Sahrens db->db_dirtycnt += 1; 1325fa9e4066Sahrens ASSERT3U(db->db_dirtycnt, <=, 3); 1326fa9e4066Sahrens 1327fa9e4066Sahrens mutex_exit(&db->db_mtx); 1328fa9e4066Sahrens 13290a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID || 13300a586ceaSMark Shellenbaum db->db_blkid == DMU_SPILL_BLKID) { 1331c717a561Smaybee mutex_enter(&dn->dn_mtx); 1332c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 1333c717a561Smaybee list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1334c717a561Smaybee mutex_exit(&dn->dn_mtx); 1335fa9e4066Sahrens dnode_setdirty(dn, tx); 1336744947dcSTom Erickson DB_DNODE_EXIT(db); 1337c717a561Smaybee return (dr); 1338d3469faaSMark Maybee } else if (do_free_accounting) { 1339d3469faaSMark Maybee blkptr_t *bp = db->db_blkptr; 1340d3469faaSMark Maybee int64_t willfree = (bp && !BP_IS_HOLE(bp)) ? 1341b24ab676SJeff Bonwick bp_get_dsize(os->os_spa, bp) : db->db.db_size; 1342d3469faaSMark Maybee /* 1343d3469faaSMark Maybee * This is only a guess -- if the dbuf is dirty 1344d3469faaSMark Maybee * in a previous txg, we don't know how much 1345d3469faaSMark Maybee * space it will use on disk yet. We should 1346d3469faaSMark Maybee * really have the struct_rwlock to access 1347d3469faaSMark Maybee * db_blkptr, but since this is just a guess, 1348d3469faaSMark Maybee * it's OK if we get an odd answer. 1349d3469faaSMark Maybee */ 1350837b568bSGeorge Wilson ddt_prefetch(os->os_spa, bp); 1351d3469faaSMark Maybee dnode_willuse_space(dn, -willfree, tx); 1352fa9e4066Sahrens } 1353fa9e4066Sahrens 1354fa9e4066Sahrens if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 1355fa9e4066Sahrens rw_enter(&dn->dn_struct_rwlock, RW_READER); 1356fa9e4066Sahrens drop_struct_lock = TRUE; 1357fa9e4066Sahrens } 1358fa9e4066Sahrens 13598346f03fSJonathan W Adams if (db->db_level == 0) { 13608346f03fSJonathan W Adams dnode_new_blkid(dn, db->db_blkid, tx, drop_struct_lock); 13618346f03fSJonathan W Adams ASSERT(dn->dn_maxblkid >= db->db_blkid); 13628346f03fSJonathan W Adams } 13638346f03fSJonathan W Adams 136444eda4d7Smaybee if (db->db_level+1 < dn->dn_nlevels) { 1365c717a561Smaybee dmu_buf_impl_t *parent = db->db_parent; 1366c717a561Smaybee dbuf_dirty_record_t *di; 1367c717a561Smaybee int parent_held = FALSE; 1368c717a561Smaybee 1369c717a561Smaybee if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) { 1370fa9e4066Sahrens int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1371c717a561Smaybee 1372fa9e4066Sahrens parent = dbuf_hold_level(dn, db->db_level+1, 1373fa9e4066Sahrens db->db_blkid >> epbs, FTAG); 137401025c89SJohn Harres ASSERT(parent != NULL); 1375c717a561Smaybee parent_held = TRUE; 1376c717a561Smaybee } 1377fa9e4066Sahrens if (drop_struct_lock) 1378fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 1379c717a561Smaybee ASSERT3U(db->db_level+1, ==, parent->db_level); 1380c717a561Smaybee di = dbuf_dirty(parent, tx); 1381c717a561Smaybee if (parent_held) 1382ea8dc4b6Seschrock dbuf_rele(parent, FTAG); 1383c717a561Smaybee 1384c717a561Smaybee mutex_enter(&db->db_mtx); 138569962b56SMatthew Ahrens /* 138669962b56SMatthew Ahrens * Since we've dropped the mutex, it's possible that 138769962b56SMatthew Ahrens * dbuf_undirty() might have changed this out from under us. 138869962b56SMatthew Ahrens */ 1389c717a561Smaybee if (db->db_last_dirty == dr || 1390c717a561Smaybee dn->dn_object == DMU_META_DNODE_OBJECT) { 1391c717a561Smaybee mutex_enter(&di->dt.di.dr_mtx); 1392c717a561Smaybee ASSERT3U(di->dr_txg, ==, tx->tx_txg); 1393c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 1394c717a561Smaybee list_insert_tail(&di->dt.di.dr_children, dr); 1395c717a561Smaybee mutex_exit(&di->dt.di.dr_mtx); 1396c717a561Smaybee dr->dr_parent = di; 1397c717a561Smaybee } 1398c717a561Smaybee mutex_exit(&db->db_mtx); 1399fa9e4066Sahrens } else { 1400c717a561Smaybee ASSERT(db->db_level+1 == dn->dn_nlevels); 1401c717a561Smaybee ASSERT(db->db_blkid < dn->dn_nblkptr); 1402744947dcSTom Erickson ASSERT(db->db_parent == NULL || db->db_parent == dn->dn_dbuf); 1403c717a561Smaybee mutex_enter(&dn->dn_mtx); 1404c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 1405c717a561Smaybee list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1406c717a561Smaybee mutex_exit(&dn->dn_mtx); 1407fa9e4066Sahrens if (drop_struct_lock) 1408fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 1409fa9e4066Sahrens } 1410fa9e4066Sahrens 1411fa9e4066Sahrens dnode_setdirty(dn, tx); 1412744947dcSTom Erickson DB_DNODE_EXIT(db); 1413c717a561Smaybee return (dr); 1414fa9e4066Sahrens } 1415fa9e4066Sahrens 14163b2aab18SMatthew Ahrens /* 14173e30c24aSWill Andrews * Undirty a buffer in the transaction group referenced by the given 14183e30c24aSWill Andrews * transaction. Return whether this evicted the dbuf. 14193b2aab18SMatthew Ahrens */ 14203b2aab18SMatthew Ahrens static boolean_t 1421fa9e4066Sahrens dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1422fa9e4066Sahrens { 1423744947dcSTom Erickson dnode_t *dn; 1424c717a561Smaybee uint64_t txg = tx->tx_txg; 142517f17c2dSbonwick dbuf_dirty_record_t *dr, **drp; 1426fa9e4066Sahrens 1427c717a561Smaybee ASSERT(txg != 0); 142846e1baa6SMatthew Ahrens 142946e1baa6SMatthew Ahrens /* 143046e1baa6SMatthew Ahrens * Due to our use of dn_nlevels below, this can only be called 143146e1baa6SMatthew Ahrens * in open context, unless we are operating on the MOS. 143246e1baa6SMatthew Ahrens * From syncing context, dn_nlevels may be different from the 143346e1baa6SMatthew Ahrens * dn_nlevels used when dbuf was dirtied. 143446e1baa6SMatthew Ahrens */ 143546e1baa6SMatthew Ahrens ASSERT(db->db_objset == 143646e1baa6SMatthew Ahrens dmu_objset_pool(db->db_objset)->dp_meta_objset || 143746e1baa6SMatthew Ahrens txg != spa_syncing_txg(dmu_objset_spa(db->db_objset))); 14380a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 14393b2aab18SMatthew Ahrens ASSERT0(db->db_level); 14403b2aab18SMatthew Ahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 1441fa9e4066Sahrens 1442fa9e4066Sahrens /* 1443fa9e4066Sahrens * If this buffer is not dirty, we're done. 1444fa9e4066Sahrens */ 144517f17c2dSbonwick for (drp = &db->db_last_dirty; (dr = *drp) != NULL; drp = &dr->dr_next) 1446c717a561Smaybee if (dr->dr_txg <= txg) 1447c717a561Smaybee break; 14483b2aab18SMatthew Ahrens if (dr == NULL || dr->dr_txg < txg) 14493b2aab18SMatthew Ahrens return (B_FALSE); 1450c717a561Smaybee ASSERT(dr->dr_txg == txg); 1451b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 1452fa9e4066Sahrens 1453744947dcSTom Erickson DB_DNODE_ENTER(db); 1454744947dcSTom Erickson dn = DB_DNODE(db); 1455744947dcSTom Erickson 1456fa9e4066Sahrens dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1457fa9e4066Sahrens 1458fa9e4066Sahrens ASSERT(db->db.db_size != 0); 1459fa9e4066Sahrens 146046e1baa6SMatthew Ahrens dsl_pool_undirty_space(dmu_objset_pool(dn->dn_objset), 146146e1baa6SMatthew Ahrens dr->dr_accounted, txg); 1462fa9e4066Sahrens 146317f17c2dSbonwick *drp = dr->dr_next; 1464c717a561Smaybee 14653f2366c2SGordon Ross /* 14663f2366c2SGordon Ross * Note that there are three places in dbuf_dirty() 14673f2366c2SGordon Ross * where this dirty record may be put on a list. 14683f2366c2SGordon Ross * Make sure to do a list_remove corresponding to 14693f2366c2SGordon Ross * every one of those list_insert calls. 14703f2366c2SGordon Ross */ 1471c717a561Smaybee if (dr->dr_parent) { 1472c717a561Smaybee mutex_enter(&dr->dr_parent->dt.di.dr_mtx); 1473c717a561Smaybee list_remove(&dr->dr_parent->dt.di.dr_children, dr); 1474c717a561Smaybee mutex_exit(&dr->dr_parent->dt.di.dr_mtx); 14753f2366c2SGordon Ross } else if (db->db_blkid == DMU_SPILL_BLKID || 14763f2366c2SGordon Ross db->db_level + 1 == dn->dn_nlevels) { 1477cdb0ab79Smaybee ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf); 1478fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 1479c717a561Smaybee list_remove(&dn->dn_dirty_records[txg & TXG_MASK], dr); 1480fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1481c717a561Smaybee } 1482744947dcSTom Erickson DB_DNODE_EXIT(db); 1483c717a561Smaybee 148482c9918fSTim Haley if (db->db_state != DB_NOFILL) { 1485c717a561Smaybee dbuf_unoverride(dr); 1486c717a561Smaybee 1487c717a561Smaybee ASSERT(db->db_buf != NULL); 1488c717a561Smaybee ASSERT(dr->dt.dl.dr_data != NULL); 1489c717a561Smaybee if (dr->dt.dl.dr_data != db->db_buf) 14903b2aab18SMatthew Ahrens VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, db)); 1491c717a561Smaybee } 1492d2b3cbbdSJorgen Lundman 1493c717a561Smaybee kmem_free(dr, sizeof (dbuf_dirty_record_t)); 1494fa9e4066Sahrens 1495fa9e4066Sahrens ASSERT(db->db_dirtycnt > 0); 1496fa9e4066Sahrens db->db_dirtycnt -= 1; 1497fa9e4066Sahrens 1498c717a561Smaybee if (refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) { 1499ea8dc4b6Seschrock arc_buf_t *buf = db->db_buf; 1500fa9e4066Sahrens 1501b24ab676SJeff Bonwick ASSERT(db->db_state == DB_NOFILL || arc_released(buf)); 1502bc9014e6SJustin Gibbs dbuf_clear_data(db); 15033b2aab18SMatthew Ahrens VERIFY(arc_buf_remove_ref(buf, db)); 1504fa9e4066Sahrens dbuf_evict(db); 15053b2aab18SMatthew Ahrens return (B_TRUE); 1506fa9e4066Sahrens } 1507fa9e4066Sahrens 15083b2aab18SMatthew Ahrens return (B_FALSE); 1509fa9e4066Sahrens } 1510fa9e4066Sahrens 1511fa9e4066Sahrens void 151243466aaeSMax Grossman dmu_buf_will_dirty(dmu_buf_t *db_fake, dmu_tx_t *tx) 1513fa9e4066Sahrens { 151443466aaeSMax Grossman dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 15151ab7f2deSmaybee int rf = DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH; 1516fa9e4066Sahrens 1517fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 1518fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 1519fa9e4066Sahrens 1520*0f2e7d03SMatthew Ahrens /* 1521*0f2e7d03SMatthew Ahrens * Quick check for dirtyness. For already dirty blocks, this 1522*0f2e7d03SMatthew Ahrens * reduces runtime of this function by >90%, and overall performance 1523*0f2e7d03SMatthew Ahrens * by 50% for some workloads (e.g. file deletion with indirect blocks 1524*0f2e7d03SMatthew Ahrens * cached). 1525*0f2e7d03SMatthew Ahrens */ 1526*0f2e7d03SMatthew Ahrens mutex_enter(&db->db_mtx); 1527*0f2e7d03SMatthew Ahrens dbuf_dirty_record_t *dr; 1528*0f2e7d03SMatthew Ahrens for (dr = db->db_last_dirty; 1529*0f2e7d03SMatthew Ahrens dr != NULL && dr->dr_txg >= tx->tx_txg; dr = dr->dr_next) { 1530*0f2e7d03SMatthew Ahrens /* 1531*0f2e7d03SMatthew Ahrens * It's possible that it is already dirty but not cached, 1532*0f2e7d03SMatthew Ahrens * because there are some calls to dbuf_dirty() that don't 1533*0f2e7d03SMatthew Ahrens * go through dmu_buf_will_dirty(). 1534*0f2e7d03SMatthew Ahrens */ 1535*0f2e7d03SMatthew Ahrens if (dr->dr_txg == tx->tx_txg && db->db_state == DB_CACHED) { 1536*0f2e7d03SMatthew Ahrens /* This dbuf is already dirty and cached. */ 1537*0f2e7d03SMatthew Ahrens dbuf_redirty(dr); 1538*0f2e7d03SMatthew Ahrens mutex_exit(&db->db_mtx); 1539*0f2e7d03SMatthew Ahrens return; 1540*0f2e7d03SMatthew Ahrens } 1541*0f2e7d03SMatthew Ahrens } 1542*0f2e7d03SMatthew Ahrens mutex_exit(&db->db_mtx); 1543*0f2e7d03SMatthew Ahrens 1544744947dcSTom Erickson DB_DNODE_ENTER(db); 1545744947dcSTom Erickson if (RW_WRITE_HELD(&DB_DNODE(db)->dn_struct_rwlock)) 1546fa9e4066Sahrens rf |= DB_RF_HAVESTRUCT; 1547744947dcSTom Erickson DB_DNODE_EXIT(db); 1548ea8dc4b6Seschrock (void) dbuf_read(db, NULL, rf); 1549c717a561Smaybee (void) dbuf_dirty(db, tx); 1550fa9e4066Sahrens } 1551fa9e4066Sahrens 1552fa9e4066Sahrens void 155382c9918fSTim Haley dmu_buf_will_not_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 155482c9918fSTim Haley { 155582c9918fSTim Haley dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 155682c9918fSTim Haley 155782c9918fSTim Haley db->db_state = DB_NOFILL; 155882c9918fSTim Haley 155982c9918fSTim Haley dmu_buf_will_fill(db_fake, tx); 156082c9918fSTim Haley } 156182c9918fSTim Haley 156282c9918fSTim Haley void 1563ea8dc4b6Seschrock dmu_buf_will_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 1564fa9e4066Sahrens { 1565ea8dc4b6Seschrock dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1566ea8dc4b6Seschrock 15670a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1568fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 1569fa9e4066Sahrens ASSERT(db->db_level == 0); 1570fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 1571fa9e4066Sahrens 1572ea8dc4b6Seschrock ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT || 1573fa9e4066Sahrens dmu_tx_private_ok(tx)); 1574fa9e4066Sahrens 1575fa9e4066Sahrens dbuf_noread(db); 1576c717a561Smaybee (void) dbuf_dirty(db, tx); 1577fa9e4066Sahrens } 1578fa9e4066Sahrens 1579fa9e4066Sahrens #pragma weak dmu_buf_fill_done = dbuf_fill_done 1580fa9e4066Sahrens /* ARGSUSED */ 1581fa9e4066Sahrens void 1582fa9e4066Sahrens dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx) 1583fa9e4066Sahrens { 1584fa9e4066Sahrens mutex_enter(&db->db_mtx); 15859c9dc39aSek110237 DBUF_VERIFY(db); 1586fa9e4066Sahrens 1587fa9e4066Sahrens if (db->db_state == DB_FILL) { 1588c717a561Smaybee if (db->db_level == 0 && db->db_freed_in_flight) { 15890a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1590fa9e4066Sahrens /* we were freed while filling */ 1591fa9e4066Sahrens /* XXX dbuf_undirty? */ 1592fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 1593c717a561Smaybee db->db_freed_in_flight = FALSE; 1594fa9e4066Sahrens } 1595fa9e4066Sahrens db->db_state = DB_CACHED; 1596fa9e4066Sahrens cv_broadcast(&db->db_changed); 1597fa9e4066Sahrens } 1598fa9e4066Sahrens mutex_exit(&db->db_mtx); 1599fa9e4066Sahrens } 1600fa9e4066Sahrens 16015d7b4d43SMatthew Ahrens void 16025d7b4d43SMatthew Ahrens dmu_buf_write_embedded(dmu_buf_t *dbuf, void *data, 16035d7b4d43SMatthew Ahrens bp_embedded_type_t etype, enum zio_compress comp, 16045d7b4d43SMatthew Ahrens int uncompressed_size, int compressed_size, int byteorder, 16055d7b4d43SMatthew Ahrens dmu_tx_t *tx) 16065d7b4d43SMatthew Ahrens { 16075d7b4d43SMatthew Ahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf; 16085d7b4d43SMatthew Ahrens struct dirty_leaf *dl; 16095d7b4d43SMatthew Ahrens dmu_object_type_t type; 16105d7b4d43SMatthew Ahrens 1611ca0cc391SMatthew Ahrens if (etype == BP_EMBEDDED_TYPE_DATA) { 1612ca0cc391SMatthew Ahrens ASSERT(spa_feature_is_active(dmu_objset_spa(db->db_objset), 1613ca0cc391SMatthew Ahrens SPA_FEATURE_EMBEDDED_DATA)); 1614ca0cc391SMatthew Ahrens } 1615ca0cc391SMatthew Ahrens 16165d7b4d43SMatthew Ahrens DB_DNODE_ENTER(db); 16175d7b4d43SMatthew Ahrens type = DB_DNODE(db)->dn_type; 16185d7b4d43SMatthew Ahrens DB_DNODE_EXIT(db); 16195d7b4d43SMatthew Ahrens 16205d7b4d43SMatthew Ahrens ASSERT0(db->db_level); 16215d7b4d43SMatthew Ahrens ASSERT(db->db_blkid != DMU_BONUS_BLKID); 16225d7b4d43SMatthew Ahrens 16235d7b4d43SMatthew Ahrens dmu_buf_will_not_fill(dbuf, tx); 16245d7b4d43SMatthew Ahrens 16255d7b4d43SMatthew Ahrens ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg); 16265d7b4d43SMatthew Ahrens dl = &db->db_last_dirty->dt.dl; 16275d7b4d43SMatthew Ahrens encode_embedded_bp_compressed(&dl->dr_overridden_by, 16285d7b4d43SMatthew Ahrens data, comp, uncompressed_size, compressed_size); 16295d7b4d43SMatthew Ahrens BPE_SET_ETYPE(&dl->dr_overridden_by, etype); 16305d7b4d43SMatthew Ahrens BP_SET_TYPE(&dl->dr_overridden_by, type); 16315d7b4d43SMatthew Ahrens BP_SET_LEVEL(&dl->dr_overridden_by, 0); 16325d7b4d43SMatthew Ahrens BP_SET_BYTEORDER(&dl->dr_overridden_by, byteorder); 16335d7b4d43SMatthew Ahrens 16345d7b4d43SMatthew Ahrens dl->dr_override_state = DR_OVERRIDDEN; 16355d7b4d43SMatthew Ahrens dl->dr_overridden_by.blk_birth = db->db_last_dirty->dr_txg; 16365d7b4d43SMatthew Ahrens } 16375d7b4d43SMatthew Ahrens 1638ea8dc4b6Seschrock /* 16392fdbea25SAleksandr Guzovskiy * Directly assign a provided arc buf to a given dbuf if it's not referenced 16402fdbea25SAleksandr Guzovskiy * by anybody except our caller. Otherwise copy arcbuf's contents to dbuf. 16412fdbea25SAleksandr Guzovskiy */ 16422fdbea25SAleksandr Guzovskiy void 16432fdbea25SAleksandr Guzovskiy dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx) 16442fdbea25SAleksandr Guzovskiy { 16452fdbea25SAleksandr Guzovskiy ASSERT(!refcount_is_zero(&db->db_holds)); 16460a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 16472fdbea25SAleksandr Guzovskiy ASSERT(db->db_level == 0); 16482fdbea25SAleksandr Guzovskiy ASSERT(DBUF_GET_BUFC_TYPE(db) == ARC_BUFC_DATA); 16492fdbea25SAleksandr Guzovskiy ASSERT(buf != NULL); 16502fdbea25SAleksandr Guzovskiy ASSERT(arc_buf_size(buf) == db->db.db_size); 16512fdbea25SAleksandr Guzovskiy ASSERT(tx->tx_txg != 0); 16522fdbea25SAleksandr Guzovskiy 16532fdbea25SAleksandr Guzovskiy arc_return_buf(buf, db); 16542fdbea25SAleksandr Guzovskiy ASSERT(arc_released(buf)); 16552fdbea25SAleksandr Guzovskiy 16562fdbea25SAleksandr Guzovskiy mutex_enter(&db->db_mtx); 16572fdbea25SAleksandr Guzovskiy 16582fdbea25SAleksandr Guzovskiy while (db->db_state == DB_READ || db->db_state == DB_FILL) 16592fdbea25SAleksandr Guzovskiy cv_wait(&db->db_changed, &db->db_mtx); 16602fdbea25SAleksandr Guzovskiy 16612fdbea25SAleksandr Guzovskiy ASSERT(db->db_state == DB_CACHED || db->db_state == DB_UNCACHED); 16622fdbea25SAleksandr Guzovskiy 16632fdbea25SAleksandr Guzovskiy if (db->db_state == DB_CACHED && 16642fdbea25SAleksandr Guzovskiy refcount_count(&db->db_holds) - 1 > db->db_dirtycnt) { 16652fdbea25SAleksandr Guzovskiy mutex_exit(&db->db_mtx); 16662fdbea25SAleksandr Guzovskiy (void) dbuf_dirty(db, tx); 16672fdbea25SAleksandr Guzovskiy bcopy(buf->b_data, db->db.db_data, db->db.db_size); 16683b2aab18SMatthew Ahrens VERIFY(arc_buf_remove_ref(buf, db)); 1669c242f9a0Schunli zhang - Sun Microsystems - Irvine United States xuio_stat_wbuf_copied(); 16702fdbea25SAleksandr Guzovskiy return; 16712fdbea25SAleksandr Guzovskiy } 16722fdbea25SAleksandr Guzovskiy 1673c242f9a0Schunli zhang - Sun Microsystems - Irvine United States xuio_stat_wbuf_nocopy(); 16742fdbea25SAleksandr Guzovskiy if (db->db_state == DB_CACHED) { 16752fdbea25SAleksandr Guzovskiy dbuf_dirty_record_t *dr = db->db_last_dirty; 16762fdbea25SAleksandr Guzovskiy 16772fdbea25SAleksandr Guzovskiy ASSERT(db->db_buf != NULL); 16782fdbea25SAleksandr Guzovskiy if (dr != NULL && dr->dr_txg == tx->tx_txg) { 16792fdbea25SAleksandr Guzovskiy ASSERT(dr->dt.dl.dr_data == db->db_buf); 16802fdbea25SAleksandr Guzovskiy if (!arc_released(db->db_buf)) { 16812fdbea25SAleksandr Guzovskiy ASSERT(dr->dt.dl.dr_override_state == 16822fdbea25SAleksandr Guzovskiy DR_OVERRIDDEN); 16832fdbea25SAleksandr Guzovskiy arc_release(db->db_buf, db); 16842fdbea25SAleksandr Guzovskiy } 16852fdbea25SAleksandr Guzovskiy dr->dt.dl.dr_data = buf; 16863b2aab18SMatthew Ahrens VERIFY(arc_buf_remove_ref(db->db_buf, db)); 16872fdbea25SAleksandr Guzovskiy } else if (dr == NULL || dr->dt.dl.dr_data != db->db_buf) { 16882fdbea25SAleksandr Guzovskiy arc_release(db->db_buf, db); 16893b2aab18SMatthew Ahrens VERIFY(arc_buf_remove_ref(db->db_buf, db)); 16902fdbea25SAleksandr Guzovskiy } 16912fdbea25SAleksandr Guzovskiy db->db_buf = NULL; 16922fdbea25SAleksandr Guzovskiy } 16932fdbea25SAleksandr Guzovskiy ASSERT(db->db_buf == NULL); 16942fdbea25SAleksandr Guzovskiy dbuf_set_data(db, buf); 16952fdbea25SAleksandr Guzovskiy db->db_state = DB_FILL; 16962fdbea25SAleksandr Guzovskiy mutex_exit(&db->db_mtx); 16972fdbea25SAleksandr Guzovskiy (void) dbuf_dirty(db, tx); 169843466aaeSMax Grossman dmu_buf_fill_done(&db->db, tx); 16992fdbea25SAleksandr Guzovskiy } 17002fdbea25SAleksandr Guzovskiy 17012fdbea25SAleksandr Guzovskiy /* 1702ea8dc4b6Seschrock * "Clear" the contents of this dbuf. This will mark the dbuf 170369962b56SMatthew Ahrens * EVICTING and clear *most* of its references. Unfortunately, 1704ea8dc4b6Seschrock * when we are not holding the dn_dbufs_mtx, we can't clear the 1705ea8dc4b6Seschrock * entry in the dn_dbufs list. We have to wait until dbuf_destroy() 1706ea8dc4b6Seschrock * in this case. For callers from the DMU we will usually see: 1707bbfa8ea8SMatthew Ahrens * dbuf_clear()->arc_clear_callback()->dbuf_do_evict()->dbuf_destroy() 1708ea8dc4b6Seschrock * For the arc callback, we will usually see: 1709ea8dc4b6Seschrock * dbuf_do_evict()->dbuf_clear();dbuf_destroy() 1710ea8dc4b6Seschrock * Sometimes, though, we will get a mix of these two: 1711bbfa8ea8SMatthew Ahrens * DMU: dbuf_clear()->arc_clear_callback() 1712ea8dc4b6Seschrock * ARC: dbuf_do_evict()->dbuf_destroy() 1713bbfa8ea8SMatthew Ahrens * 1714bbfa8ea8SMatthew Ahrens * This routine will dissociate the dbuf from the arc, by calling 1715bbfa8ea8SMatthew Ahrens * arc_clear_callback(), but will not evict the data from the ARC. 1716ea8dc4b6Seschrock */ 1717ea8dc4b6Seschrock void 1718fa9e4066Sahrens dbuf_clear(dmu_buf_impl_t *db) 1719fa9e4066Sahrens { 1720744947dcSTom Erickson dnode_t *dn; 1721ea8dc4b6Seschrock dmu_buf_impl_t *parent = db->db_parent; 1722744947dcSTom Erickson dmu_buf_impl_t *dndb; 1723bbfa8ea8SMatthew Ahrens boolean_t dbuf_gone = B_FALSE; 1724fa9e4066Sahrens 1725fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 1726fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 1727fa9e4066Sahrens 1728ea8dc4b6Seschrock dbuf_evict_user(db); 1729ea8dc4b6Seschrock 1730fa9e4066Sahrens if (db->db_state == DB_CACHED) { 1731ea8dc4b6Seschrock ASSERT(db->db.db_data != NULL); 17320a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 1733ea8dc4b6Seschrock zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN); 17345a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 17350e8c6158Smaybee } 1736fa9e4066Sahrens db->db.db_data = NULL; 1737fa9e4066Sahrens db->db_state = DB_UNCACHED; 1738fa9e4066Sahrens } 1739fa9e4066Sahrens 174082c9918fSTim Haley ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL); 1741fa9e4066Sahrens ASSERT(db->db_data_pending == NULL); 1742fa9e4066Sahrens 1743ea8dc4b6Seschrock db->db_state = DB_EVICTING; 1744ea8dc4b6Seschrock db->db_blkptr = NULL; 1745ea8dc4b6Seschrock 1746744947dcSTom Erickson DB_DNODE_ENTER(db); 1747744947dcSTom Erickson dn = DB_DNODE(db); 1748744947dcSTom Erickson dndb = dn->dn_dbuf; 17490a586ceaSMark Shellenbaum if (db->db_blkid != DMU_BONUS_BLKID && MUTEX_HELD(&dn->dn_dbufs_mtx)) { 17500f6d88adSAlex Reece avl_remove(&dn->dn_dbufs, db); 1751640c1670SJosef 'Jeff' Sipek atomic_dec_32(&dn->dn_dbufs_count); 1752744947dcSTom Erickson membar_producer(); 1753744947dcSTom Erickson DB_DNODE_EXIT(db); 1754744947dcSTom Erickson /* 1755744947dcSTom Erickson * Decrementing the dbuf count means that the hold corresponding 1756744947dcSTom Erickson * to the removed dbuf is no longer discounted in dnode_move(), 1757744947dcSTom Erickson * so the dnode cannot be moved until after we release the hold. 1758744947dcSTom Erickson * The membar_producer() ensures visibility of the decremented 1759744947dcSTom Erickson * value in dnode_move(), since DB_DNODE_EXIT doesn't actually 1760744947dcSTom Erickson * release any lock. 1761744947dcSTom Erickson */ 1762ea8dc4b6Seschrock dnode_rele(dn, db); 1763744947dcSTom Erickson db->db_dnode_handle = NULL; 1764744947dcSTom Erickson } else { 1765744947dcSTom Erickson DB_DNODE_EXIT(db); 1766ea8dc4b6Seschrock } 1767ea8dc4b6Seschrock 1768ea8dc4b6Seschrock if (db->db_buf) 1769bbfa8ea8SMatthew Ahrens dbuf_gone = arc_clear_callback(db->db_buf); 1770ea8dc4b6Seschrock 1771ea8dc4b6Seschrock if (!dbuf_gone) 1772fa9e4066Sahrens mutex_exit(&db->db_mtx); 1773fa9e4066Sahrens 1774fa9e4066Sahrens /* 1775744947dcSTom Erickson * If this dbuf is referenced from an indirect dbuf, 1776fa9e4066Sahrens * decrement the ref count on the indirect dbuf. 1777fa9e4066Sahrens */ 1778c543ec06Sahrens if (parent && parent != dndb) 1779ea8dc4b6Seschrock dbuf_rele(parent, db); 1780fa9e4066Sahrens } 1781fa9e4066Sahrens 1782a2cdcdd2SPaul Dagnelie /* 1783a2cdcdd2SPaul Dagnelie * Note: While bpp will always be updated if the function returns success, 1784a2cdcdd2SPaul Dagnelie * parentp will not be updated if the dnode does not have dn_dbuf filled in; 1785a2cdcdd2SPaul Dagnelie * this happens when the dnode is the meta-dnode, or a userused or groupused 1786a2cdcdd2SPaul Dagnelie * object. 1787a2cdcdd2SPaul Dagnelie */ 1788fa9e4066Sahrens static int 1789fa9e4066Sahrens dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse, 1790fa9e4066Sahrens dmu_buf_impl_t **parentp, blkptr_t **bpp) 1791fa9e4066Sahrens { 1792fa9e4066Sahrens int nlevels, epbs; 1793fa9e4066Sahrens 17940b69c2f0Sahrens *parentp = NULL; 17950b69c2f0Sahrens *bpp = NULL; 17960b69c2f0Sahrens 17970a586ceaSMark Shellenbaum ASSERT(blkid != DMU_BONUS_BLKID); 17980a586ceaSMark Shellenbaum 17990a586ceaSMark Shellenbaum if (blkid == DMU_SPILL_BLKID) { 18000a586ceaSMark Shellenbaum mutex_enter(&dn->dn_mtx); 180106e0070dSMark Shellenbaum if (dn->dn_have_spill && 180206e0070dSMark Shellenbaum (dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR)) 18030a586ceaSMark Shellenbaum *bpp = &dn->dn_phys->dn_spill; 18040a586ceaSMark Shellenbaum else 18050a586ceaSMark Shellenbaum *bpp = NULL; 18060a586ceaSMark Shellenbaum dbuf_add_ref(dn->dn_dbuf, NULL); 18070a586ceaSMark Shellenbaum *parentp = dn->dn_dbuf; 18080a586ceaSMark Shellenbaum mutex_exit(&dn->dn_mtx); 18090a586ceaSMark Shellenbaum return (0); 18100a586ceaSMark Shellenbaum } 1811ea8dc4b6Seschrock 1812fa9e4066Sahrens if (dn->dn_phys->dn_nlevels == 0) 1813fa9e4066Sahrens nlevels = 1; 1814fa9e4066Sahrens else 1815fa9e4066Sahrens nlevels = dn->dn_phys->dn_nlevels; 1816fa9e4066Sahrens 1817fa9e4066Sahrens epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1818fa9e4066Sahrens 1819fa9e4066Sahrens ASSERT3U(level * epbs, <, 64); 1820fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1821ea8dc4b6Seschrock if (level >= nlevels || 1822fa9e4066Sahrens (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) { 1823fa9e4066Sahrens /* the buffer has no parent yet */ 1824be6fd75aSMatthew Ahrens return (SET_ERROR(ENOENT)); 1825fa9e4066Sahrens } else if (level < nlevels-1) { 1826fa9e4066Sahrens /* this block is referenced from an indirect block */ 1827fa9e4066Sahrens int err = dbuf_hold_impl(dn, level+1, 1828a2cdcdd2SPaul Dagnelie blkid >> epbs, fail_sparse, FALSE, NULL, parentp); 1829fa9e4066Sahrens if (err) 1830fa9e4066Sahrens return (err); 1831ea8dc4b6Seschrock err = dbuf_read(*parentp, NULL, 1832ea8dc4b6Seschrock (DB_RF_HAVESTRUCT | DB_RF_NOPREFETCH | DB_RF_CANFAIL)); 1833c543ec06Sahrens if (err) { 1834c543ec06Sahrens dbuf_rele(*parentp, NULL); 1835c543ec06Sahrens *parentp = NULL; 1836c543ec06Sahrens return (err); 1837c543ec06Sahrens } 1838fa9e4066Sahrens *bpp = ((blkptr_t *)(*parentp)->db.db_data) + 1839fa9e4066Sahrens (blkid & ((1ULL << epbs) - 1)); 1840c543ec06Sahrens return (0); 1841fa9e4066Sahrens } else { 1842fa9e4066Sahrens /* the block is referenced from the dnode */ 1843fa9e4066Sahrens ASSERT3U(level, ==, nlevels-1); 1844fa9e4066Sahrens ASSERT(dn->dn_phys->dn_nblkptr == 0 || 1845fa9e4066Sahrens blkid < dn->dn_phys->dn_nblkptr); 1846c543ec06Sahrens if (dn->dn_dbuf) { 1847c543ec06Sahrens dbuf_add_ref(dn->dn_dbuf, NULL); 1848fa9e4066Sahrens *parentp = dn->dn_dbuf; 1849c543ec06Sahrens } 1850fa9e4066Sahrens *bpp = &dn->dn_phys->dn_blkptr[blkid]; 1851fa9e4066Sahrens return (0); 1852fa9e4066Sahrens } 1853fa9e4066Sahrens } 1854fa9e4066Sahrens 1855fa9e4066Sahrens static dmu_buf_impl_t * 1856fa9e4066Sahrens dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid, 1857fa9e4066Sahrens dmu_buf_impl_t *parent, blkptr_t *blkptr) 1858fa9e4066Sahrens { 1859503ad85cSMatthew Ahrens objset_t *os = dn->dn_objset; 1860fa9e4066Sahrens dmu_buf_impl_t *db, *odb; 1861fa9e4066Sahrens 1862fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1863fa9e4066Sahrens ASSERT(dn->dn_type != DMU_OT_NONE); 1864fa9e4066Sahrens 1865fa9e4066Sahrens db = kmem_cache_alloc(dbuf_cache, KM_SLEEP); 1866fa9e4066Sahrens 1867fa9e4066Sahrens db->db_objset = os; 1868fa9e4066Sahrens db->db.db_object = dn->dn_object; 1869fa9e4066Sahrens db->db_level = level; 1870fa9e4066Sahrens db->db_blkid = blkid; 1871c717a561Smaybee db->db_last_dirty = NULL; 1872ea8dc4b6Seschrock db->db_dirtycnt = 0; 1873744947dcSTom Erickson db->db_dnode_handle = dn->dn_handle; 1874ea8dc4b6Seschrock db->db_parent = parent; 1875ea8dc4b6Seschrock db->db_blkptr = blkptr; 1876fa9e4066Sahrens 1877bc9014e6SJustin Gibbs db->db_user = NULL; 1878c717a561Smaybee db->db_immediate_evict = 0; 1879c717a561Smaybee db->db_freed_in_flight = 0; 1880ea8dc4b6Seschrock 18810a586ceaSMark Shellenbaum if (blkid == DMU_BONUS_BLKID) { 1882ea8dc4b6Seschrock ASSERT3P(parent, ==, dn->dn_dbuf); 18831934e92fSmaybee db->db.db_size = DN_MAX_BONUSLEN - 18841934e92fSmaybee (dn->dn_nblkptr-1) * sizeof (blkptr_t); 18851934e92fSmaybee ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 18860a586ceaSMark Shellenbaum db->db.db_offset = DMU_BONUS_BLKID; 1887ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 1888ea8dc4b6Seschrock /* the bonus dbuf is not placed in the hash table */ 18895a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1890ea8dc4b6Seschrock return (db); 18910a586ceaSMark Shellenbaum } else if (blkid == DMU_SPILL_BLKID) { 18920a586ceaSMark Shellenbaum db->db.db_size = (blkptr != NULL) ? 18930a586ceaSMark Shellenbaum BP_GET_LSIZE(blkptr) : SPA_MINBLOCKSIZE; 18940a586ceaSMark Shellenbaum db->db.db_offset = 0; 1895fa9e4066Sahrens } else { 1896fa9e4066Sahrens int blocksize = 1897fa9e4066Sahrens db->db_level ? 1 << dn->dn_indblkshift : dn->dn_datablksz; 1898fa9e4066Sahrens db->db.db_size = blocksize; 1899fa9e4066Sahrens db->db.db_offset = db->db_blkid * blocksize; 1900fa9e4066Sahrens } 1901fa9e4066Sahrens 1902fa9e4066Sahrens /* 1903fa9e4066Sahrens * Hold the dn_dbufs_mtx while we get the new dbuf 1904fa9e4066Sahrens * in the hash table *and* added to the dbufs list. 1905fa9e4066Sahrens * This prevents a possible deadlock with someone 1906fa9e4066Sahrens * trying to look up this dbuf before its added to the 1907fa9e4066Sahrens * dn_dbufs list. 1908fa9e4066Sahrens */ 1909fa9e4066Sahrens mutex_enter(&dn->dn_dbufs_mtx); 1910ea8dc4b6Seschrock db->db_state = DB_EVICTING; 1911fa9e4066Sahrens if ((odb = dbuf_hash_insert(db)) != NULL) { 1912fa9e4066Sahrens /* someone else inserted it first */ 1913fa9e4066Sahrens kmem_cache_free(dbuf_cache, db); 1914fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 1915fa9e4066Sahrens return (odb); 1916fa9e4066Sahrens } 19170f6d88adSAlex Reece avl_add(&dn->dn_dbufs, db); 1918713d6c20SMatthew Ahrens if (db->db_level == 0 && db->db_blkid >= 1919713d6c20SMatthew Ahrens dn->dn_unlisted_l0_blkid) 1920713d6c20SMatthew Ahrens dn->dn_unlisted_l0_blkid = db->db_blkid + 1; 1921ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 1922fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 19235a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1924fa9e4066Sahrens 1925fa9e4066Sahrens if (parent && parent != dn->dn_dbuf) 1926fa9e4066Sahrens dbuf_add_ref(parent, db); 1927fa9e4066Sahrens 1928ea8dc4b6Seschrock ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 1929ea8dc4b6Seschrock refcount_count(&dn->dn_holds) > 0); 1930fa9e4066Sahrens (void) refcount_add(&dn->dn_holds, db); 1931640c1670SJosef 'Jeff' Sipek atomic_inc_32(&dn->dn_dbufs_count); 1932fa9e4066Sahrens 1933fa9e4066Sahrens dprintf_dbuf(db, "db=%p\n", db); 1934fa9e4066Sahrens 1935fa9e4066Sahrens return (db); 1936fa9e4066Sahrens } 1937fa9e4066Sahrens 1938fa9e4066Sahrens static int 1939ea8dc4b6Seschrock dbuf_do_evict(void *private) 1940fa9e4066Sahrens { 1941bbfa8ea8SMatthew Ahrens dmu_buf_impl_t *db = private; 1942fa9e4066Sahrens 1943ea8dc4b6Seschrock if (!MUTEX_HELD(&db->db_mtx)) 1944ea8dc4b6Seschrock mutex_enter(&db->db_mtx); 1945ea8dc4b6Seschrock 1946ea8dc4b6Seschrock ASSERT(refcount_is_zero(&db->db_holds)); 1947ea8dc4b6Seschrock 1948ea8dc4b6Seschrock if (db->db_state != DB_EVICTING) { 1949ea8dc4b6Seschrock ASSERT(db->db_state == DB_CACHED); 19509c9dc39aSek110237 DBUF_VERIFY(db); 1951ea8dc4b6Seschrock db->db_buf = NULL; 1952ea8dc4b6Seschrock dbuf_evict(db); 1953ea8dc4b6Seschrock } else { 1954ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 1955ea8dc4b6Seschrock dbuf_destroy(db); 1956fa9e4066Sahrens } 1957ea8dc4b6Seschrock return (0); 1958fa9e4066Sahrens } 1959fa9e4066Sahrens 1960fa9e4066Sahrens static void 1961fa9e4066Sahrens dbuf_destroy(dmu_buf_impl_t *db) 1962fa9e4066Sahrens { 1963fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 1964fa9e4066Sahrens 19650a586ceaSMark Shellenbaum if (db->db_blkid != DMU_BONUS_BLKID) { 1966ea8dc4b6Seschrock /* 1967ea8dc4b6Seschrock * If this dbuf is still on the dn_dbufs list, 1968ea8dc4b6Seschrock * remove it from that list. 1969ea8dc4b6Seschrock */ 1970744947dcSTom Erickson if (db->db_dnode_handle != NULL) { 1971744947dcSTom Erickson dnode_t *dn; 19721934e92fSmaybee 1973744947dcSTom Erickson DB_DNODE_ENTER(db); 1974744947dcSTom Erickson dn = DB_DNODE(db); 19751934e92fSmaybee mutex_enter(&dn->dn_dbufs_mtx); 19760f6d88adSAlex Reece avl_remove(&dn->dn_dbufs, db); 1977640c1670SJosef 'Jeff' Sipek atomic_dec_32(&dn->dn_dbufs_count); 1978ea8dc4b6Seschrock mutex_exit(&dn->dn_dbufs_mtx); 1979744947dcSTom Erickson DB_DNODE_EXIT(db); 1980744947dcSTom Erickson /* 1981744947dcSTom Erickson * Decrementing the dbuf count means that the hold 1982744947dcSTom Erickson * corresponding to the removed dbuf is no longer 1983744947dcSTom Erickson * discounted in dnode_move(), so the dnode cannot be 1984744947dcSTom Erickson * moved until after we release the hold. 1985744947dcSTom Erickson */ 1986ea8dc4b6Seschrock dnode_rele(dn, db); 1987744947dcSTom Erickson db->db_dnode_handle = NULL; 1988ea8dc4b6Seschrock } 1989ea8dc4b6Seschrock dbuf_hash_remove(db); 1990ea8dc4b6Seschrock } 1991ea8dc4b6Seschrock db->db_parent = NULL; 1992ea8dc4b6Seschrock db->db_buf = NULL; 1993ea8dc4b6Seschrock 1994fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 1995fa9e4066Sahrens ASSERT(db->db_hash_next == NULL); 1996fa9e4066Sahrens ASSERT(db->db_blkptr == NULL); 1997fa9e4066Sahrens ASSERT(db->db_data_pending == NULL); 1998fa9e4066Sahrens 1999fa9e4066Sahrens kmem_cache_free(dbuf_cache, db); 20005a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 2001fa9e4066Sahrens } 2002fa9e4066Sahrens 2003a2cdcdd2SPaul Dagnelie typedef struct dbuf_prefetch_arg { 2004a2cdcdd2SPaul Dagnelie spa_t *dpa_spa; /* The spa to issue the prefetch in. */ 2005a2cdcdd2SPaul Dagnelie zbookmark_phys_t dpa_zb; /* The target block to prefetch. */ 2006a2cdcdd2SPaul Dagnelie int dpa_epbs; /* Entries (blkptr_t's) Per Block Shift. */ 2007a2cdcdd2SPaul Dagnelie int dpa_curlevel; /* The current level that we're reading */ 2008a2cdcdd2SPaul Dagnelie zio_priority_t dpa_prio; /* The priority I/Os should be issued at. */ 2009a2cdcdd2SPaul Dagnelie zio_t *dpa_zio; /* The parent zio_t for all prefetches. */ 2010a2cdcdd2SPaul Dagnelie arc_flags_t dpa_aflags; /* Flags to pass to the final prefetch. */ 2011a2cdcdd2SPaul Dagnelie } dbuf_prefetch_arg_t; 2012a2cdcdd2SPaul Dagnelie 2013a2cdcdd2SPaul Dagnelie /* 2014a2cdcdd2SPaul Dagnelie * Actually issue the prefetch read for the block given. 2015a2cdcdd2SPaul Dagnelie */ 2016a2cdcdd2SPaul Dagnelie static void 2017a2cdcdd2SPaul Dagnelie dbuf_issue_final_prefetch(dbuf_prefetch_arg_t *dpa, blkptr_t *bp) 2018fa9e4066Sahrens { 2019a2cdcdd2SPaul Dagnelie if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp)) 2020a2cdcdd2SPaul Dagnelie return; 2021a2cdcdd2SPaul Dagnelie 2022a2cdcdd2SPaul Dagnelie arc_flags_t aflags = 2023a2cdcdd2SPaul Dagnelie dpa->dpa_aflags | ARC_FLAG_NOWAIT | ARC_FLAG_PREFETCH; 2024a2cdcdd2SPaul Dagnelie 2025a2cdcdd2SPaul Dagnelie ASSERT3U(dpa->dpa_curlevel, ==, BP_GET_LEVEL(bp)); 2026a2cdcdd2SPaul Dagnelie ASSERT3U(dpa->dpa_curlevel, ==, dpa->dpa_zb.zb_level); 2027a2cdcdd2SPaul Dagnelie ASSERT(dpa->dpa_zio != NULL); 2028a2cdcdd2SPaul Dagnelie (void) arc_read(dpa->dpa_zio, dpa->dpa_spa, bp, NULL, NULL, 2029a2cdcdd2SPaul Dagnelie dpa->dpa_prio, ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 2030a2cdcdd2SPaul Dagnelie &aflags, &dpa->dpa_zb); 2031a2cdcdd2SPaul Dagnelie } 2032a2cdcdd2SPaul Dagnelie 2033a2cdcdd2SPaul Dagnelie /* 2034a2cdcdd2SPaul Dagnelie * Called when an indirect block above our prefetch target is read in. This 2035a2cdcdd2SPaul Dagnelie * will either read in the next indirect block down the tree or issue the actual 2036a2cdcdd2SPaul Dagnelie * prefetch if the next block down is our target. 2037a2cdcdd2SPaul Dagnelie */ 2038a2cdcdd2SPaul Dagnelie static void 2039a2cdcdd2SPaul Dagnelie dbuf_prefetch_indirect_done(zio_t *zio, arc_buf_t *abuf, void *private) 2040a2cdcdd2SPaul Dagnelie { 2041a2cdcdd2SPaul Dagnelie dbuf_prefetch_arg_t *dpa = private; 2042a2cdcdd2SPaul Dagnelie 2043a2cdcdd2SPaul Dagnelie ASSERT3S(dpa->dpa_zb.zb_level, <, dpa->dpa_curlevel); 2044a2cdcdd2SPaul Dagnelie ASSERT3S(dpa->dpa_curlevel, >, 0); 2045a2cdcdd2SPaul Dagnelie if (zio != NULL) { 2046a2cdcdd2SPaul Dagnelie ASSERT3S(BP_GET_LEVEL(zio->io_bp), ==, dpa->dpa_curlevel); 2047a2cdcdd2SPaul Dagnelie ASSERT3U(BP_GET_LSIZE(zio->io_bp), ==, zio->io_size); 2048a2cdcdd2SPaul Dagnelie ASSERT3P(zio->io_spa, ==, dpa->dpa_spa); 2049a2cdcdd2SPaul Dagnelie } 2050a2cdcdd2SPaul Dagnelie 2051a2cdcdd2SPaul Dagnelie dpa->dpa_curlevel--; 2052a2cdcdd2SPaul Dagnelie 2053a2cdcdd2SPaul Dagnelie uint64_t nextblkid = dpa->dpa_zb.zb_blkid >> 2054a2cdcdd2SPaul Dagnelie (dpa->dpa_epbs * (dpa->dpa_curlevel - dpa->dpa_zb.zb_level)); 2055a2cdcdd2SPaul Dagnelie blkptr_t *bp = ((blkptr_t *)abuf->b_data) + 2056a2cdcdd2SPaul Dagnelie P2PHASE(nextblkid, 1ULL << dpa->dpa_epbs); 2057a2cdcdd2SPaul Dagnelie if (BP_IS_HOLE(bp) || (zio != NULL && zio->io_error != 0)) { 2058a2cdcdd2SPaul Dagnelie kmem_free(dpa, sizeof (*dpa)); 2059a2cdcdd2SPaul Dagnelie } else if (dpa->dpa_curlevel == dpa->dpa_zb.zb_level) { 2060a2cdcdd2SPaul Dagnelie ASSERT3U(nextblkid, ==, dpa->dpa_zb.zb_blkid); 2061a2cdcdd2SPaul Dagnelie dbuf_issue_final_prefetch(dpa, bp); 2062a2cdcdd2SPaul Dagnelie kmem_free(dpa, sizeof (*dpa)); 2063a2cdcdd2SPaul Dagnelie } else { 2064a2cdcdd2SPaul Dagnelie arc_flags_t iter_aflags = ARC_FLAG_NOWAIT; 2065a2cdcdd2SPaul Dagnelie zbookmark_phys_t zb; 2066a2cdcdd2SPaul Dagnelie 2067a2cdcdd2SPaul Dagnelie ASSERT3U(dpa->dpa_curlevel, ==, BP_GET_LEVEL(bp)); 2068a2cdcdd2SPaul Dagnelie 2069a2cdcdd2SPaul Dagnelie SET_BOOKMARK(&zb, dpa->dpa_zb.zb_objset, 2070a2cdcdd2SPaul Dagnelie dpa->dpa_zb.zb_object, dpa->dpa_curlevel, nextblkid); 2071a2cdcdd2SPaul Dagnelie 2072a2cdcdd2SPaul Dagnelie (void) arc_read(dpa->dpa_zio, dpa->dpa_spa, 2073a2cdcdd2SPaul Dagnelie bp, dbuf_prefetch_indirect_done, dpa, dpa->dpa_prio, 2074a2cdcdd2SPaul Dagnelie ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 2075a2cdcdd2SPaul Dagnelie &iter_aflags, &zb); 2076a2cdcdd2SPaul Dagnelie } 2077a2cdcdd2SPaul Dagnelie (void) arc_buf_remove_ref(abuf, private); 2078a2cdcdd2SPaul Dagnelie } 2079a2cdcdd2SPaul Dagnelie 2080a2cdcdd2SPaul Dagnelie /* 2081a2cdcdd2SPaul Dagnelie * Issue prefetch reads for the given block on the given level. If the indirect 2082a2cdcdd2SPaul Dagnelie * blocks above that block are not in memory, we will read them in 2083a2cdcdd2SPaul Dagnelie * asynchronously. As a result, this call never blocks waiting for a read to 2084a2cdcdd2SPaul Dagnelie * complete. 2085a2cdcdd2SPaul Dagnelie */ 2086a2cdcdd2SPaul Dagnelie void 2087a2cdcdd2SPaul Dagnelie dbuf_prefetch(dnode_t *dn, int64_t level, uint64_t blkid, zio_priority_t prio, 2088a2cdcdd2SPaul Dagnelie arc_flags_t aflags) 2089a2cdcdd2SPaul Dagnelie { 2090a2cdcdd2SPaul Dagnelie blkptr_t bp; 2091a2cdcdd2SPaul Dagnelie int epbs, nlevels, curlevel; 2092a2cdcdd2SPaul Dagnelie uint64_t curblkid; 2093fa9e4066Sahrens 20940a586ceaSMark Shellenbaum ASSERT(blkid != DMU_BONUS_BLKID); 2095fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 2096fa9e4066Sahrens 2097cf6106c8SMatthew Ahrens if (blkid > dn->dn_maxblkid) 2098cf6106c8SMatthew Ahrens return; 2099cf6106c8SMatthew Ahrens 2100fa9e4066Sahrens if (dnode_block_freed(dn, blkid)) 2101fa9e4066Sahrens return; 2102fa9e4066Sahrens 2103fa9e4066Sahrens /* 2104a2cdcdd2SPaul Dagnelie * This dnode hasn't been written to disk yet, so there's nothing to 2105a2cdcdd2SPaul Dagnelie * prefetch. 2106fa9e4066Sahrens */ 2107a2cdcdd2SPaul Dagnelie nlevels = dn->dn_phys->dn_nlevels; 2108a2cdcdd2SPaul Dagnelie if (level >= nlevels || dn->dn_phys->dn_nblkptr == 0) 2109a2cdcdd2SPaul Dagnelie return; 2110a2cdcdd2SPaul Dagnelie 2111a2cdcdd2SPaul Dagnelie epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2112a2cdcdd2SPaul Dagnelie if (dn->dn_phys->dn_maxblkid < blkid << (epbs * level)) 2113a2cdcdd2SPaul Dagnelie return; 2114a2cdcdd2SPaul Dagnelie 2115a2cdcdd2SPaul Dagnelie dmu_buf_impl_t *db = dbuf_find(dn->dn_objset, dn->dn_object, 2116a2cdcdd2SPaul Dagnelie level, blkid); 2117a2cdcdd2SPaul Dagnelie if (db != NULL) { 2118fa9e4066Sahrens mutex_exit(&db->db_mtx); 2119a2cdcdd2SPaul Dagnelie /* 2120a2cdcdd2SPaul Dagnelie * This dbuf already exists. It is either CACHED, or 2121a2cdcdd2SPaul Dagnelie * (we assume) about to be read or filled. 2122a2cdcdd2SPaul Dagnelie */ 2123fa9e4066Sahrens return; 2124fa9e4066Sahrens } 2125fa9e4066Sahrens 2126a2cdcdd2SPaul Dagnelie /* 2127a2cdcdd2SPaul Dagnelie * Find the closest ancestor (indirect block) of the target block 2128a2cdcdd2SPaul Dagnelie * that is present in the cache. In this indirect block, we will 2129a2cdcdd2SPaul Dagnelie * find the bp that is at curlevel, curblkid. 2130a2cdcdd2SPaul Dagnelie */ 2131a2cdcdd2SPaul Dagnelie curlevel = level; 2132a2cdcdd2SPaul Dagnelie curblkid = blkid; 2133a2cdcdd2SPaul Dagnelie while (curlevel < nlevels - 1) { 2134a2cdcdd2SPaul Dagnelie int parent_level = curlevel + 1; 2135a2cdcdd2SPaul Dagnelie uint64_t parent_blkid = curblkid >> epbs; 2136a2cdcdd2SPaul Dagnelie dmu_buf_impl_t *db; 2137a2cdcdd2SPaul Dagnelie 2138a2cdcdd2SPaul Dagnelie if (dbuf_hold_impl(dn, parent_level, parent_blkid, 2139a2cdcdd2SPaul Dagnelie FALSE, TRUE, FTAG, &db) == 0) { 2140a2cdcdd2SPaul Dagnelie blkptr_t *bpp = db->db_buf->b_data; 2141a2cdcdd2SPaul Dagnelie bp = bpp[P2PHASE(curblkid, 1 << epbs)]; 2142a2cdcdd2SPaul Dagnelie dbuf_rele(db, FTAG); 2143a2cdcdd2SPaul Dagnelie break; 2144a2cdcdd2SPaul Dagnelie } 2145a2cdcdd2SPaul Dagnelie 2146a2cdcdd2SPaul Dagnelie curlevel = parent_level; 2147a2cdcdd2SPaul Dagnelie curblkid = parent_blkid; 2148a2cdcdd2SPaul Dagnelie } 2149a2cdcdd2SPaul Dagnelie 2150a2cdcdd2SPaul Dagnelie if (curlevel == nlevels - 1) { 2151a2cdcdd2SPaul Dagnelie /* No cached indirect blocks found. */ 2152a2cdcdd2SPaul Dagnelie ASSERT3U(curblkid, <, dn->dn_phys->dn_nblkptr); 2153a2cdcdd2SPaul Dagnelie bp = dn->dn_phys->dn_blkptr[curblkid]; 2154a2cdcdd2SPaul Dagnelie } 2155a2cdcdd2SPaul Dagnelie if (BP_IS_HOLE(&bp)) 2156a2cdcdd2SPaul Dagnelie return; 2157a2cdcdd2SPaul Dagnelie 2158a2cdcdd2SPaul Dagnelie ASSERT3U(curlevel, ==, BP_GET_LEVEL(&bp)); 2159a2cdcdd2SPaul Dagnelie 2160a2cdcdd2SPaul Dagnelie zio_t *pio = zio_root(dmu_objset_spa(dn->dn_objset), NULL, NULL, 2161a2cdcdd2SPaul Dagnelie ZIO_FLAG_CANFAIL); 2162a2cdcdd2SPaul Dagnelie 2163a2cdcdd2SPaul Dagnelie dbuf_prefetch_arg_t *dpa = kmem_zalloc(sizeof (*dpa), KM_SLEEP); 2164b24ab676SJeff Bonwick dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset; 2165a2cdcdd2SPaul Dagnelie SET_BOOKMARK(&dpa->dpa_zb, ds != NULL ? ds->ds_object : DMU_META_OBJSET, 2166a2cdcdd2SPaul Dagnelie dn->dn_object, level, blkid); 2167a2cdcdd2SPaul Dagnelie dpa->dpa_curlevel = curlevel; 2168a2cdcdd2SPaul Dagnelie dpa->dpa_prio = prio; 2169a2cdcdd2SPaul Dagnelie dpa->dpa_aflags = aflags; 2170a2cdcdd2SPaul Dagnelie dpa->dpa_spa = dn->dn_objset->os_spa; 2171a2cdcdd2SPaul Dagnelie dpa->dpa_epbs = epbs; 2172a2cdcdd2SPaul Dagnelie dpa->dpa_zio = pio; 2173a2cdcdd2SPaul Dagnelie 2174a2cdcdd2SPaul Dagnelie /* 2175a2cdcdd2SPaul Dagnelie * If we have the indirect just above us, no need to do the asynchronous 2176a2cdcdd2SPaul Dagnelie * prefetch chain; we'll just run the last step ourselves. If we're at 2177a2cdcdd2SPaul Dagnelie * a higher level, though, we want to issue the prefetches for all the 2178a2cdcdd2SPaul Dagnelie * indirect blocks asynchronously, so we can go on with whatever we were 2179a2cdcdd2SPaul Dagnelie * doing. 2180a2cdcdd2SPaul Dagnelie */ 2181a2cdcdd2SPaul Dagnelie if (curlevel == level) { 2182a2cdcdd2SPaul Dagnelie ASSERT3U(curblkid, ==, blkid); 2183a2cdcdd2SPaul Dagnelie dbuf_issue_final_prefetch(dpa, &bp); 2184a2cdcdd2SPaul Dagnelie kmem_free(dpa, sizeof (*dpa)); 2185a2cdcdd2SPaul Dagnelie } else { 2186a2cdcdd2SPaul Dagnelie arc_flags_t iter_aflags = ARC_FLAG_NOWAIT; 21877802d7bfSMatthew Ahrens zbookmark_phys_t zb; 2188b24ab676SJeff Bonwick 2189a2cdcdd2SPaul Dagnelie SET_BOOKMARK(&zb, ds != NULL ? ds->ds_object : DMU_META_OBJSET, 2190a2cdcdd2SPaul Dagnelie dn->dn_object, curlevel, curblkid); 2191a2cdcdd2SPaul Dagnelie (void) arc_read(dpa->dpa_zio, dpa->dpa_spa, 2192a2cdcdd2SPaul Dagnelie &bp, dbuf_prefetch_indirect_done, dpa, prio, 2193fa9e4066Sahrens ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 2194a2cdcdd2SPaul Dagnelie &iter_aflags, &zb); 2195fa9e4066Sahrens } 2196a2cdcdd2SPaul Dagnelie /* 2197a2cdcdd2SPaul Dagnelie * We use pio here instead of dpa_zio since it's possible that 2198a2cdcdd2SPaul Dagnelie * dpa may have already been freed. 2199a2cdcdd2SPaul Dagnelie */ 2200a2cdcdd2SPaul Dagnelie zio_nowait(pio); 2201fa9e4066Sahrens } 2202fa9e4066Sahrens 2203fa9e4066Sahrens /* 2204fa9e4066Sahrens * Returns with db_holds incremented, and db_mtx not held. 2205fa9e4066Sahrens * Note: dn_struct_rwlock must be held. 2206fa9e4066Sahrens */ 2207fa9e4066Sahrens int 2208a2cdcdd2SPaul Dagnelie dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, 2209a2cdcdd2SPaul Dagnelie boolean_t fail_sparse, boolean_t fail_uncached, 2210fa9e4066Sahrens void *tag, dmu_buf_impl_t **dbp) 2211fa9e4066Sahrens { 2212fa9e4066Sahrens dmu_buf_impl_t *db, *parent = NULL; 2213fa9e4066Sahrens 22140a586ceaSMark Shellenbaum ASSERT(blkid != DMU_BONUS_BLKID); 2215fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 2216fa9e4066Sahrens ASSERT3U(dn->dn_nlevels, >, level); 2217fa9e4066Sahrens 2218fa9e4066Sahrens *dbp = NULL; 2219ea8dc4b6Seschrock top: 2220fa9e4066Sahrens /* dbuf_find() returns with db_mtx held */ 2221e57a022bSJustin T. Gibbs db = dbuf_find(dn->dn_objset, dn->dn_object, level, blkid); 2222fa9e4066Sahrens 2223fa9e4066Sahrens if (db == NULL) { 2224fa9e4066Sahrens blkptr_t *bp = NULL; 2225fa9e4066Sahrens int err; 2226fa9e4066Sahrens 2227a2cdcdd2SPaul Dagnelie if (fail_uncached) 2228a2cdcdd2SPaul Dagnelie return (SET_ERROR(ENOENT)); 2229a2cdcdd2SPaul Dagnelie 2230c543ec06Sahrens ASSERT3P(parent, ==, NULL); 2231fa9e4066Sahrens err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp); 2232fa9e4066Sahrens if (fail_sparse) { 2233fa9e4066Sahrens if (err == 0 && bp && BP_IS_HOLE(bp)) 2234be6fd75aSMatthew Ahrens err = SET_ERROR(ENOENT); 2235fa9e4066Sahrens if (err) { 2236c543ec06Sahrens if (parent) 2237ea8dc4b6Seschrock dbuf_rele(parent, NULL); 2238fa9e4066Sahrens return (err); 2239fa9e4066Sahrens } 2240fa9e4066Sahrens } 2241ea8dc4b6Seschrock if (err && err != ENOENT) 2242ea8dc4b6Seschrock return (err); 2243fa9e4066Sahrens db = dbuf_create(dn, level, blkid, parent, bp); 2244fa9e4066Sahrens } 2245fa9e4066Sahrens 2246a2cdcdd2SPaul Dagnelie if (fail_uncached && db->db_state != DB_CACHED) { 2247a2cdcdd2SPaul Dagnelie mutex_exit(&db->db_mtx); 2248a2cdcdd2SPaul Dagnelie return (SET_ERROR(ENOENT)); 2249a2cdcdd2SPaul Dagnelie } 2250a2cdcdd2SPaul Dagnelie 2251ea8dc4b6Seschrock if (db->db_buf && refcount_is_zero(&db->db_holds)) { 2252ea8dc4b6Seschrock arc_buf_add_ref(db->db_buf, db); 2253ea8dc4b6Seschrock if (db->db_buf->b_data == NULL) { 2254ea8dc4b6Seschrock dbuf_clear(db); 2255c543ec06Sahrens if (parent) { 2256c543ec06Sahrens dbuf_rele(parent, NULL); 2257c543ec06Sahrens parent = NULL; 2258c543ec06Sahrens } 2259ea8dc4b6Seschrock goto top; 2260ea8dc4b6Seschrock } 2261ea8dc4b6Seschrock ASSERT3P(db->db.db_data, ==, db->db_buf->b_data); 2262ea8dc4b6Seschrock } 2263ea8dc4b6Seschrock 2264ea8dc4b6Seschrock ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf)); 2265ea8dc4b6Seschrock 2266fa9e4066Sahrens /* 2267c717a561Smaybee * If this buffer is currently syncing out, and we are are 2268c717a561Smaybee * still referencing it from db_data, we need to make a copy 2269c717a561Smaybee * of it in case we decide we want to dirty it again in this txg. 2270fa9e4066Sahrens */ 22710a586ceaSMark Shellenbaum if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 2272ea8dc4b6Seschrock dn->dn_object != DMU_META_DNODE_OBJECT && 2273c717a561Smaybee db->db_state == DB_CACHED && db->db_data_pending) { 2274c717a561Smaybee dbuf_dirty_record_t *dr = db->db_data_pending; 2275c717a561Smaybee 2276c717a561Smaybee if (dr->dt.dl.dr_data == db->db_buf) { 2277ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 2278fa9e4066Sahrens 2279c717a561Smaybee dbuf_set_data(db, 2280744947dcSTom Erickson arc_buf_alloc(dn->dn_objset->os_spa, 2281c717a561Smaybee db->db.db_size, db, type)); 2282c717a561Smaybee bcopy(dr->dt.dl.dr_data->b_data, db->db.db_data, 2283fa9e4066Sahrens db->db.db_size); 2284fa9e4066Sahrens } 2285c717a561Smaybee } 2286fa9e4066Sahrens 2287ea8dc4b6Seschrock (void) refcount_add(&db->db_holds, tag); 22889c9dc39aSek110237 DBUF_VERIFY(db); 2289fa9e4066Sahrens mutex_exit(&db->db_mtx); 2290fa9e4066Sahrens 2291fa9e4066Sahrens /* NOTE: we can't rele the parent until after we drop the db_mtx */ 2292c543ec06Sahrens if (parent) 2293ea8dc4b6Seschrock dbuf_rele(parent, NULL); 2294fa9e4066Sahrens 2295744947dcSTom Erickson ASSERT3P(DB_DNODE(db), ==, dn); 2296fa9e4066Sahrens ASSERT3U(db->db_blkid, ==, blkid); 2297fa9e4066Sahrens ASSERT3U(db->db_level, ==, level); 2298fa9e4066Sahrens *dbp = db; 2299fa9e4066Sahrens 2300fa9e4066Sahrens return (0); 2301fa9e4066Sahrens } 2302fa9e4066Sahrens 2303fa9e4066Sahrens dmu_buf_impl_t * 2304ea8dc4b6Seschrock dbuf_hold(dnode_t *dn, uint64_t blkid, void *tag) 2305fa9e4066Sahrens { 2306a2cdcdd2SPaul Dagnelie return (dbuf_hold_level(dn, 0, blkid, tag)); 2307fa9e4066Sahrens } 2308fa9e4066Sahrens 2309fa9e4066Sahrens dmu_buf_impl_t * 2310fa9e4066Sahrens dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, void *tag) 2311fa9e4066Sahrens { 2312fa9e4066Sahrens dmu_buf_impl_t *db; 2313a2cdcdd2SPaul Dagnelie int err = dbuf_hold_impl(dn, level, blkid, FALSE, FALSE, tag, &db); 2314ea8dc4b6Seschrock return (err ? NULL : db); 2315fa9e4066Sahrens } 2316fa9e4066Sahrens 23171934e92fSmaybee void 2318ea8dc4b6Seschrock dbuf_create_bonus(dnode_t *dn) 2319fa9e4066Sahrens { 2320ea8dc4b6Seschrock ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 2321ea8dc4b6Seschrock 2322ea8dc4b6Seschrock ASSERT(dn->dn_bonus == NULL); 23230a586ceaSMark Shellenbaum dn->dn_bonus = dbuf_create(dn, 0, DMU_BONUS_BLKID, dn->dn_dbuf, NULL); 23240a586ceaSMark Shellenbaum } 23250a586ceaSMark Shellenbaum 23260a586ceaSMark Shellenbaum int 23270a586ceaSMark Shellenbaum dbuf_spill_set_blksz(dmu_buf_t *db_fake, uint64_t blksz, dmu_tx_t *tx) 23280a586ceaSMark Shellenbaum { 23290a586ceaSMark Shellenbaum dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2330744947dcSTom Erickson dnode_t *dn; 2331744947dcSTom Erickson 23320a586ceaSMark Shellenbaum if (db->db_blkid != DMU_SPILL_BLKID) 2333be6fd75aSMatthew Ahrens return (SET_ERROR(ENOTSUP)); 23340a586ceaSMark Shellenbaum if (blksz == 0) 23350a586ceaSMark Shellenbaum blksz = SPA_MINBLOCKSIZE; 2336b5152584SMatthew Ahrens ASSERT3U(blksz, <=, spa_maxblocksize(dmu_objset_spa(db->db_objset))); 23370a586ceaSMark Shellenbaum blksz = P2ROUNDUP(blksz, SPA_MINBLOCKSIZE); 23380a586ceaSMark Shellenbaum 2339744947dcSTom Erickson DB_DNODE_ENTER(db); 2340744947dcSTom Erickson dn = DB_DNODE(db); 2341744947dcSTom Erickson rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 23420a586ceaSMark Shellenbaum dbuf_new_size(db, blksz, tx); 2343744947dcSTom Erickson rw_exit(&dn->dn_struct_rwlock); 2344744947dcSTom Erickson DB_DNODE_EXIT(db); 23450a586ceaSMark Shellenbaum 23460a586ceaSMark Shellenbaum return (0); 23470a586ceaSMark Shellenbaum } 23480a586ceaSMark Shellenbaum 23490a586ceaSMark Shellenbaum void 23500a586ceaSMark Shellenbaum dbuf_rm_spill(dnode_t *dn, dmu_tx_t *tx) 23510a586ceaSMark Shellenbaum { 23520a586ceaSMark Shellenbaum dbuf_free_range(dn, DMU_SPILL_BLKID, DMU_SPILL_BLKID, tx); 2353fa9e4066Sahrens } 2354fa9e4066Sahrens 2355ea8dc4b6Seschrock #pragma weak dmu_buf_add_ref = dbuf_add_ref 2356fa9e4066Sahrens void 2357fa9e4066Sahrens dbuf_add_ref(dmu_buf_impl_t *db, void *tag) 2358fa9e4066Sahrens { 2359ea8dc4b6Seschrock int64_t holds = refcount_add(&db->db_holds, tag); 2360ea8dc4b6Seschrock ASSERT(holds > 1); 2361fa9e4066Sahrens } 2362fa9e4066Sahrens 2363e57a022bSJustin T. Gibbs #pragma weak dmu_buf_try_add_ref = dbuf_try_add_ref 2364e57a022bSJustin T. Gibbs boolean_t 2365e57a022bSJustin T. Gibbs dbuf_try_add_ref(dmu_buf_t *db_fake, objset_t *os, uint64_t obj, uint64_t blkid, 2366e57a022bSJustin T. Gibbs void *tag) 2367e57a022bSJustin T. Gibbs { 2368e57a022bSJustin T. Gibbs dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2369e57a022bSJustin T. Gibbs dmu_buf_impl_t *found_db; 2370e57a022bSJustin T. Gibbs boolean_t result = B_FALSE; 2371e57a022bSJustin T. Gibbs 2372e57a022bSJustin T. Gibbs if (db->db_blkid == DMU_BONUS_BLKID) 2373e57a022bSJustin T. Gibbs found_db = dbuf_find_bonus(os, obj); 2374e57a022bSJustin T. Gibbs else 2375e57a022bSJustin T. Gibbs found_db = dbuf_find(os, obj, 0, blkid); 2376e57a022bSJustin T. Gibbs 2377e57a022bSJustin T. Gibbs if (found_db != NULL) { 2378e57a022bSJustin T. Gibbs if (db == found_db && dbuf_refcount(db) > db->db_dirtycnt) { 2379e57a022bSJustin T. Gibbs (void) refcount_add(&db->db_holds, tag); 2380e57a022bSJustin T. Gibbs result = B_TRUE; 2381e57a022bSJustin T. Gibbs } 2382e57a022bSJustin T. Gibbs mutex_exit(&db->db_mtx); 2383e57a022bSJustin T. Gibbs } 2384e57a022bSJustin T. Gibbs return (result); 2385e57a022bSJustin T. Gibbs } 2386e57a022bSJustin T. Gibbs 2387744947dcSTom Erickson /* 2388744947dcSTom Erickson * If you call dbuf_rele() you had better not be referencing the dnode handle 2389744947dcSTom Erickson * unless you have some other direct or indirect hold on the dnode. (An indirect 2390744947dcSTom Erickson * hold is a hold on one of the dnode's dbufs, including the bonus buffer.) 2391744947dcSTom Erickson * Without that, the dbuf_rele() could lead to a dnode_rele() followed by the 2392744947dcSTom Erickson * dnode's parent dbuf evicting its dnode handles. 2393744947dcSTom Erickson */ 2394fa9e4066Sahrens void 2395ea8dc4b6Seschrock dbuf_rele(dmu_buf_impl_t *db, void *tag) 2396fa9e4066Sahrens { 2397b24ab676SJeff Bonwick mutex_enter(&db->db_mtx); 2398b24ab676SJeff Bonwick dbuf_rele_and_unlock(db, tag); 2399b24ab676SJeff Bonwick } 2400b24ab676SJeff Bonwick 240143466aaeSMax Grossman void 240243466aaeSMax Grossman dmu_buf_rele(dmu_buf_t *db, void *tag) 240343466aaeSMax Grossman { 240443466aaeSMax Grossman dbuf_rele((dmu_buf_impl_t *)db, tag); 240543466aaeSMax Grossman } 240643466aaeSMax Grossman 2407b24ab676SJeff Bonwick /* 2408b24ab676SJeff Bonwick * dbuf_rele() for an already-locked dbuf. This is necessary to allow 2409b24ab676SJeff Bonwick * db_dirtycnt and db_holds to be updated atomically. 2410b24ab676SJeff Bonwick */ 2411b24ab676SJeff Bonwick void 2412b24ab676SJeff Bonwick dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag) 2413b24ab676SJeff Bonwick { 2414fa9e4066Sahrens int64_t holds; 2415fa9e4066Sahrens 2416b24ab676SJeff Bonwick ASSERT(MUTEX_HELD(&db->db_mtx)); 24179c9dc39aSek110237 DBUF_VERIFY(db); 2418fa9e4066Sahrens 2419744947dcSTom Erickson /* 2420744947dcSTom Erickson * Remove the reference to the dbuf before removing its hold on the 2421744947dcSTom Erickson * dnode so we can guarantee in dnode_move() that a referenced bonus 2422744947dcSTom Erickson * buffer has a corresponding dnode hold. 2423744947dcSTom Erickson */ 2424fa9e4066Sahrens holds = refcount_remove(&db->db_holds, tag); 2425ea8dc4b6Seschrock ASSERT(holds >= 0); 2426fa9e4066Sahrens 2427c717a561Smaybee /* 2428c717a561Smaybee * We can't freeze indirects if there is a possibility that they 2429c717a561Smaybee * may be modified in the current syncing context. 2430c717a561Smaybee */ 2431c717a561Smaybee if (db->db_buf && holds == (db->db_level == 0 ? db->db_dirtycnt : 0)) 24326b4acc8bSahrens arc_buf_freeze(db->db_buf); 24336b4acc8bSahrens 2434fa9e4066Sahrens if (holds == db->db_dirtycnt && 2435c717a561Smaybee db->db_level == 0 && db->db_immediate_evict) 2436fa9e4066Sahrens dbuf_evict_user(db); 2437ea8dc4b6Seschrock 2438ea8dc4b6Seschrock if (holds == 0) { 24390a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 2440cd485b49SJustin T. Gibbs dnode_t *dn; 2441cd485b49SJustin T. Gibbs 2442cd485b49SJustin T. Gibbs /* 2443cd485b49SJustin T. Gibbs * If the dnode moves here, we cannot cross this 2444cd485b49SJustin T. Gibbs * barrier until the move completes. 2445cd485b49SJustin T. Gibbs */ 2446cd485b49SJustin T. Gibbs DB_DNODE_ENTER(db); 2447cd485b49SJustin T. Gibbs 2448cd485b49SJustin T. Gibbs dn = DB_DNODE(db); 2449cd485b49SJustin T. Gibbs atomic_dec_32(&dn->dn_dbufs_count); 2450cd485b49SJustin T. Gibbs 2451cd485b49SJustin T. Gibbs /* 2452cd485b49SJustin T. Gibbs * Decrementing the dbuf count means that the bonus 2453cd485b49SJustin T. Gibbs * buffer's dnode hold is no longer discounted in 2454cd485b49SJustin T. Gibbs * dnode_move(). The dnode cannot move until after 2455cd485b49SJustin T. Gibbs * the dnode_rele_and_unlock() below. 2456cd485b49SJustin T. Gibbs */ 2457cd485b49SJustin T. Gibbs DB_DNODE_EXIT(db); 2458cd485b49SJustin T. Gibbs 2459cd485b49SJustin T. Gibbs /* 2460cd485b49SJustin T. Gibbs * Do not reference db after its lock is dropped. 2461cd485b49SJustin T. Gibbs * Another thread may evict it. 2462cd485b49SJustin T. Gibbs */ 2463ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 2464744947dcSTom Erickson 2465744947dcSTom Erickson /* 2466cd485b49SJustin T. Gibbs * If the dnode has been freed, evict the bonus 2467cd485b49SJustin T. Gibbs * buffer immediately. The data in the bonus 2468cd485b49SJustin T. Gibbs * buffer is no longer relevant and this prevents 2469cd485b49SJustin T. Gibbs * a stale bonus buffer from being associated 2470cd485b49SJustin T. Gibbs * with this dnode_t should the dnode_t be reused 2471cd485b49SJustin T. Gibbs * prior to being destroyed. 2472744947dcSTom Erickson */ 2473cd485b49SJustin T. Gibbs mutex_enter(&dn->dn_mtx); 2474cd485b49SJustin T. Gibbs if (dn->dn_type == DMU_OT_NONE || 2475cd485b49SJustin T. Gibbs dn->dn_free_txg != 0) { 2476744947dcSTom Erickson /* 2477cd485b49SJustin T. Gibbs * Drop dn_mtx. It is a leaf lock and 2478cd485b49SJustin T. Gibbs * cannot be held when dnode_evict_bonus() 2479cd485b49SJustin T. Gibbs * acquires other locks in order to 2480cd485b49SJustin T. Gibbs * perform the eviction. 2481cd485b49SJustin T. Gibbs * 2482cd485b49SJustin T. Gibbs * Freed dnodes cannot be reused until the 2483cd485b49SJustin T. Gibbs * last hold is released. Since this bonus 2484cd485b49SJustin T. Gibbs * buffer has a hold, the dnode will remain 2485cd485b49SJustin T. Gibbs * in the free state, even without dn_mtx 2486cd485b49SJustin T. Gibbs * held, until the dnode_rele_and_unlock() 2487cd485b49SJustin T. Gibbs * below. 2488744947dcSTom Erickson */ 2489cd485b49SJustin T. Gibbs mutex_exit(&dn->dn_mtx); 2490cd485b49SJustin T. Gibbs dnode_evict_bonus(dn); 2491cd485b49SJustin T. Gibbs mutex_enter(&dn->dn_mtx); 2492cd485b49SJustin T. Gibbs } 2493cd485b49SJustin T. Gibbs dnode_rele_and_unlock(dn, db); 2494ea8dc4b6Seschrock } else if (db->db_buf == NULL) { 2495ea8dc4b6Seschrock /* 2496ea8dc4b6Seschrock * This is a special case: we never associated this 2497ea8dc4b6Seschrock * dbuf with any data allocated from the ARC. 2498ea8dc4b6Seschrock */ 249982c9918fSTim Haley ASSERT(db->db_state == DB_UNCACHED || 250082c9918fSTim Haley db->db_state == DB_NOFILL); 2501ea8dc4b6Seschrock dbuf_evict(db); 2502ea8dc4b6Seschrock } else if (arc_released(db->db_buf)) { 2503ea8dc4b6Seschrock arc_buf_t *buf = db->db_buf; 2504ea8dc4b6Seschrock /* 2505ea8dc4b6Seschrock * This dbuf has anonymous data associated with it. 2506ea8dc4b6Seschrock */ 2507bc9014e6SJustin Gibbs dbuf_clear_data(db); 25083b2aab18SMatthew Ahrens VERIFY(arc_buf_remove_ref(buf, db)); 2509ea8dc4b6Seschrock dbuf_evict(db); 2510ea8dc4b6Seschrock } else { 25113b2aab18SMatthew Ahrens VERIFY(!arc_buf_remove_ref(db->db_buf, db)); 25129253d63dSGeorge Wilson 25139253d63dSGeorge Wilson /* 25149253d63dSGeorge Wilson * A dbuf will be eligible for eviction if either the 25159253d63dSGeorge Wilson * 'primarycache' property is set or a duplicate 25169253d63dSGeorge Wilson * copy of this buffer is already cached in the arc. 25179253d63dSGeorge Wilson * 25189253d63dSGeorge Wilson * In the case of the 'primarycache' a buffer 25199253d63dSGeorge Wilson * is considered for eviction if it matches the 25209253d63dSGeorge Wilson * criteria set in the property. 25219253d63dSGeorge Wilson * 25229253d63dSGeorge Wilson * To decide if our buffer is considered a 25239253d63dSGeorge Wilson * duplicate, we must call into the arc to determine 25249253d63dSGeorge Wilson * if multiple buffers are referencing the same 25259253d63dSGeorge Wilson * block on-disk. If so, then we simply evict 25269253d63dSGeorge Wilson * ourselves. 25279253d63dSGeorge Wilson */ 2528bbfa8ea8SMatthew Ahrens if (!DBUF_IS_CACHEABLE(db)) { 2529bbfa8ea8SMatthew Ahrens if (db->db_blkptr != NULL && 2530bbfa8ea8SMatthew Ahrens !BP_IS_HOLE(db->db_blkptr) && 2531bbfa8ea8SMatthew Ahrens !BP_IS_EMBEDDED(db->db_blkptr)) { 2532bbfa8ea8SMatthew Ahrens spa_t *spa = 2533bbfa8ea8SMatthew Ahrens dmu_objset_spa(db->db_objset); 2534bbfa8ea8SMatthew Ahrens blkptr_t bp = *db->db_blkptr; 25353baa08fcSek110237 dbuf_clear(db); 2536bbfa8ea8SMatthew Ahrens arc_freed(spa, &bp); 2537bbfa8ea8SMatthew Ahrens } else { 2538bbfa8ea8SMatthew Ahrens dbuf_clear(db); 2539bbfa8ea8SMatthew Ahrens } 2540bc9014e6SJustin Gibbs } else if (db->db_objset->os_evicting || 2541bc9014e6SJustin Gibbs arc_buf_eviction_needed(db->db_buf)) { 2542bbfa8ea8SMatthew Ahrens dbuf_clear(db); 2543bbfa8ea8SMatthew Ahrens } else { 2544fa9e4066Sahrens mutex_exit(&db->db_mtx); 2545fa9e4066Sahrens } 2546bbfa8ea8SMatthew Ahrens } 2547ea8dc4b6Seschrock } else { 2548ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 2549fa9e4066Sahrens } 2550fa9e4066Sahrens } 2551fa9e4066Sahrens 2552fa9e4066Sahrens #pragma weak dmu_buf_refcount = dbuf_refcount 2553fa9e4066Sahrens uint64_t 2554fa9e4066Sahrens dbuf_refcount(dmu_buf_impl_t *db) 2555fa9e4066Sahrens { 2556fa9e4066Sahrens return (refcount_count(&db->db_holds)); 2557fa9e4066Sahrens } 2558fa9e4066Sahrens 2559fa9e4066Sahrens void * 2560bc9014e6SJustin Gibbs dmu_buf_replace_user(dmu_buf_t *db_fake, dmu_buf_user_t *old_user, 2561bc9014e6SJustin Gibbs dmu_buf_user_t *new_user) 2562fa9e4066Sahrens { 2563bc9014e6SJustin Gibbs dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2564bc9014e6SJustin Gibbs 2565bc9014e6SJustin Gibbs mutex_enter(&db->db_mtx); 2566bc9014e6SJustin Gibbs dbuf_verify_user(db, DBVU_NOT_EVICTING); 2567bc9014e6SJustin Gibbs if (db->db_user == old_user) 2568bc9014e6SJustin Gibbs db->db_user = new_user; 2569bc9014e6SJustin Gibbs else 2570bc9014e6SJustin Gibbs old_user = db->db_user; 2571bc9014e6SJustin Gibbs dbuf_verify_user(db, DBVU_NOT_EVICTING); 2572bc9014e6SJustin Gibbs mutex_exit(&db->db_mtx); 2573bc9014e6SJustin Gibbs 2574bc9014e6SJustin Gibbs return (old_user); 2575fa9e4066Sahrens } 2576fa9e4066Sahrens 2577fa9e4066Sahrens void * 2578bc9014e6SJustin Gibbs dmu_buf_set_user(dmu_buf_t *db_fake, dmu_buf_user_t *user) 2579bc9014e6SJustin Gibbs { 2580bc9014e6SJustin Gibbs return (dmu_buf_replace_user(db_fake, NULL, user)); 2581bc9014e6SJustin Gibbs } 2582bc9014e6SJustin Gibbs 2583bc9014e6SJustin Gibbs void * 2584bc9014e6SJustin Gibbs dmu_buf_set_user_ie(dmu_buf_t *db_fake, dmu_buf_user_t *user) 2585fa9e4066Sahrens { 2586fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2587fa9e4066Sahrens 2588c717a561Smaybee db->db_immediate_evict = TRUE; 2589bc9014e6SJustin Gibbs return (dmu_buf_set_user(db_fake, user)); 2590fa9e4066Sahrens } 2591fa9e4066Sahrens 2592fa9e4066Sahrens void * 2593bc9014e6SJustin Gibbs dmu_buf_remove_user(dmu_buf_t *db_fake, dmu_buf_user_t *user) 2594fa9e4066Sahrens { 2595bc9014e6SJustin Gibbs return (dmu_buf_replace_user(db_fake, user, NULL)); 2596fa9e4066Sahrens } 2597fa9e4066Sahrens 2598fa9e4066Sahrens void * 2599fa9e4066Sahrens dmu_buf_get_user(dmu_buf_t *db_fake) 2600fa9e4066Sahrens { 2601fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2602fa9e4066Sahrens 2603bc9014e6SJustin Gibbs dbuf_verify_user(db, DBVU_NOT_EVICTING); 2604bc9014e6SJustin Gibbs return (db->db_user); 2605bc9014e6SJustin Gibbs } 2606bc9014e6SJustin Gibbs 2607bc9014e6SJustin Gibbs void 2608bc9014e6SJustin Gibbs dmu_buf_user_evict_wait() 2609bc9014e6SJustin Gibbs { 2610bc9014e6SJustin Gibbs taskq_wait(dbu_evict_taskq); 2611fa9e4066Sahrens } 2612fa9e4066Sahrens 26133d692628SSanjeev Bagewadi boolean_t 26143d692628SSanjeev Bagewadi dmu_buf_freeable(dmu_buf_t *dbuf) 26153d692628SSanjeev Bagewadi { 26163d692628SSanjeev Bagewadi boolean_t res = B_FALSE; 26173d692628SSanjeev Bagewadi dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf; 26183d692628SSanjeev Bagewadi 26193d692628SSanjeev Bagewadi if (db->db_blkptr) 26203d692628SSanjeev Bagewadi res = dsl_dataset_block_freeable(db->db_objset->os_dsl_dataset, 2621c7cd2421SGeorge Wilson db->db_blkptr, db->db_blkptr->blk_birth); 26223d692628SSanjeev Bagewadi 26233d692628SSanjeev Bagewadi return (res); 26243d692628SSanjeev Bagewadi } 26253d692628SSanjeev Bagewadi 262680901aeaSGeorge Wilson blkptr_t * 262780901aeaSGeorge Wilson dmu_buf_get_blkptr(dmu_buf_t *db) 262880901aeaSGeorge Wilson { 262980901aeaSGeorge Wilson dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db; 263080901aeaSGeorge Wilson return (dbi->db_blkptr); 263180901aeaSGeorge Wilson } 263280901aeaSGeorge Wilson 2633c717a561Smaybee static void 2634c717a561Smaybee dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db) 2635fa9e4066Sahrens { 2636c717a561Smaybee /* ASSERT(dmu_tx_is_syncing(tx) */ 2637c717a561Smaybee ASSERT(MUTEX_HELD(&db->db_mtx)); 2638c717a561Smaybee 2639c717a561Smaybee if (db->db_blkptr != NULL) 2640c717a561Smaybee return; 2641c717a561Smaybee 26420a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) { 26430a586ceaSMark Shellenbaum db->db_blkptr = &dn->dn_phys->dn_spill; 26440a586ceaSMark Shellenbaum BP_ZERO(db->db_blkptr); 26450a586ceaSMark Shellenbaum return; 26460a586ceaSMark Shellenbaum } 2647c717a561Smaybee if (db->db_level == dn->dn_phys->dn_nlevels-1) { 2648c717a561Smaybee /* 2649c717a561Smaybee * This buffer was allocated at a time when there was 2650c717a561Smaybee * no available blkptrs from the dnode, or it was 2651c717a561Smaybee * inappropriate to hook it in (i.e., nlevels mis-match). 2652c717a561Smaybee */ 2653c717a561Smaybee ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr); 2654c717a561Smaybee ASSERT(db->db_parent == NULL); 2655c717a561Smaybee db->db_parent = dn->dn_dbuf; 2656c717a561Smaybee db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid]; 2657c717a561Smaybee DBUF_VERIFY(db); 2658c717a561Smaybee } else { 2659c717a561Smaybee dmu_buf_impl_t *parent = db->db_parent; 2660c717a561Smaybee int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2661c717a561Smaybee 2662c717a561Smaybee ASSERT(dn->dn_phys->dn_nlevels > 1); 2663c717a561Smaybee if (parent == NULL) { 2664c717a561Smaybee mutex_exit(&db->db_mtx); 2665c717a561Smaybee rw_enter(&dn->dn_struct_rwlock, RW_READER); 2666a2cdcdd2SPaul Dagnelie parent = dbuf_hold_level(dn, db->db_level + 1, 2667a2cdcdd2SPaul Dagnelie db->db_blkid >> epbs, db); 2668c717a561Smaybee rw_exit(&dn->dn_struct_rwlock); 2669c717a561Smaybee mutex_enter(&db->db_mtx); 2670c717a561Smaybee db->db_parent = parent; 2671c717a561Smaybee } 2672c717a561Smaybee db->db_blkptr = (blkptr_t *)parent->db.db_data + 2673c717a561Smaybee (db->db_blkid & ((1ULL << epbs) - 1)); 2674c717a561Smaybee DBUF_VERIFY(db); 2675c717a561Smaybee } 2676c717a561Smaybee } 2677c717a561Smaybee 2678c717a561Smaybee static void 2679c717a561Smaybee dbuf_sync_indirect(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 2680c717a561Smaybee { 2681c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 2682744947dcSTom Erickson dnode_t *dn; 2683c717a561Smaybee zio_t *zio; 2684c717a561Smaybee 2685c717a561Smaybee ASSERT(dmu_tx_is_syncing(tx)); 2686c717a561Smaybee 2687c717a561Smaybee dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 2688c717a561Smaybee 2689c717a561Smaybee mutex_enter(&db->db_mtx); 2690c717a561Smaybee 2691c717a561Smaybee ASSERT(db->db_level > 0); 2692c717a561Smaybee DBUF_VERIFY(db); 2693c717a561Smaybee 26943e30c24aSWill Andrews /* Read the block if it hasn't been read yet. */ 2695c717a561Smaybee if (db->db_buf == NULL) { 2696c717a561Smaybee mutex_exit(&db->db_mtx); 2697c717a561Smaybee (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED); 2698c717a561Smaybee mutex_enter(&db->db_mtx); 2699c717a561Smaybee } 2700c717a561Smaybee ASSERT3U(db->db_state, ==, DB_CACHED); 2701c717a561Smaybee ASSERT(db->db_buf != NULL); 2702c717a561Smaybee 2703744947dcSTom Erickson DB_DNODE_ENTER(db); 2704744947dcSTom Erickson dn = DB_DNODE(db); 27053e30c24aSWill Andrews /* Indirect block size must match what the dnode thinks it is. */ 2706744947dcSTom Erickson ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2707c717a561Smaybee dbuf_check_blkptr(dn, db); 2708744947dcSTom Erickson DB_DNODE_EXIT(db); 2709c717a561Smaybee 27103e30c24aSWill Andrews /* Provide the pending dirty record to child dbufs */ 2711c717a561Smaybee db->db_data_pending = dr; 2712c717a561Smaybee 2713af2c4821Smaybee mutex_exit(&db->db_mtx); 2714088f3894Sahrens dbuf_write(dr, db->db_buf, tx); 2715c717a561Smaybee 2716c717a561Smaybee zio = dr->dr_zio; 2717c717a561Smaybee mutex_enter(&dr->dt.di.dr_mtx); 271846e1baa6SMatthew Ahrens dbuf_sync_list(&dr->dt.di.dr_children, db->db_level - 1, tx); 2719c717a561Smaybee ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 2720c717a561Smaybee mutex_exit(&dr->dt.di.dr_mtx); 2721c717a561Smaybee zio_nowait(zio); 2722c717a561Smaybee } 2723c717a561Smaybee 2724c717a561Smaybee static void 2725c717a561Smaybee dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 2726c717a561Smaybee { 2727c717a561Smaybee arc_buf_t **datap = &dr->dt.dl.dr_data; 2728c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 2729744947dcSTom Erickson dnode_t *dn; 2730744947dcSTom Erickson objset_t *os; 2731c717a561Smaybee uint64_t txg = tx->tx_txg; 2732fa9e4066Sahrens 2733fa9e4066Sahrens ASSERT(dmu_tx_is_syncing(tx)); 2734fa9e4066Sahrens 2735fa9e4066Sahrens dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 2736fa9e4066Sahrens 2737fa9e4066Sahrens mutex_enter(&db->db_mtx); 2738fa9e4066Sahrens /* 2739fa9e4066Sahrens * To be synced, we must be dirtied. But we 2740fa9e4066Sahrens * might have been freed after the dirty. 2741fa9e4066Sahrens */ 2742fa9e4066Sahrens if (db->db_state == DB_UNCACHED) { 2743fa9e4066Sahrens /* This buffer has been freed since it was dirtied */ 2744fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 2745fa9e4066Sahrens } else if (db->db_state == DB_FILL) { 2746fa9e4066Sahrens /* This buffer was freed and is now being re-filled */ 2747c717a561Smaybee ASSERT(db->db.db_data != dr->dt.dl.dr_data); 2748fa9e4066Sahrens } else { 274982c9918fSTim Haley ASSERT(db->db_state == DB_CACHED || db->db_state == DB_NOFILL); 2750fa9e4066Sahrens } 27519c9dc39aSek110237 DBUF_VERIFY(db); 2752fa9e4066Sahrens 2753744947dcSTom Erickson DB_DNODE_ENTER(db); 2754744947dcSTom Erickson dn = DB_DNODE(db); 2755744947dcSTom Erickson 27560a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) { 27570a586ceaSMark Shellenbaum mutex_enter(&dn->dn_mtx); 27580a586ceaSMark Shellenbaum dn->dn_phys->dn_flags |= DNODE_FLAG_SPILL_BLKPTR; 27590a586ceaSMark Shellenbaum mutex_exit(&dn->dn_mtx); 27600a586ceaSMark Shellenbaum } 27610a586ceaSMark Shellenbaum 2762fa9e4066Sahrens /* 2763c717a561Smaybee * If this is a bonus buffer, simply copy the bonus data into the 2764c717a561Smaybee * dnode. It will be written out when the dnode is synced (and it 2765c717a561Smaybee * will be synced, since it must have been dirty for dbuf_sync to 2766c717a561Smaybee * be called). 2767fa9e4066Sahrens */ 27680a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 2769c717a561Smaybee dbuf_dirty_record_t **drp; 27701934e92fSmaybee 2771ea8dc4b6Seschrock ASSERT(*datap != NULL); 2772fb09f5aaSMadhav Suresh ASSERT0(db->db_level); 2773ea8dc4b6Seschrock ASSERT3U(dn->dn_phys->dn_bonuslen, <=, DN_MAX_BONUSLEN); 2774ea8dc4b6Seschrock bcopy(*datap, DN_BONUS(dn->dn_phys), dn->dn_phys->dn_bonuslen); 2775744947dcSTom Erickson DB_DNODE_EXIT(db); 2776744947dcSTom Erickson 27770e8c6158Smaybee if (*datap != db->db.db_data) { 2778ea8dc4b6Seschrock zio_buf_free(*datap, DN_MAX_BONUSLEN); 27795a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 27800e8c6158Smaybee } 2781ea8dc4b6Seschrock db->db_data_pending = NULL; 2782c717a561Smaybee drp = &db->db_last_dirty; 2783c717a561Smaybee while (*drp != dr) 2784c717a561Smaybee drp = &(*drp)->dr_next; 278517f17c2dSbonwick ASSERT(dr->dr_next == NULL); 2786b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 278717f17c2dSbonwick *drp = dr->dr_next; 2788c717a561Smaybee kmem_free(dr, sizeof (dbuf_dirty_record_t)); 2789ea8dc4b6Seschrock ASSERT(db->db_dirtycnt > 0); 2790ea8dc4b6Seschrock db->db_dirtycnt -= 1; 2791b24ab676SJeff Bonwick dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg); 2792ea8dc4b6Seschrock return; 2793ea8dc4b6Seschrock } 2794ea8dc4b6Seschrock 2795744947dcSTom Erickson os = dn->dn_objset; 2796744947dcSTom Erickson 2797c5c6ffa0Smaybee /* 2798f82bfe17Sgw25295 * This function may have dropped the db_mtx lock allowing a dmu_sync 2799f82bfe17Sgw25295 * operation to sneak in. As a result, we need to ensure that we 2800f82bfe17Sgw25295 * don't check the dr_override_state until we have returned from 2801f82bfe17Sgw25295 * dbuf_check_blkptr. 2802f82bfe17Sgw25295 */ 2803f82bfe17Sgw25295 dbuf_check_blkptr(dn, db); 2804f82bfe17Sgw25295 2805f82bfe17Sgw25295 /* 2806744947dcSTom Erickson * If this buffer is in the middle of an immediate write, 2807c717a561Smaybee * wait for the synchronous IO to complete. 2808c5c6ffa0Smaybee */ 2809c717a561Smaybee while (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC) { 2810c5c6ffa0Smaybee ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT); 2811c5c6ffa0Smaybee cv_wait(&db->db_changed, &db->db_mtx); 2812c717a561Smaybee ASSERT(dr->dt.dl.dr_override_state != DR_NOT_OVERRIDDEN); 2813c5c6ffa0Smaybee } 2814c717a561Smaybee 2815ab69d62fSMatthew Ahrens if (db->db_state != DB_NOFILL && 2816ab69d62fSMatthew Ahrens dn->dn_object != DMU_META_DNODE_OBJECT && 2817ab69d62fSMatthew Ahrens refcount_count(&db->db_holds) > 1 && 2818b24ab676SJeff Bonwick dr->dt.dl.dr_override_state != DR_OVERRIDDEN && 2819ab69d62fSMatthew Ahrens *datap == db->db_buf) { 2820fa9e4066Sahrens /* 282182c9918fSTim Haley * If this buffer is currently "in use" (i.e., there 282282c9918fSTim Haley * are active holds and db_data still references it), 282382c9918fSTim Haley * then make a copy before we start the write so that 282482c9918fSTim Haley * any modifications from the open txg will not leak 282582c9918fSTim Haley * into this write. 2826fa9e4066Sahrens * 282782c9918fSTim Haley * NOTE: this copy does not need to be made for 282882c9918fSTim Haley * objects only modified in the syncing context (e.g. 282982c9918fSTim Haley * DNONE_DNODE blocks). 2830fa9e4066Sahrens */ 2831ab69d62fSMatthew Ahrens int blksz = arc_buf_size(*datap); 2832ab69d62fSMatthew Ahrens arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 2833ab69d62fSMatthew Ahrens *datap = arc_buf_alloc(os->os_spa, blksz, db, type); 2834c717a561Smaybee bcopy(db->db.db_data, (*datap)->b_data, blksz); 2835fa9e4066Sahrens } 2836c717a561Smaybee db->db_data_pending = dr; 2837fa9e4066Sahrens 2838fa9e4066Sahrens mutex_exit(&db->db_mtx); 2839fa9e4066Sahrens 2840088f3894Sahrens dbuf_write(dr, *datap, tx); 2841c717a561Smaybee 2842c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 2843744947dcSTom Erickson if (dn->dn_object == DMU_META_DNODE_OBJECT) { 2844c717a561Smaybee list_insert_tail(&dn->dn_dirty_records[txg&TXG_MASK], dr); 2845744947dcSTom Erickson DB_DNODE_EXIT(db); 2846744947dcSTom Erickson } else { 2847744947dcSTom Erickson /* 2848744947dcSTom Erickson * Although zio_nowait() does not "wait for an IO", it does 2849744947dcSTom Erickson * initiate the IO. If this is an empty write it seems plausible 2850744947dcSTom Erickson * that the IO could actually be completed before the nowait 2851744947dcSTom Erickson * returns. We need to DB_DNODE_EXIT() first in case 2852744947dcSTom Erickson * zio_nowait() invalidates the dbuf. 2853744947dcSTom Erickson */ 2854744947dcSTom Erickson DB_DNODE_EXIT(db); 2855c717a561Smaybee zio_nowait(dr->dr_zio); 2856fa9e4066Sahrens } 2857744947dcSTom Erickson } 2858c717a561Smaybee 2859c717a561Smaybee void 286046e1baa6SMatthew Ahrens dbuf_sync_list(list_t *list, int level, dmu_tx_t *tx) 2861c717a561Smaybee { 2862c717a561Smaybee dbuf_dirty_record_t *dr; 2863c717a561Smaybee 2864c717a561Smaybee while (dr = list_head(list)) { 2865c717a561Smaybee if (dr->dr_zio != NULL) { 2866c717a561Smaybee /* 2867c717a561Smaybee * If we find an already initialized zio then we 2868c717a561Smaybee * are processing the meta-dnode, and we have finished. 2869c717a561Smaybee * The dbufs for all dnodes are put back on the list 2870c717a561Smaybee * during processing, so that we can zio_wait() 2871c717a561Smaybee * these IOs after initiating all child IOs. 2872c717a561Smaybee */ 2873c717a561Smaybee ASSERT3U(dr->dr_dbuf->db.db_object, ==, 2874c717a561Smaybee DMU_META_DNODE_OBJECT); 2875c717a561Smaybee break; 2876fa9e4066Sahrens } 287746e1baa6SMatthew Ahrens if (dr->dr_dbuf->db_blkid != DMU_BONUS_BLKID && 287846e1baa6SMatthew Ahrens dr->dr_dbuf->db_blkid != DMU_SPILL_BLKID) { 287946e1baa6SMatthew Ahrens VERIFY3U(dr->dr_dbuf->db_level, ==, level); 288046e1baa6SMatthew Ahrens } 2881c717a561Smaybee list_remove(list, dr); 2882c717a561Smaybee if (dr->dr_dbuf->db_level > 0) 2883c717a561Smaybee dbuf_sync_indirect(dr, tx); 2884c717a561Smaybee else 2885c717a561Smaybee dbuf_sync_leaf(dr, tx); 2886c717a561Smaybee } 2887c717a561Smaybee } 2888c717a561Smaybee 2889fa9e4066Sahrens /* ARGSUSED */ 2890fa9e4066Sahrens static void 2891c717a561Smaybee dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb) 2892fa9e4066Sahrens { 2893fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 2894744947dcSTom Erickson dnode_t *dn; 2895e14bb325SJeff Bonwick blkptr_t *bp = zio->io_bp; 2896c717a561Smaybee blkptr_t *bp_orig = &zio->io_bp_orig; 2897b24ab676SJeff Bonwick spa_t *spa = zio->io_spa; 2898b24ab676SJeff Bonwick int64_t delta; 2899fa9e4066Sahrens uint64_t fill = 0; 2900b24ab676SJeff Bonwick int i; 2901fa9e4066Sahrens 29025d7b4d43SMatthew Ahrens ASSERT3P(db->db_blkptr, ==, bp); 2903e14bb325SJeff Bonwick 2904744947dcSTom Erickson DB_DNODE_ENTER(db); 2905744947dcSTom Erickson dn = DB_DNODE(db); 2906b24ab676SJeff Bonwick delta = bp_get_dsize_sync(spa, bp) - bp_get_dsize_sync(spa, bp_orig); 2907b24ab676SJeff Bonwick dnode_diduse_space(dn, delta - zio->io_prev_space_delta); 2908b24ab676SJeff Bonwick zio->io_prev_space_delta = delta; 2909fa9e4066Sahrens 291043466aaeSMax Grossman if (bp->blk_birth != 0) { 29110a586ceaSMark Shellenbaum ASSERT((db->db_blkid != DMU_SPILL_BLKID && 29120a586ceaSMark Shellenbaum BP_GET_TYPE(bp) == dn->dn_type) || 29130a586ceaSMark Shellenbaum (db->db_blkid == DMU_SPILL_BLKID && 29145d7b4d43SMatthew Ahrens BP_GET_TYPE(bp) == dn->dn_bonustype) || 29155d7b4d43SMatthew Ahrens BP_IS_EMBEDDED(bp)); 2916e14bb325SJeff Bonwick ASSERT(BP_GET_LEVEL(bp) == db->db_level); 291743466aaeSMax Grossman } 2918e14bb325SJeff Bonwick 2919fa9e4066Sahrens mutex_enter(&db->db_mtx); 2920fa9e4066Sahrens 29210a586ceaSMark Shellenbaum #ifdef ZFS_DEBUG 29220a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) { 29230a586ceaSMark Shellenbaum ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); 29240a586ceaSMark Shellenbaum ASSERT(!(BP_IS_HOLE(db->db_blkptr)) && 29250a586ceaSMark Shellenbaum db->db_blkptr == &dn->dn_phys->dn_spill); 29260a586ceaSMark Shellenbaum } 29270a586ceaSMark Shellenbaum #endif 29280a586ceaSMark Shellenbaum 2929fa9e4066Sahrens if (db->db_level == 0) { 2930fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 29310a586ceaSMark Shellenbaum if (db->db_blkid > dn->dn_phys->dn_maxblkid && 29320a586ceaSMark Shellenbaum db->db_blkid != DMU_SPILL_BLKID) 2933fa9e4066Sahrens dn->dn_phys->dn_maxblkid = db->db_blkid; 2934fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 2935fa9e4066Sahrens 2936fa9e4066Sahrens if (dn->dn_type == DMU_OT_DNODE) { 2937fa9e4066Sahrens dnode_phys_t *dnp = db->db.db_data; 2938fa9e4066Sahrens for (i = db->db.db_size >> DNODE_SHIFT; i > 0; 2939fa9e4066Sahrens i--, dnp++) { 2940fa9e4066Sahrens if (dnp->dn_type != DMU_OT_NONE) 2941fa9e4066Sahrens fill++; 2942fa9e4066Sahrens } 2943fa9e4066Sahrens } else { 294443466aaeSMax Grossman if (BP_IS_HOLE(bp)) { 294543466aaeSMax Grossman fill = 0; 294643466aaeSMax Grossman } else { 2947fa9e4066Sahrens fill = 1; 2948fa9e4066Sahrens } 294943466aaeSMax Grossman } 2950fa9e4066Sahrens } else { 2951e14bb325SJeff Bonwick blkptr_t *ibp = db->db.db_data; 2952fa9e4066Sahrens ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2953e14bb325SJeff Bonwick for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, ibp++) { 2954e14bb325SJeff Bonwick if (BP_IS_HOLE(ibp)) 2955fa9e4066Sahrens continue; 29565d7b4d43SMatthew Ahrens fill += BP_GET_FILL(ibp); 2957fa9e4066Sahrens } 2958fa9e4066Sahrens } 2959744947dcSTom Erickson DB_DNODE_EXIT(db); 2960fa9e4066Sahrens 29615d7b4d43SMatthew Ahrens if (!BP_IS_EMBEDDED(bp)) 2962e14bb325SJeff Bonwick bp->blk_fill = fill; 2963fa9e4066Sahrens 2964fa9e4066Sahrens mutex_exit(&db->db_mtx); 2965fa9e4066Sahrens } 2966fa9e4066Sahrens 296769962b56SMatthew Ahrens /* 296869962b56SMatthew Ahrens * The SPA will call this callback several times for each zio - once 296969962b56SMatthew Ahrens * for every physical child i/o (zio->io_phys_children times). This 297069962b56SMatthew Ahrens * allows the DMU to monitor the progress of each logical i/o. For example, 297169962b56SMatthew Ahrens * there may be 2 copies of an indirect block, or many fragments of a RAID-Z 297269962b56SMatthew Ahrens * block. There may be a long delay before all copies/fragments are completed, 297369962b56SMatthew Ahrens * so this callback allows us to retire dirty space gradually, as the physical 297469962b56SMatthew Ahrens * i/os complete. 297569962b56SMatthew Ahrens */ 297669962b56SMatthew Ahrens /* ARGSUSED */ 297769962b56SMatthew Ahrens static void 297869962b56SMatthew Ahrens dbuf_write_physdone(zio_t *zio, arc_buf_t *buf, void *arg) 297969962b56SMatthew Ahrens { 298069962b56SMatthew Ahrens dmu_buf_impl_t *db = arg; 298169962b56SMatthew Ahrens objset_t *os = db->db_objset; 298269962b56SMatthew Ahrens dsl_pool_t *dp = dmu_objset_pool(os); 298369962b56SMatthew Ahrens dbuf_dirty_record_t *dr; 298469962b56SMatthew Ahrens int delta = 0; 298569962b56SMatthew Ahrens 298669962b56SMatthew Ahrens dr = db->db_data_pending; 298769962b56SMatthew Ahrens ASSERT3U(dr->dr_txg, ==, zio->io_txg); 298869962b56SMatthew Ahrens 298969962b56SMatthew Ahrens /* 299069962b56SMatthew Ahrens * The callback will be called io_phys_children times. Retire one 299169962b56SMatthew Ahrens * portion of our dirty space each time we are called. Any rounding 299269962b56SMatthew Ahrens * error will be cleaned up by dsl_pool_sync()'s call to 299369962b56SMatthew Ahrens * dsl_pool_undirty_space(). 299469962b56SMatthew Ahrens */ 299569962b56SMatthew Ahrens delta = dr->dr_accounted / zio->io_phys_children; 299669962b56SMatthew Ahrens dsl_pool_undirty_space(dp, delta, zio->io_txg); 299769962b56SMatthew Ahrens } 299869962b56SMatthew Ahrens 2999c717a561Smaybee /* ARGSUSED */ 3000c717a561Smaybee static void 3001c717a561Smaybee dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb) 3002c717a561Smaybee { 3003c717a561Smaybee dmu_buf_impl_t *db = vdb; 3004b24ab676SJeff Bonwick blkptr_t *bp_orig = &zio->io_bp_orig; 300543466aaeSMax Grossman blkptr_t *bp = db->db_blkptr; 300643466aaeSMax Grossman objset_t *os = db->db_objset; 300743466aaeSMax Grossman dmu_tx_t *tx = os->os_synctx; 3008c717a561Smaybee dbuf_dirty_record_t **drp, *dr; 3009c717a561Smaybee 3010fb09f5aaSMadhav Suresh ASSERT0(zio->io_error); 3011b24ab676SJeff Bonwick ASSERT(db->db_blkptr == bp); 3012b24ab676SJeff Bonwick 301380901aeaSGeorge Wilson /* 301480901aeaSGeorge Wilson * For nopwrites and rewrites we ensure that the bp matches our 301580901aeaSGeorge Wilson * original and bypass all the accounting. 301680901aeaSGeorge Wilson */ 301780901aeaSGeorge Wilson if (zio->io_flags & (ZIO_FLAG_IO_REWRITE | ZIO_FLAG_NOPWRITE)) { 3018b24ab676SJeff Bonwick ASSERT(BP_EQUAL(bp, bp_orig)); 3019b24ab676SJeff Bonwick } else { 302043466aaeSMax Grossman dsl_dataset_t *ds = os->os_dsl_dataset; 3021b24ab676SJeff Bonwick (void) dsl_dataset_block_kill(ds, bp_orig, tx, B_TRUE); 3022b24ab676SJeff Bonwick dsl_dataset_block_born(ds, bp, tx); 3023b24ab676SJeff Bonwick } 3024c717a561Smaybee 3025c717a561Smaybee mutex_enter(&db->db_mtx); 3026c717a561Smaybee 3027b24ab676SJeff Bonwick DBUF_VERIFY(db); 3028b24ab676SJeff Bonwick 3029c717a561Smaybee drp = &db->db_last_dirty; 303017f17c2dSbonwick while ((dr = *drp) != db->db_data_pending) 303117f17c2dSbonwick drp = &dr->dr_next; 303217f17c2dSbonwick ASSERT(!list_link_active(&dr->dr_dirty_node)); 3033b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 303417f17c2dSbonwick ASSERT(dr->dr_next == NULL); 303517f17c2dSbonwick *drp = dr->dr_next; 3036c717a561Smaybee 30370a586ceaSMark Shellenbaum #ifdef ZFS_DEBUG 30380a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) { 3039744947dcSTom Erickson dnode_t *dn; 3040744947dcSTom Erickson 3041744947dcSTom Erickson DB_DNODE_ENTER(db); 3042744947dcSTom Erickson dn = DB_DNODE(db); 30430a586ceaSMark Shellenbaum ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); 30440a586ceaSMark Shellenbaum ASSERT(!(BP_IS_HOLE(db->db_blkptr)) && 30450a586ceaSMark Shellenbaum db->db_blkptr == &dn->dn_phys->dn_spill); 3046744947dcSTom Erickson DB_DNODE_EXIT(db); 30470a586ceaSMark Shellenbaum } 30480a586ceaSMark Shellenbaum #endif 30490a586ceaSMark Shellenbaum 3050c717a561Smaybee if (db->db_level == 0) { 30510a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 3052c717a561Smaybee ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN); 305382c9918fSTim Haley if (db->db_state != DB_NOFILL) { 3054c717a561Smaybee if (dr->dt.dl.dr_data != db->db_buf) 305582c9918fSTim Haley VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, 30563b2aab18SMatthew Ahrens db)); 3057b24ab676SJeff Bonwick else if (!arc_released(db->db_buf)) 3058c717a561Smaybee arc_set_callback(db->db_buf, dbuf_do_evict, db); 305982c9918fSTim Haley } 3060c717a561Smaybee } else { 3061744947dcSTom Erickson dnode_t *dn; 3062744947dcSTom Erickson 3063744947dcSTom Erickson DB_DNODE_ENTER(db); 3064744947dcSTom Erickson dn = DB_DNODE(db); 3065c717a561Smaybee ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 3066c717a561Smaybee ASSERT3U(db->db.db_size, ==, 1 << dn->dn_phys->dn_indblkshift); 3067c717a561Smaybee if (!BP_IS_HOLE(db->db_blkptr)) { 3068c717a561Smaybee int epbs = 3069c717a561Smaybee dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 307043466aaeSMax Grossman ASSERT3U(db->db_blkid, <=, 307143466aaeSMax Grossman dn->dn_phys->dn_maxblkid >> (db->db_level * epbs)); 3072c717a561Smaybee ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, 3073c717a561Smaybee db->db.db_size); 30745d7b4d43SMatthew Ahrens if (!arc_released(db->db_buf)) 3075c717a561Smaybee arc_set_callback(db->db_buf, dbuf_do_evict, db); 3076c717a561Smaybee } 3077744947dcSTom Erickson DB_DNODE_EXIT(db); 3078c25056deSgw25295 mutex_destroy(&dr->dt.di.dr_mtx); 3079c25056deSgw25295 list_destroy(&dr->dt.di.dr_children); 3080c717a561Smaybee } 3081c717a561Smaybee kmem_free(dr, sizeof (dbuf_dirty_record_t)); 3082c717a561Smaybee 3083c717a561Smaybee cv_broadcast(&db->db_changed); 3084c717a561Smaybee ASSERT(db->db_dirtycnt > 0); 3085c717a561Smaybee db->db_dirtycnt -= 1; 3086c717a561Smaybee db->db_data_pending = NULL; 308743466aaeSMax Grossman dbuf_rele_and_unlock(db, (void *)(uintptr_t)tx->tx_txg); 3088b24ab676SJeff Bonwick } 3089b24ab676SJeff Bonwick 3090b24ab676SJeff Bonwick static void 3091b24ab676SJeff Bonwick dbuf_write_nofill_ready(zio_t *zio) 3092b24ab676SJeff Bonwick { 3093b24ab676SJeff Bonwick dbuf_write_ready(zio, NULL, zio->io_private); 3094b24ab676SJeff Bonwick } 3095b24ab676SJeff Bonwick 3096b24ab676SJeff Bonwick static void 3097b24ab676SJeff Bonwick dbuf_write_nofill_done(zio_t *zio) 3098b24ab676SJeff Bonwick { 3099b24ab676SJeff Bonwick dbuf_write_done(zio, NULL, zio->io_private); 3100b24ab676SJeff Bonwick } 3101b24ab676SJeff Bonwick 3102b24ab676SJeff Bonwick static void 3103b24ab676SJeff Bonwick dbuf_write_override_ready(zio_t *zio) 3104b24ab676SJeff Bonwick { 3105b24ab676SJeff Bonwick dbuf_dirty_record_t *dr = zio->io_private; 3106b24ab676SJeff Bonwick dmu_buf_impl_t *db = dr->dr_dbuf; 3107b24ab676SJeff Bonwick 3108b24ab676SJeff Bonwick dbuf_write_ready(zio, NULL, db); 3109b24ab676SJeff Bonwick } 3110b24ab676SJeff Bonwick 3111b24ab676SJeff Bonwick static void 3112b24ab676SJeff Bonwick dbuf_write_override_done(zio_t *zio) 3113b24ab676SJeff Bonwick { 3114b24ab676SJeff Bonwick dbuf_dirty_record_t *dr = zio->io_private; 3115b24ab676SJeff Bonwick dmu_buf_impl_t *db = dr->dr_dbuf; 3116b24ab676SJeff Bonwick blkptr_t *obp = &dr->dt.dl.dr_overridden_by; 3117b24ab676SJeff Bonwick 3118b24ab676SJeff Bonwick mutex_enter(&db->db_mtx); 3119b24ab676SJeff Bonwick if (!BP_EQUAL(zio->io_bp, obp)) { 3120b24ab676SJeff Bonwick if (!BP_IS_HOLE(obp)) 3121b24ab676SJeff Bonwick dsl_free(spa_get_dsl(zio->io_spa), zio->io_txg, obp); 3122b24ab676SJeff Bonwick arc_release(dr->dt.dl.dr_data, db); 3123b24ab676SJeff Bonwick } 3124c717a561Smaybee mutex_exit(&db->db_mtx); 3125c717a561Smaybee 3126b24ab676SJeff Bonwick dbuf_write_done(zio, NULL, db); 3127b24ab676SJeff Bonwick } 3128c717a561Smaybee 31293e30c24aSWill Andrews /* Issue I/O to commit a dirty buffer to disk. */ 3130b24ab676SJeff Bonwick static void 3131b24ab676SJeff Bonwick dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx) 3132b24ab676SJeff Bonwick { 3133b24ab676SJeff Bonwick dmu_buf_impl_t *db = dr->dr_dbuf; 3134744947dcSTom Erickson dnode_t *dn; 3135744947dcSTom Erickson objset_t *os; 3136b24ab676SJeff Bonwick dmu_buf_impl_t *parent = db->db_parent; 3137b24ab676SJeff Bonwick uint64_t txg = tx->tx_txg; 31387802d7bfSMatthew Ahrens zbookmark_phys_t zb; 3139b24ab676SJeff Bonwick zio_prop_t zp; 3140b24ab676SJeff Bonwick zio_t *zio; 31410a586ceaSMark Shellenbaum int wp_flag = 0; 3142b24ab676SJeff Bonwick 3143744947dcSTom Erickson DB_DNODE_ENTER(db); 3144744947dcSTom Erickson dn = DB_DNODE(db); 3145744947dcSTom Erickson os = dn->dn_objset; 3146744947dcSTom Erickson 3147b24ab676SJeff Bonwick if (db->db_state != DB_NOFILL) { 3148b24ab676SJeff Bonwick if (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE) { 3149b24ab676SJeff Bonwick /* 3150b24ab676SJeff Bonwick * Private object buffers are released here rather 3151b24ab676SJeff Bonwick * than in dbuf_dirty() since they are only modified 3152b24ab676SJeff Bonwick * in the syncing context and we don't want the 3153b24ab676SJeff Bonwick * overhead of making multiple copies of the data. 3154b24ab676SJeff Bonwick */ 3155b24ab676SJeff Bonwick if (BP_IS_HOLE(db->db_blkptr)) { 3156b24ab676SJeff Bonwick arc_buf_thaw(data); 3157b24ab676SJeff Bonwick } else { 31583f9d6ad7SLin Ling dbuf_release_bp(db); 3159b24ab676SJeff Bonwick } 3160b24ab676SJeff Bonwick } 3161b24ab676SJeff Bonwick } 3162b24ab676SJeff Bonwick 3163b24ab676SJeff Bonwick if (parent != dn->dn_dbuf) { 31643e30c24aSWill Andrews /* Our parent is an indirect block. */ 31653e30c24aSWill Andrews /* We have a dirty parent that has been scheduled for write. */ 3166b24ab676SJeff Bonwick ASSERT(parent && parent->db_data_pending); 31673e30c24aSWill Andrews /* Our parent's buffer is one level closer to the dnode. */ 3168b24ab676SJeff Bonwick ASSERT(db->db_level == parent->db_level-1); 31693e30c24aSWill Andrews /* 31703e30c24aSWill Andrews * We're about to modify our parent's db_data by modifying 31713e30c24aSWill Andrews * our block pointer, so the parent must be released. 31723e30c24aSWill Andrews */ 3173b24ab676SJeff Bonwick ASSERT(arc_released(parent->db_buf)); 3174b24ab676SJeff Bonwick zio = parent->db_data_pending->dr_zio; 3175b24ab676SJeff Bonwick } else { 31763e30c24aSWill Andrews /* Our parent is the dnode itself. */ 31770a586ceaSMark Shellenbaum ASSERT((db->db_level == dn->dn_phys->dn_nlevels-1 && 31780a586ceaSMark Shellenbaum db->db_blkid != DMU_SPILL_BLKID) || 31790a586ceaSMark Shellenbaum (db->db_blkid == DMU_SPILL_BLKID && db->db_level == 0)); 31800a586ceaSMark Shellenbaum if (db->db_blkid != DMU_SPILL_BLKID) 3181b24ab676SJeff Bonwick ASSERT3P(db->db_blkptr, ==, 3182b24ab676SJeff Bonwick &dn->dn_phys->dn_blkptr[db->db_blkid]); 3183b24ab676SJeff Bonwick zio = dn->dn_zio; 3184b24ab676SJeff Bonwick } 3185b24ab676SJeff Bonwick 3186b24ab676SJeff Bonwick ASSERT(db->db_level == 0 || data == db->db_buf); 3187b24ab676SJeff Bonwick ASSERT3U(db->db_blkptr->blk_birth, <=, txg); 3188b24ab676SJeff Bonwick ASSERT(zio); 3189b24ab676SJeff Bonwick 3190b24ab676SJeff Bonwick SET_BOOKMARK(&zb, os->os_dsl_dataset ? 3191b24ab676SJeff Bonwick os->os_dsl_dataset->ds_object : DMU_META_OBJSET, 3192b24ab676SJeff Bonwick db->db.db_object, db->db_level, db->db_blkid); 3193b24ab676SJeff Bonwick 31940a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) 31950a586ceaSMark Shellenbaum wp_flag = WP_SPILL; 31960a586ceaSMark Shellenbaum wp_flag |= (db->db_state == DB_NOFILL) ? WP_NOFILL : 0; 31970a586ceaSMark Shellenbaum 31980a586ceaSMark Shellenbaum dmu_write_policy(os, dn, db->db_level, wp_flag, &zp); 3199744947dcSTom Erickson DB_DNODE_EXIT(db); 3200b24ab676SJeff Bonwick 32015d7b4d43SMatthew Ahrens if (db->db_level == 0 && 32025d7b4d43SMatthew Ahrens dr->dt.dl.dr_override_state == DR_OVERRIDDEN) { 32035d7b4d43SMatthew Ahrens /* 32045d7b4d43SMatthew Ahrens * The BP for this block has been provided by open context 32055d7b4d43SMatthew Ahrens * (by dmu_sync() or dmu_buf_write_embedded()). 32065d7b4d43SMatthew Ahrens */ 32075d7b4d43SMatthew Ahrens void *contents = (data != NULL) ? data->b_data : NULL; 32085d7b4d43SMatthew Ahrens 3209b24ab676SJeff Bonwick dr->dr_zio = zio_write(zio, os->os_spa, txg, 32105d7b4d43SMatthew Ahrens db->db_blkptr, contents, db->db.db_size, &zp, 321169962b56SMatthew Ahrens dbuf_write_override_ready, NULL, dbuf_write_override_done, 321269962b56SMatthew Ahrens dr, ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); 3213b24ab676SJeff Bonwick mutex_enter(&db->db_mtx); 3214b24ab676SJeff Bonwick dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 3215b24ab676SJeff Bonwick zio_write_override(dr->dr_zio, &dr->dt.dl.dr_overridden_by, 321680901aeaSGeorge Wilson dr->dt.dl.dr_copies, dr->dt.dl.dr_nopwrite); 3217b24ab676SJeff Bonwick mutex_exit(&db->db_mtx); 3218b24ab676SJeff Bonwick } else if (db->db_state == DB_NOFILL) { 3219810e43b2SBill Pijewski ASSERT(zp.zp_checksum == ZIO_CHECKSUM_OFF || 3220810e43b2SBill Pijewski zp.zp_checksum == ZIO_CHECKSUM_NOPARITY); 3221b24ab676SJeff Bonwick dr->dr_zio = zio_write(zio, os->os_spa, txg, 3222b24ab676SJeff Bonwick db->db_blkptr, NULL, db->db.db_size, &zp, 322369962b56SMatthew Ahrens dbuf_write_nofill_ready, NULL, dbuf_write_nofill_done, db, 3224b24ab676SJeff Bonwick ZIO_PRIORITY_ASYNC_WRITE, 3225b24ab676SJeff Bonwick ZIO_FLAG_MUSTSUCCEED | ZIO_FLAG_NODATA, &zb); 3226b24ab676SJeff Bonwick } else { 3227b24ab676SJeff Bonwick ASSERT(arc_released(data)); 3228b24ab676SJeff Bonwick dr->dr_zio = arc_write(zio, os->os_spa, txg, 3229aad02571SSaso Kiselkov db->db_blkptr, data, DBUF_IS_L2CACHEABLE(db), 3230aad02571SSaso Kiselkov DBUF_IS_L2COMPRESSIBLE(db), &zp, dbuf_write_ready, 323169962b56SMatthew Ahrens dbuf_write_physdone, dbuf_write_done, db, 323269962b56SMatthew Ahrens ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); 3233b24ab676SJeff Bonwick } 3234fa9e4066Sahrens } 3235