1fa9e4066Sahrens /* 2fa9e4066Sahrens * CDDL HEADER START 3fa9e4066Sahrens * 4fa9e4066Sahrens * The contents of this file are subject to the terms of the 5f65e61c0Sahrens * Common Development and Distribution License (the "License"). 6f65e61c0Sahrens * You may not use this file except in compliance with the License. 7fa9e4066Sahrens * 8fa9e4066Sahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9fa9e4066Sahrens * or http://www.opensolaris.org/os/licensing. 10fa9e4066Sahrens * See the License for the specific language governing permissions 11fa9e4066Sahrens * and limitations under the License. 12fa9e4066Sahrens * 13fa9e4066Sahrens * When distributing Covered Code, include this CDDL HEADER in each 14fa9e4066Sahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15fa9e4066Sahrens * If applicable, add the following below this CDDL HEADER, with the 16fa9e4066Sahrens * fields enclosed by brackets "[]" replaced with your own identifying 17fa9e4066Sahrens * information: Portions Copyright [yyyy] [name of copyright owner] 18fa9e4066Sahrens * 19fa9e4066Sahrens * CDDL HEADER END 20fa9e4066Sahrens */ 21fa9e4066Sahrens /* 2206e0070dSMark Shellenbaum * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 233f2366c2SGordon Ross * Copyright 2011 Nexenta Systems, Inc. All rights reserved. 2446e1baa6SMatthew Ahrens * Copyright (c) 2012, 2015 by Delphix. All rights reserved. 25aad02571SSaso Kiselkov * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. 26810e43b2SBill Pijewski * Copyright (c) 2013, Joyent, Inc. All rights reserved. 27bc9014e6SJustin Gibbs * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. 28c3d26abcSMatthew Ahrens * Copyright (c) 2014 Integros [integros.com] 29fa9e4066Sahrens */ 30fa9e4066Sahrens 31fa9e4066Sahrens #include <sys/zfs_context.h> 32fa9e4066Sahrens #include <sys/dmu.h> 332f3d8780SMatthew Ahrens #include <sys/dmu_send.h> 34fa9e4066Sahrens #include <sys/dmu_impl.h> 35fa9e4066Sahrens #include <sys/dbuf.h> 36fa9e4066Sahrens #include <sys/dmu_objset.h> 37fa9e4066Sahrens #include <sys/dsl_dataset.h> 38fa9e4066Sahrens #include <sys/dsl_dir.h> 39fa9e4066Sahrens #include <sys/dmu_tx.h> 40fa9e4066Sahrens #include <sys/spa.h> 41fa9e4066Sahrens #include <sys/zio.h> 42fa9e4066Sahrens #include <sys/dmu_zfetch.h> 430a586ceaSMark Shellenbaum #include <sys/sa.h> 440a586ceaSMark Shellenbaum #include <sys/sa_impl.h> 455d7b4d43SMatthew Ahrens #include <sys/zfeature.h> 465d7b4d43SMatthew Ahrens #include <sys/blkptr.h> 47bf16b11eSMatthew Ahrens #include <sys/range_tree.h> 48*dcbf3bd6SGeorge Wilson #include <sys/callb.h> 49*dcbf3bd6SGeorge Wilson 50*dcbf3bd6SGeorge Wilson uint_t zfs_dbuf_evict_key; 51fa9e4066Sahrens 52713d6c20SMatthew Ahrens /* 53713d6c20SMatthew Ahrens * Number of times that zfs_free_range() took the slow path while doing 54713d6c20SMatthew Ahrens * a zfs receive. A nonzero value indicates a potential performance problem. 55713d6c20SMatthew Ahrens */ 56713d6c20SMatthew Ahrens uint64_t zfs_free_range_recv_miss; 57713d6c20SMatthew Ahrens 583b2aab18SMatthew Ahrens static boolean_t dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx); 59088f3894Sahrens static void dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx); 60fa9e4066Sahrens 61bc9014e6SJustin Gibbs #ifndef __lint 62bc9014e6SJustin Gibbs extern inline void dmu_buf_init_user(dmu_buf_user_t *dbu, 63bc9014e6SJustin Gibbs dmu_buf_evict_func_t *evict_func, dmu_buf_t **clear_on_evict_dbufp); 64bc9014e6SJustin Gibbs #endif /* ! __lint */ 65bc9014e6SJustin Gibbs 66fa9e4066Sahrens /* 67fa9e4066Sahrens * Global data structures and functions for the dbuf cache. 68fa9e4066Sahrens */ 69*dcbf3bd6SGeorge Wilson static kmem_cache_t *dbuf_kmem_cache; 70bc9014e6SJustin Gibbs static taskq_t *dbu_evict_taskq; 71fa9e4066Sahrens 72*dcbf3bd6SGeorge Wilson static kthread_t *dbuf_cache_evict_thread; 73*dcbf3bd6SGeorge Wilson static kmutex_t dbuf_evict_lock; 74*dcbf3bd6SGeorge Wilson static kcondvar_t dbuf_evict_cv; 75*dcbf3bd6SGeorge Wilson static boolean_t dbuf_evict_thread_exit; 76*dcbf3bd6SGeorge Wilson 77*dcbf3bd6SGeorge Wilson /* 78*dcbf3bd6SGeorge Wilson * LRU cache of dbufs. The dbuf cache maintains a list of dbufs that 79*dcbf3bd6SGeorge Wilson * are not currently held but have been recently released. These dbufs 80*dcbf3bd6SGeorge Wilson * are not eligible for arc eviction until they are aged out of the cache. 81*dcbf3bd6SGeorge Wilson * Dbufs are added to the dbuf cache once the last hold is released. If a 82*dcbf3bd6SGeorge Wilson * dbuf is later accessed and still exists in the dbuf cache, then it will 83*dcbf3bd6SGeorge Wilson * be removed from the cache and later re-added to the head of the cache. 84*dcbf3bd6SGeorge Wilson * Dbufs that are aged out of the cache will be immediately destroyed and 85*dcbf3bd6SGeorge Wilson * become eligible for arc eviction. 86*dcbf3bd6SGeorge Wilson */ 87*dcbf3bd6SGeorge Wilson static multilist_t dbuf_cache; 88*dcbf3bd6SGeorge Wilson static refcount_t dbuf_cache_size; 89*dcbf3bd6SGeorge Wilson uint64_t dbuf_cache_max_bytes = 100 * 1024 * 1024; 90*dcbf3bd6SGeorge Wilson 91*dcbf3bd6SGeorge Wilson /* Cap the size of the dbuf cache to log2 fraction of arc size. */ 92*dcbf3bd6SGeorge Wilson int dbuf_cache_max_shift = 5; 93*dcbf3bd6SGeorge Wilson 94*dcbf3bd6SGeorge Wilson /* 95*dcbf3bd6SGeorge Wilson * The dbuf cache uses a three-stage eviction policy: 96*dcbf3bd6SGeorge Wilson * - A low water marker designates when the dbuf eviction thread 97*dcbf3bd6SGeorge Wilson * should stop evicting from the dbuf cache. 98*dcbf3bd6SGeorge Wilson * - When we reach the maximum size (aka mid water mark), we 99*dcbf3bd6SGeorge Wilson * signal the eviction thread to run. 100*dcbf3bd6SGeorge Wilson * - The high water mark indicates when the eviction thread 101*dcbf3bd6SGeorge Wilson * is unable to keep up with the incoming load and eviction must 102*dcbf3bd6SGeorge Wilson * happen in the context of the calling thread. 103*dcbf3bd6SGeorge Wilson * 104*dcbf3bd6SGeorge Wilson * The dbuf cache: 105*dcbf3bd6SGeorge Wilson * (max size) 106*dcbf3bd6SGeorge Wilson * low water mid water hi water 107*dcbf3bd6SGeorge Wilson * +----------------------------------------+----------+----------+ 108*dcbf3bd6SGeorge Wilson * | | | | 109*dcbf3bd6SGeorge Wilson * | | | | 110*dcbf3bd6SGeorge Wilson * | | | | 111*dcbf3bd6SGeorge Wilson * | | | | 112*dcbf3bd6SGeorge Wilson * +----------------------------------------+----------+----------+ 113*dcbf3bd6SGeorge Wilson * stop signal evict 114*dcbf3bd6SGeorge Wilson * evicting eviction directly 115*dcbf3bd6SGeorge Wilson * thread 116*dcbf3bd6SGeorge Wilson * 117*dcbf3bd6SGeorge Wilson * The high and low water marks indicate the operating range for the eviction 118*dcbf3bd6SGeorge Wilson * thread. The low water mark is, by default, 90% of the total size of the 119*dcbf3bd6SGeorge Wilson * cache and the high water mark is at 110% (both of these percentages can be 120*dcbf3bd6SGeorge Wilson * changed by setting dbuf_cache_lowater_pct and dbuf_cache_hiwater_pct, 121*dcbf3bd6SGeorge Wilson * respectively). The eviction thread will try to ensure that the cache remains 122*dcbf3bd6SGeorge Wilson * within this range by waking up every second and checking if the cache is 123*dcbf3bd6SGeorge Wilson * above the low water mark. The thread can also be woken up by callers adding 124*dcbf3bd6SGeorge Wilson * elements into the cache if the cache is larger than the mid water (i.e max 125*dcbf3bd6SGeorge Wilson * cache size). Once the eviction thread is woken up and eviction is required, 126*dcbf3bd6SGeorge Wilson * it will continue evicting buffers until it's able to reduce the cache size 127*dcbf3bd6SGeorge Wilson * to the low water mark. If the cache size continues to grow and hits the high 128*dcbf3bd6SGeorge Wilson * water mark, then callers adding elments to the cache will begin to evict 129*dcbf3bd6SGeorge Wilson * directly from the cache until the cache is no longer above the high water 130*dcbf3bd6SGeorge Wilson * mark. 131*dcbf3bd6SGeorge Wilson */ 132*dcbf3bd6SGeorge Wilson 133*dcbf3bd6SGeorge Wilson /* 134*dcbf3bd6SGeorge Wilson * The percentage above and below the maximum cache size. 135*dcbf3bd6SGeorge Wilson */ 136*dcbf3bd6SGeorge Wilson uint_t dbuf_cache_hiwater_pct = 10; 137*dcbf3bd6SGeorge Wilson uint_t dbuf_cache_lowater_pct = 10; 138*dcbf3bd6SGeorge Wilson 139fa9e4066Sahrens /* ARGSUSED */ 140fa9e4066Sahrens static int 141fa9e4066Sahrens dbuf_cons(void *vdb, void *unused, int kmflag) 142fa9e4066Sahrens { 143fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 144fa9e4066Sahrens bzero(db, sizeof (dmu_buf_impl_t)); 145fa9e4066Sahrens 146fa9e4066Sahrens mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL); 147fa9e4066Sahrens cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL); 148*dcbf3bd6SGeorge Wilson multilist_link_init(&db->db_cache_link); 149fa9e4066Sahrens refcount_create(&db->db_holds); 1500f6d88adSAlex Reece 151fa9e4066Sahrens return (0); 152fa9e4066Sahrens } 153fa9e4066Sahrens 154fa9e4066Sahrens /* ARGSUSED */ 155fa9e4066Sahrens static void 156fa9e4066Sahrens dbuf_dest(void *vdb, void *unused) 157fa9e4066Sahrens { 158fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 159fa9e4066Sahrens mutex_destroy(&db->db_mtx); 160fa9e4066Sahrens cv_destroy(&db->db_changed); 161*dcbf3bd6SGeorge Wilson ASSERT(!multilist_link_active(&db->db_cache_link)); 162fa9e4066Sahrens refcount_destroy(&db->db_holds); 163fa9e4066Sahrens } 164fa9e4066Sahrens 165fa9e4066Sahrens /* 166fa9e4066Sahrens * dbuf hash table routines 167fa9e4066Sahrens */ 168fa9e4066Sahrens static dbuf_hash_table_t dbuf_hash_table; 169fa9e4066Sahrens 170fa9e4066Sahrens static uint64_t dbuf_hash_count; 171fa9e4066Sahrens 172fa9e4066Sahrens static uint64_t 173fa9e4066Sahrens dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid) 174fa9e4066Sahrens { 175fa9e4066Sahrens uintptr_t osv = (uintptr_t)os; 176fa9e4066Sahrens uint64_t crc = -1ULL; 177fa9e4066Sahrens 178fa9e4066Sahrens ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 179fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (lvl)) & 0xFF]; 180fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF]; 181fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF]; 182fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF]; 183fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 0)) & 0xFF]; 184fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 8)) & 0xFF]; 185fa9e4066Sahrens 186fa9e4066Sahrens crc ^= (osv>>14) ^ (obj>>16) ^ (blkid>>16); 187fa9e4066Sahrens 188fa9e4066Sahrens return (crc); 189fa9e4066Sahrens } 190fa9e4066Sahrens 191fa9e4066Sahrens #define DBUF_EQUAL(dbuf, os, obj, level, blkid) \ 192fa9e4066Sahrens ((dbuf)->db.db_object == (obj) && \ 193fa9e4066Sahrens (dbuf)->db_objset == (os) && \ 194fa9e4066Sahrens (dbuf)->db_level == (level) && \ 195fa9e4066Sahrens (dbuf)->db_blkid == (blkid)) 196fa9e4066Sahrens 197fa9e4066Sahrens dmu_buf_impl_t * 198e57a022bSJustin T. Gibbs dbuf_find(objset_t *os, uint64_t obj, uint8_t level, uint64_t blkid) 199fa9e4066Sahrens { 200fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 201*dcbf3bd6SGeorge Wilson uint64_t hv = dbuf_hash(os, obj, level, blkid); 202fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 203fa9e4066Sahrens dmu_buf_impl_t *db; 204fa9e4066Sahrens 205fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 206fa9e4066Sahrens for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) { 207fa9e4066Sahrens if (DBUF_EQUAL(db, os, obj, level, blkid)) { 208fa9e4066Sahrens mutex_enter(&db->db_mtx); 209ea8dc4b6Seschrock if (db->db_state != DB_EVICTING) { 210fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 211fa9e4066Sahrens return (db); 212fa9e4066Sahrens } 213fa9e4066Sahrens mutex_exit(&db->db_mtx); 214fa9e4066Sahrens } 215fa9e4066Sahrens } 216fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 217fa9e4066Sahrens return (NULL); 218fa9e4066Sahrens } 219fa9e4066Sahrens 220e57a022bSJustin T. Gibbs static dmu_buf_impl_t * 221e57a022bSJustin T. Gibbs dbuf_find_bonus(objset_t *os, uint64_t object) 222e57a022bSJustin T. Gibbs { 223e57a022bSJustin T. Gibbs dnode_t *dn; 224e57a022bSJustin T. Gibbs dmu_buf_impl_t *db = NULL; 225e57a022bSJustin T. Gibbs 226e57a022bSJustin T. Gibbs if (dnode_hold(os, object, FTAG, &dn) == 0) { 227e57a022bSJustin T. Gibbs rw_enter(&dn->dn_struct_rwlock, RW_READER); 228e57a022bSJustin T. Gibbs if (dn->dn_bonus != NULL) { 229e57a022bSJustin T. Gibbs db = dn->dn_bonus; 230e57a022bSJustin T. Gibbs mutex_enter(&db->db_mtx); 231e57a022bSJustin T. Gibbs } 232e57a022bSJustin T. Gibbs rw_exit(&dn->dn_struct_rwlock); 233e57a022bSJustin T. Gibbs dnode_rele(dn, FTAG); 234e57a022bSJustin T. Gibbs } 235e57a022bSJustin T. Gibbs return (db); 236e57a022bSJustin T. Gibbs } 237e57a022bSJustin T. Gibbs 238fa9e4066Sahrens /* 239fa9e4066Sahrens * Insert an entry into the hash table. If there is already an element 240fa9e4066Sahrens * equal to elem in the hash table, then the already existing element 241fa9e4066Sahrens * will be returned and the new element will not be inserted. 242fa9e4066Sahrens * Otherwise returns NULL. 243fa9e4066Sahrens */ 244fa9e4066Sahrens static dmu_buf_impl_t * 245fa9e4066Sahrens dbuf_hash_insert(dmu_buf_impl_t *db) 246fa9e4066Sahrens { 247fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 248503ad85cSMatthew Ahrens objset_t *os = db->db_objset; 249fa9e4066Sahrens uint64_t obj = db->db.db_object; 250fa9e4066Sahrens int level = db->db_level; 251fa9e4066Sahrens uint64_t blkid = db->db_blkid; 252*dcbf3bd6SGeorge Wilson uint64_t hv = dbuf_hash(os, obj, level, blkid); 253fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 254fa9e4066Sahrens dmu_buf_impl_t *dbf; 255fa9e4066Sahrens 256fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 257fa9e4066Sahrens for (dbf = h->hash_table[idx]; dbf != NULL; dbf = dbf->db_hash_next) { 258fa9e4066Sahrens if (DBUF_EQUAL(dbf, os, obj, level, blkid)) { 259fa9e4066Sahrens mutex_enter(&dbf->db_mtx); 260ea8dc4b6Seschrock if (dbf->db_state != DB_EVICTING) { 261fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 262fa9e4066Sahrens return (dbf); 263fa9e4066Sahrens } 264fa9e4066Sahrens mutex_exit(&dbf->db_mtx); 265fa9e4066Sahrens } 266fa9e4066Sahrens } 267fa9e4066Sahrens 268fa9e4066Sahrens mutex_enter(&db->db_mtx); 269fa9e4066Sahrens db->db_hash_next = h->hash_table[idx]; 270fa9e4066Sahrens h->hash_table[idx] = db; 271fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 2721a5e258fSJosef 'Jeff' Sipek atomic_inc_64(&dbuf_hash_count); 273fa9e4066Sahrens 274fa9e4066Sahrens return (NULL); 275fa9e4066Sahrens } 276fa9e4066Sahrens 277fa9e4066Sahrens /* 278bbfa8ea8SMatthew Ahrens * Remove an entry from the hash table. It must be in the EVICTING state. 279fa9e4066Sahrens */ 280fa9e4066Sahrens static void 281fa9e4066Sahrens dbuf_hash_remove(dmu_buf_impl_t *db) 282fa9e4066Sahrens { 283fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 284*dcbf3bd6SGeorge Wilson uint64_t hv = dbuf_hash(db->db_objset, db->db.db_object, 285fa9e4066Sahrens db->db_level, db->db_blkid); 286fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 287fa9e4066Sahrens dmu_buf_impl_t *dbf, **dbp; 288fa9e4066Sahrens 289fa9e4066Sahrens /* 290bbfa8ea8SMatthew Ahrens * We musn't hold db_mtx to maintain lock ordering: 291fa9e4066Sahrens * DBUF_HASH_MUTEX > db_mtx. 292fa9e4066Sahrens */ 293fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 294ea8dc4b6Seschrock ASSERT(db->db_state == DB_EVICTING); 295fa9e4066Sahrens ASSERT(!MUTEX_HELD(&db->db_mtx)); 296fa9e4066Sahrens 297fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 298fa9e4066Sahrens dbp = &h->hash_table[idx]; 299fa9e4066Sahrens while ((dbf = *dbp) != db) { 300fa9e4066Sahrens dbp = &dbf->db_hash_next; 301fa9e4066Sahrens ASSERT(dbf != NULL); 302fa9e4066Sahrens } 303fa9e4066Sahrens *dbp = db->db_hash_next; 304fa9e4066Sahrens db->db_hash_next = NULL; 305fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 3061a5e258fSJosef 'Jeff' Sipek atomic_dec_64(&dbuf_hash_count); 307fa9e4066Sahrens } 308fa9e4066Sahrens 309bc9014e6SJustin Gibbs typedef enum { 310bc9014e6SJustin Gibbs DBVU_EVICTING, 311bc9014e6SJustin Gibbs DBVU_NOT_EVICTING 312bc9014e6SJustin Gibbs } dbvu_verify_type_t; 313bc9014e6SJustin Gibbs 314bc9014e6SJustin Gibbs static void 315bc9014e6SJustin Gibbs dbuf_verify_user(dmu_buf_impl_t *db, dbvu_verify_type_t verify_type) 316bc9014e6SJustin Gibbs { 317bc9014e6SJustin Gibbs #ifdef ZFS_DEBUG 318bc9014e6SJustin Gibbs int64_t holds; 319bc9014e6SJustin Gibbs 320bc9014e6SJustin Gibbs if (db->db_user == NULL) 321bc9014e6SJustin Gibbs return; 322bc9014e6SJustin Gibbs 323bc9014e6SJustin Gibbs /* Only data blocks support the attachment of user data. */ 324bc9014e6SJustin Gibbs ASSERT(db->db_level == 0); 325bc9014e6SJustin Gibbs 326bc9014e6SJustin Gibbs /* Clients must resolve a dbuf before attaching user data. */ 327bc9014e6SJustin Gibbs ASSERT(db->db.db_data != NULL); 328bc9014e6SJustin Gibbs ASSERT3U(db->db_state, ==, DB_CACHED); 329bc9014e6SJustin Gibbs 330bc9014e6SJustin Gibbs holds = refcount_count(&db->db_holds); 331bc9014e6SJustin Gibbs if (verify_type == DBVU_EVICTING) { 332bc9014e6SJustin Gibbs /* 333bc9014e6SJustin Gibbs * Immediate eviction occurs when holds == dirtycnt. 334bc9014e6SJustin Gibbs * For normal eviction buffers, holds is zero on 335bc9014e6SJustin Gibbs * eviction, except when dbuf_fix_old_data() calls 336bc9014e6SJustin Gibbs * dbuf_clear_data(). However, the hold count can grow 337bc9014e6SJustin Gibbs * during eviction even though db_mtx is held (see 338bc9014e6SJustin Gibbs * dmu_bonus_hold() for an example), so we can only 339bc9014e6SJustin Gibbs * test the generic invariant that holds >= dirtycnt. 340bc9014e6SJustin Gibbs */ 341bc9014e6SJustin Gibbs ASSERT3U(holds, >=, db->db_dirtycnt); 342bc9014e6SJustin Gibbs } else { 343d2058105SJustin T. Gibbs if (db->db_user_immediate_evict == TRUE) 344bc9014e6SJustin Gibbs ASSERT3U(holds, >=, db->db_dirtycnt); 345bc9014e6SJustin Gibbs else 346bc9014e6SJustin Gibbs ASSERT3U(holds, >, 0); 347bc9014e6SJustin Gibbs } 348bc9014e6SJustin Gibbs #endif 349bc9014e6SJustin Gibbs } 350bc9014e6SJustin Gibbs 351fa9e4066Sahrens static void 352fa9e4066Sahrens dbuf_evict_user(dmu_buf_impl_t *db) 353fa9e4066Sahrens { 354bc9014e6SJustin Gibbs dmu_buf_user_t *dbu = db->db_user; 355bc9014e6SJustin Gibbs 356fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 357fa9e4066Sahrens 358bc9014e6SJustin Gibbs if (dbu == NULL) 359fa9e4066Sahrens return; 360fa9e4066Sahrens 361bc9014e6SJustin Gibbs dbuf_verify_user(db, DBVU_EVICTING); 362bc9014e6SJustin Gibbs db->db_user = NULL; 363bc9014e6SJustin Gibbs 364bc9014e6SJustin Gibbs #ifdef ZFS_DEBUG 365bc9014e6SJustin Gibbs if (dbu->dbu_clear_on_evict_dbufp != NULL) 366bc9014e6SJustin Gibbs *dbu->dbu_clear_on_evict_dbufp = NULL; 367bc9014e6SJustin Gibbs #endif 368bc9014e6SJustin Gibbs 369bc9014e6SJustin Gibbs /* 370bc9014e6SJustin Gibbs * Invoke the callback from a taskq to avoid lock order reversals 371bc9014e6SJustin Gibbs * and limit stack depth. 372bc9014e6SJustin Gibbs */ 373bc9014e6SJustin Gibbs taskq_dispatch_ent(dbu_evict_taskq, dbu->dbu_evict_func, dbu, 0, 374bc9014e6SJustin Gibbs &dbu->dbu_tqent); 375fa9e4066Sahrens } 376fa9e4066Sahrens 377744947dcSTom Erickson boolean_t 378744947dcSTom Erickson dbuf_is_metadata(dmu_buf_impl_t *db) 379744947dcSTom Erickson { 380744947dcSTom Erickson if (db->db_level > 0) { 381744947dcSTom Erickson return (B_TRUE); 382744947dcSTom Erickson } else { 383744947dcSTom Erickson boolean_t is_metadata; 384744947dcSTom Erickson 385744947dcSTom Erickson DB_DNODE_ENTER(db); 386ad135b5dSChristopher Siden is_metadata = DMU_OT_IS_METADATA(DB_DNODE(db)->dn_type); 387744947dcSTom Erickson DB_DNODE_EXIT(db); 388744947dcSTom Erickson 389744947dcSTom Erickson return (is_metadata); 390744947dcSTom Erickson } 391744947dcSTom Erickson } 392744947dcSTom Erickson 393*dcbf3bd6SGeorge Wilson /* 394*dcbf3bd6SGeorge Wilson * This function *must* return indices evenly distributed between all 395*dcbf3bd6SGeorge Wilson * sublists of the multilist. This is needed due to how the dbuf eviction 396*dcbf3bd6SGeorge Wilson * code is laid out; dbuf_evict_thread() assumes dbufs are evenly 397*dcbf3bd6SGeorge Wilson * distributed between all sublists and uses this assumption when 398*dcbf3bd6SGeorge Wilson * deciding which sublist to evict from and how much to evict from it. 399*dcbf3bd6SGeorge Wilson */ 400*dcbf3bd6SGeorge Wilson unsigned int 401*dcbf3bd6SGeorge Wilson dbuf_cache_multilist_index_func(multilist_t *ml, void *obj) 402ea8dc4b6Seschrock { 403*dcbf3bd6SGeorge Wilson dmu_buf_impl_t *db = obj; 404ea8dc4b6Seschrock 405*dcbf3bd6SGeorge Wilson /* 406*dcbf3bd6SGeorge Wilson * The assumption here, is the hash value for a given 407*dcbf3bd6SGeorge Wilson * dmu_buf_impl_t will remain constant throughout it's lifetime 408*dcbf3bd6SGeorge Wilson * (i.e. it's objset, object, level and blkid fields don't change). 409*dcbf3bd6SGeorge Wilson * Thus, we don't need to store the dbuf's sublist index 410*dcbf3bd6SGeorge Wilson * on insertion, as this index can be recalculated on removal. 411*dcbf3bd6SGeorge Wilson * 412*dcbf3bd6SGeorge Wilson * Also, the low order bits of the hash value are thought to be 413*dcbf3bd6SGeorge Wilson * distributed evenly. Otherwise, in the case that the multilist 414*dcbf3bd6SGeorge Wilson * has a power of two number of sublists, each sublists' usage 415*dcbf3bd6SGeorge Wilson * would not be evenly distributed. 416*dcbf3bd6SGeorge Wilson */ 417*dcbf3bd6SGeorge Wilson return (dbuf_hash(db->db_objset, db->db.db_object, 418*dcbf3bd6SGeorge Wilson db->db_level, db->db_blkid) % 419*dcbf3bd6SGeorge Wilson multilist_get_num_sublists(ml)); 420*dcbf3bd6SGeorge Wilson } 421*dcbf3bd6SGeorge Wilson 422*dcbf3bd6SGeorge Wilson static inline boolean_t 423*dcbf3bd6SGeorge Wilson dbuf_cache_above_hiwater(void) 424*dcbf3bd6SGeorge Wilson { 425*dcbf3bd6SGeorge Wilson uint64_t dbuf_cache_hiwater_bytes = 426*dcbf3bd6SGeorge Wilson (dbuf_cache_max_bytes * dbuf_cache_hiwater_pct) / 100; 427*dcbf3bd6SGeorge Wilson 428*dcbf3bd6SGeorge Wilson return (refcount_count(&dbuf_cache_size) > 429*dcbf3bd6SGeorge Wilson dbuf_cache_max_bytes + dbuf_cache_hiwater_bytes); 430*dcbf3bd6SGeorge Wilson } 431*dcbf3bd6SGeorge Wilson 432*dcbf3bd6SGeorge Wilson static inline boolean_t 433*dcbf3bd6SGeorge Wilson dbuf_cache_above_lowater(void) 434*dcbf3bd6SGeorge Wilson { 435*dcbf3bd6SGeorge Wilson uint64_t dbuf_cache_lowater_bytes = 436*dcbf3bd6SGeorge Wilson (dbuf_cache_max_bytes * dbuf_cache_lowater_pct) / 100; 437*dcbf3bd6SGeorge Wilson 438*dcbf3bd6SGeorge Wilson return (refcount_count(&dbuf_cache_size) > 439*dcbf3bd6SGeorge Wilson dbuf_cache_max_bytes - dbuf_cache_lowater_bytes); 440*dcbf3bd6SGeorge Wilson } 441*dcbf3bd6SGeorge Wilson 442*dcbf3bd6SGeorge Wilson /* 443*dcbf3bd6SGeorge Wilson * Evict the oldest eligible dbuf from the dbuf cache. 444*dcbf3bd6SGeorge Wilson */ 445*dcbf3bd6SGeorge Wilson static void 446*dcbf3bd6SGeorge Wilson dbuf_evict_one(void) 447*dcbf3bd6SGeorge Wilson { 448*dcbf3bd6SGeorge Wilson int idx = multilist_get_random_index(&dbuf_cache); 449*dcbf3bd6SGeorge Wilson multilist_sublist_t *mls = multilist_sublist_lock(&dbuf_cache, idx); 450*dcbf3bd6SGeorge Wilson 451*dcbf3bd6SGeorge Wilson ASSERT(!MUTEX_HELD(&dbuf_evict_lock)); 452*dcbf3bd6SGeorge Wilson 453*dcbf3bd6SGeorge Wilson /* 454*dcbf3bd6SGeorge Wilson * Set the thread's tsd to indicate that it's processing evictions. 455*dcbf3bd6SGeorge Wilson * Once a thread stops evicting from the dbuf cache it will 456*dcbf3bd6SGeorge Wilson * reset its tsd to NULL. 457*dcbf3bd6SGeorge Wilson */ 458*dcbf3bd6SGeorge Wilson ASSERT3P(tsd_get(zfs_dbuf_evict_key), ==, NULL); 459*dcbf3bd6SGeorge Wilson (void) tsd_set(zfs_dbuf_evict_key, (void *)B_TRUE); 460*dcbf3bd6SGeorge Wilson 461*dcbf3bd6SGeorge Wilson dmu_buf_impl_t *db = multilist_sublist_tail(mls); 462*dcbf3bd6SGeorge Wilson while (db != NULL && mutex_tryenter(&db->db_mtx) == 0) { 463*dcbf3bd6SGeorge Wilson db = multilist_sublist_prev(mls, db); 464*dcbf3bd6SGeorge Wilson } 465*dcbf3bd6SGeorge Wilson 466*dcbf3bd6SGeorge Wilson DTRACE_PROBE2(dbuf__evict__one, dmu_buf_impl_t *, db, 467*dcbf3bd6SGeorge Wilson multilist_sublist_t *, mls); 468*dcbf3bd6SGeorge Wilson 469*dcbf3bd6SGeorge Wilson if (db != NULL) { 470*dcbf3bd6SGeorge Wilson multilist_sublist_remove(mls, db); 471*dcbf3bd6SGeorge Wilson multilist_sublist_unlock(mls); 472*dcbf3bd6SGeorge Wilson (void) refcount_remove_many(&dbuf_cache_size, 473*dcbf3bd6SGeorge Wilson db->db.db_size, db); 474ea8dc4b6Seschrock dbuf_destroy(db); 475*dcbf3bd6SGeorge Wilson } else { 476*dcbf3bd6SGeorge Wilson multilist_sublist_unlock(mls); 477*dcbf3bd6SGeorge Wilson } 478*dcbf3bd6SGeorge Wilson (void) tsd_set(zfs_dbuf_evict_key, NULL); 479*dcbf3bd6SGeorge Wilson } 480*dcbf3bd6SGeorge Wilson 481*dcbf3bd6SGeorge Wilson /* 482*dcbf3bd6SGeorge Wilson * The dbuf evict thread is responsible for aging out dbufs from the 483*dcbf3bd6SGeorge Wilson * cache. Once the cache has reached it's maximum size, dbufs are removed 484*dcbf3bd6SGeorge Wilson * and destroyed. The eviction thread will continue running until the size 485*dcbf3bd6SGeorge Wilson * of the dbuf cache is at or below the maximum size. Once the dbuf is aged 486*dcbf3bd6SGeorge Wilson * out of the cache it is destroyed and becomes eligible for arc eviction. 487*dcbf3bd6SGeorge Wilson */ 488*dcbf3bd6SGeorge Wilson static void 489*dcbf3bd6SGeorge Wilson dbuf_evict_thread(void) 490*dcbf3bd6SGeorge Wilson { 491*dcbf3bd6SGeorge Wilson callb_cpr_t cpr; 492*dcbf3bd6SGeorge Wilson 493*dcbf3bd6SGeorge Wilson CALLB_CPR_INIT(&cpr, &dbuf_evict_lock, callb_generic_cpr, FTAG); 494*dcbf3bd6SGeorge Wilson 495*dcbf3bd6SGeorge Wilson mutex_enter(&dbuf_evict_lock); 496*dcbf3bd6SGeorge Wilson while (!dbuf_evict_thread_exit) { 497*dcbf3bd6SGeorge Wilson while (!dbuf_cache_above_lowater() && !dbuf_evict_thread_exit) { 498*dcbf3bd6SGeorge Wilson CALLB_CPR_SAFE_BEGIN(&cpr); 499*dcbf3bd6SGeorge Wilson (void) cv_timedwait_hires(&dbuf_evict_cv, 500*dcbf3bd6SGeorge Wilson &dbuf_evict_lock, SEC2NSEC(1), MSEC2NSEC(1), 0); 501*dcbf3bd6SGeorge Wilson CALLB_CPR_SAFE_END(&cpr, &dbuf_evict_lock); 502*dcbf3bd6SGeorge Wilson } 503*dcbf3bd6SGeorge Wilson mutex_exit(&dbuf_evict_lock); 504*dcbf3bd6SGeorge Wilson 505*dcbf3bd6SGeorge Wilson /* 506*dcbf3bd6SGeorge Wilson * Keep evicting as long as we're above the low water mark 507*dcbf3bd6SGeorge Wilson * for the cache. We do this without holding the locks to 508*dcbf3bd6SGeorge Wilson * minimize lock contention. 509*dcbf3bd6SGeorge Wilson */ 510*dcbf3bd6SGeorge Wilson while (dbuf_cache_above_lowater() && !dbuf_evict_thread_exit) { 511*dcbf3bd6SGeorge Wilson dbuf_evict_one(); 512*dcbf3bd6SGeorge Wilson } 513*dcbf3bd6SGeorge Wilson 514*dcbf3bd6SGeorge Wilson mutex_enter(&dbuf_evict_lock); 515*dcbf3bd6SGeorge Wilson } 516*dcbf3bd6SGeorge Wilson 517*dcbf3bd6SGeorge Wilson dbuf_evict_thread_exit = B_FALSE; 518*dcbf3bd6SGeorge Wilson cv_broadcast(&dbuf_evict_cv); 519*dcbf3bd6SGeorge Wilson CALLB_CPR_EXIT(&cpr); /* drops dbuf_evict_lock */ 520*dcbf3bd6SGeorge Wilson thread_exit(); 521*dcbf3bd6SGeorge Wilson } 522*dcbf3bd6SGeorge Wilson 523*dcbf3bd6SGeorge Wilson /* 524*dcbf3bd6SGeorge Wilson * Wake up the dbuf eviction thread if the dbuf cache is at its max size. 525*dcbf3bd6SGeorge Wilson * If the dbuf cache is at its high water mark, then evict a dbuf from the 526*dcbf3bd6SGeorge Wilson * dbuf cache using the callers context. 527*dcbf3bd6SGeorge Wilson */ 528*dcbf3bd6SGeorge Wilson static void 529*dcbf3bd6SGeorge Wilson dbuf_evict_notify(void) 530*dcbf3bd6SGeorge Wilson { 531*dcbf3bd6SGeorge Wilson 532*dcbf3bd6SGeorge Wilson /* 533*dcbf3bd6SGeorge Wilson * We use thread specific data to track when a thread has 534*dcbf3bd6SGeorge Wilson * started processing evictions. This allows us to avoid deeply 535*dcbf3bd6SGeorge Wilson * nested stacks that would have a call flow similar to this: 536*dcbf3bd6SGeorge Wilson * 537*dcbf3bd6SGeorge Wilson * dbuf_rele()-->dbuf_rele_and_unlock()-->dbuf_evict_notify() 538*dcbf3bd6SGeorge Wilson * ^ | 539*dcbf3bd6SGeorge Wilson * | | 540*dcbf3bd6SGeorge Wilson * +-----dbuf_destroy()<--dbuf_evict_one()<--------+ 541*dcbf3bd6SGeorge Wilson * 542*dcbf3bd6SGeorge Wilson * The dbuf_eviction_thread will always have its tsd set until 543*dcbf3bd6SGeorge Wilson * that thread exits. All other threads will only set their tsd 544*dcbf3bd6SGeorge Wilson * if they are participating in the eviction process. This only 545*dcbf3bd6SGeorge Wilson * happens if the eviction thread is unable to process evictions 546*dcbf3bd6SGeorge Wilson * fast enough. To keep the dbuf cache size in check, other threads 547*dcbf3bd6SGeorge Wilson * can evict from the dbuf cache directly. Those threads will set 548*dcbf3bd6SGeorge Wilson * their tsd values so that we ensure that they only evict one dbuf 549*dcbf3bd6SGeorge Wilson * from the dbuf cache. 550*dcbf3bd6SGeorge Wilson */ 551*dcbf3bd6SGeorge Wilson if (tsd_get(zfs_dbuf_evict_key) != NULL) 552*dcbf3bd6SGeorge Wilson return; 553*dcbf3bd6SGeorge Wilson 554*dcbf3bd6SGeorge Wilson if (refcount_count(&dbuf_cache_size) > dbuf_cache_max_bytes) { 555*dcbf3bd6SGeorge Wilson boolean_t evict_now = B_FALSE; 556*dcbf3bd6SGeorge Wilson 557*dcbf3bd6SGeorge Wilson mutex_enter(&dbuf_evict_lock); 558*dcbf3bd6SGeorge Wilson if (refcount_count(&dbuf_cache_size) > dbuf_cache_max_bytes) { 559*dcbf3bd6SGeorge Wilson evict_now = dbuf_cache_above_hiwater(); 560*dcbf3bd6SGeorge Wilson cv_signal(&dbuf_evict_cv); 561*dcbf3bd6SGeorge Wilson } 562*dcbf3bd6SGeorge Wilson mutex_exit(&dbuf_evict_lock); 563*dcbf3bd6SGeorge Wilson 564*dcbf3bd6SGeorge Wilson if (evict_now) { 565*dcbf3bd6SGeorge Wilson dbuf_evict_one(); 566*dcbf3bd6SGeorge Wilson } 567*dcbf3bd6SGeorge Wilson } 568ea8dc4b6Seschrock } 569ea8dc4b6Seschrock 570ea8dc4b6Seschrock void 571fa9e4066Sahrens dbuf_init(void) 572fa9e4066Sahrens { 573ea8dc4b6Seschrock uint64_t hsize = 1ULL << 16; 574fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 575fa9e4066Sahrens int i; 576fa9e4066Sahrens 577fa9e4066Sahrens /* 578fa9e4066Sahrens * The hash table is big enough to fill all of physical memory 579ea8dc4b6Seschrock * with an average 4K block size. The table will take up 580ea8dc4b6Seschrock * totalmem*sizeof(void*)/4K (i.e. 2MB/GB with 8-byte pointers). 581fa9e4066Sahrens */ 582ea8dc4b6Seschrock while (hsize * 4096 < physmem * PAGESIZE) 583fa9e4066Sahrens hsize <<= 1; 584fa9e4066Sahrens 585ea8dc4b6Seschrock retry: 586fa9e4066Sahrens h->hash_table_mask = hsize - 1; 587ea8dc4b6Seschrock h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP); 588ea8dc4b6Seschrock if (h->hash_table == NULL) { 589ea8dc4b6Seschrock /* XXX - we should really return an error instead of assert */ 590ea8dc4b6Seschrock ASSERT(hsize > (1ULL << 10)); 591ea8dc4b6Seschrock hsize >>= 1; 592ea8dc4b6Seschrock goto retry; 593ea8dc4b6Seschrock } 594fa9e4066Sahrens 595*dcbf3bd6SGeorge Wilson dbuf_kmem_cache = kmem_cache_create("dmu_buf_impl_t", 596fa9e4066Sahrens sizeof (dmu_buf_impl_t), 597fa9e4066Sahrens 0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0); 598fa9e4066Sahrens 599fa9e4066Sahrens for (i = 0; i < DBUF_MUTEXES; i++) 600fa9e4066Sahrens mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL); 601bc9014e6SJustin Gibbs 602bc9014e6SJustin Gibbs /* 603*dcbf3bd6SGeorge Wilson * Setup the parameters for the dbuf cache. We cap the size of the 604*dcbf3bd6SGeorge Wilson * dbuf cache to 1/32nd (default) of the size of the ARC. 605*dcbf3bd6SGeorge Wilson */ 606*dcbf3bd6SGeorge Wilson dbuf_cache_max_bytes = MIN(dbuf_cache_max_bytes, 607*dcbf3bd6SGeorge Wilson arc_max_bytes() >> dbuf_cache_max_shift); 608*dcbf3bd6SGeorge Wilson 609*dcbf3bd6SGeorge Wilson /* 610bc9014e6SJustin Gibbs * All entries are queued via taskq_dispatch_ent(), so min/maxalloc 611bc9014e6SJustin Gibbs * configuration is not required. 612bc9014e6SJustin Gibbs */ 613bc9014e6SJustin Gibbs dbu_evict_taskq = taskq_create("dbu_evict", 1, minclsyspri, 0, 0, 0); 614*dcbf3bd6SGeorge Wilson 615*dcbf3bd6SGeorge Wilson multilist_create(&dbuf_cache, sizeof (dmu_buf_impl_t), 616*dcbf3bd6SGeorge Wilson offsetof(dmu_buf_impl_t, db_cache_link), 617*dcbf3bd6SGeorge Wilson zfs_arc_num_sublists_per_state, 618*dcbf3bd6SGeorge Wilson dbuf_cache_multilist_index_func); 619*dcbf3bd6SGeorge Wilson refcount_create(&dbuf_cache_size); 620*dcbf3bd6SGeorge Wilson 621*dcbf3bd6SGeorge Wilson tsd_create(&zfs_dbuf_evict_key, NULL); 622*dcbf3bd6SGeorge Wilson dbuf_evict_thread_exit = B_FALSE; 623*dcbf3bd6SGeorge Wilson mutex_init(&dbuf_evict_lock, NULL, MUTEX_DEFAULT, NULL); 624*dcbf3bd6SGeorge Wilson cv_init(&dbuf_evict_cv, NULL, CV_DEFAULT, NULL); 625*dcbf3bd6SGeorge Wilson dbuf_cache_evict_thread = thread_create(NULL, 0, dbuf_evict_thread, 626*dcbf3bd6SGeorge Wilson NULL, 0, &p0, TS_RUN, minclsyspri); 627fa9e4066Sahrens } 628fa9e4066Sahrens 629fa9e4066Sahrens void 630fa9e4066Sahrens dbuf_fini(void) 631fa9e4066Sahrens { 632fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 633fa9e4066Sahrens int i; 634fa9e4066Sahrens 635fa9e4066Sahrens for (i = 0; i < DBUF_MUTEXES; i++) 636fa9e4066Sahrens mutex_destroy(&h->hash_mutexes[i]); 637fa9e4066Sahrens kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *)); 638*dcbf3bd6SGeorge Wilson kmem_cache_destroy(dbuf_kmem_cache); 639bc9014e6SJustin Gibbs taskq_destroy(dbu_evict_taskq); 640*dcbf3bd6SGeorge Wilson 641*dcbf3bd6SGeorge Wilson mutex_enter(&dbuf_evict_lock); 642*dcbf3bd6SGeorge Wilson dbuf_evict_thread_exit = B_TRUE; 643*dcbf3bd6SGeorge Wilson while (dbuf_evict_thread_exit) { 644*dcbf3bd6SGeorge Wilson cv_signal(&dbuf_evict_cv); 645*dcbf3bd6SGeorge Wilson cv_wait(&dbuf_evict_cv, &dbuf_evict_lock); 646*dcbf3bd6SGeorge Wilson } 647*dcbf3bd6SGeorge Wilson mutex_exit(&dbuf_evict_lock); 648*dcbf3bd6SGeorge Wilson tsd_destroy(&zfs_dbuf_evict_key); 649*dcbf3bd6SGeorge Wilson 650*dcbf3bd6SGeorge Wilson mutex_destroy(&dbuf_evict_lock); 651*dcbf3bd6SGeorge Wilson cv_destroy(&dbuf_evict_cv); 652*dcbf3bd6SGeorge Wilson 653*dcbf3bd6SGeorge Wilson refcount_destroy(&dbuf_cache_size); 654*dcbf3bd6SGeorge Wilson multilist_destroy(&dbuf_cache); 655fa9e4066Sahrens } 656fa9e4066Sahrens 657fa9e4066Sahrens /* 658fa9e4066Sahrens * Other stuff. 659fa9e4066Sahrens */ 660fa9e4066Sahrens 6619c9dc39aSek110237 #ifdef ZFS_DEBUG 662fa9e4066Sahrens static void 663fa9e4066Sahrens dbuf_verify(dmu_buf_impl_t *db) 664fa9e4066Sahrens { 665744947dcSTom Erickson dnode_t *dn; 666b24ab676SJeff Bonwick dbuf_dirty_record_t *dr; 667fa9e4066Sahrens 668fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 669fa9e4066Sahrens 670fa9e4066Sahrens if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY)) 671fa9e4066Sahrens return; 672fa9e4066Sahrens 673fa9e4066Sahrens ASSERT(db->db_objset != NULL); 674744947dcSTom Erickson DB_DNODE_ENTER(db); 675744947dcSTom Erickson dn = DB_DNODE(db); 676fa9e4066Sahrens if (dn == NULL) { 677fa9e4066Sahrens ASSERT(db->db_parent == NULL); 678fa9e4066Sahrens ASSERT(db->db_blkptr == NULL); 679fa9e4066Sahrens } else { 680fa9e4066Sahrens ASSERT3U(db->db.db_object, ==, dn->dn_object); 681fa9e4066Sahrens ASSERT3P(db->db_objset, ==, dn->dn_objset); 682fa9e4066Sahrens ASSERT3U(db->db_level, <, dn->dn_nlevels); 683744947dcSTom Erickson ASSERT(db->db_blkid == DMU_BONUS_BLKID || 684744947dcSTom Erickson db->db_blkid == DMU_SPILL_BLKID || 6850f6d88adSAlex Reece !avl_is_empty(&dn->dn_dbufs)); 686fa9e4066Sahrens } 6870a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 688fa9e4066Sahrens ASSERT(dn != NULL); 6891934e92fSmaybee ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 6900a586ceaSMark Shellenbaum ASSERT3U(db->db.db_offset, ==, DMU_BONUS_BLKID); 6910a586ceaSMark Shellenbaum } else if (db->db_blkid == DMU_SPILL_BLKID) { 6920a586ceaSMark Shellenbaum ASSERT(dn != NULL); 6930a586ceaSMark Shellenbaum ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 694fb09f5aaSMadhav Suresh ASSERT0(db->db.db_offset); 695fa9e4066Sahrens } else { 696fa9e4066Sahrens ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size); 697fa9e4066Sahrens } 698fa9e4066Sahrens 699b24ab676SJeff Bonwick for (dr = db->db_data_pending; dr != NULL; dr = dr->dr_next) 700b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 701b24ab676SJeff Bonwick 702b24ab676SJeff Bonwick for (dr = db->db_last_dirty; dr != NULL; dr = dr->dr_next) 703b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 704b24ab676SJeff Bonwick 70588b7b0f2SMatthew Ahrens /* 70688b7b0f2SMatthew Ahrens * We can't assert that db_size matches dn_datablksz because it 70788b7b0f2SMatthew Ahrens * can be momentarily different when another thread is doing 70888b7b0f2SMatthew Ahrens * dnode_set_blksz(). 70988b7b0f2SMatthew Ahrens */ 71088b7b0f2SMatthew Ahrens if (db->db_level == 0 && db->db.db_object == DMU_META_DNODE_OBJECT) { 711b24ab676SJeff Bonwick dr = db->db_data_pending; 712fa9e4066Sahrens /* 71388b7b0f2SMatthew Ahrens * It should only be modified in syncing context, so 71488b7b0f2SMatthew Ahrens * make sure we only have one copy of the data. 715fa9e4066Sahrens */ 716c717a561Smaybee ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf); 717fa9e4066Sahrens } 718fa9e4066Sahrens 719fa9e4066Sahrens /* verify db->db_blkptr */ 720fa9e4066Sahrens if (db->db_blkptr) { 721fa9e4066Sahrens if (db->db_parent == dn->dn_dbuf) { 722fa9e4066Sahrens /* db is pointed to by the dnode */ 723fa9e4066Sahrens /* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */ 72414843421SMatthew Ahrens if (DMU_OBJECT_IS_SPECIAL(db->db.db_object)) 725fa9e4066Sahrens ASSERT(db->db_parent == NULL); 726fa9e4066Sahrens else 727fa9e4066Sahrens ASSERT(db->db_parent != NULL); 7280a586ceaSMark Shellenbaum if (db->db_blkid != DMU_SPILL_BLKID) 729fa9e4066Sahrens ASSERT3P(db->db_blkptr, ==, 730fa9e4066Sahrens &dn->dn_phys->dn_blkptr[db->db_blkid]); 731fa9e4066Sahrens } else { 732fa9e4066Sahrens /* db is pointed to by an indirect block */ 733fa9e4066Sahrens int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT; 734fa9e4066Sahrens ASSERT3U(db->db_parent->db_level, ==, db->db_level+1); 735fa9e4066Sahrens ASSERT3U(db->db_parent->db.db_object, ==, 736fa9e4066Sahrens db->db.db_object); 737fa9e4066Sahrens /* 738fa9e4066Sahrens * dnode_grow_indblksz() can make this fail if we don't 739fa9e4066Sahrens * have the struct_rwlock. XXX indblksz no longer 740fa9e4066Sahrens * grows. safe to do this now? 741fa9e4066Sahrens */ 742744947dcSTom Erickson if (RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 743fa9e4066Sahrens ASSERT3P(db->db_blkptr, ==, 744fa9e4066Sahrens ((blkptr_t *)db->db_parent->db.db_data + 745fa9e4066Sahrens db->db_blkid % epb)); 746fa9e4066Sahrens } 747fa9e4066Sahrens } 748fa9e4066Sahrens } 749fa9e4066Sahrens if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) && 7503f9d6ad7SLin Ling (db->db_buf == NULL || db->db_buf->b_data) && 7510a586ceaSMark Shellenbaum db->db.db_data && db->db_blkid != DMU_BONUS_BLKID && 752fa9e4066Sahrens db->db_state != DB_FILL && !dn->dn_free_txg) { 753fa9e4066Sahrens /* 754fa9e4066Sahrens * If the blkptr isn't set but they have nonzero data, 755fa9e4066Sahrens * it had better be dirty, otherwise we'll lose that 756fa9e4066Sahrens * data when we evict this buffer. 7578df0bcf0SPaul Dagnelie * 7588df0bcf0SPaul Dagnelie * There is an exception to this rule for indirect blocks; in 7598df0bcf0SPaul Dagnelie * this case, if the indirect block is a hole, we fill in a few 7608df0bcf0SPaul Dagnelie * fields on each of the child blocks (importantly, birth time) 7618df0bcf0SPaul Dagnelie * to prevent hole birth times from being lost when you 7628df0bcf0SPaul Dagnelie * partially fill in a hole. 763fa9e4066Sahrens */ 764fa9e4066Sahrens if (db->db_dirtycnt == 0) { 7658df0bcf0SPaul Dagnelie if (db->db_level == 0) { 766fa9e4066Sahrens uint64_t *buf = db->db.db_data; 767fa9e4066Sahrens int i; 768fa9e4066Sahrens 769fa9e4066Sahrens for (i = 0; i < db->db.db_size >> 3; i++) { 770fa9e4066Sahrens ASSERT(buf[i] == 0); 771fa9e4066Sahrens } 7728df0bcf0SPaul Dagnelie } else { 7738df0bcf0SPaul Dagnelie blkptr_t *bps = db->db.db_data; 7748df0bcf0SPaul Dagnelie ASSERT3U(1 << DB_DNODE(db)->dn_indblkshift, ==, 7758df0bcf0SPaul Dagnelie db->db.db_size); 7768df0bcf0SPaul Dagnelie /* 7778df0bcf0SPaul Dagnelie * We want to verify that all the blkptrs in the 7788df0bcf0SPaul Dagnelie * indirect block are holes, but we may have 7798df0bcf0SPaul Dagnelie * automatically set up a few fields for them. 7808df0bcf0SPaul Dagnelie * We iterate through each blkptr and verify 7818df0bcf0SPaul Dagnelie * they only have those fields set. 7828df0bcf0SPaul Dagnelie */ 7838df0bcf0SPaul Dagnelie for (int i = 0; 7848df0bcf0SPaul Dagnelie i < db->db.db_size / sizeof (blkptr_t); 7858df0bcf0SPaul Dagnelie i++) { 7868df0bcf0SPaul Dagnelie blkptr_t *bp = &bps[i]; 7878df0bcf0SPaul Dagnelie ASSERT(ZIO_CHECKSUM_IS_ZERO( 7888df0bcf0SPaul Dagnelie &bp->blk_cksum)); 7898df0bcf0SPaul Dagnelie ASSERT( 7908df0bcf0SPaul Dagnelie DVA_IS_EMPTY(&bp->blk_dva[0]) && 7918df0bcf0SPaul Dagnelie DVA_IS_EMPTY(&bp->blk_dva[1]) && 7928df0bcf0SPaul Dagnelie DVA_IS_EMPTY(&bp->blk_dva[2])); 7938df0bcf0SPaul Dagnelie ASSERT0(bp->blk_fill); 7948df0bcf0SPaul Dagnelie ASSERT0(bp->blk_pad[0]); 7958df0bcf0SPaul Dagnelie ASSERT0(bp->blk_pad[1]); 7968df0bcf0SPaul Dagnelie ASSERT(!BP_IS_EMBEDDED(bp)); 7978df0bcf0SPaul Dagnelie ASSERT(BP_IS_HOLE(bp)); 7988df0bcf0SPaul Dagnelie ASSERT0(bp->blk_phys_birth); 7998df0bcf0SPaul Dagnelie } 8008df0bcf0SPaul Dagnelie } 801fa9e4066Sahrens } 802fa9e4066Sahrens } 803744947dcSTom Erickson DB_DNODE_EXIT(db); 804fa9e4066Sahrens } 8059c9dc39aSek110237 #endif 806fa9e4066Sahrens 807fa9e4066Sahrens static void 808bc9014e6SJustin Gibbs dbuf_clear_data(dmu_buf_impl_t *db) 809fa9e4066Sahrens { 810fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 811ea8dc4b6Seschrock dbuf_evict_user(db); 812*dcbf3bd6SGeorge Wilson ASSERT3P(db->db_buf, ==, NULL); 813ea8dc4b6Seschrock db->db.db_data = NULL; 81482c9918fSTim Haley if (db->db_state != DB_NOFILL) 815ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 816ea8dc4b6Seschrock } 817bc9014e6SJustin Gibbs 818bc9014e6SJustin Gibbs static void 819bc9014e6SJustin Gibbs dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf) 820bc9014e6SJustin Gibbs { 821bc9014e6SJustin Gibbs ASSERT(MUTEX_HELD(&db->db_mtx)); 822bc9014e6SJustin Gibbs ASSERT(buf != NULL); 823bc9014e6SJustin Gibbs 824bc9014e6SJustin Gibbs db->db_buf = buf; 825bc9014e6SJustin Gibbs ASSERT(buf->b_data != NULL); 826bc9014e6SJustin Gibbs db->db.db_data = buf->b_data; 827fa9e4066Sahrens } 828fa9e4066Sahrens 829c242f9a0Schunli zhang - Sun Microsystems - Irvine United States /* 830c242f9a0Schunli zhang - Sun Microsystems - Irvine United States * Loan out an arc_buf for read. Return the loaned arc_buf. 831c242f9a0Schunli zhang - Sun Microsystems - Irvine United States */ 832c242f9a0Schunli zhang - Sun Microsystems - Irvine United States arc_buf_t * 833c242f9a0Schunli zhang - Sun Microsystems - Irvine United States dbuf_loan_arcbuf(dmu_buf_impl_t *db) 834c242f9a0Schunli zhang - Sun Microsystems - Irvine United States { 835c242f9a0Schunli zhang - Sun Microsystems - Irvine United States arc_buf_t *abuf; 836c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 837*dcbf3bd6SGeorge Wilson ASSERT(db->db_blkid != DMU_BONUS_BLKID); 838c242f9a0Schunli zhang - Sun Microsystems - Irvine United States mutex_enter(&db->db_mtx); 839c242f9a0Schunli zhang - Sun Microsystems - Irvine United States if (arc_released(db->db_buf) || refcount_count(&db->db_holds) > 1) { 840c242f9a0Schunli zhang - Sun Microsystems - Irvine United States int blksz = db->db.db_size; 84143466aaeSMax Grossman spa_t *spa = db->db_objset->os_spa; 842744947dcSTom Erickson 843c242f9a0Schunli zhang - Sun Microsystems - Irvine United States mutex_exit(&db->db_mtx); 844744947dcSTom Erickson abuf = arc_loan_buf(spa, blksz); 845c242f9a0Schunli zhang - Sun Microsystems - Irvine United States bcopy(db->db.db_data, abuf->b_data, blksz); 846c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } else { 847c242f9a0Schunli zhang - Sun Microsystems - Irvine United States abuf = db->db_buf; 848c242f9a0Schunli zhang - Sun Microsystems - Irvine United States arc_loan_inuse_buf(abuf, db); 849*dcbf3bd6SGeorge Wilson db->db_buf = NULL; 850bc9014e6SJustin Gibbs dbuf_clear_data(db); 851c242f9a0Schunli zhang - Sun Microsystems - Irvine United States mutex_exit(&db->db_mtx); 852c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } 853c242f9a0Schunli zhang - Sun Microsystems - Irvine United States return (abuf); 854c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } 855c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 856a2cdcdd2SPaul Dagnelie /* 857a2cdcdd2SPaul Dagnelie * Calculate which level n block references the data at the level 0 offset 858a2cdcdd2SPaul Dagnelie * provided. 859a2cdcdd2SPaul Dagnelie */ 860fa9e4066Sahrens uint64_t 861a2cdcdd2SPaul Dagnelie dbuf_whichblock(dnode_t *dn, int64_t level, uint64_t offset) 862fa9e4066Sahrens { 863a2cdcdd2SPaul Dagnelie if (dn->dn_datablkshift != 0 && dn->dn_indblkshift != 0) { 864a2cdcdd2SPaul Dagnelie /* 865a2cdcdd2SPaul Dagnelie * The level n blkid is equal to the level 0 blkid divided by 866a2cdcdd2SPaul Dagnelie * the number of level 0s in a level n block. 867a2cdcdd2SPaul Dagnelie * 868a2cdcdd2SPaul Dagnelie * The level 0 blkid is offset >> datablkshift = 869a2cdcdd2SPaul Dagnelie * offset / 2^datablkshift. 870a2cdcdd2SPaul Dagnelie * 871a2cdcdd2SPaul Dagnelie * The number of level 0s in a level n is the number of block 872a2cdcdd2SPaul Dagnelie * pointers in an indirect block, raised to the power of level. 873a2cdcdd2SPaul Dagnelie * This is 2^(indblkshift - SPA_BLKPTRSHIFT)^level = 874a2cdcdd2SPaul Dagnelie * 2^(level*(indblkshift - SPA_BLKPTRSHIFT)). 875a2cdcdd2SPaul Dagnelie * 876a2cdcdd2SPaul Dagnelie * Thus, the level n blkid is: offset / 877a2cdcdd2SPaul Dagnelie * ((2^datablkshift)*(2^(level*(indblkshift - SPA_BLKPTRSHIFT))) 878a2cdcdd2SPaul Dagnelie * = offset / 2^(datablkshift + level * 879a2cdcdd2SPaul Dagnelie * (indblkshift - SPA_BLKPTRSHIFT)) 880a2cdcdd2SPaul Dagnelie * = offset >> (datablkshift + level * 881a2cdcdd2SPaul Dagnelie * (indblkshift - SPA_BLKPTRSHIFT)) 882a2cdcdd2SPaul Dagnelie */ 883a2cdcdd2SPaul Dagnelie return (offset >> (dn->dn_datablkshift + level * 884a2cdcdd2SPaul Dagnelie (dn->dn_indblkshift - SPA_BLKPTRSHIFT))); 885fa9e4066Sahrens } else { 886fa9e4066Sahrens ASSERT3U(offset, <, dn->dn_datablksz); 887fa9e4066Sahrens return (0); 888fa9e4066Sahrens } 889fa9e4066Sahrens } 890fa9e4066Sahrens 891fa9e4066Sahrens static void 892fa9e4066Sahrens dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb) 893fa9e4066Sahrens { 894fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 895fa9e4066Sahrens 896fa9e4066Sahrens mutex_enter(&db->db_mtx); 897fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_READ); 898fa9e4066Sahrens /* 899fa9e4066Sahrens * All reads are synchronous, so we must have a hold on the dbuf 900fa9e4066Sahrens */ 901fa9e4066Sahrens ASSERT(refcount_count(&db->db_holds) > 0); 902ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 903fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 904c717a561Smaybee if (db->db_level == 0 && db->db_freed_in_flight) { 905fa9e4066Sahrens /* we were freed in flight; disregard any error */ 906fa9e4066Sahrens arc_release(buf, db); 907fa9e4066Sahrens bzero(buf->b_data, db->db.db_size); 9086b4acc8bSahrens arc_buf_freeze(buf); 909c717a561Smaybee db->db_freed_in_flight = FALSE; 910fa9e4066Sahrens dbuf_set_data(db, buf); 911fa9e4066Sahrens db->db_state = DB_CACHED; 912fa9e4066Sahrens } else if (zio == NULL || zio->io_error == 0) { 913fa9e4066Sahrens dbuf_set_data(db, buf); 914fa9e4066Sahrens db->db_state = DB_CACHED; 915fa9e4066Sahrens } else { 9160a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 917fa9e4066Sahrens ASSERT3P(db->db_buf, ==, NULL); 918*dcbf3bd6SGeorge Wilson arc_buf_destroy(buf, db); 919ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 920fa9e4066Sahrens } 921fa9e4066Sahrens cv_broadcast(&db->db_changed); 9223f9d6ad7SLin Ling dbuf_rele_and_unlock(db, NULL); 923fa9e4066Sahrens } 924fa9e4066Sahrens 925ea8dc4b6Seschrock static void 926cf6106c8SMatthew Ahrens dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags) 927fa9e4066Sahrens { 928744947dcSTom Erickson dnode_t *dn; 9297802d7bfSMatthew Ahrens zbookmark_phys_t zb; 9307adb730bSGeorge Wilson arc_flags_t aflags = ARC_FLAG_NOWAIT; 931fa9e4066Sahrens 932744947dcSTom Erickson DB_DNODE_ENTER(db); 933744947dcSTom Erickson dn = DB_DNODE(db); 934fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 935fa9e4066Sahrens /* We need the struct_rwlock to prevent db_blkptr from changing. */ 936088f3894Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 937ea8dc4b6Seschrock ASSERT(MUTEX_HELD(&db->db_mtx)); 938ea8dc4b6Seschrock ASSERT(db->db_state == DB_UNCACHED); 939ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 940fa9e4066Sahrens 9410a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 942cf04dda1SMark Maybee int bonuslen = MIN(dn->dn_bonuslen, dn->dn_phys->dn_bonuslen); 9431934e92fSmaybee 9441934e92fSmaybee ASSERT3U(bonuslen, <=, db->db.db_size); 945ea8dc4b6Seschrock db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN); 9465a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 9471934e92fSmaybee if (bonuslen < DN_MAX_BONUSLEN) 948ea8dc4b6Seschrock bzero(db->db.db_data, DN_MAX_BONUSLEN); 949cf04dda1SMark Maybee if (bonuslen) 950cf04dda1SMark Maybee bcopy(DN_BONUS(dn->dn_phys), db->db.db_data, bonuslen); 951744947dcSTom Erickson DB_DNODE_EXIT(db); 952fa9e4066Sahrens db->db_state = DB_CACHED; 953fa9e4066Sahrens mutex_exit(&db->db_mtx); 954fa9e4066Sahrens return; 955fa9e4066Sahrens } 956fa9e4066Sahrens 9571c8564a7SMark Maybee /* 9581c8564a7SMark Maybee * Recheck BP_IS_HOLE() after dnode_block_freed() in case dnode_sync() 9591c8564a7SMark Maybee * processes the delete record and clears the bp while we are waiting 9601c8564a7SMark Maybee * for the dn_mtx (resulting in a "no" from block_freed). 9611c8564a7SMark Maybee */ 962088f3894Sahrens if (db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr) || 9631c8564a7SMark Maybee (db->db_level == 0 && (dnode_block_freed(dn, db->db_blkid) || 9641c8564a7SMark Maybee BP_IS_HOLE(db->db_blkptr)))) { 965ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 966ad23a2dbSjohansen 967*dcbf3bd6SGeorge Wilson dbuf_set_data(db, arc_alloc_buf(db->db_objset->os_spa, 96843466aaeSMax Grossman db->db.db_size, db, type)); 969fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 9708df0bcf0SPaul Dagnelie 9718df0bcf0SPaul Dagnelie if (db->db_blkptr != NULL && db->db_level > 0 && 9728df0bcf0SPaul Dagnelie BP_IS_HOLE(db->db_blkptr) && 9738df0bcf0SPaul Dagnelie db->db_blkptr->blk_birth != 0) { 9748df0bcf0SPaul Dagnelie blkptr_t *bps = db->db.db_data; 9758df0bcf0SPaul Dagnelie for (int i = 0; i < ((1 << 9768df0bcf0SPaul Dagnelie DB_DNODE(db)->dn_indblkshift) / sizeof (blkptr_t)); 9778df0bcf0SPaul Dagnelie i++) { 9788df0bcf0SPaul Dagnelie blkptr_t *bp = &bps[i]; 9798df0bcf0SPaul Dagnelie ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, 9808df0bcf0SPaul Dagnelie 1 << dn->dn_indblkshift); 9818df0bcf0SPaul Dagnelie BP_SET_LSIZE(bp, 9828df0bcf0SPaul Dagnelie BP_GET_LEVEL(db->db_blkptr) == 1 ? 9838df0bcf0SPaul Dagnelie dn->dn_datablksz : 9848df0bcf0SPaul Dagnelie BP_GET_LSIZE(db->db_blkptr)); 9858df0bcf0SPaul Dagnelie BP_SET_TYPE(bp, BP_GET_TYPE(db->db_blkptr)); 9868df0bcf0SPaul Dagnelie BP_SET_LEVEL(bp, 9878df0bcf0SPaul Dagnelie BP_GET_LEVEL(db->db_blkptr) - 1); 9888df0bcf0SPaul Dagnelie BP_SET_BIRTH(bp, db->db_blkptr->blk_birth, 0); 9898df0bcf0SPaul Dagnelie } 9908df0bcf0SPaul Dagnelie } 9918df0bcf0SPaul Dagnelie DB_DNODE_EXIT(db); 992fa9e4066Sahrens db->db_state = DB_CACHED; 993fa9e4066Sahrens mutex_exit(&db->db_mtx); 994fa9e4066Sahrens return; 995fa9e4066Sahrens } 996fa9e4066Sahrens 997744947dcSTom Erickson DB_DNODE_EXIT(db); 998744947dcSTom Erickson 999fa9e4066Sahrens db->db_state = DB_READ; 1000fa9e4066Sahrens mutex_exit(&db->db_mtx); 1001fa9e4066Sahrens 10023baa08fcSek110237 if (DBUF_IS_L2CACHEABLE(db)) 10037adb730bSGeorge Wilson aflags |= ARC_FLAG_L2CACHE; 10043baa08fcSek110237 1005b24ab676SJeff Bonwick SET_BOOKMARK(&zb, db->db_objset->os_dsl_dataset ? 1006b24ab676SJeff Bonwick db->db_objset->os_dsl_dataset->ds_object : DMU_META_OBJSET, 1007b24ab676SJeff Bonwick db->db.db_object, db->db_level, db->db_blkid); 1008ea8dc4b6Seschrock 1009ea8dc4b6Seschrock dbuf_add_ref(db, NULL); 1010088f3894Sahrens 101143466aaeSMax Grossman (void) arc_read(zio, db->db_objset->os_spa, db->db_blkptr, 1012fa9e4066Sahrens dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ, 1013cf6106c8SMatthew Ahrens (flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED, 101413506d1eSmaybee &aflags, &zb); 1015fa9e4066Sahrens } 1016fa9e4066Sahrens 1017ea8dc4b6Seschrock int 1018ea8dc4b6Seschrock dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags) 1019fa9e4066Sahrens { 1020ea8dc4b6Seschrock int err = 0; 102143466aaeSMax Grossman boolean_t havepzio = (zio != NULL); 102243466aaeSMax Grossman boolean_t prefetch; 1023744947dcSTom Erickson dnode_t *dn; 1024fa9e4066Sahrens 1025fa9e4066Sahrens /* 1026fa9e4066Sahrens * We don't have to hold the mutex to check db_state because it 1027fa9e4066Sahrens * can't be freed while we have a hold on the buffer. 1028fa9e4066Sahrens */ 1029fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 1030fa9e4066Sahrens 103182c9918fSTim Haley if (db->db_state == DB_NOFILL) 1032be6fd75aSMatthew Ahrens return (SET_ERROR(EIO)); 103382c9918fSTim Haley 1034744947dcSTom Erickson DB_DNODE_ENTER(db); 1035744947dcSTom Erickson dn = DB_DNODE(db); 1036fa9e4066Sahrens if ((flags & DB_RF_HAVESTRUCT) == 0) 1037744947dcSTom Erickson rw_enter(&dn->dn_struct_rwlock, RW_READER); 1038fa9e4066Sahrens 10390a586ceaSMark Shellenbaum prefetch = db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 1040744947dcSTom Erickson (flags & DB_RF_NOPREFETCH) == 0 && dn != NULL && 10413baa08fcSek110237 DBUF_IS_CACHEABLE(db); 104213506d1eSmaybee 1043fa9e4066Sahrens mutex_enter(&db->db_mtx); 1044ea8dc4b6Seschrock if (db->db_state == DB_CACHED) { 1045ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 104613506d1eSmaybee if (prefetch) 1047cb92f413SAlexander Motin dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1, B_TRUE); 1048ea8dc4b6Seschrock if ((flags & DB_RF_HAVESTRUCT) == 0) 1049744947dcSTom Erickson rw_exit(&dn->dn_struct_rwlock); 1050744947dcSTom Erickson DB_DNODE_EXIT(db); 1051ea8dc4b6Seschrock } else if (db->db_state == DB_UNCACHED) { 1052744947dcSTom Erickson spa_t *spa = dn->dn_objset->os_spa; 1053744947dcSTom Erickson 1054744947dcSTom Erickson if (zio == NULL) 1055744947dcSTom Erickson zio = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL); 1056cf6106c8SMatthew Ahrens dbuf_read_impl(db, zio, flags); 105713506d1eSmaybee 1058ea8dc4b6Seschrock /* dbuf_read_impl has dropped db_mtx for us */ 1059ea8dc4b6Seschrock 106013506d1eSmaybee if (prefetch) 1061cb92f413SAlexander Motin dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1, B_TRUE); 1062ea8dc4b6Seschrock 1063ea8dc4b6Seschrock if ((flags & DB_RF_HAVESTRUCT) == 0) 1064744947dcSTom Erickson rw_exit(&dn->dn_struct_rwlock); 1065744947dcSTom Erickson DB_DNODE_EXIT(db); 1066ea8dc4b6Seschrock 1067ea8dc4b6Seschrock if (!havepzio) 1068ea8dc4b6Seschrock err = zio_wait(zio); 1069ea8dc4b6Seschrock } else { 10703e30c24aSWill Andrews /* 10713e30c24aSWill Andrews * Another reader came in while the dbuf was in flight 10723e30c24aSWill Andrews * between UNCACHED and CACHED. Either a writer will finish 10733e30c24aSWill Andrews * writing the buffer (sending the dbuf to CACHED) or the 10743e30c24aSWill Andrews * first reader's request will reach the read_done callback 10753e30c24aSWill Andrews * and send the dbuf to CACHED. Otherwise, a failure 10763e30c24aSWill Andrews * occurred and the dbuf went to UNCACHED. 10773e30c24aSWill Andrews */ 107813506d1eSmaybee mutex_exit(&db->db_mtx); 107913506d1eSmaybee if (prefetch) 1080cb92f413SAlexander Motin dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1, B_TRUE); 1081ea8dc4b6Seschrock if ((flags & DB_RF_HAVESTRUCT) == 0) 1082744947dcSTom Erickson rw_exit(&dn->dn_struct_rwlock); 1083744947dcSTom Erickson DB_DNODE_EXIT(db); 108413506d1eSmaybee 10853e30c24aSWill Andrews /* Skip the wait per the caller's request. */ 108613506d1eSmaybee mutex_enter(&db->db_mtx); 1087ea8dc4b6Seschrock if ((flags & DB_RF_NEVERWAIT) == 0) { 1088ea8dc4b6Seschrock while (db->db_state == DB_READ || 1089ea8dc4b6Seschrock db->db_state == DB_FILL) { 1090fa9e4066Sahrens ASSERT(db->db_state == DB_READ || 1091fa9e4066Sahrens (flags & DB_RF_HAVESTRUCT) == 0); 1092f6164ad6SAdam H. Leventhal DTRACE_PROBE2(blocked__read, dmu_buf_impl_t *, 1093f6164ad6SAdam H. Leventhal db, zio_t *, zio); 1094fa9e4066Sahrens cv_wait(&db->db_changed, &db->db_mtx); 1095fa9e4066Sahrens } 1096ea8dc4b6Seschrock if (db->db_state == DB_UNCACHED) 1097be6fd75aSMatthew Ahrens err = SET_ERROR(EIO); 1098ea8dc4b6Seschrock } 1099fa9e4066Sahrens mutex_exit(&db->db_mtx); 1100fa9e4066Sahrens } 1101fa9e4066Sahrens 1102ea8dc4b6Seschrock ASSERT(err || havepzio || db->db_state == DB_CACHED); 1103ea8dc4b6Seschrock return (err); 1104fa9e4066Sahrens } 1105fa9e4066Sahrens 1106fa9e4066Sahrens static void 1107fa9e4066Sahrens dbuf_noread(dmu_buf_impl_t *db) 1108fa9e4066Sahrens { 1109fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 11100a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1111fa9e4066Sahrens mutex_enter(&db->db_mtx); 1112fa9e4066Sahrens while (db->db_state == DB_READ || db->db_state == DB_FILL) 1113fa9e4066Sahrens cv_wait(&db->db_changed, &db->db_mtx); 1114fa9e4066Sahrens if (db->db_state == DB_UNCACHED) { 1115ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 111643466aaeSMax Grossman spa_t *spa = db->db_objset->os_spa; 1117ad23a2dbSjohansen 1118ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 1119fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 1120*dcbf3bd6SGeorge Wilson dbuf_set_data(db, arc_alloc_buf(spa, db->db.db_size, db, type)); 1121fa9e4066Sahrens db->db_state = DB_FILL; 112282c9918fSTim Haley } else if (db->db_state == DB_NOFILL) { 1123bc9014e6SJustin Gibbs dbuf_clear_data(db); 1124fa9e4066Sahrens } else { 1125fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_CACHED); 1126fa9e4066Sahrens } 1127fa9e4066Sahrens mutex_exit(&db->db_mtx); 1128fa9e4066Sahrens } 1129fa9e4066Sahrens 1130fa9e4066Sahrens /* 1131fa9e4066Sahrens * This is our just-in-time copy function. It makes a copy of 1132fa9e4066Sahrens * buffers, that have been modified in a previous transaction 1133fa9e4066Sahrens * group, before we modify them in the current active group. 1134fa9e4066Sahrens * 1135fa9e4066Sahrens * This function is used in two places: when we are dirtying a 1136fa9e4066Sahrens * buffer for the first time in a txg, and when we are freeing 1137fa9e4066Sahrens * a range in a dnode that includes this buffer. 1138fa9e4066Sahrens * 1139fa9e4066Sahrens * Note that when we are called from dbuf_free_range() we do 1140fa9e4066Sahrens * not put a hold on the buffer, we just traverse the active 1141fa9e4066Sahrens * dbuf list for the dnode. 1142fa9e4066Sahrens */ 1143fa9e4066Sahrens static void 1144fa9e4066Sahrens dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg) 1145fa9e4066Sahrens { 1146c717a561Smaybee dbuf_dirty_record_t *dr = db->db_last_dirty; 1147fa9e4066Sahrens 1148fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 1149fa9e4066Sahrens ASSERT(db->db.db_data != NULL); 1150c717a561Smaybee ASSERT(db->db_level == 0); 1151c717a561Smaybee ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT); 1152fa9e4066Sahrens 11534d31c452Smaybee if (dr == NULL || 11544d31c452Smaybee (dr->dt.dl.dr_data != 11550a586ceaSMark Shellenbaum ((db->db_blkid == DMU_BONUS_BLKID) ? db->db.db_data : db->db_buf))) 1156fa9e4066Sahrens return; 1157fa9e4066Sahrens 1158fa9e4066Sahrens /* 1159c717a561Smaybee * If the last dirty record for this dbuf has not yet synced 1160c717a561Smaybee * and its referencing the dbuf data, either: 1161c717a561Smaybee * reset the reference to point to a new copy, 1162c717a561Smaybee * or (if there a no active holders) 1163c717a561Smaybee * just null out the current db_data pointer. 1164fa9e4066Sahrens */ 1165c717a561Smaybee ASSERT(dr->dr_txg >= txg - 2); 11660a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 1167c717a561Smaybee /* Note that the data bufs here are zio_bufs */ 1168c717a561Smaybee dr->dt.dl.dr_data = zio_buf_alloc(DN_MAX_BONUSLEN); 11695a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 1170c717a561Smaybee bcopy(db->db.db_data, dr->dt.dl.dr_data, DN_MAX_BONUSLEN); 1171c717a561Smaybee } else if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 1172ea8dc4b6Seschrock int size = db->db.db_size; 1173c717a561Smaybee arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 117443466aaeSMax Grossman spa_t *spa = db->db_objset->os_spa; 1175744947dcSTom Erickson 1176*dcbf3bd6SGeorge Wilson dr->dt.dl.dr_data = arc_alloc_buf(spa, size, db, type); 1177c717a561Smaybee bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size); 1178fa9e4066Sahrens } else { 1179*dcbf3bd6SGeorge Wilson db->db_buf = NULL; 1180bc9014e6SJustin Gibbs dbuf_clear_data(db); 1181fa9e4066Sahrens } 1182fa9e4066Sahrens } 1183ea8dc4b6Seschrock 1184fa9e4066Sahrens void 1185c717a561Smaybee dbuf_unoverride(dbuf_dirty_record_t *dr) 1186fa9e4066Sahrens { 1187c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 1188b24ab676SJeff Bonwick blkptr_t *bp = &dr->dt.dl.dr_overridden_by; 1189c717a561Smaybee uint64_t txg = dr->dr_txg; 1190c5c6ffa0Smaybee 1191c717a561Smaybee ASSERT(MUTEX_HELD(&db->db_mtx)); 1192c717a561Smaybee ASSERT(dr->dt.dl.dr_override_state != DR_IN_DMU_SYNC); 1193c717a561Smaybee ASSERT(db->db_level == 0); 1194c717a561Smaybee 11950a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID || 1196c717a561Smaybee dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN) 1197c717a561Smaybee return; 1198c717a561Smaybee 1199b24ab676SJeff Bonwick ASSERT(db->db_data_pending != dr); 1200b24ab676SJeff Bonwick 1201fa9e4066Sahrens /* free this block */ 120243466aaeSMax Grossman if (!BP_IS_HOLE(bp) && !dr->dt.dl.dr_nopwrite) 120343466aaeSMax Grossman zio_free(db->db_objset->os_spa, txg, bp); 1204b24ab676SJeff Bonwick 1205c717a561Smaybee dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 120680901aeaSGeorge Wilson dr->dt.dl.dr_nopwrite = B_FALSE; 120780901aeaSGeorge Wilson 12086b4acc8bSahrens /* 12096b4acc8bSahrens * Release the already-written buffer, so we leave it in 12106b4acc8bSahrens * a consistent dirty state. Note that all callers are 12116b4acc8bSahrens * modifying the buffer, so they will immediately do 12126b4acc8bSahrens * another (redundant) arc_release(). Therefore, leave 12136b4acc8bSahrens * the buf thawed to save the effort of freezing & 12146b4acc8bSahrens * immediately re-thawing it. 12156b4acc8bSahrens */ 1216c717a561Smaybee arc_release(dr->dt.dl.dr_data, db); 1217fa9e4066Sahrens } 1218fa9e4066Sahrens 1219cdb0ab79Smaybee /* 1220cdb0ab79Smaybee * Evict (if its unreferenced) or clear (if its referenced) any level-0 1221cdb0ab79Smaybee * data blocks in the free range, so that any future readers will find 122243466aaeSMax Grossman * empty blocks. 12232f3d8780SMatthew Ahrens * 12242f3d8780SMatthew Ahrens * This is a no-op if the dataset is in the middle of an incremental 12252f3d8780SMatthew Ahrens * receive; see comment below for details. 1226cdb0ab79Smaybee */ 1227fa9e4066Sahrens void 12280f6d88adSAlex Reece dbuf_free_range(dnode_t *dn, uint64_t start_blkid, uint64_t end_blkid, 12290f6d88adSAlex Reece dmu_tx_t *tx) 1230fa9e4066Sahrens { 1231bc9014e6SJustin Gibbs dmu_buf_impl_t db_search; 1232bc9014e6SJustin Gibbs dmu_buf_impl_t *db, *db_next; 1233fa9e4066Sahrens uint64_t txg = tx->tx_txg; 12340f6d88adSAlex Reece avl_index_t where; 1235445e6780STim Chase boolean_t freespill = 1236445e6780STim Chase (start_blkid == DMU_SPILL_BLKID || end_blkid == DMU_SPILL_BLKID); 1237fa9e4066Sahrens 1238445e6780STim Chase if (end_blkid > dn->dn_maxblkid && !freespill) 12390f6d88adSAlex Reece end_blkid = dn->dn_maxblkid; 12400f6d88adSAlex Reece dprintf_dnode(dn, "start=%llu end=%llu\n", start_blkid, end_blkid); 12410f6d88adSAlex Reece 12420f6d88adSAlex Reece db_search.db_level = 0; 12430f6d88adSAlex Reece db_search.db_blkid = start_blkid; 124486bb58aeSAlex Reece db_search.db_state = DB_SEARCH; 12452f3d8780SMatthew Ahrens 1246713d6c20SMatthew Ahrens mutex_enter(&dn->dn_dbufs_mtx); 1247445e6780STim Chase if (start_blkid >= dn->dn_unlisted_l0_blkid && !freespill) { 1248713d6c20SMatthew Ahrens /* There can't be any dbufs in this range; no need to search. */ 12490f6d88adSAlex Reece #ifdef DEBUG 12500f6d88adSAlex Reece db = avl_find(&dn->dn_dbufs, &db_search, &where); 12510f6d88adSAlex Reece ASSERT3P(db, ==, NULL); 12520f6d88adSAlex Reece db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER); 12530f6d88adSAlex Reece ASSERT(db == NULL || db->db_level > 0); 12540f6d88adSAlex Reece #endif 1255713d6c20SMatthew Ahrens mutex_exit(&dn->dn_dbufs_mtx); 12562f3d8780SMatthew Ahrens return; 1257713d6c20SMatthew Ahrens } else if (dmu_objset_is_receiving(dn->dn_objset)) { 1258713d6c20SMatthew Ahrens /* 1259713d6c20SMatthew Ahrens * If we are receiving, we expect there to be no dbufs in 1260713d6c20SMatthew Ahrens * the range to be freed, because receive modifies each 1261713d6c20SMatthew Ahrens * block at most once, and in offset order. If this is 1262713d6c20SMatthew Ahrens * not the case, it can lead to performance problems, 1263713d6c20SMatthew Ahrens * so note that we unexpectedly took the slow path. 1264713d6c20SMatthew Ahrens */ 1265713d6c20SMatthew Ahrens atomic_inc_64(&zfs_free_range_recv_miss); 12662f3d8780SMatthew Ahrens } 12672f3d8780SMatthew Ahrens 12680f6d88adSAlex Reece db = avl_find(&dn->dn_dbufs, &db_search, &where); 12690f6d88adSAlex Reece ASSERT3P(db, ==, NULL); 12700f6d88adSAlex Reece db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER); 12710f6d88adSAlex Reece 12720f6d88adSAlex Reece for (; db != NULL; db = db_next) { 12730f6d88adSAlex Reece db_next = AVL_NEXT(&dn->dn_dbufs, db); 12740a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1275cdb0ab79Smaybee 12760f6d88adSAlex Reece if (db->db_level != 0 || db->db_blkid > end_blkid) { 12770f6d88adSAlex Reece break; 12780f6d88adSAlex Reece } 12790f6d88adSAlex Reece ASSERT3U(db->db_blkid, >=, start_blkid); 1280fa9e4066Sahrens 1281fa9e4066Sahrens /* found a level 0 buffer in the range */ 1282fa9e4066Sahrens mutex_enter(&db->db_mtx); 12833b2aab18SMatthew Ahrens if (dbuf_undirty(db, tx)) { 12843b2aab18SMatthew Ahrens /* mutex has been dropped and dbuf destroyed */ 12853b2aab18SMatthew Ahrens continue; 12863b2aab18SMatthew Ahrens } 12873b2aab18SMatthew Ahrens 1288ea8dc4b6Seschrock if (db->db_state == DB_UNCACHED || 128982c9918fSTim Haley db->db_state == DB_NOFILL || 1290ea8dc4b6Seschrock db->db_state == DB_EVICTING) { 1291fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 1292fa9e4066Sahrens mutex_exit(&db->db_mtx); 1293fa9e4066Sahrens continue; 1294fa9e4066Sahrens } 1295c543ec06Sahrens if (db->db_state == DB_READ || db->db_state == DB_FILL) { 1296c543ec06Sahrens /* will be handled in dbuf_read_done or dbuf_rele */ 1297c717a561Smaybee db->db_freed_in_flight = TRUE; 1298fa9e4066Sahrens mutex_exit(&db->db_mtx); 1299fa9e4066Sahrens continue; 1300fa9e4066Sahrens } 1301ea8dc4b6Seschrock if (refcount_count(&db->db_holds) == 0) { 1302ea8dc4b6Seschrock ASSERT(db->db_buf); 1303*dcbf3bd6SGeorge Wilson dbuf_destroy(db); 1304ea8dc4b6Seschrock continue; 1305ea8dc4b6Seschrock } 1306c717a561Smaybee /* The dbuf is referenced */ 1307fa9e4066Sahrens 1308c717a561Smaybee if (db->db_last_dirty != NULL) { 1309c717a561Smaybee dbuf_dirty_record_t *dr = db->db_last_dirty; 1310c717a561Smaybee 1311c717a561Smaybee if (dr->dr_txg == txg) { 1312ea8dc4b6Seschrock /* 1313c717a561Smaybee * This buffer is "in-use", re-adjust the file 1314c717a561Smaybee * size to reflect that this buffer may 1315c717a561Smaybee * contain new data when we sync. 1316ea8dc4b6Seschrock */ 131706e0070dSMark Shellenbaum if (db->db_blkid != DMU_SPILL_BLKID && 131806e0070dSMark Shellenbaum db->db_blkid > dn->dn_maxblkid) 131944eda4d7Smaybee dn->dn_maxblkid = db->db_blkid; 1320c717a561Smaybee dbuf_unoverride(dr); 1321c717a561Smaybee } else { 1322c717a561Smaybee /* 1323c717a561Smaybee * This dbuf is not dirty in the open context. 1324c717a561Smaybee * Either uncache it (if its not referenced in 1325c717a561Smaybee * the open context) or reset its contents to 1326c717a561Smaybee * empty. 1327c717a561Smaybee */ 1328c717a561Smaybee dbuf_fix_old_data(db, txg); 132944eda4d7Smaybee } 1330c717a561Smaybee } 1331c717a561Smaybee /* clear the contents if its cached */ 1332ea8dc4b6Seschrock if (db->db_state == DB_CACHED) { 1333ea8dc4b6Seschrock ASSERT(db->db.db_data != NULL); 1334fa9e4066Sahrens arc_release(db->db_buf, db); 1335fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 13366b4acc8bSahrens arc_buf_freeze(db->db_buf); 1337fa9e4066Sahrens } 1338ea8dc4b6Seschrock 1339fa9e4066Sahrens mutex_exit(&db->db_mtx); 1340fa9e4066Sahrens } 1341fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 1342fa9e4066Sahrens } 1343fa9e4066Sahrens 1344fa9e4066Sahrens static int 13451934e92fSmaybee dbuf_block_freeable(dmu_buf_impl_t *db) 1346fa9e4066Sahrens { 1347fa9e4066Sahrens dsl_dataset_t *ds = db->db_objset->os_dsl_dataset; 1348fa9e4066Sahrens uint64_t birth_txg = 0; 1349fa9e4066Sahrens 1350fa9e4066Sahrens /* 1351fa9e4066Sahrens * We don't need any locking to protect db_blkptr: 1352c717a561Smaybee * If it's syncing, then db_last_dirty will be set 1353c717a561Smaybee * so we'll ignore db_blkptr. 135443466aaeSMax Grossman * 135543466aaeSMax Grossman * This logic ensures that only block births for 135643466aaeSMax Grossman * filled blocks are considered. 1357fa9e4066Sahrens */ 1358c717a561Smaybee ASSERT(MUTEX_HELD(&db->db_mtx)); 135943466aaeSMax Grossman if (db->db_last_dirty && (db->db_blkptr == NULL || 136043466aaeSMax Grossman !BP_IS_HOLE(db->db_blkptr))) { 1361c717a561Smaybee birth_txg = db->db_last_dirty->dr_txg; 136243466aaeSMax Grossman } else if (db->db_blkptr != NULL && !BP_IS_HOLE(db->db_blkptr)) { 1363fa9e4066Sahrens birth_txg = db->db_blkptr->blk_birth; 136443466aaeSMax Grossman } 1365fa9e4066Sahrens 1366837b568bSGeorge Wilson /* 136743466aaeSMax Grossman * If this block don't exist or is in a snapshot, it can't be freed. 1368837b568bSGeorge Wilson * Don't pass the bp to dsl_dataset_block_freeable() since we 1369837b568bSGeorge Wilson * are holding the db_mtx lock and might deadlock if we are 1370837b568bSGeorge Wilson * prefetching a dedup-ed block. 1371837b568bSGeorge Wilson */ 137243466aaeSMax Grossman if (birth_txg != 0) 13731934e92fSmaybee return (ds == NULL || 1374837b568bSGeorge Wilson dsl_dataset_block_freeable(ds, NULL, birth_txg)); 1375fa9e4066Sahrens else 137643466aaeSMax Grossman return (B_FALSE); 1377fa9e4066Sahrens } 1378fa9e4066Sahrens 1379fa9e4066Sahrens void 1380fa9e4066Sahrens dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx) 1381fa9e4066Sahrens { 1382fa9e4066Sahrens arc_buf_t *buf, *obuf; 1383fa9e4066Sahrens int osize = db->db.db_size; 1384ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 1385744947dcSTom Erickson dnode_t *dn; 1386fa9e4066Sahrens 13870a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1388ea8dc4b6Seschrock 1389744947dcSTom Erickson DB_DNODE_ENTER(db); 1390744947dcSTom Erickson dn = DB_DNODE(db); 1391744947dcSTom Erickson 1392fa9e4066Sahrens /* XXX does *this* func really need the lock? */ 1393744947dcSTom Erickson ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 1394fa9e4066Sahrens 1395fa9e4066Sahrens /* 139643466aaeSMax Grossman * This call to dmu_buf_will_dirty() with the dn_struct_rwlock held 1397fa9e4066Sahrens * is OK, because there can be no other references to the db 1398fa9e4066Sahrens * when we are changing its size, so no concurrent DB_FILL can 1399fa9e4066Sahrens * be happening. 1400fa9e4066Sahrens */ 1401ea8dc4b6Seschrock /* 1402ea8dc4b6Seschrock * XXX we should be doing a dbuf_read, checking the return 1403ea8dc4b6Seschrock * value and returning that up to our callers 1404ea8dc4b6Seschrock */ 140543466aaeSMax Grossman dmu_buf_will_dirty(&db->db, tx); 1406fa9e4066Sahrens 1407fa9e4066Sahrens /* create the data buffer for the new block */ 1408*dcbf3bd6SGeorge Wilson buf = arc_alloc_buf(dn->dn_objset->os_spa, size, db, type); 1409fa9e4066Sahrens 1410fa9e4066Sahrens /* copy old block data to the new block */ 1411fa9e4066Sahrens obuf = db->db_buf; 1412f65e61c0Sahrens bcopy(obuf->b_data, buf->b_data, MIN(osize, size)); 1413fa9e4066Sahrens /* zero the remainder */ 1414f65e61c0Sahrens if (size > osize) 1415fa9e4066Sahrens bzero((uint8_t *)buf->b_data + osize, size - osize); 1416fa9e4066Sahrens 1417fa9e4066Sahrens mutex_enter(&db->db_mtx); 1418fa9e4066Sahrens dbuf_set_data(db, buf); 1419*dcbf3bd6SGeorge Wilson arc_buf_destroy(obuf, db); 1420fa9e4066Sahrens db->db.db_size = size; 1421fa9e4066Sahrens 1422c717a561Smaybee if (db->db_level == 0) { 1423c717a561Smaybee ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg); 1424c717a561Smaybee db->db_last_dirty->dt.dl.dr_data = buf; 1425c717a561Smaybee } 1426fa9e4066Sahrens mutex_exit(&db->db_mtx); 1427fa9e4066Sahrens 1428744947dcSTom Erickson dnode_willuse_space(dn, size-osize, tx); 1429744947dcSTom Erickson DB_DNODE_EXIT(db); 1430fa9e4066Sahrens } 1431fa9e4066Sahrens 14323f9d6ad7SLin Ling void 14333f9d6ad7SLin Ling dbuf_release_bp(dmu_buf_impl_t *db) 14343f9d6ad7SLin Ling { 143543466aaeSMax Grossman objset_t *os = db->db_objset; 14363f9d6ad7SLin Ling 14373f9d6ad7SLin Ling ASSERT(dsl_pool_sync_context(dmu_objset_pool(os))); 14383f9d6ad7SLin Ling ASSERT(arc_released(os->os_phys_buf) || 14393f9d6ad7SLin Ling list_link_active(&os->os_dsl_dataset->ds_synced_link)); 14403f9d6ad7SLin Ling ASSERT(db->db_parent == NULL || arc_released(db->db_parent->db_buf)); 14413f9d6ad7SLin Ling 14421b912ec7SGeorge Wilson (void) arc_release(db->db_buf, db); 14433f9d6ad7SLin Ling } 14443f9d6ad7SLin Ling 14450f2e7d03SMatthew Ahrens /* 14460f2e7d03SMatthew Ahrens * We already have a dirty record for this TXG, and we are being 14470f2e7d03SMatthew Ahrens * dirtied again. 14480f2e7d03SMatthew Ahrens */ 14490f2e7d03SMatthew Ahrens static void 14500f2e7d03SMatthew Ahrens dbuf_redirty(dbuf_dirty_record_t *dr) 14510f2e7d03SMatthew Ahrens { 14520f2e7d03SMatthew Ahrens dmu_buf_impl_t *db = dr->dr_dbuf; 14530f2e7d03SMatthew Ahrens 14540f2e7d03SMatthew Ahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 14550f2e7d03SMatthew Ahrens 14560f2e7d03SMatthew Ahrens if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID) { 14570f2e7d03SMatthew Ahrens /* 14580f2e7d03SMatthew Ahrens * If this buffer has already been written out, 14590f2e7d03SMatthew Ahrens * we now need to reset its state. 14600f2e7d03SMatthew Ahrens */ 14610f2e7d03SMatthew Ahrens dbuf_unoverride(dr); 14620f2e7d03SMatthew Ahrens if (db->db.db_object != DMU_META_DNODE_OBJECT && 14630f2e7d03SMatthew Ahrens db->db_state != DB_NOFILL) { 14640f2e7d03SMatthew Ahrens /* Already released on initial dirty, so just thaw. */ 14650f2e7d03SMatthew Ahrens ASSERT(arc_released(db->db_buf)); 14660f2e7d03SMatthew Ahrens arc_buf_thaw(db->db_buf); 14670f2e7d03SMatthew Ahrens } 14680f2e7d03SMatthew Ahrens } 14690f2e7d03SMatthew Ahrens } 14700f2e7d03SMatthew Ahrens 1471c717a561Smaybee dbuf_dirty_record_t * 1472fa9e4066Sahrens dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1473fa9e4066Sahrens { 1474744947dcSTom Erickson dnode_t *dn; 1475744947dcSTom Erickson objset_t *os; 1476c717a561Smaybee dbuf_dirty_record_t **drp, *dr; 1477fa9e4066Sahrens int drop_struct_lock = FALSE; 1478d3469faaSMark Maybee boolean_t do_free_accounting = B_FALSE; 1479fa9e4066Sahrens int txgoff = tx->tx_txg & TXG_MASK; 1480fa9e4066Sahrens 1481fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 1482fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 14839c9dc39aSek110237 DMU_TX_DIRTY_BUF(tx, db); 1484fa9e4066Sahrens 1485744947dcSTom Erickson DB_DNODE_ENTER(db); 1486744947dcSTom Erickson dn = DB_DNODE(db); 1487fa9e4066Sahrens /* 1488fa9e4066Sahrens * Shouldn't dirty a regular buffer in syncing context. Private 1489fa9e4066Sahrens * objects may be dirtied in syncing context, but only if they 1490fa9e4066Sahrens * were already pre-dirtied in open context. 1491fa9e4066Sahrens */ 1492c717a561Smaybee ASSERT(!dmu_tx_is_syncing(tx) || 1493c717a561Smaybee BP_IS_HOLE(dn->dn_objset->os_rootbp) || 149414843421SMatthew Ahrens DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 149514843421SMatthew Ahrens dn->dn_objset->os_dsl_dataset == NULL); 1496fa9e4066Sahrens /* 1497fa9e4066Sahrens * We make this assert for private objects as well, but after we 1498fa9e4066Sahrens * check if we're already dirty. They are allowed to re-dirty 1499fa9e4066Sahrens * in syncing context. 1500fa9e4066Sahrens */ 1501ea8dc4b6Seschrock ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 1502c717a561Smaybee dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 1503fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 1504fa9e4066Sahrens 1505fa9e4066Sahrens mutex_enter(&db->db_mtx); 1506fa9e4066Sahrens /* 1507c717a561Smaybee * XXX make this true for indirects too? The problem is that 1508c717a561Smaybee * transactions created with dmu_tx_create_assigned() from 1509c717a561Smaybee * syncing context don't bother holding ahead. 1510fa9e4066Sahrens */ 1511c717a561Smaybee ASSERT(db->db_level != 0 || 151282c9918fSTim Haley db->db_state == DB_CACHED || db->db_state == DB_FILL || 151382c9918fSTim Haley db->db_state == DB_NOFILL); 1514fa9e4066Sahrens 1515fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 1516fa9e4066Sahrens /* 1517fa9e4066Sahrens * Don't set dirtyctx to SYNC if we're just modifying this as we 1518fa9e4066Sahrens * initialize the objset. 1519fa9e4066Sahrens */ 1520fa9e4066Sahrens if (dn->dn_dirtyctx == DN_UNDIRTIED && 1521c717a561Smaybee !BP_IS_HOLE(dn->dn_objset->os_rootbp)) { 1522fa9e4066Sahrens dn->dn_dirtyctx = 1523fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN); 1524fa9e4066Sahrens ASSERT(dn->dn_dirtyctx_firstset == NULL); 1525fa9e4066Sahrens dn->dn_dirtyctx_firstset = kmem_alloc(1, KM_SLEEP); 1526fa9e4066Sahrens } 1527fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1528fa9e4066Sahrens 15290a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) 15300a586ceaSMark Shellenbaum dn->dn_have_spill = B_TRUE; 15310a586ceaSMark Shellenbaum 1532fa9e4066Sahrens /* 1533fa9e4066Sahrens * If this buffer is already dirty, we're done. 1534fa9e4066Sahrens */ 1535c717a561Smaybee drp = &db->db_last_dirty; 1536c717a561Smaybee ASSERT(*drp == NULL || (*drp)->dr_txg <= tx->tx_txg || 1537c717a561Smaybee db->db.db_object == DMU_META_DNODE_OBJECT); 15387e2186e3Sbonwick while ((dr = *drp) != NULL && dr->dr_txg > tx->tx_txg) 15397e2186e3Sbonwick drp = &dr->dr_next; 15407e2186e3Sbonwick if (dr && dr->dr_txg == tx->tx_txg) { 1541744947dcSTom Erickson DB_DNODE_EXIT(db); 1542744947dcSTom Erickson 15430f2e7d03SMatthew Ahrens dbuf_redirty(dr); 1544fa9e4066Sahrens mutex_exit(&db->db_mtx); 15457e2186e3Sbonwick return (dr); 1546fa9e4066Sahrens } 1547fa9e4066Sahrens 1548fa9e4066Sahrens /* 1549fa9e4066Sahrens * Only valid if not already dirty. 1550fa9e4066Sahrens */ 155114843421SMatthew Ahrens ASSERT(dn->dn_object == 0 || 155214843421SMatthew Ahrens dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 1553fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 1554fa9e4066Sahrens 1555fa9e4066Sahrens ASSERT3U(dn->dn_nlevels, >, db->db_level); 1556fa9e4066Sahrens ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) || 1557fa9e4066Sahrens dn->dn_phys->dn_nlevels > db->db_level || 1558fa9e4066Sahrens dn->dn_next_nlevels[txgoff] > db->db_level || 1559fa9e4066Sahrens dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level || 1560fa9e4066Sahrens dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level); 1561fa9e4066Sahrens 1562fa9e4066Sahrens /* 1563fa9e4066Sahrens * We should only be dirtying in syncing context if it's the 156414843421SMatthew Ahrens * mos or we're initializing the os or it's a special object. 156514843421SMatthew Ahrens * However, we are allowed to dirty in syncing context provided 156614843421SMatthew Ahrens * we already dirtied it in open context. Hence we must make 156714843421SMatthew Ahrens * this assertion only if we're not already dirty. 1568fa9e4066Sahrens */ 1569744947dcSTom Erickson os = dn->dn_objset; 157014843421SMatthew Ahrens ASSERT(!dmu_tx_is_syncing(tx) || DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 157114843421SMatthew Ahrens os->os_dsl_dataset == NULL || BP_IS_HOLE(os->os_rootbp)); 1572fa9e4066Sahrens ASSERT(db->db.db_size != 0); 1573fa9e4066Sahrens 1574fa9e4066Sahrens dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1575fa9e4066Sahrens 15760a586ceaSMark Shellenbaum if (db->db_blkid != DMU_BONUS_BLKID) { 15771934e92fSmaybee /* 15781934e92fSmaybee * Update the accounting. 1579d3469faaSMark Maybee * Note: we delay "free accounting" until after we drop 1580d3469faaSMark Maybee * the db_mtx. This keeps us from grabbing other locks 1581b24ab676SJeff Bonwick * (and possibly deadlocking) in bp_get_dsize() while 1582d3469faaSMark Maybee * also holding the db_mtx. 15831934e92fSmaybee */ 15841934e92fSmaybee dnode_willuse_space(dn, db->db.db_size, tx); 1585d3469faaSMark Maybee do_free_accounting = dbuf_block_freeable(db); 15861934e92fSmaybee } 15871934e92fSmaybee 1588ea8dc4b6Seschrock /* 1589ea8dc4b6Seschrock * If this buffer is dirty in an old transaction group we need 1590ea8dc4b6Seschrock * to make a copy of it so that the changes we make in this 1591ea8dc4b6Seschrock * transaction group won't leak out when we sync the older txg. 1592ea8dc4b6Seschrock */ 1593c717a561Smaybee dr = kmem_zalloc(sizeof (dbuf_dirty_record_t), KM_SLEEP); 1594c717a561Smaybee if (db->db_level == 0) { 1595c717a561Smaybee void *data_old = db->db_buf; 1596c717a561Smaybee 159782c9918fSTim Haley if (db->db_state != DB_NOFILL) { 15980a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 1599c717a561Smaybee dbuf_fix_old_data(db, tx->tx_txg); 1600c717a561Smaybee data_old = db->db.db_data; 1601c717a561Smaybee } else if (db->db.db_object != DMU_META_DNODE_OBJECT) { 1602fa9e4066Sahrens /* 160382c9918fSTim Haley * Release the data buffer from the cache so 160482c9918fSTim Haley * that we can modify it without impacting 160582c9918fSTim Haley * possible other users of this cached data 160682c9918fSTim Haley * block. Note that indirect blocks and 160782c9918fSTim Haley * private objects are not released until the 160882c9918fSTim Haley * syncing state (since they are only modified 160982c9918fSTim Haley * then). 1610fa9e4066Sahrens */ 1611fa9e4066Sahrens arc_release(db->db_buf, db); 1612fa9e4066Sahrens dbuf_fix_old_data(db, tx->tx_txg); 1613c717a561Smaybee data_old = db->db_buf; 1614fa9e4066Sahrens } 1615c717a561Smaybee ASSERT(data_old != NULL); 161682c9918fSTim Haley } 1617c717a561Smaybee dr->dt.dl.dr_data = data_old; 1618c717a561Smaybee } else { 1619c717a561Smaybee mutex_init(&dr->dt.di.dr_mtx, NULL, MUTEX_DEFAULT, NULL); 1620c717a561Smaybee list_create(&dr->dt.di.dr_children, 1621c717a561Smaybee sizeof (dbuf_dirty_record_t), 1622c717a561Smaybee offsetof(dbuf_dirty_record_t, dr_dirty_node)); 1623fa9e4066Sahrens } 162469962b56SMatthew Ahrens if (db->db_blkid != DMU_BONUS_BLKID && os->os_dsl_dataset != NULL) 162569962b56SMatthew Ahrens dr->dr_accounted = db->db.db_size; 1626c717a561Smaybee dr->dr_dbuf = db; 1627c717a561Smaybee dr->dr_txg = tx->tx_txg; 1628c717a561Smaybee dr->dr_next = *drp; 1629c717a561Smaybee *drp = dr; 1630fa9e4066Sahrens 1631fa9e4066Sahrens /* 1632fa9e4066Sahrens * We could have been freed_in_flight between the dbuf_noread 1633fa9e4066Sahrens * and dbuf_dirty. We win, as though the dbuf_noread() had 1634fa9e4066Sahrens * happened after the free. 1635fa9e4066Sahrens */ 16360a586ceaSMark Shellenbaum if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 16370a586ceaSMark Shellenbaum db->db_blkid != DMU_SPILL_BLKID) { 1638c717a561Smaybee mutex_enter(&dn->dn_mtx); 1639bf16b11eSMatthew Ahrens if (dn->dn_free_ranges[txgoff] != NULL) { 1640bf16b11eSMatthew Ahrens range_tree_clear(dn->dn_free_ranges[txgoff], 1641bf16b11eSMatthew Ahrens db->db_blkid, 1); 1642bf16b11eSMatthew Ahrens } 1643fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1644c717a561Smaybee db->db_freed_in_flight = FALSE; 1645c717a561Smaybee } 1646fa9e4066Sahrens 1647fa9e4066Sahrens /* 1648fa9e4066Sahrens * This buffer is now part of this txg 1649fa9e4066Sahrens */ 1650fa9e4066Sahrens dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg); 1651fa9e4066Sahrens db->db_dirtycnt += 1; 1652fa9e4066Sahrens ASSERT3U(db->db_dirtycnt, <=, 3); 1653fa9e4066Sahrens 1654fa9e4066Sahrens mutex_exit(&db->db_mtx); 1655fa9e4066Sahrens 16560a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID || 16570a586ceaSMark Shellenbaum db->db_blkid == DMU_SPILL_BLKID) { 1658c717a561Smaybee mutex_enter(&dn->dn_mtx); 1659c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 1660c717a561Smaybee list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1661c717a561Smaybee mutex_exit(&dn->dn_mtx); 1662fa9e4066Sahrens dnode_setdirty(dn, tx); 1663744947dcSTom Erickson DB_DNODE_EXIT(db); 1664c717a561Smaybee return (dr); 1665d3469faaSMark Maybee } else if (do_free_accounting) { 1666d3469faaSMark Maybee blkptr_t *bp = db->db_blkptr; 1667d3469faaSMark Maybee int64_t willfree = (bp && !BP_IS_HOLE(bp)) ? 1668b24ab676SJeff Bonwick bp_get_dsize(os->os_spa, bp) : db->db.db_size; 1669d3469faaSMark Maybee /* 1670d3469faaSMark Maybee * This is only a guess -- if the dbuf is dirty 1671d3469faaSMark Maybee * in a previous txg, we don't know how much 1672d3469faaSMark Maybee * space it will use on disk yet. We should 1673d3469faaSMark Maybee * really have the struct_rwlock to access 1674d3469faaSMark Maybee * db_blkptr, but since this is just a guess, 1675d3469faaSMark Maybee * it's OK if we get an odd answer. 1676d3469faaSMark Maybee */ 1677837b568bSGeorge Wilson ddt_prefetch(os->os_spa, bp); 1678d3469faaSMark Maybee dnode_willuse_space(dn, -willfree, tx); 1679fa9e4066Sahrens } 1680fa9e4066Sahrens 1681fa9e4066Sahrens if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 1682fa9e4066Sahrens rw_enter(&dn->dn_struct_rwlock, RW_READER); 1683fa9e4066Sahrens drop_struct_lock = TRUE; 1684fa9e4066Sahrens } 1685fa9e4066Sahrens 16868346f03fSJonathan W Adams if (db->db_level == 0) { 16878346f03fSJonathan W Adams dnode_new_blkid(dn, db->db_blkid, tx, drop_struct_lock); 16888346f03fSJonathan W Adams ASSERT(dn->dn_maxblkid >= db->db_blkid); 16898346f03fSJonathan W Adams } 16908346f03fSJonathan W Adams 169144eda4d7Smaybee if (db->db_level+1 < dn->dn_nlevels) { 1692c717a561Smaybee dmu_buf_impl_t *parent = db->db_parent; 1693c717a561Smaybee dbuf_dirty_record_t *di; 1694c717a561Smaybee int parent_held = FALSE; 1695c717a561Smaybee 1696c717a561Smaybee if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) { 1697fa9e4066Sahrens int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1698c717a561Smaybee 1699fa9e4066Sahrens parent = dbuf_hold_level(dn, db->db_level+1, 1700fa9e4066Sahrens db->db_blkid >> epbs, FTAG); 170101025c89SJohn Harres ASSERT(parent != NULL); 1702c717a561Smaybee parent_held = TRUE; 1703c717a561Smaybee } 1704fa9e4066Sahrens if (drop_struct_lock) 1705fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 1706c717a561Smaybee ASSERT3U(db->db_level+1, ==, parent->db_level); 1707c717a561Smaybee di = dbuf_dirty(parent, tx); 1708c717a561Smaybee if (parent_held) 1709ea8dc4b6Seschrock dbuf_rele(parent, FTAG); 1710c717a561Smaybee 1711c717a561Smaybee mutex_enter(&db->db_mtx); 171269962b56SMatthew Ahrens /* 171369962b56SMatthew Ahrens * Since we've dropped the mutex, it's possible that 171469962b56SMatthew Ahrens * dbuf_undirty() might have changed this out from under us. 171569962b56SMatthew Ahrens */ 1716c717a561Smaybee if (db->db_last_dirty == dr || 1717c717a561Smaybee dn->dn_object == DMU_META_DNODE_OBJECT) { 1718c717a561Smaybee mutex_enter(&di->dt.di.dr_mtx); 1719c717a561Smaybee ASSERT3U(di->dr_txg, ==, tx->tx_txg); 1720c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 1721c717a561Smaybee list_insert_tail(&di->dt.di.dr_children, dr); 1722c717a561Smaybee mutex_exit(&di->dt.di.dr_mtx); 1723c717a561Smaybee dr->dr_parent = di; 1724c717a561Smaybee } 1725c717a561Smaybee mutex_exit(&db->db_mtx); 1726fa9e4066Sahrens } else { 1727c717a561Smaybee ASSERT(db->db_level+1 == dn->dn_nlevels); 1728c717a561Smaybee ASSERT(db->db_blkid < dn->dn_nblkptr); 1729744947dcSTom Erickson ASSERT(db->db_parent == NULL || db->db_parent == dn->dn_dbuf); 1730c717a561Smaybee mutex_enter(&dn->dn_mtx); 1731c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 1732c717a561Smaybee list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1733c717a561Smaybee mutex_exit(&dn->dn_mtx); 1734fa9e4066Sahrens if (drop_struct_lock) 1735fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 1736fa9e4066Sahrens } 1737fa9e4066Sahrens 1738fa9e4066Sahrens dnode_setdirty(dn, tx); 1739744947dcSTom Erickson DB_DNODE_EXIT(db); 1740c717a561Smaybee return (dr); 1741fa9e4066Sahrens } 1742fa9e4066Sahrens 17433b2aab18SMatthew Ahrens /* 17443e30c24aSWill Andrews * Undirty a buffer in the transaction group referenced by the given 17453e30c24aSWill Andrews * transaction. Return whether this evicted the dbuf. 17463b2aab18SMatthew Ahrens */ 17473b2aab18SMatthew Ahrens static boolean_t 1748fa9e4066Sahrens dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1749fa9e4066Sahrens { 1750744947dcSTom Erickson dnode_t *dn; 1751c717a561Smaybee uint64_t txg = tx->tx_txg; 175217f17c2dSbonwick dbuf_dirty_record_t *dr, **drp; 1753fa9e4066Sahrens 1754c717a561Smaybee ASSERT(txg != 0); 175546e1baa6SMatthew Ahrens 175646e1baa6SMatthew Ahrens /* 175746e1baa6SMatthew Ahrens * Due to our use of dn_nlevels below, this can only be called 175846e1baa6SMatthew Ahrens * in open context, unless we are operating on the MOS. 175946e1baa6SMatthew Ahrens * From syncing context, dn_nlevels may be different from the 176046e1baa6SMatthew Ahrens * dn_nlevels used when dbuf was dirtied. 176146e1baa6SMatthew Ahrens */ 176246e1baa6SMatthew Ahrens ASSERT(db->db_objset == 176346e1baa6SMatthew Ahrens dmu_objset_pool(db->db_objset)->dp_meta_objset || 176446e1baa6SMatthew Ahrens txg != spa_syncing_txg(dmu_objset_spa(db->db_objset))); 17650a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 17663b2aab18SMatthew Ahrens ASSERT0(db->db_level); 17673b2aab18SMatthew Ahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 1768fa9e4066Sahrens 1769fa9e4066Sahrens /* 1770fa9e4066Sahrens * If this buffer is not dirty, we're done. 1771fa9e4066Sahrens */ 177217f17c2dSbonwick for (drp = &db->db_last_dirty; (dr = *drp) != NULL; drp = &dr->dr_next) 1773c717a561Smaybee if (dr->dr_txg <= txg) 1774c717a561Smaybee break; 17753b2aab18SMatthew Ahrens if (dr == NULL || dr->dr_txg < txg) 17763b2aab18SMatthew Ahrens return (B_FALSE); 1777c717a561Smaybee ASSERT(dr->dr_txg == txg); 1778b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 1779fa9e4066Sahrens 1780744947dcSTom Erickson DB_DNODE_ENTER(db); 1781744947dcSTom Erickson dn = DB_DNODE(db); 1782744947dcSTom Erickson 1783fa9e4066Sahrens dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1784fa9e4066Sahrens 1785fa9e4066Sahrens ASSERT(db->db.db_size != 0); 1786fa9e4066Sahrens 178746e1baa6SMatthew Ahrens dsl_pool_undirty_space(dmu_objset_pool(dn->dn_objset), 178846e1baa6SMatthew Ahrens dr->dr_accounted, txg); 1789fa9e4066Sahrens 179017f17c2dSbonwick *drp = dr->dr_next; 1791c717a561Smaybee 17923f2366c2SGordon Ross /* 17933f2366c2SGordon Ross * Note that there are three places in dbuf_dirty() 17943f2366c2SGordon Ross * where this dirty record may be put on a list. 17953f2366c2SGordon Ross * Make sure to do a list_remove corresponding to 17963f2366c2SGordon Ross * every one of those list_insert calls. 17973f2366c2SGordon Ross */ 1798c717a561Smaybee if (dr->dr_parent) { 1799c717a561Smaybee mutex_enter(&dr->dr_parent->dt.di.dr_mtx); 1800c717a561Smaybee list_remove(&dr->dr_parent->dt.di.dr_children, dr); 1801c717a561Smaybee mutex_exit(&dr->dr_parent->dt.di.dr_mtx); 18023f2366c2SGordon Ross } else if (db->db_blkid == DMU_SPILL_BLKID || 18033f2366c2SGordon Ross db->db_level + 1 == dn->dn_nlevels) { 1804cdb0ab79Smaybee ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf); 1805fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 1806c717a561Smaybee list_remove(&dn->dn_dirty_records[txg & TXG_MASK], dr); 1807fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1808c717a561Smaybee } 1809744947dcSTom Erickson DB_DNODE_EXIT(db); 1810c717a561Smaybee 181182c9918fSTim Haley if (db->db_state != DB_NOFILL) { 1812c717a561Smaybee dbuf_unoverride(dr); 1813c717a561Smaybee 1814c717a561Smaybee ASSERT(db->db_buf != NULL); 1815c717a561Smaybee ASSERT(dr->dt.dl.dr_data != NULL); 1816c717a561Smaybee if (dr->dt.dl.dr_data != db->db_buf) 1817*dcbf3bd6SGeorge Wilson arc_buf_destroy(dr->dt.dl.dr_data, db); 1818c717a561Smaybee } 1819d2b3cbbdSJorgen Lundman 1820c717a561Smaybee kmem_free(dr, sizeof (dbuf_dirty_record_t)); 1821fa9e4066Sahrens 1822fa9e4066Sahrens ASSERT(db->db_dirtycnt > 0); 1823fa9e4066Sahrens db->db_dirtycnt -= 1; 1824fa9e4066Sahrens 1825c717a561Smaybee if (refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) { 1826*dcbf3bd6SGeorge Wilson ASSERT(db->db_state == DB_NOFILL || arc_released(db->db_buf)); 1827*dcbf3bd6SGeorge Wilson dbuf_destroy(db); 18283b2aab18SMatthew Ahrens return (B_TRUE); 1829fa9e4066Sahrens } 1830fa9e4066Sahrens 18313b2aab18SMatthew Ahrens return (B_FALSE); 1832fa9e4066Sahrens } 1833fa9e4066Sahrens 1834fa9e4066Sahrens void 183543466aaeSMax Grossman dmu_buf_will_dirty(dmu_buf_t *db_fake, dmu_tx_t *tx) 1836fa9e4066Sahrens { 183743466aaeSMax Grossman dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 18381ab7f2deSmaybee int rf = DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH; 1839fa9e4066Sahrens 1840fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 1841fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 1842fa9e4066Sahrens 18430f2e7d03SMatthew Ahrens /* 18440f2e7d03SMatthew Ahrens * Quick check for dirtyness. For already dirty blocks, this 18450f2e7d03SMatthew Ahrens * reduces runtime of this function by >90%, and overall performance 18460f2e7d03SMatthew Ahrens * by 50% for some workloads (e.g. file deletion with indirect blocks 18470f2e7d03SMatthew Ahrens * cached). 18480f2e7d03SMatthew Ahrens */ 18490f2e7d03SMatthew Ahrens mutex_enter(&db->db_mtx); 18500f2e7d03SMatthew Ahrens dbuf_dirty_record_t *dr; 18510f2e7d03SMatthew Ahrens for (dr = db->db_last_dirty; 18520f2e7d03SMatthew Ahrens dr != NULL && dr->dr_txg >= tx->tx_txg; dr = dr->dr_next) { 18530f2e7d03SMatthew Ahrens /* 18540f2e7d03SMatthew Ahrens * It's possible that it is already dirty but not cached, 18550f2e7d03SMatthew Ahrens * because there are some calls to dbuf_dirty() that don't 18560f2e7d03SMatthew Ahrens * go through dmu_buf_will_dirty(). 18570f2e7d03SMatthew Ahrens */ 18580f2e7d03SMatthew Ahrens if (dr->dr_txg == tx->tx_txg && db->db_state == DB_CACHED) { 18590f2e7d03SMatthew Ahrens /* This dbuf is already dirty and cached. */ 18600f2e7d03SMatthew Ahrens dbuf_redirty(dr); 18610f2e7d03SMatthew Ahrens mutex_exit(&db->db_mtx); 18620f2e7d03SMatthew Ahrens return; 18630f2e7d03SMatthew Ahrens } 18640f2e7d03SMatthew Ahrens } 18650f2e7d03SMatthew Ahrens mutex_exit(&db->db_mtx); 18660f2e7d03SMatthew Ahrens 1867744947dcSTom Erickson DB_DNODE_ENTER(db); 1868744947dcSTom Erickson if (RW_WRITE_HELD(&DB_DNODE(db)->dn_struct_rwlock)) 1869fa9e4066Sahrens rf |= DB_RF_HAVESTRUCT; 1870744947dcSTom Erickson DB_DNODE_EXIT(db); 1871ea8dc4b6Seschrock (void) dbuf_read(db, NULL, rf); 1872c717a561Smaybee (void) dbuf_dirty(db, tx); 1873fa9e4066Sahrens } 1874fa9e4066Sahrens 1875fa9e4066Sahrens void 187682c9918fSTim Haley dmu_buf_will_not_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 187782c9918fSTim Haley { 187882c9918fSTim Haley dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 187982c9918fSTim Haley 188082c9918fSTim Haley db->db_state = DB_NOFILL; 188182c9918fSTim Haley 188282c9918fSTim Haley dmu_buf_will_fill(db_fake, tx); 188382c9918fSTim Haley } 188482c9918fSTim Haley 188582c9918fSTim Haley void 1886ea8dc4b6Seschrock dmu_buf_will_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 1887fa9e4066Sahrens { 1888ea8dc4b6Seschrock dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1889ea8dc4b6Seschrock 18900a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1891fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 1892fa9e4066Sahrens ASSERT(db->db_level == 0); 1893fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 1894fa9e4066Sahrens 1895ea8dc4b6Seschrock ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT || 1896fa9e4066Sahrens dmu_tx_private_ok(tx)); 1897fa9e4066Sahrens 1898fa9e4066Sahrens dbuf_noread(db); 1899c717a561Smaybee (void) dbuf_dirty(db, tx); 1900fa9e4066Sahrens } 1901fa9e4066Sahrens 1902fa9e4066Sahrens #pragma weak dmu_buf_fill_done = dbuf_fill_done 1903fa9e4066Sahrens /* ARGSUSED */ 1904fa9e4066Sahrens void 1905fa9e4066Sahrens dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx) 1906fa9e4066Sahrens { 1907fa9e4066Sahrens mutex_enter(&db->db_mtx); 19089c9dc39aSek110237 DBUF_VERIFY(db); 1909fa9e4066Sahrens 1910fa9e4066Sahrens if (db->db_state == DB_FILL) { 1911c717a561Smaybee if (db->db_level == 0 && db->db_freed_in_flight) { 19120a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1913fa9e4066Sahrens /* we were freed while filling */ 1914fa9e4066Sahrens /* XXX dbuf_undirty? */ 1915fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 1916c717a561Smaybee db->db_freed_in_flight = FALSE; 1917fa9e4066Sahrens } 1918fa9e4066Sahrens db->db_state = DB_CACHED; 1919fa9e4066Sahrens cv_broadcast(&db->db_changed); 1920fa9e4066Sahrens } 1921fa9e4066Sahrens mutex_exit(&db->db_mtx); 1922fa9e4066Sahrens } 1923fa9e4066Sahrens 19245d7b4d43SMatthew Ahrens void 19255d7b4d43SMatthew Ahrens dmu_buf_write_embedded(dmu_buf_t *dbuf, void *data, 19265d7b4d43SMatthew Ahrens bp_embedded_type_t etype, enum zio_compress comp, 19275d7b4d43SMatthew Ahrens int uncompressed_size, int compressed_size, int byteorder, 19285d7b4d43SMatthew Ahrens dmu_tx_t *tx) 19295d7b4d43SMatthew Ahrens { 19305d7b4d43SMatthew Ahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf; 19315d7b4d43SMatthew Ahrens struct dirty_leaf *dl; 19325d7b4d43SMatthew Ahrens dmu_object_type_t type; 19335d7b4d43SMatthew Ahrens 1934ca0cc391SMatthew Ahrens if (etype == BP_EMBEDDED_TYPE_DATA) { 1935ca0cc391SMatthew Ahrens ASSERT(spa_feature_is_active(dmu_objset_spa(db->db_objset), 1936ca0cc391SMatthew Ahrens SPA_FEATURE_EMBEDDED_DATA)); 1937ca0cc391SMatthew Ahrens } 1938ca0cc391SMatthew Ahrens 19395d7b4d43SMatthew Ahrens DB_DNODE_ENTER(db); 19405d7b4d43SMatthew Ahrens type = DB_DNODE(db)->dn_type; 19415d7b4d43SMatthew Ahrens DB_DNODE_EXIT(db); 19425d7b4d43SMatthew Ahrens 19435d7b4d43SMatthew Ahrens ASSERT0(db->db_level); 19445d7b4d43SMatthew Ahrens ASSERT(db->db_blkid != DMU_BONUS_BLKID); 19455d7b4d43SMatthew Ahrens 19465d7b4d43SMatthew Ahrens dmu_buf_will_not_fill(dbuf, tx); 19475d7b4d43SMatthew Ahrens 19485d7b4d43SMatthew Ahrens ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg); 19495d7b4d43SMatthew Ahrens dl = &db->db_last_dirty->dt.dl; 19505d7b4d43SMatthew Ahrens encode_embedded_bp_compressed(&dl->dr_overridden_by, 19515d7b4d43SMatthew Ahrens data, comp, uncompressed_size, compressed_size); 19525d7b4d43SMatthew Ahrens BPE_SET_ETYPE(&dl->dr_overridden_by, etype); 19535d7b4d43SMatthew Ahrens BP_SET_TYPE(&dl->dr_overridden_by, type); 19545d7b4d43SMatthew Ahrens BP_SET_LEVEL(&dl->dr_overridden_by, 0); 19555d7b4d43SMatthew Ahrens BP_SET_BYTEORDER(&dl->dr_overridden_by, byteorder); 19565d7b4d43SMatthew Ahrens 19575d7b4d43SMatthew Ahrens dl->dr_override_state = DR_OVERRIDDEN; 19585d7b4d43SMatthew Ahrens dl->dr_overridden_by.blk_birth = db->db_last_dirty->dr_txg; 19595d7b4d43SMatthew Ahrens } 19605d7b4d43SMatthew Ahrens 1961ea8dc4b6Seschrock /* 19622fdbea25SAleksandr Guzovskiy * Directly assign a provided arc buf to a given dbuf if it's not referenced 19632fdbea25SAleksandr Guzovskiy * by anybody except our caller. Otherwise copy arcbuf's contents to dbuf. 19642fdbea25SAleksandr Guzovskiy */ 19652fdbea25SAleksandr Guzovskiy void 19662fdbea25SAleksandr Guzovskiy dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx) 19672fdbea25SAleksandr Guzovskiy { 19682fdbea25SAleksandr Guzovskiy ASSERT(!refcount_is_zero(&db->db_holds)); 19690a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 19702fdbea25SAleksandr Guzovskiy ASSERT(db->db_level == 0); 19712fdbea25SAleksandr Guzovskiy ASSERT(DBUF_GET_BUFC_TYPE(db) == ARC_BUFC_DATA); 19722fdbea25SAleksandr Guzovskiy ASSERT(buf != NULL); 19732fdbea25SAleksandr Guzovskiy ASSERT(arc_buf_size(buf) == db->db.db_size); 19742fdbea25SAleksandr Guzovskiy ASSERT(tx->tx_txg != 0); 19752fdbea25SAleksandr Guzovskiy 19762fdbea25SAleksandr Guzovskiy arc_return_buf(buf, db); 19772fdbea25SAleksandr Guzovskiy ASSERT(arc_released(buf)); 19782fdbea25SAleksandr Guzovskiy 19792fdbea25SAleksandr Guzovskiy mutex_enter(&db->db_mtx); 19802fdbea25SAleksandr Guzovskiy 19812fdbea25SAleksandr Guzovskiy while (db->db_state == DB_READ || db->db_state == DB_FILL) 19822fdbea25SAleksandr Guzovskiy cv_wait(&db->db_changed, &db->db_mtx); 19832fdbea25SAleksandr Guzovskiy 19842fdbea25SAleksandr Guzovskiy ASSERT(db->db_state == DB_CACHED || db->db_state == DB_UNCACHED); 19852fdbea25SAleksandr Guzovskiy 19862fdbea25SAleksandr Guzovskiy if (db->db_state == DB_CACHED && 19872fdbea25SAleksandr Guzovskiy refcount_count(&db->db_holds) - 1 > db->db_dirtycnt) { 19882fdbea25SAleksandr Guzovskiy mutex_exit(&db->db_mtx); 19892fdbea25SAleksandr Guzovskiy (void) dbuf_dirty(db, tx); 19902fdbea25SAleksandr Guzovskiy bcopy(buf->b_data, db->db.db_data, db->db.db_size); 1991*dcbf3bd6SGeorge Wilson arc_buf_destroy(buf, db); 1992c242f9a0Schunli zhang - Sun Microsystems - Irvine United States xuio_stat_wbuf_copied(); 19932fdbea25SAleksandr Guzovskiy return; 19942fdbea25SAleksandr Guzovskiy } 19952fdbea25SAleksandr Guzovskiy 1996c242f9a0Schunli zhang - Sun Microsystems - Irvine United States xuio_stat_wbuf_nocopy(); 19972fdbea25SAleksandr Guzovskiy if (db->db_state == DB_CACHED) { 19982fdbea25SAleksandr Guzovskiy dbuf_dirty_record_t *dr = db->db_last_dirty; 19992fdbea25SAleksandr Guzovskiy 20002fdbea25SAleksandr Guzovskiy ASSERT(db->db_buf != NULL); 20012fdbea25SAleksandr Guzovskiy if (dr != NULL && dr->dr_txg == tx->tx_txg) { 20022fdbea25SAleksandr Guzovskiy ASSERT(dr->dt.dl.dr_data == db->db_buf); 20032fdbea25SAleksandr Guzovskiy if (!arc_released(db->db_buf)) { 20042fdbea25SAleksandr Guzovskiy ASSERT(dr->dt.dl.dr_override_state == 20052fdbea25SAleksandr Guzovskiy DR_OVERRIDDEN); 20062fdbea25SAleksandr Guzovskiy arc_release(db->db_buf, db); 20072fdbea25SAleksandr Guzovskiy } 20082fdbea25SAleksandr Guzovskiy dr->dt.dl.dr_data = buf; 2009*dcbf3bd6SGeorge Wilson arc_buf_destroy(db->db_buf, db); 20102fdbea25SAleksandr Guzovskiy } else if (dr == NULL || dr->dt.dl.dr_data != db->db_buf) { 20112fdbea25SAleksandr Guzovskiy arc_release(db->db_buf, db); 2012*dcbf3bd6SGeorge Wilson arc_buf_destroy(db->db_buf, db); 20132fdbea25SAleksandr Guzovskiy } 20142fdbea25SAleksandr Guzovskiy db->db_buf = NULL; 20152fdbea25SAleksandr Guzovskiy } 20162fdbea25SAleksandr Guzovskiy ASSERT(db->db_buf == NULL); 20172fdbea25SAleksandr Guzovskiy dbuf_set_data(db, buf); 20182fdbea25SAleksandr Guzovskiy db->db_state = DB_FILL; 20192fdbea25SAleksandr Guzovskiy mutex_exit(&db->db_mtx); 20202fdbea25SAleksandr Guzovskiy (void) dbuf_dirty(db, tx); 202143466aaeSMax Grossman dmu_buf_fill_done(&db->db, tx); 20222fdbea25SAleksandr Guzovskiy } 20232fdbea25SAleksandr Guzovskiy 2024ea8dc4b6Seschrock void 2025*dcbf3bd6SGeorge Wilson dbuf_destroy(dmu_buf_impl_t *db) 2026fa9e4066Sahrens { 2027744947dcSTom Erickson dnode_t *dn; 2028ea8dc4b6Seschrock dmu_buf_impl_t *parent = db->db_parent; 2029744947dcSTom Erickson dmu_buf_impl_t *dndb; 2030fa9e4066Sahrens 2031fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 2032fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 2033fa9e4066Sahrens 2034*dcbf3bd6SGeorge Wilson if (db->db_buf != NULL) { 2035*dcbf3bd6SGeorge Wilson arc_buf_destroy(db->db_buf, db); 2036*dcbf3bd6SGeorge Wilson db->db_buf = NULL; 2037*dcbf3bd6SGeorge Wilson } 2038ea8dc4b6Seschrock 20390a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 2040*dcbf3bd6SGeorge Wilson ASSERT(db->db.db_data != NULL); 2041ea8dc4b6Seschrock zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN); 20425a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 2043fa9e4066Sahrens db->db_state = DB_UNCACHED; 2044fa9e4066Sahrens } 2045fa9e4066Sahrens 2046*dcbf3bd6SGeorge Wilson dbuf_clear_data(db); 2047*dcbf3bd6SGeorge Wilson 2048*dcbf3bd6SGeorge Wilson if (multilist_link_active(&db->db_cache_link)) { 2049*dcbf3bd6SGeorge Wilson multilist_remove(&dbuf_cache, db); 2050*dcbf3bd6SGeorge Wilson (void) refcount_remove_many(&dbuf_cache_size, 2051*dcbf3bd6SGeorge Wilson db->db.db_size, db); 2052*dcbf3bd6SGeorge Wilson } 2053*dcbf3bd6SGeorge Wilson 205482c9918fSTim Haley ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL); 2055fa9e4066Sahrens ASSERT(db->db_data_pending == NULL); 2056fa9e4066Sahrens 2057ea8dc4b6Seschrock db->db_state = DB_EVICTING; 2058ea8dc4b6Seschrock db->db_blkptr = NULL; 2059ea8dc4b6Seschrock 2060*dcbf3bd6SGeorge Wilson /* 2061*dcbf3bd6SGeorge Wilson * Now that db_state is DB_EVICTING, nobody else can find this via 2062*dcbf3bd6SGeorge Wilson * the hash table. We can now drop db_mtx, which allows us to 2063*dcbf3bd6SGeorge Wilson * acquire the dn_dbufs_mtx. 2064*dcbf3bd6SGeorge Wilson */ 2065*dcbf3bd6SGeorge Wilson mutex_exit(&db->db_mtx); 2066*dcbf3bd6SGeorge Wilson 2067744947dcSTom Erickson DB_DNODE_ENTER(db); 2068744947dcSTom Erickson dn = DB_DNODE(db); 2069744947dcSTom Erickson dndb = dn->dn_dbuf; 2070*dcbf3bd6SGeorge Wilson if (db->db_blkid != DMU_BONUS_BLKID) { 2071*dcbf3bd6SGeorge Wilson boolean_t needlock = !MUTEX_HELD(&dn->dn_dbufs_mtx); 2072*dcbf3bd6SGeorge Wilson if (needlock) 2073*dcbf3bd6SGeorge Wilson mutex_enter(&dn->dn_dbufs_mtx); 20740f6d88adSAlex Reece avl_remove(&dn->dn_dbufs, db); 2075640c1670SJosef 'Jeff' Sipek atomic_dec_32(&dn->dn_dbufs_count); 2076744947dcSTom Erickson membar_producer(); 2077744947dcSTom Erickson DB_DNODE_EXIT(db); 2078*dcbf3bd6SGeorge Wilson if (needlock) 2079*dcbf3bd6SGeorge Wilson mutex_exit(&dn->dn_dbufs_mtx); 2080744947dcSTom Erickson /* 2081744947dcSTom Erickson * Decrementing the dbuf count means that the hold corresponding 2082744947dcSTom Erickson * to the removed dbuf is no longer discounted in dnode_move(), 2083744947dcSTom Erickson * so the dnode cannot be moved until after we release the hold. 2084744947dcSTom Erickson * The membar_producer() ensures visibility of the decremented 2085744947dcSTom Erickson * value in dnode_move(), since DB_DNODE_EXIT doesn't actually 2086744947dcSTom Erickson * release any lock. 2087744947dcSTom Erickson */ 2088ea8dc4b6Seschrock dnode_rele(dn, db); 2089744947dcSTom Erickson db->db_dnode_handle = NULL; 2090*dcbf3bd6SGeorge Wilson 2091*dcbf3bd6SGeorge Wilson dbuf_hash_remove(db); 2092744947dcSTom Erickson } else { 2093744947dcSTom Erickson DB_DNODE_EXIT(db); 2094ea8dc4b6Seschrock } 2095ea8dc4b6Seschrock 2096*dcbf3bd6SGeorge Wilson ASSERT(refcount_is_zero(&db->db_holds)); 2097ea8dc4b6Seschrock 2098*dcbf3bd6SGeorge Wilson db->db_parent = NULL; 2099*dcbf3bd6SGeorge Wilson 2100*dcbf3bd6SGeorge Wilson ASSERT(db->db_buf == NULL); 2101*dcbf3bd6SGeorge Wilson ASSERT(db->db.db_data == NULL); 2102*dcbf3bd6SGeorge Wilson ASSERT(db->db_hash_next == NULL); 2103*dcbf3bd6SGeorge Wilson ASSERT(db->db_blkptr == NULL); 2104*dcbf3bd6SGeorge Wilson ASSERT(db->db_data_pending == NULL); 2105*dcbf3bd6SGeorge Wilson ASSERT(!multilist_link_active(&db->db_cache_link)); 2106*dcbf3bd6SGeorge Wilson 2107*dcbf3bd6SGeorge Wilson kmem_cache_free(dbuf_kmem_cache, db); 2108*dcbf3bd6SGeorge Wilson arc_space_return(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 2109fa9e4066Sahrens 2110fa9e4066Sahrens /* 2111744947dcSTom Erickson * If this dbuf is referenced from an indirect dbuf, 2112fa9e4066Sahrens * decrement the ref count on the indirect dbuf. 2113fa9e4066Sahrens */ 2114c543ec06Sahrens if (parent && parent != dndb) 2115ea8dc4b6Seschrock dbuf_rele(parent, db); 2116fa9e4066Sahrens } 2117fa9e4066Sahrens 2118a2cdcdd2SPaul Dagnelie /* 2119a2cdcdd2SPaul Dagnelie * Note: While bpp will always be updated if the function returns success, 2120a2cdcdd2SPaul Dagnelie * parentp will not be updated if the dnode does not have dn_dbuf filled in; 2121a2cdcdd2SPaul Dagnelie * this happens when the dnode is the meta-dnode, or a userused or groupused 2122a2cdcdd2SPaul Dagnelie * object. 2123a2cdcdd2SPaul Dagnelie */ 2124fa9e4066Sahrens static int 2125fa9e4066Sahrens dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse, 2126fa9e4066Sahrens dmu_buf_impl_t **parentp, blkptr_t **bpp) 2127fa9e4066Sahrens { 2128fa9e4066Sahrens int nlevels, epbs; 2129fa9e4066Sahrens 21300b69c2f0Sahrens *parentp = NULL; 21310b69c2f0Sahrens *bpp = NULL; 21320b69c2f0Sahrens 21330a586ceaSMark Shellenbaum ASSERT(blkid != DMU_BONUS_BLKID); 21340a586ceaSMark Shellenbaum 21350a586ceaSMark Shellenbaum if (blkid == DMU_SPILL_BLKID) { 21360a586ceaSMark Shellenbaum mutex_enter(&dn->dn_mtx); 213706e0070dSMark Shellenbaum if (dn->dn_have_spill && 213806e0070dSMark Shellenbaum (dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR)) 21390a586ceaSMark Shellenbaum *bpp = &dn->dn_phys->dn_spill; 21400a586ceaSMark Shellenbaum else 21410a586ceaSMark Shellenbaum *bpp = NULL; 21420a586ceaSMark Shellenbaum dbuf_add_ref(dn->dn_dbuf, NULL); 21430a586ceaSMark Shellenbaum *parentp = dn->dn_dbuf; 21440a586ceaSMark Shellenbaum mutex_exit(&dn->dn_mtx); 21450a586ceaSMark Shellenbaum return (0); 21460a586ceaSMark Shellenbaum } 2147ea8dc4b6Seschrock 2148fa9e4066Sahrens if (dn->dn_phys->dn_nlevels == 0) 2149fa9e4066Sahrens nlevels = 1; 2150fa9e4066Sahrens else 2151fa9e4066Sahrens nlevels = dn->dn_phys->dn_nlevels; 2152fa9e4066Sahrens 2153fa9e4066Sahrens epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 2154fa9e4066Sahrens 2155fa9e4066Sahrens ASSERT3U(level * epbs, <, 64); 2156fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 2157ea8dc4b6Seschrock if (level >= nlevels || 2158fa9e4066Sahrens (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) { 2159fa9e4066Sahrens /* the buffer has no parent yet */ 2160be6fd75aSMatthew Ahrens return (SET_ERROR(ENOENT)); 2161fa9e4066Sahrens } else if (level < nlevels-1) { 2162fa9e4066Sahrens /* this block is referenced from an indirect block */ 2163fa9e4066Sahrens int err = dbuf_hold_impl(dn, level+1, 2164a2cdcdd2SPaul Dagnelie blkid >> epbs, fail_sparse, FALSE, NULL, parentp); 2165fa9e4066Sahrens if (err) 2166fa9e4066Sahrens return (err); 2167ea8dc4b6Seschrock err = dbuf_read(*parentp, NULL, 2168ea8dc4b6Seschrock (DB_RF_HAVESTRUCT | DB_RF_NOPREFETCH | DB_RF_CANFAIL)); 2169c543ec06Sahrens if (err) { 2170c543ec06Sahrens dbuf_rele(*parentp, NULL); 2171c543ec06Sahrens *parentp = NULL; 2172c543ec06Sahrens return (err); 2173c543ec06Sahrens } 2174fa9e4066Sahrens *bpp = ((blkptr_t *)(*parentp)->db.db_data) + 2175fa9e4066Sahrens (blkid & ((1ULL << epbs) - 1)); 2176c543ec06Sahrens return (0); 2177fa9e4066Sahrens } else { 2178fa9e4066Sahrens /* the block is referenced from the dnode */ 2179fa9e4066Sahrens ASSERT3U(level, ==, nlevels-1); 2180fa9e4066Sahrens ASSERT(dn->dn_phys->dn_nblkptr == 0 || 2181fa9e4066Sahrens blkid < dn->dn_phys->dn_nblkptr); 2182c543ec06Sahrens if (dn->dn_dbuf) { 2183c543ec06Sahrens dbuf_add_ref(dn->dn_dbuf, NULL); 2184fa9e4066Sahrens *parentp = dn->dn_dbuf; 2185c543ec06Sahrens } 2186fa9e4066Sahrens *bpp = &dn->dn_phys->dn_blkptr[blkid]; 2187fa9e4066Sahrens return (0); 2188fa9e4066Sahrens } 2189fa9e4066Sahrens } 2190fa9e4066Sahrens 2191fa9e4066Sahrens static dmu_buf_impl_t * 2192fa9e4066Sahrens dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid, 2193fa9e4066Sahrens dmu_buf_impl_t *parent, blkptr_t *blkptr) 2194fa9e4066Sahrens { 2195503ad85cSMatthew Ahrens objset_t *os = dn->dn_objset; 2196fa9e4066Sahrens dmu_buf_impl_t *db, *odb; 2197fa9e4066Sahrens 2198fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 2199fa9e4066Sahrens ASSERT(dn->dn_type != DMU_OT_NONE); 2200fa9e4066Sahrens 2201*dcbf3bd6SGeorge Wilson db = kmem_cache_alloc(dbuf_kmem_cache, KM_SLEEP); 2202fa9e4066Sahrens 2203fa9e4066Sahrens db->db_objset = os; 2204fa9e4066Sahrens db->db.db_object = dn->dn_object; 2205fa9e4066Sahrens db->db_level = level; 2206fa9e4066Sahrens db->db_blkid = blkid; 2207c717a561Smaybee db->db_last_dirty = NULL; 2208ea8dc4b6Seschrock db->db_dirtycnt = 0; 2209744947dcSTom Erickson db->db_dnode_handle = dn->dn_handle; 2210ea8dc4b6Seschrock db->db_parent = parent; 2211ea8dc4b6Seschrock db->db_blkptr = blkptr; 2212fa9e4066Sahrens 2213bc9014e6SJustin Gibbs db->db_user = NULL; 2214d2058105SJustin T. Gibbs db->db_user_immediate_evict = FALSE; 2215d2058105SJustin T. Gibbs db->db_freed_in_flight = FALSE; 2216d2058105SJustin T. Gibbs db->db_pending_evict = FALSE; 2217ea8dc4b6Seschrock 22180a586ceaSMark Shellenbaum if (blkid == DMU_BONUS_BLKID) { 2219ea8dc4b6Seschrock ASSERT3P(parent, ==, dn->dn_dbuf); 22201934e92fSmaybee db->db.db_size = DN_MAX_BONUSLEN - 22211934e92fSmaybee (dn->dn_nblkptr-1) * sizeof (blkptr_t); 22221934e92fSmaybee ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 22230a586ceaSMark Shellenbaum db->db.db_offset = DMU_BONUS_BLKID; 2224ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 2225ea8dc4b6Seschrock /* the bonus dbuf is not placed in the hash table */ 22265a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 2227ea8dc4b6Seschrock return (db); 22280a586ceaSMark Shellenbaum } else if (blkid == DMU_SPILL_BLKID) { 22290a586ceaSMark Shellenbaum db->db.db_size = (blkptr != NULL) ? 22300a586ceaSMark Shellenbaum BP_GET_LSIZE(blkptr) : SPA_MINBLOCKSIZE; 22310a586ceaSMark Shellenbaum db->db.db_offset = 0; 2232fa9e4066Sahrens } else { 2233fa9e4066Sahrens int blocksize = 2234fa9e4066Sahrens db->db_level ? 1 << dn->dn_indblkshift : dn->dn_datablksz; 2235fa9e4066Sahrens db->db.db_size = blocksize; 2236fa9e4066Sahrens db->db.db_offset = db->db_blkid * blocksize; 2237fa9e4066Sahrens } 2238fa9e4066Sahrens 2239fa9e4066Sahrens /* 2240fa9e4066Sahrens * Hold the dn_dbufs_mtx while we get the new dbuf 2241fa9e4066Sahrens * in the hash table *and* added to the dbufs list. 2242fa9e4066Sahrens * This prevents a possible deadlock with someone 2243fa9e4066Sahrens * trying to look up this dbuf before its added to the 2244fa9e4066Sahrens * dn_dbufs list. 2245fa9e4066Sahrens */ 2246fa9e4066Sahrens mutex_enter(&dn->dn_dbufs_mtx); 2247ea8dc4b6Seschrock db->db_state = DB_EVICTING; 2248fa9e4066Sahrens if ((odb = dbuf_hash_insert(db)) != NULL) { 2249fa9e4066Sahrens /* someone else inserted it first */ 2250*dcbf3bd6SGeorge Wilson kmem_cache_free(dbuf_kmem_cache, db); 2251fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 2252fa9e4066Sahrens return (odb); 2253fa9e4066Sahrens } 22540f6d88adSAlex Reece avl_add(&dn->dn_dbufs, db); 2255713d6c20SMatthew Ahrens if (db->db_level == 0 && db->db_blkid >= 2256713d6c20SMatthew Ahrens dn->dn_unlisted_l0_blkid) 2257713d6c20SMatthew Ahrens dn->dn_unlisted_l0_blkid = db->db_blkid + 1; 2258ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 2259fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 22605a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 2261fa9e4066Sahrens 2262fa9e4066Sahrens if (parent && parent != dn->dn_dbuf) 2263fa9e4066Sahrens dbuf_add_ref(parent, db); 2264fa9e4066Sahrens 2265ea8dc4b6Seschrock ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 2266ea8dc4b6Seschrock refcount_count(&dn->dn_holds) > 0); 2267fa9e4066Sahrens (void) refcount_add(&dn->dn_holds, db); 2268640c1670SJosef 'Jeff' Sipek atomic_inc_32(&dn->dn_dbufs_count); 2269fa9e4066Sahrens 2270fa9e4066Sahrens dprintf_dbuf(db, "db=%p\n", db); 2271fa9e4066Sahrens 2272fa9e4066Sahrens return (db); 2273fa9e4066Sahrens } 2274fa9e4066Sahrens 2275a2cdcdd2SPaul Dagnelie typedef struct dbuf_prefetch_arg { 2276a2cdcdd2SPaul Dagnelie spa_t *dpa_spa; /* The spa to issue the prefetch in. */ 2277a2cdcdd2SPaul Dagnelie zbookmark_phys_t dpa_zb; /* The target block to prefetch. */ 2278a2cdcdd2SPaul Dagnelie int dpa_epbs; /* Entries (blkptr_t's) Per Block Shift. */ 2279a2cdcdd2SPaul Dagnelie int dpa_curlevel; /* The current level that we're reading */ 2280*dcbf3bd6SGeorge Wilson dnode_t *dpa_dnode; /* The dnode associated with the prefetch */ 2281a2cdcdd2SPaul Dagnelie zio_priority_t dpa_prio; /* The priority I/Os should be issued at. */ 2282a2cdcdd2SPaul Dagnelie zio_t *dpa_zio; /* The parent zio_t for all prefetches. */ 2283a2cdcdd2SPaul Dagnelie arc_flags_t dpa_aflags; /* Flags to pass to the final prefetch. */ 2284a2cdcdd2SPaul Dagnelie } dbuf_prefetch_arg_t; 2285a2cdcdd2SPaul Dagnelie 2286a2cdcdd2SPaul Dagnelie /* 2287a2cdcdd2SPaul Dagnelie * Actually issue the prefetch read for the block given. 2288a2cdcdd2SPaul Dagnelie */ 2289a2cdcdd2SPaul Dagnelie static void 2290a2cdcdd2SPaul Dagnelie dbuf_issue_final_prefetch(dbuf_prefetch_arg_t *dpa, blkptr_t *bp) 2291fa9e4066Sahrens { 2292a2cdcdd2SPaul Dagnelie if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp)) 2293a2cdcdd2SPaul Dagnelie return; 2294a2cdcdd2SPaul Dagnelie 2295a2cdcdd2SPaul Dagnelie arc_flags_t aflags = 2296a2cdcdd2SPaul Dagnelie dpa->dpa_aflags | ARC_FLAG_NOWAIT | ARC_FLAG_PREFETCH; 2297a2cdcdd2SPaul Dagnelie 2298a2cdcdd2SPaul Dagnelie ASSERT3U(dpa->dpa_curlevel, ==, BP_GET_LEVEL(bp)); 2299a2cdcdd2SPaul Dagnelie ASSERT3U(dpa->dpa_curlevel, ==, dpa->dpa_zb.zb_level); 2300a2cdcdd2SPaul Dagnelie ASSERT(dpa->dpa_zio != NULL); 2301a2cdcdd2SPaul Dagnelie (void) arc_read(dpa->dpa_zio, dpa->dpa_spa, bp, NULL, NULL, 2302a2cdcdd2SPaul Dagnelie dpa->dpa_prio, ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 2303a2cdcdd2SPaul Dagnelie &aflags, &dpa->dpa_zb); 2304a2cdcdd2SPaul Dagnelie } 2305a2cdcdd2SPaul Dagnelie 2306a2cdcdd2SPaul Dagnelie /* 2307a2cdcdd2SPaul Dagnelie * Called when an indirect block above our prefetch target is read in. This 2308a2cdcdd2SPaul Dagnelie * will either read in the next indirect block down the tree or issue the actual 2309a2cdcdd2SPaul Dagnelie * prefetch if the next block down is our target. 2310a2cdcdd2SPaul Dagnelie */ 2311a2cdcdd2SPaul Dagnelie static void 2312a2cdcdd2SPaul Dagnelie dbuf_prefetch_indirect_done(zio_t *zio, arc_buf_t *abuf, void *private) 2313a2cdcdd2SPaul Dagnelie { 2314a2cdcdd2SPaul Dagnelie dbuf_prefetch_arg_t *dpa = private; 2315a2cdcdd2SPaul Dagnelie 2316a2cdcdd2SPaul Dagnelie ASSERT3S(dpa->dpa_zb.zb_level, <, dpa->dpa_curlevel); 2317a2cdcdd2SPaul Dagnelie ASSERT3S(dpa->dpa_curlevel, >, 0); 2318*dcbf3bd6SGeorge Wilson 2319*dcbf3bd6SGeorge Wilson /* 2320*dcbf3bd6SGeorge Wilson * The dpa_dnode is only valid if we are called with a NULL 2321*dcbf3bd6SGeorge Wilson * zio. This indicates that the arc_read() returned without 2322*dcbf3bd6SGeorge Wilson * first calling zio_read() to issue a physical read. Once 2323*dcbf3bd6SGeorge Wilson * a physical read is made the dpa_dnode must be invalidated 2324*dcbf3bd6SGeorge Wilson * as the locks guarding it may have been dropped. If the 2325*dcbf3bd6SGeorge Wilson * dpa_dnode is still valid, then we want to add it to the dbuf 2326*dcbf3bd6SGeorge Wilson * cache. To do so, we must hold the dbuf associated with the block 2327*dcbf3bd6SGeorge Wilson * we just prefetched, read its contents so that we associate it 2328*dcbf3bd6SGeorge Wilson * with an arc_buf_t, and then release it. 2329*dcbf3bd6SGeorge Wilson */ 2330a2cdcdd2SPaul Dagnelie if (zio != NULL) { 2331a2cdcdd2SPaul Dagnelie ASSERT3S(BP_GET_LEVEL(zio->io_bp), ==, dpa->dpa_curlevel); 2332*dcbf3bd6SGeorge Wilson if (zio->io_flags & ZIO_FLAG_RAW) { 2333*dcbf3bd6SGeorge Wilson ASSERT3U(BP_GET_PSIZE(zio->io_bp), ==, zio->io_size); 2334*dcbf3bd6SGeorge Wilson } else { 2335a2cdcdd2SPaul Dagnelie ASSERT3U(BP_GET_LSIZE(zio->io_bp), ==, zio->io_size); 2336*dcbf3bd6SGeorge Wilson } 2337a2cdcdd2SPaul Dagnelie ASSERT3P(zio->io_spa, ==, dpa->dpa_spa); 2338*dcbf3bd6SGeorge Wilson 2339*dcbf3bd6SGeorge Wilson dpa->dpa_dnode = NULL; 2340*dcbf3bd6SGeorge Wilson } else if (dpa->dpa_dnode != NULL) { 2341*dcbf3bd6SGeorge Wilson uint64_t curblkid = dpa->dpa_zb.zb_blkid >> 2342*dcbf3bd6SGeorge Wilson (dpa->dpa_epbs * (dpa->dpa_curlevel - 2343*dcbf3bd6SGeorge Wilson dpa->dpa_zb.zb_level)); 2344*dcbf3bd6SGeorge Wilson dmu_buf_impl_t *db = dbuf_hold_level(dpa->dpa_dnode, 2345*dcbf3bd6SGeorge Wilson dpa->dpa_curlevel, curblkid, FTAG); 2346*dcbf3bd6SGeorge Wilson (void) dbuf_read(db, NULL, 2347*dcbf3bd6SGeorge Wilson DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH | DB_RF_HAVESTRUCT); 2348*dcbf3bd6SGeorge Wilson dbuf_rele(db, FTAG); 2349a2cdcdd2SPaul Dagnelie } 2350a2cdcdd2SPaul Dagnelie 2351a2cdcdd2SPaul Dagnelie dpa->dpa_curlevel--; 2352a2cdcdd2SPaul Dagnelie 2353a2cdcdd2SPaul Dagnelie uint64_t nextblkid = dpa->dpa_zb.zb_blkid >> 2354a2cdcdd2SPaul Dagnelie (dpa->dpa_epbs * (dpa->dpa_curlevel - dpa->dpa_zb.zb_level)); 2355a2cdcdd2SPaul Dagnelie blkptr_t *bp = ((blkptr_t *)abuf->b_data) + 2356a2cdcdd2SPaul Dagnelie P2PHASE(nextblkid, 1ULL << dpa->dpa_epbs); 2357a2cdcdd2SPaul Dagnelie if (BP_IS_HOLE(bp) || (zio != NULL && zio->io_error != 0)) { 2358a2cdcdd2SPaul Dagnelie kmem_free(dpa, sizeof (*dpa)); 2359a2cdcdd2SPaul Dagnelie } else if (dpa->dpa_curlevel == dpa->dpa_zb.zb_level) { 2360a2cdcdd2SPaul Dagnelie ASSERT3U(nextblkid, ==, dpa->dpa_zb.zb_blkid); 2361a2cdcdd2SPaul Dagnelie dbuf_issue_final_prefetch(dpa, bp); 2362a2cdcdd2SPaul Dagnelie kmem_free(dpa, sizeof (*dpa)); 2363a2cdcdd2SPaul Dagnelie } else { 2364a2cdcdd2SPaul Dagnelie arc_flags_t iter_aflags = ARC_FLAG_NOWAIT; 2365a2cdcdd2SPaul Dagnelie zbookmark_phys_t zb; 2366a2cdcdd2SPaul Dagnelie 2367a2cdcdd2SPaul Dagnelie ASSERT3U(dpa->dpa_curlevel, ==, BP_GET_LEVEL(bp)); 2368a2cdcdd2SPaul Dagnelie 2369a2cdcdd2SPaul Dagnelie SET_BOOKMARK(&zb, dpa->dpa_zb.zb_objset, 2370a2cdcdd2SPaul Dagnelie dpa->dpa_zb.zb_object, dpa->dpa_curlevel, nextblkid); 2371a2cdcdd2SPaul Dagnelie 2372a2cdcdd2SPaul Dagnelie (void) arc_read(dpa->dpa_zio, dpa->dpa_spa, 2373a2cdcdd2SPaul Dagnelie bp, dbuf_prefetch_indirect_done, dpa, dpa->dpa_prio, 2374a2cdcdd2SPaul Dagnelie ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 2375a2cdcdd2SPaul Dagnelie &iter_aflags, &zb); 2376a2cdcdd2SPaul Dagnelie } 2377*dcbf3bd6SGeorge Wilson 2378*dcbf3bd6SGeorge Wilson arc_buf_destroy(abuf, private); 2379a2cdcdd2SPaul Dagnelie } 2380a2cdcdd2SPaul Dagnelie 2381a2cdcdd2SPaul Dagnelie /* 2382a2cdcdd2SPaul Dagnelie * Issue prefetch reads for the given block on the given level. If the indirect 2383a2cdcdd2SPaul Dagnelie * blocks above that block are not in memory, we will read them in 2384a2cdcdd2SPaul Dagnelie * asynchronously. As a result, this call never blocks waiting for a read to 2385a2cdcdd2SPaul Dagnelie * complete. 2386a2cdcdd2SPaul Dagnelie */ 2387a2cdcdd2SPaul Dagnelie void 2388a2cdcdd2SPaul Dagnelie dbuf_prefetch(dnode_t *dn, int64_t level, uint64_t blkid, zio_priority_t prio, 2389a2cdcdd2SPaul Dagnelie arc_flags_t aflags) 2390a2cdcdd2SPaul Dagnelie { 2391a2cdcdd2SPaul Dagnelie blkptr_t bp; 2392a2cdcdd2SPaul Dagnelie int epbs, nlevels, curlevel; 2393a2cdcdd2SPaul Dagnelie uint64_t curblkid; 2394fa9e4066Sahrens 23950a586ceaSMark Shellenbaum ASSERT(blkid != DMU_BONUS_BLKID); 2396fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 2397fa9e4066Sahrens 2398cf6106c8SMatthew Ahrens if (blkid > dn->dn_maxblkid) 2399cf6106c8SMatthew Ahrens return; 2400cf6106c8SMatthew Ahrens 2401fa9e4066Sahrens if (dnode_block_freed(dn, blkid)) 2402fa9e4066Sahrens return; 2403fa9e4066Sahrens 2404fa9e4066Sahrens /* 2405a2cdcdd2SPaul Dagnelie * This dnode hasn't been written to disk yet, so there's nothing to 2406a2cdcdd2SPaul Dagnelie * prefetch. 2407fa9e4066Sahrens */ 2408a2cdcdd2SPaul Dagnelie nlevels = dn->dn_phys->dn_nlevels; 2409a2cdcdd2SPaul Dagnelie if (level >= nlevels || dn->dn_phys->dn_nblkptr == 0) 2410a2cdcdd2SPaul Dagnelie return; 2411a2cdcdd2SPaul Dagnelie 2412a2cdcdd2SPaul Dagnelie epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2413a2cdcdd2SPaul Dagnelie if (dn->dn_phys->dn_maxblkid < blkid << (epbs * level)) 2414a2cdcdd2SPaul Dagnelie return; 2415a2cdcdd2SPaul Dagnelie 2416a2cdcdd2SPaul Dagnelie dmu_buf_impl_t *db = dbuf_find(dn->dn_objset, dn->dn_object, 2417a2cdcdd2SPaul Dagnelie level, blkid); 2418a2cdcdd2SPaul Dagnelie if (db != NULL) { 2419fa9e4066Sahrens mutex_exit(&db->db_mtx); 2420a2cdcdd2SPaul Dagnelie /* 2421a2cdcdd2SPaul Dagnelie * This dbuf already exists. It is either CACHED, or 2422a2cdcdd2SPaul Dagnelie * (we assume) about to be read or filled. 2423a2cdcdd2SPaul Dagnelie */ 2424fa9e4066Sahrens return; 2425fa9e4066Sahrens } 2426fa9e4066Sahrens 2427a2cdcdd2SPaul Dagnelie /* 2428a2cdcdd2SPaul Dagnelie * Find the closest ancestor (indirect block) of the target block 2429a2cdcdd2SPaul Dagnelie * that is present in the cache. In this indirect block, we will 2430a2cdcdd2SPaul Dagnelie * find the bp that is at curlevel, curblkid. 2431a2cdcdd2SPaul Dagnelie */ 2432a2cdcdd2SPaul Dagnelie curlevel = level; 2433a2cdcdd2SPaul Dagnelie curblkid = blkid; 2434a2cdcdd2SPaul Dagnelie while (curlevel < nlevels - 1) { 2435a2cdcdd2SPaul Dagnelie int parent_level = curlevel + 1; 2436a2cdcdd2SPaul Dagnelie uint64_t parent_blkid = curblkid >> epbs; 2437a2cdcdd2SPaul Dagnelie dmu_buf_impl_t *db; 2438a2cdcdd2SPaul Dagnelie 2439a2cdcdd2SPaul Dagnelie if (dbuf_hold_impl(dn, parent_level, parent_blkid, 2440a2cdcdd2SPaul Dagnelie FALSE, TRUE, FTAG, &db) == 0) { 2441a2cdcdd2SPaul Dagnelie blkptr_t *bpp = db->db_buf->b_data; 2442a2cdcdd2SPaul Dagnelie bp = bpp[P2PHASE(curblkid, 1 << epbs)]; 2443a2cdcdd2SPaul Dagnelie dbuf_rele(db, FTAG); 2444a2cdcdd2SPaul Dagnelie break; 2445a2cdcdd2SPaul Dagnelie } 2446a2cdcdd2SPaul Dagnelie 2447a2cdcdd2SPaul Dagnelie curlevel = parent_level; 2448a2cdcdd2SPaul Dagnelie curblkid = parent_blkid; 2449a2cdcdd2SPaul Dagnelie } 2450a2cdcdd2SPaul Dagnelie 2451a2cdcdd2SPaul Dagnelie if (curlevel == nlevels - 1) { 2452a2cdcdd2SPaul Dagnelie /* No cached indirect blocks found. */ 2453a2cdcdd2SPaul Dagnelie ASSERT3U(curblkid, <, dn->dn_phys->dn_nblkptr); 2454a2cdcdd2SPaul Dagnelie bp = dn->dn_phys->dn_blkptr[curblkid]; 2455a2cdcdd2SPaul Dagnelie } 2456a2cdcdd2SPaul Dagnelie if (BP_IS_HOLE(&bp)) 2457a2cdcdd2SPaul Dagnelie return; 2458a2cdcdd2SPaul Dagnelie 2459a2cdcdd2SPaul Dagnelie ASSERT3U(curlevel, ==, BP_GET_LEVEL(&bp)); 2460a2cdcdd2SPaul Dagnelie 2461a2cdcdd2SPaul Dagnelie zio_t *pio = zio_root(dmu_objset_spa(dn->dn_objset), NULL, NULL, 2462a2cdcdd2SPaul Dagnelie ZIO_FLAG_CANFAIL); 2463a2cdcdd2SPaul Dagnelie 2464a2cdcdd2SPaul Dagnelie dbuf_prefetch_arg_t *dpa = kmem_zalloc(sizeof (*dpa), KM_SLEEP); 2465b24ab676SJeff Bonwick dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset; 2466a2cdcdd2SPaul Dagnelie SET_BOOKMARK(&dpa->dpa_zb, ds != NULL ? ds->ds_object : DMU_META_OBJSET, 2467a2cdcdd2SPaul Dagnelie dn->dn_object, level, blkid); 2468a2cdcdd2SPaul Dagnelie dpa->dpa_curlevel = curlevel; 2469a2cdcdd2SPaul Dagnelie dpa->dpa_prio = prio; 2470a2cdcdd2SPaul Dagnelie dpa->dpa_aflags = aflags; 2471a2cdcdd2SPaul Dagnelie dpa->dpa_spa = dn->dn_objset->os_spa; 2472*dcbf3bd6SGeorge Wilson dpa->dpa_dnode = dn; 2473a2cdcdd2SPaul Dagnelie dpa->dpa_epbs = epbs; 2474a2cdcdd2SPaul Dagnelie dpa->dpa_zio = pio; 2475a2cdcdd2SPaul Dagnelie 2476a2cdcdd2SPaul Dagnelie /* 2477a2cdcdd2SPaul Dagnelie * If we have the indirect just above us, no need to do the asynchronous 2478a2cdcdd2SPaul Dagnelie * prefetch chain; we'll just run the last step ourselves. If we're at 2479a2cdcdd2SPaul Dagnelie * a higher level, though, we want to issue the prefetches for all the 2480a2cdcdd2SPaul Dagnelie * indirect blocks asynchronously, so we can go on with whatever we were 2481a2cdcdd2SPaul Dagnelie * doing. 2482a2cdcdd2SPaul Dagnelie */ 2483a2cdcdd2SPaul Dagnelie if (curlevel == level) { 2484a2cdcdd2SPaul Dagnelie ASSERT3U(curblkid, ==, blkid); 2485a2cdcdd2SPaul Dagnelie dbuf_issue_final_prefetch(dpa, &bp); 2486a2cdcdd2SPaul Dagnelie kmem_free(dpa, sizeof (*dpa)); 2487a2cdcdd2SPaul Dagnelie } else { 2488a2cdcdd2SPaul Dagnelie arc_flags_t iter_aflags = ARC_FLAG_NOWAIT; 24897802d7bfSMatthew Ahrens zbookmark_phys_t zb; 2490b24ab676SJeff Bonwick 2491a2cdcdd2SPaul Dagnelie SET_BOOKMARK(&zb, ds != NULL ? ds->ds_object : DMU_META_OBJSET, 2492a2cdcdd2SPaul Dagnelie dn->dn_object, curlevel, curblkid); 2493a2cdcdd2SPaul Dagnelie (void) arc_read(dpa->dpa_zio, dpa->dpa_spa, 2494a2cdcdd2SPaul Dagnelie &bp, dbuf_prefetch_indirect_done, dpa, prio, 2495fa9e4066Sahrens ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 2496a2cdcdd2SPaul Dagnelie &iter_aflags, &zb); 2497fa9e4066Sahrens } 2498a2cdcdd2SPaul Dagnelie /* 2499a2cdcdd2SPaul Dagnelie * We use pio here instead of dpa_zio since it's possible that 2500a2cdcdd2SPaul Dagnelie * dpa may have already been freed. 2501a2cdcdd2SPaul Dagnelie */ 2502a2cdcdd2SPaul Dagnelie zio_nowait(pio); 2503fa9e4066Sahrens } 2504fa9e4066Sahrens 2505fa9e4066Sahrens /* 2506fa9e4066Sahrens * Returns with db_holds incremented, and db_mtx not held. 2507fa9e4066Sahrens * Note: dn_struct_rwlock must be held. 2508fa9e4066Sahrens */ 2509fa9e4066Sahrens int 2510a2cdcdd2SPaul Dagnelie dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, 2511a2cdcdd2SPaul Dagnelie boolean_t fail_sparse, boolean_t fail_uncached, 2512fa9e4066Sahrens void *tag, dmu_buf_impl_t **dbp) 2513fa9e4066Sahrens { 2514fa9e4066Sahrens dmu_buf_impl_t *db, *parent = NULL; 2515fa9e4066Sahrens 25160a586ceaSMark Shellenbaum ASSERT(blkid != DMU_BONUS_BLKID); 2517fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 2518fa9e4066Sahrens ASSERT3U(dn->dn_nlevels, >, level); 2519fa9e4066Sahrens 2520fa9e4066Sahrens *dbp = NULL; 2521ea8dc4b6Seschrock top: 2522fa9e4066Sahrens /* dbuf_find() returns with db_mtx held */ 2523e57a022bSJustin T. Gibbs db = dbuf_find(dn->dn_objset, dn->dn_object, level, blkid); 2524fa9e4066Sahrens 2525fa9e4066Sahrens if (db == NULL) { 2526fa9e4066Sahrens blkptr_t *bp = NULL; 2527fa9e4066Sahrens int err; 2528fa9e4066Sahrens 2529a2cdcdd2SPaul Dagnelie if (fail_uncached) 2530a2cdcdd2SPaul Dagnelie return (SET_ERROR(ENOENT)); 2531a2cdcdd2SPaul Dagnelie 2532c543ec06Sahrens ASSERT3P(parent, ==, NULL); 2533fa9e4066Sahrens err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp); 2534fa9e4066Sahrens if (fail_sparse) { 2535fa9e4066Sahrens if (err == 0 && bp && BP_IS_HOLE(bp)) 2536be6fd75aSMatthew Ahrens err = SET_ERROR(ENOENT); 2537fa9e4066Sahrens if (err) { 2538c543ec06Sahrens if (parent) 2539ea8dc4b6Seschrock dbuf_rele(parent, NULL); 2540fa9e4066Sahrens return (err); 2541fa9e4066Sahrens } 2542fa9e4066Sahrens } 2543ea8dc4b6Seschrock if (err && err != ENOENT) 2544ea8dc4b6Seschrock return (err); 2545fa9e4066Sahrens db = dbuf_create(dn, level, blkid, parent, bp); 2546fa9e4066Sahrens } 2547fa9e4066Sahrens 2548a2cdcdd2SPaul Dagnelie if (fail_uncached && db->db_state != DB_CACHED) { 2549a2cdcdd2SPaul Dagnelie mutex_exit(&db->db_mtx); 2550a2cdcdd2SPaul Dagnelie return (SET_ERROR(ENOENT)); 2551a2cdcdd2SPaul Dagnelie } 2552a2cdcdd2SPaul Dagnelie 2553*dcbf3bd6SGeorge Wilson if (db->db_buf != NULL) 2554ea8dc4b6Seschrock ASSERT3P(db->db.db_data, ==, db->db_buf->b_data); 2555ea8dc4b6Seschrock 2556ea8dc4b6Seschrock ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf)); 2557ea8dc4b6Seschrock 2558fa9e4066Sahrens /* 2559c717a561Smaybee * If this buffer is currently syncing out, and we are are 2560c717a561Smaybee * still referencing it from db_data, we need to make a copy 2561c717a561Smaybee * of it in case we decide we want to dirty it again in this txg. 2562fa9e4066Sahrens */ 25630a586ceaSMark Shellenbaum if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 2564ea8dc4b6Seschrock dn->dn_object != DMU_META_DNODE_OBJECT && 2565c717a561Smaybee db->db_state == DB_CACHED && db->db_data_pending) { 2566c717a561Smaybee dbuf_dirty_record_t *dr = db->db_data_pending; 2567c717a561Smaybee 2568c717a561Smaybee if (dr->dt.dl.dr_data == db->db_buf) { 2569ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 2570fa9e4066Sahrens 2571c717a561Smaybee dbuf_set_data(db, 2572*dcbf3bd6SGeorge Wilson arc_alloc_buf(dn->dn_objset->os_spa, 2573c717a561Smaybee db->db.db_size, db, type)); 2574c717a561Smaybee bcopy(dr->dt.dl.dr_data->b_data, db->db.db_data, 2575fa9e4066Sahrens db->db.db_size); 2576fa9e4066Sahrens } 2577c717a561Smaybee } 2578fa9e4066Sahrens 2579*dcbf3bd6SGeorge Wilson if (multilist_link_active(&db->db_cache_link)) { 2580*dcbf3bd6SGeorge Wilson ASSERT(refcount_is_zero(&db->db_holds)); 2581*dcbf3bd6SGeorge Wilson multilist_remove(&dbuf_cache, db); 2582*dcbf3bd6SGeorge Wilson (void) refcount_remove_many(&dbuf_cache_size, 2583*dcbf3bd6SGeorge Wilson db->db.db_size, db); 2584*dcbf3bd6SGeorge Wilson } 2585ea8dc4b6Seschrock (void) refcount_add(&db->db_holds, tag); 25869c9dc39aSek110237 DBUF_VERIFY(db); 2587fa9e4066Sahrens mutex_exit(&db->db_mtx); 2588fa9e4066Sahrens 2589fa9e4066Sahrens /* NOTE: we can't rele the parent until after we drop the db_mtx */ 2590c543ec06Sahrens if (parent) 2591ea8dc4b6Seschrock dbuf_rele(parent, NULL); 2592fa9e4066Sahrens 2593744947dcSTom Erickson ASSERT3P(DB_DNODE(db), ==, dn); 2594fa9e4066Sahrens ASSERT3U(db->db_blkid, ==, blkid); 2595fa9e4066Sahrens ASSERT3U(db->db_level, ==, level); 2596fa9e4066Sahrens *dbp = db; 2597fa9e4066Sahrens 2598fa9e4066Sahrens return (0); 2599fa9e4066Sahrens } 2600fa9e4066Sahrens 2601fa9e4066Sahrens dmu_buf_impl_t * 2602ea8dc4b6Seschrock dbuf_hold(dnode_t *dn, uint64_t blkid, void *tag) 2603fa9e4066Sahrens { 2604a2cdcdd2SPaul Dagnelie return (dbuf_hold_level(dn, 0, blkid, tag)); 2605fa9e4066Sahrens } 2606fa9e4066Sahrens 2607fa9e4066Sahrens dmu_buf_impl_t * 2608fa9e4066Sahrens dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, void *tag) 2609fa9e4066Sahrens { 2610fa9e4066Sahrens dmu_buf_impl_t *db; 2611a2cdcdd2SPaul Dagnelie int err = dbuf_hold_impl(dn, level, blkid, FALSE, FALSE, tag, &db); 2612ea8dc4b6Seschrock return (err ? NULL : db); 2613fa9e4066Sahrens } 2614fa9e4066Sahrens 26151934e92fSmaybee void 2616ea8dc4b6Seschrock dbuf_create_bonus(dnode_t *dn) 2617fa9e4066Sahrens { 2618ea8dc4b6Seschrock ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 2619ea8dc4b6Seschrock 2620ea8dc4b6Seschrock ASSERT(dn->dn_bonus == NULL); 26210a586ceaSMark Shellenbaum dn->dn_bonus = dbuf_create(dn, 0, DMU_BONUS_BLKID, dn->dn_dbuf, NULL); 26220a586ceaSMark Shellenbaum } 26230a586ceaSMark Shellenbaum 26240a586ceaSMark Shellenbaum int 26250a586ceaSMark Shellenbaum dbuf_spill_set_blksz(dmu_buf_t *db_fake, uint64_t blksz, dmu_tx_t *tx) 26260a586ceaSMark Shellenbaum { 26270a586ceaSMark Shellenbaum dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2628744947dcSTom Erickson dnode_t *dn; 2629744947dcSTom Erickson 26300a586ceaSMark Shellenbaum if (db->db_blkid != DMU_SPILL_BLKID) 2631be6fd75aSMatthew Ahrens return (SET_ERROR(ENOTSUP)); 26320a586ceaSMark Shellenbaum if (blksz == 0) 26330a586ceaSMark Shellenbaum blksz = SPA_MINBLOCKSIZE; 2634b5152584SMatthew Ahrens ASSERT3U(blksz, <=, spa_maxblocksize(dmu_objset_spa(db->db_objset))); 26350a586ceaSMark Shellenbaum blksz = P2ROUNDUP(blksz, SPA_MINBLOCKSIZE); 26360a586ceaSMark Shellenbaum 2637744947dcSTom Erickson DB_DNODE_ENTER(db); 2638744947dcSTom Erickson dn = DB_DNODE(db); 2639744947dcSTom Erickson rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 26400a586ceaSMark Shellenbaum dbuf_new_size(db, blksz, tx); 2641744947dcSTom Erickson rw_exit(&dn->dn_struct_rwlock); 2642744947dcSTom Erickson DB_DNODE_EXIT(db); 26430a586ceaSMark Shellenbaum 26440a586ceaSMark Shellenbaum return (0); 26450a586ceaSMark Shellenbaum } 26460a586ceaSMark Shellenbaum 26470a586ceaSMark Shellenbaum void 26480a586ceaSMark Shellenbaum dbuf_rm_spill(dnode_t *dn, dmu_tx_t *tx) 26490a586ceaSMark Shellenbaum { 26500a586ceaSMark Shellenbaum dbuf_free_range(dn, DMU_SPILL_BLKID, DMU_SPILL_BLKID, tx); 2651fa9e4066Sahrens } 2652fa9e4066Sahrens 2653ea8dc4b6Seschrock #pragma weak dmu_buf_add_ref = dbuf_add_ref 2654fa9e4066Sahrens void 2655fa9e4066Sahrens dbuf_add_ref(dmu_buf_impl_t *db, void *tag) 2656fa9e4066Sahrens { 2657ea8dc4b6Seschrock int64_t holds = refcount_add(&db->db_holds, tag); 2658*dcbf3bd6SGeorge Wilson ASSERT3S(holds, >, 1); 2659fa9e4066Sahrens } 2660fa9e4066Sahrens 2661e57a022bSJustin T. Gibbs #pragma weak dmu_buf_try_add_ref = dbuf_try_add_ref 2662e57a022bSJustin T. Gibbs boolean_t 2663e57a022bSJustin T. Gibbs dbuf_try_add_ref(dmu_buf_t *db_fake, objset_t *os, uint64_t obj, uint64_t blkid, 2664e57a022bSJustin T. Gibbs void *tag) 2665e57a022bSJustin T. Gibbs { 2666e57a022bSJustin T. Gibbs dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2667e57a022bSJustin T. Gibbs dmu_buf_impl_t *found_db; 2668e57a022bSJustin T. Gibbs boolean_t result = B_FALSE; 2669e57a022bSJustin T. Gibbs 2670e57a022bSJustin T. Gibbs if (db->db_blkid == DMU_BONUS_BLKID) 2671e57a022bSJustin T. Gibbs found_db = dbuf_find_bonus(os, obj); 2672e57a022bSJustin T. Gibbs else 2673e57a022bSJustin T. Gibbs found_db = dbuf_find(os, obj, 0, blkid); 2674e57a022bSJustin T. Gibbs 2675e57a022bSJustin T. Gibbs if (found_db != NULL) { 2676e57a022bSJustin T. Gibbs if (db == found_db && dbuf_refcount(db) > db->db_dirtycnt) { 2677e57a022bSJustin T. Gibbs (void) refcount_add(&db->db_holds, tag); 2678e57a022bSJustin T. Gibbs result = B_TRUE; 2679e57a022bSJustin T. Gibbs } 2680e57a022bSJustin T. Gibbs mutex_exit(&db->db_mtx); 2681e57a022bSJustin T. Gibbs } 2682e57a022bSJustin T. Gibbs return (result); 2683e57a022bSJustin T. Gibbs } 2684e57a022bSJustin T. Gibbs 2685744947dcSTom Erickson /* 2686744947dcSTom Erickson * If you call dbuf_rele() you had better not be referencing the dnode handle 2687744947dcSTom Erickson * unless you have some other direct or indirect hold on the dnode. (An indirect 2688744947dcSTom Erickson * hold is a hold on one of the dnode's dbufs, including the bonus buffer.) 2689744947dcSTom Erickson * Without that, the dbuf_rele() could lead to a dnode_rele() followed by the 2690744947dcSTom Erickson * dnode's parent dbuf evicting its dnode handles. 2691744947dcSTom Erickson */ 2692fa9e4066Sahrens void 2693ea8dc4b6Seschrock dbuf_rele(dmu_buf_impl_t *db, void *tag) 2694fa9e4066Sahrens { 2695b24ab676SJeff Bonwick mutex_enter(&db->db_mtx); 2696b24ab676SJeff Bonwick dbuf_rele_and_unlock(db, tag); 2697b24ab676SJeff Bonwick } 2698b24ab676SJeff Bonwick 269943466aaeSMax Grossman void 270043466aaeSMax Grossman dmu_buf_rele(dmu_buf_t *db, void *tag) 270143466aaeSMax Grossman { 270243466aaeSMax Grossman dbuf_rele((dmu_buf_impl_t *)db, tag); 270343466aaeSMax Grossman } 270443466aaeSMax Grossman 2705b24ab676SJeff Bonwick /* 2706b24ab676SJeff Bonwick * dbuf_rele() for an already-locked dbuf. This is necessary to allow 2707b24ab676SJeff Bonwick * db_dirtycnt and db_holds to be updated atomically. 2708b24ab676SJeff Bonwick */ 2709b24ab676SJeff Bonwick void 2710b24ab676SJeff Bonwick dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag) 2711b24ab676SJeff Bonwick { 2712fa9e4066Sahrens int64_t holds; 2713fa9e4066Sahrens 2714b24ab676SJeff Bonwick ASSERT(MUTEX_HELD(&db->db_mtx)); 27159c9dc39aSek110237 DBUF_VERIFY(db); 2716fa9e4066Sahrens 2717744947dcSTom Erickson /* 2718744947dcSTom Erickson * Remove the reference to the dbuf before removing its hold on the 2719744947dcSTom Erickson * dnode so we can guarantee in dnode_move() that a referenced bonus 2720744947dcSTom Erickson * buffer has a corresponding dnode hold. 2721744947dcSTom Erickson */ 2722fa9e4066Sahrens holds = refcount_remove(&db->db_holds, tag); 2723ea8dc4b6Seschrock ASSERT(holds >= 0); 2724fa9e4066Sahrens 2725c717a561Smaybee /* 2726c717a561Smaybee * We can't freeze indirects if there is a possibility that they 2727c717a561Smaybee * may be modified in the current syncing context. 2728c717a561Smaybee */ 2729*dcbf3bd6SGeorge Wilson if (db->db_buf != NULL && 2730*dcbf3bd6SGeorge Wilson holds == (db->db_level == 0 ? db->db_dirtycnt : 0)) { 27316b4acc8bSahrens arc_buf_freeze(db->db_buf); 2732*dcbf3bd6SGeorge Wilson } 27336b4acc8bSahrens 2734fa9e4066Sahrens if (holds == db->db_dirtycnt && 2735d2058105SJustin T. Gibbs db->db_level == 0 && db->db_user_immediate_evict) 2736fa9e4066Sahrens dbuf_evict_user(db); 2737ea8dc4b6Seschrock 2738ea8dc4b6Seschrock if (holds == 0) { 27390a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 2740cd485b49SJustin T. Gibbs dnode_t *dn; 2741d2058105SJustin T. Gibbs boolean_t evict_dbuf = db->db_pending_evict; 2742cd485b49SJustin T. Gibbs 2743cd485b49SJustin T. Gibbs /* 2744cd485b49SJustin T. Gibbs * If the dnode moves here, we cannot cross this 2745cd485b49SJustin T. Gibbs * barrier until the move completes. 2746cd485b49SJustin T. Gibbs */ 2747cd485b49SJustin T. Gibbs DB_DNODE_ENTER(db); 2748cd485b49SJustin T. Gibbs 2749cd485b49SJustin T. Gibbs dn = DB_DNODE(db); 2750cd485b49SJustin T. Gibbs atomic_dec_32(&dn->dn_dbufs_count); 2751cd485b49SJustin T. Gibbs 2752cd485b49SJustin T. Gibbs /* 2753cd485b49SJustin T. Gibbs * Decrementing the dbuf count means that the bonus 2754cd485b49SJustin T. Gibbs * buffer's dnode hold is no longer discounted in 2755cd485b49SJustin T. Gibbs * dnode_move(). The dnode cannot move until after 2756d2058105SJustin T. Gibbs * the dnode_rele() below. 2757cd485b49SJustin T. Gibbs */ 2758cd485b49SJustin T. Gibbs DB_DNODE_EXIT(db); 2759cd485b49SJustin T. Gibbs 2760cd485b49SJustin T. Gibbs /* 2761cd485b49SJustin T. Gibbs * Do not reference db after its lock is dropped. 2762cd485b49SJustin T. Gibbs * Another thread may evict it. 2763cd485b49SJustin T. Gibbs */ 2764ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 2765744947dcSTom Erickson 2766d2058105SJustin T. Gibbs if (evict_dbuf) 2767cd485b49SJustin T. Gibbs dnode_evict_bonus(dn); 2768d2058105SJustin T. Gibbs 2769d2058105SJustin T. Gibbs dnode_rele(dn, db); 2770ea8dc4b6Seschrock } else if (db->db_buf == NULL) { 2771ea8dc4b6Seschrock /* 2772ea8dc4b6Seschrock * This is a special case: we never associated this 2773ea8dc4b6Seschrock * dbuf with any data allocated from the ARC. 2774ea8dc4b6Seschrock */ 277582c9918fSTim Haley ASSERT(db->db_state == DB_UNCACHED || 277682c9918fSTim Haley db->db_state == DB_NOFILL); 2777*dcbf3bd6SGeorge Wilson dbuf_destroy(db); 2778ea8dc4b6Seschrock } else if (arc_released(db->db_buf)) { 2779ea8dc4b6Seschrock /* 2780ea8dc4b6Seschrock * This dbuf has anonymous data associated with it. 2781ea8dc4b6Seschrock */ 2782*dcbf3bd6SGeorge Wilson dbuf_destroy(db); 2783ea8dc4b6Seschrock } else { 2784*dcbf3bd6SGeorge Wilson boolean_t do_arc_evict = B_FALSE; 2785*dcbf3bd6SGeorge Wilson blkptr_t bp; 2786*dcbf3bd6SGeorge Wilson spa_t *spa = dmu_objset_spa(db->db_objset); 27879253d63dSGeorge Wilson 2788*dcbf3bd6SGeorge Wilson if (!DBUF_IS_CACHEABLE(db) && 2789*dcbf3bd6SGeorge Wilson db->db_blkptr != NULL && 2790bbfa8ea8SMatthew Ahrens !BP_IS_HOLE(db->db_blkptr) && 2791bbfa8ea8SMatthew Ahrens !BP_IS_EMBEDDED(db->db_blkptr)) { 2792*dcbf3bd6SGeorge Wilson do_arc_evict = B_TRUE; 2793*dcbf3bd6SGeorge Wilson bp = *db->db_blkptr; 2794*dcbf3bd6SGeorge Wilson } 2795*dcbf3bd6SGeorge Wilson 2796*dcbf3bd6SGeorge Wilson if (!DBUF_IS_CACHEABLE(db) || 2797*dcbf3bd6SGeorge Wilson db->db_pending_evict) { 2798*dcbf3bd6SGeorge Wilson dbuf_destroy(db); 2799*dcbf3bd6SGeorge Wilson } else if (!multilist_link_active(&db->db_cache_link)) { 2800*dcbf3bd6SGeorge Wilson multilist_insert(&dbuf_cache, db); 2801*dcbf3bd6SGeorge Wilson (void) refcount_add_many(&dbuf_cache_size, 2802*dcbf3bd6SGeorge Wilson db->db.db_size, db); 2803*dcbf3bd6SGeorge Wilson mutex_exit(&db->db_mtx); 2804*dcbf3bd6SGeorge Wilson 2805*dcbf3bd6SGeorge Wilson dbuf_evict_notify(); 2806*dcbf3bd6SGeorge Wilson } 2807*dcbf3bd6SGeorge Wilson 2808*dcbf3bd6SGeorge Wilson if (do_arc_evict) 2809bbfa8ea8SMatthew Ahrens arc_freed(spa, &bp); 2810bbfa8ea8SMatthew Ahrens } 2811ea8dc4b6Seschrock } else { 2812ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 2813fa9e4066Sahrens } 2814*dcbf3bd6SGeorge Wilson 2815fa9e4066Sahrens } 2816fa9e4066Sahrens 2817fa9e4066Sahrens #pragma weak dmu_buf_refcount = dbuf_refcount 2818fa9e4066Sahrens uint64_t 2819fa9e4066Sahrens dbuf_refcount(dmu_buf_impl_t *db) 2820fa9e4066Sahrens { 2821fa9e4066Sahrens return (refcount_count(&db->db_holds)); 2822fa9e4066Sahrens } 2823fa9e4066Sahrens 2824fa9e4066Sahrens void * 2825bc9014e6SJustin Gibbs dmu_buf_replace_user(dmu_buf_t *db_fake, dmu_buf_user_t *old_user, 2826bc9014e6SJustin Gibbs dmu_buf_user_t *new_user) 2827fa9e4066Sahrens { 2828bc9014e6SJustin Gibbs dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2829bc9014e6SJustin Gibbs 2830bc9014e6SJustin Gibbs mutex_enter(&db->db_mtx); 2831bc9014e6SJustin Gibbs dbuf_verify_user(db, DBVU_NOT_EVICTING); 2832bc9014e6SJustin Gibbs if (db->db_user == old_user) 2833bc9014e6SJustin Gibbs db->db_user = new_user; 2834bc9014e6SJustin Gibbs else 2835bc9014e6SJustin Gibbs old_user = db->db_user; 2836bc9014e6SJustin Gibbs dbuf_verify_user(db, DBVU_NOT_EVICTING); 2837bc9014e6SJustin Gibbs mutex_exit(&db->db_mtx); 2838bc9014e6SJustin Gibbs 2839bc9014e6SJustin Gibbs return (old_user); 2840fa9e4066Sahrens } 2841fa9e4066Sahrens 2842fa9e4066Sahrens void * 2843bc9014e6SJustin Gibbs dmu_buf_set_user(dmu_buf_t *db_fake, dmu_buf_user_t *user) 2844bc9014e6SJustin Gibbs { 2845bc9014e6SJustin Gibbs return (dmu_buf_replace_user(db_fake, NULL, user)); 2846bc9014e6SJustin Gibbs } 2847bc9014e6SJustin Gibbs 2848bc9014e6SJustin Gibbs void * 2849bc9014e6SJustin Gibbs dmu_buf_set_user_ie(dmu_buf_t *db_fake, dmu_buf_user_t *user) 2850fa9e4066Sahrens { 2851fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2852fa9e4066Sahrens 2853d2058105SJustin T. Gibbs db->db_user_immediate_evict = TRUE; 2854bc9014e6SJustin Gibbs return (dmu_buf_set_user(db_fake, user)); 2855fa9e4066Sahrens } 2856fa9e4066Sahrens 2857fa9e4066Sahrens void * 2858bc9014e6SJustin Gibbs dmu_buf_remove_user(dmu_buf_t *db_fake, dmu_buf_user_t *user) 2859fa9e4066Sahrens { 2860bc9014e6SJustin Gibbs return (dmu_buf_replace_user(db_fake, user, NULL)); 2861fa9e4066Sahrens } 2862fa9e4066Sahrens 2863fa9e4066Sahrens void * 2864fa9e4066Sahrens dmu_buf_get_user(dmu_buf_t *db_fake) 2865fa9e4066Sahrens { 2866fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2867fa9e4066Sahrens 2868bc9014e6SJustin Gibbs dbuf_verify_user(db, DBVU_NOT_EVICTING); 2869bc9014e6SJustin Gibbs return (db->db_user); 2870bc9014e6SJustin Gibbs } 2871bc9014e6SJustin Gibbs 2872bc9014e6SJustin Gibbs void 2873bc9014e6SJustin Gibbs dmu_buf_user_evict_wait() 2874bc9014e6SJustin Gibbs { 2875bc9014e6SJustin Gibbs taskq_wait(dbu_evict_taskq); 2876fa9e4066Sahrens } 2877fa9e4066Sahrens 28783d692628SSanjeev Bagewadi boolean_t 28793d692628SSanjeev Bagewadi dmu_buf_freeable(dmu_buf_t *dbuf) 28803d692628SSanjeev Bagewadi { 28813d692628SSanjeev Bagewadi boolean_t res = B_FALSE; 28823d692628SSanjeev Bagewadi dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf; 28833d692628SSanjeev Bagewadi 28843d692628SSanjeev Bagewadi if (db->db_blkptr) 28853d692628SSanjeev Bagewadi res = dsl_dataset_block_freeable(db->db_objset->os_dsl_dataset, 2886c7cd2421SGeorge Wilson db->db_blkptr, db->db_blkptr->blk_birth); 28873d692628SSanjeev Bagewadi 28883d692628SSanjeev Bagewadi return (res); 28893d692628SSanjeev Bagewadi } 28903d692628SSanjeev Bagewadi 289180901aeaSGeorge Wilson blkptr_t * 289280901aeaSGeorge Wilson dmu_buf_get_blkptr(dmu_buf_t *db) 289380901aeaSGeorge Wilson { 289480901aeaSGeorge Wilson dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db; 289580901aeaSGeorge Wilson return (dbi->db_blkptr); 289680901aeaSGeorge Wilson } 289780901aeaSGeorge Wilson 2898c717a561Smaybee static void 2899c717a561Smaybee dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db) 2900fa9e4066Sahrens { 2901c717a561Smaybee /* ASSERT(dmu_tx_is_syncing(tx) */ 2902c717a561Smaybee ASSERT(MUTEX_HELD(&db->db_mtx)); 2903c717a561Smaybee 2904c717a561Smaybee if (db->db_blkptr != NULL) 2905c717a561Smaybee return; 2906c717a561Smaybee 29070a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) { 29080a586ceaSMark Shellenbaum db->db_blkptr = &dn->dn_phys->dn_spill; 29090a586ceaSMark Shellenbaum BP_ZERO(db->db_blkptr); 29100a586ceaSMark Shellenbaum return; 29110a586ceaSMark Shellenbaum } 2912c717a561Smaybee if (db->db_level == dn->dn_phys->dn_nlevels-1) { 2913c717a561Smaybee /* 2914c717a561Smaybee * This buffer was allocated at a time when there was 2915c717a561Smaybee * no available blkptrs from the dnode, or it was 2916c717a561Smaybee * inappropriate to hook it in (i.e., nlevels mis-match). 2917c717a561Smaybee */ 2918c717a561Smaybee ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr); 2919c717a561Smaybee ASSERT(db->db_parent == NULL); 2920c717a561Smaybee db->db_parent = dn->dn_dbuf; 2921c717a561Smaybee db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid]; 2922c717a561Smaybee DBUF_VERIFY(db); 2923c717a561Smaybee } else { 2924c717a561Smaybee dmu_buf_impl_t *parent = db->db_parent; 2925c717a561Smaybee int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2926c717a561Smaybee 2927c717a561Smaybee ASSERT(dn->dn_phys->dn_nlevels > 1); 2928c717a561Smaybee if (parent == NULL) { 2929c717a561Smaybee mutex_exit(&db->db_mtx); 2930c717a561Smaybee rw_enter(&dn->dn_struct_rwlock, RW_READER); 2931a2cdcdd2SPaul Dagnelie parent = dbuf_hold_level(dn, db->db_level + 1, 2932a2cdcdd2SPaul Dagnelie db->db_blkid >> epbs, db); 2933c717a561Smaybee rw_exit(&dn->dn_struct_rwlock); 2934c717a561Smaybee mutex_enter(&db->db_mtx); 2935c717a561Smaybee db->db_parent = parent; 2936c717a561Smaybee } 2937c717a561Smaybee db->db_blkptr = (blkptr_t *)parent->db.db_data + 2938c717a561Smaybee (db->db_blkid & ((1ULL << epbs) - 1)); 2939c717a561Smaybee DBUF_VERIFY(db); 2940c717a561Smaybee } 2941c717a561Smaybee } 2942c717a561Smaybee 2943c717a561Smaybee static void 2944c717a561Smaybee dbuf_sync_indirect(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 2945c717a561Smaybee { 2946c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 2947744947dcSTom Erickson dnode_t *dn; 2948c717a561Smaybee zio_t *zio; 2949c717a561Smaybee 2950c717a561Smaybee ASSERT(dmu_tx_is_syncing(tx)); 2951c717a561Smaybee 2952c717a561Smaybee dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 2953c717a561Smaybee 2954c717a561Smaybee mutex_enter(&db->db_mtx); 2955c717a561Smaybee 2956c717a561Smaybee ASSERT(db->db_level > 0); 2957c717a561Smaybee DBUF_VERIFY(db); 2958c717a561Smaybee 29593e30c24aSWill Andrews /* Read the block if it hasn't been read yet. */ 2960c717a561Smaybee if (db->db_buf == NULL) { 2961c717a561Smaybee mutex_exit(&db->db_mtx); 2962c717a561Smaybee (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED); 2963c717a561Smaybee mutex_enter(&db->db_mtx); 2964c717a561Smaybee } 2965c717a561Smaybee ASSERT3U(db->db_state, ==, DB_CACHED); 2966c717a561Smaybee ASSERT(db->db_buf != NULL); 2967c717a561Smaybee 2968744947dcSTom Erickson DB_DNODE_ENTER(db); 2969744947dcSTom Erickson dn = DB_DNODE(db); 29703e30c24aSWill Andrews /* Indirect block size must match what the dnode thinks it is. */ 2971744947dcSTom Erickson ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2972c717a561Smaybee dbuf_check_blkptr(dn, db); 2973744947dcSTom Erickson DB_DNODE_EXIT(db); 2974c717a561Smaybee 29753e30c24aSWill Andrews /* Provide the pending dirty record to child dbufs */ 2976c717a561Smaybee db->db_data_pending = dr; 2977c717a561Smaybee 2978af2c4821Smaybee mutex_exit(&db->db_mtx); 2979088f3894Sahrens dbuf_write(dr, db->db_buf, tx); 2980c717a561Smaybee 2981c717a561Smaybee zio = dr->dr_zio; 2982c717a561Smaybee mutex_enter(&dr->dt.di.dr_mtx); 298346e1baa6SMatthew Ahrens dbuf_sync_list(&dr->dt.di.dr_children, db->db_level - 1, tx); 2984c717a561Smaybee ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 2985c717a561Smaybee mutex_exit(&dr->dt.di.dr_mtx); 2986c717a561Smaybee zio_nowait(zio); 2987c717a561Smaybee } 2988c717a561Smaybee 2989c717a561Smaybee static void 2990c717a561Smaybee dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 2991c717a561Smaybee { 2992c717a561Smaybee arc_buf_t **datap = &dr->dt.dl.dr_data; 2993c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 2994744947dcSTom Erickson dnode_t *dn; 2995744947dcSTom Erickson objset_t *os; 2996c717a561Smaybee uint64_t txg = tx->tx_txg; 2997fa9e4066Sahrens 2998fa9e4066Sahrens ASSERT(dmu_tx_is_syncing(tx)); 2999fa9e4066Sahrens 3000fa9e4066Sahrens dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 3001fa9e4066Sahrens 3002fa9e4066Sahrens mutex_enter(&db->db_mtx); 3003fa9e4066Sahrens /* 3004fa9e4066Sahrens * To be synced, we must be dirtied. But we 3005fa9e4066Sahrens * might have been freed after the dirty. 3006fa9e4066Sahrens */ 3007fa9e4066Sahrens if (db->db_state == DB_UNCACHED) { 3008fa9e4066Sahrens /* This buffer has been freed since it was dirtied */ 3009fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 3010fa9e4066Sahrens } else if (db->db_state == DB_FILL) { 3011fa9e4066Sahrens /* This buffer was freed and is now being re-filled */ 3012c717a561Smaybee ASSERT(db->db.db_data != dr->dt.dl.dr_data); 3013fa9e4066Sahrens } else { 301482c9918fSTim Haley ASSERT(db->db_state == DB_CACHED || db->db_state == DB_NOFILL); 3015fa9e4066Sahrens } 30169c9dc39aSek110237 DBUF_VERIFY(db); 3017fa9e4066Sahrens 3018744947dcSTom Erickson DB_DNODE_ENTER(db); 3019744947dcSTom Erickson dn = DB_DNODE(db); 3020744947dcSTom Erickson 30210a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) { 30220a586ceaSMark Shellenbaum mutex_enter(&dn->dn_mtx); 30230a586ceaSMark Shellenbaum dn->dn_phys->dn_flags |= DNODE_FLAG_SPILL_BLKPTR; 30240a586ceaSMark Shellenbaum mutex_exit(&dn->dn_mtx); 30250a586ceaSMark Shellenbaum } 30260a586ceaSMark Shellenbaum 3027fa9e4066Sahrens /* 3028c717a561Smaybee * If this is a bonus buffer, simply copy the bonus data into the 3029c717a561Smaybee * dnode. It will be written out when the dnode is synced (and it 3030c717a561Smaybee * will be synced, since it must have been dirty for dbuf_sync to 3031c717a561Smaybee * be called). 3032fa9e4066Sahrens */ 30330a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 3034c717a561Smaybee dbuf_dirty_record_t **drp; 30351934e92fSmaybee 3036ea8dc4b6Seschrock ASSERT(*datap != NULL); 3037fb09f5aaSMadhav Suresh ASSERT0(db->db_level); 3038ea8dc4b6Seschrock ASSERT3U(dn->dn_phys->dn_bonuslen, <=, DN_MAX_BONUSLEN); 3039ea8dc4b6Seschrock bcopy(*datap, DN_BONUS(dn->dn_phys), dn->dn_phys->dn_bonuslen); 3040744947dcSTom Erickson DB_DNODE_EXIT(db); 3041744947dcSTom Erickson 30420e8c6158Smaybee if (*datap != db->db.db_data) { 3043ea8dc4b6Seschrock zio_buf_free(*datap, DN_MAX_BONUSLEN); 30445a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 30450e8c6158Smaybee } 3046ea8dc4b6Seschrock db->db_data_pending = NULL; 3047c717a561Smaybee drp = &db->db_last_dirty; 3048c717a561Smaybee while (*drp != dr) 3049c717a561Smaybee drp = &(*drp)->dr_next; 305017f17c2dSbonwick ASSERT(dr->dr_next == NULL); 3051b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 305217f17c2dSbonwick *drp = dr->dr_next; 3053c717a561Smaybee kmem_free(dr, sizeof (dbuf_dirty_record_t)); 3054ea8dc4b6Seschrock ASSERT(db->db_dirtycnt > 0); 3055ea8dc4b6Seschrock db->db_dirtycnt -= 1; 3056b24ab676SJeff Bonwick dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg); 3057ea8dc4b6Seschrock return; 3058ea8dc4b6Seschrock } 3059ea8dc4b6Seschrock 3060744947dcSTom Erickson os = dn->dn_objset; 3061744947dcSTom Erickson 3062c5c6ffa0Smaybee /* 3063f82bfe17Sgw25295 * This function may have dropped the db_mtx lock allowing a dmu_sync 3064f82bfe17Sgw25295 * operation to sneak in. As a result, we need to ensure that we 3065f82bfe17Sgw25295 * don't check the dr_override_state until we have returned from 3066f82bfe17Sgw25295 * dbuf_check_blkptr. 3067f82bfe17Sgw25295 */ 3068f82bfe17Sgw25295 dbuf_check_blkptr(dn, db); 3069f82bfe17Sgw25295 3070f82bfe17Sgw25295 /* 3071744947dcSTom Erickson * If this buffer is in the middle of an immediate write, 3072c717a561Smaybee * wait for the synchronous IO to complete. 3073c5c6ffa0Smaybee */ 3074c717a561Smaybee while (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC) { 3075c5c6ffa0Smaybee ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT); 3076c5c6ffa0Smaybee cv_wait(&db->db_changed, &db->db_mtx); 3077c717a561Smaybee ASSERT(dr->dt.dl.dr_override_state != DR_NOT_OVERRIDDEN); 3078c5c6ffa0Smaybee } 3079c717a561Smaybee 3080ab69d62fSMatthew Ahrens if (db->db_state != DB_NOFILL && 3081ab69d62fSMatthew Ahrens dn->dn_object != DMU_META_DNODE_OBJECT && 3082ab69d62fSMatthew Ahrens refcount_count(&db->db_holds) > 1 && 3083b24ab676SJeff Bonwick dr->dt.dl.dr_override_state != DR_OVERRIDDEN && 3084ab69d62fSMatthew Ahrens *datap == db->db_buf) { 3085fa9e4066Sahrens /* 308682c9918fSTim Haley * If this buffer is currently "in use" (i.e., there 308782c9918fSTim Haley * are active holds and db_data still references it), 308882c9918fSTim Haley * then make a copy before we start the write so that 308982c9918fSTim Haley * any modifications from the open txg will not leak 309082c9918fSTim Haley * into this write. 3091fa9e4066Sahrens * 309282c9918fSTim Haley * NOTE: this copy does not need to be made for 309382c9918fSTim Haley * objects only modified in the syncing context (e.g. 309482c9918fSTim Haley * DNONE_DNODE blocks). 3095fa9e4066Sahrens */ 3096ab69d62fSMatthew Ahrens int blksz = arc_buf_size(*datap); 3097ab69d62fSMatthew Ahrens arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 3098*dcbf3bd6SGeorge Wilson *datap = arc_alloc_buf(os->os_spa, blksz, db, type); 3099c717a561Smaybee bcopy(db->db.db_data, (*datap)->b_data, blksz); 3100fa9e4066Sahrens } 3101c717a561Smaybee db->db_data_pending = dr; 3102fa9e4066Sahrens 3103fa9e4066Sahrens mutex_exit(&db->db_mtx); 3104fa9e4066Sahrens 3105088f3894Sahrens dbuf_write(dr, *datap, tx); 3106c717a561Smaybee 3107c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 3108744947dcSTom Erickson if (dn->dn_object == DMU_META_DNODE_OBJECT) { 3109c717a561Smaybee list_insert_tail(&dn->dn_dirty_records[txg&TXG_MASK], dr); 3110744947dcSTom Erickson DB_DNODE_EXIT(db); 3111744947dcSTom Erickson } else { 3112744947dcSTom Erickson /* 3113744947dcSTom Erickson * Although zio_nowait() does not "wait for an IO", it does 3114744947dcSTom Erickson * initiate the IO. If this is an empty write it seems plausible 3115744947dcSTom Erickson * that the IO could actually be completed before the nowait 3116744947dcSTom Erickson * returns. We need to DB_DNODE_EXIT() first in case 3117744947dcSTom Erickson * zio_nowait() invalidates the dbuf. 3118744947dcSTom Erickson */ 3119744947dcSTom Erickson DB_DNODE_EXIT(db); 3120c717a561Smaybee zio_nowait(dr->dr_zio); 3121fa9e4066Sahrens } 3122744947dcSTom Erickson } 3123c717a561Smaybee 3124c717a561Smaybee void 312546e1baa6SMatthew Ahrens dbuf_sync_list(list_t *list, int level, dmu_tx_t *tx) 3126c717a561Smaybee { 3127c717a561Smaybee dbuf_dirty_record_t *dr; 3128c717a561Smaybee 3129c717a561Smaybee while (dr = list_head(list)) { 3130c717a561Smaybee if (dr->dr_zio != NULL) { 3131c717a561Smaybee /* 3132c717a561Smaybee * If we find an already initialized zio then we 3133c717a561Smaybee * are processing the meta-dnode, and we have finished. 3134c717a561Smaybee * The dbufs for all dnodes are put back on the list 3135c717a561Smaybee * during processing, so that we can zio_wait() 3136c717a561Smaybee * these IOs after initiating all child IOs. 3137c717a561Smaybee */ 3138c717a561Smaybee ASSERT3U(dr->dr_dbuf->db.db_object, ==, 3139c717a561Smaybee DMU_META_DNODE_OBJECT); 3140c717a561Smaybee break; 3141fa9e4066Sahrens } 314246e1baa6SMatthew Ahrens if (dr->dr_dbuf->db_blkid != DMU_BONUS_BLKID && 314346e1baa6SMatthew Ahrens dr->dr_dbuf->db_blkid != DMU_SPILL_BLKID) { 314446e1baa6SMatthew Ahrens VERIFY3U(dr->dr_dbuf->db_level, ==, level); 314546e1baa6SMatthew Ahrens } 3146c717a561Smaybee list_remove(list, dr); 3147c717a561Smaybee if (dr->dr_dbuf->db_level > 0) 3148c717a561Smaybee dbuf_sync_indirect(dr, tx); 3149c717a561Smaybee else 3150c717a561Smaybee dbuf_sync_leaf(dr, tx); 3151c717a561Smaybee } 3152c717a561Smaybee } 3153c717a561Smaybee 3154fa9e4066Sahrens /* ARGSUSED */ 3155fa9e4066Sahrens static void 3156c717a561Smaybee dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb) 3157fa9e4066Sahrens { 3158fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 3159744947dcSTom Erickson dnode_t *dn; 3160e14bb325SJeff Bonwick blkptr_t *bp = zio->io_bp; 3161c717a561Smaybee blkptr_t *bp_orig = &zio->io_bp_orig; 3162b24ab676SJeff Bonwick spa_t *spa = zio->io_spa; 3163b24ab676SJeff Bonwick int64_t delta; 3164fa9e4066Sahrens uint64_t fill = 0; 3165b24ab676SJeff Bonwick int i; 3166fa9e4066Sahrens 316711ceac77SAlex Reece ASSERT3P(db->db_blkptr, !=, NULL); 316811ceac77SAlex Reece ASSERT3P(&db->db_data_pending->dr_bp_copy, ==, bp); 3169e14bb325SJeff Bonwick 3170744947dcSTom Erickson DB_DNODE_ENTER(db); 3171744947dcSTom Erickson dn = DB_DNODE(db); 3172b24ab676SJeff Bonwick delta = bp_get_dsize_sync(spa, bp) - bp_get_dsize_sync(spa, bp_orig); 3173b24ab676SJeff Bonwick dnode_diduse_space(dn, delta - zio->io_prev_space_delta); 3174b24ab676SJeff Bonwick zio->io_prev_space_delta = delta; 3175fa9e4066Sahrens 317643466aaeSMax Grossman if (bp->blk_birth != 0) { 31770a586ceaSMark Shellenbaum ASSERT((db->db_blkid != DMU_SPILL_BLKID && 31780a586ceaSMark Shellenbaum BP_GET_TYPE(bp) == dn->dn_type) || 31790a586ceaSMark Shellenbaum (db->db_blkid == DMU_SPILL_BLKID && 31805d7b4d43SMatthew Ahrens BP_GET_TYPE(bp) == dn->dn_bonustype) || 31815d7b4d43SMatthew Ahrens BP_IS_EMBEDDED(bp)); 3182e14bb325SJeff Bonwick ASSERT(BP_GET_LEVEL(bp) == db->db_level); 318343466aaeSMax Grossman } 3184e14bb325SJeff Bonwick 3185fa9e4066Sahrens mutex_enter(&db->db_mtx); 3186fa9e4066Sahrens 31870a586ceaSMark Shellenbaum #ifdef ZFS_DEBUG 31880a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) { 31890a586ceaSMark Shellenbaum ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); 319011ceac77SAlex Reece ASSERT(!(BP_IS_HOLE(bp)) && 31910a586ceaSMark Shellenbaum db->db_blkptr == &dn->dn_phys->dn_spill); 31920a586ceaSMark Shellenbaum } 31930a586ceaSMark Shellenbaum #endif 31940a586ceaSMark Shellenbaum 3195fa9e4066Sahrens if (db->db_level == 0) { 3196fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 31970a586ceaSMark Shellenbaum if (db->db_blkid > dn->dn_phys->dn_maxblkid && 31980a586ceaSMark Shellenbaum db->db_blkid != DMU_SPILL_BLKID) 3199fa9e4066Sahrens dn->dn_phys->dn_maxblkid = db->db_blkid; 3200fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 3201fa9e4066Sahrens 3202fa9e4066Sahrens if (dn->dn_type == DMU_OT_DNODE) { 3203fa9e4066Sahrens dnode_phys_t *dnp = db->db.db_data; 3204fa9e4066Sahrens for (i = db->db.db_size >> DNODE_SHIFT; i > 0; 3205fa9e4066Sahrens i--, dnp++) { 3206fa9e4066Sahrens if (dnp->dn_type != DMU_OT_NONE) 3207fa9e4066Sahrens fill++; 3208fa9e4066Sahrens } 3209fa9e4066Sahrens } else { 321043466aaeSMax Grossman if (BP_IS_HOLE(bp)) { 321143466aaeSMax Grossman fill = 0; 321243466aaeSMax Grossman } else { 3213fa9e4066Sahrens fill = 1; 3214fa9e4066Sahrens } 321543466aaeSMax Grossman } 3216fa9e4066Sahrens } else { 3217e14bb325SJeff Bonwick blkptr_t *ibp = db->db.db_data; 3218fa9e4066Sahrens ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 3219e14bb325SJeff Bonwick for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, ibp++) { 3220e14bb325SJeff Bonwick if (BP_IS_HOLE(ibp)) 3221fa9e4066Sahrens continue; 32225d7b4d43SMatthew Ahrens fill += BP_GET_FILL(ibp); 3223fa9e4066Sahrens } 3224fa9e4066Sahrens } 3225744947dcSTom Erickson DB_DNODE_EXIT(db); 3226fa9e4066Sahrens 32275d7b4d43SMatthew Ahrens if (!BP_IS_EMBEDDED(bp)) 3228e14bb325SJeff Bonwick bp->blk_fill = fill; 3229fa9e4066Sahrens 3230fa9e4066Sahrens mutex_exit(&db->db_mtx); 323111ceac77SAlex Reece 323211ceac77SAlex Reece rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 323311ceac77SAlex Reece *db->db_blkptr = *bp; 323411ceac77SAlex Reece rw_exit(&dn->dn_struct_rwlock); 3235fa9e4066Sahrens } 3236fa9e4066Sahrens 32378df0bcf0SPaul Dagnelie /* ARGSUSED */ 32388df0bcf0SPaul Dagnelie /* 32398df0bcf0SPaul Dagnelie * This function gets called just prior to running through the compression 32408df0bcf0SPaul Dagnelie * stage of the zio pipeline. If we're an indirect block comprised of only 32418df0bcf0SPaul Dagnelie * holes, then we want this indirect to be compressed away to a hole. In 32428df0bcf0SPaul Dagnelie * order to do that we must zero out any information about the holes that 32438df0bcf0SPaul Dagnelie * this indirect points to prior to before we try to compress it. 32448df0bcf0SPaul Dagnelie */ 32458df0bcf0SPaul Dagnelie static void 32468df0bcf0SPaul Dagnelie dbuf_write_children_ready(zio_t *zio, arc_buf_t *buf, void *vdb) 32478df0bcf0SPaul Dagnelie { 32488df0bcf0SPaul Dagnelie dmu_buf_impl_t *db = vdb; 32498df0bcf0SPaul Dagnelie dnode_t *dn; 32508df0bcf0SPaul Dagnelie blkptr_t *bp; 32518df0bcf0SPaul Dagnelie uint64_t i; 32528df0bcf0SPaul Dagnelie int epbs; 32538df0bcf0SPaul Dagnelie 32548df0bcf0SPaul Dagnelie ASSERT3U(db->db_level, >, 0); 32558df0bcf0SPaul Dagnelie DB_DNODE_ENTER(db); 32568df0bcf0SPaul Dagnelie dn = DB_DNODE(db); 32578df0bcf0SPaul Dagnelie epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 32588df0bcf0SPaul Dagnelie 32598df0bcf0SPaul Dagnelie /* Determine if all our children are holes */ 32608df0bcf0SPaul Dagnelie for (i = 0, bp = db->db.db_data; i < 1 << epbs; i++, bp++) { 32618df0bcf0SPaul Dagnelie if (!BP_IS_HOLE(bp)) 32628df0bcf0SPaul Dagnelie break; 32638df0bcf0SPaul Dagnelie } 32648df0bcf0SPaul Dagnelie 32658df0bcf0SPaul Dagnelie /* 32668df0bcf0SPaul Dagnelie * If all the children are holes, then zero them all out so that 32678df0bcf0SPaul Dagnelie * we may get compressed away. 32688df0bcf0SPaul Dagnelie */ 32698df0bcf0SPaul Dagnelie if (i == 1 << epbs) { 32708df0bcf0SPaul Dagnelie /* didn't find any non-holes */ 32718df0bcf0SPaul Dagnelie bzero(db->db.db_data, db->db.db_size); 32728df0bcf0SPaul Dagnelie } 32738df0bcf0SPaul Dagnelie DB_DNODE_EXIT(db); 32748df0bcf0SPaul Dagnelie } 32758df0bcf0SPaul Dagnelie 327669962b56SMatthew Ahrens /* 327769962b56SMatthew Ahrens * The SPA will call this callback several times for each zio - once 327869962b56SMatthew Ahrens * for every physical child i/o (zio->io_phys_children times). This 327969962b56SMatthew Ahrens * allows the DMU to monitor the progress of each logical i/o. For example, 328069962b56SMatthew Ahrens * there may be 2 copies of an indirect block, or many fragments of a RAID-Z 328169962b56SMatthew Ahrens * block. There may be a long delay before all copies/fragments are completed, 328269962b56SMatthew Ahrens * so this callback allows us to retire dirty space gradually, as the physical 328369962b56SMatthew Ahrens * i/os complete. 328469962b56SMatthew Ahrens */ 328569962b56SMatthew Ahrens /* ARGSUSED */ 328669962b56SMatthew Ahrens static void 328769962b56SMatthew Ahrens dbuf_write_physdone(zio_t *zio, arc_buf_t *buf, void *arg) 328869962b56SMatthew Ahrens { 328969962b56SMatthew Ahrens dmu_buf_impl_t *db = arg; 329069962b56SMatthew Ahrens objset_t *os = db->db_objset; 329169962b56SMatthew Ahrens dsl_pool_t *dp = dmu_objset_pool(os); 329269962b56SMatthew Ahrens dbuf_dirty_record_t *dr; 329369962b56SMatthew Ahrens int delta = 0; 329469962b56SMatthew Ahrens 329569962b56SMatthew Ahrens dr = db->db_data_pending; 329669962b56SMatthew Ahrens ASSERT3U(dr->dr_txg, ==, zio->io_txg); 329769962b56SMatthew Ahrens 329869962b56SMatthew Ahrens /* 329969962b56SMatthew Ahrens * The callback will be called io_phys_children times. Retire one 330069962b56SMatthew Ahrens * portion of our dirty space each time we are called. Any rounding 330169962b56SMatthew Ahrens * error will be cleaned up by dsl_pool_sync()'s call to 330269962b56SMatthew Ahrens * dsl_pool_undirty_space(). 330369962b56SMatthew Ahrens */ 330469962b56SMatthew Ahrens delta = dr->dr_accounted / zio->io_phys_children; 330569962b56SMatthew Ahrens dsl_pool_undirty_space(dp, delta, zio->io_txg); 330669962b56SMatthew Ahrens } 330769962b56SMatthew Ahrens 3308c717a561Smaybee /* ARGSUSED */ 3309c717a561Smaybee static void 3310c717a561Smaybee dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb) 3311c717a561Smaybee { 3312c717a561Smaybee dmu_buf_impl_t *db = vdb; 3313b24ab676SJeff Bonwick blkptr_t *bp_orig = &zio->io_bp_orig; 331443466aaeSMax Grossman blkptr_t *bp = db->db_blkptr; 331543466aaeSMax Grossman objset_t *os = db->db_objset; 331643466aaeSMax Grossman dmu_tx_t *tx = os->os_synctx; 3317c717a561Smaybee dbuf_dirty_record_t **drp, *dr; 3318c717a561Smaybee 3319fb09f5aaSMadhav Suresh ASSERT0(zio->io_error); 3320b24ab676SJeff Bonwick ASSERT(db->db_blkptr == bp); 3321b24ab676SJeff Bonwick 332280901aeaSGeorge Wilson /* 332380901aeaSGeorge Wilson * For nopwrites and rewrites we ensure that the bp matches our 332480901aeaSGeorge Wilson * original and bypass all the accounting. 332580901aeaSGeorge Wilson */ 332680901aeaSGeorge Wilson if (zio->io_flags & (ZIO_FLAG_IO_REWRITE | ZIO_FLAG_NOPWRITE)) { 3327b24ab676SJeff Bonwick ASSERT(BP_EQUAL(bp, bp_orig)); 3328b24ab676SJeff Bonwick } else { 332943466aaeSMax Grossman dsl_dataset_t *ds = os->os_dsl_dataset; 3330b24ab676SJeff Bonwick (void) dsl_dataset_block_kill(ds, bp_orig, tx, B_TRUE); 3331b24ab676SJeff Bonwick dsl_dataset_block_born(ds, bp, tx); 3332b24ab676SJeff Bonwick } 3333c717a561Smaybee 3334c717a561Smaybee mutex_enter(&db->db_mtx); 3335c717a561Smaybee 3336b24ab676SJeff Bonwick DBUF_VERIFY(db); 3337b24ab676SJeff Bonwick 3338c717a561Smaybee drp = &db->db_last_dirty; 333917f17c2dSbonwick while ((dr = *drp) != db->db_data_pending) 334017f17c2dSbonwick drp = &dr->dr_next; 334117f17c2dSbonwick ASSERT(!list_link_active(&dr->dr_dirty_node)); 3342b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 334317f17c2dSbonwick ASSERT(dr->dr_next == NULL); 334417f17c2dSbonwick *drp = dr->dr_next; 3345c717a561Smaybee 33460a586ceaSMark Shellenbaum #ifdef ZFS_DEBUG 33470a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) { 3348744947dcSTom Erickson dnode_t *dn; 3349744947dcSTom Erickson 3350744947dcSTom Erickson DB_DNODE_ENTER(db); 3351744947dcSTom Erickson dn = DB_DNODE(db); 33520a586ceaSMark Shellenbaum ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); 33530a586ceaSMark Shellenbaum ASSERT(!(BP_IS_HOLE(db->db_blkptr)) && 33540a586ceaSMark Shellenbaum db->db_blkptr == &dn->dn_phys->dn_spill); 3355744947dcSTom Erickson DB_DNODE_EXIT(db); 33560a586ceaSMark Shellenbaum } 33570a586ceaSMark Shellenbaum #endif 33580a586ceaSMark Shellenbaum 3359c717a561Smaybee if (db->db_level == 0) { 33600a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 3361c717a561Smaybee ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN); 336282c9918fSTim Haley if (db->db_state != DB_NOFILL) { 3363c717a561Smaybee if (dr->dt.dl.dr_data != db->db_buf) 3364*dcbf3bd6SGeorge Wilson arc_buf_destroy(dr->dt.dl.dr_data, db); 336582c9918fSTim Haley } 3366c717a561Smaybee } else { 3367744947dcSTom Erickson dnode_t *dn; 3368744947dcSTom Erickson 3369744947dcSTom Erickson DB_DNODE_ENTER(db); 3370744947dcSTom Erickson dn = DB_DNODE(db); 3371c717a561Smaybee ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 3372c717a561Smaybee ASSERT3U(db->db.db_size, ==, 1 << dn->dn_phys->dn_indblkshift); 3373c717a561Smaybee if (!BP_IS_HOLE(db->db_blkptr)) { 3374c717a561Smaybee int epbs = 3375c717a561Smaybee dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 337643466aaeSMax Grossman ASSERT3U(db->db_blkid, <=, 337743466aaeSMax Grossman dn->dn_phys->dn_maxblkid >> (db->db_level * epbs)); 3378c717a561Smaybee ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, 3379c717a561Smaybee db->db.db_size); 3380c717a561Smaybee } 3381744947dcSTom Erickson DB_DNODE_EXIT(db); 3382c25056deSgw25295 mutex_destroy(&dr->dt.di.dr_mtx); 3383c25056deSgw25295 list_destroy(&dr->dt.di.dr_children); 3384c717a561Smaybee } 3385c717a561Smaybee kmem_free(dr, sizeof (dbuf_dirty_record_t)); 3386c717a561Smaybee 3387c717a561Smaybee cv_broadcast(&db->db_changed); 3388c717a561Smaybee ASSERT(db->db_dirtycnt > 0); 3389c717a561Smaybee db->db_dirtycnt -= 1; 3390c717a561Smaybee db->db_data_pending = NULL; 339143466aaeSMax Grossman dbuf_rele_and_unlock(db, (void *)(uintptr_t)tx->tx_txg); 3392b24ab676SJeff Bonwick } 3393b24ab676SJeff Bonwick 3394b24ab676SJeff Bonwick static void 3395b24ab676SJeff Bonwick dbuf_write_nofill_ready(zio_t *zio) 3396b24ab676SJeff Bonwick { 3397b24ab676SJeff Bonwick dbuf_write_ready(zio, NULL, zio->io_private); 3398b24ab676SJeff Bonwick } 3399b24ab676SJeff Bonwick 3400b24ab676SJeff Bonwick static void 3401b24ab676SJeff Bonwick dbuf_write_nofill_done(zio_t *zio) 3402b24ab676SJeff Bonwick { 3403b24ab676SJeff Bonwick dbuf_write_done(zio, NULL, zio->io_private); 3404b24ab676SJeff Bonwick } 3405b24ab676SJeff Bonwick 3406b24ab676SJeff Bonwick static void 3407b24ab676SJeff Bonwick dbuf_write_override_ready(zio_t *zio) 3408b24ab676SJeff Bonwick { 3409b24ab676SJeff Bonwick dbuf_dirty_record_t *dr = zio->io_private; 3410b24ab676SJeff Bonwick dmu_buf_impl_t *db = dr->dr_dbuf; 3411b24ab676SJeff Bonwick 3412b24ab676SJeff Bonwick dbuf_write_ready(zio, NULL, db); 3413b24ab676SJeff Bonwick } 3414b24ab676SJeff Bonwick 3415b24ab676SJeff Bonwick static void 3416b24ab676SJeff Bonwick dbuf_write_override_done(zio_t *zio) 3417b24ab676SJeff Bonwick { 3418b24ab676SJeff Bonwick dbuf_dirty_record_t *dr = zio->io_private; 3419b24ab676SJeff Bonwick dmu_buf_impl_t *db = dr->dr_dbuf; 3420b24ab676SJeff Bonwick blkptr_t *obp = &dr->dt.dl.dr_overridden_by; 3421b24ab676SJeff Bonwick 3422b24ab676SJeff Bonwick mutex_enter(&db->db_mtx); 3423b24ab676SJeff Bonwick if (!BP_EQUAL(zio->io_bp, obp)) { 3424b24ab676SJeff Bonwick if (!BP_IS_HOLE(obp)) 3425b24ab676SJeff Bonwick dsl_free(spa_get_dsl(zio->io_spa), zio->io_txg, obp); 3426b24ab676SJeff Bonwick arc_release(dr->dt.dl.dr_data, db); 3427b24ab676SJeff Bonwick } 3428c717a561Smaybee mutex_exit(&db->db_mtx); 3429c717a561Smaybee 3430b24ab676SJeff Bonwick dbuf_write_done(zio, NULL, db); 3431b24ab676SJeff Bonwick } 3432c717a561Smaybee 34333e30c24aSWill Andrews /* Issue I/O to commit a dirty buffer to disk. */ 3434b24ab676SJeff Bonwick static void 3435b24ab676SJeff Bonwick dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx) 3436b24ab676SJeff Bonwick { 3437b24ab676SJeff Bonwick dmu_buf_impl_t *db = dr->dr_dbuf; 3438744947dcSTom Erickson dnode_t *dn; 3439744947dcSTom Erickson objset_t *os; 3440b24ab676SJeff Bonwick dmu_buf_impl_t *parent = db->db_parent; 3441b24ab676SJeff Bonwick uint64_t txg = tx->tx_txg; 34427802d7bfSMatthew Ahrens zbookmark_phys_t zb; 3443b24ab676SJeff Bonwick zio_prop_t zp; 3444b24ab676SJeff Bonwick zio_t *zio; 34450a586ceaSMark Shellenbaum int wp_flag = 0; 3446b24ab676SJeff Bonwick 344711ceac77SAlex Reece ASSERT(dmu_tx_is_syncing(tx)); 344811ceac77SAlex Reece 3449744947dcSTom Erickson DB_DNODE_ENTER(db); 3450744947dcSTom Erickson dn = DB_DNODE(db); 3451744947dcSTom Erickson os = dn->dn_objset; 3452744947dcSTom Erickson 3453b24ab676SJeff Bonwick if (db->db_state != DB_NOFILL) { 3454b24ab676SJeff Bonwick if (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE) { 3455b24ab676SJeff Bonwick /* 3456b24ab676SJeff Bonwick * Private object buffers are released here rather 3457b24ab676SJeff Bonwick * than in dbuf_dirty() since they are only modified 3458b24ab676SJeff Bonwick * in the syncing context and we don't want the 3459b24ab676SJeff Bonwick * overhead of making multiple copies of the data. 3460b24ab676SJeff Bonwick */ 3461b24ab676SJeff Bonwick if (BP_IS_HOLE(db->db_blkptr)) { 3462b24ab676SJeff Bonwick arc_buf_thaw(data); 3463b24ab676SJeff Bonwick } else { 34643f9d6ad7SLin Ling dbuf_release_bp(db); 3465b24ab676SJeff Bonwick } 3466b24ab676SJeff Bonwick } 3467b24ab676SJeff Bonwick } 3468b24ab676SJeff Bonwick 3469b24ab676SJeff Bonwick if (parent != dn->dn_dbuf) { 34703e30c24aSWill Andrews /* Our parent is an indirect block. */ 34713e30c24aSWill Andrews /* We have a dirty parent that has been scheduled for write. */ 3472b24ab676SJeff Bonwick ASSERT(parent && parent->db_data_pending); 34733e30c24aSWill Andrews /* Our parent's buffer is one level closer to the dnode. */ 3474b24ab676SJeff Bonwick ASSERT(db->db_level == parent->db_level-1); 34753e30c24aSWill Andrews /* 34763e30c24aSWill Andrews * We're about to modify our parent's db_data by modifying 34773e30c24aSWill Andrews * our block pointer, so the parent must be released. 34783e30c24aSWill Andrews */ 3479b24ab676SJeff Bonwick ASSERT(arc_released(parent->db_buf)); 3480b24ab676SJeff Bonwick zio = parent->db_data_pending->dr_zio; 3481b24ab676SJeff Bonwick } else { 34823e30c24aSWill Andrews /* Our parent is the dnode itself. */ 34830a586ceaSMark Shellenbaum ASSERT((db->db_level == dn->dn_phys->dn_nlevels-1 && 34840a586ceaSMark Shellenbaum db->db_blkid != DMU_SPILL_BLKID) || 34850a586ceaSMark Shellenbaum (db->db_blkid == DMU_SPILL_BLKID && db->db_level == 0)); 34860a586ceaSMark Shellenbaum if (db->db_blkid != DMU_SPILL_BLKID) 3487b24ab676SJeff Bonwick ASSERT3P(db->db_blkptr, ==, 3488b24ab676SJeff Bonwick &dn->dn_phys->dn_blkptr[db->db_blkid]); 3489b24ab676SJeff Bonwick zio = dn->dn_zio; 3490b24ab676SJeff Bonwick } 3491b24ab676SJeff Bonwick 3492b24ab676SJeff Bonwick ASSERT(db->db_level == 0 || data == db->db_buf); 3493b24ab676SJeff Bonwick ASSERT3U(db->db_blkptr->blk_birth, <=, txg); 3494b24ab676SJeff Bonwick ASSERT(zio); 3495b24ab676SJeff Bonwick 3496b24ab676SJeff Bonwick SET_BOOKMARK(&zb, os->os_dsl_dataset ? 3497b24ab676SJeff Bonwick os->os_dsl_dataset->ds_object : DMU_META_OBJSET, 3498b24ab676SJeff Bonwick db->db.db_object, db->db_level, db->db_blkid); 3499b24ab676SJeff Bonwick 35000a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) 35010a586ceaSMark Shellenbaum wp_flag = WP_SPILL; 35020a586ceaSMark Shellenbaum wp_flag |= (db->db_state == DB_NOFILL) ? WP_NOFILL : 0; 35030a586ceaSMark Shellenbaum 35040a586ceaSMark Shellenbaum dmu_write_policy(os, dn, db->db_level, wp_flag, &zp); 3505744947dcSTom Erickson DB_DNODE_EXIT(db); 3506b24ab676SJeff Bonwick 350711ceac77SAlex Reece /* 350811ceac77SAlex Reece * We copy the blkptr now (rather than when we instantiate the dirty 350911ceac77SAlex Reece * record), because its value can change between open context and 351011ceac77SAlex Reece * syncing context. We do not need to hold dn_struct_rwlock to read 351111ceac77SAlex Reece * db_blkptr because we are in syncing context. 351211ceac77SAlex Reece */ 351311ceac77SAlex Reece dr->dr_bp_copy = *db->db_blkptr; 351411ceac77SAlex Reece 35155d7b4d43SMatthew Ahrens if (db->db_level == 0 && 35165d7b4d43SMatthew Ahrens dr->dt.dl.dr_override_state == DR_OVERRIDDEN) { 35175d7b4d43SMatthew Ahrens /* 35185d7b4d43SMatthew Ahrens * The BP for this block has been provided by open context 35195d7b4d43SMatthew Ahrens * (by dmu_sync() or dmu_buf_write_embedded()). 35205d7b4d43SMatthew Ahrens */ 35215d7b4d43SMatthew Ahrens void *contents = (data != NULL) ? data->b_data : NULL; 35225d7b4d43SMatthew Ahrens 3523b24ab676SJeff Bonwick dr->dr_zio = zio_write(zio, os->os_spa, txg, 352411ceac77SAlex Reece &dr->dr_bp_copy, contents, db->db.db_size, &zp, 35258df0bcf0SPaul Dagnelie dbuf_write_override_ready, NULL, NULL, 35268df0bcf0SPaul Dagnelie dbuf_write_override_done, 352769962b56SMatthew Ahrens dr, ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); 3528b24ab676SJeff Bonwick mutex_enter(&db->db_mtx); 3529b24ab676SJeff Bonwick dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 3530b24ab676SJeff Bonwick zio_write_override(dr->dr_zio, &dr->dt.dl.dr_overridden_by, 353180901aeaSGeorge Wilson dr->dt.dl.dr_copies, dr->dt.dl.dr_nopwrite); 3532b24ab676SJeff Bonwick mutex_exit(&db->db_mtx); 3533b24ab676SJeff Bonwick } else if (db->db_state == DB_NOFILL) { 3534810e43b2SBill Pijewski ASSERT(zp.zp_checksum == ZIO_CHECKSUM_OFF || 3535810e43b2SBill Pijewski zp.zp_checksum == ZIO_CHECKSUM_NOPARITY); 3536b24ab676SJeff Bonwick dr->dr_zio = zio_write(zio, os->os_spa, txg, 353711ceac77SAlex Reece &dr->dr_bp_copy, NULL, db->db.db_size, &zp, 35388df0bcf0SPaul Dagnelie dbuf_write_nofill_ready, NULL, NULL, 35398df0bcf0SPaul Dagnelie dbuf_write_nofill_done, db, 3540b24ab676SJeff Bonwick ZIO_PRIORITY_ASYNC_WRITE, 3541b24ab676SJeff Bonwick ZIO_FLAG_MUSTSUCCEED | ZIO_FLAG_NODATA, &zb); 3542b24ab676SJeff Bonwick } else { 3543b24ab676SJeff Bonwick ASSERT(arc_released(data)); 35448df0bcf0SPaul Dagnelie 35458df0bcf0SPaul Dagnelie /* 35468df0bcf0SPaul Dagnelie * For indirect blocks, we want to setup the children 35478df0bcf0SPaul Dagnelie * ready callback so that we can properly handle an indirect 35488df0bcf0SPaul Dagnelie * block that only contains holes. 35498df0bcf0SPaul Dagnelie */ 35508df0bcf0SPaul Dagnelie arc_done_func_t *children_ready_cb = NULL; 35518df0bcf0SPaul Dagnelie if (db->db_level != 0) 35528df0bcf0SPaul Dagnelie children_ready_cb = dbuf_write_children_ready; 35538df0bcf0SPaul Dagnelie 3554b24ab676SJeff Bonwick dr->dr_zio = arc_write(zio, os->os_spa, txg, 355511ceac77SAlex Reece &dr->dr_bp_copy, data, DBUF_IS_L2CACHEABLE(db), 3556*dcbf3bd6SGeorge Wilson &zp, dbuf_write_ready, children_ready_cb, 355769962b56SMatthew Ahrens dbuf_write_physdone, dbuf_write_done, db, 355869962b56SMatthew Ahrens ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); 3559b24ab676SJeff Bonwick } 3560fa9e4066Sahrens } 3561