1*61145dc2SMartin Matuska // SPDX-License-Identifier: CDDL-1.0 2eda14cbcSMatt Macy /* 3eda14cbcSMatt Macy * CDDL HEADER START 4eda14cbcSMatt Macy * 5eda14cbcSMatt Macy * The contents of this file are subject to the terms of the 6eda14cbcSMatt Macy * Common Development and Distribution License (the "License"). 7eda14cbcSMatt Macy * You may not use this file except in compliance with the License. 8eda14cbcSMatt Macy * 9eda14cbcSMatt Macy * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10271171e0SMartin Matuska * or https://opensource.org/licenses/CDDL-1.0. 11eda14cbcSMatt Macy * See the License for the specific language governing permissions 12eda14cbcSMatt Macy * and limitations under the License. 13eda14cbcSMatt Macy * 14eda14cbcSMatt Macy * When distributing Covered Code, include this CDDL HEADER in each 15eda14cbcSMatt Macy * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16eda14cbcSMatt Macy * If applicable, add the following below this CDDL HEADER, with the 17eda14cbcSMatt Macy * fields enclosed by brackets "[]" replaced with your own identifying 18eda14cbcSMatt Macy * information: Portions Copyright [yyyy] [name of copyright owner] 19eda14cbcSMatt Macy * 20eda14cbcSMatt Macy * CDDL HEADER END 21eda14cbcSMatt Macy */ 22eda14cbcSMatt Macy 23eda14cbcSMatt Macy /* 24eda14cbcSMatt Macy * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 257877fdebSMatt Macy * Copyright (c) 2012, 2020 by Delphix. All rights reserved. 26eda14cbcSMatt Macy * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. 27ac0bf12eSMatt Macy * Copyright 2020 Oxide Computer Company 28eda14cbcSMatt Macy */ 29eda14cbcSMatt Macy 30eda14cbcSMatt Macy #include <sys/zfs_context.h> 31eda14cbcSMatt Macy #include <sys/dbuf.h> 32eda14cbcSMatt Macy #include <sys/dnode.h> 33eda14cbcSMatt Macy #include <sys/dmu.h> 34eda14cbcSMatt Macy #include <sys/dmu_tx.h> 35eda14cbcSMatt Macy #include <sys/dmu_objset.h> 36eda14cbcSMatt Macy #include <sys/dmu_recv.h> 37eda14cbcSMatt Macy #include <sys/dsl_dataset.h> 38eda14cbcSMatt Macy #include <sys/spa.h> 39eda14cbcSMatt Macy #include <sys/range_tree.h> 40eda14cbcSMatt Macy #include <sys/zfeature.h> 41eda14cbcSMatt Macy 42eda14cbcSMatt Macy static void 43eda14cbcSMatt Macy dnode_increase_indirection(dnode_t *dn, dmu_tx_t *tx) 44eda14cbcSMatt Macy { 45eda14cbcSMatt Macy dmu_buf_impl_t *db; 46eda14cbcSMatt Macy int txgoff = tx->tx_txg & TXG_MASK; 47eda14cbcSMatt Macy int nblkptr = dn->dn_phys->dn_nblkptr; 48eda14cbcSMatt Macy int old_toplvl = dn->dn_phys->dn_nlevels - 1; 49eda14cbcSMatt Macy int new_level = dn->dn_next_nlevels[txgoff]; 50eda14cbcSMatt Macy int i; 51eda14cbcSMatt Macy 52eda14cbcSMatt Macy rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 53eda14cbcSMatt Macy 54eda14cbcSMatt Macy /* this dnode can't be paged out because it's dirty */ 55eda14cbcSMatt Macy ASSERT(dn->dn_phys->dn_type != DMU_OT_NONE); 56eda14cbcSMatt Macy ASSERT(new_level > 1 && dn->dn_phys->dn_nlevels > 0); 57eda14cbcSMatt Macy 58eda14cbcSMatt Macy db = dbuf_hold_level(dn, dn->dn_phys->dn_nlevels, 0, FTAG); 59eda14cbcSMatt Macy ASSERT(db != NULL); 60eda14cbcSMatt Macy 61eda14cbcSMatt Macy dn->dn_phys->dn_nlevels = new_level; 62eda14cbcSMatt Macy dprintf("os=%p obj=%llu, increase to %d\n", dn->dn_objset, 6333b8c039SMartin Matuska (u_longlong_t)dn->dn_object, dn->dn_phys->dn_nlevels); 64eda14cbcSMatt Macy 65eda14cbcSMatt Macy /* 66eda14cbcSMatt Macy * Lock ordering requires that we hold the children's db_mutexes (by 67eda14cbcSMatt Macy * calling dbuf_find()) before holding the parent's db_rwlock. The lock 68eda14cbcSMatt Macy * order is imposed by dbuf_read's steps of "grab the lock to protect 69eda14cbcSMatt Macy * db_parent, get db_parent, hold db_parent's db_rwlock". 70eda14cbcSMatt Macy */ 71eda14cbcSMatt Macy dmu_buf_impl_t *children[DN_MAX_NBLKPTR]; 72eda14cbcSMatt Macy ASSERT3U(nblkptr, <=, DN_MAX_NBLKPTR); 73eda14cbcSMatt Macy for (i = 0; i < nblkptr; i++) { 7415f0b8c3SMartin Matuska children[i] = dbuf_find(dn->dn_objset, dn->dn_object, 7515f0b8c3SMartin Matuska old_toplvl, i, NULL); 76eda14cbcSMatt Macy } 77eda14cbcSMatt Macy 78eda14cbcSMatt Macy /* transfer dnode's block pointers to new indirect block */ 79eda14cbcSMatt Macy (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED|DB_RF_HAVESTRUCT); 80eda14cbcSMatt Macy if (dn->dn_dbuf != NULL) 81eda14cbcSMatt Macy rw_enter(&dn->dn_dbuf->db_rwlock, RW_WRITER); 82eda14cbcSMatt Macy rw_enter(&db->db_rwlock, RW_WRITER); 83eda14cbcSMatt Macy ASSERT(db->db.db_data); 84eda14cbcSMatt Macy ASSERT(arc_released(db->db_buf)); 85eda14cbcSMatt Macy ASSERT3U(sizeof (blkptr_t) * nblkptr, <=, db->db.db_size); 86da5137abSMartin Matuska memcpy(db->db.db_data, dn->dn_phys->dn_blkptr, 87eda14cbcSMatt Macy sizeof (blkptr_t) * nblkptr); 88eda14cbcSMatt Macy arc_buf_freeze(db->db_buf); 89eda14cbcSMatt Macy 90eda14cbcSMatt Macy /* set dbuf's parent pointers to new indirect buf */ 91eda14cbcSMatt Macy for (i = 0; i < nblkptr; i++) { 92eda14cbcSMatt Macy dmu_buf_impl_t *child = children[i]; 93eda14cbcSMatt Macy 94eda14cbcSMatt Macy if (child == NULL) 95eda14cbcSMatt Macy continue; 96eda14cbcSMatt Macy #ifdef ZFS_DEBUG 97eda14cbcSMatt Macy DB_DNODE_ENTER(child); 98eda14cbcSMatt Macy ASSERT3P(DB_DNODE(child), ==, dn); 99eda14cbcSMatt Macy DB_DNODE_EXIT(child); 100eda14cbcSMatt Macy #endif /* DEBUG */ 101eda14cbcSMatt Macy if (child->db_parent && child->db_parent != dn->dn_dbuf) { 102eda14cbcSMatt Macy ASSERT(child->db_parent->db_level == db->db_level); 103eda14cbcSMatt Macy ASSERT(child->db_blkptr != 104eda14cbcSMatt Macy &dn->dn_phys->dn_blkptr[child->db_blkid]); 105eda14cbcSMatt Macy mutex_exit(&child->db_mtx); 106eda14cbcSMatt Macy continue; 107eda14cbcSMatt Macy } 108eda14cbcSMatt Macy ASSERT(child->db_parent == NULL || 109eda14cbcSMatt Macy child->db_parent == dn->dn_dbuf); 110eda14cbcSMatt Macy 111eda14cbcSMatt Macy child->db_parent = db; 112eda14cbcSMatt Macy dbuf_add_ref(db, child); 113eda14cbcSMatt Macy if (db->db.db_data) 114eda14cbcSMatt Macy child->db_blkptr = (blkptr_t *)db->db.db_data + i; 115eda14cbcSMatt Macy else 116eda14cbcSMatt Macy child->db_blkptr = NULL; 117eda14cbcSMatt Macy dprintf_dbuf_bp(child, child->db_blkptr, 118eda14cbcSMatt Macy "changed db_blkptr to new indirect %s", ""); 119eda14cbcSMatt Macy 120eda14cbcSMatt Macy mutex_exit(&child->db_mtx); 121eda14cbcSMatt Macy } 122eda14cbcSMatt Macy 123da5137abSMartin Matuska memset(dn->dn_phys->dn_blkptr, 0, sizeof (blkptr_t) * nblkptr); 124eda14cbcSMatt Macy 125eda14cbcSMatt Macy rw_exit(&db->db_rwlock); 126eda14cbcSMatt Macy if (dn->dn_dbuf != NULL) 127eda14cbcSMatt Macy rw_exit(&dn->dn_dbuf->db_rwlock); 128eda14cbcSMatt Macy 129eda14cbcSMatt Macy dbuf_rele(db, FTAG); 130eda14cbcSMatt Macy 131eda14cbcSMatt Macy rw_exit(&dn->dn_struct_rwlock); 132eda14cbcSMatt Macy } 133eda14cbcSMatt Macy 134eda14cbcSMatt Macy static void 135eda14cbcSMatt Macy free_blocks(dnode_t *dn, blkptr_t *bp, int num, dmu_tx_t *tx) 136eda14cbcSMatt Macy { 137eda14cbcSMatt Macy dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset; 138eda14cbcSMatt Macy uint64_t bytesfreed = 0; 139eda14cbcSMatt Macy 14033b8c039SMartin Matuska dprintf("ds=%p obj=%llx num=%d\n", ds, (u_longlong_t)dn->dn_object, 14133b8c039SMartin Matuska num); 142eda14cbcSMatt Macy 143eda14cbcSMatt Macy for (int i = 0; i < num; i++, bp++) { 144eda14cbcSMatt Macy if (BP_IS_HOLE(bp)) 145eda14cbcSMatt Macy continue; 146eda14cbcSMatt Macy 147eda14cbcSMatt Macy bytesfreed += dsl_dataset_block_kill(ds, bp, tx, B_FALSE); 148eda14cbcSMatt Macy ASSERT3U(bytesfreed, <=, DN_USED_BYTES(dn->dn_phys)); 149eda14cbcSMatt Macy 150eda14cbcSMatt Macy /* 151eda14cbcSMatt Macy * Save some useful information on the holes being 152eda14cbcSMatt Macy * punched, including logical size, type, and indirection 153eda14cbcSMatt Macy * level. Retaining birth time enables detection of when 154eda14cbcSMatt Macy * holes are punched for reducing the number of free 155eda14cbcSMatt Macy * records transmitted during a zfs send. 156eda14cbcSMatt Macy */ 157eda14cbcSMatt Macy 158eda14cbcSMatt Macy uint64_t lsize = BP_GET_LSIZE(bp); 159eda14cbcSMatt Macy dmu_object_type_t type = BP_GET_TYPE(bp); 160eda14cbcSMatt Macy uint64_t lvl = BP_GET_LEVEL(bp); 161eda14cbcSMatt Macy 162da5137abSMartin Matuska memset(bp, 0, sizeof (blkptr_t)); 163eda14cbcSMatt Macy 164eda14cbcSMatt Macy if (spa_feature_is_active(dn->dn_objset->os_spa, 165eda14cbcSMatt Macy SPA_FEATURE_HOLE_BIRTH)) { 166eda14cbcSMatt Macy BP_SET_LSIZE(bp, lsize); 167eda14cbcSMatt Macy BP_SET_TYPE(bp, type); 168eda14cbcSMatt Macy BP_SET_LEVEL(bp, lvl); 169eda14cbcSMatt Macy BP_SET_BIRTH(bp, dmu_tx_get_txg(tx), 0); 170eda14cbcSMatt Macy } 171eda14cbcSMatt Macy } 172eda14cbcSMatt Macy dnode_diduse_space(dn, -bytesfreed); 173eda14cbcSMatt Macy } 174eda14cbcSMatt Macy 175eda14cbcSMatt Macy #ifdef ZFS_DEBUG 176eda14cbcSMatt Macy static void 177eda14cbcSMatt Macy free_verify(dmu_buf_impl_t *db, uint64_t start, uint64_t end, dmu_tx_t *tx) 178eda14cbcSMatt Macy { 17915f0b8c3SMartin Matuska uint64_t off, num, i, j; 18015f0b8c3SMartin Matuska unsigned int epbs; 18115f0b8c3SMartin Matuska int err; 182eda14cbcSMatt Macy uint64_t txg = tx->tx_txg; 183eda14cbcSMatt Macy dnode_t *dn; 184eda14cbcSMatt Macy 185eda14cbcSMatt Macy DB_DNODE_ENTER(db); 186eda14cbcSMatt Macy dn = DB_DNODE(db); 187eda14cbcSMatt Macy epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 18815f0b8c3SMartin Matuska off = start - (db->db_blkid << epbs); 189eda14cbcSMatt Macy num = end - start + 1; 190eda14cbcSMatt Macy 19115f0b8c3SMartin Matuska ASSERT3U(dn->dn_phys->dn_indblkshift, >=, SPA_BLKPTRSHIFT); 19215f0b8c3SMartin Matuska ASSERT3U(end + 1, >=, start); 19315f0b8c3SMartin Matuska ASSERT3U(start, >=, (db->db_blkid << epbs)); 194eda14cbcSMatt Macy ASSERT3U(db->db_level, >, 0); 195eda14cbcSMatt Macy ASSERT3U(db->db.db_size, ==, 1 << dn->dn_phys->dn_indblkshift); 196eda14cbcSMatt Macy ASSERT3U(off+num, <=, db->db.db_size >> SPA_BLKPTRSHIFT); 197eda14cbcSMatt Macy ASSERT(db->db_blkptr != NULL); 198eda14cbcSMatt Macy 199eda14cbcSMatt Macy for (i = off; i < off+num; i++) { 200eda14cbcSMatt Macy uint64_t *buf; 201eda14cbcSMatt Macy dmu_buf_impl_t *child; 202eda14cbcSMatt Macy dbuf_dirty_record_t *dr; 203eda14cbcSMatt Macy 204eda14cbcSMatt Macy ASSERT(db->db_level == 1); 205eda14cbcSMatt Macy 206eda14cbcSMatt Macy rw_enter(&dn->dn_struct_rwlock, RW_READER); 207eda14cbcSMatt Macy err = dbuf_hold_impl(dn, db->db_level - 1, 208eda14cbcSMatt Macy (db->db_blkid << epbs) + i, TRUE, FALSE, FTAG, &child); 209eda14cbcSMatt Macy rw_exit(&dn->dn_struct_rwlock); 210eda14cbcSMatt Macy if (err == ENOENT) 211eda14cbcSMatt Macy continue; 212eda14cbcSMatt Macy ASSERT(err == 0); 213eda14cbcSMatt Macy ASSERT(child->db_level == 0); 214eda14cbcSMatt Macy dr = dbuf_find_dirty_eq(child, txg); 215eda14cbcSMatt Macy 216eda14cbcSMatt Macy /* data_old better be zeroed */ 217eda14cbcSMatt Macy if (dr) { 218eda14cbcSMatt Macy buf = dr->dt.dl.dr_data->b_data; 219eda14cbcSMatt Macy for (j = 0; j < child->db.db_size >> 3; j++) { 220eda14cbcSMatt Macy if (buf[j] != 0) { 221eda14cbcSMatt Macy panic("freed data not zero: " 22215f0b8c3SMartin Matuska "child=%p i=%llu off=%llu " 22315f0b8c3SMartin Matuska "num=%llu\n", 22415f0b8c3SMartin Matuska (void *)child, (u_longlong_t)i, 22515f0b8c3SMartin Matuska (u_longlong_t)off, 22615f0b8c3SMartin Matuska (u_longlong_t)num); 227eda14cbcSMatt Macy } 228eda14cbcSMatt Macy } 229eda14cbcSMatt Macy } 230eda14cbcSMatt Macy 231eda14cbcSMatt Macy /* 232eda14cbcSMatt Macy * db_data better be zeroed unless it's dirty in a 233eda14cbcSMatt Macy * future txg. 234eda14cbcSMatt Macy */ 235eda14cbcSMatt Macy mutex_enter(&child->db_mtx); 236eda14cbcSMatt Macy buf = child->db.db_data; 237eda14cbcSMatt Macy if (buf != NULL && child->db_state != DB_FILL && 238eda14cbcSMatt Macy list_is_empty(&child->db_dirty_records)) { 239eda14cbcSMatt Macy for (j = 0; j < child->db.db_size >> 3; j++) { 240eda14cbcSMatt Macy if (buf[j] != 0) { 241eda14cbcSMatt Macy panic("freed data not zero: " 24215f0b8c3SMartin Matuska "child=%p i=%llu off=%llu " 24315f0b8c3SMartin Matuska "num=%llu\n", 24415f0b8c3SMartin Matuska (void *)child, (u_longlong_t)i, 24515f0b8c3SMartin Matuska (u_longlong_t)off, 24615f0b8c3SMartin Matuska (u_longlong_t)num); 247eda14cbcSMatt Macy } 248eda14cbcSMatt Macy } 249eda14cbcSMatt Macy } 250eda14cbcSMatt Macy mutex_exit(&child->db_mtx); 251eda14cbcSMatt Macy 252eda14cbcSMatt Macy dbuf_rele(child, FTAG); 253eda14cbcSMatt Macy } 254eda14cbcSMatt Macy DB_DNODE_EXIT(db); 255eda14cbcSMatt Macy } 256eda14cbcSMatt Macy #endif 257eda14cbcSMatt Macy 258eda14cbcSMatt Macy /* 259eda14cbcSMatt Macy * We don't usually free the indirect blocks here. If in one txg we have a 260eda14cbcSMatt Macy * free_range and a write to the same indirect block, it's important that we 261eda14cbcSMatt Macy * preserve the hole's birth times. Therefore, we don't free any any indirect 262eda14cbcSMatt Macy * blocks in free_children(). If an indirect block happens to turn into all 263eda14cbcSMatt Macy * holes, it will be freed by dbuf_write_children_ready, which happens at a 264eda14cbcSMatt Macy * point in the syncing process where we know for certain the contents of the 265eda14cbcSMatt Macy * indirect block. 266eda14cbcSMatt Macy * 267eda14cbcSMatt Macy * However, if we're freeing a dnode, its space accounting must go to zero 268eda14cbcSMatt Macy * before we actually try to free the dnode, or we will trip an assertion. In 269eda14cbcSMatt Macy * addition, we know the case described above cannot occur, because the dnode is 270eda14cbcSMatt Macy * being freed. Therefore, we free the indirect blocks immediately in that 271eda14cbcSMatt Macy * case. 272eda14cbcSMatt Macy */ 273eda14cbcSMatt Macy static void 274eda14cbcSMatt Macy free_children(dmu_buf_impl_t *db, uint64_t blkid, uint64_t nblks, 275eda14cbcSMatt Macy boolean_t free_indirects, dmu_tx_t *tx) 276eda14cbcSMatt Macy { 277eda14cbcSMatt Macy dnode_t *dn; 278eda14cbcSMatt Macy blkptr_t *bp; 279eda14cbcSMatt Macy dmu_buf_impl_t *subdb; 280eda14cbcSMatt Macy uint64_t start, end, dbstart, dbend; 281eda14cbcSMatt Macy unsigned int epbs, shift, i; 282eda14cbcSMatt Macy 283eda14cbcSMatt Macy /* 284eda14cbcSMatt Macy * There is a small possibility that this block will not be cached: 285eda14cbcSMatt Macy * 1 - if level > 1 and there are no children with level <= 1 286eda14cbcSMatt Macy * 2 - if this block was evicted since we read it from 287eda14cbcSMatt Macy * dmu_tx_hold_free(). 288eda14cbcSMatt Macy */ 289eda14cbcSMatt Macy if (db->db_state != DB_CACHED) 290eda14cbcSMatt Macy (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED); 291eda14cbcSMatt Macy 292eda14cbcSMatt Macy /* 293eda14cbcSMatt Macy * If we modify this indirect block, and we are not freeing the 294eda14cbcSMatt Macy * dnode (!free_indirects), then this indirect block needs to get 295eda14cbcSMatt Macy * written to disk by dbuf_write(). If it is dirty, we know it will 296eda14cbcSMatt Macy * be written (otherwise, we would have incorrect on-disk state 297eda14cbcSMatt Macy * because the space would be freed but still referenced by the BP 298eda14cbcSMatt Macy * in this indirect block). Therefore we VERIFY that it is 299eda14cbcSMatt Macy * dirty. 300eda14cbcSMatt Macy * 301eda14cbcSMatt Macy * Our VERIFY covers some cases that do not actually have to be 302eda14cbcSMatt Macy * dirty, but the open-context code happens to dirty. E.g. if the 303eda14cbcSMatt Macy * blocks we are freeing are all holes, because in that case, we 304eda14cbcSMatt Macy * are only freeing part of this indirect block, so it is an 305eda14cbcSMatt Macy * ancestor of the first or last block to be freed. The first and 306eda14cbcSMatt Macy * last L1 indirect blocks are always dirtied by dnode_free_range(). 307eda14cbcSMatt Macy */ 308eda14cbcSMatt Macy db_lock_type_t dblt = dmu_buf_lock_parent(db, RW_READER, FTAG); 309eda14cbcSMatt Macy VERIFY(BP_GET_FILL(db->db_blkptr) == 0 || db->db_dirtycnt > 0); 310eda14cbcSMatt Macy dmu_buf_unlock_parent(db, dblt, FTAG); 311eda14cbcSMatt Macy 312eda14cbcSMatt Macy dbuf_release_bp(db); 313eda14cbcSMatt Macy bp = db->db.db_data; 314eda14cbcSMatt Macy 315eda14cbcSMatt Macy DB_DNODE_ENTER(db); 316eda14cbcSMatt Macy dn = DB_DNODE(db); 317eda14cbcSMatt Macy epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 318eda14cbcSMatt Macy ASSERT3U(epbs, <, 31); 319eda14cbcSMatt Macy shift = (db->db_level - 1) * epbs; 320eda14cbcSMatt Macy dbstart = db->db_blkid << epbs; 321eda14cbcSMatt Macy start = blkid >> shift; 322eda14cbcSMatt Macy if (dbstart < start) { 323eda14cbcSMatt Macy bp += start - dbstart; 324eda14cbcSMatt Macy } else { 325eda14cbcSMatt Macy start = dbstart; 326eda14cbcSMatt Macy } 327eda14cbcSMatt Macy dbend = ((db->db_blkid + 1) << epbs) - 1; 328eda14cbcSMatt Macy end = (blkid + nblks - 1) >> shift; 329eda14cbcSMatt Macy if (dbend <= end) 330eda14cbcSMatt Macy end = dbend; 331eda14cbcSMatt Macy 332eda14cbcSMatt Macy ASSERT3U(start, <=, end); 333eda14cbcSMatt Macy 334eda14cbcSMatt Macy if (db->db_level == 1) { 335eda14cbcSMatt Macy FREE_VERIFY(db, start, end, tx); 336eda14cbcSMatt Macy rw_enter(&db->db_rwlock, RW_WRITER); 337eda14cbcSMatt Macy free_blocks(dn, bp, end - start + 1, tx); 338eda14cbcSMatt Macy rw_exit(&db->db_rwlock); 339eda14cbcSMatt Macy } else { 340eda14cbcSMatt Macy for (uint64_t id = start; id <= end; id++, bp++) { 341eda14cbcSMatt Macy if (BP_IS_HOLE(bp)) 342eda14cbcSMatt Macy continue; 343eda14cbcSMatt Macy rw_enter(&dn->dn_struct_rwlock, RW_READER); 344eda14cbcSMatt Macy VERIFY0(dbuf_hold_impl(dn, db->db_level - 1, 345eda14cbcSMatt Macy id, TRUE, FALSE, FTAG, &subdb)); 346eda14cbcSMatt Macy rw_exit(&dn->dn_struct_rwlock); 347eda14cbcSMatt Macy ASSERT3P(bp, ==, subdb->db_blkptr); 348eda14cbcSMatt Macy 349eda14cbcSMatt Macy free_children(subdb, blkid, nblks, free_indirects, tx); 350eda14cbcSMatt Macy dbuf_rele(subdb, FTAG); 351eda14cbcSMatt Macy } 352eda14cbcSMatt Macy } 353eda14cbcSMatt Macy 354eda14cbcSMatt Macy if (free_indirects) { 355eda14cbcSMatt Macy rw_enter(&db->db_rwlock, RW_WRITER); 356eda14cbcSMatt Macy for (i = 0, bp = db->db.db_data; i < 1 << epbs; i++, bp++) 357eda14cbcSMatt Macy ASSERT(BP_IS_HOLE(bp)); 358da5137abSMartin Matuska memset(db->db.db_data, 0, db->db.db_size); 359eda14cbcSMatt Macy free_blocks(dn, db->db_blkptr, 1, tx); 360eda14cbcSMatt Macy rw_exit(&db->db_rwlock); 361eda14cbcSMatt Macy } 362eda14cbcSMatt Macy 363eda14cbcSMatt Macy DB_DNODE_EXIT(db); 364eda14cbcSMatt Macy arc_buf_freeze(db->db_buf); 365eda14cbcSMatt Macy } 366eda14cbcSMatt Macy 367eda14cbcSMatt Macy /* 368eda14cbcSMatt Macy * Traverse the indicated range of the provided file 369eda14cbcSMatt Macy * and "free" all the blocks contained there. 370eda14cbcSMatt Macy */ 371eda14cbcSMatt Macy static void 372eda14cbcSMatt Macy dnode_sync_free_range_impl(dnode_t *dn, uint64_t blkid, uint64_t nblks, 373eda14cbcSMatt Macy boolean_t free_indirects, dmu_tx_t *tx) 374eda14cbcSMatt Macy { 375eda14cbcSMatt Macy blkptr_t *bp = dn->dn_phys->dn_blkptr; 376eda14cbcSMatt Macy int dnlevel = dn->dn_phys->dn_nlevels; 377eda14cbcSMatt Macy boolean_t trunc = B_FALSE; 378eda14cbcSMatt Macy 379eda14cbcSMatt Macy if (blkid > dn->dn_phys->dn_maxblkid) 380eda14cbcSMatt Macy return; 381eda14cbcSMatt Macy 382eda14cbcSMatt Macy ASSERT(dn->dn_phys->dn_maxblkid < UINT64_MAX); 383eda14cbcSMatt Macy if (blkid + nblks > dn->dn_phys->dn_maxblkid) { 384eda14cbcSMatt Macy nblks = dn->dn_phys->dn_maxblkid - blkid + 1; 385eda14cbcSMatt Macy trunc = B_TRUE; 386eda14cbcSMatt Macy } 387eda14cbcSMatt Macy 388eda14cbcSMatt Macy /* There are no indirect blocks in the object */ 389eda14cbcSMatt Macy if (dnlevel == 1) { 390eda14cbcSMatt Macy if (blkid >= dn->dn_phys->dn_nblkptr) { 391eda14cbcSMatt Macy /* this range was never made persistent */ 392eda14cbcSMatt Macy return; 393eda14cbcSMatt Macy } 394eda14cbcSMatt Macy ASSERT3U(blkid + nblks, <=, dn->dn_phys->dn_nblkptr); 395eda14cbcSMatt Macy free_blocks(dn, bp + blkid, nblks, tx); 396eda14cbcSMatt Macy } else { 397eda14cbcSMatt Macy int shift = (dnlevel - 1) * 398eda14cbcSMatt Macy (dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT); 399eda14cbcSMatt Macy int start = blkid >> shift; 400eda14cbcSMatt Macy int end = (blkid + nblks - 1) >> shift; 401eda14cbcSMatt Macy dmu_buf_impl_t *db; 402eda14cbcSMatt Macy 403eda14cbcSMatt Macy ASSERT(start < dn->dn_phys->dn_nblkptr); 404eda14cbcSMatt Macy bp += start; 405eda14cbcSMatt Macy for (int i = start; i <= end; i++, bp++) { 406eda14cbcSMatt Macy if (BP_IS_HOLE(bp)) 407eda14cbcSMatt Macy continue; 408eda14cbcSMatt Macy rw_enter(&dn->dn_struct_rwlock, RW_READER); 409eda14cbcSMatt Macy VERIFY0(dbuf_hold_impl(dn, dnlevel - 1, i, 410eda14cbcSMatt Macy TRUE, FALSE, FTAG, &db)); 411eda14cbcSMatt Macy rw_exit(&dn->dn_struct_rwlock); 412eda14cbcSMatt Macy free_children(db, blkid, nblks, free_indirects, tx); 413eda14cbcSMatt Macy dbuf_rele(db, FTAG); 414eda14cbcSMatt Macy } 415eda14cbcSMatt Macy } 416eda14cbcSMatt Macy 417eda14cbcSMatt Macy /* 418eda14cbcSMatt Macy * Do not truncate the maxblkid if we are performing a raw 419eda14cbcSMatt Macy * receive. The raw receive sets the maxblkid manually and 420eda14cbcSMatt Macy * must not be overridden. Usually, the last DRR_FREE record 421eda14cbcSMatt Macy * will be at the maxblkid, because the source system sets 422eda14cbcSMatt Macy * the maxblkid when truncating. However, if the last block 423eda14cbcSMatt Macy * was freed by overwriting with zeros and being compressed 424eda14cbcSMatt Macy * away to a hole, the source system will generate a DRR_FREE 425eda14cbcSMatt Macy * record while leaving the maxblkid after the end of that 426eda14cbcSMatt Macy * record. In this case we need to leave the maxblkid as 427eda14cbcSMatt Macy * indicated in the DRR_OBJECT record, so that it matches the 428eda14cbcSMatt Macy * source system, ensuring that the cryptographic hashes will 429eda14cbcSMatt Macy * match. 430eda14cbcSMatt Macy */ 431eda14cbcSMatt Macy if (trunc && !dn->dn_objset->os_raw_receive) { 432eda14cbcSMatt Macy uint64_t off __maybe_unused; 433eda14cbcSMatt Macy dn->dn_phys->dn_maxblkid = blkid == 0 ? 0 : blkid - 1; 434eda14cbcSMatt Macy 435eda14cbcSMatt Macy off = (dn->dn_phys->dn_maxblkid + 1) * 436eda14cbcSMatt Macy (dn->dn_phys->dn_datablkszsec << SPA_MINBLOCKSHIFT); 437eda14cbcSMatt Macy ASSERT(off < dn->dn_phys->dn_maxblkid || 438eda14cbcSMatt Macy dn->dn_phys->dn_maxblkid == 0 || 439eda14cbcSMatt Macy dnode_next_offset(dn, 0, &off, 1, 1, 0) != 0); 440eda14cbcSMatt Macy } 441eda14cbcSMatt Macy } 442eda14cbcSMatt Macy 443eda14cbcSMatt Macy typedef struct dnode_sync_free_range_arg { 444eda14cbcSMatt Macy dnode_t *dsfra_dnode; 445eda14cbcSMatt Macy dmu_tx_t *dsfra_tx; 446eda14cbcSMatt Macy boolean_t dsfra_free_indirects; 447eda14cbcSMatt Macy } dnode_sync_free_range_arg_t; 448eda14cbcSMatt Macy 449eda14cbcSMatt Macy static void 450eda14cbcSMatt Macy dnode_sync_free_range(void *arg, uint64_t blkid, uint64_t nblks) 451eda14cbcSMatt Macy { 452eda14cbcSMatt Macy dnode_sync_free_range_arg_t *dsfra = arg; 453eda14cbcSMatt Macy dnode_t *dn = dsfra->dsfra_dnode; 454eda14cbcSMatt Macy 455eda14cbcSMatt Macy mutex_exit(&dn->dn_mtx); 456eda14cbcSMatt Macy dnode_sync_free_range_impl(dn, blkid, nblks, 457eda14cbcSMatt Macy dsfra->dsfra_free_indirects, dsfra->dsfra_tx); 458eda14cbcSMatt Macy mutex_enter(&dn->dn_mtx); 459eda14cbcSMatt Macy } 460eda14cbcSMatt Macy 461eda14cbcSMatt Macy /* 462eda14cbcSMatt Macy * Try to kick all the dnode's dbufs out of the cache... 463eda14cbcSMatt Macy */ 464eda14cbcSMatt Macy void 465eda14cbcSMatt Macy dnode_evict_dbufs(dnode_t *dn) 466eda14cbcSMatt Macy { 467eda14cbcSMatt Macy dmu_buf_impl_t *db_marker; 468eda14cbcSMatt Macy dmu_buf_impl_t *db, *db_next; 469eda14cbcSMatt Macy 470eda14cbcSMatt Macy db_marker = kmem_alloc(sizeof (dmu_buf_impl_t), KM_SLEEP); 471eda14cbcSMatt Macy 472eda14cbcSMatt Macy mutex_enter(&dn->dn_dbufs_mtx); 473eda14cbcSMatt Macy for (db = avl_first(&dn->dn_dbufs); db != NULL; db = db_next) { 474eda14cbcSMatt Macy 475eda14cbcSMatt Macy #ifdef ZFS_DEBUG 476eda14cbcSMatt Macy DB_DNODE_ENTER(db); 477eda14cbcSMatt Macy ASSERT3P(DB_DNODE(db), ==, dn); 478eda14cbcSMatt Macy DB_DNODE_EXIT(db); 479eda14cbcSMatt Macy #endif /* DEBUG */ 480eda14cbcSMatt Macy 481eda14cbcSMatt Macy mutex_enter(&db->db_mtx); 482eda14cbcSMatt Macy if (db->db_state != DB_EVICTING && 483eda14cbcSMatt Macy zfs_refcount_is_zero(&db->db_holds)) { 484eda14cbcSMatt Macy db_marker->db_level = db->db_level; 485eda14cbcSMatt Macy db_marker->db_blkid = db->db_blkid; 4865fb307d2SMartin Matuska /* 4875fb307d2SMartin Matuska * Insert a MARKER node with the same level and blkid. 4885fb307d2SMartin Matuska * And to resolve any ties in dbuf_compare() use the 4895fb307d2SMartin Matuska * pointer of the dbuf that we are evicting. Pass the 4905fb307d2SMartin Matuska * address in db_parent. 4915fb307d2SMartin Matuska */ 4925fb307d2SMartin Matuska db_marker->db_state = DB_MARKER; 4935fb307d2SMartin Matuska db_marker->db_parent = (void *)((uintptr_t)db - 1); 494eda14cbcSMatt Macy avl_insert_here(&dn->dn_dbufs, db_marker, db, 495eda14cbcSMatt Macy AVL_BEFORE); 496eda14cbcSMatt Macy 497eda14cbcSMatt Macy /* 498eda14cbcSMatt Macy * We need to use the "marker" dbuf rather than 499eda14cbcSMatt Macy * simply getting the next dbuf, because 500eda14cbcSMatt Macy * dbuf_destroy() may actually remove multiple dbufs. 501eda14cbcSMatt Macy * It can call itself recursively on the parent dbuf, 502eda14cbcSMatt Macy * which may also be removed from dn_dbufs. The code 503eda14cbcSMatt Macy * flow would look like: 504eda14cbcSMatt Macy * 505eda14cbcSMatt Macy * dbuf_destroy(): 506eda14cbcSMatt Macy * dnode_rele_and_unlock(parent_dbuf, evicting=TRUE): 507eda14cbcSMatt Macy * if (!cacheable || pending_evict) 508eda14cbcSMatt Macy * dbuf_destroy() 509eda14cbcSMatt Macy */ 510eda14cbcSMatt Macy dbuf_destroy(db); 511eda14cbcSMatt Macy 512eda14cbcSMatt Macy db_next = AVL_NEXT(&dn->dn_dbufs, db_marker); 513eda14cbcSMatt Macy avl_remove(&dn->dn_dbufs, db_marker); 514eda14cbcSMatt Macy } else { 515eda14cbcSMatt Macy db->db_pending_evict = TRUE; 516eda14cbcSMatt Macy mutex_exit(&db->db_mtx); 517eda14cbcSMatt Macy db_next = AVL_NEXT(&dn->dn_dbufs, db); 518eda14cbcSMatt Macy } 519eda14cbcSMatt Macy } 520eda14cbcSMatt Macy mutex_exit(&dn->dn_dbufs_mtx); 521eda14cbcSMatt Macy 522eda14cbcSMatt Macy kmem_free(db_marker, sizeof (dmu_buf_impl_t)); 523eda14cbcSMatt Macy 524eda14cbcSMatt Macy dnode_evict_bonus(dn); 525eda14cbcSMatt Macy } 526eda14cbcSMatt Macy 527eda14cbcSMatt Macy void 528eda14cbcSMatt Macy dnode_evict_bonus(dnode_t *dn) 529eda14cbcSMatt Macy { 530eda14cbcSMatt Macy rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 531eda14cbcSMatt Macy if (dn->dn_bonus != NULL) { 532eda14cbcSMatt Macy if (zfs_refcount_is_zero(&dn->dn_bonus->db_holds)) { 533eda14cbcSMatt Macy mutex_enter(&dn->dn_bonus->db_mtx); 534eda14cbcSMatt Macy dbuf_destroy(dn->dn_bonus); 535eda14cbcSMatt Macy dn->dn_bonus = NULL; 536eda14cbcSMatt Macy } else { 537eda14cbcSMatt Macy dn->dn_bonus->db_pending_evict = TRUE; 538eda14cbcSMatt Macy } 539eda14cbcSMatt Macy } 540eda14cbcSMatt Macy rw_exit(&dn->dn_struct_rwlock); 541eda14cbcSMatt Macy } 542eda14cbcSMatt Macy 543eda14cbcSMatt Macy static void 544eda14cbcSMatt Macy dnode_undirty_dbufs(list_t *list) 545eda14cbcSMatt Macy { 546eda14cbcSMatt Macy dbuf_dirty_record_t *dr; 547eda14cbcSMatt Macy 548eda14cbcSMatt Macy while ((dr = list_head(list))) { 549eda14cbcSMatt Macy dmu_buf_impl_t *db = dr->dr_dbuf; 550eda14cbcSMatt Macy uint64_t txg = dr->dr_txg; 551eda14cbcSMatt Macy 552eda14cbcSMatt Macy if (db->db_level != 0) 553eda14cbcSMatt Macy dnode_undirty_dbufs(&dr->dt.di.dr_children); 554eda14cbcSMatt Macy 555eda14cbcSMatt Macy mutex_enter(&db->db_mtx); 556eda14cbcSMatt Macy /* XXX - use dbuf_undirty()? */ 557eda14cbcSMatt Macy list_remove(list, dr); 558eda14cbcSMatt Macy ASSERT(list_head(&db->db_dirty_records) == dr); 559eda14cbcSMatt Macy list_remove_head(&db->db_dirty_records); 560eda14cbcSMatt Macy ASSERT(list_is_empty(&db->db_dirty_records)); 561eda14cbcSMatt Macy db->db_dirtycnt -= 1; 562eda14cbcSMatt Macy if (db->db_level == 0) { 563eda14cbcSMatt Macy ASSERT(db->db_blkid == DMU_BONUS_BLKID || 564eda14cbcSMatt Macy dr->dt.dl.dr_data == db->db_buf); 565eda14cbcSMatt Macy dbuf_unoverride(dr); 566eda14cbcSMatt Macy } else { 567eda14cbcSMatt Macy mutex_destroy(&dr->dt.di.dr_mtx); 568eda14cbcSMatt Macy list_destroy(&dr->dt.di.dr_children); 569eda14cbcSMatt Macy } 5705c65a0a9SMartin Matuska kmem_cache_free(dbuf_dirty_kmem_cache, dr); 571eda14cbcSMatt Macy dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg, B_FALSE); 572eda14cbcSMatt Macy } 573eda14cbcSMatt Macy } 574eda14cbcSMatt Macy 575eda14cbcSMatt Macy static void 576eda14cbcSMatt Macy dnode_sync_free(dnode_t *dn, dmu_tx_t *tx) 577eda14cbcSMatt Macy { 578eda14cbcSMatt Macy int txgoff = tx->tx_txg & TXG_MASK; 579eda14cbcSMatt Macy 580eda14cbcSMatt Macy ASSERT(dmu_tx_is_syncing(tx)); 581eda14cbcSMatt Macy 582eda14cbcSMatt Macy /* 583eda14cbcSMatt Macy * Our contents should have been freed in dnode_sync() by the 584eda14cbcSMatt Macy * free range record inserted by the caller of dnode_free(). 585eda14cbcSMatt Macy */ 586eda14cbcSMatt Macy ASSERT0(DN_USED_BYTES(dn->dn_phys)); 587eda14cbcSMatt Macy ASSERT(BP_IS_HOLE(dn->dn_phys->dn_blkptr)); 588eda14cbcSMatt Macy 589eda14cbcSMatt Macy dnode_undirty_dbufs(&dn->dn_dirty_records[txgoff]); 590eda14cbcSMatt Macy dnode_evict_dbufs(dn); 591eda14cbcSMatt Macy 592eda14cbcSMatt Macy /* 593eda14cbcSMatt Macy * XXX - It would be nice to assert this, but we may still 594eda14cbcSMatt Macy * have residual holds from async evictions from the arc... 595eda14cbcSMatt Macy * 596eda14cbcSMatt Macy * zfs_obj_to_path() also depends on this being 597eda14cbcSMatt Macy * commented out. 598eda14cbcSMatt Macy * 599eda14cbcSMatt Macy * ASSERT3U(zfs_refcount_count(&dn->dn_holds), ==, 1); 600eda14cbcSMatt Macy */ 601eda14cbcSMatt Macy 602eda14cbcSMatt Macy /* Undirty next bits */ 603eda14cbcSMatt Macy dn->dn_next_nlevels[txgoff] = 0; 604eda14cbcSMatt Macy dn->dn_next_indblkshift[txgoff] = 0; 605eda14cbcSMatt Macy dn->dn_next_blksz[txgoff] = 0; 606eda14cbcSMatt Macy dn->dn_next_maxblkid[txgoff] = 0; 607eda14cbcSMatt Macy 608eda14cbcSMatt Macy /* ASSERT(blkptrs are zero); */ 609eda14cbcSMatt Macy ASSERT(dn->dn_phys->dn_type != DMU_OT_NONE); 610eda14cbcSMatt Macy ASSERT(dn->dn_type != DMU_OT_NONE); 611eda14cbcSMatt Macy 612eda14cbcSMatt Macy ASSERT(dn->dn_free_txg > 0); 613eda14cbcSMatt Macy if (dn->dn_allocated_txg != dn->dn_free_txg) 614eda14cbcSMatt Macy dmu_buf_will_dirty(&dn->dn_dbuf->db, tx); 615da5137abSMartin Matuska memset(dn->dn_phys, 0, sizeof (dnode_phys_t) * dn->dn_num_slots); 616eda14cbcSMatt Macy dnode_free_interior_slots(dn); 617eda14cbcSMatt Macy 618eda14cbcSMatt Macy mutex_enter(&dn->dn_mtx); 619eda14cbcSMatt Macy dn->dn_type = DMU_OT_NONE; 620eda14cbcSMatt Macy dn->dn_maxblkid = 0; 621eda14cbcSMatt Macy dn->dn_allocated_txg = 0; 622eda14cbcSMatt Macy dn->dn_free_txg = 0; 623eda14cbcSMatt Macy dn->dn_have_spill = B_FALSE; 624eda14cbcSMatt Macy dn->dn_num_slots = 1; 625eda14cbcSMatt Macy mutex_exit(&dn->dn_mtx); 626eda14cbcSMatt Macy 627eda14cbcSMatt Macy ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT); 628eda14cbcSMatt Macy 629eda14cbcSMatt Macy dnode_rele(dn, (void *)(uintptr_t)tx->tx_txg); 630eda14cbcSMatt Macy /* 631eda14cbcSMatt Macy * Now that we've released our hold, the dnode may 632eda14cbcSMatt Macy * be evicted, so we mustn't access it. 633eda14cbcSMatt Macy */ 634eda14cbcSMatt Macy } 635eda14cbcSMatt Macy 636eda14cbcSMatt Macy /* 637eda14cbcSMatt Macy * Write out the dnode's dirty buffers. 63814c2e0a0SMartin Matuska * Does not wait for zio completions. 639eda14cbcSMatt Macy */ 640eda14cbcSMatt Macy void 641eda14cbcSMatt Macy dnode_sync(dnode_t *dn, dmu_tx_t *tx) 642eda14cbcSMatt Macy { 643eda14cbcSMatt Macy objset_t *os = dn->dn_objset; 644eda14cbcSMatt Macy dnode_phys_t *dnp = dn->dn_phys; 645eda14cbcSMatt Macy int txgoff = tx->tx_txg & TXG_MASK; 646eda14cbcSMatt Macy list_t *list = &dn->dn_dirty_records[txgoff]; 647eda14cbcSMatt Macy static const dnode_phys_t zerodn __maybe_unused = { 0 }; 648eda14cbcSMatt Macy boolean_t kill_spill = B_FALSE; 649eda14cbcSMatt Macy 650eda14cbcSMatt Macy ASSERT(dmu_tx_is_syncing(tx)); 651eda14cbcSMatt Macy ASSERT(dnp->dn_type != DMU_OT_NONE || dn->dn_allocated_txg); 652eda14cbcSMatt Macy ASSERT(dnp->dn_type != DMU_OT_NONE || 653da5137abSMartin Matuska memcmp(dnp, &zerodn, DNODE_MIN_SIZE) == 0); 654eda14cbcSMatt Macy DNODE_VERIFY(dn); 655eda14cbcSMatt Macy 656eda14cbcSMatt Macy ASSERT(dn->dn_dbuf == NULL || arc_released(dn->dn_dbuf->db_buf)); 657eda14cbcSMatt Macy 658eda14cbcSMatt Macy /* 659eda14cbcSMatt Macy * Do user accounting if it is enabled and this is not 660eda14cbcSMatt Macy * an encrypted receive. 661eda14cbcSMatt Macy */ 662eda14cbcSMatt Macy if (dmu_objset_userused_enabled(os) && 663eda14cbcSMatt Macy !DMU_OBJECT_IS_SPECIAL(dn->dn_object) && 664eda14cbcSMatt Macy (!os->os_encrypted || !dmu_objset_is_receiving(os))) { 665eda14cbcSMatt Macy mutex_enter(&dn->dn_mtx); 666eda14cbcSMatt Macy dn->dn_oldused = DN_USED_BYTES(dn->dn_phys); 667eda14cbcSMatt Macy dn->dn_oldflags = dn->dn_phys->dn_flags; 668eda14cbcSMatt Macy dn->dn_phys->dn_flags |= DNODE_FLAG_USERUSED_ACCOUNTED; 669eda14cbcSMatt Macy if (dmu_objset_userobjused_enabled(dn->dn_objset)) 670eda14cbcSMatt Macy dn->dn_phys->dn_flags |= 671eda14cbcSMatt Macy DNODE_FLAG_USEROBJUSED_ACCOUNTED; 672eda14cbcSMatt Macy mutex_exit(&dn->dn_mtx); 673eda14cbcSMatt Macy dmu_objset_userquota_get_ids(dn, B_FALSE, tx); 674e92ffd9bSMartin Matuska } else if (!(os->os_encrypted && dmu_objset_is_receiving(os))) { 675e92ffd9bSMartin Matuska /* 676e92ffd9bSMartin Matuska * Once we account for it, we should always account for it, 677e92ffd9bSMartin Matuska * except for the case of a raw receive. We will not be able 678e92ffd9bSMartin Matuska * to account for it until the receiving dataset has been 679e92ffd9bSMartin Matuska * mounted. 680e92ffd9bSMartin Matuska */ 681eda14cbcSMatt Macy ASSERT(!(dn->dn_phys->dn_flags & 682eda14cbcSMatt Macy DNODE_FLAG_USERUSED_ACCOUNTED)); 683eda14cbcSMatt Macy ASSERT(!(dn->dn_phys->dn_flags & 684eda14cbcSMatt Macy DNODE_FLAG_USEROBJUSED_ACCOUNTED)); 685eda14cbcSMatt Macy } 686eda14cbcSMatt Macy 687eda14cbcSMatt Macy mutex_enter(&dn->dn_mtx); 688eda14cbcSMatt Macy if (dn->dn_allocated_txg == tx->tx_txg) { 689eda14cbcSMatt Macy /* The dnode is newly allocated or reallocated */ 690eda14cbcSMatt Macy if (dnp->dn_type == DMU_OT_NONE) { 691eda14cbcSMatt Macy /* this is a first alloc, not a realloc */ 692eda14cbcSMatt Macy dnp->dn_nlevels = 1; 693eda14cbcSMatt Macy dnp->dn_nblkptr = dn->dn_nblkptr; 694eda14cbcSMatt Macy } 695eda14cbcSMatt Macy 696eda14cbcSMatt Macy dnp->dn_type = dn->dn_type; 697eda14cbcSMatt Macy dnp->dn_bonustype = dn->dn_bonustype; 698eda14cbcSMatt Macy dnp->dn_bonuslen = dn->dn_bonuslen; 699eda14cbcSMatt Macy } 700eda14cbcSMatt Macy 701eda14cbcSMatt Macy dnp->dn_extra_slots = dn->dn_num_slots - 1; 702eda14cbcSMatt Macy 703eda14cbcSMatt Macy ASSERT(dnp->dn_nlevels > 1 || 704eda14cbcSMatt Macy BP_IS_HOLE(&dnp->dn_blkptr[0]) || 705eda14cbcSMatt Macy BP_IS_EMBEDDED(&dnp->dn_blkptr[0]) || 706eda14cbcSMatt Macy BP_GET_LSIZE(&dnp->dn_blkptr[0]) == 707eda14cbcSMatt Macy dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT); 708eda14cbcSMatt Macy ASSERT(dnp->dn_nlevels < 2 || 709eda14cbcSMatt Macy BP_IS_HOLE(&dnp->dn_blkptr[0]) || 710eda14cbcSMatt Macy BP_GET_LSIZE(&dnp->dn_blkptr[0]) == 1 << dnp->dn_indblkshift); 711eda14cbcSMatt Macy 712eda14cbcSMatt Macy if (dn->dn_next_type[txgoff] != 0) { 713eda14cbcSMatt Macy dnp->dn_type = dn->dn_type; 714eda14cbcSMatt Macy dn->dn_next_type[txgoff] = 0; 715eda14cbcSMatt Macy } 716eda14cbcSMatt Macy 717eda14cbcSMatt Macy if (dn->dn_next_blksz[txgoff] != 0) { 718eda14cbcSMatt Macy ASSERT(P2PHASE(dn->dn_next_blksz[txgoff], 719eda14cbcSMatt Macy SPA_MINBLOCKSIZE) == 0); 720eda14cbcSMatt Macy ASSERT(BP_IS_HOLE(&dnp->dn_blkptr[0]) || 721eda14cbcSMatt Macy dn->dn_maxblkid == 0 || list_head(list) != NULL || 722eda14cbcSMatt Macy dn->dn_next_blksz[txgoff] >> SPA_MINBLOCKSHIFT == 723eda14cbcSMatt Macy dnp->dn_datablkszsec || 724b59a0cdeSMartin Matuska !zfs_range_tree_is_empty(dn->dn_free_ranges[txgoff])); 725eda14cbcSMatt Macy dnp->dn_datablkszsec = 726eda14cbcSMatt Macy dn->dn_next_blksz[txgoff] >> SPA_MINBLOCKSHIFT; 727eda14cbcSMatt Macy dn->dn_next_blksz[txgoff] = 0; 728eda14cbcSMatt Macy } 729eda14cbcSMatt Macy 730eda14cbcSMatt Macy if (dn->dn_next_bonuslen[txgoff] != 0) { 731eda14cbcSMatt Macy if (dn->dn_next_bonuslen[txgoff] == DN_ZERO_BONUSLEN) 732eda14cbcSMatt Macy dnp->dn_bonuslen = 0; 733eda14cbcSMatt Macy else 734eda14cbcSMatt Macy dnp->dn_bonuslen = dn->dn_next_bonuslen[txgoff]; 735eda14cbcSMatt Macy ASSERT(dnp->dn_bonuslen <= 736eda14cbcSMatt Macy DN_SLOTS_TO_BONUSLEN(dnp->dn_extra_slots + 1)); 737eda14cbcSMatt Macy dn->dn_next_bonuslen[txgoff] = 0; 738eda14cbcSMatt Macy } 739eda14cbcSMatt Macy 740eda14cbcSMatt Macy if (dn->dn_next_bonustype[txgoff] != 0) { 741eda14cbcSMatt Macy ASSERT(DMU_OT_IS_VALID(dn->dn_next_bonustype[txgoff])); 742eda14cbcSMatt Macy dnp->dn_bonustype = dn->dn_next_bonustype[txgoff]; 743eda14cbcSMatt Macy dn->dn_next_bonustype[txgoff] = 0; 744eda14cbcSMatt Macy } 745eda14cbcSMatt Macy 746eda14cbcSMatt Macy boolean_t freeing_dnode = dn->dn_free_txg > 0 && 747eda14cbcSMatt Macy dn->dn_free_txg <= tx->tx_txg; 748eda14cbcSMatt Macy 749eda14cbcSMatt Macy /* 750eda14cbcSMatt Macy * Remove the spill block if we have been explicitly asked to 751eda14cbcSMatt Macy * remove it, or if the object is being removed. 752eda14cbcSMatt Macy */ 753eda14cbcSMatt Macy if (dn->dn_rm_spillblk[txgoff] || freeing_dnode) { 754eda14cbcSMatt Macy if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) 755eda14cbcSMatt Macy kill_spill = B_TRUE; 756eda14cbcSMatt Macy dn->dn_rm_spillblk[txgoff] = 0; 757eda14cbcSMatt Macy } 758eda14cbcSMatt Macy 759eda14cbcSMatt Macy if (dn->dn_next_indblkshift[txgoff] != 0) { 760eda14cbcSMatt Macy ASSERT(dnp->dn_nlevels == 1); 761eda14cbcSMatt Macy dnp->dn_indblkshift = dn->dn_next_indblkshift[txgoff]; 762eda14cbcSMatt Macy dn->dn_next_indblkshift[txgoff] = 0; 763eda14cbcSMatt Macy } 764eda14cbcSMatt Macy 765eda14cbcSMatt Macy /* 766eda14cbcSMatt Macy * Just take the live (open-context) values for checksum and compress. 767eda14cbcSMatt Macy * Strictly speaking it's a future leak, but nothing bad happens if we 768eda14cbcSMatt Macy * start using the new checksum or compress algorithm a little early. 769eda14cbcSMatt Macy */ 770eda14cbcSMatt Macy dnp->dn_checksum = dn->dn_checksum; 771eda14cbcSMatt Macy dnp->dn_compress = dn->dn_compress; 772eda14cbcSMatt Macy 773eda14cbcSMatt Macy mutex_exit(&dn->dn_mtx); 774eda14cbcSMatt Macy 775eda14cbcSMatt Macy if (kill_spill) { 776eda14cbcSMatt Macy free_blocks(dn, DN_SPILL_BLKPTR(dn->dn_phys), 1, tx); 777eda14cbcSMatt Macy mutex_enter(&dn->dn_mtx); 778eda14cbcSMatt Macy dnp->dn_flags &= ~DNODE_FLAG_SPILL_BLKPTR; 779eda14cbcSMatt Macy mutex_exit(&dn->dn_mtx); 780eda14cbcSMatt Macy } 781eda14cbcSMatt Macy 782eda14cbcSMatt Macy /* process all the "freed" ranges in the file */ 783eda14cbcSMatt Macy if (dn->dn_free_ranges[txgoff] != NULL) { 784eda14cbcSMatt Macy dnode_sync_free_range_arg_t dsfra; 785eda14cbcSMatt Macy dsfra.dsfra_dnode = dn; 786eda14cbcSMatt Macy dsfra.dsfra_tx = tx; 787eda14cbcSMatt Macy dsfra.dsfra_free_indirects = freeing_dnode; 788ac0bf12eSMatt Macy mutex_enter(&dn->dn_mtx); 789eda14cbcSMatt Macy if (freeing_dnode) { 790b59a0cdeSMartin Matuska ASSERT(zfs_range_tree_contains( 791b59a0cdeSMartin Matuska dn->dn_free_ranges[txgoff], 0, 792b59a0cdeSMartin Matuska dn->dn_maxblkid + 1)); 793eda14cbcSMatt Macy } 794ac0bf12eSMatt Macy /* 795ac0bf12eSMatt Macy * Because dnode_sync_free_range() must drop dn_mtx during its 796b59a0cdeSMartin Matuska * processing, using it as a callback to zfs_range_tree_vacate() 797b59a0cdeSMartin Matuska * is not safe. No other operations (besides destroy) are 798b59a0cdeSMartin Matuska * allowed once zfs_range_tree_vacate() has begun, and dropping 799b59a0cdeSMartin Matuska * dn_mtx would leave a window open for another thread to 800b59a0cdeSMartin Matuska * observe that invalid (and unsafe) state. 801ac0bf12eSMatt Macy */ 802b59a0cdeSMartin Matuska zfs_range_tree_walk(dn->dn_free_ranges[txgoff], 803eda14cbcSMatt Macy dnode_sync_free_range, &dsfra); 804b59a0cdeSMartin Matuska zfs_range_tree_vacate(dn->dn_free_ranges[txgoff], NULL, NULL); 805b59a0cdeSMartin Matuska zfs_range_tree_destroy(dn->dn_free_ranges[txgoff]); 806eda14cbcSMatt Macy dn->dn_free_ranges[txgoff] = NULL; 807eda14cbcSMatt Macy mutex_exit(&dn->dn_mtx); 808eda14cbcSMatt Macy } 809eda14cbcSMatt Macy 810eda14cbcSMatt Macy if (freeing_dnode) { 811eda14cbcSMatt Macy dn->dn_objset->os_freed_dnodes++; 812eda14cbcSMatt Macy dnode_sync_free(dn, tx); 813eda14cbcSMatt Macy return; 814eda14cbcSMatt Macy } 815eda14cbcSMatt Macy 816eda14cbcSMatt Macy if (dn->dn_num_slots > DNODE_MIN_SLOTS) { 817eda14cbcSMatt Macy dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset; 818eda14cbcSMatt Macy mutex_enter(&ds->ds_lock); 819eda14cbcSMatt Macy ds->ds_feature_activation[SPA_FEATURE_LARGE_DNODE] = 820eda14cbcSMatt Macy (void *)B_TRUE; 821eda14cbcSMatt Macy mutex_exit(&ds->ds_lock); 822eda14cbcSMatt Macy } 823eda14cbcSMatt Macy 824eda14cbcSMatt Macy if (dn->dn_next_nlevels[txgoff]) { 825eda14cbcSMatt Macy dnode_increase_indirection(dn, tx); 826eda14cbcSMatt Macy dn->dn_next_nlevels[txgoff] = 0; 827eda14cbcSMatt Macy } 828eda14cbcSMatt Macy 829eda14cbcSMatt Macy /* 830eda14cbcSMatt Macy * This must be done after dnode_sync_free_range() 831eda14cbcSMatt Macy * and dnode_increase_indirection(). See dnode_new_blkid() 832eda14cbcSMatt Macy * for an explanation of the high bit being set. 833eda14cbcSMatt Macy */ 834eda14cbcSMatt Macy if (dn->dn_next_maxblkid[txgoff]) { 835eda14cbcSMatt Macy mutex_enter(&dn->dn_mtx); 836eda14cbcSMatt Macy dnp->dn_maxblkid = 837eda14cbcSMatt Macy dn->dn_next_maxblkid[txgoff] & ~DMU_NEXT_MAXBLKID_SET; 838eda14cbcSMatt Macy dn->dn_next_maxblkid[txgoff] = 0; 839eda14cbcSMatt Macy mutex_exit(&dn->dn_mtx); 840eda14cbcSMatt Macy } 841eda14cbcSMatt Macy 842eda14cbcSMatt Macy if (dn->dn_next_nblkptr[txgoff]) { 843eda14cbcSMatt Macy /* this should only happen on a realloc */ 844eda14cbcSMatt Macy ASSERT(dn->dn_allocated_txg == tx->tx_txg); 845eda14cbcSMatt Macy if (dn->dn_next_nblkptr[txgoff] > dnp->dn_nblkptr) { 846eda14cbcSMatt Macy /* zero the new blkptrs we are gaining */ 847da5137abSMartin Matuska memset(dnp->dn_blkptr + dnp->dn_nblkptr, 0, 848eda14cbcSMatt Macy sizeof (blkptr_t) * 849eda14cbcSMatt Macy (dn->dn_next_nblkptr[txgoff] - dnp->dn_nblkptr)); 850eda14cbcSMatt Macy #ifdef ZFS_DEBUG 851eda14cbcSMatt Macy } else { 852eda14cbcSMatt Macy int i; 853eda14cbcSMatt Macy ASSERT(dn->dn_next_nblkptr[txgoff] < dnp->dn_nblkptr); 854eda14cbcSMatt Macy /* the blkptrs we are losing better be unallocated */ 855eda14cbcSMatt Macy for (i = 0; i < dnp->dn_nblkptr; i++) { 856eda14cbcSMatt Macy if (i >= dn->dn_next_nblkptr[txgoff]) 857eda14cbcSMatt Macy ASSERT(BP_IS_HOLE(&dnp->dn_blkptr[i])); 858eda14cbcSMatt Macy } 859eda14cbcSMatt Macy #endif 860eda14cbcSMatt Macy } 861eda14cbcSMatt Macy mutex_enter(&dn->dn_mtx); 862eda14cbcSMatt Macy dnp->dn_nblkptr = dn->dn_next_nblkptr[txgoff]; 863eda14cbcSMatt Macy dn->dn_next_nblkptr[txgoff] = 0; 864eda14cbcSMatt Macy mutex_exit(&dn->dn_mtx); 865eda14cbcSMatt Macy } 866eda14cbcSMatt Macy 867eda14cbcSMatt Macy dbuf_sync_list(list, dn->dn_phys->dn_nlevels - 1, tx); 868eda14cbcSMatt Macy 869eda14cbcSMatt Macy if (!DMU_OBJECT_IS_SPECIAL(dn->dn_object)) { 870eda14cbcSMatt Macy ASSERT3P(list_head(list), ==, NULL); 871eda14cbcSMatt Macy dnode_rele(dn, (void *)(uintptr_t)tx->tx_txg); 872eda14cbcSMatt Macy } 873eda14cbcSMatt Macy 874c03c5b1cSMartin Matuska ASSERT3U(dnp->dn_bonuslen, <=, DN_MAX_BONUS_LEN(dnp)); 875c03c5b1cSMartin Matuska 876eda14cbcSMatt Macy /* 877eda14cbcSMatt Macy * Although we have dropped our reference to the dnode, it 878eda14cbcSMatt Macy * can't be evicted until its written, and we haven't yet 8797877fdebSMatt Macy * initiated the IO for the dnode's dbuf. Additionally, the caller 8807877fdebSMatt Macy * has already added a reference to the dnode because it's on the 8817877fdebSMatt Macy * os_synced_dnodes list. 882eda14cbcSMatt Macy */ 883eda14cbcSMatt Macy } 884