1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright 2011 Nexenta Systems, Inc. All rights reserved. 24 * Copyright (c) 2012, 2020 by Delphix. All rights reserved. 25 * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. 26 * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. 27 * Copyright (c) 2019, Klara Inc. 28 * Copyright (c) 2019, Allan Jude 29 */ 30 31 #include <sys/zfs_context.h> 32 #include <sys/arc.h> 33 #include <sys/dmu.h> 34 #include <sys/dmu_send.h> 35 #include <sys/dmu_impl.h> 36 #include <sys/dbuf.h> 37 #include <sys/dmu_objset.h> 38 #include <sys/dsl_dataset.h> 39 #include <sys/dsl_dir.h> 40 #include <sys/dmu_tx.h> 41 #include <sys/spa.h> 42 #include <sys/zio.h> 43 #include <sys/dmu_zfetch.h> 44 #include <sys/sa.h> 45 #include <sys/sa_impl.h> 46 #include <sys/zfeature.h> 47 #include <sys/blkptr.h> 48 #include <sys/range_tree.h> 49 #include <sys/trace_zfs.h> 50 #include <sys/callb.h> 51 #include <sys/abd.h> 52 #include <sys/vdev.h> 53 #include <cityhash.h> 54 #include <sys/spa_impl.h> 55 #include <sys/wmsum.h> 56 #include <sys/vdev_impl.h> 57 58 static kstat_t *dbuf_ksp; 59 60 typedef struct dbuf_stats { 61 /* 62 * Various statistics about the size of the dbuf cache. 63 */ 64 kstat_named_t cache_count; 65 kstat_named_t cache_size_bytes; 66 kstat_named_t cache_size_bytes_max; 67 /* 68 * Statistics regarding the bounds on the dbuf cache size. 69 */ 70 kstat_named_t cache_target_bytes; 71 kstat_named_t cache_lowater_bytes; 72 kstat_named_t cache_hiwater_bytes; 73 /* 74 * Total number of dbuf cache evictions that have occurred. 75 */ 76 kstat_named_t cache_total_evicts; 77 /* 78 * The distribution of dbuf levels in the dbuf cache and 79 * the total size of all dbufs at each level. 80 */ 81 kstat_named_t cache_levels[DN_MAX_LEVELS]; 82 kstat_named_t cache_levels_bytes[DN_MAX_LEVELS]; 83 /* 84 * Statistics about the dbuf hash table. 85 */ 86 kstat_named_t hash_hits; 87 kstat_named_t hash_misses; 88 kstat_named_t hash_collisions; 89 kstat_named_t hash_elements; 90 kstat_named_t hash_elements_max; 91 /* 92 * Number of sublists containing more than one dbuf in the dbuf 93 * hash table. Keep track of the longest hash chain. 94 */ 95 kstat_named_t hash_chains; 96 kstat_named_t hash_chain_max; 97 /* 98 * Number of times a dbuf_create() discovers that a dbuf was 99 * already created and in the dbuf hash table. 100 */ 101 kstat_named_t hash_insert_race; 102 /* 103 * Statistics about the size of the metadata dbuf cache. 104 */ 105 kstat_named_t metadata_cache_count; 106 kstat_named_t metadata_cache_size_bytes; 107 kstat_named_t metadata_cache_size_bytes_max; 108 /* 109 * For diagnostic purposes, this is incremented whenever we can't add 110 * something to the metadata cache because it's full, and instead put 111 * the data in the regular dbuf cache. 112 */ 113 kstat_named_t metadata_cache_overflow; 114 } dbuf_stats_t; 115 116 dbuf_stats_t dbuf_stats = { 117 { "cache_count", KSTAT_DATA_UINT64 }, 118 { "cache_size_bytes", KSTAT_DATA_UINT64 }, 119 { "cache_size_bytes_max", KSTAT_DATA_UINT64 }, 120 { "cache_target_bytes", KSTAT_DATA_UINT64 }, 121 { "cache_lowater_bytes", KSTAT_DATA_UINT64 }, 122 { "cache_hiwater_bytes", KSTAT_DATA_UINT64 }, 123 { "cache_total_evicts", KSTAT_DATA_UINT64 }, 124 { { "cache_levels_N", KSTAT_DATA_UINT64 } }, 125 { { "cache_levels_bytes_N", KSTAT_DATA_UINT64 } }, 126 { "hash_hits", KSTAT_DATA_UINT64 }, 127 { "hash_misses", KSTAT_DATA_UINT64 }, 128 { "hash_collisions", KSTAT_DATA_UINT64 }, 129 { "hash_elements", KSTAT_DATA_UINT64 }, 130 { "hash_elements_max", KSTAT_DATA_UINT64 }, 131 { "hash_chains", KSTAT_DATA_UINT64 }, 132 { "hash_chain_max", KSTAT_DATA_UINT64 }, 133 { "hash_insert_race", KSTAT_DATA_UINT64 }, 134 { "metadata_cache_count", KSTAT_DATA_UINT64 }, 135 { "metadata_cache_size_bytes", KSTAT_DATA_UINT64 }, 136 { "metadata_cache_size_bytes_max", KSTAT_DATA_UINT64 }, 137 { "metadata_cache_overflow", KSTAT_DATA_UINT64 } 138 }; 139 140 struct { 141 wmsum_t cache_count; 142 wmsum_t cache_total_evicts; 143 wmsum_t cache_levels[DN_MAX_LEVELS]; 144 wmsum_t cache_levels_bytes[DN_MAX_LEVELS]; 145 wmsum_t hash_hits; 146 wmsum_t hash_misses; 147 wmsum_t hash_collisions; 148 wmsum_t hash_chains; 149 wmsum_t hash_insert_race; 150 wmsum_t metadata_cache_count; 151 wmsum_t metadata_cache_overflow; 152 } dbuf_sums; 153 154 #define DBUF_STAT_INCR(stat, val) \ 155 wmsum_add(&dbuf_sums.stat, val); 156 #define DBUF_STAT_DECR(stat, val) \ 157 DBUF_STAT_INCR(stat, -(val)); 158 #define DBUF_STAT_BUMP(stat) \ 159 DBUF_STAT_INCR(stat, 1); 160 #define DBUF_STAT_BUMPDOWN(stat) \ 161 DBUF_STAT_INCR(stat, -1); 162 #define DBUF_STAT_MAX(stat, v) { \ 163 uint64_t _m; \ 164 while ((v) > (_m = dbuf_stats.stat.value.ui64) && \ 165 (_m != atomic_cas_64(&dbuf_stats.stat.value.ui64, _m, (v))))\ 166 continue; \ 167 } 168 169 static boolean_t dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx); 170 static void dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx); 171 static void dbuf_sync_leaf_verify_bonus_dnode(dbuf_dirty_record_t *dr); 172 static int dbuf_read_verify_dnode_crypt(dmu_buf_impl_t *db, uint32_t flags); 173 174 /* 175 * Global data structures and functions for the dbuf cache. 176 */ 177 static kmem_cache_t *dbuf_kmem_cache; 178 static taskq_t *dbu_evict_taskq; 179 180 static kthread_t *dbuf_cache_evict_thread; 181 static kmutex_t dbuf_evict_lock; 182 static kcondvar_t dbuf_evict_cv; 183 static boolean_t dbuf_evict_thread_exit; 184 185 /* 186 * There are two dbuf caches; each dbuf can only be in one of them at a time. 187 * 188 * 1. Cache of metadata dbufs, to help make read-heavy administrative commands 189 * from /sbin/zfs run faster. The "metadata cache" specifically stores dbufs 190 * that represent the metadata that describes filesystems/snapshots/ 191 * bookmarks/properties/etc. We only evict from this cache when we export a 192 * pool, to short-circuit as much I/O as possible for all administrative 193 * commands that need the metadata. There is no eviction policy for this 194 * cache, because we try to only include types in it which would occupy a 195 * very small amount of space per object but create a large impact on the 196 * performance of these commands. Instead, after it reaches a maximum size 197 * (which should only happen on very small memory systems with a very large 198 * number of filesystem objects), we stop taking new dbufs into the 199 * metadata cache, instead putting them in the normal dbuf cache. 200 * 201 * 2. LRU cache of dbufs. The dbuf cache maintains a list of dbufs that 202 * are not currently held but have been recently released. These dbufs 203 * are not eligible for arc eviction until they are aged out of the cache. 204 * Dbufs that are aged out of the cache will be immediately destroyed and 205 * become eligible for arc eviction. 206 * 207 * Dbufs are added to these caches once the last hold is released. If a dbuf is 208 * later accessed and still exists in the dbuf cache, then it will be removed 209 * from the cache and later re-added to the head of the cache. 210 * 211 * If a given dbuf meets the requirements for the metadata cache, it will go 212 * there, otherwise it will be considered for the generic LRU dbuf cache. The 213 * caches and the refcounts tracking their sizes are stored in an array indexed 214 * by those caches' matching enum values (from dbuf_cached_state_t). 215 */ 216 typedef struct dbuf_cache { 217 multilist_t cache; 218 zfs_refcount_t size ____cacheline_aligned; 219 } dbuf_cache_t; 220 dbuf_cache_t dbuf_caches[DB_CACHE_MAX]; 221 222 /* Size limits for the caches */ 223 static unsigned long dbuf_cache_max_bytes = ULONG_MAX; 224 static unsigned long dbuf_metadata_cache_max_bytes = ULONG_MAX; 225 226 /* Set the default sizes of the caches to log2 fraction of arc size */ 227 static int dbuf_cache_shift = 5; 228 static int dbuf_metadata_cache_shift = 6; 229 230 static unsigned long dbuf_cache_target_bytes(void); 231 static unsigned long dbuf_metadata_cache_target_bytes(void); 232 233 /* 234 * The LRU dbuf cache uses a three-stage eviction policy: 235 * - A low water marker designates when the dbuf eviction thread 236 * should stop evicting from the dbuf cache. 237 * - When we reach the maximum size (aka mid water mark), we 238 * signal the eviction thread to run. 239 * - The high water mark indicates when the eviction thread 240 * is unable to keep up with the incoming load and eviction must 241 * happen in the context of the calling thread. 242 * 243 * The dbuf cache: 244 * (max size) 245 * low water mid water hi water 246 * +----------------------------------------+----------+----------+ 247 * | | | | 248 * | | | | 249 * | | | | 250 * | | | | 251 * +----------------------------------------+----------+----------+ 252 * stop signal evict 253 * evicting eviction directly 254 * thread 255 * 256 * The high and low water marks indicate the operating range for the eviction 257 * thread. The low water mark is, by default, 90% of the total size of the 258 * cache and the high water mark is at 110% (both of these percentages can be 259 * changed by setting dbuf_cache_lowater_pct and dbuf_cache_hiwater_pct, 260 * respectively). The eviction thread will try to ensure that the cache remains 261 * within this range by waking up every second and checking if the cache is 262 * above the low water mark. The thread can also be woken up by callers adding 263 * elements into the cache if the cache is larger than the mid water (i.e max 264 * cache size). Once the eviction thread is woken up and eviction is required, 265 * it will continue evicting buffers until it's able to reduce the cache size 266 * to the low water mark. If the cache size continues to grow and hits the high 267 * water mark, then callers adding elements to the cache will begin to evict 268 * directly from the cache until the cache is no longer above the high water 269 * mark. 270 */ 271 272 /* 273 * The percentage above and below the maximum cache size. 274 */ 275 static uint_t dbuf_cache_hiwater_pct = 10; 276 static uint_t dbuf_cache_lowater_pct = 10; 277 278 static int 279 dbuf_cons(void *vdb, void *unused, int kmflag) 280 { 281 (void) unused, (void) kmflag; 282 dmu_buf_impl_t *db = vdb; 283 memset(db, 0, sizeof (dmu_buf_impl_t)); 284 285 mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL); 286 rw_init(&db->db_rwlock, NULL, RW_DEFAULT, NULL); 287 cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL); 288 multilist_link_init(&db->db_cache_link); 289 zfs_refcount_create(&db->db_holds); 290 291 return (0); 292 } 293 294 static void 295 dbuf_dest(void *vdb, void *unused) 296 { 297 (void) unused; 298 dmu_buf_impl_t *db = vdb; 299 mutex_destroy(&db->db_mtx); 300 rw_destroy(&db->db_rwlock); 301 cv_destroy(&db->db_changed); 302 ASSERT(!multilist_link_active(&db->db_cache_link)); 303 zfs_refcount_destroy(&db->db_holds); 304 } 305 306 /* 307 * dbuf hash table routines 308 */ 309 static dbuf_hash_table_t dbuf_hash_table; 310 311 /* 312 * We use Cityhash for this. It's fast, and has good hash properties without 313 * requiring any large static buffers. 314 */ 315 static uint64_t 316 dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid) 317 { 318 return (cityhash4((uintptr_t)os, obj, (uint64_t)lvl, blkid)); 319 } 320 321 #define DTRACE_SET_STATE(db, why) \ 322 DTRACE_PROBE2(dbuf__state_change, dmu_buf_impl_t *, db, \ 323 const char *, why) 324 325 #define DBUF_EQUAL(dbuf, os, obj, level, blkid) \ 326 ((dbuf)->db.db_object == (obj) && \ 327 (dbuf)->db_objset == (os) && \ 328 (dbuf)->db_level == (level) && \ 329 (dbuf)->db_blkid == (blkid)) 330 331 dmu_buf_impl_t * 332 dbuf_find(objset_t *os, uint64_t obj, uint8_t level, uint64_t blkid) 333 { 334 dbuf_hash_table_t *h = &dbuf_hash_table; 335 uint64_t hv; 336 uint64_t idx; 337 dmu_buf_impl_t *db; 338 339 hv = dbuf_hash(os, obj, level, blkid); 340 idx = hv & h->hash_table_mask; 341 342 rw_enter(DBUF_HASH_RWLOCK(h, idx), RW_READER); 343 for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) { 344 if (DBUF_EQUAL(db, os, obj, level, blkid)) { 345 mutex_enter(&db->db_mtx); 346 if (db->db_state != DB_EVICTING) { 347 rw_exit(DBUF_HASH_RWLOCK(h, idx)); 348 return (db); 349 } 350 mutex_exit(&db->db_mtx); 351 } 352 } 353 rw_exit(DBUF_HASH_RWLOCK(h, idx)); 354 return (NULL); 355 } 356 357 static dmu_buf_impl_t * 358 dbuf_find_bonus(objset_t *os, uint64_t object) 359 { 360 dnode_t *dn; 361 dmu_buf_impl_t *db = NULL; 362 363 if (dnode_hold(os, object, FTAG, &dn) == 0) { 364 rw_enter(&dn->dn_struct_rwlock, RW_READER); 365 if (dn->dn_bonus != NULL) { 366 db = dn->dn_bonus; 367 mutex_enter(&db->db_mtx); 368 } 369 rw_exit(&dn->dn_struct_rwlock); 370 dnode_rele(dn, FTAG); 371 } 372 return (db); 373 } 374 375 /* 376 * Insert an entry into the hash table. If there is already an element 377 * equal to elem in the hash table, then the already existing element 378 * will be returned and the new element will not be inserted. 379 * Otherwise returns NULL. 380 */ 381 static dmu_buf_impl_t * 382 dbuf_hash_insert(dmu_buf_impl_t *db) 383 { 384 dbuf_hash_table_t *h = &dbuf_hash_table; 385 objset_t *os = db->db_objset; 386 uint64_t obj = db->db.db_object; 387 int level = db->db_level; 388 uint64_t blkid, hv, idx; 389 dmu_buf_impl_t *dbf; 390 uint32_t i; 391 392 blkid = db->db_blkid; 393 hv = dbuf_hash(os, obj, level, blkid); 394 idx = hv & h->hash_table_mask; 395 396 rw_enter(DBUF_HASH_RWLOCK(h, idx), RW_WRITER); 397 for (dbf = h->hash_table[idx], i = 0; dbf != NULL; 398 dbf = dbf->db_hash_next, i++) { 399 if (DBUF_EQUAL(dbf, os, obj, level, blkid)) { 400 mutex_enter(&dbf->db_mtx); 401 if (dbf->db_state != DB_EVICTING) { 402 rw_exit(DBUF_HASH_RWLOCK(h, idx)); 403 return (dbf); 404 } 405 mutex_exit(&dbf->db_mtx); 406 } 407 } 408 409 if (i > 0) { 410 DBUF_STAT_BUMP(hash_collisions); 411 if (i == 1) 412 DBUF_STAT_BUMP(hash_chains); 413 414 DBUF_STAT_MAX(hash_chain_max, i); 415 } 416 417 mutex_enter(&db->db_mtx); 418 db->db_hash_next = h->hash_table[idx]; 419 h->hash_table[idx] = db; 420 rw_exit(DBUF_HASH_RWLOCK(h, idx)); 421 uint64_t he = atomic_inc_64_nv(&dbuf_stats.hash_elements.value.ui64); 422 DBUF_STAT_MAX(hash_elements_max, he); 423 424 return (NULL); 425 } 426 427 /* 428 * This returns whether this dbuf should be stored in the metadata cache, which 429 * is based on whether it's from one of the dnode types that store data related 430 * to traversing dataset hierarchies. 431 */ 432 static boolean_t 433 dbuf_include_in_metadata_cache(dmu_buf_impl_t *db) 434 { 435 DB_DNODE_ENTER(db); 436 dmu_object_type_t type = DB_DNODE(db)->dn_type; 437 DB_DNODE_EXIT(db); 438 439 /* Check if this dbuf is one of the types we care about */ 440 if (DMU_OT_IS_METADATA_CACHED(type)) { 441 /* If we hit this, then we set something up wrong in dmu_ot */ 442 ASSERT(DMU_OT_IS_METADATA(type)); 443 444 /* 445 * Sanity check for small-memory systems: don't allocate too 446 * much memory for this purpose. 447 */ 448 if (zfs_refcount_count( 449 &dbuf_caches[DB_DBUF_METADATA_CACHE].size) > 450 dbuf_metadata_cache_target_bytes()) { 451 DBUF_STAT_BUMP(metadata_cache_overflow); 452 return (B_FALSE); 453 } 454 455 return (B_TRUE); 456 } 457 458 return (B_FALSE); 459 } 460 461 /* 462 * Remove an entry from the hash table. It must be in the EVICTING state. 463 */ 464 static void 465 dbuf_hash_remove(dmu_buf_impl_t *db) 466 { 467 dbuf_hash_table_t *h = &dbuf_hash_table; 468 uint64_t hv, idx; 469 dmu_buf_impl_t *dbf, **dbp; 470 471 hv = dbuf_hash(db->db_objset, db->db.db_object, 472 db->db_level, db->db_blkid); 473 idx = hv & h->hash_table_mask; 474 475 /* 476 * We mustn't hold db_mtx to maintain lock ordering: 477 * DBUF_HASH_RWLOCK > db_mtx. 478 */ 479 ASSERT(zfs_refcount_is_zero(&db->db_holds)); 480 ASSERT(db->db_state == DB_EVICTING); 481 ASSERT(!MUTEX_HELD(&db->db_mtx)); 482 483 rw_enter(DBUF_HASH_RWLOCK(h, idx), RW_WRITER); 484 dbp = &h->hash_table[idx]; 485 while ((dbf = *dbp) != db) { 486 dbp = &dbf->db_hash_next; 487 ASSERT(dbf != NULL); 488 } 489 *dbp = db->db_hash_next; 490 db->db_hash_next = NULL; 491 if (h->hash_table[idx] && 492 h->hash_table[idx]->db_hash_next == NULL) 493 DBUF_STAT_BUMPDOWN(hash_chains); 494 rw_exit(DBUF_HASH_RWLOCK(h, idx)); 495 atomic_dec_64(&dbuf_stats.hash_elements.value.ui64); 496 } 497 498 typedef enum { 499 DBVU_EVICTING, 500 DBVU_NOT_EVICTING 501 } dbvu_verify_type_t; 502 503 static void 504 dbuf_verify_user(dmu_buf_impl_t *db, dbvu_verify_type_t verify_type) 505 { 506 #ifdef ZFS_DEBUG 507 int64_t holds; 508 509 if (db->db_user == NULL) 510 return; 511 512 /* Only data blocks support the attachment of user data. */ 513 ASSERT(db->db_level == 0); 514 515 /* Clients must resolve a dbuf before attaching user data. */ 516 ASSERT(db->db.db_data != NULL); 517 ASSERT3U(db->db_state, ==, DB_CACHED); 518 519 holds = zfs_refcount_count(&db->db_holds); 520 if (verify_type == DBVU_EVICTING) { 521 /* 522 * Immediate eviction occurs when holds == dirtycnt. 523 * For normal eviction buffers, holds is zero on 524 * eviction, except when dbuf_fix_old_data() calls 525 * dbuf_clear_data(). However, the hold count can grow 526 * during eviction even though db_mtx is held (see 527 * dmu_bonus_hold() for an example), so we can only 528 * test the generic invariant that holds >= dirtycnt. 529 */ 530 ASSERT3U(holds, >=, db->db_dirtycnt); 531 } else { 532 if (db->db_user_immediate_evict == TRUE) 533 ASSERT3U(holds, >=, db->db_dirtycnt); 534 else 535 ASSERT3U(holds, >, 0); 536 } 537 #endif 538 } 539 540 static void 541 dbuf_evict_user(dmu_buf_impl_t *db) 542 { 543 dmu_buf_user_t *dbu = db->db_user; 544 545 ASSERT(MUTEX_HELD(&db->db_mtx)); 546 547 if (dbu == NULL) 548 return; 549 550 dbuf_verify_user(db, DBVU_EVICTING); 551 db->db_user = NULL; 552 553 #ifdef ZFS_DEBUG 554 if (dbu->dbu_clear_on_evict_dbufp != NULL) 555 *dbu->dbu_clear_on_evict_dbufp = NULL; 556 #endif 557 558 /* 559 * There are two eviction callbacks - one that we call synchronously 560 * and one that we invoke via a taskq. The async one is useful for 561 * avoiding lock order reversals and limiting stack depth. 562 * 563 * Note that if we have a sync callback but no async callback, 564 * it's likely that the sync callback will free the structure 565 * containing the dbu. In that case we need to take care to not 566 * dereference dbu after calling the sync evict func. 567 */ 568 boolean_t has_async = (dbu->dbu_evict_func_async != NULL); 569 570 if (dbu->dbu_evict_func_sync != NULL) 571 dbu->dbu_evict_func_sync(dbu); 572 573 if (has_async) { 574 taskq_dispatch_ent(dbu_evict_taskq, dbu->dbu_evict_func_async, 575 dbu, 0, &dbu->dbu_tqent); 576 } 577 } 578 579 boolean_t 580 dbuf_is_metadata(dmu_buf_impl_t *db) 581 { 582 /* 583 * Consider indirect blocks and spill blocks to be meta data. 584 */ 585 if (db->db_level > 0 || db->db_blkid == DMU_SPILL_BLKID) { 586 return (B_TRUE); 587 } else { 588 boolean_t is_metadata; 589 590 DB_DNODE_ENTER(db); 591 is_metadata = DMU_OT_IS_METADATA(DB_DNODE(db)->dn_type); 592 DB_DNODE_EXIT(db); 593 594 return (is_metadata); 595 } 596 } 597 598 /* 599 * We want to exclude buffers that are on a special allocation class from 600 * L2ARC. 601 */ 602 boolean_t 603 dbuf_is_l2cacheable(dmu_buf_impl_t *db) 604 { 605 vdev_t *vd = NULL; 606 zfs_cache_type_t cache = db->db_objset->os_secondary_cache; 607 blkptr_t *bp = db->db_blkptr; 608 609 if (bp != NULL && !BP_IS_HOLE(bp)) { 610 uint64_t vdev = DVA_GET_VDEV(bp->blk_dva); 611 vdev_t *rvd = db->db_objset->os_spa->spa_root_vdev; 612 613 if (vdev < rvd->vdev_children) 614 vd = rvd->vdev_child[vdev]; 615 616 if (cache == ZFS_CACHE_ALL || 617 (dbuf_is_metadata(db) && cache == ZFS_CACHE_METADATA)) { 618 if (vd == NULL) 619 return (B_TRUE); 620 621 if ((vd->vdev_alloc_bias != VDEV_BIAS_SPECIAL && 622 vd->vdev_alloc_bias != VDEV_BIAS_DEDUP) || 623 l2arc_exclude_special == 0) 624 return (B_TRUE); 625 } 626 } 627 628 return (B_FALSE); 629 } 630 631 static inline boolean_t 632 dnode_level_is_l2cacheable(blkptr_t *bp, dnode_t *dn, int64_t level) 633 { 634 vdev_t *vd = NULL; 635 zfs_cache_type_t cache = dn->dn_objset->os_secondary_cache; 636 637 if (bp != NULL && !BP_IS_HOLE(bp)) { 638 uint64_t vdev = DVA_GET_VDEV(bp->blk_dva); 639 vdev_t *rvd = dn->dn_objset->os_spa->spa_root_vdev; 640 641 if (vdev < rvd->vdev_children) 642 vd = rvd->vdev_child[vdev]; 643 644 if (cache == ZFS_CACHE_ALL || ((level > 0 || 645 DMU_OT_IS_METADATA(dn->dn_handle->dnh_dnode->dn_type)) && 646 cache == ZFS_CACHE_METADATA)) { 647 if (vd == NULL) 648 return (B_TRUE); 649 650 if ((vd->vdev_alloc_bias != VDEV_BIAS_SPECIAL && 651 vd->vdev_alloc_bias != VDEV_BIAS_DEDUP) || 652 l2arc_exclude_special == 0) 653 return (B_TRUE); 654 } 655 } 656 657 return (B_FALSE); 658 } 659 660 661 /* 662 * This function *must* return indices evenly distributed between all 663 * sublists of the multilist. This is needed due to how the dbuf eviction 664 * code is laid out; dbuf_evict_thread() assumes dbufs are evenly 665 * distributed between all sublists and uses this assumption when 666 * deciding which sublist to evict from and how much to evict from it. 667 */ 668 static unsigned int 669 dbuf_cache_multilist_index_func(multilist_t *ml, void *obj) 670 { 671 dmu_buf_impl_t *db = obj; 672 673 /* 674 * The assumption here, is the hash value for a given 675 * dmu_buf_impl_t will remain constant throughout it's lifetime 676 * (i.e. it's objset, object, level and blkid fields don't change). 677 * Thus, we don't need to store the dbuf's sublist index 678 * on insertion, as this index can be recalculated on removal. 679 * 680 * Also, the low order bits of the hash value are thought to be 681 * distributed evenly. Otherwise, in the case that the multilist 682 * has a power of two number of sublists, each sublists' usage 683 * would not be evenly distributed. In this context full 64bit 684 * division would be a waste of time, so limit it to 32 bits. 685 */ 686 return ((unsigned int)dbuf_hash(db->db_objset, db->db.db_object, 687 db->db_level, db->db_blkid) % 688 multilist_get_num_sublists(ml)); 689 } 690 691 /* 692 * The target size of the dbuf cache can grow with the ARC target, 693 * unless limited by the tunable dbuf_cache_max_bytes. 694 */ 695 static inline unsigned long 696 dbuf_cache_target_bytes(void) 697 { 698 return (MIN(dbuf_cache_max_bytes, 699 arc_target_bytes() >> dbuf_cache_shift)); 700 } 701 702 /* 703 * The target size of the dbuf metadata cache can grow with the ARC target, 704 * unless limited by the tunable dbuf_metadata_cache_max_bytes. 705 */ 706 static inline unsigned long 707 dbuf_metadata_cache_target_bytes(void) 708 { 709 return (MIN(dbuf_metadata_cache_max_bytes, 710 arc_target_bytes() >> dbuf_metadata_cache_shift)); 711 } 712 713 static inline uint64_t 714 dbuf_cache_hiwater_bytes(void) 715 { 716 uint64_t dbuf_cache_target = dbuf_cache_target_bytes(); 717 return (dbuf_cache_target + 718 (dbuf_cache_target * dbuf_cache_hiwater_pct) / 100); 719 } 720 721 static inline uint64_t 722 dbuf_cache_lowater_bytes(void) 723 { 724 uint64_t dbuf_cache_target = dbuf_cache_target_bytes(); 725 return (dbuf_cache_target - 726 (dbuf_cache_target * dbuf_cache_lowater_pct) / 100); 727 } 728 729 static inline boolean_t 730 dbuf_cache_above_lowater(void) 731 { 732 return (zfs_refcount_count(&dbuf_caches[DB_DBUF_CACHE].size) > 733 dbuf_cache_lowater_bytes()); 734 } 735 736 /* 737 * Evict the oldest eligible dbuf from the dbuf cache. 738 */ 739 static void 740 dbuf_evict_one(void) 741 { 742 int idx = multilist_get_random_index(&dbuf_caches[DB_DBUF_CACHE].cache); 743 multilist_sublist_t *mls = multilist_sublist_lock( 744 &dbuf_caches[DB_DBUF_CACHE].cache, idx); 745 746 ASSERT(!MUTEX_HELD(&dbuf_evict_lock)); 747 748 dmu_buf_impl_t *db = multilist_sublist_tail(mls); 749 while (db != NULL && mutex_tryenter(&db->db_mtx) == 0) { 750 db = multilist_sublist_prev(mls, db); 751 } 752 753 DTRACE_PROBE2(dbuf__evict__one, dmu_buf_impl_t *, db, 754 multilist_sublist_t *, mls); 755 756 if (db != NULL) { 757 multilist_sublist_remove(mls, db); 758 multilist_sublist_unlock(mls); 759 (void) zfs_refcount_remove_many( 760 &dbuf_caches[DB_DBUF_CACHE].size, db->db.db_size, db); 761 DBUF_STAT_BUMPDOWN(cache_levels[db->db_level]); 762 DBUF_STAT_BUMPDOWN(cache_count); 763 DBUF_STAT_DECR(cache_levels_bytes[db->db_level], 764 db->db.db_size); 765 ASSERT3U(db->db_caching_status, ==, DB_DBUF_CACHE); 766 db->db_caching_status = DB_NO_CACHE; 767 dbuf_destroy(db); 768 DBUF_STAT_BUMP(cache_total_evicts); 769 } else { 770 multilist_sublist_unlock(mls); 771 } 772 } 773 774 /* 775 * The dbuf evict thread is responsible for aging out dbufs from the 776 * cache. Once the cache has reached it's maximum size, dbufs are removed 777 * and destroyed. The eviction thread will continue running until the size 778 * of the dbuf cache is at or below the maximum size. Once the dbuf is aged 779 * out of the cache it is destroyed and becomes eligible for arc eviction. 780 */ 781 static __attribute__((noreturn)) void 782 dbuf_evict_thread(void *unused) 783 { 784 (void) unused; 785 callb_cpr_t cpr; 786 787 CALLB_CPR_INIT(&cpr, &dbuf_evict_lock, callb_generic_cpr, FTAG); 788 789 mutex_enter(&dbuf_evict_lock); 790 while (!dbuf_evict_thread_exit) { 791 while (!dbuf_cache_above_lowater() && !dbuf_evict_thread_exit) { 792 CALLB_CPR_SAFE_BEGIN(&cpr); 793 (void) cv_timedwait_idle_hires(&dbuf_evict_cv, 794 &dbuf_evict_lock, SEC2NSEC(1), MSEC2NSEC(1), 0); 795 CALLB_CPR_SAFE_END(&cpr, &dbuf_evict_lock); 796 } 797 mutex_exit(&dbuf_evict_lock); 798 799 /* 800 * Keep evicting as long as we're above the low water mark 801 * for the cache. We do this without holding the locks to 802 * minimize lock contention. 803 */ 804 while (dbuf_cache_above_lowater() && !dbuf_evict_thread_exit) { 805 dbuf_evict_one(); 806 } 807 808 mutex_enter(&dbuf_evict_lock); 809 } 810 811 dbuf_evict_thread_exit = B_FALSE; 812 cv_broadcast(&dbuf_evict_cv); 813 CALLB_CPR_EXIT(&cpr); /* drops dbuf_evict_lock */ 814 thread_exit(); 815 } 816 817 /* 818 * Wake up the dbuf eviction thread if the dbuf cache is at its max size. 819 * If the dbuf cache is at its high water mark, then evict a dbuf from the 820 * dbuf cache using the caller's context. 821 */ 822 static void 823 dbuf_evict_notify(uint64_t size) 824 { 825 /* 826 * We check if we should evict without holding the dbuf_evict_lock, 827 * because it's OK to occasionally make the wrong decision here, 828 * and grabbing the lock results in massive lock contention. 829 */ 830 if (size > dbuf_cache_target_bytes()) { 831 if (size > dbuf_cache_hiwater_bytes()) 832 dbuf_evict_one(); 833 cv_signal(&dbuf_evict_cv); 834 } 835 } 836 837 static int 838 dbuf_kstat_update(kstat_t *ksp, int rw) 839 { 840 dbuf_stats_t *ds = ksp->ks_data; 841 842 if (rw == KSTAT_WRITE) 843 return (SET_ERROR(EACCES)); 844 845 ds->cache_count.value.ui64 = 846 wmsum_value(&dbuf_sums.cache_count); 847 ds->cache_size_bytes.value.ui64 = 848 zfs_refcount_count(&dbuf_caches[DB_DBUF_CACHE].size); 849 ds->cache_target_bytes.value.ui64 = dbuf_cache_target_bytes(); 850 ds->cache_hiwater_bytes.value.ui64 = dbuf_cache_hiwater_bytes(); 851 ds->cache_lowater_bytes.value.ui64 = dbuf_cache_lowater_bytes(); 852 ds->cache_total_evicts.value.ui64 = 853 wmsum_value(&dbuf_sums.cache_total_evicts); 854 for (int i = 0; i < DN_MAX_LEVELS; i++) { 855 ds->cache_levels[i].value.ui64 = 856 wmsum_value(&dbuf_sums.cache_levels[i]); 857 ds->cache_levels_bytes[i].value.ui64 = 858 wmsum_value(&dbuf_sums.cache_levels_bytes[i]); 859 } 860 ds->hash_hits.value.ui64 = 861 wmsum_value(&dbuf_sums.hash_hits); 862 ds->hash_misses.value.ui64 = 863 wmsum_value(&dbuf_sums.hash_misses); 864 ds->hash_collisions.value.ui64 = 865 wmsum_value(&dbuf_sums.hash_collisions); 866 ds->hash_chains.value.ui64 = 867 wmsum_value(&dbuf_sums.hash_chains); 868 ds->hash_insert_race.value.ui64 = 869 wmsum_value(&dbuf_sums.hash_insert_race); 870 ds->metadata_cache_count.value.ui64 = 871 wmsum_value(&dbuf_sums.metadata_cache_count); 872 ds->metadata_cache_size_bytes.value.ui64 = zfs_refcount_count( 873 &dbuf_caches[DB_DBUF_METADATA_CACHE].size); 874 ds->metadata_cache_overflow.value.ui64 = 875 wmsum_value(&dbuf_sums.metadata_cache_overflow); 876 return (0); 877 } 878 879 void 880 dbuf_init(void) 881 { 882 uint64_t hsize = 1ULL << 16; 883 dbuf_hash_table_t *h = &dbuf_hash_table; 884 int i; 885 886 /* 887 * The hash table is big enough to fill one eighth of physical memory 888 * with an average block size of zfs_arc_average_blocksize (default 8K). 889 * By default, the table will take up 890 * totalmem * sizeof(void*) / 8K (1MB per GB with 8-byte pointers). 891 */ 892 while (hsize * zfs_arc_average_blocksize < arc_all_memory() / 8) 893 hsize <<= 1; 894 895 retry: 896 h->hash_table_mask = hsize - 1; 897 #if defined(_KERNEL) 898 /* 899 * Large allocations which do not require contiguous pages 900 * should be using vmem_alloc() in the linux kernel 901 */ 902 h->hash_table = vmem_zalloc(hsize * sizeof (void *), KM_SLEEP); 903 #else 904 h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP); 905 #endif 906 if (h->hash_table == NULL) { 907 /* XXX - we should really return an error instead of assert */ 908 ASSERT(hsize > (1ULL << 10)); 909 hsize >>= 1; 910 goto retry; 911 } 912 913 dbuf_kmem_cache = kmem_cache_create("dmu_buf_impl_t", 914 sizeof (dmu_buf_impl_t), 915 0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0); 916 917 for (i = 0; i < DBUF_RWLOCKS; i++) 918 rw_init(&h->hash_rwlocks[i], NULL, RW_DEFAULT, NULL); 919 920 dbuf_stats_init(h); 921 922 /* 923 * All entries are queued via taskq_dispatch_ent(), so min/maxalloc 924 * configuration is not required. 925 */ 926 dbu_evict_taskq = taskq_create("dbu_evict", 1, defclsyspri, 0, 0, 0); 927 928 for (dbuf_cached_state_t dcs = 0; dcs < DB_CACHE_MAX; dcs++) { 929 multilist_create(&dbuf_caches[dcs].cache, 930 sizeof (dmu_buf_impl_t), 931 offsetof(dmu_buf_impl_t, db_cache_link), 932 dbuf_cache_multilist_index_func); 933 zfs_refcount_create(&dbuf_caches[dcs].size); 934 } 935 936 dbuf_evict_thread_exit = B_FALSE; 937 mutex_init(&dbuf_evict_lock, NULL, MUTEX_DEFAULT, NULL); 938 cv_init(&dbuf_evict_cv, NULL, CV_DEFAULT, NULL); 939 dbuf_cache_evict_thread = thread_create(NULL, 0, dbuf_evict_thread, 940 NULL, 0, &p0, TS_RUN, minclsyspri); 941 942 wmsum_init(&dbuf_sums.cache_count, 0); 943 wmsum_init(&dbuf_sums.cache_total_evicts, 0); 944 for (i = 0; i < DN_MAX_LEVELS; i++) { 945 wmsum_init(&dbuf_sums.cache_levels[i], 0); 946 wmsum_init(&dbuf_sums.cache_levels_bytes[i], 0); 947 } 948 wmsum_init(&dbuf_sums.hash_hits, 0); 949 wmsum_init(&dbuf_sums.hash_misses, 0); 950 wmsum_init(&dbuf_sums.hash_collisions, 0); 951 wmsum_init(&dbuf_sums.hash_chains, 0); 952 wmsum_init(&dbuf_sums.hash_insert_race, 0); 953 wmsum_init(&dbuf_sums.metadata_cache_count, 0); 954 wmsum_init(&dbuf_sums.metadata_cache_overflow, 0); 955 956 dbuf_ksp = kstat_create("zfs", 0, "dbufstats", "misc", 957 KSTAT_TYPE_NAMED, sizeof (dbuf_stats) / sizeof (kstat_named_t), 958 KSTAT_FLAG_VIRTUAL); 959 if (dbuf_ksp != NULL) { 960 for (i = 0; i < DN_MAX_LEVELS; i++) { 961 snprintf(dbuf_stats.cache_levels[i].name, 962 KSTAT_STRLEN, "cache_level_%d", i); 963 dbuf_stats.cache_levels[i].data_type = 964 KSTAT_DATA_UINT64; 965 snprintf(dbuf_stats.cache_levels_bytes[i].name, 966 KSTAT_STRLEN, "cache_level_%d_bytes", i); 967 dbuf_stats.cache_levels_bytes[i].data_type = 968 KSTAT_DATA_UINT64; 969 } 970 dbuf_ksp->ks_data = &dbuf_stats; 971 dbuf_ksp->ks_update = dbuf_kstat_update; 972 kstat_install(dbuf_ksp); 973 } 974 } 975 976 void 977 dbuf_fini(void) 978 { 979 dbuf_hash_table_t *h = &dbuf_hash_table; 980 int i; 981 982 dbuf_stats_destroy(); 983 984 for (i = 0; i < DBUF_RWLOCKS; i++) 985 rw_destroy(&h->hash_rwlocks[i]); 986 #if defined(_KERNEL) 987 /* 988 * Large allocations which do not require contiguous pages 989 * should be using vmem_free() in the linux kernel 990 */ 991 vmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *)); 992 #else 993 kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *)); 994 #endif 995 kmem_cache_destroy(dbuf_kmem_cache); 996 taskq_destroy(dbu_evict_taskq); 997 998 mutex_enter(&dbuf_evict_lock); 999 dbuf_evict_thread_exit = B_TRUE; 1000 while (dbuf_evict_thread_exit) { 1001 cv_signal(&dbuf_evict_cv); 1002 cv_wait(&dbuf_evict_cv, &dbuf_evict_lock); 1003 } 1004 mutex_exit(&dbuf_evict_lock); 1005 1006 mutex_destroy(&dbuf_evict_lock); 1007 cv_destroy(&dbuf_evict_cv); 1008 1009 for (dbuf_cached_state_t dcs = 0; dcs < DB_CACHE_MAX; dcs++) { 1010 zfs_refcount_destroy(&dbuf_caches[dcs].size); 1011 multilist_destroy(&dbuf_caches[dcs].cache); 1012 } 1013 1014 if (dbuf_ksp != NULL) { 1015 kstat_delete(dbuf_ksp); 1016 dbuf_ksp = NULL; 1017 } 1018 1019 wmsum_fini(&dbuf_sums.cache_count); 1020 wmsum_fini(&dbuf_sums.cache_total_evicts); 1021 for (i = 0; i < DN_MAX_LEVELS; i++) { 1022 wmsum_fini(&dbuf_sums.cache_levels[i]); 1023 wmsum_fini(&dbuf_sums.cache_levels_bytes[i]); 1024 } 1025 wmsum_fini(&dbuf_sums.hash_hits); 1026 wmsum_fini(&dbuf_sums.hash_misses); 1027 wmsum_fini(&dbuf_sums.hash_collisions); 1028 wmsum_fini(&dbuf_sums.hash_chains); 1029 wmsum_fini(&dbuf_sums.hash_insert_race); 1030 wmsum_fini(&dbuf_sums.metadata_cache_count); 1031 wmsum_fini(&dbuf_sums.metadata_cache_overflow); 1032 } 1033 1034 /* 1035 * Other stuff. 1036 */ 1037 1038 #ifdef ZFS_DEBUG 1039 static void 1040 dbuf_verify(dmu_buf_impl_t *db) 1041 { 1042 dnode_t *dn; 1043 dbuf_dirty_record_t *dr; 1044 uint32_t txg_prev; 1045 1046 ASSERT(MUTEX_HELD(&db->db_mtx)); 1047 1048 if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY)) 1049 return; 1050 1051 ASSERT(db->db_objset != NULL); 1052 DB_DNODE_ENTER(db); 1053 dn = DB_DNODE(db); 1054 if (dn == NULL) { 1055 ASSERT(db->db_parent == NULL); 1056 ASSERT(db->db_blkptr == NULL); 1057 } else { 1058 ASSERT3U(db->db.db_object, ==, dn->dn_object); 1059 ASSERT3P(db->db_objset, ==, dn->dn_objset); 1060 ASSERT3U(db->db_level, <, dn->dn_nlevels); 1061 ASSERT(db->db_blkid == DMU_BONUS_BLKID || 1062 db->db_blkid == DMU_SPILL_BLKID || 1063 !avl_is_empty(&dn->dn_dbufs)); 1064 } 1065 if (db->db_blkid == DMU_BONUS_BLKID) { 1066 ASSERT(dn != NULL); 1067 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 1068 ASSERT3U(db->db.db_offset, ==, DMU_BONUS_BLKID); 1069 } else if (db->db_blkid == DMU_SPILL_BLKID) { 1070 ASSERT(dn != NULL); 1071 ASSERT0(db->db.db_offset); 1072 } else { 1073 ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size); 1074 } 1075 1076 if ((dr = list_head(&db->db_dirty_records)) != NULL) { 1077 ASSERT(dr->dr_dbuf == db); 1078 txg_prev = dr->dr_txg; 1079 for (dr = list_next(&db->db_dirty_records, dr); dr != NULL; 1080 dr = list_next(&db->db_dirty_records, dr)) { 1081 ASSERT(dr->dr_dbuf == db); 1082 ASSERT(txg_prev > dr->dr_txg); 1083 txg_prev = dr->dr_txg; 1084 } 1085 } 1086 1087 /* 1088 * We can't assert that db_size matches dn_datablksz because it 1089 * can be momentarily different when another thread is doing 1090 * dnode_set_blksz(). 1091 */ 1092 if (db->db_level == 0 && db->db.db_object == DMU_META_DNODE_OBJECT) { 1093 dr = db->db_data_pending; 1094 /* 1095 * It should only be modified in syncing context, so 1096 * make sure we only have one copy of the data. 1097 */ 1098 ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf); 1099 } 1100 1101 /* verify db->db_blkptr */ 1102 if (db->db_blkptr) { 1103 if (db->db_parent == dn->dn_dbuf) { 1104 /* db is pointed to by the dnode */ 1105 /* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */ 1106 if (DMU_OBJECT_IS_SPECIAL(db->db.db_object)) 1107 ASSERT(db->db_parent == NULL); 1108 else 1109 ASSERT(db->db_parent != NULL); 1110 if (db->db_blkid != DMU_SPILL_BLKID) 1111 ASSERT3P(db->db_blkptr, ==, 1112 &dn->dn_phys->dn_blkptr[db->db_blkid]); 1113 } else { 1114 /* db is pointed to by an indirect block */ 1115 int epb __maybe_unused = db->db_parent->db.db_size >> 1116 SPA_BLKPTRSHIFT; 1117 ASSERT3U(db->db_parent->db_level, ==, db->db_level+1); 1118 ASSERT3U(db->db_parent->db.db_object, ==, 1119 db->db.db_object); 1120 /* 1121 * dnode_grow_indblksz() can make this fail if we don't 1122 * have the parent's rwlock. XXX indblksz no longer 1123 * grows. safe to do this now? 1124 */ 1125 if (RW_LOCK_HELD(&db->db_parent->db_rwlock)) { 1126 ASSERT3P(db->db_blkptr, ==, 1127 ((blkptr_t *)db->db_parent->db.db_data + 1128 db->db_blkid % epb)); 1129 } 1130 } 1131 } 1132 if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) && 1133 (db->db_buf == NULL || db->db_buf->b_data) && 1134 db->db.db_data && db->db_blkid != DMU_BONUS_BLKID && 1135 db->db_state != DB_FILL && !dn->dn_free_txg) { 1136 /* 1137 * If the blkptr isn't set but they have nonzero data, 1138 * it had better be dirty, otherwise we'll lose that 1139 * data when we evict this buffer. 1140 * 1141 * There is an exception to this rule for indirect blocks; in 1142 * this case, if the indirect block is a hole, we fill in a few 1143 * fields on each of the child blocks (importantly, birth time) 1144 * to prevent hole birth times from being lost when you 1145 * partially fill in a hole. 1146 */ 1147 if (db->db_dirtycnt == 0) { 1148 if (db->db_level == 0) { 1149 uint64_t *buf = db->db.db_data; 1150 int i; 1151 1152 for (i = 0; i < db->db.db_size >> 3; i++) { 1153 ASSERT(buf[i] == 0); 1154 } 1155 } else { 1156 blkptr_t *bps = db->db.db_data; 1157 ASSERT3U(1 << DB_DNODE(db)->dn_indblkshift, ==, 1158 db->db.db_size); 1159 /* 1160 * We want to verify that all the blkptrs in the 1161 * indirect block are holes, but we may have 1162 * automatically set up a few fields for them. 1163 * We iterate through each blkptr and verify 1164 * they only have those fields set. 1165 */ 1166 for (int i = 0; 1167 i < db->db.db_size / sizeof (blkptr_t); 1168 i++) { 1169 blkptr_t *bp = &bps[i]; 1170 ASSERT(ZIO_CHECKSUM_IS_ZERO( 1171 &bp->blk_cksum)); 1172 ASSERT( 1173 DVA_IS_EMPTY(&bp->blk_dva[0]) && 1174 DVA_IS_EMPTY(&bp->blk_dva[1]) && 1175 DVA_IS_EMPTY(&bp->blk_dva[2])); 1176 ASSERT0(bp->blk_fill); 1177 ASSERT0(bp->blk_pad[0]); 1178 ASSERT0(bp->blk_pad[1]); 1179 ASSERT(!BP_IS_EMBEDDED(bp)); 1180 ASSERT(BP_IS_HOLE(bp)); 1181 ASSERT0(bp->blk_phys_birth); 1182 } 1183 } 1184 } 1185 } 1186 DB_DNODE_EXIT(db); 1187 } 1188 #endif 1189 1190 static void 1191 dbuf_clear_data(dmu_buf_impl_t *db) 1192 { 1193 ASSERT(MUTEX_HELD(&db->db_mtx)); 1194 dbuf_evict_user(db); 1195 ASSERT3P(db->db_buf, ==, NULL); 1196 db->db.db_data = NULL; 1197 if (db->db_state != DB_NOFILL) { 1198 db->db_state = DB_UNCACHED; 1199 DTRACE_SET_STATE(db, "clear data"); 1200 } 1201 } 1202 1203 static void 1204 dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf) 1205 { 1206 ASSERT(MUTEX_HELD(&db->db_mtx)); 1207 ASSERT(buf != NULL); 1208 1209 db->db_buf = buf; 1210 ASSERT(buf->b_data != NULL); 1211 db->db.db_data = buf->b_data; 1212 } 1213 1214 static arc_buf_t * 1215 dbuf_alloc_arcbuf(dmu_buf_impl_t *db) 1216 { 1217 spa_t *spa = db->db_objset->os_spa; 1218 1219 return (arc_alloc_buf(spa, db, DBUF_GET_BUFC_TYPE(db), db->db.db_size)); 1220 } 1221 1222 /* 1223 * Loan out an arc_buf for read. Return the loaned arc_buf. 1224 */ 1225 arc_buf_t * 1226 dbuf_loan_arcbuf(dmu_buf_impl_t *db) 1227 { 1228 arc_buf_t *abuf; 1229 1230 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1231 mutex_enter(&db->db_mtx); 1232 if (arc_released(db->db_buf) || zfs_refcount_count(&db->db_holds) > 1) { 1233 int blksz = db->db.db_size; 1234 spa_t *spa = db->db_objset->os_spa; 1235 1236 mutex_exit(&db->db_mtx); 1237 abuf = arc_loan_buf(spa, B_FALSE, blksz); 1238 memcpy(abuf->b_data, db->db.db_data, blksz); 1239 } else { 1240 abuf = db->db_buf; 1241 arc_loan_inuse_buf(abuf, db); 1242 db->db_buf = NULL; 1243 dbuf_clear_data(db); 1244 mutex_exit(&db->db_mtx); 1245 } 1246 return (abuf); 1247 } 1248 1249 /* 1250 * Calculate which level n block references the data at the level 0 offset 1251 * provided. 1252 */ 1253 uint64_t 1254 dbuf_whichblock(const dnode_t *dn, const int64_t level, const uint64_t offset) 1255 { 1256 if (dn->dn_datablkshift != 0 && dn->dn_indblkshift != 0) { 1257 /* 1258 * The level n blkid is equal to the level 0 blkid divided by 1259 * the number of level 0s in a level n block. 1260 * 1261 * The level 0 blkid is offset >> datablkshift = 1262 * offset / 2^datablkshift. 1263 * 1264 * The number of level 0s in a level n is the number of block 1265 * pointers in an indirect block, raised to the power of level. 1266 * This is 2^(indblkshift - SPA_BLKPTRSHIFT)^level = 1267 * 2^(level*(indblkshift - SPA_BLKPTRSHIFT)). 1268 * 1269 * Thus, the level n blkid is: offset / 1270 * ((2^datablkshift)*(2^(level*(indblkshift-SPA_BLKPTRSHIFT)))) 1271 * = offset / 2^(datablkshift + level * 1272 * (indblkshift - SPA_BLKPTRSHIFT)) 1273 * = offset >> (datablkshift + level * 1274 * (indblkshift - SPA_BLKPTRSHIFT)) 1275 */ 1276 1277 const unsigned exp = dn->dn_datablkshift + 1278 level * (dn->dn_indblkshift - SPA_BLKPTRSHIFT); 1279 1280 if (exp >= 8 * sizeof (offset)) { 1281 /* This only happens on the highest indirection level */ 1282 ASSERT3U(level, ==, dn->dn_nlevels - 1); 1283 return (0); 1284 } 1285 1286 ASSERT3U(exp, <, 8 * sizeof (offset)); 1287 1288 return (offset >> exp); 1289 } else { 1290 ASSERT3U(offset, <, dn->dn_datablksz); 1291 return (0); 1292 } 1293 } 1294 1295 /* 1296 * This function is used to lock the parent of the provided dbuf. This should be 1297 * used when modifying or reading db_blkptr. 1298 */ 1299 db_lock_type_t 1300 dmu_buf_lock_parent(dmu_buf_impl_t *db, krw_t rw, const void *tag) 1301 { 1302 enum db_lock_type ret = DLT_NONE; 1303 if (db->db_parent != NULL) { 1304 rw_enter(&db->db_parent->db_rwlock, rw); 1305 ret = DLT_PARENT; 1306 } else if (dmu_objset_ds(db->db_objset) != NULL) { 1307 rrw_enter(&dmu_objset_ds(db->db_objset)->ds_bp_rwlock, rw, 1308 tag); 1309 ret = DLT_OBJSET; 1310 } 1311 /* 1312 * We only return a DLT_NONE lock when it's the top-most indirect block 1313 * of the meta-dnode of the MOS. 1314 */ 1315 return (ret); 1316 } 1317 1318 /* 1319 * We need to pass the lock type in because it's possible that the block will 1320 * move from being the topmost indirect block in a dnode (and thus, have no 1321 * parent) to not the top-most via an indirection increase. This would cause a 1322 * panic if we didn't pass the lock type in. 1323 */ 1324 void 1325 dmu_buf_unlock_parent(dmu_buf_impl_t *db, db_lock_type_t type, const void *tag) 1326 { 1327 if (type == DLT_PARENT) 1328 rw_exit(&db->db_parent->db_rwlock); 1329 else if (type == DLT_OBJSET) 1330 rrw_exit(&dmu_objset_ds(db->db_objset)->ds_bp_rwlock, tag); 1331 } 1332 1333 static void 1334 dbuf_read_done(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp, 1335 arc_buf_t *buf, void *vdb) 1336 { 1337 (void) zb, (void) bp; 1338 dmu_buf_impl_t *db = vdb; 1339 1340 mutex_enter(&db->db_mtx); 1341 ASSERT3U(db->db_state, ==, DB_READ); 1342 /* 1343 * All reads are synchronous, so we must have a hold on the dbuf 1344 */ 1345 ASSERT(zfs_refcount_count(&db->db_holds) > 0); 1346 ASSERT(db->db_buf == NULL); 1347 ASSERT(db->db.db_data == NULL); 1348 if (buf == NULL) { 1349 /* i/o error */ 1350 ASSERT(zio == NULL || zio->io_error != 0); 1351 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1352 ASSERT3P(db->db_buf, ==, NULL); 1353 db->db_state = DB_UNCACHED; 1354 DTRACE_SET_STATE(db, "i/o error"); 1355 } else if (db->db_level == 0 && db->db_freed_in_flight) { 1356 /* freed in flight */ 1357 ASSERT(zio == NULL || zio->io_error == 0); 1358 arc_release(buf, db); 1359 memset(buf->b_data, 0, db->db.db_size); 1360 arc_buf_freeze(buf); 1361 db->db_freed_in_flight = FALSE; 1362 dbuf_set_data(db, buf); 1363 db->db_state = DB_CACHED; 1364 DTRACE_SET_STATE(db, "freed in flight"); 1365 } else { 1366 /* success */ 1367 ASSERT(zio == NULL || zio->io_error == 0); 1368 dbuf_set_data(db, buf); 1369 db->db_state = DB_CACHED; 1370 DTRACE_SET_STATE(db, "successful read"); 1371 } 1372 cv_broadcast(&db->db_changed); 1373 dbuf_rele_and_unlock(db, NULL, B_FALSE); 1374 } 1375 1376 /* 1377 * Shortcut for performing reads on bonus dbufs. Returns 1378 * an error if we fail to verify the dnode associated with 1379 * a decrypted block. Otherwise success. 1380 */ 1381 static int 1382 dbuf_read_bonus(dmu_buf_impl_t *db, dnode_t *dn, uint32_t flags) 1383 { 1384 int bonuslen, max_bonuslen, err; 1385 1386 err = dbuf_read_verify_dnode_crypt(db, flags); 1387 if (err) 1388 return (err); 1389 1390 bonuslen = MIN(dn->dn_bonuslen, dn->dn_phys->dn_bonuslen); 1391 max_bonuslen = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots); 1392 ASSERT(MUTEX_HELD(&db->db_mtx)); 1393 ASSERT(DB_DNODE_HELD(db)); 1394 ASSERT3U(bonuslen, <=, db->db.db_size); 1395 db->db.db_data = kmem_alloc(max_bonuslen, KM_SLEEP); 1396 arc_space_consume(max_bonuslen, ARC_SPACE_BONUS); 1397 if (bonuslen < max_bonuslen) 1398 memset(db->db.db_data, 0, max_bonuslen); 1399 if (bonuslen) 1400 memcpy(db->db.db_data, DN_BONUS(dn->dn_phys), bonuslen); 1401 db->db_state = DB_CACHED; 1402 DTRACE_SET_STATE(db, "bonus buffer filled"); 1403 return (0); 1404 } 1405 1406 static void 1407 dbuf_handle_indirect_hole(dmu_buf_impl_t *db, dnode_t *dn) 1408 { 1409 blkptr_t *bps = db->db.db_data; 1410 uint32_t indbs = 1ULL << dn->dn_indblkshift; 1411 int n_bps = indbs >> SPA_BLKPTRSHIFT; 1412 1413 for (int i = 0; i < n_bps; i++) { 1414 blkptr_t *bp = &bps[i]; 1415 1416 ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, indbs); 1417 BP_SET_LSIZE(bp, BP_GET_LEVEL(db->db_blkptr) == 1 ? 1418 dn->dn_datablksz : BP_GET_LSIZE(db->db_blkptr)); 1419 BP_SET_TYPE(bp, BP_GET_TYPE(db->db_blkptr)); 1420 BP_SET_LEVEL(bp, BP_GET_LEVEL(db->db_blkptr) - 1); 1421 BP_SET_BIRTH(bp, db->db_blkptr->blk_birth, 0); 1422 } 1423 } 1424 1425 /* 1426 * Handle reads on dbufs that are holes, if necessary. This function 1427 * requires that the dbuf's mutex is held. Returns success (0) if action 1428 * was taken, ENOENT if no action was taken. 1429 */ 1430 static int 1431 dbuf_read_hole(dmu_buf_impl_t *db, dnode_t *dn) 1432 { 1433 ASSERT(MUTEX_HELD(&db->db_mtx)); 1434 1435 int is_hole = db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr); 1436 /* 1437 * For level 0 blocks only, if the above check fails: 1438 * Recheck BP_IS_HOLE() after dnode_block_freed() in case dnode_sync() 1439 * processes the delete record and clears the bp while we are waiting 1440 * for the dn_mtx (resulting in a "no" from block_freed). 1441 */ 1442 if (!is_hole && db->db_level == 0) { 1443 is_hole = dnode_block_freed(dn, db->db_blkid) || 1444 BP_IS_HOLE(db->db_blkptr); 1445 } 1446 1447 if (is_hole) { 1448 dbuf_set_data(db, dbuf_alloc_arcbuf(db)); 1449 memset(db->db.db_data, 0, db->db.db_size); 1450 1451 if (db->db_blkptr != NULL && db->db_level > 0 && 1452 BP_IS_HOLE(db->db_blkptr) && 1453 db->db_blkptr->blk_birth != 0) { 1454 dbuf_handle_indirect_hole(db, dn); 1455 } 1456 db->db_state = DB_CACHED; 1457 DTRACE_SET_STATE(db, "hole read satisfied"); 1458 return (0); 1459 } 1460 return (ENOENT); 1461 } 1462 1463 /* 1464 * This function ensures that, when doing a decrypting read of a block, 1465 * we make sure we have decrypted the dnode associated with it. We must do 1466 * this so that we ensure we are fully authenticating the checksum-of-MACs 1467 * tree from the root of the objset down to this block. Indirect blocks are 1468 * always verified against their secure checksum-of-MACs assuming that the 1469 * dnode containing them is correct. Now that we are doing a decrypting read, 1470 * we can be sure that the key is loaded and verify that assumption. This is 1471 * especially important considering that we always read encrypted dnode 1472 * blocks as raw data (without verifying their MACs) to start, and 1473 * decrypt / authenticate them when we need to read an encrypted bonus buffer. 1474 */ 1475 static int 1476 dbuf_read_verify_dnode_crypt(dmu_buf_impl_t *db, uint32_t flags) 1477 { 1478 int err = 0; 1479 objset_t *os = db->db_objset; 1480 arc_buf_t *dnode_abuf; 1481 dnode_t *dn; 1482 zbookmark_phys_t zb; 1483 1484 ASSERT(MUTEX_HELD(&db->db_mtx)); 1485 1486 if (!os->os_encrypted || os->os_raw_receive || 1487 (flags & DB_RF_NO_DECRYPT) != 0) 1488 return (0); 1489 1490 DB_DNODE_ENTER(db); 1491 dn = DB_DNODE(db); 1492 dnode_abuf = (dn->dn_dbuf != NULL) ? dn->dn_dbuf->db_buf : NULL; 1493 1494 if (dnode_abuf == NULL || !arc_is_encrypted(dnode_abuf)) { 1495 DB_DNODE_EXIT(db); 1496 return (0); 1497 } 1498 1499 SET_BOOKMARK(&zb, dmu_objset_id(os), 1500 DMU_META_DNODE_OBJECT, 0, dn->dn_dbuf->db_blkid); 1501 err = arc_untransform(dnode_abuf, os->os_spa, &zb, B_TRUE); 1502 1503 /* 1504 * An error code of EACCES tells us that the key is still not 1505 * available. This is ok if we are only reading authenticated 1506 * (and therefore non-encrypted) blocks. 1507 */ 1508 if (err == EACCES && ((db->db_blkid != DMU_BONUS_BLKID && 1509 !DMU_OT_IS_ENCRYPTED(dn->dn_type)) || 1510 (db->db_blkid == DMU_BONUS_BLKID && 1511 !DMU_OT_IS_ENCRYPTED(dn->dn_bonustype)))) 1512 err = 0; 1513 1514 DB_DNODE_EXIT(db); 1515 1516 return (err); 1517 } 1518 1519 /* 1520 * Drops db_mtx and the parent lock specified by dblt and tag before 1521 * returning. 1522 */ 1523 static int 1524 dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags, 1525 db_lock_type_t dblt, const void *tag) 1526 { 1527 dnode_t *dn; 1528 zbookmark_phys_t zb; 1529 uint32_t aflags = ARC_FLAG_NOWAIT; 1530 int err, zio_flags; 1531 1532 err = zio_flags = 0; 1533 DB_DNODE_ENTER(db); 1534 dn = DB_DNODE(db); 1535 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); 1536 ASSERT(MUTEX_HELD(&db->db_mtx)); 1537 ASSERT(db->db_state == DB_UNCACHED); 1538 ASSERT(db->db_buf == NULL); 1539 ASSERT(db->db_parent == NULL || 1540 RW_LOCK_HELD(&db->db_parent->db_rwlock)); 1541 1542 if (db->db_blkid == DMU_BONUS_BLKID) { 1543 err = dbuf_read_bonus(db, dn, flags); 1544 goto early_unlock; 1545 } 1546 1547 err = dbuf_read_hole(db, dn); 1548 if (err == 0) 1549 goto early_unlock; 1550 1551 /* 1552 * Any attempt to read a redacted block should result in an error. This 1553 * will never happen under normal conditions, but can be useful for 1554 * debugging purposes. 1555 */ 1556 if (BP_IS_REDACTED(db->db_blkptr)) { 1557 ASSERT(dsl_dataset_feature_is_active( 1558 db->db_objset->os_dsl_dataset, 1559 SPA_FEATURE_REDACTED_DATASETS)); 1560 err = SET_ERROR(EIO); 1561 goto early_unlock; 1562 } 1563 1564 SET_BOOKMARK(&zb, dmu_objset_id(db->db_objset), 1565 db->db.db_object, db->db_level, db->db_blkid); 1566 1567 /* 1568 * All bps of an encrypted os should have the encryption bit set. 1569 * If this is not true it indicates tampering and we report an error. 1570 */ 1571 if (db->db_objset->os_encrypted && !BP_USES_CRYPT(db->db_blkptr)) { 1572 spa_log_error(db->db_objset->os_spa, &zb); 1573 zfs_panic_recover("unencrypted block in encrypted " 1574 "object set %llu", dmu_objset_id(db->db_objset)); 1575 err = SET_ERROR(EIO); 1576 goto early_unlock; 1577 } 1578 1579 err = dbuf_read_verify_dnode_crypt(db, flags); 1580 if (err != 0) 1581 goto early_unlock; 1582 1583 DB_DNODE_EXIT(db); 1584 1585 db->db_state = DB_READ; 1586 DTRACE_SET_STATE(db, "read issued"); 1587 mutex_exit(&db->db_mtx); 1588 1589 if (dbuf_is_l2cacheable(db)) 1590 aflags |= ARC_FLAG_L2CACHE; 1591 1592 dbuf_add_ref(db, NULL); 1593 1594 zio_flags = (flags & DB_RF_CANFAIL) ? 1595 ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED; 1596 1597 if ((flags & DB_RF_NO_DECRYPT) && BP_IS_PROTECTED(db->db_blkptr)) 1598 zio_flags |= ZIO_FLAG_RAW; 1599 /* 1600 * The zio layer will copy the provided blkptr later, but we need to 1601 * do this now so that we can release the parent's rwlock. We have to 1602 * do that now so that if dbuf_read_done is called synchronously (on 1603 * an l1 cache hit) we don't acquire the db_mtx while holding the 1604 * parent's rwlock, which would be a lock ordering violation. 1605 */ 1606 blkptr_t bp = *db->db_blkptr; 1607 dmu_buf_unlock_parent(db, dblt, tag); 1608 (void) arc_read(zio, db->db_objset->os_spa, &bp, 1609 dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ, zio_flags, 1610 &aflags, &zb); 1611 return (err); 1612 early_unlock: 1613 DB_DNODE_EXIT(db); 1614 mutex_exit(&db->db_mtx); 1615 dmu_buf_unlock_parent(db, dblt, tag); 1616 return (err); 1617 } 1618 1619 /* 1620 * This is our just-in-time copy function. It makes a copy of buffers that 1621 * have been modified in a previous transaction group before we access them in 1622 * the current active group. 1623 * 1624 * This function is used in three places: when we are dirtying a buffer for the 1625 * first time in a txg, when we are freeing a range in a dnode that includes 1626 * this buffer, and when we are accessing a buffer which was received compressed 1627 * and later referenced in a WRITE_BYREF record. 1628 * 1629 * Note that when we are called from dbuf_free_range() we do not put a hold on 1630 * the buffer, we just traverse the active dbuf list for the dnode. 1631 */ 1632 static void 1633 dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg) 1634 { 1635 dbuf_dirty_record_t *dr = list_head(&db->db_dirty_records); 1636 1637 ASSERT(MUTEX_HELD(&db->db_mtx)); 1638 ASSERT(db->db.db_data != NULL); 1639 ASSERT(db->db_level == 0); 1640 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT); 1641 1642 if (dr == NULL || 1643 (dr->dt.dl.dr_data != 1644 ((db->db_blkid == DMU_BONUS_BLKID) ? db->db.db_data : db->db_buf))) 1645 return; 1646 1647 /* 1648 * If the last dirty record for this dbuf has not yet synced 1649 * and its referencing the dbuf data, either: 1650 * reset the reference to point to a new copy, 1651 * or (if there a no active holders) 1652 * just null out the current db_data pointer. 1653 */ 1654 ASSERT3U(dr->dr_txg, >=, txg - 2); 1655 if (db->db_blkid == DMU_BONUS_BLKID) { 1656 dnode_t *dn = DB_DNODE(db); 1657 int bonuslen = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots); 1658 dr->dt.dl.dr_data = kmem_alloc(bonuslen, KM_SLEEP); 1659 arc_space_consume(bonuslen, ARC_SPACE_BONUS); 1660 memcpy(dr->dt.dl.dr_data, db->db.db_data, bonuslen); 1661 } else if (zfs_refcount_count(&db->db_holds) > db->db_dirtycnt) { 1662 dnode_t *dn = DB_DNODE(db); 1663 int size = arc_buf_size(db->db_buf); 1664 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 1665 spa_t *spa = db->db_objset->os_spa; 1666 enum zio_compress compress_type = 1667 arc_get_compression(db->db_buf); 1668 uint8_t complevel = arc_get_complevel(db->db_buf); 1669 1670 if (arc_is_encrypted(db->db_buf)) { 1671 boolean_t byteorder; 1672 uint8_t salt[ZIO_DATA_SALT_LEN]; 1673 uint8_t iv[ZIO_DATA_IV_LEN]; 1674 uint8_t mac[ZIO_DATA_MAC_LEN]; 1675 1676 arc_get_raw_params(db->db_buf, &byteorder, salt, 1677 iv, mac); 1678 dr->dt.dl.dr_data = arc_alloc_raw_buf(spa, db, 1679 dmu_objset_id(dn->dn_objset), byteorder, salt, iv, 1680 mac, dn->dn_type, size, arc_buf_lsize(db->db_buf), 1681 compress_type, complevel); 1682 } else if (compress_type != ZIO_COMPRESS_OFF) { 1683 ASSERT3U(type, ==, ARC_BUFC_DATA); 1684 dr->dt.dl.dr_data = arc_alloc_compressed_buf(spa, db, 1685 size, arc_buf_lsize(db->db_buf), compress_type, 1686 complevel); 1687 } else { 1688 dr->dt.dl.dr_data = arc_alloc_buf(spa, db, type, size); 1689 } 1690 memcpy(dr->dt.dl.dr_data->b_data, db->db.db_data, size); 1691 } else { 1692 db->db_buf = NULL; 1693 dbuf_clear_data(db); 1694 } 1695 } 1696 1697 int 1698 dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags) 1699 { 1700 int err = 0; 1701 boolean_t prefetch; 1702 dnode_t *dn; 1703 1704 /* 1705 * We don't have to hold the mutex to check db_state because it 1706 * can't be freed while we have a hold on the buffer. 1707 */ 1708 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); 1709 1710 if (db->db_state == DB_NOFILL) 1711 return (SET_ERROR(EIO)); 1712 1713 DB_DNODE_ENTER(db); 1714 dn = DB_DNODE(db); 1715 1716 prefetch = db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 1717 (flags & DB_RF_NOPREFETCH) == 0 && dn != NULL && 1718 DBUF_IS_CACHEABLE(db); 1719 1720 mutex_enter(&db->db_mtx); 1721 if (db->db_state == DB_CACHED) { 1722 spa_t *spa = dn->dn_objset->os_spa; 1723 1724 /* 1725 * Ensure that this block's dnode has been decrypted if 1726 * the caller has requested decrypted data. 1727 */ 1728 err = dbuf_read_verify_dnode_crypt(db, flags); 1729 1730 /* 1731 * If the arc buf is compressed or encrypted and the caller 1732 * requested uncompressed data, we need to untransform it 1733 * before returning. We also call arc_untransform() on any 1734 * unauthenticated blocks, which will verify their MAC if 1735 * the key is now available. 1736 */ 1737 if (err == 0 && db->db_buf != NULL && 1738 (flags & DB_RF_NO_DECRYPT) == 0 && 1739 (arc_is_encrypted(db->db_buf) || 1740 arc_is_unauthenticated(db->db_buf) || 1741 arc_get_compression(db->db_buf) != ZIO_COMPRESS_OFF)) { 1742 zbookmark_phys_t zb; 1743 1744 SET_BOOKMARK(&zb, dmu_objset_id(db->db_objset), 1745 db->db.db_object, db->db_level, db->db_blkid); 1746 dbuf_fix_old_data(db, spa_syncing_txg(spa)); 1747 err = arc_untransform(db->db_buf, spa, &zb, B_FALSE); 1748 dbuf_set_data(db, db->db_buf); 1749 } 1750 mutex_exit(&db->db_mtx); 1751 if (err == 0 && prefetch) { 1752 dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1, B_TRUE, 1753 B_FALSE, flags & DB_RF_HAVESTRUCT); 1754 } 1755 DB_DNODE_EXIT(db); 1756 DBUF_STAT_BUMP(hash_hits); 1757 } else if (db->db_state == DB_UNCACHED) { 1758 spa_t *spa = dn->dn_objset->os_spa; 1759 boolean_t need_wait = B_FALSE; 1760 1761 db_lock_type_t dblt = dmu_buf_lock_parent(db, RW_READER, FTAG); 1762 1763 if (zio == NULL && 1764 db->db_blkptr != NULL && !BP_IS_HOLE(db->db_blkptr)) { 1765 zio = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL); 1766 need_wait = B_TRUE; 1767 } 1768 err = dbuf_read_impl(db, zio, flags, dblt, FTAG); 1769 /* 1770 * dbuf_read_impl has dropped db_mtx and our parent's rwlock 1771 * for us 1772 */ 1773 if (!err && prefetch) { 1774 dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1, B_TRUE, 1775 db->db_state != DB_CACHED, 1776 flags & DB_RF_HAVESTRUCT); 1777 } 1778 1779 DB_DNODE_EXIT(db); 1780 DBUF_STAT_BUMP(hash_misses); 1781 1782 /* 1783 * If we created a zio_root we must execute it to avoid 1784 * leaking it, even if it isn't attached to any work due 1785 * to an error in dbuf_read_impl(). 1786 */ 1787 if (need_wait) { 1788 if (err == 0) 1789 err = zio_wait(zio); 1790 else 1791 VERIFY0(zio_wait(zio)); 1792 } 1793 } else { 1794 /* 1795 * Another reader came in while the dbuf was in flight 1796 * between UNCACHED and CACHED. Either a writer will finish 1797 * writing the buffer (sending the dbuf to CACHED) or the 1798 * first reader's request will reach the read_done callback 1799 * and send the dbuf to CACHED. Otherwise, a failure 1800 * occurred and the dbuf went to UNCACHED. 1801 */ 1802 mutex_exit(&db->db_mtx); 1803 if (prefetch) { 1804 dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1, B_TRUE, 1805 B_TRUE, flags & DB_RF_HAVESTRUCT); 1806 } 1807 DB_DNODE_EXIT(db); 1808 DBUF_STAT_BUMP(hash_misses); 1809 1810 /* Skip the wait per the caller's request. */ 1811 if ((flags & DB_RF_NEVERWAIT) == 0) { 1812 mutex_enter(&db->db_mtx); 1813 while (db->db_state == DB_READ || 1814 db->db_state == DB_FILL) { 1815 ASSERT(db->db_state == DB_READ || 1816 (flags & DB_RF_HAVESTRUCT) == 0); 1817 DTRACE_PROBE2(blocked__read, dmu_buf_impl_t *, 1818 db, zio_t *, zio); 1819 cv_wait(&db->db_changed, &db->db_mtx); 1820 } 1821 if (db->db_state == DB_UNCACHED) 1822 err = SET_ERROR(EIO); 1823 mutex_exit(&db->db_mtx); 1824 } 1825 } 1826 1827 return (err); 1828 } 1829 1830 static void 1831 dbuf_noread(dmu_buf_impl_t *db) 1832 { 1833 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); 1834 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1835 mutex_enter(&db->db_mtx); 1836 while (db->db_state == DB_READ || db->db_state == DB_FILL) 1837 cv_wait(&db->db_changed, &db->db_mtx); 1838 if (db->db_state == DB_UNCACHED) { 1839 ASSERT(db->db_buf == NULL); 1840 ASSERT(db->db.db_data == NULL); 1841 dbuf_set_data(db, dbuf_alloc_arcbuf(db)); 1842 db->db_state = DB_FILL; 1843 DTRACE_SET_STATE(db, "assigning filled buffer"); 1844 } else if (db->db_state == DB_NOFILL) { 1845 dbuf_clear_data(db); 1846 } else { 1847 ASSERT3U(db->db_state, ==, DB_CACHED); 1848 } 1849 mutex_exit(&db->db_mtx); 1850 } 1851 1852 void 1853 dbuf_unoverride(dbuf_dirty_record_t *dr) 1854 { 1855 dmu_buf_impl_t *db = dr->dr_dbuf; 1856 blkptr_t *bp = &dr->dt.dl.dr_overridden_by; 1857 uint64_t txg = dr->dr_txg; 1858 1859 ASSERT(MUTEX_HELD(&db->db_mtx)); 1860 /* 1861 * This assert is valid because dmu_sync() expects to be called by 1862 * a zilog's get_data while holding a range lock. This call only 1863 * comes from dbuf_dirty() callers who must also hold a range lock. 1864 */ 1865 ASSERT(dr->dt.dl.dr_override_state != DR_IN_DMU_SYNC); 1866 ASSERT(db->db_level == 0); 1867 1868 if (db->db_blkid == DMU_BONUS_BLKID || 1869 dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN) 1870 return; 1871 1872 ASSERT(db->db_data_pending != dr); 1873 1874 /* free this block */ 1875 if (!BP_IS_HOLE(bp) && !dr->dt.dl.dr_nopwrite) 1876 zio_free(db->db_objset->os_spa, txg, bp); 1877 1878 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 1879 dr->dt.dl.dr_nopwrite = B_FALSE; 1880 dr->dt.dl.dr_has_raw_params = B_FALSE; 1881 1882 /* 1883 * Release the already-written buffer, so we leave it in 1884 * a consistent dirty state. Note that all callers are 1885 * modifying the buffer, so they will immediately do 1886 * another (redundant) arc_release(). Therefore, leave 1887 * the buf thawed to save the effort of freezing & 1888 * immediately re-thawing it. 1889 */ 1890 arc_release(dr->dt.dl.dr_data, db); 1891 } 1892 1893 /* 1894 * Evict (if its unreferenced) or clear (if its referenced) any level-0 1895 * data blocks in the free range, so that any future readers will find 1896 * empty blocks. 1897 */ 1898 void 1899 dbuf_free_range(dnode_t *dn, uint64_t start_blkid, uint64_t end_blkid, 1900 dmu_tx_t *tx) 1901 { 1902 dmu_buf_impl_t *db_search; 1903 dmu_buf_impl_t *db, *db_next; 1904 uint64_t txg = tx->tx_txg; 1905 avl_index_t where; 1906 dbuf_dirty_record_t *dr; 1907 1908 if (end_blkid > dn->dn_maxblkid && 1909 !(start_blkid == DMU_SPILL_BLKID || end_blkid == DMU_SPILL_BLKID)) 1910 end_blkid = dn->dn_maxblkid; 1911 dprintf_dnode(dn, "start=%llu end=%llu\n", (u_longlong_t)start_blkid, 1912 (u_longlong_t)end_blkid); 1913 1914 db_search = kmem_alloc(sizeof (dmu_buf_impl_t), KM_SLEEP); 1915 db_search->db_level = 0; 1916 db_search->db_blkid = start_blkid; 1917 db_search->db_state = DB_SEARCH; 1918 1919 mutex_enter(&dn->dn_dbufs_mtx); 1920 db = avl_find(&dn->dn_dbufs, db_search, &where); 1921 ASSERT3P(db, ==, NULL); 1922 1923 db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER); 1924 1925 for (; db != NULL; db = db_next) { 1926 db_next = AVL_NEXT(&dn->dn_dbufs, db); 1927 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1928 1929 if (db->db_level != 0 || db->db_blkid > end_blkid) { 1930 break; 1931 } 1932 ASSERT3U(db->db_blkid, >=, start_blkid); 1933 1934 /* found a level 0 buffer in the range */ 1935 mutex_enter(&db->db_mtx); 1936 if (dbuf_undirty(db, tx)) { 1937 /* mutex has been dropped and dbuf destroyed */ 1938 continue; 1939 } 1940 1941 if (db->db_state == DB_UNCACHED || 1942 db->db_state == DB_NOFILL || 1943 db->db_state == DB_EVICTING) { 1944 ASSERT(db->db.db_data == NULL); 1945 mutex_exit(&db->db_mtx); 1946 continue; 1947 } 1948 if (db->db_state == DB_READ || db->db_state == DB_FILL) { 1949 /* will be handled in dbuf_read_done or dbuf_rele */ 1950 db->db_freed_in_flight = TRUE; 1951 mutex_exit(&db->db_mtx); 1952 continue; 1953 } 1954 if (zfs_refcount_count(&db->db_holds) == 0) { 1955 ASSERT(db->db_buf); 1956 dbuf_destroy(db); 1957 continue; 1958 } 1959 /* The dbuf is referenced */ 1960 1961 dr = list_head(&db->db_dirty_records); 1962 if (dr != NULL) { 1963 if (dr->dr_txg == txg) { 1964 /* 1965 * This buffer is "in-use", re-adjust the file 1966 * size to reflect that this buffer may 1967 * contain new data when we sync. 1968 */ 1969 if (db->db_blkid != DMU_SPILL_BLKID && 1970 db->db_blkid > dn->dn_maxblkid) 1971 dn->dn_maxblkid = db->db_blkid; 1972 dbuf_unoverride(dr); 1973 } else { 1974 /* 1975 * This dbuf is not dirty in the open context. 1976 * Either uncache it (if its not referenced in 1977 * the open context) or reset its contents to 1978 * empty. 1979 */ 1980 dbuf_fix_old_data(db, txg); 1981 } 1982 } 1983 /* clear the contents if its cached */ 1984 if (db->db_state == DB_CACHED) { 1985 ASSERT(db->db.db_data != NULL); 1986 arc_release(db->db_buf, db); 1987 rw_enter(&db->db_rwlock, RW_WRITER); 1988 memset(db->db.db_data, 0, db->db.db_size); 1989 rw_exit(&db->db_rwlock); 1990 arc_buf_freeze(db->db_buf); 1991 } 1992 1993 mutex_exit(&db->db_mtx); 1994 } 1995 1996 mutex_exit(&dn->dn_dbufs_mtx); 1997 kmem_free(db_search, sizeof (dmu_buf_impl_t)); 1998 } 1999 2000 void 2001 dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx) 2002 { 2003 arc_buf_t *buf, *old_buf; 2004 dbuf_dirty_record_t *dr; 2005 int osize = db->db.db_size; 2006 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 2007 dnode_t *dn; 2008 2009 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 2010 2011 DB_DNODE_ENTER(db); 2012 dn = DB_DNODE(db); 2013 2014 /* 2015 * XXX we should be doing a dbuf_read, checking the return 2016 * value and returning that up to our callers 2017 */ 2018 dmu_buf_will_dirty(&db->db, tx); 2019 2020 /* create the data buffer for the new block */ 2021 buf = arc_alloc_buf(dn->dn_objset->os_spa, db, type, size); 2022 2023 /* copy old block data to the new block */ 2024 old_buf = db->db_buf; 2025 memcpy(buf->b_data, old_buf->b_data, MIN(osize, size)); 2026 /* zero the remainder */ 2027 if (size > osize) 2028 memset((uint8_t *)buf->b_data + osize, 0, size - osize); 2029 2030 mutex_enter(&db->db_mtx); 2031 dbuf_set_data(db, buf); 2032 arc_buf_destroy(old_buf, db); 2033 db->db.db_size = size; 2034 2035 dr = list_head(&db->db_dirty_records); 2036 /* dirty record added by dmu_buf_will_dirty() */ 2037 VERIFY(dr != NULL); 2038 if (db->db_level == 0) 2039 dr->dt.dl.dr_data = buf; 2040 ASSERT3U(dr->dr_txg, ==, tx->tx_txg); 2041 ASSERT3U(dr->dr_accounted, ==, osize); 2042 dr->dr_accounted = size; 2043 mutex_exit(&db->db_mtx); 2044 2045 dmu_objset_willuse_space(dn->dn_objset, size - osize, tx); 2046 DB_DNODE_EXIT(db); 2047 } 2048 2049 void 2050 dbuf_release_bp(dmu_buf_impl_t *db) 2051 { 2052 objset_t *os __maybe_unused = db->db_objset; 2053 2054 ASSERT(dsl_pool_sync_context(dmu_objset_pool(os))); 2055 ASSERT(arc_released(os->os_phys_buf) || 2056 list_link_active(&os->os_dsl_dataset->ds_synced_link)); 2057 ASSERT(db->db_parent == NULL || arc_released(db->db_parent->db_buf)); 2058 2059 (void) arc_release(db->db_buf, db); 2060 } 2061 2062 /* 2063 * We already have a dirty record for this TXG, and we are being 2064 * dirtied again. 2065 */ 2066 static void 2067 dbuf_redirty(dbuf_dirty_record_t *dr) 2068 { 2069 dmu_buf_impl_t *db = dr->dr_dbuf; 2070 2071 ASSERT(MUTEX_HELD(&db->db_mtx)); 2072 2073 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID) { 2074 /* 2075 * If this buffer has already been written out, 2076 * we now need to reset its state. 2077 */ 2078 dbuf_unoverride(dr); 2079 if (db->db.db_object != DMU_META_DNODE_OBJECT && 2080 db->db_state != DB_NOFILL) { 2081 /* Already released on initial dirty, so just thaw. */ 2082 ASSERT(arc_released(db->db_buf)); 2083 arc_buf_thaw(db->db_buf); 2084 } 2085 } 2086 } 2087 2088 dbuf_dirty_record_t * 2089 dbuf_dirty_lightweight(dnode_t *dn, uint64_t blkid, dmu_tx_t *tx) 2090 { 2091 rw_enter(&dn->dn_struct_rwlock, RW_READER); 2092 IMPLY(dn->dn_objset->os_raw_receive, dn->dn_maxblkid >= blkid); 2093 dnode_new_blkid(dn, blkid, tx, B_TRUE, B_FALSE); 2094 ASSERT(dn->dn_maxblkid >= blkid); 2095 2096 dbuf_dirty_record_t *dr = kmem_zalloc(sizeof (*dr), KM_SLEEP); 2097 list_link_init(&dr->dr_dirty_node); 2098 list_link_init(&dr->dr_dbuf_node); 2099 dr->dr_dnode = dn; 2100 dr->dr_txg = tx->tx_txg; 2101 dr->dt.dll.dr_blkid = blkid; 2102 dr->dr_accounted = dn->dn_datablksz; 2103 2104 /* 2105 * There should not be any dbuf for the block that we're dirtying. 2106 * Otherwise the buffer contents could be inconsistent between the 2107 * dbuf and the lightweight dirty record. 2108 */ 2109 ASSERT3P(NULL, ==, dbuf_find(dn->dn_objset, dn->dn_object, 0, blkid)); 2110 2111 mutex_enter(&dn->dn_mtx); 2112 int txgoff = tx->tx_txg & TXG_MASK; 2113 if (dn->dn_free_ranges[txgoff] != NULL) { 2114 range_tree_clear(dn->dn_free_ranges[txgoff], blkid, 1); 2115 } 2116 2117 if (dn->dn_nlevels == 1) { 2118 ASSERT3U(blkid, <, dn->dn_nblkptr); 2119 list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 2120 mutex_exit(&dn->dn_mtx); 2121 rw_exit(&dn->dn_struct_rwlock); 2122 dnode_setdirty(dn, tx); 2123 } else { 2124 mutex_exit(&dn->dn_mtx); 2125 2126 int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 2127 dmu_buf_impl_t *parent_db = dbuf_hold_level(dn, 2128 1, blkid >> epbs, FTAG); 2129 rw_exit(&dn->dn_struct_rwlock); 2130 if (parent_db == NULL) { 2131 kmem_free(dr, sizeof (*dr)); 2132 return (NULL); 2133 } 2134 int err = dbuf_read(parent_db, NULL, 2135 (DB_RF_NOPREFETCH | DB_RF_CANFAIL)); 2136 if (err != 0) { 2137 dbuf_rele(parent_db, FTAG); 2138 kmem_free(dr, sizeof (*dr)); 2139 return (NULL); 2140 } 2141 2142 dbuf_dirty_record_t *parent_dr = dbuf_dirty(parent_db, tx); 2143 dbuf_rele(parent_db, FTAG); 2144 mutex_enter(&parent_dr->dt.di.dr_mtx); 2145 ASSERT3U(parent_dr->dr_txg, ==, tx->tx_txg); 2146 list_insert_tail(&parent_dr->dt.di.dr_children, dr); 2147 mutex_exit(&parent_dr->dt.di.dr_mtx); 2148 dr->dr_parent = parent_dr; 2149 } 2150 2151 dmu_objset_willuse_space(dn->dn_objset, dr->dr_accounted, tx); 2152 2153 return (dr); 2154 } 2155 2156 dbuf_dirty_record_t * 2157 dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 2158 { 2159 dnode_t *dn; 2160 objset_t *os; 2161 dbuf_dirty_record_t *dr, *dr_next, *dr_head; 2162 int txgoff = tx->tx_txg & TXG_MASK; 2163 boolean_t drop_struct_rwlock = B_FALSE; 2164 2165 ASSERT(tx->tx_txg != 0); 2166 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); 2167 DMU_TX_DIRTY_BUF(tx, db); 2168 2169 DB_DNODE_ENTER(db); 2170 dn = DB_DNODE(db); 2171 /* 2172 * Shouldn't dirty a regular buffer in syncing context. Private 2173 * objects may be dirtied in syncing context, but only if they 2174 * were already pre-dirtied in open context. 2175 */ 2176 #ifdef ZFS_DEBUG 2177 if (dn->dn_objset->os_dsl_dataset != NULL) { 2178 rrw_enter(&dn->dn_objset->os_dsl_dataset->ds_bp_rwlock, 2179 RW_READER, FTAG); 2180 } 2181 ASSERT(!dmu_tx_is_syncing(tx) || 2182 BP_IS_HOLE(dn->dn_objset->os_rootbp) || 2183 DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 2184 dn->dn_objset->os_dsl_dataset == NULL); 2185 if (dn->dn_objset->os_dsl_dataset != NULL) 2186 rrw_exit(&dn->dn_objset->os_dsl_dataset->ds_bp_rwlock, FTAG); 2187 #endif 2188 /* 2189 * We make this assert for private objects as well, but after we 2190 * check if we're already dirty. They are allowed to re-dirty 2191 * in syncing context. 2192 */ 2193 ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 2194 dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 2195 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 2196 2197 mutex_enter(&db->db_mtx); 2198 /* 2199 * XXX make this true for indirects too? The problem is that 2200 * transactions created with dmu_tx_create_assigned() from 2201 * syncing context don't bother holding ahead. 2202 */ 2203 ASSERT(db->db_level != 0 || 2204 db->db_state == DB_CACHED || db->db_state == DB_FILL || 2205 db->db_state == DB_NOFILL); 2206 2207 mutex_enter(&dn->dn_mtx); 2208 dnode_set_dirtyctx(dn, tx, db); 2209 if (tx->tx_txg > dn->dn_dirty_txg) 2210 dn->dn_dirty_txg = tx->tx_txg; 2211 mutex_exit(&dn->dn_mtx); 2212 2213 if (db->db_blkid == DMU_SPILL_BLKID) 2214 dn->dn_have_spill = B_TRUE; 2215 2216 /* 2217 * If this buffer is already dirty, we're done. 2218 */ 2219 dr_head = list_head(&db->db_dirty_records); 2220 ASSERT(dr_head == NULL || dr_head->dr_txg <= tx->tx_txg || 2221 db->db.db_object == DMU_META_DNODE_OBJECT); 2222 dr_next = dbuf_find_dirty_lte(db, tx->tx_txg); 2223 if (dr_next && dr_next->dr_txg == tx->tx_txg) { 2224 DB_DNODE_EXIT(db); 2225 2226 dbuf_redirty(dr_next); 2227 mutex_exit(&db->db_mtx); 2228 return (dr_next); 2229 } 2230 2231 /* 2232 * Only valid if not already dirty. 2233 */ 2234 ASSERT(dn->dn_object == 0 || 2235 dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 2236 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 2237 2238 ASSERT3U(dn->dn_nlevels, >, db->db_level); 2239 2240 /* 2241 * We should only be dirtying in syncing context if it's the 2242 * mos or we're initializing the os or it's a special object. 2243 * However, we are allowed to dirty in syncing context provided 2244 * we already dirtied it in open context. Hence we must make 2245 * this assertion only if we're not already dirty. 2246 */ 2247 os = dn->dn_objset; 2248 VERIFY3U(tx->tx_txg, <=, spa_final_dirty_txg(os->os_spa)); 2249 #ifdef ZFS_DEBUG 2250 if (dn->dn_objset->os_dsl_dataset != NULL) 2251 rrw_enter(&os->os_dsl_dataset->ds_bp_rwlock, RW_READER, FTAG); 2252 ASSERT(!dmu_tx_is_syncing(tx) || DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 2253 os->os_dsl_dataset == NULL || BP_IS_HOLE(os->os_rootbp)); 2254 if (dn->dn_objset->os_dsl_dataset != NULL) 2255 rrw_exit(&os->os_dsl_dataset->ds_bp_rwlock, FTAG); 2256 #endif 2257 ASSERT(db->db.db_size != 0); 2258 2259 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 2260 2261 if (db->db_blkid != DMU_BONUS_BLKID) { 2262 dmu_objset_willuse_space(os, db->db.db_size, tx); 2263 } 2264 2265 /* 2266 * If this buffer is dirty in an old transaction group we need 2267 * to make a copy of it so that the changes we make in this 2268 * transaction group won't leak out when we sync the older txg. 2269 */ 2270 dr = kmem_zalloc(sizeof (dbuf_dirty_record_t), KM_SLEEP); 2271 list_link_init(&dr->dr_dirty_node); 2272 list_link_init(&dr->dr_dbuf_node); 2273 dr->dr_dnode = dn; 2274 if (db->db_level == 0) { 2275 void *data_old = db->db_buf; 2276 2277 if (db->db_state != DB_NOFILL) { 2278 if (db->db_blkid == DMU_BONUS_BLKID) { 2279 dbuf_fix_old_data(db, tx->tx_txg); 2280 data_old = db->db.db_data; 2281 } else if (db->db.db_object != DMU_META_DNODE_OBJECT) { 2282 /* 2283 * Release the data buffer from the cache so 2284 * that we can modify it without impacting 2285 * possible other users of this cached data 2286 * block. Note that indirect blocks and 2287 * private objects are not released until the 2288 * syncing state (since they are only modified 2289 * then). 2290 */ 2291 arc_release(db->db_buf, db); 2292 dbuf_fix_old_data(db, tx->tx_txg); 2293 data_old = db->db_buf; 2294 } 2295 ASSERT(data_old != NULL); 2296 } 2297 dr->dt.dl.dr_data = data_old; 2298 } else { 2299 mutex_init(&dr->dt.di.dr_mtx, NULL, MUTEX_NOLOCKDEP, NULL); 2300 list_create(&dr->dt.di.dr_children, 2301 sizeof (dbuf_dirty_record_t), 2302 offsetof(dbuf_dirty_record_t, dr_dirty_node)); 2303 } 2304 if (db->db_blkid != DMU_BONUS_BLKID) 2305 dr->dr_accounted = db->db.db_size; 2306 dr->dr_dbuf = db; 2307 dr->dr_txg = tx->tx_txg; 2308 list_insert_before(&db->db_dirty_records, dr_next, dr); 2309 2310 /* 2311 * We could have been freed_in_flight between the dbuf_noread 2312 * and dbuf_dirty. We win, as though the dbuf_noread() had 2313 * happened after the free. 2314 */ 2315 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 2316 db->db_blkid != DMU_SPILL_BLKID) { 2317 mutex_enter(&dn->dn_mtx); 2318 if (dn->dn_free_ranges[txgoff] != NULL) { 2319 range_tree_clear(dn->dn_free_ranges[txgoff], 2320 db->db_blkid, 1); 2321 } 2322 mutex_exit(&dn->dn_mtx); 2323 db->db_freed_in_flight = FALSE; 2324 } 2325 2326 /* 2327 * This buffer is now part of this txg 2328 */ 2329 dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg); 2330 db->db_dirtycnt += 1; 2331 ASSERT3U(db->db_dirtycnt, <=, 3); 2332 2333 mutex_exit(&db->db_mtx); 2334 2335 if (db->db_blkid == DMU_BONUS_BLKID || 2336 db->db_blkid == DMU_SPILL_BLKID) { 2337 mutex_enter(&dn->dn_mtx); 2338 ASSERT(!list_link_active(&dr->dr_dirty_node)); 2339 list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 2340 mutex_exit(&dn->dn_mtx); 2341 dnode_setdirty(dn, tx); 2342 DB_DNODE_EXIT(db); 2343 return (dr); 2344 } 2345 2346 if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 2347 rw_enter(&dn->dn_struct_rwlock, RW_READER); 2348 drop_struct_rwlock = B_TRUE; 2349 } 2350 2351 /* 2352 * If we are overwriting a dedup BP, then unless it is snapshotted, 2353 * when we get to syncing context we will need to decrement its 2354 * refcount in the DDT. Prefetch the relevant DDT block so that 2355 * syncing context won't have to wait for the i/o. 2356 */ 2357 if (db->db_blkptr != NULL) { 2358 db_lock_type_t dblt = dmu_buf_lock_parent(db, RW_READER, FTAG); 2359 ddt_prefetch(os->os_spa, db->db_blkptr); 2360 dmu_buf_unlock_parent(db, dblt, FTAG); 2361 } 2362 2363 /* 2364 * We need to hold the dn_struct_rwlock to make this assertion, 2365 * because it protects dn_phys / dn_next_nlevels from changing. 2366 */ 2367 ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) || 2368 dn->dn_phys->dn_nlevels > db->db_level || 2369 dn->dn_next_nlevels[txgoff] > db->db_level || 2370 dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level || 2371 dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level); 2372 2373 2374 if (db->db_level == 0) { 2375 ASSERT(!db->db_objset->os_raw_receive || 2376 dn->dn_maxblkid >= db->db_blkid); 2377 dnode_new_blkid(dn, db->db_blkid, tx, 2378 drop_struct_rwlock, B_FALSE); 2379 ASSERT(dn->dn_maxblkid >= db->db_blkid); 2380 } 2381 2382 if (db->db_level+1 < dn->dn_nlevels) { 2383 dmu_buf_impl_t *parent = db->db_parent; 2384 dbuf_dirty_record_t *di; 2385 int parent_held = FALSE; 2386 2387 if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) { 2388 int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 2389 parent = dbuf_hold_level(dn, db->db_level + 1, 2390 db->db_blkid >> epbs, FTAG); 2391 ASSERT(parent != NULL); 2392 parent_held = TRUE; 2393 } 2394 if (drop_struct_rwlock) 2395 rw_exit(&dn->dn_struct_rwlock); 2396 ASSERT3U(db->db_level + 1, ==, parent->db_level); 2397 di = dbuf_dirty(parent, tx); 2398 if (parent_held) 2399 dbuf_rele(parent, FTAG); 2400 2401 mutex_enter(&db->db_mtx); 2402 /* 2403 * Since we've dropped the mutex, it's possible that 2404 * dbuf_undirty() might have changed this out from under us. 2405 */ 2406 if (list_head(&db->db_dirty_records) == dr || 2407 dn->dn_object == DMU_META_DNODE_OBJECT) { 2408 mutex_enter(&di->dt.di.dr_mtx); 2409 ASSERT3U(di->dr_txg, ==, tx->tx_txg); 2410 ASSERT(!list_link_active(&dr->dr_dirty_node)); 2411 list_insert_tail(&di->dt.di.dr_children, dr); 2412 mutex_exit(&di->dt.di.dr_mtx); 2413 dr->dr_parent = di; 2414 } 2415 mutex_exit(&db->db_mtx); 2416 } else { 2417 ASSERT(db->db_level + 1 == dn->dn_nlevels); 2418 ASSERT(db->db_blkid < dn->dn_nblkptr); 2419 ASSERT(db->db_parent == NULL || db->db_parent == dn->dn_dbuf); 2420 mutex_enter(&dn->dn_mtx); 2421 ASSERT(!list_link_active(&dr->dr_dirty_node)); 2422 list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 2423 mutex_exit(&dn->dn_mtx); 2424 if (drop_struct_rwlock) 2425 rw_exit(&dn->dn_struct_rwlock); 2426 } 2427 2428 dnode_setdirty(dn, tx); 2429 DB_DNODE_EXIT(db); 2430 return (dr); 2431 } 2432 2433 static void 2434 dbuf_undirty_bonus(dbuf_dirty_record_t *dr) 2435 { 2436 dmu_buf_impl_t *db = dr->dr_dbuf; 2437 2438 if (dr->dt.dl.dr_data != db->db.db_data) { 2439 struct dnode *dn = dr->dr_dnode; 2440 int max_bonuslen = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots); 2441 2442 kmem_free(dr->dt.dl.dr_data, max_bonuslen); 2443 arc_space_return(max_bonuslen, ARC_SPACE_BONUS); 2444 } 2445 db->db_data_pending = NULL; 2446 ASSERT(list_next(&db->db_dirty_records, dr) == NULL); 2447 list_remove(&db->db_dirty_records, dr); 2448 if (dr->dr_dbuf->db_level != 0) { 2449 mutex_destroy(&dr->dt.di.dr_mtx); 2450 list_destroy(&dr->dt.di.dr_children); 2451 } 2452 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 2453 ASSERT3U(db->db_dirtycnt, >, 0); 2454 db->db_dirtycnt -= 1; 2455 } 2456 2457 /* 2458 * Undirty a buffer in the transaction group referenced by the given 2459 * transaction. Return whether this evicted the dbuf. 2460 */ 2461 static boolean_t 2462 dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 2463 { 2464 uint64_t txg = tx->tx_txg; 2465 2466 ASSERT(txg != 0); 2467 2468 /* 2469 * Due to our use of dn_nlevels below, this can only be called 2470 * in open context, unless we are operating on the MOS. 2471 * From syncing context, dn_nlevels may be different from the 2472 * dn_nlevels used when dbuf was dirtied. 2473 */ 2474 ASSERT(db->db_objset == 2475 dmu_objset_pool(db->db_objset)->dp_meta_objset || 2476 txg != spa_syncing_txg(dmu_objset_spa(db->db_objset))); 2477 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 2478 ASSERT0(db->db_level); 2479 ASSERT(MUTEX_HELD(&db->db_mtx)); 2480 2481 /* 2482 * If this buffer is not dirty, we're done. 2483 */ 2484 dbuf_dirty_record_t *dr = dbuf_find_dirty_eq(db, txg); 2485 if (dr == NULL) 2486 return (B_FALSE); 2487 ASSERT(dr->dr_dbuf == db); 2488 2489 dnode_t *dn = dr->dr_dnode; 2490 2491 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 2492 2493 ASSERT(db->db.db_size != 0); 2494 2495 dsl_pool_undirty_space(dmu_objset_pool(dn->dn_objset), 2496 dr->dr_accounted, txg); 2497 2498 list_remove(&db->db_dirty_records, dr); 2499 2500 /* 2501 * Note that there are three places in dbuf_dirty() 2502 * where this dirty record may be put on a list. 2503 * Make sure to do a list_remove corresponding to 2504 * every one of those list_insert calls. 2505 */ 2506 if (dr->dr_parent) { 2507 mutex_enter(&dr->dr_parent->dt.di.dr_mtx); 2508 list_remove(&dr->dr_parent->dt.di.dr_children, dr); 2509 mutex_exit(&dr->dr_parent->dt.di.dr_mtx); 2510 } else if (db->db_blkid == DMU_SPILL_BLKID || 2511 db->db_level + 1 == dn->dn_nlevels) { 2512 ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf); 2513 mutex_enter(&dn->dn_mtx); 2514 list_remove(&dn->dn_dirty_records[txg & TXG_MASK], dr); 2515 mutex_exit(&dn->dn_mtx); 2516 } 2517 2518 if (db->db_state != DB_NOFILL) { 2519 dbuf_unoverride(dr); 2520 2521 ASSERT(db->db_buf != NULL); 2522 ASSERT(dr->dt.dl.dr_data != NULL); 2523 if (dr->dt.dl.dr_data != db->db_buf) 2524 arc_buf_destroy(dr->dt.dl.dr_data, db); 2525 } 2526 2527 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 2528 2529 ASSERT(db->db_dirtycnt > 0); 2530 db->db_dirtycnt -= 1; 2531 2532 if (zfs_refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) { 2533 ASSERT(db->db_state == DB_NOFILL || arc_released(db->db_buf)); 2534 dbuf_destroy(db); 2535 return (B_TRUE); 2536 } 2537 2538 return (B_FALSE); 2539 } 2540 2541 static void 2542 dmu_buf_will_dirty_impl(dmu_buf_t *db_fake, int flags, dmu_tx_t *tx) 2543 { 2544 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2545 2546 ASSERT(tx->tx_txg != 0); 2547 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); 2548 2549 /* 2550 * Quick check for dirtiness. For already dirty blocks, this 2551 * reduces runtime of this function by >90%, and overall performance 2552 * by 50% for some workloads (e.g. file deletion with indirect blocks 2553 * cached). 2554 */ 2555 mutex_enter(&db->db_mtx); 2556 2557 if (db->db_state == DB_CACHED) { 2558 dbuf_dirty_record_t *dr = dbuf_find_dirty_eq(db, tx->tx_txg); 2559 /* 2560 * It's possible that it is already dirty but not cached, 2561 * because there are some calls to dbuf_dirty() that don't 2562 * go through dmu_buf_will_dirty(). 2563 */ 2564 if (dr != NULL) { 2565 /* This dbuf is already dirty and cached. */ 2566 dbuf_redirty(dr); 2567 mutex_exit(&db->db_mtx); 2568 return; 2569 } 2570 } 2571 mutex_exit(&db->db_mtx); 2572 2573 DB_DNODE_ENTER(db); 2574 if (RW_WRITE_HELD(&DB_DNODE(db)->dn_struct_rwlock)) 2575 flags |= DB_RF_HAVESTRUCT; 2576 DB_DNODE_EXIT(db); 2577 (void) dbuf_read(db, NULL, flags); 2578 (void) dbuf_dirty(db, tx); 2579 } 2580 2581 void 2582 dmu_buf_will_dirty(dmu_buf_t *db_fake, dmu_tx_t *tx) 2583 { 2584 dmu_buf_will_dirty_impl(db_fake, 2585 DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH, tx); 2586 } 2587 2588 boolean_t 2589 dmu_buf_is_dirty(dmu_buf_t *db_fake, dmu_tx_t *tx) 2590 { 2591 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2592 dbuf_dirty_record_t *dr; 2593 2594 mutex_enter(&db->db_mtx); 2595 dr = dbuf_find_dirty_eq(db, tx->tx_txg); 2596 mutex_exit(&db->db_mtx); 2597 return (dr != NULL); 2598 } 2599 2600 void 2601 dmu_buf_will_not_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 2602 { 2603 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2604 2605 db->db_state = DB_NOFILL; 2606 DTRACE_SET_STATE(db, "allocating NOFILL buffer"); 2607 dmu_buf_will_fill(db_fake, tx); 2608 } 2609 2610 void 2611 dmu_buf_will_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 2612 { 2613 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2614 2615 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 2616 ASSERT(tx->tx_txg != 0); 2617 ASSERT(db->db_level == 0); 2618 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); 2619 2620 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT || 2621 dmu_tx_private_ok(tx)); 2622 2623 dbuf_noread(db); 2624 (void) dbuf_dirty(db, tx); 2625 } 2626 2627 /* 2628 * This function is effectively the same as dmu_buf_will_dirty(), but 2629 * indicates the caller expects raw encrypted data in the db, and provides 2630 * the crypt params (byteorder, salt, iv, mac) which should be stored in the 2631 * blkptr_t when this dbuf is written. This is only used for blocks of 2632 * dnodes, during raw receive. 2633 */ 2634 void 2635 dmu_buf_set_crypt_params(dmu_buf_t *db_fake, boolean_t byteorder, 2636 const uint8_t *salt, const uint8_t *iv, const uint8_t *mac, dmu_tx_t *tx) 2637 { 2638 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2639 dbuf_dirty_record_t *dr; 2640 2641 /* 2642 * dr_has_raw_params is only processed for blocks of dnodes 2643 * (see dbuf_sync_dnode_leaf_crypt()). 2644 */ 2645 ASSERT3U(db->db.db_object, ==, DMU_META_DNODE_OBJECT); 2646 ASSERT3U(db->db_level, ==, 0); 2647 ASSERT(db->db_objset->os_raw_receive); 2648 2649 dmu_buf_will_dirty_impl(db_fake, 2650 DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH | DB_RF_NO_DECRYPT, tx); 2651 2652 dr = dbuf_find_dirty_eq(db, tx->tx_txg); 2653 2654 ASSERT3P(dr, !=, NULL); 2655 2656 dr->dt.dl.dr_has_raw_params = B_TRUE; 2657 dr->dt.dl.dr_byteorder = byteorder; 2658 memcpy(dr->dt.dl.dr_salt, salt, ZIO_DATA_SALT_LEN); 2659 memcpy(dr->dt.dl.dr_iv, iv, ZIO_DATA_IV_LEN); 2660 memcpy(dr->dt.dl.dr_mac, mac, ZIO_DATA_MAC_LEN); 2661 } 2662 2663 static void 2664 dbuf_override_impl(dmu_buf_impl_t *db, const blkptr_t *bp, dmu_tx_t *tx) 2665 { 2666 struct dirty_leaf *dl; 2667 dbuf_dirty_record_t *dr; 2668 2669 dr = list_head(&db->db_dirty_records); 2670 ASSERT3U(dr->dr_txg, ==, tx->tx_txg); 2671 dl = &dr->dt.dl; 2672 dl->dr_overridden_by = *bp; 2673 dl->dr_override_state = DR_OVERRIDDEN; 2674 dl->dr_overridden_by.blk_birth = dr->dr_txg; 2675 } 2676 2677 void 2678 dmu_buf_fill_done(dmu_buf_t *dbuf, dmu_tx_t *tx) 2679 { 2680 (void) tx; 2681 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf; 2682 dbuf_states_t old_state; 2683 mutex_enter(&db->db_mtx); 2684 DBUF_VERIFY(db); 2685 2686 old_state = db->db_state; 2687 db->db_state = DB_CACHED; 2688 if (old_state == DB_FILL) { 2689 if (db->db_level == 0 && db->db_freed_in_flight) { 2690 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 2691 /* we were freed while filling */ 2692 /* XXX dbuf_undirty? */ 2693 memset(db->db.db_data, 0, db->db.db_size); 2694 db->db_freed_in_flight = FALSE; 2695 DTRACE_SET_STATE(db, 2696 "fill done handling freed in flight"); 2697 } else { 2698 DTRACE_SET_STATE(db, "fill done"); 2699 } 2700 cv_broadcast(&db->db_changed); 2701 } 2702 mutex_exit(&db->db_mtx); 2703 } 2704 2705 void 2706 dmu_buf_write_embedded(dmu_buf_t *dbuf, void *data, 2707 bp_embedded_type_t etype, enum zio_compress comp, 2708 int uncompressed_size, int compressed_size, int byteorder, 2709 dmu_tx_t *tx) 2710 { 2711 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf; 2712 struct dirty_leaf *dl; 2713 dmu_object_type_t type; 2714 dbuf_dirty_record_t *dr; 2715 2716 if (etype == BP_EMBEDDED_TYPE_DATA) { 2717 ASSERT(spa_feature_is_active(dmu_objset_spa(db->db_objset), 2718 SPA_FEATURE_EMBEDDED_DATA)); 2719 } 2720 2721 DB_DNODE_ENTER(db); 2722 type = DB_DNODE(db)->dn_type; 2723 DB_DNODE_EXIT(db); 2724 2725 ASSERT0(db->db_level); 2726 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 2727 2728 dmu_buf_will_not_fill(dbuf, tx); 2729 2730 dr = list_head(&db->db_dirty_records); 2731 ASSERT3U(dr->dr_txg, ==, tx->tx_txg); 2732 dl = &dr->dt.dl; 2733 encode_embedded_bp_compressed(&dl->dr_overridden_by, 2734 data, comp, uncompressed_size, compressed_size); 2735 BPE_SET_ETYPE(&dl->dr_overridden_by, etype); 2736 BP_SET_TYPE(&dl->dr_overridden_by, type); 2737 BP_SET_LEVEL(&dl->dr_overridden_by, 0); 2738 BP_SET_BYTEORDER(&dl->dr_overridden_by, byteorder); 2739 2740 dl->dr_override_state = DR_OVERRIDDEN; 2741 dl->dr_overridden_by.blk_birth = dr->dr_txg; 2742 } 2743 2744 void 2745 dmu_buf_redact(dmu_buf_t *dbuf, dmu_tx_t *tx) 2746 { 2747 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf; 2748 dmu_object_type_t type; 2749 ASSERT(dsl_dataset_feature_is_active(db->db_objset->os_dsl_dataset, 2750 SPA_FEATURE_REDACTED_DATASETS)); 2751 2752 DB_DNODE_ENTER(db); 2753 type = DB_DNODE(db)->dn_type; 2754 DB_DNODE_EXIT(db); 2755 2756 ASSERT0(db->db_level); 2757 dmu_buf_will_not_fill(dbuf, tx); 2758 2759 blkptr_t bp = { { { {0} } } }; 2760 BP_SET_TYPE(&bp, type); 2761 BP_SET_LEVEL(&bp, 0); 2762 BP_SET_BIRTH(&bp, tx->tx_txg, 0); 2763 BP_SET_REDACTED(&bp); 2764 BPE_SET_LSIZE(&bp, dbuf->db_size); 2765 2766 dbuf_override_impl(db, &bp, tx); 2767 } 2768 2769 /* 2770 * Directly assign a provided arc buf to a given dbuf if it's not referenced 2771 * by anybody except our caller. Otherwise copy arcbuf's contents to dbuf. 2772 */ 2773 void 2774 dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx) 2775 { 2776 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); 2777 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 2778 ASSERT(db->db_level == 0); 2779 ASSERT3U(dbuf_is_metadata(db), ==, arc_is_metadata(buf)); 2780 ASSERT(buf != NULL); 2781 ASSERT3U(arc_buf_lsize(buf), ==, db->db.db_size); 2782 ASSERT(tx->tx_txg != 0); 2783 2784 arc_return_buf(buf, db); 2785 ASSERT(arc_released(buf)); 2786 2787 mutex_enter(&db->db_mtx); 2788 2789 while (db->db_state == DB_READ || db->db_state == DB_FILL) 2790 cv_wait(&db->db_changed, &db->db_mtx); 2791 2792 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_UNCACHED); 2793 2794 if (db->db_state == DB_CACHED && 2795 zfs_refcount_count(&db->db_holds) - 1 > db->db_dirtycnt) { 2796 /* 2797 * In practice, we will never have a case where we have an 2798 * encrypted arc buffer while additional holds exist on the 2799 * dbuf. We don't handle this here so we simply assert that 2800 * fact instead. 2801 */ 2802 ASSERT(!arc_is_encrypted(buf)); 2803 mutex_exit(&db->db_mtx); 2804 (void) dbuf_dirty(db, tx); 2805 memcpy(db->db.db_data, buf->b_data, db->db.db_size); 2806 arc_buf_destroy(buf, db); 2807 return; 2808 } 2809 2810 if (db->db_state == DB_CACHED) { 2811 dbuf_dirty_record_t *dr = list_head(&db->db_dirty_records); 2812 2813 ASSERT(db->db_buf != NULL); 2814 if (dr != NULL && dr->dr_txg == tx->tx_txg) { 2815 ASSERT(dr->dt.dl.dr_data == db->db_buf); 2816 2817 if (!arc_released(db->db_buf)) { 2818 ASSERT(dr->dt.dl.dr_override_state == 2819 DR_OVERRIDDEN); 2820 arc_release(db->db_buf, db); 2821 } 2822 dr->dt.dl.dr_data = buf; 2823 arc_buf_destroy(db->db_buf, db); 2824 } else if (dr == NULL || dr->dt.dl.dr_data != db->db_buf) { 2825 arc_release(db->db_buf, db); 2826 arc_buf_destroy(db->db_buf, db); 2827 } 2828 db->db_buf = NULL; 2829 } 2830 ASSERT(db->db_buf == NULL); 2831 dbuf_set_data(db, buf); 2832 db->db_state = DB_FILL; 2833 DTRACE_SET_STATE(db, "filling assigned arcbuf"); 2834 mutex_exit(&db->db_mtx); 2835 (void) dbuf_dirty(db, tx); 2836 dmu_buf_fill_done(&db->db, tx); 2837 } 2838 2839 void 2840 dbuf_destroy(dmu_buf_impl_t *db) 2841 { 2842 dnode_t *dn; 2843 dmu_buf_impl_t *parent = db->db_parent; 2844 dmu_buf_impl_t *dndb; 2845 2846 ASSERT(MUTEX_HELD(&db->db_mtx)); 2847 ASSERT(zfs_refcount_is_zero(&db->db_holds)); 2848 2849 if (db->db_buf != NULL) { 2850 arc_buf_destroy(db->db_buf, db); 2851 db->db_buf = NULL; 2852 } 2853 2854 if (db->db_blkid == DMU_BONUS_BLKID) { 2855 int slots = DB_DNODE(db)->dn_num_slots; 2856 int bonuslen = DN_SLOTS_TO_BONUSLEN(slots); 2857 if (db->db.db_data != NULL) { 2858 kmem_free(db->db.db_data, bonuslen); 2859 arc_space_return(bonuslen, ARC_SPACE_BONUS); 2860 db->db_state = DB_UNCACHED; 2861 DTRACE_SET_STATE(db, "buffer cleared"); 2862 } 2863 } 2864 2865 dbuf_clear_data(db); 2866 2867 if (multilist_link_active(&db->db_cache_link)) { 2868 ASSERT(db->db_caching_status == DB_DBUF_CACHE || 2869 db->db_caching_status == DB_DBUF_METADATA_CACHE); 2870 2871 multilist_remove(&dbuf_caches[db->db_caching_status].cache, db); 2872 (void) zfs_refcount_remove_many( 2873 &dbuf_caches[db->db_caching_status].size, 2874 db->db.db_size, db); 2875 2876 if (db->db_caching_status == DB_DBUF_METADATA_CACHE) { 2877 DBUF_STAT_BUMPDOWN(metadata_cache_count); 2878 } else { 2879 DBUF_STAT_BUMPDOWN(cache_levels[db->db_level]); 2880 DBUF_STAT_BUMPDOWN(cache_count); 2881 DBUF_STAT_DECR(cache_levels_bytes[db->db_level], 2882 db->db.db_size); 2883 } 2884 db->db_caching_status = DB_NO_CACHE; 2885 } 2886 2887 ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL); 2888 ASSERT(db->db_data_pending == NULL); 2889 ASSERT(list_is_empty(&db->db_dirty_records)); 2890 2891 db->db_state = DB_EVICTING; 2892 DTRACE_SET_STATE(db, "buffer eviction started"); 2893 db->db_blkptr = NULL; 2894 2895 /* 2896 * Now that db_state is DB_EVICTING, nobody else can find this via 2897 * the hash table. We can now drop db_mtx, which allows us to 2898 * acquire the dn_dbufs_mtx. 2899 */ 2900 mutex_exit(&db->db_mtx); 2901 2902 DB_DNODE_ENTER(db); 2903 dn = DB_DNODE(db); 2904 dndb = dn->dn_dbuf; 2905 if (db->db_blkid != DMU_BONUS_BLKID) { 2906 boolean_t needlock = !MUTEX_HELD(&dn->dn_dbufs_mtx); 2907 if (needlock) 2908 mutex_enter_nested(&dn->dn_dbufs_mtx, 2909 NESTED_SINGLE); 2910 avl_remove(&dn->dn_dbufs, db); 2911 membar_producer(); 2912 DB_DNODE_EXIT(db); 2913 if (needlock) 2914 mutex_exit(&dn->dn_dbufs_mtx); 2915 /* 2916 * Decrementing the dbuf count means that the hold corresponding 2917 * to the removed dbuf is no longer discounted in dnode_move(), 2918 * so the dnode cannot be moved until after we release the hold. 2919 * The membar_producer() ensures visibility of the decremented 2920 * value in dnode_move(), since DB_DNODE_EXIT doesn't actually 2921 * release any lock. 2922 */ 2923 mutex_enter(&dn->dn_mtx); 2924 dnode_rele_and_unlock(dn, db, B_TRUE); 2925 db->db_dnode_handle = NULL; 2926 2927 dbuf_hash_remove(db); 2928 } else { 2929 DB_DNODE_EXIT(db); 2930 } 2931 2932 ASSERT(zfs_refcount_is_zero(&db->db_holds)); 2933 2934 db->db_parent = NULL; 2935 2936 ASSERT(db->db_buf == NULL); 2937 ASSERT(db->db.db_data == NULL); 2938 ASSERT(db->db_hash_next == NULL); 2939 ASSERT(db->db_blkptr == NULL); 2940 ASSERT(db->db_data_pending == NULL); 2941 ASSERT3U(db->db_caching_status, ==, DB_NO_CACHE); 2942 ASSERT(!multilist_link_active(&db->db_cache_link)); 2943 2944 /* 2945 * If this dbuf is referenced from an indirect dbuf, 2946 * decrement the ref count on the indirect dbuf. 2947 */ 2948 if (parent && parent != dndb) { 2949 mutex_enter(&parent->db_mtx); 2950 dbuf_rele_and_unlock(parent, db, B_TRUE); 2951 } 2952 2953 kmem_cache_free(dbuf_kmem_cache, db); 2954 arc_space_return(sizeof (dmu_buf_impl_t), ARC_SPACE_DBUF); 2955 } 2956 2957 /* 2958 * Note: While bpp will always be updated if the function returns success, 2959 * parentp will not be updated if the dnode does not have dn_dbuf filled in; 2960 * this happens when the dnode is the meta-dnode, or {user|group|project}used 2961 * object. 2962 */ 2963 __attribute__((always_inline)) 2964 static inline int 2965 dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse, 2966 dmu_buf_impl_t **parentp, blkptr_t **bpp) 2967 { 2968 *parentp = NULL; 2969 *bpp = NULL; 2970 2971 ASSERT(blkid != DMU_BONUS_BLKID); 2972 2973 if (blkid == DMU_SPILL_BLKID) { 2974 mutex_enter(&dn->dn_mtx); 2975 if (dn->dn_have_spill && 2976 (dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR)) 2977 *bpp = DN_SPILL_BLKPTR(dn->dn_phys); 2978 else 2979 *bpp = NULL; 2980 dbuf_add_ref(dn->dn_dbuf, NULL); 2981 *parentp = dn->dn_dbuf; 2982 mutex_exit(&dn->dn_mtx); 2983 return (0); 2984 } 2985 2986 int nlevels = 2987 (dn->dn_phys->dn_nlevels == 0) ? 1 : dn->dn_phys->dn_nlevels; 2988 int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 2989 2990 ASSERT3U(level * epbs, <, 64); 2991 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 2992 /* 2993 * This assertion shouldn't trip as long as the max indirect block size 2994 * is less than 1M. The reason for this is that up to that point, 2995 * the number of levels required to address an entire object with blocks 2996 * of size SPA_MINBLOCKSIZE satisfies nlevels * epbs + 1 <= 64. In 2997 * other words, if N * epbs + 1 > 64, then if (N-1) * epbs + 1 > 55 2998 * (i.e. we can address the entire object), objects will all use at most 2999 * N-1 levels and the assertion won't overflow. However, once epbs is 3000 * 13, 4 * 13 + 1 = 53, but 5 * 13 + 1 = 66. Then, 4 levels will not be 3001 * enough to address an entire object, so objects will have 5 levels, 3002 * but then this assertion will overflow. 3003 * 3004 * All this is to say that if we ever increase DN_MAX_INDBLKSHIFT, we 3005 * need to redo this logic to handle overflows. 3006 */ 3007 ASSERT(level >= nlevels || 3008 ((nlevels - level - 1) * epbs) + 3009 highbit64(dn->dn_phys->dn_nblkptr) <= 64); 3010 if (level >= nlevels || 3011 blkid >= ((uint64_t)dn->dn_phys->dn_nblkptr << 3012 ((nlevels - level - 1) * epbs)) || 3013 (fail_sparse && 3014 blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) { 3015 /* the buffer has no parent yet */ 3016 return (SET_ERROR(ENOENT)); 3017 } else if (level < nlevels-1) { 3018 /* this block is referenced from an indirect block */ 3019 int err; 3020 3021 err = dbuf_hold_impl(dn, level + 1, 3022 blkid >> epbs, fail_sparse, FALSE, NULL, parentp); 3023 3024 if (err) 3025 return (err); 3026 err = dbuf_read(*parentp, NULL, 3027 (DB_RF_HAVESTRUCT | DB_RF_NOPREFETCH | DB_RF_CANFAIL)); 3028 if (err) { 3029 dbuf_rele(*parentp, NULL); 3030 *parentp = NULL; 3031 return (err); 3032 } 3033 rw_enter(&(*parentp)->db_rwlock, RW_READER); 3034 *bpp = ((blkptr_t *)(*parentp)->db.db_data) + 3035 (blkid & ((1ULL << epbs) - 1)); 3036 if (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs))) 3037 ASSERT(BP_IS_HOLE(*bpp)); 3038 rw_exit(&(*parentp)->db_rwlock); 3039 return (0); 3040 } else { 3041 /* the block is referenced from the dnode */ 3042 ASSERT3U(level, ==, nlevels-1); 3043 ASSERT(dn->dn_phys->dn_nblkptr == 0 || 3044 blkid < dn->dn_phys->dn_nblkptr); 3045 if (dn->dn_dbuf) { 3046 dbuf_add_ref(dn->dn_dbuf, NULL); 3047 *parentp = dn->dn_dbuf; 3048 } 3049 *bpp = &dn->dn_phys->dn_blkptr[blkid]; 3050 return (0); 3051 } 3052 } 3053 3054 static dmu_buf_impl_t * 3055 dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid, 3056 dmu_buf_impl_t *parent, blkptr_t *blkptr) 3057 { 3058 objset_t *os = dn->dn_objset; 3059 dmu_buf_impl_t *db, *odb; 3060 3061 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 3062 ASSERT(dn->dn_type != DMU_OT_NONE); 3063 3064 db = kmem_cache_alloc(dbuf_kmem_cache, KM_SLEEP); 3065 3066 list_create(&db->db_dirty_records, sizeof (dbuf_dirty_record_t), 3067 offsetof(dbuf_dirty_record_t, dr_dbuf_node)); 3068 3069 db->db_objset = os; 3070 db->db.db_object = dn->dn_object; 3071 db->db_level = level; 3072 db->db_blkid = blkid; 3073 db->db_dirtycnt = 0; 3074 db->db_dnode_handle = dn->dn_handle; 3075 db->db_parent = parent; 3076 db->db_blkptr = blkptr; 3077 3078 db->db_user = NULL; 3079 db->db_user_immediate_evict = FALSE; 3080 db->db_freed_in_flight = FALSE; 3081 db->db_pending_evict = FALSE; 3082 3083 if (blkid == DMU_BONUS_BLKID) { 3084 ASSERT3P(parent, ==, dn->dn_dbuf); 3085 db->db.db_size = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots) - 3086 (dn->dn_nblkptr-1) * sizeof (blkptr_t); 3087 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 3088 db->db.db_offset = DMU_BONUS_BLKID; 3089 db->db_state = DB_UNCACHED; 3090 DTRACE_SET_STATE(db, "bonus buffer created"); 3091 db->db_caching_status = DB_NO_CACHE; 3092 /* the bonus dbuf is not placed in the hash table */ 3093 arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_DBUF); 3094 return (db); 3095 } else if (blkid == DMU_SPILL_BLKID) { 3096 db->db.db_size = (blkptr != NULL) ? 3097 BP_GET_LSIZE(blkptr) : SPA_MINBLOCKSIZE; 3098 db->db.db_offset = 0; 3099 } else { 3100 int blocksize = 3101 db->db_level ? 1 << dn->dn_indblkshift : dn->dn_datablksz; 3102 db->db.db_size = blocksize; 3103 db->db.db_offset = db->db_blkid * blocksize; 3104 } 3105 3106 /* 3107 * Hold the dn_dbufs_mtx while we get the new dbuf 3108 * in the hash table *and* added to the dbufs list. 3109 * This prevents a possible deadlock with someone 3110 * trying to look up this dbuf before it's added to the 3111 * dn_dbufs list. 3112 */ 3113 mutex_enter(&dn->dn_dbufs_mtx); 3114 db->db_state = DB_EVICTING; /* not worth logging this state change */ 3115 if ((odb = dbuf_hash_insert(db)) != NULL) { 3116 /* someone else inserted it first */ 3117 mutex_exit(&dn->dn_dbufs_mtx); 3118 kmem_cache_free(dbuf_kmem_cache, db); 3119 DBUF_STAT_BUMP(hash_insert_race); 3120 return (odb); 3121 } 3122 avl_add(&dn->dn_dbufs, db); 3123 3124 db->db_state = DB_UNCACHED; 3125 DTRACE_SET_STATE(db, "regular buffer created"); 3126 db->db_caching_status = DB_NO_CACHE; 3127 mutex_exit(&dn->dn_dbufs_mtx); 3128 arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_DBUF); 3129 3130 if (parent && parent != dn->dn_dbuf) 3131 dbuf_add_ref(parent, db); 3132 3133 ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 3134 zfs_refcount_count(&dn->dn_holds) > 0); 3135 (void) zfs_refcount_add(&dn->dn_holds, db); 3136 3137 dprintf_dbuf(db, "db=%p\n", db); 3138 3139 return (db); 3140 } 3141 3142 /* 3143 * This function returns a block pointer and information about the object, 3144 * given a dnode and a block. This is a publicly accessible version of 3145 * dbuf_findbp that only returns some information, rather than the 3146 * dbuf. Note that the dnode passed in must be held, and the dn_struct_rwlock 3147 * should be locked as (at least) a reader. 3148 */ 3149 int 3150 dbuf_dnode_findbp(dnode_t *dn, uint64_t level, uint64_t blkid, 3151 blkptr_t *bp, uint16_t *datablkszsec, uint8_t *indblkshift) 3152 { 3153 dmu_buf_impl_t *dbp = NULL; 3154 blkptr_t *bp2; 3155 int err = 0; 3156 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 3157 3158 err = dbuf_findbp(dn, level, blkid, B_FALSE, &dbp, &bp2); 3159 if (err == 0) { 3160 *bp = *bp2; 3161 if (dbp != NULL) 3162 dbuf_rele(dbp, NULL); 3163 if (datablkszsec != NULL) 3164 *datablkszsec = dn->dn_phys->dn_datablkszsec; 3165 if (indblkshift != NULL) 3166 *indblkshift = dn->dn_phys->dn_indblkshift; 3167 } 3168 3169 return (err); 3170 } 3171 3172 typedef struct dbuf_prefetch_arg { 3173 spa_t *dpa_spa; /* The spa to issue the prefetch in. */ 3174 zbookmark_phys_t dpa_zb; /* The target block to prefetch. */ 3175 int dpa_epbs; /* Entries (blkptr_t's) Per Block Shift. */ 3176 int dpa_curlevel; /* The current level that we're reading */ 3177 dnode_t *dpa_dnode; /* The dnode associated with the prefetch */ 3178 zio_priority_t dpa_prio; /* The priority I/Os should be issued at. */ 3179 zio_t *dpa_zio; /* The parent zio_t for all prefetches. */ 3180 arc_flags_t dpa_aflags; /* Flags to pass to the final prefetch. */ 3181 dbuf_prefetch_fn dpa_cb; /* prefetch completion callback */ 3182 void *dpa_arg; /* prefetch completion arg */ 3183 } dbuf_prefetch_arg_t; 3184 3185 static void 3186 dbuf_prefetch_fini(dbuf_prefetch_arg_t *dpa, boolean_t io_done) 3187 { 3188 if (dpa->dpa_cb != NULL) { 3189 dpa->dpa_cb(dpa->dpa_arg, dpa->dpa_zb.zb_level, 3190 dpa->dpa_zb.zb_blkid, io_done); 3191 } 3192 kmem_free(dpa, sizeof (*dpa)); 3193 } 3194 3195 static void 3196 dbuf_issue_final_prefetch_done(zio_t *zio, const zbookmark_phys_t *zb, 3197 const blkptr_t *iobp, arc_buf_t *abuf, void *private) 3198 { 3199 (void) zio, (void) zb, (void) iobp; 3200 dbuf_prefetch_arg_t *dpa = private; 3201 3202 if (abuf != NULL) 3203 arc_buf_destroy(abuf, private); 3204 3205 dbuf_prefetch_fini(dpa, B_TRUE); 3206 } 3207 3208 /* 3209 * Actually issue the prefetch read for the block given. 3210 */ 3211 static void 3212 dbuf_issue_final_prefetch(dbuf_prefetch_arg_t *dpa, blkptr_t *bp) 3213 { 3214 ASSERT(!BP_IS_REDACTED(bp) || 3215 dsl_dataset_feature_is_active( 3216 dpa->dpa_dnode->dn_objset->os_dsl_dataset, 3217 SPA_FEATURE_REDACTED_DATASETS)); 3218 3219 if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp) || BP_IS_REDACTED(bp)) 3220 return (dbuf_prefetch_fini(dpa, B_FALSE)); 3221 3222 int zio_flags = ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE; 3223 arc_flags_t aflags = 3224 dpa->dpa_aflags | ARC_FLAG_NOWAIT | ARC_FLAG_PREFETCH | 3225 ARC_FLAG_NO_BUF; 3226 3227 /* dnodes are always read as raw and then converted later */ 3228 if (BP_GET_TYPE(bp) == DMU_OT_DNODE && BP_IS_PROTECTED(bp) && 3229 dpa->dpa_curlevel == 0) 3230 zio_flags |= ZIO_FLAG_RAW; 3231 3232 ASSERT3U(dpa->dpa_curlevel, ==, BP_GET_LEVEL(bp)); 3233 ASSERT3U(dpa->dpa_curlevel, ==, dpa->dpa_zb.zb_level); 3234 ASSERT(dpa->dpa_zio != NULL); 3235 (void) arc_read(dpa->dpa_zio, dpa->dpa_spa, bp, 3236 dbuf_issue_final_prefetch_done, dpa, 3237 dpa->dpa_prio, zio_flags, &aflags, &dpa->dpa_zb); 3238 } 3239 3240 /* 3241 * Called when an indirect block above our prefetch target is read in. This 3242 * will either read in the next indirect block down the tree or issue the actual 3243 * prefetch if the next block down is our target. 3244 */ 3245 static void 3246 dbuf_prefetch_indirect_done(zio_t *zio, const zbookmark_phys_t *zb, 3247 const blkptr_t *iobp, arc_buf_t *abuf, void *private) 3248 { 3249 (void) zb, (void) iobp; 3250 dbuf_prefetch_arg_t *dpa = private; 3251 3252 ASSERT3S(dpa->dpa_zb.zb_level, <, dpa->dpa_curlevel); 3253 ASSERT3S(dpa->dpa_curlevel, >, 0); 3254 3255 if (abuf == NULL) { 3256 ASSERT(zio == NULL || zio->io_error != 0); 3257 return (dbuf_prefetch_fini(dpa, B_TRUE)); 3258 } 3259 ASSERT(zio == NULL || zio->io_error == 0); 3260 3261 /* 3262 * The dpa_dnode is only valid if we are called with a NULL 3263 * zio. This indicates that the arc_read() returned without 3264 * first calling zio_read() to issue a physical read. Once 3265 * a physical read is made the dpa_dnode must be invalidated 3266 * as the locks guarding it may have been dropped. If the 3267 * dpa_dnode is still valid, then we want to add it to the dbuf 3268 * cache. To do so, we must hold the dbuf associated with the block 3269 * we just prefetched, read its contents so that we associate it 3270 * with an arc_buf_t, and then release it. 3271 */ 3272 if (zio != NULL) { 3273 ASSERT3S(BP_GET_LEVEL(zio->io_bp), ==, dpa->dpa_curlevel); 3274 if (zio->io_flags & ZIO_FLAG_RAW_COMPRESS) { 3275 ASSERT3U(BP_GET_PSIZE(zio->io_bp), ==, zio->io_size); 3276 } else { 3277 ASSERT3U(BP_GET_LSIZE(zio->io_bp), ==, zio->io_size); 3278 } 3279 ASSERT3P(zio->io_spa, ==, dpa->dpa_spa); 3280 3281 dpa->dpa_dnode = NULL; 3282 } else if (dpa->dpa_dnode != NULL) { 3283 uint64_t curblkid = dpa->dpa_zb.zb_blkid >> 3284 (dpa->dpa_epbs * (dpa->dpa_curlevel - 3285 dpa->dpa_zb.zb_level)); 3286 dmu_buf_impl_t *db = dbuf_hold_level(dpa->dpa_dnode, 3287 dpa->dpa_curlevel, curblkid, FTAG); 3288 if (db == NULL) { 3289 arc_buf_destroy(abuf, private); 3290 return (dbuf_prefetch_fini(dpa, B_TRUE)); 3291 } 3292 (void) dbuf_read(db, NULL, 3293 DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH | DB_RF_HAVESTRUCT); 3294 dbuf_rele(db, FTAG); 3295 } 3296 3297 dpa->dpa_curlevel--; 3298 uint64_t nextblkid = dpa->dpa_zb.zb_blkid >> 3299 (dpa->dpa_epbs * (dpa->dpa_curlevel - dpa->dpa_zb.zb_level)); 3300 blkptr_t *bp = ((blkptr_t *)abuf->b_data) + 3301 P2PHASE(nextblkid, 1ULL << dpa->dpa_epbs); 3302 3303 ASSERT(!BP_IS_REDACTED(bp) || 3304 dsl_dataset_feature_is_active( 3305 dpa->dpa_dnode->dn_objset->os_dsl_dataset, 3306 SPA_FEATURE_REDACTED_DATASETS)); 3307 if (BP_IS_HOLE(bp) || BP_IS_REDACTED(bp)) { 3308 dbuf_prefetch_fini(dpa, B_TRUE); 3309 } else if (dpa->dpa_curlevel == dpa->dpa_zb.zb_level) { 3310 ASSERT3U(nextblkid, ==, dpa->dpa_zb.zb_blkid); 3311 dbuf_issue_final_prefetch(dpa, bp); 3312 } else { 3313 arc_flags_t iter_aflags = ARC_FLAG_NOWAIT; 3314 zbookmark_phys_t zb; 3315 3316 /* flag if L2ARC eligible, l2arc_noprefetch then decides */ 3317 if (dpa->dpa_aflags & ARC_FLAG_L2CACHE) 3318 iter_aflags |= ARC_FLAG_L2CACHE; 3319 3320 ASSERT3U(dpa->dpa_curlevel, ==, BP_GET_LEVEL(bp)); 3321 3322 SET_BOOKMARK(&zb, dpa->dpa_zb.zb_objset, 3323 dpa->dpa_zb.zb_object, dpa->dpa_curlevel, nextblkid); 3324 3325 (void) arc_read(dpa->dpa_zio, dpa->dpa_spa, 3326 bp, dbuf_prefetch_indirect_done, dpa, 3327 ZIO_PRIORITY_SYNC_READ, 3328 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 3329 &iter_aflags, &zb); 3330 } 3331 3332 arc_buf_destroy(abuf, private); 3333 } 3334 3335 /* 3336 * Issue prefetch reads for the given block on the given level. If the indirect 3337 * blocks above that block are not in memory, we will read them in 3338 * asynchronously. As a result, this call never blocks waiting for a read to 3339 * complete. Note that the prefetch might fail if the dataset is encrypted and 3340 * the encryption key is unmapped before the IO completes. 3341 */ 3342 int 3343 dbuf_prefetch_impl(dnode_t *dn, int64_t level, uint64_t blkid, 3344 zio_priority_t prio, arc_flags_t aflags, dbuf_prefetch_fn cb, 3345 void *arg) 3346 { 3347 blkptr_t bp; 3348 int epbs, nlevels, curlevel; 3349 uint64_t curblkid; 3350 3351 ASSERT(blkid != DMU_BONUS_BLKID); 3352 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 3353 3354 if (blkid > dn->dn_maxblkid) 3355 goto no_issue; 3356 3357 if (level == 0 && dnode_block_freed(dn, blkid)) 3358 goto no_issue; 3359 3360 /* 3361 * This dnode hasn't been written to disk yet, so there's nothing to 3362 * prefetch. 3363 */ 3364 nlevels = dn->dn_phys->dn_nlevels; 3365 if (level >= nlevels || dn->dn_phys->dn_nblkptr == 0) 3366 goto no_issue; 3367 3368 epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 3369 if (dn->dn_phys->dn_maxblkid < blkid << (epbs * level)) 3370 goto no_issue; 3371 3372 dmu_buf_impl_t *db = dbuf_find(dn->dn_objset, dn->dn_object, 3373 level, blkid); 3374 if (db != NULL) { 3375 mutex_exit(&db->db_mtx); 3376 /* 3377 * This dbuf already exists. It is either CACHED, or 3378 * (we assume) about to be read or filled. 3379 */ 3380 goto no_issue; 3381 } 3382 3383 /* 3384 * Find the closest ancestor (indirect block) of the target block 3385 * that is present in the cache. In this indirect block, we will 3386 * find the bp that is at curlevel, curblkid. 3387 */ 3388 curlevel = level; 3389 curblkid = blkid; 3390 while (curlevel < nlevels - 1) { 3391 int parent_level = curlevel + 1; 3392 uint64_t parent_blkid = curblkid >> epbs; 3393 dmu_buf_impl_t *db; 3394 3395 if (dbuf_hold_impl(dn, parent_level, parent_blkid, 3396 FALSE, TRUE, FTAG, &db) == 0) { 3397 blkptr_t *bpp = db->db_buf->b_data; 3398 bp = bpp[P2PHASE(curblkid, 1 << epbs)]; 3399 dbuf_rele(db, FTAG); 3400 break; 3401 } 3402 3403 curlevel = parent_level; 3404 curblkid = parent_blkid; 3405 } 3406 3407 if (curlevel == nlevels - 1) { 3408 /* No cached indirect blocks found. */ 3409 ASSERT3U(curblkid, <, dn->dn_phys->dn_nblkptr); 3410 bp = dn->dn_phys->dn_blkptr[curblkid]; 3411 } 3412 ASSERT(!BP_IS_REDACTED(&bp) || 3413 dsl_dataset_feature_is_active(dn->dn_objset->os_dsl_dataset, 3414 SPA_FEATURE_REDACTED_DATASETS)); 3415 if (BP_IS_HOLE(&bp) || BP_IS_REDACTED(&bp)) 3416 goto no_issue; 3417 3418 ASSERT3U(curlevel, ==, BP_GET_LEVEL(&bp)); 3419 3420 zio_t *pio = zio_root(dmu_objset_spa(dn->dn_objset), NULL, NULL, 3421 ZIO_FLAG_CANFAIL); 3422 3423 dbuf_prefetch_arg_t *dpa = kmem_zalloc(sizeof (*dpa), KM_SLEEP); 3424 dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset; 3425 SET_BOOKMARK(&dpa->dpa_zb, ds != NULL ? ds->ds_object : DMU_META_OBJSET, 3426 dn->dn_object, level, blkid); 3427 dpa->dpa_curlevel = curlevel; 3428 dpa->dpa_prio = prio; 3429 dpa->dpa_aflags = aflags; 3430 dpa->dpa_spa = dn->dn_objset->os_spa; 3431 dpa->dpa_dnode = dn; 3432 dpa->dpa_epbs = epbs; 3433 dpa->dpa_zio = pio; 3434 dpa->dpa_cb = cb; 3435 dpa->dpa_arg = arg; 3436 3437 /* flag if L2ARC eligible, l2arc_noprefetch then decides */ 3438 if (dnode_level_is_l2cacheable(&bp, dn, level)) 3439 dpa->dpa_aflags |= ARC_FLAG_L2CACHE; 3440 3441 /* 3442 * If we have the indirect just above us, no need to do the asynchronous 3443 * prefetch chain; we'll just run the last step ourselves. If we're at 3444 * a higher level, though, we want to issue the prefetches for all the 3445 * indirect blocks asynchronously, so we can go on with whatever we were 3446 * doing. 3447 */ 3448 if (curlevel == level) { 3449 ASSERT3U(curblkid, ==, blkid); 3450 dbuf_issue_final_prefetch(dpa, &bp); 3451 } else { 3452 arc_flags_t iter_aflags = ARC_FLAG_NOWAIT; 3453 zbookmark_phys_t zb; 3454 3455 /* flag if L2ARC eligible, l2arc_noprefetch then decides */ 3456 if (dnode_level_is_l2cacheable(&bp, dn, level)) 3457 iter_aflags |= ARC_FLAG_L2CACHE; 3458 3459 SET_BOOKMARK(&zb, ds != NULL ? ds->ds_object : DMU_META_OBJSET, 3460 dn->dn_object, curlevel, curblkid); 3461 (void) arc_read(dpa->dpa_zio, dpa->dpa_spa, 3462 &bp, dbuf_prefetch_indirect_done, dpa, 3463 ZIO_PRIORITY_SYNC_READ, 3464 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 3465 &iter_aflags, &zb); 3466 } 3467 /* 3468 * We use pio here instead of dpa_zio since it's possible that 3469 * dpa may have already been freed. 3470 */ 3471 zio_nowait(pio); 3472 return (1); 3473 no_issue: 3474 if (cb != NULL) 3475 cb(arg, level, blkid, B_FALSE); 3476 return (0); 3477 } 3478 3479 int 3480 dbuf_prefetch(dnode_t *dn, int64_t level, uint64_t blkid, zio_priority_t prio, 3481 arc_flags_t aflags) 3482 { 3483 3484 return (dbuf_prefetch_impl(dn, level, blkid, prio, aflags, NULL, NULL)); 3485 } 3486 3487 /* 3488 * Helper function for dbuf_hold_impl() to copy a buffer. Handles 3489 * the case of encrypted, compressed and uncompressed buffers by 3490 * allocating the new buffer, respectively, with arc_alloc_raw_buf(), 3491 * arc_alloc_compressed_buf() or arc_alloc_buf().* 3492 * 3493 * NOTE: Declared noinline to avoid stack bloat in dbuf_hold_impl(). 3494 */ 3495 noinline static void 3496 dbuf_hold_copy(dnode_t *dn, dmu_buf_impl_t *db) 3497 { 3498 dbuf_dirty_record_t *dr = db->db_data_pending; 3499 arc_buf_t *data = dr->dt.dl.dr_data; 3500 enum zio_compress compress_type = arc_get_compression(data); 3501 uint8_t complevel = arc_get_complevel(data); 3502 3503 if (arc_is_encrypted(data)) { 3504 boolean_t byteorder; 3505 uint8_t salt[ZIO_DATA_SALT_LEN]; 3506 uint8_t iv[ZIO_DATA_IV_LEN]; 3507 uint8_t mac[ZIO_DATA_MAC_LEN]; 3508 3509 arc_get_raw_params(data, &byteorder, salt, iv, mac); 3510 dbuf_set_data(db, arc_alloc_raw_buf(dn->dn_objset->os_spa, db, 3511 dmu_objset_id(dn->dn_objset), byteorder, salt, iv, mac, 3512 dn->dn_type, arc_buf_size(data), arc_buf_lsize(data), 3513 compress_type, complevel)); 3514 } else if (compress_type != ZIO_COMPRESS_OFF) { 3515 dbuf_set_data(db, arc_alloc_compressed_buf( 3516 dn->dn_objset->os_spa, db, arc_buf_size(data), 3517 arc_buf_lsize(data), compress_type, complevel)); 3518 } else { 3519 dbuf_set_data(db, arc_alloc_buf(dn->dn_objset->os_spa, db, 3520 DBUF_GET_BUFC_TYPE(db), db->db.db_size)); 3521 } 3522 3523 rw_enter(&db->db_rwlock, RW_WRITER); 3524 memcpy(db->db.db_data, data->b_data, arc_buf_size(data)); 3525 rw_exit(&db->db_rwlock); 3526 } 3527 3528 /* 3529 * Returns with db_holds incremented, and db_mtx not held. 3530 * Note: dn_struct_rwlock must be held. 3531 */ 3532 int 3533 dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, 3534 boolean_t fail_sparse, boolean_t fail_uncached, 3535 const void *tag, dmu_buf_impl_t **dbp) 3536 { 3537 dmu_buf_impl_t *db, *parent = NULL; 3538 3539 /* If the pool has been created, verify the tx_sync_lock is not held */ 3540 spa_t *spa = dn->dn_objset->os_spa; 3541 dsl_pool_t *dp = spa->spa_dsl_pool; 3542 if (dp != NULL) { 3543 ASSERT(!MUTEX_HELD(&dp->dp_tx.tx_sync_lock)); 3544 } 3545 3546 ASSERT(blkid != DMU_BONUS_BLKID); 3547 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 3548 ASSERT3U(dn->dn_nlevels, >, level); 3549 3550 *dbp = NULL; 3551 3552 /* dbuf_find() returns with db_mtx held */ 3553 db = dbuf_find(dn->dn_objset, dn->dn_object, level, blkid); 3554 3555 if (db == NULL) { 3556 blkptr_t *bp = NULL; 3557 int err; 3558 3559 if (fail_uncached) 3560 return (SET_ERROR(ENOENT)); 3561 3562 ASSERT3P(parent, ==, NULL); 3563 err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp); 3564 if (fail_sparse) { 3565 if (err == 0 && bp && BP_IS_HOLE(bp)) 3566 err = SET_ERROR(ENOENT); 3567 if (err) { 3568 if (parent) 3569 dbuf_rele(parent, NULL); 3570 return (err); 3571 } 3572 } 3573 if (err && err != ENOENT) 3574 return (err); 3575 db = dbuf_create(dn, level, blkid, parent, bp); 3576 } 3577 3578 if (fail_uncached && db->db_state != DB_CACHED) { 3579 mutex_exit(&db->db_mtx); 3580 return (SET_ERROR(ENOENT)); 3581 } 3582 3583 if (db->db_buf != NULL) { 3584 arc_buf_access(db->db_buf); 3585 ASSERT3P(db->db.db_data, ==, db->db_buf->b_data); 3586 } 3587 3588 ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf)); 3589 3590 /* 3591 * If this buffer is currently syncing out, and we are 3592 * still referencing it from db_data, we need to make a copy 3593 * of it in case we decide we want to dirty it again in this txg. 3594 */ 3595 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 3596 dn->dn_object != DMU_META_DNODE_OBJECT && 3597 db->db_state == DB_CACHED && db->db_data_pending) { 3598 dbuf_dirty_record_t *dr = db->db_data_pending; 3599 if (dr->dt.dl.dr_data == db->db_buf) 3600 dbuf_hold_copy(dn, db); 3601 } 3602 3603 if (multilist_link_active(&db->db_cache_link)) { 3604 ASSERT(zfs_refcount_is_zero(&db->db_holds)); 3605 ASSERT(db->db_caching_status == DB_DBUF_CACHE || 3606 db->db_caching_status == DB_DBUF_METADATA_CACHE); 3607 3608 multilist_remove(&dbuf_caches[db->db_caching_status].cache, db); 3609 (void) zfs_refcount_remove_many( 3610 &dbuf_caches[db->db_caching_status].size, 3611 db->db.db_size, db); 3612 3613 if (db->db_caching_status == DB_DBUF_METADATA_CACHE) { 3614 DBUF_STAT_BUMPDOWN(metadata_cache_count); 3615 } else { 3616 DBUF_STAT_BUMPDOWN(cache_levels[db->db_level]); 3617 DBUF_STAT_BUMPDOWN(cache_count); 3618 DBUF_STAT_DECR(cache_levels_bytes[db->db_level], 3619 db->db.db_size); 3620 } 3621 db->db_caching_status = DB_NO_CACHE; 3622 } 3623 (void) zfs_refcount_add(&db->db_holds, tag); 3624 DBUF_VERIFY(db); 3625 mutex_exit(&db->db_mtx); 3626 3627 /* NOTE: we can't rele the parent until after we drop the db_mtx */ 3628 if (parent) 3629 dbuf_rele(parent, NULL); 3630 3631 ASSERT3P(DB_DNODE(db), ==, dn); 3632 ASSERT3U(db->db_blkid, ==, blkid); 3633 ASSERT3U(db->db_level, ==, level); 3634 *dbp = db; 3635 3636 return (0); 3637 } 3638 3639 dmu_buf_impl_t * 3640 dbuf_hold(dnode_t *dn, uint64_t blkid, const void *tag) 3641 { 3642 return (dbuf_hold_level(dn, 0, blkid, tag)); 3643 } 3644 3645 dmu_buf_impl_t * 3646 dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, const void *tag) 3647 { 3648 dmu_buf_impl_t *db; 3649 int err = dbuf_hold_impl(dn, level, blkid, FALSE, FALSE, tag, &db); 3650 return (err ? NULL : db); 3651 } 3652 3653 void 3654 dbuf_create_bonus(dnode_t *dn) 3655 { 3656 ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 3657 3658 ASSERT(dn->dn_bonus == NULL); 3659 dn->dn_bonus = dbuf_create(dn, 0, DMU_BONUS_BLKID, dn->dn_dbuf, NULL); 3660 } 3661 3662 int 3663 dbuf_spill_set_blksz(dmu_buf_t *db_fake, uint64_t blksz, dmu_tx_t *tx) 3664 { 3665 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 3666 3667 if (db->db_blkid != DMU_SPILL_BLKID) 3668 return (SET_ERROR(ENOTSUP)); 3669 if (blksz == 0) 3670 blksz = SPA_MINBLOCKSIZE; 3671 ASSERT3U(blksz, <=, spa_maxblocksize(dmu_objset_spa(db->db_objset))); 3672 blksz = P2ROUNDUP(blksz, SPA_MINBLOCKSIZE); 3673 3674 dbuf_new_size(db, blksz, tx); 3675 3676 return (0); 3677 } 3678 3679 void 3680 dbuf_rm_spill(dnode_t *dn, dmu_tx_t *tx) 3681 { 3682 dbuf_free_range(dn, DMU_SPILL_BLKID, DMU_SPILL_BLKID, tx); 3683 } 3684 3685 #pragma weak dmu_buf_add_ref = dbuf_add_ref 3686 void 3687 dbuf_add_ref(dmu_buf_impl_t *db, const void *tag) 3688 { 3689 int64_t holds = zfs_refcount_add(&db->db_holds, tag); 3690 VERIFY3S(holds, >, 1); 3691 } 3692 3693 #pragma weak dmu_buf_try_add_ref = dbuf_try_add_ref 3694 boolean_t 3695 dbuf_try_add_ref(dmu_buf_t *db_fake, objset_t *os, uint64_t obj, uint64_t blkid, 3696 const void *tag) 3697 { 3698 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 3699 dmu_buf_impl_t *found_db; 3700 boolean_t result = B_FALSE; 3701 3702 if (blkid == DMU_BONUS_BLKID) 3703 found_db = dbuf_find_bonus(os, obj); 3704 else 3705 found_db = dbuf_find(os, obj, 0, blkid); 3706 3707 if (found_db != NULL) { 3708 if (db == found_db && dbuf_refcount(db) > db->db_dirtycnt) { 3709 (void) zfs_refcount_add(&db->db_holds, tag); 3710 result = B_TRUE; 3711 } 3712 mutex_exit(&found_db->db_mtx); 3713 } 3714 return (result); 3715 } 3716 3717 /* 3718 * If you call dbuf_rele() you had better not be referencing the dnode handle 3719 * unless you have some other direct or indirect hold on the dnode. (An indirect 3720 * hold is a hold on one of the dnode's dbufs, including the bonus buffer.) 3721 * Without that, the dbuf_rele() could lead to a dnode_rele() followed by the 3722 * dnode's parent dbuf evicting its dnode handles. 3723 */ 3724 void 3725 dbuf_rele(dmu_buf_impl_t *db, const void *tag) 3726 { 3727 mutex_enter(&db->db_mtx); 3728 dbuf_rele_and_unlock(db, tag, B_FALSE); 3729 } 3730 3731 void 3732 dmu_buf_rele(dmu_buf_t *db, const void *tag) 3733 { 3734 dbuf_rele((dmu_buf_impl_t *)db, tag); 3735 } 3736 3737 /* 3738 * dbuf_rele() for an already-locked dbuf. This is necessary to allow 3739 * db_dirtycnt and db_holds to be updated atomically. The 'evicting' 3740 * argument should be set if we are already in the dbuf-evicting code 3741 * path, in which case we don't want to recursively evict. This allows us to 3742 * avoid deeply nested stacks that would have a call flow similar to this: 3743 * 3744 * dbuf_rele()-->dbuf_rele_and_unlock()-->dbuf_evict_notify() 3745 * ^ | 3746 * | | 3747 * +-----dbuf_destroy()<--dbuf_evict_one()<--------+ 3748 * 3749 */ 3750 void 3751 dbuf_rele_and_unlock(dmu_buf_impl_t *db, const void *tag, boolean_t evicting) 3752 { 3753 int64_t holds; 3754 uint64_t size; 3755 3756 ASSERT(MUTEX_HELD(&db->db_mtx)); 3757 DBUF_VERIFY(db); 3758 3759 /* 3760 * Remove the reference to the dbuf before removing its hold on the 3761 * dnode so we can guarantee in dnode_move() that a referenced bonus 3762 * buffer has a corresponding dnode hold. 3763 */ 3764 holds = zfs_refcount_remove(&db->db_holds, tag); 3765 ASSERT(holds >= 0); 3766 3767 /* 3768 * We can't freeze indirects if there is a possibility that they 3769 * may be modified in the current syncing context. 3770 */ 3771 if (db->db_buf != NULL && 3772 holds == (db->db_level == 0 ? db->db_dirtycnt : 0)) { 3773 arc_buf_freeze(db->db_buf); 3774 } 3775 3776 if (holds == db->db_dirtycnt && 3777 db->db_level == 0 && db->db_user_immediate_evict) 3778 dbuf_evict_user(db); 3779 3780 if (holds == 0) { 3781 if (db->db_blkid == DMU_BONUS_BLKID) { 3782 dnode_t *dn; 3783 boolean_t evict_dbuf = db->db_pending_evict; 3784 3785 /* 3786 * If the dnode moves here, we cannot cross this 3787 * barrier until the move completes. 3788 */ 3789 DB_DNODE_ENTER(db); 3790 3791 dn = DB_DNODE(db); 3792 atomic_dec_32(&dn->dn_dbufs_count); 3793 3794 /* 3795 * Decrementing the dbuf count means that the bonus 3796 * buffer's dnode hold is no longer discounted in 3797 * dnode_move(). The dnode cannot move until after 3798 * the dnode_rele() below. 3799 */ 3800 DB_DNODE_EXIT(db); 3801 3802 /* 3803 * Do not reference db after its lock is dropped. 3804 * Another thread may evict it. 3805 */ 3806 mutex_exit(&db->db_mtx); 3807 3808 if (evict_dbuf) 3809 dnode_evict_bonus(dn); 3810 3811 dnode_rele(dn, db); 3812 } else if (db->db_buf == NULL) { 3813 /* 3814 * This is a special case: we never associated this 3815 * dbuf with any data allocated from the ARC. 3816 */ 3817 ASSERT(db->db_state == DB_UNCACHED || 3818 db->db_state == DB_NOFILL); 3819 dbuf_destroy(db); 3820 } else if (arc_released(db->db_buf)) { 3821 /* 3822 * This dbuf has anonymous data associated with it. 3823 */ 3824 dbuf_destroy(db); 3825 } else { 3826 boolean_t do_arc_evict = B_FALSE; 3827 blkptr_t bp; 3828 spa_t *spa = dmu_objset_spa(db->db_objset); 3829 3830 if (!DBUF_IS_CACHEABLE(db) && 3831 db->db_blkptr != NULL && 3832 !BP_IS_HOLE(db->db_blkptr) && 3833 !BP_IS_EMBEDDED(db->db_blkptr)) { 3834 do_arc_evict = B_TRUE; 3835 bp = *db->db_blkptr; 3836 } 3837 3838 if (!DBUF_IS_CACHEABLE(db) || 3839 db->db_pending_evict) { 3840 dbuf_destroy(db); 3841 } else if (!multilist_link_active(&db->db_cache_link)) { 3842 ASSERT3U(db->db_caching_status, ==, 3843 DB_NO_CACHE); 3844 3845 dbuf_cached_state_t dcs = 3846 dbuf_include_in_metadata_cache(db) ? 3847 DB_DBUF_METADATA_CACHE : DB_DBUF_CACHE; 3848 db->db_caching_status = dcs; 3849 3850 multilist_insert(&dbuf_caches[dcs].cache, db); 3851 uint64_t db_size = db->db.db_size; 3852 size = zfs_refcount_add_many( 3853 &dbuf_caches[dcs].size, db_size, db); 3854 uint8_t db_level = db->db_level; 3855 mutex_exit(&db->db_mtx); 3856 3857 if (dcs == DB_DBUF_METADATA_CACHE) { 3858 DBUF_STAT_BUMP(metadata_cache_count); 3859 DBUF_STAT_MAX( 3860 metadata_cache_size_bytes_max, 3861 size); 3862 } else { 3863 DBUF_STAT_BUMP(cache_count); 3864 DBUF_STAT_MAX(cache_size_bytes_max, 3865 size); 3866 DBUF_STAT_BUMP(cache_levels[db_level]); 3867 DBUF_STAT_INCR( 3868 cache_levels_bytes[db_level], 3869 db_size); 3870 } 3871 3872 if (dcs == DB_DBUF_CACHE && !evicting) 3873 dbuf_evict_notify(size); 3874 } 3875 3876 if (do_arc_evict) 3877 arc_freed(spa, &bp); 3878 } 3879 } else { 3880 mutex_exit(&db->db_mtx); 3881 } 3882 3883 } 3884 3885 #pragma weak dmu_buf_refcount = dbuf_refcount 3886 uint64_t 3887 dbuf_refcount(dmu_buf_impl_t *db) 3888 { 3889 return (zfs_refcount_count(&db->db_holds)); 3890 } 3891 3892 uint64_t 3893 dmu_buf_user_refcount(dmu_buf_t *db_fake) 3894 { 3895 uint64_t holds; 3896 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 3897 3898 mutex_enter(&db->db_mtx); 3899 ASSERT3U(zfs_refcount_count(&db->db_holds), >=, db->db_dirtycnt); 3900 holds = zfs_refcount_count(&db->db_holds) - db->db_dirtycnt; 3901 mutex_exit(&db->db_mtx); 3902 3903 return (holds); 3904 } 3905 3906 void * 3907 dmu_buf_replace_user(dmu_buf_t *db_fake, dmu_buf_user_t *old_user, 3908 dmu_buf_user_t *new_user) 3909 { 3910 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 3911 3912 mutex_enter(&db->db_mtx); 3913 dbuf_verify_user(db, DBVU_NOT_EVICTING); 3914 if (db->db_user == old_user) 3915 db->db_user = new_user; 3916 else 3917 old_user = db->db_user; 3918 dbuf_verify_user(db, DBVU_NOT_EVICTING); 3919 mutex_exit(&db->db_mtx); 3920 3921 return (old_user); 3922 } 3923 3924 void * 3925 dmu_buf_set_user(dmu_buf_t *db_fake, dmu_buf_user_t *user) 3926 { 3927 return (dmu_buf_replace_user(db_fake, NULL, user)); 3928 } 3929 3930 void * 3931 dmu_buf_set_user_ie(dmu_buf_t *db_fake, dmu_buf_user_t *user) 3932 { 3933 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 3934 3935 db->db_user_immediate_evict = TRUE; 3936 return (dmu_buf_set_user(db_fake, user)); 3937 } 3938 3939 void * 3940 dmu_buf_remove_user(dmu_buf_t *db_fake, dmu_buf_user_t *user) 3941 { 3942 return (dmu_buf_replace_user(db_fake, user, NULL)); 3943 } 3944 3945 void * 3946 dmu_buf_get_user(dmu_buf_t *db_fake) 3947 { 3948 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 3949 3950 dbuf_verify_user(db, DBVU_NOT_EVICTING); 3951 return (db->db_user); 3952 } 3953 3954 void 3955 dmu_buf_user_evict_wait(void) 3956 { 3957 taskq_wait(dbu_evict_taskq); 3958 } 3959 3960 blkptr_t * 3961 dmu_buf_get_blkptr(dmu_buf_t *db) 3962 { 3963 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db; 3964 return (dbi->db_blkptr); 3965 } 3966 3967 objset_t * 3968 dmu_buf_get_objset(dmu_buf_t *db) 3969 { 3970 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db; 3971 return (dbi->db_objset); 3972 } 3973 3974 dnode_t * 3975 dmu_buf_dnode_enter(dmu_buf_t *db) 3976 { 3977 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db; 3978 DB_DNODE_ENTER(dbi); 3979 return (DB_DNODE(dbi)); 3980 } 3981 3982 void 3983 dmu_buf_dnode_exit(dmu_buf_t *db) 3984 { 3985 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db; 3986 DB_DNODE_EXIT(dbi); 3987 } 3988 3989 static void 3990 dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db) 3991 { 3992 /* ASSERT(dmu_tx_is_syncing(tx) */ 3993 ASSERT(MUTEX_HELD(&db->db_mtx)); 3994 3995 if (db->db_blkptr != NULL) 3996 return; 3997 3998 if (db->db_blkid == DMU_SPILL_BLKID) { 3999 db->db_blkptr = DN_SPILL_BLKPTR(dn->dn_phys); 4000 BP_ZERO(db->db_blkptr); 4001 return; 4002 } 4003 if (db->db_level == dn->dn_phys->dn_nlevels-1) { 4004 /* 4005 * This buffer was allocated at a time when there was 4006 * no available blkptrs from the dnode, or it was 4007 * inappropriate to hook it in (i.e., nlevels mismatch). 4008 */ 4009 ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr); 4010 ASSERT(db->db_parent == NULL); 4011 db->db_parent = dn->dn_dbuf; 4012 db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid]; 4013 DBUF_VERIFY(db); 4014 } else { 4015 dmu_buf_impl_t *parent = db->db_parent; 4016 int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 4017 4018 ASSERT(dn->dn_phys->dn_nlevels > 1); 4019 if (parent == NULL) { 4020 mutex_exit(&db->db_mtx); 4021 rw_enter(&dn->dn_struct_rwlock, RW_READER); 4022 parent = dbuf_hold_level(dn, db->db_level + 1, 4023 db->db_blkid >> epbs, db); 4024 rw_exit(&dn->dn_struct_rwlock); 4025 mutex_enter(&db->db_mtx); 4026 db->db_parent = parent; 4027 } 4028 db->db_blkptr = (blkptr_t *)parent->db.db_data + 4029 (db->db_blkid & ((1ULL << epbs) - 1)); 4030 DBUF_VERIFY(db); 4031 } 4032 } 4033 4034 static void 4035 dbuf_sync_bonus(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 4036 { 4037 dmu_buf_impl_t *db = dr->dr_dbuf; 4038 void *data = dr->dt.dl.dr_data; 4039 4040 ASSERT0(db->db_level); 4041 ASSERT(MUTEX_HELD(&db->db_mtx)); 4042 ASSERT(db->db_blkid == DMU_BONUS_BLKID); 4043 ASSERT(data != NULL); 4044 4045 dnode_t *dn = dr->dr_dnode; 4046 ASSERT3U(DN_MAX_BONUS_LEN(dn->dn_phys), <=, 4047 DN_SLOTS_TO_BONUSLEN(dn->dn_phys->dn_extra_slots + 1)); 4048 memcpy(DN_BONUS(dn->dn_phys), data, DN_MAX_BONUS_LEN(dn->dn_phys)); 4049 4050 dbuf_sync_leaf_verify_bonus_dnode(dr); 4051 4052 dbuf_undirty_bonus(dr); 4053 dbuf_rele_and_unlock(db, (void *)(uintptr_t)tx->tx_txg, B_FALSE); 4054 } 4055 4056 /* 4057 * When syncing out a blocks of dnodes, adjust the block to deal with 4058 * encryption. Normally, we make sure the block is decrypted before writing 4059 * it. If we have crypt params, then we are writing a raw (encrypted) block, 4060 * from a raw receive. In this case, set the ARC buf's crypt params so 4061 * that the BP will be filled with the correct byteorder, salt, iv, and mac. 4062 */ 4063 static void 4064 dbuf_prepare_encrypted_dnode_leaf(dbuf_dirty_record_t *dr) 4065 { 4066 int err; 4067 dmu_buf_impl_t *db = dr->dr_dbuf; 4068 4069 ASSERT(MUTEX_HELD(&db->db_mtx)); 4070 ASSERT3U(db->db.db_object, ==, DMU_META_DNODE_OBJECT); 4071 ASSERT3U(db->db_level, ==, 0); 4072 4073 if (!db->db_objset->os_raw_receive && arc_is_encrypted(db->db_buf)) { 4074 zbookmark_phys_t zb; 4075 4076 /* 4077 * Unfortunately, there is currently no mechanism for 4078 * syncing context to handle decryption errors. An error 4079 * here is only possible if an attacker maliciously 4080 * changed a dnode block and updated the associated 4081 * checksums going up the block tree. 4082 */ 4083 SET_BOOKMARK(&zb, dmu_objset_id(db->db_objset), 4084 db->db.db_object, db->db_level, db->db_blkid); 4085 err = arc_untransform(db->db_buf, db->db_objset->os_spa, 4086 &zb, B_TRUE); 4087 if (err) 4088 panic("Invalid dnode block MAC"); 4089 } else if (dr->dt.dl.dr_has_raw_params) { 4090 (void) arc_release(dr->dt.dl.dr_data, db); 4091 arc_convert_to_raw(dr->dt.dl.dr_data, 4092 dmu_objset_id(db->db_objset), 4093 dr->dt.dl.dr_byteorder, DMU_OT_DNODE, 4094 dr->dt.dl.dr_salt, dr->dt.dl.dr_iv, dr->dt.dl.dr_mac); 4095 } 4096 } 4097 4098 /* 4099 * dbuf_sync_indirect() is called recursively from dbuf_sync_list() so it 4100 * is critical the we not allow the compiler to inline this function in to 4101 * dbuf_sync_list() thereby drastically bloating the stack usage. 4102 */ 4103 noinline static void 4104 dbuf_sync_indirect(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 4105 { 4106 dmu_buf_impl_t *db = dr->dr_dbuf; 4107 dnode_t *dn = dr->dr_dnode; 4108 4109 ASSERT(dmu_tx_is_syncing(tx)); 4110 4111 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 4112 4113 mutex_enter(&db->db_mtx); 4114 4115 ASSERT(db->db_level > 0); 4116 DBUF_VERIFY(db); 4117 4118 /* Read the block if it hasn't been read yet. */ 4119 if (db->db_buf == NULL) { 4120 mutex_exit(&db->db_mtx); 4121 (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED); 4122 mutex_enter(&db->db_mtx); 4123 } 4124 ASSERT3U(db->db_state, ==, DB_CACHED); 4125 ASSERT(db->db_buf != NULL); 4126 4127 /* Indirect block size must match what the dnode thinks it is. */ 4128 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 4129 dbuf_check_blkptr(dn, db); 4130 4131 /* Provide the pending dirty record to child dbufs */ 4132 db->db_data_pending = dr; 4133 4134 mutex_exit(&db->db_mtx); 4135 4136 dbuf_write(dr, db->db_buf, tx); 4137 4138 zio_t *zio = dr->dr_zio; 4139 mutex_enter(&dr->dt.di.dr_mtx); 4140 dbuf_sync_list(&dr->dt.di.dr_children, db->db_level - 1, tx); 4141 ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 4142 mutex_exit(&dr->dt.di.dr_mtx); 4143 zio_nowait(zio); 4144 } 4145 4146 /* 4147 * Verify that the size of the data in our bonus buffer does not exceed 4148 * its recorded size. 4149 * 4150 * The purpose of this verification is to catch any cases in development 4151 * where the size of a phys structure (i.e space_map_phys_t) grows and, 4152 * due to incorrect feature management, older pools expect to read more 4153 * data even though they didn't actually write it to begin with. 4154 * 4155 * For a example, this would catch an error in the feature logic where we 4156 * open an older pool and we expect to write the space map histogram of 4157 * a space map with size SPACE_MAP_SIZE_V0. 4158 */ 4159 static void 4160 dbuf_sync_leaf_verify_bonus_dnode(dbuf_dirty_record_t *dr) 4161 { 4162 #ifdef ZFS_DEBUG 4163 dnode_t *dn = dr->dr_dnode; 4164 4165 /* 4166 * Encrypted bonus buffers can have data past their bonuslen. 4167 * Skip the verification of these blocks. 4168 */ 4169 if (DMU_OT_IS_ENCRYPTED(dn->dn_bonustype)) 4170 return; 4171 4172 uint16_t bonuslen = dn->dn_phys->dn_bonuslen; 4173 uint16_t maxbonuslen = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots); 4174 ASSERT3U(bonuslen, <=, maxbonuslen); 4175 4176 arc_buf_t *datap = dr->dt.dl.dr_data; 4177 char *datap_end = ((char *)datap) + bonuslen; 4178 char *datap_max = ((char *)datap) + maxbonuslen; 4179 4180 /* ensure that everything is zero after our data */ 4181 for (; datap_end < datap_max; datap_end++) 4182 ASSERT(*datap_end == 0); 4183 #endif 4184 } 4185 4186 static blkptr_t * 4187 dbuf_lightweight_bp(dbuf_dirty_record_t *dr) 4188 { 4189 /* This must be a lightweight dirty record. */ 4190 ASSERT3P(dr->dr_dbuf, ==, NULL); 4191 dnode_t *dn = dr->dr_dnode; 4192 4193 if (dn->dn_phys->dn_nlevels == 1) { 4194 VERIFY3U(dr->dt.dll.dr_blkid, <, dn->dn_phys->dn_nblkptr); 4195 return (&dn->dn_phys->dn_blkptr[dr->dt.dll.dr_blkid]); 4196 } else { 4197 dmu_buf_impl_t *parent_db = dr->dr_parent->dr_dbuf; 4198 int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 4199 VERIFY3U(parent_db->db_level, ==, 1); 4200 VERIFY3P(parent_db->db_dnode_handle->dnh_dnode, ==, dn); 4201 VERIFY3U(dr->dt.dll.dr_blkid >> epbs, ==, parent_db->db_blkid); 4202 blkptr_t *bp = parent_db->db.db_data; 4203 return (&bp[dr->dt.dll.dr_blkid & ((1 << epbs) - 1)]); 4204 } 4205 } 4206 4207 static void 4208 dbuf_lightweight_ready(zio_t *zio) 4209 { 4210 dbuf_dirty_record_t *dr = zio->io_private; 4211 blkptr_t *bp = zio->io_bp; 4212 4213 if (zio->io_error != 0) 4214 return; 4215 4216 dnode_t *dn = dr->dr_dnode; 4217 4218 blkptr_t *bp_orig = dbuf_lightweight_bp(dr); 4219 spa_t *spa = dmu_objset_spa(dn->dn_objset); 4220 int64_t delta = bp_get_dsize_sync(spa, bp) - 4221 bp_get_dsize_sync(spa, bp_orig); 4222 dnode_diduse_space(dn, delta); 4223 4224 uint64_t blkid = dr->dt.dll.dr_blkid; 4225 mutex_enter(&dn->dn_mtx); 4226 if (blkid > dn->dn_phys->dn_maxblkid) { 4227 ASSERT0(dn->dn_objset->os_raw_receive); 4228 dn->dn_phys->dn_maxblkid = blkid; 4229 } 4230 mutex_exit(&dn->dn_mtx); 4231 4232 if (!BP_IS_EMBEDDED(bp)) { 4233 uint64_t fill = BP_IS_HOLE(bp) ? 0 : 1; 4234 BP_SET_FILL(bp, fill); 4235 } 4236 4237 dmu_buf_impl_t *parent_db; 4238 EQUIV(dr->dr_parent == NULL, dn->dn_phys->dn_nlevels == 1); 4239 if (dr->dr_parent == NULL) { 4240 parent_db = dn->dn_dbuf; 4241 } else { 4242 parent_db = dr->dr_parent->dr_dbuf; 4243 } 4244 rw_enter(&parent_db->db_rwlock, RW_WRITER); 4245 *bp_orig = *bp; 4246 rw_exit(&parent_db->db_rwlock); 4247 } 4248 4249 static void 4250 dbuf_lightweight_physdone(zio_t *zio) 4251 { 4252 dbuf_dirty_record_t *dr = zio->io_private; 4253 dsl_pool_t *dp = spa_get_dsl(zio->io_spa); 4254 ASSERT3U(dr->dr_txg, ==, zio->io_txg); 4255 4256 /* 4257 * The callback will be called io_phys_children times. Retire one 4258 * portion of our dirty space each time we are called. Any rounding 4259 * error will be cleaned up by dbuf_lightweight_done(). 4260 */ 4261 int delta = dr->dr_accounted / zio->io_phys_children; 4262 dsl_pool_undirty_space(dp, delta, zio->io_txg); 4263 } 4264 4265 static void 4266 dbuf_lightweight_done(zio_t *zio) 4267 { 4268 dbuf_dirty_record_t *dr = zio->io_private; 4269 4270 VERIFY0(zio->io_error); 4271 4272 objset_t *os = dr->dr_dnode->dn_objset; 4273 dmu_tx_t *tx = os->os_synctx; 4274 4275 if (zio->io_flags & (ZIO_FLAG_IO_REWRITE | ZIO_FLAG_NOPWRITE)) { 4276 ASSERT(BP_EQUAL(zio->io_bp, &zio->io_bp_orig)); 4277 } else { 4278 dsl_dataset_t *ds = os->os_dsl_dataset; 4279 (void) dsl_dataset_block_kill(ds, &zio->io_bp_orig, tx, B_TRUE); 4280 dsl_dataset_block_born(ds, zio->io_bp, tx); 4281 } 4282 4283 /* 4284 * See comment in dbuf_write_done(). 4285 */ 4286 if (zio->io_phys_children == 0) { 4287 dsl_pool_undirty_space(dmu_objset_pool(os), 4288 dr->dr_accounted, zio->io_txg); 4289 } else { 4290 dsl_pool_undirty_space(dmu_objset_pool(os), 4291 dr->dr_accounted % zio->io_phys_children, zio->io_txg); 4292 } 4293 4294 abd_free(dr->dt.dll.dr_abd); 4295 kmem_free(dr, sizeof (*dr)); 4296 } 4297 4298 noinline static void 4299 dbuf_sync_lightweight(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 4300 { 4301 dnode_t *dn = dr->dr_dnode; 4302 zio_t *pio; 4303 if (dn->dn_phys->dn_nlevels == 1) { 4304 pio = dn->dn_zio; 4305 } else { 4306 pio = dr->dr_parent->dr_zio; 4307 } 4308 4309 zbookmark_phys_t zb = { 4310 .zb_objset = dmu_objset_id(dn->dn_objset), 4311 .zb_object = dn->dn_object, 4312 .zb_level = 0, 4313 .zb_blkid = dr->dt.dll.dr_blkid, 4314 }; 4315 4316 /* 4317 * See comment in dbuf_write(). This is so that zio->io_bp_orig 4318 * will have the old BP in dbuf_lightweight_done(). 4319 */ 4320 dr->dr_bp_copy = *dbuf_lightweight_bp(dr); 4321 4322 dr->dr_zio = zio_write(pio, dmu_objset_spa(dn->dn_objset), 4323 dmu_tx_get_txg(tx), &dr->dr_bp_copy, dr->dt.dll.dr_abd, 4324 dn->dn_datablksz, abd_get_size(dr->dt.dll.dr_abd), 4325 &dr->dt.dll.dr_props, dbuf_lightweight_ready, NULL, 4326 dbuf_lightweight_physdone, dbuf_lightweight_done, dr, 4327 ZIO_PRIORITY_ASYNC_WRITE, 4328 ZIO_FLAG_MUSTSUCCEED | dr->dt.dll.dr_flags, &zb); 4329 4330 zio_nowait(dr->dr_zio); 4331 } 4332 4333 /* 4334 * dbuf_sync_leaf() is called recursively from dbuf_sync_list() so it is 4335 * critical the we not allow the compiler to inline this function in to 4336 * dbuf_sync_list() thereby drastically bloating the stack usage. 4337 */ 4338 noinline static void 4339 dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 4340 { 4341 arc_buf_t **datap = &dr->dt.dl.dr_data; 4342 dmu_buf_impl_t *db = dr->dr_dbuf; 4343 dnode_t *dn = dr->dr_dnode; 4344 objset_t *os; 4345 uint64_t txg = tx->tx_txg; 4346 4347 ASSERT(dmu_tx_is_syncing(tx)); 4348 4349 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 4350 4351 mutex_enter(&db->db_mtx); 4352 /* 4353 * To be synced, we must be dirtied. But we 4354 * might have been freed after the dirty. 4355 */ 4356 if (db->db_state == DB_UNCACHED) { 4357 /* This buffer has been freed since it was dirtied */ 4358 ASSERT(db->db.db_data == NULL); 4359 } else if (db->db_state == DB_FILL) { 4360 /* This buffer was freed and is now being re-filled */ 4361 ASSERT(db->db.db_data != dr->dt.dl.dr_data); 4362 } else { 4363 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_NOFILL); 4364 } 4365 DBUF_VERIFY(db); 4366 4367 if (db->db_blkid == DMU_SPILL_BLKID) { 4368 mutex_enter(&dn->dn_mtx); 4369 if (!(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR)) { 4370 /* 4371 * In the previous transaction group, the bonus buffer 4372 * was entirely used to store the attributes for the 4373 * dnode which overrode the dn_spill field. However, 4374 * when adding more attributes to the file a spill 4375 * block was required to hold the extra attributes. 4376 * 4377 * Make sure to clear the garbage left in the dn_spill 4378 * field from the previous attributes in the bonus 4379 * buffer. Otherwise, after writing out the spill 4380 * block to the new allocated dva, it will free 4381 * the old block pointed to by the invalid dn_spill. 4382 */ 4383 db->db_blkptr = NULL; 4384 } 4385 dn->dn_phys->dn_flags |= DNODE_FLAG_SPILL_BLKPTR; 4386 mutex_exit(&dn->dn_mtx); 4387 } 4388 4389 /* 4390 * If this is a bonus buffer, simply copy the bonus data into the 4391 * dnode. It will be written out when the dnode is synced (and it 4392 * will be synced, since it must have been dirty for dbuf_sync to 4393 * be called). 4394 */ 4395 if (db->db_blkid == DMU_BONUS_BLKID) { 4396 ASSERT(dr->dr_dbuf == db); 4397 dbuf_sync_bonus(dr, tx); 4398 return; 4399 } 4400 4401 os = dn->dn_objset; 4402 4403 /* 4404 * This function may have dropped the db_mtx lock allowing a dmu_sync 4405 * operation to sneak in. As a result, we need to ensure that we 4406 * don't check the dr_override_state until we have returned from 4407 * dbuf_check_blkptr. 4408 */ 4409 dbuf_check_blkptr(dn, db); 4410 4411 /* 4412 * If this buffer is in the middle of an immediate write, 4413 * wait for the synchronous IO to complete. 4414 */ 4415 while (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC) { 4416 ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT); 4417 cv_wait(&db->db_changed, &db->db_mtx); 4418 ASSERT(dr->dt.dl.dr_override_state != DR_NOT_OVERRIDDEN); 4419 } 4420 4421 /* 4422 * If this is a dnode block, ensure it is appropriately encrypted 4423 * or decrypted, depending on what we are writing to it this txg. 4424 */ 4425 if (os->os_encrypted && dn->dn_object == DMU_META_DNODE_OBJECT) 4426 dbuf_prepare_encrypted_dnode_leaf(dr); 4427 4428 if (db->db_state != DB_NOFILL && 4429 dn->dn_object != DMU_META_DNODE_OBJECT && 4430 zfs_refcount_count(&db->db_holds) > 1 && 4431 dr->dt.dl.dr_override_state != DR_OVERRIDDEN && 4432 *datap == db->db_buf) { 4433 /* 4434 * If this buffer is currently "in use" (i.e., there 4435 * are active holds and db_data still references it), 4436 * then make a copy before we start the write so that 4437 * any modifications from the open txg will not leak 4438 * into this write. 4439 * 4440 * NOTE: this copy does not need to be made for 4441 * objects only modified in the syncing context (e.g. 4442 * DNONE_DNODE blocks). 4443 */ 4444 int psize = arc_buf_size(*datap); 4445 int lsize = arc_buf_lsize(*datap); 4446 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 4447 enum zio_compress compress_type = arc_get_compression(*datap); 4448 uint8_t complevel = arc_get_complevel(*datap); 4449 4450 if (arc_is_encrypted(*datap)) { 4451 boolean_t byteorder; 4452 uint8_t salt[ZIO_DATA_SALT_LEN]; 4453 uint8_t iv[ZIO_DATA_IV_LEN]; 4454 uint8_t mac[ZIO_DATA_MAC_LEN]; 4455 4456 arc_get_raw_params(*datap, &byteorder, salt, iv, mac); 4457 *datap = arc_alloc_raw_buf(os->os_spa, db, 4458 dmu_objset_id(os), byteorder, salt, iv, mac, 4459 dn->dn_type, psize, lsize, compress_type, 4460 complevel); 4461 } else if (compress_type != ZIO_COMPRESS_OFF) { 4462 ASSERT3U(type, ==, ARC_BUFC_DATA); 4463 *datap = arc_alloc_compressed_buf(os->os_spa, db, 4464 psize, lsize, compress_type, complevel); 4465 } else { 4466 *datap = arc_alloc_buf(os->os_spa, db, type, psize); 4467 } 4468 memcpy((*datap)->b_data, db->db.db_data, psize); 4469 } 4470 db->db_data_pending = dr; 4471 4472 mutex_exit(&db->db_mtx); 4473 4474 dbuf_write(dr, *datap, tx); 4475 4476 ASSERT(!list_link_active(&dr->dr_dirty_node)); 4477 if (dn->dn_object == DMU_META_DNODE_OBJECT) { 4478 list_insert_tail(&dn->dn_dirty_records[txg & TXG_MASK], dr); 4479 } else { 4480 zio_nowait(dr->dr_zio); 4481 } 4482 } 4483 4484 void 4485 dbuf_sync_list(list_t *list, int level, dmu_tx_t *tx) 4486 { 4487 dbuf_dirty_record_t *dr; 4488 4489 while ((dr = list_head(list))) { 4490 if (dr->dr_zio != NULL) { 4491 /* 4492 * If we find an already initialized zio then we 4493 * are processing the meta-dnode, and we have finished. 4494 * The dbufs for all dnodes are put back on the list 4495 * during processing, so that we can zio_wait() 4496 * these IOs after initiating all child IOs. 4497 */ 4498 ASSERT3U(dr->dr_dbuf->db.db_object, ==, 4499 DMU_META_DNODE_OBJECT); 4500 break; 4501 } 4502 list_remove(list, dr); 4503 if (dr->dr_dbuf == NULL) { 4504 dbuf_sync_lightweight(dr, tx); 4505 } else { 4506 if (dr->dr_dbuf->db_blkid != DMU_BONUS_BLKID && 4507 dr->dr_dbuf->db_blkid != DMU_SPILL_BLKID) { 4508 VERIFY3U(dr->dr_dbuf->db_level, ==, level); 4509 } 4510 if (dr->dr_dbuf->db_level > 0) 4511 dbuf_sync_indirect(dr, tx); 4512 else 4513 dbuf_sync_leaf(dr, tx); 4514 } 4515 } 4516 } 4517 4518 static void 4519 dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb) 4520 { 4521 (void) buf; 4522 dmu_buf_impl_t *db = vdb; 4523 dnode_t *dn; 4524 blkptr_t *bp = zio->io_bp; 4525 blkptr_t *bp_orig = &zio->io_bp_orig; 4526 spa_t *spa = zio->io_spa; 4527 int64_t delta; 4528 uint64_t fill = 0; 4529 int i; 4530 4531 ASSERT3P(db->db_blkptr, !=, NULL); 4532 ASSERT3P(&db->db_data_pending->dr_bp_copy, ==, bp); 4533 4534 DB_DNODE_ENTER(db); 4535 dn = DB_DNODE(db); 4536 delta = bp_get_dsize_sync(spa, bp) - bp_get_dsize_sync(spa, bp_orig); 4537 dnode_diduse_space(dn, delta - zio->io_prev_space_delta); 4538 zio->io_prev_space_delta = delta; 4539 4540 if (bp->blk_birth != 0) { 4541 ASSERT((db->db_blkid != DMU_SPILL_BLKID && 4542 BP_GET_TYPE(bp) == dn->dn_type) || 4543 (db->db_blkid == DMU_SPILL_BLKID && 4544 BP_GET_TYPE(bp) == dn->dn_bonustype) || 4545 BP_IS_EMBEDDED(bp)); 4546 ASSERT(BP_GET_LEVEL(bp) == db->db_level); 4547 } 4548 4549 mutex_enter(&db->db_mtx); 4550 4551 #ifdef ZFS_DEBUG 4552 if (db->db_blkid == DMU_SPILL_BLKID) { 4553 ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); 4554 ASSERT(!(BP_IS_HOLE(bp)) && 4555 db->db_blkptr == DN_SPILL_BLKPTR(dn->dn_phys)); 4556 } 4557 #endif 4558 4559 if (db->db_level == 0) { 4560 mutex_enter(&dn->dn_mtx); 4561 if (db->db_blkid > dn->dn_phys->dn_maxblkid && 4562 db->db_blkid != DMU_SPILL_BLKID) { 4563 ASSERT0(db->db_objset->os_raw_receive); 4564 dn->dn_phys->dn_maxblkid = db->db_blkid; 4565 } 4566 mutex_exit(&dn->dn_mtx); 4567 4568 if (dn->dn_type == DMU_OT_DNODE) { 4569 i = 0; 4570 while (i < db->db.db_size) { 4571 dnode_phys_t *dnp = 4572 (void *)(((char *)db->db.db_data) + i); 4573 4574 i += DNODE_MIN_SIZE; 4575 if (dnp->dn_type != DMU_OT_NONE) { 4576 fill++; 4577 i += dnp->dn_extra_slots * 4578 DNODE_MIN_SIZE; 4579 } 4580 } 4581 } else { 4582 if (BP_IS_HOLE(bp)) { 4583 fill = 0; 4584 } else { 4585 fill = 1; 4586 } 4587 } 4588 } else { 4589 blkptr_t *ibp = db->db.db_data; 4590 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 4591 for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, ibp++) { 4592 if (BP_IS_HOLE(ibp)) 4593 continue; 4594 fill += BP_GET_FILL(ibp); 4595 } 4596 } 4597 DB_DNODE_EXIT(db); 4598 4599 if (!BP_IS_EMBEDDED(bp)) 4600 BP_SET_FILL(bp, fill); 4601 4602 mutex_exit(&db->db_mtx); 4603 4604 db_lock_type_t dblt = dmu_buf_lock_parent(db, RW_WRITER, FTAG); 4605 *db->db_blkptr = *bp; 4606 dmu_buf_unlock_parent(db, dblt, FTAG); 4607 } 4608 4609 /* 4610 * This function gets called just prior to running through the compression 4611 * stage of the zio pipeline. If we're an indirect block comprised of only 4612 * holes, then we want this indirect to be compressed away to a hole. In 4613 * order to do that we must zero out any information about the holes that 4614 * this indirect points to prior to before we try to compress it. 4615 */ 4616 static void 4617 dbuf_write_children_ready(zio_t *zio, arc_buf_t *buf, void *vdb) 4618 { 4619 (void) zio, (void) buf; 4620 dmu_buf_impl_t *db = vdb; 4621 dnode_t *dn; 4622 blkptr_t *bp; 4623 unsigned int epbs, i; 4624 4625 ASSERT3U(db->db_level, >, 0); 4626 DB_DNODE_ENTER(db); 4627 dn = DB_DNODE(db); 4628 epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 4629 ASSERT3U(epbs, <, 31); 4630 4631 /* Determine if all our children are holes */ 4632 for (i = 0, bp = db->db.db_data; i < 1ULL << epbs; i++, bp++) { 4633 if (!BP_IS_HOLE(bp)) 4634 break; 4635 } 4636 4637 /* 4638 * If all the children are holes, then zero them all out so that 4639 * we may get compressed away. 4640 */ 4641 if (i == 1ULL << epbs) { 4642 /* 4643 * We only found holes. Grab the rwlock to prevent 4644 * anybody from reading the blocks we're about to 4645 * zero out. 4646 */ 4647 rw_enter(&db->db_rwlock, RW_WRITER); 4648 memset(db->db.db_data, 0, db->db.db_size); 4649 rw_exit(&db->db_rwlock); 4650 } 4651 DB_DNODE_EXIT(db); 4652 } 4653 4654 /* 4655 * The SPA will call this callback several times for each zio - once 4656 * for every physical child i/o (zio->io_phys_children times). This 4657 * allows the DMU to monitor the progress of each logical i/o. For example, 4658 * there may be 2 copies of an indirect block, or many fragments of a RAID-Z 4659 * block. There may be a long delay before all copies/fragments are completed, 4660 * so this callback allows us to retire dirty space gradually, as the physical 4661 * i/os complete. 4662 */ 4663 static void 4664 dbuf_write_physdone(zio_t *zio, arc_buf_t *buf, void *arg) 4665 { 4666 (void) buf; 4667 dmu_buf_impl_t *db = arg; 4668 objset_t *os = db->db_objset; 4669 dsl_pool_t *dp = dmu_objset_pool(os); 4670 dbuf_dirty_record_t *dr; 4671 int delta = 0; 4672 4673 dr = db->db_data_pending; 4674 ASSERT3U(dr->dr_txg, ==, zio->io_txg); 4675 4676 /* 4677 * The callback will be called io_phys_children times. Retire one 4678 * portion of our dirty space each time we are called. Any rounding 4679 * error will be cleaned up by dbuf_write_done(). 4680 */ 4681 delta = dr->dr_accounted / zio->io_phys_children; 4682 dsl_pool_undirty_space(dp, delta, zio->io_txg); 4683 } 4684 4685 static void 4686 dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb) 4687 { 4688 (void) buf; 4689 dmu_buf_impl_t *db = vdb; 4690 blkptr_t *bp_orig = &zio->io_bp_orig; 4691 blkptr_t *bp = db->db_blkptr; 4692 objset_t *os = db->db_objset; 4693 dmu_tx_t *tx = os->os_synctx; 4694 4695 ASSERT0(zio->io_error); 4696 ASSERT(db->db_blkptr == bp); 4697 4698 /* 4699 * For nopwrites and rewrites we ensure that the bp matches our 4700 * original and bypass all the accounting. 4701 */ 4702 if (zio->io_flags & (ZIO_FLAG_IO_REWRITE | ZIO_FLAG_NOPWRITE)) { 4703 ASSERT(BP_EQUAL(bp, bp_orig)); 4704 } else { 4705 dsl_dataset_t *ds = os->os_dsl_dataset; 4706 (void) dsl_dataset_block_kill(ds, bp_orig, tx, B_TRUE); 4707 dsl_dataset_block_born(ds, bp, tx); 4708 } 4709 4710 mutex_enter(&db->db_mtx); 4711 4712 DBUF_VERIFY(db); 4713 4714 dbuf_dirty_record_t *dr = db->db_data_pending; 4715 dnode_t *dn = dr->dr_dnode; 4716 ASSERT(!list_link_active(&dr->dr_dirty_node)); 4717 ASSERT(dr->dr_dbuf == db); 4718 ASSERT(list_next(&db->db_dirty_records, dr) == NULL); 4719 list_remove(&db->db_dirty_records, dr); 4720 4721 #ifdef ZFS_DEBUG 4722 if (db->db_blkid == DMU_SPILL_BLKID) { 4723 ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); 4724 ASSERT(!(BP_IS_HOLE(db->db_blkptr)) && 4725 db->db_blkptr == DN_SPILL_BLKPTR(dn->dn_phys)); 4726 } 4727 #endif 4728 4729 if (db->db_level == 0) { 4730 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 4731 ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN); 4732 if (db->db_state != DB_NOFILL) { 4733 if (dr->dt.dl.dr_data != db->db_buf) 4734 arc_buf_destroy(dr->dt.dl.dr_data, db); 4735 } 4736 } else { 4737 ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 4738 ASSERT3U(db->db.db_size, ==, 1 << dn->dn_phys->dn_indblkshift); 4739 if (!BP_IS_HOLE(db->db_blkptr)) { 4740 int epbs __maybe_unused = dn->dn_phys->dn_indblkshift - 4741 SPA_BLKPTRSHIFT; 4742 ASSERT3U(db->db_blkid, <=, 4743 dn->dn_phys->dn_maxblkid >> (db->db_level * epbs)); 4744 ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, 4745 db->db.db_size); 4746 } 4747 mutex_destroy(&dr->dt.di.dr_mtx); 4748 list_destroy(&dr->dt.di.dr_children); 4749 } 4750 4751 cv_broadcast(&db->db_changed); 4752 ASSERT(db->db_dirtycnt > 0); 4753 db->db_dirtycnt -= 1; 4754 db->db_data_pending = NULL; 4755 dbuf_rele_and_unlock(db, (void *)(uintptr_t)tx->tx_txg, B_FALSE); 4756 4757 /* 4758 * If we didn't do a physical write in this ZIO and we 4759 * still ended up here, it means that the space of the 4760 * dbuf that we just released (and undirtied) above hasn't 4761 * been marked as undirtied in the pool's accounting. 4762 * 4763 * Thus, we undirty that space in the pool's view of the 4764 * world here. For physical writes this type of update 4765 * happens in dbuf_write_physdone(). 4766 * 4767 * If we did a physical write, cleanup any rounding errors 4768 * that came up due to writing multiple copies of a block 4769 * on disk [see dbuf_write_physdone()]. 4770 */ 4771 if (zio->io_phys_children == 0) { 4772 dsl_pool_undirty_space(dmu_objset_pool(os), 4773 dr->dr_accounted, zio->io_txg); 4774 } else { 4775 dsl_pool_undirty_space(dmu_objset_pool(os), 4776 dr->dr_accounted % zio->io_phys_children, zio->io_txg); 4777 } 4778 4779 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 4780 } 4781 4782 static void 4783 dbuf_write_nofill_ready(zio_t *zio) 4784 { 4785 dbuf_write_ready(zio, NULL, zio->io_private); 4786 } 4787 4788 static void 4789 dbuf_write_nofill_done(zio_t *zio) 4790 { 4791 dbuf_write_done(zio, NULL, zio->io_private); 4792 } 4793 4794 static void 4795 dbuf_write_override_ready(zio_t *zio) 4796 { 4797 dbuf_dirty_record_t *dr = zio->io_private; 4798 dmu_buf_impl_t *db = dr->dr_dbuf; 4799 4800 dbuf_write_ready(zio, NULL, db); 4801 } 4802 4803 static void 4804 dbuf_write_override_done(zio_t *zio) 4805 { 4806 dbuf_dirty_record_t *dr = zio->io_private; 4807 dmu_buf_impl_t *db = dr->dr_dbuf; 4808 blkptr_t *obp = &dr->dt.dl.dr_overridden_by; 4809 4810 mutex_enter(&db->db_mtx); 4811 if (!BP_EQUAL(zio->io_bp, obp)) { 4812 if (!BP_IS_HOLE(obp)) 4813 dsl_free(spa_get_dsl(zio->io_spa), zio->io_txg, obp); 4814 arc_release(dr->dt.dl.dr_data, db); 4815 } 4816 mutex_exit(&db->db_mtx); 4817 4818 dbuf_write_done(zio, NULL, db); 4819 4820 if (zio->io_abd != NULL) 4821 abd_free(zio->io_abd); 4822 } 4823 4824 typedef struct dbuf_remap_impl_callback_arg { 4825 objset_t *drica_os; 4826 uint64_t drica_blk_birth; 4827 dmu_tx_t *drica_tx; 4828 } dbuf_remap_impl_callback_arg_t; 4829 4830 static void 4831 dbuf_remap_impl_callback(uint64_t vdev, uint64_t offset, uint64_t size, 4832 void *arg) 4833 { 4834 dbuf_remap_impl_callback_arg_t *drica = arg; 4835 objset_t *os = drica->drica_os; 4836 spa_t *spa = dmu_objset_spa(os); 4837 dmu_tx_t *tx = drica->drica_tx; 4838 4839 ASSERT(dsl_pool_sync_context(spa_get_dsl(spa))); 4840 4841 if (os == spa_meta_objset(spa)) { 4842 spa_vdev_indirect_mark_obsolete(spa, vdev, offset, size, tx); 4843 } else { 4844 dsl_dataset_block_remapped(dmu_objset_ds(os), vdev, offset, 4845 size, drica->drica_blk_birth, tx); 4846 } 4847 } 4848 4849 static void 4850 dbuf_remap_impl(dnode_t *dn, blkptr_t *bp, krwlock_t *rw, dmu_tx_t *tx) 4851 { 4852 blkptr_t bp_copy = *bp; 4853 spa_t *spa = dmu_objset_spa(dn->dn_objset); 4854 dbuf_remap_impl_callback_arg_t drica; 4855 4856 ASSERT(dsl_pool_sync_context(spa_get_dsl(spa))); 4857 4858 drica.drica_os = dn->dn_objset; 4859 drica.drica_blk_birth = bp->blk_birth; 4860 drica.drica_tx = tx; 4861 if (spa_remap_blkptr(spa, &bp_copy, dbuf_remap_impl_callback, 4862 &drica)) { 4863 /* 4864 * If the blkptr being remapped is tracked by a livelist, 4865 * then we need to make sure the livelist reflects the update. 4866 * First, cancel out the old blkptr by appending a 'FREE' 4867 * entry. Next, add an 'ALLOC' to track the new version. This 4868 * way we avoid trying to free an inaccurate blkptr at delete. 4869 * Note that embedded blkptrs are not tracked in livelists. 4870 */ 4871 if (dn->dn_objset != spa_meta_objset(spa)) { 4872 dsl_dataset_t *ds = dmu_objset_ds(dn->dn_objset); 4873 if (dsl_deadlist_is_open(&ds->ds_dir->dd_livelist) && 4874 bp->blk_birth > ds->ds_dir->dd_origin_txg) { 4875 ASSERT(!BP_IS_EMBEDDED(bp)); 4876 ASSERT(dsl_dir_is_clone(ds->ds_dir)); 4877 ASSERT(spa_feature_is_enabled(spa, 4878 SPA_FEATURE_LIVELIST)); 4879 bplist_append(&ds->ds_dir->dd_pending_frees, 4880 bp); 4881 bplist_append(&ds->ds_dir->dd_pending_allocs, 4882 &bp_copy); 4883 } 4884 } 4885 4886 /* 4887 * The db_rwlock prevents dbuf_read_impl() from 4888 * dereferencing the BP while we are changing it. To 4889 * avoid lock contention, only grab it when we are actually 4890 * changing the BP. 4891 */ 4892 if (rw != NULL) 4893 rw_enter(rw, RW_WRITER); 4894 *bp = bp_copy; 4895 if (rw != NULL) 4896 rw_exit(rw); 4897 } 4898 } 4899 4900 /* 4901 * Remap any existing BP's to concrete vdevs, if possible. 4902 */ 4903 static void 4904 dbuf_remap(dnode_t *dn, dmu_buf_impl_t *db, dmu_tx_t *tx) 4905 { 4906 spa_t *spa = dmu_objset_spa(db->db_objset); 4907 ASSERT(dsl_pool_sync_context(spa_get_dsl(spa))); 4908 4909 if (!spa_feature_is_active(spa, SPA_FEATURE_DEVICE_REMOVAL)) 4910 return; 4911 4912 if (db->db_level > 0) { 4913 blkptr_t *bp = db->db.db_data; 4914 for (int i = 0; i < db->db.db_size >> SPA_BLKPTRSHIFT; i++) { 4915 dbuf_remap_impl(dn, &bp[i], &db->db_rwlock, tx); 4916 } 4917 } else if (db->db.db_object == DMU_META_DNODE_OBJECT) { 4918 dnode_phys_t *dnp = db->db.db_data; 4919 ASSERT3U(db->db_dnode_handle->dnh_dnode->dn_type, ==, 4920 DMU_OT_DNODE); 4921 for (int i = 0; i < db->db.db_size >> DNODE_SHIFT; 4922 i += dnp[i].dn_extra_slots + 1) { 4923 for (int j = 0; j < dnp[i].dn_nblkptr; j++) { 4924 krwlock_t *lock = (dn->dn_dbuf == NULL ? NULL : 4925 &dn->dn_dbuf->db_rwlock); 4926 dbuf_remap_impl(dn, &dnp[i].dn_blkptr[j], lock, 4927 tx); 4928 } 4929 } 4930 } 4931 } 4932 4933 4934 /* Issue I/O to commit a dirty buffer to disk. */ 4935 static void 4936 dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx) 4937 { 4938 dmu_buf_impl_t *db = dr->dr_dbuf; 4939 dnode_t *dn = dr->dr_dnode; 4940 objset_t *os; 4941 dmu_buf_impl_t *parent = db->db_parent; 4942 uint64_t txg = tx->tx_txg; 4943 zbookmark_phys_t zb; 4944 zio_prop_t zp; 4945 zio_t *pio; /* parent I/O */ 4946 int wp_flag = 0; 4947 4948 ASSERT(dmu_tx_is_syncing(tx)); 4949 4950 os = dn->dn_objset; 4951 4952 if (db->db_state != DB_NOFILL) { 4953 if (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE) { 4954 /* 4955 * Private object buffers are released here rather 4956 * than in dbuf_dirty() since they are only modified 4957 * in the syncing context and we don't want the 4958 * overhead of making multiple copies of the data. 4959 */ 4960 if (BP_IS_HOLE(db->db_blkptr)) { 4961 arc_buf_thaw(data); 4962 } else { 4963 dbuf_release_bp(db); 4964 } 4965 dbuf_remap(dn, db, tx); 4966 } 4967 } 4968 4969 if (parent != dn->dn_dbuf) { 4970 /* Our parent is an indirect block. */ 4971 /* We have a dirty parent that has been scheduled for write. */ 4972 ASSERT(parent && parent->db_data_pending); 4973 /* Our parent's buffer is one level closer to the dnode. */ 4974 ASSERT(db->db_level == parent->db_level-1); 4975 /* 4976 * We're about to modify our parent's db_data by modifying 4977 * our block pointer, so the parent must be released. 4978 */ 4979 ASSERT(arc_released(parent->db_buf)); 4980 pio = parent->db_data_pending->dr_zio; 4981 } else { 4982 /* Our parent is the dnode itself. */ 4983 ASSERT((db->db_level == dn->dn_phys->dn_nlevels-1 && 4984 db->db_blkid != DMU_SPILL_BLKID) || 4985 (db->db_blkid == DMU_SPILL_BLKID && db->db_level == 0)); 4986 if (db->db_blkid != DMU_SPILL_BLKID) 4987 ASSERT3P(db->db_blkptr, ==, 4988 &dn->dn_phys->dn_blkptr[db->db_blkid]); 4989 pio = dn->dn_zio; 4990 } 4991 4992 ASSERT(db->db_level == 0 || data == db->db_buf); 4993 ASSERT3U(db->db_blkptr->blk_birth, <=, txg); 4994 ASSERT(pio); 4995 4996 SET_BOOKMARK(&zb, os->os_dsl_dataset ? 4997 os->os_dsl_dataset->ds_object : DMU_META_OBJSET, 4998 db->db.db_object, db->db_level, db->db_blkid); 4999 5000 if (db->db_blkid == DMU_SPILL_BLKID) 5001 wp_flag = WP_SPILL; 5002 wp_flag |= (db->db_state == DB_NOFILL) ? WP_NOFILL : 0; 5003 5004 dmu_write_policy(os, dn, db->db_level, wp_flag, &zp); 5005 5006 /* 5007 * We copy the blkptr now (rather than when we instantiate the dirty 5008 * record), because its value can change between open context and 5009 * syncing context. We do not need to hold dn_struct_rwlock to read 5010 * db_blkptr because we are in syncing context. 5011 */ 5012 dr->dr_bp_copy = *db->db_blkptr; 5013 5014 if (db->db_level == 0 && 5015 dr->dt.dl.dr_override_state == DR_OVERRIDDEN) { 5016 /* 5017 * The BP for this block has been provided by open context 5018 * (by dmu_sync() or dmu_buf_write_embedded()). 5019 */ 5020 abd_t *contents = (data != NULL) ? 5021 abd_get_from_buf(data->b_data, arc_buf_size(data)) : NULL; 5022 5023 dr->dr_zio = zio_write(pio, os->os_spa, txg, &dr->dr_bp_copy, 5024 contents, db->db.db_size, db->db.db_size, &zp, 5025 dbuf_write_override_ready, NULL, NULL, 5026 dbuf_write_override_done, 5027 dr, ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); 5028 mutex_enter(&db->db_mtx); 5029 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 5030 zio_write_override(dr->dr_zio, &dr->dt.dl.dr_overridden_by, 5031 dr->dt.dl.dr_copies, dr->dt.dl.dr_nopwrite); 5032 mutex_exit(&db->db_mtx); 5033 } else if (db->db_state == DB_NOFILL) { 5034 ASSERT(zp.zp_checksum == ZIO_CHECKSUM_OFF || 5035 zp.zp_checksum == ZIO_CHECKSUM_NOPARITY); 5036 dr->dr_zio = zio_write(pio, os->os_spa, txg, 5037 &dr->dr_bp_copy, NULL, db->db.db_size, db->db.db_size, &zp, 5038 dbuf_write_nofill_ready, NULL, NULL, 5039 dbuf_write_nofill_done, db, 5040 ZIO_PRIORITY_ASYNC_WRITE, 5041 ZIO_FLAG_MUSTSUCCEED | ZIO_FLAG_NODATA, &zb); 5042 } else { 5043 ASSERT(arc_released(data)); 5044 5045 /* 5046 * For indirect blocks, we want to setup the children 5047 * ready callback so that we can properly handle an indirect 5048 * block that only contains holes. 5049 */ 5050 arc_write_done_func_t *children_ready_cb = NULL; 5051 if (db->db_level != 0) 5052 children_ready_cb = dbuf_write_children_ready; 5053 5054 dr->dr_zio = arc_write(pio, os->os_spa, txg, 5055 &dr->dr_bp_copy, data, dbuf_is_l2cacheable(db), 5056 &zp, dbuf_write_ready, 5057 children_ready_cb, dbuf_write_physdone, 5058 dbuf_write_done, db, ZIO_PRIORITY_ASYNC_WRITE, 5059 ZIO_FLAG_MUSTSUCCEED, &zb); 5060 } 5061 } 5062 5063 EXPORT_SYMBOL(dbuf_find); 5064 EXPORT_SYMBOL(dbuf_is_metadata); 5065 EXPORT_SYMBOL(dbuf_destroy); 5066 EXPORT_SYMBOL(dbuf_loan_arcbuf); 5067 EXPORT_SYMBOL(dbuf_whichblock); 5068 EXPORT_SYMBOL(dbuf_read); 5069 EXPORT_SYMBOL(dbuf_unoverride); 5070 EXPORT_SYMBOL(dbuf_free_range); 5071 EXPORT_SYMBOL(dbuf_new_size); 5072 EXPORT_SYMBOL(dbuf_release_bp); 5073 EXPORT_SYMBOL(dbuf_dirty); 5074 EXPORT_SYMBOL(dmu_buf_set_crypt_params); 5075 EXPORT_SYMBOL(dmu_buf_will_dirty); 5076 EXPORT_SYMBOL(dmu_buf_is_dirty); 5077 EXPORT_SYMBOL(dmu_buf_will_not_fill); 5078 EXPORT_SYMBOL(dmu_buf_will_fill); 5079 EXPORT_SYMBOL(dmu_buf_fill_done); 5080 EXPORT_SYMBOL(dmu_buf_rele); 5081 EXPORT_SYMBOL(dbuf_assign_arcbuf); 5082 EXPORT_SYMBOL(dbuf_prefetch); 5083 EXPORT_SYMBOL(dbuf_hold_impl); 5084 EXPORT_SYMBOL(dbuf_hold); 5085 EXPORT_SYMBOL(dbuf_hold_level); 5086 EXPORT_SYMBOL(dbuf_create_bonus); 5087 EXPORT_SYMBOL(dbuf_spill_set_blksz); 5088 EXPORT_SYMBOL(dbuf_rm_spill); 5089 EXPORT_SYMBOL(dbuf_add_ref); 5090 EXPORT_SYMBOL(dbuf_rele); 5091 EXPORT_SYMBOL(dbuf_rele_and_unlock); 5092 EXPORT_SYMBOL(dbuf_refcount); 5093 EXPORT_SYMBOL(dbuf_sync_list); 5094 EXPORT_SYMBOL(dmu_buf_set_user); 5095 EXPORT_SYMBOL(dmu_buf_set_user_ie); 5096 EXPORT_SYMBOL(dmu_buf_get_user); 5097 EXPORT_SYMBOL(dmu_buf_get_blkptr); 5098 5099 ZFS_MODULE_PARAM(zfs_dbuf_cache, dbuf_cache_, max_bytes, ULONG, ZMOD_RW, 5100 "Maximum size in bytes of the dbuf cache."); 5101 5102 ZFS_MODULE_PARAM(zfs_dbuf_cache, dbuf_cache_, hiwater_pct, UINT, ZMOD_RW, 5103 "Percentage over dbuf_cache_max_bytes for direct dbuf eviction."); 5104 5105 ZFS_MODULE_PARAM(zfs_dbuf_cache, dbuf_cache_, lowater_pct, UINT, ZMOD_RW, 5106 "Percentage below dbuf_cache_max_bytes when dbuf eviction stops."); 5107 5108 ZFS_MODULE_PARAM(zfs_dbuf, dbuf_, metadata_cache_max_bytes, ULONG, ZMOD_RW, 5109 "Maximum size in bytes of dbuf metadata cache."); 5110 5111 ZFS_MODULE_PARAM(zfs_dbuf, dbuf_, cache_shift, INT, ZMOD_RW, 5112 "Set size of dbuf cache to log2 fraction of arc size."); 5113 5114 ZFS_MODULE_PARAM(zfs_dbuf, dbuf_, metadata_cache_shift, INT, ZMOD_RW, 5115 "Set size of dbuf metadata cache to log2 fraction of arc size."); 5116