1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright 2011 Nexenta Systems, Inc. All rights reserved. 24 * Copyright (c) 2013 by Delphix. All rights reserved. 25 * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. 26 * Copyright (c) 2013, Joyent, Inc. All rights reserved. 27 */ 28 29 #include <sys/zfs_context.h> 30 #include <sys/dmu.h> 31 #include <sys/dmu_send.h> 32 #include <sys/dmu_impl.h> 33 #include <sys/dbuf.h> 34 #include <sys/dmu_objset.h> 35 #include <sys/dsl_dataset.h> 36 #include <sys/dsl_dir.h> 37 #include <sys/dmu_tx.h> 38 #include <sys/spa.h> 39 #include <sys/zio.h> 40 #include <sys/dmu_zfetch.h> 41 #include <sys/sa.h> 42 #include <sys/sa_impl.h> 43 44 /* 45 * Number of times that zfs_free_range() took the slow path while doing 46 * a zfs receive. A nonzero value indicates a potential performance problem. 47 */ 48 uint64_t zfs_free_range_recv_miss; 49 50 static void dbuf_destroy(dmu_buf_impl_t *db); 51 static boolean_t dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx); 52 static void dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx); 53 54 /* 55 * Global data structures and functions for the dbuf cache. 56 */ 57 static kmem_cache_t *dbuf_cache; 58 59 /* ARGSUSED */ 60 static int 61 dbuf_cons(void *vdb, void *unused, int kmflag) 62 { 63 dmu_buf_impl_t *db = vdb; 64 bzero(db, sizeof (dmu_buf_impl_t)); 65 66 mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL); 67 cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL); 68 refcount_create(&db->db_holds); 69 return (0); 70 } 71 72 /* ARGSUSED */ 73 static void 74 dbuf_dest(void *vdb, void *unused) 75 { 76 dmu_buf_impl_t *db = vdb; 77 mutex_destroy(&db->db_mtx); 78 cv_destroy(&db->db_changed); 79 refcount_destroy(&db->db_holds); 80 } 81 82 /* 83 * dbuf hash table routines 84 */ 85 static dbuf_hash_table_t dbuf_hash_table; 86 87 static uint64_t dbuf_hash_count; 88 89 static uint64_t 90 dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid) 91 { 92 uintptr_t osv = (uintptr_t)os; 93 uint64_t crc = -1ULL; 94 95 ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 96 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (lvl)) & 0xFF]; 97 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF]; 98 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF]; 99 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF]; 100 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 0)) & 0xFF]; 101 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 8)) & 0xFF]; 102 103 crc ^= (osv>>14) ^ (obj>>16) ^ (blkid>>16); 104 105 return (crc); 106 } 107 108 #define DBUF_HASH(os, obj, level, blkid) dbuf_hash(os, obj, level, blkid); 109 110 #define DBUF_EQUAL(dbuf, os, obj, level, blkid) \ 111 ((dbuf)->db.db_object == (obj) && \ 112 (dbuf)->db_objset == (os) && \ 113 (dbuf)->db_level == (level) && \ 114 (dbuf)->db_blkid == (blkid)) 115 116 dmu_buf_impl_t * 117 dbuf_find(dnode_t *dn, uint8_t level, uint64_t blkid) 118 { 119 dbuf_hash_table_t *h = &dbuf_hash_table; 120 objset_t *os = dn->dn_objset; 121 uint64_t obj = dn->dn_object; 122 uint64_t hv = DBUF_HASH(os, obj, level, blkid); 123 uint64_t idx = hv & h->hash_table_mask; 124 dmu_buf_impl_t *db; 125 126 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 127 for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) { 128 if (DBUF_EQUAL(db, os, obj, level, blkid)) { 129 mutex_enter(&db->db_mtx); 130 if (db->db_state != DB_EVICTING) { 131 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 132 return (db); 133 } 134 mutex_exit(&db->db_mtx); 135 } 136 } 137 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 138 return (NULL); 139 } 140 141 /* 142 * Insert an entry into the hash table. If there is already an element 143 * equal to elem in the hash table, then the already existing element 144 * will be returned and the new element will not be inserted. 145 * Otherwise returns NULL. 146 */ 147 static dmu_buf_impl_t * 148 dbuf_hash_insert(dmu_buf_impl_t *db) 149 { 150 dbuf_hash_table_t *h = &dbuf_hash_table; 151 objset_t *os = db->db_objset; 152 uint64_t obj = db->db.db_object; 153 int level = db->db_level; 154 uint64_t blkid = db->db_blkid; 155 uint64_t hv = DBUF_HASH(os, obj, level, blkid); 156 uint64_t idx = hv & h->hash_table_mask; 157 dmu_buf_impl_t *dbf; 158 159 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 160 for (dbf = h->hash_table[idx]; dbf != NULL; dbf = dbf->db_hash_next) { 161 if (DBUF_EQUAL(dbf, os, obj, level, blkid)) { 162 mutex_enter(&dbf->db_mtx); 163 if (dbf->db_state != DB_EVICTING) { 164 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 165 return (dbf); 166 } 167 mutex_exit(&dbf->db_mtx); 168 } 169 } 170 171 mutex_enter(&db->db_mtx); 172 db->db_hash_next = h->hash_table[idx]; 173 h->hash_table[idx] = db; 174 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 175 atomic_add_64(&dbuf_hash_count, 1); 176 177 return (NULL); 178 } 179 180 /* 181 * Remove an entry from the hash table. This operation will 182 * fail if there are any existing holds on the db. 183 */ 184 static void 185 dbuf_hash_remove(dmu_buf_impl_t *db) 186 { 187 dbuf_hash_table_t *h = &dbuf_hash_table; 188 uint64_t hv = DBUF_HASH(db->db_objset, db->db.db_object, 189 db->db_level, db->db_blkid); 190 uint64_t idx = hv & h->hash_table_mask; 191 dmu_buf_impl_t *dbf, **dbp; 192 193 /* 194 * We musn't hold db_mtx to maintin lock ordering: 195 * DBUF_HASH_MUTEX > db_mtx. 196 */ 197 ASSERT(refcount_is_zero(&db->db_holds)); 198 ASSERT(db->db_state == DB_EVICTING); 199 ASSERT(!MUTEX_HELD(&db->db_mtx)); 200 201 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 202 dbp = &h->hash_table[idx]; 203 while ((dbf = *dbp) != db) { 204 dbp = &dbf->db_hash_next; 205 ASSERT(dbf != NULL); 206 } 207 *dbp = db->db_hash_next; 208 db->db_hash_next = NULL; 209 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 210 atomic_add_64(&dbuf_hash_count, -1); 211 } 212 213 static arc_evict_func_t dbuf_do_evict; 214 215 static void 216 dbuf_evict_user(dmu_buf_impl_t *db) 217 { 218 ASSERT(MUTEX_HELD(&db->db_mtx)); 219 220 if (db->db_level != 0 || db->db_evict_func == NULL) 221 return; 222 223 if (db->db_user_data_ptr_ptr) 224 *db->db_user_data_ptr_ptr = db->db.db_data; 225 db->db_evict_func(&db->db, db->db_user_ptr); 226 db->db_user_ptr = NULL; 227 db->db_user_data_ptr_ptr = NULL; 228 db->db_evict_func = NULL; 229 } 230 231 boolean_t 232 dbuf_is_metadata(dmu_buf_impl_t *db) 233 { 234 if (db->db_level > 0) { 235 return (B_TRUE); 236 } else { 237 boolean_t is_metadata; 238 239 DB_DNODE_ENTER(db); 240 is_metadata = DMU_OT_IS_METADATA(DB_DNODE(db)->dn_type); 241 DB_DNODE_EXIT(db); 242 243 return (is_metadata); 244 } 245 } 246 247 void 248 dbuf_evict(dmu_buf_impl_t *db) 249 { 250 ASSERT(MUTEX_HELD(&db->db_mtx)); 251 ASSERT(db->db_buf == NULL); 252 ASSERT(db->db_data_pending == NULL); 253 254 dbuf_clear(db); 255 dbuf_destroy(db); 256 } 257 258 void 259 dbuf_init(void) 260 { 261 uint64_t hsize = 1ULL << 16; 262 dbuf_hash_table_t *h = &dbuf_hash_table; 263 int i; 264 265 /* 266 * The hash table is big enough to fill all of physical memory 267 * with an average 4K block size. The table will take up 268 * totalmem*sizeof(void*)/4K (i.e. 2MB/GB with 8-byte pointers). 269 */ 270 while (hsize * 4096 < physmem * PAGESIZE) 271 hsize <<= 1; 272 273 retry: 274 h->hash_table_mask = hsize - 1; 275 h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP); 276 if (h->hash_table == NULL) { 277 /* XXX - we should really return an error instead of assert */ 278 ASSERT(hsize > (1ULL << 10)); 279 hsize >>= 1; 280 goto retry; 281 } 282 283 dbuf_cache = kmem_cache_create("dmu_buf_impl_t", 284 sizeof (dmu_buf_impl_t), 285 0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0); 286 287 for (i = 0; i < DBUF_MUTEXES; i++) 288 mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL); 289 } 290 291 void 292 dbuf_fini(void) 293 { 294 dbuf_hash_table_t *h = &dbuf_hash_table; 295 int i; 296 297 for (i = 0; i < DBUF_MUTEXES; i++) 298 mutex_destroy(&h->hash_mutexes[i]); 299 kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *)); 300 kmem_cache_destroy(dbuf_cache); 301 } 302 303 /* 304 * Other stuff. 305 */ 306 307 #ifdef ZFS_DEBUG 308 static void 309 dbuf_verify(dmu_buf_impl_t *db) 310 { 311 dnode_t *dn; 312 dbuf_dirty_record_t *dr; 313 314 ASSERT(MUTEX_HELD(&db->db_mtx)); 315 316 if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY)) 317 return; 318 319 ASSERT(db->db_objset != NULL); 320 DB_DNODE_ENTER(db); 321 dn = DB_DNODE(db); 322 if (dn == NULL) { 323 ASSERT(db->db_parent == NULL); 324 ASSERT(db->db_blkptr == NULL); 325 } else { 326 ASSERT3U(db->db.db_object, ==, dn->dn_object); 327 ASSERT3P(db->db_objset, ==, dn->dn_objset); 328 ASSERT3U(db->db_level, <, dn->dn_nlevels); 329 ASSERT(db->db_blkid == DMU_BONUS_BLKID || 330 db->db_blkid == DMU_SPILL_BLKID || 331 !list_is_empty(&dn->dn_dbufs)); 332 } 333 if (db->db_blkid == DMU_BONUS_BLKID) { 334 ASSERT(dn != NULL); 335 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 336 ASSERT3U(db->db.db_offset, ==, DMU_BONUS_BLKID); 337 } else if (db->db_blkid == DMU_SPILL_BLKID) { 338 ASSERT(dn != NULL); 339 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 340 ASSERT0(db->db.db_offset); 341 } else { 342 ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size); 343 } 344 345 for (dr = db->db_data_pending; dr != NULL; dr = dr->dr_next) 346 ASSERT(dr->dr_dbuf == db); 347 348 for (dr = db->db_last_dirty; dr != NULL; dr = dr->dr_next) 349 ASSERT(dr->dr_dbuf == db); 350 351 /* 352 * We can't assert that db_size matches dn_datablksz because it 353 * can be momentarily different when another thread is doing 354 * dnode_set_blksz(). 355 */ 356 if (db->db_level == 0 && db->db.db_object == DMU_META_DNODE_OBJECT) { 357 dr = db->db_data_pending; 358 /* 359 * It should only be modified in syncing context, so 360 * make sure we only have one copy of the data. 361 */ 362 ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf); 363 } 364 365 /* verify db->db_blkptr */ 366 if (db->db_blkptr) { 367 if (db->db_parent == dn->dn_dbuf) { 368 /* db is pointed to by the dnode */ 369 /* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */ 370 if (DMU_OBJECT_IS_SPECIAL(db->db.db_object)) 371 ASSERT(db->db_parent == NULL); 372 else 373 ASSERT(db->db_parent != NULL); 374 if (db->db_blkid != DMU_SPILL_BLKID) 375 ASSERT3P(db->db_blkptr, ==, 376 &dn->dn_phys->dn_blkptr[db->db_blkid]); 377 } else { 378 /* db is pointed to by an indirect block */ 379 int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT; 380 ASSERT3U(db->db_parent->db_level, ==, db->db_level+1); 381 ASSERT3U(db->db_parent->db.db_object, ==, 382 db->db.db_object); 383 /* 384 * dnode_grow_indblksz() can make this fail if we don't 385 * have the struct_rwlock. XXX indblksz no longer 386 * grows. safe to do this now? 387 */ 388 if (RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 389 ASSERT3P(db->db_blkptr, ==, 390 ((blkptr_t *)db->db_parent->db.db_data + 391 db->db_blkid % epb)); 392 } 393 } 394 } 395 if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) && 396 (db->db_buf == NULL || db->db_buf->b_data) && 397 db->db.db_data && db->db_blkid != DMU_BONUS_BLKID && 398 db->db_state != DB_FILL && !dn->dn_free_txg) { 399 /* 400 * If the blkptr isn't set but they have nonzero data, 401 * it had better be dirty, otherwise we'll lose that 402 * data when we evict this buffer. 403 */ 404 if (db->db_dirtycnt == 0) { 405 uint64_t *buf = db->db.db_data; 406 int i; 407 408 for (i = 0; i < db->db.db_size >> 3; i++) { 409 ASSERT(buf[i] == 0); 410 } 411 } 412 } 413 DB_DNODE_EXIT(db); 414 } 415 #endif 416 417 static void 418 dbuf_update_data(dmu_buf_impl_t *db) 419 { 420 ASSERT(MUTEX_HELD(&db->db_mtx)); 421 if (db->db_level == 0 && db->db_user_data_ptr_ptr) { 422 ASSERT(!refcount_is_zero(&db->db_holds)); 423 *db->db_user_data_ptr_ptr = db->db.db_data; 424 } 425 } 426 427 static void 428 dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf) 429 { 430 ASSERT(MUTEX_HELD(&db->db_mtx)); 431 ASSERT(db->db_buf == NULL || !arc_has_callback(db->db_buf)); 432 db->db_buf = buf; 433 if (buf != NULL) { 434 ASSERT(buf->b_data != NULL); 435 db->db.db_data = buf->b_data; 436 if (!arc_released(buf)) 437 arc_set_callback(buf, dbuf_do_evict, db); 438 dbuf_update_data(db); 439 } else { 440 dbuf_evict_user(db); 441 db->db.db_data = NULL; 442 if (db->db_state != DB_NOFILL) 443 db->db_state = DB_UNCACHED; 444 } 445 } 446 447 /* 448 * Loan out an arc_buf for read. Return the loaned arc_buf. 449 */ 450 arc_buf_t * 451 dbuf_loan_arcbuf(dmu_buf_impl_t *db) 452 { 453 arc_buf_t *abuf; 454 455 mutex_enter(&db->db_mtx); 456 if (arc_released(db->db_buf) || refcount_count(&db->db_holds) > 1) { 457 int blksz = db->db.db_size; 458 spa_t *spa; 459 460 mutex_exit(&db->db_mtx); 461 DB_GET_SPA(&spa, db); 462 abuf = arc_loan_buf(spa, blksz); 463 bcopy(db->db.db_data, abuf->b_data, blksz); 464 } else { 465 abuf = db->db_buf; 466 arc_loan_inuse_buf(abuf, db); 467 dbuf_set_data(db, NULL); 468 mutex_exit(&db->db_mtx); 469 } 470 return (abuf); 471 } 472 473 uint64_t 474 dbuf_whichblock(dnode_t *dn, uint64_t offset) 475 { 476 if (dn->dn_datablkshift) { 477 return (offset >> dn->dn_datablkshift); 478 } else { 479 ASSERT3U(offset, <, dn->dn_datablksz); 480 return (0); 481 } 482 } 483 484 static void 485 dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb) 486 { 487 dmu_buf_impl_t *db = vdb; 488 489 mutex_enter(&db->db_mtx); 490 ASSERT3U(db->db_state, ==, DB_READ); 491 /* 492 * All reads are synchronous, so we must have a hold on the dbuf 493 */ 494 ASSERT(refcount_count(&db->db_holds) > 0); 495 ASSERT(db->db_buf == NULL); 496 ASSERT(db->db.db_data == NULL); 497 if (db->db_level == 0 && db->db_freed_in_flight) { 498 /* we were freed in flight; disregard any error */ 499 arc_release(buf, db); 500 bzero(buf->b_data, db->db.db_size); 501 arc_buf_freeze(buf); 502 db->db_freed_in_flight = FALSE; 503 dbuf_set_data(db, buf); 504 db->db_state = DB_CACHED; 505 } else if (zio == NULL || zio->io_error == 0) { 506 dbuf_set_data(db, buf); 507 db->db_state = DB_CACHED; 508 } else { 509 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 510 ASSERT3P(db->db_buf, ==, NULL); 511 VERIFY(arc_buf_remove_ref(buf, db)); 512 db->db_state = DB_UNCACHED; 513 } 514 cv_broadcast(&db->db_changed); 515 dbuf_rele_and_unlock(db, NULL); 516 } 517 518 static void 519 dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t *flags) 520 { 521 dnode_t *dn; 522 spa_t *spa; 523 zbookmark_t zb; 524 uint32_t aflags = ARC_NOWAIT; 525 526 DB_DNODE_ENTER(db); 527 dn = DB_DNODE(db); 528 ASSERT(!refcount_is_zero(&db->db_holds)); 529 /* We need the struct_rwlock to prevent db_blkptr from changing. */ 530 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 531 ASSERT(MUTEX_HELD(&db->db_mtx)); 532 ASSERT(db->db_state == DB_UNCACHED); 533 ASSERT(db->db_buf == NULL); 534 535 if (db->db_blkid == DMU_BONUS_BLKID) { 536 int bonuslen = MIN(dn->dn_bonuslen, dn->dn_phys->dn_bonuslen); 537 538 ASSERT3U(bonuslen, <=, db->db.db_size); 539 db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN); 540 arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 541 if (bonuslen < DN_MAX_BONUSLEN) 542 bzero(db->db.db_data, DN_MAX_BONUSLEN); 543 if (bonuslen) 544 bcopy(DN_BONUS(dn->dn_phys), db->db.db_data, bonuslen); 545 DB_DNODE_EXIT(db); 546 dbuf_update_data(db); 547 db->db_state = DB_CACHED; 548 mutex_exit(&db->db_mtx); 549 return; 550 } 551 552 /* 553 * Recheck BP_IS_HOLE() after dnode_block_freed() in case dnode_sync() 554 * processes the delete record and clears the bp while we are waiting 555 * for the dn_mtx (resulting in a "no" from block_freed). 556 */ 557 if (db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr) || 558 (db->db_level == 0 && (dnode_block_freed(dn, db->db_blkid) || 559 BP_IS_HOLE(db->db_blkptr)))) { 560 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 561 562 dbuf_set_data(db, arc_buf_alloc(dn->dn_objset->os_spa, 563 db->db.db_size, db, type)); 564 DB_DNODE_EXIT(db); 565 bzero(db->db.db_data, db->db.db_size); 566 db->db_state = DB_CACHED; 567 *flags |= DB_RF_CACHED; 568 mutex_exit(&db->db_mtx); 569 return; 570 } 571 572 spa = dn->dn_objset->os_spa; 573 DB_DNODE_EXIT(db); 574 575 db->db_state = DB_READ; 576 mutex_exit(&db->db_mtx); 577 578 if (DBUF_IS_L2CACHEABLE(db)) 579 aflags |= ARC_L2CACHE; 580 if (DBUF_IS_L2COMPRESSIBLE(db)) 581 aflags |= ARC_L2COMPRESS; 582 583 SET_BOOKMARK(&zb, db->db_objset->os_dsl_dataset ? 584 db->db_objset->os_dsl_dataset->ds_object : DMU_META_OBJSET, 585 db->db.db_object, db->db_level, db->db_blkid); 586 587 dbuf_add_ref(db, NULL); 588 589 (void) arc_read(zio, spa, db->db_blkptr, 590 dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ, 591 (*flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED, 592 &aflags, &zb); 593 if (aflags & ARC_CACHED) 594 *flags |= DB_RF_CACHED; 595 } 596 597 int 598 dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags) 599 { 600 int err = 0; 601 int havepzio = (zio != NULL); 602 int prefetch; 603 dnode_t *dn; 604 605 /* 606 * We don't have to hold the mutex to check db_state because it 607 * can't be freed while we have a hold on the buffer. 608 */ 609 ASSERT(!refcount_is_zero(&db->db_holds)); 610 611 if (db->db_state == DB_NOFILL) 612 return (SET_ERROR(EIO)); 613 614 DB_DNODE_ENTER(db); 615 dn = DB_DNODE(db); 616 if ((flags & DB_RF_HAVESTRUCT) == 0) 617 rw_enter(&dn->dn_struct_rwlock, RW_READER); 618 619 prefetch = db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 620 (flags & DB_RF_NOPREFETCH) == 0 && dn != NULL && 621 DBUF_IS_CACHEABLE(db); 622 623 mutex_enter(&db->db_mtx); 624 if (db->db_state == DB_CACHED) { 625 mutex_exit(&db->db_mtx); 626 if (prefetch) 627 dmu_zfetch(&dn->dn_zfetch, db->db.db_offset, 628 db->db.db_size, TRUE); 629 if ((flags & DB_RF_HAVESTRUCT) == 0) 630 rw_exit(&dn->dn_struct_rwlock); 631 DB_DNODE_EXIT(db); 632 } else if (db->db_state == DB_UNCACHED) { 633 spa_t *spa = dn->dn_objset->os_spa; 634 635 if (zio == NULL) 636 zio = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL); 637 dbuf_read_impl(db, zio, &flags); 638 639 /* dbuf_read_impl has dropped db_mtx for us */ 640 641 if (prefetch) 642 dmu_zfetch(&dn->dn_zfetch, db->db.db_offset, 643 db->db.db_size, flags & DB_RF_CACHED); 644 645 if ((flags & DB_RF_HAVESTRUCT) == 0) 646 rw_exit(&dn->dn_struct_rwlock); 647 DB_DNODE_EXIT(db); 648 649 if (!havepzio) 650 err = zio_wait(zio); 651 } else { 652 /* 653 * Another reader came in while the dbuf was in flight 654 * between UNCACHED and CACHED. Either a writer will finish 655 * writing the buffer (sending the dbuf to CACHED) or the 656 * first reader's request will reach the read_done callback 657 * and send the dbuf to CACHED. Otherwise, a failure 658 * occurred and the dbuf went to UNCACHED. 659 */ 660 mutex_exit(&db->db_mtx); 661 if (prefetch) 662 dmu_zfetch(&dn->dn_zfetch, db->db.db_offset, 663 db->db.db_size, TRUE); 664 if ((flags & DB_RF_HAVESTRUCT) == 0) 665 rw_exit(&dn->dn_struct_rwlock); 666 DB_DNODE_EXIT(db); 667 668 /* Skip the wait per the caller's request. */ 669 mutex_enter(&db->db_mtx); 670 if ((flags & DB_RF_NEVERWAIT) == 0) { 671 while (db->db_state == DB_READ || 672 db->db_state == DB_FILL) { 673 ASSERT(db->db_state == DB_READ || 674 (flags & DB_RF_HAVESTRUCT) == 0); 675 cv_wait(&db->db_changed, &db->db_mtx); 676 } 677 if (db->db_state == DB_UNCACHED) 678 err = SET_ERROR(EIO); 679 } 680 mutex_exit(&db->db_mtx); 681 } 682 683 ASSERT(err || havepzio || db->db_state == DB_CACHED); 684 return (err); 685 } 686 687 static void 688 dbuf_noread(dmu_buf_impl_t *db) 689 { 690 ASSERT(!refcount_is_zero(&db->db_holds)); 691 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 692 mutex_enter(&db->db_mtx); 693 while (db->db_state == DB_READ || db->db_state == DB_FILL) 694 cv_wait(&db->db_changed, &db->db_mtx); 695 if (db->db_state == DB_UNCACHED) { 696 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 697 spa_t *spa; 698 699 ASSERT(db->db_buf == NULL); 700 ASSERT(db->db.db_data == NULL); 701 DB_GET_SPA(&spa, db); 702 dbuf_set_data(db, arc_buf_alloc(spa, db->db.db_size, db, type)); 703 db->db_state = DB_FILL; 704 } else if (db->db_state == DB_NOFILL) { 705 dbuf_set_data(db, NULL); 706 } else { 707 ASSERT3U(db->db_state, ==, DB_CACHED); 708 } 709 mutex_exit(&db->db_mtx); 710 } 711 712 /* 713 * This is our just-in-time copy function. It makes a copy of 714 * buffers, that have been modified in a previous transaction 715 * group, before we modify them in the current active group. 716 * 717 * This function is used in two places: when we are dirtying a 718 * buffer for the first time in a txg, and when we are freeing 719 * a range in a dnode that includes this buffer. 720 * 721 * Note that when we are called from dbuf_free_range() we do 722 * not put a hold on the buffer, we just traverse the active 723 * dbuf list for the dnode. 724 */ 725 static void 726 dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg) 727 { 728 dbuf_dirty_record_t *dr = db->db_last_dirty; 729 730 ASSERT(MUTEX_HELD(&db->db_mtx)); 731 ASSERT(db->db.db_data != NULL); 732 ASSERT(db->db_level == 0); 733 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT); 734 735 if (dr == NULL || 736 (dr->dt.dl.dr_data != 737 ((db->db_blkid == DMU_BONUS_BLKID) ? db->db.db_data : db->db_buf))) 738 return; 739 740 /* 741 * If the last dirty record for this dbuf has not yet synced 742 * and its referencing the dbuf data, either: 743 * reset the reference to point to a new copy, 744 * or (if there a no active holders) 745 * just null out the current db_data pointer. 746 */ 747 ASSERT(dr->dr_txg >= txg - 2); 748 if (db->db_blkid == DMU_BONUS_BLKID) { 749 /* Note that the data bufs here are zio_bufs */ 750 dr->dt.dl.dr_data = zio_buf_alloc(DN_MAX_BONUSLEN); 751 arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 752 bcopy(db->db.db_data, dr->dt.dl.dr_data, DN_MAX_BONUSLEN); 753 } else if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 754 int size = db->db.db_size; 755 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 756 spa_t *spa; 757 758 DB_GET_SPA(&spa, db); 759 dr->dt.dl.dr_data = arc_buf_alloc(spa, size, db, type); 760 bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size); 761 } else { 762 dbuf_set_data(db, NULL); 763 } 764 } 765 766 void 767 dbuf_unoverride(dbuf_dirty_record_t *dr) 768 { 769 dmu_buf_impl_t *db = dr->dr_dbuf; 770 blkptr_t *bp = &dr->dt.dl.dr_overridden_by; 771 uint64_t txg = dr->dr_txg; 772 773 ASSERT(MUTEX_HELD(&db->db_mtx)); 774 ASSERT(dr->dt.dl.dr_override_state != DR_IN_DMU_SYNC); 775 ASSERT(db->db_level == 0); 776 777 if (db->db_blkid == DMU_BONUS_BLKID || 778 dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN) 779 return; 780 781 ASSERT(db->db_data_pending != dr); 782 783 /* free this block */ 784 if (!BP_IS_HOLE(bp) && !dr->dt.dl.dr_nopwrite) { 785 spa_t *spa; 786 787 DB_GET_SPA(&spa, db); 788 zio_free(spa, txg, bp); 789 } 790 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 791 dr->dt.dl.dr_nopwrite = B_FALSE; 792 793 /* 794 * Release the already-written buffer, so we leave it in 795 * a consistent dirty state. Note that all callers are 796 * modifying the buffer, so they will immediately do 797 * another (redundant) arc_release(). Therefore, leave 798 * the buf thawed to save the effort of freezing & 799 * immediately re-thawing it. 800 */ 801 arc_release(dr->dt.dl.dr_data, db); 802 } 803 804 /* 805 * Evict (if its unreferenced) or clear (if its referenced) any level-0 806 * data blocks in the free range, so that any future readers will find 807 * empty blocks. Also, if we happen across any level-1 dbufs in the 808 * range that have not already been marked dirty, mark them dirty so 809 * they stay in memory. 810 * 811 * This is a no-op if the dataset is in the middle of an incremental 812 * receive; see comment below for details. 813 */ 814 void 815 dbuf_free_range(dnode_t *dn, uint64_t start, uint64_t end, dmu_tx_t *tx) 816 { 817 dmu_buf_impl_t *db, *db_next; 818 uint64_t txg = tx->tx_txg; 819 int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 820 uint64_t first_l1 = start >> epbs; 821 uint64_t last_l1 = end >> epbs; 822 823 if (end > dn->dn_maxblkid && (end != DMU_SPILL_BLKID)) { 824 end = dn->dn_maxblkid; 825 last_l1 = end >> epbs; 826 } 827 dprintf_dnode(dn, "start=%llu end=%llu\n", start, end); 828 829 mutex_enter(&dn->dn_dbufs_mtx); 830 if (start >= dn->dn_unlisted_l0_blkid * dn->dn_datablksz) { 831 /* There can't be any dbufs in this range; no need to search. */ 832 mutex_exit(&dn->dn_dbufs_mtx); 833 return; 834 } else if (dmu_objset_is_receiving(dn->dn_objset)) { 835 /* 836 * If we are receiving, we expect there to be no dbufs in 837 * the range to be freed, because receive modifies each 838 * block at most once, and in offset order. If this is 839 * not the case, it can lead to performance problems, 840 * so note that we unexpectedly took the slow path. 841 */ 842 atomic_inc_64(&zfs_free_range_recv_miss); 843 } 844 845 for (db = list_head(&dn->dn_dbufs); db != NULL; db = db_next) { 846 db_next = list_next(&dn->dn_dbufs, db); 847 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 848 849 if (db->db_level == 1 && 850 db->db_blkid >= first_l1 && db->db_blkid <= last_l1) { 851 mutex_enter(&db->db_mtx); 852 if (db->db_last_dirty && 853 db->db_last_dirty->dr_txg < txg) { 854 dbuf_add_ref(db, FTAG); 855 mutex_exit(&db->db_mtx); 856 dbuf_will_dirty(db, tx); 857 dbuf_rele(db, FTAG); 858 } else { 859 mutex_exit(&db->db_mtx); 860 } 861 } 862 863 if (db->db_level != 0) 864 continue; 865 dprintf_dbuf(db, "found buf %s\n", ""); 866 if (db->db_blkid < start || db->db_blkid > end) 867 continue; 868 869 /* found a level 0 buffer in the range */ 870 mutex_enter(&db->db_mtx); 871 if (dbuf_undirty(db, tx)) { 872 /* mutex has been dropped and dbuf destroyed */ 873 continue; 874 } 875 876 if (db->db_state == DB_UNCACHED || 877 db->db_state == DB_NOFILL || 878 db->db_state == DB_EVICTING) { 879 ASSERT(db->db.db_data == NULL); 880 mutex_exit(&db->db_mtx); 881 continue; 882 } 883 if (db->db_state == DB_READ || db->db_state == DB_FILL) { 884 /* will be handled in dbuf_read_done or dbuf_rele */ 885 db->db_freed_in_flight = TRUE; 886 mutex_exit(&db->db_mtx); 887 continue; 888 } 889 if (refcount_count(&db->db_holds) == 0) { 890 ASSERT(db->db_buf); 891 dbuf_clear(db); 892 continue; 893 } 894 /* The dbuf is referenced */ 895 896 if (db->db_last_dirty != NULL) { 897 dbuf_dirty_record_t *dr = db->db_last_dirty; 898 899 if (dr->dr_txg == txg) { 900 /* 901 * This buffer is "in-use", re-adjust the file 902 * size to reflect that this buffer may 903 * contain new data when we sync. 904 */ 905 if (db->db_blkid != DMU_SPILL_BLKID && 906 db->db_blkid > dn->dn_maxblkid) 907 dn->dn_maxblkid = db->db_blkid; 908 dbuf_unoverride(dr); 909 } else { 910 /* 911 * This dbuf is not dirty in the open context. 912 * Either uncache it (if its not referenced in 913 * the open context) or reset its contents to 914 * empty. 915 */ 916 dbuf_fix_old_data(db, txg); 917 } 918 } 919 /* clear the contents if its cached */ 920 if (db->db_state == DB_CACHED) { 921 ASSERT(db->db.db_data != NULL); 922 arc_release(db->db_buf, db); 923 bzero(db->db.db_data, db->db.db_size); 924 arc_buf_freeze(db->db_buf); 925 } 926 927 mutex_exit(&db->db_mtx); 928 } 929 mutex_exit(&dn->dn_dbufs_mtx); 930 } 931 932 static int 933 dbuf_block_freeable(dmu_buf_impl_t *db) 934 { 935 dsl_dataset_t *ds = db->db_objset->os_dsl_dataset; 936 uint64_t birth_txg = 0; 937 938 /* 939 * We don't need any locking to protect db_blkptr: 940 * If it's syncing, then db_last_dirty will be set 941 * so we'll ignore db_blkptr. 942 */ 943 ASSERT(MUTEX_HELD(&db->db_mtx)); 944 if (db->db_last_dirty) 945 birth_txg = db->db_last_dirty->dr_txg; 946 else if (db->db_blkptr) 947 birth_txg = db->db_blkptr->blk_birth; 948 949 /* 950 * If we don't exist or are in a snapshot, we can't be freed. 951 * Don't pass the bp to dsl_dataset_block_freeable() since we 952 * are holding the db_mtx lock and might deadlock if we are 953 * prefetching a dedup-ed block. 954 */ 955 if (birth_txg) 956 return (ds == NULL || 957 dsl_dataset_block_freeable(ds, NULL, birth_txg)); 958 else 959 return (FALSE); 960 } 961 962 void 963 dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx) 964 { 965 arc_buf_t *buf, *obuf; 966 int osize = db->db.db_size; 967 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 968 dnode_t *dn; 969 970 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 971 972 DB_DNODE_ENTER(db); 973 dn = DB_DNODE(db); 974 975 /* XXX does *this* func really need the lock? */ 976 ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 977 978 /* 979 * This call to dbuf_will_dirty() with the dn_struct_rwlock held 980 * is OK, because there can be no other references to the db 981 * when we are changing its size, so no concurrent DB_FILL can 982 * be happening. 983 */ 984 /* 985 * XXX we should be doing a dbuf_read, checking the return 986 * value and returning that up to our callers 987 */ 988 dbuf_will_dirty(db, tx); 989 990 /* create the data buffer for the new block */ 991 buf = arc_buf_alloc(dn->dn_objset->os_spa, size, db, type); 992 993 /* copy old block data to the new block */ 994 obuf = db->db_buf; 995 bcopy(obuf->b_data, buf->b_data, MIN(osize, size)); 996 /* zero the remainder */ 997 if (size > osize) 998 bzero((uint8_t *)buf->b_data + osize, size - osize); 999 1000 mutex_enter(&db->db_mtx); 1001 dbuf_set_data(db, buf); 1002 VERIFY(arc_buf_remove_ref(obuf, db)); 1003 db->db.db_size = size; 1004 1005 if (db->db_level == 0) { 1006 ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg); 1007 db->db_last_dirty->dt.dl.dr_data = buf; 1008 } 1009 mutex_exit(&db->db_mtx); 1010 1011 dnode_willuse_space(dn, size-osize, tx); 1012 DB_DNODE_EXIT(db); 1013 } 1014 1015 void 1016 dbuf_release_bp(dmu_buf_impl_t *db) 1017 { 1018 objset_t *os; 1019 1020 DB_GET_OBJSET(&os, db); 1021 ASSERT(dsl_pool_sync_context(dmu_objset_pool(os))); 1022 ASSERT(arc_released(os->os_phys_buf) || 1023 list_link_active(&os->os_dsl_dataset->ds_synced_link)); 1024 ASSERT(db->db_parent == NULL || arc_released(db->db_parent->db_buf)); 1025 1026 (void) arc_release(db->db_buf, db); 1027 } 1028 1029 dbuf_dirty_record_t * 1030 dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1031 { 1032 dnode_t *dn; 1033 objset_t *os; 1034 dbuf_dirty_record_t **drp, *dr; 1035 int drop_struct_lock = FALSE; 1036 boolean_t do_free_accounting = B_FALSE; 1037 int txgoff = tx->tx_txg & TXG_MASK; 1038 1039 ASSERT(tx->tx_txg != 0); 1040 ASSERT(!refcount_is_zero(&db->db_holds)); 1041 DMU_TX_DIRTY_BUF(tx, db); 1042 1043 DB_DNODE_ENTER(db); 1044 dn = DB_DNODE(db); 1045 /* 1046 * Shouldn't dirty a regular buffer in syncing context. Private 1047 * objects may be dirtied in syncing context, but only if they 1048 * were already pre-dirtied in open context. 1049 */ 1050 ASSERT(!dmu_tx_is_syncing(tx) || 1051 BP_IS_HOLE(dn->dn_objset->os_rootbp) || 1052 DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 1053 dn->dn_objset->os_dsl_dataset == NULL); 1054 /* 1055 * We make this assert for private objects as well, but after we 1056 * check if we're already dirty. They are allowed to re-dirty 1057 * in syncing context. 1058 */ 1059 ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 1060 dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 1061 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 1062 1063 mutex_enter(&db->db_mtx); 1064 /* 1065 * XXX make this true for indirects too? The problem is that 1066 * transactions created with dmu_tx_create_assigned() from 1067 * syncing context don't bother holding ahead. 1068 */ 1069 ASSERT(db->db_level != 0 || 1070 db->db_state == DB_CACHED || db->db_state == DB_FILL || 1071 db->db_state == DB_NOFILL); 1072 1073 mutex_enter(&dn->dn_mtx); 1074 /* 1075 * Don't set dirtyctx to SYNC if we're just modifying this as we 1076 * initialize the objset. 1077 */ 1078 if (dn->dn_dirtyctx == DN_UNDIRTIED && 1079 !BP_IS_HOLE(dn->dn_objset->os_rootbp)) { 1080 dn->dn_dirtyctx = 1081 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN); 1082 ASSERT(dn->dn_dirtyctx_firstset == NULL); 1083 dn->dn_dirtyctx_firstset = kmem_alloc(1, KM_SLEEP); 1084 } 1085 mutex_exit(&dn->dn_mtx); 1086 1087 if (db->db_blkid == DMU_SPILL_BLKID) 1088 dn->dn_have_spill = B_TRUE; 1089 1090 /* 1091 * If this buffer is already dirty, we're done. 1092 */ 1093 drp = &db->db_last_dirty; 1094 ASSERT(*drp == NULL || (*drp)->dr_txg <= tx->tx_txg || 1095 db->db.db_object == DMU_META_DNODE_OBJECT); 1096 while ((dr = *drp) != NULL && dr->dr_txg > tx->tx_txg) 1097 drp = &dr->dr_next; 1098 if (dr && dr->dr_txg == tx->tx_txg) { 1099 DB_DNODE_EXIT(db); 1100 1101 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID) { 1102 /* 1103 * If this buffer has already been written out, 1104 * we now need to reset its state. 1105 */ 1106 dbuf_unoverride(dr); 1107 if (db->db.db_object != DMU_META_DNODE_OBJECT && 1108 db->db_state != DB_NOFILL) 1109 arc_buf_thaw(db->db_buf); 1110 } 1111 mutex_exit(&db->db_mtx); 1112 return (dr); 1113 } 1114 1115 /* 1116 * Only valid if not already dirty. 1117 */ 1118 ASSERT(dn->dn_object == 0 || 1119 dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 1120 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 1121 1122 ASSERT3U(dn->dn_nlevels, >, db->db_level); 1123 ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) || 1124 dn->dn_phys->dn_nlevels > db->db_level || 1125 dn->dn_next_nlevels[txgoff] > db->db_level || 1126 dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level || 1127 dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level); 1128 1129 /* 1130 * We should only be dirtying in syncing context if it's the 1131 * mos or we're initializing the os or it's a special object. 1132 * However, we are allowed to dirty in syncing context provided 1133 * we already dirtied it in open context. Hence we must make 1134 * this assertion only if we're not already dirty. 1135 */ 1136 os = dn->dn_objset; 1137 ASSERT(!dmu_tx_is_syncing(tx) || DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 1138 os->os_dsl_dataset == NULL || BP_IS_HOLE(os->os_rootbp)); 1139 ASSERT(db->db.db_size != 0); 1140 1141 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1142 1143 if (db->db_blkid != DMU_BONUS_BLKID) { 1144 /* 1145 * Update the accounting. 1146 * Note: we delay "free accounting" until after we drop 1147 * the db_mtx. This keeps us from grabbing other locks 1148 * (and possibly deadlocking) in bp_get_dsize() while 1149 * also holding the db_mtx. 1150 */ 1151 dnode_willuse_space(dn, db->db.db_size, tx); 1152 do_free_accounting = dbuf_block_freeable(db); 1153 } 1154 1155 /* 1156 * If this buffer is dirty in an old transaction group we need 1157 * to make a copy of it so that the changes we make in this 1158 * transaction group won't leak out when we sync the older txg. 1159 */ 1160 dr = kmem_zalloc(sizeof (dbuf_dirty_record_t), KM_SLEEP); 1161 if (db->db_level == 0) { 1162 void *data_old = db->db_buf; 1163 1164 if (db->db_state != DB_NOFILL) { 1165 if (db->db_blkid == DMU_BONUS_BLKID) { 1166 dbuf_fix_old_data(db, tx->tx_txg); 1167 data_old = db->db.db_data; 1168 } else if (db->db.db_object != DMU_META_DNODE_OBJECT) { 1169 /* 1170 * Release the data buffer from the cache so 1171 * that we can modify it without impacting 1172 * possible other users of this cached data 1173 * block. Note that indirect blocks and 1174 * private objects are not released until the 1175 * syncing state (since they are only modified 1176 * then). 1177 */ 1178 arc_release(db->db_buf, db); 1179 dbuf_fix_old_data(db, tx->tx_txg); 1180 data_old = db->db_buf; 1181 } 1182 ASSERT(data_old != NULL); 1183 } 1184 dr->dt.dl.dr_data = data_old; 1185 } else { 1186 mutex_init(&dr->dt.di.dr_mtx, NULL, MUTEX_DEFAULT, NULL); 1187 list_create(&dr->dt.di.dr_children, 1188 sizeof (dbuf_dirty_record_t), 1189 offsetof(dbuf_dirty_record_t, dr_dirty_node)); 1190 } 1191 if (db->db_blkid != DMU_BONUS_BLKID && os->os_dsl_dataset != NULL) 1192 dr->dr_accounted = db->db.db_size; 1193 dr->dr_dbuf = db; 1194 dr->dr_txg = tx->tx_txg; 1195 dr->dr_next = *drp; 1196 *drp = dr; 1197 1198 /* 1199 * We could have been freed_in_flight between the dbuf_noread 1200 * and dbuf_dirty. We win, as though the dbuf_noread() had 1201 * happened after the free. 1202 */ 1203 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 1204 db->db_blkid != DMU_SPILL_BLKID) { 1205 mutex_enter(&dn->dn_mtx); 1206 dnode_clear_range(dn, db->db_blkid, 1, tx); 1207 mutex_exit(&dn->dn_mtx); 1208 db->db_freed_in_flight = FALSE; 1209 } 1210 1211 /* 1212 * This buffer is now part of this txg 1213 */ 1214 dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg); 1215 db->db_dirtycnt += 1; 1216 ASSERT3U(db->db_dirtycnt, <=, 3); 1217 1218 mutex_exit(&db->db_mtx); 1219 1220 if (db->db_blkid == DMU_BONUS_BLKID || 1221 db->db_blkid == DMU_SPILL_BLKID) { 1222 mutex_enter(&dn->dn_mtx); 1223 ASSERT(!list_link_active(&dr->dr_dirty_node)); 1224 list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1225 mutex_exit(&dn->dn_mtx); 1226 dnode_setdirty(dn, tx); 1227 DB_DNODE_EXIT(db); 1228 return (dr); 1229 } else if (do_free_accounting) { 1230 blkptr_t *bp = db->db_blkptr; 1231 int64_t willfree = (bp && !BP_IS_HOLE(bp)) ? 1232 bp_get_dsize(os->os_spa, bp) : db->db.db_size; 1233 /* 1234 * This is only a guess -- if the dbuf is dirty 1235 * in a previous txg, we don't know how much 1236 * space it will use on disk yet. We should 1237 * really have the struct_rwlock to access 1238 * db_blkptr, but since this is just a guess, 1239 * it's OK if we get an odd answer. 1240 */ 1241 ddt_prefetch(os->os_spa, bp); 1242 dnode_willuse_space(dn, -willfree, tx); 1243 } 1244 1245 if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 1246 rw_enter(&dn->dn_struct_rwlock, RW_READER); 1247 drop_struct_lock = TRUE; 1248 } 1249 1250 if (db->db_level == 0) { 1251 dnode_new_blkid(dn, db->db_blkid, tx, drop_struct_lock); 1252 ASSERT(dn->dn_maxblkid >= db->db_blkid); 1253 } 1254 1255 if (db->db_level+1 < dn->dn_nlevels) { 1256 dmu_buf_impl_t *parent = db->db_parent; 1257 dbuf_dirty_record_t *di; 1258 int parent_held = FALSE; 1259 1260 if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) { 1261 int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1262 1263 parent = dbuf_hold_level(dn, db->db_level+1, 1264 db->db_blkid >> epbs, FTAG); 1265 ASSERT(parent != NULL); 1266 parent_held = TRUE; 1267 } 1268 if (drop_struct_lock) 1269 rw_exit(&dn->dn_struct_rwlock); 1270 ASSERT3U(db->db_level+1, ==, parent->db_level); 1271 di = dbuf_dirty(parent, tx); 1272 if (parent_held) 1273 dbuf_rele(parent, FTAG); 1274 1275 mutex_enter(&db->db_mtx); 1276 /* 1277 * Since we've dropped the mutex, it's possible that 1278 * dbuf_undirty() might have changed this out from under us. 1279 */ 1280 if (db->db_last_dirty == dr || 1281 dn->dn_object == DMU_META_DNODE_OBJECT) { 1282 mutex_enter(&di->dt.di.dr_mtx); 1283 ASSERT3U(di->dr_txg, ==, tx->tx_txg); 1284 ASSERT(!list_link_active(&dr->dr_dirty_node)); 1285 list_insert_tail(&di->dt.di.dr_children, dr); 1286 mutex_exit(&di->dt.di.dr_mtx); 1287 dr->dr_parent = di; 1288 } 1289 mutex_exit(&db->db_mtx); 1290 } else { 1291 ASSERT(db->db_level+1 == dn->dn_nlevels); 1292 ASSERT(db->db_blkid < dn->dn_nblkptr); 1293 ASSERT(db->db_parent == NULL || db->db_parent == dn->dn_dbuf); 1294 mutex_enter(&dn->dn_mtx); 1295 ASSERT(!list_link_active(&dr->dr_dirty_node)); 1296 list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1297 mutex_exit(&dn->dn_mtx); 1298 if (drop_struct_lock) 1299 rw_exit(&dn->dn_struct_rwlock); 1300 } 1301 1302 dnode_setdirty(dn, tx); 1303 DB_DNODE_EXIT(db); 1304 return (dr); 1305 } 1306 1307 /* 1308 * Undirty a buffer in the transaction group referenced by the given 1309 * transaction. Return whether this evicted the dbuf. 1310 */ 1311 static boolean_t 1312 dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1313 { 1314 dnode_t *dn; 1315 uint64_t txg = tx->tx_txg; 1316 dbuf_dirty_record_t *dr, **drp; 1317 1318 ASSERT(txg != 0); 1319 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1320 ASSERT0(db->db_level); 1321 ASSERT(MUTEX_HELD(&db->db_mtx)); 1322 1323 /* 1324 * If this buffer is not dirty, we're done. 1325 */ 1326 for (drp = &db->db_last_dirty; (dr = *drp) != NULL; drp = &dr->dr_next) 1327 if (dr->dr_txg <= txg) 1328 break; 1329 if (dr == NULL || dr->dr_txg < txg) 1330 return (B_FALSE); 1331 ASSERT(dr->dr_txg == txg); 1332 ASSERT(dr->dr_dbuf == db); 1333 1334 DB_DNODE_ENTER(db); 1335 dn = DB_DNODE(db); 1336 1337 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1338 1339 ASSERT(db->db.db_size != 0); 1340 1341 /* 1342 * Any space we accounted for in dp_dirty_* will be cleaned up by 1343 * dsl_pool_sync(). This is relatively rare so the discrepancy 1344 * is not a big deal. 1345 */ 1346 1347 *drp = dr->dr_next; 1348 1349 /* 1350 * Note that there are three places in dbuf_dirty() 1351 * where this dirty record may be put on a list. 1352 * Make sure to do a list_remove corresponding to 1353 * every one of those list_insert calls. 1354 */ 1355 if (dr->dr_parent) { 1356 mutex_enter(&dr->dr_parent->dt.di.dr_mtx); 1357 list_remove(&dr->dr_parent->dt.di.dr_children, dr); 1358 mutex_exit(&dr->dr_parent->dt.di.dr_mtx); 1359 } else if (db->db_blkid == DMU_SPILL_BLKID || 1360 db->db_level+1 == dn->dn_nlevels) { 1361 ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf); 1362 mutex_enter(&dn->dn_mtx); 1363 list_remove(&dn->dn_dirty_records[txg & TXG_MASK], dr); 1364 mutex_exit(&dn->dn_mtx); 1365 } 1366 DB_DNODE_EXIT(db); 1367 1368 if (db->db_state != DB_NOFILL) { 1369 dbuf_unoverride(dr); 1370 1371 ASSERT(db->db_buf != NULL); 1372 ASSERT(dr->dt.dl.dr_data != NULL); 1373 if (dr->dt.dl.dr_data != db->db_buf) 1374 VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, db)); 1375 } 1376 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 1377 1378 ASSERT(db->db_dirtycnt > 0); 1379 db->db_dirtycnt -= 1; 1380 1381 if (refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) { 1382 arc_buf_t *buf = db->db_buf; 1383 1384 ASSERT(db->db_state == DB_NOFILL || arc_released(buf)); 1385 dbuf_set_data(db, NULL); 1386 VERIFY(arc_buf_remove_ref(buf, db)); 1387 dbuf_evict(db); 1388 return (B_TRUE); 1389 } 1390 1391 return (B_FALSE); 1392 } 1393 1394 #pragma weak dmu_buf_will_dirty = dbuf_will_dirty 1395 void 1396 dbuf_will_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1397 { 1398 int rf = DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH; 1399 1400 ASSERT(tx->tx_txg != 0); 1401 ASSERT(!refcount_is_zero(&db->db_holds)); 1402 1403 DB_DNODE_ENTER(db); 1404 if (RW_WRITE_HELD(&DB_DNODE(db)->dn_struct_rwlock)) 1405 rf |= DB_RF_HAVESTRUCT; 1406 DB_DNODE_EXIT(db); 1407 (void) dbuf_read(db, NULL, rf); 1408 (void) dbuf_dirty(db, tx); 1409 } 1410 1411 void 1412 dmu_buf_will_not_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 1413 { 1414 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1415 1416 db->db_state = DB_NOFILL; 1417 1418 dmu_buf_will_fill(db_fake, tx); 1419 } 1420 1421 void 1422 dmu_buf_will_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 1423 { 1424 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1425 1426 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1427 ASSERT(tx->tx_txg != 0); 1428 ASSERT(db->db_level == 0); 1429 ASSERT(!refcount_is_zero(&db->db_holds)); 1430 1431 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT || 1432 dmu_tx_private_ok(tx)); 1433 1434 dbuf_noread(db); 1435 (void) dbuf_dirty(db, tx); 1436 } 1437 1438 #pragma weak dmu_buf_fill_done = dbuf_fill_done 1439 /* ARGSUSED */ 1440 void 1441 dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx) 1442 { 1443 mutex_enter(&db->db_mtx); 1444 DBUF_VERIFY(db); 1445 1446 if (db->db_state == DB_FILL) { 1447 if (db->db_level == 0 && db->db_freed_in_flight) { 1448 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1449 /* we were freed while filling */ 1450 /* XXX dbuf_undirty? */ 1451 bzero(db->db.db_data, db->db.db_size); 1452 db->db_freed_in_flight = FALSE; 1453 } 1454 db->db_state = DB_CACHED; 1455 cv_broadcast(&db->db_changed); 1456 } 1457 mutex_exit(&db->db_mtx); 1458 } 1459 1460 /* 1461 * Directly assign a provided arc buf to a given dbuf if it's not referenced 1462 * by anybody except our caller. Otherwise copy arcbuf's contents to dbuf. 1463 */ 1464 void 1465 dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx) 1466 { 1467 ASSERT(!refcount_is_zero(&db->db_holds)); 1468 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1469 ASSERT(db->db_level == 0); 1470 ASSERT(DBUF_GET_BUFC_TYPE(db) == ARC_BUFC_DATA); 1471 ASSERT(buf != NULL); 1472 ASSERT(arc_buf_size(buf) == db->db.db_size); 1473 ASSERT(tx->tx_txg != 0); 1474 1475 arc_return_buf(buf, db); 1476 ASSERT(arc_released(buf)); 1477 1478 mutex_enter(&db->db_mtx); 1479 1480 while (db->db_state == DB_READ || db->db_state == DB_FILL) 1481 cv_wait(&db->db_changed, &db->db_mtx); 1482 1483 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_UNCACHED); 1484 1485 if (db->db_state == DB_CACHED && 1486 refcount_count(&db->db_holds) - 1 > db->db_dirtycnt) { 1487 mutex_exit(&db->db_mtx); 1488 (void) dbuf_dirty(db, tx); 1489 bcopy(buf->b_data, db->db.db_data, db->db.db_size); 1490 VERIFY(arc_buf_remove_ref(buf, db)); 1491 xuio_stat_wbuf_copied(); 1492 return; 1493 } 1494 1495 xuio_stat_wbuf_nocopy(); 1496 if (db->db_state == DB_CACHED) { 1497 dbuf_dirty_record_t *dr = db->db_last_dirty; 1498 1499 ASSERT(db->db_buf != NULL); 1500 if (dr != NULL && dr->dr_txg == tx->tx_txg) { 1501 ASSERT(dr->dt.dl.dr_data == db->db_buf); 1502 if (!arc_released(db->db_buf)) { 1503 ASSERT(dr->dt.dl.dr_override_state == 1504 DR_OVERRIDDEN); 1505 arc_release(db->db_buf, db); 1506 } 1507 dr->dt.dl.dr_data = buf; 1508 VERIFY(arc_buf_remove_ref(db->db_buf, db)); 1509 } else if (dr == NULL || dr->dt.dl.dr_data != db->db_buf) { 1510 arc_release(db->db_buf, db); 1511 VERIFY(arc_buf_remove_ref(db->db_buf, db)); 1512 } 1513 db->db_buf = NULL; 1514 } 1515 ASSERT(db->db_buf == NULL); 1516 dbuf_set_data(db, buf); 1517 db->db_state = DB_FILL; 1518 mutex_exit(&db->db_mtx); 1519 (void) dbuf_dirty(db, tx); 1520 dbuf_fill_done(db, tx); 1521 } 1522 1523 /* 1524 * "Clear" the contents of this dbuf. This will mark the dbuf 1525 * EVICTING and clear *most* of its references. Unfortunately, 1526 * when we are not holding the dn_dbufs_mtx, we can't clear the 1527 * entry in the dn_dbufs list. We have to wait until dbuf_destroy() 1528 * in this case. For callers from the DMU we will usually see: 1529 * dbuf_clear()->arc_buf_evict()->dbuf_do_evict()->dbuf_destroy() 1530 * For the arc callback, we will usually see: 1531 * dbuf_do_evict()->dbuf_clear();dbuf_destroy() 1532 * Sometimes, though, we will get a mix of these two: 1533 * DMU: dbuf_clear()->arc_buf_evict() 1534 * ARC: dbuf_do_evict()->dbuf_destroy() 1535 */ 1536 void 1537 dbuf_clear(dmu_buf_impl_t *db) 1538 { 1539 dnode_t *dn; 1540 dmu_buf_impl_t *parent = db->db_parent; 1541 dmu_buf_impl_t *dndb; 1542 int dbuf_gone = FALSE; 1543 1544 ASSERT(MUTEX_HELD(&db->db_mtx)); 1545 ASSERT(refcount_is_zero(&db->db_holds)); 1546 1547 dbuf_evict_user(db); 1548 1549 if (db->db_state == DB_CACHED) { 1550 ASSERT(db->db.db_data != NULL); 1551 if (db->db_blkid == DMU_BONUS_BLKID) { 1552 zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN); 1553 arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 1554 } 1555 db->db.db_data = NULL; 1556 db->db_state = DB_UNCACHED; 1557 } 1558 1559 ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL); 1560 ASSERT(db->db_data_pending == NULL); 1561 1562 db->db_state = DB_EVICTING; 1563 db->db_blkptr = NULL; 1564 1565 DB_DNODE_ENTER(db); 1566 dn = DB_DNODE(db); 1567 dndb = dn->dn_dbuf; 1568 if (db->db_blkid != DMU_BONUS_BLKID && MUTEX_HELD(&dn->dn_dbufs_mtx)) { 1569 list_remove(&dn->dn_dbufs, db); 1570 (void) atomic_dec_32_nv(&dn->dn_dbufs_count); 1571 membar_producer(); 1572 DB_DNODE_EXIT(db); 1573 /* 1574 * Decrementing the dbuf count means that the hold corresponding 1575 * to the removed dbuf is no longer discounted in dnode_move(), 1576 * so the dnode cannot be moved until after we release the hold. 1577 * The membar_producer() ensures visibility of the decremented 1578 * value in dnode_move(), since DB_DNODE_EXIT doesn't actually 1579 * release any lock. 1580 */ 1581 dnode_rele(dn, db); 1582 db->db_dnode_handle = NULL; 1583 } else { 1584 DB_DNODE_EXIT(db); 1585 } 1586 1587 if (db->db_buf) 1588 dbuf_gone = arc_buf_evict(db->db_buf); 1589 1590 if (!dbuf_gone) 1591 mutex_exit(&db->db_mtx); 1592 1593 /* 1594 * If this dbuf is referenced from an indirect dbuf, 1595 * decrement the ref count on the indirect dbuf. 1596 */ 1597 if (parent && parent != dndb) 1598 dbuf_rele(parent, db); 1599 } 1600 1601 static int 1602 dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse, 1603 dmu_buf_impl_t **parentp, blkptr_t **bpp) 1604 { 1605 int nlevels, epbs; 1606 1607 *parentp = NULL; 1608 *bpp = NULL; 1609 1610 ASSERT(blkid != DMU_BONUS_BLKID); 1611 1612 if (blkid == DMU_SPILL_BLKID) { 1613 mutex_enter(&dn->dn_mtx); 1614 if (dn->dn_have_spill && 1615 (dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR)) 1616 *bpp = &dn->dn_phys->dn_spill; 1617 else 1618 *bpp = NULL; 1619 dbuf_add_ref(dn->dn_dbuf, NULL); 1620 *parentp = dn->dn_dbuf; 1621 mutex_exit(&dn->dn_mtx); 1622 return (0); 1623 } 1624 1625 if (dn->dn_phys->dn_nlevels == 0) 1626 nlevels = 1; 1627 else 1628 nlevels = dn->dn_phys->dn_nlevels; 1629 1630 epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1631 1632 ASSERT3U(level * epbs, <, 64); 1633 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1634 if (level >= nlevels || 1635 (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) { 1636 /* the buffer has no parent yet */ 1637 return (SET_ERROR(ENOENT)); 1638 } else if (level < nlevels-1) { 1639 /* this block is referenced from an indirect block */ 1640 int err = dbuf_hold_impl(dn, level+1, 1641 blkid >> epbs, fail_sparse, NULL, parentp); 1642 if (err) 1643 return (err); 1644 err = dbuf_read(*parentp, NULL, 1645 (DB_RF_HAVESTRUCT | DB_RF_NOPREFETCH | DB_RF_CANFAIL)); 1646 if (err) { 1647 dbuf_rele(*parentp, NULL); 1648 *parentp = NULL; 1649 return (err); 1650 } 1651 *bpp = ((blkptr_t *)(*parentp)->db.db_data) + 1652 (blkid & ((1ULL << epbs) - 1)); 1653 return (0); 1654 } else { 1655 /* the block is referenced from the dnode */ 1656 ASSERT3U(level, ==, nlevels-1); 1657 ASSERT(dn->dn_phys->dn_nblkptr == 0 || 1658 blkid < dn->dn_phys->dn_nblkptr); 1659 if (dn->dn_dbuf) { 1660 dbuf_add_ref(dn->dn_dbuf, NULL); 1661 *parentp = dn->dn_dbuf; 1662 } 1663 *bpp = &dn->dn_phys->dn_blkptr[blkid]; 1664 return (0); 1665 } 1666 } 1667 1668 static dmu_buf_impl_t * 1669 dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid, 1670 dmu_buf_impl_t *parent, blkptr_t *blkptr) 1671 { 1672 objset_t *os = dn->dn_objset; 1673 dmu_buf_impl_t *db, *odb; 1674 1675 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1676 ASSERT(dn->dn_type != DMU_OT_NONE); 1677 1678 db = kmem_cache_alloc(dbuf_cache, KM_SLEEP); 1679 1680 db->db_objset = os; 1681 db->db.db_object = dn->dn_object; 1682 db->db_level = level; 1683 db->db_blkid = blkid; 1684 db->db_last_dirty = NULL; 1685 db->db_dirtycnt = 0; 1686 db->db_dnode_handle = dn->dn_handle; 1687 db->db_parent = parent; 1688 db->db_blkptr = blkptr; 1689 1690 db->db_user_ptr = NULL; 1691 db->db_user_data_ptr_ptr = NULL; 1692 db->db_evict_func = NULL; 1693 db->db_immediate_evict = 0; 1694 db->db_freed_in_flight = 0; 1695 1696 if (blkid == DMU_BONUS_BLKID) { 1697 ASSERT3P(parent, ==, dn->dn_dbuf); 1698 db->db.db_size = DN_MAX_BONUSLEN - 1699 (dn->dn_nblkptr-1) * sizeof (blkptr_t); 1700 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 1701 db->db.db_offset = DMU_BONUS_BLKID; 1702 db->db_state = DB_UNCACHED; 1703 /* the bonus dbuf is not placed in the hash table */ 1704 arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1705 return (db); 1706 } else if (blkid == DMU_SPILL_BLKID) { 1707 db->db.db_size = (blkptr != NULL) ? 1708 BP_GET_LSIZE(blkptr) : SPA_MINBLOCKSIZE; 1709 db->db.db_offset = 0; 1710 } else { 1711 int blocksize = 1712 db->db_level ? 1 << dn->dn_indblkshift : dn->dn_datablksz; 1713 db->db.db_size = blocksize; 1714 db->db.db_offset = db->db_blkid * blocksize; 1715 } 1716 1717 /* 1718 * Hold the dn_dbufs_mtx while we get the new dbuf 1719 * in the hash table *and* added to the dbufs list. 1720 * This prevents a possible deadlock with someone 1721 * trying to look up this dbuf before its added to the 1722 * dn_dbufs list. 1723 */ 1724 mutex_enter(&dn->dn_dbufs_mtx); 1725 db->db_state = DB_EVICTING; 1726 if ((odb = dbuf_hash_insert(db)) != NULL) { 1727 /* someone else inserted it first */ 1728 kmem_cache_free(dbuf_cache, db); 1729 mutex_exit(&dn->dn_dbufs_mtx); 1730 return (odb); 1731 } 1732 list_insert_head(&dn->dn_dbufs, db); 1733 if (db->db_level == 0 && db->db_blkid >= 1734 dn->dn_unlisted_l0_blkid) 1735 dn->dn_unlisted_l0_blkid = db->db_blkid + 1; 1736 db->db_state = DB_UNCACHED; 1737 mutex_exit(&dn->dn_dbufs_mtx); 1738 arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1739 1740 if (parent && parent != dn->dn_dbuf) 1741 dbuf_add_ref(parent, db); 1742 1743 ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 1744 refcount_count(&dn->dn_holds) > 0); 1745 (void) refcount_add(&dn->dn_holds, db); 1746 (void) atomic_inc_32_nv(&dn->dn_dbufs_count); 1747 1748 dprintf_dbuf(db, "db=%p\n", db); 1749 1750 return (db); 1751 } 1752 1753 static int 1754 dbuf_do_evict(void *private) 1755 { 1756 arc_buf_t *buf = private; 1757 dmu_buf_impl_t *db = buf->b_private; 1758 1759 if (!MUTEX_HELD(&db->db_mtx)) 1760 mutex_enter(&db->db_mtx); 1761 1762 ASSERT(refcount_is_zero(&db->db_holds)); 1763 1764 if (db->db_state != DB_EVICTING) { 1765 ASSERT(db->db_state == DB_CACHED); 1766 DBUF_VERIFY(db); 1767 db->db_buf = NULL; 1768 dbuf_evict(db); 1769 } else { 1770 mutex_exit(&db->db_mtx); 1771 dbuf_destroy(db); 1772 } 1773 return (0); 1774 } 1775 1776 static void 1777 dbuf_destroy(dmu_buf_impl_t *db) 1778 { 1779 ASSERT(refcount_is_zero(&db->db_holds)); 1780 1781 if (db->db_blkid != DMU_BONUS_BLKID) { 1782 /* 1783 * If this dbuf is still on the dn_dbufs list, 1784 * remove it from that list. 1785 */ 1786 if (db->db_dnode_handle != NULL) { 1787 dnode_t *dn; 1788 1789 DB_DNODE_ENTER(db); 1790 dn = DB_DNODE(db); 1791 mutex_enter(&dn->dn_dbufs_mtx); 1792 list_remove(&dn->dn_dbufs, db); 1793 (void) atomic_dec_32_nv(&dn->dn_dbufs_count); 1794 mutex_exit(&dn->dn_dbufs_mtx); 1795 DB_DNODE_EXIT(db); 1796 /* 1797 * Decrementing the dbuf count means that the hold 1798 * corresponding to the removed dbuf is no longer 1799 * discounted in dnode_move(), so the dnode cannot be 1800 * moved until after we release the hold. 1801 */ 1802 dnode_rele(dn, db); 1803 db->db_dnode_handle = NULL; 1804 } 1805 dbuf_hash_remove(db); 1806 } 1807 db->db_parent = NULL; 1808 db->db_buf = NULL; 1809 1810 ASSERT(!list_link_active(&db->db_link)); 1811 ASSERT(db->db.db_data == NULL); 1812 ASSERT(db->db_hash_next == NULL); 1813 ASSERT(db->db_blkptr == NULL); 1814 ASSERT(db->db_data_pending == NULL); 1815 1816 kmem_cache_free(dbuf_cache, db); 1817 arc_space_return(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1818 } 1819 1820 void 1821 dbuf_prefetch(dnode_t *dn, uint64_t blkid, zio_priority_t prio) 1822 { 1823 dmu_buf_impl_t *db = NULL; 1824 blkptr_t *bp = NULL; 1825 1826 ASSERT(blkid != DMU_BONUS_BLKID); 1827 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1828 1829 if (dnode_block_freed(dn, blkid)) 1830 return; 1831 1832 /* dbuf_find() returns with db_mtx held */ 1833 if (db = dbuf_find(dn, 0, blkid)) { 1834 /* 1835 * This dbuf is already in the cache. We assume that 1836 * it is already CACHED, or else about to be either 1837 * read or filled. 1838 */ 1839 mutex_exit(&db->db_mtx); 1840 return; 1841 } 1842 1843 if (dbuf_findbp(dn, 0, blkid, TRUE, &db, &bp) == 0) { 1844 if (bp && !BP_IS_HOLE(bp)) { 1845 dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset; 1846 uint32_t aflags = ARC_NOWAIT | ARC_PREFETCH; 1847 zbookmark_t zb; 1848 1849 SET_BOOKMARK(&zb, ds ? ds->ds_object : DMU_META_OBJSET, 1850 dn->dn_object, 0, blkid); 1851 1852 (void) arc_read(NULL, dn->dn_objset->os_spa, 1853 bp, NULL, NULL, prio, 1854 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 1855 &aflags, &zb); 1856 } 1857 if (db) 1858 dbuf_rele(db, NULL); 1859 } 1860 } 1861 1862 /* 1863 * Returns with db_holds incremented, and db_mtx not held. 1864 * Note: dn_struct_rwlock must be held. 1865 */ 1866 int 1867 dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, int fail_sparse, 1868 void *tag, dmu_buf_impl_t **dbp) 1869 { 1870 dmu_buf_impl_t *db, *parent = NULL; 1871 1872 ASSERT(blkid != DMU_BONUS_BLKID); 1873 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1874 ASSERT3U(dn->dn_nlevels, >, level); 1875 1876 *dbp = NULL; 1877 top: 1878 /* dbuf_find() returns with db_mtx held */ 1879 db = dbuf_find(dn, level, blkid); 1880 1881 if (db == NULL) { 1882 blkptr_t *bp = NULL; 1883 int err; 1884 1885 ASSERT3P(parent, ==, NULL); 1886 err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp); 1887 if (fail_sparse) { 1888 if (err == 0 && bp && BP_IS_HOLE(bp)) 1889 err = SET_ERROR(ENOENT); 1890 if (err) { 1891 if (parent) 1892 dbuf_rele(parent, NULL); 1893 return (err); 1894 } 1895 } 1896 if (err && err != ENOENT) 1897 return (err); 1898 db = dbuf_create(dn, level, blkid, parent, bp); 1899 } 1900 1901 if (db->db_buf && refcount_is_zero(&db->db_holds)) { 1902 arc_buf_add_ref(db->db_buf, db); 1903 if (db->db_buf->b_data == NULL) { 1904 dbuf_clear(db); 1905 if (parent) { 1906 dbuf_rele(parent, NULL); 1907 parent = NULL; 1908 } 1909 goto top; 1910 } 1911 ASSERT3P(db->db.db_data, ==, db->db_buf->b_data); 1912 } 1913 1914 ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf)); 1915 1916 /* 1917 * If this buffer is currently syncing out, and we are are 1918 * still referencing it from db_data, we need to make a copy 1919 * of it in case we decide we want to dirty it again in this txg. 1920 */ 1921 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 1922 dn->dn_object != DMU_META_DNODE_OBJECT && 1923 db->db_state == DB_CACHED && db->db_data_pending) { 1924 dbuf_dirty_record_t *dr = db->db_data_pending; 1925 1926 if (dr->dt.dl.dr_data == db->db_buf) { 1927 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 1928 1929 dbuf_set_data(db, 1930 arc_buf_alloc(dn->dn_objset->os_spa, 1931 db->db.db_size, db, type)); 1932 bcopy(dr->dt.dl.dr_data->b_data, db->db.db_data, 1933 db->db.db_size); 1934 } 1935 } 1936 1937 (void) refcount_add(&db->db_holds, tag); 1938 dbuf_update_data(db); 1939 DBUF_VERIFY(db); 1940 mutex_exit(&db->db_mtx); 1941 1942 /* NOTE: we can't rele the parent until after we drop the db_mtx */ 1943 if (parent) 1944 dbuf_rele(parent, NULL); 1945 1946 ASSERT3P(DB_DNODE(db), ==, dn); 1947 ASSERT3U(db->db_blkid, ==, blkid); 1948 ASSERT3U(db->db_level, ==, level); 1949 *dbp = db; 1950 1951 return (0); 1952 } 1953 1954 dmu_buf_impl_t * 1955 dbuf_hold(dnode_t *dn, uint64_t blkid, void *tag) 1956 { 1957 dmu_buf_impl_t *db; 1958 int err = dbuf_hold_impl(dn, 0, blkid, FALSE, tag, &db); 1959 return (err ? NULL : db); 1960 } 1961 1962 dmu_buf_impl_t * 1963 dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, void *tag) 1964 { 1965 dmu_buf_impl_t *db; 1966 int err = dbuf_hold_impl(dn, level, blkid, FALSE, tag, &db); 1967 return (err ? NULL : db); 1968 } 1969 1970 void 1971 dbuf_create_bonus(dnode_t *dn) 1972 { 1973 ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 1974 1975 ASSERT(dn->dn_bonus == NULL); 1976 dn->dn_bonus = dbuf_create(dn, 0, DMU_BONUS_BLKID, dn->dn_dbuf, NULL); 1977 } 1978 1979 int 1980 dbuf_spill_set_blksz(dmu_buf_t *db_fake, uint64_t blksz, dmu_tx_t *tx) 1981 { 1982 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1983 dnode_t *dn; 1984 1985 if (db->db_blkid != DMU_SPILL_BLKID) 1986 return (SET_ERROR(ENOTSUP)); 1987 if (blksz == 0) 1988 blksz = SPA_MINBLOCKSIZE; 1989 if (blksz > SPA_MAXBLOCKSIZE) 1990 blksz = SPA_MAXBLOCKSIZE; 1991 else 1992 blksz = P2ROUNDUP(blksz, SPA_MINBLOCKSIZE); 1993 1994 DB_DNODE_ENTER(db); 1995 dn = DB_DNODE(db); 1996 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 1997 dbuf_new_size(db, blksz, tx); 1998 rw_exit(&dn->dn_struct_rwlock); 1999 DB_DNODE_EXIT(db); 2000 2001 return (0); 2002 } 2003 2004 void 2005 dbuf_rm_spill(dnode_t *dn, dmu_tx_t *tx) 2006 { 2007 dbuf_free_range(dn, DMU_SPILL_BLKID, DMU_SPILL_BLKID, tx); 2008 } 2009 2010 #pragma weak dmu_buf_add_ref = dbuf_add_ref 2011 void 2012 dbuf_add_ref(dmu_buf_impl_t *db, void *tag) 2013 { 2014 int64_t holds = refcount_add(&db->db_holds, tag); 2015 ASSERT(holds > 1); 2016 } 2017 2018 /* 2019 * If you call dbuf_rele() you had better not be referencing the dnode handle 2020 * unless you have some other direct or indirect hold on the dnode. (An indirect 2021 * hold is a hold on one of the dnode's dbufs, including the bonus buffer.) 2022 * Without that, the dbuf_rele() could lead to a dnode_rele() followed by the 2023 * dnode's parent dbuf evicting its dnode handles. 2024 */ 2025 #pragma weak dmu_buf_rele = dbuf_rele 2026 void 2027 dbuf_rele(dmu_buf_impl_t *db, void *tag) 2028 { 2029 mutex_enter(&db->db_mtx); 2030 dbuf_rele_and_unlock(db, tag); 2031 } 2032 2033 /* 2034 * dbuf_rele() for an already-locked dbuf. This is necessary to allow 2035 * db_dirtycnt and db_holds to be updated atomically. 2036 */ 2037 void 2038 dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag) 2039 { 2040 int64_t holds; 2041 2042 ASSERT(MUTEX_HELD(&db->db_mtx)); 2043 DBUF_VERIFY(db); 2044 2045 /* 2046 * Remove the reference to the dbuf before removing its hold on the 2047 * dnode so we can guarantee in dnode_move() that a referenced bonus 2048 * buffer has a corresponding dnode hold. 2049 */ 2050 holds = refcount_remove(&db->db_holds, tag); 2051 ASSERT(holds >= 0); 2052 2053 /* 2054 * We can't freeze indirects if there is a possibility that they 2055 * may be modified in the current syncing context. 2056 */ 2057 if (db->db_buf && holds == (db->db_level == 0 ? db->db_dirtycnt : 0)) 2058 arc_buf_freeze(db->db_buf); 2059 2060 if (holds == db->db_dirtycnt && 2061 db->db_level == 0 && db->db_immediate_evict) 2062 dbuf_evict_user(db); 2063 2064 if (holds == 0) { 2065 if (db->db_blkid == DMU_BONUS_BLKID) { 2066 mutex_exit(&db->db_mtx); 2067 2068 /* 2069 * If the dnode moves here, we cannot cross this barrier 2070 * until the move completes. 2071 */ 2072 DB_DNODE_ENTER(db); 2073 (void) atomic_dec_32_nv(&DB_DNODE(db)->dn_dbufs_count); 2074 DB_DNODE_EXIT(db); 2075 /* 2076 * The bonus buffer's dnode hold is no longer discounted 2077 * in dnode_move(). The dnode cannot move until after 2078 * the dnode_rele(). 2079 */ 2080 dnode_rele(DB_DNODE(db), db); 2081 } else if (db->db_buf == NULL) { 2082 /* 2083 * This is a special case: we never associated this 2084 * dbuf with any data allocated from the ARC. 2085 */ 2086 ASSERT(db->db_state == DB_UNCACHED || 2087 db->db_state == DB_NOFILL); 2088 dbuf_evict(db); 2089 } else if (arc_released(db->db_buf)) { 2090 arc_buf_t *buf = db->db_buf; 2091 /* 2092 * This dbuf has anonymous data associated with it. 2093 */ 2094 dbuf_set_data(db, NULL); 2095 VERIFY(arc_buf_remove_ref(buf, db)); 2096 dbuf_evict(db); 2097 } else { 2098 VERIFY(!arc_buf_remove_ref(db->db_buf, db)); 2099 2100 /* 2101 * A dbuf will be eligible for eviction if either the 2102 * 'primarycache' property is set or a duplicate 2103 * copy of this buffer is already cached in the arc. 2104 * 2105 * In the case of the 'primarycache' a buffer 2106 * is considered for eviction if it matches the 2107 * criteria set in the property. 2108 * 2109 * To decide if our buffer is considered a 2110 * duplicate, we must call into the arc to determine 2111 * if multiple buffers are referencing the same 2112 * block on-disk. If so, then we simply evict 2113 * ourselves. 2114 */ 2115 if (!DBUF_IS_CACHEABLE(db) || 2116 arc_buf_eviction_needed(db->db_buf)) 2117 dbuf_clear(db); 2118 else 2119 mutex_exit(&db->db_mtx); 2120 } 2121 } else { 2122 mutex_exit(&db->db_mtx); 2123 } 2124 } 2125 2126 #pragma weak dmu_buf_refcount = dbuf_refcount 2127 uint64_t 2128 dbuf_refcount(dmu_buf_impl_t *db) 2129 { 2130 return (refcount_count(&db->db_holds)); 2131 } 2132 2133 void * 2134 dmu_buf_set_user(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 2135 dmu_buf_evict_func_t *evict_func) 2136 { 2137 return (dmu_buf_update_user(db_fake, NULL, user_ptr, 2138 user_data_ptr_ptr, evict_func)); 2139 } 2140 2141 void * 2142 dmu_buf_set_user_ie(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 2143 dmu_buf_evict_func_t *evict_func) 2144 { 2145 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2146 2147 db->db_immediate_evict = TRUE; 2148 return (dmu_buf_update_user(db_fake, NULL, user_ptr, 2149 user_data_ptr_ptr, evict_func)); 2150 } 2151 2152 void * 2153 dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr, void *user_ptr, 2154 void *user_data_ptr_ptr, dmu_buf_evict_func_t *evict_func) 2155 { 2156 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2157 ASSERT(db->db_level == 0); 2158 2159 ASSERT((user_ptr == NULL) == (evict_func == NULL)); 2160 2161 mutex_enter(&db->db_mtx); 2162 2163 if (db->db_user_ptr == old_user_ptr) { 2164 db->db_user_ptr = user_ptr; 2165 db->db_user_data_ptr_ptr = user_data_ptr_ptr; 2166 db->db_evict_func = evict_func; 2167 2168 dbuf_update_data(db); 2169 } else { 2170 old_user_ptr = db->db_user_ptr; 2171 } 2172 2173 mutex_exit(&db->db_mtx); 2174 return (old_user_ptr); 2175 } 2176 2177 void * 2178 dmu_buf_get_user(dmu_buf_t *db_fake) 2179 { 2180 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2181 ASSERT(!refcount_is_zero(&db->db_holds)); 2182 2183 return (db->db_user_ptr); 2184 } 2185 2186 boolean_t 2187 dmu_buf_freeable(dmu_buf_t *dbuf) 2188 { 2189 boolean_t res = B_FALSE; 2190 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf; 2191 2192 if (db->db_blkptr) 2193 res = dsl_dataset_block_freeable(db->db_objset->os_dsl_dataset, 2194 db->db_blkptr, db->db_blkptr->blk_birth); 2195 2196 return (res); 2197 } 2198 2199 blkptr_t * 2200 dmu_buf_get_blkptr(dmu_buf_t *db) 2201 { 2202 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db; 2203 return (dbi->db_blkptr); 2204 } 2205 2206 static void 2207 dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db) 2208 { 2209 /* ASSERT(dmu_tx_is_syncing(tx) */ 2210 ASSERT(MUTEX_HELD(&db->db_mtx)); 2211 2212 if (db->db_blkptr != NULL) 2213 return; 2214 2215 if (db->db_blkid == DMU_SPILL_BLKID) { 2216 db->db_blkptr = &dn->dn_phys->dn_spill; 2217 BP_ZERO(db->db_blkptr); 2218 return; 2219 } 2220 if (db->db_level == dn->dn_phys->dn_nlevels-1) { 2221 /* 2222 * This buffer was allocated at a time when there was 2223 * no available blkptrs from the dnode, or it was 2224 * inappropriate to hook it in (i.e., nlevels mis-match). 2225 */ 2226 ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr); 2227 ASSERT(db->db_parent == NULL); 2228 db->db_parent = dn->dn_dbuf; 2229 db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid]; 2230 DBUF_VERIFY(db); 2231 } else { 2232 dmu_buf_impl_t *parent = db->db_parent; 2233 int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2234 2235 ASSERT(dn->dn_phys->dn_nlevels > 1); 2236 if (parent == NULL) { 2237 mutex_exit(&db->db_mtx); 2238 rw_enter(&dn->dn_struct_rwlock, RW_READER); 2239 (void) dbuf_hold_impl(dn, db->db_level+1, 2240 db->db_blkid >> epbs, FALSE, db, &parent); 2241 rw_exit(&dn->dn_struct_rwlock); 2242 mutex_enter(&db->db_mtx); 2243 db->db_parent = parent; 2244 } 2245 db->db_blkptr = (blkptr_t *)parent->db.db_data + 2246 (db->db_blkid & ((1ULL << epbs) - 1)); 2247 DBUF_VERIFY(db); 2248 } 2249 } 2250 2251 static void 2252 dbuf_sync_indirect(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 2253 { 2254 dmu_buf_impl_t *db = dr->dr_dbuf; 2255 dnode_t *dn; 2256 zio_t *zio; 2257 2258 ASSERT(dmu_tx_is_syncing(tx)); 2259 2260 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 2261 2262 mutex_enter(&db->db_mtx); 2263 2264 ASSERT(db->db_level > 0); 2265 DBUF_VERIFY(db); 2266 2267 /* Read the block if it hasn't been read yet. */ 2268 if (db->db_buf == NULL) { 2269 mutex_exit(&db->db_mtx); 2270 (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED); 2271 mutex_enter(&db->db_mtx); 2272 } 2273 ASSERT3U(db->db_state, ==, DB_CACHED); 2274 ASSERT(db->db_buf != NULL); 2275 2276 DB_DNODE_ENTER(db); 2277 dn = DB_DNODE(db); 2278 /* Indirect block size must match what the dnode thinks it is. */ 2279 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2280 dbuf_check_blkptr(dn, db); 2281 DB_DNODE_EXIT(db); 2282 2283 /* Provide the pending dirty record to child dbufs */ 2284 db->db_data_pending = dr; 2285 2286 mutex_exit(&db->db_mtx); 2287 dbuf_write(dr, db->db_buf, tx); 2288 2289 zio = dr->dr_zio; 2290 mutex_enter(&dr->dt.di.dr_mtx); 2291 dbuf_sync_list(&dr->dt.di.dr_children, tx); 2292 ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 2293 mutex_exit(&dr->dt.di.dr_mtx); 2294 zio_nowait(zio); 2295 } 2296 2297 static void 2298 dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 2299 { 2300 arc_buf_t **datap = &dr->dt.dl.dr_data; 2301 dmu_buf_impl_t *db = dr->dr_dbuf; 2302 dnode_t *dn; 2303 objset_t *os; 2304 uint64_t txg = tx->tx_txg; 2305 2306 ASSERT(dmu_tx_is_syncing(tx)); 2307 2308 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 2309 2310 mutex_enter(&db->db_mtx); 2311 /* 2312 * To be synced, we must be dirtied. But we 2313 * might have been freed after the dirty. 2314 */ 2315 if (db->db_state == DB_UNCACHED) { 2316 /* This buffer has been freed since it was dirtied */ 2317 ASSERT(db->db.db_data == NULL); 2318 } else if (db->db_state == DB_FILL) { 2319 /* This buffer was freed and is now being re-filled */ 2320 ASSERT(db->db.db_data != dr->dt.dl.dr_data); 2321 } else { 2322 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_NOFILL); 2323 } 2324 DBUF_VERIFY(db); 2325 2326 DB_DNODE_ENTER(db); 2327 dn = DB_DNODE(db); 2328 2329 if (db->db_blkid == DMU_SPILL_BLKID) { 2330 mutex_enter(&dn->dn_mtx); 2331 dn->dn_phys->dn_flags |= DNODE_FLAG_SPILL_BLKPTR; 2332 mutex_exit(&dn->dn_mtx); 2333 } 2334 2335 /* 2336 * If this is a bonus buffer, simply copy the bonus data into the 2337 * dnode. It will be written out when the dnode is synced (and it 2338 * will be synced, since it must have been dirty for dbuf_sync to 2339 * be called). 2340 */ 2341 if (db->db_blkid == DMU_BONUS_BLKID) { 2342 dbuf_dirty_record_t **drp; 2343 2344 ASSERT(*datap != NULL); 2345 ASSERT0(db->db_level); 2346 ASSERT3U(dn->dn_phys->dn_bonuslen, <=, DN_MAX_BONUSLEN); 2347 bcopy(*datap, DN_BONUS(dn->dn_phys), dn->dn_phys->dn_bonuslen); 2348 DB_DNODE_EXIT(db); 2349 2350 if (*datap != db->db.db_data) { 2351 zio_buf_free(*datap, DN_MAX_BONUSLEN); 2352 arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 2353 } 2354 db->db_data_pending = NULL; 2355 drp = &db->db_last_dirty; 2356 while (*drp != dr) 2357 drp = &(*drp)->dr_next; 2358 ASSERT(dr->dr_next == NULL); 2359 ASSERT(dr->dr_dbuf == db); 2360 *drp = dr->dr_next; 2361 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 2362 ASSERT(db->db_dirtycnt > 0); 2363 db->db_dirtycnt -= 1; 2364 dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg); 2365 return; 2366 } 2367 2368 os = dn->dn_objset; 2369 2370 /* 2371 * This function may have dropped the db_mtx lock allowing a dmu_sync 2372 * operation to sneak in. As a result, we need to ensure that we 2373 * don't check the dr_override_state until we have returned from 2374 * dbuf_check_blkptr. 2375 */ 2376 dbuf_check_blkptr(dn, db); 2377 2378 /* 2379 * If this buffer is in the middle of an immediate write, 2380 * wait for the synchronous IO to complete. 2381 */ 2382 while (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC) { 2383 ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT); 2384 cv_wait(&db->db_changed, &db->db_mtx); 2385 ASSERT(dr->dt.dl.dr_override_state != DR_NOT_OVERRIDDEN); 2386 } 2387 2388 if (db->db_state != DB_NOFILL && 2389 dn->dn_object != DMU_META_DNODE_OBJECT && 2390 refcount_count(&db->db_holds) > 1 && 2391 dr->dt.dl.dr_override_state != DR_OVERRIDDEN && 2392 *datap == db->db_buf) { 2393 /* 2394 * If this buffer is currently "in use" (i.e., there 2395 * are active holds and db_data still references it), 2396 * then make a copy before we start the write so that 2397 * any modifications from the open txg will not leak 2398 * into this write. 2399 * 2400 * NOTE: this copy does not need to be made for 2401 * objects only modified in the syncing context (e.g. 2402 * DNONE_DNODE blocks). 2403 */ 2404 int blksz = arc_buf_size(*datap); 2405 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 2406 *datap = arc_buf_alloc(os->os_spa, blksz, db, type); 2407 bcopy(db->db.db_data, (*datap)->b_data, blksz); 2408 } 2409 db->db_data_pending = dr; 2410 2411 mutex_exit(&db->db_mtx); 2412 2413 dbuf_write(dr, *datap, tx); 2414 2415 ASSERT(!list_link_active(&dr->dr_dirty_node)); 2416 if (dn->dn_object == DMU_META_DNODE_OBJECT) { 2417 list_insert_tail(&dn->dn_dirty_records[txg&TXG_MASK], dr); 2418 DB_DNODE_EXIT(db); 2419 } else { 2420 /* 2421 * Although zio_nowait() does not "wait for an IO", it does 2422 * initiate the IO. If this is an empty write it seems plausible 2423 * that the IO could actually be completed before the nowait 2424 * returns. We need to DB_DNODE_EXIT() first in case 2425 * zio_nowait() invalidates the dbuf. 2426 */ 2427 DB_DNODE_EXIT(db); 2428 zio_nowait(dr->dr_zio); 2429 } 2430 } 2431 2432 void 2433 dbuf_sync_list(list_t *list, dmu_tx_t *tx) 2434 { 2435 dbuf_dirty_record_t *dr; 2436 2437 while (dr = list_head(list)) { 2438 if (dr->dr_zio != NULL) { 2439 /* 2440 * If we find an already initialized zio then we 2441 * are processing the meta-dnode, and we have finished. 2442 * The dbufs for all dnodes are put back on the list 2443 * during processing, so that we can zio_wait() 2444 * these IOs after initiating all child IOs. 2445 */ 2446 ASSERT3U(dr->dr_dbuf->db.db_object, ==, 2447 DMU_META_DNODE_OBJECT); 2448 break; 2449 } 2450 list_remove(list, dr); 2451 if (dr->dr_dbuf->db_level > 0) 2452 dbuf_sync_indirect(dr, tx); 2453 else 2454 dbuf_sync_leaf(dr, tx); 2455 } 2456 } 2457 2458 /* ARGSUSED */ 2459 static void 2460 dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb) 2461 { 2462 dmu_buf_impl_t *db = vdb; 2463 dnode_t *dn; 2464 blkptr_t *bp = zio->io_bp; 2465 blkptr_t *bp_orig = &zio->io_bp_orig; 2466 spa_t *spa = zio->io_spa; 2467 int64_t delta; 2468 uint64_t fill = 0; 2469 int i; 2470 2471 ASSERT(db->db_blkptr == bp); 2472 2473 DB_DNODE_ENTER(db); 2474 dn = DB_DNODE(db); 2475 delta = bp_get_dsize_sync(spa, bp) - bp_get_dsize_sync(spa, bp_orig); 2476 dnode_diduse_space(dn, delta - zio->io_prev_space_delta); 2477 zio->io_prev_space_delta = delta; 2478 2479 if (BP_IS_HOLE(bp)) { 2480 ASSERT(bp->blk_fill == 0); 2481 DB_DNODE_EXIT(db); 2482 return; 2483 } 2484 2485 ASSERT((db->db_blkid != DMU_SPILL_BLKID && 2486 BP_GET_TYPE(bp) == dn->dn_type) || 2487 (db->db_blkid == DMU_SPILL_BLKID && 2488 BP_GET_TYPE(bp) == dn->dn_bonustype)); 2489 ASSERT(BP_GET_LEVEL(bp) == db->db_level); 2490 2491 mutex_enter(&db->db_mtx); 2492 2493 #ifdef ZFS_DEBUG 2494 if (db->db_blkid == DMU_SPILL_BLKID) { 2495 ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); 2496 ASSERT(!(BP_IS_HOLE(db->db_blkptr)) && 2497 db->db_blkptr == &dn->dn_phys->dn_spill); 2498 } 2499 #endif 2500 2501 if (db->db_level == 0) { 2502 mutex_enter(&dn->dn_mtx); 2503 if (db->db_blkid > dn->dn_phys->dn_maxblkid && 2504 db->db_blkid != DMU_SPILL_BLKID) 2505 dn->dn_phys->dn_maxblkid = db->db_blkid; 2506 mutex_exit(&dn->dn_mtx); 2507 2508 if (dn->dn_type == DMU_OT_DNODE) { 2509 dnode_phys_t *dnp = db->db.db_data; 2510 for (i = db->db.db_size >> DNODE_SHIFT; i > 0; 2511 i--, dnp++) { 2512 if (dnp->dn_type != DMU_OT_NONE) 2513 fill++; 2514 } 2515 } else { 2516 fill = 1; 2517 } 2518 } else { 2519 blkptr_t *ibp = db->db.db_data; 2520 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2521 for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, ibp++) { 2522 if (BP_IS_HOLE(ibp)) 2523 continue; 2524 fill += ibp->blk_fill; 2525 } 2526 } 2527 DB_DNODE_EXIT(db); 2528 2529 bp->blk_fill = fill; 2530 2531 mutex_exit(&db->db_mtx); 2532 } 2533 2534 /* 2535 * The SPA will call this callback several times for each zio - once 2536 * for every physical child i/o (zio->io_phys_children times). This 2537 * allows the DMU to monitor the progress of each logical i/o. For example, 2538 * there may be 2 copies of an indirect block, or many fragments of a RAID-Z 2539 * block. There may be a long delay before all copies/fragments are completed, 2540 * so this callback allows us to retire dirty space gradually, as the physical 2541 * i/os complete. 2542 */ 2543 /* ARGSUSED */ 2544 static void 2545 dbuf_write_physdone(zio_t *zio, arc_buf_t *buf, void *arg) 2546 { 2547 dmu_buf_impl_t *db = arg; 2548 objset_t *os = db->db_objset; 2549 dsl_pool_t *dp = dmu_objset_pool(os); 2550 dbuf_dirty_record_t *dr; 2551 int delta = 0; 2552 2553 dr = db->db_data_pending; 2554 ASSERT3U(dr->dr_txg, ==, zio->io_txg); 2555 2556 /* 2557 * The callback will be called io_phys_children times. Retire one 2558 * portion of our dirty space each time we are called. Any rounding 2559 * error will be cleaned up by dsl_pool_sync()'s call to 2560 * dsl_pool_undirty_space(). 2561 */ 2562 delta = dr->dr_accounted / zio->io_phys_children; 2563 dsl_pool_undirty_space(dp, delta, zio->io_txg); 2564 } 2565 2566 /* ARGSUSED */ 2567 static void 2568 dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb) 2569 { 2570 dmu_buf_impl_t *db = vdb; 2571 blkptr_t *bp = zio->io_bp; 2572 blkptr_t *bp_orig = &zio->io_bp_orig; 2573 uint64_t txg = zio->io_txg; 2574 dbuf_dirty_record_t **drp, *dr; 2575 2576 ASSERT0(zio->io_error); 2577 ASSERT(db->db_blkptr == bp); 2578 2579 /* 2580 * For nopwrites and rewrites we ensure that the bp matches our 2581 * original and bypass all the accounting. 2582 */ 2583 if (zio->io_flags & (ZIO_FLAG_IO_REWRITE | ZIO_FLAG_NOPWRITE)) { 2584 ASSERT(BP_EQUAL(bp, bp_orig)); 2585 } else { 2586 objset_t *os; 2587 dsl_dataset_t *ds; 2588 dmu_tx_t *tx; 2589 2590 DB_GET_OBJSET(&os, db); 2591 ds = os->os_dsl_dataset; 2592 tx = os->os_synctx; 2593 2594 (void) dsl_dataset_block_kill(ds, bp_orig, tx, B_TRUE); 2595 dsl_dataset_block_born(ds, bp, tx); 2596 } 2597 2598 mutex_enter(&db->db_mtx); 2599 2600 DBUF_VERIFY(db); 2601 2602 drp = &db->db_last_dirty; 2603 while ((dr = *drp) != db->db_data_pending) 2604 drp = &dr->dr_next; 2605 ASSERT(!list_link_active(&dr->dr_dirty_node)); 2606 ASSERT(dr->dr_txg == txg); 2607 ASSERT(dr->dr_dbuf == db); 2608 ASSERT(dr->dr_next == NULL); 2609 *drp = dr->dr_next; 2610 2611 #ifdef ZFS_DEBUG 2612 if (db->db_blkid == DMU_SPILL_BLKID) { 2613 dnode_t *dn; 2614 2615 DB_DNODE_ENTER(db); 2616 dn = DB_DNODE(db); 2617 ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); 2618 ASSERT(!(BP_IS_HOLE(db->db_blkptr)) && 2619 db->db_blkptr == &dn->dn_phys->dn_spill); 2620 DB_DNODE_EXIT(db); 2621 } 2622 #endif 2623 2624 if (db->db_level == 0) { 2625 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 2626 ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN); 2627 if (db->db_state != DB_NOFILL) { 2628 if (dr->dt.dl.dr_data != db->db_buf) 2629 VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, 2630 db)); 2631 else if (!arc_released(db->db_buf)) 2632 arc_set_callback(db->db_buf, dbuf_do_evict, db); 2633 } 2634 } else { 2635 dnode_t *dn; 2636 2637 DB_DNODE_ENTER(db); 2638 dn = DB_DNODE(db); 2639 ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 2640 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2641 if (!BP_IS_HOLE(db->db_blkptr)) { 2642 int epbs = 2643 dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2644 ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, 2645 db->db.db_size); 2646 ASSERT3U(dn->dn_phys->dn_maxblkid 2647 >> (db->db_level * epbs), >=, db->db_blkid); 2648 arc_set_callback(db->db_buf, dbuf_do_evict, db); 2649 } 2650 DB_DNODE_EXIT(db); 2651 mutex_destroy(&dr->dt.di.dr_mtx); 2652 list_destroy(&dr->dt.di.dr_children); 2653 } 2654 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 2655 2656 cv_broadcast(&db->db_changed); 2657 ASSERT(db->db_dirtycnt > 0); 2658 db->db_dirtycnt -= 1; 2659 db->db_data_pending = NULL; 2660 2661 dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg); 2662 } 2663 2664 static void 2665 dbuf_write_nofill_ready(zio_t *zio) 2666 { 2667 dbuf_write_ready(zio, NULL, zio->io_private); 2668 } 2669 2670 static void 2671 dbuf_write_nofill_done(zio_t *zio) 2672 { 2673 dbuf_write_done(zio, NULL, zio->io_private); 2674 } 2675 2676 static void 2677 dbuf_write_override_ready(zio_t *zio) 2678 { 2679 dbuf_dirty_record_t *dr = zio->io_private; 2680 dmu_buf_impl_t *db = dr->dr_dbuf; 2681 2682 dbuf_write_ready(zio, NULL, db); 2683 } 2684 2685 static void 2686 dbuf_write_override_done(zio_t *zio) 2687 { 2688 dbuf_dirty_record_t *dr = zio->io_private; 2689 dmu_buf_impl_t *db = dr->dr_dbuf; 2690 blkptr_t *obp = &dr->dt.dl.dr_overridden_by; 2691 2692 mutex_enter(&db->db_mtx); 2693 if (!BP_EQUAL(zio->io_bp, obp)) { 2694 if (!BP_IS_HOLE(obp)) 2695 dsl_free(spa_get_dsl(zio->io_spa), zio->io_txg, obp); 2696 arc_release(dr->dt.dl.dr_data, db); 2697 } 2698 mutex_exit(&db->db_mtx); 2699 2700 dbuf_write_done(zio, NULL, db); 2701 } 2702 2703 /* Issue I/O to commit a dirty buffer to disk. */ 2704 static void 2705 dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx) 2706 { 2707 dmu_buf_impl_t *db = dr->dr_dbuf; 2708 dnode_t *dn; 2709 objset_t *os; 2710 dmu_buf_impl_t *parent = db->db_parent; 2711 uint64_t txg = tx->tx_txg; 2712 zbookmark_t zb; 2713 zio_prop_t zp; 2714 zio_t *zio; 2715 int wp_flag = 0; 2716 2717 DB_DNODE_ENTER(db); 2718 dn = DB_DNODE(db); 2719 os = dn->dn_objset; 2720 2721 if (db->db_state != DB_NOFILL) { 2722 if (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE) { 2723 /* 2724 * Private object buffers are released here rather 2725 * than in dbuf_dirty() since they are only modified 2726 * in the syncing context and we don't want the 2727 * overhead of making multiple copies of the data. 2728 */ 2729 if (BP_IS_HOLE(db->db_blkptr)) { 2730 arc_buf_thaw(data); 2731 } else { 2732 dbuf_release_bp(db); 2733 } 2734 } 2735 } 2736 2737 if (parent != dn->dn_dbuf) { 2738 /* Our parent is an indirect block. */ 2739 /* We have a dirty parent that has been scheduled for write. */ 2740 ASSERT(parent && parent->db_data_pending); 2741 /* Our parent's buffer is one level closer to the dnode. */ 2742 ASSERT(db->db_level == parent->db_level-1); 2743 /* 2744 * We're about to modify our parent's db_data by modifying 2745 * our block pointer, so the parent must be released. 2746 */ 2747 ASSERT(arc_released(parent->db_buf)); 2748 zio = parent->db_data_pending->dr_zio; 2749 } else { 2750 /* Our parent is the dnode itself. */ 2751 ASSERT((db->db_level == dn->dn_phys->dn_nlevels-1 && 2752 db->db_blkid != DMU_SPILL_BLKID) || 2753 (db->db_blkid == DMU_SPILL_BLKID && db->db_level == 0)); 2754 if (db->db_blkid != DMU_SPILL_BLKID) 2755 ASSERT3P(db->db_blkptr, ==, 2756 &dn->dn_phys->dn_blkptr[db->db_blkid]); 2757 zio = dn->dn_zio; 2758 } 2759 2760 ASSERT(db->db_level == 0 || data == db->db_buf); 2761 ASSERT3U(db->db_blkptr->blk_birth, <=, txg); 2762 ASSERT(zio); 2763 2764 SET_BOOKMARK(&zb, os->os_dsl_dataset ? 2765 os->os_dsl_dataset->ds_object : DMU_META_OBJSET, 2766 db->db.db_object, db->db_level, db->db_blkid); 2767 2768 if (db->db_blkid == DMU_SPILL_BLKID) 2769 wp_flag = WP_SPILL; 2770 wp_flag |= (db->db_state == DB_NOFILL) ? WP_NOFILL : 0; 2771 2772 dmu_write_policy(os, dn, db->db_level, wp_flag, &zp); 2773 DB_DNODE_EXIT(db); 2774 2775 if (db->db_level == 0 && dr->dt.dl.dr_override_state == DR_OVERRIDDEN) { 2776 ASSERT(db->db_state != DB_NOFILL); 2777 dr->dr_zio = zio_write(zio, os->os_spa, txg, 2778 db->db_blkptr, data->b_data, arc_buf_size(data), &zp, 2779 dbuf_write_override_ready, NULL, dbuf_write_override_done, 2780 dr, ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); 2781 mutex_enter(&db->db_mtx); 2782 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 2783 zio_write_override(dr->dr_zio, &dr->dt.dl.dr_overridden_by, 2784 dr->dt.dl.dr_copies, dr->dt.dl.dr_nopwrite); 2785 mutex_exit(&db->db_mtx); 2786 } else if (db->db_state == DB_NOFILL) { 2787 ASSERT(zp.zp_checksum == ZIO_CHECKSUM_OFF || 2788 zp.zp_checksum == ZIO_CHECKSUM_NOPARITY); 2789 dr->dr_zio = zio_write(zio, os->os_spa, txg, 2790 db->db_blkptr, NULL, db->db.db_size, &zp, 2791 dbuf_write_nofill_ready, NULL, dbuf_write_nofill_done, db, 2792 ZIO_PRIORITY_ASYNC_WRITE, 2793 ZIO_FLAG_MUSTSUCCEED | ZIO_FLAG_NODATA, &zb); 2794 } else { 2795 ASSERT(arc_released(data)); 2796 dr->dr_zio = arc_write(zio, os->os_spa, txg, 2797 db->db_blkptr, data, DBUF_IS_L2CACHEABLE(db), 2798 DBUF_IS_L2COMPRESSIBLE(db), &zp, dbuf_write_ready, 2799 dbuf_write_physdone, dbuf_write_done, db, 2800 ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); 2801 } 2802 } 2803