1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright 2011 Nexenta Systems, Inc. All rights reserved. 24 * Copyright (c) 2012 by Delphix. All rights reserved. 25 */ 26 27 #include <sys/zfs_context.h> 28 #include <sys/dmu.h> 29 #include <sys/dmu_impl.h> 30 #include <sys/dbuf.h> 31 #include <sys/dmu_objset.h> 32 #include <sys/dsl_dataset.h> 33 #include <sys/dsl_dir.h> 34 #include <sys/dmu_tx.h> 35 #include <sys/spa.h> 36 #include <sys/zio.h> 37 #include <sys/dmu_zfetch.h> 38 #include <sys/sa.h> 39 #include <sys/sa_impl.h> 40 41 static void dbuf_destroy(dmu_buf_impl_t *db); 42 static int dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx); 43 static void dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx); 44 45 /* 46 * Global data structures and functions for the dbuf cache. 47 */ 48 static kmem_cache_t *dbuf_cache; 49 50 /* ARGSUSED */ 51 static int 52 dbuf_cons(void *vdb, void *unused, int kmflag) 53 { 54 dmu_buf_impl_t *db = vdb; 55 bzero(db, sizeof (dmu_buf_impl_t)); 56 57 mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL); 58 cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL); 59 refcount_create(&db->db_holds); 60 return (0); 61 } 62 63 /* ARGSUSED */ 64 static void 65 dbuf_dest(void *vdb, void *unused) 66 { 67 dmu_buf_impl_t *db = vdb; 68 mutex_destroy(&db->db_mtx); 69 cv_destroy(&db->db_changed); 70 refcount_destroy(&db->db_holds); 71 } 72 73 /* 74 * dbuf hash table routines 75 */ 76 static dbuf_hash_table_t dbuf_hash_table; 77 78 static uint64_t dbuf_hash_count; 79 80 static uint64_t 81 dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid) 82 { 83 uintptr_t osv = (uintptr_t)os; 84 uint64_t crc = -1ULL; 85 86 ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 87 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (lvl)) & 0xFF]; 88 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF]; 89 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF]; 90 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF]; 91 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 0)) & 0xFF]; 92 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 8)) & 0xFF]; 93 94 crc ^= (osv>>14) ^ (obj>>16) ^ (blkid>>16); 95 96 return (crc); 97 } 98 99 #define DBUF_HASH(os, obj, level, blkid) dbuf_hash(os, obj, level, blkid); 100 101 #define DBUF_EQUAL(dbuf, os, obj, level, blkid) \ 102 ((dbuf)->db.db_object == (obj) && \ 103 (dbuf)->db_objset == (os) && \ 104 (dbuf)->db_level == (level) && \ 105 (dbuf)->db_blkid == (blkid)) 106 107 dmu_buf_impl_t * 108 dbuf_find(dnode_t *dn, uint8_t level, uint64_t blkid) 109 { 110 dbuf_hash_table_t *h = &dbuf_hash_table; 111 objset_t *os = dn->dn_objset; 112 uint64_t obj = dn->dn_object; 113 uint64_t hv = DBUF_HASH(os, obj, level, blkid); 114 uint64_t idx = hv & h->hash_table_mask; 115 dmu_buf_impl_t *db; 116 117 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 118 for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) { 119 if (DBUF_EQUAL(db, os, obj, level, blkid)) { 120 mutex_enter(&db->db_mtx); 121 if (db->db_state != DB_EVICTING) { 122 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 123 return (db); 124 } 125 mutex_exit(&db->db_mtx); 126 } 127 } 128 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 129 return (NULL); 130 } 131 132 /* 133 * Insert an entry into the hash table. If there is already an element 134 * equal to elem in the hash table, then the already existing element 135 * will be returned and the new element will not be inserted. 136 * Otherwise returns NULL. 137 */ 138 static dmu_buf_impl_t * 139 dbuf_hash_insert(dmu_buf_impl_t *db) 140 { 141 dbuf_hash_table_t *h = &dbuf_hash_table; 142 objset_t *os = db->db_objset; 143 uint64_t obj = db->db.db_object; 144 int level = db->db_level; 145 uint64_t blkid = db->db_blkid; 146 uint64_t hv = DBUF_HASH(os, obj, level, blkid); 147 uint64_t idx = hv & h->hash_table_mask; 148 dmu_buf_impl_t *dbf; 149 150 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 151 for (dbf = h->hash_table[idx]; dbf != NULL; dbf = dbf->db_hash_next) { 152 if (DBUF_EQUAL(dbf, os, obj, level, blkid)) { 153 mutex_enter(&dbf->db_mtx); 154 if (dbf->db_state != DB_EVICTING) { 155 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 156 return (dbf); 157 } 158 mutex_exit(&dbf->db_mtx); 159 } 160 } 161 162 mutex_enter(&db->db_mtx); 163 db->db_hash_next = h->hash_table[idx]; 164 h->hash_table[idx] = db; 165 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 166 atomic_add_64(&dbuf_hash_count, 1); 167 168 return (NULL); 169 } 170 171 /* 172 * Remove an entry from the hash table. This operation will 173 * fail if there are any existing holds on the db. 174 */ 175 static void 176 dbuf_hash_remove(dmu_buf_impl_t *db) 177 { 178 dbuf_hash_table_t *h = &dbuf_hash_table; 179 uint64_t hv = DBUF_HASH(db->db_objset, db->db.db_object, 180 db->db_level, db->db_blkid); 181 uint64_t idx = hv & h->hash_table_mask; 182 dmu_buf_impl_t *dbf, **dbp; 183 184 /* 185 * We musn't hold db_mtx to maintin lock ordering: 186 * DBUF_HASH_MUTEX > db_mtx. 187 */ 188 ASSERT(refcount_is_zero(&db->db_holds)); 189 ASSERT(db->db_state == DB_EVICTING); 190 ASSERT(!MUTEX_HELD(&db->db_mtx)); 191 192 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 193 dbp = &h->hash_table[idx]; 194 while ((dbf = *dbp) != db) { 195 dbp = &dbf->db_hash_next; 196 ASSERT(dbf != NULL); 197 } 198 *dbp = db->db_hash_next; 199 db->db_hash_next = NULL; 200 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 201 atomic_add_64(&dbuf_hash_count, -1); 202 } 203 204 static arc_evict_func_t dbuf_do_evict; 205 206 static void 207 dbuf_evict_user(dmu_buf_impl_t *db) 208 { 209 ASSERT(MUTEX_HELD(&db->db_mtx)); 210 211 if (db->db_level != 0 || db->db_evict_func == NULL) 212 return; 213 214 if (db->db_user_data_ptr_ptr) 215 *db->db_user_data_ptr_ptr = db->db.db_data; 216 db->db_evict_func(&db->db, db->db_user_ptr); 217 db->db_user_ptr = NULL; 218 db->db_user_data_ptr_ptr = NULL; 219 db->db_evict_func = NULL; 220 } 221 222 boolean_t 223 dbuf_is_metadata(dmu_buf_impl_t *db) 224 { 225 if (db->db_level > 0) { 226 return (B_TRUE); 227 } else { 228 boolean_t is_metadata; 229 230 DB_DNODE_ENTER(db); 231 is_metadata = DMU_OT_IS_METADATA(DB_DNODE(db)->dn_type); 232 DB_DNODE_EXIT(db); 233 234 return (is_metadata); 235 } 236 } 237 238 void 239 dbuf_evict(dmu_buf_impl_t *db) 240 { 241 ASSERT(MUTEX_HELD(&db->db_mtx)); 242 ASSERT(db->db_buf == NULL); 243 ASSERT(db->db_data_pending == NULL); 244 245 dbuf_clear(db); 246 dbuf_destroy(db); 247 } 248 249 void 250 dbuf_init(void) 251 { 252 uint64_t hsize = 1ULL << 16; 253 dbuf_hash_table_t *h = &dbuf_hash_table; 254 int i; 255 256 /* 257 * The hash table is big enough to fill all of physical memory 258 * with an average 4K block size. The table will take up 259 * totalmem*sizeof(void*)/4K (i.e. 2MB/GB with 8-byte pointers). 260 */ 261 while (hsize * 4096 < physmem * PAGESIZE) 262 hsize <<= 1; 263 264 retry: 265 h->hash_table_mask = hsize - 1; 266 h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP); 267 if (h->hash_table == NULL) { 268 /* XXX - we should really return an error instead of assert */ 269 ASSERT(hsize > (1ULL << 10)); 270 hsize >>= 1; 271 goto retry; 272 } 273 274 dbuf_cache = kmem_cache_create("dmu_buf_impl_t", 275 sizeof (dmu_buf_impl_t), 276 0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0); 277 278 for (i = 0; i < DBUF_MUTEXES; i++) 279 mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL); 280 } 281 282 void 283 dbuf_fini(void) 284 { 285 dbuf_hash_table_t *h = &dbuf_hash_table; 286 int i; 287 288 for (i = 0; i < DBUF_MUTEXES; i++) 289 mutex_destroy(&h->hash_mutexes[i]); 290 kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *)); 291 kmem_cache_destroy(dbuf_cache); 292 } 293 294 /* 295 * Other stuff. 296 */ 297 298 #ifdef ZFS_DEBUG 299 static void 300 dbuf_verify(dmu_buf_impl_t *db) 301 { 302 dnode_t *dn; 303 dbuf_dirty_record_t *dr; 304 305 ASSERT(MUTEX_HELD(&db->db_mtx)); 306 307 if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY)) 308 return; 309 310 ASSERT(db->db_objset != NULL); 311 DB_DNODE_ENTER(db); 312 dn = DB_DNODE(db); 313 if (dn == NULL) { 314 ASSERT(db->db_parent == NULL); 315 ASSERT(db->db_blkptr == NULL); 316 } else { 317 ASSERT3U(db->db.db_object, ==, dn->dn_object); 318 ASSERT3P(db->db_objset, ==, dn->dn_objset); 319 ASSERT3U(db->db_level, <, dn->dn_nlevels); 320 ASSERT(db->db_blkid == DMU_BONUS_BLKID || 321 db->db_blkid == DMU_SPILL_BLKID || 322 !list_is_empty(&dn->dn_dbufs)); 323 } 324 if (db->db_blkid == DMU_BONUS_BLKID) { 325 ASSERT(dn != NULL); 326 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 327 ASSERT3U(db->db.db_offset, ==, DMU_BONUS_BLKID); 328 } else if (db->db_blkid == DMU_SPILL_BLKID) { 329 ASSERT(dn != NULL); 330 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 331 ASSERT3U(db->db.db_offset, ==, 0); 332 } else { 333 ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size); 334 } 335 336 for (dr = db->db_data_pending; dr != NULL; dr = dr->dr_next) 337 ASSERT(dr->dr_dbuf == db); 338 339 for (dr = db->db_last_dirty; dr != NULL; dr = dr->dr_next) 340 ASSERT(dr->dr_dbuf == db); 341 342 /* 343 * We can't assert that db_size matches dn_datablksz because it 344 * can be momentarily different when another thread is doing 345 * dnode_set_blksz(). 346 */ 347 if (db->db_level == 0 && db->db.db_object == DMU_META_DNODE_OBJECT) { 348 dr = db->db_data_pending; 349 /* 350 * It should only be modified in syncing context, so 351 * make sure we only have one copy of the data. 352 */ 353 ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf); 354 } 355 356 /* verify db->db_blkptr */ 357 if (db->db_blkptr) { 358 if (db->db_parent == dn->dn_dbuf) { 359 /* db is pointed to by the dnode */ 360 /* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */ 361 if (DMU_OBJECT_IS_SPECIAL(db->db.db_object)) 362 ASSERT(db->db_parent == NULL); 363 else 364 ASSERT(db->db_parent != NULL); 365 if (db->db_blkid != DMU_SPILL_BLKID) 366 ASSERT3P(db->db_blkptr, ==, 367 &dn->dn_phys->dn_blkptr[db->db_blkid]); 368 } else { 369 /* db is pointed to by an indirect block */ 370 int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT; 371 ASSERT3U(db->db_parent->db_level, ==, db->db_level+1); 372 ASSERT3U(db->db_parent->db.db_object, ==, 373 db->db.db_object); 374 /* 375 * dnode_grow_indblksz() can make this fail if we don't 376 * have the struct_rwlock. XXX indblksz no longer 377 * grows. safe to do this now? 378 */ 379 if (RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 380 ASSERT3P(db->db_blkptr, ==, 381 ((blkptr_t *)db->db_parent->db.db_data + 382 db->db_blkid % epb)); 383 } 384 } 385 } 386 if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) && 387 (db->db_buf == NULL || db->db_buf->b_data) && 388 db->db.db_data && db->db_blkid != DMU_BONUS_BLKID && 389 db->db_state != DB_FILL && !dn->dn_free_txg) { 390 /* 391 * If the blkptr isn't set but they have nonzero data, 392 * it had better be dirty, otherwise we'll lose that 393 * data when we evict this buffer. 394 */ 395 if (db->db_dirtycnt == 0) { 396 uint64_t *buf = db->db.db_data; 397 int i; 398 399 for (i = 0; i < db->db.db_size >> 3; i++) { 400 ASSERT(buf[i] == 0); 401 } 402 } 403 } 404 DB_DNODE_EXIT(db); 405 } 406 #endif 407 408 static void 409 dbuf_update_data(dmu_buf_impl_t *db) 410 { 411 ASSERT(MUTEX_HELD(&db->db_mtx)); 412 if (db->db_level == 0 && db->db_user_data_ptr_ptr) { 413 ASSERT(!refcount_is_zero(&db->db_holds)); 414 *db->db_user_data_ptr_ptr = db->db.db_data; 415 } 416 } 417 418 static void 419 dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf) 420 { 421 ASSERT(MUTEX_HELD(&db->db_mtx)); 422 ASSERT(db->db_buf == NULL || !arc_has_callback(db->db_buf)); 423 db->db_buf = buf; 424 if (buf != NULL) { 425 ASSERT(buf->b_data != NULL); 426 db->db.db_data = buf->b_data; 427 if (!arc_released(buf)) 428 arc_set_callback(buf, dbuf_do_evict, db); 429 dbuf_update_data(db); 430 } else { 431 dbuf_evict_user(db); 432 db->db.db_data = NULL; 433 if (db->db_state != DB_NOFILL) 434 db->db_state = DB_UNCACHED; 435 } 436 } 437 438 /* 439 * Loan out an arc_buf for read. Return the loaned arc_buf. 440 */ 441 arc_buf_t * 442 dbuf_loan_arcbuf(dmu_buf_impl_t *db) 443 { 444 arc_buf_t *abuf; 445 446 mutex_enter(&db->db_mtx); 447 if (arc_released(db->db_buf) || refcount_count(&db->db_holds) > 1) { 448 int blksz = db->db.db_size; 449 spa_t *spa; 450 451 mutex_exit(&db->db_mtx); 452 DB_GET_SPA(&spa, db); 453 abuf = arc_loan_buf(spa, blksz); 454 bcopy(db->db.db_data, abuf->b_data, blksz); 455 } else { 456 abuf = db->db_buf; 457 arc_loan_inuse_buf(abuf, db); 458 dbuf_set_data(db, NULL); 459 mutex_exit(&db->db_mtx); 460 } 461 return (abuf); 462 } 463 464 uint64_t 465 dbuf_whichblock(dnode_t *dn, uint64_t offset) 466 { 467 if (dn->dn_datablkshift) { 468 return (offset >> dn->dn_datablkshift); 469 } else { 470 ASSERT3U(offset, <, dn->dn_datablksz); 471 return (0); 472 } 473 } 474 475 static void 476 dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb) 477 { 478 dmu_buf_impl_t *db = vdb; 479 480 mutex_enter(&db->db_mtx); 481 ASSERT3U(db->db_state, ==, DB_READ); 482 /* 483 * All reads are synchronous, so we must have a hold on the dbuf 484 */ 485 ASSERT(refcount_count(&db->db_holds) > 0); 486 ASSERT(db->db_buf == NULL); 487 ASSERT(db->db.db_data == NULL); 488 if (db->db_level == 0 && db->db_freed_in_flight) { 489 /* we were freed in flight; disregard any error */ 490 arc_release(buf, db); 491 bzero(buf->b_data, db->db.db_size); 492 arc_buf_freeze(buf); 493 db->db_freed_in_flight = FALSE; 494 dbuf_set_data(db, buf); 495 db->db_state = DB_CACHED; 496 } else if (zio == NULL || zio->io_error == 0) { 497 dbuf_set_data(db, buf); 498 db->db_state = DB_CACHED; 499 } else { 500 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 501 ASSERT3P(db->db_buf, ==, NULL); 502 VERIFY(arc_buf_remove_ref(buf, db) == 1); 503 db->db_state = DB_UNCACHED; 504 } 505 cv_broadcast(&db->db_changed); 506 dbuf_rele_and_unlock(db, NULL); 507 } 508 509 static void 510 dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t *flags) 511 { 512 dnode_t *dn; 513 spa_t *spa; 514 zbookmark_t zb; 515 uint32_t aflags = ARC_NOWAIT; 516 arc_buf_t *pbuf; 517 518 DB_DNODE_ENTER(db); 519 dn = DB_DNODE(db); 520 ASSERT(!refcount_is_zero(&db->db_holds)); 521 /* We need the struct_rwlock to prevent db_blkptr from changing. */ 522 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 523 ASSERT(MUTEX_HELD(&db->db_mtx)); 524 ASSERT(db->db_state == DB_UNCACHED); 525 ASSERT(db->db_buf == NULL); 526 527 if (db->db_blkid == DMU_BONUS_BLKID) { 528 int bonuslen = MIN(dn->dn_bonuslen, dn->dn_phys->dn_bonuslen); 529 530 ASSERT3U(bonuslen, <=, db->db.db_size); 531 db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN); 532 arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 533 if (bonuslen < DN_MAX_BONUSLEN) 534 bzero(db->db.db_data, DN_MAX_BONUSLEN); 535 if (bonuslen) 536 bcopy(DN_BONUS(dn->dn_phys), db->db.db_data, bonuslen); 537 DB_DNODE_EXIT(db); 538 dbuf_update_data(db); 539 db->db_state = DB_CACHED; 540 mutex_exit(&db->db_mtx); 541 return; 542 } 543 544 /* 545 * Recheck BP_IS_HOLE() after dnode_block_freed() in case dnode_sync() 546 * processes the delete record and clears the bp while we are waiting 547 * for the dn_mtx (resulting in a "no" from block_freed). 548 */ 549 if (db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr) || 550 (db->db_level == 0 && (dnode_block_freed(dn, db->db_blkid) || 551 BP_IS_HOLE(db->db_blkptr)))) { 552 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 553 554 dbuf_set_data(db, arc_buf_alloc(dn->dn_objset->os_spa, 555 db->db.db_size, db, type)); 556 DB_DNODE_EXIT(db); 557 bzero(db->db.db_data, db->db.db_size); 558 db->db_state = DB_CACHED; 559 *flags |= DB_RF_CACHED; 560 mutex_exit(&db->db_mtx); 561 return; 562 } 563 564 spa = dn->dn_objset->os_spa; 565 DB_DNODE_EXIT(db); 566 567 db->db_state = DB_READ; 568 mutex_exit(&db->db_mtx); 569 570 if (DBUF_IS_L2CACHEABLE(db)) 571 aflags |= ARC_L2CACHE; 572 573 SET_BOOKMARK(&zb, db->db_objset->os_dsl_dataset ? 574 db->db_objset->os_dsl_dataset->ds_object : DMU_META_OBJSET, 575 db->db.db_object, db->db_level, db->db_blkid); 576 577 dbuf_add_ref(db, NULL); 578 /* ZIO_FLAG_CANFAIL callers have to check the parent zio's error */ 579 580 if (db->db_parent) 581 pbuf = db->db_parent->db_buf; 582 else 583 pbuf = db->db_objset->os_phys_buf; 584 585 (void) dsl_read(zio, spa, db->db_blkptr, pbuf, 586 dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ, 587 (*flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED, 588 &aflags, &zb); 589 if (aflags & ARC_CACHED) 590 *flags |= DB_RF_CACHED; 591 } 592 593 int 594 dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags) 595 { 596 int err = 0; 597 int havepzio = (zio != NULL); 598 int prefetch; 599 dnode_t *dn; 600 601 /* 602 * We don't have to hold the mutex to check db_state because it 603 * can't be freed while we have a hold on the buffer. 604 */ 605 ASSERT(!refcount_is_zero(&db->db_holds)); 606 607 if (db->db_state == DB_NOFILL) 608 return (EIO); 609 610 DB_DNODE_ENTER(db); 611 dn = DB_DNODE(db); 612 if ((flags & DB_RF_HAVESTRUCT) == 0) 613 rw_enter(&dn->dn_struct_rwlock, RW_READER); 614 615 prefetch = db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 616 (flags & DB_RF_NOPREFETCH) == 0 && dn != NULL && 617 DBUF_IS_CACHEABLE(db); 618 619 mutex_enter(&db->db_mtx); 620 if (db->db_state == DB_CACHED) { 621 mutex_exit(&db->db_mtx); 622 if (prefetch) 623 dmu_zfetch(&dn->dn_zfetch, db->db.db_offset, 624 db->db.db_size, TRUE); 625 if ((flags & DB_RF_HAVESTRUCT) == 0) 626 rw_exit(&dn->dn_struct_rwlock); 627 DB_DNODE_EXIT(db); 628 } else if (db->db_state == DB_UNCACHED) { 629 spa_t *spa = dn->dn_objset->os_spa; 630 631 if (zio == NULL) 632 zio = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL); 633 dbuf_read_impl(db, zio, &flags); 634 635 /* dbuf_read_impl has dropped db_mtx for us */ 636 637 if (prefetch) 638 dmu_zfetch(&dn->dn_zfetch, db->db.db_offset, 639 db->db.db_size, flags & DB_RF_CACHED); 640 641 if ((flags & DB_RF_HAVESTRUCT) == 0) 642 rw_exit(&dn->dn_struct_rwlock); 643 DB_DNODE_EXIT(db); 644 645 if (!havepzio) 646 err = zio_wait(zio); 647 } else { 648 mutex_exit(&db->db_mtx); 649 if (prefetch) 650 dmu_zfetch(&dn->dn_zfetch, db->db.db_offset, 651 db->db.db_size, TRUE); 652 if ((flags & DB_RF_HAVESTRUCT) == 0) 653 rw_exit(&dn->dn_struct_rwlock); 654 DB_DNODE_EXIT(db); 655 656 mutex_enter(&db->db_mtx); 657 if ((flags & DB_RF_NEVERWAIT) == 0) { 658 while (db->db_state == DB_READ || 659 db->db_state == DB_FILL) { 660 ASSERT(db->db_state == DB_READ || 661 (flags & DB_RF_HAVESTRUCT) == 0); 662 cv_wait(&db->db_changed, &db->db_mtx); 663 } 664 if (db->db_state == DB_UNCACHED) 665 err = EIO; 666 } 667 mutex_exit(&db->db_mtx); 668 } 669 670 ASSERT(err || havepzio || db->db_state == DB_CACHED); 671 return (err); 672 } 673 674 static void 675 dbuf_noread(dmu_buf_impl_t *db) 676 { 677 ASSERT(!refcount_is_zero(&db->db_holds)); 678 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 679 mutex_enter(&db->db_mtx); 680 while (db->db_state == DB_READ || db->db_state == DB_FILL) 681 cv_wait(&db->db_changed, &db->db_mtx); 682 if (db->db_state == DB_UNCACHED) { 683 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 684 spa_t *spa; 685 686 ASSERT(db->db_buf == NULL); 687 ASSERT(db->db.db_data == NULL); 688 DB_GET_SPA(&spa, db); 689 dbuf_set_data(db, arc_buf_alloc(spa, db->db.db_size, db, type)); 690 db->db_state = DB_FILL; 691 } else if (db->db_state == DB_NOFILL) { 692 dbuf_set_data(db, NULL); 693 } else { 694 ASSERT3U(db->db_state, ==, DB_CACHED); 695 } 696 mutex_exit(&db->db_mtx); 697 } 698 699 /* 700 * This is our just-in-time copy function. It makes a copy of 701 * buffers, that have been modified in a previous transaction 702 * group, before we modify them in the current active group. 703 * 704 * This function is used in two places: when we are dirtying a 705 * buffer for the first time in a txg, and when we are freeing 706 * a range in a dnode that includes this buffer. 707 * 708 * Note that when we are called from dbuf_free_range() we do 709 * not put a hold on the buffer, we just traverse the active 710 * dbuf list for the dnode. 711 */ 712 static void 713 dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg) 714 { 715 dbuf_dirty_record_t *dr = db->db_last_dirty; 716 717 ASSERT(MUTEX_HELD(&db->db_mtx)); 718 ASSERT(db->db.db_data != NULL); 719 ASSERT(db->db_level == 0); 720 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT); 721 722 if (dr == NULL || 723 (dr->dt.dl.dr_data != 724 ((db->db_blkid == DMU_BONUS_BLKID) ? db->db.db_data : db->db_buf))) 725 return; 726 727 /* 728 * If the last dirty record for this dbuf has not yet synced 729 * and its referencing the dbuf data, either: 730 * reset the reference to point to a new copy, 731 * or (if there a no active holders) 732 * just null out the current db_data pointer. 733 */ 734 ASSERT(dr->dr_txg >= txg - 2); 735 if (db->db_blkid == DMU_BONUS_BLKID) { 736 /* Note that the data bufs here are zio_bufs */ 737 dr->dt.dl.dr_data = zio_buf_alloc(DN_MAX_BONUSLEN); 738 arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 739 bcopy(db->db.db_data, dr->dt.dl.dr_data, DN_MAX_BONUSLEN); 740 } else if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 741 int size = db->db.db_size; 742 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 743 spa_t *spa; 744 745 DB_GET_SPA(&spa, db); 746 dr->dt.dl.dr_data = arc_buf_alloc(spa, size, db, type); 747 bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size); 748 } else { 749 dbuf_set_data(db, NULL); 750 } 751 } 752 753 void 754 dbuf_unoverride(dbuf_dirty_record_t *dr) 755 { 756 dmu_buf_impl_t *db = dr->dr_dbuf; 757 blkptr_t *bp = &dr->dt.dl.dr_overridden_by; 758 uint64_t txg = dr->dr_txg; 759 760 ASSERT(MUTEX_HELD(&db->db_mtx)); 761 ASSERT(dr->dt.dl.dr_override_state != DR_IN_DMU_SYNC); 762 ASSERT(db->db_level == 0); 763 764 if (db->db_blkid == DMU_BONUS_BLKID || 765 dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN) 766 return; 767 768 ASSERT(db->db_data_pending != dr); 769 770 /* free this block */ 771 if (!BP_IS_HOLE(bp)) { 772 spa_t *spa; 773 774 DB_GET_SPA(&spa, db); 775 zio_free(spa, txg, bp); 776 } 777 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 778 /* 779 * Release the already-written buffer, so we leave it in 780 * a consistent dirty state. Note that all callers are 781 * modifying the buffer, so they will immediately do 782 * another (redundant) arc_release(). Therefore, leave 783 * the buf thawed to save the effort of freezing & 784 * immediately re-thawing it. 785 */ 786 arc_release(dr->dt.dl.dr_data, db); 787 } 788 789 /* 790 * Evict (if its unreferenced) or clear (if its referenced) any level-0 791 * data blocks in the free range, so that any future readers will find 792 * empty blocks. Also, if we happen accross any level-1 dbufs in the 793 * range that have not already been marked dirty, mark them dirty so 794 * they stay in memory. 795 */ 796 void 797 dbuf_free_range(dnode_t *dn, uint64_t start, uint64_t end, dmu_tx_t *tx) 798 { 799 dmu_buf_impl_t *db, *db_next; 800 uint64_t txg = tx->tx_txg; 801 int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 802 uint64_t first_l1 = start >> epbs; 803 uint64_t last_l1 = end >> epbs; 804 805 if (end > dn->dn_maxblkid && (end != DMU_SPILL_BLKID)) { 806 end = dn->dn_maxblkid; 807 last_l1 = end >> epbs; 808 } 809 dprintf_dnode(dn, "start=%llu end=%llu\n", start, end); 810 mutex_enter(&dn->dn_dbufs_mtx); 811 for (db = list_head(&dn->dn_dbufs); db; db = db_next) { 812 db_next = list_next(&dn->dn_dbufs, db); 813 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 814 815 if (db->db_level == 1 && 816 db->db_blkid >= first_l1 && db->db_blkid <= last_l1) { 817 mutex_enter(&db->db_mtx); 818 if (db->db_last_dirty && 819 db->db_last_dirty->dr_txg < txg) { 820 dbuf_add_ref(db, FTAG); 821 mutex_exit(&db->db_mtx); 822 dbuf_will_dirty(db, tx); 823 dbuf_rele(db, FTAG); 824 } else { 825 mutex_exit(&db->db_mtx); 826 } 827 } 828 829 if (db->db_level != 0) 830 continue; 831 dprintf_dbuf(db, "found buf %s\n", ""); 832 if (db->db_blkid < start || db->db_blkid > end) 833 continue; 834 835 /* found a level 0 buffer in the range */ 836 if (dbuf_undirty(db, tx)) 837 continue; 838 839 mutex_enter(&db->db_mtx); 840 if (db->db_state == DB_UNCACHED || 841 db->db_state == DB_NOFILL || 842 db->db_state == DB_EVICTING) { 843 ASSERT(db->db.db_data == NULL); 844 mutex_exit(&db->db_mtx); 845 continue; 846 } 847 if (db->db_state == DB_READ || db->db_state == DB_FILL) { 848 /* will be handled in dbuf_read_done or dbuf_rele */ 849 db->db_freed_in_flight = TRUE; 850 mutex_exit(&db->db_mtx); 851 continue; 852 } 853 if (refcount_count(&db->db_holds) == 0) { 854 ASSERT(db->db_buf); 855 dbuf_clear(db); 856 continue; 857 } 858 /* The dbuf is referenced */ 859 860 if (db->db_last_dirty != NULL) { 861 dbuf_dirty_record_t *dr = db->db_last_dirty; 862 863 if (dr->dr_txg == txg) { 864 /* 865 * This buffer is "in-use", re-adjust the file 866 * size to reflect that this buffer may 867 * contain new data when we sync. 868 */ 869 if (db->db_blkid != DMU_SPILL_BLKID && 870 db->db_blkid > dn->dn_maxblkid) 871 dn->dn_maxblkid = db->db_blkid; 872 dbuf_unoverride(dr); 873 } else { 874 /* 875 * This dbuf is not dirty in the open context. 876 * Either uncache it (if its not referenced in 877 * the open context) or reset its contents to 878 * empty. 879 */ 880 dbuf_fix_old_data(db, txg); 881 } 882 } 883 /* clear the contents if its cached */ 884 if (db->db_state == DB_CACHED) { 885 ASSERT(db->db.db_data != NULL); 886 arc_release(db->db_buf, db); 887 bzero(db->db.db_data, db->db.db_size); 888 arc_buf_freeze(db->db_buf); 889 } 890 891 mutex_exit(&db->db_mtx); 892 } 893 mutex_exit(&dn->dn_dbufs_mtx); 894 } 895 896 static int 897 dbuf_block_freeable(dmu_buf_impl_t *db) 898 { 899 dsl_dataset_t *ds = db->db_objset->os_dsl_dataset; 900 uint64_t birth_txg = 0; 901 902 /* 903 * We don't need any locking to protect db_blkptr: 904 * If it's syncing, then db_last_dirty will be set 905 * so we'll ignore db_blkptr. 906 */ 907 ASSERT(MUTEX_HELD(&db->db_mtx)); 908 if (db->db_last_dirty) 909 birth_txg = db->db_last_dirty->dr_txg; 910 else if (db->db_blkptr) 911 birth_txg = db->db_blkptr->blk_birth; 912 913 /* 914 * If we don't exist or are in a snapshot, we can't be freed. 915 * Don't pass the bp to dsl_dataset_block_freeable() since we 916 * are holding the db_mtx lock and might deadlock if we are 917 * prefetching a dedup-ed block. 918 */ 919 if (birth_txg) 920 return (ds == NULL || 921 dsl_dataset_block_freeable(ds, NULL, birth_txg)); 922 else 923 return (FALSE); 924 } 925 926 void 927 dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx) 928 { 929 arc_buf_t *buf, *obuf; 930 int osize = db->db.db_size; 931 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 932 dnode_t *dn; 933 934 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 935 936 DB_DNODE_ENTER(db); 937 dn = DB_DNODE(db); 938 939 /* XXX does *this* func really need the lock? */ 940 ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 941 942 /* 943 * This call to dbuf_will_dirty() with the dn_struct_rwlock held 944 * is OK, because there can be no other references to the db 945 * when we are changing its size, so no concurrent DB_FILL can 946 * be happening. 947 */ 948 /* 949 * XXX we should be doing a dbuf_read, checking the return 950 * value and returning that up to our callers 951 */ 952 dbuf_will_dirty(db, tx); 953 954 /* create the data buffer for the new block */ 955 buf = arc_buf_alloc(dn->dn_objset->os_spa, size, db, type); 956 957 /* copy old block data to the new block */ 958 obuf = db->db_buf; 959 bcopy(obuf->b_data, buf->b_data, MIN(osize, size)); 960 /* zero the remainder */ 961 if (size > osize) 962 bzero((uint8_t *)buf->b_data + osize, size - osize); 963 964 mutex_enter(&db->db_mtx); 965 dbuf_set_data(db, buf); 966 VERIFY(arc_buf_remove_ref(obuf, db) == 1); 967 db->db.db_size = size; 968 969 if (db->db_level == 0) { 970 ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg); 971 db->db_last_dirty->dt.dl.dr_data = buf; 972 } 973 mutex_exit(&db->db_mtx); 974 975 dnode_willuse_space(dn, size-osize, tx); 976 DB_DNODE_EXIT(db); 977 } 978 979 void 980 dbuf_release_bp(dmu_buf_impl_t *db) 981 { 982 objset_t *os; 983 zbookmark_t zb; 984 985 DB_GET_OBJSET(&os, db); 986 ASSERT(dsl_pool_sync_context(dmu_objset_pool(os))); 987 ASSERT(arc_released(os->os_phys_buf) || 988 list_link_active(&os->os_dsl_dataset->ds_synced_link)); 989 ASSERT(db->db_parent == NULL || arc_released(db->db_parent->db_buf)); 990 991 zb.zb_objset = os->os_dsl_dataset ? 992 os->os_dsl_dataset->ds_object : 0; 993 zb.zb_object = db->db.db_object; 994 zb.zb_level = db->db_level; 995 zb.zb_blkid = db->db_blkid; 996 (void) arc_release_bp(db->db_buf, db, 997 db->db_blkptr, os->os_spa, &zb); 998 } 999 1000 dbuf_dirty_record_t * 1001 dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1002 { 1003 dnode_t *dn; 1004 objset_t *os; 1005 dbuf_dirty_record_t **drp, *dr; 1006 int drop_struct_lock = FALSE; 1007 boolean_t do_free_accounting = B_FALSE; 1008 int txgoff = tx->tx_txg & TXG_MASK; 1009 1010 ASSERT(tx->tx_txg != 0); 1011 ASSERT(!refcount_is_zero(&db->db_holds)); 1012 DMU_TX_DIRTY_BUF(tx, db); 1013 1014 DB_DNODE_ENTER(db); 1015 dn = DB_DNODE(db); 1016 /* 1017 * Shouldn't dirty a regular buffer in syncing context. Private 1018 * objects may be dirtied in syncing context, but only if they 1019 * were already pre-dirtied in open context. 1020 */ 1021 ASSERT(!dmu_tx_is_syncing(tx) || 1022 BP_IS_HOLE(dn->dn_objset->os_rootbp) || 1023 DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 1024 dn->dn_objset->os_dsl_dataset == NULL); 1025 /* 1026 * We make this assert for private objects as well, but after we 1027 * check if we're already dirty. They are allowed to re-dirty 1028 * in syncing context. 1029 */ 1030 ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 1031 dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 1032 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 1033 1034 mutex_enter(&db->db_mtx); 1035 /* 1036 * XXX make this true for indirects too? The problem is that 1037 * transactions created with dmu_tx_create_assigned() from 1038 * syncing context don't bother holding ahead. 1039 */ 1040 ASSERT(db->db_level != 0 || 1041 db->db_state == DB_CACHED || db->db_state == DB_FILL || 1042 db->db_state == DB_NOFILL); 1043 1044 mutex_enter(&dn->dn_mtx); 1045 /* 1046 * Don't set dirtyctx to SYNC if we're just modifying this as we 1047 * initialize the objset. 1048 */ 1049 if (dn->dn_dirtyctx == DN_UNDIRTIED && 1050 !BP_IS_HOLE(dn->dn_objset->os_rootbp)) { 1051 dn->dn_dirtyctx = 1052 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN); 1053 ASSERT(dn->dn_dirtyctx_firstset == NULL); 1054 dn->dn_dirtyctx_firstset = kmem_alloc(1, KM_SLEEP); 1055 } 1056 mutex_exit(&dn->dn_mtx); 1057 1058 if (db->db_blkid == DMU_SPILL_BLKID) 1059 dn->dn_have_spill = B_TRUE; 1060 1061 /* 1062 * If this buffer is already dirty, we're done. 1063 */ 1064 drp = &db->db_last_dirty; 1065 ASSERT(*drp == NULL || (*drp)->dr_txg <= tx->tx_txg || 1066 db->db.db_object == DMU_META_DNODE_OBJECT); 1067 while ((dr = *drp) != NULL && dr->dr_txg > tx->tx_txg) 1068 drp = &dr->dr_next; 1069 if (dr && dr->dr_txg == tx->tx_txg) { 1070 DB_DNODE_EXIT(db); 1071 1072 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID) { 1073 /* 1074 * If this buffer has already been written out, 1075 * we now need to reset its state. 1076 */ 1077 dbuf_unoverride(dr); 1078 if (db->db.db_object != DMU_META_DNODE_OBJECT && 1079 db->db_state != DB_NOFILL) 1080 arc_buf_thaw(db->db_buf); 1081 } 1082 mutex_exit(&db->db_mtx); 1083 return (dr); 1084 } 1085 1086 /* 1087 * Only valid if not already dirty. 1088 */ 1089 ASSERT(dn->dn_object == 0 || 1090 dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 1091 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 1092 1093 ASSERT3U(dn->dn_nlevels, >, db->db_level); 1094 ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) || 1095 dn->dn_phys->dn_nlevels > db->db_level || 1096 dn->dn_next_nlevels[txgoff] > db->db_level || 1097 dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level || 1098 dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level); 1099 1100 /* 1101 * We should only be dirtying in syncing context if it's the 1102 * mos or we're initializing the os or it's a special object. 1103 * However, we are allowed to dirty in syncing context provided 1104 * we already dirtied it in open context. Hence we must make 1105 * this assertion only if we're not already dirty. 1106 */ 1107 os = dn->dn_objset; 1108 ASSERT(!dmu_tx_is_syncing(tx) || DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 1109 os->os_dsl_dataset == NULL || BP_IS_HOLE(os->os_rootbp)); 1110 ASSERT(db->db.db_size != 0); 1111 1112 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1113 1114 if (db->db_blkid != DMU_BONUS_BLKID) { 1115 /* 1116 * Update the accounting. 1117 * Note: we delay "free accounting" until after we drop 1118 * the db_mtx. This keeps us from grabbing other locks 1119 * (and possibly deadlocking) in bp_get_dsize() while 1120 * also holding the db_mtx. 1121 */ 1122 dnode_willuse_space(dn, db->db.db_size, tx); 1123 do_free_accounting = dbuf_block_freeable(db); 1124 } 1125 1126 /* 1127 * If this buffer is dirty in an old transaction group we need 1128 * to make a copy of it so that the changes we make in this 1129 * transaction group won't leak out when we sync the older txg. 1130 */ 1131 dr = kmem_zalloc(sizeof (dbuf_dirty_record_t), KM_SLEEP); 1132 if (db->db_level == 0) { 1133 void *data_old = db->db_buf; 1134 1135 if (db->db_state != DB_NOFILL) { 1136 if (db->db_blkid == DMU_BONUS_BLKID) { 1137 dbuf_fix_old_data(db, tx->tx_txg); 1138 data_old = db->db.db_data; 1139 } else if (db->db.db_object != DMU_META_DNODE_OBJECT) { 1140 /* 1141 * Release the data buffer from the cache so 1142 * that we can modify it without impacting 1143 * possible other users of this cached data 1144 * block. Note that indirect blocks and 1145 * private objects are not released until the 1146 * syncing state (since they are only modified 1147 * then). 1148 */ 1149 arc_release(db->db_buf, db); 1150 dbuf_fix_old_data(db, tx->tx_txg); 1151 data_old = db->db_buf; 1152 } 1153 ASSERT(data_old != NULL); 1154 } 1155 dr->dt.dl.dr_data = data_old; 1156 } else { 1157 mutex_init(&dr->dt.di.dr_mtx, NULL, MUTEX_DEFAULT, NULL); 1158 list_create(&dr->dt.di.dr_children, 1159 sizeof (dbuf_dirty_record_t), 1160 offsetof(dbuf_dirty_record_t, dr_dirty_node)); 1161 } 1162 dr->dr_dbuf = db; 1163 dr->dr_txg = tx->tx_txg; 1164 dr->dr_next = *drp; 1165 *drp = dr; 1166 1167 /* 1168 * We could have been freed_in_flight between the dbuf_noread 1169 * and dbuf_dirty. We win, as though the dbuf_noread() had 1170 * happened after the free. 1171 */ 1172 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 1173 db->db_blkid != DMU_SPILL_BLKID) { 1174 mutex_enter(&dn->dn_mtx); 1175 dnode_clear_range(dn, db->db_blkid, 1, tx); 1176 mutex_exit(&dn->dn_mtx); 1177 db->db_freed_in_flight = FALSE; 1178 } 1179 1180 /* 1181 * This buffer is now part of this txg 1182 */ 1183 dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg); 1184 db->db_dirtycnt += 1; 1185 ASSERT3U(db->db_dirtycnt, <=, 3); 1186 1187 mutex_exit(&db->db_mtx); 1188 1189 if (db->db_blkid == DMU_BONUS_BLKID || 1190 db->db_blkid == DMU_SPILL_BLKID) { 1191 mutex_enter(&dn->dn_mtx); 1192 ASSERT(!list_link_active(&dr->dr_dirty_node)); 1193 list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1194 mutex_exit(&dn->dn_mtx); 1195 dnode_setdirty(dn, tx); 1196 DB_DNODE_EXIT(db); 1197 return (dr); 1198 } else if (do_free_accounting) { 1199 blkptr_t *bp = db->db_blkptr; 1200 int64_t willfree = (bp && !BP_IS_HOLE(bp)) ? 1201 bp_get_dsize(os->os_spa, bp) : db->db.db_size; 1202 /* 1203 * This is only a guess -- if the dbuf is dirty 1204 * in a previous txg, we don't know how much 1205 * space it will use on disk yet. We should 1206 * really have the struct_rwlock to access 1207 * db_blkptr, but since this is just a guess, 1208 * it's OK if we get an odd answer. 1209 */ 1210 ddt_prefetch(os->os_spa, bp); 1211 dnode_willuse_space(dn, -willfree, tx); 1212 } 1213 1214 if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 1215 rw_enter(&dn->dn_struct_rwlock, RW_READER); 1216 drop_struct_lock = TRUE; 1217 } 1218 1219 if (db->db_level == 0) { 1220 dnode_new_blkid(dn, db->db_blkid, tx, drop_struct_lock); 1221 ASSERT(dn->dn_maxblkid >= db->db_blkid); 1222 } 1223 1224 if (db->db_level+1 < dn->dn_nlevels) { 1225 dmu_buf_impl_t *parent = db->db_parent; 1226 dbuf_dirty_record_t *di; 1227 int parent_held = FALSE; 1228 1229 if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) { 1230 int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1231 1232 parent = dbuf_hold_level(dn, db->db_level+1, 1233 db->db_blkid >> epbs, FTAG); 1234 ASSERT(parent != NULL); 1235 parent_held = TRUE; 1236 } 1237 if (drop_struct_lock) 1238 rw_exit(&dn->dn_struct_rwlock); 1239 ASSERT3U(db->db_level+1, ==, parent->db_level); 1240 di = dbuf_dirty(parent, tx); 1241 if (parent_held) 1242 dbuf_rele(parent, FTAG); 1243 1244 mutex_enter(&db->db_mtx); 1245 /* possible race with dbuf_undirty() */ 1246 if (db->db_last_dirty == dr || 1247 dn->dn_object == DMU_META_DNODE_OBJECT) { 1248 mutex_enter(&di->dt.di.dr_mtx); 1249 ASSERT3U(di->dr_txg, ==, tx->tx_txg); 1250 ASSERT(!list_link_active(&dr->dr_dirty_node)); 1251 list_insert_tail(&di->dt.di.dr_children, dr); 1252 mutex_exit(&di->dt.di.dr_mtx); 1253 dr->dr_parent = di; 1254 } 1255 mutex_exit(&db->db_mtx); 1256 } else { 1257 ASSERT(db->db_level+1 == dn->dn_nlevels); 1258 ASSERT(db->db_blkid < dn->dn_nblkptr); 1259 ASSERT(db->db_parent == NULL || db->db_parent == dn->dn_dbuf); 1260 mutex_enter(&dn->dn_mtx); 1261 ASSERT(!list_link_active(&dr->dr_dirty_node)); 1262 list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1263 mutex_exit(&dn->dn_mtx); 1264 if (drop_struct_lock) 1265 rw_exit(&dn->dn_struct_rwlock); 1266 } 1267 1268 dnode_setdirty(dn, tx); 1269 DB_DNODE_EXIT(db); 1270 return (dr); 1271 } 1272 1273 static int 1274 dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1275 { 1276 dnode_t *dn; 1277 uint64_t txg = tx->tx_txg; 1278 dbuf_dirty_record_t *dr, **drp; 1279 1280 ASSERT(txg != 0); 1281 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1282 1283 mutex_enter(&db->db_mtx); 1284 /* 1285 * If this buffer is not dirty, we're done. 1286 */ 1287 for (drp = &db->db_last_dirty; (dr = *drp) != NULL; drp = &dr->dr_next) 1288 if (dr->dr_txg <= txg) 1289 break; 1290 if (dr == NULL || dr->dr_txg < txg) { 1291 mutex_exit(&db->db_mtx); 1292 return (0); 1293 } 1294 ASSERT(dr->dr_txg == txg); 1295 ASSERT(dr->dr_dbuf == db); 1296 1297 DB_DNODE_ENTER(db); 1298 dn = DB_DNODE(db); 1299 1300 /* 1301 * If this buffer is currently held, we cannot undirty 1302 * it, since one of the current holders may be in the 1303 * middle of an update. Note that users of dbuf_undirty() 1304 * should not place a hold on the dbuf before the call. 1305 * Also note: we can get here with a spill block, so 1306 * test for that similar to how dbuf_dirty does. 1307 */ 1308 if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 1309 mutex_exit(&db->db_mtx); 1310 /* Make sure we don't toss this buffer at sync phase */ 1311 if (db->db_blkid != DMU_SPILL_BLKID) { 1312 mutex_enter(&dn->dn_mtx); 1313 dnode_clear_range(dn, db->db_blkid, 1, tx); 1314 mutex_exit(&dn->dn_mtx); 1315 } 1316 DB_DNODE_EXIT(db); 1317 return (0); 1318 } 1319 1320 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1321 1322 ASSERT(db->db.db_size != 0); 1323 1324 /* XXX would be nice to fix up dn_towrite_space[] */ 1325 1326 *drp = dr->dr_next; 1327 1328 /* 1329 * Note that there are three places in dbuf_dirty() 1330 * where this dirty record may be put on a list. 1331 * Make sure to do a list_remove corresponding to 1332 * every one of those list_insert calls. 1333 */ 1334 if (dr->dr_parent) { 1335 mutex_enter(&dr->dr_parent->dt.di.dr_mtx); 1336 list_remove(&dr->dr_parent->dt.di.dr_children, dr); 1337 mutex_exit(&dr->dr_parent->dt.di.dr_mtx); 1338 } else if (db->db_blkid == DMU_SPILL_BLKID || 1339 db->db_level+1 == dn->dn_nlevels) { 1340 ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf); 1341 mutex_enter(&dn->dn_mtx); 1342 list_remove(&dn->dn_dirty_records[txg & TXG_MASK], dr); 1343 mutex_exit(&dn->dn_mtx); 1344 } 1345 DB_DNODE_EXIT(db); 1346 1347 if (db->db_level == 0) { 1348 if (db->db_state != DB_NOFILL) { 1349 dbuf_unoverride(dr); 1350 1351 ASSERT(db->db_buf != NULL); 1352 ASSERT(dr->dt.dl.dr_data != NULL); 1353 if (dr->dt.dl.dr_data != db->db_buf) 1354 VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, 1355 db) == 1); 1356 } 1357 } else { 1358 ASSERT(db->db_buf != NULL); 1359 ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 1360 mutex_destroy(&dr->dt.di.dr_mtx); 1361 list_destroy(&dr->dt.di.dr_children); 1362 } 1363 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 1364 1365 ASSERT(db->db_dirtycnt > 0); 1366 db->db_dirtycnt -= 1; 1367 1368 if (refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) { 1369 arc_buf_t *buf = db->db_buf; 1370 1371 ASSERT(db->db_state == DB_NOFILL || arc_released(buf)); 1372 dbuf_set_data(db, NULL); 1373 VERIFY(arc_buf_remove_ref(buf, db) == 1); 1374 dbuf_evict(db); 1375 return (1); 1376 } 1377 1378 mutex_exit(&db->db_mtx); 1379 return (0); 1380 } 1381 1382 #pragma weak dmu_buf_will_dirty = dbuf_will_dirty 1383 void 1384 dbuf_will_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1385 { 1386 int rf = DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH; 1387 1388 ASSERT(tx->tx_txg != 0); 1389 ASSERT(!refcount_is_zero(&db->db_holds)); 1390 1391 DB_DNODE_ENTER(db); 1392 if (RW_WRITE_HELD(&DB_DNODE(db)->dn_struct_rwlock)) 1393 rf |= DB_RF_HAVESTRUCT; 1394 DB_DNODE_EXIT(db); 1395 (void) dbuf_read(db, NULL, rf); 1396 (void) dbuf_dirty(db, tx); 1397 } 1398 1399 void 1400 dmu_buf_will_not_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 1401 { 1402 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1403 1404 db->db_state = DB_NOFILL; 1405 1406 dmu_buf_will_fill(db_fake, tx); 1407 } 1408 1409 void 1410 dmu_buf_will_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 1411 { 1412 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1413 1414 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1415 ASSERT(tx->tx_txg != 0); 1416 ASSERT(db->db_level == 0); 1417 ASSERT(!refcount_is_zero(&db->db_holds)); 1418 1419 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT || 1420 dmu_tx_private_ok(tx)); 1421 1422 dbuf_noread(db); 1423 (void) dbuf_dirty(db, tx); 1424 } 1425 1426 #pragma weak dmu_buf_fill_done = dbuf_fill_done 1427 /* ARGSUSED */ 1428 void 1429 dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx) 1430 { 1431 mutex_enter(&db->db_mtx); 1432 DBUF_VERIFY(db); 1433 1434 if (db->db_state == DB_FILL) { 1435 if (db->db_level == 0 && db->db_freed_in_flight) { 1436 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1437 /* we were freed while filling */ 1438 /* XXX dbuf_undirty? */ 1439 bzero(db->db.db_data, db->db.db_size); 1440 db->db_freed_in_flight = FALSE; 1441 } 1442 db->db_state = DB_CACHED; 1443 cv_broadcast(&db->db_changed); 1444 } 1445 mutex_exit(&db->db_mtx); 1446 } 1447 1448 /* 1449 * Directly assign a provided arc buf to a given dbuf if it's not referenced 1450 * by anybody except our caller. Otherwise copy arcbuf's contents to dbuf. 1451 */ 1452 void 1453 dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx) 1454 { 1455 ASSERT(!refcount_is_zero(&db->db_holds)); 1456 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1457 ASSERT(db->db_level == 0); 1458 ASSERT(DBUF_GET_BUFC_TYPE(db) == ARC_BUFC_DATA); 1459 ASSERT(buf != NULL); 1460 ASSERT(arc_buf_size(buf) == db->db.db_size); 1461 ASSERT(tx->tx_txg != 0); 1462 1463 arc_return_buf(buf, db); 1464 ASSERT(arc_released(buf)); 1465 1466 mutex_enter(&db->db_mtx); 1467 1468 while (db->db_state == DB_READ || db->db_state == DB_FILL) 1469 cv_wait(&db->db_changed, &db->db_mtx); 1470 1471 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_UNCACHED); 1472 1473 if (db->db_state == DB_CACHED && 1474 refcount_count(&db->db_holds) - 1 > db->db_dirtycnt) { 1475 mutex_exit(&db->db_mtx); 1476 (void) dbuf_dirty(db, tx); 1477 bcopy(buf->b_data, db->db.db_data, db->db.db_size); 1478 VERIFY(arc_buf_remove_ref(buf, db) == 1); 1479 xuio_stat_wbuf_copied(); 1480 return; 1481 } 1482 1483 xuio_stat_wbuf_nocopy(); 1484 if (db->db_state == DB_CACHED) { 1485 dbuf_dirty_record_t *dr = db->db_last_dirty; 1486 1487 ASSERT(db->db_buf != NULL); 1488 if (dr != NULL && dr->dr_txg == tx->tx_txg) { 1489 ASSERT(dr->dt.dl.dr_data == db->db_buf); 1490 if (!arc_released(db->db_buf)) { 1491 ASSERT(dr->dt.dl.dr_override_state == 1492 DR_OVERRIDDEN); 1493 arc_release(db->db_buf, db); 1494 } 1495 dr->dt.dl.dr_data = buf; 1496 VERIFY(arc_buf_remove_ref(db->db_buf, db) == 1); 1497 } else if (dr == NULL || dr->dt.dl.dr_data != db->db_buf) { 1498 arc_release(db->db_buf, db); 1499 VERIFY(arc_buf_remove_ref(db->db_buf, db) == 1); 1500 } 1501 db->db_buf = NULL; 1502 } 1503 ASSERT(db->db_buf == NULL); 1504 dbuf_set_data(db, buf); 1505 db->db_state = DB_FILL; 1506 mutex_exit(&db->db_mtx); 1507 (void) dbuf_dirty(db, tx); 1508 dbuf_fill_done(db, tx); 1509 } 1510 1511 /* 1512 * "Clear" the contents of this dbuf. This will mark the dbuf 1513 * EVICTING and clear *most* of its references. Unfortunetely, 1514 * when we are not holding the dn_dbufs_mtx, we can't clear the 1515 * entry in the dn_dbufs list. We have to wait until dbuf_destroy() 1516 * in this case. For callers from the DMU we will usually see: 1517 * dbuf_clear()->arc_buf_evict()->dbuf_do_evict()->dbuf_destroy() 1518 * For the arc callback, we will usually see: 1519 * dbuf_do_evict()->dbuf_clear();dbuf_destroy() 1520 * Sometimes, though, we will get a mix of these two: 1521 * DMU: dbuf_clear()->arc_buf_evict() 1522 * ARC: dbuf_do_evict()->dbuf_destroy() 1523 */ 1524 void 1525 dbuf_clear(dmu_buf_impl_t *db) 1526 { 1527 dnode_t *dn; 1528 dmu_buf_impl_t *parent = db->db_parent; 1529 dmu_buf_impl_t *dndb; 1530 int dbuf_gone = FALSE; 1531 1532 ASSERT(MUTEX_HELD(&db->db_mtx)); 1533 ASSERT(refcount_is_zero(&db->db_holds)); 1534 1535 dbuf_evict_user(db); 1536 1537 if (db->db_state == DB_CACHED) { 1538 ASSERT(db->db.db_data != NULL); 1539 if (db->db_blkid == DMU_BONUS_BLKID) { 1540 zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN); 1541 arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 1542 } 1543 db->db.db_data = NULL; 1544 db->db_state = DB_UNCACHED; 1545 } 1546 1547 ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL); 1548 ASSERT(db->db_data_pending == NULL); 1549 1550 db->db_state = DB_EVICTING; 1551 db->db_blkptr = NULL; 1552 1553 DB_DNODE_ENTER(db); 1554 dn = DB_DNODE(db); 1555 dndb = dn->dn_dbuf; 1556 if (db->db_blkid != DMU_BONUS_BLKID && MUTEX_HELD(&dn->dn_dbufs_mtx)) { 1557 list_remove(&dn->dn_dbufs, db); 1558 (void) atomic_dec_32_nv(&dn->dn_dbufs_count); 1559 membar_producer(); 1560 DB_DNODE_EXIT(db); 1561 /* 1562 * Decrementing the dbuf count means that the hold corresponding 1563 * to the removed dbuf is no longer discounted in dnode_move(), 1564 * so the dnode cannot be moved until after we release the hold. 1565 * The membar_producer() ensures visibility of the decremented 1566 * value in dnode_move(), since DB_DNODE_EXIT doesn't actually 1567 * release any lock. 1568 */ 1569 dnode_rele(dn, db); 1570 db->db_dnode_handle = NULL; 1571 } else { 1572 DB_DNODE_EXIT(db); 1573 } 1574 1575 if (db->db_buf) 1576 dbuf_gone = arc_buf_evict(db->db_buf); 1577 1578 if (!dbuf_gone) 1579 mutex_exit(&db->db_mtx); 1580 1581 /* 1582 * If this dbuf is referenced from an indirect dbuf, 1583 * decrement the ref count on the indirect dbuf. 1584 */ 1585 if (parent && parent != dndb) 1586 dbuf_rele(parent, db); 1587 } 1588 1589 static int 1590 dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse, 1591 dmu_buf_impl_t **parentp, blkptr_t **bpp) 1592 { 1593 int nlevels, epbs; 1594 1595 *parentp = NULL; 1596 *bpp = NULL; 1597 1598 ASSERT(blkid != DMU_BONUS_BLKID); 1599 1600 if (blkid == DMU_SPILL_BLKID) { 1601 mutex_enter(&dn->dn_mtx); 1602 if (dn->dn_have_spill && 1603 (dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR)) 1604 *bpp = &dn->dn_phys->dn_spill; 1605 else 1606 *bpp = NULL; 1607 dbuf_add_ref(dn->dn_dbuf, NULL); 1608 *parentp = dn->dn_dbuf; 1609 mutex_exit(&dn->dn_mtx); 1610 return (0); 1611 } 1612 1613 if (dn->dn_phys->dn_nlevels == 0) 1614 nlevels = 1; 1615 else 1616 nlevels = dn->dn_phys->dn_nlevels; 1617 1618 epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1619 1620 ASSERT3U(level * epbs, <, 64); 1621 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1622 if (level >= nlevels || 1623 (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) { 1624 /* the buffer has no parent yet */ 1625 return (ENOENT); 1626 } else if (level < nlevels-1) { 1627 /* this block is referenced from an indirect block */ 1628 int err = dbuf_hold_impl(dn, level+1, 1629 blkid >> epbs, fail_sparse, NULL, parentp); 1630 if (err) 1631 return (err); 1632 err = dbuf_read(*parentp, NULL, 1633 (DB_RF_HAVESTRUCT | DB_RF_NOPREFETCH | DB_RF_CANFAIL)); 1634 if (err) { 1635 dbuf_rele(*parentp, NULL); 1636 *parentp = NULL; 1637 return (err); 1638 } 1639 *bpp = ((blkptr_t *)(*parentp)->db.db_data) + 1640 (blkid & ((1ULL << epbs) - 1)); 1641 return (0); 1642 } else { 1643 /* the block is referenced from the dnode */ 1644 ASSERT3U(level, ==, nlevels-1); 1645 ASSERT(dn->dn_phys->dn_nblkptr == 0 || 1646 blkid < dn->dn_phys->dn_nblkptr); 1647 if (dn->dn_dbuf) { 1648 dbuf_add_ref(dn->dn_dbuf, NULL); 1649 *parentp = dn->dn_dbuf; 1650 } 1651 *bpp = &dn->dn_phys->dn_blkptr[blkid]; 1652 return (0); 1653 } 1654 } 1655 1656 static dmu_buf_impl_t * 1657 dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid, 1658 dmu_buf_impl_t *parent, blkptr_t *blkptr) 1659 { 1660 objset_t *os = dn->dn_objset; 1661 dmu_buf_impl_t *db, *odb; 1662 1663 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1664 ASSERT(dn->dn_type != DMU_OT_NONE); 1665 1666 db = kmem_cache_alloc(dbuf_cache, KM_SLEEP); 1667 1668 db->db_objset = os; 1669 db->db.db_object = dn->dn_object; 1670 db->db_level = level; 1671 db->db_blkid = blkid; 1672 db->db_last_dirty = NULL; 1673 db->db_dirtycnt = 0; 1674 db->db_dnode_handle = dn->dn_handle; 1675 db->db_parent = parent; 1676 db->db_blkptr = blkptr; 1677 1678 db->db_user_ptr = NULL; 1679 db->db_user_data_ptr_ptr = NULL; 1680 db->db_evict_func = NULL; 1681 db->db_immediate_evict = 0; 1682 db->db_freed_in_flight = 0; 1683 1684 if (blkid == DMU_BONUS_BLKID) { 1685 ASSERT3P(parent, ==, dn->dn_dbuf); 1686 db->db.db_size = DN_MAX_BONUSLEN - 1687 (dn->dn_nblkptr-1) * sizeof (blkptr_t); 1688 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 1689 db->db.db_offset = DMU_BONUS_BLKID; 1690 db->db_state = DB_UNCACHED; 1691 /* the bonus dbuf is not placed in the hash table */ 1692 arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1693 return (db); 1694 } else if (blkid == DMU_SPILL_BLKID) { 1695 db->db.db_size = (blkptr != NULL) ? 1696 BP_GET_LSIZE(blkptr) : SPA_MINBLOCKSIZE; 1697 db->db.db_offset = 0; 1698 } else { 1699 int blocksize = 1700 db->db_level ? 1<<dn->dn_indblkshift : dn->dn_datablksz; 1701 db->db.db_size = blocksize; 1702 db->db.db_offset = db->db_blkid * blocksize; 1703 } 1704 1705 /* 1706 * Hold the dn_dbufs_mtx while we get the new dbuf 1707 * in the hash table *and* added to the dbufs list. 1708 * This prevents a possible deadlock with someone 1709 * trying to look up this dbuf before its added to the 1710 * dn_dbufs list. 1711 */ 1712 mutex_enter(&dn->dn_dbufs_mtx); 1713 db->db_state = DB_EVICTING; 1714 if ((odb = dbuf_hash_insert(db)) != NULL) { 1715 /* someone else inserted it first */ 1716 kmem_cache_free(dbuf_cache, db); 1717 mutex_exit(&dn->dn_dbufs_mtx); 1718 return (odb); 1719 } 1720 list_insert_head(&dn->dn_dbufs, db); 1721 db->db_state = DB_UNCACHED; 1722 mutex_exit(&dn->dn_dbufs_mtx); 1723 arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1724 1725 if (parent && parent != dn->dn_dbuf) 1726 dbuf_add_ref(parent, db); 1727 1728 ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 1729 refcount_count(&dn->dn_holds) > 0); 1730 (void) refcount_add(&dn->dn_holds, db); 1731 (void) atomic_inc_32_nv(&dn->dn_dbufs_count); 1732 1733 dprintf_dbuf(db, "db=%p\n", db); 1734 1735 return (db); 1736 } 1737 1738 static int 1739 dbuf_do_evict(void *private) 1740 { 1741 arc_buf_t *buf = private; 1742 dmu_buf_impl_t *db = buf->b_private; 1743 1744 if (!MUTEX_HELD(&db->db_mtx)) 1745 mutex_enter(&db->db_mtx); 1746 1747 ASSERT(refcount_is_zero(&db->db_holds)); 1748 1749 if (db->db_state != DB_EVICTING) { 1750 ASSERT(db->db_state == DB_CACHED); 1751 DBUF_VERIFY(db); 1752 db->db_buf = NULL; 1753 dbuf_evict(db); 1754 } else { 1755 mutex_exit(&db->db_mtx); 1756 dbuf_destroy(db); 1757 } 1758 return (0); 1759 } 1760 1761 static void 1762 dbuf_destroy(dmu_buf_impl_t *db) 1763 { 1764 ASSERT(refcount_is_zero(&db->db_holds)); 1765 1766 if (db->db_blkid != DMU_BONUS_BLKID) { 1767 /* 1768 * If this dbuf is still on the dn_dbufs list, 1769 * remove it from that list. 1770 */ 1771 if (db->db_dnode_handle != NULL) { 1772 dnode_t *dn; 1773 1774 DB_DNODE_ENTER(db); 1775 dn = DB_DNODE(db); 1776 mutex_enter(&dn->dn_dbufs_mtx); 1777 list_remove(&dn->dn_dbufs, db); 1778 (void) atomic_dec_32_nv(&dn->dn_dbufs_count); 1779 mutex_exit(&dn->dn_dbufs_mtx); 1780 DB_DNODE_EXIT(db); 1781 /* 1782 * Decrementing the dbuf count means that the hold 1783 * corresponding to the removed dbuf is no longer 1784 * discounted in dnode_move(), so the dnode cannot be 1785 * moved until after we release the hold. 1786 */ 1787 dnode_rele(dn, db); 1788 db->db_dnode_handle = NULL; 1789 } 1790 dbuf_hash_remove(db); 1791 } 1792 db->db_parent = NULL; 1793 db->db_buf = NULL; 1794 1795 ASSERT(!list_link_active(&db->db_link)); 1796 ASSERT(db->db.db_data == NULL); 1797 ASSERT(db->db_hash_next == NULL); 1798 ASSERT(db->db_blkptr == NULL); 1799 ASSERT(db->db_data_pending == NULL); 1800 1801 kmem_cache_free(dbuf_cache, db); 1802 arc_space_return(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1803 } 1804 1805 void 1806 dbuf_prefetch(dnode_t *dn, uint64_t blkid) 1807 { 1808 dmu_buf_impl_t *db = NULL; 1809 blkptr_t *bp = NULL; 1810 1811 ASSERT(blkid != DMU_BONUS_BLKID); 1812 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1813 1814 if (dnode_block_freed(dn, blkid)) 1815 return; 1816 1817 /* dbuf_find() returns with db_mtx held */ 1818 if (db = dbuf_find(dn, 0, blkid)) { 1819 /* 1820 * This dbuf is already in the cache. We assume that 1821 * it is already CACHED, or else about to be either 1822 * read or filled. 1823 */ 1824 mutex_exit(&db->db_mtx); 1825 return; 1826 } 1827 1828 if (dbuf_findbp(dn, 0, blkid, TRUE, &db, &bp) == 0) { 1829 if (bp && !BP_IS_HOLE(bp)) { 1830 int priority = dn->dn_type == DMU_OT_DDT_ZAP ? 1831 ZIO_PRIORITY_DDT_PREFETCH : ZIO_PRIORITY_ASYNC_READ; 1832 arc_buf_t *pbuf; 1833 dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset; 1834 uint32_t aflags = ARC_NOWAIT | ARC_PREFETCH; 1835 zbookmark_t zb; 1836 1837 SET_BOOKMARK(&zb, ds ? ds->ds_object : DMU_META_OBJSET, 1838 dn->dn_object, 0, blkid); 1839 1840 if (db) 1841 pbuf = db->db_buf; 1842 else 1843 pbuf = dn->dn_objset->os_phys_buf; 1844 1845 (void) dsl_read(NULL, dn->dn_objset->os_spa, 1846 bp, pbuf, NULL, NULL, priority, 1847 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 1848 &aflags, &zb); 1849 } 1850 if (db) 1851 dbuf_rele(db, NULL); 1852 } 1853 } 1854 1855 /* 1856 * Returns with db_holds incremented, and db_mtx not held. 1857 * Note: dn_struct_rwlock must be held. 1858 */ 1859 int 1860 dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, int fail_sparse, 1861 void *tag, dmu_buf_impl_t **dbp) 1862 { 1863 dmu_buf_impl_t *db, *parent = NULL; 1864 1865 ASSERT(blkid != DMU_BONUS_BLKID); 1866 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1867 ASSERT3U(dn->dn_nlevels, >, level); 1868 1869 *dbp = NULL; 1870 top: 1871 /* dbuf_find() returns with db_mtx held */ 1872 db = dbuf_find(dn, level, blkid); 1873 1874 if (db == NULL) { 1875 blkptr_t *bp = NULL; 1876 int err; 1877 1878 ASSERT3P(parent, ==, NULL); 1879 err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp); 1880 if (fail_sparse) { 1881 if (err == 0 && bp && BP_IS_HOLE(bp)) 1882 err = ENOENT; 1883 if (err) { 1884 if (parent) 1885 dbuf_rele(parent, NULL); 1886 return (err); 1887 } 1888 } 1889 if (err && err != ENOENT) 1890 return (err); 1891 db = dbuf_create(dn, level, blkid, parent, bp); 1892 } 1893 1894 if (db->db_buf && refcount_is_zero(&db->db_holds)) { 1895 arc_buf_add_ref(db->db_buf, db); 1896 if (db->db_buf->b_data == NULL) { 1897 dbuf_clear(db); 1898 if (parent) { 1899 dbuf_rele(parent, NULL); 1900 parent = NULL; 1901 } 1902 goto top; 1903 } 1904 ASSERT3P(db->db.db_data, ==, db->db_buf->b_data); 1905 } 1906 1907 ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf)); 1908 1909 /* 1910 * If this buffer is currently syncing out, and we are are 1911 * still referencing it from db_data, we need to make a copy 1912 * of it in case we decide we want to dirty it again in this txg. 1913 */ 1914 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 1915 dn->dn_object != DMU_META_DNODE_OBJECT && 1916 db->db_state == DB_CACHED && db->db_data_pending) { 1917 dbuf_dirty_record_t *dr = db->db_data_pending; 1918 1919 if (dr->dt.dl.dr_data == db->db_buf) { 1920 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 1921 1922 dbuf_set_data(db, 1923 arc_buf_alloc(dn->dn_objset->os_spa, 1924 db->db.db_size, db, type)); 1925 bcopy(dr->dt.dl.dr_data->b_data, db->db.db_data, 1926 db->db.db_size); 1927 } 1928 } 1929 1930 (void) refcount_add(&db->db_holds, tag); 1931 dbuf_update_data(db); 1932 DBUF_VERIFY(db); 1933 mutex_exit(&db->db_mtx); 1934 1935 /* NOTE: we can't rele the parent until after we drop the db_mtx */ 1936 if (parent) 1937 dbuf_rele(parent, NULL); 1938 1939 ASSERT3P(DB_DNODE(db), ==, dn); 1940 ASSERT3U(db->db_blkid, ==, blkid); 1941 ASSERT3U(db->db_level, ==, level); 1942 *dbp = db; 1943 1944 return (0); 1945 } 1946 1947 dmu_buf_impl_t * 1948 dbuf_hold(dnode_t *dn, uint64_t blkid, void *tag) 1949 { 1950 dmu_buf_impl_t *db; 1951 int err = dbuf_hold_impl(dn, 0, blkid, FALSE, tag, &db); 1952 return (err ? NULL : db); 1953 } 1954 1955 dmu_buf_impl_t * 1956 dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, void *tag) 1957 { 1958 dmu_buf_impl_t *db; 1959 int err = dbuf_hold_impl(dn, level, blkid, FALSE, tag, &db); 1960 return (err ? NULL : db); 1961 } 1962 1963 void 1964 dbuf_create_bonus(dnode_t *dn) 1965 { 1966 ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 1967 1968 ASSERT(dn->dn_bonus == NULL); 1969 dn->dn_bonus = dbuf_create(dn, 0, DMU_BONUS_BLKID, dn->dn_dbuf, NULL); 1970 } 1971 1972 int 1973 dbuf_spill_set_blksz(dmu_buf_t *db_fake, uint64_t blksz, dmu_tx_t *tx) 1974 { 1975 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1976 dnode_t *dn; 1977 1978 if (db->db_blkid != DMU_SPILL_BLKID) 1979 return (ENOTSUP); 1980 if (blksz == 0) 1981 blksz = SPA_MINBLOCKSIZE; 1982 if (blksz > SPA_MAXBLOCKSIZE) 1983 blksz = SPA_MAXBLOCKSIZE; 1984 else 1985 blksz = P2ROUNDUP(blksz, SPA_MINBLOCKSIZE); 1986 1987 DB_DNODE_ENTER(db); 1988 dn = DB_DNODE(db); 1989 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 1990 dbuf_new_size(db, blksz, tx); 1991 rw_exit(&dn->dn_struct_rwlock); 1992 DB_DNODE_EXIT(db); 1993 1994 return (0); 1995 } 1996 1997 void 1998 dbuf_rm_spill(dnode_t *dn, dmu_tx_t *tx) 1999 { 2000 dbuf_free_range(dn, DMU_SPILL_BLKID, DMU_SPILL_BLKID, tx); 2001 } 2002 2003 #pragma weak dmu_buf_add_ref = dbuf_add_ref 2004 void 2005 dbuf_add_ref(dmu_buf_impl_t *db, void *tag) 2006 { 2007 int64_t holds = refcount_add(&db->db_holds, tag); 2008 ASSERT(holds > 1); 2009 } 2010 2011 /* 2012 * If you call dbuf_rele() you had better not be referencing the dnode handle 2013 * unless you have some other direct or indirect hold on the dnode. (An indirect 2014 * hold is a hold on one of the dnode's dbufs, including the bonus buffer.) 2015 * Without that, the dbuf_rele() could lead to a dnode_rele() followed by the 2016 * dnode's parent dbuf evicting its dnode handles. 2017 */ 2018 #pragma weak dmu_buf_rele = dbuf_rele 2019 void 2020 dbuf_rele(dmu_buf_impl_t *db, void *tag) 2021 { 2022 mutex_enter(&db->db_mtx); 2023 dbuf_rele_and_unlock(db, tag); 2024 } 2025 2026 /* 2027 * dbuf_rele() for an already-locked dbuf. This is necessary to allow 2028 * db_dirtycnt and db_holds to be updated atomically. 2029 */ 2030 void 2031 dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag) 2032 { 2033 int64_t holds; 2034 2035 ASSERT(MUTEX_HELD(&db->db_mtx)); 2036 DBUF_VERIFY(db); 2037 2038 /* 2039 * Remove the reference to the dbuf before removing its hold on the 2040 * dnode so we can guarantee in dnode_move() that a referenced bonus 2041 * buffer has a corresponding dnode hold. 2042 */ 2043 holds = refcount_remove(&db->db_holds, tag); 2044 ASSERT(holds >= 0); 2045 2046 /* 2047 * We can't freeze indirects if there is a possibility that they 2048 * may be modified in the current syncing context. 2049 */ 2050 if (db->db_buf && holds == (db->db_level == 0 ? db->db_dirtycnt : 0)) 2051 arc_buf_freeze(db->db_buf); 2052 2053 if (holds == db->db_dirtycnt && 2054 db->db_level == 0 && db->db_immediate_evict) 2055 dbuf_evict_user(db); 2056 2057 if (holds == 0) { 2058 if (db->db_blkid == DMU_BONUS_BLKID) { 2059 mutex_exit(&db->db_mtx); 2060 2061 /* 2062 * If the dnode moves here, we cannot cross this barrier 2063 * until the move completes. 2064 */ 2065 DB_DNODE_ENTER(db); 2066 (void) atomic_dec_32_nv(&DB_DNODE(db)->dn_dbufs_count); 2067 DB_DNODE_EXIT(db); 2068 /* 2069 * The bonus buffer's dnode hold is no longer discounted 2070 * in dnode_move(). The dnode cannot move until after 2071 * the dnode_rele(). 2072 */ 2073 dnode_rele(DB_DNODE(db), db); 2074 } else if (db->db_buf == NULL) { 2075 /* 2076 * This is a special case: we never associated this 2077 * dbuf with any data allocated from the ARC. 2078 */ 2079 ASSERT(db->db_state == DB_UNCACHED || 2080 db->db_state == DB_NOFILL); 2081 dbuf_evict(db); 2082 } else if (arc_released(db->db_buf)) { 2083 arc_buf_t *buf = db->db_buf; 2084 /* 2085 * This dbuf has anonymous data associated with it. 2086 */ 2087 dbuf_set_data(db, NULL); 2088 VERIFY(arc_buf_remove_ref(buf, db) == 1); 2089 dbuf_evict(db); 2090 } else { 2091 VERIFY(arc_buf_remove_ref(db->db_buf, db) == 0); 2092 if (!DBUF_IS_CACHEABLE(db)) 2093 dbuf_clear(db); 2094 else 2095 mutex_exit(&db->db_mtx); 2096 } 2097 } else { 2098 mutex_exit(&db->db_mtx); 2099 } 2100 } 2101 2102 #pragma weak dmu_buf_refcount = dbuf_refcount 2103 uint64_t 2104 dbuf_refcount(dmu_buf_impl_t *db) 2105 { 2106 return (refcount_count(&db->db_holds)); 2107 } 2108 2109 void * 2110 dmu_buf_set_user(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 2111 dmu_buf_evict_func_t *evict_func) 2112 { 2113 return (dmu_buf_update_user(db_fake, NULL, user_ptr, 2114 user_data_ptr_ptr, evict_func)); 2115 } 2116 2117 void * 2118 dmu_buf_set_user_ie(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 2119 dmu_buf_evict_func_t *evict_func) 2120 { 2121 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2122 2123 db->db_immediate_evict = TRUE; 2124 return (dmu_buf_update_user(db_fake, NULL, user_ptr, 2125 user_data_ptr_ptr, evict_func)); 2126 } 2127 2128 void * 2129 dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr, void *user_ptr, 2130 void *user_data_ptr_ptr, dmu_buf_evict_func_t *evict_func) 2131 { 2132 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2133 ASSERT(db->db_level == 0); 2134 2135 ASSERT((user_ptr == NULL) == (evict_func == NULL)); 2136 2137 mutex_enter(&db->db_mtx); 2138 2139 if (db->db_user_ptr == old_user_ptr) { 2140 db->db_user_ptr = user_ptr; 2141 db->db_user_data_ptr_ptr = user_data_ptr_ptr; 2142 db->db_evict_func = evict_func; 2143 2144 dbuf_update_data(db); 2145 } else { 2146 old_user_ptr = db->db_user_ptr; 2147 } 2148 2149 mutex_exit(&db->db_mtx); 2150 return (old_user_ptr); 2151 } 2152 2153 void * 2154 dmu_buf_get_user(dmu_buf_t *db_fake) 2155 { 2156 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2157 ASSERT(!refcount_is_zero(&db->db_holds)); 2158 2159 return (db->db_user_ptr); 2160 } 2161 2162 boolean_t 2163 dmu_buf_freeable(dmu_buf_t *dbuf) 2164 { 2165 boolean_t res = B_FALSE; 2166 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf; 2167 2168 if (db->db_blkptr) 2169 res = dsl_dataset_block_freeable(db->db_objset->os_dsl_dataset, 2170 db->db_blkptr, db->db_blkptr->blk_birth); 2171 2172 return (res); 2173 } 2174 2175 static void 2176 dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db) 2177 { 2178 /* ASSERT(dmu_tx_is_syncing(tx) */ 2179 ASSERT(MUTEX_HELD(&db->db_mtx)); 2180 2181 if (db->db_blkptr != NULL) 2182 return; 2183 2184 if (db->db_blkid == DMU_SPILL_BLKID) { 2185 db->db_blkptr = &dn->dn_phys->dn_spill; 2186 BP_ZERO(db->db_blkptr); 2187 return; 2188 } 2189 if (db->db_level == dn->dn_phys->dn_nlevels-1) { 2190 /* 2191 * This buffer was allocated at a time when there was 2192 * no available blkptrs from the dnode, or it was 2193 * inappropriate to hook it in (i.e., nlevels mis-match). 2194 */ 2195 ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr); 2196 ASSERT(db->db_parent == NULL); 2197 db->db_parent = dn->dn_dbuf; 2198 db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid]; 2199 DBUF_VERIFY(db); 2200 } else { 2201 dmu_buf_impl_t *parent = db->db_parent; 2202 int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2203 2204 ASSERT(dn->dn_phys->dn_nlevels > 1); 2205 if (parent == NULL) { 2206 mutex_exit(&db->db_mtx); 2207 rw_enter(&dn->dn_struct_rwlock, RW_READER); 2208 (void) dbuf_hold_impl(dn, db->db_level+1, 2209 db->db_blkid >> epbs, FALSE, db, &parent); 2210 rw_exit(&dn->dn_struct_rwlock); 2211 mutex_enter(&db->db_mtx); 2212 db->db_parent = parent; 2213 } 2214 db->db_blkptr = (blkptr_t *)parent->db.db_data + 2215 (db->db_blkid & ((1ULL << epbs) - 1)); 2216 DBUF_VERIFY(db); 2217 } 2218 } 2219 2220 static void 2221 dbuf_sync_indirect(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 2222 { 2223 dmu_buf_impl_t *db = dr->dr_dbuf; 2224 dnode_t *dn; 2225 zio_t *zio; 2226 2227 ASSERT(dmu_tx_is_syncing(tx)); 2228 2229 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 2230 2231 mutex_enter(&db->db_mtx); 2232 2233 ASSERT(db->db_level > 0); 2234 DBUF_VERIFY(db); 2235 2236 if (db->db_buf == NULL) { 2237 mutex_exit(&db->db_mtx); 2238 (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED); 2239 mutex_enter(&db->db_mtx); 2240 } 2241 ASSERT3U(db->db_state, ==, DB_CACHED); 2242 ASSERT(db->db_buf != NULL); 2243 2244 DB_DNODE_ENTER(db); 2245 dn = DB_DNODE(db); 2246 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2247 dbuf_check_blkptr(dn, db); 2248 DB_DNODE_EXIT(db); 2249 2250 db->db_data_pending = dr; 2251 2252 mutex_exit(&db->db_mtx); 2253 dbuf_write(dr, db->db_buf, tx); 2254 2255 zio = dr->dr_zio; 2256 mutex_enter(&dr->dt.di.dr_mtx); 2257 dbuf_sync_list(&dr->dt.di.dr_children, tx); 2258 ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 2259 mutex_exit(&dr->dt.di.dr_mtx); 2260 zio_nowait(zio); 2261 } 2262 2263 static void 2264 dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 2265 { 2266 arc_buf_t **datap = &dr->dt.dl.dr_data; 2267 dmu_buf_impl_t *db = dr->dr_dbuf; 2268 dnode_t *dn; 2269 objset_t *os; 2270 uint64_t txg = tx->tx_txg; 2271 2272 ASSERT(dmu_tx_is_syncing(tx)); 2273 2274 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 2275 2276 mutex_enter(&db->db_mtx); 2277 /* 2278 * To be synced, we must be dirtied. But we 2279 * might have been freed after the dirty. 2280 */ 2281 if (db->db_state == DB_UNCACHED) { 2282 /* This buffer has been freed since it was dirtied */ 2283 ASSERT(db->db.db_data == NULL); 2284 } else if (db->db_state == DB_FILL) { 2285 /* This buffer was freed and is now being re-filled */ 2286 ASSERT(db->db.db_data != dr->dt.dl.dr_data); 2287 } else { 2288 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_NOFILL); 2289 } 2290 DBUF_VERIFY(db); 2291 2292 DB_DNODE_ENTER(db); 2293 dn = DB_DNODE(db); 2294 2295 if (db->db_blkid == DMU_SPILL_BLKID) { 2296 mutex_enter(&dn->dn_mtx); 2297 dn->dn_phys->dn_flags |= DNODE_FLAG_SPILL_BLKPTR; 2298 mutex_exit(&dn->dn_mtx); 2299 } 2300 2301 /* 2302 * If this is a bonus buffer, simply copy the bonus data into the 2303 * dnode. It will be written out when the dnode is synced (and it 2304 * will be synced, since it must have been dirty for dbuf_sync to 2305 * be called). 2306 */ 2307 if (db->db_blkid == DMU_BONUS_BLKID) { 2308 dbuf_dirty_record_t **drp; 2309 2310 ASSERT(*datap != NULL); 2311 ASSERT3U(db->db_level, ==, 0); 2312 ASSERT3U(dn->dn_phys->dn_bonuslen, <=, DN_MAX_BONUSLEN); 2313 bcopy(*datap, DN_BONUS(dn->dn_phys), dn->dn_phys->dn_bonuslen); 2314 DB_DNODE_EXIT(db); 2315 2316 if (*datap != db->db.db_data) { 2317 zio_buf_free(*datap, DN_MAX_BONUSLEN); 2318 arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 2319 } 2320 db->db_data_pending = NULL; 2321 drp = &db->db_last_dirty; 2322 while (*drp != dr) 2323 drp = &(*drp)->dr_next; 2324 ASSERT(dr->dr_next == NULL); 2325 ASSERT(dr->dr_dbuf == db); 2326 *drp = dr->dr_next; 2327 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 2328 ASSERT(db->db_dirtycnt > 0); 2329 db->db_dirtycnt -= 1; 2330 dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg); 2331 return; 2332 } 2333 2334 os = dn->dn_objset; 2335 2336 /* 2337 * This function may have dropped the db_mtx lock allowing a dmu_sync 2338 * operation to sneak in. As a result, we need to ensure that we 2339 * don't check the dr_override_state until we have returned from 2340 * dbuf_check_blkptr. 2341 */ 2342 dbuf_check_blkptr(dn, db); 2343 2344 /* 2345 * If this buffer is in the middle of an immediate write, 2346 * wait for the synchronous IO to complete. 2347 */ 2348 while (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC) { 2349 ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT); 2350 cv_wait(&db->db_changed, &db->db_mtx); 2351 ASSERT(dr->dt.dl.dr_override_state != DR_NOT_OVERRIDDEN); 2352 } 2353 2354 if (db->db_state != DB_NOFILL && 2355 dn->dn_object != DMU_META_DNODE_OBJECT && 2356 refcount_count(&db->db_holds) > 1 && 2357 dr->dt.dl.dr_override_state != DR_OVERRIDDEN && 2358 *datap == db->db_buf) { 2359 /* 2360 * If this buffer is currently "in use" (i.e., there 2361 * are active holds and db_data still references it), 2362 * then make a copy before we start the write so that 2363 * any modifications from the open txg will not leak 2364 * into this write. 2365 * 2366 * NOTE: this copy does not need to be made for 2367 * objects only modified in the syncing context (e.g. 2368 * DNONE_DNODE blocks). 2369 */ 2370 int blksz = arc_buf_size(*datap); 2371 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 2372 *datap = arc_buf_alloc(os->os_spa, blksz, db, type); 2373 bcopy(db->db.db_data, (*datap)->b_data, blksz); 2374 } 2375 db->db_data_pending = dr; 2376 2377 mutex_exit(&db->db_mtx); 2378 2379 dbuf_write(dr, *datap, tx); 2380 2381 ASSERT(!list_link_active(&dr->dr_dirty_node)); 2382 if (dn->dn_object == DMU_META_DNODE_OBJECT) { 2383 list_insert_tail(&dn->dn_dirty_records[txg&TXG_MASK], dr); 2384 DB_DNODE_EXIT(db); 2385 } else { 2386 /* 2387 * Although zio_nowait() does not "wait for an IO", it does 2388 * initiate the IO. If this is an empty write it seems plausible 2389 * that the IO could actually be completed before the nowait 2390 * returns. We need to DB_DNODE_EXIT() first in case 2391 * zio_nowait() invalidates the dbuf. 2392 */ 2393 DB_DNODE_EXIT(db); 2394 zio_nowait(dr->dr_zio); 2395 } 2396 } 2397 2398 void 2399 dbuf_sync_list(list_t *list, dmu_tx_t *tx) 2400 { 2401 dbuf_dirty_record_t *dr; 2402 2403 while (dr = list_head(list)) { 2404 if (dr->dr_zio != NULL) { 2405 /* 2406 * If we find an already initialized zio then we 2407 * are processing the meta-dnode, and we have finished. 2408 * The dbufs for all dnodes are put back on the list 2409 * during processing, so that we can zio_wait() 2410 * these IOs after initiating all child IOs. 2411 */ 2412 ASSERT3U(dr->dr_dbuf->db.db_object, ==, 2413 DMU_META_DNODE_OBJECT); 2414 break; 2415 } 2416 list_remove(list, dr); 2417 if (dr->dr_dbuf->db_level > 0) 2418 dbuf_sync_indirect(dr, tx); 2419 else 2420 dbuf_sync_leaf(dr, tx); 2421 } 2422 } 2423 2424 /* ARGSUSED */ 2425 static void 2426 dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb) 2427 { 2428 dmu_buf_impl_t *db = vdb; 2429 dnode_t *dn; 2430 blkptr_t *bp = zio->io_bp; 2431 blkptr_t *bp_orig = &zio->io_bp_orig; 2432 spa_t *spa = zio->io_spa; 2433 int64_t delta; 2434 uint64_t fill = 0; 2435 int i; 2436 2437 ASSERT(db->db_blkptr == bp); 2438 2439 DB_DNODE_ENTER(db); 2440 dn = DB_DNODE(db); 2441 delta = bp_get_dsize_sync(spa, bp) - bp_get_dsize_sync(spa, bp_orig); 2442 dnode_diduse_space(dn, delta - zio->io_prev_space_delta); 2443 zio->io_prev_space_delta = delta; 2444 2445 if (BP_IS_HOLE(bp)) { 2446 ASSERT(bp->blk_fill == 0); 2447 DB_DNODE_EXIT(db); 2448 return; 2449 } 2450 2451 ASSERT((db->db_blkid != DMU_SPILL_BLKID && 2452 BP_GET_TYPE(bp) == dn->dn_type) || 2453 (db->db_blkid == DMU_SPILL_BLKID && 2454 BP_GET_TYPE(bp) == dn->dn_bonustype)); 2455 ASSERT(BP_GET_LEVEL(bp) == db->db_level); 2456 2457 mutex_enter(&db->db_mtx); 2458 2459 #ifdef ZFS_DEBUG 2460 if (db->db_blkid == DMU_SPILL_BLKID) { 2461 ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); 2462 ASSERT(!(BP_IS_HOLE(db->db_blkptr)) && 2463 db->db_blkptr == &dn->dn_phys->dn_spill); 2464 } 2465 #endif 2466 2467 if (db->db_level == 0) { 2468 mutex_enter(&dn->dn_mtx); 2469 if (db->db_blkid > dn->dn_phys->dn_maxblkid && 2470 db->db_blkid != DMU_SPILL_BLKID) 2471 dn->dn_phys->dn_maxblkid = db->db_blkid; 2472 mutex_exit(&dn->dn_mtx); 2473 2474 if (dn->dn_type == DMU_OT_DNODE) { 2475 dnode_phys_t *dnp = db->db.db_data; 2476 for (i = db->db.db_size >> DNODE_SHIFT; i > 0; 2477 i--, dnp++) { 2478 if (dnp->dn_type != DMU_OT_NONE) 2479 fill++; 2480 } 2481 } else { 2482 fill = 1; 2483 } 2484 } else { 2485 blkptr_t *ibp = db->db.db_data; 2486 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2487 for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, ibp++) { 2488 if (BP_IS_HOLE(ibp)) 2489 continue; 2490 fill += ibp->blk_fill; 2491 } 2492 } 2493 DB_DNODE_EXIT(db); 2494 2495 bp->blk_fill = fill; 2496 2497 mutex_exit(&db->db_mtx); 2498 } 2499 2500 /* ARGSUSED */ 2501 static void 2502 dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb) 2503 { 2504 dmu_buf_impl_t *db = vdb; 2505 blkptr_t *bp = zio->io_bp; 2506 blkptr_t *bp_orig = &zio->io_bp_orig; 2507 uint64_t txg = zio->io_txg; 2508 dbuf_dirty_record_t **drp, *dr; 2509 2510 ASSERT3U(zio->io_error, ==, 0); 2511 ASSERT(db->db_blkptr == bp); 2512 2513 if (zio->io_flags & ZIO_FLAG_IO_REWRITE) { 2514 ASSERT(BP_EQUAL(bp, bp_orig)); 2515 } else { 2516 objset_t *os; 2517 dsl_dataset_t *ds; 2518 dmu_tx_t *tx; 2519 2520 DB_GET_OBJSET(&os, db); 2521 ds = os->os_dsl_dataset; 2522 tx = os->os_synctx; 2523 2524 (void) dsl_dataset_block_kill(ds, bp_orig, tx, B_TRUE); 2525 dsl_dataset_block_born(ds, bp, tx); 2526 } 2527 2528 mutex_enter(&db->db_mtx); 2529 2530 DBUF_VERIFY(db); 2531 2532 drp = &db->db_last_dirty; 2533 while ((dr = *drp) != db->db_data_pending) 2534 drp = &dr->dr_next; 2535 ASSERT(!list_link_active(&dr->dr_dirty_node)); 2536 ASSERT(dr->dr_txg == txg); 2537 ASSERT(dr->dr_dbuf == db); 2538 ASSERT(dr->dr_next == NULL); 2539 *drp = dr->dr_next; 2540 2541 #ifdef ZFS_DEBUG 2542 if (db->db_blkid == DMU_SPILL_BLKID) { 2543 dnode_t *dn; 2544 2545 DB_DNODE_ENTER(db); 2546 dn = DB_DNODE(db); 2547 ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); 2548 ASSERT(!(BP_IS_HOLE(db->db_blkptr)) && 2549 db->db_blkptr == &dn->dn_phys->dn_spill); 2550 DB_DNODE_EXIT(db); 2551 } 2552 #endif 2553 2554 if (db->db_level == 0) { 2555 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 2556 ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN); 2557 if (db->db_state != DB_NOFILL) { 2558 if (dr->dt.dl.dr_data != db->db_buf) 2559 VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, 2560 db) == 1); 2561 else if (!arc_released(db->db_buf)) 2562 arc_set_callback(db->db_buf, dbuf_do_evict, db); 2563 } 2564 } else { 2565 dnode_t *dn; 2566 2567 DB_DNODE_ENTER(db); 2568 dn = DB_DNODE(db); 2569 ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 2570 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2571 if (!BP_IS_HOLE(db->db_blkptr)) { 2572 int epbs = 2573 dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2574 ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, 2575 db->db.db_size); 2576 ASSERT3U(dn->dn_phys->dn_maxblkid 2577 >> (db->db_level * epbs), >=, db->db_blkid); 2578 arc_set_callback(db->db_buf, dbuf_do_evict, db); 2579 } 2580 DB_DNODE_EXIT(db); 2581 mutex_destroy(&dr->dt.di.dr_mtx); 2582 list_destroy(&dr->dt.di.dr_children); 2583 } 2584 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 2585 2586 cv_broadcast(&db->db_changed); 2587 ASSERT(db->db_dirtycnt > 0); 2588 db->db_dirtycnt -= 1; 2589 db->db_data_pending = NULL; 2590 dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg); 2591 } 2592 2593 static void 2594 dbuf_write_nofill_ready(zio_t *zio) 2595 { 2596 dbuf_write_ready(zio, NULL, zio->io_private); 2597 } 2598 2599 static void 2600 dbuf_write_nofill_done(zio_t *zio) 2601 { 2602 dbuf_write_done(zio, NULL, zio->io_private); 2603 } 2604 2605 static void 2606 dbuf_write_override_ready(zio_t *zio) 2607 { 2608 dbuf_dirty_record_t *dr = zio->io_private; 2609 dmu_buf_impl_t *db = dr->dr_dbuf; 2610 2611 dbuf_write_ready(zio, NULL, db); 2612 } 2613 2614 static void 2615 dbuf_write_override_done(zio_t *zio) 2616 { 2617 dbuf_dirty_record_t *dr = zio->io_private; 2618 dmu_buf_impl_t *db = dr->dr_dbuf; 2619 blkptr_t *obp = &dr->dt.dl.dr_overridden_by; 2620 2621 mutex_enter(&db->db_mtx); 2622 if (!BP_EQUAL(zio->io_bp, obp)) { 2623 if (!BP_IS_HOLE(obp)) 2624 dsl_free(spa_get_dsl(zio->io_spa), zio->io_txg, obp); 2625 arc_release(dr->dt.dl.dr_data, db); 2626 } 2627 mutex_exit(&db->db_mtx); 2628 2629 dbuf_write_done(zio, NULL, db); 2630 } 2631 2632 static void 2633 dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx) 2634 { 2635 dmu_buf_impl_t *db = dr->dr_dbuf; 2636 dnode_t *dn; 2637 objset_t *os; 2638 dmu_buf_impl_t *parent = db->db_parent; 2639 uint64_t txg = tx->tx_txg; 2640 zbookmark_t zb; 2641 zio_prop_t zp; 2642 zio_t *zio; 2643 int wp_flag = 0; 2644 2645 DB_DNODE_ENTER(db); 2646 dn = DB_DNODE(db); 2647 os = dn->dn_objset; 2648 2649 if (db->db_state != DB_NOFILL) { 2650 if (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE) { 2651 /* 2652 * Private object buffers are released here rather 2653 * than in dbuf_dirty() since they are only modified 2654 * in the syncing context and we don't want the 2655 * overhead of making multiple copies of the data. 2656 */ 2657 if (BP_IS_HOLE(db->db_blkptr)) { 2658 arc_buf_thaw(data); 2659 } else { 2660 dbuf_release_bp(db); 2661 } 2662 } 2663 } 2664 2665 if (parent != dn->dn_dbuf) { 2666 ASSERT(parent && parent->db_data_pending); 2667 ASSERT(db->db_level == parent->db_level-1); 2668 ASSERT(arc_released(parent->db_buf)); 2669 zio = parent->db_data_pending->dr_zio; 2670 } else { 2671 ASSERT((db->db_level == dn->dn_phys->dn_nlevels-1 && 2672 db->db_blkid != DMU_SPILL_BLKID) || 2673 (db->db_blkid == DMU_SPILL_BLKID && db->db_level == 0)); 2674 if (db->db_blkid != DMU_SPILL_BLKID) 2675 ASSERT3P(db->db_blkptr, ==, 2676 &dn->dn_phys->dn_blkptr[db->db_blkid]); 2677 zio = dn->dn_zio; 2678 } 2679 2680 ASSERT(db->db_level == 0 || data == db->db_buf); 2681 ASSERT3U(db->db_blkptr->blk_birth, <=, txg); 2682 ASSERT(zio); 2683 2684 SET_BOOKMARK(&zb, os->os_dsl_dataset ? 2685 os->os_dsl_dataset->ds_object : DMU_META_OBJSET, 2686 db->db.db_object, db->db_level, db->db_blkid); 2687 2688 if (db->db_blkid == DMU_SPILL_BLKID) 2689 wp_flag = WP_SPILL; 2690 wp_flag |= (db->db_state == DB_NOFILL) ? WP_NOFILL : 0; 2691 2692 dmu_write_policy(os, dn, db->db_level, wp_flag, &zp); 2693 DB_DNODE_EXIT(db); 2694 2695 if (db->db_level == 0 && dr->dt.dl.dr_override_state == DR_OVERRIDDEN) { 2696 ASSERT(db->db_state != DB_NOFILL); 2697 dr->dr_zio = zio_write(zio, os->os_spa, txg, 2698 db->db_blkptr, data->b_data, arc_buf_size(data), &zp, 2699 dbuf_write_override_ready, dbuf_write_override_done, dr, 2700 ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); 2701 mutex_enter(&db->db_mtx); 2702 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 2703 zio_write_override(dr->dr_zio, &dr->dt.dl.dr_overridden_by, 2704 dr->dt.dl.dr_copies); 2705 mutex_exit(&db->db_mtx); 2706 } else if (db->db_state == DB_NOFILL) { 2707 ASSERT(zp.zp_checksum == ZIO_CHECKSUM_OFF); 2708 dr->dr_zio = zio_write(zio, os->os_spa, txg, 2709 db->db_blkptr, NULL, db->db.db_size, &zp, 2710 dbuf_write_nofill_ready, dbuf_write_nofill_done, db, 2711 ZIO_PRIORITY_ASYNC_WRITE, 2712 ZIO_FLAG_MUSTSUCCEED | ZIO_FLAG_NODATA, &zb); 2713 } else { 2714 ASSERT(arc_released(data)); 2715 dr->dr_zio = arc_write(zio, os->os_spa, txg, 2716 db->db_blkptr, data, DBUF_IS_L2CACHEABLE(db), &zp, 2717 dbuf_write_ready, dbuf_write_done, db, 2718 ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); 2719 } 2720 } 2721