1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <sys/zfs_context.h> 27 #include <sys/dmu.h> 28 #include <sys/dmu_impl.h> 29 #include <sys/dbuf.h> 30 #include <sys/dmu_objset.h> 31 #include <sys/dsl_dataset.h> 32 #include <sys/dsl_dir.h> 33 #include <sys/dmu_tx.h> 34 #include <sys/spa.h> 35 #include <sys/zio.h> 36 #include <sys/dmu_zfetch.h> 37 #include <sys/sa.h> 38 #include <sys/sa_impl.h> 39 40 static void dbuf_destroy(dmu_buf_impl_t *db); 41 static int dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx); 42 static void dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx); 43 44 /* 45 * Global data structures and functions for the dbuf cache. 46 */ 47 static kmem_cache_t *dbuf_cache; 48 49 /* ARGSUSED */ 50 static int 51 dbuf_cons(void *vdb, void *unused, int kmflag) 52 { 53 dmu_buf_impl_t *db = vdb; 54 bzero(db, sizeof (dmu_buf_impl_t)); 55 56 mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL); 57 cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL); 58 refcount_create(&db->db_holds); 59 return (0); 60 } 61 62 /* ARGSUSED */ 63 static void 64 dbuf_dest(void *vdb, void *unused) 65 { 66 dmu_buf_impl_t *db = vdb; 67 mutex_destroy(&db->db_mtx); 68 cv_destroy(&db->db_changed); 69 refcount_destroy(&db->db_holds); 70 } 71 72 /* 73 * dbuf hash table routines 74 */ 75 static dbuf_hash_table_t dbuf_hash_table; 76 77 static uint64_t dbuf_hash_count; 78 79 static uint64_t 80 dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid) 81 { 82 uintptr_t osv = (uintptr_t)os; 83 uint64_t crc = -1ULL; 84 85 ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 86 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (lvl)) & 0xFF]; 87 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF]; 88 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF]; 89 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF]; 90 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 0)) & 0xFF]; 91 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 8)) & 0xFF]; 92 93 crc ^= (osv>>14) ^ (obj>>16) ^ (blkid>>16); 94 95 return (crc); 96 } 97 98 #define DBUF_HASH(os, obj, level, blkid) dbuf_hash(os, obj, level, blkid); 99 100 #define DBUF_EQUAL(dbuf, os, obj, level, blkid) \ 101 ((dbuf)->db.db_object == (obj) && \ 102 (dbuf)->db_objset == (os) && \ 103 (dbuf)->db_level == (level) && \ 104 (dbuf)->db_blkid == (blkid)) 105 106 dmu_buf_impl_t * 107 dbuf_find(dnode_t *dn, uint8_t level, uint64_t blkid) 108 { 109 dbuf_hash_table_t *h = &dbuf_hash_table; 110 objset_t *os = dn->dn_objset; 111 uint64_t obj = dn->dn_object; 112 uint64_t hv = DBUF_HASH(os, obj, level, blkid); 113 uint64_t idx = hv & h->hash_table_mask; 114 dmu_buf_impl_t *db; 115 116 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 117 for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) { 118 if (DBUF_EQUAL(db, os, obj, level, blkid)) { 119 mutex_enter(&db->db_mtx); 120 if (db->db_state != DB_EVICTING) { 121 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 122 return (db); 123 } 124 mutex_exit(&db->db_mtx); 125 } 126 } 127 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 128 return (NULL); 129 } 130 131 /* 132 * Insert an entry into the hash table. If there is already an element 133 * equal to elem in the hash table, then the already existing element 134 * will be returned and the new element will not be inserted. 135 * Otherwise returns NULL. 136 */ 137 static dmu_buf_impl_t * 138 dbuf_hash_insert(dmu_buf_impl_t *db) 139 { 140 dbuf_hash_table_t *h = &dbuf_hash_table; 141 objset_t *os = db->db_objset; 142 uint64_t obj = db->db.db_object; 143 int level = db->db_level; 144 uint64_t blkid = db->db_blkid; 145 uint64_t hv = DBUF_HASH(os, obj, level, blkid); 146 uint64_t idx = hv & h->hash_table_mask; 147 dmu_buf_impl_t *dbf; 148 149 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 150 for (dbf = h->hash_table[idx]; dbf != NULL; dbf = dbf->db_hash_next) { 151 if (DBUF_EQUAL(dbf, os, obj, level, blkid)) { 152 mutex_enter(&dbf->db_mtx); 153 if (dbf->db_state != DB_EVICTING) { 154 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 155 return (dbf); 156 } 157 mutex_exit(&dbf->db_mtx); 158 } 159 } 160 161 mutex_enter(&db->db_mtx); 162 db->db_hash_next = h->hash_table[idx]; 163 h->hash_table[idx] = db; 164 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 165 atomic_add_64(&dbuf_hash_count, 1); 166 167 return (NULL); 168 } 169 170 /* 171 * Remove an entry from the hash table. This operation will 172 * fail if there are any existing holds on the db. 173 */ 174 static void 175 dbuf_hash_remove(dmu_buf_impl_t *db) 176 { 177 dbuf_hash_table_t *h = &dbuf_hash_table; 178 uint64_t hv = DBUF_HASH(db->db_objset, db->db.db_object, 179 db->db_level, db->db_blkid); 180 uint64_t idx = hv & h->hash_table_mask; 181 dmu_buf_impl_t *dbf, **dbp; 182 183 /* 184 * We musn't hold db_mtx to maintin lock ordering: 185 * DBUF_HASH_MUTEX > db_mtx. 186 */ 187 ASSERT(refcount_is_zero(&db->db_holds)); 188 ASSERT(db->db_state == DB_EVICTING); 189 ASSERT(!MUTEX_HELD(&db->db_mtx)); 190 191 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 192 dbp = &h->hash_table[idx]; 193 while ((dbf = *dbp) != db) { 194 dbp = &dbf->db_hash_next; 195 ASSERT(dbf != NULL); 196 } 197 *dbp = db->db_hash_next; 198 db->db_hash_next = NULL; 199 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 200 atomic_add_64(&dbuf_hash_count, -1); 201 } 202 203 static arc_evict_func_t dbuf_do_evict; 204 205 static void 206 dbuf_evict_user(dmu_buf_impl_t *db) 207 { 208 ASSERT(MUTEX_HELD(&db->db_mtx)); 209 210 if (db->db_level != 0 || db->db_evict_func == NULL) 211 return; 212 213 if (db->db_user_data_ptr_ptr) 214 *db->db_user_data_ptr_ptr = db->db.db_data; 215 db->db_evict_func(&db->db, db->db_user_ptr); 216 db->db_user_ptr = NULL; 217 db->db_user_data_ptr_ptr = NULL; 218 db->db_evict_func = NULL; 219 } 220 221 void 222 dbuf_evict(dmu_buf_impl_t *db) 223 { 224 ASSERT(MUTEX_HELD(&db->db_mtx)); 225 ASSERT(db->db_buf == NULL); 226 ASSERT(db->db_data_pending == NULL); 227 228 dbuf_clear(db); 229 dbuf_destroy(db); 230 } 231 232 void 233 dbuf_init(void) 234 { 235 uint64_t hsize = 1ULL << 16; 236 dbuf_hash_table_t *h = &dbuf_hash_table; 237 int i; 238 239 /* 240 * The hash table is big enough to fill all of physical memory 241 * with an average 4K block size. The table will take up 242 * totalmem*sizeof(void*)/4K (i.e. 2MB/GB with 8-byte pointers). 243 */ 244 while (hsize * 4096 < physmem * PAGESIZE) 245 hsize <<= 1; 246 247 retry: 248 h->hash_table_mask = hsize - 1; 249 h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP); 250 if (h->hash_table == NULL) { 251 /* XXX - we should really return an error instead of assert */ 252 ASSERT(hsize > (1ULL << 10)); 253 hsize >>= 1; 254 goto retry; 255 } 256 257 dbuf_cache = kmem_cache_create("dmu_buf_impl_t", 258 sizeof (dmu_buf_impl_t), 259 0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0); 260 261 for (i = 0; i < DBUF_MUTEXES; i++) 262 mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL); 263 } 264 265 void 266 dbuf_fini(void) 267 { 268 dbuf_hash_table_t *h = &dbuf_hash_table; 269 int i; 270 271 for (i = 0; i < DBUF_MUTEXES; i++) 272 mutex_destroy(&h->hash_mutexes[i]); 273 kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *)); 274 kmem_cache_destroy(dbuf_cache); 275 } 276 277 /* 278 * Other stuff. 279 */ 280 281 #ifdef ZFS_DEBUG 282 static void 283 dbuf_verify(dmu_buf_impl_t *db) 284 { 285 dnode_t *dn = db->db_dnode; 286 dbuf_dirty_record_t *dr; 287 288 ASSERT(MUTEX_HELD(&db->db_mtx)); 289 290 if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY)) 291 return; 292 293 ASSERT(db->db_objset != NULL); 294 if (dn == NULL) { 295 ASSERT(db->db_parent == NULL); 296 ASSERT(db->db_blkptr == NULL); 297 } else { 298 ASSERT3U(db->db.db_object, ==, dn->dn_object); 299 ASSERT3P(db->db_objset, ==, dn->dn_objset); 300 ASSERT3U(db->db_level, <, dn->dn_nlevels); 301 ASSERT(db->db_blkid == DMU_BONUS_BLKID || db->db_blkid == 302 DMU_SPILL_BLKID || list_head(&dn->dn_dbufs)); 303 } 304 if (db->db_blkid == DMU_BONUS_BLKID) { 305 ASSERT(dn != NULL); 306 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 307 ASSERT3U(db->db.db_offset, ==, DMU_BONUS_BLKID); 308 } else if (db->db_blkid == DMU_SPILL_BLKID) { 309 ASSERT(dn != NULL); 310 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 311 ASSERT3U(db->db.db_offset, ==, 0); 312 } else { 313 ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size); 314 } 315 316 for (dr = db->db_data_pending; dr != NULL; dr = dr->dr_next) 317 ASSERT(dr->dr_dbuf == db); 318 319 for (dr = db->db_last_dirty; dr != NULL; dr = dr->dr_next) 320 ASSERT(dr->dr_dbuf == db); 321 322 /* 323 * We can't assert that db_size matches dn_datablksz because it 324 * can be momentarily different when another thread is doing 325 * dnode_set_blksz(). 326 */ 327 if (db->db_level == 0 && db->db.db_object == DMU_META_DNODE_OBJECT) { 328 dr = db->db_data_pending; 329 /* 330 * It should only be modified in syncing context, so 331 * make sure we only have one copy of the data. 332 */ 333 ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf); 334 } 335 336 /* verify db->db_blkptr */ 337 if (db->db_blkptr) { 338 if (db->db_parent == dn->dn_dbuf) { 339 /* db is pointed to by the dnode */ 340 /* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */ 341 if (DMU_OBJECT_IS_SPECIAL(db->db.db_object)) 342 ASSERT(db->db_parent == NULL); 343 else 344 ASSERT(db->db_parent != NULL); 345 if (db->db_blkid != DMU_SPILL_BLKID) 346 ASSERT3P(db->db_blkptr, ==, 347 &dn->dn_phys->dn_blkptr[db->db_blkid]); 348 } else { 349 /* db is pointed to by an indirect block */ 350 int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT; 351 ASSERT3U(db->db_parent->db_level, ==, db->db_level+1); 352 ASSERT3U(db->db_parent->db.db_object, ==, 353 db->db.db_object); 354 /* 355 * dnode_grow_indblksz() can make this fail if we don't 356 * have the struct_rwlock. XXX indblksz no longer 357 * grows. safe to do this now? 358 */ 359 if (RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)) { 360 ASSERT3P(db->db_blkptr, ==, 361 ((blkptr_t *)db->db_parent->db.db_data + 362 db->db_blkid % epb)); 363 } 364 } 365 } 366 if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) && 367 db->db.db_data && db->db_blkid != DMU_BONUS_BLKID && 368 db->db_state != DB_FILL && !dn->dn_free_txg) { 369 /* 370 * If the blkptr isn't set but they have nonzero data, 371 * it had better be dirty, otherwise we'll lose that 372 * data when we evict this buffer. 373 */ 374 if (db->db_dirtycnt == 0) { 375 uint64_t *buf = db->db.db_data; 376 int i; 377 378 for (i = 0; i < db->db.db_size >> 3; i++) { 379 ASSERT(buf[i] == 0); 380 } 381 } 382 } 383 } 384 #endif 385 386 static void 387 dbuf_update_data(dmu_buf_impl_t *db) 388 { 389 ASSERT(MUTEX_HELD(&db->db_mtx)); 390 if (db->db_level == 0 && db->db_user_data_ptr_ptr) { 391 ASSERT(!refcount_is_zero(&db->db_holds)); 392 *db->db_user_data_ptr_ptr = db->db.db_data; 393 } 394 } 395 396 static void 397 dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf) 398 { 399 ASSERT(MUTEX_HELD(&db->db_mtx)); 400 ASSERT(db->db_buf == NULL || !arc_has_callback(db->db_buf)); 401 db->db_buf = buf; 402 if (buf != NULL) { 403 ASSERT(buf->b_data != NULL); 404 db->db.db_data = buf->b_data; 405 if (!arc_released(buf)) 406 arc_set_callback(buf, dbuf_do_evict, db); 407 dbuf_update_data(db); 408 } else { 409 dbuf_evict_user(db); 410 db->db.db_data = NULL; 411 if (db->db_state != DB_NOFILL) 412 db->db_state = DB_UNCACHED; 413 } 414 } 415 416 /* 417 * Loan out an arc_buf for read. Return the loaned arc_buf. 418 */ 419 arc_buf_t * 420 dbuf_loan_arcbuf(dmu_buf_impl_t *db) 421 { 422 arc_buf_t *abuf; 423 424 mutex_enter(&db->db_mtx); 425 if (arc_released(db->db_buf) || refcount_count(&db->db_holds) > 1) { 426 int blksz = db->db.db_size; 427 mutex_exit(&db->db_mtx); 428 abuf = arc_loan_buf(db->db_dnode->dn_objset->os_spa, blksz); 429 bcopy(db->db.db_data, abuf->b_data, blksz); 430 } else { 431 abuf = db->db_buf; 432 arc_loan_inuse_buf(abuf, db); 433 dbuf_set_data(db, NULL); 434 mutex_exit(&db->db_mtx); 435 } 436 return (abuf); 437 } 438 439 uint64_t 440 dbuf_whichblock(dnode_t *dn, uint64_t offset) 441 { 442 if (dn->dn_datablkshift) { 443 return (offset >> dn->dn_datablkshift); 444 } else { 445 ASSERT3U(offset, <, dn->dn_datablksz); 446 return (0); 447 } 448 } 449 450 static void 451 dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb) 452 { 453 dmu_buf_impl_t *db = vdb; 454 455 mutex_enter(&db->db_mtx); 456 ASSERT3U(db->db_state, ==, DB_READ); 457 /* 458 * All reads are synchronous, so we must have a hold on the dbuf 459 */ 460 ASSERT(refcount_count(&db->db_holds) > 0); 461 ASSERT(db->db_buf == NULL); 462 ASSERT(db->db.db_data == NULL); 463 if (db->db_level == 0 && db->db_freed_in_flight) { 464 /* we were freed in flight; disregard any error */ 465 arc_release(buf, db); 466 bzero(buf->b_data, db->db.db_size); 467 arc_buf_freeze(buf); 468 db->db_freed_in_flight = FALSE; 469 dbuf_set_data(db, buf); 470 db->db_state = DB_CACHED; 471 } else if (zio == NULL || zio->io_error == 0) { 472 dbuf_set_data(db, buf); 473 db->db_state = DB_CACHED; 474 } else { 475 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 476 ASSERT3P(db->db_buf, ==, NULL); 477 VERIFY(arc_buf_remove_ref(buf, db) == 1); 478 db->db_state = DB_UNCACHED; 479 } 480 cv_broadcast(&db->db_changed); 481 mutex_exit(&db->db_mtx); 482 dbuf_rele(db, NULL); 483 } 484 485 static void 486 dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t *flags) 487 { 488 dnode_t *dn = db->db_dnode; 489 zbookmark_t zb; 490 uint32_t aflags = ARC_NOWAIT; 491 arc_buf_t *pbuf; 492 493 ASSERT(!refcount_is_zero(&db->db_holds)); 494 /* We need the struct_rwlock to prevent db_blkptr from changing. */ 495 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 496 ASSERT(MUTEX_HELD(&db->db_mtx)); 497 ASSERT(db->db_state == DB_UNCACHED); 498 ASSERT(db->db_buf == NULL); 499 500 if (db->db_blkid == DMU_BONUS_BLKID) { 501 int bonuslen = MIN(dn->dn_bonuslen, dn->dn_phys->dn_bonuslen); 502 503 ASSERT3U(bonuslen, <=, db->db.db_size); 504 db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN); 505 arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 506 if (bonuslen < DN_MAX_BONUSLEN) 507 bzero(db->db.db_data, DN_MAX_BONUSLEN); 508 if (bonuslen) 509 bcopy(DN_BONUS(dn->dn_phys), db->db.db_data, bonuslen); 510 dbuf_update_data(db); 511 db->db_state = DB_CACHED; 512 mutex_exit(&db->db_mtx); 513 return; 514 } 515 516 /* 517 * Recheck BP_IS_HOLE() after dnode_block_freed() in case dnode_sync() 518 * processes the delete record and clears the bp while we are waiting 519 * for the dn_mtx (resulting in a "no" from block_freed). 520 */ 521 if (db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr) || 522 (db->db_level == 0 && (dnode_block_freed(dn, db->db_blkid) || 523 BP_IS_HOLE(db->db_blkptr)))) { 524 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 525 526 dbuf_set_data(db, arc_buf_alloc(dn->dn_objset->os_spa, 527 db->db.db_size, db, type)); 528 bzero(db->db.db_data, db->db.db_size); 529 db->db_state = DB_CACHED; 530 *flags |= DB_RF_CACHED; 531 mutex_exit(&db->db_mtx); 532 return; 533 } 534 535 db->db_state = DB_READ; 536 mutex_exit(&db->db_mtx); 537 538 if (DBUF_IS_L2CACHEABLE(db)) 539 aflags |= ARC_L2CACHE; 540 541 SET_BOOKMARK(&zb, db->db_objset->os_dsl_dataset ? 542 db->db_objset->os_dsl_dataset->ds_object : DMU_META_OBJSET, 543 db->db.db_object, db->db_level, db->db_blkid); 544 545 dbuf_add_ref(db, NULL); 546 /* ZIO_FLAG_CANFAIL callers have to check the parent zio's error */ 547 548 if (db->db_parent) 549 pbuf = db->db_parent->db_buf; 550 else 551 pbuf = db->db_objset->os_phys_buf; 552 553 (void) arc_read(zio, dn->dn_objset->os_spa, db->db_blkptr, pbuf, 554 dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ, 555 (*flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED, 556 &aflags, &zb); 557 if (aflags & ARC_CACHED) 558 *flags |= DB_RF_CACHED; 559 } 560 561 int 562 dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags) 563 { 564 int err = 0; 565 int havepzio = (zio != NULL); 566 int prefetch; 567 568 /* 569 * We don't have to hold the mutex to check db_state because it 570 * can't be freed while we have a hold on the buffer. 571 */ 572 ASSERT(!refcount_is_zero(&db->db_holds)); 573 574 if (db->db_state == DB_NOFILL) 575 return (EIO); 576 577 if ((flags & DB_RF_HAVESTRUCT) == 0) 578 rw_enter(&db->db_dnode->dn_struct_rwlock, RW_READER); 579 580 prefetch = db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 581 (flags & DB_RF_NOPREFETCH) == 0 && db->db_dnode != NULL && 582 DBUF_IS_CACHEABLE(db); 583 584 mutex_enter(&db->db_mtx); 585 if (db->db_state == DB_CACHED) { 586 mutex_exit(&db->db_mtx); 587 if (prefetch) 588 dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset, 589 db->db.db_size, TRUE); 590 if ((flags & DB_RF_HAVESTRUCT) == 0) 591 rw_exit(&db->db_dnode->dn_struct_rwlock); 592 } else if (db->db_state == DB_UNCACHED) { 593 if (zio == NULL) { 594 zio = zio_root(db->db_dnode->dn_objset->os_spa, 595 NULL, NULL, ZIO_FLAG_CANFAIL); 596 } 597 dbuf_read_impl(db, zio, &flags); 598 599 /* dbuf_read_impl has dropped db_mtx for us */ 600 601 if (prefetch) 602 dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset, 603 db->db.db_size, flags & DB_RF_CACHED); 604 605 if ((flags & DB_RF_HAVESTRUCT) == 0) 606 rw_exit(&db->db_dnode->dn_struct_rwlock); 607 608 if (!havepzio) 609 err = zio_wait(zio); 610 } else { 611 mutex_exit(&db->db_mtx); 612 if (prefetch) 613 dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset, 614 db->db.db_size, TRUE); 615 if ((flags & DB_RF_HAVESTRUCT) == 0) 616 rw_exit(&db->db_dnode->dn_struct_rwlock); 617 618 mutex_enter(&db->db_mtx); 619 if ((flags & DB_RF_NEVERWAIT) == 0) { 620 while (db->db_state == DB_READ || 621 db->db_state == DB_FILL) { 622 ASSERT(db->db_state == DB_READ || 623 (flags & DB_RF_HAVESTRUCT) == 0); 624 cv_wait(&db->db_changed, &db->db_mtx); 625 } 626 if (db->db_state == DB_UNCACHED) 627 err = EIO; 628 } 629 mutex_exit(&db->db_mtx); 630 } 631 632 ASSERT(err || havepzio || db->db_state == DB_CACHED); 633 return (err); 634 } 635 636 static void 637 dbuf_noread(dmu_buf_impl_t *db) 638 { 639 ASSERT(!refcount_is_zero(&db->db_holds)); 640 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 641 mutex_enter(&db->db_mtx); 642 while (db->db_state == DB_READ || db->db_state == DB_FILL) 643 cv_wait(&db->db_changed, &db->db_mtx); 644 if (db->db_state == DB_UNCACHED) { 645 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 646 647 ASSERT(db->db_buf == NULL); 648 ASSERT(db->db.db_data == NULL); 649 dbuf_set_data(db, arc_buf_alloc(db->db_dnode->dn_objset->os_spa, 650 db->db.db_size, db, type)); 651 db->db_state = DB_FILL; 652 } else if (db->db_state == DB_NOFILL) { 653 dbuf_set_data(db, NULL); 654 } else { 655 ASSERT3U(db->db_state, ==, DB_CACHED); 656 } 657 mutex_exit(&db->db_mtx); 658 } 659 660 /* 661 * This is our just-in-time copy function. It makes a copy of 662 * buffers, that have been modified in a previous transaction 663 * group, before we modify them in the current active group. 664 * 665 * This function is used in two places: when we are dirtying a 666 * buffer for the first time in a txg, and when we are freeing 667 * a range in a dnode that includes this buffer. 668 * 669 * Note that when we are called from dbuf_free_range() we do 670 * not put a hold on the buffer, we just traverse the active 671 * dbuf list for the dnode. 672 */ 673 static void 674 dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg) 675 { 676 dbuf_dirty_record_t *dr = db->db_last_dirty; 677 678 ASSERT(MUTEX_HELD(&db->db_mtx)); 679 ASSERT(db->db.db_data != NULL); 680 ASSERT(db->db_level == 0); 681 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT); 682 683 if (dr == NULL || 684 (dr->dt.dl.dr_data != 685 ((db->db_blkid == DMU_BONUS_BLKID) ? db->db.db_data : db->db_buf))) 686 return; 687 688 /* 689 * If the last dirty record for this dbuf has not yet synced 690 * and its referencing the dbuf data, either: 691 * reset the reference to point to a new copy, 692 * or (if there a no active holders) 693 * just null out the current db_data pointer. 694 */ 695 ASSERT(dr->dr_txg >= txg - 2); 696 if (db->db_blkid == DMU_BONUS_BLKID) { 697 /* Note that the data bufs here are zio_bufs */ 698 dr->dt.dl.dr_data = zio_buf_alloc(DN_MAX_BONUSLEN); 699 arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 700 bcopy(db->db.db_data, dr->dt.dl.dr_data, DN_MAX_BONUSLEN); 701 } else if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 702 int size = db->db.db_size; 703 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 704 dr->dt.dl.dr_data = arc_buf_alloc( 705 db->db_dnode->dn_objset->os_spa, size, db, type); 706 bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size); 707 } else { 708 dbuf_set_data(db, NULL); 709 } 710 } 711 712 void 713 dbuf_unoverride(dbuf_dirty_record_t *dr) 714 { 715 dmu_buf_impl_t *db = dr->dr_dbuf; 716 blkptr_t *bp = &dr->dt.dl.dr_overridden_by; 717 uint64_t txg = dr->dr_txg; 718 719 ASSERT(MUTEX_HELD(&db->db_mtx)); 720 ASSERT(dr->dt.dl.dr_override_state != DR_IN_DMU_SYNC); 721 ASSERT(db->db_level == 0); 722 723 if (db->db_blkid == DMU_BONUS_BLKID || 724 dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN) 725 return; 726 727 ASSERT(db->db_data_pending != dr); 728 729 /* free this block */ 730 if (!BP_IS_HOLE(bp)) 731 dsl_free(spa_get_dsl(db->db_dnode->dn_objset->os_spa), txg, bp); 732 733 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 734 /* 735 * Release the already-written buffer, so we leave it in 736 * a consistent dirty state. Note that all callers are 737 * modifying the buffer, so they will immediately do 738 * another (redundant) arc_release(). Therefore, leave 739 * the buf thawed to save the effort of freezing & 740 * immediately re-thawing it. 741 */ 742 arc_release(dr->dt.dl.dr_data, db); 743 } 744 745 /* 746 * Evict (if its unreferenced) or clear (if its referenced) any level-0 747 * data blocks in the free range, so that any future readers will find 748 * empty blocks. Also, if we happen accross any level-1 dbufs in the 749 * range that have not already been marked dirty, mark them dirty so 750 * they stay in memory. 751 */ 752 void 753 dbuf_free_range(dnode_t *dn, uint64_t start, uint64_t end, dmu_tx_t *tx) 754 { 755 dmu_buf_impl_t *db, *db_next; 756 uint64_t txg = tx->tx_txg; 757 int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 758 uint64_t first_l1 = start >> epbs; 759 uint64_t last_l1 = end >> epbs; 760 761 if (end > dn->dn_maxblkid && (end != DMU_SPILL_BLKID)) { 762 end = dn->dn_maxblkid; 763 last_l1 = end >> epbs; 764 } 765 dprintf_dnode(dn, "start=%llu end=%llu\n", start, end); 766 mutex_enter(&dn->dn_dbufs_mtx); 767 for (db = list_head(&dn->dn_dbufs); db; db = db_next) { 768 db_next = list_next(&dn->dn_dbufs, db); 769 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 770 771 if (db->db_level == 1 && 772 db->db_blkid >= first_l1 && db->db_blkid <= last_l1) { 773 mutex_enter(&db->db_mtx); 774 if (db->db_last_dirty && 775 db->db_last_dirty->dr_txg < txg) { 776 dbuf_add_ref(db, FTAG); 777 mutex_exit(&db->db_mtx); 778 dbuf_will_dirty(db, tx); 779 dbuf_rele(db, FTAG); 780 } else { 781 mutex_exit(&db->db_mtx); 782 } 783 } 784 785 if (db->db_level != 0) 786 continue; 787 dprintf_dbuf(db, "found buf %s\n", ""); 788 if (db->db_blkid < start || db->db_blkid > end) 789 continue; 790 791 /* found a level 0 buffer in the range */ 792 if (dbuf_undirty(db, tx)) 793 continue; 794 795 mutex_enter(&db->db_mtx); 796 if (db->db_state == DB_UNCACHED || 797 db->db_state == DB_NOFILL || 798 db->db_state == DB_EVICTING) { 799 ASSERT(db->db.db_data == NULL); 800 mutex_exit(&db->db_mtx); 801 continue; 802 } 803 if (db->db_state == DB_READ || db->db_state == DB_FILL) { 804 /* will be handled in dbuf_read_done or dbuf_rele */ 805 db->db_freed_in_flight = TRUE; 806 mutex_exit(&db->db_mtx); 807 continue; 808 } 809 if (refcount_count(&db->db_holds) == 0) { 810 ASSERT(db->db_buf); 811 dbuf_clear(db); 812 continue; 813 } 814 /* The dbuf is referenced */ 815 816 if (db->db_last_dirty != NULL) { 817 dbuf_dirty_record_t *dr = db->db_last_dirty; 818 819 if (dr->dr_txg == txg) { 820 /* 821 * This buffer is "in-use", re-adjust the file 822 * size to reflect that this buffer may 823 * contain new data when we sync. 824 */ 825 if (db->db_blkid > dn->dn_maxblkid) 826 dn->dn_maxblkid = db->db_blkid; 827 dbuf_unoverride(dr); 828 } else { 829 /* 830 * This dbuf is not dirty in the open context. 831 * Either uncache it (if its not referenced in 832 * the open context) or reset its contents to 833 * empty. 834 */ 835 dbuf_fix_old_data(db, txg); 836 } 837 } 838 /* clear the contents if its cached */ 839 if (db->db_state == DB_CACHED) { 840 ASSERT(db->db.db_data != NULL); 841 arc_release(db->db_buf, db); 842 bzero(db->db.db_data, db->db.db_size); 843 arc_buf_freeze(db->db_buf); 844 } 845 846 mutex_exit(&db->db_mtx); 847 } 848 mutex_exit(&dn->dn_dbufs_mtx); 849 } 850 851 static int 852 dbuf_block_freeable(dmu_buf_impl_t *db) 853 { 854 dsl_dataset_t *ds = db->db_objset->os_dsl_dataset; 855 uint64_t birth_txg = 0; 856 857 /* 858 * We don't need any locking to protect db_blkptr: 859 * If it's syncing, then db_last_dirty will be set 860 * so we'll ignore db_blkptr. 861 */ 862 ASSERT(MUTEX_HELD(&db->db_mtx)); 863 if (db->db_last_dirty) 864 birth_txg = db->db_last_dirty->dr_txg; 865 else if (db->db_blkptr) 866 birth_txg = db->db_blkptr->blk_birth; 867 868 /* If we don't exist or are in a snapshot, we can't be freed */ 869 if (birth_txg) 870 return (ds == NULL || 871 dsl_dataset_block_freeable(ds, birth_txg)); 872 else 873 return (FALSE); 874 } 875 876 void 877 dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx) 878 { 879 arc_buf_t *buf, *obuf; 880 int osize = db->db.db_size; 881 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 882 883 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 884 885 /* XXX does *this* func really need the lock? */ 886 ASSERT(RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)); 887 888 /* 889 * This call to dbuf_will_dirty() with the dn_struct_rwlock held 890 * is OK, because there can be no other references to the db 891 * when we are changing its size, so no concurrent DB_FILL can 892 * be happening. 893 */ 894 /* 895 * XXX we should be doing a dbuf_read, checking the return 896 * value and returning that up to our callers 897 */ 898 dbuf_will_dirty(db, tx); 899 900 /* create the data buffer for the new block */ 901 buf = arc_buf_alloc(db->db_dnode->dn_objset->os_spa, size, db, type); 902 903 /* copy old block data to the new block */ 904 obuf = db->db_buf; 905 bcopy(obuf->b_data, buf->b_data, MIN(osize, size)); 906 /* zero the remainder */ 907 if (size > osize) 908 bzero((uint8_t *)buf->b_data + osize, size - osize); 909 910 mutex_enter(&db->db_mtx); 911 dbuf_set_data(db, buf); 912 VERIFY(arc_buf_remove_ref(obuf, db) == 1); 913 db->db.db_size = size; 914 915 if (db->db_level == 0) { 916 ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg); 917 db->db_last_dirty->dt.dl.dr_data = buf; 918 } 919 mutex_exit(&db->db_mtx); 920 921 dnode_willuse_space(db->db_dnode, size-osize, tx); 922 } 923 924 dbuf_dirty_record_t * 925 dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 926 { 927 dnode_t *dn = db->db_dnode; 928 objset_t *os = dn->dn_objset; 929 dbuf_dirty_record_t **drp, *dr; 930 int drop_struct_lock = FALSE; 931 boolean_t do_free_accounting = B_FALSE; 932 int txgoff = tx->tx_txg & TXG_MASK; 933 934 ASSERT(tx->tx_txg != 0); 935 ASSERT(!refcount_is_zero(&db->db_holds)); 936 DMU_TX_DIRTY_BUF(tx, db); 937 938 /* 939 * Shouldn't dirty a regular buffer in syncing context. Private 940 * objects may be dirtied in syncing context, but only if they 941 * were already pre-dirtied in open context. 942 */ 943 ASSERT(!dmu_tx_is_syncing(tx) || 944 BP_IS_HOLE(dn->dn_objset->os_rootbp) || 945 DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 946 dn->dn_objset->os_dsl_dataset == NULL); 947 /* 948 * We make this assert for private objects as well, but after we 949 * check if we're already dirty. They are allowed to re-dirty 950 * in syncing context. 951 */ 952 ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 953 dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 954 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 955 956 mutex_enter(&db->db_mtx); 957 /* 958 * XXX make this true for indirects too? The problem is that 959 * transactions created with dmu_tx_create_assigned() from 960 * syncing context don't bother holding ahead. 961 */ 962 ASSERT(db->db_level != 0 || 963 db->db_state == DB_CACHED || db->db_state == DB_FILL || 964 db->db_state == DB_NOFILL); 965 966 mutex_enter(&dn->dn_mtx); 967 /* 968 * Don't set dirtyctx to SYNC if we're just modifying this as we 969 * initialize the objset. 970 */ 971 if (dn->dn_dirtyctx == DN_UNDIRTIED && 972 !BP_IS_HOLE(dn->dn_objset->os_rootbp)) { 973 dn->dn_dirtyctx = 974 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN); 975 ASSERT(dn->dn_dirtyctx_firstset == NULL); 976 dn->dn_dirtyctx_firstset = kmem_alloc(1, KM_SLEEP); 977 } 978 mutex_exit(&dn->dn_mtx); 979 980 if (db->db_blkid == DMU_SPILL_BLKID) 981 dn->dn_have_spill = B_TRUE; 982 983 /* 984 * If this buffer is already dirty, we're done. 985 */ 986 drp = &db->db_last_dirty; 987 ASSERT(*drp == NULL || (*drp)->dr_txg <= tx->tx_txg || 988 db->db.db_object == DMU_META_DNODE_OBJECT); 989 while ((dr = *drp) != NULL && dr->dr_txg > tx->tx_txg) 990 drp = &dr->dr_next; 991 if (dr && dr->dr_txg == tx->tx_txg) { 992 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID) { 993 /* 994 * If this buffer has already been written out, 995 * we now need to reset its state. 996 */ 997 dbuf_unoverride(dr); 998 if (db->db.db_object != DMU_META_DNODE_OBJECT && 999 db->db_state != DB_NOFILL) 1000 arc_buf_thaw(db->db_buf); 1001 } 1002 mutex_exit(&db->db_mtx); 1003 return (dr); 1004 } 1005 1006 /* 1007 * Only valid if not already dirty. 1008 */ 1009 ASSERT(dn->dn_object == 0 || 1010 dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 1011 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 1012 1013 ASSERT3U(dn->dn_nlevels, >, db->db_level); 1014 ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) || 1015 dn->dn_phys->dn_nlevels > db->db_level || 1016 dn->dn_next_nlevels[txgoff] > db->db_level || 1017 dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level || 1018 dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level); 1019 1020 /* 1021 * We should only be dirtying in syncing context if it's the 1022 * mos or we're initializing the os or it's a special object. 1023 * However, we are allowed to dirty in syncing context provided 1024 * we already dirtied it in open context. Hence we must make 1025 * this assertion only if we're not already dirty. 1026 */ 1027 ASSERT(!dmu_tx_is_syncing(tx) || DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 1028 os->os_dsl_dataset == NULL || BP_IS_HOLE(os->os_rootbp)); 1029 ASSERT(db->db.db_size != 0); 1030 1031 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1032 1033 if (db->db_blkid != DMU_BONUS_BLKID) { 1034 /* 1035 * Update the accounting. 1036 * Note: we delay "free accounting" until after we drop 1037 * the db_mtx. This keeps us from grabbing other locks 1038 * (and possibly deadlocking) in bp_get_dsize() while 1039 * also holding the db_mtx. 1040 */ 1041 dnode_willuse_space(dn, db->db.db_size, tx); 1042 do_free_accounting = dbuf_block_freeable(db); 1043 } 1044 1045 /* 1046 * If this buffer is dirty in an old transaction group we need 1047 * to make a copy of it so that the changes we make in this 1048 * transaction group won't leak out when we sync the older txg. 1049 */ 1050 dr = kmem_zalloc(sizeof (dbuf_dirty_record_t), KM_SLEEP); 1051 if (db->db_level == 0) { 1052 void *data_old = db->db_buf; 1053 1054 if (db->db_state != DB_NOFILL) { 1055 if (db->db_blkid == DMU_BONUS_BLKID) { 1056 dbuf_fix_old_data(db, tx->tx_txg); 1057 data_old = db->db.db_data; 1058 } else if (db->db.db_object != DMU_META_DNODE_OBJECT) { 1059 /* 1060 * Release the data buffer from the cache so 1061 * that we can modify it without impacting 1062 * possible other users of this cached data 1063 * block. Note that indirect blocks and 1064 * private objects are not released until the 1065 * syncing state (since they are only modified 1066 * then). 1067 */ 1068 arc_release(db->db_buf, db); 1069 dbuf_fix_old_data(db, tx->tx_txg); 1070 data_old = db->db_buf; 1071 } 1072 ASSERT(data_old != NULL); 1073 } 1074 dr->dt.dl.dr_data = data_old; 1075 } else { 1076 mutex_init(&dr->dt.di.dr_mtx, NULL, MUTEX_DEFAULT, NULL); 1077 list_create(&dr->dt.di.dr_children, 1078 sizeof (dbuf_dirty_record_t), 1079 offsetof(dbuf_dirty_record_t, dr_dirty_node)); 1080 } 1081 dr->dr_dbuf = db; 1082 dr->dr_txg = tx->tx_txg; 1083 dr->dr_next = *drp; 1084 *drp = dr; 1085 1086 /* 1087 * We could have been freed_in_flight between the dbuf_noread 1088 * and dbuf_dirty. We win, as though the dbuf_noread() had 1089 * happened after the free. 1090 */ 1091 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 1092 db->db_blkid != DMU_SPILL_BLKID) { 1093 mutex_enter(&dn->dn_mtx); 1094 dnode_clear_range(dn, db->db_blkid, 1, tx); 1095 mutex_exit(&dn->dn_mtx); 1096 db->db_freed_in_flight = FALSE; 1097 } 1098 1099 /* 1100 * This buffer is now part of this txg 1101 */ 1102 dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg); 1103 db->db_dirtycnt += 1; 1104 ASSERT3U(db->db_dirtycnt, <=, 3); 1105 1106 mutex_exit(&db->db_mtx); 1107 1108 if (db->db_blkid == DMU_BONUS_BLKID || 1109 db->db_blkid == DMU_SPILL_BLKID) { 1110 mutex_enter(&dn->dn_mtx); 1111 ASSERT(!list_link_active(&dr->dr_dirty_node)); 1112 list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1113 mutex_exit(&dn->dn_mtx); 1114 dnode_setdirty(dn, tx); 1115 return (dr); 1116 } else if (do_free_accounting) { 1117 blkptr_t *bp = db->db_blkptr; 1118 int64_t willfree = (bp && !BP_IS_HOLE(bp)) ? 1119 bp_get_dsize(os->os_spa, bp) : db->db.db_size; 1120 /* 1121 * This is only a guess -- if the dbuf is dirty 1122 * in a previous txg, we don't know how much 1123 * space it will use on disk yet. We should 1124 * really have the struct_rwlock to access 1125 * db_blkptr, but since this is just a guess, 1126 * it's OK if we get an odd answer. 1127 */ 1128 dnode_willuse_space(dn, -willfree, tx); 1129 } 1130 1131 if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 1132 rw_enter(&dn->dn_struct_rwlock, RW_READER); 1133 drop_struct_lock = TRUE; 1134 } 1135 1136 if (db->db_level == 0) { 1137 dnode_new_blkid(dn, db->db_blkid, tx, drop_struct_lock); 1138 ASSERT(dn->dn_maxblkid >= db->db_blkid); 1139 } 1140 1141 if (db->db_level+1 < dn->dn_nlevels) { 1142 dmu_buf_impl_t *parent = db->db_parent; 1143 dbuf_dirty_record_t *di; 1144 int parent_held = FALSE; 1145 1146 if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) { 1147 int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1148 1149 parent = dbuf_hold_level(dn, db->db_level+1, 1150 db->db_blkid >> epbs, FTAG); 1151 parent_held = TRUE; 1152 } 1153 if (drop_struct_lock) 1154 rw_exit(&dn->dn_struct_rwlock); 1155 ASSERT3U(db->db_level+1, ==, parent->db_level); 1156 di = dbuf_dirty(parent, tx); 1157 if (parent_held) 1158 dbuf_rele(parent, FTAG); 1159 1160 mutex_enter(&db->db_mtx); 1161 /* possible race with dbuf_undirty() */ 1162 if (db->db_last_dirty == dr || 1163 dn->dn_object == DMU_META_DNODE_OBJECT) { 1164 mutex_enter(&di->dt.di.dr_mtx); 1165 ASSERT3U(di->dr_txg, ==, tx->tx_txg); 1166 ASSERT(!list_link_active(&dr->dr_dirty_node)); 1167 list_insert_tail(&di->dt.di.dr_children, dr); 1168 mutex_exit(&di->dt.di.dr_mtx); 1169 dr->dr_parent = di; 1170 } 1171 mutex_exit(&db->db_mtx); 1172 } else { 1173 ASSERT(db->db_level+1 == dn->dn_nlevels); 1174 ASSERT(db->db_blkid < dn->dn_nblkptr); 1175 ASSERT(db->db_parent == NULL || 1176 db->db_parent == db->db_dnode->dn_dbuf); 1177 mutex_enter(&dn->dn_mtx); 1178 ASSERT(!list_link_active(&dr->dr_dirty_node)); 1179 list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1180 mutex_exit(&dn->dn_mtx); 1181 if (drop_struct_lock) 1182 rw_exit(&dn->dn_struct_rwlock); 1183 } 1184 1185 dnode_setdirty(dn, tx); 1186 return (dr); 1187 } 1188 1189 static int 1190 dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1191 { 1192 dnode_t *dn = db->db_dnode; 1193 uint64_t txg = tx->tx_txg; 1194 dbuf_dirty_record_t *dr, **drp; 1195 1196 ASSERT(txg != 0); 1197 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1198 1199 mutex_enter(&db->db_mtx); 1200 /* 1201 * If this buffer is not dirty, we're done. 1202 */ 1203 for (drp = &db->db_last_dirty; (dr = *drp) != NULL; drp = &dr->dr_next) 1204 if (dr->dr_txg <= txg) 1205 break; 1206 if (dr == NULL || dr->dr_txg < txg) { 1207 mutex_exit(&db->db_mtx); 1208 return (0); 1209 } 1210 ASSERT(dr->dr_txg == txg); 1211 ASSERT(dr->dr_dbuf == db); 1212 1213 /* 1214 * If this buffer is currently held, we cannot undirty 1215 * it, since one of the current holders may be in the 1216 * middle of an update. Note that users of dbuf_undirty() 1217 * should not place a hold on the dbuf before the call. 1218 */ 1219 if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 1220 mutex_exit(&db->db_mtx); 1221 /* Make sure we don't toss this buffer at sync phase */ 1222 mutex_enter(&dn->dn_mtx); 1223 dnode_clear_range(dn, db->db_blkid, 1, tx); 1224 mutex_exit(&dn->dn_mtx); 1225 return (0); 1226 } 1227 1228 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1229 1230 ASSERT(db->db.db_size != 0); 1231 1232 /* XXX would be nice to fix up dn_towrite_space[] */ 1233 1234 *drp = dr->dr_next; 1235 1236 if (dr->dr_parent) { 1237 mutex_enter(&dr->dr_parent->dt.di.dr_mtx); 1238 list_remove(&dr->dr_parent->dt.di.dr_children, dr); 1239 mutex_exit(&dr->dr_parent->dt.di.dr_mtx); 1240 } else if (db->db_level+1 == dn->dn_nlevels) { 1241 ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf); 1242 mutex_enter(&dn->dn_mtx); 1243 list_remove(&dn->dn_dirty_records[txg & TXG_MASK], dr); 1244 mutex_exit(&dn->dn_mtx); 1245 } 1246 1247 if (db->db_level == 0) { 1248 if (db->db_state != DB_NOFILL) { 1249 dbuf_unoverride(dr); 1250 1251 ASSERT(db->db_buf != NULL); 1252 ASSERT(dr->dt.dl.dr_data != NULL); 1253 if (dr->dt.dl.dr_data != db->db_buf) 1254 VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, 1255 db) == 1); 1256 } 1257 } else { 1258 ASSERT(db->db_buf != NULL); 1259 ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 1260 mutex_destroy(&dr->dt.di.dr_mtx); 1261 list_destroy(&dr->dt.di.dr_children); 1262 } 1263 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 1264 1265 ASSERT(db->db_dirtycnt > 0); 1266 db->db_dirtycnt -= 1; 1267 1268 if (refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) { 1269 arc_buf_t *buf = db->db_buf; 1270 1271 ASSERT(db->db_state == DB_NOFILL || arc_released(buf)); 1272 dbuf_set_data(db, NULL); 1273 VERIFY(arc_buf_remove_ref(buf, db) == 1); 1274 dbuf_evict(db); 1275 return (1); 1276 } 1277 1278 mutex_exit(&db->db_mtx); 1279 return (0); 1280 } 1281 1282 #pragma weak dmu_buf_will_dirty = dbuf_will_dirty 1283 void 1284 dbuf_will_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1285 { 1286 int rf = DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH; 1287 1288 ASSERT(tx->tx_txg != 0); 1289 ASSERT(!refcount_is_zero(&db->db_holds)); 1290 1291 if (RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)) 1292 rf |= DB_RF_HAVESTRUCT; 1293 (void) dbuf_read(db, NULL, rf); 1294 (void) dbuf_dirty(db, tx); 1295 } 1296 1297 void 1298 dmu_buf_will_not_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 1299 { 1300 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1301 1302 db->db_state = DB_NOFILL; 1303 1304 dmu_buf_will_fill(db_fake, tx); 1305 } 1306 1307 void 1308 dmu_buf_will_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 1309 { 1310 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1311 1312 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1313 ASSERT(tx->tx_txg != 0); 1314 ASSERT(db->db_level == 0); 1315 ASSERT(!refcount_is_zero(&db->db_holds)); 1316 1317 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT || 1318 dmu_tx_private_ok(tx)); 1319 1320 dbuf_noread(db); 1321 (void) dbuf_dirty(db, tx); 1322 } 1323 1324 #pragma weak dmu_buf_fill_done = dbuf_fill_done 1325 /* ARGSUSED */ 1326 void 1327 dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx) 1328 { 1329 mutex_enter(&db->db_mtx); 1330 DBUF_VERIFY(db); 1331 1332 if (db->db_state == DB_FILL) { 1333 if (db->db_level == 0 && db->db_freed_in_flight) { 1334 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1335 /* we were freed while filling */ 1336 /* XXX dbuf_undirty? */ 1337 bzero(db->db.db_data, db->db.db_size); 1338 db->db_freed_in_flight = FALSE; 1339 } 1340 db->db_state = DB_CACHED; 1341 cv_broadcast(&db->db_changed); 1342 } 1343 mutex_exit(&db->db_mtx); 1344 } 1345 1346 /* 1347 * Directly assign a provided arc buf to a given dbuf if it's not referenced 1348 * by anybody except our caller. Otherwise copy arcbuf's contents to dbuf. 1349 */ 1350 void 1351 dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx) 1352 { 1353 ASSERT(!refcount_is_zero(&db->db_holds)); 1354 ASSERT(db->db_dnode->dn_object != DMU_META_DNODE_OBJECT); 1355 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1356 ASSERT(db->db_level == 0); 1357 ASSERT(DBUF_GET_BUFC_TYPE(db) == ARC_BUFC_DATA); 1358 ASSERT(buf != NULL); 1359 ASSERT(arc_buf_size(buf) == db->db.db_size); 1360 ASSERT(tx->tx_txg != 0); 1361 1362 arc_return_buf(buf, db); 1363 ASSERT(arc_released(buf)); 1364 1365 mutex_enter(&db->db_mtx); 1366 1367 while (db->db_state == DB_READ || db->db_state == DB_FILL) 1368 cv_wait(&db->db_changed, &db->db_mtx); 1369 1370 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_UNCACHED); 1371 1372 if (db->db_state == DB_CACHED && 1373 refcount_count(&db->db_holds) - 1 > db->db_dirtycnt) { 1374 mutex_exit(&db->db_mtx); 1375 (void) dbuf_dirty(db, tx); 1376 bcopy(buf->b_data, db->db.db_data, db->db.db_size); 1377 VERIFY(arc_buf_remove_ref(buf, db) == 1); 1378 xuio_stat_wbuf_copied(); 1379 return; 1380 } 1381 1382 xuio_stat_wbuf_nocopy(); 1383 if (db->db_state == DB_CACHED) { 1384 dbuf_dirty_record_t *dr = db->db_last_dirty; 1385 1386 ASSERT(db->db_buf != NULL); 1387 if (dr != NULL && dr->dr_txg == tx->tx_txg) { 1388 ASSERT(dr->dt.dl.dr_data == db->db_buf); 1389 if (!arc_released(db->db_buf)) { 1390 ASSERT(dr->dt.dl.dr_override_state == 1391 DR_OVERRIDDEN); 1392 arc_release(db->db_buf, db); 1393 } 1394 dr->dt.dl.dr_data = buf; 1395 VERIFY(arc_buf_remove_ref(db->db_buf, db) == 1); 1396 } else if (dr == NULL || dr->dt.dl.dr_data != db->db_buf) { 1397 arc_release(db->db_buf, db); 1398 VERIFY(arc_buf_remove_ref(db->db_buf, db) == 1); 1399 } 1400 db->db_buf = NULL; 1401 } 1402 ASSERT(db->db_buf == NULL); 1403 dbuf_set_data(db, buf); 1404 db->db_state = DB_FILL; 1405 mutex_exit(&db->db_mtx); 1406 (void) dbuf_dirty(db, tx); 1407 dbuf_fill_done(db, tx); 1408 } 1409 1410 /* 1411 * "Clear" the contents of this dbuf. This will mark the dbuf 1412 * EVICTING and clear *most* of its references. Unfortunetely, 1413 * when we are not holding the dn_dbufs_mtx, we can't clear the 1414 * entry in the dn_dbufs list. We have to wait until dbuf_destroy() 1415 * in this case. For callers from the DMU we will usually see: 1416 * dbuf_clear()->arc_buf_evict()->dbuf_do_evict()->dbuf_destroy() 1417 * For the arc callback, we will usually see: 1418 * dbuf_do_evict()->dbuf_clear();dbuf_destroy() 1419 * Sometimes, though, we will get a mix of these two: 1420 * DMU: dbuf_clear()->arc_buf_evict() 1421 * ARC: dbuf_do_evict()->dbuf_destroy() 1422 */ 1423 void 1424 dbuf_clear(dmu_buf_impl_t *db) 1425 { 1426 dnode_t *dn = db->db_dnode; 1427 dmu_buf_impl_t *parent = db->db_parent; 1428 dmu_buf_impl_t *dndb = dn->dn_dbuf; 1429 int dbuf_gone = FALSE; 1430 1431 ASSERT(MUTEX_HELD(&db->db_mtx)); 1432 ASSERT(refcount_is_zero(&db->db_holds)); 1433 1434 dbuf_evict_user(db); 1435 1436 if (db->db_state == DB_CACHED) { 1437 ASSERT(db->db.db_data != NULL); 1438 if (db->db_blkid == DMU_BONUS_BLKID) { 1439 zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN); 1440 arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 1441 } 1442 db->db.db_data = NULL; 1443 db->db_state = DB_UNCACHED; 1444 } 1445 1446 ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL); 1447 ASSERT(db->db_data_pending == NULL); 1448 1449 db->db_state = DB_EVICTING; 1450 db->db_blkptr = NULL; 1451 1452 if (db->db_blkid != DMU_BONUS_BLKID && MUTEX_HELD(&dn->dn_dbufs_mtx)) { 1453 list_remove(&dn->dn_dbufs, db); 1454 dnode_rele(dn, db); 1455 db->db_dnode = NULL; 1456 } 1457 1458 if (db->db_buf) 1459 dbuf_gone = arc_buf_evict(db->db_buf); 1460 1461 if (!dbuf_gone) 1462 mutex_exit(&db->db_mtx); 1463 1464 /* 1465 * If this dbuf is referened from an indirect dbuf, 1466 * decrement the ref count on the indirect dbuf. 1467 */ 1468 if (parent && parent != dndb) 1469 dbuf_rele(parent, db); 1470 } 1471 1472 static int 1473 dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse, 1474 dmu_buf_impl_t **parentp, blkptr_t **bpp) 1475 { 1476 int nlevels, epbs; 1477 1478 *parentp = NULL; 1479 *bpp = NULL; 1480 1481 ASSERT(blkid != DMU_BONUS_BLKID); 1482 1483 if (blkid == DMU_SPILL_BLKID) { 1484 mutex_enter(&dn->dn_mtx); 1485 if (dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR) 1486 *bpp = &dn->dn_phys->dn_spill; 1487 else 1488 *bpp = NULL; 1489 dbuf_add_ref(dn->dn_dbuf, NULL); 1490 *parentp = dn->dn_dbuf; 1491 mutex_exit(&dn->dn_mtx); 1492 return (0); 1493 } 1494 1495 if (dn->dn_phys->dn_nlevels == 0) 1496 nlevels = 1; 1497 else 1498 nlevels = dn->dn_phys->dn_nlevels; 1499 1500 epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1501 1502 ASSERT3U(level * epbs, <, 64); 1503 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1504 if (level >= nlevels || 1505 (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) { 1506 /* the buffer has no parent yet */ 1507 return (ENOENT); 1508 } else if (level < nlevels-1) { 1509 /* this block is referenced from an indirect block */ 1510 int err = dbuf_hold_impl(dn, level+1, 1511 blkid >> epbs, fail_sparse, NULL, parentp); 1512 if (err) 1513 return (err); 1514 err = dbuf_read(*parentp, NULL, 1515 (DB_RF_HAVESTRUCT | DB_RF_NOPREFETCH | DB_RF_CANFAIL)); 1516 if (err) { 1517 dbuf_rele(*parentp, NULL); 1518 *parentp = NULL; 1519 return (err); 1520 } 1521 *bpp = ((blkptr_t *)(*parentp)->db.db_data) + 1522 (blkid & ((1ULL << epbs) - 1)); 1523 return (0); 1524 } else { 1525 /* the block is referenced from the dnode */ 1526 ASSERT3U(level, ==, nlevels-1); 1527 ASSERT(dn->dn_phys->dn_nblkptr == 0 || 1528 blkid < dn->dn_phys->dn_nblkptr); 1529 if (dn->dn_dbuf) { 1530 dbuf_add_ref(dn->dn_dbuf, NULL); 1531 *parentp = dn->dn_dbuf; 1532 } 1533 *bpp = &dn->dn_phys->dn_blkptr[blkid]; 1534 return (0); 1535 } 1536 } 1537 1538 static dmu_buf_impl_t * 1539 dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid, 1540 dmu_buf_impl_t *parent, blkptr_t *blkptr) 1541 { 1542 objset_t *os = dn->dn_objset; 1543 dmu_buf_impl_t *db, *odb; 1544 1545 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1546 ASSERT(dn->dn_type != DMU_OT_NONE); 1547 1548 db = kmem_cache_alloc(dbuf_cache, KM_SLEEP); 1549 1550 db->db_objset = os; 1551 db->db.db_object = dn->dn_object; 1552 db->db_level = level; 1553 db->db_blkid = blkid; 1554 db->db_last_dirty = NULL; 1555 db->db_dirtycnt = 0; 1556 db->db_dnode = dn; 1557 db->db_parent = parent; 1558 db->db_blkptr = blkptr; 1559 1560 db->db_user_ptr = NULL; 1561 db->db_user_data_ptr_ptr = NULL; 1562 db->db_evict_func = NULL; 1563 db->db_immediate_evict = 0; 1564 db->db_freed_in_flight = 0; 1565 1566 if (blkid == DMU_BONUS_BLKID) { 1567 ASSERT3P(parent, ==, dn->dn_dbuf); 1568 db->db.db_size = DN_MAX_BONUSLEN - 1569 (dn->dn_nblkptr-1) * sizeof (blkptr_t); 1570 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 1571 db->db.db_offset = DMU_BONUS_BLKID; 1572 db->db_state = DB_UNCACHED; 1573 /* the bonus dbuf is not placed in the hash table */ 1574 arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1575 return (db); 1576 } else if (blkid == DMU_SPILL_BLKID) { 1577 db->db.db_size = (blkptr != NULL) ? 1578 BP_GET_LSIZE(blkptr) : SPA_MINBLOCKSIZE; 1579 db->db.db_offset = 0; 1580 } else { 1581 int blocksize = 1582 db->db_level ? 1<<dn->dn_indblkshift : dn->dn_datablksz; 1583 db->db.db_size = blocksize; 1584 db->db.db_offset = db->db_blkid * blocksize; 1585 } 1586 1587 /* 1588 * Hold the dn_dbufs_mtx while we get the new dbuf 1589 * in the hash table *and* added to the dbufs list. 1590 * This prevents a possible deadlock with someone 1591 * trying to look up this dbuf before its added to the 1592 * dn_dbufs list. 1593 */ 1594 mutex_enter(&dn->dn_dbufs_mtx); 1595 db->db_state = DB_EVICTING; 1596 if ((odb = dbuf_hash_insert(db)) != NULL) { 1597 /* someone else inserted it first */ 1598 kmem_cache_free(dbuf_cache, db); 1599 mutex_exit(&dn->dn_dbufs_mtx); 1600 return (odb); 1601 } 1602 list_insert_head(&dn->dn_dbufs, db); 1603 db->db_state = DB_UNCACHED; 1604 mutex_exit(&dn->dn_dbufs_mtx); 1605 arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1606 1607 if (parent && parent != dn->dn_dbuf) 1608 dbuf_add_ref(parent, db); 1609 1610 ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 1611 refcount_count(&dn->dn_holds) > 0); 1612 (void) refcount_add(&dn->dn_holds, db); 1613 1614 dprintf_dbuf(db, "db=%p\n", db); 1615 1616 return (db); 1617 } 1618 1619 static int 1620 dbuf_do_evict(void *private) 1621 { 1622 arc_buf_t *buf = private; 1623 dmu_buf_impl_t *db = buf->b_private; 1624 1625 if (!MUTEX_HELD(&db->db_mtx)) 1626 mutex_enter(&db->db_mtx); 1627 1628 ASSERT(refcount_is_zero(&db->db_holds)); 1629 1630 if (db->db_state != DB_EVICTING) { 1631 ASSERT(db->db_state == DB_CACHED); 1632 DBUF_VERIFY(db); 1633 db->db_buf = NULL; 1634 dbuf_evict(db); 1635 } else { 1636 mutex_exit(&db->db_mtx); 1637 dbuf_destroy(db); 1638 } 1639 return (0); 1640 } 1641 1642 static void 1643 dbuf_destroy(dmu_buf_impl_t *db) 1644 { 1645 ASSERT(refcount_is_zero(&db->db_holds)); 1646 1647 if (db->db_blkid != DMU_BONUS_BLKID) { 1648 /* 1649 * If this dbuf is still on the dn_dbufs list, 1650 * remove it from that list. 1651 */ 1652 if (db->db_dnode) { 1653 dnode_t *dn = db->db_dnode; 1654 1655 mutex_enter(&dn->dn_dbufs_mtx); 1656 list_remove(&dn->dn_dbufs, db); 1657 mutex_exit(&dn->dn_dbufs_mtx); 1658 1659 dnode_rele(dn, db); 1660 db->db_dnode = NULL; 1661 } 1662 dbuf_hash_remove(db); 1663 } 1664 db->db_parent = NULL; 1665 db->db_buf = NULL; 1666 1667 ASSERT(!list_link_active(&db->db_link)); 1668 ASSERT(db->db.db_data == NULL); 1669 ASSERT(db->db_hash_next == NULL); 1670 ASSERT(db->db_blkptr == NULL); 1671 ASSERT(db->db_data_pending == NULL); 1672 1673 kmem_cache_free(dbuf_cache, db); 1674 arc_space_return(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1675 } 1676 1677 void 1678 dbuf_prefetch(dnode_t *dn, uint64_t blkid) 1679 { 1680 dmu_buf_impl_t *db = NULL; 1681 blkptr_t *bp = NULL; 1682 1683 ASSERT(blkid != DMU_BONUS_BLKID); 1684 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1685 1686 if (dnode_block_freed(dn, blkid)) 1687 return; 1688 1689 /* dbuf_find() returns with db_mtx held */ 1690 if (db = dbuf_find(dn, 0, blkid)) { 1691 if (refcount_count(&db->db_holds) > 0) { 1692 /* 1693 * This dbuf is active. We assume that it is 1694 * already CACHED, or else about to be either 1695 * read or filled. 1696 */ 1697 mutex_exit(&db->db_mtx); 1698 return; 1699 } 1700 mutex_exit(&db->db_mtx); 1701 db = NULL; 1702 } 1703 1704 if (dbuf_findbp(dn, 0, blkid, TRUE, &db, &bp) == 0) { 1705 if (bp && !BP_IS_HOLE(bp)) { 1706 arc_buf_t *pbuf; 1707 dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset; 1708 uint32_t aflags = ARC_NOWAIT | ARC_PREFETCH; 1709 zbookmark_t zb; 1710 1711 SET_BOOKMARK(&zb, ds ? ds->ds_object : DMU_META_OBJSET, 1712 dn->dn_object, 0, blkid); 1713 1714 if (db) 1715 pbuf = db->db_buf; 1716 else 1717 pbuf = dn->dn_objset->os_phys_buf; 1718 1719 (void) arc_read(NULL, dn->dn_objset->os_spa, 1720 bp, pbuf, NULL, NULL, ZIO_PRIORITY_ASYNC_READ, 1721 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 1722 &aflags, &zb); 1723 } 1724 if (db) 1725 dbuf_rele(db, NULL); 1726 } 1727 } 1728 1729 /* 1730 * Returns with db_holds incremented, and db_mtx not held. 1731 * Note: dn_struct_rwlock must be held. 1732 */ 1733 int 1734 dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, int fail_sparse, 1735 void *tag, dmu_buf_impl_t **dbp) 1736 { 1737 dmu_buf_impl_t *db, *parent = NULL; 1738 1739 ASSERT(blkid != DMU_BONUS_BLKID); 1740 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1741 ASSERT3U(dn->dn_nlevels, >, level); 1742 1743 *dbp = NULL; 1744 top: 1745 /* dbuf_find() returns with db_mtx held */ 1746 db = dbuf_find(dn, level, blkid); 1747 1748 if (db == NULL) { 1749 blkptr_t *bp = NULL; 1750 int err; 1751 1752 ASSERT3P(parent, ==, NULL); 1753 err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp); 1754 if (fail_sparse) { 1755 if (err == 0 && bp && BP_IS_HOLE(bp)) 1756 err = ENOENT; 1757 if (err) { 1758 if (parent) 1759 dbuf_rele(parent, NULL); 1760 return (err); 1761 } 1762 } 1763 if (err && err != ENOENT) 1764 return (err); 1765 db = dbuf_create(dn, level, blkid, parent, bp); 1766 } 1767 1768 if (db->db_buf && refcount_is_zero(&db->db_holds)) { 1769 arc_buf_add_ref(db->db_buf, db); 1770 if (db->db_buf->b_data == NULL) { 1771 dbuf_clear(db); 1772 if (parent) { 1773 dbuf_rele(parent, NULL); 1774 parent = NULL; 1775 } 1776 goto top; 1777 } 1778 ASSERT3P(db->db.db_data, ==, db->db_buf->b_data); 1779 } 1780 1781 ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf)); 1782 1783 /* 1784 * If this buffer is currently syncing out, and we are are 1785 * still referencing it from db_data, we need to make a copy 1786 * of it in case we decide we want to dirty it again in this txg. 1787 */ 1788 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 1789 dn->dn_object != DMU_META_DNODE_OBJECT && 1790 db->db_state == DB_CACHED && db->db_data_pending) { 1791 dbuf_dirty_record_t *dr = db->db_data_pending; 1792 1793 if (dr->dt.dl.dr_data == db->db_buf) { 1794 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 1795 1796 dbuf_set_data(db, 1797 arc_buf_alloc(db->db_dnode->dn_objset->os_spa, 1798 db->db.db_size, db, type)); 1799 bcopy(dr->dt.dl.dr_data->b_data, db->db.db_data, 1800 db->db.db_size); 1801 } 1802 } 1803 1804 (void) refcount_add(&db->db_holds, tag); 1805 dbuf_update_data(db); 1806 DBUF_VERIFY(db); 1807 mutex_exit(&db->db_mtx); 1808 1809 /* NOTE: we can't rele the parent until after we drop the db_mtx */ 1810 if (parent) 1811 dbuf_rele(parent, NULL); 1812 1813 ASSERT3P(db->db_dnode, ==, dn); 1814 ASSERT3U(db->db_blkid, ==, blkid); 1815 ASSERT3U(db->db_level, ==, level); 1816 *dbp = db; 1817 1818 return (0); 1819 } 1820 1821 dmu_buf_impl_t * 1822 dbuf_hold(dnode_t *dn, uint64_t blkid, void *tag) 1823 { 1824 dmu_buf_impl_t *db; 1825 int err = dbuf_hold_impl(dn, 0, blkid, FALSE, tag, &db); 1826 return (err ? NULL : db); 1827 } 1828 1829 dmu_buf_impl_t * 1830 dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, void *tag) 1831 { 1832 dmu_buf_impl_t *db; 1833 int err = dbuf_hold_impl(dn, level, blkid, FALSE, tag, &db); 1834 return (err ? NULL : db); 1835 } 1836 1837 void 1838 dbuf_create_bonus(dnode_t *dn) 1839 { 1840 ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 1841 1842 ASSERT(dn->dn_bonus == NULL); 1843 dn->dn_bonus = dbuf_create(dn, 0, DMU_BONUS_BLKID, dn->dn_dbuf, NULL); 1844 } 1845 1846 int 1847 dbuf_spill_set_blksz(dmu_buf_t *db_fake, uint64_t blksz, dmu_tx_t *tx) 1848 { 1849 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1850 if (db->db_blkid != DMU_SPILL_BLKID) 1851 return (ENOTSUP); 1852 if (blksz == 0) 1853 blksz = SPA_MINBLOCKSIZE; 1854 if (blksz > SPA_MAXBLOCKSIZE) 1855 blksz = SPA_MAXBLOCKSIZE; 1856 else 1857 blksz = P2ROUNDUP(blksz, SPA_MINBLOCKSIZE); 1858 1859 rw_enter(&db->db_dnode->dn_struct_rwlock, RW_WRITER); 1860 dbuf_new_size(db, blksz, tx); 1861 rw_exit(&db->db_dnode->dn_struct_rwlock); 1862 1863 return (0); 1864 } 1865 1866 void 1867 dbuf_rm_spill(dnode_t *dn, dmu_tx_t *tx) 1868 { 1869 dbuf_free_range(dn, DMU_SPILL_BLKID, DMU_SPILL_BLKID, tx); 1870 dnode_rm_spill(dn, tx); 1871 } 1872 1873 #pragma weak dmu_buf_add_ref = dbuf_add_ref 1874 void 1875 dbuf_add_ref(dmu_buf_impl_t *db, void *tag) 1876 { 1877 int64_t holds = refcount_add(&db->db_holds, tag); 1878 ASSERT(holds > 1); 1879 } 1880 1881 #pragma weak dmu_buf_rele = dbuf_rele 1882 void 1883 dbuf_rele(dmu_buf_impl_t *db, void *tag) 1884 { 1885 mutex_enter(&db->db_mtx); 1886 dbuf_rele_and_unlock(db, tag); 1887 } 1888 1889 /* 1890 * dbuf_rele() for an already-locked dbuf. This is necessary to allow 1891 * db_dirtycnt and db_holds to be updated atomically. 1892 */ 1893 void 1894 dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag) 1895 { 1896 int64_t holds; 1897 1898 ASSERT(MUTEX_HELD(&db->db_mtx)); 1899 DBUF_VERIFY(db); 1900 1901 holds = refcount_remove(&db->db_holds, tag); 1902 ASSERT(holds >= 0); 1903 1904 /* 1905 * We can't freeze indirects if there is a possibility that they 1906 * may be modified in the current syncing context. 1907 */ 1908 if (db->db_buf && holds == (db->db_level == 0 ? db->db_dirtycnt : 0)) 1909 arc_buf_freeze(db->db_buf); 1910 1911 if (holds == db->db_dirtycnt && 1912 db->db_level == 0 && db->db_immediate_evict) 1913 dbuf_evict_user(db); 1914 1915 if (holds == 0) { 1916 if (db->db_blkid == DMU_BONUS_BLKID) { 1917 mutex_exit(&db->db_mtx); 1918 dnode_rele(db->db_dnode, db); 1919 } else if (db->db_buf == NULL) { 1920 /* 1921 * This is a special case: we never associated this 1922 * dbuf with any data allocated from the ARC. 1923 */ 1924 ASSERT(db->db_state == DB_UNCACHED || 1925 db->db_state == DB_NOFILL); 1926 dbuf_evict(db); 1927 } else if (arc_released(db->db_buf)) { 1928 arc_buf_t *buf = db->db_buf; 1929 /* 1930 * This dbuf has anonymous data associated with it. 1931 */ 1932 dbuf_set_data(db, NULL); 1933 VERIFY(arc_buf_remove_ref(buf, db) == 1); 1934 dbuf_evict(db); 1935 } else { 1936 VERIFY(arc_buf_remove_ref(db->db_buf, db) == 0); 1937 if (!DBUF_IS_CACHEABLE(db)) 1938 dbuf_clear(db); 1939 else 1940 mutex_exit(&db->db_mtx); 1941 } 1942 } else { 1943 mutex_exit(&db->db_mtx); 1944 } 1945 } 1946 1947 #pragma weak dmu_buf_refcount = dbuf_refcount 1948 uint64_t 1949 dbuf_refcount(dmu_buf_impl_t *db) 1950 { 1951 return (refcount_count(&db->db_holds)); 1952 } 1953 1954 void * 1955 dmu_buf_set_user(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 1956 dmu_buf_evict_func_t *evict_func) 1957 { 1958 return (dmu_buf_update_user(db_fake, NULL, user_ptr, 1959 user_data_ptr_ptr, evict_func)); 1960 } 1961 1962 void * 1963 dmu_buf_set_user_ie(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 1964 dmu_buf_evict_func_t *evict_func) 1965 { 1966 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1967 1968 db->db_immediate_evict = TRUE; 1969 return (dmu_buf_update_user(db_fake, NULL, user_ptr, 1970 user_data_ptr_ptr, evict_func)); 1971 } 1972 1973 void * 1974 dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr, void *user_ptr, 1975 void *user_data_ptr_ptr, dmu_buf_evict_func_t *evict_func) 1976 { 1977 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1978 ASSERT(db->db_level == 0); 1979 1980 ASSERT((user_ptr == NULL) == (evict_func == NULL)); 1981 1982 mutex_enter(&db->db_mtx); 1983 1984 if (db->db_user_ptr == old_user_ptr) { 1985 db->db_user_ptr = user_ptr; 1986 db->db_user_data_ptr_ptr = user_data_ptr_ptr; 1987 db->db_evict_func = evict_func; 1988 1989 dbuf_update_data(db); 1990 } else { 1991 old_user_ptr = db->db_user_ptr; 1992 } 1993 1994 mutex_exit(&db->db_mtx); 1995 return (old_user_ptr); 1996 } 1997 1998 void * 1999 dmu_buf_get_user(dmu_buf_t *db_fake) 2000 { 2001 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2002 ASSERT(!refcount_is_zero(&db->db_holds)); 2003 2004 return (db->db_user_ptr); 2005 } 2006 2007 boolean_t 2008 dmu_buf_freeable(dmu_buf_t *dbuf) 2009 { 2010 boolean_t res = B_FALSE; 2011 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf; 2012 2013 if (db->db_blkptr) 2014 res = dsl_dataset_block_freeable(db->db_objset->os_dsl_dataset, 2015 db->db_blkptr->blk_birth); 2016 2017 return (res); 2018 } 2019 2020 static void 2021 dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db) 2022 { 2023 /* ASSERT(dmu_tx_is_syncing(tx) */ 2024 ASSERT(MUTEX_HELD(&db->db_mtx)); 2025 2026 if (db->db_blkptr != NULL) 2027 return; 2028 2029 if (db->db_blkid == DMU_SPILL_BLKID) { 2030 db->db_blkptr = &dn->dn_phys->dn_spill; 2031 BP_ZERO(db->db_blkptr); 2032 return; 2033 } 2034 if (db->db_level == dn->dn_phys->dn_nlevels-1) { 2035 /* 2036 * This buffer was allocated at a time when there was 2037 * no available blkptrs from the dnode, or it was 2038 * inappropriate to hook it in (i.e., nlevels mis-match). 2039 */ 2040 ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr); 2041 ASSERT(db->db_parent == NULL); 2042 db->db_parent = dn->dn_dbuf; 2043 db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid]; 2044 DBUF_VERIFY(db); 2045 } else { 2046 dmu_buf_impl_t *parent = db->db_parent; 2047 int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2048 2049 ASSERT(dn->dn_phys->dn_nlevels > 1); 2050 if (parent == NULL) { 2051 mutex_exit(&db->db_mtx); 2052 rw_enter(&dn->dn_struct_rwlock, RW_READER); 2053 (void) dbuf_hold_impl(dn, db->db_level+1, 2054 db->db_blkid >> epbs, FALSE, db, &parent); 2055 rw_exit(&dn->dn_struct_rwlock); 2056 mutex_enter(&db->db_mtx); 2057 db->db_parent = parent; 2058 } 2059 db->db_blkptr = (blkptr_t *)parent->db.db_data + 2060 (db->db_blkid & ((1ULL << epbs) - 1)); 2061 DBUF_VERIFY(db); 2062 } 2063 } 2064 2065 static void 2066 dbuf_sync_indirect(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 2067 { 2068 dmu_buf_impl_t *db = dr->dr_dbuf; 2069 dnode_t *dn = db->db_dnode; 2070 zio_t *zio; 2071 2072 ASSERT(dmu_tx_is_syncing(tx)); 2073 2074 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 2075 2076 mutex_enter(&db->db_mtx); 2077 2078 ASSERT(db->db_level > 0); 2079 DBUF_VERIFY(db); 2080 2081 if (db->db_buf == NULL) { 2082 mutex_exit(&db->db_mtx); 2083 (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED); 2084 mutex_enter(&db->db_mtx); 2085 } 2086 ASSERT3U(db->db_state, ==, DB_CACHED); 2087 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2088 ASSERT(db->db_buf != NULL); 2089 2090 dbuf_check_blkptr(dn, db); 2091 2092 db->db_data_pending = dr; 2093 2094 mutex_exit(&db->db_mtx); 2095 dbuf_write(dr, db->db_buf, tx); 2096 2097 zio = dr->dr_zio; 2098 mutex_enter(&dr->dt.di.dr_mtx); 2099 dbuf_sync_list(&dr->dt.di.dr_children, tx); 2100 ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 2101 mutex_exit(&dr->dt.di.dr_mtx); 2102 zio_nowait(zio); 2103 } 2104 2105 static void 2106 dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 2107 { 2108 arc_buf_t **datap = &dr->dt.dl.dr_data; 2109 dmu_buf_impl_t *db = dr->dr_dbuf; 2110 dnode_t *dn = db->db_dnode; 2111 objset_t *os = dn->dn_objset; 2112 uint64_t txg = tx->tx_txg; 2113 2114 ASSERT(dmu_tx_is_syncing(tx)); 2115 2116 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 2117 2118 mutex_enter(&db->db_mtx); 2119 /* 2120 * To be synced, we must be dirtied. But we 2121 * might have been freed after the dirty. 2122 */ 2123 if (db->db_state == DB_UNCACHED) { 2124 /* This buffer has been freed since it was dirtied */ 2125 ASSERT(db->db.db_data == NULL); 2126 } else if (db->db_state == DB_FILL) { 2127 /* This buffer was freed and is now being re-filled */ 2128 ASSERT(db->db.db_data != dr->dt.dl.dr_data); 2129 } else { 2130 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_NOFILL); 2131 } 2132 DBUF_VERIFY(db); 2133 2134 if (db->db_blkid == DMU_SPILL_BLKID) { 2135 mutex_enter(&dn->dn_mtx); 2136 dn->dn_phys->dn_flags |= DNODE_FLAG_SPILL_BLKPTR; 2137 mutex_exit(&dn->dn_mtx); 2138 } 2139 2140 /* 2141 * If this is a bonus buffer, simply copy the bonus data into the 2142 * dnode. It will be written out when the dnode is synced (and it 2143 * will be synced, since it must have been dirty for dbuf_sync to 2144 * be called). 2145 */ 2146 if (db->db_blkid == DMU_BONUS_BLKID) { 2147 dbuf_dirty_record_t **drp; 2148 2149 ASSERT(*datap != NULL); 2150 ASSERT3U(db->db_level, ==, 0); 2151 ASSERT3U(dn->dn_phys->dn_bonuslen, <=, DN_MAX_BONUSLEN); 2152 bcopy(*datap, DN_BONUS(dn->dn_phys), dn->dn_phys->dn_bonuslen); 2153 if (*datap != db->db.db_data) { 2154 zio_buf_free(*datap, DN_MAX_BONUSLEN); 2155 arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 2156 } 2157 db->db_data_pending = NULL; 2158 drp = &db->db_last_dirty; 2159 while (*drp != dr) 2160 drp = &(*drp)->dr_next; 2161 ASSERT(dr->dr_next == NULL); 2162 ASSERT(dr->dr_dbuf == db); 2163 *drp = dr->dr_next; 2164 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 2165 ASSERT(db->db_dirtycnt > 0); 2166 db->db_dirtycnt -= 1; 2167 dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg); 2168 return; 2169 } 2170 2171 /* 2172 * This function may have dropped the db_mtx lock allowing a dmu_sync 2173 * operation to sneak in. As a result, we need to ensure that we 2174 * don't check the dr_override_state until we have returned from 2175 * dbuf_check_blkptr. 2176 */ 2177 dbuf_check_blkptr(dn, db); 2178 2179 /* 2180 * If this buffer is in the middle of an immdiate write, 2181 * wait for the synchronous IO to complete. 2182 */ 2183 while (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC) { 2184 ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT); 2185 cv_wait(&db->db_changed, &db->db_mtx); 2186 ASSERT(dr->dt.dl.dr_override_state != DR_NOT_OVERRIDDEN); 2187 } 2188 2189 if (db->db_state != DB_NOFILL && 2190 dn->dn_object != DMU_META_DNODE_OBJECT && 2191 refcount_count(&db->db_holds) > 1 && 2192 dr->dt.dl.dr_override_state != DR_OVERRIDDEN && 2193 *datap == db->db_buf) { 2194 /* 2195 * If this buffer is currently "in use" (i.e., there 2196 * are active holds and db_data still references it), 2197 * then make a copy before we start the write so that 2198 * any modifications from the open txg will not leak 2199 * into this write. 2200 * 2201 * NOTE: this copy does not need to be made for 2202 * objects only modified in the syncing context (e.g. 2203 * DNONE_DNODE blocks). 2204 */ 2205 int blksz = arc_buf_size(*datap); 2206 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 2207 *datap = arc_buf_alloc(os->os_spa, blksz, db, type); 2208 bcopy(db->db.db_data, (*datap)->b_data, blksz); 2209 } 2210 db->db_data_pending = dr; 2211 2212 mutex_exit(&db->db_mtx); 2213 2214 dbuf_write(dr, *datap, tx); 2215 2216 ASSERT(!list_link_active(&dr->dr_dirty_node)); 2217 if (dn->dn_object == DMU_META_DNODE_OBJECT) 2218 list_insert_tail(&dn->dn_dirty_records[txg&TXG_MASK], dr); 2219 else 2220 zio_nowait(dr->dr_zio); 2221 } 2222 2223 void 2224 dbuf_sync_list(list_t *list, dmu_tx_t *tx) 2225 { 2226 dbuf_dirty_record_t *dr; 2227 2228 while (dr = list_head(list)) { 2229 if (dr->dr_zio != NULL) { 2230 /* 2231 * If we find an already initialized zio then we 2232 * are processing the meta-dnode, and we have finished. 2233 * The dbufs for all dnodes are put back on the list 2234 * during processing, so that we can zio_wait() 2235 * these IOs after initiating all child IOs. 2236 */ 2237 ASSERT3U(dr->dr_dbuf->db.db_object, ==, 2238 DMU_META_DNODE_OBJECT); 2239 break; 2240 } 2241 list_remove(list, dr); 2242 if (dr->dr_dbuf->db_level > 0) 2243 dbuf_sync_indirect(dr, tx); 2244 else 2245 dbuf_sync_leaf(dr, tx); 2246 } 2247 } 2248 2249 /* ARGSUSED */ 2250 static void 2251 dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb) 2252 { 2253 dmu_buf_impl_t *db = vdb; 2254 blkptr_t *bp = zio->io_bp; 2255 blkptr_t *bp_orig = &zio->io_bp_orig; 2256 dnode_t *dn = db->db_dnode; 2257 spa_t *spa = zio->io_spa; 2258 int64_t delta; 2259 uint64_t fill = 0; 2260 int i; 2261 2262 ASSERT(db->db_blkptr == bp); 2263 2264 delta = bp_get_dsize_sync(spa, bp) - bp_get_dsize_sync(spa, bp_orig); 2265 dnode_diduse_space(dn, delta - zio->io_prev_space_delta); 2266 zio->io_prev_space_delta = delta; 2267 2268 if (BP_IS_HOLE(bp)) { 2269 ASSERT(bp->blk_fill == 0); 2270 return; 2271 } 2272 2273 ASSERT((db->db_blkid != DMU_SPILL_BLKID && 2274 BP_GET_TYPE(bp) == dn->dn_type) || 2275 (db->db_blkid == DMU_SPILL_BLKID && 2276 BP_GET_TYPE(bp) == dn->dn_bonustype)); 2277 ASSERT(BP_GET_LEVEL(bp) == db->db_level); 2278 2279 mutex_enter(&db->db_mtx); 2280 2281 #ifdef ZFS_DEBUG 2282 if (db->db_blkid == DMU_SPILL_BLKID) { 2283 dnode_t *dn = db->db_dnode; 2284 ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); 2285 ASSERT(!(BP_IS_HOLE(db->db_blkptr)) && 2286 db->db_blkptr == &dn->dn_phys->dn_spill); 2287 } 2288 #endif 2289 2290 if (db->db_level == 0) { 2291 mutex_enter(&dn->dn_mtx); 2292 if (db->db_blkid > dn->dn_phys->dn_maxblkid && 2293 db->db_blkid != DMU_SPILL_BLKID) 2294 dn->dn_phys->dn_maxblkid = db->db_blkid; 2295 mutex_exit(&dn->dn_mtx); 2296 2297 if (dn->dn_type == DMU_OT_DNODE) { 2298 dnode_phys_t *dnp = db->db.db_data; 2299 for (i = db->db.db_size >> DNODE_SHIFT; i > 0; 2300 i--, dnp++) { 2301 if (dnp->dn_type != DMU_OT_NONE) 2302 fill++; 2303 } 2304 } else { 2305 fill = 1; 2306 } 2307 } else { 2308 blkptr_t *ibp = db->db.db_data; 2309 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2310 for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, ibp++) { 2311 if (BP_IS_HOLE(ibp)) 2312 continue; 2313 fill += ibp->blk_fill; 2314 } 2315 } 2316 2317 bp->blk_fill = fill; 2318 2319 mutex_exit(&db->db_mtx); 2320 } 2321 2322 /* ARGSUSED */ 2323 static void 2324 dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb) 2325 { 2326 dmu_buf_impl_t *db = vdb; 2327 blkptr_t *bp = zio->io_bp; 2328 blkptr_t *bp_orig = &zio->io_bp_orig; 2329 dnode_t *dn = db->db_dnode; 2330 objset_t *os = dn->dn_objset; 2331 uint64_t txg = zio->io_txg; 2332 dbuf_dirty_record_t **drp, *dr; 2333 2334 ASSERT3U(zio->io_error, ==, 0); 2335 ASSERT(db->db_blkptr == bp); 2336 2337 if (zio->io_flags & ZIO_FLAG_IO_REWRITE) { 2338 ASSERT(BP_EQUAL(bp, bp_orig)); 2339 } else { 2340 dsl_dataset_t *ds = os->os_dsl_dataset; 2341 dmu_tx_t *tx = os->os_synctx; 2342 2343 (void) dsl_dataset_block_kill(ds, bp_orig, tx, B_TRUE); 2344 dsl_dataset_block_born(ds, bp, tx); 2345 } 2346 2347 mutex_enter(&db->db_mtx); 2348 2349 DBUF_VERIFY(db); 2350 2351 drp = &db->db_last_dirty; 2352 while ((dr = *drp) != db->db_data_pending) 2353 drp = &dr->dr_next; 2354 ASSERT(!list_link_active(&dr->dr_dirty_node)); 2355 ASSERT(dr->dr_txg == txg); 2356 ASSERT(dr->dr_dbuf == db); 2357 ASSERT(dr->dr_next == NULL); 2358 *drp = dr->dr_next; 2359 2360 #ifdef ZFS_DEBUG 2361 if (db->db_blkid == DMU_SPILL_BLKID) { 2362 dnode_t *dn = db->db_dnode; 2363 ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); 2364 ASSERT(!(BP_IS_HOLE(db->db_blkptr)) && 2365 db->db_blkptr == &dn->dn_phys->dn_spill); 2366 } 2367 #endif 2368 2369 if (db->db_level == 0) { 2370 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 2371 ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN); 2372 if (db->db_state != DB_NOFILL) { 2373 if (dr->dt.dl.dr_data != db->db_buf) 2374 VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, 2375 db) == 1); 2376 else if (!arc_released(db->db_buf)) 2377 arc_set_callback(db->db_buf, dbuf_do_evict, db); 2378 } 2379 } else { 2380 ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 2381 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2382 if (!BP_IS_HOLE(db->db_blkptr)) { 2383 int epbs = 2384 dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2385 ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, 2386 db->db.db_size); 2387 ASSERT3U(dn->dn_phys->dn_maxblkid 2388 >> (db->db_level * epbs), >=, db->db_blkid); 2389 arc_set_callback(db->db_buf, dbuf_do_evict, db); 2390 } 2391 mutex_destroy(&dr->dt.di.dr_mtx); 2392 list_destroy(&dr->dt.di.dr_children); 2393 } 2394 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 2395 2396 cv_broadcast(&db->db_changed); 2397 ASSERT(db->db_dirtycnt > 0); 2398 db->db_dirtycnt -= 1; 2399 db->db_data_pending = NULL; 2400 dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg); 2401 } 2402 2403 static void 2404 dbuf_write_nofill_ready(zio_t *zio) 2405 { 2406 dbuf_write_ready(zio, NULL, zio->io_private); 2407 } 2408 2409 static void 2410 dbuf_write_nofill_done(zio_t *zio) 2411 { 2412 dbuf_write_done(zio, NULL, zio->io_private); 2413 } 2414 2415 static void 2416 dbuf_write_override_ready(zio_t *zio) 2417 { 2418 dbuf_dirty_record_t *dr = zio->io_private; 2419 dmu_buf_impl_t *db = dr->dr_dbuf; 2420 2421 dbuf_write_ready(zio, NULL, db); 2422 } 2423 2424 static void 2425 dbuf_write_override_done(zio_t *zio) 2426 { 2427 dbuf_dirty_record_t *dr = zio->io_private; 2428 dmu_buf_impl_t *db = dr->dr_dbuf; 2429 blkptr_t *obp = &dr->dt.dl.dr_overridden_by; 2430 2431 mutex_enter(&db->db_mtx); 2432 if (!BP_EQUAL(zio->io_bp, obp)) { 2433 if (!BP_IS_HOLE(obp)) 2434 dsl_free(spa_get_dsl(zio->io_spa), zio->io_txg, obp); 2435 arc_release(dr->dt.dl.dr_data, db); 2436 } 2437 mutex_exit(&db->db_mtx); 2438 2439 dbuf_write_done(zio, NULL, db); 2440 } 2441 2442 static void 2443 dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx) 2444 { 2445 dmu_buf_impl_t *db = dr->dr_dbuf; 2446 dnode_t *dn = db->db_dnode; 2447 objset_t *os = dn->dn_objset; 2448 dmu_buf_impl_t *parent = db->db_parent; 2449 uint64_t txg = tx->tx_txg; 2450 zbookmark_t zb; 2451 zio_prop_t zp; 2452 zio_t *zio; 2453 int wp_flag = 0; 2454 2455 if (db->db_state != DB_NOFILL) { 2456 if (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE) { 2457 /* 2458 * Private object buffers are released here rather 2459 * than in dbuf_dirty() since they are only modified 2460 * in the syncing context and we don't want the 2461 * overhead of making multiple copies of the data. 2462 */ 2463 if (BP_IS_HOLE(db->db_blkptr)) { 2464 arc_buf_thaw(data); 2465 } else { 2466 arc_release(data, db); 2467 } 2468 } 2469 } 2470 2471 if (parent != dn->dn_dbuf) { 2472 ASSERT(parent && parent->db_data_pending); 2473 ASSERT(db->db_level == parent->db_level-1); 2474 ASSERT(arc_released(parent->db_buf)); 2475 zio = parent->db_data_pending->dr_zio; 2476 } else { 2477 ASSERT((db->db_level == dn->dn_phys->dn_nlevels-1 && 2478 db->db_blkid != DMU_SPILL_BLKID) || 2479 (db->db_blkid == DMU_SPILL_BLKID && db->db_level == 0)); 2480 if (db->db_blkid != DMU_SPILL_BLKID) 2481 ASSERT3P(db->db_blkptr, ==, 2482 &dn->dn_phys->dn_blkptr[db->db_blkid]); 2483 zio = dn->dn_zio; 2484 } 2485 2486 ASSERT(db->db_level == 0 || data == db->db_buf); 2487 ASSERT3U(db->db_blkptr->blk_birth, <=, txg); 2488 ASSERT(zio); 2489 2490 SET_BOOKMARK(&zb, os->os_dsl_dataset ? 2491 os->os_dsl_dataset->ds_object : DMU_META_OBJSET, 2492 db->db.db_object, db->db_level, db->db_blkid); 2493 2494 if (db->db_blkid == DMU_SPILL_BLKID) 2495 wp_flag = WP_SPILL; 2496 wp_flag |= (db->db_state == DB_NOFILL) ? WP_NOFILL : 0; 2497 2498 dmu_write_policy(os, dn, db->db_level, wp_flag, &zp); 2499 2500 if (db->db_level == 0 && dr->dt.dl.dr_override_state == DR_OVERRIDDEN) { 2501 ASSERT(db->db_state != DB_NOFILL); 2502 dr->dr_zio = zio_write(zio, os->os_spa, txg, 2503 db->db_blkptr, data->b_data, arc_buf_size(data), &zp, 2504 dbuf_write_override_ready, dbuf_write_override_done, dr, 2505 ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); 2506 mutex_enter(&db->db_mtx); 2507 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 2508 zio_write_override(dr->dr_zio, &dr->dt.dl.dr_overridden_by, 2509 dr->dt.dl.dr_copies); 2510 mutex_exit(&db->db_mtx); 2511 } else if (db->db_state == DB_NOFILL) { 2512 ASSERT(zp.zp_checksum == ZIO_CHECKSUM_OFF); 2513 dr->dr_zio = zio_write(zio, os->os_spa, txg, 2514 db->db_blkptr, NULL, db->db.db_size, &zp, 2515 dbuf_write_nofill_ready, dbuf_write_nofill_done, db, 2516 ZIO_PRIORITY_ASYNC_WRITE, 2517 ZIO_FLAG_MUSTSUCCEED | ZIO_FLAG_NODATA, &zb); 2518 } else { 2519 ASSERT(arc_released(data)); 2520 dr->dr_zio = arc_write(zio, os->os_spa, txg, 2521 db->db_blkptr, data, DBUF_IS_L2CACHEABLE(db), &zp, 2522 dbuf_write_ready, dbuf_write_done, db, 2523 ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); 2524 } 2525 } 2526