1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #include <sys/zfs_context.h> 29 #include <sys/dmu.h> 30 #include <sys/dmu_impl.h> 31 #include <sys/dbuf.h> 32 #include <sys/dmu_objset.h> 33 #include <sys/dsl_dataset.h> 34 #include <sys/dsl_dir.h> 35 #include <sys/dmu_tx.h> 36 #include <sys/spa.h> 37 #include <sys/zio.h> 38 #include <sys/dmu_zfetch.h> 39 40 static void dbuf_destroy(dmu_buf_impl_t *db); 41 static int dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx); 42 static void dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, int checksum, 43 int compress, dmu_tx_t *tx); 44 static arc_done_func_t dbuf_write_ready; 45 static arc_done_func_t dbuf_write_done; 46 47 int zfs_mdcomp_disable = 0; 48 49 /* 50 * Global data structures and functions for the dbuf cache. 51 */ 52 static kmem_cache_t *dbuf_cache; 53 54 /* ARGSUSED */ 55 static int 56 dbuf_cons(void *vdb, void *unused, int kmflag) 57 { 58 dmu_buf_impl_t *db = vdb; 59 bzero(db, sizeof (dmu_buf_impl_t)); 60 61 mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL); 62 cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL); 63 refcount_create(&db->db_holds); 64 return (0); 65 } 66 67 /* ARGSUSED */ 68 static void 69 dbuf_dest(void *vdb, void *unused) 70 { 71 dmu_buf_impl_t *db = vdb; 72 mutex_destroy(&db->db_mtx); 73 cv_destroy(&db->db_changed); 74 refcount_destroy(&db->db_holds); 75 } 76 77 /* 78 * dbuf hash table routines 79 */ 80 static dbuf_hash_table_t dbuf_hash_table; 81 82 static uint64_t dbuf_hash_count; 83 84 static uint64_t 85 dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid) 86 { 87 uintptr_t osv = (uintptr_t)os; 88 uint64_t crc = -1ULL; 89 90 ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 91 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (lvl)) & 0xFF]; 92 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF]; 93 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF]; 94 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF]; 95 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 0)) & 0xFF]; 96 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 8)) & 0xFF]; 97 98 crc ^= (osv>>14) ^ (obj>>16) ^ (blkid>>16); 99 100 return (crc); 101 } 102 103 #define DBUF_HASH(os, obj, level, blkid) dbuf_hash(os, obj, level, blkid); 104 105 #define DBUF_EQUAL(dbuf, os, obj, level, blkid) \ 106 ((dbuf)->db.db_object == (obj) && \ 107 (dbuf)->db_objset == (os) && \ 108 (dbuf)->db_level == (level) && \ 109 (dbuf)->db_blkid == (blkid)) 110 111 dmu_buf_impl_t * 112 dbuf_find(dnode_t *dn, uint8_t level, uint64_t blkid) 113 { 114 dbuf_hash_table_t *h = &dbuf_hash_table; 115 objset_impl_t *os = dn->dn_objset; 116 uint64_t obj = dn->dn_object; 117 uint64_t hv = DBUF_HASH(os, obj, level, blkid); 118 uint64_t idx = hv & h->hash_table_mask; 119 dmu_buf_impl_t *db; 120 121 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 122 for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) { 123 if (DBUF_EQUAL(db, os, obj, level, blkid)) { 124 mutex_enter(&db->db_mtx); 125 if (db->db_state != DB_EVICTING) { 126 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 127 return (db); 128 } 129 mutex_exit(&db->db_mtx); 130 } 131 } 132 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 133 return (NULL); 134 } 135 136 /* 137 * Insert an entry into the hash table. If there is already an element 138 * equal to elem in the hash table, then the already existing element 139 * will be returned and the new element will not be inserted. 140 * Otherwise returns NULL. 141 */ 142 static dmu_buf_impl_t * 143 dbuf_hash_insert(dmu_buf_impl_t *db) 144 { 145 dbuf_hash_table_t *h = &dbuf_hash_table; 146 objset_impl_t *os = db->db_objset; 147 uint64_t obj = db->db.db_object; 148 int level = db->db_level; 149 uint64_t blkid = db->db_blkid; 150 uint64_t hv = DBUF_HASH(os, obj, level, blkid); 151 uint64_t idx = hv & h->hash_table_mask; 152 dmu_buf_impl_t *dbf; 153 154 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 155 for (dbf = h->hash_table[idx]; dbf != NULL; dbf = dbf->db_hash_next) { 156 if (DBUF_EQUAL(dbf, os, obj, level, blkid)) { 157 mutex_enter(&dbf->db_mtx); 158 if (dbf->db_state != DB_EVICTING) { 159 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 160 return (dbf); 161 } 162 mutex_exit(&dbf->db_mtx); 163 } 164 } 165 166 mutex_enter(&db->db_mtx); 167 db->db_hash_next = h->hash_table[idx]; 168 h->hash_table[idx] = db; 169 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 170 atomic_add_64(&dbuf_hash_count, 1); 171 172 return (NULL); 173 } 174 175 /* 176 * Remove an entry from the hash table. This operation will 177 * fail if there are any existing holds on the db. 178 */ 179 static void 180 dbuf_hash_remove(dmu_buf_impl_t *db) 181 { 182 dbuf_hash_table_t *h = &dbuf_hash_table; 183 uint64_t hv = DBUF_HASH(db->db_objset, db->db.db_object, 184 db->db_level, db->db_blkid); 185 uint64_t idx = hv & h->hash_table_mask; 186 dmu_buf_impl_t *dbf, **dbp; 187 188 /* 189 * We musn't hold db_mtx to maintin lock ordering: 190 * DBUF_HASH_MUTEX > db_mtx. 191 */ 192 ASSERT(refcount_is_zero(&db->db_holds)); 193 ASSERT(db->db_state == DB_EVICTING); 194 ASSERT(!MUTEX_HELD(&db->db_mtx)); 195 196 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 197 dbp = &h->hash_table[idx]; 198 while ((dbf = *dbp) != db) { 199 dbp = &dbf->db_hash_next; 200 ASSERT(dbf != NULL); 201 } 202 *dbp = db->db_hash_next; 203 db->db_hash_next = NULL; 204 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 205 atomic_add_64(&dbuf_hash_count, -1); 206 } 207 208 static arc_evict_func_t dbuf_do_evict; 209 210 static void 211 dbuf_evict_user(dmu_buf_impl_t *db) 212 { 213 ASSERT(MUTEX_HELD(&db->db_mtx)); 214 215 if (db->db_level != 0 || db->db_evict_func == NULL) 216 return; 217 218 if (db->db_user_data_ptr_ptr) 219 *db->db_user_data_ptr_ptr = db->db.db_data; 220 db->db_evict_func(&db->db, db->db_user_ptr); 221 db->db_user_ptr = NULL; 222 db->db_user_data_ptr_ptr = NULL; 223 db->db_evict_func = NULL; 224 } 225 226 void 227 dbuf_evict(dmu_buf_impl_t *db) 228 { 229 ASSERT(MUTEX_HELD(&db->db_mtx)); 230 ASSERT(db->db_buf == NULL); 231 ASSERT(db->db_data_pending == NULL); 232 233 dbuf_clear(db); 234 dbuf_destroy(db); 235 } 236 237 void 238 dbuf_init(void) 239 { 240 uint64_t hsize = 1ULL << 16; 241 dbuf_hash_table_t *h = &dbuf_hash_table; 242 int i; 243 244 /* 245 * The hash table is big enough to fill all of physical memory 246 * with an average 4K block size. The table will take up 247 * totalmem*sizeof(void*)/4K (i.e. 2MB/GB with 8-byte pointers). 248 */ 249 while (hsize * 4096 < physmem * PAGESIZE) 250 hsize <<= 1; 251 252 retry: 253 h->hash_table_mask = hsize - 1; 254 h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP); 255 if (h->hash_table == NULL) { 256 /* XXX - we should really return an error instead of assert */ 257 ASSERT(hsize > (1ULL << 10)); 258 hsize >>= 1; 259 goto retry; 260 } 261 262 dbuf_cache = kmem_cache_create("dmu_buf_impl_t", 263 sizeof (dmu_buf_impl_t), 264 0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0); 265 266 for (i = 0; i < DBUF_MUTEXES; i++) 267 mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL); 268 } 269 270 void 271 dbuf_fini(void) 272 { 273 dbuf_hash_table_t *h = &dbuf_hash_table; 274 int i; 275 276 for (i = 0; i < DBUF_MUTEXES; i++) 277 mutex_destroy(&h->hash_mutexes[i]); 278 kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *)); 279 kmem_cache_destroy(dbuf_cache); 280 } 281 282 /* 283 * Other stuff. 284 */ 285 286 #ifdef ZFS_DEBUG 287 static void 288 dbuf_verify(dmu_buf_impl_t *db) 289 { 290 dnode_t *dn = db->db_dnode; 291 292 ASSERT(MUTEX_HELD(&db->db_mtx)); 293 294 if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY)) 295 return; 296 297 ASSERT(db->db_objset != NULL); 298 if (dn == NULL) { 299 ASSERT(db->db_parent == NULL); 300 ASSERT(db->db_blkptr == NULL); 301 } else { 302 ASSERT3U(db->db.db_object, ==, dn->dn_object); 303 ASSERT3P(db->db_objset, ==, dn->dn_objset); 304 ASSERT3U(db->db_level, <, dn->dn_nlevels); 305 ASSERT(db->db_blkid == DB_BONUS_BLKID || 306 list_head(&dn->dn_dbufs)); 307 } 308 if (db->db_blkid == DB_BONUS_BLKID) { 309 ASSERT(dn != NULL); 310 ASSERT3U(db->db.db_size, ==, dn->dn_bonuslen); 311 ASSERT3U(db->db.db_offset, ==, DB_BONUS_BLKID); 312 } else { 313 ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size); 314 } 315 316 if (db->db_level == 0) { 317 /* we can be momentarily larger in dnode_set_blksz() */ 318 if (db->db_blkid != DB_BONUS_BLKID && dn) { 319 ASSERT3U(db->db.db_size, >=, dn->dn_datablksz); 320 } 321 if (db->db.db_object == DMU_META_DNODE_OBJECT) { 322 dbuf_dirty_record_t *dr = db->db_data_pending; 323 /* 324 * it should only be modified in syncing 325 * context, so make sure we only have 326 * one copy of the data. 327 */ 328 ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf); 329 } 330 } 331 332 /* verify db->db_blkptr */ 333 if (db->db_blkptr) { 334 if (db->db_parent == dn->dn_dbuf) { 335 /* db is pointed to by the dnode */ 336 /* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */ 337 if (db->db.db_object == DMU_META_DNODE_OBJECT) 338 ASSERT(db->db_parent == NULL); 339 else 340 ASSERT(db->db_parent != NULL); 341 ASSERT3P(db->db_blkptr, ==, 342 &dn->dn_phys->dn_blkptr[db->db_blkid]); 343 } else { 344 /* db is pointed to by an indirect block */ 345 int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT; 346 ASSERT3U(db->db_parent->db_level, ==, db->db_level+1); 347 ASSERT3U(db->db_parent->db.db_object, ==, 348 db->db.db_object); 349 /* 350 * dnode_grow_indblksz() can make this fail if we don't 351 * have the struct_rwlock. XXX indblksz no longer 352 * grows. safe to do this now? 353 */ 354 if (RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)) { 355 ASSERT3P(db->db_blkptr, ==, 356 ((blkptr_t *)db->db_parent->db.db_data + 357 db->db_blkid % epb)); 358 } 359 } 360 } 361 if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) && 362 db->db.db_data && db->db_blkid != DB_BONUS_BLKID && 363 db->db_state != DB_FILL && !dn->dn_free_txg) { 364 /* 365 * If the blkptr isn't set but they have nonzero data, 366 * it had better be dirty, otherwise we'll lose that 367 * data when we evict this buffer. 368 */ 369 if (db->db_dirtycnt == 0) { 370 uint64_t *buf = db->db.db_data; 371 int i; 372 373 for (i = 0; i < db->db.db_size >> 3; i++) { 374 ASSERT(buf[i] == 0); 375 } 376 } 377 } 378 } 379 #endif 380 381 static void 382 dbuf_update_data(dmu_buf_impl_t *db) 383 { 384 ASSERT(MUTEX_HELD(&db->db_mtx)); 385 if (db->db_level == 0 && db->db_user_data_ptr_ptr) { 386 ASSERT(!refcount_is_zero(&db->db_holds)); 387 *db->db_user_data_ptr_ptr = db->db.db_data; 388 } 389 } 390 391 static void 392 dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf) 393 { 394 ASSERT(MUTEX_HELD(&db->db_mtx)); 395 ASSERT(db->db_buf == NULL || !arc_has_callback(db->db_buf)); 396 db->db_buf = buf; 397 if (buf != NULL) { 398 ASSERT(buf->b_data != NULL); 399 db->db.db_data = buf->b_data; 400 if (!arc_released(buf)) 401 arc_set_callback(buf, dbuf_do_evict, db); 402 dbuf_update_data(db); 403 } else { 404 dbuf_evict_user(db); 405 db->db.db_data = NULL; 406 db->db_state = DB_UNCACHED; 407 } 408 } 409 410 uint64_t 411 dbuf_whichblock(dnode_t *dn, uint64_t offset) 412 { 413 if (dn->dn_datablkshift) { 414 return (offset >> dn->dn_datablkshift); 415 } else { 416 ASSERT3U(offset, <, dn->dn_datablksz); 417 return (0); 418 } 419 } 420 421 static void 422 dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb) 423 { 424 dmu_buf_impl_t *db = vdb; 425 426 mutex_enter(&db->db_mtx); 427 ASSERT3U(db->db_state, ==, DB_READ); 428 /* 429 * All reads are synchronous, so we must have a hold on the dbuf 430 */ 431 ASSERT(refcount_count(&db->db_holds) > 0); 432 ASSERT(db->db_buf == NULL); 433 ASSERT(db->db.db_data == NULL); 434 if (db->db_level == 0 && db->db_freed_in_flight) { 435 /* we were freed in flight; disregard any error */ 436 arc_release(buf, db); 437 bzero(buf->b_data, db->db.db_size); 438 arc_buf_freeze(buf); 439 db->db_freed_in_flight = FALSE; 440 dbuf_set_data(db, buf); 441 db->db_state = DB_CACHED; 442 } else if (zio == NULL || zio->io_error == 0) { 443 dbuf_set_data(db, buf); 444 db->db_state = DB_CACHED; 445 } else { 446 ASSERT(db->db_blkid != DB_BONUS_BLKID); 447 ASSERT3P(db->db_buf, ==, NULL); 448 VERIFY(arc_buf_remove_ref(buf, db) == 1); 449 db->db_state = DB_UNCACHED; 450 } 451 cv_broadcast(&db->db_changed); 452 mutex_exit(&db->db_mtx); 453 dbuf_rele(db, NULL); 454 } 455 456 static void 457 dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t *flags) 458 { 459 blkptr_t *bp; 460 zbookmark_t zb; 461 uint32_t aflags = ARC_NOWAIT; 462 463 ASSERT(!refcount_is_zero(&db->db_holds)); 464 /* We need the struct_rwlock to prevent db_blkptr from changing. */ 465 ASSERT(RW_LOCK_HELD(&db->db_dnode->dn_struct_rwlock)); 466 ASSERT(MUTEX_HELD(&db->db_mtx)); 467 ASSERT(db->db_state == DB_UNCACHED); 468 ASSERT(db->db_buf == NULL); 469 470 if (db->db_blkid == DB_BONUS_BLKID) { 471 ASSERT3U(db->db_dnode->dn_bonuslen, ==, db->db.db_size); 472 db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN); 473 arc_space_consume(DN_MAX_BONUSLEN); 474 if (db->db.db_size < DN_MAX_BONUSLEN) 475 bzero(db->db.db_data, DN_MAX_BONUSLEN); 476 bcopy(DN_BONUS(db->db_dnode->dn_phys), db->db.db_data, 477 db->db.db_size); 478 dbuf_update_data(db); 479 db->db_state = DB_CACHED; 480 mutex_exit(&db->db_mtx); 481 return; 482 } 483 484 if (db->db_level == 0 && dnode_block_freed(db->db_dnode, db->db_blkid)) 485 bp = NULL; 486 else 487 bp = db->db_blkptr; 488 489 if (bp == NULL) 490 dprintf_dbuf(db, "blkptr: %s\n", "NULL"); 491 else 492 dprintf_dbuf_bp(db, bp, "%s", "blkptr:"); 493 494 if (bp == NULL || BP_IS_HOLE(bp)) { 495 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 496 497 ASSERT(bp == NULL || BP_IS_HOLE(bp)); 498 dbuf_set_data(db, arc_buf_alloc(db->db_dnode->dn_objset->os_spa, 499 db->db.db_size, db, type)); 500 bzero(db->db.db_data, db->db.db_size); 501 db->db_state = DB_CACHED; 502 *flags |= DB_RF_CACHED; 503 mutex_exit(&db->db_mtx); 504 return; 505 } 506 507 db->db_state = DB_READ; 508 mutex_exit(&db->db_mtx); 509 510 zb.zb_objset = db->db_objset->os_dsl_dataset ? 511 db->db_objset->os_dsl_dataset->ds_object : 0; 512 zb.zb_object = db->db.db_object; 513 zb.zb_level = db->db_level; 514 zb.zb_blkid = db->db_blkid; 515 516 dbuf_add_ref(db, NULL); 517 /* ZIO_FLAG_CANFAIL callers have to check the parent zio's error */ 518 ASSERT3U(db->db_dnode->dn_type, <, DMU_OT_NUMTYPES); 519 (void) arc_read(zio, db->db_dnode->dn_objset->os_spa, bp, 520 db->db_level > 0 ? byteswap_uint64_array : 521 dmu_ot[db->db_dnode->dn_type].ot_byteswap, 522 dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ, 523 (*flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED, 524 &aflags, &zb); 525 if (aflags & ARC_CACHED) 526 *flags |= DB_RF_CACHED; 527 } 528 529 int 530 dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags) 531 { 532 int err = 0; 533 int havepzio = (zio != NULL); 534 int prefetch; 535 536 /* 537 * We don't have to hold the mutex to check db_state because it 538 * can't be freed while we have a hold on the buffer. 539 */ 540 ASSERT(!refcount_is_zero(&db->db_holds)); 541 542 if ((flags & DB_RF_HAVESTRUCT) == 0) 543 rw_enter(&db->db_dnode->dn_struct_rwlock, RW_READER); 544 545 prefetch = db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID && 546 (flags & DB_RF_NOPREFETCH) == 0 && db->db_dnode != NULL; 547 548 mutex_enter(&db->db_mtx); 549 if (db->db_state == DB_CACHED) { 550 mutex_exit(&db->db_mtx); 551 if (prefetch) 552 dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset, 553 db->db.db_size, TRUE); 554 if ((flags & DB_RF_HAVESTRUCT) == 0) 555 rw_exit(&db->db_dnode->dn_struct_rwlock); 556 } else if (db->db_state == DB_UNCACHED) { 557 if (zio == NULL) { 558 zio = zio_root(db->db_dnode->dn_objset->os_spa, 559 NULL, NULL, ZIO_FLAG_CANFAIL); 560 } 561 dbuf_read_impl(db, zio, &flags); 562 563 /* dbuf_read_impl has dropped db_mtx for us */ 564 565 if (prefetch) 566 dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset, 567 db->db.db_size, flags & DB_RF_CACHED); 568 569 if ((flags & DB_RF_HAVESTRUCT) == 0) 570 rw_exit(&db->db_dnode->dn_struct_rwlock); 571 572 if (!havepzio) 573 err = zio_wait(zio); 574 } else { 575 mutex_exit(&db->db_mtx); 576 if (prefetch) 577 dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset, 578 db->db.db_size, TRUE); 579 if ((flags & DB_RF_HAVESTRUCT) == 0) 580 rw_exit(&db->db_dnode->dn_struct_rwlock); 581 582 mutex_enter(&db->db_mtx); 583 if ((flags & DB_RF_NEVERWAIT) == 0) { 584 while (db->db_state == DB_READ || 585 db->db_state == DB_FILL) { 586 ASSERT(db->db_state == DB_READ || 587 (flags & DB_RF_HAVESTRUCT) == 0); 588 cv_wait(&db->db_changed, &db->db_mtx); 589 } 590 if (db->db_state == DB_UNCACHED) 591 err = EIO; 592 } 593 mutex_exit(&db->db_mtx); 594 } 595 596 ASSERT(err || havepzio || db->db_state == DB_CACHED); 597 return (err); 598 } 599 600 static void 601 dbuf_noread(dmu_buf_impl_t *db) 602 { 603 ASSERT(!refcount_is_zero(&db->db_holds)); 604 ASSERT(db->db_blkid != DB_BONUS_BLKID); 605 mutex_enter(&db->db_mtx); 606 while (db->db_state == DB_READ || db->db_state == DB_FILL) 607 cv_wait(&db->db_changed, &db->db_mtx); 608 if (db->db_state == DB_UNCACHED) { 609 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 610 611 ASSERT(db->db_buf == NULL); 612 ASSERT(db->db.db_data == NULL); 613 dbuf_set_data(db, arc_buf_alloc(db->db_dnode->dn_objset->os_spa, 614 db->db.db_size, db, type)); 615 db->db_state = DB_FILL; 616 } else { 617 ASSERT3U(db->db_state, ==, DB_CACHED); 618 } 619 mutex_exit(&db->db_mtx); 620 } 621 622 /* 623 * This is our just-in-time copy function. It makes a copy of 624 * buffers, that have been modified in a previous transaction 625 * group, before we modify them in the current active group. 626 * 627 * This function is used in two places: when we are dirtying a 628 * buffer for the first time in a txg, and when we are freeing 629 * a range in a dnode that includes this buffer. 630 * 631 * Note that when we are called from dbuf_free_range() we do 632 * not put a hold on the buffer, we just traverse the active 633 * dbuf list for the dnode. 634 */ 635 static void 636 dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg) 637 { 638 dbuf_dirty_record_t *dr = db->db_last_dirty; 639 640 ASSERT(MUTEX_HELD(&db->db_mtx)); 641 ASSERT(db->db.db_data != NULL); 642 ASSERT(db->db_level == 0); 643 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT); 644 645 if (dr == NULL || 646 (dr->dt.dl.dr_data != 647 ((db->db_blkid == DB_BONUS_BLKID) ? db->db.db_data : db->db_buf))) 648 return; 649 650 /* 651 * If the last dirty record for this dbuf has not yet synced 652 * and its referencing the dbuf data, either: 653 * reset the reference to point to a new copy, 654 * or (if there a no active holders) 655 * just null out the current db_data pointer. 656 */ 657 ASSERT(dr->dr_txg >= txg - 2); 658 if (db->db_blkid == DB_BONUS_BLKID) { 659 /* Note that the data bufs here are zio_bufs */ 660 dr->dt.dl.dr_data = zio_buf_alloc(DN_MAX_BONUSLEN); 661 arc_space_consume(DN_MAX_BONUSLEN); 662 bcopy(db->db.db_data, dr->dt.dl.dr_data, DN_MAX_BONUSLEN); 663 } else if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 664 int size = db->db.db_size; 665 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 666 dr->dt.dl.dr_data = arc_buf_alloc( 667 db->db_dnode->dn_objset->os_spa, size, db, type); 668 bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size); 669 } else { 670 dbuf_set_data(db, NULL); 671 } 672 } 673 674 void 675 dbuf_unoverride(dbuf_dirty_record_t *dr) 676 { 677 dmu_buf_impl_t *db = dr->dr_dbuf; 678 uint64_t txg = dr->dr_txg; 679 680 ASSERT(MUTEX_HELD(&db->db_mtx)); 681 ASSERT(dr->dt.dl.dr_override_state != DR_IN_DMU_SYNC); 682 ASSERT(db->db_level == 0); 683 684 if (db->db_blkid == DB_BONUS_BLKID || 685 dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN) 686 return; 687 688 /* free this block */ 689 if (!BP_IS_HOLE(&dr->dt.dl.dr_overridden_by)) { 690 /* XXX can get silent EIO here */ 691 (void) arc_free(NULL, db->db_dnode->dn_objset->os_spa, 692 txg, &dr->dt.dl.dr_overridden_by, NULL, NULL, ARC_WAIT); 693 } 694 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 695 /* 696 * Release the already-written buffer, so we leave it in 697 * a consistent dirty state. Note that all callers are 698 * modifying the buffer, so they will immediately do 699 * another (redundant) arc_release(). Therefore, leave 700 * the buf thawed to save the effort of freezing & 701 * immediately re-thawing it. 702 */ 703 arc_release(dr->dt.dl.dr_data, db); 704 } 705 706 void 707 dbuf_free_range(dnode_t *dn, uint64_t blkid, uint64_t nblks, dmu_tx_t *tx) 708 { 709 dmu_buf_impl_t *db, *db_next; 710 uint64_t txg = tx->tx_txg; 711 712 dprintf_dnode(dn, "blkid=%llu nblks=%llu\n", blkid, nblks); 713 mutex_enter(&dn->dn_dbufs_mtx); 714 for (db = list_head(&dn->dn_dbufs); db; db = db_next) { 715 db_next = list_next(&dn->dn_dbufs, db); 716 ASSERT(db->db_blkid != DB_BONUS_BLKID); 717 if (db->db_level != 0) 718 continue; 719 dprintf_dbuf(db, "found buf %s\n", ""); 720 if (db->db_blkid < blkid || 721 db->db_blkid >= blkid+nblks) 722 continue; 723 724 /* found a level 0 buffer in the range */ 725 if (dbuf_undirty(db, tx)) 726 continue; 727 728 mutex_enter(&db->db_mtx); 729 if (db->db_state == DB_UNCACHED || 730 db->db_state == DB_EVICTING) { 731 ASSERT(db->db.db_data == NULL); 732 mutex_exit(&db->db_mtx); 733 continue; 734 } 735 if (db->db_state == DB_READ || db->db_state == DB_FILL) { 736 /* will be handled in dbuf_read_done or dbuf_rele */ 737 db->db_freed_in_flight = TRUE; 738 mutex_exit(&db->db_mtx); 739 continue; 740 } 741 if (refcount_count(&db->db_holds) == 0) { 742 ASSERT(db->db_buf); 743 dbuf_clear(db); 744 continue; 745 } 746 /* The dbuf is referenced */ 747 748 if (db->db_last_dirty != NULL) { 749 dbuf_dirty_record_t *dr = db->db_last_dirty; 750 751 if (dr->dr_txg == txg) { 752 /* 753 * This buffer is "in-use", re-adjust the file 754 * size to reflect that this buffer may 755 * contain new data when we sync. 756 */ 757 if (db->db_blkid > dn->dn_maxblkid) 758 dn->dn_maxblkid = db->db_blkid; 759 dbuf_unoverride(dr); 760 } else { 761 /* 762 * This dbuf is not dirty in the open context. 763 * Either uncache it (if its not referenced in 764 * the open context) or reset its contents to 765 * empty. 766 */ 767 dbuf_fix_old_data(db, txg); 768 } 769 } 770 /* clear the contents if its cached */ 771 if (db->db_state == DB_CACHED) { 772 ASSERT(db->db.db_data != NULL); 773 arc_release(db->db_buf, db); 774 bzero(db->db.db_data, db->db.db_size); 775 arc_buf_freeze(db->db_buf); 776 } 777 778 mutex_exit(&db->db_mtx); 779 } 780 mutex_exit(&dn->dn_dbufs_mtx); 781 } 782 783 static int 784 dbuf_new_block(dmu_buf_impl_t *db) 785 { 786 dsl_dataset_t *ds = db->db_objset->os_dsl_dataset; 787 uint64_t birth_txg = 0; 788 789 /* Don't count meta-objects */ 790 if (ds == NULL) 791 return (FALSE); 792 793 /* 794 * We don't need any locking to protect db_blkptr: 795 * If it's syncing, then db_last_dirty will be set 796 * so we'll ignore db_blkptr. 797 */ 798 ASSERT(MUTEX_HELD(&db->db_mtx)); 799 /* If we have been dirtied since the last snapshot, its not new */ 800 if (db->db_last_dirty) 801 birth_txg = db->db_last_dirty->dr_txg; 802 else if (db->db_blkptr) 803 birth_txg = db->db_blkptr->blk_birth; 804 805 if (birth_txg) 806 return (!dsl_dataset_block_freeable(ds, birth_txg)); 807 else 808 return (TRUE); 809 } 810 811 void 812 dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx) 813 { 814 arc_buf_t *buf, *obuf; 815 int osize = db->db.db_size; 816 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 817 818 ASSERT(db->db_blkid != DB_BONUS_BLKID); 819 820 /* XXX does *this* func really need the lock? */ 821 ASSERT(RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)); 822 823 /* 824 * This call to dbuf_will_dirty() with the dn_struct_rwlock held 825 * is OK, because there can be no other references to the db 826 * when we are changing its size, so no concurrent DB_FILL can 827 * be happening. 828 */ 829 /* 830 * XXX we should be doing a dbuf_read, checking the return 831 * value and returning that up to our callers 832 */ 833 dbuf_will_dirty(db, tx); 834 835 /* create the data buffer for the new block */ 836 buf = arc_buf_alloc(db->db_dnode->dn_objset->os_spa, size, db, type); 837 838 /* copy old block data to the new block */ 839 obuf = db->db_buf; 840 bcopy(obuf->b_data, buf->b_data, MIN(osize, size)); 841 /* zero the remainder */ 842 if (size > osize) 843 bzero((uint8_t *)buf->b_data + osize, size - osize); 844 845 mutex_enter(&db->db_mtx); 846 dbuf_set_data(db, buf); 847 VERIFY(arc_buf_remove_ref(obuf, db) == 1); 848 db->db.db_size = size; 849 850 if (db->db_level == 0) { 851 ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg); 852 db->db_last_dirty->dt.dl.dr_data = buf; 853 } 854 mutex_exit(&db->db_mtx); 855 856 dnode_willuse_space(db->db_dnode, size-osize, tx); 857 } 858 859 dbuf_dirty_record_t * 860 dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 861 { 862 dnode_t *dn = db->db_dnode; 863 objset_impl_t *os = dn->dn_objset; 864 dbuf_dirty_record_t **drp, *dr; 865 int drop_struct_lock = FALSE; 866 int txgoff = tx->tx_txg & TXG_MASK; 867 868 ASSERT(tx->tx_txg != 0); 869 ASSERT(!refcount_is_zero(&db->db_holds)); 870 DMU_TX_DIRTY_BUF(tx, db); 871 872 /* 873 * Shouldn't dirty a regular buffer in syncing context. Private 874 * objects may be dirtied in syncing context, but only if they 875 * were already pre-dirtied in open context. 876 * XXX We may want to prohibit dirtying in syncing context even 877 * if they did pre-dirty. 878 */ 879 ASSERT(!dmu_tx_is_syncing(tx) || 880 BP_IS_HOLE(dn->dn_objset->os_rootbp) || 881 dn->dn_object == DMU_META_DNODE_OBJECT || 882 dn->dn_objset->os_dsl_dataset == NULL || 883 dsl_dir_is_private(dn->dn_objset->os_dsl_dataset->ds_dir)); 884 885 /* 886 * We make this assert for private objects as well, but after we 887 * check if we're already dirty. They are allowed to re-dirty 888 * in syncing context. 889 */ 890 ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 891 dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 892 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 893 894 mutex_enter(&db->db_mtx); 895 /* 896 * XXX make this true for indirects too? The problem is that 897 * transactions created with dmu_tx_create_assigned() from 898 * syncing context don't bother holding ahead. 899 */ 900 ASSERT(db->db_level != 0 || 901 db->db_state == DB_CACHED || db->db_state == DB_FILL); 902 903 mutex_enter(&dn->dn_mtx); 904 /* 905 * Don't set dirtyctx to SYNC if we're just modifying this as we 906 * initialize the objset. 907 */ 908 if (dn->dn_dirtyctx == DN_UNDIRTIED && 909 !BP_IS_HOLE(dn->dn_objset->os_rootbp)) { 910 dn->dn_dirtyctx = 911 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN); 912 ASSERT(dn->dn_dirtyctx_firstset == NULL); 913 dn->dn_dirtyctx_firstset = kmem_alloc(1, KM_SLEEP); 914 } 915 mutex_exit(&dn->dn_mtx); 916 917 /* 918 * If this buffer is already dirty, we're done. 919 */ 920 drp = &db->db_last_dirty; 921 ASSERT(*drp == NULL || (*drp)->dr_txg <= tx->tx_txg || 922 db->db.db_object == DMU_META_DNODE_OBJECT); 923 while (*drp && (*drp)->dr_txg > tx->tx_txg) 924 drp = &(*drp)->dr_next; 925 if (*drp && (*drp)->dr_txg == tx->tx_txg) { 926 if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID) { 927 /* 928 * If this buffer has already been written out, 929 * we now need to reset its state. 930 */ 931 dbuf_unoverride(*drp); 932 if (db->db.db_object != DMU_META_DNODE_OBJECT) 933 arc_buf_thaw(db->db_buf); 934 } 935 mutex_exit(&db->db_mtx); 936 return (*drp); 937 } 938 939 /* 940 * Only valid if not already dirty. 941 */ 942 ASSERT(dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 943 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 944 945 ASSERT3U(dn->dn_nlevels, >, db->db_level); 946 ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) || 947 dn->dn_phys->dn_nlevels > db->db_level || 948 dn->dn_next_nlevels[txgoff] > db->db_level || 949 dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level || 950 dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level); 951 952 /* 953 * We should only be dirtying in syncing context if it's the 954 * mos, a spa os, or we're initializing the os. However, we are 955 * allowed to dirty in syncing context provided we already 956 * dirtied it in open context. Hence we must make this 957 * assertion only if we're not already dirty. 958 */ 959 ASSERT(!dmu_tx_is_syncing(tx) || 960 os->os_dsl_dataset == NULL || 961 !dsl_dir_is_private(os->os_dsl_dataset->ds_dir) || 962 !BP_IS_HOLE(os->os_rootbp)); 963 ASSERT(db->db.db_size != 0); 964 965 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 966 967 /* 968 * If this buffer is dirty in an old transaction group we need 969 * to make a copy of it so that the changes we make in this 970 * transaction group won't leak out when we sync the older txg. 971 */ 972 dr = kmem_zalloc(sizeof (dbuf_dirty_record_t), KM_SLEEP); 973 if (db->db_level == 0) { 974 void *data_old = db->db_buf; 975 976 if (db->db_blkid == DB_BONUS_BLKID) { 977 dbuf_fix_old_data(db, tx->tx_txg); 978 data_old = db->db.db_data; 979 } else if (db->db.db_object != DMU_META_DNODE_OBJECT) { 980 /* 981 * Release the data buffer from the cache so that we 982 * can modify it without impacting possible other users 983 * of this cached data block. Note that indirect 984 * blocks and private objects are not released until the 985 * syncing state (since they are only modified then). 986 */ 987 arc_release(db->db_buf, db); 988 dbuf_fix_old_data(db, tx->tx_txg); 989 data_old = db->db_buf; 990 } 991 ASSERT(data_old != NULL); 992 dr->dt.dl.dr_data = data_old; 993 } else { 994 mutex_init(&dr->dt.di.dr_mtx, NULL, MUTEX_DEFAULT, NULL); 995 list_create(&dr->dt.di.dr_children, 996 sizeof (dbuf_dirty_record_t), 997 offsetof(dbuf_dirty_record_t, dr_dirty_node)); 998 } 999 dr->dr_dbuf = db; 1000 dr->dr_txg = tx->tx_txg; 1001 dr->dr_next = *drp; 1002 *drp = dr; 1003 1004 /* 1005 * We could have been freed_in_flight between the dbuf_noread 1006 * and dbuf_dirty. We win, as though the dbuf_noread() had 1007 * happened after the free. 1008 */ 1009 if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID) { 1010 mutex_enter(&dn->dn_mtx); 1011 dnode_clear_range(dn, db->db_blkid, 1, tx); 1012 mutex_exit(&dn->dn_mtx); 1013 db->db_freed_in_flight = FALSE; 1014 } 1015 1016 if (db->db_blkid != DB_BONUS_BLKID) { 1017 /* 1018 * Update the accounting. 1019 */ 1020 if (!dbuf_new_block(db) && db->db_blkptr) { 1021 /* 1022 * This is only a guess -- if the dbuf is dirty 1023 * in a previous txg, we don't know how much 1024 * space it will use on disk yet. We should 1025 * really have the struct_rwlock to access 1026 * db_blkptr, but since this is just a guess, 1027 * it's OK if we get an odd answer. 1028 */ 1029 dnode_willuse_space(dn, 1030 -bp_get_dasize(os->os_spa, db->db_blkptr), tx); 1031 } 1032 dnode_willuse_space(dn, db->db.db_size, tx); 1033 } 1034 1035 /* 1036 * This buffer is now part of this txg 1037 */ 1038 dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg); 1039 db->db_dirtycnt += 1; 1040 ASSERT3U(db->db_dirtycnt, <=, 3); 1041 1042 mutex_exit(&db->db_mtx); 1043 1044 if (db->db_blkid == DB_BONUS_BLKID) { 1045 mutex_enter(&dn->dn_mtx); 1046 ASSERT(!list_link_active(&dr->dr_dirty_node)); 1047 list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1048 mutex_exit(&dn->dn_mtx); 1049 dnode_setdirty(dn, tx); 1050 return (dr); 1051 } 1052 1053 if (db->db_level == 0) { 1054 dnode_new_blkid(dn, db->db_blkid, tx); 1055 ASSERT(dn->dn_maxblkid >= db->db_blkid); 1056 } 1057 1058 if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 1059 rw_enter(&dn->dn_struct_rwlock, RW_READER); 1060 drop_struct_lock = TRUE; 1061 } 1062 1063 if (db->db_level+1 < dn->dn_nlevels) { 1064 dmu_buf_impl_t *parent = db->db_parent; 1065 dbuf_dirty_record_t *di; 1066 int parent_held = FALSE; 1067 1068 if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) { 1069 int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1070 1071 parent = dbuf_hold_level(dn, db->db_level+1, 1072 db->db_blkid >> epbs, FTAG); 1073 parent_held = TRUE; 1074 } 1075 if (drop_struct_lock) 1076 rw_exit(&dn->dn_struct_rwlock); 1077 ASSERT3U(db->db_level+1, ==, parent->db_level); 1078 di = dbuf_dirty(parent, tx); 1079 if (parent_held) 1080 dbuf_rele(parent, FTAG); 1081 1082 mutex_enter(&db->db_mtx); 1083 /* possible race with dbuf_undirty() */ 1084 if (db->db_last_dirty == dr || 1085 dn->dn_object == DMU_META_DNODE_OBJECT) { 1086 mutex_enter(&di->dt.di.dr_mtx); 1087 ASSERT3U(di->dr_txg, ==, tx->tx_txg); 1088 ASSERT(!list_link_active(&dr->dr_dirty_node)); 1089 list_insert_tail(&di->dt.di.dr_children, dr); 1090 mutex_exit(&di->dt.di.dr_mtx); 1091 dr->dr_parent = di; 1092 } 1093 mutex_exit(&db->db_mtx); 1094 } else { 1095 ASSERT(db->db_level+1 == dn->dn_nlevels); 1096 ASSERT(db->db_blkid < dn->dn_nblkptr); 1097 ASSERT(db->db_parent == NULL || 1098 db->db_parent == db->db_dnode->dn_dbuf); 1099 mutex_enter(&dn->dn_mtx); 1100 ASSERT(!list_link_active(&dr->dr_dirty_node)); 1101 list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1102 mutex_exit(&dn->dn_mtx); 1103 if (drop_struct_lock) 1104 rw_exit(&dn->dn_struct_rwlock); 1105 } 1106 1107 dnode_setdirty(dn, tx); 1108 return (dr); 1109 } 1110 1111 static int 1112 dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1113 { 1114 dnode_t *dn = db->db_dnode; 1115 uint64_t txg = tx->tx_txg; 1116 dbuf_dirty_record_t *dr; 1117 1118 ASSERT(txg != 0); 1119 ASSERT(db->db_blkid != DB_BONUS_BLKID); 1120 1121 mutex_enter(&db->db_mtx); 1122 1123 /* 1124 * If this buffer is not dirty, we're done. 1125 */ 1126 for (dr = db->db_last_dirty; dr; dr = dr->dr_next) 1127 if (dr->dr_txg <= txg) 1128 break; 1129 if (dr == NULL || dr->dr_txg < txg) { 1130 mutex_exit(&db->db_mtx); 1131 return (0); 1132 } 1133 ASSERT(dr->dr_txg == txg); 1134 1135 /* 1136 * If this buffer is currently held, we cannot undirty 1137 * it, since one of the current holders may be in the 1138 * middle of an update. Note that users of dbuf_undirty() 1139 * should not place a hold on the dbuf before the call. 1140 */ 1141 if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 1142 mutex_exit(&db->db_mtx); 1143 /* Make sure we don't toss this buffer at sync phase */ 1144 mutex_enter(&dn->dn_mtx); 1145 dnode_clear_range(dn, db->db_blkid, 1, tx); 1146 mutex_exit(&dn->dn_mtx); 1147 return (0); 1148 } 1149 1150 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1151 1152 ASSERT(db->db.db_size != 0); 1153 1154 /* XXX would be nice to fix up dn_towrite_space[] */ 1155 1156 db->db_last_dirty = dr->dr_next; 1157 1158 if (dr->dr_parent) { 1159 mutex_enter(&dr->dr_parent->dt.di.dr_mtx); 1160 list_remove(&dr->dr_parent->dt.di.dr_children, dr); 1161 mutex_exit(&dr->dr_parent->dt.di.dr_mtx); 1162 } else if (db->db_level+1 == dn->dn_nlevels) { 1163 ASSERT3P(db->db_parent, ==, dn->dn_dbuf); 1164 mutex_enter(&dn->dn_mtx); 1165 list_remove(&dn->dn_dirty_records[txg & TXG_MASK], dr); 1166 mutex_exit(&dn->dn_mtx); 1167 } 1168 1169 if (db->db_level == 0) { 1170 dbuf_unoverride(dr); 1171 1172 ASSERT(db->db_buf != NULL); 1173 ASSERT(dr->dt.dl.dr_data != NULL); 1174 if (dr->dt.dl.dr_data != db->db_buf) 1175 VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, db) == 1); 1176 } else { 1177 ASSERT(db->db_buf != NULL); 1178 ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 1179 mutex_destroy(&dr->dt.di.dr_mtx); 1180 list_destroy(&dr->dt.di.dr_children); 1181 } 1182 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 1183 1184 ASSERT(db->db_dirtycnt > 0); 1185 db->db_dirtycnt -= 1; 1186 1187 if (refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) { 1188 arc_buf_t *buf = db->db_buf; 1189 1190 ASSERT(arc_released(buf)); 1191 dbuf_set_data(db, NULL); 1192 VERIFY(arc_buf_remove_ref(buf, db) == 1); 1193 dbuf_evict(db); 1194 return (1); 1195 } 1196 1197 mutex_exit(&db->db_mtx); 1198 return (0); 1199 } 1200 1201 #pragma weak dmu_buf_will_dirty = dbuf_will_dirty 1202 void 1203 dbuf_will_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1204 { 1205 int rf = DB_RF_MUST_SUCCEED; 1206 1207 ASSERT(tx->tx_txg != 0); 1208 ASSERT(!refcount_is_zero(&db->db_holds)); 1209 1210 if (RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)) 1211 rf |= DB_RF_HAVESTRUCT; 1212 (void) dbuf_read(db, NULL, rf); 1213 (void) dbuf_dirty(db, tx); 1214 } 1215 1216 void 1217 dmu_buf_will_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 1218 { 1219 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1220 1221 ASSERT(db->db_blkid != DB_BONUS_BLKID); 1222 ASSERT(tx->tx_txg != 0); 1223 ASSERT(db->db_level == 0); 1224 ASSERT(!refcount_is_zero(&db->db_holds)); 1225 1226 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT || 1227 dmu_tx_private_ok(tx)); 1228 1229 dbuf_noread(db); 1230 (void) dbuf_dirty(db, tx); 1231 } 1232 1233 #pragma weak dmu_buf_fill_done = dbuf_fill_done 1234 /* ARGSUSED */ 1235 void 1236 dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx) 1237 { 1238 mutex_enter(&db->db_mtx); 1239 DBUF_VERIFY(db); 1240 1241 if (db->db_state == DB_FILL) { 1242 if (db->db_level == 0 && db->db_freed_in_flight) { 1243 ASSERT(db->db_blkid != DB_BONUS_BLKID); 1244 /* we were freed while filling */ 1245 /* XXX dbuf_undirty? */ 1246 bzero(db->db.db_data, db->db.db_size); 1247 db->db_freed_in_flight = FALSE; 1248 } 1249 db->db_state = DB_CACHED; 1250 cv_broadcast(&db->db_changed); 1251 } 1252 mutex_exit(&db->db_mtx); 1253 } 1254 1255 /* 1256 * "Clear" the contents of this dbuf. This will mark the dbuf 1257 * EVICTING and clear *most* of its references. Unfortunetely, 1258 * when we are not holding the dn_dbufs_mtx, we can't clear the 1259 * entry in the dn_dbufs list. We have to wait until dbuf_destroy() 1260 * in this case. For callers from the DMU we will usually see: 1261 * dbuf_clear()->arc_buf_evict()->dbuf_do_evict()->dbuf_destroy() 1262 * For the arc callback, we will usually see: 1263 * dbuf_do_evict()->dbuf_clear();dbuf_destroy() 1264 * Sometimes, though, we will get a mix of these two: 1265 * DMU: dbuf_clear()->arc_buf_evict() 1266 * ARC: dbuf_do_evict()->dbuf_destroy() 1267 */ 1268 void 1269 dbuf_clear(dmu_buf_impl_t *db) 1270 { 1271 dnode_t *dn = db->db_dnode; 1272 dmu_buf_impl_t *parent = db->db_parent; 1273 dmu_buf_impl_t *dndb = dn->dn_dbuf; 1274 int dbuf_gone = FALSE; 1275 1276 ASSERT(MUTEX_HELD(&db->db_mtx)); 1277 ASSERT(refcount_is_zero(&db->db_holds)); 1278 1279 dbuf_evict_user(db); 1280 1281 if (db->db_state == DB_CACHED) { 1282 ASSERT(db->db.db_data != NULL); 1283 if (db->db_blkid == DB_BONUS_BLKID) { 1284 zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN); 1285 arc_space_return(DN_MAX_BONUSLEN); 1286 } 1287 db->db.db_data = NULL; 1288 db->db_state = DB_UNCACHED; 1289 } 1290 1291 ASSERT3U(db->db_state, ==, DB_UNCACHED); 1292 ASSERT(db->db_data_pending == NULL); 1293 1294 db->db_state = DB_EVICTING; 1295 db->db_blkptr = NULL; 1296 1297 if (db->db_blkid != DB_BONUS_BLKID && MUTEX_HELD(&dn->dn_dbufs_mtx)) { 1298 list_remove(&dn->dn_dbufs, db); 1299 dnode_rele(dn, db); 1300 } 1301 1302 if (db->db_buf) 1303 dbuf_gone = arc_buf_evict(db->db_buf); 1304 1305 if (!dbuf_gone) 1306 mutex_exit(&db->db_mtx); 1307 1308 /* 1309 * If this dbuf is referened from an indirect dbuf, 1310 * decrement the ref count on the indirect dbuf. 1311 */ 1312 if (parent && parent != dndb) 1313 dbuf_rele(parent, db); 1314 } 1315 1316 static int 1317 dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse, 1318 dmu_buf_impl_t **parentp, blkptr_t **bpp) 1319 { 1320 int nlevels, epbs; 1321 1322 *parentp = NULL; 1323 *bpp = NULL; 1324 1325 ASSERT(blkid != DB_BONUS_BLKID); 1326 1327 if (dn->dn_phys->dn_nlevels == 0) 1328 nlevels = 1; 1329 else 1330 nlevels = dn->dn_phys->dn_nlevels; 1331 1332 epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1333 1334 ASSERT3U(level * epbs, <, 64); 1335 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1336 if (level >= nlevels || 1337 (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) { 1338 /* the buffer has no parent yet */ 1339 return (ENOENT); 1340 } else if (level < nlevels-1) { 1341 /* this block is referenced from an indirect block */ 1342 int err = dbuf_hold_impl(dn, level+1, 1343 blkid >> epbs, fail_sparse, NULL, parentp); 1344 if (err) 1345 return (err); 1346 err = dbuf_read(*parentp, NULL, 1347 (DB_RF_HAVESTRUCT | DB_RF_NOPREFETCH | DB_RF_CANFAIL)); 1348 if (err) { 1349 dbuf_rele(*parentp, NULL); 1350 *parentp = NULL; 1351 return (err); 1352 } 1353 *bpp = ((blkptr_t *)(*parentp)->db.db_data) + 1354 (blkid & ((1ULL << epbs) - 1)); 1355 return (0); 1356 } else { 1357 /* the block is referenced from the dnode */ 1358 ASSERT3U(level, ==, nlevels-1); 1359 ASSERT(dn->dn_phys->dn_nblkptr == 0 || 1360 blkid < dn->dn_phys->dn_nblkptr); 1361 if (dn->dn_dbuf) { 1362 dbuf_add_ref(dn->dn_dbuf, NULL); 1363 *parentp = dn->dn_dbuf; 1364 } 1365 *bpp = &dn->dn_phys->dn_blkptr[blkid]; 1366 return (0); 1367 } 1368 } 1369 1370 static dmu_buf_impl_t * 1371 dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid, 1372 dmu_buf_impl_t *parent, blkptr_t *blkptr) 1373 { 1374 objset_impl_t *os = dn->dn_objset; 1375 dmu_buf_impl_t *db, *odb; 1376 1377 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1378 ASSERT(dn->dn_type != DMU_OT_NONE); 1379 1380 db = kmem_cache_alloc(dbuf_cache, KM_SLEEP); 1381 1382 db->db_objset = os; 1383 db->db.db_object = dn->dn_object; 1384 db->db_level = level; 1385 db->db_blkid = blkid; 1386 db->db_last_dirty = NULL; 1387 db->db_dirtycnt = 0; 1388 db->db_dnode = dn; 1389 db->db_parent = parent; 1390 db->db_blkptr = blkptr; 1391 1392 db->db_user_ptr = NULL; 1393 db->db_user_data_ptr_ptr = NULL; 1394 db->db_evict_func = NULL; 1395 db->db_immediate_evict = 0; 1396 db->db_freed_in_flight = 0; 1397 1398 if (blkid == DB_BONUS_BLKID) { 1399 ASSERT3P(parent, ==, dn->dn_dbuf); 1400 db->db.db_size = dn->dn_bonuslen; 1401 db->db.db_offset = DB_BONUS_BLKID; 1402 db->db_state = DB_UNCACHED; 1403 /* the bonus dbuf is not placed in the hash table */ 1404 arc_space_consume(sizeof (dmu_buf_impl_t)); 1405 return (db); 1406 } else { 1407 int blocksize = 1408 db->db_level ? 1<<dn->dn_indblkshift : dn->dn_datablksz; 1409 db->db.db_size = blocksize; 1410 db->db.db_offset = db->db_blkid * blocksize; 1411 } 1412 1413 /* 1414 * Hold the dn_dbufs_mtx while we get the new dbuf 1415 * in the hash table *and* added to the dbufs list. 1416 * This prevents a possible deadlock with someone 1417 * trying to look up this dbuf before its added to the 1418 * dn_dbufs list. 1419 */ 1420 mutex_enter(&dn->dn_dbufs_mtx); 1421 db->db_state = DB_EVICTING; 1422 if ((odb = dbuf_hash_insert(db)) != NULL) { 1423 /* someone else inserted it first */ 1424 kmem_cache_free(dbuf_cache, db); 1425 mutex_exit(&dn->dn_dbufs_mtx); 1426 return (odb); 1427 } 1428 list_insert_head(&dn->dn_dbufs, db); 1429 db->db_state = DB_UNCACHED; 1430 mutex_exit(&dn->dn_dbufs_mtx); 1431 arc_space_consume(sizeof (dmu_buf_impl_t)); 1432 1433 if (parent && parent != dn->dn_dbuf) 1434 dbuf_add_ref(parent, db); 1435 1436 ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 1437 refcount_count(&dn->dn_holds) > 0); 1438 (void) refcount_add(&dn->dn_holds, db); 1439 1440 dprintf_dbuf(db, "db=%p\n", db); 1441 1442 return (db); 1443 } 1444 1445 static int 1446 dbuf_do_evict(void *private) 1447 { 1448 arc_buf_t *buf = private; 1449 dmu_buf_impl_t *db = buf->b_private; 1450 1451 if (!MUTEX_HELD(&db->db_mtx)) 1452 mutex_enter(&db->db_mtx); 1453 1454 ASSERT(refcount_is_zero(&db->db_holds)); 1455 1456 if (db->db_state != DB_EVICTING) { 1457 ASSERT(db->db_state == DB_CACHED); 1458 DBUF_VERIFY(db); 1459 db->db_buf = NULL; 1460 dbuf_evict(db); 1461 } else { 1462 mutex_exit(&db->db_mtx); 1463 dbuf_destroy(db); 1464 } 1465 return (0); 1466 } 1467 1468 static void 1469 dbuf_destroy(dmu_buf_impl_t *db) 1470 { 1471 ASSERT(refcount_is_zero(&db->db_holds)); 1472 1473 if (db->db_blkid != DB_BONUS_BLKID) { 1474 dnode_t *dn = db->db_dnode; 1475 boolean_t need_mutex = !MUTEX_HELD(&dn->dn_dbufs_mtx); 1476 1477 if (need_mutex) 1478 mutex_enter(&dn->dn_dbufs_mtx); 1479 1480 /* 1481 * If this dbuf is still on the dn_dbufs list, 1482 * remove it from that list. 1483 */ 1484 if (list_link_active(&db->db_link)) { 1485 ASSERT(need_mutex); 1486 list_remove(&dn->dn_dbufs, db); 1487 mutex_exit(&dn->dn_dbufs_mtx); 1488 1489 dnode_rele(dn, db); 1490 } else if (need_mutex) { 1491 mutex_exit(&dn->dn_dbufs_mtx); 1492 } 1493 dbuf_hash_remove(db); 1494 } 1495 db->db_parent = NULL; 1496 db->db_dnode = NULL; 1497 db->db_buf = NULL; 1498 1499 ASSERT(!list_link_active(&db->db_link)); 1500 ASSERT(db->db.db_data == NULL); 1501 ASSERT(db->db_hash_next == NULL); 1502 ASSERT(db->db_blkptr == NULL); 1503 ASSERT(db->db_data_pending == NULL); 1504 1505 kmem_cache_free(dbuf_cache, db); 1506 arc_space_return(sizeof (dmu_buf_impl_t)); 1507 } 1508 1509 void 1510 dbuf_prefetch(dnode_t *dn, uint64_t blkid) 1511 { 1512 dmu_buf_impl_t *db = NULL; 1513 blkptr_t *bp = NULL; 1514 1515 ASSERT(blkid != DB_BONUS_BLKID); 1516 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1517 1518 if (dnode_block_freed(dn, blkid)) 1519 return; 1520 1521 /* dbuf_find() returns with db_mtx held */ 1522 if (db = dbuf_find(dn, 0, blkid)) { 1523 if (refcount_count(&db->db_holds) > 0) { 1524 /* 1525 * This dbuf is active. We assume that it is 1526 * already CACHED, or else about to be either 1527 * read or filled. 1528 */ 1529 mutex_exit(&db->db_mtx); 1530 return; 1531 } 1532 mutex_exit(&db->db_mtx); 1533 db = NULL; 1534 } 1535 1536 if (dbuf_findbp(dn, 0, blkid, TRUE, &db, &bp) == 0) { 1537 if (bp && !BP_IS_HOLE(bp)) { 1538 uint32_t aflags = ARC_NOWAIT | ARC_PREFETCH; 1539 zbookmark_t zb; 1540 zb.zb_objset = dn->dn_objset->os_dsl_dataset ? 1541 dn->dn_objset->os_dsl_dataset->ds_object : 0; 1542 zb.zb_object = dn->dn_object; 1543 zb.zb_level = 0; 1544 zb.zb_blkid = blkid; 1545 1546 (void) arc_read(NULL, dn->dn_objset->os_spa, bp, 1547 dmu_ot[dn->dn_type].ot_byteswap, 1548 NULL, NULL, ZIO_PRIORITY_ASYNC_READ, 1549 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 1550 &aflags, &zb); 1551 } 1552 if (db) 1553 dbuf_rele(db, NULL); 1554 } 1555 } 1556 1557 /* 1558 * Returns with db_holds incremented, and db_mtx not held. 1559 * Note: dn_struct_rwlock must be held. 1560 */ 1561 int 1562 dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, int fail_sparse, 1563 void *tag, dmu_buf_impl_t **dbp) 1564 { 1565 dmu_buf_impl_t *db, *parent = NULL; 1566 1567 ASSERT(blkid != DB_BONUS_BLKID); 1568 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1569 ASSERT3U(dn->dn_nlevels, >, level); 1570 1571 *dbp = NULL; 1572 top: 1573 /* dbuf_find() returns with db_mtx held */ 1574 db = dbuf_find(dn, level, blkid); 1575 1576 if (db == NULL) { 1577 blkptr_t *bp = NULL; 1578 int err; 1579 1580 ASSERT3P(parent, ==, NULL); 1581 err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp); 1582 if (fail_sparse) { 1583 if (err == 0 && bp && BP_IS_HOLE(bp)) 1584 err = ENOENT; 1585 if (err) { 1586 if (parent) 1587 dbuf_rele(parent, NULL); 1588 return (err); 1589 } 1590 } 1591 if (err && err != ENOENT) 1592 return (err); 1593 db = dbuf_create(dn, level, blkid, parent, bp); 1594 } 1595 1596 if (db->db_buf && refcount_is_zero(&db->db_holds)) { 1597 arc_buf_add_ref(db->db_buf, db); 1598 if (db->db_buf->b_data == NULL) { 1599 dbuf_clear(db); 1600 if (parent) { 1601 dbuf_rele(parent, NULL); 1602 parent = NULL; 1603 } 1604 goto top; 1605 } 1606 ASSERT3P(db->db.db_data, ==, db->db_buf->b_data); 1607 } 1608 1609 ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf)); 1610 1611 /* 1612 * If this buffer is currently syncing out, and we are are 1613 * still referencing it from db_data, we need to make a copy 1614 * of it in case we decide we want to dirty it again in this txg. 1615 */ 1616 if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID && 1617 dn->dn_object != DMU_META_DNODE_OBJECT && 1618 db->db_state == DB_CACHED && db->db_data_pending) { 1619 dbuf_dirty_record_t *dr = db->db_data_pending; 1620 1621 if (dr->dt.dl.dr_data == db->db_buf) { 1622 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 1623 1624 dbuf_set_data(db, 1625 arc_buf_alloc(db->db_dnode->dn_objset->os_spa, 1626 db->db.db_size, db, type)); 1627 bcopy(dr->dt.dl.dr_data->b_data, db->db.db_data, 1628 db->db.db_size); 1629 } 1630 } 1631 1632 (void) refcount_add(&db->db_holds, tag); 1633 dbuf_update_data(db); 1634 DBUF_VERIFY(db); 1635 mutex_exit(&db->db_mtx); 1636 1637 /* NOTE: we can't rele the parent until after we drop the db_mtx */ 1638 if (parent) 1639 dbuf_rele(parent, NULL); 1640 1641 ASSERT3P(db->db_dnode, ==, dn); 1642 ASSERT3U(db->db_blkid, ==, blkid); 1643 ASSERT3U(db->db_level, ==, level); 1644 *dbp = db; 1645 1646 return (0); 1647 } 1648 1649 dmu_buf_impl_t * 1650 dbuf_hold(dnode_t *dn, uint64_t blkid, void *tag) 1651 { 1652 dmu_buf_impl_t *db; 1653 int err = dbuf_hold_impl(dn, 0, blkid, FALSE, tag, &db); 1654 return (err ? NULL : db); 1655 } 1656 1657 dmu_buf_impl_t * 1658 dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, void *tag) 1659 { 1660 dmu_buf_impl_t *db; 1661 int err = dbuf_hold_impl(dn, level, blkid, FALSE, tag, &db); 1662 return (err ? NULL : db); 1663 } 1664 1665 dmu_buf_impl_t * 1666 dbuf_create_bonus(dnode_t *dn) 1667 { 1668 dmu_buf_impl_t *db = dn->dn_bonus; 1669 1670 ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 1671 1672 ASSERT(dn->dn_bonus == NULL); 1673 db = dbuf_create(dn, 0, DB_BONUS_BLKID, dn->dn_dbuf, NULL); 1674 return (db); 1675 } 1676 1677 #pragma weak dmu_buf_add_ref = dbuf_add_ref 1678 void 1679 dbuf_add_ref(dmu_buf_impl_t *db, void *tag) 1680 { 1681 int64_t holds = refcount_add(&db->db_holds, tag); 1682 ASSERT(holds > 1); 1683 } 1684 1685 #pragma weak dmu_buf_rele = dbuf_rele 1686 void 1687 dbuf_rele(dmu_buf_impl_t *db, void *tag) 1688 { 1689 int64_t holds; 1690 1691 mutex_enter(&db->db_mtx); 1692 DBUF_VERIFY(db); 1693 1694 holds = refcount_remove(&db->db_holds, tag); 1695 ASSERT(holds >= 0); 1696 1697 /* 1698 * We can't freeze indirects if there is a possibility that they 1699 * may be modified in the current syncing context. 1700 */ 1701 if (db->db_buf && holds == (db->db_level == 0 ? db->db_dirtycnt : 0)) 1702 arc_buf_freeze(db->db_buf); 1703 1704 if (holds == db->db_dirtycnt && 1705 db->db_level == 0 && db->db_immediate_evict) 1706 dbuf_evict_user(db); 1707 1708 if (holds == 0) { 1709 if (db->db_blkid == DB_BONUS_BLKID) { 1710 mutex_exit(&db->db_mtx); 1711 dnode_rele(db->db_dnode, db); 1712 } else if (db->db_buf == NULL) { 1713 /* 1714 * This is a special case: we never associated this 1715 * dbuf with any data allocated from the ARC. 1716 */ 1717 ASSERT3U(db->db_state, ==, DB_UNCACHED); 1718 dbuf_evict(db); 1719 } else if (arc_released(db->db_buf)) { 1720 arc_buf_t *buf = db->db_buf; 1721 /* 1722 * This dbuf has anonymous data associated with it. 1723 */ 1724 dbuf_set_data(db, NULL); 1725 VERIFY(arc_buf_remove_ref(buf, db) == 1); 1726 dbuf_evict(db); 1727 } else { 1728 VERIFY(arc_buf_remove_ref(db->db_buf, db) == 0); 1729 mutex_exit(&db->db_mtx); 1730 } 1731 } else { 1732 mutex_exit(&db->db_mtx); 1733 } 1734 } 1735 1736 #pragma weak dmu_buf_refcount = dbuf_refcount 1737 uint64_t 1738 dbuf_refcount(dmu_buf_impl_t *db) 1739 { 1740 return (refcount_count(&db->db_holds)); 1741 } 1742 1743 void * 1744 dmu_buf_set_user(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 1745 dmu_buf_evict_func_t *evict_func) 1746 { 1747 return (dmu_buf_update_user(db_fake, NULL, user_ptr, 1748 user_data_ptr_ptr, evict_func)); 1749 } 1750 1751 void * 1752 dmu_buf_set_user_ie(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 1753 dmu_buf_evict_func_t *evict_func) 1754 { 1755 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1756 1757 db->db_immediate_evict = TRUE; 1758 return (dmu_buf_update_user(db_fake, NULL, user_ptr, 1759 user_data_ptr_ptr, evict_func)); 1760 } 1761 1762 void * 1763 dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr, void *user_ptr, 1764 void *user_data_ptr_ptr, dmu_buf_evict_func_t *evict_func) 1765 { 1766 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1767 ASSERT(db->db_level == 0); 1768 1769 ASSERT((user_ptr == NULL) == (evict_func == NULL)); 1770 1771 mutex_enter(&db->db_mtx); 1772 1773 if (db->db_user_ptr == old_user_ptr) { 1774 db->db_user_ptr = user_ptr; 1775 db->db_user_data_ptr_ptr = user_data_ptr_ptr; 1776 db->db_evict_func = evict_func; 1777 1778 dbuf_update_data(db); 1779 } else { 1780 old_user_ptr = db->db_user_ptr; 1781 } 1782 1783 mutex_exit(&db->db_mtx); 1784 return (old_user_ptr); 1785 } 1786 1787 void * 1788 dmu_buf_get_user(dmu_buf_t *db_fake) 1789 { 1790 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1791 ASSERT(!refcount_is_zero(&db->db_holds)); 1792 1793 return (db->db_user_ptr); 1794 } 1795 1796 static void 1797 dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db) 1798 { 1799 /* ASSERT(dmu_tx_is_syncing(tx) */ 1800 ASSERT(MUTEX_HELD(&db->db_mtx)); 1801 1802 if (db->db_blkptr != NULL) 1803 return; 1804 1805 if (db->db_level == dn->dn_phys->dn_nlevels-1) { 1806 /* 1807 * This buffer was allocated at a time when there was 1808 * no available blkptrs from the dnode, or it was 1809 * inappropriate to hook it in (i.e., nlevels mis-match). 1810 */ 1811 ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr); 1812 ASSERT(db->db_parent == NULL); 1813 db->db_parent = dn->dn_dbuf; 1814 db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid]; 1815 DBUF_VERIFY(db); 1816 } else { 1817 dmu_buf_impl_t *parent = db->db_parent; 1818 int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 1819 1820 ASSERT(dn->dn_phys->dn_nlevels > 1); 1821 if (parent == NULL) { 1822 mutex_exit(&db->db_mtx); 1823 rw_enter(&dn->dn_struct_rwlock, RW_READER); 1824 (void) dbuf_hold_impl(dn, db->db_level+1, 1825 db->db_blkid >> epbs, FALSE, db, &parent); 1826 rw_exit(&dn->dn_struct_rwlock); 1827 mutex_enter(&db->db_mtx); 1828 db->db_parent = parent; 1829 } 1830 db->db_blkptr = (blkptr_t *)parent->db.db_data + 1831 (db->db_blkid & ((1ULL << epbs) - 1)); 1832 DBUF_VERIFY(db); 1833 } 1834 } 1835 1836 static void 1837 dbuf_sync_indirect(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 1838 { 1839 dmu_buf_impl_t *db = dr->dr_dbuf; 1840 dnode_t *dn = db->db_dnode; 1841 zio_t *zio; 1842 1843 ASSERT(dmu_tx_is_syncing(tx)); 1844 1845 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 1846 1847 mutex_enter(&db->db_mtx); 1848 1849 ASSERT(db->db_level > 0); 1850 DBUF_VERIFY(db); 1851 1852 if (db->db_buf == NULL) { 1853 mutex_exit(&db->db_mtx); 1854 (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED); 1855 mutex_enter(&db->db_mtx); 1856 } 1857 ASSERT3U(db->db_state, ==, DB_CACHED); 1858 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 1859 ASSERT(db->db_buf != NULL); 1860 1861 dbuf_check_blkptr(dn, db); 1862 1863 db->db_data_pending = dr; 1864 1865 arc_release(db->db_buf, db); 1866 mutex_exit(&db->db_mtx); 1867 1868 /* 1869 * XXX -- we should design a compression algorithm 1870 * that specializes in arrays of bps. 1871 */ 1872 dbuf_write(dr, db->db_buf, ZIO_CHECKSUM_FLETCHER_4, 1873 zfs_mdcomp_disable ? ZIO_COMPRESS_EMPTY : ZIO_COMPRESS_LZJB, tx); 1874 1875 zio = dr->dr_zio; 1876 mutex_enter(&dr->dt.di.dr_mtx); 1877 dbuf_sync_list(&dr->dt.di.dr_children, tx); 1878 ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 1879 mutex_exit(&dr->dt.di.dr_mtx); 1880 zio_nowait(zio); 1881 } 1882 1883 static void 1884 dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 1885 { 1886 arc_buf_t **datap = &dr->dt.dl.dr_data; 1887 dmu_buf_impl_t *db = dr->dr_dbuf; 1888 dnode_t *dn = db->db_dnode; 1889 objset_impl_t *os = dn->dn_objset; 1890 uint64_t txg = tx->tx_txg; 1891 int checksum, compress; 1892 int blksz; 1893 1894 ASSERT(dmu_tx_is_syncing(tx)); 1895 1896 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 1897 1898 mutex_enter(&db->db_mtx); 1899 /* 1900 * To be synced, we must be dirtied. But we 1901 * might have been freed after the dirty. 1902 */ 1903 if (db->db_state == DB_UNCACHED) { 1904 /* This buffer has been freed since it was dirtied */ 1905 ASSERT(db->db.db_data == NULL); 1906 } else if (db->db_state == DB_FILL) { 1907 /* This buffer was freed and is now being re-filled */ 1908 ASSERT(db->db.db_data != dr->dt.dl.dr_data); 1909 } else { 1910 ASSERT3U(db->db_state, ==, DB_CACHED); 1911 } 1912 DBUF_VERIFY(db); 1913 1914 /* 1915 * If this is a bonus buffer, simply copy the bonus data into the 1916 * dnode. It will be written out when the dnode is synced (and it 1917 * will be synced, since it must have been dirty for dbuf_sync to 1918 * be called). 1919 */ 1920 if (db->db_blkid == DB_BONUS_BLKID) { 1921 dbuf_dirty_record_t **drp; 1922 /* 1923 * Use dn_phys->dn_bonuslen since db.db_size is the length 1924 * of the bonus buffer in the open transaction rather than 1925 * the syncing transaction. 1926 */ 1927 ASSERT(*datap != NULL); 1928 ASSERT3U(db->db_level, ==, 0); 1929 ASSERT3U(dn->dn_phys->dn_bonuslen, <=, DN_MAX_BONUSLEN); 1930 bcopy(*datap, DN_BONUS(dn->dn_phys), dn->dn_phys->dn_bonuslen); 1931 if (*datap != db->db.db_data) { 1932 zio_buf_free(*datap, DN_MAX_BONUSLEN); 1933 arc_space_return(DN_MAX_BONUSLEN); 1934 } 1935 db->db_data_pending = NULL; 1936 drp = &db->db_last_dirty; 1937 while (*drp != dr) 1938 drp = &(*drp)->dr_next; 1939 ASSERT((*drp)->dr_next == NULL); 1940 *drp = NULL; 1941 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 1942 ASSERT(db->db_dirtycnt > 0); 1943 db->db_dirtycnt -= 1; 1944 mutex_exit(&db->db_mtx); 1945 dbuf_rele(db, (void *)(uintptr_t)txg); 1946 return; 1947 } 1948 1949 /* 1950 * This function may have dropped the db_mtx lock allowing a dmu_sync 1951 * operation to sneak in. As a result, we need to ensure that we 1952 * don't check the dr_override_state until we have returned from 1953 * dbuf_check_blkptr. 1954 */ 1955 dbuf_check_blkptr(dn, db); 1956 1957 /* 1958 * If this buffer is in the middle of an immdiate write, 1959 * wait for the synchronous IO to complete. 1960 */ 1961 while (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC) { 1962 ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT); 1963 cv_wait(&db->db_changed, &db->db_mtx); 1964 ASSERT(dr->dt.dl.dr_override_state != DR_NOT_OVERRIDDEN); 1965 } 1966 1967 /* 1968 * If this dbuf has already been written out via an immediate write, 1969 * just complete the write by copying over the new block pointer and 1970 * updating the accounting via the write-completion functions. 1971 */ 1972 if (dr->dt.dl.dr_override_state == DR_OVERRIDDEN) { 1973 zio_t zio_fake; 1974 1975 zio_fake.io_private = &db; 1976 zio_fake.io_error = 0; 1977 zio_fake.io_bp = db->db_blkptr; 1978 zio_fake.io_bp_orig = *db->db_blkptr; 1979 zio_fake.io_txg = txg; 1980 1981 *db->db_blkptr = dr->dt.dl.dr_overridden_by; 1982 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 1983 db->db_data_pending = dr; 1984 dr->dr_zio = &zio_fake; 1985 mutex_exit(&db->db_mtx); 1986 1987 if (BP_IS_OLDER(&zio_fake.io_bp_orig, txg)) 1988 dsl_dataset_block_kill(os->os_dsl_dataset, 1989 &zio_fake.io_bp_orig, dn->dn_zio, tx); 1990 1991 dbuf_write_ready(&zio_fake, db->db_buf, db); 1992 dbuf_write_done(&zio_fake, db->db_buf, db); 1993 1994 return; 1995 } 1996 1997 blksz = arc_buf_size(*datap); 1998 1999 if (dn->dn_object != DMU_META_DNODE_OBJECT) { 2000 /* 2001 * If this buffer is currently "in use" (i.e., there are 2002 * active holds and db_data still references it), then make 2003 * a copy before we start the write so that any modifications 2004 * from the open txg will not leak into this write. 2005 * 2006 * NOTE: this copy does not need to be made for objects only 2007 * modified in the syncing context (e.g. DNONE_DNODE blocks). 2008 */ 2009 if (refcount_count(&db->db_holds) > 1 && *datap == db->db_buf) { 2010 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 2011 *datap = arc_buf_alloc(os->os_spa, blksz, db, type); 2012 bcopy(db->db.db_data, (*datap)->b_data, blksz); 2013 } 2014 } else { 2015 /* 2016 * Private object buffers are released here rather 2017 * than in dbuf_dirty() since they are only modified 2018 * in the syncing context and we don't want the 2019 * overhead of making multiple copies of the data. 2020 */ 2021 arc_release(db->db_buf, db); 2022 } 2023 2024 ASSERT(*datap != NULL); 2025 db->db_data_pending = dr; 2026 2027 mutex_exit(&db->db_mtx); 2028 2029 /* 2030 * Allow dnode settings to override objset settings, 2031 * except for metadata checksums. 2032 */ 2033 if (dmu_ot[dn->dn_type].ot_metadata) { 2034 checksum = os->os_md_checksum; 2035 compress = zio_compress_select(dn->dn_compress, 2036 os->os_md_compress); 2037 } else { 2038 checksum = zio_checksum_select(dn->dn_checksum, 2039 os->os_checksum); 2040 compress = zio_compress_select(dn->dn_compress, 2041 os->os_compress); 2042 } 2043 2044 dbuf_write(dr, *datap, checksum, compress, tx); 2045 2046 ASSERT(!list_link_active(&dr->dr_dirty_node)); 2047 if (dn->dn_object == DMU_META_DNODE_OBJECT) 2048 list_insert_tail(&dn->dn_dirty_records[txg&TXG_MASK], dr); 2049 else 2050 zio_nowait(dr->dr_zio); 2051 } 2052 2053 void 2054 dbuf_sync_list(list_t *list, dmu_tx_t *tx) 2055 { 2056 dbuf_dirty_record_t *dr; 2057 2058 while (dr = list_head(list)) { 2059 if (dr->dr_zio != NULL) { 2060 /* 2061 * If we find an already initialized zio then we 2062 * are processing the meta-dnode, and we have finished. 2063 * The dbufs for all dnodes are put back on the list 2064 * during processing, so that we can zio_wait() 2065 * these IOs after initiating all child IOs. 2066 */ 2067 ASSERT3U(dr->dr_dbuf->db.db_object, ==, 2068 DMU_META_DNODE_OBJECT); 2069 break; 2070 } 2071 list_remove(list, dr); 2072 if (dr->dr_dbuf->db_level > 0) 2073 dbuf_sync_indirect(dr, tx); 2074 else 2075 dbuf_sync_leaf(dr, tx); 2076 } 2077 } 2078 2079 static void 2080 dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, int checksum, 2081 int compress, dmu_tx_t *tx) 2082 { 2083 dmu_buf_impl_t *db = dr->dr_dbuf; 2084 dnode_t *dn = db->db_dnode; 2085 objset_impl_t *os = dn->dn_objset; 2086 dmu_buf_impl_t *parent = db->db_parent; 2087 uint64_t txg = tx->tx_txg; 2088 zbookmark_t zb; 2089 zio_t *zio; 2090 int zio_flags; 2091 2092 if (parent != dn->dn_dbuf) { 2093 ASSERT(parent && parent->db_data_pending); 2094 ASSERT(db->db_level == parent->db_level-1); 2095 ASSERT(arc_released(parent->db_buf)); 2096 zio = parent->db_data_pending->dr_zio; 2097 } else { 2098 ASSERT(db->db_level == dn->dn_phys->dn_nlevels-1); 2099 ASSERT3P(db->db_blkptr, ==, 2100 &dn->dn_phys->dn_blkptr[db->db_blkid]); 2101 zio = dn->dn_zio; 2102 } 2103 2104 ASSERT(db->db_level == 0 || data == db->db_buf); 2105 ASSERT3U(db->db_blkptr->blk_birth, <=, txg); 2106 ASSERT(zio); 2107 2108 zb.zb_objset = os->os_dsl_dataset ? os->os_dsl_dataset->ds_object : 0; 2109 zb.zb_object = db->db.db_object; 2110 zb.zb_level = db->db_level; 2111 zb.zb_blkid = db->db_blkid; 2112 2113 zio_flags = ZIO_FLAG_MUSTSUCCEED; 2114 if (dmu_ot[dn->dn_type].ot_metadata || zb.zb_level != 0) 2115 zio_flags |= ZIO_FLAG_METADATA; 2116 if (BP_IS_OLDER(db->db_blkptr, txg)) 2117 dsl_dataset_block_kill( 2118 os->os_dsl_dataset, db->db_blkptr, zio, tx); 2119 2120 dr->dr_zio = arc_write(zio, os->os_spa, checksum, compress, 2121 dmu_get_replication_level(os, &zb, dn->dn_type), txg, 2122 db->db_blkptr, data, dbuf_write_ready, dbuf_write_done, db, 2123 ZIO_PRIORITY_ASYNC_WRITE, zio_flags, &zb); 2124 } 2125 2126 /* ARGSUSED */ 2127 static void 2128 dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb) 2129 { 2130 dmu_buf_impl_t *db = vdb; 2131 dnode_t *dn = db->db_dnode; 2132 objset_impl_t *os = dn->dn_objset; 2133 blkptr_t *bp_orig = &zio->io_bp_orig; 2134 uint64_t fill = 0; 2135 int old_size, new_size, i; 2136 2137 dprintf_dbuf_bp(db, bp_orig, "bp_orig: %s", ""); 2138 2139 old_size = bp_get_dasize(os->os_spa, bp_orig); 2140 new_size = bp_get_dasize(os->os_spa, zio->io_bp); 2141 2142 dnode_diduse_space(dn, new_size-old_size); 2143 2144 if (BP_IS_HOLE(zio->io_bp)) { 2145 dsl_dataset_t *ds = os->os_dsl_dataset; 2146 dmu_tx_t *tx = os->os_synctx; 2147 2148 if (bp_orig->blk_birth == tx->tx_txg) 2149 dsl_dataset_block_kill(ds, bp_orig, NULL, tx); 2150 ASSERT3U(db->db_blkptr->blk_fill, ==, 0); 2151 return; 2152 } 2153 2154 mutex_enter(&db->db_mtx); 2155 2156 if (db->db_level == 0) { 2157 mutex_enter(&dn->dn_mtx); 2158 if (db->db_blkid > dn->dn_phys->dn_maxblkid) 2159 dn->dn_phys->dn_maxblkid = db->db_blkid; 2160 mutex_exit(&dn->dn_mtx); 2161 2162 if (dn->dn_type == DMU_OT_DNODE) { 2163 dnode_phys_t *dnp = db->db.db_data; 2164 for (i = db->db.db_size >> DNODE_SHIFT; i > 0; 2165 i--, dnp++) { 2166 if (dnp->dn_type != DMU_OT_NONE) 2167 fill++; 2168 } 2169 } else { 2170 fill = 1; 2171 } 2172 } else { 2173 blkptr_t *bp = db->db.db_data; 2174 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2175 for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, bp++) { 2176 if (BP_IS_HOLE(bp)) 2177 continue; 2178 ASSERT3U(BP_GET_LSIZE(bp), ==, 2179 db->db_level == 1 ? dn->dn_datablksz : 2180 (1<<dn->dn_phys->dn_indblkshift)); 2181 fill += bp->blk_fill; 2182 } 2183 } 2184 2185 db->db_blkptr->blk_fill = fill; 2186 BP_SET_TYPE(db->db_blkptr, dn->dn_type); 2187 BP_SET_LEVEL(db->db_blkptr, db->db_level); 2188 2189 mutex_exit(&db->db_mtx); 2190 2191 /* We must do this after we've set the bp's type and level */ 2192 if (!DVA_EQUAL(BP_IDENTITY(zio->io_bp), BP_IDENTITY(bp_orig))) { 2193 dsl_dataset_t *ds = os->os_dsl_dataset; 2194 dmu_tx_t *tx = os->os_synctx; 2195 2196 if (bp_orig->blk_birth == tx->tx_txg) 2197 dsl_dataset_block_kill(ds, bp_orig, NULL, tx); 2198 dsl_dataset_block_born(ds, zio->io_bp, tx); 2199 } 2200 } 2201 2202 /* ARGSUSED */ 2203 static void 2204 dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb) 2205 { 2206 dmu_buf_impl_t *db = vdb; 2207 uint64_t txg = zio->io_txg; 2208 dbuf_dirty_record_t **drp, *dr; 2209 2210 ASSERT3U(zio->io_error, ==, 0); 2211 2212 mutex_enter(&db->db_mtx); 2213 2214 drp = &db->db_last_dirty; 2215 while (*drp != db->db_data_pending) 2216 drp = &(*drp)->dr_next; 2217 ASSERT(!list_link_active(&(*drp)->dr_dirty_node)); 2218 ASSERT((*drp)->dr_txg == txg); 2219 ASSERT((*drp)->dr_next == NULL); 2220 dr = *drp; 2221 *drp = NULL; 2222 2223 if (db->db_level == 0) { 2224 ASSERT(db->db_blkid != DB_BONUS_BLKID); 2225 ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN); 2226 2227 if (dr->dt.dl.dr_data != db->db_buf) 2228 VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, db) == 1); 2229 else if (!BP_IS_HOLE(db->db_blkptr)) 2230 arc_set_callback(db->db_buf, dbuf_do_evict, db); 2231 else 2232 ASSERT(arc_released(db->db_buf)); 2233 } else { 2234 dnode_t *dn = db->db_dnode; 2235 2236 ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 2237 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2238 if (!BP_IS_HOLE(db->db_blkptr)) { 2239 int epbs = 2240 dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2241 ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, 2242 db->db.db_size); 2243 ASSERT3U(dn->dn_phys->dn_maxblkid 2244 >> (db->db_level * epbs), >=, db->db_blkid); 2245 arc_set_callback(db->db_buf, dbuf_do_evict, db); 2246 } 2247 mutex_destroy(&dr->dt.di.dr_mtx); 2248 list_destroy(&dr->dt.di.dr_children); 2249 } 2250 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 2251 2252 cv_broadcast(&db->db_changed); 2253 ASSERT(db->db_dirtycnt > 0); 2254 db->db_dirtycnt -= 1; 2255 db->db_data_pending = NULL; 2256 mutex_exit(&db->db_mtx); 2257 2258 dprintf_dbuf_bp(db, zio->io_bp, "bp: %s", ""); 2259 2260 dbuf_rele(db, (void *)(uintptr_t)txg); 2261 } 2262