1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright 2011 Nexenta Systems, Inc. All rights reserved. 24 * Copyright (c) 2013 by Delphix. All rights reserved. 25 * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. 26 * Copyright (c) 2013, Joyent, Inc. All rights reserved. 27 */ 28 29 #include <sys/zfs_context.h> 30 #include <sys/dmu.h> 31 #include <sys/dmu_send.h> 32 #include <sys/dmu_impl.h> 33 #include <sys/dbuf.h> 34 #include <sys/dmu_objset.h> 35 #include <sys/dsl_dataset.h> 36 #include <sys/dsl_dir.h> 37 #include <sys/dmu_tx.h> 38 #include <sys/spa.h> 39 #include <sys/zio.h> 40 #include <sys/dmu_zfetch.h> 41 #include <sys/sa.h> 42 #include <sys/sa_impl.h> 43 44 /* 45 * Number of times that zfs_free_range() took the slow path while doing 46 * a zfs receive. A nonzero value indicates a potential performance problem. 47 */ 48 uint64_t zfs_free_range_recv_miss; 49 50 static void dbuf_destroy(dmu_buf_impl_t *db); 51 static boolean_t dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx); 52 static void dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx); 53 54 /* 55 * Global data structures and functions for the dbuf cache. 56 */ 57 static kmem_cache_t *dbuf_cache; 58 59 /* ARGSUSED */ 60 static int 61 dbuf_cons(void *vdb, void *unused, int kmflag) 62 { 63 dmu_buf_impl_t *db = vdb; 64 bzero(db, sizeof (dmu_buf_impl_t)); 65 66 mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL); 67 cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL); 68 refcount_create(&db->db_holds); 69 return (0); 70 } 71 72 /* ARGSUSED */ 73 static void 74 dbuf_dest(void *vdb, void *unused) 75 { 76 dmu_buf_impl_t *db = vdb; 77 mutex_destroy(&db->db_mtx); 78 cv_destroy(&db->db_changed); 79 refcount_destroy(&db->db_holds); 80 } 81 82 /* 83 * dbuf hash table routines 84 */ 85 static dbuf_hash_table_t dbuf_hash_table; 86 87 static uint64_t dbuf_hash_count; 88 89 static uint64_t 90 dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid) 91 { 92 uintptr_t osv = (uintptr_t)os; 93 uint64_t crc = -1ULL; 94 95 ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 96 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (lvl)) & 0xFF]; 97 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF]; 98 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF]; 99 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF]; 100 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 0)) & 0xFF]; 101 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 8)) & 0xFF]; 102 103 crc ^= (osv>>14) ^ (obj>>16) ^ (blkid>>16); 104 105 return (crc); 106 } 107 108 #define DBUF_HASH(os, obj, level, blkid) dbuf_hash(os, obj, level, blkid); 109 110 #define DBUF_EQUAL(dbuf, os, obj, level, blkid) \ 111 ((dbuf)->db.db_object == (obj) && \ 112 (dbuf)->db_objset == (os) && \ 113 (dbuf)->db_level == (level) && \ 114 (dbuf)->db_blkid == (blkid)) 115 116 dmu_buf_impl_t * 117 dbuf_find(dnode_t *dn, uint8_t level, uint64_t blkid) 118 { 119 dbuf_hash_table_t *h = &dbuf_hash_table; 120 objset_t *os = dn->dn_objset; 121 uint64_t obj = dn->dn_object; 122 uint64_t hv = DBUF_HASH(os, obj, level, blkid); 123 uint64_t idx = hv & h->hash_table_mask; 124 dmu_buf_impl_t *db; 125 126 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 127 for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) { 128 if (DBUF_EQUAL(db, os, obj, level, blkid)) { 129 mutex_enter(&db->db_mtx); 130 if (db->db_state != DB_EVICTING) { 131 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 132 return (db); 133 } 134 mutex_exit(&db->db_mtx); 135 } 136 } 137 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 138 return (NULL); 139 } 140 141 /* 142 * Insert an entry into the hash table. If there is already an element 143 * equal to elem in the hash table, then the already existing element 144 * will be returned and the new element will not be inserted. 145 * Otherwise returns NULL. 146 */ 147 static dmu_buf_impl_t * 148 dbuf_hash_insert(dmu_buf_impl_t *db) 149 { 150 dbuf_hash_table_t *h = &dbuf_hash_table; 151 objset_t *os = db->db_objset; 152 uint64_t obj = db->db.db_object; 153 int level = db->db_level; 154 uint64_t blkid = db->db_blkid; 155 uint64_t hv = DBUF_HASH(os, obj, level, blkid); 156 uint64_t idx = hv & h->hash_table_mask; 157 dmu_buf_impl_t *dbf; 158 159 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 160 for (dbf = h->hash_table[idx]; dbf != NULL; dbf = dbf->db_hash_next) { 161 if (DBUF_EQUAL(dbf, os, obj, level, blkid)) { 162 mutex_enter(&dbf->db_mtx); 163 if (dbf->db_state != DB_EVICTING) { 164 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 165 return (dbf); 166 } 167 mutex_exit(&dbf->db_mtx); 168 } 169 } 170 171 mutex_enter(&db->db_mtx); 172 db->db_hash_next = h->hash_table[idx]; 173 h->hash_table[idx] = db; 174 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 175 atomic_add_64(&dbuf_hash_count, 1); 176 177 return (NULL); 178 } 179 180 /* 181 * Remove an entry from the hash table. This operation will 182 * fail if there are any existing holds on the db. 183 */ 184 static void 185 dbuf_hash_remove(dmu_buf_impl_t *db) 186 { 187 dbuf_hash_table_t *h = &dbuf_hash_table; 188 uint64_t hv = DBUF_HASH(db->db_objset, db->db.db_object, 189 db->db_level, db->db_blkid); 190 uint64_t idx = hv & h->hash_table_mask; 191 dmu_buf_impl_t *dbf, **dbp; 192 193 /* 194 * We musn't hold db_mtx to maintin lock ordering: 195 * DBUF_HASH_MUTEX > db_mtx. 196 */ 197 ASSERT(refcount_is_zero(&db->db_holds)); 198 ASSERT(db->db_state == DB_EVICTING); 199 ASSERT(!MUTEX_HELD(&db->db_mtx)); 200 201 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 202 dbp = &h->hash_table[idx]; 203 while ((dbf = *dbp) != db) { 204 dbp = &dbf->db_hash_next; 205 ASSERT(dbf != NULL); 206 } 207 *dbp = db->db_hash_next; 208 db->db_hash_next = NULL; 209 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 210 atomic_add_64(&dbuf_hash_count, -1); 211 } 212 213 static arc_evict_func_t dbuf_do_evict; 214 215 static void 216 dbuf_evict_user(dmu_buf_impl_t *db) 217 { 218 ASSERT(MUTEX_HELD(&db->db_mtx)); 219 220 if (db->db_level != 0 || db->db_evict_func == NULL) 221 return; 222 223 if (db->db_user_data_ptr_ptr) 224 *db->db_user_data_ptr_ptr = db->db.db_data; 225 db->db_evict_func(&db->db, db->db_user_ptr); 226 db->db_user_ptr = NULL; 227 db->db_user_data_ptr_ptr = NULL; 228 db->db_evict_func = NULL; 229 } 230 231 boolean_t 232 dbuf_is_metadata(dmu_buf_impl_t *db) 233 { 234 if (db->db_level > 0) { 235 return (B_TRUE); 236 } else { 237 boolean_t is_metadata; 238 239 DB_DNODE_ENTER(db); 240 is_metadata = DMU_OT_IS_METADATA(DB_DNODE(db)->dn_type); 241 DB_DNODE_EXIT(db); 242 243 return (is_metadata); 244 } 245 } 246 247 void 248 dbuf_evict(dmu_buf_impl_t *db) 249 { 250 ASSERT(MUTEX_HELD(&db->db_mtx)); 251 ASSERT(db->db_buf == NULL); 252 ASSERT(db->db_data_pending == NULL); 253 254 dbuf_clear(db); 255 dbuf_destroy(db); 256 } 257 258 void 259 dbuf_init(void) 260 { 261 uint64_t hsize = 1ULL << 16; 262 dbuf_hash_table_t *h = &dbuf_hash_table; 263 int i; 264 265 /* 266 * The hash table is big enough to fill all of physical memory 267 * with an average 4K block size. The table will take up 268 * totalmem*sizeof(void*)/4K (i.e. 2MB/GB with 8-byte pointers). 269 */ 270 while (hsize * 4096 < physmem * PAGESIZE) 271 hsize <<= 1; 272 273 retry: 274 h->hash_table_mask = hsize - 1; 275 h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP); 276 if (h->hash_table == NULL) { 277 /* XXX - we should really return an error instead of assert */ 278 ASSERT(hsize > (1ULL << 10)); 279 hsize >>= 1; 280 goto retry; 281 } 282 283 dbuf_cache = kmem_cache_create("dmu_buf_impl_t", 284 sizeof (dmu_buf_impl_t), 285 0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0); 286 287 for (i = 0; i < DBUF_MUTEXES; i++) 288 mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL); 289 } 290 291 void 292 dbuf_fini(void) 293 { 294 dbuf_hash_table_t *h = &dbuf_hash_table; 295 int i; 296 297 for (i = 0; i < DBUF_MUTEXES; i++) 298 mutex_destroy(&h->hash_mutexes[i]); 299 kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *)); 300 kmem_cache_destroy(dbuf_cache); 301 } 302 303 /* 304 * Other stuff. 305 */ 306 307 #ifdef ZFS_DEBUG 308 static void 309 dbuf_verify(dmu_buf_impl_t *db) 310 { 311 dnode_t *dn; 312 dbuf_dirty_record_t *dr; 313 314 ASSERT(MUTEX_HELD(&db->db_mtx)); 315 316 if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY)) 317 return; 318 319 ASSERT(db->db_objset != NULL); 320 DB_DNODE_ENTER(db); 321 dn = DB_DNODE(db); 322 if (dn == NULL) { 323 ASSERT(db->db_parent == NULL); 324 ASSERT(db->db_blkptr == NULL); 325 } else { 326 ASSERT3U(db->db.db_object, ==, dn->dn_object); 327 ASSERT3P(db->db_objset, ==, dn->dn_objset); 328 ASSERT3U(db->db_level, <, dn->dn_nlevels); 329 ASSERT(db->db_blkid == DMU_BONUS_BLKID || 330 db->db_blkid == DMU_SPILL_BLKID || 331 !list_is_empty(&dn->dn_dbufs)); 332 } 333 if (db->db_blkid == DMU_BONUS_BLKID) { 334 ASSERT(dn != NULL); 335 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 336 ASSERT3U(db->db.db_offset, ==, DMU_BONUS_BLKID); 337 } else if (db->db_blkid == DMU_SPILL_BLKID) { 338 ASSERT(dn != NULL); 339 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 340 ASSERT0(db->db.db_offset); 341 } else { 342 ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size); 343 } 344 345 for (dr = db->db_data_pending; dr != NULL; dr = dr->dr_next) 346 ASSERT(dr->dr_dbuf == db); 347 348 for (dr = db->db_last_dirty; dr != NULL; dr = dr->dr_next) 349 ASSERT(dr->dr_dbuf == db); 350 351 /* 352 * We can't assert that db_size matches dn_datablksz because it 353 * can be momentarily different when another thread is doing 354 * dnode_set_blksz(). 355 */ 356 if (db->db_level == 0 && db->db.db_object == DMU_META_DNODE_OBJECT) { 357 dr = db->db_data_pending; 358 /* 359 * It should only be modified in syncing context, so 360 * make sure we only have one copy of the data. 361 */ 362 ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf); 363 } 364 365 /* verify db->db_blkptr */ 366 if (db->db_blkptr) { 367 if (db->db_parent == dn->dn_dbuf) { 368 /* db is pointed to by the dnode */ 369 /* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */ 370 if (DMU_OBJECT_IS_SPECIAL(db->db.db_object)) 371 ASSERT(db->db_parent == NULL); 372 else 373 ASSERT(db->db_parent != NULL); 374 if (db->db_blkid != DMU_SPILL_BLKID) 375 ASSERT3P(db->db_blkptr, ==, 376 &dn->dn_phys->dn_blkptr[db->db_blkid]); 377 } else { 378 /* db is pointed to by an indirect block */ 379 int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT; 380 ASSERT3U(db->db_parent->db_level, ==, db->db_level+1); 381 ASSERT3U(db->db_parent->db.db_object, ==, 382 db->db.db_object); 383 /* 384 * dnode_grow_indblksz() can make this fail if we don't 385 * have the struct_rwlock. XXX indblksz no longer 386 * grows. safe to do this now? 387 */ 388 if (RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 389 ASSERT3P(db->db_blkptr, ==, 390 ((blkptr_t *)db->db_parent->db.db_data + 391 db->db_blkid % epb)); 392 } 393 } 394 } 395 if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) && 396 (db->db_buf == NULL || db->db_buf->b_data) && 397 db->db.db_data && db->db_blkid != DMU_BONUS_BLKID && 398 db->db_state != DB_FILL && !dn->dn_free_txg) { 399 /* 400 * If the blkptr isn't set but they have nonzero data, 401 * it had better be dirty, otherwise we'll lose that 402 * data when we evict this buffer. 403 */ 404 if (db->db_dirtycnt == 0) { 405 uint64_t *buf = db->db.db_data; 406 int i; 407 408 for (i = 0; i < db->db.db_size >> 3; i++) { 409 ASSERT(buf[i] == 0); 410 } 411 } 412 } 413 DB_DNODE_EXIT(db); 414 } 415 #endif 416 417 static void 418 dbuf_update_data(dmu_buf_impl_t *db) 419 { 420 ASSERT(MUTEX_HELD(&db->db_mtx)); 421 if (db->db_level == 0 && db->db_user_data_ptr_ptr) { 422 ASSERT(!refcount_is_zero(&db->db_holds)); 423 *db->db_user_data_ptr_ptr = db->db.db_data; 424 } 425 } 426 427 static void 428 dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf) 429 { 430 ASSERT(MUTEX_HELD(&db->db_mtx)); 431 ASSERT(db->db_buf == NULL || !arc_has_callback(db->db_buf)); 432 db->db_buf = buf; 433 if (buf != NULL) { 434 ASSERT(buf->b_data != NULL); 435 db->db.db_data = buf->b_data; 436 if (!arc_released(buf)) 437 arc_set_callback(buf, dbuf_do_evict, db); 438 dbuf_update_data(db); 439 } else { 440 dbuf_evict_user(db); 441 db->db.db_data = NULL; 442 if (db->db_state != DB_NOFILL) 443 db->db_state = DB_UNCACHED; 444 } 445 } 446 447 /* 448 * Loan out an arc_buf for read. Return the loaned arc_buf. 449 */ 450 arc_buf_t * 451 dbuf_loan_arcbuf(dmu_buf_impl_t *db) 452 { 453 arc_buf_t *abuf; 454 455 mutex_enter(&db->db_mtx); 456 if (arc_released(db->db_buf) || refcount_count(&db->db_holds) > 1) { 457 int blksz = db->db.db_size; 458 spa_t *spa; 459 460 mutex_exit(&db->db_mtx); 461 DB_GET_SPA(&spa, db); 462 abuf = arc_loan_buf(spa, blksz); 463 bcopy(db->db.db_data, abuf->b_data, blksz); 464 } else { 465 abuf = db->db_buf; 466 arc_loan_inuse_buf(abuf, db); 467 dbuf_set_data(db, NULL); 468 mutex_exit(&db->db_mtx); 469 } 470 return (abuf); 471 } 472 473 uint64_t 474 dbuf_whichblock(dnode_t *dn, uint64_t offset) 475 { 476 if (dn->dn_datablkshift) { 477 return (offset >> dn->dn_datablkshift); 478 } else { 479 ASSERT3U(offset, <, dn->dn_datablksz); 480 return (0); 481 } 482 } 483 484 static void 485 dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb) 486 { 487 dmu_buf_impl_t *db = vdb; 488 489 mutex_enter(&db->db_mtx); 490 ASSERT3U(db->db_state, ==, DB_READ); 491 /* 492 * All reads are synchronous, so we must have a hold on the dbuf 493 */ 494 ASSERT(refcount_count(&db->db_holds) > 0); 495 ASSERT(db->db_buf == NULL); 496 ASSERT(db->db.db_data == NULL); 497 if (db->db_level == 0 && db->db_freed_in_flight) { 498 /* we were freed in flight; disregard any error */ 499 arc_release(buf, db); 500 bzero(buf->b_data, db->db.db_size); 501 arc_buf_freeze(buf); 502 db->db_freed_in_flight = FALSE; 503 dbuf_set_data(db, buf); 504 db->db_state = DB_CACHED; 505 } else if (zio == NULL || zio->io_error == 0) { 506 dbuf_set_data(db, buf); 507 db->db_state = DB_CACHED; 508 } else { 509 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 510 ASSERT3P(db->db_buf, ==, NULL); 511 VERIFY(arc_buf_remove_ref(buf, db)); 512 db->db_state = DB_UNCACHED; 513 } 514 cv_broadcast(&db->db_changed); 515 dbuf_rele_and_unlock(db, NULL); 516 } 517 518 static void 519 dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t *flags) 520 { 521 dnode_t *dn; 522 spa_t *spa; 523 zbookmark_t zb; 524 uint32_t aflags = ARC_NOWAIT; 525 526 DB_DNODE_ENTER(db); 527 dn = DB_DNODE(db); 528 ASSERT(!refcount_is_zero(&db->db_holds)); 529 /* We need the struct_rwlock to prevent db_blkptr from changing. */ 530 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 531 ASSERT(MUTEX_HELD(&db->db_mtx)); 532 ASSERT(db->db_state == DB_UNCACHED); 533 ASSERT(db->db_buf == NULL); 534 535 if (db->db_blkid == DMU_BONUS_BLKID) { 536 int bonuslen = MIN(dn->dn_bonuslen, dn->dn_phys->dn_bonuslen); 537 538 ASSERT3U(bonuslen, <=, db->db.db_size); 539 db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN); 540 arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 541 if (bonuslen < DN_MAX_BONUSLEN) 542 bzero(db->db.db_data, DN_MAX_BONUSLEN); 543 if (bonuslen) 544 bcopy(DN_BONUS(dn->dn_phys), db->db.db_data, bonuslen); 545 DB_DNODE_EXIT(db); 546 dbuf_update_data(db); 547 db->db_state = DB_CACHED; 548 mutex_exit(&db->db_mtx); 549 return; 550 } 551 552 /* 553 * Recheck BP_IS_HOLE() after dnode_block_freed() in case dnode_sync() 554 * processes the delete record and clears the bp while we are waiting 555 * for the dn_mtx (resulting in a "no" from block_freed). 556 */ 557 if (db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr) || 558 (db->db_level == 0 && (dnode_block_freed(dn, db->db_blkid) || 559 BP_IS_HOLE(db->db_blkptr)))) { 560 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 561 562 dbuf_set_data(db, arc_buf_alloc(dn->dn_objset->os_spa, 563 db->db.db_size, db, type)); 564 DB_DNODE_EXIT(db); 565 bzero(db->db.db_data, db->db.db_size); 566 db->db_state = DB_CACHED; 567 *flags |= DB_RF_CACHED; 568 mutex_exit(&db->db_mtx); 569 return; 570 } 571 572 spa = dn->dn_objset->os_spa; 573 DB_DNODE_EXIT(db); 574 575 db->db_state = DB_READ; 576 mutex_exit(&db->db_mtx); 577 578 if (DBUF_IS_L2CACHEABLE(db)) 579 aflags |= ARC_L2CACHE; 580 if (DBUF_IS_L2COMPRESSIBLE(db)) 581 aflags |= ARC_L2COMPRESS; 582 583 SET_BOOKMARK(&zb, db->db_objset->os_dsl_dataset ? 584 db->db_objset->os_dsl_dataset->ds_object : DMU_META_OBJSET, 585 db->db.db_object, db->db_level, db->db_blkid); 586 587 dbuf_add_ref(db, NULL); 588 589 (void) arc_read(zio, spa, db->db_blkptr, 590 dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ, 591 (*flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED, 592 &aflags, &zb); 593 if (aflags & ARC_CACHED) 594 *flags |= DB_RF_CACHED; 595 } 596 597 int 598 dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags) 599 { 600 int err = 0; 601 int havepzio = (zio != NULL); 602 int prefetch; 603 dnode_t *dn; 604 605 /* 606 * We don't have to hold the mutex to check db_state because it 607 * can't be freed while we have a hold on the buffer. 608 */ 609 ASSERT(!refcount_is_zero(&db->db_holds)); 610 611 if (db->db_state == DB_NOFILL) 612 return (SET_ERROR(EIO)); 613 614 DB_DNODE_ENTER(db); 615 dn = DB_DNODE(db); 616 if ((flags & DB_RF_HAVESTRUCT) == 0) 617 rw_enter(&dn->dn_struct_rwlock, RW_READER); 618 619 prefetch = db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 620 (flags & DB_RF_NOPREFETCH) == 0 && dn != NULL && 621 DBUF_IS_CACHEABLE(db); 622 623 mutex_enter(&db->db_mtx); 624 if (db->db_state == DB_CACHED) { 625 mutex_exit(&db->db_mtx); 626 if (prefetch) 627 dmu_zfetch(&dn->dn_zfetch, db->db.db_offset, 628 db->db.db_size, TRUE); 629 if ((flags & DB_RF_HAVESTRUCT) == 0) 630 rw_exit(&dn->dn_struct_rwlock); 631 DB_DNODE_EXIT(db); 632 } else if (db->db_state == DB_UNCACHED) { 633 spa_t *spa = dn->dn_objset->os_spa; 634 635 if (zio == NULL) 636 zio = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL); 637 dbuf_read_impl(db, zio, &flags); 638 639 /* dbuf_read_impl has dropped db_mtx for us */ 640 641 if (prefetch) 642 dmu_zfetch(&dn->dn_zfetch, db->db.db_offset, 643 db->db.db_size, flags & DB_RF_CACHED); 644 645 if ((flags & DB_RF_HAVESTRUCT) == 0) 646 rw_exit(&dn->dn_struct_rwlock); 647 DB_DNODE_EXIT(db); 648 649 if (!havepzio) 650 err = zio_wait(zio); 651 } else { 652 /* 653 * Another reader came in while the dbuf was in flight 654 * between UNCACHED and CACHED. Either a writer will finish 655 * writing the buffer (sending the dbuf to CACHED) or the 656 * first reader's request will reach the read_done callback 657 * and send the dbuf to CACHED. Otherwise, a failure 658 * occurred and the dbuf went to UNCACHED. 659 */ 660 mutex_exit(&db->db_mtx); 661 if (prefetch) 662 dmu_zfetch(&dn->dn_zfetch, db->db.db_offset, 663 db->db.db_size, TRUE); 664 if ((flags & DB_RF_HAVESTRUCT) == 0) 665 rw_exit(&dn->dn_struct_rwlock); 666 DB_DNODE_EXIT(db); 667 668 /* Skip the wait per the caller's request. */ 669 mutex_enter(&db->db_mtx); 670 if ((flags & DB_RF_NEVERWAIT) == 0) { 671 while (db->db_state == DB_READ || 672 db->db_state == DB_FILL) { 673 ASSERT(db->db_state == DB_READ || 674 (flags & DB_RF_HAVESTRUCT) == 0); 675 cv_wait(&db->db_changed, &db->db_mtx); 676 } 677 if (db->db_state == DB_UNCACHED) 678 err = SET_ERROR(EIO); 679 } 680 mutex_exit(&db->db_mtx); 681 } 682 683 ASSERT(err || havepzio || db->db_state == DB_CACHED); 684 return (err); 685 } 686 687 static void 688 dbuf_noread(dmu_buf_impl_t *db) 689 { 690 ASSERT(!refcount_is_zero(&db->db_holds)); 691 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 692 mutex_enter(&db->db_mtx); 693 while (db->db_state == DB_READ || db->db_state == DB_FILL) 694 cv_wait(&db->db_changed, &db->db_mtx); 695 if (db->db_state == DB_UNCACHED) { 696 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 697 spa_t *spa; 698 699 ASSERT(db->db_buf == NULL); 700 ASSERT(db->db.db_data == NULL); 701 DB_GET_SPA(&spa, db); 702 dbuf_set_data(db, arc_buf_alloc(spa, db->db.db_size, db, type)); 703 db->db_state = DB_FILL; 704 } else if (db->db_state == DB_NOFILL) { 705 dbuf_set_data(db, NULL); 706 } else { 707 ASSERT3U(db->db_state, ==, DB_CACHED); 708 } 709 mutex_exit(&db->db_mtx); 710 } 711 712 /* 713 * This is our just-in-time copy function. It makes a copy of 714 * buffers, that have been modified in a previous transaction 715 * group, before we modify them in the current active group. 716 * 717 * This function is used in two places: when we are dirtying a 718 * buffer for the first time in a txg, and when we are freeing 719 * a range in a dnode that includes this buffer. 720 * 721 * Note that when we are called from dbuf_free_range() we do 722 * not put a hold on the buffer, we just traverse the active 723 * dbuf list for the dnode. 724 */ 725 static void 726 dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg) 727 { 728 dbuf_dirty_record_t *dr = db->db_last_dirty; 729 730 ASSERT(MUTEX_HELD(&db->db_mtx)); 731 ASSERT(db->db.db_data != NULL); 732 ASSERT(db->db_level == 0); 733 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT); 734 735 if (dr == NULL || 736 (dr->dt.dl.dr_data != 737 ((db->db_blkid == DMU_BONUS_BLKID) ? db->db.db_data : db->db_buf))) 738 return; 739 740 /* 741 * If the last dirty record for this dbuf has not yet synced 742 * and its referencing the dbuf data, either: 743 * reset the reference to point to a new copy, 744 * or (if there a no active holders) 745 * just null out the current db_data pointer. 746 */ 747 ASSERT(dr->dr_txg >= txg - 2); 748 if (db->db_blkid == DMU_BONUS_BLKID) { 749 /* Note that the data bufs here are zio_bufs */ 750 dr->dt.dl.dr_data = zio_buf_alloc(DN_MAX_BONUSLEN); 751 arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 752 bcopy(db->db.db_data, dr->dt.dl.dr_data, DN_MAX_BONUSLEN); 753 } else if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 754 int size = db->db.db_size; 755 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 756 spa_t *spa; 757 758 DB_GET_SPA(&spa, db); 759 dr->dt.dl.dr_data = arc_buf_alloc(spa, size, db, type); 760 bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size); 761 } else { 762 dbuf_set_data(db, NULL); 763 } 764 } 765 766 void 767 dbuf_unoverride(dbuf_dirty_record_t *dr) 768 { 769 dmu_buf_impl_t *db = dr->dr_dbuf; 770 blkptr_t *bp = &dr->dt.dl.dr_overridden_by; 771 uint64_t txg = dr->dr_txg; 772 773 ASSERT(MUTEX_HELD(&db->db_mtx)); 774 ASSERT(dr->dt.dl.dr_override_state != DR_IN_DMU_SYNC); 775 ASSERT(db->db_level == 0); 776 777 if (db->db_blkid == DMU_BONUS_BLKID || 778 dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN) 779 return; 780 781 ASSERT(db->db_data_pending != dr); 782 783 /* free this block */ 784 if (!BP_IS_HOLE(bp) && !dr->dt.dl.dr_nopwrite) { 785 spa_t *spa; 786 787 DB_GET_SPA(&spa, db); 788 zio_free(spa, txg, bp); 789 } 790 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 791 dr->dt.dl.dr_nopwrite = B_FALSE; 792 793 /* 794 * Release the already-written buffer, so we leave it in 795 * a consistent dirty state. Note that all callers are 796 * modifying the buffer, so they will immediately do 797 * another (redundant) arc_release(). Therefore, leave 798 * the buf thawed to save the effort of freezing & 799 * immediately re-thawing it. 800 */ 801 arc_release(dr->dt.dl.dr_data, db); 802 } 803 804 /* 805 * Evict (if its unreferenced) or clear (if its referenced) any level-0 806 * data blocks in the free range, so that any future readers will find 807 * empty blocks. Also, if we happen across any level-1 dbufs in the 808 * range that have not already been marked dirty, mark them dirty so 809 * they stay in memory. 810 * 811 * This is a no-op if the dataset is in the middle of an incremental 812 * receive; see comment below for details. 813 */ 814 void 815 dbuf_free_range(dnode_t *dn, uint64_t start, uint64_t end, dmu_tx_t *tx) 816 { 817 dmu_buf_impl_t *db, *db_next; 818 uint64_t txg = tx->tx_txg; 819 int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 820 uint64_t first_l1 = start >> epbs; 821 uint64_t last_l1 = end >> epbs; 822 823 if (end > dn->dn_maxblkid && (end != DMU_SPILL_BLKID)) { 824 end = dn->dn_maxblkid; 825 last_l1 = end >> epbs; 826 } 827 dprintf_dnode(dn, "start=%llu end=%llu\n", start, end); 828 829 mutex_enter(&dn->dn_dbufs_mtx); 830 if (start >= dn->dn_unlisted_l0_blkid * dn->dn_datablksz) { 831 /* There can't be any dbufs in this range; no need to search. */ 832 mutex_exit(&dn->dn_dbufs_mtx); 833 return; 834 } else if (dmu_objset_is_receiving(dn->dn_objset)) { 835 /* 836 * If we are receiving, we expect there to be no dbufs in 837 * the range to be freed, because receive modifies each 838 * block at most once, and in offset order. If this is 839 * not the case, it can lead to performance problems, 840 * so note that we unexpectedly took the slow path. 841 */ 842 atomic_inc_64(&zfs_free_range_recv_miss); 843 } 844 845 for (db = list_head(&dn->dn_dbufs); db != NULL; db = db_next) { 846 db_next = list_next(&dn->dn_dbufs, db); 847 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 848 849 if (db->db_level == 1 && 850 db->db_blkid >= first_l1 && db->db_blkid <= last_l1) { 851 mutex_enter(&db->db_mtx); 852 if (db->db_last_dirty && 853 db->db_last_dirty->dr_txg < txg) { 854 dbuf_add_ref(db, FTAG); 855 mutex_exit(&db->db_mtx); 856 dbuf_will_dirty(db, tx); 857 dbuf_rele(db, FTAG); 858 } else { 859 mutex_exit(&db->db_mtx); 860 } 861 } 862 863 if (db->db_level != 0) 864 continue; 865 dprintf_dbuf(db, "found buf %s\n", ""); 866 if (db->db_blkid < start || db->db_blkid > end) 867 continue; 868 869 /* found a level 0 buffer in the range */ 870 mutex_enter(&db->db_mtx); 871 if (dbuf_undirty(db, tx)) { 872 /* mutex has been dropped and dbuf destroyed */ 873 continue; 874 } 875 876 if (db->db_state == DB_UNCACHED || 877 db->db_state == DB_NOFILL || 878 db->db_state == DB_EVICTING) { 879 ASSERT(db->db.db_data == NULL); 880 mutex_exit(&db->db_mtx); 881 continue; 882 } 883 if (db->db_state == DB_READ || db->db_state == DB_FILL) { 884 /* will be handled in dbuf_read_done or dbuf_rele */ 885 db->db_freed_in_flight = TRUE; 886 mutex_exit(&db->db_mtx); 887 continue; 888 } 889 if (refcount_count(&db->db_holds) == 0) { 890 ASSERT(db->db_buf); 891 dbuf_clear(db); 892 continue; 893 } 894 /* The dbuf is referenced */ 895 896 if (db->db_last_dirty != NULL) { 897 dbuf_dirty_record_t *dr = db->db_last_dirty; 898 899 if (dr->dr_txg == txg) { 900 /* 901 * This buffer is "in-use", re-adjust the file 902 * size to reflect that this buffer may 903 * contain new data when we sync. 904 */ 905 if (db->db_blkid != DMU_SPILL_BLKID && 906 db->db_blkid > dn->dn_maxblkid) 907 dn->dn_maxblkid = db->db_blkid; 908 dbuf_unoverride(dr); 909 } else { 910 /* 911 * This dbuf is not dirty in the open context. 912 * Either uncache it (if its not referenced in 913 * the open context) or reset its contents to 914 * empty. 915 */ 916 dbuf_fix_old_data(db, txg); 917 } 918 } 919 /* clear the contents if its cached */ 920 if (db->db_state == DB_CACHED) { 921 ASSERT(db->db.db_data != NULL); 922 arc_release(db->db_buf, db); 923 bzero(db->db.db_data, db->db.db_size); 924 arc_buf_freeze(db->db_buf); 925 } 926 927 mutex_exit(&db->db_mtx); 928 } 929 mutex_exit(&dn->dn_dbufs_mtx); 930 } 931 932 static int 933 dbuf_block_freeable(dmu_buf_impl_t *db) 934 { 935 dsl_dataset_t *ds = db->db_objset->os_dsl_dataset; 936 uint64_t birth_txg = 0; 937 938 /* 939 * We don't need any locking to protect db_blkptr: 940 * If it's syncing, then db_last_dirty will be set 941 * so we'll ignore db_blkptr. 942 */ 943 ASSERT(MUTEX_HELD(&db->db_mtx)); 944 if (db->db_last_dirty) 945 birth_txg = db->db_last_dirty->dr_txg; 946 else if (db->db_blkptr) 947 birth_txg = db->db_blkptr->blk_birth; 948 949 /* 950 * If we don't exist or are in a snapshot, we can't be freed. 951 * Don't pass the bp to dsl_dataset_block_freeable() since we 952 * are holding the db_mtx lock and might deadlock if we are 953 * prefetching a dedup-ed block. 954 */ 955 if (birth_txg) 956 return (ds == NULL || 957 dsl_dataset_block_freeable(ds, NULL, birth_txg)); 958 else 959 return (FALSE); 960 } 961 962 void 963 dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx) 964 { 965 arc_buf_t *buf, *obuf; 966 int osize = db->db.db_size; 967 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 968 dnode_t *dn; 969 970 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 971 972 DB_DNODE_ENTER(db); 973 dn = DB_DNODE(db); 974 975 /* XXX does *this* func really need the lock? */ 976 ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 977 978 /* 979 * This call to dbuf_will_dirty() with the dn_struct_rwlock held 980 * is OK, because there can be no other references to the db 981 * when we are changing its size, so no concurrent DB_FILL can 982 * be happening. 983 */ 984 /* 985 * XXX we should be doing a dbuf_read, checking the return 986 * value and returning that up to our callers 987 */ 988 dbuf_will_dirty(db, tx); 989 990 /* create the data buffer for the new block */ 991 buf = arc_buf_alloc(dn->dn_objset->os_spa, size, db, type); 992 993 /* copy old block data to the new block */ 994 obuf = db->db_buf; 995 bcopy(obuf->b_data, buf->b_data, MIN(osize, size)); 996 /* zero the remainder */ 997 if (size > osize) 998 bzero((uint8_t *)buf->b_data + osize, size - osize); 999 1000 mutex_enter(&db->db_mtx); 1001 dbuf_set_data(db, buf); 1002 VERIFY(arc_buf_remove_ref(obuf, db)); 1003 db->db.db_size = size; 1004 1005 if (db->db_level == 0) { 1006 ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg); 1007 db->db_last_dirty->dt.dl.dr_data = buf; 1008 } 1009 mutex_exit(&db->db_mtx); 1010 1011 dnode_willuse_space(dn, size-osize, tx); 1012 DB_DNODE_EXIT(db); 1013 } 1014 1015 void 1016 dbuf_release_bp(dmu_buf_impl_t *db) 1017 { 1018 objset_t *os; 1019 1020 DB_GET_OBJSET(&os, db); 1021 ASSERT(dsl_pool_sync_context(dmu_objset_pool(os))); 1022 ASSERT(arc_released(os->os_phys_buf) || 1023 list_link_active(&os->os_dsl_dataset->ds_synced_link)); 1024 ASSERT(db->db_parent == NULL || arc_released(db->db_parent->db_buf)); 1025 1026 (void) arc_release(db->db_buf, db); 1027 } 1028 1029 dbuf_dirty_record_t * 1030 dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1031 { 1032 dnode_t *dn; 1033 objset_t *os; 1034 dbuf_dirty_record_t **drp, *dr; 1035 int drop_struct_lock = FALSE; 1036 boolean_t do_free_accounting = B_FALSE; 1037 int txgoff = tx->tx_txg & TXG_MASK; 1038 1039 ASSERT(tx->tx_txg != 0); 1040 ASSERT(!refcount_is_zero(&db->db_holds)); 1041 DMU_TX_DIRTY_BUF(tx, db); 1042 1043 DB_DNODE_ENTER(db); 1044 dn = DB_DNODE(db); 1045 /* 1046 * Shouldn't dirty a regular buffer in syncing context. Private 1047 * objects may be dirtied in syncing context, but only if they 1048 * were already pre-dirtied in open context. 1049 */ 1050 ASSERT(!dmu_tx_is_syncing(tx) || 1051 BP_IS_HOLE(dn->dn_objset->os_rootbp) || 1052 DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 1053 dn->dn_objset->os_dsl_dataset == NULL); 1054 /* 1055 * We make this assert for private objects as well, but after we 1056 * check if we're already dirty. They are allowed to re-dirty 1057 * in syncing context. 1058 */ 1059 ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 1060 dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 1061 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 1062 1063 mutex_enter(&db->db_mtx); 1064 /* 1065 * XXX make this true for indirects too? The problem is that 1066 * transactions created with dmu_tx_create_assigned() from 1067 * syncing context don't bother holding ahead. 1068 */ 1069 ASSERT(db->db_level != 0 || 1070 db->db_state == DB_CACHED || db->db_state == DB_FILL || 1071 db->db_state == DB_NOFILL); 1072 1073 mutex_enter(&dn->dn_mtx); 1074 /* 1075 * Don't set dirtyctx to SYNC if we're just modifying this as we 1076 * initialize the objset. 1077 */ 1078 if (dn->dn_dirtyctx == DN_UNDIRTIED && 1079 !BP_IS_HOLE(dn->dn_objset->os_rootbp)) { 1080 dn->dn_dirtyctx = 1081 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN); 1082 ASSERT(dn->dn_dirtyctx_firstset == NULL); 1083 dn->dn_dirtyctx_firstset = kmem_alloc(1, KM_SLEEP); 1084 } 1085 mutex_exit(&dn->dn_mtx); 1086 1087 if (db->db_blkid == DMU_SPILL_BLKID) 1088 dn->dn_have_spill = B_TRUE; 1089 1090 /* 1091 * If this buffer is already dirty, we're done. 1092 */ 1093 drp = &db->db_last_dirty; 1094 ASSERT(*drp == NULL || (*drp)->dr_txg <= tx->tx_txg || 1095 db->db.db_object == DMU_META_DNODE_OBJECT); 1096 while ((dr = *drp) != NULL && dr->dr_txg > tx->tx_txg) 1097 drp = &dr->dr_next; 1098 if (dr && dr->dr_txg == tx->tx_txg) { 1099 DB_DNODE_EXIT(db); 1100 1101 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID) { 1102 /* 1103 * If this buffer has already been written out, 1104 * we now need to reset its state. 1105 */ 1106 dbuf_unoverride(dr); 1107 if (db->db.db_object != DMU_META_DNODE_OBJECT && 1108 db->db_state != DB_NOFILL) 1109 arc_buf_thaw(db->db_buf); 1110 } 1111 mutex_exit(&db->db_mtx); 1112 return (dr); 1113 } 1114 1115 /* 1116 * Only valid if not already dirty. 1117 */ 1118 ASSERT(dn->dn_object == 0 || 1119 dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 1120 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 1121 1122 ASSERT3U(dn->dn_nlevels, >, db->db_level); 1123 ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) || 1124 dn->dn_phys->dn_nlevels > db->db_level || 1125 dn->dn_next_nlevels[txgoff] > db->db_level || 1126 dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level || 1127 dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level); 1128 1129 /* 1130 * We should only be dirtying in syncing context if it's the 1131 * mos or we're initializing the os or it's a special object. 1132 * However, we are allowed to dirty in syncing context provided 1133 * we already dirtied it in open context. Hence we must make 1134 * this assertion only if we're not already dirty. 1135 */ 1136 os = dn->dn_objset; 1137 ASSERT(!dmu_tx_is_syncing(tx) || DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 1138 os->os_dsl_dataset == NULL || BP_IS_HOLE(os->os_rootbp)); 1139 ASSERT(db->db.db_size != 0); 1140 1141 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1142 1143 if (db->db_blkid != DMU_BONUS_BLKID) { 1144 /* 1145 * Update the accounting. 1146 * Note: we delay "free accounting" until after we drop 1147 * the db_mtx. This keeps us from grabbing other locks 1148 * (and possibly deadlocking) in bp_get_dsize() while 1149 * also holding the db_mtx. 1150 */ 1151 dnode_willuse_space(dn, db->db.db_size, tx); 1152 do_free_accounting = dbuf_block_freeable(db); 1153 } 1154 1155 /* 1156 * If this buffer is dirty in an old transaction group we need 1157 * to make a copy of it so that the changes we make in this 1158 * transaction group won't leak out when we sync the older txg. 1159 */ 1160 dr = kmem_zalloc(sizeof (dbuf_dirty_record_t), KM_SLEEP); 1161 if (db->db_level == 0) { 1162 void *data_old = db->db_buf; 1163 1164 if (db->db_state != DB_NOFILL) { 1165 if (db->db_blkid == DMU_BONUS_BLKID) { 1166 dbuf_fix_old_data(db, tx->tx_txg); 1167 data_old = db->db.db_data; 1168 } else if (db->db.db_object != DMU_META_DNODE_OBJECT) { 1169 /* 1170 * Release the data buffer from the cache so 1171 * that we can modify it without impacting 1172 * possible other users of this cached data 1173 * block. Note that indirect blocks and 1174 * private objects are not released until the 1175 * syncing state (since they are only modified 1176 * then). 1177 */ 1178 arc_release(db->db_buf, db); 1179 dbuf_fix_old_data(db, tx->tx_txg); 1180 data_old = db->db_buf; 1181 } 1182 ASSERT(data_old != NULL); 1183 } 1184 dr->dt.dl.dr_data = data_old; 1185 } else { 1186 mutex_init(&dr->dt.di.dr_mtx, NULL, MUTEX_DEFAULT, NULL); 1187 list_create(&dr->dt.di.dr_children, 1188 sizeof (dbuf_dirty_record_t), 1189 offsetof(dbuf_dirty_record_t, dr_dirty_node)); 1190 } 1191 if (db->db_blkid != DMU_BONUS_BLKID && os->os_dsl_dataset != NULL) 1192 dr->dr_accounted = db->db.db_size; 1193 dr->dr_dbuf = db; 1194 dr->dr_txg = tx->tx_txg; 1195 dr->dr_next = *drp; 1196 *drp = dr; 1197 1198 /* 1199 * We could have been freed_in_flight between the dbuf_noread 1200 * and dbuf_dirty. We win, as though the dbuf_noread() had 1201 * happened after the free. 1202 */ 1203 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 1204 db->db_blkid != DMU_SPILL_BLKID) { 1205 mutex_enter(&dn->dn_mtx); 1206 dnode_clear_range(dn, db->db_blkid, 1, tx); 1207 mutex_exit(&dn->dn_mtx); 1208 db->db_freed_in_flight = FALSE; 1209 } 1210 1211 /* 1212 * This buffer is now part of this txg 1213 */ 1214 dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg); 1215 db->db_dirtycnt += 1; 1216 ASSERT3U(db->db_dirtycnt, <=, 3); 1217 1218 mutex_exit(&db->db_mtx); 1219 1220 if (db->db_blkid == DMU_BONUS_BLKID || 1221 db->db_blkid == DMU_SPILL_BLKID) { 1222 mutex_enter(&dn->dn_mtx); 1223 ASSERT(!list_link_active(&dr->dr_dirty_node)); 1224 list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1225 mutex_exit(&dn->dn_mtx); 1226 dnode_setdirty(dn, tx); 1227 DB_DNODE_EXIT(db); 1228 return (dr); 1229 } else if (do_free_accounting) { 1230 blkptr_t *bp = db->db_blkptr; 1231 int64_t willfree = (bp && !BP_IS_HOLE(bp)) ? 1232 bp_get_dsize(os->os_spa, bp) : db->db.db_size; 1233 /* 1234 * This is only a guess -- if the dbuf is dirty 1235 * in a previous txg, we don't know how much 1236 * space it will use on disk yet. We should 1237 * really have the struct_rwlock to access 1238 * db_blkptr, but since this is just a guess, 1239 * it's OK if we get an odd answer. 1240 */ 1241 ddt_prefetch(os->os_spa, bp); 1242 dnode_willuse_space(dn, -willfree, tx); 1243 } 1244 1245 if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 1246 rw_enter(&dn->dn_struct_rwlock, RW_READER); 1247 drop_struct_lock = TRUE; 1248 } 1249 1250 if (db->db_level == 0) { 1251 dnode_new_blkid(dn, db->db_blkid, tx, drop_struct_lock); 1252 ASSERT(dn->dn_maxblkid >= db->db_blkid); 1253 } 1254 1255 if (db->db_level+1 < dn->dn_nlevels) { 1256 dmu_buf_impl_t *parent = db->db_parent; 1257 dbuf_dirty_record_t *di; 1258 int parent_held = FALSE; 1259 1260 if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) { 1261 int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1262 1263 parent = dbuf_hold_level(dn, db->db_level+1, 1264 db->db_blkid >> epbs, FTAG); 1265 ASSERT(parent != NULL); 1266 parent_held = TRUE; 1267 } 1268 if (drop_struct_lock) 1269 rw_exit(&dn->dn_struct_rwlock); 1270 ASSERT3U(db->db_level+1, ==, parent->db_level); 1271 di = dbuf_dirty(parent, tx); 1272 if (parent_held) 1273 dbuf_rele(parent, FTAG); 1274 1275 mutex_enter(&db->db_mtx); 1276 /* 1277 * Since we've dropped the mutex, it's possible that 1278 * dbuf_undirty() might have changed this out from under us. 1279 */ 1280 if (db->db_last_dirty == dr || 1281 dn->dn_object == DMU_META_DNODE_OBJECT) { 1282 mutex_enter(&di->dt.di.dr_mtx); 1283 ASSERT3U(di->dr_txg, ==, tx->tx_txg); 1284 ASSERT(!list_link_active(&dr->dr_dirty_node)); 1285 list_insert_tail(&di->dt.di.dr_children, dr); 1286 mutex_exit(&di->dt.di.dr_mtx); 1287 dr->dr_parent = di; 1288 } 1289 mutex_exit(&db->db_mtx); 1290 } else { 1291 ASSERT(db->db_level+1 == dn->dn_nlevels); 1292 ASSERT(db->db_blkid < dn->dn_nblkptr); 1293 ASSERT(db->db_parent == NULL || db->db_parent == dn->dn_dbuf); 1294 mutex_enter(&dn->dn_mtx); 1295 ASSERT(!list_link_active(&dr->dr_dirty_node)); 1296 list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1297 mutex_exit(&dn->dn_mtx); 1298 if (drop_struct_lock) 1299 rw_exit(&dn->dn_struct_rwlock); 1300 } 1301 1302 dnode_setdirty(dn, tx); 1303 DB_DNODE_EXIT(db); 1304 return (dr); 1305 } 1306 1307 /* 1308 * Undirty a buffer in the transaction group referenced by the given 1309 * transaction. Return whether this evicted the dbuf. 1310 */ 1311 static boolean_t 1312 dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1313 { 1314 dnode_t *dn; 1315 uint64_t txg = tx->tx_txg; 1316 dbuf_dirty_record_t *dr, **drp; 1317 1318 ASSERT(txg != 0); 1319 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1320 ASSERT0(db->db_level); 1321 ASSERT(MUTEX_HELD(&db->db_mtx)); 1322 1323 /* 1324 * If this buffer is not dirty, we're done. 1325 */ 1326 for (drp = &db->db_last_dirty; (dr = *drp) != NULL; drp = &dr->dr_next) 1327 if (dr->dr_txg <= txg) 1328 break; 1329 if (dr == NULL || dr->dr_txg < txg) 1330 return (B_FALSE); 1331 ASSERT(dr->dr_txg == txg); 1332 ASSERT(dr->dr_dbuf == db); 1333 1334 DB_DNODE_ENTER(db); 1335 dn = DB_DNODE(db); 1336 1337 /* 1338 * Note: This code will probably work even if there are concurrent 1339 * holders, but it is untested in that scenerio, as the ZPL and 1340 * ztest have additional locking (the range locks) that prevents 1341 * that type of concurrent access. 1342 */ 1343 ASSERT3U(refcount_count(&db->db_holds), ==, db->db_dirtycnt); 1344 1345 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1346 1347 ASSERT(db->db.db_size != 0); 1348 1349 /* 1350 * Any space we accounted for in dp_dirty_* will be cleaned up by 1351 * dsl_pool_sync(). This is relatively rare so the discrepancy 1352 * is not a big deal. 1353 */ 1354 1355 *drp = dr->dr_next; 1356 1357 /* 1358 * Note that there are three places in dbuf_dirty() 1359 * where this dirty record may be put on a list. 1360 * Make sure to do a list_remove corresponding to 1361 * every one of those list_insert calls. 1362 */ 1363 if (dr->dr_parent) { 1364 mutex_enter(&dr->dr_parent->dt.di.dr_mtx); 1365 list_remove(&dr->dr_parent->dt.di.dr_children, dr); 1366 mutex_exit(&dr->dr_parent->dt.di.dr_mtx); 1367 } else if (db->db_blkid == DMU_SPILL_BLKID || 1368 db->db_level+1 == dn->dn_nlevels) { 1369 ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf); 1370 mutex_enter(&dn->dn_mtx); 1371 list_remove(&dn->dn_dirty_records[txg & TXG_MASK], dr); 1372 mutex_exit(&dn->dn_mtx); 1373 } 1374 DB_DNODE_EXIT(db); 1375 1376 if (db->db_state != DB_NOFILL) { 1377 dbuf_unoverride(dr); 1378 1379 ASSERT(db->db_buf != NULL); 1380 ASSERT(dr->dt.dl.dr_data != NULL); 1381 if (dr->dt.dl.dr_data != db->db_buf) 1382 VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, db)); 1383 } 1384 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 1385 1386 ASSERT(db->db_dirtycnt > 0); 1387 db->db_dirtycnt -= 1; 1388 1389 if (refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) { 1390 arc_buf_t *buf = db->db_buf; 1391 1392 ASSERT(db->db_state == DB_NOFILL || arc_released(buf)); 1393 dbuf_set_data(db, NULL); 1394 VERIFY(arc_buf_remove_ref(buf, db)); 1395 dbuf_evict(db); 1396 return (B_TRUE); 1397 } 1398 1399 return (B_FALSE); 1400 } 1401 1402 #pragma weak dmu_buf_will_dirty = dbuf_will_dirty 1403 void 1404 dbuf_will_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1405 { 1406 int rf = DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH; 1407 1408 ASSERT(tx->tx_txg != 0); 1409 ASSERT(!refcount_is_zero(&db->db_holds)); 1410 1411 DB_DNODE_ENTER(db); 1412 if (RW_WRITE_HELD(&DB_DNODE(db)->dn_struct_rwlock)) 1413 rf |= DB_RF_HAVESTRUCT; 1414 DB_DNODE_EXIT(db); 1415 (void) dbuf_read(db, NULL, rf); 1416 (void) dbuf_dirty(db, tx); 1417 } 1418 1419 void 1420 dmu_buf_will_not_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 1421 { 1422 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1423 1424 db->db_state = DB_NOFILL; 1425 1426 dmu_buf_will_fill(db_fake, tx); 1427 } 1428 1429 void 1430 dmu_buf_will_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 1431 { 1432 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1433 1434 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1435 ASSERT(tx->tx_txg != 0); 1436 ASSERT(db->db_level == 0); 1437 ASSERT(!refcount_is_zero(&db->db_holds)); 1438 1439 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT || 1440 dmu_tx_private_ok(tx)); 1441 1442 dbuf_noread(db); 1443 (void) dbuf_dirty(db, tx); 1444 } 1445 1446 #pragma weak dmu_buf_fill_done = dbuf_fill_done 1447 /* ARGSUSED */ 1448 void 1449 dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx) 1450 { 1451 mutex_enter(&db->db_mtx); 1452 DBUF_VERIFY(db); 1453 1454 if (db->db_state == DB_FILL) { 1455 if (db->db_level == 0 && db->db_freed_in_flight) { 1456 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1457 /* we were freed while filling */ 1458 /* XXX dbuf_undirty? */ 1459 bzero(db->db.db_data, db->db.db_size); 1460 db->db_freed_in_flight = FALSE; 1461 } 1462 db->db_state = DB_CACHED; 1463 cv_broadcast(&db->db_changed); 1464 } 1465 mutex_exit(&db->db_mtx); 1466 } 1467 1468 /* 1469 * Directly assign a provided arc buf to a given dbuf if it's not referenced 1470 * by anybody except our caller. Otherwise copy arcbuf's contents to dbuf. 1471 */ 1472 void 1473 dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx) 1474 { 1475 ASSERT(!refcount_is_zero(&db->db_holds)); 1476 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1477 ASSERT(db->db_level == 0); 1478 ASSERT(DBUF_GET_BUFC_TYPE(db) == ARC_BUFC_DATA); 1479 ASSERT(buf != NULL); 1480 ASSERT(arc_buf_size(buf) == db->db.db_size); 1481 ASSERT(tx->tx_txg != 0); 1482 1483 arc_return_buf(buf, db); 1484 ASSERT(arc_released(buf)); 1485 1486 mutex_enter(&db->db_mtx); 1487 1488 while (db->db_state == DB_READ || db->db_state == DB_FILL) 1489 cv_wait(&db->db_changed, &db->db_mtx); 1490 1491 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_UNCACHED); 1492 1493 if (db->db_state == DB_CACHED && 1494 refcount_count(&db->db_holds) - 1 > db->db_dirtycnt) { 1495 mutex_exit(&db->db_mtx); 1496 (void) dbuf_dirty(db, tx); 1497 bcopy(buf->b_data, db->db.db_data, db->db.db_size); 1498 VERIFY(arc_buf_remove_ref(buf, db)); 1499 xuio_stat_wbuf_copied(); 1500 return; 1501 } 1502 1503 xuio_stat_wbuf_nocopy(); 1504 if (db->db_state == DB_CACHED) { 1505 dbuf_dirty_record_t *dr = db->db_last_dirty; 1506 1507 ASSERT(db->db_buf != NULL); 1508 if (dr != NULL && dr->dr_txg == tx->tx_txg) { 1509 ASSERT(dr->dt.dl.dr_data == db->db_buf); 1510 if (!arc_released(db->db_buf)) { 1511 ASSERT(dr->dt.dl.dr_override_state == 1512 DR_OVERRIDDEN); 1513 arc_release(db->db_buf, db); 1514 } 1515 dr->dt.dl.dr_data = buf; 1516 VERIFY(arc_buf_remove_ref(db->db_buf, db)); 1517 } else if (dr == NULL || dr->dt.dl.dr_data != db->db_buf) { 1518 arc_release(db->db_buf, db); 1519 VERIFY(arc_buf_remove_ref(db->db_buf, db)); 1520 } 1521 db->db_buf = NULL; 1522 } 1523 ASSERT(db->db_buf == NULL); 1524 dbuf_set_data(db, buf); 1525 db->db_state = DB_FILL; 1526 mutex_exit(&db->db_mtx); 1527 (void) dbuf_dirty(db, tx); 1528 dbuf_fill_done(db, tx); 1529 } 1530 1531 /* 1532 * "Clear" the contents of this dbuf. This will mark the dbuf 1533 * EVICTING and clear *most* of its references. Unfortunately, 1534 * when we are not holding the dn_dbufs_mtx, we can't clear the 1535 * entry in the dn_dbufs list. We have to wait until dbuf_destroy() 1536 * in this case. For callers from the DMU we will usually see: 1537 * dbuf_clear()->arc_buf_evict()->dbuf_do_evict()->dbuf_destroy() 1538 * For the arc callback, we will usually see: 1539 * dbuf_do_evict()->dbuf_clear();dbuf_destroy() 1540 * Sometimes, though, we will get a mix of these two: 1541 * DMU: dbuf_clear()->arc_buf_evict() 1542 * ARC: dbuf_do_evict()->dbuf_destroy() 1543 */ 1544 void 1545 dbuf_clear(dmu_buf_impl_t *db) 1546 { 1547 dnode_t *dn; 1548 dmu_buf_impl_t *parent = db->db_parent; 1549 dmu_buf_impl_t *dndb; 1550 int dbuf_gone = FALSE; 1551 1552 ASSERT(MUTEX_HELD(&db->db_mtx)); 1553 ASSERT(refcount_is_zero(&db->db_holds)); 1554 1555 dbuf_evict_user(db); 1556 1557 if (db->db_state == DB_CACHED) { 1558 ASSERT(db->db.db_data != NULL); 1559 if (db->db_blkid == DMU_BONUS_BLKID) { 1560 zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN); 1561 arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 1562 } 1563 db->db.db_data = NULL; 1564 db->db_state = DB_UNCACHED; 1565 } 1566 1567 ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL); 1568 ASSERT(db->db_data_pending == NULL); 1569 1570 db->db_state = DB_EVICTING; 1571 db->db_blkptr = NULL; 1572 1573 DB_DNODE_ENTER(db); 1574 dn = DB_DNODE(db); 1575 dndb = dn->dn_dbuf; 1576 if (db->db_blkid != DMU_BONUS_BLKID && MUTEX_HELD(&dn->dn_dbufs_mtx)) { 1577 list_remove(&dn->dn_dbufs, db); 1578 (void) atomic_dec_32_nv(&dn->dn_dbufs_count); 1579 membar_producer(); 1580 DB_DNODE_EXIT(db); 1581 /* 1582 * Decrementing the dbuf count means that the hold corresponding 1583 * to the removed dbuf is no longer discounted in dnode_move(), 1584 * so the dnode cannot be moved until after we release the hold. 1585 * The membar_producer() ensures visibility of the decremented 1586 * value in dnode_move(), since DB_DNODE_EXIT doesn't actually 1587 * release any lock. 1588 */ 1589 dnode_rele(dn, db); 1590 db->db_dnode_handle = NULL; 1591 } else { 1592 DB_DNODE_EXIT(db); 1593 } 1594 1595 if (db->db_buf) 1596 dbuf_gone = arc_buf_evict(db->db_buf); 1597 1598 if (!dbuf_gone) 1599 mutex_exit(&db->db_mtx); 1600 1601 /* 1602 * If this dbuf is referenced from an indirect dbuf, 1603 * decrement the ref count on the indirect dbuf. 1604 */ 1605 if (parent && parent != dndb) 1606 dbuf_rele(parent, db); 1607 } 1608 1609 static int 1610 dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse, 1611 dmu_buf_impl_t **parentp, blkptr_t **bpp) 1612 { 1613 int nlevels, epbs; 1614 1615 *parentp = NULL; 1616 *bpp = NULL; 1617 1618 ASSERT(blkid != DMU_BONUS_BLKID); 1619 1620 if (blkid == DMU_SPILL_BLKID) { 1621 mutex_enter(&dn->dn_mtx); 1622 if (dn->dn_have_spill && 1623 (dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR)) 1624 *bpp = &dn->dn_phys->dn_spill; 1625 else 1626 *bpp = NULL; 1627 dbuf_add_ref(dn->dn_dbuf, NULL); 1628 *parentp = dn->dn_dbuf; 1629 mutex_exit(&dn->dn_mtx); 1630 return (0); 1631 } 1632 1633 if (dn->dn_phys->dn_nlevels == 0) 1634 nlevels = 1; 1635 else 1636 nlevels = dn->dn_phys->dn_nlevels; 1637 1638 epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1639 1640 ASSERT3U(level * epbs, <, 64); 1641 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1642 if (level >= nlevels || 1643 (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) { 1644 /* the buffer has no parent yet */ 1645 return (SET_ERROR(ENOENT)); 1646 } else if (level < nlevels-1) { 1647 /* this block is referenced from an indirect block */ 1648 int err = dbuf_hold_impl(dn, level+1, 1649 blkid >> epbs, fail_sparse, NULL, parentp); 1650 if (err) 1651 return (err); 1652 err = dbuf_read(*parentp, NULL, 1653 (DB_RF_HAVESTRUCT | DB_RF_NOPREFETCH | DB_RF_CANFAIL)); 1654 if (err) { 1655 dbuf_rele(*parentp, NULL); 1656 *parentp = NULL; 1657 return (err); 1658 } 1659 *bpp = ((blkptr_t *)(*parentp)->db.db_data) + 1660 (blkid & ((1ULL << epbs) - 1)); 1661 return (0); 1662 } else { 1663 /* the block is referenced from the dnode */ 1664 ASSERT3U(level, ==, nlevels-1); 1665 ASSERT(dn->dn_phys->dn_nblkptr == 0 || 1666 blkid < dn->dn_phys->dn_nblkptr); 1667 if (dn->dn_dbuf) { 1668 dbuf_add_ref(dn->dn_dbuf, NULL); 1669 *parentp = dn->dn_dbuf; 1670 } 1671 *bpp = &dn->dn_phys->dn_blkptr[blkid]; 1672 return (0); 1673 } 1674 } 1675 1676 static dmu_buf_impl_t * 1677 dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid, 1678 dmu_buf_impl_t *parent, blkptr_t *blkptr) 1679 { 1680 objset_t *os = dn->dn_objset; 1681 dmu_buf_impl_t *db, *odb; 1682 1683 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1684 ASSERT(dn->dn_type != DMU_OT_NONE); 1685 1686 db = kmem_cache_alloc(dbuf_cache, KM_SLEEP); 1687 1688 db->db_objset = os; 1689 db->db.db_object = dn->dn_object; 1690 db->db_level = level; 1691 db->db_blkid = blkid; 1692 db->db_last_dirty = NULL; 1693 db->db_dirtycnt = 0; 1694 db->db_dnode_handle = dn->dn_handle; 1695 db->db_parent = parent; 1696 db->db_blkptr = blkptr; 1697 1698 db->db_user_ptr = NULL; 1699 db->db_user_data_ptr_ptr = NULL; 1700 db->db_evict_func = NULL; 1701 db->db_immediate_evict = 0; 1702 db->db_freed_in_flight = 0; 1703 1704 if (blkid == DMU_BONUS_BLKID) { 1705 ASSERT3P(parent, ==, dn->dn_dbuf); 1706 db->db.db_size = DN_MAX_BONUSLEN - 1707 (dn->dn_nblkptr-1) * sizeof (blkptr_t); 1708 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 1709 db->db.db_offset = DMU_BONUS_BLKID; 1710 db->db_state = DB_UNCACHED; 1711 /* the bonus dbuf is not placed in the hash table */ 1712 arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1713 return (db); 1714 } else if (blkid == DMU_SPILL_BLKID) { 1715 db->db.db_size = (blkptr != NULL) ? 1716 BP_GET_LSIZE(blkptr) : SPA_MINBLOCKSIZE; 1717 db->db.db_offset = 0; 1718 } else { 1719 int blocksize = 1720 db->db_level ? 1 << dn->dn_indblkshift : dn->dn_datablksz; 1721 db->db.db_size = blocksize; 1722 db->db.db_offset = db->db_blkid * blocksize; 1723 } 1724 1725 /* 1726 * Hold the dn_dbufs_mtx while we get the new dbuf 1727 * in the hash table *and* added to the dbufs list. 1728 * This prevents a possible deadlock with someone 1729 * trying to look up this dbuf before its added to the 1730 * dn_dbufs list. 1731 */ 1732 mutex_enter(&dn->dn_dbufs_mtx); 1733 db->db_state = DB_EVICTING; 1734 if ((odb = dbuf_hash_insert(db)) != NULL) { 1735 /* someone else inserted it first */ 1736 kmem_cache_free(dbuf_cache, db); 1737 mutex_exit(&dn->dn_dbufs_mtx); 1738 return (odb); 1739 } 1740 list_insert_head(&dn->dn_dbufs, db); 1741 if (db->db_level == 0 && db->db_blkid >= 1742 dn->dn_unlisted_l0_blkid) 1743 dn->dn_unlisted_l0_blkid = db->db_blkid + 1; 1744 db->db_state = DB_UNCACHED; 1745 mutex_exit(&dn->dn_dbufs_mtx); 1746 arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1747 1748 if (parent && parent != dn->dn_dbuf) 1749 dbuf_add_ref(parent, db); 1750 1751 ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 1752 refcount_count(&dn->dn_holds) > 0); 1753 (void) refcount_add(&dn->dn_holds, db); 1754 (void) atomic_inc_32_nv(&dn->dn_dbufs_count); 1755 1756 dprintf_dbuf(db, "db=%p\n", db); 1757 1758 return (db); 1759 } 1760 1761 static int 1762 dbuf_do_evict(void *private) 1763 { 1764 arc_buf_t *buf = private; 1765 dmu_buf_impl_t *db = buf->b_private; 1766 1767 if (!MUTEX_HELD(&db->db_mtx)) 1768 mutex_enter(&db->db_mtx); 1769 1770 ASSERT(refcount_is_zero(&db->db_holds)); 1771 1772 if (db->db_state != DB_EVICTING) { 1773 ASSERT(db->db_state == DB_CACHED); 1774 DBUF_VERIFY(db); 1775 db->db_buf = NULL; 1776 dbuf_evict(db); 1777 } else { 1778 mutex_exit(&db->db_mtx); 1779 dbuf_destroy(db); 1780 } 1781 return (0); 1782 } 1783 1784 static void 1785 dbuf_destroy(dmu_buf_impl_t *db) 1786 { 1787 ASSERT(refcount_is_zero(&db->db_holds)); 1788 1789 if (db->db_blkid != DMU_BONUS_BLKID) { 1790 /* 1791 * If this dbuf is still on the dn_dbufs list, 1792 * remove it from that list. 1793 */ 1794 if (db->db_dnode_handle != NULL) { 1795 dnode_t *dn; 1796 1797 DB_DNODE_ENTER(db); 1798 dn = DB_DNODE(db); 1799 mutex_enter(&dn->dn_dbufs_mtx); 1800 list_remove(&dn->dn_dbufs, db); 1801 (void) atomic_dec_32_nv(&dn->dn_dbufs_count); 1802 mutex_exit(&dn->dn_dbufs_mtx); 1803 DB_DNODE_EXIT(db); 1804 /* 1805 * Decrementing the dbuf count means that the hold 1806 * corresponding to the removed dbuf is no longer 1807 * discounted in dnode_move(), so the dnode cannot be 1808 * moved until after we release the hold. 1809 */ 1810 dnode_rele(dn, db); 1811 db->db_dnode_handle = NULL; 1812 } 1813 dbuf_hash_remove(db); 1814 } 1815 db->db_parent = NULL; 1816 db->db_buf = NULL; 1817 1818 ASSERT(!list_link_active(&db->db_link)); 1819 ASSERT(db->db.db_data == NULL); 1820 ASSERT(db->db_hash_next == NULL); 1821 ASSERT(db->db_blkptr == NULL); 1822 ASSERT(db->db_data_pending == NULL); 1823 1824 kmem_cache_free(dbuf_cache, db); 1825 arc_space_return(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1826 } 1827 1828 void 1829 dbuf_prefetch(dnode_t *dn, uint64_t blkid, zio_priority_t prio) 1830 { 1831 dmu_buf_impl_t *db = NULL; 1832 blkptr_t *bp = NULL; 1833 1834 ASSERT(blkid != DMU_BONUS_BLKID); 1835 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1836 1837 if (dnode_block_freed(dn, blkid)) 1838 return; 1839 1840 /* dbuf_find() returns with db_mtx held */ 1841 if (db = dbuf_find(dn, 0, blkid)) { 1842 /* 1843 * This dbuf is already in the cache. We assume that 1844 * it is already CACHED, or else about to be either 1845 * read or filled. 1846 */ 1847 mutex_exit(&db->db_mtx); 1848 return; 1849 } 1850 1851 if (dbuf_findbp(dn, 0, blkid, TRUE, &db, &bp) == 0) { 1852 if (bp && !BP_IS_HOLE(bp)) { 1853 dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset; 1854 uint32_t aflags = ARC_NOWAIT | ARC_PREFETCH; 1855 zbookmark_t zb; 1856 1857 SET_BOOKMARK(&zb, ds ? ds->ds_object : DMU_META_OBJSET, 1858 dn->dn_object, 0, blkid); 1859 1860 (void) arc_read(NULL, dn->dn_objset->os_spa, 1861 bp, NULL, NULL, prio, 1862 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 1863 &aflags, &zb); 1864 } 1865 if (db) 1866 dbuf_rele(db, NULL); 1867 } 1868 } 1869 1870 /* 1871 * Returns with db_holds incremented, and db_mtx not held. 1872 * Note: dn_struct_rwlock must be held. 1873 */ 1874 int 1875 dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, int fail_sparse, 1876 void *tag, dmu_buf_impl_t **dbp) 1877 { 1878 dmu_buf_impl_t *db, *parent = NULL; 1879 1880 ASSERT(blkid != DMU_BONUS_BLKID); 1881 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1882 ASSERT3U(dn->dn_nlevels, >, level); 1883 1884 *dbp = NULL; 1885 top: 1886 /* dbuf_find() returns with db_mtx held */ 1887 db = dbuf_find(dn, level, blkid); 1888 1889 if (db == NULL) { 1890 blkptr_t *bp = NULL; 1891 int err; 1892 1893 ASSERT3P(parent, ==, NULL); 1894 err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp); 1895 if (fail_sparse) { 1896 if (err == 0 && bp && BP_IS_HOLE(bp)) 1897 err = SET_ERROR(ENOENT); 1898 if (err) { 1899 if (parent) 1900 dbuf_rele(parent, NULL); 1901 return (err); 1902 } 1903 } 1904 if (err && err != ENOENT) 1905 return (err); 1906 db = dbuf_create(dn, level, blkid, parent, bp); 1907 } 1908 1909 if (db->db_buf && refcount_is_zero(&db->db_holds)) { 1910 arc_buf_add_ref(db->db_buf, db); 1911 if (db->db_buf->b_data == NULL) { 1912 dbuf_clear(db); 1913 if (parent) { 1914 dbuf_rele(parent, NULL); 1915 parent = NULL; 1916 } 1917 goto top; 1918 } 1919 ASSERT3P(db->db.db_data, ==, db->db_buf->b_data); 1920 } 1921 1922 ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf)); 1923 1924 /* 1925 * If this buffer is currently syncing out, and we are are 1926 * still referencing it from db_data, we need to make a copy 1927 * of it in case we decide we want to dirty it again in this txg. 1928 */ 1929 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 1930 dn->dn_object != DMU_META_DNODE_OBJECT && 1931 db->db_state == DB_CACHED && db->db_data_pending) { 1932 dbuf_dirty_record_t *dr = db->db_data_pending; 1933 1934 if (dr->dt.dl.dr_data == db->db_buf) { 1935 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 1936 1937 dbuf_set_data(db, 1938 arc_buf_alloc(dn->dn_objset->os_spa, 1939 db->db.db_size, db, type)); 1940 bcopy(dr->dt.dl.dr_data->b_data, db->db.db_data, 1941 db->db.db_size); 1942 } 1943 } 1944 1945 (void) refcount_add(&db->db_holds, tag); 1946 dbuf_update_data(db); 1947 DBUF_VERIFY(db); 1948 mutex_exit(&db->db_mtx); 1949 1950 /* NOTE: we can't rele the parent until after we drop the db_mtx */ 1951 if (parent) 1952 dbuf_rele(parent, NULL); 1953 1954 ASSERT3P(DB_DNODE(db), ==, dn); 1955 ASSERT3U(db->db_blkid, ==, blkid); 1956 ASSERT3U(db->db_level, ==, level); 1957 *dbp = db; 1958 1959 return (0); 1960 } 1961 1962 dmu_buf_impl_t * 1963 dbuf_hold(dnode_t *dn, uint64_t blkid, void *tag) 1964 { 1965 dmu_buf_impl_t *db; 1966 int err = dbuf_hold_impl(dn, 0, blkid, FALSE, tag, &db); 1967 return (err ? NULL : db); 1968 } 1969 1970 dmu_buf_impl_t * 1971 dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, void *tag) 1972 { 1973 dmu_buf_impl_t *db; 1974 int err = dbuf_hold_impl(dn, level, blkid, FALSE, tag, &db); 1975 return (err ? NULL : db); 1976 } 1977 1978 void 1979 dbuf_create_bonus(dnode_t *dn) 1980 { 1981 ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 1982 1983 ASSERT(dn->dn_bonus == NULL); 1984 dn->dn_bonus = dbuf_create(dn, 0, DMU_BONUS_BLKID, dn->dn_dbuf, NULL); 1985 } 1986 1987 int 1988 dbuf_spill_set_blksz(dmu_buf_t *db_fake, uint64_t blksz, dmu_tx_t *tx) 1989 { 1990 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1991 dnode_t *dn; 1992 1993 if (db->db_blkid != DMU_SPILL_BLKID) 1994 return (SET_ERROR(ENOTSUP)); 1995 if (blksz == 0) 1996 blksz = SPA_MINBLOCKSIZE; 1997 if (blksz > SPA_MAXBLOCKSIZE) 1998 blksz = SPA_MAXBLOCKSIZE; 1999 else 2000 blksz = P2ROUNDUP(blksz, SPA_MINBLOCKSIZE); 2001 2002 DB_DNODE_ENTER(db); 2003 dn = DB_DNODE(db); 2004 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 2005 dbuf_new_size(db, blksz, tx); 2006 rw_exit(&dn->dn_struct_rwlock); 2007 DB_DNODE_EXIT(db); 2008 2009 return (0); 2010 } 2011 2012 void 2013 dbuf_rm_spill(dnode_t *dn, dmu_tx_t *tx) 2014 { 2015 dbuf_free_range(dn, DMU_SPILL_BLKID, DMU_SPILL_BLKID, tx); 2016 } 2017 2018 #pragma weak dmu_buf_add_ref = dbuf_add_ref 2019 void 2020 dbuf_add_ref(dmu_buf_impl_t *db, void *tag) 2021 { 2022 int64_t holds = refcount_add(&db->db_holds, tag); 2023 ASSERT(holds > 1); 2024 } 2025 2026 /* 2027 * If you call dbuf_rele() you had better not be referencing the dnode handle 2028 * unless you have some other direct or indirect hold on the dnode. (An indirect 2029 * hold is a hold on one of the dnode's dbufs, including the bonus buffer.) 2030 * Without that, the dbuf_rele() could lead to a dnode_rele() followed by the 2031 * dnode's parent dbuf evicting its dnode handles. 2032 */ 2033 #pragma weak dmu_buf_rele = dbuf_rele 2034 void 2035 dbuf_rele(dmu_buf_impl_t *db, void *tag) 2036 { 2037 mutex_enter(&db->db_mtx); 2038 dbuf_rele_and_unlock(db, tag); 2039 } 2040 2041 /* 2042 * dbuf_rele() for an already-locked dbuf. This is necessary to allow 2043 * db_dirtycnt and db_holds to be updated atomically. 2044 */ 2045 void 2046 dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag) 2047 { 2048 int64_t holds; 2049 2050 ASSERT(MUTEX_HELD(&db->db_mtx)); 2051 DBUF_VERIFY(db); 2052 2053 /* 2054 * Remove the reference to the dbuf before removing its hold on the 2055 * dnode so we can guarantee in dnode_move() that a referenced bonus 2056 * buffer has a corresponding dnode hold. 2057 */ 2058 holds = refcount_remove(&db->db_holds, tag); 2059 ASSERT(holds >= 0); 2060 2061 /* 2062 * We can't freeze indirects if there is a possibility that they 2063 * may be modified in the current syncing context. 2064 */ 2065 if (db->db_buf && holds == (db->db_level == 0 ? db->db_dirtycnt : 0)) 2066 arc_buf_freeze(db->db_buf); 2067 2068 if (holds == db->db_dirtycnt && 2069 db->db_level == 0 && db->db_immediate_evict) 2070 dbuf_evict_user(db); 2071 2072 if (holds == 0) { 2073 if (db->db_blkid == DMU_BONUS_BLKID) { 2074 mutex_exit(&db->db_mtx); 2075 2076 /* 2077 * If the dnode moves here, we cannot cross this barrier 2078 * until the move completes. 2079 */ 2080 DB_DNODE_ENTER(db); 2081 (void) atomic_dec_32_nv(&DB_DNODE(db)->dn_dbufs_count); 2082 DB_DNODE_EXIT(db); 2083 /* 2084 * The bonus buffer's dnode hold is no longer discounted 2085 * in dnode_move(). The dnode cannot move until after 2086 * the dnode_rele(). 2087 */ 2088 dnode_rele(DB_DNODE(db), db); 2089 } else if (db->db_buf == NULL) { 2090 /* 2091 * This is a special case: we never associated this 2092 * dbuf with any data allocated from the ARC. 2093 */ 2094 ASSERT(db->db_state == DB_UNCACHED || 2095 db->db_state == DB_NOFILL); 2096 dbuf_evict(db); 2097 } else if (arc_released(db->db_buf)) { 2098 arc_buf_t *buf = db->db_buf; 2099 /* 2100 * This dbuf has anonymous data associated with it. 2101 */ 2102 dbuf_set_data(db, NULL); 2103 VERIFY(arc_buf_remove_ref(buf, db)); 2104 dbuf_evict(db); 2105 } else { 2106 VERIFY(!arc_buf_remove_ref(db->db_buf, db)); 2107 2108 /* 2109 * A dbuf will be eligible for eviction if either the 2110 * 'primarycache' property is set or a duplicate 2111 * copy of this buffer is already cached in the arc. 2112 * 2113 * In the case of the 'primarycache' a buffer 2114 * is considered for eviction if it matches the 2115 * criteria set in the property. 2116 * 2117 * To decide if our buffer is considered a 2118 * duplicate, we must call into the arc to determine 2119 * if multiple buffers are referencing the same 2120 * block on-disk. If so, then we simply evict 2121 * ourselves. 2122 */ 2123 if (!DBUF_IS_CACHEABLE(db) || 2124 arc_buf_eviction_needed(db->db_buf)) 2125 dbuf_clear(db); 2126 else 2127 mutex_exit(&db->db_mtx); 2128 } 2129 } else { 2130 mutex_exit(&db->db_mtx); 2131 } 2132 } 2133 2134 #pragma weak dmu_buf_refcount = dbuf_refcount 2135 uint64_t 2136 dbuf_refcount(dmu_buf_impl_t *db) 2137 { 2138 return (refcount_count(&db->db_holds)); 2139 } 2140 2141 void * 2142 dmu_buf_set_user(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 2143 dmu_buf_evict_func_t *evict_func) 2144 { 2145 return (dmu_buf_update_user(db_fake, NULL, user_ptr, 2146 user_data_ptr_ptr, evict_func)); 2147 } 2148 2149 void * 2150 dmu_buf_set_user_ie(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 2151 dmu_buf_evict_func_t *evict_func) 2152 { 2153 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2154 2155 db->db_immediate_evict = TRUE; 2156 return (dmu_buf_update_user(db_fake, NULL, user_ptr, 2157 user_data_ptr_ptr, evict_func)); 2158 } 2159 2160 void * 2161 dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr, void *user_ptr, 2162 void *user_data_ptr_ptr, dmu_buf_evict_func_t *evict_func) 2163 { 2164 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2165 ASSERT(db->db_level == 0); 2166 2167 ASSERT((user_ptr == NULL) == (evict_func == NULL)); 2168 2169 mutex_enter(&db->db_mtx); 2170 2171 if (db->db_user_ptr == old_user_ptr) { 2172 db->db_user_ptr = user_ptr; 2173 db->db_user_data_ptr_ptr = user_data_ptr_ptr; 2174 db->db_evict_func = evict_func; 2175 2176 dbuf_update_data(db); 2177 } else { 2178 old_user_ptr = db->db_user_ptr; 2179 } 2180 2181 mutex_exit(&db->db_mtx); 2182 return (old_user_ptr); 2183 } 2184 2185 void * 2186 dmu_buf_get_user(dmu_buf_t *db_fake) 2187 { 2188 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2189 ASSERT(!refcount_is_zero(&db->db_holds)); 2190 2191 return (db->db_user_ptr); 2192 } 2193 2194 boolean_t 2195 dmu_buf_freeable(dmu_buf_t *dbuf) 2196 { 2197 boolean_t res = B_FALSE; 2198 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf; 2199 2200 if (db->db_blkptr) 2201 res = dsl_dataset_block_freeable(db->db_objset->os_dsl_dataset, 2202 db->db_blkptr, db->db_blkptr->blk_birth); 2203 2204 return (res); 2205 } 2206 2207 blkptr_t * 2208 dmu_buf_get_blkptr(dmu_buf_t *db) 2209 { 2210 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db; 2211 return (dbi->db_blkptr); 2212 } 2213 2214 static void 2215 dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db) 2216 { 2217 /* ASSERT(dmu_tx_is_syncing(tx) */ 2218 ASSERT(MUTEX_HELD(&db->db_mtx)); 2219 2220 if (db->db_blkptr != NULL) 2221 return; 2222 2223 if (db->db_blkid == DMU_SPILL_BLKID) { 2224 db->db_blkptr = &dn->dn_phys->dn_spill; 2225 BP_ZERO(db->db_blkptr); 2226 return; 2227 } 2228 if (db->db_level == dn->dn_phys->dn_nlevels-1) { 2229 /* 2230 * This buffer was allocated at a time when there was 2231 * no available blkptrs from the dnode, or it was 2232 * inappropriate to hook it in (i.e., nlevels mis-match). 2233 */ 2234 ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr); 2235 ASSERT(db->db_parent == NULL); 2236 db->db_parent = dn->dn_dbuf; 2237 db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid]; 2238 DBUF_VERIFY(db); 2239 } else { 2240 dmu_buf_impl_t *parent = db->db_parent; 2241 int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2242 2243 ASSERT(dn->dn_phys->dn_nlevels > 1); 2244 if (parent == NULL) { 2245 mutex_exit(&db->db_mtx); 2246 rw_enter(&dn->dn_struct_rwlock, RW_READER); 2247 (void) dbuf_hold_impl(dn, db->db_level+1, 2248 db->db_blkid >> epbs, FALSE, db, &parent); 2249 rw_exit(&dn->dn_struct_rwlock); 2250 mutex_enter(&db->db_mtx); 2251 db->db_parent = parent; 2252 } 2253 db->db_blkptr = (blkptr_t *)parent->db.db_data + 2254 (db->db_blkid & ((1ULL << epbs) - 1)); 2255 DBUF_VERIFY(db); 2256 } 2257 } 2258 2259 static void 2260 dbuf_sync_indirect(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 2261 { 2262 dmu_buf_impl_t *db = dr->dr_dbuf; 2263 dnode_t *dn; 2264 zio_t *zio; 2265 2266 ASSERT(dmu_tx_is_syncing(tx)); 2267 2268 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 2269 2270 mutex_enter(&db->db_mtx); 2271 2272 ASSERT(db->db_level > 0); 2273 DBUF_VERIFY(db); 2274 2275 /* Read the block if it hasn't been read yet. */ 2276 if (db->db_buf == NULL) { 2277 mutex_exit(&db->db_mtx); 2278 (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED); 2279 mutex_enter(&db->db_mtx); 2280 } 2281 ASSERT3U(db->db_state, ==, DB_CACHED); 2282 ASSERT(db->db_buf != NULL); 2283 2284 DB_DNODE_ENTER(db); 2285 dn = DB_DNODE(db); 2286 /* Indirect block size must match what the dnode thinks it is. */ 2287 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2288 dbuf_check_blkptr(dn, db); 2289 DB_DNODE_EXIT(db); 2290 2291 /* Provide the pending dirty record to child dbufs */ 2292 db->db_data_pending = dr; 2293 2294 mutex_exit(&db->db_mtx); 2295 dbuf_write(dr, db->db_buf, tx); 2296 2297 zio = dr->dr_zio; 2298 mutex_enter(&dr->dt.di.dr_mtx); 2299 dbuf_sync_list(&dr->dt.di.dr_children, tx); 2300 ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 2301 mutex_exit(&dr->dt.di.dr_mtx); 2302 zio_nowait(zio); 2303 } 2304 2305 static void 2306 dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 2307 { 2308 arc_buf_t **datap = &dr->dt.dl.dr_data; 2309 dmu_buf_impl_t *db = dr->dr_dbuf; 2310 dnode_t *dn; 2311 objset_t *os; 2312 uint64_t txg = tx->tx_txg; 2313 2314 ASSERT(dmu_tx_is_syncing(tx)); 2315 2316 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 2317 2318 mutex_enter(&db->db_mtx); 2319 /* 2320 * To be synced, we must be dirtied. But we 2321 * might have been freed after the dirty. 2322 */ 2323 if (db->db_state == DB_UNCACHED) { 2324 /* This buffer has been freed since it was dirtied */ 2325 ASSERT(db->db.db_data == NULL); 2326 } else if (db->db_state == DB_FILL) { 2327 /* This buffer was freed and is now being re-filled */ 2328 ASSERT(db->db.db_data != dr->dt.dl.dr_data); 2329 } else { 2330 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_NOFILL); 2331 } 2332 DBUF_VERIFY(db); 2333 2334 DB_DNODE_ENTER(db); 2335 dn = DB_DNODE(db); 2336 2337 if (db->db_blkid == DMU_SPILL_BLKID) { 2338 mutex_enter(&dn->dn_mtx); 2339 dn->dn_phys->dn_flags |= DNODE_FLAG_SPILL_BLKPTR; 2340 mutex_exit(&dn->dn_mtx); 2341 } 2342 2343 /* 2344 * If this is a bonus buffer, simply copy the bonus data into the 2345 * dnode. It will be written out when the dnode is synced (and it 2346 * will be synced, since it must have been dirty for dbuf_sync to 2347 * be called). 2348 */ 2349 if (db->db_blkid == DMU_BONUS_BLKID) { 2350 dbuf_dirty_record_t **drp; 2351 2352 ASSERT(*datap != NULL); 2353 ASSERT0(db->db_level); 2354 ASSERT3U(dn->dn_phys->dn_bonuslen, <=, DN_MAX_BONUSLEN); 2355 bcopy(*datap, DN_BONUS(dn->dn_phys), dn->dn_phys->dn_bonuslen); 2356 DB_DNODE_EXIT(db); 2357 2358 if (*datap != db->db.db_data) { 2359 zio_buf_free(*datap, DN_MAX_BONUSLEN); 2360 arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 2361 } 2362 db->db_data_pending = NULL; 2363 drp = &db->db_last_dirty; 2364 while (*drp != dr) 2365 drp = &(*drp)->dr_next; 2366 ASSERT(dr->dr_next == NULL); 2367 ASSERT(dr->dr_dbuf == db); 2368 *drp = dr->dr_next; 2369 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 2370 ASSERT(db->db_dirtycnt > 0); 2371 db->db_dirtycnt -= 1; 2372 dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg); 2373 return; 2374 } 2375 2376 os = dn->dn_objset; 2377 2378 /* 2379 * This function may have dropped the db_mtx lock allowing a dmu_sync 2380 * operation to sneak in. As a result, we need to ensure that we 2381 * don't check the dr_override_state until we have returned from 2382 * dbuf_check_blkptr. 2383 */ 2384 dbuf_check_blkptr(dn, db); 2385 2386 /* 2387 * If this buffer is in the middle of an immediate write, 2388 * wait for the synchronous IO to complete. 2389 */ 2390 while (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC) { 2391 ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT); 2392 cv_wait(&db->db_changed, &db->db_mtx); 2393 ASSERT(dr->dt.dl.dr_override_state != DR_NOT_OVERRIDDEN); 2394 } 2395 2396 if (db->db_state != DB_NOFILL && 2397 dn->dn_object != DMU_META_DNODE_OBJECT && 2398 refcount_count(&db->db_holds) > 1 && 2399 dr->dt.dl.dr_override_state != DR_OVERRIDDEN && 2400 *datap == db->db_buf) { 2401 /* 2402 * If this buffer is currently "in use" (i.e., there 2403 * are active holds and db_data still references it), 2404 * then make a copy before we start the write so that 2405 * any modifications from the open txg will not leak 2406 * into this write. 2407 * 2408 * NOTE: this copy does not need to be made for 2409 * objects only modified in the syncing context (e.g. 2410 * DNONE_DNODE blocks). 2411 */ 2412 int blksz = arc_buf_size(*datap); 2413 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 2414 *datap = arc_buf_alloc(os->os_spa, blksz, db, type); 2415 bcopy(db->db.db_data, (*datap)->b_data, blksz); 2416 } 2417 db->db_data_pending = dr; 2418 2419 mutex_exit(&db->db_mtx); 2420 2421 dbuf_write(dr, *datap, tx); 2422 2423 ASSERT(!list_link_active(&dr->dr_dirty_node)); 2424 if (dn->dn_object == DMU_META_DNODE_OBJECT) { 2425 list_insert_tail(&dn->dn_dirty_records[txg&TXG_MASK], dr); 2426 DB_DNODE_EXIT(db); 2427 } else { 2428 /* 2429 * Although zio_nowait() does not "wait for an IO", it does 2430 * initiate the IO. If this is an empty write it seems plausible 2431 * that the IO could actually be completed before the nowait 2432 * returns. We need to DB_DNODE_EXIT() first in case 2433 * zio_nowait() invalidates the dbuf. 2434 */ 2435 DB_DNODE_EXIT(db); 2436 zio_nowait(dr->dr_zio); 2437 } 2438 } 2439 2440 void 2441 dbuf_sync_list(list_t *list, dmu_tx_t *tx) 2442 { 2443 dbuf_dirty_record_t *dr; 2444 2445 while (dr = list_head(list)) { 2446 if (dr->dr_zio != NULL) { 2447 /* 2448 * If we find an already initialized zio then we 2449 * are processing the meta-dnode, and we have finished. 2450 * The dbufs for all dnodes are put back on the list 2451 * during processing, so that we can zio_wait() 2452 * these IOs after initiating all child IOs. 2453 */ 2454 ASSERT3U(dr->dr_dbuf->db.db_object, ==, 2455 DMU_META_DNODE_OBJECT); 2456 break; 2457 } 2458 list_remove(list, dr); 2459 if (dr->dr_dbuf->db_level > 0) 2460 dbuf_sync_indirect(dr, tx); 2461 else 2462 dbuf_sync_leaf(dr, tx); 2463 } 2464 } 2465 2466 /* ARGSUSED */ 2467 static void 2468 dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb) 2469 { 2470 dmu_buf_impl_t *db = vdb; 2471 dnode_t *dn; 2472 blkptr_t *bp = zio->io_bp; 2473 blkptr_t *bp_orig = &zio->io_bp_orig; 2474 spa_t *spa = zio->io_spa; 2475 int64_t delta; 2476 uint64_t fill = 0; 2477 int i; 2478 2479 ASSERT(db->db_blkptr == bp); 2480 2481 DB_DNODE_ENTER(db); 2482 dn = DB_DNODE(db); 2483 delta = bp_get_dsize_sync(spa, bp) - bp_get_dsize_sync(spa, bp_orig); 2484 dnode_diduse_space(dn, delta - zio->io_prev_space_delta); 2485 zio->io_prev_space_delta = delta; 2486 2487 if (BP_IS_HOLE(bp)) { 2488 ASSERT(bp->blk_fill == 0); 2489 DB_DNODE_EXIT(db); 2490 return; 2491 } 2492 2493 ASSERT((db->db_blkid != DMU_SPILL_BLKID && 2494 BP_GET_TYPE(bp) == dn->dn_type) || 2495 (db->db_blkid == DMU_SPILL_BLKID && 2496 BP_GET_TYPE(bp) == dn->dn_bonustype)); 2497 ASSERT(BP_GET_LEVEL(bp) == db->db_level); 2498 2499 mutex_enter(&db->db_mtx); 2500 2501 #ifdef ZFS_DEBUG 2502 if (db->db_blkid == DMU_SPILL_BLKID) { 2503 ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); 2504 ASSERT(!(BP_IS_HOLE(db->db_blkptr)) && 2505 db->db_blkptr == &dn->dn_phys->dn_spill); 2506 } 2507 #endif 2508 2509 if (db->db_level == 0) { 2510 mutex_enter(&dn->dn_mtx); 2511 if (db->db_blkid > dn->dn_phys->dn_maxblkid && 2512 db->db_blkid != DMU_SPILL_BLKID) 2513 dn->dn_phys->dn_maxblkid = db->db_blkid; 2514 mutex_exit(&dn->dn_mtx); 2515 2516 if (dn->dn_type == DMU_OT_DNODE) { 2517 dnode_phys_t *dnp = db->db.db_data; 2518 for (i = db->db.db_size >> DNODE_SHIFT; i > 0; 2519 i--, dnp++) { 2520 if (dnp->dn_type != DMU_OT_NONE) 2521 fill++; 2522 } 2523 } else { 2524 fill = 1; 2525 } 2526 } else { 2527 blkptr_t *ibp = db->db.db_data; 2528 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2529 for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, ibp++) { 2530 if (BP_IS_HOLE(ibp)) 2531 continue; 2532 fill += ibp->blk_fill; 2533 } 2534 } 2535 DB_DNODE_EXIT(db); 2536 2537 bp->blk_fill = fill; 2538 2539 mutex_exit(&db->db_mtx); 2540 } 2541 2542 /* 2543 * The SPA will call this callback several times for each zio - once 2544 * for every physical child i/o (zio->io_phys_children times). This 2545 * allows the DMU to monitor the progress of each logical i/o. For example, 2546 * there may be 2 copies of an indirect block, or many fragments of a RAID-Z 2547 * block. There may be a long delay before all copies/fragments are completed, 2548 * so this callback allows us to retire dirty space gradually, as the physical 2549 * i/os complete. 2550 */ 2551 /* ARGSUSED */ 2552 static void 2553 dbuf_write_physdone(zio_t *zio, arc_buf_t *buf, void *arg) 2554 { 2555 dmu_buf_impl_t *db = arg; 2556 objset_t *os = db->db_objset; 2557 dsl_pool_t *dp = dmu_objset_pool(os); 2558 dbuf_dirty_record_t *dr; 2559 int delta = 0; 2560 2561 dr = db->db_data_pending; 2562 ASSERT3U(dr->dr_txg, ==, zio->io_txg); 2563 2564 /* 2565 * The callback will be called io_phys_children times. Retire one 2566 * portion of our dirty space each time we are called. Any rounding 2567 * error will be cleaned up by dsl_pool_sync()'s call to 2568 * dsl_pool_undirty_space(). 2569 */ 2570 delta = dr->dr_accounted / zio->io_phys_children; 2571 dsl_pool_undirty_space(dp, delta, zio->io_txg); 2572 } 2573 2574 /* ARGSUSED */ 2575 static void 2576 dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb) 2577 { 2578 dmu_buf_impl_t *db = vdb; 2579 blkptr_t *bp = zio->io_bp; 2580 blkptr_t *bp_orig = &zio->io_bp_orig; 2581 uint64_t txg = zio->io_txg; 2582 dbuf_dirty_record_t **drp, *dr; 2583 2584 ASSERT0(zio->io_error); 2585 ASSERT(db->db_blkptr == bp); 2586 2587 /* 2588 * For nopwrites and rewrites we ensure that the bp matches our 2589 * original and bypass all the accounting. 2590 */ 2591 if (zio->io_flags & (ZIO_FLAG_IO_REWRITE | ZIO_FLAG_NOPWRITE)) { 2592 ASSERT(BP_EQUAL(bp, bp_orig)); 2593 } else { 2594 objset_t *os; 2595 dsl_dataset_t *ds; 2596 dmu_tx_t *tx; 2597 2598 DB_GET_OBJSET(&os, db); 2599 ds = os->os_dsl_dataset; 2600 tx = os->os_synctx; 2601 2602 (void) dsl_dataset_block_kill(ds, bp_orig, tx, B_TRUE); 2603 dsl_dataset_block_born(ds, bp, tx); 2604 } 2605 2606 mutex_enter(&db->db_mtx); 2607 2608 DBUF_VERIFY(db); 2609 2610 drp = &db->db_last_dirty; 2611 while ((dr = *drp) != db->db_data_pending) 2612 drp = &dr->dr_next; 2613 ASSERT(!list_link_active(&dr->dr_dirty_node)); 2614 ASSERT(dr->dr_txg == txg); 2615 ASSERT(dr->dr_dbuf == db); 2616 ASSERT(dr->dr_next == NULL); 2617 *drp = dr->dr_next; 2618 2619 #ifdef ZFS_DEBUG 2620 if (db->db_blkid == DMU_SPILL_BLKID) { 2621 dnode_t *dn; 2622 2623 DB_DNODE_ENTER(db); 2624 dn = DB_DNODE(db); 2625 ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); 2626 ASSERT(!(BP_IS_HOLE(db->db_blkptr)) && 2627 db->db_blkptr == &dn->dn_phys->dn_spill); 2628 DB_DNODE_EXIT(db); 2629 } 2630 #endif 2631 2632 if (db->db_level == 0) { 2633 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 2634 ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN); 2635 if (db->db_state != DB_NOFILL) { 2636 if (dr->dt.dl.dr_data != db->db_buf) 2637 VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, 2638 db)); 2639 else if (!arc_released(db->db_buf)) 2640 arc_set_callback(db->db_buf, dbuf_do_evict, db); 2641 } 2642 } else { 2643 dnode_t *dn; 2644 2645 DB_DNODE_ENTER(db); 2646 dn = DB_DNODE(db); 2647 ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 2648 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2649 if (!BP_IS_HOLE(db->db_blkptr)) { 2650 int epbs = 2651 dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2652 ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, 2653 db->db.db_size); 2654 ASSERT3U(dn->dn_phys->dn_maxblkid 2655 >> (db->db_level * epbs), >=, db->db_blkid); 2656 arc_set_callback(db->db_buf, dbuf_do_evict, db); 2657 } 2658 DB_DNODE_EXIT(db); 2659 mutex_destroy(&dr->dt.di.dr_mtx); 2660 list_destroy(&dr->dt.di.dr_children); 2661 } 2662 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 2663 2664 cv_broadcast(&db->db_changed); 2665 ASSERT(db->db_dirtycnt > 0); 2666 db->db_dirtycnt -= 1; 2667 db->db_data_pending = NULL; 2668 2669 dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg); 2670 } 2671 2672 static void 2673 dbuf_write_nofill_ready(zio_t *zio) 2674 { 2675 dbuf_write_ready(zio, NULL, zio->io_private); 2676 } 2677 2678 static void 2679 dbuf_write_nofill_done(zio_t *zio) 2680 { 2681 dbuf_write_done(zio, NULL, zio->io_private); 2682 } 2683 2684 static void 2685 dbuf_write_override_ready(zio_t *zio) 2686 { 2687 dbuf_dirty_record_t *dr = zio->io_private; 2688 dmu_buf_impl_t *db = dr->dr_dbuf; 2689 2690 dbuf_write_ready(zio, NULL, db); 2691 } 2692 2693 static void 2694 dbuf_write_override_done(zio_t *zio) 2695 { 2696 dbuf_dirty_record_t *dr = zio->io_private; 2697 dmu_buf_impl_t *db = dr->dr_dbuf; 2698 blkptr_t *obp = &dr->dt.dl.dr_overridden_by; 2699 2700 mutex_enter(&db->db_mtx); 2701 if (!BP_EQUAL(zio->io_bp, obp)) { 2702 if (!BP_IS_HOLE(obp)) 2703 dsl_free(spa_get_dsl(zio->io_spa), zio->io_txg, obp); 2704 arc_release(dr->dt.dl.dr_data, db); 2705 } 2706 mutex_exit(&db->db_mtx); 2707 2708 dbuf_write_done(zio, NULL, db); 2709 } 2710 2711 /* Issue I/O to commit a dirty buffer to disk. */ 2712 static void 2713 dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx) 2714 { 2715 dmu_buf_impl_t *db = dr->dr_dbuf; 2716 dnode_t *dn; 2717 objset_t *os; 2718 dmu_buf_impl_t *parent = db->db_parent; 2719 uint64_t txg = tx->tx_txg; 2720 zbookmark_t zb; 2721 zio_prop_t zp; 2722 zio_t *zio; 2723 int wp_flag = 0; 2724 2725 DB_DNODE_ENTER(db); 2726 dn = DB_DNODE(db); 2727 os = dn->dn_objset; 2728 2729 if (db->db_state != DB_NOFILL) { 2730 if (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE) { 2731 /* 2732 * Private object buffers are released here rather 2733 * than in dbuf_dirty() since they are only modified 2734 * in the syncing context and we don't want the 2735 * overhead of making multiple copies of the data. 2736 */ 2737 if (BP_IS_HOLE(db->db_blkptr)) { 2738 arc_buf_thaw(data); 2739 } else { 2740 dbuf_release_bp(db); 2741 } 2742 } 2743 } 2744 2745 if (parent != dn->dn_dbuf) { 2746 /* Our parent is an indirect block. */ 2747 /* We have a dirty parent that has been scheduled for write. */ 2748 ASSERT(parent && parent->db_data_pending); 2749 /* Our parent's buffer is one level closer to the dnode. */ 2750 ASSERT(db->db_level == parent->db_level-1); 2751 /* 2752 * We're about to modify our parent's db_data by modifying 2753 * our block pointer, so the parent must be released. 2754 */ 2755 ASSERT(arc_released(parent->db_buf)); 2756 zio = parent->db_data_pending->dr_zio; 2757 } else { 2758 /* Our parent is the dnode itself. */ 2759 ASSERT((db->db_level == dn->dn_phys->dn_nlevels-1 && 2760 db->db_blkid != DMU_SPILL_BLKID) || 2761 (db->db_blkid == DMU_SPILL_BLKID && db->db_level == 0)); 2762 if (db->db_blkid != DMU_SPILL_BLKID) 2763 ASSERT3P(db->db_blkptr, ==, 2764 &dn->dn_phys->dn_blkptr[db->db_blkid]); 2765 zio = dn->dn_zio; 2766 } 2767 2768 ASSERT(db->db_level == 0 || data == db->db_buf); 2769 ASSERT3U(db->db_blkptr->blk_birth, <=, txg); 2770 ASSERT(zio); 2771 2772 SET_BOOKMARK(&zb, os->os_dsl_dataset ? 2773 os->os_dsl_dataset->ds_object : DMU_META_OBJSET, 2774 db->db.db_object, db->db_level, db->db_blkid); 2775 2776 if (db->db_blkid == DMU_SPILL_BLKID) 2777 wp_flag = WP_SPILL; 2778 wp_flag |= (db->db_state == DB_NOFILL) ? WP_NOFILL : 0; 2779 2780 dmu_write_policy(os, dn, db->db_level, wp_flag, &zp); 2781 DB_DNODE_EXIT(db); 2782 2783 if (db->db_level == 0 && dr->dt.dl.dr_override_state == DR_OVERRIDDEN) { 2784 ASSERT(db->db_state != DB_NOFILL); 2785 dr->dr_zio = zio_write(zio, os->os_spa, txg, 2786 db->db_blkptr, data->b_data, arc_buf_size(data), &zp, 2787 dbuf_write_override_ready, NULL, dbuf_write_override_done, 2788 dr, ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); 2789 mutex_enter(&db->db_mtx); 2790 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 2791 zio_write_override(dr->dr_zio, &dr->dt.dl.dr_overridden_by, 2792 dr->dt.dl.dr_copies, dr->dt.dl.dr_nopwrite); 2793 mutex_exit(&db->db_mtx); 2794 } else if (db->db_state == DB_NOFILL) { 2795 ASSERT(zp.zp_checksum == ZIO_CHECKSUM_OFF || 2796 zp.zp_checksum == ZIO_CHECKSUM_NOPARITY); 2797 dr->dr_zio = zio_write(zio, os->os_spa, txg, 2798 db->db_blkptr, NULL, db->db.db_size, &zp, 2799 dbuf_write_nofill_ready, NULL, dbuf_write_nofill_done, db, 2800 ZIO_PRIORITY_ASYNC_WRITE, 2801 ZIO_FLAG_MUSTSUCCEED | ZIO_FLAG_NODATA, &zb); 2802 } else { 2803 ASSERT(arc_released(data)); 2804 dr->dr_zio = arc_write(zio, os->os_spa, txg, 2805 db->db_blkptr, data, DBUF_IS_L2CACHEABLE(db), 2806 DBUF_IS_L2COMPRESSIBLE(db), &zp, dbuf_write_ready, 2807 dbuf_write_physdone, dbuf_write_done, db, 2808 ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); 2809 } 2810 } 2811