1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #include <sys/zfs_context.h> 29 #include <sys/dmu.h> 30 #include <sys/dmu_impl.h> 31 #include <sys/dbuf.h> 32 #include <sys/dmu_objset.h> 33 #include <sys/dsl_dataset.h> 34 #include <sys/dsl_dir.h> 35 #include <sys/dmu_tx.h> 36 #include <sys/spa.h> 37 #include <sys/zio.h> 38 #include <sys/dmu_zfetch.h> 39 40 static void dbuf_destroy(dmu_buf_impl_t *db); 41 static int dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx); 42 static arc_done_func_t dbuf_write_done; 43 44 /* 45 * Global data structures and functions for the dbuf cache. 46 */ 47 taskq_t *dbuf_tq; 48 static kmem_cache_t *dbuf_cache; 49 50 /* ARGSUSED */ 51 static int 52 dbuf_cons(void *vdb, void *unused, int kmflag) 53 { 54 dmu_buf_impl_t *db = vdb; 55 bzero(db, sizeof (dmu_buf_impl_t)); 56 57 mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL); 58 cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL); 59 refcount_create(&db->db_holds); 60 return (0); 61 } 62 63 /* ARGSUSED */ 64 static void 65 dbuf_dest(void *vdb, void *unused) 66 { 67 dmu_buf_impl_t *db = vdb; 68 mutex_destroy(&db->db_mtx); 69 cv_destroy(&db->db_changed); 70 refcount_destroy(&db->db_holds); 71 } 72 73 /* 74 * dbuf hash table routines 75 */ 76 static dbuf_hash_table_t dbuf_hash_table; 77 78 static uint64_t dbuf_hash_count; 79 80 static uint64_t 81 dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid) 82 { 83 uintptr_t osv = (uintptr_t)os; 84 uint64_t crc = -1ULL; 85 86 ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 87 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (lvl)) & 0xFF]; 88 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF]; 89 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF]; 90 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF]; 91 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 0)) & 0xFF]; 92 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 8)) & 0xFF]; 93 94 crc ^= (osv>>14) ^ (obj>>16) ^ (blkid>>16); 95 96 return (crc); 97 } 98 99 #define DBUF_HASH(os, obj, level, blkid) dbuf_hash(os, obj, level, blkid); 100 101 #define DBUF_EQUAL(dbuf, os, obj, level, blkid) \ 102 ((dbuf)->db.db_object == (obj) && \ 103 (dbuf)->db_objset == (os) && \ 104 (dbuf)->db_level == (level) && \ 105 (dbuf)->db_blkid == (blkid)) 106 107 dmu_buf_impl_t * 108 dbuf_find(dnode_t *dn, uint8_t level, uint64_t blkid) 109 { 110 dbuf_hash_table_t *h = &dbuf_hash_table; 111 objset_impl_t *os = dn->dn_objset; 112 uint64_t obj = dn->dn_object; 113 uint64_t hv = DBUF_HASH(os, obj, level, blkid); 114 uint64_t idx = hv & h->hash_table_mask; 115 dmu_buf_impl_t *db; 116 117 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 118 for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) { 119 if (DBUF_EQUAL(db, os, obj, level, blkid)) { 120 mutex_enter(&db->db_mtx); 121 if (db->db_state != DB_EVICTING) { 122 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 123 return (db); 124 } 125 mutex_exit(&db->db_mtx); 126 } 127 } 128 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 129 return (NULL); 130 } 131 132 /* 133 * Insert an entry into the hash table. If there is already an element 134 * equal to elem in the hash table, then the already existing element 135 * will be returned and the new element will not be inserted. 136 * Otherwise returns NULL. 137 */ 138 static dmu_buf_impl_t * 139 dbuf_hash_insert(dmu_buf_impl_t *db) 140 { 141 dbuf_hash_table_t *h = &dbuf_hash_table; 142 objset_impl_t *os = db->db_objset; 143 uint64_t obj = db->db.db_object; 144 int level = db->db_level; 145 uint64_t blkid = db->db_blkid; 146 uint64_t hv = DBUF_HASH(os, obj, level, blkid); 147 uint64_t idx = hv & h->hash_table_mask; 148 dmu_buf_impl_t *dbf; 149 150 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 151 for (dbf = h->hash_table[idx]; dbf != NULL; dbf = dbf->db_hash_next) { 152 if (DBUF_EQUAL(dbf, os, obj, level, blkid)) { 153 mutex_enter(&dbf->db_mtx); 154 if (dbf->db_state != DB_EVICTING) { 155 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 156 return (dbf); 157 } 158 mutex_exit(&dbf->db_mtx); 159 } 160 } 161 162 mutex_enter(&db->db_mtx); 163 db->db_hash_next = h->hash_table[idx]; 164 h->hash_table[idx] = db; 165 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 166 atomic_add_64(&dbuf_hash_count, 1); 167 168 return (NULL); 169 } 170 171 /* 172 * Remove an entry from the hash table. This operation will 173 * fail if there are any existing holds on the db. 174 */ 175 static void 176 dbuf_hash_remove(dmu_buf_impl_t *db) 177 { 178 dbuf_hash_table_t *h = &dbuf_hash_table; 179 uint64_t hv = DBUF_HASH(db->db_objset, db->db.db_object, 180 db->db_level, db->db_blkid); 181 uint64_t idx = hv & h->hash_table_mask; 182 dmu_buf_impl_t *dbf, **dbp; 183 184 /* 185 * We musn't hold db_mtx to maintin lock ordering: 186 * DBUF_HASH_MUTEX > db_mtx. 187 */ 188 ASSERT(refcount_is_zero(&db->db_holds)); 189 ASSERT(db->db_state == DB_EVICTING); 190 ASSERT(!MUTEX_HELD(&db->db_mtx)); 191 192 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 193 dbp = &h->hash_table[idx]; 194 while ((dbf = *dbp) != db) { 195 dbp = &dbf->db_hash_next; 196 ASSERT(dbf != NULL); 197 } 198 *dbp = db->db_hash_next; 199 db->db_hash_next = NULL; 200 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 201 atomic_add_64(&dbuf_hash_count, -1); 202 } 203 204 static arc_evict_func_t dbuf_do_evict; 205 206 static void 207 dbuf_evict_user(dmu_buf_impl_t *db) 208 { 209 ASSERT(MUTEX_HELD(&db->db_mtx)); 210 211 if (db->db_level != 0 || db->db_d.db_evict_func == NULL) 212 return; 213 214 if (db->db_d.db_user_data_ptr_ptr) 215 *db->db_d.db_user_data_ptr_ptr = db->db.db_data; 216 db->db_d.db_evict_func(&db->db, db->db_d.db_user_ptr); 217 db->db_d.db_user_ptr = NULL; 218 db->db_d.db_user_data_ptr_ptr = NULL; 219 db->db_d.db_evict_func = NULL; 220 } 221 222 void 223 dbuf_evict(dmu_buf_impl_t *db) 224 { 225 int i; 226 227 ASSERT(MUTEX_HELD(&db->db_mtx)); 228 ASSERT(db->db_buf == NULL); 229 230 #ifdef ZFS_DEBUG 231 for (i = 0; i < TXG_SIZE; i++) { 232 ASSERT(!list_link_active(&db->db_dirty_node[i])); 233 ASSERT(db->db_level != 0 || db->db_d.db_data_old[i] == NULL); 234 } 235 #endif 236 dbuf_clear(db); 237 dbuf_destroy(db); 238 } 239 240 void 241 dbuf_init(void) 242 { 243 uint64_t hsize = 1ULL << 16; 244 dbuf_hash_table_t *h = &dbuf_hash_table; 245 int i; 246 247 /* 248 * The hash table is big enough to fill all of physical memory 249 * with an average 4K block size. The table will take up 250 * totalmem*sizeof(void*)/4K (i.e. 2MB/GB with 8-byte pointers). 251 */ 252 while (hsize * 4096 < physmem * PAGESIZE) 253 hsize <<= 1; 254 255 retry: 256 h->hash_table_mask = hsize - 1; 257 h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP); 258 if (h->hash_table == NULL) { 259 /* XXX - we should really return an error instead of assert */ 260 ASSERT(hsize > (1ULL << 10)); 261 hsize >>= 1; 262 goto retry; 263 } 264 265 dbuf_cache = kmem_cache_create("dmu_buf_impl_t", 266 sizeof (dmu_buf_impl_t), 267 0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0); 268 dbuf_tq = taskq_create("dbuf_tq", 8, maxclsyspri, 50, INT_MAX, 269 TASKQ_PREPOPULATE); 270 271 for (i = 0; i < DBUF_MUTEXES; i++) 272 mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL); 273 } 274 275 void 276 dbuf_fini(void) 277 { 278 dbuf_hash_table_t *h = &dbuf_hash_table; 279 int i; 280 281 taskq_destroy(dbuf_tq); 282 dbuf_tq = NULL; 283 284 for (i = 0; i < DBUF_MUTEXES; i++) 285 mutex_destroy(&h->hash_mutexes[i]); 286 kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *)); 287 kmem_cache_destroy(dbuf_cache); 288 } 289 290 /* 291 * Other stuff. 292 */ 293 294 #ifdef ZFS_DEBUG 295 static void 296 dbuf_verify(dmu_buf_impl_t *db) 297 { 298 int i; 299 dnode_t *dn = db->db_dnode; 300 301 ASSERT(MUTEX_HELD(&db->db_mtx)); 302 303 if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY)) 304 return; 305 306 ASSERT(db->db_objset != NULL); 307 if (dn == NULL) { 308 ASSERT(db->db_parent == NULL); 309 ASSERT(db->db_blkptr == NULL); 310 } else { 311 ASSERT3U(db->db.db_object, ==, dn->dn_object); 312 ASSERT3P(db->db_objset, ==, dn->dn_objset); 313 ASSERT3U(db->db_level, <, dn->dn_nlevels); 314 ASSERT(db->db_blkid == DB_BONUS_BLKID || 315 list_head(&dn->dn_dbufs)); 316 } 317 if (db->db_blkid == DB_BONUS_BLKID) { 318 ASSERT(dn != NULL); 319 ASSERT3U(db->db.db_size, ==, dn->dn_bonuslen); 320 ASSERT3U(db->db.db_offset, ==, DB_BONUS_BLKID); 321 } else { 322 ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size); 323 } 324 325 if (db->db_level == 0) { 326 /* we can be momentarily larger in dnode_set_blksz() */ 327 if (db->db_blkid != DB_BONUS_BLKID && dn) { 328 ASSERT3U(db->db.db_size, >=, dn->dn_datablksz); 329 } 330 if (db->db.db_object == DMU_META_DNODE_OBJECT) { 331 for (i = 0; i < TXG_SIZE; i++) { 332 /* 333 * it should only be modified in syncing 334 * context, so make sure we only have 335 * one copy of the data. 336 */ 337 ASSERT(db->db_d.db_data_old[i] == NULL || 338 db->db_d.db_data_old[i] == db->db_buf); 339 } 340 } 341 } 342 343 /* verify db->db_blkptr */ 344 if (db->db_blkptr) { 345 if (db->db_parent == dn->dn_dbuf) { 346 /* db is pointed to by the dnode */ 347 /* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */ 348 if (db->db.db_object == DMU_META_DNODE_OBJECT) 349 ASSERT(db->db_parent == NULL); 350 else 351 ASSERT(db->db_parent != NULL); 352 ASSERT3P(db->db_blkptr, ==, 353 &dn->dn_phys->dn_blkptr[db->db_blkid]); 354 } else { 355 /* db is pointed to by an indirect block */ 356 int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT; 357 ASSERT3U(db->db_parent->db_level, ==, db->db_level+1); 358 ASSERT3U(db->db_parent->db.db_object, ==, 359 db->db.db_object); 360 /* 361 * dnode_grow_indblksz() can make this fail if we don't 362 * have the struct_rwlock. XXX indblksz no longer 363 * grows. safe to do this now? 364 */ 365 if (RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)) { 366 ASSERT3P(db->db_blkptr, ==, 367 ((blkptr_t *)db->db_parent->db.db_data + 368 db->db_blkid % epb)); 369 } 370 } 371 } 372 if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) && 373 db->db.db_data && db->db_blkid != DB_BONUS_BLKID && 374 db->db_state != DB_FILL && !dn->dn_free_txg) { 375 /* 376 * If the blkptr isn't set but they have nonzero data, 377 * it had better be dirty, otherwise we'll lose that 378 * data when we evict this buffer. 379 */ 380 if (db->db_dirtycnt == 0) { 381 uint64_t *buf = db->db.db_data; 382 int i; 383 384 for (i = 0; i < db->db.db_size >> 3; i++) { 385 ASSERT(buf[i] == 0); 386 } 387 } 388 } 389 } 390 #endif 391 392 static void 393 dbuf_update_data(dmu_buf_impl_t *db) 394 { 395 ASSERT(MUTEX_HELD(&db->db_mtx)); 396 if (db->db_level == 0 && db->db_d.db_user_data_ptr_ptr) { 397 ASSERT(!refcount_is_zero(&db->db_holds)); 398 *db->db_d.db_user_data_ptr_ptr = db->db.db_data; 399 } 400 } 401 402 static void 403 dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf) 404 { 405 ASSERT(MUTEX_HELD(&db->db_mtx)); 406 ASSERT(db->db_buf == NULL || !arc_has_callback(db->db_buf)); 407 db->db_buf = buf; 408 if (buf != NULL) { 409 ASSERT(buf->b_data != NULL); 410 db->db.db_data = buf->b_data; 411 if (!arc_released(buf)) 412 arc_set_callback(buf, dbuf_do_evict, db); 413 dbuf_update_data(db); 414 } else { 415 dbuf_evict_user(db); 416 db->db.db_data = NULL; 417 db->db_state = DB_UNCACHED; 418 } 419 } 420 421 uint64_t 422 dbuf_whichblock(dnode_t *dn, uint64_t offset) 423 { 424 if (dn->dn_datablkshift) { 425 return (offset >> dn->dn_datablkshift); 426 } else { 427 ASSERT3U(offset, <, dn->dn_datablksz); 428 return (0); 429 } 430 } 431 432 static void 433 dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb) 434 { 435 dmu_buf_impl_t *db = vdb; 436 437 mutex_enter(&db->db_mtx); 438 ASSERT3U(db->db_state, ==, DB_READ); 439 /* 440 * All reads are synchronous, so we must have a hold on the dbuf 441 */ 442 ASSERT(refcount_count(&db->db_holds) > 0); 443 ASSERT(db->db_buf == NULL); 444 ASSERT(db->db.db_data == NULL); 445 if (db->db_level == 0 && db->db_d.db_freed_in_flight) { 446 /* we were freed in flight; disregard any error */ 447 arc_release(buf, db); 448 bzero(buf->b_data, db->db.db_size); 449 db->db_d.db_freed_in_flight = FALSE; 450 dbuf_set_data(db, buf); 451 db->db_state = DB_CACHED; 452 } else if (zio == NULL || zio->io_error == 0) { 453 dbuf_set_data(db, buf); 454 db->db_state = DB_CACHED; 455 } else { 456 ASSERT(db->db_blkid != DB_BONUS_BLKID); 457 ASSERT3P(db->db_buf, ==, NULL); 458 VERIFY(arc_buf_remove_ref(buf, db) == 1); 459 db->db_state = DB_UNCACHED; 460 } 461 cv_broadcast(&db->db_changed); 462 mutex_exit(&db->db_mtx); 463 dbuf_rele(db, NULL); 464 } 465 466 static void 467 dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags) 468 { 469 blkptr_t *bp; 470 zbookmark_t zb; 471 472 ASSERT(!refcount_is_zero(&db->db_holds)); 473 /* We need the struct_rwlock to prevent db_blkptr from changing. */ 474 ASSERT(RW_LOCK_HELD(&db->db_dnode->dn_struct_rwlock)); 475 ASSERT(MUTEX_HELD(&db->db_mtx)); 476 ASSERT(db->db_state == DB_UNCACHED); 477 ASSERT(db->db_buf == NULL); 478 479 if (db->db_blkid == DB_BONUS_BLKID) { 480 ASSERT3U(db->db_dnode->dn_bonuslen, ==, db->db.db_size); 481 db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN); 482 if (db->db.db_size < DN_MAX_BONUSLEN) 483 bzero(db->db.db_data, DN_MAX_BONUSLEN); 484 bcopy(DN_BONUS(db->db_dnode->dn_phys), db->db.db_data, 485 db->db.db_size); 486 dbuf_update_data(db); 487 db->db_state = DB_CACHED; 488 mutex_exit(&db->db_mtx); 489 return; 490 } 491 492 if (db->db_level == 0 && dnode_block_freed(db->db_dnode, db->db_blkid)) 493 bp = NULL; 494 else 495 bp = db->db_blkptr; 496 497 if (bp == NULL) 498 dprintf_dbuf(db, "blkptr: %s\n", "NULL"); 499 else 500 dprintf_dbuf_bp(db, bp, "%s", "blkptr:"); 501 502 if (bp == NULL || BP_IS_HOLE(bp)) { 503 ASSERT(bp == NULL || BP_IS_HOLE(bp)); 504 dbuf_set_data(db, arc_buf_alloc(db->db_dnode->dn_objset->os_spa, 505 db->db.db_size, db)); 506 bzero(db->db.db_data, db->db.db_size); 507 db->db_state = DB_CACHED; 508 mutex_exit(&db->db_mtx); 509 return; 510 } 511 512 db->db_state = DB_READ; 513 mutex_exit(&db->db_mtx); 514 515 zb.zb_objset = db->db_objset->os_dsl_dataset ? 516 db->db_objset->os_dsl_dataset->ds_object : 0; 517 zb.zb_object = db->db.db_object; 518 zb.zb_level = db->db_level; 519 zb.zb_blkid = db->db_blkid; 520 521 dbuf_add_ref(db, NULL); 522 /* ZIO_FLAG_CANFAIL callers have to check the parent zio's error */ 523 (void) arc_read(zio, db->db_dnode->dn_objset->os_spa, bp, 524 db->db_level > 0 ? byteswap_uint64_array : 525 dmu_ot[db->db_dnode->dn_type].ot_byteswap, 526 dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ, 527 (flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED, 528 ARC_NOWAIT, &zb); 529 } 530 531 int 532 dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags) 533 { 534 int err = 0; 535 int havepzio = (zio != NULL); 536 537 /* 538 * We don't have to hold the mutex to check db_state because it 539 * can't be freed while we have a hold on the buffer. 540 */ 541 ASSERT(!refcount_is_zero(&db->db_holds)); 542 if (db->db_state == DB_CACHED) 543 return (0); 544 545 if ((flags & DB_RF_HAVESTRUCT) == 0) 546 rw_enter(&db->db_dnode->dn_struct_rwlock, RW_READER); 547 548 mutex_enter(&db->db_mtx); 549 if (db->db_state == DB_CACHED) { 550 mutex_exit(&db->db_mtx); 551 if ((flags & DB_RF_HAVESTRUCT) == 0) 552 rw_exit(&db->db_dnode->dn_struct_rwlock); 553 } else if (db->db_state == DB_UNCACHED) { 554 if (zio == NULL) { 555 zio = zio_root(db->db_dnode->dn_objset->os_spa, 556 NULL, NULL, ZIO_FLAG_CANFAIL); 557 } 558 dbuf_read_impl(db, zio, flags); 559 /* dbuf_read_impl has dropped db_mtx for us */ 560 561 if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID && 562 (flags & DB_RF_NOPREFETCH) == 0 && 563 db->db_dnode != NULL) { 564 dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset, 565 db->db.db_size); 566 } 567 568 if ((flags & DB_RF_HAVESTRUCT) == 0) 569 rw_exit(&db->db_dnode->dn_struct_rwlock); 570 571 if (!havepzio) 572 err = zio_wait(zio); 573 } else { 574 if ((flags & DB_RF_HAVESTRUCT) == 0) 575 rw_exit(&db->db_dnode->dn_struct_rwlock); 576 if ((flags & DB_RF_NEVERWAIT) == 0) { 577 while (db->db_state == DB_READ || 578 db->db_state == DB_FILL) { 579 ASSERT(db->db_state == DB_READ || 580 (flags & DB_RF_HAVESTRUCT) == 0); 581 cv_wait(&db->db_changed, &db->db_mtx); 582 } 583 if (db->db_state == DB_UNCACHED) 584 err = EIO; 585 } 586 mutex_exit(&db->db_mtx); 587 } 588 589 ASSERT(err || havepzio || db->db_state == DB_CACHED); 590 return (err); 591 } 592 593 static void 594 dbuf_noread(dmu_buf_impl_t *db) 595 { 596 ASSERT(!refcount_is_zero(&db->db_holds)); 597 ASSERT(db->db_blkid != DB_BONUS_BLKID); 598 mutex_enter(&db->db_mtx); 599 while (db->db_state == DB_READ || db->db_state == DB_FILL) 600 cv_wait(&db->db_changed, &db->db_mtx); 601 if (db->db_state == DB_UNCACHED) { 602 ASSERT(db->db_buf == NULL); 603 ASSERT(db->db.db_data == NULL); 604 dbuf_set_data(db, arc_buf_alloc(db->db_dnode->dn_objset->os_spa, 605 db->db.db_size, db)); 606 db->db_state = DB_FILL; 607 } else { 608 ASSERT3U(db->db_state, ==, DB_CACHED); 609 } 610 mutex_exit(&db->db_mtx); 611 } 612 613 /* 614 * This is our just-in-time copy function. It makes a copy of 615 * buffers, that have been modified in a previous transaction 616 * group, before we modify them in the current active group. 617 * 618 * This function is used in two places: when we are dirtying a 619 * buffer for the first time in a txg, and when we are freeing 620 * a range in a dnode that includes this buffer. 621 * 622 * Note that when we are called from dbuf_free_range() we do 623 * not put a hold on the buffer, we just traverse the active 624 * dbuf list for the dnode. 625 */ 626 static void 627 dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg) 628 { 629 arc_buf_t **quiescing, **syncing; 630 631 ASSERT(MUTEX_HELD(&db->db_mtx)); 632 ASSERT(db->db.db_data != NULL); 633 ASSERT(db->db_blkid != DB_BONUS_BLKID); 634 635 quiescing = (arc_buf_t **)&db->db_d.db_data_old[(txg-1)&TXG_MASK]; 636 syncing = (arc_buf_t **)&db->db_d.db_data_old[(txg-2)&TXG_MASK]; 637 638 /* 639 * If this buffer is referenced from the current quiescing 640 * transaction group: either make a copy and reset the reference 641 * to point to the copy, or (if there a no active holders) just 642 * null out the current db_data pointer. 643 */ 644 if (*quiescing == db->db_buf) { 645 /* 646 * If the quiescing txg is "dirty", then we better not 647 * be referencing the same buffer from the syncing txg. 648 */ 649 ASSERT(*syncing != db->db_buf); 650 if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 651 int size = db->db.db_size; 652 *quiescing = arc_buf_alloc( 653 db->db_dnode->dn_objset->os_spa, size, db); 654 bcopy(db->db.db_data, (*quiescing)->b_data, size); 655 } else { 656 dbuf_set_data(db, NULL); 657 } 658 return; 659 } 660 661 /* 662 * If this buffer is referenced from the current syncing 663 * transaction group: either 664 * 1 - make a copy and reset the reference, or 665 * 2 - if there are no holders, just null the current db_data. 666 */ 667 if (*syncing == db->db_buf) { 668 ASSERT3P(*quiescing, ==, NULL); 669 ASSERT3U(db->db_dirtycnt, ==, 1); 670 if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 671 int size = db->db.db_size; 672 /* we can't copy if we have already started a write */ 673 ASSERT(*syncing != db->db_data_pending); 674 *syncing = arc_buf_alloc( 675 db->db_dnode->dn_objset->os_spa, size, db); 676 bcopy(db->db.db_data, (*syncing)->b_data, size); 677 } else { 678 dbuf_set_data(db, NULL); 679 } 680 } 681 } 682 683 /* 684 * This is the "bonus buffer" version of the above routine 685 */ 686 static void 687 dbuf_fix_old_bonus_data(dmu_buf_impl_t *db, uint64_t txg) 688 { 689 void **quiescing, **syncing; 690 691 ASSERT(MUTEX_HELD(&db->db_mtx)); 692 ASSERT(db->db.db_data != NULL); 693 ASSERT(db->db_blkid == DB_BONUS_BLKID); 694 695 quiescing = &db->db_d.db_data_old[(txg-1)&TXG_MASK]; 696 syncing = &db->db_d.db_data_old[(txg-2)&TXG_MASK]; 697 698 if (*quiescing == db->db.db_data) { 699 ASSERT(*syncing != db->db.db_data); 700 *quiescing = zio_buf_alloc(DN_MAX_BONUSLEN); 701 bcopy(db->db.db_data, *quiescing, DN_MAX_BONUSLEN); 702 } else if (*syncing == db->db.db_data) { 703 ASSERT3P(*quiescing, ==, NULL); 704 ASSERT3U(db->db_dirtycnt, ==, 1); 705 *syncing = zio_buf_alloc(DN_MAX_BONUSLEN); 706 bcopy(db->db.db_data, *syncing, DN_MAX_BONUSLEN); 707 } 708 } 709 710 void 711 dbuf_unoverride(dmu_buf_impl_t *db, uint64_t txg) 712 { 713 ASSERT(db->db_blkid != DB_BONUS_BLKID); 714 ASSERT(MUTEX_HELD(&db->db_mtx)); 715 if (db->db_d.db_overridden_by[txg&TXG_MASK] == IN_DMU_SYNC) { 716 db->db_d.db_overridden_by[txg&TXG_MASK] = NULL; 717 } else if (db->db_d.db_overridden_by[txg&TXG_MASK] != NULL) { 718 /* free this block */ 719 ASSERT(list_link_active(&db->db_dirty_node[txg&TXG_MASK]) || 720 db->db_dnode->dn_free_txg == txg); 721 if (!BP_IS_HOLE(db->db_d.db_overridden_by[txg&TXG_MASK])) { 722 /* XXX can get silent EIO here */ 723 (void) arc_free(NULL, db->db_dnode->dn_objset->os_spa, 724 txg, db->db_d.db_overridden_by[txg&TXG_MASK], 725 NULL, NULL, ARC_WAIT); 726 } 727 kmem_free(db->db_d.db_overridden_by[txg&TXG_MASK], 728 sizeof (blkptr_t)); 729 db->db_d.db_overridden_by[txg&TXG_MASK] = NULL; 730 /* release the already-written buffer */ 731 arc_release(db->db_d.db_data_old[txg&TXG_MASK], db); 732 } 733 } 734 735 void 736 dbuf_free_range(dnode_t *dn, uint64_t blkid, uint64_t nblks, dmu_tx_t *tx) 737 { 738 dmu_buf_impl_t *db, *db_next; 739 uint64_t txg = tx->tx_txg; 740 741 dprintf_dnode(dn, "blkid=%llu nblks=%llu\n", blkid, nblks); 742 mutex_enter(&dn->dn_dbufs_mtx); 743 for (db = list_head(&dn->dn_dbufs); db; db = db_next) { 744 db_next = list_next(&dn->dn_dbufs, db); 745 ASSERT(db->db_blkid != DB_BONUS_BLKID); 746 if (db->db_level != 0) 747 continue; 748 dprintf_dbuf(db, "found buf %s\n", ""); 749 if (db->db_blkid < blkid || 750 db->db_blkid >= blkid+nblks) 751 continue; 752 753 /* found a level 0 buffer in the range */ 754 if (dbuf_undirty(db, tx)) 755 continue; 756 757 mutex_enter(&db->db_mtx); 758 if (db->db_state == DB_UNCACHED || 759 db->db_state == DB_EVICTING) { 760 ASSERT(db->db.db_data == NULL); 761 mutex_exit(&db->db_mtx); 762 continue; 763 } 764 if (db->db_state == DB_READ) { 765 /* this will be handled in dbuf_read_done() */ 766 db->db_d.db_freed_in_flight = TRUE; 767 mutex_exit(&db->db_mtx); 768 continue; 769 } 770 if (db->db_state == DB_FILL) { 771 /* this will be handled in dbuf_rele() */ 772 db->db_d.db_freed_in_flight = TRUE; 773 mutex_exit(&db->db_mtx); 774 continue; 775 } 776 if (refcount_count(&db->db_holds) == 0) { 777 ASSERT(db->db_buf); 778 dbuf_clear(db); 779 continue; 780 } 781 /* The dbuf is CACHED and referenced */ 782 783 if (!list_link_active(&db->db_dirty_node[txg & TXG_MASK])) { 784 /* 785 * This dbuf is not currently dirty. We will either 786 * uncache it (if its not referenced in the open 787 * context) or reset its contents to empty. 788 */ 789 dbuf_fix_old_data(db, txg); 790 } else if (db->db_d.db_overridden_by[txg & TXG_MASK] != NULL) { 791 /* 792 * This dbuf is overridden. Clear that state. 793 */ 794 dbuf_unoverride(db, txg); 795 } 796 /* fill in with appropriate data */ 797 if (db->db_state == DB_CACHED) { 798 ASSERT(db->db.db_data != NULL); 799 arc_release(db->db_buf, db); 800 bzero(db->db.db_data, db->db.db_size); 801 } 802 803 mutex_exit(&db->db_mtx); 804 } 805 mutex_exit(&dn->dn_dbufs_mtx); 806 } 807 808 static int 809 dbuf_new_block(dmu_buf_impl_t *db) 810 { 811 dsl_dataset_t *ds = db->db_objset->os_dsl_dataset; 812 uint64_t birth_txg = 0; 813 814 /* Don't count meta-objects */ 815 if (ds == NULL) 816 return (FALSE); 817 818 /* 819 * We don't need any locking to protect db_blkptr: 820 * If it's syncing, then db_dirtied will be set so we'll 821 * ignore db_blkptr. 822 */ 823 ASSERT(MUTEX_HELD(&db->db_mtx)); /* XXX strictly necessary? */ 824 /* If we have been dirtied since the last snapshot, its not new */ 825 if (db->db_dirtied) 826 birth_txg = db->db_dirtied; 827 else if (db->db_blkptr) 828 birth_txg = db->db_blkptr->blk_birth; 829 830 if (birth_txg) 831 return (!dsl_dataset_block_freeable(ds, birth_txg)); 832 else 833 return (TRUE); 834 } 835 836 void 837 dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx) 838 { 839 arc_buf_t *buf, *obuf; 840 int osize = db->db.db_size; 841 842 ASSERT(db->db_blkid != DB_BONUS_BLKID); 843 844 /* XXX does *this* func really need the lock? */ 845 ASSERT(RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)); 846 847 if (osize == size) 848 return; 849 850 /* 851 * This call to dbuf_will_dirty() with the dn_struct_rwlock held 852 * is OK, because there can be no other references to the db 853 * when we are changing its size, so no concurrent DB_FILL can 854 * be happening. 855 */ 856 /* Make a copy of the data if necessary */ 857 /* 858 * XXX we should be doing a dbuf_read, checking the return 859 * value and returning that up to our callers 860 */ 861 dbuf_will_dirty(db, tx); 862 863 /* create the data buffer for the new block */ 864 buf = arc_buf_alloc(db->db_dnode->dn_objset->os_spa, size, db); 865 866 /* copy old block data to the new block */ 867 obuf = db->db_buf; 868 bcopy(obuf->b_data, buf->b_data, MIN(osize, size)); 869 /* zero the remainder */ 870 if (size > osize) 871 bzero((uint8_t *)buf->b_data + osize, size - osize); 872 873 mutex_enter(&db->db_mtx); 874 /* ASSERT3U(refcount_count(&db->db_holds), ==, 1); */ 875 dbuf_set_data(db, buf); 876 VERIFY(arc_buf_remove_ref(obuf, db) == 1); 877 db->db.db_size = size; 878 879 /* fix up the dirty info */ 880 if (db->db_level == 0) 881 db->db_d.db_data_old[tx->tx_txg&TXG_MASK] = buf; 882 mutex_exit(&db->db_mtx); 883 884 dnode_willuse_space(db->db_dnode, size-osize, tx); 885 } 886 887 void 888 dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 889 { 890 dnode_t *dn = db->db_dnode; 891 objset_impl_t *os = dn->dn_objset; 892 int drop_struct_lock = FALSE; 893 int txgoff = tx->tx_txg & TXG_MASK; 894 895 ASSERT(tx->tx_txg != 0); 896 ASSERT(!refcount_is_zero(&db->db_holds)); 897 DMU_TX_DIRTY_BUF(tx, db); 898 899 /* 900 * Shouldn't dirty a regular buffer in syncing context. Private 901 * objects may be dirtied in syncing context, but only if they 902 * were already pre-dirtied in open context. 903 * XXX We may want to prohibit dirtying in syncing context even 904 * if they did pre-dirty. 905 */ 906 ASSERT(!(dmu_tx_is_syncing(tx) && 907 !BP_IS_HOLE(&dn->dn_objset->os_rootbp) && 908 dn->dn_object != DMU_META_DNODE_OBJECT && 909 dn->dn_objset->os_dsl_dataset != NULL && 910 !dsl_dir_is_private( 911 dn->dn_objset->os_dsl_dataset->ds_dir))); 912 913 /* 914 * We make this assert for private objects as well, but after we 915 * check if we're already dirty. They are allowed to re-dirty 916 * in syncing context. 917 */ 918 ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 919 dn->dn_dirtyctx == DN_UNDIRTIED || 920 dn->dn_dirtyctx == 921 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 922 923 mutex_enter(&db->db_mtx); 924 /* XXX make this true for indirects too? */ 925 ASSERT(db->db_level != 0 || db->db_state == DB_CACHED || 926 db->db_state == DB_FILL); 927 928 /* 929 * If this buffer is currently part of an "overridden" region, 930 * we now need to remove it from that region. 931 */ 932 if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID && 933 db->db_d.db_overridden_by[txgoff] != NULL) { 934 dbuf_unoverride(db, tx->tx_txg); 935 } 936 937 mutex_enter(&dn->dn_mtx); 938 /* 939 * Don't set dirtyctx to SYNC if we're just modifying this as we 940 * initialize the objset. 941 */ 942 if (dn->dn_dirtyctx == DN_UNDIRTIED && 943 !BP_IS_HOLE(&dn->dn_objset->os_rootbp)) { 944 dn->dn_dirtyctx = 945 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN); 946 ASSERT(dn->dn_dirtyctx_firstset == NULL); 947 dn->dn_dirtyctx_firstset = kmem_alloc(1, KM_SLEEP); 948 } 949 mutex_exit(&dn->dn_mtx); 950 951 /* 952 * If this buffer is already dirty, we're done. 953 */ 954 if (list_link_active(&db->db_dirty_node[txgoff])) { 955 mutex_exit(&db->db_mtx); 956 return; 957 } 958 959 /* 960 * Only valid if not already dirty. 961 */ 962 ASSERT(dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 963 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 964 965 ASSERT3U(dn->dn_nlevels, >, db->db_level); 966 ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) || 967 dn->dn_phys->dn_nlevels > db->db_level || 968 dn->dn_next_nlevels[txgoff] > db->db_level || 969 dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level || 970 dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level); 971 972 /* 973 * We should only be dirtying in syncing context if it's the 974 * mos, a spa os, or we're initializing the os. However, we are 975 * allowed to dirty in syncing context provided we already 976 * dirtied it in open context. Hence we must make this 977 * assertion only if we're not already dirty. 978 */ 979 ASSERT(!dmu_tx_is_syncing(tx) || 980 os->os_dsl_dataset == NULL || 981 !dsl_dir_is_private(os->os_dsl_dataset->ds_dir) || 982 !BP_IS_HOLE(&os->os_rootbp)); 983 ASSERT(db->db.db_size != 0); 984 985 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 986 987 /* 988 * If this buffer is dirty in an old transaction group we need 989 * to make a copy of it so that the changes we make in this 990 * transaction group won't leak out when we sync the older txg. 991 */ 992 if (db->db_blkid == DB_BONUS_BLKID) { 993 ASSERT(db->db.db_data != NULL); 994 ASSERT(db->db_d.db_data_old[txgoff] == NULL); 995 dbuf_fix_old_bonus_data(db, tx->tx_txg); 996 db->db_d.db_data_old[txgoff] = db->db.db_data; 997 } else if (db->db_level == 0) { 998 /* 999 * Release the data buffer from the cache so that we 1000 * can modify it without impacting possible other users 1001 * of this cached data block. Note that indirect blocks 1002 * and private objects are not released until the syncing 1003 * state (since they are only modified then). 1004 */ 1005 ASSERT(db->db_buf != NULL); 1006 ASSERT(db->db_d.db_data_old[txgoff] == NULL); 1007 if (db->db.db_object != DMU_META_DNODE_OBJECT) { 1008 arc_release(db->db_buf, db); 1009 dbuf_fix_old_data(db, tx->tx_txg); 1010 ASSERT(db->db_buf != NULL); 1011 } 1012 db->db_d.db_data_old[txgoff] = db->db_buf; 1013 } 1014 1015 mutex_enter(&dn->dn_mtx); 1016 /* 1017 * We could have been freed_in_flight between the dbuf_noread 1018 * and dbuf_dirty. We win, as though the dbuf_noread() had 1019 * happened after the free. 1020 */ 1021 if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID) { 1022 dnode_clear_range(dn, db->db_blkid, 1, tx); 1023 db->db_d.db_freed_in_flight = FALSE; 1024 } 1025 1026 db->db_dirtied = tx->tx_txg; 1027 list_insert_tail(&dn->dn_dirty_dbufs[txgoff], db); 1028 mutex_exit(&dn->dn_mtx); 1029 1030 if (db->db_blkid != DB_BONUS_BLKID) { 1031 /* 1032 * Update the accounting. 1033 */ 1034 if (!dbuf_new_block(db) && db->db_blkptr) { 1035 /* 1036 * This is only a guess -- if the dbuf is dirty 1037 * in a previous txg, we don't know how much 1038 * space it will use on disk yet. We should 1039 * really have the struct_rwlock to access 1040 * db_blkptr, but since this is just a guess, 1041 * it's OK if we get an odd answer. 1042 */ 1043 dnode_willuse_space(dn, 1044 -BP_GET_ASIZE(db->db_blkptr), tx); 1045 } 1046 dnode_willuse_space(dn, db->db.db_size, tx); 1047 } 1048 1049 /* 1050 * This buffer is now part of this txg 1051 */ 1052 dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg); 1053 db->db_dirtycnt += 1; 1054 ASSERT3U(db->db_dirtycnt, <=, 3); 1055 1056 mutex_exit(&db->db_mtx); 1057 1058 if (db->db_blkid == DB_BONUS_BLKID) { 1059 dnode_setdirty(dn, tx); 1060 return; 1061 } 1062 1063 if (db->db_level == 0) 1064 dnode_new_blkid(dn, db->db_blkid, tx); 1065 1066 if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 1067 rw_enter(&dn->dn_struct_rwlock, RW_READER); 1068 drop_struct_lock = TRUE; 1069 } 1070 1071 if (db->db_level < dn->dn_nlevels-1) { 1072 int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1073 dmu_buf_impl_t *parent; 1074 parent = dbuf_hold_level(dn, db->db_level+1, 1075 db->db_blkid >> epbs, FTAG); 1076 if (drop_struct_lock) 1077 rw_exit(&dn->dn_struct_rwlock); 1078 dbuf_dirty(parent, tx); 1079 dbuf_rele(parent, FTAG); 1080 } else { 1081 if (drop_struct_lock) 1082 rw_exit(&dn->dn_struct_rwlock); 1083 } 1084 1085 dnode_setdirty(dn, tx); 1086 } 1087 1088 static int 1089 dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1090 { 1091 dnode_t *dn = db->db_dnode; 1092 int txgoff = tx->tx_txg & TXG_MASK; 1093 int64_t holds; 1094 1095 ASSERT(tx->tx_txg != 0); 1096 ASSERT(db->db_blkid != DB_BONUS_BLKID); 1097 1098 mutex_enter(&db->db_mtx); 1099 1100 /* 1101 * If this buffer is not dirty, we're done. 1102 */ 1103 if (!list_link_active(&db->db_dirty_node[txgoff])) { 1104 mutex_exit(&db->db_mtx); 1105 return (0); 1106 } 1107 1108 /* 1109 * If this buffer is currently held, we cannot undirty 1110 * it, since one of the current holders may be in the 1111 * middle of an update. Note that users of dbuf_undirty() 1112 * should not place a hold on the dbuf before the call. 1113 * XXX - this check assumes we are being called from 1114 * dbuf_free_range(), perhaps we should move it there? 1115 */ 1116 if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 1117 mutex_exit(&db->db_mtx); 1118 mutex_enter(&dn->dn_mtx); 1119 dnode_clear_range(dn, db->db_blkid, 1, tx); 1120 mutex_exit(&dn->dn_mtx); 1121 return (0); 1122 } 1123 1124 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1125 1126 dbuf_unoverride(db, tx->tx_txg); 1127 1128 ASSERT(db->db.db_size != 0); 1129 if (db->db_level == 0) { 1130 ASSERT(db->db_buf != NULL); 1131 ASSERT(db->db_d.db_data_old[txgoff] != NULL); 1132 if (db->db_d.db_data_old[txgoff] != db->db_buf) 1133 VERIFY(arc_buf_remove_ref( 1134 db->db_d.db_data_old[txgoff], db) == 1); 1135 db->db_d.db_data_old[txgoff] = NULL; 1136 } 1137 1138 /* XXX would be nice to fix up dn_towrite_space[] */ 1139 /* XXX undo db_dirtied? but how? */ 1140 /* db->db_dirtied = tx->tx_txg; */ 1141 1142 mutex_enter(&dn->dn_mtx); 1143 list_remove(&dn->dn_dirty_dbufs[txgoff], db); 1144 mutex_exit(&dn->dn_mtx); 1145 1146 ASSERT(db->db_dirtycnt > 0); 1147 db->db_dirtycnt -= 1; 1148 1149 if ((holds = refcount_remove(&db->db_holds, 1150 (void *)(uintptr_t)tx->tx_txg)) == 0) { 1151 arc_buf_t *buf = db->db_buf; 1152 1153 ASSERT(arc_released(buf)); 1154 dbuf_set_data(db, NULL); 1155 VERIFY(arc_buf_remove_ref(buf, db) == 1); 1156 dbuf_evict(db); 1157 return (1); 1158 } 1159 ASSERT(holds > 0); 1160 1161 mutex_exit(&db->db_mtx); 1162 return (0); 1163 } 1164 1165 #pragma weak dmu_buf_will_dirty = dbuf_will_dirty 1166 void 1167 dbuf_will_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1168 { 1169 int rf = DB_RF_MUST_SUCCEED; 1170 1171 ASSERT(tx->tx_txg != 0); 1172 ASSERT(!refcount_is_zero(&db->db_holds)); 1173 1174 if (RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)) 1175 rf |= DB_RF_HAVESTRUCT; 1176 (void) dbuf_read(db, NULL, rf); 1177 dbuf_dirty(db, tx); 1178 } 1179 1180 void 1181 dmu_buf_will_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 1182 { 1183 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1184 1185 ASSERT(db->db_blkid != DB_BONUS_BLKID); 1186 ASSERT(tx->tx_txg != 0); 1187 ASSERT(db->db_level == 0); 1188 ASSERT(!refcount_is_zero(&db->db_holds)); 1189 1190 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT || 1191 dmu_tx_private_ok(tx)); 1192 1193 dbuf_noread(db); 1194 dbuf_dirty(db, tx); 1195 } 1196 1197 #pragma weak dmu_buf_fill_done = dbuf_fill_done 1198 /* ARGSUSED */ 1199 void 1200 dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx) 1201 { 1202 mutex_enter(&db->db_mtx); 1203 DBUF_VERIFY(db); 1204 1205 if (db->db_state == DB_FILL) { 1206 if (db->db_level == 0 && db->db_d.db_freed_in_flight) { 1207 ASSERT(db->db_blkid != DB_BONUS_BLKID); 1208 /* we were freed while filling */ 1209 /* XXX dbuf_undirty? */ 1210 bzero(db->db.db_data, db->db.db_size); 1211 db->db_d.db_freed_in_flight = FALSE; 1212 } 1213 db->db_state = DB_CACHED; 1214 cv_broadcast(&db->db_changed); 1215 } 1216 mutex_exit(&db->db_mtx); 1217 } 1218 1219 /* 1220 * "Clear" the contents of this dbuf. This will mark the dbuf 1221 * EVICTING and clear *most* of its references. Unfortunetely, 1222 * when we are not holding the dn_dbufs_mtx, we can't clear the 1223 * entry in the dn_dbufs list. We have to wait until dbuf_destroy() 1224 * in this case. For callers from the DMU we will usually see: 1225 * dbuf_clear()->arc_buf_evict()->dbuf_do_evict()->dbuf_destroy() 1226 * For the arc callback, we will usually see: 1227 * dbuf_do_evict()->dbuf_clear();dbuf_destroy() 1228 * Sometimes, though, we will get a mix of these two: 1229 * DMU: dbuf_clear()->arc_buf_evict() 1230 * ARC: dbuf_do_evict()->dbuf_destroy() 1231 */ 1232 void 1233 dbuf_clear(dmu_buf_impl_t *db) 1234 { 1235 dnode_t *dn = db->db_dnode; 1236 dmu_buf_impl_t *parent = db->db_parent; 1237 int dbuf_gone = FALSE; 1238 1239 ASSERT(MUTEX_HELD(&db->db_mtx)); 1240 ASSERT(refcount_is_zero(&db->db_holds)); 1241 1242 dbuf_evict_user(db); 1243 1244 if (db->db_state == DB_CACHED) { 1245 ASSERT(db->db.db_data != NULL); 1246 if (db->db_blkid == DB_BONUS_BLKID) 1247 zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN); 1248 db->db.db_data = NULL; 1249 db->db_state = DB_UNCACHED; 1250 } 1251 1252 ASSERT3U(db->db_state, ==, DB_UNCACHED); 1253 ASSERT(db->db_data_pending == NULL); 1254 1255 db->db_state = DB_EVICTING; 1256 db->db_blkptr = NULL; 1257 1258 if (db->db_blkid != DB_BONUS_BLKID && MUTEX_HELD(&dn->dn_dbufs_mtx)) { 1259 list_remove(&dn->dn_dbufs, db); 1260 dnode_rele(dn, db); 1261 } 1262 1263 if (db->db_buf) 1264 dbuf_gone = arc_buf_evict(db->db_buf); 1265 1266 if (!dbuf_gone) 1267 mutex_exit(&db->db_mtx); 1268 1269 /* 1270 * If this dbuf is referened from an indirect dbuf, 1271 * decrement the ref count on the indirect dbuf. 1272 */ 1273 if (parent && parent != dn->dn_dbuf) 1274 dbuf_rele(parent, db); 1275 } 1276 1277 static int 1278 dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse, 1279 dmu_buf_impl_t **parentp, blkptr_t **bpp) 1280 { 1281 int nlevels, epbs; 1282 1283 ASSERT(blkid != DB_BONUS_BLKID); 1284 1285 if (dn->dn_phys->dn_nlevels == 0) 1286 nlevels = 1; 1287 else 1288 nlevels = dn->dn_phys->dn_nlevels; 1289 1290 epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1291 1292 ASSERT3U(level * epbs, <, 64); 1293 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1294 if (level >= nlevels || 1295 (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) { 1296 /* the buffer has no parent yet */ 1297 *parentp = NULL; 1298 *bpp = NULL; 1299 return (ENOENT); 1300 } else if (level < nlevels-1) { 1301 /* this block is referenced from an indirect block */ 1302 int err = dbuf_hold_impl(dn, level+1, 1303 blkid >> epbs, fail_sparse, NULL, parentp); 1304 if (err) 1305 return (err); 1306 err = dbuf_read(*parentp, NULL, 1307 (DB_RF_HAVESTRUCT | DB_RF_NOPREFETCH | DB_RF_CANFAIL)); 1308 if (err == 0) { 1309 *bpp = ((blkptr_t *)(*parentp)->db.db_data) + 1310 (blkid & ((1ULL << epbs) - 1)); 1311 } 1312 return (err); 1313 } else { 1314 /* the block is referenced from the dnode */ 1315 ASSERT3U(level, ==, nlevels-1); 1316 ASSERT(dn->dn_phys->dn_nblkptr == 0 || 1317 blkid < dn->dn_phys->dn_nblkptr); 1318 *parentp = dn->dn_dbuf; 1319 *bpp = &dn->dn_phys->dn_blkptr[blkid]; 1320 return (0); 1321 } 1322 } 1323 1324 static dmu_buf_impl_t * 1325 dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid, 1326 dmu_buf_impl_t *parent, blkptr_t *blkptr) 1327 { 1328 objset_impl_t *os = dn->dn_objset; 1329 dmu_buf_impl_t *db, *odb; 1330 1331 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1332 ASSERT(dn->dn_type != DMU_OT_NONE); 1333 1334 db = kmem_cache_alloc(dbuf_cache, KM_SLEEP); 1335 1336 db->db_objset = os; 1337 db->db.db_object = dn->dn_object; 1338 db->db_level = level; 1339 db->db_blkid = blkid; 1340 db->db_dirtied = 0; 1341 db->db_dirtycnt = 0; 1342 db->db_dnode = dn; 1343 db->db_parent = parent; 1344 db->db_blkptr = blkptr; 1345 1346 bzero(&db->db_d, sizeof (db->db_d)); 1347 1348 if (blkid == DB_BONUS_BLKID) { 1349 ASSERT3P(parent, ==, dn->dn_dbuf); 1350 db->db.db_size = dn->dn_bonuslen; 1351 db->db.db_offset = DB_BONUS_BLKID; 1352 db->db_state = DB_UNCACHED; 1353 /* the bonus dbuf is not placed in the hash table */ 1354 return (db); 1355 } else { 1356 int blocksize = 1357 db->db_level ? 1<<dn->dn_indblkshift : dn->dn_datablksz; 1358 db->db.db_size = blocksize; 1359 db->db.db_offset = db->db_blkid * blocksize; 1360 } 1361 1362 /* 1363 * Hold the dn_dbufs_mtx while we get the new dbuf 1364 * in the hash table *and* added to the dbufs list. 1365 * This prevents a possible deadlock with someone 1366 * trying to look up this dbuf before its added to the 1367 * dn_dbufs list. 1368 */ 1369 mutex_enter(&dn->dn_dbufs_mtx); 1370 db->db_state = DB_EVICTING; 1371 if ((odb = dbuf_hash_insert(db)) != NULL) { 1372 /* someone else inserted it first */ 1373 kmem_cache_free(dbuf_cache, db); 1374 mutex_exit(&dn->dn_dbufs_mtx); 1375 return (odb); 1376 } 1377 list_insert_head(&dn->dn_dbufs, db); 1378 db->db_state = DB_UNCACHED; 1379 mutex_exit(&dn->dn_dbufs_mtx); 1380 1381 if (parent && parent != dn->dn_dbuf) 1382 dbuf_add_ref(parent, db); 1383 1384 ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 1385 refcount_count(&dn->dn_holds) > 0); 1386 (void) refcount_add(&dn->dn_holds, db); 1387 1388 dprintf_dbuf(db, "db=%p\n", db); 1389 1390 return (db); 1391 } 1392 1393 static int 1394 dbuf_do_evict(void *private) 1395 { 1396 arc_buf_t *buf = private; 1397 dmu_buf_impl_t *db = buf->b_private; 1398 1399 if (!MUTEX_HELD(&db->db_mtx)) 1400 mutex_enter(&db->db_mtx); 1401 1402 ASSERT(db->db_buf == buf); 1403 ASSERT(refcount_is_zero(&db->db_holds)); 1404 1405 if (db->db_state != DB_EVICTING) { 1406 ASSERT(db->db_state == DB_CACHED); 1407 DBUF_VERIFY(db); 1408 db->db_buf = NULL; 1409 dbuf_evict(db); 1410 } else { 1411 mutex_exit(&db->db_mtx); 1412 dbuf_destroy(db); 1413 } 1414 return (0); 1415 } 1416 1417 static void 1418 dbuf_destroy(dmu_buf_impl_t *db) 1419 { 1420 ASSERT(refcount_is_zero(&db->db_holds)); 1421 1422 if (db->db_blkid != DB_BONUS_BLKID) { 1423 dnode_t *dn = db->db_dnode; 1424 1425 /* 1426 * If this dbuf is still on the dn_dbufs list, 1427 * remove it from that list. 1428 */ 1429 if (list_link_active(&db->db_link)) { 1430 int need_mutex; 1431 1432 ASSERT(!MUTEX_HELD(&dn->dn_dbufs_mtx)); 1433 need_mutex = !MUTEX_HELD(&dn->dn_dbufs_mtx); 1434 if (need_mutex) 1435 mutex_enter(&dn->dn_dbufs_mtx); 1436 1437 /* remove from dn_dbufs */ 1438 list_remove(&dn->dn_dbufs, db); 1439 1440 if (need_mutex) 1441 mutex_exit(&dn->dn_dbufs_mtx); 1442 1443 dnode_rele(dn, db); 1444 } 1445 dbuf_hash_remove(db); 1446 } 1447 db->db_parent = NULL; 1448 db->db_dnode = NULL; 1449 db->db_buf = NULL; 1450 1451 ASSERT(db->db.db_data == NULL); 1452 ASSERT(db->db_hash_next == NULL); 1453 ASSERT(db->db_blkptr == NULL); 1454 ASSERT(db->db_data_pending == NULL); 1455 1456 kmem_cache_free(dbuf_cache, db); 1457 } 1458 1459 void 1460 dbuf_prefetch(dnode_t *dn, uint64_t blkid) 1461 { 1462 dmu_buf_impl_t *db, *parent = NULL; 1463 blkptr_t *bp = NULL; 1464 1465 ASSERT(blkid != DB_BONUS_BLKID); 1466 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1467 1468 if (dnode_block_freed(dn, blkid)) 1469 return; 1470 1471 /* dbuf_find() returns with db_mtx held */ 1472 if (db = dbuf_find(dn, 0, blkid)) { 1473 /* 1474 * This dbuf is already in the cache. We assume that 1475 * it is already CACHED, or else about to be either 1476 * read or filled. 1477 */ 1478 mutex_exit(&db->db_mtx); 1479 return; 1480 } 1481 1482 if (dbuf_findbp(dn, 0, blkid, TRUE, &parent, &bp) == 0) { 1483 if (bp && !BP_IS_HOLE(bp)) { 1484 zbookmark_t zb; 1485 zb.zb_objset = dn->dn_objset->os_dsl_dataset ? 1486 dn->dn_objset->os_dsl_dataset->ds_object : 0; 1487 zb.zb_object = dn->dn_object; 1488 zb.zb_level = 0; 1489 zb.zb_blkid = blkid; 1490 1491 (void) arc_read(NULL, dn->dn_objset->os_spa, bp, 1492 dmu_ot[dn->dn_type].ot_byteswap, 1493 NULL, NULL, ZIO_PRIORITY_ASYNC_READ, 1494 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 1495 (ARC_NOWAIT | ARC_PREFETCH), &zb); 1496 } 1497 if (parent && parent != dn->dn_dbuf) 1498 dbuf_rele(parent, NULL); 1499 } 1500 } 1501 1502 /* 1503 * Returns with db_holds incremented, and db_mtx not held. 1504 * Note: dn_struct_rwlock must be held. 1505 */ 1506 int 1507 dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, int fail_sparse, 1508 void *tag, dmu_buf_impl_t **dbp) 1509 { 1510 dmu_buf_impl_t *db, *parent = NULL; 1511 1512 ASSERT(blkid != DB_BONUS_BLKID); 1513 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1514 ASSERT3U(dn->dn_nlevels, >, level); 1515 1516 *dbp = NULL; 1517 top: 1518 /* dbuf_find() returns with db_mtx held */ 1519 db = dbuf_find(dn, level, blkid); 1520 1521 if (db == NULL) { 1522 blkptr_t *bp = NULL; 1523 int err; 1524 1525 err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp); 1526 if (fail_sparse) { 1527 if (err == 0 && bp && BP_IS_HOLE(bp)) 1528 err = ENOENT; 1529 if (err) { 1530 if (parent && parent != dn->dn_dbuf) 1531 dbuf_rele(parent, NULL); 1532 return (err); 1533 } 1534 } 1535 if (err && err != ENOENT) 1536 return (err); 1537 db = dbuf_create(dn, level, blkid, parent, bp); 1538 } 1539 1540 if (db->db_buf && refcount_is_zero(&db->db_holds)) { 1541 arc_buf_add_ref(db->db_buf, db); 1542 if (db->db_buf->b_data == NULL) { 1543 dbuf_clear(db); 1544 goto top; 1545 } 1546 ASSERT3P(db->db.db_data, ==, db->db_buf->b_data); 1547 } 1548 1549 ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf)); 1550 1551 /* 1552 * If this buffer is currently syncing out, and we are 1553 * are still referencing it from db_data, we need to make 1554 * a copy of it in case we decide we want to dirty it 1555 * again in this txg. 1556 */ 1557 if (db->db_level == 0 && db->db_state == DB_CACHED && 1558 dn->dn_object != DMU_META_DNODE_OBJECT && 1559 db->db_data_pending == db->db_buf) { 1560 int size = (db->db_blkid == DB_BONUS_BLKID) ? 1561 DN_MAX_BONUSLEN : db->db.db_size; 1562 1563 dbuf_set_data(db, arc_buf_alloc(db->db_dnode->dn_objset->os_spa, 1564 size, db)); 1565 bcopy(db->db_data_pending->b_data, db->db.db_data, 1566 db->db.db_size); 1567 } 1568 1569 (void) refcount_add(&db->db_holds, tag); 1570 dbuf_update_data(db); 1571 DBUF_VERIFY(db); 1572 mutex_exit(&db->db_mtx); 1573 1574 /* NOTE: we can't rele the parent until after we drop the db_mtx */ 1575 if (parent && parent != dn->dn_dbuf) 1576 dbuf_rele(parent, NULL); 1577 1578 ASSERT3P(db->db_dnode, ==, dn); 1579 ASSERT3U(db->db_blkid, ==, blkid); 1580 ASSERT3U(db->db_level, ==, level); 1581 *dbp = db; 1582 1583 return (0); 1584 } 1585 1586 dmu_buf_impl_t * 1587 dbuf_hold(dnode_t *dn, uint64_t blkid, void *tag) 1588 { 1589 dmu_buf_impl_t *db; 1590 int err = dbuf_hold_impl(dn, 0, blkid, FALSE, tag, &db); 1591 return (err ? NULL : db); 1592 } 1593 1594 dmu_buf_impl_t * 1595 dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, void *tag) 1596 { 1597 dmu_buf_impl_t *db; 1598 int err = dbuf_hold_impl(dn, level, blkid, FALSE, tag, &db); 1599 return (err ? NULL : db); 1600 } 1601 1602 dmu_buf_impl_t * 1603 dbuf_create_bonus(dnode_t *dn) 1604 { 1605 dmu_buf_impl_t *db = dn->dn_bonus; 1606 1607 ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 1608 1609 ASSERT(dn->dn_bonus == NULL); 1610 db = dbuf_create(dn, 0, DB_BONUS_BLKID, dn->dn_dbuf, NULL); 1611 return (db); 1612 } 1613 1614 #pragma weak dmu_buf_add_ref = dbuf_add_ref 1615 void 1616 dbuf_add_ref(dmu_buf_impl_t *db, void *tag) 1617 { 1618 int64_t holds = refcount_add(&db->db_holds, tag); 1619 ASSERT(holds > 1); 1620 } 1621 1622 #pragma weak dmu_buf_rele = dbuf_rele 1623 void 1624 dbuf_rele(dmu_buf_impl_t *db, void *tag) 1625 { 1626 int64_t holds; 1627 1628 mutex_enter(&db->db_mtx); 1629 DBUF_VERIFY(db); 1630 1631 holds = refcount_remove(&db->db_holds, tag); 1632 ASSERT(holds >= 0); 1633 1634 if (holds == db->db_dirtycnt && 1635 db->db_level == 0 && db->db_d.db_immediate_evict) 1636 dbuf_evict_user(db); 1637 1638 if (holds == 0) { 1639 if (db->db_blkid == DB_BONUS_BLKID) { 1640 mutex_exit(&db->db_mtx); 1641 dnode_rele(db->db_dnode, db); 1642 } else if (db->db_buf == NULL) { 1643 /* 1644 * This is a special case: we never associated this 1645 * dbuf with any data allocated from the ARC. 1646 */ 1647 ASSERT3U(db->db_state, ==, DB_UNCACHED); 1648 dbuf_evict(db); 1649 } else if (arc_released(db->db_buf)) { 1650 arc_buf_t *buf = db->db_buf; 1651 /* 1652 * This dbuf has anonymous data associated with it. 1653 */ 1654 dbuf_set_data(db, NULL); 1655 VERIFY(arc_buf_remove_ref(buf, db) == 1); 1656 dbuf_evict(db); 1657 } else { 1658 VERIFY(arc_buf_remove_ref(db->db_buf, db) == 0); 1659 mutex_exit(&db->db_mtx); 1660 } 1661 } else { 1662 mutex_exit(&db->db_mtx); 1663 } 1664 } 1665 1666 #pragma weak dmu_buf_refcount = dbuf_refcount 1667 uint64_t 1668 dbuf_refcount(dmu_buf_impl_t *db) 1669 { 1670 return (refcount_count(&db->db_holds)); 1671 } 1672 1673 void * 1674 dmu_buf_set_user(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 1675 dmu_buf_evict_func_t *evict_func) 1676 { 1677 return (dmu_buf_update_user(db_fake, NULL, user_ptr, 1678 user_data_ptr_ptr, evict_func)); 1679 } 1680 1681 void * 1682 dmu_buf_set_user_ie(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 1683 dmu_buf_evict_func_t *evict_func) 1684 { 1685 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1686 1687 db->db_d.db_immediate_evict = TRUE; 1688 return (dmu_buf_update_user(db_fake, NULL, user_ptr, 1689 user_data_ptr_ptr, evict_func)); 1690 } 1691 1692 void * 1693 dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr, void *user_ptr, 1694 void *user_data_ptr_ptr, dmu_buf_evict_func_t *evict_func) 1695 { 1696 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1697 ASSERT(db->db_level == 0); 1698 1699 ASSERT((user_ptr == NULL) == (evict_func == NULL)); 1700 1701 mutex_enter(&db->db_mtx); 1702 1703 if (db->db_d.db_user_ptr == old_user_ptr) { 1704 db->db_d.db_user_ptr = user_ptr; 1705 db->db_d.db_user_data_ptr_ptr = user_data_ptr_ptr; 1706 db->db_d.db_evict_func = evict_func; 1707 1708 dbuf_update_data(db); 1709 } else { 1710 old_user_ptr = db->db_d.db_user_ptr; 1711 } 1712 1713 mutex_exit(&db->db_mtx); 1714 return (old_user_ptr); 1715 } 1716 1717 void * 1718 dmu_buf_get_user(dmu_buf_t *db_fake) 1719 { 1720 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1721 ASSERT(!refcount_is_zero(&db->db_holds)); 1722 1723 return (db->db_d.db_user_ptr); 1724 } 1725 1726 void 1727 dbuf_sync(dmu_buf_impl_t *db, zio_t *zio, dmu_tx_t *tx) 1728 { 1729 arc_buf_t **data; 1730 uint64_t txg = tx->tx_txg; 1731 dnode_t *dn = db->db_dnode; 1732 objset_impl_t *os = dn->dn_objset; 1733 int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 1734 int checksum, compress; 1735 zbookmark_t zb; 1736 int blksz; 1737 1738 ASSERT(dmu_tx_is_syncing(tx)); 1739 1740 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 1741 1742 mutex_enter(&db->db_mtx); 1743 /* 1744 * To be synced, we must be dirtied. But we 1745 * might have been freed after the dirty. 1746 */ 1747 if (db->db_state == DB_UNCACHED) { 1748 /* This buffer has been freed since it was dirtied */ 1749 ASSERT(db->db.db_data == NULL); 1750 } else if (db->db_state == DB_FILL) { 1751 /* This buffer was freed and is now being re-filled */ 1752 ASSERT(db->db.db_data != db->db_d.db_data_old[txg&TXG_MASK]); 1753 } else { 1754 ASSERT3U(db->db_state, ==, DB_CACHED); 1755 } 1756 DBUF_VERIFY(db); 1757 1758 /* 1759 * Don't need a lock on db_dirty (dn_mtx), because it can't 1760 * be modified yet. 1761 */ 1762 1763 if (db->db_blkid == DB_BONUS_BLKID) { 1764 void **datap = &db->db_d.db_data_old[txg&TXG_MASK]; 1765 /* 1766 * Simply copy the bonus data into the dnode. It will 1767 * be written out when the dnode is synced (and it will 1768 * be synced, since it must have been dirty for dbuf_sync 1769 * to be called). 1770 */ 1771 /* 1772 * Use dn_phys->dn_bonuslen since db.db_size is the length 1773 * of the bonus buffer in the open transaction rather than 1774 * the syncing transaction. 1775 */ 1776 ASSERT(*datap != NULL); 1777 ASSERT3U(db->db_level, ==, 0); 1778 ASSERT3U(dn->dn_phys->dn_bonuslen, <=, DN_MAX_BONUSLEN); 1779 bcopy(*datap, DN_BONUS(dn->dn_phys), dn->dn_phys->dn_bonuslen); 1780 if (*datap != db->db.db_data) 1781 zio_buf_free(*datap, DN_MAX_BONUSLEN); 1782 db->db_d.db_data_old[txg&TXG_MASK] = NULL; 1783 db->db_data_pending = NULL; 1784 if (db->db_dirtied == txg) 1785 db->db_dirtied = 0; 1786 ASSERT(db->db_dirtycnt > 0); 1787 db->db_dirtycnt -= 1; 1788 mutex_exit(&db->db_mtx); 1789 dbuf_rele(db, (void *)(uintptr_t)txg); 1790 return; 1791 } 1792 1793 if (db->db_level == 0) { 1794 data = (arc_buf_t **)&db->db_d.db_data_old[txg&TXG_MASK]; 1795 blksz = arc_buf_size(*data); 1796 /* 1797 * If this buffer is currently "in use" (i.e., there are 1798 * active holds and db_data still references it), then make 1799 * a copy before we start the write so that any modifications 1800 * from the open txg will not leak into this write. 1801 * 1802 * NOTE: this copy does not need to be made for objects only 1803 * modified in the syncing context (e.g. DNONE_DNODE blocks) 1804 * or if there is no actual write involved (bonus blocks). 1805 */ 1806 if (dn->dn_object != DMU_META_DNODE_OBJECT && 1807 db->db_d.db_overridden_by[txg&TXG_MASK] == NULL) { 1808 if (refcount_count(&db->db_holds) > 1 && 1809 *data == db->db_buf) { 1810 *data = arc_buf_alloc(os->os_spa, blksz, db); 1811 bcopy(db->db.db_data, (*data)->b_data, blksz); 1812 } 1813 db->db_data_pending = *data; 1814 } else if (dn->dn_object == DMU_META_DNODE_OBJECT) { 1815 /* 1816 * Private object buffers are released here rather 1817 * than in dbuf_dirty() since they are only modified 1818 * in the syncing context and we don't want the 1819 * overhead of making multiple copies of the data. 1820 */ 1821 arc_release(db->db_buf, db); 1822 } 1823 } else { 1824 data = &db->db_buf; 1825 if (*data == NULL) { 1826 /* 1827 * This can happen if we dirty and then free 1828 * the level-0 data blocks in the same txg. So 1829 * this indirect remains unchanged. 1830 */ 1831 if (db->db_dirtied == txg) 1832 db->db_dirtied = 0; 1833 ASSERT(db->db_dirtycnt > 0); 1834 db->db_dirtycnt -= 1; 1835 mutex_exit(&db->db_mtx); 1836 dbuf_rele(db, (void *)(uintptr_t)txg); 1837 return; 1838 } 1839 blksz = db->db.db_size; 1840 ASSERT3U(blksz, ==, 1<<dn->dn_phys->dn_indblkshift); 1841 } 1842 1843 ASSERT(*data != NULL); 1844 1845 if (db->db_level > 0 && !arc_released(db->db_buf)) { 1846 /* 1847 * This indirect buffer was marked dirty, but 1848 * never modified (if it had been modified, then 1849 * we would have released the buffer). There is 1850 * no reason to write anything. 1851 */ 1852 db->db_data_pending = NULL; 1853 if (db->db_dirtied == txg) 1854 db->db_dirtied = 0; 1855 ASSERT(db->db_dirtycnt > 0); 1856 db->db_dirtycnt -= 1; 1857 mutex_exit(&db->db_mtx); 1858 dbuf_rele(db, (void *)(uintptr_t)txg); 1859 return; 1860 } else if (db->db_blkptr == NULL && 1861 db->db_level == dn->dn_phys->dn_nlevels-1 && 1862 db->db_blkid < dn->dn_phys->dn_nblkptr) { 1863 /* 1864 * This buffer was allocated at a time when there was 1865 * no available blkptrs from the dnode, or it was 1866 * inappropriate to hook it in (i.e., nlevels mis-match). 1867 */ 1868 ASSERT(db->db_blkptr == NULL); 1869 ASSERT(db->db_parent == NULL); 1870 db->db_parent = dn->dn_dbuf; 1871 db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid]; 1872 DBUF_VERIFY(db); 1873 mutex_exit(&db->db_mtx); 1874 } else if (db->db_blkptr == NULL) { 1875 dmu_buf_impl_t *parent = db->db_parent; 1876 1877 mutex_exit(&db->db_mtx); 1878 ASSERT(dn->dn_phys->dn_nlevels > 1); 1879 if (parent == NULL) { 1880 rw_enter(&dn->dn_struct_rwlock, RW_READER); 1881 (void) dbuf_hold_impl(dn, db->db_level+1, 1882 db->db_blkid >> epbs, FALSE, FTAG, &parent); 1883 rw_exit(&dn->dn_struct_rwlock); 1884 dbuf_add_ref(parent, db); 1885 db->db_parent = parent; 1886 dbuf_rele(parent, FTAG); 1887 } 1888 (void) dbuf_read(parent, NULL, DB_RF_MUST_SUCCEED); 1889 } else { 1890 mutex_exit(&db->db_mtx); 1891 } 1892 1893 ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || db->db_parent != NULL); 1894 1895 if (db->db_level > 0 && 1896 db->db_blkid > dn->dn_phys->dn_maxblkid >> (db->db_level * epbs)) { 1897 /* 1898 * Don't write indirect blocks past EOF. 1899 * We get these when we truncate a file *after* dirtying 1900 * blocks in the truncate range (we undirty the level 0 1901 * blocks in dbuf_free_range(), but not the indirects). 1902 */ 1903 #ifdef ZFS_DEBUG 1904 /* 1905 * Verify that this indirect block is empty. 1906 */ 1907 blkptr_t *bplist; 1908 int i; 1909 1910 mutex_enter(&db->db_mtx); 1911 bplist = db->db.db_data; 1912 for (i = 0; i < (1 << epbs); i++) { 1913 if (!BP_IS_HOLE(&bplist[i])) { 1914 panic("data past EOF: " 1915 "db=%p level=%d id=%llu i=%d\n", 1916 db, db->db_level, 1917 (u_longlong_t)db->db_blkid, i); 1918 } 1919 } 1920 mutex_exit(&db->db_mtx); 1921 #endif 1922 ASSERT(db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)); 1923 mutex_enter(&db->db_mtx); 1924 db->db_dirtycnt -= 1; 1925 mutex_exit(&db->db_mtx); 1926 dbuf_rele(db, (void *)(uintptr_t)txg); 1927 return; 1928 } 1929 1930 if (db->db_parent != dn->dn_dbuf) { 1931 dmu_buf_impl_t *parent = db->db_parent; 1932 1933 mutex_enter(&db->db_mtx); 1934 ASSERT(db->db_level == parent->db_level-1); 1935 ASSERT(list_link_active(&parent->db_dirty_node[txg&TXG_MASK])); 1936 /* 1937 * We may have read this indirect block after we dirtied it, 1938 * so never released it from the cache. 1939 */ 1940 arc_release(parent->db_buf, db->db_parent); 1941 1942 db->db_blkptr = (blkptr_t *)parent->db.db_data + 1943 (db->db_blkid & ((1ULL << epbs) - 1)); 1944 DBUF_VERIFY(db); 1945 mutex_exit(&db->db_mtx); 1946 #ifdef ZFS_DEBUG 1947 } else { 1948 /* 1949 * We don't need to dnode_setdirty(dn) because if we got 1950 * here then the parent is already dirty. 1951 */ 1952 ASSERT(db->db_level == dn->dn_phys->dn_nlevels-1); 1953 ASSERT3P(db->db_blkptr, ==, 1954 &dn->dn_phys->dn_blkptr[db->db_blkid]); 1955 #endif 1956 } 1957 ASSERT(db->db_parent == NULL || arc_released(db->db_parent->db_buf)); 1958 1959 if (db->db_level == 0 && 1960 db->db_d.db_overridden_by[txg&TXG_MASK] != NULL) { 1961 arc_buf_t **old = 1962 (arc_buf_t **)&db->db_d.db_data_old[txg&TXG_MASK]; 1963 blkptr_t **bpp = &db->db_d.db_overridden_by[txg&TXG_MASK]; 1964 int old_size = BP_GET_ASIZE(db->db_blkptr); 1965 int new_size = BP_GET_ASIZE(*bpp); 1966 1967 ASSERT(db->db_blkid != DB_BONUS_BLKID); 1968 1969 dnode_diduse_space(dn, new_size-old_size); 1970 mutex_enter(&dn->dn_mtx); 1971 if (db->db_blkid > dn->dn_phys->dn_maxblkid) 1972 dn->dn_phys->dn_maxblkid = db->db_blkid; 1973 mutex_exit(&dn->dn_mtx); 1974 1975 dsl_dataset_block_born(os->os_dsl_dataset, *bpp, tx); 1976 if (!BP_IS_HOLE(db->db_blkptr)) 1977 dsl_dataset_block_kill(os->os_dsl_dataset, 1978 db->db_blkptr, os->os_synctx); 1979 1980 mutex_enter(&db->db_mtx); 1981 *db->db_blkptr = **bpp; 1982 kmem_free(*bpp, sizeof (blkptr_t)); 1983 *bpp = NULL; 1984 1985 if (*old != db->db_buf) 1986 VERIFY(arc_buf_remove_ref(*old, db) == 1); 1987 else if (!BP_IS_HOLE(db->db_blkptr)) 1988 arc_set_callback(db->db_buf, dbuf_do_evict, db); 1989 else 1990 ASSERT(arc_released(db->db_buf)); 1991 *old = NULL; 1992 db->db_data_pending = NULL; 1993 1994 cv_broadcast(&db->db_changed); 1995 1996 ASSERT(db->db_dirtycnt > 0); 1997 db->db_dirtycnt -= 1; 1998 mutex_exit(&db->db_mtx); 1999 dbuf_rele(db, (void *)(uintptr_t)txg); 2000 return; 2001 } 2002 2003 if (db->db_level > 0) { 2004 /* 2005 * XXX -- we should design a compression algorithm 2006 * that specializes in arrays of bps. 2007 */ 2008 checksum = ZIO_CHECKSUM_FLETCHER_4; 2009 compress = ZIO_COMPRESS_LZJB; 2010 } else { 2011 /* 2012 * Allow dnode settings to override objset settings, 2013 * except for metadata checksums. 2014 */ 2015 if (dmu_ot[dn->dn_type].ot_metadata) { 2016 checksum = os->os_md_checksum; 2017 compress = zio_compress_select(dn->dn_compress, 2018 os->os_md_compress); 2019 } else { 2020 checksum = zio_checksum_select(dn->dn_checksum, 2021 os->os_checksum); 2022 compress = zio_compress_select(dn->dn_compress, 2023 os->os_compress); 2024 } 2025 } 2026 #ifdef ZFS_DEBUG 2027 if (db->db_parent) { 2028 ASSERT(list_link_active( 2029 &db->db_parent->db_dirty_node[txg&TXG_MASK])); 2030 ASSERT(db->db_parent == dn->dn_dbuf || 2031 db->db_parent->db_level > 0); 2032 if (dn->dn_object == DMU_META_DNODE_OBJECT || db->db_level > 0) 2033 ASSERT(*data == db->db_buf); 2034 } 2035 #endif 2036 ASSERT3U(db->db_blkptr->blk_birth, <=, tx->tx_txg); 2037 zb.zb_objset = os->os_dsl_dataset ? os->os_dsl_dataset->ds_object : 0; 2038 zb.zb_object = db->db.db_object; 2039 zb.zb_level = db->db_level; 2040 zb.zb_blkid = db->db_blkid; 2041 (void) arc_write(zio, os->os_spa, checksum, compress, txg, 2042 db->db_blkptr, *data, dbuf_write_done, db, 2043 ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, ARC_NOWAIT, &zb); 2044 /* 2045 * We can't access db after arc_write, since it could finish 2046 * and be freed, and we have no locks on it. 2047 */ 2048 } 2049 2050 struct dbuf_arg { 2051 objset_impl_t *os; 2052 blkptr_t bp; 2053 }; 2054 2055 static void 2056 dbuf_do_born(void *arg) 2057 { 2058 struct dbuf_arg *da = arg; 2059 dsl_dataset_block_born(da->os->os_dsl_dataset, 2060 &da->bp, da->os->os_synctx); 2061 kmem_free(da, sizeof (struct dbuf_arg)); 2062 } 2063 2064 static void 2065 dbuf_do_kill(void *arg) 2066 { 2067 struct dbuf_arg *da = arg; 2068 dsl_dataset_block_kill(da->os->os_dsl_dataset, 2069 &da->bp, da->os->os_synctx); 2070 kmem_free(da, sizeof (struct dbuf_arg)); 2071 } 2072 2073 /* ARGSUSED */ 2074 static void 2075 dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb) 2076 { 2077 dmu_buf_impl_t *db = vdb; 2078 dnode_t *dn = db->db_dnode; 2079 objset_impl_t *os = dn->dn_objset; 2080 uint64_t txg = zio->io_txg; 2081 uint64_t fill = 0; 2082 int i; 2083 int old_size, new_size; 2084 2085 ASSERT3U(zio->io_error, ==, 0); 2086 2087 dprintf_dbuf_bp(db, &zio->io_bp_orig, "bp_orig: %s", ""); 2088 2089 old_size = BP_GET_ASIZE(&zio->io_bp_orig); 2090 new_size = BP_GET_ASIZE(zio->io_bp); 2091 2092 dnode_diduse_space(dn, new_size-old_size); 2093 2094 mutex_enter(&db->db_mtx); 2095 2096 if (db->db_dirtied == txg) 2097 db->db_dirtied = 0; 2098 2099 if (db->db_level == 0) { 2100 arc_buf_t **old = 2101 (arc_buf_t **)&db->db_d.db_data_old[txg&TXG_MASK]; 2102 2103 ASSERT(db->db_blkid != DB_BONUS_BLKID); 2104 2105 if (*old != db->db_buf) 2106 VERIFY(arc_buf_remove_ref(*old, db) == 1); 2107 else if (!BP_IS_HOLE(db->db_blkptr)) 2108 arc_set_callback(db->db_buf, dbuf_do_evict, db); 2109 else 2110 ASSERT(arc_released(db->db_buf)); 2111 *old = NULL; 2112 db->db_data_pending = NULL; 2113 2114 mutex_enter(&dn->dn_mtx); 2115 if (db->db_blkid > dn->dn_phys->dn_maxblkid && 2116 !BP_IS_HOLE(db->db_blkptr)) 2117 dn->dn_phys->dn_maxblkid = db->db_blkid; 2118 mutex_exit(&dn->dn_mtx); 2119 2120 if (dn->dn_type == DMU_OT_DNODE) { 2121 dnode_phys_t *dnp = db->db.db_data; 2122 for (i = db->db.db_size >> DNODE_SHIFT; i > 0; 2123 i--, dnp++) { 2124 if (dnp->dn_type != DMU_OT_NONE) 2125 fill++; 2126 } 2127 } else { 2128 if (!BP_IS_HOLE(db->db_blkptr)) 2129 fill = 1; 2130 } 2131 } else { 2132 blkptr_t *bp = db->db.db_data; 2133 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2134 if (!BP_IS_HOLE(db->db_blkptr)) { 2135 int epbs = 2136 dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2137 ASSERT3U(BP_GET_LSIZE(zio->io_bp), ==, db->db.db_size); 2138 ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, 2139 db->db.db_size); 2140 ASSERT3U(dn->dn_phys->dn_maxblkid 2141 >> (db->db_level * epbs), >=, db->db_blkid); 2142 arc_set_callback(db->db_buf, dbuf_do_evict, db); 2143 } 2144 for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, bp++) { 2145 if (BP_IS_HOLE(bp)) 2146 continue; 2147 ASSERT3U(BP_GET_LSIZE(bp), ==, 2148 db->db_level == 1 ? dn->dn_datablksz : 2149 (1<<dn->dn_phys->dn_indblkshift)); 2150 fill += bp->blk_fill; 2151 } 2152 } 2153 2154 if (!BP_IS_HOLE(db->db_blkptr)) { 2155 db->db_blkptr->blk_fill = fill; 2156 BP_SET_TYPE(db->db_blkptr, dn->dn_type); 2157 BP_SET_LEVEL(db->db_blkptr, db->db_level); 2158 } else { 2159 ASSERT3U(fill, ==, 0); 2160 ASSERT3U(db->db_blkptr->blk_fill, ==, 0); 2161 } 2162 2163 dprintf_dbuf_bp(db, db->db_blkptr, 2164 "wrote %llu bytes to blkptr:", zio->io_size); 2165 2166 ASSERT(db->db_parent == NULL || 2167 list_link_active(&db->db_parent->db_dirty_node[txg&TXG_MASK])); 2168 cv_broadcast(&db->db_changed); 2169 ASSERT(db->db_dirtycnt > 0); 2170 db->db_dirtycnt -= 1; 2171 mutex_exit(&db->db_mtx); 2172 2173 /* We must do this after we've set the bp's type and level */ 2174 if (!DVA_EQUAL(BP_IDENTITY(zio->io_bp), 2175 BP_IDENTITY(&zio->io_bp_orig))) { 2176 struct dbuf_arg *da; 2177 da = kmem_alloc(sizeof (struct dbuf_arg), KM_SLEEP); 2178 da->os = os; 2179 da->bp = *zio->io_bp; 2180 (void) taskq_dispatch(dbuf_tq, dbuf_do_born, da, 0); 2181 if (!BP_IS_HOLE(&zio->io_bp_orig)) { 2182 da = kmem_alloc(sizeof (struct dbuf_arg), KM_SLEEP); 2183 da->os = os; 2184 da->bp = zio->io_bp_orig; 2185 (void) taskq_dispatch(dbuf_tq, dbuf_do_kill, da, 0); 2186 } 2187 } 2188 2189 dbuf_rele(db, (void *)(uintptr_t)txg); 2190 } 2191