1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <sys/zfs_context.h> 27 #include <sys/dmu.h> 28 #include <sys/dmu_impl.h> 29 #include <sys/dbuf.h> 30 #include <sys/dmu_objset.h> 31 #include <sys/dsl_dataset.h> 32 #include <sys/dsl_dir.h> 33 #include <sys/dmu_tx.h> 34 #include <sys/spa.h> 35 #include <sys/zio.h> 36 #include <sys/dmu_zfetch.h> 37 38 static void dbuf_destroy(dmu_buf_impl_t *db); 39 static int dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx); 40 static void dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx); 41 static arc_done_func_t dbuf_write_ready; 42 static arc_done_func_t dbuf_write_done; 43 44 /* 45 * Global data structures and functions for the dbuf cache. 46 */ 47 static kmem_cache_t *dbuf_cache; 48 49 /* ARGSUSED */ 50 static int 51 dbuf_cons(void *vdb, void *unused, int kmflag) 52 { 53 dmu_buf_impl_t *db = vdb; 54 bzero(db, sizeof (dmu_buf_impl_t)); 55 56 mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL); 57 cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL); 58 refcount_create(&db->db_holds); 59 return (0); 60 } 61 62 /* ARGSUSED */ 63 static void 64 dbuf_dest(void *vdb, void *unused) 65 { 66 dmu_buf_impl_t *db = vdb; 67 mutex_destroy(&db->db_mtx); 68 cv_destroy(&db->db_changed); 69 refcount_destroy(&db->db_holds); 70 } 71 72 /* 73 * dbuf hash table routines 74 */ 75 static dbuf_hash_table_t dbuf_hash_table; 76 77 static uint64_t dbuf_hash_count; 78 79 static uint64_t 80 dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid) 81 { 82 uintptr_t osv = (uintptr_t)os; 83 uint64_t crc = -1ULL; 84 85 ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 86 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (lvl)) & 0xFF]; 87 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF]; 88 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF]; 89 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF]; 90 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 0)) & 0xFF]; 91 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 8)) & 0xFF]; 92 93 crc ^= (osv>>14) ^ (obj>>16) ^ (blkid>>16); 94 95 return (crc); 96 } 97 98 #define DBUF_HASH(os, obj, level, blkid) dbuf_hash(os, obj, level, blkid); 99 100 #define DBUF_EQUAL(dbuf, os, obj, level, blkid) \ 101 ((dbuf)->db.db_object == (obj) && \ 102 (dbuf)->db_objset == (os) && \ 103 (dbuf)->db_level == (level) && \ 104 (dbuf)->db_blkid == (blkid)) 105 106 dmu_buf_impl_t * 107 dbuf_find(dnode_t *dn, uint8_t level, uint64_t blkid) 108 { 109 dbuf_hash_table_t *h = &dbuf_hash_table; 110 objset_impl_t *os = dn->dn_objset; 111 uint64_t obj = dn->dn_object; 112 uint64_t hv = DBUF_HASH(os, obj, level, blkid); 113 uint64_t idx = hv & h->hash_table_mask; 114 dmu_buf_impl_t *db; 115 116 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 117 for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) { 118 if (DBUF_EQUAL(db, os, obj, level, blkid)) { 119 mutex_enter(&db->db_mtx); 120 if (db->db_state != DB_EVICTING) { 121 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 122 return (db); 123 } 124 mutex_exit(&db->db_mtx); 125 } 126 } 127 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 128 return (NULL); 129 } 130 131 /* 132 * Insert an entry into the hash table. If there is already an element 133 * equal to elem in the hash table, then the already existing element 134 * will be returned and the new element will not be inserted. 135 * Otherwise returns NULL. 136 */ 137 static dmu_buf_impl_t * 138 dbuf_hash_insert(dmu_buf_impl_t *db) 139 { 140 dbuf_hash_table_t *h = &dbuf_hash_table; 141 objset_impl_t *os = db->db_objset; 142 uint64_t obj = db->db.db_object; 143 int level = db->db_level; 144 uint64_t blkid = db->db_blkid; 145 uint64_t hv = DBUF_HASH(os, obj, level, blkid); 146 uint64_t idx = hv & h->hash_table_mask; 147 dmu_buf_impl_t *dbf; 148 149 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 150 for (dbf = h->hash_table[idx]; dbf != NULL; dbf = dbf->db_hash_next) { 151 if (DBUF_EQUAL(dbf, os, obj, level, blkid)) { 152 mutex_enter(&dbf->db_mtx); 153 if (dbf->db_state != DB_EVICTING) { 154 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 155 return (dbf); 156 } 157 mutex_exit(&dbf->db_mtx); 158 } 159 } 160 161 mutex_enter(&db->db_mtx); 162 db->db_hash_next = h->hash_table[idx]; 163 h->hash_table[idx] = db; 164 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 165 atomic_add_64(&dbuf_hash_count, 1); 166 167 return (NULL); 168 } 169 170 /* 171 * Remove an entry from the hash table. This operation will 172 * fail if there are any existing holds on the db. 173 */ 174 static void 175 dbuf_hash_remove(dmu_buf_impl_t *db) 176 { 177 dbuf_hash_table_t *h = &dbuf_hash_table; 178 uint64_t hv = DBUF_HASH(db->db_objset, db->db.db_object, 179 db->db_level, db->db_blkid); 180 uint64_t idx = hv & h->hash_table_mask; 181 dmu_buf_impl_t *dbf, **dbp; 182 183 /* 184 * We musn't hold db_mtx to maintin lock ordering: 185 * DBUF_HASH_MUTEX > db_mtx. 186 */ 187 ASSERT(refcount_is_zero(&db->db_holds)); 188 ASSERT(db->db_state == DB_EVICTING); 189 ASSERT(!MUTEX_HELD(&db->db_mtx)); 190 191 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 192 dbp = &h->hash_table[idx]; 193 while ((dbf = *dbp) != db) { 194 dbp = &dbf->db_hash_next; 195 ASSERT(dbf != NULL); 196 } 197 *dbp = db->db_hash_next; 198 db->db_hash_next = NULL; 199 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 200 atomic_add_64(&dbuf_hash_count, -1); 201 } 202 203 static arc_evict_func_t dbuf_do_evict; 204 205 static void 206 dbuf_evict_user(dmu_buf_impl_t *db) 207 { 208 ASSERT(MUTEX_HELD(&db->db_mtx)); 209 210 if (db->db_level != 0 || db->db_evict_func == NULL) 211 return; 212 213 if (db->db_user_data_ptr_ptr) 214 *db->db_user_data_ptr_ptr = db->db.db_data; 215 db->db_evict_func(&db->db, db->db_user_ptr); 216 db->db_user_ptr = NULL; 217 db->db_user_data_ptr_ptr = NULL; 218 db->db_evict_func = NULL; 219 } 220 221 void 222 dbuf_evict(dmu_buf_impl_t *db) 223 { 224 ASSERT(MUTEX_HELD(&db->db_mtx)); 225 ASSERT(db->db_buf == NULL); 226 ASSERT(db->db_data_pending == NULL); 227 228 dbuf_clear(db); 229 dbuf_destroy(db); 230 } 231 232 void 233 dbuf_init(void) 234 { 235 uint64_t hsize = 1ULL << 16; 236 dbuf_hash_table_t *h = &dbuf_hash_table; 237 int i; 238 239 /* 240 * The hash table is big enough to fill all of physical memory 241 * with an average 4K block size. The table will take up 242 * totalmem*sizeof(void*)/4K (i.e. 2MB/GB with 8-byte pointers). 243 */ 244 while (hsize * 4096 < physmem * PAGESIZE) 245 hsize <<= 1; 246 247 retry: 248 h->hash_table_mask = hsize - 1; 249 h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP); 250 if (h->hash_table == NULL) { 251 /* XXX - we should really return an error instead of assert */ 252 ASSERT(hsize > (1ULL << 10)); 253 hsize >>= 1; 254 goto retry; 255 } 256 257 dbuf_cache = kmem_cache_create("dmu_buf_impl_t", 258 sizeof (dmu_buf_impl_t), 259 0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0); 260 261 for (i = 0; i < DBUF_MUTEXES; i++) 262 mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL); 263 } 264 265 void 266 dbuf_fini(void) 267 { 268 dbuf_hash_table_t *h = &dbuf_hash_table; 269 int i; 270 271 for (i = 0; i < DBUF_MUTEXES; i++) 272 mutex_destroy(&h->hash_mutexes[i]); 273 kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *)); 274 kmem_cache_destroy(dbuf_cache); 275 } 276 277 /* 278 * Other stuff. 279 */ 280 281 #ifdef ZFS_DEBUG 282 static void 283 dbuf_verify(dmu_buf_impl_t *db) 284 { 285 dnode_t *dn = db->db_dnode; 286 287 ASSERT(MUTEX_HELD(&db->db_mtx)); 288 289 if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY)) 290 return; 291 292 ASSERT(db->db_objset != NULL); 293 if (dn == NULL) { 294 ASSERT(db->db_parent == NULL); 295 ASSERT(db->db_blkptr == NULL); 296 } else { 297 ASSERT3U(db->db.db_object, ==, dn->dn_object); 298 ASSERT3P(db->db_objset, ==, dn->dn_objset); 299 ASSERT3U(db->db_level, <, dn->dn_nlevels); 300 ASSERT(db->db_blkid == DB_BONUS_BLKID || 301 list_head(&dn->dn_dbufs)); 302 } 303 if (db->db_blkid == DB_BONUS_BLKID) { 304 ASSERT(dn != NULL); 305 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 306 ASSERT3U(db->db.db_offset, ==, DB_BONUS_BLKID); 307 } else { 308 ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size); 309 } 310 311 if (db->db_level == 0) { 312 /* we can be momentarily larger in dnode_set_blksz() */ 313 if (db->db_blkid != DB_BONUS_BLKID && dn) { 314 ASSERT3U(db->db.db_size, >=, dn->dn_datablksz); 315 } 316 if (db->db.db_object == DMU_META_DNODE_OBJECT) { 317 dbuf_dirty_record_t *dr = db->db_data_pending; 318 /* 319 * it should only be modified in syncing 320 * context, so make sure we only have 321 * one copy of the data. 322 */ 323 ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf); 324 } 325 } 326 327 /* verify db->db_blkptr */ 328 if (db->db_blkptr) { 329 if (db->db_parent == dn->dn_dbuf) { 330 /* db is pointed to by the dnode */ 331 /* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */ 332 if (db->db.db_object == DMU_META_DNODE_OBJECT) 333 ASSERT(db->db_parent == NULL); 334 else 335 ASSERT(db->db_parent != NULL); 336 ASSERT3P(db->db_blkptr, ==, 337 &dn->dn_phys->dn_blkptr[db->db_blkid]); 338 } else { 339 /* db is pointed to by an indirect block */ 340 int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT; 341 ASSERT3U(db->db_parent->db_level, ==, db->db_level+1); 342 ASSERT3U(db->db_parent->db.db_object, ==, 343 db->db.db_object); 344 /* 345 * dnode_grow_indblksz() can make this fail if we don't 346 * have the struct_rwlock. XXX indblksz no longer 347 * grows. safe to do this now? 348 */ 349 if (RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)) { 350 ASSERT3P(db->db_blkptr, ==, 351 ((blkptr_t *)db->db_parent->db.db_data + 352 db->db_blkid % epb)); 353 } 354 } 355 } 356 if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) && 357 db->db.db_data && db->db_blkid != DB_BONUS_BLKID && 358 db->db_state != DB_FILL && !dn->dn_free_txg) { 359 /* 360 * If the blkptr isn't set but they have nonzero data, 361 * it had better be dirty, otherwise we'll lose that 362 * data when we evict this buffer. 363 */ 364 if (db->db_dirtycnt == 0) { 365 uint64_t *buf = db->db.db_data; 366 int i; 367 368 for (i = 0; i < db->db.db_size >> 3; i++) { 369 ASSERT(buf[i] == 0); 370 } 371 } 372 } 373 } 374 #endif 375 376 static void 377 dbuf_update_data(dmu_buf_impl_t *db) 378 { 379 ASSERT(MUTEX_HELD(&db->db_mtx)); 380 if (db->db_level == 0 && db->db_user_data_ptr_ptr) { 381 ASSERT(!refcount_is_zero(&db->db_holds)); 382 *db->db_user_data_ptr_ptr = db->db.db_data; 383 } 384 } 385 386 static void 387 dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf) 388 { 389 ASSERT(MUTEX_HELD(&db->db_mtx)); 390 ASSERT(db->db_buf == NULL || !arc_has_callback(db->db_buf)); 391 db->db_buf = buf; 392 if (buf != NULL) { 393 ASSERT(buf->b_data != NULL); 394 db->db.db_data = buf->b_data; 395 if (!arc_released(buf)) 396 arc_set_callback(buf, dbuf_do_evict, db); 397 dbuf_update_data(db); 398 } else { 399 dbuf_evict_user(db); 400 db->db.db_data = NULL; 401 db->db_state = DB_UNCACHED; 402 } 403 } 404 405 uint64_t 406 dbuf_whichblock(dnode_t *dn, uint64_t offset) 407 { 408 if (dn->dn_datablkshift) { 409 return (offset >> dn->dn_datablkshift); 410 } else { 411 ASSERT3U(offset, <, dn->dn_datablksz); 412 return (0); 413 } 414 } 415 416 static void 417 dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb) 418 { 419 dmu_buf_impl_t *db = vdb; 420 421 mutex_enter(&db->db_mtx); 422 ASSERT3U(db->db_state, ==, DB_READ); 423 /* 424 * All reads are synchronous, so we must have a hold on the dbuf 425 */ 426 ASSERT(refcount_count(&db->db_holds) > 0); 427 ASSERT(db->db_buf == NULL); 428 ASSERT(db->db.db_data == NULL); 429 if (db->db_level == 0 && db->db_freed_in_flight) { 430 /* we were freed in flight; disregard any error */ 431 arc_release(buf, db); 432 bzero(buf->b_data, db->db.db_size); 433 arc_buf_freeze(buf); 434 db->db_freed_in_flight = FALSE; 435 dbuf_set_data(db, buf); 436 db->db_state = DB_CACHED; 437 } else if (zio == NULL || zio->io_error == 0) { 438 dbuf_set_data(db, buf); 439 db->db_state = DB_CACHED; 440 } else { 441 ASSERT(db->db_blkid != DB_BONUS_BLKID); 442 ASSERT3P(db->db_buf, ==, NULL); 443 VERIFY(arc_buf_remove_ref(buf, db) == 1); 444 db->db_state = DB_UNCACHED; 445 } 446 cv_broadcast(&db->db_changed); 447 mutex_exit(&db->db_mtx); 448 dbuf_rele(db, NULL); 449 } 450 451 static void 452 dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t *flags) 453 { 454 dnode_t *dn = db->db_dnode; 455 zbookmark_t zb; 456 uint32_t aflags = ARC_NOWAIT; 457 arc_buf_t *pbuf; 458 459 ASSERT(!refcount_is_zero(&db->db_holds)); 460 /* We need the struct_rwlock to prevent db_blkptr from changing. */ 461 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 462 ASSERT(MUTEX_HELD(&db->db_mtx)); 463 ASSERT(db->db_state == DB_UNCACHED); 464 ASSERT(db->db_buf == NULL); 465 466 if (db->db_blkid == DB_BONUS_BLKID) { 467 int bonuslen = dn->dn_bonuslen; 468 469 ASSERT3U(bonuslen, <=, db->db.db_size); 470 db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN); 471 arc_space_consume(DN_MAX_BONUSLEN); 472 if (bonuslen < DN_MAX_BONUSLEN) 473 bzero(db->db.db_data, DN_MAX_BONUSLEN); 474 bcopy(DN_BONUS(dn->dn_phys), db->db.db_data, 475 bonuslen); 476 dbuf_update_data(db); 477 db->db_state = DB_CACHED; 478 mutex_exit(&db->db_mtx); 479 return; 480 } 481 482 if (db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr) || 483 (db->db_level == 0 && dnode_block_freed(dn, db->db_blkid))) { 484 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 485 486 dbuf_set_data(db, arc_buf_alloc(dn->dn_objset->os_spa, 487 db->db.db_size, db, type)); 488 bzero(db->db.db_data, db->db.db_size); 489 db->db_state = DB_CACHED; 490 *flags |= DB_RF_CACHED; 491 mutex_exit(&db->db_mtx); 492 return; 493 } 494 495 db->db_state = DB_READ; 496 mutex_exit(&db->db_mtx); 497 498 if (DBUF_IS_L2CACHEABLE(db)) 499 aflags |= ARC_L2CACHE; 500 501 zb.zb_objset = db->db_objset->os_dsl_dataset ? 502 db->db_objset->os_dsl_dataset->ds_object : 0; 503 zb.zb_object = db->db.db_object; 504 zb.zb_level = db->db_level; 505 zb.zb_blkid = db->db_blkid; 506 507 dbuf_add_ref(db, NULL); 508 /* ZIO_FLAG_CANFAIL callers have to check the parent zio's error */ 509 510 if (db->db_parent) 511 pbuf = db->db_parent->db_buf; 512 else 513 pbuf = db->db_objset->os_phys_buf; 514 515 (void) arc_read(zio, dn->dn_objset->os_spa, db->db_blkptr, pbuf, 516 dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ, 517 (*flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED, 518 &aflags, &zb); 519 if (aflags & ARC_CACHED) 520 *flags |= DB_RF_CACHED; 521 } 522 523 int 524 dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags) 525 { 526 int err = 0; 527 int havepzio = (zio != NULL); 528 int prefetch; 529 530 /* 531 * We don't have to hold the mutex to check db_state because it 532 * can't be freed while we have a hold on the buffer. 533 */ 534 ASSERT(!refcount_is_zero(&db->db_holds)); 535 536 if ((flags & DB_RF_HAVESTRUCT) == 0) 537 rw_enter(&db->db_dnode->dn_struct_rwlock, RW_READER); 538 539 prefetch = db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID && 540 (flags & DB_RF_NOPREFETCH) == 0 && db->db_dnode != NULL && 541 DBUF_IS_CACHEABLE(db); 542 543 mutex_enter(&db->db_mtx); 544 if (db->db_state == DB_CACHED) { 545 mutex_exit(&db->db_mtx); 546 if (prefetch) 547 dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset, 548 db->db.db_size, TRUE); 549 if ((flags & DB_RF_HAVESTRUCT) == 0) 550 rw_exit(&db->db_dnode->dn_struct_rwlock); 551 } else if (db->db_state == DB_UNCACHED) { 552 if (zio == NULL) { 553 zio = zio_root(db->db_dnode->dn_objset->os_spa, 554 NULL, NULL, ZIO_FLAG_CANFAIL); 555 } 556 dbuf_read_impl(db, zio, &flags); 557 558 /* dbuf_read_impl has dropped db_mtx for us */ 559 560 if (prefetch) 561 dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset, 562 db->db.db_size, flags & DB_RF_CACHED); 563 564 if ((flags & DB_RF_HAVESTRUCT) == 0) 565 rw_exit(&db->db_dnode->dn_struct_rwlock); 566 567 if (!havepzio) 568 err = zio_wait(zio); 569 } else { 570 mutex_exit(&db->db_mtx); 571 if (prefetch) 572 dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset, 573 db->db.db_size, TRUE); 574 if ((flags & DB_RF_HAVESTRUCT) == 0) 575 rw_exit(&db->db_dnode->dn_struct_rwlock); 576 577 mutex_enter(&db->db_mtx); 578 if ((flags & DB_RF_NEVERWAIT) == 0) { 579 while (db->db_state == DB_READ || 580 db->db_state == DB_FILL) { 581 ASSERT(db->db_state == DB_READ || 582 (flags & DB_RF_HAVESTRUCT) == 0); 583 cv_wait(&db->db_changed, &db->db_mtx); 584 } 585 if (db->db_state == DB_UNCACHED) 586 err = EIO; 587 } 588 mutex_exit(&db->db_mtx); 589 } 590 591 ASSERT(err || havepzio || db->db_state == DB_CACHED); 592 return (err); 593 } 594 595 static void 596 dbuf_noread(dmu_buf_impl_t *db) 597 { 598 ASSERT(!refcount_is_zero(&db->db_holds)); 599 ASSERT(db->db_blkid != DB_BONUS_BLKID); 600 mutex_enter(&db->db_mtx); 601 while (db->db_state == DB_READ || db->db_state == DB_FILL) 602 cv_wait(&db->db_changed, &db->db_mtx); 603 if (db->db_state == DB_UNCACHED) { 604 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 605 606 ASSERT(db->db_buf == NULL); 607 ASSERT(db->db.db_data == NULL); 608 dbuf_set_data(db, arc_buf_alloc(db->db_dnode->dn_objset->os_spa, 609 db->db.db_size, db, type)); 610 db->db_state = DB_FILL; 611 } else { 612 ASSERT3U(db->db_state, ==, DB_CACHED); 613 } 614 mutex_exit(&db->db_mtx); 615 } 616 617 /* 618 * This is our just-in-time copy function. It makes a copy of 619 * buffers, that have been modified in a previous transaction 620 * group, before we modify them in the current active group. 621 * 622 * This function is used in two places: when we are dirtying a 623 * buffer for the first time in a txg, and when we are freeing 624 * a range in a dnode that includes this buffer. 625 * 626 * Note that when we are called from dbuf_free_range() we do 627 * not put a hold on the buffer, we just traverse the active 628 * dbuf list for the dnode. 629 */ 630 static void 631 dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg) 632 { 633 dbuf_dirty_record_t *dr = db->db_last_dirty; 634 635 ASSERT(MUTEX_HELD(&db->db_mtx)); 636 ASSERT(db->db.db_data != NULL); 637 ASSERT(db->db_level == 0); 638 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT); 639 640 if (dr == NULL || 641 (dr->dt.dl.dr_data != 642 ((db->db_blkid == DB_BONUS_BLKID) ? db->db.db_data : db->db_buf))) 643 return; 644 645 /* 646 * If the last dirty record for this dbuf has not yet synced 647 * and its referencing the dbuf data, either: 648 * reset the reference to point to a new copy, 649 * or (if there a no active holders) 650 * just null out the current db_data pointer. 651 */ 652 ASSERT(dr->dr_txg >= txg - 2); 653 if (db->db_blkid == DB_BONUS_BLKID) { 654 /* Note that the data bufs here are zio_bufs */ 655 dr->dt.dl.dr_data = zio_buf_alloc(DN_MAX_BONUSLEN); 656 arc_space_consume(DN_MAX_BONUSLEN); 657 bcopy(db->db.db_data, dr->dt.dl.dr_data, DN_MAX_BONUSLEN); 658 } else if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 659 int size = db->db.db_size; 660 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 661 dr->dt.dl.dr_data = arc_buf_alloc( 662 db->db_dnode->dn_objset->os_spa, size, db, type); 663 bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size); 664 } else { 665 dbuf_set_data(db, NULL); 666 } 667 } 668 669 void 670 dbuf_unoverride(dbuf_dirty_record_t *dr) 671 { 672 dmu_buf_impl_t *db = dr->dr_dbuf; 673 uint64_t txg = dr->dr_txg; 674 675 ASSERT(MUTEX_HELD(&db->db_mtx)); 676 ASSERT(dr->dt.dl.dr_override_state != DR_IN_DMU_SYNC); 677 ASSERT(db->db_level == 0); 678 679 if (db->db_blkid == DB_BONUS_BLKID || 680 dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN) 681 return; 682 683 /* free this block */ 684 if (!BP_IS_HOLE(&dr->dt.dl.dr_overridden_by)) { 685 /* XXX can get silent EIO here */ 686 (void) dsl_free(NULL, 687 spa_get_dsl(db->db_dnode->dn_objset->os_spa), 688 txg, &dr->dt.dl.dr_overridden_by, NULL, NULL, ARC_WAIT); 689 } 690 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 691 /* 692 * Release the already-written buffer, so we leave it in 693 * a consistent dirty state. Note that all callers are 694 * modifying the buffer, so they will immediately do 695 * another (redundant) arc_release(). Therefore, leave 696 * the buf thawed to save the effort of freezing & 697 * immediately re-thawing it. 698 */ 699 arc_release(dr->dt.dl.dr_data, db); 700 } 701 702 /* 703 * Evict (if its unreferenced) or clear (if its referenced) any level-0 704 * data blocks in the free range, so that any future readers will find 705 * empty blocks. Also, if we happen accross any level-1 dbufs in the 706 * range that have not already been marked dirty, mark them dirty so 707 * they stay in memory. 708 */ 709 void 710 dbuf_free_range(dnode_t *dn, uint64_t start, uint64_t end, dmu_tx_t *tx) 711 { 712 dmu_buf_impl_t *db, *db_next; 713 uint64_t txg = tx->tx_txg; 714 int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 715 uint64_t first_l1 = start >> epbs; 716 uint64_t last_l1 = end >> epbs; 717 718 if (end > dn->dn_maxblkid) { 719 end = dn->dn_maxblkid; 720 last_l1 = end >> epbs; 721 } 722 dprintf_dnode(dn, "start=%llu end=%llu\n", start, end); 723 mutex_enter(&dn->dn_dbufs_mtx); 724 for (db = list_head(&dn->dn_dbufs); db; db = db_next) { 725 db_next = list_next(&dn->dn_dbufs, db); 726 ASSERT(db->db_blkid != DB_BONUS_BLKID); 727 728 if (db->db_level == 1 && 729 db->db_blkid >= first_l1 && db->db_blkid <= last_l1) { 730 mutex_enter(&db->db_mtx); 731 if (db->db_last_dirty && 732 db->db_last_dirty->dr_txg < txg) { 733 dbuf_add_ref(db, FTAG); 734 mutex_exit(&db->db_mtx); 735 dbuf_will_dirty(db, tx); 736 dbuf_rele(db, FTAG); 737 } else { 738 mutex_exit(&db->db_mtx); 739 } 740 } 741 742 if (db->db_level != 0) 743 continue; 744 dprintf_dbuf(db, "found buf %s\n", ""); 745 if (db->db_blkid < start || db->db_blkid > end) 746 continue; 747 748 /* found a level 0 buffer in the range */ 749 if (dbuf_undirty(db, tx)) 750 continue; 751 752 mutex_enter(&db->db_mtx); 753 if (db->db_state == DB_UNCACHED || 754 db->db_state == DB_EVICTING) { 755 ASSERT(db->db.db_data == NULL); 756 mutex_exit(&db->db_mtx); 757 continue; 758 } 759 if (db->db_state == DB_READ || db->db_state == DB_FILL) { 760 /* will be handled in dbuf_read_done or dbuf_rele */ 761 db->db_freed_in_flight = TRUE; 762 mutex_exit(&db->db_mtx); 763 continue; 764 } 765 if (refcount_count(&db->db_holds) == 0) { 766 ASSERT(db->db_buf); 767 dbuf_clear(db); 768 continue; 769 } 770 /* The dbuf is referenced */ 771 772 if (db->db_last_dirty != NULL) { 773 dbuf_dirty_record_t *dr = db->db_last_dirty; 774 775 if (dr->dr_txg == txg) { 776 /* 777 * This buffer is "in-use", re-adjust the file 778 * size to reflect that this buffer may 779 * contain new data when we sync. 780 */ 781 if (db->db_blkid > dn->dn_maxblkid) 782 dn->dn_maxblkid = db->db_blkid; 783 dbuf_unoverride(dr); 784 } else { 785 /* 786 * This dbuf is not dirty in the open context. 787 * Either uncache it (if its not referenced in 788 * the open context) or reset its contents to 789 * empty. 790 */ 791 dbuf_fix_old_data(db, txg); 792 } 793 } 794 /* clear the contents if its cached */ 795 if (db->db_state == DB_CACHED) { 796 ASSERT(db->db.db_data != NULL); 797 arc_release(db->db_buf, db); 798 bzero(db->db.db_data, db->db.db_size); 799 arc_buf_freeze(db->db_buf); 800 } 801 802 mutex_exit(&db->db_mtx); 803 } 804 mutex_exit(&dn->dn_dbufs_mtx); 805 } 806 807 static int 808 dbuf_block_freeable(dmu_buf_impl_t *db) 809 { 810 dsl_dataset_t *ds = db->db_objset->os_dsl_dataset; 811 uint64_t birth_txg = 0; 812 813 /* 814 * We don't need any locking to protect db_blkptr: 815 * If it's syncing, then db_last_dirty will be set 816 * so we'll ignore db_blkptr. 817 */ 818 ASSERT(MUTEX_HELD(&db->db_mtx)); 819 if (db->db_last_dirty) 820 birth_txg = db->db_last_dirty->dr_txg; 821 else if (db->db_blkptr) 822 birth_txg = db->db_blkptr->blk_birth; 823 824 /* If we don't exist or are in a snapshot, we can't be freed */ 825 if (birth_txg) 826 return (ds == NULL || 827 dsl_dataset_block_freeable(ds, birth_txg)); 828 else 829 return (FALSE); 830 } 831 832 void 833 dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx) 834 { 835 arc_buf_t *buf, *obuf; 836 int osize = db->db.db_size; 837 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 838 839 ASSERT(db->db_blkid != DB_BONUS_BLKID); 840 841 /* XXX does *this* func really need the lock? */ 842 ASSERT(RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)); 843 844 /* 845 * This call to dbuf_will_dirty() with the dn_struct_rwlock held 846 * is OK, because there can be no other references to the db 847 * when we are changing its size, so no concurrent DB_FILL can 848 * be happening. 849 */ 850 /* 851 * XXX we should be doing a dbuf_read, checking the return 852 * value and returning that up to our callers 853 */ 854 dbuf_will_dirty(db, tx); 855 856 /* create the data buffer for the new block */ 857 buf = arc_buf_alloc(db->db_dnode->dn_objset->os_spa, size, db, type); 858 859 /* copy old block data to the new block */ 860 obuf = db->db_buf; 861 bcopy(obuf->b_data, buf->b_data, MIN(osize, size)); 862 /* zero the remainder */ 863 if (size > osize) 864 bzero((uint8_t *)buf->b_data + osize, size - osize); 865 866 mutex_enter(&db->db_mtx); 867 dbuf_set_data(db, buf); 868 VERIFY(arc_buf_remove_ref(obuf, db) == 1); 869 db->db.db_size = size; 870 871 if (db->db_level == 0) { 872 ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg); 873 db->db_last_dirty->dt.dl.dr_data = buf; 874 } 875 mutex_exit(&db->db_mtx); 876 877 dnode_willuse_space(db->db_dnode, size-osize, tx); 878 } 879 880 dbuf_dirty_record_t * 881 dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 882 { 883 dnode_t *dn = db->db_dnode; 884 objset_impl_t *os = dn->dn_objset; 885 dbuf_dirty_record_t **drp, *dr; 886 int drop_struct_lock = FALSE; 887 int txgoff = tx->tx_txg & TXG_MASK; 888 889 ASSERT(tx->tx_txg != 0); 890 ASSERT(!refcount_is_zero(&db->db_holds)); 891 DMU_TX_DIRTY_BUF(tx, db); 892 893 /* 894 * Shouldn't dirty a regular buffer in syncing context. Private 895 * objects may be dirtied in syncing context, but only if they 896 * were already pre-dirtied in open context. 897 * XXX We may want to prohibit dirtying in syncing context even 898 * if they did pre-dirty. 899 */ 900 ASSERT(!dmu_tx_is_syncing(tx) || 901 BP_IS_HOLE(dn->dn_objset->os_rootbp) || 902 dn->dn_object == DMU_META_DNODE_OBJECT || 903 dn->dn_objset->os_dsl_dataset == NULL || 904 dsl_dir_is_private(dn->dn_objset->os_dsl_dataset->ds_dir)); 905 906 /* 907 * We make this assert for private objects as well, but after we 908 * check if we're already dirty. They are allowed to re-dirty 909 * in syncing context. 910 */ 911 ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 912 dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 913 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 914 915 mutex_enter(&db->db_mtx); 916 /* 917 * XXX make this true for indirects too? The problem is that 918 * transactions created with dmu_tx_create_assigned() from 919 * syncing context don't bother holding ahead. 920 */ 921 ASSERT(db->db_level != 0 || 922 db->db_state == DB_CACHED || db->db_state == DB_FILL); 923 924 mutex_enter(&dn->dn_mtx); 925 /* 926 * Don't set dirtyctx to SYNC if we're just modifying this as we 927 * initialize the objset. 928 */ 929 if (dn->dn_dirtyctx == DN_UNDIRTIED && 930 !BP_IS_HOLE(dn->dn_objset->os_rootbp)) { 931 dn->dn_dirtyctx = 932 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN); 933 ASSERT(dn->dn_dirtyctx_firstset == NULL); 934 dn->dn_dirtyctx_firstset = kmem_alloc(1, KM_SLEEP); 935 } 936 mutex_exit(&dn->dn_mtx); 937 938 /* 939 * If this buffer is already dirty, we're done. 940 */ 941 drp = &db->db_last_dirty; 942 ASSERT(*drp == NULL || (*drp)->dr_txg <= tx->tx_txg || 943 db->db.db_object == DMU_META_DNODE_OBJECT); 944 while ((dr = *drp) != NULL && dr->dr_txg > tx->tx_txg) 945 drp = &dr->dr_next; 946 if (dr && dr->dr_txg == tx->tx_txg) { 947 if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID) { 948 /* 949 * If this buffer has already been written out, 950 * we now need to reset its state. 951 */ 952 dbuf_unoverride(dr); 953 if (db->db.db_object != DMU_META_DNODE_OBJECT) 954 arc_buf_thaw(db->db_buf); 955 } 956 mutex_exit(&db->db_mtx); 957 return (dr); 958 } 959 960 /* 961 * Only valid if not already dirty. 962 */ 963 ASSERT(dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 964 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 965 966 ASSERT3U(dn->dn_nlevels, >, db->db_level); 967 ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) || 968 dn->dn_phys->dn_nlevels > db->db_level || 969 dn->dn_next_nlevels[txgoff] > db->db_level || 970 dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level || 971 dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level); 972 973 /* 974 * We should only be dirtying in syncing context if it's the 975 * mos, a spa os, or we're initializing the os. However, we are 976 * allowed to dirty in syncing context provided we already 977 * dirtied it in open context. Hence we must make this 978 * assertion only if we're not already dirty. 979 */ 980 ASSERT(!dmu_tx_is_syncing(tx) || 981 os->os_dsl_dataset == NULL || 982 !dsl_dir_is_private(os->os_dsl_dataset->ds_dir) || 983 !BP_IS_HOLE(os->os_rootbp)); 984 ASSERT(db->db.db_size != 0); 985 986 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 987 988 if (db->db_blkid != DB_BONUS_BLKID) { 989 /* 990 * Update the accounting. 991 */ 992 if (dbuf_block_freeable(db)) { 993 blkptr_t *bp = db->db_blkptr; 994 int64_t willfree = (bp && !BP_IS_HOLE(bp)) ? 995 bp_get_dasize(os->os_spa, bp) : db->db.db_size; 996 /* 997 * This is only a guess -- if the dbuf is dirty 998 * in a previous txg, we don't know how much 999 * space it will use on disk yet. We should 1000 * really have the struct_rwlock to access 1001 * db_blkptr, but since this is just a guess, 1002 * it's OK if we get an odd answer. 1003 */ 1004 dnode_willuse_space(dn, -willfree, tx); 1005 } 1006 dnode_willuse_space(dn, db->db.db_size, tx); 1007 } 1008 1009 /* 1010 * If this buffer is dirty in an old transaction group we need 1011 * to make a copy of it so that the changes we make in this 1012 * transaction group won't leak out when we sync the older txg. 1013 */ 1014 dr = kmem_zalloc(sizeof (dbuf_dirty_record_t), KM_SLEEP); 1015 if (db->db_level == 0) { 1016 void *data_old = db->db_buf; 1017 1018 if (db->db_blkid == DB_BONUS_BLKID) { 1019 dbuf_fix_old_data(db, tx->tx_txg); 1020 data_old = db->db.db_data; 1021 } else if (db->db.db_object != DMU_META_DNODE_OBJECT) { 1022 /* 1023 * Release the data buffer from the cache so that we 1024 * can modify it without impacting possible other users 1025 * of this cached data block. Note that indirect 1026 * blocks and private objects are not released until the 1027 * syncing state (since they are only modified then). 1028 */ 1029 arc_release(db->db_buf, db); 1030 dbuf_fix_old_data(db, tx->tx_txg); 1031 data_old = db->db_buf; 1032 } 1033 ASSERT(data_old != NULL); 1034 dr->dt.dl.dr_data = data_old; 1035 } else { 1036 mutex_init(&dr->dt.di.dr_mtx, NULL, MUTEX_DEFAULT, NULL); 1037 list_create(&dr->dt.di.dr_children, 1038 sizeof (dbuf_dirty_record_t), 1039 offsetof(dbuf_dirty_record_t, dr_dirty_node)); 1040 } 1041 dr->dr_dbuf = db; 1042 dr->dr_txg = tx->tx_txg; 1043 dr->dr_next = *drp; 1044 *drp = dr; 1045 1046 /* 1047 * We could have been freed_in_flight between the dbuf_noread 1048 * and dbuf_dirty. We win, as though the dbuf_noread() had 1049 * happened after the free. 1050 */ 1051 if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID) { 1052 mutex_enter(&dn->dn_mtx); 1053 dnode_clear_range(dn, db->db_blkid, 1, tx); 1054 mutex_exit(&dn->dn_mtx); 1055 db->db_freed_in_flight = FALSE; 1056 } 1057 1058 /* 1059 * This buffer is now part of this txg 1060 */ 1061 dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg); 1062 db->db_dirtycnt += 1; 1063 ASSERT3U(db->db_dirtycnt, <=, 3); 1064 1065 mutex_exit(&db->db_mtx); 1066 1067 if (db->db_blkid == DB_BONUS_BLKID) { 1068 mutex_enter(&dn->dn_mtx); 1069 ASSERT(!list_link_active(&dr->dr_dirty_node)); 1070 list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1071 mutex_exit(&dn->dn_mtx); 1072 dnode_setdirty(dn, tx); 1073 return (dr); 1074 } 1075 1076 if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 1077 rw_enter(&dn->dn_struct_rwlock, RW_READER); 1078 drop_struct_lock = TRUE; 1079 } 1080 1081 if (db->db_level == 0) { 1082 dnode_new_blkid(dn, db->db_blkid, tx, drop_struct_lock); 1083 ASSERT(dn->dn_maxblkid >= db->db_blkid); 1084 } 1085 1086 if (db->db_level+1 < dn->dn_nlevels) { 1087 dmu_buf_impl_t *parent = db->db_parent; 1088 dbuf_dirty_record_t *di; 1089 int parent_held = FALSE; 1090 1091 if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) { 1092 int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1093 1094 parent = dbuf_hold_level(dn, db->db_level+1, 1095 db->db_blkid >> epbs, FTAG); 1096 parent_held = TRUE; 1097 } 1098 if (drop_struct_lock) 1099 rw_exit(&dn->dn_struct_rwlock); 1100 ASSERT3U(db->db_level+1, ==, parent->db_level); 1101 di = dbuf_dirty(parent, tx); 1102 if (parent_held) 1103 dbuf_rele(parent, FTAG); 1104 1105 mutex_enter(&db->db_mtx); 1106 /* possible race with dbuf_undirty() */ 1107 if (db->db_last_dirty == dr || 1108 dn->dn_object == DMU_META_DNODE_OBJECT) { 1109 mutex_enter(&di->dt.di.dr_mtx); 1110 ASSERT3U(di->dr_txg, ==, tx->tx_txg); 1111 ASSERT(!list_link_active(&dr->dr_dirty_node)); 1112 list_insert_tail(&di->dt.di.dr_children, dr); 1113 mutex_exit(&di->dt.di.dr_mtx); 1114 dr->dr_parent = di; 1115 } 1116 mutex_exit(&db->db_mtx); 1117 } else { 1118 ASSERT(db->db_level+1 == dn->dn_nlevels); 1119 ASSERT(db->db_blkid < dn->dn_nblkptr); 1120 ASSERT(db->db_parent == NULL || 1121 db->db_parent == db->db_dnode->dn_dbuf); 1122 mutex_enter(&dn->dn_mtx); 1123 ASSERT(!list_link_active(&dr->dr_dirty_node)); 1124 list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1125 mutex_exit(&dn->dn_mtx); 1126 if (drop_struct_lock) 1127 rw_exit(&dn->dn_struct_rwlock); 1128 } 1129 1130 dnode_setdirty(dn, tx); 1131 return (dr); 1132 } 1133 1134 static int 1135 dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1136 { 1137 dnode_t *dn = db->db_dnode; 1138 uint64_t txg = tx->tx_txg; 1139 dbuf_dirty_record_t *dr, **drp; 1140 1141 ASSERT(txg != 0); 1142 ASSERT(db->db_blkid != DB_BONUS_BLKID); 1143 1144 mutex_enter(&db->db_mtx); 1145 1146 /* 1147 * If this buffer is not dirty, we're done. 1148 */ 1149 for (drp = &db->db_last_dirty; (dr = *drp) != NULL; drp = &dr->dr_next) 1150 if (dr->dr_txg <= txg) 1151 break; 1152 if (dr == NULL || dr->dr_txg < txg) { 1153 mutex_exit(&db->db_mtx); 1154 return (0); 1155 } 1156 ASSERT(dr->dr_txg == txg); 1157 1158 /* 1159 * If this buffer is currently held, we cannot undirty 1160 * it, since one of the current holders may be in the 1161 * middle of an update. Note that users of dbuf_undirty() 1162 * should not place a hold on the dbuf before the call. 1163 */ 1164 if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 1165 mutex_exit(&db->db_mtx); 1166 /* Make sure we don't toss this buffer at sync phase */ 1167 mutex_enter(&dn->dn_mtx); 1168 dnode_clear_range(dn, db->db_blkid, 1, tx); 1169 mutex_exit(&dn->dn_mtx); 1170 return (0); 1171 } 1172 1173 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1174 1175 ASSERT(db->db.db_size != 0); 1176 1177 /* XXX would be nice to fix up dn_towrite_space[] */ 1178 1179 *drp = dr->dr_next; 1180 1181 if (dr->dr_parent) { 1182 mutex_enter(&dr->dr_parent->dt.di.dr_mtx); 1183 list_remove(&dr->dr_parent->dt.di.dr_children, dr); 1184 mutex_exit(&dr->dr_parent->dt.di.dr_mtx); 1185 } else if (db->db_level+1 == dn->dn_nlevels) { 1186 ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf); 1187 mutex_enter(&dn->dn_mtx); 1188 list_remove(&dn->dn_dirty_records[txg & TXG_MASK], dr); 1189 mutex_exit(&dn->dn_mtx); 1190 } 1191 1192 if (db->db_level == 0) { 1193 dbuf_unoverride(dr); 1194 1195 ASSERT(db->db_buf != NULL); 1196 ASSERT(dr->dt.dl.dr_data != NULL); 1197 if (dr->dt.dl.dr_data != db->db_buf) 1198 VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, db) == 1); 1199 } else { 1200 ASSERT(db->db_buf != NULL); 1201 ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 1202 mutex_destroy(&dr->dt.di.dr_mtx); 1203 list_destroy(&dr->dt.di.dr_children); 1204 } 1205 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 1206 1207 ASSERT(db->db_dirtycnt > 0); 1208 db->db_dirtycnt -= 1; 1209 1210 if (refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) { 1211 arc_buf_t *buf = db->db_buf; 1212 1213 ASSERT(arc_released(buf)); 1214 dbuf_set_data(db, NULL); 1215 VERIFY(arc_buf_remove_ref(buf, db) == 1); 1216 dbuf_evict(db); 1217 return (1); 1218 } 1219 1220 mutex_exit(&db->db_mtx); 1221 return (0); 1222 } 1223 1224 #pragma weak dmu_buf_will_dirty = dbuf_will_dirty 1225 void 1226 dbuf_will_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1227 { 1228 int rf = DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH; 1229 1230 ASSERT(tx->tx_txg != 0); 1231 ASSERT(!refcount_is_zero(&db->db_holds)); 1232 1233 if (RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)) 1234 rf |= DB_RF_HAVESTRUCT; 1235 (void) dbuf_read(db, NULL, rf); 1236 (void) dbuf_dirty(db, tx); 1237 } 1238 1239 void 1240 dmu_buf_will_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 1241 { 1242 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1243 1244 ASSERT(db->db_blkid != DB_BONUS_BLKID); 1245 ASSERT(tx->tx_txg != 0); 1246 ASSERT(db->db_level == 0); 1247 ASSERT(!refcount_is_zero(&db->db_holds)); 1248 1249 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT || 1250 dmu_tx_private_ok(tx)); 1251 1252 dbuf_noread(db); 1253 (void) dbuf_dirty(db, tx); 1254 } 1255 1256 #pragma weak dmu_buf_fill_done = dbuf_fill_done 1257 /* ARGSUSED */ 1258 void 1259 dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx) 1260 { 1261 mutex_enter(&db->db_mtx); 1262 DBUF_VERIFY(db); 1263 1264 if (db->db_state == DB_FILL) { 1265 if (db->db_level == 0 && db->db_freed_in_flight) { 1266 ASSERT(db->db_blkid != DB_BONUS_BLKID); 1267 /* we were freed while filling */ 1268 /* XXX dbuf_undirty? */ 1269 bzero(db->db.db_data, db->db.db_size); 1270 db->db_freed_in_flight = FALSE; 1271 } 1272 db->db_state = DB_CACHED; 1273 cv_broadcast(&db->db_changed); 1274 } 1275 mutex_exit(&db->db_mtx); 1276 } 1277 1278 /* 1279 * "Clear" the contents of this dbuf. This will mark the dbuf 1280 * EVICTING and clear *most* of its references. Unfortunetely, 1281 * when we are not holding the dn_dbufs_mtx, we can't clear the 1282 * entry in the dn_dbufs list. We have to wait until dbuf_destroy() 1283 * in this case. For callers from the DMU we will usually see: 1284 * dbuf_clear()->arc_buf_evict()->dbuf_do_evict()->dbuf_destroy() 1285 * For the arc callback, we will usually see: 1286 * dbuf_do_evict()->dbuf_clear();dbuf_destroy() 1287 * Sometimes, though, we will get a mix of these two: 1288 * DMU: dbuf_clear()->arc_buf_evict() 1289 * ARC: dbuf_do_evict()->dbuf_destroy() 1290 */ 1291 void 1292 dbuf_clear(dmu_buf_impl_t *db) 1293 { 1294 dnode_t *dn = db->db_dnode; 1295 dmu_buf_impl_t *parent = db->db_parent; 1296 dmu_buf_impl_t *dndb = dn->dn_dbuf; 1297 int dbuf_gone = FALSE; 1298 1299 ASSERT(MUTEX_HELD(&db->db_mtx)); 1300 ASSERT(refcount_is_zero(&db->db_holds)); 1301 1302 dbuf_evict_user(db); 1303 1304 if (db->db_state == DB_CACHED) { 1305 ASSERT(db->db.db_data != NULL); 1306 if (db->db_blkid == DB_BONUS_BLKID) { 1307 zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN); 1308 arc_space_return(DN_MAX_BONUSLEN); 1309 } 1310 db->db.db_data = NULL; 1311 db->db_state = DB_UNCACHED; 1312 } 1313 1314 ASSERT3U(db->db_state, ==, DB_UNCACHED); 1315 ASSERT(db->db_data_pending == NULL); 1316 1317 db->db_state = DB_EVICTING; 1318 db->db_blkptr = NULL; 1319 1320 if (db->db_blkid != DB_BONUS_BLKID && MUTEX_HELD(&dn->dn_dbufs_mtx)) { 1321 list_remove(&dn->dn_dbufs, db); 1322 dnode_rele(dn, db); 1323 db->db_dnode = NULL; 1324 } 1325 1326 if (db->db_buf) 1327 dbuf_gone = arc_buf_evict(db->db_buf); 1328 1329 if (!dbuf_gone) 1330 mutex_exit(&db->db_mtx); 1331 1332 /* 1333 * If this dbuf is referened from an indirect dbuf, 1334 * decrement the ref count on the indirect dbuf. 1335 */ 1336 if (parent && parent != dndb) 1337 dbuf_rele(parent, db); 1338 } 1339 1340 static int 1341 dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse, 1342 dmu_buf_impl_t **parentp, blkptr_t **bpp) 1343 { 1344 int nlevels, epbs; 1345 1346 *parentp = NULL; 1347 *bpp = NULL; 1348 1349 ASSERT(blkid != DB_BONUS_BLKID); 1350 1351 if (dn->dn_phys->dn_nlevels == 0) 1352 nlevels = 1; 1353 else 1354 nlevels = dn->dn_phys->dn_nlevels; 1355 1356 epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1357 1358 ASSERT3U(level * epbs, <, 64); 1359 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1360 if (level >= nlevels || 1361 (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) { 1362 /* the buffer has no parent yet */ 1363 return (ENOENT); 1364 } else if (level < nlevels-1) { 1365 /* this block is referenced from an indirect block */ 1366 int err = dbuf_hold_impl(dn, level+1, 1367 blkid >> epbs, fail_sparse, NULL, parentp); 1368 if (err) 1369 return (err); 1370 err = dbuf_read(*parentp, NULL, 1371 (DB_RF_HAVESTRUCT | DB_RF_NOPREFETCH | DB_RF_CANFAIL)); 1372 if (err) { 1373 dbuf_rele(*parentp, NULL); 1374 *parentp = NULL; 1375 return (err); 1376 } 1377 *bpp = ((blkptr_t *)(*parentp)->db.db_data) + 1378 (blkid & ((1ULL << epbs) - 1)); 1379 return (0); 1380 } else { 1381 /* the block is referenced from the dnode */ 1382 ASSERT3U(level, ==, nlevels-1); 1383 ASSERT(dn->dn_phys->dn_nblkptr == 0 || 1384 blkid < dn->dn_phys->dn_nblkptr); 1385 if (dn->dn_dbuf) { 1386 dbuf_add_ref(dn->dn_dbuf, NULL); 1387 *parentp = dn->dn_dbuf; 1388 } 1389 *bpp = &dn->dn_phys->dn_blkptr[blkid]; 1390 return (0); 1391 } 1392 } 1393 1394 static dmu_buf_impl_t * 1395 dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid, 1396 dmu_buf_impl_t *parent, blkptr_t *blkptr) 1397 { 1398 objset_impl_t *os = dn->dn_objset; 1399 dmu_buf_impl_t *db, *odb; 1400 1401 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1402 ASSERT(dn->dn_type != DMU_OT_NONE); 1403 1404 db = kmem_cache_alloc(dbuf_cache, KM_SLEEP); 1405 1406 db->db_objset = os; 1407 db->db.db_object = dn->dn_object; 1408 db->db_level = level; 1409 db->db_blkid = blkid; 1410 db->db_last_dirty = NULL; 1411 db->db_dirtycnt = 0; 1412 db->db_dnode = dn; 1413 db->db_parent = parent; 1414 db->db_blkptr = blkptr; 1415 1416 db->db_user_ptr = NULL; 1417 db->db_user_data_ptr_ptr = NULL; 1418 db->db_evict_func = NULL; 1419 db->db_immediate_evict = 0; 1420 db->db_freed_in_flight = 0; 1421 1422 if (blkid == DB_BONUS_BLKID) { 1423 ASSERT3P(parent, ==, dn->dn_dbuf); 1424 db->db.db_size = DN_MAX_BONUSLEN - 1425 (dn->dn_nblkptr-1) * sizeof (blkptr_t); 1426 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 1427 db->db.db_offset = DB_BONUS_BLKID; 1428 db->db_state = DB_UNCACHED; 1429 /* the bonus dbuf is not placed in the hash table */ 1430 arc_space_consume(sizeof (dmu_buf_impl_t)); 1431 return (db); 1432 } else { 1433 int blocksize = 1434 db->db_level ? 1<<dn->dn_indblkshift : dn->dn_datablksz; 1435 db->db.db_size = blocksize; 1436 db->db.db_offset = db->db_blkid * blocksize; 1437 } 1438 1439 /* 1440 * Hold the dn_dbufs_mtx while we get the new dbuf 1441 * in the hash table *and* added to the dbufs list. 1442 * This prevents a possible deadlock with someone 1443 * trying to look up this dbuf before its added to the 1444 * dn_dbufs list. 1445 */ 1446 mutex_enter(&dn->dn_dbufs_mtx); 1447 db->db_state = DB_EVICTING; 1448 if ((odb = dbuf_hash_insert(db)) != NULL) { 1449 /* someone else inserted it first */ 1450 kmem_cache_free(dbuf_cache, db); 1451 mutex_exit(&dn->dn_dbufs_mtx); 1452 return (odb); 1453 } 1454 list_insert_head(&dn->dn_dbufs, db); 1455 db->db_state = DB_UNCACHED; 1456 mutex_exit(&dn->dn_dbufs_mtx); 1457 arc_space_consume(sizeof (dmu_buf_impl_t)); 1458 1459 if (parent && parent != dn->dn_dbuf) 1460 dbuf_add_ref(parent, db); 1461 1462 ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 1463 refcount_count(&dn->dn_holds) > 0); 1464 (void) refcount_add(&dn->dn_holds, db); 1465 1466 dprintf_dbuf(db, "db=%p\n", db); 1467 1468 return (db); 1469 } 1470 1471 static int 1472 dbuf_do_evict(void *private) 1473 { 1474 arc_buf_t *buf = private; 1475 dmu_buf_impl_t *db = buf->b_private; 1476 1477 if (!MUTEX_HELD(&db->db_mtx)) 1478 mutex_enter(&db->db_mtx); 1479 1480 ASSERT(refcount_is_zero(&db->db_holds)); 1481 1482 if (db->db_state != DB_EVICTING) { 1483 ASSERT(db->db_state == DB_CACHED); 1484 DBUF_VERIFY(db); 1485 db->db_buf = NULL; 1486 dbuf_evict(db); 1487 } else { 1488 mutex_exit(&db->db_mtx); 1489 dbuf_destroy(db); 1490 } 1491 return (0); 1492 } 1493 1494 static void 1495 dbuf_destroy(dmu_buf_impl_t *db) 1496 { 1497 ASSERT(refcount_is_zero(&db->db_holds)); 1498 1499 if (db->db_blkid != DB_BONUS_BLKID) { 1500 /* 1501 * If this dbuf is still on the dn_dbufs list, 1502 * remove it from that list. 1503 */ 1504 if (db->db_dnode) { 1505 dnode_t *dn = db->db_dnode; 1506 1507 mutex_enter(&dn->dn_dbufs_mtx); 1508 list_remove(&dn->dn_dbufs, db); 1509 mutex_exit(&dn->dn_dbufs_mtx); 1510 1511 dnode_rele(dn, db); 1512 db->db_dnode = NULL; 1513 } 1514 dbuf_hash_remove(db); 1515 } 1516 db->db_parent = NULL; 1517 db->db_buf = NULL; 1518 1519 ASSERT(!list_link_active(&db->db_link)); 1520 ASSERT(db->db.db_data == NULL); 1521 ASSERT(db->db_hash_next == NULL); 1522 ASSERT(db->db_blkptr == NULL); 1523 ASSERT(db->db_data_pending == NULL); 1524 1525 kmem_cache_free(dbuf_cache, db); 1526 arc_space_return(sizeof (dmu_buf_impl_t)); 1527 } 1528 1529 void 1530 dbuf_prefetch(dnode_t *dn, uint64_t blkid) 1531 { 1532 dmu_buf_impl_t *db = NULL; 1533 blkptr_t *bp = NULL; 1534 1535 ASSERT(blkid != DB_BONUS_BLKID); 1536 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1537 1538 if (dnode_block_freed(dn, blkid)) 1539 return; 1540 1541 /* dbuf_find() returns with db_mtx held */ 1542 if (db = dbuf_find(dn, 0, blkid)) { 1543 if (refcount_count(&db->db_holds) > 0) { 1544 /* 1545 * This dbuf is active. We assume that it is 1546 * already CACHED, or else about to be either 1547 * read or filled. 1548 */ 1549 mutex_exit(&db->db_mtx); 1550 return; 1551 } 1552 mutex_exit(&db->db_mtx); 1553 db = NULL; 1554 } 1555 1556 if (dbuf_findbp(dn, 0, blkid, TRUE, &db, &bp) == 0) { 1557 if (bp && !BP_IS_HOLE(bp)) { 1558 arc_buf_t *pbuf; 1559 uint32_t aflags = ARC_NOWAIT | ARC_PREFETCH; 1560 zbookmark_t zb; 1561 zb.zb_objset = dn->dn_objset->os_dsl_dataset ? 1562 dn->dn_objset->os_dsl_dataset->ds_object : 0; 1563 zb.zb_object = dn->dn_object; 1564 zb.zb_level = 0; 1565 zb.zb_blkid = blkid; 1566 1567 if (db) 1568 pbuf = db->db_buf; 1569 else 1570 pbuf = dn->dn_objset->os_phys_buf; 1571 1572 (void) arc_read(NULL, dn->dn_objset->os_spa, 1573 bp, pbuf, NULL, NULL, ZIO_PRIORITY_ASYNC_READ, 1574 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 1575 &aflags, &zb); 1576 } 1577 if (db) 1578 dbuf_rele(db, NULL); 1579 } 1580 } 1581 1582 /* 1583 * Returns with db_holds incremented, and db_mtx not held. 1584 * Note: dn_struct_rwlock must be held. 1585 */ 1586 int 1587 dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, int fail_sparse, 1588 void *tag, dmu_buf_impl_t **dbp) 1589 { 1590 dmu_buf_impl_t *db, *parent = NULL; 1591 1592 ASSERT(blkid != DB_BONUS_BLKID); 1593 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1594 ASSERT3U(dn->dn_nlevels, >, level); 1595 1596 *dbp = NULL; 1597 top: 1598 /* dbuf_find() returns with db_mtx held */ 1599 db = dbuf_find(dn, level, blkid); 1600 1601 if (db == NULL) { 1602 blkptr_t *bp = NULL; 1603 int err; 1604 1605 ASSERT3P(parent, ==, NULL); 1606 err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp); 1607 if (fail_sparse) { 1608 if (err == 0 && bp && BP_IS_HOLE(bp)) 1609 err = ENOENT; 1610 if (err) { 1611 if (parent) 1612 dbuf_rele(parent, NULL); 1613 return (err); 1614 } 1615 } 1616 if (err && err != ENOENT) 1617 return (err); 1618 db = dbuf_create(dn, level, blkid, parent, bp); 1619 } 1620 1621 if (db->db_buf && refcount_is_zero(&db->db_holds)) { 1622 arc_buf_add_ref(db->db_buf, db); 1623 if (db->db_buf->b_data == NULL) { 1624 dbuf_clear(db); 1625 if (parent) { 1626 dbuf_rele(parent, NULL); 1627 parent = NULL; 1628 } 1629 goto top; 1630 } 1631 ASSERT3P(db->db.db_data, ==, db->db_buf->b_data); 1632 } 1633 1634 ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf)); 1635 1636 /* 1637 * If this buffer is currently syncing out, and we are are 1638 * still referencing it from db_data, we need to make a copy 1639 * of it in case we decide we want to dirty it again in this txg. 1640 */ 1641 if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID && 1642 dn->dn_object != DMU_META_DNODE_OBJECT && 1643 db->db_state == DB_CACHED && db->db_data_pending) { 1644 dbuf_dirty_record_t *dr = db->db_data_pending; 1645 1646 if (dr->dt.dl.dr_data == db->db_buf) { 1647 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 1648 1649 dbuf_set_data(db, 1650 arc_buf_alloc(db->db_dnode->dn_objset->os_spa, 1651 db->db.db_size, db, type)); 1652 bcopy(dr->dt.dl.dr_data->b_data, db->db.db_data, 1653 db->db.db_size); 1654 } 1655 } 1656 1657 (void) refcount_add(&db->db_holds, tag); 1658 dbuf_update_data(db); 1659 DBUF_VERIFY(db); 1660 mutex_exit(&db->db_mtx); 1661 1662 /* NOTE: we can't rele the parent until after we drop the db_mtx */ 1663 if (parent) 1664 dbuf_rele(parent, NULL); 1665 1666 ASSERT3P(db->db_dnode, ==, dn); 1667 ASSERT3U(db->db_blkid, ==, blkid); 1668 ASSERT3U(db->db_level, ==, level); 1669 *dbp = db; 1670 1671 return (0); 1672 } 1673 1674 dmu_buf_impl_t * 1675 dbuf_hold(dnode_t *dn, uint64_t blkid, void *tag) 1676 { 1677 dmu_buf_impl_t *db; 1678 int err = dbuf_hold_impl(dn, 0, blkid, FALSE, tag, &db); 1679 return (err ? NULL : db); 1680 } 1681 1682 dmu_buf_impl_t * 1683 dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, void *tag) 1684 { 1685 dmu_buf_impl_t *db; 1686 int err = dbuf_hold_impl(dn, level, blkid, FALSE, tag, &db); 1687 return (err ? NULL : db); 1688 } 1689 1690 void 1691 dbuf_create_bonus(dnode_t *dn) 1692 { 1693 ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 1694 1695 ASSERT(dn->dn_bonus == NULL); 1696 dn->dn_bonus = dbuf_create(dn, 0, DB_BONUS_BLKID, dn->dn_dbuf, NULL); 1697 } 1698 1699 #pragma weak dmu_buf_add_ref = dbuf_add_ref 1700 void 1701 dbuf_add_ref(dmu_buf_impl_t *db, void *tag) 1702 { 1703 int64_t holds = refcount_add(&db->db_holds, tag); 1704 ASSERT(holds > 1); 1705 } 1706 1707 #pragma weak dmu_buf_rele = dbuf_rele 1708 void 1709 dbuf_rele(dmu_buf_impl_t *db, void *tag) 1710 { 1711 int64_t holds; 1712 1713 mutex_enter(&db->db_mtx); 1714 DBUF_VERIFY(db); 1715 1716 holds = refcount_remove(&db->db_holds, tag); 1717 ASSERT(holds >= 0); 1718 1719 /* 1720 * We can't freeze indirects if there is a possibility that they 1721 * may be modified in the current syncing context. 1722 */ 1723 if (db->db_buf && holds == (db->db_level == 0 ? db->db_dirtycnt : 0)) 1724 arc_buf_freeze(db->db_buf); 1725 1726 if (holds == db->db_dirtycnt && 1727 db->db_level == 0 && db->db_immediate_evict) 1728 dbuf_evict_user(db); 1729 1730 if (holds == 0) { 1731 if (db->db_blkid == DB_BONUS_BLKID) { 1732 mutex_exit(&db->db_mtx); 1733 dnode_rele(db->db_dnode, db); 1734 } else if (db->db_buf == NULL) { 1735 /* 1736 * This is a special case: we never associated this 1737 * dbuf with any data allocated from the ARC. 1738 */ 1739 ASSERT3U(db->db_state, ==, DB_UNCACHED); 1740 dbuf_evict(db); 1741 } else if (arc_released(db->db_buf)) { 1742 arc_buf_t *buf = db->db_buf; 1743 /* 1744 * This dbuf has anonymous data associated with it. 1745 */ 1746 dbuf_set_data(db, NULL); 1747 VERIFY(arc_buf_remove_ref(buf, db) == 1); 1748 dbuf_evict(db); 1749 } else { 1750 VERIFY(arc_buf_remove_ref(db->db_buf, db) == 0); 1751 if (!DBUF_IS_CACHEABLE(db)) 1752 dbuf_clear(db); 1753 else 1754 mutex_exit(&db->db_mtx); 1755 } 1756 } else { 1757 mutex_exit(&db->db_mtx); 1758 } 1759 } 1760 1761 #pragma weak dmu_buf_refcount = dbuf_refcount 1762 uint64_t 1763 dbuf_refcount(dmu_buf_impl_t *db) 1764 { 1765 return (refcount_count(&db->db_holds)); 1766 } 1767 1768 void * 1769 dmu_buf_set_user(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 1770 dmu_buf_evict_func_t *evict_func) 1771 { 1772 return (dmu_buf_update_user(db_fake, NULL, user_ptr, 1773 user_data_ptr_ptr, evict_func)); 1774 } 1775 1776 void * 1777 dmu_buf_set_user_ie(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 1778 dmu_buf_evict_func_t *evict_func) 1779 { 1780 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1781 1782 db->db_immediate_evict = TRUE; 1783 return (dmu_buf_update_user(db_fake, NULL, user_ptr, 1784 user_data_ptr_ptr, evict_func)); 1785 } 1786 1787 void * 1788 dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr, void *user_ptr, 1789 void *user_data_ptr_ptr, dmu_buf_evict_func_t *evict_func) 1790 { 1791 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1792 ASSERT(db->db_level == 0); 1793 1794 ASSERT((user_ptr == NULL) == (evict_func == NULL)); 1795 1796 mutex_enter(&db->db_mtx); 1797 1798 if (db->db_user_ptr == old_user_ptr) { 1799 db->db_user_ptr = user_ptr; 1800 db->db_user_data_ptr_ptr = user_data_ptr_ptr; 1801 db->db_evict_func = evict_func; 1802 1803 dbuf_update_data(db); 1804 } else { 1805 old_user_ptr = db->db_user_ptr; 1806 } 1807 1808 mutex_exit(&db->db_mtx); 1809 return (old_user_ptr); 1810 } 1811 1812 void * 1813 dmu_buf_get_user(dmu_buf_t *db_fake) 1814 { 1815 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1816 ASSERT(!refcount_is_zero(&db->db_holds)); 1817 1818 return (db->db_user_ptr); 1819 } 1820 1821 static void 1822 dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db) 1823 { 1824 /* ASSERT(dmu_tx_is_syncing(tx) */ 1825 ASSERT(MUTEX_HELD(&db->db_mtx)); 1826 1827 if (db->db_blkptr != NULL) 1828 return; 1829 1830 if (db->db_level == dn->dn_phys->dn_nlevels-1) { 1831 /* 1832 * This buffer was allocated at a time when there was 1833 * no available blkptrs from the dnode, or it was 1834 * inappropriate to hook it in (i.e., nlevels mis-match). 1835 */ 1836 ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr); 1837 ASSERT(db->db_parent == NULL); 1838 db->db_parent = dn->dn_dbuf; 1839 db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid]; 1840 DBUF_VERIFY(db); 1841 } else { 1842 dmu_buf_impl_t *parent = db->db_parent; 1843 int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 1844 1845 ASSERT(dn->dn_phys->dn_nlevels > 1); 1846 if (parent == NULL) { 1847 mutex_exit(&db->db_mtx); 1848 rw_enter(&dn->dn_struct_rwlock, RW_READER); 1849 (void) dbuf_hold_impl(dn, db->db_level+1, 1850 db->db_blkid >> epbs, FALSE, db, &parent); 1851 rw_exit(&dn->dn_struct_rwlock); 1852 mutex_enter(&db->db_mtx); 1853 db->db_parent = parent; 1854 } 1855 db->db_blkptr = (blkptr_t *)parent->db.db_data + 1856 (db->db_blkid & ((1ULL << epbs) - 1)); 1857 DBUF_VERIFY(db); 1858 } 1859 } 1860 1861 static void 1862 dbuf_sync_indirect(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 1863 { 1864 dmu_buf_impl_t *db = dr->dr_dbuf; 1865 dnode_t *dn = db->db_dnode; 1866 zio_t *zio; 1867 1868 ASSERT(dmu_tx_is_syncing(tx)); 1869 1870 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 1871 1872 mutex_enter(&db->db_mtx); 1873 1874 ASSERT(db->db_level > 0); 1875 DBUF_VERIFY(db); 1876 1877 if (db->db_buf == NULL) { 1878 mutex_exit(&db->db_mtx); 1879 (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED); 1880 mutex_enter(&db->db_mtx); 1881 } 1882 ASSERT3U(db->db_state, ==, DB_CACHED); 1883 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 1884 ASSERT(db->db_buf != NULL); 1885 1886 dbuf_check_blkptr(dn, db); 1887 1888 db->db_data_pending = dr; 1889 1890 mutex_exit(&db->db_mtx); 1891 dbuf_write(dr, db->db_buf, tx); 1892 1893 zio = dr->dr_zio; 1894 mutex_enter(&dr->dt.di.dr_mtx); 1895 dbuf_sync_list(&dr->dt.di.dr_children, tx); 1896 ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 1897 mutex_exit(&dr->dt.di.dr_mtx); 1898 zio_nowait(zio); 1899 } 1900 1901 static void 1902 dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 1903 { 1904 arc_buf_t **datap = &dr->dt.dl.dr_data; 1905 dmu_buf_impl_t *db = dr->dr_dbuf; 1906 dnode_t *dn = db->db_dnode; 1907 objset_impl_t *os = dn->dn_objset; 1908 uint64_t txg = tx->tx_txg; 1909 int blksz; 1910 1911 ASSERT(dmu_tx_is_syncing(tx)); 1912 1913 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 1914 1915 mutex_enter(&db->db_mtx); 1916 /* 1917 * To be synced, we must be dirtied. But we 1918 * might have been freed after the dirty. 1919 */ 1920 if (db->db_state == DB_UNCACHED) { 1921 /* This buffer has been freed since it was dirtied */ 1922 ASSERT(db->db.db_data == NULL); 1923 } else if (db->db_state == DB_FILL) { 1924 /* This buffer was freed and is now being re-filled */ 1925 ASSERT(db->db.db_data != dr->dt.dl.dr_data); 1926 } else { 1927 ASSERT3U(db->db_state, ==, DB_CACHED); 1928 } 1929 DBUF_VERIFY(db); 1930 1931 /* 1932 * If this is a bonus buffer, simply copy the bonus data into the 1933 * dnode. It will be written out when the dnode is synced (and it 1934 * will be synced, since it must have been dirty for dbuf_sync to 1935 * be called). 1936 */ 1937 if (db->db_blkid == DB_BONUS_BLKID) { 1938 dbuf_dirty_record_t **drp; 1939 1940 ASSERT(*datap != NULL); 1941 ASSERT3U(db->db_level, ==, 0); 1942 ASSERT3U(dn->dn_phys->dn_bonuslen, <=, DN_MAX_BONUSLEN); 1943 bcopy(*datap, DN_BONUS(dn->dn_phys), dn->dn_phys->dn_bonuslen); 1944 if (*datap != db->db.db_data) { 1945 zio_buf_free(*datap, DN_MAX_BONUSLEN); 1946 arc_space_return(DN_MAX_BONUSLEN); 1947 } 1948 db->db_data_pending = NULL; 1949 drp = &db->db_last_dirty; 1950 while (*drp != dr) 1951 drp = &(*drp)->dr_next; 1952 ASSERT(dr->dr_next == NULL); 1953 *drp = dr->dr_next; 1954 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 1955 ASSERT(db->db_dirtycnt > 0); 1956 db->db_dirtycnt -= 1; 1957 mutex_exit(&db->db_mtx); 1958 dbuf_rele(db, (void *)(uintptr_t)txg); 1959 return; 1960 } 1961 1962 /* 1963 * This function may have dropped the db_mtx lock allowing a dmu_sync 1964 * operation to sneak in. As a result, we need to ensure that we 1965 * don't check the dr_override_state until we have returned from 1966 * dbuf_check_blkptr. 1967 */ 1968 dbuf_check_blkptr(dn, db); 1969 1970 /* 1971 * If this buffer is in the middle of an immdiate write, 1972 * wait for the synchronous IO to complete. 1973 */ 1974 while (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC) { 1975 ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT); 1976 cv_wait(&db->db_changed, &db->db_mtx); 1977 ASSERT(dr->dt.dl.dr_override_state != DR_NOT_OVERRIDDEN); 1978 } 1979 1980 /* 1981 * If this dbuf has already been written out via an immediate write, 1982 * just complete the write by copying over the new block pointer and 1983 * updating the accounting via the write-completion functions. 1984 */ 1985 if (dr->dt.dl.dr_override_state == DR_OVERRIDDEN) { 1986 zio_t zio_fake; 1987 1988 zio_fake.io_private = &db; 1989 zio_fake.io_error = 0; 1990 zio_fake.io_bp = db->db_blkptr; 1991 zio_fake.io_bp_orig = *db->db_blkptr; 1992 zio_fake.io_txg = txg; 1993 1994 *db->db_blkptr = dr->dt.dl.dr_overridden_by; 1995 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 1996 db->db_data_pending = dr; 1997 dr->dr_zio = &zio_fake; 1998 mutex_exit(&db->db_mtx); 1999 2000 if (BP_IS_OLDER(&zio_fake.io_bp_orig, txg)) 2001 (void) dsl_dataset_block_kill(os->os_dsl_dataset, 2002 &zio_fake.io_bp_orig, dn->dn_zio, tx); 2003 2004 dbuf_write_ready(&zio_fake, db->db_buf, db); 2005 dbuf_write_done(&zio_fake, db->db_buf, db); 2006 2007 return; 2008 } 2009 2010 blksz = arc_buf_size(*datap); 2011 2012 if (dn->dn_object != DMU_META_DNODE_OBJECT) { 2013 /* 2014 * If this buffer is currently "in use" (i.e., there are 2015 * active holds and db_data still references it), then make 2016 * a copy before we start the write so that any modifications 2017 * from the open txg will not leak into this write. 2018 * 2019 * NOTE: this copy does not need to be made for objects only 2020 * modified in the syncing context (e.g. DNONE_DNODE blocks). 2021 */ 2022 if (refcount_count(&db->db_holds) > 1 && *datap == db->db_buf) { 2023 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 2024 *datap = arc_buf_alloc(os->os_spa, blksz, db, type); 2025 bcopy(db->db.db_data, (*datap)->b_data, blksz); 2026 } 2027 } 2028 2029 ASSERT(*datap != NULL); 2030 db->db_data_pending = dr; 2031 2032 mutex_exit(&db->db_mtx); 2033 2034 dbuf_write(dr, *datap, tx); 2035 2036 ASSERT(!list_link_active(&dr->dr_dirty_node)); 2037 if (dn->dn_object == DMU_META_DNODE_OBJECT) 2038 list_insert_tail(&dn->dn_dirty_records[txg&TXG_MASK], dr); 2039 else 2040 zio_nowait(dr->dr_zio); 2041 } 2042 2043 void 2044 dbuf_sync_list(list_t *list, dmu_tx_t *tx) 2045 { 2046 dbuf_dirty_record_t *dr; 2047 2048 while (dr = list_head(list)) { 2049 if (dr->dr_zio != NULL) { 2050 /* 2051 * If we find an already initialized zio then we 2052 * are processing the meta-dnode, and we have finished. 2053 * The dbufs for all dnodes are put back on the list 2054 * during processing, so that we can zio_wait() 2055 * these IOs after initiating all child IOs. 2056 */ 2057 ASSERT3U(dr->dr_dbuf->db.db_object, ==, 2058 DMU_META_DNODE_OBJECT); 2059 break; 2060 } 2061 list_remove(list, dr); 2062 if (dr->dr_dbuf->db_level > 0) 2063 dbuf_sync_indirect(dr, tx); 2064 else 2065 dbuf_sync_leaf(dr, tx); 2066 } 2067 } 2068 2069 static void 2070 dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx) 2071 { 2072 dmu_buf_impl_t *db = dr->dr_dbuf; 2073 dnode_t *dn = db->db_dnode; 2074 objset_impl_t *os = dn->dn_objset; 2075 dmu_buf_impl_t *parent = db->db_parent; 2076 uint64_t txg = tx->tx_txg; 2077 zbookmark_t zb; 2078 writeprops_t wp = { 0 }; 2079 zio_t *zio; 2080 int zio_flags; 2081 2082 if (!BP_IS_HOLE(db->db_blkptr) && 2083 (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE)) { 2084 /* 2085 * Private object buffers are released here rather 2086 * than in dbuf_dirty() since they are only modified 2087 * in the syncing context and we don't want the 2088 * overhead of making multiple copies of the data. 2089 */ 2090 arc_release(data, db); 2091 } else { 2092 ASSERT(arc_released(data)); 2093 /* XXX why do we need to thaw here? */ 2094 arc_buf_thaw(data); 2095 } 2096 2097 if (parent != dn->dn_dbuf) { 2098 ASSERT(parent && parent->db_data_pending); 2099 ASSERT(db->db_level == parent->db_level-1); 2100 ASSERT(arc_released(parent->db_buf)); 2101 zio = parent->db_data_pending->dr_zio; 2102 } else { 2103 ASSERT(db->db_level == dn->dn_phys->dn_nlevels-1); 2104 ASSERT3P(db->db_blkptr, ==, 2105 &dn->dn_phys->dn_blkptr[db->db_blkid]); 2106 zio = dn->dn_zio; 2107 } 2108 2109 ASSERT(db->db_level == 0 || data == db->db_buf); 2110 ASSERT3U(db->db_blkptr->blk_birth, <=, txg); 2111 ASSERT(zio); 2112 2113 zb.zb_objset = os->os_dsl_dataset ? os->os_dsl_dataset->ds_object : 0; 2114 zb.zb_object = db->db.db_object; 2115 zb.zb_level = db->db_level; 2116 zb.zb_blkid = db->db_blkid; 2117 2118 zio_flags = ZIO_FLAG_MUSTSUCCEED; 2119 if (dmu_ot[dn->dn_type].ot_metadata || zb.zb_level != 0) 2120 zio_flags |= ZIO_FLAG_METADATA; 2121 wp.wp_type = dn->dn_type; 2122 wp.wp_level = db->db_level; 2123 wp.wp_copies = os->os_copies; 2124 wp.wp_dncompress = dn->dn_compress; 2125 wp.wp_oscompress = os->os_compress; 2126 wp.wp_dnchecksum = dn->dn_checksum; 2127 wp.wp_oschecksum = os->os_checksum; 2128 2129 if (BP_IS_OLDER(db->db_blkptr, txg)) 2130 (void) dsl_dataset_block_kill( 2131 os->os_dsl_dataset, db->db_blkptr, zio, tx); 2132 2133 dr->dr_zio = arc_write(zio, os->os_spa, &wp, 2134 DBUF_IS_L2CACHEABLE(db), txg, db->db_blkptr, 2135 data, dbuf_write_ready, dbuf_write_done, db, 2136 ZIO_PRIORITY_ASYNC_WRITE, zio_flags, &zb); 2137 } 2138 2139 /* ARGSUSED */ 2140 static void 2141 dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb) 2142 { 2143 dmu_buf_impl_t *db = vdb; 2144 dnode_t *dn = db->db_dnode; 2145 objset_impl_t *os = dn->dn_objset; 2146 blkptr_t *bp_orig = &zio->io_bp_orig; 2147 uint64_t fill = 0; 2148 int old_size, new_size, i; 2149 2150 dprintf_dbuf_bp(db, bp_orig, "bp_orig: %s", ""); 2151 2152 old_size = bp_get_dasize(os->os_spa, bp_orig); 2153 new_size = bp_get_dasize(os->os_spa, zio->io_bp); 2154 2155 dnode_diduse_space(dn, new_size-old_size); 2156 2157 if (BP_IS_HOLE(zio->io_bp)) { 2158 dsl_dataset_t *ds = os->os_dsl_dataset; 2159 dmu_tx_t *tx = os->os_synctx; 2160 2161 if (bp_orig->blk_birth == tx->tx_txg) 2162 (void) dsl_dataset_block_kill(ds, bp_orig, NULL, tx); 2163 ASSERT3U(db->db_blkptr->blk_fill, ==, 0); 2164 return; 2165 } 2166 2167 mutex_enter(&db->db_mtx); 2168 2169 if (db->db_level == 0) { 2170 mutex_enter(&dn->dn_mtx); 2171 if (db->db_blkid > dn->dn_phys->dn_maxblkid) 2172 dn->dn_phys->dn_maxblkid = db->db_blkid; 2173 mutex_exit(&dn->dn_mtx); 2174 2175 if (dn->dn_type == DMU_OT_DNODE) { 2176 dnode_phys_t *dnp = db->db.db_data; 2177 for (i = db->db.db_size >> DNODE_SHIFT; i > 0; 2178 i--, dnp++) { 2179 if (dnp->dn_type != DMU_OT_NONE) 2180 fill++; 2181 } 2182 } else { 2183 fill = 1; 2184 } 2185 } else { 2186 blkptr_t *bp = db->db.db_data; 2187 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2188 for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, bp++) { 2189 if (BP_IS_HOLE(bp)) 2190 continue; 2191 ASSERT3U(BP_GET_LSIZE(bp), ==, 2192 db->db_level == 1 ? dn->dn_datablksz : 2193 (1<<dn->dn_phys->dn_indblkshift)); 2194 fill += bp->blk_fill; 2195 } 2196 } 2197 2198 db->db_blkptr->blk_fill = fill; 2199 BP_SET_TYPE(db->db_blkptr, dn->dn_type); 2200 BP_SET_LEVEL(db->db_blkptr, db->db_level); 2201 2202 mutex_exit(&db->db_mtx); 2203 2204 /* We must do this after we've set the bp's type and level */ 2205 if (!DVA_EQUAL(BP_IDENTITY(zio->io_bp), BP_IDENTITY(bp_orig))) { 2206 dsl_dataset_t *ds = os->os_dsl_dataset; 2207 dmu_tx_t *tx = os->os_synctx; 2208 2209 if (bp_orig->blk_birth == tx->tx_txg) 2210 (void) dsl_dataset_block_kill(ds, bp_orig, NULL, tx); 2211 dsl_dataset_block_born(ds, zio->io_bp, tx); 2212 } 2213 } 2214 2215 /* ARGSUSED */ 2216 static void 2217 dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb) 2218 { 2219 dmu_buf_impl_t *db = vdb; 2220 uint64_t txg = zio->io_txg; 2221 dbuf_dirty_record_t **drp, *dr; 2222 2223 ASSERT3U(zio->io_error, ==, 0); 2224 2225 mutex_enter(&db->db_mtx); 2226 2227 drp = &db->db_last_dirty; 2228 while ((dr = *drp) != db->db_data_pending) 2229 drp = &dr->dr_next; 2230 ASSERT(!list_link_active(&dr->dr_dirty_node)); 2231 ASSERT(dr->dr_txg == txg); 2232 ASSERT(dr->dr_next == NULL); 2233 *drp = dr->dr_next; 2234 2235 if (db->db_level == 0) { 2236 ASSERT(db->db_blkid != DB_BONUS_BLKID); 2237 ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN); 2238 2239 if (dr->dt.dl.dr_data != db->db_buf) 2240 VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, db) == 1); 2241 else if (!BP_IS_HOLE(db->db_blkptr)) 2242 arc_set_callback(db->db_buf, dbuf_do_evict, db); 2243 else 2244 ASSERT(arc_released(db->db_buf)); 2245 } else { 2246 dnode_t *dn = db->db_dnode; 2247 2248 ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 2249 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2250 if (!BP_IS_HOLE(db->db_blkptr)) { 2251 int epbs = 2252 dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2253 ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, 2254 db->db.db_size); 2255 ASSERT3U(dn->dn_phys->dn_maxblkid 2256 >> (db->db_level * epbs), >=, db->db_blkid); 2257 arc_set_callback(db->db_buf, dbuf_do_evict, db); 2258 } 2259 mutex_destroy(&dr->dt.di.dr_mtx); 2260 list_destroy(&dr->dt.di.dr_children); 2261 } 2262 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 2263 2264 cv_broadcast(&db->db_changed); 2265 ASSERT(db->db_dirtycnt > 0); 2266 db->db_dirtycnt -= 1; 2267 db->db_data_pending = NULL; 2268 mutex_exit(&db->db_mtx); 2269 2270 dprintf_dbuf_bp(db, zio->io_bp, "bp: %s", ""); 2271 2272 dbuf_rele(db, (void *)(uintptr_t)txg); 2273 } 2274