Lines Matching +full:write +full:- +full:1 +full:- +full:bps
1 // SPDX-License-Identifier: CDDL-1.0
10 * or https://opensource.org/licenses/CDDL-1.0.
69 static int zfs_nopwrite_enabled = 1;
86 static int zfs_dmu_offset_next_sync = 1;
182 rw_enter(&dn->dn_struct_rwlock, RW_READER); in dmu_buf_hold_noread_by_dnode()
185 rw_exit(&dn->dn_struct_rwlock); in dmu_buf_hold_noread_by_dnode()
192 *dbp = &db->db; in dmu_buf_hold_noread_by_dnode()
208 rw_enter(&dn->dn_struct_rwlock, RW_READER); in dmu_buf_hold_noread()
211 rw_exit(&dn->dn_struct_rwlock); in dmu_buf_hold_noread()
219 *dbp = &db->db; in dmu_buf_hold_noread()
274 if (newsize < 0 || newsize > db_fake->db_size) in dmu_set_bonus()
280 if (dn->dn_bonus != db) { in dmu_set_bonus()
304 if (dn->dn_bonus != db) { in dmu_set_bonustype()
322 type = DB_DNODE(db)->dn_bonustype; in dmu_get_bonustype()
336 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); in dmu_rm_spill()
338 rw_exit(&dn->dn_struct_rwlock); in dmu_rm_spill()
354 rw_enter(&dn->dn_struct_rwlock, RW_READER); in dmu_bonus_hold_by_dnode()
355 if (dn->dn_bonus == NULL) { in dmu_bonus_hold_by_dnode()
356 if (!rw_tryupgrade(&dn->dn_struct_rwlock)) { in dmu_bonus_hold_by_dnode()
357 rw_exit(&dn->dn_struct_rwlock); in dmu_bonus_hold_by_dnode()
358 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); in dmu_bonus_hold_by_dnode()
360 if (dn->dn_bonus == NULL) in dmu_bonus_hold_by_dnode()
363 db = dn->dn_bonus; in dmu_bonus_hold_by_dnode()
366 if (zfs_refcount_add(&db->db_holds, tag) == 1) { in dmu_bonus_hold_by_dnode()
368 atomic_inc_32(&dn->dn_dbufs_count); in dmu_bonus_hold_by_dnode()
376 rw_exit(&dn->dn_struct_rwlock); in dmu_bonus_hold_by_dnode()
386 *dbp = &db->db; in dmu_bonus_hold_by_dnode()
423 rw_enter(&dn->dn_struct_rwlock, RW_READER); in dmu_spill_hold_by_dnode()
428 rw_exit(&dn->dn_struct_rwlock); in dmu_spill_hold_by_dnode()
436 *dbp = &db->db; in dmu_spill_hold_by_dnode()
454 if (spa_version(dn->dn_objset->os_spa) < SPA_VERSION_SA) { in dmu_spill_hold_existing()
457 rw_enter(&dn->dn_struct_rwlock, RW_READER); in dmu_spill_hold_existing()
459 if (!dn->dn_have_spill) { in dmu_spill_hold_existing()
466 rw_exit(&dn->dn_struct_rwlock); in dmu_spill_hold_existing()
488 * Note: longer-term, we should modify all of the dmu_buf_*() interfaces
489 * to take a held dnode rather than <os, object> -- the lookup is wasteful,
510 * we can tell it about the multi-block read. dbuf_read() only knows in dmu_buf_hold_array_by_dnode()
516 rw_enter(&dn->dn_struct_rwlock, RW_READER); in dmu_buf_hold_array_by_dnode()
517 if (dn->dn_datablkshift) { in dmu_buf_hold_array_by_dnode()
518 int blkshift = dn->dn_datablkshift; in dmu_buf_hold_array_by_dnode()
519 nblks = (P2ROUNDUP(offset + length, 1ULL << blkshift) - in dmu_buf_hold_array_by_dnode()
520 P2ALIGN_TYPED(offset, 1ULL << blkshift, uint64_t)) in dmu_buf_hold_array_by_dnode()
523 if (offset + length > dn->dn_datablksz) { in dmu_buf_hold_array_by_dnode()
526 (longlong_t)dn->dn_objset-> in dmu_buf_hold_array_by_dnode()
527 os_dsl_dataset->ds_object, in dmu_buf_hold_array_by_dnode()
528 (longlong_t)dn->dn_object, dn->dn_datablksz, in dmu_buf_hold_array_by_dnode()
530 rw_exit(&dn->dn_struct_rwlock); in dmu_buf_hold_array_by_dnode()
533 nblks = 1; in dmu_buf_hold_array_by_dnode()
538 zio = zio_root(dn->dn_objset->os_spa, NULL, NULL, in dmu_buf_hold_array_by_dnode()
547 zs = dmu_zfetch_prepare(&dn->dn_zfetch, blkid, nblks, in dmu_buf_hold_array_by_dnode()
554 dmu_zfetch_run(&dn->dn_zfetch, zs, missed, in dmu_buf_hold_array_by_dnode()
557 rw_exit(&dn->dn_struct_rwlock); in dmu_buf_hold_array_by_dnode()
567 * (1) dbuf_read() may change the state to CACHED due to a in dmu_buf_hold_array_by_dnode()
573 if (i == nblks - 1 && blkid + i < dn->dn_maxblkid && in dmu_buf_hold_array_by_dnode()
574 offset + length < db->db.db_offset + in dmu_buf_hold_array_by_dnode()
575 db->db.db_size) { in dmu_buf_hold_array_by_dnode()
576 if (offset <= db->db.db_offset) in dmu_buf_hold_array_by_dnode()
582 if (db->db_state != DB_CACHED) in dmu_buf_hold_array_by_dnode()
585 dbp[i] = &db->db; in dmu_buf_hold_array_by_dnode()
593 * O_DIRECT write/read accounting takes place in in dmu_buf_hold_array_by_dnode()
594 * dmu_{write/read}_abd(). in dmu_buf_hold_array_by_dnode()
597 zfs_racct_write(dn->dn_objset->os_spa, length, nblks, flags); in dmu_buf_hold_array_by_dnode()
600 dmu_zfetch_run(&dn->dn_zfetch, zs, missed, B_TRUE, in dmu_buf_hold_array_by_dnode()
603 rw_exit(&dn->dn_struct_rwlock); in dmu_buf_hold_array_by_dnode()
616 mutex_enter(&db->db_mtx); in dmu_buf_hold_array_by_dnode()
617 while (db->db_state == DB_READ || in dmu_buf_hold_array_by_dnode()
618 db->db_state == DB_FILL) in dmu_buf_hold_array_by_dnode()
619 cv_wait(&db->db_changed, &db->db_mtx); in dmu_buf_hold_array_by_dnode()
620 if (db->db_state == DB_UNCACHED) in dmu_buf_hold_array_by_dnode()
622 mutex_exit(&db->db_mtx); in dmu_buf_hold_array_by_dnode()
731 rw_enter(&dn->dn_struct_rwlock, RW_READER); in dmu_prefetch_by_dnode()
732 if (dn->dn_datablkshift != 0) { in dmu_prefetch_by_dnode()
737 uint64_t size = (dn->dn_maxblkid + 1) << dn->dn_datablkshift; in dmu_prefetch_by_dnode()
739 rw_exit(&dn->dn_struct_rwlock); in dmu_prefetch_by_dnode()
743 len = size - offset; in dmu_prefetch_by_dnode()
751 end2 = dbuf_whichblock(dn, level, offset + len - 1) + 1; in dmu_prefetch_by_dnode()
752 uint8_t ibs = dn->dn_indblkshift; in dmu_prefetch_by_dnode()
753 uint8_t bs = (level == 0) ? dn->dn_datablkshift : ibs; in dmu_prefetch_by_dnode()
754 uint_t limit = P2ROUNDUP(dmu_prefetch_max, 1 << bs) >> bs; in dmu_prefetch_by_dnode()
760 uint8_t ibps = ibs - SPA_BLKPTRSHIFT; in dmu_prefetch_by_dnode()
761 limit = P2ROUNDUP(dmu_prefetch_max, 1 << ibs) >> ibs; in dmu_prefetch_by_dnode()
766 start2 = P2ROUNDUP(start2, 1 << ibps) >> ibps; in dmu_prefetch_by_dnode()
767 end2 = P2ROUNDUP(end2, 1 << ibps) >> ibps; in dmu_prefetch_by_dnode()
768 } while (end2 - start2 > limit); in dmu_prefetch_by_dnode()
772 end = start + (level == 0 && offset < dn->dn_datablksz); in dmu_prefetch_by_dnode()
779 rw_exit(&dn->dn_struct_rwlock); in dmu_prefetch_by_dnode()
796 mutex_enter(&dpa->dpa_lock); in dmu_prefetch_done()
797 ASSERT3U(dpa->dpa_pending_io, >, 0); in dmu_prefetch_done()
798 if (--dpa->dpa_pending_io == 0) in dmu_prefetch_done()
799 cv_broadcast(&dpa->dpa_cv); in dmu_prefetch_done()
800 mutex_exit(&dpa->dpa_lock); in dmu_prefetch_done()
811 rw_enter(&dn->dn_struct_rwlock, RW_READER); in dmu_prefetch_wait_by_dnode()
814 uint64_t end = dbuf_whichblock(dn, 0, offset + len - 1) + 1; in dmu_prefetch_wait_by_dnode()
815 dpa.dpa_pending_io = end - start; in dmu_prefetch_wait_by_dnode()
822 rw_exit(&dn->dn_struct_rwlock); in dmu_prefetch_wait_by_dnode()
856 if (dn->dn_indblkshift) { in dmu_prefetch_wait()
857 uint64_t nbps = bp_span_in_blocks(dn->dn_indblkshift, 1); in dmu_prefetch_wait()
858 chunksize = (nbps * 16) << dn->dn_datablkshift; in dmu_prefetch_wait()
860 chunksize = dn->dn_datablksz; in dmu_prefetch_wait()
869 size -= mylen; in dmu_prefetch_wait()
892 rw_enter(&dn->dn_struct_rwlock, RW_READER); in dmu_prefetch_dnode()
895 rw_exit(&dn->dn_struct_rwlock); in dmu_prefetch_dnode()
902 * data by simply searching the allocated level 1 indirects.
906 * offset that should be freed and l1blks is set to the number of level 1
913 uint64_t maxblks = DMU_MAX_ACCESS >> (dn->dn_indblkshift + 1); in get_next_chunk()
914 /* bytes of data covered by a level-1 indirect block */ in get_next_chunk()
915 uint64_t iblkrange = (uint64_t)dn->dn_datablksz * in get_next_chunk()
916 EPB(dn->dn_indblkshift, SPA_BLKPTRSHIFT); in get_next_chunk()
920 /* dn_nlevels == 1 means we don't have any L1 blocks */ in get_next_chunk()
921 if (dn->dn_nlevels <= 1) { in get_next_chunk()
934 (roundup(*start, iblkrange) - (minimum / iblkrange * iblkrange)) / in get_next_chunk()
952 (*start)--; in get_next_chunk()
955 DNODE_FIND_BACKWARDS, start, 2, 1, 0); in get_next_chunk()
1005 object_size = (dn->dn_maxblkid + 1) * dn->dn_datablksz; in dmu_free_long_range_impl()
1016 length = object_size - offset; in dmu_free_long_range_impl()
1023 if (dmu_objset_zfs_unmounting(dn->dn_objset)) in dmu_free_long_range_impl()
1035 chunk_len = chunk_end - chunk_begin; in dmu_free_long_range_impl()
1038 dmu_tx_hold_free(tx, dn->dn_object, chunk_begin, chunk_len); in dmu_free_long_range_impl()
1053 mutex_enter(&dp->dp_lock); in dmu_free_long_range_impl()
1055 dp->dp_long_free_dirty_pertxg[txg & TXG_MASK]; in dmu_free_long_range_impl()
1056 mutex_exit(&dp->dp_lock); in dmu_free_long_range_impl()
1072 * In order to prevent unnecessary write throttling, for each in dmu_free_long_range_impl()
1078 * prevents write throttle activation due to dirty freeing L1 in dmu_free_long_range_impl()
1081 mutex_enter(&dp->dp_lock); in dmu_free_long_range_impl()
1082 dp->dp_long_free_dirty_pertxg[txg & TXG_MASK] += in dmu_free_long_range_impl()
1083 l1blks << dn->dn_indblkshift; in dmu_free_long_range_impl()
1084 mutex_exit(&dp->dp_lock); in dmu_free_long_range_impl()
1092 length -= chunk_len; in dmu_free_long_range_impl()
1116 dn->dn_maxblkid = 0; in dmu_free_long_range()
1156 ASSERT(size == DMU_OBJECT_END || size <= UINT64_MAX - offset); in dmu_free_range()
1174 if (dn->dn_maxblkid == 0) { in dmu_read_impl()
1175 uint64_t newsz = offset > dn->dn_datablksz ? 0 : in dmu_read_impl()
1176 MIN(size, dn->dn_datablksz - offset); in dmu_read_impl()
1177 memset((char *)buf + newsz, 0, size - newsz); in dmu_read_impl()
1199 * NB: we could do this block-at-a-time, but it's nice in dmu_read_impl()
1214 bufoff = offset - db->db_offset; in dmu_read_impl()
1215 tocpy = MIN(db->db_size - bufoff, size); in dmu_read_impl()
1217 ASSERT(db->db_data != NULL); in dmu_read_impl()
1218 (void) memcpy(buf, (char *)db->db_data + bufoff, tocpy); in dmu_read_impl()
1221 size -= tocpy; in dmu_read_impl()
1265 bufoff = offset - db->db_offset; in dmu_write_impl()
1266 tocpy = MIN(db->db_size - bufoff, size); in dmu_write_impl()
1268 ASSERT(i == 0 || i == numbufs-1 || tocpy == db->db_size); in dmu_write_impl()
1270 if (tocpy == db->db_size) { in dmu_write_impl()
1273 if (i == numbufs - 1 && bufoff + tocpy < db->db_size) { in dmu_write_impl()
1282 ASSERT(db->db_data != NULL); in dmu_write_impl()
1283 (void) memcpy((char *)db->db_data + bufoff, buf, tocpy); in dmu_write_impl()
1285 if (tocpy == db->db_size) in dmu_write_impl()
1289 size -= tocpy; in dmu_write_impl()
1323 zfs_dio_aligned(offset, size, dn->dn_datablksz)) { in dmu_write_by_dnode()
1400 if ((flags & DMU_DIRECTIO) && (uio->uio_extflg & UIO_DIRECT)) in dmu_read_uio_dnode()
1405 * NB: we could do this block-at-a-time, but it's nice in dmu_read_uio_dnode()
1420 bufoff = zfs_uio_offset(uio) - db->db_offset; in dmu_read_uio_dnode()
1421 tocpy = MIN(db->db_size - bufoff, size); in dmu_read_uio_dnode()
1423 ASSERT(db->db_data != NULL); in dmu_read_uio_dnode()
1424 err = zfs_uio_fault_move((char *)db->db_data + bufoff, tocpy, in dmu_read_uio_dnode()
1430 size -= tocpy; in dmu_read_uio_dnode()
1439 * From object zdb->db_object.
1504 * sized aligned. Otherwise, we pass the write off to the ARC. in dmu_write_uio_dnode()
1506 if ((flags & DMU_DIRECTIO) && (uio->uio_extflg & UIO_DIRECT) && in dmu_write_uio_dnode()
1507 (write_size >= dn->dn_datablksz)) { in dmu_write_uio_dnode()
1509 dn->dn_datablksz)) { in dmu_write_uio_dnode()
1511 } else if (write_size > dn->dn_datablksz && in dmu_write_uio_dnode()
1513 dn->dn_datablksz)) { in dmu_write_uio_dnode()
1515 dn->dn_datablksz * (write_size / dn->dn_datablksz); in dmu_write_uio_dnode()
1519 size -= write_size; in dmu_write_uio_dnode()
1526 P2PHASE(zfs_uio_offset(uio), dn->dn_datablksz); in dmu_write_uio_dnode()
1544 bufoff = off - db->db_offset; in dmu_write_uio_dnode()
1545 tocpy = MIN(db->db_size - bufoff, write_size); in dmu_write_uio_dnode()
1547 ASSERT(i == 0 || i == numbufs-1 || tocpy == db->db_size); in dmu_write_uio_dnode()
1549 if (tocpy == db->db_size) { in dmu_write_uio_dnode()
1552 if (i == numbufs - 1 && bufoff + tocpy < db->db_size) { in dmu_write_uio_dnode()
1561 ASSERT(db->db_data != NULL); in dmu_write_uio_dnode()
1562 err = zfs_uio_fault_move((char *)db->db_data + bufoff, in dmu_write_uio_dnode()
1565 if (tocpy == db->db_size && dmu_buf_fill_done(db, tx, err)) { in dmu_write_uio_dnode()
1567 zfs_uio_advance(uio, off - zfs_uio_offset(uio)); in dmu_write_uio_dnode()
1573 write_size -= tocpy; in dmu_write_uio_dnode()
1574 size -= tocpy; in dmu_write_uio_dnode()
1581 if ((oflags & DMU_DIRECTIO) && (uio->uio_extflg & UIO_DIRECT) && in dmu_write_uio_dnode()
1592 * Write 'size' bytes from the uio buffer.
1593 * To object zdb->db_object.
1618 * Write 'size' bytes from the uio buffer.
1645 dmu_cached_bps(spa_t *spa, blkptr_t *bps, uint_t nbps, in dmu_cached_bps() argument
1650 if (bps == NULL) in dmu_cached_bps()
1654 blkptr_t *bp = &bps[blk_off]; in dmu_cached_bps()
1687 if (dn->dn_nlevels < 2) { in dmu_object_cached_size()
1697 dmu_prefetch_by_dnode(dn, 1, off, in dmu_object_cached_size()
1705 uint_t nbps = bp_span_in_blocks(dn->dn_indblkshift, 1); in dmu_object_cached_size()
1706 uint64_t l1blks = 1 + (dn->dn_maxblkid / nbps); in dmu_object_cached_size()
1708 rw_enter(&dn->dn_struct_rwlock, RW_READER); in dmu_object_cached_size()
1726 err = dbuf_hold_impl(dn, 1, blk, B_TRUE, B_FALSE, FTAG, &db); in dmu_object_cached_size()
1737 dmu_cached_bps(dmu_objset_spa(os), db->db.db_data, in dmu_object_cached_size()
1746 rw_exit(&dn->dn_struct_rwlock); in dmu_object_cached_size()
1760 return (arc_loan_buf(db->db_objset->os_spa, B_FALSE, size)); in dmu_request_arcbuf()
1774 * A "lightweight" write is faster than a regular write (e.g.
1779 * (temporarily) write-only, like "zfs receive".
1793 dr->dt.dll.dr_abd = abd; in dmu_lightweight_write_by_dnode()
1794 dr->dt.dll.dr_props = *zp; in dmu_lightweight_write_by_dnode()
1795 dr->dt.dll.dr_flags = flags; in dmu_lightweight_write_by_dnode()
1809 objset_t *os = dn->dn_objset; in dmu_assign_arcbuf_by_dnode()
1813 rw_enter(&dn->dn_struct_rwlock, RW_READER); in dmu_assign_arcbuf_by_dnode()
1816 rw_exit(&dn->dn_struct_rwlock); in dmu_assign_arcbuf_by_dnode()
1824 if (offset == db->db.db_offset && blksz == db->db.db_size) { in dmu_assign_arcbuf_by_dnode()
1825 zfs_racct_write(os->os_spa, blksz, 1, flags); in dmu_assign_arcbuf_by_dnode()
1831 ASSERT(!(buf->b_flags & ARC_BUF_FLAG_COMPRESSED)); in dmu_assign_arcbuf_by_dnode()
1834 dmu_write_by_dnode(dn, offset, blksz, buf->b_data, tx, flags); in dmu_assign_arcbuf_by_dnode()
1861 if (zio->io_error == 0) { in dmu_sync_ready()
1862 dbuf_dirty_record_t *dr = dsa->dsa_dr; in dmu_sync_ready()
1863 blkptr_t *bp = zio->io_bp; in dmu_sync_ready()
1868 db = &(dr->dr_dbuf->db); in dmu_sync_ready()
1870 db = dsa->dsa_zgd->zgd_db; in dmu_sync_ready()
1875 BP_SET_LSIZE(bp, db->db_size); in dmu_sync_ready()
1878 BP_SET_FILL(bp, 1); in dmu_sync_ready()
1886 dmu_sync_ready(zio, NULL, zio->io_private); in dmu_sync_late_arrival_ready()
1894 dbuf_dirty_record_t *dr = dsa->dsa_dr; in dmu_sync_done()
1895 dmu_buf_impl_t *db = dr->dr_dbuf; in dmu_sync_done()
1896 zgd_t *zgd = dsa->dsa_zgd; in dmu_sync_done()
1902 if (zgd && zio->io_error == 0) { in dmu_sync_done()
1903 zil_lwb_add_block(zgd->zgd_lwb, zgd->zgd_bp); in dmu_sync_done()
1906 mutex_enter(&db->db_mtx); in dmu_sync_done()
1907 ASSERT(dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC); in dmu_sync_done()
1908 if (zio->io_error == 0) { in dmu_sync_done()
1909 ASSERT0(dr->dt.dl.dr_has_raw_params); in dmu_sync_done()
1910 dr->dt.dl.dr_nopwrite = !!(zio->io_flags & ZIO_FLAG_NOPWRITE); in dmu_sync_done()
1911 if (dr->dt.dl.dr_nopwrite) { in dmu_sync_done()
1912 blkptr_t *bp = zio->io_bp; in dmu_sync_done()
1913 blkptr_t *bp_orig = &zio->io_bp_orig; in dmu_sync_done()
1917 VERIFY(BP_EQUAL(bp, db->db_blkptr)); in dmu_sync_done()
1918 ASSERT(zio->io_prop.zp_compress != ZIO_COMPRESS_OFF); in dmu_sync_done()
1922 dr->dt.dl.dr_overridden_by = *zio->io_bp; in dmu_sync_done()
1923 dr->dt.dl.dr_override_state = DR_OVERRIDDEN; in dmu_sync_done()
1924 dr->dt.dl.dr_copies = zio->io_prop.zp_copies; in dmu_sync_done()
1925 dr->dt.dl.dr_gang_copies = zio->io_prop.zp_gang_copies; in dmu_sync_done()
1929 * new-style holes maintain their lsize, type, level, in dmu_sync_done()
1937 if (BP_IS_HOLE(&dr->dt.dl.dr_overridden_by) && in dmu_sync_done()
1938 BP_GET_LOGICAL_BIRTH(&dr->dt.dl.dr_overridden_by) == 0) in dmu_sync_done()
1939 BP_ZERO(&dr->dt.dl.dr_overridden_by); in dmu_sync_done()
1941 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; in dmu_sync_done()
1944 cv_broadcast(&db->db_changed); in dmu_sync_done()
1945 mutex_exit(&db->db_mtx); in dmu_sync_done()
1947 if (dsa->dsa_done) in dmu_sync_done()
1948 dsa->dsa_done(dsa->dsa_zgd, zio->io_error); in dmu_sync_done()
1956 blkptr_t *bp = zio->io_bp; in dmu_sync_late_arrival_done()
1957 dmu_sync_arg_t *dsa = zio->io_private; in dmu_sync_late_arrival_done()
1958 zgd_t *zgd = dsa->dsa_zgd; in dmu_sync_late_arrival_done()
1960 if (zio->io_error == 0) { in dmu_sync_late_arrival_done()
1965 zil_lwb_add_block(zgd->zgd_lwb, zgd->zgd_bp); in dmu_sync_late_arrival_done()
1968 blkptr_t *bp_orig __maybe_unused = &zio->io_bp_orig; in dmu_sync_late_arrival_done()
1969 ASSERT(!(zio->io_flags & ZIO_FLAG_NOPWRITE)); in dmu_sync_late_arrival_done()
1971 ASSERT(BP_GET_BIRTH(zio->io_bp) == zio->io_txg); in dmu_sync_late_arrival_done()
1972 ASSERT(zio->io_txg > spa_syncing_txg(zio->io_spa)); in dmu_sync_late_arrival_done()
1973 zio_free(zio->io_spa, zio->io_txg, zio->io_bp); in dmu_sync_late_arrival_done()
1977 dmu_tx_commit(dsa->dsa_tx); in dmu_sync_late_arrival_done()
1979 dsa->dsa_done(dsa->dsa_zgd, zio->io_error); in dmu_sync_late_arrival_done()
1981 abd_free(zio->io_abd); in dmu_sync_late_arrival_done()
1993 error = dbuf_read((dmu_buf_impl_t *)zgd->zgd_db, NULL, in dmu_sync_late_arrival()
1999 dmu_tx_hold_space(tx, zgd->zgd_db->db_size); in dmu_sync_late_arrival()
2017 zil_lwb_add_txg(zgd->zgd_lwb, dmu_tx_get_txg(tx)); in dmu_sync_late_arrival()
2020 dsa->dsa_dr = NULL; in dmu_sync_late_arrival()
2021 dsa->dsa_done = done; in dmu_sync_late_arrival()
2022 dsa->dsa_zgd = zgd; in dmu_sync_late_arrival()
2023 dsa->dsa_tx = tx; in dmu_sync_late_arrival()
2031 * (this is similar to the non-late-arrival case where the dbuf is in dmu_sync_late_arrival()
2034 * Then dbuf_write_ready() sets bp_blkptr to the location we will write. in dmu_sync_late_arrival()
2041 * this txg is the same data that we are trying to write. However we in dmu_sync_late_arrival()
2046 zp->zp_nopwrite = B_FALSE; in dmu_sync_late_arrival()
2048 zio_nowait(zio_write(pio, os->os_spa, dmu_tx_get_txg(tx), zgd->zgd_bp, in dmu_sync_late_arrival()
2049 abd_get_from_buf(zgd->zgd_db->db_data, zgd->zgd_db->db_size), in dmu_sync_late_arrival()
2050 zgd->zgd_db->db_size, zgd->zgd_db->db_size, zp, in dmu_sync_late_arrival()
2065 * The caller should not log the write.
2068 * The caller should not log the write.
2085 dmu_buf_impl_t *db = (dmu_buf_impl_t *)zgd->zgd_db; in dmu_sync()
2086 objset_t *os = db->db_objset; in dmu_sync()
2087 dsl_dataset_t *ds = os->os_dsl_dataset; in dmu_sync()
2096 SET_BOOKMARK(&zb, ds->ds_object, in dmu_sync()
2097 db->db.db_object, db->db_level, db->db_blkid); in dmu_sync()
2100 dmu_write_policy(os, DB_DNODE(db), db->db_level, WP_DMU_SYNC, &zp); in dmu_sync()
2106 if (txg > spa_freeze_txg(os->os_spa)) in dmu_sync()
2115 mutex_enter(&db->db_mtx); in dmu_sync()
2117 if (txg <= spa_last_synced_txg(os->os_spa)) { in dmu_sync()
2121 mutex_exit(&db->db_mtx); in dmu_sync()
2125 if (txg <= spa_syncing_txg(os->os_spa)) { in dmu_sync()
2128 * the dirty record anymore; just write a new log block. in dmu_sync()
2130 mutex_exit(&db->db_mtx); in dmu_sync()
2141 mutex_exit(&db->db_mtx); in dmu_sync()
2145 dr_next = list_next(&db->db_dirty_records, dr); in dmu_sync()
2146 ASSERT(dr_next == NULL || dr_next->dr_txg < txg); in dmu_sync()
2148 if (db->db_blkptr != NULL) { in dmu_sync()
2160 *zgd->zgd_bp = *db->db_blkptr; in dmu_sync()
2164 * Assume the on-disk data is X, the current syncing data (in in dmu_sync()
2165 * txg - 1) is Y, and the current in-memory data is Z (currently in dmu_sync()
2170 * change before this write takes effect), then a nopwrite will in dmu_sync()
2171 * be incorrect - we would override with X, which could have in dmu_sync()
2174 * (Note that this is not a concern when we are nop-writing from in dmu_sync()
2180 * being dirty (dr_next is non-NULL), or by being freed in dmu_sync()
2183 * to the on-disk BP. in dmu_sync()
2189 if (dnode_block_freed(DB_DNODE(db), db->db_blkid)) in dmu_sync()
2194 ASSERT(dr->dr_txg == txg); in dmu_sync()
2195 if (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC || in dmu_sync()
2196 dr->dt.dl.dr_override_state == DR_OVERRIDDEN) { in dmu_sync()
2198 * We have already issued a sync write for this buffer, in dmu_sync()
2202 mutex_exit(&db->db_mtx); in dmu_sync()
2206 ASSERT0(dr->dt.dl.dr_has_raw_params); in dmu_sync()
2207 ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN); in dmu_sync()
2208 dr->dt.dl.dr_override_state = DR_IN_DMU_SYNC; in dmu_sync()
2209 mutex_exit(&db->db_mtx); in dmu_sync()
2212 dsa->dsa_dr = dr; in dmu_sync()
2213 dsa->dsa_done = done; in dmu_sync()
2214 dsa->dsa_zgd = zgd; in dmu_sync()
2215 dsa->dsa_tx = NULL; in dmu_sync()
2217 zio_nowait(arc_write(pio, os->os_spa, txg, zgd->zgd_bp, in dmu_sync()
2218 dr->dt.dl.dr_data, !DBUF_IS_CACHEABLE(db), in dmu_sync()
2265 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); in dmu_object_set_maxblkid()
2267 rw_exit(&dn->dn_struct_rwlock); in dmu_object_set_maxblkid()
2287 dn->dn_checksum = checksum; in dmu_object_set_checksum()
2306 dn->dn_compress = compress; in dmu_object_set_compress()
2320 dmu_object_type_t type = dn ? dn->dn_type : DMU_OT_OBJSET; in dmu_write_policy()
2323 enum zio_checksum checksum = os->os_checksum; in dmu_write_policy()
2324 enum zio_compress compress = os->os_compress; in dmu_write_policy()
2325 uint8_t complevel = os->os_complevel; in dmu_write_policy()
2326 enum zio_checksum dedup_checksum = os->os_dedup_checksum; in dmu_write_policy()
2329 boolean_t dedup_verify = os->os_dedup_verify; in dmu_write_policy()
2331 int copies = os->os_copies; in dmu_write_policy()
2332 int gang_copies = os->os_copies; in dmu_write_policy()
2335 * We maintain different write policies for each of the following in dmu_write_policy()
2337 * 1. metadata in dmu_write_policy()
2338 * 2. preallocated blocks (i.e. level-0 blocks of a dump device) in dmu_write_policy()
2343 * XXX -- we should design a compression algorithm in dmu_write_policy()
2344 * that specializes in arrays of bps. in dmu_write_policy()
2346 compress = zio_compress_select(os->os_spa, in dmu_write_policy()
2351 * checksum is multi-bit correctable, and it's not a in dmu_write_policy()
2352 * ZBT-style checksum, then it's suitable for metadata in dmu_write_policy()
2362 switch (os->os_redundant_metadata) { in dmu_write_policy()
2371 if (level + 1 >= in dmu_write_policy()
2390 * If this tunable is set, and this is a write for a in dmu_write_policy()
2399 dn ? dn->dn_storage_type : DMU_OT_NONE; in dmu_write_policy()
2422 compress = zio_compress_select(os->os_spa, dn->dn_compress, in dmu_write_policy()
2424 complevel = zio_complevel_select(os->os_spa, compress, in dmu_write_policy()
2437 zio_checksum_select(dn->dn_checksum, checksum) : in dmu_write_policy()
2466 if (os->os_redundant_metadata == ZFS_REDUNDANT_METADATA_ALL || in dmu_write_policy()
2467 (os->os_redundant_metadata == in dmu_write_policy()
2469 zfs_redundant_metadata_most_ditto_level <= 1)) in dmu_write_policy()
2482 if (os->os_encrypted && (wp & WP_NOFILL) == 0) { in dmu_write_policy()
2486 copies = MIN(copies, SPA_DVAS_PER_BP - 1); in dmu_write_policy()
2487 gang_copies = MIN(gang_copies, SPA_DVAS_PER_BP - 1); in dmu_write_policy()
2499 zp->zp_compress = compress; in dmu_write_policy()
2500 zp->zp_complevel = complevel; in dmu_write_policy()
2501 zp->zp_checksum = checksum; in dmu_write_policy()
2502 zp->zp_type = (wp & WP_SPILL) ? dn->dn_bonustype : type; in dmu_write_policy()
2503 zp->zp_level = level; in dmu_write_policy()
2504 zp->zp_copies = MIN(copies, spa_max_replication(os->os_spa)); in dmu_write_policy()
2505 zp->zp_gang_copies = MIN(MAX(gang_copies, copies), in dmu_write_policy()
2506 spa_max_replication(os->os_spa)); in dmu_write_policy()
2507 zp->zp_dedup = dedup; in dmu_write_policy()
2508 zp->zp_dedup_verify = dedup && dedup_verify; in dmu_write_policy()
2509 zp->zp_nopwrite = nopwrite; in dmu_write_policy()
2510 zp->zp_encrypt = encrypt; in dmu_write_policy()
2511 zp->zp_byteorder = ZFS_HOST_BYTEORDER; in dmu_write_policy()
2512 zp->zp_direct_write = (wp & WP_DIRECT_WR) ? B_TRUE : B_FALSE; in dmu_write_policy()
2513 zp->zp_rewrite = B_FALSE; in dmu_write_policy()
2514 memset(zp->zp_salt, 0, ZIO_DATA_SALT_LEN); in dmu_write_policy()
2515 memset(zp->zp_iv, 0, ZIO_DATA_IV_LEN); in dmu_write_policy()
2516 memset(zp->zp_mac, 0, ZIO_DATA_MAC_LEN); in dmu_write_policy()
2517 zp->zp_zpl_smallblk = (DMU_OT_IS_FILE(zp->zp_type) || in dmu_write_policy()
2518 zp->zp_type == DMU_OT_ZVOL) ? in dmu_write_policy()
2519 os->os_zpl_special_smallblock : 0; in dmu_write_policy()
2520 zp->zp_storage_type = dn ? dn->dn_storage_type : DMU_OT_NONE; in dmu_write_policy()
2522 ASSERT3U(zp->zp_compress, !=, ZIO_COMPRESS_INHERIT); in dmu_write_policy()
2545 rw_enter(&dn->dn_struct_rwlock, RW_READER); in dmu_offset_next()
2553 * Provided a RL_READER rangelock spanning 0-UINT64_MAX is in dmu_offset_next()
2560 rw_exit(&dn->dn_struct_rwlock); in dmu_offset_next()
2576 (hole ? DNODE_FIND_HOLE : 0), off, 1, 1, 0); in dmu_offset_next()
2579 rw_exit(&dn->dn_struct_rwlock); in dmu_offset_next()
2587 blkptr_t *bps, size_t *nbpsp) in dmu_read_l0_bps() argument
2609 mutex_enter(&db->db_mtx); in dmu_read_l0_bps()
2611 if (!list_is_empty(&db->db_dirty_records)) { in dmu_read_l0_bps()
2614 dr = list_head(&db->db_dirty_records); in dmu_read_l0_bps()
2615 if (dr->dt.dl.dr_brtwrite) { in dmu_read_l0_bps()
2621 bp = &dr->dt.dl.dr_overridden_by; in dmu_read_l0_bps()
2627 mutex_exit(&db->db_mtx); in dmu_read_l0_bps()
2632 bp = db->db_blkptr; in dmu_read_l0_bps()
2635 mutex_exit(&db->db_mtx); in dmu_read_l0_bps()
2644 BP_ZERO(&bps[i]); in dmu_read_l0_bps()
2657 * yet synced, we could clone it, but we couldn't write this in dmu_read_l0_bps()
2661 if (BP_GET_PHYSICAL_BIRTH(bp) > spa_freeze_txg(os->os_spa)) { in dmu_read_l0_bps()
2666 spa_last_synced_txg(os->os_spa)) { in dmu_read_l0_bps()
2671 bps[i] = *bp; in dmu_read_l0_bps()
2683 dmu_tx_t *tx, const blkptr_t *bps, size_t nbps) in dmu_brt_clone() argument
2693 spa = os->os_spa; in dmu_brt_clone()
2700 * Before we start cloning make sure that the dbufs sizes match new BPs in dmu_brt_clone()
2701 * sizes. If they don't, that's a no-go, as we are not able to shrink in dmu_brt_clone()
2707 bp = &bps[i]; in dmu_brt_clone()
2709 ASSERT3U(db->db.db_object, !=, DMU_META_DNODE_OBJECT); in dmu_brt_clone()
2710 ASSERT0(db->db_level); in dmu_brt_clone()
2711 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dmu_brt_clone()
2712 ASSERT(db->db_blkid != DMU_SPILL_BLKID); in dmu_brt_clone()
2714 if (!BP_IS_HOLE(bp) && BP_GET_LSIZE(bp) != dbuf->db_size) { in dmu_brt_clone()
2723 bp = &bps[i]; in dmu_brt_clone()
2727 mutex_enter(&db->db_mtx); in dmu_brt_clone()
2729 dr = list_head(&db->db_dirty_records); in dmu_brt_clone()
2731 ASSERT3U(dr->dr_txg, ==, tx->tx_txg); in dmu_brt_clone()
2732 dl = &dr->dt.dl; in dmu_brt_clone()
2733 ASSERT0(dl->dr_has_raw_params); in dmu_brt_clone()
2734 dl->dr_overridden_by = *bp; in dmu_brt_clone()
2737 BP_SET_BIRTH(&dl->dr_overridden_by, dr->dr_txg, in dmu_brt_clone()
2739 BP_SET_REWRITE(&dl->dr_overridden_by, 0); in dmu_brt_clone()
2741 BP_SET_LOGICAL_BIRTH(&dl->dr_overridden_by, in dmu_brt_clone()
2742 dr->dr_txg); in dmu_brt_clone()
2745 dl->dr_brtwrite = B_TRUE; in dmu_brt_clone()
2746 dl->dr_override_state = DR_OVERRIDDEN; in dmu_brt_clone()
2748 mutex_exit(&db->db_mtx); in dmu_brt_clone()
2768 dnode_phys_t *dnp = dn->dn_phys; in __dmu_object_info_from_dnode()
2770 doi->doi_data_block_size = dn->dn_datablksz; in __dmu_object_info_from_dnode()
2771 doi->doi_metadata_block_size = dn->dn_indblkshift ? in __dmu_object_info_from_dnode()
2772 1ULL << dn->dn_indblkshift : 0; in __dmu_object_info_from_dnode()
2773 doi->doi_type = dn->dn_type; in __dmu_object_info_from_dnode()
2774 doi->doi_bonus_type = dn->dn_bonustype; in __dmu_object_info_from_dnode()
2775 doi->doi_bonus_size = dn->dn_bonuslen; in __dmu_object_info_from_dnode()
2776 doi->doi_dnodesize = dn->dn_num_slots << DNODE_SHIFT; in __dmu_object_info_from_dnode()
2777 doi->doi_indirection = dn->dn_nlevels; in __dmu_object_info_from_dnode()
2778 doi->doi_checksum = dn->dn_checksum; in __dmu_object_info_from_dnode()
2779 doi->doi_compress = dn->dn_compress; in __dmu_object_info_from_dnode()
2780 doi->doi_nblkptr = dn->dn_nblkptr; in __dmu_object_info_from_dnode()
2781 doi->doi_physical_blocks_512 = (DN_USED_BYTES(dnp) + 256) >> 9; in __dmu_object_info_from_dnode()
2782 doi->doi_max_offset = (dn->dn_maxblkid + 1) * dn->dn_datablksz; in __dmu_object_info_from_dnode()
2783 doi->doi_fill_count = 0; in __dmu_object_info_from_dnode()
2784 for (int i = 0; i < dnp->dn_nblkptr; i++) in __dmu_object_info_from_dnode()
2785 doi->doi_fill_count += BP_GET_FILL(&dnp->dn_blkptr[i]); in __dmu_object_info_from_dnode()
2791 rw_enter(&dn->dn_struct_rwlock, RW_READER); in dmu_object_info_from_dnode()
2792 mutex_enter(&dn->dn_mtx); in dmu_object_info_from_dnode()
2796 mutex_exit(&dn->dn_mtx); in dmu_object_info_from_dnode()
2797 rw_exit(&dn->dn_struct_rwlock); in dmu_object_info_from_dnode()
2846 *blksize = dn->dn_datablksz; in dmu_object_size_from_db()
2848 *nblk512 = ((DN_USED_BYTES(dn->dn_phys) + SPA_MINBLOCKSIZE/2) >> in dmu_object_size_from_db()
2849 SPA_MINBLOCKSHIFT) + dn->dn_num_slots; in dmu_object_size_from_db()
2859 *dnsize = DB_DNODE(db)->dn_num_slots << DNODE_SHIFT; in dmu_object_dnsize_from_db()
2893 size_t count = size >> 1; in byteswap_uint16_array()
2896 ASSERT0((size & 1)); in byteswap_uint16_array()