Lines Matching +full:two +full:- +full:word
9 * or https://opensource.org/licenses/CDDL-1.0.
44 * DMU to keep more data in-core, and also to waste more I/O bandwidth
49 * Enabled whenever we want to stress test the use of double-word
58 * or two indirect blocks (16K-32K, rather than 128K).
82 * Iterate through the space map, invoking the callback on each (non-debug)
88 uint64_t blksz = sm->sm_blksz; in space_map_iterate()
94 dmu_prefetch(sm->sm_os, space_map_object(sm), 0, 0, end, in space_map_iterate()
102 error = dmu_buf_hold(sm->sm_os, space_map_object(sm), in space_map_iterate()
107 uint64_t *block_start = db->db_data; in space_map_iterate()
108 uint64_t block_length = MIN(end - block_base, blksz); in space_map_iterate()
114 ASSERT3U(blksz, ==, db->db_size); in space_map_iterate()
127 * at the end of space map blocks in-order in space_map_iterate()
128 * to not split a double-word entry in the in space_map_iterate()
129 * middle between two blocks. These entries in space_map_iterate()
152 /* it is a two-word entry */ in space_map_iterate()
157 /* move on to the second word */ in space_map_iterate()
166 uint64_t entry_offset = (raw_offset << sm->sm_shift) + in space_map_iterate()
167 sm->sm_start; in space_map_iterate()
168 uint64_t entry_run = raw_run << sm->sm_shift; in space_map_iterate()
170 VERIFY0(P2PHASE(entry_offset, 1ULL << sm->sm_shift)); in space_map_iterate()
171 VERIFY0(P2PHASE(entry_run, 1ULL << sm->sm_shift)); in space_map_iterate()
172 ASSERT3U(entry_offset, >=, sm->sm_start); in space_map_iterate()
173 ASSERT3U(entry_offset, <, sm->sm_start + sm->sm_size); in space_map_iterate()
174 ASSERT3U(entry_run, <=, sm->sm_size); in space_map_iterate()
176 sm->sm_start + sm->sm_size); in space_map_iterate()
209 * Find the offset of the last word in the space map and use in space_map_reversed_last_block_entries()
214 sm->sm_phys->smp_length - sizeof (uint64_t); in space_map_reversed_last_block_entries()
215 error = dmu_buf_hold(sm->sm_os, space_map_object(sm), last_word_offset, in space_map_reversed_last_block_entries()
220 ASSERT3U(sm->sm_object, ==, db->db_object); in space_map_reversed_last_block_entries()
221 ASSERT3U(sm->sm_blksz, ==, db->db_size); in space_map_reversed_last_block_entries()
222 ASSERT3U(bufsz, >=, db->db_size); in space_map_reversed_last_block_entries()
225 uint64_t *words = db->db_data; in space_map_reversed_last_block_entries()
227 (sm->sm_phys->smp_length - db->db_offset) / sizeof (uint64_t); in space_map_reversed_last_block_entries()
232 uint64_t j = n - 1; in space_map_reversed_last_block_entries()
238 * we have to be extra careful and add the two in space_map_reversed_last_block_entries()
239 * words of the double-word entry in the right in space_map_reversed_last_block_entries()
243 buf[j - 1] = entry; in space_map_reversed_last_block_entries()
249 j -= 2; in space_map_reversed_last_block_entries()
254 j--; in space_map_reversed_last_block_entries()
262 ASSERT3S(j, ==, -1); in space_map_reversed_last_block_entries()
269 * Note: This function performs destructive actions - specifically
278 uint64_t bufsz = MAX(sm->sm_blksz, SPA_MINBLOCKSIZE); in space_map_incremental_destroy()
281 dmu_buf_will_dirty(sm->sm_dbuf, tx); in space_map_incremental_destroy()
286 * approach is that we don't have any field on-disk that points in space_map_incremental_destroy()
296 * can't distinguish two-word space map entries from their second in space_map_incremental_destroy()
297 * word. Thus we do the following: in space_map_incremental_destroy()
326 sm->sm_phys->smp_length -= sizeof (uint64_t); in space_map_incremental_destroy()
345 /* move to the second word */ in space_map_incremental_destroy()
356 (raw_offset << sm->sm_shift) + sm->sm_start; in space_map_incremental_destroy()
357 uint64_t entry_run = raw_run << sm->sm_shift; in space_map_incremental_destroy()
359 VERIFY0(P2PHASE(entry_offset, 1ULL << sm->sm_shift)); in space_map_incremental_destroy()
360 VERIFY0(P2PHASE(entry_run, 1ULL << sm->sm_shift)); in space_map_incremental_destroy()
361 VERIFY3U(entry_offset, >=, sm->sm_start); in space_map_incremental_destroy()
362 VERIFY3U(entry_offset, <, sm->sm_start + sm->sm_size); in space_map_incremental_destroy()
363 VERIFY3U(entry_run, <=, sm->sm_size); in space_map_incremental_destroy()
365 sm->sm_start + sm->sm_size); in space_map_incremental_destroy()
378 sm->sm_phys->smp_alloc -= entry_run; in space_map_incremental_destroy()
380 sm->sm_phys->smp_alloc += entry_run; in space_map_incremental_destroy()
381 sm->sm_phys->smp_length -= words * sizeof (uint64_t); in space_map_incremental_destroy()
404 if (sme->sme_type == smla->smla_type) { in space_map_load_callback()
405 VERIFY3U(zfs_range_tree_space(smla->smla_rt) + sme->sme_run, <=, in space_map_load_callback()
406 smla->smla_sm->sm_size); in space_map_load_callback()
407 zfs_range_tree_add(smla->smla_rt, sme->sme_offset, in space_map_load_callback()
408 sme->sme_run); in space_map_load_callback()
410 zfs_range_tree_remove(smla->smla_rt, sme->sme_offset, in space_map_load_callback()
411 sme->sme_run); in space_map_load_callback()
430 zfs_range_tree_add(rt, sm->sm_start, sm->sm_size); in space_map_load_length()
457 if (sm->sm_dbuf->db_size != sizeof (space_map_phys_t)) in space_map_histogram_clear()
460 memset(sm->sm_phys->smp_histogram, 0, in space_map_histogram_clear()
461 sizeof (sm->sm_phys->smp_histogram)); in space_map_histogram_clear()
468 * Verify that the in-core range tree does not have any in space_map_histogram_verify()
471 for (int i = 0; i < sm->sm_shift; i++) { in space_map_histogram_verify()
472 if (rt->rt_histogram[i] != 0) in space_map_histogram_verify()
486 if (sm->sm_dbuf->db_size != sizeof (space_map_phys_t)) in space_map_histogram_add()
489 dmu_buf_will_dirty(sm->sm_dbuf, tx); in space_map_histogram_add()
495 * between 2^sm_shift to 2^(32+sm_shift-1). The range tree, in space_map_histogram_add()
500 for (int i = sm->sm_shift; i < ZFS_RANGE_TREE_HISTOGRAM_SIZE; i++) { in space_map_histogram_add()
504 * 2^(32+sm_shift-1), we need to normalize the values in in space_map_histogram_add()
511 ASSERT3U(i, >=, idx + sm->sm_shift); in space_map_histogram_add()
512 sm->sm_phys->smp_histogram[idx] += in space_map_histogram_add()
513 rt->rt_histogram[i] << (i - idx - sm->sm_shift); in space_map_histogram_add()
520 if (idx < SPACE_MAP_HISTOGRAM_SIZE - 1) { in space_map_histogram_add()
521 ASSERT3U(idx + sm->sm_shift, ==, i); in space_map_histogram_add()
531 dmu_buf_will_dirty(sm->sm_dbuf, tx); in space_map_write_intro_debug()
535 SM_DEBUG_SYNCPASS_ENCODE(spa_sync_pass(tx->tx_pool->dp_spa)) | in space_map_write_intro_debug()
538 dmu_write(sm->sm_os, space_map_object(sm), sm->sm_phys->smp_length, in space_map_write_intro_debug()
541 sm->sm_phys->smp_length += sizeof (dentry); in space_map_write_intro_debug()
563 * if this is a single word entry, ensure that no vdev was in space_map_write_seg()
569 ASSERT3U(db->db_size, ==, sm->sm_blksz); in space_map_write_seg()
571 uint64_t *block_base = db->db_data; in space_map_write_seg()
572 uint64_t *block_end = block_base + (sm->sm_blksz / sizeof (uint64_t)); in space_map_write_seg()
574 (sm->sm_phys->smp_length - db->db_offset) / sizeof (uint64_t); in space_map_write_seg()
578 uint64_t size = (rend - rstart) >> sm->sm_shift; in space_map_write_seg()
579 uint64_t start = (rstart - sm->sm_start) >> sm->sm_shift; in space_map_write_seg()
582 ASSERT3U(rstart, >=, sm->sm_start); in space_map_write_seg()
583 ASSERT3U(rstart, <, sm->sm_start + sm->sm_size); in space_map_write_seg()
584 ASSERT3U(rend - rstart, <=, sm->sm_size); in space_map_write_seg()
585 ASSERT3U(rend, <=, sm->sm_start + sm->sm_size); in space_map_write_seg()
597 uint64_t next_word_offset = sm->sm_phys->smp_length; in space_map_write_seg()
598 VERIFY0(dmu_buf_hold(sm->sm_os, in space_map_write_seg()
606 ASSERT3U(db->db_size, ==, sm->sm_blksz); in space_map_write_seg()
608 block_base = db->db_data; in space_map_write_seg()
611 (db->db_size / sizeof (uint64_t)); in space_map_write_seg()
615 * If we are writing a two-word entry and we only have one in space_map_write_seg()
616 * word left on this block, just pad it with an empty debug in space_map_write_seg()
617 * entry and write the two-word entry in the next block. in space_map_write_seg()
627 sm->sm_phys->smp_length += sizeof (uint64_t); in space_map_write_seg()
641 /* write the first word of the entry */ in space_map_write_seg()
647 /* move on to the second word of the entry */ in space_map_write_seg()
654 panic("%d-word space map entries are not supported", in space_map_write_seg()
658 sm->sm_phys->smp_length += words * sizeof (uint64_t); in space_map_write_seg()
661 size -= run_len; in space_map_write_seg()
675 spa_t *spa = tx->tx_pool->dp_spa; in space_map_write_impl()
685 uint64_t initial_objsize = sm->sm_phys->smp_length; in space_map_write_impl()
692 * Find the offset right after the last word in the space map in space_map_write_impl()
696 uint64_t next_word_offset = sm->sm_phys->smp_length; in space_map_write_impl()
697 VERIFY0(dmu_buf_hold(sm->sm_os, space_map_object(sm), in space_map_write_impl()
699 ASSERT3U(db->db_size, ==, sm->sm_blksz); in space_map_write_impl()
703 zfs_btree_t *t = &rt->rt_root; in space_map_write_impl()
707 uint64_t offset = (zfs_rs_get_start(rs, rt) - sm->sm_start) >> in space_map_write_impl()
708 sm->sm_shift; in space_map_write_impl()
709 uint64_t length = (zfs_rs_get_end(rs, rt) - in space_map_write_impl()
710 zfs_rs_get_start(rs, rt)) >> sm->sm_shift; in space_map_write_impl()
714 * We only write two-word entries when both of the following in space_map_write_impl()
718 * [2] The offset or run is too big for a single-word entry, in space_map_write_impl()
723 * we write two-word entries occasionally when the feature is in space_map_write_impl()
749 ASSERT3U(estimated_final_objsize, >=, sm->sm_phys->smp_length); in space_map_write_impl()
762 ASSERT(dsl_pool_sync_context(dmu_objset_pool(sm->sm_os))); in space_map_write()
765 dmu_buf_will_dirty(sm->sm_dbuf, tx); in space_map_write()
768 * This field is no longer necessary since the in-core space map in space_map_write()
772 sm->sm_phys->smp_object = sm->sm_object; in space_map_write()
775 VERIFY3U(sm->sm_object, ==, sm->sm_phys->smp_object); in space_map_write()
780 sm->sm_phys->smp_alloc += zfs_range_tree_space(rt); in space_map_write()
782 sm->sm_phys->smp_alloc -= zfs_range_tree_space(rt); in space_map_write()
784 uint64_t nodes = zfs_btree_numnodes(&rt->rt_root); in space_map_write()
793 VERIFY3U(nodes, ==, zfs_btree_numnodes(&rt->rt_root)); in space_map_write()
803 error = dmu_bonus_hold(sm->sm_os, sm->sm_object, sm, &sm->sm_dbuf); in space_map_open_impl()
807 dmu_object_size_from_db(sm->sm_dbuf, &sm->sm_blksz, &blocks); in space_map_open_impl()
808 sm->sm_phys = sm->sm_dbuf->db_data; in space_map_open_impl()
825 sm->sm_start = start; in space_map_open()
826 sm->sm_size = size; in space_map_open()
827 sm->sm_shift = shift; in space_map_open()
828 sm->sm_os = os; in space_map_open()
829 sm->sm_object = object; in space_map_open()
830 sm->sm_blksz = 0; in space_map_open()
831 sm->sm_dbuf = NULL; in space_map_open()
832 sm->sm_phys = NULL; in space_map_open()
850 if (sm->sm_dbuf != NULL) in space_map_close()
851 dmu_buf_rele(sm->sm_dbuf, sm); in space_map_close()
852 sm->sm_dbuf = NULL; in space_map_close()
853 sm->sm_phys = NULL; in space_map_close()
861 objset_t *os = sm->sm_os; in space_map_truncate()
869 dmu_object_info_from_db(sm->sm_dbuf, &doi); in space_map_truncate()
875 * free and re-allocate its object with the updated sizes. in space_map_truncate()
886 (u_longlong_t)sm->sm_object, in space_map_truncate()
891 dmu_buf_rele(sm->sm_dbuf, sm); in space_map_truncate()
893 sm->sm_object = space_map_alloc(sm->sm_os, blocksize, tx); in space_map_truncate()
896 VERIFY0(dmu_free_range(os, space_map_object(sm), 0, -1ULL, tx)); in space_map_truncate()
903 memset(sm->sm_phys->smp_histogram, 0, in space_map_truncate()
904 sizeof (sm->sm_phys->smp_histogram)); in space_map_truncate()
907 dmu_buf_will_dirty(sm->sm_dbuf, tx); in space_map_truncate()
908 sm->sm_phys->smp_length = 0; in space_map_truncate()
909 sm->sm_phys->smp_alloc = 0; in space_map_truncate()
956 space_map_free_obj(sm->sm_os, space_map_object(sm), tx); in space_map_free()
957 sm->sm_object = 0; in space_map_free()
961 * Given a range tree, it makes a worst-case estimate of how much
969 spa_t *spa = dmu_objset_spa(sm->sm_os); in space_map_estimate_optimal_size()
970 uint64_t shift = sm->sm_shift; in space_map_estimate_optimal_size()
971 uint64_t *histogram = rt->rt_histogram; in space_map_estimate_optimal_size()
976 * range tree would have on-disk as a space map, we iterate through in space_map_estimate_optimal_size()
979 * Note that this is a highest-bound/worst-case estimate for the in space_map_estimate_optimal_size()
983 * we write and we also assume that we start at the last word in space_map_estimate_optimal_size()
984 * of a block attempting to write a two-word entry. in space_map_estimate_optimal_size()
993 * of entries in [2^i, (2^(i+1))-1] of that range_tree. Given in space_map_estimate_optimal_size()
995 * can be represented using a single-word entry, ones that can in space_map_estimate_optimal_size()
996 * be represented with a double-word entry, and ones that can in space_map_estimate_optimal_size()
997 * only be represented with multiple two-word entries. in space_map_estimate_optimal_size()
1000 * are only two groups: single-word entry buckets and multiple in space_map_estimate_optimal_size()
1001 * single-word entry buckets. The information below assumes in space_map_estimate_optimal_size()
1002 * two-word entries enabled, but it can easily applied when in space_map_estimate_optimal_size()
1006 * single-word entry we look at the maximum run that such entry in space_map_estimate_optimal_size()
1010 * maximum run that can be represented by a single-word entry, in space_map_estimate_optimal_size()
1012 * SM_RUN_BITS + shift - 1. in space_map_estimate_optimal_size()
1015 * double-word entry, we follow the same approach. Finally, any in space_map_estimate_optimal_size()
1016 * bucket higher than that are represented with multiple two-word in space_map_estimate_optimal_size()
1018 * segments can be represented with a single two-word entry is X, in space_map_estimate_optimal_size()
1019 * then bucket X+1 will need 2 two-word entries for each of its in space_map_estimate_optimal_size()
1024 * the example with the one-word entry, the maximum run that can in space_map_estimate_optimal_size()
1025 * be represented in a one-word entry 2^(SM_RUN_BITS + shift) is in space_map_estimate_optimal_size()
1026 * not part of bucket SM_RUN_BITS + shift - 1. Thus, segments of in space_map_estimate_optimal_size()
1028 * we start counting two-word entries and this is one more reason in space_map_estimate_optimal_size()
1036 (vdev_id == SM_NO_VDEVID && sm->sm_size < SM_OFFSET_MAX)) { in space_map_estimate_optimal_size()
1039 * If we are trying to force some double word entries just in space_map_estimate_optimal_size()
1040 * assume the worst-case of every single word entry being in space_map_estimate_optimal_size()
1041 * written as a double word entry. in space_map_estimate_optimal_size()
1048 uint64_t single_entry_max_bucket = SM_RUN_BITS + shift - 1; in space_map_estimate_optimal_size()
1056 1ULL << (idx - single_entry_max_bucket); in space_map_estimate_optimal_size()
1066 uint64_t double_entry_max_bucket = SM2_RUN_BITS + shift - 1; in space_map_estimate_optimal_size()
1072 entries_for_seg = 1ULL << (idx - double_entry_max_bucket); in space_map_estimate_optimal_size()
1082 size += ((size / sm->sm_blksz) + 1) * sizeof (uint64_t); in space_map_estimate_optimal_size()
1090 return (sm != NULL ? sm->sm_object : 0); in space_map_object()
1096 return (sm != NULL ? sm->sm_phys->smp_alloc : 0); in space_map_allocated()
1102 return (sm != NULL ? sm->sm_phys->smp_length : 0); in space_map_length()
1110 return (DIV_ROUND_UP(space_map_length(sm), sm->sm_blksz)); in space_map_nblocks()