1 // SPDX-License-Identifier: GPL-2.0-only
5 #include "volume-index.h"
15 #include "memory-alloc.h"
18 #include "thread-utils.h"
22 #include "hash-utils.h"
28  * dense chapters). If there are no sparse chapters, only the non-hook sub index is used, and it
42  * or after all requests have been flushed in order to shut down. The only multi-threaded operation
50  * high-order bits. When the index needs to deal with ordering of index chapter numbers, as when
87 	/* The non-hook subindex configuration */
99 static const char MAGIC_START_5[] = "MI5-0005";
110 static const char MAGIC_START_6[] = "MI6-0001";
120 	return uds_extract_volume_index_bytes(name) & sub_index->address_mask;  in extract_address()
128 	return (bits >> sub_index->address_bits) % sub_index->list_count;  in extract_dlist_num()
134 	return &record->sub_index->zones[record->zone_number];  in get_zone_for_record()
141 	u32 rolling_chapter = ((index_chapter - volume_index_zone->virtual_chapter_low) &  in convert_index_to_virtual()
142 			       record->sub_index->chapter_mask);  in convert_index_to_virtual()
144 	return volume_index_zone->virtual_chapter_low + rolling_chapter;  in convert_index_to_virtual()
150 	return virtual_chapter & sub_index->chapter_mask;  in convert_virtual_to_index()
158 	return ((virtual_chapter >= volume_index_zone->virtual_chapter_low) &&  in is_virtual_chapter_indexed()
159 		(virtual_chapter <= volume_index_zone->virtual_chapter_high));  in is_virtual_chapter_indexed()
164 	return volume_index->sparse_sample_rate > 0;  in has_sparse()
173 	return (uds_extract_sampling_bytes(name) % volume_index->sparse_sample_rate) == 0;  in uds_is_volume_index_sample()
181 		&volume_index->vi_hook :  in get_volume_sub_index()
182 		&volume_index->vi_non_hook);  in get_volume_sub_index()
188 	return extract_dlist_num(sub_index, name) / sub_index->delta_index.lists_per_zone;  in get_volume_sub_index_zone()
210 	struct index_geometry *geometry = config->geometry;  in compute_volume_sub_index_parameters()
211 	u64 records_per_chapter = geometry->records_per_chapter;  in compute_volume_sub_index_parameters()
213 	params->chapter_count = geometry->chapters_per_volume;  in compute_volume_sub_index_parameters()
219 	rounded_chapters = params->chapter_count;  in compute_volume_sub_index_parameters()
223 	address_count = config->volume_index_mean_delta * DELTA_LIST_SIZE;  in compute_volume_sub_index_parameters()
224 	params->list_count = max(delta_list_records / DELTA_LIST_SIZE, min_delta_lists);  in compute_volume_sub_index_parameters()
225 	params->address_bits = bits_per(address_count - 1);  in compute_volume_sub_index_parameters()
226 	params->chapter_bits = bits_per(rounded_chapters - 1);  in compute_volume_sub_index_parameters()
227 	if ((u32) params->list_count != params->list_count) {  in compute_volume_sub_index_parameters()
230 						(unsigned long long) params->list_count);  in compute_volume_sub_index_parameters()
233 	if (params->address_bits > 31) {  in compute_volume_sub_index_parameters()
236 						params->address_bits);  in compute_volume_sub_index_parameters()
243 	 * double p_not_touched = pow((double) (params->list_count - 1) / params->list_count,  in compute_volume_sub_index_parameters()
251 	 * double invalid_chapters = p_not_touched / (1.0 - p_not_touched);  in compute_volume_sub_index_parameters()
261 	address_span = params->list_count << params->address_bits;  in compute_volume_sub_index_parameters()
262 	params->mean_delta = address_span / entries_in_volume_index;  in compute_volume_sub_index_parameters()
269 	params->chapter_size_in_bits = uds_compute_delta_index_size(records_per_chapter,  in compute_volume_sub_index_parameters()
270 								    params->mean_delta,  in compute_volume_sub_index_parameters()
271 								    params->chapter_bits);  in compute_volume_sub_index_parameters()
272 	index_size_in_bits = params->chapter_size_in_bits * chapters_in_volume_index;  in compute_volume_sub_index_parameters()
274 	params->memory_size = expected_index_size * 106 / 100;  in compute_volume_sub_index_parameters()
276 	params->target_free_bytes = expected_index_size / 20;  in compute_volume_sub_index_parameters()
282 	vdo_free(vdo_forget(sub_index->flush_chapters));  in uninitialize_volume_sub_index()
283 	vdo_free(vdo_forget(sub_index->zones));  in uninitialize_volume_sub_index()
284 	uds_uninitialize_delta_index(&sub_index->delta_index);  in uninitialize_volume_sub_index()
292 	if (volume_index->zones != NULL)  in uds_free_volume_index()
293 		vdo_free(vdo_forget(volume_index->zones));  in uds_free_volume_index()
295 	uninitialize_volume_sub_index(&volume_index->vi_non_hook);  in uds_free_volume_index()
296 	uninitialize_volume_sub_index(&volume_index->vi_hook);  in uds_free_volume_index()
325 	split->hook_config = *config;  in split_configuration()
326 	split->hook_geometry = *config->geometry;  in split_configuration()
327 	split->hook_config.geometry = &split->hook_geometry;  in split_configuration()
328 	split->non_hook_config = *config;  in split_configuration()
329 	split->non_hook_geometry = *config->geometry;  in split_configuration()
330 	split->non_hook_config.geometry = &split->non_hook_geometry;  in split_configuration()
332 	sample_rate = config->sparse_sample_rate;  in split_configuration()
333 	sparse_chapters = config->geometry->sparse_chapters_per_volume;  in split_configuration()
334 	dense_chapters = config->geometry->chapters_per_volume - sparse_chapters;  in split_configuration()
335 	sample_records = config->geometry->records_per_chapter / sample_rate;  in split_configuration()
338 	split->hook_geometry.records_per_chapter = sample_records;  in split_configuration()
339 	split->non_hook_geometry.records_per_chapter -= sample_records;  in split_configuration()
342 	split->hook_geometry.sparse_chapters_per_volume = 0;  in split_configuration()
343 	split->non_hook_geometry.sparse_chapters_per_volume = 0;  in split_configuration()
344 	split->non_hook_geometry.chapters_per_volume = dense_chapters;  in split_configuration()
354 	if (!uds_is_sparse_index_geometry(config->geometry))  in compute_volume_index_save_bytes()
393 	result = uds_next_delta_index_entry(&record->delta_entry);  in flush_invalid_entries()
397 	while (!record->delta_entry.at_end) {  in flush_invalid_entries()
398 		u32 index_chapter = uds_get_delta_entry_value(&record->delta_entry);  in flush_invalid_entries()
399 		u32 relative_chapter = ((index_chapter - flush_range->chapter_start) &  in flush_invalid_entries()
400 					record->sub_index->chapter_mask);  in flush_invalid_entries()
402 		if (likely(relative_chapter >= flush_range->chapter_count)) {  in flush_invalid_entries()
408 		result = uds_remove_delta_index_entry(&record->delta_entry);  in flush_invalid_entries()
421 	const struct volume_sub_index *sub_index = record->sub_index;  in get_volume_index_entry()
422 	u32 next_chapter_to_invalidate = sub_index->chapter_mask;  in get_volume_index_entry()
425 	result = uds_start_delta_index_search(&sub_index->delta_index, list_number, 0,  in get_volume_index_entry()
426 					      &record->delta_entry);  in get_volume_index_entry()
435 	} while (!record->delta_entry.at_end && (key > record->delta_entry.key));  in get_volume_index_entry()
437 	result = uds_remember_delta_index_offset(&record->delta_entry);  in get_volume_index_entry()
461 			if (memcmp(collision_name, record->name, UDS_RECORD_NAME_SIZE) == 0) {  in get_volume_index_entry()
473 	next_chapter_to_invalidate += flush_range->chapter_start;  in get_volume_index_entry()
474 	next_chapter_to_invalidate &= sub_index->chapter_mask;  in get_volume_index_entry()
475 	flush_range->chapter_start = next_chapter_to_invalidate;  in get_volume_index_entry()
476 	flush_range->chapter_count = 0;  in get_volume_index_entry()
488 	u64 flush_chapter = sub_index->flush_chapters[delta_list_number];  in get_volume_sub_index_record()
490 	record->sub_index = sub_index;  in get_volume_sub_index_record()
491 	record->mutex = NULL;  in get_volume_sub_index_record()
492 	record->name = name;  in get_volume_sub_index_record()
493 	record->zone_number = delta_list_number / sub_index->delta_index.lists_per_zone;  in get_volume_sub_index_record()
496 	if (flush_chapter < volume_index_zone->virtual_chapter_low) {  in get_volume_sub_index_record()
498 		u64 flush_count = volume_index_zone->virtual_chapter_low - flush_chapter;  in get_volume_sub_index_record()
501 		range.chapter_count = (flush_count > sub_index->chapter_mask ?  in get_volume_sub_index_record()
502 				       sub_index->chapter_mask + 1 :  in get_volume_sub_index_record()
507 		if (flush_chapter > volume_index_zone->virtual_chapter_high)  in get_volume_sub_index_record()
508 			flush_chapter = volume_index_zone->virtual_chapter_high;  in get_volume_sub_index_record()
509 		sub_index->flush_chapters[delta_list_number] = flush_chapter;  in get_volume_sub_index_record()
511 		result = uds_get_delta_index_entry(&sub_index->delta_index,  in get_volume_sub_index_record()
513 						   name->name, &record->delta_entry);  in get_volume_sub_index_record()
519 	record->is_found =  in get_volume_sub_index_record()
520 		(!record->delta_entry.at_end && (record->delta_entry.key == address));  in get_volume_sub_index_record()
521 	if (record->is_found) {  in get_volume_sub_index_record()
522 		u32 index_chapter = uds_get_delta_entry_value(&record->delta_entry);  in get_volume_sub_index_record()
524 		record->virtual_chapter = convert_index_to_virtual(record, index_chapter);  in get_volume_sub_index_record()
527 	record->is_collision = record->delta_entry.is_collision;  in get_volume_sub_index_record()
541 		 * the volume index, uds_get_volume_index_record() is not a read-only operation.  in uds_get_volume_index_record()
544 			get_volume_sub_index_zone(&volume_index->vi_hook, name);  in uds_get_volume_index_record()
545 		struct mutex *mutex = &volume_index->zones[zone].hook_mutex;  in uds_get_volume_index_record()
548 		result = get_volume_sub_index_record(&volume_index->vi_hook, name,  in uds_get_volume_index_record()
552 		record->mutex = mutex;  in uds_get_volume_index_record()
554 		result = get_volume_sub_index_record(&volume_index->vi_non_hook, name,  in uds_get_volume_index_record()
565 	const struct volume_sub_index *sub_index = record->sub_index;  in uds_put_volume_index_record()
568 		u64 low = get_zone_for_record(record)->virtual_chapter_low;  in uds_put_volume_index_record()
569 		u64 high = get_zone_for_record(record)->virtual_chapter_high;  in uds_put_volume_index_record()  local
575 						(unsigned long long) high);  in uds_put_volume_index_record()
577 	address = extract_address(sub_index, record->name);  in uds_put_volume_index_record()
578 	if (unlikely(record->mutex != NULL))  in uds_put_volume_index_record()
579 		mutex_lock(record->mutex);  in uds_put_volume_index_record()
580 	result = uds_put_delta_index_entry(&record->delta_entry, address,  in uds_put_volume_index_record()
583 					   record->is_found ? record->name->name : NULL);  in uds_put_volume_index_record()
584 	if (unlikely(record->mutex != NULL))  in uds_put_volume_index_record()
585 		mutex_unlock(record->mutex);  in uds_put_volume_index_record()
588 		record->virtual_chapter = virtual_chapter;  in uds_put_volume_index_record()
589 		record->is_collision = record->delta_entry.is_collision;  in uds_put_volume_index_record()
590 		record->is_found = true;  in uds_put_volume_index_record()
595 		uds_log_delta_index_entry(&record->delta_entry);  in uds_put_volume_index_record()
608 	if (!record->is_found)  in uds_remove_volume_index_record()
613 	record->is_found = false;  in uds_remove_volume_index_record()
614 	if (unlikely(record->mutex != NULL))  in uds_remove_volume_index_record()
615 		mutex_lock(record->mutex);  in uds_remove_volume_index_record()
616 	result = uds_remove_delta_index_entry(&record->delta_entry);  in uds_remove_volume_index_record()
617 	if (unlikely(record->mutex != NULL))  in uds_remove_volume_index_record()
618 		mutex_unlock(record->mutex);  in uds_remove_volume_index_record()
627 	struct volume_sub_index_zone *zone = &sub_index->zones[zone_number];  in set_volume_sub_index_zone_open_chapter()
631 	zone->virtual_chapter_low = (virtual_chapter >= sub_index->chapter_count ?  in set_volume_sub_index_zone_open_chapter()
632 				     virtual_chapter - sub_index->chapter_count + 1 :  in set_volume_sub_index_zone_open_chapter()
634 	zone->virtual_chapter_high = virtual_chapter;  in set_volume_sub_index_zone_open_chapter()
637 	delta_zone = &sub_index->delta_index.delta_zones[zone_number];  in set_volume_sub_index_zone_open_chapter()
638 	for (i = 1; i <= delta_zone->list_count; i++)  in set_volume_sub_index_zone_open_chapter()
639 		used_bits += delta_zone->delta_lists[i].size;  in set_volume_sub_index_zone_open_chapter()
641 	if (used_bits > sub_index->max_zone_bits) {  in set_volume_sub_index_zone_open_chapter()
644 			1 + (used_bits - sub_index->max_zone_bits) / sub_index->chapter_zone_bits;  in set_volume_sub_index_zone_open_chapter()
651 					  (unsigned long long) zone->virtual_chapter_low);  in set_volume_sub_index_zone_open_chapter()
652 			zone->early_flushes++;  in set_volume_sub_index_zone_open_chapter()
653 			zone->virtual_chapter_low++;  in set_volume_sub_index_zone_open_chapter()
655 			u64 first_expired = zone->virtual_chapter_low;  in set_volume_sub_index_zone_open_chapter()
657 			if (first_expired + expire_count < zone->virtual_chapter_high) {  in set_volume_sub_index_zone_open_chapter()
658 				zone->early_flushes += expire_count;  in set_volume_sub_index_zone_open_chapter()
659 				zone->virtual_chapter_low += expire_count;  in set_volume_sub_index_zone_open_chapter()
661 				zone->early_flushes +=  in set_volume_sub_index_zone_open_chapter()
662 					zone->virtual_chapter_high - zone->virtual_chapter_low;  in set_volume_sub_index_zone_open_chapter()
663 				zone->virtual_chapter_low = zone->virtual_chapter_high;  in set_volume_sub_index_zone_open_chapter()
670 					  (unsigned long long) zone->virtual_chapter_low - 1);  in set_volume_sub_index_zone_open_chapter()
679 	struct mutex *mutex = &volume_index->zones[zone_number].hook_mutex;  in uds_set_volume_index_zone_open_chapter()
681 	set_volume_sub_index_zone_open_chapter(&volume_index->vi_non_hook, zone_number,  in uds_set_volume_index_zone_open_chapter()
690 		set_volume_sub_index_zone_open_chapter(&volume_index->vi_hook,  in uds_set_volume_index_zone_open_chapter()
705 	for (zone = 0; zone < volume_index->zone_count; zone++)  in uds_set_volume_index_open_chapter()
712 	const struct volume_sub_index *sub_index = record->sub_index;  in uds_set_volume_index_record_chapter()
715 	if (!record->is_found)  in uds_set_volume_index_record_chapter()
720 		u64 low = get_zone_for_record(record)->virtual_chapter_low;  in uds_set_volume_index_record_chapter()
721 		u64 high = get_zone_for_record(record)->virtual_chapter_high;  in uds_set_volume_index_record_chapter()  local
727 						(unsigned long long) high);  in uds_set_volume_index_record_chapter()
730 	if (unlikely(record->mutex != NULL))  in uds_set_volume_index_record_chapter()
731 		mutex_lock(record->mutex);  in uds_set_volume_index_record_chapter()
732 	result = uds_set_delta_entry_value(&record->delta_entry,  in uds_set_volume_index_record_chapter()
735 	if (unlikely(record->mutex != NULL))  in uds_set_volume_index_record_chapter()
736 		mutex_unlock(record->mutex);  in uds_set_volume_index_record_chapter()
740 	record->virtual_chapter = virtual_chapter;  in uds_set_volume_index_record_chapter()
751 	const struct volume_sub_index_zone *zone = &sub_index->zones[zone_number];  in lookup_volume_sub_index_name()
757 	result = uds_get_delta_index_entry(&sub_index->delta_index, delta_list_number,  in lookup_volume_sub_index_name()
758 					   address, name->name, &delta_entry);  in lookup_volume_sub_index_name()
766 	rolling_chapter = (index_chapter - zone->virtual_chapter_low) & sub_index->chapter_mask;  in lookup_volume_sub_index_name()
768 	virtual_chapter = zone->virtual_chapter_low + rolling_chapter;  in lookup_volume_sub_index_name()
769 	if (virtual_chapter > zone->virtual_chapter_high)  in lookup_volume_sub_index_name()
775 /* Do a read-only lookup of the record name for sparse cache management. */
780 	struct mutex *mutex = &volume_index->zones[zone_number].hook_mutex;  in uds_lookup_volume_index_name()
787 	virtual_chapter = lookup_volume_sub_index_name(&volume_index->vi_hook, name);  in uds_lookup_volume_index_name()
795 	uds_reset_delta_index(&sub_index->delta_index);  in abort_restoring_volume_sub_index()
800 	abort_restoring_volume_sub_index(&volume_index->vi_non_hook);  in abort_restoring_volume_index()
802 		abort_restoring_volume_sub_index(&volume_index->vi_hook);  in abort_restoring_volume_index()
809 	unsigned int z;  in start_restoring_volume_sub_index()  local
846 		if (sub_index->volume_nonce == 0) {  in start_restoring_volume_sub_index()
847 			sub_index->volume_nonce = header.volume_nonce;  in start_restoring_volume_sub_index()
848 		} else if (header.volume_nonce != sub_index->volume_nonce) {  in start_restoring_volume_sub_index()
858 			u64 high = header.virtual_chapter_high;  in start_restoring_volume_sub_index()  local
865 							(unsigned long long) high);  in start_restoring_volume_sub_index()
880 			sub_index->flush_chapters[header.first_list + j] =  in start_restoring_volume_sub_index()
885 	for (z = 0; z < sub_index->zone_count; z++) {  in start_restoring_volume_sub_index()
886 		memset(&sub_index->zones[z], 0, sizeof(struct volume_sub_index_zone));  in start_restoring_volume_sub_index()
887 		sub_index->zones[z].virtual_chapter_low = virtual_chapter_low;  in start_restoring_volume_sub_index()
888 		sub_index->zones[z].virtual_chapter_high = virtual_chapter_high;  in start_restoring_volume_sub_index()
891 	result = uds_start_restoring_delta_index(&sub_index->delta_index, readers,  in start_restoring_volume_sub_index()
907 		return start_restoring_volume_sub_index(&volume_index->vi_non_hook,  in start_restoring_volume_index()
938 			volume_index->sparse_sample_rate = header.sparse_sample_rate;  in start_restoring_volume_index()
939 		} else if (volume_index->sparse_sample_rate != header.sparse_sample_rate) {  in start_restoring_volume_index()
942 						 volume_index->sparse_sample_rate,  in start_restoring_volume_index()
948 	result = start_restoring_volume_sub_index(&volume_index->vi_non_hook,  in start_restoring_volume_index()
953 	return start_restoring_volume_sub_index(&volume_index->vi_hook, buffered_readers,  in start_restoring_volume_index()
961 	return uds_finish_restoring_delta_index(&sub_index->delta_index,  in finish_restoring_volume_sub_index()
971 	result = finish_restoring_volume_sub_index(&volume_index->vi_non_hook,  in finish_restoring_volume_index()
974 		result = finish_restoring_volume_sub_index(&volume_index->vi_hook,  in finish_restoring_volume_index()
1011 	struct volume_sub_index_zone *volume_index_zone = &sub_index->zones[zone_number];  in start_saving_volume_sub_index()
1012 	u32 first_list = sub_index->delta_index.delta_zones[zone_number].first_list;  in start_saving_volume_sub_index()
1013 	u32 list_count = sub_index->delta_index.delta_zones[zone_number].list_count;  in start_saving_volume_sub_index()
1020 	encode_u64_le(buffer, &offset, sub_index->volume_nonce);  in start_saving_volume_sub_index()
1021 	encode_u64_le(buffer, &offset, volume_index_zone->virtual_chapter_low);  in start_saving_volume_sub_index()
1022 	encode_u64_le(buffer, &offset, volume_index_zone->virtual_chapter_high);  in start_saving_volume_sub_index()
1040 		put_unaligned_le64(sub_index->flush_chapters[first_list + i], &encoded);  in start_saving_volume_sub_index()
1049 	return uds_start_saving_delta_index(&sub_index->delta_index, zone_number,  in start_saving_volume_sub_index()
1062 		return start_saving_volume_sub_index(&volume_index->vi_non_hook,  in start_saving_volume_index()
1068 	encode_u32_le(buffer, &offset, volume_index->sparse_sample_rate);  in start_saving_volume_index()
1081 	result = start_saving_volume_sub_index(&volume_index->vi_non_hook, zone_number,  in start_saving_volume_index()
1086 	return start_saving_volume_sub_index(&volume_index->vi_hook, zone_number,  in start_saving_volume_index()
1093 	return uds_finish_saving_delta_index(&sub_index->delta_index, zone_number);  in finish_saving_volume_sub_index()
1101 	result = finish_saving_volume_sub_index(&volume_index->vi_non_hook, zone_number);  in finish_saving_volume_index()
1103 		result = finish_saving_volume_sub_index(&volume_index->vi_hook, zone_number);  in finish_saving_volume_index()
1138 	unsigned int z;  in get_volume_sub_index_stats()  local
1140 	uds_get_delta_index_stats(&sub_index->delta_index, &dis);  in get_volume_sub_index_stats()
1141 	stats->rebalance_time = dis.rebalance_time;  in get_volume_sub_index_stats()
1142 	stats->rebalance_count = dis.rebalance_count;  in get_volume_sub_index_stats()
1143 	stats->record_count = dis.record_count;  in get_volume_sub_index_stats()
1144 	stats->collision_count = dis.collision_count;  in get_volume_sub_index_stats()
1145 	stats->discard_count = dis.discard_count;  in get_volume_sub_index_stats()
1146 	stats->overflow_count = dis.overflow_count;  in get_volume_sub_index_stats()
1147 	stats->delta_lists = dis.list_count;  in get_volume_sub_index_stats()
1148 	stats->early_flushes = 0;  in get_volume_sub_index_stats()
1149 	for (z = 0; z < sub_index->zone_count; z++)  in get_volume_sub_index_stats()
1150 		stats->early_flushes += sub_index->zones[z].early_flushes;  in get_volume_sub_index_stats()
1158 	get_volume_sub_index_stats(&volume_index->vi_non_hook, stats);  in uds_get_volume_index_stats()
1162 	get_volume_sub_index_stats(&volume_index->vi_hook, &sparse_stats);  in uds_get_volume_index_stats()
1163 	stats->rebalance_time += sparse_stats.rebalance_time;  in uds_get_volume_index_stats()
1164 	stats->rebalance_count += sparse_stats.rebalance_count;  in uds_get_volume_index_stats()
1165 	stats->record_count += sparse_stats.record_count;  in uds_get_volume_index_stats()
1166 	stats->collision_count += sparse_stats.collision_count;  in uds_get_volume_index_stats()
1167 	stats->discard_count += sparse_stats.discard_count;  in uds_get_volume_index_stats()
1168 	stats->overflow_count += sparse_stats.overflow_count;  in uds_get_volume_index_stats()
1169 	stats->delta_lists += sparse_stats.delta_lists;  in uds_get_volume_index_stats()
1170 	stats->early_flushes += sparse_stats.early_flushes;  in uds_get_volume_index_stats()
1178 	unsigned int zone_count = config->zone_count;  in initialize_volume_sub_index()
1180 	unsigned int z;  in initialize_volume_sub_index()  local
1187 	sub_index->address_bits = params.address_bits;  in initialize_volume_sub_index()
1188 	sub_index->address_mask = (1u << params.address_bits) - 1;  in initialize_volume_sub_index()
1189 	sub_index->chapter_bits = params.chapter_bits;  in initialize_volume_sub_index()
1190 	sub_index->chapter_mask = (1u << params.chapter_bits) - 1;  in initialize_volume_sub_index()
1191 	sub_index->chapter_count = params.chapter_count;  in initialize_volume_sub_index()
1192 	sub_index->list_count = params.list_count;  in initialize_volume_sub_index()
1193 	sub_index->zone_count = zone_count;  in initialize_volume_sub_index()
1194 	sub_index->chapter_zone_bits = params.chapter_size_in_bits / zone_count;  in initialize_volume_sub_index()
1195 	sub_index->volume_nonce = volume_nonce;  in initialize_volume_sub_index()
1197 	result = uds_initialize_delta_index(&sub_index->delta_index, zone_count,  in initialize_volume_sub_index()
1204 	for (z = 0; z < sub_index->delta_index.zone_count; z++)  in initialize_volume_sub_index()
1205 		available_bytes += sub_index->delta_index.delta_zones[z].size;  in initialize_volume_sub_index()
1206 	available_bytes -= params.target_free_bytes;  in initialize_volume_sub_index()
1207 	sub_index->max_zone_bits = (available_bytes * BITS_PER_BYTE) / zone_count;  in initialize_volume_sub_index()
1208 	sub_index->memory_size = (sub_index->delta_index.memory_size +  in initialize_volume_sub_index()
1215 			      &sub_index->flush_chapters);  in initialize_volume_sub_index()
1220 			    "volume index zones", &sub_index->zones);  in initialize_volume_sub_index()
1235 	volume_index->zone_count = config->zone_count;  in uds_make_volume_index()
1237 	if (!uds_is_sparse_index_geometry(config->geometry)) {  in uds_make_volume_index()
1239 						     &volume_index->vi_non_hook);  in uds_make_volume_index()
1245 		volume_index->memory_size = volume_index->vi_non_hook.memory_size;  in uds_make_volume_index()
1250 	volume_index->sparse_sample_rate = config->sparse_sample_rate;  in uds_make_volume_index()
1252 	result = vdo_allocate(config->zone_count, struct volume_index_zone,  in uds_make_volume_index()
1253 			      "volume index zones", &volume_index->zones);  in uds_make_volume_index()
1259 	for (zone = 0; zone < config->zone_count; zone++)  in uds_make_volume_index()
1260 		mutex_init(&volume_index->zones[zone].hook_mutex);  in uds_make_volume_index()
1264 					     &volume_index->vi_non_hook);  in uds_make_volume_index()
1272 					     &volume_index->vi_hook);  in uds_make_volume_index()
1279 	volume_index->memory_size =  in uds_make_volume_index()
1280 		volume_index->vi_non_hook.memory_size + volume_index->vi_hook.memory_size;  in uds_make_volume_index()