296 	/* The number of zones */
298 	/* The hash zones themselves */
299 	struct hash_zone zones[];  member
2038 static void close_index(struct hash_zones *zones)  in close_index()  argument
2039 	__must_hold(&zones->lock)  in close_index()
2047 	zones->index_state = IS_CHANGING;  in close_index()
2049 	spin_unlock(&zones->lock);  in close_index()
2050 	result = uds_close_index(zones->index_session);  in close_index()
2054 	spin_lock(&zones->lock);  in close_index()
2055 	zones->index_state = IS_CLOSED;  in close_index()
2056 	zones->error_flag |= result != UDS_SUCCESS;  in close_index()
2060 static void open_index(struct hash_zones *zones)  in open_index()  argument
2061 	__must_hold(&zones->lock)  in open_index()
2065 	bool create_flag = zones->create_flag;  in open_index()
2067 	zones->create_flag = false;  in open_index()
2072 	zones->index_state = IS_CHANGING;  in open_index()
2073 	zones->error_flag = false;  in open_index()
2076 	spin_unlock(&zones->lock);  in open_index()
2078 				&zones->parameters, zones->index_session);  in open_index()
2082 	spin_lock(&zones->lock);  in open_index()
2090 			zones->index_state = IS_CLOSED;  in open_index()
2091 			zones->create_flag = true;  in open_index()
2098 		zones->index_state = IS_OPENED;  in open_index()
2100 		zones->index_state = IS_CLOSED;  in open_index()
2101 		zones->index_target = IS_CLOSED;  in open_index()
2102 		zones->error_flag = true;  in open_index()
2103 		spin_unlock(&zones->lock);  in open_index()
2105 		spin_lock(&zones->lock);  in open_index()
2115 	struct hash_zones *zones = as_hash_zones(completion);  in change_dedupe_state()  local
2117 	spin_lock(&zones->lock);  in change_dedupe_state()
2120 	while (vdo_is_state_normal(&zones->state) &&  in change_dedupe_state()
2121 	       ((zones->index_state != zones->index_target) || zones->create_flag)) {  in change_dedupe_state()
2122 		if (zones->index_state == IS_OPENED)  in change_dedupe_state()
2123 			close_index(zones);  in change_dedupe_state()
2125 			open_index(zones);  in change_dedupe_state()
2128 	zones->changing = false;  in change_dedupe_state()
2129 	spin_unlock(&zones->lock);  in change_dedupe_state()
2149  * @zones: the hash zones.
2152 static void report_dedupe_timeouts(struct hash_zones *zones, unsigned int timeouts)  in report_dedupe_timeouts()  argument
2154 	atomic64_add(timeouts, &zones->timeouts);  in report_dedupe_timeouts()
2155 	spin_lock(&zones->lock);  in report_dedupe_timeouts()
2156 	if (__ratelimit(&zones->ratelimiter)) {  in report_dedupe_timeouts()
2157 		u64 unreported = atomic64_read(&zones->timeouts);  in report_dedupe_timeouts()
2159 		unreported -= zones->reported_timeouts;  in report_dedupe_timeouts()
2162 		zones->reported_timeouts += unreported;  in report_dedupe_timeouts()
2164 	spin_unlock(&zones->lock);  in report_dedupe_timeouts()
2167 static int initialize_index(struct vdo *vdo, struct hash_zones *zones)  in initialize_index()  argument
2181 	spin_lock_init(&zones->lock);  in initialize_index()
2187 	ratelimit_default_init(&zones->ratelimiter);  in initialize_index()
2188 	ratelimit_set_flags(&zones->ratelimiter, RATELIMIT_MSG_ON_RELEASE);  in initialize_index()
2191 	zones->parameters = (struct uds_parameters) {  in initialize_index()
2200 	result = uds_create_index_session(&zones->index_session);  in initialize_index()
2207 		uds_destroy_index_session(vdo_forget(zones->index_session));  in initialize_index()
2212 	vdo_initialize_completion(&zones->completion, vdo, VDO_HASH_ZONES_COMPLETION);  in initialize_index()
2213 	vdo_set_completion_callback(&zones->completion, change_dedupe_state,  in initialize_index()
2347 static int __must_check initialize_zone(struct vdo *vdo, struct hash_zones *zones,  in initialize_zone()  argument
2352 	struct hash_zone *zone = &zones->zones[zone_number];  in initialize_zone()
2386 		context->request.session = zones->index_session;  in initialize_zone()
2396 	struct hash_zones *zones = context;  in get_thread_id_for_zone()  local
2398 	return zones->zones[zone_number].thread_id;  in get_thread_id_for_zone()
2402  * vdo_make_hash_zones() - Create the hash zones.
2405  * @zones_ptr: A pointer to hold the zones.
2412 	struct hash_zones *zones;  in vdo_make_hash_zones()  local
2420 				       __func__, &zones);  in vdo_make_hash_zones()
2424 	result = initialize_index(vdo, zones);  in vdo_make_hash_zones()
2426 		vdo_free(zones);  in vdo_make_hash_zones()
2430 	vdo_set_admin_state_code(&zones->state, VDO_ADMIN_STATE_NEW);  in vdo_make_hash_zones()
2432 	zones->zone_count = zone_count;  in vdo_make_hash_zones()
2434 		result = initialize_zone(vdo, zones, z);  in vdo_make_hash_zones()
2436 			vdo_free_hash_zones(zones);  in vdo_make_hash_zones()
2441 	result = vdo_make_action_manager(zones->zone_count, get_thread_id_for_zone,  in vdo_make_hash_zones()
2442 					 vdo->thread_config.admin_thread, zones, NULL,  in vdo_make_hash_zones()
2443 					 vdo, &zones->manager);  in vdo_make_hash_zones()
2445 		vdo_free_hash_zones(zones);  in vdo_make_hash_zones()
2449 	*zones_ptr = zones;  in vdo_make_hash_zones()
2453 void vdo_finish_dedupe_index(struct hash_zones *zones)  in vdo_finish_dedupe_index()  argument
2455 	if (zones == NULL)  in vdo_finish_dedupe_index()
2458 	uds_destroy_index_session(vdo_forget(zones->index_session));  in vdo_finish_dedupe_index()
2462  * vdo_free_hash_zones() - Free the hash zones.
2463  * @zones: The zone to free.
2465 void vdo_free_hash_zones(struct hash_zones *zones)  in vdo_free_hash_zones()  argument
2469 	if (zones == NULL)  in vdo_free_hash_zones()
2472 	vdo_free(vdo_forget(zones->manager));  in vdo_free_hash_zones()
2474 	for (i = 0; i < zones->zone_count; i++) {  in vdo_free_hash_zones()
2475 		struct hash_zone *zone = &zones->zones[i];  in vdo_free_hash_zones()
2482 	if (zones->index_session != NULL)  in vdo_free_hash_zones()
2483 		vdo_finish_dedupe_index(zones);  in vdo_free_hash_zones()
2485 	ratelimit_state_exit(&zones->ratelimiter);  in vdo_free_hash_zones()
2486 	vdo_free(zones);  in vdo_free_hash_zones()
2491 	struct hash_zones *zones = container_of(state, struct hash_zones, state);  in initiate_suspend_index()  local
2494 	spin_lock(&zones->lock);  in initiate_suspend_index()
2495 	index_state = zones->index_state;  in initiate_suspend_index()
2496 	spin_unlock(&zones->lock);  in initiate_suspend_index()
2499 		bool save = vdo_is_state_saving(&zones->state);  in initiate_suspend_index()
2502 		result = uds_suspend_index_session(zones->index_session, save);  in initiate_suspend_index()
2511  * suspend_index() - Suspend the UDS index prior to draining hash zones.
2517 	struct hash_zones *zones = context;  in suspend_index()  local
2519 	vdo_start_draining(&zones->state,  in suspend_index()
2520 			   vdo_get_current_manager_operation(zones->manager), completion,  in suspend_index()
2542 	struct hash_zones *zones = context;  in drain_hash_zone()  local
2544 	vdo_start_draining(&zones->zones[zone_number].state,  in drain_hash_zone()
2545 			   vdo_get_current_manager_operation(zones->manager), parent,  in drain_hash_zone()
2549 /** vdo_drain_hash_zones() - Drain all hash zones. */
2550 void vdo_drain_hash_zones(struct hash_zones *zones, struct vdo_completion *parent)  in vdo_drain_hash_zones()  argument
2552 	vdo_schedule_operation(zones->manager, parent->vdo->suspend_type, suspend_index,  in vdo_drain_hash_zones()
2556 static void launch_dedupe_state_change(struct hash_zones *zones)  in launch_dedupe_state_change()  argument
2557 	__must_hold(&zones->lock)  in launch_dedupe_state_change()
2560 	if (zones->changing || !vdo_is_state_normal(&zones->state))  in launch_dedupe_state_change()
2564 	if (zones->create_flag || (zones->index_state != zones->index_target)) {  in launch_dedupe_state_change()
2565 		zones->changing = true;  in launch_dedupe_state_change()
2566 		vdo_launch_completion(&zones->completion);  in launch_dedupe_state_change()
2574  * resume_index() - Resume the UDS index prior to resuming hash zones.
2580 	struct hash_zones *zones = context;  in resume_index()  local
2584 	zones->parameters.bdev = config->owned_device->bdev;  in resume_index()
2585 	result = uds_resume_index_session(zones->index_session, zones->parameters.bdev);  in resume_index()
2589 	spin_lock(&zones->lock);  in resume_index()
2590 	vdo_resume_if_quiescent(&zones->state);  in resume_index()
2593 		zones->index_target = IS_OPENED;  in resume_index()
2594 		WRITE_ONCE(zones->dedupe_flag, true);  in resume_index()
2596 		zones->index_target = IS_CLOSED;  in resume_index()
2599 	launch_dedupe_state_change(zones);  in resume_index()
2600 	spin_unlock(&zones->lock);  in resume_index()
2613 	struct hash_zone *zone = &(((struct hash_zones *) context)->zones[zone_number]);  in resume_hash_zone()
2619  * vdo_resume_hash_zones() - Resume a set of hash zones.
2620  * @zones: The hash zones to resume.
2621  * @parent: The object to notify when the zones have resumed.
2623 void vdo_resume_hash_zones(struct hash_zones *zones, struct vdo_completion *parent)  in vdo_resume_hash_zones()  argument
2630 	vdo_schedule_operation(zones->manager, VDO_ADMIN_STATE_RESUMING, resume_index,  in vdo_resume_hash_zones()
2635  * get_hash_zone_statistics() - Add the statistics for this hash zone to the tally for all zones.
2651 static void get_index_statistics(struct hash_zones *zones,  in get_index_statistics()  argument
2658 	spin_lock(&zones->lock);  in get_index_statistics()
2659 	state = zones->index_state;  in get_index_statistics()
2660 	spin_unlock(&zones->lock);  in get_index_statistics()
2665 	result = uds_get_index_session_stats(zones->index_session, &index_stats);  in get_index_statistics()
2682  * vdo_get_dedupe_statistics() - Tally the statistics from all the hash zones and the UDS index.
2683  * @zones: The hash zones to query
2686  * Return: The sum of the hash lock statistics from all hash zones plus the statistics from the UDS
2689 void vdo_get_dedupe_statistics(struct hash_zones *zones, struct vdo_statistics *stats)  in vdo_get_dedupe_statistics()  argument
2694 	for (zone = 0; zone < zones->zone_count; zone++)  in vdo_get_dedupe_statistics()
2695 		get_hash_zone_statistics(&zones->zones[zone], &stats->hash_lock);  in vdo_get_dedupe_statistics()
2697 	get_index_statistics(zones, &stats->index);  in vdo_get_dedupe_statistics()
2700 	 * zones->timeouts gives the number of timeouts, and dedupe_context_busy gives the number  in vdo_get_dedupe_statistics()
2704 		(atomic64_read(&zones->timeouts) + atomic64_read(&zones->dedupe_context_busy));  in vdo_get_dedupe_statistics()
2709  * @zones: The hash_zones from which to select.
2714 struct hash_zone *vdo_select_hash_zone(struct hash_zones *zones,  in vdo_select_hash_zone()  argument
2719 	 * since the number of hash zones is small.  in vdo_select_hash_zone()
2730 	hash = (hash * zones->zone_count) >> 8;  in vdo_select_hash_zone()
2731 	return &zones->zones[hash];  in vdo_select_hash_zone()
2760 static const char *index_state_to_string(struct hash_zones *zones,  in index_state_to_string()  argument
2763 	if (!vdo_is_state_normal(&zones->state))  in index_state_to_string()
2768 		return zones->error_flag ? ERROR : CLOSED;  in index_state_to_string()
2770 		return zones->index_target == IS_OPENED ? OPENING : CLOSING;  in index_state_to_string()
2772 		return READ_ONCE(zones->dedupe_flag) ? ONLINE : OFFLINE;  in index_state_to_string()
2798  * vdo_dump_hash_zones() - Dump information about the hash zones to the log for debugging.
2799  * @zones: The zones to dump.
2801 void vdo_dump_hash_zones(struct hash_zones *zones)  in vdo_dump_hash_zones()  argument
2806 	spin_lock(&zones->lock);  in vdo_dump_hash_zones()
2807 	state = index_state_to_string(zones, zones->index_state);  in vdo_dump_hash_zones()
2808 	target = (zones->changing ? index_state_to_string(zones, zones->index_target) : NULL);  in vdo_dump_hash_zones()
2809 	spin_unlock(&zones->lock);  in vdo_dump_hash_zones()
2815 	for (zone = 0; zone < zones->zone_count; zone++)  in vdo_dump_hash_zones()
2816 		dump_hash_zone(&zones->zones[zone]);  in vdo_dump_hash_zones()
2941 static void set_target_state(struct hash_zones *zones, enum index_state target,  in set_target_state()  argument
2946 	spin_lock(&zones->lock);  in set_target_state()
2947 	old_state = index_state_to_string(zones, zones->index_target);  in set_target_state()
2949 		WRITE_ONCE(zones->dedupe_flag, dedupe);  in set_target_state()
2952 		zones->create_flag = true;  in set_target_state()
2954 	zones->index_target = target;  in set_target_state()
2955 	launch_dedupe_state_change(zones);  in set_target_state()
2956 	new_state = index_state_to_string(zones, zones->index_target);  in set_target_state()
2957 	spin_unlock(&zones->lock);  in set_target_state()
2963 const char *vdo_get_dedupe_index_state_name(struct hash_zones *zones)  in vdo_get_dedupe_index_state_name()  argument
2967 	spin_lock(&zones->lock);  in vdo_get_dedupe_index_state_name()
2968 	state = index_state_to_string(zones, zones->index_state);  in vdo_get_dedupe_index_state_name()
2969 	spin_unlock(&zones->lock);  in vdo_get_dedupe_index_state_name()
2975 int vdo_message_dedupe_index(struct hash_zones *zones, const char *name)  in vdo_message_dedupe_index()  argument
2978 		set_target_state(zones, IS_CLOSED, false, false, false);  in vdo_message_dedupe_index()
2981 		set_target_state(zones, IS_OPENED, false, false, true);  in vdo_message_dedupe_index()
2984 		set_target_state(zones, IS_OPENED, true, false, false);  in vdo_message_dedupe_index()
2987 		set_target_state(zones, IS_OPENED, true, true, false);  in vdo_message_dedupe_index()
2994 void vdo_set_dedupe_state_normal(struct hash_zones *zones)  in vdo_set_dedupe_state_normal()  argument
2996 	vdo_set_admin_state_code(&zones->state, VDO_ADMIN_STATE_NORMAL_OPERATION);  in vdo_set_dedupe_state_normal()
3000 void vdo_start_dedupe_index(struct hash_zones *zones, bool create_flag)  in vdo_start_dedupe_index()  argument
3002 	set_target_state(zones, IS_OPENED, true, true, create_flag);  in vdo_start_dedupe_index()