283 	spinlock_t lock;  member
342 static void return_hash_lock_to_pool(struct hash_zone *zone, struct hash_lock *lock)  in return_hash_lock_to_pool()  argument
344 	memset(lock, 0, sizeof(*lock));  in return_hash_lock_to_pool()
345 	INIT_LIST_HEAD(&lock->pool_node);  in return_hash_lock_to_pool()
346 	INIT_LIST_HEAD(&lock->duplicate_ring);  in return_hash_lock_to_pool()
347 	vdo_waitq_init(&lock->waiters);  in return_hash_lock_to_pool()
348 	list_add_tail(&lock->pool_node, &zone->lock_pool);  in return_hash_lock_to_pool()
372 static inline u64 hash_lock_key(struct hash_lock *lock)  in hash_lock_key()  argument
374 	return get_unaligned_le64(&lock->hash.name);  in hash_lock_key()
424 static inline struct data_vio *dequeue_lock_waiter(struct hash_lock *lock)  in dequeue_lock_waiter()  argument
426 	return vdo_waiter_as_data_vio(vdo_waitq_dequeue_waiter(&lock->waiters));  in dequeue_lock_waiter()
480 static void start_deduping(struct hash_lock *lock, struct data_vio *agent,
482 static void start_locking(struct hash_lock *lock, struct data_vio *agent);
483 static void start_writing(struct hash_lock *lock, struct data_vio *agent);
523 static struct data_vio *retire_lock_agent(struct hash_lock *lock)  in retire_lock_agent()  argument
525 	struct data_vio *old_agent = lock->agent;  in retire_lock_agent()
526 	struct data_vio *new_agent = dequeue_lock_waiter(lock);  in retire_lock_agent()
528 	lock->agent = new_agent;  in retire_lock_agent()
531 		set_duplicate_location(new_agent, lock->duplicate);  in retire_lock_agent()
540 static void wait_on_hash_lock(struct hash_lock *lock, struct data_vio *data_vio)  in wait_on_hash_lock()  argument
542 	vdo_waitq_enqueue_waiter(&lock->waiters, &data_vio->waiter);  in wait_on_hash_lock()
548 	if ((lock->state != VDO_HASH_LOCK_WRITING) || !cancel_data_vio_compression(lock->agent))  in wait_on_hash_lock()
558 	data_vio->compression.lock_holder = lock->agent;  in wait_on_hash_lock()
581 static void start_bypassing(struct hash_lock *lock, struct data_vio *agent)  in start_bypassing()  argument
583 	lock->state = VDO_HASH_LOCK_BYPASSING;  in start_bypassing()
589 	struct hash_lock *lock = data_vio->hash_lock;  in vdo_clean_failed_hash_lock()  local
591 	if (lock->state == VDO_HASH_LOCK_BYPASSING) {  in vdo_clean_failed_hash_lock()
596 	if (lock->agent == NULL) {  in vdo_clean_failed_hash_lock()
597 		lock->agent = data_vio;  in vdo_clean_failed_hash_lock()
598 	} else if (data_vio != lock->agent) {  in vdo_clean_failed_hash_lock()
603 	lock->state = VDO_HASH_LOCK_BYPASSING;  in vdo_clean_failed_hash_lock()
606 	lock->update_advice = false;  in vdo_clean_failed_hash_lock()
608 	vdo_waitq_notify_all_waiters(&lock->waiters, abort_waiter, NULL);  in vdo_clean_failed_hash_lock()
610 	if (lock->duplicate_lock != NULL) {  in vdo_clean_failed_hash_lock()
612 		data_vio->duplicate = lock->duplicate;  in vdo_clean_failed_hash_lock()
617 	lock->agent = NULL;  in vdo_clean_failed_hash_lock()
632 	struct hash_lock *lock = agent->hash_lock;  in finish_unlocking()  local
636 	VDO_ASSERT_LOG_ONLY(lock->duplicate_lock == NULL,  in finish_unlocking()
639 	if (!lock->verified) {  in finish_unlocking()
646 		start_writing(lock, agent);  in finish_unlocking()
654 	lock->verified = false;  in finish_unlocking()
656 	if (vdo_waitq_has_waiters(&lock->waiters)) {  in finish_unlocking()
665 		agent = retire_lock_agent(lock);  in finish_unlocking()
666 		start_locking(lock, agent);  in finish_unlocking()
674 	start_bypassing(lock, agent);  in finish_unlocking()
688 	struct hash_lock *lock = agent->hash_lock;  in unlock_duplicate_pbn()  local
691 	VDO_ASSERT_LOG_ONLY(lock->duplicate_lock != NULL,  in unlock_duplicate_pbn()
695 					   vdo_forget(lock->duplicate_lock));  in unlock_duplicate_pbn()
696 	if (lock->state == VDO_HASH_LOCK_BYPASSING) {  in unlock_duplicate_pbn()
710 static void start_unlocking(struct hash_lock *lock, struct data_vio *agent)  in start_unlocking()  argument
712 	lock->state = VDO_HASH_LOCK_UNLOCKING;  in start_unlocking()
745 	struct hash_lock *lock = agent->hash_lock;  in finish_updating()  local
755 	lock->update_advice = false;  in finish_updating()
757 	if (vdo_waitq_has_waiters(&lock->waiters)) {  in finish_updating()
763 		start_deduping(lock, agent, true);  in finish_updating()
767 	if (lock->duplicate_lock != NULL) {  in finish_updating()
772 		start_unlocking(lock, agent);  in finish_updating()
780 	start_bypassing(lock, agent);  in finish_updating()
791 static void start_updating(struct hash_lock *lock, struct data_vio *agent)  in start_updating()  argument
793 	lock->state = VDO_HASH_LOCK_UPDATING;  in start_updating()
795 	VDO_ASSERT_LOG_ONLY(lock->verified, "new advice should have been verified");  in start_updating()
796 	VDO_ASSERT_LOG_ONLY(lock->update_advice, "should only update advice if needed");  in start_updating()
814 static void finish_deduping(struct hash_lock *lock, struct data_vio *data_vio)  in finish_deduping()  argument
818 	VDO_ASSERT_LOG_ONLY(lock->agent == NULL, "shouldn't have an agent in DEDUPING");  in finish_deduping()
819 	VDO_ASSERT_LOG_ONLY(!vdo_waitq_has_waiters(&lock->waiters),  in finish_deduping()
823 	if (lock->reference_count > 1) {  in finish_deduping()
829 	lock->agent = agent;  in finish_deduping()
830 	if (lock->update_advice) {  in finish_deduping()
838 		start_updating(lock, agent);  in finish_deduping()
845 		start_unlocking(lock, agent);  in finish_deduping()
868 	struct hash_lock *lock, *new_lock;  in acquire_lock()  local
890 				 new_lock, (replace_lock != NULL), (void **) &lock);  in acquire_lock()
898 		VDO_ASSERT_LOG_ONLY(lock == replace_lock,  in acquire_lock()
906 	if (lock == replace_lock) {  in acquire_lock()
907 		lock = new_lock;  in acquire_lock()
908 		lock->registered = true;  in acquire_lock()
914 	*lock_ptr = lock;  in acquire_lock()
979 static void launch_dedupe(struct hash_lock *lock, struct data_vio *data_vio,  in launch_dedupe()  argument
982 	if (!has_claim && !vdo_claim_pbn_lock_increment(lock->duplicate_lock)) {  in launch_dedupe()
984 		fork_hash_lock(lock, data_vio);  in launch_dedupe()
989 	set_duplicate_location(data_vio, lock->duplicate);  in launch_dedupe()
991 	update_metadata_for_data_vio_write(data_vio, lock->duplicate_lock);  in launch_dedupe()
1004 static void start_deduping(struct hash_lock *lock, struct data_vio *agent,  in start_deduping()  argument
1007 	lock->state = VDO_HASH_LOCK_DEDUPING;  in start_deduping()
1013 	if (lock->duplicate_lock == NULL) {  in start_deduping()
1021 	VDO_ASSERT_LOG_ONLY(vdo_is_pbn_read_lock(lock->duplicate_lock),  in start_deduping()
1029 	lock->agent = NULL;  in start_deduping()
1037 		launch_dedupe(lock, agent, true);  in start_deduping()
1040 	while (vdo_waitq_has_waiters(&lock->waiters))  in start_deduping()
1041 		launch_dedupe(lock, dequeue_lock_waiter(lock), false);  in start_deduping()
1049 		finish_deduping(lock, agent);  in start_deduping()
1076 	struct hash_lock *lock = agent->hash_lock;  in finish_verifying()  local
1080 	lock->verified = agent->is_duplicate;  in finish_verifying()
1086 	if (!lock->verify_counted) {  in finish_verifying()
1087 		lock->verify_counted = true;  in finish_verifying()
1088 		if (lock->verified)  in finish_verifying()
1098 	if (lock->verified && !vdo_claim_pbn_lock_increment(lock->duplicate_lock)) {  in finish_verifying()
1100 		lock->verified = false;  in finish_verifying()
1103 	if (lock->verified) {  in finish_verifying()
1108 		start_deduping(lock, agent, false);  in finish_verifying()
1117 		lock->update_advice = true;  in finish_verifying()
1118 		start_unlocking(lock, agent);  in finish_verifying()
1191 static void start_verifying(struct hash_lock *lock, struct data_vio *agent)  in start_verifying()  argument
1199 	lock->state = VDO_HASH_LOCK_VERIFYING;  in start_verifying()
1200 	VDO_ASSERT_LOG_ONLY(!lock->verified, "hash lock only verifies advice once");  in start_verifying()
1225 	struct hash_lock *lock = agent->hash_lock;  in finish_locking()  local
1230 		VDO_ASSERT_LOG_ONLY(lock->duplicate_lock == NULL,  in finish_locking()
1238 		lock->update_advice = true;  in finish_locking()
1239 		start_writing(lock, agent);  in finish_locking()
1243 	VDO_ASSERT_LOG_ONLY(lock->duplicate_lock != NULL,  in finish_locking()
1246 	if (!lock->verified) {  in finish_locking()
1252 		start_verifying(lock, agent);  in finish_locking()
1256 	if (!vdo_claim_pbn_lock_increment(lock->duplicate_lock)) {  in finish_locking()
1263 		lock->verified = false;  in finish_locking()
1264 		lock->update_advice = true;  in finish_locking()
1265 		start_unlocking(lock, agent);  in finish_locking()
1273 	start_deduping(lock, agent, false);  in finish_locking()
1276 static bool acquire_provisional_reference(struct data_vio *agent, struct pbn_lock *lock,  in acquire_provisional_reference()  argument
1281 	int result = vdo_acquire_provisional_reference(slab, agent->duplicate.pbn, lock);  in acquire_provisional_reference()
1290 					   agent->duplicate.pbn, lock);  in acquire_provisional_reference()
1308 	struct pbn_lock *lock;  in lock_duplicate_pbn()  local
1335 						    VIO_READ_LOCK, &lock);  in lock_duplicate_pbn()
1341 	if (!vdo_is_pbn_read_lock(lock)) {  in lock_duplicate_pbn()
1382 	if (lock->holder_count == 0) {  in lock_duplicate_pbn()
1383 		if (!acquire_provisional_reference(agent, lock, depot))  in lock_duplicate_pbn()
1391 		lock->increment_limit = increment_limit;  in lock_duplicate_pbn()
1397 	set_duplicate_lock(agent->hash_lock, lock);  in lock_duplicate_pbn()
1412 static void start_locking(struct hash_lock *lock, struct data_vio *agent)  in start_locking()  argument
1414 	VDO_ASSERT_LOG_ONLY(lock->duplicate_lock == NULL,  in start_locking()
1417 	lock->state = VDO_HASH_LOCK_LOCKING;  in start_locking()
1442 static void finish_writing(struct hash_lock *lock, struct data_vio *agent)  in finish_writing()  argument
1448 	lock->duplicate = agent->new_mapped;  in finish_writing()
1449 	lock->verified = true;  in finish_writing()
1451 	if (vdo_is_state_compressed(lock->duplicate.state) && lock->registered) {  in finish_writing()
1456 		lock->update_advice = true;  in finish_writing()
1460 	if (vdo_waitq_has_waiters(&lock->waiters)) {  in finish_writing()
1467 		start_deduping(lock, agent, true);  in finish_writing()
1475 	if (lock->update_advice) {  in finish_writing()
1481 		start_updating(lock, agent);  in finish_writing()
1482 	} else if (lock->duplicate_lock != NULL) {  in finish_writing()
1487 		set_duplicate_location(agent, lock->duplicate);  in finish_writing()
1488 		start_unlocking(lock, agent);  in finish_writing()
1495 		start_bypassing(lock, agent);  in finish_writing()
1506 static struct data_vio *select_writing_agent(struct hash_lock *lock)  in select_writing_agent()  argument
1517 	while (((data_vio = dequeue_lock_waiter(lock)) != NULL) &&  in select_writing_agent()
1528 		vdo_waitq_transfer_all_waiters(&lock->waiters, &temp_queue);  in select_writing_agent()
1534 		vdo_waitq_enqueue_waiter(&lock->waiters, &lock->agent->waiter);  in select_writing_agent()
1535 		lock->agent = data_vio;  in select_writing_agent()
1538 		data_vio = lock->agent;  in select_writing_agent()
1542 	vdo_waitq_transfer_all_waiters(&temp_queue, &lock->waiters);  in select_writing_agent()
1555 static void start_writing(struct hash_lock *lock, struct data_vio *agent)  in start_writing()  argument
1557 	lock->state = VDO_HASH_LOCK_WRITING;  in start_writing()
1564 		agent = select_writing_agent(lock);  in start_writing()
1583 	if (vdo_waitq_has_waiters(&lock->waiters))  in start_writing()
1666 	struct hash_lock *lock = agent->hash_lock;  in finish_querying()  local
1673 		lock->duplicate = agent->duplicate;  in finish_querying()
1679 		start_locking(lock, agent);  in finish_querying()
1685 		lock->update_advice = !data_vio_has_allocation(agent);  in finish_querying()
1690 		start_writing(lock, agent);  in finish_querying()
1703 static void start_querying(struct hash_lock *lock, struct data_vio *data_vio)  in start_querying()  argument
1705 	lock->agent = data_vio;  in start_querying()
1706 	lock->state = VDO_HASH_LOCK_QUERYING;  in start_querying()
1720 static void report_bogus_lock_state(struct hash_lock *lock, struct data_vio *data_vio)  in report_bogus_lock_state()  argument
1723 			    get_hash_lock_state_name(lock->state));  in report_bogus_lock_state()
1741 	struct hash_lock *lock = data_vio->hash_lock;  in vdo_continue_hash_lock()  local
1743 	switch (lock->state) {  in vdo_continue_hash_lock()
1745 		VDO_ASSERT_LOG_ONLY(data_vio == lock->agent,  in vdo_continue_hash_lock()
1747 		finish_writing(lock, data_vio);  in vdo_continue_hash_lock()
1751 		finish_deduping(lock, data_vio);  in vdo_continue_hash_lock()
1766 		report_bogus_lock_state(lock, data_vio);  in vdo_continue_hash_lock()
1770 		report_bogus_lock_state(lock, data_vio);  in vdo_continue_hash_lock()
1786 static bool is_hash_collision(struct hash_lock *lock, struct data_vio *candidate)  in is_hash_collision()  argument
1792 	if (list_empty(&lock->duplicate_ring))  in is_hash_collision()
1795 	lock_holder = list_first_entry(&lock->duplicate_ring, struct data_vio,  in is_hash_collision()
1838 	struct hash_lock *lock;  in vdo_acquire_hash_lock()  local
1849 	result = acquire_lock(data_vio->hash_zone, &data_vio->record_name, NULL, &lock);  in vdo_acquire_hash_lock()
1855 	if (is_hash_collision(lock, data_vio)) {  in vdo_acquire_hash_lock()
1866 	set_hash_lock(data_vio, lock);  in vdo_acquire_hash_lock()
1867 	switch (lock->state) {  in vdo_acquire_hash_lock()
1869 		start_querying(lock, data_vio);  in vdo_acquire_hash_lock()
1879 		wait_on_hash_lock(lock, data_vio);  in vdo_acquire_hash_lock()
1889 		launch_dedupe(lock, data_vio, false);  in vdo_acquire_hash_lock()
1894 		report_bogus_lock_state(lock, data_vio);  in vdo_acquire_hash_lock()
1912 	struct hash_lock *lock = data_vio->hash_lock;  in vdo_release_hash_lock()  local
1915 	if (lock == NULL)  in vdo_release_hash_lock()
1920 	if (lock->reference_count > 0) {  in vdo_release_hash_lock()
1925 	lock_key = hash_lock_key(lock);  in vdo_release_hash_lock()
1926 	if (lock->registered) {  in vdo_release_hash_lock()
1930 		VDO_ASSERT_LOG_ONLY(lock == removed,  in vdo_release_hash_lock()
1933 		VDO_ASSERT_LOG_ONLY(lock != vdo_int_map_get(zone->hash_lock_map, lock_key),  in vdo_release_hash_lock()
1937 	VDO_ASSERT_LOG_ONLY(!vdo_waitq_has_waiters(&lock->waiters),  in vdo_release_hash_lock()
1939 	VDO_ASSERT_LOG_ONLY((lock->duplicate_lock == NULL),  in vdo_release_hash_lock()
1941 	VDO_ASSERT_LOG_ONLY((lock->state == VDO_HASH_LOCK_BYPASSING),  in vdo_release_hash_lock()
1943 			    get_hash_lock_state_name(lock->state));  in vdo_release_hash_lock()
1944 	VDO_ASSERT_LOG_ONLY(list_empty(&lock->pool_node),  in vdo_release_hash_lock()
1946 	VDO_ASSERT_LOG_ONLY(list_empty(&lock->duplicate_ring),  in vdo_release_hash_lock()
1949 	return_hash_lock_to_pool(zone, lock);  in vdo_release_hash_lock()
1967 	VDO_ASSERT_LOG_ONLY(vdo_is_pbn_read_lock(allocation->lock),  in transfer_allocation_lock()
1977 	hash_lock->duplicate_lock = vdo_forget(allocation->lock);  in transfer_allocation_lock()
2039 	__must_hold(&zones->lock)  in close_index()
2049 	spin_unlock(&zones->lock);  in close_index()
2054 	spin_lock(&zones->lock);  in close_index()
2061 	__must_hold(&zones->lock)  in open_index()
2076 	spin_unlock(&zones->lock);  in open_index()
2082 	spin_lock(&zones->lock);  in open_index()
2103 		spin_unlock(&zones->lock);  in open_index()
2105 		spin_lock(&zones->lock);  in open_index()
2117 	spin_lock(&zones->lock);  in change_dedupe_state()
2129 	spin_unlock(&zones->lock);  in change_dedupe_state()
2155 	spin_lock(&zones->lock);  in report_dedupe_timeouts()
2164 	spin_unlock(&zones->lock);  in report_dedupe_timeouts()
2181 	spin_lock_init(&zones->lock);  in initialize_index()
2494 	spin_lock(&zones->lock);  in initiate_suspend_index()
2496 	spin_unlock(&zones->lock);  in initiate_suspend_index()
2557 	__must_hold(&zones->lock)  in launch_dedupe_state_change()
2589 	spin_lock(&zones->lock);  in resume_index()
2600 	spin_unlock(&zones->lock);  in resume_index()
2658 	spin_lock(&zones->lock);  in get_index_statistics()
2660 	spin_unlock(&zones->lock);  in get_index_statistics()
2739 static void dump_hash_lock(const struct hash_lock *lock)  in dump_hash_lock()  argument
2743 	if (!list_empty(&lock->pool_node)) {  in dump_hash_lock()
2752 	state = get_hash_lock_state_name(lock->state);  in dump_hash_lock()
2754 		     lock, state, (lock->registered ? 'D' : 'U'),  in dump_hash_lock()
2755 		     (unsigned long long) lock->duplicate.pbn,  in dump_hash_lock()
2756 		     lock->duplicate.state, lock->reference_count,  in dump_hash_lock()
2757 		     vdo_waitq_num_waiters(&lock->waiters), lock->agent);  in dump_hash_lock()
2806 	spin_lock(&zones->lock);  in vdo_dump_hash_zones()
2809 	spin_unlock(&zones->lock);  in vdo_dump_hash_zones()
2946 	spin_lock(&zones->lock);  in set_target_state()
2957 	spin_unlock(&zones->lock);  in set_target_state()
2967 	spin_lock(&zones->lock);  in vdo_get_dedupe_index_state_name()
2969 	spin_unlock(&zones->lock);  in vdo_get_dedupe_index_state_name()