227 typedef void (*process_cell_fn)(struct thin_c *tc, struct dm_bio_prison_cell *cell);
468 			 struct dm_bio_prison_cell *cell,  in cell_release()  argument
471 	dm_cell_release(pool->prison, cell, bios);  in cell_release()
472 	dm_bio_prison_free_cell(pool->prison, cell);  in cell_release()
478 			       struct dm_bio_prison_cell *cell)  in cell_visit_release()  argument
480 	dm_cell_visit_release(pool->prison, fn, context, cell);  in cell_visit_release()
481 	dm_bio_prison_free_cell(pool->prison, cell);  in cell_visit_release()
485 				   struct dm_bio_prison_cell *cell,  in cell_release_no_holder()  argument
488 	dm_cell_release_no_holder(pool->prison, cell, bios);  in cell_release_no_holder()
489 	dm_bio_prison_free_cell(pool->prison, cell);  in cell_release_no_holder()
493 		struct dm_bio_prison_cell *cell, blk_status_t error_code)  in cell_error_with_code()  argument
495 	dm_cell_error(pool->prison, cell, error_code);  in cell_error_with_code()
496 	dm_bio_prison_free_cell(pool->prison, cell);  in cell_error_with_code()
504 static void cell_error(struct pool *pool, struct dm_bio_prison_cell *cell)  in cell_error()  argument
506 	cell_error_with_code(pool, cell, get_pool_io_error_code(pool));  in cell_error()
509 static void cell_success(struct pool *pool, struct dm_bio_prison_cell *cell)  in cell_success()  argument
511 	cell_error_with_code(pool, cell, 0);  in cell_success()
514 static void cell_requeue(struct pool *pool, struct dm_bio_prison_cell *cell)  in cell_requeue()  argument
516 	cell_error_with_code(pool, cell, BLK_STS_DM_REQUEUE);  in cell_requeue()
592 	struct dm_bio_prison_cell *cell;  member
623 	struct dm_bio_prison_cell *cell, *tmp;  in requeue_deferred_cells()  local
631 	list_for_each_entry_safe(cell, tmp, &cells, user_list)  in requeue_deferred_cells()
632 		cell_requeue(pool, cell);  in requeue_deferred_cells()
815 	struct dm_bio_prison_cell *cell;  member
880 static void cell_defer_no_holder(struct thin_c *tc, struct dm_bio_prison_cell *cell)  in cell_defer_no_holder()  argument
887 	cell_release_no_holder(pool, cell, &bios);  in cell_defer_no_holder()
906 				       struct dm_bio_prison_cell *cell)  in __inc_remap_and_issue_cell()  argument
911 	while ((bio = bio_list_pop(&cell->bios))) {  in __inc_remap_and_issue_cell()
928 				     struct dm_bio_prison_cell *cell,  in inc_remap_and_issue_cell()  argument
944 			   &info, cell);  in inc_remap_and_issue_cell()
955 	cell_error(m->tc->pool, m->cell);  in process_prepared_mapping_fail()
1000 		cell_error(pool, m->cell);  in process_prepared_mapping()
1012 		cell_error(pool, m->cell);  in process_prepared_mapping()
1023 		inc_remap_and_issue_cell(tc, m->cell, m->data_block);  in process_prepared_mapping()
1026 		inc_all_io_entry(tc->pool, m->cell->holder);  in process_prepared_mapping()
1027 		remap_and_issue(tc, m->cell->holder, m->data_block);  in process_prepared_mapping()
1028 		inc_remap_and_issue_cell(tc, m->cell, m->data_block);  in process_prepared_mapping()
1042 	if (m->cell)  in free_discard_mapping()
1043 		cell_defer_no_holder(tc, m->cell);  in free_discard_mapping()
1064 	r = dm_thin_remove_range(tc->td, m->cell->key.block_begin, m->cell->key.block_end);  in process_prepared_discard_no_passdown()
1071 	cell_defer_no_holder(tc, m->cell);  in process_prepared_discard_no_passdown()
1162 		cell_defer_no_holder(tc, m->cell);  in process_prepared_discard_passdown_pt1()
1175 		cell_defer_no_holder(tc, m->cell);  in process_prepared_discard_passdown_pt1()
1212 	cell_defer_no_holder(tc, m->cell);  in process_prepared_discard_passdown_pt2()
1310 			  struct dm_bio_prison_cell *cell, struct bio *bio,  in schedule_copy()  argument
1320 	m->cell = cell;  in schedule_copy()
1370 				   struct dm_bio_prison_cell *cell, struct bio *bio)  in schedule_internal_copy()  argument
1373 		      data_origin, data_dest, cell, bio,  in schedule_internal_copy()
1378 			  dm_block_t data_block, struct dm_bio_prison_cell *cell,  in schedule_zero()  argument
1389 	m->cell = cell;  in schedule_zero()
1409 				   struct dm_bio_prison_cell *cell, struct bio *bio)  in schedule_external_copy()  argument
1417 			      virt_block, data_dest, cell, bio,  in schedule_external_copy()
1422 			      virt_block, data_dest, cell, bio,  in schedule_external_copy()
1426 		schedule_zero(tc, virt_block, data_dest, cell, bio);  in schedule_external_copy()
1625 static void retry_bios_on_resume(struct pool *pool, struct dm_bio_prison_cell *cell)  in retry_bios_on_resume()  argument
1633 		cell_error_with_code(pool, cell, error);  in retry_bios_on_resume()
1638 	cell_release(pool, cell, &bios);  in retry_bios_on_resume()
1657 	m->cell = virt_cell;  in process_discard_cell_no_passdown()
1720 			m->cell = data_cell;  in break_up_discard_bio()
1753 	h->cell = virt_cell;  in process_discard_cell_passdown()
1802 			  struct dm_bio_prison_cell *cell)  in break_sharing()  argument
1812 				       data_block, cell, bio);  in break_sharing()
1816 		retry_bios_on_resume(pool, cell);  in break_sharing()
1822 		cell_error(pool, cell);  in break_sharing()
1828 					  struct dm_bio_prison_cell *cell)  in __remap_and_issue_shared_cell()  argument
1833 	while ((bio = bio_list_pop(&cell->bios))) {  in __remap_and_issue_shared_cell()
1848 					struct dm_bio_prison_cell *cell,  in remap_and_issue_shared_cell()  argument
1859 			   &info, cell);  in remap_and_issue_shared_cell()
1903 			    struct dm_bio_prison_cell *cell)  in provision_block()  argument
1914 		cell_defer_no_holder(tc, cell);  in provision_block()
1925 		cell_defer_no_holder(tc, cell);  in provision_block()
1934 			schedule_external_copy(tc, block, data_block, cell, bio);  in provision_block()
1936 			schedule_zero(tc, block, data_block, cell, bio);  in provision_block()
1940 		retry_bios_on_resume(pool, cell);  in provision_block()
1946 		cell_error(pool, cell);  in provision_block()
1951 static void process_cell(struct thin_c *tc, struct dm_bio_prison_cell *cell)  in process_cell()  argument
1955 	struct bio *bio = cell->holder;  in process_cell()
1960 		cell_requeue(pool, cell);  in process_cell()
1968 			process_shared_bio(tc, bio, block, &lookup_result, cell);  in process_cell()
1972 			inc_remap_and_issue_cell(tc, cell, lookup_result.block);  in process_cell()
1979 			cell_defer_no_holder(tc, cell);  in process_cell()
1994 			provision_block(tc, bio, block, cell);  in process_cell()
2000 		cell_defer_no_holder(tc, cell);  in process_cell()
2010 	struct dm_bio_prison_cell *cell;  in process_bio()  local
2018 	if (bio_detain(pool, &key, bio, &cell))  in process_bio()
2021 	process_cell(tc, cell);  in process_bio()
2025 				    struct dm_bio_prison_cell *cell)  in __process_bio_read_only()  argument
2037 			if (cell)  in __process_bio_read_only()
2038 				cell_defer_no_holder(tc, cell);  in __process_bio_read_only()
2042 			if (cell)  in __process_bio_read_only()
2043 				inc_remap_and_issue_cell(tc, cell, lookup_result.block);  in __process_bio_read_only()
2048 		if (cell)  in __process_bio_read_only()
2049 			cell_defer_no_holder(tc, cell);  in __process_bio_read_only()
2068 		if (cell)  in __process_bio_read_only()
2069 			cell_defer_no_holder(tc, cell);  in __process_bio_read_only()
2080 static void process_cell_read_only(struct thin_c *tc, struct dm_bio_prison_cell *cell)  in process_cell_read_only()  argument
2082 	__process_bio_read_only(tc, cell->holder, cell);  in process_cell_read_only()
2095 static void process_cell_success(struct thin_c *tc, struct dm_bio_prison_cell *cell)  in process_cell_success()  argument
2097 	cell_success(tc->pool, cell);  in process_cell_success()
2100 static void process_cell_fail(struct thin_c *tc, struct dm_bio_prison_cell *cell)  in process_cell_fail()  argument
2102 	cell_error(tc->pool, cell);  in process_cell_fail()
2258 	struct dm_bio_prison_cell *cell, *tmp;  in sort_cells()  local
2260 	list_for_each_entry_safe(cell, tmp, cells, user_list) {  in sort_cells()
2264 		pool->cell_sort_array[count++] = cell;  in sort_cells()
2265 		list_del(&cell->user_list);  in sort_cells()
2268 	sort(pool->cell_sort_array, count, sizeof(cell), cmp_cells, NULL);  in sort_cells()
2277 	struct dm_bio_prison_cell *cell;  in process_thin_deferred_cells()  local
2293 			cell = pool->cell_sort_array[i];  in process_thin_deferred_cells()
2294 			BUG_ON(!cell->holder);  in process_thin_deferred_cells()
2311 			if (bio_op(cell->holder) == REQ_OP_DISCARD)  in process_thin_deferred_cells()
2312 				pool->process_discard_cell(tc, cell);  in process_thin_deferred_cells()
2314 				pool->process_cell(tc, cell);  in process_thin_deferred_cells()
2692 static void thin_defer_cell(struct thin_c *tc, struct dm_bio_prison_cell *cell)  in thin_defer_cell()  argument
2698 	list_add_tail(&cell->user_list, &tc->deferred_cells);  in thin_defer_cell()
2713 	h->cell = NULL;  in thin_hook_bio()
4366 	if (h->cell)  in thin_endio()
4367 		cell_defer_no_holder(h->tc, h->cell);  in thin_endio()