477 	struct dm_cache_migration *mg;  in alloc_migration()  local
479 	mg = mempool_alloc(&cache->migration_pool, GFP_NOIO);  in alloc_migration()
481 	memset(mg, 0, sizeof(*mg));  in alloc_migration()
483 	mg->cache = cache;  in alloc_migration()
486 	return mg;  in alloc_migration()
489 static void free_migration(struct dm_cache_migration *mg)  in free_migration()  argument
491 	struct cache *cache = mg->cache;  in free_migration()
496 	mempool_free(mg, &cache->migration_pool);  in free_migration()
1070 static void quiesce(struct dm_cache_migration *mg,  in quiesce()  argument
1073 	init_continuation(&mg->k, continuation);  in quiesce()
1074 	dm_cell_quiesce_v2(mg->cache->prison, mg->cell, &mg->k.ws);  in quiesce()
1086 	struct dm_cache_migration *mg = container_of(context, struct dm_cache_migration, k);  in copy_complete()  local
1089 		mg->k.input = BLK_STS_IOERR;  in copy_complete()
1091 	queue_continuation(mg->cache->wq, &mg->k);  in copy_complete()
1094 static void copy(struct dm_cache_migration *mg, bool promote)  in copy()  argument
1097 	struct cache *cache = mg->cache;  in copy()
1100 	o_region.sector = from_oblock(mg->op->oblock) * cache->sectors_per_block;  in copy()
1104 	c_region.sector = from_cblock(mg->op->cblock) * cache->sectors_per_block;  in copy()
1108 		dm_kcopyd_copy(cache->copier, &o_region, 1, &c_region, 0, copy_complete, &mg->k);  in copy()
1110 		dm_kcopyd_copy(cache->copier, &c_region, 1, &o_region, 0, copy_complete, &mg->k);  in copy()
1124 	struct dm_cache_migration *mg = bio->bi_private;  in overwrite_endio()  local
1125 	struct cache *cache = mg->cache;  in overwrite_endio()
1131 		mg->k.input = bio->bi_status;  in overwrite_endio()
1133 	queue_continuation(cache->wq, &mg->k);  in overwrite_endio()
1136 static void overwrite(struct dm_cache_migration *mg,  in overwrite()  argument
1139 	struct bio *bio = mg->overwrite_bio;  in overwrite()
1142 	dm_hook_bio(&pb->hook_info, bio, overwrite_endio, mg);  in overwrite()
1148 	if (mg->op->op == POLICY_PROMOTE)  in overwrite()
1149 		remap_to_cache(mg->cache, bio, mg->op->cblock);  in overwrite()
1151 		remap_to_origin(mg->cache, bio);  in overwrite()
1153 	init_continuation(&mg->k, continuation);  in overwrite()
1154 	accounted_request(mg->cache, bio);  in overwrite()
1164  * 5) quiesce
1168 static void mg_complete(struct dm_cache_migration *mg, bool success)  in mg_complete()  argument
1171 	struct cache *cache = mg->cache;  in mg_complete()
1172 	struct policy_work *op = mg->op;  in mg_complete()
1183 		if (mg->overwrite_bio) {  in mg_complete()
1186 			else if (mg->k.input)  in mg_complete()
1187 				mg->overwrite_bio->bi_status = mg->k.input;  in mg_complete()
1189 				mg->overwrite_bio->bi_status = BLK_STS_IOERR;  in mg_complete()
1190 			bio_endio(mg->overwrite_bio);  in mg_complete()
1217 	if (mg->cell) {  in mg_complete()
1218 		if (dm_cell_unlock_v2(cache->prison, mg->cell, &bios))  in mg_complete()
1219 			free_prison_cell(cache, mg->cell);  in mg_complete()
1222 	free_migration(mg);  in mg_complete()
1231 	struct dm_cache_migration *mg = ws_to_mg(ws);  in mg_success()  local
1233 	mg_complete(mg, mg->k.input == 0);  in mg_success()
1239 	struct dm_cache_migration *mg = ws_to_mg(ws);  in mg_update_metadata()  local
1240 	struct cache *cache = mg->cache;  in mg_update_metadata()
1241 	struct policy_work *op = mg->op;  in mg_update_metadata()
1251 			mg_complete(mg, false);  in mg_update_metadata()
1254 		mg_complete(mg, true);  in mg_update_metadata()
1264 			mg_complete(mg, false);  in mg_update_metadata()
1287 		init_continuation(&mg->k, mg_success);  in mg_update_metadata()
1288 		continue_after_commit(&cache->committer, &mg->k);  in mg_update_metadata()
1293 		mg_complete(mg, true);  in mg_update_metadata()
1300 	struct dm_cache_migration *mg = ws_to_mg(ws);  in mg_update_metadata_after_copy()  local
1305 	if (mg->k.input)  in mg_update_metadata_after_copy()
1306 		mg_complete(mg, false);  in mg_update_metadata_after_copy()
1314 	struct dm_cache_migration *mg = ws_to_mg(ws);  in mg_upgrade_lock()  local
1319 	if (mg->k.input)  in mg_upgrade_lock()
1320 		mg_complete(mg, false);  in mg_upgrade_lock()
1326 		r = dm_cell_lock_promote_v2(mg->cache->prison, mg->cell,  in mg_upgrade_lock()
1329 			mg_complete(mg, false);  in mg_upgrade_lock()
1332 			quiesce(mg, mg_update_metadata);  in mg_upgrade_lock()
1341 	struct dm_cache_migration *mg = ws_to_mg(ws);  in mg_full_copy()  local
1342 	struct cache *cache = mg->cache;  in mg_full_copy()
1343 	struct policy_work *op = mg->op;  in mg_full_copy()
1352 	init_continuation(&mg->k, mg_upgrade_lock);  in mg_full_copy()
1353 	copy(mg, is_policy_promote);  in mg_full_copy()
1358 	struct dm_cache_migration *mg = ws_to_mg(ws);  in mg_copy()  local
1360 	if (mg->overwrite_bio) {  in mg_copy()
1366 		if (!optimisable_bio(mg->cache, mg->overwrite_bio, mg->op->oblock)) {  in mg_copy()
1370 			bool rb = bio_detain_shared(mg->cache, mg->op->oblock, mg->overwrite_bio);  in mg_copy()
1373 			mg->overwrite_bio = NULL;  in mg_copy()
1374 			inc_io_migrations(mg->cache);  in mg_copy()
1386 		overwrite(mg, mg_update_metadata_after_copy);  in mg_copy()
1392 static int mg_lock_writes(struct dm_cache_migration *mg)  in mg_lock_writes()  argument
1396 	struct cache *cache = mg->cache;  in mg_lock_writes()
1406 	build_key(mg->op->oblock, oblock_succ(mg->op->oblock), &key);  in mg_lock_writes()
1408 			    mg->overwrite_bio ?  READ_WRITE_LOCK_LEVEL : WRITE_LOCK_LEVEL,  in mg_lock_writes()
1409 			    prealloc, &mg->cell);  in mg_lock_writes()
1412 		mg_complete(mg, false);  in mg_lock_writes()
1416 	if (mg->cell != prealloc)  in mg_lock_writes()
1420 		mg_copy(&mg->k.ws);  in mg_lock_writes()
1422 		quiesce(mg, mg_copy);  in mg_lock_writes()
1429 	struct dm_cache_migration *mg;  in mg_start()  local
1436 	mg = alloc_migration(cache);  in mg_start()
1438 	mg->op = op;  in mg_start()
1439 	mg->overwrite_bio = bio;  in mg_start()
1444 	return mg_lock_writes(mg);  in mg_start()
1453 static void invalidate_complete(struct dm_cache_migration *mg, bool success)  in invalidate_complete()  argument
1456 	struct cache *cache = mg->cache;  in invalidate_complete()
1459 	if (dm_cell_unlock_v2(cache->prison, mg->cell, &bios))  in invalidate_complete()
1460 		free_prison_cell(cache, mg->cell);  in invalidate_complete()
1462 	if (!success && mg->overwrite_bio)  in invalidate_complete()
1463 		bio_io_error(mg->overwrite_bio);  in invalidate_complete()
1465 	free_migration(mg);  in invalidate_complete()
1473 	struct dm_cache_migration *mg = ws_to_mg(ws);  in invalidate_completed()  local
1475 	invalidate_complete(mg, !mg->k.input);  in invalidate_completed()
1506 	struct dm_cache_migration *mg = ws_to_mg(ws);  in invalidate_remove()  local
1507 	struct cache *cache = mg->cache;  in invalidate_remove()
1509 	r = invalidate_cblock(cache, mg->invalidate_cblock);  in invalidate_remove()
1511 		invalidate_complete(mg, false);  in invalidate_remove()
1515 	init_continuation(&mg->k, invalidate_completed);  in invalidate_remove()
1516 	continue_after_commit(&cache->committer, &mg->k);  in invalidate_remove()
1517 	remap_to_origin_clear_discard(cache, mg->overwrite_bio, mg->invalidate_oblock);  in invalidate_remove()
1518 	mg->overwrite_bio = NULL;  in invalidate_remove()
1522 static int invalidate_lock(struct dm_cache_migration *mg)  in invalidate_lock()  argument
1526 	struct cache *cache = mg->cache;  in invalidate_lock()
1531 	build_key(mg->invalidate_oblock, oblock_succ(mg->invalidate_oblock), &key);  in invalidate_lock()
1533 			    READ_WRITE_LOCK_LEVEL, prealloc, &mg->cell);  in invalidate_lock()
1536 		invalidate_complete(mg, false);  in invalidate_lock()
1540 	if (mg->cell != prealloc)  in invalidate_lock()
1544 		quiesce(mg, invalidate_remove);  in invalidate_lock()
1551 		init_continuation(&mg->k, invalidate_remove);  in invalidate_lock()
1552 		queue_work(cache->wq, &mg->k.ws);  in invalidate_lock()
1561 	struct dm_cache_migration *mg;  in invalidate_start()  local
1566 	mg = alloc_migration(cache);  in invalidate_start()
1568 	mg->overwrite_bio = bio;  in invalidate_start()
1569 	mg->invalidate_cblock = cblock;  in invalidate_start()
1570 	mg->invalidate_oblock = oblock;  in invalidate_start()
1572 	return invalidate_lock(mg);  in invalidate_start()