170 	struct zloop_device *zlo = rq->q->queuedata;  in rq_zone_no()  local
172 	return blk_rq_pos(rq) >> zlo->zone_shift;  in rq_zone_no()
181 static inline void zloop_lru_rotate_open_zone(struct zloop_device *zlo,  in zloop_lru_rotate_open_zone()  argument
184 	if (zlo->max_open_zones) {  in zloop_lru_rotate_open_zone()
185 		spin_lock(&zlo->open_zones_lock);  in zloop_lru_rotate_open_zone()
187 			       &zlo->open_zones_lru_list);  in zloop_lru_rotate_open_zone()
188 		spin_unlock(&zlo->open_zones_lock);  in zloop_lru_rotate_open_zone()
192 static inline void zloop_lru_remove_open_zone(struct zloop_device *zlo,  in zloop_lru_remove_open_zone()  argument
197 		spin_lock(&zlo->open_zones_lock);  in zloop_lru_remove_open_zone()
199 		zlo->nr_open_zones--;  in zloop_lru_remove_open_zone()
200 		spin_unlock(&zlo->open_zones_lock);  in zloop_lru_remove_open_zone()
204 static inline bool zloop_can_open_zone(struct zloop_device *zlo)  in zloop_can_open_zone()  argument
206 	return !zlo->max_open_zones || zlo->nr_open_zones < zlo->max_open_zones;  in zloop_can_open_zone()
214 static bool zloop_close_imp_open_zone(struct zloop_device *zlo)  in zloop_close_imp_open_zone()  argument
218 	lockdep_assert_held(&zlo->open_zones_lock);  in zloop_close_imp_open_zone()
220 	if (zloop_can_open_zone(zlo))  in zloop_close_imp_open_zone()
223 	list_for_each_entry(zone, &zlo->open_zones_lru_list, open_zone_entry) {  in zloop_close_imp_open_zone()
227 			zlo->nr_open_zones--;  in zloop_close_imp_open_zone()
235 static bool zloop_open_closed_or_empty_zone(struct zloop_device *zlo,  in zloop_open_closed_or_empty_zone()  argument
239 	spin_lock(&zlo->open_zones_lock);  in zloop_open_closed_or_empty_zone()
246 		if (!zloop_can_open_zone(zlo))  in zloop_open_closed_or_empty_zone()
254 		if (!zloop_close_imp_open_zone(zlo))  in zloop_open_closed_or_empty_zone()
259 	zlo->nr_open_zones++;  in zloop_open_closed_or_empty_zone()
261 		      &zlo->open_zones_lru_list);  in zloop_open_closed_or_empty_zone()
263 	spin_unlock(&zlo->open_zones_lock);  in zloop_open_closed_or_empty_zone()
268 	spin_unlock(&zlo->open_zones_lock);  in zloop_open_closed_or_empty_zone()
273 static bool zloop_do_open_zone(struct zloop_device *zlo,  in zloop_do_open_zone()  argument
281 		zloop_lru_rotate_open_zone(zlo, zone);  in zloop_do_open_zone()
285 		return zloop_open_closed_or_empty_zone(zlo, zone, explicit);  in zloop_do_open_zone()
291 static void zloop_mark_full(struct zloop_device *zlo, struct zloop_zone *zone)  in zloop_mark_full()  argument
295 	zloop_lru_remove_open_zone(zlo, zone);  in zloop_mark_full()
300 static void zloop_mark_empty(struct zloop_device *zlo, struct zloop_zone *zone)  in zloop_mark_empty()  argument
304 	zloop_lru_remove_open_zone(zlo, zone);  in zloop_mark_empty()
309 static int zloop_update_seq_zone(struct zloop_device *zlo, unsigned int zone_no)  in zloop_update_seq_zone()  argument
311 	struct zloop_zone *zone = &zlo->zones[zone_no];  in zloop_update_seq_zone()
327 	if (file_sectors > zlo->zone_capacity) {  in zloop_update_seq_zone()
329 		       zone_no, file_sectors, zlo->zone_capacity);  in zloop_update_seq_zone()
333 	if (!IS_ALIGNED(stat.size, zlo->block_size)) {  in zloop_update_seq_zone()
335 		       zone_no, stat.size, zlo->block_size);  in zloop_update_seq_zone()
341 		zloop_mark_empty(zlo, zone);  in zloop_update_seq_zone()
342 	} else if (file_sectors == zlo->zone_capacity) {  in zloop_update_seq_zone()
343 		zloop_mark_full(zlo, zone);  in zloop_update_seq_zone()
355 static int zloop_open_zone(struct zloop_device *zlo, unsigned int zone_no)  in zloop_open_zone()  argument
357 	struct zloop_zone *zone = &zlo->zones[zone_no];  in zloop_open_zone()
366 		ret = zloop_update_seq_zone(zlo, zone_no);  in zloop_open_zone()
371 	if (!zloop_do_open_zone(zlo, zone, true))  in zloop_open_zone()
380 static int zloop_close_zone(struct zloop_device *zlo, unsigned int zone_no)  in zloop_close_zone()  argument
382 	struct zloop_zone *zone = &zlo->zones[zone_no];  in zloop_close_zone()
391 		ret = zloop_update_seq_zone(zlo, zone_no);  in zloop_close_zone()
402 		zloop_lru_remove_open_zone(zlo, zone);  in zloop_close_zone()
422 static int zloop_reset_zone(struct zloop_device *zlo, unsigned int zone_no)  in zloop_reset_zone()  argument
424 	struct zloop_zone *zone = &zlo->zones[zone_no];  in zloop_reset_zone()
443 	zloop_mark_empty(zlo, zone);  in zloop_reset_zone()
453 static int zloop_reset_all_zones(struct zloop_device *zlo)  in zloop_reset_all_zones()  argument
458 	for (i = zlo->nr_conv_zones; i < zlo->nr_zones; i++) {  in zloop_reset_all_zones()
459 		ret = zloop_reset_zone(zlo, i);  in zloop_reset_all_zones()
467 static int zloop_finish_zone(struct zloop_device *zlo, unsigned int zone_no)  in zloop_finish_zone()  argument
469 	struct zloop_zone *zone = &zlo->zones[zone_no];  in zloop_finish_zone()
481 	if (vfs_truncate(&zone->file->f_path, zlo->zone_size << SECTOR_SHIFT)) {  in zloop_finish_zone()
488 	zloop_mark_full(zlo, zone);  in zloop_finish_zone()
523 	struct zloop_device *zlo = rq->q->queuedata;  in zloop_do_rw()  local
524 	struct zloop_zone *zone = &zlo->zones[rq_zone_no(rq)];  in zloop_do_rw()
562 	if (!zlo->buffered_io)  in zloop_do_rw()
574 	struct zloop_device *zlo = rq->q->queuedata;  in zloop_seq_write_prep()  local
578 	struct zloop_zone *zone = &zlo->zones[zone_no];  in zloop_seq_write_prep()
579 	sector_t zone_end = zone->start + zlo->zone_capacity;  in zloop_seq_write_prep()
594 		if (!zlo->ordered_zone_append) {  in zloop_seq_write_prep()
612 	if (!zloop_do_open_zone(zlo, zone, false)) {  in zloop_seq_write_prep()
622 	if (!is_append || !zlo->ordered_zone_append) {  in zloop_seq_write_prep()
625 			zloop_mark_full(zlo, zone);  in zloop_seq_write_prep()
635 	struct zloop_device *zlo = rq->q->queuedata;  in zloop_rw()  local
648 	if (WARN_ON_ONCE(is_append && !zlo->zone_append))  in zloop_rw()
652 	if (WARN_ON_ONCE(zone_no >= zlo->nr_zones))  in zloop_rw()
655 	zone = &zlo->zones[zone_no];  in zloop_rw()
662 			 zone->start + zlo->zone_size))  in zloop_rw()
667 		ret = zloop_update_seq_zone(zlo, zone_no);  in zloop_rw()
700 static int zloop_record_safe_wps(struct zloop_device *zlo)  in zloop_record_safe_wps()  argument
705 	for (i = 0; i < zlo->nr_zones; i++) {  in zloop_record_safe_wps()
706 		struct zloop_zone *zone = &zlo->zones[i];  in zloop_record_safe_wps()
715 				zlo->disk->part0, ret);  in zloop_record_safe_wps()
726 static int zloop_flush(struct zloop_device *zlo)  in zloop_flush()  argument
728 	struct super_block *sb = file_inode(zlo->data_dir)->i_sb;  in zloop_flush()
731 	if (zlo->discard_write_cache) {  in zloop_flush()
732 		ret = zloop_record_safe_wps(zlo);  in zloop_flush()
747 	struct zloop_device *zlo = rq->q->queuedata;  in zloop_handle_cmd()  local
764 		cmd->ret = zloop_flush(zlo);  in zloop_handle_cmd()
767 		cmd->ret = zloop_reset_zone(zlo, rq_zone_no(rq));  in zloop_handle_cmd()
770 		cmd->ret = zloop_reset_all_zones(zlo);  in zloop_handle_cmd()
773 		cmd->ret = zloop_finish_zone(zlo, rq_zone_no(rq));  in zloop_handle_cmd()
776 		cmd->ret = zloop_open_zone(zlo, rq_zone_no(rq));  in zloop_handle_cmd()
779 		cmd->ret = zloop_close_zone(zlo, rq_zone_no(rq));  in zloop_handle_cmd()
804 	struct zloop_device *zlo = rq->q->queuedata;  in zloop_complete_rq()  local
805 	unsigned int zone_no = cmd->sector >> zlo->zone_shift;  in zloop_complete_rq()
806 	struct zloop_zone *zone = &zlo->zones[zone_no];  in zloop_complete_rq()
861 	struct zloop_device *zlo = rq->q->queuedata;  in zloop_set_zone_append_sector()  local
863 	struct zloop_zone *zone = &zlo->zones[zone_no];  in zloop_set_zone_append_sector()
864 	sector_t zone_end = zone->start + zlo->zone_capacity;  in zloop_set_zone_append_sector()
878 		zloop_mark_full(zlo, zone);  in zloop_set_zone_append_sector()
890 	struct zloop_device *zlo = rq->q->queuedata;  in zloop_queue_rq()  local
892 	if (data_race(READ_ONCE(zlo->state)) == Zlo_deleting) {  in zloop_queue_rq()
902 	if (zlo->ordered_zone_append && req_op(rq) == REQ_OP_ZONE_APPEND) {  in zloop_queue_rq()
910 	queue_work(zlo->workqueue, &cmd->work);  in zloop_queue_rq()
922 	struct zloop_device *zlo = disk->private_data;  in zloop_open()  local
929 	if (zlo->state != Zlo_live)  in zloop_open()
938 	struct zloop_device *zlo = disk->private_data;  in zloop_report_zones()  local
944 	if (first >= zlo->nr_zones)  in zloop_report_zones()
946 	nr_zones = min(nr_zones, zlo->nr_zones - first);  in zloop_report_zones()
950 		struct zloop_zone *zone = &zlo->zones[zone_no];  in zloop_report_zones()
955 			ret = zloop_update_seq_zone(zlo, zone_no);  in zloop_report_zones()
963 		blkz.len = zlo->zone_size;  in zloop_report_zones()
970 			blkz.capacity = zlo->zone_size;  in zloop_report_zones()
973 			blkz.capacity = zlo->zone_capacity;  in zloop_report_zones()
988 	struct zloop_device *zlo = disk->private_data;  in zloop_free_disk()  local
991 	blk_mq_free_tag_set(&zlo->tag_set);  in zloop_free_disk()
993 	for (i = 0; i < zlo->nr_zones; i++) {  in zloop_free_disk()
994 		struct zloop_zone *zone = &zlo->zones[i];  in zloop_free_disk()
1001 	fput(zlo->data_dir);  in zloop_free_disk()
1002 	destroy_workqueue(zlo->workqueue);  in zloop_free_disk()
1003 	kfree(zlo->base_dir);  in zloop_free_disk()
1004 	kvfree(zlo);  in zloop_free_disk()
1033 static int zloop_get_block_size(struct zloop_device *zlo,  in zloop_get_block_size()  argument
1046 		zlo->block_size = file_inode(zone->file)->i_sb->s_blocksize;  in zloop_get_block_size()
1049 		zlo->block_size = st.dio_offset_align;  in zloop_get_block_size()
1051 		zlo->block_size = bdev_physical_block_size(sb_bdev);  in zloop_get_block_size()
1053 		zlo->block_size = SECTOR_SIZE;  in zloop_get_block_size()
1055 	if (zlo->zone_capacity & ((zlo->block_size >> SECTOR_SHIFT) - 1)) {  in zloop_get_block_size()
1057 		       zlo->block_size);  in zloop_get_block_size()
1064 static int zloop_init_zone(struct zloop_device *zlo, struct zloop_options *opts,  in zloop_init_zone()  argument
1067 	struct zloop_zone *zone = &zlo->zones[zone_no];  in zloop_init_zone()
1076 	zone->start = (sector_t)zone_no << zlo->zone_shift;  in zloop_init_zone()
1084 	if (zone_no < zlo->nr_conv_zones) {  in zloop_init_zone()
1091 					zlo->base_dir, zlo->id, zone_no);  in zloop_init_zone()
1094 			       zone_no, zlo->base_dir, zlo->id, zone_no,  in zloop_init_zone()
1099 		if (!zlo->block_size) {  in zloop_init_zone()
1100 			ret = zloop_get_block_size(zlo, zone);  in zloop_init_zone()
1112 		if (restore && file_sectors != zlo->zone_size) {  in zloop_init_zone()
1114 			       zone_no, file_sectors, zlo->zone_capacity);  in zloop_init_zone()
1119 				   zlo->zone_size << SECTOR_SHIFT);  in zloop_init_zone()
1131 					 zlo->base_dir, zlo->id, zone_no);  in zloop_init_zone()
1134 		       zone_no, zlo->base_dir, zlo->id, zone_no,  in zloop_init_zone()
1139 	if (!zlo->block_size) {  in zloop_init_zone()
1140 		ret = zloop_get_block_size(zlo, zone);  in zloop_init_zone()
1145 	zloop_get_block_size(zlo, zone);  in zloop_init_zone()
1148 	ret = zloop_update_seq_zone(zlo, zone_no);  in zloop_init_zone()
1154 static bool zloop_dev_exists(struct zloop_device *zlo)  in zloop_dev_exists()  argument
1160 				  zlo->base_dir, zlo->id, 0);  in zloop_dev_exists()
1162 				  zlo->base_dir, zlo->id, 0);  in zloop_dev_exists()
1182 	struct zloop_device *zlo;  in zloop_ctl_add()  local
1201 	zlo = kvzalloc_flex(*zlo, zones, nr_zones);  in zloop_ctl_add()
1202 	if (!zlo) {  in zloop_ctl_add()
1206 	WRITE_ONCE(zlo->state, Zlo_creating);  in zloop_ctl_add()
1207 	spin_lock_init(&zlo->open_zones_lock);  in zloop_ctl_add()
1208 	INIT_LIST_HEAD(&zlo->open_zones_lru_list);  in zloop_ctl_add()
1216 		ret = idr_alloc(&zloop_index_idr, zlo,  in zloop_ctl_add()
1221 		ret = idr_alloc(&zloop_index_idr, zlo, 0, 0, GFP_KERNEL);  in zloop_ctl_add()
1227 	zlo->id = ret;  in zloop_ctl_add()
1228 	zlo->zone_shift = ilog2(opts->zone_size);  in zloop_ctl_add()
1229 	zlo->zone_size = opts->zone_size;  in zloop_ctl_add()
1231 		zlo->zone_capacity = opts->zone_capacity;  in zloop_ctl_add()
1233 		zlo->zone_capacity = zlo->zone_size;  in zloop_ctl_add()
1234 	zlo->nr_zones = nr_zones;  in zloop_ctl_add()
1235 	zlo->nr_conv_zones = opts->nr_conv_zones;  in zloop_ctl_add()
1236 	zlo->max_open_zones = opts->max_open_zones;  in zloop_ctl_add()
1237 	zlo->buffered_io = opts->buffered_io;  in zloop_ctl_add()
1238 	zlo->zone_append = opts->zone_append;  in zloop_ctl_add()
1239 	if (zlo->zone_append)  in zloop_ctl_add()
1240 		zlo->ordered_zone_append = opts->ordered_zone_append;  in zloop_ctl_add()
1241 	zlo->discard_write_cache = opts->discard_write_cache;  in zloop_ctl_add()
1243 	zlo->workqueue = alloc_workqueue("zloop%d", WQ_UNBOUND | WQ_FREEZABLE,  in zloop_ctl_add()
1244 				opts->nr_queues * opts->queue_depth, zlo->id);  in zloop_ctl_add()
1245 	if (!zlo->workqueue) {  in zloop_ctl_add()
1251 		zlo->base_dir = kstrdup(opts->base_dir, GFP_KERNEL);  in zloop_ctl_add()
1253 		zlo->base_dir = kstrdup(ZLOOP_DEF_BASE_DIR, GFP_KERNEL);  in zloop_ctl_add()
1254 	if (!zlo->base_dir) {  in zloop_ctl_add()
1259 	zlo->data_dir = zloop_filp_open_fmt(O_RDONLY | O_DIRECTORY, 0, "%s/%u",  in zloop_ctl_add()
1260 					    zlo->base_dir, zlo->id);  in zloop_ctl_add()
1261 	if (IS_ERR(zlo->data_dir)) {  in zloop_ctl_add()
1262 		ret = PTR_ERR(zlo->data_dir);  in zloop_ctl_add()
1264 			zlo->base_dir, zlo->id, ret);  in zloop_ctl_add()
1273 	restore = zloop_dev_exists(zlo);  in zloop_ctl_add()
1275 		ret = zloop_init_zone(zlo, opts, i, restore);  in zloop_ctl_add()
1280 	lim.physical_block_size = zlo->block_size;  in zloop_ctl_add()
1281 	lim.logical_block_size = zlo->block_size;  in zloop_ctl_add()
1282 	if (zlo->zone_append)  in zloop_ctl_add()
1284 	lim.max_open_zones = zlo->max_open_zones;  in zloop_ctl_add()
1286 	zlo->tag_set.ops = &zloop_mq_ops;  in zloop_ctl_add()
1287 	zlo->tag_set.nr_hw_queues = opts->nr_queues;  in zloop_ctl_add()
1288 	zlo->tag_set.queue_depth = opts->queue_depth;  in zloop_ctl_add()
1289 	zlo->tag_set.numa_node = NUMA_NO_NODE;  in zloop_ctl_add()
1290 	zlo->tag_set.cmd_size = sizeof(struct zloop_cmd);  in zloop_ctl_add()
1291 	zlo->tag_set.driver_data = zlo;  in zloop_ctl_add()
1293 	ret = blk_mq_alloc_tag_set(&zlo->tag_set);  in zloop_ctl_add()
1299 	zlo->disk = blk_mq_alloc_disk(&zlo->tag_set, &lim, zlo);  in zloop_ctl_add()
1300 	if (IS_ERR(zlo->disk)) {  in zloop_ctl_add()
1302 		ret = PTR_ERR(zlo->disk);  in zloop_ctl_add()
1305 	zlo->disk->flags = GENHD_FL_NO_PART;  in zloop_ctl_add()
1306 	zlo->disk->fops = &zloop_fops;  in zloop_ctl_add()
1307 	zlo->disk->private_data = zlo;  in zloop_ctl_add()
1308 	sprintf(zlo->disk->disk_name, "zloop%d", zlo->id);  in zloop_ctl_add()
1309 	set_capacity(zlo->disk, (u64)lim.chunk_sectors * zlo->nr_zones);  in zloop_ctl_add()
1311 	ret = blk_revalidate_disk_zones(zlo->disk);  in zloop_ctl_add()
1315 	ret = add_disk(zlo->disk);  in zloop_ctl_add()
1322 	WRITE_ONCE(zlo->state, Zlo_live);  in zloop_ctl_add()
1326 		zlo->id, zlo->nr_zones,  in zloop_ctl_add()
1327 		((sector_t)zlo->zone_size << SECTOR_SHIFT) >> 20,  in zloop_ctl_add()
1328 		zlo->block_size);  in zloop_ctl_add()
1330 		zlo->id,  in zloop_ctl_add()
1331 		zlo->ordered_zone_append ? "ordered " : "",  in zloop_ctl_add()
1332 		zlo->zone_append ? "native" : "emulated");  in zloop_ctl_add()
1337 	put_disk(zlo->disk);  in zloop_ctl_add()
1339 	blk_mq_free_tag_set(&zlo->tag_set);  in zloop_ctl_add()
1342 		struct zloop_zone *zone = &zlo->zones[j];  in zloop_ctl_add()
1347 	fput(zlo->data_dir);  in zloop_ctl_add()
1349 	kfree(zlo->base_dir);  in zloop_ctl_add()
1351 	destroy_workqueue(zlo->workqueue);  in zloop_ctl_add()
1354 	idr_remove(&zloop_index_idr, zlo->id);  in zloop_ctl_add()
1357 	kvfree(zlo);  in zloop_ctl_add()
1365 static void zloop_forget_cache(struct zloop_device *zlo)  in zloop_forget_cache()  argument
1370 	pr_info("%pg: discarding volatile write cache\n", zlo->disk->part0);  in zloop_forget_cache()
1372 	for (i = 0; i < zlo->nr_zones; i++) {  in zloop_forget_cache()
1373 		struct zloop_zone *zone = &zlo->zones[i];  in zloop_forget_cache()
1386 				zlo->disk->part0, ret);  in zloop_forget_cache()
1406 	struct zloop_device *zlo;  in zloop_ctl_remove()  local
1423 	zlo = idr_find(&zloop_index_idr, opts->id);  in zloop_ctl_remove()
1424 	if (!zlo || zlo->state == Zlo_creating) {  in zloop_ctl_remove()
1426 	} else if (zlo->state == Zlo_deleting) {  in zloop_ctl_remove()
1429 		idr_remove(&zloop_index_idr, zlo->id);  in zloop_ctl_remove()
1430 		WRITE_ONCE(zlo->state, Zlo_deleting);  in zloop_ctl_remove()
1437 	del_gendisk(zlo->disk);  in zloop_ctl_remove()
1439 	if (zlo->discard_write_cache)  in zloop_ctl_remove()
1440 		zloop_forget_cache(zlo);  in zloop_ctl_remove()
1442 	put_disk(zlo->disk);  in zloop_ctl_remove()