Lines Matching defs:lo
100 * @lo: struct loop_device
101 * @global: true if @lo is about to bind another "struct loop_device", false otherwise
109 static int loop_global_lock_killable(struct loop_device *lo, bool global)
118 err = mutex_lock_killable(&lo->lo_mutex);
127 * @lo: struct loop_device
128 * @global: true if @lo was about to bind another "struct loop_device", false otherwise
130 static void loop_global_unlock(struct loop_device *lo, bool global)
132 mutex_unlock(&lo->lo_mutex);
140 static loff_t lo_calculate_size(struct loop_device *lo, struct file *file)
162 if (lo->lo_offset > 0)
163 loopsize -= lo->lo_offset;
167 if (lo->lo_sizelimit > 0 && lo->lo_sizelimit < loopsize)
168 loopsize = lo->lo_sizelimit;
181 static bool lo_can_use_dio(struct loop_device *lo)
183 if (!(lo->lo_backing_file->f_mode & FMODE_CAN_ODIRECT))
185 if (queue_logical_block_size(lo->lo_queue) < lo->lo_min_dio_size)
187 if (lo->lo_offset & (lo->lo_min_dio_size - 1))
200 static inline void loop_update_dio(struct loop_device *lo)
202 lockdep_assert_held(&lo->lo_mutex);
203 WARN_ON_ONCE(lo->lo_state == Lo_bound &&
204 lo->lo_queue->mq_freeze_depth == 0);
206 if ((lo->lo_flags & LO_FLAGS_DIRECT_IO) && !lo_can_use_dio(lo))
207 lo->lo_flags &= ~LO_FLAGS_DIRECT_IO;
212 * @lo: struct loop_device to set the size for
218 static void loop_set_size(struct loop_device *lo, loff_t size)
220 if (!set_capacity_and_notify(lo->lo_disk, size))
221 kobject_uevent(&disk_to_dev(lo->lo_disk)->kobj, KOBJ_CHANGE);
224 static void loop_clear_limits(struct loop_device *lo, int mode)
226 struct queue_limits lim = queue_limits_start_update(lo->lo_queue);
243 queue_limits_commit_update(lo->lo_queue, &lim);
246 static int lo_fallocate(struct loop_device *lo, struct request *rq, loff_t pos,
253 struct file *file = lo->lo_backing_file;
258 if (!bdev_max_discard_sectors(lo->lo_device))
270 loop_clear_limits(lo, mode);
275 static int lo_req_flush(struct loop_device *lo, struct request *rq)
277 int ret = vfs_fsync(lo->lo_backing_file, 0);
340 static int lo_rw_aio(struct loop_device *lo, struct loop_cmd *cmd,
346 struct file *file = lo->lo_backing_file;
409 static int do_req_filebacked(struct loop_device *lo, struct request *rq)
412 loff_t pos = ((loff_t) blk_rq_pos(rq) << 9) + lo->lo_offset;
416 return lo_req_flush(lo, rq);
422 return lo_fallocate(lo, rq, pos,
427 return lo_fallocate(lo, rq, pos, FALLOC_FL_PUNCH_HOLE);
429 return lo_rw_aio(lo, cmd, pos, ITER_SOURCE);
431 return lo_rw_aio(lo, cmd, pos, ITER_DEST);
438 static void loop_reread_partitions(struct loop_device *lo)
442 mutex_lock(&lo->lo_disk->open_mutex);
443 rc = bdev_disk_changed(lo->lo_disk, false);
444 mutex_unlock(&lo->lo_disk->open_mutex);
447 __func__, lo->lo_number, lo->lo_file_name, rc);
450 static unsigned int loop_query_min_dio_size(struct loop_device *lo)
452 struct file *file = lo->lo_backing_file;
495 /* Order wrt setting lo->lo_backing_file in loop_configure(). */
504 static void loop_assign_backing_file(struct loop_device *lo, struct file *file)
506 lo->lo_backing_file = file;
507 lo->old_gfp_mask = mapping_gfp_mask(file->f_mapping);
509 lo->old_gfp_mask & ~(__GFP_IO | __GFP_FS));
510 if (lo->lo_backing_file->f_flags & O_DIRECT)
511 lo->lo_flags |= LO_FLAGS_DIRECT_IO;
512 lo->lo_min_dio_size = loop_query_min_dio_size(lo);
534 static int loop_change_fd(struct loop_device *lo, struct block_device *bdev,
554 dev_set_uevent_suppress(disk_to_dev(lo->lo_disk), 1);
557 error = loop_global_lock_killable(lo, is_loop);
561 if (lo->lo_state != Lo_bound)
566 if (!(lo->lo_flags & LO_FLAGS_READ_ONLY))
573 old_file = lo->lo_backing_file;
578 if (lo_calculate_size(lo, file) != lo_calculate_size(lo, old_file))
589 disk_force_media_change(lo->lo_disk);
590 memflags = blk_mq_freeze_queue(lo->lo_queue);
591 mapping_set_gfp_mask(old_file->f_mapping, lo->old_gfp_mask);
592 loop_assign_backing_file(lo, file);
593 loop_update_dio(lo);
594 blk_mq_unfreeze_queue(lo->lo_queue, memflags);
595 partscan = lo->lo_flags & LO_FLAGS_PARTSCAN;
596 loop_global_unlock(lo, is_loop);
612 dev_set_uevent_suppress(disk_to_dev(lo->lo_disk), 0);
614 loop_reread_partitions(lo);
618 kobject_uevent(&disk_to_dev(lo->lo_disk)->kobj, KOBJ_CHANGE);
622 loop_global_unlock(lo, is_loop);
625 dev_set_uevent_suppress(disk_to_dev(lo->lo_disk), 0);
635 struct loop_device *lo = disk->private_data;
637 return callback(lo, page);
650 static ssize_t loop_attr_backing_file_show(struct loop_device *lo, char *buf)
655 spin_lock_irq(&lo->lo_lock);
656 if (lo->lo_backing_file)
657 p = file_path(lo->lo_backing_file, buf, PAGE_SIZE - 1);
658 spin_unlock_irq(&lo->lo_lock);
672 static ssize_t loop_attr_offset_show(struct loop_device *lo, char *buf)
674 return sysfs_emit(buf, "%llu\n", (unsigned long long)lo->lo_offset);
677 static ssize_t loop_attr_sizelimit_show(struct loop_device *lo, char *buf)
679 return sysfs_emit(buf, "%llu\n", (unsigned long long)lo->lo_sizelimit);
682 static ssize_t loop_attr_autoclear_show(struct loop_device *lo, char *buf)
684 int autoclear = (lo->lo_flags & LO_FLAGS_AUTOCLEAR);
689 static ssize_t loop_attr_partscan_show(struct loop_device *lo, char *buf)
691 int partscan = (lo->lo_flags & LO_FLAGS_PARTSCAN);
696 static ssize_t loop_attr_dio_show(struct loop_device *lo, char *buf)
698 int dio = (lo->lo_flags & LO_FLAGS_DIRECT_IO);
725 static void loop_sysfs_init(struct loop_device *lo)
727 lo->sysfs_inited = !sysfs_create_group(&disk_to_dev(lo->lo_disk)->kobj,
731 static void loop_sysfs_exit(struct loop_device *lo)
733 if (lo->sysfs_inited)
734 sysfs_remove_group(&disk_to_dev(lo->lo_disk)->kobj,
738 static void loop_get_discard_config(struct loop_device *lo,
741 struct file *file = lo->lo_backing_file;
773 struct loop_device *lo;
792 static void loop_queue_work(struct loop_device *lo, struct loop_cmd *cmd)
799 spin_lock_irq(&lo->lo_work_lock);
804 node = &lo->worker_tree.rb_node;
839 worker->lo = lo;
841 rb_insert_color(&worker->rb_node, &lo->worker_tree);
854 work = &lo->rootcg_work;
855 cmd_list = &lo->rootcg_cmd_list;
858 queue_work(lo->workqueue, work);
859 spin_unlock_irq(&lo->lo_work_lock);
862 static void loop_set_timer(struct loop_device *lo)
864 timer_reduce(&lo->timer, jiffies + LOOP_IDLE_WORKER_TIMEOUT);
867 static void loop_free_idle_workers(struct loop_device *lo, bool delete_all)
871 spin_lock_irq(&lo->lo_work_lock);
872 list_for_each_entry_safe(worker, pos, &lo->idle_worker_list,
879 rb_erase(&worker->rb_node, &lo->worker_tree);
883 if (!list_empty(&lo->idle_worker_list))
884 loop_set_timer(lo);
885 spin_unlock_irq(&lo->lo_work_lock);
890 struct loop_device *lo = container_of(timer, struct loop_device, timer);
892 return loop_free_idle_workers(lo, false);
897 * @lo: struct loop_device to configure
904 loop_set_status_from_info(struct loop_device *lo,
927 lo->lo_offset = info->lo_offset;
928 lo->lo_sizelimit = info->lo_sizelimit;
930 memcpy(lo->lo_file_name, info->lo_file_name, LO_NAME_SIZE);
931 lo->lo_file_name[LO_NAME_SIZE-1] = 0;
935 static unsigned int loop_default_blocksize(struct loop_device *lo)
938 if (lo->lo_flags & LO_FLAGS_DIRECT_IO)
939 return lo->lo_min_dio_size;
943 static void loop_update_limits(struct loop_device *lo, struct queue_limits *lim,
946 struct file *file = lo->lo_backing_file;
957 bsize = loop_default_blocksize(lo);
959 loop_get_discard_config(lo, &granularity, &max_discard_sectors);
965 if (file->f_op->fsync && !(lo->lo_flags & LO_FLAGS_READ_ONLY))
977 static int loop_configure(struct loop_device *lo, blk_mode_t mode,
1012 error = loop_global_lock_killable(lo, is_loop);
1017 if (lo->lo_state != Lo_unbound)
1029 error = loop_set_status_from_info(lo, &config->info);
1032 lo->lo_flags = config->info.lo_flags;
1036 lo->lo_flags |= LO_FLAGS_READ_ONLY;
1038 if (!lo->workqueue) {
1039 lo->workqueue = alloc_workqueue("loop%d",
1041 0, lo->lo_number);
1042 if (!lo->workqueue) {
1049 dev_set_uevent_suppress(disk_to_dev(lo->lo_disk), 1);
1051 disk_force_media_change(lo->lo_disk);
1052 set_disk_ro(lo->lo_disk, (lo->lo_flags & LO_FLAGS_READ_ONLY) != 0);
1054 lo->lo_device = bdev;
1055 loop_assign_backing_file(lo, file);
1057 lim = queue_limits_start_update(lo->lo_queue);
1058 loop_update_limits(lo, &lim, config->block_size);
1060 error = queue_limits_commit_update(lo->lo_queue, &lim);
1071 loop_update_dio(lo);
1072 loop_sysfs_init(lo);
1074 size = lo_calculate_size(lo, file);
1075 loop_set_size(lo, size);
1080 WRITE_ONCE(lo->lo_state, Lo_bound);
1082 lo->lo_flags |= LO_FLAGS_PARTSCAN;
1083 partscan = lo->lo_flags & LO_FLAGS_PARTSCAN;
1085 clear_bit(GD_SUPPRESS_PART_SCAN, &lo->lo_disk->state);
1087 dev_set_uevent_suppress(disk_to_dev(lo->lo_disk), 0);
1088 kobject_uevent(&disk_to_dev(lo->lo_disk)->kobj, KOBJ_CHANGE);
1090 loop_global_unlock(lo, is_loop);
1092 loop_reread_partitions(lo);
1100 loop_global_unlock(lo, is_loop);
1111 static void __loop_clr_fd(struct loop_device *lo)
1115 gfp_t gfp = lo->old_gfp_mask;
1117 spin_lock_irq(&lo->lo_lock);
1118 filp = lo->lo_backing_file;
1119 lo->lo_backing_file = NULL;
1120 spin_unlock_irq(&lo->lo_lock);
1122 lo->lo_device = NULL;
1123 lo->lo_offset = 0;
1124 lo->lo_sizelimit = 0;
1125 memset(lo->lo_file_name, 0, LO_NAME_SIZE);
1133 lim = queue_limits_start_update(lo->lo_queue);
1137 queue_limits_commit_update(lo->lo_queue, &lim);
1139 invalidate_disk(lo->lo_disk);
1140 loop_sysfs_exit(lo);
1142 kobject_uevent(&disk_to_dev(lo->lo_disk)->kobj, KOBJ_CHANGE);
1147 disk_force_media_change(lo->lo_disk);
1149 if (lo->lo_flags & LO_FLAGS_PARTSCAN) {
1160 err = bdev_disk_changed(lo->lo_disk, false);
1163 __func__, lo->lo_number, err);
1168 * lo->lo_state is set to Lo_unbound here after above partscan has
1171 * change the 'lo' device.
1173 lo->lo_flags = 0;
1175 set_bit(GD_SUPPRESS_PART_SCAN, &lo->lo_disk->state);
1176 mutex_lock(&lo->lo_mutex);
1177 WRITE_ONCE(lo->lo_state, Lo_unbound);
1178 mutex_unlock(&lo->lo_mutex);
1188 static int loop_clr_fd(struct loop_device *lo)
1201 err = loop_global_lock_killable(lo, true);
1204 if (lo->lo_state != Lo_bound) {
1205 loop_global_unlock(lo, true);
1214 lo->lo_flags |= LO_FLAGS_AUTOCLEAR;
1215 if (disk_openers(lo->lo_disk) == 1)
1216 WRITE_ONCE(lo->lo_state, Lo_rundown);
1217 loop_global_unlock(lo, true);
1223 loop_set_status(struct loop_device *lo, const struct loop_info64 *info)
1230 err = mutex_lock_killable(&lo->lo_mutex);
1233 if (lo->lo_state != Lo_bound) {
1238 if (lo->lo_offset != info->lo_offset ||
1239 lo->lo_sizelimit != info->lo_sizelimit) {
1241 sync_blockdev(lo->lo_device);
1242 invalidate_bdev(lo->lo_device);
1246 memflags = blk_mq_freeze_queue(lo->lo_queue);
1248 err = loop_set_status_from_info(lo, info);
1252 partscan = !(lo->lo_flags & LO_FLAGS_PARTSCAN) &&
1255 lo->lo_flags &= ~LOOP_SET_STATUS_CLEARABLE_FLAGS;
1256 lo->lo_flags |= (info->lo_flags & LOOP_SET_STATUS_SETTABLE_FLAGS);
1259 loop_update_dio(lo);
1262 blk_mq_unfreeze_queue(lo->lo_queue, memflags);
1264 clear_bit(GD_SUPPRESS_PART_SCAN, &lo->lo_disk->state);
1266 loff_t new_size = lo_calculate_size(lo, lo->lo_backing_file);
1267 loop_set_size(lo, new_size);
1270 mutex_unlock(&lo->lo_mutex);
1272 loop_reread_partitions(lo);
1278 loop_get_status(struct loop_device *lo, struct loop_info64 *info)
1284 ret = mutex_lock_killable(&lo->lo_mutex);
1287 if (lo->lo_state != Lo_bound) {
1288 mutex_unlock(&lo->lo_mutex);
1293 info->lo_number = lo->lo_number;
1294 info->lo_offset = lo->lo_offset;
1295 info->lo_sizelimit = lo->lo_sizelimit;
1296 info->lo_flags = lo->lo_flags;
1297 memcpy(info->lo_file_name, lo->lo_file_name, LO_NAME_SIZE);
1300 path = lo->lo_backing_file->f_path;
1302 mutex_unlock(&lo->lo_mutex);
1350 loop_set_status_old(struct loop_device *lo, const struct loop_info __user *arg)
1358 return loop_set_status(lo, &info64);
1362 loop_set_status64(struct loop_device *lo, const struct loop_info64 __user *arg)
1368 return loop_set_status(lo, &info64);
1372 loop_get_status_old(struct loop_device *lo, struct loop_info __user *arg) {
1379 err = loop_get_status(lo, &info64);
1389 loop_get_status64(struct loop_device *lo, struct loop_info64 __user *arg) {
1395 err = loop_get_status(lo, &info64);
1402 static int loop_set_capacity(struct loop_device *lo)
1406 if (unlikely(lo->lo_state != Lo_bound))
1409 size = lo_calculate_size(lo, lo->lo_backing_file);
1410 loop_set_size(lo, size);
1415 static int loop_set_dio(struct loop_device *lo, unsigned long arg)
1420 if (lo->lo_state != Lo_bound)
1422 if (use_dio == !!(lo->lo_flags & LO_FLAGS_DIRECT_IO))
1426 if (!lo_can_use_dio(lo))
1429 vfs_fsync(lo->lo_backing_file, 0);
1432 memflags = blk_mq_freeze_queue(lo->lo_queue);
1434 lo->lo_flags |= LO_FLAGS_DIRECT_IO;
1436 lo->lo_flags &= ~LO_FLAGS_DIRECT_IO;
1437 blk_mq_unfreeze_queue(lo->lo_queue, memflags);
1441 static int loop_set_block_size(struct loop_device *lo, blk_mode_t mode,
1458 err = mutex_lock_killable(&lo->lo_mutex);
1462 if (lo->lo_state != Lo_bound) {
1467 if (lo->lo_queue->limits.logical_block_size == arg)
1470 sync_blockdev(lo->lo_device);
1471 invalidate_bdev(lo->lo_device);
1473 lim = queue_limits_start_update(lo->lo_queue);
1474 loop_update_limits(lo, &lim, arg);
1476 memflags = blk_mq_freeze_queue(lo->lo_queue);
1477 err = queue_limits_commit_update(lo->lo_queue, &lim);
1478 loop_update_dio(lo);
1479 blk_mq_unfreeze_queue(lo->lo_queue, memflags);
1482 mutex_unlock(&lo->lo_mutex);
1489 static int lo_simple_ioctl(struct loop_device *lo, unsigned int cmd,
1494 err = mutex_lock_killable(&lo->lo_mutex);
1499 err = loop_set_capacity(lo);
1502 err = loop_set_dio(lo, arg);
1507 mutex_unlock(&lo->lo_mutex);
1514 struct loop_device *lo = bdev->bd_disk->private_data;
1530 return loop_configure(lo, mode, bdev, &config);
1538 return loop_configure(lo, mode, bdev, &config);
1541 return loop_change_fd(lo, bdev, arg);
1543 return loop_clr_fd(lo);
1547 err = loop_set_status_old(lo, argp);
1550 return loop_get_status_old(lo, argp);
1554 err = loop_set_status64(lo, argp);
1557 return loop_get_status64(lo, argp);
1561 return loop_set_block_size(lo, mode, bdev, arg);
1568 err = lo_simple_ioctl(lo, cmd, arg);
1648 loop_set_status_compat(struct loop_device *lo,
1657 return loop_set_status(lo, &info64);
1661 loop_get_status_compat(struct loop_device *lo,
1669 err = loop_get_status(lo, &info64);
1678 struct loop_device *lo = bdev->bd_disk->private_data;
1683 err = loop_set_status_compat(lo,
1687 err = loop_get_status_compat(lo,
1713 struct loop_device *lo = disk->private_data;
1716 err = mutex_lock_killable(&lo->lo_mutex);
1720 if (lo->lo_state == Lo_deleting || lo->lo_state == Lo_rundown)
1722 mutex_unlock(&lo->lo_mutex);
1728 struct loop_device *lo = disk->private_data;
1739 mutex_lock(&lo->lo_mutex);
1740 if (lo->lo_state == Lo_bound && (lo->lo_flags & LO_FLAGS_AUTOCLEAR))
1741 WRITE_ONCE(lo->lo_state, Lo_rundown);
1743 need_clear = (lo->lo_state == Lo_rundown);
1744 mutex_unlock(&lo->lo_mutex);
1747 __loop_clr_fd(lo);
1752 struct loop_device *lo = disk->private_data;
1754 if (lo->workqueue)
1755 destroy_workqueue(lo->workqueue);
1756 loop_free_idle_workers(lo, true);
1757 timer_shutdown_sync(&lo->timer);
1758 mutex_destroy(&lo->lo_mutex);
1759 kfree(lo);
1852 struct loop_device *lo = rq->q->queuedata;
1856 if (data_race(READ_ONCE(lo->lo_state)) != Lo_bound)
1866 cmd->use_aio = lo->lo_flags & LO_FLAGS_DIRECT_IO;
1885 loop_queue_work(lo, cmd);
1896 struct loop_device *lo = rq->q->queuedata;
1900 if (write && (lo->lo_flags & LO_FLAGS_READ_ONLY)) {
1921 ret = do_req_filebacked(lo, rq);
1943 struct list_head *cmd_list, struct loop_device *lo)
1949 spin_lock_irq(&lo->lo_work_lock);
1954 spin_unlock_irq(&lo->lo_work_lock);
1959 spin_lock_irq(&lo->lo_work_lock);
1969 list_add_tail(&worker->idle_list, &lo->idle_worker_list);
1970 loop_set_timer(lo);
1972 spin_unlock_irq(&lo->lo_work_lock);
1980 loop_process_work(worker, &worker->cmd_list, worker->lo);
1985 struct loop_device *lo =
1987 loop_process_work(NULL, &lo->rootcg_cmd_list, lo);
2003 struct loop_device *lo;
2008 lo = kzalloc_obj(*lo);
2009 if (!lo)
2011 lo->worker_tree = RB_ROOT;
2012 INIT_LIST_HEAD(&lo->idle_worker_list);
2013 timer_setup(&lo->timer, loop_free_idle_workers_timer, TIMER_DEFERRABLE);
2014 WRITE_ONCE(lo->lo_state, Lo_unbound);
2022 err = idr_alloc(&loop_index_idr, lo, i, i + 1, GFP_KERNEL);
2026 err = idr_alloc(&loop_index_idr, lo, 0, 0, GFP_KERNEL);
2033 lo->tag_set.ops = &loop_mq_ops;
2034 lo->tag_set.nr_hw_queues = 1;
2035 lo->tag_set.queue_depth = hw_queue_depth;
2036 lo->tag_set.numa_node = NUMA_NO_NODE;
2037 lo->tag_set.cmd_size = sizeof(struct loop_cmd);
2038 lo->tag_set.flags = BLK_MQ_F_STACKING | BLK_MQ_F_NO_SCHED_BY_DEFAULT;
2039 lo->tag_set.driver_data = lo;
2041 err = blk_mq_alloc_tag_set(&lo->tag_set);
2045 disk = lo->lo_disk = blk_mq_alloc_disk(&lo->tag_set, &lim, lo);
2050 lo->lo_queue = lo->lo_disk->queue;
2072 mutex_init(&lo->lo_mutex);
2073 lo->lo_number = i;
2074 spin_lock_init(&lo->lo_lock);
2075 spin_lock_init(&lo->lo_work_lock);
2076 INIT_WORK(&lo->rootcg_work, loop_rootcg_workfn);
2077 INIT_LIST_HEAD(&lo->rootcg_cmd_list);
2082 disk->private_data = lo;
2083 disk->queue = lo->lo_queue;
2094 lo->idr_visible = true;
2102 blk_mq_free_tag_set(&lo->tag_set);
2108 kfree(lo);
2113 static void loop_remove(struct loop_device *lo)
2116 del_gendisk(lo->lo_disk);
2117 blk_mq_free_tag_set(&lo->tag_set);
2120 idr_remove(&loop_index_idr, lo->lo_number);
2123 put_disk(lo->lo_disk);
2141 struct loop_device *lo;
2153 lo = idr_find(&loop_index_idr, idx);
2154 if (!lo || !lo->idr_visible)
2157 lo->idr_visible = false;
2163 ret = mutex_lock_killable(&lo->lo_mutex);
2166 if (lo->lo_state != Lo_unbound || disk_openers(lo->lo_disk) > 0) {
2167 mutex_unlock(&lo->lo_mutex);
2172 WRITE_ONCE(lo->lo_state, Lo_deleting);
2173 mutex_unlock(&lo->lo_mutex);
2175 loop_remove(lo);
2181 lo->idr_visible = true;
2188 struct loop_device *lo;
2194 idr_for_each_entry(&loop_index_idr, lo, id) {
2199 if (lo->idr_visible &&
2200 data_race(READ_ONCE(lo->lo_state)) == Lo_unbound)
2297 struct loop_device *lo;
2309 idr_for_each_entry(&loop_index_idr, lo, id)
2310 loop_remove(lo);